def __call__(self, args):
super(DbCommand, self).__call__(args)
self.configure_logging(args)
# Since EnvDefault.__action__ is only called if a user actively passes
# a variable, there's no way to do the string replacing in the action
# itself. Instead, we're post-processing them here, but this could be
# improved.
names = sorted(x.dest for x in self.parser._actions)
for dest in names:
if dest in ("help", "verbose", "quiet"):
continue
value = getattr(args, dest)
if value and isinstance(value, basestring):
replacement = value % dict(args._get_kwargs())
log.debug("% 20s => %s" % (dest, replacement))
setattr(args, dest, replacement)
if args.serverdir:
d = args.serverdir
else:
raise Stop(1, 'OMERO server directory required')
ext = External(d)
ext.setup_omero_cli()
DbAdmin(d, args.dbcommand, args, ext)
def __init__(self, cmd, args):
self.args = args
log.info("%s: %s", self.__class__.__name__, cmd)
log.debug("Current directory: %s", os.getcwd())
if cmd == 'upgrade':
newinstall = False
if not os.path.exists(args.sym):
raise Stop(30, 'Symlink is missing: %s' % args.sym)
elif cmd == 'install':
newinstall = True
if os.path.exists(args.sym):
raise Stop(30, 'Symlink already exists: %s' % args.sym)
else:
raise Exception('Unexpected command: %s' % cmd)
server_dir = self.get_server_dir()
if newinstall:
# Create a symlink to simplify the rest of the logic-
# just need to check if OLD == NEW
self.symlink(server_dir, args.sym)
log.info("Installing %s (%s)...", server_dir, args.sym)
else:
log.info("Upgrading %s (%s)...", server_dir, args.sym)
self.external = External(server_dir)
self.external.setup_omero_cli()
if not newinstall:
self.external.setup_previous_omero_env(args.sym, args.savevarsfile)
# Need lib/python set above
import path
self.dir = path.path(server_dir)
if not newinstall:
self.stop()
self.archive_logs()
copyold = not newinstall and not args.ignoreconfig
self.configure(copyold, args.prestartfile)
self.directories()
if newinstall:
self.init_db()
self.upgrade_db()
self.external.save_env_vars(args.savevarsfile, args.savevars.split())
self.start()
def __init__(self, cmd, args):
self.args, newinstall = self._handle_args(cmd, args)
log.info("%s: %s", self.__class__.__name__, cmd)
log.debug("Current directory: %s", os.getcwd())
self.symlink_check_and_set()
if newinstall is None:
# Automatically install or upgrade
newinstall = not os.path.exists(args.sym)
elif newinstall is False:
if not os.path.exists(args.sym):
raise Stop(30, 'Symlink is missing: %s' % args.sym)
elif newinstall is True:
if os.path.exists(args.sym):
raise Stop(30, 'Symlink already exists: %s' % args.sym)
else:
assert False
server_dir = self.get_server_dir()
if newinstall:
# Create a symlink to simplify the rest of the logic-
# just need to check if OLD == NEW
self.symlink(server_dir, args.sym)
log.info("Installing %s (%s)...", server_dir, args.sym)
else:
log.info("Upgrading %s (%s)...", server_dir, args.sym)
self.external = External(server_dir)
self.external.setup_omero_cli()
if not newinstall:
self.external.setup_previous_omero_env(args.sym, args.savevarsfile)
# Need lib/python set above
import path
self.dir = path.path(server_dir)
if not newinstall:
self.stop()
self.archive_logs()
copyold = not newinstall and not args.ignoreconfig
self.configure(copyold, args.prestartfile)
self.directories()
self.handle_database()
self.external.save_env_vars(args.savevarsfile, args.savevars.split())
self.start()
def run(self, wd=None):
self.log.info("Starting RunController")
try:
while True:
time.sleep(0.2)
if self._stop.isSet():
break
if self.current and self.current.status == 'completed':
if self.current.retval:
self.retval[self.current.pid] = self.current.retval
self.log.info("Retval: %s" % self.current.retval)
self.current_process = None
self.current.join()
self.current = None
if self.current and self.current.status == 'aborted':
self.log.warning("%s aborted without finishing" %
self.current.name)
if self.run_queue and not self.current:
pid = sorted(self.run_queue.keys())[0]
cmd = self.run_queue[pid]
if cmd[0] == 'mkdir':
try:
os.stat(cmd[1])
except:
os.makedirs(cmd[1])
elif cmd[0] == 'cd':
os.chdir(cmd[1])
else:
self.current = External(cmd[0],
cmd[1:],
pid=pid,
log_name=self.log_name)
cwd = os.getcwd()
if self.wd:
os.chdir(self.wd)
self.current.start()
while not self.current.started:
time.sleep(0.1)
os.chdir(cwd)
del self.run_queue[pid]
self.state = 'running'
if not self.run_queue and not self.current and self.state == 'running':
self.state = 'finished'
if self.current:
self.current.stop()
self.current.join()
except Exception as e:
print "RunController: %s" % e
self.log.info("RunController finishing")
def __init__(self, cmd, args):
self.args = args
log.info("%s: %s", self.__class__.__name__, cmd)
log.debug("Current directory: %s", os.getcwd())
if cmd == "upgrade":
newinstall = False
if not os.path.exists(args.sym):
raise Stop(30, "Symlink is missing: %s" % args.sym)
elif cmd == "install":
newinstall = True
if os.path.exists(args.sym):
raise Stop(30, "Symlink already exists: %s" % args.sym)
else:
raise Exception("Unexpected command: %s" % cmd)
server_dir = self.get_server_dir()
if newinstall:
# Create a symlink to simplify the rest of the logic-
# just need to check if OLD == NEW
self.symlink(server_dir, args.sym)
log.info("Installing %s (%s)...", server_dir, args.sym)
else:
log.info("Upgrading %s (%s)...", server_dir, args.sym)
self.external = External(server_dir)
self.external.setup_omero_cli()
if not newinstall:
self.external.setup_previous_omero_env(args.sym, args.savevarsfile)
# Need lib/python set above
import path
self.dir = path.path(server_dir)
if not newinstall:
self.stop()
self.archive_logs()
copyold = not newinstall and not args.ignoreconfig
self.configure(copyold, args.prestartfile)
self.directories()
if newinstall:
self.init_db()
self.upgrade_db()
self.external.save_env_vars(args.savevarsfile, args.savevars.split())
self.start()
def taskProcess(self):
logging.debug(self.__class__)
logging.debug(self.currentTask)
logging.debug(self.objects)
jobId = self.currentTask.job
taskId = self.currentTask.task_id
external = External()
if len(self.objects) == 0:
raise ObjectStoreException("Task processing didn't find task object.")
if not self.objects[0].isSet("pcap_content"):
raise ParamException("pcap_content param is missing.")
outputDir = "/tmp/%s" % uuid.uuid4()
pcapPath = self.getPcapFilePath()
config = Config().getConfig()
configVar = config.get("verifier", "default")
external.runExternal(["mkdir", "-p", outputDir]);
pcapFilePath = self.getPcapFilePath()
external.runExternal(["tcpxtract", "-f", pcapFilePath, "-o", outputDir]);
dirList=os.listdir(outputDir)
verifierFactory = VerifierFactory()
objects = list()
for fname in dirList:
filepath = "%s/%s" % (outputDir, fname)
m = magic.open(magic.MAGIC_MIME)
m.load()
result = m.file(filepath)
mimetype = result.split(";")[0]
fileName, fileExtension = os.path.splitext(filepath)
fileExtension = fileExtension[1:]
verifierList = verifierFactory.getVerifierList(fileExtension, config)
result = True;
for ver in verifierList:
logging.info("%s, %s, %s" % (filepath, mimetype, fileExtension))
result = result and ver.verify(filepath, mimetype, fileExtension, config)
if not result:
break
if result:
objects.append(self.createNewObject(filepath, fileExtension))
if len(objects) > 0:
newObjIds = self.osAdapter.objectsPut(jobId, taskId, objects)
self.newObjects.extend(newObjIds)
external.runExternal(["rm", "-rf", outputDir, pcapFilePath]);
return []
def psql(self, *psqlargs):
"""
Run a psql command
"""
db, env = self.get_db_args_env()
args = ['-d', db['name'], '-h', db['host'], '-U', db['user'],
'-w', '-A', '-t'] + list(psqlargs)
stdout, stderr = External.run('psql', args, capturestd=True, env=env)
if stderr:
log.warn('stderr: %s', stderr)
log.debug('stdout: %s', stdout)
return stdout
def __init__(self, cmd, args):
self.args, newinstall = self._handle_args(cmd, args)
log.info("%s: %s", self.__class__.__name__, cmd)
log.debug("Current directory: %s", os.getcwd())
self.symlink_check_and_set()
if newinstall is None:
# Automatically install or upgrade
newinstall = not os.path.exists(args.sym)
elif newinstall is False:
if not os.path.exists(args.sym):
raise Stop(30, 'Symlink is missing: %s' % args.sym)
elif newinstall is True:
if os.path.exists(args.sym):
raise Stop(30, 'Symlink already exists: %s' % args.sym)
else:
assert False
server_dir = self.get_server_dir()
if newinstall:
# Create a symlink to simplify the rest of the logic-
# just need to check if OLD == NEW
self.symlink(server_dir, args.sym)
log.info("Installing %s (%s)...", server_dir, args.sym)
else:
log.info("Upgrading %s (%s)...", server_dir, args.sym)
self.external = External(server_dir)
self.external.setup_omero_cli()
if not newinstall:
self.external.setup_previous_omero_env(args.sym, args.savevarsfile)
# Need lib/python set above
import path
self.dir = path.path(server_dir)
if not newinstall:
self.stop()
self.archive_logs()
copyold = not newinstall and not args.ignoreconfig
self.configure(copyold, args.prestartfile)
self.directories()
self.handle_database()
self.external.save_env_vars(args.savevarsfile, args.savevars.split())
self.start()
def pgdump(self, *pgdumpargs):
"""
Run a pg_dump command
"""
db, env = self.get_db_args_env()
args = ['-d', db['name'], '-h', db['host'], '-U', db['user'], '-w'
] + list(pgdumpargs)
stdout, stderr = External.run(
'pg_dump', args, capturestd=True, env=env)
if stderr:
log.warn('stderr: %s', stderr)
log.debug('stdout: %s', stdout)
return stdout
def run(self, wd = None):
self.log.info("Starting RunController")
try:
while True:
time.sleep(0.2)
if self._stop.isSet():
break
if self.current and self.current.status == 'completed':
if self.current.retval:
self.retval[self.current.pid] = self.current.retval
self.log.info("Retval: %s" % self.current.retval)
self.current_process = None
self.current.join()
self.current = None
if self.current and self.current.status in ['failed', 'aborted']:
self.log.error("%s %s" % (self.current.name, self.current.status))
self.current_process = None
self.current.join()
self.current = None
self.run_queue.clear()
if self.run_queue and not self.current:
pid = sorted(self.run_queue.keys())[0]
cmd = self.run_queue[pid]
if cmd[0] == 'mkdir':
try:
os.stat(cmd[1])
except:
os.makedirs(cmd[1])
elif cmd[0] == 'cd':
os.chdir(cmd[1])
else:
cwd = os.getcwd()
os.chdir(self.wd)
self.current = External(cmd[0], cmd[1:], pid = pid, log_name = self.log_name, wd=self.wd)
self.current.start()
while not self.current.started:
time.sleep(0.1)
os.chdir(cwd)
del self.run_queue[pid]
self.state = 'running'
if not self.run_queue and not self.current and self.state == 'running':
self.state = 'finished'
if self.current:
self.current.stop()
self.current.join()
except Exception as e:
print "RunController: %s" % e
self.log.info("RunController finishing")
def psql(self, *psqlargs):
"""
Run a psql command
"""
if not self.args.dbname:
raise Exception('Database name required')
env = os.environ.copy()
env['PGPASSWORD'] = self.args.dbpass
args = ['-d', self.args.dbname, '-h', self.args.dbhost, '-U',
self.args.dbuser, '-w', '-A', '-t'] + list(psqlargs)
stdout, stderr = External.run('psql', args, capturestd=True, env=env)
if stderr:
log.warn('stderr: %s', stderr)
log.debug('stdout: %s', stdout)
return stdout
def psql(self, *psqlargs):
"""
Run a psql command
"""
db, env = self.get_db_args_env()
args = [
'-v', 'ON_ERROR_STOP=on',
'-d', db['name'],
'-h', db['host'],
'-U', db['user'],
'-w', '-A', '-t'
] + list(psqlargs)
stdout, stderr = External.run('psql', args, capturestd=True, env=env)
if stderr:
log.warn('stderr: %s', stderr)
log.debug('stdout: %s', stdout)
return stdout
def getRequestList(self, pcapFilePath, whitelist):
self.external = External()
outputLines = self.external.runExternal(["tshark", whitelist, "-r", pcapFilePath])[0].split("\n")
result = []
for line in outputLines:
match = re.search(self.regex, line.strip())
if match is None:
continue
tmp = {}
tmp["id"] = int(match.group("id").strip())
tmp["time"] = match.group("time").strip()
tmp["from"] = match.group("from").strip()
tmp["to"] = match.group("to").strip()
tmp["protocol"] = match.group("protocol").strip()
tmp["port"] = match.group("port").strip()
tmp["rest"] = match.group("rest").strip()
result.append(tmp)
return result
def main(argv):
parser = argparse.ArgumentParser(description='Create a skeleton application using some command line options.')
parser.add_argument('appname', help='The application name')
parser.add_argument('-b', '--bower', help='Dependencies installed using bower')
parser.add_argument('-n', '--no-debug', action='store_false')
parser.add_argument('-v', '--virtualenv', action='store_true')
parser.add_argument('-d', '--database', action='store_true')
parser.add_argument('-g', '--git', action='store_true')
args = parser.parse_args()
bower = None
if args.bower:
bower = args.bower.split(',')
virtualenv = args.virtualenv
debug = args.no_debug
appname = args.appname
database = args.database
git = args.git
if database:
project = FlaskDbProject(appname)
else:
project = FlaskProject(appname)
project.debug = debug
project.bower = bower
project.virtualenv = virtualenv
project.git = git
print(generate_brief(project.brief_var))
errors = External().errors
if len(errors) > 0:
print(generate_errorlist({'errors': errors, }))
sys.exit(1)
if query_yes_no("Is this correct ?"):
project.install()
else:
print("Aborting")
sys.exit(0)
class RunController(threading.Thread):
def __init__(self, log_level = logging.INFO, log_name = "MetLab"):
super(RunController, self).__init__()
self.log_name = log_name
self.log = logging.getLogger( log_name )
self.log.setLevel( log_level )
self._stop = threading.Event()
self.current = None
self.state = 'idle'
self.run_queue = {}
self.retval = {}
self.process_counter = 0
self.wd = None
self._init_db()
def _init_db(self):
for table in DATABASE_SCHEMA:
self._query( table )
def _query(self, query, *args):
con = None
data = None
try:
con = sqlite3.connect( DATABASE )
cur = con.cursor()
cur.execute(query, tuple(args))
data = cur.fetchall()
if not data:
con.commit()
except sqlite3.Error as e:
self.log.error("Database error: %s" % e)
except Exception as e:
self.log.error("Exception in _query: %s" % e)
finally:
if con:
con.close()
return data
def get_paths(self):
return self._query("SELECT `name`,`path` FROM paths")
def get_retval(self, pid):
if int(pid) in self.retval:
return self.retval[int(pid)]
return None
def queue(self, cmd):
pid = self.process_counter
if cmd[0][0] == '.':
cmd[0] = os.path.abspath(cmd[0])
self.run_queue[pid] = cmd
self.log.info("adding step: %s" % cmd)
if hasattr(self, "project_id"):
self._query("INSERT INTO steps(project_id, command) VALUES (?, ?)", str(self.project_id), " ".join(cmd))
self.process_counter += 1
return pid
def clear(self):
self.run_queue = {}
def run(self, wd = None):
self.log.info("Starting RunController")
try:
while True:
time.sleep(0.2)
if self._stop.isSet():
break
if self.current and self.current.status == 'completed':
if self.current.retval:
self.retval[self.current.pid] = self.current.retval
self.log.info("Retval: %s" % self.current.retval)
self.current_process = None
self.current.join()
self.current = None
if self.current and self.current.status in ['failed', 'aborted']:
self.log.error("%s %s" % (self.current.name, self.current.status))
self.current_process = None
self.current.join()
self.current = None
self.run_queue.clear()
if self.run_queue and not self.current:
pid = sorted(self.run_queue.keys())[0]
cmd = self.run_queue[pid]
if cmd[0] == 'mkdir':
try:
os.stat(cmd[1])
except:
os.makedirs(cmd[1])
elif cmd[0] == 'cd':
os.chdir(cmd[1])
else:
cwd = os.getcwd()
os.chdir(self.wd)
self.current = External(cmd[0], cmd[1:], pid = pid, log_name = self.log_name, wd=self.wd)
self.current.start()
while not self.current.started:
time.sleep(0.1)
os.chdir(cwd)
del self.run_queue[pid]
self.state = 'running'
if not self.run_queue and not self.current and self.state == 'running':
self.state = 'finished'
if self.current:
self.current.stop()
self.current.join()
except Exception as e:
print "RunController: %s" % e
self.log.info("RunController finishing")
def set_wd(self, dir_name):
self.log.info("Setting new WD to %s" % dir_name)
if dir_name[0] == ".":
self.wd = "%s%s" % (os.getcwd(), dir_name[1:])
elif dir_name[0] != "/":
self.wd = "%s/%s" % (os.getcwd(), dir_name)
else:
self.wd = dir_name
def stop(self):
self._stop.set()
def stop_current(self):
if self.current:
self.current.stop()
def start_project(self, name):
self.queue(["mkdir", name])
self._query("INSERT INTO projects (name, directory, started) VALUES ('%s', '%s', %i)" % (name, self.wd, time.time()))
self.project_id = self._query("SELECT id FROM projects ORDER BY id DESC LIMIT 1")[0][0]
self.log.info("Settings project_id = %i" % self.project_id)
def verify(self, filepath, mimetype, extension, config):
external = External()
output = external.runExternal(["zip", "--test", filepath])
if re.match("(.*)OK(.*)", output[0], re.I) is not None:
return True
return False
class Install(object):
def __init__(self, cmd, args):
self.args = args
log.info("%s: %s", self.__class__.__name__, cmd)
log.debug("Current directory: %s", os.getcwd())
if cmd == 'upgrade':
newinstall = False
if not os.path.exists(args.sym):
raise Stop(30, 'Symlink is missing: %s' % args.sym)
elif cmd == 'install':
newinstall = True
if os.path.exists(args.sym):
raise Stop(30, 'Symlink already exists: %s' % args.sym)
else:
raise Exception('Unexpected command: %s' % cmd)
server_dir = self.get_server_dir()
if newinstall:
# Create a symlink to simplify the rest of the logic-
# just need to check if OLD == NEW
self.symlink(server_dir, args.sym)
log.info("Installing %s (%s)...", server_dir, args.sym)
else:
log.info("Upgrading %s (%s)...", server_dir, args.sym)
self.external = External(server_dir)
self.external.setup_omero_cli()
if not newinstall:
self.external.setup_previous_omero_env(args.sym, args.savevarsfile)
# Need lib/python set above
import path
self.dir = path.path(server_dir)
if not newinstall:
self.stop()
self.archive_logs()
copyold = not newinstall and not args.ignoreconfig
self.configure(copyold, args.prestartfile)
self.directories()
if newinstall:
self.init_db()
self.upgrade_db()
self.external.save_env_vars(args.savevarsfile, args.savevars.split())
self.start()
def get_server_dir(self):
"""
Either downloads and/or unzips the server if necessary
return: the directory of the unzipped server
"""
if not self.args.server:
if self.args.skipunzip:
raise Stop(0, 'Unzip disabled, exiting')
log.info('Downloading server')
artifacts = Artifacts(self.args)
server = artifacts.download('server')
else:
progress = 0
if self.args.verbose:
progress = 20
ptype, server = fileutils.get_as_local_path(
self.args.server,
self.args.overwrite,
progress=progress,
httpuser=self.args.httpuser,
httppassword=self.args.httppassword)
if ptype == 'file':
if self.args.skipunzip:
raise Stop(0, 'Unzip disabled, exiting')
log.info('Unzipping %s', server)
server = fileutils.unzip(server,
match_dir=True,
destdir=self.args.unzipdir)
log.debug('Server directory: %s', server)
return server
def stop(self):
try:
log.info("Stopping server")
self.bin("admin status --nodeonly")
self.bin("admin stop")
except Exception as e:
log.error('Error whilst stopping server: %s', e)
if not self.args.no_web:
try:
log.info("Stopping web")
self.stopweb()
except Exception as e:
log.error('Error whilst stopping web: %s', e)
def configure(self, copyold, prestartfile):
def samecontents(a, b):
# os.path.samefile is not available on Windows
try:
return os.path.samefile(a, b)
except AttributeError:
with open(a) as fa:
with open(b) as fb:
return fa.read() == fb.read()
target = self.dir / "etc" / "grid" / "config.xml"
if copyold:
from path import path
old_grid = path(self.args.sym) / "etc" / "grid"
old_cfg = old_grid / "config.xml"
log.info("Copying old configuration from %s", old_cfg)
if not old_cfg.exists():
raise Stop(40, 'config.xml not found')
if target.exists() and samecontents(old_cfg, target):
# This likely is caused by the symlink being
# created early on an initial install.
pass
else:
old_cfg.copy(target)
else:
if target.exists():
log.info('Deleting configuration file %s', target)
target.remove()
if prestartfile:
for f in prestartfile:
log.info('Loading prestart file %s', f)
ftype, fpath = fileutils.get_as_local_path(f, 'backup')
if ftype != 'file':
raise Stop(50, 'Expected file, found: %s %s' % (ftype, f))
self.run(['load', fpath])
def archive_logs(self):
if self.args.archivelogs:
logdir = os.path.join(self.args.sym, 'var', 'log')
archive = self.args.archivelogs
log.info('Archiving logs to %s', archive)
fileutils.zip(archive, logdir, os.path.join(self.args.sym, 'var'))
return archive
def directories(self):
if self.samedir(self.dir, self.args.sym):
log.warn("Upgraded server was the same, not deleting")
return
try:
target = self.readlink(self.args.sym)
targetzip = target + '.zip'
except IOError:
log.error('Unable to get symlink target: %s', self.args.sym)
target = None
targetzip = None
if self.args.delete_old and target:
try:
log.info("Deleting %s", target)
shutil.rmtree(target)
except OSError as e:
log.error("Failed to delete %s: %s", target, e)
if not self.args.keep_old_zip and targetzip:
try:
log.info("Deleting %s", targetzip)
os.unlink(targetzip)
except OSError as e:
log.error("Failed to delete %s: %s", targetzip, e)
self.rmlink(self.args.sym)
self.symlink(self.dir, self.args.sym)
def init_db(self):
if self.args.initdb:
log.debug('Initialising database')
DbAdmin(self.dir, 'init', self.args, self.external)
def upgrade_db(self):
if self.args.upgradedb:
log.debug('Upgrading database')
DbAdmin(self.dir, 'upgrade', self.args, self.external)
def start(self):
if self.args.no_start:
log.debug('Not starting OMERO')
return
self.run("admin start")
if not self.args.no_web:
log.info("Starting web")
self.startweb()
def run(self, command):
"""
Runs a command as if from the command-line
without the need for using popen or subprocess
"""
if isinstance(command, basestring):
command = command.split()
else:
command = list(command)
self.external.omero_cli(command)
def bin(self, command):
"""
Runs the omero command-line client with an array of arguments using the
old environment
"""
if isinstance(command, basestring):
command = command.split()
self.external.omero_bin(command)
class RunController(threading.Thread):
def __init__(self, log_level=logging.INFO, log_name="MetLab"):
super(RunController, self).__init__()
self.log_name = log_name
self.log = logging.getLogger(log_name)
self.log.setLevel(log_level)
self._stop = threading.Event()
self.current = None
self.state = 'idle'
self.run_queue = {}
self.retval = {}
self.process_counter = 0
self.wd = None
self._init_db()
def _init_db(self):
for table in DATABASE_SCHEMA:
self._query(table)
def _query(self, query, *args):
con = None
data = None
try:
con = sqlite3.connect(DATABASE)
cur = con.cursor()
cur.execute(query, tuple(args))
data = cur.fetchall()
if not data:
con.commit()
except sqlite3.Error as e:
self.log.error("Database error: %s" % e)
except Exception as e:
self.log.error("Exception in _query: %s" % e)
finally:
if con:
con.close()
return data
def get_paths(self):
return self._query("SELECT `name`,`path` FROM paths")
def get_retval(self, pid):
if int(pid) in self.retval:
return self.retval[int(pid)]
return None
def queue(self, cmd):
pid = self.process_counter
if cmd[0][0] == '.':
cmd[0] = os.path.abspath(cmd[0])
self.run_queue[pid] = cmd
self.log.info("adding step: %s" % cmd)
if hasattr(self, "project_id"):
self._query("INSERT INTO steps(project_id, command) VALUES (?, ?)",
str(self.project_id), " ".join(cmd))
self.process_counter += 1
return pid
def clear(self):
self.run_queue = {}
def run(self, wd=None):
self.log.info("Starting RunController")
try:
while True:
time.sleep(0.2)
if self._stop.isSet():
break
if self.current and self.current.status == 'completed':
if self.current.retval:
self.retval[self.current.pid] = self.current.retval
self.log.info("Retval: %s" % self.current.retval)
self.current_process = None
self.current.join()
self.current = None
if self.current and self.current.status == 'aborted':
self.log.warning("%s aborted without finishing" %
self.current.name)
if self.run_queue and not self.current:
pid = sorted(self.run_queue.keys())[0]
cmd = self.run_queue[pid]
if cmd[0] == 'mkdir':
try:
os.stat(cmd[1])
except:
os.makedirs(cmd[1])
elif cmd[0] == 'cd':
os.chdir(cmd[1])
else:
self.current = External(cmd[0],
cmd[1:],
pid=pid,
log_name=self.log_name)
cwd = os.getcwd()
if self.wd:
os.chdir(self.wd)
self.current.start()
while not self.current.started:
time.sleep(0.1)
os.chdir(cwd)
del self.run_queue[pid]
self.state = 'running'
if not self.run_queue and not self.current and self.state == 'running':
self.state = 'finished'
if self.current:
self.current.stop()
self.current.join()
except Exception as e:
print "RunController: %s" % e
self.log.info("RunController finishing")
def set_wd(self, dir_name):
if dir_name[0] == ".":
self.wd = "%s%s" % (os.getcwd(), dir_name[1:])
elif dir_name[0] != "/":
self.wd = "%s/%s" % (os.getcwd(), dir_name)
else:
self.wd = dir_name
def stop(self):
self._stop.set()
def stop_current(self):
if self.current:
self.current.stop()
def start_project(self, name):
self.queue(["mkdir", name])
self._query(
"INSERT INTO projects (name, directory, started) VALUES ('%s', '%s', %i)"
% (name, self.wd, time.time()))
self.project_id = self._query(
"SELECT id FROM projects ORDER BY id DESC LIMIT 1")[0][0]
self.log.info("Settings project_id = %i" % self.project_id)
class Install(object):
def __init__(self, cmd, args):
self.args, newinstall = self._handle_args(cmd, args)
log.info("%s: %s", self.__class__.__name__, cmd)
log.debug("Current directory: %s", os.getcwd())
self.symlink_check_and_set()
if newinstall is None:
# Automatically install or upgrade
newinstall = not os.path.exists(args.sym)
elif newinstall is False:
if not os.path.exists(args.sym):
raise Stop(30, 'Symlink is missing: %s' % args.sym)
elif newinstall is True:
if os.path.exists(args.sym):
raise Stop(30, 'Symlink already exists: %s' % args.sym)
else:
assert False
server_dir = self.get_server_dir()
if newinstall:
# Create a symlink to simplify the rest of the logic-
# just need to check if OLD == NEW
self.symlink(server_dir, args.sym)
log.info("Installing %s (%s)...", server_dir, args.sym)
else:
log.info("Upgrading %s (%s)...", server_dir, args.sym)
self.external = External(server_dir)
self.external.setup_omero_cli()
if not newinstall:
self.external.setup_previous_omero_env(args.sym, args.savevarsfile)
# Need lib/python set above
import path
self.dir = path.path(server_dir)
if not newinstall:
self.stop()
self.archive_logs()
copyold = not newinstall and not args.ignoreconfig
self.configure(copyold, args.prestartfile)
self.directories()
self.handle_database()
self.external.save_env_vars(args.savevarsfile, args.savevars.split())
self.start()
def _handle_args(self, cmd, args):
"""
We need to support deprecated behaviour for now which makes this
quite complicated
Current behaviour:
- install: Installs a new server, existing server causes an error
- install --upgrade: Installs or upgrades a server
- install --managedb: Automatically initialise or upgrade the db
Deprecated:
- install --upgradedb --initdb: Replaced by install --managedb
- install --upgradedb: upgrade the db, must exist
- install --initdb: initialise the db
- upgrade: Upgrades a server, must already exist
- upgrade --upgradedb: Automatically upgrade the db
returns:
- Modified args object, flag to indicate new/existing/auto install
"""
if cmd == 'install':
if args.upgrade:
# Current behaviour: install or upgrade
if args.initdb or args.upgradedb:
raise Stop(10, (
'Deprecated --initdb --upgradedb flags '
'are incompatible with --upgrade'))
newinstall = None
else:
# Current behaviour: Server must not exist
newinstall = True
if args.managedb:
# Current behaviour
if args.initdb or args.upgradedb:
raise Stop(10, (
'Deprecated --initdb --upgradedb flags '
'are incompatible with --managedb'))
args.initdb = True
args.upgradedb = True
else:
if args.initdb or args.upgradedb:
log.warn('--initdb and --upgradedb are deprecated, '
'use --managedb')
elif cmd == 'upgrade':
# Deprecated behaviour
log.warn(
'"omero upgrade" is deprecated, use "omego install --upgrade"')
cmd = 'install'
args.upgrade = True
# Deprecated behaviour: Server must exist
newinstall = False
else:
raise Exception('Unexpected command: %s' % cmd)
return args, newinstall
def get_server_dir(self):
"""
Either downloads and/or unzips the server if necessary
return: the directory of the unzipped server
"""
if not self.args.server:
if self.args.skipunzip:
raise Stop(0, 'Unzip disabled, exiting')
log.info('Downloading server')
# The downloader automatically symlinks the server, however if
# we are upgrading we want to delay the symlink swap, so this
# overrides args.sym
# TODO: Find a nicer way to do this?
artifact_args = copy.copy(self.args)
artifact_args.sym = ''
artifacts = Artifacts(artifact_args)
server = artifacts.download('server')
else:
progress = 0
if self.args.verbose:
progress = 20
ptype, server = fileutils.get_as_local_path(
self.args.server, self.args.overwrite, progress=progress,
httpuser=self.args.httpuser,
httppassword=self.args.httppassword)
if ptype == 'file':
if self.args.skipunzip:
raise Stop(0, 'Unzip disabled, exiting')
log.info('Unzipping %s', server)
server = fileutils.unzip(
server, match_dir=True, destdir=self.args.unzipdir)
log.debug('Server directory: %s', server)
return server
def stop(self):
try:
log.info("Stopping server")
self.bin("admin status --nodeonly")
self.bin("admin stop")
except Exception as e:
log.error('Error whilst stopping server: %s', e)
if not self.args.no_web:
try:
log.info("Stopping web")
self.stopweb()
except Exception as e:
log.error('Error whilst stopping web: %s', e)
def configure(self, copyold, prestartfile):
def samecontents(a, b):
# os.path.samefile is not available on Windows
try:
return os.path.samefile(a, b)
except AttributeError:
with open(a) as fa:
with open(b) as fb:
return fa.read() == fb.read()
target = self.dir / "etc" / "grid" / "config.xml"
if copyold:
from path import path
old_grid = path(self.args.sym) / "etc" / "grid"
old_cfg = old_grid / "config.xml"
log.info("Copying old configuration from %s", old_cfg)
if not old_cfg.exists():
raise Stop(40, 'config.xml not found')
if target.exists() and samecontents(old_cfg, target):
# This likely is caused by the symlink being
# created early on an initial install.
pass
else:
old_cfg.copy(target)
else:
if target.exists():
log.info('Deleting configuration file %s', target)
target.remove()
if prestartfile:
for f in prestartfile:
log.info('Loading prestart file %s', f)
ftype, fpath = fileutils.get_as_local_path(f, 'backup')
if ftype != 'file':
raise Stop(50, 'Expected file, found: %s %s' % (
ftype, f))
self.run(['load', fpath])
def archive_logs(self):
if self.args.archivelogs:
logdir = os.path.join(self.args.sym, 'var', 'log')
archive = self.args.archivelogs
log.info('Archiving logs to %s', archive)
fileutils.zip(archive, logdir, os.path.join(self.args.sym, 'var'))
return archive
def directories(self):
if self.samedir(self.dir, self.args.sym):
log.warn("Upgraded server was the same, not deleting")
return
try:
target = self.readlink(self.args.sym)
targetzip = target + '.zip'
except IOError:
log.error('Unable to get symlink target: %s', self.args.sym)
target = None
targetzip = None
if self.args.delete_old and target:
try:
log.info("Deleting %s", target)
shutil.rmtree(target)
except OSError as e:
log.error("Failed to delete %s: %s", target, e)
if not self.args.keep_old_zip and targetzip:
try:
log.info("Deleting %s", targetzip)
os.unlink(targetzip)
except OSError as e:
log.error("Failed to delete %s: %s", targetzip, e)
self.rmlink(self.args.sym)
self.symlink(self.dir, self.args.sym)
def handle_database(self):
"""
Handle database initialisation and upgrade, taking into account
command line arguments
"""
# TODO: When initdb and upgradedb are dropped we can just test
# managedb, but for backwards compatibility we need to support
# initdb without upgradedb and vice-versa
if self.args.initdb or self.args.upgradedb:
db = DbAdmin(self.dir, None, self.args, self.external)
status = db.check()
log.debug('OMERO database upgrade status: %s', status)
else:
log.warn('OMERO database check disabled')
return DB_INIT_NEEDED
if status == DB_INIT_NEEDED:
if self.args.initdb:
log.debug('Initialising OMERO database')
db.init()
else:
log.error('OMERO database not found')
raise Stop(DB_INIT_NEEDED,
'Install/Upgrade failed: OMERO database not found')
elif status == DB_UPGRADE_NEEDED:
log.warn('OMERO database exists but is out of date')
if self.args.upgradedb:
log.debug('Upgrading OMERO database')
db.upgrade()
else:
raise Stop(
DB_UPGRADE_NEEDED,
'Pass --managedb or upgrade your OMERO database manually')
else:
assert status == DB_UPTODATE
return status
def start(self):
if self.args.no_start:
log.debug('Not starting OMERO')
return
self.run("admin start")
if not self.args.no_web:
log.info("Starting web")
self.startweb()
def run(self, command):
"""
Runs a command as if from the command-line
without the need for using popen or subprocess
"""
if isinstance(command, basestring):
command = command.split()
else:
command = list(command)
self.external.omero_cli(command)
def bin(self, command):
"""
Runs the omero command-line client with an array of arguments using the
old environment
"""
if isinstance(command, basestring):
command = command.split()
self.external.omero_bin(command)
def symlink_check_and_set(self):
"""
The default symlink was changed from OMERO-CURRENT to OMERO.server.
If `--sym` was not specified and OMERO-CURRENT exists in the current
directory stop and warn.
"""
if self.args.sym == '':
if os.path.exists('OMERO-CURRENT'):
log.error('Deprecated OMERO-CURRENT found but --sym not set')
raise Stop(
30, 'The default for --sym has changed to OMERO.server '
'but the current directory contains OMERO-CURRENT. '
'Either remove OMERO-CURRENT or explicity pass --sym.')
if self.args.sym in ('', 'auto'):
self.args.sym = 'OMERO.server'
class Install(object):
def __init__(self, cmd, args):
self.args = args
log.info("%s: %s", self.__class__.__name__, cmd)
log.debug("Current directory: %s", os.getcwd())
if cmd == "upgrade":
newinstall = False
if not os.path.exists(args.sym):
raise Stop(30, "Symlink is missing: %s" % args.sym)
elif cmd == "install":
newinstall = True
if os.path.exists(args.sym):
raise Stop(30, "Symlink already exists: %s" % args.sym)
else:
raise Exception("Unexpected command: %s" % cmd)
server_dir = self.get_server_dir()
if newinstall:
# Create a symlink to simplify the rest of the logic-
# just need to check if OLD == NEW
self.symlink(server_dir, args.sym)
log.info("Installing %s (%s)...", server_dir, args.sym)
else:
log.info("Upgrading %s (%s)...", server_dir, args.sym)
self.external = External(server_dir)
self.external.setup_omero_cli()
if not newinstall:
self.external.setup_previous_omero_env(args.sym, args.savevarsfile)
# Need lib/python set above
import path
self.dir = path.path(server_dir)
if not newinstall:
self.stop()
self.archive_logs()
copyold = not newinstall and not args.ignoreconfig
self.configure(copyold, args.prestartfile)
self.directories()
if newinstall:
self.init_db()
self.upgrade_db()
self.external.save_env_vars(args.savevarsfile, args.savevars.split())
self.start()
def get_server_dir(self):
"""
Either downloads and/or unzips the server if necessary
return: the directory of the unzipped server
"""
if not self.args.server:
if self.args.skipunzip:
raise Stop(0, "Unzip disabled, exiting")
log.info("Downloading server")
artifacts = Artifacts(self.args)
server = artifacts.download("server")
else:
progress = 0
if self.args.verbose:
progress = 20
ptype, server = fileutils.get_as_local_path(
self.args.server,
self.args.overwrite,
progress=progress,
httpuser=self.args.httpuser,
httppassword=self.args.httppassword,
)
if ptype == "file":
if self.args.skipunzip:
raise Stop(0, "Unzip disabled, exiting")
log.info("Unzipping %s", server)
server = fileutils.unzip(server, match_dir=True, destdir=self.args.unzipdir)
log.debug("Server directory: %s", server)
return server
def stop(self):
try:
log.info("Stopping server")
self.bin("admin status --nodeonly")
self.bin("admin stop")
except Exception as e:
log.error("Error whilst stopping server: %s", e)
if not self.args.no_web:
try:
log.info("Stopping web")
self.stopweb()
except Exception as e:
log.error("Error whilst stopping web: %s", e)
def configure(self, copyold, prestartfile):
def samecontents(a, b):
# os.path.samefile is not available on Windows
try:
return os.path.samefile(a, b)
except AttributeError:
with open(a) as fa:
with open(b) as fb:
return fa.read() == fb.read()
target = self.dir / "etc" / "grid" / "config.xml"
if copyold:
from path import path
old_grid = path(self.args.sym) / "etc" / "grid"
old_cfg = old_grid / "config.xml"
log.info("Copying old configuration from %s", old_cfg)
if not old_cfg.exists():
raise Stop(40, "config.xml not found")
if target.exists() and samecontents(old_cfg, target):
# This likely is caused by the symlink being
# created early on an initial install.
pass
else:
old_cfg.copy(target)
else:
if target.exists():
log.info("Deleting configuration file %s", target)
target.remove()
if prestartfile:
for f in prestartfile:
log.info("Loading prestart file %s", f)
ftype, fpath = fileutils.get_as_local_path(f, "backup")
if ftype != "file":
raise Stop(50, "Expected file, found: %s %s" % (ftype, f))
self.run(["load", fpath])
def archive_logs(self):
if self.args.archivelogs:
logdir = os.path.join(self.args.sym, "var", "log")
archive = self.args.archivelogs
log.info("Archiving logs to %s", archive)
fileutils.zip(archive, logdir, os.path.join(self.args.sym, "var"))
return archive
def directories(self):
if self.samedir(self.dir, self.args.sym):
log.warn("Upgraded server was the same, not deleting")
return
try:
target = self.readlink(self.args.sym)
targetzip = target + ".zip"
except IOError:
log.error("Unable to get symlink target: %s", self.args.sym)
target = None
targetzip = None
if self.args.delete_old and target:
try:
log.info("Deleting %s", target)
shutil.rmtree(target)
except OSError as e:
log.error("Failed to delete %s: %s", target, e)
if not self.args.keep_old_zip and targetzip:
try:
log.info("Deleting %s", targetzip)
os.unlink(targetzip)
except OSError as e:
log.error("Failed to delete %s: %s", targetzip, e)
self.rmlink(self.args.sym)
self.symlink(self.dir, self.args.sym)
def init_db(self):
if self.args.initdb:
log.debug("Initialising database")
DbAdmin(self.dir, "init", self.args, self.external)
def upgrade_db(self):
if self.args.upgradedb:
log.debug("Upgrading database")
DbAdmin(self.dir, "upgrade", self.args, self.external)
def start(self):
if self.args.no_start:
log.debug("Not starting OMERO")
return
self.run("admin start")
if not self.args.no_web:
log.info("Starting web")
self.startweb()
def run(self, command):
"""
Runs a command as if from the command-line
without the need for using popen or subprocess
"""
if isinstance(command, basestring):
command = command.split()
else:
command = list(command)
self.external.omero_cli(command)
def bin(self, command):
"""
Runs the omero command-line client with an array of arguments using the
old environment
"""
if isinstance(command, basestring):
command = command.split()
self.external.omero_bin(command)
class PcapAnalyzeTaskProcessor(HSN2TaskProcessor):
parser = None
external = None
regex = re.compile("(?P<id>.*?)\s+(?P<time>.*?)\s+(?P<from>.+?)\s*->\s{1,3}(?P<to>.+?)\s{1,3}(?P<protocol>\w+)\s*(?P<port>\d+)(?P<rest>.*)")
ipRegex = re.compile("[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}", re.I)
dnsRegex = re.compile('Running as user "root"(.*)')
def __init__(self, connector, datastore, serviceName, serviceQueue, objectStoreQueue, **extra):
HSN2TaskProcessor.__init__(self, connector, datastore, serviceName, serviceQueue, objectStoreQueue, **extra)
def getPcapFilePath(self):
return self.dsAdapter.saveTmp(self.currentTask.job, self.objects[0].pcap_content.getKey())
def getRequestList(self, pcapFilePath, whitelist):
self.external = External()
outputLines = self.external.runExternal(["tshark", whitelist, "-r", pcapFilePath])[0].split("\n")
result = []
for line in outputLines:
match = re.search(self.regex, line.strip())
if match is None:
continue
tmp = {}
tmp["id"] = int(match.group("id").strip())
tmp["time"] = match.group("time").strip()
tmp["from"] = match.group("from").strip()
tmp["to"] = match.group("to").strip()
tmp["protocol"] = match.group("protocol").strip()
tmp["port"] = match.group("port").strip()
tmp["rest"] = match.group("rest").strip()
result.append(tmp)
return result
def processSaveIp(self, pcapFilePath):
outputLines = self.external.runExternal(["tshark", "-r", pcapFilePath, "-Tfields", "-e", "ip.addr"])[0].split("\n")
resultSet = set()
for line in outputLines:
line = line.strip()
if line == "":
continue
arr = line.split(",");
for ip in arr:
if re.match(self.ipRegex, ip) is not None:
resultSet.add(ip.strip())
resultList = list()
for ip in resultSet:
resultList.append(ip)
ipList = ow.toIpAddressList(resultList)
self.objects[0].addBytes("pcap_ip", self.dsAdapter.putBytes(bytes(ipList.SerializeToString()), self.currentTask.job))
def processSaveDns(self, pcapFilePath):
outputLines = self.external.runExternal(["tshark", "dns", "-r", pcapFilePath])[0].split("\n")
resultList = list()
for line in outputLines:
match = re.search(self.regex, line.strip())
if match is None:
continue
resultList.append(line)
dnsList = ow.toDnsList(resultList)
self.objects[0].addBytes("pcap_dns", self.dsAdapter.putBytes(bytes(dnsList.SerializeToString()), self.currentTask.job))
def processProtocolFilter(self, pcapFilePath, protocolFilter, protocolFields, protocolUnique):
protocolFieldsArray = protocolFields.split(",")
def prepare(x): return x.strip()
protocolFieldsArray = map(prepare, protocolFieldsArray)
tsharkExec = ["tshark", protocolFilter, "-r", pcapFilePath, "-Tfields"]
for field in protocolFieldsArray:
tsharkExec.append("-e")
tsharkExec.append(field)
outputLines = self.external.runExternal(tsharkExec)[0].split("\n")
if protocolUnique:
resultSet = set()
for line in outputLines:
line = line.strip();
if re.match(self.dnsRegex, line) or line == "":
continue
resultSet.add(line)
resultList = list()
for elem in resultSet:
resultList.append(elem)
else:
resultList = list()
for line in outputLines:
line = line.strip();
if re.match(self.dnsRegex, line) or line == "":
continue
resultList.append(line)
filterList = ow.toFilterList(resultList)
self.objects[0].addBytes("pcap_protocol_%s" % protocolFilter, self.dsAdapter.putBytes(bytes(filterList.SerializeToString()), self.currentTask.job))
def taskProcess(self):
logging.debug(self.__class__)
logging.debug(self.currentTask)
logging.debug(self.objects)
jobId = self.currentTask.job
taskId = self.currentTask.task_id
if len(self.objects) == 0:
raise ObjectStoreException("Task processing didn't find task object.")
if not self.objects[0].isSet("pcap_content"):
raise ParamException("pcap_content param is missing.")
configObj = Config()
configObj.getConfig()
whitelist = configObj.getWhiteList()
pcapFilePath = self.getPcapFilePath()
tsharkResultAll = self.getRequestList(pcapFilePath, "")
tsharkResultClean = self.getRequestList(pcapFilePath, whitelist)
cleanLen = len(tsharkResultClean)
allLen = len(tsharkResultAll)
pcapNetworkTrafic = (whitelist != "" and cleanLen < allLen) or (whitelist == "" and allLen == 0)
self.objects[0].addBool("pcap_network_traffic", pcapNetworkTrafic)
if configObj.getSaveIp():
self.processSaveIp(pcapFilePath)
if configObj.getSaveDns():
self.processSaveDns(pcapFilePath)
if configObj.hasProtocolFilter():
self.processProtocolFilter(pcapFilePath, configObj.getProtocolFilter(), configObj.getProtocolFields(), configObj.getProtocolUnique())
return []
class Install(object):
def __init__(self, cmd, args):
self.args, newinstall = self._handle_args(cmd, args)
log.info("%s: %s", self.__class__.__name__, cmd)
log.debug("Current directory: %s", os.getcwd())
self.symlink_check_and_set()
if newinstall is None:
# Automatically install or upgrade
newinstall = not os.path.exists(args.sym)
elif newinstall is False:
if not os.path.exists(args.sym):
raise Stop(30, 'Symlink is missing: %s' % args.sym)
elif newinstall is True:
if os.path.exists(args.sym):
raise Stop(30, 'Symlink already exists: %s' % args.sym)
else:
assert False
server_dir = self.get_server_dir()
if newinstall:
# Create a symlink to simplify the rest of the logic-
# just need to check if OLD == NEW
self.symlink(server_dir, args.sym)
log.info("Installing %s (%s)...", server_dir, args.sym)
else:
log.info("Upgrading %s (%s)...", server_dir, args.sym)
self.external = External(server_dir)
self.external.setup_omero_cli()
if not newinstall:
self.external.setup_previous_omero_env(args.sym, args.savevarsfile)
# Need lib/python set above
import path
self.dir = path.path(server_dir)
if not newinstall:
self.stop()
self.archive_logs()
copyold = not newinstall and not args.ignoreconfig
self.configure(copyold, args.prestartfile)
self.directories()
self.handle_database()
self.external.save_env_vars(args.savevarsfile, args.savevars.split())
self.start()
def _handle_args(self, cmd, args):
"""
We need to support deprecated behaviour for now which makes this
quite complicated
Current behaviour:
- install: Installs a new server, existing server causes an error
- install --upgrade: Installs or upgrades a server
- install --managedb: Automatically initialise or upgrade the db
Deprecated:
- install --upgradedb --initdb: Replaced by install --managedb
- install --upgradedb: upgrade the db, must exist
- install --initdb: initialise the db
- upgrade: Upgrades a server, must already exist
- upgrade --upgradedb: Automatically upgrade the db
returns:
- Modified args object, flag to indicate new/existing/auto install
"""
if cmd == 'install':
if args.upgrade:
# Current behaviour: install or upgrade
if args.initdb or args.upgradedb:
raise Stop(10, ('Deprecated --initdb --upgradedb flags '
'are incompatible with --upgrade'))
newinstall = None
else:
# Current behaviour: Server must not exist
newinstall = True
if args.managedb:
# Current behaviour
if args.initdb or args.upgradedb:
raise Stop(10, ('Deprecated --initdb --upgradedb flags '
'are incompatible with --managedb'))
args.initdb = True
args.upgradedb = True
else:
if args.initdb or args.upgradedb:
log.warn('--initdb and --upgradedb are deprecated, '
'use --managedb')
elif cmd == 'upgrade':
# Deprecated behaviour
log.warn(
'"omero upgrade" is deprecated, use "omego install --upgrade"')
cmd = 'install'
args.upgrade = True
# Deprecated behaviour: Server must exist
newinstall = False
else:
raise Exception('Unexpected command: %s' % cmd)
return args, newinstall
def get_server_dir(self):
"""
Either downloads and/or unzips the server if necessary
return: the directory of the unzipped server
"""
if not self.args.server:
if self.args.skipunzip:
raise Stop(0, 'Unzip disabled, exiting')
log.info('Downloading server')
# The downloader automatically symlinks the server, however if
# we are upgrading we want to delay the symlink swap, so this
# overrides args.sym
# TODO: Find a nicer way to do this?
artifact_args = copy.copy(self.args)
artifact_args.sym = ''
artifacts = Artifacts(artifact_args)
server = artifacts.download('server')
else:
progress = 0
if self.args.verbose:
progress = 20
ptype, server = fileutils.get_as_local_path(
self.args.server,
self.args.overwrite,
progress=progress,
httpuser=self.args.httpuser,
httppassword=self.args.httppassword)
if ptype == 'file':
if self.args.skipunzip:
raise Stop(0, 'Unzip disabled, exiting')
log.info('Unzipping %s', server)
server = fileutils.unzip(server,
match_dir=True,
destdir=self.args.unzipdir)
log.debug('Server directory: %s', server)
return server
def stop(self):
try:
log.info("Stopping server")
self.bin("admin status --nodeonly")
self.bin("admin stop")
except Exception as e:
log.error('Error whilst stopping server: %s', e)
if not self.args.no_web:
try:
log.info("Stopping web")
self.stopweb()
except Exception as e:
log.error('Error whilst stopping web: %s', e)
def configure(self, copyold, prestartfile):
def samecontents(a, b):
# os.path.samefile is not available on Windows
try:
return os.path.samefile(a, b)
except AttributeError:
with open(a) as fa:
with open(b) as fb:
return fa.read() == fb.read()
target = self.dir / "etc" / "grid" / "config.xml"
if copyold:
from path import path
old_grid = path(self.args.sym) / "etc" / "grid"
old_cfg = old_grid / "config.xml"
log.info("Copying old configuration from %s", old_cfg)
if not old_cfg.exists():
raise Stop(40, 'config.xml not found')
if target.exists() and samecontents(old_cfg, target):
# This likely is caused by the symlink being
# created early on an initial install.
pass
else:
old_cfg.copy(target)
else:
if target.exists():
log.info('Deleting configuration file %s', target)
target.remove()
if prestartfile:
for f in prestartfile:
log.info('Loading prestart file %s', f)
ftype, fpath = fileutils.get_as_local_path(f, 'backup')
if ftype != 'file':
raise Stop(50, 'Expected file, found: %s %s' % (ftype, f))
self.run(['load', fpath])
def archive_logs(self):
if self.args.archivelogs:
logdir = os.path.join(self.args.sym, 'var', 'log')
archive = self.args.archivelogs
log.info('Archiving logs to %s', archive)
fileutils.zip(archive, logdir, os.path.join(self.args.sym, 'var'))
return archive
def directories(self):
if self.samedir(self.dir, self.args.sym):
log.warn("Upgraded server was the same, not deleting")
return
try:
target = self.readlink(self.args.sym)
targetzip = target + '.zip'
except IOError:
log.error('Unable to get symlink target: %s', self.args.sym)
target = None
targetzip = None
if self.args.delete_old and target:
try:
log.info("Deleting %s", target)
shutil.rmtree(target)
except OSError as e:
log.error("Failed to delete %s: %s", target, e)
if not self.args.keep_old_zip and targetzip:
try:
log.info("Deleting %s", targetzip)
os.unlink(targetzip)
except OSError as e:
log.error("Failed to delete %s: %s", targetzip, e)
self.rmlink(self.args.sym)
self.symlink(self.dir, self.args.sym)
def handle_database(self):
"""
Handle database initialisation and upgrade, taking into account
command line arguments
"""
# TODO: When initdb and upgradedb are dropped we can just test
# managedb, but for backwards compatibility we need to support
# initdb without upgradedb and vice-versa
if self.args.initdb or self.args.upgradedb:
db = DbAdmin(self.dir, None, self.args, self.external)
status = db.check()
log.debug('OMERO database upgrade status: %s', status)
else:
log.warn('OMERO database check disabled')
return DB_INIT_NEEDED
if status == DB_INIT_NEEDED:
if self.args.initdb:
log.debug('Initialising OMERO database')
db.init()
else:
log.error('OMERO database not found')
raise Stop(DB_INIT_NEEDED,
'Install/Upgrade failed: OMERO database not found')
elif status == DB_UPGRADE_NEEDED:
log.warn('OMERO database exists but is out of date')
if self.args.upgradedb:
log.debug('Upgrading OMERO database')
db.upgrade()
else:
raise Stop(
DB_UPGRADE_NEEDED,
'Pass --managedb or upgrade your OMERO database manually')
else:
assert status == DB_UPTODATE
return status
def start(self):
if self.args.no_start:
log.debug('Not starting OMERO')
return
self.run("admin start")
if not self.args.no_web:
log.info("Starting web")
self.startweb()
def run(self, command):
"""
Runs a command as if from the command-line
without the need for using popen or subprocess
"""
if isinstance(command, basestring):
command = command.split()
else:
command = list(command)
self.external.omero_cli(command)
def bin(self, command):
"""
Runs the omero command-line client with an array of arguments using the
old environment
"""
if isinstance(command, basestring):
command = command.split()
self.external.omero_bin(command)
def symlink_check_and_set(self):
"""
The default symlink was changed from OMERO-CURRENT to OMERO.server.
If `--sym` was not specified and OMERO-CURRENT exists in the current
directory stop and warn.
"""
if self.args.sym == '':
if os.path.exists('OMERO-CURRENT'):
log.error('Deprecated OMERO-CURRENT found but --sym not set')
raise Stop(
30, 'The default for --sym has changed to OMERO.server '
'but the current directory contains OMERO-CURRENT. '
'Either remove OMERO-CURRENT or explicity pass --sym.')
if self.args.sym in ('', 'auto'):
self.args.sym = 'OMERO.server'