def get_server_dir(self):
"""
Either downloads and/or unzips the server if necessary
return: the directory of the unzipped server
"""
if not self.args.server:
if self.args.skipunzip:
raise Stop(0, 'Unzip disabled, exiting')
log.info('Downloading server')
artifacts = Artifacts(self.args)
server = artifacts.download('server')
else:
progress = 0
if self.args.verbose:
progress = 20
ptype, server = fileutils.get_as_local_path(
self.args.server,
self.args.overwrite,
progress=progress,
httpuser=self.args.httpuser,
httppassword=self.args.httppassword)
if ptype == 'file':
if self.args.skipunzip:
raise Stop(0, 'Unzip disabled, exiting')
log.info('Unzipping %s', server)
server = fileutils.unzip(server,
match_dir=True,
destdir=self.args.unzipdir)
log.debug('Server directory: %s', server)
return server
def doc_deploy(self):
"""
Deploy a new content using symlink swapping.
Two symlinks get replaced during the lifetime of the script. Both
operations are atomic.
"""
if not os.path.exists(self.live_folder):
raise Stop(5, "The following path does not exist: %s. "
"Pass --init to the scc deploy command to initialize "
"the symlink swapping." % self.live_folder)
if not os.path.islink(self.folder):
raise Stop(5, "The following path is not a symlink: %s. "
"Pass --init to the scc deploy command to initialize "
"the symlink swapping." % self.folder)
if not os.path.exists(self.tmp_folder):
raise Stop(5, "The following path does not exist: %s. "
"Copy the new content to be deployed to this folder "
"and run scc deploy again." % self.tmp_folder)
self.symlink(self.tmp_folder, self.folder)
self.rmtree(self.live_folder)
self.copytree(self.tmp_folder, self.live_folder)
self.symlink(self.live_folder, self.folder)
self.rmtree(self.tmp_folder)
def upgrade(self, check=False):
try:
currentsqlv = '%s__%s' % self.get_current_db_version()
except RunException as e:
log.error(e)
if check:
return DB_INIT_NEEDED
raise Stop(DB_INIT_NEEDED, 'Unable to get database version')
M, versions = self.sql_version_matrix()
latestsqlv = versions[-1]
if latestsqlv == currentsqlv:
log.info('Database is already at %s', latestsqlv)
if check:
return DB_UPTODATE
else:
ugpath = self.sql_version_resolve(M, versions, currentsqlv)
log.debug('Database upgrade path: %s', ugpath)
if check:
return DB_UPGRADE_NEEDED
if self.args.dry_run:
raise Stop(
DB_UPGRADE_NEEDED, 'Database upgrade required %s->%s' % (
currentsqlv, latestsqlv))
for upgradesql in ugpath:
log.info('Upgrading database using %s', upgradesql)
self.psql('-f', upgradesql)
def get_latest_runs(self, root):
"""
Jenkins has a bug whereby it may return matrix sub-builds for older
runs from different nodes in addition to the latest one, so we need
to compare each run with the current build number
"""
rurl = [u.text for u in root.findall('./url')]
if len(rurl) != 1:
log.error('Expected one root url, found %d: %s', len(rurl), rurl)
raise Stop(20, 'Failed to parse CI XML')
rurl = rurl[0]
log.debug('Root url: %s', rurl)
try:
build = re.search('/(\d+)/?$', rurl).group(1)
except:
log.error('Failed to extract build number from url: %s', rurl)
raise Stop(20, 'Failed to parse CI XML')
runs = root.findall('./run')
runurls = [
r.find('url').text for r in runs if r.find('number').text == build
]
log.debug('Child runs: %s', runurls)
if len(runurls) < 1:
log.error('No runs found in build: %s', rurl)
raise Stop(20, 'Failed to parse CI XML')
return runurls
def _handle_args(self, cmd, args):
"""
We need to support deprecated behaviour for now which makes this
quite complicated
Current behaviour:
- install: Installs a new server, existing server causes an error
- install --upgrade: Installs or upgrades a server
- install --managedb: Automatically initialise or upgrade the db
Deprecated:
- install --upgradedb --initdb: Replaced by install --managedb
- install --upgradedb: upgrade the db, must exist
- install --initdb: initialise the db
- upgrade: Upgrades a server, must already exist
- upgrade --upgradedb: Automatically upgrade the db
returns:
- Modified args object, flag to indicate new/existing/auto install
"""
if cmd == 'install':
if args.upgrade:
# Current behaviour: install or upgrade
if args.initdb or args.upgradedb:
raise Stop(10, ('Deprecated --initdb --upgradedb flags '
'are incompatible with --upgrade'))
newinstall = None
else:
# Current behaviour: Server must not exist
newinstall = True
if args.managedb:
# Current behaviour
if args.initdb or args.upgradedb:
raise Stop(10, ('Deprecated --initdb --upgradedb flags '
'are incompatible with --managedb'))
args.initdb = True
args.upgradedb = True
else:
if args.initdb or args.upgradedb:
log.warn('--initdb and --upgradedb are deprecated, '
'use --managedb')
elif cmd == 'upgrade':
# Deprecated behaviour
log.warn(
'"omero upgrade" is deprecated, use "omego install --upgrade"')
cmd = 'install'
args.upgrade = True
# Deprecated behaviour: Server must exist
newinstall = False
else:
raise Exception('Unexpected command: %s' % cmd)
return args, newinstall
def __init__(self, cmd, args):
self.args = args
log.info("%s: %s", self.__class__.__name__, cmd)
log.debug("Current directory: %s", os.getcwd())
if cmd == 'upgrade':
newinstall = False
if not os.path.exists(args.sym):
raise Stop(30, 'Symlink is missing: %s' % args.sym)
elif cmd == 'install':
newinstall = True
if os.path.exists(args.sym):
raise Stop(30, 'Symlink already exists: %s' % args.sym)
else:
raise Exception('Unexpected command: %s' % cmd)
server_dir = self.get_server_dir()
if newinstall:
# Create a symlink to simplify the rest of the logic-
# just need to check if OLD == NEW
self.symlink(server_dir, args.sym)
log.info("Installing %s (%s)...", server_dir, args.sym)
else:
log.info("Upgrading %s (%s)...", server_dir, args.sym)
self.external = External(server_dir)
self.external.setup_omero_cli()
if not newinstall:
self.external.setup_previous_omero_env(args.sym, args.savevarsfile)
# Need lib/python set above
import path
self.dir = path.path(server_dir)
if not newinstall:
self.stop()
self.archive_logs()
copyold = not newinstall and not args.ignoreconfig
self.configure(copyold, args.prestartfile)
self.directories()
if newinstall:
self.init_db()
self.upgrade_db()
self.external.save_env_vars(args.savevarsfile, args.savevars.split())
self.start()
def __init__(self, cmd, args):
self.args, newinstall = self._handle_args(cmd, args)
log.info("%s: %s", self.__class__.__name__, cmd)
log.debug("Current directory: %s", os.getcwd())
self.symlink_check_and_set()
if newinstall is None:
# Automatically install or upgrade
newinstall = not os.path.exists(args.sym)
elif newinstall is False:
if not os.path.exists(args.sym):
raise Stop(30, 'Symlink is missing: %s' % args.sym)
elif newinstall is True:
if os.path.exists(args.sym):
raise Stop(30, 'Symlink already exists: %s' % args.sym)
else:
assert False
server_dir = self.get_server_dir()
if newinstall:
# Create a symlink to simplify the rest of the logic-
# just need to check if OLD == NEW
self.symlink(server_dir, args.sym)
log.info("Installing %s (%s)...", server_dir, args.sym)
else:
log.info("Upgrading %s (%s)...", server_dir, args.sym)
self.external = External(server_dir)
self.external.setup_omero_cli()
if not newinstall:
self.external.setup_previous_omero_env(args.sym, args.savevarsfile)
# Need lib/python set above
import path
self.dir = path.path(server_dir)
if not newinstall:
self.stop()
self.archive_logs()
copyold = not newinstall and not args.ignoreconfig
self.configure(copyold, args.prestartfile)
self.directories()
self.handle_database()
self.external.save_env_vars(args.savevarsfile, args.savevars.split())
self.start()
def __call__(self, args):
super(DbCommand, self).__call__(args)
self.configure_logging(args)
# Since EnvDefault.__action__ is only called if a user actively passes
# a variable, there's no way to do the string replacing in the action
# itself. Instead, we're post-processing them here, but this could be
# improved.
names = sorted(x.dest for x in self.parser._actions)
for dest in names:
if dest in ("help", "verbose", "quiet"):
continue
value = getattr(args, dest)
if value and isinstance(value, basestring):
replacement = value % dict(args._get_kwargs())
log.debug("% 20s => %s" % (dest, replacement))
setattr(args, dest, replacement)
if args.serverdir:
d = args.serverdir
else:
raise Stop(1, 'OMERO server directory required')
ext = External(d)
ext.setup_omero_cli()
DbAdmin(d, args.dbcommand, args, ext)
def __init__(self, args):
self.args = args
if re.match('[A-Za-z]\w+-\w+', args.branch):
self.artifacts = JenkinsArtifacts(args)
elif re.match('[0-9]+|latest$', args.branch):
self.artifacts = ReleaseArtifacts(args)
else:
log.error('Invalid release or job name: %s', args.branch)
raise Stop(20, 'Invalid release or job name: %s', args.branch)
def doc_init(self):
"""
Set up the symlink swapping structure to use the deployment script.
"""
if not os.path.exists(self.folder):
raise Stop(5, "The following path does not exist: %s. "
"Copy some contents to this folder and run"
" scc deploy --init again." % self.folder)
if os.path.exists(self.live_folder):
raise Stop(5, "The following path already exists: %s. "
"Run the scc deploy command without the --init"
" argument." % self.live_folder)
self.copytree(self.folder, self.live_folder)
self.rmtree(self.folder)
self.symlink(self.live_folder, self.folder)
def read_xml(self, buildurl):
url = None
try:
url = fileutils.open_url(buildurl + 'api/xml')
log.debug('Fetching xml from %s code:%d', url.url, url.code)
if url.code != 200:
log.error('Failed to get CI XML from %s (code %d)', url.url,
url.code)
raise Stop(20, 'Job lookup failed, is the job name correct?')
ci_xml = url.read()
except HTTPError as e:
log.error('Failed to get CI XML (%s)', e)
raise Stop(20, 'Job lookup failed, is the job name correct?')
finally:
if url:
url.close()
root = XML(ci_xml)
return root
def handle_database(self):
"""
Handle database initialisation and upgrade, taking into account
command line arguments
"""
# TODO: When initdb and upgradedb are dropped we can just test
# managedb, but for backwards compatibility we need to support
# initdb without upgradedb and vice-versa
if self.args.initdb or self.args.upgradedb:
db = DbAdmin(self.dir, None, self.args, self.external)
status = db.check()
log.debug('OMERO database upgrade status: %s', status)
else:
log.warn('OMERO database check disabled')
return DB_INIT_NEEDED
if status == DB_INIT_NEEDED:
if self.args.initdb:
log.debug('Initialising OMERO database')
db.init()
else:
log.error('OMERO database not found')
raise Stop(DB_INIT_NEEDED,
'Install/Upgrade failed: OMERO database not found')
elif status == DB_UPGRADE_NEEDED:
log.warn('OMERO database exists but is out of date')
if self.args.upgradedb:
log.debug('Upgrading OMERO database')
db.upgrade()
else:
raise Stop(
DB_UPGRADE_NEEDED,
'Pass --managedb or upgrade your OMERO database manually')
else:
assert status == DB_UPTODATE
return status
def get_server_dir(self):
"""
Either downloads and/or unzips the server if necessary
return: the directory of the unzipped server
"""
if not self.args.server:
if self.args.skipunzip:
raise Stop(0, 'Unzip disabled, exiting')
log.info('Downloading server')
# The downloader automatically symlinks the server, however if
# we are upgrading we want to delay the symlink swap, so this
# overrides args.sym
# TODO: Find a nicer way to do this?
artifact_args = copy.copy(self.args)
artifact_args.sym = ''
artifacts = Artifacts(artifact_args)
server = artifacts.download('server')
else:
progress = 0
if self.args.verbose:
progress = 20
ptype, server = fileutils.get_as_local_path(
self.args.server,
self.args.overwrite,
progress=progress,
httpuser=self.args.httpuser,
httppassword=self.args.httppassword)
if ptype == 'file':
if self.args.skipunzip:
raise Stop(0, 'Unzip disabled, exiting')
log.info('Unzipping %s', server)
server = fileutils.unzip(server,
match_dir=True,
destdir=self.args.unzipdir)
log.debug('Server directory: %s', server)
return server
def follow_latest_redirect(self, args):
ver = ''
if args.branch != 'latest':
ver = args.branch
try:
latesturl = '%s/latest/omero%s' % (args.downloadurl, ver)
finalurl = fileutils.dereference_url(latesturl)
log.debug('Checked %s: %s', latesturl, finalurl)
except HTTPError as e:
log.error('Invalid URL %s: %s', latesturl, e)
raise Stop(20, 'Invalid latest URL, is the version correct?')
return finalurl
def configure(self, copyold, prestartfile):
def samecontents(a, b):
# os.path.samefile is not available on Windows
try:
return os.path.samefile(a, b)
except AttributeError:
with open(a) as fa:
with open(b) as fb:
return fa.read() == fb.read()
target = self.dir / "etc" / "grid" / "config.xml"
if copyold:
from path import path
old_grid = path(self.args.sym) / "etc" / "grid"
old_cfg = old_grid / "config.xml"
log.info("Copying old configuration from %s", old_cfg)
if not old_cfg.exists():
raise Stop(40, 'config.xml not found')
if target.exists() and samecontents(old_cfg, target):
# This likely is caused by the symlink being
# created early on an initial install.
pass
else:
old_cfg.copy(target)
else:
if target.exists():
log.info('Deleting configuration file %s', target)
target.remove()
if prestartfile:
for f in prestartfile:
log.info('Loading prestart file %s', f)
ftype, fpath = fileutils.get_as_local_path(f, 'backup')
if ftype != 'file':
raise Stop(50, 'Expected file, found: %s %s' % (ftype, f))
self.run(['load', fpath])
def symlink_check_and_set(self):
"""
The default symlink was changed from OMERO-CURRENT to OMERO.server.
If `--sym` was not specified and OMERO-CURRENT exists in the current
directory stop and warn.
"""
if self.args.sym == '':
if os.path.exists('OMERO-CURRENT'):
log.error('Deprecated OMERO-CURRENT found but --sym not set')
raise Stop(
30, 'The default for --sym has changed to OMERO.server '
'but the current directory contains OMERO-CURRENT. '
'Either remove OMERO-CURRENT or explicity pass --sym.')
if self.args.sym in ('', 'auto'):
self.args.sym = 'OMERO.server'
def read_downloads(dlurl):
url = None
parser = HtmlHrefParser()
try:
url = fileutils.open_url(dlurl)
log.debug('Fetching html from %s code:%d', url.url, url.code)
if url.code != 200:
log.error('Failed to get HTML from %s (code %d)', url.url,
url.code)
raise Stop(20,
'Downloads page failed, is the version correct?')
parser.feed(url.read())
except HTTPError as e:
log.error('Failed to get HTML from %s (%s)', dlurl, e)
raise Stop(20, 'Downloads page failed, is the version correct?')
finally:
if url:
url.close()
dl_icever = {}
for href in parser.hrefs:
try:
icever = re.search('-(ice\d+).*zip$', href).group(1)
if re.match('\w+://', href):
fullurl = href
else:
fullurl = dlurl + href
try:
dl_icever[icever].append(fullurl)
except KeyError:
dl_icever[icever] = [fullurl]
log.debug('Found artifact: %s', fullurl)
except AttributeError:
pass
return dl_icever
def find_label_matches(self, urls):
required = set(self.args.labels.split(','))
if '' in required:
required.remove('')
log.debug('Searching for matrix runs matching: %s', required)
matches = []
for url in urls:
url_labels = self.label_list_parser(url)
if len(required.intersection(url_labels)) == len(required):
matches.append(url)
if len(matches) != 1:
log.error('Found %d matching matrix build runs: %s', len(matches),
matches)
raise Stop(30,
'Expected one matching run, found %d' % len(matches))
return matches[0]
def open_url(url, httpuser=None, httppassword=None, method=None):
"""
Open a URL using an opener that will simulate a browser user-agent
url: The URL
httpuser, httppassword: HTTP authentication credentials (either both or
neither must be provided)
method: The HTTP method
Caller is reponsible for calling close() on the returned object
"""
if os.getenv('OMEGO_SSL_NO_VERIFY') == '1':
# This needs to come first to override the default HTTPS handler
log.debug('OMEGO_SSL_NO_VERIFY=1')
try:
sslctx = ssl.create_default_context()
except Exception as e:
log.error('Failed to create Default SSL context: %s' % e)
raise Stop(
'Failed to create Default SSL context, OMEGO_SSL_NO_VERIFY '
'is not supported on older versions of Python')
sslctx.check_hostname = False
sslctx.verify_mode = ssl.CERT_NONE
opener = urllib2.build_opener(urllib2.HTTPSHandler(context=sslctx))
else:
opener = urllib2.build_opener()
if 'USER_AGENT' in os.environ:
opener.addheaders = [('User-agent', os.environ.get('USER_AGENT'))]
log.debug('Setting user-agent: %s', os.environ.get('USER_AGENT'))
if httpuser and httppassword:
mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
mgr.add_password(None, url, httpuser, httppassword)
log.debug('Enabling HTTP authentication')
opener.add_handler(urllib2.HTTPBasicAuthHandler(mgr))
opener.add_handler(urllib2.HTTPDigestAuthHandler(mgr))
elif httpuser or httppassword:
raise FileException('httpuser and httppassword must be used together',
url)
# Override method http://stackoverflow.com/a/4421485
req = urllib2.Request(url)
if method:
req.get_method = lambda: method
return opener.open(req)
def download(self, component):
componenturl = self.artifacts.get(component)
if not componenturl:
raise Exception("No %s found" % component)
filename = os.path.basename(componenturl)
unzipped = filename.replace(".zip", "")
if os.path.exists(unzipped):
self.create_symlink(unzipped)
return unzipped
log.info("Checking %s", componenturl)
if self.args.dry_run:
return
progress = 0
if self.args.verbose:
progress = 20
ptype, localpath = fileutils.get_as_local_path(
componenturl,
self.args.overwrite,
progress=progress,
httpuser=self.args.httpuser,
httppassword=self.args.httppassword)
if ptype != 'file':
raise ArtifactException('Expected local file', localpath)
if not self.args.skipunzip:
if localpath.endswith('.zip'):
try:
log.info('Unzipping %s', localpath)
unzipped = fileutils.unzip(localpath,
match_dir=True,
destdir=self.args.unzipdir)
self.create_symlink(unzipped)
return unzipped
except Exception as e:
log.error('Unzip failed: %s', e)
print e
raise Stop(20, 'Unzip failed, try unzipping manually')
else:
log.warn('Not unzipping %s', localpath)
return localpath
def find_label_matches(self, urls, icever=None):
# The Ice version is handled as a matrix label in the CI jobs
required = set(self.args.labels.split(','))
if '' in required:
required.remove('')
if icever:
required.add('ICE=%s' % icever)
log.debug('Searching for matrix runs matching: %s', required)
matches = []
for url in urls:
url_labels = self.label_list_parser(url)
if len(required.intersection(url_labels)) == len(required):
matches.append(url)
if len(matches) != 1:
log.error('Found %d matching matrix build runs: %s', len(matches),
matches)
raise Stop(30,
'Expected one matching run, found %d' % len(matches))
return matches[0]
def init(self):
omerosql = self.args.omerosql
autoupgrade = False
if not omerosql:
omerosql = fileutils.timestamp_filename('omero', 'sql')
log.info('Creating SQL: %s', omerosql)
if not self.args.dry_run:
self.external.omero_cli(
["db", "script", "-f", omerosql, "", "",
self.args.rootpass])
elif os.path.exists(omerosql):
log.info('Using existing SQL: %s', omerosql)
autoupgrade = True
else:
log.error('SQL file not found: %s', omerosql)
raise Stop(40, 'SQL file not found')
log.info('Creating database using %s', omerosql)
if not self.args.dry_run:
self.psql('-f', omerosql)
if autoupgrade:
self.upgrade()
def __init__(self, dir, command, args, external):
self.dir = dir
self.args = args
log.info("%s: DbAdmin %s ...", self.__class__.__name__, dir)
# TODO: If the server has already been configured we should use the
# OMERO db credentials if not explicitly provided in args
# Server directory
if not os.path.exists(dir):
raise Exception("%s does not exist!" % dir)
self.external = external
psqlv = self.psql('--version')
log.info('psql version: %s', psqlv)
self.check_connection()
if command in ('init', 'upgrade', 'dump'):
getattr(self, command)()
elif command is not None:
raise Stop('Invalid db command: %s', command)
def check_connection(self):
try:
self.psql('-c', r'\conninfo')
except RunException as e:
log.error(e)
raise Stop(30, 'Database connection check failed')