def __init__(self, args):
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running()
self.connection = Connection(server_url, token=get_token())
self.filemanager = FileManager(server_url)
def __init__(self, args=None):
if args is None:
args = self._get_args()
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
class FileTagList(object):
def __init__(self, args=None, silent=False):
if args is None:
args = self._get_args()
self.args = args
self.silent = silent
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
def _get_args(self):
self.parser = self.get_parser()
return self.parser.parse_args()
@classmethod
def get_parser(cls, parser=None):
# If called from main, use the subparser provided.
# Otherwise create a top-level parser here.
if parser is None:
parser = argparse.ArgumentParser(__file__)
parser.add_argument(
'target',
metavar='TARGET',
nargs='?',
help='show tags only for the specified file')
return parser
def run(self):
if self.args.target:
try:
files = self.connection.get_data_object_index(
min=1, max=1,
query_string=self.args.target, type='file')
except LoomengineUtilsError as e:
raise SystemExit(
"ERROR! Failed to get data object list: '%s'" % e)
try:
tag_data = self.connection.list_data_tags(files[0]['uuid'])
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to get tag list: '%s'" % e)
tags = tag_data.get('tags', [])
else:
try:
tag_list = self.connection.get_data_tag_index()
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to get tag list: '%s'" % e)
tags = [item.get('tag') for item in tag_list]
if not self.silent:
print '[showing %s tags]' % len(tags)
for tag in tags:
print tag
class RunLabelAdd(object):
"""Add a new run labels
"""
def __init__(self, args=None, silent=False):
# Args may be given as an input argument for testing purposes
# or from the main parser.
# Otherwise get them from the parser.
if args is None:
args = self._get_args()
self.args = args
self.silent = silent
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
def _get_args(self):
self.parser = self.get_parser()
return self.parser.parse_args()
@classmethod
def get_parser(cls, parser=None):
# If called from main, use the subparser provided.
# Otherwise create a top-level parser here.
if parser is None:
parser = argparse.ArgumentParser(__file__)
parser.add_argument(
'target',
metavar='TARGET',
help='identifier for run to be labeled')
parser.add_argument(
'label',
metavar='LABEL', help='label name to be added')
return parser
def run(self):
try:
runs = self.connection.get_run_index(
min=1, max=1,
query_string=self.args.target)
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to get run list: '%s'" % e)
label_data = {'label': self.args.label}
try:
label = self.connection.post_run_label(runs[0]['uuid'], label_data)
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to create label: '%s'" % e)
if not self.silent:
print 'Target "%s@%s" has been labeled as "%s"' % \
(runs[0].get('name'),
runs[0].get('uuid'),
label.get('label'))
def __init__(self, args):
"""Common init tasks for all Importer classes
"""
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
token = get_token()
self.filemanager = FileManager(server_url, token=token)
self.connection = Connection(server_url, token=token)
def __init__(self, args=None):
# Parse arguments
if args is None:
args = _get_args()
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.args = args
self._set_run_function()
self.connection = Connection(server_url, token=None)
class FileTagAdd(object):
"""Add a new file tags
"""
def __init__(self, args=None, silent=False):
# Args may be given as an input argument for testing purposes
# or from the main parser.
# Otherwise get them from the parser.
if args is None:
args = self._get_args()
self.args = args
self.silent = silent
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
def _get_args(self):
self.parser = self.get_parser()
return self.parser.parse_args()
@classmethod
def get_parser(cls, parser=None):
# If called from main, use the subparser provided.
# Otherwise create a top-level parser here.
if parser is None:
parser = argparse.ArgumentParser(__file__)
parser.add_argument(
'target',
metavar='TARGET',
help='identifier for file to be tagged')
parser.add_argument(
'tag',
metavar='TAG', help='tag name to be added')
return parser
def run(self):
try:
files = self.connection.get_data_object_index(
min=1, max=1,
query_string=self.args.target, type='file')
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to get data object list: '%s'" % e)
tag_data = {'tag': self.args.tag}
try:
tag = self.connection.post_data_tag(files[0]['uuid'], tag_data)
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to create tag: '%s'" % e)
if not self.silent:
print 'Target "%s@%s" has been tagged as "%s"' % \
(files[0]['value'].get('filename'),
files[0].get('uuid'),
tag.get('tag'))
def __init__(self, args=None):
# Args may be given as an input argument for testing purposes
# or from the main parser.
# Otherwise get them from the parser.
if args is None:
args = self._get_args()
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
class TemplateLabelRemove(object):
"""Remove a template label
"""
def __init__(self, args=None, silent=False):
if args is None:
args = self._get_args()
self.args = args
self.silent = silent
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
def _get_args(self):
self.parser = self.get_parser()
return self.parser.parse_args()
@classmethod
def get_parser(cls, parser=None):
# If called from main, use the subparser provided.
# Otherwise create a top-level parser here.
if parser is None:
parser = argparse.ArgumentParser(__file__)
parser.add_argument(
'target',
metavar='TARGET',
help='identifier for template to be unlabeled')
parser.add_argument(
'label',
metavar='LABEL', help='label name to be removed')
return parser
def run(self):
try:
templates = self.connection.get_template_index(
min=1, max=1,
query_string=self.args.target)
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to get template list: '%s'" % e)
label_data = {'label': self.args.label}
try:
label = self.connection.remove_template_label(
templates[0]['uuid'], label_data)
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to remove label: '%s'" % e)
if not self.silent:
print 'Label %s has been removed from template "%s@%s"' % \
(label.get('label'),
templates[0].get('name'),
templates[0].get('uuid'))
class RunTagRemove(object):
"""Remove a run tag
"""
def __init__(self, args=None, silent=False):
if args is None:
args = self._get_args()
self.args = args
self.silent = silent
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
def _get_args(self):
self.parser = self.get_parser()
return self.parser.parse_args()
@classmethod
def get_parser(cls, parser=None):
# If called from main, use the subparser provided.
# Otherwise create a top-level parser here.
if parser is None:
parser = argparse.ArgumentParser(__file__)
parser.add_argument(
'target',
metavar='TARGET',
help='identifier for run to be untagged')
parser.add_argument(
'tag',
metavar='TAG', help='tag name to be removed')
return parser
def run(self):
try:
runs = self.connection.get_run_index(
min=1, max=1,
query_string=self.args.target)
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to get run list: '%s'" % e)
tag_data = {'tag': self.args.tag}
try:
tag = self.connection.remove_run_tag(runs[0]['uuid'], tag_data)
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to remove tag: '%s'" % e)
print 'Tag %s has been removed from run "%s@%s"' % \
(tag.get('tag'),
runs[0].get('name'),
runs[0].get('uuid'))
def __call__(self, parser, namespace, values, option_string):
if not has_connection_settings():
server_version = 'not connected'
else:
url = get_server_url()
if not is_server_running(url=url):
server_version = 'no response'
else:
connection = Connection(url)
server_version = connection.get_version()
print "client version: %s" % loomengine_utils.version.version()
print "server version: %s" % server_version
exit(0)
class RunLabelList(object):
def __init__(self, args=None):
if args is None:
args = self._get_args()
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
def _get_args(self):
self.parser = self.get_parser()
return self.parser.parse_args()
@classmethod
def get_parser(cls, parser=None):
# If called from main, use the subparser provided.
# Otherwise create a top-level parser here.
if parser is None:
parser = argparse.ArgumentParser(__file__)
parser.add_argument(
'target',
metavar='TARGET',
nargs='?',
help='show labels only for the specified run')
return parser
def run(self):
if self.args.target:
runs = self.connection.get_run_index(
min=1, max=1,
query_string=self.args.target)
label_data = self.connection.list_run_labels(runs[0]['uuid'])
labels = label_data.get('labels', [])
print '[showing %s labels]' % len(labels)
for label in labels:
print label
else:
label_list = self.connection.get_run_label_index()
label_counts = {}
for item in label_list:
label_counts.setdefault(item.get('label'), 0)
label_counts[item.get('label')] += 1
print '[showing %s labels]' % len(label_counts)
for key in label_counts:
print "%s (%s)" % (key, label_counts[key])
class RunLabelAdd(object):
"""Add a new run labels
"""
def __init__(self, args=None):
# Args may be given as an input argument for testing purposes
# or from the main parser.
# Otherwise get them from the parser.
if args is None:
args = self._get_args()
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
def _get_args(self):
self.parser = self.get_parser()
return self.parser.parse_args()
@classmethod
def get_parser(cls, parser=None):
# If called from main, use the subparser provided.
# Otherwise create a top-level parser here.
if parser is None:
parser = argparse.ArgumentParser(__file__)
parser.add_argument(
'target',
metavar='TARGET',
help='identifier for run to be labeled')
parser.add_argument(
'label',
metavar='LABEL', help='label name to be added')
return parser
def run(self):
runs = self.connection.get_run_index(
min=1, max=1,
query_string=self.args.target)
label_data = {'label': self.args.label}
label = self.connection.post_run_label(runs[0]['uuid'], label_data)
print 'Target "%s@%s" has been labeled as "%s"' % \
(runs[0].get('name'),
runs[0].get('uuid'),
label.get('label'))
class TemplateTagAdd(object):
"""Add a new template tags
"""
def __init__(self, args=None):
# Args may be given as an input argument for testing purposes
# or from the main parser.
# Otherwise get them from the parser.
if args is None:
args = self._get_args()
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
def _get_args(self):
self.parser = self.get_parser()
return self.parser.parse_args()
@classmethod
def get_parser(cls, parser=None):
# If called from main, use the subparser provided.
# Otherwise create a top-level parser here.
if parser is None:
parser = argparse.ArgumentParser(__file__)
parser.add_argument(
'target',
metavar='TARGET',
help='identifier for template to be tagged')
parser.add_argument(
'tag',
metavar='TAG', help='tag name to be added')
return parser
def run(self):
templates = self.connection.get_template_index(
min=1, max=1,
query_string=self.args.target)
tag_data = {'tag': self.args.tag}
tag = self.connection.post_template_tag(templates[0]['uuid'], tag_data)
print 'Target "%s@%s" has been tagged as "%s"' % \
(templates[0].get('name'),
templates[0].get('uuid'),
tag.get('tag'))
class AbstractRunSubcommand(object):
def __init__(self, args=None, silent=False):
self._validate_args(args)
self.args = args
self.silent = silent
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
token = get_token()
self.connection = Connection(server_url, token=token)
try:
self.storage_settings = self.connection.get_storage_settings()
self.import_manager = ImportManager(
self.connection, storage_settings=self.storage_settings)
self.export_manager = ExportManager(
self.connection, storage_settings=self.storage_settings)
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to initialize client: '%s'" % e)
@classmethod
def _validate_args(cls, args):
pass
def _print(self, text):
if not self.silent:
print text
def __init__(self, args=None, mock_connection=None, mock_filemanager=None):
if args is None:
args = self._get_args()
self.settings = {
'TASK_ATTEMPT_ID': args.task_attempt_id,
'SERVER_URL': args.server_url,
'LOG_LEVEL': args.log_level,
}
self.is_failed=False
self.logger = get_stdout_logger(
__name__, self.settings['LOG_LEVEL'])
if mock_connection is not None:
self.connection = mock_connection
else:
try:
self.connection = Connection(self.settings['SERVER_URL'],
token=args.token)
except Exception as e:
error = self._get_error_text(e)
self.logger.error(
'TaskMonitor for attempt %s failed to initialize server '\
'connection. %s' \
% (self.settings.get('TASK_ATTEMPT_ID'), error))
raise
self._event('Initializing TaskMonitor')
self._init_task_attempt()
# From here on errors can be reported to Loom
if mock_filemanager is not None:
self.filemanager = mock_filemanager
else:
try:
self.filemanager = FileManager(self.settings['SERVER_URL'],
token=args.token)
self.settings.update(self._get_settings())
self._init_docker_client()
self._init_working_dir()
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(
detail='Initializing TaskMonitor failed. %s'\
% error)
raise
def __call__(self, parser, namespace, values, option_string):
if not has_connection_settings():
server_version = 'not connected'
else:
url = get_server_url()
connection = Connection(url)
try:
server_version = connection.get_version()
except ServerConnectionHttpError as e:
server_version = '[server error! %s]' % e
except ServerConnectionError:
server_version = '[no response]'
except LoomengineUtilsError as e:
server_version = '[client error! %s]' % e
print "client version: %s" % loomengine_utils.version.version()
print "server version: %s" % server_version
exit(0)
def __init__(self, args=None, silent=False):
if args is None:
args = self._get_args()
self.args = args
self.silent = silent
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
class RunLabelRemove(object):
"""Remove a run label
"""
def __init__(self, args=None):
if args is None:
args = self._get_args()
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
def _get_args(self):
self.parser = self.get_parser()
return self.parser.parse_args()
@classmethod
def get_parser(cls, parser=None):
# If called from main, use the subparser provided.
# Otherwise create a top-level parser here.
if parser is None:
parser = argparse.ArgumentParser(__file__)
parser.add_argument(
'target',
metavar='TARGET',
help='identifier for run to be unlabeled')
parser.add_argument(
'label',
metavar='LABEL', help='label name to be removed')
return parser
def run(self):
runs = self.connection.get_run_index(
min=1, max=1,
query_string=self.args.target)
label_data = {'label': self.args.label}
label = self.connection.remove_run_label(runs[0]['uuid'], label_data)
print 'Label %s has been removed from run "%s@%s"' % \
(label.get('label'),
runs[0].get('name'),
runs[0].get('uuid'))
class TemplateTagList(object):
def __init__(self, args=None):
if args is None:
args = self._get_args()
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
def _get_args(self):
self.parser = self.get_parser()
return self.parser.parse_args()
@classmethod
def get_parser(cls, parser=None):
# If called from main, use the subparser provided.
# Otherwise create a top-level parser here.
if parser is None:
parser = argparse.ArgumentParser(__file__)
parser.add_argument(
'target',
metavar='TARGET',
nargs='?',
help='show tags only for the specified template')
return parser
def run(self):
if self.args.target:
templates = self.connection.get_template_index(
min=1, max=1,
query_string=self.args.target)
tag_data = self.connection.list_template_tags(templates[0]['uuid'])
tags = tag_data.get('tags', [])
else:
tag_list = self.connection.get_template_tag_index()
tags = [item.get('tag') for item in tag_list]
print '[showing %s tags]' % len(tags)
for tag in tags:
print tag
class TemplateExport(object):
def __init__(self, args):
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running()
self.connection = Connection(server_url, token=get_token())
self.filemanager = FileManager(server_url)
@classmethod
def get_parser(cls, parser):
parser.add_argument('template_id',
metavar='TEMPLATE_ID',
help='template to be downloaded')
parser.add_argument('-d',
'--destination',
metavar='DESTINATION',
help='destination filename or directory')
parser.add_argument('-f',
'--format',
choices=['json', 'yaml'],
default='yaml',
help='data format for downloaded template')
parser.add_argument(
'-r', '--retry', action='store_true',
default=False,
help='allow retries if there is a failure '\
'connecting to storage')
return parser
def run(self):
template = self.connection.get_template_index(
query_string=self.args.template_id, min=1, max=1)[0]
destination_url = self._get_destination_url(template,
retry=self.args.retry)
self._save_template(template, destination_url, retry=self.args.retry)
def _get_destination_url(self, template, retry=False):
default_name = '%s.%s' % (template['name'], self.args.format)
return self.filemanager.get_destination_file_url(self.args.destination,
default_name,
retry=retry)
def _save_template(self, template, destination, retry=False):
print 'Exporting template %s@%s to %s...' % (
template.get('name'), template.get('uuid'), destination)
if self.args.format == 'json':
template_text = json.dumps(template,
indent=4,
separators=(',', ': '))
elif self.args.format == 'yaml':
template_text = yaml.safe_dump(template, default_flow_style=False)
else:
raise Exception('Invalid format type %s' % self.args.format)
self.filemanager.write_to_file(destination, template_text, retry=retry)
print '...finished exporting template'
def __init__(self, args=None, silent=False):
# Parse arguments
if args is None:
args = _get_args()
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.args = args
self.silent = silent
self._set_run_function()
self.connection = Connection(server_url, token=None)
class FileTagRemove(object):
"""Remove a file tag
"""
def __init__(self, args=None):
if args is None:
args = self._get_args()
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
def _get_args(self):
self.parser = self.get_parser()
return self.parser.parse_args()
@classmethod
def get_parser(cls, parser=None):
# If called from main, use the subparser provided.
# Otherwise create a top-level parser here.
if parser is None:
parser = argparse.ArgumentParser(__file__)
parser.add_argument('target',
metavar='TARGET',
help='identifier for file to be untagged')
parser.add_argument('tag',
metavar='TAG',
help='tag name to be removed')
return parser
def run(self):
files = self.connection.get_data_object_index(
min=1, max=1, query_string=self.args.target, type='file')
tag_data = {'tag': self.args.tag}
tag = self.connection.remove_data_tag(files[0]['uuid'], tag_data)
print 'Tag %s has been removed from file "%s@%s"' % \
(tag.get('tag'),
files[0]['value'].get('filename'),
files[0].get('uuid'))
def __init__(self, args=None, silent=False):
# Args may be given as an input argument for testing purposes
# or from the main parser.
# Otherwise get them from the parser.
if args is None:
args = self._get_args()
self.args = args
self.silent = silent
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
def __init__(self, args=None, mock_connection=None,
mock_import_manager=None, mock_export_manager=None):
if args is None:
args = self._get_args()
self.settings = {
'TASK_ATTEMPT_ID': args.task_attempt_id,
'SERVER_URL': args.server_url,
'LOG_LEVEL': args.log_level,
}
self.is_failed = False
self.logger = get_stdout_logger(
__name__, self.settings['LOG_LEVEL'])
if mock_connection is not None:
self.connection = mock_connection
else:
try:
self.connection = Connection(self.settings['SERVER_URL'],
token=args.token)
except Exception as e:
error = self._get_error_text(e)
self.logger.error(
'TaskMonitor for attempt %s failed to initialize server '
'connection. %s'
% (self.settings.get('TASK_ATTEMPT_ID'), error))
raise
self._event('Initializing TaskMonitor')
self._init_task_attempt()
# From here on errors can be reported to Loom
if mock_import_manager is not None:
self.import_manager = mock_import_manager
else:
try:
self.storage_settings = self.connection.get_storage_settings()
self.import_manager = ImportManager(
self.connection, storage_settings=self.storage_settings)
self.export_manager = ExportManager(
self.connection, storage_settings=self.storage_settings)
self.settings.update(self._get_settings())
self._init_docker_client()
self._init_working_dir()
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(
detail='Initializing TaskMonitor failed. %s'
% error)
raise
class AuthClient(object):
def __init__(self, args=None, silent=False):
# Parse arguments
if args is None:
args = _get_args()
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.args = args
self.silent = silent
self._set_run_function()
self.connection = Connection(server_url, token=None)
def _print(self, text):
if not self.silent:
print text
def _set_run_function(self):
# Map user input command to method
commands = {
'login': self.login,
'logout': self.logout,
'print-token': self.print_token,
}
self.run = commands[self.args.command]
def login(self):
username = self.args.username
password = self.args.password
if password is None:
password = getpass("Password: "******"ERROR! Login failed")
save_token(token)
self._print("Login was successful. Token saved.")
def logout(self):
token = get_token()
if token is None:
self._print("No token found. You are logged out.")
else:
delete_token()
self._print("Token deleted.")
def print_token(self):
print get_token()
def __init__(self, args=None, silent=False):
self._validate_args(args)
self.args = args
self.silent = silent
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
token = get_token()
self.connection = Connection(server_url, token=token)
try:
self.storage_settings = self.connection.get_storage_settings()
self.import_manager = ImportManager(
self.connection, storage_settings=self.storage_settings)
self.export_manager = ExportManager(
self.connection, storage_settings=self.storage_settings)
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to initialize client: '%s'" % e)
class AuthClient(object):
def __init__(self, args=None):
# Parse arguments
if args is None:
args = _get_args()
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.args = args
self._set_run_function()
self.connection = Connection(server_url, token=None)
def _set_run_function(self):
# Map user input command to method
commands = {
'login': self.login,
'logout': self.logout,
'print-token': self.print_token,
}
self.run = commands[self.args.command]
def login(self):
username = self.args.username
password = self.args.password
if password is None:
password = getpass("Password: "******"ERROR! Login failed")
save_token(token)
print "Login was successful. Token saved."
def logout(self):
token = get_token()
if token is None:
print "No token found. You are logged out."
else:
delete_token()
print "Token deleted."
def print_token(self):
print get_token()
class TemplateList(object):
def __init__(self, args):
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
@classmethod
def get_parser(cls, parser):
parser.add_argument('template_id',
nargs='?',
metavar='TEMPLATE_IDENTIFIER',
help='Name or ID of template(s) to list.')
parser.add_argument('-d',
'--detail',
action='store_true',
help='Show detailed view of templates')
parser.add_argument(
'-a', '--all',
action='store_true',
help='List all templates, including nested children. '\
'(ignored when TEMPLATE_IDENTIFIER is given)')
parser.add_argument('-l',
'--label',
metavar='LABEL',
action='append',
help='Filter by label')
return parser
def run(self):
if self.args.template_id:
imported = False
else:
imported = not self.args.all
offset = 0
limit = 10
while True:
data = self.connection.get_template_index_with_limit(
labels=self.args.label,
limit=limit,
offset=offset,
query_string=self.args.template_id,
imported=imported)
if offset == 0:
print '[showing %s templates]' % data.get('count')
self._list_templates(data['results'])
if data.get('next'):
offset += limit
else:
break
def _list_templates(self, templates):
for template in templates:
print self._render_template(template)
def _render_template(self, template):
template_identifier = '%s@%s' % (template['name'], template['uuid'])
if self.args.detail:
text = '---------------------------------------\n'
text += 'Template: %s\n' % template_identifier
text += ' - md5: %s\n' % template.get('md5')
text += ' - Imported: %s\n' % \
_render_time(template['datetime_created'])
if template.get('inputs'):
text += ' - Inputs\n'
for input in template['inputs']:
text += ' - %s\n' % input['channel']
if template.get('outputs'):
text += ' - Outputs\n'
for output in template['outputs']:
text += ' - %s\n' % output['channel']
if template.get('steps'):
text += ' - Steps\n'
for step in template['steps']:
text += ' - %s@%s\n' % (step['name'], step['uuid'])
if template.get('command'):
text += ' - Command: %s\n' % template['command']
else:
text = 'Template: %s' % template_identifier
return text
class TemplateImport(object):
def __init__(self, args):
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
token = get_token()
self.filemanager = FileManager(server_url, token=token)
self.connection = Connection(server_url, token=token)
@classmethod
def get_parser(cls, parser):
parser.add_argument(
'template',
metavar='TEMPLATE_FILE', help='template to be imported, '\
'in YAML or JSON format')
parser.add_argument(
'-c', '--comments',
metavar='COMMENTS',
help='comments. '\
'Give enough detail for traceability')
parser.add_argument('-d', '--force-duplicates', action='store_true',
default=False,
help='force upload even if another template with '\
'the same name and md5 exists')
parser.add_argument('-r', '--retry', action='store_true',
default=False,
help='allow retries if there is a failure '\
'connecting to storage')
parser.add_argument('-t',
'--tag',
metavar='TAG',
action='append',
help='tag the template when it is created')
parser.add_argument('-l',
'--label',
metavar='LABEL',
action='append',
help='label the template when it is created')
return parser
def run(self):
template = self.import_template(
self.args.template,
self.args.comments,
self.filemanager,
self.connection,
force_duplicates=self.args.force_duplicates,
retry=self.args.retry)
self._apply_tags(template)
self._apply_labels(template)
return template
@classmethod
def import_template(cls,
template_file,
comments,
filemanager,
connection,
force_duplicates=False,
retry=False):
print 'Importing template from "%s".' % filemanager.normalize_url(
template_file)
(template, source_url) = cls._get_template(template_file, filemanager,
retry)
if not force_duplicates:
templates = filemanager.get_template_duplicates(template)
if len(templates) > 0:
name = templates[-1]['name']
md5 = templates[-1]['md5']
uuid = templates[-1]['uuid']
warnings.warn(
'WARNING! The name and md5 hash "%s$%s" is already in use by one '
'or more templates. '\
'Use "--force-duplicates" to create another copy, but if you '\
'do you will have to use @uuid to reference these templates.'
% (name, md5))
print 'Matching template already exists as "%s@%s".' % (name,
uuid)
return templates[0]
if comments:
template.update({'import_comments': comments})
if source_url:
template.update({'imported_from_url': source_url})
try:
template_from_server = connection.post_template(template)
except HTTPError as e:
if e.response.status_code == 400:
errors = e.response.json()
raise SystemExit("ERROR! %s" % errors)
else:
raise
print 'Imported template "%s@%s".' % (template_from_server['name'],
template_from_server['uuid'])
return template_from_server
@classmethod
def _get_template(cls, template_file, filemanager, retry):
md5 = filemanager.calculate_md5(template_file, retry=retry)
try:
(template_text, source_url) = filemanager.read_file(template_file,
retry=retry)
except Exception as e:
raise SystemExit('ERROR! Unable to read file "%s". %s' %
(template_file, str(e)))
template = parse_as_json_or_yaml(template_text)
try:
template.update({'md5': md5})
except AttributeError:
raise SystemExit(
'ERROR! Template at "%s" could not be parsed into a dict.' %
os.path.abspath(template_file))
return template, source_url
def _apply_tags(self, template):
if not self.args.tag:
return
for tagname in self.args.tag:
tag_data = {'tag': tagname}
tag = self.connection.post_template_tag(template.get('uuid'),
tag_data)
print 'Template "%s@%s" has been tagged as "%s"' % \
(template.get('name'),
template.get('uuid'),
tag.get('tag'))
def _apply_labels(self, template):
if not self.args.label:
return
for labelname in self.args.label:
label_data = {'label': labelname}
label = self.connection.post_template_label(
template.get('uuid'), label_data)
print 'Template "%s@%s" has been labeled as "%s"' % \
(template.get('name'),
template.get('uuid'),
label.get('label'))
class RunLabelList(object):
def __init__(self, args=None, silent=False):
if args is None:
args = self._get_args()
self.args = args
self.silent = silent
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
def _get_args(self):
self.parser = self.get_parser()
return self.parser.parse_args()
@classmethod
def get_parser(cls, parser=None):
# If called from main, use the subparser provided.
# Otherwise create a top-level parser here.
if parser is None:
parser = argparse.ArgumentParser(__file__)
parser.add_argument(
'target',
metavar='TARGET',
nargs='?',
help='show labels only for the specified run')
return parser
def run(self):
if self.args.target:
try:
runs = self.connection.get_run_index(
min=1, max=1,
query_string=self.args.target)
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to get run list: '%s'" % e)
try:
label_data = self.connection.list_run_labels(runs[0]['uuid'])
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to get label list: '%s'" % e)
labels = label_data.get('labels', [])
if not self.silent:
print '[showing %s labels]' % len(labels)
for label in labels:
print label
else:
try:
label_list = self.connection.get_run_label_index()
except LoomengineUtilsError as e:
raise SystemExit("ERROR! Failed to get label list: '%s'" % e)
label_counts = {}
for item in label_list:
label_counts.setdefault(item.get('label'), 0)
label_counts[item.get('label')] += 1
if not self.silent:
print '[showing %s labels]' % len(label_counts)
for key in label_counts:
print "%s (%s)" % (key, label_counts[key])
class RunStart(object):
"""Run a template.
"""
def __init__(self, args=None):
if args is None:
args = self._get_args()
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
token = get_token()
self.connection = Connection(server_url, token=token)
self.filemanager = FileManager(server_url, token=token)
@classmethod
def _get_args(cls):
parser = cls.get_parser()
args = parser.parse_args()
self._validate_args(args)
return args
@classmethod
def get_parser(cls, parser=None):
if parser is None:
parser = argparse.ArgumentParser(__file__)
parser.add_argument('template',
metavar='TEMPLATE',
help='ID of template to run')
parser.add_argument('inputs',
metavar='INPUT_NAME=DATA_ID',
nargs='*',
help='ID or value of data inputs')
parser.add_argument('-n',
'--name',
metavar='RUN_NAME',
help='run name (default is template name)')
parser.add_argument('-e',
'--notify',
action='append',
metavar='EMAIL/URL',
help='recipients of completed run notifications')
parser.add_argument('-t',
'--tag',
metavar='TAG',
action='append',
help='tag the run when it is started')
parser.add_argument('-l',
'--label',
metavar='LABEL',
action='append',
help='label the run when it is started')
return parser
@classmethod
def _validate_args(cls, args):
if not args.inputs:
return
for input in arg.inputs:
vals = input.split('=')
if not len(vals) == 2 or vals[0] == '':
raise InvalidInputError(
'Invalid input key-value pair "%s". Must be of the form key=value or key=value1,value2,...'
% input)
def run(self):
run_data = {
'template': self.args.template,
'user_inputs': self._get_inputs(),
'notification_addresses': self.args.notify,
}
if self.args.name:
run_data['name'] = self.args.name
try:
run = self.connection.post_run(run_data)
except requests.exceptions.HTTPError as e:
if e.response.status_code >= 400:
try:
message = e.response.json()
except:
message = e.response.text
if isinstance(message, list):
message = '; '.join(message)
raise SystemExit(message)
else:
raise e
print 'Created run %s@%s' % (run['name'], run['uuid'])
self._apply_tags(run)
self._apply_labels(run)
return run
def _get_inputs(self):
"""Converts command line args into a list of template inputs
"""
inputs = []
if self.args.inputs:
for kv_pair in self.args.inputs:
(channel, input_id) = kv_pair.split('=')
inputs.append({
'channel': channel,
'data': {
'contents':
self._parse_string_to_nested_lists(input_id)
}
})
return inputs
def _apply_tags(self, run):
if not self.args.tag:
return
for tagname in self.args.tag:
tag_data = {'tag': tagname}
tag = self.connection.post_run_tag(run.get('uuid'), tag_data)
print 'Run "%s@%s" has been tagged as "%s"' % \
(run.get('name'),
run.get('uuid'),
tag.get('tag'))
def _apply_labels(self, run):
if not self.args.label:
return
for labelname in self.args.label:
label_data = {'label': labelname}
label = self.connection.post_run_label(run.get('uuid'), label_data)
print 'Run "%s@%s" has been labeled as "%s"' % \
(run.get('name'),
run.get('uuid'),
label.get('label'))
def _parse_string_to_nested_lists(self, value):
"""e.g., convert "[[a,b,c],[d,e],[f,g]]"
into [["a","b","c"],["d","e"],["f","g"]]
"""
if not re.match('\[.*\]', value.strip()):
if '[' in value or ']' in value or ',' in value:
raise Exception('Missing outer brace')
elif len(value.strip()) == 0:
raise Exception('Missing value')
else:
terms = value.split(',')
terms = [term.strip() for term in terms]
if len(terms) == 1:
return terms[0]
else:
return terms
# remove outer braces
value = value[1:-1]
terms = []
depth = 0
leftmost = 0
first_open_brace = None
break_on_commas = False
for i in range(len(value)):
if value[i] == ',' and depth == 0:
terms.append(
self._parse_string_to_nested_lists(value[leftmost:i]))
leftmost = i + 1
if value[i] == '[':
if first_open_brace is None:
first_open_brace = i
depth += 1
if value[i] == ']':
depth -= 1
if depth < 0:
raise Exception('Unbalanced close brace')
i += i
if depth > 0:
raise Exception('Expected "]"')
terms.append(
self._parse_string_to_nested_lists(value[leftmost:len(value)]))
return terms
class TaskMonitor(object):
DOCKER_SOCKET = 'unix://var/run/docker.sock'
LOOM_RUN_SCRIPT_NAME = '.loom_run_script'
def __init__(self, args=None, mock_connection=None, mock_filemanager=None):
if args is None:
args = self._get_args()
self.settings = {
'TASK_ATTEMPT_ID': args.task_attempt_id,
'SERVER_URL': args.server_url,
'LOG_LEVEL': args.log_level,
}
self.is_failed=False
self.logger = get_stdout_logger(
__name__, self.settings['LOG_LEVEL'])
if mock_connection is not None:
self.connection = mock_connection
else:
try:
self.connection = Connection(self.settings['SERVER_URL'],
token=args.token)
except Exception as e:
error = self._get_error_text(e)
self.logger.error(
'TaskMonitor for attempt %s failed to initialize server '\
'connection. %s' \
% (self.settings.get('TASK_ATTEMPT_ID'), error))
raise
self._event('Initializing TaskMonitor')
self._init_task_attempt()
# From here on errors can be reported to Loom
if mock_filemanager is not None:
self.filemanager = mock_filemanager
else:
try:
self.filemanager = FileManager(self.settings['SERVER_URL'],
token=args.token)
self.settings.update(self._get_settings())
self._init_docker_client()
self._init_working_dir()
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(
detail='Initializing TaskMonitor failed. %s'\
% error)
raise
def _init_task_attempt(self):
self.task_attempt = self.connection.get_task_attempt(
self.settings['TASK_ATTEMPT_ID'])
if self.task_attempt is None:
raise Exception(
'TaskAttempt ID "%s" not found' % self.settings['TASK_ATTEMPT_ID'])
def _get_settings(self):
settings = self.connection.get_task_attempt_settings(
self.settings['TASK_ATTEMPT_ID'])
if settings is None:
raise Exception('Worker settings not found')
return settings
def _init_docker_client(self):
self.docker_client = docker.Client(base_url=self.DOCKER_SOCKET)
self._verify_docker()
def _verify_docker(self):
try:
self.docker_client.info()
except requests.exceptions.ConnectionError:
raise Exception('Failed to connect to Docker daemon')
def _init_working_dir(self):
init_directory(self.settings['WORKING_DIR'], new=True)
def run_with_heartbeats(self, function):
heartbeat_interval = int(self.settings['HEARTBEAT_INTERVAL_SECONDS'])
polling_interval = 1
t = threading.Thread(target=function)
t.start()
last_heartbeat = self._send_heartbeat()
while t.is_alive():
time.sleep(polling_interval)
if (datetime.utcnow().replace(tzinfo=pytz.utc) - last_heartbeat)\
.total_seconds() > \
(heartbeat_interval - polling_interval):
last_heartbeat = self._send_heartbeat()
def run(self):
try:
self._copy_inputs()
self._create_run_script()
self._pull_image()
self._create_container()
self._run_container()
self._stream_docker_logs()
self._get_returncode()
self._save_process_logs()
if not self.is_failed:
self._save_outputs()
self._finish()
finally:
self._delete_container()
def _copy_inputs(self):
self._event('Copying inputs')
if self.task_attempt.get('inputs') is None:
return
try:
for input in self.task_attempt['inputs']:
TaskAttemptInput(input, self).copy()
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(detail='Copying inputs failed. %s' % error)
raise
def _create_run_script(self):
try:
user_command = self.task_attempt['command']
with open(os.path.join(
self.settings['WORKING_DIR'],
self.LOOM_RUN_SCRIPT_NAME),
'w') as f:
f.write(user_command.encode('utf-8') + '\n')
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(detail='Creating run script failed. %s' % error)
raise
def _pull_image(self):
self._event('Pulling image')
try:
docker_image = self._get_docker_image()
raw_pull_data = execute_with_retries(
lambda: self.docker_client.pull(docker_image),
(docker.errors.NotFound),
self.logger,
"Pull docker image")
pull_data = self._parse_docker_output(raw_pull_data)
if pull_data[-1].get('errorDetail'):
raise Exception('Failed to pull docker image: "%s"'
% pull_data[-1].get('errorDetail'))
container_info = self.docker_client.inspect_image(
self._get_docker_image())
self._save_environment_info(container_info)
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(detail='Pulling Docker image failed with "%s" error: "%s"' % (e.__class__, str(e)))
raise
def _get_docker_image(self):
docker_image = self.task_attempt['environment']['docker_image']
# If no registry is specified, set to default.
# If the first term contains "." or ends in ":", it is a registry.
part1=docker_image.split('/')[0]
if not '.' in part1 and not part1.endswith(':'):
default_registry = self.settings.get('DEFAULT_DOCKER_REGISTRY', None)
# Don't add default_registry without the owner. Default ower is library
if len(docker_image.split('/')) == 1:
docker_image = 'library/' + docker_image
if default_registry:
docker_image = '%s/%s' % (default_registry, docker_image)
# Tag is required. Otherwise docker-py pull will download all tags.
if not '@' in docker_image and not ':' in docker_image:
docker_image += ':latest'
return docker_image
def _parse_docker_output(self, data):
return [json.loads(line) for line in data.strip().split('\r\n')]
def _create_container(self):
self._event('Creating container')
try:
docker_image = self._get_docker_image()
interpreter = self.task_attempt['interpreter']
host_dir = self.settings['WORKING_DIR']
container_dir = '/loom_workspace'
command = interpreter.split(' ')
command.append(self.LOOM_RUN_SCRIPT_NAME)
self.container = self.docker_client.create_container(
image=docker_image,
command=command,
volumes=[container_dir],
host_config=self.docker_client.create_host_config(
binds={host_dir: {
'bind': container_dir,
'mode': 'rw',
}}),
working_dir=container_dir,
name=self.settings['SERVER_NAME']+'-attempt-'+self.settings[
'TASK_ATTEMPT_ID'],
)
self._set_container_id(self.container['Id'])
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(detail='Creating container failed. %s' % error)
raise
def _run_container(self):
self._event('Starting analysis')
try:
self.docker_client.start(self.container)
self._verify_container_started_running()
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(detail='Starting analysis failed. %s' % error)
raise
def _verify_container_started_running(self):
status = self.docker_client.inspect_container(
self.container)['State'].get('Status')
if status == 'running' or status == 'exited':
return
else:
raise Exception('Unexpected container status "%s"' % status)
def _stream_docker_logs(self):
"""Stream stdout and stderr from the task container to this process's
stdout and stderr, respectively.
"""
thread = threading.Thread(target=self._stderr_stream_worker)
thread.start()
for line in self.docker_client.logs(self.container, stdout=True,
stderr=False, stream=True):
sys.stdout.write(line)
thread.join()
def _stderr_stream_worker(self):
for line in self.docker_client.logs(self.container, stdout=False,
stderr=True, stream=True):
sys.stderr.write(line)
def _get_returncode(self):
self._event('Running analysis')
try:
returncode = self._poll_for_returncode()
if returncode == 0:
return
else:
# bad returncode
self._report_analysis_error(
'Analysis finished with returncode %s. '\
'Check stderr/stdout logs for errors.'
% returncode)
# Do not raise error. Attempt to save log files.
except Exception as e:
error = self._get_error_text(e)
self._report_system_error('Failed to run analysis. %s' % error)
# Do not raise error. Attempt to save log files.
def _poll_for_returncode(self, poll_interval_seconds=1):
while True:
try:
container_data = self.docker_client.inspect_container(self.container)
except Exception as e:
raise Exception('Unable to inspect Docker container: "%s"' % str(e))
if not container_data.get('State'):
raise Exception(
'Could not parse container info from Docker: "%s"' % container_data)
if container_data['State'].get('Status') == 'exited':
# Success
return container_data['State'].get('ExitCode')
elif container_data['State'].get('Status') == 'running':
time.sleep(poll_interval_seconds)
else:
# Error -- process did not complete
message = 'Docker container has unexpected status "%s"' % \
container_data['State'].get('Status')
raise Exception(message)
def _save_process_logs(self):
self._event('Saving logfiles')
try:
init_directory(
os.path.dirname(os.path.abspath(self.settings['STDOUT_LOG_FILE'])))
with open(self.settings['STDOUT_LOG_FILE'], 'w') as stdoutlog:
stdoutlog.write(self._get_stdout())
init_directory(
os.path.dirname(os.path.abspath(self.settings['STDERR_LOG_FILE'])))
with open(self.settings['STDERR_LOG_FILE'], 'w') as stderrlog:
stderrlog.write(self._get_stderr())
self._import_log_file(self.settings['STDOUT_LOG_FILE'], retry=True)
self._import_log_file(self.settings['STDERR_LOG_FILE'], retry=True)
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(detail='Saving log files failed. %s' % error)
raise
def _get_stdout(self):
return self.docker_client.logs(self.container, stderr=False, stdout=True)
def _get_stderr(self):
return self.docker_client.logs(self.container, stderr=True, stdout=False)
def _import_log_file(self, filepath, retry=True):
try:
self.filemanager.import_log_file(
self.task_attempt,
filepath,
retry=retry,
)
except IOError:
message = 'Failed to upload log file %s' % filepath
raise Exception(message)
def _save_outputs(self):
self._event('Saving outputs')
try:
for output in self.task_attempt['outputs']:
TaskAttemptOutput(output, self).save()
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(detail='Saving outputs failed. %s' % error)
raise
def _finish(self):
try:
self._finish()
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(detail='Setting finished status failed. %s' % error)
raise
# Updates to TaskAttempt
def _send_heartbeat(self):
task_attempt = self.connection.update_task_attempt(
self.settings['TASK_ATTEMPT_ID'],
{}
)
return parse(task_attempt.get('last_heartbeat'))
def _set_container_id(self, container_id):
self.connection.update_task_attempt(
self.settings['TASK_ATTEMPT_ID'],
{'container_id': container_id}
)
def _save_environment_info(self, container_info):
self.connection.update_task_attempt(
self.settings['TASK_ATTEMPT_ID'],
{'environment_info': container_info}
)
def _event(self, event, detail='', is_error=False):
if is_error:
self.logger.error("%s. %s" % (event, detail))
else:
self.logger.info("%s. %s" % (event, detail))
self.connection.post_task_attempt_event(
self.settings['TASK_ATTEMPT_ID'],
{
'event': event,
'detail': detail,
'is_error': is_error
})
def _report_system_error(self, detail=''):
self.is_failed=True
try:
self._event("TaskAttempt execution failed.", detail=detail, is_error=True)
self.connection.post_task_attempt_system_error(
self.settings['TASK_ATTEMPT_ID'])
except:
# If there is an error reporting failure, don't raise it
# because it will mask the root cause of failure
pass
def _report_analysis_error(self, detail=''):
self.is_failed=True
try:
self._event("TaskAttempt execution failed.", detail=detail, is_error=True)
self.connection.post_task_attempt_analysis_error(
self.settings['TASK_ATTEMPT_ID'])
except:
# If there is an error reporting failure, don't raise it
# because it will mask the root cause of failure
pass
def _finish(self):
self.connection.post_task_attempt_finish(self.settings['TASK_ATTEMPT_ID'])
def _delete_container(self):
try:
if not self.container:
return
except AttributeError:
return
if self.settings.get('PRESERVE_ALL'):
return
if self.is_failed and self.settings.get('PRESERVE_ON_FAILURE'):
return
self.docker_client.stop(self.container)
self.docker_client.remove_container(self.container)
def _get_error_text(self, e):
if hasattr(self, 'settings') and self.settings.get('DEBUG'):
return traceback.format_exc()
else:
return "%s.%s: %s" % (
e.__class__.__module__, e.__class__.__name__, str(e))
# Parser
def _get_args(self):
parser = self.get_parser()
return parser.parse_args()
@classmethod
def get_parser(self):
parser = argparse.ArgumentParser(__file__)
parser.add_argument('-i',
'--task_attempt_id',
required=True,
help='ID of TaskAttempt to be processed')
parser.add_argument('-u',
'--server_url',
required=True,
help='URL of the Loom server')
parser.add_argument('-l',
'--log_level',
required=False,
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
default='WARNING',
help='Log level')
parser.add_argument('-t',
'--token',
required=False,
default=None,
help='Authentication token')
return parser
class RunList(object):
def __init__(self, args):
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
@classmethod
def get_parser(cls, parser):
parser.add_argument('run_id',
nargs='?',
metavar='RUN_IDENTIFIER',
help='name or ID of run(s) to list.')
parser.add_argument('-d',
'--detail',
action='store_true',
help='show detailed view of runs')
parser.add_argument(
'-a', '--all',
action='store_true',
help='list all runs, including nested children '\
'(ignored when RUN_IDENTIFIER is given)')
parser.add_argument('-l',
'--label',
metavar='LABEL',
action='append',
help='filter by label')
return parser
def run(self):
if self.args.run_id:
parent_only = False
else:
parent_only = not self.args.all
offset = 0
limit = 10
while True:
data = self.connection.get_run_index_with_limit(
query_string=self.args.run_id,
limit=limit,
offset=offset,
labels=self.args.label,
parent_only=parent_only)
if offset == 0:
print '[showing %s runs]' % data.get('count')
self._list_runs(data['results'])
if data.get('next'):
offset += limit
else:
break
def _list_runs(self, runs):
for run in runs:
print self._render_run(run)
def _render_run(self, run):
run_identifier = '%s@%s' % (run['name'], run['uuid'])
if self.args.detail:
text = '---------------------------------------\n'
text += 'Run: %s\n' % run_identifier
text += ' - Created: %s\n' % _render_time(run['datetime_created'])
text += ' - Status: %s\n' % run.get('status')
if run.get('steps'):
text += ' - Steps:\n'
for step in run['steps']:
text += ' - %s@%s (%s)\n' % (step['name'], step['uuid'],
step.get('status'))
else:
text = "Run: %s (%s)" % (run_identifier, run.get('status'))
return text
class TaskMonitor(object):
DOCKER_SOCKET = 'unix://var/run/docker.sock'
LOOM_RUN_SCRIPT_NAME = '.loom_run_script'
def __init__(self, args=None, mock_connection=None,
mock_import_manager=None, mock_export_manager=None):
if args is None:
args = self._get_args()
self.settings = {
'TASK_ATTEMPT_ID': args.task_attempt_id,
'SERVER_URL': args.server_url,
'LOG_LEVEL': args.log_level,
}
self.is_failed = False
self.logger = get_stdout_logger(
__name__, self.settings['LOG_LEVEL'])
if mock_connection is not None:
self.connection = mock_connection
else:
try:
self.connection = Connection(self.settings['SERVER_URL'],
token=args.token)
except Exception as e:
error = self._get_error_text(e)
self.logger.error(
'TaskMonitor for attempt %s failed to initialize server '
'connection. %s'
% (self.settings.get('TASK_ATTEMPT_ID'), error))
raise
self._event('Initializing TaskMonitor')
self._init_task_attempt()
# From here on errors can be reported to Loom
if mock_import_manager is not None:
self.import_manager = mock_import_manager
else:
try:
self.storage_settings = self.connection.get_storage_settings()
self.import_manager = ImportManager(
self.connection, storage_settings=self.storage_settings)
self.export_manager = ExportManager(
self.connection, storage_settings=self.storage_settings)
self.settings.update(self._get_settings())
self._init_docker_client()
self._init_working_dir()
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(
detail='Initializing TaskMonitor failed. %s'
% error)
raise
def _init_task_attempt(self):
self.task_attempt = self.connection.get_task_attempt(
self.settings['TASK_ATTEMPT_ID'])
if self.task_attempt is None:
raise Exception(
'TaskAttempt ID "%s" not found'
% self.settings['TASK_ATTEMPT_ID'])
def _get_settings(self):
settings = self.connection.get_task_attempt_settings(
self.settings['TASK_ATTEMPT_ID'])
if settings is None:
raise Exception('Worker settings not found')
return settings
def _init_docker_client(self):
self.docker_client = docker.Client(base_url=self.DOCKER_SOCKET)
self._verify_docker()
def _verify_docker(self):
try:
self.docker_client.info()
except requests.exceptions.ConnectionError:
raise Exception('Failed to connect to Docker daemon')
def _init_working_dir(self):
init_directory(self.settings['WORKING_DIR_ROOT'], new=True)
self.working_dir = os.path.join(
self.settings['WORKING_DIR_ROOT'], 'work')
self.log_dir = os.path.join(self.settings['WORKING_DIR_ROOT'], 'logs')
init_directory(self.working_dir, new=True)
init_directory(self.log_dir, new=True)
def _delete_working_dir(self):
# Skip delete if blank or root!
if self.settings['WORKING_DIR_ROOT'].strip('/'):
shutil.rmtree(self.settings['WORKING_DIR_ROOT'])
def run_with_heartbeats(self, function):
heartbeat_interval = int(self.settings['HEARTBEAT_INTERVAL_SECONDS'])
polling_interval = 1
t = threading.Thread(target=function)
t.start()
last_heartbeat = self._send_heartbeat()
while t.is_alive():
time.sleep(polling_interval)
if (datetime.utcnow().replace(tzinfo=pytz.utc) - last_heartbeat)\
.total_seconds() > \
(heartbeat_interval - polling_interval):
last_heartbeat = self._send_heartbeat()
def run(self):
try:
self._copy_inputs()
self._create_run_script()
self._pull_image()
self._create_container()
self._run_container()
self._stream_docker_logs()
self._get_returncode()
self._save_process_logs()
if not self.is_failed:
self._save_outputs()
self._finish()
finally:
self._delete_working_dir()
self._delete_container()
def _copy_inputs(self):
self._event('Copying inputs')
if self.task_attempt.get('inputs') is None:
return
try:
for input in self.task_attempt['inputs']:
TaskAttemptInput(input, self).copy()
except FileAlreadyExistsError as e:
error = self._get_error_text(e)
self._report_system_error(
detail='Copying inputs failed because file already exists. '
'Are there multiple inputs with the same name? %s' % error)
raise
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(
detail='Copying inputs failed. %s' % error)
raise
def _create_run_script(self):
try:
user_command = self.task_attempt['command']
with open(os.path.join(
self.working_dir,
self.LOOM_RUN_SCRIPT_NAME),
'w') as f:
f.write(user_command.encode('utf-8') + '\n')
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(
detail='Creating run script failed. %s' % error)
raise
def _pull_image(self):
self._event('Pulling image')
try:
docker_image = self._get_docker_image()
raw_pull_data = execute_with_retries(
lambda: self.docker_client.pull(docker_image),
(docker.errors.NotFound,),
self.logger,
"Pull docker image")
pull_data = self._parse_docker_output(raw_pull_data)
if pull_data[-1].get('errorDetail'):
raise Exception('Failed to pull docker image: "%s"'
% pull_data[-1].get('errorDetail'))
container_info = self.docker_client.inspect_image(
self._get_docker_image())
self._save_environment_info(container_info)
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(
detail='Pulling Docker image failed with "%s" error: '
'"%s"' % (e.__class__, str(e)))
raise
def _get_docker_image(self):
docker_image = self.task_attempt['environment']['docker_image']
# If no registry is specified, set to default.
# If the first term contains "." or ends in ":", it is a registry.
part1 = docker_image.split('/')[0]
if '.' not in part1 and not part1.endswith(':'):
default_registry = self.settings.get(
'DEFAULT_DOCKER_REGISTRY', None)
# Don't add default_registry without the owner.
# Default ower is library
if len(docker_image.split('/')) == 1:
docker_image = 'library/' + docker_image
if default_registry:
docker_image = '%s/%s' % (default_registry, docker_image)
# Tag is required. Otherwise docker-py pull will download all tags.
if '@' not in docker_image and ':' not in docker_image:
docker_image += ':latest'
return docker_image
def _parse_docker_output(self, data):
return [json.loads(line) for line in data.strip().split('\r\n')]
def _create_container(self):
self._event('Creating container')
try:
docker_image = self._get_docker_image()
interpreter = self.task_attempt['interpreter']
host_dir = self.working_dir
container_dir = '/loom_workspace'
command = interpreter.split(' ')
command.append(self.LOOM_RUN_SCRIPT_NAME)
self.container = self.docker_client.create_container(
image=docker_image,
command=command,
volumes=[container_dir],
host_config=self.docker_client.create_host_config(
binds={host_dir: {
'bind': container_dir,
'mode': 'rw',
}}),
working_dir=container_dir,
name=self.settings['PROCESS_CONTAINER_NAME'],
)
self._set_container_id(self.container['Id'])
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(
detail='Creating container failed. %s' % error)
raise
def _run_container(self):
self._event('Starting analysis')
try:
self.docker_client.start(self.container)
self._verify_container_started_running()
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(
detail='Starting analysis failed. %s' % error)
raise
def _verify_container_started_running(self):
status = self.docker_client.inspect_container(
self.container)['State'].get('Status')
if status == 'running' or status == 'exited':
return
else:
raise Exception('Unexpected container status "%s"' % status)
def _stream_docker_logs(self):
"""Stream stdout and stderr from the task container to this
process's stdout and stderr, respectively.
"""
thread = threading.Thread(target=self._stderr_stream_worker)
thread.start()
for line in self.docker_client.logs(self.container, stdout=True,
stderr=False, stream=True):
sys.stdout.write(line)
thread.join()
def _stderr_stream_worker(self):
for line in self.docker_client.logs(self.container, stdout=False,
stderr=True, stream=True):
sys.stderr.write(line)
def _get_returncode(self):
self._event('Running analysis')
try:
returncode = self._poll_for_returncode()
if returncode == 0:
return
else:
# bad returncode
self._report_analysis_error(
'Analysis finished with returncode %s. '
'Check stderr/stdout logs for errors.'
% returncode)
# Do not raise error. Attempt to save log files.
except Exception as e:
error = self._get_error_text(e)
self._report_system_error('Failed to run analysis. %s' % error)
# Do not raise error. Attempt to save log files.
def _poll_for_returncode(self, poll_interval_seconds=1):
while True:
try:
container_data = self.docker_client.inspect_container(
self.container)
except Exception as e:
raise Exception(
'Unable to inspect Docker container: "%s"' % str(e))
if not container_data.get('State'):
raise Exception(
'Could not parse container info from Docker: "%s"'
% container_data)
if container_data['State'].get('Status') == 'exited':
# Success
return container_data['State'].get('ExitCode')
elif container_data['State'].get('Status') == 'running':
time.sleep(poll_interval_seconds)
else:
# Error -- process did not complete
message = 'Docker container has unexpected status "%s"' % \
container_data['State'].get('Status')
raise Exception(message)
def _save_process_logs(self):
self._event('Saving logfiles')
try:
stdout_file = os.path.join(self.log_dir, 'stdout.log')
stderr_file = os.path.join(self.log_dir, 'stderr.log')
init_directory(
os.path.dirname(self.log_dir))
with open(stdout_file, 'w') as stdoutlog:
stdoutlog.write(self._get_stdout())
with open(stderr_file, 'w') as stderrlog:
stderrlog.write(self._get_stderr())
self._import_log_file(stderr_file, retry=True)
self._import_log_file(stdout_file, retry=True)
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(
detail='Saving log files failed. %s' % error)
raise
def _get_stdout(self):
return self.docker_client.logs(
self.container, stderr=False, stdout=True)
def _get_stderr(self):
return self.docker_client.logs(
self.container, stderr=True, stdout=False)
def _import_log_file(self, filepath, retry=True):
try:
self.import_manager.import_log_file(
self.task_attempt,
filepath,
retry=retry,
)
except IOError:
message = 'Failed to upload log file %s' % filepath
raise Exception(message)
def _save_outputs(self):
self._event('Saving outputs')
try:
for output in self.task_attempt['outputs']:
TaskAttemptOutput(output, self).save()
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(
detail='Saving outputs failed. %s' % error)
raise
def _finish(self):
try:
self._finish()
except Exception as e:
error = self._get_error_text(e)
self._report_system_error(
detail='Setting finished status failed. %s' % error)
raise
# Updates to TaskAttempt
def _send_heartbeat(self):
task_attempt = self.connection.update_task_attempt(
self.settings['TASK_ATTEMPT_ID'],
{}
)
return parse(task_attempt.get('last_heartbeat'))
def _set_container_id(self, container_id):
self.connection.update_task_attempt(
self.settings['TASK_ATTEMPT_ID'],
{'container_id': container_id}
)
def _save_environment_info(self, container_info):
self.connection.update_task_attempt(
self.settings['TASK_ATTEMPT_ID'],
{'environment_info': container_info}
)
def _event(self, event, detail='', is_error=False):
if is_error:
self.logger.error("%s. %s" % (event, detail))
else:
self.logger.info("%s. %s" % (event, detail))
self.connection.post_task_attempt_event(
self.settings['TASK_ATTEMPT_ID'],
{
'event': event,
'detail': detail,
'is_error': is_error
})
def _report_system_error(self, detail=''):
self.is_failed = True
try:
self._event(
"TaskAttempt execution failed.", detail=detail, is_error=True)
self.connection.post_task_attempt_system_error(
self.settings['TASK_ATTEMPT_ID'])
except Exception:
# If there is an error reporting failure, don't raise it
# because it will mask the root cause of failure
pass
def _report_analysis_error(self, detail=''):
self.is_failed = True
try:
self._event(
"TaskAttempt execution failed.", detail=detail, is_error=True)
self.connection.post_task_attempt_analysis_error(
self.settings['TASK_ATTEMPT_ID'])
except Exception:
# If there is an error reporting failure, don't raise it
# because it will mask the root cause of failure
pass
def _finish(self):
self.connection.finish_task_attempt(self.settings['TASK_ATTEMPT_ID'])
def _delete_container(self):
try:
if not self.container:
return
except AttributeError:
return
if self.settings.get('PRESERVE_ALL'):
return
if self.is_failed and self.settings.get('PRESERVE_ON_FAILURE'):
return
self.docker_client.stop(self.container)
self.docker_client.remove_container(self.container)
def _get_error_text(self, e):
if hasattr(self, 'settings') and self.settings.get('DEBUG'):
return traceback.format_exc()
else:
return "%s.%s: %s" % (
e.__class__.__module__, e.__class__.__name__, str(e))
# Parser
def _get_args(self):
parser = self.get_parser()
return parser.parse_args()
@classmethod
def get_parser(self):
parser = argparse.ArgumentParser(__file__)
parser.add_argument('-i',
'--task_attempt_id',
required=True,
help='ID of TaskAttempt to be processed')
parser.add_argument('-u',
'--server_url',
required=True,
help='URL of the Loom server')
parser.add_argument('-l',
'--log_level',
required=False,
choices=['DEBUG', 'INFO', 'WARNING',
'ERROR', 'CRITICAL'],
default='WARNING',
help='Log level')
parser.add_argument('-t',
'--token',
required=False,
default=None,
help='Authentication token')
return parser
class FileImport(object):
def __init__(self, args):
"""Common init tasks for all Importer classes
"""
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
token = get_token()
self.filemanager = FileManager(server_url, token=token)
self.connection = Connection(server_url, token=token)
@classmethod
def get_parser(cls, parser):
parser.add_argument(
'files',
metavar='FILE', nargs='+', help='file path or Google Storage URL '\
'of file(s) to be imported. Wildcards are allowed')
parser.add_argument(
'-c', '--comments',
metavar='COMMENTS',
help='comments. '\
'Give enough detail for traceability.')
parser.add_argument('-d', '--force-duplicates', action='store_true',
default=False,
help='force upload even if another file with '\
'the same name and md5 exists')
parser.add_argument('-o', '--original-copy', action='store_true',
default=False,
help='use existing copy instead of copying to storage '\
'managed by Loom')
parser.add_argument('-r', '--retry', action='store_true',
default=False,
help='allow retries if there is a failure '\
'connecting to storage')
parser.add_argument('-t',
'--tag',
metavar='TAG',
action='append',
help='tag the file when it is created')
parser.add_argument('-l',
'--label',
metavar='LABEL',
action='append',
help='label the file when it is created')
return parser
def run(self):
files_imported = self.filemanager.import_from_patterns(
self.args.files,
self.args.comments,
original_copy=self.args.original_copy,
force_duplicates=self.args.force_duplicates,
retry=self.args.retry)
if len(files_imported) == 0:
raise SystemExit('ERROR! Did not find any files matching "%s"' %
'", "'.join(self.args.files))
self._apply_tags(files_imported)
self._apply_labels(files_imported)
return files_imported
def _apply_tags(self, files_imported):
if not self.args.tag:
return
if len(files_imported) > 1:
print ('WARNING! No tags were applied, because tags '\
'must be unique but multiple files were imported.')
return
else:
for tagname in self.args.tag:
tag_data = {'tag': tagname}
tag = self.connection.post_data_tag(
files_imported[0].get('uuid'), tag_data)
print 'File "%s@%s" has been tagged as "%s"' % \
(files_imported[0]['value'].get('filename'),
files_imported[0].get('uuid'),
tag.get('tag'))
def _apply_labels(self, files_imported):
if not self.args.label:
return
for labelname in self.args.label:
for file_imported in files_imported:
label_data = {'label': labelname}
label = self.connection.post_data_label(
file_imported.get('uuid'), label_data)
print 'File "%s@%s" has been labeled as "%s"' % \
(file_imported['value'].get('filename'),
file_imported.get('uuid'),
label.get('label'))
class FileList(object):
def __init__(self, args):
self.args = args
verify_has_connection_settings()
server_url = get_server_url()
verify_server_is_running(url=server_url)
self.connection = Connection(server_url, token=get_token())
@classmethod
def get_parser(cls, parser):
parser.add_argument('file_id',
nargs='?',
metavar='FILE_IDENTIFIER',
help='Name or ID of file(s) to list.')
parser.add_argument('-d',
'--detail',
action='store_true',
help='Show detailed view of files')
parser.add_argument(
'-t', '--type',
choices=['imported', 'result', 'log', 'all'],
default='imported',
help='List only files of the specified type. '\
'(ignored when FILE_IDENTIFIER is given)')
parser.add_argument('-l',
'--label',
metavar='LABEL',
action='append',
help='Filter by label')
return parser
def run(self):
if self.args.file_id:
source_type = None
else:
source_type = self.args.type
offset = 0
limit = 10
while True:
data = self.connection.get_data_object_index_with_limit(
limit=limit,
offset=offset,
query_string=self.args.file_id,
source_type=source_type,
labels=self.args.label,
type='file')
if offset == 0:
print '[showing %s files]' % data.get('count')
self._list_files(data['results'])
if data.get('next'):
offset += limit
else:
break
def _list_files(self, files):
for file_data_object in files:
text = self._render_file(file_data_object)
if text is not None:
print text
def _render_file(self, file_data_object):
try:
file_identifier = '%s@%s' % (file_data_object['value'].get(
'filename'), file_data_object['uuid'])
except TypeError:
file_identifier = '@%s' % file_data_object['uuid']
if self.args.detail:
text = '---------------------------------------\n'
text += 'File: %s\n' % file_identifier
try:
text += ' - Imported: %s\n' % \
_render_time(file_data_object['datetime_created'])
text += ' - md5: %s\n' % file_data_object['value'].get('md5')
if file_data_object['value'].get('imported_from_url'):
text += ' - Source URL: %s\n' % \
file_data_object['value'].get('imported_from_url')
if file_data_object['value'].get('import_comments'):
text += ' - Import note: %s\n' % \
file_data_object['value']['import_comments']
except TypeError:
pass
else:
text = 'File: %s' % file_identifier
return text
