def set_params(self, params):
"""
Add a new set of parameters.
:type params: dict
:param params: dictionary of parameters indexed by step id (see :class:`WorkflowTestCase`)
"""
for step_id, step_params in _iteritems(params):
for name, value in _iteritems(step_params):
self.add_param(step_id, name, value)
def __init__(self,
test_id,
workflow,
inputs,
outputs,
output_history,
expected_outputs,
missing_tools,
results,
output_file_map,
output_folder=WorkflowTestCase.DEFAULT_OUTPUT_FOLDER,
errors=None):
self.test_id = test_id
self.workflow = workflow
self.inputs = inputs
self.outputs = outputs
self.errors = [] if errors is None else errors
self.output_history = output_history
self.expected_outputs = expected_outputs
self.output_folder = output_folder
self.missing_tools = missing_tools
self.output_file_map = output_file_map
self.results = results
self.failed_outputs = {
out[0]: out[1]
for out in _iteritems(self.results) if not out[1]
}
def write_header(self):
self.file.seek(0)
self.header[self.time_dimension] = self.image_i
header = "#INRIMAGE-4#{\n" + "\n".join(
[str(k) + '=' + str(v) for k, v in _iteritems(self.header)])
header = header + ('\n' * (252 - len(header))) + "##}"
self.file.write(header)
def find_missing_tools(self, workflow=None):
"""
Find tools required by the workflow to test and not installed on the configured Galaxy server.
:type workflow: :class:`bioblend.galaxy.objects.wrappers.Workflow`
:param workflow: an optional instance of :class:`bioblend.galaxy.objects.wrappers.Workflow`
:rtype: list
:return: the list of missing tools
"""
_logger.debug("Checking required tools ...")
workflow = self.get_galaxy_workflow() if not workflow else workflow
available_tools = self._galaxy_instance.tools.list()
missing_tools = []
_logger.debug(
"Available tools: %s", ", ".join(
["{0}, {1}".format(t.id, t.version) for t in available_tools]))
for order, step in _iteritems(workflow.steps):
if step.tool_id and len([
t for t in available_tools
if t.id == step.tool_id and t.version == step.tool_version
]) == 0:
missing_tools.append((step.tool_id, step.tool_version))
_logger.debug("Missing tools: {0}".format(
"None" if len(missing_tools) == 0 else ", ".join([
"{0} (version {1})".format(x[0], x[1]) for x in missing_tools
])))
_logger.debug("Checking required tools: DONE")
return missing_tools
def unload_workflows(self):
"""
Unload all workflows loaded by this :class:`WorkflowLoader` instance.
"""
if not self._galaxy_instance:
raise RuntimeError("WorkflowLoader not initialized")
for _, wf in _iteritems(self._workflows):
self.unload_workflow(wf.id)
def save_dict_to_json(filename,d):
"""
Saves a (flat) dictionary that can also contain numpy
arrays to a json file.
"""
with open(filename,'w') as fp:
dat = [(p,_var_to_json_safe(param)) for (p,param) in _iteritems(d)]
json.dump(dict(dat), fp)
def dict_recursive_update(d, u):
for k, v in _iteritems(u):
if isinstance(v, collections.Mapping):
r = dict_recursive_update(d.get(k, {}), v)
d[k] = r
else:
d[k] = u[k]
return d
def show_outputs(self, stream=_sys.stdout):
"""
Print workflow outputs (indexed by workflow step) to file.
"""
for step_id, step_outputs in _iteritems(self.outputs):
print("'{0}': {1}".format(
step_id,
", ".join([x["label"] for x in step_outputs.values()])),
file=stream)
def load_dict_from_json(filename):
"""
Loads a (flat) dictionary from a json file and converts
lists back into numpy arrays.
"""
with open(filename,'r') as fp:
dat = json.load(fp)
assert(type(dat) == dict)
dat = dict([(p,_json_safe_to_value(param)) for (p,param) in _iteritems(dat)])
return dat
def set_expected_outputs(self, expected_outputs):
"""
Add a new set of expected outputs (see :class:`WorkflowTestCase`).
:type expected_outputs: dict
:param expected_outputs: a dictionary structured as specified in :class:`WorkflowTestCase`
"""
for name, config in _iteritems(expected_outputs):
self.add_expected_output(name, config["file"],
config.get("comparator"))
def set_inputs(self, inputs):
"""
Update the mapping between workflow inputs and test datasets.
:param inputs: dict
:return: a dictionary of mappings (see :class:`WorkflowTestCase`)
"""
for name, config in _iteritems(inputs):
self.add_input(name, config["file"],
config["type"] if "type" in config else None)
def __init__(self, **kwargs):
for k, v in _iteritems(kwargs):
if not k.startswith('_'):
self.__dict__[save_name(k)] = self.__make_Ox(v)
else:
self.__dict__[k] = v
#super(Ox,self).__init__(**kwargs)
if not hasattr(self, '_item'):
self.__dict__['_item'] = kwargs
if not hasattr(self, '_original_type'):
self.__dict__['_original_type'] = 'dict'
def _parse_dict(elements):
results = {}
for name, value in _iteritems(elements):
result = value
if isinstance(value, _basestring):
result = {"name": name, "file": value}
elif isinstance(value, dict):
result["name"] = name
else:
raise ValueError("Configuration error: %r", elements)
results[name] = result
return results
def cleanup_output_folder(self, test_result=None):
"""
Perform a clean up of the temporary files produced during the workflow test execution.
"""
test_results = self._test_cases.values() if not test_result else [
test_result
]
for _test in test_results:
for output_name, output_map in _iteritems(_test.output_file_map):
_logger.debug("Cleaning output folder: %s", output_name)
if _os.path.exists(output_map["filename"]):
_os.remove(output_map["filename"])
_logger.debug("Deleted output file '%s'.",
output_map["filename"])
def pngsave(A, file, info={}):
"""
wrapper around PIL 1.1.7 Image.save to preserve PNG metadata
based on public domain script by Nick Galbreath
http://blog.modp.com/2007/08/python-pil-and-png-metadata-take-2.html
"""
from PIL import Image, PngImagePlugin
im = Image.fromarray(256.0 * A).convert('RGB')
reserved = ('interlace', 'gamma', 'dpi', 'transparency', 'aspect')
meta = PngImagePlugin.PngInfo()
for k, v in _iteritems(info):
if k in reserved: continue
meta.add_text(k, v, 0)
im.save(file, "PNG", pnginfo=meta)
def to_dict(self):
"""
Return a dictionary representation of the current class instance.
:rtype: dict
:return:
"""
return dict({
"name": self.name,
"file": self.filename,
"inputs": {
name: input_["file"][0]
for name, input_ in _iteritems(self.inputs)
},
"params": self.params,
"expected": self.expected_outputs
})
def _load_configuration(config_filename):
with open(config_filename) as config_file:
workflows_conf = None
try:
workflows_conf = _yaml_load(config_file)
except ValueError as e:
_logger.error(
"Configuration file '%s' is not a valid YAML or JSON file",
config_filename)
raise ValueError(
"Not valid format for the configuration file '%s'.",
config_filename)
# update inputs/expected fields
for wf_name, wf in _iteritems(workflows_conf["workflows"]):
wf["inputs"] = _parse_dict(wf["inputs"])
wf["expected"] = _parse_dict(wf["expected"])
return workflows_conf
def cleanup(self, output_folder=None):
"""
Perform a complete clean up of the data produced during the execution of a workflow test,
i.e., the uploaded workflow and the created history are removed from Galaxy and the actual
output datasets (downloaded from Galaxy) are deleted from the output path of the local file system.
"""
_logger.debug("Cleanup of workflow test '%s'...", self._uuid)
for test_uuid, test_result in _iteritems(self._test_cases):
if test_result.output_history:
self._galaxy_instance.histories.delete(
test_result.output_history.id)
self.cleanup_output_folder(test_result)
if self._galaxy_workflow:
self._workflow_loader.unload_workflow(self._galaxy_workflow.id)
self._galaxy_workflow = None
_logger.debug("Cleanup of workflow test '%s': DONE", self._uuid)
if output_folder and _os.path.exists(output_folder):
_shutil.rmtree(output_folder)
_logger.debug("Deleted WF output folder '%s': DONE", output_folder)
def __init__(self,filename,z=False, slice_at = None):
self.filename = filename
self.file = open(filename,'r')
self.raw_header = self.file.read(256)
self.header = dict([h.split('=') for h in self.raw_header.split('\n') if '=' in h])
self.header = dict([(k,litus._make_an_int_if_possible(v)) for k,v in _iteritems(self.header)])
self.z = z
self.last_image = np.zeros((50,50))
self.start_at = 0
self.stop_at = None
self.step_at = None
if slice_at is not None:
self.start_at = slice_at.start
self.stop_at = slice_at.stop
self.step_at = slice_at.step
if self.start_at is None:
self.start_at = 0
if self.step_at is None:
self.step_at = 1
self.image_i = self.start_at
def load(filename, output_folder=None):
if _os.path.exists(filename):
# TODO: catch YAML parsing errors
file_configuration = _load_configuration(filename)
base_path = file_configuration.get(
"base_path", _os.path.dirname(_os.path.abspath(filename)))
suite = WorkflowTestSuite(
galaxy_url=file_configuration.get("galaxy_url"),
galaxy_api_key=file_configuration.get("galaxy_api_key"),
enable_logger=file_configuration.get("enable_logger", False),
enable_debug=file_configuration.get("enable_debug", False),
disable_cleanup=file_configuration.get("disable_cleanup", False),
disable_assertions=file_configuration.get("disable_assertions", False),
output_folder=output_folder \
or file_configuration.get("output_folder") \
or WorkflowTestCase.DEFAULT_OUTPUT_FOLDER,
max_retries=file_configuration.get("max_retries", None),
retry_delay=file_configuration.get("retry_delay", None),
polling_interval=file_configuration.get("polling_interval", None)
)
for wf_name, wf_config in _iteritems(
file_configuration.get("workflows")):
wf_base_path = _os.path.join(base_path,
wf_config.get("base_path", ""))
wf_config["output_folder"] = _os.path.join(
suite.output_folder, wf_config.get("output_folder",
wf_name))
# add the workflow
w = WorkflowTestCase(name=wf_name,
base_path=wf_base_path,
workflow_filename=wf_config["file"],
inputs=wf_config["inputs"],
params=wf_config.get("params", {}),
expected_outputs=wf_config["expected"],
output_folder=wf_config["output_folder"])
suite.add_workflow_test(w)
return suite
else:
raise ValueError("Filename '{0}' not found".format(filename))
def png_client(images,
info={},
port=10000,
host='localhost',
compress_level=0,
resize=(1.0, 1.0)):
if len(images.shape) == 2:
images = [images]
# Create a TCP/IP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Connect the socket to the port where the server is listening
server_address = (host, port)
sock.connect(server_address)
try:
for A in images:
im = Image.fromarray(256.0 * A).convert('RGB')
if resize != (1.0, 1.0) and resize is not None:
if not type(resize) == tuple:
resize = (resize, resize)
im = im.resize(
(int(resize[0] * im.size[0]), int(resize[1] * im.size[1])),
PIL.Image.ANTIALIAS)
output = StringIO.StringIO()
meta = PngImagePlugin.PngInfo()
reserved = ('interlace', 'gamma', 'dpi', 'transparency', 'aspect')
for k, v in _iteritems(info):
if k in reserved:
continue
meta.add_text(str(k), str(v), 0)
im.save(output,
format="PNG",
pnginfo=meta,
compress_level=compress_level)
message = output.getvalue()
output.close()
sock.sendall(message)
finally:
sock.close()
def describe_html(v, wrap_in_html=True, **kwargs):
from IPython.display import HTML
import uuid
try:
import html
except:
import cgi as html # fallback escape function
if isinstance(v, variables.Parameter) or isinstance(
v, variables.Variable) or isinstance(v,
torch.nn.parameter.Parameter):
d = {}
for k in [
'name', '_name', 'simple_name', 'doc', 'config_key',
'optimizable', 'node', 'save', 'init', 'get', 'set',
'variable_type', 'auto_name'
]:
if has_convis_attribute(v, k):
d[k] = get_convis_attribute(v, k)
name = d.get('name', '') # optional: None handling
if not type(name) is str or name is '':
name = d.get('_name', '')
if not type(name) is str or name is '':
name = repr(v)
if type(v) is torch.nn.parameter.Parameter:
name = 'torch.nn.Parameter'
if has_convis_attribute(v, 'html_name'):
name += ' ' + str(get_convis_attribute(v, 'html_name'))
#simple_name = str(d.get('simple_name',''))
s = """<div class='convis_description variable'><b """ + on_click_toggle + """>""" + name + """</b> <small>""" + d.get(
'variable_type', '') + """</small>"""
# default: show everything, hide on click;
s += "<div class='description_content_replacer' style='border-left: 2px solid #eee; padding-left: 5px; margin-bottom: 10px; display: none;'>(…)</div>"
s += "<div class='description_content' style='border-left: 2px solid #eee; border-top: 2px solid #f8f8f8; padding-left: 5px; margin-bottom: 10px; margin-top: 2px;'>"
if has_convis_attribute(v, 'path'):
s += "<small>" + full_path(v) + "</small><br/>"
if has_convis_attribute(
v, 'doc') and get_convis_attribute(v, 'doc') != '':
s += '<p class="doc" style="padding:2px;">' + get_convis_attribute(
v, 'doc') + '</p>'
if has_convis_attribute(v, 'owner'):
s += "<tt style='color: gray;'><small>" + str(
v.owner) + "</small></tt><br/>"
for k in [
'auto_name', 'config_key', 'optimizable', 'node', 'save',
'init', 'get', 'set', 'state_out_state', 'param_init',
'state_init', 'state_in_state', 'copied_from', 'config_key',
'config_default'
]:
if has_convis_attribute(v, k):
if isinstance(get_convis_attribute(v, k), MethodType):
s += '<div><b>' + str(k) + '</b>: <tt>method</tt></div>'
elif isinstance(get_convis_attribute(v, k), FunctionType):
s += '<div><b>' + str(k) + '</b>: <tt>function</tt></div>'
else:
s += '<div><b>' + str(k) + '</b>: <tt>' + html.escape(
str(get_convis_attribute(v, k))) + '</tt></div>'
try:
if hasattr(v, 'get_value'):
s += '<b>value</b>: ' + str(
_tensor_to_html(v.get_value(), title=name, **kwargs))
except Exception as e:
s += '<b>value</b>: ' + str(e)
pass
try:
s += '<b>got</b>: ' + _tensor_to_html(get_convis_attribute(
v, 'get')(variables.create_context_O(v)),
title=name,
**kwargs)
except:
pass
vv = v
if hasattr(v, 'detach'):
vv = v.detach()
if hasattr(vv, '__array__'):
s += '<b>value </b>: ' + _tensor_to_html(vv.__array__())
elif hasattr(vv, 'data'):
s += '<b>value </b>: ' + _tensor_to_html(vv.data.__array__())
s += """</div>"""
s += """</div>"""
if not wrap_in_html:
return s
return HTML(s)
##
# Handeling other datatypes
#
if type(v) == int or type(v) == float:
s = str(v)
if not wrap_in_html:
return s
return HTML(s)
elif hasattr(v, '__array__'):
if hasattr(v, 'detach'):
s = _tensor_to_html(v.detach().__array__(), **kwargs)
else:
s = _tensor_to_html(v.__array__(), **kwargs)
if not wrap_in_html:
return s
return HTML(s)
if isinstance(v, torch.nn.Module):
return describe_layer_with_html(v, 4, wrap_in_html)
if isinstance(v, ModuleType):
uid = uuid.uuid4().hex
s = """<div class='convis_description module'><b """ + on_click_toggle + """>""" + getattr(
v, '__name__', '(nameless module)') + """</b>"""
s += "<div class='description_content_replacer' style='border-left: 2px solid #eee; padding-left: 5px; margin-bottom: 10px; display: none;'>(…)</div>"
s += "<div class='description_content' style='border-left: 2px solid #eee; border-top: 2px solid #f8f8f8; padding-left: 5px; margin-bottom: 10px; margin-top: 2px;'>"
s += '<pre>' + str(getattr(v, '__doc__',
"(no doc string found)")) + '</pre>'
for f in dir(v):
if f.startswith('_'):
continue
vv = getattr(v, f)
if isinstance(vv, ModuleType):
s += "<div class='convis_description dict_item'><b>" + str(
f) + "</b> (module " + str(getattr(vv, '__name__',
'')) + ")</div>"
continue
s += "<div class='convis_description dict_item'><b id=" + uid + save_name(
f
) + " " + on_click_toggle + " >" + str(
f
) + "</b> <a style=\"text-decoration: none;\" href='#" + uid + "''>↩</a>"
s += "<div class='description_content_replacer' style='border-left: 0px solid #ddd; padding-left: 5px; display: none;'>(…)</div>"
s += "<div class='description_content' style='border-left: 0px solid #ddd; padding-left: 5px;'>"
s += describe_html(vv, wrap_in_html=False, **kwargs)
s += "</div>"
s += "</div>"
s += """</div>"""
s += """</div>"""
if not wrap_in_html:
return s
return HTML(s)
if isinstance(v, FunctionType):
s = """<div class='convis_description module'><b """ + on_click_toggle + """>""" + getattr(
v, '__name__', '(nameless function)') + """</b>"""
s += "<div class='description_content_replacer' style='border-left: 2px solid #eee; padding-left: 5px; margin-bottom: 10px; display: none;'>(…)</div>"
s += "<div class='description_content' style='border-left: 2px solid #eee; border-top: 2px solid #f8f8f8; padding-left: 5px; margin-bottom: 10px; margin-top: 2px;'>"
s += '<pre>' + str(getattr(v, '__doc__',
"(no doc string found)")) + '</pre>'
s += """</div>"""
s += """</div>"""
if not wrap_in_html:
return s
return HTML(s)
if inspect.isclass(v):
s = """<div class='convis_description module'><b """ + on_click_toggle + """>""" + getattr(
v, '__name__', '(nameless class)') + """</b>"""
s += "<div class='description_content_replacer' style='border-left: 2px solid #eee; padding-left: 5px; margin-bottom: 10px; display: none;'>(…)</div>"
s += "<div class='description_content' style='border-left: 2px solid #eee; border-top: 2px solid #f8f8f8; padding-left: 5px; margin-bottom: 10px; margin-top: 2px;'>"
s += '<pre>' + str(getattr(v, '__doc__',
"(no doc string found)")) + '</pre>'
s += """</div>"""
s += """</div>"""
if not wrap_in_html:
return s
return HTML(s)
if type(v) in [dict] or hasattr(v, '__iteritems__'):
uid = uuid.uuid4().hex
s = "<div class='convis_description list'>"
iteration = list(v.__iteritems__() if hasattr(v, '__iteritems__'
) else _iteritems(v))
s += "<b id=" + uid + " " + on_click_toggle + " >+</b> "
for (k, vv) in iteration:
s += '| <a style="text-decoration: none; font-size: 8pt;" href="#' + uid + save_name(
k) + '">' + str(k) + '</a> '
s += "<div class='description_content_replacer' style='border-left: 4px solid #f0f0f0; border-top: 4px solid #f8f8f8; padding-left: 10px; margin-bottom: 10px; display: none;'>(…)</div>"
s += "<div class='description_content' style='border-left: 4px solid #f0f0f0; border-top: 4px solid #f8f8f8; padding-left: 10px; margin-bottom: 10px;'>"
iteration = list(v.__iteritems__() if hasattr(v, '__iteritems__'
) else _iteritems(v))
path = kwargs.pop('path', '')
for (k, vv) in iteration:
s += "<div class='convis_description dict_item'><small>" + path + ".</small><b id=" + uid + save_name(
k
) + " " + on_click_toggle + " >" + str(
k
) + "</b> <a style=\"text-decoration: none;\" href='#" + uid + "''>↩</a>"
s += "<div class='description_content_replacer' style='border-left: 0px solid #ddd; padding-left: 5px; display: none;'>(…)</div>"
s += "<div class='description_content' style='border-left: 0px solid #ddd; padding-left: 5px;'>"
s += describe_html(vv,
wrap_in_html=False,
path=path + '.' + k,
**kwargs)
s += "</div>"
s += "</div>"
s += "</div>"
s += "</div>"
if not wrap_in_html:
return s
return HTML(s)
if type(v) in [list, tuple] or hasattr(v, '__iter__'):
try:
s = "<div class='convis_description list'><b " + on_click_toggle + ">List (" + str(
len(v)) + "):</b>"
s += "<div class='description_content_replacer' style='border-left: 4px solid #f0f0f0; border-top: 4px solid #f8f8f8; padding-left: 10px; margin-bottom: 10px; display: none;'>(…)</div>"
s += "<div class='description_content' style='border-left: 4px solid #f0f0f0; border-top: 4px solid #f8f8f8; padding-left: 10px; margin-bottom: 10px;'>"
s += '\n'.join(
[describe_html(vv, wrap_in_html=False, **kwargs) for vv in v])
s += "</div>"
s += "</div>"
if not wrap_in_html:
return s
return HTML(s)
except:
# Tensor Variables love to raise TypeErrors when iterated over
pass
##
# Assuming its a annotated variable:
#
d = {}
for k in [
'name', 'simple_name', 'doc', 'config_key', 'optimizable', 'node',
'save', 'init', 'get', 'set', 'variable_type', 'auto_name'
]:
if has_convis_attribute(v, k):
d[k] = get_convis_attribute(v, k)
name = d.get('name', '') # optional: None handling
if not type(name) is str or name is '':
name = repr(v)
if has_convis_attribute(v, 'html_name'):
name += ' ' + str(get_convis_attribute(v, 'html_name'))
#simple_name = str(d.get('simple_name',''))
s = """<div class='convis_description variable'><b """ + on_click_toggle + """>""" + name + """</b> <small>""" + d.get(
'variable_type', '') + """</small>"""
# default: show everything, hide on click;
s += "<div class='description_content_replacer' style='border-left: 2px solid #eee; padding-left: 5px; margin-bottom: 10px; display: none;'>(…)</div>"
s += "<div class='description_content' style='border-left: 2px solid #eee; border-top: 2px solid #f8f8f8; padding-left: 5px; margin-bottom: 10px; margin-top: 2px;'>"
if has_convis_attribute(v, 'path'):
s += "<small>" + full_path(v) + "</small><br/>"
if has_convis_attribute(v, 'doc') and get_convis_attribute(v, 'doc') != '':
s += '<p class="doc" style="padding:2px;">' + get_convis_attribute(
v, 'doc') + '</p>'
if has_convis_attribute(v, 'owner'):
s += "<tt style='color: gray;'><small>" + str(
v.owner) + "</small></tt><br/>"
for k in [
'auto_name', 'config_key', 'optimizable', 'node', 'save', 'init',
'get', 'set', 'state_out_state', 'param_init', 'state_init',
'state_in_state', 'copied_from', 'config_key', 'config_default'
]:
if has_convis_attribute(v, k):
if isinstance(get_convis_attribute(v, k), MethodType):
s += '<div><b>' + str(k) + '</b>: <tt>method</tt></div>'
elif isinstance(get_convis_attribute(v, k), FunctionType):
s += '<div><b>' + str(k) + '</b>: <tt>function</tt></div>'
else:
s += '<div><b>' + str(k) + '</b>: <tt>' + html.escape(
str(get_convis_attribute(v, k))) + '</tt></div>'
try:
if hasattr(v, 'get_value'):
s += '<b>value</b>: ' + str(
_tensor_to_html(v.get_value(), title=name, **kwargs))
except Exception as e:
s += '<b>value</b>: ' + str(e)
pass
try:
s += '<b>got</b>: ' + _tensor_to_html(get_convis_attribute(v, 'get')(
variables.create_context_O(v)),
title=name,
**kwargs)
except:
pass
s += """</div>"""
s += """</div>"""
if not wrap_in_html:
return s
return HTML(s)
def register_users(galaxy_instance,
users_list,
create_api_key=False,
report_filename=None):
if report_filename:
if _os.path.exists(report_filename):
raise ValueError(
"Output file {} already exists. Won't overwrite".format(
report_filename))
ofile = open(report_filename, 'w')
else:
ofile = _sys.stdout
try:
actual_users = {
u["email"]: u
for u in galaxy_instance.users.get_users()
}
logger.debug("Number of users to register: %d", len(users_list))
galaxy_users = [] # registered_galaxy_users.values()
for username, u_data in _iteritems(users_list):
try:
if not u_data["email"] in actual_users:
logger.debug("Registering user: %s - %s",
u_data["username"], u_data["email"])
user = galaxy_instance.users.create_local_user(
u_data["username"], u_data["email"],
u_data["password"])
u_data["galaxy_id"] = user["id"]
if create_api_key:
api_key = galaxy_instance.users.create_user_apikey(
user['id'])
logger.debug("Create API_KEY %s for user %s", api_key,
username)
galaxy_users.append({
"galaxy_id":
user["id"],
"uid":
u_data["uid"],
"username":
u_data["username"],
"email":
u_data["email"],
"api_key":
api_key,
"password":
u_data["password"],
"hashed_password":
u_data["hashed_password"]
})
logger.info(
"User registered with ID '%s' and API_KEY '%s'",
u_data["id"], api_key)
else:
galaxy_users.append({
"galaxy_id":
user["id"],
"uid":
u_data["uid"],
"username":
u_data["username"],
"email":
u_data["email"],
"password":
u_data["password"],
"hashed_password":
u_data["hashed_password"]
})
logger.info("User registered with ID: %s", user["id"])
except Exception as e:
logger.debug(e)
logger.info("Registered users: %d", len(galaxy_users))
logger.debug("Writing report to '%s' %d users", report_filename,
len(galaxy_users))
writer = _csv.DictWriter(ofile,
fieldnames=[
"uid", "galaxy_id", "username", "email",
"password", "hashed_password", "api_key"
])
writer.writeheader()
writer.writerows(galaxy_users)
finally:
if report_filename:
ofile.close()
def extract_workflow(self,
filename=None,
workflow_name=None,
v_step=100,
h_step=400):
if workflow_name is None:
workflow_name = "Workflow extracted from history {0}".format(
self._history.id)
# start
self._logger.info("Extracting Workflow from history...")
# wf object representation
wf = _collections.OrderedDict({
"a_galaxy_workflow": "true",
"annotation": "",
"format-version": "0.1",
"name": workflow_name,
"uuid": str(_uuid.uuid1()),
"steps": _collections.OrderedDict()
})
# position
p_left = 50
p_top = 0
# process steps
inputs = {
k: v
for k, v in _iteritems(self._input_order_map)
if k in self.input_datasets
}
for hds_id, index in sorted(_iteritems(inputs),
key=_operator.itemgetter(1)):
input_name = self.input_dataset_labels[hds_id]
input_description = ""
p_top += v_step
wf["steps"][str(index)] = {
"annotation": "",
"content_id": None,
"id": index,
"input_connections": {},
"inputs": [{
"description": input_description,
"name": input_name
}],
"label": None,
"name": "Input dataset",
"outputs": [],
"position": {
"left": p_left,
"top": p_top
},
"tool_errors": None,
"tool_id": None,
"tool_state": _json.dumps({"name": input_name}),
"tool_version": None,
"type": "data_input",
"uuid": str(_uuid.uuid1()),
"workflow_outputs": []
}
# reset top position
p_top = p_top - (v_step * (len(self.input_datasets) / 2))
# process intermediate and final steps
for job_id, job in _iteritems(self.processing_jobs):
# update top position
p_left += h_step
# compute the step index
index = len(wf["steps"])
# get the tool related to the current job
tool = self._get_tool(job.wrapped["tool_id"])
# compute params
params = {"__page__": 0, "__rerun_remap_job_id__": None}
tool_inputs = {
param["name"]: param
for param in tool.wrapped["inputs"]
}
for param_name, param_info in _iteritems(job.wrapped["params"]):
if param_name in tool_inputs:
params[param_name] = param_info
# add inputs to tool state and inputs
inputs = []
input_connections = {}
for job_input_name, job_input_info in _iteritems(
job.wrapped["inputs"]):
params[job_input_name] = _json.dumps(
{"__class__": "RuntimeValue"})
inputs.append({
"description":
"Runtime input value {0}".format(job_input_name),
"name":
job_input_name
})
output_name = self.intermediate_dataset_labels[job_input_info["id"]] \
if job_input_info["id"] in self.intermediate_dataset_labels else "output"
input_connections[job_input_name] = {
"id": self._input_order_map[job_input_info["id"]],
"output_name": output_name
}
# add outputs
outputs = []
workflow_outputs = []
tool_outputs = {
param["name"]: param
for param in tool.wrapped["outputs"]
}
for job_output_name, job_output_info in _iteritems(
job.wrapped["outputs"]):
outputs.append({
"name": job_output_name,
"type": tool_outputs[job_output_name]["format"]
})
# TODO: check if exists cryteria for detecting if an output
# is a workflow output or a simple intermediate output
workflow_outputs.append({
"label": job_output_name,
"output_name": job_output_name,
"uuid": str(_uuid.uuid1())
})
wf["steps"][str(index)] = {
"annotation": "",
"content_id": tool.id,
"id": index,
"input_connections": input_connections,
"inputs": inputs,
"label": None,
"name": tool.name,
"outputs": outputs,
"position": {
"left": p_left,
"top": p_top
},
"tool_errors": None,
"tool_id": tool.id,
"tool_state": _json.dumps(params),
"tool_version": tool.version,
"type": "tool",
"uuid": str(_uuid.uuid1()),
"workflow_outputs": workflow_outputs
}
# save workflow
if filename is not None:
self._logger.info("Saving workflow to file...")
with open(filename, "w") as fp:
self._logger.debug("Workflow file path: %s", filename)
_json.dump(wf, fp, indent=4)
self._logger.info("Saving workflow to file: done")
# extraction wf end
self._logger.info("Extracting Workflow from history: done")
return wf
def _process_history(self):
# check if a history has been assigned
if self._history is None:
raise RuntimeError("No history found!")
# auxiliary infos
ds_input_info = {}
ds_output_info = {}
intermediate_datasets = []
# get history datasets
history = self._history
self.datasets = history.get_datasets()
# process jobs chain (through their created datasets)
for ds in self.datasets:
self._logger.info("Processing dataset %s ... ", ds.id)
# load job info
creating_job = self._get_job(ds.wrapped["creating_job"])
job_inputs = {
in_info["id"]: in_name
for in_name, in_info in _iteritems(
creating_job.wrapped["inputs"])
}
job_outputs = {
out_info["id"]: out_name
for out_name, out_info in _iteritems(
creating_job.wrapped["outputs"])
}
intermediate_datasets += list(job_inputs)
# update auxiliary data info
for in_id, in_name in _iteritems(job_inputs):
if in_id not in ds_input_info:
ds_input_info[in_id] = {}
ds_input_info[in_id][creating_job.id] = in_name
for out_id, out_name in _iteritems(job_outputs):
if out_id not in ds_output_info:
ds_output_info[out_id] = {}
ds_output_info[out_id][creating_job.id] = out_name
# register the job as the creating of this DS
self.creating_jobs[ds.id] = creating_job
# detect if the job creates an input DS
# or it is a processing job
if len(job_inputs) == 0:
self.input_datasets[ds.id] = ds
else:
# add the processing job
self.processing_jobs[creating_job.id] = creating_job
# compute the processing job level
self.processing_jobs[creating_job.id]
# update in/out maps
if creating_job.id not in self.job_input_ids:
self.job_input_ids[creating_job.id] = list(job_inputs)
self.job_output_ids[creating_job.id] = list(job_outputs)
self._logger.info("Process dataset %s: done ", ds.id)
self._logger.info("Processing extra info...")
# Auxiliary function which computes the label for a given dataset
def __set_label(labels, ds_id, info_matrix, label=None, prefix=None):
if label is not None:
labels[ds_id] = label
elif ds_id in info_matrix and len(info_matrix[ds_id]) == 1:
# use job
labels[ds_id] = info_matrix[ds_id][list(info_matrix[ds_id])[0]]
else:
# use a default label if the same dataset if used by more than one job
labels[ds_id] = "{0}_{1}".format(prefix, len(labels))
# process datasets to:
# - determine intermediate and output datasets
# - determine input/output labels
for ds in self.datasets:
if ds.id in self.input_datasets:
__set_label(self.input_dataset_labels,
ds.id,
ds_input_info,
prefix="input")
__set_label(self.intermediate_dataset_labels,
ds.id,
ds_input_info,
label="output")
else:
if ds.id in intermediate_datasets:
self.intermediate_datasets[ds.id] = ds
__set_label(self.input_dataset_labels,
ds.id,
ds_input_info,
prefix="input")
__set_label(self.intermediate_dataset_labels,
ds.id,
ds_output_info,
prefix="output")
else:
self.output_datasets[ds.id] = ds
__set_label(self.output_dataset_labels,
ds.id,
ds_output_info,
prefix="output")
intermediate_inputs = []
not_ordered_inputs = list(self.input_datasets)
input_datasets = _collections.OrderedDict()
inputs = list(self.input_datasets)
self._input_order_map = {x: inputs.index(x) for x in inputs}
# determine the job level
self._logger.debug("Processing JOB levels ...")
for job_id, job in _iteritems(self.processing_jobs):
# compute and set the job level
self.processing_job_levels[
job_id] = self.compute_processing_job_level(job_id)
# order inputs
tool = self._get_tool(job.wrapped["tool_id"])
ordered_names = [x["name"] for x in tool.wrapped["inputs"]]
for name in ordered_names:
if name in job.wrapped["inputs"]:
in_id = job.wrapped["inputs"][name]["id"]
if in_id in not_ordered_inputs:
input_datasets[in_id] = self.input_datasets[in_id]
if in_id not in self.input_datasets:
self._input_order_map[in_id] = len(
self.input_datasets
) + self.processing_job_levels[job_id]
not_ordered_inputs.remove(in_id)
# add intermediate inputs
for x in job.wrapped["outputs"].values():
if x["id"] not in self.output_datasets:
intermediate_inputs.append(x["id"])
if x["id"] not in self.input_datasets:
self._input_order_map[x["id"]] = len(
self.input_datasets
) + self.processing_job_levels[job_id]
self._logger.debug("JOB levels processing: done")
# copy remaining inputs
for ds_in in not_ordered_inputs:
input_datasets[ds_in] = self.input_datasets[ds_in]
self.input_datasets = input_datasets
self._logger.info("Processing extra info: done")
def _get_workflow_info(filename,
galaxy_url,
galaxy_api_key,
tool_folder=DEFAULT_TOOLS_FOLDER):
inputs = []
params = _CommentedMap()
outputs = {}
# loading wf info start
_logger.debug("Loading workflow definition from %s file...", filename)
# setup galaxy instance
galaxy_instance = _common.get_galaxy_instance(galaxy_url, galaxy_api_key)
galaxy_tool_client = _ToolClient(
galaxy_instance.gi) # get the non-object version of the GI
if not _os.path.exists(DEFAULT_TOOLS_FOLDER):
_os.makedirs(DEFAULT_TOOLS_FOLDER)
with open(filename) as fp:
wf_config = _json.load(fp)
for sid, step in _iteritems(wf_config["steps"]):
# tool = gi.tools.get()
_logger.debug("Processing step '%s' -- '%s'", sid, step["name"])
# an input step....
if not step["tool_id"] and step["type"] == "data_input":
for input_ in step["inputs"]:
_logger.debug("Processing input: '%s' (%s)", input_["name"],
input_["description"])
inputs.append(input_)
# a processing step (with outputs) ...
if step["tool_id"] and step["type"] == "tool":
# tool parameters
tool_params = _CommentedMap()
# process tool info to extract parameters
tool_id = step["tool_id"]
# tool = galaxy_instance.tools.get(tool_id)
## LP: re-write this using the bioblend.objects API to fetch the tool
# inputs. See the comment above `def _process_tool_param_element`
# tool_config_xml = _os.path.basename(tool.wrapped["config_file"])
# _logger.debug("Processing step tool '%s'", tool_id)
#
# try:
# _logger.debug("Download TOOL '%s' definition file XML: %s....", tool_id, tool_config_xml)
# targz_filename = _os.path.join(DEFAULT_TOOLS_FOLDER, tool_id + ".tar.gz")
# targz_content = galaxy_tool_client._get(_os.path.join(tool_id, "download"), json=False)
# if targz_content.status_code == 200:
# with open(targz_filename, "w") as tfp:
# tfp.write(targz_content.content)
# tar = _tarfile.open(targz_filename)
# tar.extractall(path=tool_folder)
# tar.close()
# _os.remove(targz_filename)
# _logger.debug("Download TOOL '%s' definition file XML: %s....: DONE", tool_id, tool_config_xml)
# else:
# _logger.debug("Download TOOL '%s' definition file XML: %s....: ERROR %r",
# tool_id, tool_config_xml, targz_content.status_code)
#
# tool_config_xml = _os.path.join(DEFAULT_TOOLS_FOLDER, tool_config_xml)
# if _os.path.exists(tool_config_xml):
# tree = _etree.parse(tool_config_xml)
# root = tree.getroot()
# inputs_el = root.find("inputs")
# for input_el in inputs_el:
# _process_tool_param_element(input_el, tool_params)
# if len(tool_params) > 0:
# params.insert(int(sid), sid, tool_params)
#
# except _StandardError as e:
# _logger.debug("Download TOOL '%s' definition file XML: %s....: ERROR", tool_id, tool_config_xml)
# _logger.error(e)
# process
outputs[str(sid)] = {}
for output in step["workflow_outputs"]:
outputs[str(sid)][output["uuid"]] = output
# loading wf info end
_logger.debug("Workflow definition loaded from %s file...", filename)
# return loaded info
return wf_config, inputs, params, outputs
def __init__(self, **enums):
for name, val in _iteritems(enums):
self.__dict__[name] = val
def run_test(self,
base_path=None,
inputs=None,
params=None,
expected_outputs=None,
output_folder=None,
disable_assertions=None,
disable_cleanup=None,
enable_logger=None,
enable_debug=None):
"""
Run the workflow test which this runner is associated to.
The parameters ``base_path``, ``inputs``, ``outputs``, ``expected_outputs``
``output_folder``, ``disable_assertions``, ``disable_cleanup``, ``enable_logger``, ``enable_debug``
can be provided to override the corresponding defined in the :class:`WorkflowTestCase` instance
which this runner is related to (see :class:`WorkflowTestCase` for more details).
:rtype: :class:`WorkflowTestResult`
:return: the :class:`WorkflowTestResult` instance which represents the test result
"""
# update test settings
if enable_logger is None \
and self._workflow_test_config is not None \
and hasattr(self._workflow_test_config, "enable_logger"):
enable_logger = self._workflow_test_config.enable_logger
if enable_debug is None \
and self._workflow_test_config is not None \
and hasattr(self._workflow_test_config, "enable_debug"):
enable_debug = self._workflow_test_config.enable_debug
if disable_cleanup is None:
disable_cleanup = self._workflow_test_config.disable_cleanup
if disable_assertions is None:
disable_assertions = self._workflow_test_config.disable_assertions
# set basepath
base_path = self._base_path if not base_path else base_path
# load workflow
workflow = self.get_galaxy_workflow()
# output folder
if output_folder is None:
output_folder = self._workflow_test_config.output_folder
# update logger
if enable_logger or enable_debug:
_common.LoggerManager.update_log_level(
_logging.DEBUG if enable_debug else _logging.INFO)
if disable_cleanup:
self._file_handler = _common.LoggerManager.enable_log_to_file(
output_folder=output_folder,
log_filename="-".join([
"WorkflowTestCase", self.worflow_test_name, self.uuid
]) + ".log")
else:
_common.LoggerManager.update_log_level(_logging.ERROR)
_empty_logger.info("")
_logger.info("Running workflow testcase: %r",
self._workflow_test_config.name)
_logger.debug("TestCase configuration: %r",
self._workflow_test_config.__dict__)
# check input_map
if inputs is None:
if len(self._workflow_test_config.inputs) > 0:
inputs = self._workflow_test_config.inputs
else:
raise ValueError("No input configured !!!")
# check params
if params is None:
params = self._workflow_test_config.params
_logger.debug("Using default params")
# check expected_output_map
if expected_outputs is None:
if len(self._workflow_test_config.expected_outputs) > 0:
expected_outputs = self._workflow_test_config.expected_outputs
else:
raise ValueError("No output configured !!!")
# update config options
disable_cleanup = disable_cleanup if disable_cleanup is not None else self._disable_cleanup
disable_assertions = disable_assertions if disable_assertions is not None else self._disable_assertions
output_folder = output_folder if output_folder is not None else self._output_folder
# uuid of the current test
test_uuid = self.uuid
# store the current message
error_msg = None
# test restul
test_result = None
# check tools
errors = []
missing_tools = self.find_missing_tools()
if len(missing_tools) == 0:
try:
# create a new history for the current test
history = self._galaxy_instance.histories.create("-".join([
_core.WorkflowTestCase.DEFAULT_HISTORY_NAME_PREFIX,
self._workflow_test_config.name.replace(" ", ""), test_uuid
]))
_logger.info("Create a history '%s' (id: %r)", history.name,
history.id)
# upload input data to the current history
# and generate the datamap INPUT --> DATASET
datamap = {}
for label, config in _iteritems(inputs):
datamap[label] = []
for filename in config["file"]:
dataset_filename = filename if _os.path.isabs(
filename) else _os.path.join(base_path, filename)
if config["type"]:
datamap[label].append(
history.upload_dataset(
dataset_filename,
file_type=config["type"]))
else:
datamap[label].append(
history.upload_dataset(dataset_filename))
# run the workflow
_logger.debug("About to launch workflow.")
_logger.debug("history: %r", history)
_logger.debug("datamap: %r", datamap)
_logger.debug("params: %r", params)
_logger.info("Workflow '%s' (id: %s) running ...",
workflow.name, workflow.id)
outputs, output_history = workflow.run(
datamap,
history,
params=params,
wait=True,
polling_interval=self._galaxy_instance.polling_interval)
_logger.info("Workflow '%s' (id: %s) executed", workflow.name,
workflow.id)
# check outputs
results, output_file_map = self._check_outputs(
base_path, outputs, expected_outputs, output_folder)
# instantiate the result object
test_result = _core.WorkflowTestResult(
test_uuid, workflow, inputs, outputs, output_history,
expected_outputs, missing_tools, results, output_file_map,
output_folder)
if test_result.failed():
error_msg = "The actual output{0} {2} differ{1} from the expected one{0}." \
.format("" if len(test_result.failed_outputs) == 1 else "s",
"" if len(test_result.failed_outputs) > 1 else "s",
", ".join(["'{0}'".format(n) for n in test_result.failed_outputs]))
except RuntimeError as e:
error_msg = "Runtime error: {0}".format(e.message)
errors.append(error_msg)
_logger.debug(error_msg)
else:
error_msg = "Some workflow tools are not available in Galaxy: {0}".format(
", ".join([
"{0} (ver. {1})".format(t[0], t[1]) for t in missing_tools
]))
errors.append(error_msg)
_logger.debug(error_msg)
# instantiate the result object
if not test_result:
test_result = _core.WorkflowTestResult(test_uuid, workflow, inputs,
[], None, expected_outputs,
missing_tools, {}, {},
output_folder, errors)
# store result
self._test_cases[test_uuid] = test_result
if self._test_suite_runner:
self._test_suite_runner._add_test_result(test_result)
# FIXME
self.test_result = test_result
# cleanup
if not disable_cleanup:
self.cleanup(output_folder)
# disable file logger
if self._file_handler is not None:
_common.LoggerManager.remove_file_handler(self._file_handler,
not disable_cleanup)
self._file_handler = None
# raise error message
if error_msg:
if not disable_assertions:
raise AssertionError(error_msg)
return test_result
def delete_users(galaxy_instance, users):
logger.debug("Number of users to delete: %d", len(users))
for _, user in _iteritems(users):
galaxy_instance.users.delete_user(user["galaxy_id"])
logger.info("Deleted user %s: %s", user["galaxy_id"], user["email"])
def __str__(self):
return "Test {0}: workflow {1}, intputs=[{2}], outputs=[{3}]" \
.format(self.test_id, self.workflow.name,
",".join([i for i in self.inputs]),
", ".join(["{0}: {1}".format(x[0], "OK" if x[1] else "ERROR")
for x in _iteritems(self.results)]))
