def set_params(self, params): """ Add a new set of parameters. :type params: dict :param params: dictionary of parameters indexed by step id (see :class:`WorkflowTestCase`) """ for step_id, step_params in _iteritems(params): for name, value in _iteritems(step_params): self.add_param(step_id, name, value)
def __init__(self, test_id, workflow, inputs, outputs, output_history, expected_outputs, missing_tools, results, output_file_map, output_folder=WorkflowTestCase.DEFAULT_OUTPUT_FOLDER, errors=None): self.test_id = test_id self.workflow = workflow self.inputs = inputs self.outputs = outputs self.errors = [] if errors is None else errors self.output_history = output_history self.expected_outputs = expected_outputs self.output_folder = output_folder self.missing_tools = missing_tools self.output_file_map = output_file_map self.results = results self.failed_outputs = { out[0]: out[1] for out in _iteritems(self.results) if not out[1] }
def write_header(self): self.file.seek(0) self.header[self.time_dimension] = self.image_i header = "#INRIMAGE-4#{\n" + "\n".join( [str(k) + '=' + str(v) for k, v in _iteritems(self.header)]) header = header + ('\n' * (252 - len(header))) + "##}" self.file.write(header)
def find_missing_tools(self, workflow=None): """ Find tools required by the workflow to test and not installed on the configured Galaxy server. :type workflow: :class:`bioblend.galaxy.objects.wrappers.Workflow` :param workflow: an optional instance of :class:`bioblend.galaxy.objects.wrappers.Workflow` :rtype: list :return: the list of missing tools """ _logger.debug("Checking required tools ...") workflow = self.get_galaxy_workflow() if not workflow else workflow available_tools = self._galaxy_instance.tools.list() missing_tools = [] _logger.debug( "Available tools: %s", ", ".join( ["{0}, {1}".format(t.id, t.version) for t in available_tools])) for order, step in _iteritems(workflow.steps): if step.tool_id and len([ t for t in available_tools if t.id == step.tool_id and t.version == step.tool_version ]) == 0: missing_tools.append((step.tool_id, step.tool_version)) _logger.debug("Missing tools: {0}".format( "None" if len(missing_tools) == 0 else ", ".join([ "{0} (version {1})".format(x[0], x[1]) for x in missing_tools ]))) _logger.debug("Checking required tools: DONE") return missing_tools
def unload_workflows(self): """ Unload all workflows loaded by this :class:`WorkflowLoader` instance. """ if not self._galaxy_instance: raise RuntimeError("WorkflowLoader not initialized") for _, wf in _iteritems(self._workflows): self.unload_workflow(wf.id)
def save_dict_to_json(filename,d): """ Saves a (flat) dictionary that can also contain numpy arrays to a json file. """ with open(filename,'w') as fp: dat = [(p,_var_to_json_safe(param)) for (p,param) in _iteritems(d)] json.dump(dict(dat), fp)
def dict_recursive_update(d, u): for k, v in _iteritems(u): if isinstance(v, collections.Mapping): r = dict_recursive_update(d.get(k, {}), v) d[k] = r else: d[k] = u[k] return d
def show_outputs(self, stream=_sys.stdout): """ Print workflow outputs (indexed by workflow step) to file. """ for step_id, step_outputs in _iteritems(self.outputs): print("'{0}': {1}".format( step_id, ", ".join([x["label"] for x in step_outputs.values()])), file=stream)
def load_dict_from_json(filename): """ Loads a (flat) dictionary from a json file and converts lists back into numpy arrays. """ with open(filename,'r') as fp: dat = json.load(fp) assert(type(dat) == dict) dat = dict([(p,_json_safe_to_value(param)) for (p,param) in _iteritems(dat)]) return dat
def set_expected_outputs(self, expected_outputs): """ Add a new set of expected outputs (see :class:`WorkflowTestCase`). :type expected_outputs: dict :param expected_outputs: a dictionary structured as specified in :class:`WorkflowTestCase` """ for name, config in _iteritems(expected_outputs): self.add_expected_output(name, config["file"], config.get("comparator"))
def set_inputs(self, inputs): """ Update the mapping between workflow inputs and test datasets. :param inputs: dict :return: a dictionary of mappings (see :class:`WorkflowTestCase`) """ for name, config in _iteritems(inputs): self.add_input(name, config["file"], config["type"] if "type" in config else None)
def __init__(self, **kwargs): for k, v in _iteritems(kwargs): if not k.startswith('_'): self.__dict__[save_name(k)] = self.__make_Ox(v) else: self.__dict__[k] = v #super(Ox,self).__init__(**kwargs) if not hasattr(self, '_item'): self.__dict__['_item'] = kwargs if not hasattr(self, '_original_type'): self.__dict__['_original_type'] = 'dict'
def _parse_dict(elements): results = {} for name, value in _iteritems(elements): result = value if isinstance(value, _basestring): result = {"name": name, "file": value} elif isinstance(value, dict): result["name"] = name else: raise ValueError("Configuration error: %r", elements) results[name] = result return results
def cleanup_output_folder(self, test_result=None): """ Perform a clean up of the temporary files produced during the workflow test execution. """ test_results = self._test_cases.values() if not test_result else [ test_result ] for _test in test_results: for output_name, output_map in _iteritems(_test.output_file_map): _logger.debug("Cleaning output folder: %s", output_name) if _os.path.exists(output_map["filename"]): _os.remove(output_map["filename"]) _logger.debug("Deleted output file '%s'.", output_map["filename"])
def pngsave(A, file, info={}): """ wrapper around PIL 1.1.7 Image.save to preserve PNG metadata based on public domain script by Nick Galbreath http://blog.modp.com/2007/08/python-pil-and-png-metadata-take-2.html """ from PIL import Image, PngImagePlugin im = Image.fromarray(256.0 * A).convert('RGB') reserved = ('interlace', 'gamma', 'dpi', 'transparency', 'aspect') meta = PngImagePlugin.PngInfo() for k, v in _iteritems(info): if k in reserved: continue meta.add_text(k, v, 0) im.save(file, "PNG", pnginfo=meta)
def to_dict(self): """ Return a dictionary representation of the current class instance. :rtype: dict :return: """ return dict({ "name": self.name, "file": self.filename, "inputs": { name: input_["file"][0] for name, input_ in _iteritems(self.inputs) }, "params": self.params, "expected": self.expected_outputs })
def _load_configuration(config_filename): with open(config_filename) as config_file: workflows_conf = None try: workflows_conf = _yaml_load(config_file) except ValueError as e: _logger.error( "Configuration file '%s' is not a valid YAML or JSON file", config_filename) raise ValueError( "Not valid format for the configuration file '%s'.", config_filename) # update inputs/expected fields for wf_name, wf in _iteritems(workflows_conf["workflows"]): wf["inputs"] = _parse_dict(wf["inputs"]) wf["expected"] = _parse_dict(wf["expected"]) return workflows_conf
def cleanup(self, output_folder=None): """ Perform a complete clean up of the data produced during the execution of a workflow test, i.e., the uploaded workflow and the created history are removed from Galaxy and the actual output datasets (downloaded from Galaxy) are deleted from the output path of the local file system. """ _logger.debug("Cleanup of workflow test '%s'...", self._uuid) for test_uuid, test_result in _iteritems(self._test_cases): if test_result.output_history: self._galaxy_instance.histories.delete( test_result.output_history.id) self.cleanup_output_folder(test_result) if self._galaxy_workflow: self._workflow_loader.unload_workflow(self._galaxy_workflow.id) self._galaxy_workflow = None _logger.debug("Cleanup of workflow test '%s': DONE", self._uuid) if output_folder and _os.path.exists(output_folder): _shutil.rmtree(output_folder) _logger.debug("Deleted WF output folder '%s': DONE", output_folder)
def __init__(self,filename,z=False, slice_at = None): self.filename = filename self.file = open(filename,'r') self.raw_header = self.file.read(256) self.header = dict([h.split('=') for h in self.raw_header.split('\n') if '=' in h]) self.header = dict([(k,litus._make_an_int_if_possible(v)) for k,v in _iteritems(self.header)]) self.z = z self.last_image = np.zeros((50,50)) self.start_at = 0 self.stop_at = None self.step_at = None if slice_at is not None: self.start_at = slice_at.start self.stop_at = slice_at.stop self.step_at = slice_at.step if self.start_at is None: self.start_at = 0 if self.step_at is None: self.step_at = 1 self.image_i = self.start_at
def load(filename, output_folder=None): if _os.path.exists(filename): # TODO: catch YAML parsing errors file_configuration = _load_configuration(filename) base_path = file_configuration.get( "base_path", _os.path.dirname(_os.path.abspath(filename))) suite = WorkflowTestSuite( galaxy_url=file_configuration.get("galaxy_url"), galaxy_api_key=file_configuration.get("galaxy_api_key"), enable_logger=file_configuration.get("enable_logger", False), enable_debug=file_configuration.get("enable_debug", False), disable_cleanup=file_configuration.get("disable_cleanup", False), disable_assertions=file_configuration.get("disable_assertions", False), output_folder=output_folder \ or file_configuration.get("output_folder") \ or WorkflowTestCase.DEFAULT_OUTPUT_FOLDER, max_retries=file_configuration.get("max_retries", None), retry_delay=file_configuration.get("retry_delay", None), polling_interval=file_configuration.get("polling_interval", None) ) for wf_name, wf_config in _iteritems( file_configuration.get("workflows")): wf_base_path = _os.path.join(base_path, wf_config.get("base_path", "")) wf_config["output_folder"] = _os.path.join( suite.output_folder, wf_config.get("output_folder", wf_name)) # add the workflow w = WorkflowTestCase(name=wf_name, base_path=wf_base_path, workflow_filename=wf_config["file"], inputs=wf_config["inputs"], params=wf_config.get("params", {}), expected_outputs=wf_config["expected"], output_folder=wf_config["output_folder"]) suite.add_workflow_test(w) return suite else: raise ValueError("Filename '{0}' not found".format(filename))
def png_client(images, info={}, port=10000, host='localhost', compress_level=0, resize=(1.0, 1.0)): if len(images.shape) == 2: images = [images] # Create a TCP/IP socket sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # Connect the socket to the port where the server is listening server_address = (host, port) sock.connect(server_address) try: for A in images: im = Image.fromarray(256.0 * A).convert('RGB') if resize != (1.0, 1.0) and resize is not None: if not type(resize) == tuple: resize = (resize, resize) im = im.resize( (int(resize[0] * im.size[0]), int(resize[1] * im.size[1])), PIL.Image.ANTIALIAS) output = StringIO.StringIO() meta = PngImagePlugin.PngInfo() reserved = ('interlace', 'gamma', 'dpi', 'transparency', 'aspect') for k, v in _iteritems(info): if k in reserved: continue meta.add_text(str(k), str(v), 0) im.save(output, format="PNG", pnginfo=meta, compress_level=compress_level) message = output.getvalue() output.close() sock.sendall(message) finally: sock.close()
def describe_html(v, wrap_in_html=True, **kwargs): from IPython.display import HTML import uuid try: import html except: import cgi as html # fallback escape function if isinstance(v, variables.Parameter) or isinstance( v, variables.Variable) or isinstance(v, torch.nn.parameter.Parameter): d = {} for k in [ 'name', '_name', 'simple_name', 'doc', 'config_key', 'optimizable', 'node', 'save', 'init', 'get', 'set', 'variable_type', 'auto_name' ]: if has_convis_attribute(v, k): d[k] = get_convis_attribute(v, k) name = d.get('name', '') # optional: None handling if not type(name) is str or name is '': name = d.get('_name', '') if not type(name) is str or name is '': name = repr(v) if type(v) is torch.nn.parameter.Parameter: name = 'torch.nn.Parameter' if has_convis_attribute(v, 'html_name'): name += ' ' + str(get_convis_attribute(v, 'html_name')) #simple_name = str(d.get('simple_name','')) s = """<div class='convis_description variable'><b """ + on_click_toggle + """>""" + name + """</b> <small>""" + d.get( 'variable_type', '') + """</small>""" # default: show everything, hide on click; s += "<div class='description_content_replacer' style='border-left: 2px solid #eee; padding-left: 5px; margin-bottom: 10px; display: none;'>(…)</div>" s += "<div class='description_content' style='border-left: 2px solid #eee; border-top: 2px solid #f8f8f8; padding-left: 5px; margin-bottom: 10px; margin-top: 2px;'>" if has_convis_attribute(v, 'path'): s += "<small>" + full_path(v) + "</small><br/>" if has_convis_attribute( v, 'doc') and get_convis_attribute(v, 'doc') != '': s += '<p class="doc" style="padding:2px;">' + get_convis_attribute( v, 'doc') + '</p>' if has_convis_attribute(v, 'owner'): s += "<tt style='color: gray;'><small>" + str( v.owner) + "</small></tt><br/>" for k in [ 'auto_name', 'config_key', 'optimizable', 'node', 'save', 'init', 'get', 'set', 'state_out_state', 'param_init', 'state_init', 'state_in_state', 'copied_from', 'config_key', 'config_default' ]: if has_convis_attribute(v, k): if isinstance(get_convis_attribute(v, k), MethodType): s += '<div><b>' + str(k) + '</b>: <tt>method</tt></div>' elif isinstance(get_convis_attribute(v, k), FunctionType): s += '<div><b>' + str(k) + '</b>: <tt>function</tt></div>' else: s += '<div><b>' + str(k) + '</b>: <tt>' + html.escape( str(get_convis_attribute(v, k))) + '</tt></div>' try: if hasattr(v, 'get_value'): s += '<b>value</b>: ' + str( _tensor_to_html(v.get_value(), title=name, **kwargs)) except Exception as e: s += '<b>value</b>: ' + str(e) pass try: s += '<b>got</b>: ' + _tensor_to_html(get_convis_attribute( v, 'get')(variables.create_context_O(v)), title=name, **kwargs) except: pass vv = v if hasattr(v, 'detach'): vv = v.detach() if hasattr(vv, '__array__'): s += '<b>value </b>: ' + _tensor_to_html(vv.__array__()) elif hasattr(vv, 'data'): s += '<b>value </b>: ' + _tensor_to_html(vv.data.__array__()) s += """</div>""" s += """</div>""" if not wrap_in_html: return s return HTML(s) ## # Handeling other datatypes # if type(v) == int or type(v) == float: s = str(v) if not wrap_in_html: return s return HTML(s) elif hasattr(v, '__array__'): if hasattr(v, 'detach'): s = _tensor_to_html(v.detach().__array__(), **kwargs) else: s = _tensor_to_html(v.__array__(), **kwargs) if not wrap_in_html: return s return HTML(s) if isinstance(v, torch.nn.Module): return describe_layer_with_html(v, 4, wrap_in_html) if isinstance(v, ModuleType): uid = uuid.uuid4().hex s = """<div class='convis_description module'><b """ + on_click_toggle + """>""" + getattr( v, '__name__', '(nameless module)') + """</b>""" s += "<div class='description_content_replacer' style='border-left: 2px solid #eee; padding-left: 5px; margin-bottom: 10px; display: none;'>(…)</div>" s += "<div class='description_content' style='border-left: 2px solid #eee; border-top: 2px solid #f8f8f8; padding-left: 5px; margin-bottom: 10px; margin-top: 2px;'>" s += '<pre>' + str(getattr(v, '__doc__', "(no doc string found)")) + '</pre>' for f in dir(v): if f.startswith('_'): continue vv = getattr(v, f) if isinstance(vv, ModuleType): s += "<div class='convis_description dict_item'><b>" + str( f) + "</b> (module " + str(getattr(vv, '__name__', '')) + ")</div>" continue s += "<div class='convis_description dict_item'><b id=" + uid + save_name( f ) + " " + on_click_toggle + " >" + str( f ) + "</b> <a style=\"text-decoration: none;\" href='#" + uid + "''>↩</a>" s += "<div class='description_content_replacer' style='border-left: 0px solid #ddd; padding-left: 5px; display: none;'>(…)</div>" s += "<div class='description_content' style='border-left: 0px solid #ddd; padding-left: 5px;'>" s += describe_html(vv, wrap_in_html=False, **kwargs) s += "</div>" s += "</div>" s += """</div>""" s += """</div>""" if not wrap_in_html: return s return HTML(s) if isinstance(v, FunctionType): s = """<div class='convis_description module'><b """ + on_click_toggle + """>""" + getattr( v, '__name__', '(nameless function)') + """</b>""" s += "<div class='description_content_replacer' style='border-left: 2px solid #eee; padding-left: 5px; margin-bottom: 10px; display: none;'>(…)</div>" s += "<div class='description_content' style='border-left: 2px solid #eee; border-top: 2px solid #f8f8f8; padding-left: 5px; margin-bottom: 10px; margin-top: 2px;'>" s += '<pre>' + str(getattr(v, '__doc__', "(no doc string found)")) + '</pre>' s += """</div>""" s += """</div>""" if not wrap_in_html: return s return HTML(s) if inspect.isclass(v): s = """<div class='convis_description module'><b """ + on_click_toggle + """>""" + getattr( v, '__name__', '(nameless class)') + """</b>""" s += "<div class='description_content_replacer' style='border-left: 2px solid #eee; padding-left: 5px; margin-bottom: 10px; display: none;'>(…)</div>" s += "<div class='description_content' style='border-left: 2px solid #eee; border-top: 2px solid #f8f8f8; padding-left: 5px; margin-bottom: 10px; margin-top: 2px;'>" s += '<pre>' + str(getattr(v, '__doc__', "(no doc string found)")) + '</pre>' s += """</div>""" s += """</div>""" if not wrap_in_html: return s return HTML(s) if type(v) in [dict] or hasattr(v, '__iteritems__'): uid = uuid.uuid4().hex s = "<div class='convis_description list'>" iteration = list(v.__iteritems__() if hasattr(v, '__iteritems__' ) else _iteritems(v)) s += "<b id=" + uid + " " + on_click_toggle + " >+</b> " for (k, vv) in iteration: s += '| <a style="text-decoration: none; font-size: 8pt;" href="#' + uid + save_name( k) + '">' + str(k) + '</a> ' s += "<div class='description_content_replacer' style='border-left: 4px solid #f0f0f0; border-top: 4px solid #f8f8f8; padding-left: 10px; margin-bottom: 10px; display: none;'>(…)</div>" s += "<div class='description_content' style='border-left: 4px solid #f0f0f0; border-top: 4px solid #f8f8f8; padding-left: 10px; margin-bottom: 10px;'>" iteration = list(v.__iteritems__() if hasattr(v, '__iteritems__' ) else _iteritems(v)) path = kwargs.pop('path', '') for (k, vv) in iteration: s += "<div class='convis_description dict_item'><small>" + path + ".</small><b id=" + uid + save_name( k ) + " " + on_click_toggle + " >" + str( k ) + "</b> <a style=\"text-decoration: none;\" href='#" + uid + "''>↩</a>" s += "<div class='description_content_replacer' style='border-left: 0px solid #ddd; padding-left: 5px; display: none;'>(…)</div>" s += "<div class='description_content' style='border-left: 0px solid #ddd; padding-left: 5px;'>" s += describe_html(vv, wrap_in_html=False, path=path + '.' + k, **kwargs) s += "</div>" s += "</div>" s += "</div>" s += "</div>" if not wrap_in_html: return s return HTML(s) if type(v) in [list, tuple] or hasattr(v, '__iter__'): try: s = "<div class='convis_description list'><b " + on_click_toggle + ">List (" + str( len(v)) + "):</b>" s += "<div class='description_content_replacer' style='border-left: 4px solid #f0f0f0; border-top: 4px solid #f8f8f8; padding-left: 10px; margin-bottom: 10px; display: none;'>(…)</div>" s += "<div class='description_content' style='border-left: 4px solid #f0f0f0; border-top: 4px solid #f8f8f8; padding-left: 10px; margin-bottom: 10px;'>" s += '\n'.join( [describe_html(vv, wrap_in_html=False, **kwargs) for vv in v]) s += "</div>" s += "</div>" if not wrap_in_html: return s return HTML(s) except: # Tensor Variables love to raise TypeErrors when iterated over pass ## # Assuming its a annotated variable: # d = {} for k in [ 'name', 'simple_name', 'doc', 'config_key', 'optimizable', 'node', 'save', 'init', 'get', 'set', 'variable_type', 'auto_name' ]: if has_convis_attribute(v, k): d[k] = get_convis_attribute(v, k) name = d.get('name', '') # optional: None handling if not type(name) is str or name is '': name = repr(v) if has_convis_attribute(v, 'html_name'): name += ' ' + str(get_convis_attribute(v, 'html_name')) #simple_name = str(d.get('simple_name','')) s = """<div class='convis_description variable'><b """ + on_click_toggle + """>""" + name + """</b> <small>""" + d.get( 'variable_type', '') + """</small>""" # default: show everything, hide on click; s += "<div class='description_content_replacer' style='border-left: 2px solid #eee; padding-left: 5px; margin-bottom: 10px; display: none;'>(…)</div>" s += "<div class='description_content' style='border-left: 2px solid #eee; border-top: 2px solid #f8f8f8; padding-left: 5px; margin-bottom: 10px; margin-top: 2px;'>" if has_convis_attribute(v, 'path'): s += "<small>" + full_path(v) + "</small><br/>" if has_convis_attribute(v, 'doc') and get_convis_attribute(v, 'doc') != '': s += '<p class="doc" style="padding:2px;">' + get_convis_attribute( v, 'doc') + '</p>' if has_convis_attribute(v, 'owner'): s += "<tt style='color: gray;'><small>" + str( v.owner) + "</small></tt><br/>" for k in [ 'auto_name', 'config_key', 'optimizable', 'node', 'save', 'init', 'get', 'set', 'state_out_state', 'param_init', 'state_init', 'state_in_state', 'copied_from', 'config_key', 'config_default' ]: if has_convis_attribute(v, k): if isinstance(get_convis_attribute(v, k), MethodType): s += '<div><b>' + str(k) + '</b>: <tt>method</tt></div>' elif isinstance(get_convis_attribute(v, k), FunctionType): s += '<div><b>' + str(k) + '</b>: <tt>function</tt></div>' else: s += '<div><b>' + str(k) + '</b>: <tt>' + html.escape( str(get_convis_attribute(v, k))) + '</tt></div>' try: if hasattr(v, 'get_value'): s += '<b>value</b>: ' + str( _tensor_to_html(v.get_value(), title=name, **kwargs)) except Exception as e: s += '<b>value</b>: ' + str(e) pass try: s += '<b>got</b>: ' + _tensor_to_html(get_convis_attribute(v, 'get')( variables.create_context_O(v)), title=name, **kwargs) except: pass s += """</div>""" s += """</div>""" if not wrap_in_html: return s return HTML(s)
def register_users(galaxy_instance, users_list, create_api_key=False, report_filename=None): if report_filename: if _os.path.exists(report_filename): raise ValueError( "Output file {} already exists. Won't overwrite".format( report_filename)) ofile = open(report_filename, 'w') else: ofile = _sys.stdout try: actual_users = { u["email"]: u for u in galaxy_instance.users.get_users() } logger.debug("Number of users to register: %d", len(users_list)) galaxy_users = [] # registered_galaxy_users.values() for username, u_data in _iteritems(users_list): try: if not u_data["email"] in actual_users: logger.debug("Registering user: %s - %s", u_data["username"], u_data["email"]) user = galaxy_instance.users.create_local_user( u_data["username"], u_data["email"], u_data["password"]) u_data["galaxy_id"] = user["id"] if create_api_key: api_key = galaxy_instance.users.create_user_apikey( user['id']) logger.debug("Create API_KEY %s for user %s", api_key, username) galaxy_users.append({ "galaxy_id": user["id"], "uid": u_data["uid"], "username": u_data["username"], "email": u_data["email"], "api_key": api_key, "password": u_data["password"], "hashed_password": u_data["hashed_password"] }) logger.info( "User registered with ID '%s' and API_KEY '%s'", u_data["id"], api_key) else: galaxy_users.append({ "galaxy_id": user["id"], "uid": u_data["uid"], "username": u_data["username"], "email": u_data["email"], "password": u_data["password"], "hashed_password": u_data["hashed_password"] }) logger.info("User registered with ID: %s", user["id"]) except Exception as e: logger.debug(e) logger.info("Registered users: %d", len(galaxy_users)) logger.debug("Writing report to '%s' %d users", report_filename, len(galaxy_users)) writer = _csv.DictWriter(ofile, fieldnames=[ "uid", "galaxy_id", "username", "email", "password", "hashed_password", "api_key" ]) writer.writeheader() writer.writerows(galaxy_users) finally: if report_filename: ofile.close()
def extract_workflow(self, filename=None, workflow_name=None, v_step=100, h_step=400): if workflow_name is None: workflow_name = "Workflow extracted from history {0}".format( self._history.id) # start self._logger.info("Extracting Workflow from history...") # wf object representation wf = _collections.OrderedDict({ "a_galaxy_workflow": "true", "annotation": "", "format-version": "0.1", "name": workflow_name, "uuid": str(_uuid.uuid1()), "steps": _collections.OrderedDict() }) # position p_left = 50 p_top = 0 # process steps inputs = { k: v for k, v in _iteritems(self._input_order_map) if k in self.input_datasets } for hds_id, index in sorted(_iteritems(inputs), key=_operator.itemgetter(1)): input_name = self.input_dataset_labels[hds_id] input_description = "" p_top += v_step wf["steps"][str(index)] = { "annotation": "", "content_id": None, "id": index, "input_connections": {}, "inputs": [{ "description": input_description, "name": input_name }], "label": None, "name": "Input dataset", "outputs": [], "position": { "left": p_left, "top": p_top }, "tool_errors": None, "tool_id": None, "tool_state": _json.dumps({"name": input_name}), "tool_version": None, "type": "data_input", "uuid": str(_uuid.uuid1()), "workflow_outputs": [] } # reset top position p_top = p_top - (v_step * (len(self.input_datasets) / 2)) # process intermediate and final steps for job_id, job in _iteritems(self.processing_jobs): # update top position p_left += h_step # compute the step index index = len(wf["steps"]) # get the tool related to the current job tool = self._get_tool(job.wrapped["tool_id"]) # compute params params = {"__page__": 0, "__rerun_remap_job_id__": None} tool_inputs = { param["name"]: param for param in tool.wrapped["inputs"] } for param_name, param_info in _iteritems(job.wrapped["params"]): if param_name in tool_inputs: params[param_name] = param_info # add inputs to tool state and inputs inputs = [] input_connections = {} for job_input_name, job_input_info in _iteritems( job.wrapped["inputs"]): params[job_input_name] = _json.dumps( {"__class__": "RuntimeValue"}) inputs.append({ "description": "Runtime input value {0}".format(job_input_name), "name": job_input_name }) output_name = self.intermediate_dataset_labels[job_input_info["id"]] \ if job_input_info["id"] in self.intermediate_dataset_labels else "output" input_connections[job_input_name] = { "id": self._input_order_map[job_input_info["id"]], "output_name": output_name } # add outputs outputs = [] workflow_outputs = [] tool_outputs = { param["name"]: param for param in tool.wrapped["outputs"] } for job_output_name, job_output_info in _iteritems( job.wrapped["outputs"]): outputs.append({ "name": job_output_name, "type": tool_outputs[job_output_name]["format"] }) # TODO: check if exists cryteria for detecting if an output # is a workflow output or a simple intermediate output workflow_outputs.append({ "label": job_output_name, "output_name": job_output_name, "uuid": str(_uuid.uuid1()) }) wf["steps"][str(index)] = { "annotation": "", "content_id": tool.id, "id": index, "input_connections": input_connections, "inputs": inputs, "label": None, "name": tool.name, "outputs": outputs, "position": { "left": p_left, "top": p_top }, "tool_errors": None, "tool_id": tool.id, "tool_state": _json.dumps(params), "tool_version": tool.version, "type": "tool", "uuid": str(_uuid.uuid1()), "workflow_outputs": workflow_outputs } # save workflow if filename is not None: self._logger.info("Saving workflow to file...") with open(filename, "w") as fp: self._logger.debug("Workflow file path: %s", filename) _json.dump(wf, fp, indent=4) self._logger.info("Saving workflow to file: done") # extraction wf end self._logger.info("Extracting Workflow from history: done") return wf
def _process_history(self): # check if a history has been assigned if self._history is None: raise RuntimeError("No history found!") # auxiliary infos ds_input_info = {} ds_output_info = {} intermediate_datasets = [] # get history datasets history = self._history self.datasets = history.get_datasets() # process jobs chain (through their created datasets) for ds in self.datasets: self._logger.info("Processing dataset %s ... ", ds.id) # load job info creating_job = self._get_job(ds.wrapped["creating_job"]) job_inputs = { in_info["id"]: in_name for in_name, in_info in _iteritems( creating_job.wrapped["inputs"]) } job_outputs = { out_info["id"]: out_name for out_name, out_info in _iteritems( creating_job.wrapped["outputs"]) } intermediate_datasets += list(job_inputs) # update auxiliary data info for in_id, in_name in _iteritems(job_inputs): if in_id not in ds_input_info: ds_input_info[in_id] = {} ds_input_info[in_id][creating_job.id] = in_name for out_id, out_name in _iteritems(job_outputs): if out_id not in ds_output_info: ds_output_info[out_id] = {} ds_output_info[out_id][creating_job.id] = out_name # register the job as the creating of this DS self.creating_jobs[ds.id] = creating_job # detect if the job creates an input DS # or it is a processing job if len(job_inputs) == 0: self.input_datasets[ds.id] = ds else: # add the processing job self.processing_jobs[creating_job.id] = creating_job # compute the processing job level self.processing_jobs[creating_job.id] # update in/out maps if creating_job.id not in self.job_input_ids: self.job_input_ids[creating_job.id] = list(job_inputs) self.job_output_ids[creating_job.id] = list(job_outputs) self._logger.info("Process dataset %s: done ", ds.id) self._logger.info("Processing extra info...") # Auxiliary function which computes the label for a given dataset def __set_label(labels, ds_id, info_matrix, label=None, prefix=None): if label is not None: labels[ds_id] = label elif ds_id in info_matrix and len(info_matrix[ds_id]) == 1: # use job labels[ds_id] = info_matrix[ds_id][list(info_matrix[ds_id])[0]] else: # use a default label if the same dataset if used by more than one job labels[ds_id] = "{0}_{1}".format(prefix, len(labels)) # process datasets to: # - determine intermediate and output datasets # - determine input/output labels for ds in self.datasets: if ds.id in self.input_datasets: __set_label(self.input_dataset_labels, ds.id, ds_input_info, prefix="input") __set_label(self.intermediate_dataset_labels, ds.id, ds_input_info, label="output") else: if ds.id in intermediate_datasets: self.intermediate_datasets[ds.id] = ds __set_label(self.input_dataset_labels, ds.id, ds_input_info, prefix="input") __set_label(self.intermediate_dataset_labels, ds.id, ds_output_info, prefix="output") else: self.output_datasets[ds.id] = ds __set_label(self.output_dataset_labels, ds.id, ds_output_info, prefix="output") intermediate_inputs = [] not_ordered_inputs = list(self.input_datasets) input_datasets = _collections.OrderedDict() inputs = list(self.input_datasets) self._input_order_map = {x: inputs.index(x) for x in inputs} # determine the job level self._logger.debug("Processing JOB levels ...") for job_id, job in _iteritems(self.processing_jobs): # compute and set the job level self.processing_job_levels[ job_id] = self.compute_processing_job_level(job_id) # order inputs tool = self._get_tool(job.wrapped["tool_id"]) ordered_names = [x["name"] for x in tool.wrapped["inputs"]] for name in ordered_names: if name in job.wrapped["inputs"]: in_id = job.wrapped["inputs"][name]["id"] if in_id in not_ordered_inputs: input_datasets[in_id] = self.input_datasets[in_id] if in_id not in self.input_datasets: self._input_order_map[in_id] = len( self.input_datasets ) + self.processing_job_levels[job_id] not_ordered_inputs.remove(in_id) # add intermediate inputs for x in job.wrapped["outputs"].values(): if x["id"] not in self.output_datasets: intermediate_inputs.append(x["id"]) if x["id"] not in self.input_datasets: self._input_order_map[x["id"]] = len( self.input_datasets ) + self.processing_job_levels[job_id] self._logger.debug("JOB levels processing: done") # copy remaining inputs for ds_in in not_ordered_inputs: input_datasets[ds_in] = self.input_datasets[ds_in] self.input_datasets = input_datasets self._logger.info("Processing extra info: done")
def _get_workflow_info(filename, galaxy_url, galaxy_api_key, tool_folder=DEFAULT_TOOLS_FOLDER): inputs = [] params = _CommentedMap() outputs = {} # loading wf info start _logger.debug("Loading workflow definition from %s file...", filename) # setup galaxy instance galaxy_instance = _common.get_galaxy_instance(galaxy_url, galaxy_api_key) galaxy_tool_client = _ToolClient( galaxy_instance.gi) # get the non-object version of the GI if not _os.path.exists(DEFAULT_TOOLS_FOLDER): _os.makedirs(DEFAULT_TOOLS_FOLDER) with open(filename) as fp: wf_config = _json.load(fp) for sid, step in _iteritems(wf_config["steps"]): # tool = gi.tools.get() _logger.debug("Processing step '%s' -- '%s'", sid, step["name"]) # an input step.... if not step["tool_id"] and step["type"] == "data_input": for input_ in step["inputs"]: _logger.debug("Processing input: '%s' (%s)", input_["name"], input_["description"]) inputs.append(input_) # a processing step (with outputs) ... if step["tool_id"] and step["type"] == "tool": # tool parameters tool_params = _CommentedMap() # process tool info to extract parameters tool_id = step["tool_id"] # tool = galaxy_instance.tools.get(tool_id) ## LP: re-write this using the bioblend.objects API to fetch the tool # inputs. See the comment above `def _process_tool_param_element` # tool_config_xml = _os.path.basename(tool.wrapped["config_file"]) # _logger.debug("Processing step tool '%s'", tool_id) # # try: # _logger.debug("Download TOOL '%s' definition file XML: %s....", tool_id, tool_config_xml) # targz_filename = _os.path.join(DEFAULT_TOOLS_FOLDER, tool_id + ".tar.gz") # targz_content = galaxy_tool_client._get(_os.path.join(tool_id, "download"), json=False) # if targz_content.status_code == 200: # with open(targz_filename, "w") as tfp: # tfp.write(targz_content.content) # tar = _tarfile.open(targz_filename) # tar.extractall(path=tool_folder) # tar.close() # _os.remove(targz_filename) # _logger.debug("Download TOOL '%s' definition file XML: %s....: DONE", tool_id, tool_config_xml) # else: # _logger.debug("Download TOOL '%s' definition file XML: %s....: ERROR %r", # tool_id, tool_config_xml, targz_content.status_code) # # tool_config_xml = _os.path.join(DEFAULT_TOOLS_FOLDER, tool_config_xml) # if _os.path.exists(tool_config_xml): # tree = _etree.parse(tool_config_xml) # root = tree.getroot() # inputs_el = root.find("inputs") # for input_el in inputs_el: # _process_tool_param_element(input_el, tool_params) # if len(tool_params) > 0: # params.insert(int(sid), sid, tool_params) # # except _StandardError as e: # _logger.debug("Download TOOL '%s' definition file XML: %s....: ERROR", tool_id, tool_config_xml) # _logger.error(e) # process outputs[str(sid)] = {} for output in step["workflow_outputs"]: outputs[str(sid)][output["uuid"]] = output # loading wf info end _logger.debug("Workflow definition loaded from %s file...", filename) # return loaded info return wf_config, inputs, params, outputs
def __init__(self, **enums): for name, val in _iteritems(enums): self.__dict__[name] = val
def run_test(self, base_path=None, inputs=None, params=None, expected_outputs=None, output_folder=None, disable_assertions=None, disable_cleanup=None, enable_logger=None, enable_debug=None): """ Run the workflow test which this runner is associated to. The parameters ``base_path``, ``inputs``, ``outputs``, ``expected_outputs`` ``output_folder``, ``disable_assertions``, ``disable_cleanup``, ``enable_logger``, ``enable_debug`` can be provided to override the corresponding defined in the :class:`WorkflowTestCase` instance which this runner is related to (see :class:`WorkflowTestCase` for more details). :rtype: :class:`WorkflowTestResult` :return: the :class:`WorkflowTestResult` instance which represents the test result """ # update test settings if enable_logger is None \ and self._workflow_test_config is not None \ and hasattr(self._workflow_test_config, "enable_logger"): enable_logger = self._workflow_test_config.enable_logger if enable_debug is None \ and self._workflow_test_config is not None \ and hasattr(self._workflow_test_config, "enable_debug"): enable_debug = self._workflow_test_config.enable_debug if disable_cleanup is None: disable_cleanup = self._workflow_test_config.disable_cleanup if disable_assertions is None: disable_assertions = self._workflow_test_config.disable_assertions # set basepath base_path = self._base_path if not base_path else base_path # load workflow workflow = self.get_galaxy_workflow() # output folder if output_folder is None: output_folder = self._workflow_test_config.output_folder # update logger if enable_logger or enable_debug: _common.LoggerManager.update_log_level( _logging.DEBUG if enable_debug else _logging.INFO) if disable_cleanup: self._file_handler = _common.LoggerManager.enable_log_to_file( output_folder=output_folder, log_filename="-".join([ "WorkflowTestCase", self.worflow_test_name, self.uuid ]) + ".log") else: _common.LoggerManager.update_log_level(_logging.ERROR) _empty_logger.info("") _logger.info("Running workflow testcase: %r", self._workflow_test_config.name) _logger.debug("TestCase configuration: %r", self._workflow_test_config.__dict__) # check input_map if inputs is None: if len(self._workflow_test_config.inputs) > 0: inputs = self._workflow_test_config.inputs else: raise ValueError("No input configured !!!") # check params if params is None: params = self._workflow_test_config.params _logger.debug("Using default params") # check expected_output_map if expected_outputs is None: if len(self._workflow_test_config.expected_outputs) > 0: expected_outputs = self._workflow_test_config.expected_outputs else: raise ValueError("No output configured !!!") # update config options disable_cleanup = disable_cleanup if disable_cleanup is not None else self._disable_cleanup disable_assertions = disable_assertions if disable_assertions is not None else self._disable_assertions output_folder = output_folder if output_folder is not None else self._output_folder # uuid of the current test test_uuid = self.uuid # store the current message error_msg = None # test restul test_result = None # check tools errors = [] missing_tools = self.find_missing_tools() if len(missing_tools) == 0: try: # create a new history for the current test history = self._galaxy_instance.histories.create("-".join([ _core.WorkflowTestCase.DEFAULT_HISTORY_NAME_PREFIX, self._workflow_test_config.name.replace(" ", ""), test_uuid ])) _logger.info("Create a history '%s' (id: %r)", history.name, history.id) # upload input data to the current history # and generate the datamap INPUT --> DATASET datamap = {} for label, config in _iteritems(inputs): datamap[label] = [] for filename in config["file"]: dataset_filename = filename if _os.path.isabs( filename) else _os.path.join(base_path, filename) if config["type"]: datamap[label].append( history.upload_dataset( dataset_filename, file_type=config["type"])) else: datamap[label].append( history.upload_dataset(dataset_filename)) # run the workflow _logger.debug("About to launch workflow.") _logger.debug("history: %r", history) _logger.debug("datamap: %r", datamap) _logger.debug("params: %r", params) _logger.info("Workflow '%s' (id: %s) running ...", workflow.name, workflow.id) outputs, output_history = workflow.run( datamap, history, params=params, wait=True, polling_interval=self._galaxy_instance.polling_interval) _logger.info("Workflow '%s' (id: %s) executed", workflow.name, workflow.id) # check outputs results, output_file_map = self._check_outputs( base_path, outputs, expected_outputs, output_folder) # instantiate the result object test_result = _core.WorkflowTestResult( test_uuid, workflow, inputs, outputs, output_history, expected_outputs, missing_tools, results, output_file_map, output_folder) if test_result.failed(): error_msg = "The actual output{0} {2} differ{1} from the expected one{0}." \ .format("" if len(test_result.failed_outputs) == 1 else "s", "" if len(test_result.failed_outputs) > 1 else "s", ", ".join(["'{0}'".format(n) for n in test_result.failed_outputs])) except RuntimeError as e: error_msg = "Runtime error: {0}".format(e.message) errors.append(error_msg) _logger.debug(error_msg) else: error_msg = "Some workflow tools are not available in Galaxy: {0}".format( ", ".join([ "{0} (ver. {1})".format(t[0], t[1]) for t in missing_tools ])) errors.append(error_msg) _logger.debug(error_msg) # instantiate the result object if not test_result: test_result = _core.WorkflowTestResult(test_uuid, workflow, inputs, [], None, expected_outputs, missing_tools, {}, {}, output_folder, errors) # store result self._test_cases[test_uuid] = test_result if self._test_suite_runner: self._test_suite_runner._add_test_result(test_result) # FIXME self.test_result = test_result # cleanup if not disable_cleanup: self.cleanup(output_folder) # disable file logger if self._file_handler is not None: _common.LoggerManager.remove_file_handler(self._file_handler, not disable_cleanup) self._file_handler = None # raise error message if error_msg: if not disable_assertions: raise AssertionError(error_msg) return test_result
def delete_users(galaxy_instance, users): logger.debug("Number of users to delete: %d", len(users)) for _, user in _iteritems(users): galaxy_instance.users.delete_user(user["galaxy_id"]) logger.info("Deleted user %s: %s", user["galaxy_id"], user["email"])
def __str__(self): return "Test {0}: workflow {1}, intputs=[{2}], outputs=[{3}]" \ .format(self.test_id, self.workflow.name, ",".join([i for i in self.inputs]), ", ".join(["{0}: {1}".format(x[0], "OK" if x[1] else "ERROR") for x in _iteritems(self.results)]))