def __init__(self, arg=None): initialize() if arg is None: # Copied from VistrailsApplicationInterface#open_vistrail() locator = UntitledLocator() loaded_objs = vistrails.core.db.io.load_vistrail(locator) self.controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:]) elif isinstance(arg, (_Pipeline, Pipeline)): if isinstance(arg, Pipeline): pipeline = arg.pipeline else: pipeline = arg # Copied from VistrailsApplicationInterface#open_workflow() vistrail = _Vistrail() ops = [] for module in pipeline.module_list: ops.append(('add', module)) for connection in pipeline.connection_list: ops.append(('add', connection)) action = vistrails.core.db.action.create_action(ops) vistrail.add_action(action, 0L) vistrail.update_id_scope() vistrail.change_description("Imported pipeline", 0L) self.controller = VistrailController(vistrail, UntitledLocator()) elif isinstance(arg, VistrailController): self.controller = arg elif isinstance(arg, basestring): raise TypeError("Vistrail was constructed from %r.\n" "Use load_vistrail() to get a Vistrail from a " "file." % type(arg).__name__) else: raise TypeError("Vistrail was constructed from unexpected " "argument type %r" % type(arg).__name__)
def test_vtl_files(self): from vistrails.tests.utils import run_file for root, dirs, file_names in os.walk(self.vtl_path): for file_name in sorted(file_names): if file_name.endswith('.vtl'): # update available packages from vistrails.core.packagemanager import get_package_manager get_package_manager().build_available_package_names_list() f = os.path.join(root, file_name) locator = FileLocator(f) version = locator._vnode # if there is a version specified try to execute it, # else just load the pipeline if version: errors = run_file(f, lambda x: x == version) self.assertEqual( errors, [], 'Errors processing %s: %s' % (f, str(errors))) else: import vistrails.core.db.io from vistrails.core.vistrail.controller import \ VistrailController loaded_objs = vistrails.core.db.io.load_vistrail( locator) controller = VistrailController( loaded_objs[0], locator, *loaded_objs[1:]) controller.change_selected_version( controller.vistrail.get_latest_version()) self.assertTrue(controller.current_pipeline.is_valid, "Latest pipeline is invalid: %s" % f)
def add_vistrail(self, vistrail, locator, abstraction_files=None, thumbnail_files=None, mashups=None): objs = vistrail, locator, abstraction_files, thumbnail_files, mashups controller = VistrailController(*objs) self._controllers[locator] = controller self._cur_controller = controller return self._cur_controller
def pipeline_from_generator(variable_gen): # Get the original OutputPort module orig_controller = variable_gen._generator.controller base_pipeline = orig_controller.vistrail.getPipeline('dat-vars') if len(base_pipeline.module_list) != 1: raise ValueError("dat-vars version is invalid") output_port = base_pipeline.module_list[0] controller = VistrailController(Vistrail()) # OutputPort operations = [('add', output_port)] # Rest of the pipeline operations += variable_gen._generator.operations # Connection connection = controller.create_connection( variable_gen._output_module, variable_gen._outputport_name, output_port, 'InternalPipe') operations.append(('add', connection)) # Materialize this action = create_action(operations) controller.add_new_action(action) version = controller.perform_action(action) controller.change_selected_version(version) assert version == controller.current_version == 1 return controller.current_pipeline, 1
def make_pipeline(self): """Creates an example pipeline that is used to conduct tests. """ vistrail = Vistrail() controller = VistrailController(vistrail) controller.change_selected_version(0) # 0 1 2 7 8 # | / \ \ / # 3 4 5 9 # | / \ # 6 10 11 modules = [ controller.add_module('org.vistrails.vistrails.basic', 'String') for i in xrange(12) ] def connect(outmod, inmod): controller.add_connection(modules[outmod].id, 'value', modules[inmod].id, 'value') for (outmod, inmod) in [(0, 3), (1, 4), (1, 5), (5, 6), (7, 9), (8, 9), (9, 10), (9, 11)]: connect(outmod, inmod) return controller, modules
def test_find_modules_by_type(self): """Tests the find_modules_by_type() function. """ vistrail = Vistrail() controller = VistrailController(vistrail) controller.change_selected_version(0) mod1 = controller.add_module('org.vistrails.vistrails.basic', 'String') mod2 = controller.add_module('org.vistrails.vistrails.basic', 'Float') mod3 = controller.add_module('org.vistrails.vistrails.basic', 'String') mod4 = controller.add_module('org.vistrails.vistrails.basic', 'Integer') from dat.vistrails_interface import find_modules_by_type from vistrails.core.modules.basic_modules import Boolean, Float, String self.assertEqual( set(m.id for m in find_modules_by_type(controller.current_pipeline, [String])), set([mod1.id, mod3.id])) self.assertEqual([ m.id for m in find_modules_by_type(controller.current_pipeline, [Float]) ], [mod2.id, mod4.id]) self.assertEqual( find_modules_by_type(controller.current_pipeline, [Boolean]), [])
def load_vistrail(filename, version=None): """Loads a Vistrail from a filename. """ initialize() if not isinstance(filename, basestring): raise TypeError("load_vistrails() expects a filename, got %r" % type(filename).__name__) locator = FileLocator(filename) # Copied from VistrailsApplicationInterface#open_vistrail() loaded_objs = vistrails.core.db.io.load_vistrail(locator) controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:]) return Vistrail(controller)
def test_cache(self): from vistrails.core.modules.basic_modules import StandardOutput old_compute = StandardOutput.compute StandardOutput.compute = lambda s: None try: from vistrails.core.db.locator import XMLFileLocator from vistrails.core.vistrail.controller import VistrailController from vistrails.core.db.io import load_vistrail """Test if basic caching is working.""" locator = XMLFileLocator( vistrails.core.system.vistrails_root_directory() + '/tests/resources/dummy.xml') (v, abstractions, thumbnails, mashups) = load_vistrail(locator) # the controller will take care of upgrades controller = VistrailController(v, locator, abstractions, thumbnails, mashups) p1 = v.getPipeline('int chain') n = v.get_version_number('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p1 = controller.current_pipeline view = DummyView() interpreter = CachedInterpreter.get() result = interpreter.execute( p1, locator=v, current_version=n, view=view, ) # to force fresh params p2 = v.getPipeline('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p2 = controller.current_pipeline result = interpreter.execute( p2, locator=v, current_version=n, view=view, ) self.assertEqual(len(result.modules_added), 1) finally: StandardOutput.compute = old_compute
def execute(self): """ execute() -> None Execute the selected pipeline with the edited aliases """ aliases = {} for r in xrange(self.aliasTable.rowCount()): name = str(self.aliasTable.item(r, 0).text()) value = str(self.aliasTable.item(r, 1).text()) aliases[name] = value versionNumber = self.versions[ self.pipelineList.currentIndex().row()][0] pipeline = self.vistrail.getPipelineVersionNumber(versionNumber) controller = VistrailController(self.vistrail) controller.execute_workflow_list([ (self.vistrail.locator, versionNumber, pipeline, DummyView(), aliases, None) ])
def get_upgraded_pipeline(vistrail, version=None): """This is similar to Vistrail#getPipeline() but performs upgrades. getPipeline() can fail if the original pipeline has a different version. In contrast, this function will update the pipeline first using a controller. """ if version is None: version = vistrail.get_latest_version() elif isinstance(version, (int, long)): pass elif isinstance(version, basestring): version = vistrail.get_tag_str(version).action_id else: raise TypeError controller = VistrailController(vistrail) controller.recompute_terse_graph() # FIXME : this shouldn't be needed... controller.do_version_switch(version) return controller.current_pipeline
def run_file(filename, tag_filter=lambda x: True): """Loads a .vt file and runs all the tagged versions in it. """ import vistrails.core.db.io from vistrails.core.db.locator import FileLocator from vistrails.core.system import vistrails_root_directory from vistrails.core.vistrail.controller import VistrailController filename = os.path.join(vistrails_root_directory(), '..', filename) locator = FileLocator(filename) loaded_objs = vistrails.core.db.io.load_vistrail(locator) controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:]) errors = [] for version, name in controller.vistrail.get_tagMap().iteritems(): if tag_filter(name): controller.change_selected_version(version) (result,), _ = controller.execute_current_workflow() if result.errors: errors.append(("%d: %s" % (version, name), result.errors)) return errors
def execute_wf(wf, output_port): # Save the workflow in a temporary file temp_wf_fd, temp_wf = tempfile.mkstemp() try: f = open(temp_wf, 'w') f.write(wf) f.close() os.close(temp_wf_fd) # Clean the cache interpreter = get_default_interpreter() interpreter.flush() # Load the Pipeline from the temporary file vistrail = Vistrail() locator = XMLFileLocator(temp_wf) workflow = locator.load(Pipeline) # Build a Vistrail from this single Pipeline action_list = [] for module in workflow.module_list: action_list.append(('add', module)) for connection in workflow.connection_list: action_list.append(('add', connection)) action = vistrails.core.db.action.create_action(action_list) vistrail.add_action(action, 0L) vistrail.update_id_scope() tag = 'parallel flow' vistrail.addTag(tag, action.id) # Build a controller and execute controller = VistrailController() controller.set_vistrail(vistrail, None) controller.change_selected_version(vistrail.get_version_number(tag)) execution = controller.execute_current_workflow( custom_aliases=None, custom_params=None, extra_info=None, reason='API Pipeline Execution') # Build a list of errors errors = [] pipeline = vistrail.getPipeline(tag) execution_errors = execution[0][0].errors if execution_errors: for key in execution_errors: module = pipeline.modules[key] msg = '%s: %s' % (module.name, execution_errors[key]) errors.append(msg) # Get the execution log from the controller try: module_log = controller.log.workflow_execs[0].item_execs[0] except IndexError: errors.append("Module log not found") return dict(errors=errors) else: machine = controller.log.workflow_execs[0].machines[ module_log.machine_id] xml_log = serialize(module_log) machine_log = serialize(machine) # Get the output value output = None serializable = None if not execution_errors: executed_module, = execution[0][0].executed executed_module = execution[0][0].objects[executed_module] try: output = executed_module.get_output(output_port) except ModuleError: errors.append("Output port not found: %s" % output_port) return dict(errors=errors) reg = vistrails.core.modules.module_registry.get_module_registry() base_classes = inspect.getmro(type(output)) if Module in base_classes: serializable = reg.get_descriptor(type(output)).sigstring output = output.serialize() # Return the dictionary, that will be sent back to the client return dict(errors=errors, output=output, serializable=serializable, xml_log=xml_log, machine_log=machine_log) finally: os.unlink(temp_wf)
def run_and_get_results(w_list, parameters='', update_vistrail=True, extra_info=None, reason='Console Mode Execution'): """run_and_get_results(w_list: list of (locator, version), parameters: str, output_dir:str, update_vistrail: boolean, extra_info:dict) Run all workflows in w_list, and returns an interpreter result object. version can be a tag name or a version id. """ elements = parameters.split("$&$") aliases = {} params = [] result = [] for locator, workflow in w_list: (v, abstractions, thumbnails, mashups) = load_vistrail(locator) controller = VistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=update_vistrail) if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) for e in elements: pos = e.find("=") if pos != -1: key = e[:pos].strip() value = e[pos + 1:].strip() if controller.current_pipeline.has_alias(key): aliases[key] = value elif 'mashup_id' in extra_info: # new-style mashups can have aliases not existing in pipeline for mashuptrail in mashups: if mashuptrail.vtVersion == version: mashup = mashuptrail.getMashup( extra_info['mashup_id']) c = mashup.getAliasByName(key).component params.append((c.vttype, c.vtid, value)) if not update_vistrail: conf = get_vistrails_configuration() if conf.has('thumbs'): conf.thumbs.autoSave = False jobMonitor = controller.jobMonitor current_workflow = jobMonitor.currentWorkflow() if not current_workflow: for job in jobMonitor.workflows.itervalues(): try: job_version = int(job.version) except ValueError: try: job_version = v.get_version_number(job.version) except KeyError: # this is a PE or mashup continue if version == job_version: current_workflow = job jobMonitor.startWorkflow(job) if not current_workflow: current_workflow = JobWorkflow(version) jobMonitor.startWorkflow(current_workflow) try: (results, _) = \ controller.execute_current_workflow(custom_aliases=aliases, custom_params=params, extra_info=extra_info, reason=reason) finally: jobMonitor.finishWorkflow() new_version = controller.current_version if new_version != version: debug.log("Version '%s' (%s) was upgraded. The actual " "version executed was %s" % (workflow, version, new_version)) run = results[0] run.workflow_info = (locator.name, new_version) run.pipeline = controller.current_pipeline if update_vistrail: controller.write_vistrail(locator) result.append(run) if current_workflow.jobs: if current_workflow.completed(): run.job = "COMPLETED" else: run.job = "RUNNING: %s" % current_workflow.id for job in current_workflow.jobs.itervalues(): if not job.finished: run.job += "\n %s %s %s" % (job.start, job.name, job.description()) print run.job return result
sender = context.socket(zmq.PUSH) sender.connect("tcp://{0}:{1}".format(HOST, SEND)) while True: # Receiving pipeline instance configuration data = receiver.recv() logging.debug('Receiving: ' + data) fields = data.split("|") filename = fields[0] parameter_list = ast.literal_eval(fields[1]) inputs = ast.literal_eval(fields[2]) outputs = ast.literal_eval(fields[3]) locator = FileLocator(filename) loaded_objs = vistrails.core.db.io.load_vistrail(locator) controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:]) controller.do_version_switch(controller.get_latest_version_in_graph()) pipeline = Pipeline(controller) kwargs = {} for i in range(len(parameter_list)): kwargs[inputs[i]] = parameter_list[i] try: #Executing pipeline instance and retieving the result result = pipeline.execute(**kwargs) for output in outputs: parameter_list.append(str(result.output_port(output))) except: traceback.print_exc(file=sys.stdout) parameter_list.append(str(False)) kwargs['result'] = parameter_list[-1]