def pipeline_from_generator(variable_gen): # Get the original OutputPort module orig_controller = variable_gen._generator.controller base_pipeline = orig_controller.vistrail.getPipeline('dat-vars') if len(base_pipeline.module_list) != 1: raise ValueError("dat-vars version is invalid") output_port = base_pipeline.module_list[0] controller = VistrailController(Vistrail()) # OutputPort operations = [('add', output_port)] # Rest of the pipeline operations += variable_gen._generator.operations # Connection connection = controller.create_connection( variable_gen._output_module, variable_gen._outputport_name, output_port, 'InternalPipe') operations.append(('add', connection)) # Materialize this action = create_action(operations) controller.add_new_action(action) version = controller.perform_action(action) controller.change_selected_version(version) assert version == controller.current_version == 1 return controller.current_pipeline, 1
def make_pipeline(self): """Creates an example pipeline that is used to conduct tests. """ vistrail = Vistrail() controller = VistrailController(vistrail) controller.change_selected_version(0) # 0 1 2 7 8 # | / \ \ / # 3 4 5 9 # | / \ # 6 10 11 modules = [controller.add_module('org.vistrails.vistrails.basic', 'String') for i in xrange(12)] def connect(outmod, inmod): controller.add_connection( modules[outmod].id, 'value', modules[inmod].id, 'value') for (outmod, inmod) in [(0, 3), (1, 4), (1, 5), (5, 6), (7, 9), (8, 9), (9, 10), (9, 11)]: connect(outmod, inmod) return controller, modules
def test_vtl_files(self): from vistrails.tests.utils import run_file for root, dirs, file_names in os.walk(self.vtl_path): for file_name in file_names: if file_name.endswith('.vtl'): f = os.path.join(root, file_name) locator = FileLocator(f) version = locator._vnode # if there is a version specified try to execute it, # else just load the pipeline if version: errors = run_file(f, lambda x: x == version) self.assertEqual(errors, [], 'Errors processing %s: %s' % (f, str(errors))) else: import vistrails.core.db.io from vistrails.core.vistrail.controller import \ VistrailController loaded_objs = vistrails.core.db.io.load_vistrail(locator) controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:]) controller.change_selected_version( controller.vistrail.get_latest_version()) self.assertTrue(controller.current_pipeline.is_valid, "Latest pipeline is invalid: %s" % f)
def test_vtl_files(self): from vistrails.tests.utils import run_file for root, dirs, file_names in os.walk(self.vtl_path): for file_name in sorted(file_names): if file_name.endswith('.vtl'): # update available packages from vistrails.core.packagemanager import get_package_manager get_package_manager().build_available_package_names_list() f = os.path.join(root, file_name) locator = FileLocator(f) version = locator._vnode # if there is a version specified try to execute it, # else just load the pipeline if version: errors = run_file(f, lambda x: x == version) self.assertEqual( errors, [], 'Errors processing %s: %s' % (f, str(errors))) else: import vistrails.core.db.io from vistrails.core.vistrail.controller import \ VistrailController loaded_objs = vistrails.core.db.io.load_vistrail( locator) controller = VistrailController( loaded_objs[0], locator, *loaded_objs[1:]) controller.change_selected_version( controller.vistrail.get_latest_version()) self.assertTrue(controller.current_pipeline.is_valid, "Latest pipeline is invalid: %s" % f)
def make_pipeline(self): """Creates an example pipeline that is used to conduct tests. """ vistrail = Vistrail() controller = VistrailController(vistrail) controller.change_selected_version(0) # 0 1 2 7 8 # | / \ \ / # 3 4 5 9 # | / \ # 6 10 11 modules = [ controller.add_module('org.vistrails.vistrails.basic', 'String') for i in xrange(12) ] def connect(outmod, inmod): controller.add_connection(modules[outmod].id, 'value', modules[inmod].id, 'value') for (outmod, inmod) in [(0, 3), (1, 4), (1, 5), (5, 6), (7, 9), (8, 9), (9, 10), (9, 11)]: connect(outmod, inmod) return controller, modules
def test_find_modules_by_type(self): """Tests the find_modules_by_type() function. """ vistrail = Vistrail() controller = VistrailController(vistrail) controller.change_selected_version(0) mod1 = controller.add_module('org.vistrails.vistrails.basic', 'String') mod2 = controller.add_module('org.vistrails.vistrails.basic', 'Float') mod3 = controller.add_module('org.vistrails.vistrails.basic', 'String') mod4 = controller.add_module('org.vistrails.vistrails.basic', 'Integer') from dat.vistrails_interface import find_modules_by_type from vistrails.core.modules.basic_modules import Boolean, Float, String self.assertEqual( set(m.id for m in find_modules_by_type(controller.current_pipeline, [String])), set([mod1.id, mod3.id])) self.assertEqual([ m.id for m in find_modules_by_type(controller.current_pipeline, [Float]) ], [mod2.id, mod4.id]) self.assertEqual( find_modules_by_type(controller.current_pipeline, [Boolean]), [])
def previewChanges(self, aliases): print "previewChanges", aliases # we will just execute the pipeline with the given alias dictionary controller = VistrailController() controller.set_vistrail(self.plot_vistrail, self.locator) version = self.plot_vistrail.get_version_number(self.workflow_tag) if self.workflow_tag else controller.get_latest_version_in_graph() controller.change_selected_version(version) (results, _) = controller.execute_current_workflow(aliases)
def test_cache(self): from vistrails.core.modules.basic_modules import StandardOutput old_compute = StandardOutput.compute StandardOutput.compute = lambda s: None try: from vistrails.core.db.locator import XMLFileLocator from vistrails.core.vistrail.controller import VistrailController from vistrails.core.db.io import load_vistrail """Test if basic caching is working.""" locator = XMLFileLocator( vistrails.core.system.vistrails_root_directory() + '/tests/resources/dummy.xml') (v, abstractions, thumbnails, mashups) = load_vistrail(locator) # the controller will take care of upgrades controller = VistrailController(v, locator, abstractions, thumbnails, mashups) p1 = v.getPipeline('int chain') n = v.get_version_number('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p1 = controller.current_pipeline view = DummyView() interpreter = CachedInterpreter.get() result = interpreter.execute( p1, locator=v, current_version=n, view=view, ) # to force fresh params p2 = v.getPipeline('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p2 = controller.current_pipeline result = interpreter.execute( p2, locator=v, current_version=n, view=view, ) self.assertEqual(len(result.modules_added), 1) finally: StandardOutput.compute = old_compute
def test_cache(self): from vistrails.core.modules.basic_modules import StandardOutput old_compute = StandardOutput.compute StandardOutput.compute = lambda s: None try: from vistrails.core.db.locator import XMLFileLocator from vistrails.core.vistrail.controller import VistrailController from vistrails.core.db.io import load_vistrail """Test if basic caching is working.""" locator = XMLFileLocator(vistrails.core.system.vistrails_root_directory() + '/tests/resources/dummy.xml') (v, abstractions, thumbnails, mashups) = load_vistrail(locator) # the controller will take care of upgrades controller = VistrailController(v, locator, abstractions, thumbnails, mashups) p1 = v.getPipeline('int chain') n = v.get_version_number('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p1 = controller.current_pipeline view = DummyView() interpreter = CachedInterpreter.get() result = interpreter.execute(p1, locator=v, current_version=n, view=view, ) # to force fresh params p2 = v.getPipeline('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p2 = controller.current_pipeline result = interpreter.execute(p2, locator=v, current_version=n, view=view, ) self.assertEqual(len(result.modules_added), 1) finally: StandardOutput.compute = old_compute
def run_file(filename, tag_filter=lambda x: True): """Loads a .vt file and runs all the tagged versions in it. """ import vistrails.core.db.io from vistrails.core.db.locator import FileLocator from vistrails.core.system import vistrails_root_directory from vistrails.core.vistrail.controller import VistrailController filename = os.path.join(vistrails_root_directory(), '..', filename) locator = FileLocator(filename) loaded_objs = vistrails.core.db.io.load_vistrail(locator) controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:]) errors = [] for version, name in controller.vistrail.get_tagMap().iteritems(): if tag_filter(name): controller.change_selected_version(version) (result,), _ = controller.execute_current_workflow() if result.errors: errors.append(("%d: %s" % (version, name), result.errors)) return errors
def test_find_modules_by_type(self): """Tests the find_modules_by_type() function. """ vistrail = Vistrail() controller = VistrailController(vistrail) controller.change_selected_version(0) mod1 = controller.add_module('org.vistrails.vistrails.basic', 'String') mod2 = controller.add_module('org.vistrails.vistrails.basic', 'Float') mod3 = controller.add_module('org.vistrails.vistrails.basic', 'String') mod4 = controller.add_module('org.vistrails.vistrails.basic', 'Integer') from dat.vistrails_interface import find_modules_by_type from vistrails.core.modules.basic_modules import Boolean, Float, String self.assertEqual( set(m.id for m in find_modules_by_type( controller.current_pipeline, [String])), set([mod1.id, mod3.id])) self.assertEqual( [m.id for m in find_modules_by_type( controller.current_pipeline, [Float])], [mod2.id, mod4.id]) self.assertEqual( find_modules_by_type( controller.current_pipeline, [Boolean]), [])
def run_and_get_results(w_list, parameters='', output_dir=None, update_vistrail=True, extra_info=None, reason='Console Mode Execution'): """run_and_get_results(w_list: list of (locator, version), parameters: str, output_dir:str, update_vistrail: boolean, extra_info:dict) Run all workflows in w_list, and returns an interpreter result object. version can be a tag name or a version id. """ elements = parameters.split("$&$") aliases = {} params = [] result = [] for locator, workflow in w_list: (v, abstractions , thumbnails, mashups) = load_vistrail(locator) controller = VistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=update_vistrail) if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) for e in elements: pos = e.find("=") if pos != -1: key = e[:pos].strip() value = e[pos+1:].strip() if controller.current_pipeline.has_alias(key): aliases[key] = value elif 'mashup_id' in extra_info: # new-style mashups can have aliases not existing in pipeline for mashuptrail in mashups: if mashuptrail.vtVersion == version: mashup = mashuptrail.getMashup(extra_info['mashup_id']) c = mashup.getAliasByName(key).component params.append((c.vttype, c.vtid, value)) if output_dir is not None and controller.current_pipeline is not None: # FIXME DAK: why is this always done?!? there is a flag for it... if is_running_gui(): controller.updatePipelineScene() base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version) filename = os.path.join(output_dir, base_fname) controller.current_pipeline_scene.saveToPDF(filename) else: debug.critical("Cannot save pipeline figure when not " "running in gui mode") base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version) filename = os.path.join(output_dir, base_fname) vistrails.core.db.io.save_workflow(controller.current_pipeline, filename) if not update_vistrail: conf = get_vistrails_configuration() if conf.has('thumbs'): conf.thumbs.autoSave = False jobMonitor = controller.jobMonitor current_workflow = jobMonitor.currentWorkflow() if not current_workflow: for job in jobMonitor.workflows.itervalues(): try: job_version = int(job.version) except ValueError: job_version = v.get_version_number(job.version) if version == job_version: current_workflow = job jobMonitor.startWorkflow(job) if not current_workflow: current_workflow = JobWorkflow(version) jobMonitor.startWorkflow(current_workflow) try: (results, _) = \ controller.execute_current_workflow(custom_aliases=aliases, custom_params=params, extra_info=extra_info, reason=reason) finally: jobMonitor.finishWorkflow() new_version = controller.current_version if new_version != version: debug.log("Version '%s' (%s) was upgraded. The actual " "version executed was %s" % ( workflow, version, new_version)) run = results[0] run.workflow_info = (locator.name, new_version) run.pipeline = controller.current_pipeline if update_vistrail: controller.write_vistrail(locator) result.append(run) if current_workflow.jobs: if current_workflow.completed(): run.job = "COMPLETED" else: run.job = "RUNNING: %s" % current_workflow.id for job in current_workflow.jobs.itervalues(): if not job.finished: run.job += "\n %s %s %s" % (job.start, job.name, job.description()) print run.job return result
def execute_wf(wf, output_port): # Save the workflow in a temporary file temp_wf_fd, temp_wf = tempfile.mkstemp() try: f = open(temp_wf, 'w') f.write(wf) f.close() os.close(temp_wf_fd) # Clean the cache interpreter = get_default_interpreter() interpreter.flush() # Load the Pipeline from the temporary file vistrail = Vistrail() locator = XMLFileLocator(temp_wf) workflow = locator.load(Pipeline) # Build a Vistrail from this single Pipeline action_list = [] for module in workflow.module_list: action_list.append(('add', module)) for connection in workflow.connection_list: action_list.append(('add', connection)) action = vistrails.core.db.action.create_action(action_list) vistrail.add_action(action, 0L) vistrail.update_id_scope() tag = 'parallel flow' vistrail.addTag(tag, action.id) # Build a controller and execute controller = VistrailController() controller.set_vistrail(vistrail, None) controller.change_selected_version(vistrail.get_version_number(tag)) execution = controller.execute_current_workflow( custom_aliases=None, custom_params=None, extra_info=None, reason='API Pipeline Execution') # Build a list of errors errors = [] pipeline = vistrail.getPipeline(tag) execution_errors = execution[0][0].errors if execution_errors: for key in execution_errors: module = pipeline.modules[key] msg = '%s: %s' %(module.name, execution_errors[key]) errors.append(msg) # Get the execution log from the controller try: module_log = controller.log.workflow_execs[0].item_execs[0] except IndexError: errors.append("Module log not found") return dict(errors=errors) else: machine = controller.log.workflow_execs[0].machines[ module_log.machine_id] xml_log = serialize(module_log) machine_log = serialize(machine) # Get the output value output = None serializable = None if not execution_errors: executed_module, = execution[0][0].executed executed_module = execution[0][0].objects[executed_module] try: output = executed_module.get_output(output_port) except ModuleError: errors.append("Output port not found: %s" % output_port) return dict(errors=errors) reg = vistrails.core.modules.module_registry.get_module_registry() base_classes = inspect.getmro(type(output)) if Module in base_classes: serializable = reg.get_descriptor(type(output)).sigstring output = output.serialize() # Return the dictionary, that will be sent back to the client return dict(errors=errors, output=output, serializable=serializable, xml_log=xml_log, machine_log=machine_log) finally: os.unlink(temp_wf)
def execute_wf(wf, output_port): # Save the workflow in a temporary file temp_wf_fd, temp_wf = tempfile.mkstemp() try: f = open(temp_wf, 'w') f.write(wf) f.close() os.close(temp_wf_fd) # Clean the cache interpreter = get_default_interpreter() interpreter.flush() # Load the Pipeline from the temporary file vistrail = Vistrail() locator = XMLFileLocator(temp_wf) workflow = locator.load(Pipeline) # Build a Vistrail from this single Pipeline action_list = [] for module in workflow.module_list: action_list.append(('add', module)) for connection in workflow.connection_list: action_list.append(('add', connection)) action = vistrails.core.db.action.create_action(action_list) vistrail.add_action(action, 0L) vistrail.update_id_scope() tag = 'parallel flow' vistrail.addTag(tag, action.id) # Build a controller and execute controller = VistrailController() controller.set_vistrail(vistrail, None) controller.change_selected_version(vistrail.get_version_number(tag)) execution = controller.execute_current_workflow( custom_aliases=None, custom_params=None, extra_info=None, reason='API Pipeline Execution') # Build a list of errors errors = [] pipeline = vistrail.getPipeline(tag) execution_errors = execution[0][0].errors if execution_errors: for key in execution_errors: module = pipeline.modules[key] msg = '%s: %s' % (module.name, execution_errors[key]) errors.append(msg) # Get the execution log from the controller try: module_log = controller.log.workflow_execs[0].item_execs[0] except IndexError: errors.append("Module log not found") return dict(errors=errors) else: machine = controller.log.workflow_execs[0].machines[ module_log.machine_id] xml_log = serialize(module_log) machine_log = serialize(machine) # Get the output value output = None serializable = None if not execution_errors: executed_module, = execution[0][0].executed executed_module = execution[0][0].objects[executed_module] try: output = executed_module.get_output(output_port) except ModuleError: errors.append("Output port not found: %s" % output_port) return dict(errors=errors) reg = vistrails.core.modules.module_registry.get_module_registry() base_classes = inspect.getmro(type(output)) if Module in base_classes: serializable = reg.get_descriptor(type(output)).sigstring output = output.serialize() # Return the dictionary, that will be sent back to the client return dict(errors=errors, output=output, serializable=serializable, xml_log=xml_log, machine_log=machine_log) finally: os.unlink(temp_wf)
def run_and_get_results(w_list, parameters='', update_vistrail=True, extra_info=None, reason='Console Mode Execution'): """run_and_get_results(w_list: list of (locator, version), parameters: str, output_dir:str, update_vistrail: boolean, extra_info:dict) Run all workflows in w_list, and returns an interpreter result object. version can be a tag name or a version id. """ elements = parameters.split("$&$") aliases = {} params = [] result = [] for locator, workflow in w_list: (v, abstractions, thumbnails, mashups) = load_vistrail(locator) controller = VistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=update_vistrail) if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) for e in elements: pos = e.find("=") if pos != -1: key = e[:pos].strip() value = e[pos + 1:].strip() if controller.current_pipeline.has_alias(key): aliases[key] = value elif 'mashup_id' in extra_info: # new-style mashups can have aliases not existing in pipeline for mashuptrail in mashups: if mashuptrail.vtVersion == version: mashup = mashuptrail.getMashup( extra_info['mashup_id']) c = mashup.getAliasByName(key).component params.append((c.vttype, c.vtid, value)) if not update_vistrail: conf = get_vistrails_configuration() if conf.has('thumbs'): conf.thumbs.autoSave = False jobMonitor = controller.jobMonitor current_workflow = jobMonitor.currentWorkflow() if not current_workflow: for job in jobMonitor.workflows.itervalues(): try: job_version = int(job.version) except ValueError: try: job_version = v.get_version_number(job.version) except KeyError: # this is a PE or mashup continue if version == job_version: current_workflow = job jobMonitor.startWorkflow(job) if not current_workflow: current_workflow = JobWorkflow(version) jobMonitor.startWorkflow(current_workflow) try: (results, _) = \ controller.execute_current_workflow(custom_aliases=aliases, custom_params=params, extra_info=extra_info, reason=reason) finally: jobMonitor.finishWorkflow() new_version = controller.current_version if new_version != version: debug.log("Version '%s' (%s) was upgraded. The actual " "version executed was %s" % (workflow, version, new_version)) run = results[0] run.workflow_info = (locator.name, new_version) run.pipeline = controller.current_pipeline if update_vistrail: controller.write_vistrail(locator) result.append(run) if current_workflow.jobs: if current_workflow.completed(): run.job = "COMPLETED" else: run.job = "RUNNING: %s" % current_workflow.id for job in current_workflow.jobs.itervalues(): if not job.finished: run.job += "\n %s %s %s" % (job.start, job.name, job.description()) print run.job return result
def load(self, loadworkflow=True): config = ConfigParser.ConfigParser() if config.read(self.config_file): if config.has_section('global'): if config.has_option('global', 'cellnum'): self.cellnum = config.getint('global', 'cellnum') if config.has_option('global', 'filenum'): self.filenum = config.getint('global', 'filenum') if config.has_option('global', 'varnum'): self.varnum = config.getint('global', 'varnum') print " ------ Loaded plot %s, varnum = %d ------ " % ( self.name, self.varnum ) if config.has_option('global', 'workflow_tag'): self.workflow_tag = config.get('global', 'workflow_tag') # else: # debug.warning("CDAT Package: file %s does not contain a required option 'workflow_tag'. Widget will not be loaded."%self.config_file) # self.loaded = False # return if config.has_option('global', 'filetypes'): types = config.get('global', 'filetypes') tlist = [t.strip() for t in types.split(";")] for t in tlist: kv = t.split(":") self.filetypes[kv[0].strip()] = [v.strip() for v in kv[1].split(",")] if config.has_option('global', 'qt_filter'): self.qt_filter = config.get('global', 'qt_filter') if config.has_option('global', 'dependencies'): deps = config.get('global', 'dependencies') self.dependencies = [d.strip() for d in deps.split(",")] if config.has_option('global', 'serialized_config_alias'): self.serializedConfigAlias = config.get('global', 'serialized_config_alias') for y in range(self.filenum): self.files.append( 'Filename' + str(y+1) ) for v in range(self.varnum): self.vars.append( 'VariableName' + str(v+1) ) self.axes.append( 'Axes' + str(v+1) ) for x in range(self.cellnum): section_name = 'cell' + str(x+1) if config.has_section(section_name): cellType = config.get(section_name, 'celltype') if config.has_option(section_name, 'address_alias'): self.cells.append( Cell( cellType, None, None, config.get(section_name, 'address_alias') ) ) else: self.cells.append(Cell( cellType,"Row"+str(x+1), "Column"+str(x+1) ) ) else: for y in range(self.filenum): option_name = 'filename_alias' + str(y+1) if config.has_option('global', option_name): self.files.append(config.get('global', option_name)) for v in range(self.varnum): option_name = 'varname_alias' + str(v+1) if config.has_option('global', option_name): self.vars.append(config.get('global', option_name)) axes_name = 'axes_alias' + str(v+1) if config.has_option('global', axes_name): self.axes.append(config.get('global', axes_name)) for x in range(self.cellnum): section_name = 'cell' + str(x+1) if (config.has_section(section_name) and config.has_option(section_name, 'celltype') and config.has_option(section_name, 'row_alias') and config.has_option(section_name, 'col_alias')): self.cells.append(Cell(config.get(section_name, 'celltype'), config.get(section_name, 'row_alias'), config.get(section_name, 'col_alias'))) if loadworkflow: #load workflow in vistrail #only if dependencies are enabled manager = get_package_manager() self.unsatisfied_deps = [] for dep in self.dependencies: if not manager.has_package(dep): self.unsatisfied_deps.append(dep) if len(self.unsatisfied_deps) == 0: try: (self.plot_vistrail, abstractions , thumbnails, mashups) = load_vistrail(self.locator) controller = VistrailController() controller.set_vistrail(self.plot_vistrail, self.locator, abstractions, thumbnails, mashups) self.workflow_version = self.plot_vistrail.get_version_number(self.workflow_tag) if self.workflow_tag else controller.get_latest_version_in_graph() print " Loaded %s version: %s" % ( self.name, str( self.workflow_version ) ) controller.change_selected_version(self.workflow_version) self.workflow = controller.current_pipeline self.loaded = True except Exception, err: debug.warning( "Error loading workflow %s: %s" % ( self.name, err ) ) self.loaded = False else: debug.warning("UV-CDAT: %s widget could not be loaded \ because it depends on packages that are not loaded:"%self.name) debug.warning(" %s"%", ".join(self.unsatisfied_deps)) self.loaded = False else: debug.warning("UV-CDAT: file %s does not contain a 'global'\ section. Widget will not be loaded."%self.config_file) self.loaded = False
class Vistrail(object): """This class wraps both Vistrail and VistrailController. From it, you can get any pipeline from a tag name or version number. It has a concept of "current version", from which you can create new versions by performing actions. """ _current_pipeline = None _html = None def __init__(self, arg=None): initialize() if arg is None: # Copied from VistrailsApplicationInterface#open_vistrail() locator = UntitledLocator() loaded_objs = vistrails.core.db.io.load_vistrail(locator) self.controller = VistrailController(*loaded_objs) elif isinstance(arg, (_Pipeline, Pipeline)): if isinstance(arg, Pipeline): pipeline = arg.pipeline else: pipeline = arg # Copied from VistrailsApplicationInterface#open_workflow() vistrail = _Vistrail() ops = [] for module in pipeline.module_list: ops.append(('add', module)) for connection in pipeline.connection_list: ops.append(('add', connection)) action = vistrails.core.db.action.create_action(ops) vistrail.add_action(action, 0L) vistrail.update_id_scope() vistrail.change_description("Imported pipeline", 0L) self.controller = VistrailController(vistrail, UntitledLocator()) elif isinstance(arg, VistrailController): self.controller = arg elif isinstance(arg, basestring): raise TypeError("Vistrail was constructed from %r.\n" "Use load_vistrail() to get a Vistrail from a " "file." % type(arg).__name__) else: raise TypeError("Vistrail was constructed from unexpected " "argument type %r" % type(arg).__name__) def get_pipeline(self, version): """Returns a pipeline from a version number of tag. This does not change the currently selected version in this Vistrail. """ vistrail = self.controller.vistrail if isinstance(version, (int, long)): if not vistrail.db_has_action_with_id(version): raise NoSuchVersion("Vistrail doesn't have a version %r" % version) return Pipeline(vistrail.getPipelineVersionNumber(version)) elif isinstance(version, basestring): if not vistrail.has_tag_str(version): raise NoSuchVersion("Vistrail doesn't have a tag %r" % version) return Pipeline(vistrail.getPipelineVersionName(version)) else: raise TypeError("get_pipeline() argument must be a string or " "integer, not %r" % type(version).__name__) def select_version(self, version): """Sets a different version as current. The current workflow is accessible via current_workflow; it is the one that gets executed when calling execute(), and the version from which new versions are created if you perform actions. """ vistrail = self.controller.vistrail if isinstance(version, (int, long)): if not vistrail.db_has_action_with_id(version): raise NoSuchVersion("Vistrail doesn't have a version %r" % version) elif (isinstance(version, basestring)): if not vistrail.has_tag_str(version): raise NoSuchVersion("Vistrail doesn't have a tag %r" % version) version = vistrail.get_tag_str(version).action_id else: raise TypeError("select_version() argument must be a string " "or integer, not %r" % type(version).__name__) self.controller.change_selected_version(version) self._current_pipeline = None self._html = None def select_latest_version(self): """Sets the most recent version in the vistrail as current. """ self.controller.select_latest_version() self._current_pipeline = None self._html = None @property def current_pipeline(self): if self._current_pipeline is None: self._current_pipeline = Pipeline( self.controller.current_pipeline, vistrail=(self, self.current_version)) return self._current_pipeline @property def current_version(self): return self.controller.current_version def set_tag(self, *args): """Sets a tag for the current or specified version. """ if len(args) == 1: version, (tag,) = self.controller.current_version, args elif len(args) == 2: version, tag = args else: raise TypeError("set_tag() takes 1 or 2 arguments (%r given)" % len(args)) if isinstance(version, (int, long)): if not self.controller.vistrail.db_has_action_with_id(version): raise NoSuchVersion("Vistrail doesn't have a version %r" % version) elif isinstance(version, basestring): if not self.controller.vistrail.has_tag_str(version): raise NoSuchVersion("Vistrail doesn't have a tag %r" % version) else: raise TypeError("set_tag() expects the version to be a string or " "integer, not %r" % type(version).__name__) self.controller.vistrail.set_tag(version, tag) def tag(self, tag): """Sets a tag for the current version. """ self.set_tag(tag) def execute(self, *args, **kwargs): """Executes the current workflow. """ return self.current_pipeline.execute(*args, **kwargs) @property def changed(self): return self.controller.changed # TODO : vistrail modification methods def __repr__(self): version_nb = self.controller.current_version if self.controller.vistrail.has_tag(version_nb): version = "%s (tag %s)" % ( version_nb, self.controller.vistrail.get_tag(version_nb)) else: version = version_nb return "<%s: %s, version %s, %s>" % ( self.__class__.__name__, self.controller.name, version, ('not changed', 'changed')[self.controller.changed]) def _repr_html_(self): if self._html is None: import cgi try: from cStringIO import StringIO except ImportError: from StringIO import StringIO self._html = '' stream = StringIO() self.controller.recompute_terse_graph() self.controller.save_version_graph( stream, highlight=self.controller.current_version) stream.seek(0) dot = stream.read() try: proc = subprocess.Popen(['dot', '-Tsvg'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) svg, _ = proc.communicate(dot) if proc.wait() == 0: self._html += svg except OSError: pass self._html += '<pre>' + cgi.escape(repr(self)) + '</pre>' return self._html
def execute(workflowJSON): ''' Execute a workflow from it's JSON representation ''' debug('convert json to xml') workflowXML = json2xml(workflowJSON) #temp_wf_fd, temp_wf = tempfile.mkstemp('.xml') debug('create temporary file') temp_wf_fd, temp_wf = tempfile.mkstemp() try: f = open(temp_wf, 'w') f.write(workflowXML) f.close() os.close(temp_wf_fd) #load workflow temp file into vistrails #vt.load_workflow(temp_wf) #execute workflow #execution = vt.execute() debug('Load the Pipeline from the temporary file') vistrail = Vistrail() locator = XMLFileLocator(temp_wf) workflow = locator.load(Pipeline) debug('Build a Vistrail from this single Pipeline') action_list = [] for module in workflow.module_list: action_list.append(('add', module)) for connection in workflow.connection_list: action_list.append(('add', connection)) action = vistrails.core.db.action.create_action(action_list) debug('add actions') vistrail.add_action(action, 0L) vistrail.update_id_scope() tag = 'climatepipes' vistrail.addTag(tag, action.id) debug('Build a controller and execute') controller = VistrailController() controller.set_vistrail(vistrail, None) controller.change_selected_version(vistrail.get_version_number(tag)) execution = controller.execute_current_workflow( custom_aliases=None, custom_params=None, extra_info=None, reason='API Pipeline Execution') debug('get result') execution_pipeline = execution[0][0] if len(execution_pipeline.errors) > 0: error("Executing workflow") for key in execution_pipeline.errors: error(execution_pipeline.errors[key]) print execution_pipeline.errors[key] return None modules = execution_pipeline.objects for id, module in modules.iteritems(): if isinstance(module, ToGeoJSON): return json.dumps({'result': module.JSON, 'error': None }) finally: os.unlink(temp_wf)
def run_and_get_results(w_list, parameters='', workflow_info=None, update_vistrail=True, extra_info=None, reason='Console Mode Execution'): """run_and_get_results(w_list: list of (locator, version), parameters: str, workflow_info:str, update_vistrail: boolean, extra_info:dict) Run all workflows in w_list, and returns an interpreter result object. version can be a tag name or a version id. """ elements = parameters.split("$&$") aliases = {} result = [] for locator, workflow in w_list: (v, abstractions , thumbnails, mashups) = load_vistrail(locator) controller = VistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=update_vistrail) if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) for e in elements: pos = e.find("=") if pos != -1: key = e[:pos].strip() value = e[pos+1:].strip() if controller.current_pipeline.has_alias(key): aliases[key] = value if workflow_info is not None and controller.current_pipeline is not None: # FIXME DAK: why is this always done?!? there is a flag for it... if is_running_gui(): controller.updatePipelineScene() base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version) filename = os.path.join(workflow_info, base_fname) controller.current_pipeline_scene.saveToPDF(filename) else: debug.critical("Cannot save pipeline figure when not " "running in gui mode") base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version) filename = os.path.join(workflow_info, base_fname) vistrails.core.db.io.save_workflow(controller.current_pipeline, filename) if not update_vistrail: conf = get_vistrails_configuration() if conf.has('thumbs'): conf.thumbs.autoSave = False (results, _) = \ controller.execute_current_workflow(custom_aliases=aliases, extra_info=extra_info, reason=reason) new_version = controller.current_version if new_version != version: debug.warning("Version '%s' (%s) was upgraded. The actual " "version executed was %s" % \ (workflow, version, new_version)) run = results[0] run.workflow_info = (locator.name, new_version) run.pipeline = controller.current_pipeline if update_vistrail: controller.write_vistrail(locator) result.append(run) return result