def copyVistrailController(vt_controller, view=DummyView()): vistrail = vt_controller.vistrail newvt_controller = VistrailController(vistrail, None, pipeline_view=view) current_log = vt_controller.log newvt_controller.log = current_log newvt_controller.disable_autosave() return newvt_controller
def createMashupApp(vtcontroller, mashuptrail, version): from vistrails.gui.mashups.mashup_app import QMashupAppMainWindow vistrail_view = vtcontroller.vistrail_view vtVersion = mashuptrail.vtVersion view = DummyView() view.scene().current_pipeline = vtcontroller.vistrail.getPipeline(vtVersion) view.scene().current_pipeline.validate() new_vtcontroller = MashupsManager.copyBaseVistrailController(vtcontroller) new_vtcontroller.change_selected_version(vtVersion) mshpController = MashupController(vtcontroller, new_vtcontroller, vtVersion, mashuptrail) mshpController.setCurrentVersion(version) app = QMashupAppMainWindow(parent=None, vistrail_view=vistrail_view, controller=mshpController, version=version) return app
def createMashupApp(vtcontroller, mashuptrail, version): from vistrails.gui.mashups.mashup_app import QMashupAppMainWindow vistrail_view = vtcontroller.vistrail_view vtVersion = mashuptrail.vtVersion view = DummyView() view.scene().current_pipeline = vtcontroller.vistrail.getPipeline(vtVersion) view.scene().current_pipeline.validate() new_vtcontroller = MashupsManager.copyBaseVistrailController(vtcontroller) new_vtcontroller.change_selected_version(vtVersion) mshpController = MashupController(vtcontroller, new_vtcontroller, vtVersion, mashuptrail) mshpController.setCurrentVersion(version) app = QMashupAppMainWindow(parent=None, vistrail_view=vistrail_view, controller=mshpController, version=version) return app
def test_cache(self): from vistrails.core.modules.basic_modules import StandardOutput old_compute = StandardOutput.compute StandardOutput.compute = lambda s: None try: from vistrails.core.db.locator import XMLFileLocator from vistrails.core.vistrail.controller import VistrailController from vistrails.core.db.io import load_vistrail """Test if basic caching is working.""" locator = XMLFileLocator( vistrails.core.system.vistrails_root_directory() + '/tests/resources/dummy.xml') (v, abstractions, thumbnails, mashups) = load_vistrail(locator) # the controller will take care of upgrades controller = VistrailController(v, locator, abstractions, thumbnails, mashups) p1 = v.getPipeline('int chain') n = v.get_version_number('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p1 = controller.current_pipeline view = DummyView() interpreter = CachedInterpreter.get() result = interpreter.execute( p1, locator=v, current_version=n, view=view, ) # to force fresh params p2 = v.getPipeline('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p2 = controller.current_pipeline result = interpreter.execute( p2, locator=v, current_version=n, view=view, ) self.assertEqual(len(result.modules_added), 1) finally: StandardOutput.compute = old_compute
def test_pipeline_creation(self): import dat.tests.pkg_test_plots.init as pkg_test_plots controller = self.vt_controller() vistraildata = VistrailManager(controller) loader = Test_generation._loaders.get('StrMaker') loader.v = 'Hello' vistraildata.new_variable('var1', loader.load()) loader.v = 'world' vistraildata.new_variable('var2', loader.load()) cellInfo = FakeObj( row=0, column=0, tab=FakeObj(tabWidget=FakeObj(tabText=lambda w: 'Sheet 1'))) recipe = DATRecipe( pkg_test_plots.concat_plot, { 'param1': (RecipeParameterValue( variable=vistraildata.get_variable('var1')), ), 'param2': (RecipeParameterValue( variable=vistraildata.get_variable('var2')), ), 'param3': (RecipeParameterValue(constant="!"), ), }) pipelineInfo = vistrails_interface.create_pipeline( controller, recipe, cellInfo.row, cellInfo.column, None) # This plot has no cell module so this is fine controller.change_selected_version(pipelineInfo.version) result = CallRecorder() pkg_test_plots.Recorder.callback = result interpreter = get_default_interpreter() interpreter.execute(controller.current_pipeline, view=DummyView(), locator=controller.locator, current_version=pipelineInfo.version) call = (['Hello, world!'], dict()) self.assertEqual(result.calls, [call])
def execute(self): """ execute() -> None Execute the selected pipeline with the edited aliases """ aliases = {} for r in xrange(self.aliasTable.rowCount()): name = str(self.aliasTable.item(r, 0).text()) value = str(self.aliasTable.item(r, 1).text()) aliases[name] = value versionNumber = self.versions[ self.pipelineList.currentIndex().row()][0] pipeline = self.vistrail.getPipelineVersionNumber(versionNumber) controller = VistrailController(self.vistrail) controller.execute_workflow_list([ (self.vistrail.locator, versionNumber, pipeline, DummyView(), aliases, None) ])
def test_tuple(self): from vistrails.core.vistrail.module_param import ModuleParam from vistrails.core.vistrail.module_function import ModuleFunction from vistrails.core.utils import DummyView from vistrails.core.vistrail.module import Module import vistrails.db.domain id_scope = vistrails.db.domain.IdScope() interpreter = vistrails.core.interpreter.default.get_default_interpreter( ) v = DummyView() p = vistrails.core.vistrail.pipeline.Pipeline() params = [ ModuleParam( id=id_scope.getNewId(ModuleParam.vtType), pos=0, type='Float', val='2.0', ), ModuleParam( id=id_scope.getNewId(ModuleParam.vtType), pos=1, type='Float', val='2.0', ) ] function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType), name='input') function.add_parameters(params) module = Module(id=id_scope.getNewId(Module.vtType), name='TestTupleExecution', package='org.vistrails.vistrails.console_mode_test', version='0.9.1') module.add_function(function) p.add_module(module) interpreter.execute(p, locator=XMLFileLocator('foo'), current_version=1L, view=v)
def execute_pipeline(controller, pipeline, reason, locator, version, **kwargs): """Execute the pipeline while showing a progress dialog. """ _ = translate('execute_pipeline') totalProgress = len(pipeline.modules) progress = QtGui.QProgressDialog(_("Executing..."), None, 0, totalProgress) progress.setWindowTitle(_("Pipeline Execution")) progress.setWindowModality(QtCore.Qt.WindowModal) progress.show() def moduleExecuted(objId): progress.setValue(progress.value() + 1) QtCore.QCoreApplication.processEvents() if 'module_executed_hook' in kwargs: kwargs['module_executed_hook'].append(moduleExecuted) else: kwargs['module_executed_hook'] = [moduleExecuted] results, changed = controller.execute_workflow_list([( locator, # locator version, # version pipeline, # pipeline DummyView(), # view None, # custom_aliases None, # custom_params reason, # reason None, # sinks kwargs)]) # extra_info get_vistrails_application().send_notification('execution_updated') progress.setValue(totalProgress) progress.hide() progress.deleteLater() if not results[0].errors: return None else: module_id, error = next(results[0].errors.iteritems()) return str(error)
def executePipelineWithProgress(pipeline, pTitle='Pipeline Execution', pCaption='Executing...', pCancel='&Cancel', **kwargs): """ executePipelineWithProgress(pipeline: Pipeline, pTitle: str, pCaption: str, pCancel: str, kwargs: keyword arguments) -> bool Execute the pipeline while showing a progress dialog with title pTitle, caption pCaption and the cancel button text pCancel. kwargs is the keyword arguments that will be passed to the interpreter. A bool will be returned indicating if the execution was performed without cancel or not. """ withoutCancel = True totalProgress = len(pipeline.modules) progress = QtGui.QProgressDialog(pCaption, pCancel, 0, totalProgress) progress.setWindowTitle(pTitle) progress.setWindowModality(QtCore.Qt.WindowModal) progress.show() def moduleExecuted(objId): if not progress.wasCanceled(): progress.setValue(progress.value()+1) QtCore.QCoreApplication.processEvents() else: withoutCancel = False interpreter = get_default_interpreter() if kwargs.has_key('module_executed_hook'): kwargs['module_executed_hook'].append(moduleExecuted) else: kwargs['module_executed_hook'] = [moduleExecuted] kwargs['view'] = DummyView() interpreter.execute(pipeline, **kwargs) progress.setValue(totalProgress) return withoutCancel
def openSpreadsheet(self, fileName): """ openSpreadsheet(fileName: str) -> None Open a saved spreadsheet assuming that all VTK files must exist and have all the version using the saved spreadsheet """ def parse_locator(text): locator = None wrapper = XMLWrapper() dom = wrapper.create_document_from_string(text) root = dom.documentElement version = None version = root.getAttribute('version') if version == '1.0': for element in named_elements(root, 'locator'): if str(element.getAttribute('type')) == 'file': locator = FileLocator.parse(element) elif str(element.getAttribute('type')) == 'db': locator = DBLocator.parse(element) return locator locators = {} indexFile = open(fileName, 'r') contents = indexFile.read() self.clearTabs() lidx = 0 lines = contents.split('\n') tabCount = int(lines[lidx]) lidx += 1 for tabIdx in xrange(tabCount): # FIXME: eval should pretty much never be used tabInfo = literal_eval(lines[lidx]) lidx += 1 sheet = spreadsheetRegistry.getSheet(tabInfo[1])(self) sheet.setDimension(tabInfo[2], tabInfo[3]) self.addTabWidget(sheet, tabInfo[0]) while lines[lidx] != '---': (r, c, vistrail, pid, cid) = literal_eval(lines[lidx]) locator = vistrail['locator'] if locators.has_key(locator): vistrail['locator'] = locators[locator] else: locators[locator] = parse_locator(vistrail['locator']) vistrail['locator'] = locators[locator] self.appendMonitoredLocations((vistrail, pid, cid), (sheet, r, c)) lidx += 1 lidx += 1 pipelineCount = int(lines[lidx]) lidx += 1 self.loadingMode = True progress = QtGui.QProgressDialog("Loading spreadsheet...", "&Cancel", 0, pipelineCount, self, QtCore.Qt.WindowStaysOnTopHint) progress.show() for pipelineIdx in xrange(pipelineCount): # FIXME: eval should pretty much never be used (serializedLocator, version) = literal_eval(lines[lidx]) try: locator = locators[serializedLocator] except KeyError: locator = parse_locator(serializedLocator) if locator: bundle = locator.load() if isinstance(bundle, SaveBundle): pipeline = bundle.vistrail.getPipeline(version) else: pipeline = bundle.getPipeline(version) execution = get_default_interpreter() progress.setValue(pipelineIdx) QtCore.QCoreApplication.processEvents() if progress.wasCanceled(): break kwargs = { 'locator': locator, 'current_version': version, 'view': DummyView(), } execution.execute(pipeline, **kwargs) else: raise RuntimeError("Couldn't load spreadsheet") lidx += 1 progress.setValue(pipelineCount) QtCore.QCoreApplication.processEvents() self.changeSpreadsheetFileName(fileName) self.loadingMode = False indexFile.close()
def createMashupController(self, vt_controller, version, view=DummyView()): #print "Manager creating mashup controller ", vt_controller, version newvt_controller = MashupsManager.copyVistrailController( vt_controller, view) mashuptrail = \ MashupsManager.getMashuptrailforVersionInVistrailController(vt_controller, version) if mashuptrail is None: (p_mashuptrail, p_version) = \ MashupsManager.findClosestParentMashuptrail(vt_controller, version) id_scope = IdScope(1L) if p_mashuptrail is not None: version_name = vt_controller.get_pipeline_name(p_version) (res, mshpv) = MashupsManager.showFoundMashupsDialog( p_mashuptrail, version_name) if res in ['Copy', 'Move']: pipeline = newvt_controller.vistrail.getPipeline(version) if res == 'Copy': # we will copy the mashup from the parent trail and # validate it to the current pipeline before adding # to the current mashup trail mashuptrail = Mashuptrail(self.getNewMashuptrailId(), version, id_scope) p_mashup = p_mashuptrail.getMashup(mshpv) mashup = p_mashup.do_copy() mashup.id_scope = id_scope mashup.version = version mashup.validateForPipeline(pipeline) currVersion = mashuptrail.addVersion( parent_id=mashuptrail.getLatestVersion(), mashup=mashup, user=vistrails.core.system.current_user(), date=vistrails.core.system.current_time()) mashuptrail.currentVersion = currVersion mashuptrail.updateIdScope() p_tag = p_mashuptrail.getTagForActionId(mshpv) if p_tag == '': tag = "<latest>" tag = "Copy from %s" % p_tag MashupsManager.addMashuptrailtoVistrailController( vt_controller, mashuptrail) else: # res == 'Move' # we will move the parent trail and validate all mashups # for the current pipeline to make sure they will be # executable for the current version mashuptrail = p_mashuptrail currVersion = mashuptrail.getLatestVersion() mashuptrail.currentVersion = currVersion mashuptrail.validateMashupsForPipeline( version, pipeline) tag = None mashuptrail.vtVersion = version mshpController = MashupController(vt_controller, newvt_controller, version, mashuptrail) mshpController.setCurrentVersion( mashuptrail.currentVersion) # this is to make sure the pipeline displayed in the mashup # view is consistent with the list of aliases in the central # panel mshpController.updatePipelineAliasesFromCurrentMashup() if tag is not None: mshpController.updateCurrentTag(tag) return mshpController mashuptrail = Mashuptrail(self.getNewMashuptrailId(), version, id_scope) pipeline = newvt_controller.vistrail.getPipeline(version) id = id_scope.getNewId('mashup') mashup = Mashup(id=id, name="mashup%s" % id, vtid=vt_controller.locator, version=version) mashup.loadAliasesFromPipeline(pipeline, id_scope) currVersion = mashuptrail.addVersion( parent_id=mashuptrail.getLatestVersion(), mashup=mashup, user=vistrails.core.system.current_user(), date=vistrails.core.system.current_time()) mashuptrail.currentVersion = currVersion MashupsManager.addMashuptrailtoVistrailController( vt_controller, mashuptrail) mshpController = MashupController(vt_controller, newvt_controller, version, mashuptrail) mshpController.setCurrentVersion(mashuptrail.currentVersion) if mshpController.currentVersion == 1L: mshpController.updateCurrentTag("ROOT") else: #print "----> found mashuptrail ", mashuptrail.currentVersion mshpController = MashupController(vt_controller, newvt_controller, version, mashuptrail) mshpController.setCurrentVersion(mashuptrail.currentVersion) mshpController.updatePipelineAliasesFromCurrentMashup() return mshpController
def execute(self, *args, **kwargs): """Execute the pipeline. Positional arguments are either input values (created from ``module == value``, where `module` is a Module from the pipeline and `value` is some value or Function instance) for the pipeline's InputPorts, or Module instances (to select sink modules). Keyword arguments are also used to set InputPort by looking up inputs by name. Example:: input_bound = pipeline.get_input('higher_bound') input_url = pipeline.get_input('url') sinkmodule = pipeline.get_module(32) pipeline.execute(sinkmodule, input_bound == vt.Function(Integer, 10), input_url == 'http://www.vistrails.org/', resolution=15) # kwarg: only one equal sign """ sinks = set() inputs = {} reg = get_module_registry() InputPort_desc = reg.get_descriptor_by_name( get_vistrails_basic_pkg_id(), 'InputPort') # Read args for arg in args: if isinstance(arg, ModuleValuePair): if arg.module.id in inputs: raise ValueError("Multiple values set for InputPort %r" % get_inputoutput_name(arg.module)) if not reg.is_descriptor_subclass(arg.module.module_descriptor, InputPort_desc): raise ValueError("Module %d is not an InputPort" % arg.module.id) inputs[arg.module.id] = arg.value elif isinstance(arg, Module): sinks.add(arg.module_id) # Read kwargs for key, value in kwargs.iteritems(): key = self.get_input(key) # Might raise KeyError if key.module_id in inputs: raise ValueError("Multiple values set for InputPort %r" % get_inputoutput_name(key.module)) inputs[key.module_id] = value reason = "API pipeline execution" sinks = sinks or None # Use controller only if no inputs were passed in if (not inputs and self.vistrail is not None and self.vistrail.current_version == self.version): controller = self.vistrail.controller results, changed = controller.execute_workflow_list([[ controller.locator, # locator self.version, # version self.pipeline, # pipeline DummyView(), # view None, # custom_aliases None, # custom_params reason, # reason sinks, # sinks None, # extra_info ]]) result, = results else: pipeline = self.pipeline if inputs: id_scope = IdScope(1) pipeline = pipeline.do_copy(False, id_scope) # A hach to get ids from id_scope that we know won't collide: # make them negative id_scope.getNewId = lambda t, g=id_scope.getNewId: -g(t) create_module = \ VistrailController.create_module_from_descriptor_static create_function = VistrailController.create_function_static create_connection = VistrailController.create_connection_static # Fills in the ExternalPipe ports for module_id, values in inputs.iteritems(): module = pipeline.modules[module_id] if not isinstance(values, (list, tuple)): values = [values] # Guess the type of the InputPort _, sigstrings, _, _, _ = get_port_spec_info( pipeline, module) sigstrings = parse_port_spec_string(sigstrings) # Convert whatever we got to a list of strings, for the # pipeline values = [ reg.convert_port_val(val, sigstring, None) for val, sigstring in izip(values, sigstrings) ] if len(values) == 1: # Create the constant module constant_desc = reg.get_descriptor_by_name( *sigstrings[0]) constant_mod = create_module(id_scope, constant_desc) func = create_function(id_scope, constant_mod, 'value', values) constant_mod.add_function(func) pipeline.add_module(constant_mod) # Connect it to the ExternalPipe port conn = create_connection(id_scope, constant_mod, 'value', module, 'ExternalPipe') pipeline.db_add_connection(conn) else: raise RuntimeError("TODO : create tuple") interpreter = get_default_interpreter() result = interpreter.execute(pipeline, reason=reason, sinks=sinks) if result.errors: raise ExecutionErrors(self, result) else: return ExecutionResults(self, result)
def execute(self, pipeline, **kwargs): """execute(pipeline, **kwargs): kwargs: controller = fetch('controller', None) locator = fetch('locator', None) current_version = fetch('current_version', None) view = fetch('view', DummyView()) aliases = fetch('aliases', None) params = fetch('params', None) extra_info = fetch('extra_info', None) logger = fetch('logger', DummyLogController) reason = fetch('reason', None) actions = fetch('actions', None) done_summon_hooks = fetch('done_summon_hooks', []) module_executed_hook = fetch('module_executed_hook', []) job_monitor = fetch('job_monitor', None) Executes a pipeline using caching. Caching works by reusing pipelines directly. This means that there exists one global pipeline whose parts get executed over and over again. This function returns a triple of dictionaries (objs, errs, execs). objs is a mapping from local ids (the ids in the pipeline) to objects **in the persistent pipeline**. Notice, these are not the objects inside the passed pipeline, but the objects they were mapped to in the persistent pipeline. errs is a dictionary from local ids to error messages of modules that might have returns errors. execs is a dictionary from local ids to boolean values indicating whether they were executed or not. If modules have no error associated with but were not executed, it means they were cached.""" # Setup named arguments. We don't use named parameters so # that positional parameter calls fail earlier new_kwargs = {} def fetch(name, default): new_kwargs[name] = r = kwargs.pop(name, default) return r controller = fetch('controller', None) locator = fetch('locator', None) current_version = fetch('current_version', None) view = fetch('view', DummyView()) vistrail_variables = fetch('vistrail_variables', None) aliases = fetch('aliases', None) params = fetch('params', None) extra_info = fetch('extra_info', None) logger = fetch('logger', DummyLogController) sinks = fetch('sinks', None) reason = fetch('reason', None) actions = fetch('actions', None) done_summon_hooks = fetch('done_summon_hooks', []) module_executed_hook = fetch('module_executed_hook', []) stop_on_error = fetch('stop_on_error', True) parent_exec = fetch('parent_exec', None) job_monitor = fetch('job_monitor', None) if len(kwargs) > 0: raise VistrailsInternalError('Wrong parameters passed ' 'to execute: %s' % kwargs) self.clean_non_cacheable_modules() record_usage(execute=True) # if controller is not None: # vistrail = controller.vistrail # (pipeline, module_remap) = \ # core.db.io.expand_workflow(vistrail, pipeline) # new_kwargs['module_remap'] = module_remap # else: # vistrail = None if controller is not None: vistrail = controller.vistrail else: vistrail = None time_start = time.time() logger = logger.start_workflow_execution(parent_exec, vistrail, pipeline, current_version) new_kwargs['logger'] = logger self.annotate_workflow_execution(logger, reason, aliases, params) res = self.setup_pipeline(pipeline, **new_kwargs) modules_added = res[2] conns_added = res[3] to_delete = res[4] errors = res[5] if len(errors) == 0: res = self.execute_pipeline(pipeline, *(res[:2]), **new_kwargs) else: res = (to_delete, res[0], errors, {}, {}, {}, []) for (i, error) in errors.iteritems(): view.set_module_error(i, error.msg, error.errorTrace) self.finalize_pipeline(pipeline, *(res[:-1]), **new_kwargs) time_end = time.time() result = InstanceObject(objects=res[1], errors=res[2], executed=res[3], suspended=res[4], parameter_changes=res[6], modules_added=modules_added, conns_added=conns_added) logger.finish_workflow_execution(result.errors, suspended=result.suspended) record_usage(time=time_end - time_start, modules=len(res[1]), errors=len(res[2]), executed=len(res[3]), suspended=len(res[4])) return result
def execute_pipeline(self, pipeline, tmp_id_to_module_map, persistent_to_tmp_id_map, **kwargs): def fetch(name, default): return kwargs.pop(name, default) controller = fetch('controller', None) locator = fetch('locator', None) current_version = fetch('current_version', None) view = fetch('view', DummyView()) vistrail_variables = fetch('vistrail_variables', None) aliases = fetch('aliases', None) params = fetch('params', None) extra_info = fetch('extra_info', None) logger = fetch('logger', DummyLogController) sinks = fetch('sinks', None) reason = fetch('reason', None) actions = fetch('actions', None) module_executed_hook = fetch('module_executed_hook', []) done_summon_hooks = fetch('done_summon_hooks', []) clean_pipeline = fetch('clean_pipeline', False) stop_on_error = fetch('stop_on_error', True) parent_exec = fetch('parent_exec', None) job_monitor = fetch('job_monitor', None) if len(kwargs) > 0: raise VistrailsInternalError('Wrong parameters passed ' 'to execute_pipeline: %s' % kwargs) # LOGGING SETUP def get_remapped_id(id): return persistent_to_tmp_id_map[id] logging_obj = ViewUpdatingLogController( logger=logger, view=view, remap_id=get_remapped_id, ids=pipeline.modules.keys(), module_executed_hook=module_executed_hook) # PARAMETER CHANGES SETUP parameter_changes = [] def change_parameter(obj, name, value): parameter_changes.append((get_remapped_id(obj.id), name, value)) def make_change_parameter(obj): return lambda *args: change_parameter(obj, *args) # Update **all** modules in the current pipeline for i, obj in tmp_id_to_module_map.iteritems(): obj.in_pipeline = True # set flag to indicate in pipeline obj.logging = logging_obj obj.change_parameter = make_change_parameter(obj) # Update object pipeline information obj.moduleInfo['locator'] = locator obj.moduleInfo['version'] = current_version obj.moduleInfo['moduleId'] = i obj.moduleInfo['pipeline'] = pipeline obj.moduleInfo['controller'] = controller # extract job monitor from controller if this is the top level if controller: obj.moduleInfo['job_monitor'] = controller.jobMonitor else: obj.moduleInfo['job_monitor'] = job_monitor if extra_info is not None: obj.moduleInfo['extra_info'] = extra_info if reason is not None: obj.moduleInfo['reason'] = reason if actions is not None: obj.moduleInfo['actions'] = actions ## Checking 'sinks' from kwargs to resolve only requested sinks # Note that we accept any module in 'sinks', even if it's not actually # a sink in the graph if sinks is not None: persistent_sinks = [ tmp_id_to_module_map[sink] for sink in sinks if sink in tmp_id_to_module_map ] else: persistent_sinks = [ tmp_id_to_module_map[sink] for sink in pipeline.graph.sinks() ] self._streams.append(Generator.generators) Generator.generators = [] # Update new sinks for obj in persistent_sinks: abort = False try: obj.update() continue except ModuleWasSuspended: continue except ModuleHadError: pass except AbortExecution: break except ModuleSuspended, ms: ms.module.logging.end_update(ms.module, ms, was_suspended=True) continue except ModuleErrors, mes: for me in mes.module_errors: me.module.logging.end_update(me.module, me) logging_obj.signalError(me.module, me) abort = abort or me.abort
def setup_pipeline(self, pipeline, **kwargs): """setup_pipeline(controller, pipeline, locator, currentVersion, view, aliases, **kwargs) Matches a pipeline with the persistent pipeline and creates instances of modules that aren't in the cache. """ def fetch(name, default): return kwargs.pop(name, default) controller = fetch('controller', None) locator = fetch('locator', None) current_version = fetch('current_version', None) view = fetch('view', DummyView()) vistrail_variables = fetch('vistrail_variables', None) aliases = fetch('aliases', None) params = fetch('params', None) extra_info = fetch('extra_info', None) logger = fetch('logger', DummyLogController) sinks = fetch('sinks', None) reason = fetch('reason', None) actions = fetch('actions', None) done_summon_hooks = fetch('done_summon_hooks', []) module_executed_hook = fetch('module_executed_hook', []) stop_on_error = fetch('stop_on_error', True) parent_exec = fetch('parent_exec', None) job_monitor = fetch('job_monitor', None) reg = get_module_registry() if len(kwargs) > 0: raise VistrailsInternalError('Wrong parameters passed ' 'to setup_pipeline: %s' % kwargs) def create_null(): """Creates a Null value""" getter = reg.get_descriptor_by_name descriptor = getter(basic_pkg, 'Null') return descriptor.module() def create_constant(param, module): """Creates a Constant from a parameter spec""" getter = reg.get_descriptor_by_name desc = getter(param.identifier, param.type, param.namespace) constant = desc.module() constant.id = module.id # if param.evaluatedStrValue: # constant.setValue(param.evaluatedStrValue) if param.strValue != '': constant.setValue(param.strValue) else: constant.setValue( constant.translate_to_string(constant.default_value)) return constant ### BEGIN METHOD ### # if self.debugger: # self.debugger.update() to_delete = [] errors = {} if controller is not None: # Controller is none for sub_modules controller.validate(pipeline) else: pipeline.validate() self.resolve_aliases(pipeline, aliases) if vistrail_variables: self.resolve_variables(vistrail_variables, pipeline) self.update_params(pipeline, params) (tmp_to_persistent_module_map, conn_map, module_added_set, conn_added_set) = self.add_to_persistent_pipeline(pipeline) # Create the new objects for i in module_added_set: persistent_id = tmp_to_persistent_module_map[i] module = self._persistent_pipeline.modules[persistent_id] obj = self._objects[persistent_id] = module.summon() obj.interpreter = self obj.id = persistent_id obj.signature = module._signature # Checking if output should be stored if module.has_annotation_with_key('annotate_output'): annotate_output = module.get_annotation_by_key( 'annotate_output') #print annotate_output if annotate_output: obj.annotate_output = True for f in module.functions: connector = None if len(f.params) == 0: connector = ModuleConnector(create_null(), 'value', f.get_spec('output')) elif len(f.params) == 1: p = f.params[0] try: constant = create_constant(p, module) connector = ModuleConnector(constant, 'value', f.get_spec('output')) except Exception, e: debug.unexpected_exception(e) err = ModuleError( module, "Uncaught exception creating Constant from " "%r: %s" % (p.strValue, debug.format_exception(e))) errors[i] = err to_delete.append(obj.id) else: tupleModule = vistrails.core.interpreter.base.InternalTuple( ) tupleModule.length = len(f.params) for (j, p) in enumerate(f.params): try: constant = create_constant(p, module) constant.update() connector = ModuleConnector( constant, 'value', f.get_spec('output')) tupleModule.set_input_port(j, connector) except Exception, e: debug.unexpected_exception(e) err = ModuleError( module, "Uncaught exception creating Constant " "from %r: %s" % (p.strValue, debug.format_exception(e))) errors[i] = err to_delete.append(obj.id) connector = ModuleConnector(tupleModule, 'value', f.get_spec('output')) if connector: obj.set_input_port(f.name, connector, is_method=True)
def execute(modules, connections=[], add_port_specs=[], enable_pkg=True, full_results=False): """Build a pipeline and execute it. This is useful to simply build a pipeline in a test case, and run it. When doing that, intercept_result() can be used to check the results of each module. modules is a list of module tuples describing the modules to be created, with the following format: [('ModuleName', 'package.identifier', [ # Functions ('port_name', [ # Function parameters ('Signature', 'value-as-string'), ]), ])] connections is a list of tuples describing the connections to make, with the following format: [ (source_module_index, 'source_port_name', dest_module_index, 'dest_module_name'), ] add_port_specs is a list of specs to add to modules, with the following format: [ (mod_id, 'input'/'output', 'portname', '(port_sig)'), ] It is useful to test modules that can have custom ports through a configuration widget. The function returns the 'errors' dict it gets from the interpreter, so you should use a construct like self.assertFalse(execute(...)) if the execution is not supposed to fail. For example, this creates (and runs) an Integer module with its value set to 44, connected to a PythonCalc module, connected to a StandardOutput: self.assertFalse(execute([ ('Float', 'org.vistrails.vistrails.basic', [ ('value', [('Float', '44.0')]), ]), ('PythonCalc', 'org.vistrails.vistrails.pythoncalc', [ ('value2', [('Float', '2.0')]), ('op', [('String', '-')]), ]), ('StandardOutput', 'org.vistrails.vistrails.basic', []), ], [ (0, 'value', 1, 'value1'), (1, 'value', 2, 'value'), ])) """ from vistrails.core.db.locator import XMLFileLocator from vistrails.core.modules.module_registry import MissingPackage from vistrails.core.packagemanager import get_package_manager from vistrails.core.utils import DummyView from vistrails.core.vistrail.connection import Connection from vistrails.core.vistrail.module import Module from vistrails.core.vistrail.module_function import ModuleFunction from vistrails.core.vistrail.module_param import ModuleParam from vistrails.core.vistrail.pipeline import Pipeline from vistrails.core.vistrail.port import Port from vistrails.core.vistrail.port_spec import PortSpec from vistrails.core.interpreter.noncached import Interpreter pm = get_package_manager() port_spec_per_module = {} # mod_id -> [portspec: PortSpec] j = 0 for i, (mod_id, inout, name, sig) in enumerate(add_port_specs): mod_specs = port_spec_per_module.setdefault(mod_id, []) ps = PortSpec(id=i, name=name, type=inout, sigstring=sig, sort_key=-1) for psi in ps.port_spec_items: psi.id = j j += 1 mod_specs.append(ps) pipeline = Pipeline() module_list = [] for i, (name, identifier, functions) in enumerate(modules): function_list = [] try: pkg = pm.get_package(identifier) except MissingPackage: if not enable_pkg: raise dep_graph = pm.build_dependency_graph([identifier]) for pkg_id in pm.get_ordered_dependencies(dep_graph): pkg = pm.identifier_is_available(pkg_id) if pkg is None: raise pm.late_enable_package(pkg.codepath) pkg = pm.get_package(identifier) for func_name, params in functions: param_list = [] for j, (param_type, param_val) in enumerate(params): param_list.append( ModuleParam(pos=j, type=param_type, val=param_val)) function_list.append( ModuleFunction(name=func_name, parameters=param_list)) name = name.rsplit('|', 1) if len(name) == 2: namespace, name = name else: namespace = None name, = name module = Module(name=name, namespace=namespace, package=identifier, version=pkg.version, id=i, functions=function_list) for port_spec in port_spec_per_module.get(i, []): module.add_port_spec(port_spec) pipeline.add_module(module) module_list.append(module) for i, (sid, sport, did, dport) in enumerate(connections): s_sig = module_list[sid].get_port_spec(sport, 'output').sigstring d_sig = module_list[did].get_port_spec(dport, 'input').sigstring pipeline.add_connection( Connection(id=i, ports=[ Port(id=i * 2, type='source', moduleId=sid, name=sport, signature=s_sig), Port(id=i * 2 + 1, type='destination', moduleId=did, name=dport, signature=d_sig), ])) interpreter = Interpreter.get() result = interpreter.execute(pipeline, locator=XMLFileLocator('foo.xml'), current_version=1, view=DummyView()) if full_results: return result else: # Allows to do self.assertFalse(execute(...)) return result.errors