def createMashupApp(vtcontroller, mashuptrail, version): from gui.mashups.mashup_app import QMashupAppMainWindow vistrail_view = vtcontroller.vistrail_view vtVersion = mashuptrail.vtVersion view = DummyView() view.scene().current_pipeline = vtcontroller.vistrail.getPipeline(vtVersion) view.scene().current_pipeline.validate() new_vtcontroller = MashupsManager.copyBaseVistrailController(vtcontroller) new_vtcontroller.change_selected_version(vtVersion) mshpController = MashupController(vtcontroller, new_vtcontroller, vtVersion, mashuptrail) mshpController.setCurrentVersion(version) app = QMashupAppMainWindow(parent=None, vistrail_view=vistrail_view, controller=mshpController, version=version) return app
def finalize_pipeline(self, pipeline, to_delete, objs, errs, execs, suspended, cached, **kwargs): def fetch(name, default): r = kwargs.get(name, default) try: del kwargs[name] except KeyError: pass return r view = fetch('view', DummyView()) reset_computed = fetch('reset_computed', True) self.clean_modules(to_delete) for i in objs: if i in errs: view.set_module_error(i, errs[i].msg, errs[i].errorTrace) elif i in suspended and suspended[i]: view.set_module_suspended(i, suspended[i]) elif i in execs and execs[i]: view.set_module_success(i) elif i in cached and cached[i]: view.set_module_not_executed(i) else: view.set_module_persistent(i) if reset_computed: for module in self._objects.itervalues(): module.computed = False
def testIncorrectURL_2(self): import core.vistrail from core.db.locator import XMLFileLocator from core.vistrail.module import Module from core.vistrail.module_function import ModuleFunction from core.vistrail.module_param import ModuleParam import core.interpreter p = core.vistrail.pipeline.Pipeline() m_param = ModuleParam( type='String', val='http://neitherodesthisohrly', ) m_function = ModuleFunction( name='url', parameters=[m_param], ) p.add_module( Module( name='HTTPFile', package=identifier, version=version, id=0, functions=[m_function], )) interpreter = core.interpreter.default.get_default_interpreter() kwargs = { 'locator': XMLFileLocator('foo'), 'current_version': 1L, 'view': DummyView(), }
def copyVistrailController(vt_controller, view=DummyView()): newvt_controller = VistrailController() current_log = vt_controller.log vistrail = vt_controller.vistrail newvt_controller.log = current_log newvt_controller.current_pipeline_view = view.scene() newvt_controller.set_vistrail(vistrail, None) newvt_controller.disable_autosave() return newvt_controller
def createMashupApp(vtcontroller, mashuptrail, version): from gui.mashups.mashup_app import QMashupAppMainWindow vistrail_view = vtcontroller.vistrail_view vtVersion = mashuptrail.vtVersion view = DummyView() view.scene().current_pipeline = vtcontroller.vistrail.getPipeline( vtVersion) view.scene().current_pipeline.validate() new_vtcontroller = MashupsManager.copyBaseVistrailController( vtcontroller) new_vtcontroller.change_selected_version(vtVersion) mshpController = MashupController(vtcontroller, new_vtcontroller, vtVersion, mashuptrail) mshpController.setCurrentVersion(version) app = QMashupAppMainWindow(parent=None, vistrail_view=vistrail_view, controller=mshpController, version=version) return app
def execute(self): """ execute() -> None Execute the selected pipeline with the edited aliases """ aliases = {} for r in xrange(self.aliasTable.rowCount()): name = str(self.aliasTable.item(r, 0).text()) value = str(self.aliasTable.item(r, 1).text()) aliases[name] = value versionNumber = self.versions[self.pipelineList.currentIndex().row()][0] pipeline = self.vistrail.getPipelineVersionNumber(versionNumber) controller = VistrailController(self.vistrail) controller.execute_workflow_list([(self.vistrail.locator, versionNumber, pipeline, DummyView(), aliases, None)])
def test_tuple(self): from core.vistrail.module_param import ModuleParam from core.vistrail.module_function import ModuleFunction from core.utils import DummyView from core.vistrail.module import Module import db.domain id_scope = db.domain.IdScope() interpreter = core.interpreter.default.get_default_interpreter() v = DummyView() p = core.vistrail.pipeline.Pipeline() params = [ ModuleParam( id=id_scope.getNewId(ModuleParam.vtType), pos=0, type='Float', val='2.0', ), ModuleParam( id=id_scope.getNewId(ModuleParam.vtType), pos=1, type='Float', val='2.0', ) ] function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType), name='input') function.add_parameters(params) module = Module(id=id_scope.getNewId(Module.vtType), name='TestTupleExecution', package='edu.utah.sci.vistrails.console_mode_test', version='0.9.0') module.add_function(function) p.add_module(module) kwargs = { 'locator': XMLFileLocator('foo'), 'current_version': 1L, 'view': v, }
def executePipeline(self, pipeline): from core.db.io import unserialize from core.vistrail.pipeline import Pipeline from core.interpreter.default import get_default_interpreter as getDefaultInterpreter from core.utils import DummyView import api tabController = self.spreadsheetWindow.get_current_tab_controller() pip = unserialize(str(pipeline), Pipeline) # print " **** Client-%s ---Received Pipeline--- modules:" % str( self.dimensions ) # for module in pip.module_list: # print " ", str(module.id) self.current_pipeline = pip interpreter = getDefaultInterpreter() kwargs = { "locator": None, "current_version": None, "view": DummyView(), "aliases": {} } interpreter.execute(pip, **kwargs) print "Finished Executing Pipeline"
def executePipelineWithProgress(pipeline, pTitle='Pipeline Execution', pCaption='Executing...', pCancel='&Cancel', **kwargs): """ executePipelineWithProgress(pipeline: Pipeline, pTitle: str, pCaption: str, pCancel: str, kwargs: keyword arguments) -> bool Execute the pipeline while showing a progress dialog with title pTitle, caption pCaption and the cancel button text pCancel. kwargs is the keyword arguments that will be passed to the interpreter. A bool will be returned indicating if the execution was performed without cancel or not. """ withoutCancel = True totalProgress = len(pipeline.modules) progress = QtGui.QProgressDialog(pCaption, pCancel, 0, totalProgress) progress.setWindowTitle(pTitle) progress.setWindowModality(QtCore.Qt.WindowModal) progress.show() def moduleExecuted(objId): if not progress.wasCanceled(): progress.setValue(progress.value() + 1) QtCore.QCoreApplication.processEvents() else: withoutCancel = False interpreter = get_default_interpreter() if kwargs.has_key('module_executed_hook'): kwargs['module_executed_hook'].append(moduleExecuted) else: kwargs['module_executed_hook'] = [moduleExecuted] kwargs['view'] = DummyView() interpreter.execute(pipeline, **kwargs) progress.setValue(totalProgress) return withoutCancel
def test_cache(self): from core.db.locator import XMLFileLocator from core.vistrail.controller import VistrailController from core.db.io import load_vistrail """Test if basic caching is working.""" locator = XMLFileLocator(core.system.vistrails_root_directory() + '/tests/resources/dummy.xml') (v, abstractions, thumbnails, mashups) = load_vistrail(locator) # the controller will take care of upgrades controller = VistrailController() controller.set_vistrail(v, locator, abstractions, thumbnails, mashups) p1 = v.getPipeline('int chain') n = v.get_version_number('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p1 = controller.current_pipeline view = DummyView() interpreter = core.interpreter.cached.CachedInterpreter.get() result = interpreter.execute(p1, locator=v, current_version=n, view=view, ) # to force fresh params p2 = v.getPipeline('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p2 = controller.current_pipeline result = interpreter.execute(p2, locator=v, current_version=n, view=view, ) assert len(result.modules_added) == 1
def createMashupController(self, vt_controller, version, view=DummyView()): #print "Manager creating mashup controller ", vt_controller, version newvt_controller = MashupsManager.copyVistrailController( vt_controller, view) mashuptrail = \ MashupsManager.getMashuptrailforVersionInVistrailController(vt_controller, version) if mashuptrail is None: (p_mashuptrail, p_version) = \ MashupsManager.findClosestParentMashuptrail(vt_controller, version) id_scope = IdScope(1L) if p_mashuptrail is not None: version_name = vt_controller.get_pipeline_name(p_version) (res, mshpv) = MashupsManager.showFoundMashupsDialog( p_mashuptrail, version_name) if res in ['Copy', 'Move']: pipeline = newvt_controller.vistrail.getPipeline(version) if res == 'Copy': # we will copy the mashup from the parent trail and # validate it to the current pipeline before adding # to the current mashup trail mashuptrail = Mashuptrail(self.getNewMashuptrailId(), version, id_scope) p_mashup = p_mashuptrail.getMashup(mshpv) mashup = p_mashup.doCopy() mashup.id_scope = id_scope mashup.version = version mashup.validateForPipeline(pipeline) currVersion = mashuptrail.addVersion( parent_id=mashuptrail.getLatestVersion(), mashup=mashup, user=core.system.current_user(), date=core.system.current_time()) mashuptrail.currentVersion = currVersion mashuptrail.updateIdScope() p_tag = p_mashuptrail.getTagForActionId(mshpv) if p_tag == '': tag = "<latest>" tag = "Copy from %s" % p_tag MashupsManager.addMashuptrailtoVistrailController( vt_controller, mashuptrail) elif res == 'Move': # we will move the parent trail and validate all mashups # for the current pipeline to make sure they will be # executable for the current version mashuptrail = p_mashuptrail currVersion = mashuptrail.getLatestVersion() mashuptrail.currentVersion = currVersion mashuptrail.validateMashupsForPipeline( version, pipeline) tag = None mashuptrail.vtVersion = version mshpController = MashupController(vt_controller, newvt_controller, version, mashuptrail) mshpController.setCurrentVersion( mashuptrail.currentVersion) # this is to make sure the pipeline displayed in the mashup # view is consistent with the list of aliases in the central # panel mshpController.updatePipelineAliasesFromCurrentMashup() if tag is not None: mshpController.updateCurrentTag(tag) return mshpController mashuptrail = Mashuptrail(self.getNewMashuptrailId(), version, id_scope) pipeline = newvt_controller.vistrail.getPipeline(version) id = id_scope.getNewId('mashup') mashup = Mashup(id=id, name="mashup%s" % id, vtid=vt_controller.locator, version=version) mashup.loadAliasesFromPipeline(pipeline, id_scope) currVersion = mashuptrail.addVersion( parent_id=mashuptrail.getLatestVersion(), mashup=mashup, user=core.system.current_user(), date=core.system.current_time()) mashuptrail.currentVersion = currVersion MashupsManager.addMashuptrailtoVistrailController( vt_controller, mashuptrail) mshpController = MashupController(vt_controller, newvt_controller, version, mashuptrail) mshpController.setCurrentVersion(mashuptrail.currentVersion) if mshpController.currentVersion == 1L: mshpController.updateCurrentTag("ROOT") else: #print "----> found mashuptrail ", mashuptrail.currentVersion mshpController = MashupController(vt_controller, newvt_controller, version, mashuptrail) mshpController.setCurrentVersion(mashuptrail.currentVersion) mshpController.updatePipelineAliasesFromCurrentMashup() return mshpController
def execute(self, pipeline, **kwargs): """execute(pipeline, **kwargs): kwargs: controller = fetch('controller', None) locator = fetch('locator', None) current_version = fetch('current_version', None) view = fetch('view', DummyView()) aliases = fetch('aliases', None) params = fetch('params', None) extra_info = fetch('extra_info', None) logger = fetch('logger', DummyLogController()) reason = fetch('reason', None) actions = fetch('actions', None) done_summon_hooks = fetch('done_summon_hooks', []) module_executed_hook = fetch('module_executed_hook', []) Executes a pipeline using caching. Caching works by reusing pipelines directly. This means that there exists one global pipeline whose parts get executed over and over again. This function returns a triple of dictionaries (objs, errs, execs). objs is a mapping from local ids (the ids in the pipeline) to objects **in the persistent pipeline**. Notice, these are not the objects inside the passed pipeline, but the objects they were mapped to in the persistent pipeline. errs is a dictionary from local ids to error messages of modules that might have returns errors. execs is a dictionary from local ids to boolean values indicating whether they were executed or not. If modules have no error associated with but were not executed, it means they were cached.""" # Setup named arguments. We don't use named parameters so # that positional parameter calls fail earlier new_kwargs = {} def fetch(name, default): r = kwargs.get(name, default) new_kwargs[name] = r try: del kwargs[name] except KeyError: pass return r controller = fetch('controller', None) locator = fetch('locator', None) current_version = fetch('current_version', None) view = fetch('view', DummyView()) aliases = fetch('aliases', None) params = fetch('params', None) extra_info = fetch('extra_info', None) logger = fetch('logger', DummyLogController()) sinks = fetch('sinks', None) reason = fetch('reason', None) actions = fetch('actions', None) done_summon_hooks = fetch('done_summon_hooks', []) module_executed_hook = fetch('module_executed_hook', []) if len(kwargs) > 0: raise VistrailsInternalError('Wrong parameters passed ' 'to execute: %s' % kwargs) self.clean_non_cacheable_modules() # if controller is not None: # vistrail = controller.vistrail # (pipeline, module_remap) = \ # core.db.io.expand_workflow(vistrail, pipeline) # new_kwargs['module_remap'] = module_remap # else: # vistrail = None if controller is not None: vistrail = controller.vistrail else: vistrail = None self.parent_execs = [None] logger.start_workflow_execution(vistrail, pipeline, current_version) self.annotate_workflow_execution(logger, reason, aliases, params) result = self.unlocked_execute(pipeline, **new_kwargs) logger.finish_workflow_execution(result.errors, suspended=result.suspended) self.parent_execs = [None] return result
def execute_pipeline(self, pipeline, tmp_id_to_module_map, persistent_to_tmp_id_map, **kwargs): def fetch(name, default): r = kwargs.get(name, default) try: del kwargs[name] except KeyError: pass return r controller = fetch('controller', None) locator = fetch('locator', None) current_version = fetch('current_version', None) view = fetch('view', DummyView()) aliases = fetch('aliases', None) params = fetch('params', None) extra_info = fetch('extra_info', None) logger = fetch('logger', DummyLogController()) sinks = fetch('sinks', None) reason = fetch('reason', None) actions = fetch('actions', None) module_executed_hook = fetch('module_executed_hook', []) module_suspended_hook = fetch('module_suspended_hook', []) done_summon_hooks = fetch('done_summon_hooks', []) clean_pipeline = fetch('clean_pipeline', False) # parent_exec = fetch('parent_exec', None) if len(kwargs) > 0: raise VistrailsInternalError('Wrong parameters passed ' 'to execute_pipeline: %s' % kwargs) errors = {} executed = {} suspended = {} cached = {} # LOGGING SETUP def get_remapped_id(id): return persistent_to_tmp_id_map[id] # the executed dict works on persistent ids def add_to_executed(obj): executed[obj.id] = True for callable_ in module_executed_hook: callable_(obj.id) # the suspended dict works on persistent ids def add_to_suspended(obj): suspended[obj.id] = obj.suspended for callable_ in module_suspended_hook: callable_(obj.id) # views work on local ids def begin_compute(obj): i = get_remapped_id(obj.id) view.set_module_computing(i) reg = modules.module_registry.get_module_registry() module_name = reg.get_descriptor(obj.__class__).name # !!!self.parent_execs is mutated!!! logger.start_execution(obj, i, module_name, parent_execs=self.parent_execs) # views and loggers work on local ids def begin_update(obj): i = get_remapped_id(obj.id) view.set_module_active(i) def update_cached(obj): cached[obj.id] = True i = get_remapped_id(obj.id) reg = modules.module_registry.get_module_registry() module_name = reg.get_descriptor(obj.__class__).name # !!!self.parent_execs is mutated!!! logger.start_execution(obj, i, module_name, parent_execs=self.parent_execs, cached=1) view.set_module_not_executed(i) num_pops = logger.finish_execution(obj,'', self.parent_execs) # views and loggers work on local ids def end_update(obj, error='', errorTrace=None, was_suspended = False): i = get_remapped_id(obj.id) if was_suspended: view.set_module_suspended(i, error) elif not error: view.set_module_success(i) else: view.set_module_error(i, error) # !!!self.parent_execs is mutated!!! logger.finish_execution(obj, error, self.parent_execs, errorTrace, was_suspended) # views and loggers work on local ids def annotate(obj, d): i = get_remapped_id(obj.id) logger.insert_module_annotations(obj, d) # views and loggers work on local ids def update_progress(obj, percentage=0.0): i = get_remapped_id(obj.id) view.set_module_progress(i, percentage) logging_obj = InstanceObject(signalSuccess=add_to_executed, signalSuspended=add_to_suspended, begin_update=begin_update, begin_compute=begin_compute, update_progress=update_progress, end_update=end_update, update_cached=update_cached, annotate=annotate, log=logger) # PARAMETER CHANGES SETUP parameter_changes = [] def change_parameter(obj, name, value): parameter_changes.append((get_remapped_id(obj.id), name, value)) def make_change_parameter(obj): return lambda *args: change_parameter(obj, *args) # Update **all** modules in the current pipeline for i, obj in tmp_id_to_module_map.iteritems(): obj.logging = logging_obj obj.change_parameter = make_change_parameter(obj) # Update object pipeline information obj.moduleInfo['controller'] = controller obj.moduleInfo['locator'] = locator obj.moduleInfo['version'] = current_version obj.moduleInfo['moduleId'] = i obj.moduleInfo['pipeline'] = pipeline if extra_info is not None: obj.moduleInfo['extra_info'] = extra_info if reason is not None: obj.moduleInfo['reason'] = reason if actions is not None: obj.moduleInfo['actions'] = actions ## Checking 'sinks' from kwargs to resolve only requested sinks if sinks is not None: requestedSinks = sinks persistent_sinks = [tmp_id_to_module_map[sink] for sink in pipeline.graph.sinks() if sink in requestedSinks] else: persistent_sinks = [tmp_id_to_module_map[sink] for sink in pipeline.graph.sinks()] # Update new sinks for obj in persistent_sinks: try: obj.update() except ModuleErrors, mes: for me in mes.module_errors: me.module.logging.end_update(me.module, me.msg) errors[me.module.id] = me break except ModuleError, me: me.module.logging.end_update(me.module, me.msg, me.errorTrace) errors[me.module.id] = me break
def setup_pipeline(self, pipeline, **kwargs): """setup_pipeline(controller, pipeline, locator, currentVersion, view, aliases, **kwargs) Matches a pipeline with the persistent pipeline and creates instances of modules that aren't in the cache. """ def fetch(name, default): r = kwargs.get(name, default) try: del kwargs[name] except KeyError: pass return r controller = fetch('controller', None) locator = fetch('locator', None) current_version = fetch('current_version', None) view = fetch('view', DummyView()) aliases = fetch('aliases', None) params = fetch('params', None) extra_info = fetch('extra_info', None) logger = fetch('logger', DummyLogController()) sinks = fetch('sinks', None) reason = fetch('reason', None) actions = fetch('actions', None) done_summon_hooks = fetch('done_summon_hooks', []) module_executed_hook = fetch('module_executed_hook', []) if len(kwargs) > 0: raise VistrailsInternalError('Wrong parameters passed ' 'to setup_pipeline: %s' % kwargs) def create_null(): """Creates a Null value""" getter = modules.module_registry.registry.get_descriptor_by_name descriptor = getter('edu.utah.sci.vistrails.basic', 'Null') return descriptor.module() def create_constant(param, module): """Creates a Constant from a parameter spec""" reg = modules.module_registry.get_module_registry() getter = reg.get_descriptor_by_name desc = getter(param.identifier, param.type, param.namespace) constant = desc.module() constant.id = module.id # if param.evaluatedStrValue: # constant.setValue(param.evaluatedStrValue) if param.strValue != '': constant.setValue(param.strValue) else: constant.setValue( \ constant.translate_to_string(constant.default_value)) return constant ### BEGIN METHOD ### # if self.debugger: # self.debugger.update() to_delete = [] errors = {} if controller is not None: # Controller is none for sub_modules controller.validate(pipeline) else: pipeline.validate() self.resolve_aliases(pipeline, aliases) if controller is not None: # Controller is none for sub_modules, so we can't resolve variables self.resolve_variables(controller, pipeline) self.update_params(pipeline, params) (tmp_to_persistent_module_map, conn_map, module_added_set, conn_added_set) = self.add_to_persistent_pipeline(pipeline) # Create the new objects for i in module_added_set: persistent_id = tmp_to_persistent_module_map[i] module = self._persistent_pipeline.modules[persistent_id] self._objects[persistent_id] = module.summon() obj = self._objects[persistent_id] obj.interpreter = self obj.id = persistent_id obj.is_breakpoint = module.is_breakpoint obj.signature = module._signature reg = modules.module_registry.get_module_registry() for f in module.functions: connector = None if len(f.params) == 0: connector = ModuleConnector(create_null(), 'value') elif len(f.params) == 1: p = f.params[0] try: constant = create_constant(p, module) connector = ModuleConnector(constant, 'value') except ValueError, e: err = ModuleError(self, 'Cannot convert parameter ' 'value "%s"\n' % p.strValue + str(e)) errors[i] = err to_delete.append(obj.id) except Exception, e: import traceback traceback.print_exc() err = ModuleError(self, 'Uncaught exception: "%s"' % str(e) ) errors[i] = err to_delete.append(obj.id) else: tupleModule = core.interpreter.base.InternalTuple() tupleModule.length = len(f.params) if f.name == 'levelRangeScale': pass for (j,p) in enumerate(f.params): try: constant = create_constant(p, module) constant.update() connector = ModuleConnector(constant, 'value') tupleModule.set_input_port(j, connector) except ValueError, e: err = ModuleError(self, "Cannot convert parameter " "value '%s'\n" % p.strValue + \ str(e)) errors[i] = err to_delete.append(obj.id) except Exception, e: import traceback traceback.print_exc() err = ModuleError(self, 'Uncaught exception: "%s"' % str(e)) errors[i] = err to_delete.append(obj.id)
def openSpreadsheet(self, fileName): """ openSpreadsheet(fileName: str) -> None Open a saved spreadsheet assuming that all VTK files must exist and have all the version using the saved spreadsheet """ def parse_locator(text): locator = None wrapper = XMLWrapper() dom = wrapper.create_document_from_string(text) root = dom.documentElement version = None version = root.getAttribute('version') if version == '1.0': for element in named_elements(root, 'locator'): if str(element.getAttribute('type')) == 'file': locator = FileLocator.parse(element) elif str(element.getAttribute('type')) == 'db': locator = DBLocator.parse(element) return locator locators = {} indexFile = open(fileName, 'r') contents = indexFile.read() self.clearTabs() lidx = 0 lines = contents.split('\n') tabCount = int(lines[lidx]) lidx += 1 for tabIdx in xrange(tabCount): # FIXME: eval should pretty much never be used tabInfo = eval(lines[lidx]) lidx += 1 sheet = spreadsheetRegistry.getSheet(tabInfo[1])(self) sheet.setDimension(tabInfo[2], tabInfo[3]) self.addTabWidget(sheet, tabInfo[0]) while lines[lidx] != '---': (r, c, vistrail, pid, cid) = eval(lines[lidx]) locator = vistrail['locator'] if locators.has_key(locator): vistrail['locator'] = locators[locator] else: locators[locator] = parse_locator(vistrail['locator']) vistrail['locator'] = locators[locator] self.appendMonitoredLocations((vistrail, pid, cid), (sheet, r, c)) lidx += 1 lidx += 1 pipelineCount = int(lines[lidx]) lidx += 1 self.loadingMode = True progress = QtGui.QProgressDialog("Loading spreadsheet...", "&Cancel", 0, pipelineCount, self, QtCore.Qt.WindowStaysOnTopHint) progress.show() for pipelineIdx in xrange(pipelineCount): # FIXME: eval should pretty much never be used (serializedLocator, version) = eval(lines[lidx]) try: locator = locators[serializedLocator] except KeyError: locator = parse_locator(serializedLocator) if locator: bundle = locator.load() if isinstance(bundle, SaveBundle): pipeline = bundle.vistrail.getPipeline(version) else: pipeline = bundle.getPipeline(version) execution = get_default_interpreter() progress.setValue(pipelineIdx) QtCore.QCoreApplication.processEvents() if progress.wasCanceled(): break kwargs = { 'locator': locator, 'current_version': version, 'view': DummyView(), } execution.execute(pipeline, **kwargs) else: raise Exception("Couldn't load spreadsheet") lidx += 1 progress.setValue(pipelineCount) QtCore.QCoreApplication.processEvents() self.changeSpreadsheetFileName(fileName) self.loadingMode = False indexFile.close()