def get_upgraded_pipeline(vistrail, version=None): """This is similar to Vistrail#getPipeline() but performs upgrades. getPipeline() can fail if the original pipeline has a different version. In contrast, this function will update the pipeline first using a controller. """ if version is None: version = vistrail.get_latest_version() elif isinstance(version, (int, long)): pass elif isinstance(version, basestring): version = vistrail.get_tag_str(version).action_id else: raise TypeError controller = VistrailController(vistrail) controller.recompute_terse_graph() # FIXME : this shouldn't be needed... controller.do_version_switch(version) return controller.current_pipeline
def test_infinite_looping_upgrade(self): """Test that circular upgrades fail gracefully""" # Expected actions are as follow: # - loads workflow2.xml # * pipeline is missing looping_fix.x version 0.1 # - enables looping_fix.x (version 0.2) # * pipeline is still missing looping_fix.x version 0.1 # - runs upgrade for looping_fix.x, 0.1 -> 0.2 # - upgrade changes modules to package looping_fix.y version 0.1 # * pipeline is missing looping_fix.y version 0.1 # - enables looping_fix.y (version 0.2) # * pipeline is still missing looping_fix.y version 0.1 # Loop 50 times: # - runs upgrade for looping_fix.y, 0.1 -> 0.2 # - upgrade changes modules to package looping_fix.x version 0.1 # * pipeline is missing looping_fix.x version 0.1 # - runs upgrade for looping_fix.x, 0.1 -> 0.2 # - upgrade changes modules to package looping_fix.y version 0.1 # * pipeline is missing looping_fix.y version 0.1 # 50 calls to handle_invalid_pipeline() # Pre-adds packages so that the package manager can find them packages = ["pkg_x", "pkg_y"] prefix = "vistrails.tests.resources.looping_upgrades." pm = get_package_manager() for pkg in packages: pm.get_available_package(pkg, prefix=prefix) # Hooks handle_invalid_pipeline() from vistrails.core.vistrail.controller import VistrailController orig_hip = VistrailController.handle_invalid_pipeline count = [0] def new_hip(*args, **kwargs): count[0] += 1 return orig_hip(*args, **kwargs) VistrailController.handle_invalid_pipeline = new_hip try: # Loads workflow.xml from vistrails.core.db.io import load_vistrail from vistrails.core.db.locator import FileLocator from vistrails.core.system import vistrails_root_directory locator = FileLocator( os.path.join(vistrails_root_directory(), "tests", "resources", "looping_upgrades", "workflow2.xml") ) loaded_objs = load_vistrail(locator) controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:]) # Select version (triggers all the validation/upgrade/loading) self.assertEqual(controller.get_latest_version_in_graph(), 1) try: controller.do_version_switch(1) except InvalidPipeline: pass else: self.fail("No InvalidPipeline exception raised!") # Restores handle_invalid_pipeline() finally: VistrailController.handle_invalid_pipeline = orig_hip # disable packages for pkg in reversed(packages): try: pm.late_disable_package(pkg) except MissingPackage: pass # make sure it looped 50 times before failing max_loops = getattr(get_vistrails_configuration(), "maxPipelineFixAttempts", 50) self.assertEqual(count[0], max_loops) # Check that original version gets selected self.assertEqual(1, controller.current_version)
def test_looping_pipeline_fix(self): """Chains upgrades and automatic package initialization.""" # Expected actions are as follow: # - loads workflow.xml # * pipeline is missing looping_fix.a version 0.1 # - enables looping_fix.a (version 0.2) # * pipeline is still missing looping_fix.a version 0.1 # - runs upgrade for looping_fix.a, 0.1 -> 0.2 # - upgrade changes modules to package looping_fix.b version 0.1 # * pipeline is missing looping_fix.b version 0.1 # - enables looping_fix.b (version 0.2) # * pipeline is still missing looping_fix.b version 0.1 # - runs upgrade for looping_fix.b, 0.1 -> 0.2 # - upgrade changes modules to package looping_fix.c version 1.0 # * pipeline is missing looping_fix.c version 1.0 # - enables looping_fix.c (version 1.0) # * pipeline is valid # 5 calls to handle_invalid_pipeline() # Pre-adds packages so that the package manager can find them packages = ["pkg_a", "pkg_b", "pkg_c"] prefix = "vistrails.tests.resources.looping_upgrades." pm = get_package_manager() for pkg in packages: pm.get_available_package(pkg, prefix=prefix) self.assertFalse(set(pkg.codepath for pkg in pm.enabled_package_list()).intersection(packages)) # Hooks handle_invalid_pipeline() from vistrails.core.vistrail.controller import VistrailController orig_hip = VistrailController.handle_invalid_pipeline count = [0] def new_hip(*args, **kwargs): count[0] += 1 return orig_hip(*args, **kwargs) VistrailController.handle_invalid_pipeline = new_hip try: # Loads workflow.xml from vistrails.core.db.io import load_vistrail from vistrails.core.db.locator import FileLocator from vistrails.core.system import vistrails_root_directory locator = FileLocator( os.path.join(vistrails_root_directory(), "tests", "resources", "looping_upgrades", "workflow.xml") ) loaded_objs = load_vistrail(locator) controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:]) # Select version (triggers all the validation/upgrade/loading) self.assertEqual(controller.get_latest_version_in_graph(), 1) controller.do_version_switch(1) self.assertEqual(count[0], 5) # Restores handle_invalid_pipeline() finally: VistrailController.handle_invalid_pipeline = orig_hip # disable packages for pkg in reversed(packages): try: pm.late_disable_package(pkg) except MissingPackage: pass
class Vistrail(object): """This class wraps both Vistrail and VistrailController. From it, you can get any pipeline from a tag name or version number. It has a concept of "current version", from which you can create new versions by performing actions. """ _current_pipeline = None _html = None def __init__(self, arg=None): initialize() if arg is None: # Copied from VistrailsApplicationInterface#open_vistrail() locator = UntitledLocator() loaded_objs = vistrails.core.db.io.load_vistrail(locator) self.controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:]) elif isinstance(arg, (_Pipeline, Pipeline)): if isinstance(arg, Pipeline): pipeline = arg.pipeline else: pipeline = arg # Copied from VistrailsApplicationInterface#open_workflow() vistrail = _Vistrail() ops = [] for module in pipeline.module_list: ops.append(('add', module)) for connection in pipeline.connection_list: ops.append(('add', connection)) action = vistrails.core.db.action.create_action(ops) vistrail.add_action(action, 0L) vistrail.update_id_scope() vistrail.change_description("Imported pipeline", 0L) self.controller = VistrailController(vistrail, UntitledLocator()) elif isinstance(arg, VistrailController): self.controller = arg elif isinstance(arg, basestring): raise TypeError("Vistrail was constructed from %r.\n" "Use load_vistrail() to get a Vistrail from a " "file." % type(arg).__name__) else: raise TypeError("Vistrail was constructed from unexpected " "argument type %r" % type(arg).__name__) def get_pipeline(self, version): """Returns a pipeline from a version number of tag. This does not change the currently selected version in this Vistrail. """ vistrail = self.controller.vistrail if isinstance(version, (int, long)): if not vistrail.db_has_action_with_id(version): raise NoSuchVersion("Vistrail doesn't have a version %r" % version) return Pipeline(vistrail.getPipelineVersionNumber(version)) elif isinstance(version, basestring): if not vistrail.has_tag_str(version): raise NoSuchVersion("Vistrail doesn't have a tag %r" % version) return Pipeline(vistrail.getPipelineVersionName(version)) else: raise TypeError("get_pipeline() argument must be a string or " "integer, not %r" % type(version).__name__) def select_version(self, version): """Sets a different version as current. The current workflow is accessible via current_workflow; it is the one that gets executed when calling execute(), and the version from which new versions are created if you perform actions. """ vistrail = self.controller.vistrail if isinstance(version, (int, long)): if not vistrail.db_has_action_with_id(version): raise NoSuchVersion("Vistrail doesn't have a version %r" % version) elif (isinstance(version, basestring)): if not vistrail.has_tag_str(version): raise NoSuchVersion("Vistrail doesn't have a tag %r" % version) version = vistrail.get_tag_str(version).action_id else: raise TypeError("select_version() argument must be a string " "or integer, not %r" % type(version).__name__) self.controller.do_version_switch(version) self._current_pipeline = None self._html = None def select_latest_version(self): """Sets the most recent version in the vistrail as current. """ self.controller.do_version_switch( self.controller.get_latest_version_in_graph()) self._current_pipeline = None self._html = None @property def current_pipeline(self): if self._current_pipeline is None: self._current_pipeline = Pipeline(self.controller.current_pipeline, vistrail=(self, self.current_version)) return self._current_pipeline @property def current_version(self): return self.controller.current_version def set_tag(self, *args): """Sets a tag for the current or specified version. """ if len(args) == 1: version, (tag, ) = self.controller.current_version, args elif len(args) == 2: version, tag = args else: raise TypeError("set_tag() takes 1 or 2 arguments (%r given)" % len(args)) if isinstance(version, (int, long)): if not self.controller.vistrail.db_has_action_with_id(version): raise NoSuchVersion("Vistrail doesn't have a version %r" % version) elif isinstance(version, basestring): if not self.controller.vistrail.has_tag_str(version): raise NoSuchVersion("Vistrail doesn't have a tag %r" % version) else: raise TypeError("set_tag() expects the version to be a string or " "integer, not %r" % type(version).__name__) self.controller.vistrail.set_tag(version, tag) def tag(self, tag): """Sets a tag for the current version. """ self.set_tag(tag) def execute(self, *args, **kwargs): """Executes the current workflow. """ return self.current_pipeline.execute(*args, **kwargs) @property def changed(self): return self.controller.changed # TODO : vistrail modification methods def __repr__(self): version_nb = self.controller.current_version if self.controller.vistrail.has_tag(version_nb): version = "%s (tag %s)" % ( version_nb, self.controller.vistrail.get_tag(version_nb)) else: version = version_nb return "<%s: %s, version %s, %s>" % ( self.__class__.__name__, self.controller.name, version, ('not changed', 'changed')[self.controller.changed]) def _repr_html_(self): if self._html is None: import cgi try: from cStringIO import StringIO except ImportError: from StringIO import StringIO self._html = '' stream = StringIO() self.controller.recompute_terse_graph() self.controller.save_version_graph( stream, highlight=self.controller.current_version) stream.seek(0) dot = stream.read() try: proc = subprocess.Popen(['dot', '-Tsvg'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) svg, _ = proc.communicate(dot) if proc.wait() == 0: self._html += svg except OSError: pass self._html += '<pre>' + cgi.escape(repr(self)) + '</pre>' return self._html
sender.connect("tcp://{0}:{1}".format(HOST, SEND)) while True: # Receiving pipeline instance configuration data = receiver.recv() logging.debug('Receiving: ' + data) fields = data.split("|") filename = fields[0] parameter_list = ast.literal_eval(fields[1]) inputs = ast.literal_eval(fields[2]) outputs = ast.literal_eval(fields[3]) locator = FileLocator(filename) loaded_objs = vistrails.core.db.io.load_vistrail(locator) controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:]) controller.do_version_switch(controller.get_latest_version_in_graph()) pipeline = Pipeline(controller) kwargs = {} for i in range(len(parameter_list)): kwargs[inputs[i]] = parameter_list[i] try: #Executing pipeline instance and retieving the result result = pipeline.execute(**kwargs) for output in outputs: parameter_list.append(str(result.output_port(output))) except: traceback.print_exc(file=sys.stdout) parameter_list.append(str(False)) kwargs['result'] = parameter_list[-1] record_python_run(kwargs, filename)
class Vistrail(object): """This class wraps both Vistrail and VistrailController. From it, you can get any pipeline from a tag name or version number. It has a concept of "current version", from which you can create new versions by performing actions. """ _current_pipeline = None _html = None def __init__(self, arg=None): initialize() if arg is None: # Copied from VistrailsApplicationInterface#open_vistrail() locator = UntitledLocator() loaded_objs = vistrails.core.db.io.load_vistrail(locator) self.controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:]) elif isinstance(arg, (_Pipeline, Pipeline)): if isinstance(arg, Pipeline): pipeline = arg.pipeline else: pipeline = arg # Copied from VistrailsApplicationInterface#open_workflow() vistrail = _Vistrail() ops = [] for module in pipeline.module_list: ops.append(('add', module)) for connection in pipeline.connection_list: ops.append(('add', connection)) action = vistrails.core.db.action.create_action(ops) vistrail.add_action(action, 0L) vistrail.update_id_scope() vistrail.change_description("Imported pipeline", 0L) self.controller = VistrailController(vistrail, UntitledLocator()) elif isinstance(arg, VistrailController): self.controller = arg elif isinstance(arg, basestring): raise TypeError("Vistrail was constructed from %r.\n" "Use load_vistrail() to get a Vistrail from a " "file." % type(arg).__name__) else: raise TypeError("Vistrail was constructed from unexpected " "argument type %r" % type(arg).__name__) def get_pipeline(self, version): """Returns a pipeline from a version number of tag. This does not change the currently selected version in this Vistrail. """ vistrail = self.controller.vistrail if isinstance(version, (int, long)): if not vistrail.db_has_action_with_id(version): raise NoSuchVersion("Vistrail doesn't have a version %r" % version) return Pipeline(vistrail.getPipelineVersionNumber(version)) elif isinstance(version, basestring): if not vistrail.has_tag_str(version): raise NoSuchVersion("Vistrail doesn't have a tag %r" % version) return Pipeline(vistrail.getPipelineVersionName(version)) else: raise TypeError("get_pipeline() argument must be a string or " "integer, not %r" % type(version).__name__) def select_version(self, version): """Sets a different version as current. The current workflow is accessible via current_workflow; it is the one that gets executed when calling execute(), and the version from which new versions are created if you perform actions. """ vistrail = self.controller.vistrail if isinstance(version, (int, long)): if not vistrail.db_has_action_with_id(version): raise NoSuchVersion("Vistrail doesn't have a version %r" % version) elif (isinstance(version, basestring)): if not vistrail.has_tag_str(version): raise NoSuchVersion("Vistrail doesn't have a tag %r" % version) version = vistrail.get_tag_str(version).action_id else: raise TypeError("select_version() argument must be a string " "or integer, not %r" % type(version).__name__) self.controller.do_version_switch(version) self._current_pipeline = None self._html = None def select_latest_version(self): """Sets the most recent version in the vistrail as current. """ self.controller.do_version_switch( self.controller.get_latest_version_in_graph()) self._current_pipeline = None self._html = None @property def current_pipeline(self): if self._current_pipeline is None: self._current_pipeline = Pipeline( self.controller.current_pipeline, vistrail=(self, self.current_version)) return self._current_pipeline @property def current_version(self): return self.controller.current_version def set_tag(self, *args): """Sets a tag for the current or specified version. """ if len(args) == 1: version, (tag,) = self.controller.current_version, args elif len(args) == 2: version, tag = args else: raise TypeError("set_tag() takes 1 or 2 arguments (%r given)" % len(args)) if isinstance(version, (int, long)): if not self.controller.vistrail.db_has_action_with_id(version): raise NoSuchVersion("Vistrail doesn't have a version %r" % version) elif isinstance(version, basestring): if not self.controller.vistrail.has_tag_str(version): raise NoSuchVersion("Vistrail doesn't have a tag %r" % version) else: raise TypeError("set_tag() expects the version to be a string or " "integer, not %r" % type(version).__name__) self.controller.vistrail.set_tag(version, tag) def tag(self, tag): """Sets a tag for the current version. """ self.set_tag(tag) def execute(self, *args, **kwargs): """Executes the current workflow. """ return self.current_pipeline.execute(*args, **kwargs) @property def changed(self): return self.controller.changed # TODO : vistrail modification methods def __repr__(self): version_nb = self.controller.current_version if self.controller.vistrail.has_tag(version_nb): version = "%s (tag %s)" % ( version_nb, self.controller.vistrail.get_tag(version_nb)) else: version = version_nb return "<%s: %s, version %s, %s>" % ( self.__class__.__name__, self.controller.name, version, ('not changed', 'changed')[self.controller.changed]) def _repr_html_(self): if self._html is None: import cgi try: from cStringIO import StringIO except ImportError: from StringIO import StringIO self._html = '' stream = StringIO() self.controller.recompute_terse_graph() self.controller.save_version_graph( stream, highlight=self.controller.current_version) stream.seek(0) dot = stream.read() try: proc = subprocess.Popen(['dot', '-Tsvg'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) svg, _ = proc.communicate(dot) if proc.wait() == 0: self._html += svg except OSError: pass self._html += '<pre>' + cgi.escape(repr(self)) + '</pre>' return self._html