def get_upgraded_pipeline(vistrail, version=None):
"""This is similar to Vistrail#getPipeline() but performs upgrades.
getPipeline() can fail if the original pipeline has a different version.
In contrast, this function will update the pipeline first using a
controller.
"""
if version is None:
version = vistrail.get_latest_version()
elif isinstance(version, (int, long)):
pass
elif isinstance(version, basestring):
version = vistrail.get_tag_str(version).action_id
else:
raise TypeError
controller = VistrailController(vistrail)
controller.recompute_terse_graph() # FIXME : this shouldn't be needed...
controller.do_version_switch(version)
return controller.current_pipeline
def test_infinite_looping_upgrade(self):
"""Test that circular upgrades fail gracefully"""
# Expected actions are as follow:
# - loads workflow2.xml
# * pipeline is missing looping_fix.x version 0.1
# - enables looping_fix.x (version 0.2)
# * pipeline is still missing looping_fix.x version 0.1
# - runs upgrade for looping_fix.x, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.y version 0.1
# * pipeline is missing looping_fix.y version 0.1
# - enables looping_fix.y (version 0.2)
# * pipeline is still missing looping_fix.y version 0.1
# Loop 50 times:
# - runs upgrade for looping_fix.y, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.x version 0.1
# * pipeline is missing looping_fix.x version 0.1
# - runs upgrade for looping_fix.x, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.y version 0.1
# * pipeline is missing looping_fix.y version 0.1
# 50 calls to handle_invalid_pipeline()
# Pre-adds packages so that the package manager can find them
packages = ["pkg_x", "pkg_y"]
prefix = "vistrails.tests.resources.looping_upgrades."
pm = get_package_manager()
for pkg in packages:
pm.get_available_package(pkg, prefix=prefix)
# Hooks handle_invalid_pipeline()
from vistrails.core.vistrail.controller import VistrailController
orig_hip = VistrailController.handle_invalid_pipeline
count = [0]
def new_hip(*args, **kwargs):
count[0] += 1
return orig_hip(*args, **kwargs)
VistrailController.handle_invalid_pipeline = new_hip
try:
# Loads workflow.xml
from vistrails.core.db.io import load_vistrail
from vistrails.core.db.locator import FileLocator
from vistrails.core.system import vistrails_root_directory
locator = FileLocator(
os.path.join(vistrails_root_directory(), "tests", "resources", "looping_upgrades", "workflow2.xml")
)
loaded_objs = load_vistrail(locator)
controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
# Select version (triggers all the validation/upgrade/loading)
self.assertEqual(controller.get_latest_version_in_graph(), 1)
try:
controller.do_version_switch(1)
except InvalidPipeline:
pass
else:
self.fail("No InvalidPipeline exception raised!")
# Restores handle_invalid_pipeline()
finally:
VistrailController.handle_invalid_pipeline = orig_hip
# disable packages
for pkg in reversed(packages):
try:
pm.late_disable_package(pkg)
except MissingPackage:
pass
# make sure it looped 50 times before failing
max_loops = getattr(get_vistrails_configuration(), "maxPipelineFixAttempts", 50)
self.assertEqual(count[0], max_loops)
# Check that original version gets selected
self.assertEqual(1, controller.current_version)
def test_looping_pipeline_fix(self):
"""Chains upgrades and automatic package initialization."""
# Expected actions are as follow:
# - loads workflow.xml
# * pipeline is missing looping_fix.a version 0.1
# - enables looping_fix.a (version 0.2)
# * pipeline is still missing looping_fix.a version 0.1
# - runs upgrade for looping_fix.a, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.b version 0.1
# * pipeline is missing looping_fix.b version 0.1
# - enables looping_fix.b (version 0.2)
# * pipeline is still missing looping_fix.b version 0.1
# - runs upgrade for looping_fix.b, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.c version 1.0
# * pipeline is missing looping_fix.c version 1.0
# - enables looping_fix.c (version 1.0)
# * pipeline is valid
# 5 calls to handle_invalid_pipeline()
# Pre-adds packages so that the package manager can find them
packages = ["pkg_a", "pkg_b", "pkg_c"]
prefix = "vistrails.tests.resources.looping_upgrades."
pm = get_package_manager()
for pkg in packages:
pm.get_available_package(pkg, prefix=prefix)
self.assertFalse(set(pkg.codepath for pkg in pm.enabled_package_list()).intersection(packages))
# Hooks handle_invalid_pipeline()
from vistrails.core.vistrail.controller import VistrailController
orig_hip = VistrailController.handle_invalid_pipeline
count = [0]
def new_hip(*args, **kwargs):
count[0] += 1
return orig_hip(*args, **kwargs)
VistrailController.handle_invalid_pipeline = new_hip
try:
# Loads workflow.xml
from vistrails.core.db.io import load_vistrail
from vistrails.core.db.locator import FileLocator
from vistrails.core.system import vistrails_root_directory
locator = FileLocator(
os.path.join(vistrails_root_directory(), "tests", "resources", "looping_upgrades", "workflow.xml")
)
loaded_objs = load_vistrail(locator)
controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
# Select version (triggers all the validation/upgrade/loading)
self.assertEqual(controller.get_latest_version_in_graph(), 1)
controller.do_version_switch(1)
self.assertEqual(count[0], 5)
# Restores handle_invalid_pipeline()
finally:
VistrailController.handle_invalid_pipeline = orig_hip
# disable packages
for pkg in reversed(packages):
try:
pm.late_disable_package(pkg)
except MissingPackage:
pass
class Vistrail(object):
"""This class wraps both Vistrail and VistrailController.
From it, you can get any pipeline from a tag name or version number.
It has a concept of "current version", from which you can create new
versions by performing actions.
"""
_current_pipeline = None
_html = None
def __init__(self, arg=None):
initialize()
if arg is None:
# Copied from VistrailsApplicationInterface#open_vistrail()
locator = UntitledLocator()
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
self.controller = VistrailController(loaded_objs[0], locator,
*loaded_objs[1:])
elif isinstance(arg, (_Pipeline, Pipeline)):
if isinstance(arg, Pipeline):
pipeline = arg.pipeline
else:
pipeline = arg
# Copied from VistrailsApplicationInterface#open_workflow()
vistrail = _Vistrail()
ops = []
for module in pipeline.module_list:
ops.append(('add', module))
for connection in pipeline.connection_list:
ops.append(('add', connection))
action = vistrails.core.db.action.create_action(ops)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
vistrail.change_description("Imported pipeline", 0L)
self.controller = VistrailController(vistrail, UntitledLocator())
elif isinstance(arg, VistrailController):
self.controller = arg
elif isinstance(arg, basestring):
raise TypeError("Vistrail was constructed from %r.\n"
"Use load_vistrail() to get a Vistrail from a "
"file." % type(arg).__name__)
else:
raise TypeError("Vistrail was constructed from unexpected "
"argument type %r" % type(arg).__name__)
def get_pipeline(self, version):
"""Returns a pipeline from a version number of tag.
This does not change the currently selected version in this Vistrail.
"""
vistrail = self.controller.vistrail
if isinstance(version, (int, long)):
if not vistrail.db_has_action_with_id(version):
raise NoSuchVersion("Vistrail doesn't have a version %r" %
version)
return Pipeline(vistrail.getPipelineVersionNumber(version))
elif isinstance(version, basestring):
if not vistrail.has_tag_str(version):
raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
return Pipeline(vistrail.getPipelineVersionName(version))
else:
raise TypeError("get_pipeline() argument must be a string or "
"integer, not %r" % type(version).__name__)
def select_version(self, version):
"""Sets a different version as current.
The current workflow is accessible via current_workflow; it is the one
that gets executed when calling execute(), and the version from which
new versions are created if you perform actions.
"""
vistrail = self.controller.vistrail
if isinstance(version, (int, long)):
if not vistrail.db_has_action_with_id(version):
raise NoSuchVersion("Vistrail doesn't have a version %r" %
version)
elif (isinstance(version, basestring)):
if not vistrail.has_tag_str(version):
raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
version = vistrail.get_tag_str(version).action_id
else:
raise TypeError("select_version() argument must be a string "
"or integer, not %r" % type(version).__name__)
self.controller.do_version_switch(version)
self._current_pipeline = None
self._html = None
def select_latest_version(self):
"""Sets the most recent version in the vistrail as current.
"""
self.controller.do_version_switch(
self.controller.get_latest_version_in_graph())
self._current_pipeline = None
self._html = None
@property
def current_pipeline(self):
if self._current_pipeline is None:
self._current_pipeline = Pipeline(self.controller.current_pipeline,
vistrail=(self,
self.current_version))
return self._current_pipeline
@property
def current_version(self):
return self.controller.current_version
def set_tag(self, *args):
"""Sets a tag for the current or specified version.
"""
if len(args) == 1:
version, (tag, ) = self.controller.current_version, args
elif len(args) == 2:
version, tag = args
else:
raise TypeError("set_tag() takes 1 or 2 arguments (%r given)" %
len(args))
if isinstance(version, (int, long)):
if not self.controller.vistrail.db_has_action_with_id(version):
raise NoSuchVersion("Vistrail doesn't have a version %r" %
version)
elif isinstance(version, basestring):
if not self.controller.vistrail.has_tag_str(version):
raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
else:
raise TypeError("set_tag() expects the version to be a string or "
"integer, not %r" % type(version).__name__)
self.controller.vistrail.set_tag(version, tag)
def tag(self, tag):
"""Sets a tag for the current version.
"""
self.set_tag(tag)
def execute(self, *args, **kwargs):
"""Executes the current workflow.
"""
return self.current_pipeline.execute(*args, **kwargs)
@property
def changed(self):
return self.controller.changed
# TODO : vistrail modification methods
def __repr__(self):
version_nb = self.controller.current_version
if self.controller.vistrail.has_tag(version_nb):
version = "%s (tag %s)" % (
version_nb, self.controller.vistrail.get_tag(version_nb))
else:
version = version_nb
return "<%s: %s, version %s, %s>" % (
self.__class__.__name__, self.controller.name, version,
('not changed', 'changed')[self.controller.changed])
def _repr_html_(self):
if self._html is None:
import cgi
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
self._html = ''
stream = StringIO()
self.controller.recompute_terse_graph()
self.controller.save_version_graph(
stream, highlight=self.controller.current_version)
stream.seek(0)
dot = stream.read()
try:
proc = subprocess.Popen(['dot', '-Tsvg'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
svg, _ = proc.communicate(dot)
if proc.wait() == 0:
self._html += svg
except OSError:
pass
self._html += '<pre>' + cgi.escape(repr(self)) + '</pre>'
return self._html
sender.connect("tcp://{0}:{1}".format(HOST, SEND))
while True:
# Receiving pipeline instance configuration
data = receiver.recv()
logging.debug('Receiving: ' + data)
fields = data.split("|")
filename = fields[0]
parameter_list = ast.literal_eval(fields[1])
inputs = ast.literal_eval(fields[2])
outputs = ast.literal_eval(fields[3])
locator = FileLocator(filename)
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
controller.do_version_switch(controller.get_latest_version_in_graph())
pipeline = Pipeline(controller)
kwargs = {}
for i in range(len(parameter_list)):
kwargs[inputs[i]] = parameter_list[i]
try:
#Executing pipeline instance and retieving the result
result = pipeline.execute(**kwargs)
for output in outputs:
parameter_list.append(str(result.output_port(output)))
except:
traceback.print_exc(file=sys.stdout)
parameter_list.append(str(False))
kwargs['result'] = parameter_list[-1]
record_python_run(kwargs, filename)
class Vistrail(object):
"""This class wraps both Vistrail and VistrailController.
From it, you can get any pipeline from a tag name or version number.
It has a concept of "current version", from which you can create new
versions by performing actions.
"""
_current_pipeline = None
_html = None
def __init__(self, arg=None):
initialize()
if arg is None:
# Copied from VistrailsApplicationInterface#open_vistrail()
locator = UntitledLocator()
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
self.controller = VistrailController(loaded_objs[0], locator,
*loaded_objs[1:])
elif isinstance(arg, (_Pipeline, Pipeline)):
if isinstance(arg, Pipeline):
pipeline = arg.pipeline
else:
pipeline = arg
# Copied from VistrailsApplicationInterface#open_workflow()
vistrail = _Vistrail()
ops = []
for module in pipeline.module_list:
ops.append(('add', module))
for connection in pipeline.connection_list:
ops.append(('add', connection))
action = vistrails.core.db.action.create_action(ops)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
vistrail.change_description("Imported pipeline", 0L)
self.controller = VistrailController(vistrail, UntitledLocator())
elif isinstance(arg, VistrailController):
self.controller = arg
elif isinstance(arg, basestring):
raise TypeError("Vistrail was constructed from %r.\n"
"Use load_vistrail() to get a Vistrail from a "
"file." % type(arg).__name__)
else:
raise TypeError("Vistrail was constructed from unexpected "
"argument type %r" % type(arg).__name__)
def get_pipeline(self, version):
"""Returns a pipeline from a version number of tag.
This does not change the currently selected version in this Vistrail.
"""
vistrail = self.controller.vistrail
if isinstance(version, (int, long)):
if not vistrail.db_has_action_with_id(version):
raise NoSuchVersion("Vistrail doesn't have a version %r" %
version)
return Pipeline(vistrail.getPipelineVersionNumber(version))
elif isinstance(version, basestring):
if not vistrail.has_tag_str(version):
raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
return Pipeline(vistrail.getPipelineVersionName(version))
else:
raise TypeError("get_pipeline() argument must be a string or "
"integer, not %r" % type(version).__name__)
def select_version(self, version):
"""Sets a different version as current.
The current workflow is accessible via current_workflow; it is the one
that gets executed when calling execute(), and the version from which
new versions are created if you perform actions.
"""
vistrail = self.controller.vistrail
if isinstance(version, (int, long)):
if not vistrail.db_has_action_with_id(version):
raise NoSuchVersion("Vistrail doesn't have a version %r" %
version)
elif (isinstance(version, basestring)):
if not vistrail.has_tag_str(version):
raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
version = vistrail.get_tag_str(version).action_id
else:
raise TypeError("select_version() argument must be a string "
"or integer, not %r" % type(version).__name__)
self.controller.do_version_switch(version)
self._current_pipeline = None
self._html = None
def select_latest_version(self):
"""Sets the most recent version in the vistrail as current.
"""
self.controller.do_version_switch(
self.controller.get_latest_version_in_graph())
self._current_pipeline = None
self._html = None
@property
def current_pipeline(self):
if self._current_pipeline is None:
self._current_pipeline = Pipeline(
self.controller.current_pipeline,
vistrail=(self, self.current_version))
return self._current_pipeline
@property
def current_version(self):
return self.controller.current_version
def set_tag(self, *args):
"""Sets a tag for the current or specified version.
"""
if len(args) == 1:
version, (tag,) = self.controller.current_version, args
elif len(args) == 2:
version, tag = args
else:
raise TypeError("set_tag() takes 1 or 2 arguments (%r given)" %
len(args))
if isinstance(version, (int, long)):
if not self.controller.vistrail.db_has_action_with_id(version):
raise NoSuchVersion("Vistrail doesn't have a version %r" %
version)
elif isinstance(version, basestring):
if not self.controller.vistrail.has_tag_str(version):
raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
else:
raise TypeError("set_tag() expects the version to be a string or "
"integer, not %r" % type(version).__name__)
self.controller.vistrail.set_tag(version, tag)
def tag(self, tag):
"""Sets a tag for the current version.
"""
self.set_tag(tag)
def execute(self, *args, **kwargs):
"""Executes the current workflow.
"""
return self.current_pipeline.execute(*args, **kwargs)
@property
def changed(self):
return self.controller.changed
# TODO : vistrail modification methods
def __repr__(self):
version_nb = self.controller.current_version
if self.controller.vistrail.has_tag(version_nb):
version = "%s (tag %s)" % (
version_nb,
self.controller.vistrail.get_tag(version_nb))
else:
version = version_nb
return "<%s: %s, version %s, %s>" % (
self.__class__.__name__,
self.controller.name,
version,
('not changed', 'changed')[self.controller.changed])
def _repr_html_(self):
if self._html is None:
import cgi
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
self._html = ''
stream = StringIO()
self.controller.recompute_terse_graph()
self.controller.save_version_graph(
stream,
highlight=self.controller.current_version)
stream.seek(0)
dot = stream.read()
try:
proc = subprocess.Popen(['dot', '-Tsvg'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
svg, _ = proc.communicate(dot)
if proc.wait() == 0:
self._html += svg
except OSError:
pass
self._html += '<pre>' + cgi.escape(repr(self)) + '</pre>'
return self._html
