def make_pipeline(self):
"""Creates an example pipeline that is used to conduct tests.
"""
vistrail = Vistrail()
controller = VistrailController(vistrail)
controller.change_selected_version(0)
# 0 1 2 7 8
# | / \ \ /
# 3 4 5 9
# | / \
# 6 10 11
modules = [controller.add_module('org.vistrails.vistrails.basic',
'String')
for i in xrange(12)]
def connect(outmod, inmod):
controller.add_connection(
modules[outmod].id,
'value',
modules[inmod].id,
'value')
for (outmod, inmod) in [(0, 3),
(1, 4), (1, 5), (5, 6),
(7, 9), (8, 9), (9, 10), (9, 11)]:
connect(outmod, inmod)
return controller, modules
def __init__(self, arg=None):
initialize()
if arg is None:
# Copied from VistrailsApplicationInterface#open_vistrail()
locator = UntitledLocator()
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
self.controller = VistrailController(loaded_objs[0], locator,
*loaded_objs[1:])
elif isinstance(arg, (_Pipeline, Pipeline)):
if isinstance(arg, Pipeline):
pipeline = arg.pipeline
else:
pipeline = arg
# Copied from VistrailsApplicationInterface#open_workflow()
vistrail = _Vistrail()
ops = []
for module in pipeline.module_list:
ops.append(('add', module))
for connection in pipeline.connection_list:
ops.append(('add', connection))
action = vistrails.core.db.action.create_action(ops)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
vistrail.change_description("Imported pipeline", 0L)
self.controller = VistrailController(vistrail, UntitledLocator())
elif isinstance(arg, VistrailController):
self.controller = arg
elif isinstance(arg, basestring):
raise TypeError("Vistrail was constructed from %r.\n"
"Use load_vistrail() to get a Vistrail from a "
"file." % type(arg).__name__)
else:
raise TypeError("Vistrail was constructed from unexpected "
"argument type %r" % type(arg).__name__)
def test_vtl_files(self):
from vistrails.tests.utils import run_file
for root, dirs, file_names in os.walk(self.vtl_path):
for file_name in file_names:
if file_name.endswith('.vtl'):
f = os.path.join(root, file_name)
locator = FileLocator(f)
version = locator._vnode
# if there is a version specified try to execute it,
# else just load the pipeline
if version:
errors = run_file(f, lambda x: x == version)
self.assertEqual(errors, [], 'Errors processing %s: %s' % (f, str(errors)))
else:
import vistrails.core.db.io
from vistrails.core.vistrail.controller import \
VistrailController
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
controller = VistrailController(loaded_objs[0],
locator,
*loaded_objs[1:])
controller.change_selected_version(
controller.vistrail.get_latest_version())
self.assertTrue(controller.current_pipeline.is_valid,
"Latest pipeline is invalid: %s" % f)
def __init__(self, arg=None):
initialize()
if arg is None:
# Copied from VistrailsApplicationInterface#open_vistrail()
locator = UntitledLocator()
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
self.controller = VistrailController(loaded_objs[0], locator,
*loaded_objs[1:])
elif isinstance(arg, (_Pipeline, Pipeline)):
if isinstance(arg, Pipeline):
pipeline = arg.pipeline
else:
pipeline = arg
# Copied from VistrailsApplicationInterface#open_workflow()
vistrail = _Vistrail()
ops = []
for module in pipeline.module_list:
ops.append(('add', module))
for connection in pipeline.connection_list:
ops.append(('add', connection))
action = vistrails.core.db.action.create_action(ops)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
vistrail.change_description("Imported pipeline", 0L)
self.controller = VistrailController(vistrail, UntitledLocator())
elif isinstance(arg, VistrailController):
self.controller = arg
elif isinstance(arg, basestring):
raise TypeError("Vistrail was constructed from %r.\n"
"Use load_vistrail() to get a Vistrail from a "
"file." % type(arg).__name__)
else:
raise TypeError("Vistrail was constructed from unexpected "
"argument type %r" % type(arg).__name__)
def test_vtl_files(self):
from vistrails.tests.utils import run_file
for root, dirs, file_names in os.walk(self.vtl_path):
for file_name in sorted(file_names):
if file_name.endswith('.vtl'):
# update available packages
from vistrails.core.packagemanager import get_package_manager
get_package_manager().build_available_package_names_list()
f = os.path.join(root, file_name)
locator = FileLocator(f)
version = locator._vnode
# if there is a version specified try to execute it,
# else just load the pipeline
if version:
errors = run_file(f, lambda x: x == version)
self.assertEqual(
errors, [],
'Errors processing %s: %s' % (f, str(errors)))
else:
import vistrails.core.db.io
from vistrails.core.vistrail.controller import \
VistrailController
loaded_objs = vistrails.core.db.io.load_vistrail(
locator)
controller = VistrailController(
loaded_objs[0], locator, *loaded_objs[1:])
controller.change_selected_version(
controller.vistrail.get_latest_version())
self.assertTrue(controller.current_pipeline.is_valid,
"Latest pipeline is invalid: %s" % f)
def make_pipeline(self):
"""Creates an example pipeline that is used to conduct tests.
"""
vistrail = Vistrail()
controller = VistrailController(vistrail)
controller.change_selected_version(0)
# 0 1 2 7 8
# | / \ \ /
# 3 4 5 9
# | / \
# 6 10 11
modules = [
controller.add_module('org.vistrails.vistrails.basic', 'String')
for i in xrange(12)
]
def connect(outmod, inmod):
controller.add_connection(modules[outmod].id, 'value',
modules[inmod].id, 'value')
for (outmod, inmod) in [(0, 3), (1, 4), (1, 5), (5, 6), (7, 9), (8, 9),
(9, 10), (9, 11)]:
connect(outmod, inmod)
return controller, modules
def copyBaseVistrailController(vt_controller):
vistrail = vt_controller.vistrail
newvt_controller = BaseVistrailController(vistrail, None)
current_log = vt_controller.log
newvt_controller.log = current_log
# DAK: why does the base version copy the _mashups but the non-base
# version doesn't??
for m in vt_controller._mashups:
newvt_controller._mashups.append(copy.copy(m))
return newvt_controller
def copyBaseVistrailController(vt_controller):
vistrail = vt_controller.vistrail
newvt_controller = BaseVistrailController(vistrail, None)
current_log = vt_controller.log
newvt_controller.log = current_log
# DAK: why does the base version copy the _mashups but the non-base
# version doesn't??
for m in vt_controller._mashups:
newvt_controller._mashups.append(copy.copy(m))
return newvt_controller
def pipeline_from_generator(variable_gen):
# Get the original OutputPort module
orig_controller = variable_gen._generator.controller
base_pipeline = orig_controller.vistrail.getPipeline('dat-vars')
if len(base_pipeline.module_list) != 1:
raise ValueError("dat-vars version is invalid")
output_port = base_pipeline.module_list[0]
controller = VistrailController(Vistrail())
# OutputPort
operations = [('add', output_port)]
# Rest of the pipeline
operations += variable_gen._generator.operations
# Connection
connection = controller.create_connection(
variable_gen._output_module, variable_gen._outputport_name,
output_port, 'InternalPipe')
operations.append(('add', connection))
# Materialize this
action = create_action(operations)
controller.add_new_action(action)
version = controller.perform_action(action)
controller.change_selected_version(version)
assert version == controller.current_version == 1
return controller.current_pipeline, 1
def test_find_modules_by_type(self):
"""Tests the find_modules_by_type() function.
"""
vistrail = Vistrail()
controller = VistrailController(vistrail)
controller.change_selected_version(0)
mod1 = controller.add_module('org.vistrails.vistrails.basic', 'String')
mod2 = controller.add_module('org.vistrails.vistrails.basic', 'Float')
mod3 = controller.add_module('org.vistrails.vistrails.basic', 'String')
mod4 = controller.add_module('org.vistrails.vistrails.basic',
'Integer')
from dat.vistrails_interface import find_modules_by_type
from vistrails.core.modules.basic_modules import Boolean, Float, String
self.assertEqual(
set(m.id for m in find_modules_by_type(controller.current_pipeline,
[String])),
set([mod1.id, mod3.id]))
self.assertEqual([
m.id
for m in find_modules_by_type(controller.current_pipeline, [Float])
], [mod2.id, mod4.id])
self.assertEqual(
find_modules_by_type(controller.current_pipeline, [Boolean]), [])
def add_vistrail(self, vistrail, locator,
abstraction_files=None, thumbnail_files=None, mashups=None):
objs = vistrail, locator, abstraction_files, thumbnail_files, mashups
controller = VistrailController(*objs)
self._controllers[locator] = controller
self._cur_controller = controller
return self._cur_controller
def test_cache(self):
from vistrails.core.modules.basic_modules import StandardOutput
old_compute = StandardOutput.compute
StandardOutput.compute = lambda s: None
try:
from vistrails.core.db.locator import XMLFileLocator
from vistrails.core.vistrail.controller import VistrailController
from vistrails.core.db.io import load_vistrail
"""Test if basic caching is working."""
locator = XMLFileLocator(
vistrails.core.system.vistrails_root_directory() +
'/tests/resources/dummy.xml')
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
# the controller will take care of upgrades
controller = VistrailController(v, locator, abstractions,
thumbnails, mashups)
p1 = v.getPipeline('int chain')
n = v.get_version_number('int chain')
controller.change_selected_version(n)
controller.flush_delayed_actions()
p1 = controller.current_pipeline
view = DummyView()
interpreter = CachedInterpreter.get()
result = interpreter.execute(
p1,
locator=v,
current_version=n,
view=view,
)
# to force fresh params
p2 = v.getPipeline('int chain')
controller.change_selected_version(n)
controller.flush_delayed_actions()
p2 = controller.current_pipeline
result = interpreter.execute(
p2,
locator=v,
current_version=n,
view=view,
)
self.assertEqual(len(result.modules_added), 1)
finally:
StandardOutput.compute = old_compute
def pipeline_from_generator(variable_gen):
# Get the original OutputPort module
orig_controller = variable_gen._generator.controller
base_pipeline = orig_controller.vistrail.getPipeline('dat-vars')
if len(base_pipeline.module_list) != 1:
raise ValueError("dat-vars version is invalid")
output_port = base_pipeline.module_list[0]
controller = VistrailController(Vistrail())
# OutputPort
operations = [('add', output_port)]
# Rest of the pipeline
operations += variable_gen._generator.operations
# Connection
connection = controller.create_connection(
variable_gen._output_module,
variable_gen._outputport_name,
output_port,
'InternalPipe')
operations.append(('add', connection))
# Materialize this
action = create_action(operations)
controller.add_new_action(action)
version = controller.perform_action(action)
controller.change_selected_version(version)
assert version == controller.current_version == 1
return controller.current_pipeline, 1
def execute(self):
""" execute() -> None
Execute the selected pipeline with the edited aliases
"""
aliases = {}
for r in xrange(self.aliasTable.rowCount()):
name = str(self.aliasTable.item(r, 0).text())
value = str(self.aliasTable.item(r, 1).text())
aliases[name] = value
versionNumber = self.versions[
self.pipelineList.currentIndex().row()][0]
pipeline = self.vistrail.getPipelineVersionNumber(versionNumber)
controller = VistrailController(self.vistrail)
controller.execute_workflow_list([
(self.vistrail.locator, versionNumber, pipeline, DummyView(),
aliases, None)
])
def execute(self):
""" execute() -> None
Execute the selected pipeline with the edited aliases
"""
aliases = {}
for r in xrange(self.aliasTable.rowCount()):
name = str(self.aliasTable.item(r, 0).text())
value = str(self.aliasTable.item(r, 1).text())
aliases[name] = value
versionNumber = self.versions[self.pipelineList.currentIndex().row()][0]
pipeline = self.vistrail.getPipelineVersionNumber(versionNumber)
controller = VistrailController(self.vistrail)
controller.execute_workflow_list([(self.vistrail.locator,
versionNumber,
pipeline,
DummyView(),
aliases,
None)])
def run_file(filename, tag_filter=lambda x: True):
"""Loads a .vt file and runs all the tagged versions in it.
"""
import vistrails.core.db.io
from vistrails.core.db.locator import FileLocator
from vistrails.core.system import vistrails_root_directory
from vistrails.core.vistrail.controller import VistrailController
filename = os.path.join(vistrails_root_directory(), '..', filename)
locator = FileLocator(filename)
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
errors = []
for version, name in controller.vistrail.get_tagMap().iteritems():
if tag_filter(name):
controller.change_selected_version(version)
(result,), _ = controller.execute_current_workflow()
if result.errors:
errors.append(("%d: %s" % (version, name), result.errors))
return errors
def test_cache(self):
from vistrails.core.modules.basic_modules import StandardOutput
old_compute = StandardOutput.compute
StandardOutput.compute = lambda s: None
try:
from vistrails.core.db.locator import XMLFileLocator
from vistrails.core.vistrail.controller import VistrailController
from vistrails.core.db.io import load_vistrail
"""Test if basic caching is working."""
locator = XMLFileLocator(vistrails.core.system.vistrails_root_directory() +
'/tests/resources/dummy.xml')
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
# the controller will take care of upgrades
controller = VistrailController(v, locator, abstractions,
thumbnails, mashups)
p1 = v.getPipeline('int chain')
n = v.get_version_number('int chain')
controller.change_selected_version(n)
controller.flush_delayed_actions()
p1 = controller.current_pipeline
view = DummyView()
interpreter = CachedInterpreter.get()
result = interpreter.execute(p1,
locator=v,
current_version=n,
view=view,
)
# to force fresh params
p2 = v.getPipeline('int chain')
controller.change_selected_version(n)
controller.flush_delayed_actions()
p2 = controller.current_pipeline
result = interpreter.execute(p2,
locator=v,
current_version=n,
view=view,
)
self.assertEqual(len(result.modules_added), 1)
finally:
StandardOutput.compute = old_compute
def previewChanges(self, aliases):
print "previewChanges", aliases
# we will just execute the pipeline with the given alias dictionary
controller = VistrailController()
controller.set_vistrail(self.plot_vistrail, self.locator)
version = self.plot_vistrail.get_version_number(self.workflow_tag) if self.workflow_tag else controller.get_latest_version_in_graph()
controller.change_selected_version(version)
(results, _) = controller.execute_current_workflow(aliases)
def load_vistrail(filename, version=None):
"""Loads a Vistrail from a filename.
"""
initialize()
if not isinstance(filename, basestring):
raise TypeError("load_vistrails() expects a filename, got %r" %
type(filename).__name__)
locator = FileLocator(filename)
# Copied from VistrailsApplicationInterface#open_vistrail()
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
return Vistrail(controller)
def get_upgraded_pipeline(vistrail, version=None):
"""This is similar to Vistrail#getPipeline() but performs upgrades.
getPipeline() can fail if the original pipeline has a different version.
In contrast, this function will update the pipeline first using a
controller.
"""
if version is None:
version = vistrail.get_latest_version()
elif isinstance(version, (int, long)):
pass
elif isinstance(version, basestring):
version = vistrail.get_tag_str(version).action_id
else:
raise TypeError
controller = VistrailController(vistrail)
controller.recompute_terse_graph() # FIXME : this shouldn't be needed...
controller.do_version_switch(version)
return controller.current_pipeline
def test_find_modules_by_type(self):
"""Tests the find_modules_by_type() function.
"""
vistrail = Vistrail()
controller = VistrailController(vistrail)
controller.change_selected_version(0)
mod1 = controller.add_module('org.vistrails.vistrails.basic',
'String')
mod2 = controller.add_module('org.vistrails.vistrails.basic',
'Float')
mod3 = controller.add_module('org.vistrails.vistrails.basic',
'String')
mod4 = controller.add_module('org.vistrails.vistrails.basic',
'Integer')
from dat.vistrails_interface import find_modules_by_type
from vistrails.core.modules.basic_modules import Boolean, Float, String
self.assertEqual(
set(m.id for m in find_modules_by_type(
controller.current_pipeline,
[String])),
set([mod1.id, mod3.id]))
self.assertEqual(
[m.id for m in find_modules_by_type(
controller.current_pipeline,
[Float])],
[mod2.id, mod4.id])
self.assertEqual(
find_modules_by_type(
controller.current_pipeline,
[Boolean]),
[])
def load(self, loadworkflow=True):
config = ConfigParser.ConfigParser()
if config.read(self.config_file):
if config.has_section('global'):
if config.has_option('global', 'cellnum'):
self.cellnum = config.getint('global', 'cellnum')
if config.has_option('global', 'filenum'):
self.filenum = config.getint('global', 'filenum')
if config.has_option('global', 'varnum'):
self.varnum = config.getint('global', 'varnum')
print " ------ Loaded plot %s, varnum = %d ------ " % ( self.name, self.varnum )
if config.has_option('global', 'workflow_tag'):
self.workflow_tag = config.get('global', 'workflow_tag')
# else:
# debug.warning("CDAT Package: file %s does not contain a required option 'workflow_tag'. Widget will not be loaded."%self.config_file)
# self.loaded = False
# return
if config.has_option('global', 'filetypes'):
types = config.get('global', 'filetypes')
tlist = [t.strip() for t in types.split(";")]
for t in tlist:
kv = t.split(":")
self.filetypes[kv[0].strip()] = [v.strip()
for v in kv[1].split(",")]
if config.has_option('global', 'qt_filter'):
self.qt_filter = config.get('global', 'qt_filter')
if config.has_option('global', 'dependencies'):
deps = config.get('global', 'dependencies')
self.dependencies = [d.strip() for d in deps.split(",")]
if config.has_option('global', 'serialized_config_alias'):
self.serializedConfigAlias = config.get('global', 'serialized_config_alias')
for y in range(self.filenum):
self.files.append( 'Filename' + str(y+1) )
for v in range(self.varnum):
self.vars.append( 'VariableName' + str(v+1) )
self.axes.append( 'Axes' + str(v+1) )
for x in range(self.cellnum):
section_name = 'cell' + str(x+1)
if config.has_section(section_name):
cellType = config.get(section_name, 'celltype')
if config.has_option(section_name, 'address_alias'):
self.cells.append( Cell( cellType, None, None,
config.get(section_name, 'address_alias') ) )
else:
self.cells.append(Cell( cellType,"Row"+str(x+1), "Column"+str(x+1) ) )
else:
for y in range(self.filenum):
option_name = 'filename_alias' + str(y+1)
if config.has_option('global', option_name):
self.files.append(config.get('global', option_name))
for v in range(self.varnum):
option_name = 'varname_alias' + str(v+1)
if config.has_option('global', option_name):
self.vars.append(config.get('global', option_name))
axes_name = 'axes_alias' + str(v+1)
if config.has_option('global', axes_name):
self.axes.append(config.get('global', axes_name))
for x in range(self.cellnum):
section_name = 'cell' + str(x+1)
if (config.has_section(section_name) and
config.has_option(section_name, 'celltype') and
config.has_option(section_name, 'row_alias') and
config.has_option(section_name, 'col_alias')):
self.cells.append(Cell(config.get(section_name, 'celltype'),
config.get(section_name, 'row_alias'),
config.get(section_name, 'col_alias')))
if loadworkflow:
#load workflow in vistrail
#only if dependencies are enabled
manager = get_package_manager()
self.unsatisfied_deps = []
for dep in self.dependencies:
if not manager.has_package(dep):
self.unsatisfied_deps.append(dep)
if len(self.unsatisfied_deps) == 0:
try:
(self.plot_vistrail, abstractions , thumbnails, mashups) = load_vistrail(self.locator)
controller = VistrailController()
controller.set_vistrail(self.plot_vistrail, self.locator,
abstractions, thumbnails,
mashups)
self.workflow_version = self.plot_vistrail.get_version_number(self.workflow_tag) if self.workflow_tag else controller.get_latest_version_in_graph()
print " Loaded %s version: %s" % ( self.name, str( self.workflow_version ) )
controller.change_selected_version(self.workflow_version)
self.workflow = controller.current_pipeline
self.loaded = True
except Exception, err:
debug.warning( "Error loading workflow %s: %s" % ( self.name, err ) )
self.loaded = False
else:
debug.warning("UV-CDAT: %s widget could not be loaded \
because it depends on packages that are not loaded:"%self.name)
debug.warning(" %s"%", ".join(self.unsatisfied_deps))
self.loaded = False
else:
debug.warning("UV-CDAT: file %s does not contain a 'global'\
section. Widget will not be loaded."%self.config_file)
self.loaded = False
def exportMashup(filename, vtcontroller, mashuptrail, mashupversion, etype):
"""exportMashup(filename: str, vtcontroller: VistrailController,
mashuptrail: Mashuptrail, type: int) -> bool
where etype is
0: include full tree
1: include only workflow and mashup identified by version
2: as a link, it will point to a local file.
"""
result = False
if vtcontroller is not None and mashuptrail is not None:
locator = vtcontroller.locator
version = mashuptrail.vtVersion
node = ElementTree.Element('vtlink')
if isinstance(locator, DBLocator):
node.set('host', str(locator.host))
node.set('port', str(locator.port))
node.set('database', str(locator.db))
node.set('vtid', str(locator.obj_id))
else:
node.set('filename', str(locator.name))
node.set('version', str(version))
node.set('execute', "True")
node.set('forceDB', "False")
node.set('showSpreadsheetOnly', "True")
node.set('mashuptrail', str(mashuptrail.id))
node.set('mashupVersion', str(mashupversion))
if etype in [0,1]:
if etype == 1: #minimal
pip = vtcontroller.vistrail.getPipeline(version)
vistrail = Vistrail()
id_remap = {}
action = vistrails.core.db.action.create_paste_action(pip,
vistrail.idScope,
id_remap)
vistrail.add_action(action, 0L, 0)
tag = vtcontroller.vistrail.get_tag(version)
if tag is None:
tag = "Imported workflow"
vistrail.addTag(tag, action.id)
node.set('version', str(action.id))
id_scope = IdScope(1L)
newmashuptrail = Mashuptrail(
MashupsManager.getNewMashuptrailId(),
action.id,
id_scope)
maction = mashuptrail.actionMap[mashupversion]
mtag = mashuptrail.getTagForActionId(mashupversion)
newmashup = copy.copy(maction.mashup)
newmashup.remapPipelineObjects(id_remap)
currVersion = newmashuptrail.addVersion(
newmashuptrail.getLatestVersion(),
newmashup, maction.user,
maction.date)
newmashuptrail.currentVersion = currVersion
newmashuptrail.changeTag(currVersion, mtag, maction.user,
maction.date)
newvtcontroller = BaseVistrailController(vistrail, None)
MashupsManager.addMashuptrailtoVistrailController(newvtcontroller,
newmashuptrail)
node.set('mashuptrail', str(newmashuptrail.id))
node.set('mashupVersion', str(newmashuptrail.currentVersion))
else:
vistrail = vtcontroller.vistrail
newvtcontroller = MashupsManager.copyBaseVistrailController(vtcontroller)
#create temporary file
(fd, name) = tempfile.mkstemp(prefix='vt_tmp',
suffix='.vt')
os.close(fd)
try:
fileLocator = FileLocator(name)
newvtcontroller.write_vistrail(fileLocator)
contents = open(name).read()
vtcontent = base64.b64encode(contents)
finally:
os.unlink(name)
#if not vistrail.db_version:
# vistrail.db_version = currentVersion
node.set('vtcontent',vtcontent)
xmlstring = ElementTree.tostring(node)
file_ = open(filename,'w')
file_.write(xmlstring)
file_.close()
result = True
return result
def execute_wf(wf, output_port):
# Save the workflow in a temporary file
temp_wf_fd, temp_wf = tempfile.mkstemp()
try:
f = open(temp_wf, 'w')
f.write(wf)
f.close()
os.close(temp_wf_fd)
# Clean the cache
interpreter = get_default_interpreter()
interpreter.flush()
# Load the Pipeline from the temporary file
vistrail = Vistrail()
locator = XMLFileLocator(temp_wf)
workflow = locator.load(Pipeline)
# Build a Vistrail from this single Pipeline
action_list = []
for module in workflow.module_list:
action_list.append(('add', module))
for connection in workflow.connection_list:
action_list.append(('add', connection))
action = vistrails.core.db.action.create_action(action_list)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
tag = 'parallel flow'
vistrail.addTag(tag, action.id)
# Build a controller and execute
controller = VistrailController()
controller.set_vistrail(vistrail, None)
controller.change_selected_version(vistrail.get_version_number(tag))
execution = controller.execute_current_workflow(
custom_aliases=None,
custom_params=None,
extra_info=None,
reason='API Pipeline Execution')
# Build a list of errors
errors = []
pipeline = vistrail.getPipeline(tag)
execution_errors = execution[0][0].errors
if execution_errors:
for key in execution_errors:
module = pipeline.modules[key]
msg = '%s: %s' % (module.name, execution_errors[key])
errors.append(msg)
# Get the execution log from the controller
try:
module_log = controller.log.workflow_execs[0].item_execs[0]
except IndexError:
errors.append("Module log not found")
return dict(errors=errors)
else:
machine = controller.log.workflow_execs[0].machines[
module_log.machine_id]
xml_log = serialize(module_log)
machine_log = serialize(machine)
# Get the output value
output = None
serializable = None
if not execution_errors:
executed_module, = execution[0][0].executed
executed_module = execution[0][0].objects[executed_module]
try:
output = executed_module.get_output(output_port)
except ModuleError:
errors.append("Output port not found: %s" % output_port)
return dict(errors=errors)
reg = vistrails.core.modules.module_registry.get_module_registry()
base_classes = inspect.getmro(type(output))
if Module in base_classes:
serializable = reg.get_descriptor(type(output)).sigstring
output = output.serialize()
# Return the dictionary, that will be sent back to the client
return dict(errors=errors,
output=output,
serializable=serializable,
xml_log=xml_log,
machine_log=machine_log)
finally:
os.unlink(temp_wf)
def execute_wf(wf, output_port):
# Save the workflow in a temporary file
temp_wf_fd, temp_wf = tempfile.mkstemp()
try:
f = open(temp_wf, 'w')
f.write(wf)
f.close()
os.close(temp_wf_fd)
# Clean the cache
interpreter = get_default_interpreter()
interpreter.flush()
# Load the Pipeline from the temporary file
vistrail = Vistrail()
locator = XMLFileLocator(temp_wf)
workflow = locator.load(Pipeline)
# Build a Vistrail from this single Pipeline
action_list = []
for module in workflow.module_list:
action_list.append(('add', module))
for connection in workflow.connection_list:
action_list.append(('add', connection))
action = vistrails.core.db.action.create_action(action_list)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
tag = 'parallel flow'
vistrail.addTag(tag, action.id)
# Build a controller and execute
controller = VistrailController()
controller.set_vistrail(vistrail, None)
controller.change_selected_version(vistrail.get_version_number(tag))
execution = controller.execute_current_workflow(
custom_aliases=None,
custom_params=None,
extra_info=None,
reason='API Pipeline Execution')
# Build a list of errors
errors = []
pipeline = vistrail.getPipeline(tag)
execution_errors = execution[0][0].errors
if execution_errors:
for key in execution_errors:
module = pipeline.modules[key]
msg = '%s: %s' %(module.name, execution_errors[key])
errors.append(msg)
# Get the execution log from the controller
try:
module_log = controller.log.workflow_execs[0].item_execs[0]
except IndexError:
errors.append("Module log not found")
return dict(errors=errors)
else:
machine = controller.log.workflow_execs[0].machines[
module_log.machine_id]
xml_log = serialize(module_log)
machine_log = serialize(machine)
# Get the output value
output = None
serializable = None
if not execution_errors:
executed_module, = execution[0][0].executed
executed_module = execution[0][0].objects[executed_module]
try:
output = executed_module.get_output(output_port)
except ModuleError:
errors.append("Output port not found: %s" % output_port)
return dict(errors=errors)
reg = vistrails.core.modules.module_registry.get_module_registry()
base_classes = inspect.getmro(type(output))
if Module in base_classes:
serializable = reg.get_descriptor(type(output)).sigstring
output = output.serialize()
# Return the dictionary, that will be sent back to the client
return dict(errors=errors,
output=output,
serializable=serializable,
xml_log=xml_log,
machine_log=machine_log)
finally:
os.unlink(temp_wf)
def test_infinite_looping_upgrade(self):
"""Test that circular upgrades fail gracefully"""
# Expected actions are as follow:
# - loads workflow2.xml
# * pipeline is missing looping_fix.x version 0.1
# - enables looping_fix.x (version 0.2)
# * pipeline is still missing looping_fix.x version 0.1
# - runs upgrade for looping_fix.x, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.y version 0.1
# * pipeline is missing looping_fix.y version 0.1
# - enables looping_fix.y (version 0.2)
# * pipeline is still missing looping_fix.y version 0.1
# Loop 50 times:
# - runs upgrade for looping_fix.y, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.x version 0.1
# * pipeline is missing looping_fix.x version 0.1
# - runs upgrade for looping_fix.x, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.y version 0.1
# * pipeline is missing looping_fix.y version 0.1
# 50 calls to handle_invalid_pipeline()
# Pre-adds packages so that the package manager can find them
packages = ["pkg_x", "pkg_y"]
prefix = "vistrails.tests.resources.looping_upgrades."
pm = get_package_manager()
for pkg in packages:
pm.get_available_package(pkg, prefix=prefix)
# Hooks handle_invalid_pipeline()
from vistrails.core.vistrail.controller import VistrailController
orig_hip = VistrailController.handle_invalid_pipeline
count = [0]
def new_hip(*args, **kwargs):
count[0] += 1
return orig_hip(*args, **kwargs)
VistrailController.handle_invalid_pipeline = new_hip
try:
# Loads workflow.xml
from vistrails.core.db.io import load_vistrail
from vistrails.core.db.locator import FileLocator
from vistrails.core.system import vistrails_root_directory
locator = FileLocator(
os.path.join(vistrails_root_directory(), "tests", "resources", "looping_upgrades", "workflow2.xml")
)
loaded_objs = load_vistrail(locator)
controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
# Select version (triggers all the validation/upgrade/loading)
self.assertEqual(controller.get_latest_version_in_graph(), 1)
try:
controller.do_version_switch(1)
except InvalidPipeline:
pass
else:
self.fail("No InvalidPipeline exception raised!")
# Restores handle_invalid_pipeline()
finally:
VistrailController.handle_invalid_pipeline = orig_hip
# disable packages
for pkg in reversed(packages):
try:
pm.late_disable_package(pkg)
except MissingPackage:
pass
# make sure it looped 50 times before failing
max_loops = getattr(get_vistrails_configuration(), "maxPipelineFixAttempts", 50)
self.assertEqual(count[0], max_loops)
# Check that original version gets selected
self.assertEqual(1, controller.current_version)
def run_and_get_results(w_list,
parameters='',
update_vistrail=True,
extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
output_dir:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
params = []
result = []
for locator, workflow in w_list:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(v,
locator,
abstractions,
thumbnails,
mashups,
auto_save=update_vistrail)
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos + 1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
elif 'mashup_id' in extra_info:
# new-style mashups can have aliases not existing in pipeline
for mashuptrail in mashups:
if mashuptrail.vtVersion == version:
mashup = mashuptrail.getMashup(
extra_info['mashup_id'])
c = mashup.getAliasByName(key).component
params.append((c.vttype, c.vtid, value))
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
jobMonitor = controller.jobMonitor
current_workflow = jobMonitor.currentWorkflow()
if not current_workflow:
for job in jobMonitor.workflows.itervalues():
try:
job_version = int(job.version)
except ValueError:
try:
job_version = v.get_version_number(job.version)
except KeyError:
# this is a PE or mashup
continue
if version == job_version:
current_workflow = job
jobMonitor.startWorkflow(job)
if not current_workflow:
current_workflow = JobWorkflow(version)
jobMonitor.startWorkflow(current_workflow)
try:
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
custom_params=params,
extra_info=extra_info,
reason=reason)
finally:
jobMonitor.finishWorkflow()
new_version = controller.current_version
if new_version != version:
debug.log("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" %
(workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
if current_workflow.jobs:
if current_workflow.completed():
run.job = "COMPLETED"
else:
run.job = "RUNNING: %s" % current_workflow.id
for job in current_workflow.jobs.itervalues():
if not job.finished:
run.job += "\n %s %s %s" % (job.start, job.name,
job.description())
print run.job
return result
def run_and_get_results(w_list, parameters='', workflow_info=None,
update_vistrail=True, extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
workflow_info:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
result = []
for locator, workflow in w_list:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(v, locator, abstractions, thumbnails,
mashups, auto_save=update_vistrail)
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos+1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
if workflow_info is not None and controller.current_pipeline is not None:
# FIXME DAK: why is this always done?!? there is a flag for it...
if is_running_gui():
controller.updatePipelineScene()
base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version)
filename = os.path.join(workflow_info, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
debug.critical("Cannot save pipeline figure when not "
"running in gui mode")
base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version)
filename = os.path.join(workflow_info, base_fname)
vistrails.core.db.io.save_workflow(controller.current_pipeline, filename)
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
extra_info=extra_info,
reason=reason)
new_version = controller.current_version
if new_version != version:
debug.warning("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" % \
(workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
return result
def test_looping_pipeline_fix(self):
"""Chains upgrades and automatic package initialization."""
# Expected actions are as follow:
# - loads workflow.xml
# * pipeline is missing looping_fix.a version 0.1
# - enables looping_fix.a (version 0.2)
# * pipeline is still missing looping_fix.a version 0.1
# - runs upgrade for looping_fix.a, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.b version 0.1
# * pipeline is missing looping_fix.b version 0.1
# - enables looping_fix.b (version 0.2)
# * pipeline is still missing looping_fix.b version 0.1
# - runs upgrade for looping_fix.b, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.c version 1.0
# * pipeline is missing looping_fix.c version 1.0
# - enables looping_fix.c (version 1.0)
# * pipeline is valid
# 5 calls to handle_invalid_pipeline()
# Pre-adds packages so that the package manager can find them
packages = ["pkg_a", "pkg_b", "pkg_c"]
prefix = "vistrails.tests.resources.looping_upgrades."
pm = get_package_manager()
for pkg in packages:
pm.get_available_package(pkg, prefix=prefix)
self.assertFalse(set(pkg.codepath for pkg in pm.enabled_package_list()).intersection(packages))
# Hooks handle_invalid_pipeline()
from vistrails.core.vistrail.controller import VistrailController
orig_hip = VistrailController.handle_invalid_pipeline
count = [0]
def new_hip(*args, **kwargs):
count[0] += 1
return orig_hip(*args, **kwargs)
VistrailController.handle_invalid_pipeline = new_hip
try:
# Loads workflow.xml
from vistrails.core.db.io import load_vistrail
from vistrails.core.db.locator import FileLocator
from vistrails.core.system import vistrails_root_directory
locator = FileLocator(
os.path.join(vistrails_root_directory(), "tests", "resources", "looping_upgrades", "workflow.xml")
)
loaded_objs = load_vistrail(locator)
controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
# Select version (triggers all the validation/upgrade/loading)
self.assertEqual(controller.get_latest_version_in_graph(), 1)
controller.do_version_switch(1)
self.assertEqual(count[0], 5)
# Restores handle_invalid_pipeline()
finally:
VistrailController.handle_invalid_pipeline = orig_hip
# disable packages
for pkg in reversed(packages):
try:
pm.late_disable_package(pkg)
except MissingPackage:
pass
def run_and_get_results(w_list, parameters='', output_dir=None,
update_vistrail=True, extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
output_dir:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
params = []
result = []
for locator, workflow in w_list:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(v, locator, abstractions, thumbnails,
mashups, auto_save=update_vistrail)
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos+1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
elif 'mashup_id' in extra_info:
# new-style mashups can have aliases not existing in pipeline
for mashuptrail in mashups:
if mashuptrail.vtVersion == version:
mashup = mashuptrail.getMashup(extra_info['mashup_id'])
c = mashup.getAliasByName(key).component
params.append((c.vttype, c.vtid, value))
if output_dir is not None and controller.current_pipeline is not None:
# FIXME DAK: why is this always done?!? there is a flag for it...
if is_running_gui():
controller.updatePipelineScene()
base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
debug.critical("Cannot save pipeline figure when not "
"running in gui mode")
base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
vistrails.core.db.io.save_workflow(controller.current_pipeline, filename)
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
jobMonitor = controller.jobMonitor
current_workflow = jobMonitor.currentWorkflow()
if not current_workflow:
for job in jobMonitor.workflows.itervalues():
try:
job_version = int(job.version)
except ValueError:
job_version = v.get_version_number(job.version)
if version == job_version:
current_workflow = job
jobMonitor.startWorkflow(job)
if not current_workflow:
current_workflow = JobWorkflow(version)
jobMonitor.startWorkflow(current_workflow)
try:
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
custom_params=params,
extra_info=extra_info,
reason=reason)
finally:
jobMonitor.finishWorkflow()
new_version = controller.current_version
if new_version != version:
debug.log("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" % (
workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
if current_workflow.jobs:
if current_workflow.completed():
run.job = "COMPLETED"
else:
run.job = "RUNNING: %s" % current_workflow.id
for job in current_workflow.jobs.itervalues():
if not job.finished:
run.job += "\n %s %s %s" % (job.start, job.name, job.description())
print run.job
return result
def push_vistrail_to_repository(self, branching=False):
""" uploads current VisTrail to web repository """
self._repository_status['details'] = "Pushing to repository..."
self._push_button.setEnabled(False)
self._branch_button.setEnabled(False)
self.update_push_information()
try:
# create temp file
(fd, filename) = tempfile.mkstemp(suffix='.vt', prefix='vt_tmp')
os.close(fd)
# writing tmp vt and switching back to orginal vt
locator = ZIPFileLocator(filename)
controller = vistrails.api.get_current_controller()
tmp_controller = VistrailController(controller.vistrail.do_copy(),
locator)
tmp_controller.changed = True
tmp_controller.write_vistrail(locator)
# check if this vt is from the repository
if controller.vistrail.get_annotation('repository_vt_id'):
repository_vt_id = controller.vistrail.get_annotation('repository_vt_id').value
else:
repository_vt_id = -1
# upload vistrail temp file to repository
register_openers(cookiejar=self.dialog.cookiejar)
project = self.serverCombo.itemData(self.serverCombo.currentIndex())[0]
if project == "Default": project = ""
params = {'vistrail_file': open(filename, 'rb'),
'action': 'upload',
'name': controller.locator.short_name,
'repository_vt_id': repository_vt_id if not branching else -1,
'is_runnable': not bool(len(self.unsupported_packages)+ \
len(self.local_data_modules)),
'vt_id': 0,
'branched_from': "" if not branching else repository_vt_id,
'project': project,
'everyone_can_view': self.perm_view.checkState(),
'everyone_can_edit': self.perm_edit.checkState(),
'everyone_can_download': self.perm_download.checkState()
}
upload_url = "%s/vistrails/remote_upload/" % \
self.config.webRepositoryURL
datagen, headers = multipart_encode(params)
request = urllib2.Request(upload_url, datagen, headers)
result = urllib2.urlopen(request)
updated_response = result.read()
os.unlink(filename)
if updated_response[:6] == "upload":
# No update, just upload
if result.code != 200:
self._repository_status['details'] = \
"Push to repository failed"
debug.critical("Push to repository failed (Please contact an administrator)")
else:
repository_vt_id = int(updated_response[8:])
controller.vistrail.set_annotation('repository_vt_id',
repository_vt_id)
controller.vistrail.set_annotation('repository_creator',
self.dialog.loginUser)
# ensure that the annotations get saved
controller.set_changed(True)
self._repository_status['details'] = \
"Push to repository was successful"
else:
# update, load updated vistrail
if result.code != 200:
self._repository_status['details'] = "Update Failed"
debug.critical("Update vistrail in web repository failed (Please contact an administrator)")
else:
debug.log("getting version from web")
# request file to download
download_url = "%s/vistrails/download/%s/" % \
(self.config.webRepositoryURL, updated_response)
request = urllib2.Request(download_url)
result = urllib2.urlopen(request)
updated_file = result.read()
# create temp file of updated vistrail
(fd, updated_filename) = tempfile.mkstemp(suffix='.vtl',
prefix='vtl_tmp')
os.close(fd)
updated_vt = open(updated_filename, 'w')
updated_vt.write(updated_file)
updated_vt.close()
# switch vistrails to updated one
controller = vistrails.api.get_current_controller()
updated_locator = FileLocator(updated_filename)
(up_vistrail, abstractions, thumbnails, mashups) = \
load_vistrail(updated_locator)
# FIXME need to figure out what to do with this !!!
controller.set_vistrail(up_vistrail,
controller.vistrail.locator,
abstractions, thumbnails, mashups)
# update version tree drawing
controller.recompute_terse_graph()
controller.invalidate_version_tree()
os.remove(updated_filename)
os.remove(updated_filename[:-1])
self._repository_status['details'] = \
"Update to repository was successful"
except Exception, e:
debug.critical("An error occurred", str(e))
self._repository_status['details'] = "An error occurred"
def execute(workflowJSON):
''' Execute a workflow from it's JSON representation
'''
debug('convert json to xml')
workflowXML = json2xml(workflowJSON)
#temp_wf_fd, temp_wf = tempfile.mkstemp('.xml')
debug('create temporary file')
temp_wf_fd, temp_wf = tempfile.mkstemp()
try:
f = open(temp_wf, 'w')
f.write(workflowXML)
f.close()
os.close(temp_wf_fd)
#load workflow temp file into vistrails
#vt.load_workflow(temp_wf)
#execute workflow
#execution = vt.execute()
debug('Load the Pipeline from the temporary file')
vistrail = Vistrail()
locator = XMLFileLocator(temp_wf)
workflow = locator.load(Pipeline)
debug('Build a Vistrail from this single Pipeline')
action_list = []
for module in workflow.module_list:
action_list.append(('add', module))
for connection in workflow.connection_list:
action_list.append(('add', connection))
action = vistrails.core.db.action.create_action(action_list)
debug('add actions')
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
tag = 'climatepipes'
vistrail.addTag(tag, action.id)
debug('Build a controller and execute')
controller = VistrailController()
controller.set_vistrail(vistrail, None)
controller.change_selected_version(vistrail.get_version_number(tag))
execution = controller.execute_current_workflow(
custom_aliases=None,
custom_params=None,
extra_info=None,
reason='API Pipeline Execution')
debug('get result')
execution_pipeline = execution[0][0]
if len(execution_pipeline.errors) > 0:
error("Executing workflow")
for key in execution_pipeline.errors:
error(execution_pipeline.errors[key])
print execution_pipeline.errors[key]
return None
modules = execution_pipeline.objects
for id, module in modules.iteritems():
if isinstance(module, ToGeoJSON):
return json.dumps({'result': module.JSON, 'error': None })
finally:
os.unlink(temp_wf)
class Vistrail(object):
"""This class wraps both Vistrail and VistrailController.
From it, you can get any pipeline from a tag name or version number.
It has a concept of "current version", from which you can create new
versions by performing actions.
"""
_current_pipeline = None
_html = None
def __init__(self, arg=None):
initialize()
if arg is None:
# Copied from VistrailsApplicationInterface#open_vistrail()
locator = UntitledLocator()
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
self.controller = VistrailController(loaded_objs[0], locator,
*loaded_objs[1:])
elif isinstance(arg, (_Pipeline, Pipeline)):
if isinstance(arg, Pipeline):
pipeline = arg.pipeline
else:
pipeline = arg
# Copied from VistrailsApplicationInterface#open_workflow()
vistrail = _Vistrail()
ops = []
for module in pipeline.module_list:
ops.append(('add', module))
for connection in pipeline.connection_list:
ops.append(('add', connection))
action = vistrails.core.db.action.create_action(ops)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
vistrail.change_description("Imported pipeline", 0L)
self.controller = VistrailController(vistrail, UntitledLocator())
elif isinstance(arg, VistrailController):
self.controller = arg
elif isinstance(arg, basestring):
raise TypeError("Vistrail was constructed from %r.\n"
"Use load_vistrail() to get a Vistrail from a "
"file." % type(arg).__name__)
else:
raise TypeError("Vistrail was constructed from unexpected "
"argument type %r" % type(arg).__name__)
def get_pipeline(self, version):
"""Returns a pipeline from a version number of tag.
This does not change the currently selected version in this Vistrail.
"""
vistrail = self.controller.vistrail
if isinstance(version, (int, long)):
if not vistrail.db_has_action_with_id(version):
raise NoSuchVersion("Vistrail doesn't have a version %r" %
version)
return Pipeline(vistrail.getPipelineVersionNumber(version))
elif isinstance(version, basestring):
if not vistrail.has_tag_str(version):
raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
return Pipeline(vistrail.getPipelineVersionName(version))
else:
raise TypeError("get_pipeline() argument must be a string or "
"integer, not %r" % type(version).__name__)
def select_version(self, version):
"""Sets a different version as current.
The current workflow is accessible via current_workflow; it is the one
that gets executed when calling execute(), and the version from which
new versions are created if you perform actions.
"""
vistrail = self.controller.vistrail
if isinstance(version, (int, long)):
if not vistrail.db_has_action_with_id(version):
raise NoSuchVersion("Vistrail doesn't have a version %r" %
version)
elif (isinstance(version, basestring)):
if not vistrail.has_tag_str(version):
raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
version = vistrail.get_tag_str(version).action_id
else:
raise TypeError("select_version() argument must be a string "
"or integer, not %r" % type(version).__name__)
self.controller.do_version_switch(version)
self._current_pipeline = None
self._html = None
def select_latest_version(self):
"""Sets the most recent version in the vistrail as current.
"""
self.controller.do_version_switch(
self.controller.get_latest_version_in_graph())
self._current_pipeline = None
self._html = None
@property
def current_pipeline(self):
if self._current_pipeline is None:
self._current_pipeline = Pipeline(
self.controller.current_pipeline,
vistrail=(self, self.current_version))
return self._current_pipeline
@property
def current_version(self):
return self.controller.current_version
def set_tag(self, *args):
"""Sets a tag for the current or specified version.
"""
if len(args) == 1:
version, (tag,) = self.controller.current_version, args
elif len(args) == 2:
version, tag = args
else:
raise TypeError("set_tag() takes 1 or 2 arguments (%r given)" %
len(args))
if isinstance(version, (int, long)):
if not self.controller.vistrail.db_has_action_with_id(version):
raise NoSuchVersion("Vistrail doesn't have a version %r" %
version)
elif isinstance(version, basestring):
if not self.controller.vistrail.has_tag_str(version):
raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
else:
raise TypeError("set_tag() expects the version to be a string or "
"integer, not %r" % type(version).__name__)
self.controller.vistrail.set_tag(version, tag)
def tag(self, tag):
"""Sets a tag for the current version.
"""
self.set_tag(tag)
def execute(self, *args, **kwargs):
"""Executes the current workflow.
"""
return self.current_pipeline.execute(*args, **kwargs)
@property
def changed(self):
return self.controller.changed
# TODO : vistrail modification methods
def __repr__(self):
version_nb = self.controller.current_version
if self.controller.vistrail.has_tag(version_nb):
version = "%s (tag %s)" % (
version_nb,
self.controller.vistrail.get_tag(version_nb))
else:
version = version_nb
return "<%s: %s, version %s, %s>" % (
self.__class__.__name__,
self.controller.name,
version,
('not changed', 'changed')[self.controller.changed])
def _repr_html_(self):
if self._html is None:
import cgi
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
self._html = ''
stream = StringIO()
self.controller.recompute_terse_graph()
self.controller.save_version_graph(
stream,
highlight=self.controller.current_version)
stream.seek(0)
dot = stream.read()
try:
proc = subprocess.Popen(['dot', '-Tsvg'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
svg, _ = proc.communicate(dot)
if proc.wait() == 0:
self._html += svg
except OSError:
pass
self._html += '<pre>' + cgi.escape(repr(self)) + '</pre>'
return self._html
class Vistrail(object):
"""This class wraps both Vistrail and VistrailController.
From it, you can get any pipeline from a tag name or version number.
It has a concept of "current version", from which you can create new
versions by performing actions.
"""
_current_pipeline = None
_html = None
def __init__(self, arg=None):
initialize()
if arg is None:
# Copied from VistrailsApplicationInterface#open_vistrail()
locator = UntitledLocator()
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
self.controller = VistrailController(loaded_objs[0], locator,
*loaded_objs[1:])
elif isinstance(arg, (_Pipeline, Pipeline)):
if isinstance(arg, Pipeline):
pipeline = arg.pipeline
else:
pipeline = arg
# Copied from VistrailsApplicationInterface#open_workflow()
vistrail = _Vistrail()
ops = []
for module in pipeline.module_list:
ops.append(('add', module))
for connection in pipeline.connection_list:
ops.append(('add', connection))
action = vistrails.core.db.action.create_action(ops)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
vistrail.change_description("Imported pipeline", 0L)
self.controller = VistrailController(vistrail, UntitledLocator())
elif isinstance(arg, VistrailController):
self.controller = arg
elif isinstance(arg, basestring):
raise TypeError("Vistrail was constructed from %r.\n"
"Use load_vistrail() to get a Vistrail from a "
"file." % type(arg).__name__)
else:
raise TypeError("Vistrail was constructed from unexpected "
"argument type %r" % type(arg).__name__)
def get_pipeline(self, version):
"""Returns a pipeline from a version number of tag.
This does not change the currently selected version in this Vistrail.
"""
vistrail = self.controller.vistrail
if isinstance(version, (int, long)):
if not vistrail.db_has_action_with_id(version):
raise NoSuchVersion("Vistrail doesn't have a version %r" %
version)
return Pipeline(vistrail.getPipelineVersionNumber(version))
elif isinstance(version, basestring):
if not vistrail.has_tag_str(version):
raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
return Pipeline(vistrail.getPipelineVersionName(version))
else:
raise TypeError("get_pipeline() argument must be a string or "
"integer, not %r" % type(version).__name__)
def select_version(self, version):
"""Sets a different version as current.
The current workflow is accessible via current_workflow; it is the one
that gets executed when calling execute(), and the version from which
new versions are created if you perform actions.
"""
vistrail = self.controller.vistrail
if isinstance(version, (int, long)):
if not vistrail.db_has_action_with_id(version):
raise NoSuchVersion("Vistrail doesn't have a version %r" %
version)
elif (isinstance(version, basestring)):
if not vistrail.has_tag_str(version):
raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
version = vistrail.get_tag_str(version).action_id
else:
raise TypeError("select_version() argument must be a string "
"or integer, not %r" % type(version).__name__)
self.controller.do_version_switch(version)
self._current_pipeline = None
self._html = None
def select_latest_version(self):
"""Sets the most recent version in the vistrail as current.
"""
self.controller.do_version_switch(
self.controller.get_latest_version_in_graph())
self._current_pipeline = None
self._html = None
@property
def current_pipeline(self):
if self._current_pipeline is None:
self._current_pipeline = Pipeline(self.controller.current_pipeline,
vistrail=(self,
self.current_version))
return self._current_pipeline
@property
def current_version(self):
return self.controller.current_version
def set_tag(self, *args):
"""Sets a tag for the current or specified version.
"""
if len(args) == 1:
version, (tag, ) = self.controller.current_version, args
elif len(args) == 2:
version, tag = args
else:
raise TypeError("set_tag() takes 1 or 2 arguments (%r given)" %
len(args))
if isinstance(version, (int, long)):
if not self.controller.vistrail.db_has_action_with_id(version):
raise NoSuchVersion("Vistrail doesn't have a version %r" %
version)
elif isinstance(version, basestring):
if not self.controller.vistrail.has_tag_str(version):
raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
else:
raise TypeError("set_tag() expects the version to be a string or "
"integer, not %r" % type(version).__name__)
self.controller.vistrail.set_tag(version, tag)
def tag(self, tag):
"""Sets a tag for the current version.
"""
self.set_tag(tag)
def execute(self, *args, **kwargs):
"""Executes the current workflow.
"""
return self.current_pipeline.execute(*args, **kwargs)
@property
def changed(self):
return self.controller.changed
# TODO : vistrail modification methods
def __repr__(self):
version_nb = self.controller.current_version
if self.controller.vistrail.has_tag(version_nb):
version = "%s (tag %s)" % (
version_nb, self.controller.vistrail.get_tag(version_nb))
else:
version = version_nb
return "<%s: %s, version %s, %s>" % (
self.__class__.__name__, self.controller.name, version,
('not changed', 'changed')[self.controller.changed])
def _repr_html_(self):
if self._html is None:
import cgi
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
self._html = ''
stream = StringIO()
self.controller.recompute_terse_graph()
self.controller.save_version_graph(
stream, highlight=self.controller.current_version)
stream.seek(0)
dot = stream.read()
try:
proc = subprocess.Popen(['dot', '-Tsvg'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
svg, _ = proc.communicate(dot)
if proc.wait() == 0:
self._html += svg
except OSError:
pass
self._html += '<pre>' + cgi.escape(repr(self)) + '</pre>'
return self._html
def exportMashup(filename, vtcontroller, mashuptrail, mashupversion,
etype):
"""exportMashup(filename: str, vtcontroller: VistrailController,
mashuptrail: Mashuptrail, type: int) -> bool
where etype is
0: include full tree
1: include only workflow and mashup identified by version
2: as a link, it will point to a local file.
"""
result = False
if vtcontroller is not None and mashuptrail is not None:
locator = vtcontroller.locator
version = mashuptrail.vtVersion
node = ElementTree.Element('vtlink')
if isinstance(locator, DBLocator):
node.set('host', str(locator.host))
node.set('port', str(locator.port))
node.set('database', str(locator.db))
node.set('vtid', str(locator.obj_id))
else:
node.set('filename', str(locator.name))
node.set('version', str(version))
node.set('execute', "True")
node.set('forceDB', "False")
node.set('showSpreadsheetOnly', "True")
node.set('mashuptrail', str(mashuptrail.id))
node.set('mashupVersion', str(mashupversion))
if etype in [0, 1]:
if etype == 1: #minimal
pip = vtcontroller.vistrail.getPipeline(version)
vistrail = Vistrail()
id_remap = {}
action = vistrails.core.db.action.create_paste_action(
pip, vistrail.idScope, id_remap)
vistrail.add_action(action, 0L, 0)
tag = vtcontroller.vistrail.get_tag(version)
if tag is None:
tag = "Imported workflow"
vistrail.addTag(tag, action.id)
node.set('version', str(action.id))
id_scope = IdScope(1L)
newmashuptrail = Mashuptrail(
MashupsManager.getNewMashuptrailId(), action.id,
id_scope)
maction = mashuptrail.actionMap[mashupversion]
mtag = mashuptrail.getTagForActionId(mashupversion)
newmashup = copy.copy(maction.mashup)
newmashup.remapPipelineObjects(id_remap)
currVersion = newmashuptrail.addVersion(
newmashuptrail.getLatestVersion(), newmashup,
maction.user, maction.date)
newmashuptrail.currentVersion = currVersion
newmashuptrail.changeTag(currVersion, mtag, maction.user,
maction.date)
newvtcontroller = BaseVistrailController(vistrail, None)
MashupsManager.addMashuptrailtoVistrailController(
newvtcontroller, newmashuptrail)
node.set('mashuptrail', str(newmashuptrail.id))
node.set('mashupVersion',
str(newmashuptrail.currentVersion))
else:
vistrail = vtcontroller.vistrail
newvtcontroller = MashupsManager.copyBaseVistrailController(
vtcontroller)
#create temporary file
(fd, name) = tempfile.mkstemp(prefix='vt_tmp', suffix='.vt')
os.close(fd)
try:
fileLocator = FileLocator(name)
newvtcontroller.write_vistrail(fileLocator)
contents = open(name).read()
vtcontent = base64.b64encode(contents)
finally:
os.unlink(name)
#if not vistrail.db_version:
# vistrail.db_version = currentVersion
node.set('vtcontent', vtcontent)
xmlstring = ElementTree.tostring(node)
file_ = open(filename, 'w')
file_.write(xmlstring)
file_.close()
result = True
return result
sender = context.socket(zmq.PUSH)
sender.connect("tcp://{0}:{1}".format(HOST, SEND))
while True:
# Receiving pipeline instance configuration
data = receiver.recv()
logging.debug('Receiving: ' + data)
fields = data.split("|")
filename = fields[0]
parameter_list = ast.literal_eval(fields[1])
inputs = ast.literal_eval(fields[2])
outputs = ast.literal_eval(fields[3])
locator = FileLocator(filename)
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
controller.do_version_switch(controller.get_latest_version_in_graph())
pipeline = Pipeline(controller)
kwargs = {}
for i in range(len(parameter_list)):
kwargs[inputs[i]] = parameter_list[i]
try:
#Executing pipeline instance and retieving the result
result = pipeline.execute(**kwargs)
for output in outputs:
parameter_list.append(str(result.output_port(output)))
except:
traceback.print_exc(file=sys.stdout)
parameter_list.append(str(False))
kwargs['result'] = parameter_list[-1]