def copyVistrailController(vt_controller, view=DummyView()):
vistrail = vt_controller.vistrail
newvt_controller = VistrailController(vistrail, None,
pipeline_view=view)
current_log = vt_controller.log
newvt_controller.log = current_log
newvt_controller.disable_autosave()
return newvt_controller
def createMashupApp(vtcontroller, mashuptrail, version):
from vistrails.gui.mashups.mashup_app import QMashupAppMainWindow
vistrail_view = vtcontroller.vistrail_view
vtVersion = mashuptrail.vtVersion
view = DummyView()
view.scene().current_pipeline = vtcontroller.vistrail.getPipeline(vtVersion)
view.scene().current_pipeline.validate()
new_vtcontroller = MashupsManager.copyBaseVistrailController(vtcontroller)
new_vtcontroller.change_selected_version(vtVersion)
mshpController = MashupController(vtcontroller,
new_vtcontroller,
vtVersion, mashuptrail)
mshpController.setCurrentVersion(version)
app = QMashupAppMainWindow(parent=None,
vistrail_view=vistrail_view,
controller=mshpController,
version=version)
return app
def createMashupApp(vtcontroller, mashuptrail, version):
from vistrails.gui.mashups.mashup_app import QMashupAppMainWindow
vistrail_view = vtcontroller.vistrail_view
vtVersion = mashuptrail.vtVersion
view = DummyView()
view.scene().current_pipeline = vtcontroller.vistrail.getPipeline(vtVersion)
view.scene().current_pipeline.validate()
new_vtcontroller = MashupsManager.copyBaseVistrailController(vtcontroller)
new_vtcontroller.change_selected_version(vtVersion)
mshpController = MashupController(vtcontroller,
new_vtcontroller,
vtVersion, mashuptrail)
mshpController.setCurrentVersion(version)
app = QMashupAppMainWindow(parent=None,
vistrail_view=vistrail_view,
controller=mshpController,
version=version)
return app
def test_cache(self):
from vistrails.core.modules.basic_modules import StandardOutput
old_compute = StandardOutput.compute
StandardOutput.compute = lambda s: None
try:
from vistrails.core.db.locator import XMLFileLocator
from vistrails.core.vistrail.controller import VistrailController
from vistrails.core.db.io import load_vistrail
"""Test if basic caching is working."""
locator = XMLFileLocator(
vistrails.core.system.vistrails_root_directory() +
'/tests/resources/dummy.xml')
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
# the controller will take care of upgrades
controller = VistrailController(v, locator, abstractions,
thumbnails, mashups)
p1 = v.getPipeline('int chain')
n = v.get_version_number('int chain')
controller.change_selected_version(n)
controller.flush_delayed_actions()
p1 = controller.current_pipeline
view = DummyView()
interpreter = CachedInterpreter.get()
result = interpreter.execute(
p1,
locator=v,
current_version=n,
view=view,
)
# to force fresh params
p2 = v.getPipeline('int chain')
controller.change_selected_version(n)
controller.flush_delayed_actions()
p2 = controller.current_pipeline
result = interpreter.execute(
p2,
locator=v,
current_version=n,
view=view,
)
self.assertEqual(len(result.modules_added), 1)
finally:
StandardOutput.compute = old_compute
def test_pipeline_creation(self):
import dat.tests.pkg_test_plots.init as pkg_test_plots
controller = self.vt_controller()
vistraildata = VistrailManager(controller)
loader = Test_generation._loaders.get('StrMaker')
loader.v = 'Hello'
vistraildata.new_variable('var1', loader.load())
loader.v = 'world'
vistraildata.new_variable('var2', loader.load())
cellInfo = FakeObj(
row=0,
column=0,
tab=FakeObj(tabWidget=FakeObj(tabText=lambda w: 'Sheet 1')))
recipe = DATRecipe(
pkg_test_plots.concat_plot, {
'param1': (RecipeParameterValue(
variable=vistraildata.get_variable('var1')), ),
'param2': (RecipeParameterValue(
variable=vistraildata.get_variable('var2')), ),
'param3': (RecipeParameterValue(constant="!"), ),
})
pipelineInfo = vistrails_interface.create_pipeline(
controller, recipe, cellInfo.row, cellInfo.column,
None) # This plot has no cell module so this is fine
controller.change_selected_version(pipelineInfo.version)
result = CallRecorder()
pkg_test_plots.Recorder.callback = result
interpreter = get_default_interpreter()
interpreter.execute(controller.current_pipeline,
view=DummyView(),
locator=controller.locator,
current_version=pipelineInfo.version)
call = (['Hello, world!'], dict())
self.assertEqual(result.calls, [call])
def execute(self):
""" execute() -> None
Execute the selected pipeline with the edited aliases
"""
aliases = {}
for r in xrange(self.aliasTable.rowCount()):
name = str(self.aliasTable.item(r, 0).text())
value = str(self.aliasTable.item(r, 1).text())
aliases[name] = value
versionNumber = self.versions[
self.pipelineList.currentIndex().row()][0]
pipeline = self.vistrail.getPipelineVersionNumber(versionNumber)
controller = VistrailController(self.vistrail)
controller.execute_workflow_list([
(self.vistrail.locator, versionNumber, pipeline, DummyView(),
aliases, None)
])
def test_tuple(self):
from vistrails.core.vistrail.module_param import ModuleParam
from vistrails.core.vistrail.module_function import ModuleFunction
from vistrails.core.utils import DummyView
from vistrails.core.vistrail.module import Module
import vistrails.db.domain
id_scope = vistrails.db.domain.IdScope()
interpreter = vistrails.core.interpreter.default.get_default_interpreter(
)
v = DummyView()
p = vistrails.core.vistrail.pipeline.Pipeline()
params = [
ModuleParam(
id=id_scope.getNewId(ModuleParam.vtType),
pos=0,
type='Float',
val='2.0',
),
ModuleParam(
id=id_scope.getNewId(ModuleParam.vtType),
pos=1,
type='Float',
val='2.0',
)
]
function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType),
name='input')
function.add_parameters(params)
module = Module(id=id_scope.getNewId(Module.vtType),
name='TestTupleExecution',
package='org.vistrails.vistrails.console_mode_test',
version='0.9.1')
module.add_function(function)
p.add_module(module)
interpreter.execute(p,
locator=XMLFileLocator('foo'),
current_version=1L,
view=v)
def execute_pipeline(controller, pipeline, reason, locator, version, **kwargs):
"""Execute the pipeline while showing a progress dialog.
"""
_ = translate('execute_pipeline')
totalProgress = len(pipeline.modules)
progress = QtGui.QProgressDialog(_("Executing..."), None, 0, totalProgress)
progress.setWindowTitle(_("Pipeline Execution"))
progress.setWindowModality(QtCore.Qt.WindowModal)
progress.show()
def moduleExecuted(objId):
progress.setValue(progress.value() + 1)
QtCore.QCoreApplication.processEvents()
if 'module_executed_hook' in kwargs:
kwargs['module_executed_hook'].append(moduleExecuted)
else:
kwargs['module_executed_hook'] = [moduleExecuted]
results, changed = controller.execute_workflow_list([(
locator, # locator
version, # version
pipeline, # pipeline
DummyView(), # view
None, # custom_aliases
None, # custom_params
reason, # reason
None, # sinks
kwargs)]) # extra_info
get_vistrails_application().send_notification('execution_updated')
progress.setValue(totalProgress)
progress.hide()
progress.deleteLater()
if not results[0].errors:
return None
else:
module_id, error = next(results[0].errors.iteritems())
return str(error)
def executePipelineWithProgress(pipeline,
pTitle='Pipeline Execution',
pCaption='Executing...',
pCancel='&Cancel',
**kwargs):
""" executePipelineWithProgress(pipeline: Pipeline,
pTitle: str, pCaption: str, pCancel: str,
kwargs: keyword arguments) -> bool
Execute the pipeline while showing a progress dialog with title
pTitle, caption pCaption and the cancel button text
pCancel. kwargs is the keyword arguments that will be passed to
the interpreter. A bool will be returned indicating if the
execution was performed without cancel or not.
"""
withoutCancel = True
totalProgress = len(pipeline.modules)
progress = QtGui.QProgressDialog(pCaption,
pCancel,
0, totalProgress)
progress.setWindowTitle(pTitle)
progress.setWindowModality(QtCore.Qt.WindowModal)
progress.show()
def moduleExecuted(objId):
if not progress.wasCanceled():
progress.setValue(progress.value()+1)
QtCore.QCoreApplication.processEvents()
else:
withoutCancel = False
interpreter = get_default_interpreter()
if kwargs.has_key('module_executed_hook'):
kwargs['module_executed_hook'].append(moduleExecuted)
else:
kwargs['module_executed_hook'] = [moduleExecuted]
kwargs['view'] = DummyView()
interpreter.execute(pipeline, **kwargs)
progress.setValue(totalProgress)
return withoutCancel
def openSpreadsheet(self, fileName):
""" openSpreadsheet(fileName: str) -> None
Open a saved spreadsheet assuming that all VTK files must exist and have
all the version using the saved spreadsheet
"""
def parse_locator(text):
locator = None
wrapper = XMLWrapper()
dom = wrapper.create_document_from_string(text)
root = dom.documentElement
version = None
version = root.getAttribute('version')
if version == '1.0':
for element in named_elements(root, 'locator'):
if str(element.getAttribute('type')) == 'file':
locator = FileLocator.parse(element)
elif str(element.getAttribute('type')) == 'db':
locator = DBLocator.parse(element)
return locator
locators = {}
indexFile = open(fileName, 'r')
contents = indexFile.read()
self.clearTabs()
lidx = 0
lines = contents.split('\n')
tabCount = int(lines[lidx])
lidx += 1
for tabIdx in xrange(tabCount):
# FIXME: eval should pretty much never be used
tabInfo = literal_eval(lines[lidx])
lidx += 1
sheet = spreadsheetRegistry.getSheet(tabInfo[1])(self)
sheet.setDimension(tabInfo[2], tabInfo[3])
self.addTabWidget(sheet, tabInfo[0])
while lines[lidx] != '---':
(r, c, vistrail, pid, cid) = literal_eval(lines[lidx])
locator = vistrail['locator']
if locators.has_key(locator):
vistrail['locator'] = locators[locator]
else:
locators[locator] = parse_locator(vistrail['locator'])
vistrail['locator'] = locators[locator]
self.appendMonitoredLocations((vistrail, pid, cid),
(sheet, r, c))
lidx += 1
lidx += 1
pipelineCount = int(lines[lidx])
lidx += 1
self.loadingMode = True
progress = QtGui.QProgressDialog("Loading spreadsheet...", "&Cancel",
0, pipelineCount, self,
QtCore.Qt.WindowStaysOnTopHint)
progress.show()
for pipelineIdx in xrange(pipelineCount):
# FIXME: eval should pretty much never be used
(serializedLocator, version) = literal_eval(lines[lidx])
try:
locator = locators[serializedLocator]
except KeyError:
locator = parse_locator(serializedLocator)
if locator:
bundle = locator.load()
if isinstance(bundle, SaveBundle):
pipeline = bundle.vistrail.getPipeline(version)
else:
pipeline = bundle.getPipeline(version)
execution = get_default_interpreter()
progress.setValue(pipelineIdx)
QtCore.QCoreApplication.processEvents()
if progress.wasCanceled():
break
kwargs = {
'locator': locator,
'current_version': version,
'view': DummyView(),
}
execution.execute(pipeline, **kwargs)
else:
raise RuntimeError("Couldn't load spreadsheet")
lidx += 1
progress.setValue(pipelineCount)
QtCore.QCoreApplication.processEvents()
self.changeSpreadsheetFileName(fileName)
self.loadingMode = False
indexFile.close()
def createMashupController(self, vt_controller, version, view=DummyView()):
#print "Manager creating mashup controller ", vt_controller, version
newvt_controller = MashupsManager.copyVistrailController(
vt_controller, view)
mashuptrail = \
MashupsManager.getMashuptrailforVersionInVistrailController(vt_controller,
version)
if mashuptrail is None:
(p_mashuptrail, p_version) = \
MashupsManager.findClosestParentMashuptrail(vt_controller,
version)
id_scope = IdScope(1L)
if p_mashuptrail is not None:
version_name = vt_controller.get_pipeline_name(p_version)
(res, mshpv) = MashupsManager.showFoundMashupsDialog(
p_mashuptrail, version_name)
if res in ['Copy', 'Move']:
pipeline = newvt_controller.vistrail.getPipeline(version)
if res == 'Copy':
# we will copy the mashup from the parent trail and
# validate it to the current pipeline before adding
# to the current mashup trail
mashuptrail = Mashuptrail(self.getNewMashuptrailId(),
version, id_scope)
p_mashup = p_mashuptrail.getMashup(mshpv)
mashup = p_mashup.do_copy()
mashup.id_scope = id_scope
mashup.version = version
mashup.validateForPipeline(pipeline)
currVersion = mashuptrail.addVersion(
parent_id=mashuptrail.getLatestVersion(),
mashup=mashup,
user=vistrails.core.system.current_user(),
date=vistrails.core.system.current_time())
mashuptrail.currentVersion = currVersion
mashuptrail.updateIdScope()
p_tag = p_mashuptrail.getTagForActionId(mshpv)
if p_tag == '':
tag = "<latest>"
tag = "Copy from %s" % p_tag
MashupsManager.addMashuptrailtoVistrailController(
vt_controller, mashuptrail)
else: # res == 'Move'
# we will move the parent trail and validate all mashups
# for the current pipeline to make sure they will be
# executable for the current version
mashuptrail = p_mashuptrail
currVersion = mashuptrail.getLatestVersion()
mashuptrail.currentVersion = currVersion
mashuptrail.validateMashupsForPipeline(
version, pipeline)
tag = None
mashuptrail.vtVersion = version
mshpController = MashupController(vt_controller,
newvt_controller,
version, mashuptrail)
mshpController.setCurrentVersion(
mashuptrail.currentVersion)
# this is to make sure the pipeline displayed in the mashup
# view is consistent with the list of aliases in the central
# panel
mshpController.updatePipelineAliasesFromCurrentMashup()
if tag is not None:
mshpController.updateCurrentTag(tag)
return mshpController
mashuptrail = Mashuptrail(self.getNewMashuptrailId(), version,
id_scope)
pipeline = newvt_controller.vistrail.getPipeline(version)
id = id_scope.getNewId('mashup')
mashup = Mashup(id=id,
name="mashup%s" % id,
vtid=vt_controller.locator,
version=version)
mashup.loadAliasesFromPipeline(pipeline, id_scope)
currVersion = mashuptrail.addVersion(
parent_id=mashuptrail.getLatestVersion(),
mashup=mashup,
user=vistrails.core.system.current_user(),
date=vistrails.core.system.current_time())
mashuptrail.currentVersion = currVersion
MashupsManager.addMashuptrailtoVistrailController(
vt_controller, mashuptrail)
mshpController = MashupController(vt_controller, newvt_controller,
version, mashuptrail)
mshpController.setCurrentVersion(mashuptrail.currentVersion)
if mshpController.currentVersion == 1L:
mshpController.updateCurrentTag("ROOT")
else:
#print "----> found mashuptrail ", mashuptrail.currentVersion
mshpController = MashupController(vt_controller, newvt_controller,
version, mashuptrail)
mshpController.setCurrentVersion(mashuptrail.currentVersion)
mshpController.updatePipelineAliasesFromCurrentMashup()
return mshpController
def execute(self, *args, **kwargs):
"""Execute the pipeline.
Positional arguments are either input values (created from
``module == value``, where `module` is a Module from the pipeline and
`value` is some value or Function instance) for the pipeline's
InputPorts, or Module instances (to select sink modules).
Keyword arguments are also used to set InputPort by looking up inputs
by name.
Example::
input_bound = pipeline.get_input('higher_bound')
input_url = pipeline.get_input('url')
sinkmodule = pipeline.get_module(32)
pipeline.execute(sinkmodule,
input_bound == vt.Function(Integer, 10),
input_url == 'http://www.vistrails.org/',
resolution=15) # kwarg: only one equal sign
"""
sinks = set()
inputs = {}
reg = get_module_registry()
InputPort_desc = reg.get_descriptor_by_name(
get_vistrails_basic_pkg_id(), 'InputPort')
# Read args
for arg in args:
if isinstance(arg, ModuleValuePair):
if arg.module.id in inputs:
raise ValueError("Multiple values set for InputPort %r" %
get_inputoutput_name(arg.module))
if not reg.is_descriptor_subclass(arg.module.module_descriptor,
InputPort_desc):
raise ValueError("Module %d is not an InputPort" %
arg.module.id)
inputs[arg.module.id] = arg.value
elif isinstance(arg, Module):
sinks.add(arg.module_id)
# Read kwargs
for key, value in kwargs.iteritems():
key = self.get_input(key) # Might raise KeyError
if key.module_id in inputs:
raise ValueError("Multiple values set for InputPort %r" %
get_inputoutput_name(key.module))
inputs[key.module_id] = value
reason = "API pipeline execution"
sinks = sinks or None
# Use controller only if no inputs were passed in
if (not inputs and self.vistrail is not None
and self.vistrail.current_version == self.version):
controller = self.vistrail.controller
results, changed = controller.execute_workflow_list([[
controller.locator, # locator
self.version, # version
self.pipeline, # pipeline
DummyView(), # view
None, # custom_aliases
None, # custom_params
reason, # reason
sinks, # sinks
None, # extra_info
]])
result, = results
else:
pipeline = self.pipeline
if inputs:
id_scope = IdScope(1)
pipeline = pipeline.do_copy(False, id_scope)
# A hach to get ids from id_scope that we know won't collide:
# make them negative
id_scope.getNewId = lambda t, g=id_scope.getNewId: -g(t)
create_module = \
VistrailController.create_module_from_descriptor_static
create_function = VistrailController.create_function_static
create_connection = VistrailController.create_connection_static
# Fills in the ExternalPipe ports
for module_id, values in inputs.iteritems():
module = pipeline.modules[module_id]
if not isinstance(values, (list, tuple)):
values = [values]
# Guess the type of the InputPort
_, sigstrings, _, _, _ = get_port_spec_info(
pipeline, module)
sigstrings = parse_port_spec_string(sigstrings)
# Convert whatever we got to a list of strings, for the
# pipeline
values = [
reg.convert_port_val(val, sigstring, None)
for val, sigstring in izip(values, sigstrings)
]
if len(values) == 1:
# Create the constant module
constant_desc = reg.get_descriptor_by_name(
*sigstrings[0])
constant_mod = create_module(id_scope, constant_desc)
func = create_function(id_scope, constant_mod, 'value',
values)
constant_mod.add_function(func)
pipeline.add_module(constant_mod)
# Connect it to the ExternalPipe port
conn = create_connection(id_scope, constant_mod,
'value', module,
'ExternalPipe')
pipeline.db_add_connection(conn)
else:
raise RuntimeError("TODO : create tuple")
interpreter = get_default_interpreter()
result = interpreter.execute(pipeline, reason=reason, sinks=sinks)
if result.errors:
raise ExecutionErrors(self, result)
else:
return ExecutionResults(self, result)
def execute(self, pipeline, **kwargs):
"""execute(pipeline, **kwargs):
kwargs:
controller = fetch('controller', None)
locator = fetch('locator', None)
current_version = fetch('current_version', None)
view = fetch('view', DummyView())
aliases = fetch('aliases', None)
params = fetch('params', None)
extra_info = fetch('extra_info', None)
logger = fetch('logger', DummyLogController)
reason = fetch('reason', None)
actions = fetch('actions', None)
done_summon_hooks = fetch('done_summon_hooks', [])
module_executed_hook = fetch('module_executed_hook', [])
job_monitor = fetch('job_monitor', None)
Executes a pipeline using caching. Caching works by reusing
pipelines directly. This means that there exists one global
pipeline whose parts get executed over and over again.
This function returns a triple of dictionaries (objs, errs, execs).
objs is a mapping from local ids (the ids in the pipeline) to
objects **in the persistent pipeline**. Notice, these are not
the objects inside the passed pipeline, but the objects they
were mapped to in the persistent pipeline.
errs is a dictionary from local ids to error messages of modules
that might have returns errors.
execs is a dictionary from local ids to boolean values indicating
whether they were executed or not.
If modules have no error associated with but were not executed, it
means they were cached."""
# Setup named arguments. We don't use named parameters so
# that positional parameter calls fail earlier
new_kwargs = {}
def fetch(name, default):
new_kwargs[name] = r = kwargs.pop(name, default)
return r
controller = fetch('controller', None)
locator = fetch('locator', None)
current_version = fetch('current_version', None)
view = fetch('view', DummyView())
vistrail_variables = fetch('vistrail_variables', None)
aliases = fetch('aliases', None)
params = fetch('params', None)
extra_info = fetch('extra_info', None)
logger = fetch('logger', DummyLogController)
sinks = fetch('sinks', None)
reason = fetch('reason', None)
actions = fetch('actions', None)
done_summon_hooks = fetch('done_summon_hooks', [])
module_executed_hook = fetch('module_executed_hook', [])
stop_on_error = fetch('stop_on_error', True)
parent_exec = fetch('parent_exec', None)
job_monitor = fetch('job_monitor', None)
if len(kwargs) > 0:
raise VistrailsInternalError('Wrong parameters passed '
'to execute: %s' % kwargs)
self.clean_non_cacheable_modules()
record_usage(execute=True)
# if controller is not None:
# vistrail = controller.vistrail
# (pipeline, module_remap) = \
# core.db.io.expand_workflow(vistrail, pipeline)
# new_kwargs['module_remap'] = module_remap
# else:
# vistrail = None
if controller is not None:
vistrail = controller.vistrail
else:
vistrail = None
time_start = time.time()
logger = logger.start_workflow_execution(parent_exec, vistrail,
pipeline, current_version)
new_kwargs['logger'] = logger
self.annotate_workflow_execution(logger, reason, aliases, params)
res = self.setup_pipeline(pipeline, **new_kwargs)
modules_added = res[2]
conns_added = res[3]
to_delete = res[4]
errors = res[5]
if len(errors) == 0:
res = self.execute_pipeline(pipeline, *(res[:2]), **new_kwargs)
else:
res = (to_delete, res[0], errors, {}, {}, {}, [])
for (i, error) in errors.iteritems():
view.set_module_error(i, error.msg, error.errorTrace)
self.finalize_pipeline(pipeline, *(res[:-1]), **new_kwargs)
time_end = time.time()
result = InstanceObject(objects=res[1],
errors=res[2],
executed=res[3],
suspended=res[4],
parameter_changes=res[6],
modules_added=modules_added,
conns_added=conns_added)
logger.finish_workflow_execution(result.errors,
suspended=result.suspended)
record_usage(time=time_end - time_start,
modules=len(res[1]),
errors=len(res[2]),
executed=len(res[3]),
suspended=len(res[4]))
return result
def execute_pipeline(self, pipeline, tmp_id_to_module_map,
persistent_to_tmp_id_map, **kwargs):
def fetch(name, default):
return kwargs.pop(name, default)
controller = fetch('controller', None)
locator = fetch('locator', None)
current_version = fetch('current_version', None)
view = fetch('view', DummyView())
vistrail_variables = fetch('vistrail_variables', None)
aliases = fetch('aliases', None)
params = fetch('params', None)
extra_info = fetch('extra_info', None)
logger = fetch('logger', DummyLogController)
sinks = fetch('sinks', None)
reason = fetch('reason', None)
actions = fetch('actions', None)
module_executed_hook = fetch('module_executed_hook', [])
done_summon_hooks = fetch('done_summon_hooks', [])
clean_pipeline = fetch('clean_pipeline', False)
stop_on_error = fetch('stop_on_error', True)
parent_exec = fetch('parent_exec', None)
job_monitor = fetch('job_monitor', None)
if len(kwargs) > 0:
raise VistrailsInternalError('Wrong parameters passed '
'to execute_pipeline: %s' % kwargs)
# LOGGING SETUP
def get_remapped_id(id):
return persistent_to_tmp_id_map[id]
logging_obj = ViewUpdatingLogController(
logger=logger,
view=view,
remap_id=get_remapped_id,
ids=pipeline.modules.keys(),
module_executed_hook=module_executed_hook)
# PARAMETER CHANGES SETUP
parameter_changes = []
def change_parameter(obj, name, value):
parameter_changes.append((get_remapped_id(obj.id), name, value))
def make_change_parameter(obj):
return lambda *args: change_parameter(obj, *args)
# Update **all** modules in the current pipeline
for i, obj in tmp_id_to_module_map.iteritems():
obj.in_pipeline = True # set flag to indicate in pipeline
obj.logging = logging_obj
obj.change_parameter = make_change_parameter(obj)
# Update object pipeline information
obj.moduleInfo['locator'] = locator
obj.moduleInfo['version'] = current_version
obj.moduleInfo['moduleId'] = i
obj.moduleInfo['pipeline'] = pipeline
obj.moduleInfo['controller'] = controller
# extract job monitor from controller if this is the top level
if controller:
obj.moduleInfo['job_monitor'] = controller.jobMonitor
else:
obj.moduleInfo['job_monitor'] = job_monitor
if extra_info is not None:
obj.moduleInfo['extra_info'] = extra_info
if reason is not None:
obj.moduleInfo['reason'] = reason
if actions is not None:
obj.moduleInfo['actions'] = actions
## Checking 'sinks' from kwargs to resolve only requested sinks
# Note that we accept any module in 'sinks', even if it's not actually
# a sink in the graph
if sinks is not None:
persistent_sinks = [
tmp_id_to_module_map[sink] for sink in sinks
if sink in tmp_id_to_module_map
]
else:
persistent_sinks = [
tmp_id_to_module_map[sink] for sink in pipeline.graph.sinks()
]
self._streams.append(Generator.generators)
Generator.generators = []
# Update new sinks
for obj in persistent_sinks:
abort = False
try:
obj.update()
continue
except ModuleWasSuspended:
continue
except ModuleHadError:
pass
except AbortExecution:
break
except ModuleSuspended, ms:
ms.module.logging.end_update(ms.module, ms, was_suspended=True)
continue
except ModuleErrors, mes:
for me in mes.module_errors:
me.module.logging.end_update(me.module, me)
logging_obj.signalError(me.module, me)
abort = abort or me.abort
def setup_pipeline(self, pipeline, **kwargs):
"""setup_pipeline(controller, pipeline, locator, currentVersion,
view, aliases, **kwargs)
Matches a pipeline with the persistent pipeline and creates
instances of modules that aren't in the cache.
"""
def fetch(name, default):
return kwargs.pop(name, default)
controller = fetch('controller', None)
locator = fetch('locator', None)
current_version = fetch('current_version', None)
view = fetch('view', DummyView())
vistrail_variables = fetch('vistrail_variables', None)
aliases = fetch('aliases', None)
params = fetch('params', None)
extra_info = fetch('extra_info', None)
logger = fetch('logger', DummyLogController)
sinks = fetch('sinks', None)
reason = fetch('reason', None)
actions = fetch('actions', None)
done_summon_hooks = fetch('done_summon_hooks', [])
module_executed_hook = fetch('module_executed_hook', [])
stop_on_error = fetch('stop_on_error', True)
parent_exec = fetch('parent_exec', None)
job_monitor = fetch('job_monitor', None)
reg = get_module_registry()
if len(kwargs) > 0:
raise VistrailsInternalError('Wrong parameters passed '
'to setup_pipeline: %s' % kwargs)
def create_null():
"""Creates a Null value"""
getter = reg.get_descriptor_by_name
descriptor = getter(basic_pkg, 'Null')
return descriptor.module()
def create_constant(param, module):
"""Creates a Constant from a parameter spec"""
getter = reg.get_descriptor_by_name
desc = getter(param.identifier, param.type, param.namespace)
constant = desc.module()
constant.id = module.id
# if param.evaluatedStrValue:
# constant.setValue(param.evaluatedStrValue)
if param.strValue != '':
constant.setValue(param.strValue)
else:
constant.setValue(
constant.translate_to_string(constant.default_value))
return constant
### BEGIN METHOD ###
# if self.debugger:
# self.debugger.update()
to_delete = []
errors = {}
if controller is not None:
# Controller is none for sub_modules
controller.validate(pipeline)
else:
pipeline.validate()
self.resolve_aliases(pipeline, aliases)
if vistrail_variables:
self.resolve_variables(vistrail_variables, pipeline)
self.update_params(pipeline, params)
(tmp_to_persistent_module_map, conn_map, module_added_set,
conn_added_set) = self.add_to_persistent_pipeline(pipeline)
# Create the new objects
for i in module_added_set:
persistent_id = tmp_to_persistent_module_map[i]
module = self._persistent_pipeline.modules[persistent_id]
obj = self._objects[persistent_id] = module.summon()
obj.interpreter = self
obj.id = persistent_id
obj.signature = module._signature
# Checking if output should be stored
if module.has_annotation_with_key('annotate_output'):
annotate_output = module.get_annotation_by_key(
'annotate_output')
#print annotate_output
if annotate_output:
obj.annotate_output = True
for f in module.functions:
connector = None
if len(f.params) == 0:
connector = ModuleConnector(create_null(), 'value',
f.get_spec('output'))
elif len(f.params) == 1:
p = f.params[0]
try:
constant = create_constant(p, module)
connector = ModuleConnector(constant, 'value',
f.get_spec('output'))
except Exception, e:
debug.unexpected_exception(e)
err = ModuleError(
module,
"Uncaught exception creating Constant from "
"%r: %s" % (p.strValue, debug.format_exception(e)))
errors[i] = err
to_delete.append(obj.id)
else:
tupleModule = vistrails.core.interpreter.base.InternalTuple(
)
tupleModule.length = len(f.params)
for (j, p) in enumerate(f.params):
try:
constant = create_constant(p, module)
constant.update()
connector = ModuleConnector(
constant, 'value', f.get_spec('output'))
tupleModule.set_input_port(j, connector)
except Exception, e:
debug.unexpected_exception(e)
err = ModuleError(
module, "Uncaught exception creating Constant "
"from %r: %s" %
(p.strValue, debug.format_exception(e)))
errors[i] = err
to_delete.append(obj.id)
connector = ModuleConnector(tupleModule, 'value',
f.get_spec('output'))
if connector:
obj.set_input_port(f.name, connector, is_method=True)
def execute(modules,
connections=[],
add_port_specs=[],
enable_pkg=True,
full_results=False):
"""Build a pipeline and execute it.
This is useful to simply build a pipeline in a test case, and run it. When
doing that, intercept_result() can be used to check the results of each
module.
modules is a list of module tuples describing the modules to be created,
with the following format:
[('ModuleName', 'package.identifier', [
# Functions
('port_name', [
# Function parameters
('Signature', 'value-as-string'),
]),
])]
connections is a list of tuples describing the connections to make, with
the following format:
[
(source_module_index, 'source_port_name',
dest_module_index, 'dest_module_name'),
]
add_port_specs is a list of specs to add to modules, with the following
format:
[
(mod_id, 'input'/'output', 'portname',
'(port_sig)'),
]
It is useful to test modules that can have custom ports through a
configuration widget.
The function returns the 'errors' dict it gets from the interpreter, so you
should use a construct like self.assertFalse(execute(...)) if the execution
is not supposed to fail.
For example, this creates (and runs) an Integer module with its value set
to 44, connected to a PythonCalc module, connected to a StandardOutput:
self.assertFalse(execute([
('Float', 'org.vistrails.vistrails.basic', [
('value', [('Float', '44.0')]),
]),
('PythonCalc', 'org.vistrails.vistrails.pythoncalc', [
('value2', [('Float', '2.0')]),
('op', [('String', '-')]),
]),
('StandardOutput', 'org.vistrails.vistrails.basic', []),
],
[
(0, 'value', 1, 'value1'),
(1, 'value', 2, 'value'),
]))
"""
from vistrails.core.db.locator import XMLFileLocator
from vistrails.core.modules.module_registry import MissingPackage
from vistrails.core.packagemanager import get_package_manager
from vistrails.core.utils import DummyView
from vistrails.core.vistrail.connection import Connection
from vistrails.core.vistrail.module import Module
from vistrails.core.vistrail.module_function import ModuleFunction
from vistrails.core.vistrail.module_param import ModuleParam
from vistrails.core.vistrail.pipeline import Pipeline
from vistrails.core.vistrail.port import Port
from vistrails.core.vistrail.port_spec import PortSpec
from vistrails.core.interpreter.noncached import Interpreter
pm = get_package_manager()
port_spec_per_module = {} # mod_id -> [portspec: PortSpec]
j = 0
for i, (mod_id, inout, name, sig) in enumerate(add_port_specs):
mod_specs = port_spec_per_module.setdefault(mod_id, [])
ps = PortSpec(id=i, name=name, type=inout, sigstring=sig, sort_key=-1)
for psi in ps.port_spec_items:
psi.id = j
j += 1
mod_specs.append(ps)
pipeline = Pipeline()
module_list = []
for i, (name, identifier, functions) in enumerate(modules):
function_list = []
try:
pkg = pm.get_package(identifier)
except MissingPackage:
if not enable_pkg:
raise
dep_graph = pm.build_dependency_graph([identifier])
for pkg_id in pm.get_ordered_dependencies(dep_graph):
pkg = pm.identifier_is_available(pkg_id)
if pkg is None:
raise
pm.late_enable_package(pkg.codepath)
pkg = pm.get_package(identifier)
for func_name, params in functions:
param_list = []
for j, (param_type, param_val) in enumerate(params):
param_list.append(
ModuleParam(pos=j, type=param_type, val=param_val))
function_list.append(
ModuleFunction(name=func_name, parameters=param_list))
name = name.rsplit('|', 1)
if len(name) == 2:
namespace, name = name
else:
namespace = None
name, = name
module = Module(name=name,
namespace=namespace,
package=identifier,
version=pkg.version,
id=i,
functions=function_list)
for port_spec in port_spec_per_module.get(i, []):
module.add_port_spec(port_spec)
pipeline.add_module(module)
module_list.append(module)
for i, (sid, sport, did, dport) in enumerate(connections):
s_sig = module_list[sid].get_port_spec(sport, 'output').sigstring
d_sig = module_list[did].get_port_spec(dport, 'input').sigstring
pipeline.add_connection(
Connection(id=i,
ports=[
Port(id=i * 2,
type='source',
moduleId=sid,
name=sport,
signature=s_sig),
Port(id=i * 2 + 1,
type='destination',
moduleId=did,
name=dport,
signature=d_sig),
]))
interpreter = Interpreter.get()
result = interpreter.execute(pipeline,
locator=XMLFileLocator('foo.xml'),
current_version=1,
view=DummyView())
if full_results:
return result
else:
# Allows to do self.assertFalse(execute(...))
return result.errors
