def __init__(self, parent=None):
RichIPythonWidget.__init__(self, parent)
self.old_streams = None
self.running_workflow = False
self.kernel_manager = km
self.kernel_client = kernel_client
self.exit_requested.connect(self.stop)
self.setWindowTitle("Console")
self.vistrails_interpreter = get_default_interpreter()
def __init__(self, parent=None):
RichIPythonWidget.__init__(self, parent)
self.old_streams = None
self.running_workflow = False
self.kernel_manager = km
self.kernel_client = kernel_client
self.exit_requested.connect(self.stop)
self.setWindowTitle("Console")
self.vistrails_interpreter = get_default_interpreter()
def __init__(self, parent=None):
QtGui.QWidget.__init__(self, parent=parent)
self.app = vistrails.gui.application.get_vistrails_application()
self.inspector = QObjectInspector()
layout = QtGui.QVBoxLayout()
layout.setMargin(0)
layout.setSpacing(0)
layout.addWidget(self.inspector)
self.setLayout(layout)
# self.setTitleBarWidget(QtGui.QLabel("Debugger"))
self.setWindowTitle("Debugger")
self.controller = None
self.vistrails_interpreter = get_default_interpreter()
self.vistrails_interpreter.debugger = self
def __init__(self, parent=None):
QtGui.QWidget.__init__(self, parent=parent)
self.app = vistrails.gui.application.get_vistrails_application()
self.inspector = QObjectInspector()
layout = QtGui.QVBoxLayout()
layout.setMargin(0)
layout.setSpacing(0)
layout.addWidget(self.inspector)
self.setLayout(layout)
# self.setTitleBarWidget(QtGui.QLabel("Debugger"))
self.setWindowTitle("Debugger")
self.controller = None
self.vistrails_interpreter = get_default_interpreter()
self.vistrails_interpreter.debugger = self
def __init__(self, parent=None):
QtGui.QWidget.__init__(self, parent=parent)
#locals() returns the original dictionary, not a copy as
#the docs say
self.firstLocals = copy.copy(locals())
self.shell = QShell(self.firstLocals,None)
layout = QtGui.QVBoxLayout()
layout.setMargin(0)
layout.setSpacing(0)
layout.addWidget(self.shell)
self.setLayout(layout)
# self.setWidget(self.shell)
self.setWindowTitle(self.shell.windowTitle())
# self.setTitleBarWidget(QtGui.QLabel(self.shell.windowTitle()))
# self.monitorWindowTitle(self.shell)
self.vistrails_interpreter = get_default_interpreter()
def test_pipeline_creation(self):
import dat.tests.pkg_test_plots.init as pkg_test_plots
controller = self.vt_controller()
vistraildata = VistrailManager(controller)
loader = Test_generation._loaders.get('StrMaker')
loader.v = 'Hello'
vistraildata.new_variable('var1', loader.load())
loader.v = 'world'
vistraildata.new_variable('var2', loader.load())
cellInfo = FakeObj(
row=0,
column=0,
tab=FakeObj(tabWidget=FakeObj(tabText=lambda w: 'Sheet 1')))
recipe = DATRecipe(
pkg_test_plots.concat_plot, {
'param1': (RecipeParameterValue(
variable=vistraildata.get_variable('var1')), ),
'param2': (RecipeParameterValue(
variable=vistraildata.get_variable('var2')), ),
'param3': (RecipeParameterValue(constant="!"), ),
})
pipelineInfo = vistrails_interface.create_pipeline(
controller, recipe, cellInfo.row, cellInfo.column,
None) # This plot has no cell module so this is fine
controller.change_selected_version(pipelineInfo.version)
result = CallRecorder()
pkg_test_plots.Recorder.callback = result
interpreter = get_default_interpreter()
interpreter.execute(controller.current_pipeline,
view=DummyView(),
locator=controller.locator,
current_version=pipelineInfo.version)
call = (['Hello, world!'], dict())
self.assertEqual(result.calls, [call])
def __init__(self, parent=None):
RichIPythonWidget.__init__(self, parent)
self.kernel_manager = km
self.kernel_client = kernel_client
self.exit_requested.connect(stop)
#locals() returns the original dictionary, not a copy as
#the docs say
# self.firstLocals = copy.copy(locals())
# self.shell = IPythonXXX(self.firstLocals,None)
# layout = QtGui.QVBoxLayout()
# layout.setMargin(0)
# layout.setSpacing(0)
# layout.addWidget(self.shell)
# self.setLayout(layout)
# self.setWidget(self.shell)
self.setWindowTitle("Console")
# self.setTitleBarWidget(QtGui.QLabel(self.shell.windowTitle()))
# self.monitorWindowTitle(self.shell)
self.vistrails_interpreter = get_default_interpreter()
def executePipelineWithProgress(pipeline,
pTitle='Pipeline Execution',
pCaption='Executing...',
pCancel='&Cancel',
**kwargs):
""" executePipelineWithProgress(pipeline: Pipeline,
pTitle: str, pCaption: str, pCancel: str,
kwargs: keyword arguments) -> bool
Execute the pipeline while showing a progress dialog with title
pTitle, caption pCaption and the cancel button text
pCancel. kwargs is the keyword arguments that will be passed to
the interpreter. A bool will be returned indicating if the
execution was performed without cancel or not.
"""
withoutCancel = True
totalProgress = len(pipeline.modules)
progress = QtGui.QProgressDialog(pCaption,
pCancel,
0, totalProgress)
progress.setWindowTitle(pTitle)
progress.setWindowModality(QtCore.Qt.WindowModal)
progress.show()
def moduleExecuted(objId):
if not progress.wasCanceled():
progress.setValue(progress.value()+1)
QtCore.QCoreApplication.processEvents()
else:
withoutCancel = False
interpreter = get_default_interpreter()
if kwargs.has_key('module_executed_hook'):
kwargs['module_executed_hook'].append(moduleExecuted)
else:
kwargs['module_executed_hook'] = [moduleExecuted]
kwargs['view'] = DummyView()
interpreter.execute(pipeline, **kwargs)
progress.setValue(totalProgress)
return withoutCancel
def execute(self, *args, **kwargs):
"""Execute the pipeline.
Positional arguments are either input values (created from
``module == value``, where `module` is a Module from the pipeline and
`value` is some value or Function instance) for the pipeline's
InputPorts, or Module instances (to select sink modules).
Keyword arguments are also used to set InputPort by looking up inputs
by name.
Example::
input_bound = pipeline.get_input('higher_bound')
input_url = pipeline.get_input('url')
sinkmodule = pipeline.get_module(32)
pipeline.execute(sinkmodule,
input_bound == vt.Function(Integer, 10),
input_url == 'http://www.vistrails.org/',
resolution=15) # kwarg: only one equal sign
"""
sinks = set()
inputs = {}
reg = get_module_registry()
InputPort_desc = reg.get_descriptor_by_name(
get_vistrails_basic_pkg_id(), 'InputPort')
# Read args
for arg in args:
if isinstance(arg, ModuleValuePair):
if arg.module.id in inputs:
raise ValueError("Multiple values set for InputPort %r" %
get_inputoutput_name(arg.module))
if not reg.is_descriptor_subclass(arg.module.module_descriptor,
InputPort_desc):
raise ValueError("Module %d is not an InputPort" %
arg.module.id)
inputs[arg.module.id] = arg.value
elif isinstance(arg, Module):
sinks.add(arg.module_id)
# Read kwargs
for key, value in kwargs.iteritems():
key = self.get_input(key) # Might raise KeyError
if key.module_id in inputs:
raise ValueError("Multiple values set for InputPort %r" %
get_inputoutput_name(key.module))
inputs[key.module_id] = value
reason = "API pipeline execution"
sinks = sinks or None
# Use controller only if no inputs were passed in
if (not inputs and self.vistrail is not None
and self.vistrail.current_version == self.version):
controller = self.vistrail.controller
results, changed = controller.execute_workflow_list([[
controller.locator, # locator
self.version, # version
self.pipeline, # pipeline
DummyView(), # view
None, # custom_aliases
None, # custom_params
reason, # reason
sinks, # sinks
None, # extra_info
]])
result, = results
else:
pipeline = self.pipeline
if inputs:
id_scope = IdScope(1)
pipeline = pipeline.do_copy(False, id_scope)
# A hach to get ids from id_scope that we know won't collide:
# make them negative
id_scope.getNewId = lambda t, g=id_scope.getNewId: -g(t)
create_module = \
VistrailController.create_module_from_descriptor_static
create_function = VistrailController.create_function_static
create_connection = VistrailController.create_connection_static
# Fills in the ExternalPipe ports
for module_id, values in inputs.iteritems():
module = pipeline.modules[module_id]
if not isinstance(values, (list, tuple)):
values = [values]
# Guess the type of the InputPort
_, sigstrings, _, _, _ = get_port_spec_info(
pipeline, module)
sigstrings = parse_port_spec_string(sigstrings)
# Convert whatever we got to a list of strings, for the
# pipeline
values = [
reg.convert_port_val(val, sigstring, None)
for val, sigstring in izip(values, sigstrings)
]
if len(values) == 1:
# Create the constant module
constant_desc = reg.get_descriptor_by_name(
*sigstrings[0])
constant_mod = create_module(id_scope, constant_desc)
func = create_function(id_scope, constant_mod, 'value',
values)
constant_mod.add_function(func)
pipeline.add_module(constant_mod)
# Connect it to the ExternalPipe port
conn = create_connection(id_scope, constant_mod,
'value', module,
'ExternalPipe')
pipeline.db_add_connection(conn)
else:
raise RuntimeError("TODO : create tuple")
interpreter = get_default_interpreter()
result = interpreter.execute(pipeline, reason=reason, sinks=sinks)
if result.errors:
raise ExecutionErrors(self, result)
else:
return ExecutionResults(self, result)
def execute_wf(wf, output_port):
# Save the workflow in a temporary file
temp_wf_fd, temp_wf = tempfile.mkstemp()
try:
f = open(temp_wf, 'w')
f.write(wf)
f.close()
os.close(temp_wf_fd)
# Clean the cache
interpreter = get_default_interpreter()
interpreter.flush()
# Load the Pipeline from the temporary file
vistrail = Vistrail()
locator = XMLFileLocator(temp_wf)
workflow = locator.load(Pipeline)
# Build a Vistrail from this single Pipeline
action_list = []
for module in workflow.module_list:
action_list.append(('add', module))
for connection in workflow.connection_list:
action_list.append(('add', connection))
action = vistrails.core.db.action.create_action(action_list)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
tag = 'parallel flow'
vistrail.addTag(tag, action.id)
# Build a controller and execute
controller = VistrailController()
controller.set_vistrail(vistrail, None)
controller.change_selected_version(vistrail.get_version_number(tag))
execution = controller.execute_current_workflow(
custom_aliases=None,
custom_params=None,
extra_info=None,
reason='API Pipeline Execution')
# Build a list of errors
errors = []
pipeline = vistrail.getPipeline(tag)
execution_errors = execution[0][0].errors
if execution_errors:
for key in execution_errors:
module = pipeline.modules[key]
msg = '%s: %s' %(module.name, execution_errors[key])
errors.append(msg)
# Get the execution log from the controller
try:
module_log = controller.log.workflow_execs[0].item_execs[0]
except IndexError:
errors.append("Module log not found")
return dict(errors=errors)
else:
machine = controller.log.workflow_execs[0].machines[
module_log.machine_id]
xml_log = serialize(module_log)
machine_log = serialize(machine)
# Get the output value
output = None
serializable = None
if not execution_errors:
executed_module, = execution[0][0].executed
executed_module = execution[0][0].objects[executed_module]
try:
output = executed_module.get_output(output_port)
except ModuleError:
errors.append("Output port not found: %s" % output_port)
return dict(errors=errors)
reg = vistrails.core.modules.module_registry.get_module_registry()
base_classes = inspect.getmro(type(output))
if Module in base_classes:
serializable = reg.get_descriptor(type(output)).sigstring
output = output.serialize()
# Return the dictionary, that will be sent back to the client
return dict(errors=errors,
output=output,
serializable=serializable,
xml_log=xml_log,
machine_log=machine_log)
finally:
os.unlink(temp_wf)
def execute_wf(wf, output_port):
# Save the workflow in a temporary file
temp_wf_fd, temp_wf = tempfile.mkstemp()
try:
f = open(temp_wf, 'w')
f.write(wf)
f.close()
os.close(temp_wf_fd)
# Clean the cache
interpreter = get_default_interpreter()
interpreter.flush()
# Load the Pipeline from the temporary file
vistrail = Vistrail()
locator = XMLFileLocator(temp_wf)
workflow = locator.load(Pipeline)
# Build a Vistrail from this single Pipeline
action_list = []
for module in workflow.module_list:
action_list.append(('add', module))
for connection in workflow.connection_list:
action_list.append(('add', connection))
action = vistrails.core.db.action.create_action(action_list)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
tag = 'parallel flow'
vistrail.addTag(tag, action.id)
# Build a controller and execute
controller = VistrailController()
controller.set_vistrail(vistrail, None)
controller.change_selected_version(vistrail.get_version_number(tag))
execution = controller.execute_current_workflow(
custom_aliases=None,
custom_params=None,
extra_info=None,
reason='API Pipeline Execution')
# Build a list of errors
errors = []
pipeline = vistrail.getPipeline(tag)
execution_errors = execution[0][0].errors
if execution_errors:
for key in execution_errors:
module = pipeline.modules[key]
msg = '%s: %s' % (module.name, execution_errors[key])
errors.append(msg)
# Get the execution log from the controller
try:
module_log = controller.log.workflow_execs[0].item_execs[0]
except IndexError:
errors.append("Module log not found")
return dict(errors=errors)
else:
machine = controller.log.workflow_execs[0].machines[
module_log.machine_id]
xml_log = serialize(module_log)
machine_log = serialize(machine)
# Get the output value
output = None
serializable = None
if not execution_errors:
executed_module, = execution[0][0].executed
executed_module = execution[0][0].objects[executed_module]
try:
output = executed_module.get_output(output_port)
except ModuleError:
errors.append("Output port not found: %s" % output_port)
return dict(errors=errors)
reg = vistrails.core.modules.module_registry.get_module_registry()
base_classes = inspect.getmro(type(output))
if Module in base_classes:
serializable = reg.get_descriptor(type(output)).sigstring
output = output.serialize()
# Return the dictionary, that will be sent back to the client
return dict(errors=errors,
output=output,
serializable=serializable,
xml_log=xml_log,
machine_log=machine_log)
finally:
os.unlink(temp_wf)
def performParameterExploration(self):
""" performParameterExploration() -> None
Perform the exploration by collecting a list of actions
corresponding to each dimension
"""
registry = get_module_registry()
actions = self.peWidget.table.collectParameterActions()
spreadsheet_pkg = '%s.spreadsheet' % get_vistrails_default_pkg_prefix()
# Set the annotation to persist the parameter exploration
# TODO: For now, we just replace the existing exploration - Later we should append them.
xmlString = "<paramexps>\n" + self.getParameterExploration() + "\n</paramexps>"
self.controller.vistrail.set_paramexp(self.currentVersion, xmlString)
self.controller.set_changed(True)
if self.controller.current_pipeline and actions:
explorer = ActionBasedParameterExploration()
(pipelines, performedActions) = explorer.explore(
self.controller.current_pipeline, actions)
dim = [max(1, len(a)) for a in actions]
if (registry.has_module(spreadsheet_pkg, 'CellLocation') and
registry.has_module(spreadsheet_pkg, 'SheetReference')):
modifiedPipelines = self.virtualCell.positionPipelines(
'PE#%d %s' % (QParameterExplorationTab.explorationId,
self.controller.name),
dim[2], dim[1], dim[0], pipelines, self.controller)
else:
modifiedPipelines = pipelines
mCount = []
for p in modifiedPipelines:
if len(mCount)==0:
mCount.append(0)
else:
mCount.append(len(p.modules)+mCount[len(mCount)-1])
# Now execute the pipelines
totalProgress = sum([len(p.modules) for p in modifiedPipelines])
progress = QtGui.QProgressDialog('Performing Parameter '
'Exploration...',
'&Cancel',
0, totalProgress)
progress.setWindowTitle('Parameter Exploration')
progress.setWindowModality(QtCore.Qt.WindowModal)
progress.show()
QParameterExplorationTab.explorationId += 1
interpreter = get_default_interpreter()
for pi in xrange(len(modifiedPipelines)):
progress.setValue(mCount[pi])
QtCore.QCoreApplication.processEvents()
if progress.wasCanceled():
break
def moduleExecuted(objId):
if not progress.wasCanceled():
#progress.setValue(progress.value()+1)
#the call above was crashing when used by multithreaded
#code, replacing with the call below (thanks to Terence
#for submitting this fix).
QtCore.QMetaObject.invokeMethod(progress, "setValue",
QtCore.Q_ARG(int,progress.value()+1))
QtCore.QCoreApplication.processEvents()
kwargs = {'locator': self.controller.locator,
'current_version': self.controller.current_version,
'view': self.controller.current_pipeline_scene,
'module_executed_hook': [moduleExecuted],
'reason': 'Parameter Exploration',
'actions': performedActions[pi],
}
interpreter.execute(modifiedPipelines[pi], **kwargs)
progress.setValue(totalProgress)
def openSpreadsheet(self, fileName):
""" openSpreadsheet(fileName: str) -> None
Open a saved spreadsheet assuming that all VTK files must exist and have
all the version using the saved spreadsheet
"""
def parse_locator(text):
locator = None
wrapper = XMLWrapper()
dom = wrapper.create_document_from_string(text)
root = dom.documentElement
version = None
version = root.getAttribute('version')
if version == '1.0':
for element in named_elements(root, 'locator'):
if str(element.getAttribute('type')) == 'file':
locator = FileLocator.parse(element)
elif str(element.getAttribute('type')) == 'db':
locator = DBLocator.parse(element)
return locator
locators = {}
indexFile = open(fileName, 'r')
contents = indexFile.read()
self.clearTabs()
lidx = 0
lines = contents.split('\n')
tabCount = int(lines[lidx])
lidx += 1
for tabIdx in xrange(tabCount):
# FIXME: eval should pretty much never be used
tabInfo = literal_eval(lines[lidx])
lidx += 1
sheet = spreadsheetRegistry.getSheet(tabInfo[1])(self)
sheet.setDimension(tabInfo[2], tabInfo[3])
self.addTabWidget(sheet, tabInfo[0])
while lines[lidx]!='---':
(r, c, vistrail, pid, cid) = literal_eval(lines[lidx])
locator = vistrail['locator']
if locators.has_key(locator):
vistrail['locator'] = locators[locator]
else:
locators[locator] = parse_locator(vistrail['locator'])
vistrail['locator'] = locators[locator]
self.appendMonitoredLocations((vistrail, pid, cid),
(sheet, r, c))
lidx += 1
lidx += 1
pipelineCount = int(lines[lidx])
lidx += 1
self.loadingMode = True
progress = QtGui.QProgressDialog("Loading spreadsheet...",
"&Cancel", 0, pipelineCount,
self,
QtCore.Qt.WindowStaysOnTopHint
)
progress.show()
for pipelineIdx in xrange(pipelineCount):
# FIXME: eval should pretty much never be used
(serializedLocator, version) = literal_eval(lines[lidx])
try:
locator = locators[serializedLocator]
except KeyError:
locator = parse_locator(serializedLocator)
if locator:
bundle = locator.load()
if isinstance(bundle, SaveBundle):
pipeline = bundle.vistrail.getPipeline(version)
else:
pipeline = bundle.getPipeline(version)
execution = get_default_interpreter()
progress.setValue(pipelineIdx)
QtCore.QCoreApplication.processEvents()
if progress.wasCanceled():
break
kwargs = {'locator': locator,
'current_version': version,
'view': DummyView(),
}
execution.execute(pipeline, **kwargs)
else:
raise RuntimeError("Couldn't load spreadsheet")
lidx += 1
progress.setValue(pipelineCount)
QtCore.QCoreApplication.processEvents()
self.changeSpreadsheetFileName(fileName)
self.loadingMode = False
indexFile.close()
def performParameterExploration(self):
""" performParameterExploration() -> None
Perform the exploration by collecting a list of actions
corresponding to each dimension
"""
registry = get_module_registry()
actions = self.peWidget.table.collectParameterActions()
spreadsheet_pkg = 'org.vistrails.vistrails.spreadsheet'
# Set the annotation to persist the parameter exploration
# TODO: For now, we just replace the existing exploration - Later we should append them.
xmlString = "<paramexps>\n" + self.getParameterExploration() + "\n</paramexps>"
self.controller.vistrail.set_paramexp(self.currentVersion, xmlString)
self.controller.set_changed(True)
if self.controller.current_pipeline and actions:
explorer = ActionBasedParameterExploration()
(pipelines, performedActions) = explorer.explore(
self.controller.current_pipeline, actions)
dim = [max(1, len(a)) for a in actions]
if (registry.has_module(spreadsheet_pkg, 'CellLocation') and
registry.has_module(spreadsheet_pkg, 'SheetReference')):
modifiedPipelines = self.virtualCell.positionPipelines(
'PE#%d %s' % (QParameterExplorationTab.explorationId,
self.controller.name),
dim[2], dim[1], dim[0], pipelines, self.controller)
else:
modifiedPipelines = pipelines
mCount = []
for p in modifiedPipelines:
if len(mCount)==0:
mCount.append(0)
else:
mCount.append(len(p.modules)+mCount[len(mCount)-1])
# Now execute the pipelines
totalProgress = sum([len(p.modules) for p in modifiedPipelines])
progress = QtGui.QProgressDialog('Performing Parameter '
'Exploration...',
'&Cancel',
0, totalProgress)
progress.setWindowTitle('Parameter Exploration')
progress.setWindowModality(QtCore.Qt.WindowModal)
progress.show()
QParameterExplorationTab.explorationId += 1
interpreter = get_default_interpreter()
for pi in xrange(len(modifiedPipelines)):
progress.setValue(mCount[pi])
QtCore.QCoreApplication.processEvents()
if progress.wasCanceled():
break
def moduleExecuted(objId):
if not progress.wasCanceled():
#progress.setValue(progress.value()+1)
#the call above was crashing when used by multithreaded
#code, replacing with the call below (thanks to Terence
#for submitting this fix).
QtCore.QMetaObject.invokeMethod(progress, "setValue",
QtCore.Q_ARG(int,progress.value()+1))
QtCore.QCoreApplication.processEvents()
kwargs = {'locator': self.controller.locator,
'current_version': self.controller.current_version,
'view': self.controller.current_pipeline_scene,
'module_executed_hook': [moduleExecuted],
'reason': 'Parameter Exploration',
'actions': performedActions[pi],
}
interpreter.execute(modifiedPipelines[pi], **kwargs)
progress.setValue(totalProgress)
def get_variable_value(variable):
"""Get the value of a variable, i.e. the result of its pipeline.
The 'variable' can either be a Variable, from which a temporary pipeline
will be built, or a VariableInformation, representing an existing pipeline.
"""
def pipeline_from_info(variableinfo):
controller = variableinfo._controller
version = controller.vistrail.get_version_number(
'dat-var-%s' % variable.name)
return controller.vistrail.getPipeline(version), version
def pipeline_from_generator(variable_gen):
# Get the original OutputPort module
orig_controller = variable_gen._generator.controller
base_pipeline = orig_controller.vistrail.getPipeline('dat-vars')
if len(base_pipeline.module_list) != 1:
raise ValueError("dat-vars version is invalid")
output_port = base_pipeline.module_list[0]
controller = VistrailController(Vistrail())
# OutputPort
operations = [('add', output_port)]
# Rest of the pipeline
operations += variable_gen._generator.operations
# Connection
connection = controller.create_connection(
variable_gen._output_module,
variable_gen._outputport_name,
output_port,
'InternalPipe')
operations.append(('add', connection))
# Materialize this
action = create_action(operations)
controller.add_new_action(action)
version = controller.perform_action(action)
controller.change_selected_version(version)
assert version == controller.current_version == 1
return controller.current_pipeline, 1
# Obtain 'pipeline' and 'version' from 'variable'
if isinstance(variable, Variable.VariableInformation):
# Pipeline already exists
pipeline, version = pipeline_from_info(variable)
elif isinstance(variable, Variable):
if variable._materialized is not None:
# Pipeline already exists
pipeline, version = pipeline_from_info(variable._materialized)
else:
# Pipeline doesn't exist
# We need to make one from the operations
pipeline, version = pipeline_from_generator(variable)
else:
raise TypeError
# Setup the interpreter for execution
interpreter = get_default_interpreter()
interpreter.clean_non_cacheable_modules()
interpreter.parent_execs = [None]
res = interpreter.setup_pipeline(pipeline)
if len(res[5]) > 0:
raise ValueError("Variable pipeline has errors:\n%s" %
'\n'.join(me.msg for me in res[5].itervalues()))
tmp_id_to_module_map = res[0]
# Execute
res = interpreter.execute_pipeline(
pipeline,
res[0], # tmp_id_to_module_map
res[1], # persistent_to_tmp_id_map
current_version=version,
reason="getting variable value")
if len(res[2]) > 0:
raise ValueError("Error while executing variable pipeline:\n%s" %
'\n'.join('%s: %s' % (me.module.__class__.__name__,
me.msg)
for me in res[2].itervalues()))
if len(res[4]) > 0:
# extract messages and previous ModuleSuspended exceptions
raise ValueError("Module got suspended while executing variable "
"pipeline:\n%s" %
'\n'.join(msg for msg in res[4].itervalues()))
# Get the result
outputport_desc = get_module_registry().get_descriptor_by_name(
'org.vistrails.vistrails.basic', 'OutputPort')
for module in pipeline.module_list:
if module.module_descriptor is outputport_desc:
if get_function(module, 'name') == 'value':
module_obj = tmp_id_to_module_map[module.id]
result = module_obj.get_output('ExternalPipe')
break
else:
result = None
interpreter.finalize_pipeline(pipeline, *res[:-1])
interpreter.parent_execs = [None]
return result
def get_variable_value(variable):
"""Get the value of a variable, i.e. the result of its pipeline.
The 'variable' can either be a Variable, from which a temporary pipeline
will be built, or a VariableInformation, representing an existing pipeline.
"""
def pipeline_from_info(variableinfo):
controller = variableinfo._controller
version = controller.vistrail.get_version_number('dat-var-%s' %
variable.name)
return controller.vistrail.getPipeline(version), version
def pipeline_from_generator(variable_gen):
# Get the original OutputPort module
orig_controller = variable_gen._generator.controller
base_pipeline = orig_controller.vistrail.getPipeline('dat-vars')
if len(base_pipeline.module_list) != 1:
raise ValueError("dat-vars version is invalid")
output_port = base_pipeline.module_list[0]
controller = VistrailController(Vistrail())
# OutputPort
operations = [('add', output_port)]
# Rest of the pipeline
operations += variable_gen._generator.operations
# Connection
connection = controller.create_connection(
variable_gen._output_module, variable_gen._outputport_name,
output_port, 'InternalPipe')
operations.append(('add', connection))
# Materialize this
action = create_action(operations)
controller.add_new_action(action)
version = controller.perform_action(action)
controller.change_selected_version(version)
assert version == controller.current_version == 1
return controller.current_pipeline, 1
# Obtain 'pipeline' and 'version' from 'variable'
if isinstance(variable, Variable.VariableInformation):
# Pipeline already exists
pipeline, version = pipeline_from_info(variable)
elif isinstance(variable, Variable):
if variable._materialized is not None:
# Pipeline already exists
pipeline, version = pipeline_from_info(variable._materialized)
else:
# Pipeline doesn't exist
# We need to make one from the operations
pipeline, version = pipeline_from_generator(variable)
else:
raise TypeError
# Setup the interpreter for execution
interpreter = get_default_interpreter()
interpreter.clean_non_cacheable_modules()
interpreter.parent_execs = [None]
res = interpreter.setup_pipeline(pipeline)
if len(res[5]) > 0:
raise ValueError("Variable pipeline has errors:\n%s" %
'\n'.join(me.msg for me in res[5].itervalues()))
tmp_id_to_module_map = res[0]
# Execute
res = interpreter.execute_pipeline(
pipeline,
res[0], # tmp_id_to_module_map
res[1], # persistent_to_tmp_id_map
current_version=version,
reason="getting variable value")
if len(res[2]) > 0:
raise ValueError("Error while executing variable pipeline:\n%s" %
'\n'.join('%s: %s' %
(me.module.__class__.__name__, me.msg)
for me in res[2].itervalues()))
if len(res[4]) > 0:
# extract messages and previous ModuleSuspended exceptions
raise ValueError("Module got suspended while executing variable "
"pipeline:\n%s" %
'\n'.join(msg for msg in res[4].itervalues()))
# Get the result
outputport_desc = get_module_registry().get_descriptor_by_name(
'org.vistrails.vistrails.basic', 'OutputPort')
for module in pipeline.module_list:
if module.module_descriptor is outputport_desc:
if get_function(module, 'name') == 'value':
module_obj = tmp_id_to_module_map[module.id]
result = module_obj.get_output('ExternalPipe')
break
else:
result = None
interpreter.finalize_pipeline(pipeline, *res[:-1])
interpreter.parent_execs = [None]
return result
def execute(self, *args, **kwargs):
"""Execute the pipeline.
Positional arguments are either input values (created from
``module == value``, where `module` is a Module from the pipeline and
`value` is some value or Function instance) for the pipeline's
InputPorts, or Module instances (to select sink modules).
Keyword arguments are also used to set InputPort by looking up inputs
by name.
Example::
input_bound = pipeline.get_input('higher_bound')
input_url = pipeline.get_input('url')
sinkmodule = pipeline.get_module(32)
pipeline.execute(sinkmodule,
input_bound == vt.Function(Integer, 10),
input_url == 'http://www.vistrails.org/',
resolution=15) # kwarg: only one equal sign
"""
sinks = set()
inputs = {}
reg = get_module_registry()
InputPort_desc = reg.get_descriptor_by_name(
get_vistrails_basic_pkg_id(),
'InputPort')
# Read args
for arg in args:
if isinstance(arg, ModuleValuePair):
if arg.module.id in inputs:
raise ValueError(
"Multiple values set for InputPort %r" %
get_inputoutput_name(arg.module))
if not reg.is_descriptor_subclass(arg.module.module_descriptor,
InputPort_desc):
raise ValueError("Module %d is not an InputPort" %
arg.module.id)
inputs[arg.module.id] = arg.value
elif isinstance(arg, Module):
sinks.add(arg.module_id)
# Read kwargs
for key, value in kwargs.iteritems():
key = self.get_input(key) # Might raise KeyError
if key.module_id in inputs:
raise ValueError("Multiple values set for InputPort %r" %
get_inputoutput_name(key.module))
inputs[key.module_id] = value
reason = "API pipeline execution"
sinks = sinks or None
# Use controller only if no inputs were passed in
if (not inputs and self.vistrail is not None and
self.vistrail.current_version == self.version):
controller = self.vistrail.controller
results, changed = controller.execute_workflow_list([[
controller.locator, # locator
self.version, # version
self.pipeline, # pipeline
DummyView(), # view
None, # custom_aliases
None, # custom_params
reason, # reason
sinks, # sinks
None, # extra_info
]])
result, = results
else:
pipeline = self.pipeline
if inputs:
id_scope = IdScope(1)
pipeline = pipeline.do_copy(False, id_scope)
# A hach to get ids from id_scope that we know won't collide:
# make them negative
id_scope.getNewId = lambda t, g=id_scope.getNewId: -g(t)
create_module = \
VistrailController.create_module_from_descriptor_static
create_function = VistrailController.create_function_static
create_connection = VistrailController.create_connection_static
# Fills in the ExternalPipe ports
for module_id, values in inputs.iteritems():
module = pipeline.modules[module_id]
if not isinstance(values, (list, tuple)):
values = [values]
# Guess the type of the InputPort
_, sigstrings, _, _, _ = get_port_spec_info(pipeline, module)
sigstrings = parse_port_spec_string(sigstrings)
# Convert whatever we got to a list of strings, for the
# pipeline
values = [reg.convert_port_val(val, sigstring, None)
for val, sigstring in izip(values, sigstrings)]
if len(values) == 1:
# Create the constant module
constant_desc = reg.get_descriptor_by_name(
*sigstrings[0])
constant_mod = create_module(id_scope, constant_desc)
func = create_function(id_scope, constant_mod,
'value', values)
constant_mod.add_function(func)
pipeline.add_module(constant_mod)
# Connect it to the ExternalPipe port
conn = create_connection(id_scope,
constant_mod, 'value',
module, 'ExternalPipe')
pipeline.db_add_connection(conn)
else:
raise RuntimeError("TODO : create tuple")
interpreter = get_default_interpreter()
result = interpreter.execute(pipeline,
reason=reason,
sinks=sinks)
if result.errors:
raise ExecutionErrors(self, result)
else:
return ExecutionResults(self, result)
def openSpreadsheet(self, fileName):
""" openSpreadsheet(fileName: str) -> None
Open a saved spreadsheet assuming that all VTK files must exist and have
all the version using the saved spreadsheet
"""
def parse_locator(text):
locator = None
wrapper = XMLWrapper()
dom = wrapper.create_document_from_string(text)
root = dom.documentElement
version = None
version = root.getAttribute('version')
if version == '1.0':
for element in named_elements(root, 'locator'):
if str(element.getAttribute('type')) == 'file':
locator = FileLocator.parse(element)
elif str(element.getAttribute('type')) == 'db':
locator = DBLocator.parse(element)
return locator
locators = {}
indexFile = open(fileName, 'r')
contents = indexFile.read()
self.clearTabs()
lidx = 0
lines = contents.split('\n')
tabCount = int(lines[lidx])
lidx += 1
for tabIdx in xrange(tabCount):
# FIXME: eval should pretty much never be used
tabInfo = literal_eval(lines[lidx])
lidx += 1
sheet = spreadsheetRegistry.getSheet(tabInfo[1])(self)
sheet.setDimension(tabInfo[2], tabInfo[3])
self.addTabWidget(sheet, tabInfo[0])
while lines[lidx] != '---':
(r, c, vistrail, pid, cid) = literal_eval(lines[lidx])
locator = vistrail['locator']
if locators.has_key(locator):
vistrail['locator'] = locators[locator]
else:
locators[locator] = parse_locator(vistrail['locator'])
vistrail['locator'] = locators[locator]
self.appendMonitoredLocations((vistrail, pid, cid),
(sheet, r, c))
lidx += 1
lidx += 1
pipelineCount = int(lines[lidx])
lidx += 1
self.loadingMode = True
progress = QtGui.QProgressDialog("Loading spreadsheet...", "&Cancel",
0, pipelineCount, self,
QtCore.Qt.WindowStaysOnTopHint)
progress.show()
for pipelineIdx in xrange(pipelineCount):
# FIXME: eval should pretty much never be used
(serializedLocator, version) = literal_eval(lines[lidx])
try:
locator = locators[serializedLocator]
except KeyError:
locator = parse_locator(serializedLocator)
if locator:
bundle = locator.load()
if isinstance(bundle, SaveBundle):
pipeline = bundle.vistrail.getPipeline(version)
else:
pipeline = bundle.getPipeline(version)
execution = get_default_interpreter()
progress.setValue(pipelineIdx)
QtCore.QCoreApplication.processEvents()
if progress.wasCanceled():
break
kwargs = {
'locator': locator,
'current_version': version,
'view': DummyView(),
}
execution.execute(pipeline, **kwargs)
else:
raise RuntimeError("Couldn't load spreadsheet")
lidx += 1
progress.setValue(pipelineCount)
QtCore.QCoreApplication.processEvents()
self.changeSpreadsheetFileName(fileName)
self.loadingMode = False
indexFile.close()
def execute(self, *args, **kwargs):
"""Execute the pipeline.
Positional arguments are either input values (created from
``module == value``, where `module` is a Module from the pipeline and
`value` is some value or Function instance) for the pipeline's
InputPorts, or Module instances (to select sink modules).
Keyword arguments are also used to set InputPort by looking up inputs
by name.
Example::
input_bound = pipeline.get_input('higher_bound')
input_url = pipeline.get_input('url')
sinkmodule = pipeline.get_module(32)
pipeline.execute(sinkmodule,
input_bound == vt.Function(Integer, 10),
input_url == 'http://www.vistrails.org/',
resolution=15) # kwarg: only one equal sign
"""
pipeline = self.pipeline
sinks = set()
inputs = {}
reg = get_module_registry()
InputPort_desc = reg.get_descriptor_by_name(
get_vistrails_basic_pkg_id(), 'InputPort')
# Read args
for arg in args:
if isinstance(arg, ModuleValuePair):
if arg.module.id in inputs:
raise ValueError("Multiple values set for InputPort %r" %
get_inputoutput_name(arg.module))
if not reg.is_descriptor_subclass(arg.module.module_descriptor,
InputPort_desc):
raise ValueError("Module %d is not an InputPort" %
arg.module.id)
inputs[arg.module.id] = arg.value
elif isinstance(arg, Module):
sinks.add(arg.module_id)
# Read kwargs
for key, value in kwargs.iteritems():
name = key
key = self.get_python_parameter(key) # Might raise KeyError
if name in inputs:
raise ValueError("Multiple values set for input %r" % name)
inputs[name] = [key.module_id, value]
reason = "API pipeline execution"
sinks = sinks or None
# Use controller only if no inputs were passed in
if (not inputs and self.vistrail is not None
and self.vistrail.current_version == self.version):
controller = self.vistrail.controller
results, changed = controller.execute_workflow_list([[
controller.locator, # locator
self.version, # version
self.pipeline, # pipeline
DummyView(), # view
None, # custom_aliases
None, # custom_params
reason, # reason
sinks, # sinks
None, # extra_info
]])
result, = results
else:
# pipeline = self.pipeline
if inputs:
# id_scope = IdScope(1)
# pipeline = pipeline.do_copy(False, id_scope)
# A hach to get ids from id_scope that we know won't collide:
# make them negative
# id_scope.getNewId = lambda t, g=id_scope.getNewId: -g(t)
# create_module = \
# VistrailController.create_module_from_descriptor_static
# create_function = VistrailController.create_function_static
# create_connection = VistrailController.create_connection_static
# Fills in the ExternalPipe ports
for name, input_list in inputs.iteritems():
module_id, values = input_list
module = pipeline.modules[module_id]
if not isinstance(values, (list, tuple)):
values = [values]
'''
# Guess the type of the InputPort
_, sigstrings, _, _, _ = get_port_spec_info(pipeline, module)
sigstrings = parse_port_spec_string(sigstrings)
# Convert whatever we got to a list of strings, for the
# pipeline
values = [reg.convert_port_val(val, sigstring, None)
for val, sigstring in izip(values, sigstrings)]
if len(values) == 1:
# Create the constant module
constant_desc = reg.get_descriptor_by_name(
*sigstrings[0])
#print('Setting desription: ',str(constant_desc),str(sigstrings[0]))
constant_mod = create_module(id_scope, constant_desc)
func = create_function(id_scope, constant_mod,
'value', values)
constant_mod.add_function(func)
pipeline.add_module(constant_mod)
# Connect it to the ExternalPipe port
conn = create_connection(id_scope,
constant_mod, 'value',
module, 'ExternalPipe')
pipeline.db_add_connection(conn)
else:
raise RuntimeError("TODO : create tuple")
'''
port_spec = reg.get_input_port_spec(module, name)
added_functions = {}
tmp_f_id = -1L
tmp_p_id = -1L
function = [
f for f in module.functions if f.name == port_spec.name
]
if function:
function = function[0]
else:
try:
function = added_functions[(module.id,
port_spec.name)]
except KeyError:
# add to function list
params = []
for psi in port_spec.port_spec_items:
parameter = ModuleParam(
id=tmp_p_id,
pos=psi.pos,
name='<no description>',
val=psi.default,
type=psi.descriptor.sigstring)
params.append(parameter)
tmp_p_id -= 1
function = ModuleFunction(
id=tmp_f_id,
pos=module.getNumFunctions(),
name=port_spec.name,
parameters=params)
tmp_f_id -= 1
added_functions[(module.id,
port_spec.name)] = function
action = vistrails.core.db.action.create_action([
('add', function, module.vtType, module.id)
])
# function_actions.append(action)
parameter = function.params[0]
# find old parameter
old_param = parameter
actions = []
for v in values:
desc = reg.get_descriptor_by_name(
'org.vistrails.vistrails.basic', 'String', None)
if not isinstance(v, str):
str_value = desc.module.translate_to_string(v)
else:
str_value = v
new_param = ModuleParam(id=tmp_p_id,
pos=old_param.pos,
name=old_param.name,
alias=old_param.alias,
val=str_value,
type=old_param.type)
tmp_p_id -= 1
action_spec = ('change', old_param, new_param,
function.vtType, function.real_id)
action = vistrails.core.db.action.create_action(
[action_spec])
actions.append(action)
# controller = self.vistrail.controller
self.controller.perform_action(action)
#########################################################################
interpreter = get_default_interpreter()
result = interpreter.execute(pipeline,
locator=self.locator,
reason=reason,
sinks=sinks,
actions=actions)
if result.errors:
raise ExecutionErrors(self, result)
else:
return ExecutionResults(self, result)
