def test_outputportspec_cache(self):
"""
Test that signature hash includes port specs
If it is not included, a module with different port spec may
be used in the cache, leading to an exception.
"""
from vistrails import api
from vistrails.core.vistrail.port_spec import PortSpec
api.new_vistrail()
c = api.get_current_controller()
ps = api.add_module(0, 0, 'org.vistrails.vistrails.basic',
'PythonSource', '')
api.change_parameter(ps.id, 'source', ['a = b = 1\ncache_this()'])
so = api.add_module(0, 0, 'org.vistrails.vistrails.basic', 'Integer',
'')
api.add_port_spec(
ps.id,
PortSpec(name='a',
type='output',
sigstring='org.vistrails.vistrails.basic:Integer'))
api.add_connection(ps.id, 'a', so.id, 'value')
# adds ps to cache
self.assertEqual(c.execute_current_workflow()[0][0].errors, {})
api.add_port_spec(
ps.id,
PortSpec(name='b',
type='output',
sigstring='org.vistrails.vistrails.basic:Integer'))
api.add_connection(ps.id, 'b', so.id, 'value')
# will fail if outputportspec is not hashed and cache is reused
self.assertEqual(c.execute_current_workflow()[0][0].errors, {})
def convert(_module):
if _module.__class__ == Module:
return
_module.__class__ = Module
for _port_spec in _module.db_portSpecs:
PortSpec.convert(_port_spec)
if _module.db_location:
Location.convert(_module.db_location)
for _function in _module.db_functions:
ModuleFunction.convert(_function)
for _annotation in _module.db_get_annotations():
Annotation.convert(_annotation)
_module.set_defaults()
def convert(_desc):
if _desc.__class__ == ModuleDescriptor:
return
_desc.__class__ = ModuleDescriptor
for port_spec in _desc.db_portSpecs:
PortSpec.convert(port_spec)
# do more init stuff
_desc.children = []
_desc.module = None
_desc._base_descriptor = None
_desc._port_count = 0
_desc.set_defaults()
def convert(_desc):
if _desc.__class__ == ModuleDescriptor:
return
_desc.__class__ = ModuleDescriptor
for port_spec in _desc.db_portSpecs:
PortSpec.convert(port_spec)
# do more init stuff
_desc.children = []
_desc.module = None
_desc._base_descriptor = None
_desc._port_count = 0
_desc.set_defaults()
def get_spec(self, port_type):
""" get_spec(port_type) -> PortSpec
Returns a PortSpec corresponding to the function parameter
types set. This is useful to make module functions look more
like they are 'regular' modules and connections (which is what
they get compiled down to in execution).
port_type is either 'input' or 'output', as strings, which
simply gets set on the spec being returned.
"""
assert port_type == 'input' or port_type == 'output'
result = PortSpec(sigstring=self.sigstring)
result.type = port_type
return result
def get_spec(self, port_type):
""" get_spec(port_type) -> PortSpec
Returns a PortSpec corresponding to the function parameter
types set. This is useful to make module functions look more
like they are 'regular' modules and connections (which is what
they get compiled down to in execution).
port_type is either 'input' or 'output', as strings, which
simply gets set on the spec being returned.
"""
assert port_type == 'input' or port_type == 'output'
result = PortSpec(sigstring=self.sigstring)
result.type = port_type
return result
def convert(_module):
if _module.__class__ == Module:
return
_module.__class__ = Module
for _port_spec in _module.db_portSpecs:
PortSpec.convert(_port_spec)
if _module.db_location:
Location.convert(_module.db_location)
for _function in _module.db_functions:
ModuleFunction.convert(_function)
for _annotation in _module.db_get_annotations():
Annotation.convert(_annotation)
for _control_parameter in _module.db_get_controlParameters():
ModuleControlParam.convert(_control_parameter)
_module.set_defaults()
def create_workflow(self, c):
upgrade_test_pkg = 'org.vistrails.vistrails.tests.upgrade'
d1 = ModuleDescriptor(package=upgrade_test_pkg,
name='TestUpgradeA',
namespace='',
package_version='0.8')
m1 = c.create_module_from_descriptor(d1, use_desc_pkg_version=True)
m1.is_valid = False
c.add_module_action(m1)
d2 = ModuleDescriptor(package=upgrade_test_pkg,
name='TestUpgradeB',
namespace='',
package_version='0.8')
m2 = c.create_module_from_descriptor(d2, use_desc_pkg_version=True)
m2.is_valid = False
c.add_module_action(m2)
basic_pkg = get_vistrails_basic_pkg_id()
psi = PortSpecItem(module="Float",
package=basic_pkg,
namespace="",
pos=0)
function_port_spec = PortSpec(name="a", type="input", items=[psi])
f = c.create_function(m1, function_port_spec, [12])
c.add_function_action(m1, f)
conn_out_psi = PortSpecItem(module="Integer",
package=basic_pkg,
namespace="",
pos=0)
conn_out_spec = PortSpec(name="z", type="output", items=[conn_out_psi])
conn_in_psi = PortSpecItem(module="Integer",
package=basic_pkg,
namespace="",
pos=0)
conn_in_spec = PortSpec(name="b", type="input", items=[conn_in_psi])
conn = c.create_connection(m1, conn_out_spec, m2, conn_in_spec)
c.add_connection_action(conn)
return c.current_version
def remap_functions(old_module, new_module, function_remap):
# FIXME need use_registry flag passed through!
use_registry = True
function_ops = []
for function in old_module.functions:
if function.name in function_remap:
remap = function_remap[function.name]
if remap is None:
# don't add the function back in
continue
elif not isinstance(remap, basestring):
function_ops.extend(remap(function, new_module))
continue
else:
function_name = remap
if len(function.parameters) > 0:
new_param_vals, aliases = zip(
*[(p.strValue, p.alias) for p in function.parameters])
else:
new_param_vals = []
aliases = []
if use_registry:
function_port_spec = function_name
else:
def mk_psi(pos):
psi = PortSpecItem(module="Module",
package=basic_pkg,
namespace="",
pos=pos)
return psi
n_items = len(new_param_vals)
function_port_spec = PortSpec(
name=function_name,
items=[mk_psi(i) for i in xrange(n_items)])
new_function = controller.create_function(
new_module, function_port_spec, new_param_vals, aliases)
new_module.add_function(new_function)
if None in function_remap:
# used to add new functions
remap = function_remap[None]
function_ops.extend(remap(None, new_module))
return function_ops
def replace_generic(controller,
pipeline,
old_module,
new_module,
function_remap=None,
src_port_remap=None,
dst_port_remap=None,
annotation_remap=None,
control_param_remap=None,
use_registry=True):
if function_remap is None:
function_remap = {}
if src_port_remap is None:
src_port_remap = {}
if dst_port_remap is None:
dst_port_remap = {}
if annotation_remap is None:
annotation_remap = {}
if control_param_remap is None:
control_param_remap = {}
basic_pkg = get_vistrails_basic_pkg_id()
ops = []
ops.extend(controller.delete_module_list_ops(pipeline,
[old_module.id]))
for annotation in old_module.annotations:
if annotation.key not in annotation_remap:
annotation_key = annotation.key
else:
remap = annotation_remap[annotation.key]
if remap is None:
# don't add the annotation back in
continue
elif not isinstance(remap, basestring):
ops.extend(remap(annotation))
continue
else:
annotation_key = remap
new_annotation = \
Annotation(id=controller.id_scope.getNewId(Annotation.vtType),
key=annotation_key,
value=annotation.value)
new_module.add_annotation(new_annotation)
for control_param in old_module.control_parameters:
if control_param.name not in control_param_remap:
control_param_name = control_param.name
else:
remap = control_param_remap[control_param.name]
if remap is None:
# don't add the control param back in
continue
elif not isinstance(remap, basestring):
ops.extend(remap(control_param))
continue
else:
control_param_name = remap
new_control_param = \
ModuleControlParam(id=controller.id_scope.getNewId(
ModuleControlParam.vtType),
name=control_param_name,
value=control_param.value)
new_module.add_control_parameter(new_control_param)
if not old_module.is_group() and not old_module.is_abstraction():
for port_spec in old_module.port_spec_list:
if port_spec.type == 'input':
if port_spec.name not in dst_port_remap:
spec_name = port_spec.name
else:
remap = dst_port_remap[port_spec.name]
if remap is None:
continue
elif not isinstance(remap, basestring):
ops.extend(remap(port_spec))
continue
else:
spec_name = remap
elif port_spec.type == 'output':
if port_spec.name not in src_port_remap:
spec_name = port_spec.name
else:
remap = src_port_remap[port_spec.name]
if remap is None:
continue
elif not isinstance(remap, basestring):
ops.extend(remap(port_spec))
continue
else:
spec_name = remap
new_spec = port_spec.do_copy(True, controller.id_scope, {})
new_spec.name = spec_name
new_module.add_port_spec(new_spec)
function_ops = []
for function in old_module.functions:
if function.name not in function_remap:
function_name = function.name
else:
remap = function_remap[function.name]
if remap is None:
# don't add the function back in
continue
elif not isinstance(remap, basestring):
function_ops.extend(remap(function, new_module))
continue
else:
function_name = remap
if len(function.parameters) > 0:
new_param_vals, aliases = zip(*[(p.strValue, p.alias)
for p in function.parameters])
else:
new_param_vals = []
aliases = []
if use_registry:
function_port_spec = function_name
else:
def mk_psi(pos):
psi = PortSpecItem(module="Module",
package=basic_pkg,
namespace="",
pos=pos)
return psi
n_items = len(new_param_vals)
function_port_spec = PortSpec(
name=function_name,
items=[mk_psi(i) for i in xrange(n_items)])
new_function = controller.create_function(new_module,
function_port_spec,
new_param_vals, aliases)
new_module.add_function(new_function)
if None in function_remap:
# used to add new functions
remap = function_remap[None]
function_ops.extend(remap(None, new_module))
# add the new module
ops.append(('add', new_module))
ops.extend(function_ops)
create_new_connection = UpgradeWorkflowHandler.create_new_connection
for _, conn_id in pipeline.graph.edges_from(old_module.id):
old_conn = pipeline.connections[conn_id]
if old_conn.source.name not in src_port_remap:
source_name = old_conn.source.name
else:
remap = src_port_remap[old_conn.source.name]
if remap is None:
# don't add this connection back in
continue
elif not isinstance(remap, basestring):
ops.extend(remap(old_conn, new_module))
continue
else:
source_name = remap
old_dst_module = pipeline.modules[old_conn.destination.moduleId]
if use_registry:
source_port = source_name
else:
source_port = Port(name=source_name,
type='source',
signature=create_port_spec_string([
(basic_pkg, 'Variant', '')
]))
new_conn = create_new_connection(controller, new_module,
source_port, old_dst_module,
old_conn.destination)
ops.append(('add', new_conn))
for _, conn_id in pipeline.graph.edges_to(old_module.id):
old_conn = pipeline.connections[conn_id]
if old_conn.destination.name not in dst_port_remap:
destination_name = old_conn.destination.name
else:
remap = dst_port_remap[old_conn.destination.name]
if remap is None:
# don't add this connection back in
continue
elif not isinstance(remap, basestring):
ops.extend(remap(old_conn, new_module))
continue
else:
destination_name = remap
old_src_module = pipeline.modules[old_conn.source.moduleId]
if use_registry:
destination_port = destination_name
else:
destination_port = Port(name=destination_name,
type='destination',
signature=create_port_spec_string([
(basic_pkg, 'Variant', '')
]))
new_conn = create_new_connection(controller, old_src_module,
old_conn.source, new_module,
destination_port)
ops.append(('add', new_conn))
return [vistrails.core.db.action.create_action(ops)]
def update_module(self, module):
""" update_module(module: Module) -> None
Setup this tree widget to show functions of module
"""
self.setColumnHidden(0, True)
self.setColumnHidden(1, not self.ports_visible)
# this is strange but if you try to clear the widget when the focus is
# in one of the items (after setting a parameter for example),
# VisTrails crashes on a Mac (Emanuele) This is probably a Qt bug
w = QtGui.QApplication.focusWidget()
if self.isAncestorOf(w):
w.clearFocus()
self.clear()
self.module = module
self.port_spec_items = {}
self.function_map = {}
if module and module.is_valid:
reg = get_module_registry()
descriptor = module.module_descriptor
if self.port_type == 'input':
self.setColumnHidden(
0, not get_vistrails_configuration().check(
'showInlineParameterWidgets'))
port_specs = module.destinationPorts()
connected_ports = module.connected_input_ports
visible_ports = module.visible_input_ports
elif self.port_type == 'output':
port_specs = module.sourcePorts()
connected_ports = module.connected_output_ports
visible_ports = module.visible_output_ports
else:
raise TypeError("Unknown port type: '%s'" % self.port_type)
for port_spec in sorted(port_specs, key=lambda x: x.name):
connected = port_spec.name in connected_ports and \
connected_ports[port_spec.name] > 0
item = PortItem(port_spec, connected, port_spec.optional,
port_spec.name in visible_ports, port_spec.name
in module.editable_input_ports)
self.addTopLevelItem(item)
self.port_spec_items[port_spec.name] = (port_spec, item)
if self.port_type == 'input':
for function in module.functions:
if not function.is_valid:
continue
port_spec, item = self.port_spec_items[function.name]
subitem = self.entry_klass(port_spec, function,
self.types_visible)
self.function_map[function.real_id] = subitem
item.addChild(subitem)
subitem.setFirstColumnSpanned(True)
self.setItemWidget(subitem, 2, subitem.get_widget())
item.setExpanded(True)
# self.setItemWidget(item, 0, item.get_visible())
# self.setItemWidget(item, 1, item.get_connected())
# i = QTreeWidgetItem(self)
# self.addTopLevelItem(i)
# i.setText(2, port_spec.name)
# visible_checkbox = QtGui.QCheckBox()
# self.setItemWidget(i, 0, visible_checkbox)
# connceted_checkbox = QtGui.QCheckBox()
# connected_checkbox.setEnabled(False)
# self.setItemWidget(i, 1, connected_checkbox)
# Highlight unset ports
for _, item in self.port_spec_items.itervalues():
item.calcUnset()
self.sortItems(0, QtCore.Qt.AscendingOrder)
# base_items = {}
# # Create the base widget item for each descriptor
# for descriptor in moduleHierarchy:
# baseName = descriptor.name
# base_package = descriptor.identifier
# baseItem = QMethodTreeWidgetItem(None,
# None,
# self,
# ([]
# << baseName
# << ''))
# base_items[descriptor] = baseItem
# method_specs = {}
# # do this in reverse to ensure proper overloading
# # !!! NOTE: we have to use ***all*** input ports !!!
# # because a subclass can overload a port with a
# # type that isn't a method
# for descriptor in reversed(moduleHierarchy):
# method_specs.update((name, (descriptor, spec))
# for name, spec in \
# registry.module_ports('input',
# descriptor))
# # add local registry last so that it takes precedence
# method_specs.update((spec.name, (descriptor, spec))
# for spec in module.port_spec_list
# if spec.type == 'input')
# for _, (desc, method_spec) in sorted(method_specs.iteritems()):
# if registry.is_method(method_spec):
# baseItem = base_items[desc]
# sig = method_spec.short_sigstring
# QMethodTreeWidgetItem(module,
# method_spec,
# baseItem,
# ([]
# << method_spec.name
# << sig))
# self.expandAll()
# self.resizeColumnToContents(2)
# show invalid module attributes
if module and not module.is_valid and self.port_type == 'input':
for function in module.functions:
if function.name in self.port_spec_items:
port_spec, item = self.port_spec_items[function.name]
else:
sigstring = create_port_spec_string([
(basic_identifier, "String")
for i in xrange(len(function.parameters))
])
port_spec = PortSpec(name=function.name,
type='input',
sigstring=sigstring)
item = PortItem(port_spec, False, False, False)
self.addTopLevelItem(item)
self.port_spec_items[port_spec.name] = (port_spec, item)
subitem = self.entry_klass(port_spec, function)
self.function_map[function.real_id] = subitem
item.addChild(subitem)
subitem.setFirstColumnSpanned(True)
self.setItemWidget(subitem, 2, subitem.get_widget())
item.setExpanded(True)
def execute(modules,
connections=[],
add_port_specs=[],
enable_pkg=True,
full_results=False):
"""Build a pipeline and execute it.
This is useful to simply build a pipeline in a test case, and run it. When
doing that, intercept_result() can be used to check the results of each
module.
modules is a list of module tuples describing the modules to be created,
with the following format:
[('ModuleName', 'package.identifier', [
# Functions
('port_name', [
# Function parameters
('Signature', 'value-as-string'),
]),
])]
connections is a list of tuples describing the connections to make, with
the following format:
[
(source_module_index, 'source_port_name',
dest_module_index, 'dest_module_name'),
]
add_port_specs is a list of specs to add to modules, with the following
format:
[
(mod_id, 'input'/'output', 'portname',
'(port_sig)'),
]
It is useful to test modules that can have custom ports through a
configuration widget.
The function returns the 'errors' dict it gets from the interpreter, so you
should use a construct like self.assertFalse(execute(...)) if the execution
is not supposed to fail.
For example, this creates (and runs) an Integer module with its value set
to 44, connected to a PythonCalc module, connected to a StandardOutput:
self.assertFalse(execute([
('Float', 'org.vistrails.vistrails.basic', [
('value', [('Float', '44.0')]),
]),
('PythonCalc', 'org.vistrails.vistrails.pythoncalc', [
('value2', [('Float', '2.0')]),
('op', [('String', '-')]),
]),
('StandardOutput', 'org.vistrails.vistrails.basic', []),
],
[
(0, 'value', 1, 'value1'),
(1, 'value', 2, 'value'),
]))
"""
from vistrails.core.db.locator import XMLFileLocator
from vistrails.core.modules.module_registry import MissingPackage
from vistrails.core.packagemanager import get_package_manager
from vistrails.core.utils import DummyView
from vistrails.core.vistrail.connection import Connection
from vistrails.core.vistrail.module import Module
from vistrails.core.vistrail.module_function import ModuleFunction
from vistrails.core.vistrail.module_param import ModuleParam
from vistrails.core.vistrail.pipeline import Pipeline
from vistrails.core.vistrail.port import Port
from vistrails.core.vistrail.port_spec import PortSpec
from vistrails.core.interpreter.noncached import Interpreter
pm = get_package_manager()
port_spec_per_module = {} # mod_id -> [portspec: PortSpec]
j = 0
for i, (mod_id, inout, name, sig) in enumerate(add_port_specs):
mod_specs = port_spec_per_module.setdefault(mod_id, [])
ps = PortSpec(id=i, name=name, type=inout, sigstring=sig, sort_key=-1)
for psi in ps.port_spec_items:
psi.id = j
j += 1
mod_specs.append(ps)
pipeline = Pipeline()
module_list = []
for i, (name, identifier, functions) in enumerate(modules):
function_list = []
try:
pkg = pm.get_package(identifier)
except MissingPackage:
if not enable_pkg:
raise
dep_graph = pm.build_dependency_graph([identifier])
for pkg_id in pm.get_ordered_dependencies(dep_graph):
pkg = pm.identifier_is_available(pkg_id)
if pkg is None:
raise
pm.late_enable_package(pkg.codepath)
pkg = pm.get_package(identifier)
for func_name, params in functions:
param_list = []
for j, (param_type, param_val) in enumerate(params):
param_list.append(
ModuleParam(pos=j, type=param_type, val=param_val))
function_list.append(
ModuleFunction(name=func_name, parameters=param_list))
name = name.rsplit('|', 1)
if len(name) == 2:
namespace, name = name
else:
namespace = None
name, = name
module = Module(name=name,
namespace=namespace,
package=identifier,
version=pkg.version,
id=i,
functions=function_list)
for port_spec in port_spec_per_module.get(i, []):
module.add_port_spec(port_spec)
pipeline.add_module(module)
module_list.append(module)
for i, (sid, sport, did, dport) in enumerate(connections):
s_sig = module_list[sid].get_port_spec(sport, 'output').sigstring
d_sig = module_list[did].get_port_spec(dport, 'input').sigstring
pipeline.add_connection(
Connection(id=i,
ports=[
Port(id=i * 2,
type='source',
moduleId=sid,
name=sport,
signature=s_sig),
Port(id=i * 2 + 1,
type='destination',
moduleId=did,
name=dport,
signature=d_sig),
]))
interpreter = Interpreter.get()
result = interpreter.execute(pipeline,
locator=XMLFileLocator('foo.xml'),
current_version=1,
view=DummyView())
if full_results:
return result
else:
# Allows to do self.assertFalse(execute(...))
return result.errors
def _update_func(self, port_spec, *args, **kwargs):
# print 'running _update_func', port_spec.name
# print args
vt_api = get_api()
if port_spec.type != 'input':
if self._module.has_port_spec(port_spec.name, 'input'):
port_spec = \
self._module.get_port_spec(port_spec.name, 'input')
else:
raise TypeError("cannot update an output port spec")
# FIXME deal with kwargs
num_ports = 0
num_params = 0
for value in args:
# print 'processing', type(value), value
if isinstance(value, Port):
# make connection to specified output port
# print 'updating port'
num_ports += 1
elif isinstance(value, Module):
# make connection to 'self' output port of value
# print 'updating module'
num_ports += 1
else:
# print 'update literal', type(value), value
num_params += 1
if num_ports > 1 or (num_ports == 1 and num_params > 0):
reg = get_module_registry()
tuple_desc = reg.get_descriptor_by_name(basic_pkg, 'Tuple')
tuple_module = vt_api.add_module_from_descriptor(tuple_desc)
output_port_spec = PortSpec(id=-1,
name='value',
type='output',
sigstring=port_spec.sigstring)
vt_api.add_port_spec(tuple_module, output_port_spec)
self._update_func(port_spec, tuple_module.value())
assert len(port_spec.descriptors()) == len(args)
for i, descriptor in enumerate(port_spec.descriptors()):
arg_name = 'arg%d' % i
sigstring = create_port_spec_string([descriptor.spec_tuple])
tuple_port_spec = PortSpec(id=-1,
name=arg_name,
type='input',
sigstring=sigstring)
vt_api.add_port_spec(tuple_module, tuple_port_spec)
tuple_module._process_attr_value(arg_name, args[i])
elif num_ports == 1:
other = args[0]
if isinstance(other, Port):
if other._port_spec.type != 'output':
other_module = other._vistrails_module._module
if other_module.has_port_spec(port_spec.name,
'output'):
other_port_spec = \
other_module.get_port_spec(port_spec.name,
'output')
else:
raise TypeError("cannot update an input "
"port spec")
else:
other_port_spec = other._port_spec
vt_api.add_connection(other._vistrails_module,
other_port_spec,
self,
port_spec)
elif isinstance(other, Module):
other_port_spec = \
other._module.get_port_spec('self', 'output')
vt_api.add_connection(other,
other_port_spec,
self,
port_spec)
else:
vt_api.change_parameter(self,
port_spec.name,
[str(x) for x in args])