def test_outputportspec_cache(self): """ Test that signature hash includes port specs If it is not included, a module with different port spec may be used in the cache, leading to an exception. """ from vistrails import api from vistrails.core.vistrail.port_spec import PortSpec api.new_vistrail() c = api.get_current_controller() ps = api.add_module(0, 0, 'org.vistrails.vistrails.basic', 'PythonSource', '') api.change_parameter(ps.id, 'source', ['a = b = 1\ncache_this()']) so = api.add_module(0, 0, 'org.vistrails.vistrails.basic', 'Integer', '') api.add_port_spec( ps.id, PortSpec(name='a', type='output', sigstring='org.vistrails.vistrails.basic:Integer')) api.add_connection(ps.id, 'a', so.id, 'value') # adds ps to cache self.assertEqual(c.execute_current_workflow()[0][0].errors, {}) api.add_port_spec( ps.id, PortSpec(name='b', type='output', sigstring='org.vistrails.vistrails.basic:Integer')) api.add_connection(ps.id, 'b', so.id, 'value') # will fail if outputportspec is not hashed and cache is reused self.assertEqual(c.execute_current_workflow()[0][0].errors, {})
def convert(_module): if _module.__class__ == Module: return _module.__class__ = Module for _port_spec in _module.db_portSpecs: PortSpec.convert(_port_spec) if _module.db_location: Location.convert(_module.db_location) for _function in _module.db_functions: ModuleFunction.convert(_function) for _annotation in _module.db_get_annotations(): Annotation.convert(_annotation) _module.set_defaults()
def convert(_desc): if _desc.__class__ == ModuleDescriptor: return _desc.__class__ = ModuleDescriptor for port_spec in _desc.db_portSpecs: PortSpec.convert(port_spec) # do more init stuff _desc.children = [] _desc.module = None _desc._base_descriptor = None _desc._port_count = 0 _desc.set_defaults()
def get_spec(self, port_type): """ get_spec(port_type) -> PortSpec Returns a PortSpec corresponding to the function parameter types set. This is useful to make module functions look more like they are 'regular' modules and connections (which is what they get compiled down to in execution). port_type is either 'input' or 'output', as strings, which simply gets set on the spec being returned. """ assert port_type == 'input' or port_type == 'output' result = PortSpec(sigstring=self.sigstring) result.type = port_type return result
def convert(_module): if _module.__class__ == Module: return _module.__class__ = Module for _port_spec in _module.db_portSpecs: PortSpec.convert(_port_spec) if _module.db_location: Location.convert(_module.db_location) for _function in _module.db_functions: ModuleFunction.convert(_function) for _annotation in _module.db_get_annotations(): Annotation.convert(_annotation) for _control_parameter in _module.db_get_controlParameters(): ModuleControlParam.convert(_control_parameter) _module.set_defaults()
def create_workflow(self, c): upgrade_test_pkg = 'org.vistrails.vistrails.tests.upgrade' d1 = ModuleDescriptor(package=upgrade_test_pkg, name='TestUpgradeA', namespace='', package_version='0.8') m1 = c.create_module_from_descriptor(d1, use_desc_pkg_version=True) m1.is_valid = False c.add_module_action(m1) d2 = ModuleDescriptor(package=upgrade_test_pkg, name='TestUpgradeB', namespace='', package_version='0.8') m2 = c.create_module_from_descriptor(d2, use_desc_pkg_version=True) m2.is_valid = False c.add_module_action(m2) basic_pkg = get_vistrails_basic_pkg_id() psi = PortSpecItem(module="Float", package=basic_pkg, namespace="", pos=0) function_port_spec = PortSpec(name="a", type="input", items=[psi]) f = c.create_function(m1, function_port_spec, [12]) c.add_function_action(m1, f) conn_out_psi = PortSpecItem(module="Integer", package=basic_pkg, namespace="", pos=0) conn_out_spec = PortSpec(name="z", type="output", items=[conn_out_psi]) conn_in_psi = PortSpecItem(module="Integer", package=basic_pkg, namespace="", pos=0) conn_in_spec = PortSpec(name="b", type="input", items=[conn_in_psi]) conn = c.create_connection(m1, conn_out_spec, m2, conn_in_spec) c.add_connection_action(conn) return c.current_version
def remap_functions(old_module, new_module, function_remap): # FIXME need use_registry flag passed through! use_registry = True function_ops = [] for function in old_module.functions: if function.name in function_remap: remap = function_remap[function.name] if remap is None: # don't add the function back in continue elif not isinstance(remap, basestring): function_ops.extend(remap(function, new_module)) continue else: function_name = remap if len(function.parameters) > 0: new_param_vals, aliases = zip( *[(p.strValue, p.alias) for p in function.parameters]) else: new_param_vals = [] aliases = [] if use_registry: function_port_spec = function_name else: def mk_psi(pos): psi = PortSpecItem(module="Module", package=basic_pkg, namespace="", pos=pos) return psi n_items = len(new_param_vals) function_port_spec = PortSpec( name=function_name, items=[mk_psi(i) for i in xrange(n_items)]) new_function = controller.create_function( new_module, function_port_spec, new_param_vals, aliases) new_module.add_function(new_function) if None in function_remap: # used to add new functions remap = function_remap[None] function_ops.extend(remap(None, new_module)) return function_ops
def replace_generic(controller, pipeline, old_module, new_module, function_remap=None, src_port_remap=None, dst_port_remap=None, annotation_remap=None, control_param_remap=None, use_registry=True): if function_remap is None: function_remap = {} if src_port_remap is None: src_port_remap = {} if dst_port_remap is None: dst_port_remap = {} if annotation_remap is None: annotation_remap = {} if control_param_remap is None: control_param_remap = {} basic_pkg = get_vistrails_basic_pkg_id() ops = [] ops.extend(controller.delete_module_list_ops(pipeline, [old_module.id])) for annotation in old_module.annotations: if annotation.key not in annotation_remap: annotation_key = annotation.key else: remap = annotation_remap[annotation.key] if remap is None: # don't add the annotation back in continue elif not isinstance(remap, basestring): ops.extend(remap(annotation)) continue else: annotation_key = remap new_annotation = \ Annotation(id=controller.id_scope.getNewId(Annotation.vtType), key=annotation_key, value=annotation.value) new_module.add_annotation(new_annotation) for control_param in old_module.control_parameters: if control_param.name not in control_param_remap: control_param_name = control_param.name else: remap = control_param_remap[control_param.name] if remap is None: # don't add the control param back in continue elif not isinstance(remap, basestring): ops.extend(remap(control_param)) continue else: control_param_name = remap new_control_param = \ ModuleControlParam(id=controller.id_scope.getNewId( ModuleControlParam.vtType), name=control_param_name, value=control_param.value) new_module.add_control_parameter(new_control_param) if not old_module.is_group() and not old_module.is_abstraction(): for port_spec in old_module.port_spec_list: if port_spec.type == 'input': if port_spec.name not in dst_port_remap: spec_name = port_spec.name else: remap = dst_port_remap[port_spec.name] if remap is None: continue elif not isinstance(remap, basestring): ops.extend(remap(port_spec)) continue else: spec_name = remap elif port_spec.type == 'output': if port_spec.name not in src_port_remap: spec_name = port_spec.name else: remap = src_port_remap[port_spec.name] if remap is None: continue elif not isinstance(remap, basestring): ops.extend(remap(port_spec)) continue else: spec_name = remap new_spec = port_spec.do_copy(True, controller.id_scope, {}) new_spec.name = spec_name new_module.add_port_spec(new_spec) function_ops = [] for function in old_module.functions: if function.name not in function_remap: function_name = function.name else: remap = function_remap[function.name] if remap is None: # don't add the function back in continue elif not isinstance(remap, basestring): function_ops.extend(remap(function, new_module)) continue else: function_name = remap if len(function.parameters) > 0: new_param_vals, aliases = zip(*[(p.strValue, p.alias) for p in function.parameters]) else: new_param_vals = [] aliases = [] if use_registry: function_port_spec = function_name else: def mk_psi(pos): psi = PortSpecItem(module="Module", package=basic_pkg, namespace="", pos=pos) return psi n_items = len(new_param_vals) function_port_spec = PortSpec( name=function_name, items=[mk_psi(i) for i in xrange(n_items)]) new_function = controller.create_function(new_module, function_port_spec, new_param_vals, aliases) new_module.add_function(new_function) if None in function_remap: # used to add new functions remap = function_remap[None] function_ops.extend(remap(None, new_module)) # add the new module ops.append(('add', new_module)) ops.extend(function_ops) create_new_connection = UpgradeWorkflowHandler.create_new_connection for _, conn_id in pipeline.graph.edges_from(old_module.id): old_conn = pipeline.connections[conn_id] if old_conn.source.name not in src_port_remap: source_name = old_conn.source.name else: remap = src_port_remap[old_conn.source.name] if remap is None: # don't add this connection back in continue elif not isinstance(remap, basestring): ops.extend(remap(old_conn, new_module)) continue else: source_name = remap old_dst_module = pipeline.modules[old_conn.destination.moduleId] if use_registry: source_port = source_name else: source_port = Port(name=source_name, type='source', signature=create_port_spec_string([ (basic_pkg, 'Variant', '') ])) new_conn = create_new_connection(controller, new_module, source_port, old_dst_module, old_conn.destination) ops.append(('add', new_conn)) for _, conn_id in pipeline.graph.edges_to(old_module.id): old_conn = pipeline.connections[conn_id] if old_conn.destination.name not in dst_port_remap: destination_name = old_conn.destination.name else: remap = dst_port_remap[old_conn.destination.name] if remap is None: # don't add this connection back in continue elif not isinstance(remap, basestring): ops.extend(remap(old_conn, new_module)) continue else: destination_name = remap old_src_module = pipeline.modules[old_conn.source.moduleId] if use_registry: destination_port = destination_name else: destination_port = Port(name=destination_name, type='destination', signature=create_port_spec_string([ (basic_pkg, 'Variant', '') ])) new_conn = create_new_connection(controller, old_src_module, old_conn.source, new_module, destination_port) ops.append(('add', new_conn)) return [vistrails.core.db.action.create_action(ops)]
def update_module(self, module): """ update_module(module: Module) -> None Setup this tree widget to show functions of module """ self.setColumnHidden(0, True) self.setColumnHidden(1, not self.ports_visible) # this is strange but if you try to clear the widget when the focus is # in one of the items (after setting a parameter for example), # VisTrails crashes on a Mac (Emanuele) This is probably a Qt bug w = QtGui.QApplication.focusWidget() if self.isAncestorOf(w): w.clearFocus() self.clear() self.module = module self.port_spec_items = {} self.function_map = {} if module and module.is_valid: reg = get_module_registry() descriptor = module.module_descriptor if self.port_type == 'input': self.setColumnHidden( 0, not get_vistrails_configuration().check( 'showInlineParameterWidgets')) port_specs = module.destinationPorts() connected_ports = module.connected_input_ports visible_ports = module.visible_input_ports elif self.port_type == 'output': port_specs = module.sourcePorts() connected_ports = module.connected_output_ports visible_ports = module.visible_output_ports else: raise TypeError("Unknown port type: '%s'" % self.port_type) for port_spec in sorted(port_specs, key=lambda x: x.name): connected = port_spec.name in connected_ports and \ connected_ports[port_spec.name] > 0 item = PortItem(port_spec, connected, port_spec.optional, port_spec.name in visible_ports, port_spec.name in module.editable_input_ports) self.addTopLevelItem(item) self.port_spec_items[port_spec.name] = (port_spec, item) if self.port_type == 'input': for function in module.functions: if not function.is_valid: continue port_spec, item = self.port_spec_items[function.name] subitem = self.entry_klass(port_spec, function, self.types_visible) self.function_map[function.real_id] = subitem item.addChild(subitem) subitem.setFirstColumnSpanned(True) self.setItemWidget(subitem, 2, subitem.get_widget()) item.setExpanded(True) # self.setItemWidget(item, 0, item.get_visible()) # self.setItemWidget(item, 1, item.get_connected()) # i = QTreeWidgetItem(self) # self.addTopLevelItem(i) # i.setText(2, port_spec.name) # visible_checkbox = QtGui.QCheckBox() # self.setItemWidget(i, 0, visible_checkbox) # connceted_checkbox = QtGui.QCheckBox() # connected_checkbox.setEnabled(False) # self.setItemWidget(i, 1, connected_checkbox) # Highlight unset ports for _, item in self.port_spec_items.itervalues(): item.calcUnset() self.sortItems(0, QtCore.Qt.AscendingOrder) # base_items = {} # # Create the base widget item for each descriptor # for descriptor in moduleHierarchy: # baseName = descriptor.name # base_package = descriptor.identifier # baseItem = QMethodTreeWidgetItem(None, # None, # self, # ([] # << baseName # << '')) # base_items[descriptor] = baseItem # method_specs = {} # # do this in reverse to ensure proper overloading # # !!! NOTE: we have to use ***all*** input ports !!! # # because a subclass can overload a port with a # # type that isn't a method # for descriptor in reversed(moduleHierarchy): # method_specs.update((name, (descriptor, spec)) # for name, spec in \ # registry.module_ports('input', # descriptor)) # # add local registry last so that it takes precedence # method_specs.update((spec.name, (descriptor, spec)) # for spec in module.port_spec_list # if spec.type == 'input') # for _, (desc, method_spec) in sorted(method_specs.iteritems()): # if registry.is_method(method_spec): # baseItem = base_items[desc] # sig = method_spec.short_sigstring # QMethodTreeWidgetItem(module, # method_spec, # baseItem, # ([] # << method_spec.name # << sig)) # self.expandAll() # self.resizeColumnToContents(2) # show invalid module attributes if module and not module.is_valid and self.port_type == 'input': for function in module.functions: if function.name in self.port_spec_items: port_spec, item = self.port_spec_items[function.name] else: sigstring = create_port_spec_string([ (basic_identifier, "String") for i in xrange(len(function.parameters)) ]) port_spec = PortSpec(name=function.name, type='input', sigstring=sigstring) item = PortItem(port_spec, False, False, False) self.addTopLevelItem(item) self.port_spec_items[port_spec.name] = (port_spec, item) subitem = self.entry_klass(port_spec, function) self.function_map[function.real_id] = subitem item.addChild(subitem) subitem.setFirstColumnSpanned(True) self.setItemWidget(subitem, 2, subitem.get_widget()) item.setExpanded(True)
def execute(modules, connections=[], add_port_specs=[], enable_pkg=True, full_results=False): """Build a pipeline and execute it. This is useful to simply build a pipeline in a test case, and run it. When doing that, intercept_result() can be used to check the results of each module. modules is a list of module tuples describing the modules to be created, with the following format: [('ModuleName', 'package.identifier', [ # Functions ('port_name', [ # Function parameters ('Signature', 'value-as-string'), ]), ])] connections is a list of tuples describing the connections to make, with the following format: [ (source_module_index, 'source_port_name', dest_module_index, 'dest_module_name'), ] add_port_specs is a list of specs to add to modules, with the following format: [ (mod_id, 'input'/'output', 'portname', '(port_sig)'), ] It is useful to test modules that can have custom ports through a configuration widget. The function returns the 'errors' dict it gets from the interpreter, so you should use a construct like self.assertFalse(execute(...)) if the execution is not supposed to fail. For example, this creates (and runs) an Integer module with its value set to 44, connected to a PythonCalc module, connected to a StandardOutput: self.assertFalse(execute([ ('Float', 'org.vistrails.vistrails.basic', [ ('value', [('Float', '44.0')]), ]), ('PythonCalc', 'org.vistrails.vistrails.pythoncalc', [ ('value2', [('Float', '2.0')]), ('op', [('String', '-')]), ]), ('StandardOutput', 'org.vistrails.vistrails.basic', []), ], [ (0, 'value', 1, 'value1'), (1, 'value', 2, 'value'), ])) """ from vistrails.core.db.locator import XMLFileLocator from vistrails.core.modules.module_registry import MissingPackage from vistrails.core.packagemanager import get_package_manager from vistrails.core.utils import DummyView from vistrails.core.vistrail.connection import Connection from vistrails.core.vistrail.module import Module from vistrails.core.vistrail.module_function import ModuleFunction from vistrails.core.vistrail.module_param import ModuleParam from vistrails.core.vistrail.pipeline import Pipeline from vistrails.core.vistrail.port import Port from vistrails.core.vistrail.port_spec import PortSpec from vistrails.core.interpreter.noncached import Interpreter pm = get_package_manager() port_spec_per_module = {} # mod_id -> [portspec: PortSpec] j = 0 for i, (mod_id, inout, name, sig) in enumerate(add_port_specs): mod_specs = port_spec_per_module.setdefault(mod_id, []) ps = PortSpec(id=i, name=name, type=inout, sigstring=sig, sort_key=-1) for psi in ps.port_spec_items: psi.id = j j += 1 mod_specs.append(ps) pipeline = Pipeline() module_list = [] for i, (name, identifier, functions) in enumerate(modules): function_list = [] try: pkg = pm.get_package(identifier) except MissingPackage: if not enable_pkg: raise dep_graph = pm.build_dependency_graph([identifier]) for pkg_id in pm.get_ordered_dependencies(dep_graph): pkg = pm.identifier_is_available(pkg_id) if pkg is None: raise pm.late_enable_package(pkg.codepath) pkg = pm.get_package(identifier) for func_name, params in functions: param_list = [] for j, (param_type, param_val) in enumerate(params): param_list.append( ModuleParam(pos=j, type=param_type, val=param_val)) function_list.append( ModuleFunction(name=func_name, parameters=param_list)) name = name.rsplit('|', 1) if len(name) == 2: namespace, name = name else: namespace = None name, = name module = Module(name=name, namespace=namespace, package=identifier, version=pkg.version, id=i, functions=function_list) for port_spec in port_spec_per_module.get(i, []): module.add_port_spec(port_spec) pipeline.add_module(module) module_list.append(module) for i, (sid, sport, did, dport) in enumerate(connections): s_sig = module_list[sid].get_port_spec(sport, 'output').sigstring d_sig = module_list[did].get_port_spec(dport, 'input').sigstring pipeline.add_connection( Connection(id=i, ports=[ Port(id=i * 2, type='source', moduleId=sid, name=sport, signature=s_sig), Port(id=i * 2 + 1, type='destination', moduleId=did, name=dport, signature=d_sig), ])) interpreter = Interpreter.get() result = interpreter.execute(pipeline, locator=XMLFileLocator('foo.xml'), current_version=1, view=DummyView()) if full_results: return result else: # Allows to do self.assertFalse(execute(...)) return result.errors
def _update_func(self, port_spec, *args, **kwargs): # print 'running _update_func', port_spec.name # print args vt_api = get_api() if port_spec.type != 'input': if self._module.has_port_spec(port_spec.name, 'input'): port_spec = \ self._module.get_port_spec(port_spec.name, 'input') else: raise TypeError("cannot update an output port spec") # FIXME deal with kwargs num_ports = 0 num_params = 0 for value in args: # print 'processing', type(value), value if isinstance(value, Port): # make connection to specified output port # print 'updating port' num_ports += 1 elif isinstance(value, Module): # make connection to 'self' output port of value # print 'updating module' num_ports += 1 else: # print 'update literal', type(value), value num_params += 1 if num_ports > 1 or (num_ports == 1 and num_params > 0): reg = get_module_registry() tuple_desc = reg.get_descriptor_by_name(basic_pkg, 'Tuple') tuple_module = vt_api.add_module_from_descriptor(tuple_desc) output_port_spec = PortSpec(id=-1, name='value', type='output', sigstring=port_spec.sigstring) vt_api.add_port_spec(tuple_module, output_port_spec) self._update_func(port_spec, tuple_module.value()) assert len(port_spec.descriptors()) == len(args) for i, descriptor in enumerate(port_spec.descriptors()): arg_name = 'arg%d' % i sigstring = create_port_spec_string([descriptor.spec_tuple]) tuple_port_spec = PortSpec(id=-1, name=arg_name, type='input', sigstring=sigstring) vt_api.add_port_spec(tuple_module, tuple_port_spec) tuple_module._process_attr_value(arg_name, args[i]) elif num_ports == 1: other = args[0] if isinstance(other, Port): if other._port_spec.type != 'output': other_module = other._vistrails_module._module if other_module.has_port_spec(port_spec.name, 'output'): other_port_spec = \ other_module.get_port_spec(port_spec.name, 'output') else: raise TypeError("cannot update an input " "port spec") else: other_port_spec = other._port_spec vt_api.add_connection(other._vistrails_module, other_port_spec, self, port_spec) elif isinstance(other, Module): other_port_spec = \ other._module.get_port_spec('self', 'output') vt_api.add_connection(other, other_port_spec, self, port_spec) else: vt_api.change_parameter(self, port_spec.name, [str(x) for x in args])