def test_constructor(self): m1_param = ModuleParam(val="1.2", type="Float", alias="", ) m1_function = ModuleFunction(name="value", parameters=[m1_param], ) m1 = Module(id=0, name='Float', functions=[m1_function], ) m2 = Module() m2.name = "Float" m2.id = 0 f = ModuleFunction() f.name = "value" m2.functions.append(f) param = ModuleParam() param.strValue = "1.2" param.type = "Float" param.alias = "" f.params.append(param) assert m1 == m2
def test_constructor(self): m1_param = ModuleParam( val="1.2", type="Float", alias="", ) m1_function = ModuleFunction( name="value", parameters=[m1_param], ) m1 = Module( id=0, name='Float', functions=[m1_function], ) m2 = Module() m2.name = "Float" m2.id = 0 f = ModuleFunction() f.name = "value" m2.functions.append(f) param = ModuleParam() param.strValue = "1.2" param.type = "Float" param.alias = "" f.params.append(param) assert m1 == m2
def convert(_abstraction): if _abstraction.__class__ == Abstraction: return _abstraction.__class__ = Abstraction if _abstraction.db_location: Location.convert(_abstraction.db_location) for _function in _abstraction.db_functions: ModuleFunction.convert(_function) for _annotation in _abstraction.db_get_annotations(): Annotation.convert(_annotation) _abstraction.set_defaults()
def convert(_abstraction): if _abstraction.__class__ == Abstraction: return _abstraction.__class__ = Abstraction if _abstraction.db_location: Location.convert(_abstraction.db_location) for _function in _abstraction.db_functions: ModuleFunction.convert(_function) for _annotation in _abstraction.db_get_annotations(): Annotation.convert(_annotation) for _control_parameter in _abstraction.db_get_controlParameters(): ModuleControlParam.convert(_control_parameter) _abstraction.set_defaults()
def convert(_module): if _module.__class__ == Module: return _module.__class__ = Module for _port_spec in _module.db_portSpecs: PortSpec.convert(_port_spec) if _module.db_location: Location.convert(_module.db_location) for _function in _module.db_functions: ModuleFunction.convert(_function) for _annotation in _module.db_get_annotations(): Annotation.convert(_annotation) _module.set_defaults()
def convert(_group): if _group.__class__ == Group: return _group.__class__ = Group if _group.db_location: Location.convert(_group.db_location) if _group.db_workflow: from vistrails.core.vistrail.pipeline import Pipeline Pipeline.convert(_group.db_workflow) for _function in _group.db_functions: ModuleFunction.convert(_function) for _annotation in _group.db_get_annotations(): Annotation.convert(_annotation) _group.set_defaults()
def convert(_module): if _module.__class__ == Module: return _module.__class__ = Module for _port_spec in _module.db_portSpecs: PortSpec.convert(_port_spec) if _module.db_location: Location.convert(_module.db_location) for _function in _module.db_functions: ModuleFunction.convert(_function) for _annotation in _module.db_get_annotations(): Annotation.convert(_annotation) for _control_parameter in _module.db_get_controlParameters(): ModuleControlParam.convert(_control_parameter) _module.set_defaults()
def create_module(self, id_scope=None): from vistrails.core.modules.basic_modules import identifier as basic_pkg from vistrails.db.domain import IdScope if id_scope is None: id_scope = IdScope() params = [ ModuleParam(id=id_scope.getNewId(ModuleParam.vtType), type='Int', val='1') ] functions = [ ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType), name='value', parameters=params) ] control_parameters = [ ModuleControlParam(id=id_scope.getNewId(ModuleControlParam.vtType), name='combiner', value='pairwise') ] module = Module(id=id_scope.getNewId(Module.vtType), name='Float', package=basic_pkg, functions=functions, controlParameters=control_parameters) return module
def create_action(self, id_scope=None): from vistrails.core.modules.basic_modules import identifier as basic_pkg from vistrails.core.vistrail.action import Action from vistrails.core.vistrail.module import Module from vistrails.core.vistrail.module_function import ModuleFunction from vistrails.core.vistrail.module_param import ModuleParam from vistrails.db.domain import IdScope if id_scope is None: id_scope = IdScope() param = ModuleParam(id=id_scope.getNewId(ModuleParam.vtType), type='Integer', val='1') function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType), name='value', parameters=[param]) m = Module(id=id_scope.getNewId(Module.vtType), name='Float', package=basic_pkg, functions=[function]) add_op = AddOp(id=id_scope.getNewId('operation'), what='module', objectId=m.id, data=m) action = Action(id=id_scope.getNewId(Action.vtType), prevId=0, date=datetime(2007, 11, 18), operations=[add_op]) return action
def convert(_group): if _group.__class__ == Group: return _group.__class__ = Group if _group.db_location: Location.convert(_group.db_location) if _group.db_workflow: from vistrails.core.vistrail.pipeline import Pipeline Pipeline.convert(_group.db_workflow) for _function in _group.db_functions: ModuleFunction.convert(_function) for _annotation in _group.db_get_annotations(): Annotation.convert(_annotation) for _control_parameter in _group.db_get_controlParameters(): ModuleControlParam.convert(_control_parameter) _group.set_defaults()
def create_ops(self, id_scope=IdScope()): from vistrails.core.modules.basic_modules import identifier as basic_pkg from vistrails.core.vistrail.module import Module from vistrails.core.vistrail.module_function import ModuleFunction from vistrails.core.vistrail.module_param import ModuleParam from vistrails.core.vistrail.annotation import Annotation if id_scope is None: id_scope = IdScope( remap={ AddOp.vtType: 'operation', ChangeOp.vtType: 'operation', DeleteOp.vtType: 'operation' }) m = Module(id=id_scope.getNewId(Module.vtType), name='Float', package=basic_pkg) add_op = AddOp(id=id_scope.getNewId(AddOp.vtType), what=Module.vtType, objectId=m.id, data=m) function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType), name='value') change_op = ChangeOp(id=id_scope.getNewId(ChangeOp.vtType), what=ModuleFunction.vtType, oldObjId=2, newObjId=function.real_id, parentObjId=m.id, parentObjType=Module.vtType, data=function) param = ModuleParam(id=id_scope.getNewId(ModuleParam.vtType), type='Float', val='1.0') delete_op = DeleteOp(id=id_scope.getNewId(DeleteOp.vtType), what=ModuleParam.vtType, objectId=param.real_id, parentObjId=function.real_id, parentObjType=ModuleFunction.vtType) annotation = Annotation(id=id_scope.getNewId(Annotation.vtType), key='foo', value='bar') add_annotation = AddOp(id=id_scope.getNewId(AddOp.vtType), what=Annotation.vtType, objectId=m.id, data=annotation) cparam = ModuleControlParam(id=id_scope.getNewId( ModuleControlParam.vtType), name='foo', value='bar') add_cparam = AddOp(id=id_scope.getNewId(AddOp.vtType), what=ModuleControlParam.vtType, objectId=m.id, data=cparam) return [add_op, change_op, delete_op, add_annotation]
def perform(self, pipeline, step): """ perform(pipeline: VisPipeline, step: int) -> None This will takes a pipeline and apply the interpolated values at step 'step' to the pipeline. Then return the updated pipeline """ m = pipeline.modules[self.module.id] f = ModuleFunction() f.name = self.function f.returnType = 'void' value = self.values[step] for v in value: p = ModuleParam() convert = {'int':'Integer', 'str':'String', 'float':'Float', 'double':'Float'} p.type = convert[type(v).__name__] p.strValue = str(v) f.params.append(p) m.functions.append(f)
def test_tuple(self): from vistrails.core.vistrail.module_param import ModuleParam from vistrails.core.vistrail.module_function import ModuleFunction from vistrails.core.utils import DummyView from vistrails.core.vistrail.module import Module import vistrails.db.domain id_scope = vistrails.db.domain.IdScope() interpreter = vistrails.core.interpreter.default.get_default_interpreter( ) v = DummyView() p = vistrails.core.vistrail.pipeline.Pipeline() params = [ ModuleParam( id=id_scope.getNewId(ModuleParam.vtType), pos=0, type='Float', val='2.0', ), ModuleParam( id=id_scope.getNewId(ModuleParam.vtType), pos=1, type='Float', val='2.0', ) ] function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType), name='input') function.add_parameters(params) module = Module(id=id_scope.getNewId(Module.vtType), name='TestTupleExecution', package='org.vistrails.vistrails.console_mode_test', version='0.9.1') module.add_function(function) p.add_module(module) interpreter.execute(p, locator=XMLFileLocator('foo'), current_version=1L, view=v)
def test_str(self): m = Module( id=0, name='Float', functions=[ ModuleFunction( name='value', parameters=[ModuleParam( type='Int', val='1', )], ) ], ) str(m)
def test_tuple(self): from vistrails.core.vistrail.module_param import ModuleParam from vistrails.core.vistrail.module_function import ModuleFunction from vistrails.core.utils import DummyView from vistrails.core.vistrail.module import Module import vistrails.db.domain id_scope = vistrails.db.domain.IdScope() interpreter = vistrails.core.interpreter.default.get_default_interpreter() v = DummyView() p = vistrails.core.vistrail.pipeline.Pipeline() params = [ModuleParam(id=id_scope.getNewId(ModuleParam.vtType), pos=0, type='Float', val='2.0', ), ModuleParam(id=id_scope.getNewId(ModuleParam.vtType), pos=1, type='Float', val='2.0', )] function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType), name='input') function.add_parameters(params) module = Module(id=id_scope.getNewId(Module.vtType), name='TestTupleExecution', package='org.vistrails.vistrails.console_mode_test', version='0.9.1') module.add_function(function) p.add_module(module) interpreter.execute(p, locator=XMLFileLocator('foo'), current_version=1L, view=v)
def perform(self, pipeline, step): """ perform(pipeline: VisPipeline, step: int) -> None This will takes a pipeline and apply the interpolated values at step 'step' to the pipeline. Then return the updated pipeline """ m = pipeline.modules[self.module.id] f = ModuleFunction() f.name = self.function f.returnType = 'void' value = self.values[step] for v in value: p = ModuleParam() convert = { 'int': 'Integer', 'str': 'String', 'float': 'Float', 'double': 'Float' } p.type = convert[type(v).__name__] p.strValue = str(v) f.params.append(p) m.functions.append(f)
def create_group(self, id_scope=IdScope()): from vistrails.core.vistrail.location import Location from vistrails.core.vistrail.module_function import ModuleFunction from vistrails.core.vistrail.module_param import ModuleParam params = [ModuleParam(id=id_scope.getNewId(ModuleParam.vtType), type='Int', val='1')] functions = [ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType), name='value', parameters=params)] location = Location(id=id_scope.getNewId(Location.vtType), x=12.342, y=-19.432) module = \ Group(id=id_scope.getNewId(Group.vtType), location=location, functions=functions, ) return module
def create_vistrail(self): vistrail = Vistrail() m = Module(id=vistrail.idScope.getNewId(Module.vtType), name='Float', package=get_vistrails_basic_pkg_id()) add_op = AddOp(id=vistrail.idScope.getNewId(AddOp.vtType), what=Module.vtType, objectId=m.id, data=m) function_id = vistrail.idScope.getNewId(ModuleFunction.vtType) function = ModuleFunction(id=function_id, name='value') change_op = ChangeOp(id=vistrail.idScope.getNewId(ChangeOp.vtType), what=ModuleFunction.vtType, oldObjId=2, newObjId=function.real_id, parentObjId=m.id, parentObjType=Module.vtType, data=function) param = ModuleParam(id=vistrail.idScope.getNewId(ModuleParam.vtType), type='Integer', val='1') delete_op = DeleteOp(id=vistrail.idScope.getNewId(DeleteOp.vtType), what=ModuleParam.vtType, objectId=param.real_id, parentObjId=function.real_id, parentObjType=ModuleFunction.vtType) action1 = Action(id=vistrail.idScope.getNewId(Action.vtType), operations=[add_op]) action2 = Action(id=vistrail.idScope.getNewId(Action.vtType), operations=[change_op, delete_op]) vistrail.add_action(action1, 0) vistrail.add_action(action2, action1.id) vistrail.addTag('first action', action1.id) vistrail.addTag('second action', action2.id) return vistrail
def execute(self, *args, **kwargs): """Execute the pipeline. Positional arguments are either input values (created from ``module == value``, where `module` is a Module from the pipeline and `value` is some value or Function instance) for the pipeline's InputPorts, or Module instances (to select sink modules). Keyword arguments are also used to set InputPort by looking up inputs by name. Example:: input_bound = pipeline.get_input('higher_bound') input_url = pipeline.get_input('url') sinkmodule = pipeline.get_module(32) pipeline.execute(sinkmodule, input_bound == vt.Function(Integer, 10), input_url == 'http://www.vistrails.org/', resolution=15) # kwarg: only one equal sign """ pipeline = self.pipeline sinks = set() inputs = {} reg = get_module_registry() InputPort_desc = reg.get_descriptor_by_name( get_vistrails_basic_pkg_id(), 'InputPort') # Read args for arg in args: if isinstance(arg, ModuleValuePair): if arg.module.id in inputs: raise ValueError("Multiple values set for InputPort %r" % get_inputoutput_name(arg.module)) if not reg.is_descriptor_subclass(arg.module.module_descriptor, InputPort_desc): raise ValueError("Module %d is not an InputPort" % arg.module.id) inputs[arg.module.id] = arg.value elif isinstance(arg, Module): sinks.add(arg.module_id) # Read kwargs for key, value in kwargs.iteritems(): name = key key = self.get_python_parameter(key) # Might raise KeyError if name in inputs: raise ValueError("Multiple values set for input %r" % name) inputs[name] = [key.module_id, value] reason = "API pipeline execution" sinks = sinks or None # Use controller only if no inputs were passed in if (not inputs and self.vistrail is not None and self.vistrail.current_version == self.version): controller = self.vistrail.controller results, changed = controller.execute_workflow_list([[ controller.locator, # locator self.version, # version self.pipeline, # pipeline DummyView(), # view None, # custom_aliases None, # custom_params reason, # reason sinks, # sinks None, # extra_info ]]) result, = results else: # pipeline = self.pipeline if inputs: # id_scope = IdScope(1) # pipeline = pipeline.do_copy(False, id_scope) # A hach to get ids from id_scope that we know won't collide: # make them negative # id_scope.getNewId = lambda t, g=id_scope.getNewId: -g(t) # create_module = \ # VistrailController.create_module_from_descriptor_static # create_function = VistrailController.create_function_static # create_connection = VistrailController.create_connection_static # Fills in the ExternalPipe ports for name, input_list in inputs.iteritems(): module_id, values = input_list module = pipeline.modules[module_id] if not isinstance(values, (list, tuple)): values = [values] ''' # Guess the type of the InputPort _, sigstrings, _, _, _ = get_port_spec_info(pipeline, module) sigstrings = parse_port_spec_string(sigstrings) # Convert whatever we got to a list of strings, for the # pipeline values = [reg.convert_port_val(val, sigstring, None) for val, sigstring in izip(values, sigstrings)] if len(values) == 1: # Create the constant module constant_desc = reg.get_descriptor_by_name( *sigstrings[0]) #print('Setting desription: ',str(constant_desc),str(sigstrings[0])) constant_mod = create_module(id_scope, constant_desc) func = create_function(id_scope, constant_mod, 'value', values) constant_mod.add_function(func) pipeline.add_module(constant_mod) # Connect it to the ExternalPipe port conn = create_connection(id_scope, constant_mod, 'value', module, 'ExternalPipe') pipeline.db_add_connection(conn) else: raise RuntimeError("TODO : create tuple") ''' port_spec = reg.get_input_port_spec(module, name) added_functions = {} tmp_f_id = -1L tmp_p_id = -1L function = [ f for f in module.functions if f.name == port_spec.name ] if function: function = function[0] else: try: function = added_functions[(module.id, port_spec.name)] except KeyError: # add to function list params = [] for psi in port_spec.port_spec_items: parameter = ModuleParam( id=tmp_p_id, pos=psi.pos, name='<no description>', val=psi.default, type=psi.descriptor.sigstring) params.append(parameter) tmp_p_id -= 1 function = ModuleFunction( id=tmp_f_id, pos=module.getNumFunctions(), name=port_spec.name, parameters=params) tmp_f_id -= 1 added_functions[(module.id, port_spec.name)] = function action = vistrails.core.db.action.create_action([ ('add', function, module.vtType, module.id) ]) # function_actions.append(action) parameter = function.params[0] # find old parameter old_param = parameter actions = [] for v in values: desc = reg.get_descriptor_by_name( 'org.vistrails.vistrails.basic', 'String', None) if not isinstance(v, str): str_value = desc.module.translate_to_string(v) else: str_value = v new_param = ModuleParam(id=tmp_p_id, pos=old_param.pos, name=old_param.name, alias=old_param.alias, val=str_value, type=old_param.type) tmp_p_id -= 1 action_spec = ('change', old_param, new_param, function.vtType, function.real_id) action = vistrails.core.db.action.create_action( [action_spec]) actions.append(action) # controller = self.vistrail.controller self.controller.perform_action(action) ######################################################################### interpreter = get_default_interpreter() result = interpreter.execute(pipeline, locator=self.locator, reason=reason, sinks=sinks, actions=actions) if result.errors: raise ExecutionErrors(self, result) else: return ExecutionResults(self, result)
def collectParameterActions(self, pipeline): """ collectParameterActions() -> list Return a list of action lists corresponding to each dimension """ if not pipeline: return unescape_dict = { "'":"'", '"':'"', '
':'\n' } from vistrails.core.modules.module_registry import get_module_registry reg = get_module_registry() parameterValues = [[], [], [], []] # a list of added functions [(module_id, function_name)] = function added_functions = {} vistrail_vars = [] function_actions = [] tmp_f_id = -1L tmp_p_id = -1L for i in xrange(len(self.functions)): pe_function = self.functions[i] module = pipeline.db_get_object(Module.vtType, pe_function.module_id) # collect overridden vistrail vars if module.is_vistrail_var(): vistrail_vars.append(module.get_vistrail_var()) port_spec = reg.get_input_port_spec(module, pe_function.port_name) for param in pe_function.parameters: port_spec_item = port_spec.port_spec_items[param.pos] dim = param.dimension if dim not in [0, 1, 2, 3]: continue count = self.dims[dim] # find interpolator values values = [] text = '%s' % unescape(param.value, unescape_dict) if param.interpolator == 'Linear Interpolation': # need to figure out type if port_spec_item.module == "Integer": i_range = literal_eval(text) p_min = int(i_range[0]) p_max =int(i_range[1]) values = IntegerLinearInterpolator(p_min, p_max, count).get_values() if port_spec_item.module == "Float": i_range = literal_eval(text) p_min = float(i_range[0]) p_max =float(i_range[1]) values = FloatLinearInterpolator(p_min, p_max, count).get_values() elif param.interpolator == 'RGB Interpolation': i_range = literal_eval(text) p_min = str(i_range[0]) p_max =str(i_range[1]) values = RGBColorInterpolator(p_min, p_max, count).get_values() elif param.interpolator == 'HSV Interpolation': i_range = literal_eval(text) p_min = str(i_range[0]) p_max =str(i_range[1]) values = HSVColorInterpolator(p_min, p_max, count).get_values() elif param.interpolator == 'List': p_module = port_spec_item.descriptor.module values = [p_module.translate_to_python(m) for m in literal_eval(text)] elif param.interpolator == 'User-defined Function': p_module = port_spec_item.descriptor.module values = UserDefinedFunctionInterpolator(p_module, text, count).get_values() if not values: return None # find parameter or create one function = [f for f in module.functions if f.name == port_spec.name] if function: function = function[0] else: try: function = added_functions[(module.id,port_spec.name)] except KeyError: # add to function list params = [] for psi in port_spec.port_spec_items: parameter = ModuleParam(id=tmp_p_id, pos=psi.pos, name='<no description>', val=psi.default, type=psi.descriptor.sigstring) params.append(parameter) tmp_p_id -= 1 function = ModuleFunction(id=tmp_f_id, pos=module.getNumFunctions(), name=port_spec.name, parameters=params) tmp_f_id -= 1 added_functions[(module.id, port_spec.name)]=function action = vistrails.core.db.action.create_action([('add', function, module.vtType, module.id)]) function_actions.append(action) parameter = function.params[port_spec_item.pos] # find old parameter old_param = parameter actions = [] for v in values: desc = port_spec_item.descriptor if not isinstance(v, str): str_value = desc.module.translate_to_string(v) else: str_value = v new_param = ModuleParam(id=tmp_p_id, pos=old_param.pos, name=old_param.name, alias=old_param.alias, val=str_value, type=old_param.type) tmp_p_id -= 1 action_spec = ('change', old_param, new_param, function.vtType, function.real_id) action = vistrails.core.db.action.create_action([action_spec]) actions.append(action) parameterValues[dim].append(actions) return [zip(*p) for p in parameterValues], function_actions, vistrail_vars
def updateFunctionPort(self): """ Function to be used inside the updateUsptream method of the Map module. It updates the module connected to the FunctionPort port, executing it in parallel. """ nameInput = self.getInputFromPort('InputPort') nameOutput = self.getInputFromPort('OutputPort') rawInputList = self.getInputFromPort('InputList') # Create inputList to always have iterable elements # to simplify code if len(nameInput) == 1: element_is_iter = False inputList = [[element] for element in rawInputList] else: element_is_iter = True inputList = rawInputList workflows = [] module = None vtType = None # iterating through the connectors for connector in self.inputPorts.get('FunctionPort'): module = connector.obj # pipeline original_pipeline = connector.obj.moduleInfo['pipeline'] # module module_id = connector.obj.moduleInfo['moduleId'] vtType = original_pipeline.modules[module_id].vtType # serialize the module for each value in the list for i, element in enumerate(inputList): if element_is_iter: self.element = element else: self.element = element[0] # checking type and setting input in the module self.typeChecking(connector.obj, nameInput, inputList) self.setInputValues(connector.obj, nameInput, element) pipeline_db_module = original_pipeline.modules[module_id].do_copy() # transforming a subworkflow in a group # TODO: should we also transform inner subworkflows? if pipeline_db_module.is_abstraction(): group = Group(id=pipeline_db_module.id, cache=pipeline_db_module.cache, location=pipeline_db_module.location, functions=pipeline_db_module.functions, annotations=pipeline_db_module.annotations) source_port_specs = pipeline_db_module.sourcePorts() dest_port_specs = pipeline_db_module.destinationPorts() for source_port_spec in source_port_specs: group.add_port_spec(source_port_spec) for dest_port_spec in dest_port_specs: group.add_port_spec(dest_port_spec) group.pipeline = pipeline_db_module.pipeline pipeline_db_module = group # getting highest id between functions to guarantee unique ids # TODO: can get current IdScope here? if pipeline_db_module.functions: high_id = max(function.db_id for function in pipeline_db_module.functions) else: high_id = 0 # adding function and parameter to module in pipeline # TODO: 'pos' should not be always 0 here id_scope = IdScope(beginId=long(high_id+1)) for elementValue, inputPort in izip(element, nameInput): p_spec = pipeline_db_module.get_port_spec(inputPort, 'input') descrs = p_spec.descriptors() if len(descrs) != 1: raise ModuleError( self, "Tuple input ports are not supported") if not issubclass(descrs[0].module, Constant): raise ModuleError( self, "Module inputs should be Constant types") type = p_spec.sigstring[1:-1] mod_function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType), pos=0, name=inputPort) mod_param = ModuleParam(id=0L, pos=0, type=type, val=elementValue) mod_function.add_parameter(mod_param) pipeline_db_module.add_function(mod_function) # serializing module wf = self.serialize_module(pipeline_db_module) workflows.append(wf) # getting first connector, ignoring the rest break # IPython stuff try: rc = get_client() except Exception, error: raise ModuleError(self, "Exception while loading IPython: " "%s" % error)
def updateFunctionPort(self): """ Function to be used inside the updateUsptream method of the Map module. It updates the module connected to the FunctionPort port, executing it in parallel. """ nameInput = self.get_input('InputPort') nameOutput = self.get_input('OutputPort') rawInputList = self.get_input('InputList') # Create inputList to always have iterable elements # to simplify code if len(nameInput) == 1: element_is_iter = False inputList = [[element] for element in rawInputList] else: element_is_iter = True inputList = rawInputList workflows = [] module = None vtType = None # iterating through the connectors for connector in self.inputPorts.get('FunctionPort'): module = connector.obj # pipeline original_pipeline = connector.obj.moduleInfo['pipeline'] # module module_id = connector.obj.moduleInfo['moduleId'] vtType = original_pipeline.modules[module_id].vtType # serialize the module for each value in the list for i, element in enumerate(inputList): if element_is_iter: self.element = element else: self.element = element[0] # checking type and setting input in the module self.typeChecking(connector.obj, nameInput, inputList) self.setInputValues(connector.obj, nameInput, element, i) pipeline_db_module = original_pipeline.modules[ module_id].do_copy() # transforming a subworkflow in a group # TODO: should we also transform inner subworkflows? if pipeline_db_module.is_abstraction(): group = Group(id=pipeline_db_module.id, cache=pipeline_db_module.cache, location=pipeline_db_module.location, functions=pipeline_db_module.functions, annotations=pipeline_db_module.annotations) source_port_specs = pipeline_db_module.sourcePorts() dest_port_specs = pipeline_db_module.destinationPorts() for source_port_spec in source_port_specs: group.add_port_spec(source_port_spec) for dest_port_spec in dest_port_specs: group.add_port_spec(dest_port_spec) group.pipeline = pipeline_db_module.pipeline pipeline_db_module = group # getting highest id between functions to guarantee unique ids # TODO: can get current IdScope here? if pipeline_db_module.functions: high_id = max(function.db_id for function in pipeline_db_module.functions) else: high_id = 0 # adding function and parameter to module in pipeline # TODO: 'pos' should not be always 0 here id_scope = IdScope(beginId=long(high_id + 1)) for elementValue, inputPort in izip(element, nameInput): p_spec = pipeline_db_module.get_port_spec( inputPort, 'input') descrs = p_spec.descriptors() if len(descrs) != 1: raise ModuleError( self, "Tuple input ports are not supported") if not issubclass(descrs[0].module, Constant): raise ModuleError( self, "Module inputs should be Constant types") type = p_spec.sigstring[1:-1] mod_function = ModuleFunction(id=id_scope.getNewId( ModuleFunction.vtType), pos=0, name=inputPort) mod_param = ModuleParam(id=0L, pos=0, type=type, val=elementValue) mod_function.add_parameter(mod_param) pipeline_db_module.add_function(mod_function) # serializing module wf = self.serialize_module(pipeline_db_module) workflows.append(wf) # getting first connector, ignoring the rest break # IPython stuff try: rc = get_client() except Exception, error: raise ModuleError( self, "Exception while loading IPython: %s" % debug.format_exception(error))
def execute(modules, connections=[], add_port_specs=[], enable_pkg=True, full_results=False): """Build a pipeline and execute it. This is useful to simply build a pipeline in a test case, and run it. When doing that, intercept_result() can be used to check the results of each module. modules is a list of module tuples describing the modules to be created, with the following format: [('ModuleName', 'package.identifier', [ # Functions ('port_name', [ # Function parameters ('Signature', 'value-as-string'), ]), ])] connections is a list of tuples describing the connections to make, with the following format: [ (source_module_index, 'source_port_name', dest_module_index, 'dest_module_name'), ] add_port_specs is a list of specs to add to modules, with the following format: [ (mod_id, 'input'/'output', 'portname', '(port_sig)'), ] It is useful to test modules that can have custom ports through a configuration widget. The function returns the 'errors' dict it gets from the interpreter, so you should use a construct like self.assertFalse(execute(...)) if the execution is not supposed to fail. For example, this creates (and runs) an Integer module with its value set to 44, connected to a PythonCalc module, connected to a StandardOutput: self.assertFalse(execute([ ('Float', 'org.vistrails.vistrails.basic', [ ('value', [('Float', '44.0')]), ]), ('PythonCalc', 'org.vistrails.vistrails.pythoncalc', [ ('value2', [('Float', '2.0')]), ('op', [('String', '-')]), ]), ('StandardOutput', 'org.vistrails.vistrails.basic', []), ], [ (0, 'value', 1, 'value1'), (1, 'value', 2, 'value'), ])) """ from vistrails.core.db.locator import XMLFileLocator from vistrails.core.modules.module_registry import MissingPackage from vistrails.core.packagemanager import get_package_manager from vistrails.core.utils import DummyView from vistrails.core.vistrail.connection import Connection from vistrails.core.vistrail.module import Module from vistrails.core.vistrail.module_function import ModuleFunction from vistrails.core.vistrail.module_param import ModuleParam from vistrails.core.vistrail.pipeline import Pipeline from vistrails.core.vistrail.port import Port from vistrails.core.vistrail.port_spec import PortSpec from vistrails.core.interpreter.noncached import Interpreter pm = get_package_manager() port_spec_per_module = {} # mod_id -> [portspec: PortSpec] j = 0 for i, (mod_id, inout, name, sig) in enumerate(add_port_specs): mod_specs = port_spec_per_module.setdefault(mod_id, []) ps = PortSpec(id=i, name=name, type=inout, sigstring=sig, sort_key=-1) for psi in ps.port_spec_items: psi.id = j j += 1 mod_specs.append(ps) pipeline = Pipeline() module_list = [] for i, (name, identifier, functions) in enumerate(modules): function_list = [] try: pkg = pm.get_package(identifier) except MissingPackage: if not enable_pkg: raise dep_graph = pm.build_dependency_graph([identifier]) for pkg_id in pm.get_ordered_dependencies(dep_graph): pkg = pm.identifier_is_available(pkg_id) if pkg is None: raise pm.late_enable_package(pkg.codepath) pkg = pm.get_package(identifier) for func_name, params in functions: param_list = [] for j, (param_type, param_val) in enumerate(params): param_list.append( ModuleParam(pos=j, type=param_type, val=param_val)) function_list.append( ModuleFunction(name=func_name, parameters=param_list)) name = name.rsplit('|', 1) if len(name) == 2: namespace, name = name else: namespace = None name, = name module = Module(name=name, namespace=namespace, package=identifier, version=pkg.version, id=i, functions=function_list) for port_spec in port_spec_per_module.get(i, []): module.add_port_spec(port_spec) pipeline.add_module(module) module_list.append(module) for i, (sid, sport, did, dport) in enumerate(connections): s_sig = module_list[sid].get_port_spec(sport, 'output').sigstring d_sig = module_list[did].get_port_spec(dport, 'input').sigstring pipeline.add_connection( Connection(id=i, ports=[ Port(id=i * 2, type='source', moduleId=sid, name=sport, signature=s_sig), Port(id=i * 2 + 1, type='destination', moduleId=did, name=dport, signature=d_sig), ])) interpreter = Interpreter.get() result = interpreter.execute(pipeline, locator=XMLFileLocator('foo.xml'), current_version=1, view=DummyView()) if full_results: return result else: # Allows to do self.assertFalse(execute(...)) return result.errors