def test_constructor(self):
m1_param = ModuleParam(val="1.2",
type="Float",
alias="",
)
m1_function = ModuleFunction(name="value",
parameters=[m1_param],
)
m1 = Module(id=0,
name='Float',
functions=[m1_function],
)
m2 = Module()
m2.name = "Float"
m2.id = 0
f = ModuleFunction()
f.name = "value"
m2.functions.append(f)
param = ModuleParam()
param.strValue = "1.2"
param.type = "Float"
param.alias = ""
f.params.append(param)
assert m1 == m2
def test_constructor(self):
m1_param = ModuleParam(
val="1.2",
type="Float",
alias="",
)
m1_function = ModuleFunction(
name="value",
parameters=[m1_param],
)
m1 = Module(
id=0,
name='Float',
functions=[m1_function],
)
m2 = Module()
m2.name = "Float"
m2.id = 0
f = ModuleFunction()
f.name = "value"
m2.functions.append(f)
param = ModuleParam()
param.strValue = "1.2"
param.type = "Float"
param.alias = ""
f.params.append(param)
assert m1 == m2
def convert(_abstraction):
if _abstraction.__class__ == Abstraction:
return
_abstraction.__class__ = Abstraction
if _abstraction.db_location:
Location.convert(_abstraction.db_location)
for _function in _abstraction.db_functions:
ModuleFunction.convert(_function)
for _annotation in _abstraction.db_get_annotations():
Annotation.convert(_annotation)
_abstraction.set_defaults()
def convert(_abstraction):
if _abstraction.__class__ == Abstraction:
return
_abstraction.__class__ = Abstraction
if _abstraction.db_location:
Location.convert(_abstraction.db_location)
for _function in _abstraction.db_functions:
ModuleFunction.convert(_function)
for _annotation in _abstraction.db_get_annotations():
Annotation.convert(_annotation)
for _control_parameter in _abstraction.db_get_controlParameters():
ModuleControlParam.convert(_control_parameter)
_abstraction.set_defaults()
def convert(_abstraction):
if _abstraction.__class__ == Abstraction:
return
_abstraction.__class__ = Abstraction
if _abstraction.db_location:
Location.convert(_abstraction.db_location)
for _function in _abstraction.db_functions:
ModuleFunction.convert(_function)
for _annotation in _abstraction.db_get_annotations():
Annotation.convert(_annotation)
for _control_parameter in _abstraction.db_get_controlParameters():
ModuleControlParam.convert(_control_parameter)
_abstraction.set_defaults()
def convert(_module):
if _module.__class__ == Module:
return
_module.__class__ = Module
for _port_spec in _module.db_portSpecs:
PortSpec.convert(_port_spec)
if _module.db_location:
Location.convert(_module.db_location)
for _function in _module.db_functions:
ModuleFunction.convert(_function)
for _annotation in _module.db_get_annotations():
Annotation.convert(_annotation)
_module.set_defaults()
def convert(_group):
if _group.__class__ == Group:
return
_group.__class__ = Group
if _group.db_location:
Location.convert(_group.db_location)
if _group.db_workflow:
from vistrails.core.vistrail.pipeline import Pipeline
Pipeline.convert(_group.db_workflow)
for _function in _group.db_functions:
ModuleFunction.convert(_function)
for _annotation in _group.db_get_annotations():
Annotation.convert(_annotation)
_group.set_defaults()
def convert(_module):
if _module.__class__ == Module:
return
_module.__class__ = Module
for _port_spec in _module.db_portSpecs:
PortSpec.convert(_port_spec)
if _module.db_location:
Location.convert(_module.db_location)
for _function in _module.db_functions:
ModuleFunction.convert(_function)
for _annotation in _module.db_get_annotations():
Annotation.convert(_annotation)
for _control_parameter in _module.db_get_controlParameters():
ModuleControlParam.convert(_control_parameter)
_module.set_defaults()
def create_module(self, id_scope=None):
from vistrails.core.modules.basic_modules import identifier as basic_pkg
from vistrails.db.domain import IdScope
if id_scope is None:
id_scope = IdScope()
params = [
ModuleParam(id=id_scope.getNewId(ModuleParam.vtType),
type='Int',
val='1')
]
functions = [
ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType),
name='value',
parameters=params)
]
control_parameters = [
ModuleControlParam(id=id_scope.getNewId(ModuleControlParam.vtType),
name='combiner',
value='pairwise')
]
module = Module(id=id_scope.getNewId(Module.vtType),
name='Float',
package=basic_pkg,
functions=functions,
controlParameters=control_parameters)
return module
def create_action(self, id_scope=None):
from vistrails.core.modules.basic_modules import identifier as basic_pkg
from vistrails.core.vistrail.action import Action
from vistrails.core.vistrail.module import Module
from vistrails.core.vistrail.module_function import ModuleFunction
from vistrails.core.vistrail.module_param import ModuleParam
from vistrails.db.domain import IdScope
if id_scope is None:
id_scope = IdScope()
param = ModuleParam(id=id_scope.getNewId(ModuleParam.vtType),
type='Integer',
val='1')
function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType),
name='value',
parameters=[param])
m = Module(id=id_scope.getNewId(Module.vtType),
name='Float',
package=basic_pkg,
functions=[function])
add_op = AddOp(id=id_scope.getNewId('operation'),
what='module',
objectId=m.id,
data=m)
action = Action(id=id_scope.getNewId(Action.vtType),
prevId=0,
date=datetime(2007, 11, 18),
operations=[add_op])
return action
def convert(_group):
if _group.__class__ == Group:
return
_group.__class__ = Group
if _group.db_location:
Location.convert(_group.db_location)
if _group.db_workflow:
from vistrails.core.vistrail.pipeline import Pipeline
Pipeline.convert(_group.db_workflow)
for _function in _group.db_functions:
ModuleFunction.convert(_function)
for _annotation in _group.db_get_annotations():
Annotation.convert(_annotation)
for _control_parameter in _group.db_get_controlParameters():
ModuleControlParam.convert(_control_parameter)
_group.set_defaults()
def create_ops(self, id_scope=IdScope()):
from vistrails.core.modules.basic_modules import identifier as basic_pkg
from vistrails.core.vistrail.module import Module
from vistrails.core.vistrail.module_function import ModuleFunction
from vistrails.core.vistrail.module_param import ModuleParam
from vistrails.core.vistrail.annotation import Annotation
if id_scope is None:
id_scope = IdScope(
remap={
AddOp.vtType: 'operation',
ChangeOp.vtType: 'operation',
DeleteOp.vtType: 'operation'
})
m = Module(id=id_scope.getNewId(Module.vtType),
name='Float',
package=basic_pkg)
add_op = AddOp(id=id_scope.getNewId(AddOp.vtType),
what=Module.vtType,
objectId=m.id,
data=m)
function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType),
name='value')
change_op = ChangeOp(id=id_scope.getNewId(ChangeOp.vtType),
what=ModuleFunction.vtType,
oldObjId=2,
newObjId=function.real_id,
parentObjId=m.id,
parentObjType=Module.vtType,
data=function)
param = ModuleParam(id=id_scope.getNewId(ModuleParam.vtType),
type='Float',
val='1.0')
delete_op = DeleteOp(id=id_scope.getNewId(DeleteOp.vtType),
what=ModuleParam.vtType,
objectId=param.real_id,
parentObjId=function.real_id,
parentObjType=ModuleFunction.vtType)
annotation = Annotation(id=id_scope.getNewId(Annotation.vtType),
key='foo',
value='bar')
add_annotation = AddOp(id=id_scope.getNewId(AddOp.vtType),
what=Annotation.vtType,
objectId=m.id,
data=annotation)
cparam = ModuleControlParam(id=id_scope.getNewId(
ModuleControlParam.vtType),
name='foo',
value='bar')
add_cparam = AddOp(id=id_scope.getNewId(AddOp.vtType),
what=ModuleControlParam.vtType,
objectId=m.id,
data=cparam)
return [add_op, change_op, delete_op, add_annotation]
def perform(self, pipeline, step):
""" perform(pipeline: VisPipeline, step: int) -> None
This will takes a pipeline and apply the interpolated values
at step 'step' to the pipeline. Then return the updated
pipeline
"""
m = pipeline.modules[self.module.id]
f = ModuleFunction()
f.name = self.function
f.returnType = 'void'
value = self.values[step]
for v in value:
p = ModuleParam()
convert = {'int':'Integer', 'str':'String',
'float':'Float', 'double':'Float'}
p.type = convert[type(v).__name__]
p.strValue = str(v)
f.params.append(p)
m.functions.append(f)
def test_tuple(self):
from vistrails.core.vistrail.module_param import ModuleParam
from vistrails.core.vistrail.module_function import ModuleFunction
from vistrails.core.utils import DummyView
from vistrails.core.vistrail.module import Module
import vistrails.db.domain
id_scope = vistrails.db.domain.IdScope()
interpreter = vistrails.core.interpreter.default.get_default_interpreter(
)
v = DummyView()
p = vistrails.core.vistrail.pipeline.Pipeline()
params = [
ModuleParam(
id=id_scope.getNewId(ModuleParam.vtType),
pos=0,
type='Float',
val='2.0',
),
ModuleParam(
id=id_scope.getNewId(ModuleParam.vtType),
pos=1,
type='Float',
val='2.0',
)
]
function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType),
name='input')
function.add_parameters(params)
module = Module(id=id_scope.getNewId(Module.vtType),
name='TestTupleExecution',
package='org.vistrails.vistrails.console_mode_test',
version='0.9.1')
module.add_function(function)
p.add_module(module)
interpreter.execute(p,
locator=XMLFileLocator('foo'),
current_version=1L,
view=v)
def test_str(self):
m = Module(
id=0,
name='Float',
functions=[
ModuleFunction(
name='value',
parameters=[ModuleParam(
type='Int',
val='1',
)],
)
],
)
str(m)
def test_tuple(self):
from vistrails.core.vistrail.module_param import ModuleParam
from vistrails.core.vistrail.module_function import ModuleFunction
from vistrails.core.utils import DummyView
from vistrails.core.vistrail.module import Module
import vistrails.db.domain
id_scope = vistrails.db.domain.IdScope()
interpreter = vistrails.core.interpreter.default.get_default_interpreter()
v = DummyView()
p = vistrails.core.vistrail.pipeline.Pipeline()
params = [ModuleParam(id=id_scope.getNewId(ModuleParam.vtType),
pos=0,
type='Float',
val='2.0',
),
ModuleParam(id=id_scope.getNewId(ModuleParam.vtType),
pos=1,
type='Float',
val='2.0',
)]
function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType),
name='input')
function.add_parameters(params)
module = Module(id=id_scope.getNewId(Module.vtType),
name='TestTupleExecution',
package='org.vistrails.vistrails.console_mode_test',
version='0.9.1')
module.add_function(function)
p.add_module(module)
interpreter.execute(p,
locator=XMLFileLocator('foo'),
current_version=1L,
view=v)
def perform(self, pipeline, step):
""" perform(pipeline: VisPipeline, step: int) -> None
This will takes a pipeline and apply the interpolated values
at step 'step' to the pipeline. Then return the updated
pipeline
"""
m = pipeline.modules[self.module.id]
f = ModuleFunction()
f.name = self.function
f.returnType = 'void'
value = self.values[step]
for v in value:
p = ModuleParam()
convert = {
'int': 'Integer',
'str': 'String',
'float': 'Float',
'double': 'Float'
}
p.type = convert[type(v).__name__]
p.strValue = str(v)
f.params.append(p)
m.functions.append(f)
def create_group(self, id_scope=IdScope()):
from vistrails.core.vistrail.location import Location
from vistrails.core.vistrail.module_function import ModuleFunction
from vistrails.core.vistrail.module_param import ModuleParam
params = [ModuleParam(id=id_scope.getNewId(ModuleParam.vtType),
type='Int',
val='1')]
functions = [ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType),
name='value',
parameters=params)]
location = Location(id=id_scope.getNewId(Location.vtType),
x=12.342,
y=-19.432)
module = \
Group(id=id_scope.getNewId(Group.vtType),
location=location,
functions=functions,
)
return module
def create_vistrail(self):
vistrail = Vistrail()
m = Module(id=vistrail.idScope.getNewId(Module.vtType),
name='Float',
package=get_vistrails_basic_pkg_id())
add_op = AddOp(id=vistrail.idScope.getNewId(AddOp.vtType),
what=Module.vtType,
objectId=m.id,
data=m)
function_id = vistrail.idScope.getNewId(ModuleFunction.vtType)
function = ModuleFunction(id=function_id, name='value')
change_op = ChangeOp(id=vistrail.idScope.getNewId(ChangeOp.vtType),
what=ModuleFunction.vtType,
oldObjId=2,
newObjId=function.real_id,
parentObjId=m.id,
parentObjType=Module.vtType,
data=function)
param = ModuleParam(id=vistrail.idScope.getNewId(ModuleParam.vtType),
type='Integer',
val='1')
delete_op = DeleteOp(id=vistrail.idScope.getNewId(DeleteOp.vtType),
what=ModuleParam.vtType,
objectId=param.real_id,
parentObjId=function.real_id,
parentObjType=ModuleFunction.vtType)
action1 = Action(id=vistrail.idScope.getNewId(Action.vtType),
operations=[add_op])
action2 = Action(id=vistrail.idScope.getNewId(Action.vtType),
operations=[change_op, delete_op])
vistrail.add_action(action1, 0)
vistrail.add_action(action2, action1.id)
vistrail.addTag('first action', action1.id)
vistrail.addTag('second action', action2.id)
return vistrail
def execute(self, *args, **kwargs):
"""Execute the pipeline.
Positional arguments are either input values (created from
``module == value``, where `module` is a Module from the pipeline and
`value` is some value or Function instance) for the pipeline's
InputPorts, or Module instances (to select sink modules).
Keyword arguments are also used to set InputPort by looking up inputs
by name.
Example::
input_bound = pipeline.get_input('higher_bound')
input_url = pipeline.get_input('url')
sinkmodule = pipeline.get_module(32)
pipeline.execute(sinkmodule,
input_bound == vt.Function(Integer, 10),
input_url == 'http://www.vistrails.org/',
resolution=15) # kwarg: only one equal sign
"""
pipeline = self.pipeline
sinks = set()
inputs = {}
reg = get_module_registry()
InputPort_desc = reg.get_descriptor_by_name(
get_vistrails_basic_pkg_id(), 'InputPort')
# Read args
for arg in args:
if isinstance(arg, ModuleValuePair):
if arg.module.id in inputs:
raise ValueError("Multiple values set for InputPort %r" %
get_inputoutput_name(arg.module))
if not reg.is_descriptor_subclass(arg.module.module_descriptor,
InputPort_desc):
raise ValueError("Module %d is not an InputPort" %
arg.module.id)
inputs[arg.module.id] = arg.value
elif isinstance(arg, Module):
sinks.add(arg.module_id)
# Read kwargs
for key, value in kwargs.iteritems():
name = key
key = self.get_python_parameter(key) # Might raise KeyError
if name in inputs:
raise ValueError("Multiple values set for input %r" % name)
inputs[name] = [key.module_id, value]
reason = "API pipeline execution"
sinks = sinks or None
# Use controller only if no inputs were passed in
if (not inputs and self.vistrail is not None
and self.vistrail.current_version == self.version):
controller = self.vistrail.controller
results, changed = controller.execute_workflow_list([[
controller.locator, # locator
self.version, # version
self.pipeline, # pipeline
DummyView(), # view
None, # custom_aliases
None, # custom_params
reason, # reason
sinks, # sinks
None, # extra_info
]])
result, = results
else:
# pipeline = self.pipeline
if inputs:
# id_scope = IdScope(1)
# pipeline = pipeline.do_copy(False, id_scope)
# A hach to get ids from id_scope that we know won't collide:
# make them negative
# id_scope.getNewId = lambda t, g=id_scope.getNewId: -g(t)
# create_module = \
# VistrailController.create_module_from_descriptor_static
# create_function = VistrailController.create_function_static
# create_connection = VistrailController.create_connection_static
# Fills in the ExternalPipe ports
for name, input_list in inputs.iteritems():
module_id, values = input_list
module = pipeline.modules[module_id]
if not isinstance(values, (list, tuple)):
values = [values]
'''
# Guess the type of the InputPort
_, sigstrings, _, _, _ = get_port_spec_info(pipeline, module)
sigstrings = parse_port_spec_string(sigstrings)
# Convert whatever we got to a list of strings, for the
# pipeline
values = [reg.convert_port_val(val, sigstring, None)
for val, sigstring in izip(values, sigstrings)]
if len(values) == 1:
# Create the constant module
constant_desc = reg.get_descriptor_by_name(
*sigstrings[0])
#print('Setting desription: ',str(constant_desc),str(sigstrings[0]))
constant_mod = create_module(id_scope, constant_desc)
func = create_function(id_scope, constant_mod,
'value', values)
constant_mod.add_function(func)
pipeline.add_module(constant_mod)
# Connect it to the ExternalPipe port
conn = create_connection(id_scope,
constant_mod, 'value',
module, 'ExternalPipe')
pipeline.db_add_connection(conn)
else:
raise RuntimeError("TODO : create tuple")
'''
port_spec = reg.get_input_port_spec(module, name)
added_functions = {}
tmp_f_id = -1L
tmp_p_id = -1L
function = [
f for f in module.functions if f.name == port_spec.name
]
if function:
function = function[0]
else:
try:
function = added_functions[(module.id,
port_spec.name)]
except KeyError:
# add to function list
params = []
for psi in port_spec.port_spec_items:
parameter = ModuleParam(
id=tmp_p_id,
pos=psi.pos,
name='<no description>',
val=psi.default,
type=psi.descriptor.sigstring)
params.append(parameter)
tmp_p_id -= 1
function = ModuleFunction(
id=tmp_f_id,
pos=module.getNumFunctions(),
name=port_spec.name,
parameters=params)
tmp_f_id -= 1
added_functions[(module.id,
port_spec.name)] = function
action = vistrails.core.db.action.create_action([
('add', function, module.vtType, module.id)
])
# function_actions.append(action)
parameter = function.params[0]
# find old parameter
old_param = parameter
actions = []
for v in values:
desc = reg.get_descriptor_by_name(
'org.vistrails.vistrails.basic', 'String', None)
if not isinstance(v, str):
str_value = desc.module.translate_to_string(v)
else:
str_value = v
new_param = ModuleParam(id=tmp_p_id,
pos=old_param.pos,
name=old_param.name,
alias=old_param.alias,
val=str_value,
type=old_param.type)
tmp_p_id -= 1
action_spec = ('change', old_param, new_param,
function.vtType, function.real_id)
action = vistrails.core.db.action.create_action(
[action_spec])
actions.append(action)
# controller = self.vistrail.controller
self.controller.perform_action(action)
#########################################################################
interpreter = get_default_interpreter()
result = interpreter.execute(pipeline,
locator=self.locator,
reason=reason,
sinks=sinks,
actions=actions)
if result.errors:
raise ExecutionErrors(self, result)
else:
return ExecutionResults(self, result)
def collectParameterActions(self, pipeline):
""" collectParameterActions() -> list
Return a list of action lists corresponding to each dimension
"""
if not pipeline:
return
unescape_dict = { "'":"'", '"':'"', '
':'\n' }
from vistrails.core.modules.module_registry import get_module_registry
reg = get_module_registry()
parameterValues = [[], [], [], []]
# a list of added functions [(module_id, function_name)] = function
added_functions = {}
vistrail_vars = []
function_actions = []
tmp_f_id = -1L
tmp_p_id = -1L
for i in xrange(len(self.functions)):
pe_function = self.functions[i]
module = pipeline.db_get_object(Module.vtType, pe_function.module_id)
# collect overridden vistrail vars
if module.is_vistrail_var():
vistrail_vars.append(module.get_vistrail_var())
port_spec = reg.get_input_port_spec(module, pe_function.port_name)
for param in pe_function.parameters:
port_spec_item = port_spec.port_spec_items[param.pos]
dim = param.dimension
if dim not in [0, 1, 2, 3]:
continue
count = self.dims[dim]
# find interpolator values
values = []
text = '%s' % unescape(param.value, unescape_dict)
if param.interpolator == 'Linear Interpolation':
# need to figure out type
if port_spec_item.module == "Integer":
i_range = literal_eval(text)
p_min = int(i_range[0])
p_max =int(i_range[1])
values = IntegerLinearInterpolator(p_min, p_max,
count).get_values()
if port_spec_item.module == "Float":
i_range = literal_eval(text)
p_min = float(i_range[0])
p_max =float(i_range[1])
values = FloatLinearInterpolator(p_min, p_max,
count).get_values()
elif param.interpolator == 'RGB Interpolation':
i_range = literal_eval(text)
p_min = str(i_range[0])
p_max =str(i_range[1])
values = RGBColorInterpolator(p_min, p_max,
count).get_values()
elif param.interpolator == 'HSV Interpolation':
i_range = literal_eval(text)
p_min = str(i_range[0])
p_max =str(i_range[1])
values = HSVColorInterpolator(p_min, p_max,
count).get_values()
elif param.interpolator == 'List':
p_module = port_spec_item.descriptor.module
values = [p_module.translate_to_python(m)
for m in literal_eval(text)]
elif param.interpolator == 'User-defined Function':
p_module = port_spec_item.descriptor.module
values = UserDefinedFunctionInterpolator(p_module,
text, count).get_values()
if not values:
return None
# find parameter or create one
function = [f for f in module.functions
if f.name == port_spec.name]
if function:
function = function[0]
else:
try:
function = added_functions[(module.id,port_spec.name)]
except KeyError:
# add to function list
params = []
for psi in port_spec.port_spec_items:
parameter = ModuleParam(id=tmp_p_id,
pos=psi.pos,
name='<no description>',
val=psi.default,
type=psi.descriptor.sigstring)
params.append(parameter)
tmp_p_id -= 1
function = ModuleFunction(id=tmp_f_id,
pos=module.getNumFunctions(),
name=port_spec.name,
parameters=params)
tmp_f_id -= 1
added_functions[(module.id, port_spec.name)]=function
action = vistrails.core.db.action.create_action([('add',
function,
module.vtType,
module.id)])
function_actions.append(action)
parameter = function.params[port_spec_item.pos]
# find old parameter
old_param = parameter
actions = []
for v in values:
desc = port_spec_item.descriptor
if not isinstance(v, str):
str_value = desc.module.translate_to_string(v)
else:
str_value = v
new_param = ModuleParam(id=tmp_p_id,
pos=old_param.pos,
name=old_param.name,
alias=old_param.alias,
val=str_value,
type=old_param.type)
tmp_p_id -= 1
action_spec = ('change', old_param, new_param,
function.vtType, function.real_id)
action = vistrails.core.db.action.create_action([action_spec])
actions.append(action)
parameterValues[dim].append(actions)
return [zip(*p) for p in parameterValues], function_actions, vistrail_vars
def updateFunctionPort(self):
"""
Function to be used inside the updateUsptream method of the Map module. It
updates the module connected to the FunctionPort port, executing it in
parallel.
"""
nameInput = self.getInputFromPort('InputPort')
nameOutput = self.getInputFromPort('OutputPort')
rawInputList = self.getInputFromPort('InputList')
# Create inputList to always have iterable elements
# to simplify code
if len(nameInput) == 1:
element_is_iter = False
inputList = [[element] for element in rawInputList]
else:
element_is_iter = True
inputList = rawInputList
workflows = []
module = None
vtType = None
# iterating through the connectors
for connector in self.inputPorts.get('FunctionPort'):
module = connector.obj
# pipeline
original_pipeline = connector.obj.moduleInfo['pipeline']
# module
module_id = connector.obj.moduleInfo['moduleId']
vtType = original_pipeline.modules[module_id].vtType
# serialize the module for each value in the list
for i, element in enumerate(inputList):
if element_is_iter:
self.element = element
else:
self.element = element[0]
# checking type and setting input in the module
self.typeChecking(connector.obj, nameInput, inputList)
self.setInputValues(connector.obj, nameInput, element)
pipeline_db_module = original_pipeline.modules[module_id].do_copy()
# transforming a subworkflow in a group
# TODO: should we also transform inner subworkflows?
if pipeline_db_module.is_abstraction():
group = Group(id=pipeline_db_module.id,
cache=pipeline_db_module.cache,
location=pipeline_db_module.location,
functions=pipeline_db_module.functions,
annotations=pipeline_db_module.annotations)
source_port_specs = pipeline_db_module.sourcePorts()
dest_port_specs = pipeline_db_module.destinationPorts()
for source_port_spec in source_port_specs:
group.add_port_spec(source_port_spec)
for dest_port_spec in dest_port_specs:
group.add_port_spec(dest_port_spec)
group.pipeline = pipeline_db_module.pipeline
pipeline_db_module = group
# getting highest id between functions to guarantee unique ids
# TODO: can get current IdScope here?
if pipeline_db_module.functions:
high_id = max(function.db_id
for function in pipeline_db_module.functions)
else:
high_id = 0
# adding function and parameter to module in pipeline
# TODO: 'pos' should not be always 0 here
id_scope = IdScope(beginId=long(high_id+1))
for elementValue, inputPort in izip(element, nameInput):
p_spec = pipeline_db_module.get_port_spec(inputPort, 'input')
descrs = p_spec.descriptors()
if len(descrs) != 1:
raise ModuleError(
self,
"Tuple input ports are not supported")
if not issubclass(descrs[0].module, Constant):
raise ModuleError(
self,
"Module inputs should be Constant types")
type = p_spec.sigstring[1:-1]
mod_function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType),
pos=0,
name=inputPort)
mod_param = ModuleParam(id=0L,
pos=0,
type=type,
val=elementValue)
mod_function.add_parameter(mod_param)
pipeline_db_module.add_function(mod_function)
# serializing module
wf = self.serialize_module(pipeline_db_module)
workflows.append(wf)
# getting first connector, ignoring the rest
break
# IPython stuff
try:
rc = get_client()
except Exception, error:
raise ModuleError(self, "Exception while loading IPython: "
"%s" % error)
def updateFunctionPort(self):
"""
Function to be used inside the updateUsptream method of the Map module. It
updates the module connected to the FunctionPort port, executing it in
parallel.
"""
nameInput = self.get_input('InputPort')
nameOutput = self.get_input('OutputPort')
rawInputList = self.get_input('InputList')
# Create inputList to always have iterable elements
# to simplify code
if len(nameInput) == 1:
element_is_iter = False
inputList = [[element] for element in rawInputList]
else:
element_is_iter = True
inputList = rawInputList
workflows = []
module = None
vtType = None
# iterating through the connectors
for connector in self.inputPorts.get('FunctionPort'):
module = connector.obj
# pipeline
original_pipeline = connector.obj.moduleInfo['pipeline']
# module
module_id = connector.obj.moduleInfo['moduleId']
vtType = original_pipeline.modules[module_id].vtType
# serialize the module for each value in the list
for i, element in enumerate(inputList):
if element_is_iter:
self.element = element
else:
self.element = element[0]
# checking type and setting input in the module
self.typeChecking(connector.obj, nameInput, inputList)
self.setInputValues(connector.obj, nameInput, element, i)
pipeline_db_module = original_pipeline.modules[
module_id].do_copy()
# transforming a subworkflow in a group
# TODO: should we also transform inner subworkflows?
if pipeline_db_module.is_abstraction():
group = Group(id=pipeline_db_module.id,
cache=pipeline_db_module.cache,
location=pipeline_db_module.location,
functions=pipeline_db_module.functions,
annotations=pipeline_db_module.annotations)
source_port_specs = pipeline_db_module.sourcePorts()
dest_port_specs = pipeline_db_module.destinationPorts()
for source_port_spec in source_port_specs:
group.add_port_spec(source_port_spec)
for dest_port_spec in dest_port_specs:
group.add_port_spec(dest_port_spec)
group.pipeline = pipeline_db_module.pipeline
pipeline_db_module = group
# getting highest id between functions to guarantee unique ids
# TODO: can get current IdScope here?
if pipeline_db_module.functions:
high_id = max(function.db_id
for function in pipeline_db_module.functions)
else:
high_id = 0
# adding function and parameter to module in pipeline
# TODO: 'pos' should not be always 0 here
id_scope = IdScope(beginId=long(high_id + 1))
for elementValue, inputPort in izip(element, nameInput):
p_spec = pipeline_db_module.get_port_spec(
inputPort, 'input')
descrs = p_spec.descriptors()
if len(descrs) != 1:
raise ModuleError(
self, "Tuple input ports are not supported")
if not issubclass(descrs[0].module, Constant):
raise ModuleError(
self, "Module inputs should be Constant types")
type = p_spec.sigstring[1:-1]
mod_function = ModuleFunction(id=id_scope.getNewId(
ModuleFunction.vtType),
pos=0,
name=inputPort)
mod_param = ModuleParam(id=0L,
pos=0,
type=type,
val=elementValue)
mod_function.add_parameter(mod_param)
pipeline_db_module.add_function(mod_function)
# serializing module
wf = self.serialize_module(pipeline_db_module)
workflows.append(wf)
# getting first connector, ignoring the rest
break
# IPython stuff
try:
rc = get_client()
except Exception, error:
raise ModuleError(
self, "Exception while loading IPython: %s" %
debug.format_exception(error))
def execute(modules,
connections=[],
add_port_specs=[],
enable_pkg=True,
full_results=False):
"""Build a pipeline and execute it.
This is useful to simply build a pipeline in a test case, and run it. When
doing that, intercept_result() can be used to check the results of each
module.
modules is a list of module tuples describing the modules to be created,
with the following format:
[('ModuleName', 'package.identifier', [
# Functions
('port_name', [
# Function parameters
('Signature', 'value-as-string'),
]),
])]
connections is a list of tuples describing the connections to make, with
the following format:
[
(source_module_index, 'source_port_name',
dest_module_index, 'dest_module_name'),
]
add_port_specs is a list of specs to add to modules, with the following
format:
[
(mod_id, 'input'/'output', 'portname',
'(port_sig)'),
]
It is useful to test modules that can have custom ports through a
configuration widget.
The function returns the 'errors' dict it gets from the interpreter, so you
should use a construct like self.assertFalse(execute(...)) if the execution
is not supposed to fail.
For example, this creates (and runs) an Integer module with its value set
to 44, connected to a PythonCalc module, connected to a StandardOutput:
self.assertFalse(execute([
('Float', 'org.vistrails.vistrails.basic', [
('value', [('Float', '44.0')]),
]),
('PythonCalc', 'org.vistrails.vistrails.pythoncalc', [
('value2', [('Float', '2.0')]),
('op', [('String', '-')]),
]),
('StandardOutput', 'org.vistrails.vistrails.basic', []),
],
[
(0, 'value', 1, 'value1'),
(1, 'value', 2, 'value'),
]))
"""
from vistrails.core.db.locator import XMLFileLocator
from vistrails.core.modules.module_registry import MissingPackage
from vistrails.core.packagemanager import get_package_manager
from vistrails.core.utils import DummyView
from vistrails.core.vistrail.connection import Connection
from vistrails.core.vistrail.module import Module
from vistrails.core.vistrail.module_function import ModuleFunction
from vistrails.core.vistrail.module_param import ModuleParam
from vistrails.core.vistrail.pipeline import Pipeline
from vistrails.core.vistrail.port import Port
from vistrails.core.vistrail.port_spec import PortSpec
from vistrails.core.interpreter.noncached import Interpreter
pm = get_package_manager()
port_spec_per_module = {} # mod_id -> [portspec: PortSpec]
j = 0
for i, (mod_id, inout, name, sig) in enumerate(add_port_specs):
mod_specs = port_spec_per_module.setdefault(mod_id, [])
ps = PortSpec(id=i, name=name, type=inout, sigstring=sig, sort_key=-1)
for psi in ps.port_spec_items:
psi.id = j
j += 1
mod_specs.append(ps)
pipeline = Pipeline()
module_list = []
for i, (name, identifier, functions) in enumerate(modules):
function_list = []
try:
pkg = pm.get_package(identifier)
except MissingPackage:
if not enable_pkg:
raise
dep_graph = pm.build_dependency_graph([identifier])
for pkg_id in pm.get_ordered_dependencies(dep_graph):
pkg = pm.identifier_is_available(pkg_id)
if pkg is None:
raise
pm.late_enable_package(pkg.codepath)
pkg = pm.get_package(identifier)
for func_name, params in functions:
param_list = []
for j, (param_type, param_val) in enumerate(params):
param_list.append(
ModuleParam(pos=j, type=param_type, val=param_val))
function_list.append(
ModuleFunction(name=func_name, parameters=param_list))
name = name.rsplit('|', 1)
if len(name) == 2:
namespace, name = name
else:
namespace = None
name, = name
module = Module(name=name,
namespace=namespace,
package=identifier,
version=pkg.version,
id=i,
functions=function_list)
for port_spec in port_spec_per_module.get(i, []):
module.add_port_spec(port_spec)
pipeline.add_module(module)
module_list.append(module)
for i, (sid, sport, did, dport) in enumerate(connections):
s_sig = module_list[sid].get_port_spec(sport, 'output').sigstring
d_sig = module_list[did].get_port_spec(dport, 'input').sigstring
pipeline.add_connection(
Connection(id=i,
ports=[
Port(id=i * 2,
type='source',
moduleId=sid,
name=sport,
signature=s_sig),
Port(id=i * 2 + 1,
type='destination',
moduleId=did,
name=dport,
signature=d_sig),
]))
interpreter = Interpreter.get()
result = interpreter.execute(pipeline,
locator=XMLFileLocator('foo.xml'),
current_version=1,
view=DummyView())
if full_results:
return result
else:
# Allows to do self.assertFalse(execute(...))
return result.errors
