def process_group(group):
def get_port_name(module):
port_name = None
for function in module.db_functions:
if function.db_name == 'name':
port_name = function.db_parameters[0].db_val
return port_name
g_workflow = group.db_workflow
group.db_portSpecs_name_index = {}
for module in g_workflow.db_modules:
if module.db_name == 'InputPort' and module.db_package == basic_pkg:
port_name = get_port_name(module)
# FIXME add sigstring to DBPortSpec
group.db_portSpecs_name_index[(port_name, 'input')] = \
DBPortSpec(id=-1,
name=port_name,
type='input')
elif module.db_name == 'OutputPort' and \
module.db_package == basic_package:
port_name = get_port_name(module)
# FIXME add sigstring to DBPortSpec
group.db_portSpecs_name_index[(port_name, 'output')] = \
DBPortSpec(id=-1,
name=port_name,
type='output')
elif module.db_name == 'Group' and module.db_package == basic_pkg:
process_group(module)
def do_copy(self, new_ids=False, id_scope=None, id_remap=None):
cp = DBPortSpec.do_copy(self, new_ids, id_scope, id_remap)
cp._short_sigstring = self._short_sigstring
cp._tooltip = self._tooltip
cp._shape = self._shape
cp._docstring = self._docstring
cp.is_valid = self.is_valid
cp.__class__ = PortSpec
# if cp._entries is not None:
# cp.create_tooltip()
return cp
def do_copy(self, new_ids=False, id_scope=None, id_remap=None):
cp = DBPortSpec.do_copy(self, new_ids, id_scope, id_remap)
cp._short_sigstring = self._short_sigstring
cp._tooltip = self._tooltip
cp._shape = self._shape
cp._docstring = self._docstring
cp.is_valid = self.is_valid
cp.__class__ = PortSpec
# if cp._entries is not None:
# cp.create_tooltip()
return cp
def test_port_spec_copy(self):
id_scope = IdScope()
psi = DBPortSpecItem(id=id_scope.getNewId(DBPortSpecItem.vtType),
pos=0,
module="File",
package="org.vistrails.vistrails.basic")
ps1 = DBPortSpec(id=id_scope.getNewId(DBPortSpec.vtType),
portSpecItems=[psi,])
ops = create_copy_op_chain(ps1, id_scope=id_scope)
self.assertNotEqual(ps1.db_portSpecItems[0].db_id,
ops[0].db_data.db_portSpecItems[0].db_id)
def __init__(self, *args, **kwargs):
signature = None
if 'signature' in kwargs:
signature = kwargs['signature']
del kwargs['signature']
sigstring = None
if 'sigstring' in kwargs:
sigstring = kwargs['sigstring']
del kwargs['sigstring']
defaults = None
if 'defaults' in kwargs:
defaults = kwargs['defaults']
del kwargs['defaults']
labels = None
if 'labels' in kwargs:
labels = kwargs['labels']
del kwargs['labels']
values = None
if 'values' in kwargs:
values = kwargs['values']
del kwargs['values']
entry_types = None
if 'entry_types' in kwargs:
entry_types = kwargs['entry_types']
del kwargs['entry_types']
if 'items' in kwargs and 'portSpecItems' not in kwargs:
kwargs['portSpecItems'] = kwargs['items']
del kwargs['items']
if 'optional' not in kwargs:
kwargs['optional'] = 0 # False
elif not isinstance(kwargs['optional'], (int, long)):
if isinstance(kwargs['optional'], bool):
if kwargs['optional']:
kwargs['optional'] = 1
else:
kwargs['optional'] = 0
else:
raise VistrailsInternalError("Cannot parse 'optional' kw "
"-- must be an int or bool")
if 'min_conns' not in kwargs:
kwargs['min_conns'] = 0
elif kwargs['optional'] == 1 and kwargs['min_conns'] > 0:
raise VistrailsInternalError("A mandatory port cannot be set "
"to optional")
if 'max_conns' not in kwargs:
kwargs['max_conns'] = -1
if kwargs['min_conns'] >= 0 and kwargs['max_conns'] >= 0 and \
kwargs['min_conns'] > kwargs['max_conns']:
raise VistrailsInternalError("Minimum number of connections "
"cannot be greater than maximum "
"number of connections")
if 'sort_key' not in kwargs:
kwargs['sort_key'] = -1
if 'id' not in kwargs:
kwargs['id'] = -1
if 'tooltip' in kwargs:
self._tooltip = kwargs['tooltip']
del kwargs['tooltip']
else:
self._tooltip = None
if 'docstring' in kwargs:
self._docstring = kwargs['docstring']
del kwargs['docstring']
else:
self._docstring = None
if 'shape' in kwargs:
self._shape = kwargs['shape']
del kwargs['shape']
else:
self._shape = None
DBPortSpec.__init__(self, *args, **kwargs)
if sum(1 for container in (self.port_spec_items, signature, sigstring)
if container) > 1:
raise ValueError("Please specify only one of portSpecItems,"
" signature, or sigstring kwargs.")
self.create_spec_items(self.port_spec_items, signature, sigstring,
defaults, labels, values, entry_types)
self._short_sigstring = None
# if signature is not None:
# self.create_entries(signature)
# if not self.sigstring and self._entries is not None:
# # create sigstring from entries
# self.create_sigstring_and_descriptors()
# DAKOOP: removed this---we will check in module_registry and pipeline
# validation, this way, we can let errors go all the way up
# elif self._entries is None and self.sigstring:
# # create entries from sigstring
# self.create_entries_and_descriptors()
# else:
# raise VistrailsInternalError("Need to specify signature or "
# "sigstring to create PortSpec")
# if self._entries is not None and self._tooltip is None:
# self.create_tooltip()
self.is_valid = True
def __init__(self, *args, **kwargs):
signature = None
if 'signature' in kwargs:
signature = kwargs['signature']
del kwargs['signature']
sigstring = None
if 'sigstring' in kwargs:
sigstring = kwargs['sigstring']
del kwargs['sigstring']
defaults = None
if 'defaults' in kwargs:
defaults = kwargs['defaults']
del kwargs['defaults']
labels = None
if 'labels' in kwargs:
labels = kwargs['labels']
del kwargs['labels']
values = None
if 'values' in kwargs:
values = kwargs['values']
del kwargs['values']
entry_types = None
if 'entry_types' in kwargs:
entry_types = kwargs['entry_types']
del kwargs['entry_types']
if 'items' in kwargs and 'portSpecItems' not in kwargs:
kwargs['portSpecItems'] = kwargs['items']
del kwargs['items']
if 'optional' not in kwargs:
kwargs['optional'] = 0 # False
elif not isinstance(kwargs['optional'], (int, long)):
if isinstance(kwargs['optional'], bool):
if kwargs['optional']:
kwargs['optional'] = 1
else:
kwargs['optional'] = 0
else:
raise VistrailsInternalError("Cannot parse 'optional' kw "
"-- must be an int or bool")
if 'min_conns' not in kwargs:
kwargs['min_conns'] = 0
elif kwargs['optional'] == 1 and kwargs['min_conns'] > 0:
raise VistrailsInternalError("A mandatory port cannot be set "
"to optional")
if 'max_conns' not in kwargs:
kwargs['max_conns'] = -1
if kwargs['min_conns'] >= 0 and kwargs['max_conns'] >= 0 and \
kwargs['min_conns'] > kwargs['max_conns']:
raise VistrailsInternalError("Minimum number of connections "
"cannot be greater than maximum "
"number of connections")
if 'sort_key' not in kwargs:
kwargs['sort_key'] = -1
if 'depth' not in kwargs:
kwargs['depth'] = 0
if 'id' not in kwargs:
kwargs['id'] = -1
if 'tooltip' in kwargs:
self._tooltip = kwargs['tooltip']
del kwargs['tooltip']
else:
self._tooltip = None
if 'docstring' in kwargs:
self._docstring = kwargs['docstring']
del kwargs['docstring']
else:
self._docstring = None
if 'shape' in kwargs:
self._shape = kwargs['shape']
del kwargs['shape']
else:
self._shape = None
DBPortSpec.__init__(self, *args, **kwargs)
if sum(1 for container in (self.port_spec_items, signature, sigstring)
if container) > 1:
raise ValueError("Please specify only one of portSpecItems,"
" signature, or sigstring kwargs.")
self.create_spec_items(self.port_spec_items, signature, sigstring,
defaults, labels, values, entry_types)
self._short_sigstring = None
# if signature is not None:
# self.create_entries(signature)
# if not self.sigstring and self._entries is not None:
# # create sigstring from entries
# self.create_sigstring_and_descriptors()
# DAKOOP: removed this---we will check in module_registry and pipeline
# validation, this way, we can let errors go all the way up
# elif self._entries is None and self.sigstring:
# # create entries from sigstring
# self.create_entries_and_descriptors()
# else:
# raise VistrailsInternalError("Need to specify signature or "
# "sigstring to create PortSpec")
# if self._entries is not None and self._tooltip is None:
# self.create_tooltip()
self.is_valid = True
def process_module_loop(module, found_input_ports, found_output_ports):
print "*** Processing Module with loops"
if depth + 1 in depth_accounts:
account = depth_accounts[depth + 1]
else:
account = create_account(depth + 1, id_scope)
accounts.append(account)
depth_accounts[depth + 1] = account
# need to have process that extracts artifacts for each iteration
result_artifacts = [
a for r in found_output_ports
if found_output_ports[r] is not None
for a in found_output_ports[r]
]
s_process = create_process_manual('Split', account, id_scope)
processes.append(s_process)
for input_port in found_input_ports:
for input_name in input_port:
dependencies.append(
create_used(s_process, found_input_ports[input_name],
account, id_scope))
# need to have process that condenses artifacts from each iteration
if result_artifacts:
j_process = create_process_manual('Join', account, id_scope)
processes.append(j_process)
for loop_exec in item_exec.db_loop_execs:
for loop_iteration in loop_exec.db_loop_iterations:
loop_up_artifacts = {}
loop_down_artifacts = {}
for input_port in found_input_ports:
for input_name in input_port:
port_spec = DBPortSpec(id=-1,
name=input_name,
type='output')
s_artifact = \
create_artifact_from_port_spec(port_spec, account,
id_scope)
artifacts.append(s_artifact)
dependencies.append(
create_was_generated_by(
s_artifact, s_process, account, id_scope))
if input_name not in loop_up_artifacts:
loop_up_artifacts[input_name] = []
loop_up_artifacts[input_name].append(s_artifact)
# process output_port
if loop_iteration.db_completed == 1:
for output_name in found_output_ports:
port_spec = DBPortSpec(id=-1,
name=output_name,
type='output')
o_artifact = \
create_artifact_from_port_spec(port_spec, account,
id_scope)
artifacts.append(o_artifact)
if output_name not in loop_down_artifacts:
loop_down_artifacts[output_name] = []
loop_down_artifacts[output_name].append(o_artifact)
if result_artifacts:
dependencies.append(
create_used(j_process, o_artifact, account,
id_scope))
# now process a loop_exec
for child_exec in loop_iteration.db_item_execs:
do_create_process(workflow, child_exec, account,
module_processes)
for child_exec in loop_iteration.db_item_execs:
process_exec(child_exec, workflow, account,
upstream_lookup, downstream_lookup,
depth + 1, conn_artifacts,
function_artifacts, module_processes,
loop_up_artifacts, loop_down_artifacts,
True)
# need to set Return artifacts and connect j_process to it
for result_artifact in result_artifacts:
dependencies.append(
create_was_generated_by(result_artifact, j_process,
account, id_scope))
def process_map(module, found_input_ports, found_output_ports):
print "*** Processing Map"
if depth + 1 in depth_accounts:
account = depth_accounts[depth + 1]
else:
account = create_account(depth + 1, id_scope)
accounts.append(account)
depth_accounts[depth + 1] = account
# need to have process that extracts artifacts for each iteration
input_list_artifact = found_input_ports['InputList']
result_artifact = found_output_ports.get('Result', None)
# if InputPort or OutputPort is a Connection we cannot do anything
if (found_input_ports['InputPort'].vtType == DBConnection.vtType
or found_input_ports['OutputPort'].vtType
== DBConnection.vtType):
return
input_port_list = \
literal_eval(found_input_ports['InputPort'].db_parameters[0].db_val)
output_port = \
found_input_ports['OutputPort'].db_parameters[0].db_val
s_process = create_process_manual('Split', account, id_scope)
processes.append(s_process)
dependencies.append(
create_used(s_process, input_list_artifact, account, id_scope))
# need to have process that condenses artifacts from each iteration
if result_artifact is not None:
j_process = create_process_manual('Join', account, id_scope)
processes.append(j_process)
for loop_exec in item_exec.db_loop_execs:
for loop_iteration in loop_exec.db_loop_iterations:
loop_up_artifacts = {}
loop_down_artifacts = {}
for input_name in input_port_list:
port_spec = DBPortSpec(id=-1,
name=input_name,
type='output')
s_artifact = \
create_artifact_from_port_spec(port_spec, account,
id_scope)
artifacts.append(s_artifact)
dependencies.append(
create_was_generated_by(s_artifact, s_process,
account, id_scope))
if input_name not in loop_up_artifacts:
loop_up_artifacts[input_name] = []
loop_up_artifacts[input_name].append(s_artifact)
# process output_port
if loop_iteration.db_completed == 1:
port_spec = DBPortSpec(id=-1,
name=output_port,
type='output')
o_artifact = \
create_artifact_from_port_spec(port_spec, account,
id_scope)
artifacts.append(o_artifact)
if output_port not in loop_down_artifacts:
loop_down_artifacts[output_port] = []
loop_down_artifacts[output_port].append(o_artifact)
if result_artifact is not None:
dependencies.append(
create_used(j_process, o_artifact, account,
id_scope))
# now process a loop_exec
for child_exec in loop_iteration.db_item_execs:
do_create_process(workflow, child_exec, account,
module_processes)
for child_exec in loop_iteration.db_item_execs:
process_exec(child_exec, workflow, account,
upstream_lookup, downstream_lookup,
depth + 1, conn_artifacts,
function_artifacts, module_processes,
loop_up_artifacts, loop_down_artifacts,
True)
# need to set Return artifact and connect j_process to it
if result_artifact is not None:
dependencies.append(
create_was_generated_by(result_artifact, j_process,
account, id_scope))
