def process_group(group): def get_port_name(module): port_name = None for function in module.db_functions: if function.db_name == 'name': port_name = function.db_parameters[0].db_val return port_name g_workflow = group.db_workflow group.db_portSpecs_name_index = {} for module in g_workflow.db_modules: if module.db_name == 'InputPort' and module.db_package == basic_pkg: port_name = get_port_name(module) # FIXME add sigstring to DBPortSpec group.db_portSpecs_name_index[(port_name, 'input')] = \ DBPortSpec(id=-1, name=port_name, type='input') elif module.db_name == 'OutputPort' and \ module.db_package == basic_package: port_name = get_port_name(module) # FIXME add sigstring to DBPortSpec group.db_portSpecs_name_index[(port_name, 'output')] = \ DBPortSpec(id=-1, name=port_name, type='output') elif module.db_name == 'Group' and module.db_package == basic_pkg: process_group(module)
def do_copy(self, new_ids=False, id_scope=None, id_remap=None): cp = DBPortSpec.do_copy(self, new_ids, id_scope, id_remap) cp._short_sigstring = self._short_sigstring cp._tooltip = self._tooltip cp._shape = self._shape cp._docstring = self._docstring cp.is_valid = self.is_valid cp.__class__ = PortSpec # if cp._entries is not None: # cp.create_tooltip() return cp
def test_port_spec_copy(self): id_scope = IdScope() psi = DBPortSpecItem(id=id_scope.getNewId(DBPortSpecItem.vtType), pos=0, module="File", package="org.vistrails.vistrails.basic") ps1 = DBPortSpec(id=id_scope.getNewId(DBPortSpec.vtType), portSpecItems=[psi,]) ops = create_copy_op_chain(ps1, id_scope=id_scope) self.assertNotEqual(ps1.db_portSpecItems[0].db_id, ops[0].db_data.db_portSpecItems[0].db_id)
def __init__(self, *args, **kwargs): signature = None if 'signature' in kwargs: signature = kwargs['signature'] del kwargs['signature'] sigstring = None if 'sigstring' in kwargs: sigstring = kwargs['sigstring'] del kwargs['sigstring'] defaults = None if 'defaults' in kwargs: defaults = kwargs['defaults'] del kwargs['defaults'] labels = None if 'labels' in kwargs: labels = kwargs['labels'] del kwargs['labels'] values = None if 'values' in kwargs: values = kwargs['values'] del kwargs['values'] entry_types = None if 'entry_types' in kwargs: entry_types = kwargs['entry_types'] del kwargs['entry_types'] if 'items' in kwargs and 'portSpecItems' not in kwargs: kwargs['portSpecItems'] = kwargs['items'] del kwargs['items'] if 'optional' not in kwargs: kwargs['optional'] = 0 # False elif not isinstance(kwargs['optional'], (int, long)): if isinstance(kwargs['optional'], bool): if kwargs['optional']: kwargs['optional'] = 1 else: kwargs['optional'] = 0 else: raise VistrailsInternalError("Cannot parse 'optional' kw " "-- must be an int or bool") if 'min_conns' not in kwargs: kwargs['min_conns'] = 0 elif kwargs['optional'] == 1 and kwargs['min_conns'] > 0: raise VistrailsInternalError("A mandatory port cannot be set " "to optional") if 'max_conns' not in kwargs: kwargs['max_conns'] = -1 if kwargs['min_conns'] >= 0 and kwargs['max_conns'] >= 0 and \ kwargs['min_conns'] > kwargs['max_conns']: raise VistrailsInternalError("Minimum number of connections " "cannot be greater than maximum " "number of connections") if 'sort_key' not in kwargs: kwargs['sort_key'] = -1 if 'id' not in kwargs: kwargs['id'] = -1 if 'tooltip' in kwargs: self._tooltip = kwargs['tooltip'] del kwargs['tooltip'] else: self._tooltip = None if 'docstring' in kwargs: self._docstring = kwargs['docstring'] del kwargs['docstring'] else: self._docstring = None if 'shape' in kwargs: self._shape = kwargs['shape'] del kwargs['shape'] else: self._shape = None DBPortSpec.__init__(self, *args, **kwargs) if sum(1 for container in (self.port_spec_items, signature, sigstring) if container) > 1: raise ValueError("Please specify only one of portSpecItems," " signature, or sigstring kwargs.") self.create_spec_items(self.port_spec_items, signature, sigstring, defaults, labels, values, entry_types) self._short_sigstring = None # if signature is not None: # self.create_entries(signature) # if not self.sigstring and self._entries is not None: # # create sigstring from entries # self.create_sigstring_and_descriptors() # DAKOOP: removed this---we will check in module_registry and pipeline # validation, this way, we can let errors go all the way up # elif self._entries is None and self.sigstring: # # create entries from sigstring # self.create_entries_and_descriptors() # else: # raise VistrailsInternalError("Need to specify signature or " # "sigstring to create PortSpec") # if self._entries is not None and self._tooltip is None: # self.create_tooltip() self.is_valid = True
def __init__(self, *args, **kwargs): signature = None if 'signature' in kwargs: signature = kwargs['signature'] del kwargs['signature'] sigstring = None if 'sigstring' in kwargs: sigstring = kwargs['sigstring'] del kwargs['sigstring'] defaults = None if 'defaults' in kwargs: defaults = kwargs['defaults'] del kwargs['defaults'] labels = None if 'labels' in kwargs: labels = kwargs['labels'] del kwargs['labels'] values = None if 'values' in kwargs: values = kwargs['values'] del kwargs['values'] entry_types = None if 'entry_types' in kwargs: entry_types = kwargs['entry_types'] del kwargs['entry_types'] if 'items' in kwargs and 'portSpecItems' not in kwargs: kwargs['portSpecItems'] = kwargs['items'] del kwargs['items'] if 'optional' not in kwargs: kwargs['optional'] = 0 # False elif not isinstance(kwargs['optional'], (int, long)): if isinstance(kwargs['optional'], bool): if kwargs['optional']: kwargs['optional'] = 1 else: kwargs['optional'] = 0 else: raise VistrailsInternalError("Cannot parse 'optional' kw " "-- must be an int or bool") if 'min_conns' not in kwargs: kwargs['min_conns'] = 0 elif kwargs['optional'] == 1 and kwargs['min_conns'] > 0: raise VistrailsInternalError("A mandatory port cannot be set " "to optional") if 'max_conns' not in kwargs: kwargs['max_conns'] = -1 if kwargs['min_conns'] >= 0 and kwargs['max_conns'] >= 0 and \ kwargs['min_conns'] > kwargs['max_conns']: raise VistrailsInternalError("Minimum number of connections " "cannot be greater than maximum " "number of connections") if 'sort_key' not in kwargs: kwargs['sort_key'] = -1 if 'depth' not in kwargs: kwargs['depth'] = 0 if 'id' not in kwargs: kwargs['id'] = -1 if 'tooltip' in kwargs: self._tooltip = kwargs['tooltip'] del kwargs['tooltip'] else: self._tooltip = None if 'docstring' in kwargs: self._docstring = kwargs['docstring'] del kwargs['docstring'] else: self._docstring = None if 'shape' in kwargs: self._shape = kwargs['shape'] del kwargs['shape'] else: self._shape = None DBPortSpec.__init__(self, *args, **kwargs) if sum(1 for container in (self.port_spec_items, signature, sigstring) if container) > 1: raise ValueError("Please specify only one of portSpecItems," " signature, or sigstring kwargs.") self.create_spec_items(self.port_spec_items, signature, sigstring, defaults, labels, values, entry_types) self._short_sigstring = None # if signature is not None: # self.create_entries(signature) # if not self.sigstring and self._entries is not None: # # create sigstring from entries # self.create_sigstring_and_descriptors() # DAKOOP: removed this---we will check in module_registry and pipeline # validation, this way, we can let errors go all the way up # elif self._entries is None and self.sigstring: # # create entries from sigstring # self.create_entries_and_descriptors() # else: # raise VistrailsInternalError("Need to specify signature or " # "sigstring to create PortSpec") # if self._entries is not None and self._tooltip is None: # self.create_tooltip() self.is_valid = True
def process_module_loop(module, found_input_ports, found_output_ports): print "*** Processing Module with loops" if depth + 1 in depth_accounts: account = depth_accounts[depth + 1] else: account = create_account(depth + 1, id_scope) accounts.append(account) depth_accounts[depth + 1] = account # need to have process that extracts artifacts for each iteration result_artifacts = [ a for r in found_output_ports if found_output_ports[r] is not None for a in found_output_ports[r] ] s_process = create_process_manual('Split', account, id_scope) processes.append(s_process) for input_port in found_input_ports: for input_name in input_port: dependencies.append( create_used(s_process, found_input_ports[input_name], account, id_scope)) # need to have process that condenses artifacts from each iteration if result_artifacts: j_process = create_process_manual('Join', account, id_scope) processes.append(j_process) for loop_exec in item_exec.db_loop_execs: for loop_iteration in loop_exec.db_loop_iterations: loop_up_artifacts = {} loop_down_artifacts = {} for input_port in found_input_ports: for input_name in input_port: port_spec = DBPortSpec(id=-1, name=input_name, type='output') s_artifact = \ create_artifact_from_port_spec(port_spec, account, id_scope) artifacts.append(s_artifact) dependencies.append( create_was_generated_by( s_artifact, s_process, account, id_scope)) if input_name not in loop_up_artifacts: loop_up_artifacts[input_name] = [] loop_up_artifacts[input_name].append(s_artifact) # process output_port if loop_iteration.db_completed == 1: for output_name in found_output_ports: port_spec = DBPortSpec(id=-1, name=output_name, type='output') o_artifact = \ create_artifact_from_port_spec(port_spec, account, id_scope) artifacts.append(o_artifact) if output_name not in loop_down_artifacts: loop_down_artifacts[output_name] = [] loop_down_artifacts[output_name].append(o_artifact) if result_artifacts: dependencies.append( create_used(j_process, o_artifact, account, id_scope)) # now process a loop_exec for child_exec in loop_iteration.db_item_execs: do_create_process(workflow, child_exec, account, module_processes) for child_exec in loop_iteration.db_item_execs: process_exec(child_exec, workflow, account, upstream_lookup, downstream_lookup, depth + 1, conn_artifacts, function_artifacts, module_processes, loop_up_artifacts, loop_down_artifacts, True) # need to set Return artifacts and connect j_process to it for result_artifact in result_artifacts: dependencies.append( create_was_generated_by(result_artifact, j_process, account, id_scope))
def process_map(module, found_input_ports, found_output_ports): print "*** Processing Map" if depth + 1 in depth_accounts: account = depth_accounts[depth + 1] else: account = create_account(depth + 1, id_scope) accounts.append(account) depth_accounts[depth + 1] = account # need to have process that extracts artifacts for each iteration input_list_artifact = found_input_ports['InputList'] result_artifact = found_output_ports.get('Result', None) # if InputPort or OutputPort is a Connection we cannot do anything if (found_input_ports['InputPort'].vtType == DBConnection.vtType or found_input_ports['OutputPort'].vtType == DBConnection.vtType): return input_port_list = \ literal_eval(found_input_ports['InputPort'].db_parameters[0].db_val) output_port = \ found_input_ports['OutputPort'].db_parameters[0].db_val s_process = create_process_manual('Split', account, id_scope) processes.append(s_process) dependencies.append( create_used(s_process, input_list_artifact, account, id_scope)) # need to have process that condenses artifacts from each iteration if result_artifact is not None: j_process = create_process_manual('Join', account, id_scope) processes.append(j_process) for loop_exec in item_exec.db_loop_execs: for loop_iteration in loop_exec.db_loop_iterations: loop_up_artifacts = {} loop_down_artifacts = {} for input_name in input_port_list: port_spec = DBPortSpec(id=-1, name=input_name, type='output') s_artifact = \ create_artifact_from_port_spec(port_spec, account, id_scope) artifacts.append(s_artifact) dependencies.append( create_was_generated_by(s_artifact, s_process, account, id_scope)) if input_name not in loop_up_artifacts: loop_up_artifacts[input_name] = [] loop_up_artifacts[input_name].append(s_artifact) # process output_port if loop_iteration.db_completed == 1: port_spec = DBPortSpec(id=-1, name=output_port, type='output') o_artifact = \ create_artifact_from_port_spec(port_spec, account, id_scope) artifacts.append(o_artifact) if output_port not in loop_down_artifacts: loop_down_artifacts[output_port] = [] loop_down_artifacts[output_port].append(o_artifact) if result_artifact is not None: dependencies.append( create_used(j_process, o_artifact, account, id_scope)) # now process a loop_exec for child_exec in loop_iteration.db_item_execs: do_create_process(workflow, child_exec, account, module_processes) for child_exec in loop_iteration.db_item_execs: process_exec(child_exec, workflow, account, upstream_lookup, downstream_lookup, depth + 1, conn_artifacts, function_artifacts, module_processes, loop_up_artifacts, loop_down_artifacts, True) # need to set Return artifact and connect j_process to it if result_artifact is not None: dependencies.append( create_was_generated_by(result_artifact, j_process, account, id_scope))