def upgrade_cell_to_output(module_remap, module_id, pipeline, old_name, new_module, end_version, input_port_name, start_version=None, output_version=None): """This function upgrades a *Cell module to a *Output module. The upgrade only happens if the original module doesn't have any connection on the cell input ports that can't be translated. This is to ease the transition to *Output modules, but we don't want (or need) to break anything; the *Cell modules still exist, so they can stay. """ if not isinstance(module_remap, UpgradePackageRemap): module_remap = UpgradePackageRemap.from_dict(module_remap) old_module = pipeline.modules[module_id] old_module_name = create_descriptor_string(old_module.package, old_module.name, old_module.namespace, False) if old_module_name != old_name: return module_remap used_input_ports = set(old_module.connected_input_ports.keys()) for func in old_module.functions: used_input_ports.add(func.name) if used_input_ports != set([input_port_name]): return module_remap _old_remap = module_remap module_remap = copy.copy(module_remap) assert _old_remap.remaps is not module_remap.remaps remap = UpgradeModuleRemap(start_version, end_version, output_version, module_name=old_name, new_module=new_module) remap.add_remap('dst_port_remap', input_port_name, 'value') remap.add_remap('function_remap', input_port_name, 'value') module_remap.add_module_remap(remap) return module_remap
def remap_module(controller, module_id, pipeline, pkg_remap): """remap_module offers a method to shortcut the specification of upgrades. It is useful when just changing the names of ports or modules, but can also be used to add intermediate modules or change the format of parameters. It is usually called from handle_module_upgrade_request, and the first three arguments are passed from the arguments to that method. pkg_remap specifies all of the changes and is of the format {<old_module_name>: [(<start_version>, <end_version>, <new_module_klass> | <new_module_id> | None, <remap_dictionary>)]} where new_module_klass is the class and new_module_id is a string of the format <package_name>:[<namespace> | ]<module_name> passing None keeps the original name, and remap_dictionary is {<remap_type>: <name_changes>} and <name_changes> is a map from <old_name> to <new_name> or <remap_function> The remap functions are passed the old object and the new module and should return a list of operations with elements of the form ('add', <obj>). For example: def outputName_remap(old_conn, new_module): ops = [] ... return ops pkg_remap = {'FileSink': [(None, '1.5.1', FileSink, {'dst_port_remap': {'overrideFile': 'overwrite', 'outputName': outputName_remap}, 'function_remap': {'overrideFile': 'overwrite', 'outputName': 'outputPath'}}), } """ reg = get_module_registry() old_module = pipeline.modules[module_id] old_version = old_module.version old_desc_str = create_descriptor_string(old_module.package, old_module.name, old_module.namespace, False) # print 'running module_upgrade_request', old_module.name if not isinstance(pkg_remap, UpgradePackageRemap): pkg_remap = UpgradePackageRemap.from_dict(pkg_remap) action_list = [] old_module_t = \ (old_module.package, old_module.name, old_module.namespace) module_remap = pkg_remap.get_module_upgrade(old_desc_str, old_version) tmp_pipeline = copy.copy(pipeline) while module_remap is not None: new_module_type = module_remap.new_module if new_module_type is None: new_module_t = old_module_t elif isinstance(new_module_type, basestring): new_module_t = parse_descriptor_string(new_module_type, old_module_t[0]) else: new_module_desc = reg.get_descriptor(new_module_type) new_module_t = new_module_desc.spec_tuple() new_pkg_version = module_remap.output_version if (new_pkg_version is None or reg.get_package_by_name(new_module_t[0]).version == new_pkg_version): # upgrading to the current version try: new_module_desc = reg.get_descriptor_by_name(*new_module_t) except MissingModule, e: # if the replacement is an abstraction, # and it has been upgraded, we use that if reg.has_abs_upgrade(*new_module_t): new_module_desc = reg.get_abs_upgrade(*new_module_t) else: raise e use_registry = True next_module_remap = None else: new_module_desc = ModuleDescriptor(package=new_module_t[0], name=new_module_t[1], namespace=new_module_t[2], version=new_pkg_version) use_registry = False # need to try more upgrades since this one isn't current old_desc_str = create_descriptor_string(new_module_t[0], new_module_t[1], new_module_t[2], False) old_version = new_pkg_version next_module_remap = pkg_remap.get_module_upgrade(old_desc_str, old_version) old_module_t = new_module_t replace_module = UpgradeWorkflowHandler.replace_module actions = replace_module(controller, tmp_pipeline, module_id, new_module_desc, module_remap.function_remap, module_remap.src_port_remap, module_remap.dst_port_remap, module_remap.annotation_remap, module_remap.control_param_remap, use_registry) for a in actions: for op in a.operations: # Update the id of the module being updated if op.vtType == 'add' and op.what == 'module': module_id = op.objectId tmp_pipeline.perform_action(a) action_list.extend(actions) module_remap = next_module_remap
def _get_sigstring(self): return create_descriptor_string(self.db_package, self.db_name, self.db_namespace)
def remap_module(controller, module_id, pipeline, module_remap): """remap_module offers a method to shortcut the specification of upgrades. It is useful when just changing the names of ports or modules, but can also be used to add intermediate modules or change the format of parameters. It is usually called from handle_module_upgrade_request, and the first three arguments are passed from the arguments to that method. module_remap specifies all of the changes and is of the format {<old_module_name>: [(<start_version>, <end_version>, <new_module_klass> | <new_module_id> | None, <remap_dictionary>)]} where new_module_klass is the class and new_module_id is a string of the format <package_name>:[<namespace> | ]<module_name> passing None keeps the original name, and remap_dictionary is {<remap_type>: <name_changes>} and <name_changes> is a map from <old_name> to <new_name> or <remap_function> The remap functions are passed the old object and the new module and should return a list of operations with elements of the form ('add', <obj>). For example: def outputName_remap(old_conn, new_module): ops = [] ... return ops module_remap = {'FileSink': [(None, '1.5.1', FileSink, {'dst_port_remap': {'overrideFile': 'overwrite', 'outputName': outputName_remap}, 'function_remap': {'overrideFile': 'overwrite', 'outputName': 'outputPath'}}), } """ reg = get_module_registry() old_module = pipeline.modules[module_id] old_desc_str = create_descriptor_string(old_module.package, old_module.name, old_module.namespace, False) # print 'running module_upgrade_request', old_module.name if old_desc_str in module_remap: for upgrade_tuple in module_remap[old_desc_str]: (start_version, end_version, new_module_type, remap) = \ upgrade_tuple old_version = old_module.version if ((start_version is None or not versions_increasing(old_version, start_version)) and (end_version is None or versions_increasing(old_version, end_version))): # do upgrade if new_module_type is None: try: new_module_desc = \ reg.get_descriptor_by_name(old_module.package, old_module.name, old_module.namespace) except MissingModule, e: # if the replacement is an abstraction, # and it has been upgraded, we use that if reg.has_abs_upgrade(old_module.package, old_module.name, old_module.namespace): new_module_desc = \ reg.get_abs_upgrade(old_module.package, old_module.name, old_module.namespace) else: raise e elif isinstance(new_module_type, basestring): d_tuple = parse_descriptor_string(new_module_type, old_module.package) try: new_module_desc = \ reg.get_descriptor_by_name(*d_tuple) except MissingModule, e: # if the replacement is an abstraction, # and it has been upgraded, we use that if reg.has_abs_upgrade(*d_tuple): new_module_desc = reg.get_abs_upgrade(*d_tuple) else: raise e else: # we have a klass for get_descriptor new_module_desc = reg.get_descriptor(new_module_type) src_port_remap = remap.get('src_port_remap', {}) dst_port_remap = remap.get('dst_port_remap', {}) # !!! we're going to let dst_port_remap serve as a # base for function_remap but the developer is # responsible for knowing that anything beyond name # remaps requires different functions function_remap = copy.copy(dst_port_remap) function_remap.update(remap.get('function_remap', {})) annotation_remap = remap.get('annotation_remap', {}) action_list = \ UpgradeWorkflowHandler.replace_module(controller, pipeline, module_id, new_module_desc, function_remap, src_port_remap, dst_port_remap, annotation_remap) return action_list
def _read_metadata(self, package_identifier): """Reads a plot's ports from the subworkflow file Finds each InputPort module and gets the parameter name, optional flag and type from its 'name', 'optional' and 'spec' input functions. If input ports were declared in this Plot, we check that they are indeed present and were all listed (either list all of them or none). If the module type is a subclass of Constant, we will assume the port is to be set via direct input (ConstantPort), else by dragging a variable (DataPort). We also automatically add aliased input ports of compatible constant types as optional ConstantPort's. """ pipeline = self.get_pipeline() inputports = find_modules_by_type(pipeline, [InputPort]) if not inputports: raise ValueError("No InputPort module") currentports = {port.name: port for port in self.ports} seenports = set() for port in inputports: name = get_function(port, 'name') if not name: raise ValueError( "Subworkflow of plot '%s' in package '%s' has an " "InputPort with no name" % ( self.name, package_identifier)) if name in seenports: raise ValueError( "Subworkflow of plot '%s' in package '%s' has several " "InputPort modules with name '%s'" % ( self.name, package_identifier, name)) spec = get_function(port, 'spec') optional = get_function(port, 'optional') if optional == 'True': optional = True elif optional == 'False': optional = False else: optional = None try: currentport = currentports[name] except KeyError: # If the package didn't provide any port, it's ok, we can # discover them. But if some were present and some were # forgotten, emit a warning if currentports: warnings.warn( "Declaration of plot '%s' in package '%s' omitted " "port '%s'" % ( self.name, package_identifier, name)) if not spec: warnings.warn( "Subworkflow of plot '%s' in package '%s' has an " "InputPort '%s' with no type; assuming Module" % ( self.name, package_identifier, name)) spec = 'org.vistrails.vistrails.basic:Module' if not optional: optional = False type = resolve_descriptor(spec, package_identifier) if issubclass(type.module, Constant): currentport = ConstantPort( name=name, type=type, optional=optional) else: currentport = DataPort( name=name, type=type, optional=optional) self.ports.append(currentport) else: currentspec = (currentport.type.identifier + ':' + currentport.type.name) # Canonicalize spec spec = create_descriptor_string(*parse_descriptor_string( spec, package_identifier)) if ((spec and spec != currentspec) or (optional is not None and optional != currentport.optional)): warnings.warn( "Declaration of port '%s' from plot '%s' in " "package '%s' differs from subworkflow " "contents" % ( name, self.name, package_identifier)) spec = currentspec type = resolve_descriptor(currentspec, package_identifier) # Get info from the PortSpec try: (default_type, default_value, entry_type, enum_values) = read_port_specs( pipeline, port) if default_value is not None: if not issubclass(default_type, type.module): raise ValueError("incompatible type %r" % (( default_type, type.module),)) elif default_type is type.module: currentport.default_value = default_value currentport.entry_type = entry_type currentport.enum_values = enum_values except ValueError, e: raise ValueError( "Error reading specs for port '%s' from plot '%s' of " "package '%s': %s" % ( name, self.name, package_identifier, e.args[0])) seenports.add(name)
def remap_module(controller, module_id, pipeline, pkg_remap): """remap_module offers a method to shortcut the specification of upgrades. It is useful when just changing the names of ports or modules, but can also be used to add intermediate modules or change the format of parameters. It is usually called from handle_module_upgrade_request, and the first three arguments are passed from the arguments to that method. pkg_remap specifies all of the changes and is of the format {<old_module_name>: [(<start_version>, <end_version>, <new_module_klass> | <new_module_id> | None, <remap_dictionary>)]} where new_module_klass is the class and new_module_id is a string of the format <package_name>:[<namespace> | ]<module_name> passing None keeps the original name, and remap_dictionary is {<remap_type>: <name_changes>} and <name_changes> is a map from <old_name> to <new_name> or <remap_function> The remap functions are passed the old object and the new module and should return a list of operations with elements of the form ('add', <obj>). For example: def outputName_remap(old_conn, new_module): ops = [] ... return ops pkg_remap = {'FileSink': [(None, '1.5.1', FileSink, {'dst_port_remap': {'overrideFile': 'overwrite', 'outputName': outputName_remap}, 'function_remap': {'overrideFile': 'overwrite', 'outputName': 'outputPath'}}), } """ reg = get_module_registry() old_module = pipeline.modules[module_id] old_version = old_module.version old_desc_str = create_descriptor_string(old_module.package, old_module.name, old_module.namespace, False) # print 'running module_upgrade_request', old_module.name if not isinstance(pkg_remap, UpgradePackageRemap): pkg_remap = UpgradePackageRemap.from_dict(pkg_remap) action_list = [] old_module_t = \ (old_module.package, old_module.name, old_module.namespace) module_remap = pkg_remap.get_module_upgrade(old_desc_str, old_version) tmp_pipeline = copy.copy(pipeline) while module_remap is not None: new_module_type = module_remap.new_module if new_module_type is None: new_module_t = old_module_t elif isinstance(new_module_type, basestring): new_module_t = parse_descriptor_string(new_module_type, old_module_t[0]) elif isinstance(new_module_type, ModuleDescriptor): new_module_t = new_module_type.spec_tuple else: new_module_desc = reg.get_descriptor(new_module_type) new_module_t = new_module_desc.spec_tuple new_pkg_version = module_remap.output_version if (new_pkg_version is None or reg.get_package_by_name( new_module_t[0]).version == new_pkg_version): # upgrading to the current version try: new_module_desc = reg.get_descriptor_by_name(*new_module_t) except MissingModule, e: # if the replacement is an abstraction, # and it has been upgraded, we use that if reg.has_abs_upgrade(*new_module_t): new_module_desc = reg.get_abs_upgrade(*new_module_t) else: raise e use_registry = True next_module_remap = None else: new_module_desc = ModuleDescriptor( package=new_module_t[0], name=new_module_t[1], namespace=new_module_t[2], package_version=new_pkg_version) use_registry = False # need to try more upgrades since this one isn't current old_desc_str = create_descriptor_string( new_module_t[0], new_module_t[1], new_module_t[2], False) old_version = new_pkg_version next_module_remap = pkg_remap.get_module_upgrade( old_desc_str, old_version) old_module_t = new_module_t replace_module = UpgradeWorkflowHandler.replace_module actions = replace_module( controller, tmp_pipeline, module_id, new_module_desc, module_remap.function_remap, module_remap.src_port_remap, module_remap.dst_port_remap, module_remap.annotation_remap, module_remap.control_param_remap, use_registry) for a in actions: for op in a.operations: # Update the id of the module being updated # FIXME: This is brittle # This assumes first added module is the correct one if op.vtType == 'add' and op.what == 'module': module_id = op.objectId break tmp_pipeline.perform_action(a) action_list.extend(actions) module_remap = next_module_remap