def resolve_descriptor(param, package_identifier=None): """Resolve a type specifier to a ModuleDescriptor. This accepts different arguments and turns it into a ModuleDescriptor: * a ModuleDescriptor * a Module object * a descriptor string It should be used when accepting type specifiers from third-party code. The optional 'package_identifier' parameter gives the context in which to resolve module names; it is passed to parse_descriptor_string(). """ reg = get_module_registry() if isinstance(param, str): d_tuple = parse_descriptor_string(param, package_identifier) return reg.get_descriptor_by_name(*d_tuple) elif isinstance(param, type) and issubclass(param, Module): return reg.get_descriptor(param) elif isinstance(param, ModuleDescriptor): return param else: raise TypeError("resolve_descriptor() argument must be a Module " "subclass or str object, not '%s'" % type(param))
def remap_module(controller, module_id, pipeline, pkg_remap): """remap_module offers a method to shortcut the specification of upgrades. It is useful when just changing the names of ports or modules, but can also be used to add intermediate modules or change the format of parameters. It is usually called from handle_module_upgrade_request, and the first three arguments are passed from the arguments to that method. pkg_remap specifies all of the changes and is of the format {<old_module_name>: [(<start_version>, <end_version>, <new_module_klass> | <new_module_id> | None, <remap_dictionary>)]} where new_module_klass is the class and new_module_id is a string of the format <package_name>:[<namespace> | ]<module_name> passing None keeps the original name, and remap_dictionary is {<remap_type>: <name_changes>} and <name_changes> is a map from <old_name> to <new_name> or <remap_function> The remap functions are passed the old object and the new module and should return a list of operations with elements of the form ('add', <obj>). For example: def outputName_remap(old_conn, new_module): ops = [] ... return ops pkg_remap = {'FileSink': [(None, '1.5.1', FileSink, {'dst_port_remap': {'overrideFile': 'overwrite', 'outputName': outputName_remap}, 'function_remap': {'overrideFile': 'overwrite', 'outputName': 'outputPath'}}), } """ reg = get_module_registry() old_module = pipeline.modules[module_id] old_version = old_module.version old_desc_str = create_descriptor_string(old_module.package, old_module.name, old_module.namespace, False) # print 'running module_upgrade_request', old_module.name if not isinstance(pkg_remap, UpgradePackageRemap): pkg_remap = UpgradePackageRemap.from_dict(pkg_remap) action_list = [] old_module_t = \ (old_module.package, old_module.name, old_module.namespace) module_remap = pkg_remap.get_module_upgrade(old_desc_str, old_version) tmp_pipeline = copy.copy(pipeline) while module_remap is not None: new_module_type = module_remap.new_module if new_module_type is None: new_module_t = old_module_t elif isinstance(new_module_type, basestring): new_module_t = parse_descriptor_string(new_module_type, old_module_t[0]) else: new_module_desc = reg.get_descriptor(new_module_type) new_module_t = new_module_desc.spec_tuple() new_pkg_version = module_remap.output_version if (new_pkg_version is None or reg.get_package_by_name(new_module_t[0]).version == new_pkg_version): # upgrading to the current version try: new_module_desc = reg.get_descriptor_by_name(*new_module_t) except MissingModule, e: # if the replacement is an abstraction, # and it has been upgraded, we use that if reg.has_abs_upgrade(*new_module_t): new_module_desc = reg.get_abs_upgrade(*new_module_t) else: raise e use_registry = True next_module_remap = None else: new_module_desc = ModuleDescriptor(package=new_module_t[0], name=new_module_t[1], namespace=new_module_t[2], version=new_pkg_version) use_registry = False # need to try more upgrades since this one isn't current old_desc_str = create_descriptor_string(new_module_t[0], new_module_t[1], new_module_t[2], False) old_version = new_pkg_version next_module_remap = pkg_remap.get_module_upgrade(old_desc_str, old_version) old_module_t = new_module_t replace_module = UpgradeWorkflowHandler.replace_module actions = replace_module(controller, tmp_pipeline, module_id, new_module_desc, module_remap.function_remap, module_remap.src_port_remap, module_remap.dst_port_remap, module_remap.annotation_remap, module_remap.control_param_remap, use_registry) for a in actions: for op in a.operations: # Update the id of the module being updated if op.vtType == 'add' and op.what == 'module': module_id = op.objectId tmp_pipeline.perform_action(a) action_list.extend(actions) module_remap = next_module_remap
def remap_module(controller, module_id, pipeline, module_remap): """remap_module offers a method to shortcut the specification of upgrades. It is useful when just changing the names of ports or modules, but can also be used to add intermediate modules or change the format of parameters. It is usually called from handle_module_upgrade_request, and the first three arguments are passed from the arguments to that method. module_remap specifies all of the changes and is of the format {<old_module_name>: [(<start_version>, <end_version>, <new_module_klass> | <new_module_id> | None, <remap_dictionary>)]} where new_module_klass is the class and new_module_id is a string of the format <package_name>:[<namespace> | ]<module_name> passing None keeps the original name, and remap_dictionary is {<remap_type>: <name_changes>} and <name_changes> is a map from <old_name> to <new_name> or <remap_function> The remap functions are passed the old object and the new module and should return a list of operations with elements of the form ('add', <obj>). For example: def outputName_remap(old_conn, new_module): ops = [] ... return ops module_remap = {'FileSink': [(None, '1.5.1', FileSink, {'dst_port_remap': {'overrideFile': 'overwrite', 'outputName': outputName_remap}, 'function_remap': {'overrideFile': 'overwrite', 'outputName': 'outputPath'}}), } """ reg = get_module_registry() old_module = pipeline.modules[module_id] old_desc_str = create_descriptor_string(old_module.package, old_module.name, old_module.namespace, False) # print 'running module_upgrade_request', old_module.name if old_desc_str in module_remap: for upgrade_tuple in module_remap[old_desc_str]: (start_version, end_version, new_module_type, remap) = \ upgrade_tuple old_version = old_module.version if ((start_version is None or not versions_increasing(old_version, start_version)) and (end_version is None or versions_increasing(old_version, end_version))): # do upgrade if new_module_type is None: try: new_module_desc = \ reg.get_descriptor_by_name(old_module.package, old_module.name, old_module.namespace) except MissingModule, e: # if the replacement is an abstraction, # and it has been upgraded, we use that if reg.has_abs_upgrade(old_module.package, old_module.name, old_module.namespace): new_module_desc = \ reg.get_abs_upgrade(old_module.package, old_module.name, old_module.namespace) else: raise e elif isinstance(new_module_type, basestring): d_tuple = parse_descriptor_string(new_module_type, old_module.package) try: new_module_desc = \ reg.get_descriptor_by_name(*d_tuple) except MissingModule, e: # if the replacement is an abstraction, # and it has been upgraded, we use that if reg.has_abs_upgrade(*d_tuple): new_module_desc = reg.get_abs_upgrade(*d_tuple) else: raise e else: # we have a klass for get_descriptor new_module_desc = reg.get_descriptor(new_module_type) src_port_remap = remap.get('src_port_remap', {}) dst_port_remap = remap.get('dst_port_remap', {}) # !!! we're going to let dst_port_remap serve as a # base for function_remap but the developer is # responsible for knowing that anything beyond name # remaps requires different functions function_remap = copy.copy(dst_port_remap) function_remap.update(remap.get('function_remap', {})) annotation_remap = remap.get('annotation_remap', {}) action_list = \ UpgradeWorkflowHandler.replace_module(controller, pipeline, module_id, new_module_desc, function_remap, src_port_remap, dst_port_remap, annotation_remap) return action_list
def _read_metadata(self, package_identifier): """Reads a plot's ports from the subworkflow file Finds each InputPort module and gets the parameter name, optional flag and type from its 'name', 'optional' and 'spec' input functions. If input ports were declared in this Plot, we check that they are indeed present and were all listed (either list all of them or none). If the module type is a subclass of Constant, we will assume the port is to be set via direct input (ConstantPort), else by dragging a variable (DataPort). We also automatically add aliased input ports of compatible constant types as optional ConstantPort's. """ pipeline = self.get_pipeline() inputports = find_modules_by_type(pipeline, [InputPort]) if not inputports: raise ValueError("No InputPort module") currentports = {port.name: port for port in self.ports} seenports = set() for port in inputports: name = get_function(port, 'name') if not name: raise ValueError( "Subworkflow of plot '%s' in package '%s' has an " "InputPort with no name" % ( self.name, package_identifier)) if name in seenports: raise ValueError( "Subworkflow of plot '%s' in package '%s' has several " "InputPort modules with name '%s'" % ( self.name, package_identifier, name)) spec = get_function(port, 'spec') optional = get_function(port, 'optional') if optional == 'True': optional = True elif optional == 'False': optional = False else: optional = None try: currentport = currentports[name] except KeyError: # If the package didn't provide any port, it's ok, we can # discover them. But if some were present and some were # forgotten, emit a warning if currentports: warnings.warn( "Declaration of plot '%s' in package '%s' omitted " "port '%s'" % ( self.name, package_identifier, name)) if not spec: warnings.warn( "Subworkflow of plot '%s' in package '%s' has an " "InputPort '%s' with no type; assuming Module" % ( self.name, package_identifier, name)) spec = 'org.vistrails.vistrails.basic:Module' if not optional: optional = False type = resolve_descriptor(spec, package_identifier) if issubclass(type.module, Constant): currentport = ConstantPort( name=name, type=type, optional=optional) else: currentport = DataPort( name=name, type=type, optional=optional) self.ports.append(currentport) else: currentspec = (currentport.type.identifier + ':' + currentport.type.name) # Canonicalize spec spec = create_descriptor_string(*parse_descriptor_string( spec, package_identifier)) if ((spec and spec != currentspec) or (optional is not None and optional != currentport.optional)): warnings.warn( "Declaration of port '%s' from plot '%s' in " "package '%s' differs from subworkflow " "contents" % ( name, self.name, package_identifier)) spec = currentspec type = resolve_descriptor(currentspec, package_identifier) # Get info from the PortSpec try: (default_type, default_value, entry_type, enum_values) = read_port_specs( pipeline, port) if default_value is not None: if not issubclass(default_type, type.module): raise ValueError("incompatible type %r" % (( default_type, type.module),)) elif default_type is type.module: currentport.default_value = default_value currentport.entry_type = entry_type currentport.enum_values = enum_values except ValueError, e: raise ValueError( "Error reading specs for port '%s' from plot '%s' of " "package '%s': %s" % ( name, self.name, package_identifier, e.args[0])) seenports.add(name)
def remap_module(controller, module_id, pipeline, pkg_remap): """remap_module offers a method to shortcut the specification of upgrades. It is useful when just changing the names of ports or modules, but can also be used to add intermediate modules or change the format of parameters. It is usually called from handle_module_upgrade_request, and the first three arguments are passed from the arguments to that method. pkg_remap specifies all of the changes and is of the format {<old_module_name>: [(<start_version>, <end_version>, <new_module_klass> | <new_module_id> | None, <remap_dictionary>)]} where new_module_klass is the class and new_module_id is a string of the format <package_name>:[<namespace> | ]<module_name> passing None keeps the original name, and remap_dictionary is {<remap_type>: <name_changes>} and <name_changes> is a map from <old_name> to <new_name> or <remap_function> The remap functions are passed the old object and the new module and should return a list of operations with elements of the form ('add', <obj>). For example: def outputName_remap(old_conn, new_module): ops = [] ... return ops pkg_remap = {'FileSink': [(None, '1.5.1', FileSink, {'dst_port_remap': {'overrideFile': 'overwrite', 'outputName': outputName_remap}, 'function_remap': {'overrideFile': 'overwrite', 'outputName': 'outputPath'}}), } """ reg = get_module_registry() old_module = pipeline.modules[module_id] old_version = old_module.version old_desc_str = create_descriptor_string(old_module.package, old_module.name, old_module.namespace, False) # print 'running module_upgrade_request', old_module.name if not isinstance(pkg_remap, UpgradePackageRemap): pkg_remap = UpgradePackageRemap.from_dict(pkg_remap) action_list = [] old_module_t = \ (old_module.package, old_module.name, old_module.namespace) module_remap = pkg_remap.get_module_upgrade(old_desc_str, old_version) tmp_pipeline = copy.copy(pipeline) while module_remap is not None: new_module_type = module_remap.new_module if new_module_type is None: new_module_t = old_module_t elif isinstance(new_module_type, basestring): new_module_t = parse_descriptor_string(new_module_type, old_module_t[0]) elif isinstance(new_module_type, ModuleDescriptor): new_module_t = new_module_type.spec_tuple else: new_module_desc = reg.get_descriptor(new_module_type) new_module_t = new_module_desc.spec_tuple new_pkg_version = module_remap.output_version if (new_pkg_version is None or reg.get_package_by_name( new_module_t[0]).version == new_pkg_version): # upgrading to the current version try: new_module_desc = reg.get_descriptor_by_name(*new_module_t) except MissingModule, e: # if the replacement is an abstraction, # and it has been upgraded, we use that if reg.has_abs_upgrade(*new_module_t): new_module_desc = reg.get_abs_upgrade(*new_module_t) else: raise e use_registry = True next_module_remap = None else: new_module_desc = ModuleDescriptor( package=new_module_t[0], name=new_module_t[1], namespace=new_module_t[2], package_version=new_pkg_version) use_registry = False # need to try more upgrades since this one isn't current old_desc_str = create_descriptor_string( new_module_t[0], new_module_t[1], new_module_t[2], False) old_version = new_pkg_version next_module_remap = pkg_remap.get_module_upgrade( old_desc_str, old_version) old_module_t = new_module_t replace_module = UpgradeWorkflowHandler.replace_module actions = replace_module( controller, tmp_pipeline, module_id, new_module_desc, module_remap.function_remap, module_remap.src_port_remap, module_remap.dst_port_remap, module_remap.annotation_remap, module_remap.control_param_remap, use_registry) for a in actions: for op in a.operations: # Update the id of the module being updated # FIXME: This is brittle # This assumes first added module is the correct one if op.vtType == 'add' and op.what == 'module': module_id = op.objectId break tmp_pipeline.perform_action(a) action_list.extend(actions) module_remap = next_module_remap