def upgrade_cell_to_output(module_remap,
module_id,
pipeline,
old_name,
new_module,
end_version,
input_port_name,
start_version=None,
output_version=None):
"""This function upgrades a *Cell module to a *Output module.
The upgrade only happens if the original module doesn't have any connection
on the cell input ports that can't be translated.
This is to ease the transition to *Output modules, but we don't want (or
need) to break anything; the *Cell modules still exist, so they can stay.
"""
if not isinstance(module_remap, UpgradePackageRemap):
module_remap = UpgradePackageRemap.from_dict(module_remap)
old_module = pipeline.modules[module_id]
old_module_name = create_descriptor_string(old_module.package,
old_module.name,
old_module.namespace, False)
if old_module_name != old_name:
return module_remap
used_input_ports = set(old_module.connected_input_ports.keys())
for func in old_module.functions:
used_input_ports.add(func.name)
if used_input_ports != set([input_port_name]):
return module_remap
_old_remap = module_remap
module_remap = copy.copy(module_remap)
assert _old_remap.remaps is not module_remap.remaps
remap = UpgradeModuleRemap(start_version,
end_version,
output_version,
module_name=old_name,
new_module=new_module)
remap.add_remap('dst_port_remap', input_port_name, 'value')
remap.add_remap('function_remap', input_port_name, 'value')
module_remap.add_module_remap(remap)
return module_remap
def upgrade_cell_to_output(module_remap, module_id, pipeline,
old_name, new_module,
end_version, input_port_name,
start_version=None, output_version=None):
"""This function upgrades a *Cell module to a *Output module.
The upgrade only happens if the original module doesn't have any connection
on the cell input ports that can't be translated.
This is to ease the transition to *Output modules, but we don't want (or
need) to break anything; the *Cell modules still exist, so they can stay.
"""
if not isinstance(module_remap, UpgradePackageRemap):
module_remap = UpgradePackageRemap.from_dict(module_remap)
old_module = pipeline.modules[module_id]
old_module_name = create_descriptor_string(old_module.package,
old_module.name,
old_module.namespace,
False)
if old_module_name != old_name:
return module_remap
used_input_ports = set(old_module.connected_input_ports.keys())
for func in old_module.functions:
used_input_ports.add(func.name)
if used_input_ports != set([input_port_name]):
return module_remap
_old_remap = module_remap
module_remap = copy.copy(module_remap)
assert _old_remap.remaps is not module_remap.remaps
remap = UpgradeModuleRemap(start_version, end_version, output_version,
module_name=old_name,
new_module=new_module)
remap.add_remap('dst_port_remap', input_port_name, 'value')
remap.add_remap('function_remap', input_port_name, 'value')
module_remap.add_module_remap(remap)
return module_remap
def remap_module(controller, module_id, pipeline, pkg_remap):
"""remap_module offers a method to shortcut the
specification of upgrades. It is useful when just changing
the names of ports or modules, but can also be used to add
intermediate modules or change the format of parameters. It
is usually called from handle_module_upgrade_request, and the
first three arguments are passed from the arguments to that
method.
pkg_remap specifies all of the changes and is of the format
{<old_module_name>: [(<start_version>, <end_version>,
<new_module_klass> | <new_module_id> | None,
<remap_dictionary>)]}
where new_module_klass is the class and new_module_id
is a string of the format
<package_name>:[<namespace> | ]<module_name>
passing None keeps the original name,
and remap_dictionary is {<remap_type>:
<name_changes>} and <name_changes> is a map from <old_name> to
<new_name> or <remap_function>
The remap functions are passed the old object and the new
module and should return a list of operations with elements of
the form ('add', <obj>).
For example:
def outputName_remap(old_conn, new_module):
ops = []
...
return ops
pkg_remap = {'FileSink': [(None, '1.5.1', FileSink,
{'dst_port_remap':
{'overrideFile': 'overwrite',
'outputName': outputName_remap},
'function_remap':
{'overrideFile': 'overwrite',
'outputName': 'outputPath'}}),
}
"""
reg = get_module_registry()
old_module = pipeline.modules[module_id]
old_version = old_module.version
old_desc_str = create_descriptor_string(old_module.package,
old_module.name,
old_module.namespace,
False)
# print 'running module_upgrade_request', old_module.name
if not isinstance(pkg_remap, UpgradePackageRemap):
pkg_remap = UpgradePackageRemap.from_dict(pkg_remap)
action_list = []
old_module_t = \
(old_module.package, old_module.name, old_module.namespace)
module_remap = pkg_remap.get_module_upgrade(old_desc_str, old_version)
tmp_pipeline = copy.copy(pipeline)
while module_remap is not None:
new_module_type = module_remap.new_module
if new_module_type is None:
new_module_t = old_module_t
elif isinstance(new_module_type, basestring):
new_module_t = parse_descriptor_string(new_module_type,
old_module_t[0])
else:
new_module_desc = reg.get_descriptor(new_module_type)
new_module_t = new_module_desc.spec_tuple()
new_pkg_version = module_remap.output_version
if (new_pkg_version is None or
reg.get_package_by_name(new_module_t[0]).version == new_pkg_version):
# upgrading to the current version
try:
new_module_desc = reg.get_descriptor_by_name(*new_module_t)
except MissingModule, e:
# if the replacement is an abstraction,
# and it has been upgraded, we use that
if reg.has_abs_upgrade(*new_module_t):
new_module_desc = reg.get_abs_upgrade(*new_module_t)
else:
raise e
use_registry = True
next_module_remap = None
else:
new_module_desc = ModuleDescriptor(package=new_module_t[0],
name=new_module_t[1],
namespace=new_module_t[2],
version=new_pkg_version)
use_registry = False
# need to try more upgrades since this one isn't current
old_desc_str = create_descriptor_string(new_module_t[0],
new_module_t[1],
new_module_t[2],
False)
old_version = new_pkg_version
next_module_remap = pkg_remap.get_module_upgrade(old_desc_str,
old_version)
old_module_t = new_module_t
replace_module = UpgradeWorkflowHandler.replace_module
actions = replace_module(controller,
tmp_pipeline,
module_id,
new_module_desc,
module_remap.function_remap,
module_remap.src_port_remap,
module_remap.dst_port_remap,
module_remap.annotation_remap,
module_remap.control_param_remap,
use_registry)
for a in actions:
for op in a.operations:
# Update the id of the module being updated
if op.vtType == 'add' and op.what == 'module':
module_id = op.objectId
tmp_pipeline.perform_action(a)
action_list.extend(actions)
module_remap = next_module_remap
def _get_sigstring(self):
return create_descriptor_string(self.db_package, self.db_name,
self.db_namespace)
def remap_module(controller, module_id, pipeline, module_remap):
"""remap_module offers a method to shortcut the
specification of upgrades. It is useful when just changing
the names of ports or modules, but can also be used to add
intermediate modules or change the format of parameters. It
is usually called from handle_module_upgrade_request, and the
first three arguments are passed from the arguments to that
method.
module_remap specifies all of the changes and is of the format
{<old_module_name>: [(<start_version>, <end_version>,
<new_module_klass> | <new_module_id> | None,
<remap_dictionary>)]}
where new_module_klass is the class and new_module_id
is a string of the format
<package_name>:[<namespace> | ]<module_name>
passing None keeps the original name,
and remap_dictionary is {<remap_type>:
<name_changes>} and <name_changes> is a map from <old_name> to
<new_name> or <remap_function>
The remap functions are passed the old object and the new
module and should return a list of operations with elements of
the form ('add', <obj>).
For example:
def outputName_remap(old_conn, new_module):
ops = []
...
return ops
module_remap = {'FileSink': [(None, '1.5.1', FileSink,
{'dst_port_remap':
{'overrideFile': 'overwrite',
'outputName': outputName_remap},
'function_remap':
{'overrideFile': 'overwrite',
'outputName': 'outputPath'}}),
}
"""
reg = get_module_registry()
old_module = pipeline.modules[module_id]
old_desc_str = create_descriptor_string(old_module.package,
old_module.name,
old_module.namespace,
False)
# print 'running module_upgrade_request', old_module.name
if old_desc_str in module_remap:
for upgrade_tuple in module_remap[old_desc_str]:
(start_version, end_version, new_module_type, remap) = \
upgrade_tuple
old_version = old_module.version
if ((start_version is None or
not versions_increasing(old_version, start_version)) and
(end_version is None or
versions_increasing(old_version, end_version))):
# do upgrade
if new_module_type is None:
try:
new_module_desc = \
reg.get_descriptor_by_name(old_module.package,
old_module.name,
old_module.namespace)
except MissingModule, e:
# if the replacement is an abstraction,
# and it has been upgraded, we use that
if reg.has_abs_upgrade(old_module.package,
old_module.name,
old_module.namespace):
new_module_desc = \
reg.get_abs_upgrade(old_module.package,
old_module.name,
old_module.namespace)
else:
raise e
elif isinstance(new_module_type, basestring):
d_tuple = parse_descriptor_string(new_module_type,
old_module.package)
try:
new_module_desc = \
reg.get_descriptor_by_name(*d_tuple)
except MissingModule, e:
# if the replacement is an abstraction,
# and it has been upgraded, we use that
if reg.has_abs_upgrade(*d_tuple):
new_module_desc = reg.get_abs_upgrade(*d_tuple)
else:
raise e
else: # we have a klass for get_descriptor
new_module_desc = reg.get_descriptor(new_module_type)
src_port_remap = remap.get('src_port_remap', {})
dst_port_remap = remap.get('dst_port_remap', {})
# !!! we're going to let dst_port_remap serve as a
# base for function_remap but the developer is
# responsible for knowing that anything beyond name
# remaps requires different functions
function_remap = copy.copy(dst_port_remap)
function_remap.update(remap.get('function_remap', {}))
annotation_remap = remap.get('annotation_remap', {})
action_list = \
UpgradeWorkflowHandler.replace_module(controller,
pipeline,
module_id,
new_module_desc,
function_remap,
src_port_remap,
dst_port_remap,
annotation_remap)
return action_list
def _get_sigstring(self):
return create_descriptor_string(self.db_package, self.db_name,
self.db_namespace)
def _read_metadata(self, package_identifier):
"""Reads a plot's ports from the subworkflow file
Finds each InputPort module and gets the parameter name, optional flag
and type from its 'name', 'optional' and 'spec' input functions.
If input ports were declared in this Plot, we check that they are
indeed present and were all listed (either list all of them or none).
If the module type is a subclass of Constant, we will assume the port
is to be set via direct input (ConstantPort), else by dragging a
variable (DataPort).
We also automatically add aliased input ports of compatible constant
types as optional ConstantPort's.
"""
pipeline = self.get_pipeline()
inputports = find_modules_by_type(pipeline, [InputPort])
if not inputports:
raise ValueError("No InputPort module")
currentports = {port.name: port for port in self.ports}
seenports = set()
for port in inputports:
name = get_function(port, 'name')
if not name:
raise ValueError(
"Subworkflow of plot '%s' in package '%s' has an "
"InputPort with no name" % (
self.name, package_identifier))
if name in seenports:
raise ValueError(
"Subworkflow of plot '%s' in package '%s' has several "
"InputPort modules with name '%s'" % (
self.name, package_identifier, name))
spec = get_function(port, 'spec')
optional = get_function(port, 'optional')
if optional == 'True':
optional = True
elif optional == 'False':
optional = False
else:
optional = None
try:
currentport = currentports[name]
except KeyError:
# If the package didn't provide any port, it's ok, we can
# discover them. But if some were present and some were
# forgotten, emit a warning
if currentports:
warnings.warn(
"Declaration of plot '%s' in package '%s' omitted "
"port '%s'" % (
self.name, package_identifier, name))
if not spec:
warnings.warn(
"Subworkflow of plot '%s' in package '%s' has an "
"InputPort '%s' with no type; assuming Module" % (
self.name, package_identifier, name))
spec = 'org.vistrails.vistrails.basic:Module'
if not optional:
optional = False
type = resolve_descriptor(spec, package_identifier)
if issubclass(type.module, Constant):
currentport = ConstantPort(
name=name,
type=type,
optional=optional)
else:
currentport = DataPort(
name=name,
type=type,
optional=optional)
self.ports.append(currentport)
else:
currentspec = (currentport.type.identifier +
':' +
currentport.type.name)
# Canonicalize spec
spec = create_descriptor_string(*parse_descriptor_string(
spec, package_identifier))
if ((spec and spec != currentspec) or
(optional is not None and
optional != currentport.optional)):
warnings.warn(
"Declaration of port '%s' from plot '%s' in "
"package '%s' differs from subworkflow "
"contents" % (
name, self.name, package_identifier))
spec = currentspec
type = resolve_descriptor(currentspec, package_identifier)
# Get info from the PortSpec
try:
(default_type, default_value,
entry_type, enum_values) = read_port_specs(
pipeline,
port)
if default_value is not None:
if not issubclass(default_type, type.module):
raise ValueError("incompatible type %r" % ((
default_type,
type.module),))
elif default_type is type.module:
currentport.default_value = default_value
currentport.entry_type = entry_type
currentport.enum_values = enum_values
except ValueError, e:
raise ValueError(
"Error reading specs for port '%s' from plot '%s' of "
"package '%s': %s" % (
name, self.name, package_identifier, e.args[0]))
seenports.add(name)
def remap_module(controller, module_id, pipeline, pkg_remap):
"""remap_module offers a method to shortcut the
specification of upgrades. It is useful when just changing
the names of ports or modules, but can also be used to add
intermediate modules or change the format of parameters. It
is usually called from handle_module_upgrade_request, and the
first three arguments are passed from the arguments to that
method.
pkg_remap specifies all of the changes and is of the format
{<old_module_name>: [(<start_version>, <end_version>,
<new_module_klass> | <new_module_id> | None,
<remap_dictionary>)]}
where new_module_klass is the class and new_module_id
is a string of the format
<package_name>:[<namespace> | ]<module_name>
passing None keeps the original name,
and remap_dictionary is {<remap_type>:
<name_changes>} and <name_changes> is a map from <old_name> to
<new_name> or <remap_function>
The remap functions are passed the old object and the new
module and should return a list of operations with elements of
the form ('add', <obj>).
For example:
def outputName_remap(old_conn, new_module):
ops = []
...
return ops
pkg_remap = {'FileSink': [(None, '1.5.1', FileSink,
{'dst_port_remap':
{'overrideFile': 'overwrite',
'outputName': outputName_remap},
'function_remap':
{'overrideFile': 'overwrite',
'outputName': 'outputPath'}}),
}
"""
reg = get_module_registry()
old_module = pipeline.modules[module_id]
old_version = old_module.version
old_desc_str = create_descriptor_string(old_module.package,
old_module.name,
old_module.namespace, False)
# print 'running module_upgrade_request', old_module.name
if not isinstance(pkg_remap, UpgradePackageRemap):
pkg_remap = UpgradePackageRemap.from_dict(pkg_remap)
action_list = []
old_module_t = \
(old_module.package, old_module.name, old_module.namespace)
module_remap = pkg_remap.get_module_upgrade(old_desc_str, old_version)
tmp_pipeline = copy.copy(pipeline)
while module_remap is not None:
new_module_type = module_remap.new_module
if new_module_type is None:
new_module_t = old_module_t
elif isinstance(new_module_type, basestring):
new_module_t = parse_descriptor_string(new_module_type,
old_module_t[0])
elif isinstance(new_module_type, ModuleDescriptor):
new_module_t = new_module_type.spec_tuple
else:
new_module_desc = reg.get_descriptor(new_module_type)
new_module_t = new_module_desc.spec_tuple
new_pkg_version = module_remap.output_version
if (new_pkg_version is None or reg.get_package_by_name(
new_module_t[0]).version == new_pkg_version):
# upgrading to the current version
try:
new_module_desc = reg.get_descriptor_by_name(*new_module_t)
except MissingModule, e:
# if the replacement is an abstraction,
# and it has been upgraded, we use that
if reg.has_abs_upgrade(*new_module_t):
new_module_desc = reg.get_abs_upgrade(*new_module_t)
else:
raise e
use_registry = True
next_module_remap = None
else:
new_module_desc = ModuleDescriptor(
package=new_module_t[0],
name=new_module_t[1],
namespace=new_module_t[2],
package_version=new_pkg_version)
use_registry = False
# need to try more upgrades since this one isn't current
old_desc_str = create_descriptor_string(
new_module_t[0], new_module_t[1], new_module_t[2], False)
old_version = new_pkg_version
next_module_remap = pkg_remap.get_module_upgrade(
old_desc_str, old_version)
old_module_t = new_module_t
replace_module = UpgradeWorkflowHandler.replace_module
actions = replace_module(
controller, tmp_pipeline, module_id, new_module_desc,
module_remap.function_remap, module_remap.src_port_remap,
module_remap.dst_port_remap, module_remap.annotation_remap,
module_remap.control_param_remap, use_registry)
for a in actions:
for op in a.operations:
# Update the id of the module being updated
# FIXME: This is brittle
# This assumes first added module is the correct one
if op.vtType == 'add' and op.what == 'module':
module_id = op.objectId
break
tmp_pipeline.perform_action(a)
action_list.extend(actions)
module_remap = next_module_remap
