def convert(_module_exec):
if _module_exec.__class__ == ModuleExec:
return
_module_exec.__class__ = ModuleExec
for annotation in _module_exec.annotations:
Annotation.convert(annotation)
for loop_exec in _module_exec.loop_execs:
LoopExec.convert(loop_exec)
def convert(_abstraction):
if _abstraction.__class__ == Abstraction:
return
_abstraction.__class__ = Abstraction
if _abstraction.db_location:
Location.convert(_abstraction.db_location)
for _function in _abstraction.db_functions:
ModuleFunction.convert(_function)
for _annotation in _abstraction.db_get_annotations():
Annotation.convert(_annotation)
_abstraction.set_defaults()
def convert(_group_exec):
if _group_exec.__class__ == GroupExec:
return
_group_exec.__class__ = GroupExec
for annotation in _group_exec.annotations:
Annotation.convert(annotation)
for item_exec in _group_exec.item_execs:
if item_exec.vtType == ModuleExec.vtType:
ModuleExec.convert(item_exec)
elif item_exec.vtType == GroupExec.vtType:
GroupExec.convert(item_exec)
elif item_exec.vtType == LoopExec.vtType:
LoopExec.convert(item_exec)
def convert(_abstraction):
if _abstraction.__class__ == Abstraction:
return
_abstraction.__class__ = Abstraction
if _abstraction.db_location:
Location.convert(_abstraction.db_location)
for _function in _abstraction.db_functions:
ModuleFunction.convert(_function)
for _annotation in _abstraction.db_get_annotations():
Annotation.convert(_annotation)
for _control_parameter in _abstraction.db_get_controlParameters():
ModuleControlParam.convert(_control_parameter)
_abstraction.set_defaults()
def convert(_abstraction):
if _abstraction.__class__ == Abstraction:
return
_abstraction.__class__ = Abstraction
if _abstraction.db_location:
Location.convert(_abstraction.db_location)
for _function in _abstraction.db_functions:
ModuleFunction.convert(_function)
for _annotation in _abstraction.db_get_annotations():
Annotation.convert(_annotation)
for _control_parameter in _abstraction.db_get_controlParameters():
ModuleControlParam.convert(_control_parameter)
_abstraction.set_defaults()
def convert(_mtrail):
_mtrail.__class__ = Mashuptrail
for action in _mtrail.actions:
Action.convert(action)
for annotation in _mtrail.annotations:
Annotation.convert(annotation)
for aannotation in _mtrail.actionAnnotations:
ActionAnnotation.convert(aannotation)
_mtrail.id_scope = IdScope(1L)
_mtrail.updateIdScope()
def convert(_group_exec):
if _group_exec.__class__ == GroupExec:
return
_group_exec.__class__ = GroupExec
for annotation in _group_exec.annotations:
Annotation.convert(annotation)
for item_exec in _group_exec.item_execs:
if item_exec.vtType == ModuleExec.vtType:
ModuleExec.convert(item_exec)
elif item_exec.vtType == GroupExec.vtType:
GroupExec.convert(item_exec)
elif item_exec.vtType == LoopExec.vtType:
LoopExec.convert(item_exec)
def convert(_module):
if _module.__class__ == Module:
return
_module.__class__ = Module
for _port_spec in _module.db_portSpecs:
PortSpec.convert(_port_spec)
if _module.db_location:
Location.convert(_module.db_location)
for _function in _module.db_functions:
ModuleFunction.convert(_function)
for _annotation in _module.db_get_annotations():
Annotation.convert(_annotation)
_module.set_defaults()
def convert(_mtrail):
_mtrail.__class__ = Mashuptrail
for action in _mtrail.actions:
Action.convert(action)
for annotation in _mtrail.annotations:
Annotation.convert(annotation)
for aannotation in _mtrail.actionAnnotations:
ActionAnnotation.convert(aannotation)
_mtrail.id_scope = IdScope(1L)
_mtrail.updateIdScope()
def convert(_group):
if _group.__class__ == Group:
return
_group.__class__ = Group
if _group.db_location:
Location.convert(_group.db_location)
if _group.db_workflow:
from vistrails.core.vistrail.pipeline import Pipeline
Pipeline.convert(_group.db_workflow)
for _function in _group.db_functions:
ModuleFunction.convert(_function)
for _annotation in _group.db_get_annotations():
Annotation.convert(_annotation)
_group.set_defaults()
def convert(_module):
if _module.__class__ == Module:
return
_module.__class__ = Module
for _port_spec in _module.db_portSpecs:
PortSpec.convert(_port_spec)
if _module.db_location:
Location.convert(_module.db_location)
for _function in _module.db_functions:
ModuleFunction.convert(_function)
for _annotation in _module.db_get_annotations():
Annotation.convert(_annotation)
for _control_parameter in _module.db_get_controlParameters():
ModuleControlParam.convert(_control_parameter)
_module.set_defaults()
def convert(_wf_exec):
if _wf_exec.__class__ == WorkflowExec:
return
_wf_exec.__class__ = WorkflowExec
for annotation in _wf_exec.annotations:
Annotation.convert(annotation)
for machine in _wf_exec.machine_list:
Machine.convert(machine)
for item_exec in _wf_exec.item_execs:
if item_exec.vtType == ModuleExec.vtType:
ModuleExec.convert(item_exec)
elif item_exec.vtType == GroupExec.vtType:
GroupExec.convert(item_exec)
elif item_exec.vtType == LoopExec.vtType:
LoopExec.convert(item_exec)
def convert(_action):
if _action.__class__ == Action:
return
_action.__class__ = Action
for _annotation in _action.annotations:
Annotation.convert(_annotation)
for _operation in _action.operations:
if _operation.vtType == 'add':
AddOp.convert(_operation)
elif _operation.vtType == 'change':
ChangeOp.convert(_operation)
elif _operation.vtType == 'delete':
DeleteOp.convert(_operation)
else:
raise TypeError("Unknown operation type '%s'" %
_operation.vtType)
def convert(_action):
if _action.__class__ == Action:
return
_action.__class__ = Action
for _annotation in _action.annotations:
Annotation.convert(_annotation)
for _operation in _action.operations:
if _operation.vtType == 'add':
AddOp.convert(_operation)
elif _operation.vtType == 'change':
ChangeOp.convert(_operation)
elif _operation.vtType == 'delete':
DeleteOp.convert(_operation)
else:
raise TypeError("Unknown operation type '%s'" %
_operation.vtType)
def convert(_group):
if _group.__class__ == Group:
return
_group.__class__ = Group
if _group.db_location:
Location.convert(_group.db_location)
if _group.db_workflow:
from vistrails.core.vistrail.pipeline import Pipeline
Pipeline.convert(_group.db_workflow)
for _function in _group.db_functions:
ModuleFunction.convert(_function)
for _annotation in _group.db_get_annotations():
Annotation.convert(_annotation)
for _control_parameter in _group.db_get_controlParameters():
ModuleControlParam.convert(_control_parameter)
_group.set_defaults()
def insert_module_annotations(self, module, a_dict):
"""Adds an annotation on the execution object for this module.
"""
for k, v in a_dict.iteritems():
a_id = self.log.id_scope.getNewId(Annotation.vtType)
annotation = Annotation(id=a_id, key=k, value=v)
self.module_execs[module].add_annotation(annotation)
def create_ops(self, id_scope=IdScope()):
from vistrails.core.modules.basic_modules import identifier as basic_pkg
from vistrails.core.vistrail.module import Module
from vistrails.core.vistrail.module_function import ModuleFunction
from vistrails.core.vistrail.module_param import ModuleParam
from vistrails.core.vistrail.annotation import Annotation
if id_scope is None:
id_scope = IdScope(
remap={
AddOp.vtType: 'operation',
ChangeOp.vtType: 'operation',
DeleteOp.vtType: 'operation'
})
m = Module(id=id_scope.getNewId(Module.vtType),
name='Float',
package=basic_pkg)
add_op = AddOp(id=id_scope.getNewId(AddOp.vtType),
what=Module.vtType,
objectId=m.id,
data=m)
function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType),
name='value')
change_op = ChangeOp(id=id_scope.getNewId(ChangeOp.vtType),
what=ModuleFunction.vtType,
oldObjId=2,
newObjId=function.real_id,
parentObjId=m.id,
parentObjType=Module.vtType,
data=function)
param = ModuleParam(id=id_scope.getNewId(ModuleParam.vtType),
type='Float',
val='1.0')
delete_op = DeleteOp(id=id_scope.getNewId(DeleteOp.vtType),
what=ModuleParam.vtType,
objectId=param.real_id,
parentObjId=function.real_id,
parentObjType=ModuleFunction.vtType)
annotation = Annotation(id=id_scope.getNewId(Annotation.vtType),
key='foo',
value='bar')
add_annotation = AddOp(id=id_scope.getNewId(AddOp.vtType),
what=Annotation.vtType,
objectId=m.id,
data=annotation)
cparam = ModuleControlParam(id=id_scope.getNewId(
ModuleControlParam.vtType),
name='foo',
value='bar')
add_cparam = AddOp(id=id_scope.getNewId(AddOp.vtType),
what=ModuleControlParam.vtType,
objectId=m.id,
data=cparam)
return [add_op, change_op, delete_op, add_annotation]
def convert(_vistrail):
_vistrail.__class__ = Vistrail
for action in _vistrail.actions:
Action.convert(action)
# for tag in _vistrail.tags:
# Tag.convert(tag)
for annotation in _vistrail.annotations:
Annotation.convert(annotation)
for annotation in _vistrail.action_annotations:
ActionAnnotation.convert(annotation)
for variable in _vistrail.vistrail_variables:
VistrailVariable.convert(variable)
for pe in _vistrail.parameter_explorations:
ParameterExploration.convert(pe)
_vistrail.set_defaults()
def convert(_vistrail):
_vistrail.__class__ = Vistrail
for action in _vistrail.actions:
Action.convert(action)
# for tag in _vistrail.tags:
# Tag.convert(tag)
for annotation in _vistrail.annotations:
Annotation.convert(annotation)
for annotation in _vistrail.action_annotations:
ActionAnnotation.convert(annotation)
for variable in _vistrail.vistrail_variables:
VistrailVariable.convert(variable)
for pe in _vistrail.parameter_explorations:
ParameterExploration.convert(pe)
_vistrail.set_defaults()
def insert_workflow_exec_annotations(self, a_dict):
"""Adds an annotation on the whole workflow log object.
The information is not associated with the execution of a specific
module but of the whole pipeline.
"""
if self.workflow_exec:
for k, v in a_dict.iteritems():
a_id = self.log.id_scope.getNewId(Annotation.vtType)
annotation = Annotation(id=a_id, key=k, value=v)
self.workflow_exec.add_annotation(annotation)
def set_annotation(self, key, value):
if self.db_has_annotation_with_key(key):
old_annotation = self.db_get_annotation_by_key(key)
if old_annotation.value == value:
return False
self.db_delete_annotation(old_annotation)
if not (value is None or
(isinstance(value, str) and value.strip() == '')):
annotation = Annotation(
id=self.idScope.getNewId(Annotation.vtType),
key=key,
value=value,
)
self.db_add_annotation(annotation)
return True
def finish_execution(self,
module,
error,
errorTrace=None,
suspended=False):
"""Signals the end of the execution of a module.
Called by a module after succeeded of suspended, or called by the
interpreter after an exception.
"""
module_exec = self.module_execs.pop(module, None)
if module_exec is None:
# The module can finish execution without starting (if it was
# suspended, etc...)
return
module_exec.ts_end = vistrails.core.system.current_time()
if suspended:
module_exec.completed = -2
module_exec.error = error
elif error:
module_exec.completed = -1
module_exec.error = error
if errorTrace:
a_id = self.log.id_scope.getNewId(Annotation.vtType)
annotation = Annotation(id=a_id,
key="errorTrace",
value=errorTrace)
module_exec.add_annotation(annotation)
else:
module_exec.completed = 1
for child in self.children_execs.pop(module, ()):
child.ts_end = vistrails.core.system.current_time()
if suspended:
child.completed = -2
child.error = error
elif not error:
child.completed = 1
else:
child.completed = -1
child.error = error
def change_annotation(self, key, value, version_number):
""" change_annotation(key:str, value:str, version_number:long) -> None
Changes the annotation of (key, value) for version version_number
"""
if version_number in self.actionMap:
action = self.actionMap[version_number]
if action.has_annotation_with_key(key):
old_annotation = action.get_annotation_by_key(key)
if old_annotation.value == value:
return False
action.delete_annotation(old_annotation)
if value.strip() != '':
annotation = \
Annotation(id=self.idScope.getNewId(Annotation.vtType),
key=key,
value=value,
)
action.add_annotation(annotation)
self.changed = True
return True
return False
log = map_result[engine]['xml_log']
exec_ = None
if (vtType == 'abstraction') or (vtType == 'group'):
exec_ = unserialize(log, GroupExec)
elif (vtType == 'module'):
exec_ = unserialize(log, ModuleExec)
else:
# something is wrong...
continue
# assigning new ids to existing annotations
exec_annotations = exec_.annotations
for i in range(len(exec_annotations)):
exec_annotations[i].id = self.logging.log.log.id_scope.getNewId(Annotation.vtType)
parallel_annotation = Annotation(key='parallel_execution', value=True)
parallel_annotation.id = self.logging.log.log.id_scope.getNewId(Annotation.vtType)
annotations = [parallel_annotation] + exec_annotations
exec_.annotations = annotations
# before adding the execution log, we need to get the machine information
machine = unserialize(map_result[engine]['machine_log'], Machine)
machine_id = self.logging.add_machine(machine)
# recursively add machine information to execution items
def add_machine_recursive(exec_):
for item in exec_.item_execs:
if hasattr(item, 'machine_id'):
item.machine_id = machine_id
if item.vtType in ('abstraction', 'group'):
add_machine_recursive(item)
if (vtType == 'abstraction') or (vtType == 'group'):
exec_ = unserialize(log, GroupExec)
elif (vtType == 'module'):
exec_ = unserialize(log, ModuleExec)
else:
# something is wrong...
continue
# assigning new ids to existing annotations
exec_annotations = exec_.annotations
for i in range(len(exec_annotations)):
exec_annotations[
i].id = self.logging.log.log.id_scope.getNewId(
Annotation.vtType)
parallel_annotation = Annotation(key='parallel_execution',
value=True)
parallel_annotation.id = self.logging.log.log.id_scope.getNewId(
Annotation.vtType)
annotations = [parallel_annotation] + exec_annotations
exec_.annotations = annotations
# before adding the execution log, we need to get the machine information
machine = unserialize(map_result[engine]['machine_log'], Machine)
machine_id = self.logging.add_machine(machine)
# recursively add machine information to execution items
def add_machine_recursive(exec_):
for item in exec_.item_execs:
if hasattr(item, 'machine_id'):
item.machine_id = machine_id
if item.vtType in ('abstraction', 'group'):
def replace_generic(controller,
pipeline,
old_module,
new_module,
function_remap=None,
src_port_remap=None,
dst_port_remap=None,
annotation_remap=None,
control_param_remap=None,
use_registry=True):
if function_remap is None:
function_remap = {}
if src_port_remap is None:
src_port_remap = {}
if dst_port_remap is None:
dst_port_remap = {}
if annotation_remap is None:
annotation_remap = {}
if control_param_remap is None:
control_param_remap = {}
basic_pkg = get_vistrails_basic_pkg_id()
ops = []
ops.extend(controller.delete_module_list_ops(pipeline,
[old_module.id]))
for annotation in old_module.annotations:
if annotation.key not in annotation_remap:
annotation_key = annotation.key
else:
remap = annotation_remap[annotation.key]
if remap is None:
# don't add the annotation back in
continue
elif not isinstance(remap, basestring):
ops.extend(remap(annotation))
continue
else:
annotation_key = remap
new_annotation = \
Annotation(id=controller.id_scope.getNewId(Annotation.vtType),
key=annotation_key,
value=annotation.value)
new_module.add_annotation(new_annotation)
for control_param in old_module.control_parameters:
if control_param.name not in control_param_remap:
control_param_name = control_param.name
else:
remap = control_param_remap[control_param.name]
if remap is None:
# don't add the control param back in
continue
elif not isinstance(remap, basestring):
ops.extend(remap(control_param))
continue
else:
control_param_name = remap
new_control_param = \
ModuleControlParam(id=controller.id_scope.getNewId(
ModuleControlParam.vtType),
name=control_param_name,
value=control_param.value)
new_module.add_control_parameter(new_control_param)
if not old_module.is_group() and not old_module.is_abstraction():
for port_spec in old_module.port_spec_list:
if port_spec.type == 'input':
if port_spec.name not in dst_port_remap:
spec_name = port_spec.name
else:
remap = dst_port_remap[port_spec.name]
if remap is None:
continue
elif not isinstance(remap, basestring):
ops.extend(remap(port_spec))
continue
else:
spec_name = remap
elif port_spec.type == 'output':
if port_spec.name not in src_port_remap:
spec_name = port_spec.name
else:
remap = src_port_remap[port_spec.name]
if remap is None:
continue
elif not isinstance(remap, basestring):
ops.extend(remap(port_spec))
continue
else:
spec_name = remap
new_spec = port_spec.do_copy(True, controller.id_scope, {})
new_spec.name = spec_name
new_module.add_port_spec(new_spec)
function_ops = []
for function in old_module.functions:
if function.name not in function_remap:
function_name = function.name
else:
remap = function_remap[function.name]
if remap is None:
# don't add the function back in
continue
elif not isinstance(remap, basestring):
function_ops.extend(remap(function, new_module))
continue
else:
function_name = remap
if len(function.parameters) > 0:
new_param_vals, aliases = zip(*[(p.strValue, p.alias)
for p in function.parameters])
else:
new_param_vals = []
aliases = []
if use_registry:
function_port_spec = function_name
else:
def mk_psi(pos):
psi = PortSpecItem(module="Module",
package=basic_pkg,
namespace="",
pos=pos)
return psi
n_items = len(new_param_vals)
function_port_spec = PortSpec(
name=function_name,
items=[mk_psi(i) for i in xrange(n_items)])
new_function = controller.create_function(new_module,
function_port_spec,
new_param_vals, aliases)
new_module.add_function(new_function)
if None in function_remap:
# used to add new functions
remap = function_remap[None]
function_ops.extend(remap(None, new_module))
# add the new module
ops.append(('add', new_module))
ops.extend(function_ops)
create_new_connection = UpgradeWorkflowHandler.create_new_connection
for _, conn_id in pipeline.graph.edges_from(old_module.id):
old_conn = pipeline.connections[conn_id]
if old_conn.source.name not in src_port_remap:
source_name = old_conn.source.name
else:
remap = src_port_remap[old_conn.source.name]
if remap is None:
# don't add this connection back in
continue
elif not isinstance(remap, basestring):
ops.extend(remap(old_conn, new_module))
continue
else:
source_name = remap
old_dst_module = pipeline.modules[old_conn.destination.moduleId]
if use_registry:
source_port = source_name
else:
source_port = Port(name=source_name,
type='source',
signature=create_port_spec_string([
(basic_pkg, 'Variant', '')
]))
new_conn = create_new_connection(controller, new_module,
source_port, old_dst_module,
old_conn.destination)
ops.append(('add', new_conn))
for _, conn_id in pipeline.graph.edges_to(old_module.id):
old_conn = pipeline.connections[conn_id]
if old_conn.destination.name not in dst_port_remap:
destination_name = old_conn.destination.name
else:
remap = dst_port_remap[old_conn.destination.name]
if remap is None:
# don't add this connection back in
continue
elif not isinstance(remap, basestring):
ops.extend(remap(old_conn, new_module))
continue
else:
destination_name = remap
old_src_module = pipeline.modules[old_conn.source.moduleId]
if use_registry:
destination_port = destination_name
else:
destination_port = Port(name=destination_name,
type='destination',
signature=create_port_spec_string([
(basic_pkg, 'Variant', '')
]))
new_conn = create_new_connection(controller, old_src_module,
old_conn.source, new_module,
destination_port)
ops.append(('add', new_conn))
return [vistrails.core.db.action.create_action(ops)]
