def create(self):
# FIXME moved here because otherwise we hit the registry too early
from vistrails.core.modules.module_utils import FilePool
self._file_pool = FilePool()
self._persistent_pipeline = vistrails.core.vistrail.pipeline.Pipeline()
self._objects = {}
self.filePool = self._file_pool
self._streams = []
class CachedInterpreter(vistrails.core.interpreter.base.BaseInterpreter):
def __init__(self):
vistrails.core.interpreter.base.BaseInterpreter.__init__(self)
self.debugger = None
self.create()
def create(self):
# FIXME moved here because otherwise we hit the registry too early
from vistrails.core.modules.module_utils import FilePool
self._file_pool = FilePool()
self._persistent_pipeline = vistrails.core.vistrail.pipeline.Pipeline()
self._objects = {}
self.filePool = self._file_pool
self._streams = []
def clear(self):
self._file_pool.cleanup()
self._persistent_pipeline.clear()
for obj in self._objects.itervalues():
obj.clear()
self._objects = {}
def __del__(self):
self.clear()
def clean_modules(self, modules_to_clean):
"""clean_modules(modules_to_clean: list of persistent module ids)
Removes modules from the persistent pipeline, and the modules that
depend on them."""
if not modules_to_clean:
return
g = self._persistent_pipeline.graph
modules_to_clean = (set(modules_to_clean) &
set(self._persistent_pipeline.modules.iterkeys()))
dependencies = g.vertices_topological_sort(modules_to_clean)
for v in dependencies:
self._persistent_pipeline.delete_module(v)
del self._objects[v]
def clean_non_cacheable_modules(self):
"""clean_non_cacheable_modules() -> None
Removes all modules that are not cacheable from the persistent
pipeline, and the modules that depend on them.
"""
non_cacheable_modules = [i for
(i, mod) in self._objects.iteritems()
if not mod.is_cacheable()]
self.clean_modules(non_cacheable_modules)
def _clear_package(self, identifier):
"""clear_package(identifier: str) -> None
Removes all modules from the given package from the persistent
pipeline.
"""
modules = [mod.id
for mod in self._persistent_pipeline.module_list
if mod.module_descriptor.identifier == identifier]
self.clean_modules(modules)
def make_connection(self, conn, src, dst):
"""make_connection(self, conn, src, dst)
Builds a execution-time connection between modules.
"""
iport = conn.destination.name
oport = conn.source.name
src.enable_output_port(oport)
src.load_type_check_descs()
if isinstance(src, src.InputPort_desc.module):
typecheck = [False]
else:
typecheck = src.get_type_checks(conn.source.spec)
dst.set_input_port(iport,
ModuleConnector(src, oport, conn.source.spec,
typecheck))
def setup_pipeline(self, pipeline, **kwargs):
"""setup_pipeline(controller, pipeline, locator, currentVersion,
view, aliases, **kwargs)
Matches a pipeline with the persistent pipeline and creates
instances of modules that aren't in the cache.
"""
def fetch(name, default):
return kwargs.pop(name, default)
controller = fetch('controller', None)
locator = fetch('locator', None)
current_version = fetch('current_version', None)
view = fetch('view', DummyView())
vistrail_variables = fetch('vistrail_variables', None)
aliases = fetch('aliases', None)
params = fetch('params', None)
extra_info = fetch('extra_info', None)
logger = fetch('logger', DummyLogController)
sinks = fetch('sinks', None)
reason = fetch('reason', None)
actions = fetch('actions', None)
done_summon_hooks = fetch('done_summon_hooks', [])
module_executed_hook = fetch('module_executed_hook', [])
stop_on_error = fetch('stop_on_error', True)
parent_exec = fetch('parent_exec', None)
reg = get_module_registry()
if len(kwargs) > 0:
raise VistrailsInternalError('Wrong parameters passed '
'to setup_pipeline: %s' % kwargs)
def create_null():
"""Creates a Null value"""
getter = reg.get_descriptor_by_name
descriptor = getter(basic_pkg, 'Null')
return descriptor.module()
def create_constant(param, module):
"""Creates a Constant from a parameter spec"""
getter = reg.get_descriptor_by_name
desc = getter(param.identifier, param.type, param.namespace)
constant = desc.module()
constant.id = module.id
# if param.evaluatedStrValue:
# constant.setValue(param.evaluatedStrValue)
if param.strValue != '':
constant.setValue(param.strValue)
else:
constant.setValue( \
constant.translate_to_string(constant.default_value))
return constant
### BEGIN METHOD ###
# if self.debugger:
# self.debugger.update()
to_delete = []
errors = {}
if controller is not None:
# Controller is none for sub_modules
controller.validate(pipeline)
else:
pipeline.validate()
self.resolve_aliases(pipeline, aliases)
if vistrail_variables:
self.resolve_variables(vistrail_variables, pipeline)
self.update_params(pipeline, params)
(tmp_to_persistent_module_map,
conn_map,
module_added_set,
conn_added_set) = self.add_to_persistent_pipeline(pipeline)
# Create the new objects
for i in module_added_set:
persistent_id = tmp_to_persistent_module_map[i]
module = self._persistent_pipeline.modules[persistent_id]
obj = self._objects[persistent_id] = module.summon()
obj.interpreter = self
obj.id = persistent_id
obj.signature = module._signature
# Checking if output should be stored
if module.has_annotation_with_key('annotate_output'):
annotate_output = module.get_annotation_by_key('annotate_output')
#print annotate_output
if annotate_output:
obj.annotate_output = True
for f in module.functions:
connector = None
if len(f.params) == 0:
connector = ModuleConnector(create_null(), 'value',
f.get_spec('output'))
elif len(f.params) == 1:
p = f.params[0]
try:
constant = create_constant(p, module)
connector = ModuleConnector(constant, 'value',
f.get_spec('output'))
except Exception, e:
debug.unexpected_exception(e)
err = ModuleError(
module,
"Uncaught exception creating Constant from "
"%r: %s" % (
p.strValue,
debug.format_exception(e)))
errors[i] = err
to_delete.append(obj.id)
else:
tupleModule = vistrails.core.interpreter.base.InternalTuple()
tupleModule.length = len(f.params)
for (j,p) in enumerate(f.params):
try:
constant = create_constant(p, module)
constant.update()
connector = ModuleConnector(constant, 'value',
f.get_spec('output'))
tupleModule.set_input_port(j, connector)
except Exception, e:
debug.unexpected_exception(e)
err = ModuleError(
module,
"Uncaught exception creating Constant "
"from %r: %s" % (
p.strValue,
debug.format_exception(e)))
errors[i] = err
to_delete.append(obj.id)
connector = ModuleConnector(tupleModule, 'value',
f.get_spec('output'))
if connector:
obj.set_input_port(f.name, connector, is_method=True)
class CachedInterpreter(vistrails.core.interpreter.base.BaseInterpreter):
def __init__(self):
vistrails.core.interpreter.base.BaseInterpreter.__init__(self)
self.debugger = None
self.create()
def create(self):
# FIXME moved here because otherwise we hit the registry too early
from vistrails.core.modules.module_utils import FilePool
self._file_pool = FilePool()
self._persistent_pipeline = vistrails.core.vistrail.pipeline.Pipeline()
self._objects = {}
self.filePool = self._file_pool
self._streams = []
def clear(self):
self._file_pool.cleanup()
self._persistent_pipeline.clear()
for obj in self._objects.itervalues():
obj.clear()
self._objects = {}
def __del__(self):
self.clear()
def clean_modules(self, modules_to_clean):
"""clean_modules(modules_to_clean: list of persistent module ids)
Removes modules from the persistent pipeline, and the modules that
depend on them."""
if not modules_to_clean:
return
g = self._persistent_pipeline.graph
modules_to_clean = (set(modules_to_clean) & set(
self._persistent_pipeline.modules.iterkeys()))
dependencies = g.vertices_topological_sort(modules_to_clean)
for v in dependencies:
self._persistent_pipeline.delete_module(v)
del self._objects[v]
def clean_non_cacheable_modules(self):
"""clean_non_cacheable_modules() -> None
Removes all modules that are not cacheable from the persistent
pipeline, and the modules that depend on them.
"""
non_cacheable_modules = [
i for (i, mod) in self._objects.iteritems()
if not mod.is_cacheable()
]
self.clean_modules(non_cacheable_modules)
def _clear_package(self, identifier):
"""clear_package(identifier: str) -> None
Removes all modules from the given package from the persistent
pipeline.
"""
modules = [
mod.id for mod in self._persistent_pipeline.module_list
if mod.module_descriptor.identifier == identifier
]
self.clean_modules(modules)
def make_connection(self, conn, src, dst):
"""make_connection(self, conn, src, dst)
Builds a execution-time connection between modules.
"""
iport = conn.destination.name
oport = conn.source.name
src.enable_output_port(oport)
src.load_type_check_descs()
if isinstance(src, src.InputPort_desc.module):
typecheck = [False]
else:
typecheck = src.get_type_checks(conn.source.spec)
dst.set_input_port(
iport, ModuleConnector(src, oport, conn.source.spec, typecheck))
def setup_pipeline(self, pipeline, **kwargs):
"""setup_pipeline(controller, pipeline, locator, currentVersion,
view, aliases, **kwargs)
Matches a pipeline with the persistent pipeline and creates
instances of modules that aren't in the cache.
"""
def fetch(name, default):
return kwargs.pop(name, default)
controller = fetch('controller', None)
locator = fetch('locator', None)
current_version = fetch('current_version', None)
view = fetch('view', DummyView())
vistrail_variables = fetch('vistrail_variables', None)
aliases = fetch('aliases', None)
params = fetch('params', None)
extra_info = fetch('extra_info', None)
logger = fetch('logger', DummyLogController)
sinks = fetch('sinks', None)
reason = fetch('reason', None)
actions = fetch('actions', None)
done_summon_hooks = fetch('done_summon_hooks', [])
module_executed_hook = fetch('module_executed_hook', [])
stop_on_error = fetch('stop_on_error', True)
parent_exec = fetch('parent_exec', None)
job_monitor = fetch('job_monitor', None)
reg = get_module_registry()
if len(kwargs) > 0:
raise VistrailsInternalError('Wrong parameters passed '
'to setup_pipeline: %s' % kwargs)
def create_null():
"""Creates a Null value"""
getter = reg.get_descriptor_by_name
descriptor = getter(basic_pkg, 'Null')
return descriptor.module()
def create_constant(param, module):
"""Creates a Constant from a parameter spec"""
getter = reg.get_descriptor_by_name
desc = getter(param.identifier, param.type, param.namespace)
constant = desc.module()
constant.id = module.id
# if param.evaluatedStrValue:
# constant.setValue(param.evaluatedStrValue)
if param.strValue != '':
constant.setValue(param.strValue)
else:
constant.setValue(
constant.translate_to_string(constant.default_value))
return constant
### BEGIN METHOD ###
# if self.debugger:
# self.debugger.update()
to_delete = []
errors = {}
if controller is not None:
# Controller is none for sub_modules
controller.validate(pipeline)
else:
pipeline.validate()
self.resolve_aliases(pipeline, aliases)
if vistrail_variables:
self.resolve_variables(vistrail_variables, pipeline)
self.update_params(pipeline, params)
(tmp_to_persistent_module_map, conn_map, module_added_set,
conn_added_set) = self.add_to_persistent_pipeline(pipeline)
# Create the new objects
for i in module_added_set:
persistent_id = tmp_to_persistent_module_map[i]
module = self._persistent_pipeline.modules[persistent_id]
obj = self._objects[persistent_id] = module.summon()
obj.interpreter = self
obj.id = persistent_id
obj.signature = module._signature
# Checking if output should be stored
if module.has_annotation_with_key('annotate_output'):
annotate_output = module.get_annotation_by_key(
'annotate_output')
#print annotate_output
if annotate_output:
obj.annotate_output = True
for f in module.functions:
connector = None
if len(f.params) == 0:
connector = ModuleConnector(create_null(), 'value',
f.get_spec('output'))
elif len(f.params) == 1:
p = f.params[0]
try:
constant = create_constant(p, module)
connector = ModuleConnector(constant, 'value',
f.get_spec('output'))
except Exception, e:
debug.unexpected_exception(e)
err = ModuleError(
module,
"Uncaught exception creating Constant from "
"%r: %s" % (p.strValue, debug.format_exception(e)))
errors[i] = err
to_delete.append(obj.id)
else:
tupleModule = vistrails.core.interpreter.base.InternalTuple(
)
tupleModule.length = len(f.params)
for (j, p) in enumerate(f.params):
try:
constant = create_constant(p, module)
constant.update()
connector = ModuleConnector(
constant, 'value', f.get_spec('output'))
tupleModule.set_input_port(j, connector)
except Exception, e:
debug.unexpected_exception(e)
err = ModuleError(
module, "Uncaught exception creating Constant "
"from %r: %s" %
(p.strValue, debug.format_exception(e)))
errors[i] = err
to_delete.append(obj.id)
connector = ModuleConnector(tupleModule, 'value',
f.get_spec('output'))
if connector:
obj.set_input_port(f.name, connector, is_method=True)