def create(self): # FIXME moved here because otherwise we hit the registry too early from vistrails.core.modules.module_utils import FilePool self._file_pool = FilePool() self._persistent_pipeline = vistrails.core.vistrail.pipeline.Pipeline() self._objects = {} self.filePool = self._file_pool self._streams = []
class CachedInterpreter(vistrails.core.interpreter.base.BaseInterpreter): def __init__(self): vistrails.core.interpreter.base.BaseInterpreter.__init__(self) self.debugger = None self.create() def create(self): # FIXME moved here because otherwise we hit the registry too early from vistrails.core.modules.module_utils import FilePool self._file_pool = FilePool() self._persistent_pipeline = vistrails.core.vistrail.pipeline.Pipeline() self._objects = {} self.filePool = self._file_pool self._streams = [] def clear(self): self._file_pool.cleanup() self._persistent_pipeline.clear() for obj in self._objects.itervalues(): obj.clear() self._objects = {} def __del__(self): self.clear() def clean_modules(self, modules_to_clean): """clean_modules(modules_to_clean: list of persistent module ids) Removes modules from the persistent pipeline, and the modules that depend on them.""" if not modules_to_clean: return g = self._persistent_pipeline.graph modules_to_clean = (set(modules_to_clean) & set(self._persistent_pipeline.modules.iterkeys())) dependencies = g.vertices_topological_sort(modules_to_clean) for v in dependencies: self._persistent_pipeline.delete_module(v) del self._objects[v] def clean_non_cacheable_modules(self): """clean_non_cacheable_modules() -> None Removes all modules that are not cacheable from the persistent pipeline, and the modules that depend on them. """ non_cacheable_modules = [i for (i, mod) in self._objects.iteritems() if not mod.is_cacheable()] self.clean_modules(non_cacheable_modules) def _clear_package(self, identifier): """clear_package(identifier: str) -> None Removes all modules from the given package from the persistent pipeline. """ modules = [mod.id for mod in self._persistent_pipeline.module_list if mod.module_descriptor.identifier == identifier] self.clean_modules(modules) def make_connection(self, conn, src, dst): """make_connection(self, conn, src, dst) Builds a execution-time connection between modules. """ iport = conn.destination.name oport = conn.source.name src.enable_output_port(oport) src.load_type_check_descs() if isinstance(src, src.InputPort_desc.module): typecheck = [False] else: typecheck = src.get_type_checks(conn.source.spec) dst.set_input_port(iport, ModuleConnector(src, oport, conn.source.spec, typecheck)) def setup_pipeline(self, pipeline, **kwargs): """setup_pipeline(controller, pipeline, locator, currentVersion, view, aliases, **kwargs) Matches a pipeline with the persistent pipeline and creates instances of modules that aren't in the cache. """ def fetch(name, default): return kwargs.pop(name, default) controller = fetch('controller', None) locator = fetch('locator', None) current_version = fetch('current_version', None) view = fetch('view', DummyView()) vistrail_variables = fetch('vistrail_variables', None) aliases = fetch('aliases', None) params = fetch('params', None) extra_info = fetch('extra_info', None) logger = fetch('logger', DummyLogController) sinks = fetch('sinks', None) reason = fetch('reason', None) actions = fetch('actions', None) done_summon_hooks = fetch('done_summon_hooks', []) module_executed_hook = fetch('module_executed_hook', []) stop_on_error = fetch('stop_on_error', True) parent_exec = fetch('parent_exec', None) reg = get_module_registry() if len(kwargs) > 0: raise VistrailsInternalError('Wrong parameters passed ' 'to setup_pipeline: %s' % kwargs) def create_null(): """Creates a Null value""" getter = reg.get_descriptor_by_name descriptor = getter(basic_pkg, 'Null') return descriptor.module() def create_constant(param, module): """Creates a Constant from a parameter spec""" getter = reg.get_descriptor_by_name desc = getter(param.identifier, param.type, param.namespace) constant = desc.module() constant.id = module.id # if param.evaluatedStrValue: # constant.setValue(param.evaluatedStrValue) if param.strValue != '': constant.setValue(param.strValue) else: constant.setValue( \ constant.translate_to_string(constant.default_value)) return constant ### BEGIN METHOD ### # if self.debugger: # self.debugger.update() to_delete = [] errors = {} if controller is not None: # Controller is none for sub_modules controller.validate(pipeline) else: pipeline.validate() self.resolve_aliases(pipeline, aliases) if vistrail_variables: self.resolve_variables(vistrail_variables, pipeline) self.update_params(pipeline, params) (tmp_to_persistent_module_map, conn_map, module_added_set, conn_added_set) = self.add_to_persistent_pipeline(pipeline) # Create the new objects for i in module_added_set: persistent_id = tmp_to_persistent_module_map[i] module = self._persistent_pipeline.modules[persistent_id] obj = self._objects[persistent_id] = module.summon() obj.interpreter = self obj.id = persistent_id obj.signature = module._signature # Checking if output should be stored if module.has_annotation_with_key('annotate_output'): annotate_output = module.get_annotation_by_key('annotate_output') #print annotate_output if annotate_output: obj.annotate_output = True for f in module.functions: connector = None if len(f.params) == 0: connector = ModuleConnector(create_null(), 'value', f.get_spec('output')) elif len(f.params) == 1: p = f.params[0] try: constant = create_constant(p, module) connector = ModuleConnector(constant, 'value', f.get_spec('output')) except Exception, e: debug.unexpected_exception(e) err = ModuleError( module, "Uncaught exception creating Constant from " "%r: %s" % ( p.strValue, debug.format_exception(e))) errors[i] = err to_delete.append(obj.id) else: tupleModule = vistrails.core.interpreter.base.InternalTuple() tupleModule.length = len(f.params) for (j,p) in enumerate(f.params): try: constant = create_constant(p, module) constant.update() connector = ModuleConnector(constant, 'value', f.get_spec('output')) tupleModule.set_input_port(j, connector) except Exception, e: debug.unexpected_exception(e) err = ModuleError( module, "Uncaught exception creating Constant " "from %r: %s" % ( p.strValue, debug.format_exception(e))) errors[i] = err to_delete.append(obj.id) connector = ModuleConnector(tupleModule, 'value', f.get_spec('output')) if connector: obj.set_input_port(f.name, connector, is_method=True)
class CachedInterpreter(vistrails.core.interpreter.base.BaseInterpreter): def __init__(self): vistrails.core.interpreter.base.BaseInterpreter.__init__(self) self.debugger = None self.create() def create(self): # FIXME moved here because otherwise we hit the registry too early from vistrails.core.modules.module_utils import FilePool self._file_pool = FilePool() self._persistent_pipeline = vistrails.core.vistrail.pipeline.Pipeline() self._objects = {} self.filePool = self._file_pool self._streams = [] def clear(self): self._file_pool.cleanup() self._persistent_pipeline.clear() for obj in self._objects.itervalues(): obj.clear() self._objects = {} def __del__(self): self.clear() def clean_modules(self, modules_to_clean): """clean_modules(modules_to_clean: list of persistent module ids) Removes modules from the persistent pipeline, and the modules that depend on them.""" if not modules_to_clean: return g = self._persistent_pipeline.graph modules_to_clean = (set(modules_to_clean) & set( self._persistent_pipeline.modules.iterkeys())) dependencies = g.vertices_topological_sort(modules_to_clean) for v in dependencies: self._persistent_pipeline.delete_module(v) del self._objects[v] def clean_non_cacheable_modules(self): """clean_non_cacheable_modules() -> None Removes all modules that are not cacheable from the persistent pipeline, and the modules that depend on them. """ non_cacheable_modules = [ i for (i, mod) in self._objects.iteritems() if not mod.is_cacheable() ] self.clean_modules(non_cacheable_modules) def _clear_package(self, identifier): """clear_package(identifier: str) -> None Removes all modules from the given package from the persistent pipeline. """ modules = [ mod.id for mod in self._persistent_pipeline.module_list if mod.module_descriptor.identifier == identifier ] self.clean_modules(modules) def make_connection(self, conn, src, dst): """make_connection(self, conn, src, dst) Builds a execution-time connection between modules. """ iport = conn.destination.name oport = conn.source.name src.enable_output_port(oport) src.load_type_check_descs() if isinstance(src, src.InputPort_desc.module): typecheck = [False] else: typecheck = src.get_type_checks(conn.source.spec) dst.set_input_port( iport, ModuleConnector(src, oport, conn.source.spec, typecheck)) def setup_pipeline(self, pipeline, **kwargs): """setup_pipeline(controller, pipeline, locator, currentVersion, view, aliases, **kwargs) Matches a pipeline with the persistent pipeline and creates instances of modules that aren't in the cache. """ def fetch(name, default): return kwargs.pop(name, default) controller = fetch('controller', None) locator = fetch('locator', None) current_version = fetch('current_version', None) view = fetch('view', DummyView()) vistrail_variables = fetch('vistrail_variables', None) aliases = fetch('aliases', None) params = fetch('params', None) extra_info = fetch('extra_info', None) logger = fetch('logger', DummyLogController) sinks = fetch('sinks', None) reason = fetch('reason', None) actions = fetch('actions', None) done_summon_hooks = fetch('done_summon_hooks', []) module_executed_hook = fetch('module_executed_hook', []) stop_on_error = fetch('stop_on_error', True) parent_exec = fetch('parent_exec', None) job_monitor = fetch('job_monitor', None) reg = get_module_registry() if len(kwargs) > 0: raise VistrailsInternalError('Wrong parameters passed ' 'to setup_pipeline: %s' % kwargs) def create_null(): """Creates a Null value""" getter = reg.get_descriptor_by_name descriptor = getter(basic_pkg, 'Null') return descriptor.module() def create_constant(param, module): """Creates a Constant from a parameter spec""" getter = reg.get_descriptor_by_name desc = getter(param.identifier, param.type, param.namespace) constant = desc.module() constant.id = module.id # if param.evaluatedStrValue: # constant.setValue(param.evaluatedStrValue) if param.strValue != '': constant.setValue(param.strValue) else: constant.setValue( constant.translate_to_string(constant.default_value)) return constant ### BEGIN METHOD ### # if self.debugger: # self.debugger.update() to_delete = [] errors = {} if controller is not None: # Controller is none for sub_modules controller.validate(pipeline) else: pipeline.validate() self.resolve_aliases(pipeline, aliases) if vistrail_variables: self.resolve_variables(vistrail_variables, pipeline) self.update_params(pipeline, params) (tmp_to_persistent_module_map, conn_map, module_added_set, conn_added_set) = self.add_to_persistent_pipeline(pipeline) # Create the new objects for i in module_added_set: persistent_id = tmp_to_persistent_module_map[i] module = self._persistent_pipeline.modules[persistent_id] obj = self._objects[persistent_id] = module.summon() obj.interpreter = self obj.id = persistent_id obj.signature = module._signature # Checking if output should be stored if module.has_annotation_with_key('annotate_output'): annotate_output = module.get_annotation_by_key( 'annotate_output') #print annotate_output if annotate_output: obj.annotate_output = True for f in module.functions: connector = None if len(f.params) == 0: connector = ModuleConnector(create_null(), 'value', f.get_spec('output')) elif len(f.params) == 1: p = f.params[0] try: constant = create_constant(p, module) connector = ModuleConnector(constant, 'value', f.get_spec('output')) except Exception, e: debug.unexpected_exception(e) err = ModuleError( module, "Uncaught exception creating Constant from " "%r: %s" % (p.strValue, debug.format_exception(e))) errors[i] = err to_delete.append(obj.id) else: tupleModule = vistrails.core.interpreter.base.InternalTuple( ) tupleModule.length = len(f.params) for (j, p) in enumerate(f.params): try: constant = create_constant(p, module) constant.update() connector = ModuleConnector( constant, 'value', f.get_spec('output')) tupleModule.set_input_port(j, connector) except Exception, e: debug.unexpected_exception(e) err = ModuleError( module, "Uncaught exception creating Constant " "from %r: %s" % (p.strValue, debug.format_exception(e))) errors[i] = err to_delete.append(obj.id) connector = ModuleConnector(tupleModule, 'value', f.get_spec('output')) if connector: obj.set_input_port(f.name, connector, is_method=True)