def coalesce_port_specs(neighbors, type): reg = module_registry.get_module_registry() cur_descs = None cur_depth = 0 Variant_desc = reg.get_descriptor_by_name(basic_pkg, 'Variant') if type == 'input': find_common = reg.find_descriptor_subclass common_desc = Variant_desc elif type == 'output': find_common = reg.find_descriptor_superclass common_desc = reg.get_descriptor_by_name(basic_pkg, 'Module') else: raise VistrailsInternalError("Cannot understand type '%s'" % type) for (module, port_name) in neighbors: if cur_descs is None: port_spec = module.get_port_spec(port_name, type) cur_descs = port_spec.descriptors() cur_depth = port_spec.depth else: next_port_spec = module.get_port_spec(port_name, type) next_descs = next_port_spec.descriptors() next_depth = next_port_spec.depth if len(cur_descs) != len(next_descs): raise VistrailsInternalError("Cannot have single port " "connect to incompatible " "types") if 'input' == type and cur_depth > next_depth: # use least depth cur_depth = next_depth cur_descs = next_descs if 'output' == type and cur_depth < next_depth: # use least depth cur_depth = next_depth cur_descs = next_descs descs = [] for cur_desc, next_desc in izip(cur_descs, next_descs): if cur_desc is Variant_desc: new_desc = next_desc elif next_desc is Variant_desc: new_desc = cur_desc else: new_desc = find_common(cur_desc, next_desc) if new_desc is None: new_desc = common_desc descs.append(new_desc) cur_descs = descs if cur_descs: sigstring = '(' + ','.join(d.sigstring for d in cur_descs) + ')' else: sigstring = None return (sigstring, cur_depth)
def get_port_spec_info(pipeline, module): type_map = {'OutputPort': 'output', 'InputPort': 'input'} try: type = type_map[module.name] except KeyError: raise VistrailsInternalError("cannot translate type '%s'" % module.name) if type == 'input': get_edges = pipeline.graph.edges_from get_port_name = \ lambda x: pipeline.connections[x].destination.name else: # type == 'output' get_edges = pipeline.graph.edges_to get_port_name = \ lambda x: pipeline.connections[x].source.name # conns = get_edges(module.id) # for i, m in pipeline.modules.iteritems(): # print i, m.name # for j, c in pipeline.connections.iteritems(): # print j, c.source.moduleId, c.destination.moduleId neighbors = [(pipeline.modules[m_id], get_port_name(c_id)) for (m_id, c_id) in get_edges(module.id)] port_name = neighbors[0][1] sigstring, depth = coalesce_port_specs(neighbors, type) # sigstring = neighbor.get_port_spec(port_name, type).sigstring # FIXME check old registry here? port_optional = False for function in module.functions: if function.name == 'name': port_name = function.params[0].strValue if function.name == 'optional': port_optional = function.params[0].strValue == 'True' # print 'psi:', port_name, old_name, sigstring return (port_name, sigstring, port_optional, depth, neighbors)
def delete_output_port(self, name): key = (name, 'output') if key in self.port_specs: self.delete_port_spec(self.port_specs[key]) else: raise VistrailsInternalError("delete_output_port called on " "nonexistent port '%s'" % name)
def __init__(self, registry, startup): """__init__(configuration: ConfigurationObject) -> PackageManager configuration is the persistent configuration object of the application. """ global _package_manager if _package_manager: m = "Package manager can only be constructed once." raise VistrailsInternalError(m) _package_manager = self self._registry = registry self._startup = startup # Contains packages that have not yet been enabled, but exist on the # filesystem self._available_packages = {} # codepath: str -> Package # These other lists contain enabled packages self._package_list = {} # codepath: str -> Package self._package_versions = {} # identifier: str -> version -> Package self._old_identifier_map = {} # old_id: str -> new_id: str self._dependency_graph = vistrails.core.data_structures.graph.Graph() self._default_prefix_dict = \ {'basic_modules': 'vistrails.core.modules.', 'abstraction': 'vistrails.core.modules.'} self._userpackages = None self._packages = None self._abstraction_pkg = None self._currently_importing_package = None # Setup a global __import__ hook that calls Package#import_override() # for all imports executed from that package import __builtin__ self._orig_import = __builtin__.__import__ __builtin__.__import__ = self._import_override # Compute the list of available packages, _available_packages self.build_available_package_names_list() if get_vistrails_configuration().loadPackages: for pkg in self._startup.enabled_packages.itervalues(): self.add_package(pkg.name, prefix=pkg.prefix) else: try: basic_pkg = self._startup.enabled_packages['basic_modules'] except KeyError: pass else: self.add_package(basic_pkg.name, prefix=basic_pkg.prefix) try: abs_pkg = self._startup.enabled_packages['abstraction'] except KeyError: pass else: self.add_package(abs_pkg.name, prefix=abs_pkg.prefix)
def add_connection(self, conn): """add_connection(conn: ExtConnection) -> None Adds a connection to the list """ if self.__connections.has_key(conn.id): msg = "External Connection '%s' with repeated id" % conn.name raise VistrailsInternalError(msg) self.__connections[conn.id] = conn self.current_id = max(self.current_id, conn.id+1) self.serialize()
def get_directory(self): thumbnail_dir = system.get_vistrails_directory('thumbs.cacheDir') if thumbnail_dir is not None: if not os.path.exists(thumbnail_dir): raise VistrailsInternalError("Cannot find %s" % thumbnail_dir) return thumbnail_dir # raise VistrailsInternalError("'thumbs.cacheDir' not" # " specified in configuration") if self._temp_directory is None: self._temp_directory = tempfile.mkdtemp(prefix='vt_thumbs_') return self._temp_directory
def addVersion(self, action): """ addVersion(action: Action) -> None Adds new version to vistrail """ if action.timestep in self.actionMap: raise VistrailsInternalError("existing timestep: %d" % action.timestep) self.db_add_action(action) self.changed = True # signal to update explicit tree self.tree.addVersion(action.id, action.prevId)
def get_wf_graph(w_list, output_dir, pdf=False): """run_and_get_results(w_list: list of (locator, version), output_dir:str, pdf:bool) Load all workflows in wf_list and dump their graph to output_dir. """ result = [] if is_running_gui(): from vistrails.gui.vistrail_controller import VistrailController as \ GUIVistrailController for locator, workflow in w_list: try: (v, abstractions, thumbnails, mashups) = load_vistrail(locator) controller = GUIVistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=False) # FIXME TE: why is this needed controller.current_pipeline_view.set_controller(controller) version = None if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) if controller.current_pipeline is not None: controller.updatePipelineScene() if pdf: base_fname = "%s_%s_pipeline.pdf" % \ (locator.short_filename, version) filename = os.path.join(output_dir, base_fname) controller.current_pipeline_scene.saveToPDF(filename) else: base_fname = "%s_%s_pipeline.png" % \ (locator.short_filename, version) filename = os.path.join(output_dir, base_fname) controller.current_pipeline_scene.saveToPNG(filename) result.append((True, "")) except Exception, e: result.append((False, debug.format_exception(e)))
def findModule(self, descriptor): moduleName = descriptor.name items = [ x for x in self.treeWidget.findItems( moduleName, QtCore.Qt.MatchExactly | QtCore.Qt.MatchWrap | QtCore.Qt.MatchRecursive) if not x.is_top_level() and x.descriptor == descriptor ] if len(items) <> 1: raise VistrailsInternalError( "Expected one item (%s), got %d: %s" % (moduleName, len(items), ";".join(x.descriptor.name for x in items))) item = items[0] return item
def getRegistryPorts(self, registry, type): if not registry: return [] if type == 'input': getter = registry.destination_ports_from_descriptor elif type == 'output': getter = registry.source_ports_from_descriptor else: raise VistrailsInternalError("Unrecognized port type '%s'", type) ports = [] try: ports = [(p.name, p.sigstring) for p in getter(self.module.module_descriptor)] except ModuleRegistryException: pass return ports
def parse_meminfo(): """parse_meminfo() -> dictionary Parses /proc/meminfo and returns appropriate dictionary. Only available on Linux.""" info = {} with open('/proc/meminfo') as fp: for line in fp: m = _meminfo_fmt.match(line) if m is None: raise VistrailsInternalError("Invalid format found in " "/proc/meminfo") key, value, unit = m.groups() if unit == 'kB': value = int(value) * 1000 else: value = int(value) info[key] = value return info
def initialize(self): if not self._loaded: raise VistrailsInternalError("Called initialize() on non-loaded " "Package %s" % self.codepath) self.check_requirements() try: name = self.prefix + self.codepath + '.init' try: __import__(name, globals(), locals(), []) except ImportError, e: # FIXME !!! Want to differentiate between .init not # existing and an error with an import in the .init # file !!! if str(e) != 'No module named init': raise else: self._init_module = self._module else:
def __init__(self, *args, **kwargs): if 'load_configuration' in kwargs: arg = kwargs['load_configuration'] if not isinstance(arg, (int, long)): if isinstance(arg, bool): if arg: kwargs['load_configuration'] = 1 else: kwargs['load_configuration'] = 0 # kwargs['load_configuration'] = 1 if arg else 0 else: raise VistrailsInternalError("Cannot convert " "load_configuration") DBPackage.__init__(self, *args, **kwargs) self.set_defaults() self._force_no_unload = None self._force_unload = None self._force_sys_unload = None self._imports_are_good = True
def late_enable_package(self, codepath, prefix_dictionary={}, needs_add=True): """late_enable_package enables a package 'late', that is, after VisTrails initialization. All dependencies need to be already enabled. """ if needs_add: if codepath in self._package_list: msg = 'duplicate package identifier: %s' % codepath raise VistrailsInternalError(msg) self.add_package(codepath) app = get_vistrails_application() pkg = self.get_package_by_codepath(codepath) try: pkg.load(prefix_dictionary.get(pkg.codepath, None)) # pkg.create_startup_package_node() except Exception, e: # invert self.add_package del self._package_list[codepath] raise
def __init__(self, module, port_type, port_name, parent=None): QtGui.QDialog.__init__(self, parent) if not module.has_port_spec(port_name, port_type): doc = None else: port_spec = module.get_port_spec(port_name, port_type) doc = port_spec.docstring() if doc is None: descriptor = module.module_descriptor # try the old method of accessing documentation if port_type == 'output': call_ = descriptor.module.provide_output_port_documentation elif port_type == 'input': call_ = descriptor.module.provide_input_port_documentation else: raise VistrailsInternalError("Invalid port type") doc = call_(port_name) self.setWindowTitle('Documentation for %s port %s in "%s"' % (port_type, port_name, module.name)) layout = QtGui.QVBoxLayout() layout.addStrut(600) layout.addWidget(QtGui.QLabel("Port name: %s" % port_name)) layout.addWidget(QtGui.QLabel("Module name: %s" % module.name)) layout.addWidget(QtGui.QLabel("Module package: %s" % \ module.package)) self.textEdit = QtGui.QTextEdit(self) layout.addWidget(self.textEdit, 1) if doc: self.textEdit.insertPlainText(doc) else: self.textEdit.insertPlainText("(Documentation not available)") self.textEdit.setReadOnly(True) self.textEdit.setTextCursor(QtGui.QTextCursor(self.textEdit.document())) self.buttonBox = QtGui.QDialogButtonBox(QtGui.QDialogButtonBox.Ok) layout.addWidget(self.buttonBox) self.setLayout(layout) self.connect(self.buttonBox, QtCore.SIGNAL('accepted()'), self.accept)
def cleanup(self): """cleanup() -> None Cleans up the file pool, by removing all temporary files and the directory they existed in. Module developers should never call this directly. """ if not os.path.isdir(self.directory): # cleanup has already happened return try: for root, dirs, files in os.walk(self.directory, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(self.directory) except OSError, e: debug.unexpected_exception(e) raise VistrailsInternalError("Can't remove %s: %s" % (self.directory, debug.format_exception(e)))
def run_and_get_results(w_list, parameters='', update_vistrail=True, extra_info=None, reason='Console Mode Execution'): """run_and_get_results(w_list: list of (locator, version), parameters: str, output_dir:str, update_vistrail: boolean, extra_info:dict) Run all workflows in w_list, and returns an interpreter result object. version can be a tag name or a version id. """ elements = parameters.split("$&$") aliases = {} params = [] result = [] for locator, workflow in w_list: (v, abstractions, thumbnails, mashups) = load_vistrail(locator) controller = VistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=update_vistrail) if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) for e in elements: pos = e.find("=") if pos != -1: key = e[:pos].strip() value = e[pos + 1:].strip() if controller.current_pipeline.has_alias(key): aliases[key] = value elif 'mashup_id' in extra_info: # new-style mashups can have aliases not existing in pipeline for mashuptrail in mashups: if mashuptrail.vtVersion == version: mashup = mashuptrail.getMashup( extra_info['mashup_id']) c = mashup.getAliasByName(key).component params.append((c.vttype, c.vtid, value)) if not update_vistrail: conf = get_vistrails_configuration() if conf.has('thumbs'): conf.thumbs.autoSave = False jobMonitor = controller.jobMonitor current_workflow = jobMonitor.currentWorkflow() if not current_workflow: for job in jobMonitor.workflows.itervalues(): try: job_version = int(job.version) except ValueError: try: job_version = v.get_version_number(job.version) except KeyError: # this is a PE or mashup continue if version == job_version: current_workflow = job jobMonitor.startWorkflow(job) if not current_workflow: current_workflow = JobWorkflow(version) jobMonitor.startWorkflow(current_workflow) try: (results, _) = \ controller.execute_current_workflow(custom_aliases=aliases, custom_params=params, extra_info=extra_info, reason=reason) finally: jobMonitor.finishWorkflow() new_version = controller.current_version if new_version != version: debug.log("Version '%s' (%s) was upgraded. The actual " "version executed was %s" % (workflow, version, new_version)) run = results[0] run.workflow_info = (locator.name, new_version) run.pipeline = controller.current_pipeline if update_vistrail: controller.write_vistrail(locator) result.append(run) if current_workflow.jobs: if current_workflow.completed(): run.job = "COMPLETED" else: run.job = "RUNNING: %s" % current_workflow.id for job in current_workflow.jobs.itervalues(): if not job.finished: run.job += "\n %s %s %s" % (job.start, job.name, job.description()) print run.job return result
def updateVistrail(self): msg = "Must implement updateVistrail in subclass" raise VistrailsInternalError(msg)
def __init__(self, *args, **kwargs): signature = None if 'signature' in kwargs: signature = kwargs['signature'] del kwargs['signature'] sigstring = None if 'sigstring' in kwargs: sigstring = kwargs['sigstring'] del kwargs['sigstring'] defaults = None if 'defaults' in kwargs: defaults = kwargs['defaults'] del kwargs['defaults'] labels = None if 'labels' in kwargs: labels = kwargs['labels'] del kwargs['labels'] values = None if 'values' in kwargs: values = kwargs['values'] del kwargs['values'] entry_types = None if 'entry_types' in kwargs: entry_types = kwargs['entry_types'] del kwargs['entry_types'] if 'items' in kwargs and 'portSpecItems' not in kwargs: kwargs['portSpecItems'] = kwargs['items'] del kwargs['items'] if 'optional' not in kwargs: kwargs['optional'] = 0 # False elif not isinstance(kwargs['optional'], (int, long)): if isinstance(kwargs['optional'], bool): if kwargs['optional']: kwargs['optional'] = 1 else: kwargs['optional'] = 0 else: raise VistrailsInternalError("Cannot parse 'optional' kw " "-- must be an int or bool") if 'min_conns' not in kwargs: kwargs['min_conns'] = 0 elif kwargs['optional'] == 1 and kwargs['min_conns'] > 0: raise VistrailsInternalError("A mandatory port cannot be set " "to optional") if 'max_conns' not in kwargs: kwargs['max_conns'] = -1 if kwargs['min_conns'] >= 0 and kwargs['max_conns'] >= 0 and \ kwargs['min_conns'] > kwargs['max_conns']: raise VistrailsInternalError("Minimum number of connections " "cannot be greater than maximum " "number of connections") if 'sort_key' not in kwargs: kwargs['sort_key'] = -1 if 'depth' not in kwargs: kwargs['depth'] = 0 if 'id' not in kwargs: kwargs['id'] = -1 if 'tooltip' in kwargs: self._tooltip = kwargs['tooltip'] del kwargs['tooltip'] else: self._tooltip = None if 'docstring' in kwargs: self._docstring = kwargs['docstring'] del kwargs['docstring'] else: self._docstring = None if 'shape' in kwargs: self._shape = kwargs['shape'] del kwargs['shape'] else: self._shape = None DBPortSpec.__init__(self, *args, **kwargs) if sum(1 for container in (self.port_spec_items, signature, sigstring) if container) > 1: raise ValueError("Please specify only one of portSpecItems," " signature, or sigstring kwargs.") self.create_spec_items(self.port_spec_items, signature, sigstring, defaults, labels, values, entry_types) self._short_sigstring = None # if signature is not None: # self.create_entries(signature) # if not self.sigstring and self._entries is not None: # # create sigstring from entries # self.create_sigstring_and_descriptors() # DAKOOP: removed this---we will check in module_registry and pipeline # validation, this way, we can let errors go all the way up # elif self._entries is None and self.sigstring: # # create entries from sigstring # self.create_entries_and_descriptors() # else: # raise VistrailsInternalError("Need to specify signature or " # "sigstring to create PortSpec") # if self._entries is not None and self._tooltip is None: # self.create_tooltip() self.is_valid = True
def compute(self): # Check required attributes if not hasattr(self, 'pipeline') or self.pipeline is None: raise VistrailsInternalError( "%s cannot execute -- pipeline doesn't exist" % self.__class__.__name__) elif (not hasattr(self, 'output_remap') or self.output_remap is None or not hasattr(self, 'input_remap') or self.input_remap is None): raise VistrailsInternalError( "%s cannot execute -- remap dictionaries don't exist" % self.__class__.__name__) # Setup pipeline for execution res = self.interpreter.setup_pipeline(self.pipeline) self.persistent_modules = res[0].values() if len(res[5]) > 0: raise ModuleError(self, "Error(s) inside group:\n" + "\n".join(me.msg for me in res[5].itervalues())) tmp_id_to_module_map = res[0] # Connect Group's external input ports to internal InputPort modules for iport_name, conn in self.inputPorts.iteritems(): iport_module = self.input_remap[iport_name] iport_obj = tmp_id_to_module_map[iport_module.id] iport_obj.set_input_port('ExternalPipe', conn[0]) # Execute pipeline kwargs = {'logger': self.logging.log.recursing(self), 'clean_pipeline': True, 'current_version': self.moduleInfo['version']} module_info_args = set(['locator', 'reason', 'extra_info', 'actions']) for arg in module_info_args: if arg in self.moduleInfo: kwargs[arg] = self.moduleInfo[arg] res = self.interpreter.execute_pipeline(self.pipeline, *res[:2], **kwargs) # Check and propagate errors if len(res[2]) > 0: raise ModuleError(self, "Error(s) inside group:\n" + "\n".join("%s: %s" % ( me.module.__class__.__name__, me.msg) for me in res[2].itervalues())) # Check and propagate ModuleSuspended exceptions if res[4]: message = "\n".join([ms.msg for ms in res[4].itervalues()]) children = list(res[4].values()) raise ModuleSuspended(self, message, children=children) # Connect internal OutputPort modules to Group's external output ports for oport_name, oport_module in self.output_remap.iteritems(): if oport_name is not 'self': oport_obj = tmp_id_to_module_map[oport_module.id] self.set_output(oport_name, oport_obj.get_output('ExternalPipe')) self.interpreter.finalize_pipeline(self.pipeline, *res[:-1], reset_computed=False)
def delete_port_spec(self, port_spec): raise VistrailsInternalError( "Cannot delete port spec from abstraction")
def resetTriggered(self): msg = "Must implement saveTriggered in subclass" raise VistrailsInternalError(msg)
def get_package_manager(): if not _package_manager: raise VistrailsInternalError("package manager not constructed yet.") return _package_manager
"Initialization of package <codepath %s> " "failed and will be disabled" % package.codepath, e) # We disable the package manually to skip over things # we know will not be necessary - the only thing needed is # the reference in the package list self._startup.set_package_to_disabled(package.codepath) failed.append(package) else: if package.identifier not in self._package_versions: self._package_versions[package.identifier] = {} self._dependency_graph.add_vertex(package.identifier) elif package.version in \ self._package_versions[package.identifier]: raise VistrailsInternalError("Duplicate package version: " "'%s' (version %s) in %s" % \ (package.identifier, package.version, package.codepath)) else: debug.warning('Duplicate package identifier: %s' % \ package.identifier) self._package_versions[package.identifier][package.version] = \ package for old_id in package.old_identifiers: self._old_identifier_map[old_id] = package.identifier for pkg in failed: del self._package_list[pkg.codepath] failed = [] # determine dependencies
vistrail: (str or Vistrail), registry: ModuleRegistry, vt_fname: str, internal_version: long, pipeline: Pipeline) -> type Creates a new VisTrails module that is a subclass of Abstraction according to the vistrail file provided and the version. The version can either be a tag (string) or an id (long) """ if isinstance(vistrail, basestring): vt_fname = vistrail vistrail = read_vistrail(vistrail) elif vt_fname is None: raise VistrailsInternalError("Abstraction must provide " "vt_fname with vistrail") if internal_version == -1L: internal_version = vistrail.get_latest_version() action = vistrail.actionMap[internal_version] if pipeline is None: pipeline = vistrail.getPipeline(internal_version) # try to make the subworkflow work with the package versions we have pipeline.validate() uuid = get_cur_abs_namespace(vistrail) if vistrail.has_notes(action.id): docstring = vistrail.get_notes(action.id) else: docstring = None
def add_port_spec(self, port_spec): raise VistrailsInternalError("Cannot add port spec to abstraction")