def get_directory(self):
if self.conf.check('cacheDirectory'):
thumbnail_dir = self.conf.cacheDirectory
if not os.path.exists(thumbnail_dir):
raise VistrailsInternalError("Cannot find %s" % thumbnail_dir)
return thumbnail_dir
raise VistrailsInternalError("'thumbs.cacheDirectory' not"
" specified in configuration")
return None
def create_module_function(self):
"""create_module_function() -> ModuleFunction
creates a ModuleFunction object from self.
"""
from core.vistrail.module_function import ModuleFunction
from core.vistrail.module_param import ModuleParam
def from_source_port():
f = ModuleFunction()
f.name = self.name
if len(self._descriptors) > 0:
f.returnType = self._descriptors[0].name
return f
def from_destination_port():
f = ModuleFunction()
f.name = self.name
for descriptor in self._descriptors:
p = ModuleParam()
p.identifier = descriptor.identifier
p.namespace = descriptor.namespace
p.type = descriptor.name
p.name = '<no description>'
f.addParameter(p)
return f
if self.type == 'output':
return from_source_port()
elif self.type == 'input':
return from_destination_port()
else:
raise VistrailsInternalError("Was expecting a valid endpoint")
def __init__(self, descriptor, port_type, port_name, parent=None):
QtGui.QDialog.__init__(self, parent)
self.descriptor = descriptor
self.setModal(True)
if port_type == 'output':
call_ = descriptor.module.provide_output_port_documentation
elif port_type == 'input':
call_ = descriptor.module.provide_input_port_documentation
else:
raise VistrailsInternalError("Invalid port type")
self.setWindowTitle('Documentation for %s port %s in "%s"' %
(port_type, port_name, descriptor.name))
self.setLayout(QtGui.QVBoxLayout())
self.layout().addStrut(600)
self.layout().addWidget(QtGui.QLabel("Port name: %s" % port_name))
self.layout().addWidget(QtGui.QLabel("Module name: %s" % descriptor.name))
package = descriptor.module_package()
self.layout().addWidget(QtGui.QLabel("Module package: %s" % package))
self.closeButton = QtGui.QPushButton('Ok', self)
self.textEdit = QtGui.QTextEdit(self)
self.layout().addWidget(self.textEdit, 1)
doc = call_(port_name)
if doc:
self.textEdit.insertPlainText(doc)
else:
self.textEdit.insertPlainText("Documentation not available.")
self.textEdit.setReadOnly(True)
self.textEdit.setTextCursor(QtGui.QTextCursor(self.textEdit.document()))
self.layout().addWidget(self.closeButton)
self.connect(self.closeButton, QtCore.SIGNAL('clicked(bool)'), self.close)
self.closeButton.setShortcut('Enter')
def delete_output_port(self, name):
key = (name, 'output')
if key in self.port_specs:
self.delete_port_spec(self.port_specs[key])
else:
raise VistrailsInternalError("delete_output_port called on "
"nonexistent port '%s'" % name)
def compute(self):
if not hasattr(self, 'pipeline') or self.pipeline is None:
raise VistrailsInternalError("%s cannot execute--" % \
self.__class__.__name__ + \
"pipeline doesn't exist")
elif not hasattr(self, 'input_remap') or self.input_remap is None or \
not hasattr(self, 'output_remap') or self.output_remap is None:
raise VistrailsInternalError("%s cannot execute--" % \
self.__class__.__name__ + \
"remap dictionaries don't exist")
res = self.interpreter.setup_pipeline(self.pipeline)
if len(res[5]) > 0:
raise ModuleError(self, 'Error(s) inside group:\n' +
'\n'.join(me.msg for me in res[5].itervalues()))
tmp_id_to_module_map = res[0]
for iport_name, conn in self.inputPorts.iteritems():
iport_module = self.input_remap[iport_name]
iport_obj = tmp_id_to_module_map[iport_module.id]
iport_obj.set_input_port('ExternalPipe', conn[0])
kwargs = {'logger': self.logging.log, 'clean_pipeline': True,
'current_version': self.moduleInfo['version']}
module_info_args = set(['locator', 'reason', 'extra_info', 'actions'])
for arg in module_info_args:
if arg in self.moduleInfo:
kwargs[arg] = self.moduleInfo[arg]
# if hasattr(self, 'group_exec'):
# kwargs['parent_exec'] = self.group_exec
res = self.interpreter.execute_pipeline(self.pipeline, *(res[:2]),
**kwargs)
if len(res[2]) > 0:
raise ModuleError(self, 'Error(s) inside group:\n' +
'\n '.join(me.module.__class__.__name__ + ': ' + \
me.msg for me in res[2].itervalues()))
for oport_name, oport_module in self.output_remap.iteritems():
if oport_name is not 'self':
# oport_module = self.output_remap[oport_name]
oport_obj = tmp_id_to_module_map[oport_module.id]
self.setResult(oport_name, oport_obj.get_output('ExternalPipe'))
self.interpreter.finalize_pipeline(self.pipeline, *res[:-1],
**{'reset_computed': False})
def add_connection(self, conn):
"""add_connection(conn: ExtConnection) -> None
Adds a connection to the list
"""
if self.__connections.has_key(conn.id):
msg = "External Connection '%s' with repeated id" % conn.name
raise VistrailsInternalError(msg)
self.__connections[conn.id] = conn
self.current_id = max(self.current_id, conn.id+1)
self.serialize()
def coalesce_port_specs(neighbors, type):
from core.modules.basic_modules import identifier as basic_pkg
reg = module_registry.get_module_registry()
cur_descs = None
if type == 'input':
find_common = reg.find_descriptor_subclass
common_desc = reg.get_descriptor_by_name(basic_pkg, 'Variant')
elif type == 'output':
find_common = reg.find_descriptor_superclass
common_desc = reg.get_descriptor_by_name(basic_pkg, 'Module')
else:
raise VistrailsInternalError("Cannot understand type '%s'" % type)
for (module, port_name) in neighbors:
if cur_descs is None:
port_spec = module.get_port_spec(port_name, type)
cur_descs = port_spec.descriptors()
else:
next_port_spec = module.get_port_spec(port_name, type)
next_descs = next_port_spec.descriptors()
if len(cur_descs) != len(next_descs):
raise VistrailsInternalError("Cannot have single port "
"connect to incompatible "
"types")
descs = []
for cur_desc, next_desc in izip(cur_descs, next_descs):
new_desc = find_common(cur_desc, next_desc)
if new_desc is None:
new_desc = common_desc
descs.append(new_desc)
cur_descs = descs
if cur_descs:
sigstring = '(' + ','.join(d.sigstring for d in cur_descs) + ')'
else:
sigstring = None
return sigstring
def get_wf_graph(w_list, workflow_info=None, pdf=False):
"""run_and_get_results(w_list: list of (locator, version),
workflow_info:str, pdf:bool)
Load all workflows in wf_list and dump their graph to workflow_info.
"""
result = []
if is_running_gui():
from gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator, workflow in w_list:
try:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController()
if type(workflow) == type("str"):
version = v.get_version_number(workflow)
elif type(workflow) in [type(1), long]:
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
if (workflow_info is not None
and controller.current_pipeline is not None):
from gui.pipeline_view import QPipelineView
pipeline_view = QPipelineView()
controller.current_pipeline_view = pipeline_view.scene()
controller.set_vistrail(v, locator, abstractions,
thumbnails, mashups)
pipeline_view.scene().setupScene(
controller.current_pipeline)
if pdf:
base_fname = "%s_%s_pipeline.pdf" % (
locator.short_name, version)
filename = os.path.join(workflow_info, base_fname)
pipeline_view.scene().saveToPDF(filename)
else:
base_fname = "%s_%s_pipeline.png" % (
locator.short_name, version)
filename = os.path.join(workflow_info, base_fname)
pipeline_view.scene().saveToPNG(filename)
del pipeline_view
result.append((True, ""))
except Exception, e:
result.append((False, str(e)))
def __init__(self, *args, **kwargs):
signature = None
if 'signature' in kwargs:
signature = kwargs['signature']
del kwargs['signature']
if 'optional' not in kwargs:
kwargs['optional'] = 0 # False
elif type(kwargs['optional']) != type(0):
if type(kwargs['optional']) == type(True):
if kwargs['optional']:
kwargs['optional'] = 1
else:
kwargs['optional'] = 0
else:
raise VistrailsInternalError("Cannot parse 'optional' kw "
"-- must be an int or bool")
if 'sort_key' not in kwargs:
kwargs['sort_key'] = -1
if 'id' not in kwargs:
kwargs['id'] = -1
if 'tooltip' in kwargs:
self._tooltip = kwargs['tooltip']
del kwargs['tooltip']
else:
self._tooltip = None
DBPortSpec.__init__(self, *args, **kwargs)
self._entries = None
self._descriptors = None
self._short_sigstring = None
self._labels = None
self._defaults = None
if signature is not None:
self.create_entries(signature)
if not self.sigstring and self._entries is not None:
# create sigstring from entries
self.create_sigstring_and_descriptors()
# DAKOOP: removed this---we will check in module_registry and pipeline
# validation, this way, we can let errors go all the way up
# elif self._entries is None and self.sigstring:
# # create entries from sigstring
# self.create_entries_and_descriptors()
# else:
# raise VistrailsInternalError("Need to specify signature or "
# "sigstring to create PortSpec")
if self._entries is not None and self._tooltip is None:
self.create_tooltip()
self.is_valid = True
def getRegistryPorts(self, registry, type):
if not registry:
return []
if type == 'input':
getter = registry.destination_ports_from_descriptor
elif type == 'output':
getter = registry.source_ports_from_descriptor
else:
raise VistrailsInternalError("Unrecognized port type '%s'", type)
ports = []
try:
ports = [(p.name, p.sigstring)
for p in getter(self.module.module_descriptor)]
except ModuleRegistryException:
pass
return ports
def findModule(self, descriptor):
moduleName = descriptor.name
items = [x for x in
self.treeWidget.findItems(moduleName,
QtCore.Qt.MatchExactly |
QtCore.Qt.MatchWrap |
QtCore.Qt.MatchRecursive)
if not x.is_top_level() and x.descriptor == descriptor]
if len(items) <> 1:
raise VistrailsInternalError("Expected one item (%s), got %d: %s" %
(moduleName,
len(items),
";".join(x.descriptor.name
for x in items)))
item = items[0]
return item
def __init__(self, configuration):
"""__init__(configuration: ConfigurationObject) -> PackageManager
configuration is the persistent configuration object of the application.
"""
global _package_manager
if _package_manager:
m = "Package manager can only be constructed once."
raise VistrailsInternalError(m)
_package_manager = self
self._configuration = configuration
self._package_list = {}
self._package_versions = {}
self._dependency_graph = core.data_structures.graph.Graph()
self._registry = None
self._userpackages = None
self._packages = None
self._abstraction_pkg = None
def late_enable_package(self, package_codepath, prefix_dictionary={},
needs_add=True):
"""late_enable_package enables a package 'late', that is,
after VisTrails initialization. All dependencies need to be
already enabled.
"""
if needs_add:
if package_codepath in self._package_list:
msg = 'duplicate package identifier: %s' % package_codepath
raise VistrailsInternalError(msg)
self.add_package(package_codepath)
pkg = self.get_package_by_codepath(package_codepath)
try:
pkg.load(prefix_dictionary.get(pkg.codepath, None))
except Exception, e:
# invert self.add_package
del self._package_list[package_codepath]
raise
def new_port_spec(self, name, type, signature=None, sigstring=None,
optional=False, sort_key=-1):
# DEPRECATED: create using ModuleRegistry
if signature is None and sigstring is None:
raise VistrailsInternalError("new_port_spec: signature and "
"sigstring cannot both be None")
if sigstring is not None:
return PortSpec(id=-1,
name=name,
type=type,
sigstring=sigstring,
optional=optional,
sort_key=sort_key)
return PortSpec(id=-1,
name=name,
type=type,
signature=signature,
optional=optional,
sort_key=sort_key)
def cleanup(self):
"""cleanup() -> None
Cleans up the file pool, by removing all temporary files
and the directory they existed in. Module developers should never
call this directly."""
if not os.path.isdir(self.directory):
# cleanup has already happened
return
try:
for root, dirs, files in os.walk(self.directory, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
os.rmdir(self.directory)
except OSError, e:
raise VistrailsInternalError("Can't remove %s: %s" %
(self.directory, str(e)))
def get_port_spec_info(pipeline, module):
type_map = {'OutputPort': 'output', 'InputPort': 'input'}
try:
type = type_map[module.name]
except KeyError:
raise VistrailsInternalError("cannot translate type '%s'" % type)
if type == 'input':
get_edges = pipeline.graph.edges_from
get_port_name = \
lambda x: pipeline.connections[x].destination.name
elif type == 'output':
get_edges = pipeline.graph.edges_to
get_port_name = \
lambda x: pipeline.connections[x].source.name
# conns = get_edges(module.id)
# for i, m in pipeline.modules.iteritems():
# print i, m.name
# for j, c in pipeline.connections.iteritems():
# print j, c.source.moduleId, c.destination.moduleId
neighbors = [(pipeline.modules[m_id], get_port_name(c_id))
for (m_id, c_id) in get_edges(module.id)]
port_name = neighbors[0][1]
sigstring = coalesce_port_specs(neighbors, type)
old_name = port_name
# sigstring = neighbor.get_port_spec(port_name, type).sigstring
# FIXME check old registry here?
port_optional = False
for function in module.functions:
if function.name == 'name':
port_name = function.params[0].strValue
if function.name == 'optional':
port_optional = function.params[0].strValue == 'True'
# print 'psi:', port_name, old_name, sigstring
return (port_name, sigstring, port_optional, neighbors)
"failed and will be disabled" % \
package.codepath, str(e))
# We disable the package manually to skip over things
# we know will not be necessary - the only thing needed is
# the reference in the package list
package.remove_own_dom_element()
failed.append(package)
else:
if package.identifier not in self._package_versions:
self._package_versions[package.identifier] = {}
self._dependency_graph.add_vertex(package.identifier)
elif package.version in \
self._package_versions[package.identifier]:
raise VistrailsInternalError("Duplicate package version: "
"'%s' (version %s) in %s" % \
(package.identifier,
package.version,
package.codepath))
else:
debug.warning('Duplicate package identifier: %s' % \
package.identifier)
self._package_versions[package.identifier][package.version] = \
package
for pkg in failed:
del self._package_list[pkg.codepath]
failed = []
# determine dependencies
for package in self._package_list.itervalues():
try:
def resetTriggered(self):
msg = "Must implement saveTriggered in subclass"
raise VistrailsInternalError(msg)
def add_port_spec(self, port_spec):
raise VistrailsInternalError("Cannot add port spec to abstraction")
def updateVistrail(self):
msg = "Must implement updateVistrail in subclass"
raise VistrailsInternalError(msg)
def delete_port_spec(self, port_spec):
raise VistrailsInternalError(
"Cannot delete port spec from abstraction")
def run_and_get_results(w_list,
parameters='',
workflow_info=None,
update_vistrail=True,
extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
workflow_info:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
result = []
for locator, workflow in w_list:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(auto_save=update_vistrail)
controller.set_vistrail(v, locator, abstractions, thumbnails, mashups)
if type(workflow) == type("str"):
version = v.get_version_number(workflow)
elif type(workflow) in [type(1), long]:
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos + 1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
if workflow_info is not None and controller.current_pipeline is not None:
if is_running_gui():
from gui.pipeline_view import QPipelineView
pipeline_view = QPipelineView()
pipeline_view.scene().setupScene(controller.current_pipeline)
base_fname = "%s_%s_pipeline.pdf" % (locator.short_name,
version)
filename = os.path.join(workflow_info, base_fname)
pipeline_view.scene().saveToPDF(filename)
del pipeline_view
else:
debug.critical("Cannot save pipeline figure when not "
"running in gui mode")
base_fname = "%s_%s_pipeline.xml" % (locator.short_name, version)
filename = os.path.join(workflow_info, base_fname)
core.db.io.save_workflow(controller.current_pipeline, filename)
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
extra_info=extra_info,
reason=reason)
new_version = controller.current_version
if new_version != version:
debug.warning("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" % \
(workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
return result
def get_package_manager():
global _package_manager
if not _package_manager:
raise VistrailsInternalError("package manager not constructed yet.")
return _package_manager
vistrail: (str or Vistrail),
registry: ModuleRegistry,
vt_fname: str,
internal_version: long,
pipeline: Pipeline) -> type
Creates a new VisTrails module that is a subclass of Abstraction
according to the vistrail file provided and the version. The version
can either be a tag (string) or an id (long)
"""
if type(vistrail) == type(""):
vt_fname = vistrail
vistrail = read_vistrail(vistrail)
elif vt_fname is None:
raise VistrailsInternalError("Abstraction must provide "
"vt_fname with vistrail")
if internal_version == -1L:
internal_version = vistrail.get_latest_version()
action = vistrail.actionMap[internal_version]
if pipeline is None:
pipeline = vistrail.getPipeline(internal_version)
# try to make the subworkflow work with the package versions we have
pipeline.validate()
uuid = get_cur_abs_namespace(vistrail)
if vistrail.has_notes(action.id):
docstring = vistrail.get_notes(action.id)
else:
docstring = None
