def coalesce_port_specs(neighbors, type):
reg = module_registry.get_module_registry()
cur_descs = None
cur_depth = 0
Variant_desc = reg.get_descriptor_by_name(basic_pkg, 'Variant')
if type == 'input':
find_common = reg.find_descriptor_subclass
common_desc = Variant_desc
elif type == 'output':
find_common = reg.find_descriptor_superclass
common_desc = reg.get_descriptor_by_name(basic_pkg, 'Module')
else:
raise VistrailsInternalError("Cannot understand type '%s'" % type)
for (module, port_name) in neighbors:
if cur_descs is None:
port_spec = module.get_port_spec(port_name, type)
cur_descs = port_spec.descriptors()
cur_depth = port_spec.depth
else:
next_port_spec = module.get_port_spec(port_name, type)
next_descs = next_port_spec.descriptors()
next_depth = next_port_spec.depth
if len(cur_descs) != len(next_descs):
raise VistrailsInternalError("Cannot have single port "
"connect to incompatible "
"types")
if 'input' == type and cur_depth > next_depth:
# use least depth
cur_depth = next_depth
cur_descs = next_descs
if 'output' == type and cur_depth < next_depth:
# use least depth
cur_depth = next_depth
cur_descs = next_descs
descs = []
for cur_desc, next_desc in izip(cur_descs, next_descs):
if cur_desc is Variant_desc:
new_desc = next_desc
elif next_desc is Variant_desc:
new_desc = cur_desc
else:
new_desc = find_common(cur_desc, next_desc)
if new_desc is None:
new_desc = common_desc
descs.append(new_desc)
cur_descs = descs
if cur_descs:
sigstring = '(' + ','.join(d.sigstring for d in cur_descs) + ')'
else:
sigstring = None
return (sigstring, cur_depth)
def get_port_spec_info(pipeline, module):
type_map = {'OutputPort': 'output', 'InputPort': 'input'}
try:
type = type_map[module.name]
except KeyError:
raise VistrailsInternalError("cannot translate type '%s'" % module.name)
if type == 'input':
get_edges = pipeline.graph.edges_from
get_port_name = \
lambda x: pipeline.connections[x].destination.name
else: # type == 'output'
get_edges = pipeline.graph.edges_to
get_port_name = \
lambda x: pipeline.connections[x].source.name
# conns = get_edges(module.id)
# for i, m in pipeline.modules.iteritems():
# print i, m.name
# for j, c in pipeline.connections.iteritems():
# print j, c.source.moduleId, c.destination.moduleId
neighbors = [(pipeline.modules[m_id], get_port_name(c_id))
for (m_id, c_id) in get_edges(module.id)]
port_name = neighbors[0][1]
sigstring, depth = coalesce_port_specs(neighbors, type)
# sigstring = neighbor.get_port_spec(port_name, type).sigstring
# FIXME check old registry here?
port_optional = False
for function in module.functions:
if function.name == 'name':
port_name = function.params[0].strValue
if function.name == 'optional':
port_optional = function.params[0].strValue == 'True'
# print 'psi:', port_name, old_name, sigstring
return (port_name, sigstring, port_optional, depth, neighbors)
def delete_output_port(self, name):
key = (name, 'output')
if key in self.port_specs:
self.delete_port_spec(self.port_specs[key])
else:
raise VistrailsInternalError("delete_output_port called on "
"nonexistent port '%s'" % name)
def __init__(self, registry, startup):
"""__init__(configuration: ConfigurationObject) -> PackageManager
configuration is the persistent configuration object of the application.
"""
global _package_manager
if _package_manager:
m = "Package manager can only be constructed once."
raise VistrailsInternalError(m)
_package_manager = self
self._registry = registry
self._startup = startup
# Contains packages that have not yet been enabled, but exist on the
# filesystem
self._available_packages = {} # codepath: str -> Package
# These other lists contain enabled packages
self._package_list = {} # codepath: str -> Package
self._package_versions = {} # identifier: str -> version -> Package
self._old_identifier_map = {} # old_id: str -> new_id: str
self._dependency_graph = vistrails.core.data_structures.graph.Graph()
self._default_prefix_dict = \
{'basic_modules': 'vistrails.core.modules.',
'abstraction': 'vistrails.core.modules.'}
self._userpackages = None
self._packages = None
self._abstraction_pkg = None
self._currently_importing_package = None
# Setup a global __import__ hook that calls Package#import_override()
# for all imports executed from that package
import __builtin__
self._orig_import = __builtin__.__import__
__builtin__.__import__ = self._import_override
# Compute the list of available packages, _available_packages
self.build_available_package_names_list()
if get_vistrails_configuration().loadPackages:
for pkg in self._startup.enabled_packages.itervalues():
self.add_package(pkg.name, prefix=pkg.prefix)
else:
try:
basic_pkg = self._startup.enabled_packages['basic_modules']
except KeyError:
pass
else:
self.add_package(basic_pkg.name, prefix=basic_pkg.prefix)
try:
abs_pkg = self._startup.enabled_packages['abstraction']
except KeyError:
pass
else:
self.add_package(abs_pkg.name, prefix=abs_pkg.prefix)
def add_connection(self, conn):
"""add_connection(conn: ExtConnection) -> None
Adds a connection to the list
"""
if self.__connections.has_key(conn.id):
msg = "External Connection '%s' with repeated id" % conn.name
raise VistrailsInternalError(msg)
self.__connections[conn.id] = conn
self.current_id = max(self.current_id, conn.id+1)
self.serialize()
def get_directory(self):
thumbnail_dir = system.get_vistrails_directory('thumbs.cacheDir')
if thumbnail_dir is not None:
if not os.path.exists(thumbnail_dir):
raise VistrailsInternalError("Cannot find %s" % thumbnail_dir)
return thumbnail_dir
# raise VistrailsInternalError("'thumbs.cacheDir' not"
# " specified in configuration")
if self._temp_directory is None:
self._temp_directory = tempfile.mkdtemp(prefix='vt_thumbs_')
return self._temp_directory
def addVersion(self, action):
""" addVersion(action: Action) -> None
Adds new version to vistrail
"""
if action.timestep in self.actionMap:
raise VistrailsInternalError("existing timestep: %d" %
action.timestep)
self.db_add_action(action)
self.changed = True
# signal to update explicit tree
self.tree.addVersion(action.id, action.prevId)
def get_wf_graph(w_list, output_dir, pdf=False):
"""run_and_get_results(w_list: list of (locator, version),
output_dir:str, pdf:bool)
Load all workflows in wf_list and dump their graph to output_dir.
"""
result = []
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator, workflow in w_list:
try:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v,
locator,
abstractions,
thumbnails,
mashups,
auto_save=False)
# FIXME TE: why is this needed
controller.current_pipeline_view.set_controller(controller)
version = None
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
if controller.current_pipeline is not None:
controller.updatePipelineScene()
if pdf:
base_fname = "%s_%s_pipeline.pdf" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
base_fname = "%s_%s_pipeline.png" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPNG(filename)
result.append((True, ""))
except Exception, e:
result.append((False, debug.format_exception(e)))
def findModule(self, descriptor):
moduleName = descriptor.name
items = [
x for x in self.treeWidget.findItems(
moduleName, QtCore.Qt.MatchExactly | QtCore.Qt.MatchWrap
| QtCore.Qt.MatchRecursive)
if not x.is_top_level() and x.descriptor == descriptor
]
if len(items) <> 1:
raise VistrailsInternalError(
"Expected one item (%s), got %d: %s" %
(moduleName, len(items), ";".join(x.descriptor.name
for x in items)))
item = items[0]
return item
def getRegistryPorts(self, registry, type):
if not registry:
return []
if type == 'input':
getter = registry.destination_ports_from_descriptor
elif type == 'output':
getter = registry.source_ports_from_descriptor
else:
raise VistrailsInternalError("Unrecognized port type '%s'", type)
ports = []
try:
ports = [(p.name, p.sigstring)
for p in getter(self.module.module_descriptor)]
except ModuleRegistryException:
pass
return ports
def parse_meminfo():
"""parse_meminfo() -> dictionary
Parses /proc/meminfo and returns appropriate dictionary. Only available on
Linux."""
info = {}
with open('/proc/meminfo') as fp:
for line in fp:
m = _meminfo_fmt.match(line)
if m is None:
raise VistrailsInternalError("Invalid format found in "
"/proc/meminfo")
key, value, unit = m.groups()
if unit == 'kB':
value = int(value) * 1000
else:
value = int(value)
info[key] = value
return info
def initialize(self):
if not self._loaded:
raise VistrailsInternalError("Called initialize() on non-loaded "
"Package %s" % self.codepath)
self.check_requirements()
try:
name = self.prefix + self.codepath + '.init'
try:
__import__(name, globals(), locals(), [])
except ImportError, e:
# FIXME !!! Want to differentiate between .init not
# existing and an error with an import in the .init
# file !!!
if str(e) != 'No module named init':
raise
else:
self._init_module = self._module
else:
def __init__(self, *args, **kwargs):
if 'load_configuration' in kwargs:
arg = kwargs['load_configuration']
if not isinstance(arg, (int, long)):
if isinstance(arg, bool):
if arg:
kwargs['load_configuration'] = 1
else:
kwargs['load_configuration'] = 0
# kwargs['load_configuration'] = 1 if arg else 0
else:
raise VistrailsInternalError("Cannot convert "
"load_configuration")
DBPackage.__init__(self, *args, **kwargs)
self.set_defaults()
self._force_no_unload = None
self._force_unload = None
self._force_sys_unload = None
self._imports_are_good = True
def late_enable_package(self, codepath, prefix_dictionary={},
needs_add=True):
"""late_enable_package enables a package 'late', that is,
after VisTrails initialization. All dependencies need to be
already enabled.
"""
if needs_add:
if codepath in self._package_list:
msg = 'duplicate package identifier: %s' % codepath
raise VistrailsInternalError(msg)
self.add_package(codepath)
app = get_vistrails_application()
pkg = self.get_package_by_codepath(codepath)
try:
pkg.load(prefix_dictionary.get(pkg.codepath, None))
# pkg.create_startup_package_node()
except Exception, e:
# invert self.add_package
del self._package_list[codepath]
raise
def __init__(self, module, port_type, port_name, parent=None):
QtGui.QDialog.__init__(self, parent)
if not module.has_port_spec(port_name, port_type):
doc = None
else:
port_spec = module.get_port_spec(port_name, port_type)
doc = port_spec.docstring()
if doc is None:
descriptor = module.module_descriptor
# try the old method of accessing documentation
if port_type == 'output':
call_ = descriptor.module.provide_output_port_documentation
elif port_type == 'input':
call_ = descriptor.module.provide_input_port_documentation
else:
raise VistrailsInternalError("Invalid port type")
doc = call_(port_name)
self.setWindowTitle('Documentation for %s port %s in "%s"' %
(port_type, port_name, module.name))
layout = QtGui.QVBoxLayout()
layout.addStrut(600)
layout.addWidget(QtGui.QLabel("Port name: %s" % port_name))
layout.addWidget(QtGui.QLabel("Module name: %s" % module.name))
layout.addWidget(QtGui.QLabel("Module package: %s" % \
module.package))
self.textEdit = QtGui.QTextEdit(self)
layout.addWidget(self.textEdit, 1)
if doc:
self.textEdit.insertPlainText(doc)
else:
self.textEdit.insertPlainText("(Documentation not available)")
self.textEdit.setReadOnly(True)
self.textEdit.setTextCursor(QtGui.QTextCursor(self.textEdit.document()))
self.buttonBox = QtGui.QDialogButtonBox(QtGui.QDialogButtonBox.Ok)
layout.addWidget(self.buttonBox)
self.setLayout(layout)
self.connect(self.buttonBox, QtCore.SIGNAL('accepted()'), self.accept)
def cleanup(self):
"""cleanup() -> None
Cleans up the file pool, by removing all temporary files and
the directory they existed in. Module developers should never
call this directly.
"""
if not os.path.isdir(self.directory):
# cleanup has already happened
return
try:
for root, dirs, files in os.walk(self.directory, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
os.rmdir(self.directory)
except OSError, e:
debug.unexpected_exception(e)
raise VistrailsInternalError("Can't remove %s: %s" %
(self.directory,
debug.format_exception(e)))
def run_and_get_results(w_list,
parameters='',
update_vistrail=True,
extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
output_dir:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
params = []
result = []
for locator, workflow in w_list:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(v,
locator,
abstractions,
thumbnails,
mashups,
auto_save=update_vistrail)
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos + 1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
elif 'mashup_id' in extra_info:
# new-style mashups can have aliases not existing in pipeline
for mashuptrail in mashups:
if mashuptrail.vtVersion == version:
mashup = mashuptrail.getMashup(
extra_info['mashup_id'])
c = mashup.getAliasByName(key).component
params.append((c.vttype, c.vtid, value))
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
jobMonitor = controller.jobMonitor
current_workflow = jobMonitor.currentWorkflow()
if not current_workflow:
for job in jobMonitor.workflows.itervalues():
try:
job_version = int(job.version)
except ValueError:
try:
job_version = v.get_version_number(job.version)
except KeyError:
# this is a PE or mashup
continue
if version == job_version:
current_workflow = job
jobMonitor.startWorkflow(job)
if not current_workflow:
current_workflow = JobWorkflow(version)
jobMonitor.startWorkflow(current_workflow)
try:
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
custom_params=params,
extra_info=extra_info,
reason=reason)
finally:
jobMonitor.finishWorkflow()
new_version = controller.current_version
if new_version != version:
debug.log("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" %
(workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
if current_workflow.jobs:
if current_workflow.completed():
run.job = "COMPLETED"
else:
run.job = "RUNNING: %s" % current_workflow.id
for job in current_workflow.jobs.itervalues():
if not job.finished:
run.job += "\n %s %s %s" % (job.start, job.name,
job.description())
print run.job
return result
def updateVistrail(self):
msg = "Must implement updateVistrail in subclass"
raise VistrailsInternalError(msg)
def __init__(self, *args, **kwargs):
signature = None
if 'signature' in kwargs:
signature = kwargs['signature']
del kwargs['signature']
sigstring = None
if 'sigstring' in kwargs:
sigstring = kwargs['sigstring']
del kwargs['sigstring']
defaults = None
if 'defaults' in kwargs:
defaults = kwargs['defaults']
del kwargs['defaults']
labels = None
if 'labels' in kwargs:
labels = kwargs['labels']
del kwargs['labels']
values = None
if 'values' in kwargs:
values = kwargs['values']
del kwargs['values']
entry_types = None
if 'entry_types' in kwargs:
entry_types = kwargs['entry_types']
del kwargs['entry_types']
if 'items' in kwargs and 'portSpecItems' not in kwargs:
kwargs['portSpecItems'] = kwargs['items']
del kwargs['items']
if 'optional' not in kwargs:
kwargs['optional'] = 0 # False
elif not isinstance(kwargs['optional'], (int, long)):
if isinstance(kwargs['optional'], bool):
if kwargs['optional']:
kwargs['optional'] = 1
else:
kwargs['optional'] = 0
else:
raise VistrailsInternalError("Cannot parse 'optional' kw "
"-- must be an int or bool")
if 'min_conns' not in kwargs:
kwargs['min_conns'] = 0
elif kwargs['optional'] == 1 and kwargs['min_conns'] > 0:
raise VistrailsInternalError("A mandatory port cannot be set "
"to optional")
if 'max_conns' not in kwargs:
kwargs['max_conns'] = -1
if kwargs['min_conns'] >= 0 and kwargs['max_conns'] >= 0 and \
kwargs['min_conns'] > kwargs['max_conns']:
raise VistrailsInternalError("Minimum number of connections "
"cannot be greater than maximum "
"number of connections")
if 'sort_key' not in kwargs:
kwargs['sort_key'] = -1
if 'depth' not in kwargs:
kwargs['depth'] = 0
if 'id' not in kwargs:
kwargs['id'] = -1
if 'tooltip' in kwargs:
self._tooltip = kwargs['tooltip']
del kwargs['tooltip']
else:
self._tooltip = None
if 'docstring' in kwargs:
self._docstring = kwargs['docstring']
del kwargs['docstring']
else:
self._docstring = None
if 'shape' in kwargs:
self._shape = kwargs['shape']
del kwargs['shape']
else:
self._shape = None
DBPortSpec.__init__(self, *args, **kwargs)
if sum(1 for container in (self.port_spec_items, signature, sigstring)
if container) > 1:
raise ValueError("Please specify only one of portSpecItems,"
" signature, or sigstring kwargs.")
self.create_spec_items(self.port_spec_items, signature, sigstring,
defaults, labels, values, entry_types)
self._short_sigstring = None
# if signature is not None:
# self.create_entries(signature)
# if not self.sigstring and self._entries is not None:
# # create sigstring from entries
# self.create_sigstring_and_descriptors()
# DAKOOP: removed this---we will check in module_registry and pipeline
# validation, this way, we can let errors go all the way up
# elif self._entries is None and self.sigstring:
# # create entries from sigstring
# self.create_entries_and_descriptors()
# else:
# raise VistrailsInternalError("Need to specify signature or "
# "sigstring to create PortSpec")
# if self._entries is not None and self._tooltip is None:
# self.create_tooltip()
self.is_valid = True
def compute(self):
# Check required attributes
if not hasattr(self, 'pipeline') or self.pipeline is None:
raise VistrailsInternalError(
"%s cannot execute -- pipeline doesn't exist" %
self.__class__.__name__)
elif (not hasattr(self, 'output_remap') or self.output_remap is None or
not hasattr(self, 'input_remap') or self.input_remap is None):
raise VistrailsInternalError(
"%s cannot execute -- remap dictionaries don't exist" %
self.__class__.__name__)
# Setup pipeline for execution
res = self.interpreter.setup_pipeline(self.pipeline)
self.persistent_modules = res[0].values()
if len(res[5]) > 0:
raise ModuleError(self, "Error(s) inside group:\n" +
"\n".join(me.msg for me in res[5].itervalues()))
tmp_id_to_module_map = res[0]
# Connect Group's external input ports to internal InputPort modules
for iport_name, conn in self.inputPorts.iteritems():
iport_module = self.input_remap[iport_name]
iport_obj = tmp_id_to_module_map[iport_module.id]
iport_obj.set_input_port('ExternalPipe', conn[0])
# Execute pipeline
kwargs = {'logger': self.logging.log.recursing(self),
'clean_pipeline': True,
'current_version': self.moduleInfo['version']}
module_info_args = set(['locator', 'reason', 'extra_info', 'actions'])
for arg in module_info_args:
if arg in self.moduleInfo:
kwargs[arg] = self.moduleInfo[arg]
res = self.interpreter.execute_pipeline(self.pipeline,
*res[:2],
**kwargs)
# Check and propagate errors
if len(res[2]) > 0:
raise ModuleError(self, "Error(s) inside group:\n" +
"\n".join("%s: %s" % (
me.module.__class__.__name__, me.msg)
for me in res[2].itervalues()))
# Check and propagate ModuleSuspended exceptions
if res[4]:
message = "\n".join([ms.msg for ms in res[4].itervalues()])
children = list(res[4].values())
raise ModuleSuspended(self, message, children=children)
# Connect internal OutputPort modules to Group's external output ports
for oport_name, oport_module in self.output_remap.iteritems():
if oport_name is not 'self':
oport_obj = tmp_id_to_module_map[oport_module.id]
self.set_output(oport_name,
oport_obj.get_output('ExternalPipe'))
self.interpreter.finalize_pipeline(self.pipeline, *res[:-1],
reset_computed=False)
def delete_port_spec(self, port_spec):
raise VistrailsInternalError(
"Cannot delete port spec from abstraction")
def resetTriggered(self):
msg = "Must implement saveTriggered in subclass"
raise VistrailsInternalError(msg)
def get_package_manager():
if not _package_manager:
raise VistrailsInternalError("package manager not constructed yet.")
return _package_manager
"Initialization of package <codepath %s> "
"failed and will be disabled" % package.codepath, e)
# We disable the package manually to skip over things
# we know will not be necessary - the only thing needed is
# the reference in the package list
self._startup.set_package_to_disabled(package.codepath)
failed.append(package)
else:
if package.identifier not in self._package_versions:
self._package_versions[package.identifier] = {}
self._dependency_graph.add_vertex(package.identifier)
elif package.version in \
self._package_versions[package.identifier]:
raise VistrailsInternalError("Duplicate package version: "
"'%s' (version %s) in %s" % \
(package.identifier,
package.version,
package.codepath))
else:
debug.warning('Duplicate package identifier: %s' % \
package.identifier)
self._package_versions[package.identifier][package.version] = \
package
for old_id in package.old_identifiers:
self._old_identifier_map[old_id] = package.identifier
for pkg in failed:
del self._package_list[pkg.codepath]
failed = []
# determine dependencies
vistrail: (str or Vistrail),
registry: ModuleRegistry,
vt_fname: str,
internal_version: long,
pipeline: Pipeline) -> type
Creates a new VisTrails module that is a subclass of Abstraction
according to the vistrail file provided and the version. The version
can either be a tag (string) or an id (long)
"""
if isinstance(vistrail, basestring):
vt_fname = vistrail
vistrail = read_vistrail(vistrail)
elif vt_fname is None:
raise VistrailsInternalError("Abstraction must provide "
"vt_fname with vistrail")
if internal_version == -1L:
internal_version = vistrail.get_latest_version()
action = vistrail.actionMap[internal_version]
if pipeline is None:
pipeline = vistrail.getPipeline(internal_version)
# try to make the subworkflow work with the package versions we have
pipeline.validate()
uuid = get_cur_abs_namespace(vistrail)
if vistrail.has_notes(action.id):
docstring = vistrail.get_notes(action.id)
else:
docstring = None
def add_port_spec(self, port_spec):
raise VistrailsInternalError("Cannot add port spec to abstraction")