def create_registry(self, registry_filename=None):
if registry_filename is not None:
registry = vistrails.core.db.io.open_registry(registry_filename)
registry.set_global()
else:
registry = ModuleRegistry()
registry.set_global()
return registry
def create_registry(self, registry_filename=None):
if registry_filename is not None:
registry = vistrails.core.db.io.open_registry(registry_filename)
registry.set_global()
else:
registry = ModuleRegistry()
registry.set_global()
return registry
def init_registry(self, registry_filename=None):
if registry_filename is not None:
self._registry = vistrails.core.db.io.open_registry(registry_filename)
self._registry.set_global()
else:
self._registry = ModuleRegistry()
self._registry.set_global()
def setup_basic_package():
# setup basic package
basic_package = self.add_package('basic_modules')
# FIXME need to serialize old_identifiers
basic_package.old_identifiers = ['edu.utah.sci.vistrails.basic']
self._registry._default_package = basic_package
prefix_dictionary = {'basic_modules': 'vistrails.core.modules.'}
self.initialize_packages(prefix_dictionary)
setup_basic_package()
self._abstraction_pkg = self.add_package('abstraction', False)
# FIXME need to get this info from the package, but cannot
# do this since controller isn't imported yet
self._abstraction_pkg.identifier = 'local.abstractions'
self._abstraction_pkg.name = 'My SubWorkflows'
self._abstraction_pkg.version = '1.6'
self._registry.add_package(self._abstraction_pkg)
# Setup a global __import__ hook that calls Package#import_override()
# for all imports executed from that package
import __builtin__
self._orig_import = __builtin__.__import__
__builtin__.__import__ = self._import_override
def diff_package(pkg_id, reg_fname1, reg_fname2):
reg1 = open_registry_from_xml(reg_fname1)
ModuleRegistry.convert(reg1)
reg2 = open_registry_from_xml(reg_fname2)
ModuleRegistry.convert(reg2)
pkg1 = reg1.get_package_by_name(pkg_id)
pkg2 = reg2.get_package_by_name(pkg_id)
d2_modules_dict = dict(((d2.identifier, d2.name, d2.namespace), d2)
for d2 in pkg2.descriptor_list)
for d1 in pkg1.descriptor_list:
if reg2.has_descriptor_with_name(pkg_id, d1.name, d1.namespace):
d2 = reg2.get_descriptor_by_name(pkg_id, d1.name, d1.namespace)
d1_port_specs = {}
for ps in d1.port_specs_list:
d1_port_specs[(ps.name, ps.type)] = ps.sigstring
d2_port_specs = {}
for ps in d2.port_specs_list:
d2_port_specs[(ps.name, ps.type)] = ps.sigstring
d2_port_specs_set = set(d2_port_specs.keys())
for ps_id, sig1 in d1_port_specs.iteritems():
if ps_id not in d2_port_specs:
print "added %s port: %s:%s" % \
(ps_id[1], get_module_name(d1), ps_id[0])
continue
d2_port_specs_set.discard(ps_id)
if sig1 != d2_port_specs[ps_id]:
print "changed %s port: %s:%s" % \
(ps_id[1], get_module_name(d1), ps_id[0])
print " %s -> %s" % (sig1, d2_port_specs[ps_id])
else:
# equal
pass
for ps_id in d2_port_specs_set:
print "deleted %s port: %s:%s" % \
(ps_id[1], get_module_name(d1), ps_id[0])
del d2_modules_dict[(d1.identifier, d1.name, d1.namespace)]
else:
print "deleted module: %s" % get_module_name(d1)
for d2 in d2_modules_dict.itervalues():
print "added module: %s" % get_module_name(d2)
def diff_package(pkg_id, reg_fname1, reg_fname2):
reg1 = open_registry_from_xml(reg_fname1)
ModuleRegistry.convert(reg1)
reg2 = open_registry_from_xml(reg_fname2)
ModuleRegistry.convert(reg2)
pkg1 = reg1.get_package_by_name(pkg_id)
pkg2 = reg2.get_package_by_name(pkg_id)
d2_modules_dict = dict(((d2.identifier, d2.name, d2.namespace), d2)
for d2 in pkg2.descriptor_list)
for d1 in pkg1.descriptor_list:
if reg2.has_descriptor_with_name(pkg_id, d1.name, d1.namespace):
d2 = reg2.get_descriptor_by_name(pkg_id, d1.name, d1.namespace)
d1_port_specs = {}
for ps in d1.port_specs_list:
d1_port_specs[(ps.name, ps.type)] = ps.sigstring
d2_port_specs = {}
for ps in d2.port_specs_list:
d2_port_specs[(ps.name, ps.type)] = ps.sigstring
d2_port_specs_set = set(d2_port_specs.keys())
for ps_id, sig1 in d1_port_specs.iteritems():
if ps_id not in d2_port_specs:
print "added %s port: %s:%s" % \
(ps_id[1], get_module_name(d1), ps_id[0])
continue
d2_port_specs_set.discard(ps_id)
if sig1 != d2_port_specs[ps_id]:
print "changed %s port: %s:%s" % \
(ps_id[1], get_module_name(d1), ps_id[0])
print " %s -> %s" % (sig1, d2_port_specs[ps_id])
else:
# equal
pass
for ps_id in d2_port_specs_set:
print "deleted %s port: %s:%s" % \
(ps_id[1], get_module_name(d1), ps_id[0])
del d2_modules_dict[(d1.identifier, d1.name, d1.namespace)]
else:
print "deleted module: %s" % get_module_name(d1)
for d2 in d2_modules_dict.itervalues():
print "added module: %s" % get_module_name(d2)
def open_registry(filename):
from vistrails.core.modules.module_registry import ModuleRegistry
registry = vistrails.db.services.io.open_registry_from_xml(filename)
ModuleRegistry.convert(registry)
return registry
class PackageManager(object):
# # add_package_menu_signal is emitted with a tuple containing the package
# # identifier, package name and the menu item
# add_package_menu_signal = QtCore.SIGNAL("add_package_menu")
# # remove_package_menu_signal is emitted with the package identifier
# remove_package_menu_signal = QtCore.SIGNAL("remove_package_menu")
# # package_error_message_signal is emitted with the package identifier,
# # package name and the error message
# package_error_message_signal = QtCore.SIGNAL("package_error_message_signal")
# # reloading_package_signal is emitted when a package reload has disabled
# # the packages, but has not yet enabled them
# reloading_package_signal = QtCore.SIGNAL("reloading_package_signal")
class DependencyCycle(Exception):
def __init__(self, p1, p2):
self._package_1 = p1
self._package_2 = p2
def __str__(self):
return ("Packages '%s' and '%s' have cyclic dependencies" %
(self._package_1,
self._package_2))
class PackageInternalError(Exception):
def __init__(self, n, d):
self._package_name = n
self._description = d
def __str__(self):
return "Package '%s' has a bug: %s" % (self._package_name,
self._description)
def import_packages_module(self):
"""Imports the packages module using path trickery to find it
in the right place.
"""
if self._packages is not None:
return self._packages
# Imports standard packages directory
conf = self._configuration
old_sys_path = copy.copy(sys.path)
if conf.check('packageDirectory'):
sys.path.insert(0, conf.packageDirectory)
try:
import vistrails.packages
except ImportError:
debug.critical('ImportError: "packages" sys.path: %s' % sys.path)
raise
finally:
sys.path = old_sys_path
self._packages = vistrails.packages
return vistrails.packages
def import_user_packages_module(self):
"""Imports the packages module using path trickery to find it
in the right place.
"""
if self._userpackages is not None:
return self._userpackages
# Imports user packages directory
conf = self._configuration
old_sys_path = copy.copy(sys.path)
if conf.check('userPackageDirectory'):
sys.path.insert(0, os.path.join(conf.userPackageDirectory,
os.path.pardir))
try:
import userpackages
except ImportError:
debug.critical('ImportError: "userpackages" sys.path: %s' % sys.path)
raise
finally:
sys.path = old_sys_path
os.environ['VISTRAILS_USERPACKAGES_DIR'] = conf.userPackageDirectory
self._userpackages = userpackages
return userpackages
def __init__(self, configuration):
"""__init__(configuration: ConfigurationObject) -> PackageManager
configuration is the persistent configuration object of the application.
"""
global _package_manager
if _package_manager:
m = "Package manager can only be constructed once."
raise VistrailsInternalError(m)
_package_manager = self
self._configuration = configuration
# Contains packages that have not yet been enabled, but exist on the
# filesystem
self._available_packages = {} # codepath: str -> Package
# These other lists contain enabled packages
self._package_list = {} # codepath: str -> Package
self._package_versions = {} # identifier: str -> version -> Package
self._old_identifier_map = {} # old_id: str -> new_id: str
self._dependency_graph = vistrails.core.data_structures.graph.Graph()
self._registry = None
self._userpackages = None
self._packages = None
self._abstraction_pkg = None
self._currently_importing_package = None
def init_registry(self, registry_filename=None):
if registry_filename is not None:
self._registry = vistrails.core.db.io.open_registry(registry_filename)
self._registry.set_global()
else:
self._registry = ModuleRegistry()
self._registry.set_global()
def setup_basic_package():
# setup basic package
basic_package = self.add_package('basic_modules')
# FIXME need to serialize old_identifiers
basic_package.old_identifiers = ['edu.utah.sci.vistrails.basic']
self._registry._default_package = basic_package
prefix_dictionary = {'basic_modules': 'vistrails.core.modules.'}
self.initialize_packages(prefix_dictionary)
setup_basic_package()
self._abstraction_pkg = self.add_package('abstraction', False)
# FIXME need to get this info from the package, but cannot
# do this since controller isn't imported yet
self._abstraction_pkg.identifier = 'local.abstractions'
self._abstraction_pkg.name = 'My SubWorkflows'
self._abstraction_pkg.version = '1.6'
self._registry.add_package(self._abstraction_pkg)
# Setup a global __import__ hook that calls Package#import_override()
# for all imports executed from that package
import __builtin__
self._orig_import = __builtin__.__import__
__builtin__.__import__ = self._import_override
def _import_override(self,
name, globals={}, locals={}, fromlist=[], level=-1):
# Get the caller module, using globals (like the original __import
# does)
try:
if globals is None:
raise KeyError
module = globals['__name__']
except KeyError:
# Another method of getting the caller module, using the stack
caller = inspect.currentframe().f_back
module = inspect.getmodule(caller)
# Some frames might not be associated to a module, because of the
# use of exec for instance; we just skip these until we reach a
# valid one
while module is None:
caller = caller.f_back
if caller is None:
break
module = inspect.getmodule(caller)
if module:
module = module.__name__
# Get the Package from the module name
if module:
importing_pkg = None
current = self._currently_importing_package
if (current is not None and
current.prefix and
module.startswith(current.prefix + current.codepath)):
importing_pkg = current
else:
for pkg in itertools.chain(
self._package_list.itervalues(),
self._available_packages.itervalues()):
if (pkg.prefix is not None and
module.startswith(pkg.prefix + pkg.codepath)):
importing_pkg = pkg
break
# If we are importing directly from a package
if importing_pkg is not None:
old_current = self._currently_importing_package
self._currently_importing_package = importing_pkg
result = importing_pkg.import_override(
self._orig_import,
name, globals, locals, fromlist, level,
package_importing_directly=True)
self._currently_importing_package = old_current
return result
# If we are doing it indirectly (from other stuff imported from a
# package)
elif self._currently_importing_package is not None:
return self._currently_importing_package.import_override(
self._orig_import,
name, globals, locals, fromlist, level,
package_importing_directly=False)
# Else, this is not from a package
return self._orig_import(name, globals, locals, fromlist, level)
def finalize_packages(self):
"""Finalizes all installed packages. Call this only prior to
exiting VisTrails."""
for package in self._package_list.itervalues():
package.finalize()
self._package_list = {}
self._package_versions = {}
self._old_identifier_map = {}
global _package_manager
_package_manager = None
def get_available_package(self, codepath):
try:
pkg = self._available_packages[codepath]
except KeyError:
pkg = self._registry.create_package(codepath)
self._available_packages[codepath] = pkg
return pkg
def add_package(self, codepath, add_to_package_list=True):
"""Adds a new package to the manager. This does not initialize it.
To do so, call initialize_packages()"""
package = self.get_available_package(codepath)
if add_to_package_list:
self.add_to_package_list(codepath, package)
return package
def add_to_package_list(self, codepath, package):
self._available_packages[codepath] = package
self._package_list[codepath] = package
def initialize_abstraction_pkg(self, prefix_dictionary):
if self._abstraction_pkg is None:
raise RuntimeError("Subworkflows packages is None")
self.add_to_package_list(self._abstraction_pkg.codepath,
self._abstraction_pkg)
self.late_enable_package(self._abstraction_pkg.codepath,
prefix_dictionary, False)
def remove_old_identifiers(self, identifier):
# remove refs in old_identifier_map
old_ids = []
for old_id, cur_id in self._old_identifier_map.iteritems():
if cur_id == identifier:
old_ids.append(old_id)
for old_id in old_ids:
del self._old_identifier_map[old_id]
def remove_package(self, codepath):
"""remove_package(name): Removes a package from the system."""
pkg = self._package_list[codepath]
from vistrails.core.interpreter.cached import CachedInterpreter
CachedInterpreter.clear_package(pkg.identifier)
self._dependency_graph.delete_vertex(pkg.identifier)
del self._package_versions[pkg.identifier][pkg.version]
if len(self._package_versions[pkg.identifier]) == 0:
del self._package_versions[pkg.identifier]
self.remove_old_identifiers(pkg.identifier)
self.remove_menu_items(pkg)
pkg.finalize()
del self._package_list[codepath]
self._registry.remove_package(pkg)
app = get_vistrails_application()
app.send_notification("package_removed", codepath)
def has_package(self, identifier, version=None):
"""has_package(identifer: string) -> Boolean.
Returns true if given package identifier is present."""
# check if it's an old identifier
identifier = self._old_identifier_map.get(identifier, identifier)
if identifier in self._package_versions:
return (version is None or
version in self._package_versions[identifier])
return False
def look_at_available_package(self, codepath):
"""look_at_available_package(codepath: string) -> Package
Returns a Package object for an uninstalled package. This does
NOT install a package.
"""
return self.get_available_package(codepath)
def get_package(self, identifier, version=None):
# check if it's an old identifier
identifier = self._old_identifier_map.get(identifier, identifier)
try:
package_versions = self._package_versions[identifier]
if version is not None:
return package_versions[version]
except KeyError:
# dynamic packages are only registered in the registry
try:
return self._registry.get_package_by_name(identifier, version)
except MissingPackageVersion:
return self._registry.get_package_by_name(identifier)
max_version = '0'
max_pkg = None
for version, pkg in package_versions.iteritems():
if versions_increasing(max_version, version):
max_version = version
max_pkg = pkg
return max_pkg
def get_package_by_codepath(self, codepath):
"""get_package_by_codepath(codepath: string) -> Package.
Returns a package with given codepath if it is enabled,
otherwise throws exception
"""
if codepath not in self._package_list:
raise MissingPackage(codepath)
else:
return self._package_list[codepath]
def get_package_by_identifier(self, identifier):
"""get_package_by_identifier(identifier: string) -> Package.
Deprecated, use get_package() instead.
"""
warnings.warn(
"You should use get_package instead of "
"get_package_by_identifier",
VistrailsDeprecation,
stacklevel=2)
return self.get_package(identifier)
def get_package_configuration(self, codepath):
"""get_package_configuration(codepath: string) ->
ConfigurationObject or None
Returns the configuration object for the package, if existing,
or None. Throws MissingPackage if package doesn't exist.
"""
pkg = self.get_package_by_codepath(codepath)
if not hasattr(pkg.module, 'configuration'):
return None
else:
c = pkg.module.configuration
if not isinstance(c, ConfigurationObject):
d = "'configuration' attribute should be a ConfigurationObject"
raise self.PackageInternalError(codepath, d)
return c
def check_dependencies(self, package, deps):
# want to check that necessary version also exists, if specified
missing_deps = []
for dep in deps:
min_version = None
max_version = None
if isinstance(dep, tuple):
identifier = dep[0]
if len(dep) > 1:
min_version = dep[1]
if len(dep) > 2:
max_version = dep[2]
else:
identifier = dep
# check if it's an old identifier
identifier = self._old_identifier_map.get(identifier, identifier)
if identifier not in self._package_versions:
missing_deps.append((identifier, None, None))
else:
if min_version is None and max_version is None:
continue
found_version = False
for version, pkg in \
self._package_versions[identifier].iteritems():
if ((min_version is None or
versions_increasing(min_version, version)) and
(max_version is None or
versions_increasing(version, max_version))):
found_version = True
if not found_version:
missing_deps.append((identifier, min_version, max_version))
if len(missing_deps) > 0:
raise Package.MissingDependency(package, missing_deps)
return True
def add_dependencies(self, package):
"""add_dependencies(package) -> None. Register all
dependencies a package contains by calling the appropriate
callback.
Does not add multiple dependencies - if a dependency is already there,
add_dependencies ignores it.
"""
deps = package.dependencies()
# FIXME don't hardcode this
from vistrails.core.modules.basic_modules import identifier as basic_pkg
if package.identifier != basic_pkg:
deps.append(basic_pkg)
self.check_dependencies(package, deps)
for dep in deps:
if isinstance(dep, tuple):
dep_name = dep[0]
else:
dep_name = dep
dep_name = self.get_package(dep_name).identifier
if not self._dependency_graph.has_edge(package.identifier,
dep_name):
self._dependency_graph.add_edge(package.identifier, dep_name)
def late_enable_package(self, codepath, prefix_dictionary={},
needs_add=True):
"""late_enable_package enables a package 'late', that is,
after VisTrails initialization. All dependencies need to be
already enabled.
"""
if needs_add:
if codepath in self._package_list:
msg = 'duplicate package identifier: %s' % codepath
raise VistrailsInternalError(msg)
self.add_package(codepath)
pkg = self.get_package_by_codepath(codepath)
try:
pkg.load(prefix_dictionary.get(pkg.codepath, None))
pkg.create_startup_package_node()
except Exception, e:
# invert self.add_package
del self._package_list[codepath]
raise
self._dependency_graph.add_vertex(pkg.identifier)
if pkg.identifier not in self._package_versions:
self._package_versions[pkg.identifier] = {}
self._package_versions[pkg.identifier][pkg.version] = pkg
for old_id in pkg.old_identifiers:
self._old_identifier_map[old_id] = pkg.identifier
try:
self.add_dependencies(pkg)
#check_requirements is now called in pkg.initialize()
#pkg.check_requirements()
self._registry.initialize_package(pkg)
self._registry.signals.emit_new_package(pkg.identifier, True)
app = get_vistrails_application()
app.send_notification("package_added", codepath)
self.add_menu_items(pkg)
except Exception, e:
del self._package_versions[pkg.identifier][pkg.version]
if len(self._package_versions[pkg.identifier]) == 0:
del self._package_versions[pkg.identifier]
self.remove_old_identifiers(pkg.identifier)
self._dependency_graph.delete_vertex(pkg.identifier)
# invert self.add_package
del self._package_list[codepath]
# if we adding the package to the registry, make sure we
# remove it if initialization fails
try:
self._registry.remove_package(pkg)
except MissingPackage:
pass
raise e