def __init__(self, *repo_dirs, **kwargs):
# super-namespace for all packages in the RepoPath
self.super_namespace = kwargs.get('namespace', repo_namespace)
self.repos = []
self.by_namespace = NamespaceTrie()
self.by_path = {}
self._all_package_names = None
self._provider_index = None
# If repo_dirs is empty, just use the configuration
if not repo_dirs:
import spack.config
repo_dirs = spack.config.get_config('repos')
if not repo_dirs:
raise NoRepoConfiguredError(
"Spack configuration contains no package repositories.")
# Add each repo to this path.
for root in repo_dirs:
try:
repo = Repo(root, self.super_namespace)
self.put_last(repo)
except RepoError as e:
tty.warn("Failed to initialize repository at '%s'." % root,
e.message,
"To remove the bad repository, run this command:",
" spack repo rm %s" % root)
def __init__(self, *repos):
self.repos = []
self.by_namespace = NamespaceTrie()
self._all_package_names = None
self._provider_index = None
self._patch_index = None
# Add each repo to this path.
for repo in repos:
try:
if isinstance(repo, string_types):
repo = Repo(repo)
self.put_last(repo)
except RepoError as e:
tty.warn("Failed to initialize repository: '%s'." % repo,
e.message,
"To remove the bad repository, run this command:",
" spack repo rm %s" % repo)
def __init__(self, *repos, **kwargs):
self.super_namespace = kwargs.get('namespace', repo_namespace)
self.repos = []
self.by_namespace = NamespaceTrie()
self.by_path = {}
self._all_package_names = None
self._provider_index = None
# Add each repo to this path.
for repo in repos:
try:
if isinstance(repo, string_types):
repo = Repo(repo, self.super_namespace)
self.put_last(repo)
except RepoError as e:
tty.warn("Failed to initialize repository: '%s'." % repo,
e.message,
"To remove the bad repository, run this command:",
" spack repo rm %s" % repo)
class RepoPath(object):
"""A RepoPath is a list of repos that function as one.
It functions exactly like a Repo, but it operates on the combined
results of the Repos in its list instead of on a single package
repository.
Args:
repos (list): list Repo objects or paths to put in this RepoPath
Optional Args:
repo_namespace (str): super-namespace for all packages in this
RepoPath (used when importing repos as modules)
"""
def __init__(self, *repos, **kwargs):
self.super_namespace = kwargs.get('namespace', repo_namespace)
self.repos = []
self.by_namespace = NamespaceTrie()
self._all_package_names = None
self._provider_index = None
# Add each repo to this path.
for repo in repos:
try:
if isinstance(repo, string_types):
repo = Repo(repo, self.super_namespace)
self.put_last(repo)
except RepoError as e:
tty.warn("Failed to initialize repository: '%s'." % repo,
e.message,
"To remove the bad repository, run this command:",
" spack repo rm %s" % repo)
def put_first(self, repo):
"""Add repo first in the search path."""
if isinstance(repo, RepoPath):
for r in reversed(repo.repos):
self.put_first(r)
return
self.repos.insert(0, repo)
self.by_namespace[repo.full_namespace] = repo
def put_last(self, repo):
"""Add repo last in the search path."""
if isinstance(repo, RepoPath):
for r in repo.repos:
self.put_last(r)
return
self.repos.append(repo)
# don't mask any higher-precedence repos with same namespace
if repo.full_namespace not in self.by_namespace:
self.by_namespace[repo.full_namespace] = repo
def remove(self, repo):
"""Remove a repo from the search path."""
if repo in self.repos:
self.repos.remove(repo)
def get_repo(self, namespace, default=NOT_PROVIDED):
"""Get a repository by namespace.
Arguments:
namespace:
Look up this namespace in the RepoPath, and return it if found.
Optional Arguments:
default:
If default is provided, return it when the namespace
isn't found. If not, raise an UnknownNamespaceError.
"""
fullspace = '%s.%s' % (self.super_namespace, namespace)
if fullspace not in self.by_namespace:
if default == NOT_PROVIDED:
raise UnknownNamespaceError(namespace)
return default
return self.by_namespace[fullspace]
def first_repo(self):
"""Get the first repo in precedence order."""
return self.repos[0] if self.repos else None
def all_package_names(self):
"""Return all unique package names in all repositories."""
if self._all_package_names is None:
all_pkgs = set()
for repo in self.repos:
for name in repo.all_package_names():
all_pkgs.add(name)
self._all_package_names = sorted(all_pkgs, key=lambda n: n.lower())
return self._all_package_names
def packages_with_tags(self, *tags):
r = set()
for repo in self.repos:
r |= set(repo.packages_with_tags(*tags))
return sorted(r)
def all_packages(self):
for name in self.all_package_names():
yield self.get(name)
@property
def provider_index(self):
"""Merged ProviderIndex from all Repos in the RepoPath."""
if self._provider_index is None:
self._provider_index = ProviderIndex()
for repo in reversed(self.repos):
self._provider_index.merge(repo.provider_index)
return self._provider_index
@_autospec
def providers_for(self, vpkg_spec):
providers = self.provider_index.providers_for(vpkg_spec)
if not providers:
raise UnknownPackageError(vpkg_spec.name)
return providers
@_autospec
def extensions_for(self, extendee_spec):
return [p for p in self.all_packages() if p.extends(extendee_spec)]
def find_module(self, fullname, path=None):
"""Implements precedence for overlaid namespaces.
Loop checks each namespace in self.repos for packages, and
also handles loading empty containing namespaces.
"""
# namespaces are added to repo, and package modules are leaves.
namespace, dot, module_name = fullname.rpartition('.')
# If it's a module in some repo, or if it is the repo's
# namespace, let the repo handle it.
for repo in self.repos:
if namespace == repo.full_namespace:
if repo.real_name(module_name):
return repo
elif fullname == repo.full_namespace:
return repo
# No repo provides the namespace, but it is a valid prefix of
# something in the RepoPath.
if self.by_namespace.is_prefix(fullname):
return self
return None
def load_module(self, fullname):
"""Handles loading container namespaces when necessary.
See ``Repo`` for how actual package modules are loaded.
"""
if fullname in sys.modules:
return sys.modules[fullname]
if not self.by_namespace.is_prefix(fullname):
raise ImportError("No such Spack repo: %s" % fullname)
module = SpackNamespace(fullname)
module.__loader__ = self
sys.modules[fullname] = module
return module
def repo_for_pkg(self, spec):
"""Given a spec, get the repository for its package."""
# We don't @_autospec this function b/c it's called very frequently
# and we want to avoid parsing str's into Specs unnecessarily.
namespace = None
if isinstance(spec, spack.spec.Spec):
namespace = spec.namespace
name = spec.name
else:
# handle strings directly for speed instead of @_autospec'ing
namespace, _, name = spec.rpartition('.')
# If the spec already has a namespace, then return the
# corresponding repo if we know about it.
if namespace:
fullspace = '%s.%s' % (self.super_namespace, namespace)
if fullspace not in self.by_namespace:
raise UnknownNamespaceError(spec.namespace)
return self.by_namespace[fullspace]
# If there's no namespace, search in the RepoPath.
for repo in self.repos:
if name in repo:
return repo
# If the package isn't in any repo, return the one with
# highest precedence. This is for commands like `spack edit`
# that can operate on packages that don't exist yet.
return self.first_repo()
@_autospec
def get(self, spec, new=False):
"""Find a repo that contains the supplied spec's package.
Raises UnknownPackageError if not found.
"""
return self.repo_for_pkg(spec).get(spec)
def get_pkg_class(self, pkg_name):
"""Find a class for the spec's package and return the class object."""
return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name)
@_autospec
def dump_provenance(self, spec, path):
"""Dump provenance information for a spec to a particular path.
This dumps the package file and any associated patch files.
Raises UnknownPackageError if not found.
"""
return self.repo_for_pkg(spec).dump_provenance(spec, path)
def dirname_for_package_name(self, pkg_name):
return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name)
def filename_for_package_name(self, pkg_name):
return self.repo_for_pkg(pkg_name).filename_for_package_name(pkg_name)
def exists(self, pkg_name):
"""Whether package with the give name exists in the path's repos.
Note that virtual packages do not "exist".
"""
return any(repo.exists(pkg_name) for repo in self.repos)
def is_virtual(self, pkg_name):
"""True if the package with this name is virtual, False otherwise."""
return pkg_name in self.provider_index
def __contains__(self, pkg_name):
return self.exists(pkg_name)
def setUp(self):
self.trie = NamespaceTrie()
class NamespaceTrieTest(unittest.TestCase):
def setUp(self):
self.trie = NamespaceTrie()
def test_add_single(self):
self.trie['foo'] = 'bar'
self.assertTrue(self.trie.is_prefix('foo'))
self.assertTrue(self.trie.has_value('foo'))
self.assertEqual(self.trie['foo'], 'bar')
def test_add_multiple(self):
self.trie['foo.bar'] = 'baz'
self.assertFalse(self.trie.has_value('foo'))
self.assertTrue(self.trie.is_prefix('foo'))
self.assertTrue(self.trie.is_prefix('foo.bar'))
self.assertTrue(self.trie.has_value('foo.bar'))
self.assertEqual(self.trie['foo.bar'], 'baz')
self.assertFalse(self.trie.is_prefix('foo.bar.baz'))
self.assertFalse(self.trie.has_value('foo.bar.baz'))
def test_add_three(self):
# add a three-level namespace
self.trie['foo.bar.baz'] = 'quux'
self.assertTrue(self.trie.is_prefix('foo'))
self.assertFalse(self.trie.has_value('foo'))
self.assertTrue(self.trie.is_prefix('foo.bar'))
self.assertFalse(self.trie.has_value('foo.bar'))
self.assertTrue(self.trie.is_prefix('foo.bar.baz'))
self.assertTrue(self.trie.has_value('foo.bar.baz'))
self.assertEqual(self.trie['foo.bar.baz'], 'quux')
self.assertFalse(self.trie.is_prefix('foo.bar.baz.quux'))
self.assertFalse(self.trie.has_value('foo.bar.baz.quux'))
# Try to add a second element in a prefix namespace
self.trie['foo.bar'] = 'blah'
self.assertTrue(self.trie.is_prefix('foo'))
self.assertFalse(self.trie.has_value('foo'))
self.assertTrue(self.trie.is_prefix('foo.bar'))
self.assertTrue(self.trie.has_value('foo.bar'))
self.assertEqual(self.trie['foo.bar'], 'blah')
self.assertTrue(self.trie.is_prefix('foo.bar.baz'))
self.assertTrue(self.trie.has_value('foo.bar.baz'))
self.assertEqual(self.trie['foo.bar.baz'], 'quux')
self.assertFalse(self.trie.is_prefix('foo.bar.baz.quux'))
self.assertFalse(self.trie.has_value('foo.bar.baz.quux'))
def test_add_none_single(self):
self.trie['foo'] = None
self.assertTrue(self.trie.is_prefix('foo'))
self.assertTrue(self.trie.has_value('foo'))
self.assertEqual(self.trie['foo'], None)
self.assertFalse(self.trie.is_prefix('foo.bar'))
self.assertFalse(self.trie.has_value('foo.bar'))
def test_add_none_multiple(self):
self.trie['foo.bar'] = None
self.assertTrue(self.trie.is_prefix('foo'))
self.assertFalse(self.trie.has_value('foo'))
self.assertTrue(self.trie.is_prefix('foo.bar'))
self.assertTrue(self.trie.has_value('foo.bar'))
self.assertEqual(self.trie['foo.bar'], None)
self.assertFalse(self.trie.is_prefix('foo.bar.baz'))
self.assertFalse(self.trie.has_value('foo.bar.baz'))
def setUp(self):
self.trie = NamespaceTrie()
class NamespaceTrieTest(unittest.TestCase):
def setUp(self):
self.trie = NamespaceTrie()
def test_add_single(self):
self.trie['foo'] = 'bar'
self.assertTrue(self.trie.is_prefix('foo'))
self.assertTrue(self.trie.has_value('foo'))
self.assertEqual(self.trie['foo'], 'bar')
def test_add_multiple(self):
self.trie['foo.bar'] = 'baz'
self.assertFalse(self.trie.has_value('foo'))
self.assertTrue(self.trie.is_prefix('foo'))
self.assertTrue(self.trie.is_prefix('foo.bar'))
self.assertTrue(self.trie.has_value('foo.bar'))
self.assertEqual(self.trie['foo.bar'], 'baz')
self.assertFalse(self.trie.is_prefix('foo.bar.baz'))
self.assertFalse(self.trie.has_value('foo.bar.baz'))
def test_add_three(self):
# add a three-level namespace
self.trie['foo.bar.baz'] = 'quux'
self.assertTrue(self.trie.is_prefix('foo'))
self.assertFalse(self.trie.has_value('foo'))
self.assertTrue(self.trie.is_prefix('foo.bar'))
self.assertFalse(self.trie.has_value('foo.bar'))
self.assertTrue(self.trie.is_prefix('foo.bar.baz'))
self.assertTrue(self.trie.has_value('foo.bar.baz'))
self.assertEqual(self.trie['foo.bar.baz'], 'quux')
self.assertFalse(self.trie.is_prefix('foo.bar.baz.quux'))
self.assertFalse(self.trie.has_value('foo.bar.baz.quux'))
# Try to add a second element in a prefix namespace
self.trie['foo.bar'] = 'blah'
self.assertTrue(self.trie.is_prefix('foo'))
self.assertFalse(self.trie.has_value('foo'))
self.assertTrue(self.trie.is_prefix('foo.bar'))
self.assertTrue(self.trie.has_value('foo.bar'))
self.assertEqual(self.trie['foo.bar'], 'blah')
self.assertTrue(self.trie.is_prefix('foo.bar.baz'))
self.assertTrue(self.trie.has_value('foo.bar.baz'))
self.assertEqual(self.trie['foo.bar.baz'], 'quux')
self.assertFalse(self.trie.is_prefix('foo.bar.baz.quux'))
self.assertFalse(self.trie.has_value('foo.bar.baz.quux'))
def test_add_none_single(self):
self.trie['foo'] = None
self.assertTrue(self.trie.is_prefix('foo'))
self.assertTrue(self.trie.has_value('foo'))
self.assertEqual(self.trie['foo'], None)
self.assertFalse(self.trie.is_prefix('foo.bar'))
self.assertFalse(self.trie.has_value('foo.bar'))
def test_add_none_multiple(self):
self.trie['foo.bar'] = None
self.assertTrue(self.trie.is_prefix('foo'))
self.assertFalse(self.trie.has_value('foo'))
self.assertTrue(self.trie.is_prefix('foo.bar'))
self.assertTrue(self.trie.has_value('foo.bar'))
self.assertEqual(self.trie['foo.bar'], None)
self.assertFalse(self.trie.is_prefix('foo.bar.baz'))
self.assertFalse(self.trie.has_value('foo.bar.baz'))
class RepoPath(object):
"""A RepoPath is a list of repos that function as one.
It functions exactly like a Repo, but it operates on the combined
results of the Repos in its list instead of on a single package
repository.
Args:
repos (list): list Repo objects or paths to put in this RepoPath
Optional Args:
repo_namespace (str): super-namespace for all packages in this
RepoPath (used when importing repos as modules)
"""
def __init__(self, *repos, **kwargs):
self.super_namespace = kwargs.get('namespace', repo_namespace)
self.repos = []
self.by_namespace = NamespaceTrie()
self.by_path = {}
self._all_package_names = None
self._provider_index = None
# Add each repo to this path.
for repo in repos:
try:
if isinstance(repo, string_types):
repo = Repo(repo, self.super_namespace)
self.put_last(repo)
except RepoError as e:
tty.warn("Failed to initialize repository: '%s'." % repo,
e.message,
"To remove the bad repository, run this command:",
" spack repo rm %s" % repo)
def _add(self, repo):
"""Add a repository to the namespace and path indexes.
Checks for duplicates -- two repos can't have the same root
directory, and they provide have the same namespace.
"""
if repo.root in self.by_path:
raise DuplicateRepoError("Duplicate repository: '%s'" % repo.root)
if repo.namespace in self.by_namespace:
raise DuplicateRepoError(
"Package repos '%s' and '%s' both provide namespace %s"
% (repo.root, self.by_namespace[repo.namespace].root,
repo.namespace))
# Add repo to the pkg indexes
self.by_namespace[repo.full_namespace] = repo
self.by_path[repo.root] = repo
def put_first(self, repo):
"""Add repo first in the search path."""
self._add(repo)
self.repos.insert(0, repo)
def put_last(self, repo):
"""Add repo last in the search path."""
self._add(repo)
self.repos.append(repo)
def remove(self, repo):
"""Remove a repo from the search path."""
if repo in self.repos:
self.repos.remove(repo)
def get_repo(self, namespace, default=NOT_PROVIDED):
"""Get a repository by namespace.
Arguments:
namespace:
Look up this namespace in the RepoPath, and return it if found.
Optional Arguments:
default:
If default is provided, return it when the namespace
isn't found. If not, raise an UnknownNamespaceError.
"""
fullspace = '%s.%s' % (self.super_namespace, namespace)
if fullspace not in self.by_namespace:
if default == NOT_PROVIDED:
raise UnknownNamespaceError(namespace)
return default
return self.by_namespace[fullspace]
def first_repo(self):
"""Get the first repo in precedence order."""
return self.repos[0] if self.repos else None
def all_package_names(self):
"""Return all unique package names in all repositories."""
if self._all_package_names is None:
all_pkgs = set()
for repo in self.repos:
for name in repo.all_package_names():
all_pkgs.add(name)
self._all_package_names = sorted(all_pkgs, key=lambda n: n.lower())
return self._all_package_names
def packages_with_tags(self, *tags):
r = set()
for repo in self.repos:
r |= set(repo.packages_with_tags(*tags))
return sorted(r)
def all_packages(self):
for name in self.all_package_names():
yield self.get(name)
@property
def provider_index(self):
"""Merged ProviderIndex from all Repos in the RepoPath."""
if self._provider_index is None:
self._provider_index = ProviderIndex()
for repo in reversed(self.repos):
self._provider_index.merge(repo.provider_index)
return self._provider_index
@_autospec
def providers_for(self, vpkg_spec):
providers = self.provider_index.providers_for(vpkg_spec)
if not providers:
raise UnknownPackageError(vpkg_spec.name)
return providers
@_autospec
def extensions_for(self, extendee_spec):
return [p for p in self.all_packages() if p.extends(extendee_spec)]
def find_module(self, fullname, path=None):
"""Implements precedence for overlaid namespaces.
Loop checks each namespace in self.repos for packages, and
also handles loading empty containing namespaces.
"""
# namespaces are added to repo, and package modules are leaves.
namespace, dot, module_name = fullname.rpartition('.')
# If it's a module in some repo, or if it is the repo's
# namespace, let the repo handle it.
for repo in self.repos:
if namespace == repo.full_namespace:
if repo.real_name(module_name):
return repo
elif fullname == repo.full_namespace:
return repo
# No repo provides the namespace, but it is a valid prefix of
# something in the RepoPath.
if self.by_namespace.is_prefix(fullname):
return self
return None
def load_module(self, fullname):
"""Handles loading container namespaces when necessary.
See ``Repo`` for how actual package modules are loaded.
"""
if fullname in sys.modules:
return sys.modules[fullname]
if not self.by_namespace.is_prefix(fullname):
raise ImportError("No such Spack repo: %s" % fullname)
module = SpackNamespace(fullname)
module.__loader__ = self
sys.modules[fullname] = module
return module
def repo_for_pkg(self, spec):
"""Given a spec, get the repository for its package."""
# We don't @_autospec this function b/c it's called very frequently
# and we want to avoid parsing str's into Specs unnecessarily.
namespace = None
if isinstance(spec, spack.spec.Spec):
namespace = spec.namespace
name = spec.name
else:
# handle strings directly for speed instead of @_autospec'ing
namespace, _, name = spec.rpartition('.')
# If the spec already has a namespace, then return the
# corresponding repo if we know about it.
if namespace:
fullspace = '%s.%s' % (self.super_namespace, namespace)
if fullspace not in self.by_namespace:
raise UnknownNamespaceError(spec.namespace)
return self.by_namespace[fullspace]
# If there's no namespace, search in the RepoPath.
for repo in self.repos:
if name in repo:
return repo
# If the package isn't in any repo, return the one with
# highest precedence. This is for commands like `spack edit`
# that can operate on packages that don't exist yet.
return self.first_repo()
@_autospec
def get(self, spec, new=False):
"""Find a repo that contains the supplied spec's package.
Raises UnknownPackageError if not found.
"""
return self.repo_for_pkg(spec).get(spec)
def get_pkg_class(self, pkg_name):
"""Find a class for the spec's package and return the class object."""
return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name)
@_autospec
def dump_provenance(self, spec, path):
"""Dump provenance information for a spec to a particular path.
This dumps the package file and any associated patch files.
Raises UnknownPackageError if not found.
"""
return self.repo_for_pkg(spec).dump_provenance(spec, path)
def dirname_for_package_name(self, pkg_name):
return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name)
def filename_for_package_name(self, pkg_name):
return self.repo_for_pkg(pkg_name).filename_for_package_name(pkg_name)
def exists(self, pkg_name):
"""Whether package with the give name exists in the path's repos.
Note that virtual packages do not "exist".
"""
return any(repo.exists(pkg_name) for repo in self.repos)
def is_virtual(self, pkg_name):
"""True if the package with this name is virtual, False otherwise."""
return pkg_name in self.provider_index
def __contains__(self, pkg_name):
return self.exists(pkg_name)
