class RepoPath(object): """A RepoPath is a list of repos that function as one. It functions exactly like a Repo, but it operates on the combined results of the Repos in its list instead of on a single package repository. """ def __init__(self, *repo_dirs, **kwargs): # super-namespace for all packages in the RepoPath self.super_namespace = kwargs.get('namespace', repo_namespace) self.repos = [] self.by_namespace = NamespaceTrie() self.by_path = {} self._all_package_names = None self._provider_index = None # If repo_dirs is empty, just use the configuration if not repo_dirs: import spack.config repo_dirs = spack.config.get_config('repos') if not repo_dirs: raise NoRepoConfiguredError( "Spack configuration contains no package repositories.") # Add each repo to this path. for root in repo_dirs: try: repo = Repo(root, self.super_namespace) self.put_last(repo) except RepoError as e: tty.warn("Failed to initialize repository at '%s'." % root, e.message, "To remove the bad repository, run this command:", " spack repo rm %s" % root) def swap(self, other): """Convenience function to make swapping repositories easier. This is currently used by mock tests. TODO: Maybe there is a cleaner way. """ attrs = [ 'repos', 'by_namespace', 'by_path', '_all_package_names', '_provider_index' ] for attr in attrs: tmp = getattr(self, attr) setattr(self, attr, getattr(other, attr)) setattr(other, attr, tmp) def _add(self, repo): """Add a repository to the namespace and path indexes. Checks for duplicates -- two repos can't have the same root directory, and they provide have the same namespace. """ if repo.root in self.by_path: raise DuplicateRepoError("Duplicate repository: '%s'" % repo.root) if repo.namespace in self.by_namespace: raise DuplicateRepoError( "Package repos '%s' and '%s' both provide namespace %s" % (repo.root, self.by_namespace[repo.namespace].root, repo.namespace)) # Add repo to the pkg indexes self.by_namespace[repo.full_namespace] = repo self.by_path[repo.root] = repo def put_first(self, repo): """Add repo first in the search path.""" self._add(repo) self.repos.insert(0, repo) def put_last(self, repo): """Add repo last in the search path.""" self._add(repo) self.repos.append(repo) def remove(self, repo): """Remove a repo from the search path.""" if repo in self.repos: self.repos.remove(repo) def get_repo(self, namespace, default=NOT_PROVIDED): """Get a repository by namespace. Arguments: namespace: Look up this namespace in the RepoPath, and return it if found. Optional Arguments: default: If default is provided, return it when the namespace isn't found. If not, raise an UnknownNamespaceError. """ fullspace = '%s.%s' % (self.super_namespace, namespace) if fullspace not in self.by_namespace: if default == NOT_PROVIDED: raise UnknownNamespaceError(namespace) return default return self.by_namespace[fullspace] def first_repo(self): """Get the first repo in precedence order.""" return self.repos[0] if self.repos else None def all_package_names(self): """Return all unique package names in all repositories.""" if self._all_package_names is None: all_pkgs = set() for repo in self.repos: for name in repo.all_package_names(): all_pkgs.add(name) self._all_package_names = sorted(all_pkgs, key=lambda n: n.lower()) return self._all_package_names def all_packages(self): for name in self.all_package_names(): yield self.get(name) @property def provider_index(self): """Merged ProviderIndex from all Repos in the RepoPath.""" if self._provider_index is None: self._provider_index = ProviderIndex() for repo in reversed(self.repos): self._provider_index.merge(repo.provider_index) return self._provider_index @_autospec def providers_for(self, vpkg_spec): providers = self.provider_index.providers_for(vpkg_spec) if not providers: raise UnknownPackageError(vpkg_spec.name) return providers @_autospec def extensions_for(self, extendee_spec): return [p for p in self.all_packages() if p.extends(extendee_spec)] def find_module(self, fullname, path=None): """Implements precedence for overlaid namespaces. Loop checks each namespace in self.repos for packages, and also handles loading empty containing namespaces. """ # namespaces are added to repo, and package modules are leaves. namespace, dot, module_name = fullname.rpartition('.') # If it's a module in some repo, or if it is the repo's # namespace, let the repo handle it. for repo in self.repos: if namespace == repo.full_namespace: if repo.real_name(module_name): return repo elif fullname == repo.full_namespace: return repo # No repo provides the namespace, but it is a valid prefix of # something in the RepoPath. if self.by_namespace.is_prefix(fullname): return self return None def load_module(self, fullname): """Handles loading container namespaces when necessary. See ``Repo`` for how actual package modules are loaded. """ if fullname in sys.modules: return sys.modules[fullname] if not self.by_namespace.is_prefix(fullname): raise ImportError("No such Spack repo: %s" % fullname) module = SpackNamespace(fullname) module.__loader__ = self sys.modules[fullname] = module return module @_autospec def repo_for_pkg(self, spec): """Given a spec, get the repository for its package.""" # If the spec already has a namespace, then return the # corresponding repo if we know about it. if spec.namespace: fullspace = '%s.%s' % (self.super_namespace, spec.namespace) if fullspace not in self.by_namespace: raise UnknownNamespaceError(spec.namespace) return self.by_namespace[fullspace] # If there's no namespace, search in the RepoPath. for repo in self.repos: if spec.name in repo: return repo # If the package isn't in any repo, return the one with # highest precedence. This is for commands like `spack edit` # that can operate on packages that don't exist yet. return self.first_repo() @_autospec def get(self, spec, new=False): """Find a repo that contains the supplied spec's package. Raises UnknownPackageError if not found. """ return self.repo_for_pkg(spec).get(spec) def get_pkg_class(self, pkg_name): """Find a class for the spec's package and return the class object.""" return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name) @_autospec def dump_provenance(self, spec, path): """Dump provenance information for a spec to a particular path. This dumps the package file and any associated patch files. Raises UnknownPackageError if not found. """ return self.repo_for_pkg(spec).dump_provenance(spec, path) def dirname_for_package_name(self, pkg_name): return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) def filename_for_package_name(self, pkg_name): return self.repo_for_pkg(pkg_name).filename_for_package_name(pkg_name) def exists(self, pkg_name): """Whether package with the give name exists in the path's repos. Note that virtual packages do not "exist". """ return any(repo.exists(pkg_name) for repo in self.repos) def is_virtual(self, pkg_name): """True if the package with this name is virtual, False otherwise.""" return pkg_name in self.provider_index def __contains__(self, pkg_name): return self.exists(pkg_name)
class RepoPath(object): """A RepoPath is a list of repos that function as one. It functions exactly like a Repo, but it operates on the combined results of the Repos in its list instead of on a single package repository. Args: repos (list): list Repo objects or paths to put in this RepoPath Optional Args: repo_namespace (str): super-namespace for all packages in this RepoPath (used when importing repos as modules) """ def __init__(self, *repos, **kwargs): self.super_namespace = kwargs.get('namespace', repo_namespace) self.repos = [] self.by_namespace = NamespaceTrie() self._all_package_names = None self._provider_index = None # Add each repo to this path. for repo in repos: try: if isinstance(repo, string_types): repo = Repo(repo, self.super_namespace) self.put_last(repo) except RepoError as e: tty.warn("Failed to initialize repository: '%s'." % repo, e.message, "To remove the bad repository, run this command:", " spack repo rm %s" % repo) def put_first(self, repo): """Add repo first in the search path.""" if isinstance(repo, RepoPath): for r in reversed(repo.repos): self.put_first(r) return self.repos.insert(0, repo) self.by_namespace[repo.full_namespace] = repo def put_last(self, repo): """Add repo last in the search path.""" if isinstance(repo, RepoPath): for r in repo.repos: self.put_last(r) return self.repos.append(repo) # don't mask any higher-precedence repos with same namespace if repo.full_namespace not in self.by_namespace: self.by_namespace[repo.full_namespace] = repo def remove(self, repo): """Remove a repo from the search path.""" if repo in self.repos: self.repos.remove(repo) def get_repo(self, namespace, default=NOT_PROVIDED): """Get a repository by namespace. Arguments: namespace: Look up this namespace in the RepoPath, and return it if found. Optional Arguments: default: If default is provided, return it when the namespace isn't found. If not, raise an UnknownNamespaceError. """ fullspace = '%s.%s' % (self.super_namespace, namespace) if fullspace not in self.by_namespace: if default == NOT_PROVIDED: raise UnknownNamespaceError(namespace) return default return self.by_namespace[fullspace] def first_repo(self): """Get the first repo in precedence order.""" return self.repos[0] if self.repos else None def all_package_names(self): """Return all unique package names in all repositories.""" if self._all_package_names is None: all_pkgs = set() for repo in self.repos: for name in repo.all_package_names(): all_pkgs.add(name) self._all_package_names = sorted(all_pkgs, key=lambda n: n.lower()) return self._all_package_names def packages_with_tags(self, *tags): r = set() for repo in self.repos: r |= set(repo.packages_with_tags(*tags)) return sorted(r) def all_packages(self): for name in self.all_package_names(): yield self.get(name) @property def provider_index(self): """Merged ProviderIndex from all Repos in the RepoPath.""" if self._provider_index is None: self._provider_index = ProviderIndex() for repo in reversed(self.repos): self._provider_index.merge(repo.provider_index) return self._provider_index @_autospec def providers_for(self, vpkg_spec): providers = self.provider_index.providers_for(vpkg_spec) if not providers: raise UnknownPackageError(vpkg_spec.name) return providers @_autospec def extensions_for(self, extendee_spec): return [p for p in self.all_packages() if p.extends(extendee_spec)] def find_module(self, fullname, path=None): """Implements precedence for overlaid namespaces. Loop checks each namespace in self.repos for packages, and also handles loading empty containing namespaces. """ # namespaces are added to repo, and package modules are leaves. namespace, dot, module_name = fullname.rpartition('.') # If it's a module in some repo, or if it is the repo's # namespace, let the repo handle it. for repo in self.repos: if namespace == repo.full_namespace: if repo.real_name(module_name): return repo elif fullname == repo.full_namespace: return repo # No repo provides the namespace, but it is a valid prefix of # something in the RepoPath. if self.by_namespace.is_prefix(fullname): return self return None def load_module(self, fullname): """Handles loading container namespaces when necessary. See ``Repo`` for how actual package modules are loaded. """ if fullname in sys.modules: return sys.modules[fullname] if not self.by_namespace.is_prefix(fullname): raise ImportError("No such Spack repo: %s" % fullname) module = SpackNamespace(fullname) module.__loader__ = self sys.modules[fullname] = module return module def repo_for_pkg(self, spec): """Given a spec, get the repository for its package.""" # We don't @_autospec this function b/c it's called very frequently # and we want to avoid parsing str's into Specs unnecessarily. namespace = None if isinstance(spec, spack.spec.Spec): namespace = spec.namespace name = spec.name else: # handle strings directly for speed instead of @_autospec'ing namespace, _, name = spec.rpartition('.') # If the spec already has a namespace, then return the # corresponding repo if we know about it. if namespace: fullspace = '%s.%s' % (self.super_namespace, namespace) if fullspace not in self.by_namespace: raise UnknownNamespaceError(spec.namespace) return self.by_namespace[fullspace] # If there's no namespace, search in the RepoPath. for repo in self.repos: if name in repo: return repo # If the package isn't in any repo, return the one with # highest precedence. This is for commands like `spack edit` # that can operate on packages that don't exist yet. return self.first_repo() @_autospec def get(self, spec, new=False): """Find a repo that contains the supplied spec's package. Raises UnknownPackageError if not found. """ return self.repo_for_pkg(spec).get(spec) def get_pkg_class(self, pkg_name): """Find a class for the spec's package and return the class object.""" return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name) @_autospec def dump_provenance(self, spec, path): """Dump provenance information for a spec to a particular path. This dumps the package file and any associated patch files. Raises UnknownPackageError if not found. """ return self.repo_for_pkg(spec).dump_provenance(spec, path) def dirname_for_package_name(self, pkg_name): return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) def filename_for_package_name(self, pkg_name): return self.repo_for_pkg(pkg_name).filename_for_package_name(pkg_name) def exists(self, pkg_name): """Whether package with the give name exists in the path's repos. Note that virtual packages do not "exist". """ return any(repo.exists(pkg_name) for repo in self.repos) def is_virtual(self, pkg_name): """True if the package with this name is virtual, False otherwise.""" return pkg_name in self.provider_index def __contains__(self, pkg_name): return self.exists(pkg_name)
class RepoPath(object): """A RepoPath is a list of repos that function as one. It functions exactly like a Repo, but it operates on the combined results of the Repos in its list instead of on a single package repository. """ def __init__(self, *repo_dirs, **kwargs): # super-namespace for all packages in the RepoPath self.super_namespace = kwargs.get('namespace', repo_namespace) self.repos = [] self.by_namespace = NamespaceTrie() self.by_path = {} self._all_package_names = None self._provider_index = None # If repo_dirs is empty, just use the configuration if not repo_dirs: import spack.config repo_dirs = spack.config.get_config('repos') if not repo_dirs: raise NoRepoConfiguredError( "Spack configuration contains no package repositories.") # Add each repo to this path. for root in repo_dirs: try: repo = Repo(root, self.super_namespace) self.put_last(repo) except RepoError as e: tty.warn("Failed to initialize repository at '%s'." % root, e.message, "To remove the bad repository, run this command:", " spack repo rm %s" % root) def swap(self, other): """Convenience function to make swapping repositories easier. This is currently used by mock tests. TODO: Maybe there is a cleaner way. """ attrs = ['repos', 'by_namespace', 'by_path', '_all_package_names', '_provider_index'] for attr in attrs: tmp = getattr(self, attr) setattr(self, attr, getattr(other, attr)) setattr(other, attr, tmp) def _add(self, repo): """Add a repository to the namespace and path indexes. Checks for duplicates -- two repos can't have the same root directory, and they provide have the same namespace. """ if repo.root in self.by_path: raise DuplicateRepoError("Duplicate repository: '%s'" % repo.root) if repo.namespace in self.by_namespace: raise DuplicateRepoError( "Package repos '%s' and '%s' both provide namespace %s" % (repo.root, self.by_namespace[repo.namespace].root, repo.namespace)) # Add repo to the pkg indexes self.by_namespace[repo.full_namespace] = repo self.by_path[repo.root] = repo def put_first(self, repo): """Add repo first in the search path.""" self._add(repo) self.repos.insert(0, repo) def put_last(self, repo): """Add repo last in the search path.""" self._add(repo) self.repos.append(repo) def remove(self, repo): """Remove a repo from the search path.""" if repo in self.repos: self.repos.remove(repo) def get_repo(self, namespace, default=NOT_PROVIDED): """Get a repository by namespace. Arguments: namespace: Look up this namespace in the RepoPath, and return it if found. Optional Arguments: default: If default is provided, return it when the namespace isn't found. If not, raise an UnknownNamespaceError. """ fullspace = '%s.%s' % (self.super_namespace, namespace) if fullspace not in self.by_namespace: if default == NOT_PROVIDED: raise UnknownNamespaceError(namespace) return default return self.by_namespace[fullspace] def first_repo(self): """Get the first repo in precedence order.""" return self.repos[0] if self.repos else None def all_package_names(self): """Return all unique package names in all repositories.""" if self._all_package_names is None: all_pkgs = set() for repo in self.repos: for name in repo.all_package_names(): all_pkgs.add(name) self._all_package_names = sorted(all_pkgs, key=lambda n: n.lower()) return self._all_package_names def all_packages(self): for name in self.all_package_names(): yield self.get(name) @property def provider_index(self): """Merged ProviderIndex from all Repos in the RepoPath.""" if self._provider_index is None: self._provider_index = ProviderIndex() for repo in reversed(self.repos): self._provider_index.merge(repo.provider_index) return self._provider_index @_autospec def providers_for(self, vpkg_spec): providers = self.provider_index.providers_for(vpkg_spec) if not providers: raise UnknownPackageError(vpkg_spec.name) return providers @_autospec def extensions_for(self, extendee_spec): return [p for p in self.all_packages() if p.extends(extendee_spec)] def find_module(self, fullname, path=None): """Implements precedence for overlaid namespaces. Loop checks each namespace in self.repos for packages, and also handles loading empty containing namespaces. """ # namespaces are added to repo, and package modules are leaves. namespace, dot, module_name = fullname.rpartition('.') # If it's a module in some repo, or if it is the repo's # namespace, let the repo handle it. for repo in self.repos: if namespace == repo.full_namespace: if repo.real_name(module_name): return repo elif fullname == repo.full_namespace: return repo # No repo provides the namespace, but it is a valid prefix of # something in the RepoPath. if self.by_namespace.is_prefix(fullname): return self return None def load_module(self, fullname): """Handles loading container namespaces when necessary. See ``Repo`` for how actual package modules are loaded. """ if fullname in sys.modules: return sys.modules[fullname] if not self.by_namespace.is_prefix(fullname): raise ImportError("No such Spack repo: %s" % fullname) module = SpackNamespace(fullname) module.__loader__ = self sys.modules[fullname] = module return module @_autospec def repo_for_pkg(self, spec): """Given a spec, get the repository for its package.""" # If the spec already has a namespace, then return the # corresponding repo if we know about it. if spec.namespace: fullspace = '%s.%s' % (self.super_namespace, spec.namespace) if fullspace not in self.by_namespace: raise UnknownNamespaceError(spec.namespace) return self.by_namespace[fullspace] # If there's no namespace, search in the RepoPath. for repo in self.repos: if spec.name in repo: return repo # If the package isn't in any repo, return the one with # highest precedence. This is for commands like `spack edit` # that can operate on packages that don't exist yet. return self.first_repo() @_autospec def get(self, spec, new=False): """Find a repo that contains the supplied spec's package. Raises UnknownPackageError if not found. """ return self.repo_for_pkg(spec).get(spec) def get_pkg_class(self, pkg_name): """Find a class for the spec's package and return the class object.""" return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name) @_autospec def dump_provenance(self, spec, path): """Dump provenance information for a spec to a particular path. This dumps the package file and any associated patch files. Raises UnknownPackageError if not found. """ return self.repo_for_pkg(spec).dump_provenance(spec, path) def dirname_for_package_name(self, pkg_name): return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) def filename_for_package_name(self, pkg_name): return self.repo_for_pkg(pkg_name).filename_for_package_name(pkg_name) def exists(self, pkg_name): """Whether package with the give name exists in the path's repos. Note that virtual packages do not "exist". """ return any(repo.exists(pkg_name) for repo in self.repos) def is_virtual(self, pkg_name): """True if the package with this name is virtual, False otherwise.""" return pkg_name in self.provider_index def __contains__(self, pkg_name): return self.exists(pkg_name)