def __init__(self, root, layout, **kwargs): super(YamlFilesystemView, self).__init__(root, layout, **kwargs) # Super class gets projections from the kwargs # YAML specific to get projections from YAML file projections_path = os.path.join(self._root, _projections_path) if not self.projections: if os.path.exists(projections_path): # Read projections file from view with open(projections_path, 'r') as f: projections_data = s_yaml.load(f) spack.config.validate(projections_data, spack.schema.projections.schema) self.projections = projections_data['projections'] else: # Write projections file to new view # Not strictly necessary as the empty file is the empty # projection but it makes sense for consistency mkdirp(os.path.dirname(projections_path)) with open(projections_path, 'w') as f: f.write(s_yaml.dump({'projections': self.projections})) elif not os.path.exists(projections_path): # Write projections file to new view mkdirp(os.path.dirname(projections_path)) with open(projections_path, 'w') as f: f.write(s_yaml.dump({'projections': self.projections})) else: msg = 'View at %s has projections file' % self._root msg += ' and was passed projections manually.' raise ConflictingProjectionsError(msg) self.extensions_layout = YamlViewExtensionsLayout(self, layout) self._croot = colorize_root(self._root) + " "
def activate(parser, args): # Somehow activation is sensible to quoting on the command line # The code below is a quick fix to parse the spec as if it was unquoted, # and work around a subtle bug. The bug itself is still to be found pieces = [] for x in args.spec: pieces.extend(x.split()) args.spec = pieces specs = spack.cmd.parse_specs(args.spec) if len(specs) != 1: tty.die("activate requires one spec. %d given." % len(specs)) spec = spack.cmd.disambiguate_spec(specs[0]) if not spec.package.is_extension: tty.die("%s is not an extension." % spec.name) layout = spack.store.extensions if args.view is not None: layout = YamlViewExtensionsLayout(args.view, spack.store.layout) if spec.package.is_activated(extensions_layout=layout): tty.msg("Package %s is already activated." % specs[0].short_spec) return spec.package.do_activate(extensions_layout=layout, with_dependencies=not args.force)
def deactivate(parser, args): specs = spack.cmd.parse_specs(args.spec) if len(specs) != 1: tty.die("deactivate requires one spec. %d given." % len(specs)) spec = spack.cmd.disambiguate_spec(specs[0]) pkg = spec.package layout = spack.store.extensions if args.view is not None: layout = YamlViewExtensionsLayout(args.view, spack.store.layout) if args.all: if pkg.extendable: tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec) ext_pkgs = spack.store.db.activated_extensions_for( spec, extensions_layout=layout) for ext_pkg in ext_pkgs: ext_pkg.spec.normalize() if ext_pkg.is_activated(): ext_pkg.do_deactivate(force=True, extensions_layout=layout) elif pkg.is_extension: if not args.force and \ not spec.package.is_activated(extensions_layout=layout): tty.die("%s is not activated." % pkg.spec.short_spec) tty.msg("Deactivating %s and all dependencies." % pkg.spec.short_spec) topo_order = topological_sort(spec) index = spec.index() for name in topo_order: espec = index[name] epkg = espec.package if epkg.extends(pkg.extendee_spec): if epkg.is_activated(extensions_layout=layout) or \ args.force: epkg.do_deactivate(force=args.force, extensions_layout=layout) else: tty.die("spack deactivate --all requires an extendable package " "or an extension.") else: if not pkg.is_extension: tty.die("spack deactivate requires an extension.", "Did you mean 'spack deactivate --all'?") if not args.force and \ not spec.package.is_activated(extensions_layout=layout): tty.die("Package %s is not activated." % specs[0].short_spec) spec.package.do_deactivate(force=args.force, extensions_layout=layout)
def __init__(self, root, layout, **kwargs): super(YamlFilesystemView, self).__init__(root, layout, **kwargs) # Super class gets projections from the kwargs # YAML specific to get projections from YAML file self.projections_path = os.path.join(self._root, _projections_path) if not self.projections: # Read projections file from view self.projections = self.read_projections() elif not os.path.exists(self.projections_path): # Write projections file to new view self.write_projections() else: # Ensure projections are the same from each source # Read projections file from view if self.projections != self.read_projections(): msg = 'View at %s has projections file' % self._root msg += ' which does not match projections passed manually.' raise ConflictingProjectionsError(msg) self.extensions_layout = YamlViewExtensionsLayout(self, layout) self._croot = colorize_root(self._root) + " "
def activate(parser, args): specs = spack.cmd.parse_specs(args.spec) if len(specs) != 1: tty.die("activate requires one spec. %d given." % len(specs)) spec = spack.cmd.disambiguate_spec(specs[0]) if not spec.package.is_extension: tty.die("%s is not an extension." % spec.name) layout = spack.store.extensions if args.view is not None: layout = YamlViewExtensionsLayout(args.view, spack.store.layout) if spec.package.is_activated(extensions_layout=layout): tty.die("Package %s is already activated." % specs[0].short_spec) spec.package.do_activate(extensions_layout=layout)
class YamlFilesystemView(FilesystemView): """ Filesystem view to work with a yaml based directory layout. """ def __init__(self, root, layout, **kwargs): super(YamlFilesystemView, self).__init__(root, layout, **kwargs) # Super class gets projections from the kwargs # YAML specific to get projections from YAML file self.projections_path = os.path.join(self._root, _projections_path) if not self.projections: # Read projections file from view self.projections = self.read_projections() elif not os.path.exists(self.projections_path): # Write projections file to new view self.write_projections() else: # Ensure projections are the same from each source # Read projections file from view if self.projections != self.read_projections(): msg = 'View at %s has projections file' % self._root msg += ' which does not match projections passed manually.' raise ConflictingProjectionsError(msg) self.extensions_layout = YamlViewExtensionsLayout(self, layout) self._croot = colorize_root(self._root) + " " def write_projections(self): if self.projections: mkdirp(os.path.dirname(self.projections_path)) with open(self.projections_path, 'w') as f: f.write(s_yaml.dump_config({'projections': self.projections})) def read_projections(self): if os.path.exists(self.projections_path): with open(self.projections_path, 'r') as f: projections_data = s_yaml.load(f) spack.config.validate(projections_data, spack.schema.projections.schema) return projections_data['projections'] else: return {} def add_specs(self, *specs, **kwargs): assert all((s.concrete for s in specs)) specs = set(specs) if kwargs.get("with_dependencies", True): specs.update(get_dependencies(specs)) if kwargs.get("exclude", None): specs = set(filter_exclude(specs, kwargs["exclude"])) conflicts = self.get_conflicts(*specs) if conflicts: for s, v in conflicts: self.print_conflict(v, s) return extensions = set(filter(lambda s: s.package.is_extension, specs)) standalones = specs - extensions set(map(self._check_no_ext_conflicts, extensions)) # fail on first error, otherwise link extensions as well if all(map(self.add_standalone, standalones)): all(map(self.add_extension, extensions)) def add_extension(self, spec): if not spec.package.is_extension: tty.error(self._croot + 'Package %s is not an extension.' % spec.name) return False if spec.external: tty.warn(self._croot + 'Skipping external package: %s' % colorize_spec(spec)) return True if not spec.package.is_activated(self): spec.package.do_activate(self, verbose=self.verbose, with_dependencies=False) # make sure the meta folder is linked as well (this is not done by the # extension-activation mechnism) if not self.check_added(spec): self.link_meta_folder(spec) return True def add_standalone(self, spec): if spec.package.is_extension: tty.error(self._croot + 'Package %s is an extension.' % spec.name) return False if spec.external: tty.warn(self._croot + 'Skipping external package: %s' % colorize_spec(spec)) return True if self.check_added(spec): tty.warn(self._croot + 'Skipping already linked package: %s' % colorize_spec(spec)) return True if spec.package.extendable: # Check for globally activated extensions in the extendee that # we're looking at. activated = [ p.spec for p in spack.store.db.activated_extensions_for(spec) ] if activated: tty.error("Globally activated extensions cannot be used in " "conjunction with filesystem views. " "Please deactivate the following specs: ") spack.cmd.display_specs(activated, flags=True, variants=True, long=False) return False self.merge(spec) self.link_meta_folder(spec) if self.verbose: tty.info(self._croot + 'Linked package: %s' % colorize_spec(spec)) return True def merge(self, spec, ignore=None): pkg = spec.package view_source = pkg.view_source() view_dst = pkg.view_destination(self) tree = LinkTree(view_source) ignore = ignore or (lambda f: False) ignore_file = match_predicate(self.layout.hidden_file_regexes, ignore) # check for dir conflicts conflicts = tree.find_dir_conflicts(view_dst, ignore_file) merge_map = tree.get_file_map(view_dst, ignore_file) if not self.ignore_conflicts: conflicts.extend(pkg.view_file_conflicts(self, merge_map)) if conflicts: raise MergeConflictError(conflicts[0]) # merge directories with the tree tree.merge_directories(view_dst, ignore_file) pkg.add_files_to_view(self, merge_map) def unmerge(self, spec, ignore=None): pkg = spec.package view_source = pkg.view_source() view_dst = pkg.view_destination(self) tree = LinkTree(view_source) ignore = ignore or (lambda f: False) ignore_file = match_predicate(self.layout.hidden_file_regexes, ignore) merge_map = tree.get_file_map(view_dst, ignore_file) pkg.remove_files_from_view(self, merge_map) # now unmerge the directory tree tree.unmerge_directories(view_dst, ignore_file) def remove_files(self, files): def needs_file(spec, file): # convert the file we want to remove to a source in this spec projection = self.get_projection_for_spec(spec) relative_path = os.path.relpath(file, projection) test_path = os.path.join(spec.prefix, relative_path) # check if this spec owns a file of that name (through the # manifest in the metadata dir, which we have in the view). manifest_file = os.path.join(self.get_path_meta_folder(spec), spack.store.layout.manifest_file_name) try: with open(manifest_file, 'r') as f: manifest = s_json.load(f) except (OSError, IOError): # if we can't load it, assume it doesn't know about the file. manifest = {} return test_path in manifest specs = self.get_all_specs() for file in files: if not os.path.lexists(file): tty.warn("Tried to remove %s which does not exist" % file) continue # remove if file is not owned by any other package in the view # This will only be false if two packages are merged into a prefix # and have a conflicting file # check all specs for whether they own the file. That include the spec # we are currently removing, as we remove files before unlinking the # metadata directory. if len([s for s in specs if needs_file(s, file)]) <= 1: tty.debug("Removing file " + file) os.remove(file) def check_added(self, spec): assert spec.concrete return spec == self.get_spec(spec) def remove_specs(self, *specs, **kwargs): assert all((s.concrete for s in specs)) with_dependents = kwargs.get("with_dependents", True) with_dependencies = kwargs.get("with_dependencies", False) # caller can pass this in, as get_all_specs() is expensive all_specs = kwargs.get("all_specs", None) or set(self.get_all_specs()) specs = set(specs) if with_dependencies: specs = get_dependencies(specs) if kwargs.get("exclude", None): specs = set(filter_exclude(specs, kwargs["exclude"])) to_deactivate = specs to_keep = all_specs - to_deactivate dependents = find_dependents(to_keep, to_deactivate) if with_dependents: # remove all packages depending on the ones to remove if len(dependents) > 0: tty.warn(self._croot + "The following dependents will be removed: %s" % ", ".join((s.name for s in dependents))) to_deactivate.update(dependents) elif len(dependents) > 0: tty.warn(self._croot + "The following packages will be unusable: %s" % ", ".join((s.name for s in dependents))) # Determine the order that packages should be removed from the view; # dependents come before their dependencies. to_deactivate_sorted = list() depmap = dict() for spec in to_deactivate: depmap[spec] = set(d for d in spec.traverse(root=False) if d in to_deactivate) while depmap: for spec in [s for s, d in depmap.items() if not d]: to_deactivate_sorted.append(spec) for s in depmap.keys(): depmap[s].discard(spec) depmap.pop(spec) to_deactivate_sorted.reverse() # Ensure that the sorted list contains all the packages assert set(to_deactivate_sorted) == to_deactivate # Remove the packages from the view for spec in to_deactivate_sorted: if spec.package.is_extension: self.remove_extension(spec, with_dependents=with_dependents) else: self.remove_standalone(spec) self._purge_empty_directories() def remove_extension(self, spec, with_dependents=True): """ Remove (unlink) an extension from this view. """ if not self.check_added(spec): tty.warn(self._croot + 'Skipping package not linked in view: %s' % spec.name) return if spec.package.is_activated(self): spec.package.do_deactivate(self, verbose=self.verbose, remove_dependents=with_dependents) self.unlink_meta_folder(spec) def remove_standalone(self, spec): """ Remove (unlink) a standalone package from this view. """ if not self.check_added(spec): tty.warn(self._croot + 'Skipping package not linked in view: %s' % spec.name) return self.unmerge(spec) self.unlink_meta_folder(spec) if self.verbose: tty.info(self._croot + 'Removed package: %s' % colorize_spec(spec)) def get_projection_for_spec(self, spec): """ Return the projection for a spec in this view. Relies on the ordering of projections to avoid ambiguity. """ spec = spack.spec.Spec(spec) # Extensions are placed by their extendee, not by their own spec locator_spec = spec if spec.package.extendee_spec: locator_spec = spec.package.extendee_spec proj = spack.projections.get_projection(self.projections, locator_spec) if proj: return os.path.join(self._root, locator_spec.format(proj)) return self._root def get_all_specs(self): md_dirs = [] for root, dirs, files in os.walk(self._root): if spack.store.layout.metadata_dir in dirs: md_dirs.append( os.path.join(root, spack.store.layout.metadata_dir)) specs = [] for md_dir in md_dirs: if os.path.exists(md_dir): for name_dir in os.listdir(md_dir): filename = os.path.join(md_dir, name_dir, spack.store.layout.spec_file_name) spec = get_spec_from_file(filename) if spec: specs.append(spec) return specs def get_conflicts(self, *specs): """ Return list of tuples (<spec>, <spec in view>) where the spec active in the view differs from the one to be activated. """ in_view = map(self.get_spec, specs) return [(s, v) for s, v in zip(specs, in_view) if v is not None and s != v] def get_path_meta_folder(self, spec): "Get path to meta folder for either spec or spec name." return os.path.join(self.get_projection_for_spec(spec), spack.store.layout.metadata_dir, getattr(spec, "name", spec)) def get_spec(self, spec): dotspack = self.get_path_meta_folder(spec) filename = os.path.join(dotspack, spack.store.layout.spec_file_name) return get_spec_from_file(filename) def link_meta_folder(self, spec): src = spack.store.layout.metadata_path(spec) tgt = self.get_path_meta_folder(spec) tree = LinkTree(src) # there should be no conflicts when linking the meta folder tree.merge(tgt, link=self.link) def print_conflict(self, spec_active, spec_specified, level="error"): "Singular print function for spec conflicts." cprint = getattr(tty, level) color = sys.stdout.isatty() linked = tty.color.colorize(" (@gLinked@.)", color=color) specified = tty.color.colorize("(@rSpecified@.)", color=color) cprint(self._croot + "Package conflict detected:\n" "%s %s\n" % (linked, colorize_spec(spec_active)) + "%s %s" % (specified, colorize_spec(spec_specified))) def print_status(self, *specs, **kwargs): if kwargs.get("with_dependencies", False): specs = set(get_dependencies(specs)) specs = sorted(specs, key=lambda s: s.name) in_view = list(map(self.get_spec, specs)) for s, v in zip(specs, in_view): if not v: tty.error(self._croot + 'Package not linked: %s' % s.name) elif s != v: self.print_conflict(v, s, level="warn") in_view = list(filter(None, in_view)) if len(specs) > 0: tty.msg("Packages linked in %s:" % self._croot[:-1]) # Make a dict with specs keyed by architecture and compiler. index = index_by(specs, ('architecture', 'compiler')) # Traverse the index and print out each package for i, (architecture, compiler) in enumerate(sorted(index)): if i > 0: print() header = "%s{%s} / %s{%s}" % ( spack.spec.architecture_color, architecture, spack.spec.compiler_color, compiler) tty.hline(colorize(header), char='-') specs = index[(architecture, compiler)] specs.sort() format_string = '{name}{@version}' format_string += '{%compiler}{compiler_flags}{variants}' abbreviated = [s.cformat(format_string) for s in specs] # Print one spec per line along with prefix path width = max(len(s) for s in abbreviated) width += 2 format = " %%-%ds%%s" % width for abbrv, s in zip(abbreviated, specs): prefix = '' if self.verbose: prefix = colorize('@K{%s}' % s.dag_hash(7)) print(prefix + (format % (abbrv, self.get_projection_for_spec(s)))) else: tty.warn(self._croot + "No packages found.") def _purge_empty_directories(self): remove_empty_directories(self._root) def _purge_broken_links(self): remove_dead_links(self._root) def clean(self): self._purge_broken_links() self._purge_empty_directories() def unlink_meta_folder(self, spec): path = self.get_path_meta_folder(spec) assert os.path.exists(path) shutil.rmtree(path) def _check_no_ext_conflicts(self, spec): """ Check that there is no extension conflict for specs. """ extendee = spec.package.extendee_spec try: self.extensions_layout.check_extension_conflict(extendee, spec) except ExtensionAlreadyInstalledError: # we print the warning here because later on the order in which # packages get activated is not clear (set-sorting) tty.warn(self._croot + 'Skipping already activated package: %s' % spec.name)
def __init__(self, root, layout, **kwargs): super(YamlFilesystemView, self).__init__(root, layout, **kwargs) self.extensions_layout = YamlViewExtensionsLayout(root, layout) self._croot = colorize_root(self.root) + " "
class YamlFilesystemView(FilesystemView): """ Filesystem view to work with a yaml based directory layout. """ def __init__(self, root, layout, **kwargs): super(YamlFilesystemView, self).__init__(root, layout, **kwargs) self.extensions_layout = YamlViewExtensionsLayout(root, layout) self._croot = colorize_root(self.root) + " " def add_specs(self, *specs, **kwargs): assert all((s.concrete for s in specs)) specs = set(specs) if kwargs.get("with_dependencies", True): specs.update(get_dependencies(specs)) if kwargs.get("exclude", None): specs = set(filter_exclude(specs, kwargs["exclude"])) conflicts = self.get_conflicts(*specs) if conflicts: for s, v in conflicts: self.print_conflict(v, s) return extensions = set(filter(lambda s: s.package.is_extension, specs)) standalones = specs - extensions set(map(self._check_no_ext_conflicts, extensions)) # fail on first error, otherwise link extensions as well if all(map(self.add_standalone, standalones)): all(map(self.add_extension, extensions)) def add_extension(self, spec): if not spec.package.is_extension: tty.error(self._croot + 'Package %s is not an extension.' % spec.name) return False if spec.external: tty.warn(self._croot + 'Skipping external package: %s' % colorize_spec(spec)) return True if not spec.package.is_activated(self): spec.package.do_activate( self, verbose=self.verbose, with_dependencies=False) # make sure the meta folder is linked as well (this is not done by the # extension-activation mechnism) if not self.check_added(spec): self.link_meta_folder(spec) return True def add_standalone(self, spec): if spec.package.is_extension: tty.error(self._croot + 'Package %s is an extension.' % spec.name) return False if spec.external: tty.warn(self._croot + 'Skipping external package: %s' % colorize_spec(spec)) return True if self.check_added(spec): tty.warn(self._croot + 'Skipping already linked package: %s' % colorize_spec(spec)) return True if spec.package.extendable: # Check for globally activated extensions in the extendee that # we're looking at. activated = [p.spec for p in spack.store.db.activated_extensions_for(spec)] if activated: tty.error("Globally activated extensions cannot be used in " "conjunction with filesystem views. " "Please deactivate the following specs: ") spack.cmd.display_specs(activated, flags=True, variants=True, long=False) return False self.merge(spec) self.link_meta_folder(spec) if self.verbose: tty.info(self._croot + 'Linked package: %s' % colorize_spec(spec)) return True def merge(self, spec, ignore=None): pkg = spec.package view_source = pkg.view_source() view_dst = pkg.view_destination(self) tree = LinkTree(view_source) ignore = ignore or (lambda f: False) ignore_file = match_predicate( self.layout.hidden_file_paths, ignore) # check for dir conflicts conflicts = tree.find_dir_conflicts(view_dst, ignore_file) merge_map = tree.get_file_map(view_dst, ignore_file) if not self.ignore_conflicts: conflicts.extend(pkg.view_file_conflicts(self, merge_map)) if conflicts: raise MergeConflictError(conflicts[0]) # merge directories with the tree tree.merge_directories(view_dst, ignore_file) pkg.add_files_to_view(self, merge_map) def unmerge(self, spec, ignore=None): pkg = spec.package view_source = pkg.view_source() view_dst = pkg.view_destination(self) tree = LinkTree(view_source) ignore = ignore or (lambda f: False) ignore_file = match_predicate( self.layout.hidden_file_paths, ignore) merge_map = tree.get_file_map(view_dst, ignore_file) pkg.remove_files_from_view(self, merge_map) # now unmerge the directory tree tree.unmerge_directories(view_dst, ignore_file) def remove_file(self, src, dest): if not os.path.islink(dest): raise ValueError("%s is not a link tree!" % dest) # remove if dest is a hardlink/symlink to src; this will only # be false if two packages are merged into a prefix and have a # conflicting file if filecmp.cmp(src, dest, shallow=True): os.remove(dest) def check_added(self, spec): assert spec.concrete return spec == self.get_spec(spec) def remove_specs(self, *specs, **kwargs): assert all((s.concrete for s in specs)) with_dependents = kwargs.get("with_dependents", True) with_dependencies = kwargs.get("with_dependencies", False) specs = set(specs) if with_dependencies: specs = get_dependencies(specs) if kwargs.get("exclude", None): specs = set(filter_exclude(specs, kwargs["exclude"])) all_specs = set(self.get_all_specs()) to_deactivate = specs to_keep = all_specs - to_deactivate dependents = find_dependents(to_keep, to_deactivate) if with_dependents: # remove all packages depending on the ones to remove if len(dependents) > 0: tty.warn(self._croot + "The following dependents will be removed: %s" % ", ".join((s.name for s in dependents))) to_deactivate.update(dependents) elif len(dependents) > 0: tty.warn(self._croot + "The following packages will be unusable: %s" % ", ".join((s.name for s in dependents))) extensions = set(filter(lambda s: s.package.is_extension, to_deactivate)) standalones = to_deactivate - extensions # Please note that a traversal of the DAG in post-order and then # forcibly removing each package should remove the need to specify # with_dependents for deactivating extensions/allow removal without # additional checks (force=True). If removal performance becomes # unbearable for whatever reason, this should be the first point of # attack. # # see: https://github.com/spack/spack/pull/3227#discussion_r117147475 remove_extension = ft.partial(self.remove_extension, with_dependents=with_dependents) set(map(remove_extension, extensions)) set(map(self.remove_standalone, standalones)) self.purge_empty_directories() def remove_extension(self, spec, with_dependents=True): """ Remove (unlink) an extension from this view. """ if not self.check_added(spec): tty.warn(self._croot + 'Skipping package not linked in view: %s' % spec.name) return # The spec might have been deactivated as depdency of another package # already if spec.package.is_activated(self): spec.package.do_deactivate( self, verbose=self.verbose, remove_dependents=with_dependents) self.unlink_meta_folder(spec) def remove_standalone(self, spec): """ Remove (unlink) a standalone package from this view. """ if not self.check_added(spec): tty.warn(self._croot + 'Skipping package not linked in view: %s' % spec.name) return self.unmerge(spec) self.unlink_meta_folder(spec) if self.verbose: tty.info(self._croot + 'Removed package: %s' % colorize_spec(spec)) def get_all_specs(self): dotspack = os.path.join(self.root, spack.store.layout.metadata_dir) if os.path.exists(dotspack): return list(filter(None, map(self.get_spec, os.listdir(dotspack)))) else: return [] def get_conflicts(self, *specs): """ Return list of tuples (<spec>, <spec in view>) where the spec active in the view differs from the one to be activated. """ in_view = map(self.get_spec, specs) return [(s, v) for s, v in zip(specs, in_view) if v is not None and s != v] def get_path_meta_folder(self, spec): "Get path to meta folder for either spec or spec name." return os.path.join(self.root, spack.store.layout.metadata_dir, getattr(spec, "name", spec)) def get_spec(self, spec): dotspack = self.get_path_meta_folder(spec) filename = os.path.join(dotspack, spack.store.layout.spec_file_name) try: with open(filename, "r") as f: return spack.spec.Spec.from_yaml(f) except IOError: return None def link_meta_folder(self, spec): src = spack.store.layout.metadata_path(spec) tgt = self.get_path_meta_folder(spec) tree = LinkTree(src) # there should be no conflicts when linking the meta folder tree.merge(tgt, link=self.link) def print_conflict(self, spec_active, spec_specified, level="error"): "Singular print function for spec conflicts." cprint = getattr(tty, level) color = sys.stdout.isatty() linked = tty.color.colorize(" (@gLinked@.)", color=color) specified = tty.color.colorize("(@rSpecified@.)", color=color) cprint(self._croot + "Package conflict detected:\n" "%s %s\n" % (linked, colorize_spec(spec_active)) + "%s %s" % (specified, colorize_spec(spec_specified))) def print_status(self, *specs, **kwargs): if kwargs.get("with_dependencies", False): specs = set(get_dependencies(specs)) specs = sorted(specs, key=lambda s: s.name) in_view = list(map(self.get_spec, specs)) for s, v in zip(specs, in_view): if not v: tty.error(self._croot + 'Package not linked: %s' % s.name) elif s != v: self.print_conflict(v, s, level="warn") in_view = list(filter(None, in_view)) if len(specs) > 0: tty.msg("Packages linked in %s:" % self._croot[:-1]) # avoid circular dependency import spack.cmd spack.cmd.display_specs(in_view, flags=True, variants=True, long=self.verbose) else: tty.warn(self._croot + "No packages found.") def purge_empty_directories(self): """ Ascend up from the leaves accessible from `path` and remove empty directories. """ for dirpath, subdirs, files in os.walk(self.root, topdown=False): for sd in subdirs: sdp = os.path.join(dirpath, sd) try: os.rmdir(sdp) except OSError: pass def unlink_meta_folder(self, spec): path = self.get_path_meta_folder(spec) assert os.path.exists(path) shutil.rmtree(path) def _check_no_ext_conflicts(self, spec): """ Check that there is no extension conflict for specs. """ extendee = spec.package.extendee_spec try: self.extensions_layout.check_extension_conflict(extendee, spec) except ExtensionAlreadyInstalledError: # we print the warning here because later on the order in which # packages get activated is not clear (set-sorting) tty.warn(self._croot + 'Skipping already activated package: %s' % spec.name)
def extensions(parser, args): if not args.spec: tty.die("extensions requires a package spec.") show_packages = False show_installed = False show_activated = False show_all = False if args.show == 'packages': show_packages = True elif args.show == 'installed': show_installed = True elif args.show == 'activated': show_activated = True elif args.show == 'all': show_packages = True show_installed = True show_activated = True show_all = True else: tty.die('unrecognized show type: %s' % args.show) # # Checks # spec = spack.cmd.parse_specs(args.spec) if len(spec) > 1: tty.die("Can only list extensions for one package.") if not spec[0].package.extendable: tty.die("%s is not an extendable package." % spec[0].name) spec = spack.cmd.disambiguate_spec(spec[0]) if not spec.package.extendable: tty.die("%s does not have extensions." % spec.short_spec) if not args.mode: args.mode = 'short' if show_packages: # # List package names of extensions extensions = spack.repo.extensions_for(spec) if not extensions: tty.msg("%s has no extensions." % spec.cshort_spec) else: tty.msg(spec.cshort_spec) tty.msg("%d extensions:" % len(extensions)) colify(ext.name for ext in extensions) layout = spack.store.extensions if args.view is not None: layout = YamlViewExtensionsLayout(args.view, spack.store.layout) if show_installed: # # List specs of installed extensions. # installed = [ s.spec for s in spack.store.db.installed_extensions_for(spec) ] if show_all: print if not installed: tty.msg("None installed.") else: tty.msg("%d installed:" % len(installed)) spack.cmd.find.display_specs(installed, mode=args.mode) if show_activated: # # List specs of activated extensions. # activated = layout.extension_map(spec) if show_all: print if not activated: tty.msg("None activated.") else: tty.msg("%d currently activated:" % len(activated)) spack.cmd.find.display_specs(activated.values(), mode=args.mode, long=args.long)
def __init__(self, root, layout, **kwargs): super(YamlFilesystemView, self).__init__(root, layout, **kwargs) self.extensions_layout = YamlViewExtensionsLayout(root, layout) self._croot = colorize_root(self.root) + " "
class YamlFilesystemView(FilesystemView): """ Filesystem view to work with a yaml based directory layout. """ def __init__(self, root, layout, **kwargs): super(YamlFilesystemView, self).__init__(root, layout, **kwargs) self.extensions_layout = YamlViewExtensionsLayout(root, layout) self._croot = colorize_root(self.root) + " " def add_specs(self, *specs, **kwargs): assert all((s.concrete for s in specs)) specs = set(specs) if kwargs.get("with_dependencies", True): specs.update(get_dependencies(specs)) if kwargs.get("exclude", None): specs = set(filter_exclude(specs, kwargs["exclude"])) conflicts = self.get_conflicts(*specs) if conflicts: for s, v in conflicts: self.print_conflict(v, s) return extensions = set(filter(lambda s: s.package.is_extension, specs)) standalones = specs - extensions set(map(self._check_no_ext_conflicts, extensions)) # fail on first error, otherwise link extensions as well if all(map(self.add_standalone, standalones)): all(map(self.add_extension, extensions)) def add_extension(self, spec): if not spec.package.is_extension: tty.error(self._croot + 'Package %s is not an extension.' % spec.name) return False if spec.external: tty.warn(self._croot + 'Skipping external package: %s' % colorize_spec(spec)) return True if not spec.package.is_activated(self): spec.package.do_activate( self, verbose=self.verbose, with_dependencies=False) # make sure the meta folder is linked as well (this is not done by the # extension-activation mechnism) if not self.check_added(spec): self.link_meta_folder(spec) return True def add_standalone(self, spec): if spec.package.is_extension: tty.error(self._croot + 'Package %s is an extension.' % spec.name) return False if spec.external: tty.warn(self._croot + 'Skipping external package: %s' % colorize_spec(spec)) return True if self.check_added(spec): tty.warn(self._croot + 'Skipping already linked package: %s' % colorize_spec(spec)) return True if spec.package.extendable: # Check for globally activated extensions in the extendee that # we're looking at. activated = [p.spec for p in spack.store.db.activated_extensions_for(spec)] if activated: tty.error("Globally activated extensions cannot be used in " "conjunction with filesystem views. " "Please deactivate the following specs: ") spack.cmd.display_specs(activated, flags=True, variants=True, long=False) return False self.merge(spec) self.link_meta_folder(spec) if self.verbose: tty.info(self._croot + 'Linked package: %s' % colorize_spec(spec)) return True def merge(self, spec, ignore=None): pkg = spec.package view_source = pkg.view_source() view_dst = pkg.view_destination(self) tree = LinkTree(view_source) ignore = ignore or (lambda f: False) ignore_file = match_predicate( self.layout.hidden_file_paths, ignore) # check for dir conflicts conflicts = tree.find_dir_conflicts(view_dst, ignore_file) merge_map = tree.get_file_map(view_dst, ignore_file) if not self.ignore_conflicts: conflicts.extend(pkg.view_file_conflicts(self, merge_map)) if conflicts: raise MergeConflictError(conflicts[0]) # merge directories with the tree tree.merge_directories(view_dst, ignore_file) pkg.add_files_to_view(self, merge_map) def unmerge(self, spec, ignore=None): pkg = spec.package view_source = pkg.view_source() view_dst = pkg.view_destination(self) tree = LinkTree(view_source) ignore = ignore or (lambda f: False) ignore_file = match_predicate( self.layout.hidden_file_paths, ignore) merge_map = tree.get_file_map(view_dst, ignore_file) pkg.remove_files_from_view(self, merge_map) # now unmerge the directory tree tree.unmerge_directories(view_dst, ignore_file) def remove_file(self, src, dest): if not os.path.islink(dest): raise ValueError("%s is not a link tree!" % dest) # remove if dest is a hardlink/symlink to src; this will only # be false if two packages are merged into a prefix and have a # conflicting file if filecmp.cmp(src, dest, shallow=True): os.remove(dest) def check_added(self, spec): assert spec.concrete return spec == self.get_spec(spec) def remove_specs(self, *specs, **kwargs): assert all((s.concrete for s in specs)) with_dependents = kwargs.get("with_dependents", True) with_dependencies = kwargs.get("with_dependencies", False) specs = set(specs) if with_dependencies: specs = get_dependencies(specs) if kwargs.get("exclude", None): specs = set(filter_exclude(specs, kwargs["exclude"])) all_specs = set(self.get_all_specs()) to_deactivate = specs to_keep = all_specs - to_deactivate dependents = find_dependents(to_keep, to_deactivate) if with_dependents: # remove all packages depending on the ones to remove if len(dependents) > 0: tty.warn(self._croot + "The following dependents will be removed: %s" % ", ".join((s.name for s in dependents))) to_deactivate.update(dependents) elif len(dependents) > 0: tty.warn(self._croot + "The following packages will be unusable: %s" % ", ".join((s.name for s in dependents))) extensions = set(filter(lambda s: s.package.is_extension, to_deactivate)) standalones = to_deactivate - extensions # Please note that a traversal of the DAG in post-order and then # forcibly removing each package should remove the need to specify # with_dependents for deactivating extensions/allow removal without # additional checks (force=True). If removal performance becomes # unbearable for whatever reason, this should be the first point of # attack. # # see: https://github.com/spack/spack/pull/3227#discussion_r117147475 remove_extension = ft.partial(self.remove_extension, with_dependents=with_dependents) set(map(remove_extension, extensions)) set(map(self.remove_standalone, standalones)) self.purge_empty_directories() def remove_extension(self, spec, with_dependents=True): """ Remove (unlink) an extension from this view. """ if not self.check_added(spec): tty.warn(self._croot + 'Skipping package not linked in view: %s' % spec.name) return # The spec might have been deactivated as depdency of another package # already if spec.package.is_activated(self): spec.package.do_deactivate( self, verbose=self.verbose, remove_dependents=with_dependents) self.unlink_meta_folder(spec) def remove_standalone(self, spec): """ Remove (unlink) a standalone package from this view. """ if not self.check_added(spec): tty.warn(self._croot + 'Skipping package not linked in view: %s' % spec.name) return self.unmerge(spec) self.unlink_meta_folder(spec) if self.verbose: tty.info(self._croot + 'Removed package: %s' % colorize_spec(spec)) def get_all_specs(self): dotspack = os.path.join(self.root, spack.store.layout.metadata_dir) if os.path.exists(dotspack): return list(filter(None, map(self.get_spec, os.listdir(dotspack)))) else: return [] def get_conflicts(self, *specs): """ Return list of tuples (<spec>, <spec in view>) where the spec active in the view differs from the one to be activated. """ in_view = map(self.get_spec, specs) return [(s, v) for s, v in zip(specs, in_view) if v is not None and s != v] def get_path_meta_folder(self, spec): "Get path to meta folder for either spec or spec name." return os.path.join(self.root, spack.store.layout.metadata_dir, getattr(spec, "name", spec)) def get_spec(self, spec): dotspack = self.get_path_meta_folder(spec) filename = os.path.join(dotspack, spack.store.layout.spec_file_name) try: with open(filename, "r") as f: return spack.spec.Spec.from_yaml(f) except IOError: return None def link_meta_folder(self, spec): src = spack.store.layout.metadata_path(spec) tgt = self.get_path_meta_folder(spec) tree = LinkTree(src) # there should be no conflicts when linking the meta folder tree.merge(tgt, link=self.link) def print_conflict(self, spec_active, spec_specified, level="error"): "Singular print function for spec conflicts." cprint = getattr(tty, level) color = sys.stdout.isatty() linked = tty.color.colorize(" (@gLinked@.)", color=color) specified = tty.color.colorize("(@rSpecified@.)", color=color) cprint(self._croot + "Package conflict detected:\n" "%s %s\n" % (linked, colorize_spec(spec_active)) + "%s %s" % (specified, colorize_spec(spec_specified))) def print_status(self, *specs, **kwargs): if kwargs.get("with_dependencies", False): specs = set(get_dependencies(specs)) specs = sorted(specs, key=lambda s: s.name) in_view = list(map(self.get_spec, specs)) for s, v in zip(specs, in_view): if not v: tty.error(self._croot + 'Package not linked: %s' % s.name) elif s != v: self.print_conflict(v, s, level="warn") in_view = list(filter(None, in_view)) if len(specs) > 0: tty.msg("Packages linked in %s:" % self._croot[:-1]) # avoid circular dependency import spack.cmd spack.cmd.display_specs(in_view, flags=True, variants=True, long=self.verbose) else: tty.warn(self._croot + "No packages found.") def purge_empty_directories(self): """ Ascend up from the leaves accessible from `path` and remove empty directories. """ for dirpath, subdirs, files in os.walk(self.root, topdown=False): for sd in subdirs: sdp = os.path.join(dirpath, sd) try: os.rmdir(sdp) except OSError: pass def unlink_meta_folder(self, spec): path = self.get_path_meta_folder(spec) assert os.path.exists(path) shutil.rmtree(path) def _check_no_ext_conflicts(self, spec): """ Check that there is no extension conflict for specs. """ extendee = spec.package.extendee_spec try: self.extensions_layout.check_extension_conflict(extendee, spec) except ExtensionAlreadyInstalledError: # we print the warning here because later on the order in which # packages get activated is not clear (set-sorting) tty.warn(self._croot + 'Skipping already activated package: %s' % spec.name)
class YamlFilesystemView(FilesystemView): """ Filesystem view to work with a yaml based directory layout. """ def __init__(self, root, layout, **kwargs): super(YamlFilesystemView, self).__init__(root, layout, **kwargs) # Super class gets projections from the kwargs # YAML specific to get projections from YAML file projections_path = os.path.join(self._root, _projections_path) if not self.projections: if os.path.exists(projections_path): # Read projections file from view with open(projections_path, 'r') as f: projections_data = s_yaml.load(f) spack.config.validate(projections_data, spack.schema.projections.schema) self.projections = projections_data['projections'] else: # Write projections file to new view # Not strictly necessary as the empty file is the empty # projection but it makes sense for consistency mkdirp(os.path.dirname(projections_path)) with open(projections_path, 'w') as f: f.write(s_yaml.dump({'projections': self.projections})) elif not os.path.exists(projections_path): # Write projections file to new view mkdirp(os.path.dirname(projections_path)) with open(projections_path, 'w') as f: f.write(s_yaml.dump({'projections': self.projections})) else: msg = 'View at %s has projections file' % self._root msg += ' and was passed projections manually.' raise ConflictingProjectionsError(msg) self.extensions_layout = YamlViewExtensionsLayout(self, layout) self._croot = colorize_root(self._root) + " " def add_specs(self, *specs, **kwargs): assert all((s.concrete for s in specs)) specs = set(specs) if kwargs.get("with_dependencies", True): specs.update(get_dependencies(specs)) if kwargs.get("exclude", None): specs = set(filter_exclude(specs, kwargs["exclude"])) conflicts = self.get_conflicts(*specs) if conflicts: for s, v in conflicts: self.print_conflict(v, s) return extensions = set(filter(lambda s: s.package.is_extension, specs)) standalones = specs - extensions set(map(self._check_no_ext_conflicts, extensions)) # fail on first error, otherwise link extensions as well if all(map(self.add_standalone, standalones)): all(map(self.add_extension, extensions)) def add_extension(self, spec): if not spec.package.is_extension: tty.error(self._croot + 'Package %s is not an extension.' % spec.name) return False if spec.external: tty.warn(self._croot + 'Skipping external package: %s' % colorize_spec(spec)) return True if not spec.package.is_activated(self): spec.package.do_activate(self, verbose=self.verbose, with_dependencies=False) # make sure the meta folder is linked as well (this is not done by the # extension-activation mechnism) if not self.check_added(spec): self.link_meta_folder(spec) return True def add_standalone(self, spec): if spec.package.is_extension: tty.error(self._croot + 'Package %s is an extension.' % spec.name) return False if spec.external: tty.warn(self._croot + 'Skipping external package: %s' % colorize_spec(spec)) return True if self.check_added(spec): tty.warn(self._croot + 'Skipping already linked package: %s' % colorize_spec(spec)) return True if spec.package.extendable: # Check for globally activated extensions in the extendee that # we're looking at. activated = [ p.spec for p in spack.store.db.activated_extensions_for(spec) ] if activated: tty.error("Globally activated extensions cannot be used in " "conjunction with filesystem views. " "Please deactivate the following specs: ") spack.cmd.display_specs(activated, flags=True, variants=True, long=False) return False self.merge(spec) self.link_meta_folder(spec) if self.verbose: tty.info(self._croot + 'Linked package: %s' % colorize_spec(spec)) return True def merge(self, spec, ignore=None): pkg = spec.package view_source = pkg.view_source() view_dst = pkg.view_destination(self) tree = LinkTree(view_source) ignore = ignore or (lambda f: False) ignore_file = match_predicate(self.layout.hidden_file_paths, ignore) # check for dir conflicts conflicts = tree.find_dir_conflicts(view_dst, ignore_file) merge_map = tree.get_file_map(view_dst, ignore_file) if not self.ignore_conflicts: conflicts.extend(pkg.view_file_conflicts(self, merge_map)) if conflicts: raise MergeConflictError(conflicts[0]) # merge directories with the tree tree.merge_directories(view_dst, ignore_file) pkg.add_files_to_view(self, merge_map) def unmerge(self, spec, ignore=None): pkg = spec.package view_source = pkg.view_source() view_dst = pkg.view_destination(self) tree = LinkTree(view_source) ignore = ignore or (lambda f: False) ignore_file = match_predicate(self.layout.hidden_file_paths, ignore) merge_map = tree.get_file_map(view_dst, ignore_file) pkg.remove_files_from_view(self, merge_map) # now unmerge the directory tree tree.unmerge_directories(view_dst, ignore_file) def remove_file(self, src, dest): if not os.path.islink(dest): raise ValueError("%s is not a link tree!" % dest) # remove if dest is a hardlink/symlink to src; this will only # be false if two packages are merged into a prefix and have a # conflicting file if filecmp.cmp(src, dest, shallow=True): os.remove(dest) def check_added(self, spec): assert spec.concrete return spec == self.get_spec(spec) def remove_specs(self, *specs, **kwargs): assert all((s.concrete for s in specs)) with_dependents = kwargs.get("with_dependents", True) with_dependencies = kwargs.get("with_dependencies", False) specs = set(specs) if with_dependencies: specs = get_dependencies(specs) if kwargs.get("exclude", None): specs = set(filter_exclude(specs, kwargs["exclude"])) all_specs = set(self.get_all_specs()) to_deactivate = specs to_keep = all_specs - to_deactivate dependents = find_dependents(to_keep, to_deactivate) if with_dependents: # remove all packages depending on the ones to remove if len(dependents) > 0: tty.warn(self._croot + "The following dependents will be removed: %s" % ", ".join((s.name for s in dependents))) to_deactivate.update(dependents) elif len(dependents) > 0: tty.warn(self._croot + "The following packages will be unusable: %s" % ", ".join((s.name for s in dependents))) extensions = set( filter(lambda s: s.package.is_extension, to_deactivate)) standalones = to_deactivate - extensions # Please note that a traversal of the DAG in post-order and then # forcibly removing each package should remove the need to specify # with_dependents for deactivating extensions/allow removal without # additional checks (force=True). If removal performance becomes # unbearable for whatever reason, this should be the first point of # attack. # # see: https://github.com/spack/spack/pull/3227#discussion_r117147475 remove_extension = ft.partial(self.remove_extension, with_dependents=with_dependents) set(map(remove_extension, extensions)) set(map(self.remove_standalone, standalones)) self.purge_empty_directories() def remove_extension(self, spec, with_dependents=True): """ Remove (unlink) an extension from this view. """ if not self.check_added(spec): tty.warn(self._croot + 'Skipping package not linked in view: %s' % spec.name) return if spec.package.is_activated(self): spec.package.do_deactivate(self, verbose=self.verbose, remove_dependents=with_dependents) self.unlink_meta_folder(spec) def remove_standalone(self, spec): """ Remove (unlink) a standalone package from this view. """ if not self.check_added(spec): tty.warn(self._croot + 'Skipping package not linked in view: %s' % spec.name) return self.unmerge(spec) self.unlink_meta_folder(spec) if self.verbose: tty.info(self._croot + 'Removed package: %s' % colorize_spec(spec)) def get_projection_for_spec(self, spec): """ Return the projection for a spec in this view. Relies on the ordering of projections to avoid ambiguity. """ spec = spack.spec.Spec(spec) # Extensions are placed by their extendee, not by their own spec locator_spec = spec if spec.package.extendee_spec: locator_spec = spec.package.extendee_spec all_fmt_str = None for spec_like, fmt_str in self.projections.items(): if locator_spec.satisfies(spec_like, strict=True): return os.path.join(self._root, locator_spec.format(fmt_str)) elif spec_like == 'all': all_fmt_str = fmt_str if all_fmt_str: return os.path.join(self._root, locator_spec.format(all_fmt_str)) return self._root def get_all_specs(self): md_dirs = [] for root, dirs, files in os.walk(self._root): if spack.store.layout.metadata_dir in dirs: md_dirs.append( os.path.join(root, spack.store.layout.metadata_dir)) specs = [] for md_dir in md_dirs: if os.path.exists(md_dir): for name_dir in os.listdir(md_dir): filename = os.path.join(md_dir, name_dir, spack.store.layout.spec_file_name) spec = get_spec_from_file(filename) if spec: specs.append(spec) return specs def get_conflicts(self, *specs): """ Return list of tuples (<spec>, <spec in view>) where the spec active in the view differs from the one to be activated. """ in_view = map(self.get_spec, specs) return [(s, v) for s, v in zip(specs, in_view) if v is not None and s != v] def get_path_meta_folder(self, spec): "Get path to meta folder for either spec or spec name." return os.path.join(self.get_projection_for_spec(spec), spack.store.layout.metadata_dir, getattr(spec, "name", spec)) def get_spec(self, spec): dotspack = self.get_path_meta_folder(spec) filename = os.path.join(dotspack, spack.store.layout.spec_file_name) return get_spec_from_file(filename) def link_meta_folder(self, spec): src = spack.store.layout.metadata_path(spec) tgt = self.get_path_meta_folder(spec) tree = LinkTree(src) # there should be no conflicts when linking the meta folder tree.merge(tgt, link=self.link) def print_conflict(self, spec_active, spec_specified, level="error"): "Singular print function for spec conflicts." cprint = getattr(tty, level) color = sys.stdout.isatty() linked = tty.color.colorize(" (@gLinked@.)", color=color) specified = tty.color.colorize("(@rSpecified@.)", color=color) cprint(self._croot + "Package conflict detected:\n" "%s %s\n" % (linked, colorize_spec(spec_active)) + "%s %s" % (specified, colorize_spec(spec_specified))) def print_status(self, *specs, **kwargs): if kwargs.get("with_dependencies", False): specs = set(get_dependencies(specs)) specs = sorted(specs, key=lambda s: s.name) in_view = list(map(self.get_spec, specs)) for s, v in zip(specs, in_view): if not v: tty.error(self._croot + 'Package not linked: %s' % s.name) elif s != v: self.print_conflict(v, s, level="warn") in_view = list(filter(None, in_view)) if len(specs) > 0: tty.msg("Packages linked in %s:" % self._croot[:-1]) # Make a dict with specs keyed by architecture and compiler. index = index_by(specs, ('architecture', 'compiler')) # Traverse the index and print out each package for i, (architecture, compiler) in enumerate(sorted(index)): if i > 0: print() header = "%s{%s} / %s{%s}" % ( spack.spec.architecture_color, architecture, spack.spec.compiler_color, compiler) tty.hline(colorize(header), char='-') specs = index[(architecture, compiler)] specs.sort() format_string = '$_$@$%@+$+' abbreviated = [s.cformat(format_string) for s in specs] # Print one spec per line along with prefix path width = max(len(s) for s in abbreviated) width += 2 format = " %%-%ds%%s" % width for abbrv, s in zip(abbreviated, specs): prefix = '' if self.verbose: prefix = colorize('@K{%s}' % s.dag_hash(7)) print(prefix + (format % (abbrv, self.get_projection_for_spec(s)))) else: tty.warn(self._croot + "No packages found.") def purge_empty_directories(self): """ Ascend up from the leaves accessible from `path` and remove empty directories. """ for dirpath, subdirs, files in os.walk(self._root, topdown=False): for sd in subdirs: sdp = os.path.join(dirpath, sd) try: os.rmdir(sdp) except OSError: pass def unlink_meta_folder(self, spec): path = self.get_path_meta_folder(spec) assert os.path.exists(path) shutil.rmtree(path) def _check_no_ext_conflicts(self, spec): """ Check that there is no extension conflict for specs. """ extendee = spec.package.extendee_spec try: self.extensions_layout.check_extension_conflict(extendee, spec) except ExtensionAlreadyInstalledError: # we print the warning here because later on the order in which # packages get activated is not clear (set-sorting) tty.warn(self._croot + 'Skipping already activated package: %s' % spec.name)