Esempio n. 1
0
 def add_to_loaded_collection(self, name):
     """Add a module `name` to the currently loaded collection"""
     collection_name = pymod.environ.get(pymod.names.loaded_collection)
     if collection_name is None:  # pragma: no cover
         tty.die("There is no collection currently loaded")
     data = OrderedDict(self.data.pop(collection_name))
     module = pymod.modulepath.get(name)
     if module is None:
         raise pymod.error.ModuleNotFoundError(name)
     if not module.is_loaded:
         pymod.mc.load_impl(module)
     for (mp, modules) in data.items():
         if mp != module.modulepath:
             continue
         for other in modules:
             if other["fullname"] == module.fullname:  # pragma: no cover
                 tty.warn("{0} is already in collection {1}".format(
                     name, collection_name))
                 return
     ar = pymod.mc.archive_module(module)
     ar["refcount"] = 0
     data.setdefault(module.modulepath, []).append(ar)
     data = list(data.items())
     self.data.update({collection_name: data})
     self.write(self.data, self.filename)
     return None
Esempio n. 2
0
 def save(self, name, modules):
     collection = OrderedDict()
     for module in modules:
         ar = pymod.mc.archive_module(module)
         ar["refcount"] = 0
         collection.setdefault(module.modulepath, []).append(ar)
     collection = list(collection.items())
     self.data.update({name: collection})
     self.write(self.data, self.filename)
     return None
Esempio n. 3
0
 def pop_from_loaded_collection(self, name):
     """Remove a module `name` to the currently loaded collection"""
     collection_name = pymod.environ.get(pymod.names.loaded_collection)
     if collection_name is None:
         tty.die(
             "There is no collection currently loaded")  # pragma: no cover
     data = OrderedDict(self.data.pop(collection_name))
     module = pymod.modulepath.get(name)
     if module is None:  # pragma: no cover
         raise pymod.error.ModuleNotFoundError(name)
     if module.is_loaded:
         pymod.mc.unload_impl(module)
     for (mp, modules) in data.items():
         if mp != module.modulepath:  # pragma: no cover
             continue
         data[mp] = [
             other for other in modules
             if other["fullname"] == module.fullname
         ]
         break
     data = list(data.items())
     self.data.update({collection_name: data})
     self.write(self.data, self.filename)
     return None
Esempio n. 4
0
class MirrorCollection(Mapping):
    """A mapping of mirror names to mirrors."""
    def __init__(self, mirrors=None, scope=None):
        self._mirrors = OrderedDict(
            (name, Mirror.from_dict(mirror, name)) for name, mirror in (
                mirrors.items() if mirrors is not None else spack.config.
                get('mirrors', scope=scope).items()))

    def to_json(self, stream=None):
        return sjson.dump(self.to_dict(True), stream)

    def to_yaml(self, stream=None):
        return syaml.dump(self.to_dict(True), stream)

    # TODO: this isn't called anywhere
    @staticmethod
    def from_yaml(stream, name=None):
        try:
            data = syaml.load(stream)
            return MirrorCollection(data)
        except yaml_error.MarkedYAMLError as e:
            raise syaml.SpackYAMLError("error parsing YAML spec:", str(e))

    @staticmethod
    def from_json(stream, name=None):
        d = sjson.load(stream)
        return MirrorCollection(d)

    def to_dict(self, recursive=False):
        return syaml_dict(
            sorted(((k, (v.to_dict() if recursive else v))
                    for (k, v) in self._mirrors.items()),
                   key=operator.itemgetter(0)))

    @staticmethod
    def from_dict(d):
        return MirrorCollection(d)

    def __getitem__(self, item):
        return self._mirrors[item]

    def display(self):
        max_len = max(len(mirror.name) for mirror in self._mirrors.values())
        for mirror in self._mirrors.values():
            mirror.display(max_len)

    def lookup(self, name_or_url):
        """Looks up and returns a Mirror.

        If this MirrorCollection contains a named Mirror under the name
        [name_or_url], then that mirror is returned.  Otherwise, [name_or_url]
        is assumed to be a mirror URL, and an anonymous mirror with the given
        URL is returned.
        """
        result = self.get(name_or_url)

        if result is None:
            result = Mirror(fetch_url=name_or_url)

        return result

    def __iter__(self):
        return iter(self._mirrors)

    def __len__(self):
        return len(self._mirrors)
Esempio n. 5
0
def upgrade_None_to_1_0(new, old, depth=[0]):

    depth[0] += 1
    if depth[0] > 1:
        raise ValueError("Recursion!")

    version_string = ".".join(str(_) for _ in new.version)
    tty.info(
        "Converting Modulecmd.py collections version 0.0 to "
        "version {0}".format(version_string)
    )
    new_collections = {}
    for (name, old_collection) in old.items():
        new_collection = OrderedDict()
        mp = pymod.modulepath.Modulepath([])
        for (path, m_descs) in old_collection:
            if new_collection is None:
                break
            if not os.path.isdir(path):
                tty.warn(
                    "Collection {0} contains directory {1} which "
                    "does not exist!  This collection will be skipped".format(
                        name, path
                    )
                )
                new_collection = None
                break
            avail = mp.append_path(path)
            if avail is None:
                tty.warn(
                    "Collection {0} contains directory {1} which "
                    "does not have any available modules!  "
                    "This collection will be skipped".format(name, path)
                )
                new_collection = None
                break
            for (fullname, filename, opts) in m_descs:
                m = mp.get(filename)
                if m is None:
                    tty.warn(
                        "Collection {0} requests module {1} which "
                        "can not be found! This collection will be skipped".format(
                            name, fullname
                        )
                    )
                    new_collection = None
                    break
                m.opts = opts
                m.acquired_as = m.fullname
                ar = pymod.mc.archive_module(m)
                new_collection.setdefault(m.modulepath, []).append(ar)

        if new_collection is None:
            tty.warn(
                "Skipping collection {0} because of previous " "errors".format(name)
            )
            continue

        new_collections[name] = list(new_collection.items())

    bak = new.filename + ".bak"
    with open(bak, "w") as fh:
        json.dump(old, fh, indent=2)

    new.write(list(new_collections.items()), new.filename)
    return new_collections
Esempio n. 6
0
class Environment(object):
    def __init__(self, path, init_file=None, with_view=None):
        """Create a new environment.

        The environment can be optionally initialized with either a
        spack.yaml or spack.lock file.

        Arguments:
            path (str): path to the root directory of this environment
            init_file (str or file object): filename or file object to
                initialize the environment
            with_view (str or bool): whether a view should be maintained for
                the environment. If the value is a string, it specifies the
                path to the view.
        """
        self.path = os.path.abspath(path)
        self.clear()

        if init_file:
            with fs.open_if_filename(init_file) as f:
                if hasattr(f, 'name') and f.name.endswith('.lock'):
                    self._read_manifest(default_manifest_yaml)
                    self._read_lockfile(f)
                    self._set_user_specs_from_lockfile()
                else:
                    self._read_manifest(f)
        else:
            default_manifest = not os.path.exists(self.manifest_path)
            if default_manifest:
                # No manifest, use default yaml
                self._read_manifest(default_manifest_yaml)
            else:
                with open(self.manifest_path) as f:
                    self._read_manifest(f)

            if os.path.exists(self.lock_path):
                with open(self.lock_path) as f:
                    read_lock_version = self._read_lockfile(f)
                if default_manifest:
                    # No manifest, set user specs from lockfile
                    self._set_user_specs_from_lockfile()

                if read_lock_version == 1:
                    tty.debug(
                        "Storing backup of old lockfile {0} at {1}".format(
                            self.lock_path, self._lock_backup_v1_path))
                    shutil.copy(self.lock_path, self._lock_backup_v1_path)

        if with_view is False:
            self.views = {}
        elif with_view is True:
            self.views = {
                default_view_name: ViewDescriptor(self.view_path_default)
            }
        elif isinstance(with_view, six.string_types):
            self.views = {default_view_name: ViewDescriptor(with_view)}
        # If with_view is None, then defer to the view settings determined by
        # the manifest file

    def _read_manifest(self, f):
        """Read manifest file and set up user specs."""
        self.yaml = _read_yaml(f)

        self.spec_lists = OrderedDict()

        for item in config_dict(self.yaml).get('definitions', []):
            entry = copy.deepcopy(item)
            when = _eval_conditional(entry.pop('when', 'True'))
            assert len(entry) == 1
            if when:
                name, spec_list = next(iter(entry.items()))
                user_specs = SpecList(name, spec_list, self.spec_lists.copy())
                if name in self.spec_lists:
                    self.spec_lists[name].extend(user_specs)
                else:
                    self.spec_lists[name] = user_specs

        spec_list = config_dict(self.yaml).get(user_speclist_name)
        user_specs = SpecList(user_speclist_name, [s for s in spec_list if s],
                              self.spec_lists.copy())
        self.spec_lists[user_speclist_name] = user_specs

        enable_view = config_dict(self.yaml).get('view')
        # enable_view can be boolean, string, or None
        if enable_view is True or enable_view is None:
            self.views = {
                default_view_name: ViewDescriptor(self.view_path_default)
            }
        elif isinstance(enable_view, six.string_types):
            self.views = {default_view_name: ViewDescriptor(enable_view)}
        elif enable_view:
            self.views = dict((name, ViewDescriptor.from_dict(values))
                              for name, values in enable_view.items())
        else:
            self.views = {}

    @property
    def user_specs(self):
        return self.spec_lists[user_speclist_name]

    def _set_user_specs_from_lockfile(self):
        """Copy user_specs from a read-in lockfile."""
        self.spec_lists = {
            user_speclist_name:
            SpecList(user_speclist_name,
                     [str(s) for s in self.concretized_user_specs])
        }

    def clear(self):
        self.spec_lists = {user_speclist_name: SpecList()}  # specs from yaml
        self.concretized_user_specs = []  # user specs from last concretize
        self.concretized_order = []  # roots of last concretize, in order
        self.specs_by_hash = {}  # concretized specs by hash
        self.new_specs = []  # write packages for these on write()
        self._repo = None  # RepoPath for this env (memoized)
        self._previous_active = None  # previously active environment

    @property
    def internal(self):
        """Whether this environment is managed by Spack."""
        return self.path.startswith(env_path)

    @property
    def name(self):
        """Human-readable representation of the environment.

        This is the path for directory environments, and just the name
        for named environments.
        """
        if self.internal:
            return os.path.basename(self.path)
        else:
            return self.path

    @property
    def active(self):
        """True if this environment is currently active."""
        return _active_environment and self.path == _active_environment.path

    @property
    def manifest_path(self):
        """Path to spack.yaml file in this environment."""
        return os.path.join(self.path, manifest_name)

    @property
    def lock_path(self):
        """Path to spack.lock file in this environment."""
        return os.path.join(self.path, lockfile_name)

    @property
    def _lock_backup_v1_path(self):
        """Path to backup of v1 lockfile before conversion to v2"""
        return self.lock_path + '.backup.v1'

    @property
    def env_subdir_path(self):
        """Path to directory where the env stores repos, logs, views."""
        return os.path.join(self.path, env_subdir_name)

    @property
    def repos_path(self):
        return os.path.join(self.path, env_subdir_name, 'repos')

    @property
    def log_path(self):
        return os.path.join(self.path, env_subdir_name, 'logs')

    @property
    def view_path_default(self):
        # default path for environment views
        return os.path.join(self.env_subdir_path, 'view')

    @property
    def repo(self):
        if self._repo is None:
            self._repo = make_repo_path(self.repos_path)
        return self._repo

    def included_config_scopes(self):
        """List of included configuration scopes from the environment.

        Scopes are listed in the YAML file in order from highest to
        lowest precedence, so configuration from earlier scope will take
        precedence over later ones.

        This routine returns them in the order they should be pushed onto
        the internal scope stack (so, in reverse, from lowest to highest).
        """
        scopes = []

        # load config scopes added via 'include:', in reverse so that
        # highest-precedence scopes are last.
        includes = config_dict(self.yaml).get('include', [])
        for i, config_path in enumerate(reversed(includes)):
            # allow paths to contain environment variables
            config_path = config_path.format(**os.environ)

            # treat relative paths as relative to the environment
            if not os.path.isabs(config_path):
                config_path = os.path.join(self.path, config_path)
                config_path = os.path.normpath(os.path.realpath(config_path))

            if os.path.isdir(config_path):
                # directories are treated as regular ConfigScopes
                config_name = 'env:%s:%s' % (self.name,
                                             os.path.basename(config_path))
                scope = spack.config.ConfigScope(config_name, config_path)
            else:
                # files are assumed to be SingleFileScopes
                base, ext = os.path.splitext(os.path.basename(config_path))
                config_name = 'env:%s:%s' % (self.name, base)
                scope = spack.config.SingleFileScope(
                    config_name, config_path, spack.schema.merged.schema)

            scopes.append(scope)

        return scopes

    def env_file_config_scope_name(self):
        """Name of the config scope of this environment's manifest file."""
        return 'env:%s' % self.name

    def env_file_config_scope(self):
        """Get the configuration scope for the environment's manifest file."""
        config_name = self.env_file_config_scope_name()
        return spack.config.SingleFileScope(config_name, self.manifest_path,
                                            spack.schema.env.schema,
                                            [env_schema_keys])

    def config_scopes(self):
        """A list of all configuration scopes for this environment."""
        return self.included_config_scopes() + [self.env_file_config_scope()]

    def destroy(self):
        """Remove this environment from Spack entirely."""
        shutil.rmtree(self.path)

    def update_stale_references(self, from_list=None):
        """Iterate over spec lists updating references."""
        if not from_list:
            from_list = next(iter(self.spec_lists.keys()))
        index = list(self.spec_lists.keys()).index(from_list)

        # spec_lists is an OrderedDict, all list entries after the modified
        # list may refer to the modified list. Update stale references
        for i, (name, speclist) in enumerate(
                list(self.spec_lists.items())[index + 1:], index + 1):
            new_reference = dict((n, self.spec_lists[n])
                                 for n in list(self.spec_lists.keys())[:i])
            speclist.update_reference(new_reference)

    def add(self, user_spec, list_name=user_speclist_name):
        """Add a single user_spec (non-concretized) to the Environment

        Returns:
            (bool): True if the spec was added, False if it was already
                present and did not need to be added

        """
        spec = Spec(user_spec)

        if list_name not in self.spec_lists:
            raise SpackEnvironmentError('No list %s exists in environment %s' %
                                        (list_name, self.name))

        if list_name == user_speclist_name:
            if not spec.name:
                raise SpackEnvironmentError(
                    'cannot add anonymous specs to an environment!')
            elif not spack.repo.path.exists(spec.name):
                raise SpackEnvironmentError('no such package: %s' % spec.name)

        list_to_change = self.spec_lists[list_name]
        existing = str(spec) in list_to_change.yaml_list
        if not existing:
            list_to_change.add(str(spec))
            self.update_stale_references(list_name)

        return bool(not existing)

    def remove(self, query_spec, list_name=user_speclist_name, force=False):
        """Remove specs from an environment that match a query_spec"""
        query_spec = Spec(query_spec)

        list_to_change = self.spec_lists[list_name]
        matches = []

        if not query_spec.concrete:
            matches = [s for s in list_to_change if s.satisfies(query_spec)]

        if not matches:
            # concrete specs match against concrete specs in the env
            specs_hashes = zip(self.concretized_user_specs,
                               self.concretized_order)
            matches = [
                s for s, h in specs_hashes if query_spec.dag_hash() == h
            ]

        if not matches:
            raise SpackEnvironmentError("Not found: {0}".format(query_spec))

        old_specs = set(self.user_specs)
        for spec in matches:
            if spec in list_to_change:
                list_to_change.remove(spec)

        self.update_stale_references(list_name)

        # If force, update stale concretized specs
        # Only check specs removed by this operation
        new_specs = set(self.user_specs)
        for spec in old_specs - new_specs:
            if force and spec in self.concretized_user_specs:
                i = self.concretized_user_specs.index(spec)
                del self.concretized_user_specs[i]

                dag_hash = self.concretized_order[i]
                del self.concretized_order[i]
                del self.specs_by_hash[dag_hash]

    def concretize(self, force=False):
        """Concretize user_specs in this environment.

        Only concretizes specs that haven't been concretized yet unless
        force is ``True``.

        This only modifies the environment in memory. ``write()`` will
        write out a lockfile containing concretized specs.

        Arguments:
            force (bool): re-concretize ALL specs, even those that were
               already concretized

        Returns:
            List of specs that have been concretized. Each entry is a tuple of
            the user spec and the corresponding concretized spec.
        """
        if force:
            # Clear previously concretized specs
            self.concretized_user_specs = []
            self.concretized_order = []
            self.specs_by_hash = {}

        # keep any concretized specs whose user specs are still in the manifest
        old_concretized_user_specs = self.concretized_user_specs
        old_concretized_order = self.concretized_order
        old_specs_by_hash = self.specs_by_hash

        self.concretized_user_specs = []
        self.concretized_order = []
        self.specs_by_hash = {}

        for s, h in zip(old_concretized_user_specs, old_concretized_order):
            if s in self.user_specs:
                concrete = old_specs_by_hash[h]
                self._add_concrete_spec(s, concrete, new=False)

        # Concretize any new user specs that we haven't concretized yet
        concretized_specs = []
        for uspec, uspec_constraints in zip(
                self.user_specs, self.user_specs.specs_as_constraints):
            if uspec not in old_concretized_user_specs:
                concrete = _concretize_from_constraints(uspec_constraints)
                self._add_concrete_spec(uspec, concrete)
                concretized_specs.append((uspec, concrete))
        return concretized_specs

    def install(self, user_spec, concrete_spec=None, **install_args):
        """Install a single spec into an environment.

        This will automatically concretize the single spec, but it won't
        affect other as-yet unconcretized specs.
        """
        spec = Spec(user_spec)

        if self.add(spec):
            concrete = concrete_spec if concrete_spec else spec.concretized()
            self._add_concrete_spec(spec, concrete)
        else:
            # spec might be in the user_specs, but not installed.
            # TODO: Redo name-based comparison for old style envs
            spec = next(s for s in self.user_specs if s.satisfies(user_spec))
            concrete = self.specs_by_hash.get(spec.build_hash())
            if not concrete:
                concrete = spec.concretized()
                self._add_concrete_spec(spec, concrete)

        self._install(concrete, **install_args)

    def _install(self, spec, **install_args):
        spec.package.do_install(**install_args)

        # Make sure log directory exists
        log_path = self.log_path
        fs.mkdirp(log_path)

        with fs.working_dir(self.path):
            # Link the resulting log file into logs dir
            build_log_link = os.path.join(
                log_path, '%s-%s.log' % (spec.name, spec.dag_hash(7)))
            if os.path.lexists(build_log_link):
                os.remove(build_log_link)
            os.symlink(spec.package.build_log_path, build_log_link)

    @property
    def default_view(self):
        if not self.views:
            raise SpackEnvironmentError(
                "{0} does not have a view enabled".format(self.name))

        if default_view_name not in self.views:
            raise SpackEnvironmentError(
                "{0} does not have a default view enabled".format(self.name))

        return self.views[default_view_name]

    def update_default_view(self, viewpath):
        name = default_view_name
        if name in self.views and self.default_view.root != viewpath:
            shutil.rmtree(self.default_view.root)

        if viewpath:
            if name in self.views:
                self.default_view.root = viewpath
            else:
                self.views[name] = ViewDescriptor(viewpath)
        else:
            self.views.pop(name, None)

    def regenerate_views(self):
        if not self.views:
            tty.debug("Skip view update, this environment does not"
                      " maintain a view")
            return

        specs = self._get_environment_specs()
        for view in self.views.values():
            view.regenerate(specs, self.roots())

    def _shell_vars(self):
        updates = [
            ('PATH', ['bin']),
            ('MANPATH', ['man', 'share/man']),
            ('ACLOCAL_PATH', ['share/aclocal']),
            ('LD_LIBRARY_PATH', ['lib', 'lib64']),
            ('LIBRARY_PATH', ['lib', 'lib64']),
            ('CPATH', ['include']),
            ('PKG_CONFIG_PATH', ['lib/pkgconfig', 'lib64/pkgconfig']),
            ('CMAKE_PREFIX_PATH', ['']),
        ]
        path_updates = list()
        if default_view_name in self.views:
            for var, subdirs in updates:
                paths = filter(
                    lambda x: os.path.exists(x),
                    list(
                        os.path.join(self.default_view.root, x)
                        for x in subdirs))
                path_updates.append((var, paths))
        return path_updates

    def add_default_view_to_shell(self, shell):
        env_mod = EnvironmentModifications()
        for var, paths in self._shell_vars():
            for path in paths:
                env_mod.prepend_path(var, path)
        return env_mod.shell_modifications(shell)

    def rm_default_view_from_shell(self, shell):
        env_mod = EnvironmentModifications()
        for var, paths in self._shell_vars():
            for path in paths:
                env_mod.remove_path(var, path)
        return env_mod.shell_modifications(shell)

    def _add_concrete_spec(self, spec, concrete, new=True):
        """Called when a new concretized spec is added to the environment.

        This ensures that all internal data structures are kept in sync.

        Arguments:
            spec (Spec): user spec that resulted in the concrete spec
            concrete (Spec): spec concretized within this environment
            new (bool): whether to write this spec's package to the env
                repo on write()
        """
        assert concrete.concrete

        # when a spec is newly concretized, we need to make a note so
        # that we can write its package to the env repo on write()
        if new:
            self.new_specs.append(concrete)

        # update internal lists of specs
        self.concretized_user_specs.append(spec)

        h = concrete.build_hash()
        self.concretized_order.append(h)
        self.specs_by_hash[h] = concrete

    def install_all(self, args=None):
        """Install all concretized specs in an environment."""
        for concretized_hash in self.concretized_order:
            spec = self.specs_by_hash[concretized_hash]

            # Parse cli arguments and construct a dictionary
            # that will be passed to Package.do_install API
            kwargs = dict()
            if args:
                spack.cmd.install.update_kwargs_from_args(args, kwargs)

            self._install(spec, **kwargs)

            if not spec.external:
                # Link the resulting log file into logs dir
                build_log_link = os.path.join(
                    self.log_path, '%s-%s.log' % (spec.name, spec.dag_hash(7)))
                if os.path.lexists(build_log_link):
                    os.remove(build_log_link)
                os.symlink(spec.package.build_log_path, build_log_link)

        self.regenerate_views()

    def all_specs_by_hash(self):
        """Map of hashes to spec for all specs in this environment."""
        # Note this uses dag-hashes calculated without build deps as keys,
        # whereas the environment tracks specs based on dag-hashes calculated
        # with all dependencies. This function should not be used by an
        # Environment object for management of its own data structures
        hashes = {}
        for h in self.concretized_order:
            specs = self.specs_by_hash[h].traverse(deptype=('link', 'run'))
            for spec in specs:
                hashes[spec.dag_hash()] = spec
        return hashes

    def all_specs(self):
        """Return all specs, even those a user spec would shadow."""
        return sorted(self.all_specs_by_hash().values())

    def all_hashes(self):
        """Return all specs, even those a user spec would shadow."""
        return list(self.all_specs_by_hash().keys())

    def roots(self):
        """Specs explicitly requested by the user *in this environment*.

        Yields both added and installed specs that have user specs in
        `spack.yaml`.
        """
        concretized = dict(self.concretized_specs())
        for spec in self.user_specs:
            concrete = concretized.get(spec)
            yield concrete if concrete else spec

    def added_specs(self):
        """Specs that are not yet installed.

        Yields the user spec for non-concretized specs, and the concrete
        spec for already concretized but not yet installed specs.
        """
        concretized = dict(self.concretized_specs())
        for spec in self.user_specs:
            concrete = concretized.get(spec)
            if not concrete:
                yield spec
            elif not concrete.package.installed:
                yield concrete

    def concretized_specs(self):
        """Tuples of (user spec, concrete spec) for all concrete specs."""
        for s, h in zip(self.concretized_user_specs, self.concretized_order):
            yield (s, self.specs_by_hash[h])

    def removed_specs(self):
        """Tuples of (user spec, concrete spec) for all specs that will be
           removed on nexg concretize."""
        needed = set()
        for s, c in self.concretized_specs():
            if s in self.user_specs:
                for d in c.traverse():
                    needed.add(d)

        for s, c in self.concretized_specs():
            for d in c.traverse():
                if d not in needed:
                    yield d

    def _get_environment_specs(self, recurse_dependencies=True):
        """Returns the specs of all the packages in an environment.

        If these specs appear under different user_specs, only one copy
        is added to the list returned.
        """
        spec_list = list()

        for spec_hash in self.concretized_order:
            spec = self.specs_by_hash[spec_hash]

            specs = (spec.traverse(
                deptype=('link', 'run')) if recurse_dependencies else (spec, ))

            spec_list.extend(specs)

        return spec_list

    def _to_lockfile_dict(self):
        """Create a dictionary to store a lockfile for this environment."""
        concrete_specs = {}
        for spec in self.specs_by_hash.values():
            for s in spec.traverse():
                dag_hash_all = s.build_hash()
                if dag_hash_all not in concrete_specs:
                    spec_dict = s.to_node_dict(hash=ht.build_hash)
                    spec_dict[s.name]['hash'] = s.dag_hash()
                    concrete_specs[dag_hash_all] = spec_dict

        hash_spec_list = zip(self.concretized_order,
                             self.concretized_user_specs)

        # this is the lockfile we'll write out
        data = {
            # metadata about the format
            '_meta': {
                'file-type': 'spack-lockfile',
                'lockfile-version': lockfile_format_version,
            },

            # users specs + hashes are the 'roots' of the environment
            'roots': [{
                'hash': h,
                'spec': str(s)
            } for h, s in hash_spec_list],

            # Concrete specs by hash, including dependencies
            'concrete_specs': concrete_specs,
        }

        return data

    def _read_lockfile(self, file_or_json):
        """Read a lockfile from a file or from a raw string."""
        lockfile_dict = sjson.load(file_or_json)
        self._read_lockfile_dict(lockfile_dict)
        return lockfile_dict['_meta']['lockfile-version']

    def _read_lockfile_dict(self, d):
        """Read a lockfile dictionary into this environment."""
        roots = d['roots']
        self.concretized_user_specs = [Spec(r['spec']) for r in roots]
        self.concretized_order = [r['hash'] for r in roots]

        json_specs_by_hash = d['concrete_specs']
        root_hashes = set(self.concretized_order)

        specs_by_hash = {}
        for dag_hash, node_dict in json_specs_by_hash.items():
            specs_by_hash[dag_hash] = Spec.from_node_dict(node_dict)

        for dag_hash, node_dict in json_specs_by_hash.items():
            for dep_name, dep_hash, deptypes in (
                    Spec.dependencies_from_node_dict(node_dict)):
                specs_by_hash[dag_hash]._add_dependency(
                    specs_by_hash[dep_hash], deptypes)

        # If we are reading an older lockfile format (which uses dag hashes
        # that exclude build deps), we use this to convert the old
        # concretized_order to the full hashes (preserving the order)
        old_hash_to_new = {}
        self.specs_by_hash = {}
        for _, spec in specs_by_hash.items():
            dag_hash = spec.dag_hash()
            build_hash = spec.build_hash()
            if dag_hash in root_hashes:
                old_hash_to_new[dag_hash] = build_hash

            if (dag_hash in root_hashes or build_hash in root_hashes):
                self.specs_by_hash[build_hash] = spec

        if old_hash_to_new:
            # Replace any older hashes in concretized_order with hashes
            # that include build deps
            self.concretized_order = [
                old_hash_to_new.get(h, h) for h in self.concretized_order
            ]

    def write(self):
        """Writes an in-memory environment to its location on disk.

        This will also write out package files for each newly concretized spec.
        """
        # ensure path in var/spack/environments
        fs.mkdirp(self.path)

        if self.specs_by_hash:
            # ensure the prefix/.env directory exists
            fs.mkdirp(self.env_subdir_path)

            for spec in self.new_specs:
                for dep in spec.traverse():
                    if not dep.concrete:
                        raise ValueError('specs passed to environment.write() '
                                         'must be concrete!')

                    root = os.path.join(self.repos_path, dep.namespace)
                    repo = spack.repo.create_or_construct(root, dep.namespace)
                    pkg_dir = repo.dirname_for_package_name(dep.name)

                    fs.mkdirp(pkg_dir)
                    spack.repo.path.dump_provenance(dep, pkg_dir)
            self.new_specs = []

            # write the lock file last
            with fs.write_tmp_and_move(self.lock_path) as f:
                sjson.dump(self._to_lockfile_dict(), stream=f)
        else:
            if os.path.exists(self.lock_path):
                os.unlink(self.lock_path)

        # invalidate _repo cache
        self._repo = None

        # put any changes in the definitions in the YAML
        for name, speclist in self.spec_lists.items():
            if name == user_speclist_name:
                # The primary list is handled differently
                continue

            conf = config_dict(self.yaml)
            active_yaml_lists = [
                l for l in conf.get('definitions', [])
                if name in l and _eval_conditional(l.get('when', 'True'))
            ]

            # Remove any specs in yaml that are not in internal representation
            for ayl in active_yaml_lists:
                # If it's not a string, it's a matrix. Those can't have changed
                # If it is a string that starts with '$', it's a reference.
                # Those also can't have changed.
                ayl[name][:] = [
                    s for s in ayl.setdefault(name, [])
                    if (not isinstance(s, six.string_types))
                    or s.startswith('$') or Spec(s) in speclist.specs
                ]

            # Put the new specs into the first active list from the yaml
            new_specs = [
                entry for entry in speclist.yaml_list
                if isinstance(entry, six.string_types) and not any(
                    entry in ayl[name] for ayl in active_yaml_lists)
            ]
            list_for_new_specs = active_yaml_lists[0].setdefault(name, [])
            list_for_new_specs[:] = list_for_new_specs + new_specs

        # put the new user specs in the YAML.
        # This can be done directly because there can't be multiple definitions
        # nor when clauses for `specs` list.
        yaml_spec_list = config_dict(self.yaml).setdefault(
            user_speclist_name, [])
        yaml_spec_list[:] = self.user_specs.yaml_list

        default_name = default_view_name
        if self.views and len(self.views) == 1 and default_name in self.views:
            path = self.default_view.root
            if self.default_view == ViewDescriptor(self.view_path_default):
                view = True
            elif self.default_view == ViewDescriptor(path):
                view = path
            else:
                view = dict((name, view.to_dict())
                            for name, view in self.views.items())
        elif self.views:
            view = dict(
                (name, view.to_dict()) for name, view in self.views.items())
        else:
            view = False

        yaml_dict = config_dict(self.yaml)
        if view is not True:
            # The default case is to keep an active view inside of the
            # Spack environment directory. To avoid cluttering the config,
            # we omit the setting in this case.
            yaml_dict['view'] = view
        elif 'view' in yaml_dict:
            del yaml_dict['view']

        # if all that worked, write out the manifest file at the top level
        with fs.write_tmp_and_move(self.manifest_path) as f:
            _write_yaml(self.yaml, f)

        # TODO: for operations that just add to the env (install etc.) this
        # could just call update_view
        self.regenerate_views()

    def __enter__(self):
        self._previous_active = _active_environment
        activate(self)
        return

    def __exit__(self, exc_type, exc_val, exc_tb):
        deactivate()
        if self._previous_active:
            activate(self._previous_active)