def add_to_loaded_collection(self, name):
"""Add a module `name` to the currently loaded collection"""
collection_name = pymod.environ.get(pymod.names.loaded_collection)
if collection_name is None: # pragma: no cover
tty.die("There is no collection currently loaded")
data = OrderedDict(self.data.pop(collection_name))
module = pymod.modulepath.get(name)
if module is None:
raise pymod.error.ModuleNotFoundError(name)
if not module.is_loaded:
pymod.mc.load_impl(module)
for (mp, modules) in data.items():
if mp != module.modulepath:
continue
for other in modules:
if other["fullname"] == module.fullname: # pragma: no cover
tty.warn("{0} is already in collection {1}".format(
name, collection_name))
return
ar = pymod.mc.archive_module(module)
ar["refcount"] = 0
data.setdefault(module.modulepath, []).append(ar)
data = list(data.items())
self.data.update({collection_name: data})
self.write(self.data, self.filename)
return None
def save(self, name, modules):
collection = OrderedDict()
for module in modules:
ar = pymod.mc.archive_module(module)
ar["refcount"] = 0
collection.setdefault(module.modulepath, []).append(ar)
collection = list(collection.items())
self.data.update({name: collection})
self.write(self.data, self.filename)
return None
def pop_from_loaded_collection(self, name):
"""Remove a module `name` to the currently loaded collection"""
collection_name = pymod.environ.get(pymod.names.loaded_collection)
if collection_name is None:
tty.die(
"There is no collection currently loaded") # pragma: no cover
data = OrderedDict(self.data.pop(collection_name))
module = pymod.modulepath.get(name)
if module is None: # pragma: no cover
raise pymod.error.ModuleNotFoundError(name)
if module.is_loaded:
pymod.mc.unload_impl(module)
for (mp, modules) in data.items():
if mp != module.modulepath: # pragma: no cover
continue
data[mp] = [
other for other in modules
if other["fullname"] == module.fullname
]
break
data = list(data.items())
self.data.update({collection_name: data})
self.write(self.data, self.filename)
return None
class MirrorCollection(Mapping):
"""A mapping of mirror names to mirrors."""
def __init__(self, mirrors=None, scope=None):
self._mirrors = OrderedDict(
(name, Mirror.from_dict(mirror, name)) for name, mirror in (
mirrors.items() if mirrors is not None else spack.config.
get('mirrors', scope=scope).items()))
def to_json(self, stream=None):
return sjson.dump(self.to_dict(True), stream)
def to_yaml(self, stream=None):
return syaml.dump(self.to_dict(True), stream)
# TODO: this isn't called anywhere
@staticmethod
def from_yaml(stream, name=None):
try:
data = syaml.load(stream)
return MirrorCollection(data)
except yaml_error.MarkedYAMLError as e:
raise syaml.SpackYAMLError("error parsing YAML spec:", str(e))
@staticmethod
def from_json(stream, name=None):
d = sjson.load(stream)
return MirrorCollection(d)
def to_dict(self, recursive=False):
return syaml_dict(
sorted(((k, (v.to_dict() if recursive else v))
for (k, v) in self._mirrors.items()),
key=operator.itemgetter(0)))
@staticmethod
def from_dict(d):
return MirrorCollection(d)
def __getitem__(self, item):
return self._mirrors[item]
def display(self):
max_len = max(len(mirror.name) for mirror in self._mirrors.values())
for mirror in self._mirrors.values():
mirror.display(max_len)
def lookup(self, name_or_url):
"""Looks up and returns a Mirror.
If this MirrorCollection contains a named Mirror under the name
[name_or_url], then that mirror is returned. Otherwise, [name_or_url]
is assumed to be a mirror URL, and an anonymous mirror with the given
URL is returned.
"""
result = self.get(name_or_url)
if result is None:
result = Mirror(fetch_url=name_or_url)
return result
def __iter__(self):
return iter(self._mirrors)
def __len__(self):
return len(self._mirrors)
def upgrade_None_to_1_0(new, old, depth=[0]):
depth[0] += 1
if depth[0] > 1:
raise ValueError("Recursion!")
version_string = ".".join(str(_) for _ in new.version)
tty.info(
"Converting Modulecmd.py collections version 0.0 to "
"version {0}".format(version_string)
)
new_collections = {}
for (name, old_collection) in old.items():
new_collection = OrderedDict()
mp = pymod.modulepath.Modulepath([])
for (path, m_descs) in old_collection:
if new_collection is None:
break
if not os.path.isdir(path):
tty.warn(
"Collection {0} contains directory {1} which "
"does not exist! This collection will be skipped".format(
name, path
)
)
new_collection = None
break
avail = mp.append_path(path)
if avail is None:
tty.warn(
"Collection {0} contains directory {1} which "
"does not have any available modules! "
"This collection will be skipped".format(name, path)
)
new_collection = None
break
for (fullname, filename, opts) in m_descs:
m = mp.get(filename)
if m is None:
tty.warn(
"Collection {0} requests module {1} which "
"can not be found! This collection will be skipped".format(
name, fullname
)
)
new_collection = None
break
m.opts = opts
m.acquired_as = m.fullname
ar = pymod.mc.archive_module(m)
new_collection.setdefault(m.modulepath, []).append(ar)
if new_collection is None:
tty.warn(
"Skipping collection {0} because of previous " "errors".format(name)
)
continue
new_collections[name] = list(new_collection.items())
bak = new.filename + ".bak"
with open(bak, "w") as fh:
json.dump(old, fh, indent=2)
new.write(list(new_collections.items()), new.filename)
return new_collections
class Environment(object):
def __init__(self, path, init_file=None, with_view=None):
"""Create a new environment.
The environment can be optionally initialized with either a
spack.yaml or spack.lock file.
Arguments:
path (str): path to the root directory of this environment
init_file (str or file object): filename or file object to
initialize the environment
with_view (str or bool): whether a view should be maintained for
the environment. If the value is a string, it specifies the
path to the view.
"""
self.path = os.path.abspath(path)
self.clear()
if init_file:
with fs.open_if_filename(init_file) as f:
if hasattr(f, 'name') and f.name.endswith('.lock'):
self._read_manifest(default_manifest_yaml)
self._read_lockfile(f)
self._set_user_specs_from_lockfile()
else:
self._read_manifest(f)
else:
default_manifest = not os.path.exists(self.manifest_path)
if default_manifest:
# No manifest, use default yaml
self._read_manifest(default_manifest_yaml)
else:
with open(self.manifest_path) as f:
self._read_manifest(f)
if os.path.exists(self.lock_path):
with open(self.lock_path) as f:
read_lock_version = self._read_lockfile(f)
if default_manifest:
# No manifest, set user specs from lockfile
self._set_user_specs_from_lockfile()
if read_lock_version == 1:
tty.debug(
"Storing backup of old lockfile {0} at {1}".format(
self.lock_path, self._lock_backup_v1_path))
shutil.copy(self.lock_path, self._lock_backup_v1_path)
if with_view is False:
self.views = {}
elif with_view is True:
self.views = {
default_view_name: ViewDescriptor(self.view_path_default)
}
elif isinstance(with_view, six.string_types):
self.views = {default_view_name: ViewDescriptor(with_view)}
# If with_view is None, then defer to the view settings determined by
# the manifest file
def _read_manifest(self, f):
"""Read manifest file and set up user specs."""
self.yaml = _read_yaml(f)
self.spec_lists = OrderedDict()
for item in config_dict(self.yaml).get('definitions', []):
entry = copy.deepcopy(item)
when = _eval_conditional(entry.pop('when', 'True'))
assert len(entry) == 1
if when:
name, spec_list = next(iter(entry.items()))
user_specs = SpecList(name, spec_list, self.spec_lists.copy())
if name in self.spec_lists:
self.spec_lists[name].extend(user_specs)
else:
self.spec_lists[name] = user_specs
spec_list = config_dict(self.yaml).get(user_speclist_name)
user_specs = SpecList(user_speclist_name, [s for s in spec_list if s],
self.spec_lists.copy())
self.spec_lists[user_speclist_name] = user_specs
enable_view = config_dict(self.yaml).get('view')
# enable_view can be boolean, string, or None
if enable_view is True or enable_view is None:
self.views = {
default_view_name: ViewDescriptor(self.view_path_default)
}
elif isinstance(enable_view, six.string_types):
self.views = {default_view_name: ViewDescriptor(enable_view)}
elif enable_view:
self.views = dict((name, ViewDescriptor.from_dict(values))
for name, values in enable_view.items())
else:
self.views = {}
@property
def user_specs(self):
return self.spec_lists[user_speclist_name]
def _set_user_specs_from_lockfile(self):
"""Copy user_specs from a read-in lockfile."""
self.spec_lists = {
user_speclist_name:
SpecList(user_speclist_name,
[str(s) for s in self.concretized_user_specs])
}
def clear(self):
self.spec_lists = {user_speclist_name: SpecList()} # specs from yaml
self.concretized_user_specs = [] # user specs from last concretize
self.concretized_order = [] # roots of last concretize, in order
self.specs_by_hash = {} # concretized specs by hash
self.new_specs = [] # write packages for these on write()
self._repo = None # RepoPath for this env (memoized)
self._previous_active = None # previously active environment
@property
def internal(self):
"""Whether this environment is managed by Spack."""
return self.path.startswith(env_path)
@property
def name(self):
"""Human-readable representation of the environment.
This is the path for directory environments, and just the name
for named environments.
"""
if self.internal:
return os.path.basename(self.path)
else:
return self.path
@property
def active(self):
"""True if this environment is currently active."""
return _active_environment and self.path == _active_environment.path
@property
def manifest_path(self):
"""Path to spack.yaml file in this environment."""
return os.path.join(self.path, manifest_name)
@property
def lock_path(self):
"""Path to spack.lock file in this environment."""
return os.path.join(self.path, lockfile_name)
@property
def _lock_backup_v1_path(self):
"""Path to backup of v1 lockfile before conversion to v2"""
return self.lock_path + '.backup.v1'
@property
def env_subdir_path(self):
"""Path to directory where the env stores repos, logs, views."""
return os.path.join(self.path, env_subdir_name)
@property
def repos_path(self):
return os.path.join(self.path, env_subdir_name, 'repos')
@property
def log_path(self):
return os.path.join(self.path, env_subdir_name, 'logs')
@property
def view_path_default(self):
# default path for environment views
return os.path.join(self.env_subdir_path, 'view')
@property
def repo(self):
if self._repo is None:
self._repo = make_repo_path(self.repos_path)
return self._repo
def included_config_scopes(self):
"""List of included configuration scopes from the environment.
Scopes are listed in the YAML file in order from highest to
lowest precedence, so configuration from earlier scope will take
precedence over later ones.
This routine returns them in the order they should be pushed onto
the internal scope stack (so, in reverse, from lowest to highest).
"""
scopes = []
# load config scopes added via 'include:', in reverse so that
# highest-precedence scopes are last.
includes = config_dict(self.yaml).get('include', [])
for i, config_path in enumerate(reversed(includes)):
# allow paths to contain environment variables
config_path = config_path.format(**os.environ)
# treat relative paths as relative to the environment
if not os.path.isabs(config_path):
config_path = os.path.join(self.path, config_path)
config_path = os.path.normpath(os.path.realpath(config_path))
if os.path.isdir(config_path):
# directories are treated as regular ConfigScopes
config_name = 'env:%s:%s' % (self.name,
os.path.basename(config_path))
scope = spack.config.ConfigScope(config_name, config_path)
else:
# files are assumed to be SingleFileScopes
base, ext = os.path.splitext(os.path.basename(config_path))
config_name = 'env:%s:%s' % (self.name, base)
scope = spack.config.SingleFileScope(
config_name, config_path, spack.schema.merged.schema)
scopes.append(scope)
return scopes
def env_file_config_scope_name(self):
"""Name of the config scope of this environment's manifest file."""
return 'env:%s' % self.name
def env_file_config_scope(self):
"""Get the configuration scope for the environment's manifest file."""
config_name = self.env_file_config_scope_name()
return spack.config.SingleFileScope(config_name, self.manifest_path,
spack.schema.env.schema,
[env_schema_keys])
def config_scopes(self):
"""A list of all configuration scopes for this environment."""
return self.included_config_scopes() + [self.env_file_config_scope()]
def destroy(self):
"""Remove this environment from Spack entirely."""
shutil.rmtree(self.path)
def update_stale_references(self, from_list=None):
"""Iterate over spec lists updating references."""
if not from_list:
from_list = next(iter(self.spec_lists.keys()))
index = list(self.spec_lists.keys()).index(from_list)
# spec_lists is an OrderedDict, all list entries after the modified
# list may refer to the modified list. Update stale references
for i, (name, speclist) in enumerate(
list(self.spec_lists.items())[index + 1:], index + 1):
new_reference = dict((n, self.spec_lists[n])
for n in list(self.spec_lists.keys())[:i])
speclist.update_reference(new_reference)
def add(self, user_spec, list_name=user_speclist_name):
"""Add a single user_spec (non-concretized) to the Environment
Returns:
(bool): True if the spec was added, False if it was already
present and did not need to be added
"""
spec = Spec(user_spec)
if list_name not in self.spec_lists:
raise SpackEnvironmentError('No list %s exists in environment %s' %
(list_name, self.name))
if list_name == user_speclist_name:
if not spec.name:
raise SpackEnvironmentError(
'cannot add anonymous specs to an environment!')
elif not spack.repo.path.exists(spec.name):
raise SpackEnvironmentError('no such package: %s' % spec.name)
list_to_change = self.spec_lists[list_name]
existing = str(spec) in list_to_change.yaml_list
if not existing:
list_to_change.add(str(spec))
self.update_stale_references(list_name)
return bool(not existing)
def remove(self, query_spec, list_name=user_speclist_name, force=False):
"""Remove specs from an environment that match a query_spec"""
query_spec = Spec(query_spec)
list_to_change = self.spec_lists[list_name]
matches = []
if not query_spec.concrete:
matches = [s for s in list_to_change if s.satisfies(query_spec)]
if not matches:
# concrete specs match against concrete specs in the env
specs_hashes = zip(self.concretized_user_specs,
self.concretized_order)
matches = [
s for s, h in specs_hashes if query_spec.dag_hash() == h
]
if not matches:
raise SpackEnvironmentError("Not found: {0}".format(query_spec))
old_specs = set(self.user_specs)
for spec in matches:
if spec in list_to_change:
list_to_change.remove(spec)
self.update_stale_references(list_name)
# If force, update stale concretized specs
# Only check specs removed by this operation
new_specs = set(self.user_specs)
for spec in old_specs - new_specs:
if force and spec in self.concretized_user_specs:
i = self.concretized_user_specs.index(spec)
del self.concretized_user_specs[i]
dag_hash = self.concretized_order[i]
del self.concretized_order[i]
del self.specs_by_hash[dag_hash]
def concretize(self, force=False):
"""Concretize user_specs in this environment.
Only concretizes specs that haven't been concretized yet unless
force is ``True``.
This only modifies the environment in memory. ``write()`` will
write out a lockfile containing concretized specs.
Arguments:
force (bool): re-concretize ALL specs, even those that were
already concretized
Returns:
List of specs that have been concretized. Each entry is a tuple of
the user spec and the corresponding concretized spec.
"""
if force:
# Clear previously concretized specs
self.concretized_user_specs = []
self.concretized_order = []
self.specs_by_hash = {}
# keep any concretized specs whose user specs are still in the manifest
old_concretized_user_specs = self.concretized_user_specs
old_concretized_order = self.concretized_order
old_specs_by_hash = self.specs_by_hash
self.concretized_user_specs = []
self.concretized_order = []
self.specs_by_hash = {}
for s, h in zip(old_concretized_user_specs, old_concretized_order):
if s in self.user_specs:
concrete = old_specs_by_hash[h]
self._add_concrete_spec(s, concrete, new=False)
# Concretize any new user specs that we haven't concretized yet
concretized_specs = []
for uspec, uspec_constraints in zip(
self.user_specs, self.user_specs.specs_as_constraints):
if uspec not in old_concretized_user_specs:
concrete = _concretize_from_constraints(uspec_constraints)
self._add_concrete_spec(uspec, concrete)
concretized_specs.append((uspec, concrete))
return concretized_specs
def install(self, user_spec, concrete_spec=None, **install_args):
"""Install a single spec into an environment.
This will automatically concretize the single spec, but it won't
affect other as-yet unconcretized specs.
"""
spec = Spec(user_spec)
if self.add(spec):
concrete = concrete_spec if concrete_spec else spec.concretized()
self._add_concrete_spec(spec, concrete)
else:
# spec might be in the user_specs, but not installed.
# TODO: Redo name-based comparison for old style envs
spec = next(s for s in self.user_specs if s.satisfies(user_spec))
concrete = self.specs_by_hash.get(spec.build_hash())
if not concrete:
concrete = spec.concretized()
self._add_concrete_spec(spec, concrete)
self._install(concrete, **install_args)
def _install(self, spec, **install_args):
spec.package.do_install(**install_args)
# Make sure log directory exists
log_path = self.log_path
fs.mkdirp(log_path)
with fs.working_dir(self.path):
# Link the resulting log file into logs dir
build_log_link = os.path.join(
log_path, '%s-%s.log' % (spec.name, spec.dag_hash(7)))
if os.path.lexists(build_log_link):
os.remove(build_log_link)
os.symlink(spec.package.build_log_path, build_log_link)
@property
def default_view(self):
if not self.views:
raise SpackEnvironmentError(
"{0} does not have a view enabled".format(self.name))
if default_view_name not in self.views:
raise SpackEnvironmentError(
"{0} does not have a default view enabled".format(self.name))
return self.views[default_view_name]
def update_default_view(self, viewpath):
name = default_view_name
if name in self.views and self.default_view.root != viewpath:
shutil.rmtree(self.default_view.root)
if viewpath:
if name in self.views:
self.default_view.root = viewpath
else:
self.views[name] = ViewDescriptor(viewpath)
else:
self.views.pop(name, None)
def regenerate_views(self):
if not self.views:
tty.debug("Skip view update, this environment does not"
" maintain a view")
return
specs = self._get_environment_specs()
for view in self.views.values():
view.regenerate(specs, self.roots())
def _shell_vars(self):
updates = [
('PATH', ['bin']),
('MANPATH', ['man', 'share/man']),
('ACLOCAL_PATH', ['share/aclocal']),
('LD_LIBRARY_PATH', ['lib', 'lib64']),
('LIBRARY_PATH', ['lib', 'lib64']),
('CPATH', ['include']),
('PKG_CONFIG_PATH', ['lib/pkgconfig', 'lib64/pkgconfig']),
('CMAKE_PREFIX_PATH', ['']),
]
path_updates = list()
if default_view_name in self.views:
for var, subdirs in updates:
paths = filter(
lambda x: os.path.exists(x),
list(
os.path.join(self.default_view.root, x)
for x in subdirs))
path_updates.append((var, paths))
return path_updates
def add_default_view_to_shell(self, shell):
env_mod = EnvironmentModifications()
for var, paths in self._shell_vars():
for path in paths:
env_mod.prepend_path(var, path)
return env_mod.shell_modifications(shell)
def rm_default_view_from_shell(self, shell):
env_mod = EnvironmentModifications()
for var, paths in self._shell_vars():
for path in paths:
env_mod.remove_path(var, path)
return env_mod.shell_modifications(shell)
def _add_concrete_spec(self, spec, concrete, new=True):
"""Called when a new concretized spec is added to the environment.
This ensures that all internal data structures are kept in sync.
Arguments:
spec (Spec): user spec that resulted in the concrete spec
concrete (Spec): spec concretized within this environment
new (bool): whether to write this spec's package to the env
repo on write()
"""
assert concrete.concrete
# when a spec is newly concretized, we need to make a note so
# that we can write its package to the env repo on write()
if new:
self.new_specs.append(concrete)
# update internal lists of specs
self.concretized_user_specs.append(spec)
h = concrete.build_hash()
self.concretized_order.append(h)
self.specs_by_hash[h] = concrete
def install_all(self, args=None):
"""Install all concretized specs in an environment."""
for concretized_hash in self.concretized_order:
spec = self.specs_by_hash[concretized_hash]
# Parse cli arguments and construct a dictionary
# that will be passed to Package.do_install API
kwargs = dict()
if args:
spack.cmd.install.update_kwargs_from_args(args, kwargs)
self._install(spec, **kwargs)
if not spec.external:
# Link the resulting log file into logs dir
build_log_link = os.path.join(
self.log_path, '%s-%s.log' % (spec.name, spec.dag_hash(7)))
if os.path.lexists(build_log_link):
os.remove(build_log_link)
os.symlink(spec.package.build_log_path, build_log_link)
self.regenerate_views()
def all_specs_by_hash(self):
"""Map of hashes to spec for all specs in this environment."""
# Note this uses dag-hashes calculated without build deps as keys,
# whereas the environment tracks specs based on dag-hashes calculated
# with all dependencies. This function should not be used by an
# Environment object for management of its own data structures
hashes = {}
for h in self.concretized_order:
specs = self.specs_by_hash[h].traverse(deptype=('link', 'run'))
for spec in specs:
hashes[spec.dag_hash()] = spec
return hashes
def all_specs(self):
"""Return all specs, even those a user spec would shadow."""
return sorted(self.all_specs_by_hash().values())
def all_hashes(self):
"""Return all specs, even those a user spec would shadow."""
return list(self.all_specs_by_hash().keys())
def roots(self):
"""Specs explicitly requested by the user *in this environment*.
Yields both added and installed specs that have user specs in
`spack.yaml`.
"""
concretized = dict(self.concretized_specs())
for spec in self.user_specs:
concrete = concretized.get(spec)
yield concrete if concrete else spec
def added_specs(self):
"""Specs that are not yet installed.
Yields the user spec for non-concretized specs, and the concrete
spec for already concretized but not yet installed specs.
"""
concretized = dict(self.concretized_specs())
for spec in self.user_specs:
concrete = concretized.get(spec)
if not concrete:
yield spec
elif not concrete.package.installed:
yield concrete
def concretized_specs(self):
"""Tuples of (user spec, concrete spec) for all concrete specs."""
for s, h in zip(self.concretized_user_specs, self.concretized_order):
yield (s, self.specs_by_hash[h])
def removed_specs(self):
"""Tuples of (user spec, concrete spec) for all specs that will be
removed on nexg concretize."""
needed = set()
for s, c in self.concretized_specs():
if s in self.user_specs:
for d in c.traverse():
needed.add(d)
for s, c in self.concretized_specs():
for d in c.traverse():
if d not in needed:
yield d
def _get_environment_specs(self, recurse_dependencies=True):
"""Returns the specs of all the packages in an environment.
If these specs appear under different user_specs, only one copy
is added to the list returned.
"""
spec_list = list()
for spec_hash in self.concretized_order:
spec = self.specs_by_hash[spec_hash]
specs = (spec.traverse(
deptype=('link', 'run')) if recurse_dependencies else (spec, ))
spec_list.extend(specs)
return spec_list
def _to_lockfile_dict(self):
"""Create a dictionary to store a lockfile for this environment."""
concrete_specs = {}
for spec in self.specs_by_hash.values():
for s in spec.traverse():
dag_hash_all = s.build_hash()
if dag_hash_all not in concrete_specs:
spec_dict = s.to_node_dict(hash=ht.build_hash)
spec_dict[s.name]['hash'] = s.dag_hash()
concrete_specs[dag_hash_all] = spec_dict
hash_spec_list = zip(self.concretized_order,
self.concretized_user_specs)
# this is the lockfile we'll write out
data = {
# metadata about the format
'_meta': {
'file-type': 'spack-lockfile',
'lockfile-version': lockfile_format_version,
},
# users specs + hashes are the 'roots' of the environment
'roots': [{
'hash': h,
'spec': str(s)
} for h, s in hash_spec_list],
# Concrete specs by hash, including dependencies
'concrete_specs': concrete_specs,
}
return data
def _read_lockfile(self, file_or_json):
"""Read a lockfile from a file or from a raw string."""
lockfile_dict = sjson.load(file_or_json)
self._read_lockfile_dict(lockfile_dict)
return lockfile_dict['_meta']['lockfile-version']
def _read_lockfile_dict(self, d):
"""Read a lockfile dictionary into this environment."""
roots = d['roots']
self.concretized_user_specs = [Spec(r['spec']) for r in roots]
self.concretized_order = [r['hash'] for r in roots]
json_specs_by_hash = d['concrete_specs']
root_hashes = set(self.concretized_order)
specs_by_hash = {}
for dag_hash, node_dict in json_specs_by_hash.items():
specs_by_hash[dag_hash] = Spec.from_node_dict(node_dict)
for dag_hash, node_dict in json_specs_by_hash.items():
for dep_name, dep_hash, deptypes in (
Spec.dependencies_from_node_dict(node_dict)):
specs_by_hash[dag_hash]._add_dependency(
specs_by_hash[dep_hash], deptypes)
# If we are reading an older lockfile format (which uses dag hashes
# that exclude build deps), we use this to convert the old
# concretized_order to the full hashes (preserving the order)
old_hash_to_new = {}
self.specs_by_hash = {}
for _, spec in specs_by_hash.items():
dag_hash = spec.dag_hash()
build_hash = spec.build_hash()
if dag_hash in root_hashes:
old_hash_to_new[dag_hash] = build_hash
if (dag_hash in root_hashes or build_hash in root_hashes):
self.specs_by_hash[build_hash] = spec
if old_hash_to_new:
# Replace any older hashes in concretized_order with hashes
# that include build deps
self.concretized_order = [
old_hash_to_new.get(h, h) for h in self.concretized_order
]
def write(self):
"""Writes an in-memory environment to its location on disk.
This will also write out package files for each newly concretized spec.
"""
# ensure path in var/spack/environments
fs.mkdirp(self.path)
if self.specs_by_hash:
# ensure the prefix/.env directory exists
fs.mkdirp(self.env_subdir_path)
for spec in self.new_specs:
for dep in spec.traverse():
if not dep.concrete:
raise ValueError('specs passed to environment.write() '
'must be concrete!')
root = os.path.join(self.repos_path, dep.namespace)
repo = spack.repo.create_or_construct(root, dep.namespace)
pkg_dir = repo.dirname_for_package_name(dep.name)
fs.mkdirp(pkg_dir)
spack.repo.path.dump_provenance(dep, pkg_dir)
self.new_specs = []
# write the lock file last
with fs.write_tmp_and_move(self.lock_path) as f:
sjson.dump(self._to_lockfile_dict(), stream=f)
else:
if os.path.exists(self.lock_path):
os.unlink(self.lock_path)
# invalidate _repo cache
self._repo = None
# put any changes in the definitions in the YAML
for name, speclist in self.spec_lists.items():
if name == user_speclist_name:
# The primary list is handled differently
continue
conf = config_dict(self.yaml)
active_yaml_lists = [
l for l in conf.get('definitions', [])
if name in l and _eval_conditional(l.get('when', 'True'))
]
# Remove any specs in yaml that are not in internal representation
for ayl in active_yaml_lists:
# If it's not a string, it's a matrix. Those can't have changed
# If it is a string that starts with '$', it's a reference.
# Those also can't have changed.
ayl[name][:] = [
s for s in ayl.setdefault(name, [])
if (not isinstance(s, six.string_types))
or s.startswith('$') or Spec(s) in speclist.specs
]
# Put the new specs into the first active list from the yaml
new_specs = [
entry for entry in speclist.yaml_list
if isinstance(entry, six.string_types) and not any(
entry in ayl[name] for ayl in active_yaml_lists)
]
list_for_new_specs = active_yaml_lists[0].setdefault(name, [])
list_for_new_specs[:] = list_for_new_specs + new_specs
# put the new user specs in the YAML.
# This can be done directly because there can't be multiple definitions
# nor when clauses for `specs` list.
yaml_spec_list = config_dict(self.yaml).setdefault(
user_speclist_name, [])
yaml_spec_list[:] = self.user_specs.yaml_list
default_name = default_view_name
if self.views and len(self.views) == 1 and default_name in self.views:
path = self.default_view.root
if self.default_view == ViewDescriptor(self.view_path_default):
view = True
elif self.default_view == ViewDescriptor(path):
view = path
else:
view = dict((name, view.to_dict())
for name, view in self.views.items())
elif self.views:
view = dict(
(name, view.to_dict()) for name, view in self.views.items())
else:
view = False
yaml_dict = config_dict(self.yaml)
if view is not True:
# The default case is to keep an active view inside of the
# Spack environment directory. To avoid cluttering the config,
# we omit the setting in this case.
yaml_dict['view'] = view
elif 'view' in yaml_dict:
del yaml_dict['view']
# if all that worked, write out the manifest file at the top level
with fs.write_tmp_and_move(self.manifest_path) as f:
_write_yaml(self.yaml, f)
# TODO: for operations that just add to the env (install etc.) this
# could just call update_view
self.regenerate_views()
def __enter__(self):
self._previous_active = _active_environment
activate(self)
return
def __exit__(self, exc_type, exc_val, exc_tb):
deactivate()
if self._previous_active:
activate(self._previous_active)