def fetch_tarballs(url, name, version):
"""Try to find versions of the supplied archive by scraping the web.
Prompts the user to select how many to download if many are found.
"""
versions = spack.util.web.find_versions_of_archive(url)
rkeys = sorted(versions.keys(), reverse=True)
versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))
archives_to_fetch = 1
if not versions:
# If the fetch failed for some reason, revert to what the user provided
versions = { version : url }
elif len(versions) > 1:
tty.msg("Found %s versions of %s:" % (len(versions), name),
*spack.cmd.elide_list(
["%-10s%s" % (v,u) for v, u in versions.iteritems()]))
print
archives_to_fetch = tty.get_number(
"Include how many checksums in the package file?",
default=5, abort='q')
if not archives_to_fetch:
tty.die("Aborted.")
sorted_versions = sorted(versions.keys(), reverse=True)
sorted_urls = [versions[v] for v in sorted_versions]
return sorted_versions[:archives_to_fetch], sorted_urls[:archives_to_fetch]
def fetch_tarballs(url, name, version):
"""Try to find versions of the supplied archive by scraping the web.
Prompts the user to select how many to download if many are found."""
versions = spack.util.web.find_versions_of_archive(url)
rkeys = sorted(versions.keys(), reverse=True)
versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))
archives_to_fetch = 1
if not versions:
# If the fetch failed for some reason, revert to what the user provided
versions = {version: url}
elif len(versions) > 1:
tty.msg("Found %s versions of %s:" % (len(versions), name),
*spack.cmd.elide_list(
["%-10s%s" % (v, u) for v, u in versions.iteritems()]))
print('')
archives_to_fetch = tty.get_number(
"Include how many checksums in the package file?",
default=5, abort='q')
if not archives_to_fetch:
tty.die("Aborted.")
sorted_versions = sorted(versions.keys(), reverse=True)
sorted_urls = [versions[v] for v in sorted_versions]
return sorted_versions[:archives_to_fetch], sorted_urls[:archives_to_fetch]
def _read_manifest(self, f):
"""Read manifest file and set up user specs."""
self.yaml = _read_yaml(f)
self.spec_lists = OrderedDict()
for item in config_dict(self.yaml).get('definitions', []):
entry = copy.deepcopy(item)
when = _eval_conditional(entry.pop('when', 'True'))
assert len(entry) == 1
if when:
name, spec_list = next(iter(entry.items()))
user_specs = SpecList(name, spec_list, self.spec_lists.copy())
if name in self.spec_lists:
self.spec_lists[name].extend(user_specs)
else:
self.spec_lists[name] = user_specs
spec_list = config_dict(self.yaml).get(user_speclist_name)
user_specs = SpecList(user_speclist_name, [s for s in spec_list if s],
self.spec_lists.copy())
self.spec_lists[user_speclist_name] = user_specs
enable_view = config_dict(self.yaml).get('view')
# enable_view can be boolean, string, or None
if enable_view is True or enable_view is None:
self.views = {
default_view_name: ViewDescriptor(self.view_path_default)}
elif isinstance(enable_view, six.string_types):
self.views = {default_view_name: ViewDescriptor(enable_view)}
elif enable_view:
self.views = dict((name, ViewDescriptor.from_dict(values))
for name, values in enable_view.items())
else:
self.views = {}
def save(self, name, modules):
collection = OrderedDict()
for module in modules:
ar = pymod.mc.archive_module(module)
ar["refcount"] = 0
collection.setdefault(module.modulepath, []).append(ar)
collection = list(collection.items())
self.data.update({name: collection})
self.write(self.data, self.filename)
return None
def __init__(self, *scopes):
"""Initialize a configuration with an initial list of scopes.
Args:
scopes (list of ConfigScope): list of scopes to add to this
Configuration, ordered from lowest to highest precedence
"""
self.scopes = OrderedDict()
for scope in scopes:
self.push_scope(scope)
def initmock(self):
# Use the mock packages database for these tests. This allows
# us to set up contrived packages that don't interfere with
# real ones.
self.db = RepoPath(spack.mock_packages_path)
spack.repo.swap(self.db)
# Mock up temporary configuration directories
self.temp_config = tempfile.mkdtemp()
self.mock_site_config = os.path.join(self.temp_config, 'site')
self.mock_user_config = os.path.join(self.temp_config, 'user')
mkdirp(self.mock_site_config)
mkdirp(self.mock_user_config)
for filename, data in mock_configs.items():
conf_yaml = os.path.join(self.mock_site_config, filename)
with open(conf_yaml, 'w') as f:
f.write(data)
# TODO: Mocking this up is kind of brittle b/c ConfigScope
# TODO: constructor modifies config_scopes. Make it cleaner.
spack.config.clear_config_caches()
self.real_scopes = spack.config.config_scopes
spack.config.config_scopes = OrderedDict()
spack.config.ConfigScope('site', self.mock_site_config)
spack.config.ConfigScope('user', self.mock_user_config)
# Keep tests from interfering with the actual module path.
self.real_share_path = spack.share_path
spack.share_path = tempfile.mkdtemp()
# Store changes to the package's dependencies so we can
# restore later.
self.saved_deps = {}
def initmock(self):
# Use the mock packages database for these tests. This allows
# us to set up contrived packages that don't interfere with
# real ones.
self.db = RepoPath(spack.mock_packages_path)
spack.repo.swap(self.db)
spack.config.clear_config_caches()
self.real_scopes = spack.config.config_scopes
# Mock up temporary configuration directories
self.temp_config = tempfile.mkdtemp()
self.mock_site_config = os.path.join(self.temp_config, 'site')
self.mock_user_config = os.path.join(self.temp_config, 'user')
mkdirp(self.mock_site_config)
mkdirp(self.mock_user_config)
for confs in [('compilers.yaml', mock_compiler_config), ('packages.yaml', mock_packages_config)]:
conf_yaml = os.path.join(self.mock_site_config, confs[0])
with open(conf_yaml, 'w') as f:
f.write(confs[1])
# TODO: Mocking this up is kind of brittle b/c ConfigScope
# TODO: constructor modifies config_scopes. Make it cleaner.
spack.config.config_scopes = OrderedDict()
spack.config.ConfigScope('site', self.mock_site_config)
spack.config.ConfigScope('user', self.mock_user_config)
# Store changes to the package's dependencies so we can
# restore later.
self.saved_deps = {}
def slice_items(items, ignore, scoped_argkeys_cache):
# we pick the first item which uses a fixture instance in the
# requested scope and which we haven't seen yet. We slice the input
# items list into a list of items_nomatch, items_same and
# items_other
if scoped_argkeys_cache: # do we need to do work at all?
it = iter(items)
# first find a slicing key
for i, item in enumerate(it):
argkeys = scoped_argkeys_cache.get(item)
if argkeys is not None:
newargkeys = OrderedDict.fromkeys(k for k in argkeys
if k not in ignore)
if newargkeys: # found a slicing key
slicing_argkey, _ = newargkeys.popitem()
items_before = items[:i]
items_same = [item]
items_other = []
# now slice the remainder of the list
for item in it:
argkeys = scoped_argkeys_cache.get(item)
if argkeys and slicing_argkey in argkeys and \
slicing_argkey not in ignore:
items_same.append(item)
else:
items_other.append(item)
newignore = ignore.copy()
newignore.add(slicing_argkey)
return (items_before, items_same, items_other, newignore)
return items, None, None, None
def slice_items(items, ignore, scoped_argkeys_cache):
# we pick the first item which uses a fixture instance in the
# requested scope and which we haven't seen yet. We slice the input
# items list into a list of items_nomatch, items_same and
# items_other
if scoped_argkeys_cache: # do we need to do work at all?
it = iter(items)
# first find a slicing key
for i, item in enumerate(it):
argkeys = scoped_argkeys_cache.get(item)
if argkeys is not None:
newargkeys = OrderedDict.fromkeys(k for k in argkeys if k not in ignore)
if newargkeys: # found a slicing key
slicing_argkey, _ = newargkeys.popitem()
items_before = items[:i]
items_same = [item]
items_other = []
# now slice the remainder of the list
for item in it:
argkeys = scoped_argkeys_cache.get(item)
if argkeys and slicing_argkey in argkeys and \
slicing_argkey not in ignore:
items_same.append(item)
else:
items_other.append(item)
newignore = ignore.copy()
newignore.add(slicing_argkey)
return (items_before, items_same, items_other, newignore)
return items, None, None, None
def __init__(self, args):
Reporter.__init__(self, args)
self.template_dir = os.path.join('reports', 'cdash')
self.cdash_upload_url = args.cdash_upload_url
if self.cdash_upload_url:
self.buildid_regexp = re.compile("<buildId>([0-9]+)</buildId>")
self.phase_regexp = re.compile(r"Executing phase: '(.*)'")
if args.package:
packages = args.package
else:
packages = []
for file in args.specfiles:
with open(file, 'r') as f:
s = spack.spec.Spec.from_yaml(f)
packages.append(s.format())
self.install_command = ' '.join(packages)
self.base_buildname = args.cdash_build or self.install_command
self.site = args.cdash_site or socket.gethostname()
self.osname = platform.system()
self.endtime = int(time.time())
if args.cdash_buildstamp:
self.buildstamp = args.cdash_buildstamp
else:
buildstamp_format = "%Y%m%d-%H%M-{0}".format(args.cdash_track)
self.buildstamp = time.strftime(buildstamp_format,
time.localtime(self.endtime))
self.buildIds = OrderedDict()
self.revision = ''
git = which('git')
with working_dir(spack.paths.spack_root):
self.revision = git('rev-parse', 'HEAD', output=str).strip()
self.multiple_packages = False
def setUp(self):
super(ConfigTest, self).setUp()
self.tmp_dir = mkdtemp('.tmp', 'spack-config-test-')
spack.config.config_scopes = OrderedDict()
spack.config.ConfigScope('test_low_priority',
os.path.join(self.tmp_dir, 'low'))
spack.config.ConfigScope('test_high_priority',
os.path.join(self.tmp_dir, 'high'))
def reorder_items(items):
argkeys_cache = {}
for scopenum in range(0, scopenum_function):
argkeys_cache[scopenum] = d = {}
for item in items:
keys = OrderedDict.fromkeys(get_parametrized_fixture_keys(item, scopenum))
if keys:
d[item] = keys
return reorder_items_atscope(items, set(), argkeys_cache, 0)
def reorder_items(items):
argkeys_cache = {}
for scopenum in range(0, scopenum_function):
argkeys_cache[scopenum] = d = {}
for item in items:
keys = OrderedDict.fromkeys(
get_parametrized_fixture_keys(item, scopenum))
if keys:
d[item] = keys
return reorder_items_atscope(items, set(), argkeys_cache, 0)
def __init__(self, *scopes):
"""Initialize a configuration with an initial list of scopes.
Args:
scopes (list of ConfigScope): list of scopes to add to this
Configuration, ordered from lowest to highest precedence
"""
self.scopes = OrderedDict()
for scope in scopes:
self.push_scope(scope)
def _valid_virtuals_and_externals(self, spec):
"""Returns a list of candidate virtual dep providers and external
packages that coiuld be used to concretize a spec.
Preferred specs come first in the list.
"""
# First construct a list of concrete candidates to replace spec with.
candidates = [spec]
pref_key = lambda spec: 0 # no-op pref key
if spec.virtual:
candidates = spack.repo.providers_for(spec)
if not candidates:
raise UnsatisfiableProviderSpecError(candidates[0], spec)
# Find nearest spec in the DAG (up then down) that has prefs.
spec_w_prefs = find_spec(
spec, lambda p: PackagePrefs.has_preferred_providers(
p.name, spec.name), spec) # default to spec itself.
# Create a key to sort candidates by the prefs we found
pref_key = PackagePrefs(spec_w_prefs.name, 'providers', spec.name)
# For each candidate package, if it has externals, add those
# to the usable list. if it's not buildable, then *only* add
# the externals.
#
# Use an OrderedDict to avoid duplicates (use it like a set)
usable = OrderedDict()
for cspec in candidates:
if is_spec_buildable(cspec):
usable[cspec] = True
externals = spec_externals(cspec)
for ext in externals:
if ext.satisfies(spec):
usable[ext] = True
# If nothing is in the usable list now, it's because we aren't
# allowed to build anything.
if not usable:
raise NoBuildError(spec)
# Use a sort key to order the results
return sorted(
usable,
key=lambda spec: (
not spec.external, # prefer externals
pref_key(spec), # respect prefs
spec.name, # group by name
reverse_order(spec.versions), # latest version
spec # natural order
))
def add_to_loaded_collection(self, name):
"""Add a module `name` to the currently loaded collection"""
collection_name = pymod.environ.get(pymod.names.loaded_collection)
if collection_name is None: # pragma: no cover
tty.die("There is no collection currently loaded")
data = OrderedDict(self.data.pop(collection_name))
module = pymod.modulepath.get(name)
if module is None:
raise pymod.error.ModuleNotFoundError(name)
if not module.is_loaded:
pymod.mc.load_impl(module)
for (mp, modules) in data.items():
if mp != module.modulepath:
continue
for other in modules:
if other["fullname"] == module.fullname: # pragma: no cover
tty.warn("{0} is already in collection {1}".format(
name, collection_name))
return
ar = pymod.mc.archive_module(module)
ar["refcount"] = 0
data.setdefault(module.modulepath, []).append(ar)
data = list(data.items())
self.data.update({collection_name: data})
self.write(self.data, self.filename)
return None
def test_get_header():
headers = {
'Content-type': 'text/plain'
}
# looking up headers should just work like a plain dict
# lookup when there is an entry with the right key
assert(web_util.get_header(headers, 'Content-type') == 'text/plain')
# looking up headers should still work if there is a fuzzy match
assert(web_util.get_header(headers, 'contentType') == 'text/plain')
# ...unless there is an exact match for the "fuzzy" spelling.
headers['contentType'] = 'text/html'
assert(web_util.get_header(headers, 'contentType') == 'text/html')
# If lookup has to fallback to fuzzy matching and there are more than one
# fuzzy match, the result depends on the internal ordering of the given
# mapping
headers = OrderedDict()
headers['Content-type'] = 'text/plain'
headers['contentType'] = 'text/html'
assert(web_util.get_header(headers, 'CONTENT_TYPE') == 'text/plain')
del headers['Content-type']
assert(web_util.get_header(headers, 'CONTENT_TYPE') == 'text/html')
# Same as above, but different ordering
headers = OrderedDict()
headers['contentType'] = 'text/html'
headers['Content-type'] = 'text/plain'
assert(web_util.get_header(headers, 'CONTENT_TYPE') == 'text/html')
del headers['contentType']
assert(web_util.get_header(headers, 'CONTENT_TYPE') == 'text/plain')
# If there isn't even a fuzzy match, raise KeyError
with pytest.raises(KeyError):
web_util.get_header(headers, 'ContentLength')
def setUp(self):
super(ConfigTest, self).setUp()
self.tmp_dir = mkdtemp('.tmp', 'spack-config-test-')
self.a_comp_specs = [
ac['compiler']['spec'] for ac in a_comps['compilers']
]
self.b_comp_specs = [
bc['compiler']['spec'] for bc in b_comps['compilers']
]
spack.config.config_scopes = OrderedDict()
for priority in ['low', 'high']:
scope_dir = os.path.join(self.tmp_dir, priority)
spack.config.ConfigScope(priority, scope_dir)
def pop_from_loaded_collection(self, name):
"""Remove a module `name` to the currently loaded collection"""
collection_name = pymod.environ.get(pymod.names.loaded_collection)
if collection_name is None:
tty.die(
"There is no collection currently loaded") # pragma: no cover
data = OrderedDict(self.data.pop(collection_name))
module = pymod.modulepath.get(name)
if module is None: # pragma: no cover
raise pymod.error.ModuleNotFoundError(name)
if module.is_loaded:
pymod.mc.unload_impl(module)
for (mp, modules) in data.items():
if mp != module.modulepath: # pragma: no cover
continue
data[mp] = [
other for other in modules
if other["fullname"] == module.fullname
]
break
data = list(data.items())
self.data.update({collection_name: data})
self.write(self.data, self.filename)
return None
def view_copy(src, dst, view, spec=None):
"""
Copy a file from src to dst.
Use spec and view to generate relocations
"""
shutil.copy2(src, dst)
if spec and not spec.external:
# Not metadata, we have to relocate it
# Get information on where to relocate from/to
# This is vestigial code for the *old* location of sbang. Previously,
# sbang was a bash script, and it lived in the spack prefix. It is
# now a POSIX script that lives in the install prefix. Old packages
# will have the old sbang location in their shebangs.
# TODO: Not sure which one to use...
import spack.hooks.sbang as sbang
# Break a package include cycle
import spack.relocate
orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(spack.paths.spack_root)
new_sbang = sbang.sbang_shebang_line()
prefix_to_projection = OrderedDict(
{spec.prefix: view.get_projection_for_spec(spec)})
for dep in spec.traverse():
if not dep.external:
prefix_to_projection[dep.prefix] = \
view.get_projection_for_spec(dep)
if spack.relocate.is_binary(dst):
spack.relocate.relocate_text_bin(binaries=[dst],
prefixes=prefix_to_projection)
else:
prefix_to_projection[spack.store.layout.root] = view._root
prefix_to_projection[orig_sbang] = new_sbang
spack.relocate.relocate_text(files=[dst],
prefixes=prefix_to_projection)
try:
stat = os.stat(src)
os.chown(dst, stat.st_uid, stat.st_gid)
except OSError:
tty.debug('Can\'t change the permissions for %s' % dst)
def __init__(self, args):
Reporter.__init__(self, args)
tty.set_verbose(args.verbose)
self.success = True
self.template_dir = os.path.join('reports', 'cdash')
self.cdash_upload_url = args.cdash_upload_url
if self.cdash_upload_url:
self.buildid_regexp = re.compile("<buildId>([0-9]+)</buildId>")
self.phase_regexp = re.compile(r"Executing phase: '(.*)'")
self.authtoken = None
if 'SPACK_CDASH_AUTH_TOKEN' in os.environ:
tty.verbose("Using CDash auth token from environment")
self.authtoken = os.environ.get('SPACK_CDASH_AUTH_TOKEN')
if getattr(args, 'spec', ''):
packages = args.spec
elif getattr(args, 'specs', ''):
packages = args.specs
else:
packages = []
for file in args.specfiles:
with open(file, 'r') as f:
s = spack.spec.Spec.from_yaml(f)
packages.append(s.format())
self.install_command = ' '.join(packages)
self.base_buildname = args.cdash_build or self.install_command
self.site = args.cdash_site or socket.gethostname()
self.osname = platform.system()
self.endtime = int(time.time())
if args.cdash_buildstamp:
self.buildstamp = args.cdash_buildstamp
else:
buildstamp_format = "%Y%m%d-%H%M-{0}".format(args.cdash_track)
self.buildstamp = time.strftime(buildstamp_format,
time.localtime(self.endtime))
self.buildIds = OrderedDict()
self.revision = ''
git = which('git')
with working_dir(spack.paths.spack_root):
self.revision = git('rev-parse', 'HEAD', output=str).strip()
self.multiple_packages = False
def view_copy(src, dst, view, spec=None):
"""
Copy a file from src to dst.
Use spec and view to generate relocations
"""
shutil.copyfile(src, dst)
if spec:
# Not metadata, we have to relocate it
# Get information on where to relocate from/to
# This is vestigial code for the *old* location of sbang. Previously,
# sbang was a bash script, and it lived in the spack prefix. It is
# now a POSIX script that lives in the install prefix. Old packages
# will have the old sbang location in their shebangs.
# TODO: Not sure which one to use...
import spack.hooks.sbang as sbang
orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(spack.paths.spack_root)
new_sbang = sbang.sbang_shebang_line()
prefix_to_projection = OrderedDict({
spec.prefix: view.get_projection_for_spec(spec),
spack.paths.spack_root: view._root})
for dep in spec.traverse():
prefix_to_projection[dep.prefix] = \
view.get_projection_for_spec(dep)
if spack.relocate.is_binary(dst):
spack.relocate.relocate_text_bin(
binaries=[dst],
prefixes=prefix_to_projection
)
else:
prefix_to_projection[spack.store.layout.root] = view._root
prefix_to_projection[orig_sbang] = new_sbang
spack.relocate.relocate_text(
files=[dst],
prefixes=prefix_to_projection
)
def relocate_text_bin(binaries, prefixes, concurrency=32):
"""Replace null terminated path strings hard coded into binaries.
The new install prefix must be shorter than the original one.
Args:
binaries (list): binaries to be relocated
prefixes (OrderedDict): String prefixes which need to be changed.
concurrency (int): Desired degree of parallelism.
Raises:
BinaryTextReplaceError: when the new path is longer than the old path
"""
byte_prefixes = OrderedDict({})
for orig_prefix, new_prefix in prefixes.items():
if orig_prefix != new_prefix:
if isinstance(orig_prefix, bytes):
orig_bytes = orig_prefix
else:
orig_bytes = orig_prefix.encode('utf-8')
if isinstance(new_prefix, bytes):
new_bytes = new_prefix
else:
new_bytes = new_prefix.encode('utf-8')
byte_prefixes[orig_bytes] = new_bytes
# Do relocations on text in binaries that refers to the install tree
# multiprocesing.ThreadPool.map requires single argument
args = []
for binary in binaries:
args.append((binary, byte_prefixes))
tp = multiprocessing.pool.ThreadPool(processes=concurrency)
try:
tp.map(llnl.util.lang.star(_replace_prefix_bin), args)
finally:
tp.terminate()
tp.join()
def relocate_text(files, prefixes, concurrency=32):
"""Relocate text file from the original installation prefix to the
new prefix.
Relocation also affects the the path in Spack's sbang script.
Args:
files (list): Text files to be relocated
prefixes (OrderedDict): String prefixes which need to be changed
concurrency (int): Preferred degree of parallelism
"""
# This now needs to be handled by the caller in all cases
# orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(orig_spack)
# new_sbang = '#!/bin/bash {0}/bin/sbang'.format(new_spack)
compiled_prefixes = OrderedDict({})
for orig_prefix, new_prefix in prefixes.items():
if orig_prefix != new_prefix:
orig_bytes = orig_prefix.encode('utf-8')
orig_prefix_rexp = re.compile(
b'(?<![\\w\\-_/])([\\w\\-_]*?)%s([\\w\\-_/]*)' % orig_bytes)
new_bytes = b'\\1%s\\2' % new_prefix.encode('utf-8')
compiled_prefixes[orig_prefix_rexp] = new_bytes
# Do relocations on text that refers to the install tree
# multiprocesing.ThreadPool.map requires single argument
args = []
for filename in files:
args.append((filename, compiled_prefixes))
tp = multiprocessing.pool.ThreadPool(processes=concurrency)
try:
tp.map(llnl.util.lang.star(_replace_prefix_text), args)
finally:
tp.terminate()
tp.join()
class Configuration(object):
"""A full Spack configuration, from a hierarchy of config files.
This class makes it easy to add a new scope on top of an existing one.
"""
def __init__(self, *scopes):
"""Initialize a configuration with an initial list of scopes.
Args:
scopes (list of ConfigScope): list of scopes to add to this
Configuration, ordered from lowest to highest precedence
"""
self.scopes = OrderedDict()
for scope in scopes:
self.push_scope(scope)
def push_scope(self, scope):
"""Add a higher precedence scope to the Configuration."""
cmd_line_scope = None
if self.scopes:
highest_precedence_scope = list(self.scopes.values())[-1]
if highest_precedence_scope.name == 'command_line':
# If the command-line scope is present, it should always
# be the scope of highest precedence
cmd_line_scope = self.pop_scope()
self.scopes[scope.name] = scope
if cmd_line_scope:
self.scopes['command_line'] = cmd_line_scope
def pop_scope(self):
"""Remove the highest precedence scope and return it."""
name, scope = self.scopes.popitem(last=True)
return scope
def remove_scope(self, scope_name):
return self.scopes.pop(scope_name)
@property
def file_scopes(self):
"""List of writable scopes with an associated file."""
return [s for s in self.scopes.values() if type(s) == ConfigScope]
def highest_precedence_scope(self):
"""Non-internal scope with highest precedence."""
return next(reversed(self.file_scopes), None)
def _validate_scope(self, scope):
"""Ensure that scope is valid in this configuration.
This should be used by routines in ``config.py`` to validate
scope name arguments, and to determine a default scope where no
scope is specified.
Raises:
ValueError: if ``scope`` is not valid
Returns:
ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid
"""
if scope is None:
# default to the scope with highest precedence.
return self.highest_precedence_scope()
elif scope in self.scopes:
return self.scopes[scope]
else:
raise ValueError("Invalid config scope: '%s'. Must be one of %s"
% (scope, self.scopes.keys()))
def get_config_filename(self, scope, section):
"""For some scope and section, get the name of the configuration file.
"""
scope = self._validate_scope(scope)
return scope.get_section_filename(section)
def clear_caches(self):
"""Clears the caches for configuration files,
This will cause files to be re-read upon the next request."""
for scope in self.scopes.values():
scope.clear()
def update_config(self, section, update_data, scope=None):
"""Update the configuration file for a particular scope.
Overwrites contents of a section in a scope with update_data,
then writes out the config file.
update_data should have the top-level section name stripped off
(it will be re-added). Data itself can be a list, dict, or any
other yaml-ish structure.
"""
_validate_section_name(section) # validate section name
scope = self._validate_scope(scope) # get ConfigScope object
# read only the requested section's data.
scope.sections[section] = {section: update_data}
scope.write_section(section)
def get_config(self, section, scope=None):
"""Get configuration settings for a section.
If ``scope`` is ``None`` or not provided, return the merged contents
of all of Spack's configuration scopes. If ``scope`` is provided,
return only the confiugration as specified in that scope.
This off the top-level name from the YAML section. That is, for a
YAML config file that looks like this::
config:
install_tree: $spack/opt/spack
module_roots:
lmod: $spack/share/spack/lmod
``get_config('config')`` will return::
{ 'install_tree': '$spack/opt/spack',
'module_roots: {
'lmod': '$spack/share/spack/lmod'
}
}
"""
_validate_section_name(section)
if scope is None:
scopes = self.scopes.values()
else:
scopes = [self._validate_scope(scope)]
merged_section = syaml.syaml_dict()
for scope in scopes:
# read potentially cached data from the scope.
data = scope.get_section(section)
# Skip empty configs
if not data or not isinstance(data, dict):
continue
if section not in data:
continue
merged_section = _merge_yaml(merged_section, data)
# no config files -- empty config.
if section not in merged_section:
return {}
# take the top key off before returning.
return merged_section[section]
def get(self, path, default=None, scope=None):
"""Get a config section or a single value from one.
Accepts a path syntax that allows us to grab nested config map
entries. Getting the 'config' section would look like::
spack.config.get('config')
and the ``dirty`` section in the ``config`` scope would be::
spack.config.get('config:dirty')
We use ``:`` as the separator, like YAML objects.
"""
# TODO: Currently only handles maps. Think about lists if neded.
section, _, rest = path.partition(':')
value = self.get_config(section, scope=scope)
if not rest:
return value
parts = rest.split(':')
while parts:
key = parts.pop(0)
value = value.get(key, default)
return value
def set(self, path, value, scope=None):
"""Convenience function for setting single values in config files.
Accepts the path syntax described in ``get()``.
"""
section, _, rest = path.partition(':')
if not rest:
self.update_config(section, value, scope=scope)
else:
section_data = self.get_config(section, scope=scope)
parts = rest.split(':')
data = section_data
while len(parts) > 1:
key = parts.pop(0)
data = data[key]
data[parts[0]] = value
self.update_config(section, section_data, scope=scope)
def __iter__(self):
"""Iterate over scopes in this configuration."""
for scope in self.scopes.values():
yield scope
def print_section(self, section, blame=False):
"""Print a configuration to stdout."""
try:
data = syaml.syaml_dict()
data[section] = self.get_config(section)
syaml.dump(
data, stream=sys.stdout, default_flow_style=False, blame=blame)
except (yaml.YAMLError, IOError):
raise ConfigError("Error reading configuration: %s" % section)
},
'dotkit': {
'allOf': [
{'$ref': '#/definitions/module_type_configuration'}, # Base configuration
{} # Specific dotkit extensions
]
},
}
},
},
},
}
"""OrderedDict of config scopes keyed by name.
Later scopes will override earlier scopes.
"""
config_scopes = OrderedDict()
def validate_section_name(section):
"""Exit if the section is not a valid section."""
if section not in section_schemas:
tty.die("Invalid config section: '%s'. Options are: %s" %
(section, " ".join(section_schemas.keys())))
def extend_with_default(validator_class):
"""Add support for the 'default' attribute for properties and patternProperties.
jsonschema does not handle this out of the box -- it only
validates. This allows us to set default values for configs
where certain fields are `None` b/c they're deleted or
class Configuration(object):
"""A full Spack configuration, from a hierarchy of config files.
This class makes it easy to add a new scope on top of an existing one.
"""
def __init__(self, *scopes):
"""Initialize a configuration with an initial list of scopes.
Args:
scopes (list of ConfigScope): list of scopes to add to this
Configuration, ordered from lowest to highest precedence
"""
self.scopes = OrderedDict()
for scope in scopes:
self.push_scope(scope)
self.format_updates = collections.defaultdict(list)
@_config_mutator
def push_scope(self, scope):
"""Add a higher precedence scope to the Configuration."""
cmd_line_scope = None
if self.scopes:
highest_precedence_scope = list(self.scopes.values())[-1]
if highest_precedence_scope.name == 'command_line':
# If the command-line scope is present, it should always
# be the scope of highest precedence
cmd_line_scope = self.pop_scope()
self.scopes[scope.name] = scope
if cmd_line_scope:
self.scopes['command_line'] = cmd_line_scope
@_config_mutator
def pop_scope(self):
"""Remove the highest precedence scope and return it."""
name, scope = self.scopes.popitem(last=True)
return scope
@_config_mutator
def remove_scope(self, scope_name):
return self.scopes.pop(scope_name)
@property
def file_scopes(self):
"""List of writable scopes with an associated file."""
return [s for s in self.scopes.values()
if (type(s) == ConfigScope
or type(s) == SingleFileScope)]
def highest_precedence_scope(self):
"""Non-internal scope with highest precedence."""
return next(reversed(self.file_scopes), None)
def highest_precedence_non_platform_scope(self):
"""Non-internal non-platform scope with highest precedence
Platform-specific scopes are of the form scope/platform"""
generator = reversed(self.file_scopes)
highest = next(generator, None)
while highest and highest.is_platform_dependent:
highest = next(generator, None)
return highest
def matching_scopes(self, reg_expr):
"""
List of all scopes whose names match the provided regular expression.
For example, matching_scopes(r'^command') will return all scopes
whose names begin with `command`.
"""
return [s for s in self.scopes.values() if re.search(reg_expr, s.name)]
def _validate_scope(self, scope):
"""Ensure that scope is valid in this configuration.
This should be used by routines in ``config.py`` to validate
scope name arguments, and to determine a default scope where no
scope is specified.
Raises:
ValueError: if ``scope`` is not valid
Returns:
ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid
"""
if scope is None:
# default to the scope with highest precedence.
return self.highest_precedence_scope()
elif scope in self.scopes:
return self.scopes[scope]
else:
raise ValueError("Invalid config scope: '%s'. Must be one of %s"
% (scope, self.scopes.keys()))
def get_config_filename(self, scope, section):
"""For some scope and section, get the name of the configuration file.
"""
scope = self._validate_scope(scope)
return scope.get_section_filename(section)
@_config_mutator
def clear_caches(self):
"""Clears the caches for configuration files,
This will cause files to be re-read upon the next request."""
for scope in self.scopes.values():
scope.clear()
@_config_mutator
def update_config(self, section, update_data, scope=None, force=False):
"""Update the configuration file for a particular scope.
Overwrites contents of a section in a scope with update_data,
then writes out the config file.
update_data should have the top-level section name stripped off
(it will be re-added). Data itself can be a list, dict, or any
other yaml-ish structure.
Configuration scopes that are still written in an old schema
format will fail to update unless ``force`` is True.
Args:
section (str): section of the configuration to be updated
update_data (dict): data to be used for the update
scope (str): scope to be updated
force (str): force the update
"""
if self.format_updates.get(section) and not force:
msg = ('The "{0}" section of the configuration needs to be written'
' to disk, but is currently using a deprecated format. '
'Please update it using:\n\n'
'\tspack config [--scope=<scope] update {0}\n\n'
'Note that previous versions of Spack will not be able to '
'use the updated configuration.')
msg = msg.format(section)
raise RuntimeError(msg)
_validate_section_name(section) # validate section name
scope = self._validate_scope(scope) # get ConfigScope object
# manually preserve comments
need_comment_copy = (section in scope.sections and
scope.sections[section] is not None)
if need_comment_copy:
comments = getattr(scope.sections[section][section],
yaml.comments.Comment.attrib,
None)
# read only the requested section's data.
scope.sections[section] = syaml.syaml_dict({section: update_data})
if need_comment_copy and comments:
setattr(scope.sections[section][section],
yaml.comments.Comment.attrib,
comments)
scope._write_section(section)
def get_config(self, section, scope=None):
"""Get configuration settings for a section.
If ``scope`` is ``None`` or not provided, return the merged contents
of all of Spack's configuration scopes. If ``scope`` is provided,
return only the configuration as specified in that scope.
This off the top-level name from the YAML section. That is, for a
YAML config file that looks like this::
config:
install_tree: $spack/opt/spack
module_roots:
lmod: $spack/share/spack/lmod
``get_config('config')`` will return::
{ 'install_tree': '$spack/opt/spack',
'module_roots: {
'lmod': '$spack/share/spack/lmod'
}
}
"""
return self._get_config_memoized(section, scope)
@llnl.util.lang.memoized
def _get_config_memoized(self, section, scope):
_validate_section_name(section)
if scope is None:
scopes = self.scopes.values()
else:
scopes = [self._validate_scope(scope)]
merged_section = syaml.syaml_dict()
for scope in scopes:
# read potentially cached data from the scope.
data = scope.get_section(section)
# Skip empty configs
if not data or not isinstance(data, dict):
continue
if section not in data:
continue
# We might be reading configuration files in an old format,
# thus read data and update it in memory if need be.
changed = _update_in_memory(data, section)
if changed:
self.format_updates[section].append(scope)
merged_section = merge_yaml(merged_section, data)
# no config files -- empty config.
if section not in merged_section:
return syaml.syaml_dict()
# take the top key off before returning.
ret = merged_section[section]
if isinstance(ret, dict):
ret = syaml.syaml_dict(ret)
return ret
def get(self, path, default=None, scope=None):
"""Get a config section or a single value from one.
Accepts a path syntax that allows us to grab nested config map
entries. Getting the 'config' section would look like::
spack.config.get('config')
and the ``dirty`` section in the ``config`` scope would be::
spack.config.get('config:dirty')
We use ``:`` as the separator, like YAML objects.
"""
# TODO: Currently only handles maps. Think about lists if needed.
parts = process_config_path(path)
section = parts.pop(0)
value = self.get_config(section, scope=scope)
while parts:
key = parts.pop(0)
value = value.get(key, default)
return value
@_config_mutator
def set(self, path, value, scope=None):
"""Convenience function for setting single values in config files.
Accepts the path syntax described in ``get()``.
"""
if ':' not in path:
# handle bare section name as path
self.update_config(path, value, scope=scope)
return
parts = process_config_path(path)
section = parts.pop(0)
section_data = self.get_config(section, scope=scope)
data = section_data
while len(parts) > 1:
key = parts.pop(0)
if _override(key):
new = type(data[key])()
del data[key]
else:
new = data[key]
if isinstance(new, dict):
# Make it an ordered dict
new = syaml.syaml_dict(new)
# reattach to parent object
data[key] = new
data = new
if _override(parts[0]):
data.pop(parts[0], None)
# update new value
data[parts[0]] = value
self.update_config(section, section_data, scope=scope)
def __iter__(self):
"""Iterate over scopes in this configuration."""
for scope in self.scopes.values():
yield scope
def print_section(self, section, blame=False):
"""Print a configuration to stdout."""
try:
data = syaml.syaml_dict()
data[section] = self.get_config(section)
syaml.dump_config(
data, stream=sys.stdout, default_flow_style=False, blame=blame)
except (yaml.YAMLError, IOError):
raise ConfigError("Error reading configuration: %s" % section)
try:
name = spack.url.parse_name(url, version)
except spack.url.UndetectableNameError, e:
# Use a user-supplied name if one is present
tty.die("Couldn't guess a name for this package. Try running:", "",
"spack create --name <name> <url>")
if not valid_module_name(name):
tty.die("Package name can only contain A-Z, a-z, 0-9, '_' and '-'")
tty.msg("This looks like a URL for %s version %s." % (name, version))
tty.msg("Creating template for package %s" % name)
versions = spack.util.web.find_versions_of_archive(url)
rkeys = sorted(versions.keys(), reverse=True)
versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))
archives_to_fetch = 1
if not versions:
# If the fetch failed for some reason, revert to what the user provided
versions = { version : url }
elif len(versions) > 1:
tty.msg("Found %s versions of %s:" % (len(versions), name),
*spack.cmd.elide_list(
["%-10s%s" % (v,u) for v, u in versions.iteritems()]))
print
archives_to_fetch = tty.get_number(
"Include how many checksums in the package file?",
default=5, abort='q')
if not archives_to_fetch:
import re
import inspect
import textwrap
from six import StringIO
from argparse import ArgumentParser
from ordereddict_backport import OrderedDict
import pymod.paths
import pymod.callback
category_descriptions = OrderedDict(
environment="Functions for modifying the environment",
path="Functions for modifying path-like variables",
alias="Functions for defining shell aliases and functions",
module="General module functions",
interaction="Functions for interacting with other modules",
family="Functions for interacting with module families",
modulepath="Functions for interacting with the MODULEPATH",
info="Functions for relaying information",
utility="General purpose utilities",
)
def fill_with_paragraphs(string, indent=""):
filled = []
lines = []
for line in string.split("\n"):
if line.split():
lines.append(line)
elif lines:
filled.append(
class Environment(object):
def __init__(self, path, init_file=None, with_view=None):
"""Create a new environment.
The environment can be optionally initialized with either a
spack.yaml or spack.lock file.
Arguments:
path (str): path to the root directory of this environment
init_file (str or file object): filename or file object to
initialize the environment
with_view (str or bool): whether a view should be maintained for
the environment. If the value is a string, it specifies the
path to the view.
"""
self.path = os.path.abspath(path)
self.clear()
if init_file:
with fs.open_if_filename(init_file) as f:
if hasattr(f, 'name') and f.name.endswith('.lock'):
self._read_manifest(default_manifest_yaml)
self._read_lockfile(f)
self._set_user_specs_from_lockfile()
else:
self._read_manifest(f)
else:
default_manifest = not os.path.exists(self.manifest_path)
if default_manifest:
# No manifest, use default yaml
self._read_manifest(default_manifest_yaml)
else:
with open(self.manifest_path) as f:
self._read_manifest(f)
if os.path.exists(self.lock_path):
with open(self.lock_path) as f:
read_lock_version = self._read_lockfile(f)
if default_manifest:
# No manifest, set user specs from lockfile
self._set_user_specs_from_lockfile()
if read_lock_version == 1:
tty.debug(
"Storing backup of old lockfile {0} at {1}".format(
self.lock_path, self._lock_backup_v1_path))
shutil.copy(self.lock_path, self._lock_backup_v1_path)
if with_view is False:
self.views = {}
elif with_view is True:
self.views = {
default_view_name: ViewDescriptor(self.view_path_default)
}
elif isinstance(with_view, six.string_types):
self.views = {default_view_name: ViewDescriptor(with_view)}
# If with_view is None, then defer to the view settings determined by
# the manifest file
def _read_manifest(self, f):
"""Read manifest file and set up user specs."""
self.yaml = _read_yaml(f)
self.spec_lists = OrderedDict()
for item in config_dict(self.yaml).get('definitions', []):
entry = copy.deepcopy(item)
when = _eval_conditional(entry.pop('when', 'True'))
assert len(entry) == 1
if when:
name, spec_list = next(iter(entry.items()))
user_specs = SpecList(name, spec_list, self.spec_lists.copy())
if name in self.spec_lists:
self.spec_lists[name].extend(user_specs)
else:
self.spec_lists[name] = user_specs
spec_list = config_dict(self.yaml).get(user_speclist_name)
user_specs = SpecList(user_speclist_name, [s for s in spec_list if s],
self.spec_lists.copy())
self.spec_lists[user_speclist_name] = user_specs
enable_view = config_dict(self.yaml).get('view')
# enable_view can be boolean, string, or None
if enable_view is True or enable_view is None:
self.views = {
default_view_name: ViewDescriptor(self.view_path_default)
}
elif isinstance(enable_view, six.string_types):
self.views = {default_view_name: ViewDescriptor(enable_view)}
elif enable_view:
self.views = dict((name, ViewDescriptor.from_dict(values))
for name, values in enable_view.items())
else:
self.views = {}
@property
def user_specs(self):
return self.spec_lists[user_speclist_name]
def _set_user_specs_from_lockfile(self):
"""Copy user_specs from a read-in lockfile."""
self.spec_lists = {
user_speclist_name:
SpecList(user_speclist_name,
[str(s) for s in self.concretized_user_specs])
}
def clear(self):
self.spec_lists = {user_speclist_name: SpecList()} # specs from yaml
self.concretized_user_specs = [] # user specs from last concretize
self.concretized_order = [] # roots of last concretize, in order
self.specs_by_hash = {} # concretized specs by hash
self.new_specs = [] # write packages for these on write()
self._repo = None # RepoPath for this env (memoized)
self._previous_active = None # previously active environment
@property
def internal(self):
"""Whether this environment is managed by Spack."""
return self.path.startswith(env_path)
@property
def name(self):
"""Human-readable representation of the environment.
This is the path for directory environments, and just the name
for named environments.
"""
if self.internal:
return os.path.basename(self.path)
else:
return self.path
@property
def active(self):
"""True if this environment is currently active."""
return _active_environment and self.path == _active_environment.path
@property
def manifest_path(self):
"""Path to spack.yaml file in this environment."""
return os.path.join(self.path, manifest_name)
@property
def lock_path(self):
"""Path to spack.lock file in this environment."""
return os.path.join(self.path, lockfile_name)
@property
def _lock_backup_v1_path(self):
"""Path to backup of v1 lockfile before conversion to v2"""
return self.lock_path + '.backup.v1'
@property
def env_subdir_path(self):
"""Path to directory where the env stores repos, logs, views."""
return os.path.join(self.path, env_subdir_name)
@property
def repos_path(self):
return os.path.join(self.path, env_subdir_name, 'repos')
@property
def log_path(self):
return os.path.join(self.path, env_subdir_name, 'logs')
@property
def view_path_default(self):
# default path for environment views
return os.path.join(self.env_subdir_path, 'view')
@property
def repo(self):
if self._repo is None:
self._repo = make_repo_path(self.repos_path)
return self._repo
def included_config_scopes(self):
"""List of included configuration scopes from the environment.
Scopes are listed in the YAML file in order from highest to
lowest precedence, so configuration from earlier scope will take
precedence over later ones.
This routine returns them in the order they should be pushed onto
the internal scope stack (so, in reverse, from lowest to highest).
"""
scopes = []
# load config scopes added via 'include:', in reverse so that
# highest-precedence scopes are last.
includes = config_dict(self.yaml).get('include', [])
for i, config_path in enumerate(reversed(includes)):
# allow paths to contain environment variables
config_path = config_path.format(**os.environ)
# treat relative paths as relative to the environment
if not os.path.isabs(config_path):
config_path = os.path.join(self.path, config_path)
config_path = os.path.normpath(os.path.realpath(config_path))
if os.path.isdir(config_path):
# directories are treated as regular ConfigScopes
config_name = 'env:%s:%s' % (self.name,
os.path.basename(config_path))
scope = spack.config.ConfigScope(config_name, config_path)
else:
# files are assumed to be SingleFileScopes
base, ext = os.path.splitext(os.path.basename(config_path))
config_name = 'env:%s:%s' % (self.name, base)
scope = spack.config.SingleFileScope(
config_name, config_path, spack.schema.merged.schema)
scopes.append(scope)
return scopes
def env_file_config_scope_name(self):
"""Name of the config scope of this environment's manifest file."""
return 'env:%s' % self.name
def env_file_config_scope(self):
"""Get the configuration scope for the environment's manifest file."""
config_name = self.env_file_config_scope_name()
return spack.config.SingleFileScope(config_name, self.manifest_path,
spack.schema.env.schema,
[env_schema_keys])
def config_scopes(self):
"""A list of all configuration scopes for this environment."""
return self.included_config_scopes() + [self.env_file_config_scope()]
def destroy(self):
"""Remove this environment from Spack entirely."""
shutil.rmtree(self.path)
def update_stale_references(self, from_list=None):
"""Iterate over spec lists updating references."""
if not from_list:
from_list = next(iter(self.spec_lists.keys()))
index = list(self.spec_lists.keys()).index(from_list)
# spec_lists is an OrderedDict, all list entries after the modified
# list may refer to the modified list. Update stale references
for i, (name, speclist) in enumerate(
list(self.spec_lists.items())[index + 1:], index + 1):
new_reference = dict((n, self.spec_lists[n])
for n in list(self.spec_lists.keys())[:i])
speclist.update_reference(new_reference)
def add(self, user_spec, list_name=user_speclist_name):
"""Add a single user_spec (non-concretized) to the Environment
Returns:
(bool): True if the spec was added, False if it was already
present and did not need to be added
"""
spec = Spec(user_spec)
if list_name not in self.spec_lists:
raise SpackEnvironmentError('No list %s exists in environment %s' %
(list_name, self.name))
if list_name == user_speclist_name:
if not spec.name:
raise SpackEnvironmentError(
'cannot add anonymous specs to an environment!')
elif not spack.repo.path.exists(spec.name):
raise SpackEnvironmentError('no such package: %s' % spec.name)
list_to_change = self.spec_lists[list_name]
existing = str(spec) in list_to_change.yaml_list
if not existing:
list_to_change.add(str(spec))
self.update_stale_references(list_name)
return bool(not existing)
def remove(self, query_spec, list_name=user_speclist_name, force=False):
"""Remove specs from an environment that match a query_spec"""
query_spec = Spec(query_spec)
list_to_change = self.spec_lists[list_name]
matches = []
if not query_spec.concrete:
matches = [s for s in list_to_change if s.satisfies(query_spec)]
if not matches:
# concrete specs match against concrete specs in the env
specs_hashes = zip(self.concretized_user_specs,
self.concretized_order)
matches = [
s for s, h in specs_hashes if query_spec.dag_hash() == h
]
if not matches:
raise SpackEnvironmentError("Not found: {0}".format(query_spec))
old_specs = set(self.user_specs)
for spec in matches:
if spec in list_to_change:
list_to_change.remove(spec)
self.update_stale_references(list_name)
# If force, update stale concretized specs
# Only check specs removed by this operation
new_specs = set(self.user_specs)
for spec in old_specs - new_specs:
if force and spec in self.concretized_user_specs:
i = self.concretized_user_specs.index(spec)
del self.concretized_user_specs[i]
dag_hash = self.concretized_order[i]
del self.concretized_order[i]
del self.specs_by_hash[dag_hash]
def concretize(self, force=False):
"""Concretize user_specs in this environment.
Only concretizes specs that haven't been concretized yet unless
force is ``True``.
This only modifies the environment in memory. ``write()`` will
write out a lockfile containing concretized specs.
Arguments:
force (bool): re-concretize ALL specs, even those that were
already concretized
Returns:
List of specs that have been concretized. Each entry is a tuple of
the user spec and the corresponding concretized spec.
"""
if force:
# Clear previously concretized specs
self.concretized_user_specs = []
self.concretized_order = []
self.specs_by_hash = {}
# keep any concretized specs whose user specs are still in the manifest
old_concretized_user_specs = self.concretized_user_specs
old_concretized_order = self.concretized_order
old_specs_by_hash = self.specs_by_hash
self.concretized_user_specs = []
self.concretized_order = []
self.specs_by_hash = {}
for s, h in zip(old_concretized_user_specs, old_concretized_order):
if s in self.user_specs:
concrete = old_specs_by_hash[h]
self._add_concrete_spec(s, concrete, new=False)
# Concretize any new user specs that we haven't concretized yet
concretized_specs = []
for uspec, uspec_constraints in zip(
self.user_specs, self.user_specs.specs_as_constraints):
if uspec not in old_concretized_user_specs:
concrete = _concretize_from_constraints(uspec_constraints)
self._add_concrete_spec(uspec, concrete)
concretized_specs.append((uspec, concrete))
return concretized_specs
def install(self, user_spec, concrete_spec=None, **install_args):
"""Install a single spec into an environment.
This will automatically concretize the single spec, but it won't
affect other as-yet unconcretized specs.
"""
spec = Spec(user_spec)
if self.add(spec):
concrete = concrete_spec if concrete_spec else spec.concretized()
self._add_concrete_spec(spec, concrete)
else:
# spec might be in the user_specs, but not installed.
# TODO: Redo name-based comparison for old style envs
spec = next(s for s in self.user_specs if s.satisfies(user_spec))
concrete = self.specs_by_hash.get(spec.build_hash())
if not concrete:
concrete = spec.concretized()
self._add_concrete_spec(spec, concrete)
self._install(concrete, **install_args)
def _install(self, spec, **install_args):
spec.package.do_install(**install_args)
# Make sure log directory exists
log_path = self.log_path
fs.mkdirp(log_path)
with fs.working_dir(self.path):
# Link the resulting log file into logs dir
build_log_link = os.path.join(
log_path, '%s-%s.log' % (spec.name, spec.dag_hash(7)))
if os.path.lexists(build_log_link):
os.remove(build_log_link)
os.symlink(spec.package.build_log_path, build_log_link)
@property
def default_view(self):
if not self.views:
raise SpackEnvironmentError(
"{0} does not have a view enabled".format(self.name))
if default_view_name not in self.views:
raise SpackEnvironmentError(
"{0} does not have a default view enabled".format(self.name))
return self.views[default_view_name]
def update_default_view(self, viewpath):
name = default_view_name
if name in self.views and self.default_view.root != viewpath:
shutil.rmtree(self.default_view.root)
if viewpath:
if name in self.views:
self.default_view.root = viewpath
else:
self.views[name] = ViewDescriptor(viewpath)
else:
self.views.pop(name, None)
def regenerate_views(self):
if not self.views:
tty.debug("Skip view update, this environment does not"
" maintain a view")
return
specs = self._get_environment_specs()
for view in self.views.values():
view.regenerate(specs, self.roots())
def _shell_vars(self):
updates = [
('PATH', ['bin']),
('MANPATH', ['man', 'share/man']),
('ACLOCAL_PATH', ['share/aclocal']),
('LD_LIBRARY_PATH', ['lib', 'lib64']),
('LIBRARY_PATH', ['lib', 'lib64']),
('CPATH', ['include']),
('PKG_CONFIG_PATH', ['lib/pkgconfig', 'lib64/pkgconfig']),
('CMAKE_PREFIX_PATH', ['']),
]
path_updates = list()
if default_view_name in self.views:
for var, subdirs in updates:
paths = filter(
lambda x: os.path.exists(x),
list(
os.path.join(self.default_view.root, x)
for x in subdirs))
path_updates.append((var, paths))
return path_updates
def add_default_view_to_shell(self, shell):
env_mod = EnvironmentModifications()
for var, paths in self._shell_vars():
for path in paths:
env_mod.prepend_path(var, path)
return env_mod.shell_modifications(shell)
def rm_default_view_from_shell(self, shell):
env_mod = EnvironmentModifications()
for var, paths in self._shell_vars():
for path in paths:
env_mod.remove_path(var, path)
return env_mod.shell_modifications(shell)
def _add_concrete_spec(self, spec, concrete, new=True):
"""Called when a new concretized spec is added to the environment.
This ensures that all internal data structures are kept in sync.
Arguments:
spec (Spec): user spec that resulted in the concrete spec
concrete (Spec): spec concretized within this environment
new (bool): whether to write this spec's package to the env
repo on write()
"""
assert concrete.concrete
# when a spec is newly concretized, we need to make a note so
# that we can write its package to the env repo on write()
if new:
self.new_specs.append(concrete)
# update internal lists of specs
self.concretized_user_specs.append(spec)
h = concrete.build_hash()
self.concretized_order.append(h)
self.specs_by_hash[h] = concrete
def install_all(self, args=None):
"""Install all concretized specs in an environment."""
for concretized_hash in self.concretized_order:
spec = self.specs_by_hash[concretized_hash]
# Parse cli arguments and construct a dictionary
# that will be passed to Package.do_install API
kwargs = dict()
if args:
spack.cmd.install.update_kwargs_from_args(args, kwargs)
self._install(spec, **kwargs)
if not spec.external:
# Link the resulting log file into logs dir
build_log_link = os.path.join(
self.log_path, '%s-%s.log' % (spec.name, spec.dag_hash(7)))
if os.path.lexists(build_log_link):
os.remove(build_log_link)
os.symlink(spec.package.build_log_path, build_log_link)
self.regenerate_views()
def all_specs_by_hash(self):
"""Map of hashes to spec for all specs in this environment."""
# Note this uses dag-hashes calculated without build deps as keys,
# whereas the environment tracks specs based on dag-hashes calculated
# with all dependencies. This function should not be used by an
# Environment object for management of its own data structures
hashes = {}
for h in self.concretized_order:
specs = self.specs_by_hash[h].traverse(deptype=('link', 'run'))
for spec in specs:
hashes[spec.dag_hash()] = spec
return hashes
def all_specs(self):
"""Return all specs, even those a user spec would shadow."""
return sorted(self.all_specs_by_hash().values())
def all_hashes(self):
"""Return all specs, even those a user spec would shadow."""
return list(self.all_specs_by_hash().keys())
def roots(self):
"""Specs explicitly requested by the user *in this environment*.
Yields both added and installed specs that have user specs in
`spack.yaml`.
"""
concretized = dict(self.concretized_specs())
for spec in self.user_specs:
concrete = concretized.get(spec)
yield concrete if concrete else spec
def added_specs(self):
"""Specs that are not yet installed.
Yields the user spec for non-concretized specs, and the concrete
spec for already concretized but not yet installed specs.
"""
concretized = dict(self.concretized_specs())
for spec in self.user_specs:
concrete = concretized.get(spec)
if not concrete:
yield spec
elif not concrete.package.installed:
yield concrete
def concretized_specs(self):
"""Tuples of (user spec, concrete spec) for all concrete specs."""
for s, h in zip(self.concretized_user_specs, self.concretized_order):
yield (s, self.specs_by_hash[h])
def removed_specs(self):
"""Tuples of (user spec, concrete spec) for all specs that will be
removed on nexg concretize."""
needed = set()
for s, c in self.concretized_specs():
if s in self.user_specs:
for d in c.traverse():
needed.add(d)
for s, c in self.concretized_specs():
for d in c.traverse():
if d not in needed:
yield d
def _get_environment_specs(self, recurse_dependencies=True):
"""Returns the specs of all the packages in an environment.
If these specs appear under different user_specs, only one copy
is added to the list returned.
"""
spec_list = list()
for spec_hash in self.concretized_order:
spec = self.specs_by_hash[spec_hash]
specs = (spec.traverse(
deptype=('link', 'run')) if recurse_dependencies else (spec, ))
spec_list.extend(specs)
return spec_list
def _to_lockfile_dict(self):
"""Create a dictionary to store a lockfile for this environment."""
concrete_specs = {}
for spec in self.specs_by_hash.values():
for s in spec.traverse():
dag_hash_all = s.build_hash()
if dag_hash_all not in concrete_specs:
spec_dict = s.to_node_dict(hash=ht.build_hash)
spec_dict[s.name]['hash'] = s.dag_hash()
concrete_specs[dag_hash_all] = spec_dict
hash_spec_list = zip(self.concretized_order,
self.concretized_user_specs)
# this is the lockfile we'll write out
data = {
# metadata about the format
'_meta': {
'file-type': 'spack-lockfile',
'lockfile-version': lockfile_format_version,
},
# users specs + hashes are the 'roots' of the environment
'roots': [{
'hash': h,
'spec': str(s)
} for h, s in hash_spec_list],
# Concrete specs by hash, including dependencies
'concrete_specs': concrete_specs,
}
return data
def _read_lockfile(self, file_or_json):
"""Read a lockfile from a file or from a raw string."""
lockfile_dict = sjson.load(file_or_json)
self._read_lockfile_dict(lockfile_dict)
return lockfile_dict['_meta']['lockfile-version']
def _read_lockfile_dict(self, d):
"""Read a lockfile dictionary into this environment."""
roots = d['roots']
self.concretized_user_specs = [Spec(r['spec']) for r in roots]
self.concretized_order = [r['hash'] for r in roots]
json_specs_by_hash = d['concrete_specs']
root_hashes = set(self.concretized_order)
specs_by_hash = {}
for dag_hash, node_dict in json_specs_by_hash.items():
specs_by_hash[dag_hash] = Spec.from_node_dict(node_dict)
for dag_hash, node_dict in json_specs_by_hash.items():
for dep_name, dep_hash, deptypes in (
Spec.dependencies_from_node_dict(node_dict)):
specs_by_hash[dag_hash]._add_dependency(
specs_by_hash[dep_hash], deptypes)
# If we are reading an older lockfile format (which uses dag hashes
# that exclude build deps), we use this to convert the old
# concretized_order to the full hashes (preserving the order)
old_hash_to_new = {}
self.specs_by_hash = {}
for _, spec in specs_by_hash.items():
dag_hash = spec.dag_hash()
build_hash = spec.build_hash()
if dag_hash in root_hashes:
old_hash_to_new[dag_hash] = build_hash
if (dag_hash in root_hashes or build_hash in root_hashes):
self.specs_by_hash[build_hash] = spec
if old_hash_to_new:
# Replace any older hashes in concretized_order with hashes
# that include build deps
self.concretized_order = [
old_hash_to_new.get(h, h) for h in self.concretized_order
]
def write(self):
"""Writes an in-memory environment to its location on disk.
This will also write out package files for each newly concretized spec.
"""
# ensure path in var/spack/environments
fs.mkdirp(self.path)
if self.specs_by_hash:
# ensure the prefix/.env directory exists
fs.mkdirp(self.env_subdir_path)
for spec in self.new_specs:
for dep in spec.traverse():
if not dep.concrete:
raise ValueError('specs passed to environment.write() '
'must be concrete!')
root = os.path.join(self.repos_path, dep.namespace)
repo = spack.repo.create_or_construct(root, dep.namespace)
pkg_dir = repo.dirname_for_package_name(dep.name)
fs.mkdirp(pkg_dir)
spack.repo.path.dump_provenance(dep, pkg_dir)
self.new_specs = []
# write the lock file last
with fs.write_tmp_and_move(self.lock_path) as f:
sjson.dump(self._to_lockfile_dict(), stream=f)
else:
if os.path.exists(self.lock_path):
os.unlink(self.lock_path)
# invalidate _repo cache
self._repo = None
# put any changes in the definitions in the YAML
for name, speclist in self.spec_lists.items():
if name == user_speclist_name:
# The primary list is handled differently
continue
conf = config_dict(self.yaml)
active_yaml_lists = [
l for l in conf.get('definitions', [])
if name in l and _eval_conditional(l.get('when', 'True'))
]
# Remove any specs in yaml that are not in internal representation
for ayl in active_yaml_lists:
# If it's not a string, it's a matrix. Those can't have changed
# If it is a string that starts with '$', it's a reference.
# Those also can't have changed.
ayl[name][:] = [
s for s in ayl.setdefault(name, [])
if (not isinstance(s, six.string_types))
or s.startswith('$') or Spec(s) in speclist.specs
]
# Put the new specs into the first active list from the yaml
new_specs = [
entry for entry in speclist.yaml_list
if isinstance(entry, six.string_types) and not any(
entry in ayl[name] for ayl in active_yaml_lists)
]
list_for_new_specs = active_yaml_lists[0].setdefault(name, [])
list_for_new_specs[:] = list_for_new_specs + new_specs
# put the new user specs in the YAML.
# This can be done directly because there can't be multiple definitions
# nor when clauses for `specs` list.
yaml_spec_list = config_dict(self.yaml).setdefault(
user_speclist_name, [])
yaml_spec_list[:] = self.user_specs.yaml_list
default_name = default_view_name
if self.views and len(self.views) == 1 and default_name in self.views:
path = self.default_view.root
if self.default_view == ViewDescriptor(self.view_path_default):
view = True
elif self.default_view == ViewDescriptor(path):
view = path
else:
view = dict((name, view.to_dict())
for name, view in self.views.items())
elif self.views:
view = dict(
(name, view.to_dict()) for name, view in self.views.items())
else:
view = False
yaml_dict = config_dict(self.yaml)
if view is not True:
# The default case is to keep an active view inside of the
# Spack environment directory. To avoid cluttering the config,
# we omit the setting in this case.
yaml_dict['view'] = view
elif 'view' in yaml_dict:
del yaml_dict['view']
# if all that worked, write out the manifest file at the top level
with fs.write_tmp_and_move(self.manifest_path) as f:
_write_yaml(self.yaml, f)
# TODO: for operations that just add to the env (install etc.) this
# could just call update_view
self.regenerate_views()
def __enter__(self):
self._previous_active = _active_environment
activate(self)
return
def __exit__(self, exc_type, exc_val, exc_tb):
deactivate()
if self._previous_active:
activate(self._previous_active)
class Configuration(object):
"""A full Spack configuration, from a hierarchy of config files.
This class makes it easy to add a new scope on top of an existing one.
"""
def __init__(self, *scopes):
"""Initialize a configuration with an initial list of scopes.
Args:
scopes (list of ConfigScope): list of scopes to add to this
Configuration, ordered from lowest to highest precedence
"""
self.scopes = OrderedDict()
for scope in scopes:
self.push_scope(scope)
def push_scope(self, scope):
"""Add a higher precedence scope to the Configuration."""
cmd_line_scope = None
if self.scopes:
highest_precedence_scope = list(self.scopes.values())[-1]
if highest_precedence_scope.name == 'command_line':
# If the command-line scope is present, it should always
# be the scope of highest precedence
cmd_line_scope = self.pop_scope()
self.scopes[scope.name] = scope
if cmd_line_scope:
self.scopes['command_line'] = cmd_line_scope
def pop_scope(self):
"""Remove the highest precedence scope and return it."""
name, scope = self.scopes.popitem(last=True)
return scope
def remove_scope(self, scope_name):
return self.scopes.pop(scope_name)
@property
def file_scopes(self):
"""List of writable scopes with an associated file."""
return [s for s in self.scopes.values() if type(s) == ConfigScope]
def highest_precedence_scope(self):
"""Non-internal scope with highest precedence."""
return next(reversed(self.file_scopes), None)
def matching_scopes(self, reg_expr):
"""
List of all scopes whose names match the provided regular expression.
For example, matching_scopes(r'^command') will return all scopes
whose names begin with `command`.
"""
return [s for s in self.scopes.values() if re.search(reg_expr, s.name)]
def _validate_scope(self, scope):
"""Ensure that scope is valid in this configuration.
This should be used by routines in ``config.py`` to validate
scope name arguments, and to determine a default scope where no
scope is specified.
Raises:
ValueError: if ``scope`` is not valid
Returns:
ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid
"""
if scope is None:
# default to the scope with highest precedence.
return self.highest_precedence_scope()
elif scope in self.scopes:
return self.scopes[scope]
else:
raise ValueError("Invalid config scope: '%s'. Must be one of %s"
% (scope, self.scopes.keys()))
def get_config_filename(self, scope, section):
"""For some scope and section, get the name of the configuration file.
"""
scope = self._validate_scope(scope)
return scope.get_section_filename(section)
def clear_caches(self):
"""Clears the caches for configuration files,
This will cause files to be re-read upon the next request."""
for scope in self.scopes.values():
scope.clear()
def update_config(self, section, update_data, scope=None):
"""Update the configuration file for a particular scope.
Overwrites contents of a section in a scope with update_data,
then writes out the config file.
update_data should have the top-level section name stripped off
(it will be re-added). Data itself can be a list, dict, or any
other yaml-ish structure.
"""
_validate_section_name(section) # validate section name
scope = self._validate_scope(scope) # get ConfigScope object
# read only the requested section's data.
scope.sections[section] = {section: update_data}
scope.write_section(section)
def get_config(self, section, scope=None):
"""Get configuration settings for a section.
If ``scope`` is ``None`` or not provided, return the merged contents
of all of Spack's configuration scopes. If ``scope`` is provided,
return only the confiugration as specified in that scope.
This off the top-level name from the YAML section. That is, for a
YAML config file that looks like this::
config:
install_tree: $spack/opt/spack
module_roots:
lmod: $spack/share/spack/lmod
``get_config('config')`` will return::
{ 'install_tree': '$spack/opt/spack',
'module_roots: {
'lmod': '$spack/share/spack/lmod'
}
}
"""
_validate_section_name(section)
if scope is None:
scopes = self.scopes.values()
else:
scopes = [self._validate_scope(scope)]
merged_section = syaml.syaml_dict()
for scope in scopes:
# read potentially cached data from the scope.
data = scope.get_section(section)
# Skip empty configs
if not data or not isinstance(data, dict):
continue
if section not in data:
continue
merged_section = _merge_yaml(merged_section, data)
# no config files -- empty config.
if section not in merged_section:
return {}
# take the top key off before returning.
return merged_section[section]
def get(self, path, default=None, scope=None):
"""Get a config section or a single value from one.
Accepts a path syntax that allows us to grab nested config map
entries. Getting the 'config' section would look like::
spack.config.get('config')
and the ``dirty`` section in the ``config`` scope would be::
spack.config.get('config:dirty')
We use ``:`` as the separator, like YAML objects.
"""
# TODO: Currently only handles maps. Think about lists if neded.
section, _, rest = path.partition(':')
value = self.get_config(section, scope=scope)
if not rest:
return value
parts = rest.split(':')
while parts:
key = parts.pop(0)
value = value.get(key, default)
return value
def set(self, path, value, scope=None):
"""Convenience function for setting single values in config files.
Accepts the path syntax described in ``get()``.
"""
section, _, rest = path.partition(':')
if not rest:
self.update_config(section, value, scope=scope)
else:
section_data = self.get_config(section, scope=scope)
parts = rest.split(':')
data = section_data
while len(parts) > 1:
key = parts.pop(0)
data = data[key]
data[parts[0]] = value
self.update_config(section, section_data, scope=scope)
def __iter__(self):
"""Iterate over scopes in this configuration."""
for scope in self.scopes.values():
yield scope
def print_section(self, section, blame=False):
"""Print a configuration to stdout."""
try:
data = syaml.syaml_dict()
data[section] = self.get_config(section)
syaml.dump_config(
data, stream=sys.stdout, default_flow_style=False, blame=blame)
except (yaml.YAMLError, IOError):
raise ConfigError("Error reading configuration: %s" % section)
def upgrade_None_to_1_0(new, old, depth=[0]):
depth[0] += 1
if depth[0] > 1:
raise ValueError("Recursion!")
version_string = ".".join(str(_) for _ in new.version)
tty.info(
"Converting Modulecmd.py collections version 0.0 to "
"version {0}".format(version_string)
)
new_collections = {}
for (name, old_collection) in old.items():
new_collection = OrderedDict()
mp = pymod.modulepath.Modulepath([])
for (path, m_descs) in old_collection:
if new_collection is None:
break
if not os.path.isdir(path):
tty.warn(
"Collection {0} contains directory {1} which "
"does not exist! This collection will be skipped".format(
name, path
)
)
new_collection = None
break
avail = mp.append_path(path)
if avail is None:
tty.warn(
"Collection {0} contains directory {1} which "
"does not have any available modules! "
"This collection will be skipped".format(name, path)
)
new_collection = None
break
for (fullname, filename, opts) in m_descs:
m = mp.get(filename)
if m is None:
tty.warn(
"Collection {0} requests module {1} which "
"can not be found! This collection will be skipped".format(
name, fullname
)
)
new_collection = None
break
m.opts = opts
m.acquired_as = m.fullname
ar = pymod.mc.archive_module(m)
new_collection.setdefault(m.modulepath, []).append(ar)
if new_collection is None:
tty.warn(
"Skipping collection {0} because of previous " "errors".format(name)
)
continue
new_collections[name] = list(new_collection.items())
bak = new.filename + ".bak"
with open(bak, "w") as fh:
json.dump(old, fh, indent=2)
new.write(list(new_collections.items()), new.filename)
return new_collections
class MirrorCollection(Mapping):
"""A mapping of mirror names to mirrors."""
def __init__(self, mirrors=None, scope=None):
self._mirrors = OrderedDict(
(name, Mirror.from_dict(mirror, name)) for name, mirror in (
mirrors.items() if mirrors is not None else spack.config.
get('mirrors', scope=scope).items()))
def to_json(self, stream=None):
return sjson.dump(self.to_dict(True), stream)
def to_yaml(self, stream=None):
return syaml.dump(self.to_dict(True), stream)
# TODO: this isn't called anywhere
@staticmethod
def from_yaml(stream, name=None):
try:
data = syaml.load(stream)
return MirrorCollection(data)
except yaml_error.MarkedYAMLError as e:
raise syaml.SpackYAMLError("error parsing YAML spec:", str(e))
@staticmethod
def from_json(stream, name=None):
d = sjson.load(stream)
return MirrorCollection(d)
def to_dict(self, recursive=False):
return syaml_dict(
sorted(((k, (v.to_dict() if recursive else v))
for (k, v) in self._mirrors.items()),
key=operator.itemgetter(0)))
@staticmethod
def from_dict(d):
return MirrorCollection(d)
def __getitem__(self, item):
return self._mirrors[item]
def display(self):
max_len = max(len(mirror.name) for mirror in self._mirrors.values())
for mirror in self._mirrors.values():
mirror.display(max_len)
def lookup(self, name_or_url):
"""Looks up and returns a Mirror.
If this MirrorCollection contains a named Mirror under the name
[name_or_url], then that mirror is returned. Otherwise, [name_or_url]
is assumed to be a mirror URL, and an anonymous mirror with the given
URL is returned.
"""
result = self.get(name_or_url)
if result is None:
result = Mirror(fetch_url=name_or_url)
return result
def __iter__(self):
return iter(self._mirrors)
def __len__(self):
return len(self._mirrors)
def __init__(self, mirrors=None, scope=None):
self._mirrors = OrderedDict(
(name, Mirror.from_dict(mirror, name)) for name, mirror in (
mirrors.items() if mirrors is not None else spack.config.
get('mirrors', scope=scope).items()))
