def _add_single_spec(spec, mirror, mirror_stats):
tty.msg("Adding package {pkg} to mirror".format(
pkg=spec.format("{name}{@version}")))
num_retries = 3
while num_retries > 0:
try:
with spec.package.stage as pkg_stage:
pkg_stage.cache_mirror(mirror, mirror_stats)
for patch in spec.package.all_patches():
if patch.stage:
patch.stage.cache_mirror(mirror, mirror_stats)
patch.clean()
exception = None
break
except Exception as e:
exc_tuple = sys.exc_info()
exception = e
num_retries -= 1
if exception:
if spack.config.get('config:debug'):
traceback.print_exception(file=sys.stderr, *exc_tuple)
else:
tty.warn(
"Error while fetching %s" % spec.cformat('{name}{@version}'),
getattr(exception, 'message', exception))
mirror_stats.error()
def _assign_dependencies(self, hash_key, installs, data):
# Add dependencies from other records in the install DB to
# form a full spec.
spec = data[hash_key].spec
spec_dict = installs[hash_key]['spec']
if 'dependencies' in spec_dict[spec.name]:
yaml_deps = spec_dict[spec.name]['dependencies']
for dname, dhash, dtypes in spack.spec.Spec.read_yaml_dep_specs(
yaml_deps):
# It is important that we always check upstream installations
# in the same order, and that we always check the local
# installation first: if a downstream Spack installs a package
# then dependents in that installation could be using it.
# If a hash is installed locally and upstream, there isn't
# enough information to determine which one a local package
# depends on, so the convention ensures that this isn't an
# issue.
upstream, record = self.query_by_spec_hash(dhash, data=data)
child = record.spec if record else None
if not child:
msg = ("Missing dependency not in database: "
"%s needs %s-%s" %
(spec.cformat('{name}{/hash:7}'), dname, dhash[:7]))
if self._fail_when_missing_deps:
raise MissingDependenciesError(msg)
tty.warn(msg)
continue
spec._add_dependency(child, dtypes)
def _assign_dependencies(self, hash_key, installs, data):
# Add dependencies from other records in the install DB to
# form a full spec.
spec = data[hash_key].spec
spec_dict = installs[hash_key]['spec']
if 'dependencies' in spec_dict[spec.name]:
yaml_deps = spec_dict[spec.name]['dependencies']
for dname, dhash, dtypes in spack.spec.Spec.read_yaml_dep_specs(
yaml_deps):
if dhash not in data:
tty.warn("Missing dependency not in database: ",
"%s needs %s-%s" % (
spec.cformat('$_$/'), dname, dhash[:7]))
continue
child = data[dhash].spec
spec._add_dependency(child, dtypes)
def _assign_dependencies(self, hash_key, installs, data):
# Add dependencies from other records in the install DB to
# form a full spec.
spec = data[hash_key].spec
spec_dict = installs[hash_key]['spec']
if 'dependencies' in spec_dict[spec.name]:
yaml_deps = spec_dict[spec.name]['dependencies']
for dname, dhash, dtypes in spack.spec.Spec.read_yaml_dep_specs(
yaml_deps):
if dhash not in data:
tty.warn("Missing dependency not in database: ",
"%s needs %s-%s" % (
spec.cformat('$_$/'), dname, dhash[:7]))
continue
child = data[dhash].spec
spec._add_dependency(child, dtypes)
def display_specs(specs, args=None, **kwargs):
"""Display human readable specs with customizable formatting.
Prints the supplied specs to the screen, formatted according to the
arguments provided.
Specs are grouped by architecture and compiler, and columnized if
possible. There are three possible "modes":
* ``short`` (default): short specs with name and version, columnized
* ``paths``: Two columns: one for specs, one for paths
* ``deps``: Dependency-tree style, like ``spack spec``; can get long
Options can add more information to the default display. Options can
be provided either as keyword arguments or as an argparse namespace.
Keyword arguments take precedence over settings in the argparse
namespace.
Args:
specs (list of spack.spec.Spec): the specs to display
args (optional argparse.Namespace): namespace containing
formatting arguments
Keyword Args:
mode (str): Either 'short', 'paths', or 'deps'
long (bool): Display short hashes with specs
very_long (bool): Display full hashes with specs (supersedes ``long``)
namespace (bool): Print namespaces along with names
show_flags (bool): Show compiler flags with specs
variants (bool): Show variants with specs
indent (int): indent each line this much
decorators (dict): dictionary mappng specs to decorators
header_callback (function): called at start of arch/compiler sections
all_headers (bool): show headers even when arch/compiler aren't defined
"""
def get_arg(name, default=None):
"""Prefer kwargs, then args, then default."""
if name in kwargs:
return kwargs.get(name)
elif args is not None:
return getattr(args, name, default)
else:
return default
mode = get_arg('mode', 'short')
hashes = get_arg('long', False)
namespace = get_arg('namespace', False)
flags = get_arg('show_flags', False)
full_compiler = get_arg('show_full_compiler', False)
variants = get_arg('variants', False)
all_headers = get_arg('all_headers', False)
decorator = get_arg('decorator', None)
if decorator is None:
decorator = lambda s, f: f
indent = get_arg('indent', 0)
ispace = indent * ' '
hlen = 7
if get_arg('very_long', False):
hashes = True
hlen = None
nfmt = '{fullpackage}' if namespace else '{package}'
ffmt = ''
if full_compiler or flags:
ffmt += '$%'
if full_compiler:
ffmt += '@'
ffmt += '+'
vfmt = '$+' if variants else ''
format_string = '$%s$@%s%s' % (nfmt, ffmt, vfmt)
# Make a dict with specs keyed by architecture and compiler.
index = index_by(specs, ('architecture', 'compiler'))
transform = {'package': decorator, 'fullpackage': decorator}
# Traverse the index and print out each package
for i, (architecture, compiler) in enumerate(sorted(index)):
if i > 0:
print()
header = "%s{%s} / %s{%s}" % (spack.spec.architecture_color,
architecture if architecture else
'no arch', spack.spec.compiler_color,
compiler if compiler else 'no compiler')
# Sometimes we want to display specs that are not yet concretized.
# If they don't have a compiler / architecture attached to them,
# then skip the header
if all_headers or (architecture is not None or compiler is not None):
sys.stdout.write(ispace)
tty.hline(colorize(header), char='-')
specs = index[(architecture, compiler)]
specs.sort()
if mode == 'paths':
# Print one spec per line along with prefix path
abbreviated = [
s.cformat(format_string, transform=transform) for s in specs
]
width = max(len(s) for s in abbreviated)
width += 2
for abbrv, spec in zip(abbreviated, specs):
# optional hash prefix for paths
h = gray_hash(spec, hlen) if hashes else ''
# only show prefix for concrete specs
prefix = spec.prefix if spec.concrete else ''
# print it all out at once
fmt = "%%s%%s %%-%ds%%s" % width
print(fmt % (ispace, h, abbrv, prefix))
elif mode == 'deps':
for spec in specs:
print(
spec.tree(format=format_string,
indent=4,
prefix=(lambda s: gray_hash(s, hlen))
if hashes else None))
elif mode == 'short':
def fmt(s):
string = ""
if hashes:
string += gray_hash(s, hlen) + ' '
string += s.cformat('$%s$@%s' % (nfmt, vfmt),
transform=transform)
return string
if not flags and not full_compiler:
# Print columns of output if not printing flags
colify((fmt(s) for s in specs), indent=indent)
else:
# Print one entry per line if including flags
for spec in specs:
# Print the hash if necessary
hsh = gray_hash(spec, hlen) + ' ' if hashes else ''
print(ispace + hsh +
spec.cformat(format_string, transform=transform))
else:
raise ValueError(
"Invalid mode for display_specs: %s. Must be one of (paths,"
"deps, short)." % mode)
def display_specs(specs, args=None, **kwargs):
"""Display human readable specs with customizable formatting.
Prints the supplied specs to the screen, formatted according to the
arguments provided.
Specs are grouped by architecture and compiler, and columnized if
possible. There are three possible "modes":
* ``short`` (default): short specs with name and version, columnized
* ``paths``: Two columns: one for specs, one for paths
* ``deps``: Dependency-tree style, like ``spack spec``; can get long
Options can add more information to the default display. Options can
be provided either as keyword arguments or as an argparse namespace.
Keyword arguments take precedence over settings in the argparse
namespace.
Args:
specs (list of spack.spec.Spec): the specs to display
args (optional argparse.Namespace): namespace containing
formatting arguments
Keyword Args:
mode (str): Either 'short', 'paths', or 'deps'
long (bool): Display short hashes with specs
very_long (bool): Display full hashes with specs (supersedes ``long``)
namespace (bool): Print namespaces along with names
show_flags (bool): Show compiler flags with specs
variants (bool): Show variants with specs
"""
def get_arg(name, default=None):
"""Prefer kwargs, then args, then default."""
if name in kwargs:
return kwargs.get(name)
elif args is not None:
return getattr(args, name, default)
else:
return default
mode = get_arg('mode', 'short')
hashes = get_arg('long', False)
namespace = get_arg('namespace', False)
flags = get_arg('show_flags', False)
full_compiler = get_arg('show_full_compiler', False)
variants = get_arg('variants', False)
hlen = 7
if get_arg('very_long', False):
hashes = True
hlen = None
nfmt = '.' if namespace else '_'
ffmt = ''
if full_compiler or flags:
ffmt += '$%'
if full_compiler:
ffmt += '@'
ffmt += '+'
vfmt = '$+' if variants else ''
format_string = '$%s$@%s%s' % (nfmt, ffmt, vfmt)
# Make a dict with specs keyed by architecture and compiler.
index = index_by(specs, ('architecture', 'compiler'))
# Traverse the index and print out each package
for i, (architecture, compiler) in enumerate(sorted(index)):
if i > 0:
print()
header = "%s{%s} / %s{%s}" % (spack.spec.architecture_color,
architecture, spack.spec.compiler_color,
compiler)
# Sometimes we want to display specs that are not yet concretized.
# If they don't have a compiler / architecture attached to them,
# then skip the header
if architecture is not None or compiler is not None:
tty.hline(colorize(header), char='-')
specs = index[(architecture, compiler)]
specs.sort()
abbreviated = [s.cformat(format_string) for s in specs]
if mode == 'paths':
# Print one spec per line along with prefix path
width = max(len(s) for s in abbreviated)
width += 2
format = " %%-%ds%%s" % width
for abbrv, spec in zip(abbreviated, specs):
prefix = gray_hash(spec, hlen) if hashes else ''
print(prefix + (format % (abbrv, spec.prefix)))
elif mode == 'deps':
for spec in specs:
print(spec.tree(
format=format_string,
indent=4,
prefix=(lambda s: gray_hash(s, hlen)) if hashes else None))
elif mode == 'short':
# Print columns of output if not printing flags
if not flags and not full_compiler:
def fmt(s):
string = ""
if hashes:
string += gray_hash(s, hlen) + ' '
string += s.cformat('$-%s$@%s' % (nfmt, vfmt))
return string
colify(fmt(s) for s in specs)
# Print one entry per line if including flags
else:
for spec in specs:
# Print the hash if necessary
hsh = gray_hash(spec, hlen) + ' ' if hashes else ''
print(hsh + spec.cformat(format_string) + '\n')
else:
raise ValueError(
"Invalid mode for display_specs: %s. Must be one of (paths,"
"deps, short)." % mode)
def display_specs(specs, args=None, **kwargs):
"""Display human readable specs with customizable formatting.
Prints the supplied specs to the screen, formatted according to the
arguments provided.
Specs are grouped by architecture and compiler, and columnized if
possible. There are three possible "modes":
* ``short`` (default): short specs with name and version, columnized
* ``paths``: Two columns: one for specs, one for paths
* ``deps``: Dependency-tree style, like ``spack spec``; can get long
Options can add more information to the default display. Options can
be provided either as keyword arguments or as an argparse namespace.
Keyword arguments take precedence over settings in the argparse
namespace.
Args:
specs (list of spack.spec.Spec): the specs to display
args (optional argparse.Namespace): namespace containing
formatting arguments
Keyword Args:
mode (str): Either 'short', 'paths', or 'deps'
long (bool): Display short hashes with specs
very_long (bool): Display full hashes with specs (supersedes ``long``)
namespace (bool): Print namespaces along with names
show_flags (bool): Show compiler flags with specs
variants (bool): Show variants with specs
"""
def get_arg(name, default=None):
"""Prefer kwargs, then args, then default."""
if name in kwargs:
return kwargs.get(name)
elif args is not None:
return getattr(args, name, default)
else:
return default
mode = get_arg('mode', 'short')
hashes = get_arg('long', False)
namespace = get_arg('namespace', False)
flags = get_arg('show_flags', False)
full_compiler = get_arg('show_full_compiler', False)
variants = get_arg('variants', False)
hlen = 7
if get_arg('very_long', False):
hashes = True
hlen = None
nfmt = '.' if namespace else '_'
ffmt = ''
if full_compiler or flags:
ffmt += '$%'
if full_compiler:
ffmt += '@'
ffmt += '+'
vfmt = '$+' if variants else ''
format_string = '$%s$@%s%s' % (nfmt, ffmt, vfmt)
# Make a dict with specs keyed by architecture and compiler.
index = index_by(specs, ('architecture', 'compiler'))
# Traverse the index and print out each package
for i, (architecture, compiler) in enumerate(sorted(index)):
if i > 0:
print()
header = "%s{%s} / %s{%s}" % (spack.spec.architecture_color,
architecture, spack.spec.compiler_color,
compiler)
tty.hline(colorize(header), char='-')
specs = index[(architecture, compiler)]
specs.sort()
abbreviated = [s.cformat(format_string) for s in specs]
if mode == 'paths':
# Print one spec per line along with prefix path
width = max(len(s) for s in abbreviated)
width += 2
format = " %%-%ds%%s" % width
for abbrv, spec in zip(abbreviated, specs):
prefix = gray_hash(spec, hlen) if hashes else ''
print(prefix + (format % (abbrv, spec.prefix)))
elif mode == 'deps':
for spec in specs:
print(
spec.tree(format=format_string,
indent=4,
prefix=(lambda s: gray_hash(s, hlen))
if hashes else None))
elif mode == 'short':
# Print columns of output if not printing flags
if not flags and not full_compiler:
def fmt(s):
string = ""
if hashes:
string += gray_hash(s, hlen) + ' '
string += s.cformat('$-%s$@%s' % (nfmt, vfmt))
return string
colify(fmt(s) for s in specs)
# Print one entry per line if including flags
else:
for spec in specs:
# Print the hash if necessary
hsh = gray_hash(spec, hlen) + ' ' if hashes else ''
print(hsh + spec.cformat(format_string) + '\n')
else:
raise ValueError(
"Invalid mode for display_specs: %s. Must be one of (paths,"
"deps, short)." % mode)
