def install_from_queue(work_queue, installation_result_queue, kwargs):
while True:
try:
jobs, serialized_spec = work_queue.get(block=True)
spec = Spec.from_yaml(serialized_spec).concretized()
if jobs:
kwargs['make_jobs'] = jobs
if kwargs['make_jobs'] is None:
kwargs['make_jobs'] = get_cpu_count()
kwargs['install_deps'] = False
tty.msg('Installing %s with %s jobs' %
(spec.name, kwargs['make_jobs']))
with tty.SuppressOutput(msg_enabled=False,
warn_enabled=False,
error_enabled=False):
spec.package.do_install(**kwargs)
installation_result_queue.put_nowait((None, serialized_spec))
except Exception as e:
tty.error(e)
traceback.print_exc()
installation_result_queue.put_nowait(('ERROR', serialized_spec))
def get_concrete_specs(root_spec, job_name, related_builds, compiler_action):
spec_map = {
'root': None,
'deps': {},
}
if compiler_action == 'FIND_ANY':
# This corresponds to a bootstrapping phase where we need to
# rely on any available compiler to build the package (i.e. the
# compiler needed to be stripped from the spec when we generated
# the job), and thus we need to concretize the root spec again.
tty.debug('About to concretize {0}'.format(root_spec))
concrete_root = Spec(root_spec).concretized()
tty.debug('Resulting concrete root: {0}'.format(concrete_root))
else:
# in this case, either we're relying on Spack to install missing
# compiler bootstrapped in a previous phase, or else we only had one
# phase (like a site which already knows what compilers are available
# on it's runners), so we don't want to concretize that root spec
# again. The reason we take this path in the first case (bootstrapped
# compiler), is that we can't concretize a spec at this point if we're
# going to ask spack to "install_missing_compilers".
concrete_root = Spec.from_yaml(
str(zlib.decompress(base64.b64decode(root_spec)).decode('utf-8')))
spec_map['root'] = concrete_root
spec_map[job_name] = concrete_root[job_name]
if related_builds:
for dep_job_name in related_builds.split(';'):
spec_map['deps'][dep_job_name] = concrete_root[dep_job_name]
return spec_map
def read_package_info(want_specs=True):
dev.environment.bootstrap_environment()
packages_filename = os.path.join(os.environ['SPACKDEV_BASE'],
dev.spackdev_aux_packages_sd_file)
with open(packages_filename, 'r') as f:
first_line = f.readline().rstrip()
if first_line.find('[') > -1:
tty.die(
'packages.sd in obsolete (unsafe) format: please re-execute spack init or initialize a new spackdev area.'
)
requesteds = [
DevPackageInfo(package_arg) for package_arg in first_line.split()
]
additional = [
DevPackageInfo(package_arg)
for package_arg in f.readline().rstrip().split()
]
deps = f.readline().rstrip().split()
install_specs = []
if want_specs:
specs_dir = os.path.join(os.environ['SPACKDEV_BASE'],
dev.spackdev_aux_specs_subdir)
if not os.path.exists(specs_dir):
tty.die(
'YAML spec information missing: please re-execute spack init or initialize a new spackdev area.'
)
for spec_file in os.listdir(specs_dir):
if spec_file.endswith('.yaml'):
with open(os.path.join(specs_dir, spec_file), 'r') as f:
install_specs.append(Spec.from_yaml(f))
return requesteds, additional, deps, install_specs
return requesteds, additional, deps
def test_yaml_subdag(self):
spec = Spec('mpileaks^mpich+debug')
spec.concretize()
yaml_spec = Spec.from_yaml(spec.to_yaml())
for dep in ('callpath', 'mpich', 'dyninst', 'libdwarf', 'libelf'):
self.assertTrue(spec[dep].eq_dag(yaml_spec[dep]))
def test_yaml_subdag(config, mock_packages):
spec = Spec('mpileaks^mpich+debug')
spec.concretize()
yaml_spec = Spec.from_yaml(spec.to_yaml())
for dep in ('callpath', 'mpich', 'dyninst', 'libdwarf', 'libelf'):
assert spec[dep].eq_dag(yaml_spec[dep])
def test_yaml_subdag(config, builtin_mock):
spec = Spec('mpileaks^mpich+debug')
spec.concretize()
yaml_spec = Spec.from_yaml(spec.to_yaml())
for dep in ('callpath', 'mpich', 'dyninst', 'libdwarf', 'libelf'):
assert spec[dep].eq_dag(yaml_spec[dep])
def try_download_specs(urls=None, force=False):
'''
Try to download the urls and cache them
'''
global _cached_specs
if urls is None:
return {}
for link in urls:
with Stage(link, name="build_cache", keep=True) as stage:
if force and os.path.exists(stage.save_filename):
os.remove(stage.save_filename)
if not os.path.exists(stage.save_filename):
try:
stage.fetch()
except fs.FetchError:
continue
with open(stage.save_filename, 'r') as f:
# read the spec from the build cache file. All specs
# in build caches are concrete (as they are built) so
# we need to mark this spec concrete on read-in.
spec = Spec.from_yaml(f)
spec._mark_concrete()
_cached_specs.add(spec)
return _cached_specs
def createtarball(args):
"""create a binary package from an existing install"""
if args.spec_yaml:
packages = set()
tty.msg('createtarball, reading spec from {0}'.format(args.spec_yaml))
with open(args.spec_yaml, 'r') as fd:
yaml_text = fd.read()
tty.debug('createtarball read spec yaml:')
tty.debug(yaml_text)
s = Spec.from_yaml(yaml_text)
packages.add('/{0}'.format(s.dag_hash()))
elif args.packages:
packages = args.packages
else:
tty.die("build cache file creation requires at least one" +
" installed package argument or else path to a" +
" yaml file containing a spec to install")
pkgs = set(packages)
specs = set()
outdir = '.'
if args.directory:
outdir = args.directory
signkey = None
if args.key:
signkey = args.key
# restrict matching to current environment if one is active
env = ev.get_env(args, 'buildcache create')
matches = find_matching_specs(pkgs, env=env)
if matches:
tty.msg('Found at least one matching spec')
for match in matches:
tty.msg('examining match {0}'.format(match.format()))
if match.external or match.virtual:
tty.msg('skipping external or virtual spec %s' % match.format())
else:
tty.msg('adding matching spec %s' % match.format())
specs.add(match)
tty.msg('recursing dependencies')
for d, node in match.traverse(order='post',
depth=True,
deptype=('link', 'run')):
if node.external or node.virtual:
tty.msg('skipping external or virtual dependency %s' %
node.format())
else:
tty.msg('adding dependency %s' % node.format())
specs.add(node)
tty.msg('writing tarballs to %s/build_cache' % outdir)
for spec in specs:
tty.msg('creating binary cache file for package %s ' % spec.format())
bindist.build_tarball(spec, outdir, args.force, args.rel,
args.unsigned, args.allow_root, signkey,
not args.no_rebuild_index)
def test_legacy_yaml(tmpdir, install_mockery, mock_packages):
"""Tests a simple legacy YAML with a dependency and ensures spec survives
concretization."""
yaml = """
spec:
- a:
version: '2.0'
arch:
platform: linux
platform_os: rhel7
target: x86_64
compiler:
name: gcc
version: 8.3.0
namespace: builtin.mock
parameters:
bvv: true
foo:
- bar
foobar: bar
cflags: []
cppflags: []
cxxflags: []
fflags: []
ldflags: []
ldlibs: []
dependencies:
b:
hash: iaapywazxgetn6gfv2cfba353qzzqvhn
type:
- build
- link
hash: obokmcsn3hljztrmctbscmqjs3xclazz
full_hash: avrk2tqsnzxeabmxa6r776uq7qbpeufv
build_hash: obokmcsn3hljztrmctbscmqjs3xclazy
- b:
version: '1.0'
arch:
platform: linux
platform_os: rhel7
target: x86_64
compiler:
name: gcc
version: 8.3.0
namespace: builtin.mock
parameters:
cflags: []
cppflags: []
cxxflags: []
fflags: []
ldflags: []
ldlibs: []
hash: iaapywazxgetn6gfv2cfba353qzzqvhn
full_hash: qvsxvlmjaothtpjluqijv7qfnni3kyyg
build_hash: iaapywazxgetn6gfv2cfba353qzzqvhy
"""
spec = Spec.from_yaml(yaml)
concrete_spec = spec.concretized()
assert concrete_spec.eq_dag(spec)
def read_spec(self, path):
"""Read the contents of a file and parse them as a spec"""
with open(path) as f:
spec = Spec.from_yaml(f)
# Specs read from actual installations are always concrete
spec._mark_concrete()
return spec
def test_yaml_subdag(self):
spec = Spec('mpileaks^mpich+debug')
spec.concretize()
yaml_spec = Spec.from_yaml(spec.to_yaml())
for dep in ('callpath', 'mpich', 'dyninst', 'libdwarf', 'libelf'):
self.assertTrue(spec[dep].eq_dag(yaml_spec[dep]))
def read_spec(self, path):
"""Read the contents of a file and parse them as a spec"""
with open(path) as f:
spec = Spec.from_yaml(f)
# Specs read from actual installations are always concrete
spec._mark_concrete()
return spec
def test_read_and_write_spec(self):
"""This goes through each package in spack and creates a directory for
it. It then ensures that the spec for the directory's
installed package can be read back in consistently, and
finally that the directory can be removed by the directory
layout.
"""
packages = list(spack.repo.all_packages())[:max_packages]
for pkg in packages:
if pkg.name.startswith('external'):
#External package tests cannot be installed
continue
spec = pkg.spec
# If a spec fails to concretize, just skip it. If it is a
# real error, it will be caught by concretization tests.
try:
spec.concretize()
except:
continue
self.layout.create_install_directory(spec)
install_dir = self.layout.path_for_spec(spec)
spec_path = self.layout.spec_file_path(spec)
# Ensure directory has been created in right place.
self.assertTrue(os.path.isdir(install_dir))
self.assertTrue(install_dir.startswith(self.tmpdir))
# Ensure spec file exists when directory is created
self.assertTrue(os.path.isfile(spec_path))
self.assertTrue(spec_path.startswith(install_dir))
# Make sure spec file can be read back in to get the original spec
spec_from_file = self.layout.read_spec(spec_path)
self.assertEqual(spec, spec_from_file)
self.assertTrue(spec.eq_dag, spec_from_file)
self.assertTrue(spec_from_file.concrete)
# Ensure that specs that come out "normal" are really normal.
with open(spec_path) as spec_file:
read_separately = Spec.from_yaml(spec_file.read())
read_separately.normalize()
self.assertEqual(read_separately, spec_from_file)
read_separately.concretize()
self.assertEqual(read_separately, spec_from_file)
# Make sure the hash of the read-in spec is the same
self.assertEqual(spec.dag_hash(), spec_from_file.dag_hash())
# Ensure directories are properly removed
self.layout.remove_install_directory(spec)
self.assertFalse(os.path.isdir(install_dir))
self.assertFalse(os.path.exists(install_dir))
def get_specs(force=False):
"""
Get spec.yaml's for build caches available on mirror
"""
global _cached_specs
if _cached_specs:
tty.debug("Using previously-retrieved specs")
return _cached_specs
if not spack.mirror.MirrorCollection():
tty.warn("No Spack mirrors are currently configured")
return {}
urls = set()
for mirror in spack.mirror.MirrorCollection().values():
fetch_url_build_cache = url_util.join(mirror.fetch_url,
_build_cache_relative_path)
mirror_dir = url_util.local_file_path(fetch_url_build_cache)
if mirror_dir:
tty.msg("Finding buildcaches in %s" % mirror_dir)
if os.path.exists(mirror_dir):
files = os.listdir(mirror_dir)
for file in files:
if re.search('spec.yaml', file):
link = url_util.join(fetch_url_build_cache, file)
urls.add(link)
else:
tty.msg("Finding buildcaches at %s" %
url_util.format(fetch_url_build_cache))
p, links = web_util.spider(
url_util.join(fetch_url_build_cache, 'index.html'))
for link in links:
if re.search("spec.yaml", link):
urls.add(link)
_cached_specs = []
for link in urls:
with Stage(link, name="build_cache", keep=True) as stage:
if force and os.path.exists(stage.save_filename):
os.remove(stage.save_filename)
if not os.path.exists(stage.save_filename):
try:
stage.fetch()
except fs.FetchError:
continue
with open(stage.save_filename, 'r') as f:
# read the spec from the build cache file. All specs
# in build caches are concrete (as they are built) so
# we need to mark this spec concrete on read-in.
spec = Spec.from_yaml(f)
spec._mark_concrete()
_cached_specs.append(spec)
return _cached_specs
def _concretize_spec(ser_spec):
try:
spec = Spec.from_yaml(ser_spec)
try:
return ser_spec, spec.concretized().to_yaml(all_deps=True)
except Exception as e:
tty.warn('Could not concretize %s: %s' % (spec.name, e))
except Exception as e:
tty.warn('Could not deserialize spec %s' % e.message)
return None
def get_specs(force=False):
"""
Get spec.yaml's for build caches available on mirror
"""
global _cached_specs
if _cached_specs:
tty.debug("Using previously-retrieved specs")
return _cached_specs
mirrors = spack.config.get('mirrors')
if len(mirrors) == 0:
tty.warn("No Spack mirrors are currently configured")
return {}
path = str(spack.architecture.sys_type())
urls = set()
for mirror_name, mirror_url in mirrors.items():
if mirror_url.startswith('file'):
mirror = mirror_url.replace(
'file://', '') + "/" + _build_cache_relative_path
tty.msg("Finding buildcaches in %s" % mirror)
if os.path.exists(mirror):
files = os.listdir(mirror)
for file in files:
if re.search('spec.yaml', file):
link = 'file://' + mirror + '/' + file
urls.add(link)
else:
tty.msg("Finding buildcaches on %s" % mirror_url)
p, links = spider(mirror_url + "/" + _build_cache_relative_path)
for link in links:
if re.search("spec.yaml", link) and re.search(path, link):
urls.add(link)
_cached_specs = []
for link in urls:
with Stage(link, name="build_cache", keep=True) as stage:
if force and os.path.exists(stage.save_filename):
os.remove(stage.save_filename)
if not os.path.exists(stage.save_filename):
try:
stage.fetch()
except fs.FetchError:
continue
with open(stage.save_filename, 'r') as f:
# read the spec from the build cache file. All specs
# in build caches are concrete (as they are built) so
# we need to mark this spec concrete on read-in.
spec = Spec.from_yaml(f)
spec._mark_concrete()
_cached_specs.append(spec)
return _cached_specs
def concrete_specs_gen(self, specs, print_time=False):
"""Concretizes specs across a pool of processes
Returns:
A generator that yields a tuple with the original spec and a
concretized spec. Output is not necessarily ordered.
"""
start_time = time.time()
abs_list, conc_list = [], []
for s in specs:
conc_list.append(s) if s.concrete else abs_list.append(s)
# Note: Json serialization does not support 'all_deps'
yaml_specs = [s.to_yaml(all_deps=True) for s in abs_list]
# Spin off work to the pool
conc_spec_gen = self.pool.imap_unordered(
ParallelConcretizer._concretize_spec, yaml_specs)
# Hand back any specs that are already concrete while the pool works
for concrete_spec in conc_list:
yield (concrete_spec, concrete_spec)
# Yield specs as the pool concretizes them
for spec_yaml_tuple in conc_spec_gen:
if spec_yaml_tuple is None:
if not self.ignore_error:
raise Exception("Parallel concretization Failed!")
else:
orig_spec_yaml, conc_spec_yaml = spec_yaml_tuple
yield (Spec.from_yaml(orig_spec_yaml),
Spec.from_yaml(conc_spec_yaml))
if print_time:
tot_time = time.time() - start_time
spec_per_second = len(specs) / tot_time
tty.msg(
'Added %s specs in %s seconds (%s Spec/s)' %
(len(specs), round(tot_time, 2), round(spec_per_second, 2)))
def generate_package_index(cache_prefix):
"""Create the build cache index page.
Creates (or replaces) the "index.json" page at the location given in
cache_prefix. This page contains a link for each binary package (*.yaml)
and public key (*.key) under cache_prefix.
"""
tmpdir = tempfile.mkdtemp()
db_root_dir = os.path.join(tmpdir, 'db_root')
db = spack_db.Database(None, db_dir=db_root_dir,
enable_transaction_locking=False,
record_fields=['spec', 'ref_count'])
file_list = (
entry
for entry in web_util.list_url(cache_prefix)
if entry.endswith('.yaml'))
tty.debug('Retrieving spec.yaml files from {0} to build index'.format(
cache_prefix))
for file_path in file_list:
try:
yaml_url = url_util.join(cache_prefix, file_path)
tty.debug('fetching {0}'.format(yaml_url))
_, _, yaml_file = web_util.read_from_url(yaml_url)
yaml_contents = codecs.getreader('utf-8')(yaml_file).read()
# yaml_obj = syaml.load(yaml_contents)
# s = Spec.from_yaml(yaml_obj)
s = Spec.from_yaml(yaml_contents)
db.add(s, None)
except (URLError, web_util.SpackWebError) as url_err:
tty.error('Error reading spec.yaml: {0}'.format(file_path))
tty.error(url_err)
try:
index_json_path = os.path.join(db_root_dir, 'index.json')
with open(index_json_path, 'w') as f:
db._write_to_file(f)
web_util.push_to_url(
index_json_path,
url_util.join(cache_prefix, 'index.json'),
keep_original=False,
extra_args={'ContentType': 'application/json'})
finally:
shutil.rmtree(tmpdir)
def get_concrete_spec(args):
spec_str = args.spec
spec_yaml_path = args.spec_yaml
if not spec_str and not spec_yaml_path:
tty.msg('Must provide either spec string or path to ' +
'yaml to concretize spec')
sys.exit(1)
if spec_str:
try:
spec = find_matching_specs(spec_str)[0]
spec.concretize()
except SpecError as spec_error:
tty.error('Unable to concrectize spec {0}'.format(args.spec))
tty.debug(spec_error)
sys.exit(1)
return spec
with open(spec_yaml_path, 'r') as fd:
return Spec.from_yaml(fd.read())
def copy_fn(args):
"""Copy a buildcache entry and all its files from one mirror, given as
'--base-dir', to some other mirror, specified as '--destination-url'.
The specific buildcache entry to be copied from one location to the
other is identified using the '--spec-file' argument."""
# TODO: Remove after v0.18.0 release
msg = ('"spack buildcache copy" is deprecated and will be removed from '
'Spack starting in v0.19.0')
warnings.warn(msg)
if not args.spec_file:
tty.msg('No spec yaml provided, exiting.')
sys.exit(1)
if not args.base_dir:
tty.msg('No base directory provided, exiting.')
sys.exit(1)
if not args.destination_url:
tty.msg('No destination mirror url provided, exiting.')
sys.exit(1)
dest_url = args.destination_url
if dest_url[0:7] != 'file://' and dest_url[0] != '/':
tty.msg('Only urls beginning with "file://" or "/" are supported ' +
'by buildcache copy.')
sys.exit(1)
try:
with open(args.spec_file, 'r') as fd:
spec = Spec.from_yaml(fd.read())
except Exception as e:
tty.debug(e)
tty.error('Unable to concrectize spec from yaml {0}'.format(
args.spec_file))
sys.exit(1)
dest_root_path = dest_url
if dest_url[0:7] == 'file://':
dest_root_path = dest_url[7:]
build_cache_dir = bindist.build_cache_relative_path()
tarball_rel_path = os.path.join(build_cache_dir,
bindist.tarball_path_name(spec, '.spack'))
tarball_src_path = os.path.join(args.base_dir, tarball_rel_path)
tarball_dest_path = os.path.join(dest_root_path, tarball_rel_path)
specfile_rel_path = os.path.join(build_cache_dir,
bindist.tarball_name(spec, '.spec.json'))
specfile_src_path = os.path.join(args.base_dir, specfile_rel_path)
specfile_dest_path = os.path.join(dest_root_path, specfile_rel_path)
specfile_rel_path_yaml = os.path.join(
build_cache_dir, bindist.tarball_name(spec, '.spec.yaml'))
specfile_src_path_yaml = os.path.join(args.base_dir, specfile_rel_path)
specfile_dest_path_yaml = os.path.join(dest_root_path, specfile_rel_path)
# Make sure directory structure exists before attempting to copy
os.makedirs(os.path.dirname(tarball_dest_path))
# Now copy the specfile and tarball files to the destination mirror
tty.msg('Copying {0}'.format(tarball_rel_path))
shutil.copyfile(tarball_src_path, tarball_dest_path)
tty.msg('Copying {0}'.format(specfile_rel_path))
shutil.copyfile(specfile_src_path, specfile_dest_path)
tty.msg('Copying {0}'.format(specfile_rel_path_yaml))
shutil.copyfile(specfile_src_path_yaml, specfile_dest_path_yaml)
def install_dag(self, dag_scheduler, kwargs):
"""Installs a list of specs"""
start_time = time.time()
# Initialize structures relating to the process pool
outstanding_installs = {}
work_queue = Manager().Queue()
installation_result_queue = Manager().Queue()
try:
with NonDaemonPool(processes=get_cpu_count()) as pool:
# Create a process for each core, initialize it with async
# queue for sending specs/receiving results
for _ in range(get_cpu_count()):
pool.apply_async(
install_from_queue,
(work_queue, installation_result_queue, kwargs))
# Initialize spec structures
if not dag_scheduler.schedule_is_built():
dag_scheduler.build_schedule()
ready_specs = set(dag_scheduler.pop_ready_specs())
# tty.msg(self._progress_prompt_str(
# 'Installing', 'Ready', 'Unscheduled'))
while len(ready_specs) > 0 or len(outstanding_installs) > 0:
for jobs, spec in ready_specs:
# Note: to_json does not support all_deps
work_queue.put_nowait(
(jobs, spec.to_yaml(all_deps=True)))
outstanding_installs[spec.full_hash()] = spec
ready_specs.clear()
# Block until something finishes
# TODO put a timeout and TimeoutError handler here
res, serialized_spec = installation_result_queue.get(True)
spec = Spec.from_yaml(serialized_spec)
# Message indicates an error
if res:
removed_specs = list(
dag_scheduler.install_failed(spec))
rm_specs_str = '\n\t'.join(
s.name for s in sorted(removed_specs))
tty.error(
'Installation of "%s" failed. Skipping %d'
' dependent packages: \n\t%s' %
(spec.name, len(removed_specs), rm_specs_str))
else:
dag_scheduler.install_successful(spec)
tty.msg('Installed %s' % spec.name)
# Greedily get all ready specs
for j_s in dag_scheduler.pop_ready_specs():
ready_specs.add(j_s)
outstanding_installs.pop(spec.full_hash())
except Exception as e:
import traceback
tty.error("Installation pool error, %s\n" % (str(e)))
traceback.print_exc()
finally:
tty.msg('Installation finished (%s)' % time.strftime(
'%H:%M:%S', time.gmtime(time.time() - start_time)))
def _createtarball(env,
spec_yaml=None,
packages=None,
add_spec=True,
add_deps=True,
output_location=os.getcwd(),
signing_key=None,
force=False,
make_relative=False,
unsigned=False,
allow_root=False,
rebuild_index=False):
if spec_yaml:
with open(spec_yaml, 'r') as fd:
yaml_text = fd.read()
tty.debug('createtarball read spec yaml:')
tty.debug(yaml_text)
s = Spec.from_yaml(yaml_text)
package = '/{0}'.format(s.dag_hash())
matches = find_matching_specs(package, env=env)
elif packages:
matches = find_matching_specs(packages, env=env)
elif env:
matches = [env.specs_by_hash[h] for h in env.concretized_order]
else:
tty.die("build cache file creation requires at least one" +
" installed package spec, an active environment," +
" or else a path to a yaml file containing a spec" +
" to install")
specs = set()
mirror = spack.mirror.MirrorCollection().lookup(output_location)
outdir = url_util.format(mirror.push_url)
msg = 'Buildcache files will be output to %s/build_cache' % outdir
tty.msg(msg)
if matches:
tty.debug('Found at least one matching spec')
for match in matches:
tty.debug('examining match {0}'.format(match.format()))
if match.external or match.virtual:
tty.debug('skipping external or virtual spec %s' % match.format())
else:
lookup = spack.store.db.query_one(match)
if not add_spec:
tty.debug('skipping matching root spec %s' % match.format())
elif lookup is None:
tty.debug('skipping uninstalled matching spec %s' %
match.format())
else:
tty.debug('adding matching spec %s' % match.format())
specs.add(match)
if not add_deps:
continue
tty.debug('recursing dependencies')
for d, node in match.traverse(order='post',
depth=True,
deptype=('link', 'run')):
# skip root, since it's handled above
if d == 0:
continue
lookup = spack.store.db.query_one(node)
if node.external or node.virtual:
tty.debug('skipping external or virtual dependency %s' %
node.format())
elif lookup is None:
tty.debug('skipping uninstalled depenendency %s' %
node.format())
else:
tty.debug('adding dependency %s' % node.format())
specs.add(node)
tty.debug('writing tarballs to %s/build_cache' % outdir)
for spec in specs:
tty.debug('creating binary cache file for package %s ' % spec.format())
try:
bindist.build_tarball(spec, outdir, force, make_relative, unsigned,
allow_root, signing_key, rebuild_index)
except bindist.NoOverwriteException as e:
tty.warn(e)
def test_read_and_write_spec(temporary_store, config, mock_packages):
"""This goes through each package in spack and creates a directory for
it. It then ensures that the spec for the directory's
installed package can be read back in consistently, and
finally that the directory can be removed by the directory
layout.
"""
layout = temporary_store.layout
pkg_names = list(spack.repo.path.all_package_names())[:max_packages]
for name in pkg_names:
if name.startswith('external'):
# External package tests cannot be installed
continue
# If a spec fails to concretize, just skip it. If it is a
# real error, it will be caught by concretization tests.
try:
spec = spack.spec.Spec(name).concretized()
except Exception:
continue
layout.create_install_directory(spec)
install_dir = path_to_os_path(layout.path_for_spec(spec))[0]
spec_path = layout.spec_file_path(spec)
# Ensure directory has been created in right place.
assert os.path.isdir(install_dir)
assert install_dir.startswith(temporary_store.root)
# Ensure spec file exists when directory is created
assert os.path.isfile(spec_path)
assert spec_path.startswith(install_dir)
# Make sure spec file can be read back in to get the original spec
spec_from_file = layout.read_spec(spec_path)
stored_deptypes = spack.hash_types.dag_hash
expected = spec.copy(deps=stored_deptypes)
expected._mark_concrete()
assert expected.concrete
assert expected == spec_from_file
assert expected.eq_dag(spec_from_file)
assert spec_from_file.concrete
# Ensure that specs that come out "normal" are really normal.
with open(spec_path) as spec_file:
read_separately = Spec.from_yaml(spec_file.read())
# TODO: revise this when build deps are in dag_hash
norm = read_separately.copy(deps=stored_deptypes)
assert norm == spec_from_file
assert norm.eq_dag(spec_from_file)
# TODO: revise this when build deps are in dag_hash
conc = read_separately.concretized().copy(deps=stored_deptypes)
assert conc == spec_from_file
assert conc.eq_dag(spec_from_file)
assert expected.dag_hash() == spec_from_file.dag_hash()
# Ensure directories are properly removed
layout.remove_install_directory(spec)
assert not os.path.isdir(install_dir)
assert not os.path.exists(install_dir)
def upload_spec(args):
"""Upload a spec to s3 bucket"""
if not args.spec and not args.spec_yaml:
tty.error('Cannot upload spec without spec arg or path to spec yaml')
sys.exit(1)
if not args.base_dir:
tty.error('No base directory for buildcache specified')
sys.exit(1)
if args.spec:
try:
spec = Spec(args.spec)
spec.concretize()
except Exception as e:
tty.debug(e)
tty.error('Unable to concrectize spec from string {0}'.format(
args.spec))
sys.exit(1)
else:
try:
with open(args.spec_yaml, 'r') as fd:
spec = Spec.from_yaml(fd.read())
except Exception as e:
tty.debug(e)
tty.error('Unable to concrectize spec from yaml {0}'.format(
args.spec_yaml))
sys.exit(1)
s3, bucket_name = get_s3_session(args.endpoint_url)
build_cache_dir = bindist.build_cache_relative_path()
tarball_key = os.path.join(
build_cache_dir, bindist.tarball_path_name(spec, '.spack'))
tarball_path = os.path.join(args.base_dir, tarball_key)
specfile_key = os.path.join(
build_cache_dir, bindist.tarball_name(spec, '.spec.yaml'))
specfile_path = os.path.join(args.base_dir, specfile_key)
cdashidfile_key = os.path.join(
build_cache_dir, bindist.tarball_name(spec, '.cdashid'))
cdashidfile_path = os.path.join(args.base_dir, cdashidfile_key)
tty.msg('Uploading {0}'.format(tarball_key))
s3.meta.client.upload_file(
tarball_path, bucket_name,
os.path.join('mirror', tarball_key),
ExtraArgs={'ACL': 'public-read'})
tty.msg('Uploading {0}'.format(specfile_key))
s3.meta.client.upload_file(
specfile_path, bucket_name,
os.path.join('mirror', specfile_key),
ExtraArgs={'ACL': 'public-read'})
if os.path.exists(cdashidfile_path):
tty.msg('Uploading {0}'.format(cdashidfile_key))
s3.meta.client.upload_file(
cdashidfile_path, bucket_name,
os.path.join('mirror', cdashidfile_key),
ExtraArgs={'ACL': 'public-read'})
def createtarball(args):
"""create a binary package from an existing install"""
if args.spec_yaml:
packages = set()
tty.msg('createtarball, reading spec from {0}'.format(args.spec_yaml))
with open(args.spec_yaml, 'r') as fd:
yaml_text = fd.read()
tty.debug('createtarball read spec yaml:')
tty.debug(yaml_text)
s = Spec.from_yaml(yaml_text)
packages.add('/{0}'.format(s.dag_hash()))
elif args.packages:
packages = args.packages
else:
tty.die("build cache file creation requires at least one" +
" installed package argument or else path to a" +
" yaml file containing a spec to install")
pkgs = set(packages)
specs = set()
outdir = '.'
if args.directory:
outdir = args.directory
mirror = spack.mirror.MirrorCollection().lookup(outdir)
outdir = url_util.format(mirror.push_url)
signkey = None
if args.key:
signkey = args.key
# restrict matching to current environment if one is active
env = ev.get_env(args, 'buildcache create')
matches = find_matching_specs(pkgs, env=env)
if matches:
tty.debug('Found at least one matching spec')
for match in matches:
tty.debug('examining match {0}'.format(match.format()))
if match.external or match.virtual:
tty.debug('skipping external or virtual spec %s' % match.format())
else:
tty.debug('adding matching spec %s' % match.format())
if "package" in args.target_type:
specs.add(match)
if "dependencies" not in args.target_type:
# if the user does not want dependencies, stop here
continue
tty.debug('recursing dependencies')
for d, node in match.traverse(order='post',
depth=True,
deptype=('link', 'run')):
if node.external or node.virtual:
tty.debug('skipping external or virtual dependency %s' %
node.format())
else:
tty.debug('adding dependency %s' % node.format())
specs.add(node)
tty.debug('writing tarballs to %s/build_cache' % outdir)
f_create = ft.partial(create_single_tarball,
outdir=outdir,
force=args.force,
relative=args.rel,
unsigned=args.unsigned,
allow_root=args.allow_root,
signkey=signkey,
rebuild_index=args.rebuild_index and args.jobs == 1,
catch_exceptions=args.jobs != 1)
# default behavior (early termination) for one job
if args.jobs == 1:
for spec in specs:
f_create(spec)
else:
# currently, specs cause an infinite recursion bug when pickled
# -> as multiprocessing uses pickle internally, we need to transform
# specs prior to distributing the work via worker pool
# TODO: check if specs can be pickled
specs = [s.to_dict() for s in specs]
pool = NoDaemonPool(args.jobs if args.jobs > 1 else mp.cpu_count())
# chunksize=1 because we do not want to pre-allocate specs to workers
# (since each package will need a different amount of time to be
# compressed)
retvals = pool.map(f_create, specs, chunksize=1)
errors = [rv["error"] for rv in retvals if rv["error"] is not None]
list(map(tty.error, errors))
if len(errors) > 0:
sys.exit(1)
# perform rebuild of index unless user requested not to
if args.rebuild_index:
bindist.generate_package_index(outdir)
def check_yaml_round_trip(self, spec):
yaml_text = spec.to_yaml()
spec_from_yaml = Spec.from_yaml(yaml_text)
self.assertTrue(spec.eq_dag(spec_from_yaml))
def _createtarball(env, spec_yaml, packages, directory, key, no_deps, force,
rel, unsigned, allow_root, no_rebuild_index):
if spec_yaml:
packages = set()
with open(spec_yaml, 'r') as fd:
yaml_text = fd.read()
tty.debug('createtarball read spec yaml:')
tty.debug(yaml_text)
s = Spec.from_yaml(yaml_text)
packages.add('/{0}'.format(s.dag_hash()))
elif packages:
packages = packages
else:
tty.die("build cache file creation requires at least one" +
" installed package argument or else path to a" +
" yaml file containing a spec to install")
pkgs = set(packages)
specs = set()
outdir = '.'
if directory:
outdir = directory
mirror = spack.mirror.MirrorCollection().lookup(outdir)
outdir = url_util.format(mirror.push_url)
signkey = None
if key:
signkey = key
matches = find_matching_specs(pkgs, env=env)
if matches:
tty.debug('Found at least one matching spec')
for match in matches:
tty.debug('examining match {0}'.format(match.format()))
if match.external or match.virtual:
tty.debug('skipping external or virtual spec %s' % match.format())
else:
tty.debug('adding matching spec %s' % match.format())
specs.add(match)
if no_deps is True:
continue
tty.debug('recursing dependencies')
for d, node in match.traverse(order='post',
depth=True,
deptype=('link', 'run')):
if node.external or node.virtual:
tty.debug('skipping external or virtual dependency %s' %
node.format())
else:
tty.debug('adding dependency %s' % node.format())
specs.add(node)
tty.debug('writing tarballs to %s/build_cache' % outdir)
for spec in specs:
tty.msg('creating binary cache file for package %s ' % spec.format())
try:
bindist.build_tarball(spec, outdir, force, rel, unsigned,
allow_root, signkey, not no_rebuild_index)
except Exception as e:
tty.warn('%s' % e)
pass
def buildcache_copy(args):
"""Copy a buildcache entry and all its files from one mirror, given as
'--base-dir', to some other mirror, specified as '--destination-url'.
The specific buildcache entry to be copied from one location to the
other is identified using the '--spec-yaml' argument."""
# TODO: This sub-command should go away once #11117 is merged
if not args.spec_yaml:
tty.msg('No spec yaml provided, exiting.')
sys.exit(1)
if not args.base_dir:
tty.msg('No base directory provided, exiting.')
sys.exit(1)
if not args.destination_url:
tty.msg('No destination mirror url provided, exiting.')
sys.exit(1)
dest_url = args.destination_url
if dest_url[0:7] != 'file://' and dest_url[0] != '/':
tty.msg('Only urls beginning with "file://" or "/" are supported ' +
'by buildcache copy.')
sys.exit(1)
try:
with open(args.spec_yaml, 'r') as fd:
spec = Spec.from_yaml(fd.read())
except Exception as e:
tty.debug(e)
tty.error('Unable to concrectize spec from yaml {0}'.format(
args.spec_yaml))
sys.exit(1)
dest_root_path = dest_url
if dest_url[0:7] == 'file://':
dest_root_path = dest_url[7:]
build_cache_dir = bindist.build_cache_relative_path()
tarball_rel_path = os.path.join(build_cache_dir,
bindist.tarball_path_name(spec, '.spack'))
tarball_src_path = os.path.join(args.base_dir, tarball_rel_path)
tarball_dest_path = os.path.join(dest_root_path, tarball_rel_path)
specfile_rel_path = os.path.join(build_cache_dir,
bindist.tarball_name(spec, '.spec.yaml'))
specfile_src_path = os.path.join(args.base_dir, specfile_rel_path)
specfile_dest_path = os.path.join(dest_root_path, specfile_rel_path)
cdashidfile_rel_path = os.path.join(build_cache_dir,
bindist.tarball_name(spec, '.cdashid'))
cdashid_src_path = os.path.join(args.base_dir, cdashidfile_rel_path)
cdashid_dest_path = os.path.join(dest_root_path, cdashidfile_rel_path)
# Make sure directory structure exists before attempting to copy
os.makedirs(os.path.dirname(tarball_dest_path))
# Now copy the specfile and tarball files to the destination mirror
tty.msg('Copying {0}'.format(tarball_rel_path))
shutil.copyfile(tarball_src_path, tarball_dest_path)
tty.msg('Copying {0}'.format(specfile_rel_path))
shutil.copyfile(specfile_src_path, specfile_dest_path)
# Copy the cdashid file (if exists) to the destination mirror
if os.path.exists(cdashid_src_path):
tty.msg('Copying {0}'.format(cdashidfile_rel_path))
shutil.copyfile(cdashid_src_path, cdashid_dest_path)
def check_specs_equal(original_spec, spec_yaml_path):
with open(spec_yaml_path, 'r') as fd:
spec_yaml = fd.read()
spec_from_yaml = Spec.from_yaml(spec_yaml)
return original_spec.eq_dag(spec_from_yaml)
def test_invalid_yaml_spec(invalid_yaml):
with pytest.raises(SpackYAMLError) as e:
Spec.from_yaml(invalid_yaml)
exc_msg = str(e.value)
assert exc_msg.startswith("error parsing YAML spec:")
assert invalid_yaml in exc_msg
def test_read_and_write_spec(
layout_and_dir, config, builtin_mock
):
"""This goes through each package in spack and creates a directory for
it. It then ensures that the spec for the directory's
installed package can be read back in consistently, and
finally that the directory can be removed by the directory
layout.
"""
layout, tmpdir = layout_and_dir
packages = list(spack.repo.all_packages())[:max_packages]
for pkg in packages:
if pkg.name.startswith('external'):
# External package tests cannot be installed
continue
spec = pkg.spec
# If a spec fails to concretize, just skip it. If it is a
# real error, it will be caught by concretization tests.
try:
spec.concretize()
except Exception:
continue
layout.create_install_directory(spec)
install_dir = layout.path_for_spec(spec)
spec_path = layout.spec_file_path(spec)
# Ensure directory has been created in right place.
assert os.path.isdir(install_dir)
assert install_dir.startswith(str(tmpdir))
# Ensure spec file exists when directory is created
assert os.path.isfile(spec_path)
assert spec_path.startswith(install_dir)
# Make sure spec file can be read back in to get the original spec
spec_from_file = layout.read_spec(spec_path)
# currently we don't store build dependency information when
# we write out specs to the filesystem.
# TODO: fix this when we can concretize more loosely based on
# TODO: what is installed. We currently omit these to
# TODO: increase reuse of build dependencies.
stored_deptypes = ('link', 'run')
expected = spec.copy(deps=stored_deptypes)
assert expected.concrete
assert expected == spec_from_file
assert expected.eq_dag(spec_from_file)
assert spec_from_file.concrete
# Ensure that specs that come out "normal" are really normal.
with open(spec_path) as spec_file:
read_separately = Spec.from_yaml(spec_file.read())
# TODO: revise this when build deps are in dag_hash
norm = read_separately.normalized().copy(deps=stored_deptypes)
assert norm == spec_from_file
assert norm.eq_dag(spec_from_file)
# TODO: revise this when build deps are in dag_hash
conc = read_separately.concretized().copy(deps=stored_deptypes)
assert conc == spec_from_file
assert conc.eq_dag(spec_from_file)
assert expected.dag_hash() == spec_from_file.dag_hash()
# Ensure directories are properly removed
layout.remove_install_directory(spec)
assert not os.path.isdir(install_dir)
assert not os.path.exists(install_dir)
def test_ordered_read_not_required_for_consistent_dag_hash(
config, mock_packages):
"""Make sure ordered serialization isn't required to preserve hashes.
For consistent hashes, we require that YAML and json documents
have their keys serialized in a deterministic order. However, we
don't want to require them to be serialized in order. This
ensures that is not required.
"""
specs = ['mpileaks ^zmpi', 'dttop', 'dtuse']
for spec in specs:
spec = Spec(spec)
spec.concretize()
#
# Dict & corresponding YAML & JSON from the original spec.
#
spec_dict = spec.to_dict()
spec_yaml = spec.to_yaml()
spec_json = spec.to_json()
#
# Make a spec with reversed OrderedDicts for every
# OrderedDict in the original.
#
reversed_spec_dict = reverse_all_dicts(spec.to_dict())
#
# Dump to YAML and JSON
#
yaml_string = syaml.dump(spec_dict, default_flow_style=False)
reversed_yaml_string = syaml.dump(reversed_spec_dict,
default_flow_style=False)
json_string = sjson.dump(spec_dict)
reversed_json_string = sjson.dump(reversed_spec_dict)
#
# Do many consistency checks
#
# spec yaml is ordered like the spec dict
assert yaml_string == spec_yaml
assert json_string == spec_json
# reversed string is different from the original, so it
# *would* generate a different hash
assert yaml_string != reversed_yaml_string
assert json_string != reversed_json_string
# build specs from the "wrongly" ordered data
round_trip_yaml_spec = Spec.from_yaml(yaml_string)
round_trip_json_spec = Spec.from_json(json_string)
round_trip_reversed_yaml_spec = Spec.from_yaml(reversed_yaml_string)
round_trip_reversed_json_spec = Spec.from_yaml(reversed_json_string)
# TODO: remove this when build deps are in provenance.
spec = spec.copy(deps=('link', 'run'))
# specs are equal to the original
assert spec == round_trip_yaml_spec
assert spec == round_trip_json_spec
assert spec == round_trip_reversed_yaml_spec
assert spec == round_trip_reversed_json_spec
assert round_trip_yaml_spec == round_trip_reversed_yaml_spec
assert round_trip_json_spec == round_trip_reversed_json_spec
# dag_hashes are equal
assert spec.dag_hash() == round_trip_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_json_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()
# full_hashes are equal
spec.concretize()
round_trip_yaml_spec.concretize()
round_trip_json_spec.concretize()
round_trip_reversed_yaml_spec.concretize()
round_trip_reversed_json_spec.concretize()
assert spec.full_hash() == round_trip_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_json_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_json_spec.full_hash()
def test_ordered_read_not_required_for_consistent_dag_hash(
config, builtin_mock
):
"""Make sure ordered serialization isn't required to preserve hashes.
For consistent hashes, we require that YAML and json documents
have their keys serialized in a deterministic order. However, we
don't want to require them to be serialized in order. This
ensures that is not required.
"""
specs = ['mpileaks ^zmpi', 'dttop', 'dtuse']
for spec in specs:
spec = Spec(spec)
spec.concretize()
#
# Dict & corresponding YAML & JSON from the original spec.
#
spec_dict = spec.to_dict()
spec_yaml = spec.to_yaml()
spec_json = spec.to_json()
#
# Make a spec with reversed OrderedDicts for every
# OrderedDict in the original.
#
reversed_spec_dict = reverse_all_dicts(spec.to_dict())
#
# Dump to YAML and JSON
#
yaml_string = syaml.dump(spec_dict, default_flow_style=False)
reversed_yaml_string = syaml.dump(reversed_spec_dict,
default_flow_style=False)
json_string = sjson.dump(spec_dict)
reversed_json_string = sjson.dump(reversed_spec_dict)
#
# Do many consistency checks
#
# spec yaml is ordered like the spec dict
assert yaml_string == spec_yaml
assert json_string == spec_json
# reversed string is different from the original, so it
# *would* generate a different hash
assert yaml_string != reversed_yaml_string
assert json_string != reversed_json_string
# build specs from the "wrongly" ordered data
round_trip_yaml_spec = Spec.from_yaml(yaml_string)
round_trip_json_spec = Spec.from_json(json_string)
round_trip_reversed_yaml_spec = Spec.from_yaml(
reversed_yaml_string
)
round_trip_reversed_json_spec = Spec.from_yaml(
reversed_json_string
)
# TODO: remove this when build deps are in provenance.
spec = spec.copy(deps=('link', 'run'))
# specs are equal to the original
assert spec == round_trip_yaml_spec
assert spec == round_trip_json_spec
assert spec == round_trip_reversed_yaml_spec
assert spec == round_trip_reversed_json_spec
assert round_trip_yaml_spec == round_trip_reversed_yaml_spec
assert round_trip_json_spec == round_trip_reversed_json_spec
# dag_hashes are equal
assert spec.dag_hash() == round_trip_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_json_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()
def check_yaml_round_trip(spec):
yaml_text = spec.to_yaml()
spec_from_yaml = Spec.from_yaml(yaml_text)
assert spec.eq_dag(spec_from_yaml)
def check_yaml_round_trip(spec):
yaml_text = spec.to_yaml()
spec_from_yaml = Spec.from_yaml(yaml_text)
assert spec.eq_dag(spec_from_yaml)
def test_read_and_write_spec(
layout_and_dir, config, builtin_mock
):
"""This goes through each package in spack and creates a directory for
it. It then ensures that the spec for the directory's
installed package can be read back in consistently, and
finally that the directory can be removed by the directory
layout.
"""
layout, tmpdir = layout_and_dir
packages = list(spack.repo.all_packages())[:max_packages]
for pkg in packages:
if pkg.name.startswith('external'):
# External package tests cannot be installed
continue
spec = pkg.spec
# If a spec fails to concretize, just skip it. If it is a
# real error, it will be caught by concretization tests.
try:
spec.concretize()
except Exception:
continue
layout.create_install_directory(spec)
install_dir = layout.path_for_spec(spec)
spec_path = layout.spec_file_path(spec)
# Ensure directory has been created in right place.
assert os.path.isdir(install_dir)
assert install_dir.startswith(str(tmpdir))
# Ensure spec file exists when directory is created
assert os.path.isfile(spec_path)
assert spec_path.startswith(install_dir)
# Make sure spec file can be read back in to get the original spec
spec_from_file = layout.read_spec(spec_path)
# currently we don't store build dependency information when
# we write out specs to the filesystem.
# TODO: fix this when we can concretize more loosely based on
# TODO: what is installed. We currently omit these to
# TODO: increase reuse of build dependencies.
stored_deptypes = ('link', 'run')
expected = spec.copy(deps=stored_deptypes)
expected._mark_concrete()
assert expected.concrete
assert expected == spec_from_file
assert expected.eq_dag(spec_from_file)
assert spec_from_file.concrete
# Ensure that specs that come out "normal" are really normal.
with open(spec_path) as spec_file:
read_separately = Spec.from_yaml(spec_file.read())
# TODO: revise this when build deps are in dag_hash
norm = read_separately.normalized().copy(deps=stored_deptypes)
assert norm == spec_from_file
assert norm.eq_dag(spec_from_file)
# TODO: revise this when build deps are in dag_hash
conc = read_separately.concretized().copy(deps=stored_deptypes)
assert conc == spec_from_file
assert conc.eq_dag(spec_from_file)
assert expected.dag_hash() == spec_from_file.dag_hash()
# Ensure directories are properly removed
layout.remove_install_directory(spec)
assert not os.path.isdir(install_dir)
assert not os.path.exists(install_dir)
def check_yaml_round_trip(self, spec):
yaml_text = spec.to_yaml()
spec_from_yaml = Spec.from_yaml(yaml_text)
self.assertTrue(spec.eq_dag(spec_from_yaml))
