def test_splice_dict_roundtrip(self, transitive):
spec = Spec('splice-t')
dep = Spec('splice-h+foo')
spec.concretize()
dep.concretize()
out = spec.splice(dep, transitive)
# Sanity check all hashes are unique...
assert spec.dag_hash() != dep.dag_hash()
assert out.dag_hash() != dep.dag_hash()
assert out.dag_hash() != spec.dag_hash()
out_rt_spec = Spec.from_dict(out.to_dict()) # rt is "round trip"
assert out_rt_spec.dag_hash() == out.dag_hash()
out_rt_spec_bld_hash = out_rt_spec.build_spec.dag_hash()
out_rt_spec_h_bld_hash = out_rt_spec['splice-h'].build_spec.dag_hash()
out_rt_spec_z_bld_hash = out_rt_spec['splice-z'].build_spec.dag_hash()
# In any case, the build spec for splice-t (root) should point to the
# original spec, preserving build provenance.
assert spec.dag_hash() == out_rt_spec_bld_hash
assert out_rt_spec.dag_hash() != out_rt_spec_bld_hash
# The build spec for splice-h should always point to the introduced
# spec, since that is the spec spliced in.
assert dep['splice-h'].dag_hash() == out_rt_spec_h_bld_hash
# The build spec for splice-z will depend on whether or not the splice
# was transitive.
expected_z_bld_hash = (dep['splice-z'].dag_hash()
if transitive else spec['splice-z'].dag_hash())
assert expected_z_bld_hash == out_rt_spec_z_bld_hash
def test_splice_input_unchanged(self, transitive):
spec = Spec('splice-t').concretized()
dep = Spec('splice-h+foo').concretized()
orig_spec_hash = spec.dag_hash()
orig_dep_hash = dep.dag_hash()
spec.splice(dep, transitive)
# Post-splice, dag hash should still be different; no changes should be
# made to these specs.
assert spec.dag_hash() == orig_spec_hash
assert dep.dag_hash() == orig_dep_hash
def test_regression_issue_7941(self):
# The string representation of a spec containing
# an explicit multi-valued variant and a dependency
# might be parsed differently than the originating spec
s = Spec('a foobar=bar ^b')
t = Spec(str(s))
s.concretize()
t.concretize()
assert s.dag_hash() == t.dag_hash()
def test_regression_issue_7941(self):
# The string representation of a spec containing
# an explicit multi-valued variant and a dependency
# might be parsed differently than the originating spec
s = Spec('a foobar=bar ^b')
t = Spec(str(s))
s.concretize()
t.concretize()
assert s.dag_hash() == t.dag_hash()
def test_reuse_installed_packages(self, context, mutable_database,
repo_with_changing_recipe):
# Install a spec
root = Spec('root').concretized()
dependency = root['changing'].copy()
root.package.do_install(fake=True, explicit=True)
# Modify package.py
repo_with_changing_recipe.change(context)
# Try to concretize with the spec installed previously
new_root = Spec('root ^/{0}'.format(
dependency.dag_hash())).concretized()
assert root.dag_hash() == new_root.dag_hash()
def test_dynamic_dot_graph_mpileaks(mock_packages, config):
"""Test dynamically graphing the mpileaks package."""
s = Spec('mpileaks').concretized()
stream = StringIO()
graph_dot([s], static=False, out=stream)
dot = stream.getvalue()
print(dot)
mpileaks_hash, mpileaks_lbl = s.dag_hash(), s.format('{name}')
mpi_hash, mpi_lbl = s['mpi'].dag_hash(), s['mpi'].format('{name}')
callpath_hash, callpath_lbl = (s['callpath'].dag_hash(),
s['callpath'].format('{name}'))
dyninst_hash, dyninst_lbl = (s['dyninst'].dag_hash(),
s['dyninst'].format('{name}'))
libdwarf_hash, libdwarf_lbl = (s['libdwarf'].dag_hash(),
s['libdwarf'].format('{name}'))
libelf_hash, libelf_lbl = (s['libelf'].dag_hash(),
s['libelf'].format('{name}'))
assert ' "%s" [label="%s"]\n' % (mpileaks_hash, mpileaks_lbl) in dot
assert ' "%s" [label="%s"]\n' % (callpath_hash, callpath_lbl) in dot
assert ' "%s" [label="%s"]\n' % (mpi_hash, mpi_lbl) in dot
assert ' "%s" [label="%s"]\n' % (dyninst_hash, dyninst_lbl) in dot
assert ' "%s" [label="%s"]\n' % (libdwarf_hash, libdwarf_lbl) in dot
assert ' "%s" [label="%s"]\n' % (libelf_hash, libelf_lbl) in dot
assert ' "%s" -> "%s"\n' % (dyninst_hash, libdwarf_hash) in dot
assert ' "%s" -> "%s"\n' % (callpath_hash, dyninst_hash) in dot
assert ' "%s" -> "%s"\n' % (mpileaks_hash, mpi_hash) in dot
assert ' "%s" -> "%s"\n' % (libdwarf_hash, libelf_hash) in dot
assert ' "%s" -> "%s"\n' % (callpath_hash, mpi_hash) in dot
assert ' "%s" -> "%s"\n' % (mpileaks_hash, callpath_hash) in dot
assert ' "%s" -> "%s"\n' % (dyninst_hash, libelf_hash) in dot
def test_splice(self, transitive):
# Tests the new splice function in Spec using a somewhat simple case
# with a variant with a conditional dependency.
# TODO: Test being able to splice in different provider for a virtual.
# Example: mvapich for mpich.
spec = Spec('splice-t')
dep = Spec('splice-h+foo')
spec.concretize()
dep.concretize()
# Sanity checking that these are not the same thing.
assert dep.dag_hash() != spec['splice-h'].dag_hash()
assert dep.build_hash() != spec['splice-h'].build_hash()
# Do the splice.
out = spec.splice(dep, transitive)
# Returned spec should still be concrete.
assert out.concrete
# Traverse the spec and assert that all dependencies are accounted for.
for node in spec.traverse():
assert node.name in out
# If the splice worked, then the full hash of the spliced dep should
# now match the full hash of the build spec of the dependency from the
# returned spec.
out_h_build = out['splice-h'].build_spec
assert out_h_build.full_hash() == dep.full_hash()
# Transitivity should determine whether the transitive dependency was
# changed.
expected_z = dep['splice-z'] if transitive else spec['splice-z']
assert out['splice-z'].full_hash() == expected_z.full_hash()
# Sanity check build spec of out should be the original spec.
assert (out['splice-t'].build_spec.full_hash() ==
spec['splice-t'].full_hash())
# Finally, the spec should know it's been spliced:
assert out.spliced
def test_rewire(mock_fetch, install_mockery, transitive):
spec = Spec('splice-t^splice-h~foo').concretized()
dep = Spec('splice-h+foo').concretized()
spec.package.do_install()
dep.package.do_install()
spliced_spec = spec.splice(dep, transitive=transitive)
assert spec.dag_hash() != spliced_spec.dag_hash()
spack.rewiring.rewire(spliced_spec)
# check that the prefix exists
assert os.path.exists(spliced_spec.prefix)
# test that it made it into the database
rec = spack.store.db.get_record(spliced_spec)
installed_in_db = rec.installed if rec else False
assert installed_in_db
# check the file in the prefix has the correct paths
for node in spliced_spec.traverse(root=True):
text_file_path = os.path.join(node.prefix, node.name)
with open(text_file_path, 'r') as f:
text = f.read()
for modded_spec in node.traverse(root=True):
assert modded_spec.prefix in text
def test_rewire_bin(mock_fetch, install_mockery, transitive):
spec = Spec('quux').concretized()
dep = Spec('garply cflags=-g').concretized()
spec.package.do_install()
dep.package.do_install()
spliced_spec = spec.splice(dep, transitive=transitive)
assert spec.dag_hash() != spliced_spec.dag_hash()
spack.rewiring.rewire(spliced_spec)
# check that the prefix exists
assert os.path.exists(spliced_spec.prefix)
# test that it made it into the database
rec = spack.store.db.get_record(spliced_spec)
installed_in_db = rec.installed if rec else False
assert installed_in_db
# check the file in the prefix has the correct paths
bin_names = {
'garply': 'garplinator',
'corge': 'corgegator',
'quux': 'quuxifier'
}
for node in spliced_spec.traverse(root=True):
for dep in node.traverse(root=True):
bin_file_path = os.path.join(dep.prefix.bin, bin_names[dep.name])
assert text_in_bin(dep.prefix, bin_file_path)
def remove(self, query_spec, force=False):
"""Remove specs from an environment that match a query_spec"""
query_spec = Spec(query_spec)
# try abstract specs first
matches = []
if not query_spec.concrete:
matches = [s for s in self.user_specs if s.satisfies(query_spec)]
if not matches:
# concrete specs match against concrete specs in the env
specs_hashes = zip(self.concretized_user_specs,
self.concretized_order)
matches = [
s for s, h in specs_hashes if query_spec.dag_hash() == h
]
if not matches:
raise SpackEnvironmentError("Not found: {0}".format(query_spec))
for spec in matches:
if spec in self.user_specs:
self.user_specs.remove(spec)
if force and spec in self.concretized_user_specs:
i = self.concretized_user_specs.index(spec)
del self.concretized_user_specs[i]
dag_hash = self.concretized_order[i]
del self.concretized_order[i]
del self.specs_by_hash[dag_hash]
def remove(self, query_spec, force=False):
"""Remove specs from an environment that match a query_spec"""
query_spec = Spec(query_spec)
# try abstract specs first
matches = []
if not query_spec.concrete:
matches = [s for s in self.user_specs if s.satisfies(query_spec)]
if not matches:
# concrete specs match against concrete specs in the env
specs_hashes = zip(
self.concretized_user_specs, self.concretized_order)
matches = [
s for s, h in specs_hashes if query_spec.dag_hash() == h]
if not matches:
raise SpackEnvironmentError("Not found: {0}".format(query_spec))
for spec in matches:
if spec in self.user_specs:
self.user_specs.remove(spec)
if force and spec in self.concretized_user_specs:
i = self.concretized_user_specs.index(spec)
del self.concretized_user_specs[i]
dag_hash = self.concretized_order[i]
del self.concretized_order[i]
del self.specs_by_hash[dag_hash]
def test_dynamic_dot_graph_mpileaks(mock_packages):
"""Test dynamically graphing the mpileaks package."""
s = Spec('mpileaks').normalized()
stream = StringIO()
graph_dot([s], static=False, out=stream)
dot = stream.getvalue()
mpileaks_hash, mpileaks_lbl = s.dag_hash(), s.format('$_$/')
mpi_hash, mpi_lbl = s['mpi'].dag_hash(), s['mpi'].format('$_$/')
callpath_hash, callpath_lbl = (
s['callpath'].dag_hash(), s['callpath'].format('$_$/'))
dyninst_hash, dyninst_lbl = (
s['dyninst'].dag_hash(), s['dyninst'].format('$_$/'))
libdwarf_hash, libdwarf_lbl = (
s['libdwarf'].dag_hash(), s['libdwarf'].format('$_$/'))
libelf_hash, libelf_lbl = (
s['libelf'].dag_hash(), s['libelf'].format('$_$/'))
assert ' "%s" [label="%s"]\n' % (mpileaks_hash, mpileaks_lbl) in dot
assert ' "%s" [label="%s"]\n' % (callpath_hash, callpath_lbl) in dot
assert ' "%s" [label="%s"]\n' % (mpi_hash, mpi_lbl) in dot
assert ' "%s" [label="%s"]\n' % (dyninst_hash, dyninst_lbl) in dot
assert ' "%s" [label="%s"]\n' % (libdwarf_hash, libdwarf_lbl) in dot
assert ' "%s" [label="%s"]\n' % (libelf_hash, libelf_lbl) in dot
assert ' "%s" -> "%s"\n' % (dyninst_hash, libdwarf_hash) in dot
assert ' "%s" -> "%s"\n' % (callpath_hash, dyninst_hash) in dot
assert ' "%s" -> "%s"\n' % (mpileaks_hash, mpi_hash) in dot
assert ' "%s" -> "%s"\n' % (libdwarf_hash, libelf_hash) in dot
assert ' "%s" -> "%s"\n' % (callpath_hash, mpi_hash) in dot
assert ' "%s" -> "%s"\n' % (mpileaks_hash, callpath_hash) in dot
assert ' "%s" -> "%s"\n' % (dyninst_hash, libelf_hash) in dot
def test_splice_with_cached_hashes(self, transitive):
spec = Spec('splice-t')
dep = Spec('splice-h+foo')
spec.concretize()
dep.concretize()
# monkeypatch hashes so we can test that they are cached
spec._hash = 'aaaaaa'
dep._hash = 'bbbbbb'
spec['splice-h']._hash = 'cccccc'
spec['splice-z']._hash = 'dddddd'
dep['splice-z']._hash = 'eeeeee'
out = spec.splice(dep, transitive=transitive)
out_z_expected = (dep if transitive else spec)['splice-z']
assert out.dag_hash() != spec.dag_hash()
assert (out['splice-h'].dag_hash() == dep.dag_hash()) == transitive
assert out['splice-z'].dag_hash() == out_z_expected.dag_hash()
def test_external_packages_have_consistent_hash(self):
if spack.config.get('config:concretizer') == 'original':
pytest.skip('This tests needs the ASP-based concretizer')
s, t = Spec('externaltool'), Spec('externaltool')
s._old_concretize(), t._new_concretize()
assert s.dag_hash() == t.dag_hash()
assert s.build_hash() == t.build_hash()
assert s.full_hash() == t.full_hash()
def test_installed_dependency_request_conflicts(install_mockery, mock_fetch,
mutable_mock_repo):
dependency = Spec('dependency-install')
dependency.concretize()
dependency.package.do_install()
dependency_hash = dependency.dag_hash()
dependent = Spec('conflicting-dependent ^/' + dependency_hash)
with pytest.raises(spack.error.UnsatisfiableSpecError):
dependent.concretize()
def test_installed_dependency_request_conflicts(
install_mockery, mock_fetch, mutable_mock_packages):
dependency = Spec('dependency-install')
dependency.concretize()
dependency.package.do_install()
dependency_hash = dependency.dag_hash()
dependent = Spec(
'conflicting-dependent ^/' + dependency_hash)
with pytest.raises(spack.spec.UnsatisfiableSpecError):
dependent.concretize()
def test_satisfies_same_spec_with_different_hash(self):
"""Ensure that concrete specs are matched *exactly* by hash."""
s1 = Spec('mpileaks').concretized()
s2 = s1.copy()
assert s1.satisfies(s2)
assert s2.satisfies(s1)
# Simulate specs that were installed before and after a change to
# Spack's hashing algorithm. This just reverses s2's hash.
s2._hash = s1.dag_hash()[-1::-1]
assert not s1.satisfies(s2)
assert not s2.satisfies(s1)
def test_dont_add_patches_to_installed_package(install_mockery, mock_fetch):
dependency = Spec('dependency-install')
dependency.concretize()
dependency.package.do_install()
dependency_hash = dependency.dag_hash()
dependent = Spec('dependent-install ^/' + dependency_hash)
dependent.concretize()
dependency.package.patches['dependency-install'] = [
spack.patch.UrlPatch(
dependent.package, 'file://fake.patch', sha256='unused-hash')]
assert dependent['dependency-install'] == dependency
def test_satisfies_same_spec_with_different_hash(self):
"""Ensure that concrete specs are matched *exactly* by hash."""
s1 = Spec('mpileaks').concretized()
s2 = s1.copy()
assert s1.satisfies(s2)
assert s2.satisfies(s1)
# Simulate specs that were installed before and after a change to
# Spack's hashing algorithm. This just reverses s2's hash.
s2._hash = s1.dag_hash()[-1::-1]
assert not s1.satisfies(s2)
assert not s2.satisfies(s1)
def test_dont_add_patches_to_installed_package(install_mockery, mock_fetch):
import sys
dependency = Spec('dependency-install')
dependency.concretize()
dependency.package.do_install()
dependency.package.patches['dependency-install'] = [
sys.modules['spack.patch'].Patch.create(
None, 'file://fake.patch', sha256='unused-hash')]
dependency_hash = dependency.dag_hash()
dependent = Spec('dependent-install ^/' + dependency_hash)
dependent.concretize()
assert dependent['dependency-install'] == dependency
def remove(self, query_spec, list_name=user_speclist_name, force=False):
"""Remove specs from an environment that match a query_spec"""
query_spec = Spec(query_spec)
list_to_change = self.spec_lists[list_name]
matches = []
if not query_spec.concrete:
matches = [s for s in list_to_change if s.satisfies(query_spec)]
if not matches:
# concrete specs match against concrete specs in the env
specs_hashes = zip(
self.concretized_user_specs, self.concretized_order)
matches = [
s for s, h in specs_hashes
if query_spec.dag_hash() == h
]
if not matches:
raise SpackEnvironmentError(
"Not found: {0}".format(query_spec))
old_specs = set(self.user_specs)
for spec in matches:
if spec in list_to_change:
list_to_change.remove(spec)
self.update_stale_references(list_name)
# If force, update stale concretized specs
# Only check specs removed by this operation
new_specs = set(self.user_specs)
for spec in old_specs - new_specs:
if force and spec in self.concretized_user_specs:
i = self.concretized_user_specs.index(spec)
del self.concretized_user_specs[i]
dag_hash = self.concretized_order[i]
del self.concretized_order[i]
del self.specs_by_hash[dag_hash]
def test_failing_overwrite_install_should_keep_previous_installation(
mock_fetch, install_mockery
):
"""
Make sure that whenever `spack install --overwrite` fails, spack restores
the original install prefix instead of cleaning it.
"""
# Do a successful install
s = Spec('canfail').concretized()
s.package.succeed = True
# Do a failing overwrite install
s.package.do_install()
s.package.succeed = False
kwargs = {'overwrite': [s.dag_hash()]}
with pytest.raises(Exception):
s.package.do_install(**kwargs)
assert s.package.spec.installed
assert os.path.exists(s.prefix)
def test_splice_dict(self, transitive):
spec = Spec('splice-t')
dep = Spec('splice-h+foo')
spec.concretize()
dep.concretize()
out = spec.splice(dep, transitive)
# Sanity check all hashes are unique...
assert spec.dag_hash() != dep.dag_hash()
assert out.dag_hash() != dep.dag_hash()
assert out.dag_hash() != spec.dag_hash()
node_list = out.to_dict()['spec']['nodes']
root_nodes = [n for n in node_list if n['hash'] == out.dag_hash()]
build_spec_nodes = [n for n in node_list if n['hash'] == spec.dag_hash()]
assert spec.dag_hash() == out.build_spec.dag_hash()
assert len(root_nodes) == 1
assert len(build_spec_nodes) == 1
def test_update_sbang(tmpdir, install_mockery, function_mirror):
"""
Test the creation and installation of buildcaches with default rpaths
into the non-default directory layout scheme, triggering an update of the
sbang.
"""
# Save the original store and layout before we touch ANYTHING.
real_store = spack.store.store
real_layout = spack.store.layout
# Concretize a package with some old-fashioned sbang lines.
sspec = Spec('old-sbang')
sspec.concretize()
# Need a fake mirror with *function* scope.
mirror_dir = function_mirror
# Assumes all commands will concretize sspec the same way.
install_cmd('--no-cache', sspec.name)
# Create a buildcache with the installed spec.
buildcache_cmd('create', '-u', '-a', '-d', mirror_dir,
'/%s' % sspec.dag_hash())
# Need to force an update of the buildcache index
buildcache_cmd('update-index', '-d', 'file://%s' % mirror_dir)
# Uninstall the original package.
uninstall_cmd('-y', '/%s' % sspec.dag_hash())
try:
# New install tree locations...
# Too fine-grained to do be done in a fixture
spack.store.store = spack.store.Store(str(tmpdir.join('newtree')))
spack.store.layout = YamlDirectoryLayout(
str(tmpdir.join('newtree')),
path_scheme=ndef_install_path_scheme) # noqa: E501
# Install package from buildcache
buildcache_cmd('install', '-a', '-u', '-f', sspec.name)
# Continue blowing away caches
bindist.clear_spec_cache()
spack.stage.purge()
# test that the sbang was updated by the move
sbang_style_1_expected = '''{0}
#!/usr/bin/env python
{1}
'''.format(sbang.sbang_shebang_line(), sspec.prefix.bin)
sbang_style_2_expected = '''{0}
#!/usr/bin/env python
{1}
'''.format(sbang.sbang_shebang_line(), sspec.prefix.bin)
installed_script_style_1_path = \
sspec.prefix.bin.join('sbang-style-1.sh')
assert sbang_style_1_expected == \
open(str(installed_script_style_1_path)).read()
installed_script_style_2_path = \
sspec.prefix.bin.join('sbang-style-2.sh')
assert sbang_style_2_expected == \
open(str(installed_script_style_2_path)).read()
uninstall_cmd('-y', '/%s' % sspec.dag_hash())
finally:
spack.store.store = real_store
spack.store.layout = real_layout
def test_buildcache(mock_archive, tmpdir):
# tweak patchelf to only do a download
spec = Spec("patchelf")
spec.concretize()
pkg = spack.repo.get(spec)
fake_fetchify(pkg.fetcher, pkg)
mkdirp(os.path.join(pkg.prefix, "bin"))
patchelfscr = os.path.join(pkg.prefix, "bin", "patchelf")
f = open(patchelfscr, 'w')
body = """#!/bin/bash
echo $PATH"""
f.write(body)
f.close()
st = os.stat(patchelfscr)
os.chmod(patchelfscr, st.st_mode | stat.S_IEXEC)
# Install the test package
spec = Spec('trivial-install-test-package')
spec.concretize()
assert spec.concrete
pkg = spec.package
fake_fetchify(mock_archive.url, pkg)
pkg.do_install()
pkghash = '/' + spec.dag_hash(7)
# Put some non-relocatable file in there
filename = os.path.join(spec.prefix, "dummy.txt")
with open(filename, "w") as script:
script.write(spec.prefix)
# Create the build cache and
# put it directly into the mirror
mirror_path = os.path.join(str(tmpdir), 'test-mirror')
spack.mirror.create(
mirror_path, specs=[], no_checksum=True
)
# register mirror with spack config
mirrors = {'spack-mirror-test': 'file://' + mirror_path}
spack.config.set('mirrors', mirrors)
stage = spack.stage.Stage(
mirrors['spack-mirror-test'], name="build_cache", keep=True)
stage.create()
# setup argument parser
parser = argparse.ArgumentParser()
buildcache.setup_parser(parser)
# Create a private key to sign package with if gpg2 available
if has_gnupg2():
spack.util.gpg.Gpg.create(name='test key 1', expires='0',
email='*****@*****.**',
comment='Spack test key')
# Create build cache with signing
args = parser.parse_args(['create', '-d', mirror_path, str(spec)])
buildcache.buildcache(parser, args)
# Uninstall the package
pkg.do_uninstall(force=True)
# test overwrite install
args = parser.parse_args(['install', '-f', str(pkghash)])
buildcache.buildcache(parser, args)
# create build cache with relative path and signing
args = parser.parse_args(
['create', '-d', mirror_path, '-f', '-r', str(spec)])
buildcache.buildcache(parser, args)
# Uninstall the package
pkg.do_uninstall(force=True)
# install build cache with verification
args = parser.parse_args(['install', str(spec)])
buildcache.install_tarball(spec, args)
# test overwrite install
args = parser.parse_args(['install', '-f', str(pkghash)])
buildcache.buildcache(parser, args)
else:
# create build cache without signing
args = parser.parse_args(
['create', '-d', mirror_path, '-u', str(spec)])
buildcache.buildcache(parser, args)
# Uninstall the package
pkg.do_uninstall(force=True)
# install build cache without verification
args = parser.parse_args(['install', '-u', str(spec)])
buildcache.install_tarball(spec, args)
# test overwrite install without verification
args = parser.parse_args(['install', '-f', '-u', str(pkghash)])
buildcache.buildcache(parser, args)
# create build cache with relative path
args = parser.parse_args(
['create', '-d', mirror_path, '-f', '-r', '-u', str(pkghash)])
buildcache.buildcache(parser, args)
# Uninstall the package
pkg.do_uninstall(force=True)
# install build cache
args = parser.parse_args(['install', '-u', str(spec)])
buildcache.install_tarball(spec, args)
# test overwrite install
args = parser.parse_args(['install', '-f', '-u', str(pkghash)])
buildcache.buildcache(parser, args)
# Validate the relocation information
buildinfo = bindist.read_buildinfo_file(spec.prefix)
assert(buildinfo['relocate_textfiles'] == ['dummy.txt'])
args = parser.parse_args(['list'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['list', '-f'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['list', 'trivial'])
buildcache.buildcache(parser, args)
# Copy a key to the mirror to have something to download
shutil.copyfile(mock_gpg_keys_path + '/external.key',
mirror_path + '/external.key')
args = parser.parse_args(['keys'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['keys', '-f'])
buildcache.buildcache(parser, args)
# unregister mirror with spack config
mirrors = {}
spack.config.set('mirrors', mirrors)
shutil.rmtree(mirror_path)
stage.destroy()
def test_packaging(mock_archive, tmpdir):
# tweak patchelf to only do a download
spec = Spec("patchelf")
spec.concretize()
pkg = spack.repo.get(spec)
fake_fetchify(pkg.fetcher, pkg)
mkdirp(os.path.join(pkg.prefix, "bin"))
patchelfscr = os.path.join(pkg.prefix, "bin", "patchelf")
f = open(patchelfscr, 'w')
body = """#!/bin/bash
echo $PATH"""
f.write(body)
f.close()
st = os.stat(patchelfscr)
os.chmod(patchelfscr, st.st_mode | stat.S_IEXEC)
# Install the test package
spec = Spec('trivial-install-test-package')
spec.concretize()
assert spec.concrete
pkg = spack.repo.get(spec)
fake_fetchify(mock_archive.url, pkg)
pkg.do_install()
pkghash = '/' + spec.dag_hash(7)
# Put some non-relocatable file in there
filename = os.path.join(spec.prefix, "dummy.txt")
with open(filename, "w") as script:
script.write(spec.prefix)
# Create the build cache and
# put it directly into the mirror
mirror_path = os.path.join(str(tmpdir), 'test-mirror')
specs = [spec]
spack.mirror.create(mirror_path, specs, no_checksum=True)
# register mirror with spack config
mirrors = {'spack-mirror-test': 'file://' + mirror_path}
spack.config.update_config('mirrors', mirrors)
stage = spack.stage.Stage(mirrors['spack-mirror-test'],
name="build_cache",
keep=True)
stage.create()
# setup argument parser
parser = argparse.ArgumentParser()
buildcache.setup_parser(parser)
# Create a private key to sign package with if gpg2 available
if has_gnupg2():
spack.util.gpg.Gpg.create(name='test key 1',
expires='0',
email='*****@*****.**',
comment='Spack test key')
# Create build cache with signing
args = parser.parse_args(['create', '-d', mirror_path, str(spec)])
buildcache.buildcache(parser, args)
# Uninstall the package
pkg.do_uninstall(force=True)
# test overwrite install
args = parser.parse_args(['install', '-f', str(pkghash)])
buildcache.buildcache(parser, args)
# create build cache with relative path and signing
args = parser.parse_args(
['create', '-d', mirror_path, '-f', '-r',
str(spec)])
buildcache.buildcache(parser, args)
# Uninstall the package
pkg.do_uninstall(force=True)
# install build cache with verification
args = parser.parse_args(['install', str(spec)])
buildcache.install_tarball(spec, args)
# test overwrite install
args = parser.parse_args(['install', '-f', str(pkghash)])
buildcache.buildcache(parser, args)
else:
# create build cache without signing
args = parser.parse_args(
['create', '-d', mirror_path, '-y',
str(spec)])
buildcache.buildcache(parser, args)
# Uninstall the package
pkg.do_uninstall(force=True)
# install build cache without verification
args = parser.parse_args(['install', '-y', str(spec)])
buildcache.install_tarball(spec, args)
# test overwrite install without verification
args = parser.parse_args(['install', '-f', '-y', str(pkghash)])
buildcache.buildcache(parser, args)
# create build cache with relative path
args = parser.parse_args(
['create', '-d', mirror_path, '-f', '-r', '-y',
str(pkghash)])
buildcache.buildcache(parser, args)
# Uninstall the package
pkg.do_uninstall(force=True)
# install build cache
args = parser.parse_args(['install', '-y', str(spec)])
buildcache.install_tarball(spec, args)
# test overwrite install
args = parser.parse_args(['install', '-f', '-y', str(pkghash)])
buildcache.buildcache(parser, args)
# Validate the relocation information
buildinfo = bindist.read_buildinfo_file(spec.prefix)
assert (buildinfo['relocate_textfiles'] == ['dummy.txt'])
args = parser.parse_args(['list'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['list', '-f'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['list', 'trivial'])
buildcache.buildcache(parser, args)
# Copy a key to the mirror to have something to download
shutil.copyfile(spack.mock_gpg_keys_path + '/external.key',
mirror_path + '/external.key')
args = parser.parse_args(['keys'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['keys', '-f'])
buildcache.buildcache(parser, args)
# unregister mirror with spack config
mirrors = {}
spack.config.update_config('mirrors', mirrors)
shutil.rmtree(mirror_path)
stage.destroy()
def test_update_sbang(tmpdir, test_mirror):
"""Test the creation and installation of buildcaches with default rpaths
into the non-default directory layout scheme, triggering an update of the
sbang.
"""
scheme = os.path.join(
'${name}', '${version}',
'${architecture}-${compiler.name}-${compiler.version}-${hash}')
spec_str = 'old-sbang'
# Concretize a package with some old-fashioned sbang lines.
old_spec = Spec(spec_str).concretized()
old_spec_hash_str = '/{0}'.format(old_spec.dag_hash())
# Need a fake mirror with *function* scope.
mirror_dir = test_mirror
mirror_url = 'file://{0}'.format(mirror_dir)
# Assume all commands will concretize old_spec the same way.
install_cmd('--no-cache', old_spec.name)
# Create a buildcache with the installed spec.
buildcache_cmd('create', '-u', '-a', '-d', mirror_dir, old_spec_hash_str)
# Need to force an update of the buildcache index
buildcache_cmd('update-index', '-d', mirror_url)
# Uninstall the original package.
uninstall_cmd('-y', old_spec_hash_str)
# Switch the store to the new install tree locations
newtree_dir = tmpdir.join('newtree')
s = spack.store.Store(str(newtree_dir))
s.layout = DirectoryLayout(str(newtree_dir), path_scheme=scheme)
with spack.store.use_store(s):
new_spec = Spec('old-sbang')
new_spec.concretize()
assert new_spec.dag_hash() == old_spec.dag_hash()
# Install package from buildcache
buildcache_cmd('install', '-a', '-u', '-f', new_spec.name)
# Continue blowing away caches
bindist.clear_spec_cache()
spack.stage.purge()
# test that the sbang was updated by the move
sbang_style_1_expected = '''{0}
#!/usr/bin/env python
{1}
'''.format(sbang.sbang_shebang_line(), new_spec.prefix.bin)
sbang_style_2_expected = '''{0}
#!/usr/bin/env python
{1}
'''.format(sbang.sbang_shebang_line(), new_spec.prefix.bin)
installed_script_style_1_path = new_spec.prefix.bin.join(
'sbang-style-1.sh')
assert sbang_style_1_expected == \
open(str(installed_script_style_1_path)).read()
installed_script_style_2_path = new_spec.prefix.bin.join(
'sbang-style-2.sh')
assert sbang_style_2_expected == \
open(str(installed_script_style_2_path)).read()
uninstall_cmd('-y', '/%s' % new_spec.dag_hash())
def test_buildcache(mock_archive, tmpdir):
# tweak patchelf to only do a download
pspec = Spec("patchelf")
pspec.concretize()
pkg = spack.repo.get(pspec)
fake_fetchify(pkg.fetcher, pkg)
mkdirp(os.path.join(pkg.prefix, "bin"))
patchelfscr = os.path.join(pkg.prefix, "bin", "patchelf")
f = open(patchelfscr, 'w')
body = """#!/bin/bash
echo $PATH"""
f.write(body)
f.close()
st = os.stat(patchelfscr)
os.chmod(patchelfscr, st.st_mode | stat.S_IEXEC)
# Install the test package
spec = Spec('trivial-install-test-package')
spec.concretize()
assert spec.concrete
pkg = spec.package
fake_fetchify(mock_archive.url, pkg)
pkg.do_install()
pkghash = '/' + str(spec.dag_hash(7))
# Put some non-relocatable file in there
filename = os.path.join(spec.prefix, "dummy.txt")
with open(filename, "w") as script:
script.write(spec.prefix)
# Create an absolute symlink
linkname = os.path.join(spec.prefix, "link_to_dummy.txt")
os.symlink(filename, linkname)
# Create the build cache and
# put it directly into the mirror
mirror_path = os.path.join(str(tmpdir), 'test-mirror')
spack.mirror.create(mirror_path, specs=[])
# register mirror with spack config
mirrors = {'spack-mirror-test': 'file://' + mirror_path}
spack.config.set('mirrors', mirrors)
stage = spack.stage.Stage(
mirrors['spack-mirror-test'], name="build_cache", keep=True)
stage.create()
# setup argument parser
parser = argparse.ArgumentParser()
buildcache.setup_parser(parser)
create_args = ['create', '-a', '-f', '-d', mirror_path, pkghash]
# Create a private key to sign package with if gpg2 available
spack.util.gpg.create(name='test key 1', expires='0',
email='*****@*****.**',
comment='Spack test key')
create_args.insert(create_args.index('-a'), '--rebuild-index')
args = parser.parse_args(create_args)
buildcache.buildcache(parser, args)
# trigger overwrite warning
buildcache.buildcache(parser, args)
# Uninstall the package
pkg.do_uninstall(force=True)
install_args = ['install', '-a', '-f', pkghash]
args = parser.parse_args(install_args)
# Test install
buildcache.buildcache(parser, args)
files = os.listdir(spec.prefix)
assert 'link_to_dummy.txt' in files
assert 'dummy.txt' in files
# Validate the relocation information
buildinfo = bindist.read_buildinfo_file(spec.prefix)
assert(buildinfo['relocate_textfiles'] == ['dummy.txt'])
assert(buildinfo['relocate_links'] == ['link_to_dummy.txt'])
# create build cache with relative path
create_args.insert(create_args.index('-a'), '-f')
create_args.insert(create_args.index('-a'), '-r')
args = parser.parse_args(create_args)
buildcache.buildcache(parser, args)
# Uninstall the package
pkg.do_uninstall(force=True)
args = parser.parse_args(install_args)
buildcache.buildcache(parser, args)
# test overwrite install
install_args.insert(install_args.index('-a'), '-f')
args = parser.parse_args(install_args)
buildcache.buildcache(parser, args)
files = os.listdir(spec.prefix)
assert 'link_to_dummy.txt' in files
assert 'dummy.txt' in files
# assert os.path.realpath(
# os.path.join(spec.prefix, 'link_to_dummy.txt')
# ) == os.path.realpath(os.path.join(spec.prefix, 'dummy.txt'))
args = parser.parse_args(['keys'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['list'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['list'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['list', 'trivial'])
buildcache.buildcache(parser, args)
# Copy a key to the mirror to have something to download
shutil.copyfile(mock_gpg_keys_path + '/external.key',
mirror_path + '/external.key')
args = parser.parse_args(['keys'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['keys', '-f'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['keys', '-i', '-t'])
buildcache.buildcache(parser, args)
# unregister mirror with spack config
mirrors = {}
spack.config.set('mirrors', mirrors)
shutil.rmtree(mirror_path)
stage.destroy()
# Remove cached binary specs since we deleted the mirror
bindist._cached_specs = set()
def test_ordered_read_not_required_for_consistent_dag_hash(
config, mock_packages):
"""Make sure ordered serialization isn't required to preserve hashes.
For consistent hashes, we require that YAML and json documents
have their keys serialized in a deterministic order. However, we
don't want to require them to be serialized in order. This
ensures that is not required.
"""
specs = ['mpileaks ^zmpi', 'dttop', 'dtuse']
for spec in specs:
spec = Spec(spec)
spec.concretize()
#
# Dict & corresponding YAML & JSON from the original spec.
#
spec_dict = spec.to_dict()
spec_yaml = spec.to_yaml()
spec_json = spec.to_json()
#
# Make a spec with reversed OrderedDicts for every
# OrderedDict in the original.
#
reversed_spec_dict = reverse_all_dicts(spec.to_dict())
#
# Dump to YAML and JSON
#
yaml_string = syaml.dump(spec_dict, default_flow_style=False)
reversed_yaml_string = syaml.dump(reversed_spec_dict,
default_flow_style=False)
json_string = sjson.dump(spec_dict)
reversed_json_string = sjson.dump(reversed_spec_dict)
#
# Do many consistency checks
#
# spec yaml is ordered like the spec dict
assert yaml_string == spec_yaml
assert json_string == spec_json
# reversed string is different from the original, so it
# *would* generate a different hash
assert yaml_string != reversed_yaml_string
assert json_string != reversed_json_string
# build specs from the "wrongly" ordered data
round_trip_yaml_spec = Spec.from_yaml(yaml_string)
round_trip_json_spec = Spec.from_json(json_string)
round_trip_reversed_yaml_spec = Spec.from_yaml(reversed_yaml_string)
round_trip_reversed_json_spec = Spec.from_yaml(reversed_json_string)
# TODO: remove this when build deps are in provenance.
spec = spec.copy(deps=('link', 'run'))
# specs are equal to the original
assert spec == round_trip_yaml_spec
assert spec == round_trip_json_spec
assert spec == round_trip_reversed_yaml_spec
assert spec == round_trip_reversed_json_spec
assert round_trip_yaml_spec == round_trip_reversed_yaml_spec
assert round_trip_json_spec == round_trip_reversed_json_spec
# dag_hashes are equal
assert spec.dag_hash() == round_trip_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_json_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()
# full_hashes are equal
spec.concretize()
round_trip_yaml_spec.concretize()
round_trip_json_spec.concretize()
round_trip_reversed_yaml_spec.concretize()
round_trip_reversed_json_spec.concretize()
assert spec.full_hash() == round_trip_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_json_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_json_spec.full_hash()
def test_spec_formatting(self):
spec = Spec("multivalue-variant cflags=-O2")
spec.concretize()
# Since the default is the full spec see if the string rep of
# spec is the same as the output of spec.format()
# ignoring whitespace (though should we?) and ignoring dependencies
spec_string = str(spec)
idx = spec_string.index(' ^')
assert spec_string[:idx] == spec.format().strip()
# Testing named strings ie {string} and whether we get
# the correct component
# Mixed case intentional to test both
package_segments = [("{NAME}", "name"), ("{VERSION}", "versions"),
("{compiler}", "compiler"),
("{compiler_flags}", "compiler_flags"),
("{variants}", "variants"),
("{architecture}", "architecture")]
sigil_package_segments = [("{@VERSIONS}", '@' + str(spec.version)),
("{%compiler}", '%' + str(spec.compiler)),
("{arch=architecture}",
'arch=' + str(spec.architecture))]
compiler_segments = [("{compiler.name}", "name"),
("{compiler.version}", "versions")]
sigil_compiler_segments = [
("{%compiler.name}", '%' + spec.compiler.name),
("{@compiler.version}", '@' + str(spec.compiler.version))
]
architecture_segments = [("{architecture.platform}", "platform"),
("{architecture.os}", "os"),
("{architecture.target}", "target")]
other_segments = [('{spack_root}', spack.paths.spack_root),
('{spack_install}', spack.store.layout.root),
('{hash:7}', spec.dag_hash(7)),
('{/hash}', '/' + spec.dag_hash())]
for named_str, prop in package_segments:
expected = getattr(spec, prop, "")
actual = spec.format(named_str)
assert str(expected).strip() == actual
for named_str, expected in sigil_package_segments:
actual = spec.format(named_str)
assert expected == actual
compiler = spec.compiler
for named_str, prop in compiler_segments:
expected = getattr(compiler, prop, "")
actual = spec.format(named_str)
assert str(expected) == actual
for named_str, expected in sigil_compiler_segments:
actual = spec.format(named_str)
assert expected == actual
arch = spec.architecture
for named_str, prop in architecture_segments:
expected = getattr(arch, prop, "")
actual = spec.format(named_str)
assert str(expected) == actual
for named_str, expected in other_segments:
actual = spec.format(named_str)
assert expected == actual
def test_ordered_read_not_required_for_consistent_dag_hash(
config, builtin_mock
):
"""Make sure ordered serialization isn't required to preserve hashes.
For consistent hashes, we require that YAML and json documents
have their keys serialized in a deterministic order. However, we
don't want to require them to be serialized in order. This
ensures that is not required.
"""
specs = ['mpileaks ^zmpi', 'dttop', 'dtuse']
for spec in specs:
spec = Spec(spec)
spec.concretize()
#
# Dict & corresponding YAML & JSON from the original spec.
#
spec_dict = spec.to_dict()
spec_yaml = spec.to_yaml()
spec_json = spec.to_json()
#
# Make a spec with reversed OrderedDicts for every
# OrderedDict in the original.
#
reversed_spec_dict = reverse_all_dicts(spec.to_dict())
#
# Dump to YAML and JSON
#
yaml_string = syaml.dump(spec_dict, default_flow_style=False)
reversed_yaml_string = syaml.dump(reversed_spec_dict,
default_flow_style=False)
json_string = sjson.dump(spec_dict)
reversed_json_string = sjson.dump(reversed_spec_dict)
#
# Do many consistency checks
#
# spec yaml is ordered like the spec dict
assert yaml_string == spec_yaml
assert json_string == spec_json
# reversed string is different from the original, so it
# *would* generate a different hash
assert yaml_string != reversed_yaml_string
assert json_string != reversed_json_string
# build specs from the "wrongly" ordered data
round_trip_yaml_spec = Spec.from_yaml(yaml_string)
round_trip_json_spec = Spec.from_json(json_string)
round_trip_reversed_yaml_spec = Spec.from_yaml(
reversed_yaml_string
)
round_trip_reversed_json_spec = Spec.from_yaml(
reversed_json_string
)
# TODO: remove this when build deps are in provenance.
spec = spec.copy(deps=('link', 'run'))
# specs are equal to the original
assert spec == round_trip_yaml_spec
assert spec == round_trip_json_spec
assert spec == round_trip_reversed_yaml_spec
assert spec == round_trip_reversed_json_spec
assert round_trip_yaml_spec == round_trip_reversed_yaml_spec
assert round_trip_json_spec == round_trip_reversed_json_spec
# dag_hashes are equal
assert spec.dag_hash() == round_trip_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_json_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()