def test_splice_with_cached_hashes(self, transitive):
spec = Spec('splice-t')
dep = Spec('splice-h+foo')
spec.concretize()
dep.concretize()
# monkeypatch hashes so we can test that they are cached
spec._full_hash = 'aaaaaa'
spec._build_hash = 'aaaaaa'
dep._full_hash = 'bbbbbb'
dep._build_hash = 'bbbbbb'
spec['splice-h']._full_hash = 'cccccc'
spec['splice-h']._build_hash = 'cccccc'
spec['splice-z']._full_hash = 'dddddd'
spec['splice-z']._build_hash = 'dddddd'
dep['splice-z']._full_hash = 'eeeeee'
dep['splice-z']._build_hash = 'eeeeee'
out = spec.splice(dep, transitive=transitive)
out_z_expected = (dep if transitive else spec)['splice-z']
assert out.full_hash() != spec.full_hash()
assert (out['splice-h'].full_hash() == dep.full_hash()) == transitive
assert out['splice-z'].full_hash() == out_z_expected.full_hash()
assert out.build_hash() != spec.build_hash()
assert (out['splice-h'].build_hash() == dep.build_hash()) == transitive
assert out['splice-z'].build_hash() == out_z_expected.build_hash()
def test_splice_dict_roundtrip(self, transitive):
spec = Spec('splice-t')
dep = Spec('splice-h+foo')
spec.concretize()
dep.concretize()
out = spec.splice(dep, transitive)
# Sanity check all hashes are unique...
assert spec.full_hash() != dep.full_hash()
assert out.full_hash() != dep.full_hash()
assert out.full_hash() != spec.full_hash()
out_rt_spec = Spec.from_dict(out.to_dict()) # rt is "round trip"
assert out_rt_spec.full_hash() == out.full_hash()
out_rt_spec_bld_hash = out_rt_spec.build_spec.full_hash()
out_rt_spec_h_bld_hash = out_rt_spec['splice-h'].build_spec.full_hash()
out_rt_spec_z_bld_hash = out_rt_spec['splice-z'].build_spec.full_hash()
# In any case, the build spec for splice-t (root) should point to the
# original spec, preserving build provenance.
assert spec.full_hash() == out_rt_spec_bld_hash
assert out_rt_spec.full_hash() != out_rt_spec_bld_hash
# The build spec for splice-h should always point to the introduced
# spec, since that is the spec spliced in.
assert dep['splice-h'].full_hash() == out_rt_spec_h_bld_hash
# The build spec for splice-z will depend on whether or not the splice
# was transitive.
expected_z_bld_hash = (dep['splice-z'].full_hash()
if transitive else spec['splice-z'].full_hash())
assert expected_z_bld_hash == out_rt_spec_z_bld_hash
def test_built_spec_cache(tmpdir,
install_mockery):
""" Because the buildcache list command fetches the buildcache index
and uses it to populate the binary_distribution built spec cache, when
this test calls get_mirrors_for_spec, it is testing the popluation of
that cache from a buildcache index. """
global mirror_path_rel
mparser = argparse.ArgumentParser()
mirror.setup_parser(mparser)
margs = mparser.parse_args(
['add', '--scope', 'site', 'test-mirror-rel', 'file://%s' % mirror_path_rel])
mirror.mirror(mparser, margs)
# setup argument parser
parser = argparse.ArgumentParser()
buildcache.setup_parser(parser)
list_args = ['list', '-a', '-l']
args = parser.parse_args(list_args)
buildcache.buildcache(parser, args)
gspec = Spec('garply')
gspec.concretize()
cspec = Spec('corge')
cspec.concretize()
full_hash_map = {
'garply': gspec.full_hash(),
'corge': cspec.full_hash(),
}
gspec_results = bindist.get_mirrors_for_spec(gspec)
gspec_mirrors = {}
for result in gspec_results:
s = result['spec']
assert(s._full_hash == full_hash_map[s.name])
assert(result['mirror_url'] not in gspec_mirrors)
gspec_mirrors[result['mirror_url']] = True
cspec_results = bindist.get_mirrors_for_spec(cspec, full_hash_match=True)
cspec_mirrors = {}
for result in cspec_results:
s = result['spec']
assert(s._full_hash == full_hash_map[s.name])
assert(result['mirror_url'] not in cspec_mirrors)
cspec_mirrors[result['mirror_url']] = True
bindist.clear_spec_cache()
margs = mparser.parse_args(
['rm', '--scope', 'site', 'test-mirror-rel'])
mirror.mirror(mparser, margs)
def test_splice_input_unchanged(self, transitive):
spec = Spec('splice-t').concretized()
dep = Spec('splice-h+foo').concretized()
orig_spec_hash = spec.full_hash()
orig_dep_hash = dep.full_hash()
spec.splice(dep, transitive)
# Post-splice, dag hash should still be different; no changes should be
# made to these specs.
assert spec.full_hash() == orig_spec_hash
assert dep.full_hash() == orig_dep_hash
def test_built_spec_cache(tmpdir, install_mockery):
""" Test what's the situation now """
global mirror_path_rel
mparser = argparse.ArgumentParser()
mirror.setup_parser(mparser)
margs = mparser.parse_args([
'add', '--scope', 'site', 'test-mirror-rel',
'file://%s' % mirror_path_rel
])
mirror.mirror(mparser, margs)
# setup argument parser
parser = argparse.ArgumentParser()
buildcache.setup_parser(parser)
list_args = ['list', '-a', '-l']
args = parser.parse_args(list_args)
buildcache.buildcache(parser, args)
gspec = Spec('garply')
gspec.concretize()
cspec = Spec('corge')
cspec.concretize()
full_hash_map = {
'garply': gspec.full_hash(),
'corge': cspec.full_hash(),
}
gspec_results = bindist.get_spec(gspec)
gspec_mirrors = {}
for result in gspec_results:
s = result['spec']
assert (s._full_hash == full_hash_map[s.name])
assert (result['mirror_url'] not in gspec_mirrors)
gspec_mirrors[result['mirror_url']] = True
cspec_results = bindist.get_spec(cspec, full_hash_match=True)
cspec_mirrors = {}
for result in cspec_results:
s = result['spec']
assert (s._full_hash == full_hash_map[s.name])
assert (result['mirror_url'] not in cspec_mirrors)
cspec_mirrors[result['mirror_url']] = True
bindist.clear_spec_cache()
margs = mparser.parse_args(['rm', '--scope', 'site', 'test-mirror-rel'])
mirror.mirror(mparser, margs)
def test_ci_generate_read_broken_specs_url(tmpdir, mutable_mock_env_path,
env_deactivate, install_mockery,
mock_packages, monkeypatch):
"""Verify that `broken-specs-url` works as intended"""
spec_a = Spec('a')
spec_a.concretize()
a_full_hash = spec_a.full_hash()
spec_flattendeps = Spec('flatten-deps')
spec_flattendeps.concretize()
flattendeps_full_hash = spec_flattendeps.full_hash()
# Mark 'a' as broken (but not 'flatten-deps')
broken_spec_a_path = str(tmpdir.join(a_full_hash))
with open(broken_spec_a_path, 'w') as bsf:
bsf.write('')
# Test that `spack ci generate` notices this broken spec and fails.
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
spack:
specs:
- flatten-deps
- a
mirrors:
some-mirror: https://my.fake.mirror
gitlab-ci:
broken-specs-url: "{0}"
mappings:
- match:
- archive-files
runner-attributes:
tags:
- donotcare
image: donotcare
""".format(tmpdir.strpath))
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
with ev.read('test'):
# Check output of the 'generate' subcommand
output = ci_cmd('generate', output=str, fail_on_error=False)
assert('known to be broken' in output)
ex = '({0})'.format(a_full_hash)
assert(ex in output)
ex = '({0})'.format(flattendeps_full_hash)
assert(ex not in output)
def test_splice(self, transitive):
# Tests the new splice function in Spec using a somewhat simple case
# with a variant with a conditional dependency.
# TODO: Test being able to splice in different provider for a virtual.
# Example: mvapich for mpich.
spec = Spec('splice-t')
dep = Spec('splice-h+foo')
spec.concretize()
dep.concretize()
# Sanity checking that these are not the same thing.
assert dep.dag_hash() != spec['splice-h'].dag_hash()
assert dep.build_hash() != spec['splice-h'].build_hash()
# Do the splice.
out = spec.splice(dep, transitive)
# Returned spec should still be concrete.
assert out.concrete
# Traverse the spec and assert that all dependencies are accounted for.
for node in spec.traverse():
assert node.name in out
# If the splice worked, then the full hash of the spliced dep should
# now match the full hash of the build spec of the dependency from the
# returned spec.
out_h_build = out['splice-h'].build_spec
assert out_h_build.full_hash() == dep.full_hash()
# Transitivity should determine whether the transitive dependency was
# changed.
expected_z = dep['splice-z'] if transitive else spec['splice-z']
assert out['splice-z'].full_hash() == expected_z.full_hash()
# Sanity check build spec of out should be the original spec.
assert (out['splice-t'].build_spec.full_hash() ==
spec['splice-t'].full_hash())
# Finally, the spec should know it's been spliced:
assert out.spliced
def test_built_spec_cache(mirror_dir):
""" Because the buildcache list command fetches the buildcache index
and uses it to populate the binary_distribution built spec cache, when
this test calls get_mirrors_for_spec, it is testing the popluation of
that cache from a buildcache index. """
buildcache_cmd('list', '-a', '-l')
gspec, cspec = Spec('garply').concretized(), Spec('corge').concretized()
full_hash_map = {
'garply': gspec.full_hash(),
'corge': cspec.full_hash(),
}
gspec_results = bindist.get_mirrors_for_spec(gspec)
gspec_mirrors = {}
for result in gspec_results:
s = result['spec']
assert (s._full_hash == full_hash_map[s.name])
assert (result['mirror_url'] not in gspec_mirrors)
gspec_mirrors[result['mirror_url']] = True
cspec_results = bindist.get_mirrors_for_spec(cspec, full_hash_match=True)
cspec_mirrors = {}
for result in cspec_results:
s = result['spec']
assert (s._full_hash == full_hash_map[s.name])
assert (result['mirror_url'] not in cspec_mirrors)
cspec_mirrors[result['mirror_url']] = True
def test_external_packages_have_consistent_hash(self):
if spack.config.get('config:concretizer') == 'original':
pytest.skip('This tests needs the ASP-based concretizer')
s, t = Spec('externaltool'), Spec('externaltool')
s._old_concretize(), t._new_concretize()
assert s.dag_hash() == t.dag_hash()
assert s.build_hash() == t.build_hash()
assert s.full_hash() == t.full_hash()
def test_splice_dict(self, transitive):
spec = Spec('splice-t')
dep = Spec('splice-h+foo')
spec.concretize()
dep.concretize()
out = spec.splice(dep, transitive)
# Sanity check all hashes are unique...
assert spec.full_hash() != dep.full_hash()
assert out.full_hash() != dep.full_hash()
assert out.full_hash() != spec.full_hash()
node_list = out.to_dict()['spec']['nodes']
root_nodes = [n for n in node_list if n['full_hash'] == out.full_hash()]
build_spec_nodes = [n for n in node_list if n['full_hash'] == spec.full_hash()]
assert spec.full_hash() == out.build_spec.full_hash()
assert len(root_nodes) == 1
assert len(build_spec_nodes) == 1
def test_ordered_read_not_required_for_consistent_dag_hash(
config, mock_packages):
"""Make sure ordered serialization isn't required to preserve hashes.
For consistent hashes, we require that YAML and json documents
have their keys serialized in a deterministic order. However, we
don't want to require them to be serialized in order. This
ensures that is not required.
"""
specs = ['mpileaks ^zmpi', 'dttop', 'dtuse']
for spec in specs:
spec = Spec(spec)
spec.concretize()
#
# Dict & corresponding YAML & JSON from the original spec.
#
spec_dict = spec.to_dict()
spec_yaml = spec.to_yaml()
spec_json = spec.to_json()
#
# Make a spec with reversed OrderedDicts for every
# OrderedDict in the original.
#
reversed_spec_dict = reverse_all_dicts(spec.to_dict())
#
# Dump to YAML and JSON
#
yaml_string = syaml.dump(spec_dict, default_flow_style=False)
reversed_yaml_string = syaml.dump(reversed_spec_dict,
default_flow_style=False)
json_string = sjson.dump(spec_dict)
reversed_json_string = sjson.dump(reversed_spec_dict)
#
# Do many consistency checks
#
# spec yaml is ordered like the spec dict
assert yaml_string == spec_yaml
assert json_string == spec_json
# reversed string is different from the original, so it
# *would* generate a different hash
assert yaml_string != reversed_yaml_string
assert json_string != reversed_json_string
# build specs from the "wrongly" ordered data
round_trip_yaml_spec = Spec.from_yaml(yaml_string)
round_trip_json_spec = Spec.from_json(json_string)
round_trip_reversed_yaml_spec = Spec.from_yaml(reversed_yaml_string)
round_trip_reversed_json_spec = Spec.from_yaml(reversed_json_string)
# TODO: remove this when build deps are in provenance.
spec = spec.copy(deps=('link', 'run'))
# specs are equal to the original
assert spec == round_trip_yaml_spec
assert spec == round_trip_json_spec
assert spec == round_trip_reversed_yaml_spec
assert spec == round_trip_reversed_json_spec
assert round_trip_yaml_spec == round_trip_reversed_yaml_spec
assert round_trip_json_spec == round_trip_reversed_json_spec
# dag_hashes are equal
assert spec.dag_hash() == round_trip_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_json_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()
# full_hashes are equal
spec.concretize()
round_trip_yaml_spec.concretize()
round_trip_json_spec.concretize()
round_trip_reversed_yaml_spec.concretize()
round_trip_reversed_json_spec.concretize()
assert spec.full_hash() == round_trip_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_json_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_json_spec.full_hash()
def test_ordered_read_not_required_for_consistent_dag_hash(
config, mock_packages
):
"""Make sure ordered serialization isn't required to preserve hashes.
For consistent hashes, we require that YAML and json documents
have their keys serialized in a deterministic order. However, we
don't want to require them to be serialized in order. This
ensures that is not required.
"""
specs = ['mpileaks ^zmpi', 'dttop', 'dtuse']
for spec in specs:
spec = Spec(spec)
spec.concretize()
#
# Dict & corresponding YAML & JSON from the original spec.
#
spec_dict = spec.to_dict()
spec_yaml = spec.to_yaml()
spec_json = spec.to_json()
#
# Make a spec with reversed OrderedDicts for every
# OrderedDict in the original.
#
reversed_spec_dict = reverse_all_dicts(spec.to_dict())
#
# Dump to YAML and JSON
#
yaml_string = syaml.dump(spec_dict, default_flow_style=False)
reversed_yaml_string = syaml.dump(reversed_spec_dict,
default_flow_style=False)
json_string = sjson.dump(spec_dict)
reversed_json_string = sjson.dump(reversed_spec_dict)
#
# Do many consistency checks
#
# spec yaml is ordered like the spec dict
assert yaml_string == spec_yaml
assert json_string == spec_json
# reversed string is different from the original, so it
# *would* generate a different hash
assert yaml_string != reversed_yaml_string
assert json_string != reversed_json_string
# build specs from the "wrongly" ordered data
round_trip_yaml_spec = Spec.from_yaml(yaml_string)
round_trip_json_spec = Spec.from_json(json_string)
round_trip_reversed_yaml_spec = Spec.from_yaml(
reversed_yaml_string
)
round_trip_reversed_json_spec = Spec.from_yaml(
reversed_json_string
)
# TODO: remove this when build deps are in provenance.
spec = spec.copy(deps=('link', 'run'))
# specs are equal to the original
assert spec == round_trip_yaml_spec
assert spec == round_trip_json_spec
assert spec == round_trip_reversed_yaml_spec
assert spec == round_trip_reversed_json_spec
assert round_trip_yaml_spec == round_trip_reversed_yaml_spec
assert round_trip_json_spec == round_trip_reversed_json_spec
# dag_hashes are equal
assert spec.dag_hash() == round_trip_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_json_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()
# full_hashes are equal
spec.concretize()
round_trip_yaml_spec.concretize()
round_trip_json_spec.concretize()
round_trip_reversed_yaml_spec.concretize()
round_trip_reversed_json_spec.concretize()
assert spec.full_hash() == round_trip_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_json_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_json_spec.full_hash()
