def test_splice_with_cached_hashes(self, transitive): spec = Spec('splice-t') dep = Spec('splice-h+foo') spec.concretize() dep.concretize() # monkeypatch hashes so we can test that they are cached spec._full_hash = 'aaaaaa' spec._build_hash = 'aaaaaa' dep._full_hash = 'bbbbbb' dep._build_hash = 'bbbbbb' spec['splice-h']._full_hash = 'cccccc' spec['splice-h']._build_hash = 'cccccc' spec['splice-z']._full_hash = 'dddddd' spec['splice-z']._build_hash = 'dddddd' dep['splice-z']._full_hash = 'eeeeee' dep['splice-z']._build_hash = 'eeeeee' out = spec.splice(dep, transitive=transitive) out_z_expected = (dep if transitive else spec)['splice-z'] assert out.full_hash() != spec.full_hash() assert (out['splice-h'].full_hash() == dep.full_hash()) == transitive assert out['splice-z'].full_hash() == out_z_expected.full_hash() assert out.build_hash() != spec.build_hash() assert (out['splice-h'].build_hash() == dep.build_hash()) == transitive assert out['splice-z'].build_hash() == out_z_expected.build_hash()
def test_splice_dict_roundtrip(self, transitive): spec = Spec('splice-t') dep = Spec('splice-h+foo') spec.concretize() dep.concretize() out = spec.splice(dep, transitive) # Sanity check all hashes are unique... assert spec.full_hash() != dep.full_hash() assert out.full_hash() != dep.full_hash() assert out.full_hash() != spec.full_hash() out_rt_spec = Spec.from_dict(out.to_dict()) # rt is "round trip" assert out_rt_spec.full_hash() == out.full_hash() out_rt_spec_bld_hash = out_rt_spec.build_spec.full_hash() out_rt_spec_h_bld_hash = out_rt_spec['splice-h'].build_spec.full_hash() out_rt_spec_z_bld_hash = out_rt_spec['splice-z'].build_spec.full_hash() # In any case, the build spec for splice-t (root) should point to the # original spec, preserving build provenance. assert spec.full_hash() == out_rt_spec_bld_hash assert out_rt_spec.full_hash() != out_rt_spec_bld_hash # The build spec for splice-h should always point to the introduced # spec, since that is the spec spliced in. assert dep['splice-h'].full_hash() == out_rt_spec_h_bld_hash # The build spec for splice-z will depend on whether or not the splice # was transitive. expected_z_bld_hash = (dep['splice-z'].full_hash() if transitive else spec['splice-z'].full_hash()) assert expected_z_bld_hash == out_rt_spec_z_bld_hash
def test_built_spec_cache(tmpdir, install_mockery): """ Because the buildcache list command fetches the buildcache index and uses it to populate the binary_distribution built spec cache, when this test calls get_mirrors_for_spec, it is testing the popluation of that cache from a buildcache index. """ global mirror_path_rel mparser = argparse.ArgumentParser() mirror.setup_parser(mparser) margs = mparser.parse_args( ['add', '--scope', 'site', 'test-mirror-rel', 'file://%s' % mirror_path_rel]) mirror.mirror(mparser, margs) # setup argument parser parser = argparse.ArgumentParser() buildcache.setup_parser(parser) list_args = ['list', '-a', '-l'] args = parser.parse_args(list_args) buildcache.buildcache(parser, args) gspec = Spec('garply') gspec.concretize() cspec = Spec('corge') cspec.concretize() full_hash_map = { 'garply': gspec.full_hash(), 'corge': cspec.full_hash(), } gspec_results = bindist.get_mirrors_for_spec(gspec) gspec_mirrors = {} for result in gspec_results: s = result['spec'] assert(s._full_hash == full_hash_map[s.name]) assert(result['mirror_url'] not in gspec_mirrors) gspec_mirrors[result['mirror_url']] = True cspec_results = bindist.get_mirrors_for_spec(cspec, full_hash_match=True) cspec_mirrors = {} for result in cspec_results: s = result['spec'] assert(s._full_hash == full_hash_map[s.name]) assert(result['mirror_url'] not in cspec_mirrors) cspec_mirrors[result['mirror_url']] = True bindist.clear_spec_cache() margs = mparser.parse_args( ['rm', '--scope', 'site', 'test-mirror-rel']) mirror.mirror(mparser, margs)
def test_splice_input_unchanged(self, transitive): spec = Spec('splice-t').concretized() dep = Spec('splice-h+foo').concretized() orig_spec_hash = spec.full_hash() orig_dep_hash = dep.full_hash() spec.splice(dep, transitive) # Post-splice, dag hash should still be different; no changes should be # made to these specs. assert spec.full_hash() == orig_spec_hash assert dep.full_hash() == orig_dep_hash
def test_built_spec_cache(tmpdir, install_mockery): """ Test what's the situation now """ global mirror_path_rel mparser = argparse.ArgumentParser() mirror.setup_parser(mparser) margs = mparser.parse_args([ 'add', '--scope', 'site', 'test-mirror-rel', 'file://%s' % mirror_path_rel ]) mirror.mirror(mparser, margs) # setup argument parser parser = argparse.ArgumentParser() buildcache.setup_parser(parser) list_args = ['list', '-a', '-l'] args = parser.parse_args(list_args) buildcache.buildcache(parser, args) gspec = Spec('garply') gspec.concretize() cspec = Spec('corge') cspec.concretize() full_hash_map = { 'garply': gspec.full_hash(), 'corge': cspec.full_hash(), } gspec_results = bindist.get_spec(gspec) gspec_mirrors = {} for result in gspec_results: s = result['spec'] assert (s._full_hash == full_hash_map[s.name]) assert (result['mirror_url'] not in gspec_mirrors) gspec_mirrors[result['mirror_url']] = True cspec_results = bindist.get_spec(cspec, full_hash_match=True) cspec_mirrors = {} for result in cspec_results: s = result['spec'] assert (s._full_hash == full_hash_map[s.name]) assert (result['mirror_url'] not in cspec_mirrors) cspec_mirrors[result['mirror_url']] = True bindist.clear_spec_cache() margs = mparser.parse_args(['rm', '--scope', 'site', 'test-mirror-rel']) mirror.mirror(mparser, margs)
def test_ci_generate_read_broken_specs_url(tmpdir, mutable_mock_env_path, env_deactivate, install_mockery, mock_packages, monkeypatch): """Verify that `broken-specs-url` works as intended""" spec_a = Spec('a') spec_a.concretize() a_full_hash = spec_a.full_hash() spec_flattendeps = Spec('flatten-deps') spec_flattendeps.concretize() flattendeps_full_hash = spec_flattendeps.full_hash() # Mark 'a' as broken (but not 'flatten-deps') broken_spec_a_path = str(tmpdir.join(a_full_hash)) with open(broken_spec_a_path, 'w') as bsf: bsf.write('') # Test that `spack ci generate` notices this broken spec and fails. filename = str(tmpdir.join('spack.yaml')) with open(filename, 'w') as f: f.write("""\ spack: specs: - flatten-deps - a mirrors: some-mirror: https://my.fake.mirror gitlab-ci: broken-specs-url: "{0}" mappings: - match: - archive-files runner-attributes: tags: - donotcare image: donotcare """.format(tmpdir.strpath)) with tmpdir.as_cwd(): env_cmd('create', 'test', './spack.yaml') with ev.read('test'): # Check output of the 'generate' subcommand output = ci_cmd('generate', output=str, fail_on_error=False) assert('known to be broken' in output) ex = '({0})'.format(a_full_hash) assert(ex in output) ex = '({0})'.format(flattendeps_full_hash) assert(ex not in output)
def test_splice(self, transitive): # Tests the new splice function in Spec using a somewhat simple case # with a variant with a conditional dependency. # TODO: Test being able to splice in different provider for a virtual. # Example: mvapich for mpich. spec = Spec('splice-t') dep = Spec('splice-h+foo') spec.concretize() dep.concretize() # Sanity checking that these are not the same thing. assert dep.dag_hash() != spec['splice-h'].dag_hash() assert dep.build_hash() != spec['splice-h'].build_hash() # Do the splice. out = spec.splice(dep, transitive) # Returned spec should still be concrete. assert out.concrete # Traverse the spec and assert that all dependencies are accounted for. for node in spec.traverse(): assert node.name in out # If the splice worked, then the full hash of the spliced dep should # now match the full hash of the build spec of the dependency from the # returned spec. out_h_build = out['splice-h'].build_spec assert out_h_build.full_hash() == dep.full_hash() # Transitivity should determine whether the transitive dependency was # changed. expected_z = dep['splice-z'] if transitive else spec['splice-z'] assert out['splice-z'].full_hash() == expected_z.full_hash() # Sanity check build spec of out should be the original spec. assert (out['splice-t'].build_spec.full_hash() == spec['splice-t'].full_hash()) # Finally, the spec should know it's been spliced: assert out.spliced
def test_built_spec_cache(mirror_dir): """ Because the buildcache list command fetches the buildcache index and uses it to populate the binary_distribution built spec cache, when this test calls get_mirrors_for_spec, it is testing the popluation of that cache from a buildcache index. """ buildcache_cmd('list', '-a', '-l') gspec, cspec = Spec('garply').concretized(), Spec('corge').concretized() full_hash_map = { 'garply': gspec.full_hash(), 'corge': cspec.full_hash(), } gspec_results = bindist.get_mirrors_for_spec(gspec) gspec_mirrors = {} for result in gspec_results: s = result['spec'] assert (s._full_hash == full_hash_map[s.name]) assert (result['mirror_url'] not in gspec_mirrors) gspec_mirrors[result['mirror_url']] = True cspec_results = bindist.get_mirrors_for_spec(cspec, full_hash_match=True) cspec_mirrors = {} for result in cspec_results: s = result['spec'] assert (s._full_hash == full_hash_map[s.name]) assert (result['mirror_url'] not in cspec_mirrors) cspec_mirrors[result['mirror_url']] = True
def test_external_packages_have_consistent_hash(self): if spack.config.get('config:concretizer') == 'original': pytest.skip('This tests needs the ASP-based concretizer') s, t = Spec('externaltool'), Spec('externaltool') s._old_concretize(), t._new_concretize() assert s.dag_hash() == t.dag_hash() assert s.build_hash() == t.build_hash() assert s.full_hash() == t.full_hash()
def test_splice_dict(self, transitive): spec = Spec('splice-t') dep = Spec('splice-h+foo') spec.concretize() dep.concretize() out = spec.splice(dep, transitive) # Sanity check all hashes are unique... assert spec.full_hash() != dep.full_hash() assert out.full_hash() != dep.full_hash() assert out.full_hash() != spec.full_hash() node_list = out.to_dict()['spec']['nodes'] root_nodes = [n for n in node_list if n['full_hash'] == out.full_hash()] build_spec_nodes = [n for n in node_list if n['full_hash'] == spec.full_hash()] assert spec.full_hash() == out.build_spec.full_hash() assert len(root_nodes) == 1 assert len(build_spec_nodes) == 1
def test_ordered_read_not_required_for_consistent_dag_hash( config, mock_packages): """Make sure ordered serialization isn't required to preserve hashes. For consistent hashes, we require that YAML and json documents have their keys serialized in a deterministic order. However, we don't want to require them to be serialized in order. This ensures that is not required. """ specs = ['mpileaks ^zmpi', 'dttop', 'dtuse'] for spec in specs: spec = Spec(spec) spec.concretize() # # Dict & corresponding YAML & JSON from the original spec. # spec_dict = spec.to_dict() spec_yaml = spec.to_yaml() spec_json = spec.to_json() # # Make a spec with reversed OrderedDicts for every # OrderedDict in the original. # reversed_spec_dict = reverse_all_dicts(spec.to_dict()) # # Dump to YAML and JSON # yaml_string = syaml.dump(spec_dict, default_flow_style=False) reversed_yaml_string = syaml.dump(reversed_spec_dict, default_flow_style=False) json_string = sjson.dump(spec_dict) reversed_json_string = sjson.dump(reversed_spec_dict) # # Do many consistency checks # # spec yaml is ordered like the spec dict assert yaml_string == spec_yaml assert json_string == spec_json # reversed string is different from the original, so it # *would* generate a different hash assert yaml_string != reversed_yaml_string assert json_string != reversed_json_string # build specs from the "wrongly" ordered data round_trip_yaml_spec = Spec.from_yaml(yaml_string) round_trip_json_spec = Spec.from_json(json_string) round_trip_reversed_yaml_spec = Spec.from_yaml(reversed_yaml_string) round_trip_reversed_json_spec = Spec.from_yaml(reversed_json_string) # TODO: remove this when build deps are in provenance. spec = spec.copy(deps=('link', 'run')) # specs are equal to the original assert spec == round_trip_yaml_spec assert spec == round_trip_json_spec assert spec == round_trip_reversed_yaml_spec assert spec == round_trip_reversed_json_spec assert round_trip_yaml_spec == round_trip_reversed_yaml_spec assert round_trip_json_spec == round_trip_reversed_json_spec # dag_hashes are equal assert spec.dag_hash() == round_trip_yaml_spec.dag_hash() assert spec.dag_hash() == round_trip_json_spec.dag_hash() assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash() assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash() # full_hashes are equal spec.concretize() round_trip_yaml_spec.concretize() round_trip_json_spec.concretize() round_trip_reversed_yaml_spec.concretize() round_trip_reversed_json_spec.concretize() assert spec.full_hash() == round_trip_yaml_spec.full_hash() assert spec.full_hash() == round_trip_json_spec.full_hash() assert spec.full_hash() == round_trip_reversed_yaml_spec.full_hash() assert spec.full_hash() == round_trip_reversed_json_spec.full_hash()
def test_ordered_read_not_required_for_consistent_dag_hash( config, mock_packages ): """Make sure ordered serialization isn't required to preserve hashes. For consistent hashes, we require that YAML and json documents have their keys serialized in a deterministic order. However, we don't want to require them to be serialized in order. This ensures that is not required. """ specs = ['mpileaks ^zmpi', 'dttop', 'dtuse'] for spec in specs: spec = Spec(spec) spec.concretize() # # Dict & corresponding YAML & JSON from the original spec. # spec_dict = spec.to_dict() spec_yaml = spec.to_yaml() spec_json = spec.to_json() # # Make a spec with reversed OrderedDicts for every # OrderedDict in the original. # reversed_spec_dict = reverse_all_dicts(spec.to_dict()) # # Dump to YAML and JSON # yaml_string = syaml.dump(spec_dict, default_flow_style=False) reversed_yaml_string = syaml.dump(reversed_spec_dict, default_flow_style=False) json_string = sjson.dump(spec_dict) reversed_json_string = sjson.dump(reversed_spec_dict) # # Do many consistency checks # # spec yaml is ordered like the spec dict assert yaml_string == spec_yaml assert json_string == spec_json # reversed string is different from the original, so it # *would* generate a different hash assert yaml_string != reversed_yaml_string assert json_string != reversed_json_string # build specs from the "wrongly" ordered data round_trip_yaml_spec = Spec.from_yaml(yaml_string) round_trip_json_spec = Spec.from_json(json_string) round_trip_reversed_yaml_spec = Spec.from_yaml( reversed_yaml_string ) round_trip_reversed_json_spec = Spec.from_yaml( reversed_json_string ) # TODO: remove this when build deps are in provenance. spec = spec.copy(deps=('link', 'run')) # specs are equal to the original assert spec == round_trip_yaml_spec assert spec == round_trip_json_spec assert spec == round_trip_reversed_yaml_spec assert spec == round_trip_reversed_json_spec assert round_trip_yaml_spec == round_trip_reversed_yaml_spec assert round_trip_json_spec == round_trip_reversed_json_spec # dag_hashes are equal assert spec.dag_hash() == round_trip_yaml_spec.dag_hash() assert spec.dag_hash() == round_trip_json_spec.dag_hash() assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash() assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash() # full_hashes are equal spec.concretize() round_trip_yaml_spec.concretize() round_trip_json_spec.concretize() round_trip_reversed_yaml_spec.concretize() round_trip_reversed_json_spec.concretize() assert spec.full_hash() == round_trip_yaml_spec.full_hash() assert spec.full_hash() == round_trip_json_spec.full_hash() assert spec.full_hash() == round_trip_reversed_yaml_spec.full_hash() assert spec.full_hash() == round_trip_reversed_json_spec.full_hash()