def test_yaml_subdag(config, mock_packages):
spec = Spec('mpileaks^mpich+debug')
spec.concretize()
yaml_spec = Spec.from_yaml(spec.to_yaml())
json_spec = Spec.from_json(spec.to_json())
for dep in ('callpath', 'mpich', 'dyninst', 'libdwarf', 'libelf'):
assert spec[dep].eq_dag(yaml_spec[dep])
assert spec[dep].eq_dag(json_spec[dep])
def test_ordered_read_not_required_for_consistent_dag_hash(
config, mock_packages):
"""Make sure ordered serialization isn't required to preserve hashes.
For consistent hashes, we require that YAML and json documents
have their keys serialized in a deterministic order. However, we
don't want to require them to be serialized in order. This
ensures that is not required.
"""
specs = ['mpileaks ^zmpi', 'dttop', 'dtuse']
for spec in specs:
spec = Spec(spec)
spec.concretize()
#
# Dict & corresponding YAML & JSON from the original spec.
#
spec_dict = spec.to_dict()
spec_yaml = spec.to_yaml()
spec_json = spec.to_json()
#
# Make a spec with reversed OrderedDicts for every
# OrderedDict in the original.
#
reversed_spec_dict = reverse_all_dicts(spec.to_dict())
#
# Dump to YAML and JSON
#
yaml_string = syaml.dump(spec_dict, default_flow_style=False)
reversed_yaml_string = syaml.dump(reversed_spec_dict,
default_flow_style=False)
json_string = sjson.dump(spec_dict)
reversed_json_string = sjson.dump(reversed_spec_dict)
#
# Do many consistency checks
#
# spec yaml is ordered like the spec dict
assert yaml_string == spec_yaml
assert json_string == spec_json
# reversed string is different from the original, so it
# *would* generate a different hash
assert yaml_string != reversed_yaml_string
assert json_string != reversed_json_string
# build specs from the "wrongly" ordered data
round_trip_yaml_spec = Spec.from_yaml(yaml_string)
round_trip_json_spec = Spec.from_json(json_string)
round_trip_reversed_yaml_spec = Spec.from_yaml(reversed_yaml_string)
round_trip_reversed_json_spec = Spec.from_yaml(reversed_json_string)
# TODO: remove this when build deps are in provenance.
spec = spec.copy(deps=('link', 'run'))
# specs are equal to the original
assert spec == round_trip_yaml_spec
assert spec == round_trip_json_spec
assert spec == round_trip_reversed_yaml_spec
assert spec == round_trip_reversed_json_spec
assert round_trip_yaml_spec == round_trip_reversed_yaml_spec
assert round_trip_json_spec == round_trip_reversed_json_spec
# dag_hashes are equal
assert spec.dag_hash() == round_trip_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_json_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()
# full_hashes are equal
spec.concretize()
round_trip_yaml_spec.concretize()
round_trip_json_spec.concretize()
round_trip_reversed_yaml_spec.concretize()
round_trip_reversed_json_spec.concretize()
assert spec.full_hash() == round_trip_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_json_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_json_spec.full_hash()
def check_json_round_trip(spec):
json_text = spec.to_json()
spec_from_json = Spec.from_json(json_text)
assert spec.eq_dag(spec_from_json)
def test_ordered_read_not_required_for_consistent_dag_hash(
config, builtin_mock
):
"""Make sure ordered serialization isn't required to preserve hashes.
For consistent hashes, we require that YAML and json documents
have their keys serialized in a deterministic order. However, we
don't want to require them to be serialized in order. This
ensures that is not required.
"""
specs = ['mpileaks ^zmpi', 'dttop', 'dtuse']
for spec in specs:
spec = Spec(spec)
spec.concretize()
#
# Dict & corresponding YAML & JSON from the original spec.
#
spec_dict = spec.to_dict()
spec_yaml = spec.to_yaml()
spec_json = spec.to_json()
#
# Make a spec with reversed OrderedDicts for every
# OrderedDict in the original.
#
reversed_spec_dict = reverse_all_dicts(spec.to_dict())
#
# Dump to YAML and JSON
#
yaml_string = syaml.dump(spec_dict, default_flow_style=False)
reversed_yaml_string = syaml.dump(reversed_spec_dict,
default_flow_style=False)
json_string = sjson.dump(spec_dict)
reversed_json_string = sjson.dump(reversed_spec_dict)
#
# Do many consistency checks
#
# spec yaml is ordered like the spec dict
assert yaml_string == spec_yaml
assert json_string == spec_json
# reversed string is different from the original, so it
# *would* generate a different hash
assert yaml_string != reversed_yaml_string
assert json_string != reversed_json_string
# build specs from the "wrongly" ordered data
round_trip_yaml_spec = Spec.from_yaml(yaml_string)
round_trip_json_spec = Spec.from_json(json_string)
round_trip_reversed_yaml_spec = Spec.from_yaml(
reversed_yaml_string
)
round_trip_reversed_json_spec = Spec.from_yaml(
reversed_json_string
)
# TODO: remove this when build deps are in provenance.
spec = spec.copy(deps=('link', 'run'))
# specs are equal to the original
assert spec == round_trip_yaml_spec
assert spec == round_trip_json_spec
assert spec == round_trip_reversed_yaml_spec
assert spec == round_trip_reversed_json_spec
assert round_trip_yaml_spec == round_trip_reversed_yaml_spec
assert round_trip_json_spec == round_trip_reversed_json_spec
# dag_hashes are equal
assert spec.dag_hash() == round_trip_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_json_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()
def test_invalid_json_spec(invalid_json, error_message):
with pytest.raises(sjson.SpackJSONError) as e:
Spec.from_json(invalid_json)
exc_msg = str(e.value)
assert exc_msg.startswith("error parsing JSON spec:")
assert error_message in exc_msg
def _createtarball(env,
spec_file=None,
packages=None,
add_spec=True,
add_deps=True,
output_location=os.getcwd(),
signing_key=None,
force=False,
make_relative=False,
unsigned=False,
allow_root=False,
rebuild_index=False):
if spec_file:
with open(spec_file, 'r') as fd:
specfile_contents = fd.read()
tty.debug('createtarball read specfile contents:')
tty.debug(specfile_contents)
if spec_file.endswith('.json'):
s = Spec.from_json(specfile_contents)
else:
s = Spec.from_yaml(specfile_contents)
package = '/{0}'.format(s.dag_hash())
matches = find_matching_specs(package, env=env)
elif packages:
matches = find_matching_specs(packages, env=env)
elif env:
matches = [env.specs_by_hash[h] for h in env.concretized_order]
else:
tty.die("build cache file creation requires at least one" +
" installed package spec, an active environment," +
" or else a path to a json or yaml file containing a spec" +
" to install")
specs = set()
mirror = spack.mirror.MirrorCollection().lookup(output_location)
outdir = url_util.format(mirror.push_url)
msg = 'Buildcache files will be output to %s/build_cache' % outdir
tty.msg(msg)
if matches:
tty.debug('Found at least one matching spec')
for match in matches:
tty.debug('examining match {0}'.format(match.format()))
if match.external or match.virtual:
tty.debug('skipping external or virtual spec %s' % match.format())
else:
lookup = spack.store.db.query_one(match)
if not add_spec:
tty.debug('skipping matching root spec %s' % match.format())
elif lookup is None:
tty.debug('skipping uninstalled matching spec %s' %
match.format())
else:
tty.debug('adding matching spec %s' % match.format())
specs.add(match)
if not add_deps:
continue
tty.debug('recursing dependencies')
for d, node in match.traverse(order='post',
depth=True,
deptype=('link', 'run')):
# skip root, since it's handled above
if d == 0:
continue
lookup = spack.store.db.query_one(node)
if node.external or node.virtual:
tty.debug('skipping external or virtual dependency %s' %
node.format())
elif lookup is None:
tty.debug('skipping uninstalled depenendency %s' %
node.format())
else:
tty.debug('adding dependency %s' % node.format())
specs.add(node)
tty.debug('writing tarballs to %s/build_cache' % outdir)
for spec in specs:
tty.debug('creating binary cache file for package %s ' % spec.format())
try:
bindist.build_tarball(spec, outdir, force, make_relative, unsigned,
allow_root, signing_key, rebuild_index)
except bindist.NoOverwriteException as e:
tty.warn(e)
