def __init__(self, root, layout, **kwargs):
super(YamlFilesystemView, self).__init__(root, layout, **kwargs)
# Super class gets projections from the kwargs
# YAML specific to get projections from YAML file
projections_path = os.path.join(self._root, _projections_path)
if not self.projections:
if os.path.exists(projections_path):
# Read projections file from view
with open(projections_path, 'r') as f:
projections_data = s_yaml.load(f)
spack.config.validate(projections_data,
spack.schema.projections.schema)
self.projections = projections_data['projections']
else:
# Write projections file to new view
# Not strictly necessary as the empty file is the empty
# projection but it makes sense for consistency
mkdirp(os.path.dirname(projections_path))
with open(projections_path, 'w') as f:
f.write(s_yaml.dump({'projections': self.projections}))
elif not os.path.exists(projections_path):
# Write projections file to new view
mkdirp(os.path.dirname(projections_path))
with open(projections_path, 'w') as f:
f.write(s_yaml.dump({'projections': self.projections}))
else:
msg = 'View at %s has projections file' % self._root
msg += ' and was passed projections manually.'
raise ConflictingProjectionsError(msg)
self.extensions_layout = YamlViewExtensionsLayout(self, layout)
self._croot = colorize_root(self._root) + " "
def to_yaml(self, stream=None):
provider_list = self._transform(
lambda vpkg, pset: [
vpkg.to_node_dict(), [p.to_node_dict() for p in pset]], list)
syaml.dump({'provider_index': {'providers': provider_list}},
stream=stream)
def add_view(env, extension, link_type):
view_path = os.path.join(os.environ['SPACK_MANAGER'], 'views', extension,
'snapshot')
view_dict = {
'snapshot': {
'root': view_path,
'projections': {
'all':
'{compiler.name}-{compiler.version}/{name}/'
'{version}-{hash:4}',
'^cuda':
'{compiler.name}-{compiler.version}-'
'{^cuda.name}-{^cuda.version}/{name}/{version}'
'-{hash:4}',
'^rocm':
'{compiler.name}-{compiler.version}-'
'{^rocm.name}-{^rocm.version}/{name}/{version}'
'-{hash:4}'
},
'link_type': link_type
}
}
with open(env.manifest_path, 'r') as f:
yaml = syaml.load(f)
# view yaml entry can also be a bool so first try to add to a dictionary,
# and if that fails overwrite entry as a dictionary
try:
yaml['spack']['view'].update(view_dict)
except AttributeError:
yaml['spack']['view'] = view_dict
with open(env.manifest_path, 'w') as f:
syaml.dump(yaml, stream=f, default_flow_style=False)
def to_yaml(self, stream=None):
provider_list = self._transform(
lambda vpkg, pset: [
vpkg.to_node_dict(), [p.to_node_dict() for p in pset]], list)
syaml.dump({'provider_index': {'providers': provider_list}},
stream=stream)
def print_section(section):
"""Print a configuration to stdout."""
try:
data = syaml.syaml_dict()
data[section] = get_config(section)
syaml.dump(data, stream=sys.stdout, default_flow_style=False)
except (yaml.YAMLError, IOError) as e:
raise ConfigError("Error reading configuration: %s" % section)
def print_section(section):
"""Print a configuration to stdout."""
try:
data = syaml.syaml_dict()
data[section] = get_config(section)
syaml.dump(data, stream=sys.stdout, default_flow_style=False)
except (yaml.YAMLError, IOError) as e:
raise ConfigError("Error reading configuration: %s" % section)
def generate_module_index(root, modules):
entries = syaml.syaml_dict()
for m in modules:
entry = {'path': m.layout.filename, 'use_name': m.layout.use_name}
entries[m.spec.dag_hash()] = entry
index = {'module_index': entries}
index_path = os.path.join(root, 'module-index.yaml')
llnl.util.filesystem.mkdirp(root)
with open(index_path, 'w') as index_file:
syaml.dump(index, default_flow_style=False, stream=index_file)
def write_section(self, section):
filename = self.get_section_filename(section)
data = self.get_section(section)
try:
mkdirp(self.path)
with open(filename, 'w') as f:
_validate_section(data, section_schemas[section])
syaml.dump(data, stream=f, default_flow_style=False)
except (yaml.YAMLError, IOError) as e:
raise ConfigFileError(
"Error writing to config file: '%s'" % str(e))
def write_section(self, section):
filename = self.get_section_filename(section)
data = self.get_section(section)
try:
mkdirp(self.path)
with open(filename, 'w') as f:
_validate_section(data, section_schemas[section])
syaml.dump(data, stream=f, default_flow_style=False)
except (yaml.YAMLError, IOError) as e:
raise ConfigFileError("Error writing to config file: '%s'" %
str(e))
def write_section(self, section):
_validate(self.sections, self.schema)
try:
parent = os.path.dirname(self.path)
mkdirp(parent)
tmp = os.path.join(parent, '.%s.tmp' % self.path)
with open(tmp, 'w') as f:
syaml.dump(self.sections, stream=f, default_flow_style=False)
os.path.move(tmp, self.path)
except (yaml.YAMLError, IOError) as e:
raise ConfigFileError(
"Error writing to config file: '%s'" % str(e))
def write_section(self, section):
validate(self.sections, self.schema)
try:
parent = os.path.dirname(self.path)
mkdirp(parent)
tmp = os.path.join(parent, '.%s.tmp' % self.path)
with open(tmp, 'w') as f:
syaml.dump(self.sections, stream=f, default_flow_style=False)
os.path.move(tmp, self.path)
except (yaml.YAMLError, IOError) as e:
raise ConfigFileError("Error writing to config file: '%s'" %
str(e))
def create_projection_file(tmpdir, projection):
if 'projections' not in projection:
projection = {'projections': projection}
projection_file = tmpdir.mkdir('projection').join('projection.yaml')
projection_file.write(s_yaml.dump(projection))
return projection_file
def _create(scope=None):
old_data = {
'packages': {
'cmake': {
'paths': {'[email protected]': '/usr'}
},
'gcc': {
'modules': {'[email protected]': 'gcc-8'}
}
}
}
scope = scope or spack.config.default_modify_scope()
cfg_file = spack.config.config.get_config_filename(scope, 'packages')
with open(cfg_file, 'w') as f:
syaml.dump(old_data, stream=f)
return cfg_file
def generate_module_index(root, modules, overwrite=False):
index_path = os.path.join(root, 'module-index.yaml')
if overwrite or not os.path.exists(index_path):
entries = syaml.syaml_dict()
else:
with open(index_path) as index_file:
yaml_content = syaml.load(index_file)
entries = yaml_content['module_index']
for m in modules:
entry = {'path': m.layout.filename, 'use_name': m.layout.use_name}
entries[m.spec.dag_hash()] = entry
index = {'module_index': entries}
llnl.util.filesystem.mkdirp(root)
with open(index_path, 'w') as index_file:
syaml.dump(index, default_flow_style=False, stream=index_file)
def write_buildinfo_file(spec, workdir, rel=False):
"""
Create a cache file containing information
required for the relocation
"""
prefix = spec.prefix
text_to_relocate = []
binary_to_relocate = []
link_to_relocate = []
blacklist = (".spack", "man")
prefix_to_hash = dict()
prefix_to_hash[str(spec.package.prefix)] = spec.dag_hash()
deps = spack.build_environment.get_rpath_deps(spec.package)
for d in deps:
prefix_to_hash[str(d.prefix)] = d.dag_hash()
# Do this at during tarball creation to save time when tarball unpacked.
# Used by make_package_relative to determine binaries to change.
for root, dirs, files in os.walk(prefix, topdown=True):
dirs[:] = [d for d in dirs if d not in blacklist]
for filename in files:
path_name = os.path.join(root, filename)
m_type, m_subtype = relocate.mime_type(path_name)
if os.path.islink(path_name):
link = os.readlink(path_name)
if os.path.isabs(link):
# Relocate absolute links into the spack tree
if link.startswith(spack.store.layout.root):
rel_path_name = os.path.relpath(path_name, prefix)
link_to_relocate.append(rel_path_name)
else:
msg = 'Absolute link %s to %s ' % (path_name, link)
msg += 'outside of prefix %s ' % prefix
msg += 'should not be relocated.'
tty.warn(msg)
if relocate.needs_binary_relocation(m_type, m_subtype):
if not filename.endswith('.o'):
rel_path_name = os.path.relpath(path_name, prefix)
binary_to_relocate.append(rel_path_name)
if relocate.needs_text_relocation(m_type, m_subtype):
rel_path_name = os.path.relpath(path_name, prefix)
text_to_relocate.append(rel_path_name)
# Create buildinfo data and write it to disk
buildinfo = {}
buildinfo['relative_rpaths'] = rel
buildinfo['buildpath'] = spack.store.layout.root
buildinfo['spackprefix'] = spack.paths.prefix
buildinfo['relative_prefix'] = os.path.relpath(prefix,
spack.store.layout.root)
buildinfo['relocate_textfiles'] = text_to_relocate
buildinfo['relocate_binaries'] = binary_to_relocate
buildinfo['relocate_links'] = link_to_relocate
buildinfo['prefix_to_hash'] = prefix_to_hash
filename = buildinfo_file_name(workdir)
with open(filename, 'w') as outfile:
outfile.write(syaml.dump(buildinfo, default_flow_style=True))
def write_buildinfo_file(prefix, workdir, rel=False):
"""
Create a cache file containing information
required for the relocation
"""
text_to_relocate = []
binary_to_relocate = []
link_to_relocate = []
blacklist = (".spack", "man")
# Do this at during tarball creation to save time when tarball unpacked.
# Used by make_package_relative to determine binaries to change.
for root, dirs, files in os.walk(prefix, topdown=True):
dirs[:] = [d for d in dirs if d not in blacklist]
for filename in files:
path_name = os.path.join(root, filename)
m_type, m_subtype = relocate.mime_type(path_name)
if os.path.islink(path_name):
link = os.readlink(path_name)
if os.path.isabs(link):
# Relocate absolute links into the spack tree
if link.startswith(spack.store.layout.root):
rel_path_name = os.path.relpath(path_name, prefix)
link_to_relocate.append(rel_path_name)
else:
msg = 'Absolute link %s to %s ' % (path_name, link)
msg += 'outside of stage %s ' % prefix
msg += 'cannot be relocated.'
tty.warn(msg)
# Check if the file contains a string with the installroot.
# This cuts down on the number of files added to the list
# of files potentially needing relocation
elif relocate.strings_contains_installroot(
path_name,
spack.store.layout.root):
if relocate.needs_binary_relocation(m_type, m_subtype):
rel_path_name = os.path.relpath(path_name, prefix)
binary_to_relocate.append(rel_path_name)
elif relocate.needs_text_relocation(m_type, m_subtype):
rel_path_name = os.path.relpath(path_name, prefix)
text_to_relocate.append(rel_path_name)
# Create buildinfo data and write it to disk
buildinfo = {}
buildinfo['relative_rpaths'] = rel
buildinfo['buildpath'] = spack.store.layout.root
buildinfo['spackprefix'] = spack.paths.prefix
buildinfo['relative_prefix'] = os.path.relpath(
prefix, spack.store.layout.root)
buildinfo['relocate_textfiles'] = text_to_relocate
buildinfo['relocate_binaries'] = binary_to_relocate
buildinfo['relocate_links'] = link_to_relocate
filename = buildinfo_file_name(workdir)
with open(filename, 'w') as outfile:
outfile.write(syaml.dump(buildinfo, default_flow_style=True))
def add_spec(env, extension, data, create_modules):
ev.activate(env)
add(data.spec)
ev.deactivate()
excludes = view_excludes(data)
with open(env.manifest_path, 'r') as f:
yaml = syaml.load(f)
if create_modules:
module_excludes = excludes.copy()
module_path = os.path.join(os.environ['SPACK_MANAGER'], 'modules')
module_dict = {
data.id: {
'enable': ['tcl'],
'use_view': data.id,
'prefix_inspections': {
'bin': ['PATH']
},
'roots': {
'tcl': module_path
},
'arch_folder': False,
'tcl': {
'projections': {
'all': '%s/{name}-%s' % (extension, data.id)
},
'hash_length': 0,
'blacklist_implicits': True,
'blacklist': module_excludes
}
}
}
try:
yaml['spack']['modules'].update(module_dict)
except KeyError:
yaml['spack']['modules'] = module_dict
with open(env.manifest_path, 'w') as f:
syaml.dump(yaml, stream=f, default_flow_style=False)
def test_yaml_aliases():
aliased_list_1 = ['foo']
aliased_list_2 = []
dict_with_aliases = {
'a': aliased_list_1,
'b': aliased_list_1,
'c': aliased_list_1,
'd': aliased_list_2,
'e': aliased_list_2,
'f': aliased_list_2,
}
string = syaml.dump(dict_with_aliases)
# ensure no YAML aliases appear in syaml dumps.
assert '*id' not in string
def test_yaml_aliases(self):
aliased_list_1 = ['foo']
aliased_list_2 = []
dict_with_aliases = {
'a': aliased_list_1,
'b': aliased_list_1,
'c': aliased_list_1,
'd': aliased_list_2,
'e': aliased_list_2,
'f': aliased_list_2,
}
string = syaml.dump(dict_with_aliases)
# ensure no YAML aliases appear in syaml dumps.
self.assertFalse('*id' in string)
def manifest(self):
"""The spack.yaml file that should be used in the image"""
import jsonschema
# Copy in the part of spack.yaml prescribed in the configuration file
manifest = copy.deepcopy(self.config)
manifest.pop('container')
# Ensure that a few paths are where they need to be
manifest.setdefault('config', syaml.syaml_dict())
manifest['config']['install_tree'] = self.paths.store
manifest['view'] = self.paths.view
manifest = {'spack': manifest}
# Validate the manifest file
jsonschema.validate(manifest, schema=spack.schema.env.schema)
return syaml.dump(manifest, default_flow_style=False).strip()
for comp in compilers:
comp = comp['compiler']
paths = comp['paths']
prefixes = []
for bin, binpath in paths.items():
binpath = os.path.dirname(binpath)
if os.path.basename(binpath) == 'bin':
binpath = os.path.dirname(binpath)
prefixes.append(binpath)
prefix = os.path.commonprefix(prefixes)
if prefix.startswith("/usr") or prefix == "/":
continue
ld_paths = os.environ.get("LD_LIBRARY_PATH", "")
ld_paths = ld_paths.split(os.path.pathsep)
found_ld_paths = []
for ld_elm in ld_paths:
if ld_elm.startswith(prefix) and os.path.isdir(ld_elm):
found_ld_paths.append(ld_elm)
if len(found_ld_paths) == 0:
continue
found_ld_paths = os.path.pathsep.join(found_ld_paths)
elm = comp.setdefault('environment', {})
elm = elm.setdefault('prepend-path', {})
elm["LD_LIBRARY_PATH"] = found_ld_paths
print(
"*** Fixing compiler spec %r with prefix %r to add LD_LIBRARY_PATH %r"
% (comp['spec'], prefix, found_ld_paths))
with open(fn, "w") as f:
dump(data, stream=f)
def build_tarball(spec,
outdir,
force=False,
rel=False,
unsigned=False,
allow_root=False,
key=None,
regenerate_index=False):
"""
Build a tarball from given spec and put it into the directory structure
used at the mirror (following <tarball_directory_name>).
"""
if not spec.concrete:
raise ValueError('spec must be concrete to build tarball')
# set up some paths
tmpdir = tempfile.mkdtemp()
cache_prefix = build_cache_prefix(tmpdir)
tarfile_name = tarball_name(spec, '.tar.gz')
tarfile_dir = os.path.join(cache_prefix, tarball_directory_name(spec))
tarfile_path = os.path.join(tarfile_dir, tarfile_name)
spackfile_path = os.path.join(cache_prefix,
tarball_path_name(spec, '.spack'))
remote_spackfile_path = url_util.join(
outdir, os.path.relpath(spackfile_path, tmpdir))
mkdirp(tarfile_dir)
if web_util.url_exists(remote_spackfile_path):
if force:
web_util.remove_url(remote_spackfile_path)
else:
raise NoOverwriteException(url_util.format(remote_spackfile_path))
# need to copy the spec file so the build cache can be downloaded
# without concretizing with the current spack packages
# and preferences
spec_file = os.path.join(spec.prefix, ".spack", "spec.yaml")
specfile_name = tarball_name(spec, '.spec.yaml')
specfile_path = os.path.realpath(os.path.join(cache_prefix, specfile_name))
remote_specfile_path = url_util.join(
outdir, os.path.relpath(specfile_path, os.path.realpath(tmpdir)))
if web_util.url_exists(remote_specfile_path):
if force:
web_util.remove_url(remote_specfile_path)
else:
raise NoOverwriteException(url_util.format(remote_specfile_path))
# make a copy of the install directory to work with
workdir = os.path.join(tmpdir, os.path.basename(spec.prefix))
# install_tree copies hardlinks
# create a temporary tarfile from prefix and exract it to workdir
# tarfile preserves hardlinks
temp_tarfile_name = tarball_name(spec, '.tar')
temp_tarfile_path = os.path.join(tarfile_dir, temp_tarfile_name)
with closing(tarfile.open(temp_tarfile_path, 'w')) as tar:
tar.add(name='%s' % spec.prefix, arcname='.')
with closing(tarfile.open(temp_tarfile_path, 'r')) as tar:
tar.extractall(workdir)
os.remove(temp_tarfile_path)
# create info for later relocation and create tar
write_buildinfo_file(spec, workdir, rel)
# optionally make the paths in the binaries relative to each other
# in the spack install tree before creating tarball
if rel:
try:
make_package_relative(workdir, spec, allow_root)
except Exception as e:
shutil.rmtree(workdir)
shutil.rmtree(tarfile_dir)
shutil.rmtree(tmpdir)
tty.die(e)
else:
try:
check_package_relocatable(workdir, spec, allow_root)
except Exception as e:
shutil.rmtree(workdir)
shutil.rmtree(tarfile_dir)
shutil.rmtree(tmpdir)
tty.die(e)
# create gzip compressed tarball of the install prefix
with closing(tarfile.open(tarfile_path, 'w:gz')) as tar:
tar.add(name='%s' % workdir,
arcname='%s' % os.path.basename(spec.prefix))
# remove copy of install directory
shutil.rmtree(workdir)
# get the sha256 checksum of the tarball
checksum = checksum_tarball(tarfile_path)
# add sha256 checksum to spec.yaml
with open(spec_file, 'r') as inputfile:
content = inputfile.read()
spec_dict = yaml.load(content)
bchecksum = {}
bchecksum['hash_algorithm'] = 'sha256'
bchecksum['hash'] = checksum
spec_dict['binary_cache_checksum'] = bchecksum
# Add original install prefix relative to layout root to spec.yaml.
# This will be used to determine is the directory layout has changed.
buildinfo = {}
buildinfo['relative_prefix'] = os.path.relpath(spec.prefix,
spack.store.layout.root)
buildinfo['relative_rpaths'] = rel
spec_dict['buildinfo'] = buildinfo
spec_dict['full_hash'] = spec.full_hash()
tty.debug('The full_hash ({0}) of {1} will be written into {2}'.format(
spec_dict['full_hash'], spec.name,
url_util.format(remote_specfile_path)))
tty.debug(spec.tree())
with open(specfile_path, 'w') as outfile:
outfile.write(syaml.dump(spec_dict))
# sign the tarball and spec file with gpg
if not unsigned:
key = select_signing_key(key)
sign_tarball(key, force, specfile_path)
# put tarball, spec and signature files in .spack archive
with closing(tarfile.open(spackfile_path, 'w')) as tar:
tar.add(name=tarfile_path, arcname='%s' % tarfile_name)
tar.add(name=specfile_path, arcname='%s' % specfile_name)
if not unsigned:
tar.add(name='%s.asc' % specfile_path,
arcname='%s.asc' % specfile_name)
# cleanup file moved to archive
os.remove(tarfile_path)
if not unsigned:
os.remove('%s.asc' % specfile_path)
web_util.push_to_url(spackfile_path,
remote_spackfile_path,
keep_original=False)
web_util.push_to_url(specfile_path,
remote_specfile_path,
keep_original=False)
tty.debug('Buildcache for "{0}" written to \n {1}'.format(
spec, remote_spackfile_path))
try:
# push the key to the build cache's _pgp directory so it can be
# imported
if not unsigned:
push_keys(outdir,
keys=[key],
regenerate_index=regenerate_index,
tmpdir=tmpdir)
# create an index.json for the build_cache directory so specs can be
# found
if regenerate_index:
generate_package_index(
url_util.join(outdir, os.path.relpath(cache_prefix, tmpdir)))
finally:
shutil.rmtree(tmpdir)
return None
def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
allow_root=False, key=None, regenerate_index=False):
"""
Build a tarball from given spec and put it into the directory structure
used at the mirror (following <tarball_directory_name>).
"""
if not spec.concrete:
raise ValueError('spec must be concrete to build tarball')
# set up some paths
build_cache_dir = build_cache_directory(outdir)
tarfile_name = tarball_name(spec, '.tar.gz')
tarfile_dir = os.path.join(build_cache_dir,
tarball_directory_name(spec))
tarfile_path = os.path.join(tarfile_dir, tarfile_name)
mkdirp(tarfile_dir)
spackfile_path = os.path.join(
build_cache_dir, tarball_path_name(spec, '.spack'))
if os.path.exists(spackfile_path):
if force:
os.remove(spackfile_path)
else:
raise NoOverwriteException(str(spackfile_path))
# need to copy the spec file so the build cache can be downloaded
# without concretizing with the current spack packages
# and preferences
spec_file = os.path.join(spec.prefix, ".spack", "spec.yaml")
specfile_name = tarball_name(spec, '.spec.yaml')
specfile_path = os.path.realpath(
os.path.join(build_cache_dir, specfile_name))
if os.path.exists(specfile_path):
if force:
os.remove(specfile_path)
else:
raise NoOverwriteException(str(specfile_path))
# make a copy of the install directory to work with
workdir = os.path.join(tempfile.mkdtemp(), os.path.basename(spec.prefix))
install_tree(spec.prefix, workdir, symlinks=True)
# create info for later relocation and create tar
write_buildinfo_file(spec.prefix, workdir, rel=rel)
# optinally make the paths in the binaries relative to each other
# in the spack install tree before creating tarball
if rel:
try:
make_package_relative(workdir, spec.prefix, allow_root)
except Exception as e:
shutil.rmtree(workdir)
shutil.rmtree(tarfile_dir)
tty.die(str(e))
else:
try:
make_package_placeholder(workdir, spec.prefix, allow_root)
except Exception as e:
shutil.rmtree(workdir)
shutil.rmtree(tarfile_dir)
tty.die(str(e))
# create compressed tarball of the install prefix
with closing(tarfile.open(tarfile_path, 'w:gz')) as tar:
tar.add(name='%s' % workdir,
arcname='%s' % os.path.basename(spec.prefix))
# remove copy of install directory
shutil.rmtree(workdir)
# get the sha256 checksum of the tarball
checksum = checksum_tarball(tarfile_path)
# add sha256 checksum to spec.yaml
spec_dict = {}
with open(spec_file, 'r') as inputfile:
content = inputfile.read()
spec_dict = syaml.load(content)
bchecksum = {}
bchecksum['hash_algorithm'] = 'sha256'
bchecksum['hash'] = checksum
spec_dict['binary_cache_checksum'] = bchecksum
# Add original install prefix relative to layout root to spec.yaml.
# This will be used to determine is the directory layout has changed.
buildinfo = {}
buildinfo['relative_prefix'] = os.path.relpath(
spec.prefix, spack.store.layout.root)
spec_dict['buildinfo'] = buildinfo
spec_dict['full_hash'] = spec.full_hash()
tty.debug('The full_hash ({0}) of {1} will be written into {2}'.format(
spec_dict['full_hash'], spec.name, specfile_path))
tty.debug(spec.tree())
with open(specfile_path, 'w') as outfile:
outfile.write(syaml.dump(spec_dict))
# sign the tarball and spec file with gpg
if not unsigned:
sign_tarball(key, force, specfile_path)
# put tarball, spec and signature files in .spack archive
with closing(tarfile.open(spackfile_path, 'w')) as tar:
tar.add(name='%s' % tarfile_path, arcname='%s' % tarfile_name)
tar.add(name='%s' % specfile_path, arcname='%s' % specfile_name)
if not unsigned:
tar.add(name='%s.asc' % specfile_path,
arcname='%s.asc' % specfile_name)
# cleanup file moved to archive
os.remove(tarfile_path)
if not unsigned:
os.remove('%s.asc' % specfile_path)
# create an index.html for the build_cache directory so specs can be found
if regenerate_index:
generate_package_index(build_cache_dir)
return None
def dumper(configuration):
content = syaml.dump(configuration, default_flow_style=False)
config_file = tmpdir / 'spack.yaml'
config_file.write(content)
return str(tmpdir)
def release_jobs(parser, args):
share_path = os.path.join(spack_root, 'share', 'spack', 'docker')
os_container_mapping_path = os.path.join(
share_path, 'os-container-mapping.yaml')
with open(os_container_mapping_path, 'r') as fin:
os_container_mapping = syaml.load(fin)
try:
validate(os_container_mapping, mapping_schema)
except ValidationError as val_err:
tty.error('Ill-formed os-container-mapping configuration object')
tty.error(os_container_mapping)
tty.debug(val_err)
return
containers = os_container_mapping['containers']
if args.specs:
# Just print out the spec labels and all dependency edges in
# a json format.
spec_list = [Spec(s) for s in args.specs]
with open(args.specs_deps_output, 'w') as out:
compute_spec_deps(spec_list, out)
return
current_system = sys_type() if args.resolve_deps_locally else None
release_specs_path = args.spec_set
if not release_specs_path:
raise SpackError('Must provide path to release spec-set')
release_spec_set = CombinatorialSpecSet.from_file(release_specs_path)
mirror_url = args.mirror_url
if not mirror_url:
raise SpackError('Must provide url of target binary mirror')
cdash_url = args.cdash_url
spec_labels, dependencies, stages = stage_spec_jobs(
release_spec_set, containers, current_system)
if not stages:
tty.msg('No jobs staged, exiting.')
return
if args.print_summary:
print_staging_summary(spec_labels, dependencies, stages)
output_object = {}
job_count = 0
stage_names = ['stage-{0}'.format(i) for i in range(len(stages))]
stage = 0
for stage_jobs in stages:
stage_name = stage_names[stage]
for spec_label in stage_jobs:
release_spec = spec_labels[spec_label]['spec']
root_spec = spec_labels[spec_label]['rootSpec']
pkg_compiler = release_spec.compiler
pkg_hash = release_spec.dag_hash()
osname = str(release_spec.architecture)
job_name = get_job_name(release_spec, osname)
container_info = containers[osname]
build_image = container_info['image']
job_scripts = ['./bin/rebuild-package.sh']
if 'setup_script' in container_info:
job_scripts.insert(
0, container_info['setup_script'] % pkg_compiler)
job_dependencies = []
if spec_label in dependencies:
job_dependencies = (
[get_job_name(spec_labels[dep_label]['spec'], osname)
for dep_label in dependencies[spec_label]])
job_object = {
'stage': stage_name,
'variables': {
'MIRROR_URL': mirror_url,
'CDASH_BASE_URL': cdash_url,
'HASH': pkg_hash,
'DEPENDENCIES': ';'.join(job_dependencies),
'ROOT_SPEC': str(root_spec),
},
'script': job_scripts,
'image': build_image,
'artifacts': {
'paths': [
'local_mirror/build_cache',
'jobs_scratch_dir',
'cdash_report',
],
'when': 'always',
},
'dependencies': job_dependencies,
}
# If we see 'compilers' in the container iformation, it's a
# filter for the compilers this container can handle, else we
# assume it can handle any compiler
if 'compilers' in container_info:
do_job = False
for item in container_info['compilers']:
container_compiler_spec = CompilerSpec(item['name'])
if pkg_compiler == container_compiler_spec:
do_job = True
else:
do_job = True
if args.shared_runner_tag:
job_object['tags'] = [args.shared_runner_tag]
if args.signing_key:
job_object['variables']['SIGN_KEY_HASH'] = args.signing_key
if do_job:
output_object[job_name] = job_object
job_count += 1
stage += 1
tty.msg('{0} build jobs generated in {1} stages'.format(
job_count, len(stages)))
final_stage = 'stage-rebuild-index'
final_job = {
'stage': final_stage,
'variables': {
'MIRROR_URL': mirror_url,
},
'image': build_image,
'script': './bin/rebuild-index.sh',
}
if args.shared_runner_tag:
final_job['tags'] = [args.shared_runner_tag]
output_object['rebuild-index'] = final_job
stage_names.append(final_stage)
output_object['stages'] = stage_names
with open(args.output_file, 'w') as outf:
outf.write(syaml.dump(output_object))
def _create_config(scope=None, data={}, section='packages'):
scope = scope or spack.config.default_modify_scope()
cfg_file = spack.config.config.get_config_filename(scope, section)
with open(cfg_file, 'w') as f:
syaml.dump(data, stream=f)
return cfg_file
def to_yaml(self, stream=None):
return syaml.dump(self.to_dict(), stream)
def ci_rebuild(args):
"""Check a single spec against the remote mirror, and rebuild it from
source if the mirror does not contain the full hash match of the spec
as computed locally. """
env = ev.get_env(args, 'ci rebuild', required=True)
# Make sure the environment is "gitlab-enabled", or else there's nothing
# to do.
yaml_root = ev.config_dict(env.yaml)
gitlab_ci = None
if 'gitlab-ci' in yaml_root:
gitlab_ci = yaml_root['gitlab-ci']
if not gitlab_ci:
tty.die('spack ci rebuild requires an env containing gitlab-ci cfg')
# Grab the environment variables we need. These either come from the
# pipeline generation step ("spack ci generate"), where they were written
# out as variables, or else provided by GitLab itself.
pipeline_artifacts_dir = get_env_var('SPACK_ARTIFACTS_ROOT')
job_log_dir = get_env_var('SPACK_JOB_LOG_DIR')
repro_dir = get_env_var('SPACK_JOB_REPRO_DIR')
local_mirror_dir = get_env_var('SPACK_LOCAL_MIRROR_DIR')
concrete_env_dir = get_env_var('SPACK_CONCRETE_ENV_DIR')
ci_pipeline_id = get_env_var('CI_PIPELINE_ID')
ci_job_name = get_env_var('CI_JOB_NAME')
signing_key = get_env_var('SPACK_SIGNING_KEY')
root_spec = get_env_var('SPACK_ROOT_SPEC')
job_spec_pkg_name = get_env_var('SPACK_JOB_SPEC_PKG_NAME')
compiler_action = get_env_var('SPACK_COMPILER_ACTION')
cdash_build_name = get_env_var('SPACK_CDASH_BUILD_NAME')
related_builds = get_env_var('SPACK_RELATED_BUILDS_CDASH')
spack_pipeline_type = get_env_var('SPACK_PIPELINE_TYPE')
pr_mirror_url = get_env_var('SPACK_PR_MIRROR_URL')
remote_mirror_url = get_env_var('SPACK_REMOTE_MIRROR_URL')
# Construct absolute paths relative to current $CI_PROJECT_DIR
ci_project_dir = get_env_var('CI_PROJECT_DIR')
pipeline_artifacts_dir = os.path.join(ci_project_dir,
pipeline_artifacts_dir)
job_log_dir = os.path.join(ci_project_dir, job_log_dir)
repro_dir = os.path.join(ci_project_dir, repro_dir)
local_mirror_dir = os.path.join(ci_project_dir, local_mirror_dir)
concrete_env_dir = os.path.join(ci_project_dir, concrete_env_dir)
# Debug print some of the key environment variables we should have received
tty.debug('pipeline_artifacts_dir = {0}'.format(pipeline_artifacts_dir))
tty.debug('root_spec = {0}'.format(root_spec))
tty.debug('remote_mirror_url = {0}'.format(remote_mirror_url))
tty.debug('job_spec_pkg_name = {0}'.format(job_spec_pkg_name))
tty.debug('compiler_action = {0}'.format(compiler_action))
# Query the environment manifest to find out whether we're reporting to a
# CDash instance, and if so, gather some information from the manifest to
# support that task.
enable_cdash = False
if 'cdash' in yaml_root:
enable_cdash = True
ci_cdash = yaml_root['cdash']
job_spec_buildgroup = ci_cdash['build-group']
cdash_base_url = ci_cdash['url']
cdash_project = ci_cdash['project']
proj_enc = urlencode({'project': cdash_project})
eq_idx = proj_enc.find('=') + 1
cdash_project_enc = proj_enc[eq_idx:]
cdash_site = ci_cdash['site']
tty.debug('cdash_base_url = {0}'.format(cdash_base_url))
tty.debug('cdash_project = {0}'.format(cdash_project))
tty.debug('cdash_project_enc = {0}'.format(cdash_project_enc))
tty.debug('cdash_build_name = {0}'.format(cdash_build_name))
tty.debug('cdash_site = {0}'.format(cdash_site))
tty.debug('related_builds = {0}'.format(related_builds))
tty.debug('job_spec_buildgroup = {0}'.format(job_spec_buildgroup))
# Is this a pipeline run on a spack PR or a merge to develop? It might
# be neither, e.g. a pipeline run on some environment repository.
spack_is_pr_pipeline = spack_pipeline_type == 'spack_pull_request'
spack_is_develop_pipeline = spack_pipeline_type == 'spack_protected_branch'
tty.debug('Pipeline type - PR: {0}, develop: {1}'.format(
spack_is_pr_pipeline, spack_is_develop_pipeline))
# Figure out what is our temporary storage mirror: Is it artifacts
# buildcache? Or temporary-storage-url-prefix? In some cases we need to
# force something or pipelines might not have a way to propagate build
# artifacts from upstream to downstream jobs.
pipeline_mirror_url = None
temp_storage_url_prefix = None
if 'temporary-storage-url-prefix' in gitlab_ci:
temp_storage_url_prefix = gitlab_ci['temporary-storage-url-prefix']
pipeline_mirror_url = url_util.join(temp_storage_url_prefix,
ci_pipeline_id)
enable_artifacts_mirror = False
if 'enable-artifacts-buildcache' in gitlab_ci:
enable_artifacts_mirror = gitlab_ci['enable-artifacts-buildcache']
if (enable_artifacts_mirror
or (spack_is_pr_pipeline and not enable_artifacts_mirror
and not temp_storage_url_prefix)):
# If you explicitly enabled the artifacts buildcache feature, or
# if this is a PR pipeline but you did not enable either of the
# per-pipeline temporary storage features, we force the use of
# artifacts buildcache. Otherwise jobs will not have binary
# dependencies from previous stages available since we do not
# allow pushing binaries to the remote mirror during PR pipelines.
enable_artifacts_mirror = True
pipeline_mirror_url = 'file://' + local_mirror_dir
mirror_msg = 'artifact buildcache enabled, mirror url: {0}'.format(
pipeline_mirror_url)
tty.debug(mirror_msg)
# Whatever form of root_spec we got, use it to get a map giving us concrete
# specs for this job and all of its dependencies.
spec_map = spack_ci.get_concrete_specs(env, root_spec, job_spec_pkg_name,
related_builds, compiler_action)
job_spec = spec_map[job_spec_pkg_name]
job_spec_yaml_file = '{0}.yaml'.format(job_spec_pkg_name)
job_spec_yaml_path = os.path.join(repro_dir, job_spec_yaml_file)
# To provide logs, cdash reports, etc for developer download/perusal,
# these things have to be put into artifacts. This means downstream
# jobs that "need" this job will get those artifacts too. So here we
# need to clean out the artifacts we may have got from upstream jobs.
cdash_report_dir = os.path.join(pipeline_artifacts_dir, 'cdash_report')
if os.path.exists(cdash_report_dir):
shutil.rmtree(cdash_report_dir)
if os.path.exists(job_log_dir):
shutil.rmtree(job_log_dir)
if os.path.exists(repro_dir):
shutil.rmtree(repro_dir)
# Now that we removed them if they existed, create the directories we
# need for storing artifacts. The cdash_report directory will be
# created internally if needed.
os.makedirs(job_log_dir)
os.makedirs(repro_dir)
# Copy the concrete environment files to the repro directory so we can
# expose them as artifacts and not conflict with the concrete environment
# files we got as artifacts from the upstream pipeline generation job.
# Try to cast a slightly wider net too, and hopefully get the generated
# pipeline yaml. If we miss it, the user will still be able to go to the
# pipeline generation job and get it from there.
target_dirs = [concrete_env_dir, pipeline_artifacts_dir]
for dir_to_list in target_dirs:
for file_name in os.listdir(dir_to_list):
src_file = os.path.join(dir_to_list, file_name)
if os.path.isfile(src_file):
dst_file = os.path.join(repro_dir, file_name)
shutil.copyfile(src_file, dst_file)
# If signing key was provided via "SPACK_SIGNING_KEY", then try to
# import it.
if signing_key:
spack_ci.import_signing_key(signing_key)
# Depending on the specifics of this job, we might need to turn on the
# "config:install_missing compilers" option (to build this job spec
# with a bootstrapped compiler), or possibly run "spack compiler find"
# (to build a bootstrap compiler or one of its deps in a
# compiler-agnostic way), or maybe do nothing at all (to build a spec
# using a compiler already installed on the target system).
spack_ci.configure_compilers(compiler_action)
# Write this job's spec yaml into the reproduction directory, and it will
# also be used in the generated "spack install" command to install the spec
tty.debug('job concrete spec path: {0}'.format(job_spec_yaml_path))
with open(job_spec_yaml_path, 'w') as fd:
fd.write(job_spec.to_yaml(hash=ht.build_hash))
# Write the concrete root spec yaml into the reproduction directory
root_spec_yaml_path = os.path.join(repro_dir, 'root.yaml')
with open(root_spec_yaml_path, 'w') as fd:
fd.write(spec_map['root'].to_yaml(hash=ht.build_hash))
# Write some other details to aid in reproduction into an artifact
repro_file = os.path.join(repro_dir, 'repro.json')
repro_details = {
'job_name': ci_job_name,
'job_spec_yaml': job_spec_yaml_file,
'root_spec_yaml': 'root.yaml',
'ci_project_dir': ci_project_dir
}
with open(repro_file, 'w') as fd:
fd.write(json.dumps(repro_details))
# Write information about spack into an artifact in the repro dir
spack_info = spack_ci.get_spack_info()
spack_info_file = os.path.join(repro_dir, 'spack_info.txt')
with open(spack_info_file, 'w') as fd:
fd.write('\n{0}\n'.format(spack_info))
# If we decided there should be a temporary storage mechanism, add that
# mirror now so it's used when we check for a full hash match already
# built for this spec.
if pipeline_mirror_url:
spack.mirror.add(spack_ci.TEMP_STORAGE_MIRROR_NAME,
pipeline_mirror_url, cfg.default_modify_scope())
cdash_build_id = None
cdash_build_stamp = None
# Check configured mirrors for a built spec with a matching full hash
matches = bindist.get_mirrors_for_spec(job_spec,
full_hash_match=True,
index_only=False)
if matches:
# Got a full hash match on at least one configured mirror. All
# matches represent the fully up-to-date spec, so should all be
# equivalent. If artifacts mirror is enabled, we just pick one
# of the matches and download the buildcache files from there to
# the artifacts, so they're available to be used by dependent
# jobs in subsequent stages.
tty.msg('No need to rebuild {0}, found full hash match at: '.format(
job_spec_pkg_name))
for match in matches:
tty.msg(' {0}'.format(match['mirror_url']))
if enable_artifacts_mirror:
matching_mirror = matches[0]['mirror_url']
build_cache_dir = os.path.join(local_mirror_dir, 'build_cache')
tty.debug('Getting {0} buildcache from {1}'.format(
job_spec_pkg_name, matching_mirror))
tty.debug('Downloading to {0}'.format(build_cache_dir))
buildcache.download_buildcache_files(job_spec, build_cache_dir,
False, matching_mirror)
# Now we are done and successful
sys.exit(0)
# No full hash match anywhere means we need to rebuild spec
# Start with spack arguments
install_args = [base_arg for base_arg in CI_REBUILD_INSTALL_BASE_ARGS]
config = cfg.get('config')
if not config['verify_ssl']:
install_args.append('-k')
install_args.extend([
'install',
'--keep-stage',
'--require-full-hash-match',
])
can_verify = spack_ci.can_verify_binaries()
verify_binaries = can_verify and spack_is_pr_pipeline is False
if not verify_binaries:
install_args.append('--no-check-signature')
# If CDash reporting is enabled, we first register this build with
# the specified CDash instance, then relate the build to those of
# its dependencies.
if enable_cdash:
tty.debug('CDash: Registering build')
(cdash_build_id, cdash_build_stamp) = spack_ci.register_cdash_build(
cdash_build_name, cdash_base_url, cdash_project, cdash_site,
job_spec_buildgroup)
if cdash_build_id is not None:
cdash_upload_url = '{0}/submit.php?project={1}'.format(
cdash_base_url, cdash_project_enc)
install_args.extend([
'--cdash-upload-url',
cdash_upload_url,
'--cdash-build',
cdash_build_name,
'--cdash-site',
cdash_site,
'--cdash-buildstamp',
cdash_build_stamp,
])
tty.debug('CDash: Relating build with dependency builds')
spack_ci.relate_cdash_builds(
spec_map, cdash_base_url, cdash_build_id, cdash_project,
[pipeline_mirror_url, pr_mirror_url, remote_mirror_url])
# A compiler action of 'FIND_ANY' means we are building a bootstrap
# compiler or one of its deps.
# TODO: when compilers are dependencies, we should include --no-add
if compiler_action != 'FIND_ANY':
install_args.append('--no-add')
# TODO: once we have the concrete spec registry, use the DAG hash
# to identify the spec to install, rather than the concrete spec
# yaml file.
install_args.extend(['-f', job_spec_yaml_path])
tty.debug('Installing {0} from source'.format(job_spec.name))
tty.debug('spack install arguments: {0}'.format(install_args))
# Write the install command to a shell script
with open('install.sh', 'w') as fd:
fd.write('#!/bin/bash\n\n')
fd.write('\n# spack install command\n')
fd.write(' '.join(['"{0}"'.format(i) for i in install_args]))
fd.write('\n')
st = os.stat('install.sh')
os.chmod('install.sh', st.st_mode | stat.S_IEXEC)
install_copy_path = os.path.join(repro_dir, 'install.sh')
shutil.copyfile('install.sh', install_copy_path)
# Run the generated install.sh shell script as if it were being run in
# a login shell.
try:
install_process = subprocess.Popen(['bash', '-l', './install.sh'])
install_process.wait()
install_exit_code = install_process.returncode
except (ValueError, subprocess.CalledProcessError, OSError) as inst:
tty.error('Encountered error running install script')
tty.error(inst)
# Now do the post-install tasks
tty.debug('spack install exited {0}'.format(install_exit_code))
# If a spec fails to build in a spack develop pipeline, we add it to a
# list of known broken full hashes. This allows spack PR pipelines to
# avoid wasting compute cycles attempting to build those hashes.
if install_exit_code == INSTALL_FAIL_CODE and spack_is_develop_pipeline:
tty.debug('Install failed on develop')
if 'broken-specs-url' in gitlab_ci:
broken_specs_url = gitlab_ci['broken-specs-url']
dev_fail_hash = job_spec.full_hash()
broken_spec_path = url_util.join(broken_specs_url, dev_fail_hash)
tty.msg('Reporting broken develop build as: {0}'.format(
broken_spec_path))
tmpdir = tempfile.mkdtemp()
empty_file_path = os.path.join(tmpdir, 'empty.txt')
broken_spec_details = {
'broken-spec': {
'job-url': get_env_var('CI_JOB_URL'),
'pipeline-url': get_env_var('CI_PIPELINE_URL'),
'concrete-spec-yaml': job_spec.to_dict(hash=ht.full_hash)
}
}
try:
with open(empty_file_path, 'w') as efd:
efd.write(syaml.dump(broken_spec_details))
web_util.push_to_url(empty_file_path,
broken_spec_path,
keep_original=False,
extra_args={'ContentType': 'text/plain'})
except Exception as err:
# If we got some kind of S3 (access denied or other connection
# error), the first non boto-specific class in the exception
# hierarchy is Exception. Just print a warning and return
msg = 'Error writing to broken specs list {0}: {1}'.format(
broken_spec_path, err)
tty.warn(msg)
finally:
shutil.rmtree(tmpdir)
# We generated the "spack install ..." command to "--keep-stage", copy
# any logs from the staging directory to artifacts now
spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir)
# Create buildcache on remote mirror, either on pr-specific mirror or
# on the main mirror defined in the gitlab-enabled spack environment
if spack_is_pr_pipeline:
buildcache_mirror_url = pr_mirror_url
else:
buildcache_mirror_url = remote_mirror_url
# If the install succeeded, create a buildcache entry for this job spec
# and push it to one or more mirrors. If the install did not succeed,
# print out some instructions on how to reproduce this build failure
# outside of the pipeline environment.
if install_exit_code == 0:
can_sign = spack_ci.can_sign_binaries()
sign_binaries = can_sign and spack_is_pr_pipeline is False
# Create buildcache in either the main remote mirror, or in the
# per-PR mirror, if this is a PR pipeline
if buildcache_mirror_url:
spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path,
buildcache_mirror_url, sign_binaries)
if cdash_build_id:
tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format(
cdash_build_id, buildcache_mirror_url))
spack_ci.write_cdashid_to_mirror(cdash_build_id, job_spec,
buildcache_mirror_url)
# Create another copy of that buildcache in the per-pipeline
# temporary storage mirror (this is only done if either
# artifacts buildcache is enabled or a temporary storage url
# prefix is set)
if pipeline_mirror_url:
spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path,
pipeline_mirror_url, sign_binaries)
if cdash_build_id:
tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format(
cdash_build_id, pipeline_mirror_url))
spack_ci.write_cdashid_to_mirror(cdash_build_id, job_spec,
pipeline_mirror_url)
# If this is a develop pipeline, check if the spec that we just built is
# on the broken-specs list. If so, remove it.
if spack_is_develop_pipeline and 'broken-specs-url' in gitlab_ci:
broken_specs_url = gitlab_ci['broken-specs-url']
just_built_hash = job_spec.full_hash()
broken_spec_path = url_util.join(broken_specs_url, just_built_hash)
if web_util.url_exists(broken_spec_path):
tty.msg('Removing {0} from the list of broken specs'.format(
broken_spec_path))
try:
web_util.remove_url(broken_spec_path)
except Exception as err:
# If we got some kind of S3 (access denied or other connection
# error), the first non boto-specific class in the exception
# hierarchy is Exception. Just print a warning and return
msg = 'Error removing {0} from broken specs list: {1}'.format(
broken_spec_path, err)
tty.warn(msg)
else:
tty.debug('spack install exited non-zero, will not create buildcache')
api_root_url = get_env_var('CI_API_V4_URL')
ci_project_id = get_env_var('CI_PROJECT_ID')
ci_job_id = get_env_var('CI_JOB_ID')
repro_job_url = '{0}/projects/{1}/jobs/{2}/artifacts'.format(
api_root_url, ci_project_id, ci_job_id)
# Control characters cause this to be printed in blue so it stands out
reproduce_msg = """
\033[34mTo reproduce this build locally, run:
spack ci reproduce-build {0} [--working-dir <dir>]
If this project does not have public pipelines, you will need to first:
export GITLAB_PRIVATE_TOKEN=<generated_token>
... then follow the printed instructions.\033[0;0m
""".format(repro_job_url)
print(reproduce_msg)
# Tie job success/failure to the success/failure of building the spec
return install_exit_code
def test_ordered_read_not_required_for_consistent_dag_hash(
config, mock_packages):
"""Make sure ordered serialization isn't required to preserve hashes.
For consistent hashes, we require that YAML and json documents
have their keys serialized in a deterministic order. However, we
don't want to require them to be serialized in order. This
ensures that is not required.
"""
specs = ['mpileaks ^zmpi', 'dttop', 'dtuse']
for spec in specs:
spec = Spec(spec)
spec.concretize()
#
# Dict & corresponding YAML & JSON from the original spec.
#
spec_dict = spec.to_dict()
spec_yaml = spec.to_yaml()
spec_json = spec.to_json()
#
# Make a spec with reversed OrderedDicts for every
# OrderedDict in the original.
#
reversed_spec_dict = reverse_all_dicts(spec.to_dict())
#
# Dump to YAML and JSON
#
yaml_string = syaml.dump(spec_dict, default_flow_style=False)
reversed_yaml_string = syaml.dump(reversed_spec_dict,
default_flow_style=False)
json_string = sjson.dump(spec_dict)
reversed_json_string = sjson.dump(reversed_spec_dict)
#
# Do many consistency checks
#
# spec yaml is ordered like the spec dict
assert yaml_string == spec_yaml
assert json_string == spec_json
# reversed string is different from the original, so it
# *would* generate a different hash
assert yaml_string != reversed_yaml_string
assert json_string != reversed_json_string
# build specs from the "wrongly" ordered data
round_trip_yaml_spec = Spec.from_yaml(yaml_string)
round_trip_json_spec = Spec.from_json(json_string)
round_trip_reversed_yaml_spec = Spec.from_yaml(reversed_yaml_string)
round_trip_reversed_json_spec = Spec.from_yaml(reversed_json_string)
# TODO: remove this when build deps are in provenance.
spec = spec.copy(deps=('link', 'run'))
# specs are equal to the original
assert spec == round_trip_yaml_spec
assert spec == round_trip_json_spec
assert spec == round_trip_reversed_yaml_spec
assert spec == round_trip_reversed_json_spec
assert round_trip_yaml_spec == round_trip_reversed_yaml_spec
assert round_trip_json_spec == round_trip_reversed_json_spec
# dag_hashes are equal
assert spec.dag_hash() == round_trip_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_json_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()
# full_hashes are equal
spec.concretize()
round_trip_yaml_spec.concretize()
round_trip_json_spec.concretize()
round_trip_reversed_yaml_spec.concretize()
round_trip_reversed_json_spec.concretize()
assert spec.full_hash() == round_trip_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_json_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_json_spec.full_hash()
def test_ordered_read_not_required_for_consistent_dag_hash(
config, builtin_mock
):
"""Make sure ordered serialization isn't required to preserve hashes.
For consistent hashes, we require that YAML and json documents
have their keys serialized in a deterministic order. However, we
don't want to require them to be serialized in order. This
ensures that is not required.
"""
specs = ['mpileaks ^zmpi', 'dttop', 'dtuse']
for spec in specs:
spec = Spec(spec)
spec.concretize()
#
# Dict & corresponding YAML & JSON from the original spec.
#
spec_dict = spec.to_dict()
spec_yaml = spec.to_yaml()
spec_json = spec.to_json()
#
# Make a spec with reversed OrderedDicts for every
# OrderedDict in the original.
#
reversed_spec_dict = reverse_all_dicts(spec.to_dict())
#
# Dump to YAML and JSON
#
yaml_string = syaml.dump(spec_dict, default_flow_style=False)
reversed_yaml_string = syaml.dump(reversed_spec_dict,
default_flow_style=False)
json_string = sjson.dump(spec_dict)
reversed_json_string = sjson.dump(reversed_spec_dict)
#
# Do many consistency checks
#
# spec yaml is ordered like the spec dict
assert yaml_string == spec_yaml
assert json_string == spec_json
# reversed string is different from the original, so it
# *would* generate a different hash
assert yaml_string != reversed_yaml_string
assert json_string != reversed_json_string
# build specs from the "wrongly" ordered data
round_trip_yaml_spec = Spec.from_yaml(yaml_string)
round_trip_json_spec = Spec.from_json(json_string)
round_trip_reversed_yaml_spec = Spec.from_yaml(
reversed_yaml_string
)
round_trip_reversed_json_spec = Spec.from_yaml(
reversed_json_string
)
# TODO: remove this when build deps are in provenance.
spec = spec.copy(deps=('link', 'run'))
# specs are equal to the original
assert spec == round_trip_yaml_spec
assert spec == round_trip_json_spec
assert spec == round_trip_reversed_yaml_spec
assert spec == round_trip_reversed_json_spec
assert round_trip_yaml_spec == round_trip_reversed_yaml_spec
assert round_trip_json_spec == round_trip_reversed_json_spec
# dag_hashes are equal
assert spec.dag_hash() == round_trip_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_json_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()
