def _read_spec_from_yaml(self, hash_key, installs, parent_key=None):
"""Recursively construct a spec from a hash in a YAML database.
Does not do any locking.
"""
spec_dict = installs[hash_key]['spec']
# Install records don't include hash with spec, so we add it in here
# to ensure it is read properly.
for name in spec_dict:
spec_dict[name]['hash'] = hash_key
# Build spec from dict first.
spec = Spec.from_node_dict(spec_dict)
# Add dependencies from other records in the install DB to
# form a full spec.
for dep_hash in spec_dict[spec.name]['dependencies'].values():
child = self._read_spec_from_yaml(dep_hash, installs, hash_key)
spec._add_dependency(child)
# Specs from the database need to be marked concrete because
# they represent actual installations.
spec._mark_concrete()
return spec
def installcosmo(self, args):
# Extract and concretize cosmo_spec
if not args.spec:
tty.die("spack dev-build requires a package spec argument.")
specs = spack.cmd.parse_specs(args.spec)
if len(specs) > 1:
tty.die("spack dev-build only takes one spec.")
cosmo_spec = specs[0]
# Collect user-specified versions of dependencies before first concretization
user_versioned_deps = set()
for dep in cosmo_spec.traverse():
if dep.name != "cosmo" and dep.versions and len(dep.versions) != 0:
user_versioned_deps.add(dep.name)
cosmo_spec.concretize()
package = spack.repo.get(cosmo_spec)
package.do_fetch()
package.do_stage()
# Load serialized yaml from inside cloned repo
with open(package.stage.source_path + "/cosmo/ACC/spack/spec.yaml",
"r") as f:
try:
data = yaml.load(f)
except yaml.error.MarkedYAMLError as e:
raise syaml.SpackYAMLError("error parsing YAML spec:", str(e))
# Read nodes out of list. Root spec (cosmo) is the first element;
# dependencies are the following elements.
spec_list = [Spec.from_node_dict(node) for node in data["spec"]]
if not spec_list:
raise spack.error.SpecError("YAML spec contains no nodes.")
# Take only the dependencies
deps_serialized_dict = dict((spec.name, spec) for spec in spec_list)
# Selectively substitute the dependencies' versions with those found in the deserialized list of specs
# The order of precedence in the choice of a dependency's version becomes:
# 1. the one provided by the user in the command,
# 2. the one specified in spec.yaml,
# 3. the default prescribed by the spack package.
for dep in cosmo_spec.traverse():
if dep.name in deps_serialized_dict and not dep.name in user_versioned_deps:
dep.versions = deps_serialized_dict[dep.name].versions.copy()
if dep.name == "cosmo-dycore":
dep.versions = cosmo_spec.versions.copy()
# re-concretize
cosmo_spec = spack.cmd.parse_specs(cosmo_spec.__str__())[0]
cosmo_spec.concretize()
# Dev-build cosmo
custom_install(cosmo_spec, args)
def _read_spec_from_yaml(self, hash_key, installs, parent_key=None):
"""Recursively construct a spec from a hash in a YAML database.
Does not do any locking.
"""
if hash_key not in installs:
parent = read_spec(installs[parent_key]['path'])
spec_dict = installs[hash_key]['spec']
# Build spec from dict first.
spec = Spec.from_node_dict(spec_dict)
# Add dependencies from other records in the install DB to
# form a full spec.
for dep_hash in spec_dict[spec.name]['dependencies'].values():
child = self._read_spec_from_yaml(dep_hash, installs, hash_key)
spec._add_dependency(child)
return spec
def _read_lockfile_dict(self, d):
"""Read a lockfile dictionary into this environment."""
roots = d['roots']
self.concretized_user_specs = [Spec(r['spec']) for r in roots]
self.concretized_order = [r['hash'] for r in roots]
json_specs_by_hash = d['concrete_specs']
root_hashes = set(self.concretized_order)
specs_by_hash = {}
for dag_hash, node_dict in json_specs_by_hash.items():
specs_by_hash[dag_hash] = Spec.from_node_dict(node_dict)
for dag_hash, node_dict in json_specs_by_hash.items():
for dep_name, dep_hash, deptypes in (
Spec.dependencies_from_node_dict(node_dict)):
specs_by_hash[dag_hash]._add_dependency(
specs_by_hash[dep_hash], deptypes)
self.specs_by_hash = dict(
(x, y) for x, y in specs_by_hash.items() if x in root_hashes)
def _read_lockfile_dict(self, d):
"""Read a lockfile dictionary into this environment."""
roots = d['roots']
self.concretized_user_specs = [Spec(r['spec']) for r in roots]
self.concretized_order = [r['hash'] for r in roots]
json_specs_by_hash = d['concrete_specs']
root_hashes = set(self.concretized_order)
specs_by_hash = {}
for dag_hash, node_dict in json_specs_by_hash.items():
specs_by_hash[dag_hash] = Spec.from_node_dict(node_dict)
for dag_hash, node_dict in json_specs_by_hash.items():
for dep_name, dep_hash, deptypes in (
Spec.dependencies_from_node_dict(node_dict)):
specs_by_hash[dag_hash]._add_dependency(
specs_by_hash[dep_hash], deptypes)
self.specs_by_hash = dict(
(x, y) for x, y in specs_by_hash.items() if x in root_hashes)
def _read_lockfile_dict(self, d):
"""Read a lockfile dictionary into this environment."""
roots = d['roots']
self.concretized_user_specs = [Spec(r['spec']) for r in roots]
self.concretized_order = [r['hash'] for r in roots]
json_specs_by_hash = d['concrete_specs']
root_hashes = set(self.concretized_order)
specs_by_hash = {}
for dag_hash, node_dict in json_specs_by_hash.items():
specs_by_hash[dag_hash] = Spec.from_node_dict(node_dict)
for dag_hash, node_dict in json_specs_by_hash.items():
for dep_name, dep_hash, deptypes in (
Spec.dependencies_from_node_dict(node_dict)):
specs_by_hash[dag_hash]._add_dependency(
specs_by_hash[dep_hash], deptypes)
# If we are reading an older lockfile format (which uses dag hashes
# that exclude build deps), we use this to convert the old
# concretized_order to the full hashes (preserving the order)
old_hash_to_new = {}
self.specs_by_hash = {}
for _, spec in specs_by_hash.items():
dag_hash = spec.dag_hash()
build_hash = spec.build_hash()
if dag_hash in root_hashes:
old_hash_to_new[dag_hash] = build_hash
if (dag_hash in root_hashes or build_hash in root_hashes):
self.specs_by_hash[build_hash] = spec
if old_hash_to_new:
# Replace any older hashes in concretized_order with hashes
# that include build deps
self.concretized_order = [
old_hash_to_new.get(h, h) for h in self.concretized_order
]
def _read_spec_from_yaml(self, hash_key, installs, parent_key=None):
"""Recursively construct a spec from a hash in a YAML database.
Does not do any locking.
"""
if hash_key not in installs:
parent = read_spec(installs[parent_key]['path'])
spec_dict = installs[hash_key]['spec']
# Build spec from dict first.
spec = Spec.from_node_dict(spec_dict)
# Add dependencies from other records in the install DB to
# form a full spec.
for dep_hash in spec_dict[spec.name]['dependencies'].values():
child = self._read_spec_from_yaml(dep_hash, installs, hash_key)
spec._add_dependency(child)
# Specs from the database need to be marked concrete because
# they represent actual installations.
spec._mark_concrete()
return spec
def _read_spec_from_yaml(self, hash_key, installs, parent_key=None):
"""Recursively construct a spec from a hash in a YAML database.
Does not do any locking.
"""
if hash_key not in installs:
parent = read_spec(installs[parent_key]["path"])
spec_dict = installs[hash_key]["spec"]
# Build spec from dict first.
spec = Spec.from_node_dict(spec_dict)
# Add dependencies from other records in the install DB to
# form a full spec.
for dep_hash in spec_dict[spec.name]["dependencies"].values():
child = self._read_spec_from_yaml(dep_hash, installs, hash_key)
spec._add_dependency(child)
# Specs from the database need to be marked concrete because
# they represent actual installations.
spec._mark_concrete()
return spec
def devbuildcosmo(self, args):
# Extract and concretize cosmo_spec
if not args.spec:
tty.die("spack dev-build requires a package spec argument.")
# validate specs
specs = spack.cmd.parse_specs(args.spec)
if len(specs) > 1:
tty.die("spack dev-build only takes one spec.")
cosmo_spec = spack.cmd.parse_specs(args.spec)[0]
# Collect user-specified versions of dependencies before first concretization
user_versioned_deps = set()
for dep in cosmo_spec.traverse():
if dep.name != "cosmo" and dep.versions and len(dep.versions) != 0:
user_versioned_deps.add(dep.name)
cosmo_spec.concretize()
# Setting source_path to current working directory
source_path = os.getcwd()
source_path = os.path.abspath(source_path)
if not args.no_specyaml:
# Load serialized yaml from inside cloned repo
with open(source_path + "/cosmo/ACC/spack/spec.yaml", "r") as f:
try:
data = yaml.load(f)
except yaml.error.MarkedYAMLError as e:
raise syaml.SpackYAMLError("error parsing YAML spec:", str(e))
# Read nodes out of list. Root spec (cosmo) is the first element;
# dependencies are the following elements.
spec_list = [Spec.from_node_dict(node) for node in data["spec"]]
if not spec_list:
raise spack.error.SpecError("YAML spec contains no nodes.")
# Take only the dependencies
deps_serialized_dict = dict((spec.name, spec) for spec in spec_list)
# Selectively substitute the dependencies' versions with those found in the deserialized list of specs
# The order of precedence in the choice of a dependency's version becomes:
# 1. the one specified in spec.yaml,
# 2. the one provided by the user in the command,
# 3. the default prescribed by the spack package.
for dep in cosmo_spec.traverse():
if dep.name in deps_serialized_dict and not dep.name in user_versioned_deps:
dep.versions = deps_serialized_dict[dep.name].versions.copy()
if dep.name == "cosmo-dycore":
dep.versions = cosmo_spec.versions.copy()
# Re-concretize Spec: to enforce checking of constraints after update of versions.
# Note: unfortunately there is no better way to re-concretize a Spec (e.g. via API).
# The only solution seems to be going via string representation and then back
# into a Spec object. It's not elegant but it seems to work very well.
cosmo_spec = spack.cmd.parse_specs(cosmo_spec.__str__())[0]
cosmo_spec.concretize()
# Clean if needed
if args.clean_build:
print("\033[92m" + "==> " + "\033[0m" +
"cosmo: Cleaning build directory")
# set F90 to prevent abort defined in Makefile
os.environ["F90"] = "NOTSET"
subprocess.run(["make", "clean"], cwd=source_path + "/cosmo/ACC")
os.environ.pop("F90")
if os.path.exists(source_path + "/spack-build"):
print("\033[92m" + "==> " + "\033[0m" +
"dycore: Cleaning build directory")
shutil.rmtree(source_path + "/spack-build")
if cosmo_spec.satisfies("+cppdycore"):
dycore_spec = cosmo_spec.get_dependency("cosmo-dycore").spec
custom_devbuild(source_path, dycore_spec, args)
# Dev-build cosmo
custom_devbuild(source_path, cosmo_spec, args)
# Serialize data
if "+serialize" in cosmo_spec:
print("\033[92m" + "==> " + "\033[0m" + "cosmo: Serializing data")
try:
subprocess.run([
source_path + '/cosmo/ACC/test/tools/serialize_cosmo.py',
self.spec__str__(), '-b', source_path
],
check=True,
stderr=subprocess.STDOUT)
except:
tty.die('Serialization failed')
