def output_path(self):
"""
Determines and creates an output directory for this study.
"""
if self.restart_dir is not None:
output_path = self.restart_dir
if not os.path.isdir(output_path):
raise ValueError(
f"Restart dir '{self.restart_dir}' does not exist!")
return os.path.abspath(output_path)
else:
output_path = str(self.spec.output_path)
if (self.override_vars is not None) and ("OUTPUT_PATH"
in self.override_vars):
output_path = str(self.override_vars["OUTPUT_PATH"])
output_path = expand_line(output_path, self.user_vars)
output_path = os.path.abspath(output_path)
if not os.path.isdir(output_path):
os.makedirs(output_path)
LOG.info(f"Made dir(s) to output path '{output_path}'.")
return output_path
def write_expand_by_line(self, filepath, keywords):
"""
Given a destination and keyword dictionary, expand each
line of the destination file in-place.
"""
with FileInput(filepath, inplace=True) as _file:
for line in _file:
expanded_line = expand_line(line, keywords)
print(expanded_line, end="")
def expanded_spec(self):
"""
Determines, writes to yaml, and loads into memory an expanded
specification.
"""
# If we are restarting, we don't need to re-expand, just need to read
# in the previously expanded spec
if self.restart_dir is not None:
return self.get_expanded_spec()
result = self.get_expanded_spec()
expanded_name = result.description["name"].replace(
" ", "_") + ".expanded.yaml"
# Set expanded filepath
expanded_filepath = os.path.join(self.info, expanded_name)
# expand provenance spec filename
if contains_token(self.original_spec.name) or contains_shell_ref(
self.original_spec.name):
name = f"{result.description['name'].replace(' ', '_')}_{self.timestamp}"
name = expand_line(name, {}, env_vars=True)
if "/" in name:
raise ValueError(
f"Expanded value '{name}' for field 'name' in section 'description' is not a valid filename."
)
expanded_workspace = os.path.join(self.output_path, name)
if result.merlin["samples"]:
sample_file = result.merlin["samples"]["file"]
if sample_file.startswith(self.workspace):
new_samples_file = sample_file.replace(
self.workspace, expanded_workspace)
result.merlin["samples"]["generate"][
"cmd"] = result.merlin["samples"]["generate"][
"cmd"].replace(self.workspace, expanded_workspace)
result.merlin["samples"]["file"] = new_samples_file
shutil.move(self.workspace, expanded_workspace)
self.workspace = expanded_workspace
self.info = os.path.join(self.workspace, "merlin_info")
self.special_vars["MERLIN_INFO"] = self.info
expanded_filepath = os.path.join(self.info, expanded_name)
new_spec_text = expand_by_line(result.dump(),
MerlinStudy.get_user_vars(result))
result = MerlinSpec.load_spec_from_string(new_spec_text)
result = expand_env_vars(result)
# pgen
if self.pgen_file:
env = result.get_study_environment()
result.globals = self.load_pgen(self.pgen_file, self.pargs, env)
# copy the --samplesfile (if any) into merlin_info
if self.samples_file:
shutil.copyfile(
self.samples_file,
os.path.join(self.info, os.path.basename(self.samples_file)),
)
# write expanded spec for provenance
with open(expanded_filepath, "w") as f:
f.write(result.dump())
# write original spec for provenance
result = MerlinSpec.load_spec_from_string(result.dump())
result.path = expanded_filepath
name = result.description["name"].replace(" ", "_")
self.write_original_spec(name)
# write partially-expanded spec for provenance
partial_spec = deepcopy(self.original_spec)
if "variables" in result.environment:
partial_spec.environment["variables"] = result.environment[
"variables"]
if "labels" in result.environment:
partial_spec.environment["labels"] = result.environment["labels"]
partial_spec_path = os.path.join(self.info, name + ".partial.yaml")
with open(partial_spec_path, "w") as f:
f.write(partial_spec.dump())
LOG.info(f"Study workspace is '{self.workspace}'.")
return result