def output_path(self): """ Determines and creates an output directory for this study. """ if self.restart_dir is not None: output_path = self.restart_dir if not os.path.isdir(output_path): raise ValueError( f"Restart dir '{self.restart_dir}' does not exist!") return os.path.abspath(output_path) else: output_path = str(self.spec.output_path) if (self.override_vars is not None) and ("OUTPUT_PATH" in self.override_vars): output_path = str(self.override_vars["OUTPUT_PATH"]) output_path = expand_line(output_path, self.user_vars) output_path = os.path.abspath(output_path) if not os.path.isdir(output_path): os.makedirs(output_path) LOG.info(f"Made dir(s) to output path '{output_path}'.") return output_path
def write_expand_by_line(self, filepath, keywords): """ Given a destination and keyword dictionary, expand each line of the destination file in-place. """ with FileInput(filepath, inplace=True) as _file: for line in _file: expanded_line = expand_line(line, keywords) print(expanded_line, end="")
def expanded_spec(self): """ Determines, writes to yaml, and loads into memory an expanded specification. """ # If we are restarting, we don't need to re-expand, just need to read # in the previously expanded spec if self.restart_dir is not None: return self.get_expanded_spec() result = self.get_expanded_spec() expanded_name = result.description["name"].replace( " ", "_") + ".expanded.yaml" # Set expanded filepath expanded_filepath = os.path.join(self.info, expanded_name) # expand provenance spec filename if contains_token(self.original_spec.name) or contains_shell_ref( self.original_spec.name): name = f"{result.description['name'].replace(' ', '_')}_{self.timestamp}" name = expand_line(name, {}, env_vars=True) if "/" in name: raise ValueError( f"Expanded value '{name}' for field 'name' in section 'description' is not a valid filename." ) expanded_workspace = os.path.join(self.output_path, name) if result.merlin["samples"]: sample_file = result.merlin["samples"]["file"] if sample_file.startswith(self.workspace): new_samples_file = sample_file.replace( self.workspace, expanded_workspace) result.merlin["samples"]["generate"][ "cmd"] = result.merlin["samples"]["generate"][ "cmd"].replace(self.workspace, expanded_workspace) result.merlin["samples"]["file"] = new_samples_file shutil.move(self.workspace, expanded_workspace) self.workspace = expanded_workspace self.info = os.path.join(self.workspace, "merlin_info") self.special_vars["MERLIN_INFO"] = self.info expanded_filepath = os.path.join(self.info, expanded_name) new_spec_text = expand_by_line(result.dump(), MerlinStudy.get_user_vars(result)) result = MerlinSpec.load_spec_from_string(new_spec_text) result = expand_env_vars(result) # pgen if self.pgen_file: env = result.get_study_environment() result.globals = self.load_pgen(self.pgen_file, self.pargs, env) # copy the --samplesfile (if any) into merlin_info if self.samples_file: shutil.copyfile( self.samples_file, os.path.join(self.info, os.path.basename(self.samples_file)), ) # write expanded spec for provenance with open(expanded_filepath, "w") as f: f.write(result.dump()) # write original spec for provenance result = MerlinSpec.load_spec_from_string(result.dump()) result.path = expanded_filepath name = result.description["name"].replace(" ", "_") self.write_original_spec(name) # write partially-expanded spec for provenance partial_spec = deepcopy(self.original_spec) if "variables" in result.environment: partial_spec.environment["variables"] = result.environment[ "variables"] if "labels" in result.environment: partial_spec.environment["labels"] = result.environment["labels"] partial_spec_path = os.path.join(self.info, name + ".partial.yaml") with open(partial_spec_path, "w") as f: f.write(partial_spec.dump()) LOG.info(f"Study workspace is '{self.workspace}'.") return result