def get_spec_with_expansion(filepath, override_vars=None): """ Return a MerlinSpec with overrides and expansion, without creating a MerlinStudy. """ expanded_spec_text = expand_spec_no_study(filepath, override_vars) return MerlinSpec.load_spec_from_string(expanded_spec_text)
def get_expanded_spec(self): """ Get a new yaml spec file with defaults, cli overrides, and variable expansions. Useful for provenance. """ # get specification including defaults and cli-overridden user variables full_spec_text = dump_with_overrides(self.original_spec, self.override_vars) new_spec = MerlinSpec.load_spec_from_string(full_spec_text) # expand user variables new_spec_text = expand_by_line( new_spec.dump(), MerlinStudy.get_user_vars(new_spec) ) # expand reserved words new_spec_text = expand_by_line(new_spec_text, self.special_vars) return MerlinSpec.load_spec_from_string(new_spec_text)
def get_expanded_spec(self): """ Get a new yaml spec file with defaults, cli overrides, and variable expansions. Useful for provenance. """ # get specification including defaults and cli-overridden user variables new_env = replace_override_vars(self.original_spec.environment, self.override_vars) new_spec = deepcopy(self.original_spec) new_spec.environment = new_env # expand user variables new_spec_text = expand_by_line(new_spec.dump(), MerlinStudy.get_user_vars(new_spec)) # expand reserved words new_spec_text = expand_by_line(new_spec_text, self.special_vars) result = MerlinSpec.load_spec_from_string(new_spec_text) return expand_env_vars(result)
def expand_spec_no_study(filepath, override_vars=None): """ Get the expanded text of a spec without creating a MerlinStudy. Expansion is limited to user variables (the ones defined inside the yaml spec or at the command line). """ error_override_vars(override_vars, filepath) spec = MerlinSpec.load_specification(filepath) full_spec = dump_with_overrides(spec, override_vars) spec = MerlinSpec.load_spec_from_string(full_spec) uvars = [] if "variables" in spec.environment: uvars.append(spec.environment["variables"]) if "labels" in spec.environment: uvars.append(spec.environment["labels"]) evaluated_uvars = determine_user_variables(*uvars) return expand_by_line(full_spec.split("\n"), evaluated_uvars)
def expanded_spec(self): """ Determines, writes to yaml, and loads into memory an expanded specification. """ # If we are restarting, we don't need to re-expand, just need to read # in the previously expanded spec if self.restart_dir is not None: return self.get_expanded_spec() result = self.get_expanded_spec() expanded_name = result.description["name"].replace( " ", "_") + ".expanded.yaml" # Set expanded filepath expanded_filepath = os.path.join(self.info, expanded_name) # expand provenance spec filename if contains_token(self.original_spec.name) or contains_shell_ref( self.original_spec.name): name = f"{result.description['name'].replace(' ', '_')}_{self.timestamp}" name = expand_line(name, {}, env_vars=True) if "/" in name: raise ValueError( f"Expanded value '{name}' for field 'name' in section 'description' is not a valid filename." ) expanded_workspace = os.path.join(self.output_path, name) if result.merlin["samples"]: sample_file = result.merlin["samples"]["file"] if sample_file.startswith(self.workspace): new_samples_file = sample_file.replace( self.workspace, expanded_workspace) result.merlin["samples"]["generate"][ "cmd"] = result.merlin["samples"]["generate"][ "cmd"].replace(self.workspace, expanded_workspace) result.merlin["samples"]["file"] = new_samples_file shutil.move(self.workspace, expanded_workspace) self.workspace = expanded_workspace self.info = os.path.join(self.workspace, "merlin_info") self.special_vars["MERLIN_INFO"] = self.info expanded_filepath = os.path.join(self.info, expanded_name) new_spec_text = expand_by_line(result.dump(), MerlinStudy.get_user_vars(result)) result = MerlinSpec.load_spec_from_string(new_spec_text) result = expand_env_vars(result) # pgen if self.pgen_file: env = result.get_study_environment() result.globals = self.load_pgen(self.pgen_file, self.pargs, env) # copy the --samplesfile (if any) into merlin_info if self.samples_file: shutil.copyfile( self.samples_file, os.path.join(self.info, os.path.basename(self.samples_file)), ) # write expanded spec for provenance with open(expanded_filepath, "w") as f: f.write(result.dump()) # write original spec for provenance result = MerlinSpec.load_spec_from_string(result.dump()) result.path = expanded_filepath name = result.description["name"].replace(" ", "_") self.write_original_spec(name) # write partially-expanded spec for provenance partial_spec = deepcopy(self.original_spec) if "variables" in result.environment: partial_spec.environment["variables"] = result.environment[ "variables"] if "labels" in result.environment: partial_spec.environment["labels"] = result.environment["labels"] partial_spec_path = os.path.join(self.info, name + ".partial.yaml") with open(partial_spec_path, "w") as f: f.write(partial_spec.dump()) LOG.info(f"Study workspace is '{self.workspace}'.") return result