def setUp(self): self.tmpdir = tempfile.mkdtemp() self.merlin_spec_filepath = os.path.join(self.tmpdir, "basic_ensemble.yaml") with open(self.merlin_spec_filepath, "w+") as _file: _file.write(MERLIN_SPEC) self.spec = MerlinSpec.load_specification(self.merlin_spec_filepath)
def setUp(self): self.tmpdir = tempfile.mkdtemp() self.merlin_spec_filepath = os.path.join(self.tmpdir, "no_merlin.yaml") with open(self.merlin_spec_filepath, "w+") as _file: _file.write(NO_MERLIN) self.spec = MerlinSpec.load_specification(self.merlin_spec_filepath)
def expanded_spec(self): """ Determines, writes to yaml, and loads into memory an expanded specification. """ # Write expanded yaml spec self.expanded_filepath = os.path.join( self.info, self.spec.name.replace(" ", "_") + ".yaml" ) # If we are restarting, we don't need to re-expand, just need to read # in the previously expanded spec if self.restart_dir is None: self.write_expanded_spec(self.expanded_filepath) return MerlinSpec.load_specification(self.expanded_filepath)
def __init__( self, filepath, override_vars=None, restart_dir=None, samples_file=None, dry_run=False, no_errors=False, pgen_file=None, pargs=None, ): self.original_spec = MerlinSpec.load_specification(filepath) self.override_vars = override_vars error_override_vars(self.override_vars, self.original_spec.path) self.samples_file = samples_file self.label_clash_error() self.dry_run = dry_run self.no_errors = no_errors # If we load from a file, record that in the object for provenance # downstream if self.samples_file is not None: self.original_spec.merlin["samples"]["file"] = self.samples_file self.original_spec.merlin["samples"]["generate"]["cmd"] = "" self.restart_dir = restart_dir self.special_vars = { "SPECROOT": self.original_spec.specroot, "MERLIN_TIMESTAMP": self.timestamp, "MERLIN_INFO": self.info, "MERLIN_WORKSPACE": self.workspace, "OUTPUT_PATH": self.output_path, "MERLIN_SUCCESS": str(int(ReturnCode.OK)), "MERLIN_RESTART": str(int(ReturnCode.RESTART)), "MERLIN_SOFT_FAIL": str(int(ReturnCode.SOFT_FAIL)), "MERLIN_HARD_FAIL": str(int(ReturnCode.HARD_FAIL)), "MERLIN_RETRY": str(int(ReturnCode.RETRY)), } self.pgen_file = pgen_file self.pargs = pargs self.dag = None self.load_dag()
def stop_workers(args): """ CLI command for stopping all workers. :param `args`: parsed CLI arguments """ print(banner_small) worker_names = [] if args.spec: spec_path = verify_filepath(args.spec) spec = MerlinSpec.load_specification(spec_path) worker_names = spec.get_worker_names() for worker_name in worker_names: if "$" in worker_name: LOG.warning( f"Worker '{worker_name}' is unexpanded. Target provenance spec instead?" ) router.stop_workers(args.task_server, worker_names, args.queues, args.workers)
def write_expanded_spec(self, dest): """ Write a new yaml spec file with defaults and variable expansions. Useful for provenance. :param `dest`: destination for fully expanded yaml file """ # specification text including defaults and overridden user variables full_spec = dump_with_overrides(self.spec, self.override_vars) with open(dest, "w") as dumped_file: dumped_file.write(full_spec) # update spec so that user_vars update will be accurate self.spec = MerlinSpec.load_specification(dest) # expand user variables self.write_expand_by_line(dest, self.user_vars) # expand reserved words self.write_expand_by_line(dest, self.special_vars)
def expand_spec_no_study(filepath, override_vars=None): """ Get the expanded text of a spec without creating a MerlinStudy. Expansion is limited to user variables (the ones defined inside the yaml spec or at the command line). """ error_override_vars(override_vars, filepath) spec = MerlinSpec.load_specification(filepath) full_spec = dump_with_overrides(spec, override_vars) spec = MerlinSpec.load_spec_from_string(full_spec) uvars = [] if "variables" in spec.environment: uvars.append(spec.environment["variables"]) if "labels" in spec.environment: uvars.append(spec.environment["labels"]) evaluated_uvars = determine_user_variables(*uvars) return expand_by_line(full_spec.split("\n"), evaluated_uvars)
def get_adapter_config(self, override_type=None): spec = MerlinSpec.load_specification(self.spec.path) adapter_config = dict(spec.batch) if "type" not in adapter_config.keys(): adapter_config["type"] = "local" # The type may be overriden, preserve the batch type adapter_config["batch_type"] = adapter_config["type"] if override_type is not None: adapter_config["type"] = override_type # if a dry run was ordered by the yaml spec OR the cli flag, do a dry run. adapter_config["dry_run"] = self.dry_run or adapter_config["dry_run"] # Add the version if using flux to switch the command in the step if adapter_config["batch_type"] == "flux": adapter_config["flux_command"] = self.flux_command LOG.debug(f"Adapter config = {adapter_config}") return adapter_config