def test_post_prior(tmpdir): # Generate original chain info: InputDict = { "output": os.path.join(tmpdir, "gaussian"), "force": True, "params": info_params, "sampler": info_sampler, "likelihood": {"one": None}, "prior": {"gaussian": sampled_pdf}} info_post: InputDict = { "output": info["output"], "force": True, "post": {"suffix": "foo", 'skip': 0.1, "remove": {"prior": {"gaussian": None}}, "add": {"prior": {"target": target_pdf_prior}}}} _, sampler = run(info) if mpi.is_main_process(): mcsamples_in = loadMCSamples(info["output"], settings={'ignore_rows': 0.1}) target_mean, target_cov = mpi.share(_get_targets(mcsamples_in)) else: target_mean, target_cov = mpi.share() for mem in [False, True]: post(info_post, sample=sampler.products()["sample"] if mem else None) # Load with GetDist and compare if mpi.is_main_process(): mcsamples = loadMCSamples( info_post["output"] + _post_ + info_post["post"]["suffix"]) new_mean = mcsamples.mean(["a", "b"]) new_cov = mcsamples.getCovMat().matrix mpi.share((new_mean, new_cov)) else: new_mean, new_cov = mpi.share() assert np.allclose(new_mean, target_mean) assert np.allclose(new_cov, target_cov)
def test_post_params(): # Tests: # - added simple dynamical derived parameter "a+b" # - added dynamical derived parameter that depends on *new* chi2__target # - added new fixed input "c" + new derived-from-external-function "cprime" # Generate original chain info = { "params": info_params, "sampler": info_sampler_dummy, "likelihood": {"gaussian": sampled_pdf}} updated_info_gaussian, sampler_gaussian = run(info) products_gaussian = sampler_gaussian.products() info_post = { "post": {"suffix": "foo", "remove": {"params": {"a_plus_b": None}}, "add": { "likelihood": { "target": {"external": target_pdf, "output_params": ["cprime"]}}, "params": { "c": 1.234, "a_minus_b": {"derived": "lambda a,b: a-b"}, "my_chi2__target": { "derived": "lambda chi2__target: chi2__target"}, "cprime": None}}}} info_post.update(updated_info_gaussian) products = post(info_post, products_gaussian["sample"]).products # Compare parameters assert np.allclose( products["sample"]["a"] - products["sample"]["b"], products["sample"]["a_minus_b"]) assert np.allclose( products["sample"]["cprime"], info_post["post"]["add"]["params"]["c"]) assert np.allclose( products["sample"]["my_chi2__target"], products["sample"]["chi2__target"])
def test_post_prior(tmpdir): # Generate original chain info = { _output_prefix: os.path.join(str(tmpdir), "gaussian"), _force: True, _params: info_params, kinds.sampler: info_sampler, kinds.likelihood: { "one": None }, _prior: { "gaussian": sampled_pdf } } run(info) info_post = { _output_prefix: info[_output_prefix], _force: True, _post: { _post_suffix: "foo", _post_remove: { _prior: { "gaussian": None } }, _post_add: { _prior: { "target": target_pdf_prior } } } } post(info_post) # Load with GetDist and compare mcsamples = loadMCSamples(info_post[_output_prefix] + _post_ + info_post[_post][_post_suffix]) new_mean = mcsamples.mean(["a", "b"]) new_cov = mcsamples.getCovMat().matrix assert abs(KL_norm(target["mean"], target["cov"], new_mean, new_cov)) < 0.02
def test_post_prior(tmpdir): # Generate original chain info = { "output": os.path.join(str(tmpdir), "gaussian"), "force": True, "params": info_params, "sampler": info_sampler, "likelihood": { "one": None }, "prior": { "gaussian": sampled_pdf } } run(info) info_post = { "output": info["output"], "force": True, "post": { "suffix": "foo", "remove": { "prior": { "gaussian": None } }, "add": { "prior": { "target": target_pdf } } } } post(info_post) # Load with GetDist and compare mcsamples = loadMCSamples(info_post["output"] + "_post_" + info_post["post"]["suffix"]) new_mean = mcsamples.mean(["a", "b"]) new_cov = mcsamples.getCovMat().matrix assert abs(KL_norm(target["mean"], target["cov"], new_mean, new_cov)) < 0.02
def test_post_params(): # Tests: # - added simple dynamical derived parameter "a+b" # - added dynamical derived parameter that depends on *new* chi2__target # - added new fixed input "c" + new derived-from-external-function "cprime" # Generate original chain info = { _params: info_params, _sampler: info_sampler_dummy, _likelihood: { "gaussian": sampled_pdf } } updated_info_gaussian, products_gaussian = run(info) info_post = { _post: { _post_suffix: "foo", _post_remove: { _params: { "a_plus_b": None } }, _post_add: { _likelihood: { "target": target_pdf }, _params: { "c": 1.234, "a_minus_b": { _p_derived: "lambda a,b: a-b" }, "my_chi2__target": { _p_derived: "lambda chi2__target: chi2__target" }, "cprime": None } } } } info_post.update(updated_info_gaussian) updated_info, products = post(info_post, products_gaussian["sample"]) # Compare parameters assert np.allclose(products["sample"]["a"] - products["sample"]["b"], products["sample"]["a_minus_b"]) assert np.allclose(products["sample"]["cprime"], info_post[_post][_post_add][_params]["c"]) assert np.allclose(products["sample"]["my_chi2__target"], products["sample"]["chi2__target"])
def run( info_or_yaml_or_file: Union[InputDict, str, os.PathLike], packages_path: Optional[str] = None, output: Union[str, LiteralFalse, None] = None, debug: Union[bool, int, None] = None, stop_at_error: Optional[bool] = None, resume: bool = False, force: bool = False, no_mpi: bool = False, test: bool = False, override: Optional[InputDict] = None, ) -> Union[InfoSamplerTuple, PostTuple]: """ Run from an input dictionary, file name or yaml string, with optional arguments to override settings in the input as needed. :param info_or_yaml_or_file: input options dictionary, yaml file, or yaml text :param packages_path: path where external packages were installed :param output: path name prefix for output files, or False for no file output :param debug: true for verbose debug output, or a specific logging level :param stop_at_error: stop if an error is raised :param resume: continue an existing run :param force: overwrite existing output if it exists :param no_mpi: run without MPI :param test: only test initialization rather than actually running :param override: option dictionary to merge into the input one, overriding settings (but with lower precedence than the explicit keyword arguments) :return: (updated_info, sampler) tuple of options dictionary and Sampler instance, or (updated_info, results) if using "post" post-processing """ # This function reproduces the model-->output-->sampler pipeline one would follow # when instantiating by hand, but alters the order to performs checks and dump info # as early as possible, e.g. to check if resuming possible or `force` needed. if no_mpi or test: mpi.set_mpi_disabled() with mpi.ProcessState("run"): info: InputDict = load_info_overrides(info_or_yaml_or_file, debug, stop_at_error, packages_path, override) if test: info["test"] = True # If any of resume|force given as cmd args, ignore those in the input file if resume or force: if resume and force: raise ValueError("'rename' and 'force' are exclusive options") info["resume"] = bool(resume) info["force"] = bool(force) if info.get("post"): if isinstance(output, str) or output is False: info["post"]["output"] = output or None return post(info) if isinstance(output, str) or output is False: info["output"] = output or None logger_setup(info.get("debug"), info.get("debug_file")) logger_run = get_logger(run.__name__) # MARKED FOR DEPRECATION IN v3.0 # BEHAVIOUR TO BE REPLACED BY ERROR: check_deprecated_modules_path(info) # END OF DEPRECATION BLOCK # 1. Prepare output driver, if requested by defining an output_prefix # GetDist needs to know the original sampler, so don't overwrite if minimizer try: which_sampler = list(info["sampler"])[0] except (KeyError, TypeError): raise LoggedError( logger_run, "You need to specify a sampler using the 'sampler' key " "as e.g. `sampler: {mcmc: None}.`") infix = "minimize" if which_sampler == "minimize" else None with get_output(prefix=info.get("output"), resume=info.get("resume"), force=info.get("force"), infix=infix) as out: # 2. Update the input info with the defaults for each component updated_info = update_info(info) if is_debug(logger_run): # Dump only if not doing output # (otherwise, the user can check the .updated file) if not out and mpi.is_main_process(): logger_run.info( "Input info updated with defaults (dumped to YAML):\n%s", yaml_dump(sort_cosmetic(updated_info))) # 3. If output requested, check compatibility if existing one, and dump. # 3.1 First: model only out.check_and_dump_info(info, updated_info, cache_old=True, ignore_blocks=["sampler"]) # 3.2 Then sampler -- 1st get the last sampler mentioned in the updated.yaml # TODO: ideally, using Minimizer would *append* to the sampler block. # Some code already in place, but not possible at the moment. try: last_sampler = list(updated_info["sampler"])[-1] last_sampler_info = { last_sampler: updated_info["sampler"][last_sampler] } except (KeyError, TypeError): raise LoggedError(logger_run, "No sampler requested.") sampler_name, sampler_class = get_sampler_name_and_class( last_sampler_info) check_sampler_info((out.reload_updated_info(use_cache=True) or {}).get("sampler"), updated_info["sampler"], is_resuming=out.is_resuming()) # Dump again, now including sampler info out.check_and_dump_info(info, updated_info, check_compatible=False) # Check if resumable run sampler_class.check_force_resume( out, info=updated_info["sampler"][sampler_name]) # 4. Initialize the posterior and the sampler with Model(updated_info["params"], updated_info["likelihood"], updated_info.get("prior"), updated_info.get("theory"), packages_path=info.get("packages_path"), timing=updated_info.get("timing"), allow_renames=False, stop_at_error=info.get("stop_at_error", False)) as model: # Re-dump the updated info, now containing parameter routes and version updated_info = recursive_update(updated_info, model.info()) out.check_and_dump_info(None, updated_info, check_compatible=False) sampler = sampler_class( updated_info["sampler"][sampler_name], model, out, name=sampler_name, packages_path=info.get("packages_path")) # Re-dump updated info, now also containing updates from the sampler updated_info["sampler"][sampler_name] = \ recursive_update(updated_info["sampler"][sampler_name], sampler.info()) out.check_and_dump_info(None, updated_info, check_compatible=False) mpi.sync_processes() if info.get("test", False): logger_run.info( "Test initialization successful! " "You can probably run now without `--%s`.", "test") return InfoSamplerTuple(updated_info, sampler) # Run the sampler sampler.run() return InfoSamplerTuple(updated_info, sampler)
def test_post_likelihood(): """ Swaps likelihood "gaussian" for "target". It also tests aggregated chi2's by removing and adding a likelihood to an existing type. """ # Generate original chain orig_interval = OutputOptions.output_inteveral_s try: OutputOptions.output_inteveral_s = 0 info_params_local = deepcopy(info_params) info_params_local["dummy"] = 0 dummy_loglike_add = 0.1 dummy_loglike_remove = 0.01 info = { "output": None, "force": True, "params": info_params_local, "sampler": info_sampler, "likelihood": { "gaussian": { "external": sampled_pdf, "type": "A" }, "dummy": { "external": lambda dummy: 1, "type": "BB" }, "dummy_remove": { "external": lambda dummy: dummy_loglike_add, "type": "BB" } } } info_out, sampler = run(info) samples_in = mpi.gather(sampler.products()["sample"]) if mpi.is_main_process(): mcsamples_in = MCSamplesFromCobaya(info_out, samples_in) else: mcsamples_in = None info_out.update({ "post": { "suffix": "foo", "remove": { "likelihood": { "gaussian": None, "dummy_remove": None } }, "add": { "likelihood": { "target": { "external": target_pdf, "type": "A", "output_params": ["cprime"] }, "dummy_add": { "external": lambda dummy: dummy_loglike_remove, "type": "BB" } } } } }) info_post_out, products_post = post(info_out, sampler.products()["sample"]) samples = mpi.gather(products_post["sample"]) # Load with GetDist and compare if mcsamples_in: target_mean, target_cov = mpi.share(_get_targets(mcsamples_in)) mcsamples = MCSamplesFromCobaya(info_post_out, samples, name_tag="sample") new_mean = mcsamples.mean(["a", "b"]) new_cov = mcsamples.getCovMat().matrix mpi.share((new_mean, new_cov)) else: target_mean, target_cov = mpi.share() new_mean, new_cov = mpi.share() assert np.allclose(new_mean, target_mean) assert np.allclose(new_cov, target_cov) assert allclose(products_post["sample"]["chi2__A"], products_post["sample"]["chi2__target"]) assert allclose( products_post["sample"]["chi2__BB"], products_post["sample"]["chi2__dummy"] + products_post["sample"]["chi2__dummy_add"]) finally: OutputOptions.output_inteveral_s = orig_interval
def test_post_likelihood(tmpdir): """ Swaps likelihood "gaussian" for "target". It also tests aggregated chi2's by removing and adding a likelihood to an existing type. """ # Generate original chain info_params_local = deepcopy(info_params) info_params_local["dummy"] = 0 dummy_loglike_add = 0.1 dummy_loglike_remove = 0.01 info = { _output_prefix: os.path.join(str(tmpdir), "gaussian"), _force: True, _params: info_params_local, kinds.sampler: info_sampler, kinds.likelihood: { "gaussian": { "external": sampled_pdf, "type": "A" }, "dummy": { "external": lambda dummy: 1, "type": "B" }, "dummy_remove": { "external": lambda dummy: dummy_loglike_add, "type": "B" } } } info_run_out, sampler_run = run(info) info_post = { _output_prefix: info[_output_prefix], _force: True, _post: { _post_suffix: "foo", _post_remove: { kinds.likelihood: { "gaussian": None, "dummy_remove": None } }, _post_add: { kinds.likelihood: { "target": { "external": target_pdf, "type": "A", "output_params": ["cprime"] }, "dummy_add": { "external": lambda dummy: dummy_loglike_remove, "type": "B" } } } } } info_post_out, products_post = post(info_post) # Load with GetDist and compare mcsamples = loadMCSamples(info_post[_output_prefix] + _post_ + info_post[_post][_post_suffix]) new_mean = mcsamples.mean(["a", "b"]) new_cov = mcsamples.getCovMat().matrix assert abs(KL_norm(target["mean"], target["cov"], new_mean, new_cov)) < 0.02 assert np.allclose(products_post["sample"]["chi2__A"], products_post["sample"]["chi2__target"]) assert np.allclose( products_post["sample"]["chi2__B"], products_post["sample"]["chi2__dummy"] + products_post["sample"]["chi2__dummy_add"])
def run_script(): warn_deprecation() import os import argparse parser = argparse.ArgumentParser(description="Cobaya's run script.") parser.add_argument("input_file", nargs=1, action="store", metavar="input_file.yaml", help="An input file to run.") parser.add_argument("-" + _packages_path_arg[0], "--" + _packages_path_arg_posix, action="store", nargs=1, metavar="/packages/path", default=[None], help="Path where external packages were installed.") # MARKED FOR DEPRECATION IN v3.0 modules = "modules" parser.add_argument("-" + modules[0], "--" + modules, action="store", nargs=1, required=False, metavar="/packages/path", default=[None], help="To be deprecated! " "Alias for %s, which should be used instead." % _packages_path_arg_posix) # END OF DEPRECATION BLOCK -- CONTINUES BELOW! parser.add_argument("-" + _output_prefix[0], "--" + _output_prefix, action="store", nargs=1, metavar="/some/path", default=[None], help="Path and prefix for the text output.") parser.add_argument("-" + _debug[0], "--" + _debug, action="store_true", help="Produce verbose debug output.") continuation = parser.add_mutually_exclusive_group(required=False) continuation.add_argument( "-" + _resume[0], "--" + _resume, action="store_true", help="Resume an existing chain if it has similar info " "(fails otherwise).") continuation.add_argument("-" + _force[0], "--" + _force, action="store_true", help="Overwrites previous output, if it exists " "(use with care!)") parser.add_argument("--%s" % _test_run, action="store_true", help="Initialize model and sampler, and exit.") parser.add_argument("--version", action="version", version=__version__) parser.add_argument("--no-mpi", action='store_true', help="disable MPI when mpi4py installed but MPI does " "not actually work") arguments = parser.parse_args() if arguments.no_mpi or getattr(arguments, _test_run, False): set_mpi_disabled() if any((os.path.splitext(f)[0] in ("input", "updated")) for f in arguments.input_file): raise ValueError("'input' and 'updated' are reserved file names. " "Please, use a different one.") load_input = import_MPI(".input", "load_input") given_input = arguments.input_file[0] if any(given_input.lower().endswith(ext) for ext in _yaml_extensions): info = load_input(given_input) output_prefix_cmd = getattr(arguments, _output_prefix)[0] output_prefix_input = info.get(_output_prefix) info[_output_prefix] = output_prefix_cmd or output_prefix_input else: # Passed an existing output_prefix? Try to find the corresponding *.updated.yaml updated_file = get_info_path(*split_prefix(given_input), kind="updated") try: info = load_input(updated_file) except IOError: raise ValueError( "Not a valid input file, or non-existent run to resume") # We need to update the output_prefix to resume the run *where it is* info[_output_prefix] = given_input # If input given this way, we obviously want to resume! info[_resume] = True # solve packages installation path cmd > env > input # MARKED FOR DEPRECATION IN v3.0 if getattr(arguments, modules) != [None]: logger_setup() logger = logging.getLogger(__name__.split(".")[-1]) logger.warning( "*DEPRECATION*: -m/--modules will be deprecated in favor of " "-%s/--%s in the next version. Please, use that one instead.", _packages_path_arg[0], _packages_path_arg_posix) # BEHAVIOUR TO BE REPLACED BY ERROR: if getattr(arguments, _packages_path_arg) == [None]: setattr(arguments, _packages_path_arg, getattr(arguments, modules)) # BEHAVIOUR TO BE REPLACED BY ERROR: check_deprecated_modules_path(info) # END OF DEPRECATION BLOCK info[_packages_path] = \ getattr(arguments, _packages_path_arg)[0] or info.get(_packages_path) info[_debug] = getattr(arguments, _debug) or info.get( _debug, _debug_default) info[_test_run] = getattr(arguments, _test_run, False) # If any of resume|force given as cmd args, ignore those in the input file resume_arg, force_arg = [ getattr(arguments, arg) for arg in [_resume, _force] ] if any([resume_arg, force_arg]): info[_resume], info[_force] = resume_arg, force_arg if _post in info: post(info) else: run(info)
def run_script(): warn_deprecation() import os import argparse parser = argparse.ArgumentParser(description="Cobaya's run script.") parser.add_argument("input_file", nargs=1, action="store", metavar="input_file.yaml", help="An input file to run.") parser.add_argument( "-" + _modules_path_arg[0], "--" + _modules_path_arg, action="store", nargs=1, metavar="/some/path", default=[None], help="Path where modules were automatically installed.") parser.add_argument("-" + _output_prefix[0], "--" + _output_prefix, action="store", nargs=1, metavar="/some/path", default=[None], help="Path and prefix for the text output.") parser.add_argument("-" + _debug[0], "--" + _debug, action="store_true", help="Produce verbose debug output.") continuation = parser.add_mutually_exclusive_group(required=False) continuation.add_argument( "-" + _resume[0], "--" + _resume, action="store_true", help="Resume an existing chain if it has similar info " "(fails otherwise).") continuation.add_argument("-" + _force[0], "--" + _force, action="store_true", help="Overwrites previous output, if it exists " "(use with care!)") parser.add_argument("--version", action="version", version=__version__) args = parser.parse_args() if any([(os.path.splitext(f)[0] in ("input", "updated")) for f in args.input_file]): raise ValueError("'input' and 'updated' are reserved file names. " "Please, use a different one.") load_input = import_MPI(".input", "load_input") given_input = args.input_file[0] if any(given_input.lower().endswith(ext) for ext in _yaml_extensions): info = load_input(given_input) output_prefix_cmd = getattr(args, _output_prefix)[0] output_prefix_input = info.get(_output_prefix) info[_output_prefix] = output_prefix_cmd or output_prefix_input else: # Passed an existing output_prefix? Try to find the corresponding *.updated.yaml updated_file = ( given_input + (_separator_files if not given_input.endswith(os.sep) else "") + _updated_suffix + _yaml_extensions[0]) try: info = load_input(updated_file) except IOError: raise ValueError( "Not a valid input file, or non-existent sample to resume") # We need to update the output_prefix to resume the sample *where it is* info[_output_prefix] = given_input # If input given this way, we obviously want to resume! info[_resume] = True # solve modules installation path cmd > env > input path_cmd = getattr(args, _modules_path_arg)[0] path_env = os.environ.get(_modules_path_env, None) path_input = info.get(_path_install) info[_path_install] = path_cmd or (path_env or path_input) info[_debug] = getattr(args, _debug) or info.get(_debug, _debug_default) info[_resume] = getattr(args, _resume, _resume_default) info[_force] = getattr(args, _force, False) if _post in info: post(info) else: run(info)