def load_input_file(input_file: Union[str, os.PathLike], no_mpi: bool = False, help_commands: Optional[str] = None) -> InputDict: if no_mpi: mpi.set_mpi_disabled() input_file = str(input_file) stem, suffix = os.path.splitext(input_file) if os.path.basename(stem) in ("input", "updated"): raise ValueError("'input' and 'updated' are reserved file names. " "Please, use a different one.") if suffix.lower() in Extension.yamls + (Extension.dill, ): info = load_input_MPI(input_file) root, suffix = os.path.splitext(stem) if suffix == ".updated": # path may have been removed, so put in full path and name info["output"] = root else: # Passed an existing output_prefix? # First see if there is a binary info pickle updated_file = get_info_path(*split_prefix(input_file), ext=Extension.dill) if not os.path.exists(updated_file): # Try to find the corresponding *.updated.yaml updated_file = get_info_path(*split_prefix(input_file)) try: info = load_input_MPI(updated_file) except IOError: err_msg = "Not a valid input file, or non-existent run to resume." if help_commands: err_msg += ( " Maybe you mistyped one of the following commands: " + help_commands) raise ValueError(err_msg) # We need to update the output_prefix to resume the run *where it is* info["output"] = input_file if "post" not in info: # If input given this way, we obviously want to resume! info["resume"] = True return info
def run( info_or_yaml_or_file: Union[InputDict, str, os.PathLike], packages_path: Optional[str] = None, output: Union[str, LiteralFalse, None] = None, debug: Union[bool, int, None] = None, stop_at_error: Optional[bool] = None, resume: bool = False, force: bool = False, no_mpi: bool = False, test: bool = False, override: Optional[InputDict] = None, ) -> Union[InfoSamplerTuple, PostTuple]: """ Run from an input dictionary, file name or yaml string, with optional arguments to override settings in the input as needed. :param info_or_yaml_or_file: input options dictionary, yaml file, or yaml text :param packages_path: path where external packages were installed :param output: path name prefix for output files, or False for no file output :param debug: true for verbose debug output, or a specific logging level :param stop_at_error: stop if an error is raised :param resume: continue an existing run :param force: overwrite existing output if it exists :param no_mpi: run without MPI :param test: only test initialization rather than actually running :param override: option dictionary to merge into the input one, overriding settings (but with lower precedence than the explicit keyword arguments) :return: (updated_info, sampler) tuple of options dictionary and Sampler instance, or (updated_info, results) if using "post" post-processing """ # This function reproduces the model-->output-->sampler pipeline one would follow # when instantiating by hand, but alters the order to performs checks and dump info # as early as possible, e.g. to check if resuming possible or `force` needed. if no_mpi or test: mpi.set_mpi_disabled() with mpi.ProcessState("run"): info: InputDict = load_info_overrides(info_or_yaml_or_file, debug, stop_at_error, packages_path, override) if test: info["test"] = True # If any of resume|force given as cmd args, ignore those in the input file if resume or force: if resume and force: raise ValueError("'rename' and 'force' are exclusive options") info["resume"] = bool(resume) info["force"] = bool(force) if info.get("post"): if isinstance(output, str) or output is False: info["post"]["output"] = output or None return post(info) if isinstance(output, str) or output is False: info["output"] = output or None logger_setup(info.get("debug"), info.get("debug_file")) logger_run = get_logger(run.__name__) # MARKED FOR DEPRECATION IN v3.0 # BEHAVIOUR TO BE REPLACED BY ERROR: check_deprecated_modules_path(info) # END OF DEPRECATION BLOCK # 1. Prepare output driver, if requested by defining an output_prefix # GetDist needs to know the original sampler, so don't overwrite if minimizer try: which_sampler = list(info["sampler"])[0] except (KeyError, TypeError): raise LoggedError( logger_run, "You need to specify a sampler using the 'sampler' key " "as e.g. `sampler: {mcmc: None}.`") infix = "minimize" if which_sampler == "minimize" else None with get_output(prefix=info.get("output"), resume=info.get("resume"), force=info.get("force"), infix=infix) as out: # 2. Update the input info with the defaults for each component updated_info = update_info(info) if is_debug(logger_run): # Dump only if not doing output # (otherwise, the user can check the .updated file) if not out and mpi.is_main_process(): logger_run.info( "Input info updated with defaults (dumped to YAML):\n%s", yaml_dump(sort_cosmetic(updated_info))) # 3. If output requested, check compatibility if existing one, and dump. # 3.1 First: model only out.check_and_dump_info(info, updated_info, cache_old=True, ignore_blocks=["sampler"]) # 3.2 Then sampler -- 1st get the last sampler mentioned in the updated.yaml # TODO: ideally, using Minimizer would *append* to the sampler block. # Some code already in place, but not possible at the moment. try: last_sampler = list(updated_info["sampler"])[-1] last_sampler_info = { last_sampler: updated_info["sampler"][last_sampler] } except (KeyError, TypeError): raise LoggedError(logger_run, "No sampler requested.") sampler_name, sampler_class = get_sampler_name_and_class( last_sampler_info) check_sampler_info((out.reload_updated_info(use_cache=True) or {}).get("sampler"), updated_info["sampler"], is_resuming=out.is_resuming()) # Dump again, now including sampler info out.check_and_dump_info(info, updated_info, check_compatible=False) # Check if resumable run sampler_class.check_force_resume( out, info=updated_info["sampler"][sampler_name]) # 4. Initialize the posterior and the sampler with Model(updated_info["params"], updated_info["likelihood"], updated_info.get("prior"), updated_info.get("theory"), packages_path=info.get("packages_path"), timing=updated_info.get("timing"), allow_renames=False, stop_at_error=info.get("stop_at_error", False)) as model: # Re-dump the updated info, now containing parameter routes and version updated_info = recursive_update(updated_info, model.info()) out.check_and_dump_info(None, updated_info, check_compatible=False) sampler = sampler_class( updated_info["sampler"][sampler_name], model, out, name=sampler_name, packages_path=info.get("packages_path")) # Re-dump updated info, now also containing updates from the sampler updated_info["sampler"][sampler_name] = \ recursive_update(updated_info["sampler"][sampler_name], sampler.info()) out.check_and_dump_info(None, updated_info, check_compatible=False) mpi.sync_processes() if info.get("test", False): logger_run.info( "Test initialization successful! " "You can probably run now without `--%s`.", "test") return InfoSamplerTuple(updated_info, sampler) # Run the sampler sampler.run() return InfoSamplerTuple(updated_info, sampler)
def run_script(): warn_deprecation() import os import argparse parser = argparse.ArgumentParser(description="Cobaya's run script.") parser.add_argument("input_file", nargs=1, action="store", metavar="input_file.yaml", help="An input file to run.") parser.add_argument("-" + _packages_path_arg[0], "--" + _packages_path_arg_posix, action="store", nargs=1, metavar="/packages/path", default=[None], help="Path where external packages were installed.") # MARKED FOR DEPRECATION IN v3.0 modules = "modules" parser.add_argument("-" + modules[0], "--" + modules, action="store", nargs=1, required=False, metavar="/packages/path", default=[None], help="To be deprecated! " "Alias for %s, which should be used instead." % _packages_path_arg_posix) # END OF DEPRECATION BLOCK -- CONTINUES BELOW! parser.add_argument("-" + _output_prefix[0], "--" + _output_prefix, action="store", nargs=1, metavar="/some/path", default=[None], help="Path and prefix for the text output.") parser.add_argument("-" + _debug[0], "--" + _debug, action="store_true", help="Produce verbose debug output.") continuation = parser.add_mutually_exclusive_group(required=False) continuation.add_argument( "-" + _resume[0], "--" + _resume, action="store_true", help="Resume an existing chain if it has similar info " "(fails otherwise).") continuation.add_argument("-" + _force[0], "--" + _force, action="store_true", help="Overwrites previous output, if it exists " "(use with care!)") parser.add_argument("--%s" % _test_run, action="store_true", help="Initialize model and sampler, and exit.") parser.add_argument("--version", action="version", version=__version__) parser.add_argument("--no-mpi", action='store_true', help="disable MPI when mpi4py installed but MPI does " "not actually work") arguments = parser.parse_args() if arguments.no_mpi or getattr(arguments, _test_run, False): set_mpi_disabled() if any((os.path.splitext(f)[0] in ("input", "updated")) for f in arguments.input_file): raise ValueError("'input' and 'updated' are reserved file names. " "Please, use a different one.") load_input = import_MPI(".input", "load_input") given_input = arguments.input_file[0] if any(given_input.lower().endswith(ext) for ext in _yaml_extensions): info = load_input(given_input) output_prefix_cmd = getattr(arguments, _output_prefix)[0] output_prefix_input = info.get(_output_prefix) info[_output_prefix] = output_prefix_cmd or output_prefix_input else: # Passed an existing output_prefix? Try to find the corresponding *.updated.yaml updated_file = get_info_path(*split_prefix(given_input), kind="updated") try: info = load_input(updated_file) except IOError: raise ValueError( "Not a valid input file, or non-existent run to resume") # We need to update the output_prefix to resume the run *where it is* info[_output_prefix] = given_input # If input given this way, we obviously want to resume! info[_resume] = True # solve packages installation path cmd > env > input # MARKED FOR DEPRECATION IN v3.0 if getattr(arguments, modules) != [None]: logger_setup() logger = logging.getLogger(__name__.split(".")[-1]) logger.warning( "*DEPRECATION*: -m/--modules will be deprecated in favor of " "-%s/--%s in the next version. Please, use that one instead.", _packages_path_arg[0], _packages_path_arg_posix) # BEHAVIOUR TO BE REPLACED BY ERROR: if getattr(arguments, _packages_path_arg) == [None]: setattr(arguments, _packages_path_arg, getattr(arguments, modules)) # BEHAVIOUR TO BE REPLACED BY ERROR: check_deprecated_modules_path(info) # END OF DEPRECATION BLOCK info[_packages_path] = \ getattr(arguments, _packages_path_arg)[0] or info.get(_packages_path) info[_debug] = getattr(arguments, _debug) or info.get( _debug, _debug_default) info[_test_run] = getattr(arguments, _test_run, False) # If any of resume|force given as cmd args, ignore those in the input file resume_arg, force_arg = [ getattr(arguments, arg) for arg in [_resume, _force] ] if any([resume_arg, force_arg]): info[_resume], info[_force] = resume_arg, force_arg if _post in info: post(info) else: run(info)
def install_script(): set_mpi_disabled(True) warn_deprecation() # Parse arguments import argparse parser = argparse.ArgumentParser( description="Cobaya's installation tool for external packages.") parser.add_argument( "files_or_components", action="store", nargs="+", metavar="input_file.yaml|component_name", help="One or more input files or component names " "(or simply 'cosmo' to install all the requisites for basic" " cosmological runs)") parser.add_argument( "-" + _packages_path_arg[0], "--" + _packages_path_arg_posix, action="store", nargs=1, required=False, metavar="/packages/path", default=[None], help="Desired path where to install external packages. " "Optional if one has been set globally or as an env variable" " (run with '--show_%s' to check)." % _packages_path_arg_posix) # MARKED FOR DEPRECATION IN v3.0 modules = "modules" parser.add_argument("-" + modules[0], "--" + modules, action="store", nargs=1, required=False, metavar="/packages/path", default=[None], help="To be deprecated! " "Alias for %s, which should be used instead." % _packages_path_arg_posix) # END OF DEPRECATION BLOCK -- CONTINUES BELOW! output_show_packages_path = resolve_packages_path() if output_show_packages_path and os.environ.get(_packages_path_env): output_show_packages_path += " (from env variable %r)" % _packages_path_env elif output_show_packages_path: output_show_packages_path += " (from config file)" else: output_show_packages_path = "(Not currently set.)" parser.add_argument( "--show-" + _packages_path_arg_posix, action="version", version=output_show_packages_path, help="Prints default external packages installation folder " "and exits.") parser.add_argument( "-" + _force[0], "--" + _force, action="store_true", default=False, help="Force re-installation of apparently installed packages.") parser.add_argument( "--skip", action="store", nargs="*", metavar="keyword", help="Keywords of components that will be skipped during " "installation.") parser.add_argument( "--no-progress-bars", action="store_true", default=False, help="No progress bars shown. Shorter logs (used in Travis).") parser.add_argument("--just-check", action="store_true", default=False, help="Just check whether components are installed.") parser.add_argument( "--no-set-global", action="store_true", default=False, help="Do not store the installation path for later runs.") group_just = parser.add_mutually_exclusive_group(required=False) group_just.add_argument("-C", "--just-code", action="store_false", default=True, help="Install code of the components.", dest=_data) group_just.add_argument("-D", "--just-data", action="store_false", default=True, help="Install data of the components.", dest=_code) arguments = parser.parse_args() # Configure the logger ASAP logger_setup() logger = logging.getLogger(__name__.split(".")[-1]) # Gather requests infos = [] for f in arguments.files_or_components: if f.lower() == "cosmo": logger.info("Installing basic cosmological packages.") from cobaya.cosmo_input import install_basic infos += [install_basic] elif f.lower() == "cosmo-tests": logger.info("Installing *tested* cosmological packages.") from cobaya.cosmo_input import install_tests infos += [install_tests] elif os.path.splitext(f)[1].lower() in _yaml_extensions: from cobaya.input import load_input infos += [load_input(f)] else: try: kind = get_kind(f) infos += [{kind: {f: None}}] except Exception: logger.warning("Could not identify component %r. Skipping.", f) if not infos: logger.info("Nothing to install.") return # MARKED FOR DEPRECATION IN v3.0 if getattr(arguments, modules) != [None]: logger.warning( "*DEPRECATION*: -m/--modules will be deprecated in favor of " "-%s/--%s in the next version. Please, use that one instead.", _packages_path_arg[0], _packages_path_arg_posix) # BEHAVIOUR TO BE REPLACED BY ERROR: if getattr(arguments, _packages_path_arg) == [None]: setattr(arguments, _packages_path_arg, getattr(arguments, modules)) # END OF DEPRECATION BLOCK # Launch installer install(*infos, path=getattr(arguments, _packages_path_arg)[0], **{ arg: getattr(arguments, arg) for arg in [ "force", _code, _data, "no_progress_bars", "just_check", "no_set_global", "skip" ] })
def install_script(args=None): set_mpi_disabled() warn_deprecation() # Parse arguments import argparse parser = argparse.ArgumentParser( prog="cobaya install", description="Cobaya's installation tool for external packages.") parser.add_argument( "files_or_components", action="store", nargs="+", metavar="input_file.yaml|component_name", help="One or more input files or component names " "(or simply 'cosmo' to install all the requisites for basic" " cosmological runs)") parser.add_argument( "-" + packages_path_arg[0], "--" + packages_path_arg_posix, action="store", required=False, metavar="/packages/path", default=None, help="Desired path where to install external packages. " "Optional if one has been set globally or as an env variable" " (run with '--show_%s' to check)." % packages_path_arg_posix) # MARKED FOR DEPRECATION IN v3.0 modules = "modules" parser.add_argument("-" + modules[0], "--" + modules, action="store", required=False, metavar="/packages/path", default=None, help="Deprecated! Use %s instead." % packages_path_arg_posix) # END OF DEPRECATION BLOCK -- CONTINUES BELOW! output_show_packages_path = resolve_packages_path() if output_show_packages_path and os.environ.get(packages_path_env): output_show_packages_path += " (from env variable %r)" % packages_path_env elif output_show_packages_path: output_show_packages_path += " (from config file)" else: output_show_packages_path = "(Not currently set.)" parser.add_argument( "--show-" + packages_path_arg_posix, action="version", version=output_show_packages_path, help="Prints default external packages installation folder " "and exits.") parser.add_argument( "-" + "f", "--" + "force", action="store_true", default=False, help="Force re-installation of apparently installed packages.") parser.add_argument( "--skip", action="store", nargs="*", metavar="keyword", help="Keywords of components that will be skipped during " "installation.") parser.add_argument( "--no-progress-bars", action="store_true", default=False, help="No progress bars shown. Shorter logs (used in Travis).") parser.add_argument("--%s" % "test", action="store_true", default=False, help="Just check whether components are installed.") # MARKED FOR DEPRECATION IN v3.0 parser.add_argument("--just-check", action="store_true", default=False, help="Just check whether components are installed.") # END OF DEPRECATION BLOCK -- CONTINUES BELOW! parser.add_argument( "--no-set-global", action="store_true", default=False, help="Do not store the installation path for later runs.") parser.add_argument( "--skip-global", action="store_true", default=False, help="Skip installation of already-available Python modules.") parser.add_argument("-" + "d", "--" + "debug", action="store_true", help="Produce verbose debug output.") group_just = parser.add_mutually_exclusive_group(required=False) group_just.add_argument("-C", "--just-code", action="store_false", default=True, help="Install code of the components.", dest=data_path) group_just.add_argument("-D", "--just-data", action="store_false", default=True, help="Install data of the components.", dest=code_path) arguments = parser.parse_args(args) # Configure the logger ASAP logger_setup() logger = get_logger("install") # Gather requests infos: List[InputDict] = [] for f in arguments.files_or_components: if f.lower() == "cosmo": logger.info("Installing basic cosmological packages.") from cobaya.cosmo_input import install_basic infos += [install_basic] elif f.lower() == "cosmo-tests": logger.info("Installing *tested* cosmological packages.") from cobaya.cosmo_input import install_tests infos += [install_tests] elif os.path.splitext(f)[1].lower() in Extension.yamls: from cobaya.input import load_input infos += [load_input(f)] else: try: kind = get_kind(f) infos += [{kind: {f: None}}] except Exception: logger.warning("Could not identify component %r. Skipping.", f) if not infos: logger.info("Nothing to install.") return # List of deprecation warnings, to be printed *after* installation deprecation_warnings = [] # MARKED FOR DEPRECATION IN v3.0 if getattr(arguments, modules) is not None: raise LoggedError( logger, "-m/--modules has been deprecated in favor of " "-%s/--%s", packages_path_arg[0], packages_path_arg_posix) # END OF DEPRECATION BLOCK # MARKED FOR DEPRECATION IN v3.0 if arguments.just_check is True: raise LoggedError(logger, "--just-check has been deprecated in favor of --%s", "test") # END OF DEPRECATION BLOCK # Launch installer install(*infos, path=getattr(arguments, packages_path_arg), **{ arg: getattr(arguments, arg) for arg in [ "force", code_path, data_path, "no_progress_bars", "test", "no_set_global", "skip", "skip_global", "debug" ] }) # MARKED FOR DEPRECATION IN v3.0 for warning_msg in deprecation_warnings: logger.warning(warning_msg)
def install_script(args=None): """Command line script for the installer.""" set_mpi_disabled() warn_deprecation() # Parse arguments import argparse parser = argparse.ArgumentParser( prog="cobaya install", description="Cobaya's installation tool for external packages.") parser.add_argument( "files_or_components", action="store", nargs="+", metavar="input_file.yaml|component_name", help="One or more input files or component names " "(or simply 'cosmo' to install all the requisites for basic" " cosmological runs)") parser.add_argument( "-" + packages_path_arg[0], "--" + packages_path_arg_posix, action="store", required=False, metavar="/packages/path", default=None, help="Desired path where to install external packages. " "Optional if one has been set globally or as an env variable" " (run with '--show_%s' to check)." % packages_path_arg_posix) output_show_packages_path = resolve_packages_path() if output_show_packages_path and os.environ.get(packages_path_env): output_show_packages_path += " (from env variable %r)" % packages_path_env elif output_show_packages_path: output_show_packages_path += " (from config file)" else: output_show_packages_path = "(Not currently set.)" parser.add_argument( "--show-" + packages_path_arg_posix, action="version", version=output_show_packages_path, help="Prints default external packages installation folder " "and exits.") parser.add_argument( "-" + "f", "--" + "force", action="store_true", default=False, help="Force re-installation of apparently installed packages.") parser.add_argument("--%s" % "test", action="store_true", default=False, help="Just check whether components are installed.") parser.add_argument("--upgrade", action="store_true", default=False, help="Force upgrade of obsolete components.") parser.add_argument("--skip", action="store", nargs="*", metavar="keyword", help=("Keywords of components that will be " "skipped during installation.")) parser.add_argument( "--skip-global", action="store_true", default=False, help="Skip installation of already-available Python modules.") parser.add_argument("-" + "d", "--" + "debug", action="store_true", help="Produce verbose debug output.") group_just = parser.add_mutually_exclusive_group(required=False) group_just.add_argument("-C", "--just-code", action="store_false", default=True, help="Install code of the components.", dest=data_path) group_just.add_argument("-D", "--just-data", action="store_false", default=True, help="Install data of the components.", dest=code_path) parser.add_argument( "--no-progress-bars", action="store_true", default=False, help=("No progress bars shown; use when output is saved into a " "text file (e.g. when running on a cluster).")) parser.add_argument( "--no-set-global", action="store_true", default=False, help="Do not store the installation path for later runs.") arguments = parser.parse_args(args) # Configure the logger ASAP logger_setup(arguments.debug) logger = get_logger("install") # Gather requests infos: List[Union[InputDict, str]] = [] for f in arguments.files_or_components: if f.lower() == "cosmo": logger.info("Installing basic cosmological packages.") from cobaya.cosmo_input import install_basic infos += [install_basic] elif f.lower() == "cosmo-tests": logger.info("Installing *tested* cosmological packages.") from cobaya.cosmo_input import install_tests infos += [install_tests] elif os.path.splitext(f)[1].lower() in Extension.yamls: from cobaya.input import load_input infos += [load_input(f)] else: # a single component name, no kind specified infos += [f] # Launch installer install(*infos, path=getattr(arguments, packages_path_arg), logger=logger, **{ arg: getattr(arguments, arg) for arg in [ "force", code_path, data_path, "no_progress_bars", "test", "no_set_global", "skip", "skip_global", "debug", "upgrade" ] })