def load_input_file(input_file: Union[str, os.PathLike],
no_mpi: bool = False,
help_commands: Optional[str] = None) -> InputDict:
if no_mpi:
mpi.set_mpi_disabled()
input_file = str(input_file)
stem, suffix = os.path.splitext(input_file)
if os.path.basename(stem) in ("input", "updated"):
raise ValueError("'input' and 'updated' are reserved file names. "
"Please, use a different one.")
if suffix.lower() in Extension.yamls + (Extension.dill, ):
info = load_input_MPI(input_file)
root, suffix = os.path.splitext(stem)
if suffix == ".updated":
# path may have been removed, so put in full path and name
info["output"] = root
else:
# Passed an existing output_prefix?
# First see if there is a binary info pickle
updated_file = get_info_path(*split_prefix(input_file),
ext=Extension.dill)
if not os.path.exists(updated_file):
# Try to find the corresponding *.updated.yaml
updated_file = get_info_path(*split_prefix(input_file))
try:
info = load_input_MPI(updated_file)
except IOError:
err_msg = "Not a valid input file, or non-existent run to resume."
if help_commands:
err_msg += (
" Maybe you mistyped one of the following commands: " +
help_commands)
raise ValueError(err_msg)
# We need to update the output_prefix to resume the run *where it is*
info["output"] = input_file
if "post" not in info:
# If input given this way, we obviously want to resume!
info["resume"] = True
return info
def run(
info_or_yaml_or_file: Union[InputDict, str, os.PathLike],
packages_path: Optional[str] = None,
output: Union[str, LiteralFalse, None] = None,
debug: Union[bool, int, None] = None,
stop_at_error: Optional[bool] = None,
resume: bool = False,
force: bool = False,
no_mpi: bool = False,
test: bool = False,
override: Optional[InputDict] = None,
) -> Union[InfoSamplerTuple, PostTuple]:
"""
Run from an input dictionary, file name or yaml string, with optional arguments
to override settings in the input as needed.
:param info_or_yaml_or_file: input options dictionary, yaml file, or yaml text
:param packages_path: path where external packages were installed
:param output: path name prefix for output files, or False for no file output
:param debug: true for verbose debug output, or a specific logging level
:param stop_at_error: stop if an error is raised
:param resume: continue an existing run
:param force: overwrite existing output if it exists
:param no_mpi: run without MPI
:param test: only test initialization rather than actually running
:param override: option dictionary to merge into the input one, overriding settings
(but with lower precedence than the explicit keyword arguments)
:return: (updated_info, sampler) tuple of options dictionary and Sampler instance,
or (updated_info, results) if using "post" post-processing
"""
# This function reproduces the model-->output-->sampler pipeline one would follow
# when instantiating by hand, but alters the order to performs checks and dump info
# as early as possible, e.g. to check if resuming possible or `force` needed.
if no_mpi or test:
mpi.set_mpi_disabled()
with mpi.ProcessState("run"):
info: InputDict = load_info_overrides(info_or_yaml_or_file, debug,
stop_at_error, packages_path,
override)
if test:
info["test"] = True
# If any of resume|force given as cmd args, ignore those in the input file
if resume or force:
if resume and force:
raise ValueError("'rename' and 'force' are exclusive options")
info["resume"] = bool(resume)
info["force"] = bool(force)
if info.get("post"):
if isinstance(output, str) or output is False:
info["post"]["output"] = output or None
return post(info)
if isinstance(output, str) or output is False:
info["output"] = output or None
logger_setup(info.get("debug"), info.get("debug_file"))
logger_run = get_logger(run.__name__)
# MARKED FOR DEPRECATION IN v3.0
# BEHAVIOUR TO BE REPLACED BY ERROR:
check_deprecated_modules_path(info)
# END OF DEPRECATION BLOCK
# 1. Prepare output driver, if requested by defining an output_prefix
# GetDist needs to know the original sampler, so don't overwrite if minimizer
try:
which_sampler = list(info["sampler"])[0]
except (KeyError, TypeError):
raise LoggedError(
logger_run,
"You need to specify a sampler using the 'sampler' key "
"as e.g. `sampler: {mcmc: None}.`")
infix = "minimize" if which_sampler == "minimize" else None
with get_output(prefix=info.get("output"),
resume=info.get("resume"),
force=info.get("force"),
infix=infix) as out:
# 2. Update the input info with the defaults for each component
updated_info = update_info(info)
if is_debug(logger_run):
# Dump only if not doing output
# (otherwise, the user can check the .updated file)
if not out and mpi.is_main_process():
logger_run.info(
"Input info updated with defaults (dumped to YAML):\n%s",
yaml_dump(sort_cosmetic(updated_info)))
# 3. If output requested, check compatibility if existing one, and dump.
# 3.1 First: model only
out.check_and_dump_info(info,
updated_info,
cache_old=True,
ignore_blocks=["sampler"])
# 3.2 Then sampler -- 1st get the last sampler mentioned in the updated.yaml
# TODO: ideally, using Minimizer would *append* to the sampler block.
# Some code already in place, but not possible at the moment.
try:
last_sampler = list(updated_info["sampler"])[-1]
last_sampler_info = {
last_sampler: updated_info["sampler"][last_sampler]
}
except (KeyError, TypeError):
raise LoggedError(logger_run, "No sampler requested.")
sampler_name, sampler_class = get_sampler_name_and_class(
last_sampler_info)
check_sampler_info((out.reload_updated_info(use_cache=True)
or {}).get("sampler"),
updated_info["sampler"],
is_resuming=out.is_resuming())
# Dump again, now including sampler info
out.check_and_dump_info(info, updated_info, check_compatible=False)
# Check if resumable run
sampler_class.check_force_resume(
out, info=updated_info["sampler"][sampler_name])
# 4. Initialize the posterior and the sampler
with Model(updated_info["params"],
updated_info["likelihood"],
updated_info.get("prior"),
updated_info.get("theory"),
packages_path=info.get("packages_path"),
timing=updated_info.get("timing"),
allow_renames=False,
stop_at_error=info.get("stop_at_error",
False)) as model:
# Re-dump the updated info, now containing parameter routes and version
updated_info = recursive_update(updated_info, model.info())
out.check_and_dump_info(None,
updated_info,
check_compatible=False)
sampler = sampler_class(
updated_info["sampler"][sampler_name],
model,
out,
name=sampler_name,
packages_path=info.get("packages_path"))
# Re-dump updated info, now also containing updates from the sampler
updated_info["sampler"][sampler_name] = \
recursive_update(updated_info["sampler"][sampler_name],
sampler.info())
out.check_and_dump_info(None,
updated_info,
check_compatible=False)
mpi.sync_processes()
if info.get("test", False):
logger_run.info(
"Test initialization successful! "
"You can probably run now without `--%s`.", "test")
return InfoSamplerTuple(updated_info, sampler)
# Run the sampler
sampler.run()
return InfoSamplerTuple(updated_info, sampler)
def run_script():
warn_deprecation()
import os
import argparse
parser = argparse.ArgumentParser(description="Cobaya's run script.")
parser.add_argument("input_file",
nargs=1,
action="store",
metavar="input_file.yaml",
help="An input file to run.")
parser.add_argument("-" + _packages_path_arg[0],
"--" + _packages_path_arg_posix,
action="store",
nargs=1,
metavar="/packages/path",
default=[None],
help="Path where external packages were installed.")
# MARKED FOR DEPRECATION IN v3.0
modules = "modules"
parser.add_argument("-" + modules[0],
"--" + modules,
action="store",
nargs=1,
required=False,
metavar="/packages/path",
default=[None],
help="To be deprecated! "
"Alias for %s, which should be used instead." %
_packages_path_arg_posix)
# END OF DEPRECATION BLOCK -- CONTINUES BELOW!
parser.add_argument("-" + _output_prefix[0],
"--" + _output_prefix,
action="store",
nargs=1,
metavar="/some/path",
default=[None],
help="Path and prefix for the text output.")
parser.add_argument("-" + _debug[0],
"--" + _debug,
action="store_true",
help="Produce verbose debug output.")
continuation = parser.add_mutually_exclusive_group(required=False)
continuation.add_argument(
"-" + _resume[0],
"--" + _resume,
action="store_true",
help="Resume an existing chain if it has similar info "
"(fails otherwise).")
continuation.add_argument("-" + _force[0],
"--" + _force,
action="store_true",
help="Overwrites previous output, if it exists "
"(use with care!)")
parser.add_argument("--%s" % _test_run,
action="store_true",
help="Initialize model and sampler, and exit.")
parser.add_argument("--version", action="version", version=__version__)
parser.add_argument("--no-mpi",
action='store_true',
help="disable MPI when mpi4py installed but MPI does "
"not actually work")
arguments = parser.parse_args()
if arguments.no_mpi or getattr(arguments, _test_run, False):
set_mpi_disabled()
if any((os.path.splitext(f)[0] in ("input", "updated"))
for f in arguments.input_file):
raise ValueError("'input' and 'updated' are reserved file names. "
"Please, use a different one.")
load_input = import_MPI(".input", "load_input")
given_input = arguments.input_file[0]
if any(given_input.lower().endswith(ext) for ext in _yaml_extensions):
info = load_input(given_input)
output_prefix_cmd = getattr(arguments, _output_prefix)[0]
output_prefix_input = info.get(_output_prefix)
info[_output_prefix] = output_prefix_cmd or output_prefix_input
else:
# Passed an existing output_prefix? Try to find the corresponding *.updated.yaml
updated_file = get_info_path(*split_prefix(given_input),
kind="updated")
try:
info = load_input(updated_file)
except IOError:
raise ValueError(
"Not a valid input file, or non-existent run to resume")
# We need to update the output_prefix to resume the run *where it is*
info[_output_prefix] = given_input
# If input given this way, we obviously want to resume!
info[_resume] = True
# solve packages installation path cmd > env > input
# MARKED FOR DEPRECATION IN v3.0
if getattr(arguments, modules) != [None]:
logger_setup()
logger = logging.getLogger(__name__.split(".")[-1])
logger.warning(
"*DEPRECATION*: -m/--modules will be deprecated in favor of "
"-%s/--%s in the next version. Please, use that one instead.",
_packages_path_arg[0], _packages_path_arg_posix)
# BEHAVIOUR TO BE REPLACED BY ERROR:
if getattr(arguments, _packages_path_arg) == [None]:
setattr(arguments, _packages_path_arg, getattr(arguments, modules))
# BEHAVIOUR TO BE REPLACED BY ERROR:
check_deprecated_modules_path(info)
# END OF DEPRECATION BLOCK
info[_packages_path] = \
getattr(arguments, _packages_path_arg)[0] or info.get(_packages_path)
info[_debug] = getattr(arguments, _debug) or info.get(
_debug, _debug_default)
info[_test_run] = getattr(arguments, _test_run, False)
# If any of resume|force given as cmd args, ignore those in the input file
resume_arg, force_arg = [
getattr(arguments, arg) for arg in [_resume, _force]
]
if any([resume_arg, force_arg]):
info[_resume], info[_force] = resume_arg, force_arg
if _post in info:
post(info)
else:
run(info)
def install_script():
set_mpi_disabled(True)
warn_deprecation()
# Parse arguments
import argparse
parser = argparse.ArgumentParser(
description="Cobaya's installation tool for external packages.")
parser.add_argument(
"files_or_components",
action="store",
nargs="+",
metavar="input_file.yaml|component_name",
help="One or more input files or component names "
"(or simply 'cosmo' to install all the requisites for basic"
" cosmological runs)")
parser.add_argument(
"-" + _packages_path_arg[0],
"--" + _packages_path_arg_posix,
action="store",
nargs=1,
required=False,
metavar="/packages/path",
default=[None],
help="Desired path where to install external packages. "
"Optional if one has been set globally or as an env variable"
" (run with '--show_%s' to check)." % _packages_path_arg_posix)
# MARKED FOR DEPRECATION IN v3.0
modules = "modules"
parser.add_argument("-" + modules[0],
"--" + modules,
action="store",
nargs=1,
required=False,
metavar="/packages/path",
default=[None],
help="To be deprecated! "
"Alias for %s, which should be used instead." %
_packages_path_arg_posix)
# END OF DEPRECATION BLOCK -- CONTINUES BELOW!
output_show_packages_path = resolve_packages_path()
if output_show_packages_path and os.environ.get(_packages_path_env):
output_show_packages_path += " (from env variable %r)" % _packages_path_env
elif output_show_packages_path:
output_show_packages_path += " (from config file)"
else:
output_show_packages_path = "(Not currently set.)"
parser.add_argument(
"--show-" + _packages_path_arg_posix,
action="version",
version=output_show_packages_path,
help="Prints default external packages installation folder "
"and exits.")
parser.add_argument(
"-" + _force[0],
"--" + _force,
action="store_true",
default=False,
help="Force re-installation of apparently installed packages.")
parser.add_argument(
"--skip",
action="store",
nargs="*",
metavar="keyword",
help="Keywords of components that will be skipped during "
"installation.")
parser.add_argument(
"--no-progress-bars",
action="store_true",
default=False,
help="No progress bars shown. Shorter logs (used in Travis).")
parser.add_argument("--just-check",
action="store_true",
default=False,
help="Just check whether components are installed.")
parser.add_argument(
"--no-set-global",
action="store_true",
default=False,
help="Do not store the installation path for later runs.")
group_just = parser.add_mutually_exclusive_group(required=False)
group_just.add_argument("-C",
"--just-code",
action="store_false",
default=True,
help="Install code of the components.",
dest=_data)
group_just.add_argument("-D",
"--just-data",
action="store_false",
default=True,
help="Install data of the components.",
dest=_code)
arguments = parser.parse_args()
# Configure the logger ASAP
logger_setup()
logger = logging.getLogger(__name__.split(".")[-1])
# Gather requests
infos = []
for f in arguments.files_or_components:
if f.lower() == "cosmo":
logger.info("Installing basic cosmological packages.")
from cobaya.cosmo_input import install_basic
infos += [install_basic]
elif f.lower() == "cosmo-tests":
logger.info("Installing *tested* cosmological packages.")
from cobaya.cosmo_input import install_tests
infos += [install_tests]
elif os.path.splitext(f)[1].lower() in _yaml_extensions:
from cobaya.input import load_input
infos += [load_input(f)]
else:
try:
kind = get_kind(f)
infos += [{kind: {f: None}}]
except Exception:
logger.warning("Could not identify component %r. Skipping.", f)
if not infos:
logger.info("Nothing to install.")
return
# MARKED FOR DEPRECATION IN v3.0
if getattr(arguments, modules) != [None]:
logger.warning(
"*DEPRECATION*: -m/--modules will be deprecated in favor of "
"-%s/--%s in the next version. Please, use that one instead.",
_packages_path_arg[0], _packages_path_arg_posix)
# BEHAVIOUR TO BE REPLACED BY ERROR:
if getattr(arguments, _packages_path_arg) == [None]:
setattr(arguments, _packages_path_arg, getattr(arguments, modules))
# END OF DEPRECATION BLOCK
# Launch installer
install(*infos,
path=getattr(arguments, _packages_path_arg)[0],
**{
arg: getattr(arguments, arg)
for arg in [
"force", _code, _data, "no_progress_bars", "just_check",
"no_set_global", "skip"
]
})
def install_script(args=None):
set_mpi_disabled()
warn_deprecation()
# Parse arguments
import argparse
parser = argparse.ArgumentParser(
prog="cobaya install",
description="Cobaya's installation tool for external packages.")
parser.add_argument(
"files_or_components",
action="store",
nargs="+",
metavar="input_file.yaml|component_name",
help="One or more input files or component names "
"(or simply 'cosmo' to install all the requisites for basic"
" cosmological runs)")
parser.add_argument(
"-" + packages_path_arg[0],
"--" + packages_path_arg_posix,
action="store",
required=False,
metavar="/packages/path",
default=None,
help="Desired path where to install external packages. "
"Optional if one has been set globally or as an env variable"
" (run with '--show_%s' to check)." % packages_path_arg_posix)
# MARKED FOR DEPRECATION IN v3.0
modules = "modules"
parser.add_argument("-" + modules[0],
"--" + modules,
action="store",
required=False,
metavar="/packages/path",
default=None,
help="Deprecated! Use %s instead." %
packages_path_arg_posix)
# END OF DEPRECATION BLOCK -- CONTINUES BELOW!
output_show_packages_path = resolve_packages_path()
if output_show_packages_path and os.environ.get(packages_path_env):
output_show_packages_path += " (from env variable %r)" % packages_path_env
elif output_show_packages_path:
output_show_packages_path += " (from config file)"
else:
output_show_packages_path = "(Not currently set.)"
parser.add_argument(
"--show-" + packages_path_arg_posix,
action="version",
version=output_show_packages_path,
help="Prints default external packages installation folder "
"and exits.")
parser.add_argument(
"-" + "f",
"--" + "force",
action="store_true",
default=False,
help="Force re-installation of apparently installed packages.")
parser.add_argument(
"--skip",
action="store",
nargs="*",
metavar="keyword",
help="Keywords of components that will be skipped during "
"installation.")
parser.add_argument(
"--no-progress-bars",
action="store_true",
default=False,
help="No progress bars shown. Shorter logs (used in Travis).")
parser.add_argument("--%s" % "test",
action="store_true",
default=False,
help="Just check whether components are installed.")
# MARKED FOR DEPRECATION IN v3.0
parser.add_argument("--just-check",
action="store_true",
default=False,
help="Just check whether components are installed.")
# END OF DEPRECATION BLOCK -- CONTINUES BELOW!
parser.add_argument(
"--no-set-global",
action="store_true",
default=False,
help="Do not store the installation path for later runs.")
parser.add_argument(
"--skip-global",
action="store_true",
default=False,
help="Skip installation of already-available Python modules.")
parser.add_argument("-" + "d",
"--" + "debug",
action="store_true",
help="Produce verbose debug output.")
group_just = parser.add_mutually_exclusive_group(required=False)
group_just.add_argument("-C",
"--just-code",
action="store_false",
default=True,
help="Install code of the components.",
dest=data_path)
group_just.add_argument("-D",
"--just-data",
action="store_false",
default=True,
help="Install data of the components.",
dest=code_path)
arguments = parser.parse_args(args)
# Configure the logger ASAP
logger_setup()
logger = get_logger("install")
# Gather requests
infos: List[InputDict] = []
for f in arguments.files_or_components:
if f.lower() == "cosmo":
logger.info("Installing basic cosmological packages.")
from cobaya.cosmo_input import install_basic
infos += [install_basic]
elif f.lower() == "cosmo-tests":
logger.info("Installing *tested* cosmological packages.")
from cobaya.cosmo_input import install_tests
infos += [install_tests]
elif os.path.splitext(f)[1].lower() in Extension.yamls:
from cobaya.input import load_input
infos += [load_input(f)]
else:
try:
kind = get_kind(f)
infos += [{kind: {f: None}}]
except Exception:
logger.warning("Could not identify component %r. Skipping.", f)
if not infos:
logger.info("Nothing to install.")
return
# List of deprecation warnings, to be printed *after* installation
deprecation_warnings = []
# MARKED FOR DEPRECATION IN v3.0
if getattr(arguments, modules) is not None:
raise LoggedError(
logger, "-m/--modules has been deprecated in favor of "
"-%s/--%s", packages_path_arg[0], packages_path_arg_posix)
# END OF DEPRECATION BLOCK
# MARKED FOR DEPRECATION IN v3.0
if arguments.just_check is True:
raise LoggedError(logger,
"--just-check has been deprecated in favor of --%s",
"test")
# END OF DEPRECATION BLOCK
# Launch installer
install(*infos,
path=getattr(arguments, packages_path_arg),
**{
arg: getattr(arguments, arg)
for arg in [
"force", code_path, data_path, "no_progress_bars", "test",
"no_set_global", "skip", "skip_global", "debug"
]
})
# MARKED FOR DEPRECATION IN v3.0
for warning_msg in deprecation_warnings:
logger.warning(warning_msg)
def install_script(args=None):
"""Command line script for the installer."""
set_mpi_disabled()
warn_deprecation()
# Parse arguments
import argparse
parser = argparse.ArgumentParser(
prog="cobaya install",
description="Cobaya's installation tool for external packages.")
parser.add_argument(
"files_or_components",
action="store",
nargs="+",
metavar="input_file.yaml|component_name",
help="One or more input files or component names "
"(or simply 'cosmo' to install all the requisites for basic"
" cosmological runs)")
parser.add_argument(
"-" + packages_path_arg[0],
"--" + packages_path_arg_posix,
action="store",
required=False,
metavar="/packages/path",
default=None,
help="Desired path where to install external packages. "
"Optional if one has been set globally or as an env variable"
" (run with '--show_%s' to check)." % packages_path_arg_posix)
output_show_packages_path = resolve_packages_path()
if output_show_packages_path and os.environ.get(packages_path_env):
output_show_packages_path += " (from env variable %r)" % packages_path_env
elif output_show_packages_path:
output_show_packages_path += " (from config file)"
else:
output_show_packages_path = "(Not currently set.)"
parser.add_argument(
"--show-" + packages_path_arg_posix,
action="version",
version=output_show_packages_path,
help="Prints default external packages installation folder "
"and exits.")
parser.add_argument(
"-" + "f",
"--" + "force",
action="store_true",
default=False,
help="Force re-installation of apparently installed packages.")
parser.add_argument("--%s" % "test",
action="store_true",
default=False,
help="Just check whether components are installed.")
parser.add_argument("--upgrade",
action="store_true",
default=False,
help="Force upgrade of obsolete components.")
parser.add_argument("--skip",
action="store",
nargs="*",
metavar="keyword",
help=("Keywords of components that will be "
"skipped during installation."))
parser.add_argument(
"--skip-global",
action="store_true",
default=False,
help="Skip installation of already-available Python modules.")
parser.add_argument("-" + "d",
"--" + "debug",
action="store_true",
help="Produce verbose debug output.")
group_just = parser.add_mutually_exclusive_group(required=False)
group_just.add_argument("-C",
"--just-code",
action="store_false",
default=True,
help="Install code of the components.",
dest=data_path)
group_just.add_argument("-D",
"--just-data",
action="store_false",
default=True,
help="Install data of the components.",
dest=code_path)
parser.add_argument(
"--no-progress-bars",
action="store_true",
default=False,
help=("No progress bars shown; use when output is saved into a "
"text file (e.g. when running on a cluster)."))
parser.add_argument(
"--no-set-global",
action="store_true",
default=False,
help="Do not store the installation path for later runs.")
arguments = parser.parse_args(args)
# Configure the logger ASAP
logger_setup(arguments.debug)
logger = get_logger("install")
# Gather requests
infos: List[Union[InputDict, str]] = []
for f in arguments.files_or_components:
if f.lower() == "cosmo":
logger.info("Installing basic cosmological packages.")
from cobaya.cosmo_input import install_basic
infos += [install_basic]
elif f.lower() == "cosmo-tests":
logger.info("Installing *tested* cosmological packages.")
from cobaya.cosmo_input import install_tests
infos += [install_tests]
elif os.path.splitext(f)[1].lower() in Extension.yamls:
from cobaya.input import load_input
infos += [load_input(f)]
else: # a single component name, no kind specified
infos += [f]
# Launch installer
install(*infos,
path=getattr(arguments, packages_path_arg),
logger=logger,
**{
arg: getattr(arguments, arg)
for arg in [
"force", code_path, data_path, "no_progress_bars", "test",
"no_set_global", "skip", "skip_global", "debug", "upgrade"
]
})
