def __call__(self, *args, warn=True, **kwargs):
if warn:
get_logger(self.__class__.__name__).warning(
"Do not call the instance directly. Use instead methods P(z, k) or "
"logP(z, k) to get the (log)power spectrum. (If you know what you are "
"doing, pass warn=False)")
return super().__call__(*args, **kwargs)
def check_ranges(self, z, k):
"""Checks that we are not trying to extrapolate beyond the interpolator limits."""
z = np.atleast_1d(z).flatten()
min_z, max_z = min(z), max(z)
if min_z < self.zmin and not np.allclose(min_z, self.zmin):
raise LoggedError(
get_logger(self.__class__.__name__),
f"Not possible to extrapolate to z={min(z)} "
f"(minimum z computed is {self.zmin}).")
if max_z > self.zmax and not np.allclose(max_z, self.zmax):
raise LoggedError(
get_logger(self.__class__.__name__),
f"Not possible to extrapolate to z={max(z)} "
f"(maximum z computed is {self.zmax}).")
k = np.atleast_1d(k).flatten()
min_k, max_k = min(k), max(k)
if min_k < self.kmin and not np.allclose(min_k, self.kmin):
raise LoggedError(
get_logger(self.__class__.__name__),
f"Not possible to extrapolate to k={min(k)} 1/Mpc "
f"(minimum k possible is {self.kmin} 1/Mpc).")
if max_k > self.kmax and not np.allclose(max_k, self.kmax):
raise LoggedError(
get_logger(self.__class__.__name__),
f"Not possible to extrapolate to k={max(k)} 1/Mpc "
f"(maximum k possible is {self.kmax} 1/Mpc).")
def install(cls, path=None, data=True, no_progress_bars=False, **_kwargs):
if not data:
return True
log = get_logger(cls.get_qualified_class_name())
opts = cls.get_install_options()
if not opts:
log.info("No install options. Nothing to do.")
return True
repo = opts.get("github_repository", None)
if repo:
from cobaya.install import download_github_release
log.info("Downloading %s data..." % repo)
return download_github_release(
os.path.join(path, "data"), repo, opts.get("github_release", "master"),
no_progress_bars=no_progress_bars, logger=log)
else:
full_path = cls.get_path(path)
if not os.path.exists(full_path):
os.makedirs(full_path)
if not data:
return True
url = opts["download_url"]
log.info("Downloading likelihood data file: %s...", url)
from cobaya.install import download_file
if not download_file(url, full_path, decompress=True, logger=log,
no_progress_bars=no_progress_bars):
return False
log.info("Likelihood data downloaded and uncompressed correctly.")
return True
def install(cls, path=None, force=False, code=False, data=False,
no_progress_bars=False):
if not code:
return True
log = get_logger(__name__)
log.info("Downloading PolyChord...")
success = download_github_release(os.path.join(path, "code"), cls._pc_repo_name,
cls._pc_repo_version,
no_progress_bars=no_progress_bars,
logger=log)
if not success:
log.error("Could not download PolyChord.")
return False
log.info("Compiling (Py)PolyChord...")
from subprocess import Popen, PIPE
# Needs to re-define os' PWD,
# because MakeFile calls it and is not affected by the cwd of Popen
cwd = os.path.join(path, "code",
cls._pc_repo_name[cls._pc_repo_name.find("/") + 1:])
my_env = os.environ.copy()
my_env.update({"PWD": cwd})
if "CC" not in my_env:
my_env["CC"] = "mpicc"
if "CXX" not in my_env:
my_env["CXX"] = "mpicxx"
process_make = Popen([sys.executable, "setup.py", "build"],
cwd=cwd, env=my_env, stdout=PIPE, stderr=PIPE)
out, err = process_make.communicate()
if process_make.returncode:
log.info(out.decode("utf-8"))
log.info(err.decode("utf-8"))
log.error("Python build failed!")
return False
return True
def set_lock(self, log, filename, force=False):
if self.has_lock():
return
self.lock_file = filename + '.locked'
self.lock_error_file = filename + '.lock_err'
try:
os.remove(self.lock_error_file)
except OSError:
pass
self.log = log or get_logger("file_lock")
try:
h: Any = None
if use_portalocker():
import portalocker
try:
h = open(self.lock_file, 'wb')
portalocker.lock(h, portalocker.LOCK_EX + portalocker.LOCK_NB)
self._file_handle = h
except portalocker.exceptions.BaseLockException:
if h:
h.close()
self.lock_error()
else:
# will work, but crashes will leave .lock files that will raise error
self._file_handle = open(self.lock_file, 'wb' if force else 'xb')
except OSError:
self.lock_error()
def bib_script(args=None):
"""Command line script for the bibliography."""
warn_deprecation()
# Parse arguments and launch
import argparse
parser = argparse.ArgumentParser(
prog="cobaya bib",
description=
("Prints bibliography to be cited for one or more components or input files."
))
parser.add_argument(
"files_or_components",
action="store",
nargs="+",
metavar="input_file.yaml|component_name",
help="Component(s) or input file(s) whose bib info is requested.")
arguments = parser.parse_args(args)
# Configure the logger ASAP
logger_setup()
logger = get_logger("bib")
# Gather requests
infos: List[Union[Dict, str]] = []
for f in arguments.files_or_components:
if os.path.splitext(f)[1].lower() in Extension.yamls:
infos += [load_input(f)]
else: # a single component name, no kind specified
infos += [f]
if not infos:
logger.info(
"Nothing to do. Pass input files or component names as arguments.")
return
print(pretty_repr_bib(*get_bib_info(*infos, logger=logger)))
def is_installed(cls, **kwargs):
"""
Performs an installation check and returns ``True`` if successful, ``False`` if
not, or raises :class:`tools.VersionCheckError` if there is an obsolete
installation.
"""
if kwargs.get("data", True):
path = kwargs["path"]
path = cls.get_path(path) # ensure full install path passed
opts = cls.get_install_options()
if not opts:
return True
elif not (os.path.exists(path) and len(os.listdir(path)) > 0):
log = get_logger(cls.get_qualified_class_name())
log.error("The given installation path does not exist: '%s'",
path)
return False
elif opts.get("github_release"):
try:
with open(os.path.join(path, _version_filename), "r") as f:
installed_version = version.parse(f.readlines()[0])
except FileNotFoundError: # old install: no version file
raise VersionCheckError("Could not read current version.")
min_version = version.parse(opts.get("github_release"))
if installed_version < min_version:
raise VersionCheckError(
f"Installed version ({installed_version}) "
f"older than minimum required one ({min_version}).")
return True
def download_github_release(directory,
repo_name,
release_name,
repo_rename=None,
no_progress_bars=False,
logger=None):
logger = logger or get_logger("install")
if "/" in repo_name:
github_user = repo_name[:repo_name.find("/")]
repo_name = repo_name[repo_name.find("/") + 1:]
else:
github_user = "******"
if not os.path.exists(directory):
os.makedirs(directory)
url = (r"https://github.com/" + github_user + "/" + repo_name +
"/archive/" + release_name + ".tar.gz")
if not download_file(url,
directory,
decompress=True,
no_progress_bars=no_progress_bars,
logger=logger):
return False
# Remove version number from directory name
w_version = next(d for d in os.listdir(directory)
if (d.startswith(repo_name) and len(d) != len(repo_name)))
repo_rename = repo_rename or repo_name
repo_path = os.path.join(directory, repo_rename)
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
os.rename(os.path.join(directory, w_version), repo_path)
logger.info("%s %s downloaded and decompressed correctly.", repo_name,
release_name)
return True
def install_clik(path, no_progress_bars=False):
log = get_logger("clik")
log.info("Installing pre-requisites...")
for req in ("cython", "astropy"):
exit_status = pip_install(req)
if exit_status:
raise LoggedError(log, "Failed installing '%s'.", req)
log.info("Downloading...")
click_url = pla_url_prefix + '152000'
if not download_file(click_url,
path,
decompress=True,
no_progress_bars=no_progress_bars,
logger=log):
log.error("Not possible to download clik.")
return False
source_dir = get_clik_source_folder(path)
log.info('Installing from directory %s' % source_dir)
cwd = os.getcwd()
try:
os.chdir(source_dir)
log.info("Configuring... (and maybe installing dependencies...)")
flags = ["--install_all_deps", "--extra_lib=m"
] # missing for some reason in some systems, but harmless
if not execute([sys.executable, "waf", "configure"] + flags):
log.error("Configuration failed!")
return False
log.info("Compiling...")
if not execute([sys.executable, "waf", "install"]):
log.error("Compilation failed!")
return False
finally:
os.chdir(cwd)
log.info("Finished!")
return True
def download_file(url,
path,
no_progress_bars=False,
decompress=False,
logger=None):
logger = logger or get_logger("install")
with tempfile.TemporaryDirectory() as tmp_path:
try:
req = requests.get(url, allow_redirects=True, stream=True)
# get hinted filename if available:
try:
filename = re.findall("filename=(.+)",
req.headers['content-disposition'])[0]
filename = filename.strip('"\'')
except KeyError:
filename = os.path.basename(url)
filename_tmp_path = os.path.normpath(
os.path.join(tmp_path, filename))
size = int(req.headers.get('content-length', 0))
# Adapted from https://gist.github.com/yanqd0/c13ed29e29432e3cf3e7c38467f42f51
if not no_progress_bars:
bar = tqdm.tqdm(total=size,
unit='iB',
unit_scale=True,
unit_divisor=1024)
with open(filename_tmp_path, 'wb') as f:
for data in req.iter_content(chunk_size=1024):
chunk_size = f.write(data)
if not no_progress_bars:
bar.update(chunk_size)
if not no_progress_bars:
bar.close()
logger.info('Downloaded filename %s', filename)
except Exception as e:
logger.error("Error downloading %s' to folder '%s': %s", url,
tmp_path, e)
return False
logger.debug('Got: %s', filename)
if not decompress:
return True
extension = os.path.splitext(filename)[-1][1:]
try:
if extension == "zip":
from zipfile import ZipFile
with ZipFile(filename_tmp_path, 'r') as zipObj:
zipObj.extractall(path)
else:
import tarfile
if extension == "tgz":
extension = "gz"
with tarfile.open(filename_tmp_path, "r:" + extension) as tar:
tar.extractall(path)
logger.debug('Decompressed: %s', filename)
return True
except Exception as excpt:
logger.error(
"Error decompressing downloaded file! Corrupt file? [%s]",
excpt)
return False
def is_installed_clik(path, reload=False):
# min_version here is checked inside get_clik_import_path, since it is displayed
# in the folder name and cannot be retrieved from the module.
try:
return bool(load_clik(
"clik", path=path, get_import_path=get_clik_import_path,
reload=reload, logger=get_logger("clik"), not_installed_level="debug"))
except ComponentNotInstalledError:
return False
def __exit__(self, exc_type, exc_val, exc_tb):
if log:
log.info('END %s %s', self.name, self.tag)
if exc_type:
self.set(State.ERROR)
if not self.wait_all_ended(
timeout=not issubclass(exc_type, OtherProcessError)):
from cobaya.log import get_traceback_text, LoggedError, get_logger
get_logger(self.name).critical(
"Aborting MPI due to error" if issubclass(exc_type, LoggedError) else
get_traceback_text(sys.exc_info()))
self.timeout_abort_proc()
self.wait_all_ended() # if didn't actually MPI abort
else:
self.set(State.END)
self.wait_all_ended()
set_current_process_state(self.last_process_state)
if not exc_type and any(self.states == State.ERROR):
self.fire_error()
def is_installed(cls, **kwargs):
log = get_logger(cls.__name__)
if not kwargs.get("code", True):
return True
check = kwargs.get("check", True)
func = log.info if check else log.error
path: Optional[str] = kwargs["path"]
if path is not None and path.lower() == "global":
path = None
if path and not kwargs.get("allow_global"):
if is_main_process():
log.info("Importing *local* PolyChord from '%s'.", path)
if not os.path.exists(path):
if is_main_process():
func("The given folder does not exist: '%s'", path)
return False
poly_build_path = cls.get_import_path(path)
if not poly_build_path:
return False
elif not path:
if is_main_process():
log.info("Importing *global* PolyChord.")
poly_build_path = None
else:
if is_main_process():
log.info(
"Importing *auto-installed* PolyChord (but defaulting to *global*)."
)
poly_build_path = cls.get_import_path(path)
cls._poly_build_path = poly_build_path
try:
# TODO: add min_version when polychord module version available
return load_module('pypolychord',
path=poly_build_path,
min_version=None)
except ModuleNotFoundError:
if path is not None and path.lower() != "global":
log.error(
"Couldn't find the PolyChord python interface at '%s'. "
"Are you sure it has been installed there?", path)
elif not check:
log.error(
"Could not import global PolyChord installation. "
"Specify a Cobaya or PolyChord installation path, "
"or install the PolyChord Python interface globally with "
"'cd /path/to/polychord/ ; python setup.py install'")
return False
except ImportError as e:
log.error(
"Couldn't load the PolyChord python interface in %s:\n"
"%s", poly_build_path or "global", e)
return False
except VersionCheckError as e:
log.error(str(e))
return False
def is_installed(cls, reload=False, **kwargs):
if not kwargs.get("code", True):
return True
try:
return bool(load_external_module(
"pypolychord", path=kwargs["path"],
get_import_path=get_compiled_import_path,
min_version=cls._pc_repo_version, reload=reload,
logger=get_logger(cls.__name__), not_installed_level="debug"))
except ComponentNotInstalledError:
return False
def is_installed(cls, **kwargs):
if not kwargs.get("code", True):
return True
log = get_logger(cls.__name__)
import platform
check = kwargs.get("check", True)
func = log.info if check else log.error
path = kwargs["path"]
if path is not None and path.lower() == "global":
path = None
if isinstance(path, str) and not kwargs.get("allow_global"):
log.info("Importing *local* CAMB from " + path)
if not os.path.exists(path):
func("The given folder does not exist: '%s'", path)
return False
if not os.path.exists(os.path.join(path, "setup.py")):
func(
"Either CAMB is not in the given folder, '%s', or you are using"
" a very old version without the Python interface.", path)
return False
if not os.path.isfile(
os.path.realpath(
os.path.join(
path, "camb", "cambdll.dll" if
(platform.system()
== "Windows") else "camblib.so"))):
log.error(
"CAMB installation at '%s' appears not to be compiled.",
path)
return False
elif not path:
log.info("Importing *global* CAMB.")
path = None
else:
log.info(
"Importing *auto-installed* CAMB (but defaulting to *global*)."
)
try:
return load_module("camb",
path=path,
min_version=cls._min_camb_version)
except ImportError:
if path is not None and path.lower() != "global":
func(
"Couldn't find the CAMB python interface at '%s'. "
"Are you sure it has been installed there?", path)
elif not check:
log.error("Could not import global CAMB installation. "
"Specify a Cobaya or CAMB installation path, "
"or install the 'camb' Python package globally.")
return False
except VersionCheckError as e:
log.error(str(e))
return False
def install(cls, path=None, force=False, code=True, data=True,
no_progress_bars=False):
name = cls.get_qualified_class_name()
log = get_logger(name)
path_names = {"code": common_path, "data": get_data_path(name)}
import platform
if platform.system() == "Windows":
log.error("Not compatible with Windows.")
return False
global _clik_install_failed
if _clik_install_failed:
log.info("Previous clik install failed, skipping")
return False
# Create common folders: all planck likelihoods share install
# folder for code and data
paths = {}
for s in ("code", "data"):
if eval(s):
paths[s] = os.path.realpath(os.path.join(path, s, path_names[s]))
if not os.path.exists(paths[s]):
os.makedirs(paths[s])
success = True
# Install clik
if code and (not is_installed_clik(paths["code"]) or force):
log.info("Installing the clik code.")
success *= install_clik(paths["code"], no_progress_bars=no_progress_bars)
if not success:
log.warning("clik code installation failed! "
"Try configuring+compiling by hand at " + paths["code"])
_clik_install_failed = True
if data:
# 2nd test, in case the code wasn't there but the data is:
if force or not cls.is_installed(path=path, code=False, data=True):
log.info("Downloading the likelihood data.")
product_id, _ = get_product_id_and_clik_file(name)
# Download and decompress the particular likelihood
url = pla_url_prefix + product_id
# Helper for the progress bars: some known product download sizes
# (no actual effect if missing or wrong!)
size = {"1900": 314153370, "1903": 4509715660, "151902": 60293120,
"151905": 5476083302, "151903": 8160437862}.get(product_id)
if not download_file(url, paths["data"], size=size, decompress=True,
logger=log, no_progress_bars=no_progress_bars):
log.error("Not possible to download this likelihood.")
success = False
# Additional data and covmats, stored in same repo as the
# 2018 python lensing likelihood
from cobaya.likelihoods.planck_2018_lensing import native
if not native.is_installed(data=True, path=path):
success *= native.install(path=path, force=force, code=code,
data=data,
no_progress_bars=no_progress_bars)
return success
def is_installed(cls, **kwargs):
if kwargs.get("data", True):
path = kwargs["path"]
opts = cls.get_install_options()
if not opts:
return True
elif not (os.path.exists(path) and len(os.listdir(path)) > 0):
log = get_logger(cls.get_qualified_class_name())
func = log.info if kwargs.get("check", True) else log.error
func("The given installation path does not exist: '%s'", path)
return False
return True
def get_bib_info(*infos, logger=None):
"""
Gathers and returns the descriptions and bibliographic sources for the components
mentioned in ``infos``.
``infos`` can be input dictionaries or single component names.
"""
if not logger:
logger_setup()
logger = get_logger("bib")
used_components, component_infos = get_used_components(*infos,
return_infos=True)
descs: InfoDict = {}
bibs: InfoDict = {}
used_components = get_used_components(*infos)
for kind, components in used_components.items():
if kind is None:
continue # we will deal with bare component names later, to avoid repetition
descs[kind], bibs[kind] = {}, {}
for component in components:
try:
descs[kind][component] = get_desc_component(
component, kind, component_infos[component])
bibs[kind][component] = get_bib_component(component, kind)
except ComponentNotFoundError:
sugg = similar_internal_class_names(component)
logger.error(
f"Could not identify component '{component}'. "
f"Did you mean any of the following? {sugg} (mind capitalization!)"
)
continue
# Deal with bare component names
for component in used_components.get(None, []):
try:
cls = get_component_class(component)
except ComponentNotFoundError:
sugg = similar_internal_class_names(component)
logger.error(
f"Could not identify component '{component}'. "
f"Did you mean any of the following? {sugg} (mind capitalization!)"
)
continue
kind = cls.get_kind()
if kind not in descs:
descs[kind], bibs[kind] = {}, {}
if kind in descs and component in descs[kind]:
continue # avoid repetition
descs[kind][component] = get_desc_component(cls, kind)
bibs[kind][component] = get_bib_component(cls, kind)
descs["cobaya"] = {"cobaya": cobaya_desc}
bibs["cobaya"] = {"cobaya": cobaya_bib}
return descs, bibs
def install(cls, path=None, code=True, no_progress_bars=False, **_kwargs):
log = get_logger(cls.__name__)
if not code:
log.info("Code not requested. Nothing to do.")
return True
log.info("Installing pre-requisites...")
exit_status = pip_install("cython")
if exit_status:
log.error("Could not install pre-requisite: cython")
return False
log.info("Downloading classy...")
success = download_github_release(os.path.join(path, "code"),
cls._classy_repo_name,
cls._classy_repo_version,
directory=cls.__name__,
no_progress_bars=no_progress_bars,
logger=log)
if not success:
log.error("Could not download classy.")
return False
# Compilation
# gcc check after downloading, in case the user wants to change the compiler by
# hand in the Makefile
classy_path = cls.get_path(path)
if not check_gcc_version(cls._classy_min_gcc_version,
error_returns=False):
log.error(
"Your gcc version is too low! CLASS would probably compile, "
"but it would leak memory when running a chain. Please use a "
"gcc version newer than %s. You can still compile CLASS by hand, "
"maybe changing the compiler in the Makefile. CLASS has been "
"downloaded into %r", cls._classy_min_gcc_version, classy_path)
return False
log.info("Compiling classy...")
from subprocess import Popen, PIPE
env = deepcopy(os.environ)
env.update({"PYTHON": sys.executable})
process_make = Popen(["make"],
cwd=classy_path,
stdout=PIPE,
stderr=PIPE,
env=env)
out, err = process_make.communicate()
if process_make.returncode:
log.info(out)
log.info(err)
log.error("Compilation failed!")
return False
return True
def is_installed(cls, **kwargs):
if not kwargs.get("code", True):
return True
log = get_logger(cls.__name__)
check = kwargs.get("check", True)
func = log.info if check else log.error
path = kwargs["path"]
if path is not None and path.lower() == "global":
path = None
if path and not kwargs.get("allow_global"):
log.info("Importing *local* CLASS from '%s'.", path)
assert path is not None
if not os.path.exists(path):
func("The given folder does not exist: '%s'", path)
return False
classy_build_path = cls.get_import_path(path)
if not classy_build_path:
return False
elif not path:
log.info("Importing *global* CLASS.")
classy_build_path = None
else:
log.info(
"Importing *auto-installed* CLASS (but defaulting to *global*)."
)
classy_build_path = cls.get_import_path(path)
try:
return load_module('classy',
path=classy_build_path,
min_version=cls._classy_repo_version)
except ImportError:
if path is not None and path.lower() != "global":
func(
"Couldn't find the CLASS python interface at '%s'. "
"Are you sure it has been installed there?", path)
elif not check:
log.error(
"Could not import global CLASS installation. "
"Specify a Cobaya or CLASS installation path, "
"or install the CLASS Python interface globally with "
"'cd /path/to/class/python/ ; python setup.py install'")
return False
except VersionCheckError as e:
log.error(str(e))
return False
def get_import_path(cls, path):
log = get_logger(cls.__name__)
poly_build_path = os.path.join(path, "build")
if not os.path.isdir(poly_build_path):
log.error(
"Either PolyChord is not in the given folder, "
"'%s', or you have not compiled it.", path)
return None
py_version = "%d.%d" % (sys.version_info.major, sys.version_info.minor)
try:
post = next(d for d in os.listdir(poly_build_path)
if (d.startswith("lib.") and py_version in d))
except StopIteration:
log.error(
"The PolyChord installation at '%s' has not been compiled for the "
"current Python version.", path)
return None
return os.path.join(poly_build_path, post)
def get_sampler(info_sampler: SamplersDict,
model: Model,
output: Optional[Output] = None,
packages_path: Optional[str] = None) -> 'Sampler':
assert isinstance(info_sampler, Mapping), (
"The first argument must be a dictionary with the info needed for the sampler. "
"If you were trying to pass the name of an input file instead, "
"load it first with 'cobaya.input.load_input', "
"or, if you were passing a yaml string, load it with 'cobaya.yaml.yaml_load'."
)
logger_sampler = get_logger(__name__)
info_sampler = deepcopy_where_possible(info_sampler)
if output is None:
output = OutputDummy()
# Check and update info
check_sane_info_sampler(info_sampler)
updated_info_sampler = update_info({"sampler": info_sampler
})["sampler"] # type: ignore
if is_debug(logger_sampler):
logger_sampler.debug(
"Input info updated with defaults (dumped to YAML):\n%s",
yaml_dump(updated_info_sampler))
# Get sampler class & check resume/force compatibility
sampler_name, sampler_class = get_sampler_name_and_class(
updated_info_sampler, logger=logger_sampler)
check_sampler_info((output.reload_updated_info(use_cache=True)
or {}).get("sampler"),
updated_info_sampler,
is_resuming=output.is_resuming())
# Check if resumable run
sampler_class.check_force_resume(output,
info=updated_info_sampler[sampler_name])
# Instantiate the sampler
sampler_instance = sampler_class(updated_info_sampler[sampler_name],
model,
output,
packages_path=packages_path)
# If output, dump updated
if output:
to_dump = model.info()
to_dump["sampler"] = {sampler_name: sampler_instance.info()}
to_dump["output"] = os.path.join(output.folder, output.prefix)
output.check_and_dump_info(None, to_dump, check_compatible=False)
return sampler_instance
def check_sampler_info(info_old: Optional[SamplersDict],
info_new: SamplersDict,
is_resuming=False):
"""
Checks compatibility between the new sampler info and that of a pre-existing run.
Done separately from `Output.check_compatible_and_dump` because there may be
multiple samplers mentioned in an `updated.yaml` file, e.g. `MCMC` + `Minimize`.
"""
logger_sampler = get_logger(__name__)
if not info_old:
return
# TODO: restore this at some point: just append minimize info to the old one
# There is old info, but the new one is Minimizer and the old one is not
# if (len(info_old) == 1 and list(info_old) != ["minimize"] and
# list(info_new) == ["minimize"]):
# # In-place append of old+new --> new
# aux = info_new.pop("minimize")
# info_new.update(info_old)
# info_new.update({"minimize": aux})
# info_old = {}
# keep_old = {}
if list(info_old) != list(info_new) and list(info_new) == ["minimize"]:
return
if list(info_old) == list(info_new):
# Restore some selected old values for some classes
keep_old = get_preferred_old_values({"sampler": info_old})
info_new = recursive_update(info_new, keep_old.get("sampler", {}))
if not is_equal_info({"sampler": info_old}, {"sampler": info_new},
strict=False):
if is_resuming:
raise LoggedError(
logger_sampler,
"Old and new Sampler information not compatible! "
"Resuming not possible!")
else:
raise LoggedError(
logger_sampler,
"Found old Sampler information which is not compatible "
"with the new one. Delete the previous output manually, "
"or automatically with either "
"'-f', '--force', 'force: True'")
def __init__(self, *args, **kwargs):
# Ensure check for install and version errors
# (e.g. may inherit from a class that inherits from this one, and not have them)
if self.install_options:
name = self.get_qualified_class_name()
logger = get_logger(name)
packages_path = kwargs.get(
"packages_path") or resolve_packages_path()
old = False
try:
installed = self.is_installed(path=packages_path)
except Exception as excpt: # catches VersionCheckError and unexpected ones
installed = False
old = isinstance(excpt, VersionCheckError)
logger.error(f"{type(excpt).__name__}: {excpt}")
if not installed:
not_or_old = ("is not up to date"
if old else "has not been correctly installed")
raise ComponentNotInstalledError(logger, (
f"The data for this likelihood {not_or_old}. To install it, "
f"run `cobaya-install {name}{' --upgrade' if old else ''}`"
))
super().__init__(*args, **kwargs)
def install(cls, path=None, code=True, no_progress_bars=False, **_kwargs):
log = get_logger(cls.__name__)
if not code:
log.info("Code not requested. Nothing to do.")
return True
log.info("Downloading camb...")
success = download_github_release(os.path.join(path, "code"),
cls._camb_repo_name,
cls._camb_repo_version,
no_progress_bars=no_progress_bars,
logger=log)
if not success:
log.error("Could not download camb.")
return False
camb_path = cls.get_path(path)
log.info("Compiling camb...")
from subprocess import Popen, PIPE
process_make = Popen([sys.executable, "setup.py", "build_cluster"],
cwd=camb_path,
stdout=PIPE,
stderr=PIPE)
out, err = process_make.communicate()
if process_make.returncode:
log.info(out.decode())
log.info(err.decode())
gcc_check = check_gcc_version(cls._camb_min_gcc_version,
error_returns=False)
if not gcc_check:
cause = (
" Possible cause: it looks like `gcc` does not have the correct "
"version number (CAMB requires %s); and `ifort` is also "
"probably not available." % cls._camb_min_gcc_version)
else:
cause = ""
log.error("Compilation failed!" + cause)
return False
return True
def is_installed_clik(path, allow_global=False, check=True):
log = get_logger("clik")
func = log.info if check else log.error
if path is not None and path.lower() == "global":
path = None
clik_path = None
if isinstance(path, str) and path.lower() != "global":
try:
clik_path = os.path.join(get_clik_source_folder(path),
'lib/python/site-packages')
except FileNotFoundError:
func("The given folder does not exist: '%s'", clik_path or path)
return False
if path and not allow_global:
log.info("Importing *local* clik from %s ", path)
elif not path:
log.info("Importing *global* clik.")
else:
log.info(
"Importing *auto-installed* clik (but defaulting to *global*).")
try:
return load_module("clik", path=clik_path)
except ImportError:
if path is not None and path.lower() != "global":
func(
"Couldn't find the clik python interface at '%s'. "
"Are you sure it has been installed and compiled there?", path)
elif not check:
log.error("Could not import global clik installation. "
"Specify a Cobaya or clik installation path, "
"or install the clik Python interface globally.")
except Exception as excpt:
log.error(
"Error when trying to import clik from %s [%s]. Error message: [%s].",
path, clik_path, str(excpt))
return False
def post(info_or_yaml_or_file: Union[InputDict, str, os.PathLike],
sample: Union[SampleCollection, List[SampleCollection], None] = None
) -> PostTuple:
info = load_input_dict(info_or_yaml_or_file)
logger_setup(info.get("debug"), info.get("debug_file"))
log = get_logger(__name__)
# MARKED FOR DEPRECATION IN v3.0
if info.get("modules"):
raise LoggedError(log, "The input field 'modules' has been deprecated."
"Please use instead %r", packages_path_input)
# END OF DEPRECATION BLOCK
info_post: PostDict = info.get("post") or {}
if not info_post:
raise LoggedError(log, "No 'post' block given. Nothing to do!")
if mpi.is_main_process() and info.get("resume"):
log.warning("Resuming not implemented for post-processing. Re-starting.")
if not info.get("output") and info_post.get("output") \
and not info.get("params"):
raise LoggedError(log, "The input dictionary must have be a full option "
"dictionary, or have an existing 'output' root to load "
"previous settings from ('output' to read from is in the "
"main block not under 'post'). ")
# 1. Load existing sample
output_in = get_output(prefix=info.get("output"))
if output_in:
info_in = output_in.load_updated_info() or update_info(info)
else:
info_in = update_info(info)
params_in: ExpandedParamsDict = info_in["params"] # type: ignore
dummy_model_in = DummyModel(params_in, info_in.get("likelihood", {}),
info_in.get("prior"))
in_collections = []
thin = info_post.get("thin", 1)
skip = info_post.get("skip", 0)
if info.get('thin') is not None or info.get('skip') is not None: # type: ignore
raise LoggedError(log, "'thin' and 'skip' should be "
"parameters of the 'post' block")
if sample:
# If MPI, assume for each MPI process post is passed in the list of
# collections that should be processed by that process
# (e.g. single chain output from sampler)
if isinstance(sample, SampleCollection):
in_collections = [sample]
else:
in_collections = sample
for i, collection in enumerate(in_collections):
if skip:
if 0 < skip < 1:
skip = int(round(skip * len(collection)))
collection = collection.filtered_copy(slice(skip, None))
if thin != 1:
collection = collection.thin_samples(thin)
in_collections[i] = collection
elif output_in:
files = output_in.find_collections()
numbered = files
if not numbered:
# look for un-numbered output files
files = output_in.find_collections(name=False)
if files:
if mpi.size() > len(files):
raise LoggedError(log, "Number of MPI processes (%s) is larger than "
"the number of sample files (%s)",
mpi.size(), len(files))
for num in range(mpi.rank(), len(files), mpi.size()):
in_collections += [SampleCollection(
dummy_model_in, output_in,
onload_thin=thin, onload_skip=skip, load=True, file_name=files[num],
name=str(num + 1) if numbered else "")]
else:
raise LoggedError(log, "No samples found for the input model with prefix %s",
os.path.join(output_in.folder, output_in.prefix))
else:
raise LoggedError(log, "No output from where to load from, "
"nor input collections given.")
if any(len(c) <= 1 for c in in_collections):
raise LoggedError(
log, "Not enough samples for post-processing. Try using a larger sample, "
"or skipping or thinning less.")
mpi.sync_processes()
log.info("Will process %d sample points.", sum(len(c) for c in in_collections))
# 2. Compare old and new info: determine what to do
add = info_post.get("add") or {}
if "remove" in add:
raise LoggedError(log, "remove block should be under 'post', not 'add'")
remove = info_post.get("remove") or {}
# Add a dummy 'one' likelihood, to absorb unused parameters
if not add.get("likelihood"):
add["likelihood"] = {}
add["likelihood"]["one"] = None
# Expand the "add" info, but don't add new default sampled parameters
orig_params = set(add.get("params") or [])
add = update_info(add, add_aggr_chi2=False)
add_params: ExpandedParamsDict = add["params"] # type: ignore
for p in set(add_params) - orig_params:
if p in params_in:
add_params.pop(p)
# 2.1 Adding/removing derived parameters and changes in priors of sampled parameters
out_combined_params = deepcopy_where_possible(params_in)
remove_params = list(str_to_list(remove.get("params")) or [])
for p in remove_params:
pinfo = params_in.get(p)
if pinfo is None or not is_derived_param(pinfo):
raise LoggedError(
log,
"You tried to remove parameter '%s', which is not a derived parameter. "
"Only derived parameters can be removed during post-processing.", p)
out_combined_params.pop(p)
# Force recomputation of aggregated chi2
for p in list(out_combined_params):
if p.startswith(get_chi2_name("")):
out_combined_params.pop(p)
prior_recompute_1d = False
for p, pinfo in add_params.items():
pinfo_in = params_in.get(p)
if is_sampled_param(pinfo):
if not is_sampled_param(pinfo_in):
# No added sampled parameters (de-marginalisation not implemented)
if pinfo_in is None:
raise LoggedError(
log, "You added a new sampled parameter %r (maybe accidentally "
"by adding a new likelihood that depends on it). "
"Adding new sampled parameters is not possible. Try fixing "
"it to some value.", p)
else:
raise LoggedError(
log,
"You tried to change the prior of parameter '%s', "
"but it was not a sampled parameter. "
"To change that prior, you need to define as an external one.", p)
# recompute prior if potentially changed sampled parameter priors
prior_recompute_1d = True
elif is_derived_param(pinfo):
if p in out_combined_params:
raise LoggedError(
log, "You tried to add derived parameter '%s', which is already "
"present. To force its recomputation, 'remove' it too.", p)
elif is_fixed_or_function_param(pinfo):
# Only one possibility left "fixed" parameter that was not present before:
# input of new likelihood, or just an argument for dynamical derived (dropped)
if pinfo_in and p in params_in and pinfo["value"] != pinfo_in.get("value"):
raise LoggedError(
log,
"You tried to add a fixed parameter '%s: %r' that was already present"
" but had a different value or was not fixed. This is not allowed. "
"The old info of the parameter was '%s: %r'",
p, dict(pinfo), p, dict(pinfo_in))
elif not pinfo_in: # OK as long as we have known value for it
raise LoggedError(log, "Parameter %s no known value. ", p)
out_combined_params[p] = pinfo
out_combined: InputDict = {"params": out_combined_params} # type: ignore
# Turn the rest of *derived* parameters into constants,
# so that the likelihoods do not try to recompute them
# But be careful to exclude *input* params that have a "derived: True" value
# (which in "updated info" turns into "derived: 'lambda [x]: [x]'")
# Don't assign to derived parameters to theories, only likelihoods, so they can be
# recomputed if needed. If the theory does not need to be computed, it doesn't matter
# if it is already assigned parameters in the usual way; likelihoods can get
# the required derived parameters from the stored sample derived parameter inputs.
out_params_with_computed = deepcopy_where_possible(out_combined_params)
dropped_theory = set()
for p, pinfo in out_params_with_computed.items():
if (is_derived_param(pinfo) and "value" not in pinfo
and p not in add_params):
out_params_with_computed[p] = {"value": np.nan}
dropped_theory.add(p)
# 2.2 Manage adding/removing priors and likelihoods
warn_remove = False
kind: ModelBlock
for kind in ("prior", "likelihood", "theory"):
out_combined[kind] = deepcopy_where_possible(info_in.get(kind)) or {}
for remove_item in str_to_list(remove.get(kind)) or []:
try:
out_combined[kind].pop(remove_item, None)
if remove_item not in (add.get(kind) or []) and kind != "theory":
warn_remove = True
except ValueError:
raise LoggedError(
log, "Trying to remove %s '%s', but it is not present. "
"Existing ones: %r", kind, remove_item, list(out_combined[kind]))
if kind != "theory" and kind in add:
dups = set(add.get(kind) or []).intersection(out_combined[kind]) - {"one"}
if dups:
raise LoggedError(
log, "You have added %s '%s', which was already present. If you "
"want to force its recomputation, you must also 'remove' it.",
kind, dups)
out_combined[kind].update(add[kind])
if warn_remove and mpi.is_main_process():
log.warning("You are removing a prior or likelihood pdf. "
"Notice that if the resulting posterior is much wider "
"than the original one, or displaced enough, "
"it is probably safer to explore it directly.")
mlprior_names_add = minuslogprior_names(add.get("prior") or [])
chi2_names_add = [get_chi2_name(name) for name in add["likelihood"] if
name != "one"]
out_combined["likelihood"].pop("one", None)
add_theory = add.get("theory")
if add_theory:
if len(add["likelihood"]) == 1 and not any(
is_derived_param(pinfo) for pinfo in add_params.values()):
log.warning("You are adding a theory, but this does not force recomputation "
"of any likelihood or derived parameters unless explicitly "
"removed+added.")
# Inherit from the original chain (input|output_params, renames, etc)
added_theory = add_theory.copy()
for theory, theory_info in out_combined["theory"].items():
if theory in list(added_theory):
out_combined["theory"][theory] = \
recursive_update(theory_info, added_theory.pop(theory))
out_combined["theory"].update(added_theory)
# Prepare recomputation of aggregated chi2
# (they need to be recomputed by hand, because auto-computation won't pick up
# old likelihoods for a given type)
all_types = {like: str_to_list(opts.get("type") or [])
for like, opts in out_combined["likelihood"].items()}
types = set(chain(*all_types.values()))
inv_types = {t: [like for like, like_types in all_types.items() if t in like_types]
for t in sorted(types)}
add_aggregated_chi2_params(out_combined_params, types)
# 3. Create output collection
# Use default prefix if it exists. If it does not, produce no output by default.
# {post: {output: None}} suppresses output, and if it's a string, updates it.
out_prefix = info_post.get("output", info.get("output"))
if out_prefix:
suffix = info_post.get("suffix")
if not suffix:
raise LoggedError(log, "You need to provide a '%s' for your output chains.",
"suffix")
out_prefix += separator_files + "post" + separator_files + suffix
output_out = get_output(prefix=out_prefix, force=info.get("force"))
output_out.set_lock()
if output_out and not output_out.force and output_out.find_collections():
raise LoggedError(log, "Found existing post-processing output with prefix %r. "
"Delete it manually or re-run with `force: True` "
"(or `-f`, `--force` from the shell).", out_prefix)
elif output_out and output_out.force and mpi.is_main_process():
output_out.delete_infos()
for _file in output_out.find_collections():
output_out.delete_file_or_folder(_file)
info_out = deepcopy_where_possible(info)
info_post = info_post.copy()
info_out["post"] = info_post
# Updated with input info and extended (updated) add info
info_out.update(info_in) # type: ignore
info_post["add"] = add
dummy_model_out = DummyModel(out_combined_params, out_combined["likelihood"],
info_prior=out_combined["prior"])
out_func_parameterization = Parameterization(out_params_with_computed)
# TODO: check allow_renames=False?
model_add = Model(out_params_with_computed, add["likelihood"],
info_prior=add.get("prior"), info_theory=out_combined["theory"],
packages_path=(info_post.get(packages_path_input) or
info.get(packages_path_input)),
allow_renames=False, post=True,
stop_at_error=info.get('stop_at_error', False),
skip_unused_theories=True, dropped_theory_params=dropped_theory)
# Remove auxiliary "one" before dumping -- 'add' *is* info_out["post"]["add"]
add["likelihood"].pop("one")
out_collections = [SampleCollection(dummy_model_out, output_out, name=c.name,
cache_size=OutputOptions.default_post_cache_size)
for c in in_collections]
# TODO: should maybe add skip/thin to out_combined, so can tell post-processed?
output_out.check_and_dump_info(info_out, out_combined, check_compatible=False)
collection_in = in_collections[0]
collection_out = out_collections[0]
last_percent = None
known_constants = dummy_model_out.parameterization.constant_params()
known_constants.update(dummy_model_in.parameterization.constant_params())
missing_params = dummy_model_in.parameterization.sampled_params().keys() - set(
collection_in.columns)
if missing_params:
raise LoggedError(log, "Input samples do not contain expected sampled parameter "
"values: %s", missing_params)
missing_priors = set(name for name in collection_out.minuslogprior_names if
name not in mlprior_names_add
and name not in collection_in.columns)
if _minuslogprior_1d_name in missing_priors:
prior_recompute_1d = True
if prior_recompute_1d:
missing_priors.discard(_minuslogprior_1d_name)
mlprior_names_add.insert(0, _minuslogprior_1d_name)
prior_regenerate: Optional[Prior]
if missing_priors and "prior" in info_in:
# in case there are input priors that are not stored in input samples
# e.g. when postprocessing GetDist/CosmoMC-format chains
in_names = minuslogprior_names(info_in["prior"])
info_prior = {piname: inf for (piname, inf), in_name in
zip(info_in["prior"].items(), in_names) if
in_name in missing_priors}
regenerated_prior_names = minuslogprior_names(info_prior)
missing_priors.difference_update(regenerated_prior_names)
prior_regenerate = Prior(dummy_model_in.parameterization, info_prior)
else:
prior_regenerate = None
regenerated_prior_names = None
if missing_priors:
raise LoggedError(log, "Missing priors: %s", missing_priors)
mpi.sync_processes()
output_in.check_lock()
# 4. Main loop! Loop over input samples and adjust as required.
if mpi.is_main_process():
log.info("Running post-processing...")
difflogmax: Optional[float] = None
to_do = sum(len(c) for c in in_collections)
weights = []
done = 0
last_dump_time = time.time()
for collection_in, collection_out in zip(in_collections, out_collections):
importance_weights = []
def set_difflogmax():
nonlocal difflogmax
difflog = (collection_in[OutPar.minuslogpost].to_numpy(
dtype=np.float64)[:len(collection_out)]
- collection_out[OutPar.minuslogpost].to_numpy(dtype=np.float64))
difflogmax = np.max(difflog)
if abs(difflogmax) < 1:
difflogmax = 0 # keep simple when e.g. very similar
log.debug("difflogmax: %g", difflogmax)
if mpi.more_than_one_process():
difflogmax = max(mpi.allgather(difflogmax))
if mpi.is_main_process():
log.debug("Set difflogmax: %g", difflogmax)
_weights = np.exp(difflog - difflogmax)
importance_weights.extend(_weights)
collection_out.reweight(_weights)
for i, point in collection_in.data.iterrows():
all_params = point.to_dict()
for p in remove_params:
all_params.pop(p, None)
log.debug("Point: %r", point)
sampled = np.array([all_params[param] for param in
dummy_model_in.parameterization.sampled_params()])
all_params = out_func_parameterization.to_input(all_params).copy()
# Add/remove priors
if prior_recompute_1d:
priors_add = [model_add.prior.logps_internal(sampled)]
if priors_add[0] == -np.inf:
continue
else:
priors_add = []
if model_add.prior.external:
priors_add.extend(model_add.prior.logps_external(all_params))
logpriors_add = dict(zip(mlprior_names_add, priors_add))
logpriors_new = [logpriors_add.get(name, - point.get(name, 0))
for name in collection_out.minuslogprior_names]
if prior_regenerate:
regenerated = dict(zip(regenerated_prior_names,
prior_regenerate.logps_external(all_params)))
for _i, name in enumerate(collection_out.minuslogprior_names):
if name in regenerated_prior_names:
logpriors_new[_i] = regenerated[name]
if is_debug(log):
log.debug("New set of priors: %r",
dict(zip(dummy_model_out.prior, logpriors_new)))
if -np.inf in logpriors_new:
continue
# Add/remove likelihoods and/or (re-)calculate derived parameters
loglikes_add, output_derived = model_add._loglikes_input_params(
all_params, return_output_params=True)
loglikes_add = dict(zip(chi2_names_add, loglikes_add))
output_derived = dict(zip(model_add.output_params, output_derived))
loglikes_new = [loglikes_add.get(name, -0.5 * point.get(name, 0))
for name in collection_out.chi2_names]
if is_debug(log):
log.debug("New set of likelihoods: %r",
dict(zip(dummy_model_out.likelihood, loglikes_new)))
if output_derived:
log.debug("New set of derived parameters: %r", output_derived)
if -np.inf in loglikes_new:
continue
all_params.update(output_derived)
all_params.update(out_func_parameterization.to_derived(all_params))
derived = {param: all_params.get(param) for param in
dummy_model_out.parameterization.derived_params()}
# We need to recompute the aggregated chi2 by hand
for type_, likes in inv_types.items():
derived[get_chi2_name(type_)] = sum(
-2 * lvalue for lname, lvalue
in zip(collection_out.chi2_names, loglikes_new)
if undo_chi2_name(lname) in likes)
if is_debug(log):
log.debug("New derived parameters: %r",
{p: derived[p]
for p in dummy_model_out.parameterization.derived_params()
if p in add["params"]})
# Save to the collection (keep old weight for now)
weight = point.get(OutPar.weight)
mpi.check_errors()
if difflogmax is None and i > OutputOptions.reweight_after and \
time.time() - last_dump_time > OutputOptions.output_inteveral_s / 2:
set_difflogmax()
collection_out.out_update()
if difflogmax is not None:
logpost_new = sum(logpriors_new) + sum(loglikes_new)
importance_weight = np.exp(logpost_new + point.get(OutPar.minuslogpost)
- difflogmax)
weight = weight * importance_weight
importance_weights.append(importance_weight)
if time.time() - last_dump_time > OutputOptions.output_inteveral_s:
collection_out.out_update()
last_dump_time = time.time()
if weight > 0:
collection_out.add(sampled, derived=derived.values(), weight=weight,
logpriors=logpriors_new, loglikes=loglikes_new)
# Display progress
percent = int(np.round((i + done) / to_do * 100))
if percent != last_percent and not percent % 5:
last_percent = percent
progress_bar(log, percent, " (%d/%d)" % (i + done, to_do))
if difflogmax is None:
set_difflogmax()
if not collection_out.data.last_valid_index():
raise LoggedError(
log, "No elements in the final sample. Possible causes: "
"added a prior or likelihood valued zero over the full sampled "
"domain, or the computation of the theory failed everywhere, etc.")
collection_out.out_update()
weights.append(np.array(importance_weights))
done += len(collection_in)
assert difflogmax is not None
points = 0
tot_weight = 0
min_weight = np.inf
max_weight = -np.inf
max_output_weight = -np.inf
sum_w2 = 0
points_removed = 0
for collection_in, collection_out, importance_weights in zip(in_collections,
out_collections,
weights):
output_weights = collection_out[OutPar.weight]
points += len(collection_out)
tot_weight += np.sum(output_weights)
points_removed += len(importance_weights) - len(output_weights)
min_weight = min(min_weight, np.min(importance_weights))
max_weight = max(max_weight, np.max(importance_weights))
max_output_weight = max(max_output_weight, np.max(output_weights))
sum_w2 += np.dot(output_weights, output_weights)
(tot_weights, min_weights, max_weights, max_output_weights, sum_w2s, points_s,
points_removed_s) = mpi.zip_gather(
[tot_weight, min_weight, max_weight, max_output_weight, sum_w2,
points, points_removed])
if mpi.is_main_process():
output_out.clear_lock()
log.info("Finished! Final number of distinct sample points: %s", sum(points_s))
log.info("Importance weight range: %.4g -- %.4g",
min(min_weights), max(max_weights))
if sum(points_removed_s):
log.info("Points deleted due to zero weight: %s", sum(points_removed_s))
log.info("Effective number of single samples if independent (sum w)/max(w): %s",
int(sum(tot_weights) / max(max_output_weights)))
log.info(
"Effective number of weighted samples if independent (sum w)^2/sum(w^2): "
"%s", int(sum(tot_weights) ** 2 / sum(sum_w2s)))
products: PostResultDict = {"sample": value_or_list(out_collections),
"stats": {'min_importance_weight': (min(min_weights) /
max(max_weights)),
'points_removed': sum(points_removed_s),
'tot_weight': sum(tot_weights),
'max_weight': max(max_output_weights),
'sum_w2': sum(sum_w2s),
'points': sum(points_s)},
"logpost_weight_offset": difflogmax,
"weights": value_or_list(weights)}
return PostTuple(info=out_combined, products=products)
from itertools import permutations
from typing import Mapping, Sequence, Any, List, TypeVar, Optional, Union, \
Iterable, Set, Dict
from types import ModuleType
from inspect import cleandoc, getfullargspec
from ast import parse
import traceback
# Local
from cobaya.conventions import cobaya_package, subfolders, kinds, \
packages_path_config_file, packages_path_env, packages_path_arg, dump_sort_cosmetic
from cobaya.log import LoggedError, HasLogger, get_logger
from cobaya.typing import Kind
# Set up logger
log = get_logger(__name__)
def str_to_list(x) -> List:
"""
Makes sure that the input is a list of strings (could be string).
"""
return [x] if isinstance(x, str) else x
def ensure_dict(iterable_or_dict):
"""
For iterables, returns dict with elements as keys and null values.
"""
if not isinstance(iterable_or_dict, Mapping):
return dict.fromkeys(iterable_or_dict)
def run_script(args=None):
warn_deprecation()
import argparse
parser = argparse.ArgumentParser(prog="cobaya run",
description="Cobaya's run script.")
parser.add_argument("input_file",
action="store",
metavar="input_file.yaml",
help="An input file to run.")
parser.add_argument("-" + packages_path_arg[0],
"--" + packages_path_arg_posix,
action="store",
metavar="/packages/path",
default=None,
help="Path where external packages were installed.")
# MARKED FOR DEPRECATION IN v3.0
modules = "modules"
parser.add_argument("-" + modules[0],
"--" + modules,
action="store",
required=False,
metavar="/packages/path",
default=None,
help="To be deprecated! "
"Alias for %s, which should be used instead." %
packages_path_arg_posix)
# END OF DEPRECATION BLOCK -- CONTINUES BELOW!
parser.add_argument("-" + "o",
"--" + "output",
action="store",
metavar="/some/path",
default=None,
help="Path and prefix for the text output.")
parser.add_argument("-" + "d",
"--" + "debug",
action="store_true",
help="Produce verbose debug output.")
continuation = parser.add_mutually_exclusive_group(required=False)
continuation.add_argument(
"-" + "r",
"--" + "resume",
action="store_true",
help="Resume an existing chain if it has similar info "
"(fails otherwise).")
continuation.add_argument("-" + "f",
"--" + "force",
action="store_true",
help="Overwrites previous output, if it exists "
"(use with care!)")
parser.add_argument("--%s" % "test",
action="store_true",
help="Initialize model and sampler, and exit.")
parser.add_argument("--version", action="version", version=get_version())
parser.add_argument("--no-mpi",
action='store_true',
help="disable MPI when mpi4py installed but MPI does "
"not actually work")
arguments = parser.parse_args(args)
# MARKED FOR DEPRECATION IN v3.0
if arguments.modules is not None:
logger_setup()
logger = get_logger("run")
logger.warning(
"*DEPRECATION*: -m/--modules will be deprecated in favor of "
"-%s/--%s in the next version. Please, use that one instead.",
packages_path_arg[0], packages_path_arg_posix)
# BEHAVIOUR TO BE REPLACED BY ERROR:
if getattr(arguments, packages_path_arg) is None:
setattr(arguments, packages_path_arg, arguments.modules)
del arguments.modules
# END OF DEPRECATION BLOCK
info = arguments.input_file
del arguments.input_file
run(info, **arguments.__dict__)
def run(
info_or_yaml_or_file: Union[InputDict, str, os.PathLike],
packages_path: Optional[str] = None,
output: Union[str, LiteralFalse, None] = None,
debug: Union[bool, int, None] = None,
stop_at_error: Optional[bool] = None,
resume: bool = False,
force: bool = False,
no_mpi: bool = False,
test: bool = False,
override: Optional[InputDict] = None,
) -> Union[InfoSamplerTuple, PostTuple]:
"""
Run from an input dictionary, file name or yaml string, with optional arguments
to override settings in the input as needed.
:param info_or_yaml_or_file: input options dictionary, yaml file, or yaml text
:param packages_path: path where external packages were installed
:param output: path name prefix for output files, or False for no file output
:param debug: true for verbose debug output, or a specific logging level
:param stop_at_error: stop if an error is raised
:param resume: continue an existing run
:param force: overwrite existing output if it exists
:param no_mpi: run without MPI
:param test: only test initialization rather than actually running
:param override: option dictionary to merge into the input one, overriding settings
(but with lower precedence than the explicit keyword arguments)
:return: (updated_info, sampler) tuple of options dictionary and Sampler instance,
or (updated_info, results) if using "post" post-processing
"""
# This function reproduces the model-->output-->sampler pipeline one would follow
# when instantiating by hand, but alters the order to performs checks and dump info
# as early as possible, e.g. to check if resuming possible or `force` needed.
if no_mpi or test:
mpi.set_mpi_disabled()
with mpi.ProcessState("run"):
info: InputDict = load_info_overrides(info_or_yaml_or_file, debug,
stop_at_error, packages_path,
override)
if test:
info["test"] = True
# If any of resume|force given as cmd args, ignore those in the input file
if resume or force:
if resume and force:
raise ValueError("'rename' and 'force' are exclusive options")
info["resume"] = bool(resume)
info["force"] = bool(force)
if info.get("post"):
if isinstance(output, str) or output is False:
info["post"]["output"] = output or None
return post(info)
if isinstance(output, str) or output is False:
info["output"] = output or None
logger_setup(info.get("debug"), info.get("debug_file"))
logger_run = get_logger(run.__name__)
# MARKED FOR DEPRECATION IN v3.0
# BEHAVIOUR TO BE REPLACED BY ERROR:
check_deprecated_modules_path(info)
# END OF DEPRECATION BLOCK
# 1. Prepare output driver, if requested by defining an output_prefix
# GetDist needs to know the original sampler, so don't overwrite if minimizer
try:
which_sampler = list(info["sampler"])[0]
except (KeyError, TypeError):
raise LoggedError(
logger_run,
"You need to specify a sampler using the 'sampler' key "
"as e.g. `sampler: {mcmc: None}.`")
infix = "minimize" if which_sampler == "minimize" else None
with get_output(prefix=info.get("output"),
resume=info.get("resume"),
force=info.get("force"),
infix=infix) as out:
# 2. Update the input info with the defaults for each component
updated_info = update_info(info)
if is_debug(logger_run):
# Dump only if not doing output
# (otherwise, the user can check the .updated file)
if not out and mpi.is_main_process():
logger_run.info(
"Input info updated with defaults (dumped to YAML):\n%s",
yaml_dump(sort_cosmetic(updated_info)))
# 3. If output requested, check compatibility if existing one, and dump.
# 3.1 First: model only
out.check_and_dump_info(info,
updated_info,
cache_old=True,
ignore_blocks=["sampler"])
# 3.2 Then sampler -- 1st get the last sampler mentioned in the updated.yaml
# TODO: ideally, using Minimizer would *append* to the sampler block.
# Some code already in place, but not possible at the moment.
try:
last_sampler = list(updated_info["sampler"])[-1]
last_sampler_info = {
last_sampler: updated_info["sampler"][last_sampler]
}
except (KeyError, TypeError):
raise LoggedError(logger_run, "No sampler requested.")
sampler_name, sampler_class = get_sampler_name_and_class(
last_sampler_info)
check_sampler_info((out.reload_updated_info(use_cache=True)
or {}).get("sampler"),
updated_info["sampler"],
is_resuming=out.is_resuming())
# Dump again, now including sampler info
out.check_and_dump_info(info, updated_info, check_compatible=False)
# Check if resumable run
sampler_class.check_force_resume(
out, info=updated_info["sampler"][sampler_name])
# 4. Initialize the posterior and the sampler
with Model(updated_info["params"],
updated_info["likelihood"],
updated_info.get("prior"),
updated_info.get("theory"),
packages_path=info.get("packages_path"),
timing=updated_info.get("timing"),
allow_renames=False,
stop_at_error=info.get("stop_at_error",
False)) as model:
# Re-dump the updated info, now containing parameter routes and version
updated_info = recursive_update(updated_info, model.info())
out.check_and_dump_info(None,
updated_info,
check_compatible=False)
sampler = sampler_class(
updated_info["sampler"][sampler_name],
model,
out,
name=sampler_name,
packages_path=info.get("packages_path"))
# Re-dump updated info, now also containing updates from the sampler
updated_info["sampler"][sampler_name] = \
recursive_update(updated_info["sampler"][sampler_name],
sampler.info())
out.check_and_dump_info(None,
updated_info,
check_compatible=False)
mpi.sync_processes()
if info.get("test", False):
logger_run.info(
"Test initialization successful! "
"You can probably run now without `--%s`.", "test")
return InfoSamplerTuple(updated_info, sampler)
# Run the sampler
sampler.run()
return InfoSamplerTuple(updated_info, sampler)
