def dump_info(self, input_info, full_info):
"""
Saves the info in the chain folder twice:
- the input info.
- idem, populated with the modules' defaults.
If resuming a sample, checks first that old and new infos are consistent.
"""
# trim known params of each likelihood: for internal use only
full_info_trimmed = deepcopy(full_info)
for lik_info in full_info_trimmed.get(_likelihood, {}).values():
if hasattr(lik_info, "pop"):
lik_info.pop(_params, None)
try:
# We will test the old info agains the dumped+loaded new info.
# This is because we can't actually check if python objects are the same as before.
old_info = yaml_load_file(self.file_full)
new_info = yaml_load(yaml_dump(full_info_trimmed))
if not is_equal_info(old_info, new_info, strict=False):
self.log.error(
"Old and new sample information not compatible! "
"Resuming not possible!")
raise HandledException
except IOError:
# There was no previous chain
pass
# We write the new one anyway (maybe updated debug, resuming...)
for f, info in [(self.file_input, input_info),
(self.file_full, full_info_trimmed)]:
with open(f, "w") as f_out:
try:
f_out.write(yaml_dump(info))
except OutputError as e:
self.log.error(e.message)
raise HandledException
def get_default_info(module,
kind=None,
fail_if_not_found=False,
return_yaml=False,
yaml_expand_defaults=True):
"""
Get default info for a module.
"""
try:
if kind is None:
kind = get_kind(module)
cls = get_class(module, kind, None_if_not_found=not fail_if_not_found)
if cls:
default_module_info = cls.get_defaults(
return_yaml=return_yaml,
yaml_expand_defaults=yaml_expand_defaults)
else:
default_module_info = (lambda x: yaml_dump(x)
if return_yaml else x)({
kind: {
module: {}
}
})
except Exception as e:
raise LoggedError(log, "Failed to get defaults for module '%s' [%s]",
("%s:" % kind if kind else "") + module, e)
try:
if not return_yaml:
default_module_info[kind][module]
except KeyError:
raise LoggedError(
log, "The defaults file for '%s' should be structured "
"as %s:%s:{[options]}.", module, kind, module)
return default_module_info
def create_docker_image(filenames, MPI_version=None):
log.info("Creating Docker image...")
if not MPI_version:
MPI_version = "3.2"
# log.warning("You have not specified an MPICH version. "
# "It is strongly encouraged to request the one installed in your cluster,"
# " using '--mpi-version X.Y'. Defaulting to MPICH v%s.", MPI_version)
dc = get_docker_client()
modules = yaml_dump(get_modules(*[load_input(f)
for f in filenames])).strip()
echos_reqs = "RUN " + " && \\ \n ".join([
r'echo "%s" >> %s' % (block, requirements_file_path)
for block in modules.split("\n")
])
echos_help = "RUN " + " && \\ \n ".join([
r'echo "%s" >> %s' % (line, help_file_path)
for line in image_help("docker").split("\n")
])
recipe = r"""
FROM cobaya/base_mpich_%s:latest
%s
RUN cobaya-install %s --%s %s --just-code --force ### NEEDS PYTHON UPDATE! --no-progress-bars
%s
CMD ["cat", "%s"]
""" % (MPI_version, echos_reqs, requirements_file_path, _modules_path_arg,
_modules_path, echos_help, help_file_path)
image_name = "cobaya:" + uuid.uuid4().hex[:6]
with StringIO(recipe) as stream:
dc.images.build(fileobj=stream, tag=image_name)
log.info(
"Docker image '%s' created! "
"Do 'docker save %s | gzip > some_name.tar.gz'"
"to save it to the current folder.", image_name, image_name)
def get_defaults(cls, return_yaml=False, yaml_expand_defaults=True):
"""
Return defaults for this module, with syntax:
.. code::
[kind]
[module_name]:
option: value
[...]
params:
[...] # if required
prior:
[...] # if required
If keyword `return_yaml` is set to True, it returns literally that,
whereas if False (default), it returns the corresponding Python dict.
"""
path_to_defaults = cls.get_yaml_file()
if return_yaml:
if yaml_expand_defaults:
return yaml_dump(yaml_load_file(path_to_defaults))
else:
with open(path_to_defaults, "r") as filedef:
return "".join(filedef.readlines())
else:
return yaml_load_file(path_to_defaults)
def get_model(info):
assert hasattr(info, "keys"), (
"The first argument must be a dictionary with the info needed for the model. "
"If you were trying to pass the name of an input file instead, "
"load it first with 'cobaya.input.load_input', "
"or, if you were passing a yaml string, load it with 'cobaya.yaml.yaml_load'."
)
# Configure the logger ASAP
# Just a dummy import before configuring the logger, until I fix root/individual level
import getdist
logger_setup(info.pop(_debug, _debug_default), info.pop(_debug_file, None))
# Create the full input information, including defaults for each module.
info = deepcopy(info)
ignored_info = {}
for k in list(info):
if k not in [
_params, _likelihood, _prior, _theory, _path_install, _timing
]:
ignored_info.update({k: info.pop(k)})
import logging
if ignored_info:
logging.getLogger(__name__.split(".")[-1]).warn(
"Ignored blocks/options: %r", list(ignored_info))
full_info = get_full_info(info)
if logging.root.getEffectiveLevel() <= logging.DEBUG:
logging.getLogger(__name__.split(".")[-1]).debug(
"Input info updated with defaults (dumped to YAML):\n%s",
yaml_dump(full_info, force_reproducible=False))
# Initialize the posterior and the sampler
return Model(full_info[_params],
full_info[_likelihood],
full_info.get(_prior),
full_info.get(_theory),
modules=info.get(_path_install),
timing=full_info.get(_timing))
def dump_info(self, input_info, updated_info, check_compatible=True):
"""
Saves the info in the chain folder twice:
- the input info.
- idem, populated with the modules' defaults.
If resuming a sample, checks first that old and new infos are consistent.
"""
# trim known params of each likelihood: for internal use only
updated_info_trimmed = deepcopy_where_possible(updated_info)
for lik_info in updated_info_trimmed.get(_likelihood, {}).values():
if hasattr(lik_info, "pop"):
lik_info.pop(_params, None)
if check_compatible:
try:
# We will test the old info against the dumped+loaded new info.
# This is because we can't actually check if python objects do change
old_info = self.reload_updated_info()
new_info = yaml_load(yaml_dump(updated_info_trimmed))
ignore_blocks = []
if list(new_info.get(_sampler, [None]))[0] == "minimize":
ignore_blocks = [_sampler]
if not is_equal_info(old_info, new_info, strict=False,
ignore_blocks=ignore_blocks):
# HACK!!! NEEDS TO BE FIXED
if list(updated_info.get(_sampler, [None]))[0] == "minimize":
raise LoggedError(
self.log, "Old and new sample information not compatible! "
"At this moment it is not possible to 'force' deletion of "
"and old 'minimize' run. Please delete it by hand. "
"We are working on fixing this very soon!")
raise LoggedError(
self.log, "Old and new sample information not compatible! "
"Resuming not possible!")
except IOError:
# There was no previous chain
pass
# We write the new one anyway (maybe updated debug, resuming...)
for f, info in [(self.file_input, input_info),
(self.file_updated, updated_info_trimmed)]:
if not info:
pass
with open(f, "w") as f_out:
try:
f_out.write(yaml_dump(info))
except OutputError as e:
raise LoggedError(self.log, str(e))
def refresh_display(self, info):
QApplication.setOverrideCursor(Qt.WaitCursor)
try:
comments = info.pop(input_database._comment, None)
comments_text = "\n# " + "\n# ".join(comments)
except (
TypeError, # No comments
AttributeError
): # Failed to generate info (returned str instead)
comments_text = ""
self.display["python"].setText("info = " + pformat(info) +
comments_text)
self.display["yaml"].setText(
yaml_dump(sort_cosmetic(info)) + comments_text)
self.display["bibliography"].setText(
prettyprint_bib(*get_bib_info(info)))
# Display covmat
packages_path = resolve_packages_path()
if not packages_path:
self.covmat_text.setText(
"\nIn order to find a covariance matrix, you need to define an external "
"packages installation path, e.g. via the env variable %r.\n" %
_packages_path_env)
elif any(not os.path.isdir(d.format(**{_packages_path: packages_path}))
for d in covmat_folders):
self.covmat_text.setText(
"\nThe external cosmological packages appear not to be installed where "
"expected: %r\n" % packages_path)
else:
covmat_data = get_best_covmat(info, packages_path=packages_path)
self.current_params_in_covmat = covmat_data["params"]
self.current_covmat = covmat_data["covmat"]
_, corrmat = cov_to_std_and_corr(self.current_covmat)
self.covmat_text.setText(
"\nCovariance file: %r\n\n"
"NB: you do *not* need to save or copy this covariance matrix: "
"it will be selected automatically.\n" % covmat_data["name"])
self.covmat_table.setRowCount(len(self.current_params_in_covmat))
self.covmat_table.setColumnCount(len(
self.current_params_in_covmat))
self.covmat_table.setHorizontalHeaderLabels(
list(self.current_params_in_covmat))
self.covmat_table.setVerticalHeaderLabels(
list(self.current_params_in_covmat))
for i, pi in enumerate(self.current_params_in_covmat):
for j, pj in enumerate(self.current_params_in_covmat):
self.covmat_table.setItem(
i, j,
QTableWidgetItem("%g" % self.current_covmat[i, j]))
if i != j:
color = [
256 * c
for c in cmap_corr(corrmat[i, j] / 2 + 0.5)[:3]
]
else:
color = [255.99] * 3
self.covmat_table.item(i, j).setBackground(QColor(*color))
self.covmat_table.item(i, j).setForeground(QColor(0, 0, 0))
QApplication.restoreOverrideCursor()
def run(info):
assert hasattr(info, "keys"), (
"The first argument must be a dictionary with the info needed for the run. "
"If you were trying to pass the name of an input file instead, "
"load it first with 'cobaya.input.load_input', "
"or, if you were passing a yaml string, load it with 'cobaya.yaml.yaml_load'."
)
# Configure the logger ASAP
# Just a dummy import before configuring the logger, until I fix root/individual level
import getdist
logger_setup(info.get(_debug), info.get(_debug_file))
import logging
# Initialize output, if required
output = Output(output_prefix=info.get(_output_prefix),
resume=info.get(_resume),
force_output=info.pop(_force, None))
# Create the full input information, including defaults for each module.
full_info = get_full_info(info)
if output:
full_info[_output_prefix] = output.updated_output_prefix()
full_info[_resume] = output.is_resuming()
if logging.root.getEffectiveLevel() <= logging.DEBUG:
# Don't dump unless we are doing output, just in case something not serializable
# May be fixed in the future if we find a way to serialize external functions
if info.get(_output_prefix) and am_single_or_primary_process():
logging.getLogger(__name__.split(".")[-1]).info(
"Input info updated with defaults (dumped to YAML):\n%s",
yaml_dump(full_info))
# TO BE DEPRECATED IN >1.2!!! #####################
_force_reproducible = "force_reproducible"
if _force_reproducible in info:
info.pop(_force_reproducible)
logging.getLogger(__name__.split(".")[-1]).warn(
"Option '%s' is no longer necessary. Please remove it!" %
_force_reproducible)
# CHECK THAT THIS WARNING WORKS!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
###################################################
# We dump the info now, before modules initialization, lest it is accidentally modified
# If resuming a sample, it checks that old and new infos are consistent
output.dump_info(info, full_info)
# Initialize the posterior and the sampler
with Model(full_info[_params],
full_info[_likelihood],
full_info.get(_prior),
full_info.get(_theory),
modules=info.get(_path_install),
timing=full_info.get(_timing),
allow_renames=False) as model:
with Sampler(full_info[_sampler],
model,
output,
resume=full_info.get(_resume),
modules=info.get(_path_install)) as sampler:
sampler.run()
# For scripted calls
return deepcopy(full_info), sampler.products()
def refresh_display(self, info):
try:
comments = info.pop(input_database._comment, None)
comments_text = "\n# " + "\n# ".join(comments)
except (TypeError, # No comments
AttributeError): # Failed to generate info (returned str instead)
comments_text = ""
self.display["python"].setText(
"from collections import OrderedDict\n\ninfo = " +
pformat(info) + comments_text)
self.display["yaml"].setText(yaml_dump(info) + comments_text)
self.display["citations"].setText(prettyprint_citation(citation(info)))
def dump_info(self, input_info, full_info):
"""
Saves the info in the chain folder twice:
- the input info.
- idem, populated with the modules' defaults.
"""
for f, info in [(self.file_input, input_info),
(self.file_full, full_info)]:
with open(f, "w") as f_out:
f_out.write(
yaml_dump(info,
default_flow_style=False,
trim_params_info=True,
force_reproducible=self.force_reproducible))
def create_singularity_image(filenames, MPI_version=None):
log.info("Creating Singularity image...")
if not MPI_version:
MPI_version = "2.1.1"
# log.warning("You have not specified an OpenMPI version. "
# "It is strongly encouraged to request the one installed in your cluster,"
# " using '--mpi-version X.Y.Z'. Defaulting to OpenMPI v%s.", MPI_version)
components = yaml_dump(
get_used_components(*[load_input(f) for f in filenames])).strip()
echos_reqs = "\n " + "\n ".join([""] + [
'echo "%s" >> %s' % (block, requirements_file_path)
for block in components.split("\n")
])
recipe = (
dedent("""
Bootstrap: docker
From: cobaya/base_openmpi_%s:latest\n
%%post\n""" % MPI_version) + dedent(echos_reqs) + dedent("""
export CONTAINED=TRUE
cobaya-install %s --%s %s --just-code --force ### --no-progress-bars
mkdir $COBAYA_PRODUCTS
%%help
%s
""" % (
requirements_file_path,
# TODO: this looks wrong?
packages_path_input,
os.path.join(packages_path_arg, packages_path_input, data_path),
"\n ".join(image_help("singularity").split("\n")[1:]))))
with NamedTemporaryFile(delete=False) as recipe_file:
recipe_file.write(recipe.encode('utf-8'))
recipe_file_name = recipe_file.name
image_name = "cobaya_" + uuid.uuid4().hex[:6] + ".simg"
process_build = Popen(
["singularity", "build", image_name, recipe_file_name],
stdout=PIPE,
stderr=PIPE)
out, err = process_build.communicate()
if process_build.returncode:
log.info(out)
log.info(err)
raise LoggedError(log,
"Image creation failed! See error message above.")
log.info("Singularity image '%s' created!", image_name)
def get_sampler(info_sampler: SamplersDict,
model: Model,
output: Optional[Output] = None,
packages_path: Optional[str] = None) -> 'Sampler':
assert isinstance(info_sampler, Mapping), (
"The first argument must be a dictionary with the info needed for the sampler. "
"If you were trying to pass the name of an input file instead, "
"load it first with 'cobaya.input.load_input', "
"or, if you were passing a yaml string, load it with 'cobaya.yaml.yaml_load'."
)
logger_sampler = get_logger(__name__)
info_sampler = deepcopy_where_possible(info_sampler)
if output is None:
output = OutputDummy()
# Check and update info
check_sane_info_sampler(info_sampler)
updated_info_sampler = update_info({"sampler": info_sampler
})["sampler"] # type: ignore
if is_debug(logger_sampler):
logger_sampler.debug(
"Input info updated with defaults (dumped to YAML):\n%s",
yaml_dump(updated_info_sampler))
# Get sampler class & check resume/force compatibility
sampler_name, sampler_class = get_sampler_name_and_class(
updated_info_sampler, logger=logger_sampler)
check_sampler_info((output.reload_updated_info(use_cache=True)
or {}).get("sampler"),
updated_info_sampler,
is_resuming=output.is_resuming())
# Check if resumable run
sampler_class.check_force_resume(output,
info=updated_info_sampler[sampler_name])
# Instantiate the sampler
sampler_instance = sampler_class(updated_info_sampler[sampler_name],
model,
output,
packages_path=packages_path)
# If output, dump updated
if output:
to_dump = model.info()
to_dump["sampler"] = {sampler_name: sampler_instance.info()}
to_dump["output"] = os.path.join(output.folder, output.prefix)
output.check_and_dump_info(None, to_dump, check_compatible=False)
return sampler_instance
def test_cosmo_docs_basic():
flag = True
for theo in ["camb", "classy"]:
info_new = create_input(preset=preset_pre + theo)
info_yaml_new = yaml_dump(info_new)
file_path = os.path.join(path, file_pre + theo + ".yaml")
with open(file_path) as docs_file:
info_yaml_docs = "".join(docs_file.readlines())
info_docs = yaml_load(info_yaml_docs)
if not is_equal_info(
info_new, info_docs, strict=True, print_not_log=True):
with open(file_path, "w") as docs_file:
docs_file.write(info_yaml_new)
flag = False
print("OLD:\n%s" % info_yaml_docs)
print("----------------------------------------")
print("NEW:\n%s" % info_yaml_new)
assert flag, ("Differences in example input file. "
"Files have been re-generated; check out your git diff.")
def get_sampler(info_sampler, model, output=None, packages_path=None):
assert isinstance(info_sampler, Mapping), (
"The first argument must be a dictionary with the info needed for the sampler. "
"If you were trying to pass the name of an input file instead, "
"load it first with 'cobaya.input.load_input', "
"or, if you were passing a yaml string, load it with 'cobaya.yaml.yaml_load'."
)
logger_sampler = logging.getLogger(__name__.split(".")[-1])
info_sampler = deepcopy_where_possible(info_sampler)
if output is None:
output = OutputDummy()
# Check and update info
check_sane_info_sampler(info_sampler)
updated_info_sampler = update_info({kinds.sampler:
info_sampler})[kinds.sampler]
if logging.root.getEffectiveLevel() <= logging.DEBUG:
logger_sampler.debug(
"Input info updated with defaults (dumped to YAML):\n%s",
yaml_dump(updated_info_sampler))
# Get sampler class & check resume/force compatibility
sampler_name, sampler_class = get_sampler_name_and_class(
updated_info_sampler)
check_sampler_info((output.reload_updated_info(use_cache=True)
or {}).get(kinds.sampler),
updated_info_sampler,
is_resuming=output.is_resuming())
# Check if resumable run
sampler_class.check_force_resume(output,
info=updated_info_sampler[sampler_name])
# Instantiate the sampler
sampler_instance = sampler_class(updated_info_sampler[sampler_name],
model,
output,
packages_path=packages_path)
# If output, dump updated
if output:
to_dump = model.info()
to_dump[kinds.sampler] = {sampler_name: sampler_instance.info()}
to_dump[_output_prefix] = os.path.join(output.folder, output.prefix)
output.check_and_dump_info(None, to_dump, check_compatible=False)
return sampler_instance
def get_defaults(cls,
return_yaml=False,
yaml_expand_defaults=True,
input_options=empty_dict):
"""
Return defaults for this component_or_class, with syntax:
.. code::
option: value
[...]
params:
[...] # if required
prior:
[...] # if required
If keyword `return_yaml` is set to True, it returns literally that,
whereas if False (default), it returns the corresponding Python dict.
Note that in external components installed as zip_safe=True packages files cannot
be accessed directly.
In this case using !default .yaml includes currently does not work.
Also note that if you return a dictionary it may be modified (return a deep copy
if you want to keep it).
if yaml_expand_defaults then !default: file includes will be expanded
input_options may be a dictionary of input options, e.g. in case default params
are dynamically dependent on an input variable
"""
if 'class_options' in cls.__dict__:
raise LoggedError(
log, "class_options (in %s) should now be replaced by "
"public attributes defined directly in the class" %
cls.get_qualified_class_name())
yaml_text = cls.get_associated_file_content('.yaml')
options = cls.get_class_options(input_options=input_options)
if options and yaml_text:
raise LoggedError(
log, "%s: any class can either have .yaml or class variables "
"but not both (type declarations without values are fine "
"also with yaml file). You have class attributes: %s",
cls.get_qualified_class_name(), list(options))
if return_yaml and not yaml_expand_defaults:
return yaml_text or ""
this_defaults = yaml_load_file(cls.get_yaml_file(), yaml_text) \
if yaml_text else deepcopy_where_possible(options)
# start with this one to keep the order such that most recent class options
# near the top. Update below to actually override parameters with these.
defaults = this_defaults.copy()
if not return_yaml:
for base in cls.__bases__:
if issubclass(base, HasDefaults) and base is not HasDefaults:
defaults.update(
base.get_defaults(input_options=input_options))
defaults.update(this_defaults)
if return_yaml:
return yaml_dump(defaults)
else:
return defaults
def run(
info_or_yaml_or_file: Union[InputDict, str, os.PathLike],
packages_path: Optional[str] = None,
output: Union[str, LiteralFalse, None] = None,
debug: Union[bool, int, None] = None,
stop_at_error: Optional[bool] = None,
resume: bool = False,
force: bool = False,
no_mpi: bool = False,
test: bool = False,
override: Optional[InputDict] = None,
) -> Union[InfoSamplerTuple, PostTuple]:
"""
Run from an input dictionary, file name or yaml string, with optional arguments
to override settings in the input as needed.
:param info_or_yaml_or_file: input options dictionary, yaml file, or yaml text
:param packages_path: path where external packages were installed
:param output: path name prefix for output files, or False for no file output
:param debug: true for verbose debug output, or a specific logging level
:param stop_at_error: stop if an error is raised
:param resume: continue an existing run
:param force: overwrite existing output if it exists
:param no_mpi: run without MPI
:param test: only test initialization rather than actually running
:param override: option dictionary to merge into the input one, overriding settings
(but with lower precedence than the explicit keyword arguments)
:return: (updated_info, sampler) tuple of options dictionary and Sampler instance,
or (updated_info, results) if using "post" post-processing
"""
# This function reproduces the model-->output-->sampler pipeline one would follow
# when instantiating by hand, but alters the order to performs checks and dump info
# as early as possible, e.g. to check if resuming possible or `force` needed.
if no_mpi or test:
mpi.set_mpi_disabled()
with mpi.ProcessState("run"):
info: InputDict = load_info_overrides(info_or_yaml_or_file, debug,
stop_at_error, packages_path,
override)
if test:
info["test"] = True
# If any of resume|force given as cmd args, ignore those in the input file
if resume or force:
if resume and force:
raise ValueError("'rename' and 'force' are exclusive options")
info["resume"] = bool(resume)
info["force"] = bool(force)
if info.get("post"):
if isinstance(output, str) or output is False:
info["post"]["output"] = output or None
return post(info)
if isinstance(output, str) or output is False:
info["output"] = output or None
logger_setup(info.get("debug"), info.get("debug_file"))
logger_run = get_logger(run.__name__)
# MARKED FOR DEPRECATION IN v3.0
# BEHAVIOUR TO BE REPLACED BY ERROR:
check_deprecated_modules_path(info)
# END OF DEPRECATION BLOCK
# 1. Prepare output driver, if requested by defining an output_prefix
# GetDist needs to know the original sampler, so don't overwrite if minimizer
try:
which_sampler = list(info["sampler"])[0]
except (KeyError, TypeError):
raise LoggedError(
logger_run,
"You need to specify a sampler using the 'sampler' key "
"as e.g. `sampler: {mcmc: None}.`")
infix = "minimize" if which_sampler == "minimize" else None
with get_output(prefix=info.get("output"),
resume=info.get("resume"),
force=info.get("force"),
infix=infix) as out:
# 2. Update the input info with the defaults for each component
updated_info = update_info(info)
if is_debug(logger_run):
# Dump only if not doing output
# (otherwise, the user can check the .updated file)
if not out and mpi.is_main_process():
logger_run.info(
"Input info updated with defaults (dumped to YAML):\n%s",
yaml_dump(sort_cosmetic(updated_info)))
# 3. If output requested, check compatibility if existing one, and dump.
# 3.1 First: model only
out.check_and_dump_info(info,
updated_info,
cache_old=True,
ignore_blocks=["sampler"])
# 3.2 Then sampler -- 1st get the last sampler mentioned in the updated.yaml
# TODO: ideally, using Minimizer would *append* to the sampler block.
# Some code already in place, but not possible at the moment.
try:
last_sampler = list(updated_info["sampler"])[-1]
last_sampler_info = {
last_sampler: updated_info["sampler"][last_sampler]
}
except (KeyError, TypeError):
raise LoggedError(logger_run, "No sampler requested.")
sampler_name, sampler_class = get_sampler_name_and_class(
last_sampler_info)
check_sampler_info((out.reload_updated_info(use_cache=True)
or {}).get("sampler"),
updated_info["sampler"],
is_resuming=out.is_resuming())
# Dump again, now including sampler info
out.check_and_dump_info(info, updated_info, check_compatible=False)
# Check if resumable run
sampler_class.check_force_resume(
out, info=updated_info["sampler"][sampler_name])
# 4. Initialize the posterior and the sampler
with Model(updated_info["params"],
updated_info["likelihood"],
updated_info.get("prior"),
updated_info.get("theory"),
packages_path=info.get("packages_path"),
timing=updated_info.get("timing"),
allow_renames=False,
stop_at_error=info.get("stop_at_error",
False)) as model:
# Re-dump the updated info, now containing parameter routes and version
updated_info = recursive_update(updated_info, model.info())
out.check_and_dump_info(None,
updated_info,
check_compatible=False)
sampler = sampler_class(
updated_info["sampler"][sampler_name],
model,
out,
name=sampler_name,
packages_path=info.get("packages_path"))
# Re-dump updated info, now also containing updates from the sampler
updated_info["sampler"][sampler_name] = \
recursive_update(updated_info["sampler"][sampler_name],
sampler.info())
out.check_and_dump_info(None,
updated_info,
check_compatible=False)
mpi.sync_processes()
if info.get("test", False):
logger_run.info(
"Test initialization successful! "
"You can probably run now without `--%s`.", "test")
return InfoSamplerTuple(updated_info, sampler)
# Run the sampler
sampler.run()
return InfoSamplerTuple(updated_info, sampler)
def run(info):
# This function reproduces the model-->output-->sampler pipeline one would follow
# when instantiating by hand, but alters the order to performs checks and dump info
# as early as possible, e.g. to check if resuming possible or `force` needed.
assert isinstance(info, Mapping), (
"The first argument must be a dictionary with the info needed for the run. "
"If you were trying to pass the name of an input file instead, "
"load it first with 'cobaya.input.load_input', "
"or, if you were passing a yaml string, load it with 'cobaya.yaml.yaml_load'."
)
logger_setup(info.get(_debug), info.get(_debug_file))
logger_run = logging.getLogger(__name__.split(".")[-1])
# MARKED FOR DEPRECATION IN v3.0
# BEHAVIOUR TO BE REPLACED BY ERROR:
check_deprecated_modules_path(info)
# END OF DEPRECATION BLOCK
# 1. Prepare output driver, if requested by defining an output_prefix
output = get_output(output_prefix=info.get(_output_prefix),
resume=info.get(_resume),
force=info.get(_force))
# 2. Update the input info with the defaults for each component
updated_info = update_info(info)
if logging.root.getEffectiveLevel() <= logging.DEBUG:
# Dump only if not doing output (otherwise, the user can check the .updated file)
if not output and is_main_process():
logger_run.info(
"Input info updated with defaults (dumped to YAML):\n%s",
yaml_dump(sort_cosmetic(updated_info)))
# 3. If output requested, check compatibility if existing one, and dump.
# 3.1 First: model only
output.check_and_dump_info(info,
updated_info,
cache_old=True,
ignore_blocks=[kinds.sampler])
# 3.2 Then sampler -- 1st get the last sampler mentioned in the updated.yaml
# TODO: ideally, using Minimizer would *append* to the sampler block.
# Some code already in place, but not possible at the moment.
try:
last_sampler = list(updated_info[kinds.sampler])[-1]
last_sampler_info = {
last_sampler: updated_info[kinds.sampler][last_sampler]
}
except (KeyError, TypeError):
raise LoggedError(logger_run, "No sampler requested.")
sampler_name, sampler_class = get_sampler_name_and_class(last_sampler_info)
check_sampler_info((output.reload_updated_info(use_cache=True)
or {}).get(kinds.sampler),
updated_info[kinds.sampler],
is_resuming=output.is_resuming())
# Dump again, now including sampler info
output.check_and_dump_info(info, updated_info, check_compatible=False)
# Check if resumable run
sampler_class.check_force_resume(
output, info=updated_info[kinds.sampler][sampler_name])
# 4. Initialize the posterior and the sampler
with Model(updated_info[_params], updated_info[kinds.likelihood],
updated_info.get(_prior), updated_info.get(kinds.theory),
packages_path=info.get(_packages_path), timing=updated_info.get(_timing),
allow_renames=False, stop_at_error=info.get("stop_at_error", False)) \
as model:
# Re-dump the updated info, now containing parameter routes and version info
updated_info = recursive_update(updated_info, model.info())
output.check_and_dump_info(None, updated_info, check_compatible=False)
sampler = sampler_class(updated_info[kinds.sampler][sampler_name],
model,
output,
packages_path=info.get(_packages_path))
# Re-dump updated info, now also containing updates from the sampler
updated_info[kinds.sampler][sampler.get_name()] = \
recursive_update(
updated_info[kinds.sampler][sampler.get_name()], sampler.info())
# TODO -- maybe also re-dump model info, now possibly with measured speeds
# (waiting until the camb.transfers issue is solved)
output.check_and_dump_info(None, updated_info, check_compatible=False)
if info.get(_test_run, False):
logger_run.info(
"Test initialization successful! "
"You can probably run now without `--%s`.", _test_run)
return updated_info, sampler
# Run the sampler
sampler.run()
return updated_info, sampler
def check_and_dump_info(self, input_info, updated_info, check_compatible=True,
cache_old=False, use_cache_old=False, ignore_blocks=()):
"""
Saves the info in the chain folder twice:
- the input info.
- idem, populated with the components' defaults.
If resuming a sample, checks first that old and new infos and versions are
consistent.
"""
# trim known params of each likelihood: for internal use only
self.check_lock()
updated_info_trimmed = deepcopy_where_possible(updated_info)
updated_info_trimmed["version"] = get_version()
for like_info in updated_info_trimmed.get("likelihood", {}).values():
(like_info or {}).pop("params", None)
if check_compatible:
# We will test the old info against the dumped+loaded new info.
# This is because we can't actually check if python objects do change
try:
old_info = self.reload_updated_info(cache=cache_old,
use_cache=use_cache_old)
except InputImportError:
# for example, when there's a dynamically generated class that cannot
# be found by the yaml loader (could use yaml loader that ignores them)
old_info = None
if old_info:
# use consistent yaml read-in types
# TODO: could probably just compare full infos here, with externals?
# for the moment cautiously keeping old behaviour
old_info = yaml_load(yaml_dump(old_info)) # type: ignore
new_info = yaml_load(yaml_dump(updated_info_trimmed))
if not is_equal_info(old_info, new_info, strict=False,
ignore_blocks=list(ignore_blocks) + [
"output"]):
raise LoggedError(
self.log, "Old and new run information not compatible! "
"Resuming not possible!")
# Deal with version comparison separately:
# - If not specified now, take the one used in resume info
# - If specified both now and before, check new older than old one
# (For Cobaya's own version, prefer new one always)
old_version = old_info.get("version")
new_version = new_info.get("version")
if isinstance(old_version, str) and isinstance(new_version, str):
if version.parse(old_version) > version.parse(new_version):
raise LoggedError(
self.log, "You are trying to resume a run performed with a "
"newer version of Cobaya: %r (you are using %r). "
"Please, update your Cobaya installation.",
old_version, new_version)
for k in set(kinds).intersection(updated_info):
if k in ignore_blocks or updated_info[k] is None:
continue
for c in updated_info[k]:
new_version = updated_info[k][c].get("version")
old_version = old_info[k][c].get("version") # type: ignore
if new_version is None:
updated_info[k][c]["version"] = old_version
updated_info_trimmed[k][c]["version"] = old_version
elif old_version is not None:
cls = get_resolved_class(
c, k, None_if_not_found=True,
class_name=updated_info[k][c].get("class"))
if cls and cls.compare_versions(
old_version, new_version, equal=False):
raise LoggedError(
self.log, "You have requested version %r for "
"%s:%s, but you are trying to resume a "
"run that used a newer version: %r.",
new_version, k, c, old_version)
# If resuming, we don't want to to *partial* dumps
if ignore_blocks and self.is_resuming():
return
# Work on a copy of the input info, since we are updating the prefix
# (the updated one is already a copy)
if input_info is not None:
input_info = deepcopy_where_possible(input_info)
# Write the new one
for f, info in [(self.file_input, input_info),
(self.file_updated, updated_info_trimmed)]:
if info:
for k in ignore_blocks:
info.pop(k, None)
info.pop("debug", None)
info.pop("force", None)
info.pop("resume", None)
# make sure the dumped output_prefix does only contain the file prefix,
# not the folder, since it's already been placed inside it
info["output"] = self.updated_prefix()
with open(f, "w", encoding="utf-8") as f_out:
try:
f_out.write(yaml_dump(sort_cosmetic(info)))
except OutputError as e:
raise LoggedError(self.log, str(e))
if updated_info_trimmed and has_non_yaml_reproducible(updated_info_trimmed):
try:
import dill
except ImportError:
self.mpi_info('Install "dill" to save reproducible options file.')
else:
import pickle
try:
with open(self.dump_file_updated, 'wb') as f:
dill.dump(sort_cosmetic(updated_info_trimmed), f,
pickle.HIGHEST_PROTOCOL)
except pickle.PicklingError as e:
os.remove(self.dump_file_updated)
self.mpi_info('Options file cannot be pickled %s', e)
def check_and_dump_info(self,
input_info,
updated_info,
check_compatible=True,
cache_old=False,
use_cache_old=False,
ignore_blocks=()):
"""
Saves the info in the chain folder twice:
- the input info.
- idem, populated with the components' defaults.
If resuming a sample, checks first that old and new infos and versions are
consistent.
"""
# trim known params of each likelihood: for internal use only
updated_info_trimmed = deepcopy_where_possible(updated_info)
updated_info_trimmed[_version] = __version__
for like_info in updated_info_trimmed.get(kinds.likelihood,
{}).values():
(like_info or {}).pop(_params, None)
if check_compatible:
# We will test the old info against the dumped+loaded new info.
# This is because we can't actually check if python objects do change
old_info = self.reload_updated_info(cache=cache_old,
use_cache=use_cache_old)
if old_info:
new_info = yaml_load(yaml_dump(updated_info_trimmed))
if not is_equal_info(
old_info,
new_info,
strict=False,
ignore_blocks=list(ignore_blocks) + [_output_prefix]):
raise LoggedError(
self.log,
"Old and new run information not compatible! "
"Resuming not possible!")
# Deal with version comparison separately:
# - If not specified now, take the one used in resume info
# - If specified both now and before, check new older than old one
# (For Cobaya's own version, prefer new one always)
old_version = old_info.get(_version, None)
new_version = new_info.get(_version, None)
if old_version:
if version.parse(old_version) > version.parse(new_version):
raise LoggedError(
self.log,
"You are trying to resume a run performed with a "
"newer version of Cobaya: %r (you are using %r). "
"Please, update your Cobaya installation.",
old_version, new_version)
for k in (kind for kind in kinds if kind in updated_info):
if k in ignore_blocks:
continue
for c in updated_info[k]:
new_version = updated_info[k][c].get(_version)
old_version = old_info[k][c].get(_version)
if new_version is None:
updated_info[k][c][_version] = old_version
updated_info_trimmed[k][c][_version] = old_version
elif old_version is not None:
cls = get_class(c, k, None_if_not_found=True)
if cls and cls.compare_versions(
old_version, new_version, equal=False):
raise LoggedError(
self.log,
"You have requested version %r for "
"%s:%s, but you are trying to resume a "
"run that used a newer version: %r.",
new_version, k, c, old_version)
# If resuming, we don't want to to *partial* dumps
if ignore_blocks and self.is_resuming():
return
# Work on a copy of the input info, since we are updating the prefix
# (the updated one is already a copy)
if input_info is not None:
input_info = deepcopy_where_possible(input_info)
# Write the new one
for f, info in [(self.file_input, input_info),
(self.file_updated, updated_info_trimmed)]:
if info:
for k in ignore_blocks:
info.pop(k, None)
info.pop(_debug, None)
info.pop(_force, None)
info.pop(_resume, None)
# make sure the dumped output_prefix does only contain the file prefix,
# not the folder, since it's already been placed inside it
info[_output_prefix] = self.updated_prefix()
with open(f, "w", encoding="utf-8") as f_out:
try:
f_out.write(yaml_dump(sort_cosmetic(info)))
except OutputError as e:
raise LoggedError(self.log, str(e))
def refresh_display(self, info):
self.display["python"].setText(
"from collections import OrderedDict\n\ninfo = " + pformat(info))
self.display["yaml"].setText(yaml_dump(info))
def run(info):
assert hasattr(info, "keys"), (
"The first argument must be a dictionary with the info needed for the run. "
"If you were trying to pass the name of an input file instead, "
"load it first with 'cobaya.input.load_input', "
"or, if you were passing a yaml string, load it with 'cobaya.yaml.yaml_load'."
)
# Configure the logger ASAP
# Just a dummy import before configuring the logger, until I fix root/individual level
import getdist
logger_setup(info.get(_debug), info.get(_debug_file))
import logging
# Initialize output, if required
resume, force = info.get(_resume), info.get(_force)
ignore_blocks = []
# If minimizer, always try to re-use sample to get bestfit/covmat
if list(info[_sampler])[0] == "minimize":
resume = True
force = False
output = Output(output_prefix=info.get(_output_prefix),
resume=resume,
force_output=force)
# Create the updated input information, including defaults for each module.
updated_info = update_info(info)
if output:
updated_info[_output_prefix] = output.updated_output_prefix()
updated_info[_resume] = output.is_resuming()
if logging.root.getEffectiveLevel() <= logging.DEBUG:
# Don't dump unless we are doing output, just in case something not serializable
# May be fixed in the future if we find a way to serialize external functions
if info.get(_output_prefix) and am_single_or_primary_process():
logging.getLogger(__name__.split(".")[-1]).info(
"Input info updated with defaults (dumped to YAML):\n%s",
yaml_dump(updated_info))
# TO BE DEPRECATED IN >1.2!!! #####################
_force_reproducible = "force_reproducible"
if _force_reproducible in info:
info.pop(_force_reproducible)
logging.getLogger(__name__.split(".")[-1]).warning(
"Option '%s' is no longer necessary. Please remove it!" %
_force_reproducible)
# CHECK THAT THIS WARNING WORKS!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
###################################################
# We dump the info now, before modules initialization, to better track errors and
# to check if resuming is possible asap (old and new infos are consistent)
output.dump_info(info, updated_info)
# Initialize the posterior and the sampler
with Model(updated_info[_params],
updated_info[_likelihood],
updated_info.get(_prior),
updated_info.get(_theory),
modules=info.get(_path_install),
timing=updated_info.get(_timing),
allow_renames=False) as model:
# Update the updated info with the parameter routes
keys = ([_likelihood, _theory]
if _theory in updated_info else [_likelihood])
updated_info.update(odict([[k, model.info()[k]] for k in keys]))
output.dump_info(None, updated_info, check_compatible=False)
with Sampler(updated_info[_sampler],
model,
output,
resume=updated_info.get(_resume),
modules=info.get(_path_install)) as sampler:
sampler.run()
# For scripted calls:
# Restore the original output_prefix: the script has not changed folder!
if _output_prefix in info:
updated_info[_output_prefix] = info.get(_output_prefix)
return updated_info, sampler.products()
def run(info):
assert hasattr(info, "items"), (
"The agument of `run` must be a dictionary with the info needed for the run. "
"If you were trying to pass an input file instead, load it first with "
"`cobaya.input.load_input`.")
# Import names
from cobaya.conventions import _likelihood, _prior, _params
from cobaya.conventions import _theory, _sampler, _path_install
from cobaya.conventions import _debug, _debug_file, _output_prefix
# Configure the logger ASAP
from cobaya.log import logger_setup
logger_setup(info.get(_debug), info.get(_debug_file))
# Debug (lazy call)
import logging
if logging.root.getEffectiveLevel() <= logging.DEBUG:
# Don't dump unless we are doing output, just in case something not serializable
# May be fixed in the future if we find a way to serialize external functions
if info.get(_output_prefix):
from cobaya.yaml import yaml_dump
logging.getLogger(__name__.split(".")[-1]).debug(
"Input info (dumped to YAML):\n%s", yaml_dump(info))
# Import general classes
from cobaya.prior import Prior
from cobaya.sampler import get_Sampler as Sampler
# Import the functions and classes that need MPI wrapping
from cobaya.mpi import import_MPI
# Likelihood = import_MPI(".likelihood", "LikelihoodCollection")
from cobaya.likelihood import LikelihoodCollection as Likelihood
# Initialise output, if requiered
do_output = info.get(_output_prefix)
if do_output:
Output = import_MPI(".output", "Output")
output = Output(info)
else:
from cobaya.output import Output_dummy
output = Output_dummy(info)
# Create the full input information, including defaults for each module.
from cobaya.input import get_full_info
full_info = get_full_info(info)
if logging.root.getEffectiveLevel() <= logging.DEBUG:
# Don't dump unless we are doing output, just in case something not serializable
# May be fixed in the future if we find a way to serialize external functions
if info.get(_output_prefix):
logging.getLogger(__name__.split(".")[-1]).debug(
"Updated info (dumped to YAML):\n%s", yaml_dump(full_info))
# We dump the info now, before modules initialization, lest it is accidentaly modified
output.dump_info(info, full_info)
# Set the path of the installed modules, if given
from cobaya.tools import set_path_to_installation
set_path_to_installation(info.get(_path_install))
# Initialise parametrization, likelihoods and prior
from cobaya.parametrization import Parametrization
with Parametrization(full_info[_params]) as par:
with Prior(par, full_info.get(_prior)) as prior:
with Likelihood(full_info[_likelihood], par,
full_info.get(_theory)) as lik:
with Sampler(full_info[_sampler], par, prior, lik,
output) as sampler:
sampler.run()
# For scripted calls
return deepcopy(full_info), sampler.products()