def get_default_info(component_or_class,
kind=None,
return_yaml=False,
yaml_expand_defaults=True,
component_path=None,
input_options=empty_dict,
class_name=None,
return_undefined_annotations=False):
"""
Get default info for a component_or_class.
"""
try:
cls = get_resolved_class(component_or_class, kind, component_path,
class_name)
default_component_info = \
cls.get_defaults(return_yaml=return_yaml,
yaml_expand_defaults=yaml_expand_defaults,
input_options=input_options)
except Exception as e:
raise LoggedError(
log, "Failed to get defaults for component or class '%s' [%s]",
component_or_class, e)
if return_undefined_annotations:
annotations = {
k: v
for k, v in cls.get_annotations().items()
if k not in default_component_info
}
return default_component_info, annotations
else:
return default_component_info
def get_preferred_old_values(info_old):
"""
Returns selected values in `info_old`, which are preferred at resuming.
"""
keep_old: InfoDict = {}
for block_name, block in info_old.items():
if block_name not in kinds or not block:
continue
for k in block:
try:
component_path = block[k].pop("python_path", None) \
if isinstance(block[k], dict) else None
cls = get_resolved_class(k,
kind=block_name,
component_path=component_path,
class_name=(block[k]
or {}).get("class"))
prefer_old_k_this = getattr(cls, "_at_resume_prefer_old", {})
if prefer_old_k_this:
if block_name not in keep_old:
keep_old[block_name] = {}
keep_old[block_name].update({
k: {
o: block[k][o]
for o in prefer_old_k_this if o in block[k]
}
})
except ImportError:
pass
return keep_old
def __init__(self,
info_theory: TheoriesDict,
packages_path=None,
timing=None):
super().__init__()
self.set_logger("theory")
if info_theory:
for name, info in info_theory.items():
info = info or {}
# If it has an "external" key, wrap it up. Else, load it up
if isinstance(info, Theory):
self.add_instance(name, info)
elif isinstance(info.get("external"), Theory):
self.add_instance(name, info["external"])
else:
if "external" in info:
theory_class = info["external"]
if not isinstance(theory_class, type) or \
not issubclass(theory_class, Theory):
raise LoggedError(
self.log, "Theory %s is not a Theory subclass",
name)
else:
theory_class = get_resolved_class(
name, kind="theory", class_name=info.get("class"))
self.add_instance(
name,
theory_class(info,
packages_path=packages_path,
timing=timing,
name=name))
def __init__(self, info_theory, packages_path=None, timing=None):
super().__init__()
self.set_logger("theory")
if info_theory:
for name, info in info_theory.items():
# If it has an "external" key, wrap it up. Else, load it up
if isinstance(info, Theory):
self.add_instance(name, info)
else:
if _external in info:
theory_class = info[_external]
if not inspect.isclass(theory_class) or \
not issubclass(theory_class, Theory):
raise LoggedError(
self.log, "Theory %s is not a Theory subclass",
name)
else:
theory_class = get_resolved_class(
name,
kind=kinds.theory,
class_name=info.get(_class_name))
self.add_instance(
name,
theory_class(info,
packages_path=packages_path,
timing=timing,
name=name))
def get_sampler_name_and_class(info_sampler: SamplersDict):
"""
Auxiliary function to retrieve the class of the required sampler.
"""
check_sane_info_sampler(info_sampler)
name = list(info_sampler)[0]
sampler_class = get_resolved_class(name, kind="sampler")
assert issubclass(sampler_class, Sampler)
return name, sampler_class
def __init__(self,
info_likelihood: LikesDict,
packages_path=None,
timing=None,
theory=None):
super().__init__()
self.set_logger("likelihood")
self.theory = theory
# Get the individual likelihood classes
for name, info in info_likelihood.items():
if isinstance(name, Theory):
name, info = name.get_name(), info
if isinstance(info, Theory):
self.add_instance(name, info)
elif isinstance(info, Mapping) and "external" in info:
external = info["external"]
if isinstance(external, Theory):
self.add_instance(name, external)
elif isinstance(external, type):
if not is_LikelihoodInterface(external) or \
not issubclass(external, Theory):
raise LoggedError(
self.log, "%s: external class likelihood must "
"be a subclass of Theory and have "
"logp, current_logp attributes", external.__name__)
self.add_instance(
name,
external(info,
packages_path=packages_path,
timing=timing,
standalone=False,
name=name))
else:
# If it has an "external" key, wrap it up. Else, load it up
self.add_instance(
name,
LikelihoodExternalFunction(info, name, timing=timing))
else:
assert isinstance(info, Mapping)
like_class: type = get_resolved_class(
name,
kind="likelihood",
component_path=info.get("python_path", None),
class_name=info.get("class"))
self.add_instance(
name,
like_class(info,
packages_path=packages_path,
timing=timing,
standalone=False,
name=name))
if not is_LikelihoodInterface(self[name]):
raise LoggedError(
self.log, "'Likelihood' %s is not actually a "
"likelihood (no current_logp attribute)", name)
def __init__(self, info_likelihood, packages_path=None, timing=None, theory=None):
super().__init__()
self.set_logger("likelihood")
self.theory = theory
# Get the individual likelihood classes
for name, info in info_likelihood.items():
if isinstance(name, Theory):
name, info = name.get_name(), info
if isinstance(info, Theory):
self.add_instance(name, info)
elif _external in info:
if isinstance(info[_external], Theory):
self.add_instance(name, info[_external])
elif inspect.isclass(info[_external]):
if not is_LikelihoodInterface(info[_external]) or \
not issubclass(info[_external], Theory):
raise LoggedError(self.log, "%s: external class likelihood must "
"be a subclass of Theory and have "
"logp, current_logp attributes",
info[_external].__name__)
self.add_instance(name,
info[_external](info, packages_path=packages_path,
timing=timing,
standalone=False,
name=name))
else:
# If it has an "external" key, wrap it up. Else, load it up
self.add_instance(name, LikelihoodExternalFunction(info, name,
timing=timing))
else:
like_class = get_resolved_class(
name, kind=kinds.likelihood,
component_path=info.pop(_component_path, None),
class_name=info.get(_class_name))
self.add_instance(name, like_class(info, packages_path=packages_path,
timing=timing, standalone=False,
name=name))
if not is_LikelihoodInterface(self[name]):
raise LoggedError(self.log, "'Likelihood' %s is not actually a "
"likelihood (no current_logp attribute)",
name)
def is_equal_info(info_old,
info_new,
strict=True,
print_not_log=False,
ignore_blocks=()):
"""
Compares two information dictionaries, and old one versus a new one, and updates the
new one for selected values of the old one.
Set ``strict=False`` (default: ``True``) to ignore options that would not affect
the statistics of a posterior sample, including order of params/priors/likelihoods.
"""
myprint: Callable
myprint_debug: Callable
if print_not_log:
myprint = print
myprint_debug = lambda x: x
else:
myprint = log.info
myprint_debug = log.debug
myname = inspect.stack()[0][3]
ignore = set() if strict else \
{"debug", "debug_file", "resume", "force", packages_path_input, "test", "version"}
ignore = ignore.union(ignore_blocks or [])
if set(info for info in info_old if info_old[info] is not None) - ignore \
!= set(info for info in info_new if info_new[info] is not None) - ignore:
myprint(myname +
": different blocks or options: %r (old) vs %r (new)" %
(set(info_old).difference(ignore),
set(info_new).difference(ignore)))
return False
for block_name in info_old:
if block_name in ignore or block_name not in info_new:
continue
block1 = deepcopy_where_possible(info_old[block_name])
block2 = deepcopy_where_possible(info_new[block_name])
# First, deal with root-level options (force, output, ...)
if not isinstance(block1, dict):
if block1 != block2:
myprint(myname + ": different option '%s'" % block_name)
return False
continue
# Now let's do components and params
# 1. check order (it DOES matter, but just up to 1st level)
f = list if strict else set
if f(block1) != f(block2):
myprint(myname +
": different [%s] or different order of them: %r vs %r" %
(block_name, list(block1), list(block2)))
return False
# 2. Gather general options to be ignored
ignore_k = set()
if not strict:
if block_name in ["theory", "likelihood"]:
ignore_k.update({"input_params", "output_params"})
elif block_name == "params":
for param in block1:
# Unify notation
block1[param] = expand_info_param(block1[param])
block2[param] = expand_info_param(block2[param])
ignore_k.update(
{"latex", "renames", "ref", "proposal", "min", "max"})
# Fixed params, it doesn't matter if they are saved as derived
if "value" in block1[param]:
block1[param].pop("derived", None)
if "value" in block2[param]:
block2[param].pop("derived", None)
# Renames: order does not matter
block1[param]["renames"] = set(block1[param].get(
"renames", []))
block2[param]["renames"] = set(block2[param].get(
"renames", []))
# 3. Now check component/parameters one-by-one
for k in block1:
if not strict:
# Add component-specific options to be ignored
if block_name in kinds:
ignore_k_this = ignore_k.union({"python_path"})
if "external" not in block1[k]:
try:
component_path = block1[k].pop("python_path", None) \
if isinstance(block1[k], dict) else None
cls = get_resolved_class(
k,
kind=block_name,
component_path=component_path,
class_name=(block1[k] or {}).get("class"))
ignore_k_this.update(
set(getattr(cls, "_at_resume_prefer_new", {})))
except ImportError:
pass
# Pop ignored and kept options
for j in ignore_k_this:
block1[k].pop(j, None)
block2[k].pop(j, None)
if block1[k] != block2[k]:
# For clarity, pop common stuff before printing
to_pop = [
j for j in block1[k]
if (block1[k].get(j) == block2[k].get(j))
]
[(block1[k].pop(j, None), block2[k].pop(j, None))
for j in to_pop]
myprint(myname + ": different content of [%s:%s]" %
(block_name, k) +
" -- (re-run with `debug: True` for more info)")
myprint_debug("%r (old) vs %r (new)" % (block1[k], block2[k]))
return False
return True
def install(*infos, **kwargs):
debug = kwargs.get("debug")
# noinspection PyUnresolvedReferences
if not log.root.handlers:
logger_setup(debug=debug)
path = kwargs.get("path")
if not path:
path = resolve_packages_path(infos)
if not path:
raise LoggedError(
log,
"No 'path' argument given, and none could be found in input infos "
"(as %r), the %r env variable or the config file. "
"Maybe specify one via a command line argument '-%s [...]'?",
"packages_path", packages_path_env, packages_path_arg[0])
abspath = os.path.abspath(path)
log.info("Installing external packages at '%s'", abspath)
kwargs_install = {
"force": kwargs.get("force", False),
"no_progress_bars": kwargs.get("no_progress_bars")
}
for what in (code_path, data_path):
kwargs_install[what] = kwargs.get(what, True)
spath = os.path.join(abspath, what)
if kwargs_install[what] and not os.path.exists(spath):
try:
os.makedirs(spath)
except OSError:
raise LoggedError(
log,
"Could not create the desired installation folder '%s'",
spath)
failed_components = []
skip_keywords_arg = set(kwargs.get("skip", []) or [])
# NB: if passed with quotes as `--skip "a b"`, it's interpreted as a single key
skip_keywords_arg = set(
chain(*[word.split() for word in skip_keywords_arg]))
skip_keywords_env = set(
os.environ.get(install_skip_env, "").replace(",", " ").lower().split())
skip_keywords = skip_keywords_arg.union(skip_keywords_env)
used_components, components_infos = get_used_components(*infos,
return_infos=True)
for kind, components in used_components.items():
for component in components:
print()
print(create_banner(kind + ":" + component,
symbol=_banner_symbol,
length=_banner_length),
end="")
print()
if _skip_helper(component.lower(), skip_keywords,
skip_keywords_env, log):
continue
info = components_infos[component]
if isinstance(info, str) or "external" in info:
log.warning(
"Component '%s' is a custom function. "
"Nothing to do.", component)
continue
try:
class_name = (info or {}).get("class")
if class_name:
log.info("Class to be installed for this component: %r",
class_name)
imported_class = get_resolved_class(component,
kind=kind,
component_path=info.pop(
"python_path", None),
class_name=class_name)
except ImportError as excpt:
log.error("Component '%s' not recognized. [%s].", component,
excpt)
failed_components += ["%s:%s" % (kind, component)]
continue
else:
if _skip_helper(imported_class.__name__.lower(), skip_keywords,
skip_keywords_env, log):
continue
is_compatible = getattr(imported_class, "is_compatible",
lambda: True)()
if not is_compatible:
log.info(
"Skipping %r because it is not compatible with your OS.",
component)
continue
log.info("Checking if dependencies have already been installed...")
is_installed = getattr(imported_class, "is_installed", None)
if is_installed is None:
log.info("%s.%s is a fully built-in component: nothing to do.",
kind, imported_class.__name__)
continue
install_path = abspath
get_path = getattr(imported_class, "get_path", None)
if get_path:
install_path = get_path(install_path)
has_been_installed = False
with NoLogging(None if debug else logging.ERROR):
if kwargs.get("skip_global"):
has_been_installed = is_installed(path="global",
**kwargs_install)
if not has_been_installed:
has_been_installed = is_installed(path=install_path,
**kwargs_install)
if has_been_installed:
log.info(
"External dependencies for this component already installed."
)
if kwargs.get("test", False):
continue
if kwargs_install["force"] and not kwargs.get("skip_global"):
log.info("Forcing re-installation, as requested.")
else:
log.info("Doing nothing.")
continue
else:
log.info("Check found no existing installation")
if not debug:
log.info(
"(If you expected this to be already installed, re-run "
"`cobaya-install` with --debug to get more verbose output.)"
)
if kwargs.get("test", False):
continue
log.info("Installing...")
try:
install_this = getattr(imported_class, "install", None)
success = install_this(path=abspath, **kwargs_install)
except KeyboardInterrupt:
raise
except:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
log.error(
"An unknown error occurred. Delete the external packages "
"folder %r and try again. "
"Please, notify the developers if this error persists.",
abspath)
success = False
if success:
log.info("Successfully installed! Let's check it...")
else:
log.error(
"Installation failed! Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve, "
"install the packages required by this component manually."
)
failed_components += ["%s:%s" % (kind, component)]
continue
# test installation
with NoLogging(None if debug else logging.ERROR):
successfully_installed = is_installed(path=install_path,
check=False,
**kwargs_install)
if not successfully_installed:
log.error(
"Installation apparently worked, "
"but the subsequent installation test failed! "
"Look at the error messages above, or re-run with --debug "
"for more more verbose output. "
"Try to solve the issues and try again, or, if you are unable "
"to solve them, install the packages required by this "
"component manually.")
failed_components += ["%s:%s" % (kind, component)]
else:
log.info("Installation check successful.")
print()
print(create_banner(" * Summary * ",
symbol=_banner_symbol,
length=_banner_length),
end="")
print()
if failed_components:
bullet = "\n - "
raise LoggedError(
log,
"The installation (or installation test) of some component(s) has "
"failed: %s\nCheck output of the installer of each component above "
"for precise error info.\n",
bullet + bullet.join(failed_components))
log.info("All requested components' dependencies correctly installed.")
# Set the installation path in the global config file
if not kwargs.get("no_set_global", False) and not kwargs.get(
"test", False):
write_packages_path_in_config_file(abspath)
log.info(
"The installation path has been written into the global config file: %s",
os.path.join(get_config_path(), packages_path_config_file))