def __init__(self, info_theory, packages_path=None, timing=None):
super().__init__()
self.set_logger("theory")
if info_theory:
for name, info in info_theory.items():
# If it has an "external" key, wrap it up. Else, load it up
if isinstance(info, Theory):
self.add_instance(name, info)
else:
if _external in info:
theory_class = info[_external]
if not inspect.isclass(theory_class) or \
not issubclass(theory_class, Theory):
raise LoggedError(
self.log, "Theory %s is not a Theory subclass",
name)
else:
theory_class = get_class(info.get(_class_name) or name,
kind=kinds.theory)
self.add_instance(
name,
theory_class(info,
packages_path=packages_path,
timing=timing,
name=name))
def get_default_info(module,
kind=None,
fail_if_not_found=False,
return_yaml=False,
yaml_expand_defaults=True):
"""
Get default info for a module.
"""
try:
if kind is None:
kind = get_kind(module)
cls = get_class(module, kind, None_if_not_found=not fail_if_not_found)
if cls:
default_module_info = cls.get_defaults(
return_yaml=return_yaml,
yaml_expand_defaults=yaml_expand_defaults)
else:
default_module_info = (lambda x: yaml_dump(x)
if return_yaml else x)({
kind: {
module: {}
}
})
except Exception as e:
raise LoggedError(log, "Failed to get defaults for module '%s' [%s]",
("%s:" % kind if kind else "") + module, e)
try:
if not return_yaml:
default_module_info[kind][module]
except KeyError:
raise LoggedError(
log, "The defaults file for '%s' should be structured "
"as %s:%s:{[options]}.", module, kind, module)
return default_module_info
def get_desc_component(component, kind, info=None):
cls = get_class(component, kind, None_if_not_found=True)
if cls:
lines = cleandoc(cls.get_desc(info) or "")
else:
lines = "[no description found]"
return lines + "\n"
def get_sampler_name_and_class(info_sampler):
"""
Auxiliary function to retrieve the class of the required sampler.
"""
check_sane_info_sampler(info_sampler)
name = list(info_sampler)[0]
return name, get_class(name, kind=kinds.sampler)
def get_preferred_old_values(info_old):
"""
Returns selected values in `info_old`, which are preferred at resuming.
"""
keep_old = {}
for block_name, block in info_old.items():
if block_name not in kinds or not block:
continue
for k in block:
try:
component_path = block[k].pop(_component_path, None) \
if isinstance(block[k], dict) else None
class_name = (block[k] or {}).get(_class_name) or k
cls = get_class(class_name,
block_name,
component_path=component_path)
prefer_old_k_this = getattr(cls, "_at_resume_prefer_old", {})
if prefer_old_k_this:
if block_name not in keep_old:
keep_old[block_name] = {}
keep_old[block_name].update({
k: {
o: block[k][o]
for o in prefer_old_k_this if o in block[k]
}
})
except ImportError:
pass
return keep_old
def get_default_info(component_or_class,
kind=None,
return_yaml=False,
yaml_expand_defaults=True,
component_path=None,
input_options=empty_dict,
class_name=None):
"""
Get default info for a component_or_class.
"""
_kind = kind
try:
if inspect.isclass(component_or_class):
cls = component_or_class
else:
_kind = _kind or get_kind(component_or_class,
class_name=class_name)
cls = get_class(class_name or component_or_class,
_kind,
component_path=component_path)
default_component_info = \
cls.get_defaults(return_yaml=return_yaml,
yaml_expand_defaults=yaml_expand_defaults,
input_options=input_options)
except Exception as e:
raise LoggedError(
log, "Failed to get defaults for component or class '%s' [%s]",
component_or_class, e)
return default_component_info
def get_bib_component(component, kind):
cls = get_class(component, kind, None_if_not_found=True)
if cls:
lines = ((cls.get_bibtex() or "").lstrip("\n").rstrip("\n")
or "# [no bibliography information found]")
else:
lines = "# [Component '%s.%s' not known.]" % (kind, component)
return lines + "\n"
def __init__(self, info_likelihood, parameterization, info_theory=None, modules=None,
timing=None):
self.log = logging.getLogger("Likelihood")
# *IF* there is a theory code, initialize it
if info_theory:
name = list(info_theory)[0]
# If it has an "external" key, wrap it up. Else, load it up
if _external in info_theory[name]:
theory_class = info_theory[name][_external]
else:
theory_class = get_class(name, kind=_theory)
self.theory = theory_class(info_theory[name], modules=modules, timing=timing)
else:
self.theory = None
# Initialize individual Likelihoods
self._likelihoods = odict()
for name, info in info_likelihood.items():
# If it has an "external" key, wrap it up. Else, load it up
if _external in info:
self._likelihoods[name] = LikelihoodExternalFunction(
name, info, _theory=getattr(self, _theory, None), timing=timing)
else:
like_class = get_class(name)
self._likelihoods[name] = like_class(info, modules=modules, timing=timing)
# Assign input/output parameters
self._assign_params(parameterization, info_likelihood, info_theory)
# Do the user-defined post-initialisation, and assign the theory code
if self.theory:
self.theory.initialize()
for like in self:
self[like].initialize()
self[like].theory = self.theory
self[like].add_theory()
# Store the input params and likelihoods on which each sampled params depends.
self.sampled_input_dependence = parameterization.sampled_input_dependence()
self.sampled_like_dependence = odict(
[[p, [like for like in list(self) + ([_theory] if self.theory else [])
if any([(i in self[like].input_params) for i in (i_s or [p])])]]
for p, i_s in self.sampled_input_dependence.items()])
# Theory parameters "depend" on every likelihood, since re-computing the theory
# code forces recomputation of the likelihoods
for p, ls in self.sampled_like_dependence.items():
if _theory in ls:
self.sampled_like_dependence[p] = ([_theory] + list(self._likelihoods))
# Overhead per likelihood evaluation
self.overhead = _overhead_per_param * len(parameterization.sampled_params())
def __init__(self,
info_likelihood,
packages_path=None,
timing=None,
theory=None):
super().__init__()
self.set_logger("likelihood")
self.theory = theory
# Get the individual likelihood classes
for name, info in info_likelihood.items():
if isinstance(name, Theory):
name, info = name.get_name(), info
if isinstance(info, Theory):
self.add_instance(name, info)
elif _external in info:
if isinstance(info[_external], Theory):
self.add_instance(name, info[_external])
elif inspect.isclass(info[_external]):
if not hasattr(info[_external], "get_current_logp") or \
not issubclass(info[_external], Theory):
raise LoggedError(
self.log, "%s: external class likelihood must "
"be a subclass of Theory and have "
"logp, get_current_logp functions",
info[_external].__name__)
self.add_instance(
name, info[_external](info,
packages_path=packages_path,
timing=timing,
standalone=False,
name=name))
else:
# If it has an "external" key, wrap it up. Else, load it up
self.add_instance(
name,
LikelihoodExternalFunction(info, name, timing=timing))
else:
like_class = get_class(info.get(_class_name) or name,
kind=kinds.likelihood,
component_path=info.pop(
_component_path, None))
self.add_instance(
name,
like_class(info,
packages_path=packages_path,
timing=timing,
standalone=False,
name=name))
if not hasattr(self[name], "get_current_logp"):
raise LoggedError(
self.log, "'Likelihood' %s is not actually a "
"likelihood (no get_current_logp function)", name)
def test_generic_camb(packages_path, skip_not_installed):
like = "bao.sdss_dr12_consensus_bao"
like_rename = "my_bao"
chi2_generic = deepcopy(chi2_sdss_dr12_consensus_bao)
chi2_generic[like_rename] = chi2_generic.pop(like)
likelihood_defaults = get_class(like).get_defaults()
likelihood_defaults.pop("path")
likelihood_defaults["class"] = "bao.generic"
info_likelihood = {like_rename: likelihood_defaults}
info_theory = {"camb": None}
body_of_test(packages_path, best_fit, info_likelihood, info_theory,
chi2_generic, skip_not_installed=skip_not_installed)
def get_sampler(info_sampler,
posterior,
output_file,
resume=_resume_default,
modules=None):
"""
Auxiliary function to retrieve and initialize the requested sampler.
"""
if not info_sampler:
log.error("No sampler given!")
raise HandledException
try:
name = list(info_sampler)[0]
except AttributeError:
log.error(
"The sampler block must be a dictionary 'sampler: {options}'.")
raise HandledException
sampler_class = get_class(name, kind=_sampler)
return sampler_class(info_sampler[name],
posterior,
output_file,
resume=resume,
modules=modules)
def get_sampler(info_sampler,
posterior,
output_file,
resume=_resume_default,
modules=None):
"""
Auxiliary function to retrieve and initialize the requested sampler.
"""
log = logging.getLogger(__name__.split(".")[-1])
if not info_sampler:
raise LoggedError(log, "No sampler given!")
try:
name = list(info_sampler)[0]
except AttributeError:
raise LoggedError(
log,
"The sampler block must be a dictionary 'sampler: {options}'.")
sampler_class = get_class(name, kind=_sampler)
return sampler_class(info_sampler[name],
posterior,
output_file,
resume=resume,
modules=modules)
def __init__(self, info_likelihood, parametrization, info_theory=None):
# Store the input/output parameters
self.input_params = parametrization.input_params()
self.output_params = parametrization.output_params()
# Initialise individual Likelihoods
self._likelihoods = odict()
for name, info in info_likelihood.items():
# If it does "external" key, wrap it up. Else, load it up
if _external in info:
self._likelihoods[name] = LikelihoodExternalFunction(
name,
info,
parametrization,
theory=getattr(self, _theory, None))
else:
lik_class = get_class(name)
self._likelihoods[name] = lik_class(info, parametrization)
# Check that all are recognized
requested_not_known = {}
for params in ("input_params", "output_params"):
info = getattr(parametrization, params)()
setattr(self, params, info)
requested = set(info)
known = set(chain(*[getattr(self[lik], params) for lik in self]))
requested_not_known[params] = requested.difference(known)
if requested_not_known[params]:
if info_theory:
log.debug(
"The following %s parameters are not recognised by any "
"likelihood, and will be passed to the theory code: %r",
params.split("_")[0], requested_not_known[params])
else:
log.error(
"Some of the requested %s parameters were not recognized "
"by any likelihood: %r",
params.split("_")[0], requested_not_known[params])
raise HandledException
# *IF* there is a theory code, initialise it
if info_theory:
input_params_theory = self.input_params.fromkeys(
requested_not_known["input_params"])
output_params_theory = self.output_params.fromkeys(
requested_not_known["output_params"])
name, fields = list(info_theory.items())[0]
theory_class = get_class(name, kind=_theory)
self.theory = theory_class(input_params_theory,
output_params_theory, fields)
else:
self.theory = None
for lik in self:
self[lik].theory = self.theory
self[lik].add_theory()
# Store the input params and likelihods on which each sampled params depends.
# Theory parameters "depend" on every likelihood, since re-computing the theory
# code forces recomputation of the likelihoods
self.sampled_input_dependence = parametrization.sampled_input_dependence(
)
self.sampled_lik_dependence = odict([[
p,
[
lik for lik in list(self) + ([_theory] if self.theory else [])
if any([(i in self[lik].input_params) for i in (i_s or [p])])
]
] for p, i_s in self.sampled_input_dependence.items()])
# Theory parameters "depend" on every likelihood, since re-computing the theory
# code forces recomputation of the likelihoods
for p, ls in self.sampled_lik_dependence.items():
if _theory in ls:
self.sampled_lik_dependence[p] = ([_theory] +
list(self._likelihoods))
# Pop the per-likelihood parameters info, that was auxiliary
for lik in info_likelihood:
info_likelihood[lik].pop(_params)
def __init__(self, info_likelihood, parameterization, info_theory=None, modules=None,
timing=None):
self.log = logging.getLogger("Likelihood")
# Initialize individual Likelihoods
self._likelihoods = odict()
for name, info in info_likelihood.items():
# If it has an "external" key, wrap it up. Else, load it up
if _external in info:
self._likelihoods[name] = LikelihoodExternalFunction(
name, info, parameterization, _theory=getattr(self, _theory, None),
timing=timing)
else:
# If mock likelihood and there is a theory code, make the mock ignore
# all parameters, or they would never reach the theory code
if "prefix" in info and info_theory:
info["prefix"] = None
self.log.debug("Mock likelihood '%s' will ignore all parameters, "
"since a theory code is present.", name)
like_class = get_class(name)
self._likelihoods[name] = like_class(
info, parameterization, modules=modules, timing=timing)
# Check that all are recognized
requested_not_known = {}
for params in ("input_params", "output_params"):
# Store the input/output parameters
info = getattr(parameterization, params)()
setattr(self, params, info)
requested = set(info)
known = set(chain(*[getattr(self[like], params) for like in self]))
requested_not_known[params] = requested.difference(known)
if requested_not_known[params]:
if info_theory:
self.log.debug(
"The following %s parameters are not recognized by any "
"likelihood, and will be passed to the theory code: %r",
params.split("_")[0], list(requested_not_known[params]))
else:
self.log.error(
"Some of the requested %s parameters were not recognized "
"by any likelihood: %r",
params.split("_")[0], list(requested_not_known[params]))
raise HandledException
# *IF* there is a theory code, initialize it
if info_theory:
input_params_theory = {p: v for p, v in self.input_params.items()
if p in requested_not_known["input_params"]}
output_params_theory = {p: None for p in self.output_params
if p in requested_not_known["output_params"] and
# Not the chi2's either
not p.startswith(_chi2 + _separator)}
name, fields = list(info_theory.items())[0]
# If it has an "external" key, wrap it up. Else, load it up
if _external in list(info_theory.values())[0]:
theory_class = list(info_theory.values())[0][_external]
else:
theory_class = get_class(name, kind=_theory)
self.theory = theory_class(input_params_theory, output_params_theory,
fields, modules=modules, timing=timing)
else:
self.theory = None
for like in self:
self[like].theory = self.theory
self[like].add_theory()
# Store the input params and likelihoods on which each sampled params depends.
self.sampled_input_dependence = parameterization.sampled_input_dependence()
self.sampled_like_dependence = odict(
[[p, [like for like in list(self) + ([_theory] if self.theory else [])
if any([(i in self[like].input_params) for i in (i_s or [p])])]]
for p, i_s in self.sampled_input_dependence.items()])
# Theory parameters "depend" on every likelihood, since re-computing the theory
# code forces recomputation of the likelihoods
for p, ls in self.sampled_like_dependence.items():
if _theory in ls:
self.sampled_like_dependence[p] = ([_theory] + list(self._likelihoods))
# Pop the per-likelihood parameters info, that was auxiliary
for like in info_likelihood:
info_likelihood[like].pop(_params)
# Overhead per likelihood evaluation
self.overhead = _overhead_per_param * len(parameterization.sampled_params())
def install(*infos, **kwargs):
debug = kwargs.get(_debug)
if not log.root.handlers:
logger_setup()
path = kwargs.get("path")
if not path:
path = resolve_packages_path(infos)
if not path:
raise LoggedError(
log, "No 'path' argument given, and none could be found in input infos "
"(as %r), the %r env variable or the config file. "
"Maybe specify one via a command line argument '-%s [...]'?",
_packages_path, _packages_path_env, _packages_path_arg[0])
abspath = os.path.abspath(path)
log.info("Installing external packages at '%s'", abspath)
kwargs_install = {"force": kwargs.get("force", False),
"no_progress_bars": kwargs.get("no_progress_bars")}
for what in (_code, _data):
kwargs_install[what] = kwargs.get(what, True)
spath = os.path.join(abspath, what)
if kwargs_install[what] and not os.path.exists(spath):
try:
os.makedirs(spath)
except OSError:
raise LoggedError(
log, "Could not create the desired installation folder '%s'", spath)
failed_components = []
skip_keywords_arg = set(kwargs.get("skip", []) or [])
# NB: if passed with quotes as `--skip "a b"`, it's interpreted as a single key
skip_keywords_arg = set(chain(*[word.split() for word in skip_keywords_arg]))
skip_keywords_env = set(
os.environ.get(_install_skip_env, "").replace(",", " ").lower().split())
skip_keywords = skip_keywords_arg.union(skip_keywords_env)
for kind, components in get_used_components(*infos).items():
for component in components:
print()
print(create_banner(kind + ":" + component,
symbol=_banner_symbol, length=_banner_length), end="")
print()
if _skip_helper(component.lower(), skip_keywords, skip_keywords_env, log):
continue
info = (next(info for info in infos if component in
info.get(kind, {}))[kind][component]) or {}
if isinstance(info, str) or _external in info:
log.warning("Component '%s' is a custom function. "
"Nothing to do.", component)
continue
try:
imported_class = get_class(component, kind,
component_path=info.pop(_component_path, None))
except ImportError as excpt:
log.error("Component '%s' not recognized. [%s].", component, excpt)
failed_components += ["%s:%s" % (kind, component)]
continue
else:
if _skip_helper(imported_class.__name__.lower(), skip_keywords,
skip_keywords_env, log):
continue
is_compatible = getattr(imported_class, "is_compatible", lambda: True)()
if not is_compatible:
log.info(
"Skipping %r because it is not compatible with your OS.", component)
continue
log.info("Checking if dependencies have already been installed...")
is_installed = getattr(imported_class, "is_installed", None)
if is_installed is None:
log.info("%s.%s is a fully built-in component: nothing to do.",
kind, imported_class.__name__)
continue
install_path = abspath
get_path = getattr(imported_class, "get_path", None)
if get_path:
install_path = get_path(install_path)
has_been_installed = False
if not debug:
logging.disable(logging.ERROR)
if kwargs.get("skip_global"):
has_been_installed = is_installed(path="global", **kwargs_install)
if not has_been_installed:
has_been_installed = is_installed(path=install_path, **kwargs_install)
if not debug:
logging.disable(logging.NOTSET)
if has_been_installed:
log.info("External dependencies for this component already installed.")
if kwargs.get(_test_run, False):
continue
if kwargs_install["force"] and not kwargs.get("skip_global"):
log.info("Forcing re-installation, as requested.")
else:
log.info("Doing nothing.")
continue
else:
log.info("Installation check failed!")
if not debug:
log.info(
"(If you expected this to be already installed, re-run "
"`cobaya-install` with --debug to get more verbose output.)")
if kwargs.get(_test_run, False):
continue
log.info("Installing...")
try:
install_this = getattr(imported_class, "install", None)
success = install_this(path=abspath, **kwargs_install)
except KeyboardInterrupt:
raise
except:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
log.error("An unknown error occurred. Delete the external packages "
"folder %r and try again. "
"Please, notify the developers if this error persists.",
abspath)
success = False
if success:
log.info("Successfully installed! Let's check it...")
else:
log.error("Installation failed! Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve, "
"install the packages required by this component manually.")
failed_components += ["%s:%s" % (kind, component)]
continue
# test installation
if not debug:
logging.disable(logging.ERROR)
successfully_installed = is_installed(path=install_path, **kwargs_install)
if not debug:
logging.disable(logging.NOTSET)
if not successfully_installed:
log.error("Installation apparently worked, "
"but the subsequent installation test failed! "
"Look at the error messages above, or re-run with --debug "
"for more more verbose output. "
"Try to solve the issues and try again, or, if you are unable "
"to solve them, install the packages required by this "
"component manually.")
failed_components += ["%s:%s" % (kind, component)]
else:
log.info("Installation check successful.")
print()
print(create_banner(" * Summary * ",
symbol=_banner_symbol, length=_banner_length), end="")
print()
if failed_components:
bullet = "\n - "
raise LoggedError(
log, "The installation (or installation test) of some component(s) has "
"failed: %s\nCheck output of the installer of each component above "
"for precise error info.\n",
bullet + bullet.join(failed_components))
log.info("All requested components' dependencies correctly installed.")
# Set the installation path in the global config file
if not kwargs.get("no_set_global", False) and not kwargs.get(_test_run, False):
write_packages_path_in_config_file(abspath)
log.info("The installation path has been written into the global config file: %s",
os.path.join(get_config_path(), _packages_path_config_file))
def install(*infos, **kwargs):
if not log.root.handlers:
logger_setup()
path = kwargs.get("path")
if not path:
path = resolve_packages_path(infos)
if not path:
raise LoggedError(
log,
"No 'path' argument given, and none could be found in input infos "
"(as %r), the %r env variable or the config file. "
"Maybe specify one via a command line argument '-%s [...]'?",
_packages_path, _packages_path_env, _packages_path_arg[0])
abspath = os.path.abspath(path)
log.info("Installing external packages at '%s'", abspath)
kwargs_install = {
"force": kwargs.get("force", False),
"no_progress_bars": kwargs.get("no_progress_bars")
}
for what in (_code, _data):
kwargs_install[what] = kwargs.get(what, True)
spath = os.path.join(abspath, what)
if kwargs_install[what] and not os.path.exists(spath):
try:
os.makedirs(spath)
except OSError:
raise LoggedError(
log,
"Could not create the desired installation folder '%s'",
spath)
failed_components = []
skip_keywords = set(kwargs.get("skip", []) or [])
skip_keywords_env = set(
os.environ.get(_install_skip_env, "").replace(",",
" ").lower().split())
skip_keywords = skip_keywords.union(skip_keywords_env)
for kind, components in get_used_components(*infos).items():
for component in components:
print()
print(create_banner(kind + ":" + component, symbol="=", length=80))
print()
if _skip_helper(component.lower(), skip_keywords,
skip_keywords_env, log):
continue
info = (next(
info for info in infos
if component in info.get(kind, {}))[kind][component]) or {}
if isinstance(info, str) or _external in info:
log.warning(
"Component '%s' is a custom function. "
"Nothing to do.", component)
continue
try:
imported_class = get_class(component,
kind,
component_path=info.pop(
_component_path, None))
except ImportError as e:
log.error("Component '%s' not recognized. [%s]." %
(component, e))
failed_components += ["%s:%s" % (kind, component)]
continue
else:
if _skip_helper(imported_class.__name__.lower(), skip_keywords,
skip_keywords_env, log):
continue
is_installed = getattr(imported_class, "is_installed", None)
if is_installed is None:
log.info("%s.%s is a fully built-in component: nothing to do.",
kind, imported_class.__name__)
continue
if is_installed(path=abspath, **kwargs_install):
log.info("External component already installed.")
if kwargs.get("just_check", False):
continue
if kwargs_install["force"]:
log.info("Forcing re-installation, as requested.")
else:
log.info("Doing nothing.")
continue
else:
if kwargs.get("just_check", False):
log.info("NOT INSTALLED!")
continue
try:
install_this = getattr(imported_class, "install", None)
success = install_this(path=abspath, **kwargs_install)
except KeyboardInterrupt:
raise
except:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
log.error(
"An unknown error occurred. Delete the external packages "
"folder %r and try again. "
"Please, notify the developers if this error persists.",
abspath)
success = False
if success:
log.info("Successfully installed!")
else:
log.error(
"Installation failed! Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve, "
"install the packages required by this component manually."
)
failed_components += ["%s:%s" % (kind, component)]
continue
# test installation
if not is_installed(path=abspath, **kwargs_install):
log.error(
"Installation apparently worked, "
"but the subsequent installation test failed! "
"Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve, "
"install the packages required by this component manually."
)
failed_components += ["%s:%s" % (kind, component)]
if failed_components:
bullet = "\n - "
raise LoggedError(
log,
"The installation (or installation test) of some component(s) has "
"failed: %s\nCheck output of the installer of each component above "
"for precise error info.\n",
bullet + bullet.join(failed_components))
# Set the installation path in the global config file
if not kwargs.get("no_set_global", False) and not kwargs.get(
"just_check", False):
write_packages_path_in_config_file(abspath)
log.info(
"The installation path has been written in the global config file."
)
def install(*infos, **kwargs):
if not log.root.handlers:
logger_setup()
path = kwargs.get("path", ".")
if not path:
# See if we can get one (and only one) from infos
paths = set(
[p for p in [info.get(_path_install) for info in infos] if p])
if len(paths) == 1:
path = paths[0]
else:
print("logging?")
raise LoggedError(
log,
"No 'path' argument given and could not extract one (and only one) "
"from the infos.")
abspath = os.path.abspath(path)
log.info("Installing modules at '%s'\n", abspath)
kwargs_install = {
"force": kwargs.get("force", False),
"no_progress_bars": kwargs.get("no_progress_bars")
}
for what in (_code, _data):
kwargs_install[what] = kwargs.get(what, True)
spath = os.path.join(abspath, what)
if kwargs_install[what] and not os.path.exists(spath):
try:
os.makedirs(spath)
except OSError:
raise LoggedError(
log,
"Could not create the desired installation folder '%s'",
spath)
failed_modules = []
skip_list = os.environ.get("COBAYA_TEST_SKIP",
"").replace(",", " ").lower().split()
for kind, modules in get_used_modules(*infos).items():
for module in modules:
print(create_banner(kind + ":" + module, symbol="=", length=80))
if len([s for s in skip_list if s in module.lower()]):
log.info("Skipping %s for test skip list %s" %
(module, skip_list))
continue
module_folder = get_class_module(module, kind)
try:
imported_module = import_module(module_folder,
package=_package)
imported_class = get_class(module, kind)
if len([
s for s in skip_list
if s in imported_class.__name__.lower()
]):
log.info("Skipping %s for test skip list %s" %
(imported_class.__name__, skip_list))
continue
except ImportError as e:
if kind == _likelihood:
info = (next(info for info in infos if module in info.get(
_likelihood, {}))[_likelihood][module]) or {}
if isinstance(info, string_types) or _external in info:
log.warning(
"Module '%s' is a custom likelihood. "
"Nothing to do.\n", module)
else:
log.error("Module '%s' not recognized. [%s]\n" %
(module, e))
failed_modules += ["%s:%s" % (kind, module)]
continue
is_installed = getattr(
imported_class, "is_installed",
getattr(imported_module, "is_installed", None))
if is_installed is None:
log.info("Built-in module: nothing to do.\n")
continue
if is_installed(path=abspath, **kwargs_install):
log.info("External module already installed.\n")
if kwargs.get("just_check", False):
continue
if kwargs_install["force"]:
log.info("Forcing re-installation, as requested.")
else:
log.info("Doing nothing.\n")
continue
else:
if kwargs.get("just_check", False):
log.info("NOT INSTALLED!\n")
continue
try:
install_this = getattr(
imported_class, "install",
getattr(imported_module, "install", None))
success = install_this(path=abspath, **kwargs_install)
except:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
log.error(
"An unknown error occurred. Delete the modules folder and try "
"again. Notify the developers if this error persists.")
success = False
if success:
log.info("Successfully installed!\n")
else:
log.error(
"Installation failed! Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve, "
"install this module manually.")
failed_modules += ["%s:%s" % (kind, module)]
continue
# test installation
if not is_installed(path=abspath, **kwargs_install):
log.error(
"Installation apparently worked, "
"but the subsequent installation test failed! "
"Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve, "
"install this module manually.")
failed_modules += ["%s:%s" % (kind, module)]
if failed_modules:
bullet = "\n - "
raise LoggedError(
log,
"The installation (or installation test) of some module(s) has failed: "
"%s\nCheck output of the installer of each module above "
"for precise error info.\n", bullet + bullet.join(failed_modules))
def is_equal_info(info_old,
info_new,
strict=True,
print_not_log=False,
ignore_blocks=()):
"""
Compares two information dictionaries, and old one versus a new one, and updates the
new one for selected values of the old one.
Set ``strict=False`` (default: ``True``) to ignore options that would not affect
the statistics of a posterior sample, including order of params/priors/likelihoods.
"""
if print_not_log:
myprint = print
myprint_debug = lambda x: x
else:
myprint = log.info
myprint_debug = log.debug
myname = inspect.stack()[0][3]
ignore = set() if strict else \
{_debug, _debug_file, _resume, _force, _packages_path, _test_run, _version}
ignore = ignore.union(set(ignore_blocks or []))
if set(info for info in info_old if info_old[info] is not None).difference(ignore) \
!= set(info for info in info_new if info_new[info] is not None).difference(
ignore):
myprint(myname +
": different blocks or options: %r (old) vs %r (new)" %
(set(info_old).difference(ignore),
set(info_new).difference(ignore)))
return False
for block_name in info_old:
if block_name in ignore or block_name not in info_new:
continue
block1 = deepcopy_where_possible(info_old[block_name])
block2 = deepcopy_where_possible(info_new[block_name])
# First, deal with root-level options (force, output, ...)
if not isinstance(block1, dict):
if block1 != block2:
myprint(myname + ": different option '%s'" % block_name)
return False
continue
# Now let's do components and params
# 1. check order (it DOES matter, but just up to 1st level)
f = list if strict else set
if f(block1) != f(block2):
myprint(myname +
": different [%s] or different order of them: %r vs %r" %
(block_name, list(block1), list(block2)))
return False
# 2. Gather general options to be ignored
if not strict:
ignore_k = set()
if block_name in [kinds.theory, kinds.likelihood]:
ignore_k = ignore_k.union({_input_params, _output_params})
elif block_name == _params:
for param in block1:
# Unify notation
block1[param] = expand_info_param(block1[param])
block2[param] = expand_info_param(block2[param])
ignore_k = ignore_k.union({
partag.latex, partag.renames, partag.ref,
partag.proposal, "min", "max"
})
# Fixed params, it doesn't matter if they are saved as derived
if partag.value in block1[param]:
block1[param].pop(partag.derived, None)
if partag.value in block2[param]:
block2[param].pop(partag.derived, None)
# Renames: order does not matter
block1[param][partag.renames] = set(block1[param].get(
partag.renames, []))
block2[param][partag.renames] = set(block2[param].get(
partag.renames, []))
# 3. Now check component/parameters one-by-one
for k in block1:
if not strict:
# Add component-specific options to be ignored
if block_name in kinds:
ignore_k_this = ignore_k.copy()
if _external not in block1[k]:
try:
component_path = block1[k].pop(_component_path, None) \
if isinstance(block1[k], dict) else None
class_name = (block1[k]
or {}).get(_class_name) or k
cls = get_class(class_name,
block_name,
component_path=component_path)
ignore_k_this = ignore_k_this.union(
set(getattr(cls, "_at_resume_prefer_new", {})))
except ImportError:
pass
# Pop ignored and kept options
for j in list(ignore_k_this):
block1[k].pop(j, None)
block2[k].pop(j, None)
if block1[k] != block2[k]:
# For clarity, pop common stuff before printing
to_pop = [
j for j in block1[k]
if (block1[k].get(j) == block2[k].get(j))
]
[(block1[k].pop(j, None), block2[k].pop(j, None))
for j in to_pop]
myprint(myname + ": different content of [%s:%s]" %
(block_name, k) +
" -- (re-run with `debug: True` for more info)")
myprint_debug("%r (old) vs %r (new)" % (block1[k], block2[k]))
return False
return True
