def pytest_addoption(parser):
parser.addoption("--" + packages_path_arg_posix, action="store",
default=resolve_packages_path(),
help="Path to folder of automatic installation of packages")
parser.addoption("--skip-not-installed", action="store_true",
help="Skip tests for which dependencies of used components are not "
"installed.")
def refresh_display(self, info):
QApplication.setOverrideCursor(Qt.WaitCursor)
try:
comments = info.pop(input_database._comment, None)
comments_text = "\n# " + "\n# ".join(comments)
except (
TypeError, # No comments
AttributeError
): # Failed to generate info (returned str instead)
comments_text = ""
self.display["python"].setText("info = " + pformat(info) +
comments_text)
self.display["yaml"].setText(
yaml_dump(sort_cosmetic(info)) + comments_text)
self.display["bibliography"].setText(
prettyprint_bib(*get_bib_info(info)))
# Display covmat
packages_path = resolve_packages_path()
if not packages_path:
self.covmat_text.setText(
"\nIn order to find a covariance matrix, you need to define an external "
"packages installation path, e.g. via the env variable %r.\n" %
_packages_path_env)
elif any(not os.path.isdir(d.format(**{_packages_path: packages_path}))
for d in covmat_folders):
self.covmat_text.setText(
"\nThe external cosmological packages appear not to be installed where "
"expected: %r\n" % packages_path)
else:
covmat_data = get_best_covmat(info, packages_path=packages_path)
self.current_params_in_covmat = covmat_data["params"]
self.current_covmat = covmat_data["covmat"]
_, corrmat = cov_to_std_and_corr(self.current_covmat)
self.covmat_text.setText(
"\nCovariance file: %r\n\n"
"NB: you do *not* need to save or copy this covariance matrix: "
"it will be selected automatically.\n" % covmat_data["name"])
self.covmat_table.setRowCount(len(self.current_params_in_covmat))
self.covmat_table.setColumnCount(len(
self.current_params_in_covmat))
self.covmat_table.setHorizontalHeaderLabels(
list(self.current_params_in_covmat))
self.covmat_table.setVerticalHeaderLabels(
list(self.current_params_in_covmat))
for i, pi in enumerate(self.current_params_in_covmat):
for j, pj in enumerate(self.current_params_in_covmat):
self.covmat_table.setItem(
i, j,
QTableWidgetItem("%g" % self.current_covmat[i, j]))
if i != j:
color = [
256 * c
for c in cmap_corr(corrmat[i, j] / 2 + 0.5)[:3]
]
else:
color = [255.99] * 3
self.covmat_table.item(i, j).setBackground(QColor(*color))
self.covmat_table.item(i, j).setForeground(QColor(0, 0, 0))
QApplication.restoreOverrideCursor()
def __init__(self,
info: InfoDictIn = empty_dict,
name: Optional[str] = None,
timing: Optional[bool] = None,
packages_path: Optional[str] = None,
initialize=True,
standalone=True):
if standalone:
# TODO: would probably be more natural if defaults were always read here
default_info = self.get_defaults(input_options=info)
default_info.update(info)
info = default_info
self.set_instance_defaults()
self._name = name or self.get_qualified_class_name()
self.packages_path = packages_path or resolve_packages_path()
# set attributes from the info (from yaml file or directly input dictionary)
for k, value in info.items():
try:
# MARKED FOR DEPRECATION IN v3.0
# NB: cannot ever raise an error, since users may use "path_install" for
# their own purposes. When considered *fully* deprecated, simply
# remove this whole block.
if k == "path_install":
self.log.warning(
"*DEPRECATION*: `path_install` will be deprecated "
"in the next version. Please use `packages_path` instead."
)
setattr(self, packages_path_input, value)
# END OF DEPRECATION BLOCK
setattr(self, k, value)
except AttributeError:
raise AttributeError("Cannot set {} attribute for {}!".format(
k, self))
self.set_logger(name=self._name)
self.set_timing_on(timing)
try:
if initialize:
self.initialize()
except AttributeError as e:
if '_params' in str(e):
raise LoggedError(
self.log, "use 'initialize_with_params' if you need to "
"initialize after input and output parameters"
" are set (%s, %s)", self, e)
raise
def __init__(self,
info=empty_dict,
name=None,
timing=None,
packages_path=None,
initialize=True,
standalone=True):
if standalone:
# TODO: would probably be more natural if defaults were always read here
default_info = self.get_defaults(input_options=info)
default_info.update(info)
info = default_info
self.set_instance_defaults()
self._name = name or self.get_qualified_class_name()
self.packages_path = packages_path or resolve_packages_path()
# set attributes from the info (from yaml file or directly input dictionary)
for k, value in info.items():
try:
# MARKED FOR DEPRECATION IN v3.0
if k == "path_install":
self.log.warning(
"*DEPRECATION*: `path_install` will be deprecated "
"in the next version. Please use `packages_path` instead."
)
# BEHAVIOUR TO BE REPLACED BY ERROR:
# set BOTH old and new names, just in case old one is used internally
from cobaya.conventions import _packages_path
setattr(self, _packages_path, value)
# END OF DEPRECATION BLOCK
setattr(self, k, value)
except AttributeError:
raise AttributeError("Cannot set {} attribute for {}!".format(
k, self))
self.set_logger(name=self._name)
self.set_timing_on(timing)
try:
if initialize:
self.initialize()
except AttributeError as e:
if '_params' in str(e):
raise LoggedError(
self.log, "use 'initialize_with_params' if you need to "
"initialize after input and output parameters"
" are set (%s, %s)", self, e)
raise
def __init__(self, *args, **kwargs):
# Ensure check for install and version errors
# (e.g. may inherit from a class that inherits from this one, and not have them)
if self.install_options:
name = self.get_qualified_class_name()
logger = get_logger(name)
packages_path = kwargs.get(
"packages_path") or resolve_packages_path()
old = False
try:
installed = self.is_installed(path=packages_path)
except Exception as excpt: # catches VersionCheckError and unexpected ones
installed = False
old = isinstance(excpt, VersionCheckError)
logger.error(f"{type(excpt).__name__}: {excpt}")
if not installed:
not_or_old = ("is not up to date"
if old else "has not been correctly installed")
raise ComponentNotInstalledError(logger, (
f"The data for this likelihood {not_or_old}. To install it, "
f"run `cobaya-install {name}{' --upgrade' if old else ''}`"
))
super().__init__(*args, **kwargs)
def install(*infos, **kwargs):
if not log.root.handlers:
logger_setup()
path = kwargs.get("path")
if not path:
path = resolve_packages_path(infos)
if not path:
raise LoggedError(
log,
"No 'path' argument given, and none could be found in input infos "
"(as %r), the %r env variable or the config file. "
"Maybe specify one via a command line argument '-%s [...]'?",
_packages_path, _packages_path_env, _packages_path_arg[0])
abspath = os.path.abspath(path)
log.info("Installing external packages at '%s'", abspath)
kwargs_install = {
"force": kwargs.get("force", False),
"no_progress_bars": kwargs.get("no_progress_bars")
}
for what in (_code, _data):
kwargs_install[what] = kwargs.get(what, True)
spath = os.path.join(abspath, what)
if kwargs_install[what] and not os.path.exists(spath):
try:
os.makedirs(spath)
except OSError:
raise LoggedError(
log,
"Could not create the desired installation folder '%s'",
spath)
failed_components = []
skip_keywords = set(kwargs.get("skip", []) or [])
skip_keywords_env = set(
os.environ.get(_install_skip_env, "").replace(",",
" ").lower().split())
skip_keywords = skip_keywords.union(skip_keywords_env)
for kind, components in get_used_components(*infos).items():
for component in components:
print()
print(create_banner(kind + ":" + component, symbol="=", length=80))
print()
if _skip_helper(component.lower(), skip_keywords,
skip_keywords_env, log):
continue
info = (next(
info for info in infos
if component in info.get(kind, {}))[kind][component]) or {}
if isinstance(info, str) or _external in info:
log.warning(
"Component '%s' is a custom function. "
"Nothing to do.", component)
continue
try:
imported_class = get_class(component,
kind,
component_path=info.pop(
_component_path, None))
except ImportError as e:
log.error("Component '%s' not recognized. [%s]." %
(component, e))
failed_components += ["%s:%s" % (kind, component)]
continue
else:
if _skip_helper(imported_class.__name__.lower(), skip_keywords,
skip_keywords_env, log):
continue
is_installed = getattr(imported_class, "is_installed", None)
if is_installed is None:
log.info("%s.%s is a fully built-in component: nothing to do.",
kind, imported_class.__name__)
continue
if is_installed(path=abspath, **kwargs_install):
log.info("External component already installed.")
if kwargs.get("just_check", False):
continue
if kwargs_install["force"]:
log.info("Forcing re-installation, as requested.")
else:
log.info("Doing nothing.")
continue
else:
if kwargs.get("just_check", False):
log.info("NOT INSTALLED!")
continue
try:
install_this = getattr(imported_class, "install", None)
success = install_this(path=abspath, **kwargs_install)
except KeyboardInterrupt:
raise
except:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
log.error(
"An unknown error occurred. Delete the external packages "
"folder %r and try again. "
"Please, notify the developers if this error persists.",
abspath)
success = False
if success:
log.info("Successfully installed!")
else:
log.error(
"Installation failed! Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve, "
"install the packages required by this component manually."
)
failed_components += ["%s:%s" % (kind, component)]
continue
# test installation
if not is_installed(path=abspath, **kwargs_install):
log.error(
"Installation apparently worked, "
"but the subsequent installation test failed! "
"Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve, "
"install the packages required by this component manually."
)
failed_components += ["%s:%s" % (kind, component)]
if failed_components:
bullet = "\n - "
raise LoggedError(
log,
"The installation (or installation test) of some component(s) has "
"failed: %s\nCheck output of the installer of each component above "
"for precise error info.\n",
bullet + bullet.join(failed_components))
# Set the installation path in the global config file
if not kwargs.get("no_set_global", False) and not kwargs.get(
"just_check", False):
write_packages_path_in_config_file(abspath)
log.info(
"The installation path has been written in the global config file."
)
def install_script():
set_mpi_disabled(True)
warn_deprecation()
# Parse arguments
import argparse
parser = argparse.ArgumentParser(
description="Cobaya's installation tool for external packages.")
parser.add_argument(
"files_or_components",
action="store",
nargs="+",
metavar="input_file.yaml|component_name",
help="One or more input files or component names "
"(or simply 'cosmo' to install all the requisites for basic"
" cosmological runs)")
parser.add_argument(
"-" + _packages_path_arg[0],
"--" + _packages_path_arg_posix,
action="store",
nargs=1,
required=False,
metavar="/packages/path",
default=[None],
help="Desired path where to install external packages. "
"Optional if one has been set globally or as an env variable"
" (run with '--show_%s' to check)." % _packages_path_arg_posix)
# MARKED FOR DEPRECATION IN v3.0
modules = "modules"
parser.add_argument("-" + modules[0],
"--" + modules,
action="store",
nargs=1,
required=False,
metavar="/packages/path",
default=[None],
help="To be deprecated! "
"Alias for %s, which should be used instead." %
_packages_path_arg_posix)
# END OF DEPRECATION BLOCK -- CONTINUES BELOW!
output_show_packages_path = resolve_packages_path()
if output_show_packages_path and os.environ.get(_packages_path_env):
output_show_packages_path += " (from env variable %r)" % _packages_path_env
elif output_show_packages_path:
output_show_packages_path += " (from config file)"
else:
output_show_packages_path = "(Not currently set.)"
parser.add_argument(
"--show-" + _packages_path_arg_posix,
action="version",
version=output_show_packages_path,
help="Prints default external packages installation folder "
"and exits.")
parser.add_argument(
"-" + _force[0],
"--" + _force,
action="store_true",
default=False,
help="Force re-installation of apparently installed packages.")
parser.add_argument(
"--skip",
action="store",
nargs="*",
metavar="keyword",
help="Keywords of components that will be skipped during "
"installation.")
parser.add_argument(
"--no-progress-bars",
action="store_true",
default=False,
help="No progress bars shown. Shorter logs (used in Travis).")
parser.add_argument("--just-check",
action="store_true",
default=False,
help="Just check whether components are installed.")
parser.add_argument(
"--no-set-global",
action="store_true",
default=False,
help="Do not store the installation path for later runs.")
group_just = parser.add_mutually_exclusive_group(required=False)
group_just.add_argument("-C",
"--just-code",
action="store_false",
default=True,
help="Install code of the components.",
dest=_data)
group_just.add_argument("-D",
"--just-data",
action="store_false",
default=True,
help="Install data of the components.",
dest=_code)
arguments = parser.parse_args()
# Configure the logger ASAP
logger_setup()
logger = logging.getLogger(__name__.split(".")[-1])
# Gather requests
infos = []
for f in arguments.files_or_components:
if f.lower() == "cosmo":
logger.info("Installing basic cosmological packages.")
from cobaya.cosmo_input import install_basic
infos += [install_basic]
elif f.lower() == "cosmo-tests":
logger.info("Installing *tested* cosmological packages.")
from cobaya.cosmo_input import install_tests
infos += [install_tests]
elif os.path.splitext(f)[1].lower() in _yaml_extensions:
from cobaya.input import load_input
infos += [load_input(f)]
else:
try:
kind = get_kind(f)
infos += [{kind: {f: None}}]
except Exception:
logger.warning("Could not identify component %r. Skipping.", f)
if not infos:
logger.info("Nothing to install.")
return
# MARKED FOR DEPRECATION IN v3.0
if getattr(arguments, modules) != [None]:
logger.warning(
"*DEPRECATION*: -m/--modules will be deprecated in favor of "
"-%s/--%s in the next version. Please, use that one instead.",
_packages_path_arg[0], _packages_path_arg_posix)
# BEHAVIOUR TO BE REPLACED BY ERROR:
if getattr(arguments, _packages_path_arg) == [None]:
setattr(arguments, _packages_path_arg, getattr(arguments, modules))
# END OF DEPRECATION BLOCK
# Launch installer
install(*infos,
path=getattr(arguments, _packages_path_arg)[0],
**{
arg: getattr(arguments, arg)
for arg in [
"force", _code, _data, "no_progress_bars", "just_check",
"no_set_global", "skip"
]
})
def install(*infos, **kwargs):
debug = kwargs.get(_debug)
if not log.root.handlers:
logger_setup()
path = kwargs.get("path")
if not path:
path = resolve_packages_path(infos)
if not path:
raise LoggedError(
log, "No 'path' argument given, and none could be found in input infos "
"(as %r), the %r env variable or the config file. "
"Maybe specify one via a command line argument '-%s [...]'?",
_packages_path, _packages_path_env, _packages_path_arg[0])
abspath = os.path.abspath(path)
log.info("Installing external packages at '%s'", abspath)
kwargs_install = {"force": kwargs.get("force", False),
"no_progress_bars": kwargs.get("no_progress_bars")}
for what in (_code, _data):
kwargs_install[what] = kwargs.get(what, True)
spath = os.path.join(abspath, what)
if kwargs_install[what] and not os.path.exists(spath):
try:
os.makedirs(spath)
except OSError:
raise LoggedError(
log, "Could not create the desired installation folder '%s'", spath)
failed_components = []
skip_keywords_arg = set(kwargs.get("skip", []) or [])
# NB: if passed with quotes as `--skip "a b"`, it's interpreted as a single key
skip_keywords_arg = set(chain(*[word.split() for word in skip_keywords_arg]))
skip_keywords_env = set(
os.environ.get(_install_skip_env, "").replace(",", " ").lower().split())
skip_keywords = skip_keywords_arg.union(skip_keywords_env)
for kind, components in get_used_components(*infos).items():
for component in components:
print()
print(create_banner(kind + ":" + component,
symbol=_banner_symbol, length=_banner_length), end="")
print()
if _skip_helper(component.lower(), skip_keywords, skip_keywords_env, log):
continue
info = (next(info for info in infos if component in
info.get(kind, {}))[kind][component]) or {}
if isinstance(info, str) or _external in info:
log.warning("Component '%s' is a custom function. "
"Nothing to do.", component)
continue
try:
imported_class = get_class(component, kind,
component_path=info.pop(_component_path, None))
except ImportError as excpt:
log.error("Component '%s' not recognized. [%s].", component, excpt)
failed_components += ["%s:%s" % (kind, component)]
continue
else:
if _skip_helper(imported_class.__name__.lower(), skip_keywords,
skip_keywords_env, log):
continue
is_compatible = getattr(imported_class, "is_compatible", lambda: True)()
if not is_compatible:
log.info(
"Skipping %r because it is not compatible with your OS.", component)
continue
log.info("Checking if dependencies have already been installed...")
is_installed = getattr(imported_class, "is_installed", None)
if is_installed is None:
log.info("%s.%s is a fully built-in component: nothing to do.",
kind, imported_class.__name__)
continue
install_path = abspath
get_path = getattr(imported_class, "get_path", None)
if get_path:
install_path = get_path(install_path)
has_been_installed = False
if not debug:
logging.disable(logging.ERROR)
if kwargs.get("skip_global"):
has_been_installed = is_installed(path="global", **kwargs_install)
if not has_been_installed:
has_been_installed = is_installed(path=install_path, **kwargs_install)
if not debug:
logging.disable(logging.NOTSET)
if has_been_installed:
log.info("External dependencies for this component already installed.")
if kwargs.get(_test_run, False):
continue
if kwargs_install["force"] and not kwargs.get("skip_global"):
log.info("Forcing re-installation, as requested.")
else:
log.info("Doing nothing.")
continue
else:
log.info("Installation check failed!")
if not debug:
log.info(
"(If you expected this to be already installed, re-run "
"`cobaya-install` with --debug to get more verbose output.)")
if kwargs.get(_test_run, False):
continue
log.info("Installing...")
try:
install_this = getattr(imported_class, "install", None)
success = install_this(path=abspath, **kwargs_install)
except KeyboardInterrupt:
raise
except:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
log.error("An unknown error occurred. Delete the external packages "
"folder %r and try again. "
"Please, notify the developers if this error persists.",
abspath)
success = False
if success:
log.info("Successfully installed! Let's check it...")
else:
log.error("Installation failed! Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve, "
"install the packages required by this component manually.")
failed_components += ["%s:%s" % (kind, component)]
continue
# test installation
if not debug:
logging.disable(logging.ERROR)
successfully_installed = is_installed(path=install_path, **kwargs_install)
if not debug:
logging.disable(logging.NOTSET)
if not successfully_installed:
log.error("Installation apparently worked, "
"but the subsequent installation test failed! "
"Look at the error messages above, or re-run with --debug "
"for more more verbose output. "
"Try to solve the issues and try again, or, if you are unable "
"to solve them, install the packages required by this "
"component manually.")
failed_components += ["%s:%s" % (kind, component)]
else:
log.info("Installation check successful.")
print()
print(create_banner(" * Summary * ",
symbol=_banner_symbol, length=_banner_length), end="")
print()
if failed_components:
bullet = "\n - "
raise LoggedError(
log, "The installation (or installation test) of some component(s) has "
"failed: %s\nCheck output of the installer of each component above "
"for precise error info.\n",
bullet + bullet.join(failed_components))
log.info("All requested components' dependencies correctly installed.")
# Set the installation path in the global config file
if not kwargs.get("no_set_global", False) and not kwargs.get(_test_run, False):
write_packages_path_in_config_file(abspath)
log.info("The installation path has been written into the global config file: %s",
os.path.join(get_config_path(), _packages_path_config_file))
def install_script(args=None):
set_mpi_disabled()
warn_deprecation()
# Parse arguments
import argparse
parser = argparse.ArgumentParser(
prog="cobaya install",
description="Cobaya's installation tool for external packages.")
parser.add_argument(
"files_or_components",
action="store",
nargs="+",
metavar="input_file.yaml|component_name",
help="One or more input files or component names "
"(or simply 'cosmo' to install all the requisites for basic"
" cosmological runs)")
parser.add_argument(
"-" + packages_path_arg[0],
"--" + packages_path_arg_posix,
action="store",
required=False,
metavar="/packages/path",
default=None,
help="Desired path where to install external packages. "
"Optional if one has been set globally or as an env variable"
" (run with '--show_%s' to check)." % packages_path_arg_posix)
# MARKED FOR DEPRECATION IN v3.0
modules = "modules"
parser.add_argument("-" + modules[0],
"--" + modules,
action="store",
required=False,
metavar="/packages/path",
default=None,
help="Deprecated! Use %s instead." %
packages_path_arg_posix)
# END OF DEPRECATION BLOCK -- CONTINUES BELOW!
output_show_packages_path = resolve_packages_path()
if output_show_packages_path and os.environ.get(packages_path_env):
output_show_packages_path += " (from env variable %r)" % packages_path_env
elif output_show_packages_path:
output_show_packages_path += " (from config file)"
else:
output_show_packages_path = "(Not currently set.)"
parser.add_argument(
"--show-" + packages_path_arg_posix,
action="version",
version=output_show_packages_path,
help="Prints default external packages installation folder "
"and exits.")
parser.add_argument(
"-" + "f",
"--" + "force",
action="store_true",
default=False,
help="Force re-installation of apparently installed packages.")
parser.add_argument(
"--skip",
action="store",
nargs="*",
metavar="keyword",
help="Keywords of components that will be skipped during "
"installation.")
parser.add_argument(
"--no-progress-bars",
action="store_true",
default=False,
help="No progress bars shown. Shorter logs (used in Travis).")
parser.add_argument("--%s" % "test",
action="store_true",
default=False,
help="Just check whether components are installed.")
# MARKED FOR DEPRECATION IN v3.0
parser.add_argument("--just-check",
action="store_true",
default=False,
help="Just check whether components are installed.")
# END OF DEPRECATION BLOCK -- CONTINUES BELOW!
parser.add_argument(
"--no-set-global",
action="store_true",
default=False,
help="Do not store the installation path for later runs.")
parser.add_argument(
"--skip-global",
action="store_true",
default=False,
help="Skip installation of already-available Python modules.")
parser.add_argument("-" + "d",
"--" + "debug",
action="store_true",
help="Produce verbose debug output.")
group_just = parser.add_mutually_exclusive_group(required=False)
group_just.add_argument("-C",
"--just-code",
action="store_false",
default=True,
help="Install code of the components.",
dest=data_path)
group_just.add_argument("-D",
"--just-data",
action="store_false",
default=True,
help="Install data of the components.",
dest=code_path)
arguments = parser.parse_args(args)
# Configure the logger ASAP
logger_setup()
logger = get_logger("install")
# Gather requests
infos: List[InputDict] = []
for f in arguments.files_or_components:
if f.lower() == "cosmo":
logger.info("Installing basic cosmological packages.")
from cobaya.cosmo_input import install_basic
infos += [install_basic]
elif f.lower() == "cosmo-tests":
logger.info("Installing *tested* cosmological packages.")
from cobaya.cosmo_input import install_tests
infos += [install_tests]
elif os.path.splitext(f)[1].lower() in Extension.yamls:
from cobaya.input import load_input
infos += [load_input(f)]
else:
try:
kind = get_kind(f)
infos += [{kind: {f: None}}]
except Exception:
logger.warning("Could not identify component %r. Skipping.", f)
if not infos:
logger.info("Nothing to install.")
return
# List of deprecation warnings, to be printed *after* installation
deprecation_warnings = []
# MARKED FOR DEPRECATION IN v3.0
if getattr(arguments, modules) is not None:
raise LoggedError(
logger, "-m/--modules has been deprecated in favor of "
"-%s/--%s", packages_path_arg[0], packages_path_arg_posix)
# END OF DEPRECATION BLOCK
# MARKED FOR DEPRECATION IN v3.0
if arguments.just_check is True:
raise LoggedError(logger,
"--just-check has been deprecated in favor of --%s",
"test")
# END OF DEPRECATION BLOCK
# Launch installer
install(*infos,
path=getattr(arguments, packages_path_arg),
**{
arg: getattr(arguments, arg)
for arg in [
"force", code_path, data_path, "no_progress_bars", "test",
"no_set_global", "skip", "skip_global", "debug"
]
})
# MARKED FOR DEPRECATION IN v3.0
for warning_msg in deprecation_warnings:
logger.warning(warning_msg)
def install_script(args=None):
"""Command line script for the installer."""
set_mpi_disabled()
warn_deprecation()
# Parse arguments
import argparse
parser = argparse.ArgumentParser(
prog="cobaya install",
description="Cobaya's installation tool for external packages.")
parser.add_argument(
"files_or_components",
action="store",
nargs="+",
metavar="input_file.yaml|component_name",
help="One or more input files or component names "
"(or simply 'cosmo' to install all the requisites for basic"
" cosmological runs)")
parser.add_argument(
"-" + packages_path_arg[0],
"--" + packages_path_arg_posix,
action="store",
required=False,
metavar="/packages/path",
default=None,
help="Desired path where to install external packages. "
"Optional if one has been set globally or as an env variable"
" (run with '--show_%s' to check)." % packages_path_arg_posix)
output_show_packages_path = resolve_packages_path()
if output_show_packages_path and os.environ.get(packages_path_env):
output_show_packages_path += " (from env variable %r)" % packages_path_env
elif output_show_packages_path:
output_show_packages_path += " (from config file)"
else:
output_show_packages_path = "(Not currently set.)"
parser.add_argument(
"--show-" + packages_path_arg_posix,
action="version",
version=output_show_packages_path,
help="Prints default external packages installation folder "
"and exits.")
parser.add_argument(
"-" + "f",
"--" + "force",
action="store_true",
default=False,
help="Force re-installation of apparently installed packages.")
parser.add_argument("--%s" % "test",
action="store_true",
default=False,
help="Just check whether components are installed.")
parser.add_argument("--upgrade",
action="store_true",
default=False,
help="Force upgrade of obsolete components.")
parser.add_argument("--skip",
action="store",
nargs="*",
metavar="keyword",
help=("Keywords of components that will be "
"skipped during installation."))
parser.add_argument(
"--skip-global",
action="store_true",
default=False,
help="Skip installation of already-available Python modules.")
parser.add_argument("-" + "d",
"--" + "debug",
action="store_true",
help="Produce verbose debug output.")
group_just = parser.add_mutually_exclusive_group(required=False)
group_just.add_argument("-C",
"--just-code",
action="store_false",
default=True,
help="Install code of the components.",
dest=data_path)
group_just.add_argument("-D",
"--just-data",
action="store_false",
default=True,
help="Install data of the components.",
dest=code_path)
parser.add_argument(
"--no-progress-bars",
action="store_true",
default=False,
help=("No progress bars shown; use when output is saved into a "
"text file (e.g. when running on a cluster)."))
parser.add_argument(
"--no-set-global",
action="store_true",
default=False,
help="Do not store the installation path for later runs.")
arguments = parser.parse_args(args)
# Configure the logger ASAP
logger_setup(arguments.debug)
logger = get_logger("install")
# Gather requests
infos: List[Union[InputDict, str]] = []
for f in arguments.files_or_components:
if f.lower() == "cosmo":
logger.info("Installing basic cosmological packages.")
from cobaya.cosmo_input import install_basic
infos += [install_basic]
elif f.lower() == "cosmo-tests":
logger.info("Installing *tested* cosmological packages.")
from cobaya.cosmo_input import install_tests
infos += [install_tests]
elif os.path.splitext(f)[1].lower() in Extension.yamls:
from cobaya.input import load_input
infos += [load_input(f)]
else: # a single component name, no kind specified
infos += [f]
# Launch installer
install(*infos,
path=getattr(arguments, packages_path_arg),
logger=logger,
**{
arg: getattr(arguments, arg)
for arg in [
"force", code_path, data_path, "no_progress_bars", "test",
"no_set_global", "skip", "skip_global", "debug", "upgrade"
]
})
def install(*infos, **kwargs):
"""
Installs the external packages required by the components mentioned in ``infos``.
``infos`` can be input dictionaries or single component names.
:param force: force re-installation of apparently installed packages (default:
``False``).
:param test: just check whether components are installed (default: ``False``).
:param upgrade: force upgrade of obsolete components (default: ``False``).
:param skip: keywords of components that will be skipped during installation.
:param skip_global: skip installation of already-available Python modules (default:
``False``).
:param debug: produce verbose debug output (default: ``False``).
:param code: set to ``False`` to skip code packages (default: ``True``).
:param data: set to ``False`` to skip data packages (default: ``True``).
:param no_progress_bars: no progress bars shown; use when output is saved into a text
file (e.g. when running on a cluster) (default: ``False``).
:param no_set_global: do not store the installation path for later runs (default:
``False``).
"""
debug = kwargs.get("debug", False)
logger = kwargs.get("logger")
if not logger:
logger_setup(debug=debug)
logger = get_logger("install")
path = kwargs.get("path")
infos_not_single_names = [
info for info in infos if isinstance(info, Mapping)
]
if not path:
path = resolve_packages_path(*infos_not_single_names)
if not path:
raise LoggedError(logger, (
"No 'path' argument given, and none could be found in input infos "
"(as %r), the %r env variable or the config file. "
"Maybe specify one via a command line argument '-%s [...]'?"),
packages_path_input, packages_path_env,
packages_path_arg[0])
# General install path for all dependencies
general_abspath = os.path.abspath(path)
logger.info("Installing external packages at '%s'", general_abspath)
# Set the installation path in the global config file
if not kwargs.get("no_set_global", False) and not kwargs.get(
"test", False):
write_packages_path_in_config_file(general_abspath)
logger.info(
"The installation path has been written into the global config file: %s",
os.path.join(get_config_path(), packages_path_config_file))
kwargs_install = {
"force": kwargs.get("force", False),
"no_progress_bars": kwargs.get("no_progress_bars")
}
for what in (code_path, data_path):
kwargs_install[what] = kwargs.get(what, True)
spath = os.path.join(general_abspath, what)
if kwargs_install[what] and not os.path.exists(spath):
try:
os.makedirs(spath)
except OSError:
raise LoggedError(
logger,
f"Could not create the desired installation folder '{spath}'"
)
# To check e.g. for a version upgrade, it needs to reload the component class and
# all relevant imported modules: the implementation of `is_installed` for each
# class is expected to always reload external modules if passed `reload=True`
# (should be False by default to avoid deleting objects unnecessarily).
kwargs_is_installed = {"reload": True}
unknown_components = [] # could not be identified
failed_components = [] # general errors
obsolete_components = [] # older or unknown version already installed
skip_keywords_arg = set(kwargs.get("skip", []) or [])
# NB: if passed with quotes as `--skip "a b"`, it's interpreted as a single key
skip_keywords_arg = set(
chain(*[word.split() for word in skip_keywords_arg]))
skip_keywords_env = set(
os.environ.get(install_skip_env, "").replace(",", " ").lower().split())
skip_keywords = skip_keywords_arg.union(skip_keywords_env)
# Combine all requested components and install them
# NB: components mentioned by name may be repeated with those given in dict infos.
# That's OK, because the install check will skip them in the 2nd pass
used_components, components_infos = get_used_components(*infos,
return_infos=True)
for kind, components in used_components.items():
for component in components:
name_w_kind = (kind + ":" if kind else "") + component
print()
print(create_banner(name_w_kind,
symbol=_banner_symbol,
length=_banner_length),
end="")
print()
if _skip_helper(component.lower(), skip_keywords,
skip_keywords_env, logger):
continue
info = components_infos[component]
if isinstance(info, str) or "external" in info:
logger.info(
f"Component '{name_w_kind}' is a custom function. Nothing to do."
)
continue
try:
class_name = (info or {}).get("class")
if class_name:
logger.info(
f"Class to be installed for this component: {class_name}"
)
imported_class = get_component_class(component,
kind=kind,
component_path=info.pop(
"python_path", None),
class_name=class_name,
logger=logger)
# Update the name if the kind was unknown
if not kind:
name_w_kind = imported_class.get_kind() + ":" + component
except ComponentNotFoundError:
logger.error(
f"Component '{name_w_kind}' could not be identified. Skipping."
)
unknown_components += [name_w_kind]
continue
except Exception:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
logger.error(
f"An error occurred when loading '{name_w_kind}'. Skipping."
)
failed_components += [name_w_kind]
continue
else:
if _skip_helper(imported_class.__name__.lower(), skip_keywords,
skip_keywords_env, logger):
continue
is_compatible = getattr(imported_class, "is_compatible",
lambda: True)()
if not is_compatible:
logger.error(f"Skipping '{name_w_kind}' "
"because it is not compatible with your OS.")
failed_components += [name_w_kind]
continue
logger.info(
"Checking if dependencies have already been installed...")
is_installed = getattr(imported_class, "is_installed", None)
if is_installed is None:
logger.info(
f"Component '{name_w_kind}' is a fully built-in component: "
"nothing to do.")
continue
this_component_install_path = general_abspath
get_path = getattr(imported_class, "get_path", None)
if get_path:
this_component_install_path = get_path(
this_component_install_path)
# Check previous installations and their versions
has_been_installed = False
is_old_version_msg = None
with NoLogging(None if debug else logging.ERROR):
try:
if kwargs.get("skip_global"):
has_been_installed = is_installed(
path="global",
**kwargs_install,
**kwargs_is_installed)
if not has_been_installed:
has_been_installed = is_installed(
path=this_component_install_path,
**kwargs_install,
**kwargs_is_installed)
except VersionCheckError as excpt:
is_old_version_msg = str(excpt)
if has_been_installed: # no VersionCheckError was raised
logger.info(
"External dependencies for this component already installed."
)
if kwargs.get("test", False):
continue
if kwargs_install["force"] and not kwargs.get("skip_global"):
logger.info("Forcing re-installation, as requested.")
else:
logger.info("Doing nothing.")
continue
elif is_old_version_msg:
logger.info(f"Version check failed: {is_old_version_msg}")
obsolete_components += [name_w_kind]
if kwargs.get("test", False):
continue
if not kwargs.get("upgrade", False) and not kwargs.get(
"force", False):
logger.info("Skipping because '--upgrade' not requested.")
continue
else:
logger.info("Check found no existing installation")
if not debug:
logger.info(
"(If you expected this to be already installed, re-run "
"`cobaya-install` with --debug to get more verbose output.)"
)
if kwargs.get("test", False):
# We are only testing whether it was installed, so consider it failed
failed_components += [name_w_kind]
continue
# Do the install
logger.info("Installing...")
try:
install_this = getattr(imported_class, "install", None)
success = install_this(path=general_abspath, **kwargs_install)
except Exception:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
logger.error(
"An unknown error occurred. Delete the external packages "
"folder %r and try again. "
"Please, notify the developers if this error persists.",
general_abspath)
success = False
if success:
logger.info("Successfully installed! Let's check it...")
else:
logger.error(
"Installation failed! Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve them, "
"install the packages required by this component manually."
)
failed_components += [name_w_kind]
continue
# Test installation
reloaded_class = get_component_class(component,
kind=kind,
component_path=info.pop(
"python_path", None),
class_name=class_name,
logger=logger)
reloaded_is_installed = getattr(reloaded_class, "is_installed",
None)
with NoLogging(None if debug else logging.ERROR):
try:
successfully_installed = reloaded_is_installed(
path=this_component_install_path,
**kwargs_install,
**kwargs_is_installed)
except Exception:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
successfully_installed = False
if not successfully_installed:
logger.error(
"Installation apparently worked, "
"but the subsequent installation test failed! "
"This does not always mean that there was an actual error, "
"and is sometimes fixed simply by running the installer "
"again. If not, look closely at the error messages above, "
"or re-run with --debug for more more verbose output. "
"If you are unable to fix the issues above, "
"try installing the packages required by this "
"component manually.")
failed_components += [name_w_kind]
else:
logger.info("Installation check successful.")
print()
print(create_banner(" * Summary * ",
symbol=_banner_symbol,
length=_banner_length),
end="")
print()
bullet = "\n - "
if unknown_components:
suggestions_dict = {
name: similar_internal_class_names(name)
for name in unknown_components
}
suggestions_msg = \
bullet + bullet.join(
f"{name}: did you mean any of the following? {sugg} "
"(mind capitalization!)" for name, sugg in suggestions_dict.items())
raise LoggedError(logger, (
"The following components could not be identified and were skipped:"
f"{suggestions_msg}"))
if failed_components:
raise LoggedError(logger, (
"The installation (or installation test) of some component(s) has "
"failed: %s\nCheck output of the installer of each component above "
"for precise error info.\n"),
bullet + bullet.join(failed_components))
if obsolete_components:
raise LoggedError(logger, (
"The following packages are obsolete. Re-run with `--upgrade` option"
" (not upgrading by default to preserve possible user changes): %s"
), bullet + bullet.join(obsolete_components))
if not unknown_components and not failed_components and not obsolete_components:
logger.info(
"All requested components' dependencies correctly installed at "
f"{general_abspath}")
def pytest_addoption(parser):
parser.addoption(
"--" + _packages_path_arg,
action="store",
default=resolve_packages_path(),
help="Path to folder of automatic installation of packages")
