def get_bib_info(*infos, logger=None):
"""
Gathers and returns the descriptions and bibliographic sources for the components
mentioned in ``infos``.
``infos`` can be input dictionaries or single component names.
"""
if not logger:
logger_setup()
logger = get_logger("bib")
used_components, component_infos = get_used_components(*infos,
return_infos=True)
descs: InfoDict = {}
bibs: InfoDict = {}
used_components = get_used_components(*infos)
for kind, components in used_components.items():
if kind is None:
continue # we will deal with bare component names later, to avoid repetition
descs[kind], bibs[kind] = {}, {}
for component in components:
try:
descs[kind][component] = get_desc_component(
component, kind, component_infos[component])
bibs[kind][component] = get_bib_component(component, kind)
except ComponentNotFoundError:
sugg = similar_internal_class_names(component)
logger.error(
f"Could not identify component '{component}'. "
f"Did you mean any of the following? {sugg} (mind capitalization!)"
)
continue
# Deal with bare component names
for component in used_components.get(None, []):
try:
cls = get_component_class(component)
except ComponentNotFoundError:
sugg = similar_internal_class_names(component)
logger.error(
f"Could not identify component '{component}'. "
f"Did you mean any of the following? {sugg} (mind capitalization!)"
)
continue
kind = cls.get_kind()
if kind not in descs:
descs[kind], bibs[kind] = {}, {}
if kind in descs and component in descs[kind]:
continue # avoid repetition
descs[kind][component] = get_desc_component(cls, kind)
bibs[kind][component] = get_bib_component(cls, kind)
descs["cobaya"] = {"cobaya": cobaya_desc}
bibs["cobaya"] = {"cobaya": cobaya_bib}
return descs, bibs
def get_bib_info(*infos):
used_components, component_infos = get_used_components(*infos, return_infos=True)
descs = {}
bibs = {}
for kind, components in get_used_components(*infos).items():
descs[kind], bibs[kind] = {}, {}
for component in components:
descs[kind][component] = get_desc_component(
component, kind, component_infos[component])
bibs[kind][component] = get_bib_component(component, kind)
descs["cobaya"] = {"cobaya": cobaya_desc}
bibs["cobaya"] = {"cobaya": cobaya_bib}
return descs, bibs
def create_docker_image(filenames, MPI_version=None):
log.info("Creating Docker image...")
if not MPI_version:
MPI_version = "3.2"
# log.warning("You have not specified an MPICH version. "
# "It is strongly encouraged to request the one installed in your cluster,"
# " using '--mpi-version X.Y'. Defaulting to MPICH v%s.", MPI_version)
dc = get_docker_client()
components = yaml_dump(
get_used_components(*[load_input(f) for f in filenames])).strip()
echos_reqs = "RUN " + " && \\ \n ".join([
r'echo "%s" >> %s' % (block, requirements_file_path)
for block in components.split("\n")
])
echos_help = "RUN " + " && \\ \n ".join([
r'echo "%s" >> %s' % (line, help_file_path)
for line in image_help("docker").split("\n")
])
recipe = r"""
FROM cobaya/base_mpich_%s:latest
%s
RUN cobaya-install %s --%s %s --just-code --force ### NEEDS PYTHON UPDATE! --no-progress-bars
%s
CMD ["cat", "%s"]
""" % (MPI_version, echos_reqs, requirements_file_path, _packages_path_arg,
_packages_path, echos_help, help_file_path)
image_name = "cobaya:" + uuid.uuid4().hex[:6]
with StringIO(recipe) as stream:
dc.images.build(fileobj=stream, tag=image_name)
log.info(
"Docker image '%s' created! "
"Do 'docker save %s | gzip > some_name.tar.gz'"
"to save it to the current folder.", image_name, image_name)
def get_bib_info(*infos):
blocks_text = {"Cobaya": cobaya_bib}
for kind, components in get_used_components(*infos).items():
for component in components:
blocks_text["%s:%s" % (kind, component)] = get_bib_component(
component, kind)
return blocks_text
def create_singularity_image(filenames, MPI_version=None):
log.info("Creating Singularity image...")
if not MPI_version:
MPI_version = "2.1.1"
# log.warning("You have not specified an OpenMPI version. "
# "It is strongly encouraged to request the one installed in your cluster,"
# " using '--mpi-version X.Y.Z'. Defaulting to OpenMPI v%s.", MPI_version)
components = yaml_dump(
get_used_components(*[load_input(f) for f in filenames])).strip()
echos_reqs = "\n " + "\n ".join([""] + [
'echo "%s" >> %s' % (block, requirements_file_path)
for block in components.split("\n")
])
recipe = (
dedent("""
Bootstrap: docker
From: cobaya/base_openmpi_%s:latest\n
%%post\n""" % MPI_version) + dedent(echos_reqs) + dedent("""
export CONTAINED=TRUE
cobaya-install %s --%s %s --just-code --force ### --no-progress-bars
mkdir $COBAYA_PRODUCTS
%%help
%s
""" % (
requirements_file_path,
# TODO: this looks wrong?
packages_path_input,
os.path.join(packages_path_arg, packages_path_input, data_path),
"\n ".join(image_help("singularity").split("\n")[1:]))))
with NamedTemporaryFile(delete=False) as recipe_file:
recipe_file.write(recipe.encode('utf-8'))
recipe_file_name = recipe_file.name
image_name = "cobaya_" + uuid.uuid4().hex[:6] + ".simg"
process_build = Popen(
["singularity", "build", image_name, recipe_file_name],
stdout=PIPE,
stderr=PIPE)
out, err = process_build.communicate()
if process_build.returncode:
log.info(out)
log.info(err)
raise LoggedError(log,
"Image creation failed! See error message above.")
log.info("Singularity image '%s' created!", image_name)
def install(*infos, **kwargs):
if not log.root.handlers:
logger_setup()
path = kwargs.get("path")
if not path:
path = resolve_packages_path(infos)
if not path:
raise LoggedError(
log,
"No 'path' argument given, and none could be found in input infos "
"(as %r), the %r env variable or the config file. "
"Maybe specify one via a command line argument '-%s [...]'?",
_packages_path, _packages_path_env, _packages_path_arg[0])
abspath = os.path.abspath(path)
log.info("Installing external packages at '%s'", abspath)
kwargs_install = {
"force": kwargs.get("force", False),
"no_progress_bars": kwargs.get("no_progress_bars")
}
for what in (_code, _data):
kwargs_install[what] = kwargs.get(what, True)
spath = os.path.join(abspath, what)
if kwargs_install[what] and not os.path.exists(spath):
try:
os.makedirs(spath)
except OSError:
raise LoggedError(
log,
"Could not create the desired installation folder '%s'",
spath)
failed_components = []
skip_keywords = set(kwargs.get("skip", []) or [])
skip_keywords_env = set(
os.environ.get(_install_skip_env, "").replace(",",
" ").lower().split())
skip_keywords = skip_keywords.union(skip_keywords_env)
for kind, components in get_used_components(*infos).items():
for component in components:
print()
print(create_banner(kind + ":" + component, symbol="=", length=80))
print()
if _skip_helper(component.lower(), skip_keywords,
skip_keywords_env, log):
continue
info = (next(
info for info in infos
if component in info.get(kind, {}))[kind][component]) or {}
if isinstance(info, str) or _external in info:
log.warning(
"Component '%s' is a custom function. "
"Nothing to do.", component)
continue
try:
imported_class = get_class(component,
kind,
component_path=info.pop(
_component_path, None))
except ImportError as e:
log.error("Component '%s' not recognized. [%s]." %
(component, e))
failed_components += ["%s:%s" % (kind, component)]
continue
else:
if _skip_helper(imported_class.__name__.lower(), skip_keywords,
skip_keywords_env, log):
continue
is_installed = getattr(imported_class, "is_installed", None)
if is_installed is None:
log.info("%s.%s is a fully built-in component: nothing to do.",
kind, imported_class.__name__)
continue
if is_installed(path=abspath, **kwargs_install):
log.info("External component already installed.")
if kwargs.get("just_check", False):
continue
if kwargs_install["force"]:
log.info("Forcing re-installation, as requested.")
else:
log.info("Doing nothing.")
continue
else:
if kwargs.get("just_check", False):
log.info("NOT INSTALLED!")
continue
try:
install_this = getattr(imported_class, "install", None)
success = install_this(path=abspath, **kwargs_install)
except KeyboardInterrupt:
raise
except:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
log.error(
"An unknown error occurred. Delete the external packages "
"folder %r and try again. "
"Please, notify the developers if this error persists.",
abspath)
success = False
if success:
log.info("Successfully installed!")
else:
log.error(
"Installation failed! Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve, "
"install the packages required by this component manually."
)
failed_components += ["%s:%s" % (kind, component)]
continue
# test installation
if not is_installed(path=abspath, **kwargs_install):
log.error(
"Installation apparently worked, "
"but the subsequent installation test failed! "
"Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve, "
"install the packages required by this component manually."
)
failed_components += ["%s:%s" % (kind, component)]
if failed_components:
bullet = "\n - "
raise LoggedError(
log,
"The installation (or installation test) of some component(s) has "
"failed: %s\nCheck output of the installer of each component above "
"for precise error info.\n",
bullet + bullet.join(failed_components))
# Set the installation path in the global config file
if not kwargs.get("no_set_global", False) and not kwargs.get(
"just_check", False):
write_packages_path_in_config_file(abspath)
log.info(
"The installation path has been written in the global config file."
)
def makeGrid(batchPath,
settingName=None,
settings=None,
read_only=False,
interactive=False,
install_reqs_at=None,
install_reqs_force=None):
print("Generating grid...")
batchPath = os.path.abspath(batchPath) + os.sep
if not settings:
if not settingName:
raise NotImplementedError(
"Re-using previous batch is work in progress...")
# if not pathIsGrid(batchPath):
# raise Exception('Need to give name of setting file if batchPath/config '
# 'does not exist')
# read_only = True
# sys.path.insert(0, batchPath + 'config')
# settings = __import__(IniFile(batchPath + 'config/config.ini').params['setting_file'].replace('.py', ''))
elif os.path.splitext(settingName)[-1].lower() in _yaml_extensions:
settings = yaml_load_file(settingName)
else:
raise NotImplementedError(
"Using a python script is work in progress...")
# In this case, info-as-dict would be passed
# settings = __import__(settingName, fromlist=['dummy'])
batch = batchjob.BatchJob(batchPath)
# batch.skip = settings.get("skip", False)
batch.makeItems(settings, messages=not read_only)
if read_only:
for jobItem in [b for b in batch.jobItems]:
if not jobItem.chainExists():
batch.jobItems.remove(jobItem)
batch.save()
print('OK, configured grid with %u existing chains' %
(len(batch.jobItems)))
return batch
else:
batch.makeDirectories(setting_file=None)
batch.save()
infos = {}
components_used = {}
# Default info
defaults = copy.deepcopy(settings)
grid_definition = defaults.pop("grid")
models_definitions = grid_definition["models"]
datasets_definitions = grid_definition["datasets"]
for jobItem in batch.items(wantSubItems=False):
# Model info
jobItem.makeChainPath()
try:
model_info = copy.deepcopy(models_definitions[jobItem.param_set]
or {})
except KeyError:
raise ValueError("Model '%s' must be defined." % jobItem.param_set)
model_info = merge_info(defaults, model_info)
# Dataset info
try:
dataset_info = copy.deepcopy(
datasets_definitions[jobItem.data_set.tag])
except KeyError:
raise ValueError("Data set '%s' must be defined." %
jobItem.data_set.tag)
# Combined info
combined_info = merge_info(defaults, model_info, dataset_info)
if "preset" in combined_info:
preset = combined_info.pop("preset")
combined_info = merge_info(create_input(**preset), combined_info)
combined_info[_output_prefix] = jobItem.chainRoot
# Requisites
components_used = get_used_components(components_used, combined_info)
if install_reqs_at:
combined_info[_packages_path] = os.path.abspath(install_reqs_at)
# Save the info (we will write it after installation:
# we need to install to add auto covmats
if jobItem.param_set not in infos:
infos[jobItem.param_set] = {}
infos[jobItem.param_set][jobItem.data_set.tag] = combined_info
# Installing requisites
if install_reqs_at:
print("Installing required code and data for the grid.")
from cobaya.log import logger_setup
logger_setup()
install_reqs(components_used,
path=install_reqs_at,
force=install_reqs_force)
print("Adding covmats (if necessary) and writing input files")
for jobItem in batch.items(wantSubItems=False):
info = infos[jobItem.param_set][jobItem.data_set.tag]
# Covariance matrices
# We try to find them now, instead of at run time, to check if correctly selected
try:
sampler = list(info[kinds.sampler])[0]
except KeyError:
raise ValueError("No sampler has been chosen")
if sampler == "mcmc" and info[kinds.sampler][sampler].get(
"covmat", "auto"):
packages_path = install_reqs_at or info.get(_packages_path, None)
if not packages_path:
raise ValueError(
"Cannot assign automatic covariance matrices because no "
"external packages path has been defined.")
# Need updated info for covmats: includes renames
updated_info = update_info(info)
# Ideally, we use slow+sampled parameters to look for the covariance matrix
# but since for that we'd need to initialise a model, we approximate that set
# as theory+sampled
from itertools import chain
like_params = set(
chain(*[
list(like[_params])
for like in updated_info[kinds.likelihood].values()
]))
params_info = {
p: v
for p, v in updated_info[_params].items()
if is_sampled_param(v) and p not in like_params
}
best_covmat = _get_best_covmat(os.path.abspath(packages_path),
params_info,
updated_info[kinds.likelihood])
info[kinds.sampler][sampler]["covmat"] = os.path.join(
best_covmat["folder"], best_covmat["name"])
# Write the info for this job
# Allow overwrite since often will want to regenerate grid with tweaks
yaml_dump_file(jobItem.iniFile(),
sort_cosmetic(info),
error_if_exists=False)
# Non-translated old code
# if not start_at_bestfit:
# setMinimize(jobItem, ini)
# variant = '_minimize'
# ini.saveFile(jobItem.iniFile(variant))
## NOT IMPLEMENTED: start at best fit
## ini.params['start_at_bestfit'] = start_at_bestfit
# ---
# for deffile in settings.defaults:
# ini.defaults.append(batch.commonPath + deffile)
# if hasattr(settings, 'override_defaults'):
# ini.defaults = [batch.commonPath + deffile for deffile in settings.override_defaults] + ini.defaults
# ---
# # add ini files for importance sampling runs
# for imp in jobItem.importanceJobs():
# if getattr(imp, 'importanceFilter', None): continue
# if batch.hasName(imp.name.replace('_post', '')):
# raise Exception('importance sampling something you already have?')
# for minimize in (False, True):
# if minimize and not getattr(imp, 'want_minimize', True): continue
# ini = IniFile()
# updateIniParams(ini, imp.importanceSettings, batch.commonPath)
# if cosmomcAction == 0 and not minimize:
# for deffile in settings.importanceDefaults:
# ini.defaults.append(batch.commonPath + deffile)
# ini.params['redo_outroot'] = imp.chainRoot
# ini.params['action'] = 1
# else:
# ini.params['file_root'] = imp.chainRoot
# if minimize:
# setMinimize(jobItem, ini)
# variant = '_minimize'
# else:
# variant = ''
# ini.defaults.append(jobItem.iniFile())
# ini.saveFile(imp.iniFile(variant))
# if cosmomcAction != 0: break
if not interactive:
return batch
print('Done... to run do: cobaya-grid-run %s' % batchPath)
def install(*infos, **kwargs):
debug = kwargs.get(_debug)
if not log.root.handlers:
logger_setup()
path = kwargs.get("path")
if not path:
path = resolve_packages_path(infos)
if not path:
raise LoggedError(
log, "No 'path' argument given, and none could be found in input infos "
"(as %r), the %r env variable or the config file. "
"Maybe specify one via a command line argument '-%s [...]'?",
_packages_path, _packages_path_env, _packages_path_arg[0])
abspath = os.path.abspath(path)
log.info("Installing external packages at '%s'", abspath)
kwargs_install = {"force": kwargs.get("force", False),
"no_progress_bars": kwargs.get("no_progress_bars")}
for what in (_code, _data):
kwargs_install[what] = kwargs.get(what, True)
spath = os.path.join(abspath, what)
if kwargs_install[what] and not os.path.exists(spath):
try:
os.makedirs(spath)
except OSError:
raise LoggedError(
log, "Could not create the desired installation folder '%s'", spath)
failed_components = []
skip_keywords_arg = set(kwargs.get("skip", []) or [])
# NB: if passed with quotes as `--skip "a b"`, it's interpreted as a single key
skip_keywords_arg = set(chain(*[word.split() for word in skip_keywords_arg]))
skip_keywords_env = set(
os.environ.get(_install_skip_env, "").replace(",", " ").lower().split())
skip_keywords = skip_keywords_arg.union(skip_keywords_env)
for kind, components in get_used_components(*infos).items():
for component in components:
print()
print(create_banner(kind + ":" + component,
symbol=_banner_symbol, length=_banner_length), end="")
print()
if _skip_helper(component.lower(), skip_keywords, skip_keywords_env, log):
continue
info = (next(info for info in infos if component in
info.get(kind, {}))[kind][component]) or {}
if isinstance(info, str) or _external in info:
log.warning("Component '%s' is a custom function. "
"Nothing to do.", component)
continue
try:
imported_class = get_class(component, kind,
component_path=info.pop(_component_path, None))
except ImportError as excpt:
log.error("Component '%s' not recognized. [%s].", component, excpt)
failed_components += ["%s:%s" % (kind, component)]
continue
else:
if _skip_helper(imported_class.__name__.lower(), skip_keywords,
skip_keywords_env, log):
continue
is_compatible = getattr(imported_class, "is_compatible", lambda: True)()
if not is_compatible:
log.info(
"Skipping %r because it is not compatible with your OS.", component)
continue
log.info("Checking if dependencies have already been installed...")
is_installed = getattr(imported_class, "is_installed", None)
if is_installed is None:
log.info("%s.%s is a fully built-in component: nothing to do.",
kind, imported_class.__name__)
continue
install_path = abspath
get_path = getattr(imported_class, "get_path", None)
if get_path:
install_path = get_path(install_path)
has_been_installed = False
if not debug:
logging.disable(logging.ERROR)
if kwargs.get("skip_global"):
has_been_installed = is_installed(path="global", **kwargs_install)
if not has_been_installed:
has_been_installed = is_installed(path=install_path, **kwargs_install)
if not debug:
logging.disable(logging.NOTSET)
if has_been_installed:
log.info("External dependencies for this component already installed.")
if kwargs.get(_test_run, False):
continue
if kwargs_install["force"] and not kwargs.get("skip_global"):
log.info("Forcing re-installation, as requested.")
else:
log.info("Doing nothing.")
continue
else:
log.info("Installation check failed!")
if not debug:
log.info(
"(If you expected this to be already installed, re-run "
"`cobaya-install` with --debug to get more verbose output.)")
if kwargs.get(_test_run, False):
continue
log.info("Installing...")
try:
install_this = getattr(imported_class, "install", None)
success = install_this(path=abspath, **kwargs_install)
except KeyboardInterrupt:
raise
except:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
log.error("An unknown error occurred. Delete the external packages "
"folder %r and try again. "
"Please, notify the developers if this error persists.",
abspath)
success = False
if success:
log.info("Successfully installed! Let's check it...")
else:
log.error("Installation failed! Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve, "
"install the packages required by this component manually.")
failed_components += ["%s:%s" % (kind, component)]
continue
# test installation
if not debug:
logging.disable(logging.ERROR)
successfully_installed = is_installed(path=install_path, **kwargs_install)
if not debug:
logging.disable(logging.NOTSET)
if not successfully_installed:
log.error("Installation apparently worked, "
"but the subsequent installation test failed! "
"Look at the error messages above, or re-run with --debug "
"for more more verbose output. "
"Try to solve the issues and try again, or, if you are unable "
"to solve them, install the packages required by this "
"component manually.")
failed_components += ["%s:%s" % (kind, component)]
else:
log.info("Installation check successful.")
print()
print(create_banner(" * Summary * ",
symbol=_banner_symbol, length=_banner_length), end="")
print()
if failed_components:
bullet = "\n - "
raise LoggedError(
log, "The installation (or installation test) of some component(s) has "
"failed: %s\nCheck output of the installer of each component above "
"for precise error info.\n",
bullet + bullet.join(failed_components))
log.info("All requested components' dependencies correctly installed.")
# Set the installation path in the global config file
if not kwargs.get("no_set_global", False) and not kwargs.get(_test_run, False):
write_packages_path_in_config_file(abspath)
log.info("The installation path has been written into the global config file: %s",
os.path.join(get_config_path(), _packages_path_config_file))
def install(*infos, **kwargs):
"""
Installs the external packages required by the components mentioned in ``infos``.
``infos`` can be input dictionaries or single component names.
:param force: force re-installation of apparently installed packages (default:
``False``).
:param test: just check whether components are installed (default: ``False``).
:param upgrade: force upgrade of obsolete components (default: ``False``).
:param skip: keywords of components that will be skipped during installation.
:param skip_global: skip installation of already-available Python modules (default:
``False``).
:param debug: produce verbose debug output (default: ``False``).
:param code: set to ``False`` to skip code packages (default: ``True``).
:param data: set to ``False`` to skip data packages (default: ``True``).
:param no_progress_bars: no progress bars shown; use when output is saved into a text
file (e.g. when running on a cluster) (default: ``False``).
:param no_set_global: do not store the installation path for later runs (default:
``False``).
"""
debug = kwargs.get("debug", False)
logger = kwargs.get("logger")
if not logger:
logger_setup(debug=debug)
logger = get_logger("install")
path = kwargs.get("path")
infos_not_single_names = [
info for info in infos if isinstance(info, Mapping)
]
if not path:
path = resolve_packages_path(*infos_not_single_names)
if not path:
raise LoggedError(logger, (
"No 'path' argument given, and none could be found in input infos "
"(as %r), the %r env variable or the config file. "
"Maybe specify one via a command line argument '-%s [...]'?"),
packages_path_input, packages_path_env,
packages_path_arg[0])
# General install path for all dependencies
general_abspath = os.path.abspath(path)
logger.info("Installing external packages at '%s'", general_abspath)
# Set the installation path in the global config file
if not kwargs.get("no_set_global", False) and not kwargs.get(
"test", False):
write_packages_path_in_config_file(general_abspath)
logger.info(
"The installation path has been written into the global config file: %s",
os.path.join(get_config_path(), packages_path_config_file))
kwargs_install = {
"force": kwargs.get("force", False),
"no_progress_bars": kwargs.get("no_progress_bars")
}
for what in (code_path, data_path):
kwargs_install[what] = kwargs.get(what, True)
spath = os.path.join(general_abspath, what)
if kwargs_install[what] and not os.path.exists(spath):
try:
os.makedirs(spath)
except OSError:
raise LoggedError(
logger,
f"Could not create the desired installation folder '{spath}'"
)
# To check e.g. for a version upgrade, it needs to reload the component class and
# all relevant imported modules: the implementation of `is_installed` for each
# class is expected to always reload external modules if passed `reload=True`
# (should be False by default to avoid deleting objects unnecessarily).
kwargs_is_installed = {"reload": True}
unknown_components = [] # could not be identified
failed_components = [] # general errors
obsolete_components = [] # older or unknown version already installed
skip_keywords_arg = set(kwargs.get("skip", []) or [])
# NB: if passed with quotes as `--skip "a b"`, it's interpreted as a single key
skip_keywords_arg = set(
chain(*[word.split() for word in skip_keywords_arg]))
skip_keywords_env = set(
os.environ.get(install_skip_env, "").replace(",", " ").lower().split())
skip_keywords = skip_keywords_arg.union(skip_keywords_env)
# Combine all requested components and install them
# NB: components mentioned by name may be repeated with those given in dict infos.
# That's OK, because the install check will skip them in the 2nd pass
used_components, components_infos = get_used_components(*infos,
return_infos=True)
for kind, components in used_components.items():
for component in components:
name_w_kind = (kind + ":" if kind else "") + component
print()
print(create_banner(name_w_kind,
symbol=_banner_symbol,
length=_banner_length),
end="")
print()
if _skip_helper(component.lower(), skip_keywords,
skip_keywords_env, logger):
continue
info = components_infos[component]
if isinstance(info, str) or "external" in info:
logger.info(
f"Component '{name_w_kind}' is a custom function. Nothing to do."
)
continue
try:
class_name = (info or {}).get("class")
if class_name:
logger.info(
f"Class to be installed for this component: {class_name}"
)
imported_class = get_component_class(component,
kind=kind,
component_path=info.pop(
"python_path", None),
class_name=class_name,
logger=logger)
# Update the name if the kind was unknown
if not kind:
name_w_kind = imported_class.get_kind() + ":" + component
except ComponentNotFoundError:
logger.error(
f"Component '{name_w_kind}' could not be identified. Skipping."
)
unknown_components += [name_w_kind]
continue
except Exception:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
logger.error(
f"An error occurred when loading '{name_w_kind}'. Skipping."
)
failed_components += [name_w_kind]
continue
else:
if _skip_helper(imported_class.__name__.lower(), skip_keywords,
skip_keywords_env, logger):
continue
is_compatible = getattr(imported_class, "is_compatible",
lambda: True)()
if not is_compatible:
logger.error(f"Skipping '{name_w_kind}' "
"because it is not compatible with your OS.")
failed_components += [name_w_kind]
continue
logger.info(
"Checking if dependencies have already been installed...")
is_installed = getattr(imported_class, "is_installed", None)
if is_installed is None:
logger.info(
f"Component '{name_w_kind}' is a fully built-in component: "
"nothing to do.")
continue
this_component_install_path = general_abspath
get_path = getattr(imported_class, "get_path", None)
if get_path:
this_component_install_path = get_path(
this_component_install_path)
# Check previous installations and their versions
has_been_installed = False
is_old_version_msg = None
with NoLogging(None if debug else logging.ERROR):
try:
if kwargs.get("skip_global"):
has_been_installed = is_installed(
path="global",
**kwargs_install,
**kwargs_is_installed)
if not has_been_installed:
has_been_installed = is_installed(
path=this_component_install_path,
**kwargs_install,
**kwargs_is_installed)
except VersionCheckError as excpt:
is_old_version_msg = str(excpt)
if has_been_installed: # no VersionCheckError was raised
logger.info(
"External dependencies for this component already installed."
)
if kwargs.get("test", False):
continue
if kwargs_install["force"] and not kwargs.get("skip_global"):
logger.info("Forcing re-installation, as requested.")
else:
logger.info("Doing nothing.")
continue
elif is_old_version_msg:
logger.info(f"Version check failed: {is_old_version_msg}")
obsolete_components += [name_w_kind]
if kwargs.get("test", False):
continue
if not kwargs.get("upgrade", False) and not kwargs.get(
"force", False):
logger.info("Skipping because '--upgrade' not requested.")
continue
else:
logger.info("Check found no existing installation")
if not debug:
logger.info(
"(If you expected this to be already installed, re-run "
"`cobaya-install` with --debug to get more verbose output.)"
)
if kwargs.get("test", False):
# We are only testing whether it was installed, so consider it failed
failed_components += [name_w_kind]
continue
# Do the install
logger.info("Installing...")
try:
install_this = getattr(imported_class, "install", None)
success = install_this(path=general_abspath, **kwargs_install)
except Exception:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
logger.error(
"An unknown error occurred. Delete the external packages "
"folder %r and try again. "
"Please, notify the developers if this error persists.",
general_abspath)
success = False
if success:
logger.info("Successfully installed! Let's check it...")
else:
logger.error(
"Installation failed! Look at the error messages above. "
"Solve them and try again, or, if you are unable to solve them, "
"install the packages required by this component manually."
)
failed_components += [name_w_kind]
continue
# Test installation
reloaded_class = get_component_class(component,
kind=kind,
component_path=info.pop(
"python_path", None),
class_name=class_name,
logger=logger)
reloaded_is_installed = getattr(reloaded_class, "is_installed",
None)
with NoLogging(None if debug else logging.ERROR):
try:
successfully_installed = reloaded_is_installed(
path=this_component_install_path,
**kwargs_install,
**kwargs_is_installed)
except Exception:
traceback.print_exception(*sys.exc_info(), file=sys.stdout)
successfully_installed = False
if not successfully_installed:
logger.error(
"Installation apparently worked, "
"but the subsequent installation test failed! "
"This does not always mean that there was an actual error, "
"and is sometimes fixed simply by running the installer "
"again. If not, look closely at the error messages above, "
"or re-run with --debug for more more verbose output. "
"If you are unable to fix the issues above, "
"try installing the packages required by this "
"component manually.")
failed_components += [name_w_kind]
else:
logger.info("Installation check successful.")
print()
print(create_banner(" * Summary * ",
symbol=_banner_symbol,
length=_banner_length),
end="")
print()
bullet = "\n - "
if unknown_components:
suggestions_dict = {
name: similar_internal_class_names(name)
for name in unknown_components
}
suggestions_msg = \
bullet + bullet.join(
f"{name}: did you mean any of the following? {sugg} "
"(mind capitalization!)" for name, sugg in suggestions_dict.items())
raise LoggedError(logger, (
"The following components could not be identified and were skipped:"
f"{suggestions_msg}"))
if failed_components:
raise LoggedError(logger, (
"The installation (or installation test) of some component(s) has "
"failed: %s\nCheck output of the installer of each component above "
"for precise error info.\n"),
bullet + bullet.join(failed_components))
if obsolete_components:
raise LoggedError(logger, (
"The following packages are obsolete. Re-run with `--upgrade` option"
" (not upgrading by default to preserve possible user changes): %s"
), bullet + bullet.join(obsolete_components))
if not unknown_components and not failed_components and not obsolete_components:
logger.info(
"All requested components' dependencies correctly installed at "
f"{general_abspath}")
