def __init__(self, info_theory: TheoriesDict, packages_path=None, timing=None): super().__init__() self.set_logger("theory") if info_theory: for name, info in info_theory.items(): info = info or {} # If it has an "external" key, wrap it up. Else, load it up if isinstance(info, Theory): self.add_instance(name, info) elif isinstance(info.get("external"), Theory): self.add_instance(name, info["external"]) else: if "external" in info: theory_class = info["external"] if not isinstance(theory_class, type) or \ not issubclass(theory_class, Theory): raise LoggedError(self.log, "Theory %s is not a Theory subclass", name) else: theory_class = get_component_class( name, kind="theory", class_name=info.get("class"), logger=self.log) self.add_instance( name, theory_class( info, packages_path=packages_path, timing=timing, name=name))
def get_default_info(component_or_class, kind=None, return_yaml=False, yaml_expand_defaults=True, component_path=None, input_options=empty_dict, class_name=None, return_undefined_annotations=False): """ Get default info for a component_or_class. """ try: cls = get_component_class(component_or_class, kind, component_path, class_name, logger=logger) default_component_info = \ cls.get_defaults(return_yaml=return_yaml, yaml_expand_defaults=yaml_expand_defaults, input_options=input_options) except ComponentNotFoundError: raise except Exception as e: raise LoggedError(logger, "Failed to get defaults for component or class '%s' [%s]", component_or_class, e) if return_undefined_annotations: annotations = {k: v for k, v in cls.get_annotations().items() if k not in default_component_info} return default_component_info, annotations else: return default_component_info
def get_sampler_name_and_class(info_sampler: SamplersDict, logger=None): """ Auxiliary function to retrieve the class of the required sampler. """ check_sane_info_sampler(info_sampler) name = list(info_sampler)[0] sampler_class = get_component_class(name, kind="sampler", logger=logger) assert issubclass(sampler_class, Sampler) return name, sampler_class
def __init__(self, info_likelihood: LikesDict, packages_path=None, timing=None, theory=None): super().__init__() self.set_logger("likelihood") self.theory = theory # Get the individual likelihood classes for name, info in info_likelihood.items(): if isinstance(name, Theory): name, info = name.get_name(), info if isinstance(info, Theory): self.add_instance(name, info) elif isinstance(info, Mapping) and "external" in info: external = info["external"] if isinstance(external, Theory): self.add_instance(name, external) elif isinstance(external, type): if not is_LikelihoodInterface(external) or \ not issubclass(external, Theory): raise LoggedError( self.log, "%s: external class likelihood must " "be a subclass of Theory and have " "logp, current_logp attributes", external.__name__) self.add_instance( name, external(info, packages_path=packages_path, timing=timing, standalone=False, name=name)) else: # If it has an "external" key, wrap it up. Else, load it up self.add_instance( name, LikelihoodExternalFunction(info, name, timing=timing)) else: assert isinstance(info, Mapping) like_class: type = get_component_class( name, kind="likelihood", component_path=info.get("python_path", None), class_name=info.get("class"), logger=self.log) self.add_instance( name, like_class(info, packages_path=packages_path, timing=timing, standalone=False, name=name)) if not is_LikelihoodInterface(self[name]): raise LoggedError( self.log, "'Likelihood' %s is not actually a " "likelihood (no current_logp attribute)", name)
def test_generic_camb(packages_path, skip_not_installed): like = "bao.sdss_dr12_consensus_bao" like_rename = "my_bao" chi2_generic = deepcopy(chi2_sdss_dr12_consensus_bao) chi2_generic[like_rename] = chi2_generic.pop(like) likelihood_defaults = get_component_class(like).get_defaults() likelihood_defaults.pop("path") likelihood_defaults["class"] = "bao.generic" info_likelihood = {like_rename: likelihood_defaults} info_theory = {"camb": None} body_of_test(packages_path, best_fit, info_likelihood, info_theory, chi2_generic, skip_not_installed=skip_not_installed)
def get_bib_info(*infos, logger=None): """ Gathers and returns the descriptions and bibliographic sources for the components mentioned in ``infos``. ``infos`` can be input dictionaries or single component names. """ if not logger: logger_setup() logger = get_logger("bib") used_components, component_infos = get_used_components(*infos, return_infos=True) descs: InfoDict = {} bibs: InfoDict = {} used_components = get_used_components(*infos) for kind, components in used_components.items(): if kind is None: continue # we will deal with bare component names later, to avoid repetition descs[kind], bibs[kind] = {}, {} for component in components: try: descs[kind][component] = get_desc_component( component, kind, component_infos[component]) bibs[kind][component] = get_bib_component(component, kind) except ComponentNotFoundError: sugg = similar_internal_class_names(component) logger.error( f"Could not identify component '{component}'. " f"Did you mean any of the following? {sugg} (mind capitalization!)" ) continue # Deal with bare component names for component in used_components.get(None, []): try: cls = get_component_class(component) except ComponentNotFoundError: sugg = similar_internal_class_names(component) logger.error( f"Could not identify component '{component}'. " f"Did you mean any of the following? {sugg} (mind capitalization!)" ) continue kind = cls.get_kind() if kind not in descs: descs[kind], bibs[kind] = {}, {} if kind in descs and component in descs[kind]: continue # avoid repetition descs[kind][component] = get_desc_component(cls, kind) bibs[kind][component] = get_bib_component(cls, kind) descs["cobaya"] = {"cobaya": cobaya_desc} bibs["cobaya"] = {"cobaya": cobaya_bib} return descs, bibs
def get_preferred_old_values(info_old): """ Returns selected values in `info_old`, which are preferred at resuming. """ keep_old: InfoDict = {} for block_name, block in info_old.items(): if block_name not in kinds or not block: continue for k in block: try: component_path = block[k].pop("python_path", None) \ if isinstance(block[k], dict) else None cls = get_component_class( k, kind=block_name, component_path=component_path, class_name=(block[k] or {}).get("class"), logger=logger) prefer_old_k_this = getattr(cls, "_at_resume_prefer_old", {}) if prefer_old_k_this: if block_name not in keep_old: keep_old[block_name] = {} keep_old[block_name].update( {k: {o: block[k][o] for o in prefer_old_k_this if o in block[k]}}) except ImportError: pass return keep_old
def check_and_dump_info(self, input_info, updated_info, check_compatible=True, cache_old=False, use_cache_old=False, ignore_blocks=()): """ Saves the info in the chain folder twice: - the input info. - idem, populated with the components' defaults. If resuming a sample, checks first that old and new infos and versions are consistent. """ # trim known params of each likelihood: for internal use only self.check_lock() updated_info_trimmed = deepcopy_where_possible(updated_info) updated_info_trimmed["version"] = get_version() for like_info in updated_info_trimmed.get("likelihood", {}).values(): (like_info or {}).pop("params", None) if check_compatible: # We will test the old info against the dumped+loaded new info. # This is because we can't actually check if python objects do change try: old_info = self.reload_updated_info(cache=cache_old, use_cache=use_cache_old) except InputImportError: # for example, when there's a dynamically generated class that cannot # be found by the yaml loader (could use yaml loader that ignores them) old_info = None if old_info: # use consistent yaml read-in types # TODO: could probably just compare full infos here, with externals? # for the moment cautiously keeping old behaviour old_info = yaml_load(yaml_dump(old_info)) # type: ignore new_info = yaml_load(yaml_dump(updated_info_trimmed)) if not is_equal_info( old_info, new_info, strict=False, ignore_blocks=list(ignore_blocks) + ["output"]): raise LoggedError( self.log, "Old and new run information not compatible! " "Resuming not possible!") # Deal with version comparison separately: # - If not specified now, take the one used in resume info # - If specified both now and before, check new older than old one # (For Cobaya's own version, prefer new one always) old_version = old_info.get("version") new_version = new_info.get("version") if isinstance(old_version, str) and isinstance( new_version, str): if version.parse(old_version) > version.parse(new_version): raise LoggedError( self.log, "You are trying to resume a run performed with a " "newer version of Cobaya: %r (you are using %r). " "Please, update your Cobaya installation.", old_version, new_version) for k in set(kinds).intersection(updated_info): if k in ignore_blocks or updated_info[k] is None: continue for c in updated_info[k]: new_version = updated_info[k][c].get("version") old_version = old_info[k][c].get( "version") # type: ignore if new_version is None: updated_info[k][c]["version"] = old_version updated_info_trimmed[k][c]["version"] = old_version elif old_version is not None: cls = get_component_class( c, k, class_name=updated_info[k][c].get("class"), logger=self.log) if cls and cls.compare_versions( old_version, new_version, equal=False): raise LoggedError( self.log, "You have requested version %r for " "%s:%s, but you are trying to resume a " "run that used a newer version: %r.", new_version, k, c, old_version) # If resuming, we don't want to do *partial* dumps if ignore_blocks and self.is_resuming(): return # Work on a copy of the input info, since we are updating the prefix # (the updated one is already a copy) if input_info is not None: input_info = deepcopy_where_possible(input_info) # Write the new one for f, info in [(self.file_input, input_info), (self.file_updated, updated_info_trimmed)]: if info: for k in ignore_blocks: info.pop(k, None) info.pop("debug", None) info.pop("force", None) info.pop("resume", None) # make sure the dumped output_prefix does only contain the file prefix, # not the folder, since it's already been placed inside it info["output"] = self.updated_prefix() with open(f, "w", encoding="utf-8") as f_out: try: f_out.write(yaml_dump(sort_cosmetic(info))) except OutputError as e: raise LoggedError(self.log, str(e)) if updated_info_trimmed and has_non_yaml_reproducible( updated_info_trimmed): try: import dill except ImportError: self.mpi_info( 'Install "dill" to save reproducible options file.') else: import pickle try: with open(self.dump_file_updated, 'wb') as f: dill.dump(sort_cosmetic(updated_info_trimmed), f, pickle.HIGHEST_PROTOCOL) except pickle.PicklingError as e: os.remove(self.dump_file_updated) self.mpi_info('Options file cannot be pickled %s', e)
def doc_script(args=None): """Command line script for the documentation.""" warn_deprecation() logger_setup() logger = get_logger("doc") # Parse arguments import argparse parser = argparse.ArgumentParser( prog="cobaya doc", description="Prints defaults for Cobaya's components.") parser.add_argument( "component", action="store", nargs="?", default="", metavar="component_name", help=("The component whose defaults are requested. " "Pass a component kind (sampler, theory, likelihood) to " "list all available (internal) ones, pass nothing to list " "all available (internal) components of all kinds.")) parser.add_argument("-p", "--python", action="store_true", default=False, help="Request Python instead of YAML.") expand_flag, expand_flag_ishort = "expand", 1 parser.add_argument("-" + expand_flag[expand_flag_ishort], "--" + expand_flag, action="store_true", default=False, help="Expand YAML defaults.") arguments = parser.parse_args(args) # Nothing passed: list all if not arguments.component: msg = "Available components: (some may need external code/data)" print(msg + "\n" + "-" * len(msg)) for kind in kinds: print("%s:" % kind) print(_indent + ("\n" + _indent).join(get_available_internal_class_names(kind))) return # A kind passed (plural or singular): list all of that kind if arguments.component.lower() in subfolders.values(): arguments.component = next(k for k in subfolders if arguments.component == subfolders[k]) if arguments.component.lower() in kinds: print("%s:" % arguments.component.lower()) print(_indent + ("\n" + _indent).join( get_available_internal_class_names(arguments.component.lower()))) return # Otherwise, try to identify the component try: cls = get_component_class(arguments.component, logger=logger) except ComponentNotFoundError: suggestions = similar_internal_class_names(arguments.component) logger.error( f"Could not identify component '{arguments.component}'. " f"Did you mean any of the following? {suggestions} (mind capitalization!)" ) return 1 to_print = get_default_info(cls, return_yaml=not arguments.python, yaml_expand_defaults=arguments.expand) if arguments.python: print(pformat({cls.get_kind(): {arguments.component: to_print}})) else: print(cls.get_kind() + ":\n" + _indent + arguments.component + ":\n" + 2 * _indent + ("\n" + 2 * _indent).join(to_print.split("\n"))) if "!defaults" in to_print: print("# This file contains defaults. " "To populate them, use the flag --%s (or -%s)." % (expand_flag, expand_flag[expand_flag_ishort]))
def get_bib_component(component, kind): """Extract the bibliographic sources of a component, if defined.""" cls = get_component_class(component, kind) lines = ((cls.get_bibtex() or "").lstrip("\n").rstrip("\n") or "# [no bibliography information found]") return lines + "\n"
def get_desc_component(component, kind, info=None): """Extract a short description of a component, if defined.""" cls = get_component_class(component, kind) return cleandoc(cls.get_desc(info) or "") + "\n"
def is_equal_info(info_old, info_new, strict=True, print_not_log=False, ignore_blocks=()): """ Compares two information dictionaries, and old one versus a new one, and updates the new one for selected values of the old one. Set ``strict=False`` (default: ``True``) to ignore options that would not affect the statistics of a posterior sample, including order of params/priors/likelihoods. """ myprint: Callable myprint_debug: Callable if print_not_log: myprint = print myprint_debug = lambda x: x else: myprint = logger.info myprint_debug = logger.debug myname = inspect.stack()[0][3] ignorable = {"debug", "resume", "force", packages_path_input, "test", "version"} # MARKED FOR DEPRECATION IN v3.2 ignorable.add("debug_file") # END OF DEPRECATION BLOCK ignore = set() if strict else ignorable ignore = ignore.union(ignore_blocks or []) if set(info for info in info_old if info_old[info] is not None) - ignore \ != set(info for info in info_new if info_new[info] is not None) - ignore: myprint(myname + ": different blocks or options: %r (old) vs %r (new)" % ( set(info_old).difference(ignore), set(info_new).difference(ignore))) return False for block_name in info_old: if block_name in ignore or block_name not in info_new: continue block1 = deepcopy_where_possible(info_old[block_name]) block2 = deepcopy_where_possible(info_new[block_name]) # First, deal with root-level options (force, output, ...) if not isinstance(block1, dict): if block1 != block2: myprint(myname + ": different option '%s'" % block_name) return False continue # Now let's do components and params # 1. check order (it DOES matter, but just up to 1st level) f = list if strict else set if f(block1) != f(block2): myprint( myname + ": different [%s] or different order of them: %r vs %r" % ( block_name, list(block1), list(block2))) return False # 2. Gather general options to be ignored ignore_k = set() if not strict: if block_name in ["theory", "likelihood"]: ignore_k.update({"input_params", "output_params"}) elif block_name == "params": for param in block1: # Unify notation block1[param] = expand_info_param(block1[param]) block2[param] = expand_info_param(block2[param]) ignore_k.update({"latex", "renames", "ref", "proposal", "min", "max"}) # Fixed params, it doesn't matter if they are saved as derived if "value" in block1[param]: block1[param].pop("derived", None) if "value" in block2[param]: block2[param].pop("derived", None) # Renames: order does not matter block1[param]["renames"] = set(block1[param].get("renames", [])) block2[param]["renames"] = set(block2[param].get("renames", [])) # 3. Now check component/parameters one-by-one for k in block1: if not strict: # Add component-specific options to be ignored if block_name in kinds: ignore_k_this = ignore_k.union({"python_path"}) if "external" not in block1[k]: try: component_path = block1[k].pop("python_path", None) \ if isinstance(block1[k], dict) else None cls = get_component_class( k, kind=block_name, component_path=component_path, class_name=(block1[k] or {}).get("class"), logger=logger) ignore_k_this.update(set( getattr(cls, "_at_resume_prefer_new", {}))) except ImportError: pass # Pop ignored and kept options for j in ignore_k_this: block1[k].pop(j, None) block2[k].pop(j, None) if block1[k] != block2[k]: # For clarity, pop common stuff before printing to_pop = [j for j in block1[k] if (block1[k].get(j) == block2[k].get(j))] [(block1[k].pop(j, None), block2[k].pop(j, None)) for j in to_pop] myprint( myname + ": different content of [%s:%s]" % (block_name, k) + " -- (re-run with `debug: True` for more info)") myprint_debug("%r (old) vs %r (new)" % (block1[k], block2[k])) return False return True
def install(*infos, **kwargs): """ Installs the external packages required by the components mentioned in ``infos``. ``infos`` can be input dictionaries or single component names. :param force: force re-installation of apparently installed packages (default: ``False``). :param test: just check whether components are installed (default: ``False``). :param upgrade: force upgrade of obsolete components (default: ``False``). :param skip: keywords of components that will be skipped during installation. :param skip_global: skip installation of already-available Python modules (default: ``False``). :param debug: produce verbose debug output (default: ``False``). :param code: set to ``False`` to skip code packages (default: ``True``). :param data: set to ``False`` to skip data packages (default: ``True``). :param no_progress_bars: no progress bars shown; use when output is saved into a text file (e.g. when running on a cluster) (default: ``False``). :param no_set_global: do not store the installation path for later runs (default: ``False``). """ debug = kwargs.get("debug", False) logger = kwargs.get("logger") if not logger: logger_setup(debug=debug) logger = get_logger("install") path = kwargs.get("path") infos_not_single_names = [ info for info in infos if isinstance(info, Mapping) ] if not path: path = resolve_packages_path(*infos_not_single_names) if not path: raise LoggedError(logger, ( "No 'path' argument given, and none could be found in input infos " "(as %r), the %r env variable or the config file. " "Maybe specify one via a command line argument '-%s [...]'?"), packages_path_input, packages_path_env, packages_path_arg[0]) # General install path for all dependencies general_abspath = os.path.abspath(path) logger.info("Installing external packages at '%s'", general_abspath) # Set the installation path in the global config file if not kwargs.get("no_set_global", False) and not kwargs.get( "test", False): write_packages_path_in_config_file(general_abspath) logger.info( "The installation path has been written into the global config file: %s", os.path.join(get_config_path(), packages_path_config_file)) kwargs_install = { "force": kwargs.get("force", False), "no_progress_bars": kwargs.get("no_progress_bars") } for what in (code_path, data_path): kwargs_install[what] = kwargs.get(what, True) spath = os.path.join(general_abspath, what) if kwargs_install[what] and not os.path.exists(spath): try: os.makedirs(spath) except OSError: raise LoggedError( logger, f"Could not create the desired installation folder '{spath}'" ) # To check e.g. for a version upgrade, it needs to reload the component class and # all relevant imported modules: the implementation of `is_installed` for each # class is expected to always reload external modules if passed `reload=True` # (should be False by default to avoid deleting objects unnecessarily). kwargs_is_installed = {"reload": True} unknown_components = [] # could not be identified failed_components = [] # general errors obsolete_components = [] # older or unknown version already installed skip_keywords_arg = set(kwargs.get("skip", []) or []) # NB: if passed with quotes as `--skip "a b"`, it's interpreted as a single key skip_keywords_arg = set( chain(*[word.split() for word in skip_keywords_arg])) skip_keywords_env = set( os.environ.get(install_skip_env, "").replace(",", " ").lower().split()) skip_keywords = skip_keywords_arg.union(skip_keywords_env) # Combine all requested components and install them # NB: components mentioned by name may be repeated with those given in dict infos. # That's OK, because the install check will skip them in the 2nd pass used_components, components_infos = get_used_components(*infos, return_infos=True) for kind, components in used_components.items(): for component in components: name_w_kind = (kind + ":" if kind else "") + component print() print(create_banner(name_w_kind, symbol=_banner_symbol, length=_banner_length), end="") print() if _skip_helper(component.lower(), skip_keywords, skip_keywords_env, logger): continue info = components_infos[component] if isinstance(info, str) or "external" in info: logger.info( f"Component '{name_w_kind}' is a custom function. Nothing to do." ) continue try: class_name = (info or {}).get("class") if class_name: logger.info( f"Class to be installed for this component: {class_name}" ) imported_class = get_component_class(component, kind=kind, component_path=info.pop( "python_path", None), class_name=class_name, logger=logger) # Update the name if the kind was unknown if not kind: name_w_kind = imported_class.get_kind() + ":" + component except ComponentNotFoundError: logger.error( f"Component '{name_w_kind}' could not be identified. Skipping." ) unknown_components += [name_w_kind] continue except Exception: traceback.print_exception(*sys.exc_info(), file=sys.stdout) logger.error( f"An error occurred when loading '{name_w_kind}'. Skipping." ) failed_components += [name_w_kind] continue else: if _skip_helper(imported_class.__name__.lower(), skip_keywords, skip_keywords_env, logger): continue is_compatible = getattr(imported_class, "is_compatible", lambda: True)() if not is_compatible: logger.error(f"Skipping '{name_w_kind}' " "because it is not compatible with your OS.") failed_components += [name_w_kind] continue logger.info( "Checking if dependencies have already been installed...") is_installed = getattr(imported_class, "is_installed", None) if is_installed is None: logger.info( f"Component '{name_w_kind}' is a fully built-in component: " "nothing to do.") continue this_component_install_path = general_abspath get_path = getattr(imported_class, "get_path", None) if get_path: this_component_install_path = get_path( this_component_install_path) # Check previous installations and their versions has_been_installed = False is_old_version_msg = None with NoLogging(None if debug else logging.ERROR): try: if kwargs.get("skip_global"): has_been_installed = is_installed( path="global", **kwargs_install, **kwargs_is_installed) if not has_been_installed: has_been_installed = is_installed( path=this_component_install_path, **kwargs_install, **kwargs_is_installed) except VersionCheckError as excpt: is_old_version_msg = str(excpt) if has_been_installed: # no VersionCheckError was raised logger.info( "External dependencies for this component already installed." ) if kwargs.get("test", False): continue if kwargs_install["force"] and not kwargs.get("skip_global"): logger.info("Forcing re-installation, as requested.") else: logger.info("Doing nothing.") continue elif is_old_version_msg: logger.info(f"Version check failed: {is_old_version_msg}") obsolete_components += [name_w_kind] if kwargs.get("test", False): continue if not kwargs.get("upgrade", False) and not kwargs.get( "force", False): logger.info("Skipping because '--upgrade' not requested.") continue else: logger.info("Check found no existing installation") if not debug: logger.info( "(If you expected this to be already installed, re-run " "`cobaya-install` with --debug to get more verbose output.)" ) if kwargs.get("test", False): # We are only testing whether it was installed, so consider it failed failed_components += [name_w_kind] continue # Do the install logger.info("Installing...") try: install_this = getattr(imported_class, "install", None) success = install_this(path=general_abspath, **kwargs_install) except Exception: traceback.print_exception(*sys.exc_info(), file=sys.stdout) logger.error( "An unknown error occurred. Delete the external packages " "folder %r and try again. " "Please, notify the developers if this error persists.", general_abspath) success = False if success: logger.info("Successfully installed! Let's check it...") else: logger.error( "Installation failed! Look at the error messages above. " "Solve them and try again, or, if you are unable to solve them, " "install the packages required by this component manually." ) failed_components += [name_w_kind] continue # Test installation reloaded_class = get_component_class(component, kind=kind, component_path=info.pop( "python_path", None), class_name=class_name, logger=logger) reloaded_is_installed = getattr(reloaded_class, "is_installed", None) with NoLogging(None if debug else logging.ERROR): try: successfully_installed = reloaded_is_installed( path=this_component_install_path, **kwargs_install, **kwargs_is_installed) except Exception: traceback.print_exception(*sys.exc_info(), file=sys.stdout) successfully_installed = False if not successfully_installed: logger.error( "Installation apparently worked, " "but the subsequent installation test failed! " "This does not always mean that there was an actual error, " "and is sometimes fixed simply by running the installer " "again. If not, look closely at the error messages above, " "or re-run with --debug for more more verbose output. " "If you are unable to fix the issues above, " "try installing the packages required by this " "component manually.") failed_components += [name_w_kind] else: logger.info("Installation check successful.") print() print(create_banner(" * Summary * ", symbol=_banner_symbol, length=_banner_length), end="") print() bullet = "\n - " if unknown_components: suggestions_dict = { name: similar_internal_class_names(name) for name in unknown_components } suggestions_msg = \ bullet + bullet.join( f"{name}: did you mean any of the following? {sugg} " "(mind capitalization!)" for name, sugg in suggestions_dict.items()) raise LoggedError(logger, ( "The following components could not be identified and were skipped:" f"{suggestions_msg}")) if failed_components: raise LoggedError(logger, ( "The installation (or installation test) of some component(s) has " "failed: %s\nCheck output of the installer of each component above " "for precise error info.\n"), bullet + bullet.join(failed_components)) if obsolete_components: raise LoggedError(logger, ( "The following packages are obsolete. Re-run with `--upgrade` option" " (not upgrading by default to preserve possible user changes): %s" ), bullet + bullet.join(obsolete_components)) if not unknown_components and not failed_components and not obsolete_components: logger.info( "All requested components' dependencies correctly installed at " f"{general_abspath}")