def get_default_info(component_or_class, kind=None, return_yaml=False, yaml_expand_defaults=True, component_path=None, input_options=empty_dict, class_name=None, return_undefined_annotations=False): """ Get default info for a component_or_class. """ try: cls = get_resolved_class(component_or_class, kind, component_path, class_name) default_component_info = \ cls.get_defaults(return_yaml=return_yaml, yaml_expand_defaults=yaml_expand_defaults, input_options=input_options) except Exception as e: raise LoggedError( log, "Failed to get defaults for component or class '%s' [%s]", component_or_class, e) if return_undefined_annotations: annotations = { k: v for k, v in cls.get_annotations().items() if k not in default_component_info } return default_component_info, annotations else: return default_component_info
def get_preferred_old_values(info_old): """ Returns selected values in `info_old`, which are preferred at resuming. """ keep_old: InfoDict = {} for block_name, block in info_old.items(): if block_name not in kinds or not block: continue for k in block: try: component_path = block[k].pop("python_path", None) \ if isinstance(block[k], dict) else None cls = get_resolved_class(k, kind=block_name, component_path=component_path, class_name=(block[k] or {}).get("class")) prefer_old_k_this = getattr(cls, "_at_resume_prefer_old", {}) if prefer_old_k_this: if block_name not in keep_old: keep_old[block_name] = {} keep_old[block_name].update({ k: { o: block[k][o] for o in prefer_old_k_this if o in block[k] } }) except ImportError: pass return keep_old
def __init__(self, info_theory: TheoriesDict, packages_path=None, timing=None): super().__init__() self.set_logger("theory") if info_theory: for name, info in info_theory.items(): info = info or {} # If it has an "external" key, wrap it up. Else, load it up if isinstance(info, Theory): self.add_instance(name, info) elif isinstance(info.get("external"), Theory): self.add_instance(name, info["external"]) else: if "external" in info: theory_class = info["external"] if not isinstance(theory_class, type) or \ not issubclass(theory_class, Theory): raise LoggedError( self.log, "Theory %s is not a Theory subclass", name) else: theory_class = get_resolved_class( name, kind="theory", class_name=info.get("class")) self.add_instance( name, theory_class(info, packages_path=packages_path, timing=timing, name=name))
def __init__(self, info_theory, packages_path=None, timing=None): super().__init__() self.set_logger("theory") if info_theory: for name, info in info_theory.items(): # If it has an "external" key, wrap it up. Else, load it up if isinstance(info, Theory): self.add_instance(name, info) else: if _external in info: theory_class = info[_external] if not inspect.isclass(theory_class) or \ not issubclass(theory_class, Theory): raise LoggedError( self.log, "Theory %s is not a Theory subclass", name) else: theory_class = get_resolved_class( name, kind=kinds.theory, class_name=info.get(_class_name)) self.add_instance( name, theory_class(info, packages_path=packages_path, timing=timing, name=name))
def get_sampler_name_and_class(info_sampler: SamplersDict): """ Auxiliary function to retrieve the class of the required sampler. """ check_sane_info_sampler(info_sampler) name = list(info_sampler)[0] sampler_class = get_resolved_class(name, kind="sampler") assert issubclass(sampler_class, Sampler) return name, sampler_class
def __init__(self, info_likelihood: LikesDict, packages_path=None, timing=None, theory=None): super().__init__() self.set_logger("likelihood") self.theory = theory # Get the individual likelihood classes for name, info in info_likelihood.items(): if isinstance(name, Theory): name, info = name.get_name(), info if isinstance(info, Theory): self.add_instance(name, info) elif isinstance(info, Mapping) and "external" in info: external = info["external"] if isinstance(external, Theory): self.add_instance(name, external) elif isinstance(external, type): if not is_LikelihoodInterface(external) or \ not issubclass(external, Theory): raise LoggedError( self.log, "%s: external class likelihood must " "be a subclass of Theory and have " "logp, current_logp attributes", external.__name__) self.add_instance( name, external(info, packages_path=packages_path, timing=timing, standalone=False, name=name)) else: # If it has an "external" key, wrap it up. Else, load it up self.add_instance( name, LikelihoodExternalFunction(info, name, timing=timing)) else: assert isinstance(info, Mapping) like_class: type = get_resolved_class( name, kind="likelihood", component_path=info.get("python_path", None), class_name=info.get("class")) self.add_instance( name, like_class(info, packages_path=packages_path, timing=timing, standalone=False, name=name)) if not is_LikelihoodInterface(self[name]): raise LoggedError( self.log, "'Likelihood' %s is not actually a " "likelihood (no current_logp attribute)", name)
def __init__(self, info_likelihood, packages_path=None, timing=None, theory=None): super().__init__() self.set_logger("likelihood") self.theory = theory # Get the individual likelihood classes for name, info in info_likelihood.items(): if isinstance(name, Theory): name, info = name.get_name(), info if isinstance(info, Theory): self.add_instance(name, info) elif _external in info: if isinstance(info[_external], Theory): self.add_instance(name, info[_external]) elif inspect.isclass(info[_external]): if not is_LikelihoodInterface(info[_external]) or \ not issubclass(info[_external], Theory): raise LoggedError(self.log, "%s: external class likelihood must " "be a subclass of Theory and have " "logp, current_logp attributes", info[_external].__name__) self.add_instance(name, info[_external](info, packages_path=packages_path, timing=timing, standalone=False, name=name)) else: # If it has an "external" key, wrap it up. Else, load it up self.add_instance(name, LikelihoodExternalFunction(info, name, timing=timing)) else: like_class = get_resolved_class( name, kind=kinds.likelihood, component_path=info.pop(_component_path, None), class_name=info.get(_class_name)) self.add_instance(name, like_class(info, packages_path=packages_path, timing=timing, standalone=False, name=name)) if not is_LikelihoodInterface(self[name]): raise LoggedError(self.log, "'Likelihood' %s is not actually a " "likelihood (no current_logp attribute)", name)
def is_equal_info(info_old, info_new, strict=True, print_not_log=False, ignore_blocks=()): """ Compares two information dictionaries, and old one versus a new one, and updates the new one for selected values of the old one. Set ``strict=False`` (default: ``True``) to ignore options that would not affect the statistics of a posterior sample, including order of params/priors/likelihoods. """ myprint: Callable myprint_debug: Callable if print_not_log: myprint = print myprint_debug = lambda x: x else: myprint = log.info myprint_debug = log.debug myname = inspect.stack()[0][3] ignore = set() if strict else \ {"debug", "debug_file", "resume", "force", packages_path_input, "test", "version"} ignore = ignore.union(ignore_blocks or []) if set(info for info in info_old if info_old[info] is not None) - ignore \ != set(info for info in info_new if info_new[info] is not None) - ignore: myprint(myname + ": different blocks or options: %r (old) vs %r (new)" % (set(info_old).difference(ignore), set(info_new).difference(ignore))) return False for block_name in info_old: if block_name in ignore or block_name not in info_new: continue block1 = deepcopy_where_possible(info_old[block_name]) block2 = deepcopy_where_possible(info_new[block_name]) # First, deal with root-level options (force, output, ...) if not isinstance(block1, dict): if block1 != block2: myprint(myname + ": different option '%s'" % block_name) return False continue # Now let's do components and params # 1. check order (it DOES matter, but just up to 1st level) f = list if strict else set if f(block1) != f(block2): myprint(myname + ": different [%s] or different order of them: %r vs %r" % (block_name, list(block1), list(block2))) return False # 2. Gather general options to be ignored ignore_k = set() if not strict: if block_name in ["theory", "likelihood"]: ignore_k.update({"input_params", "output_params"}) elif block_name == "params": for param in block1: # Unify notation block1[param] = expand_info_param(block1[param]) block2[param] = expand_info_param(block2[param]) ignore_k.update( {"latex", "renames", "ref", "proposal", "min", "max"}) # Fixed params, it doesn't matter if they are saved as derived if "value" in block1[param]: block1[param].pop("derived", None) if "value" in block2[param]: block2[param].pop("derived", None) # Renames: order does not matter block1[param]["renames"] = set(block1[param].get( "renames", [])) block2[param]["renames"] = set(block2[param].get( "renames", [])) # 3. Now check component/parameters one-by-one for k in block1: if not strict: # Add component-specific options to be ignored if block_name in kinds: ignore_k_this = ignore_k.union({"python_path"}) if "external" not in block1[k]: try: component_path = block1[k].pop("python_path", None) \ if isinstance(block1[k], dict) else None cls = get_resolved_class( k, kind=block_name, component_path=component_path, class_name=(block1[k] or {}).get("class")) ignore_k_this.update( set(getattr(cls, "_at_resume_prefer_new", {}))) except ImportError: pass # Pop ignored and kept options for j in ignore_k_this: block1[k].pop(j, None) block2[k].pop(j, None) if block1[k] != block2[k]: # For clarity, pop common stuff before printing to_pop = [ j for j in block1[k] if (block1[k].get(j) == block2[k].get(j)) ] [(block1[k].pop(j, None), block2[k].pop(j, None)) for j in to_pop] myprint(myname + ": different content of [%s:%s]" % (block_name, k) + " -- (re-run with `debug: True` for more info)") myprint_debug("%r (old) vs %r (new)" % (block1[k], block2[k])) return False return True
def install(*infos, **kwargs): debug = kwargs.get("debug") # noinspection PyUnresolvedReferences if not log.root.handlers: logger_setup(debug=debug) path = kwargs.get("path") if not path: path = resolve_packages_path(infos) if not path: raise LoggedError( log, "No 'path' argument given, and none could be found in input infos " "(as %r), the %r env variable or the config file. " "Maybe specify one via a command line argument '-%s [...]'?", "packages_path", packages_path_env, packages_path_arg[0]) abspath = os.path.abspath(path) log.info("Installing external packages at '%s'", abspath) kwargs_install = { "force": kwargs.get("force", False), "no_progress_bars": kwargs.get("no_progress_bars") } for what in (code_path, data_path): kwargs_install[what] = kwargs.get(what, True) spath = os.path.join(abspath, what) if kwargs_install[what] and not os.path.exists(spath): try: os.makedirs(spath) except OSError: raise LoggedError( log, "Could not create the desired installation folder '%s'", spath) failed_components = [] skip_keywords_arg = set(kwargs.get("skip", []) or []) # NB: if passed with quotes as `--skip "a b"`, it's interpreted as a single key skip_keywords_arg = set( chain(*[word.split() for word in skip_keywords_arg])) skip_keywords_env = set( os.environ.get(install_skip_env, "").replace(",", " ").lower().split()) skip_keywords = skip_keywords_arg.union(skip_keywords_env) used_components, components_infos = get_used_components(*infos, return_infos=True) for kind, components in used_components.items(): for component in components: print() print(create_banner(kind + ":" + component, symbol=_banner_symbol, length=_banner_length), end="") print() if _skip_helper(component.lower(), skip_keywords, skip_keywords_env, log): continue info = components_infos[component] if isinstance(info, str) or "external" in info: log.warning( "Component '%s' is a custom function. " "Nothing to do.", component) continue try: class_name = (info or {}).get("class") if class_name: log.info("Class to be installed for this component: %r", class_name) imported_class = get_resolved_class(component, kind=kind, component_path=info.pop( "python_path", None), class_name=class_name) except ImportError as excpt: log.error("Component '%s' not recognized. [%s].", component, excpt) failed_components += ["%s:%s" % (kind, component)] continue else: if _skip_helper(imported_class.__name__.lower(), skip_keywords, skip_keywords_env, log): continue is_compatible = getattr(imported_class, "is_compatible", lambda: True)() if not is_compatible: log.info( "Skipping %r because it is not compatible with your OS.", component) continue log.info("Checking if dependencies have already been installed...") is_installed = getattr(imported_class, "is_installed", None) if is_installed is None: log.info("%s.%s is a fully built-in component: nothing to do.", kind, imported_class.__name__) continue install_path = abspath get_path = getattr(imported_class, "get_path", None) if get_path: install_path = get_path(install_path) has_been_installed = False with NoLogging(None if debug else logging.ERROR): if kwargs.get("skip_global"): has_been_installed = is_installed(path="global", **kwargs_install) if not has_been_installed: has_been_installed = is_installed(path=install_path, **kwargs_install) if has_been_installed: log.info( "External dependencies for this component already installed." ) if kwargs.get("test", False): continue if kwargs_install["force"] and not kwargs.get("skip_global"): log.info("Forcing re-installation, as requested.") else: log.info("Doing nothing.") continue else: log.info("Check found no existing installation") if not debug: log.info( "(If you expected this to be already installed, re-run " "`cobaya-install` with --debug to get more verbose output.)" ) if kwargs.get("test", False): continue log.info("Installing...") try: install_this = getattr(imported_class, "install", None) success = install_this(path=abspath, **kwargs_install) except KeyboardInterrupt: raise except: traceback.print_exception(*sys.exc_info(), file=sys.stdout) log.error( "An unknown error occurred. Delete the external packages " "folder %r and try again. " "Please, notify the developers if this error persists.", abspath) success = False if success: log.info("Successfully installed! Let's check it...") else: log.error( "Installation failed! Look at the error messages above. " "Solve them and try again, or, if you are unable to solve, " "install the packages required by this component manually." ) failed_components += ["%s:%s" % (kind, component)] continue # test installation with NoLogging(None if debug else logging.ERROR): successfully_installed = is_installed(path=install_path, check=False, **kwargs_install) if not successfully_installed: log.error( "Installation apparently worked, " "but the subsequent installation test failed! " "Look at the error messages above, or re-run with --debug " "for more more verbose output. " "Try to solve the issues and try again, or, if you are unable " "to solve them, install the packages required by this " "component manually.") failed_components += ["%s:%s" % (kind, component)] else: log.info("Installation check successful.") print() print(create_banner(" * Summary * ", symbol=_banner_symbol, length=_banner_length), end="") print() if failed_components: bullet = "\n - " raise LoggedError( log, "The installation (or installation test) of some component(s) has " "failed: %s\nCheck output of the installer of each component above " "for precise error info.\n", bullet + bullet.join(failed_components)) log.info("All requested components' dependencies correctly installed.") # Set the installation path in the global config file if not kwargs.get("no_set_global", False) and not kwargs.get( "test", False): write_packages_path_in_config_file(abspath) log.info( "The installation path has been written into the global config file: %s", os.path.join(get_config_path(), packages_path_config_file))