def __init__(self, info_theory, packages_path=None, timing=None): super().__init__() self.set_logger("theory") if info_theory: for name, info in info_theory.items(): # If it has an "external" key, wrap it up. Else, load it up if isinstance(info, Theory): self.add_instance(name, info) else: if _external in info: theory_class = info[_external] if not inspect.isclass(theory_class) or \ not issubclass(theory_class, Theory): raise LoggedError( self.log, "Theory %s is not a Theory subclass", name) else: theory_class = get_class(info.get(_class_name) or name, kind=kinds.theory) self.add_instance( name, theory_class(info, packages_path=packages_path, timing=timing, name=name))
def get_default_info(module, kind=None, fail_if_not_found=False, return_yaml=False, yaml_expand_defaults=True): """ Get default info for a module. """ try: if kind is None: kind = get_kind(module) cls = get_class(module, kind, None_if_not_found=not fail_if_not_found) if cls: default_module_info = cls.get_defaults( return_yaml=return_yaml, yaml_expand_defaults=yaml_expand_defaults) else: default_module_info = (lambda x: yaml_dump(x) if return_yaml else x)({ kind: { module: {} } }) except Exception as e: raise LoggedError(log, "Failed to get defaults for module '%s' [%s]", ("%s:" % kind if kind else "") + module, e) try: if not return_yaml: default_module_info[kind][module] except KeyError: raise LoggedError( log, "The defaults file for '%s' should be structured " "as %s:%s:{[options]}.", module, kind, module) return default_module_info
def get_desc_component(component, kind, info=None): cls = get_class(component, kind, None_if_not_found=True) if cls: lines = cleandoc(cls.get_desc(info) or "") else: lines = "[no description found]" return lines + "\n"
def get_sampler_name_and_class(info_sampler): """ Auxiliary function to retrieve the class of the required sampler. """ check_sane_info_sampler(info_sampler) name = list(info_sampler)[0] return name, get_class(name, kind=kinds.sampler)
def get_preferred_old_values(info_old): """ Returns selected values in `info_old`, which are preferred at resuming. """ keep_old = {} for block_name, block in info_old.items(): if block_name not in kinds or not block: continue for k in block: try: component_path = block[k].pop(_component_path, None) \ if isinstance(block[k], dict) else None class_name = (block[k] or {}).get(_class_name) or k cls = get_class(class_name, block_name, component_path=component_path) prefer_old_k_this = getattr(cls, "_at_resume_prefer_old", {}) if prefer_old_k_this: if block_name not in keep_old: keep_old[block_name] = {} keep_old[block_name].update({ k: { o: block[k][o] for o in prefer_old_k_this if o in block[k] } }) except ImportError: pass return keep_old
def get_default_info(component_or_class, kind=None, return_yaml=False, yaml_expand_defaults=True, component_path=None, input_options=empty_dict, class_name=None): """ Get default info for a component_or_class. """ _kind = kind try: if inspect.isclass(component_or_class): cls = component_or_class else: _kind = _kind or get_kind(component_or_class, class_name=class_name) cls = get_class(class_name or component_or_class, _kind, component_path=component_path) default_component_info = \ cls.get_defaults(return_yaml=return_yaml, yaml_expand_defaults=yaml_expand_defaults, input_options=input_options) except Exception as e: raise LoggedError( log, "Failed to get defaults for component or class '%s' [%s]", component_or_class, e) return default_component_info
def get_bib_component(component, kind): cls = get_class(component, kind, None_if_not_found=True) if cls: lines = ((cls.get_bibtex() or "").lstrip("\n").rstrip("\n") or "# [no bibliography information found]") else: lines = "# [Component '%s.%s' not known.]" % (kind, component) return lines + "\n"
def __init__(self, info_likelihood, parameterization, info_theory=None, modules=None, timing=None): self.log = logging.getLogger("Likelihood") # *IF* there is a theory code, initialize it if info_theory: name = list(info_theory)[0] # If it has an "external" key, wrap it up. Else, load it up if _external in info_theory[name]: theory_class = info_theory[name][_external] else: theory_class = get_class(name, kind=_theory) self.theory = theory_class(info_theory[name], modules=modules, timing=timing) else: self.theory = None # Initialize individual Likelihoods self._likelihoods = odict() for name, info in info_likelihood.items(): # If it has an "external" key, wrap it up. Else, load it up if _external in info: self._likelihoods[name] = LikelihoodExternalFunction( name, info, _theory=getattr(self, _theory, None), timing=timing) else: like_class = get_class(name) self._likelihoods[name] = like_class(info, modules=modules, timing=timing) # Assign input/output parameters self._assign_params(parameterization, info_likelihood, info_theory) # Do the user-defined post-initialisation, and assign the theory code if self.theory: self.theory.initialize() for like in self: self[like].initialize() self[like].theory = self.theory self[like].add_theory() # Store the input params and likelihoods on which each sampled params depends. self.sampled_input_dependence = parameterization.sampled_input_dependence() self.sampled_like_dependence = odict( [[p, [like for like in list(self) + ([_theory] if self.theory else []) if any([(i in self[like].input_params) for i in (i_s or [p])])]] for p, i_s in self.sampled_input_dependence.items()]) # Theory parameters "depend" on every likelihood, since re-computing the theory # code forces recomputation of the likelihoods for p, ls in self.sampled_like_dependence.items(): if _theory in ls: self.sampled_like_dependence[p] = ([_theory] + list(self._likelihoods)) # Overhead per likelihood evaluation self.overhead = _overhead_per_param * len(parameterization.sampled_params())
def __init__(self, info_likelihood, packages_path=None, timing=None, theory=None): super().__init__() self.set_logger("likelihood") self.theory = theory # Get the individual likelihood classes for name, info in info_likelihood.items(): if isinstance(name, Theory): name, info = name.get_name(), info if isinstance(info, Theory): self.add_instance(name, info) elif _external in info: if isinstance(info[_external], Theory): self.add_instance(name, info[_external]) elif inspect.isclass(info[_external]): if not hasattr(info[_external], "get_current_logp") or \ not issubclass(info[_external], Theory): raise LoggedError( self.log, "%s: external class likelihood must " "be a subclass of Theory and have " "logp, get_current_logp functions", info[_external].__name__) self.add_instance( name, info[_external](info, packages_path=packages_path, timing=timing, standalone=False, name=name)) else: # If it has an "external" key, wrap it up. Else, load it up self.add_instance( name, LikelihoodExternalFunction(info, name, timing=timing)) else: like_class = get_class(info.get(_class_name) or name, kind=kinds.likelihood, component_path=info.pop( _component_path, None)) self.add_instance( name, like_class(info, packages_path=packages_path, timing=timing, standalone=False, name=name)) if not hasattr(self[name], "get_current_logp"): raise LoggedError( self.log, "'Likelihood' %s is not actually a " "likelihood (no get_current_logp function)", name)
def test_generic_camb(packages_path, skip_not_installed): like = "bao.sdss_dr12_consensus_bao" like_rename = "my_bao" chi2_generic = deepcopy(chi2_sdss_dr12_consensus_bao) chi2_generic[like_rename] = chi2_generic.pop(like) likelihood_defaults = get_class(like).get_defaults() likelihood_defaults.pop("path") likelihood_defaults["class"] = "bao.generic" info_likelihood = {like_rename: likelihood_defaults} info_theory = {"camb": None} body_of_test(packages_path, best_fit, info_likelihood, info_theory, chi2_generic, skip_not_installed=skip_not_installed)
def get_sampler(info_sampler, posterior, output_file, resume=_resume_default, modules=None): """ Auxiliary function to retrieve and initialize the requested sampler. """ if not info_sampler: log.error("No sampler given!") raise HandledException try: name = list(info_sampler)[0] except AttributeError: log.error( "The sampler block must be a dictionary 'sampler: {options}'.") raise HandledException sampler_class = get_class(name, kind=_sampler) return sampler_class(info_sampler[name], posterior, output_file, resume=resume, modules=modules)
def get_sampler(info_sampler, posterior, output_file, resume=_resume_default, modules=None): """ Auxiliary function to retrieve and initialize the requested sampler. """ log = logging.getLogger(__name__.split(".")[-1]) if not info_sampler: raise LoggedError(log, "No sampler given!") try: name = list(info_sampler)[0] except AttributeError: raise LoggedError( log, "The sampler block must be a dictionary 'sampler: {options}'.") sampler_class = get_class(name, kind=_sampler) return sampler_class(info_sampler[name], posterior, output_file, resume=resume, modules=modules)
def __init__(self, info_likelihood, parametrization, info_theory=None): # Store the input/output parameters self.input_params = parametrization.input_params() self.output_params = parametrization.output_params() # Initialise individual Likelihoods self._likelihoods = odict() for name, info in info_likelihood.items(): # If it does "external" key, wrap it up. Else, load it up if _external in info: self._likelihoods[name] = LikelihoodExternalFunction( name, info, parametrization, theory=getattr(self, _theory, None)) else: lik_class = get_class(name) self._likelihoods[name] = lik_class(info, parametrization) # Check that all are recognized requested_not_known = {} for params in ("input_params", "output_params"): info = getattr(parametrization, params)() setattr(self, params, info) requested = set(info) known = set(chain(*[getattr(self[lik], params) for lik in self])) requested_not_known[params] = requested.difference(known) if requested_not_known[params]: if info_theory: log.debug( "The following %s parameters are not recognised by any " "likelihood, and will be passed to the theory code: %r", params.split("_")[0], requested_not_known[params]) else: log.error( "Some of the requested %s parameters were not recognized " "by any likelihood: %r", params.split("_")[0], requested_not_known[params]) raise HandledException # *IF* there is a theory code, initialise it if info_theory: input_params_theory = self.input_params.fromkeys( requested_not_known["input_params"]) output_params_theory = self.output_params.fromkeys( requested_not_known["output_params"]) name, fields = list(info_theory.items())[0] theory_class = get_class(name, kind=_theory) self.theory = theory_class(input_params_theory, output_params_theory, fields) else: self.theory = None for lik in self: self[lik].theory = self.theory self[lik].add_theory() # Store the input params and likelihods on which each sampled params depends. # Theory parameters "depend" on every likelihood, since re-computing the theory # code forces recomputation of the likelihoods self.sampled_input_dependence = parametrization.sampled_input_dependence( ) self.sampled_lik_dependence = odict([[ p, [ lik for lik in list(self) + ([_theory] if self.theory else []) if any([(i in self[lik].input_params) for i in (i_s or [p])]) ] ] for p, i_s in self.sampled_input_dependence.items()]) # Theory parameters "depend" on every likelihood, since re-computing the theory # code forces recomputation of the likelihoods for p, ls in self.sampled_lik_dependence.items(): if _theory in ls: self.sampled_lik_dependence[p] = ([_theory] + list(self._likelihoods)) # Pop the per-likelihood parameters info, that was auxiliary for lik in info_likelihood: info_likelihood[lik].pop(_params)
def __init__(self, info_likelihood, parameterization, info_theory=None, modules=None, timing=None): self.log = logging.getLogger("Likelihood") # Initialize individual Likelihoods self._likelihoods = odict() for name, info in info_likelihood.items(): # If it has an "external" key, wrap it up. Else, load it up if _external in info: self._likelihoods[name] = LikelihoodExternalFunction( name, info, parameterization, _theory=getattr(self, _theory, None), timing=timing) else: # If mock likelihood and there is a theory code, make the mock ignore # all parameters, or they would never reach the theory code if "prefix" in info and info_theory: info["prefix"] = None self.log.debug("Mock likelihood '%s' will ignore all parameters, " "since a theory code is present.", name) like_class = get_class(name) self._likelihoods[name] = like_class( info, parameterization, modules=modules, timing=timing) # Check that all are recognized requested_not_known = {} for params in ("input_params", "output_params"): # Store the input/output parameters info = getattr(parameterization, params)() setattr(self, params, info) requested = set(info) known = set(chain(*[getattr(self[like], params) for like in self])) requested_not_known[params] = requested.difference(known) if requested_not_known[params]: if info_theory: self.log.debug( "The following %s parameters are not recognized by any " "likelihood, and will be passed to the theory code: %r", params.split("_")[0], list(requested_not_known[params])) else: self.log.error( "Some of the requested %s parameters were not recognized " "by any likelihood: %r", params.split("_")[0], list(requested_not_known[params])) raise HandledException # *IF* there is a theory code, initialize it if info_theory: input_params_theory = {p: v for p, v in self.input_params.items() if p in requested_not_known["input_params"]} output_params_theory = {p: None for p in self.output_params if p in requested_not_known["output_params"] and # Not the chi2's either not p.startswith(_chi2 + _separator)} name, fields = list(info_theory.items())[0] # If it has an "external" key, wrap it up. Else, load it up if _external in list(info_theory.values())[0]: theory_class = list(info_theory.values())[0][_external] else: theory_class = get_class(name, kind=_theory) self.theory = theory_class(input_params_theory, output_params_theory, fields, modules=modules, timing=timing) else: self.theory = None for like in self: self[like].theory = self.theory self[like].add_theory() # Store the input params and likelihoods on which each sampled params depends. self.sampled_input_dependence = parameterization.sampled_input_dependence() self.sampled_like_dependence = odict( [[p, [like for like in list(self) + ([_theory] if self.theory else []) if any([(i in self[like].input_params) for i in (i_s or [p])])]] for p, i_s in self.sampled_input_dependence.items()]) # Theory parameters "depend" on every likelihood, since re-computing the theory # code forces recomputation of the likelihoods for p, ls in self.sampled_like_dependence.items(): if _theory in ls: self.sampled_like_dependence[p] = ([_theory] + list(self._likelihoods)) # Pop the per-likelihood parameters info, that was auxiliary for like in info_likelihood: info_likelihood[like].pop(_params) # Overhead per likelihood evaluation self.overhead = _overhead_per_param * len(parameterization.sampled_params())
def install(*infos, **kwargs): debug = kwargs.get(_debug) if not log.root.handlers: logger_setup() path = kwargs.get("path") if not path: path = resolve_packages_path(infos) if not path: raise LoggedError( log, "No 'path' argument given, and none could be found in input infos " "(as %r), the %r env variable or the config file. " "Maybe specify one via a command line argument '-%s [...]'?", _packages_path, _packages_path_env, _packages_path_arg[0]) abspath = os.path.abspath(path) log.info("Installing external packages at '%s'", abspath) kwargs_install = {"force": kwargs.get("force", False), "no_progress_bars": kwargs.get("no_progress_bars")} for what in (_code, _data): kwargs_install[what] = kwargs.get(what, True) spath = os.path.join(abspath, what) if kwargs_install[what] and not os.path.exists(spath): try: os.makedirs(spath) except OSError: raise LoggedError( log, "Could not create the desired installation folder '%s'", spath) failed_components = [] skip_keywords_arg = set(kwargs.get("skip", []) or []) # NB: if passed with quotes as `--skip "a b"`, it's interpreted as a single key skip_keywords_arg = set(chain(*[word.split() for word in skip_keywords_arg])) skip_keywords_env = set( os.environ.get(_install_skip_env, "").replace(",", " ").lower().split()) skip_keywords = skip_keywords_arg.union(skip_keywords_env) for kind, components in get_used_components(*infos).items(): for component in components: print() print(create_banner(kind + ":" + component, symbol=_banner_symbol, length=_banner_length), end="") print() if _skip_helper(component.lower(), skip_keywords, skip_keywords_env, log): continue info = (next(info for info in infos if component in info.get(kind, {}))[kind][component]) or {} if isinstance(info, str) or _external in info: log.warning("Component '%s' is a custom function. " "Nothing to do.", component) continue try: imported_class = get_class(component, kind, component_path=info.pop(_component_path, None)) except ImportError as excpt: log.error("Component '%s' not recognized. [%s].", component, excpt) failed_components += ["%s:%s" % (kind, component)] continue else: if _skip_helper(imported_class.__name__.lower(), skip_keywords, skip_keywords_env, log): continue is_compatible = getattr(imported_class, "is_compatible", lambda: True)() if not is_compatible: log.info( "Skipping %r because it is not compatible with your OS.", component) continue log.info("Checking if dependencies have already been installed...") is_installed = getattr(imported_class, "is_installed", None) if is_installed is None: log.info("%s.%s is a fully built-in component: nothing to do.", kind, imported_class.__name__) continue install_path = abspath get_path = getattr(imported_class, "get_path", None) if get_path: install_path = get_path(install_path) has_been_installed = False if not debug: logging.disable(logging.ERROR) if kwargs.get("skip_global"): has_been_installed = is_installed(path="global", **kwargs_install) if not has_been_installed: has_been_installed = is_installed(path=install_path, **kwargs_install) if not debug: logging.disable(logging.NOTSET) if has_been_installed: log.info("External dependencies for this component already installed.") if kwargs.get(_test_run, False): continue if kwargs_install["force"] and not kwargs.get("skip_global"): log.info("Forcing re-installation, as requested.") else: log.info("Doing nothing.") continue else: log.info("Installation check failed!") if not debug: log.info( "(If you expected this to be already installed, re-run " "`cobaya-install` with --debug to get more verbose output.)") if kwargs.get(_test_run, False): continue log.info("Installing...") try: install_this = getattr(imported_class, "install", None) success = install_this(path=abspath, **kwargs_install) except KeyboardInterrupt: raise except: traceback.print_exception(*sys.exc_info(), file=sys.stdout) log.error("An unknown error occurred. Delete the external packages " "folder %r and try again. " "Please, notify the developers if this error persists.", abspath) success = False if success: log.info("Successfully installed! Let's check it...") else: log.error("Installation failed! Look at the error messages above. " "Solve them and try again, or, if you are unable to solve, " "install the packages required by this component manually.") failed_components += ["%s:%s" % (kind, component)] continue # test installation if not debug: logging.disable(logging.ERROR) successfully_installed = is_installed(path=install_path, **kwargs_install) if not debug: logging.disable(logging.NOTSET) if not successfully_installed: log.error("Installation apparently worked, " "but the subsequent installation test failed! " "Look at the error messages above, or re-run with --debug " "for more more verbose output. " "Try to solve the issues and try again, or, if you are unable " "to solve them, install the packages required by this " "component manually.") failed_components += ["%s:%s" % (kind, component)] else: log.info("Installation check successful.") print() print(create_banner(" * Summary * ", symbol=_banner_symbol, length=_banner_length), end="") print() if failed_components: bullet = "\n - " raise LoggedError( log, "The installation (or installation test) of some component(s) has " "failed: %s\nCheck output of the installer of each component above " "for precise error info.\n", bullet + bullet.join(failed_components)) log.info("All requested components' dependencies correctly installed.") # Set the installation path in the global config file if not kwargs.get("no_set_global", False) and not kwargs.get(_test_run, False): write_packages_path_in_config_file(abspath) log.info("The installation path has been written into the global config file: %s", os.path.join(get_config_path(), _packages_path_config_file))
def install(*infos, **kwargs): if not log.root.handlers: logger_setup() path = kwargs.get("path") if not path: path = resolve_packages_path(infos) if not path: raise LoggedError( log, "No 'path' argument given, and none could be found in input infos " "(as %r), the %r env variable or the config file. " "Maybe specify one via a command line argument '-%s [...]'?", _packages_path, _packages_path_env, _packages_path_arg[0]) abspath = os.path.abspath(path) log.info("Installing external packages at '%s'", abspath) kwargs_install = { "force": kwargs.get("force", False), "no_progress_bars": kwargs.get("no_progress_bars") } for what in (_code, _data): kwargs_install[what] = kwargs.get(what, True) spath = os.path.join(abspath, what) if kwargs_install[what] and not os.path.exists(spath): try: os.makedirs(spath) except OSError: raise LoggedError( log, "Could not create the desired installation folder '%s'", spath) failed_components = [] skip_keywords = set(kwargs.get("skip", []) or []) skip_keywords_env = set( os.environ.get(_install_skip_env, "").replace(",", " ").lower().split()) skip_keywords = skip_keywords.union(skip_keywords_env) for kind, components in get_used_components(*infos).items(): for component in components: print() print(create_banner(kind + ":" + component, symbol="=", length=80)) print() if _skip_helper(component.lower(), skip_keywords, skip_keywords_env, log): continue info = (next( info for info in infos if component in info.get(kind, {}))[kind][component]) or {} if isinstance(info, str) or _external in info: log.warning( "Component '%s' is a custom function. " "Nothing to do.", component) continue try: imported_class = get_class(component, kind, component_path=info.pop( _component_path, None)) except ImportError as e: log.error("Component '%s' not recognized. [%s]." % (component, e)) failed_components += ["%s:%s" % (kind, component)] continue else: if _skip_helper(imported_class.__name__.lower(), skip_keywords, skip_keywords_env, log): continue is_installed = getattr(imported_class, "is_installed", None) if is_installed is None: log.info("%s.%s is a fully built-in component: nothing to do.", kind, imported_class.__name__) continue if is_installed(path=abspath, **kwargs_install): log.info("External component already installed.") if kwargs.get("just_check", False): continue if kwargs_install["force"]: log.info("Forcing re-installation, as requested.") else: log.info("Doing nothing.") continue else: if kwargs.get("just_check", False): log.info("NOT INSTALLED!") continue try: install_this = getattr(imported_class, "install", None) success = install_this(path=abspath, **kwargs_install) except KeyboardInterrupt: raise except: traceback.print_exception(*sys.exc_info(), file=sys.stdout) log.error( "An unknown error occurred. Delete the external packages " "folder %r and try again. " "Please, notify the developers if this error persists.", abspath) success = False if success: log.info("Successfully installed!") else: log.error( "Installation failed! Look at the error messages above. " "Solve them and try again, or, if you are unable to solve, " "install the packages required by this component manually." ) failed_components += ["%s:%s" % (kind, component)] continue # test installation if not is_installed(path=abspath, **kwargs_install): log.error( "Installation apparently worked, " "but the subsequent installation test failed! " "Look at the error messages above. " "Solve them and try again, or, if you are unable to solve, " "install the packages required by this component manually." ) failed_components += ["%s:%s" % (kind, component)] if failed_components: bullet = "\n - " raise LoggedError( log, "The installation (or installation test) of some component(s) has " "failed: %s\nCheck output of the installer of each component above " "for precise error info.\n", bullet + bullet.join(failed_components)) # Set the installation path in the global config file if not kwargs.get("no_set_global", False) and not kwargs.get( "just_check", False): write_packages_path_in_config_file(abspath) log.info( "The installation path has been written in the global config file." )
def install(*infos, **kwargs): if not log.root.handlers: logger_setup() path = kwargs.get("path", ".") if not path: # See if we can get one (and only one) from infos paths = set( [p for p in [info.get(_path_install) for info in infos] if p]) if len(paths) == 1: path = paths[0] else: print("logging?") raise LoggedError( log, "No 'path' argument given and could not extract one (and only one) " "from the infos.") abspath = os.path.abspath(path) log.info("Installing modules at '%s'\n", abspath) kwargs_install = { "force": kwargs.get("force", False), "no_progress_bars": kwargs.get("no_progress_bars") } for what in (_code, _data): kwargs_install[what] = kwargs.get(what, True) spath = os.path.join(abspath, what) if kwargs_install[what] and not os.path.exists(spath): try: os.makedirs(spath) except OSError: raise LoggedError( log, "Could not create the desired installation folder '%s'", spath) failed_modules = [] skip_list = os.environ.get("COBAYA_TEST_SKIP", "").replace(",", " ").lower().split() for kind, modules in get_used_modules(*infos).items(): for module in modules: print(create_banner(kind + ":" + module, symbol="=", length=80)) if len([s for s in skip_list if s in module.lower()]): log.info("Skipping %s for test skip list %s" % (module, skip_list)) continue module_folder = get_class_module(module, kind) try: imported_module = import_module(module_folder, package=_package) imported_class = get_class(module, kind) if len([ s for s in skip_list if s in imported_class.__name__.lower() ]): log.info("Skipping %s for test skip list %s" % (imported_class.__name__, skip_list)) continue except ImportError as e: if kind == _likelihood: info = (next(info for info in infos if module in info.get( _likelihood, {}))[_likelihood][module]) or {} if isinstance(info, string_types) or _external in info: log.warning( "Module '%s' is a custom likelihood. " "Nothing to do.\n", module) else: log.error("Module '%s' not recognized. [%s]\n" % (module, e)) failed_modules += ["%s:%s" % (kind, module)] continue is_installed = getattr( imported_class, "is_installed", getattr(imported_module, "is_installed", None)) if is_installed is None: log.info("Built-in module: nothing to do.\n") continue if is_installed(path=abspath, **kwargs_install): log.info("External module already installed.\n") if kwargs.get("just_check", False): continue if kwargs_install["force"]: log.info("Forcing re-installation, as requested.") else: log.info("Doing nothing.\n") continue else: if kwargs.get("just_check", False): log.info("NOT INSTALLED!\n") continue try: install_this = getattr( imported_class, "install", getattr(imported_module, "install", None)) success = install_this(path=abspath, **kwargs_install) except: traceback.print_exception(*sys.exc_info(), file=sys.stdout) log.error( "An unknown error occurred. Delete the modules folder and try " "again. Notify the developers if this error persists.") success = False if success: log.info("Successfully installed!\n") else: log.error( "Installation failed! Look at the error messages above. " "Solve them and try again, or, if you are unable to solve, " "install this module manually.") failed_modules += ["%s:%s" % (kind, module)] continue # test installation if not is_installed(path=abspath, **kwargs_install): log.error( "Installation apparently worked, " "but the subsequent installation test failed! " "Look at the error messages above. " "Solve them and try again, or, if you are unable to solve, " "install this module manually.") failed_modules += ["%s:%s" % (kind, module)] if failed_modules: bullet = "\n - " raise LoggedError( log, "The installation (or installation test) of some module(s) has failed: " "%s\nCheck output of the installer of each module above " "for precise error info.\n", bullet + bullet.join(failed_modules))
def is_equal_info(info_old, info_new, strict=True, print_not_log=False, ignore_blocks=()): """ Compares two information dictionaries, and old one versus a new one, and updates the new one for selected values of the old one. Set ``strict=False`` (default: ``True``) to ignore options that would not affect the statistics of a posterior sample, including order of params/priors/likelihoods. """ if print_not_log: myprint = print myprint_debug = lambda x: x else: myprint = log.info myprint_debug = log.debug myname = inspect.stack()[0][3] ignore = set() if strict else \ {_debug, _debug_file, _resume, _force, _packages_path, _test_run, _version} ignore = ignore.union(set(ignore_blocks or [])) if set(info for info in info_old if info_old[info] is not None).difference(ignore) \ != set(info for info in info_new if info_new[info] is not None).difference( ignore): myprint(myname + ": different blocks or options: %r (old) vs %r (new)" % (set(info_old).difference(ignore), set(info_new).difference(ignore))) return False for block_name in info_old: if block_name in ignore or block_name not in info_new: continue block1 = deepcopy_where_possible(info_old[block_name]) block2 = deepcopy_where_possible(info_new[block_name]) # First, deal with root-level options (force, output, ...) if not isinstance(block1, dict): if block1 != block2: myprint(myname + ": different option '%s'" % block_name) return False continue # Now let's do components and params # 1. check order (it DOES matter, but just up to 1st level) f = list if strict else set if f(block1) != f(block2): myprint(myname + ": different [%s] or different order of them: %r vs %r" % (block_name, list(block1), list(block2))) return False # 2. Gather general options to be ignored if not strict: ignore_k = set() if block_name in [kinds.theory, kinds.likelihood]: ignore_k = ignore_k.union({_input_params, _output_params}) elif block_name == _params: for param in block1: # Unify notation block1[param] = expand_info_param(block1[param]) block2[param] = expand_info_param(block2[param]) ignore_k = ignore_k.union({ partag.latex, partag.renames, partag.ref, partag.proposal, "min", "max" }) # Fixed params, it doesn't matter if they are saved as derived if partag.value in block1[param]: block1[param].pop(partag.derived, None) if partag.value in block2[param]: block2[param].pop(partag.derived, None) # Renames: order does not matter block1[param][partag.renames] = set(block1[param].get( partag.renames, [])) block2[param][partag.renames] = set(block2[param].get( partag.renames, [])) # 3. Now check component/parameters one-by-one for k in block1: if not strict: # Add component-specific options to be ignored if block_name in kinds: ignore_k_this = ignore_k.copy() if _external not in block1[k]: try: component_path = block1[k].pop(_component_path, None) \ if isinstance(block1[k], dict) else None class_name = (block1[k] or {}).get(_class_name) or k cls = get_class(class_name, block_name, component_path=component_path) ignore_k_this = ignore_k_this.union( set(getattr(cls, "_at_resume_prefer_new", {}))) except ImportError: pass # Pop ignored and kept options for j in list(ignore_k_this): block1[k].pop(j, None) block2[k].pop(j, None) if block1[k] != block2[k]: # For clarity, pop common stuff before printing to_pop = [ j for j in block1[k] if (block1[k].get(j) == block2[k].get(j)) ] [(block1[k].pop(j, None), block2[k].pop(j, None)) for j in to_pop] myprint(myname + ": different content of [%s:%s]" % (block_name, k) + " -- (re-run with `debug: True` for more info)") myprint_debug("%r (old) vs %r (new)" % (block1[k], block2[k])) return False return True