def pytest_addoption(parser): parser.addoption("--" + packages_path_arg_posix, action="store", default=resolve_packages_path(), help="Path to folder of automatic installation of packages") parser.addoption("--skip-not-installed", action="store_true", help="Skip tests for which dependencies of used components are not " "installed.")
def refresh_display(self, info): QApplication.setOverrideCursor(Qt.WaitCursor) try: comments = info.pop(input_database._comment, None) comments_text = "\n# " + "\n# ".join(comments) except ( TypeError, # No comments AttributeError ): # Failed to generate info (returned str instead) comments_text = "" self.display["python"].setText("info = " + pformat(info) + comments_text) self.display["yaml"].setText( yaml_dump(sort_cosmetic(info)) + comments_text) self.display["bibliography"].setText( prettyprint_bib(*get_bib_info(info))) # Display covmat packages_path = resolve_packages_path() if not packages_path: self.covmat_text.setText( "\nIn order to find a covariance matrix, you need to define an external " "packages installation path, e.g. via the env variable %r.\n" % _packages_path_env) elif any(not os.path.isdir(d.format(**{_packages_path: packages_path})) for d in covmat_folders): self.covmat_text.setText( "\nThe external cosmological packages appear not to be installed where " "expected: %r\n" % packages_path) else: covmat_data = get_best_covmat(info, packages_path=packages_path) self.current_params_in_covmat = covmat_data["params"] self.current_covmat = covmat_data["covmat"] _, corrmat = cov_to_std_and_corr(self.current_covmat) self.covmat_text.setText( "\nCovariance file: %r\n\n" "NB: you do *not* need to save or copy this covariance matrix: " "it will be selected automatically.\n" % covmat_data["name"]) self.covmat_table.setRowCount(len(self.current_params_in_covmat)) self.covmat_table.setColumnCount(len( self.current_params_in_covmat)) self.covmat_table.setHorizontalHeaderLabels( list(self.current_params_in_covmat)) self.covmat_table.setVerticalHeaderLabels( list(self.current_params_in_covmat)) for i, pi in enumerate(self.current_params_in_covmat): for j, pj in enumerate(self.current_params_in_covmat): self.covmat_table.setItem( i, j, QTableWidgetItem("%g" % self.current_covmat[i, j])) if i != j: color = [ 256 * c for c in cmap_corr(corrmat[i, j] / 2 + 0.5)[:3] ] else: color = [255.99] * 3 self.covmat_table.item(i, j).setBackground(QColor(*color)) self.covmat_table.item(i, j).setForeground(QColor(0, 0, 0)) QApplication.restoreOverrideCursor()
def __init__(self, info: InfoDictIn = empty_dict, name: Optional[str] = None, timing: Optional[bool] = None, packages_path: Optional[str] = None, initialize=True, standalone=True): if standalone: # TODO: would probably be more natural if defaults were always read here default_info = self.get_defaults(input_options=info) default_info.update(info) info = default_info self.set_instance_defaults() self._name = name or self.get_qualified_class_name() self.packages_path = packages_path or resolve_packages_path() # set attributes from the info (from yaml file or directly input dictionary) for k, value in info.items(): try: # MARKED FOR DEPRECATION IN v3.0 # NB: cannot ever raise an error, since users may use "path_install" for # their own purposes. When considered *fully* deprecated, simply # remove this whole block. if k == "path_install": self.log.warning( "*DEPRECATION*: `path_install` will be deprecated " "in the next version. Please use `packages_path` instead." ) setattr(self, packages_path_input, value) # END OF DEPRECATION BLOCK setattr(self, k, value) except AttributeError: raise AttributeError("Cannot set {} attribute for {}!".format( k, self)) self.set_logger(name=self._name) self.set_timing_on(timing) try: if initialize: self.initialize() except AttributeError as e: if '_params' in str(e): raise LoggedError( self.log, "use 'initialize_with_params' if you need to " "initialize after input and output parameters" " are set (%s, %s)", self, e) raise
def __init__(self, info=empty_dict, name=None, timing=None, packages_path=None, initialize=True, standalone=True): if standalone: # TODO: would probably be more natural if defaults were always read here default_info = self.get_defaults(input_options=info) default_info.update(info) info = default_info self.set_instance_defaults() self._name = name or self.get_qualified_class_name() self.packages_path = packages_path or resolve_packages_path() # set attributes from the info (from yaml file or directly input dictionary) for k, value in info.items(): try: # MARKED FOR DEPRECATION IN v3.0 if k == "path_install": self.log.warning( "*DEPRECATION*: `path_install` will be deprecated " "in the next version. Please use `packages_path` instead." ) # BEHAVIOUR TO BE REPLACED BY ERROR: # set BOTH old and new names, just in case old one is used internally from cobaya.conventions import _packages_path setattr(self, _packages_path, value) # END OF DEPRECATION BLOCK setattr(self, k, value) except AttributeError: raise AttributeError("Cannot set {} attribute for {}!".format( k, self)) self.set_logger(name=self._name) self.set_timing_on(timing) try: if initialize: self.initialize() except AttributeError as e: if '_params' in str(e): raise LoggedError( self.log, "use 'initialize_with_params' if you need to " "initialize after input and output parameters" " are set (%s, %s)", self, e) raise
def __init__(self, *args, **kwargs): # Ensure check for install and version errors # (e.g. may inherit from a class that inherits from this one, and not have them) if self.install_options: name = self.get_qualified_class_name() logger = get_logger(name) packages_path = kwargs.get( "packages_path") or resolve_packages_path() old = False try: installed = self.is_installed(path=packages_path) except Exception as excpt: # catches VersionCheckError and unexpected ones installed = False old = isinstance(excpt, VersionCheckError) logger.error(f"{type(excpt).__name__}: {excpt}") if not installed: not_or_old = ("is not up to date" if old else "has not been correctly installed") raise ComponentNotInstalledError(logger, ( f"The data for this likelihood {not_or_old}. To install it, " f"run `cobaya-install {name}{' --upgrade' if old else ''}`" )) super().__init__(*args, **kwargs)
def install(*infos, **kwargs): if not log.root.handlers: logger_setup() path = kwargs.get("path") if not path: path = resolve_packages_path(infos) if not path: raise LoggedError( log, "No 'path' argument given, and none could be found in input infos " "(as %r), the %r env variable or the config file. " "Maybe specify one via a command line argument '-%s [...]'?", _packages_path, _packages_path_env, _packages_path_arg[0]) abspath = os.path.abspath(path) log.info("Installing external packages at '%s'", abspath) kwargs_install = { "force": kwargs.get("force", False), "no_progress_bars": kwargs.get("no_progress_bars") } for what in (_code, _data): kwargs_install[what] = kwargs.get(what, True) spath = os.path.join(abspath, what) if kwargs_install[what] and not os.path.exists(spath): try: os.makedirs(spath) except OSError: raise LoggedError( log, "Could not create the desired installation folder '%s'", spath) failed_components = [] skip_keywords = set(kwargs.get("skip", []) or []) skip_keywords_env = set( os.environ.get(_install_skip_env, "").replace(",", " ").lower().split()) skip_keywords = skip_keywords.union(skip_keywords_env) for kind, components in get_used_components(*infos).items(): for component in components: print() print(create_banner(kind + ":" + component, symbol="=", length=80)) print() if _skip_helper(component.lower(), skip_keywords, skip_keywords_env, log): continue info = (next( info for info in infos if component in info.get(kind, {}))[kind][component]) or {} if isinstance(info, str) or _external in info: log.warning( "Component '%s' is a custom function. " "Nothing to do.", component) continue try: imported_class = get_class(component, kind, component_path=info.pop( _component_path, None)) except ImportError as e: log.error("Component '%s' not recognized. [%s]." % (component, e)) failed_components += ["%s:%s" % (kind, component)] continue else: if _skip_helper(imported_class.__name__.lower(), skip_keywords, skip_keywords_env, log): continue is_installed = getattr(imported_class, "is_installed", None) if is_installed is None: log.info("%s.%s is a fully built-in component: nothing to do.", kind, imported_class.__name__) continue if is_installed(path=abspath, **kwargs_install): log.info("External component already installed.") if kwargs.get("just_check", False): continue if kwargs_install["force"]: log.info("Forcing re-installation, as requested.") else: log.info("Doing nothing.") continue else: if kwargs.get("just_check", False): log.info("NOT INSTALLED!") continue try: install_this = getattr(imported_class, "install", None) success = install_this(path=abspath, **kwargs_install) except KeyboardInterrupt: raise except: traceback.print_exception(*sys.exc_info(), file=sys.stdout) log.error( "An unknown error occurred. Delete the external packages " "folder %r and try again. " "Please, notify the developers if this error persists.", abspath) success = False if success: log.info("Successfully installed!") else: log.error( "Installation failed! Look at the error messages above. " "Solve them and try again, or, if you are unable to solve, " "install the packages required by this component manually." ) failed_components += ["%s:%s" % (kind, component)] continue # test installation if not is_installed(path=abspath, **kwargs_install): log.error( "Installation apparently worked, " "but the subsequent installation test failed! " "Look at the error messages above. " "Solve them and try again, or, if you are unable to solve, " "install the packages required by this component manually." ) failed_components += ["%s:%s" % (kind, component)] if failed_components: bullet = "\n - " raise LoggedError( log, "The installation (or installation test) of some component(s) has " "failed: %s\nCheck output of the installer of each component above " "for precise error info.\n", bullet + bullet.join(failed_components)) # Set the installation path in the global config file if not kwargs.get("no_set_global", False) and not kwargs.get( "just_check", False): write_packages_path_in_config_file(abspath) log.info( "The installation path has been written in the global config file." )
def install_script(): set_mpi_disabled(True) warn_deprecation() # Parse arguments import argparse parser = argparse.ArgumentParser( description="Cobaya's installation tool for external packages.") parser.add_argument( "files_or_components", action="store", nargs="+", metavar="input_file.yaml|component_name", help="One or more input files or component names " "(or simply 'cosmo' to install all the requisites for basic" " cosmological runs)") parser.add_argument( "-" + _packages_path_arg[0], "--" + _packages_path_arg_posix, action="store", nargs=1, required=False, metavar="/packages/path", default=[None], help="Desired path where to install external packages. " "Optional if one has been set globally or as an env variable" " (run with '--show_%s' to check)." % _packages_path_arg_posix) # MARKED FOR DEPRECATION IN v3.0 modules = "modules" parser.add_argument("-" + modules[0], "--" + modules, action="store", nargs=1, required=False, metavar="/packages/path", default=[None], help="To be deprecated! " "Alias for %s, which should be used instead." % _packages_path_arg_posix) # END OF DEPRECATION BLOCK -- CONTINUES BELOW! output_show_packages_path = resolve_packages_path() if output_show_packages_path and os.environ.get(_packages_path_env): output_show_packages_path += " (from env variable %r)" % _packages_path_env elif output_show_packages_path: output_show_packages_path += " (from config file)" else: output_show_packages_path = "(Not currently set.)" parser.add_argument( "--show-" + _packages_path_arg_posix, action="version", version=output_show_packages_path, help="Prints default external packages installation folder " "and exits.") parser.add_argument( "-" + _force[0], "--" + _force, action="store_true", default=False, help="Force re-installation of apparently installed packages.") parser.add_argument( "--skip", action="store", nargs="*", metavar="keyword", help="Keywords of components that will be skipped during " "installation.") parser.add_argument( "--no-progress-bars", action="store_true", default=False, help="No progress bars shown. Shorter logs (used in Travis).") parser.add_argument("--just-check", action="store_true", default=False, help="Just check whether components are installed.") parser.add_argument( "--no-set-global", action="store_true", default=False, help="Do not store the installation path for later runs.") group_just = parser.add_mutually_exclusive_group(required=False) group_just.add_argument("-C", "--just-code", action="store_false", default=True, help="Install code of the components.", dest=_data) group_just.add_argument("-D", "--just-data", action="store_false", default=True, help="Install data of the components.", dest=_code) arguments = parser.parse_args() # Configure the logger ASAP logger_setup() logger = logging.getLogger(__name__.split(".")[-1]) # Gather requests infos = [] for f in arguments.files_or_components: if f.lower() == "cosmo": logger.info("Installing basic cosmological packages.") from cobaya.cosmo_input import install_basic infos += [install_basic] elif f.lower() == "cosmo-tests": logger.info("Installing *tested* cosmological packages.") from cobaya.cosmo_input import install_tests infos += [install_tests] elif os.path.splitext(f)[1].lower() in _yaml_extensions: from cobaya.input import load_input infos += [load_input(f)] else: try: kind = get_kind(f) infos += [{kind: {f: None}}] except Exception: logger.warning("Could not identify component %r. Skipping.", f) if not infos: logger.info("Nothing to install.") return # MARKED FOR DEPRECATION IN v3.0 if getattr(arguments, modules) != [None]: logger.warning( "*DEPRECATION*: -m/--modules will be deprecated in favor of " "-%s/--%s in the next version. Please, use that one instead.", _packages_path_arg[0], _packages_path_arg_posix) # BEHAVIOUR TO BE REPLACED BY ERROR: if getattr(arguments, _packages_path_arg) == [None]: setattr(arguments, _packages_path_arg, getattr(arguments, modules)) # END OF DEPRECATION BLOCK # Launch installer install(*infos, path=getattr(arguments, _packages_path_arg)[0], **{ arg: getattr(arguments, arg) for arg in [ "force", _code, _data, "no_progress_bars", "just_check", "no_set_global", "skip" ] })
def install(*infos, **kwargs): debug = kwargs.get(_debug) if not log.root.handlers: logger_setup() path = kwargs.get("path") if not path: path = resolve_packages_path(infos) if not path: raise LoggedError( log, "No 'path' argument given, and none could be found in input infos " "(as %r), the %r env variable or the config file. " "Maybe specify one via a command line argument '-%s [...]'?", _packages_path, _packages_path_env, _packages_path_arg[0]) abspath = os.path.abspath(path) log.info("Installing external packages at '%s'", abspath) kwargs_install = {"force": kwargs.get("force", False), "no_progress_bars": kwargs.get("no_progress_bars")} for what in (_code, _data): kwargs_install[what] = kwargs.get(what, True) spath = os.path.join(abspath, what) if kwargs_install[what] and not os.path.exists(spath): try: os.makedirs(spath) except OSError: raise LoggedError( log, "Could not create the desired installation folder '%s'", spath) failed_components = [] skip_keywords_arg = set(kwargs.get("skip", []) or []) # NB: if passed with quotes as `--skip "a b"`, it's interpreted as a single key skip_keywords_arg = set(chain(*[word.split() for word in skip_keywords_arg])) skip_keywords_env = set( os.environ.get(_install_skip_env, "").replace(",", " ").lower().split()) skip_keywords = skip_keywords_arg.union(skip_keywords_env) for kind, components in get_used_components(*infos).items(): for component in components: print() print(create_banner(kind + ":" + component, symbol=_banner_symbol, length=_banner_length), end="") print() if _skip_helper(component.lower(), skip_keywords, skip_keywords_env, log): continue info = (next(info for info in infos if component in info.get(kind, {}))[kind][component]) or {} if isinstance(info, str) or _external in info: log.warning("Component '%s' is a custom function. " "Nothing to do.", component) continue try: imported_class = get_class(component, kind, component_path=info.pop(_component_path, None)) except ImportError as excpt: log.error("Component '%s' not recognized. [%s].", component, excpt) failed_components += ["%s:%s" % (kind, component)] continue else: if _skip_helper(imported_class.__name__.lower(), skip_keywords, skip_keywords_env, log): continue is_compatible = getattr(imported_class, "is_compatible", lambda: True)() if not is_compatible: log.info( "Skipping %r because it is not compatible with your OS.", component) continue log.info("Checking if dependencies have already been installed...") is_installed = getattr(imported_class, "is_installed", None) if is_installed is None: log.info("%s.%s is a fully built-in component: nothing to do.", kind, imported_class.__name__) continue install_path = abspath get_path = getattr(imported_class, "get_path", None) if get_path: install_path = get_path(install_path) has_been_installed = False if not debug: logging.disable(logging.ERROR) if kwargs.get("skip_global"): has_been_installed = is_installed(path="global", **kwargs_install) if not has_been_installed: has_been_installed = is_installed(path=install_path, **kwargs_install) if not debug: logging.disable(logging.NOTSET) if has_been_installed: log.info("External dependencies for this component already installed.") if kwargs.get(_test_run, False): continue if kwargs_install["force"] and not kwargs.get("skip_global"): log.info("Forcing re-installation, as requested.") else: log.info("Doing nothing.") continue else: log.info("Installation check failed!") if not debug: log.info( "(If you expected this to be already installed, re-run " "`cobaya-install` with --debug to get more verbose output.)") if kwargs.get(_test_run, False): continue log.info("Installing...") try: install_this = getattr(imported_class, "install", None) success = install_this(path=abspath, **kwargs_install) except KeyboardInterrupt: raise except: traceback.print_exception(*sys.exc_info(), file=sys.stdout) log.error("An unknown error occurred. Delete the external packages " "folder %r and try again. " "Please, notify the developers if this error persists.", abspath) success = False if success: log.info("Successfully installed! Let's check it...") else: log.error("Installation failed! Look at the error messages above. " "Solve them and try again, or, if you are unable to solve, " "install the packages required by this component manually.") failed_components += ["%s:%s" % (kind, component)] continue # test installation if not debug: logging.disable(logging.ERROR) successfully_installed = is_installed(path=install_path, **kwargs_install) if not debug: logging.disable(logging.NOTSET) if not successfully_installed: log.error("Installation apparently worked, " "but the subsequent installation test failed! " "Look at the error messages above, or re-run with --debug " "for more more verbose output. " "Try to solve the issues and try again, or, if you are unable " "to solve them, install the packages required by this " "component manually.") failed_components += ["%s:%s" % (kind, component)] else: log.info("Installation check successful.") print() print(create_banner(" * Summary * ", symbol=_banner_symbol, length=_banner_length), end="") print() if failed_components: bullet = "\n - " raise LoggedError( log, "The installation (or installation test) of some component(s) has " "failed: %s\nCheck output of the installer of each component above " "for precise error info.\n", bullet + bullet.join(failed_components)) log.info("All requested components' dependencies correctly installed.") # Set the installation path in the global config file if not kwargs.get("no_set_global", False) and not kwargs.get(_test_run, False): write_packages_path_in_config_file(abspath) log.info("The installation path has been written into the global config file: %s", os.path.join(get_config_path(), _packages_path_config_file))
def install_script(args=None): set_mpi_disabled() warn_deprecation() # Parse arguments import argparse parser = argparse.ArgumentParser( prog="cobaya install", description="Cobaya's installation tool for external packages.") parser.add_argument( "files_or_components", action="store", nargs="+", metavar="input_file.yaml|component_name", help="One or more input files or component names " "(or simply 'cosmo' to install all the requisites for basic" " cosmological runs)") parser.add_argument( "-" + packages_path_arg[0], "--" + packages_path_arg_posix, action="store", required=False, metavar="/packages/path", default=None, help="Desired path where to install external packages. " "Optional if one has been set globally or as an env variable" " (run with '--show_%s' to check)." % packages_path_arg_posix) # MARKED FOR DEPRECATION IN v3.0 modules = "modules" parser.add_argument("-" + modules[0], "--" + modules, action="store", required=False, metavar="/packages/path", default=None, help="Deprecated! Use %s instead." % packages_path_arg_posix) # END OF DEPRECATION BLOCK -- CONTINUES BELOW! output_show_packages_path = resolve_packages_path() if output_show_packages_path and os.environ.get(packages_path_env): output_show_packages_path += " (from env variable %r)" % packages_path_env elif output_show_packages_path: output_show_packages_path += " (from config file)" else: output_show_packages_path = "(Not currently set.)" parser.add_argument( "--show-" + packages_path_arg_posix, action="version", version=output_show_packages_path, help="Prints default external packages installation folder " "and exits.") parser.add_argument( "-" + "f", "--" + "force", action="store_true", default=False, help="Force re-installation of apparently installed packages.") parser.add_argument( "--skip", action="store", nargs="*", metavar="keyword", help="Keywords of components that will be skipped during " "installation.") parser.add_argument( "--no-progress-bars", action="store_true", default=False, help="No progress bars shown. Shorter logs (used in Travis).") parser.add_argument("--%s" % "test", action="store_true", default=False, help="Just check whether components are installed.") # MARKED FOR DEPRECATION IN v3.0 parser.add_argument("--just-check", action="store_true", default=False, help="Just check whether components are installed.") # END OF DEPRECATION BLOCK -- CONTINUES BELOW! parser.add_argument( "--no-set-global", action="store_true", default=False, help="Do not store the installation path for later runs.") parser.add_argument( "--skip-global", action="store_true", default=False, help="Skip installation of already-available Python modules.") parser.add_argument("-" + "d", "--" + "debug", action="store_true", help="Produce verbose debug output.") group_just = parser.add_mutually_exclusive_group(required=False) group_just.add_argument("-C", "--just-code", action="store_false", default=True, help="Install code of the components.", dest=data_path) group_just.add_argument("-D", "--just-data", action="store_false", default=True, help="Install data of the components.", dest=code_path) arguments = parser.parse_args(args) # Configure the logger ASAP logger_setup() logger = get_logger("install") # Gather requests infos: List[InputDict] = [] for f in arguments.files_or_components: if f.lower() == "cosmo": logger.info("Installing basic cosmological packages.") from cobaya.cosmo_input import install_basic infos += [install_basic] elif f.lower() == "cosmo-tests": logger.info("Installing *tested* cosmological packages.") from cobaya.cosmo_input import install_tests infos += [install_tests] elif os.path.splitext(f)[1].lower() in Extension.yamls: from cobaya.input import load_input infos += [load_input(f)] else: try: kind = get_kind(f) infos += [{kind: {f: None}}] except Exception: logger.warning("Could not identify component %r. Skipping.", f) if not infos: logger.info("Nothing to install.") return # List of deprecation warnings, to be printed *after* installation deprecation_warnings = [] # MARKED FOR DEPRECATION IN v3.0 if getattr(arguments, modules) is not None: raise LoggedError( logger, "-m/--modules has been deprecated in favor of " "-%s/--%s", packages_path_arg[0], packages_path_arg_posix) # END OF DEPRECATION BLOCK # MARKED FOR DEPRECATION IN v3.0 if arguments.just_check is True: raise LoggedError(logger, "--just-check has been deprecated in favor of --%s", "test") # END OF DEPRECATION BLOCK # Launch installer install(*infos, path=getattr(arguments, packages_path_arg), **{ arg: getattr(arguments, arg) for arg in [ "force", code_path, data_path, "no_progress_bars", "test", "no_set_global", "skip", "skip_global", "debug" ] }) # MARKED FOR DEPRECATION IN v3.0 for warning_msg in deprecation_warnings: logger.warning(warning_msg)
def install_script(args=None): """Command line script for the installer.""" set_mpi_disabled() warn_deprecation() # Parse arguments import argparse parser = argparse.ArgumentParser( prog="cobaya install", description="Cobaya's installation tool for external packages.") parser.add_argument( "files_or_components", action="store", nargs="+", metavar="input_file.yaml|component_name", help="One or more input files or component names " "(or simply 'cosmo' to install all the requisites for basic" " cosmological runs)") parser.add_argument( "-" + packages_path_arg[0], "--" + packages_path_arg_posix, action="store", required=False, metavar="/packages/path", default=None, help="Desired path where to install external packages. " "Optional if one has been set globally or as an env variable" " (run with '--show_%s' to check)." % packages_path_arg_posix) output_show_packages_path = resolve_packages_path() if output_show_packages_path and os.environ.get(packages_path_env): output_show_packages_path += " (from env variable %r)" % packages_path_env elif output_show_packages_path: output_show_packages_path += " (from config file)" else: output_show_packages_path = "(Not currently set.)" parser.add_argument( "--show-" + packages_path_arg_posix, action="version", version=output_show_packages_path, help="Prints default external packages installation folder " "and exits.") parser.add_argument( "-" + "f", "--" + "force", action="store_true", default=False, help="Force re-installation of apparently installed packages.") parser.add_argument("--%s" % "test", action="store_true", default=False, help="Just check whether components are installed.") parser.add_argument("--upgrade", action="store_true", default=False, help="Force upgrade of obsolete components.") parser.add_argument("--skip", action="store", nargs="*", metavar="keyword", help=("Keywords of components that will be " "skipped during installation.")) parser.add_argument( "--skip-global", action="store_true", default=False, help="Skip installation of already-available Python modules.") parser.add_argument("-" + "d", "--" + "debug", action="store_true", help="Produce verbose debug output.") group_just = parser.add_mutually_exclusive_group(required=False) group_just.add_argument("-C", "--just-code", action="store_false", default=True, help="Install code of the components.", dest=data_path) group_just.add_argument("-D", "--just-data", action="store_false", default=True, help="Install data of the components.", dest=code_path) parser.add_argument( "--no-progress-bars", action="store_true", default=False, help=("No progress bars shown; use when output is saved into a " "text file (e.g. when running on a cluster).")) parser.add_argument( "--no-set-global", action="store_true", default=False, help="Do not store the installation path for later runs.") arguments = parser.parse_args(args) # Configure the logger ASAP logger_setup(arguments.debug) logger = get_logger("install") # Gather requests infos: List[Union[InputDict, str]] = [] for f in arguments.files_or_components: if f.lower() == "cosmo": logger.info("Installing basic cosmological packages.") from cobaya.cosmo_input import install_basic infos += [install_basic] elif f.lower() == "cosmo-tests": logger.info("Installing *tested* cosmological packages.") from cobaya.cosmo_input import install_tests infos += [install_tests] elif os.path.splitext(f)[1].lower() in Extension.yamls: from cobaya.input import load_input infos += [load_input(f)] else: # a single component name, no kind specified infos += [f] # Launch installer install(*infos, path=getattr(arguments, packages_path_arg), logger=logger, **{ arg: getattr(arguments, arg) for arg in [ "force", code_path, data_path, "no_progress_bars", "test", "no_set_global", "skip", "skip_global", "debug", "upgrade" ] })
def install(*infos, **kwargs): """ Installs the external packages required by the components mentioned in ``infos``. ``infos`` can be input dictionaries or single component names. :param force: force re-installation of apparently installed packages (default: ``False``). :param test: just check whether components are installed (default: ``False``). :param upgrade: force upgrade of obsolete components (default: ``False``). :param skip: keywords of components that will be skipped during installation. :param skip_global: skip installation of already-available Python modules (default: ``False``). :param debug: produce verbose debug output (default: ``False``). :param code: set to ``False`` to skip code packages (default: ``True``). :param data: set to ``False`` to skip data packages (default: ``True``). :param no_progress_bars: no progress bars shown; use when output is saved into a text file (e.g. when running on a cluster) (default: ``False``). :param no_set_global: do not store the installation path for later runs (default: ``False``). """ debug = kwargs.get("debug", False) logger = kwargs.get("logger") if not logger: logger_setup(debug=debug) logger = get_logger("install") path = kwargs.get("path") infos_not_single_names = [ info for info in infos if isinstance(info, Mapping) ] if not path: path = resolve_packages_path(*infos_not_single_names) if not path: raise LoggedError(logger, ( "No 'path' argument given, and none could be found in input infos " "(as %r), the %r env variable or the config file. " "Maybe specify one via a command line argument '-%s [...]'?"), packages_path_input, packages_path_env, packages_path_arg[0]) # General install path for all dependencies general_abspath = os.path.abspath(path) logger.info("Installing external packages at '%s'", general_abspath) # Set the installation path in the global config file if not kwargs.get("no_set_global", False) and not kwargs.get( "test", False): write_packages_path_in_config_file(general_abspath) logger.info( "The installation path has been written into the global config file: %s", os.path.join(get_config_path(), packages_path_config_file)) kwargs_install = { "force": kwargs.get("force", False), "no_progress_bars": kwargs.get("no_progress_bars") } for what in (code_path, data_path): kwargs_install[what] = kwargs.get(what, True) spath = os.path.join(general_abspath, what) if kwargs_install[what] and not os.path.exists(spath): try: os.makedirs(spath) except OSError: raise LoggedError( logger, f"Could not create the desired installation folder '{spath}'" ) # To check e.g. for a version upgrade, it needs to reload the component class and # all relevant imported modules: the implementation of `is_installed` for each # class is expected to always reload external modules if passed `reload=True` # (should be False by default to avoid deleting objects unnecessarily). kwargs_is_installed = {"reload": True} unknown_components = [] # could not be identified failed_components = [] # general errors obsolete_components = [] # older or unknown version already installed skip_keywords_arg = set(kwargs.get("skip", []) or []) # NB: if passed with quotes as `--skip "a b"`, it's interpreted as a single key skip_keywords_arg = set( chain(*[word.split() for word in skip_keywords_arg])) skip_keywords_env = set( os.environ.get(install_skip_env, "").replace(",", " ").lower().split()) skip_keywords = skip_keywords_arg.union(skip_keywords_env) # Combine all requested components and install them # NB: components mentioned by name may be repeated with those given in dict infos. # That's OK, because the install check will skip them in the 2nd pass used_components, components_infos = get_used_components(*infos, return_infos=True) for kind, components in used_components.items(): for component in components: name_w_kind = (kind + ":" if kind else "") + component print() print(create_banner(name_w_kind, symbol=_banner_symbol, length=_banner_length), end="") print() if _skip_helper(component.lower(), skip_keywords, skip_keywords_env, logger): continue info = components_infos[component] if isinstance(info, str) or "external" in info: logger.info( f"Component '{name_w_kind}' is a custom function. Nothing to do." ) continue try: class_name = (info or {}).get("class") if class_name: logger.info( f"Class to be installed for this component: {class_name}" ) imported_class = get_component_class(component, kind=kind, component_path=info.pop( "python_path", None), class_name=class_name, logger=logger) # Update the name if the kind was unknown if not kind: name_w_kind = imported_class.get_kind() + ":" + component except ComponentNotFoundError: logger.error( f"Component '{name_w_kind}' could not be identified. Skipping." ) unknown_components += [name_w_kind] continue except Exception: traceback.print_exception(*sys.exc_info(), file=sys.stdout) logger.error( f"An error occurred when loading '{name_w_kind}'. Skipping." ) failed_components += [name_w_kind] continue else: if _skip_helper(imported_class.__name__.lower(), skip_keywords, skip_keywords_env, logger): continue is_compatible = getattr(imported_class, "is_compatible", lambda: True)() if not is_compatible: logger.error(f"Skipping '{name_w_kind}' " "because it is not compatible with your OS.") failed_components += [name_w_kind] continue logger.info( "Checking if dependencies have already been installed...") is_installed = getattr(imported_class, "is_installed", None) if is_installed is None: logger.info( f"Component '{name_w_kind}' is a fully built-in component: " "nothing to do.") continue this_component_install_path = general_abspath get_path = getattr(imported_class, "get_path", None) if get_path: this_component_install_path = get_path( this_component_install_path) # Check previous installations and their versions has_been_installed = False is_old_version_msg = None with NoLogging(None if debug else logging.ERROR): try: if kwargs.get("skip_global"): has_been_installed = is_installed( path="global", **kwargs_install, **kwargs_is_installed) if not has_been_installed: has_been_installed = is_installed( path=this_component_install_path, **kwargs_install, **kwargs_is_installed) except VersionCheckError as excpt: is_old_version_msg = str(excpt) if has_been_installed: # no VersionCheckError was raised logger.info( "External dependencies for this component already installed." ) if kwargs.get("test", False): continue if kwargs_install["force"] and not kwargs.get("skip_global"): logger.info("Forcing re-installation, as requested.") else: logger.info("Doing nothing.") continue elif is_old_version_msg: logger.info(f"Version check failed: {is_old_version_msg}") obsolete_components += [name_w_kind] if kwargs.get("test", False): continue if not kwargs.get("upgrade", False) and not kwargs.get( "force", False): logger.info("Skipping because '--upgrade' not requested.") continue else: logger.info("Check found no existing installation") if not debug: logger.info( "(If you expected this to be already installed, re-run " "`cobaya-install` with --debug to get more verbose output.)" ) if kwargs.get("test", False): # We are only testing whether it was installed, so consider it failed failed_components += [name_w_kind] continue # Do the install logger.info("Installing...") try: install_this = getattr(imported_class, "install", None) success = install_this(path=general_abspath, **kwargs_install) except Exception: traceback.print_exception(*sys.exc_info(), file=sys.stdout) logger.error( "An unknown error occurred. Delete the external packages " "folder %r and try again. " "Please, notify the developers if this error persists.", general_abspath) success = False if success: logger.info("Successfully installed! Let's check it...") else: logger.error( "Installation failed! Look at the error messages above. " "Solve them and try again, or, if you are unable to solve them, " "install the packages required by this component manually." ) failed_components += [name_w_kind] continue # Test installation reloaded_class = get_component_class(component, kind=kind, component_path=info.pop( "python_path", None), class_name=class_name, logger=logger) reloaded_is_installed = getattr(reloaded_class, "is_installed", None) with NoLogging(None if debug else logging.ERROR): try: successfully_installed = reloaded_is_installed( path=this_component_install_path, **kwargs_install, **kwargs_is_installed) except Exception: traceback.print_exception(*sys.exc_info(), file=sys.stdout) successfully_installed = False if not successfully_installed: logger.error( "Installation apparently worked, " "but the subsequent installation test failed! " "This does not always mean that there was an actual error, " "and is sometimes fixed simply by running the installer " "again. If not, look closely at the error messages above, " "or re-run with --debug for more more verbose output. " "If you are unable to fix the issues above, " "try installing the packages required by this " "component manually.") failed_components += [name_w_kind] else: logger.info("Installation check successful.") print() print(create_banner(" * Summary * ", symbol=_banner_symbol, length=_banner_length), end="") print() bullet = "\n - " if unknown_components: suggestions_dict = { name: similar_internal_class_names(name) for name in unknown_components } suggestions_msg = \ bullet + bullet.join( f"{name}: did you mean any of the following? {sugg} " "(mind capitalization!)" for name, sugg in suggestions_dict.items()) raise LoggedError(logger, ( "The following components could not be identified and were skipped:" f"{suggestions_msg}")) if failed_components: raise LoggedError(logger, ( "The installation (or installation test) of some component(s) has " "failed: %s\nCheck output of the installer of each component above " "for precise error info.\n"), bullet + bullet.join(failed_components)) if obsolete_components: raise LoggedError(logger, ( "The following packages are obsolete. Re-run with `--upgrade` option" " (not upgrading by default to preserve possible user changes): %s" ), bullet + bullet.join(obsolete_components)) if not unknown_components and not failed_components and not obsolete_components: logger.info( "All requested components' dependencies correctly installed at " f"{general_abspath}")
def pytest_addoption(parser): parser.addoption( "--" + _packages_path_arg, action="store", default=resolve_packages_path(), help="Path to folder of automatic installation of packages")