def put(self): """Update Thoth config file.""" initial_path = Path.cwd() input_data = self.get_json_body() new_runtime_environment: str = input_data["runtime_environment"] force: bool = input_data["force"] complete_path: str = input_data["complete_path"] os.chdir(complete_path) configuration = _Configuration() if not configuration.config_file_exists(): _LOGGER.info("Thoth config does not exist, creating it...") try: configuration.create_default_config() except Exception as e: raise Exception("Thoth config file could not be created! %r", e) configuration.set_runtime_environment( runtime_environment=new_runtime_environment, force=force # TODO: force should be user choice? ) configuration.save_config() _LOGGER.info("Updated Thoth config: %r", configuration.content) os.chdir(initial_path) self.finish( json.dumps({ "message": f"Successfully updated thoth config at {initial_path}!" }))
def post(self): """Retrieve or create Thoth config file.""" initial_path = Path.cwd() input_data = self.get_json_body() kernel_name: str = input_data["kernel_name"] home = Path.home() complete_path = home.joinpath(".local/share/thoth/kernels") env_path = complete_path.joinpath(kernel_name) env_path.mkdir(parents=True, exist_ok=True) os.chdir(env_path) _LOGGER.info( f"kernel_name selected: {kernel_name} and path: {env_path}") config = _Configuration() if not config.config_file_exists(): _LOGGER.info("Thoth config does not exist, creating it...") try: config.create_default_config() except Exception as e: raise Exception("Thoth config file could not be created! %r", e) config.load_config() thoth_config = config.content _LOGGER.info("Thoth config: %r", thoth_config) os.chdir(initial_path) self.finish(json.dumps(thoth_config))
def get_thoth_config( kernel_name: str, kernels_path: Path = Path.home().joinpath(".local/share/thoth/kernels"), ) -> _Configuration: """Get Thoth config.""" initial_path = Path.cwd() env_path = kernels_path.joinpath(kernel_name) env_path.mkdir(parents=True, exist_ok=True) os.chdir(env_path) _LOGGER.info(f"kernel_name selected: {kernel_name} and path: {env_path}") config = _Configuration() if not config.config_file_exists(): _LOGGER.info("Thoth config does not exist, creating it...") try: config.create_default_config() except Exception as e: raise Exception("Thoth config file could not be created! %r", e) config.load_config() os.chdir(initial_path) return config
def qeb_hwt_thamos_advise() -> None: """Qeb-Hwt Thamos Advise Task.""" if not Configuration._REPO_PATH: raise Exception( f"No path has been provided to REPO_PATH env variable.") if not Path(Configuration._REPO_PATH).exists(): raise FileNotFoundError( f"Cannot find the file on this path: {Configuration._REPO_PATH}") OpenShift.verify_github_app_inputs( github_event_type=Configuration._GITHUB_EVENT_TYPE, github_check_run_id=Configuration._GITHUB_CHECK_RUN_ID, github_installation_id=Configuration._GITHUB_INSTALLATION_ID, github_base_repo_url=Configuration._GITHUB_BASE_REPO_URL, origin=Configuration._ORIGIN, ) os.chdir(Configuration._REPO_PATH) thoth_yaml_config = _Configuration() if not thoth_yaml_config.config_file_exists(): exception_message = _create_message_config_file_error(no_file=True) trigger_finished_webhook(exception_message=exception_message, has_error=True) return try: analysis_id = advise_here( nowait=True, github_event_type=Configuration._GITHUB_EVENT_TYPE, github_check_run_id=Configuration._GITHUB_CHECK_RUN_ID, github_installation_id=Configuration._GITHUB_INSTALLATION_ID, github_base_repo_url=Configuration._GITHUB_BASE_REPO_URL, origin=Configuration._ORIGIN, source_type=ThothAdviserIntegrationEnum.GITHUB_APP, ) _LOGGER.info("Successfully submitted thamos advise call.") except Exception as exception: if isinstance( exception, ( NoRuntimeEnvironmentError, NoRequirementsFormatError, FileNotFoundError, FileLoadError, KeyError, ValueError, AttributeError, ), ): _LOGGER.debug(exception) exception_message = str(exception) else: _LOGGER.debug(json.loads(exception.body)["error"]) exception_message = json.loads(exception.body)["error"] trigger_finished_webhook(exception_message=exception_message, has_error=True)
def qeb_hwt_thamos_advise() -> None: """Qeb-Hwt Thamos Advise Task.""" if not Configuration._REPO_PATH: raise Exception("No path has been provided to REPO_PATH env variable.") if not Path(Configuration._REPO_PATH).exists(): raise FileNotFoundError( f"Cannot find the file on this path: {Configuration._REPO_PATH}") output_messages: list = [] OpenShift.verify_github_app_inputs( github_event_type=Configuration._GITHUB_EVENT_TYPE, github_check_run_id=Configuration._GITHUB_CHECK_RUN_ID, github_installation_id=Configuration._GITHUB_INSTALLATION_ID, github_base_repo_url=Configuration._GITHUB_BASE_REPO_URL, origin=Configuration._ORIGIN, ) os.chdir(Configuration._REPO_PATH) thoth_yaml_config = _Configuration() if not thoth_yaml_config.config_file_exists(): exception_message = _create_message_config_file_error(no_file=True) store_messages(output_messages) trigger_finished_webhook(exception_message=exception_message, has_error=True, error_type="MissingThothYamlFile") return try: # Consider first runtime environment runtime_environment = thoth_yaml_config.content.get( "runtime_environments")[0] # Fetch recommendation type recommendation_type = runtime_environment.get( "recommendation_type") if thoth_yaml_config else "latest" requirements_format = thoth_yaml_config.requirements_format if requirements_format == "pipenv": project = Project.from_files( without_pipfile_lock=not os.path.exists("Pipfile.lock")) elif requirements_format in ("pip", "pip-tools", "pip-compile"): project = Project.from_pip_compile_files(allow_without_lock=True) else: raise ValueError( f"Unknown configuration option for requirements format: {requirements_format!r}" ) pipfile = project.pipfile.to_string() pipfile_lock_str = project.pipfile_lock.to_string( ) if project.pipfile_lock else "" application_stack = PythonStack( requirements=pipfile, requirements_lock=pipfile_lock_str, requirements_format=requirements_format).to_dict() except Exception as exception: _LOGGER.debug(json.loads(exception.body)["error"]) # type: ignore exception_message = json.loads(exception.body)["error"] # type: ignore store_messages(output_messages) trigger_finished_webhook(exception_message=exception_message, has_error=True) return # The input for AdviserTriggerMessage if no exceptions were found message_input = { "component_name": { "type": "str", "value": __COMPONENT_NAME__ }, "service_version": { "type": "str", "value": __service_version__ }, "application_stack": { "type": "Dict", "value": application_stack }, "runtime_environment": { "type": "Dict", "value": runtime_environment }, "recommendation_type": { "type": "str", "value": recommendation_type }, "github_event_type": { "type": "str", "value": Configuration._GITHUB_EVENT_TYPE }, "github_check_run_id": { "type": "int", "value": int(Configuration._GITHUB_CHECK_RUN_ID) }, "github_installation_id": { "type": "int", "value": int(Configuration._GITHUB_INSTALLATION_ID) }, "github_base_repo_url": { "type": "str", "value": Configuration._GITHUB_BASE_REPO_URL }, "origin": { "type": "str", "value": Configuration._ORIGIN }, "source_type": { "type": "str", "value": ThothAdviserIntegrationEnum.GITHUB_APP.name }, } # We store the message to put in the output file here. output_messages = [{ "topic_name": "thoth.adviser-trigger", "message_contents": message_input }] store_messages(output_messages)
def horus_lock_command( path: str, resolution_engine: str, timeout: int, force: bool, recommendation_type: str, kernel_name: typing.Optional[str] = None, os_name: typing.Optional[str] = None, os_version: typing.Optional[str] = None, python_version: typing.Optional[str] = None, save_in_notebook: bool = True, save_on_disk: bool = False, ): """Lock requirements in notebook metadata.""" results = {} results["kernel_name"] = "" results["runtime_environment"] = "" results["dependency_resolution_engine"] = resolution_engine notebook = get_notebook_content(notebook_path=path) notebook_metadata = notebook.get("metadata") kernelspec = notebook_metadata.get("kernelspec") notebook_kernel = kernelspec.get("name") if not kernel_name: kernel = notebook_kernel else: kernel = kernel_name if kernel == "python3": kernel = "jupyterlab-requirements" results["kernel_name"] = kernel requirements = notebook_metadata.get("requirements") if not requirements: raise KeyError( "No Pipfile identified in notebook metadata." "You can start creating one with command: " "`horus requirements [NOTEBOOK].ipynb --add [PACKAGE NAME]`") pipfile_ = Pipfile.from_string(requirements) error = False if resolution_engine == "thoth": thoth_config_string = notebook_metadata.get("thoth_config") if not thoth_config_string: thoth_config = get_thoth_config(kernel_name=kernel) else: thoth_config = _Configuration() thoth_config.load_config_from_string(thoth_config_string) try: notebook_content_py = get_notebook_content(notebook_path=path, py_format=True) except Exception as e: _LOGGER.error(f"Could not get notebook content!: {e!r}") notebook_content_py = "" runtime_environment = thoth_config.get_runtime_environment() runtime_environment["name"] = kernel operating_system = { "name": runtime_environment["operating_system"]["name"], "version": runtime_environment["operating_system"]["version"], } if os_name: operating_system["name"] = os_name if os_version: operating_system["version"] = os_version runtime_environment["operating_system"] = operating_system if python_version: runtime_environment["python_version"] = python_version runtime_environment["recommendation_type"] = recommendation_type results["runtime_environment"] = runtime_environment # Assign runtime environment to thoth config runtime environment. thoth_config.set_runtime_environment( runtime_environment=runtime_environment, force=True) _, lock_results = lock_dependencies_with_thoth( kernel_name=kernel, pipfile_string=requirements, config=json.dumps(thoth_config.content), timeout=timeout, force=force, notebook_content=notebook_content_py, ) if not lock_results["error"]: requirements = lock_results["requirements"] requirements_lock = lock_results["requirement_lock"] notebook_metadata["thoth_config"] = json.dumps(thoth_config) else: error = True if resolution_engine == "pipenv": _, lock_results = lock_dependencies_with_pipenv( kernel_name=kernel, pipfile_string=pipfile_.to_string()) if not lock_results["error"]: requirements_lock = lock_results["requirements_lock"] else: error = True if save_on_disk and not error: home = Path.home() store_path: Path = home.joinpath(".local/share/thoth/kernels") complete_path: Path = store_path.joinpath(kernel) complete_path.mkdir(parents=True, exist_ok=True) _LOGGER.info("Path used to store dependencies is: %r", complete_path.as_posix()) requirements_format = "pipenv" if resolution_engine == "thoth": project = Project.from_dict(requirements, requirements_lock) else: project = Project.from_dict(pipfile_.to_dict(), requirements_lock) pipfile_path = complete_path.joinpath("Pipfile") pipfile_lock_path = complete_path.joinpath("Pipfile.lock") if requirements_format == "pipenv": _LOGGER.debug("Writing to Pipfile/Pipfile.lock in %r", complete_path) project.to_files(pipfile_path=pipfile_path, pipfile_lock_path=pipfile_lock_path) if resolution_engine == "thoth": # thoth thoth_config_string = json.dumps(thoth_config.content) config = _Configuration() config.load_config_from_string(thoth_config_string) config_path = complete_path.joinpath(".thoth.yaml") config.save_config(path=config_path) if save_in_notebook and not error: notebook_metadata["dependency_resolution_engine"] = resolution_engine notebook_metadata["requirements"] = requirements notebook_metadata["requirements_lock"] = requirements_lock # Assign kernel name to kernelspec. kernelspec["name"] = kernel notebook_metadata["kernelspec"] = kernelspec notebook["metadata"] = notebook_metadata save_notebook_content(notebook_path=path, notebook=notebook) return results, lock_results
def horus_show_command( path: str, pipfile: bool = False, pipfile_lock: bool = False, thoth_config: bool = False, ): """Horus show command.""" show_all: bool = False if not pipfile and not pipfile_lock and not thoth_config: # If no parameter to be shown is set, show all is set. show_all = True results = {} results["kernel_name"] = "" results["dependency_resolution_engine"] = "" results["pipfile"] = "" results["pipfile_lock"] = "" results["thoth_config"] = "" notebook = get_notebook_content(notebook_path=path) notebook_metadata = notebook.get("metadata") if notebook_metadata.get("language_info"): language = notebook_metadata["language_info"]["name"] if language and language != "python": raise Exception("Only Python kernels are currently supported.") if notebook_metadata.get("kernelspec"): kernelspec = notebook_metadata.get("kernelspec") kernel_name = kernelspec.get("name") else: kernel_name = "python3" results["kernel_name"] = kernel_name dependency_resolution_engine = notebook_metadata.get( "dependency_resolution_engine") results["dependency_resolution_engine"] = dependency_resolution_engine pipfile_string = notebook_metadata.get("requirements") if pipfile or pipfile_lock or show_all: if not pipfile_string: results["pipfile"] = "No Pipfile identified in notebook metadata." else: pipfile_ = Pipfile.from_string(pipfile_string) if pipfile or show_all: results["pipfile"] = f"\nPipfile:\n\n{pipfile_.to_string()}" if pipfile_lock or show_all: if pipfile_string: pipfile_lock_string = notebook_metadata.get("requirements_lock") if not pipfile_lock_string: results[ "pipfile_lock"] = "No Pipfile.lock identified in notebook metadata." else: pipfile_lock_ = PipfileLock.from_string( pipfile_content=pipfile_lock_string, pipfile=pipfile_) results[ "pipfile_lock"] = f"\nPipfile.lock:\n\n{pipfile_lock_.to_string()}" else: results[ "pipfile_lock"] = "No Pipfile identified in notebook metadata, therefore Pipfile.lock cannot be created." if thoth_config or show_all: thoth_config_string = notebook_metadata.get("thoth_config") if not thoth_config_string: results[ "thoth_config"] = "No .thoth.yaml identified in notebook metadata." else: config = _Configuration() config.load_config_from_string(thoth_config_string) results[ "thoth_config"] = f"\n.thoth.yaml:\n\n{yaml.dump(config.content)}" return results
def horus_extract_command( notebook_path: str, store_files_path: str, pipfile: bool = False, pipfile_lock: bool = False, thoth_config: bool = False, use_overlay: bool = False, force: bool = False, ): """Horus extract command.""" results = {} results["kernel_name"] = "" results["resolution_engine"] = "" extract_all: bool = False if not pipfile and not pipfile_lock and not thoth_config: # If no parameter to be extracted is set, extract all is set. extract_all = True notebook = get_notebook_content(notebook_path=notebook_path) notebook_metadata = notebook.get("metadata") if notebook_metadata.get("language_info"): language = notebook_metadata["language_info"]["name"] if language and language != "python": raise Exception("Only Python kernels are currently supported.") if notebook_metadata.get("kernelspec"): kernelspec = notebook_metadata.get("kernelspec") kernel_name = kernelspec.get("name") else: kernel_name = "python3" results["kernel_name"] = kernel_name store_path: Path = Path(store_files_path) if use_overlay: if not kernel_name: raise KeyError( "No kernel name identified in notebook metadata kernelspec.") store_path = store_path.joinpath("overlays").joinpath(kernel_name) store_path.mkdir(parents=True, exist_ok=True) dependency_resolution_engine = notebook_metadata.get( "dependency_resolution_engine") if not dependency_resolution_engine: raise KeyError("No Resolution engine identified in notebook metadata.") results["resolution_engine"] = dependency_resolution_engine if pipfile or pipfile_lock or extract_all: pipfile_string = notebook_metadata.get("requirements") if not pipfile_string: raise KeyError("No Pipfile identified in notebook metadata.") pipfile_ = Pipfile.from_string(pipfile_string) if pipfile or extract_all: pipfile_path = store_path.joinpath("Pipfile") if pipfile_path.exists() and not force: raise FileExistsError( f"Cannot store Pipfile because it already exists at path: {pipfile_path.as_posix()!r}. " "Use --force to overwrite existing content or --show-only to visualize it." ) else: pipfile_.to_file(path=pipfile_path) if pipfile_lock or extract_all: pipfile_lock_string = notebook_metadata.get("requirements_lock") if not pipfile_lock_string: raise KeyError("No Pipfile.lock identified in notebook metadata.") pipfile_lock_ = PipfileLock.from_string( pipfile_content=pipfile_lock_string, pipfile=pipfile_) pipfile_lock_path = store_path.joinpath("Pipfile.lock") if pipfile_lock_path.exists() and not force: raise FileExistsError( f"Cannot store Pipfile.lock because it already exists at path: {pipfile_lock_path.as_posix()!r}. " "Use --force to overwrite existing content or --show-only to visualize it." ) else: pipfile_lock_.to_file(path=pipfile_lock_path) if thoth_config or extract_all: thoth_config_string = notebook_metadata.get("thoth_config") if not thoth_config_string: raise KeyError("No .thoth.yaml identified in notebook metadata.") config = _Configuration() config.load_config_from_string(thoth_config_string) yaml_path = Path(".thoth.yaml") if yaml_path.exists() and not force: raise FileExistsError( f"Cannot store .thoth.yaml because it already exists at path: {yaml_path.as_posix()!r}. " "Use --force to overwrite existing content or --show-only to visualize it." ) else: config.save_config() return results
def horus_set_kernel_command( path: str, kernel_name: typing.Optional[str], save_in_notebook: bool = True, resolution_engine: typing.Optional[str] = None, is_magic_command: bool = False, ): """Create kernel using dependencies in notebook metadata.""" results = {} results["kernel_name"] = "" results["dependency_resolution_engine"] = "" # 0. Check if all metadata for dependencies are present in the notebook notebook = get_notebook_content(notebook_path=path) notebook_metadata = notebook.get("metadata") if notebook_metadata.get("language_info"): language = notebook_metadata["language_info"]["name"] if language and language != "python": raise Exception("Only Python kernels are currently supported.") kernelspec = notebook_metadata.get("kernelspec") notebook_kernel = kernelspec.get("name") if not kernel_name: kernel = notebook_kernel else: kernel = kernel_name if kernel == "python3": kernel = "jupyterlab-requirements" results["kernel_name"]: str = kernel home = Path.home() store_path: Path = home.joinpath(".local/share/thoth/kernels") if not resolution_engine: dependency_resolution_engine = notebook_metadata.get( "dependency_resolution_engine") if not dependency_resolution_engine: raise KeyError( "No Resolution engine identified in notebook metadata.") else: dependency_resolution_engine = resolution_engine results["dependency_resolution_engine"] = dependency_resolution_engine complete_path: Path = store_path.joinpath(kernel) if not is_magic_command: if complete_path.exists(): delete_kernel(kernel_name=kernel) complete_path.mkdir(parents=True, exist_ok=True) # 1. Get Pipfile, Pipfile.lock and .thoth.yaml and store them in ./.local/share/kernel/{kernel_name} # requirements if not is_magic_command: pipfile_string = notebook_metadata.get("requirements") pipfile_ = Pipfile.from_string(pipfile_string) pipfile_path = complete_path.joinpath("Pipfile") pipfile_.to_file(path=pipfile_path) # requirements lock if not is_magic_command: pipfile_lock_string = notebook_metadata.get("requirements_lock") pipfile_lock_ = PipfileLock.from_string( pipfile_content=pipfile_lock_string, pipfile=pipfile_) pipfile_lock_path = complete_path.joinpath("Pipfile.lock") pipfile_lock_.to_file(path=pipfile_lock_path) if dependency_resolution_engine == "thoth" and not is_magic_command: # thoth thoth_config_string = notebook_metadata.get("thoth_config") config = _Configuration() config.load_config_from_string(thoth_config_string) config_path = complete_path.joinpath(".thoth.yaml") config.save_config(path=config_path) # 2. Create virtualenv and install dependencies install_packages( kernel_name=kernel, resolution_engine=dependency_resolution_engine, is_cli=True, is_magic_command=is_magic_command, ) # 3. Install packages using micropipenv create_kernel(kernel_name=kernel) if save_in_notebook: # Update kernel name if different name selected. kernelspec["name"] = kernel notebook_metadata["kernelspec"] = kernelspec notebook["metadata"] = notebook_metadata save_notebook_content(notebook_path=path, notebook=notebook) return results