def _write_files(requirements: Dict[str, Any], requirements_lock: Dict[str, Any], requirements_format: str) -> None: """Write content of Pipfile/Pipfile.lock or requirements.in/txt to the current directory.""" project = Project.from_dict(requirements, requirements_lock) if requirements_format == "pipenv": _LOGGER.debug("Writing to Pipfile/Pipfile.lock in %r", os.getcwd()) project.to_files() elif requirements_format in ("pip", "pip-tools", "pip-compile"): _LOGGER.debug("Writing to requirements.in/requirements.txt in %r", os.getcwd()) project.to_pip_compile_files() _LOGGER.debug("No changes to Pipfile to write") else: raise ValueError( f"Unknown requirements format, supported are 'pipenv' and 'pip': {requirements_format!r}" )
async def post(self): """Lock dependencies using Thoth service.""" initial_path = Path.cwd() input_data = self.get_json_body() config: str = input_data["thoth_config"] kernel_name: str = input_data["kernel_name"] timeout: str = input_data["thoth_timeout"] notebook_content: str = input_data["notebook_content"] requirements: dict = json.loads(input_data["requirements"]) # Get origin before changing path origin: str = _get_origin() _LOGGER.info("Origin identified by thamos: %r", origin) home = Path.home() complete_path = home.joinpath(".local/share/thoth/kernels") env_path = complete_path.joinpath(kernel_name) # Delete and recreate folder if not env_path.exists(): _ = subprocess.call(f"rm -rf ./{kernel_name} ", shell=True, cwd=complete_path) env_path.mkdir(parents=True, exist_ok=True) os.chdir(env_path) _LOGGER.info("Resolution engine used: thoth") pipfile_string = Pipfile.from_dict(requirements).to_string() _LOGGER.info(f"Current path: %r ", env_path.as_posix()) _LOGGER.info(f"Input Pipfile: \n{pipfile_string}") advise = {"requirements": {}, "requirement_lock": {}, "error": False} temp = tempfile.NamedTemporaryFile(prefix="jl_thoth_", mode='w+t') try: adviser_inputs = { "pipfile": pipfile_string, "config": config, "origin": origin } _LOGGER.info("Adviser inputs are: %r", adviser_inputs) temp.write(notebook_content) _LOGGER.info("path to temporary file is: %r", temp.name) response = advise_using_config( pipfile=pipfile_string, pipfile_lock="", # TODO: Provide Pipfile.lock retrieved? force=False, # TODO: Provide force input from user? config=config, origin=origin, nowait=False, source_type=ThothAdviserIntegrationEnum.JUPYTER_NOTEBOOK, no_static_analysis=False, timeout=timeout, src_path=temp.name) _LOGGER.info(f"Response: {response}") if not response: raise Exception("Analysis was not successful.") result, error_result = response if error_result: advise['error'] = True else: # Use report of the best one, therefore index 0 if result["report"] and result["report"]["products"]: justifications = result["report"]["products"][0][ "justification"] _LOGGER.info(f"Justification: {justifications}") stack_info = result["report"]["stack_info"] _LOGGER.debug(f"Stack info {stack_info}") pipfile = result["report"]["products"][0]["project"][ "requirements"] pipfile_lock = result["report"]["products"][0]["project"][ "requirements_locked"] advise = { "requirements": pipfile, "requirement_lock": pipfile_lock, "error": False } except Exception as api_error: _LOGGER.warning( f"error locking dependencies using Thoth: {api_error}") advise['error'] = True finally: temp.close() _LOGGER.info(f"advise received: {advise}") if not advise['error']: try: requirements_format = "pipenv" project = Project.from_dict(pipfile, pipfile_lock) pipfile_path = env_path.joinpath("Pipfile") pipfile_lock_path = env_path.joinpath("Pipfile.lock") if requirements_format == "pipenv": _LOGGER.info("Writing to Pipfile/Pipfile.lock in %r", env_path.as_posix()) project.to_files(pipfile_path=pipfile_path, pipfile_lock_path=pipfile_lock_path) except Exception as e: _LOGGER.warning( "Requirements files have not been stored successfully %r", e) os.chdir(initial_path) self.finish(json.dumps(advise))
def horus_lock_command( path: str, resolution_engine: str, timeout: int, force: bool, recommendation_type: str, kernel_name: typing.Optional[str] = None, os_name: typing.Optional[str] = None, os_version: typing.Optional[str] = None, python_version: typing.Optional[str] = None, save_in_notebook: bool = True, save_on_disk: bool = False, ): """Lock requirements in notebook metadata.""" results = {} results["kernel_name"] = "" results["runtime_environment"] = "" results["dependency_resolution_engine"] = resolution_engine notebook = get_notebook_content(notebook_path=path) notebook_metadata = notebook.get("metadata") kernelspec = notebook_metadata.get("kernelspec") notebook_kernel = kernelspec.get("name") if not kernel_name: kernel = notebook_kernel else: kernel = kernel_name if kernel == "python3": kernel = "jupyterlab-requirements" results["kernel_name"] = kernel requirements = notebook_metadata.get("requirements") if not requirements: raise KeyError( "No Pipfile identified in notebook metadata." "You can start creating one with command: " "`horus requirements [NOTEBOOK].ipynb --add [PACKAGE NAME]`") pipfile_ = Pipfile.from_string(requirements) error = False if resolution_engine == "thoth": thoth_config_string = notebook_metadata.get("thoth_config") if not thoth_config_string: thoth_config = get_thoth_config(kernel_name=kernel) else: thoth_config = _Configuration() thoth_config.load_config_from_string(thoth_config_string) try: notebook_content_py = get_notebook_content(notebook_path=path, py_format=True) except Exception as e: _LOGGER.error(f"Could not get notebook content!: {e!r}") notebook_content_py = "" runtime_environment = thoth_config.get_runtime_environment() runtime_environment["name"] = kernel operating_system = { "name": runtime_environment["operating_system"]["name"], "version": runtime_environment["operating_system"]["version"], } if os_name: operating_system["name"] = os_name if os_version: operating_system["version"] = os_version runtime_environment["operating_system"] = operating_system if python_version: runtime_environment["python_version"] = python_version runtime_environment["recommendation_type"] = recommendation_type results["runtime_environment"] = runtime_environment # Assign runtime environment to thoth config runtime environment. thoth_config.set_runtime_environment( runtime_environment=runtime_environment, force=True) _, lock_results = lock_dependencies_with_thoth( kernel_name=kernel, pipfile_string=requirements, config=json.dumps(thoth_config.content), timeout=timeout, force=force, notebook_content=notebook_content_py, ) if not lock_results["error"]: requirements = lock_results["requirements"] requirements_lock = lock_results["requirement_lock"] notebook_metadata["thoth_config"] = json.dumps(thoth_config) else: error = True if resolution_engine == "pipenv": _, lock_results = lock_dependencies_with_pipenv( kernel_name=kernel, pipfile_string=pipfile_.to_string()) if not lock_results["error"]: requirements_lock = lock_results["requirements_lock"] else: error = True if save_on_disk and not error: home = Path.home() store_path: Path = home.joinpath(".local/share/thoth/kernels") complete_path: Path = store_path.joinpath(kernel) complete_path.mkdir(parents=True, exist_ok=True) _LOGGER.info("Path used to store dependencies is: %r", complete_path.as_posix()) requirements_format = "pipenv" if resolution_engine == "thoth": project = Project.from_dict(requirements, requirements_lock) else: project = Project.from_dict(pipfile_.to_dict(), requirements_lock) pipfile_path = complete_path.joinpath("Pipfile") pipfile_lock_path = complete_path.joinpath("Pipfile.lock") if requirements_format == "pipenv": _LOGGER.debug("Writing to Pipfile/Pipfile.lock in %r", complete_path) project.to_files(pipfile_path=pipfile_path, pipfile_lock_path=pipfile_lock_path) if resolution_engine == "thoth": # thoth thoth_config_string = json.dumps(thoth_config.content) config = _Configuration() config.load_config_from_string(thoth_config_string) config_path = complete_path.joinpath(".thoth.yaml") config.save_config(path=config_path) if save_in_notebook and not error: notebook_metadata["dependency_resolution_engine"] = resolution_engine notebook_metadata["requirements"] = requirements notebook_metadata["requirements_lock"] = requirements_lock # Assign kernel name to kernelspec. kernelspec["name"] = kernel notebook_metadata["kernelspec"] = kernelspec notebook["metadata"] = notebook_metadata save_notebook_content(notebook_path=path, notebook=notebook) return results, lock_results
def lock_dependencies_with_thoth( kernel_name: str, pipfile_string: str, config: str, timeout: int, force: bool, notebook_content: str, kernels_path: Path = Path.home().joinpath(".local/share/thoth/kernels"), ) -> typing.Tuple[int, dict]: """Lock dependencies using Thoth resolution engine.""" initial_path = Path.cwd() # Get origin before changing path origin: str = _get_origin() _LOGGER.info("Origin identified by thamos: %r", origin) env_path = kernels_path.joinpath(kernel_name) env_path.mkdir(parents=True, exist_ok=True) os.chdir(env_path) _LOGGER.info("Resolution engine used: thoth") _LOGGER.info("Current path: %r ", env_path.as_posix()) _LOGGER.info(f"Input Pipfile: \n{pipfile_string}") advise = { "requirements": {}, "requirement_lock": {}, "error": False, "error_msg": "" } returncode = 0 temp = tempfile.NamedTemporaryFile(prefix="jl_thoth_", mode="w+t") try: adviser_inputs = { "pipfile": pipfile_string, "config": config, "origin": origin } _LOGGER.info("Adviser inputs are: %r", adviser_inputs) temp.write(notebook_content) _LOGGER.info("path to temporary file is: %r", temp.name) response = advise_using_config( pipfile=pipfile_string, pipfile_lock="", # TODO: Provide Pipfile.lock retrieved? force=force, config=config, origin=origin, nowait=False, source_type=ThothAdviserIntegrationEnum.JUPYTER_NOTEBOOK, no_static_analysis=False, timeout=timeout, src_path=temp.name, ) _LOGGER.info(f"Response: {response}") if not response: raise Exception("Analysis was not successful.") result, error_result = response if error_result: advise["error"] = True advise["error_msg"] = result.get("error_msg") returncode = 1 else: # Use report of the best one, therefore index 0 if result["report"] and result["report"]["products"]: justifications = result["report"]["products"][0][ "justification"] _LOGGER.info(f"Justification: {justifications}") stack_info = result["report"]["stack_info"] _LOGGER.debug(f"Stack info {stack_info}") pipfile = result["report"]["products"][0]["project"][ "requirements"] pipfile_lock = result["report"]["products"][0]["project"][ "requirements_locked"] advise = { "requirements": pipfile, "requirement_lock": pipfile_lock, "error": False } except Exception as api_error: _LOGGER.warning(f"error locking dependencies using Thoth: {api_error}") advise["error"] = True if not advise.get("error_msg"): advise[ "error_msg"] = f"Error locking dependencies, check pod logs for more details about the error. {api_error}" returncode = 1 finally: temp.close() _LOGGER.info(f"advise received: {advise}") if not advise["error"]: try: requirements_format = "pipenv" project = Project.from_dict(pipfile, pipfile_lock) pipfile_path = env_path.joinpath("Pipfile") pipfile_lock_path = env_path.joinpath("Pipfile.lock") if requirements_format == "pipenv": _LOGGER.info("Writing to Pipfile/Pipfile.lock in %r", env_path.as_posix()) project.to_files(pipfile_path=pipfile_path, pipfile_lock_path=pipfile_lock_path) except Exception as e: _LOGGER.warning( "Requirements files have not been stored successfully %r", e) os.chdir(initial_path) return returncode, advise