def _write_files(requirements: Dict[str, Any], requirements_lock: Dict[str,
Any],
requirements_format: str) -> None:
"""Write content of Pipfile/Pipfile.lock or requirements.in/txt to the current directory."""
project = Project.from_dict(requirements, requirements_lock)
if requirements_format == "pipenv":
_LOGGER.debug("Writing to Pipfile/Pipfile.lock in %r", os.getcwd())
project.to_files()
elif requirements_format in ("pip", "pip-tools", "pip-compile"):
_LOGGER.debug("Writing to requirements.in/requirements.txt in %r",
os.getcwd())
project.to_pip_compile_files()
_LOGGER.debug("No changes to Pipfile to write")
else:
raise ValueError(
f"Unknown requirements format, supported are 'pipenv' and 'pip': {requirements_format!r}"
)
async def post(self):
"""Lock dependencies using Thoth service."""
initial_path = Path.cwd()
input_data = self.get_json_body()
config: str = input_data["thoth_config"]
kernel_name: str = input_data["kernel_name"]
timeout: str = input_data["thoth_timeout"]
notebook_content: str = input_data["notebook_content"]
requirements: dict = json.loads(input_data["requirements"])
# Get origin before changing path
origin: str = _get_origin()
_LOGGER.info("Origin identified by thamos: %r", origin)
home = Path.home()
complete_path = home.joinpath(".local/share/thoth/kernels")
env_path = complete_path.joinpath(kernel_name)
# Delete and recreate folder
if not env_path.exists():
_ = subprocess.call(f"rm -rf ./{kernel_name} ",
shell=True,
cwd=complete_path)
env_path.mkdir(parents=True, exist_ok=True)
os.chdir(env_path)
_LOGGER.info("Resolution engine used: thoth")
pipfile_string = Pipfile.from_dict(requirements).to_string()
_LOGGER.info(f"Current path: %r ", env_path.as_posix())
_LOGGER.info(f"Input Pipfile: \n{pipfile_string}")
advise = {"requirements": {}, "requirement_lock": {}, "error": False}
temp = tempfile.NamedTemporaryFile(prefix="jl_thoth_", mode='w+t')
try:
adviser_inputs = {
"pipfile": pipfile_string,
"config": config,
"origin": origin
}
_LOGGER.info("Adviser inputs are: %r", adviser_inputs)
temp.write(notebook_content)
_LOGGER.info("path to temporary file is: %r", temp.name)
response = advise_using_config(
pipfile=pipfile_string,
pipfile_lock="", # TODO: Provide Pipfile.lock retrieved?
force=False, # TODO: Provide force input from user?
config=config,
origin=origin,
nowait=False,
source_type=ThothAdviserIntegrationEnum.JUPYTER_NOTEBOOK,
no_static_analysis=False,
timeout=timeout,
src_path=temp.name)
_LOGGER.info(f"Response: {response}")
if not response:
raise Exception("Analysis was not successful.")
result, error_result = response
if error_result:
advise['error'] = True
else:
# Use report of the best one, therefore index 0
if result["report"] and result["report"]["products"]:
justifications = result["report"]["products"][0][
"justification"]
_LOGGER.info(f"Justification: {justifications}")
stack_info = result["report"]["stack_info"]
_LOGGER.debug(f"Stack info {stack_info}")
pipfile = result["report"]["products"][0]["project"][
"requirements"]
pipfile_lock = result["report"]["products"][0]["project"][
"requirements_locked"]
advise = {
"requirements": pipfile,
"requirement_lock": pipfile_lock,
"error": False
}
except Exception as api_error:
_LOGGER.warning(
f"error locking dependencies using Thoth: {api_error}")
advise['error'] = True
finally:
temp.close()
_LOGGER.info(f"advise received: {advise}")
if not advise['error']:
try:
requirements_format = "pipenv"
project = Project.from_dict(pipfile, pipfile_lock)
pipfile_path = env_path.joinpath("Pipfile")
pipfile_lock_path = env_path.joinpath("Pipfile.lock")
if requirements_format == "pipenv":
_LOGGER.info("Writing to Pipfile/Pipfile.lock in %r",
env_path.as_posix())
project.to_files(pipfile_path=pipfile_path,
pipfile_lock_path=pipfile_lock_path)
except Exception as e:
_LOGGER.warning(
"Requirements files have not been stored successfully %r",
e)
os.chdir(initial_path)
self.finish(json.dumps(advise))
def horus_lock_command(
path: str,
resolution_engine: str,
timeout: int,
force: bool,
recommendation_type: str,
kernel_name: typing.Optional[str] = None,
os_name: typing.Optional[str] = None,
os_version: typing.Optional[str] = None,
python_version: typing.Optional[str] = None,
save_in_notebook: bool = True,
save_on_disk: bool = False,
):
"""Lock requirements in notebook metadata."""
results = {}
results["kernel_name"] = ""
results["runtime_environment"] = ""
results["dependency_resolution_engine"] = resolution_engine
notebook = get_notebook_content(notebook_path=path)
notebook_metadata = notebook.get("metadata")
kernelspec = notebook_metadata.get("kernelspec")
notebook_kernel = kernelspec.get("name")
if not kernel_name:
kernel = notebook_kernel
else:
kernel = kernel_name
if kernel == "python3":
kernel = "jupyterlab-requirements"
results["kernel_name"] = kernel
requirements = notebook_metadata.get("requirements")
if not requirements:
raise KeyError(
"No Pipfile identified in notebook metadata."
"You can start creating one with command: "
"`horus requirements [NOTEBOOK].ipynb --add [PACKAGE NAME]`")
pipfile_ = Pipfile.from_string(requirements)
error = False
if resolution_engine == "thoth":
thoth_config_string = notebook_metadata.get("thoth_config")
if not thoth_config_string:
thoth_config = get_thoth_config(kernel_name=kernel)
else:
thoth_config = _Configuration()
thoth_config.load_config_from_string(thoth_config_string)
try:
notebook_content_py = get_notebook_content(notebook_path=path,
py_format=True)
except Exception as e:
_LOGGER.error(f"Could not get notebook content!: {e!r}")
notebook_content_py = ""
runtime_environment = thoth_config.get_runtime_environment()
runtime_environment["name"] = kernel
operating_system = {
"name": runtime_environment["operating_system"]["name"],
"version": runtime_environment["operating_system"]["version"],
}
if os_name:
operating_system["name"] = os_name
if os_version:
operating_system["version"] = os_version
runtime_environment["operating_system"] = operating_system
if python_version:
runtime_environment["python_version"] = python_version
runtime_environment["recommendation_type"] = recommendation_type
results["runtime_environment"] = runtime_environment
# Assign runtime environment to thoth config runtime environment.
thoth_config.set_runtime_environment(
runtime_environment=runtime_environment, force=True)
_, lock_results = lock_dependencies_with_thoth(
kernel_name=kernel,
pipfile_string=requirements,
config=json.dumps(thoth_config.content),
timeout=timeout,
force=force,
notebook_content=notebook_content_py,
)
if not lock_results["error"]:
requirements = lock_results["requirements"]
requirements_lock = lock_results["requirement_lock"]
notebook_metadata["thoth_config"] = json.dumps(thoth_config)
else:
error = True
if resolution_engine == "pipenv":
_, lock_results = lock_dependencies_with_pipenv(
kernel_name=kernel, pipfile_string=pipfile_.to_string())
if not lock_results["error"]:
requirements_lock = lock_results["requirements_lock"]
else:
error = True
if save_on_disk and not error:
home = Path.home()
store_path: Path = home.joinpath(".local/share/thoth/kernels")
complete_path: Path = store_path.joinpath(kernel)
complete_path.mkdir(parents=True, exist_ok=True)
_LOGGER.info("Path used to store dependencies is: %r",
complete_path.as_posix())
requirements_format = "pipenv"
if resolution_engine == "thoth":
project = Project.from_dict(requirements, requirements_lock)
else:
project = Project.from_dict(pipfile_.to_dict(), requirements_lock)
pipfile_path = complete_path.joinpath("Pipfile")
pipfile_lock_path = complete_path.joinpath("Pipfile.lock")
if requirements_format == "pipenv":
_LOGGER.debug("Writing to Pipfile/Pipfile.lock in %r",
complete_path)
project.to_files(pipfile_path=pipfile_path,
pipfile_lock_path=pipfile_lock_path)
if resolution_engine == "thoth":
# thoth
thoth_config_string = json.dumps(thoth_config.content)
config = _Configuration()
config.load_config_from_string(thoth_config_string)
config_path = complete_path.joinpath(".thoth.yaml")
config.save_config(path=config_path)
if save_in_notebook and not error:
notebook_metadata["dependency_resolution_engine"] = resolution_engine
notebook_metadata["requirements"] = requirements
notebook_metadata["requirements_lock"] = requirements_lock
# Assign kernel name to kernelspec.
kernelspec["name"] = kernel
notebook_metadata["kernelspec"] = kernelspec
notebook["metadata"] = notebook_metadata
save_notebook_content(notebook_path=path, notebook=notebook)
return results, lock_results
def lock_dependencies_with_thoth(
kernel_name: str,
pipfile_string: str,
config: str,
timeout: int,
force: bool,
notebook_content: str,
kernels_path: Path = Path.home().joinpath(".local/share/thoth/kernels"),
) -> typing.Tuple[int, dict]:
"""Lock dependencies using Thoth resolution engine."""
initial_path = Path.cwd()
# Get origin before changing path
origin: str = _get_origin()
_LOGGER.info("Origin identified by thamos: %r", origin)
env_path = kernels_path.joinpath(kernel_name)
env_path.mkdir(parents=True, exist_ok=True)
os.chdir(env_path)
_LOGGER.info("Resolution engine used: thoth")
_LOGGER.info("Current path: %r ", env_path.as_posix())
_LOGGER.info(f"Input Pipfile: \n{pipfile_string}")
advise = {
"requirements": {},
"requirement_lock": {},
"error": False,
"error_msg": ""
}
returncode = 0
temp = tempfile.NamedTemporaryFile(prefix="jl_thoth_", mode="w+t")
try:
adviser_inputs = {
"pipfile": pipfile_string,
"config": config,
"origin": origin
}
_LOGGER.info("Adviser inputs are: %r", adviser_inputs)
temp.write(notebook_content)
_LOGGER.info("path to temporary file is: %r", temp.name)
response = advise_using_config(
pipfile=pipfile_string,
pipfile_lock="", # TODO: Provide Pipfile.lock retrieved?
force=force,
config=config,
origin=origin,
nowait=False,
source_type=ThothAdviserIntegrationEnum.JUPYTER_NOTEBOOK,
no_static_analysis=False,
timeout=timeout,
src_path=temp.name,
)
_LOGGER.info(f"Response: {response}")
if not response:
raise Exception("Analysis was not successful.")
result, error_result = response
if error_result:
advise["error"] = True
advise["error_msg"] = result.get("error_msg")
returncode = 1
else:
# Use report of the best one, therefore index 0
if result["report"] and result["report"]["products"]:
justifications = result["report"]["products"][0][
"justification"]
_LOGGER.info(f"Justification: {justifications}")
stack_info = result["report"]["stack_info"]
_LOGGER.debug(f"Stack info {stack_info}")
pipfile = result["report"]["products"][0]["project"][
"requirements"]
pipfile_lock = result["report"]["products"][0]["project"][
"requirements_locked"]
advise = {
"requirements": pipfile,
"requirement_lock": pipfile_lock,
"error": False
}
except Exception as api_error:
_LOGGER.warning(f"error locking dependencies using Thoth: {api_error}")
advise["error"] = True
if not advise.get("error_msg"):
advise[
"error_msg"] = f"Error locking dependencies, check pod logs for more details about the error. {api_error}"
returncode = 1
finally:
temp.close()
_LOGGER.info(f"advise received: {advise}")
if not advise["error"]:
try:
requirements_format = "pipenv"
project = Project.from_dict(pipfile, pipfile_lock)
pipfile_path = env_path.joinpath("Pipfile")
pipfile_lock_path = env_path.joinpath("Pipfile.lock")
if requirements_format == "pipenv":
_LOGGER.info("Writing to Pipfile/Pipfile.lock in %r",
env_path.as_posix())
project.to_files(pipfile_path=pipfile_path,
pipfile_lock_path=pipfile_lock_path)
except Exception as e:
_LOGGER.warning(
"Requirements files have not been stored successfully %r", e)
os.chdir(initial_path)
return returncode, advise
