def put(self):
"""Update Thoth config file."""
initial_path = Path.cwd()
input_data = self.get_json_body()
new_runtime_environment: str = input_data["runtime_environment"]
force: bool = input_data["force"]
complete_path: str = input_data["complete_path"]
os.chdir(complete_path)
configuration = _Configuration()
if not configuration.config_file_exists():
_LOGGER.info("Thoth config does not exist, creating it...")
try:
configuration.create_default_config()
except Exception as e:
raise Exception("Thoth config file could not be created! %r",
e)
configuration.set_runtime_environment(
runtime_environment=new_runtime_environment,
force=force # TODO: force should be user choice?
)
configuration.save_config()
_LOGGER.info("Updated Thoth config: %r", configuration.content)
os.chdir(initial_path)
self.finish(
json.dumps({
"message":
f"Successfully updated thoth config at {initial_path}!"
}))
def post(self):
"""Retrieve or create Thoth config file."""
initial_path = Path.cwd()
input_data = self.get_json_body()
kernel_name: str = input_data["kernel_name"]
home = Path.home()
complete_path = home.joinpath(".local/share/thoth/kernels")
env_path = complete_path.joinpath(kernel_name)
env_path.mkdir(parents=True, exist_ok=True)
os.chdir(env_path)
_LOGGER.info(
f"kernel_name selected: {kernel_name} and path: {env_path}")
config = _Configuration()
if not config.config_file_exists():
_LOGGER.info("Thoth config does not exist, creating it...")
try:
config.create_default_config()
except Exception as e:
raise Exception("Thoth config file could not be created! %r",
e)
config.load_config()
thoth_config = config.content
_LOGGER.info("Thoth config: %r", thoth_config)
os.chdir(initial_path)
self.finish(json.dumps(thoth_config))
def get_thoth_config(
kernel_name: str,
kernels_path: Path = Path.home().joinpath(".local/share/thoth/kernels"),
) -> _Configuration:
"""Get Thoth config."""
initial_path = Path.cwd()
env_path = kernels_path.joinpath(kernel_name)
env_path.mkdir(parents=True, exist_ok=True)
os.chdir(env_path)
_LOGGER.info(f"kernel_name selected: {kernel_name} and path: {env_path}")
config = _Configuration()
if not config.config_file_exists():
_LOGGER.info("Thoth config does not exist, creating it...")
try:
config.create_default_config()
except Exception as e:
raise Exception("Thoth config file could not be created! %r", e)
config.load_config()
os.chdir(initial_path)
return config
def qeb_hwt_thamos_advise() -> None:
"""Qeb-Hwt Thamos Advise Task."""
if not Configuration._REPO_PATH:
raise Exception(
f"No path has been provided to REPO_PATH env variable.")
if not Path(Configuration._REPO_PATH).exists():
raise FileNotFoundError(
f"Cannot find the file on this path: {Configuration._REPO_PATH}")
OpenShift.verify_github_app_inputs(
github_event_type=Configuration._GITHUB_EVENT_TYPE,
github_check_run_id=Configuration._GITHUB_CHECK_RUN_ID,
github_installation_id=Configuration._GITHUB_INSTALLATION_ID,
github_base_repo_url=Configuration._GITHUB_BASE_REPO_URL,
origin=Configuration._ORIGIN,
)
os.chdir(Configuration._REPO_PATH)
thoth_yaml_config = _Configuration()
if not thoth_yaml_config.config_file_exists():
exception_message = _create_message_config_file_error(no_file=True)
trigger_finished_webhook(exception_message=exception_message,
has_error=True)
return
try:
analysis_id = advise_here(
nowait=True,
github_event_type=Configuration._GITHUB_EVENT_TYPE,
github_check_run_id=Configuration._GITHUB_CHECK_RUN_ID,
github_installation_id=Configuration._GITHUB_INSTALLATION_ID,
github_base_repo_url=Configuration._GITHUB_BASE_REPO_URL,
origin=Configuration._ORIGIN,
source_type=ThothAdviserIntegrationEnum.GITHUB_APP,
)
_LOGGER.info("Successfully submitted thamos advise call.")
except Exception as exception:
if isinstance(
exception,
(
NoRuntimeEnvironmentError,
NoRequirementsFormatError,
FileNotFoundError,
FileLoadError,
KeyError,
ValueError,
AttributeError,
),
):
_LOGGER.debug(exception)
exception_message = str(exception)
else:
_LOGGER.debug(json.loads(exception.body)["error"])
exception_message = json.loads(exception.body)["error"]
trigger_finished_webhook(exception_message=exception_message,
has_error=True)
def qeb_hwt_thamos_advise() -> None:
"""Qeb-Hwt Thamos Advise Task."""
if not Configuration._REPO_PATH:
raise Exception("No path has been provided to REPO_PATH env variable.")
if not Path(Configuration._REPO_PATH).exists():
raise FileNotFoundError(
f"Cannot find the file on this path: {Configuration._REPO_PATH}")
output_messages: list = []
OpenShift.verify_github_app_inputs(
github_event_type=Configuration._GITHUB_EVENT_TYPE,
github_check_run_id=Configuration._GITHUB_CHECK_RUN_ID,
github_installation_id=Configuration._GITHUB_INSTALLATION_ID,
github_base_repo_url=Configuration._GITHUB_BASE_REPO_URL,
origin=Configuration._ORIGIN,
)
os.chdir(Configuration._REPO_PATH)
thoth_yaml_config = _Configuration()
if not thoth_yaml_config.config_file_exists():
exception_message = _create_message_config_file_error(no_file=True)
store_messages(output_messages)
trigger_finished_webhook(exception_message=exception_message,
has_error=True,
error_type="MissingThothYamlFile")
return
try:
# Consider first runtime environment
runtime_environment = thoth_yaml_config.content.get(
"runtime_environments")[0]
# Fetch recommendation type
recommendation_type = runtime_environment.get(
"recommendation_type") if thoth_yaml_config else "latest"
requirements_format = thoth_yaml_config.requirements_format
if requirements_format == "pipenv":
project = Project.from_files(
without_pipfile_lock=not os.path.exists("Pipfile.lock"))
elif requirements_format in ("pip", "pip-tools", "pip-compile"):
project = Project.from_pip_compile_files(allow_without_lock=True)
else:
raise ValueError(
f"Unknown configuration option for requirements format: {requirements_format!r}"
)
pipfile = project.pipfile.to_string()
pipfile_lock_str = project.pipfile_lock.to_string(
) if project.pipfile_lock else ""
application_stack = PythonStack(
requirements=pipfile,
requirements_lock=pipfile_lock_str,
requirements_format=requirements_format).to_dict()
except Exception as exception:
_LOGGER.debug(json.loads(exception.body)["error"]) # type: ignore
exception_message = json.loads(exception.body)["error"] # type: ignore
store_messages(output_messages)
trigger_finished_webhook(exception_message=exception_message,
has_error=True)
return
# The input for AdviserTriggerMessage if no exceptions were found
message_input = {
"component_name": {
"type": "str",
"value": __COMPONENT_NAME__
},
"service_version": {
"type": "str",
"value": __service_version__
},
"application_stack": {
"type": "Dict",
"value": application_stack
},
"runtime_environment": {
"type": "Dict",
"value": runtime_environment
},
"recommendation_type": {
"type": "str",
"value": recommendation_type
},
"github_event_type": {
"type": "str",
"value": Configuration._GITHUB_EVENT_TYPE
},
"github_check_run_id": {
"type": "int",
"value": int(Configuration._GITHUB_CHECK_RUN_ID)
},
"github_installation_id": {
"type": "int",
"value": int(Configuration._GITHUB_INSTALLATION_ID)
},
"github_base_repo_url": {
"type": "str",
"value": Configuration._GITHUB_BASE_REPO_URL
},
"origin": {
"type": "str",
"value": Configuration._ORIGIN
},
"source_type": {
"type": "str",
"value": ThothAdviserIntegrationEnum.GITHUB_APP.name
},
}
# We store the message to put in the output file here.
output_messages = [{
"topic_name": "thoth.adviser-trigger",
"message_contents": message_input
}]
store_messages(output_messages)
def horus_lock_command(
path: str,
resolution_engine: str,
timeout: int,
force: bool,
recommendation_type: str,
kernel_name: typing.Optional[str] = None,
os_name: typing.Optional[str] = None,
os_version: typing.Optional[str] = None,
python_version: typing.Optional[str] = None,
save_in_notebook: bool = True,
save_on_disk: bool = False,
):
"""Lock requirements in notebook metadata."""
results = {}
results["kernel_name"] = ""
results["runtime_environment"] = ""
results["dependency_resolution_engine"] = resolution_engine
notebook = get_notebook_content(notebook_path=path)
notebook_metadata = notebook.get("metadata")
kernelspec = notebook_metadata.get("kernelspec")
notebook_kernel = kernelspec.get("name")
if not kernel_name:
kernel = notebook_kernel
else:
kernel = kernel_name
if kernel == "python3":
kernel = "jupyterlab-requirements"
results["kernel_name"] = kernel
requirements = notebook_metadata.get("requirements")
if not requirements:
raise KeyError(
"No Pipfile identified in notebook metadata."
"You can start creating one with command: "
"`horus requirements [NOTEBOOK].ipynb --add [PACKAGE NAME]`")
pipfile_ = Pipfile.from_string(requirements)
error = False
if resolution_engine == "thoth":
thoth_config_string = notebook_metadata.get("thoth_config")
if not thoth_config_string:
thoth_config = get_thoth_config(kernel_name=kernel)
else:
thoth_config = _Configuration()
thoth_config.load_config_from_string(thoth_config_string)
try:
notebook_content_py = get_notebook_content(notebook_path=path,
py_format=True)
except Exception as e:
_LOGGER.error(f"Could not get notebook content!: {e!r}")
notebook_content_py = ""
runtime_environment = thoth_config.get_runtime_environment()
runtime_environment["name"] = kernel
operating_system = {
"name": runtime_environment["operating_system"]["name"],
"version": runtime_environment["operating_system"]["version"],
}
if os_name:
operating_system["name"] = os_name
if os_version:
operating_system["version"] = os_version
runtime_environment["operating_system"] = operating_system
if python_version:
runtime_environment["python_version"] = python_version
runtime_environment["recommendation_type"] = recommendation_type
results["runtime_environment"] = runtime_environment
# Assign runtime environment to thoth config runtime environment.
thoth_config.set_runtime_environment(
runtime_environment=runtime_environment, force=True)
_, lock_results = lock_dependencies_with_thoth(
kernel_name=kernel,
pipfile_string=requirements,
config=json.dumps(thoth_config.content),
timeout=timeout,
force=force,
notebook_content=notebook_content_py,
)
if not lock_results["error"]:
requirements = lock_results["requirements"]
requirements_lock = lock_results["requirement_lock"]
notebook_metadata["thoth_config"] = json.dumps(thoth_config)
else:
error = True
if resolution_engine == "pipenv":
_, lock_results = lock_dependencies_with_pipenv(
kernel_name=kernel, pipfile_string=pipfile_.to_string())
if not lock_results["error"]:
requirements_lock = lock_results["requirements_lock"]
else:
error = True
if save_on_disk and not error:
home = Path.home()
store_path: Path = home.joinpath(".local/share/thoth/kernels")
complete_path: Path = store_path.joinpath(kernel)
complete_path.mkdir(parents=True, exist_ok=True)
_LOGGER.info("Path used to store dependencies is: %r",
complete_path.as_posix())
requirements_format = "pipenv"
if resolution_engine == "thoth":
project = Project.from_dict(requirements, requirements_lock)
else:
project = Project.from_dict(pipfile_.to_dict(), requirements_lock)
pipfile_path = complete_path.joinpath("Pipfile")
pipfile_lock_path = complete_path.joinpath("Pipfile.lock")
if requirements_format == "pipenv":
_LOGGER.debug("Writing to Pipfile/Pipfile.lock in %r",
complete_path)
project.to_files(pipfile_path=pipfile_path,
pipfile_lock_path=pipfile_lock_path)
if resolution_engine == "thoth":
# thoth
thoth_config_string = json.dumps(thoth_config.content)
config = _Configuration()
config.load_config_from_string(thoth_config_string)
config_path = complete_path.joinpath(".thoth.yaml")
config.save_config(path=config_path)
if save_in_notebook and not error:
notebook_metadata["dependency_resolution_engine"] = resolution_engine
notebook_metadata["requirements"] = requirements
notebook_metadata["requirements_lock"] = requirements_lock
# Assign kernel name to kernelspec.
kernelspec["name"] = kernel
notebook_metadata["kernelspec"] = kernelspec
notebook["metadata"] = notebook_metadata
save_notebook_content(notebook_path=path, notebook=notebook)
return results, lock_results
def horus_show_command(
path: str,
pipfile: bool = False,
pipfile_lock: bool = False,
thoth_config: bool = False,
):
"""Horus show command."""
show_all: bool = False
if not pipfile and not pipfile_lock and not thoth_config:
# If no parameter to be shown is set, show all is set.
show_all = True
results = {}
results["kernel_name"] = ""
results["dependency_resolution_engine"] = ""
results["pipfile"] = ""
results["pipfile_lock"] = ""
results["thoth_config"] = ""
notebook = get_notebook_content(notebook_path=path)
notebook_metadata = notebook.get("metadata")
if notebook_metadata.get("language_info"):
language = notebook_metadata["language_info"]["name"]
if language and language != "python":
raise Exception("Only Python kernels are currently supported.")
if notebook_metadata.get("kernelspec"):
kernelspec = notebook_metadata.get("kernelspec")
kernel_name = kernelspec.get("name")
else:
kernel_name = "python3"
results["kernel_name"] = kernel_name
dependency_resolution_engine = notebook_metadata.get(
"dependency_resolution_engine")
results["dependency_resolution_engine"] = dependency_resolution_engine
pipfile_string = notebook_metadata.get("requirements")
if pipfile or pipfile_lock or show_all:
if not pipfile_string:
results["pipfile"] = "No Pipfile identified in notebook metadata."
else:
pipfile_ = Pipfile.from_string(pipfile_string)
if pipfile or show_all:
results["pipfile"] = f"\nPipfile:\n\n{pipfile_.to_string()}"
if pipfile_lock or show_all:
if pipfile_string:
pipfile_lock_string = notebook_metadata.get("requirements_lock")
if not pipfile_lock_string:
results[
"pipfile_lock"] = "No Pipfile.lock identified in notebook metadata."
else:
pipfile_lock_ = PipfileLock.from_string(
pipfile_content=pipfile_lock_string, pipfile=pipfile_)
results[
"pipfile_lock"] = f"\nPipfile.lock:\n\n{pipfile_lock_.to_string()}"
else:
results[
"pipfile_lock"] = "No Pipfile identified in notebook metadata, therefore Pipfile.lock cannot be created."
if thoth_config or show_all:
thoth_config_string = notebook_metadata.get("thoth_config")
if not thoth_config_string:
results[
"thoth_config"] = "No .thoth.yaml identified in notebook metadata."
else:
config = _Configuration()
config.load_config_from_string(thoth_config_string)
results[
"thoth_config"] = f"\n.thoth.yaml:\n\n{yaml.dump(config.content)}"
return results
def horus_extract_command(
notebook_path: str,
store_files_path: str,
pipfile: bool = False,
pipfile_lock: bool = False,
thoth_config: bool = False,
use_overlay: bool = False,
force: bool = False,
):
"""Horus extract command."""
results = {}
results["kernel_name"] = ""
results["resolution_engine"] = ""
extract_all: bool = False
if not pipfile and not pipfile_lock and not thoth_config:
# If no parameter to be extracted is set, extract all is set.
extract_all = True
notebook = get_notebook_content(notebook_path=notebook_path)
notebook_metadata = notebook.get("metadata")
if notebook_metadata.get("language_info"):
language = notebook_metadata["language_info"]["name"]
if language and language != "python":
raise Exception("Only Python kernels are currently supported.")
if notebook_metadata.get("kernelspec"):
kernelspec = notebook_metadata.get("kernelspec")
kernel_name = kernelspec.get("name")
else:
kernel_name = "python3"
results["kernel_name"] = kernel_name
store_path: Path = Path(store_files_path)
if use_overlay:
if not kernel_name:
raise KeyError(
"No kernel name identified in notebook metadata kernelspec.")
store_path = store_path.joinpath("overlays").joinpath(kernel_name)
store_path.mkdir(parents=True, exist_ok=True)
dependency_resolution_engine = notebook_metadata.get(
"dependency_resolution_engine")
if not dependency_resolution_engine:
raise KeyError("No Resolution engine identified in notebook metadata.")
results["resolution_engine"] = dependency_resolution_engine
if pipfile or pipfile_lock or extract_all:
pipfile_string = notebook_metadata.get("requirements")
if not pipfile_string:
raise KeyError("No Pipfile identified in notebook metadata.")
pipfile_ = Pipfile.from_string(pipfile_string)
if pipfile or extract_all:
pipfile_path = store_path.joinpath("Pipfile")
if pipfile_path.exists() and not force:
raise FileExistsError(
f"Cannot store Pipfile because it already exists at path: {pipfile_path.as_posix()!r}. "
"Use --force to overwrite existing content or --show-only to visualize it."
)
else:
pipfile_.to_file(path=pipfile_path)
if pipfile_lock or extract_all:
pipfile_lock_string = notebook_metadata.get("requirements_lock")
if not pipfile_lock_string:
raise KeyError("No Pipfile.lock identified in notebook metadata.")
pipfile_lock_ = PipfileLock.from_string(
pipfile_content=pipfile_lock_string, pipfile=pipfile_)
pipfile_lock_path = store_path.joinpath("Pipfile.lock")
if pipfile_lock_path.exists() and not force:
raise FileExistsError(
f"Cannot store Pipfile.lock because it already exists at path: {pipfile_lock_path.as_posix()!r}. "
"Use --force to overwrite existing content or --show-only to visualize it."
)
else:
pipfile_lock_.to_file(path=pipfile_lock_path)
if thoth_config or extract_all:
thoth_config_string = notebook_metadata.get("thoth_config")
if not thoth_config_string:
raise KeyError("No .thoth.yaml identified in notebook metadata.")
config = _Configuration()
config.load_config_from_string(thoth_config_string)
yaml_path = Path(".thoth.yaml")
if yaml_path.exists() and not force:
raise FileExistsError(
f"Cannot store .thoth.yaml because it already exists at path: {yaml_path.as_posix()!r}. "
"Use --force to overwrite existing content or --show-only to visualize it."
)
else:
config.save_config()
return results
def horus_set_kernel_command(
path: str,
kernel_name: typing.Optional[str],
save_in_notebook: bool = True,
resolution_engine: typing.Optional[str] = None,
is_magic_command: bool = False,
):
"""Create kernel using dependencies in notebook metadata."""
results = {}
results["kernel_name"] = ""
results["dependency_resolution_engine"] = ""
# 0. Check if all metadata for dependencies are present in the notebook
notebook = get_notebook_content(notebook_path=path)
notebook_metadata = notebook.get("metadata")
if notebook_metadata.get("language_info"):
language = notebook_metadata["language_info"]["name"]
if language and language != "python":
raise Exception("Only Python kernels are currently supported.")
kernelspec = notebook_metadata.get("kernelspec")
notebook_kernel = kernelspec.get("name")
if not kernel_name:
kernel = notebook_kernel
else:
kernel = kernel_name
if kernel == "python3":
kernel = "jupyterlab-requirements"
results["kernel_name"]: str = kernel
home = Path.home()
store_path: Path = home.joinpath(".local/share/thoth/kernels")
if not resolution_engine:
dependency_resolution_engine = notebook_metadata.get(
"dependency_resolution_engine")
if not dependency_resolution_engine:
raise KeyError(
"No Resolution engine identified in notebook metadata.")
else:
dependency_resolution_engine = resolution_engine
results["dependency_resolution_engine"] = dependency_resolution_engine
complete_path: Path = store_path.joinpath(kernel)
if not is_magic_command:
if complete_path.exists():
delete_kernel(kernel_name=kernel)
complete_path.mkdir(parents=True, exist_ok=True)
# 1. Get Pipfile, Pipfile.lock and .thoth.yaml and store them in ./.local/share/kernel/{kernel_name}
# requirements
if not is_magic_command:
pipfile_string = notebook_metadata.get("requirements")
pipfile_ = Pipfile.from_string(pipfile_string)
pipfile_path = complete_path.joinpath("Pipfile")
pipfile_.to_file(path=pipfile_path)
# requirements lock
if not is_magic_command:
pipfile_lock_string = notebook_metadata.get("requirements_lock")
pipfile_lock_ = PipfileLock.from_string(
pipfile_content=pipfile_lock_string, pipfile=pipfile_)
pipfile_lock_path = complete_path.joinpath("Pipfile.lock")
pipfile_lock_.to_file(path=pipfile_lock_path)
if dependency_resolution_engine == "thoth" and not is_magic_command:
# thoth
thoth_config_string = notebook_metadata.get("thoth_config")
config = _Configuration()
config.load_config_from_string(thoth_config_string)
config_path = complete_path.joinpath(".thoth.yaml")
config.save_config(path=config_path)
# 2. Create virtualenv and install dependencies
install_packages(
kernel_name=kernel,
resolution_engine=dependency_resolution_engine,
is_cli=True,
is_magic_command=is_magic_command,
)
# 3. Install packages using micropipenv
create_kernel(kernel_name=kernel)
if save_in_notebook:
# Update kernel name if different name selected.
kernelspec["name"] = kernel
notebook_metadata["kernelspec"] = kernelspec
notebook["metadata"] = notebook_metadata
save_notebook_content(notebook_path=path, notebook=notebook)
return results
