def _instantiate_project(
requirements: str,
requirements_locked: Optional[str] = None,
runtime_environment: RuntimeEnvironment = None,
):
"""Create Project instance based on arguments passed to CLI."""
try:
with open(requirements, "r") as requirements_file:
requirements = requirements_file.read()
except (OSError, FileNotFoundError):
# We we gather values from env vars, un-escape new lines.
requirements = requirements.replace("\\n", "\n")
if requirements_locked:
try:
with open(requirements_locked, "r") as requirements_file:
requirements_locked = requirements_file.read()
except (OSError, FileNotFoundError):
# We we gather values from env vars, un-escape new lines.
requirements_locked = requirements_locked.replace("\\n", "\n")
pipfile = Pipfile.from_string(requirements)
pipfile_lock = PipfileLock.from_string(
requirements_locked, pipfile) if requirements_locked else None
project = Project(
pipfile=pipfile,
pipfile_lock=pipfile_lock,
runtime_environment=runtime_environment
or RuntimeEnvironment.from_dict({}),
)
return project
def _instantiate_project(
requirements: str,
requirements_locked: typing.Optional[str],
files: bool,
runtime_environment: RuntimeEnvironment = None,
):
"""Create Project instance based on arguments passed to CLI."""
if files:
with open(requirements, "r") as requirements_file:
requirements = requirements_file.read()
if requirements_locked:
with open(requirements_locked, "r") as requirements_file:
requirements_locked = requirements_file.read()
del requirements_file
else:
# We we gather values from env vars, un-escape new lines.
requirements = requirements.replace("\\n", "\n")
if requirements_locked:
requirements_locked = requirements_locked.replace("\\n", "\n")
pipfile = Pipfile.from_string(requirements)
pipfile_lock = (
PipfileLock.from_string(requirements_locked, pipfile)
if requirements_locked
else None
)
project = Project(
pipfile=pipfile,
pipfile_lock=pipfile_lock,
runtime_environment=runtime_environment or RuntimeEnvironment.from_dict({}),
)
return project
def _instantiate_project(
requirements: str,
requirements_locked: Optional[str] = None,
*,
runtime_environment: RuntimeEnvironment = None,
constraints: Optional[str] = None,
):
"""Create Project instance based on arguments passed to CLI."""
try:
with open(requirements, "r") as requirements_file:
requirements = requirements_file.read()
except (OSError, FileNotFoundError):
# We we gather values from env vars, un-escape new lines.
requirements = requirements.replace("\\n", "\n")
if requirements_locked:
try:
with open(requirements_locked, "r") as requirements_file:
requirements_locked = requirements_file.read()
except (OSError, FileNotFoundError):
# We we gather values from env vars, un-escape new lines.
requirements_locked = requirements_locked.replace("\\n", "\n")
pipfile = Pipfile.from_string(requirements)
pipfile_lock = None
if requirements_locked and requirements_locked != "null":
pipfile_lock = PipfileLock.from_string(requirements_locked, pipfile)
constraints_instance = None
if constraints:
try:
with open(constraints, "r") as constraints_file:
constraints_content = constraints_file.read()
except (OSError, FileNotFoundError):
# We we gather values from env vars, un-escape new lines.
constraints_content = constraints.replace("\\n", "\n")
try:
constraints_instance = Constraints.from_dict(
json.loads(constraints_content))
except json.decoder.JSONDecodeError:
constraints_instance = Constraints.from_string(constraints_content)
runtime_environment = runtime_environment or RuntimeEnvironment.from_dict(
{})
if not runtime_environment.platform:
runtime_environment.platform = _DEFAULT_PLATFORM
project = Project(
pipfile=pipfile,
pipfile_lock=pipfile_lock,
runtime_environment=runtime_environment,
constraints=constraints_instance or Constraints(),
)
return project
def test_from_string(self, pipfile_lock: str):
"""Test from string."""
with open(os.path.join(self.data_dir, "pipfiles", pipfile_lock), "r") as pipfile_lock_file:
content = pipfile_lock_file.read()
with open(os.path.join(self.data_dir, "pipfiles", pipfile_lock[: -len(".lock")]), "r") as pipfile_file:
pipfile_content = pipfile_file.read()
pipfile_instance = Pipfile.from_string(pipfile_content)
instance = PipfileLock.from_string(content, pipfile=pipfile_instance)
assert instance.to_string() == content
def test_from_string(self, pipfile_lock: str):
with open(os.path.join(self.data_dir, 'pipfiles', pipfile_lock),
'r') as pipfile_lock_file:
content = pipfile_lock_file.read()
with open(
os.path.join(self.data_dir, 'pipfiles',
pipfile_lock[:-len('.lock')]),
'r') as pipfile_file:
pipfile_content = pipfile_file.read()
pipfile_instance = Pipfile.from_string(pipfile_content)
instance = PipfileLock.from_string(content, pipfile=pipfile_instance)
assert instance.to_string() == content
def horus_show_command(
path: str,
pipfile: bool = False,
pipfile_lock: bool = False,
thoth_config: bool = False,
):
"""Horus show command."""
show_all: bool = False
if not pipfile and not pipfile_lock and not thoth_config:
# If no parameter to be shown is set, show all is set.
show_all = True
results = {}
results["kernel_name"] = ""
results["dependency_resolution_engine"] = ""
results["pipfile"] = ""
results["pipfile_lock"] = ""
results["thoth_config"] = ""
notebook = get_notebook_content(notebook_path=path)
notebook_metadata = notebook.get("metadata")
if notebook_metadata.get("language_info"):
language = notebook_metadata["language_info"]["name"]
if language and language != "python":
raise Exception("Only Python kernels are currently supported.")
if notebook_metadata.get("kernelspec"):
kernelspec = notebook_metadata.get("kernelspec")
kernel_name = kernelspec.get("name")
else:
kernel_name = "python3"
results["kernel_name"] = kernel_name
dependency_resolution_engine = notebook_metadata.get(
"dependency_resolution_engine")
results["dependency_resolution_engine"] = dependency_resolution_engine
pipfile_string = notebook_metadata.get("requirements")
if pipfile or pipfile_lock or show_all:
if not pipfile_string:
results["pipfile"] = "No Pipfile identified in notebook metadata."
else:
pipfile_ = Pipfile.from_string(pipfile_string)
if pipfile or show_all:
results["pipfile"] = f"\nPipfile:\n\n{pipfile_.to_string()}"
if pipfile_lock or show_all:
if pipfile_string:
pipfile_lock_string = notebook_metadata.get("requirements_lock")
if not pipfile_lock_string:
results[
"pipfile_lock"] = "No Pipfile.lock identified in notebook metadata."
else:
pipfile_lock_ = PipfileLock.from_string(
pipfile_content=pipfile_lock_string, pipfile=pipfile_)
results[
"pipfile_lock"] = f"\nPipfile.lock:\n\n{pipfile_lock_.to_string()}"
else:
results[
"pipfile_lock"] = "No Pipfile identified in notebook metadata, therefore Pipfile.lock cannot be created."
if thoth_config or show_all:
thoth_config_string = notebook_metadata.get("thoth_config")
if not thoth_config_string:
results[
"thoth_config"] = "No .thoth.yaml identified in notebook metadata."
else:
config = _Configuration()
config.load_config_from_string(thoth_config_string)
results[
"thoth_config"] = f"\n.thoth.yaml:\n\n{yaml.dump(config.content)}"
return results
def lock_dependencies_with_pipenv(
kernel_name: str,
pipfile_string: str,
kernels_path: Path = Path.home().joinpath(".local/share/thoth/kernels"),
) -> typing.Tuple[int, dict]:
"""Lock dependencies using Pipenv resolution engine."""
initial_path = Path.cwd()
env_path = kernels_path.joinpath(kernel_name)
# Delete and recreate folder
if not env_path.exists():
_ = subprocess.call(f"rm -rf ./{kernel_name} ",
shell=True,
cwd=kernels_path)
env_path.mkdir(parents=True, exist_ok=True)
result = {"requirements_lock": "", "error": False, "error_msg": ""}
returncode = 0
## Create virtualenv
cli_run([str(env_path)])
pipfile_path = env_path.joinpath("Pipfile")
_LOGGER.info("Resolution engine used: pipenv")
with open(pipfile_path, "w") as pipfile_file:
pipfile_file.write(pipfile_string)
_LOGGER.info(f"kernel path: {env_path}")
_LOGGER.info(f"Input Pipfile: \n{pipfile_string}")
# 2. Install pipenv if not installed already
package = "pipenv"
check_install = subprocess.run(
f"python3 -c \"import sys, pkgutil; sys.exit(0 if pkgutil.find_loader('{package}') else 1)\"",
shell=True,
cwd=kernels_path,
capture_output=True,
)
if check_install.returncode != 0:
_LOGGER.debug(
f"pipenv is not installed in the host!: {check_install.stderr!r}")
try:
subprocess.run("pip install pipenv", cwd=kernels_path, shell=True)
except Exception as pipenv_install_error:
_LOGGER.warning("error installing pipenv: %r",
pipenv_install_error)
result["error"] = True
result["error_msg"] = pipenv_install_error
returncode = 1
os.chdir(initial_path)
return returncode, result
else:
_LOGGER.debug("pipenv is already present on the host!")
pipfile_lock_path = env_path.joinpath("Pipfile.lock")
try:
output = subprocess.run(
f". {kernel_name}/bin/activate && cd {kernel_name} && pipenv lock",
env=dict(os.environ, PIPENV_CACHE_DIR="/tmp"),
cwd=kernels_path,
shell=True,
capture_output=True,
)
except Exception as pipenv_error:
_LOGGER.warning("error locking dependencies using Pipenv: %r",
pipenv_error)
result["error"] = True
result["error_msg"] = str(pipenv_error)
returncode = 1
if output.returncode != 0:
_LOGGER.warning(
"error in process trying to lock dependencies with pipenv: %r",
output.stderr)
result["error"] = True
result["error_msg"] = str(output.stderr)
returncode = 1
os.chdir(env_path)
if not result["error"]:
if pipfile_lock_path.exists():
with open(pipfile_lock_path, "r") as pipfile_lock_file:
pipfile_lock_str = pipfile_lock_file.read()
pipfile = Pipfile.from_string(pipfile_string)
pipfile_lock_: PipfileLock = PipfileLock.from_string(
pipfile_lock_str, pipfile=pipfile)
result["requirements_lock"] = pipfile_lock_.to_dict()
_LOGGER.debug(f"result from pipenv received: {result}")
else:
_LOGGER.warning("Pipfile.lock cannot be found at: %r",
str(pipfile_lock_path))
result["error"] = True
result[
"error_msg"] = "Error retrieving Pipfile.lock created from pipenv."
os.chdir(initial_path)
return returncode, result
def horus_extract_command(
notebook_path: str,
store_files_path: str,
pipfile: bool = False,
pipfile_lock: bool = False,
thoth_config: bool = False,
use_overlay: bool = False,
force: bool = False,
):
"""Horus extract command."""
results = {}
results["kernel_name"] = ""
results["resolution_engine"] = ""
extract_all: bool = False
if not pipfile and not pipfile_lock and not thoth_config:
# If no parameter to be extracted is set, extract all is set.
extract_all = True
notebook = get_notebook_content(notebook_path=notebook_path)
notebook_metadata = notebook.get("metadata")
if notebook_metadata.get("language_info"):
language = notebook_metadata["language_info"]["name"]
if language and language != "python":
raise Exception("Only Python kernels are currently supported.")
if notebook_metadata.get("kernelspec"):
kernelspec = notebook_metadata.get("kernelspec")
kernel_name = kernelspec.get("name")
else:
kernel_name = "python3"
results["kernel_name"] = kernel_name
store_path: Path = Path(store_files_path)
if use_overlay:
if not kernel_name:
raise KeyError(
"No kernel name identified in notebook metadata kernelspec.")
store_path = store_path.joinpath("overlays").joinpath(kernel_name)
store_path.mkdir(parents=True, exist_ok=True)
dependency_resolution_engine = notebook_metadata.get(
"dependency_resolution_engine")
if not dependency_resolution_engine:
raise KeyError("No Resolution engine identified in notebook metadata.")
results["resolution_engine"] = dependency_resolution_engine
if pipfile or pipfile_lock or extract_all:
pipfile_string = notebook_metadata.get("requirements")
if not pipfile_string:
raise KeyError("No Pipfile identified in notebook metadata.")
pipfile_ = Pipfile.from_string(pipfile_string)
if pipfile or extract_all:
pipfile_path = store_path.joinpath("Pipfile")
if pipfile_path.exists() and not force:
raise FileExistsError(
f"Cannot store Pipfile because it already exists at path: {pipfile_path.as_posix()!r}. "
"Use --force to overwrite existing content or --show-only to visualize it."
)
else:
pipfile_.to_file(path=pipfile_path)
if pipfile_lock or extract_all:
pipfile_lock_string = notebook_metadata.get("requirements_lock")
if not pipfile_lock_string:
raise KeyError("No Pipfile.lock identified in notebook metadata.")
pipfile_lock_ = PipfileLock.from_string(
pipfile_content=pipfile_lock_string, pipfile=pipfile_)
pipfile_lock_path = store_path.joinpath("Pipfile.lock")
if pipfile_lock_path.exists() and not force:
raise FileExistsError(
f"Cannot store Pipfile.lock because it already exists at path: {pipfile_lock_path.as_posix()!r}. "
"Use --force to overwrite existing content or --show-only to visualize it."
)
else:
pipfile_lock_.to_file(path=pipfile_lock_path)
if thoth_config or extract_all:
thoth_config_string = notebook_metadata.get("thoth_config")
if not thoth_config_string:
raise KeyError("No .thoth.yaml identified in notebook metadata.")
config = _Configuration()
config.load_config_from_string(thoth_config_string)
yaml_path = Path(".thoth.yaml")
if yaml_path.exists() and not force:
raise FileExistsError(
f"Cannot store .thoth.yaml because it already exists at path: {yaml_path.as_posix()!r}. "
"Use --force to overwrite existing content or --show-only to visualize it."
)
else:
config.save_config()
return results
def horus_set_kernel_command(
path: str,
kernel_name: typing.Optional[str],
save_in_notebook: bool = True,
resolution_engine: typing.Optional[str] = None,
is_magic_command: bool = False,
):
"""Create kernel using dependencies in notebook metadata."""
results = {}
results["kernel_name"] = ""
results["dependency_resolution_engine"] = ""
# 0. Check if all metadata for dependencies are present in the notebook
notebook = get_notebook_content(notebook_path=path)
notebook_metadata = notebook.get("metadata")
if notebook_metadata.get("language_info"):
language = notebook_metadata["language_info"]["name"]
if language and language != "python":
raise Exception("Only Python kernels are currently supported.")
kernelspec = notebook_metadata.get("kernelspec")
notebook_kernel = kernelspec.get("name")
if not kernel_name:
kernel = notebook_kernel
else:
kernel = kernel_name
if kernel == "python3":
kernel = "jupyterlab-requirements"
results["kernel_name"]: str = kernel
home = Path.home()
store_path: Path = home.joinpath(".local/share/thoth/kernels")
if not resolution_engine:
dependency_resolution_engine = notebook_metadata.get(
"dependency_resolution_engine")
if not dependency_resolution_engine:
raise KeyError(
"No Resolution engine identified in notebook metadata.")
else:
dependency_resolution_engine = resolution_engine
results["dependency_resolution_engine"] = dependency_resolution_engine
complete_path: Path = store_path.joinpath(kernel)
if not is_magic_command:
if complete_path.exists():
delete_kernel(kernel_name=kernel)
complete_path.mkdir(parents=True, exist_ok=True)
# 1. Get Pipfile, Pipfile.lock and .thoth.yaml and store them in ./.local/share/kernel/{kernel_name}
# requirements
if not is_magic_command:
pipfile_string = notebook_metadata.get("requirements")
pipfile_ = Pipfile.from_string(pipfile_string)
pipfile_path = complete_path.joinpath("Pipfile")
pipfile_.to_file(path=pipfile_path)
# requirements lock
if not is_magic_command:
pipfile_lock_string = notebook_metadata.get("requirements_lock")
pipfile_lock_ = PipfileLock.from_string(
pipfile_content=pipfile_lock_string, pipfile=pipfile_)
pipfile_lock_path = complete_path.joinpath("Pipfile.lock")
pipfile_lock_.to_file(path=pipfile_lock_path)
if dependency_resolution_engine == "thoth" and not is_magic_command:
# thoth
thoth_config_string = notebook_metadata.get("thoth_config")
config = _Configuration()
config.load_config_from_string(thoth_config_string)
config_path = complete_path.joinpath(".thoth.yaml")
config.save_config(path=config_path)
# 2. Create virtualenv and install dependencies
install_packages(
kernel_name=kernel,
resolution_engine=dependency_resolution_engine,
is_cli=True,
is_magic_command=is_magic_command,
)
# 3. Install packages using micropipenv
create_kernel(kernel_name=kernel)
if save_in_notebook:
# Update kernel name if different name selected.
kernelspec["name"] = kernel
notebook_metadata["kernelspec"] = kernelspec
notebook["metadata"] = notebook_metadata
save_notebook_content(notebook_path=path, notebook=notebook)
return results
async def post(self):
"""Lock and install dependencies using pipenv."""
initial_path = Path.cwd()
input_data = self.get_json_body()
kernel_name: str = input_data["kernel_name"]
requirements: dict = json.loads(input_data["requirements"])
home = Path.home()
complete_path = home.joinpath(".local/share/thoth/kernels")
env_path = complete_path.joinpath(kernel_name)
# Delete and recreate folder
if not env_path.exists():
_ = subprocess.call(f"rm -rf ./{kernel_name} ",
shell=True,
cwd=complete_path)
env_path.mkdir(parents=True, exist_ok=True)
result = {"requirements_lock": "", "error": False}
## Create virtualenv
cli_run([str(env_path)])
pipfile_path = env_path.joinpath("Pipfile")
pipfile_string = Pipfile.from_dict(requirements).to_string()
_LOGGER.info("Resolution engine used: pipenv")
with open(pipfile_path, "w") as pipfile_file:
pipfile_file.write(pipfile_string)
_LOGGER.info(f"kernel path: {env_path}")
_LOGGER.info(f"Input Pipfile: \n{pipfile_string}")
try:
# TODO: check if pipenv is installed
subprocess.run(
f". {kernel_name}/bin/activate && cd {kernel_name} && pip install pipenv",
cwd=complete_path,
shell=True)
except Exception as pipenv_error:
_LOGGER.warning("error installing pipenv: %r", pipenv_error)
result['error'] = True
os.chdir(initial_path)
self.finish(json.dumps(result))
try:
subprocess.run(
f". {kernel_name}/bin/activate && cd {kernel_name} && pipenv lock",
env=dict(os.environ, PIPENV_CACHE_DIR='/tmp'),
cwd=complete_path,
shell=True)
except Exception as pipenv_error:
_LOGGER.warning("error locking dependencies using Pipenv: %r",
pipenv_error)
result['error'] = True
os.chdir(env_path)
if not result['error']:
pipfile_lock_path = env_path.joinpath("Pipfile.lock")
if pipfile_lock_path.exists():
with open(pipfile_lock_path, "r") as pipfile_lock_file:
pipfile_lock_str = pipfile_lock_file.read()
pipfile = Pipfile.from_string(pipfile_string)
pipfile_lock_str: PipfileLock = PipfileLock.from_string(
pipfile_lock_str, pipfile=pipfile)
result["requirements_lock"] = pipfile_lock_str.to_dict()
_LOGGER.debug(f"result from pipenv received: {result}")
else:
_LOGGER.warning("Pipfile.lock cannot be found at: %r",
str(pipfile_lock_path))
result['error'] = True
os.chdir(initial_path)
self.finish(json.dumps(result))