def _download_and_install_wheel(self,
uri: str,
logger: Optional[
logging.Logger] = default_logger):
"""Download and install a wheel URI, and then delete the local wheel file."""
wheel_file = download_and_unpack_package(uri,
self._resources_dir,
logger=logger)
module_dir = self._get_local_dir_from_uri(uri)
pip_install_cmd = [
"pip",
"install",
wheel_file,
f"--target={module_dir}",
]
logger.info("Running py_modules wheel install command: %s",
str(pip_install_cmd))
try:
exit_code, output = exec_cmd_stream_to_logger(
pip_install_cmd, logger)
finally:
if Path(wheel_file).exists():
Path(wheel_file).unlink()
if exit_code != 0:
if Path(module_dir).exists():
Path(module_dir).unlink()
raise RuntimeError(
f"Failed to install py_modules wheel {wheel_file}"
f"to {module_dir}:\n{output}")
return module_dir
def _download_jars(
self, uri: str, logger: Optional[logging.Logger] = default_logger
):
"""Download a jar URI."""
jar_file = download_and_unpack_package(uri, self._resources_dir, logger=logger)
module_dir = self._get_local_dir_from_uri(uri)
logger.debug(f"Succeeded to download jar file {jar_file} .")
return module_dir
def _create():
if is_jar_uri(uri):
module_dir = self._download_jars(uri=uri, logger=logger)
else:
module_dir = download_and_unpack_package(
uri, self._resources_dir, logger=logger
)
return get_directory_size_bytes(module_dir)
async def create(
self,
uri: str,
runtime_env: dict,
context: RuntimeEnvContext,
logger: Optional[logging.Logger] = default_logger,
) -> int:
local_dir = download_and_unpack_package(uri,
self._resources_dir,
logger=logger)
return get_directory_size_bytes(local_dir)
def setup(self,
runtime_env: RuntimeEnv,
context: RuntimeEnvContext,
logger: Optional[logging.Logger] = default_logger):
if not runtime_env.py_modules():
return
module_dirs = []
for uri in runtime_env.py_modules():
module_dir = download_and_unpack_package(
uri, self._resources_dir, logger=logger)
module_dirs.append(module_dir)
# Insert the py_modules directories into the PYTHONPATH.
python_path = os.pathsep.join(module_dirs)
if "PYTHONPATH" in context.env_vars:
python_path += os.pathsep + context.env_vars["PYTHONPATH"]
context.env_vars["PYTHONPATH"] = python_path
def setup(self,
runtime_env: dict,
context: RuntimeEnvContext,
logger: Optional[logging.Logger] = default_logger):
if not runtime_env.get("working_dir"):
return
working_dir = download_and_unpack_package(runtime_env["working_dir"],
self._resources_dir,
logger=logger)
context.command_prefix += [f"cd {working_dir}"]
# Insert the working_dir as the first entry in PYTHONPATH. This is
# compatible with users providing their own PYTHONPATH in env_vars.
python_path = working_dir
if "PYTHONPATH" in context.env_vars:
python_path += os.pathsep + context.env_vars["PYTHONPATH"]
context.env_vars["PYTHONPATH"] = python_path
def _create():
local_dir = download_and_unpack_package(uri,
self._resources_dir,
logger=logger)
return get_directory_size_bytes(local_dir)
