async def get_package(self,
req: aiohttp.web.Request) -> aiohttp.web.Response:
package_uri = req.query["package_uri"]
resp = GetPackageResponse(package_exists=package_exists(package_uri))
return aiohttp.web.Response(
text=json.dumps(dataclasses.asdict(resp)),
content_type="application/json")
async def get_package(self, req: Request) -> Response:
package_uri = http_uri_components_to_uri(
protocol=req.match_info["protocol"],
package_name=req.match_info["package_name"])
if not package_exists(package_uri):
return Response(text=f"Package {package_uri} does not exist",
status=aiohttp.web.HTTPNotFound.status_code)
return Response()
async def get_package(self, req: Request) -> Response:
package_uri = http_uri_components_to_uri(
protocol=req.match_info["protocol"],
package_name=req.match_info["package_name"],
)
logger.debug(f"Adding temporary reference to package {package_uri}.")
try:
pin_runtime_env_uri(package_uri)
except Exception:
return Response(
text=traceback.format_exc(),
status=aiohttp.web.HTTPInternalServerError.status_code,
)
if not package_exists(package_uri):
return Response(
text=f"Package {package_uri} does not exist",
status=aiohttp.web.HTTPNotFound.status_code,
)
return Response()
def upload_py_modules_if_needed(
runtime_env: Dict[str, Any],
scratch_dir: Optional[str] = os.getcwd(),
logger: Optional[logging.Logger] = default_logger,
upload_fn=None,
) -> Dict[str, Any]:
"""Uploads the entries in py_modules and replaces them with a list of URIs.
For each entry that is already a URI, this is a no-op.
"""
py_modules = runtime_env.get("py_modules")
if py_modules is None:
return runtime_env
if not isinstance(py_modules, list):
raise TypeError(
"py_modules must be a List of local paths, imported modules, or "
f"URIs, got {type(py_modules)}.")
py_modules_uris = []
for module in py_modules:
if isinstance(module, str):
# module_path is a local path or a URI.
module_path = module
elif isinstance(module, Path):
module_path = str(module)
elif isinstance(module, ModuleType):
# NOTE(edoakes): Python allows some installed Python packages to
# be split into multiple directories. We could probably handle
# this, but it seems tricky & uncommon. If it's a problem for
# users, we can add this support on demand.
if len(module.__path__) > 1:
raise ValueError(
"py_modules only supports modules whose __path__ has length 1."
)
[module_path] = module.__path__
else:
raise TypeError("py_modules must be a list of file paths, URIs, "
f"or imported modules, got {type(module)}.")
if _check_is_uri(module_path):
module_uri = module_path
else:
# module_path is a local path.
if Path(module_path).is_dir():
excludes = runtime_env.get("excludes", None)
module_uri = get_uri_for_directory(module_path,
excludes=excludes)
if upload_fn is None:
upload_package_if_needed(
module_uri,
scratch_dir,
module_path,
excludes=excludes,
include_parent_dir=True,
logger=logger,
)
else:
upload_fn(module_path, excludes=excludes)
elif Path(module_path).suffix == ".whl":
module_uri = get_uri_for_package(Path(module_path))
if upload_fn is None:
if not package_exists(module_uri):
upload_package_to_gcs(module_uri,
Path(module_path).read_bytes())
else:
upload_fn(module_path, excludes=None, is_file=True)
else:
raise ValueError(
"py_modules entry must be a directory or a .whl file; "
f"got {module_path}")
py_modules_uris.append(module_uri)
# TODO(architkulkarni): Expose a single URI for py_modules. This plugin
# should internally handle the "sub-URIs", the individual modules.
runtime_env["py_modules"] = py_modules_uris
return runtime_env