async def test_get_always_the_same_client_session():
app = web.Application()
session = get_client_session(app)
assert session in app.values()
assert app[APP_CLIENT_SESSION_KEY] == session
for _ in range(3):
assert get_client_session(app) == session
async def update_service(
app: web.Application,
user_id: int,
service_key: str,
service_version: str,
product_name: str,
update_data: Dict[str, Any],
) -> Dict[str, Any]:
url = (URL(app[KCATALOG_ORIGIN]).with_path(
app[KCATALOG_VERSION_PREFIX] +
f"/services/{urllib.parse.quote_plus(service_key)}/{service_version}").
with_query({
"user_id": user_id,
}))
session = get_client_session(app)
try:
async with session.patch(url,
headers={X_PRODUCT_NAME_HEADER: product_name},
json=update_data) as resp:
resp.raise_for_status(
) # FIXME: error handling for session and response exceptions
return await resp.json()
except asyncio.TimeoutError as err:
logger.warning("Catalog service connection timeout error")
raise web.HTTPServiceUnavailable(
reason="catalog is currently unavailable") from err
async def copy_file_s3_datcore(self, user_id: str, dest_uuid: str,
source_uuid: str):
session = get_client_session(self.app)
# source is s3, get link and copy to datcore
bucket_name = self.simcore_bucket_name
object_name = source_uuid
filename = source_uuid.split("/")[-1]
s3_dowload_link = self.s3_client.create_presigned_get_url(
bucket_name, object_name)
with tempfile.TemporaryDirectory() as tmpdir:
# FIXME: connect download and upload streams
local_file_path = os.path.join(tmpdir, filename)
# Downloads S3 -> local
await download_to_file_or_raise(session, s3_dowload_link,
local_file_path)
# Uploads local -> DATCore
await self.upload_file_to_datcore(
_user_id=user_id,
_local_file_path=local_file_path,
_destination_id=dest_uuid,
)
def _get_storage_client(app: web.Application) -> Tuple[ClientSession, URL]:
settings: StorageSettings = get_plugin_settings(app)
# storage service API endpoint
endpoint = URL(settings.base_url)
session = get_client_session(app)
return session, endpoint
async def get_service(
app: web.Application,
user_id: int,
service_key: str,
service_version: str,
product_name: str,
) -> Dict[str, Any]:
session: ClientSession = get_client_session(app)
settings: CatalogSettings = get_plugin_settings(app)
url = (URL(settings.api_base_url) /
f"services/{urllib.parse.quote_plus(service_key)}/{service_version}"
).with_query({"user_id": user_id})
try:
async with session.get(url,
headers={X_PRODUCT_NAME_HEADER:
product_name}) as resp:
resp.raise_for_status(
) # FIXME: error handling for session and response exceptions
return await resp.json()
except asyncio.TimeoutError as err:
logger.warning("Catalog service connection timeout error")
raise web.HTTPServiceUnavailable(
reason="catalog is currently unavailable") from err
async def make_request_and_envelope_response(
app: web.Application,
method: str,
url: URL,
headers: Optional[Dict[str, str]] = None,
data: Optional[bytes] = None,
) -> web.Response:
"""
Helper to forward a request to the catalog service
"""
session = get_client_session(app)
try:
async with session.request(method, url, headers=headers,
data=data) as resp:
payload = await resp.json()
try:
resp.raise_for_status()
resp_data = wrap_as_envelope(data=payload)
except ClientResponseError as err:
if 500 <= err.status:
raise err
resp_data = wrap_as_envelope(error=payload["errors"])
return web.json_response(resp_data, status=resp.status)
except (asyncio.TimeoutError, ClientConnectionError,
ClientResponseError) as err:
logger.warning("Catalog service errors upon request %s %s: %s", method,
url.relative(), err)
raise web.HTTPServiceUnavailable(
reason="catalog is currently unavailable") from err
async def get_services_for_user_in_product(
app: web.Application, user_id: int, product_name: str, *,
only_key_versions: bool) -> List[Dict]:
url = (URL(app[KCATALOG_ORIGIN]).with_path(app[KCATALOG_VERSION_PREFIX] +
"/services").with_query({
"user_id":
user_id,
"details":
f"{not only_key_versions}"
}))
session = get_client_session(app)
try:
async with session.get(
url,
headers={X_PRODUCT_NAME_HEADER: product_name},
) as resp:
if resp.status >= 400:
logger.warning(
"Error while retrieving services for user %s. Returning an empty list",
user_id,
)
return []
return await resp.json()
except asyncio.TimeoutError as err:
logger.warning("Catalog service connection timeout error")
raise web.HTTPServiceUnavailable(
reason="catalog is currently unavailable") from err
def _get_director_client(app: web.Application) -> Tuple[ClientSession, URL]:
cfg: Dict[str, Any] = get_config(app)
api_endpoint = URL.build(scheme="http", host=cfg["host"],
port=cfg["port"]).with_path(cfg["version"])
session = get_client_session(app)
return session, api_endpoint
def acquire_instance(cls, app: MutableMapping[str, Any],
settings: SciCrunchSettings) -> "SciCrunch":
"""Returns single instance for the application and stores it"""
obj = app.get(f"{__name__}.{cls.__name__}")
if obj is None:
session = get_client_session(app)
app[f"{__name__}.{cls.__name__}"] = obj = cls(session, settings)
return obj
async def get_storage_locations_for_user(app: web.Application,
user_id: int) -> List[Dict[str, Any]]:
session = get_client_session(app)
url: URL = _get_base_storage_url(app) / "locations"
params = dict(user_id=user_id)
async with session.get(url, ssl=False, params=params) as resp:
data, _ = cast(List[Dict[str, Any]], await safe_unwrap(resp))
return data
async def get_file_upload_url(app: web.Application, location_id: str,
fileId: str, user_id: int) -> str:
session = get_client_session(app)
url: URL = (_get_base_storage_url(app) / "locations" / location_id /
"files" / urllib.parse.quote(fileId, safe=""))
params = dict(user_id=user_id)
async with session.put(url, ssl=False, params=params) as resp:
data, _ = await safe_unwrap(resp)
return extract_link(data)
def _get_client_session_info():
client: ClientSession = get_client_session(request.app)
info = {"instance": str(client)}
if not client.closed:
info.update({
"limit": client.connector.limit,
"limit_per_host": client.connector.limit_per_host,
})
return info
def app_mock():
# we only need the app as a container of the aiohttp client session and the director entrypoint
app = web.Application()
# mocks client session (normally included as upon startup)
single_client_in_app = get_client_session(app)
assert single_client_in_app
assert single_client_in_app is get_client_session(app)
# mocks loading config
settings = DirectorSettings()
app[director_config.APP_CONFIG_KEY] = {
director_config.CONFIG_SECTION_NAME:
json.loads(settings.json(exclude={"url"}, by_alias=True))
}
assert setup_director(app, disable_routes=True)
return app
async def test_app_client_session_json_serialize(
server: TestServer, fake_data_dict: Dict[str, Any]
):
session = get_client_session(server.app)
resp = await session.post(server.make_url("/echo"), json=fake_data_dict)
assert resp.status == 200
got = await resp.json()
expected = json.loads(json_dumps(fake_data_dict))
assert got == expected
async def fake_app(mock_env_devel_environment, loop):
# By using .env-devel we ensure all needed variables are at
# least defined there
print("app's environment variables", format(mock_env_devel_environment))
app = {}
setup_scicrunch_submodule(app)
assert app
yield app
client = get_client_session(app)
await client.close()
async def get_status(request: aiohttp.web.Request):
session = get_client_session(request.app)
user_id = request.get(RQT_USERID_KEY, -1)
config = request.app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
url = URL.build(
scheme="http",
host=config["prometheus_host"],
port=config["prometheus_port"],
path=f"/api/{config['prometheus_api_version']}/query",
)
results = await asyncio.gather(
get_cpu_usage(session, url, user_id),
get_memory_usage(session, url, user_id),
get_container_metric_for_labels(session, url, user_id),
return_exceptions=True,
)
cpu_usage = get_prometheus_result_or_default(results[0], [])
mem_usage = get_prometheus_result_or_default(results[1], [])
metric = get_prometheus_result_or_default(results[2], [])
res = defaultdict(dict)
for node in cpu_usage:
node_id = node["metric"]["container_label_node_id"]
usage = float(node["value"][1])
res[node_id] = {"stats": {"cpuUsage": usage}}
for node in mem_usage:
node_id = node["metric"]["container_label_node_id"]
usage = float(node["value"][1])
if node_id in res:
res[node_id]["stats"]["memUsage"] = usage
else:
res[node_id] = {"stats": {"memUsage": usage}}
for node in metric:
limits = {"cpus": 0.0, "mem": 0.0}
metric_labels = node["metric"]
limits["cpus"] = float(
metric_labels.get("container_label_nano_cpus_limit", 0)) / pow(
10, 9) # Nanocpus to cpus
limits["mem"] = float(metric_labels.get("container_label_mem_limit",
0)) / pow(1024, 2) # In MB
node_id = metric_labels.get("container_label_node_id")
res[node_id]["limits"] = limits
if not res:
raise aiohttp.web.HTTPNoContent
return dict(res)
async def fake_app(mock_env_devel_environment, loop):
# By using .env-devel we ensure all needed variables are at
# least defined there
print("app's environment variables", format(mock_env_devel_environment))
app = create_safe_application()
setup_settings(app)
setup_scicrunch(app)
yield app
client = get_client_session(app)
await client.close()
def app_mock():
# we only need the app as a container of the aiohttp client session and the director entrypoint
app = web.Application()
# mocks client session (normally included as upon startup)
single_client_in_app = get_client_session(app)
assert single_client_in_app
assert single_client_in_app is get_client_session(app)
# mocks loading config
settings = DirectorSettings()
app[director_config.APP_CONFIG_KEY] = {
director_config.CONFIG_SECTION_NAME: {
"host": settings.DIRECTOR_HOST,
"port": settings.DIRECTOR_PORT,
"version": settings.DIRECTOR_VTAG,
"enabled": True,
}
}
assert setup_director(app, disable_routes=True)
return app
async def get_status(request: aiohttp.web.Request):
session = get_client_session(request.app)
user_id = request.get(RQT_USERID_KEY, -1)
prometheus_settings = get_plugin_settings(request.app)
url = URL(prometheus_settings.base_url)
results = await asyncio.gather(
get_cpu_usage(session, url, user_id),
get_memory_usage(session, url, user_id),
get_container_metric_for_labels(session, url, user_id),
return_exceptions=True,
)
cpu_usage = get_prometheus_result_or_default(results[0], [])
mem_usage = get_prometheus_result_or_default(results[1], [])
metric = get_prometheus_result_or_default(results[2], [])
res = defaultdict(dict)
for node in cpu_usage:
node_id = node["metric"]["container_label_node_id"]
usage = float(node["value"][1])
res[node_id] = {"stats": {"cpuUsage": usage}}
for node in mem_usage:
node_id = node["metric"]["container_label_node_id"]
usage = float(node["value"][1])
if node_id in res:
res[node_id]["stats"]["memUsage"] = usage
else:
res[node_id] = {"stats": {"memUsage": usage}}
for node in metric:
limits = {"cpus": 0.0, "mem": 0.0}
metric_labels = node["metric"]
limits["cpus"] = float(
metric_labels.get("container_label_nano_cpus_limit", 0)) / pow(
10, 9) # Nanocpus to cpus
limits["mem"] = float(metric_labels.get("container_label_mem_limit",
0)) / pow(1024, 2) # In MB
node_id = metric_labels.get("container_label_node_id")
res[node_id]["limits"] = limits
if not res:
raise aiohttp.web.HTTPNoContent
return dict(res)
async def _request_storage(request: web.Request, method: str, **kwargs):
await extract_and_validate(request)
url = _resolve_storage_url(request)
# _token_data, _token_secret = _get_token_key_and_secret(request)
body = None
if request.can_read_body:
body = await request.json()
session = get_client_session(request.app)
async with session.request(method.upper(),
url,
ssl=False,
json=body,
**kwargs) as resp:
payload = await resp.json()
return payload
async def is_service_responsive(app: web.Application):
"""Returns true if catalog is ready"""
try:
session: ClientSession = get_client_session(app)
settings: CatalogSettings = get_plugin_settings(app)
await session.get(
settings.base_url,
ssl=False,
raise_for_status=True,
timeout=ClientTimeout(total=2, connect=1),
)
except (ClientConnectionError, ClientResponseError, InvalidURL,
ValueError) as err:
logger.warning("Catalog service unresponsive: %s", err)
return False
else:
return True
async def get_project_files_metadata(app: web.Application, location_id: str,
uuid_filter: str,
user_id: int) -> List[Dict[str, Any]]:
session = get_client_session(app)
url: URL = (_get_base_storage_url(app) / "locations" / location_id /
"files" / "metadata")
params = dict(user_id=user_id, uuid_filter=uuid_filter)
async with session.get(url, ssl=False, params=params) as resp:
data, _ = await safe_unwrap(resp)
if data is None:
raise web.HTTPException(
reason=f"No url found in response: '{data}'")
if not isinstance(data, list):
raise web.HTTPException(
reason=f"No list payload received as data: '{data}'")
return data
async def _assemble_cached_indexes(app: web.Application):
"""
Currently the static resources are contain 3 folders: osparc, s4l, tis
each of them contain and index.html to be served to as the root of the site
for each type of frontend.
Caching these 3 items on start. This
"""
settings: StaticWebserverModuleSettings = get_settings(app)
cached_indexes: Dict[str, str] = {}
session: ClientSession = get_client_session(app)
for frontend_name in FRONTEND_APPS_AVAILABLE:
url = URL(settings.static_web_server_url) / frontend_name
log.info("Fetching index from %s", url)
try:
# web-static server might still not be up
async for attempt in AsyncRetrying(**RETRY_ON_STARTUP_POLICY):
with attempt:
response = await session.get(url, raise_for_status=True)
body = await response.text()
except ClientError as err:
log.error("Could not fetch index from static server: %s", err)
# ANE: Yes this is supposed to fail the boot process
raise RuntimeError(
f"Could not fetch index at {str(url)}. Stopping application boot"
) from err
# fixes relative paths
body = body.replace(f"../resource/{frontend_name}",
f"resource/{frontend_name}")
body = body.replace("boot.js", f"{frontend_name}/boot.js")
log.info("Storing index for %s", url)
cached_indexes[frontend_name] = body
app[APP_FRONTEND_CACHED_INDEXES_KEY] = cached_indexes
async def copy_file_datcore_s3(
self,
user_id: str,
dest_uuid: str,
source_uuid: str,
filename_missing: bool = False,
):
session = get_client_session(self.app)
# 2 steps: Get download link for local copy, the upload link to s3
# TODO: This should be a redirect stream!
dc_link, filename = await self.download_link_datcore(
user_id=user_id, file_id=source_uuid)
if filename_missing:
dest_uuid = str(Path(dest_uuid) / filename)
s3_upload_link = await self.upload_link(user_id, dest_uuid)
with tempfile.TemporaryDirectory() as tmpdir:
# FIXME: connect download and upload streams
local_file_path = os.path.join(tmpdir, filename)
# Downloads DATCore -> local
await download_to_file_or_raise(session, dc_link, local_file_path)
# Uploads local -> S3
s3_upload_link = URL(s3_upload_link)
async with session.put(
s3_upload_link,
data=Path(local_file_path).open("rb"),
raise_for_status=True,
) as resp:
logger.debug(
"Uploaded local -> SIMCore %s . Status %s",
s3_upload_link,
resp.status,
)
return dest_uuid
async def is_service_responsive(app: web.Application):
"""Returns true if catalog is ready"""
try:
origin: Optional[URL] = app.get(KCATALOG_ORIGIN)
if not origin:
raise ValueError(
"KCATALOG_ORIGIN was not initialized (app module was not enabled?)"
)
client: ClientSession = get_client_session(app)
await client.get(
origin,
ssl=False,
raise_for_status=True,
timeout=ClientTimeout(total=2, connect=1),
)
except (ClientConnectionError, ClientResponseError, InvalidURL,
ValueError) as err:
logger.warning("Catalog service unresponsive: %s", err)
return False
else:
return True
async def get_status(request: aiohttp.web.Request):
session = get_client_session(request.app)
user_id = request.get(RQT_USERID_KEY, -1)
config = request.app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
url = URL.build(
scheme="http",
host=config["prometheus_host"],
port=config["prometheus_port"],
path=f"/api/{config['prometheus_api_version']}/query",
)
results = await asyncio.gather(
get_cpu_usage(session, url, user_id),
get_memory_usage(session, url, user_id),
get_celery_reserved(request.app),
get_container_metric_for_labels(session, url, user_id),
return_exceptions=True,
)
cpu_usage = get_prometheus_result_or_default(results[0], [])
mem_usage = get_prometheus_result_or_default(results[1], [])
metric = get_prometheus_result_or_default(results[3], [])
celery_inspect = results[2]
res = defaultdict(dict)
for node in cpu_usage:
node_id = node["metric"]["container_label_node_id"]
usage = float(node["value"][1])
res[node_id] = {"stats": {"cpuUsage": usage}}
for node in mem_usage:
node_id = node["metric"]["container_label_node_id"]
usage = float(node["value"][1])
if node_id in res:
res[node_id]["stats"]["memUsage"] = usage
else:
res[node_id] = {"stats": {"memUsage": usage}}
for node in metric:
limits = {"cpus": 0, "mem": 0}
metric_labels = node["metric"]
limits["cpus"] = float(
metric_labels.get("container_label_nano_cpus_limit", 0)
) / pow(
10, 9
) # Nanocpus to cpus
limits["mem"] = float(metric_labels.get("container_label_mem_limit", 0)) / pow(
1024, 2
) # In MB
node_id = metric_labels.get("container_label_node_id")
res[node_id]["limits"] = limits
if hasattr(celery_inspect, "items"):
for dummy_worker_id, worker in celery_inspect.items():
for task in worker:
values = task["args"][1:-1].split(", ")
if values[0] == str(user_id): # Extracts user_id from task's args
node_id = values[2][1:-1] # Extracts node_id from task's args
res[node_id]["queued"] = True
if not res:
raise aiohttp.web.HTTPNoContent
return dict(res)
def test_setup_scicrunch_submodule(fake_app):
# scicruch should be init
scicrunch = SciCrunch.get_instance(fake_app)
assert scicrunch
assert scicrunch.client == get_client_session(fake_app)
def _get_director_client(app: web.Application) -> Tuple[ClientSession, URL]:
settings: DirectorSettings = get_plugin_settings(app)
api_endpoint = settings.base_url
session = get_client_session(app)
return session, api_endpoint
