async def fetch_ingest_lines_task(
context: MetricsContext, project_id: str,
services: List[GCPService]) -> List[IngestLine]:
fetch_metric_tasks = []
topology_tasks = []
topology_task_services = []
for service in services:
if service.name in entities_extractors:
topology_task = entities_extractors[service.name](context,
project_id,
service)
topology_tasks.append(topology_task)
topology_task_services.append(service)
fetch_topology_results = await asyncio.gather(*topology_tasks,
return_exceptions=True)
skipped_services = []
for service in services:
if service in topology_task_services:
service_topology = fetch_topology_results[
topology_task_services.index(service)]
if not service_topology:
skipped_services.append(service.name)
continue # skip fetching the metrics because there are no instances
for metric in service.metrics:
fetch_metric_task = run_fetch_metric(context=context,
project_id=project_id,
service=service,
metric=metric)
fetch_metric_tasks.append(fetch_metric_task)
if skipped_services:
skipped_services_string = ', '.join(skipped_services)
context.log(
project_id,
f"Skipped fetching metrics for {skipped_services_string} due to no instances detected"
)
fetch_metric_results = await asyncio.gather(*fetch_metric_tasks,
return_exceptions=True)
entity_id_map = build_entity_id_map(fetch_topology_results)
flat_metric_results = flatten_and_enrich_metric_results(
context, fetch_metric_results, entity_id_map)
return flat_metric_results
async def fetch_ingest_lines_task(context, project_id, services):
fetch_metric_tasks = []
topology_tasks = []
for service in services:
if service.name in entities_extractors:
topology_task = entities_extractors[service.name](context, project_id, service)
topology_tasks.append(topology_task)
for metric in service.metrics:
fetch_metric_task = run_fetch_metric(
context=context,
project_id=project_id,
service=service,
metric=metric
)
fetch_metric_tasks.append(fetch_metric_task)
all_results = await asyncio.gather(
asyncio.gather(*fetch_metric_tasks, return_exceptions=True),
asyncio.gather(*topology_tasks, return_exceptions=True)
)
fetch_metric_results = all_results[0]
fetch_topology_results = all_results[1]
entity_id_map = build_entity_id_map(fetch_topology_results)
flat_metric_results = flatten_and_enrich_metric_results(fetch_metric_results, entity_id_map)
return flat_metric_results
async def fetch_ingest_lines_task(context: MetricsContext, project_id: str,
services: List[GCPService],
disabled_apis: Set[str]) -> List[IngestLine]:
fetch_metric_tasks = []
topology_tasks = []
topology_task_services = []
skipped_topology_services = set()
for service in services:
if service.name in entities_extractors:
if entities_extractors[service.name].used_api in disabled_apis:
skipped_topology_services.add(service.name)
continue
topology_task = entities_extractors[service.name].extractor(
context, project_id, service)
topology_tasks.append(topology_task)
topology_task_services.append(service)
if skipped_topology_services:
skipped_topology_services_string = ", ".join(skipped_topology_services)
context.log(
project_id,
f"Skipped fetching topology for disabled services: {skipped_topology_services_string}"
)
fetch_topology_results = await asyncio.gather(*topology_tasks,
return_exceptions=True)
skipped_services_no_instances = []
skipped_disabled_apis = set()
for service in services:
if service in topology_task_services:
service_topology = fetch_topology_results[
topology_task_services.index(service)]
if not service_topology:
skipped_services_no_instances.append(
f"{service.name}/{service.feature_set}")
continue # skip fetching the metrics because there are no instances
for metric in service.metrics:
gcp_api_last_index = metric.google_metric.find("/")
api = metric.google_metric[:gcp_api_last_index]
if api in disabled_apis:
skipped_disabled_apis.add(api)
continue # skip fetching the metrics because service API is disabled
fetch_metric_task = run_fetch_metric(context=context,
project_id=project_id,
service=service,
metric=metric)
fetch_metric_tasks.append(fetch_metric_task)
if skipped_services_no_instances:
skipped_services_string = ', '.join(skipped_services_no_instances)
context.log(
project_id,
f"Skipped fetching metrics for {skipped_services_string} due to no instances detected"
)
if skipped_disabled_apis:
skipped_disabled_apis_string = ", ".join(skipped_disabled_apis)
context.log(
project_id,
f"Skipped fetching metrics for disabled APIs: {skipped_disabled_apis_string}"
)
fetch_metric_results = await asyncio.gather(*fetch_metric_tasks,
return_exceptions=True)
entity_id_map = build_entity_id_map(fetch_topology_results)
flat_metric_results = flatten_and_enrich_metric_results(
context, fetch_metric_results, entity_id_map)
return flat_metric_results