async def extensions_fetch(gcp_session: ClientSession,
dt_session: ClientSession,
token: str) -> Optional[ExtensionsFetchResult]:
extension_fetcher_result = await ExtensionsFetcher(
dt_session=dt_session,
dynatrace_url=await
fetch_dynatrace_url(gcp_session, get_project_id_from_environment(),
token),
dynatrace_access_key=await
fetch_dynatrace_api_key(gcp_session, get_project_id_from_environment(),
token),
logging_context=logging_context).execute()
if extension_fetcher_result.services:
feature_sets_names = [
f"{service.name}/{service.feature_set}"
for service in extension_fetcher_result.services
]
logging_context.log(
f'Monitoring enabled for the following fetched extensions feature sets: {feature_sets_names}'
)
return extension_fetcher_result
else:
logging_context.log(
"Monitoring disabled. Check configured extensions.")
return None
async def execute(self) -> FastCheckResult:
_check_configuration_flags(self.logging_context,
METRICS_CONFIGURATION_FLAGS)
project_list = await get_all_accessible_projects(
self.logging_context, self.gcp_session, self.token)
ready_to_monitor = []
for project_id in project_list:
tasks = [self._check_services(project_id)]
results = await asyncio.gather(*tasks, return_exceptions=True)
if all(result is not None for result in results):
ready_to_monitor.append(project_id)
dynatrace_url = await fetch_dynatrace_url(self.gcp_session,
project_id, self.token)
dynatrace_access_key = await fetch_dynatrace_api_key(
self.gcp_session, project_id, self.token)
await check_dynatrace(logging_context=self.logging_context,
project_id=get_project_id_from_environment(),
dt_session=self.dt_session,
dynatrace_url=dynatrace_url,
dynatrace_access_key=dynatrace_access_key)
return FastCheckResult(projects=ready_to_monitor)
def create_logs_context(sfm_queue: Queue):
dynatrace_api_key = get_dynatrace_api_key_from_env()
dynatrace_url = get_dynatrace_log_ingest_url_from_env()
project_id_owner = get_project_id_from_environment()
return LogsContext(project_id_owner=project_id_owner,
dynatrace_api_key=dynatrace_api_key,
dynatrace_url=dynatrace_url,
scheduled_execution_id=str(int(time.time()))[-8:],
sfm_queue=sfm_queue)
async def import_self_monitoring_dashboards(metadata: InstanceMetadata):
if metadata:
async with init_gcp_client_session() as gcp_session:
token = await create_token(logging_context, gcp_session)
if token:
sfm_dashboards_context = SfmDashboardsContext(
project_id_owner=get_project_id_from_environment(),
token=token,
gcp_session=gcp_session,
operation_mode=OPERATION_MODE,
scheduled_execution_id=None)
await import_self_monitoring_dashboard(
context=sfm_dashboards_context)
async def handle_event(event: Dict, event_context, project_id_owner: Optional[str]):
if event_context is Dict:
context = LoggingContext(event_context.get("execution_id", None))
else:
context = LoggingContext(None)
selected_services = None
if "GCP_SERVICES" in os.environ:
selected_services_string = os.environ.get("GCP_SERVICES", "")
selected_services = selected_services_string.split(",") if selected_services_string else []
services = load_supported_services(context, selected_services)
async with aiohttp.ClientSession() as session:
setup_start_time = time.time()
token = await create_token(context, session)
if token is None:
context.log("Cannot proceed without authorization token, stopping the execution")
return
if not isinstance(token, str):
raise Exception(f"Failed to fetch access token, got non string value: {token}")
context.log("Successfully obtained access token")
if not project_id_owner:
project_id_owner = get_project_id_from_environment()
dynatrace_api_key = await fetch_dynatrace_api_key(session=session, project_id=project_id_owner, token=token)
dynatrace_url = await fetch_dynatrace_url(session=session, project_id=project_id_owner, token=token)
print_metric_ingest_input = \
"PRINT_METRIC_INGEST_INPUT" in os.environ and os.environ["PRINT_METRIC_INGEST_INPUT"].upper() == "TRUE"
context = Context(
session=session,
project_id_owner=project_id_owner,
token=token,
execution_time=datetime.utcnow(),
execution_interval_seconds=60 * 1,
dynatrace_api_key=dynatrace_api_key,
dynatrace_url=dynatrace_url,
print_metric_ingest_input=print_metric_ingest_input,
scheduled_execution_id=context.scheduled_execution_id
)
projects_ids = await get_all_accessible_projects(context, session)
context.setup_execution_time = (time.time() - setup_start_time)
fetch_gcp_data_start_time = time.time()
fetch_ingest_lines_tasks = [fetch_ingest_lines_task(context, project_id, services) for project_id in projects_ids]
ingest_lines_per_project = await asyncio.gather(*fetch_ingest_lines_tasks, return_exceptions=True)
ingest_lines = [ingest_line for sublist in ingest_lines_per_project for ingest_line in sublist]
context.fetch_gcp_data_execution_time = time.time() - fetch_gcp_data_start_time
context.log(f"Fetched GCP data in {context.fetch_gcp_data_execution_time} s")
await push_ingest_lines(context, ingest_lines)
await push_self_monitoring_time_series(context)
await session.close()
async def handle_event(event: Dict,
event_context,
project_id_owner: Optional[str],
projects_ids: Optional[List[str]] = None):
if isinstance(event_context, Dict):
context = LoggingContext(event_context.get("execution_id", None))
else:
context = LoggingContext(None)
selected_services = None
if "GCP_SERVICES" in os.environ:
selected_services_string = os.environ.get("GCP_SERVICES", "")
selected_services = selected_services_string.split(
",") if selected_services_string else []
#set default featureset if featureset not present in env variable
for i, service in enumerate(selected_services):
if "/" not in service:
selected_services[i] = f"{service}/default"
services = load_supported_services(context, selected_services)
async with init_gcp_client_session(
) as gcp_session, init_dt_client_session() as dt_session:
setup_start_time = time.time()
token = await create_token(context, gcp_session)
if token is None:
context.log(
"Cannot proceed without authorization token, stopping the execution"
)
return
if not isinstance(token, str):
raise Exception(
f"Failed to fetch access token, got non string value: {token}")
context.log("Successfully obtained access token")
if not project_id_owner:
project_id_owner = get_project_id_from_environment()
dynatrace_api_key = await fetch_dynatrace_api_key(
gcp_session=gcp_session, project_id=project_id_owner, token=token)
dynatrace_url = await fetch_dynatrace_url(gcp_session=gcp_session,
project_id=project_id_owner,
token=token)
print_metric_ingest_input = \
"PRINT_METRIC_INGEST_INPUT" in os.environ and os.environ["PRINT_METRIC_INGEST_INPUT"].upper() == "TRUE"
self_monitoring_enabled = os.environ.get('SELF_MONITORING_ENABLED',
"False").upper() == "TRUE"
context = MetricsContext(
gcp_session=gcp_session,
dt_session=dt_session,
project_id_owner=project_id_owner,
token=token,
execution_time=datetime.utcnow(),
execution_interval_seconds=60 * 1,
dynatrace_api_key=dynatrace_api_key,
dynatrace_url=dynatrace_url,
print_metric_ingest_input=print_metric_ingest_input,
self_monitoring_enabled=self_monitoring_enabled,
scheduled_execution_id=context.scheduled_execution_id)
if not projects_ids:
projects_ids = await get_all_accessible_projects(
context, gcp_session, token)
setup_time = (time.time() - setup_start_time)
context.setup_execution_time = {
project_id: setup_time
for project_id in projects_ids
}
context.start_processing_timestamp = time.time()
process_project_metrics_tasks = [
process_project_metrics(context, project_id, services)
for project_id in projects_ids
]
await asyncio.gather(*process_project_metrics_tasks,
return_exceptions=True)
context.log(
f"Fetched and pushed GCP data in {time.time() - context.start_processing_timestamp} s"
)
log_self_monitoring_data(context)
if context.self_monitoring_enabled:
await push_self_monitoring(context)
await gcp_session.close()
await dt_session.close()
async def handle_event(event: Dict,
event_context,
projects_ids: Optional[List[str]] = None,
services: Optional[List[GCPService]] = None):
if isinstance(event_context, Dict):
# for k8s installation
context = LoggingContext(event_context.get("execution_id", None))
else:
context = LoggingContext(None)
if not services:
# load services for GCP Function
services = load_supported_services(context)
async with init_gcp_client_session(
) as gcp_session, init_dt_client_session() as dt_session:
setup_start_time = time.time()
token = await create_token(context, gcp_session)
if token is None:
context.log(
"Cannot proceed without authorization token, stopping the execution"
)
return
if not isinstance(token, str):
raise Exception(
f"Failed to fetch access token, got non string value: {token}")
context.log("Successfully obtained access token")
project_id_owner = get_project_id_from_environment()
dynatrace_api_key = await fetch_dynatrace_api_key(
gcp_session=gcp_session, project_id=project_id_owner, token=token)
dynatrace_url = await fetch_dynatrace_url(gcp_session=gcp_session,
project_id=project_id_owner,
token=token)
check_version(logging_context=context)
await check_dynatrace(logging_context=context,
project_id=project_id_owner,
dt_session=dt_session,
dynatrace_url=dynatrace_url,
dynatrace_access_key=dynatrace_api_key)
query_interval_min = get_query_interval_minutes()
print_metric_ingest_input = os.environ.get(
"PRINT_METRIC_INGEST_INPUT", "FALSE").upper() in ["TRUE", "YES"]
self_monitoring_enabled = os.environ.get(
'SELF_MONITORING_ENABLED', "FALSE").upper() in ["TRUE", "YES"]
context = MetricsContext(
gcp_session=gcp_session,
dt_session=dt_session,
project_id_owner=project_id_owner,
token=token,
execution_time=datetime.utcnow(),
execution_interval_seconds=60 * query_interval_min,
dynatrace_api_key=dynatrace_api_key,
dynatrace_url=dynatrace_url,
print_metric_ingest_input=print_metric_ingest_input,
self_monitoring_enabled=self_monitoring_enabled,
scheduled_execution_id=context.scheduled_execution_id)
if not projects_ids:
projects_ids = await get_all_accessible_projects(
context, gcp_session, token)
disabled_apis = {}
disabled_projects = []
for project_id in projects_ids:
await check_x_goog_user_project_header_permissions(
context, project_id)
disabled_apis = {
project_id: await get_all_disabled_apis(context, project_id)
}
if 'monitoring.googleapis.com' in disabled_apis[project_id]:
disabled_projects.append(project_id)
if disabled_projects:
context.log(
f"monitoring.googleapis.com API disabled in the projects: " +
", ".join(disabled_projects) +
", that projects will not be monitored")
for disabled_project in disabled_projects:
projects_ids.remove(disabled_project)
setup_time = (time.time() - setup_start_time)
context.setup_execution_time = {
project_id: setup_time
for project_id in projects_ids
}
context.start_processing_timestamp = time.time()
process_project_metrics_tasks = [
process_project_metrics(context, project_id, services,
disabled_apis.get(project_id, set()))
for project_id in projects_ids
]
await asyncio.gather(*process_project_metrics_tasks,
return_exceptions=True)
context.log(
f"Fetched and pushed GCP data in {time.time() - context.start_processing_timestamp} s"
)
log_self_monitoring_data(context)
if context.self_monitoring_enabled:
await push_self_monitoring(context)
await gcp_session.close()
await dt_session.close()
