async def handle_event(event: Dict, event_context, project_id_owner: Optional[str], projects_ids: Optional[List[str]] = None): if isinstance(event_context, Dict): context = LoggingContext(event_context.get("execution_id", None)) else: context = LoggingContext(None) selected_services = None if "GCP_SERVICES" in os.environ: selected_services_string = os.environ.get("GCP_SERVICES", "") selected_services = selected_services_string.split( ",") if selected_services_string else [] #set default featureset if featureset not present in env variable for i, service in enumerate(selected_services): if "/" not in service: selected_services[i] = f"{service}/default" services = load_supported_services(context, selected_services) async with init_gcp_client_session( ) as gcp_session, init_dt_client_session() as dt_session: setup_start_time = time.time() token = await create_token(context, gcp_session) if token is None: context.log( "Cannot proceed without authorization token, stopping the execution" ) return if not isinstance(token, str): raise Exception( f"Failed to fetch access token, got non string value: {token}") context.log("Successfully obtained access token") if not project_id_owner: project_id_owner = get_project_id_from_environment() dynatrace_api_key = await fetch_dynatrace_api_key( gcp_session=gcp_session, project_id=project_id_owner, token=token) dynatrace_url = await fetch_dynatrace_url(gcp_session=gcp_session, project_id=project_id_owner, token=token) print_metric_ingest_input = \ "PRINT_METRIC_INGEST_INPUT" in os.environ and os.environ["PRINT_METRIC_INGEST_INPUT"].upper() == "TRUE" self_monitoring_enabled = os.environ.get('SELF_MONITORING_ENABLED', "False").upper() == "TRUE" context = MetricsContext( gcp_session=gcp_session, dt_session=dt_session, project_id_owner=project_id_owner, token=token, execution_time=datetime.utcnow(), execution_interval_seconds=60 * 1, dynatrace_api_key=dynatrace_api_key, dynatrace_url=dynatrace_url, print_metric_ingest_input=print_metric_ingest_input, self_monitoring_enabled=self_monitoring_enabled, scheduled_execution_id=context.scheduled_execution_id) if not projects_ids: projects_ids = await get_all_accessible_projects( context, gcp_session, token) setup_time = (time.time() - setup_start_time) context.setup_execution_time = { project_id: setup_time for project_id in projects_ids } context.start_processing_timestamp = time.time() process_project_metrics_tasks = [ process_project_metrics(context, project_id, services) for project_id in projects_ids ] await asyncio.gather(*process_project_metrics_tasks, return_exceptions=True) context.log( f"Fetched and pushed GCP data in {time.time() - context.start_processing_timestamp} s" ) log_self_monitoring_data(context) if context.self_monitoring_enabled: await push_self_monitoring(context) await gcp_session.close() await dt_session.close()
async def handle_event(event: Dict, event_context, project_id_owner: Optional[str]): if event_context is Dict: context = LoggingContext(event_context.get("execution_id", None)) else: context = LoggingContext(None) selected_services = None if "GCP_SERVICES" in os.environ: selected_services_string = os.environ.get("GCP_SERVICES", "") selected_services = selected_services_string.split(",") if selected_services_string else [] services = load_supported_services(context, selected_services) async with aiohttp.ClientSession() as session: setup_start_time = time.time() token = await create_token(context, session) if token is None: context.log("Cannot proceed without authorization token, stopping the execution") return if not isinstance(token, str): raise Exception(f"Failed to fetch access token, got non string value: {token}") context.log("Successfully obtained access token") if not project_id_owner: project_id_owner = get_project_id_from_environment() dynatrace_api_key = await fetch_dynatrace_api_key(session=session, project_id=project_id_owner, token=token) dynatrace_url = await fetch_dynatrace_url(session=session, project_id=project_id_owner, token=token) print_metric_ingest_input = \ "PRINT_METRIC_INGEST_INPUT" in os.environ and os.environ["PRINT_METRIC_INGEST_INPUT"].upper() == "TRUE" context = Context( session=session, project_id_owner=project_id_owner, token=token, execution_time=datetime.utcnow(), execution_interval_seconds=60 * 1, dynatrace_api_key=dynatrace_api_key, dynatrace_url=dynatrace_url, print_metric_ingest_input=print_metric_ingest_input, scheduled_execution_id=context.scheduled_execution_id ) projects_ids = await get_all_accessible_projects(context, session) context.setup_execution_time = (time.time() - setup_start_time) fetch_gcp_data_start_time = time.time() fetch_ingest_lines_tasks = [fetch_ingest_lines_task(context, project_id, services) for project_id in projects_ids] ingest_lines_per_project = await asyncio.gather(*fetch_ingest_lines_tasks, return_exceptions=True) ingest_lines = [ingest_line for sublist in ingest_lines_per_project for ingest_line in sublist] context.fetch_gcp_data_execution_time = time.time() - fetch_gcp_data_start_time context.log(f"Fetched GCP data in {context.fetch_gcp_data_execution_time} s") await push_ingest_lines(context, ingest_lines) await push_self_monitoring_time_series(context) await session.close()
async def handle_event(event: Dict, event_context, projects_ids: Optional[List[str]] = None, services: Optional[List[GCPService]] = None): if isinstance(event_context, Dict): # for k8s installation context = LoggingContext(event_context.get("execution_id", None)) else: context = LoggingContext(None) if not services: # load services for GCP Function services = load_supported_services(context) async with init_gcp_client_session( ) as gcp_session, init_dt_client_session() as dt_session: setup_start_time = time.time() token = await create_token(context, gcp_session) if token is None: context.log( "Cannot proceed without authorization token, stopping the execution" ) return if not isinstance(token, str): raise Exception( f"Failed to fetch access token, got non string value: {token}") context.log("Successfully obtained access token") project_id_owner = get_project_id_from_environment() dynatrace_api_key = await fetch_dynatrace_api_key( gcp_session=gcp_session, project_id=project_id_owner, token=token) dynatrace_url = await fetch_dynatrace_url(gcp_session=gcp_session, project_id=project_id_owner, token=token) check_version(logging_context=context) await check_dynatrace(logging_context=context, project_id=project_id_owner, dt_session=dt_session, dynatrace_url=dynatrace_url, dynatrace_access_key=dynatrace_api_key) query_interval_min = get_query_interval_minutes() print_metric_ingest_input = os.environ.get( "PRINT_METRIC_INGEST_INPUT", "FALSE").upper() in ["TRUE", "YES"] self_monitoring_enabled = os.environ.get( 'SELF_MONITORING_ENABLED', "FALSE").upper() in ["TRUE", "YES"] context = MetricsContext( gcp_session=gcp_session, dt_session=dt_session, project_id_owner=project_id_owner, token=token, execution_time=datetime.utcnow(), execution_interval_seconds=60 * query_interval_min, dynatrace_api_key=dynatrace_api_key, dynatrace_url=dynatrace_url, print_metric_ingest_input=print_metric_ingest_input, self_monitoring_enabled=self_monitoring_enabled, scheduled_execution_id=context.scheduled_execution_id) if not projects_ids: projects_ids = await get_all_accessible_projects( context, gcp_session, token) disabled_apis = {} disabled_projects = [] for project_id in projects_ids: await check_x_goog_user_project_header_permissions( context, project_id) disabled_apis = { project_id: await get_all_disabled_apis(context, project_id) } if 'monitoring.googleapis.com' in disabled_apis[project_id]: disabled_projects.append(project_id) if disabled_projects: context.log( f"monitoring.googleapis.com API disabled in the projects: " + ", ".join(disabled_projects) + ", that projects will not be monitored") for disabled_project in disabled_projects: projects_ids.remove(disabled_project) setup_time = (time.time() - setup_start_time) context.setup_execution_time = { project_id: setup_time for project_id in projects_ids } context.start_processing_timestamp = time.time() process_project_metrics_tasks = [ process_project_metrics(context, project_id, services, disabled_apis.get(project_id, set())) for project_id in projects_ids ] await asyncio.gather(*process_project_metrics_tasks, return_exceptions=True) context.log( f"Fetched and pushed GCP data in {time.time() - context.start_processing_timestamp} s" ) log_self_monitoring_data(context) if context.self_monitoring_enabled: await push_self_monitoring(context) await gcp_session.close() await dt_session.close()