async def extensions_fetch(gcp_session: ClientSession, dt_session: ClientSession, token: str) -> Optional[ExtensionsFetchResult]: extension_fetcher_result = await ExtensionsFetcher( dt_session=dt_session, dynatrace_url=await fetch_dynatrace_url(gcp_session, get_project_id_from_environment(), token), dynatrace_access_key=await fetch_dynatrace_api_key(gcp_session, get_project_id_from_environment(), token), logging_context=logging_context).execute() if extension_fetcher_result.services: feature_sets_names = [ f"{service.name}/{service.feature_set}" for service in extension_fetcher_result.services ] logging_context.log( f'Monitoring enabled for the following fetched extensions feature sets: {feature_sets_names}' ) return extension_fetcher_result else: logging_context.log( "Monitoring disabled. Check configured extensions.") return None
async def execute(self) -> FastCheckResult: _check_configuration_flags(self.logging_context, METRICS_CONFIGURATION_FLAGS) project_list = await get_all_accessible_projects( self.logging_context, self.gcp_session, self.token) ready_to_monitor = [] for project_id in project_list: tasks = [self._check_services(project_id)] results = await asyncio.gather(*tasks, return_exceptions=True) if all(result is not None for result in results): ready_to_monitor.append(project_id) dynatrace_url = await fetch_dynatrace_url(self.gcp_session, project_id, self.token) dynatrace_access_key = await fetch_dynatrace_api_key( self.gcp_session, project_id, self.token) await check_dynatrace(logging_context=self.logging_context, project_id=get_project_id_from_environment(), dt_session=self.dt_session, dynatrace_url=dynatrace_url, dynatrace_access_key=dynatrace_access_key) return FastCheckResult(projects=ready_to_monitor)
def create_logs_context(sfm_queue: Queue): dynatrace_api_key = get_dynatrace_api_key_from_env() dynatrace_url = get_dynatrace_log_ingest_url_from_env() project_id_owner = get_project_id_from_environment() return LogsContext(project_id_owner=project_id_owner, dynatrace_api_key=dynatrace_api_key, dynatrace_url=dynatrace_url, scheduled_execution_id=str(int(time.time()))[-8:], sfm_queue=sfm_queue)
async def import_self_monitoring_dashboards(metadata: InstanceMetadata): if metadata: async with init_gcp_client_session() as gcp_session: token = await create_token(logging_context, gcp_session) if token: sfm_dashboards_context = SfmDashboardsContext( project_id_owner=get_project_id_from_environment(), token=token, gcp_session=gcp_session, operation_mode=OPERATION_MODE, scheduled_execution_id=None) await import_self_monitoring_dashboard( context=sfm_dashboards_context)
async def handle_event(event: Dict, event_context, project_id_owner: Optional[str]): if event_context is Dict: context = LoggingContext(event_context.get("execution_id", None)) else: context = LoggingContext(None) selected_services = None if "GCP_SERVICES" in os.environ: selected_services_string = os.environ.get("GCP_SERVICES", "") selected_services = selected_services_string.split(",") if selected_services_string else [] services = load_supported_services(context, selected_services) async with aiohttp.ClientSession() as session: setup_start_time = time.time() token = await create_token(context, session) if token is None: context.log("Cannot proceed without authorization token, stopping the execution") return if not isinstance(token, str): raise Exception(f"Failed to fetch access token, got non string value: {token}") context.log("Successfully obtained access token") if not project_id_owner: project_id_owner = get_project_id_from_environment() dynatrace_api_key = await fetch_dynatrace_api_key(session=session, project_id=project_id_owner, token=token) dynatrace_url = await fetch_dynatrace_url(session=session, project_id=project_id_owner, token=token) print_metric_ingest_input = \ "PRINT_METRIC_INGEST_INPUT" in os.environ and os.environ["PRINT_METRIC_INGEST_INPUT"].upper() == "TRUE" context = Context( session=session, project_id_owner=project_id_owner, token=token, execution_time=datetime.utcnow(), execution_interval_seconds=60 * 1, dynatrace_api_key=dynatrace_api_key, dynatrace_url=dynatrace_url, print_metric_ingest_input=print_metric_ingest_input, scheduled_execution_id=context.scheduled_execution_id ) projects_ids = await get_all_accessible_projects(context, session) context.setup_execution_time = (time.time() - setup_start_time) fetch_gcp_data_start_time = time.time() fetch_ingest_lines_tasks = [fetch_ingest_lines_task(context, project_id, services) for project_id in projects_ids] ingest_lines_per_project = await asyncio.gather(*fetch_ingest_lines_tasks, return_exceptions=True) ingest_lines = [ingest_line for sublist in ingest_lines_per_project for ingest_line in sublist] context.fetch_gcp_data_execution_time = time.time() - fetch_gcp_data_start_time context.log(f"Fetched GCP data in {context.fetch_gcp_data_execution_time} s") await push_ingest_lines(context, ingest_lines) await push_self_monitoring_time_series(context) await session.close()
async def handle_event(event: Dict, event_context, project_id_owner: Optional[str], projects_ids: Optional[List[str]] = None): if isinstance(event_context, Dict): context = LoggingContext(event_context.get("execution_id", None)) else: context = LoggingContext(None) selected_services = None if "GCP_SERVICES" in os.environ: selected_services_string = os.environ.get("GCP_SERVICES", "") selected_services = selected_services_string.split( ",") if selected_services_string else [] #set default featureset if featureset not present in env variable for i, service in enumerate(selected_services): if "/" not in service: selected_services[i] = f"{service}/default" services = load_supported_services(context, selected_services) async with init_gcp_client_session( ) as gcp_session, init_dt_client_session() as dt_session: setup_start_time = time.time() token = await create_token(context, gcp_session) if token is None: context.log( "Cannot proceed without authorization token, stopping the execution" ) return if not isinstance(token, str): raise Exception( f"Failed to fetch access token, got non string value: {token}") context.log("Successfully obtained access token") if not project_id_owner: project_id_owner = get_project_id_from_environment() dynatrace_api_key = await fetch_dynatrace_api_key( gcp_session=gcp_session, project_id=project_id_owner, token=token) dynatrace_url = await fetch_dynatrace_url(gcp_session=gcp_session, project_id=project_id_owner, token=token) print_metric_ingest_input = \ "PRINT_METRIC_INGEST_INPUT" in os.environ and os.environ["PRINT_METRIC_INGEST_INPUT"].upper() == "TRUE" self_monitoring_enabled = os.environ.get('SELF_MONITORING_ENABLED', "False").upper() == "TRUE" context = MetricsContext( gcp_session=gcp_session, dt_session=dt_session, project_id_owner=project_id_owner, token=token, execution_time=datetime.utcnow(), execution_interval_seconds=60 * 1, dynatrace_api_key=dynatrace_api_key, dynatrace_url=dynatrace_url, print_metric_ingest_input=print_metric_ingest_input, self_monitoring_enabled=self_monitoring_enabled, scheduled_execution_id=context.scheduled_execution_id) if not projects_ids: projects_ids = await get_all_accessible_projects( context, gcp_session, token) setup_time = (time.time() - setup_start_time) context.setup_execution_time = { project_id: setup_time for project_id in projects_ids } context.start_processing_timestamp = time.time() process_project_metrics_tasks = [ process_project_metrics(context, project_id, services) for project_id in projects_ids ] await asyncio.gather(*process_project_metrics_tasks, return_exceptions=True) context.log( f"Fetched and pushed GCP data in {time.time() - context.start_processing_timestamp} s" ) log_self_monitoring_data(context) if context.self_monitoring_enabled: await push_self_monitoring(context) await gcp_session.close() await dt_session.close()
async def handle_event(event: Dict, event_context, projects_ids: Optional[List[str]] = None, services: Optional[List[GCPService]] = None): if isinstance(event_context, Dict): # for k8s installation context = LoggingContext(event_context.get("execution_id", None)) else: context = LoggingContext(None) if not services: # load services for GCP Function services = load_supported_services(context) async with init_gcp_client_session( ) as gcp_session, init_dt_client_session() as dt_session: setup_start_time = time.time() token = await create_token(context, gcp_session) if token is None: context.log( "Cannot proceed without authorization token, stopping the execution" ) return if not isinstance(token, str): raise Exception( f"Failed to fetch access token, got non string value: {token}") context.log("Successfully obtained access token") project_id_owner = get_project_id_from_environment() dynatrace_api_key = await fetch_dynatrace_api_key( gcp_session=gcp_session, project_id=project_id_owner, token=token) dynatrace_url = await fetch_dynatrace_url(gcp_session=gcp_session, project_id=project_id_owner, token=token) check_version(logging_context=context) await check_dynatrace(logging_context=context, project_id=project_id_owner, dt_session=dt_session, dynatrace_url=dynatrace_url, dynatrace_access_key=dynatrace_api_key) query_interval_min = get_query_interval_minutes() print_metric_ingest_input = os.environ.get( "PRINT_METRIC_INGEST_INPUT", "FALSE").upper() in ["TRUE", "YES"] self_monitoring_enabled = os.environ.get( 'SELF_MONITORING_ENABLED', "FALSE").upper() in ["TRUE", "YES"] context = MetricsContext( gcp_session=gcp_session, dt_session=dt_session, project_id_owner=project_id_owner, token=token, execution_time=datetime.utcnow(), execution_interval_seconds=60 * query_interval_min, dynatrace_api_key=dynatrace_api_key, dynatrace_url=dynatrace_url, print_metric_ingest_input=print_metric_ingest_input, self_monitoring_enabled=self_monitoring_enabled, scheduled_execution_id=context.scheduled_execution_id) if not projects_ids: projects_ids = await get_all_accessible_projects( context, gcp_session, token) disabled_apis = {} disabled_projects = [] for project_id in projects_ids: await check_x_goog_user_project_header_permissions( context, project_id) disabled_apis = { project_id: await get_all_disabled_apis(context, project_id) } if 'monitoring.googleapis.com' in disabled_apis[project_id]: disabled_projects.append(project_id) if disabled_projects: context.log( f"monitoring.googleapis.com API disabled in the projects: " + ", ".join(disabled_projects) + ", that projects will not be monitored") for disabled_project in disabled_projects: projects_ids.remove(disabled_project) setup_time = (time.time() - setup_start_time) context.setup_execution_time = { project_id: setup_time for project_id in projects_ids } context.start_processing_timestamp = time.time() process_project_metrics_tasks = [ process_project_metrics(context, project_id, services, disabled_apis.get(project_id, set())) for project_id in projects_ids ] await asyncio.gather(*process_project_metrics_tasks, return_exceptions=True) context.log( f"Fetched and pushed GCP data in {time.time() - context.start_processing_timestamp} s" ) log_self_monitoring_data(context) if context.self_monitoring_enabled: await push_self_monitoring(context) await gcp_session.close() await dt_session.close()