from typing import List, Dict, Tuple from urllib.error import HTTPError from urllib.parse import urlparse from urllib.request import Request from lib.context import get_should_require_valid_certificate, get_int_environment_value, \ DynatraceConnectivity, LogsContext from lib.logs.log_self_monitoring import LogSelfMonitoring, aggregate_self_monitoring_metrics, put_sfm_into_queue from lib.logs.logs_processor import LogProcessingJob ssl_context = ssl.create_default_context() if not get_should_require_valid_certificate(): ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE _TIMEOUT = get_int_environment_value("DYNATRACE_TIMEOUT_SECONDS", 30) def send_logs(context: LogsContext, logs: List[LogProcessingJob], batch: str): # pylint: disable=R0912 context.self_monitoring = aggregate_self_monitoring_metrics(LogSelfMonitoring(), [log.self_monitoring for log in logs]) context.self_monitoring.sending_time_start = time.perf_counter() log_ingest_url = urlparse(context.dynatrace_url + "/api/v2/logs/ingest").geturl() try: encoded_body_bytes = batch.encode("UTF-8") context.self_monitoring.all_requests += 1 status, reason, response = _perform_http_request( method="POST", url=log_ingest_url, encoded_body_bytes=encoded_body_bytes,
from aiohttp import web from lib.clientsession_provider import init_dt_client_session, init_gcp_client_session from lib.configure_dynatrace import ConfigureDynatrace from lib.context import LoggingContext, get_int_environment_value from lib.credentials import create_token from lib.fast_check import MetricsFastCheck, FastCheckResult, LogsFastCheck from lib.instance_metadata import InstanceMetadataCheck, InstanceMetadata from lib.logs.log_forwarder import run_logs from main import async_dynatrace_gcp_extension from operation_mode import OperationMode OPERATION_MODE = OperationMode.from_environment_string( os.environ.get("OPERATION_MODE", None)) or OperationMode.Metrics HEALTH_CHECK_PORT = get_int_environment_value("HEALTH_CHECK_PORT", 8080) loop = asyncio.get_event_loop() async def scheduling_loop(project_ids: Optional[List[str]] = None): while True: loop.create_task(async_dynatrace_gcp_extension(project_ids)) await asyncio.sleep(60) async def metrics_initial_check() -> Optional[FastCheckResult]: async with init_gcp_client_session( ) as gcp_session, init_dt_client_session() as dt_session: token = await create_token(logging_context, gcp_session) if token:
import os from datetime import timedelta from lib.context import get_int_environment_value PROCESSING_WORKER_PULL_REQUEST_MAX_MESSAGES = 10_000 PROCESSING_WORKERS = 4 MAX_SFM_MESSAGES_PROCESSED = 10_000 LOGS_SUBSCRIPTION_PROJECT = os.environ.get("LOGS_SUBSCRIPTION_PROJECT", None) LOGS_SUBSCRIPTION_ID = os.environ.get('LOGS_SUBSCRIPTION_ID', None) CONTENT_LENGTH_LIMIT = get_int_environment_value("DYNATRACE_LOG_INGEST_CONTENT_MAX_LENGTH", 8192) ATTRIBUTE_VALUE_LENGTH_LIMIT = get_int_environment_value("DYNATRACE_LOG_INGEST_ATTRIBUTE_VALUE_MAX_LENGTH", 250) EVENT_AGE_LIMIT_SECONDS = get_int_environment_value("DYNATRACE_LOG_INGEST_EVENT_MAX_AGE_SECONDS", int(timedelta(days=1).total_seconds())) SENDING_WORKER_EXECUTION_PERIOD_SECONDS = get_int_environment_value("DYNATRACE_LOG_INGEST_SENDING_WORKER_EXECUTION_PERIOD", 60) SFM_WORKER_EXECUTION_PERIOD_SECONDS = get_int_environment_value("DYNATRACE_LOG_INGEST_SFM_WORKER_EXECUTION_PERIOD", 60) REQUEST_BODY_MAX_SIZE = get_int_environment_value("DYNATRACE_LOG_INGEST_REQUEST_MAX_SIZE", 1048576) REQUEST_MAX_EVENTS = get_int_environment_value("DYNATRACE_LOG_INGEST_REQUEST_MAX_EVENTS", 5000) BATCH_MAX_MESSAGES = get_int_environment_value("DYNATRACE_LOG_INGEST_BATCH_MAX_MESSAGES", 10_000)