from typing import List, Dict, Tuple
from urllib.error import HTTPError
from urllib.parse import urlparse
from urllib.request import Request
from lib.context import get_should_require_valid_certificate, get_int_environment_value, \
DynatraceConnectivity, LogsContext
from lib.logs.log_self_monitoring import LogSelfMonitoring, aggregate_self_monitoring_metrics, put_sfm_into_queue
from lib.logs.logs_processor import LogProcessingJob
ssl_context = ssl.create_default_context()
if not get_should_require_valid_certificate():
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
_TIMEOUT = get_int_environment_value("DYNATRACE_TIMEOUT_SECONDS", 30)
def send_logs(context: LogsContext, logs: List[LogProcessingJob], batch: str):
# pylint: disable=R0912
context.self_monitoring = aggregate_self_monitoring_metrics(LogSelfMonitoring(), [log.self_monitoring for log in logs])
context.self_monitoring.sending_time_start = time.perf_counter()
log_ingest_url = urlparse(context.dynatrace_url + "/api/v2/logs/ingest").geturl()
try:
encoded_body_bytes = batch.encode("UTF-8")
context.self_monitoring.all_requests += 1
status, reason, response = _perform_http_request(
method="POST",
url=log_ingest_url,
encoded_body_bytes=encoded_body_bytes,
from aiohttp import web
from lib.clientsession_provider import init_dt_client_session, init_gcp_client_session
from lib.configure_dynatrace import ConfigureDynatrace
from lib.context import LoggingContext, get_int_environment_value
from lib.credentials import create_token
from lib.fast_check import MetricsFastCheck, FastCheckResult, LogsFastCheck
from lib.instance_metadata import InstanceMetadataCheck, InstanceMetadata
from lib.logs.log_forwarder import run_logs
from main import async_dynatrace_gcp_extension
from operation_mode import OperationMode
OPERATION_MODE = OperationMode.from_environment_string(
os.environ.get("OPERATION_MODE", None)) or OperationMode.Metrics
HEALTH_CHECK_PORT = get_int_environment_value("HEALTH_CHECK_PORT", 8080)
loop = asyncio.get_event_loop()
async def scheduling_loop(project_ids: Optional[List[str]] = None):
while True:
loop.create_task(async_dynatrace_gcp_extension(project_ids))
await asyncio.sleep(60)
async def metrics_initial_check() -> Optional[FastCheckResult]:
async with init_gcp_client_session(
) as gcp_session, init_dt_client_session() as dt_session:
token = await create_token(logging_context, gcp_session)
if token:
import os
from datetime import timedelta
from lib.context import get_int_environment_value
PROCESSING_WORKER_PULL_REQUEST_MAX_MESSAGES = 10_000
PROCESSING_WORKERS = 4
MAX_SFM_MESSAGES_PROCESSED = 10_000
LOGS_SUBSCRIPTION_PROJECT = os.environ.get("LOGS_SUBSCRIPTION_PROJECT", None)
LOGS_SUBSCRIPTION_ID = os.environ.get('LOGS_SUBSCRIPTION_ID', None)
CONTENT_LENGTH_LIMIT = get_int_environment_value("DYNATRACE_LOG_INGEST_CONTENT_MAX_LENGTH", 8192)
ATTRIBUTE_VALUE_LENGTH_LIMIT = get_int_environment_value("DYNATRACE_LOG_INGEST_ATTRIBUTE_VALUE_MAX_LENGTH", 250)
EVENT_AGE_LIMIT_SECONDS = get_int_environment_value("DYNATRACE_LOG_INGEST_EVENT_MAX_AGE_SECONDS", int(timedelta(days=1).total_seconds()))
SENDING_WORKER_EXECUTION_PERIOD_SECONDS = get_int_environment_value("DYNATRACE_LOG_INGEST_SENDING_WORKER_EXECUTION_PERIOD", 60)
SFM_WORKER_EXECUTION_PERIOD_SECONDS = get_int_environment_value("DYNATRACE_LOG_INGEST_SFM_WORKER_EXECUTION_PERIOD", 60)
REQUEST_BODY_MAX_SIZE = get_int_environment_value("DYNATRACE_LOG_INGEST_REQUEST_MAX_SIZE", 1048576)
REQUEST_MAX_EVENTS = get_int_environment_value("DYNATRACE_LOG_INGEST_REQUEST_MAX_EVENTS", 5000)
BATCH_MAX_MESSAGES = get_int_environment_value("DYNATRACE_LOG_INGEST_BATCH_MAX_MESSAGES", 10_000)