def __init__(self, plugin_context=None, config=None): self.hooks = { 'before:invocation': self.before_invocation, 'after:invocation': self.after_invocation } self.plugin_context = plugin_context self.logger = logging.getLogger('STDOUT') if isinstance(config, LogConfig): self.config = config else: self.config = LogConfig() with LogPlugin.lock: if (not LogPlugin.wrapped) and (not ConfigProvider.get( config_names.THUNDRA_LOG_CONSOLE_DISABLE)): if PY37 or PY38: wrapt.wrap_function_wrapper('builtins', 'print', self._wrapper) else: sys.stdout = StreamToLogger(self.logger, sys.stdout) LogPlugin.wrapped = True if not ConfigProvider.get(config_names.THUNDRA_LOG_CONSOLE_DISABLE): handler = ThundraLogHandler() has_thundra_log_handler = False for log_handlers in self.logger.handlers: if isinstance(log_handlers, ThundraLogHandler): has_thundra_log_handler = True if not has_thundra_log_handler: self.logger.addHandler(handler) self.logger.setLevel(logging.INFO) handler.setLevel(logging.INFO) self.logger.propagate = False
def test_config_correct_default_value(): ConfigProvider.__init__() assert ConfigProvider.get('thundra.agent.debug.enable') is False assert ConfigProvider.get('thundra.agent.debug.enable', True) is True assert ConfigProvider.get( 'thundra.agent.lambda.debugger.logs.enable') is False
def get_collector_url(): use_local = ConfigProvider.get(config_names.THUNDRA_REPORT_REST_LOCAL) if use_local: return 'http://' + constants.LOCAL_COLLECTOR_ENDPOINT + '/v1' return ConfigProvider.get( config_names.THUNDRA_REPORT_REST_BASEURL, 'https://' + utils.get_nearest_collector() + '/v1')
def send_reports(self, reports, **opts): if not self.api_key: debug_logger("API key not set, not sending report to Thundra.") return [] headers = { 'Content-Type': 'application/json', 'Authorization': 'ApiKey ' + self.api_key } test_run_event = opts.get("test_run_event", False) rest_composite_data_enabled = ConfigProvider.get( config_names.THUNDRA_REPORT_REST_COMPOSITE_ENABLE, True) if not test_run_event: path = constants.COMPOSITE_DATA_PATH if rest_composite_data_enabled else constants.PATH else: path = constants.PATH base_url = self.get_collector_url() request_url = base_url + path if ConfigProvider.get(config_names.THUNDRA_REPORT_CLOUDWATCH_ENABLE): if ConfigProvider.get( config_names.THUNDRA_REPORT_CLOUDWATCH_COMPOSITE_ENABLE, True): if not test_run_event: reports_json = self.prepare_composite_report_json(reports) else: reports_json = self.prepare_report_json(reports) for report in reports_json: print(report) else: for report in reports: try: print(to_json(report, separators=(',', ':'))) except TypeError: logger.error(( "Couldn't dump report with type {} to json string, " "probably it contains a byte array").format( report.get('type'))) return [] if not test_run_event and rest_composite_data_enabled: reports_json = self.prepare_composite_report_json(reports) else: reports_json = self.prepare_report_json(reports) responses = [] if len(reports_json) > 0: _futures = [ self.pool.submit(self.send_batch, (request_url, headers, data)) for data in reports_json ] responses = [ future.result() for future in futures.as_completed(_futures) ] if ConfigProvider.get(config_names.THUNDRA_DEBUG_ENABLE): debug_logger("Thundra API responses: " + str(responses)) return responses
def before_call(self, scope, wrapped, instance, args, kwargs, response, exception): scope.span.domain_name = constants.DomainNames['DB'] scope.span.class_name = constants.ClassNames['DYNAMODB'] operation_name, request_data = args operation_type = get_operation_type(scope.span.class_name, operation_name) self.request_data = request_data.copy() self.endpoint = instance._endpoint.host.split('/')[-1] tags = { constants.SpanTags['OPERATION_TYPE']: operation_type, constants.DBTags['DB_INSTANCE']: self.endpoint, constants.DBTags['DB_TYPE']: constants.DBTypes['DYNAMODB'], constants.AwsDynamoTags['TABLE_NAME']: str(self.request_data['TableName']) if 'TableName' in self.request_data else None, constants.DBTags['DB_STATEMENT_TYPE']: operation_type, constants.AwsSDKTags['REQUEST_NAME']: operation_name, } scope.span.tags = tags # Check if Key and Item fields have any byte field and convert to string if 'Key' in self.request_data: self.escape_byte_fields(self.request_data['Key']) if 'Item' in self.request_data: self.escape_byte_fields(self.request_data['Item']) # DB statement tags should not be set on span if masked if not ConfigProvider.get( config_names. THUNDRA_TRACE_INTEGRATIONS_AWS_DYNAMODB_STATEMENT_MASK): self.OPERATION.get(operation_name, dummy_func)(scope) scope.span.set_tag(constants.SpanTags['TOPOLOGY_VERTEX'], True) if ConfigProvider.get( config_names. THUNDRA_TRACE_INTEGRATIONS_AWS_DYNAMODB_TRACEINJECTION_ENABLE): if operation_name == 'PutItem': self.inject_trace_link_on_put(scope.span, request_data, instance) if operation_name == 'UpdateItem': self.inject_trace_link_on_update(scope.span, request_data, instance) if operation_name == 'DeleteItem': self.inject_trace_link_on_delete(request_data)
def get_report_batches(self, reports): batch_size = ConfigProvider.get( config_names.THUNDRA_REPORT_REST_COMPOSITE_BATCH_SIZE) if ConfigProvider.get(config_names.THUNDRA_REPORT_CLOUDWATCH_ENABLE): batch_size = ConfigProvider.get( config_names.THUNDRA_REPORT_CLOUDWATCH_COMPOSITE_BATCH_SIZE) batches = [ reports[i:i + batch_size] for i in range(0, len(reports), batch_size) ] return batches
def patch(): if not ConfigProvider.get( config_names.THUNDRA_TRACE_INTEGRATIONS_AWS_DISABLE): wrapt.wrap_function_wrapper('botocore.client', 'BaseClient._make_api_call', _wrapper) if not ConfigProvider.get( config_names.THUNDRA_TRACE_INTEGRATIONS_HTTP_DISABLE): try: wrapt.wrap_function_wrapper('botocore.vendored.requests', 'Session.send', request_wrapper) except Exception: # Vendored version of requests is removed from botocore pass
def test_config_environment_variable_override_options(monkeypatch, config_options): monkeypatch.setitem(os.environ, 'THUNDRA_AGENT_MY_KEY', 'my_value_from_env') monkeypatch.setitem(os.environ, 'THUNDRA_AGENT_LAMBDA_MY_KEY2', 'my_value_from_env2') ConfigProvider.__init__(options=config_options) assert ConfigProvider.get('thundra.agent.my.key') == 'my_value_from_env' assert ConfigProvider.get( 'thundra.agent.lambda.my.key2') == 'my_value_from_env2' assert ConfigProvider.get('thundra.agent.my.key2') == 'my_value_from_env2'
def get_application_info_from_config(): return { 'applicationId': ConfigProvider.get(config_names.THUNDRA_APPLICATION_ID), 'applicationInstanceId': ConfigProvider.get(config_names.THUNDRA_APPLICATION_INSTANCE_ID), 'applicationDomainName': ConfigProvider.get(config_names.THUNDRA_APPLICATION_DOMAIN_NAME), 'applicationClassName': ConfigProvider.get(config_names.THUNDRA_APPLICATION_CLASS_NAME), 'applicationName': ConfigProvider.get(config_names.THUNDRA_APPLICATION_NAME), 'applicationVersion': ConfigProvider.get(config_names.THUNDRA_APPLICATION_VERSION, ''), 'applicationStage': ConfigProvider.get(config_names.THUNDRA_APPLICATION_STAGE, ''), 'applicationRegion': ConfigProvider.get(config_names.THUNDRA_APPLICATION_REGION, ''), 'applicationRuntime': ApplicationInfoProvider.APPLICATION_RUNTIME, 'applicationRuntimeVersion': ApplicationInfoProvider.APPLICATION_RUNTIME_VERSION, 'applicationTags': ApplicationInfoProvider.parse_application_tags() }
def test_config_from_environment_variable(monkeypatch): monkeypatch.setitem(os.environ, 'THUNDRA_AGENT_TEST_KEY', 'test_value') monkeypatch.setitem(os.environ, 'THUNDRA_AGENT_LAMBDA_TEST_KEY2', 'test_value2') ConfigProvider.__init__() monkeypatch.delitem(os.environ, 'THUNDRA_AGENT_TEST_KEY') monkeypatch.delitem(os.environ, 'THUNDRA_AGENT_LAMBDA_TEST_KEY2') assert ConfigProvider.get('thundra.agent.test.key') == 'test_value' assert ConfigProvider.get( 'thundra.agent.lambda.test.key2') == 'test_value2' assert ConfigProvider.get('THUNDRA_AGENT_TEST_KEY') is None assert ConfigProvider.get('THUNDRA_AGENT_LAMBDA_TEST_KEY2') is None
def get_operation_name(self, wrapped, instance, args, kwargs): prepared_request = args[0] url_dict = utils.parse_http_url( prepared_request.url, ConfigProvider.get( config_names.THUNDRA_TRACE_INTEGRATIONS_HTTP_URL_DEPTH)) return url_dict.get('operation_name')
def before_call(self, scope, wrapped, instance, args, kwargs, response, exception): operation_name, request_data = args self.request_data = request_data self.response = response self.message = request_data.get('Message', '') scope.span.domain_name = constants.DomainNames['MESSAGING'] scope.span.class_name = constants.ClassNames['SNS'] tags = { constants.AwsSDKTags['REQUEST_NAME']: operation_name, constants.SpanTags['OPERATION_TYPE']: get_operation_type(scope.span.class_name, operation_name), constants.AwsSNSTags['TOPIC_NAME']: self.topicName } scope.span.tags = tags scope.span.set_tag(constants.SpanTags['TOPOLOGY_VERTEX'], True) if not ConfigProvider.get( config_names.THUNDRA_TRACE_INTEGRATIONS_AWS_SNS_MESSAGE_MASK): scope.span.set_tag(constants.AwsSNSTags['MESSAGE'], self.message)
def before_call(self, scope, cursor, connection, _args, _kwargs, response, exception): span = scope.span span.domain_name = constants.DomainNames['DB'] span.class_name = constants.ClassNames['POSTGRESQL'] dsn = parse_dsn(connection.dsn) query = '' operation = '' try: query = _args[0] if len(query) > 0: operation = query.split()[0].strip("\"").lower() except Exception: pass tags = { constants.SpanTags['OPERATION_TYPE']: PostgreIntegration._OPERATION_TO_TYPE.get(operation, ''), constants.SpanTags['DB_INSTANCE']: dsn.get('dbname', ''), constants.SpanTags['DB_HOST']: dsn.get('host', ''), constants.SpanTags['DB_TYPE']: "postgresql", constants.SpanTags['DB_STATEMENT_TYPE']: operation.upper(), constants.SpanTags['TRIGGER_CLASS_NAME']: "API", constants.SpanTags['TOPOLOGY_VERTEX']: True } if not ConfigProvider.get(config_names.THUNDRA_TRACE_INTEGRATIONS_RDB_STATEMENT_MASK): tags[constants.DBTags['DB_STATEMENT']] = query span.tags = tags
def pytest_sessionstart(session): """ Check thundra has been activated. If it has been, then start session. Args: session (pytest.Session): Pytest session class """ try: from thundra.config.config_provider import ConfigProvider from thundra.config import config_names if (session.config.getoption("thundra_disable") or session.config.getini("thundra_disable") or check_thundra_test_disabled() or ConfigProvider.get( config_names.THUNDRA_TEST_DISABLE, False)): import thundra already_configured = True if ConfigProvider.configs else False thundra._set_thundra_for_test_env(already_configured) else: api_key_env, project_id_env = check_thundra_from_env() api_key_conf, project_id_conf = check_thundra_from_conf() check_thundra_project_id = project_id_env or project_id_conf check_thundra_api_key = api_key_env or api_key_conf if check_thundra_project_id and check_thundra_api_key and not PytestHelper.check_pytest_collect_only( ): PytestHelper.set_pytest_started() patch() PytestHelper.session_setup(executor=foresight_executor) else: print( "[THUNDRA] Please make sure setting THUNDRA_APIKEY and THUNDRA_AGENT_TEST_PROJECT_ID as environment variables and pytest --collect-only args ain't activated!" ) except Exception as e: logger.error("Pytest pytest_sessionstart error: {}".format(e)) pass
def before_call(self, scope, wrapped, instance, args, kwargs, response, exception): scope.span.class_name = constants.ClassNames['ELASTICSEARCH'] scope.span.domain_name = constants.DomainNames['DB'] operation_name = self.get_operation_name(wrapped, instance, args, kwargs) hosts = self.get_hosts(instance) http_method, es_path = args es_body = kwargs.get('body', {}) es_params = kwargs.get('params', {}) tags = { constants.ESTags['ES_HOSTS']: hosts, constants.ESTags['ES_URI']: es_path, constants.ESTags['ES_NORMALIZED_URI']: operation_name, constants.ESTags['ES_METHOD']: http_method, constants.ESTags['ES_PARAMS']: es_params, constants.DBTags['DB_TYPE']: 'elasticsearch', constants.SpanTags['OPERATION_TYPE']: http_method, constants.SpanTags['TOPOLOGY_VERTEX']: True, } if not ConfigProvider.get( config_names.THUNDRA_TRACE_INTEGRATIONS_ELASTICSEARCH_BODY_MASK ): tags[constants.ESTags['ES_BODY']] = es_body scope.span.tags = tags
def finish_trace(execution_context): root_span = execution_context.root_span """ Getting request data into start trace occurs unexpected bugs into application process. After whole process finish for request data into appşication flow, getting request data and set the request_body for root_span as tag. """ try: _request = execution_context.platform_data['request'] req_data = None if _request and not ConfigProvider.get(config_names.THUNDRA_TRACE_REQUEST_SKIP, False): cl = _request.content_length if cl == None or cl <= constants.THUNDRA_MAX_STREAM_REQUEST_BODY: req_data = _request.get_data() else: req_data = None root_span.set_tag(constants.HttpTags['BODY'], req_data) except Exception as e: Logger.error("Error occured whilst setting request body to root span tag: {}".format(e)) pass if execution_context.response: status_code = get_response_status(execution_context) if status_code: root_span.set_tag(constants.HttpTags['HTTP_STATUS'], status_code) if execution_context.trigger_operation_name and execution_context.response and hasattr( execution_context.response, 'headers'): execution_context.response.headers[ constants.TRIGGER_RESOURCE_NAME_TAG] = execution_context.trigger_operation_name web_wrapper_utils.finish_trace(execution_context)
def handle_request(req): """Manipulate request for thundra tracer. If request has "more_body" field, then add it to current request body in execution context request body. Args: req (Request): Gathered from asgi receive function """ if req["type"] == "http.request": try: if "body" in req: req_body = req.get("body", b"") if execution_context.platform_data["request"]["body"]: execution_context.platform_data["request"][ "body"] += req_body else: execution_context.platform_data["request"][ "body"] = req_body if not ConfigProvider.get( config_names.THUNDRA_TRACE_REQUEST_SKIP, True): execution_context.root_span.set_tag( constants.HttpTags['BODY'], execution_context.platform_data["request"] ["body"]) except Exception as e: logger.error( "Error during getting req body in fast api: {}".format( e))
def patch(): if not ConfigProvider.get(config_names.THUNDRA_TRACE_INTEGRATIONS_RDB_DISABLE): patch_extensions() wrapt.wrap_function_wrapper( 'psycopg2', 'connect', _wrapper)
def patch(): if not ConfigProvider.get(config_names.THUNDRA_TRACE_INTEGRATIONS_ES_DISABLE): wrapt.wrap_function_wrapper( 'elasticsearch', 'transport.Transport.perform_request', _wrapper )
def patch(): if not ConfigProvider.get(config_names.THUNDRA_TRACE_INTEGRATIONS_HTTP_DISABLE): wrapt.wrap_function_wrapper( 'requests', 'Session.send', _wrapper )
def before_call(self, scope, wrapped, instance, args, kwargs, response, exception): operation_name, request_data = args self.request_data = request_data self.queueName = str(self.getQueueName(self.request_data)) scope.span.domain_name = constants.DomainNames['MESSAGING'] scope.span.class_name = constants.ClassNames['SQS'] tags = { constants.AwsSQSTags['QUEUE_NAME']: self.queueName, constants.SpanTags['OPERATION_TYPE']: get_operation_type(scope.span.class_name, operation_name), constants.AwsSDKTags['REQUEST_NAME']: operation_name, } scope.span.tags = tags scope.span.set_tag(constants.SpanTags['TOPOLOGY_VERTEX'], True) if not ConfigProvider.get( config_names.THUNDRA_TRACE_INTEGRATIONS_AWS_SQS_MESSAGE_MASK): if operation_name == "SendMessage": message = request_data.get("MessageBody", "") scope.span.set_tag(constants.AwsSQSTags['MESSAGE'], message) elif operation_name == "SendMessageBatch": entries = request_data.get('Entries', None) messages = [] if entries: for entry in entries: messages.append(entry.get("MessageBody", "")) scope.span.set_tag(constants.AwsSQSTags['MESSAGES'], messages)
def before_call(self, scope, cursor, connection, _args, _kwargs, response, exception): span = scope.span span.domain_name = constants.DomainNames['DB'] span.class_name = constants.ClassNames['MYSQL'] query = '' operation = '' try: query = _args[0] if len(query) > 0: operation = query.split()[0].strip("\"").lower() except Exception: pass tags = { constants.SpanTags['OPERATION_TYPE']: MysqlIntegration._OPERATION_TO_TYPE.get(operation, ''), constants.SpanTags['DB_INSTANCE']: connection._database, constants.SpanTags['DB_HOST']: connection._host, constants.SpanTags['DB_TYPE']: "mysql", constants.SpanTags['DB_STATEMENT_TYPE']: operation.upper(), constants.SpanTags['TOPOLOGY_VERTEX']: True, } if not ConfigProvider.get(config_names.THUNDRA_TRACE_INTEGRATIONS_RDB_STATEMENT_MASK): tags[constants.DBTags['DB_STATEMENT']] = query span.tags = tags
def before_call(self, scope, wrapped, instance, args, kwargs, response, exception): connection_kwargs = instance.connection_pool.connection_kwargs host = connection_kwargs.get('host', '') port = connection_kwargs.get('port', '6379') command_type = wrapped.__name__.upper() or "" operation_type = constants.RedisCommandTypes.get(command_type, '') command = '{} {}'.format(command_type, ' '.join([str(arg) for arg in args])) scope.span.domain_name = constants.DomainNames['CACHE'] scope.span.class_name = constants.ClassNames['REDIS'] tags = { constants.SpanTags['OPERATION_TYPE']: operation_type, constants.DBTags['DB_INSTANCE']: host, constants.DBTags['DB_STATEMENT_TYPE']: operation_type, constants.DBTags['DB_TYPE']: 'redis', constants.RedisTags['REDIS_HOST']: host, constants.RedisTags['REDIS_PORT']: port, constants.RedisTags['REDIS_COMMAND_TYPE']: command_type, constants.SpanTags['TOPOLOGY_VERTEX']: True, } if not ConfigProvider.get( config_names.THUNDRA_TRACE_INTEGRATIONS_REDIS_COMMAND_MASK): tags[constants.DBTags['DB_STATEMENT']] = command tags[constants.RedisTags['REDIS_COMMAND']] = command scope.span.tags = tags
async def on_request_end(session, trace_config_ctx, params): if not hasattr(trace_config_ctx, "scope"): return scope = trace_config_ctx.scope response = params.response if response is not None: status_code = response.status scope.span.set_tag(constants.HttpTags['HTTP_STATUS'], status_code) if response.headers and (response.headers.get("x-amz-apigw-id") or response.headers.get("apigw-requestid")): scope.span.class_name = constants.ClassNames['APIGATEWAY'] if response.headers and response.headers.get( "x-thundra-resource-name"): resource_name = response.headers.get("x-thundra-resource-name") scope.span.operation_name = resource_name if (status_code and ConfigProvider.get( config_names. THUNDRA_TRACE_INTEGRATIONS_HTTP_ERROR_STATUS_CODE_MIN) <= status_code): scope.span.set_tag('error.kind', "HttpError") scope.span.set_tag('error', True) scope.span.set_tag('error.message', response.reason) try: scope.span.finish() except Exception as e: logger.error(e) scope.close()
def __init__(self, *, plugin_executor=None, api_key=None, disable_trace=False, disable_metric=True, disable_log=True, opts=None, **ignored): super(TestWrapper, self).__init__(api_key, disable_trace, disable_metric, disable_log, opts) ExecutionContextManager.set_provider( TracingExecutionContextProvider()) #TODO self.application_info_provider = GlobalApplicationInfoProvider() self._set_application_info("Foresight", "TestSuite", "TestSuite") self.plugin_context = PluginContext( application_info=self.application_info_provider. get_application_info(), request_count=0, executor=plugin_executor, api_key=self.api_key) max_test_log_count = ConfigProvider.get( config_names.THUNDRA_TEST_LOG_COUNT_MAX) self.config.log_config = LogConfig( sampler=MaxCountAwareSampler(max_test_log_count)) self.plugins = wrapper_utils.initialize_plugins(self.plugin_context, disable_trace, disable_metric, disable_log, config=self.config) TestWrapper.__instance = self
def before_call(self, scope, conn, statement): db_config = self.get_db_config(conn) scope.span.class_name = constants.ClassNames.get( db_config.get('db_type', '').upper()) scope.span.domain_name = constants.DomainNames['DB'] try: operation = statement.split()[0].strip("\"").lower() except: operation = "" tags = { constants.SpanTags['OPERATION_TYPE']: SqlAlchemyIntegration._OPERATION_TO_TYPE.get(operation, ''), constants.SpanTags['DB_INSTANCE']: db_config.get('database', ''), constants.SpanTags['DB_HOST']: db_config.get('host', ''), constants.SpanTags['DB_TYPE']: db_config.get('db_type', ''), constants.SpanTags['DB_STATEMENT_TYPE']: operation.upper(), constants.SpanTags['TOPOLOGY_VERTEX']: True } if not ConfigProvider.get( config_names.THUNDRA_TRACE_INTEGRATIONS_RDB_STATEMENT_MASK): tags[constants.DBTags['DB_STATEMENT']] = statement scope.span.tags = tags
async def on_request_chunk_sent(session, trace_config_ctx, params): if not hasattr(trace_config_ctx, "scope"): return scope = trace_config_ctx.scope if not ConfigProvider.get(config_names.THUNDRA_TRACE_INTEGRATIONS_HTTP_BODY_MASK) and \ (scope.span.get_tag(constants.HttpTags["BODY"]) is None): body = params.chunk if params.chunk else "" scope.span.set_tag(constants.HttpTags["BODY"], body)
def get_incoming_trace_links(): if ConfigProvider.get(config_names.THUNDRA_DISABLE, False): return {} execution_context = ExecutionContextManager.get() incoming_trace_links = list(set(execution_context.incoming_trace_links) )[:constants.MAX_INCOMING_TRACE_LINKS] return {"incomingTraceLinks": incoming_trace_links}
def parse_application_tags(): application_tags = {} prefix_length = len(config_names.THUNDRA_APPLICATION_TAG_PREFIX) for key in ConfigProvider.configs: if key.startswith(config_names.THUNDRA_APPLICATION_TAG_PREFIX): app_tag_key = key[prefix_length:] val = ConfigProvider.get(key) application_tags[app_tag_key] = val return application_tags
def patch(): if not ConfigProvider.get( config_names.THUNDRA_TRACE_INTEGRATIONS_HTTP_DISABLE): try: import aiohttp wrapt.wrap_function_wrapper('aiohttp', 'ClientSession.__init__', _wrapper) except ImportError: pass