def before_request(cls, wrapped, instance, args, kwargs):
"""
Runs when new request comes in.
:param wrapped: wrapt's wrapped
:param instance: wrapt's instance
:param args: wrapt's args
:param kwargs: wrapt's kwargs
"""
print_debug('before_request Tornado request')
try:
ignored = ignore_request('', instance.request.path)
if not ignored and not is_ignored_endpoint(instance.request.path):
unique_id = str(uuid.uuid4())
trace = epsagon.trace.trace_factory.get_or_create_trace(
unique_id=unique_id)
trace.prepare()
setattr(instance, TORNADO_TRACE_ID, unique_id)
cls.RUNNERS[unique_id] = (TornadoRunner(
time.time(), instance.request))
trace.set_runner(cls.RUNNERS[unique_id])
print_debug('Created Tornado Runner')
# Collect metadata in case this is a container.
collect_container_metadata(
cls.RUNNERS[unique_id].resource['metadata'])
except Exception as instrumentation_exception: # pylint: disable=W0703
epsagon.trace.trace_factory.add_exception(
instrumentation_exception, traceback.format_exc())
return wrapped(*args, **kwargs)
async def server_call_wrapper(wrapped, _instance, args, kwargs):
"""
Wraps the main fastapi server request entrypoint - which is
the ServerErrorMiddleware __call_ function.
:param wrapped: wrapt's wrapped
:param instance: wrapt's instance
:param args: wrapt's args
:param kwargs: wrapt's kwargs
"""
# Skip on Lambda environment since it's not relevant and might be duplicate
if is_lambda_env() or not args or len(args) != 3:
return await wrapped(*args, **kwargs)
scope = args[0]
if not scope or scope.get('type', '') != 'http':
return await wrapped(*args, **kwargs)
trace = None
try:
epsagon.trace.trace_factory.switch_to_multiple_traces()
unique_id = str(uuid.uuid4())
trace = epsagon.trace.trace_factory.get_or_create_trace(
unique_id=unique_id)
trace.prepare()
scope[EPSAGON_MARKER] = {
SCOPE_UNIQUE_ID: unique_id,
}
except Exception: # pylint: disable=broad-except
_clean_trace(trace)
return await wrapped(*args, **kwargs)
response = None
raised_error = None
sent_trace = False
epsagon_scope = scope[EPSAGON_MARKER]
try:
response = await wrapped(*args, **kwargs)
except Exception as exception: # pylint: disable=broad-except
raised_error = exception
if trace.runner and not epsagon_scope.get(SCOPE_IGNORE_REQUEST):
try:
if raised_error:
traceback_data = get_traceback_data_from_exception(
raised_error)
trace.runner.set_exception(raised_error, traceback_data)
trace.runner.update_status_code(DEFAULT_ERROR_STATUS_CODE,
override=False)
await run_in_threadpool(epsagon.trace.trace_factory.send_traces,
trace=trace)
sent_trace = True
except Exception as exception: # pylint: disable=broad-except
print_debug('Failed to send traces: {}'.format(exception))
scope.pop(EPSAGON_MARKER, None)
if not sent_trace:
_clean_trace(trace)
if raised_error:
raise raised_error from None
return response
def pre_request(worker, req):
"""
Runs before process of response.
"""
if 'SyncWorker' not in type(worker).__name__:
return
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
if ignore_request('', req.path.lower()):
return
# Create a Gunicorn runner with current request.
try:
runner = GunicornRunner(time.time(), req)
trace.set_runner(runner)
# Collect metadata in case this is a container.
collect_container_metadata(runner.resource['metadata'])
# pylint: disable=W0703
except Exception as exception:
# Regress to python runner.
warnings.warn('Could not extract request', EpsagonWarning)
epsagon.trace.trace_factory.add_exception(exception,
traceback.format_exc())
async def custom_route_handler(request: Request) -> Response:
"""
Traces given request and its response.
:param request: to trace
"""
should_ignore_request = True
try:
epsagon.trace.trace_factory.switch_to_async_tracer()
if not ignore_request('', request.url.path.lower()):
should_ignore_request = False
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
except Exception as exception: # pylint: disable=W0703
return await original_route_handler(request)
if should_ignore_request:
return await original_route_handler(request)
runner = None
response = None
try:
body = await request.json()
except json.decoder.JSONDecodeError:
body = ''
try:
runner = FastapiRunner(time.time(), request, json.dumps(body))
trace.set_runner(runner)
collect_container_metadata(runner.resource['metadata'])
except Exception as exception: # pylint: disable=W0703
warnings.warn('Could not extract request', EpsagonWarning)
raised_err = None
try:
response: Response = await original_route_handler(request)
except Exception as exception: # pylint: disable=W0703
raised_err = exception
traceback_data = get_traceback_data_from_exception(exception)
trace.runner.set_exception(exception, traceback_data)
try:
if not raised_err and response is not None and runner:
if ignore_request(
response.headers.get('Content-Type', '').lower(),
''
):
return response
runner.update_response(response)
if runner:
epsagon.trace.trace_factory.send_traces()
except Exception as exception: # pylint: disable=W0703
print_debug('Failed to send traces: {}'.format(exception))
if raised_err:
raise raised_err
return response
async def AiohttpMiddleware(request, handler):
"""
aiohttp middleware to create a runner with event details
:param request: incoming request data
:param handler: original handler
:return: response data from the handler
"""
print_debug('[aiohttp] started middleware')
epsagon.trace.trace_factory.switch_to_async_tracer()
if (ignore_request('', request.path.lower())
or is_ignored_endpoint(request.path.lower())):
print_debug('[aiohttp] ignoring request')
return await handler(request)
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
runner = None
response = None
try:
body = await request.text()
print_debug('[aiohttp] got body')
runner = AiohttpRunner(time.time(), request, body, handler)
trace.set_runner(runner)
collect_container_metadata(runner.resource['metadata'])
print_debug('[aiohttp] initialized runner')
except Exception as exception: # pylint: disable=W0703
warnings.warn('Could not extract request', EpsagonWarning)
raised_err = None
try:
response = await handler(request)
print_debug('[aiohttp] got response')
except HTTPNotFound:
# Ignoring 404s
epsagon.trace.trace_factory.pop_trace(trace)
raise
except Exception as exception: # pylint: disable=W0703
raised_err = exception
traceback_data = get_traceback_data_from_exception(exception)
trace.runner.set_exception(exception, traceback_data)
if response is not None and runner:
if ignore_request(response.content_type.lower(), ''):
return response
runner.update_response(response)
if runner:
print_debug('[aiohttp] sending trace')
epsagon.trace.trace_factory.send_traces()
if raised_err:
print_debug('[aiohttp] raising error')
raise raised_err
print_debug('[aiohttp] middleware done')
return response
def before_request(self):
"""
Runs before process of response.
"""
trace = epsagon.trace.trace_factory.get_trace()
if not trace:
trace = epsagon.trace.trace_factory.get_or_create_trace()
else:
self.should_send_trace = False
trace.prepare()
# Ignoring non relevant content types.
self.ignored_request = ignore_request('', self.request.path.lower())
if self.ignored_request:
return
# Create a Django runner with current request.
try:
self.runner = epsagon.runners.django.DjangoRunner(
time.time(),
self.request
)
trace.set_runner(self.runner)
self.request.epsagon_trace = trace
# Collect metadata in case this is a container.
collect_container_metadata(self.runner.resource['metadata'])
# pylint: disable=W0703
except Exception as exception:
# Regress to python runner.
warnings.warn('Could not extract request', EpsagonWarning)
epsagon.trace.trace_factory.add_exception(
exception,
traceback.format_exc()
)
# Extract HTTP trigger data.
try:
trigger = epsagon.triggers.http.HTTPTriggerFactory.factory(
time.time(),
self.request
)
if trigger:
epsagon.trace.trace_factory.add_event(trigger)
# pylint: disable=W0703
except Exception as exception:
epsagon.trace.trace_factory.add_exception(
exception,
traceback.format_exc(),
)
def test_prepare():
trace = trace_factory.get_or_create_trace()
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
trace.prepare()
assert not trace.events
assert trace.exceptions == []
assert len(w) == 1
trace.clear_events()
trace.add_event(EventMock())
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
trace.prepare()
assert not trace.events
assert trace.exceptions == []
assert len(w) == 1
trace.clear_events()
trace.add_event(EventMock())
with warnings.catch_warnings(record=True) as w:
trace.prepare()
trace.prepare() # this call should NOT trigger a warning
assert not trace.events
assert trace.exceptions == []
assert len(w) == 1
def _gcp_wrapper(*args, **kwargs):
"""
Generic google function wrapper
"""
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
try:
runner = GoogleFunctionRunner(time.time(), )
# pylint: disable=W0703
except Exception as exception:
# Regress to python runner.
warnings.warn(
'GCP environment is invalid, using simple python wrapper',
EpsagonWarning)
trace.add_exception(exception, traceback.format_exc())
return epsagon.wrappers.python_function.wrap_python_function(
func, args, kwargs)
constants.COLD_START = False
result = None
try:
result = func(*args, **kwargs)
return result
# pylint: disable=W0703
except Exception as exception:
runner.set_exception(exception, traceback.format_exc())
raise
finally:
try:
if not trace.metadata_only:
runner.resource['metadata']['return_value'] = result
# pylint: disable=W0703
except Exception as exception:
trace.add_exception(exception, traceback.format_exc())
try:
trace.add_event(runner)
epsagon.trace.trace_factory.send_traces()
# pylint: disable=W0703
except Exception:
pass
def _teardown_request(self, exception):
"""
Runs at the end of the request. Exception will be passed if happens.
:param exception: Exception (or None).
:return: None.
"""
if self.ignored_request:
return
if exception:
self.exception_handler[sys.version_info.major](exception)
# Ignoring endpoint, only if no error happened.
if not exception and request.url_rule.rule in self.ignored_endpoints:
return
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.send_traces()
trace.prepare()
def _before_request(self):
"""
Runs when new request comes in.
:return: None.
"""
# Ignoring non relevant content types.
self.ignored_request = ignore_request('', request.path.lower())
if self.ignored_request:
return
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
# Create flask runner with current request.
try:
self.runner = epsagon.runners.flask.FlaskRunner(
time.time(), self.app, request)
trace.set_runner(self.runner)
# Collect metadata in case this is a container.
collect_container_metadata(self.runner.resource['metadata'])
# pylint: disable=W0703
except Exception as exception:
# Regress to python runner.
warnings.warn('Could not extract request', EpsagonWarning)
trace.add_exception(exception, traceback.format_exc())
# Extract HTTP trigger data.
try:
trigger = epsagon.triggers.http.HTTPTriggerFactory.factory(
time.time(), request)
if trigger:
trace.add_event(trigger)
# pylint: disable=W0703
except Exception as exception:
trace.add_exception(
exception,
traceback.format_exc(),
)
async def AiohttpMiddleware(request, handler):
"""
aiohttp middleware to create a runner with event details
:param request: incoming request data
:param handler: original handler
:return: response data from the handler
"""
epsagon.trace.trace_factory.switch_to_async_tracer()
if ignore_request('', request.path.lower()):
return await handler(request)
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
runner = None
response = None
try:
body = await request.text()
runner = AiohttpRunner(time.time(), request, body, handler)
trace.set_runner(runner)
collect_container_metadata(runner.resource['metadata'])
except Exception as exception: # pylint: disable=W0703
warnings.warn('Could not extract request', EpsagonWarning)
try:
response = await handler(request)
except Exception as exception: # pylint: disable=W0703
traceback_data = get_traceback_data_from_exception(exception)
trace.runner.set_exception(exception, traceback_data)
if response is not None and runner:
if ignore_request(response.content_type.lower(), ''):
return response
runner.update_response(response)
if runner:
epsagon.trace.trace_factory.send_traces()
return response
def _lambda_wrapper(*args, **kwargs):
"""
Generic Lambda function wrapper
"""
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
try:
event, context = args
except ValueError:
# This can happen when someone manually calls handler without
# parameters / sends kwargs. In such case we ignore this trace.
return func(*args, **kwargs)
try:
runner = epsagon.runners.aws_lambda.LambdaRunner(
time.time(), context)
trace.set_runner(runner)
# pylint: disable=W0703
except Exception as exception:
# Regress to python runner.
warnings.warn(
'Lambda context is invalid, using simple python wrapper',
EpsagonWarning)
trace.add_exception(exception, traceback.format_exc())
return epsagon.wrappers.python_function.wrap_python_function(
func, args, kwargs)
constants.COLD_START = False
try:
trace.add_event(
epsagon.triggers.aws_lambda.LambdaTriggerFactory.factory(
time.time(), event, context))
# pylint: disable=W0703
except Exception as exception:
trace.add_exception(exception,
traceback.format_exc(),
additional_data={'event': event})
if not trace.disable_timeout_send:
trace.set_timeout_handler(context)
result = None
try:
result = func(*args, **kwargs)
if trace.propagate_lambda_id and isinstance(result, dict):
result[EPSAGON_EVENT_ID_KEY] = runner.event_id
runner.resource['metadata']['propagation_enabled'] = True
return result
# pylint: disable=W0703
except Exception as exception:
runner.set_exception(exception,
traceback.format_exc(),
handled=False)
raise
finally:
try:
_add_status_code(runner, result)
if not trace.metadata_only:
runner.resource['metadata']['return_value'] = result
# pylint: disable=W0703
except Exception as exception:
trace.add_exception(
exception,
traceback.format_exc(),
)
try:
if not trace.disable_timeout_send:
epsagon.trace.Trace.reset_timeout_handler()
# pylint: disable=W0703
except Exception:
pass
try:
epsagon.trace.trace_factory.send_traces()
# pylint: disable=W0703
except Exception:
pass
def _lambda_wrapper(*args, **kwargs):
"""
Generic Step Function wrapper
"""
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
try:
event, context = args
except ValueError:
# This can happen when someone manually calls handler without
# parameters / sends kwargs. In such case we ignore this trace.
return func(*args, **kwargs)
try:
runner = epsagon.runners.aws_lambda.StepLambdaRunner(
time.time(), context)
trace.set_runner(runner)
# pylint: disable=W0703
except Exception as exception:
# Regress to python runner.
warnings.warn(
'Lambda context is invalid, using simple python wrapper',
EpsagonWarning)
trace.add_exception(exception, traceback.format_exc())
return epsagon.wrappers.python_function.wrap_python_function(
func, args, kwargs)
constants.COLD_START = False
try:
trace.add_event(
epsagon.triggers.aws_lambda.LambdaTriggerFactory.factory(
time.time(), event, context))
# pylint: disable=W0703
except Exception as exception:
trace.add_exception(exception,
traceback.format_exc(),
additional_data={'event': event})
trace.set_timeout_handler(context)
result = None
try:
result = func(*args, **kwargs)
steps_dict = epsagon.utils.find_in_object(event, STEP_DICT_NAME)
if isinstance(result, dict):
# If the step functions data is not present, then this is the
# First step.
if steps_dict is None:
steps_dict = {'id': str(uuid4()), 'step_num': 0}
# Otherwise, just advance the steps number by one.
else:
# don't change trigger data
steps_dict = copy.deepcopy(steps_dict)
steps_dict['step_num'] += 1
result[STEP_DICT_NAME] = steps_dict
runner.add_step_data(steps_dict)
return result
# pylint: disable=W0703
except Exception as exception:
runner.set_exception(exception,
traceback.format_exc(),
handled=False)
raise
finally:
try:
_add_status_code(runner, result)
if not trace.metadata_only:
runner.resource['metadata']['return_value'] = (
copy.deepcopy(result))
# pylint: disable=W0703
except Exception as exception:
trace.add_exception(
exception,
traceback.format_exc(),
)
try:
epsagon.trace.Trace.reset_timeout_handler()
# pylint: disable=W0703
except Exception:
pass
try:
epsagon.trace.trace_factory.send_traces()
# pylint: disable=W0703
except Exception:
pass
def _lambda_wrapper(*args, **kwargs):
"""
Generic Step Function wrapper
"""
cold_start_duration = time.time() - constants.COLD_START_TIME
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
try:
event, context = args
except ValueError:
# This can happen when someone manually calls handler without
# parameters / sends kwargs. In such case we ignore this trace.
return func(*args, **kwargs)
try:
runner = epsagon.runners.aws_lambda.StepLambdaRunner(
time.time(),
context
)
trace.set_runner(runner)
# pylint: disable=W0703
except Exception as exception:
# Regress to python runner.
warnings.warn(
'Lambda context is invalid, using simple python wrapper',
EpsagonWarning
)
trace.add_exception(
exception,
traceback.format_exc()
)
return epsagon.wrappers.python_function.wrap_python_function(
func,
args,
kwargs
)
if constants.COLD_START:
runner.resource['metadata'][
'aws.lambda.cold_start_duration'
] = cold_start_duration
constants.COLD_START = False
try:
trace.add_event(
epsagon.triggers.aws_lambda.LambdaTriggerFactory.factory(
time.time(),
event,
context
)
)
# pylint: disable=W0703
except Exception as exception:
trace.add_exception(
exception,
traceback.format_exc(),
additional_data={'event': event}
)
trace.set_timeout_handler(context)
result = None
try:
result = func(*args, **kwargs)
steps_data = epsagon.utils.find_in_object(
event,
STEP_DICT_NAME
)
if isinstance(result, dict):
epsagon.utils.print_debug(
'Step function result type is dict, steps_data={}'.format(
steps_data
)
)
# If the step functions data is not present, then this is the
# First step.
if steps_data is None:
epsagon.utils.print_debug(
'Could not find existing steps data'
)
steps_dict = {'id': str(uuid4()), 'step_num': 0}
path = []
# Otherwise, just advance the steps number by one.
else:
# don't change trigger data
steps_dict, path = steps_data
steps_dict = copy.deepcopy(steps_dict)
if 'step_num' in steps_dict:
steps_dict['step_num'] += 1
epsagon.utils.print_debug(
'Steps data found, new dict={}'.format(steps_dict)
)
else:
steps_dict = {'id': str(uuid4()), 'step_num': 0}
epsagon.utils.print_debug(
'Steps data not found, new dict={}'.format(
steps_dict
)
)
result_path = result
# Tries to inject the steps data in the configured
# or same path where it was found
if isinstance(trace.step_dict_output_path, list):
path = trace.step_dict_output_path
try:
for sub_path in path:
result_path = result_path.get(sub_path)
except Exception as exception: # pylint: disable=broad-except
epsagon.utils.print_debug(
'Could not put steps in path={}'.format(path)
)
if result_path:
epsagon.utils.print_debug(
'Adding steps dict to result_path={}'.format(
result_path
)
)
result_path[STEP_DICT_NAME] = steps_dict
else:
epsagon.utils.print_debug(
'Adding steps dict to root result'
)
result[STEP_DICT_NAME] = steps_dict
runner.add_step_data(steps_dict)
return result
# pylint: disable=W0703
except Exception as exception:
runner.set_exception(
exception,
traceback.format_exc(),
handled=False
)
raise
finally:
try:
_add_status_code(runner, result)
if not trace.metadata_only:
runner.resource['metadata']['return_value'] = (
copy.deepcopy(result)
)
# pylint: disable=W0703
except Exception as exception:
trace.add_exception(
exception,
traceback.format_exc(),
)
try:
epsagon.trace.Trace.reset_timeout_handler()
# pylint: disable=W0703
except Exception:
pass
try:
epsagon.trace.trace_factory.send_traces()
# pylint: disable=W0703
except Exception:
pass
def _lambda_wrapper(*args, **kwargs):
"""
Generic Lambda function wrapper
"""
cold_start_duration = time.time() - constants.COLD_START_TIME
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
try:
event, context = args
except ValueError:
# This can happen when someone manually calls handler without
# parameters / sends kwargs. In such case we ignore this trace.
return func(*args, **kwargs)
if isinstance(event, dict):
ignored_payloads = _get_ignored_payloads()
if ignored_payloads and event in ignored_payloads:
return func(*args, **kwargs)
if os.environ.get(
'AWS_LAMBDA_INITIALIZATION_TYPE'
) == 'provisioned-concurrency':
constants.COLD_START = False
try:
runner = epsagon.runners.aws_lambda.LambdaRunner(
time.time(),
context
)
trace.set_runner(runner)
# pylint: disable=W0703
except Exception as exception:
# Regress to python runner.
warnings.warn(
'Lambda context is invalid, using simple python wrapper',
EpsagonWarning
)
trace.add_exception(
exception,
traceback.format_exc()
)
return epsagon.wrappers.python_function.wrap_python_function(
func,
args,
kwargs
)
if constants.COLD_START:
runner.resource['metadata'][
'aws.lambda.cold_start_duration'
] = cold_start_duration
constants.COLD_START = False
try:
trace.add_event(
epsagon.triggers.aws_lambda.LambdaTriggerFactory.factory(
time.time(),
event,
context
)
)
# pylint: disable=W0703
except Exception as exception:
trace.add_exception(
exception,
traceback.format_exc(),
additional_data={'event': event}
)
if not trace.disable_timeout_send:
trace.set_timeout_handler(context)
result = None
try:
result = func(*args, **kwargs)
if trace.propagate_lambda_id and isinstance(result, dict):
result[EPSAGON_EVENT_ID_KEY] = runner.event_id
runner.resource['metadata']['propagation_enabled'] = True
return result
# pylint: disable=W0703
except Exception as exception:
runner.set_exception(
exception,
traceback.format_exc(),
handled=False
)
raise
finally:
try:
_add_status_code(runner, result)
if not trace.metadata_only:
runner.resource['metadata']['return_value'] = result
# pylint: disable=W0703
except Exception as exception:
trace.add_exception(
exception,
traceback.format_exc(),
)
try:
if not trace.disable_timeout_send:
epsagon.trace.Trace.reset_timeout_handler()
# pylint: disable=W0703
except Exception:
pass
try:
epsagon.trace.trace_factory.send_traces()
# pylint: disable=W0703
except Exception:
epsagon.utils.print_debug('Failed to send Lambda trace')
def _tencent_function_wrapper(*args, **kwargs):
"""
Generic SCF function wrapper
"""
start_time = time.time()
cold_start_duration = start_time - constants.COLD_START_TIME
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
try:
event, context = args
except ValueError:
# This can happen when someone manually calls handler without
# parameters / sends kwargs. In such case we ignore this trace.
return func(*args, **kwargs)
try:
runner = epsagon.runners.tencent_function.TencentFunctionRunner(
start_time, context)
trace.set_runner(runner)
# pylint: disable=W0703
except Exception as exception:
# Regress to python runner.
warnings.warn(
'SCF context is invalid, using simple python wrapper',
EpsagonWarning)
trace.add_exception(exception, traceback.format_exc())
return epsagon.wrappers.python_function.wrap_python_function(
func, args, kwargs)
if constants.COLD_START:
runner.resource['metadata'][
'tencent.scf.cold_start_duration'] = cold_start_duration
constants.COLD_START = False
try:
trace.add_event(
TencentFunctionTriggerFactory.factory(start_time, event,
context, runner))
# pylint: disable=W0703
except Exception as exception:
trace.add_exception(exception,
traceback.format_exc(),
additional_data={'event': event})
result = None
try:
result = func(*args, **kwargs)
return result
# pylint: disable=W0703
except Exception as exception:
runner.set_exception(exception,
traceback.format_exc(),
handled=False)
raise
finally:
try:
_add_status_code(runner, result)
if not trace.metadata_only:
runner.resource['metadata']['tencent.scf.return_data'] = (
result)
# pylint: disable=W0703
except Exception as exception:
trace.add_exception(
exception,
traceback.format_exc(),
)
try:
epsagon.trace.trace_factory.send_traces()
# pylint: disable=W0703
except Exception:
epsagon.utils.print_debug('Failed to send SCF trace')
