def before_request(cls, wrapped, instance, args, kwargs):
"""
Runs when new request comes in.
:param wrapped: wrapt's wrapped
:param instance: wrapt's instance
:param args: wrapt's args
:param kwargs: wrapt's kwargs
"""
print_debug('before_request Tornado request')
try:
ignored = ignore_request('', instance.request.path)
if not ignored and not is_ignored_endpoint(instance.request.path):
unique_id = str(uuid.uuid4())
trace = epsagon.trace.trace_factory.get_or_create_trace(
unique_id=unique_id)
trace.prepare()
setattr(instance, TORNADO_TRACE_ID, unique_id)
cls.RUNNERS[unique_id] = (TornadoRunner(
time.time(), instance.request))
trace.set_runner(cls.RUNNERS[unique_id])
print_debug('Created Tornado Runner')
# Collect metadata in case this is a container.
collect_container_metadata(
cls.RUNNERS[unique_id].resource['metadata'])
except Exception as instrumentation_exception: # pylint: disable=W0703
epsagon.trace.trace_factory.add_exception(
instrumentation_exception, traceback.format_exc())
return wrapped(*args, **kwargs)
def pre_request(worker, req):
"""
Runs before process of response.
"""
if 'SyncWorker' not in type(worker).__name__:
return
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
if ignore_request('', req.path.lower()):
return
# Create a Gunicorn runner with current request.
try:
runner = GunicornRunner(time.time(), req)
trace.set_runner(runner)
# Collect metadata in case this is a container.
collect_container_metadata(runner.resource['metadata'])
# pylint: disable=W0703
except Exception as exception:
# Regress to python runner.
warnings.warn('Could not extract request', EpsagonWarning)
epsagon.trace.trace_factory.add_exception(exception,
traceback.format_exc())
async def custom_route_handler(request: Request) -> Response:
"""
Traces given request and its response.
:param request: to trace
"""
should_ignore_request = True
try:
epsagon.trace.trace_factory.switch_to_async_tracer()
if not ignore_request('', request.url.path.lower()):
should_ignore_request = False
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
except Exception as exception: # pylint: disable=W0703
return await original_route_handler(request)
if should_ignore_request:
return await original_route_handler(request)
runner = None
response = None
try:
body = await request.json()
except json.decoder.JSONDecodeError:
body = ''
try:
runner = FastapiRunner(time.time(), request, json.dumps(body))
trace.set_runner(runner)
collect_container_metadata(runner.resource['metadata'])
except Exception as exception: # pylint: disable=W0703
warnings.warn('Could not extract request', EpsagonWarning)
raised_err = None
try:
response: Response = await original_route_handler(request)
except Exception as exception: # pylint: disable=W0703
raised_err = exception
traceback_data = get_traceback_data_from_exception(exception)
trace.runner.set_exception(exception, traceback_data)
try:
if not raised_err and response is not None and runner:
if ignore_request(
response.headers.get('Content-Type', '').lower(),
''
):
return response
runner.update_response(response)
if runner:
epsagon.trace.trace_factory.send_traces()
except Exception as exception: # pylint: disable=W0703
print_debug('Failed to send traces: {}'.format(exception))
if raised_err:
raise raised_err
return response
async def AiohttpMiddleware(request, handler):
"""
aiohttp middleware to create a runner with event details
:param request: incoming request data
:param handler: original handler
:return: response data from the handler
"""
print_debug('[aiohttp] started middleware')
epsagon.trace.trace_factory.switch_to_async_tracer()
if (ignore_request('', request.path.lower())
or is_ignored_endpoint(request.path.lower())):
print_debug('[aiohttp] ignoring request')
return await handler(request)
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
runner = None
response = None
try:
body = await request.text()
print_debug('[aiohttp] got body')
runner = AiohttpRunner(time.time(), request, body, handler)
trace.set_runner(runner)
collect_container_metadata(runner.resource['metadata'])
print_debug('[aiohttp] initialized runner')
except Exception as exception: # pylint: disable=W0703
warnings.warn('Could not extract request', EpsagonWarning)
raised_err = None
try:
response = await handler(request)
print_debug('[aiohttp] got response')
except HTTPNotFound:
# Ignoring 404s
epsagon.trace.trace_factory.pop_trace(trace)
raise
except Exception as exception: # pylint: disable=W0703
raised_err = exception
traceback_data = get_traceback_data_from_exception(exception)
trace.runner.set_exception(exception, traceback_data)
if response is not None and runner:
if ignore_request(response.content_type.lower(), ''):
return response
runner.update_response(response)
if runner:
print_debug('[aiohttp] sending trace')
epsagon.trace.trace_factory.send_traces()
if raised_err:
print_debug('[aiohttp] raising error')
raise raised_err
print_debug('[aiohttp] middleware done')
return response
def test_runner_duration(_wrapped_post):
runner = RunnerEventMock()
runner.terminated = False
trace = trace_factory.get_or_create_trace()
trace.token = 'a'
trace.set_runner(runner)
time.sleep(0.2)
trace_factory.send_traces()
assert 0.2 < runner.duration < 0.3
def test_set_error_sanity():
event = RunnerEventMock()
trace = trace_factory.get_or_create_trace()
trace.clear_events()
trace.set_runner(event)
msg = 'oops'
trace.set_error(ValueError(msg))
assert trace.to_dict()['events'][0]['exception']['message'] == msg
assert len(trace.to_dict()['events'][0]['exception']['traceback']) > 1
def test_trace_url_sanity():
event = RunnerEventMock()
trace = trace_factory.get_or_create_trace()
trace.clear_events()
trace.set_runner(event)
trace_url = trace.get_trace_url()
assert trace_url == TRACE_URL_PREFIX.format(
id=event.resource['metadata']['trace_id'],
start_time=int(event.start_time)
)
def test_send_traces_unicode(wrapped_post):
trace = trace_factory.get_or_create_trace()
runner = UnicodeReturnValueEventMock()
trace.set_runner(runner)
trace_factory.send_traces()
wrapped_post.assert_called_with(
'POST',
'collector',
body=json.dumps(trace.to_dict(), ensure_ascii=False),
timeout=epsagon.constants.SEND_TIMEOUT,
)
def test_custom_labels_override_trace():
event = RunnerEventMock()
trace = trace_factory.get_or_create_trace()
trace.clear_events()
trace.set_runner(event)
trace.add_label('test_label', 'test_value1')
trace.add_label('test_label', 'test_value2')
trace_metadata = trace.to_dict()['events'][0]['resource']['metadata']
assert trace_metadata.get('labels') is not None
assert json.loads(trace_metadata['labels']) == {'test_label': 'test_value2'}
def test_set_error_string():
event = RunnerEventMock()
trace = trace_factory.get_or_create_trace()
trace.clear_events()
trace.set_runner(event)
msg = 'oops'
trace.set_error(msg)
assert trace.to_dict()['events'][0]['exception']['message'] == msg
assert trace.to_dict()['events'][0]['exception']['type'] == (
EpsagonException.__name__)
def test_set_error_with_traceback():
event = RunnerEventMock()
trace = trace_factory.get_or_create_trace()
trace.clear_events()
trace.set_runner(event)
msg = 'oops'
traceback_data = 'test_value'
trace.set_error(ValueError(msg), traceback_data=traceback_data)
assert trace.to_dict()['events'][0]['exception']['message'] == msg
assert (trace.to_dict()['events'][0]['exception']['traceback'] ==
traceback_data)
def before_request(self):
"""
Runs before process of response.
"""
trace = epsagon.trace.trace_factory.get_trace()
if not trace:
trace = epsagon.trace.trace_factory.get_or_create_trace()
else:
self.should_send_trace = False
trace.prepare()
# Ignoring non relevant content types.
self.ignored_request = ignore_request('', self.request.path.lower())
if self.ignored_request:
return
# Create a Django runner with current request.
try:
self.runner = epsagon.runners.django.DjangoRunner(
time.time(),
self.request
)
trace.set_runner(self.runner)
self.request.epsagon_trace = trace
# Collect metadata in case this is a container.
collect_container_metadata(self.runner.resource['metadata'])
# pylint: disable=W0703
except Exception as exception:
# Regress to python runner.
warnings.warn('Could not extract request', EpsagonWarning)
epsagon.trace.trace_factory.add_exception(
exception,
traceback.format_exc()
)
# Extract HTTP trigger data.
try:
trigger = epsagon.triggers.http.HTTPTriggerFactory.factory(
time.time(),
self.request
)
if trigger:
epsagon.trace.trace_factory.add_event(trigger)
# pylint: disable=W0703
except Exception as exception:
epsagon.trace.trace_factory.add_exception(
exception,
traceback.format_exc(),
)
def test_lambda_trace_url_sanity():
event = LambdaRunnerEventMock()
trace = trace_factory.get_or_create_trace()
trace.clear_events()
trace.set_runner(event)
trace_url = trace.get_trace_url()
assert trace_url == LAMBDA_TRACE_URL_PREFIX.format(
aws_account=event.resource['metadata']['aws_account'],
region=event.resource['metadata']['region'],
function_name=event.resource['name'],
request_id=event.event_id,
request_time=int(event.start_time)
)
def test_timeout_handler_called(wrapped_post):
"""
Sanity
"""
context = ContextMock(DEFAULT_SEND_TIMEOUT_MS * 1.1)
runner = RunnerEventMock()
trace = trace_factory.get_or_create_trace()
trace.token = 'a'
trace.set_timeout_handler(context)
trace.set_runner(runner)
time.sleep((DEFAULT_SEND_TIMEOUT_MS / 1000) * 1.5)
trace.reset_timeout_handler()
assert trace.trace_sent
assert wrapped_post.called
def test_timeout_handler_called(wrapped_post):
"""
Sanity
"""
context = ContextMock(300)
runner = RunnerEventMock()
trace = trace_factory.get_or_create_trace()
trace.token = 'a'
trace.set_timeout_handler(context)
trace.set_runner(runner)
time.sleep(0.5)
trace.reset_timeout_handler()
assert trace.trace_sent
assert wrapped_post.called
def test_timeout_send_not_called_twice(wrapped_post):
"""
In case of a timeout send trace, validate no trace
is sent afterwards (if the flow continues)
"""
context = ContextMock(300)
runner = RunnerEventMock()
trace = trace_factory.get_or_create_trace()
trace.token = 'a'
trace.set_timeout_handler(context)
trace.set_runner(runner)
time.sleep(0.5)
trace.reset_timeout_handler()
assert trace.trace_sent
assert wrapped_post.call_count == 1
def test_multi_value_labels_sanity():
event = RunnerEventMock()
trace = trace_factory.get_or_create_trace()
trace.clear_events()
trace.set_runner(event)
trace.add_label('test_label', {
'test2_label': 15,
'test3_label': 'test',
4: 'hey'
})
trace_metadata = trace.to_dict()['events'][0]['resource']['metadata']
assert trace_metadata.get('labels') is not None
assert json.loads(trace_metadata['labels']) == {
'test_label.test2_label': '15',
'test_label.test3_label': 'test',
'test_label.4': 'hey',
}
def test_custom_labels_sanity():
event = RunnerEventMock()
trace = trace_factory.get_or_create_trace()
trace.clear_events()
trace.set_runner(event)
trace.add_label('test_label', 'test_value')
trace.add_label('test_label_2', 42)
trace.add_label('test_label_3', 42.2)
trace.add_label('test_label_invalid', {})
trace_metadata = trace.to_dict()['events'][0]['resource']['metadata']
assert trace_metadata.get('labels') is not None
assert json.loads(trace_metadata['labels']) == {
'test_label': 'test_value',
'test_label_2': '42',
'test_label_3': '42.2',
}
def test_return_value_key_to_ignore(wrapped_post):
key_to_ignore = 'key_to_ignore_in_return_value'
os.environ['EPSAGON_IGNORED_KEYS'] = key_to_ignore
keys_to_ignore = [key_to_ignore]
# reset traces created at setup function
trace_factory.traces = {}
epsagon.utils.init(token='token',
app_name='app-name',
collector_url='collector',
metadata_only=False)
trace = trace_factory.get_or_create_trace()
return_value = {
key_to_ignore: 'f',
's': {
'a': 1,
'b': 2,
'c': {
'f': 1,
key_to_ignore: '1',
'g': {
key_to_ignore: '1'
}
}
}
}
copied_return_value = return_value.copy()
runner = ReturnValueEventMock(return_value)
trace.set_runner(runner)
trace.token = 'a'
trace_factory.send_traces()
assert len(trace.to_dict()['events']) == 1
event = trace.to_dict()['events'][0]
assert event['origin'] == 'runner'
actual_return_value = event['resource']['metadata']['return_value']
_assert_key_not_exist(actual_return_value, key_to_ignore)
# check that original return value hasn't been changed
assert copied_return_value == return_value
wrapped_post.assert_called_with(
'collector',
data=json.dumps(trace.to_dict()),
timeout=epsagon.constants.SEND_TIMEOUT,
headers={'Authorization': 'Bearer {}'.format(trace.token)})
os.environ.pop('EPSAGON_IGNORED_KEYS')
def test_send_invalid_return_value(wrapped_post):
trace = trace_factory.get_or_create_trace()
runner = InvalidReturnValueEventMock()
trace.set_runner(runner)
trace_factory.send_traces()
assert len(trace.to_dict()['events']) == 1
event = trace.to_dict()['events'][0]
assert event['origin'] == 'runner'
actual_return_value = event['resource']['metadata']['return_value']
assert actual_return_value == FAILED_TO_SERIALIZE_MESSAGE
wrapped_post.assert_called_with(
'POST',
'collector',
body=json.dumps(trace.to_dict()),
timeout=epsagon.constants.SEND_TIMEOUT,
)
def test_send_invalid_return_value(wrapped_post):
trace = trace_factory.get_or_create_trace()
runner = InvalidReturnValueEventMock()
trace.set_runner(runner)
trace.token = 'a'
trace_factory.send_traces()
assert len(trace.to_dict()['events']) == 1
event = trace.to_dict()['events'][0]
assert event['origin'] == 'runner'
actual_return_value = event['resource']['metadata']['return_value']
assert actual_return_value == FAILED_TO_SERIALIZE_MESSAGE
wrapped_post.assert_called_with(
'',
data=json.dumps(trace.to_dict()),
timeout=epsagon.constants.SEND_TIMEOUT,
headers={'Authorization': 'Bearer {}'.format(trace.token)})
def test_timeout_happyflow_handler_call(wrapped_post):
"""
Test in case we already sent the traces on happy flow,
that timeout handler call won't send them again.
"""
context = ContextMock(300)
runner = RunnerEventMock()
trace = trace_factory.get_or_create_trace()
trace.set_runner(runner)
trace.token = 'a'
trace_factory.send_traces()
trace.set_timeout_handler(context)
time.sleep(0.5)
trace.reset_timeout_handler()
assert trace.trace_sent
assert wrapped_post.call_count == 1
def test_custom_labels_sanity():
event = RunnerEventMock()
trace = trace_factory.get_or_create_trace()
trace.clear_events()
trace.set_runner(event)
trace.add_label('test_label', 'test_value')
trace.add_label('test_label_2', 42)
trace.add_label('test_label_3', 42.2)
trace.add_label('test_label_4', True)
# This is not an invalid label, but it won't be added because dict is empty.
trace.add_label('test_label_invalid', {})
trace_metadata = trace.to_dict()['events'][0]['resource']['metadata']
assert trace_metadata.get('labels') is not None
assert json.loads(trace_metadata['labels']) == {
'test_label': 'test_value',
'test_label_2': 42,
'test_label_3': 42.2,
'test_label_4': True,
}
def test_metadata_field_too_big(wrapped_post):
trace = trace_factory.get_or_create_trace()
max_size = MAX_METADATA_FIELD_SIZE_LIMIT
return_value = {'1': 'a' * (max_size + 1)}
runner = ReturnValueEventMock(return_value)
trace.set_runner(runner)
trace.token = 'a'
trace_factory.send_traces()
assert len(trace.to_dict()['events']) == 1
event = trace.to_dict()['events'][0]
assert event['origin'] == 'runner'
actual_return_value = event['resource']['metadata']['return_value']
assert actual_return_value == json.dumps(return_value)[:max_size]
wrapped_post.assert_called_with(
'',
data=json.dumps(trace.to_dict()),
timeout=epsagon.constants.SEND_TIMEOUT,
headers={'Authorization': 'Bearer {}'.format(trace.token)})
def _setup_trace_runner(epsagon_scope, trace, request):
"""
Setups the trace runner event - creates the FastAPI runner event,
collects container metadata (if relevant).
Assumption: a runner event has NOT been created before for given trace.
:return: True if succeeded to setup runner event, False otherwise
"""
try:
trace.set_runner(FastapiRunner(time.time(), request))
except Exception: # pylint: disable=W0703
print_debug('Failed to add FastAPI runner event, skipping trace')
# failed to add runner event, skipping trace
return False
try:
if not epsagon_scope.get(SCOPE_CONTAINER_METADATA_COLLECTED):
collect_container_metadata(trace.runner.resource['metadata'])
epsagon_scope[SCOPE_CONTAINER_METADATA_COLLECTED] = True
except Exception: # pylint: disable=W0703
warnings.warn('Could not extract container metadata', EpsagonWarning)
return True
async def AiohttpMiddleware(request, handler):
"""
aiohttp middleware to create a runner with event details
:param request: incoming request data
:param handler: original handler
:return: response data from the handler
"""
epsagon.trace.trace_factory.switch_to_async_tracer()
if ignore_request('', request.path.lower()):
return await handler(request)
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
runner = None
response = None
try:
body = await request.text()
runner = AiohttpRunner(time.time(), request, body, handler)
trace.set_runner(runner)
collect_container_metadata(runner.resource['metadata'])
except Exception as exception: # pylint: disable=W0703
warnings.warn('Could not extract request', EpsagonWarning)
try:
response = await handler(request)
except Exception as exception: # pylint: disable=W0703
traceback_data = get_traceback_data_from_exception(exception)
trace.runner.set_exception(exception, traceback_data)
if response is not None and runner:
if ignore_request(response.content_type.lower(), ''):
return response
runner.update_response(response)
if runner:
epsagon.trace.trace_factory.send_traces()
return response
def _before_request(self):
"""
Runs when new request comes in.
:return: None.
"""
# Ignoring non relevant content types.
self.ignored_request = ignore_request('', request.path.lower())
if self.ignored_request:
return
trace = epsagon.trace.trace_factory.get_or_create_trace()
trace.prepare()
# Create flask runner with current request.
try:
self.runner = epsagon.runners.flask.FlaskRunner(
time.time(), self.app, request)
trace.set_runner(self.runner)
# Collect metadata in case this is a container.
collect_container_metadata(self.runner.resource['metadata'])
# pylint: disable=W0703
except Exception as exception:
# Regress to python runner.
warnings.warn('Could not extract request', EpsagonWarning)
trace.add_exception(exception, traceback.format_exc())
# Extract HTTP trigger data.
try:
trigger = epsagon.triggers.http.HTTPTriggerFactory.factory(
time.time(), request)
if trigger:
trace.add_event(trigger)
# pylint: disable=W0703
except Exception as exception:
trace.add_exception(
exception,
traceback.format_exc(),
)
def test_send_big_trace(wrapped_post):
trace = trace_factory.get_or_create_trace()
runner = RunnerEventMock()
trace.set_runner(runner)
trace.token = 'a'
for _ in range(2):
trace.add_event(BigEventMock())
trace_factory.send_traces()
assert len(trace.to_dict()['events']) == 3
for event in trace.to_dict()['events']:
if event['origin'] == 'runner':
assert event['resource']['metadata']['is_trimmed']
wrapped_post.assert_called_with(
'',
data=json.dumps(trace.to_dict()),
timeout=epsagon.constants.SEND_TIMEOUT,
headers={'Authorization': 'Bearer {}'.format(trace.token)})
def test_send_big_trace(wrapped_post):
trace = trace_factory.get_or_create_trace()
runner = RunnerEventMock()
trace.set_runner(runner)
for _ in range(2):
trace.add_event(BigEventMock())
trace_factory.send_traces()
assert len(trace.to_dict()['events']) == 3
for event in trace.to_dict()['events']:
if event['origin'] == 'runner':
assert event['resource']['metadata']['is_trimmed']
wrapped_post.assert_called_with(
'POST',
'collector',
body=json.dumps(trace.to_dict()),
timeout=epsagon.constants.SEND_TIMEOUT,
)
def test_return_value_key_to_ignore(wrapped_post):
key_to_ignore = 'key_to_ignore_in_return_value'
os.environ['EPSAGON_IGNORED_KEYS'] = key_to_ignore
epsagon.utils.init(
token='token',
app_name='app-name',
collector_url='collector',
metadata_only=False
)
trace = trace_factory.get_or_create_trace()
return_value = {
key_to_ignore: 'f',
's': {
'a': 1,
'b': 2,
'c': {
'f': 1,
key_to_ignore: '1',
'g': {
key_to_ignore: '1'
}
}
}
}
copied_return_value = return_value.copy()
runner = ReturnValueEventMock(return_value)
trace.set_runner(runner)
trace_factory.send_traces()
assert len(trace.to_dict()['events']) == 1
event = trace.to_dict()['events'][0]
assert event['origin'] == 'runner'
actual_return_value = event['resource']['metadata']['return_value']
_assert_key_not_exist(actual_return_value, key_to_ignore)
# check that original return value hasn't been changed
assert copied_return_value == return_value
os.environ.pop('EPSAGON_IGNORED_KEYS')
