def _finalise_queries(self):
collector = DataCollector()
self._end_queries()
assert self.profile, 'no profile was created'
diff = set(self._queries_after).difference(set(self._queries_before))
self.profile['queries'] = diff
collector.register_profile(self.profile)
def _finalise_queries(self):
collector = DataCollector()
self._queries_after = self._query_identifiers_from_collector()
assert self.profile, 'no profile was created'
diff = set(self._queries_after).difference(set(self._queries_before))
self.profile['queries'] = [collector.query_with_temp_id(x) for x in diff]
collector.register_profile(self.profile)
def wrapped_target(*args, **kwargs):
request = DataCollector().request
if request:
start_time = timezone.now()
result = target(*args, **kwargs)
end_time = timezone.now()
curr = request.meta_time or 0
request.meta_time = curr + _time_taken(start_time, end_time)
else:
result = target(*args, **kwargs)
return result
def _process_response(self, response):
with silk_meta_profiler():
collector = DataCollector()
silk_request = collector.request
if silk_request:
silk_response = ResponseModelFactory(response).construct_response_model()
silk_response.save()
silk_request.end_time = timezone.now()
collector.finalise()
silk_request.save()
else:
Logger.error('No request model was available when processing response. Did something go wrong in process_request/process_view?')
def wrapped_target(*args, **kwargs):
request = DataCollector().request
if request:
start_time = timezone.now()
result = target(*args, **kwargs)
end_time = timezone.now()
curr = request.meta_time or 0
request.meta_time = curr + _time_taken(start_time, end_time)
else:
# Logger.error('Cant perform meta profile due to no request model in DataCollector. '
# 'Has Silk middleware been installed properly?')
result = target(*args, **kwargs)
return result
def __exit__(self, exc_type, exc_val, exc_tb):
if self._should_meta_profile:
end_time = datetime.datetime.now()
exception_raised = exc_type is not None
if exception_raised:
Logger.error('Exception when performing meta profiling, dumping trace below')
traceback.print_exception(exc_type, exc_val, exc_tb)
request = DataCollector().request
if request:
curr = request.meta_time or 0
request.meta_time = curr + _time_taken(self.start_time, end_time)
else:
Logger.error('Cant perform meta profile due to no request model in DataCollector. '
'Has Silk middleware been installed properly?')
def _process_response(self, response):
with silk_meta_profiler():
collector = DataCollector()
collector.stop_python_profiler()
silk_request = collector.request
if silk_request:
try:
silk_response = ResponseModelFactory(response).construct_response_model()
except IntegrityError:
raise #collector.finalise()
else:
silk_response.save()
silk_request.end_time = datetime.datetime.now()
collector.finalise()
silk_request.save()
else:
Logger.error('No request model was available when processing response. Did something go wrong in process_request/process_view?')
def _process_response(self, response):
Logger.debug('Process response')
with silk_meta_profiler():
collector = DataCollector()
collector.stop_python_profiler()
silk_request = collector.request
if silk_request:
silk_response = ResponseModelFactory(response).construct_response_model()
silk_response.save()
else:
Logger.error(
'No request model was available when processing response. Did something go wrong in process_request/process_view?')
# Need to save the data outside the silk_meta_profiler
# Otherwise the meta time collected in the context manager
# is not taken in account
if silk_request:
silk_request.save()
Logger.debug('Process response done.')
def process_response(self, request, response):
if _should_intercept(request):
collector = DataCollector()
content_type = response.get('Content-Type', '').split(';')[0]
silk_request = collector.request
if silk_request:
Logger.debug('Creating response model for request model with pk %s' % silk_request.pk)
body = ''
if content_type in content_types_json:
# TODO: Perhaps theres a way to format the JSON without parsing it?
try:
content = response.content
try: # py3
content = content.decode('UTF-8')
except AttributeError: # py2
pass
body = json.dumps(json.loads(content), sort_keys=True, indent=4)
except (TypeError, ValueError):
Logger.warn('Response to request with pk %s has content type %s but was unable to parse it' % (silk_request.pk, content_type))
raw_headers = response._headers
headers = {}
for k, v in raw_headers.items():
try:
header, val = v
except ValueError:
header, val = k, v
finally:
headers[header] = val
content = response.content
silky_response = models.Response.objects.create(request=silk_request,
status_code=response.status_code,
encoded_headers=json.dumps(headers),
body=body)
try:
silky_response.raw_body = content
silky_response.save()
except DjangoUnicodeDecodeError:
Logger.debug('NYI: Saving of binary response body') # TODO
silk_request.end_time = timezone.now()
silk_request.save()
collector.finalise()
else:
Logger.error('No request model was available when processing response. Did something go wrong in process_request/process_view?')
return response
def process_response(self, request, response):
if _should_intercept(request):
meta_start_time = None
if SilkyConfig().SILKY_META:
meta_start_time = timezone.now()
collector = DataCollector()
silk_request = collector.request
if silk_request:
silk_response = ResponseModelFactory(response).construct_response_model()
silk_response.save()
silk_request.end_time = timezone.now()
collector.finalise()
if SilkyConfig().SILKY_META:
meta_end_time = timezone.now()
silk_request.meta_start_time = meta_start_time
silk_request.meta_end_time = meta_end_time
silk_request.save()
else:
Logger.error('No request model was available when processing response. Did something go wrong in process_request/process_view?')
return response
def __enter__(self):
if self._silk_installed() and self._should_profile():
with silk_meta_profiler():
self._start_queries()
if not self.name:
raise ValueError('silk_profile used as a context manager must have a name')
frame = inspect.currentframe()
frames = inspect.getouterframes(frame)
outer_frame = frames[1]
path = outer_frame[1]
line_num = outer_frame[2]
request = DataCollector().request
self.profile = {
'name': self.name,
'file_path': path,
'line_num': line_num,
'dynamic': self._dynamic,
'request': request,
'start_time': timezone.now(),
}
else:
Logger.warn('Cannot execute silk_profile as silk is not installed correctly.')
def _process_response(self, request, response):
Logger.debug('Process response')
with silk_meta_profiler():
collector = DataCollector()
collector.stop_python_profiler()
silk_request = collector.request
if silk_request:
silk_response = ResponseModelFactory(response).construct_response_model()
silk_response.save()
silk_request.end_time = timezone.now()
collector.finalise()
else:
Logger.error('No request model was available when processing response. '
'Did something go wrong in process_request/process_view?'
'\n' + str(request) + '\n\n' + str(response))
# Need to save the data outside the silk_meta_profiler
# Otherwise the meta time collected in the context manager
# is not taken in account
if silk_request:
silk_request.save()
Logger.debug('Process response done.')
def test_time_taken(self):
profile = list(DataCollector().profiles.values())[0]
time_taken = _time_taken(start_time=profile['start_time'],
end_time=profile['end_time'])
self.assertGreaterEqual(time_taken, 100)
self.assertLess(time_taken, 110)
def setUpClass(cls):
r = Request.objects.create()
DataCollector().configure(r)
with silk_profile(name='test_profile'):
sleep(0.1)
class ResponseModelFactory(object):
"""given a response object, craft the silk response model"""
def __init__(self, response):
super(ResponseModelFactory, self).__init__()
self.response = response
self.request = DataCollector().request
def body(self):
body = ''
content_type, char_set = _parse_content_type(self.response.get('Content-Type', ''))
content = getattr(self.response, 'content', '')
if char_set and content:
try:
content = content.decode(char_set)
except AttributeError:
pass
except LookupError: # If no encoding exists, default to UTF-8
try:
content = content.decode('UTF-8')
except AttributeError:
pass
except UnicodeDecodeError:
content = ''
except Exception as e:
Logger.error('Unable to decode response body using char_set %s due to error: %s. Will ignore. Stacktrace:' % (char_set, e))
traceback.print_exc()
else:
# Default to an attempt at UTF-8 decoding.
try:
content = content.decode('UTF-8')
except AttributeError:
pass
except UnicodeDecodeError:
content = ''
if content:
max_body_size = SilkyConfig().SILKY_MAX_RESPONSE_BODY_SIZE
if max_body_size > -1:
Logger.debug('Max size of response body defined so checking')
size = sys.getsizeof(content, None)
if not size:
Logger.error('Could not get size of response body. Ignoring')
content = ''
else:
if size > max_body_size:
content = ''
Logger.debug('Size of %d for %s is bigger than %d so ignoring response body' % (size, self.request.path, max_body_size))
else:
Logger.debug('Size of %d for %s is less than %d so saving response body' % (size, self.request.path, max_body_size))
if content_type in content_types_json:
# TODO: Perhaps theres a way to format the JSON without parsing it?
try:
body = json.dumps(json.loads(content), sort_keys=True, indent=4)
except (TypeError, ValueError):
Logger.warn('Response to request with pk %s has content type %s but was unable to parse it' % (self.request.pk, content_type))
return body, content
def construct_response_model(self):
assert self.request, 'Cant construct a response model if there is no request model'
Logger.debug('Creating response model for request model with pk %s' % self.request.pk)
b, content = self.body()
raw_headers = self.response._headers
headers = {}
for k, v in raw_headers.items():
try:
header, val = v
except ValueError:
header, val = k, v
finally:
headers[header] = val
silky_response = models.Response.objects.create(request=self.request,
status_code=self.response.status_code,
encoded_headers=json.dumps(headers),
body=b)
# Text fields are encoded as UTF-8 in Django and hence will try to coerce
# anything to we pass to UTF-8. Some stuff like binary will fail.
try:
silky_response.raw_body = content
except UnicodeDecodeError:
Logger.debug('NYI: Saving of binary response body') # TODO
self.request.response = silky_response
self.request.save()
return silky_response
def test_singleton(self):
a = DataCollector()
b = DataCollector()
c = DataCollector()
self.assertTrue(a == b == c)
def test_query_registration(self):
mock_query = {}
DataCollector().register_query(mock_query)
self.assertIn(mock_query, list(DataCollector().queries.values()))
def __enter__(self):
DataCollector().start_meta_block()
if self._should_meta_profile:
self.start_time = timezone.now()
def test_name(self):
profile = list(DataCollector().profiles)[0]
self.assertEqual(profile['name'], 'test_profile')
def test_query(self):
query = list(DataCollector().queries.values())[0]
self.assertEqual(query['query'], self.query_string)
def test_context_manager_no_request(self):
DataCollector().configure()
with silk_profile(name='test_profile'):
sleep(0.1)
self.assertFalse(DataCollector().profiles)
def test_clear(self):
self.test_query_registration()
DataCollector().clear()
self.assertFalse(DataCollector().queries)
def __init__(self, response):
super(ResponseModelFactory, self).__init__()
self.response = response
self.request = DataCollector().request
def setUp(self):
DataCollector().request = Request.objects.create()
def _get_query(self):
query = list(DataCollector().queries)[0]
return query
def __init__(self, response):
super(ResponseModelFactory, self).__init__()
self.response = response
self.request = DataCollector().request
def _should_profile(self):
return DataCollector().request is not None
def test_context_manager_request(self):
DataCollector().configure(Request.objects.create(path='/to/somewhere'))
with silk_profile(name='test_profile'):
sleep(0.1)
profile = list(DataCollector().profiles.values())[0]
self.assertEqual(DataCollector().request, profile['request'])
def __init__(self, response):
super().__init__()
self.response = response
self.request = DataCollector().request
def test_count(self):
self.assertEqual(1, len(DataCollector().queries))
def setUpClass(cls):
super(TestProfilertContextManager, cls).setUpClass()
r = Request.objects.create()
DataCollector().configure(r)
with silk_profile(name='test_profile'):
sleep(0.1)
def _query(self):
try:
query = list(DataCollector().queries)[0]
except IndexError:
self.fail('No queries created')
return query
def test_one_object(self):
self.assertTrue(len(DataCollector().profiles), 1)
def _query_identifiers_from_collector(self):
return [x for x in DataCollector().queries]
def test_name(self):
profile = list(DataCollector().profiles.values())[0]
self.assertEqual(profile['name'], 'func')
