def body(self):
body = ''
content_type, char_set = _parse_content_type(
self.response.get('Content-Type', ''))
content = getattr(self.response, 'content', '')
if content:
max_body_size = SilkyConfig().SILKY_MAX_RESPONSE_BODY_SIZE
if max_body_size > -1:
Logger.debug('Max size of response body defined so checking')
size = sys.getsizeof(content, None)
if not size:
Logger.error(
'Could not get size of response body. Ignoring')
content = ''
else:
if size > max_body_size:
content = ''
Logger.debug(
'Size of %d for %s is bigger than %d so ignoring response body'
% (size, self.request.path, max_body_size))
else:
Logger.debug(
'Size of %d for %s is less than %d so saving response body'
% (size, self.request.path, max_body_size))
if content and content_type in content_types_json:
# TODO: Perhaps theres a way to format the JSON without parsing it?
if not isinstance(content, str):
# byte string is not compatible with json.loads(...)
# and json.dumps(...) in python3
content = content.decode()
try:
body = json.dumps(json.loads(content),
sort_keys=True,
indent=4)
except (TypeError, ValueError):
Logger.warn(
'Response to request with pk %s has content type %s but was unable to parse it'
% (self.request.pk, content_type))
return body, content
def encoded_headers(self):
"""
From Django docs (https://docs.djangoproject.com/en/1.6/ref/request-response/#httprequest-objects):
"With the exception of CONTENT_LENGTH and CONTENT_TYPE, as given above, any HTTP headers in the request are converted to
META keys by converting all characters to uppercase, replacing any hyphens with underscores and adding an HTTP_ prefix
to the name. So, for example, a header called X-Bender would be mapped to the META key HTTP_X_BENDER."
"""
headers = {}
for k, v in self.request.META.items():
if k.startswith('HTTP') or k in ('CONTENT_TYPE', 'CONTENT_LENGTH'):
splt = k.split('_')
if splt[0] == 'HTTP':
splt = splt[1:]
k = '-'.join(splt)
headers[k] = v
if SilkyConfig().SILKY_HIDE_COOKIES:
try:
del headers['COOKIE']
except KeyError:
pass
return json.dumps(headers)
class SilkDBRouter:
silk_database = SilkyConfig().SILKY_DATABASE_NAME
logger.debug(f"Using silk_database '{silk_database}'")
def db_for_read(self, model, **hints):
if model._meta.app_label == "silk":
return self.silk_database
# No opinion
return None
def db_for_write(self, model, **hints):
if model._meta.app_label == "silk":
return self.silk_database
# No opinion
return None
def allow_relation(self, obj1, obj2, **hints):
if obj1._meta.app_label == obj2._meta.app_label == "silk":
return True
# No opinion
return None
def allow_migrate(self, db, app_label, model_name=None, **hints):
# Not a relevant app, no opinion
if app_label != "silk":
return None
# Allow silk models only on specified database
if db == self.silk_database:
return True
# Silk model on database that was not specified: disagree
return False
def body(self):
body = ''
content_type, char_set = _parse_content_type(
self.response.get('Content-Type', ''))
if self.response.streaming:
content = self.response.streaming_content
else:
content = self.response.content
if char_set:
try:
content = content.decode(char_set)
except AttributeError:
pass
except LookupError: # If no encoding exists, default to UTF-8
try:
content = content.decode('UTF-8')
except AttributeError:
pass
except UnicodeDecodeError:
content = ''
except Exception as e:
Logger.error(
'Unable to decode response body using char_set %s due to error: %s. Will ignore. Stacktrace:'
% (char_set, e))
traceback.print_exc()
else:
# Default to an attempt at UTF-8 decoding.
try:
content = content.decode('UTF-8')
except AttributeError:
pass
except UnicodeDecodeError:
content = ''
if content:
max_body_size = SilkyConfig().SILKY_MAX_RESPONSE_BODY_SIZE
if max_body_size > -1:
Logger.debug('Max size of response body defined so checking')
if isinstance(content, collections.Iterable):
size = 0 #cannot compute size on StreamingHttpResponse
else:
size = sys.getsizeof(content, None)
raise Exception("SIZE=%d" % size)
if not size:
Logger.error(
'Could not get size of response body. Ignoring')
content = ''
else:
if size > max_body_size:
content = ''
Logger.debug(
'Size of %d for %s is bigger than %d so ignoring response body'
% (size, self.request.path, max_body_size))
else:
Logger.debug(
'Size of %d for %s is less than %d so saving response body'
% (size, self.request.path, max_body_size))
if content_type in content_types_json:
# TODO: Perhaps theres a way to format the JSON without parsing it?
try:
body = json.dumps(json.loads(content),
sort_keys=True,
indent=4)
except (TypeError, ValueError):
Logger.warn(
'Response to request with pk %s has content type %s but was unable to parse it'
% (self.request.pk, content_type))
return body, content
def _should_meta_profile(self):
return SilkyConfig().SILKY_META
def setUpClass(cls):
super().setUpClass()
SilkyConfig().SILKY_AUTHENTICATION = False
SilkyConfig().SILKY_AUTHORISATION = False
from django.core.files.storage import get_storage_class
from django.db import models
from django.db.models import (DateTimeField, TextField, CharField, ForeignKey,
IntegerField, BooleanField, F, ManyToManyField,
OneToOneField, FloatField, FileField)
from django.utils import timezone
from django.db import transaction
from uuid import uuid4
import sqlparse
from django.utils.safestring import mark_safe
from silk.utils.profile_parser import parse_profile
from silk.config import SilkyConfig
silk_storage = get_storage_class(SilkyConfig().SILKY_STORAGE_CLASS)()
# Seperated out so can use in tests w/o models
def _time_taken(start_time, end_time):
d = end_time - start_time
return d.seconds * 1000 + d.microseconds / 1000
def time_taken(self):
return _time_taken(self.start_time, self.end_time)
class CaseInsensitiveDictionary(dict):
def __getitem__(self, key):
return super(CaseInsensitiveDictionary, self).__getitem__(key.lower())
def test_enabled(self):
SilkyConfig().SILKY_META = True
r = self._execute_request()
self.assertTrue(r.meta_time is not None
or r.meta_num_queries is not None
or r.meta_time_spent_queries is not None)
def setUpClass(cls):
super().setUpClass()
BlindFactory.create_batch(size=5)
SilkyConfig().SILKY_META = False
SilkyConfig().SILKY_ANALYZE_QUERIES = True
def tearDown(self):
SilkyConfig().SILKY_SENSITIVE_KEYS = DEFAULT_SENSITIVE_KEYS
def test_mask_credentials_masks_sensitive_values_listed_in_settings(self):
SilkyConfig().SILKY_SENSITIVE_KEYS = {"foo"}
body = "foo=hidethis"
expected = f"foo={CLEANSED}"
self.assertEqual(expected, self._mask(body))
def test_mask_credentials_masks_sensitive_values_listed_in_settings(self):
SilkyConfig().SILKY_SENSITIVE_KEYS = {"foo"}
self.assertNotIn("hidethis", self._mask({"foo": "hidethis"}))
def __init__(self):
super().__init__(
location=SilkyConfig().SILKY_PYTHON_PROFILER_RESULT_PATH,
base_url='')
self.base_url = None
def __init__(self):
super(ProfilerResultStorage, self).__init__(
location=SilkyConfig().SILKY_PYTHON_PROFILER_RESULT_PATH,
base_url='')
self.base_url = None
def tearDown(self):
SilkyConfig(
).SILKY_MAX_RECORDED_REQUESTS_CHECK_PERCENT = self.max_percent
SilkyConfig().SILKY_MAX_RECORDED_REQUESTS = self.max_requests
def setUp(self):
self.obj = RequestMinFactory.create()
self.max_percent = SilkyConfig(
).SILKY_MAX_RECORDED_REQUESTS_CHECK_PERCENT
self.max_requests = SilkyConfig().SILKY_MAX_RECORDED_REQUESTS
def _should_wrap(sql_query):
for ignore_str in SilkyConfig().SILKY_IGNORE_QUERIES:
if ignore_str in sql_query:
return False
return True
def test_config_with_no_value_should_have_default(self):
config = SilkyConfig()
self.assertEqual(config.SILKY_RESPONSE_MODEL_FACTORY_CLASS,
'silk.model_factory.ResponseModelFactory')
def tearDownClass(cls):
super().tearDownClass()
SilkyConfig().SILKLY_ANALYZE_QUERIES = False
def test_config_with_no_value_should_have_response_model_factory_property(
self):
config = SilkyConfig()
self.assertTrue(hasattr(config, 'response_model_factory'))
def test_disabled(self):
SilkyConfig().SILKY_META = False
r = self._execute_request()
self.assertFalse(r.meta_time)
def test_config_with_custom_class_should_load_custom_class(self):
config = SilkyConfig()
self.assertIs(config.response_model_factory, DummyResponseModelFactory)
def login_possibly_required(function=None, **kwargs):
if SilkyConfig().SILKY_AUTHENTICATION:
return login_required(function, **kwargs)
return function
def body(self):
content_type, char_set = self.content_type()
try:
raw_body = self.request.body
except RequestDataTooBig:
raw_body = b"Raw body exceeds DATA_UPLOAD_MAX_MEMORY_SIZE, Silk is not showing file uploads."
body = self.request.POST.copy()
for k, v in self.request.FILES.items():
body.appendlist(k, v)
return body, raw_body
if char_set:
try:
raw_body = raw_body.decode(char_set)
except AttributeError:
pass
except LookupError: # If no encoding exists, default to UTF-8
try:
raw_body = raw_body.decode('UTF-8')
except AttributeError:
pass
except UnicodeDecodeError:
raw_body = ''
except Exception as e:
Logger.error(
'Unable to decode request body using char_set %s due to error: %s. Will ignore. Stacktrace:'
% (char_set, e))
traceback.print_exc()
else:
# Default to an attempt at UTF-8 decoding.
try:
raw_body = raw_body.decode('UTF-8')
except AttributeError:
pass
except UnicodeDecodeError:
raw_body = ''
max_size = SilkyConfig().SILKY_MAX_REQUEST_BODY_SIZE
body = ''
if raw_body:
if max_size > -1:
Logger.debug('A max request size is set so checking size')
size = sys.getsizeof(raw_body, default=None)
request_identifier = self.request.path
if not size:
Logger.error(
'No way in which to get size of request body for %s, will ignore it',
request_identifier)
elif size <= max_size:
Logger.debug(
'Request %s has body of size %d which is less than %d so will save the body'
% (request_identifier, size, max_size))
body = self._body(raw_body, content_type)
else:
Logger.debug(
'Request %s has body of size %d which is greater than %d, therefore ignoring'
% (request_identifier, size, max_size))
raw_body = None
else:
Logger.debug(
'No maximum request body size is set, continuing.')
body = self._body(raw_body, content_type)
body = self._mask_credentials(body)
raw_body = self._mask_credentials(raw_body)
return body, raw_body
def setUpClass(cls):
super(TestViewClearDB, cls).setUpClass()
SilkyConfig().SILKY_AUTHENTICATION = False
SilkyConfig().SILKY_AUTHORISATION = False
def _should_display_file_name(file_name):
for ignored_file in SilkyConfig().SILKY_IGNORE_FILES:
if ignored_file in file_name:
return False
return True
def test_no_mappings(self):
middleware = SilkyMiddleware()
SilkyConfig().SILKY_DYNAMIC_PROFILING = []
middleware._apply_dynamic_mappings() # Just checking no crash
def configure(self, request=None):
self.request = request
self._configure()
if SilkyConfig().SILKY_PYTHON_PROFILER:
self.pythonprofiler = cProfile.Profile()
self.pythonprofiler.enable()
Logger = logging.getLogger('silk.middleware')
def silky_reverse(name, *args, **kwargs):
try:
r = reverse('silk:%s' % name, *args, **kwargs)
except NoReverseMatch:
# In case user forgets to set namespace, but also fixes Django 1.5 tests on Travis
# Hopefully if user has forgotten to add namespace there are no clashes with their own
# view names but I don't think there is really anything can do about this.
r = reverse(name, *args, **kwargs)
return r
fpath = silky_reverse('summary')
config = SilkyConfig()
def _should_intercept(request):
"""we want to avoid recording any requests/sql queries etc that belong to Silky"""
# Check custom intercept logic.
if config.SILKY_INTERCEPT_FUNC:
if not config.SILKY_INTERCEPT_FUNC(request):
return False
# don't trap every request
elif config.SILKY_INTERCEPT_PERCENT < 100:
if random.random() > config.SILKY_INTERCEPT_PERCENT / 100.0:
return False
silky = request.path.startswith(fpath)
ignored = request.path in config.SILKY_IGNORE_PATHS
class SilkyMiddleware(MiddlewareMixin):
edit_request_model_function = SilkyConfig().SILKY_EDIT_REQUEST_MODEL_FUNCTION
def _apply_dynamic_mappings(self):
dynamic_profile_configs = config.SILKY_DYNAMIC_PROFILING
for conf in dynamic_profile_configs:
module = conf.get('module')
function = conf.get('function')
start_line = conf.get('start_line')
end_line = conf.get('end_line')
name = conf.get('name')
if module and function:
if start_line and end_line: # Dynamic context manager
dynamic.inject_context_manager_func(module=module,
func=function,
start_line=start_line,
end_line=end_line,
name=name)
else: # Dynamic decorator
dynamic.profile_function_or_method(module=module,
func=function,
name=name)
else:
raise KeyError('Invalid dynamic mapping %s' % conf)
@silk_meta_profiler()
def process_request(self, request):
DataCollector().clear()
if not _should_intercept(request):
return
Logger.debug('process_request')
request.silk_is_intercepted = True
self._apply_dynamic_mappings()
if not hasattr(SQLCompiler, '_execute_sql'):
SQLCompiler._execute_sql = SQLCompiler.execute_sql
SQLCompiler.execute_sql = execute_sql
request_model = RequestModelFactory(request).construct_request_model()
DataCollector().configure(request_model)
@transaction.atomic()
def _process_response(self, request, response):
Logger.debug('Process response')
with silk_meta_profiler():
collector = DataCollector()
collector.stop_python_profiler()
silk_request = collector.request
if silk_request:
silk_request = self.edit_request_model_function(silk_request, request)
silk_response = ResponseModelFactory(response).construct_response_model()
silk_response.save()
silk_request.end_time = timezone.now()
collector.finalise()
else:
Logger.error(
'No request model was available when processing response. '
'Did something go wrong in process_request/process_view?'
'\n' + str(request) + '\n\n' + str(response)
)
# Need to save the data outside the silk_meta_profiler
# Otherwise the meta time collected in the context manager
# is not taken in account
if silk_request:
silk_request.save()
Logger.debug('Process response done.')
def process_response(self, request, response):
if getattr(request, 'silk_is_intercepted', False):
while True:
try:
self._process_response(request, response)
except (AttributeError, DatabaseError):
Logger.debug('Retrying _process_response')
self._process_response(request, response)
finally:
break
return response
