def body(self): body = '' content_type, char_set = _parse_content_type( self.response.get('Content-Type', '')) content = getattr(self.response, 'content', '') if content: max_body_size = SilkyConfig().SILKY_MAX_RESPONSE_BODY_SIZE if max_body_size > -1: Logger.debug('Max size of response body defined so checking') size = sys.getsizeof(content, None) if not size: Logger.error( 'Could not get size of response body. Ignoring') content = '' else: if size > max_body_size: content = '' Logger.debug( 'Size of %d for %s is bigger than %d so ignoring response body' % (size, self.request.path, max_body_size)) else: Logger.debug( 'Size of %d for %s is less than %d so saving response body' % (size, self.request.path, max_body_size)) if content and content_type in content_types_json: # TODO: Perhaps theres a way to format the JSON without parsing it? if not isinstance(content, str): # byte string is not compatible with json.loads(...) # and json.dumps(...) in python3 content = content.decode() try: body = json.dumps(json.loads(content), sort_keys=True, indent=4) except (TypeError, ValueError): Logger.warn( 'Response to request with pk %s has content type %s but was unable to parse it' % (self.request.pk, content_type)) return body, content
def encoded_headers(self): """ From Django docs (https://docs.djangoproject.com/en/1.6/ref/request-response/#httprequest-objects): "With the exception of CONTENT_LENGTH and CONTENT_TYPE, as given above, any HTTP headers in the request are converted to META keys by converting all characters to uppercase, replacing any hyphens with underscores and adding an HTTP_ prefix to the name. So, for example, a header called X-Bender would be mapped to the META key HTTP_X_BENDER." """ headers = {} for k, v in self.request.META.items(): if k.startswith('HTTP') or k in ('CONTENT_TYPE', 'CONTENT_LENGTH'): splt = k.split('_') if splt[0] == 'HTTP': splt = splt[1:] k = '-'.join(splt) headers[k] = v if SilkyConfig().SILKY_HIDE_COOKIES: try: del headers['COOKIE'] except KeyError: pass return json.dumps(headers)
class SilkDBRouter: silk_database = SilkyConfig().SILKY_DATABASE_NAME logger.debug(f"Using silk_database '{silk_database}'") def db_for_read(self, model, **hints): if model._meta.app_label == "silk": return self.silk_database # No opinion return None def db_for_write(self, model, **hints): if model._meta.app_label == "silk": return self.silk_database # No opinion return None def allow_relation(self, obj1, obj2, **hints): if obj1._meta.app_label == obj2._meta.app_label == "silk": return True # No opinion return None def allow_migrate(self, db, app_label, model_name=None, **hints): # Not a relevant app, no opinion if app_label != "silk": return None # Allow silk models only on specified database if db == self.silk_database: return True # Silk model on database that was not specified: disagree return False
def body(self): body = '' content_type, char_set = _parse_content_type( self.response.get('Content-Type', '')) if self.response.streaming: content = self.response.streaming_content else: content = self.response.content if char_set: try: content = content.decode(char_set) except AttributeError: pass except LookupError: # If no encoding exists, default to UTF-8 try: content = content.decode('UTF-8') except AttributeError: pass except UnicodeDecodeError: content = '' except Exception as e: Logger.error( 'Unable to decode response body using char_set %s due to error: %s. Will ignore. Stacktrace:' % (char_set, e)) traceback.print_exc() else: # Default to an attempt at UTF-8 decoding. try: content = content.decode('UTF-8') except AttributeError: pass except UnicodeDecodeError: content = '' if content: max_body_size = SilkyConfig().SILKY_MAX_RESPONSE_BODY_SIZE if max_body_size > -1: Logger.debug('Max size of response body defined so checking') if isinstance(content, collections.Iterable): size = 0 #cannot compute size on StreamingHttpResponse else: size = sys.getsizeof(content, None) raise Exception("SIZE=%d" % size) if not size: Logger.error( 'Could not get size of response body. Ignoring') content = '' else: if size > max_body_size: content = '' Logger.debug( 'Size of %d for %s is bigger than %d so ignoring response body' % (size, self.request.path, max_body_size)) else: Logger.debug( 'Size of %d for %s is less than %d so saving response body' % (size, self.request.path, max_body_size)) if content_type in content_types_json: # TODO: Perhaps theres a way to format the JSON without parsing it? try: body = json.dumps(json.loads(content), sort_keys=True, indent=4) except (TypeError, ValueError): Logger.warn( 'Response to request with pk %s has content type %s but was unable to parse it' % (self.request.pk, content_type)) return body, content
def _should_meta_profile(self): return SilkyConfig().SILKY_META
def setUpClass(cls): super().setUpClass() SilkyConfig().SILKY_AUTHENTICATION = False SilkyConfig().SILKY_AUTHORISATION = False
from django.core.files.storage import get_storage_class from django.db import models from django.db.models import (DateTimeField, TextField, CharField, ForeignKey, IntegerField, BooleanField, F, ManyToManyField, OneToOneField, FloatField, FileField) from django.utils import timezone from django.db import transaction from uuid import uuid4 import sqlparse from django.utils.safestring import mark_safe from silk.utils.profile_parser import parse_profile from silk.config import SilkyConfig silk_storage = get_storage_class(SilkyConfig().SILKY_STORAGE_CLASS)() # Seperated out so can use in tests w/o models def _time_taken(start_time, end_time): d = end_time - start_time return d.seconds * 1000 + d.microseconds / 1000 def time_taken(self): return _time_taken(self.start_time, self.end_time) class CaseInsensitiveDictionary(dict): def __getitem__(self, key): return super(CaseInsensitiveDictionary, self).__getitem__(key.lower())
def test_enabled(self): SilkyConfig().SILKY_META = True r = self._execute_request() self.assertTrue(r.meta_time is not None or r.meta_num_queries is not None or r.meta_time_spent_queries is not None)
def setUpClass(cls): super().setUpClass() BlindFactory.create_batch(size=5) SilkyConfig().SILKY_META = False SilkyConfig().SILKY_ANALYZE_QUERIES = True
def tearDown(self): SilkyConfig().SILKY_SENSITIVE_KEYS = DEFAULT_SENSITIVE_KEYS
def test_mask_credentials_masks_sensitive_values_listed_in_settings(self): SilkyConfig().SILKY_SENSITIVE_KEYS = {"foo"} body = "foo=hidethis" expected = f"foo={CLEANSED}" self.assertEqual(expected, self._mask(body))
def test_mask_credentials_masks_sensitive_values_listed_in_settings(self): SilkyConfig().SILKY_SENSITIVE_KEYS = {"foo"} self.assertNotIn("hidethis", self._mask({"foo": "hidethis"}))
def __init__(self): super().__init__( location=SilkyConfig().SILKY_PYTHON_PROFILER_RESULT_PATH, base_url='') self.base_url = None
def __init__(self): super(ProfilerResultStorage, self).__init__( location=SilkyConfig().SILKY_PYTHON_PROFILER_RESULT_PATH, base_url='') self.base_url = None
def tearDown(self): SilkyConfig( ).SILKY_MAX_RECORDED_REQUESTS_CHECK_PERCENT = self.max_percent SilkyConfig().SILKY_MAX_RECORDED_REQUESTS = self.max_requests
def setUp(self): self.obj = RequestMinFactory.create() self.max_percent = SilkyConfig( ).SILKY_MAX_RECORDED_REQUESTS_CHECK_PERCENT self.max_requests = SilkyConfig().SILKY_MAX_RECORDED_REQUESTS
def _should_wrap(sql_query): for ignore_str in SilkyConfig().SILKY_IGNORE_QUERIES: if ignore_str in sql_query: return False return True
def test_config_with_no_value_should_have_default(self): config = SilkyConfig() self.assertEqual(config.SILKY_RESPONSE_MODEL_FACTORY_CLASS, 'silk.model_factory.ResponseModelFactory')
def tearDownClass(cls): super().tearDownClass() SilkyConfig().SILKLY_ANALYZE_QUERIES = False
def test_config_with_no_value_should_have_response_model_factory_property( self): config = SilkyConfig() self.assertTrue(hasattr(config, 'response_model_factory'))
def test_disabled(self): SilkyConfig().SILKY_META = False r = self._execute_request() self.assertFalse(r.meta_time)
def test_config_with_custom_class_should_load_custom_class(self): config = SilkyConfig() self.assertIs(config.response_model_factory, DummyResponseModelFactory)
def login_possibly_required(function=None, **kwargs): if SilkyConfig().SILKY_AUTHENTICATION: return login_required(function, **kwargs) return function
def body(self): content_type, char_set = self.content_type() try: raw_body = self.request.body except RequestDataTooBig: raw_body = b"Raw body exceeds DATA_UPLOAD_MAX_MEMORY_SIZE, Silk is not showing file uploads." body = self.request.POST.copy() for k, v in self.request.FILES.items(): body.appendlist(k, v) return body, raw_body if char_set: try: raw_body = raw_body.decode(char_set) except AttributeError: pass except LookupError: # If no encoding exists, default to UTF-8 try: raw_body = raw_body.decode('UTF-8') except AttributeError: pass except UnicodeDecodeError: raw_body = '' except Exception as e: Logger.error( 'Unable to decode request body using char_set %s due to error: %s. Will ignore. Stacktrace:' % (char_set, e)) traceback.print_exc() else: # Default to an attempt at UTF-8 decoding. try: raw_body = raw_body.decode('UTF-8') except AttributeError: pass except UnicodeDecodeError: raw_body = '' max_size = SilkyConfig().SILKY_MAX_REQUEST_BODY_SIZE body = '' if raw_body: if max_size > -1: Logger.debug('A max request size is set so checking size') size = sys.getsizeof(raw_body, default=None) request_identifier = self.request.path if not size: Logger.error( 'No way in which to get size of request body for %s, will ignore it', request_identifier) elif size <= max_size: Logger.debug( 'Request %s has body of size %d which is less than %d so will save the body' % (request_identifier, size, max_size)) body = self._body(raw_body, content_type) else: Logger.debug( 'Request %s has body of size %d which is greater than %d, therefore ignoring' % (request_identifier, size, max_size)) raw_body = None else: Logger.debug( 'No maximum request body size is set, continuing.') body = self._body(raw_body, content_type) body = self._mask_credentials(body) raw_body = self._mask_credentials(raw_body) return body, raw_body
def setUpClass(cls): super(TestViewClearDB, cls).setUpClass() SilkyConfig().SILKY_AUTHENTICATION = False SilkyConfig().SILKY_AUTHORISATION = False
def _should_display_file_name(file_name): for ignored_file in SilkyConfig().SILKY_IGNORE_FILES: if ignored_file in file_name: return False return True
def test_no_mappings(self): middleware = SilkyMiddleware() SilkyConfig().SILKY_DYNAMIC_PROFILING = [] middleware._apply_dynamic_mappings() # Just checking no crash
def configure(self, request=None): self.request = request self._configure() if SilkyConfig().SILKY_PYTHON_PROFILER: self.pythonprofiler = cProfile.Profile() self.pythonprofiler.enable()
Logger = logging.getLogger('silk.middleware') def silky_reverse(name, *args, **kwargs): try: r = reverse('silk:%s' % name, *args, **kwargs) except NoReverseMatch: # In case user forgets to set namespace, but also fixes Django 1.5 tests on Travis # Hopefully if user has forgotten to add namespace there are no clashes with their own # view names but I don't think there is really anything can do about this. r = reverse(name, *args, **kwargs) return r fpath = silky_reverse('summary') config = SilkyConfig() def _should_intercept(request): """we want to avoid recording any requests/sql queries etc that belong to Silky""" # Check custom intercept logic. if config.SILKY_INTERCEPT_FUNC: if not config.SILKY_INTERCEPT_FUNC(request): return False # don't trap every request elif config.SILKY_INTERCEPT_PERCENT < 100: if random.random() > config.SILKY_INTERCEPT_PERCENT / 100.0: return False silky = request.path.startswith(fpath) ignored = request.path in config.SILKY_IGNORE_PATHS
class SilkyMiddleware(MiddlewareMixin): edit_request_model_function = SilkyConfig().SILKY_EDIT_REQUEST_MODEL_FUNCTION def _apply_dynamic_mappings(self): dynamic_profile_configs = config.SILKY_DYNAMIC_PROFILING for conf in dynamic_profile_configs: module = conf.get('module') function = conf.get('function') start_line = conf.get('start_line') end_line = conf.get('end_line') name = conf.get('name') if module and function: if start_line and end_line: # Dynamic context manager dynamic.inject_context_manager_func(module=module, func=function, start_line=start_line, end_line=end_line, name=name) else: # Dynamic decorator dynamic.profile_function_or_method(module=module, func=function, name=name) else: raise KeyError('Invalid dynamic mapping %s' % conf) @silk_meta_profiler() def process_request(self, request): DataCollector().clear() if not _should_intercept(request): return Logger.debug('process_request') request.silk_is_intercepted = True self._apply_dynamic_mappings() if not hasattr(SQLCompiler, '_execute_sql'): SQLCompiler._execute_sql = SQLCompiler.execute_sql SQLCompiler.execute_sql = execute_sql request_model = RequestModelFactory(request).construct_request_model() DataCollector().configure(request_model) @transaction.atomic() def _process_response(self, request, response): Logger.debug('Process response') with silk_meta_profiler(): collector = DataCollector() collector.stop_python_profiler() silk_request = collector.request if silk_request: silk_request = self.edit_request_model_function(silk_request, request) silk_response = ResponseModelFactory(response).construct_response_model() silk_response.save() silk_request.end_time = timezone.now() collector.finalise() else: Logger.error( 'No request model was available when processing response. ' 'Did something go wrong in process_request/process_view?' '\n' + str(request) + '\n\n' + str(response) ) # Need to save the data outside the silk_meta_profiler # Otherwise the meta time collected in the context manager # is not taken in account if silk_request: silk_request.save() Logger.debug('Process response done.') def process_response(self, request, response): if getattr(request, 'silk_is_intercepted', False): while True: try: self._process_response(request, response) except (AttributeError, DatabaseError): Logger.debug('Retrying _process_response') self._process_response(request, response) finally: break return response