def test_timing_request(self): def app(environ, start_response): start_response('200 OK', [('Content-Type', 'text/html')]) @time_trace(name='foo_func', min_duration=0.1) def foo(arg): time.sleep(0.2) return arg foo('a') time.sleep(0.1) try: import psycopg2 except ImportError: return conn = psycopg2.connect( "user=test host=127.0.0.1 dbname=test password=test") c = conn.cursor() psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, c) c.execute('SELECT 1, pg_sleep(0.5)') c.fetchone() c.close() conn.close() return ['Hello World!'] get_local_storage(local_timing).clear() req = Request.blank('http://localhost/test') app = make_appenlight_middleware(app, global_config=timing_conf) req.get_response(app) stats, result = get_local_storage(local_timing).get_thread_stats() self.assertGreater(stats['main'], 0) self.assertGreater(stats['sql'], 0)
def process_exception(self, request, exception): if (not getattr(self, 'appenlight_client') or not self.appenlight_client.config.get('enabled')): return None environ = getattr(request, 'environ', request.META) if not self.appenlight_client.config['report_errors'] \ or environ.get('appenlight.ignore_error'): return None user = getattr(request, 'user', None) end_time = default_timer() if user and user_is_authenticated(user): if 'appenlight.username' not in environ: environ['appenlight.username'] = six.text_type(user.pk) if not isinstance(exception, Http404): http_status = 500 traceback = self.appenlight_client.get_current_traceback() appenlight_storage = get_local_storage() appenlight_storage.thread_stats[ 'main'] = end_time - request.__start_time__ stats, slow_calls = appenlight_storage.get_thread_stats() self.appenlight_client.save_request_stats( stats, view_name=environ.get('appenlight.view_name', '')) self.appenlight_client.py_report( environ, traceback, message=None, http_status=http_status, start_time=datetime.utcfromtimestamp( request.__start_time__), end_time=datetime.utcfromtimestamp( end_time), request_stats=stats, slow_calls=slow_calls) del traceback request._errormator_create_report = True
def view_callable_wrapper(context, request): appenlight_storage = get_local_storage() view_name = '' try: original_view = getattr(appenlight_callable, '__original_view__') if original_view: view_name = fullyQualifiedName(appenlight_callable) # fix the problem with bound methods, # we have to attach the resolved view name somehow. if is_bound_method(original_view): _original_view = original_view def original_view(context, request): return _original_view(context, request) if not hasattr(original_view, '_appenlight_name'): original_view._appenlight_name = view_name except Exception: raise if 'pyramid/static' in view_name: # normalize static views view_name = 'pyramid/static' if not getattr(appenlight_storage, 'view_name', None): appenlight_storage.view_name = view_name return appenlight_callable(context, request)
def test_httplib(self): import httplib h2 = httplib.HTTPConnection("www.ubuntu.com") h2.request("GET", "/") stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_urllib_URLOpener_open(self): import urllib opener = urllib.URLopener() opener.open("http://www.ubuntu.com/") stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def process_exception(self, request, exception): if (not getattr(self, 'appenlight_client') or not self.appenlight_client.config.get('enabled')): return None environ = request.environ if not self.appenlight_client.config['report_errors'] or environ.get('appenlight.ignore_error'): return None user = getattr(request, 'user', None) end_time = default_timer() if user and user.is_authenticated(): environ['appenlight.username'] = unicode(user.pk) http_status = 500 request._errormator_create_report = True traceback = get_current_traceback(skip=1, show_hidden_frames=True, ignore_system_exceptions=True) appenlight_storage = get_local_storage(local_timing) appenlight_storage.thread_stats['main'] = end_time - request.__start_time__ stats, slow_calls = appenlight_storage.get_thread_stats() self.appenlight_client.save_request_stats(stats, view_name=environ.get('appenlight.view_name','')) self.appenlight_client.py_report(environ, traceback, message=None, http_status=http_status, start_time=datetime.utcfromtimestamp(request.__start_time__), end_time=datetime.utcfromtimestamp(end_time), request_stats=stats, slow_calls=slow_calls) del traceback
def process_request(self, request): request._errormator_create_report = False request.__traceback__ = None environ = getattr(request, 'environ', request.META) ignored_slow_paths = self.appenlight_client.config.get( 'ignore_slow_paths', []) if any(p in request.path for p in ignored_slow_paths): log.debug('appenlight.ignore_slow_path in effect') environ['appenlight.ignore_slow'] = True ignored_paths = self.appenlight_client.config.get('ignore_paths', []) if any(p in request.path for p in ignored_paths): log.debug('appenlight.ignore_path in effect') environ['appenlight.ignore_error'] = True environ['appenlight.request_id'] = str(uuid.uuid4()) # inject client instance reference to environ if 'appenlight.client' not in environ: environ['appenlight.client'] = self.appenlight_client if 'appenlight.tags' not in environ: environ['appenlight.tags'] = {} if 'appenlight.extra' not in environ: environ['appenlight.extra'] = {} environ['appenlight.post_vars'] = request.POST appenlight_storage = get_local_storage() # clear out thread stats on request start appenlight_storage.clear() request.__start_time__ = default_timer() return None
def emit(self, record): appenlight_storage = get_local_storage() if len(appenlight_storage.logs) >= self.max_logs: return r_dict = convert_record_to_dict(record, self.client_config) if r_dict: if r_dict not in appenlight_storage.logs: appenlight_storage.logs.append(r_dict)
def emit(self, record): appenlight_storage = get_local_storage() if len(appenlight_storage.logs) >= self.max_logs: return r_dict = convert_record_to_dict(record, self.ae_client_config) if r_dict: if r_dict not in appenlight_storage.logs: appenlight_storage.logs.append(r_dict)
def test_requests(self): try: import requests except ImportError: return requests.get("http://www.ubuntu.com/") stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_urllib3(self): try: import urllib3 except ImportError: return http = urllib3.PoolManager() http.request('GET', "http://www.ubuntu.com/") stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_memcache_command(self): try: import memcache except ImportError: return mc = memcache.Client(['127.0.0.1:11211'], debug=0) mc.set("some_key", "Some value") value = mc.get("some_key") stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_redis_py_command(self): try: import redis except ImportError: return client = redis.StrictRedis() client.setex('testval', 10, 'foo') stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def process_response(self, request, response): try: return response finally: environ = getattr(request, 'environ', request.META) enabled = self.appenlight_client.config.get('enabled') if enabled and not request._errormator_create_report and not environ.get( 'appenlight.ignore_slow'): end_time = default_timer() user = getattr(request, 'user', None) http_status = response.status_code if user and user_is_authenticated(user): if 'appenlight.username' not in environ: environ['appenlight.username'] = six.text_type(user.pk) if (http_status == 404 and self.appenlight_client.config[ 'report_404']): request._errormator_create_report = True delta = timedelta(seconds=(end_time - request.__start_time__)) appenlight_storage = get_local_storage() appenlight_storage.thread_stats[ 'main'] = end_time - request.__start_time__ stats, slow_calls = appenlight_storage.get_thread_stats() self.appenlight_client.save_request_stats( stats, view_name=environ.get('appenlight.view_name', '')) if self.appenlight_client.config['slow_requests']: if (delta >= self.appenlight_client.config[ 'slow_request_time'] or slow_calls): request._errormator_create_report = True if request._errormator_create_report: self.appenlight_client.py_report( environ, None, message=None, http_status=http_status, start_time=datetime.utcfromtimestamp( request.__start_time__), end_time=datetime.utcfromtimestamp(end_time), request_stats=stats, slow_calls=slow_calls) if self.appenlight_client.config['logging']: records = self.appenlight_client.log_handlers_get_records() self.appenlight_client.log_handlers_clear_records() self.appenlight_client.py_log( environ, records=records, r_uuid=environ[ 'appenlight.request_id'], created_report=request._errormator_create_report) if self.appenlight_client.config.get('enabled'): self.appenlight_client.check_if_deliver( self.appenlight_client.config['force_send'] or environ.get('appenlight.force_send'))
def test_memcache_command(self): # TODO: not finished return try: import pylibmc except ImportError: return mc = pylibmc.Client(['127.0.0.1:11211']) mc.set("some_key", "Some value") value = mc.get("some_key") stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def view_callable_wrapper(*args, **kwargs): appenlight_storage = get_local_storage() if hasattr(appenlight_storage, 'view_name'): try: split_name = appenlight_storage.view_name.split(':') # only change the name if it wasn't resolved yet if len(split_name) == 2 and '.' not in split_name[1]: appenlight_storage.view_name = '%s.%s' % (appenlight_storage.view_name, appenlight_callable.__name__) except Exception: pass return appenlight_callable(*args, **kwargs)
def test_sqlite(self): try: import sqlite3 except ImportError: return conn = sqlite3.connect(':memory:') c = conn.cursor() c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_postgresql(self): try: import postgresql except ImportError: return conn = postgresql.open('pq://*****:*****@localhost') c = conn.cursor() c.execute(self.stmt) c.fetchone()[0] c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_oursql(self): try: import oursql except ImportError: return conn = oursql.connect(passwd="test", user="******") c = conn.cursor(oursql.DictCursor) c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_pymysql(self): try: import pymysql except ImportError: return conn = pymysql.connect(host='127.0.0.1', user='******', passwd='test') c = conn.cursor() c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_pg8000(self): try: import pg8000 except ImportError: return conn = pg8000.DBAPI.connect(host="127.0.0.1", user="******", password="******") c = conn.cursor() c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def process_request(self, request): request._errormator_create_report = False request.__traceback__ = None environ = request.environ environ['appenlight.request_id'] = str(uuid.uuid4()) # inject client instance reference to environ if 'appenlight.client' not in environ: environ['appenlight.client'] = self.appenlight_client environ['appenlight.post_vars'] = request.POST appenlight_storage = get_local_storage(local_timing) # clear out thread stats on request start appenlight_storage.clear() request.__start_time__ = default_timer() return None
def test_render(self): try: import chameleon.zpt except ImportError: return import time template = chameleon.zpt.PageTemplate(''' ${sleep(0.06)} xxxxx ${1+2} yyyyyy ''') template.render(sleep=time.sleep) stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_odbc(self): try: import pyodbc except ImportError: return conn = pyodbc.connect( 'Driver={MySQL};Server=127.0.0.1;Port=3306;Database=information_schema;User=test; Password=test;Option=3;') c = conn.cursor() c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_mysqldb(self): try: import MySQLdb except ImportError: return conn = MySQLdb.connect(passwd="test", user="******", host="127.0.0.1", port=3306) c = conn.cursor() c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_render(self): try: import jinja2 except ImportError: return import time template = jinja2.Template(''' {{sleep(0.06)}} xxxxx {{1+2}} yyyyyy ''') template.render(sleep=time.sleep) stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_psycopg2(self): try: import psycopg2 except ImportError: return conn = psycopg2.connect( "user=test host=127.0.0.1 dbname=test password=test") c = conn.cursor() psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, c) c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_render_unicode(self): try: import mako except ImportError: return template = mako.template.Template(u''' <% import time time.sleep(0.01) %> xxxxx ${1+2} yyyyyy ''') template.render_unicode() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def gather_data(client, environ=None, gather_exception=True, gather_slowness=True, gather_logs=True, clear_storage=True, exc_info=None, start_time=None, end_time=None): """ exc_info is supposed to be (exc_type, exc_value, tb) - what sys.exc_info() returns """ if not client.config['enabled']: return None if environ is None: environ = {} if not environ.get('appenlight.request_id'): environ['appenlight.request_id'] = str(uuid.uuid4()) http_status = 200 traceback = None if gather_exception and not exc_info: traceback = client.get_current_traceback() if traceback: http_status = 500 elif exc_info: traceback = Traceback(*exc_info) http_status = 500 appenlight_storage = get_local_storage(local_timing) stats, slow_calls = appenlight_storage.get_thread_stats() if traceback is not None or (slow_calls and gather_slowness): client.py_report(environ, traceback, http_status=http_status, request_stats=stats, slow_calls=slow_calls, start_time=start_time, end_time=end_time) # dereference del traceback if client.config['logging']: if gather_logs: records = client.log_handlers_get_records() client.log_handlers_clear_records() client.py_log(environ, records=records, r_uuid=environ['appenlight.request_id']) if clear_storage: appenlight_storage.clear() client.check_if_deliver(client.config['force_send'] or environ.get('appenlight.force_send'))
def view_callable_wrapper(context, request): appenlight_storage = get_local_storage() view_name = '' try: original_view = getattr(appenlight_callable, '__original_view__') if original_view: view_name = fullyQualifiedName(appenlight_callable) if not hasattr(original_view, '_appenlight_name'): original_view._appenlight_name = view_name except Exception: raise if 'pyramid/static' in view_name: # normalize static views view_name = 'pyramid/static' if not getattr(appenlight_storage, 'view_name', None): appenlight_storage.view_name = view_name return appenlight_callable(context, request)
def test_template_lookup(self): try: from mako.lookup import TemplateLookup except ImportError: return lookup = TemplateLookup() lookup.put_string("base.html", ''' <% import time time.sleep(0.01) %> <html><body></body></html> ''') template = lookup.get_template("base.html") template.render_unicode() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_render_call_number(self): try: import mako except ImportError: return template = mako.template.Template(''' <% import time time.sleep(0.01) %> xxxxx ${1+2} yyyyyy ''') template.render() template.render() template.render() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(stats['tmpl_calls'], 3)
def test_render(self): try: from django import template except ImportError: return from django.conf import settings settings.configure(TEMPLATE_DIRS=("/whatever/templates",)) import time ctx = template.Context() ctx.update({'time': lambda: time.sleep(0.06)}) template = template.Template(''' xxxxx {{ time }} yyyyyy ''') template.render(ctx) stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def process_request(self, request): request._errormator_create_report = False request.__traceback__ = None environ = request.environ environ['appenlight.request_id'] = str(uuid.uuid4()) # inject client instance reference to environ if 'appenlight.client' not in environ: environ['appenlight.client'] = self.appenlight_client if 'appenlight.tags' not in environ: environ['appenlight.tags'] = {} if 'appenlight.extra' not in environ: environ['appenlight.extra'] = {} environ['appenlight.post_vars'] = request.POST appenlight_storage = get_local_storage() # clear out thread stats on request start appenlight_storage.clear() request.__start_time__ = default_timer() return None
def clear_records(self, thread=None): """ Clears ALL logs from AE storage """ appenlight_storage = get_local_storage() appenlight_storage.logs = []
def get_records(self, thread=None): """ Returns a list of records for the current thread. """ appenlight_storage = get_local_storage() return appenlight_storage.logs
def emit(self, record): appenlight_storage = get_local_storage() r_dict = convert_record_to_dict(record, self.ae_client_config) if r_dict: if r_dict not in appenlight_storage.logs: appenlight_storage.logs.append(r_dict)
import logging import socket import pkg_resources import time from appenlight_client import client, make_appenlight_middleware from appenlight_client.exceptions import get_current_traceback from appenlight_client.logger import register_logging from appenlight_client.wsgi import AppenlightWSGIWrapper timing_conf = client.get_config({'appenlight.api_key': '1234'}) for k, v in timing_conf.iteritems(): if 'appenlight.timing' in k: timing_conf[k] = 0.00000001 client.Client(config=timing_conf) from appenlight_client.timing import local_timing, get_local_storage import timeit import jinja2 print 'traced', hasattr(jinja2.Template.render, '_e_attached_tracer') s = """ template = jinja2.Template('''xxxxx {{1+2}} yyyyyy''') template.render() """ print 'time', timeit.timeit(stmt=s, number=1500, setup="import jinja2") stats, slow_calls = get_local_storage(local_timing).get_thread_stats() print 'calls', len(slow_calls) print 'stats', stats
def log_handlers_get_records(self): appenlight_storage = get_local_storage() return appenlight_storage.logs
def log_handlers_clear_records(self): appenlight_storage = get_local_storage() appenlight_storage.logs = []
def __call__(self, environ, start_response): """Run the application and conserve the traceback frames. also determine if we got 404 """ environ['appenlight.request_id'] = str(uuid.uuid4()) appenlight_storage = get_local_storage() # clear out thread stats on request start appenlight_storage.clear() app_iter = None detected_data = [] create_report = False traceback = None http_status = 200 start_time = default_timer() def detect_headers(status, headers, *k, **kw): detected_data[:] = status[:3], headers return start_response(status, headers, *k, **kw) # inject client instance reference to environ if 'appenlight.client' not in environ: environ['appenlight.client'] = self.appenlight_client # some bw. compat stubs def local_report(message, include_traceback=True, http_status=200): environ['appenlight.force_send'] = True def local_log(level, message): environ['appenlight.force_send'] = True environ['appenlight.report'] = local_report environ['appenlight.log'] = local_log if 'appenlight.tags' not in environ: environ['appenlight.tags'] = {} if 'appenlight.extra' not in environ: environ['appenlight.extra'] = {} try: app_iter = self.app(environ, detect_headers) return app_iter except Exception: if hasattr(app_iter, 'close'): app_iter.close() # we need that here traceback = self.appenlight_client.get_current_traceback() # by default reraise exceptions for app/FW to handle if self.appenlight_client.config['reraise_exceptions']: raise try: start_response('500 INTERNAL SERVER ERROR', [('Content-Type', 'text/html; charset=utf-8')]) except Exception: environ['wsgi.errors'].write( 'AppenlightWSGIWrapper middleware catched exception ' 'in streamed response at a point where response headers ' 'were already sent.\n') else: return 'Server Error' finally: # report 500's and 404's # report slowness end_time = default_timer() appenlight_storage.thread_stats['main'] = end_time - start_time delta = datetime.timedelta(seconds=(end_time - start_time)) stats, slow_calls = appenlight_storage.get_thread_stats() if 'appenlight.view_name' not in environ: environ['appenlight.view_name'] = getattr( appenlight_storage, 'view_name', '') if detected_data and detected_data[0]: http_status = int(detected_data[0]) if self.appenlight_client.config[ 'slow_requests'] and not environ.get( 'appenlight.ignore_slow'): # do we have slow calls/request ? if (delta >= self.appenlight_client.config['slow_request_time'] or slow_calls): create_report = True if 'appenlight.__traceback' in environ and not environ.get( 'appenlight.ignore_error'): # get traceback gathered by pyramid tween traceback = environ['appenlight.__traceback'] del environ['appenlight.__traceback'] http_status = 500 create_report = True if traceback and self.appenlight_client.config[ 'report_errors'] and not environ.get( 'appenlight.ignore_error'): http_status = 500 create_report = True elif (self.appenlight_client.config['report_404'] and http_status == 404): create_report = True if create_report: self.appenlight_client.py_report( environ, traceback, message=None, http_status=http_status, start_time=datetime.datetime.utcfromtimestamp(start_time), end_time=datetime.datetime.utcfromtimestamp(end_time), request_stats=stats, slow_calls=slow_calls) # dereference del traceback self.appenlight_client.save_request_stats( stats, view_name=environ.get('appenlight.view_name', '')) if self.appenlight_client.config['logging']: records = self.appenlight_client.log_handlers_get_records() self.appenlight_client.log_handlers_clear_records() self.appenlight_client.py_log( environ, records=records, r_uuid=environ['appenlight.request_id'], created_report=create_report) # send all data we gathered immediately at the end of request self.appenlight_client.check_if_deliver( self.appenlight_client.config['force_send'] or environ.get('appenlight.force_send'))
def purge_data(self): storage = get_local_storage() storage.clear() self.log_handlers_clear_records() self.transport.purge()