def test_urllib_URLOpener_open(self): import urllib opener = urllib.URLopener() opener.open("http://www.ubuntu.com/") stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_httplib(self): import httplib h2 = httplib.HTTPConnection("www.ubuntu.com") h2.request("GET", "/") stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_requests(self): try: import requests except ImportError: return requests.get("http://www.ubuntu.com/") stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_urllib3(self): try: import urllib3 except ImportError: return http = urllib3.PoolManager() http.request('GET', "http://www.ubuntu.com/") stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_redis_py_command(self): try: import redis except ImportError: return client = redis.StrictRedis() client.setex('testval', 10, 'foo') stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_memcache_command(self): try: import memcache except ImportError: return mc = memcache.Client(['127.0.0.1:11211'], debug=0) mc.set("some_key", "Some value") value = mc.get("some_key") stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_memcache_command(self): # TODO: not finished return try: import pylibmc except ImportError: return mc = pylibmc.Client(['127.0.0.1:11211']) mc.set("some_key", "Some value") value = mc.get("some_key") stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_sqlite(self): try: import sqlite3 except ImportError: return conn = sqlite3.connect(':memory:') c = conn.cursor() c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_postgresql(self): try: import postgresql except ImportError: return conn = postgresql.open('pq://*****:*****@localhost') c = conn.cursor() c.execute(self.stmt) c.fetchone()[0] c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_oursql(self): try: import oursql except ImportError: return conn = oursql.connect(passwd="test", user="******") c = conn.cursor(oursql.DictCursor) c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_pymysql(self): try: import pymysql except ImportError: return conn = pymysql.connect(host='127.0.0.1', user='******', passwd='test') c = conn.cursor() c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def process_request(self, request): request.__e_processed_exception__ = False request.__traceback__ = None environ = request.environ environ['errormator.request_id'] = str(uuid.uuid4()) # inject client instance reference to environ if 'errormator.client' not in environ: environ['errormator.client'] = self.errormator_client environ['errormator.post_vars'] = request.POST errormator_storage = get_local_storage(local_timing) # clear out thread stats on request start errormator_storage.clear() request.__start_time__ = default_timer() return None
def test_mysqldb(self): try: import MySQLdb except ImportError: return conn = MySQLdb.connect(passwd="test", user="******", host="127.0.0.1", port=3306) c = conn.cursor() c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_render(self): try: import jinja2 except ImportError: return import time template = jinja2.Template(''' {{sleep(0.06)}} xxxxx {{1+2}} yyyyyy ''') template.render(sleep=time.sleep) stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_render(self): try: import chameleon.zpt except ImportError: return import time template = chameleon.zpt.PageTemplate(''' ${sleep(0.06)} xxxxx ${1+2} yyyyyy ''') template.render(sleep=time.sleep) stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_odbc(self): try: import pyodbc except ImportError: return conn = pyodbc.connect( 'Driver={MySQL};Server=127.0.0.1;Port=3306;Database=information_schema;User=test; Password=test;Option=3;') c = conn.cursor() c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_pg8000(self): try: import pg8000 except ImportError: return conn = pg8000.DBAPI.connect(host="127.0.0.1", user="******", password="******") c = conn.cursor() c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_render_unicode(self): try: import mako except ImportError: return template = mako.template.Template(u''' <% import time time.sleep(0.01) %> xxxxx ${1+2} yyyyyy ''') template.render_unicode() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_psycopg2(self): try: import psycopg2 except ImportError: return conn = psycopg2.connect( "user=test host=127.0.0.1 dbname=test password=test") c = conn.cursor() psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, c) c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def test_odbc(self): try: import pyodbc except ImportError: return conn = pyodbc.connect( 'Driver={MySQL};Server=127.0.0.1;Port=3306;Database=information_schema;User=test; Password=test;Option=3;' ) c = conn.cursor() c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 2)
def process_response(self, request, response): try: return response finally: if self.errormator_client.config.get('enabled'): end_time = default_timer() environ = request.environ user = getattr(request, 'user', None) if user: environ['errormator.username'] = unicode(user.id) if (response.status_code == 404 and not request.__e_processed_exception__): self.process_exception(request, Http404()) delta = timedelta(seconds=(end_time - request.__start_time__)) errormator_storage = get_local_storage(local_timing) errormator_storage.thread_stats[ 'main'] = end_time - request.__start_time__ stats, slow_calls = errormator_storage.get_thread_stats() errormator_storage.clear() # report slowness self.errormator_client.save_request_stats(stats) if self.errormator_client.config['slow_requests']: # do we have slow calls ? if (delta >= self.errormator_client.config['slow_request_time'] or slow_calls): self.errormator_client.py_slow_report( environ, datetime.utcfromtimestamp(request.__start_time__), datetime.utcfromtimestamp(end_time), slow_calls, request_stats=stats) # force log fetching request.__traceback__ = True if self.errormator_client.config['logging']: records = self.errormator_client.log_handler.get_records() self.errormator_client.log_handler.clear_records() self.errormator_client.py_log( environ, records=records, r_uuid=environ['errormator.request_id'], traceback=request.__traceback__) # send all data we gathered immediately at the # end of request self.errormator_client.check_if_deliver( self.errormator_client.config['force_send'] or environ.get('errormator.force_send'))
def process_response(self, request, response): try: return response finally: if self.errormator_client.config.get('enabled'): end_time = default_timer() environ = request.environ user = getattr(request, 'user', None) if user: environ['errormator.username'] = unicode(user.id) if (response.status_code == 404 and not request.__e_processed_exception__): self.process_exception(request, Http404()) delta = timedelta( seconds=(end_time - request.__start_time__)) errormator_storage = get_local_storage(local_timing) errormator_storage.thread_stats[ 'main'] = end_time - request.__start_time__ stats, slow_calls = errormator_storage.get_thread_stats() errormator_storage.clear() # report slowness self.errormator_client.save_request_stats(stats) if self.errormator_client.config['slow_requests']: # do we have slow calls ? if (delta >= self.errormator_client.config[ 'slow_request_time'] or slow_calls): self.errormator_client.py_slow_report( environ, datetime.utcfromtimestamp(request.__start_time__), datetime.utcfromtimestamp(end_time), slow_calls, request_stats=stats) # force log fetching request.__traceback__ = True if self.errormator_client.config['logging']: records = self.errormator_client.log_handler.get_records() self.errormator_client.log_handler.clear_records() self.errormator_client.py_log( environ, records=records, r_uuid=environ['errormator.request_id'], traceback=request.__traceback__) # send all data we gathered immediately at the # end of request self.errormator_client.check_if_deliver( self.errormator_client.config['force_send'] or environ.get('errormator.force_send'))
def gather_data(client, environ, gather_exception=True, gather_slowness=True, gather_logs=True, clear_storage=True, exc_info=None, spawn_thread=True): if client.config['enabled'] == False: return None if not environ.get('wsgi.url_scheme'): environ['wsgi.url_scheme'] = '' if not environ.get('HTTP_HOST'): environ['HTTP_HOST'] = 'localhost' if not environ.get('errormator.request_id'): environ['errormator.request_id'] = str(uuid.uuid4()) if gather_exception and not exc_info: traceback = get_current_traceback(skip=1, show_hidden_frames=True, ignore_system_exceptions=True) elif exc_info: traceback = Traceback(*exc_info) else: traceback = None errormator_storage = get_local_storage(local_timing) stats, slow_calls = errormator_storage.get_thread_stats() if traceback: client.py_report(environ, traceback, http_status=500, request_stats=stats) # dereference del traceback traceback = True # report slowness now = datetime.datetime.utcnow() if clear_storage: errormator_storage.clear() if client.config['slow_requests'] and gather_slowness: # do we have slow calls ? if (slow_calls): client.py_slow_report(environ, now, now, slow_calls, request_stats=stats) # force log fetching traceback = True if client.config['logging'] and gather_logs: records = client.log_handler.get_records() client.log_handler.clear_records() client.py_log(environ, records=records, traceback=traceback, r_uuid=environ['errormator.request_id']) # send all data we gathered immediately at the end of request client.check_if_deliver(client.config['force_send'] or environ.get('errormator.force_send'), spawn_thread=spawn_thread)
def test_render_call_number(self): try: import mako except ImportError: return template = mako.template.Template(''' <% import time time.sleep(0.01) %> xxxxx ${1+2} yyyyyy ''') template.render() template.render() template.render() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(stats['tmpl_calls'], 3)
def test_template_lookup(self): try: from mako.lookup import TemplateLookup except ImportError: return lookup = TemplateLookup() lookup.put_string("base.html", ''' <% import time time.sleep(0.01) %> <html><body></body></html> ''') template = lookup.get_template("base.html") template.render_unicode() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_render(self): try: from django import template except ImportError: return from django.conf import settings settings.configure(TEMPLATE_DIRS=("/whatever/templates",)) import time ctx = template.Context() ctx.update({'time': lambda: time.sleep(0.06)}) template = template.Template(''' xxxxx {{ time }} yyyyyy ''') template.render(ctx) stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def test_sqlite_call_number(self): try: import sqlite3 except ImportError: return conn = sqlite3.connect(':memory:') c = conn.cursor() c.execute(self.stmt) c.fetchone() c.execute(self.stmt) c.fetchone() c.execute(self.stmt) c.fetchone() c.execute(self.stmt) c.fetchone() c.close() conn.close() stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(int(stats['sql_calls']), 4)
def test_timing_request(self): def app(environ, start_response): start_response('200 OK', [('Content-Type', 'text/html')]) time.sleep(0.1) try: import sqlite3 except ImportError: return conn = sqlite3.connect(':memory:') c = conn.cursor() c.execute('''SELECT 1+2+3 AS result''') c.fetchone() c.close() conn.close() return ['Hello World!'] req = Request.blank('http://localhost/test') app = make_errormator_middleware(app, global_config=timing_conf) req.get_response(app) stats, result = get_local_storage(local_timing).get_thread_stats() self.assertGreater(stats['main'], 0) self.assertGreater(stats['sql'], 0)
def process_exception(self, request, exception): if (not getattr(self, 'errormator_client') or not self.errormator_client.config.get('enabled')): return None request.__e_processed_exception__ = True environ = request.environ user = getattr(request, 'user', None) if user: environ['errormator.username'] = unicode(user.id) if isinstance(exception, Http404): http_status = 404 else: http_status = 500 request.__traceback__ = get_current_traceback( skip=1, show_hidden_frames=True, ignore_system_exceptions=True) # report 500's and 404's if not self.errormator_client.config['report_errors']: return None errormator_storage = get_local_storage(local_timing) stats, slow_calls = errormator_storage.get_thread_stats() self.errormator_client.py_report( environ, request.__traceback__, message=None, http_status=http_status, start_time=datetime.utcfromtimestamp(request.__start_time__), request_stats=stats) if request.__traceback__: # dereference tb object but set it to true afterwards for # other stuff del request.__traceback__ request.__traceback__ = True
def test_urllib2(self): import urllib2 urllib2.urlopen("http://www.ubuntu.com/") stats, result = get_local_storage(local_timing).get_thread_stats() self.assertEqual(len(result), 1)
def tearDown(self): get_local_storage(local_timing).clear()
def test_send_error_io(self): self.setUpClient() self.client.py_report(TEST_ENVIRON, http_status=404) result = self.client.submit_data() get_local_storage(local_timing).clear() self.assertEqual(result['reports'], False)
def test_send_error_failure_queue(self): self.setUpClient() self.client.py_report(TEST_ENVIRON, http_status=404) self.client.submit_data() get_local_storage(local_timing).clear() self.assertEqual(self.client.report_queue, [])
import time from errormator_client import client, make_errormator_middleware from errormator_client.exceptions import get_current_traceback from errormator_client.logger import register_logging from errormator_client.wsgi import ErrormatorWSGIWrapper fname = pkg_resources.resource_filename('errormator_client', 'templates/default_template.ini') timing_conf = client.get_config(path_to_config=fname) for k, v in timing_conf.iteritems(): if 'errormator.timing' in k: timing_conf[k] = 0.00000001 client.Client(config=timing_conf) from errormator_client.timing import local_timing, get_local_storage import timeit import jinja2 print 'traced', hasattr(jinja2.Template.render, '_e_attached_tracer') s = """ template = jinja2.Template('''xxxxx {{1+2}} yyyyyy''') template.render() """ print 'time', timeit.timeit(stmt=s, number=1500, setup="import jinja2") stats, slow_calls = get_local_storage(local_timing).get_thread_stats() print 'calls', len(slow_calls) print 'stats', stats
import pkg_resources import time from errormator_client import client, make_errormator_middleware from errormator_client.exceptions import get_current_traceback from errormator_client.logger import register_logging from errormator_client.wsgi import ErrormatorWSGIWrapper fname = pkg_resources.resource_filename('errormator_client', 'templates/default_template.ini') timing_conf = client.get_config(path_to_config=fname) for k, v in timing_conf.iteritems(): if 'errormator.timing' in k: timing_conf[k] = 0.00000001 client.Client(config=timing_conf) from errormator_client.timing import local_timing, get_local_storage import timeit import jinja2 print 'traced', hasattr(jinja2.Template.render, '_e_attached_tracer') s = """ template = jinja2.Template('''xxxxx {{1+2}} yyyyyy''') template.render() """ print 'time', timeit.timeit(stmt=s, number=1500, setup="import jinja2") stats, slow_calls = get_local_storage(local_timing).get_thread_stats() print 'calls', len(slow_calls) print 'stats', stats
def __call__(self, environ, start_response): """Run the application and conserve the traceback frames. also determine if we got 404 """ environ['errormator.request_id'] = str(uuid.uuid4()) app_iter = None detected_data = [] traceback = None start_time = default_timer() def detect_headers(status, headers, *k, **kw): detected_data[:] = status[:3], headers return start_response(status, headers, *k, **kw) # inject client instance reference to environ if 'errormator.client' not in environ: environ['errormator.client'] = self.errormator_client # some bw. compat stubs def local_report(message, include_traceback=True, http_status=200): environ['errormator.force_send'] = True def local_log(level, message): environ['errormator.force_send'] = True environ['errormator.report'] = local_report environ['errormator.log'] = local_log try: app_iter = self.app(environ, detect_headers) return app_iter except Exception as exc: if hasattr(app_iter, 'close'): app_iter.close() # we need that here traceback = get_current_traceback(skip=1, show_hidden_frames=True, ignore_system_exceptions=True) # by default reraise exceptions for app/FW to handle if self.errormator_client.config['reraise_exceptions']: raise try: start_response('500 INTERNAL SERVER ERROR', [('Content-Type', 'text/html; charset=utf-8')]) except Exception as exc: environ['wsgi.errors'].write( 'ErrormatorWSGIWrapper middleware catched exception ' 'in streamed response at a point where response headers ' 'were already sent.\n') else: return 'Server Error' finally: # report 500's and 404's # report slowness end_time = default_timer() errormator_storage = get_local_storage(local_timing) errormator_storage.thread_stats['main'] = end_time - start_time stats, slow_calls = errormator_storage.get_thread_stats() errormator_storage.clear() if 'errormator.__traceback' in environ: # get traceback gathered by tween traceback = environ['errormator.__traceback'] del environ['errormator.__traceback'] if traceback and self.errormator_client.config['report_errors']: http_status = 500 elif (self.errormator_client.config['report_404'] and detected_data and detected_data[0] == '404'): http_status = int(detected_data[0]) else: http_status = None if http_status: self.errormator_client.py_report( environ, traceback, message=None, http_status=http_status, start_time=datetime.datetime.utcfromtimestamp( start_time), request_stats=stats) # dereference del traceback traceback = True delta = datetime.timedelta(seconds=(end_time - start_time)) self.errormator_client.save_request_stats(stats) if self.errormator_client.config['slow_requests']: # do we have slow calls ? if (delta >= self.errormator_client.config['slow_request_time'] or slow_calls): self.errormator_client.py_slow_report( environ, datetime.datetime.utcfromtimestamp(start_time), datetime.datetime.utcfromtimestamp(end_time), slow_calls, request_stats=stats) # force log fetching traceback = True if self.errormator_client.config['logging']: records = self.errormator_client.log_handler.get_records() self.errormator_client.log_handler.clear_records() self.errormator_client.py_log( environ, records=records, r_uuid=environ[ 'errormator.request_id'], traceback=traceback) # send all data we gathered immediately at the end of request self.errormator_client.check_if_deliver( self.errormator_client.config['force_send'] or environ.get('errormator.force_send'))