def test_urllib_URLOpener_open(self):
import urllib
opener = urllib.URLopener()
opener.open("http://www.ubuntu.com/")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_httplib(self):
import httplib
h2 = httplib.HTTPConnection("www.ubuntu.com")
h2.request("GET", "/")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_requests(self):
try:
import requests
except ImportError:
return
requests.get("http://www.ubuntu.com/")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_urllib3(self):
try:
import urllib3
except ImportError:
return
http = urllib3.PoolManager()
http.request('GET', "http://www.ubuntu.com/")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_redis_py_command(self):
try:
import redis
except ImportError:
return
client = redis.StrictRedis()
client.setex('testval', 10, 'foo')
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_memcache_command(self):
try:
import memcache
except ImportError:
return
mc = memcache.Client(['127.0.0.1:11211'], debug=0)
mc.set("some_key", "Some value")
value = mc.get("some_key")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_memcache_command(self):
# TODO: not finished
return
try:
import pylibmc
except ImportError:
return
mc = pylibmc.Client(['127.0.0.1:11211'])
mc.set("some_key", "Some value")
value = mc.get("some_key")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_sqlite(self):
try:
import sqlite3
except ImportError:
return
conn = sqlite3.connect(':memory:')
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_postgresql(self):
try:
import postgresql
except ImportError:
return
conn = postgresql.open('pq://*****:*****@localhost')
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()[0]
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_oursql(self):
try:
import oursql
except ImportError:
return
conn = oursql.connect(passwd="test", user="******")
c = conn.cursor(oursql.DictCursor)
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_pymysql(self):
try:
import pymysql
except ImportError:
return
conn = pymysql.connect(host='127.0.0.1', user='******', passwd='test')
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def process_request(self, request):
request.__e_processed_exception__ = False
request.__traceback__ = None
environ = request.environ
environ['errormator.request_id'] = str(uuid.uuid4())
# inject client instance reference to environ
if 'errormator.client' not in environ:
environ['errormator.client'] = self.errormator_client
environ['errormator.post_vars'] = request.POST
errormator_storage = get_local_storage(local_timing)
# clear out thread stats on request start
errormator_storage.clear()
request.__start_time__ = default_timer()
return None
def test_mysqldb(self):
try:
import MySQLdb
except ImportError:
return
conn = MySQLdb.connect(passwd="test", user="******", host="127.0.0.1",
port=3306)
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def process_request(self, request):
request.__e_processed_exception__ = False
request.__traceback__ = None
environ = request.environ
environ['errormator.request_id'] = str(uuid.uuid4())
# inject client instance reference to environ
if 'errormator.client' not in environ:
environ['errormator.client'] = self.errormator_client
environ['errormator.post_vars'] = request.POST
errormator_storage = get_local_storage(local_timing)
# clear out thread stats on request start
errormator_storage.clear()
request.__start_time__ = default_timer()
return None
def test_render(self):
try:
import jinja2
except ImportError:
return
import time
template = jinja2.Template('''
{{sleep(0.06)}}
xxxxx {{1+2}} yyyyyy
''')
template.render(sleep=time.sleep)
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_render(self):
try:
import chameleon.zpt
except ImportError:
return
import time
template = chameleon.zpt.PageTemplate('''
${sleep(0.06)}
xxxxx ${1+2} yyyyyy
''')
template.render(sleep=time.sleep)
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_odbc(self):
try:
import pyodbc
except ImportError:
return
conn = pyodbc.connect(
'Driver={MySQL};Server=127.0.0.1;Port=3306;Database=information_schema;User=test; Password=test;Option=3;')
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_pg8000(self):
try:
import pg8000
except ImportError:
return
conn = pg8000.DBAPI.connect(host="127.0.0.1", user="******",
password="******")
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_render_unicode(self):
try:
import mako
except ImportError:
return
template = mako.template.Template(u'''
<%
import time
time.sleep(0.01)
%>
xxxxx ${1+2} yyyyyy
''')
template.render_unicode()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_psycopg2(self):
try:
import psycopg2
except ImportError:
return
conn = psycopg2.connect(
"user=test host=127.0.0.1 dbname=test password=test")
c = conn.cursor()
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, c)
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_pg8000(self):
try:
import pg8000
except ImportError:
return
conn = pg8000.DBAPI.connect(host="127.0.0.1",
user="******",
password="******")
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_psycopg2(self):
try:
import psycopg2
except ImportError:
return
conn = psycopg2.connect(
"user=test host=127.0.0.1 dbname=test password=test")
c = conn.cursor()
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, c)
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_odbc(self):
try:
import pyodbc
except ImportError:
return
conn = pyodbc.connect(
'Driver={MySQL};Server=127.0.0.1;Port=3306;Database=information_schema;User=test; Password=test;Option=3;'
)
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_render_unicode(self):
try:
import mako
except ImportError:
return
template = mako.template.Template(u'''
<%
import time
time.sleep(0.01)
%>
xxxxx ${1+2} yyyyyy
''')
template.render_unicode()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def process_response(self, request, response):
try:
return response
finally:
if self.errormator_client.config.get('enabled'):
end_time = default_timer()
environ = request.environ
user = getattr(request, 'user', None)
if user:
environ['errormator.username'] = unicode(user.id)
if (response.status_code == 404
and not request.__e_processed_exception__):
self.process_exception(request, Http404())
delta = timedelta(seconds=(end_time - request.__start_time__))
errormator_storage = get_local_storage(local_timing)
errormator_storage.thread_stats[
'main'] = end_time - request.__start_time__
stats, slow_calls = errormator_storage.get_thread_stats()
errormator_storage.clear()
# report slowness
self.errormator_client.save_request_stats(stats)
if self.errormator_client.config['slow_requests']:
# do we have slow calls ?
if (delta >=
self.errormator_client.config['slow_request_time']
or slow_calls):
self.errormator_client.py_slow_report(
environ,
datetime.utcfromtimestamp(request.__start_time__),
datetime.utcfromtimestamp(end_time),
slow_calls,
request_stats=stats)
# force log fetching
request.__traceback__ = True
if self.errormator_client.config['logging']:
records = self.errormator_client.log_handler.get_records()
self.errormator_client.log_handler.clear_records()
self.errormator_client.py_log(
environ,
records=records,
r_uuid=environ['errormator.request_id'],
traceback=request.__traceback__)
# send all data we gathered immediately at the
# end of request
self.errormator_client.check_if_deliver(
self.errormator_client.config['force_send']
or environ.get('errormator.force_send'))
def test_mysqldb(self):
try:
import MySQLdb
except ImportError:
return
conn = MySQLdb.connect(passwd="test",
user="******",
host="127.0.0.1",
port=3306)
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def process_response(self, request, response):
try:
return response
finally:
if self.errormator_client.config.get('enabled'):
end_time = default_timer()
environ = request.environ
user = getattr(request, 'user', None)
if user:
environ['errormator.username'] = unicode(user.id)
if (response.status_code == 404 and
not request.__e_processed_exception__):
self.process_exception(request, Http404())
delta = timedelta(
seconds=(end_time - request.__start_time__))
errormator_storage = get_local_storage(local_timing)
errormator_storage.thread_stats[
'main'] = end_time - request.__start_time__
stats, slow_calls = errormator_storage.get_thread_stats()
errormator_storage.clear()
# report slowness
self.errormator_client.save_request_stats(stats)
if self.errormator_client.config['slow_requests']:
# do we have slow calls ?
if (delta >= self.errormator_client.config[
'slow_request_time'] or slow_calls):
self.errormator_client.py_slow_report(
environ,
datetime.utcfromtimestamp(request.__start_time__),
datetime.utcfromtimestamp(end_time),
slow_calls,
request_stats=stats)
# force log fetching
request.__traceback__ = True
if self.errormator_client.config['logging']:
records = self.errormator_client.log_handler.get_records()
self.errormator_client.log_handler.clear_records()
self.errormator_client.py_log(
environ, records=records,
r_uuid=environ['errormator.request_id'],
traceback=request.__traceback__)
# send all data we gathered immediately at the
# end of request
self.errormator_client.check_if_deliver(
self.errormator_client.config['force_send'] or
environ.get('errormator.force_send'))
def gather_data(client, environ, gather_exception=True,
gather_slowness=True, gather_logs=True,
clear_storage=True, exc_info=None, spawn_thread=True):
if client.config['enabled'] == False:
return None
if not environ.get('wsgi.url_scheme'):
environ['wsgi.url_scheme'] = ''
if not environ.get('HTTP_HOST'):
environ['HTTP_HOST'] = 'localhost'
if not environ.get('errormator.request_id'):
environ['errormator.request_id'] = str(uuid.uuid4())
if gather_exception and not exc_info:
traceback = get_current_traceback(skip=1, show_hidden_frames=True,
ignore_system_exceptions=True)
elif exc_info:
traceback = Traceback(*exc_info)
else:
traceback = None
errormator_storage = get_local_storage(local_timing)
stats, slow_calls = errormator_storage.get_thread_stats()
if traceback:
client.py_report(environ, traceback, http_status=500,
request_stats=stats)
# dereference
del traceback
traceback = True
# report slowness
now = datetime.datetime.utcnow()
if clear_storage:
errormator_storage.clear()
if client.config['slow_requests'] and gather_slowness:
# do we have slow calls ?
if (slow_calls):
client.py_slow_report(environ, now, now, slow_calls,
request_stats=stats)
# force log fetching
traceback = True
if client.config['logging'] and gather_logs:
records = client.log_handler.get_records()
client.log_handler.clear_records()
client.py_log(environ, records=records, traceback=traceback,
r_uuid=environ['errormator.request_id'])
# send all data we gathered immediately at the end of request
client.check_if_deliver(client.config['force_send'] or
environ.get('errormator.force_send'),
spawn_thread=spawn_thread)
def test_render_call_number(self):
try:
import mako
except ImportError:
return
template = mako.template.Template('''
<%
import time
time.sleep(0.01)
%>
xxxxx ${1+2} yyyyyy
''')
template.render()
template.render()
template.render()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(stats['tmpl_calls'], 3)
def test_template_lookup(self):
try:
from mako.lookup import TemplateLookup
except ImportError:
return
lookup = TemplateLookup()
lookup.put_string("base.html", '''
<%
import time
time.sleep(0.01)
%>
<html><body></body></html>
''')
template = lookup.get_template("base.html")
template.render_unicode()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_template_lookup(self):
try:
from mako.lookup import TemplateLookup
except ImportError:
return
lookup = TemplateLookup()
lookup.put_string("base.html", '''
<%
import time
time.sleep(0.01)
%>
<html><body></body></html>
''')
template = lookup.get_template("base.html")
template.render_unicode()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_render_call_number(self):
try:
import mako
except ImportError:
return
template = mako.template.Template('''
<%
import time
time.sleep(0.01)
%>
xxxxx ${1+2} yyyyyy
''')
template.render()
template.render()
template.render()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(stats['tmpl_calls'], 3)
def test_render(self):
try:
from django import template
except ImportError:
return
from django.conf import settings
settings.configure(TEMPLATE_DIRS=("/whatever/templates",))
import time
ctx = template.Context()
ctx.update({'time': lambda: time.sleep(0.06)})
template = template.Template('''
xxxxx {{ time }} yyyyyy
''')
template.render(ctx)
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_render(self):
try:
from django import template
except ImportError:
return
from django.conf import settings
settings.configure(TEMPLATE_DIRS=("/whatever/templates",))
import time
ctx = template.Context()
ctx.update({'time': lambda: time.sleep(0.06)})
template = template.Template('''
xxxxx {{ time }} yyyyyy
''')
template.render(ctx)
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_sqlite_call_number(self):
try:
import sqlite3
except ImportError:
return
conn = sqlite3.connect(':memory:')
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.execute(self.stmt)
c.fetchone()
c.execute(self.stmt)
c.fetchone()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(int(stats['sql_calls']), 4)
def test_timing_request(self):
def app(environ, start_response):
start_response('200 OK', [('Content-Type', 'text/html')])
time.sleep(0.1)
try:
import sqlite3
except ImportError:
return
conn = sqlite3.connect(':memory:')
c = conn.cursor()
c.execute('''SELECT 1+2+3 AS result''')
c.fetchone()
c.close()
conn.close()
return ['Hello World!']
req = Request.blank('http://localhost/test')
app = make_errormator_middleware(app, global_config=timing_conf)
req.get_response(app)
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertGreater(stats['main'], 0)
self.assertGreater(stats['sql'], 0)
def test_timing_request(self):
def app(environ, start_response):
start_response('200 OK', [('Content-Type', 'text/html')])
time.sleep(0.1)
try:
import sqlite3
except ImportError:
return
conn = sqlite3.connect(':memory:')
c = conn.cursor()
c.execute('''SELECT 1+2+3 AS result''')
c.fetchone()
c.close()
conn.close()
return ['Hello World!']
req = Request.blank('http://localhost/test')
app = make_errormator_middleware(app, global_config=timing_conf)
req.get_response(app)
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertGreater(stats['main'], 0)
self.assertGreater(stats['sql'], 0)
def process_exception(self, request, exception):
if (not getattr(self, 'errormator_client') or
not self.errormator_client.config.get('enabled')):
return None
request.__e_processed_exception__ = True
environ = request.environ
user = getattr(request, 'user', None)
if user:
environ['errormator.username'] = unicode(user.id)
if isinstance(exception, Http404):
http_status = 404
else:
http_status = 500
request.__traceback__ = get_current_traceback(
skip=1,
show_hidden_frames=True,
ignore_system_exceptions=True)
# report 500's and 404's
if not self.errormator_client.config['report_errors']:
return None
errormator_storage = get_local_storage(local_timing)
stats, slow_calls = errormator_storage.get_thread_stats()
self.errormator_client.py_report(
environ,
request.__traceback__,
message=None,
http_status=http_status,
start_time=datetime.utcfromtimestamp(request.__start_time__),
request_stats=stats)
if request.__traceback__:
# dereference tb object but set it to true afterwards for
# other stuff
del request.__traceback__
request.__traceback__ = True
def test_urllib2(self):
import urllib2
urllib2.urlopen("http://www.ubuntu.com/")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def tearDown(self):
get_local_storage(local_timing).clear()
def tearDown(self):
get_local_storage(local_timing).clear()
def test_urllib2(self):
import urllib2
urllib2.urlopen("http://www.ubuntu.com/")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_send_error_io(self):
self.setUpClient()
self.client.py_report(TEST_ENVIRON, http_status=404)
result = self.client.submit_data()
get_local_storage(local_timing).clear()
self.assertEqual(result['reports'], False)
def test_send_error_failure_queue(self):
self.setUpClient()
self.client.py_report(TEST_ENVIRON, http_status=404)
self.client.submit_data()
get_local_storage(local_timing).clear()
self.assertEqual(self.client.report_queue, [])
import time
from errormator_client import client, make_errormator_middleware
from errormator_client.exceptions import get_current_traceback
from errormator_client.logger import register_logging
from errormator_client.wsgi import ErrormatorWSGIWrapper
fname = pkg_resources.resource_filename('errormator_client',
'templates/default_template.ini')
timing_conf = client.get_config(path_to_config=fname)
for k, v in timing_conf.iteritems():
if 'errormator.timing' in k:
timing_conf[k] = 0.00000001
client.Client(config=timing_conf)
from errormator_client.timing import local_timing, get_local_storage
import timeit
import jinja2
print 'traced', hasattr(jinja2.Template.render, '_e_attached_tracer')
s = """
template = jinja2.Template('''xxxxx {{1+2}} yyyyyy''')
template.render()
"""
print 'time', timeit.timeit(stmt=s, number=1500, setup="import jinja2")
stats, slow_calls = get_local_storage(local_timing).get_thread_stats()
print 'calls', len(slow_calls)
print 'stats', stats
import pkg_resources
import time
from errormator_client import client, make_errormator_middleware
from errormator_client.exceptions import get_current_traceback
from errormator_client.logger import register_logging
from errormator_client.wsgi import ErrormatorWSGIWrapper
fname = pkg_resources.resource_filename('errormator_client',
'templates/default_template.ini')
timing_conf = client.get_config(path_to_config=fname)
for k, v in timing_conf.iteritems():
if 'errormator.timing' in k:
timing_conf[k] = 0.00000001
client.Client(config=timing_conf)
from errormator_client.timing import local_timing, get_local_storage
import timeit
import jinja2
print 'traced', hasattr(jinja2.Template.render, '_e_attached_tracer')
s = """
template = jinja2.Template('''xxxxx {{1+2}} yyyyyy''')
template.render()
"""
print 'time', timeit.timeit(stmt=s, number=1500, setup="import jinja2")
stats, slow_calls = get_local_storage(local_timing).get_thread_stats()
print 'calls', len(slow_calls)
print 'stats', stats
def __call__(self, environ, start_response):
"""Run the application and conserve the traceback frames.
also determine if we got 404
"""
environ['errormator.request_id'] = str(uuid.uuid4())
app_iter = None
detected_data = []
traceback = None
start_time = default_timer()
def detect_headers(status, headers, *k, **kw):
detected_data[:] = status[:3], headers
return start_response(status, headers, *k, **kw)
# inject client instance reference to environ
if 'errormator.client' not in environ:
environ['errormator.client'] = self.errormator_client
# some bw. compat stubs
def local_report(message, include_traceback=True,
http_status=200):
environ['errormator.force_send'] = True
def local_log(level, message):
environ['errormator.force_send'] = True
environ['errormator.report'] = local_report
environ['errormator.log'] = local_log
try:
app_iter = self.app(environ, detect_headers)
return app_iter
except Exception as exc:
if hasattr(app_iter, 'close'):
app_iter.close()
# we need that here
traceback = get_current_traceback(skip=1, show_hidden_frames=True,
ignore_system_exceptions=True)
# by default reraise exceptions for app/FW to handle
if self.errormator_client.config['reraise_exceptions']:
raise
try:
start_response('500 INTERNAL SERVER ERROR',
[('Content-Type', 'text/html; charset=utf-8')])
except Exception as exc:
environ['wsgi.errors'].write(
'ErrormatorWSGIWrapper middleware catched exception '
'in streamed response at a point where response headers '
'were already sent.\n')
else:
return 'Server Error'
finally:
# report 500's and 404's
# report slowness
end_time = default_timer()
errormator_storage = get_local_storage(local_timing)
errormator_storage.thread_stats['main'] = end_time - start_time
stats, slow_calls = errormator_storage.get_thread_stats()
errormator_storage.clear()
if 'errormator.__traceback' in environ:
# get traceback gathered by tween
traceback = environ['errormator.__traceback']
del environ['errormator.__traceback']
if traceback and self.errormator_client.config['report_errors']:
http_status = 500
elif (self.errormator_client.config['report_404'] and
detected_data and detected_data[0] == '404'):
http_status = int(detected_data[0])
else:
http_status = None
if http_status:
self.errormator_client.py_report(
environ, traceback,
message=None,
http_status=http_status,
start_time=datetime.datetime.utcfromtimestamp(
start_time),
request_stats=stats)
# dereference
del traceback
traceback = True
delta = datetime.timedelta(seconds=(end_time - start_time))
self.errormator_client.save_request_stats(stats)
if self.errormator_client.config['slow_requests']:
# do we have slow calls ?
if (delta >= self.errormator_client.config['slow_request_time']
or slow_calls):
self.errormator_client.py_slow_report(
environ,
datetime.datetime.utcfromtimestamp(start_time),
datetime.datetime.utcfromtimestamp(end_time),
slow_calls,
request_stats=stats)
# force log fetching
traceback = True
if self.errormator_client.config['logging']:
records = self.errormator_client.log_handler.get_records()
self.errormator_client.log_handler.clear_records()
self.errormator_client.py_log(
environ,
records=records,
r_uuid=environ[
'errormator.request_id'],
traceback=traceback)
# send all data we gathered immediately at the end of request
self.errormator_client.check_if_deliver(
self.errormator_client.config['force_send'] or
environ.get('errormator.force_send'))
