def test_timing_request(self):
def app(environ, start_response):
start_response('200 OK', [('Content-Type', 'text/html')])
@time_trace(name='foo_func', min_duration=0.1)
def foo(arg):
time.sleep(0.2)
return arg
foo('a')
time.sleep(0.1)
try:
import psycopg2
except ImportError:
return
conn = psycopg2.connect(
"user=test host=127.0.0.1 dbname=test password=test")
c = conn.cursor()
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, c)
c.execute('SELECT 1, pg_sleep(0.5)')
c.fetchone()
c.close()
conn.close()
return ['Hello World!']
get_local_storage(local_timing).clear()
req = Request.blank('http://localhost/test')
app = make_appenlight_middleware(app, global_config=timing_conf)
req.get_response(app)
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertGreater(stats['main'], 0)
self.assertGreater(stats['sql'], 0)
def process_exception(self, request, exception):
if (not getattr(self, 'appenlight_client') or
not self.appenlight_client.config.get('enabled')):
return None
environ = getattr(request, 'environ', request.META)
if not self.appenlight_client.config['report_errors'] \
or environ.get('appenlight.ignore_error'):
return None
user = getattr(request, 'user', None)
end_time = default_timer()
if user and user_is_authenticated(user):
if 'appenlight.username' not in environ:
environ['appenlight.username'] = six.text_type(user.pk)
if not isinstance(exception, Http404):
http_status = 500
traceback = self.appenlight_client.get_current_traceback()
appenlight_storage = get_local_storage()
appenlight_storage.thread_stats[
'main'] = end_time - request.__start_time__
stats, slow_calls = appenlight_storage.get_thread_stats()
self.appenlight_client.save_request_stats(
stats, view_name=environ.get('appenlight.view_name', ''))
self.appenlight_client.py_report(
environ, traceback, message=None, http_status=http_status,
start_time=datetime.utcfromtimestamp(
request.__start_time__),
end_time=datetime.utcfromtimestamp(
end_time),
request_stats=stats, slow_calls=slow_calls)
del traceback
request._errormator_create_report = True
def view_callable_wrapper(context, request):
appenlight_storage = get_local_storage()
view_name = ''
try:
original_view = getattr(appenlight_callable, '__original_view__')
if original_view:
view_name = fullyQualifiedName(appenlight_callable)
# fix the problem with bound methods,
# we have to attach the resolved view name somehow.
if is_bound_method(original_view):
_original_view = original_view
def original_view(context, request):
return _original_view(context, request)
if not hasattr(original_view, '_appenlight_name'):
original_view._appenlight_name = view_name
except Exception:
raise
if 'pyramid/static' in view_name:
# normalize static views
view_name = 'pyramid/static'
if not getattr(appenlight_storage, 'view_name', None):
appenlight_storage.view_name = view_name
return appenlight_callable(context, request)
def test_httplib(self):
import httplib
h2 = httplib.HTTPConnection("www.ubuntu.com")
h2.request("GET", "/")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_urllib_URLOpener_open(self):
import urllib
opener = urllib.URLopener()
opener.open("http://www.ubuntu.com/")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def process_exception(self, request, exception):
if (not getattr(self, 'appenlight_client') or not self.appenlight_client.config.get('enabled')):
return None
environ = request.environ
if not self.appenlight_client.config['report_errors'] or environ.get('appenlight.ignore_error'):
return None
user = getattr(request, 'user', None)
end_time = default_timer()
if user and user.is_authenticated():
environ['appenlight.username'] = unicode(user.pk)
http_status = 500
request._errormator_create_report = True
traceback = get_current_traceback(skip=1,
show_hidden_frames=True,
ignore_system_exceptions=True)
appenlight_storage = get_local_storage(local_timing)
appenlight_storage.thread_stats['main'] = end_time - request.__start_time__
stats, slow_calls = appenlight_storage.get_thread_stats()
self.appenlight_client.save_request_stats(stats, view_name=environ.get('appenlight.view_name',''))
self.appenlight_client.py_report(environ,
traceback,
message=None,
http_status=http_status,
start_time=datetime.utcfromtimestamp(request.__start_time__),
end_time=datetime.utcfromtimestamp(end_time),
request_stats=stats,
slow_calls=slow_calls)
del traceback
def process_request(self, request):
request._errormator_create_report = False
request.__traceback__ = None
environ = getattr(request, 'environ', request.META)
ignored_slow_paths = self.appenlight_client.config.get(
'ignore_slow_paths', [])
if any(p in request.path for p in ignored_slow_paths):
log.debug('appenlight.ignore_slow_path in effect')
environ['appenlight.ignore_slow'] = True
ignored_paths = self.appenlight_client.config.get('ignore_paths', [])
if any(p in request.path for p in ignored_paths):
log.debug('appenlight.ignore_path in effect')
environ['appenlight.ignore_error'] = True
environ['appenlight.request_id'] = str(uuid.uuid4())
# inject client instance reference to environ
if 'appenlight.client' not in environ:
environ['appenlight.client'] = self.appenlight_client
if 'appenlight.tags' not in environ:
environ['appenlight.tags'] = {}
if 'appenlight.extra' not in environ:
environ['appenlight.extra'] = {}
environ['appenlight.post_vars'] = request.POST
appenlight_storage = get_local_storage()
# clear out thread stats on request start
appenlight_storage.clear()
request.__start_time__ = default_timer()
return None
def emit(self, record):
appenlight_storage = get_local_storage()
if len(appenlight_storage.logs) >= self.max_logs:
return
r_dict = convert_record_to_dict(record, self.client_config)
if r_dict:
if r_dict not in appenlight_storage.logs:
appenlight_storage.logs.append(r_dict)
def emit(self, record):
appenlight_storage = get_local_storage()
if len(appenlight_storage.logs) >= self.max_logs:
return
r_dict = convert_record_to_dict(record, self.ae_client_config)
if r_dict:
if r_dict not in appenlight_storage.logs:
appenlight_storage.logs.append(r_dict)
def test_requests(self):
try:
import requests
except ImportError:
return
requests.get("http://www.ubuntu.com/")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_urllib3(self):
try:
import urllib3
except ImportError:
return
http = urllib3.PoolManager()
http.request('GET', "http://www.ubuntu.com/")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_memcache_command(self):
try:
import memcache
except ImportError:
return
mc = memcache.Client(['127.0.0.1:11211'], debug=0)
mc.set("some_key", "Some value")
value = mc.get("some_key")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_redis_py_command(self):
try:
import redis
except ImportError:
return
client = redis.StrictRedis()
client.setex('testval', 10, 'foo')
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def process_response(self, request, response):
try:
return response
finally:
environ = getattr(request, 'environ', request.META)
enabled = self.appenlight_client.config.get('enabled')
if enabled and not request._errormator_create_report and not environ.get(
'appenlight.ignore_slow'):
end_time = default_timer()
user = getattr(request, 'user', None)
http_status = response.status_code
if user and user_is_authenticated(user):
if 'appenlight.username' not in environ:
environ['appenlight.username'] = six.text_type(user.pk)
if (http_status == 404 and self.appenlight_client.config[
'report_404']):
request._errormator_create_report = True
delta = timedelta(seconds=(end_time - request.__start_time__))
appenlight_storage = get_local_storage()
appenlight_storage.thread_stats[
'main'] = end_time - request.__start_time__
stats, slow_calls = appenlight_storage.get_thread_stats()
self.appenlight_client.save_request_stats(
stats, view_name=environ.get('appenlight.view_name', ''))
if self.appenlight_client.config['slow_requests']:
if (delta >= self.appenlight_client.config[
'slow_request_time'] or slow_calls):
request._errormator_create_report = True
if request._errormator_create_report:
self.appenlight_client.py_report(
environ,
None,
message=None,
http_status=http_status,
start_time=datetime.utcfromtimestamp(
request.__start_time__),
end_time=datetime.utcfromtimestamp(end_time),
request_stats=stats,
slow_calls=slow_calls)
if self.appenlight_client.config['logging']:
records = self.appenlight_client.log_handlers_get_records()
self.appenlight_client.log_handlers_clear_records()
self.appenlight_client.py_log(
environ,
records=records,
r_uuid=environ[
'appenlight.request_id'],
created_report=request._errormator_create_report)
if self.appenlight_client.config.get('enabled'):
self.appenlight_client.check_if_deliver(
self.appenlight_client.config['force_send'] or
environ.get('appenlight.force_send'))
def test_memcache_command(self):
# TODO: not finished
return
try:
import pylibmc
except ImportError:
return
mc = pylibmc.Client(['127.0.0.1:11211'])
mc.set("some_key", "Some value")
value = mc.get("some_key")
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def view_callable_wrapper(*args, **kwargs):
appenlight_storage = get_local_storage()
if hasattr(appenlight_storage, 'view_name'):
try:
split_name = appenlight_storage.view_name.split(':')
# only change the name if it wasn't resolved yet
if len(split_name) == 2 and '.' not in split_name[1]:
appenlight_storage.view_name = '%s.%s' % (appenlight_storage.view_name,
appenlight_callable.__name__)
except Exception:
pass
return appenlight_callable(*args, **kwargs)
def view_callable_wrapper(*args, **kwargs):
appenlight_storage = get_local_storage()
if hasattr(appenlight_storage, 'view_name'):
try:
split_name = appenlight_storage.view_name.split(':')
# only change the name if it wasn't resolved yet
if len(split_name) == 2 and '.' not in split_name[1]:
appenlight_storage.view_name = '%s.%s' % (appenlight_storage.view_name,
appenlight_callable.__name__)
except Exception:
pass
return appenlight_callable(*args, **kwargs)
def test_sqlite(self):
try:
import sqlite3
except ImportError:
return
conn = sqlite3.connect(':memory:')
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_postgresql(self):
try:
import postgresql
except ImportError:
return
conn = postgresql.open('pq://*****:*****@localhost')
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()[0]
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_oursql(self):
try:
import oursql
except ImportError:
return
conn = oursql.connect(passwd="test", user="******")
c = conn.cursor(oursql.DictCursor)
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_pymysql(self):
try:
import pymysql
except ImportError:
return
conn = pymysql.connect(host='127.0.0.1', user='******', passwd='test')
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_pg8000(self):
try:
import pg8000
except ImportError:
return
conn = pg8000.DBAPI.connect(host="127.0.0.1", user="******",
password="******")
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def process_request(self, request):
request._errormator_create_report = False
request.__traceback__ = None
environ = request.environ
environ['appenlight.request_id'] = str(uuid.uuid4())
# inject client instance reference to environ
if 'appenlight.client' not in environ:
environ['appenlight.client'] = self.appenlight_client
environ['appenlight.post_vars'] = request.POST
appenlight_storage = get_local_storage(local_timing)
# clear out thread stats on request start
appenlight_storage.clear()
request.__start_time__ = default_timer()
return None
def test_render(self):
try:
import chameleon.zpt
except ImportError:
return
import time
template = chameleon.zpt.PageTemplate('''
${sleep(0.06)}
xxxxx ${1+2} yyyyyy
''')
template.render(sleep=time.sleep)
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_odbc(self):
try:
import pyodbc
except ImportError:
return
conn = pyodbc.connect(
'Driver={MySQL};Server=127.0.0.1;Port=3306;Database=information_schema;User=test; Password=test;Option=3;')
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_mysqldb(self):
try:
import MySQLdb
except ImportError:
return
conn = MySQLdb.connect(passwd="test", user="******", host="127.0.0.1",
port=3306)
c = conn.cursor()
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_render(self):
try:
import jinja2
except ImportError:
return
import time
template = jinja2.Template('''
{{sleep(0.06)}}
xxxxx {{1+2}} yyyyyy
''')
template.render(sleep=time.sleep)
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_psycopg2(self):
try:
import psycopg2
except ImportError:
return
conn = psycopg2.connect(
"user=test host=127.0.0.1 dbname=test password=test")
c = conn.cursor()
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, c)
c.execute(self.stmt)
c.fetchone()
c.close()
conn.close()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 2)
def test_render_unicode(self):
try:
import mako
except ImportError:
return
template = mako.template.Template(u'''
<%
import time
time.sleep(0.01)
%>
xxxxx ${1+2} yyyyyy
''')
template.render_unicode()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def gather_data(client,
environ=None,
gather_exception=True,
gather_slowness=True,
gather_logs=True,
clear_storage=True,
exc_info=None,
start_time=None,
end_time=None):
""" exc_info is supposed to be (exc_type, exc_value, tb) - what sys.exc_info() returns """
if not client.config['enabled']:
return None
if environ is None:
environ = {}
if not environ.get('appenlight.request_id'):
environ['appenlight.request_id'] = str(uuid.uuid4())
http_status = 200
traceback = None
if gather_exception and not exc_info:
traceback = client.get_current_traceback()
if traceback:
http_status = 500
elif exc_info:
traceback = Traceback(*exc_info)
http_status = 500
appenlight_storage = get_local_storage(local_timing)
stats, slow_calls = appenlight_storage.get_thread_stats()
if traceback is not None or (slow_calls and gather_slowness):
client.py_report(environ,
traceback,
http_status=http_status,
request_stats=stats,
slow_calls=slow_calls,
start_time=start_time,
end_time=end_time)
# dereference
del traceback
if client.config['logging']:
if gather_logs:
records = client.log_handlers_get_records()
client.log_handlers_clear_records()
client.py_log(environ,
records=records,
r_uuid=environ['appenlight.request_id'])
if clear_storage:
appenlight_storage.clear()
client.check_if_deliver(client.config['force_send']
or environ.get('appenlight.force_send'))
def view_callable_wrapper(context, request):
appenlight_storage = get_local_storage()
view_name = ''
try:
original_view = getattr(appenlight_callable, '__original_view__')
if original_view:
view_name = fullyQualifiedName(appenlight_callable)
if not hasattr(original_view, '_appenlight_name'):
original_view._appenlight_name = view_name
except Exception:
raise
if 'pyramid/static' in view_name:
# normalize static views
view_name = 'pyramid/static'
if not getattr(appenlight_storage, 'view_name', None):
appenlight_storage.view_name = view_name
return appenlight_callable(context, request)
def test_template_lookup(self):
try:
from mako.lookup import TemplateLookup
except ImportError:
return
lookup = TemplateLookup()
lookup.put_string("base.html", '''
<%
import time
time.sleep(0.01)
%>
<html><body></body></html>
''')
template = lookup.get_template("base.html")
template.render_unicode()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def test_render_call_number(self):
try:
import mako
except ImportError:
return
template = mako.template.Template('''
<%
import time
time.sleep(0.01)
%>
xxxxx ${1+2} yyyyyy
''')
template.render()
template.render()
template.render()
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(stats['tmpl_calls'], 3)
def view_callable_wrapper(context, request):
appenlight_storage = get_local_storage()
view_name = ''
try:
original_view = getattr(appenlight_callable, '__original_view__')
if original_view:
view_name = fullyQualifiedName(appenlight_callable)
if not hasattr(original_view, '_appenlight_name'):
original_view._appenlight_name = view_name
except Exception:
raise
if 'pyramid/static' in view_name:
# normalize static views
view_name = 'pyramid/static'
if not getattr(appenlight_storage, 'view_name', None):
appenlight_storage.view_name = view_name
return appenlight_callable(context, request)
def test_render(self):
try:
from django import template
except ImportError:
return
from django.conf import settings
settings.configure(TEMPLATE_DIRS=("/whatever/templates",))
import time
ctx = template.Context()
ctx.update({'time': lambda: time.sleep(0.06)})
template = template.Template('''
xxxxx {{ time }} yyyyyy
''')
template.render(ctx)
stats, result = get_local_storage(local_timing).get_thread_stats()
self.assertEqual(len(result), 1)
def process_request(self, request):
request._errormator_create_report = False
request.__traceback__ = None
environ = request.environ
environ['appenlight.request_id'] = str(uuid.uuid4())
# inject client instance reference to environ
if 'appenlight.client' not in environ:
environ['appenlight.client'] = self.appenlight_client
if 'appenlight.tags' not in environ:
environ['appenlight.tags'] = {}
if 'appenlight.extra' not in environ:
environ['appenlight.extra'] = {}
environ['appenlight.post_vars'] = request.POST
appenlight_storage = get_local_storage()
# clear out thread stats on request start
appenlight_storage.clear()
request.__start_time__ = default_timer()
return None
def clear_records(self, thread=None):
""" Clears ALL logs from AE storage """
appenlight_storage = get_local_storage()
appenlight_storage.logs = []
def get_records(self, thread=None):
"""
Returns a list of records for the current thread.
"""
appenlight_storage = get_local_storage()
return appenlight_storage.logs
def emit(self, record):
appenlight_storage = get_local_storage()
r_dict = convert_record_to_dict(record, self.ae_client_config)
if r_dict:
if r_dict not in appenlight_storage.logs:
appenlight_storage.logs.append(r_dict)
import logging
import socket
import pkg_resources
import time
from appenlight_client import client, make_appenlight_middleware
from appenlight_client.exceptions import get_current_traceback
from appenlight_client.logger import register_logging
from appenlight_client.wsgi import AppenlightWSGIWrapper
timing_conf = client.get_config({'appenlight.api_key': '1234'})
for k, v in timing_conf.iteritems():
if 'appenlight.timing' in k:
timing_conf[k] = 0.00000001
client.Client(config=timing_conf)
from appenlight_client.timing import local_timing, get_local_storage
import timeit
import jinja2
print 'traced', hasattr(jinja2.Template.render, '_e_attached_tracer')
s = """
template = jinja2.Template('''xxxxx {{1+2}} yyyyyy''')
template.render()
"""
print 'time', timeit.timeit(stmt=s, number=1500, setup="import jinja2")
stats, slow_calls = get_local_storage(local_timing).get_thread_stats()
print 'calls', len(slow_calls)
print 'stats', stats
def log_handlers_get_records(self):
appenlight_storage = get_local_storage()
return appenlight_storage.logs
def log_handlers_clear_records(self):
appenlight_storage = get_local_storage()
appenlight_storage.logs = []
def __call__(self, environ, start_response):
"""Run the application and conserve the traceback frames.
also determine if we got 404
"""
environ['appenlight.request_id'] = str(uuid.uuid4())
appenlight_storage = get_local_storage()
# clear out thread stats on request start
appenlight_storage.clear()
app_iter = None
detected_data = []
create_report = False
traceback = None
http_status = 200
start_time = default_timer()
def detect_headers(status, headers, *k, **kw):
detected_data[:] = status[:3], headers
return start_response(status, headers, *k, **kw)
# inject client instance reference to environ
if 'appenlight.client' not in environ:
environ['appenlight.client'] = self.appenlight_client
# some bw. compat stubs
def local_report(message, include_traceback=True, http_status=200):
environ['appenlight.force_send'] = True
def local_log(level, message):
environ['appenlight.force_send'] = True
environ['appenlight.report'] = local_report
environ['appenlight.log'] = local_log
if 'appenlight.tags' not in environ:
environ['appenlight.tags'] = {}
if 'appenlight.extra' not in environ:
environ['appenlight.extra'] = {}
try:
app_iter = self.app(environ, detect_headers)
return app_iter
except Exception:
if hasattr(app_iter, 'close'):
app_iter.close()
# we need that here
traceback = self.appenlight_client.get_current_traceback()
# by default reraise exceptions for app/FW to handle
if self.appenlight_client.config['reraise_exceptions']:
raise
try:
start_response('500 INTERNAL SERVER ERROR',
[('Content-Type', 'text/html; charset=utf-8')])
except Exception:
environ['wsgi.errors'].write(
'AppenlightWSGIWrapper middleware catched exception '
'in streamed response at a point where response headers '
'were already sent.\n')
else:
return 'Server Error'
finally:
# report 500's and 404's
# report slowness
end_time = default_timer()
appenlight_storage.thread_stats['main'] = end_time - start_time
delta = datetime.timedelta(seconds=(end_time - start_time))
stats, slow_calls = appenlight_storage.get_thread_stats()
if 'appenlight.view_name' not in environ:
environ['appenlight.view_name'] = getattr(
appenlight_storage, 'view_name', '')
if detected_data and detected_data[0]:
http_status = int(detected_data[0])
if self.appenlight_client.config[
'slow_requests'] and not environ.get(
'appenlight.ignore_slow'):
# do we have slow calls/request ?
if (delta >= self.appenlight_client.config['slow_request_time']
or slow_calls):
create_report = True
if 'appenlight.__traceback' in environ and not environ.get(
'appenlight.ignore_error'):
# get traceback gathered by pyramid tween
traceback = environ['appenlight.__traceback']
del environ['appenlight.__traceback']
http_status = 500
create_report = True
if traceback and self.appenlight_client.config[
'report_errors'] and not environ.get(
'appenlight.ignore_error'):
http_status = 500
create_report = True
elif (self.appenlight_client.config['report_404']
and http_status == 404):
create_report = True
if create_report:
self.appenlight_client.py_report(
environ,
traceback,
message=None,
http_status=http_status,
start_time=datetime.datetime.utcfromtimestamp(start_time),
end_time=datetime.datetime.utcfromtimestamp(end_time),
request_stats=stats,
slow_calls=slow_calls)
# dereference
del traceback
self.appenlight_client.save_request_stats(
stats, view_name=environ.get('appenlight.view_name', ''))
if self.appenlight_client.config['logging']:
records = self.appenlight_client.log_handlers_get_records()
self.appenlight_client.log_handlers_clear_records()
self.appenlight_client.py_log(
environ,
records=records,
r_uuid=environ['appenlight.request_id'],
created_report=create_report)
# send all data we gathered immediately at the end of request
self.appenlight_client.check_if_deliver(
self.appenlight_client.config['force_send']
or environ.get('appenlight.force_send'))
def purge_data(self):
storage = get_local_storage()
storage.clear()
self.log_handlers_clear_records()
self.transport.purge()