def test_does_not_email_for_whitelisted_errors(self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine(
"/site-packages/lib/doghouse/authentication.py", 7,
"check_auth",
'raise errors.BadAuthenticationError("Something smells off...")'
)
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey(
"lib/doghouse/authentication.py", 7, "check_auth", 'raise errors.BadAuthenticationError("Something smells off...")'):
api.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
False,
"2020-01-01 00:00:00",
is_known_error=True,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEquals(None, self.smtp_stub.last_args)
def test_traces_up_stack_trace_for_errors_originating_from_building_blocks(
self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/coreservices/service.py", 7,
"serve", "..."),
api.StackLine(
"/site-packages/apps/shopkick/doghouse/lib/base.py", 9,
"_get_request_param",
'raise errors.BadRequestError("Missing param %s" % name)')
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
def test_doesnt_email_on_errors_before_cutoff_date(self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/coreservices/service.py", 7,
"serve", "..."),
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x")
],
exception_message="email text",
hostname="localhost",
)
self.popen_stub.stdout = StringIO.StringIO(
"75563df6e9d1efe44b48f6643fde9ebbd822b0c5 25 25 1\n"
"author John Egan\n"
"author-mail <*****@*****.**>\n"
"author-time %d\n"
"author-tz -0800\n" %
int(time.mktime(datetime.datetime(2009, 7, 30).timetuple())))
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api.ErrorInfo(1,
"*****@*****.**",
"2009-07-30 00:00:00",
False,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual(None, self.smtp_stub.last_args)
def test_records_error_only_once(self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/coreservices/service.py", 7,
"serve", "..."),
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self._set_stub_time(datetime.datetime(2020, 1, 2))
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api.ErrorInfo(2,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-02 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual(1, len(self.smtp_stub.args_list))
def test_uses_threshold_specified_in_request(self):
self.test_config.report_error_threshold = 2
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/coreservices/service.py", 7,
"serve", "..."),
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x")
],
exception_message="email text",
hostname="localhost",
error_threshold=1,
)
self.handler.record_error(req.dumps())
self.assertEmailEquals(
dict(to_addresses=["*****@*****.**"],
from_address="*****@*****.**",
cc_address=None,
bcc_address=None,
subject="Error on localhost in coreservices/service.py",
body="email text",
smtp_server_host_port=None), self.smtp_stub.last_args)
def test_removes_duplicate_emails(self):
# Almost the same setup as test_uses_threshold_specified_in_request except the traceback includes
# a path that John is watching. Since he's also the developer who committed the buggy code, he is
# the one and only email recipient.
self.test_config.report_error_threshold = 2
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/lib/no/such/path/for/testing.py",
7, "fubar", "..."),
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x")
],
exception_message="email text",
hostname="localhost",
error_threshold=1,
)
self.handler.record_error(req.dumps())
self.assertEmailEquals(
dict(
to_addresses=["*****@*****.**"],
from_address="*****@*****.**",
cc_address=None,
bcc_address=None,
subject="Error on localhost in lib/no/such/path/for/testing.py",
body="email text",
smtp_server_host_port=None), self.smtp_stub.last_args)
def test_email_includes_watchers(self):
# Almost the same setup as test_uses_threshold_specified_in_request except the traceback includes
# a path that Yen is watching
self.test_config.report_error_threshold = 2
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/tools/furminator.py", 7, "fubar",
"..."),
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x")
],
exception_message="email text",
hostname="localhost",
error_threshold=1,
)
self.handler.record_error(req.dumps())
self.assertEmailEquals(
dict(to_addresses=[
"*****@*****.**", "*****@*****.**",
"*****@*****.**"
],
from_address="*****@*****.**",
cc_address=None,
bcc_address=None,
subject="Error on localhost in tools/furminator.py",
body="email text",
smtp_server_host_port=None), self.smtp_stub.last_args)
def test_doesnt_report_errors_under_threshold(self):
self.test_config.report_error_threshold = 2
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/coreservices/service.py", 7,
"serve", "..."),
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
False,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEquals(None, self.smtp_stub.last_args)
def test_ignores_third_party_whitelisted_errors_for_facebook(self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/facebook.py", 7,
"post_treat_to_facebook",
'urllib.urlencode(args), post_data)')
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self.assertDictEquals({}, self.handler.errors_seen.dict)
def test_ignores_error_in_thrift_directory(self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x"),
api.StackLine("/site-packages/coreservices/thrift/file.py", 7,
"serve", "...")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self.assertDictEquals({}, self.handler.errors_seen.dict)
def test_ignores_third_party_whitelisted_errors(self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine(
"/site-packages/SQLAlchemy-0.5.6-py2.6.egg/sqlalchemy/pool.py",
7, "do_get",
'raise exc.TimeoutError("QueuePool limit of size %d overflow %d '
'reached, connection timed out, timeout %d" % (self.size(),'
'self.overflow(), self._timeout))')
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self.assertDictEquals({}, self.handler.errors_seen.dict)
def test_email_includes_extra_information(self):
# Traceback includes a path that has extra_information tagged on it
self.test_config.report_error_threshold = 2
req = api.RecordErrorRequest(
traceback=[
api.StackLine(
"/site-packages/coreservices/waterbowl/rewards/water.py",
5, "make_external_api_request",
"raise api.WaterbowlError(error_code=api.WaterbowlErrorCode."
"OUT_OF_WATER, message=str(e))"),
],
exception_message="email text",
hostname="localhost",
error_threshold=1,
additional_info="extra stuff",
)
self.handler.record_error(req.dumps())
self.assertEmailEquals(
dict(
to_addresses=[
"*****@*****.**", "*****@*****.**",
"*****@*****.**"
],
from_address="*****@*****.**",
cc_address=None,
bcc_address=None,
subject=
"Error on localhost in coreservices/waterbowl/rewards/water.py",
body=
'NOTE: This error typically does not require dev team involvement.',
smtp_server_host_port=None), self.smtp_stub.last_args)
body = email.message_from_string(
self.smtp_stub.last_args["body"]).get_payload(decode=True)
self.assertTrue("email text" in body)
self.assertTrue("extra stuff" in body)
def test_always_alerts_on_red_alert_errors(self):
self.test_config.report_error_threshold = 3
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/coreservices/service.py", 7,
"serve", "..."),
api.StackLine('/site-packages/coreservices/service/utils.py',
9, "check_water_levels",
'% (waterbowl_id, min_required_level))'),
api.StackLine(
"/site-packages/coreservices/waterbowl/rewards/water.py",
5, "make_external_api_request",
"raise api.WaterbowlError(error_code=api.WaterbowlErrorCode."
"OUT_OF_WATER, message=str(e))"),
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
# The 2 red alert recipients plus the developer responsible
self.assertEmailEquals(
dict(
to_addresses=[
"*****@*****.**", "*****@*****.**",
"*****@*****.**"
],
from_address="*****@*****.**",
cc_address=None,
bcc_address=None,
subject=
"Error on localhost in coreservices/waterbowl/rewards/water.py",
body="email text",
smtp_server_host_port=None), self.smtp_stub.last_args)
def test_records_error_with_thrift_in_file_name(self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/coreservices/thrift_file.py", 7,
"serve", "..."),
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey("coreservices/thrift_file.py", 7, "serve", "..."):
api.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
def test_records_error(self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/coreservices/service.py", 7,
"serve", "..."),
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual([
"git", "--git-dir=/tmp/.git", "--work-tree=/tmp", "blame", "-p",
"/tmp/coreservices/service.py", "-L", "7,+1"
], self.popen_stub.last_args)
self.assertEmailEquals(
dict(to_addresses=["*****@*****.**"],
from_address="*****@*****.**",
subject="Error on localhost in coreservices/service.py",
body="email text",
smtp_server_host_port=None), self.smtp_stub.last_args)
def record_error(hostname,
tb,
exception_message,
preceding_stack=None,
error_threshold=None,
additional_info=None):
''' Helper function to record errors to the flawless backend '''
try:
if not _get_backend_host():
warnings.warn(
"Unable to record error: flawless server hostport not set",
RuntimeWarning)
return
stack = []
while tb is not None:
stack.append(tb)
tb = tb.tb_next
stack_lines = []
for row in preceding_stack or []:
stack_lines.append(
api.StackLine(filename=os.path.abspath(row[0]),
line_number=row[1],
function_name=row[2],
text=row[3]))
for index, tb in enumerate(stack):
filename = tb.tb_frame.f_code.co_filename
func_name = tb.tb_frame.f_code.co_name
lineno = tb.tb_lineno
line = linecache.getline(filename, lineno, tb.tb_frame.f_globals)
frame_locals = None
if index >= (len(stack) - NUM_FRAMES_TO_SAVE):
# Include some limits on max string length & number of variables to keep things from getting
# out of hand
frame_locals = dict(
(k, _myrepr(v))
for k, v in tb.tb_frame.f_locals.items()[:MAX_LOCALS]
if k != "self")
if "self" in tb.tb_frame.f_locals and hasattr(
tb.tb_frame.f_locals["self"], "__dict__"):
frame_locals.update(
dict(("self." + k, _myrepr(v)) for k, v in tb.tb_frame.
f_locals["self"].__dict__.items()[:MAX_LOCALS]
if k != "self"))
# TODO (john): May need to prepend site-packages to filename to get correct path
stack_lines.append(
api.StackLine(filename=os.path.abspath(filename),
line_number=lineno,
function_name=func_name,
text=line,
frame_locals=frame_locals))
data = api.RecordErrorRequest(
traceback=stack_lines,
exception_message=exception_message,
hostname=hostname,
error_threshold=error_threshold,
additional_info=additional_info,
)
req = urllib2.Request(url="http://%s/record_error" %
_get_backend_host(),
data=data.dumps())
_send_request(req)
except:
raise