def test_removes_duplicate_emails(self):
# Almost the same setup as test_uses_threshold_specified_in_request except the traceback includes
# a path that John is watching. Since he's also the developer who committed the buggy code, he is
# the one and only email recipient.
self.test_config.report_error_threshold = 2
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine(
"/site-packages/lib/no/such/path/for/testing.py", 7,
"fubar", "..."),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
hostname="localhost",
error_threshold=1,
)
self.handler.record_error(req)
self.assertEmailEquals(
dict(
to_addresses=["*****@*****.**"],
from_address="*****@*****.**",
cc_address=None,
bcc_address=None,
subject="Error on localhost in lib/no/such/path/for/testing.py",
body="email text",
smtp_server_host_port=None), self.smtp_stub.last_args)
def test_records_error_with_thrift_in_file_name(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine(
"/site-packages/coreservices/thrift_file.py", 7, "serve",
"..."),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey("coreservices/thrift_file.py", 7, "serve", "..."):
api_ttypes.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
def test_does_not_email_for_whitelisted_errors(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine(
"/site-packages/lib/doghouse/authentication.py", 7,
"check_auth",
'raise errors.BadAuthenticationError("Something smells off...")'
)
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey(
"lib/doghouse/authentication.py", 7, "check_auth", 'raise errors.BadAuthenticationError("Something smells off...")'):
api_ttypes.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
False,
"2020-01-01 00:00:00",
is_known_error=True,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual(None, self.smtp_stub.last_args)
def test_traces_up_stack_trace_for_errors_originating_from_building_blocks(
self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/coreservices/service.py",
7, "serve", "..."),
api_ttypes.StackLine(
"/site-packages/apps/shopkick/doghouse/lib/base.py", 9,
"_get_request_param",
'raise errors.BadRequestError("Missing param %s" % name)')
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api_ttypes.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
def test_doesnt_email_on_errors_before_cutoff_date(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/coreservices/service.py",
7, "serve", "..."),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
hostname="localhost",
)
self.popen_stub.stdout = StringIO(
"75563df6e9d1efe44b48f6643fde9ebbd822b0c5 25 25 1\n"
"author John Egan\n"
"author-mail <*****@*****.**>\n"
"author-time %d\n"
"author-tz -0800\n" %
int(time.mktime(datetime.datetime(2009, 7, 30).timetuple())))
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api_ttypes.ErrorInfo(1,
"*****@*****.**",
"2009-07-30 00:00:00",
False,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual(None, self.smtp_stub.last_args)
def test_records_error_only_once(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/coreservices/service.py",
7, "serve", "..."),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self._set_stub_time(datetime.datetime(2020, 1, 2))
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api_ttypes.ErrorInfo(2,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-02 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual(1, len(self.smtp_stub.args_list))
def test_email_includes_watchers(self):
# Almost the same setup as test_uses_threshold_specified_in_request except the traceback includes
# a path that Yen is watching
self.test_config.report_error_threshold = 2
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/tools/furminator.py", 7,
"fubar", "..."),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
hostname="localhost",
error_threshold=1,
)
self.handler.record_error(req)
self.assertEmailEquals(
dict(to_addresses=[
"*****@*****.**", "*****@*****.**",
"*****@*****.**"
],
from_address="*****@*****.**",
cc_address=None,
bcc_address=None,
subject="Error on localhost in tools/furminator.py",
body="email text",
smtp_server_host_port=None), self.smtp_stub.last_args)
def test_uses_threshold_specified_in_request(self):
self.test_config.report_error_threshold = 2
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/coreservices/service.py",
7, "serve", "..."),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
hostname="localhost",
error_threshold=1,
)
self.handler.record_error(req)
self.assertEmailEquals(
dict(to_addresses=["*****@*****.**"],
from_address="*****@*****.**",
cc_address=None,
bcc_address=None,
subject="Error on localhost in coreservices/service.py",
body="email text",
smtp_server_host_port=None), self.smtp_stub.last_args)
def test_doesnt_report_errors_under_threshold(self):
self.test_config.report_error_threshold = 2
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/coreservices/service.py",
7, "serve", "..."),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api_ttypes.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
False,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual(None, self.smtp_stub.last_args)
def test_ignores_third_party_whitelisted_errors_for_facebook(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/facebook.py", 7,
"post_treat_to_facebook",
'urllib.urlencode(args), post_data)')
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals({}, self.handler.errors_seen.dict)
def test_ignores_error_in_thrift_directory(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x"),
api_ttypes.StackLine(
"/site-packages/coreservices/thrift/file.py", 7, "serve",
"...")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals({}, self.handler.errors_seen.dict)
def test_ignores_third_party_whitelisted_errors(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine(
"/site-packages/SQLAlchemy-0.5.6-py2.6.egg/sqlalchemy/pool.py",
7, "do_get",
'raise exc.TimeoutError("QueuePool limit of size %d overflow %d '
'reached, connection timed out, timeout %d" % (self.size(),'
'self.overflow(), self._timeout))')
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals({}, self.handler.errors_seen.dict)
def test_ignores_ignored_exceptions(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/coreservices/service.py",
7, "serve", "..."),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
exception_type="exceptions.BananaException",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals({}, self.handler.errors_seen.dict)
def test_email_includes_extra_information(self):
# Traceback includes a path that has extra_information tagged on it
self.test_config.report_error_threshold = 2
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine(
"/site-packages/coreservices/waterbowl/rewards/water.py",
5, "make_external_api_ttypes_request",
"raise api_ttypes.WaterbowlError(error_code=api_ttypes.WaterbowlErrorCode."
"OUT_OF_WATER, message=str(e))"),
],
exception_message="email text",
hostname="localhost",
error_threshold=1,
additional_info="extra stuff",
)
self.handler.record_error(req)
self.assertEmailEquals(
dict(
to_addresses=[
"*****@*****.**", "*****@*****.**",
"*****@*****.**"
],
from_address="*****@*****.**",
cc_address=None,
bcc_address=None,
subject=
"Error on localhost in coreservices/waterbowl/rewards/water.py",
body=
'NOTE: This error typically does not require dev team involvement.',
smtp_server_host_port=None), self.smtp_stub.last_args)
body = email.message_from_string(
self.smtp_stub.last_args["body"]).get_payload(
decode=True).decode('utf8')
self.assertTrue("email text" in body)
self.assertTrue("extra stuff" in body)
def test_always_alerts_on_red_alert_errors(self):
self.test_config.report_error_threshold = 3
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/coreservices/service.py",
7, "serve", "..."),
api_ttypes.StackLine(
'/site-packages/coreservices/service/utils.py', 9,
"check_water_levels",
'% (waterbowl_id, min_required_level))'),
api_ttypes.StackLine(
"/site-packages/coreservices/waterbowl/rewards/water.py",
5, "make_external_api_ttypes_request",
"raise api_ttypes.WaterbowlError(error_code=api_ttypes.WaterbowlErrorCode."
"OUT_OF_WATER, message=str(e))")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
# The 2 red alert recipients plus the developer responsible
self.assertEmailEquals(
dict(
to_addresses=[
"*****@*****.**", "*****@*****.**",
"*****@*****.**"
],
from_address="*****@*****.**",
cc_address=None,
bcc_address=None,
subject=
"Error on localhost in coreservices/waterbowl/rewards/water.py",
body="email text",
smtp_server_host_port=None), self.smtp_stub.last_args)
def test_handles_disowned_files(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/scripts/my_script.py", 7,
"run_all", "code"),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey("scripts/my_script.py", 7, "run_all", "code"):
api_ttypes.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual([
"git", "--git-dir=/tmp/.git", "--work-tree=/tmp", "blame", "-p",
"/tmp/scripts/my_script.py", "-L", "7,+1"
], self.popen_stub.last_args)
self.assertEmailEquals(
dict(to_addresses=["*****@*****.**"],
from_address="*****@*****.**",
subject="Error on localhost in scripts/my_script.py",
body="email text",
smtp_server_host_port=None), self.smtp_stub.last_args)
def record_error(hostname,
exc_info,
preceding_stack=None,
error_threshold=None,
additional_info=None):
''' Helper function to record errors to the flawless backend '''
stack = []
exc_type, exc_value, sys_traceback = exc_info
while sys_traceback is not None:
stack.append(sys_traceback)
sys_traceback = sys_traceback.tb_next
stack_lines = []
for row in preceding_stack or []:
stack_lines.append(
api_ttypes.StackLine(filename=os.path.abspath(row[0]),
line_number=row[1],
function_name=row[2],
text=row[3]))
for index, tb in enumerate(stack):
filename = tb.tb_frame.f_code.co_filename
func_name = tb.tb_frame.f_code.co_name
lineno = tb.tb_lineno
line = linecache.getline(filename, lineno, tb.tb_frame.f_globals)
frame_locals = None
if index >= (len(stack) - NUM_FRAMES_TO_SAVE):
# Include some limits on max string length & number of variables to keep things from getting
# out of hand
frame_locals = dict(
(k, _myrepr(k, v))
for k, v in list(tb.tb_frame.f_locals.items())[:MAX_LOCALS]
if k != "self")
if "self" in tb.tb_frame.f_locals and hasattr(
tb.tb_frame.f_locals["self"], "__dict__"):
frame_locals.update(
dict(("self." + k, _myrepr(k, v))
for k, v in list(tb.tb_frame.f_locals["self"].
__dict__.items())[:MAX_LOCALS]
if k != "self"))
stack_lines.append(
api_ttypes.StackLine(filename=os.path.abspath(filename),
line_number=lineno,
function_name=func_name,
text=line,
frame_locals=frame_locals))
# Check LRU cache & potentially do not send error report if this client has already reported this error
# several times.
key = CachedErrorInfo.get_hash_key(stack_lines)
info = ERROR_CACHE.get(key) or CachedErrorInfo()
info.increment()
ERROR_CACHE[key] = info
if info.should_report():
error_count = info.mark_reported()
_send_request(
api_ttypes.RecordErrorRequest(
traceback=stack_lines,
exception_message=repr(exc_value),
exception_type=exc_type.__module__ + "." + exc_type.__name__,
hostname=hostname,
error_threshold=error_threshold,
additional_info=additional_info,
error_count=error_count,
))
