def test_removes_duplicate_emails(self): # Almost the same setup as test_uses_threshold_specified_in_request except the traceback includes # a path that John is watching. Since he's also the developer who committed the buggy code, he is # the one and only email recipient. self.test_config.report_error_threshold = 2 req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"), api_ttypes.StackLine( "/site-packages/lib/no/such/path/for/testing.py", 7, "fubar", "..."), api_ttypes.StackLine( "/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x") ], exception_message="email text", hostname="localhost", error_threshold=1, ) self.handler.record_error(req) self.assertEmailEquals( dict( to_addresses=["*****@*****.**"], from_address="*****@*****.**", cc_address=None, bcc_address=None, subject="Error on localhost in lib/no/such/path/for/testing.py", body="email text", smtp_server_host_port=None), self.smtp_stub.last_args)
def test_records_error_with_thrift_in_file_name(self): req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine( "/site-packages/coreservices/thrift_file.py", 7, "serve", "..."), api_ttypes.StackLine( "/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x") ], exception_message="email text", hostname="localhost", ) self.handler.record_error(req) self.assertDictEquals( { api_ttypes.ErrorKey("coreservices/thrift_file.py", 7, "serve", "..."): api_ttypes.ErrorInfo(1, "*****@*****.**", "2017-07-30 00:00:00", True, "2020-01-01 00:00:00", is_known_error=False, last_error_data=req) }, self.handler.errors_seen.dict)
def test_does_not_email_for_whitelisted_errors(self): req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"), api_ttypes.StackLine( "/site-packages/lib/doghouse/authentication.py", 7, "check_auth", 'raise errors.BadAuthenticationError("Something smells off...")' ) ], exception_message="email text", hostname="localhost", ) self.handler.record_error(req) self.assertDictEquals( { api_ttypes.ErrorKey( "lib/doghouse/authentication.py", 7, "check_auth", 'raise errors.BadAuthenticationError("Something smells off...")'): api_ttypes.ErrorInfo(1, "*****@*****.**", "2017-07-30 00:00:00", False, "2020-01-01 00:00:00", is_known_error=True, last_error_data=req) }, self.handler.errors_seen.dict) self.assertEqual(None, self.smtp_stub.last_args)
def test_traces_up_stack_trace_for_errors_originating_from_building_blocks( self): req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"), api_ttypes.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."), api_ttypes.StackLine( "/site-packages/apps/shopkick/doghouse/lib/base.py", 9, "_get_request_param", 'raise errors.BadRequestError("Missing param %s" % name)') ], exception_message="email text", hostname="localhost", ) self.handler.record_error(req) self.assertDictEquals( { api_ttypes.ErrorKey("coreservices/service.py", 7, "serve", "..."): api_ttypes.ErrorInfo(1, "*****@*****.**", "2017-07-30 00:00:00", True, "2020-01-01 00:00:00", is_known_error=False, last_error_data=req) }, self.handler.errors_seen.dict)
def test_doesnt_email_on_errors_before_cutoff_date(self): req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"), api_ttypes.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."), api_ttypes.StackLine( "/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x") ], exception_message="email text", hostname="localhost", ) self.popen_stub.stdout = StringIO( "75563df6e9d1efe44b48f6643fde9ebbd822b0c5 25 25 1\n" "author John Egan\n" "author-mail <*****@*****.**>\n" "author-time %d\n" "author-tz -0800\n" % int(time.mktime(datetime.datetime(2009, 7, 30).timetuple()))) self.handler.record_error(req) self.assertDictEquals( { api_ttypes.ErrorKey("coreservices/service.py", 7, "serve", "..."): api_ttypes.ErrorInfo(1, "*****@*****.**", "2009-07-30 00:00:00", False, "2020-01-01 00:00:00", is_known_error=False, last_error_data=req) }, self.handler.errors_seen.dict) self.assertEqual(None, self.smtp_stub.last_args)
def test_records_error_only_once(self): req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"), api_ttypes.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."), api_ttypes.StackLine( "/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x") ], exception_message="email text", hostname="localhost", ) self.handler.record_error(req) self._set_stub_time(datetime.datetime(2020, 1, 2)) self.handler.record_error(req) self.assertDictEquals( { api_ttypes.ErrorKey("coreservices/service.py", 7, "serve", "..."): api_ttypes.ErrorInfo(2, "*****@*****.**", "2017-07-30 00:00:00", True, "2020-01-02 00:00:00", is_known_error=False, last_error_data=req) }, self.handler.errors_seen.dict) self.assertEqual(1, len(self.smtp_stub.args_list))
def test_email_includes_watchers(self): # Almost the same setup as test_uses_threshold_specified_in_request except the traceback includes # a path that Yen is watching self.test_config.report_error_threshold = 2 req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"), api_ttypes.StackLine("/site-packages/tools/furminator.py", 7, "fubar", "..."), api_ttypes.StackLine( "/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x") ], exception_message="email text", hostname="localhost", error_threshold=1, ) self.handler.record_error(req) self.assertEmailEquals( dict(to_addresses=[ "*****@*****.**", "*****@*****.**", "*****@*****.**" ], from_address="*****@*****.**", cc_address=None, bcc_address=None, subject="Error on localhost in tools/furminator.py", body="email text", smtp_server_host_port=None), self.smtp_stub.last_args)
def test_uses_threshold_specified_in_request(self): self.test_config.report_error_threshold = 2 req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"), api_ttypes.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."), api_ttypes.StackLine( "/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x") ], exception_message="email text", hostname="localhost", error_threshold=1, ) self.handler.record_error(req) self.assertEmailEquals( dict(to_addresses=["*****@*****.**"], from_address="*****@*****.**", cc_address=None, bcc_address=None, subject="Error on localhost in coreservices/service.py", body="email text", smtp_server_host_port=None), self.smtp_stub.last_args)
def test_doesnt_report_errors_under_threshold(self): self.test_config.report_error_threshold = 2 req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"), api_ttypes.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."), api_ttypes.StackLine( "/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x") ], exception_message="email text", hostname="localhost", ) self.handler.record_error(req) self.assertDictEquals( { api_ttypes.ErrorKey("coreservices/service.py", 7, "serve", "..."): api_ttypes.ErrorInfo(1, "*****@*****.**", "2017-07-30 00:00:00", False, "2020-01-01 00:00:00", is_known_error=False, last_error_data=req) }, self.handler.errors_seen.dict) self.assertEqual(None, self.smtp_stub.last_args)
def test_ignores_third_party_whitelisted_errors_for_facebook(self): req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"), api_ttypes.StackLine("/site-packages/facebook.py", 7, "post_treat_to_facebook", 'urllib.urlencode(args), post_data)') ], exception_message="email text", hostname="localhost", ) self.handler.record_error(req) self.assertDictEquals({}, self.handler.errors_seen.dict)
def test_ignores_error_in_thrift_directory(self): req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine( "/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x"), api_ttypes.StackLine( "/site-packages/coreservices/thrift/file.py", 7, "serve", "...") ], exception_message="email text", hostname="localhost", ) self.handler.record_error(req) self.assertDictEquals({}, self.handler.errors_seen.dict)
def test_ignores_third_party_whitelisted_errors(self): req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"), api_ttypes.StackLine( "/site-packages/SQLAlchemy-0.5.6-py2.6.egg/sqlalchemy/pool.py", 7, "do_get", 'raise exc.TimeoutError("QueuePool limit of size %d overflow %d ' 'reached, connection timed out, timeout %d" % (self.size(),' 'self.overflow(), self._timeout))') ], exception_message="email text", hostname="localhost", ) self.handler.record_error(req) self.assertDictEquals({}, self.handler.errors_seen.dict)
def test_ignores_ignored_exceptions(self): req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"), api_ttypes.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."), api_ttypes.StackLine( "/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x") ], exception_message="email text", exception_type="exceptions.BananaException", hostname="localhost", ) self.handler.record_error(req) self.assertDictEquals({}, self.handler.errors_seen.dict)
def test_always_alerts_on_red_alert_errors(self): self.test_config.report_error_threshold = 3 req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"), api_ttypes.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."), api_ttypes.StackLine( '/site-packages/coreservices/service/utils.py', 9, "check_water_levels", '% (waterbowl_id, min_required_level))'), api_ttypes.StackLine( "/site-packages/coreservices/waterbowl/rewards/water.py", 5, "make_external_api_ttypes_request", "raise api_ttypes.WaterbowlError(error_code=api_ttypes.WaterbowlErrorCode." "OUT_OF_WATER, message=str(e))") ], exception_message="email text", hostname="localhost", ) self.handler.record_error(req) # The 2 red alert recipients plus the developer responsible self.assertEmailEquals( dict( to_addresses=[ "*****@*****.**", "*****@*****.**", "*****@*****.**" ], from_address="*****@*****.**", cc_address=None, bcc_address=None, subject= "Error on localhost in coreservices/waterbowl/rewards/water.py", body="email text", smtp_server_host_port=None), self.smtp_stub.last_args)
def test_handles_disowned_files(self): req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"), api_ttypes.StackLine("/site-packages/scripts/my_script.py", 7, "run_all", "code"), api_ttypes.StackLine( "/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x") ], exception_message="email text", hostname="localhost", ) self.handler.record_error(req) self.assertDictEquals( { api_ttypes.ErrorKey("scripts/my_script.py", 7, "run_all", "code"): api_ttypes.ErrorInfo(1, "*****@*****.**", "2017-07-30 00:00:00", True, "2020-01-01 00:00:00", is_known_error=False, last_error_data=req) }, self.handler.errors_seen.dict) self.assertEqual([ "git", "--git-dir=/tmp/.git", "--work-tree=/tmp", "blame", "-p", "/tmp/scripts/my_script.py", "-L", "7,+1" ], self.popen_stub.last_args) self.assertEmailEquals( dict(to_addresses=["*****@*****.**"], from_address="*****@*****.**", subject="Error on localhost in scripts/my_script.py", body="email text", smtp_server_host_port=None), self.smtp_stub.last_args)
def test_email_includes_extra_information(self): # Traceback includes a path that has extra_information tagged on it self.test_config.report_error_threshold = 2 req = api_ttypes.RecordErrorRequest( traceback=[ api_ttypes.StackLine( "/site-packages/coreservices/waterbowl/rewards/water.py", 5, "make_external_api_ttypes_request", "raise api_ttypes.WaterbowlError(error_code=api_ttypes.WaterbowlErrorCode." "OUT_OF_WATER, message=str(e))"), ], exception_message="email text", hostname="localhost", error_threshold=1, additional_info="extra stuff", ) self.handler.record_error(req) self.assertEmailEquals( dict( to_addresses=[ "*****@*****.**", "*****@*****.**", "*****@*****.**" ], from_address="*****@*****.**", cc_address=None, bcc_address=None, subject= "Error on localhost in coreservices/waterbowl/rewards/water.py", body= 'NOTE: This error typically does not require dev team involvement.', smtp_server_host_port=None), self.smtp_stub.last_args) body = email.message_from_string( self.smtp_stub.last_args["body"]).get_payload( decode=True).decode('utf8') self.assertTrue("email text" in body) self.assertTrue("extra stuff" in body)
def record_error(hostname, exc_info, preceding_stack=None, error_threshold=None, additional_info=None): ''' Helper function to record errors to the flawless backend ''' stack = [] exc_type, exc_value, sys_traceback = exc_info while sys_traceback is not None: stack.append(sys_traceback) sys_traceback = sys_traceback.tb_next stack_lines = [] for row in preceding_stack or []: stack_lines.append( api_ttypes.StackLine(filename=os.path.abspath(row[0]), line_number=row[1], function_name=row[2], text=row[3])) for index, tb in enumerate(stack): filename = tb.tb_frame.f_code.co_filename func_name = tb.tb_frame.f_code.co_name lineno = tb.tb_lineno line = linecache.getline(filename, lineno, tb.tb_frame.f_globals) frame_locals = None if index >= (len(stack) - NUM_FRAMES_TO_SAVE): # Include some limits on max string length & number of variables to keep things from getting # out of hand frame_locals = dict( (k, _myrepr(k, v)) for k, v in list(tb.tb_frame.f_locals.items())[:MAX_LOCALS] if k != "self") if "self" in tb.tb_frame.f_locals and hasattr( tb.tb_frame.f_locals["self"], "__dict__"): frame_locals.update( dict(("self." + k, _myrepr(k, v)) for k, v in list(tb.tb_frame.f_locals["self"]. __dict__.items())[:MAX_LOCALS] if k != "self")) stack_lines.append( api_ttypes.StackLine(filename=os.path.abspath(filename), line_number=lineno, function_name=func_name, text=line, frame_locals=frame_locals)) # Check LRU cache & potentially do not send error report if this client has already reported this error # several times. key = CachedErrorInfo.get_hash_key(stack_lines) info = ERROR_CACHE.get(key) or CachedErrorInfo() info.increment() ERROR_CACHE[key] = info if info.should_report(): error_count = info.mark_reported() _send_request( api_ttypes.RecordErrorRequest( traceback=stack_lines, exception_message=repr(exc_value), exception_type=exc_type.__module__ + "." + exc_type.__name__, hostname=hostname, error_threshold=error_threshold, additional_info=additional_info, error_count=error_count, ))