def test_does_not_email_for_whitelisted_errors(self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine(
"/site-packages/lib/doghouse/authentication.py", 7,
"check_auth",
'raise errors.BadAuthenticationError("Something smells off...")'
)
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey(
"lib/doghouse/authentication.py", 7, "check_auth", 'raise errors.BadAuthenticationError("Something smells off...")'):
api.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
False,
"2020-01-01 00:00:00",
is_known_error=True,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEquals(None, self.smtp_stub.last_args)
def test_traces_up_stack_trace_for_errors_originating_from_building_blocks(
self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/coreservices/service.py", 7,
"serve", "..."),
api.StackLine(
"/site-packages/apps/shopkick/doghouse/lib/base.py", 9,
"_get_request_param",
'raise errors.BadRequestError("Missing param %s" % name)')
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
def test_records_error_only_once(self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/coreservices/service.py", 7,
"serve", "..."),
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self._set_stub_time(datetime.datetime(2020, 1, 2))
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api.ErrorInfo(2,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-02 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual(1, len(self.smtp_stub.args_list))
def test_doesnt_email_on_errors_before_cutoff_date(self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/coreservices/service.py", 7,
"serve", "..."),
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x")
],
exception_message="email text",
hostname="localhost",
)
self.popen_stub.stdout = StringIO.StringIO(
"75563df6e9d1efe44b48f6643fde9ebbd822b0c5 25 25 1\n"
"author John Egan\n"
"author-mail <*****@*****.**>\n"
"author-time %d\n"
"author-tz -0800\n" %
int(time.mktime(datetime.datetime(2009, 7, 30).timetuple())))
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api.ErrorInfo(1,
"*****@*****.**",
"2009-07-30 00:00:00",
False,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual(None, self.smtp_stub.last_args)
def test_doesnt_report_errors_under_threshold(self):
self.test_config.report_error_threshold = 2
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/coreservices/service.py", 7,
"serve", "..."),
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
False,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEquals(None, self.smtp_stub.last_args)
def test_records_error_with_thrift_in_file_name(self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/coreservices/thrift_file.py", 7,
"serve", "..."),
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey("coreservices/thrift_file.py", 7, "serve", "..."):
api.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
def test_records_error(self):
req = api.RecordErrorRequest(
traceback=[
api.StackLine("/site-packages/lib/test.py", 5, "test_func",
"code"),
api.StackLine("/site-packages/coreservices/service.py", 7,
"serve", "..."),
api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9,
"call", "x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req.dumps())
self.assertDictEquals(
{
api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual([
"git", "--git-dir=/tmp/.git", "--work-tree=/tmp", "blame", "-p",
"/tmp/coreservices/service.py", "-L", "7,+1"
], self.popen_stub.last_args)
self.assertEmailEquals(
dict(to_addresses=["*****@*****.**"],
from_address="*****@*****.**",
subject="Error on localhost in coreservices/service.py",
body="email text",
smtp_server_host_port=None), self.smtp_stub.last_args)
def _record_error(self, request):
# Parse request
request = api.RecordErrorRequest.loads(request)
# Figure out which line in the stack trace is to blame for the error
key, blamed_entry, email_recipients = self._blame_line(
request.traceback)
if not key:
return
# If this error hasn't been reported before, then find the dev responsible
err_info = None
if key not in self.errors_seen:
# If flawless is being flooded wih errors, limit the number of git blames so the
# service doesn't fall over. We don't use a thread safe counter, because 10
# git blames is just a soft limit
if self.number_of_git_blames_running > config.max_concurrent_git_blames:
log.error(
"Unable to process %s because %d git blames already running"
% (str(key), self.number_of_git_blames_running))
return
try:
self.number_of_git_blames_running += 1
email, last_touched_ts = self.repository.blame(
key.filename, key.line_number)
finally:
self.number_of_git_blames_running -= 1
dev_email = self._get_email(email)
last_touched_ts = last_touched_ts or 0
cur_time = self._convert_epoch_ms(
datetime.datetime).strftime("%Y-%m-%d %H:%M:%S")
mod_time = self._convert_epoch_ms(datetime.datetime,
epoch_ms=last_touched_ts * 1000)
mod_time = mod_time.strftime("%Y-%m-%d %H:%M:%S")
known_entry = self._get_entry(blamed_entry, self.known_errors)
err_info = api.ErrorInfo(error_count=1,
developer_email=dev_email or "unknown",
date=mod_time,
email_sent=False,
last_occurrence=cur_time,
is_known_error=bool(known_entry),
last_error_data=request)
self.errors_seen[key] = err_info
log.info("Error %s caused by %s on %s" %
(str(key), dev_email, mod_time))
if not dev_email:
self._handle_flawless_issue(
"Unable to do blame for %s. You may want to consider setting "
"only_blame_filepaths_matching in your flawless.cfg " %
str(key))
err_info.email_sent = True
return
# If we've already seen this error then update the error count
elif key in self.errors_seen:
err_info = self.errors_seen[key]
err_info.error_count += 1
err_info.last_error_data = request
cur_dt = self._convert_epoch_ms(datetime.datetime)
err_info.last_occurrence = cur_dt.strftime("%Y-%m-%d %H:%M:%S")
self.errors_seen[key] = err_info
# Figure out if we should send an email or not
send_email = False
known_entry = None
if blamed_entry not in self.known_errors[blamed_entry.filename]:
# If it is an unknown error, then it must meet certain criteria. The code must have been
# touched after report_only_after_minimum_date so errors in old code can be ignored. It
# also has to have occurred at least report_error_threshold times (although the client
# is allowed to override that value).
if (not err_info.email_sent
and err_info.date >= config.report_only_after_minimum_date
and err_info.error_count >=
(request.error_threshold or config.report_error_threshold)):
send_email = True
else:
# If it is a known error, we allow fine grainted control of how frequently emails will
# be sent. An email will be sent if it has passed the min_alert_threshold, and/or this
# is the Nth occurrence as defined alert_every_n_occurences. If it has passed
# max_alert_threshold then no emails will be sent.
known_entry = self._get_entry(blamed_entry, self.known_errors)
if (known_entry.min_alert_threshold
and err_info.error_count >= known_entry.min_alert_threshold
and not err_info.email_sent):
send_email = True
if (known_entry.alert_every_n_occurences and err_info.error_count %
known_entry.alert_every_n_occurences == 0):
send_email = True
if (known_entry.max_alert_threshold is not None and
err_info.error_count > known_entry.max_alert_threshold):
send_email = False
# Send email if applicable
if send_email:
email_body = []
dev_email = self._get_email(err_info.developer_email)
if dev_email:
email_recipients.append(dev_email)
# Add additional recipients that have registered for this error
if blamed_entry.filename in self.watch_only_if_blamed:
email_recipients.extend(
self.watch_only_if_blamed[blamed_entry.filename])
if known_entry:
email_recipients.extend(known_entry.email_recipients or [])
email_body.append(known_entry.email_header or "")
email_body.append(self._format_traceback(request))
email_body.append(
"<br /><br /><a href='http://%s/add_known_error?%s'>Add to whitelist</a>"
% (config.hostname + ":" + str(config.port),
urllib.urlencode(
dict(filename=key.filename,
function_name=key.function_name,
code_fragment=key.text))))
# Send the email
log.info("Sending email for %s to %s" %
(str(key), ", ".join(email_recipients)))
self._sendmail(
to_addresses=email_recipients,
subject="Error on %s in %s" % (request.hostname, key.filename),
body="<br />".join([s for s in email_body if s]),
)
err_info.email_sent = True
self.errors_seen[key] = err_info