def test_traces_up_stack_trace_for_errors_originating_from_building_blocks(
self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/coreservices/service.py",
7, "serve", "..."),
api_ttypes.StackLine(
"/site-packages/apps/shopkick/doghouse/lib/base.py", 9,
"_get_request_param",
'raise errors.BadRequestError("Missing param %s" % name)')
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api_ttypes.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
def test_does_not_email_for_whitelisted_errors(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine(
"/site-packages/lib/doghouse/authentication.py", 7,
"check_auth",
'raise errors.BadAuthenticationError("Something smells off...")'
)
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey(
"lib/doghouse/authentication.py", 7, "check_auth", 'raise errors.BadAuthenticationError("Something smells off...")'):
api_ttypes.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
False,
"2020-01-01 00:00:00",
is_known_error=True,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual(None, self.smtp_stub.last_args)
def test_records_error_only_once(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/coreservices/service.py",
7, "serve", "..."),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self._set_stub_time(datetime.datetime(2020, 1, 2))
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api_ttypes.ErrorInfo(2,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-02 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual(1, len(self.smtp_stub.args_list))
def test_doesnt_email_on_errors_before_cutoff_date(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/coreservices/service.py",
7, "serve", "..."),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
hostname="localhost",
)
self.popen_stub.stdout = StringIO(
"75563df6e9d1efe44b48f6643fde9ebbd822b0c5 25 25 1\n"
"author John Egan\n"
"author-mail <*****@*****.**>\n"
"author-time %d\n"
"author-tz -0800\n" %
int(time.mktime(datetime.datetime(2009, 7, 30).timetuple())))
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api_ttypes.ErrorInfo(1,
"*****@*****.**",
"2009-07-30 00:00:00",
False,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual(None, self.smtp_stub.last_args)
def test_doesnt_report_errors_under_threshold(self):
self.test_config.report_error_threshold = 2
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/coreservices/service.py",
7, "serve", "..."),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey("coreservices/service.py", 7, "serve", "..."):
api_ttypes.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
False,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual(None, self.smtp_stub.last_args)
def test_records_error_with_thrift_in_file_name(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine(
"/site-packages/coreservices/thrift_file.py", 7, "serve",
"..."),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey("coreservices/thrift_file.py", 7, "serve", "..."):
api_ttypes.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
def test_handles_disowned_files(self):
req = api_ttypes.RecordErrorRequest(
traceback=[
api_ttypes.StackLine("/site-packages/lib/test.py", 5,
"test_func", "code"),
api_ttypes.StackLine("/site-packages/scripts/my_script.py", 7,
"run_all", "code"),
api_ttypes.StackLine(
"/site-packages/thirdparty/3rdparty_lib.py", 9, "call",
"x")
],
exception_message="email text",
hostname="localhost",
)
self.handler.record_error(req)
self.assertDictEquals(
{
api_ttypes.ErrorKey("scripts/my_script.py", 7, "run_all", "code"):
api_ttypes.ErrorInfo(1,
"*****@*****.**",
"2017-07-30 00:00:00",
True,
"2020-01-01 00:00:00",
is_known_error=False,
last_error_data=req)
}, self.handler.errors_seen.dict)
self.assertEqual([
"git", "--git-dir=/tmp/.git", "--work-tree=/tmp", "blame", "-p",
"/tmp/scripts/my_script.py", "-L", "7,+1"
], self.popen_stub.last_args)
self.assertEmailEquals(
dict(to_addresses=["*****@*****.**"],
from_address="*****@*****.**",
subject="Error on localhost in scripts/my_script.py",
body="email text",
smtp_server_host_port=None), self.smtp_stub.last_args)
def _record_error(self, request):
log.debug("Recieved error from %s for %s" %
(request.hostname, request.exception_message))
# Skip ignored exceptions (ex: connection errors)
if request.exception_type and request.exception_type in self.ignored_exceptions:
return
# Figure out which line in the stack trace is to blame for the error
key, blamed_entry, email_recipients, was_whitelisted = self._blame_line(
request.traceback)
if not key:
if not was_whitelisted:
name_map = api_ttypes.LineType._VALUES_TO_NAMES
line_types = [(stack_line.filename,
name_map.get(self._get_line_type(stack_line)))
for stack_line in request.traceback]
log.info("Unable to blame: %s" % str(line_types))
return
# If this error hasn't been reported before, then find the dev responsible
err_info = None
if key not in self.errors_seen:
# If flawless is being flooded with errors, limit the number of git blames so the
# service doesn't fall over. We don't use a thread safe counter, because 10
# git blames is just a soft limit
if self.number_of_git_blames_running > config.max_concurrent_git_blames:
log.error(
"Unable to process %s because %d git blames already running"
% (str(key), self.number_of_git_blames_running))
return
try:
self.number_of_git_blames_running += 1
email, last_touched_ts = self.repository.blame(
key.filename, key.line_number)
finally:
self.number_of_git_blames_running -= 1
dev_email = self._get_email(email)
last_touched_ts = last_touched_ts or 0
cur_time = self._convert_epoch_ms(
datetime.datetime).strftime("%Y-%m-%d %H:%M:%S")
mod_time = self._convert_epoch_ms(datetime.datetime,
epoch_ms=last_touched_ts * 1000)
mod_time = mod_time.strftime("%Y-%m-%d %H:%M:%S")
known_entry = self._get_entry(blamed_entry, self.known_errors)
err_info = api_ttypes.ErrorInfo(error_count=1,
developer_email=dev_email
or "unknown",
date=mod_time,
email_sent=False,
last_occurrence=cur_time,
is_known_error=bool(known_entry),
last_error_data=request)
self.errors_seen[key] = err_info
log.info("Error %s caused by %s on %s" %
(str(key), dev_email, mod_time))
if not dev_email:
self._handle_flawless_issue(
"Unable to do blame for %s from %s. You may want to consider setting "
"only_blame_filepaths_matching in your flawless.cfg " %
(str(key), request.hostname))
err_info.email_sent = True
return
# If we've already seen this error then update the error count
elif key in self.errors_seen:
err_info = self.errors_seen[key]
err_info.error_count += request.error_count or 1
err_info.last_error_data = request
cur_dt = self._convert_epoch_ms(datetime.datetime)
err_info.last_occurrence = cur_dt.strftime("%Y-%m-%d %H:%M:%S")
self.errors_seen[key] = err_info
# Figure out if we should send an email or not
send_email = False
known_entry = None
if blamed_entry not in self.known_errors[blamed_entry.filename]:
# If it is an unknown error, then it must meet certain criteria. The code must have been
# touched after report_only_after_minimum_date so errors in old code can be ignored. It
# also has to have occurred at least report_error_threshold times (although the client
# is allowed to override that value).
if (not err_info.email_sent
and err_info.date >= config.report_only_after_minimum_date
and err_info.error_count >=
(request.error_threshold or config.report_error_threshold)):
send_email = True
else:
# If it is a known error, we allow fine grainted control of how frequently emails will
# be sent. An email will be sent if it has passed the min_alert_threshold, and/or this
# is the Nth occurrence as defined alert_every_n_occurrences. If it has passed
# max_alert_threshold then no emails will be sent.
known_entry = self._get_entry(blamed_entry, self.known_errors)
if (known_entry.min_alert_threshold
and err_info.error_count >= known_entry.min_alert_threshold
and not err_info.email_sent):
send_email = True
if (known_entry.alert_every_n_occurrences
and err_info.error_count %
known_entry.alert_every_n_occurrences == 0):
send_email = True
if (known_entry.max_alert_threshold is not None and
err_info.error_count > known_entry.max_alert_threshold):
send_email = False
# Send email if applicable
if send_email:
self._send_error_email(request, key, err_info, blamed_entry,
known_entry, email_recipients)
