def process_crashfile(crashfile):
"""Parse the contents of a crashfile and submit sentry messages."""
crash_info = read_crashfile(str(crashfile))
with sentry_sdk.push_scope() as scope:
scope.level = 'fatal'
# Extract node name
node_name = crash_info.pop('node').split('.')[-1]
scope.set_tag("node_name", node_name)
# Massage the traceback, extract the gist
traceback = crash_info.pop('traceback')
# last line is probably most informative summary
gist = traceback.splitlines()[-1]
exception_text_start = 1
for line in traceback.splitlines()[1:]:
if not line[0].isspace():
break
exception_text_start += 1
exception_text = '\n'.join(
traceback.splitlines()[exception_text_start:])
# Extract inputs, if present
inputs = crash_info.pop('inputs', None)
if inputs:
scope.set_extra('inputs', dict(inputs))
# Extract any other possible metadata in the crash file
for k, v in crash_info.items():
strv = list(_chunks(str(v)))
if len(strv) == 1:
scope.set_extra(k, strv[0])
else:
for i, chunk in enumerate(strv):
scope.set_extra('%s_%02d' % (k, i), chunk)
fingerprint = ''
issue_title = '{}: {}'.format(node_name, gist)
for new_fingerprint, error_snippets in KNOWN_ERRORS.items():
for error_snippet in error_snippets:
if error_snippet in traceback:
fingerprint = new_fingerprint
issue_title = new_fingerprint
break
if fingerprint:
break
message = issue_title + '\n\n'
message += exception_text[-(8192 - len(message)):]
if fingerprint:
sentry_sdk.add_breadcrumb(message=fingerprint, level='fatal')
else:
# remove file paths
fingerprint = re.sub(r"(/[^/ ]*)+/?", '', message)
# remove words containing numbers
fingerprint = re.sub(r"([a-zA-Z]*[0-9]+[a-zA-Z]*)+", '',
fingerprint)
# adding the return code if it exists
for line in message.splitlines():
if line.startswith("Return code"):
fingerprint += line
break
scope.fingerprint = [fingerprint]
sentry_sdk.capture_message(message, 'fatal')
def index_error_dir(self, error_dir):
"""
Crawl subjects crash directory for the corresponding run, report to sentry, and
populate self.errors.
"""
for crashfile in error_dir.glob('crash*.*'):
crash_info = read_crashfile(str(crashfile))
if self.sentry_sdk:
with self.sentry_sdk.push_scope() as scope:
node_name = crash_info['node'].split('.')[-1]
# last line is probably most informative summary
gist = crash_info['traceback'].split('\n')[-1]
exception_text_start = 1
for line in crash_info['traceback'].split('\n')[1:]:
if not line[0].isspace():
break
exception_text_start += 1
exception_text = '\n'.join(crash_info['traceback'].split(
'\n')[exception_text_start:])
scope.set_tag("node_name", node_name)
chunk_size = 16384
for k, v in crash_info.items():
if k == 'inputs':
scope.set_extra(k, dict(v))
elif isinstance(v, str) and len(v) > chunk_size:
chunks = [
v[i:i + chunk_size]
for i in range(0, len(v), chunk_size)
]
for i, chunk in enumerate(chunks):
scope.set_extra('%s_%02d' % (k, i), chunk)
else:
scope.set_extra(k, v)
scope.level = 'fatal'
# Group common events with pre specified fingerprints
fingerprint_dict = {
'permission-denied':
["PermissionError: [Errno 13] Permission denied"],
'memory-error':
["MemoryError", "Cannot allocate memory"],
'reconall-already-running': [
"ERROR: it appears that recon-all is already running"
],
'no-disk-space': [
"OSError: [Errno 28] No space left on device",
"[Errno 122] Disk quota exceeded"
],
'sigkill': ["Return code: 137"],
'keyboard-interrupt': ["KeyboardInterrupt"]
}
fingerprint = ''
issue_title = node_name + ': ' + gist
for new_fingerprint, error_snippets in fingerprint_dict.items(
):
for error_snippet in error_snippets:
if error_snippet in crash_info['traceback']:
fingerprint = new_fingerprint
issue_title = new_fingerprint
break
if fingerprint:
break
message = issue_title + '\n\n'
message += exception_text[-(8192 - len(message)):]
if fingerprint:
self.sentry_sdk.add_breadcrumb(message=fingerprint,
level='fatal')
else:
# remove file paths
fingerprint = re.sub(r"(/[^/ ]*)+/?", '', message)
# remove words containing numbers
fingerprint = re.sub(r"([a-zA-Z]*[0-9]+[a-zA-Z]*)+",
'', fingerprint)
# adding the return code if it exists
for line in message.split('\n'):
if line.startswith("Return code"):
fingerprint += line
break
scope.fingerprint = [fingerprint]
self.sentry_sdk.capture_message(message, 'fatal')
self.errors.append(crash_info)
def index_error_dir(self, error_dir):
"""
Crawl subjects crash directory for the corresponding run, report to sentry, and
populate self.errors.
"""
for crashfile in error_dir.glob('crash*.*'):
crash_info = read_crashfile(str(crashfile))
if self.sentry_sdk:
with self.sentry_sdk.push_scope() as scope:
node_name = crash_info['node'].split('.')[-1]
# last line is probably most informative summary
gist = crash_info['traceback'].split('\n')[-1]
exception_text_start = 1
for line in crash_info['traceback'].split('\n')[1:]:
if not line[0].isspace():
break
exception_text_start += 1
exception_text = '\n'.join(crash_info['traceback'].split('\n')[
exception_text_start:])
scope.set_tag("node_name", node_name)
chunk_size = 16384
for k, v in crash_info.items():
if k == 'inputs':
scope.set_extra(k, dict(v))
elif isinstance(v, str) and len(v) > chunk_size:
chunks = [v[i:i + chunk_size] for i in range(0, len(v), chunk_size)]
for i, chunk in enumerate(chunks):
scope.set_extra('%s_%02d' % (k, i), chunk)
else:
scope.set_extra(k, v)
scope.level = 'fatal'
# Group common events with pre specified fingerprints
fingerprint_dict = {'permission-denied': [
"PermissionError: [Errno 13] Permission denied"],
'memory-error': ["MemoryError", "Cannot allocate memory"],
'reconall-already-running': [
"ERROR: it appears that recon-all is already running"],
'no-disk-space': [
"OSError: [Errno 28] No space left on device",
"[Errno 122] Disk quota exceeded"],
'sigkill': ["Return code: 137"],
'keyboard-interrupt': ["KeyboardInterrupt"]}
fingerprint = ''
issue_title = node_name + ': ' + gist
for new_fingerprint, error_snippets in fingerprint_dict.items():
for error_snippet in error_snippets:
if error_snippet in crash_info['traceback']:
fingerprint = new_fingerprint
issue_title = new_fingerprint
break
if fingerprint:
break
message = issue_title + '\n\n'
message += exception_text[-(8192-len(message)):]
if fingerprint:
self.sentry_sdk.add_breadcrumb(message=fingerprint, level='fatal')
else:
# remove file paths
fingerprint = re.sub(r"(/[^/ ]*)+/?", '', message)
# remove words containing numbers
fingerprint = re.sub(r"([a-zA-Z]*[0-9]+[a-zA-Z]*)+", '', fingerprint)
# adding the return code if it exists
for line in message.split('\n'):
if line.startswith("Return code"):
fingerprint += line
break
scope.fingerprint = [fingerprint]
self.sentry_sdk.capture_message(message, 'fatal')
self.errors.append(crash_info)