def __init__(self, config):
super(SignatureGeneratorRule, self).__init__(config)
try:
sentry_dsn = self.config.sentry.dsn
except (AttributeError, KeyError):
sentry_dsn = None
self.sentry_dsn = sentry_dsn
self.generator = SignatureGenerator(error_handler=self._error_handler)
def __init__(self, config):
super(SignatureGeneratorRule, self).__init__(config)
try:
sentry_dsn = self.config.sentry.dsn
except KeyError:
# NOTE(willkg): DotDict raises a KeyError when things are missing
sentry_dsn = None
self.sentry_dsn = sentry_dsn
self.generator = SignatureGenerator(error_handler=self._error_handler)
def __init__(self, config):
super(SignatureGeneratorRule, self).__init__(config)
try:
sentry_dsn = self.config.sentry.dsn
except KeyError:
# DotDict raises a KeyError when things are missing
sentry_dsn = None
self.generator = SignatureGenerator(sentry_dsn=sentry_dsn)
def test_sentry_dsn(self, mock_raven):
class BadRule(object):
pass
sentry_dsn = 'https://*****:*****@sentry.example.com/'
generator = SignatureGenerator(pipeline=[BadRule()], sentry_dsn=sentry_dsn)
generator.generate({}, {})
# Make sure the client was instantiated with the sentry_dsn
mock_raven.Client.assert_called_once_with(dsn=sentry_dsn)
# Make sure captureExeption was called
mock_raven.Client().captureException.assert_called_once_with()
def test_sentry_dsn(self, mock_raven):
class BadRule(object):
pass
sentry_dsn = 'https://*****:*****@sentry.example.com/'
generator = SignatureGenerator(pipeline=[BadRule()],
sentry_dsn=sentry_dsn)
generator.generate({}, {})
# Make sure the client was instantiated with the sentry_dsn
mock_raven.Client.assert_called_once_with(dsn=sentry_dsn)
# Make sure captureExeption was called
mock_raven.Client().captureException.assert_called_once_with()
def test_failing_rule(self):
class BadRule(object):
pass
generator = SignatureGenerator(pipeline=[BadRule()])
ret = generator.generate({}, {})
expected = {
'notes': [
'Rule BadRule failed: \'BadRule\' object has no attribute \'predicate\''
],
'signature': ''
}
assert ret == expected
def test_failing_rule(self):
class BadRule(object):
pass
generator = SignatureGenerator(pipeline=[BadRule()])
ret = generator.generate({}, {})
expected = {
'notes': [
'Rule BadRule failed: \'BadRule\' object has no attribute \'predicate\''
],
'signature':
''
}
assert ret == expected
class SignatureGeneratorRule(Rule):
"""Generates a Socorro crash signature"""
def __init__(self, config):
super(SignatureGeneratorRule, self).__init__(config)
try:
sentry_dsn = self.config.sentry.dsn
except (AttributeError, KeyError):
sentry_dsn = None
self.sentry_dsn = sentry_dsn
self.generator = SignatureGenerator(error_handler=self._error_handler)
def _error_handler(self, crash_data, exc_info, extra):
"""Captures errors from signature generation"""
extra['uuid'] = crash_data.get('uuid', None)
raven_client.capture_error(
self.sentry_dsn, self.config.logger, exc_info=exc_info, extra=extra
)
def action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
# Generate a crash signature and capture the signature and notes
crash_data = convert_to_crash_data(raw_crash, processed_crash)
ret = self.generator.generate(crash_data)
processed_crash['signature'] = ret['signature']
if 'proto_signature' in ret:
processed_crash['proto_signature'] = ret['proto_signature']
processor_meta['processor_notes'].extend(ret['notes'])
def test_empty_dicts(self):
generator = SignatureGenerator()
ret = generator.generate({}, {})
# NOTE(willkg): This is what the current pipeline yields. If any of those parts change, this
# might change, too. The point of this test is that we can pass in empty dicts and the
# SignatureGenerator and the rules in the default pipeline don't fall over.
expected = {
'notes': [
'CSignatureTool: No signature could be created because we do not know '
'which thread crashed'
],
'signature': 'EMPTY: no crashing thread identified'
}
assert ret == expected
class SignatureGeneratorRule(Rule):
"""Generates a Socorro crash signature"""
def __init__(self, config):
super().__init__(config)
try:
sentry_dsn = self.config.sentry.dsn
except (AttributeError, KeyError):
sentry_dsn = None
self.sentry_dsn = sentry_dsn
self.generator = SignatureGenerator(error_handler=self._error_handler)
def _error_handler(self, crash_data, exc_info, extra):
"""Captures errors from signature generation"""
extra['uuid'] = crash_data.get('uuid', None)
sentry_client.capture_error(
self.sentry_dsn, self.logger, exc_info=exc_info, extra=extra
)
def action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
# Generate a crash signature and capture the signature and notes
crash_data = convert_to_crash_data(raw_crash, processed_crash)
ret = self.generator.generate(crash_data)
processed_crash['signature'] = ret.signature
processor_meta['processor_notes'].extend(ret.notes)
# NOTE(willkg): this picks up proto_signature
processed_crash.update(ret.extra)
def test_empty_dicts(self):
generator = SignatureGenerator()
ret = generator.generate({}, {})
# NOTE(willkg): This is what the current pipeline yields. If any of those parts change, this
# might change, too. The point of this test is that we can pass in empty dicts and the
# SignatureGenerator and the rules in the default pipeline don't fall over.
expected = {
'notes': [
'CSignatureTool: No signature could be created because we do not know '
'which thread crashed'
],
'signature':
'EMPTY: no crashing thread identified'
}
assert ret == expected
def __init__(self, config):
super().__init__(config)
try:
sentry_dsn = self.config.sentry.dsn
except (AttributeError, KeyError):
sentry_dsn = None
self.sentry_dsn = sentry_dsn
self.generator = SignatureGenerator(error_handler=self._error_handler)
class SignatureGeneratorRule(Rule):
"""Generates a Socorro crash signature"""
def __init__(self, config):
super(SignatureGeneratorRule, self).__init__(config)
try:
sentry_dsn = self.config.sentry.dsn
except KeyError:
# DotDict raises a KeyError when things are missing
sentry_dsn = None
self.generator = SignatureGenerator(sentry_dsn=sentry_dsn)
def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
# Generate a crash signature and capture the signature and notes
ret = self.generator.generate(raw_crash, processed_crash)
processed_crash['signature'] = ret['signature']
processor_meta['processor_notes'].extend(ret['notes'])
return True
class SignatureGeneratorRule(Rule):
"""Generates a Socorro crash signature."""
def __init__(self):
super().__init__()
self.generator = SignatureGenerator(error_handler=self._error_handler)
def _error_handler(self, crash_data, exc_info, extra):
"""Captures errors from signature generation"""
extra["uuid"] = crash_data.get("uuid", None)
sentry_client.capture_error(self.logger, exc_info=exc_info, extra=extra)
def action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
# Generate a crash signature and capture the signature and notes
crash_data = convert_to_crash_data(raw_crash, processed_crash)
ret = self.generator.generate(crash_data)
processed_crash["signature"] = ret.signature
processor_meta["processor_notes"].extend(ret.notes)
# NOTE(willkg): this picks up proto_signature
processed_crash.update(ret.extra)
def main(argv=None):
"""Takes crash data via args and generates a Socorro signature
"""
# Fix the case where the user ran "python -m socorro.signature --help" so
# it prints a helpful prog.
if sys.argv and '__main__.py' in sys.argv[0]:
sys.argv[0] = 'python -m socorro.signature'
parser = argparse.ArgumentParser(description=DESCRIPTION, epilog=EPILOG)
parser.add_argument(
'-v', '--verbose', help='increase output verbosity', action='store_true'
)
parser.add_argument(
'--format', help='specify output format: csv, text (default)'
)
parser.add_argument(
'--different-only', dest='different', action='store_true',
help='limit output to just the signatures that changed',
)
parser.add_argument(
'crashids', metavar='crashid', nargs='*', help='crash id to generate signatures for'
)
if argv is None:
args = parser.parse_args()
else:
args = parser.parse_args(argv)
if args.format == 'csv':
outputter = CSVOutput
else:
outputter = TextOutput
if args.verbose:
logging_level = 'DEBUG'
else:
logging_level = 'INFO'
api_token = os.environ.get('SOCORRO_API_TOKEN', '')
setup_logging(logging_level)
generator = SignatureGenerator(debug=args.verbose)
crashids_iterable = args.crashids or sys.stdin
with outputter() as out:
for crash_id in crashids_iterable:
crash_id = crash_id.strip()
resp = fetch('/RawCrash/', crash_id, api_token)
if resp.status_code == 404:
out.warning('%s: does not exist.' % crash_id)
continue
if resp.status_code == 429:
out.warning('API rate limit reached. %s' % resp.content)
# FIXME(willkg): Maybe there's something better we could do here. Like maybe wait a
# few minutes.
return 1
if resp.status_code == 500:
out.warning('HTTP 500: %s' % resp.content)
continue
raw_crash = resp.json()
# If there's an error in the raw crash, then something is wrong--probably with the API
# token. So print that out and exit.
if 'error' in raw_crash:
out.warning('Error fetching raw crash: %s' % raw_crash['error'])
return 1
raw_crash_minimal = {
'JavaStackTrace': raw_crash.get('JavaStackTrace', None),
'OOMAllocationSize': raw_crash.get('OOMAllocationSize', None),
'AbortMessage': raw_crash.get('AbortMessage', None),
'AsyncShutdownTimeout': raw_crash.get('AsyncShutdownTimeout', None),
'ipc_channel_error': raw_crash.get('ipc_channel_error', None),
'additional_minidumps': raw_crash.get('additional_minidumps', None),
'IPCMessageName': raw_crash.get('IPCMessageName', None),
'MozCrashReason': raw_crash.get('MozCrashReason', None),
}
resp = fetch('/ProcessedCrash/', crash_id, api_token)
if resp.status_code == 404:
out.warning('%s: does not have processed crash.' % crash_id)
continue
if resp.status_code == 429:
out.warning('API rate limit reached. %s' % resp.content)
# FIXME(willkg): Maybe there's something better we could do here. Like maybe wait a
# few minutes.
return 1
if resp.status_code == 500:
out.warning('HTTP 500: %s' % resp.content)
continue
processed_crash = resp.json()
# If there's an error in the processed crash, then something is wrong--probably with the
# API token. So print that out and exit.
if 'error' in processed_crash:
out.warning('Error fetching processed crash: %s' % processed_crash['error'])
return 1
old_signature = processed_crash['signature']
processed_crash_minimal = {
'hang_type': processed_crash.get('hang_type', None),
'json_dump': {
'threads': glom(processed_crash, 'json_dump.threads', default=[]),
'system_info': {
'os': glom(processed_crash, 'json_dump.system_info.os', default=''),
},
'crash_info': {
'crashing_thread': glom(
processed_crash, 'json_dump.crash_info.crashing_thread', default=None
),
},
},
# NOTE(willkg): Classifications aren't available via the public API.
'classifications': {
'jit': {
'category': glom(processed_crash, 'classifications.jit.category', ''),
},
},
'mdsw_status_string': processed_crash.get('mdsw_status_string', None),
# This needs to be an empty string--the signature generator fills it in.
'signature': ''
}
# We want to generate fresh signatures, so we remove the "normalized" field from stack
# frames because this is essentially cached data from processing
for thread in processed_crash_minimal['json_dump'].get('threads', []):
for frame in thread.get('frames', []):
if 'normalized' in frame:
del frame['normalized']
ret = generator.generate(raw_crash_minimal, processed_crash_minimal)
if not args.different or old_signature != ret['signature']:
out.data(crash_id, old_signature, ret['signature'], ret['notes'])
def __init__(self):
super().__init__()
self.generator = SignatureGenerator(error_handler=self._error_handler)
def main(args):
"""Takes crash data via args and generates a Socorro signature
"""
parser = argparse.ArgumentParser(description=DESCRIPTION, epilog=EPILOG)
parser.add_argument('-v',
'--verbose',
help='increase output verbosity',
action='store_true')
parser.add_argument('--format',
help='specify output format: csv, text (default)')
parser.add_argument(
'--different-only',
dest='different',
action='store_true',
help='limit output to just the signatures that changed',
)
parser.add_argument('crashids',
metavar='crashid',
nargs='*',
help='crash id to generate signatures for')
args = parser.parse_args()
if args.format == 'csv':
outputter = CSVOutput
else:
outputter = TextOutput
if args.verbose:
logging_level = 'DEBUG'
else:
logging_level = 'INFO'
api_token = os.environ.get('SOCORRO_API_TOKEN', '')
setup_logging(logging_level)
generator = SignatureGenerator(debug=args.verbose)
crashids_iterable = args.crashids or sys.stdin
with outputter() as out:
for crash_id in crashids_iterable:
crash_id = crash_id.strip()
resp = fetch('/RawCrash/', crash_id, api_token)
if resp.status_code == 404:
out.warning('%s: does not exist.' % crash_id)
continue
if resp.status_code == 429:
out.warning('API rate limit reached. %s' % resp.content)
# FIXME(willkg): Maybe there's something better we could do here. Like maybe wait a
# few minutes.
return 1
if resp.status_code == 500:
out.warning('HTTP 500: %s' % resp.content)
continue
raw_crash = resp.json()
# If there's an error in the raw crash, then something is wrong--probably with the API
# token. So print that out and exit.
if 'error' in raw_crash:
out.warning('Error fetching raw crash: %s' %
raw_crash['error'])
return 1
raw_crash_minimal = {
'JavaStackTrace': raw_crash.get('JavaStackTrace', None),
'OOMAllocationSize': raw_crash.get('OOMAllocationSize', None),
'AbortMessage': raw_crash.get('AbortMessage', None),
'AsyncShutdownTimeout': raw_crash.get('AsyncShutdownTimeout',
None),
'ipc_channel_error': raw_crash.get('ipc_channel_error', None),
'additional_minidumps': raw_crash.get('additional_minidumps',
None),
'IPCMessageName': raw_crash.get('IPCMessageName', None),
'MozCrashReason': raw_crash.get('MozCrashReason', None),
}
resp = fetch('/ProcessedCrash/', crash_id, api_token)
if resp.status_code == 404:
out.warning('%s: does not have processed crash.' % crash_id)
continue
if resp.status_code == 429:
out.warning('API rate limit reached. %s' % resp.content)
# FIXME(willkg): Maybe there's something better we could do here. Like maybe wait a
# few minutes.
return 1
if resp.status_code == 500:
out.warning('HTTP 500: %s' % resp.content)
continue
processed_crash = resp.json()
# If there's an error in the processed crash, then something is wrong--probably with the
# API token. So print that out and exit.
if 'error' in processed_crash:
out.warning('Error fetching processed crash: %s' %
processed_crash['error'])
return 1
old_signature = processed_crash['signature']
processed_crash_minimal = {
'hang_type':
processed_crash.get('hang_type', None),
'json_dump': {
'threads':
tree_get(processed_crash, 'json_dump.threads', default=[]),
'system_info': {
'os':
tree_get(processed_crash,
'json_dump.system_info.os',
default=''),
},
'crash_info': {
'crashing_thread':
tree_get(processed_crash,
'json_dump.crash_info.crashing_thread',
default=None),
},
},
# NOTE(willkg): Classifications aren't available via the public API.
'classifications': {
'jit': {
'category':
tree_get(processed_crash,
'classifications.jit.category', ''),
},
},
'mdsw_status_string':
processed_crash.get('mdsw_status_string', None),
# This needs to be an empty string--the signature generator fills it in.
'signature':
''
}
# We want to generate fresh signatures, so we remove the "normalized" field from stack
# frames because this is essentially cached data from processing
for thread in processed_crash_minimal['json_dump'].get(
'threads', []):
for frame in thread.get('frames', []):
if 'normalized' in frame:
del frame['normalized']
ret = generator.generate(raw_crash_minimal,
processed_crash_minimal)
if not args.different or old_signature != ret['signature']:
out.data(crash_id, old_signature, ret['signature'],
ret['notes'])