class BixieProcessor(RequiredConfig):
"""this class is a processor algorthim for Bixie suitable for use in the
'processor_app' introducted in 2012."""
required_config = Namespace()
required_config.add_option(
'database_class',
doc="the class of the database",
default=ConnectionContext,
from_string_converter=class_converter
)
required_config.add_option(
'transaction_executor_class',
default=TransactionExecutor,
doc='a class that will manage transactions',
from_string_converter=class_converter
)
required_config.namespace('statistics')
required_config.statistics.add_option(
'stats_class',
default='socorro.lib.statistics.StatisticsForStatsd',
doc='name of a class that will gather statistics',
from_string_converter=class_converter
)
#--------------------------------------------------------------------------
def __init__(self, config, quit_check_callback=None):
super(BixieProcessor, self).__init__()
self.config = config
if quit_check_callback:
self.quit_check = quit_check_callback
else:
self.quit_check = lambda: False
self.database = self.config.database_class(config)
self.transaction = \
self.config.transaction_executor_class(
config,
self.database,
self.quit_check
)
self.rule_system = TransformRuleSystem()
self._load_transform_rules()
self._statistics = config.statistics.stats_class(
config.statistics,
self.config.processor_name
)
self._statistics.incr('restarts')
#--------------------------------------------------------------------------
def reject_raw_crash(self, crash_id, reason):
self._log_job_start(crash_id)
self.config.logger.warning('%s rejected: %s', crash_id, reason)
self._log_job_end(False, crash_id)
#--------------------------------------------------------------------------
def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dumps):
""" This function is run only by a worker thread.
Given a job, fetch a thread local database connection and the json
document. Use these to create the record in the 'reports' table,
then start the analysis of the dump file.
input parameters:
raw_crash - a nested dict of the form outlined at
https://gist.github.com/lonnen/dafb5fdf8611201572f1
raw_dumps - for future use if we choose to add binary
attachments to crashes
"""
raw_crash = DotDict(raw_crash)
self._statistics.incr('jobs')
processor_notes = []
processed_crash = self._create_minimal_processed_crash()
try:
self.quit_check()
crash_id = raw_crash['crash_id']
started_timestamp = self._log_job_start(crash_id)
processed_crash.processor.started_timestamp = started_timestamp
processed_crash.crash_id = raw_crash['crash_id']
self.rule_system.apply_all_rules(
raw_crash,
raw_dumps,
processed_crash,
self
)
processed_crash.success = True
except Exception, x:
self.config.logger.warning(
'Error while processing %s: %s',
raw_crash['crash_id'],
str(x),
exc_info=True
)
processed_crash.success = False
processor_notes.append('unrecoverable processor error')
self._statistics.incr('errors')
processed_crash.processor.notes = processor_notes
completed_timestamp = utc_now()
processed_crash.processor.completed_timestamp = completed_timestamp
self._log_job_end(
processed_crash.success,
crash_id
)
return processed_crash
class BixieProcessor(RequiredConfig):
"""this class is a processor algorthim for Bixie suitable for use in the
'processor_app' introducted in 2012."""
required_config = Namespace()
required_config.add_option('database_class',
doc="the class of the database",
default=ConnectionContext,
from_string_converter=class_converter)
required_config.add_option('transaction_executor_class',
default="socorro.database.transaction_executor."
"TransactionExecutorWithInfiniteBackoff",
doc='a class that will manage transactions',
from_string_converter=class_converter)
required_config.namespace('statistics')
required_config.statistics.add_option(
'stats_class',
default='socorro.lib.statistics.StatisticsForStatsd',
doc='name of a class that will gather statistics',
from_string_converter=class_converter)
#--------------------------------------------------------------------------
def __init__(self, config, quit_check_callback=None):
super(BixieProcessor, self).__init__()
self.config = config
if quit_check_callback:
self.quit_check = quit_check_callback
else:
self.quit_check = lambda: False
self.database = self.config.database_class(config)
self.transaction = \
self.config.transaction_executor_class(
config,
self.database,
self.quit_check
)
self.rule_system = TransformRuleSystem()
self._load_transform_rules()
self._statistics = config.statistics.stats_class(
config.statistics, self.config.processor_name)
self._statistics.incr('restarts')
#--------------------------------------------------------------------------
def reject_raw_crash(self, crash_id, reason):
self._log_job_start(crash_id)
self.config.logger.warning('%s rejected: %s', crash_id, reason)
self._log_job_end(False, crash_id)
#--------------------------------------------------------------------------
def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dumps):
""" This function is run only by a worker thread.
Given a job, fetch a thread local database connection and the json
document. Use these to create the record in the 'reports' table,
then start the analysis of the dump file.
input parameters:
raw_crash - a nested dict of the form outlined at
https://gist.github.com/lonnen/dafb5fdf8611201572f1
raw_dumps - for future use if we choose to add binary
attachments to crashes
"""
raw_crash = DotDict(raw_crash)
self._statistics.incr('jobs')
processor_notes = []
processed_crash = self._create_minimal_processed_crash()
try:
self.quit_check()
crash_id = raw_crash['crash_id']
started_timestamp = self._log_job_start(crash_id)
processed_crash.processor.started_timestamp = started_timestamp
processed_crash.crash_id = raw_crash['crash_id']
self.rule_system.apply_all_rules(raw_crash, raw_dumps,
processed_crash, self)
processed_crash.success = True
except Exception, x:
self.config.logger.warning('Error while processing %s: %s',
raw_crash['crash_id'],
str(x),
exc_info=True)
processed_crash.success = False
processor_notes.append('unrecoverable processor error')
self._statistics.incr('errors')
processed_crash.processor.notes = processor_notes
completed_timestamp = utc_now()
processed_crash.processor.completed_timestamp = completed_timestamp
self._log_job_end(processed_crash.success, crash_id)
return processed_crash
class LegacyCrashProcessor(RequiredConfig):
"""this class is a refactoring of the original processor algorithm into
a single class. This class is suitble for use in the 'processor_app'
introducted in 2012."""
required_config = Namespace()
required_config.add_option(
'database_class',
doc="the class of the database",
default=ConnectionContext,
from_string_converter=class_converter
)
required_config.add_option(
'transaction_executor_class',
default=TransactionExecutor,
doc='a class that will manage transactions',
from_string_converter=class_converter
)
required_config.add_option(
'stackwalk_command_line',
doc='the template for the command to invoke minidump_stackwalk',
default='$minidump_stackwalk_pathname -m $dumpfilePathname '
'$processor_symbols_pathname_list 2>/dev/null',
)
required_config.add_option(
'minidump_stackwalk_pathname',
doc='the full pathname of the extern program minidump_stackwalk '
'(quote path with embedded spaces)',
default='/data/socorro/stackwalk/bin/minidump_stackwalk',
)
required_config.add_option(
'symbol_cache_path',
doc='the path where the symbol cache is found (quote path with '
'embedded spaces)',
default='/mnt/socorro/symbols',
)
required_config.add_option(
'processor_symbols_pathname_list',
doc='comma or space separated list of symbol files for '
'minidump_stackwalk (quote paths with embedded spaces)',
default='/mnt/socorro/symbols/symbols_ffx,'
'/mnt/socorro/symbols/symbols_sea,'
'/mnt/socorro/symbols/symbols_tbrd,'
'/mnt/socorro/symbols/symbols_sbrd,'
'/mnt/socorro/symbols/symbols_os',
from_string_converter=create_symbol_path_str
)
required_config.add_option(
'crashing_thread_frame_threshold',
doc='the number of frames to keep in the raw dump for the '
'crashing thread',
default=100,
)
required_config.add_option(
'crashing_thread_tail_frame_threshold',
doc='the number of frames to keep in the raw dump at the tail of the '
'frame list',
default=10,
)
required_config.add_option(
'temporary_file_system_storage_path',
doc='a local filesystem path where processor can write dumps '
'temporarily for processing',
default='/home/socorro/temp',
)
required_config.namespace('c_signature')
required_config.c_signature.add_option(
'c_signature_tool_class',
doc='the class that can generate a C signature',
default='socorro.processor.signature_utilities.CSignatureTool',
from_string_converter=class_converter
)
required_config.namespace('java_signature')
required_config.java_signature.add_option(
'java_signature_tool_class',
doc='the class that can generate a Java signature',
default='socorro.processor.signature_utilities.JavaSignatureTool',
from_string_converter=class_converter
)
required_config.add_option(
'known_flash_identifiers',
doc='A subset of the known "debug identifiers" for flash versions, '
'associated to the version',
default={
'7224164B5918E29AF52365AF3EAF7A500': '10.1.51.66',
'C6CDEFCDB58EFE5C6ECEF0C463C979F80': '10.1.51.66',
'4EDBBD7016E8871A461CCABB7F1B16120': '10.1',
'D1AAAB5D417861E6A5B835B01D3039550': '10.0.45.2',
'EBD27FDBA9D9B3880550B2446902EC4A0': '10.0.45.2',
'266780DB53C4AAC830AFF69306C5C0300': '10.0.42.34',
'C4D637F2C8494896FBD4B3EF0319EBAC0': '10.0.42.34',
'B19EE2363941C9582E040B99BB5E237A0': '10.0.32.18',
'025105C956638D665850591768FB743D0': '10.0.32.18',
'986682965B43DFA62E0A0DFFD7B7417F0': '10.0.23',
'937DDCC422411E58EF6AD13710B0EF190': '10.0.23',
'860692A215F054B7B9474B410ABEB5300': '10.0.22.87',
'77CB5AC61C456B965D0B41361B3F6CEA0': '10.0.22.87',
'38AEB67F6A0B43C6A341D7936603E84A0': '10.0.12.36',
'776944FD51654CA2B59AB26A33D8F9B30': '10.0.12.36',
'974873A0A6AD482F8F17A7C55F0A33390': '9.0.262.0',
'B482D3DFD57C23B5754966F42D4CBCB60': '9.0.262.0',
'0B03252A5C303973E320CAA6127441F80': '9.0.260.0',
'AE71D92D2812430FA05238C52F7E20310': '9.0.246.0',
'6761F4FA49B5F55833D66CAC0BBF8CB80': '9.0.246.0',
'27CC04C9588E482A948FB5A87E22687B0': '9.0.159.0',
'1C8715E734B31A2EACE3B0CFC1CF21EB0': '9.0.159.0',
'F43004FFC4944F26AF228334F2CDA80B0': '9.0.151.0',
'890664D4EF567481ACFD2A21E9D2A2420': '9.0.151.0',
'8355DCF076564B6784C517FD0ECCB2F20': '9.0.124.0',
'51C00B72112812428EFA8F4A37F683A80': '9.0.124.0',
'9FA57B6DC7FF4CFE9A518442325E91CB0': '9.0.115.0',
'03D99C42D7475B46D77E64D4D5386D6D0': '9.0.115.0',
'0CFAF1611A3C4AA382D26424D609F00B0': '9.0.47.0',
'0F3262B5501A34B963E5DF3F0386C9910': '9.0.47.0',
'C5B5651B46B7612E118339D19A6E66360': '9.0.45.0',
'BF6B3B51ACB255B38FCD8AA5AEB9F1030': '9.0.28.0',
'83CF4DC03621B778E931FC713889E8F10': '9.0.16.0',
}
)
required_config.add_option(
'collect_addon',
doc='boolean indictating if information about add-ons should be '
'collected',
default=True,
)
required_config.add_option(
'collect_crash_process',
doc='boolean indictating if information about process type should be '
'collected',
default=True,
)
#--------------------------------------------------------------------------
def __init__(self, config, quit_check_callback=None):
super(LegacyCrashProcessor, self).__init__()
self.config = config
if quit_check_callback:
self.quit_check = quit_check_callback
else:
self.quit_check = lambda: False
self.database = self.config.database_class(config)
self.transaction = \
self.config.transaction_executor_class(
config,
self.database,
quit_check_callback
)
self.raw_crash_transform_rule_system = TransformRuleSystem()
self._load_transform_rules()
# *** originally from the ExternalProcessor class
#preprocess the breakpad_stackwalk command line
strip_parens_re = re.compile(r'\$(\()(\w+)(\))')
convert_to_python_substitution_format_re = re.compile(r'\$(\w+)')
# Canonical form of $(param) is $param. Convert any that are needed
tmp = strip_parens_re.sub(
r'$\2',
config.stackwalk_command_line
)
# Convert canonical $dumpfilePathname to DUMPFILEPATHNAME
tmp = tmp.replace('$dumpfilePathname', 'DUMPFILEPATHNAME')
# finally, convert any remaining $param to pythonic %(param)s
tmp = convert_to_python_substitution_format_re.sub(r'%(\1)s', tmp)
self.command_line = tmp % config
# *** end from ExternalProcessor
self.c_signature_tool = config.c_signature.c_signature_tool_class(
config.c_signature
)
self.java_signature_tool = \
config.java_signature.java_signature_tool_class(
config.java_signature
)
#--------------------------------------------------------------------------
def reject_raw_crash(self, crash_id, reason):
self._log_job_start(crash_id)
self.config.logger.warning('%s rejected: %s', crash_id, reason)
self._log_job_end(utc_now(), False, crash_id)
#--------------------------------------------------------------------------
def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dump):
""" This function is run only by a worker thread.
Given a job, fetch a thread local database connection and the json
document. Use these to create the record in the 'reports' table,
then start the analysis of the dump file.
input parameters:
"""
try:
self.quit_check()
crash_id = raw_crash.uuid
processor_notes = []
processed_crash = DotDict()
processed_crash.uuid = raw_crash.uuid
processed_crash.success = False
started_timestamp = self._log_job_start(crash_id)
#self.config.logger.debug('about to apply rules')
self.raw_crash_transform_rule_system.apply_all_rules(raw_crash,
self)
#self.config.logger.debug('done applying transform rules')
try:
submitted_timestamp = datetimeFromISOdateString(
raw_crash.submitted_timestamp
)
except KeyError:
submitted_timestamp = dateFromOoid(crash_id)
# formerly the call to 'insertReportIntoDatabase'
processed_crash_update = self._create_basic_processed_crash(
crash_id,
raw_crash,
submitted_timestamp,
started_timestamp,
processor_notes
)
processed_crash.update(processed_crash_update)
temp_dump_pathname = self._get_temp_dump_pathname(
crash_id,
raw_dump
)
try:
#logger.debug('about to doBreakpadStackDumpAnalysis')
processed_crash_update_dict = \
self._do_breakpad_stack_dump_analysis(
crash_id,
temp_dump_pathname,
processed_crash.hang_type,
processed_crash.java_stack_trace,
submitted_timestamp,
processor_notes
)
processed_crash.update(processed_crash_update_dict)
finally:
self._cleanup_temp_file(temp_dump_pathname)
processed_crash.topmost_filenames = "|".join(
processed_crash.get('topmost_filenames', [])
)
try:
processed_crash.Winsock_LSP = raw_crash.Winsock_LSP
except KeyError:
pass # if it's not in the original raw_crash,
# it does get into the jsonz
#except (KeyboardInterrupt, SystemExit):
#self.config.logger.info("quit request detected")
#raise
except Exception, x:
self.config.logger.warning(
'Error while processing %s: %s',
crash_id,
str(x),
exc_info=True
)
processor_notes.append(str(x))
processor_notes = '; '.join(processor_notes)
processed_crash.processor_notes = processor_notes
completed_datetime = utc_now()
processed_crash.completeddatetime = completed_datetime
self._log_job_end(
completed_datetime,
processed_crash.success,
crash_id
)
return processed_crash
class LegacyCrashProcessor(RequiredConfig):
"""this class is a refactoring of the original processor algorithm into
a single class. This class is suitble for use in the 'processor_app'
introducted in 2012."""
required_config = Namespace()
required_config.add_option('database_class',
doc="the class of the database",
default=ConnectionContext,
from_string_converter=class_converter)
required_config.add_option('transaction_executor_class',
default=TransactionExecutor,
doc='a class that will manage transactions',
from_string_converter=class_converter)
required_config.add_option(
'stackwalk_command_line',
doc='the template for the command to invoke minidump_stackwalk',
default='$minidump_stackwalk_pathname -m $dumpfilePathname '
'$processor_symbols_pathname_list 2>/dev/null',
)
required_config.add_option(
'minidump_stackwalk_pathname',
doc='the full pathname of the extern program minidump_stackwalk '
'(quote path with embedded spaces)',
default='/data/socorro/stackwalk/bin/minidump_stackwalk',
)
required_config.add_option(
'symbol_cache_path',
doc='the path where the symbol cache is found (quote path with '
'embedded spaces)',
default='/mnt/socorro/symbols',
)
required_config.add_option(
'processor_symbols_pathname_list',
doc='comma or space separated list of symbol files for '
'minidump_stackwalk (quote paths with embedded spaces)',
default='/mnt/socorro/symbols/symbols_ffx,'
'/mnt/socorro/symbols/symbols_sea,'
'/mnt/socorro/symbols/symbols_tbrd,'
'/mnt/socorro/symbols/symbols_sbrd,'
'/mnt/socorro/symbols/symbols_os',
from_string_converter=create_symbol_path_str)
required_config.add_option(
'crashing_thread_frame_threshold',
doc='the number of frames to keep in the raw dump for the '
'crashing thread',
default=100,
)
required_config.add_option(
'crashing_thread_tail_frame_threshold',
doc='the number of frames to keep in the raw dump at the tail of the '
'frame list',
default=10,
)
required_config.add_option(
'temporary_file_system_storage_path',
doc='a local filesystem path where processor can write dumps '
'temporarily for processing',
default='/home/socorro/temp',
)
required_config.namespace('c_signature')
required_config.c_signature.add_option(
'c_signature_tool_class',
doc='the class that can generate a C signature',
default='socorro.processor.signature_utilities.CSignatureTool',
from_string_converter=class_converter)
required_config.namespace('java_signature')
required_config.java_signature.add_option(
'java_signature_tool_class',
doc='the class that can generate a Java signature',
default='socorro.processor.signature_utilities.JavaSignatureTool',
from_string_converter=class_converter)
required_config.add_option(
'known_flash_identifiers',
doc='A subset of the known "debug identifiers" for flash versions, '
'associated to the version',
default={
'7224164B5918E29AF52365AF3EAF7A500': '10.1.51.66',
'C6CDEFCDB58EFE5C6ECEF0C463C979F80': '10.1.51.66',
'4EDBBD7016E8871A461CCABB7F1B16120': '10.1',
'D1AAAB5D417861E6A5B835B01D3039550': '10.0.45.2',
'EBD27FDBA9D9B3880550B2446902EC4A0': '10.0.45.2',
'266780DB53C4AAC830AFF69306C5C0300': '10.0.42.34',
'C4D637F2C8494896FBD4B3EF0319EBAC0': '10.0.42.34',
'B19EE2363941C9582E040B99BB5E237A0': '10.0.32.18',
'025105C956638D665850591768FB743D0': '10.0.32.18',
'986682965B43DFA62E0A0DFFD7B7417F0': '10.0.23',
'937DDCC422411E58EF6AD13710B0EF190': '10.0.23',
'860692A215F054B7B9474B410ABEB5300': '10.0.22.87',
'77CB5AC61C456B965D0B41361B3F6CEA0': '10.0.22.87',
'38AEB67F6A0B43C6A341D7936603E84A0': '10.0.12.36',
'776944FD51654CA2B59AB26A33D8F9B30': '10.0.12.36',
'974873A0A6AD482F8F17A7C55F0A33390': '9.0.262.0',
'B482D3DFD57C23B5754966F42D4CBCB60': '9.0.262.0',
'0B03252A5C303973E320CAA6127441F80': '9.0.260.0',
'AE71D92D2812430FA05238C52F7E20310': '9.0.246.0',
'6761F4FA49B5F55833D66CAC0BBF8CB80': '9.0.246.0',
'27CC04C9588E482A948FB5A87E22687B0': '9.0.159.0',
'1C8715E734B31A2EACE3B0CFC1CF21EB0': '9.0.159.0',
'F43004FFC4944F26AF228334F2CDA80B0': '9.0.151.0',
'890664D4EF567481ACFD2A21E9D2A2420': '9.0.151.0',
'8355DCF076564B6784C517FD0ECCB2F20': '9.0.124.0',
'51C00B72112812428EFA8F4A37F683A80': '9.0.124.0',
'9FA57B6DC7FF4CFE9A518442325E91CB0': '9.0.115.0',
'03D99C42D7475B46D77E64D4D5386D6D0': '9.0.115.0',
'0CFAF1611A3C4AA382D26424D609F00B0': '9.0.47.0',
'0F3262B5501A34B963E5DF3F0386C9910': '9.0.47.0',
'C5B5651B46B7612E118339D19A6E66360': '9.0.45.0',
'BF6B3B51ACB255B38FCD8AA5AEB9F1030': '9.0.28.0',
'83CF4DC03621B778E931FC713889E8F10': '9.0.16.0',
})
required_config.add_option(
'collect_addon',
doc='boolean indictating if information about add-ons should be '
'collected',
default=True,
)
required_config.add_option(
'collect_crash_process',
doc='boolean indictating if information about process type should be '
'collected',
default=True,
)
#--------------------------------------------------------------------------
def __init__(self, config, quit_check_callback=None):
super(LegacyCrashProcessor, self).__init__()
self.config = config
if quit_check_callback:
self.quit_check = quit_check_callback
else:
self.quit_check = lambda: False
self.database = self.config.database_class(config)
self.transaction = \
self.config.transaction_executor_class(
config,
self.database,
quit_check_callback
)
self.raw_crash_transform_rule_system = TransformRuleSystem()
self._load_transform_rules()
# *** originally from the ExternalProcessor class
#preprocess the breakpad_stackwalk command line
strip_parens_re = re.compile(r'\$(\()(\w+)(\))')
convert_to_python_substitution_format_re = re.compile(r'\$(\w+)')
# Canonical form of $(param) is $param. Convert any that are needed
tmp = strip_parens_re.sub(r'$\2', config.stackwalk_command_line)
# Convert canonical $dumpfilePathname to DUMPFILEPATHNAME
tmp = tmp.replace('$dumpfilePathname', 'DUMPFILEPATHNAME')
# finally, convert any remaining $param to pythonic %(param)s
tmp = convert_to_python_substitution_format_re.sub(r'%(\1)s', tmp)
self.command_line = tmp % config
# *** end from ExternalProcessor
self.c_signature_tool = config.c_signature.c_signature_tool_class(
config.c_signature)
self.java_signature_tool = \
config.java_signature.java_signature_tool_class(
config.java_signature
)
#--------------------------------------------------------------------------
def reject_raw_crash(self, crash_id, reason):
self._log_job_start(crash_id)
self.config.logger.warning('%s rejected: %s', crash_id, reason)
self._log_job_end(utc_now(), False, crash_id)
#--------------------------------------------------------------------------
def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dump):
""" This function is run only by a worker thread.
Given a job, fetch a thread local database connection and the json
document. Use these to create the record in the 'reports' table,
then start the analysis of the dump file.
input parameters:
"""
try:
self.quit_check()
crash_id = raw_crash.uuid
processor_notes = []
processed_crash = DotDict()
processed_crash.uuid = raw_crash.uuid
processed_crash.success = False
started_timestamp = self._log_job_start(crash_id)
#self.config.logger.debug('about to apply rules')
self.raw_crash_transform_rule_system.apply_all_rules(
raw_crash, self)
#self.config.logger.debug('done applying transform rules')
try:
submitted_timestamp = datetimeFromISOdateString(
raw_crash.submitted_timestamp)
except KeyError:
submitted_timestamp = dateFromOoid(crash_id)
# formerly the call to 'insertReportIntoDatabase'
processed_crash_update = self._create_basic_processed_crash(
crash_id, raw_crash, submitted_timestamp, started_timestamp,
processor_notes)
processed_crash.update(processed_crash_update)
temp_dump_pathname = self._get_temp_dump_pathname(
crash_id, raw_dump)
try:
#logger.debug('about to doBreakpadStackDumpAnalysis')
processed_crash_update_dict = \
self._do_breakpad_stack_dump_analysis(
crash_id,
temp_dump_pathname,
processed_crash.hang_type,
processed_crash.java_stack_trace,
submitted_timestamp,
processor_notes
)
processed_crash.update(processed_crash_update_dict)
finally:
self._cleanup_temp_file(temp_dump_pathname)
processed_crash.topmost_filenames = "|".join(
processed_crash.get('topmost_filenames', []))
try:
processed_crash.Winsock_LSP = raw_crash.Winsock_LSP
except KeyError:
pass # if it's not in the original raw_crash,
# it does get into the jsonz
#except (KeyboardInterrupt, SystemExit):
#self.config.logger.info("quit request detected")
#raise
except Exception, x:
self.config.logger.warning('Error while processing %s: %s',
crash_id,
str(x),
exc_info=True)
processor_notes.append(str(x))
processor_notes = '; '.join(processor_notes)
processed_crash.processor_notes = processor_notes
completed_datetime = utc_now()
processed_crash.completeddatetime = completed_datetime
self._log_job_end(completed_datetime, processed_crash.success,
crash_id)
return processed_crash
