def __init__(self, config, quit_check_callback=None): super(LegacyCrashProcessor, self).__init__() self.config = config if quit_check_callback: self.quit_check = quit_check_callback else: self.quit_check = lambda: False self.database = self.config.database_class(config) self.transaction = \ self.config.transaction_executor_class( config, self.database, quit_check_callback ) self.raw_crash_transform_rule_system = TransformRuleSystem() self._load_transform_rules() # *** originally from the ExternalProcessor class #preprocess the breakpad_stackwalk command line strip_parens_re = re.compile(r'\$(\()(\w+)(\))') convert_to_python_substitution_format_re = re.compile(r'\$(\w+)') # Canonical form of $(param) is $param. Convert any that are needed tmp = strip_parens_re.sub(r'$\2', config.stackwalk_command_line) # Convert canonical $dumpfilePathname to DUMPFILEPATHNAME tmp = tmp.replace('$dumpfilePathname', 'DUMPFILEPATHNAME') # finally, convert any remaining $param to pythonic %(param)s tmp = convert_to_python_substitution_format_re.sub(r'%(\1)s', tmp) self.command_line = tmp % config # *** end from ExternalProcessor self.c_signature_tool = config.c_signature.c_signature_tool_class( config.c_signature) self.java_signature_tool = \ config.java_signature.java_signature_tool_class( config.java_signature )
def __init__(self, config, quit_check_callback=None): super(BixieProcessor, self).__init__() self.config = config if quit_check_callback: self.quit_check = quit_check_callback else: self.quit_check = lambda: False self.database = self.config.database_class(config) self.transaction = \ self.config.transaction_executor_class( config, self.database, self.quit_check ) self.rule_system = TransformRuleSystem() self._load_transform_rules() self._statistics = config.statistics.stats_class( config.statistics, self.config.processor_name) self._statistics.incr('restarts')
def __init__(self, config, quit_check_callback=None): super(LegacyCrashProcessor, self).__init__() self.config = config if quit_check_callback: self.quit_check = quit_check_callback else: self.quit_check = lambda: False self.database = self.config.database_class(config) self.transaction = \ self.config.transaction_executor_class( config, self.database, quit_check_callback ) self.raw_crash_transform_rule_system = TransformRuleSystem() self._load_transform_rules() # *** originally from the ExternalProcessor class #preprocess the breakpad_stackwalk command line strip_parens_re = re.compile(r'\$(\()(\w+)(\))') convert_to_python_substitution_format_re = re.compile(r'\$(\w+)') # Canonical form of $(param) is $param. Convert any that are needed tmp = strip_parens_re.sub( r'$\2', config.stackwalk_command_line ) # Convert canonical $dumpfilePathname to DUMPFILEPATHNAME tmp = tmp.replace('$dumpfilePathname', 'DUMPFILEPATHNAME') # finally, convert any remaining $param to pythonic %(param)s tmp = convert_to_python_substitution_format_re.sub(r'%(\1)s', tmp) self.command_line = tmp % config # *** end from ExternalProcessor self.c_signature_tool = config.c_signature.c_signature_tool_class( config.c_signature ) self.java_signature_tool = \ config.java_signature.java_signature_tool_class( config.java_signature )
def __init__(self, config, quit_check_callback=None): super(BixieProcessor, self).__init__() self.config = config if quit_check_callback: self.quit_check = quit_check_callback else: self.quit_check = lambda: False self.database = self.config.database_class(config) self.transaction = \ self.config.transaction_executor_class( config, self.database, self.quit_check ) self.rule_system = TransformRuleSystem() self._load_transform_rules() self._statistics = config.statistics.stats_class( config.statistics, self.config.processor_name ) self._statistics.incr('restarts')
def run(self, run_datetime): logger = self.config.logger if self.config.test_mode: logger.warning('You are running Automatic Emails cron app ' 'in test mode') delay = datetime.timedelta(days=self.config.delay_between_emails) params = { 'start_date': run_datetime - datetime.timedelta(hours=1), 'end_date': run_datetime, 'delayed_date': run_datetime - delay, 'products': tuple(self.config.restrict_products) } # Find the indexes to use to optimize the elasticsearch query. indexes = self.generate_list_of_indexes( params['start_date'], params['end_date'], self.config.elasticsearch.elasticsearch_index ) # Create and configure the search object. connection = SuperS().es( urls=self.config.elasticsearch.elasticsearch_urls, timeout=self.config.elasticsearch.elasticsearch_timeout, ) search = ( connection.indexes(*indexes).doctypes( self.config.elasticsearch.elasticsearch_doctype ).order_by('processed_crash.email') ) # Create filters. args_and = { 'processed_crash.date_processed__lt': params['end_date'], 'processed_crash.date_processed__gt': params['start_date'], 'processed_crash.product': [x.lower() for x in params['products']], } args_not = { 'processed_crash.email__missing': None, } filters = elasticutils.F(**args_and) filters &= ~elasticutils.F(**args_not) search = search.filter(filters) count = search.count() # Total number of results. search = search[:count] # Get the recently sent emails emails = self.get_list_of_emails(params, connection) validation_rules = TransformRuleSystem() validation_rules.load_rules(( (verify_email, (), {}, sanitize_email, (), {}), (verify_email, (), {}, False, (), {}), ( verify_email_last_sending, (), {'emails_list': emails}, True, (), {} ), )) template_rules = TransformRuleSystem() template_rules.load_rules(( ( verify_support_classification, ('bitguard',), {}, set_email_template, ('socorro_bitguard_en',), {} ), # If no other rule passed, fall back to the default template. ( True, (), {}, set_email_template, (self.config.email_template,), {} ), )) for hit in search.values_dict( 'processed_crash.email', 'processed_crash.classifications.support.classification', ): res = validation_rules.apply_until_predicate_fails(hit) if res is None: # All predicates succeeded! # Now apply all template rules to find which email template # to use. template_rules.apply_until_action_succeeds(hit) if not hit['email_template']: # Bug 965610 - If the email template is empty, do not send # an email. Setting the default email template to '' means # no generic email will be sent anymore. continue email = hit['processed_crash.email'] self.send_email(hit) self.update_user(email, run_datetime, connection.get_es()) emails[email] = run_datetime # logger.info('Automatic Email sent to %s', email) # Make sure the next run will have updated data, to avoid sending an # email several times. connection.get_es().refresh()
def run(self, run_datetime): logger = self.config.logger if self.config.test_mode: logger.warning('You are running Automatic Emails cron app ' 'in test mode') delay = datetime.timedelta(days=self.config.delay_between_emails) params = { 'start_date': run_datetime - datetime.timedelta(hours=1), 'end_date': run_datetime, 'delayed_date': run_datetime - delay, 'products': tuple(self.config.restrict_products) } # Find the indexes to use to optimize the elasticsearch query. indexes = self.generate_list_of_indexes( params['start_date'], params['end_date'], self.config.elasticsearch.elasticsearch_index ) # Create and configure the search object. connection = SuperS().es( urls=self.config.elasticsearch.elasticsearch_urls, timeout=self.config.elasticsearch.elasticsearch_timeout, ) search = (connection.indexes(*indexes) .doctypes( self.config.elasticsearch.elasticsearch_doctype ) .order_by('processed_crash.email')) # Create filters. args_and = { 'processed_crash.date_processed__lt': params['end_date'], 'processed_crash.date_processed__gt': params['start_date'], 'processed_crash.product': [x.lower() for x in params['products']], } args_not = { 'processed_crash.email__missing': None, } filters = elasticutils.F(**args_and) filters &= ~elasticutils.F(**args_not) search = search.filter(filters) count = search.count() # Total number of results. search = search[:count] # Get the recently sent emails emails = self.get_list_of_emails(params, connection) validation_rules = TransformRuleSystem() validation_rules.load_rules(( (verify_email, (), {}, sanitize_email, (), {}), (verify_email, (), {}, False, (), {}), ( verify_email_last_sending, (), {'emails_list': emails}, True, (), {} ), )) template_rules = TransformRuleSystem() template_rules.load_rules(( ( verify_support_classification, ('bitguard',), {}, set_email_template, ('socorro_bitguard_en',), {} ), # If no other rule passed, fall back to the default template. ( True, (), {}, set_email_template, (self.config.email_template,), {} ), )) for hit in search.values_dict( 'processed_crash.email', 'processed_crash.classifications.support.classification', ): res = validation_rules.apply_until_predicate_fails(hit) if res is None: # All predicates succeeded! # Now apply all template rules to find which email template # to use. template_rules.apply_until_action_succeeds(hit) email = hit['processed_crash.email'] self.send_email(hit) self.update_user(email, run_datetime, connection.get_es()) emails[email] = run_datetime # logger.info('Automatic Email sent to %s', email) # Make sure the next run will have updated data, to avoid sending an # email several times. connection.get_es().refresh()
class BixieProcessor(RequiredConfig): """this class is a processor algorthim for Bixie suitable for use in the 'processor_app' introducted in 2012.""" required_config = Namespace() required_config.add_option( 'database_class', doc="the class of the database", default=ConnectionContext, from_string_converter=class_converter ) required_config.add_option( 'transaction_executor_class', default=TransactionExecutor, doc='a class that will manage transactions', from_string_converter=class_converter ) required_config.namespace('statistics') required_config.statistics.add_option( 'stats_class', default='socorro.lib.statistics.StatisticsForStatsd', doc='name of a class that will gather statistics', from_string_converter=class_converter ) #-------------------------------------------------------------------------- def __init__(self, config, quit_check_callback=None): super(BixieProcessor, self).__init__() self.config = config if quit_check_callback: self.quit_check = quit_check_callback else: self.quit_check = lambda: False self.database = self.config.database_class(config) self.transaction = \ self.config.transaction_executor_class( config, self.database, self.quit_check ) self.rule_system = TransformRuleSystem() self._load_transform_rules() self._statistics = config.statistics.stats_class( config.statistics, self.config.processor_name ) self._statistics.incr('restarts') #-------------------------------------------------------------------------- def reject_raw_crash(self, crash_id, reason): self._log_job_start(crash_id) self.config.logger.warning('%s rejected: %s', crash_id, reason) self._log_job_end(False, crash_id) #-------------------------------------------------------------------------- def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dumps): """ This function is run only by a worker thread. Given a job, fetch a thread local database connection and the json document. Use these to create the record in the 'reports' table, then start the analysis of the dump file. input parameters: raw_crash - a nested dict of the form outlined at https://gist.github.com/lonnen/dafb5fdf8611201572f1 raw_dumps - for future use if we choose to add binary attachments to crashes """ raw_crash = DotDict(raw_crash) self._statistics.incr('jobs') processor_notes = [] processed_crash = self._create_minimal_processed_crash() try: self.quit_check() crash_id = raw_crash['crash_id'] started_timestamp = self._log_job_start(crash_id) processed_crash.processor.started_timestamp = started_timestamp processed_crash.crash_id = raw_crash['crash_id'] self.rule_system.apply_all_rules( raw_crash, raw_dumps, processed_crash, self ) processed_crash.success = True except Exception, x: self.config.logger.warning( 'Error while processing %s: %s', raw_crash['crash_id'], str(x), exc_info=True ) processed_crash.success = False processor_notes.append('unrecoverable processor error') self._statistics.incr('errors') processed_crash.processor.notes = processor_notes completed_timestamp = utc_now() processed_crash.processor.completed_timestamp = completed_timestamp self._log_job_end( processed_crash.success, crash_id ) return processed_crash
class LegacyCrashProcessor(RequiredConfig): """this class is a refactoring of the original processor algorithm into a single class. This class is suitble for use in the 'processor_app' introducted in 2012.""" required_config = Namespace() required_config.add_option( 'database_class', doc="the class of the database", default=ConnectionContext, from_string_converter=class_converter ) required_config.add_option( 'transaction_executor_class', default=TransactionExecutor, doc='a class that will manage transactions', from_string_converter=class_converter ) required_config.add_option( 'stackwalk_command_line', doc='the template for the command to invoke minidump_stackwalk', default='$minidump_stackwalk_pathname -m $dumpfilePathname ' '$processor_symbols_pathname_list 2>/dev/null', ) required_config.add_option( 'minidump_stackwalk_pathname', doc='the full pathname of the extern program minidump_stackwalk ' '(quote path with embedded spaces)', default='/data/socorro/stackwalk/bin/minidump_stackwalk', ) required_config.add_option( 'symbol_cache_path', doc='the path where the symbol cache is found (quote path with ' 'embedded spaces)', default='/mnt/socorro/symbols', ) required_config.add_option( 'processor_symbols_pathname_list', doc='comma or space separated list of symbol files for ' 'minidump_stackwalk (quote paths with embedded spaces)', default='/mnt/socorro/symbols/symbols_ffx,' '/mnt/socorro/symbols/symbols_sea,' '/mnt/socorro/symbols/symbols_tbrd,' '/mnt/socorro/symbols/symbols_sbrd,' '/mnt/socorro/symbols/symbols_os', from_string_converter=create_symbol_path_str ) required_config.add_option( 'crashing_thread_frame_threshold', doc='the number of frames to keep in the raw dump for the ' 'crashing thread', default=100, ) required_config.add_option( 'crashing_thread_tail_frame_threshold', doc='the number of frames to keep in the raw dump at the tail of the ' 'frame list', default=10, ) required_config.add_option( 'temporary_file_system_storage_path', doc='a local filesystem path where processor can write dumps ' 'temporarily for processing', default='/home/socorro/temp', ) required_config.namespace('c_signature') required_config.c_signature.add_option( 'c_signature_tool_class', doc='the class that can generate a C signature', default='socorro.processor.signature_utilities.CSignatureTool', from_string_converter=class_converter ) required_config.namespace('java_signature') required_config.java_signature.add_option( 'java_signature_tool_class', doc='the class that can generate a Java signature', default='socorro.processor.signature_utilities.JavaSignatureTool', from_string_converter=class_converter ) required_config.add_option( 'known_flash_identifiers', doc='A subset of the known "debug identifiers" for flash versions, ' 'associated to the version', default={ '7224164B5918E29AF52365AF3EAF7A500': '10.1.51.66', 'C6CDEFCDB58EFE5C6ECEF0C463C979F80': '10.1.51.66', '4EDBBD7016E8871A461CCABB7F1B16120': '10.1', 'D1AAAB5D417861E6A5B835B01D3039550': '10.0.45.2', 'EBD27FDBA9D9B3880550B2446902EC4A0': '10.0.45.2', '266780DB53C4AAC830AFF69306C5C0300': '10.0.42.34', 'C4D637F2C8494896FBD4B3EF0319EBAC0': '10.0.42.34', 'B19EE2363941C9582E040B99BB5E237A0': '10.0.32.18', '025105C956638D665850591768FB743D0': '10.0.32.18', '986682965B43DFA62E0A0DFFD7B7417F0': '10.0.23', '937DDCC422411E58EF6AD13710B0EF190': '10.0.23', '860692A215F054B7B9474B410ABEB5300': '10.0.22.87', '77CB5AC61C456B965D0B41361B3F6CEA0': '10.0.22.87', '38AEB67F6A0B43C6A341D7936603E84A0': '10.0.12.36', '776944FD51654CA2B59AB26A33D8F9B30': '10.0.12.36', '974873A0A6AD482F8F17A7C55F0A33390': '9.0.262.0', 'B482D3DFD57C23B5754966F42D4CBCB60': '9.0.262.0', '0B03252A5C303973E320CAA6127441F80': '9.0.260.0', 'AE71D92D2812430FA05238C52F7E20310': '9.0.246.0', '6761F4FA49B5F55833D66CAC0BBF8CB80': '9.0.246.0', '27CC04C9588E482A948FB5A87E22687B0': '9.0.159.0', '1C8715E734B31A2EACE3B0CFC1CF21EB0': '9.0.159.0', 'F43004FFC4944F26AF228334F2CDA80B0': '9.0.151.0', '890664D4EF567481ACFD2A21E9D2A2420': '9.0.151.0', '8355DCF076564B6784C517FD0ECCB2F20': '9.0.124.0', '51C00B72112812428EFA8F4A37F683A80': '9.0.124.0', '9FA57B6DC7FF4CFE9A518442325E91CB0': '9.0.115.0', '03D99C42D7475B46D77E64D4D5386D6D0': '9.0.115.0', '0CFAF1611A3C4AA382D26424D609F00B0': '9.0.47.0', '0F3262B5501A34B963E5DF3F0386C9910': '9.0.47.0', 'C5B5651B46B7612E118339D19A6E66360': '9.0.45.0', 'BF6B3B51ACB255B38FCD8AA5AEB9F1030': '9.0.28.0', '83CF4DC03621B778E931FC713889E8F10': '9.0.16.0', } ) required_config.add_option( 'collect_addon', doc='boolean indictating if information about add-ons should be ' 'collected', default=True, ) required_config.add_option( 'collect_crash_process', doc='boolean indictating if information about process type should be ' 'collected', default=True, ) #-------------------------------------------------------------------------- def __init__(self, config, quit_check_callback=None): super(LegacyCrashProcessor, self).__init__() self.config = config if quit_check_callback: self.quit_check = quit_check_callback else: self.quit_check = lambda: False self.database = self.config.database_class(config) self.transaction = \ self.config.transaction_executor_class( config, self.database, quit_check_callback ) self.raw_crash_transform_rule_system = TransformRuleSystem() self._load_transform_rules() # *** originally from the ExternalProcessor class #preprocess the breakpad_stackwalk command line strip_parens_re = re.compile(r'\$(\()(\w+)(\))') convert_to_python_substitution_format_re = re.compile(r'\$(\w+)') # Canonical form of $(param) is $param. Convert any that are needed tmp = strip_parens_re.sub( r'$\2', config.stackwalk_command_line ) # Convert canonical $dumpfilePathname to DUMPFILEPATHNAME tmp = tmp.replace('$dumpfilePathname', 'DUMPFILEPATHNAME') # finally, convert any remaining $param to pythonic %(param)s tmp = convert_to_python_substitution_format_re.sub(r'%(\1)s', tmp) self.command_line = tmp % config # *** end from ExternalProcessor self.c_signature_tool = config.c_signature.c_signature_tool_class( config.c_signature ) self.java_signature_tool = \ config.java_signature.java_signature_tool_class( config.java_signature ) #-------------------------------------------------------------------------- def reject_raw_crash(self, crash_id, reason): self._log_job_start(crash_id) self.config.logger.warning('%s rejected: %s', crash_id, reason) self._log_job_end(utc_now(), False, crash_id) #-------------------------------------------------------------------------- def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dump): """ This function is run only by a worker thread. Given a job, fetch a thread local database connection and the json document. Use these to create the record in the 'reports' table, then start the analysis of the dump file. input parameters: """ try: self.quit_check() crash_id = raw_crash.uuid processor_notes = [] processed_crash = DotDict() processed_crash.uuid = raw_crash.uuid processed_crash.success = False started_timestamp = self._log_job_start(crash_id) #self.config.logger.debug('about to apply rules') self.raw_crash_transform_rule_system.apply_all_rules(raw_crash, self) #self.config.logger.debug('done applying transform rules') try: submitted_timestamp = datetimeFromISOdateString( raw_crash.submitted_timestamp ) except KeyError: submitted_timestamp = dateFromOoid(crash_id) # formerly the call to 'insertReportIntoDatabase' processed_crash_update = self._create_basic_processed_crash( crash_id, raw_crash, submitted_timestamp, started_timestamp, processor_notes ) processed_crash.update(processed_crash_update) temp_dump_pathname = self._get_temp_dump_pathname( crash_id, raw_dump ) try: #logger.debug('about to doBreakpadStackDumpAnalysis') processed_crash_update_dict = \ self._do_breakpad_stack_dump_analysis( crash_id, temp_dump_pathname, processed_crash.hang_type, processed_crash.java_stack_trace, submitted_timestamp, processor_notes ) processed_crash.update(processed_crash_update_dict) finally: self._cleanup_temp_file(temp_dump_pathname) processed_crash.topmost_filenames = "|".join( processed_crash.get('topmost_filenames', []) ) try: processed_crash.Winsock_LSP = raw_crash.Winsock_LSP except KeyError: pass # if it's not in the original raw_crash, # it does get into the jsonz #except (KeyboardInterrupt, SystemExit): #self.config.logger.info("quit request detected") #raise except Exception, x: self.config.logger.warning( 'Error while processing %s: %s', crash_id, str(x), exc_info=True ) processor_notes.append(str(x)) processor_notes = '; '.join(processor_notes) processed_crash.processor_notes = processor_notes completed_datetime = utc_now() processed_crash.completeddatetime = completed_datetime self._log_job_end( completed_datetime, processed_crash.success, crash_id ) return processed_crash
class BixieProcessor(RequiredConfig): """this class is a processor algorthim for Bixie suitable for use in the 'processor_app' introducted in 2012.""" required_config = Namespace() required_config.add_option('database_class', doc="the class of the database", default=ConnectionContext, from_string_converter=class_converter) required_config.add_option('transaction_executor_class', default="socorro.database.transaction_executor." "TransactionExecutorWithInfiniteBackoff", doc='a class that will manage transactions', from_string_converter=class_converter) required_config.namespace('statistics') required_config.statistics.add_option( 'stats_class', default='socorro.lib.statistics.StatisticsForStatsd', doc='name of a class that will gather statistics', from_string_converter=class_converter) #-------------------------------------------------------------------------- def __init__(self, config, quit_check_callback=None): super(BixieProcessor, self).__init__() self.config = config if quit_check_callback: self.quit_check = quit_check_callback else: self.quit_check = lambda: False self.database = self.config.database_class(config) self.transaction = \ self.config.transaction_executor_class( config, self.database, self.quit_check ) self.rule_system = TransformRuleSystem() self._load_transform_rules() self._statistics = config.statistics.stats_class( config.statistics, self.config.processor_name) self._statistics.incr('restarts') #-------------------------------------------------------------------------- def reject_raw_crash(self, crash_id, reason): self._log_job_start(crash_id) self.config.logger.warning('%s rejected: %s', crash_id, reason) self._log_job_end(False, crash_id) #-------------------------------------------------------------------------- def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dumps): """ This function is run only by a worker thread. Given a job, fetch a thread local database connection and the json document. Use these to create the record in the 'reports' table, then start the analysis of the dump file. input parameters: raw_crash - a nested dict of the form outlined at https://gist.github.com/lonnen/dafb5fdf8611201572f1 raw_dumps - for future use if we choose to add binary attachments to crashes """ raw_crash = DotDict(raw_crash) self._statistics.incr('jobs') processor_notes = [] processed_crash = self._create_minimal_processed_crash() try: self.quit_check() crash_id = raw_crash['crash_id'] started_timestamp = self._log_job_start(crash_id) processed_crash.processor.started_timestamp = started_timestamp processed_crash.crash_id = raw_crash['crash_id'] self.rule_system.apply_all_rules(raw_crash, raw_dumps, processed_crash, self) processed_crash.success = True except Exception, x: self.config.logger.warning('Error while processing %s: %s', raw_crash['crash_id'], str(x), exc_info=True) processed_crash.success = False processor_notes.append('unrecoverable processor error') self._statistics.incr('errors') processed_crash.processor.notes = processor_notes completed_timestamp = utc_now() processed_crash.processor.completed_timestamp = completed_timestamp self._log_job_end(processed_crash.success, crash_id) return processed_crash
class LegacyCrashProcessor(RequiredConfig): """this class is a refactoring of the original processor algorithm into a single class. This class is suitble for use in the 'processor_app' introducted in 2012.""" required_config = Namespace() required_config.add_option('database_class', doc="the class of the database", default=ConnectionContext, from_string_converter=class_converter) required_config.add_option('transaction_executor_class', default=TransactionExecutor, doc='a class that will manage transactions', from_string_converter=class_converter) required_config.add_option( 'stackwalk_command_line', doc='the template for the command to invoke minidump_stackwalk', default='$minidump_stackwalk_pathname -m $dumpfilePathname ' '$processor_symbols_pathname_list 2>/dev/null', ) required_config.add_option( 'minidump_stackwalk_pathname', doc='the full pathname of the extern program minidump_stackwalk ' '(quote path with embedded spaces)', default='/data/socorro/stackwalk/bin/minidump_stackwalk', ) required_config.add_option( 'symbol_cache_path', doc='the path where the symbol cache is found (quote path with ' 'embedded spaces)', default='/mnt/socorro/symbols', ) required_config.add_option( 'processor_symbols_pathname_list', doc='comma or space separated list of symbol files for ' 'minidump_stackwalk (quote paths with embedded spaces)', default='/mnt/socorro/symbols/symbols_ffx,' '/mnt/socorro/symbols/symbols_sea,' '/mnt/socorro/symbols/symbols_tbrd,' '/mnt/socorro/symbols/symbols_sbrd,' '/mnt/socorro/symbols/symbols_os', from_string_converter=create_symbol_path_str) required_config.add_option( 'crashing_thread_frame_threshold', doc='the number of frames to keep in the raw dump for the ' 'crashing thread', default=100, ) required_config.add_option( 'crashing_thread_tail_frame_threshold', doc='the number of frames to keep in the raw dump at the tail of the ' 'frame list', default=10, ) required_config.add_option( 'temporary_file_system_storage_path', doc='a local filesystem path where processor can write dumps ' 'temporarily for processing', default='/home/socorro/temp', ) required_config.namespace('c_signature') required_config.c_signature.add_option( 'c_signature_tool_class', doc='the class that can generate a C signature', default='socorro.processor.signature_utilities.CSignatureTool', from_string_converter=class_converter) required_config.namespace('java_signature') required_config.java_signature.add_option( 'java_signature_tool_class', doc='the class that can generate a Java signature', default='socorro.processor.signature_utilities.JavaSignatureTool', from_string_converter=class_converter) required_config.add_option( 'known_flash_identifiers', doc='A subset of the known "debug identifiers" for flash versions, ' 'associated to the version', default={ '7224164B5918E29AF52365AF3EAF7A500': '10.1.51.66', 'C6CDEFCDB58EFE5C6ECEF0C463C979F80': '10.1.51.66', '4EDBBD7016E8871A461CCABB7F1B16120': '10.1', 'D1AAAB5D417861E6A5B835B01D3039550': '10.0.45.2', 'EBD27FDBA9D9B3880550B2446902EC4A0': '10.0.45.2', '266780DB53C4AAC830AFF69306C5C0300': '10.0.42.34', 'C4D637F2C8494896FBD4B3EF0319EBAC0': '10.0.42.34', 'B19EE2363941C9582E040B99BB5E237A0': '10.0.32.18', '025105C956638D665850591768FB743D0': '10.0.32.18', '986682965B43DFA62E0A0DFFD7B7417F0': '10.0.23', '937DDCC422411E58EF6AD13710B0EF190': '10.0.23', '860692A215F054B7B9474B410ABEB5300': '10.0.22.87', '77CB5AC61C456B965D0B41361B3F6CEA0': '10.0.22.87', '38AEB67F6A0B43C6A341D7936603E84A0': '10.0.12.36', '776944FD51654CA2B59AB26A33D8F9B30': '10.0.12.36', '974873A0A6AD482F8F17A7C55F0A33390': '9.0.262.0', 'B482D3DFD57C23B5754966F42D4CBCB60': '9.0.262.0', '0B03252A5C303973E320CAA6127441F80': '9.0.260.0', 'AE71D92D2812430FA05238C52F7E20310': '9.0.246.0', '6761F4FA49B5F55833D66CAC0BBF8CB80': '9.0.246.0', '27CC04C9588E482A948FB5A87E22687B0': '9.0.159.0', '1C8715E734B31A2EACE3B0CFC1CF21EB0': '9.0.159.0', 'F43004FFC4944F26AF228334F2CDA80B0': '9.0.151.0', '890664D4EF567481ACFD2A21E9D2A2420': '9.0.151.0', '8355DCF076564B6784C517FD0ECCB2F20': '9.0.124.0', '51C00B72112812428EFA8F4A37F683A80': '9.0.124.0', '9FA57B6DC7FF4CFE9A518442325E91CB0': '9.0.115.0', '03D99C42D7475B46D77E64D4D5386D6D0': '9.0.115.0', '0CFAF1611A3C4AA382D26424D609F00B0': '9.0.47.0', '0F3262B5501A34B963E5DF3F0386C9910': '9.0.47.0', 'C5B5651B46B7612E118339D19A6E66360': '9.0.45.0', 'BF6B3B51ACB255B38FCD8AA5AEB9F1030': '9.0.28.0', '83CF4DC03621B778E931FC713889E8F10': '9.0.16.0', }) required_config.add_option( 'collect_addon', doc='boolean indictating if information about add-ons should be ' 'collected', default=True, ) required_config.add_option( 'collect_crash_process', doc='boolean indictating if information about process type should be ' 'collected', default=True, ) #-------------------------------------------------------------------------- def __init__(self, config, quit_check_callback=None): super(LegacyCrashProcessor, self).__init__() self.config = config if quit_check_callback: self.quit_check = quit_check_callback else: self.quit_check = lambda: False self.database = self.config.database_class(config) self.transaction = \ self.config.transaction_executor_class( config, self.database, quit_check_callback ) self.raw_crash_transform_rule_system = TransformRuleSystem() self._load_transform_rules() # *** originally from the ExternalProcessor class #preprocess the breakpad_stackwalk command line strip_parens_re = re.compile(r'\$(\()(\w+)(\))') convert_to_python_substitution_format_re = re.compile(r'\$(\w+)') # Canonical form of $(param) is $param. Convert any that are needed tmp = strip_parens_re.sub(r'$\2', config.stackwalk_command_line) # Convert canonical $dumpfilePathname to DUMPFILEPATHNAME tmp = tmp.replace('$dumpfilePathname', 'DUMPFILEPATHNAME') # finally, convert any remaining $param to pythonic %(param)s tmp = convert_to_python_substitution_format_re.sub(r'%(\1)s', tmp) self.command_line = tmp % config # *** end from ExternalProcessor self.c_signature_tool = config.c_signature.c_signature_tool_class( config.c_signature) self.java_signature_tool = \ config.java_signature.java_signature_tool_class( config.java_signature ) #-------------------------------------------------------------------------- def reject_raw_crash(self, crash_id, reason): self._log_job_start(crash_id) self.config.logger.warning('%s rejected: %s', crash_id, reason) self._log_job_end(utc_now(), False, crash_id) #-------------------------------------------------------------------------- def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dump): """ This function is run only by a worker thread. Given a job, fetch a thread local database connection and the json document. Use these to create the record in the 'reports' table, then start the analysis of the dump file. input parameters: """ try: self.quit_check() crash_id = raw_crash.uuid processor_notes = [] processed_crash = DotDict() processed_crash.uuid = raw_crash.uuid processed_crash.success = False started_timestamp = self._log_job_start(crash_id) #self.config.logger.debug('about to apply rules') self.raw_crash_transform_rule_system.apply_all_rules( raw_crash, self) #self.config.logger.debug('done applying transform rules') try: submitted_timestamp = datetimeFromISOdateString( raw_crash.submitted_timestamp) except KeyError: submitted_timestamp = dateFromOoid(crash_id) # formerly the call to 'insertReportIntoDatabase' processed_crash_update = self._create_basic_processed_crash( crash_id, raw_crash, submitted_timestamp, started_timestamp, processor_notes) processed_crash.update(processed_crash_update) temp_dump_pathname = self._get_temp_dump_pathname( crash_id, raw_dump) try: #logger.debug('about to doBreakpadStackDumpAnalysis') processed_crash_update_dict = \ self._do_breakpad_stack_dump_analysis( crash_id, temp_dump_pathname, processed_crash.hang_type, processed_crash.java_stack_trace, submitted_timestamp, processor_notes ) processed_crash.update(processed_crash_update_dict) finally: self._cleanup_temp_file(temp_dump_pathname) processed_crash.topmost_filenames = "|".join( processed_crash.get('topmost_filenames', [])) try: processed_crash.Winsock_LSP = raw_crash.Winsock_LSP except KeyError: pass # if it's not in the original raw_crash, # it does get into the jsonz #except (KeyboardInterrupt, SystemExit): #self.config.logger.info("quit request detected") #raise except Exception, x: self.config.logger.warning('Error while processing %s: %s', crash_id, str(x), exc_info=True) processor_notes.append(str(x)) processor_notes = '; '.join(processor_notes) processed_crash.processor_notes = processor_notes completed_datetime = utc_now() processed_crash.completeddatetime = completed_datetime self._log_job_end(completed_datetime, processed_crash.success, crash_id) return processed_crash