def test_get(self): state = CrontabberState(config=self.config) res = state.get() self.assertEqual(res['state'], json.loads(_SAMPLE_JSON)) self.assertTrue(isinstance(res['last_updated'], basestring)) # it should be a parsable datetime datetimeutil.datetimeFromISOdateString(res['last_updated'])
def processJob (self, jobTuple): """ This function is run only by a worker thread. Given a job, fetch a thread local database connection and the json document. Use these to create the record in the 'reports' table, then start the analysis of the dump file. input parameters: jobTuple: a tuple containing up to three items: the jobId (the primary key from the jobs table), the jobUuid (a unique string with the json file basename minus the extension) and the priority (an integer) """ threadName = threading.currentThread().getName() try: threadLocalDatabaseConnection, threadLocalCursor = self.databaseConnectionPool.connectionCursorPair() except: self.quit = True socorro.lib.util.reportExceptionAndAbort(logger) # can't continue without a database connection try: processorErrorMessages = [] jobId, jobUuid, jobPriority = jobTuple logger.info("%s - starting job: %s, %s", threadName, jobId, jobUuid) startedDateTime = datetime.datetime.now() threadLocalCursor.execute("update jobs set starteddatetime = %s where id = %s", (startedDateTime, jobId)) threadLocalDatabaseConnection.commit() jobPathname = self.jsonPathForUuidInJsonDumpStorage(jobUuid) dumpfilePathname = self.dumpPathForUuidInJsonDumpStorage(jobUuid) jsonFile = open(jobPathname) try: jsonDocument = simplejson.load(jsonFile) finally: jsonFile.close() try: date_processed = sdt.datetimeFromISOdateString(jsonDocument["submitted_timestamp"]) except KeyError: date_processed = ooid.dateFromOoid(jobUuid) reportId = self.insertReportIntoDatabase(threadLocalCursor, jobUuid, jsonDocument, jobPathname, date_processed, processorErrorMessages) threadLocalDatabaseConnection.commit() truncated = self.doBreakpadStackDumpAnalysis(reportId, jobUuid, dumpfilePathname, threadLocalCursor, date_processed, processorErrorMessages) self.quitCheck() #finished a job - cleanup threadLocalCursor.execute("update jobs set completeddatetime = %s, success = True where id = %s", (datetime.datetime.now(), jobId)) threadLocalCursor.execute("update reports set started_datetime = timestamp without time zone '%s', completed_datetime = timestamp without time zone '%s', success = True, truncated = %s where id = %s and date_processed = timestamp without time zone '%s'" % (startedDateTime, datetime.datetime.now(), truncated, reportId, date_processed)) #self.updateRegistrationNoCommit(threadLocalCursor) threadLocalDatabaseConnection.commit() logger.info("%s - succeeded and committed: %s, %s", threadName, jobId, jobUuid) except (KeyboardInterrupt, SystemExit): logger.info("%s - quit request detected", threadName) self.quit = True try: logger.info("%s - abandoning job with rollback: %s, %s", threadName, jobId, jobUuid) threadLocalDatabaseConnection.rollback() threadLocalDatabaseConnection.close() except: pass except DuplicateEntryException, x: logger.warning("%s - duplicate entry: %s", threadName, jobUuid)
def post(self, *args): " Webpy method receives inputs from uri " errors = [] email_form = self.email_form() if email_form.validates(): product = email_form['product'].value versions = tuple([x.strip() for x in email_form['versions'].value.split(',')]) signature = email_form['signature'].value subject = email_form['subject'].value body = email_form['body'].value start_date = dtutil.datetimeFromISOdateString(email_form['start_date'].value) end_date = dtutil.datetimeFromISOdateString(email_form['end_date'].value) author = email_form['author'].value logger.info("%s is creating an email campaign for %s %s crashes in [%s] Between %s and %s" %(author, product, versions, signature, start_date, end_date)) connection = self.database.connection() try: cursor = connection.cursor() campaign_id, full_email_rows = self.create_email_campaign(cursor, product, versions, signature, subject, body, start_date, end_date, author) logger.info('full_email_rows: %s' % full_email_rows) email_addresses = [row['email'] for row in full_email_rows] logger.info('email_addresses: %s' % email_addresses) email_contact_ids = self.save_campaign_contacts(cursor, campaign_id, email_addresses) logger.info('email_contact_ids: %s' % email_contact_ids) connection.commit() return {'campaign_id': campaign_id} finally: connection.close() else: web.badrequest() for field in ['product', 'versions', 'signature', 'subject', 'body', 'start_date', 'end_date', 'author']: if email_form[field].note: # Example "product: Required" errors.append("%s: %s" % (field, email_form[field].note)) logger.info("Bad Request. %s" % ', '.join(errors)) return {'message': ', '.join(errors)}
def export_uuids(path, numberofdays): """Export crash report uuids from a PostgreSQL database to a CSV file path - Directory where the csv file will be created. numberofdays - Number of days of crash reports to retrieve, before the most recent crash date. """ database = db.Database(config) connection = database.connection() cur = connection.cursor() # steps # 1. pull all distinct dates sql = """ SELECT DISTINCT to_char(date_processed, 'YYYY-MM-DD') as day FROM reports ORDER BY day DESC """ if numberofdays: sql = "%s LIMIT %s" % (sql, numberofdays) print 'Calculating dates... ' days = db.execute(cur, sql) days_list = [] for day in days: days_list.append(day[0]) store_filename = 'uuids.csv' store_filename = os.path.normpath('%s/%s' % (path, store_filename)) store_file = open(store_filename, 'w') store = csv.writer(store_file, delimiter=',', quotechar='"') print 'Store file created: %s' % store_filename for day in days_list: date_from = dtu.datetimeFromISOdateString(day) date_to = date_from + datetime.timedelta(1) sql = "SELECT uuid FROM reports WHERE date_processed BETWEEN %s AND %s" print 'Getting crash reports for day %s' % date_from.date() crashes_list = db.execute(cur, sql, (date_from, date_to)) for crash in crashes_list: store.writerow(crash) store_file.close() connection.close() return store_filename
def format_date(date): """ Take a string and return a datetime object. """ if not date: return None if type(date) is not datetime: if type(date) is list: date = " ".join(date) try: date = dtutil.datetimeFromISOdateString(date) except Exception: date = None return date
def new_entry(self, crash_id, raw_crash, dumps_dict, webhead_host_name='webhead01', timestamp=None): if not isinstance(dumps_dict, collections.Mapping): dumps_dict = {self.dump_field: dumps_dict} name_dir, date_dir = super(JsonDumpStorage, self).newEntry( crash_id, datetimeFromISOdateString(raw_crash['submitted_timestamp']), webhead_host_name ) raw_crash_pathname = os.path.join( name_dir, crash_id + self.jsonSuffix ) with open(raw_crash_pathname, "w") as rcf: json.dump(raw_crash, rcf) for dump_name, dump in dumps_dict.iteritems(): full_dump_name = self.dump_file_name(crash_id, dump_name) dump_pathname = os.path.join( name_dir, full_dump_name ) with open(dump_pathname, "w") as dp: dp.write(dump) self.osModule.chmod(dump_pathname, self.dumpPermissions) name_depth = socorro_ooid.depthFromOoid(crash_id) if not name_depth: name_depth = 4 rparts = [os.path.pardir, ] * (1 + name_depth) rparts.append(self.dateName) date_depth = 2 # .../hh/mm_slot... if webhead_host_name and self.subSlotCount: date_depth = 3 # .../webHeadName_slot date_parts = date_dir.split(os.path.sep)[-date_depth:] rparts.extend(date_parts) self.osModule.symlink( os.path.sep.join(rparts), os.path.join(name_dir, crash_id) )
def new_entry(self, crash_id, raw_crash, dumps_dict, webhead_host_name='webhead01', timestamp=None): if not isinstance(dumps_dict, collections.Mapping): dumps_dict = {self.dump_field: dumps_dict} name_dir, date_dir = super(JsonDumpStorage, self).newEntry( crash_id, datetimeFromISOdateString(raw_crash['submitted_timestamp']), webhead_host_name) raw_crash_pathname = os.path.join(name_dir, crash_id + self.jsonSuffix) with open(raw_crash_pathname, "w") as rcf: json.dump(raw_crash, rcf) for dump_name, dump in dumps_dict.iteritems(): full_dump_name = self.dump_file_name(crash_id, dump_name) dump_pathname = os.path.join(name_dir, full_dump_name) with open(dump_pathname, "w") as dp: dp.write(dump) self.osModule.chmod(dump_pathname, self.dumpPermissions) name_depth = socorro_ooid.depthFromOoid(crash_id) if not name_depth: name_depth = 4 rparts = [ os.path.pardir, ] * (1 + name_depth) rparts.append(self.dateName) date_depth = 2 # .../hh/mm_slot... if webhead_host_name and self.subSlotCount: date_depth = 3 # .../webHeadName_slot date_parts = date_dir.split(os.path.sep)[-date_depth:] rparts.extend(date_parts) self.osModule.symlink(os.path.sep.join(rparts), os.path.join(name_dir, crash_id))
def test_create_basic_processed_crash_normal(self): config = setup_config_with_mocks() config.collect_addon = False config.collect_crash_process = False mocked_transform_rules_str = \ 'socorro.processor.legacy_processor.TransformRuleSystem' with mock.patch(mocked_transform_rules_str) as m_transform_class: m_transform = mock.Mock() m_transform_class.return_value = m_transform m_transform.attach_mock(mock.Mock(), 'apply_all_rules') utc_now_str = 'socorro.processor.legacy_processor.utc_now' with mock.patch(utc_now_str) as m_utc_now: m_utc_now.return_value = datetime(2012, 5, 4, 15, 11) started_timestamp = datetime(2012, 5, 4, 15, 10) raw_crash = canonical_standard_raw_crash leg_proc = LegacyCrashProcessor(config, config.mock_quit_fn) processor_notes = [] # test 01 processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) self.assertEqual(processed_crash, cannonical_basic_processed_crash) # test 02 processor_notes = [] raw_crash_missing_product = copy.deepcopy(raw_crash) del raw_crash_missing_product['ProductName'] processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_missing_product, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_missing_product = \ copy.copy(cannonical_basic_processed_crash) processed_crash_missing_product.product = None self.assertEqual(processed_crash, processed_crash_missing_product) self.assertTrue('WARNING: raw_crash missing ProductName' in processor_notes) self.assertEqual(len(processor_notes), 1) # test 03 processor_notes = [] raw_crash_missing_version = copy.deepcopy(raw_crash) del raw_crash_missing_version['Version'] processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_missing_version, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_missing_version = \ copy.copy(cannonical_basic_processed_crash) processed_crash_missing_version.version = None self.assertEqual(processed_crash, processed_crash_missing_version) self.assertTrue( 'WARNING: raw_crash missing Version' in processor_notes) self.assertEqual(len(processor_notes), 1) # test 04 processor_notes = [] raw_crash_with_hangid = copy.deepcopy(raw_crash) raw_crash_with_hangid.HangID = \ '30cb3212-b61d-4d1f-85ae-3bc4bcaa0504' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_with_hangid, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_with_hangid = \ copy.copy(cannonical_basic_processed_crash) processed_crash_with_hangid.hangid = \ raw_crash_with_hangid.HangID processed_crash_with_hangid.hang_type = -1 self.assertEqual(processed_crash, processed_crash_with_hangid) self.assertEqual(len(processor_notes), 0)
def test_create_basic_processed_crash_normal(self): config = setup_config_with_mocks() config.collect_addon = False config.collect_crash_process = False mocked_transform_rules_str = \ 'socorro.processor.hybrid_processor.TransformRuleSystem' with mock.patch(mocked_transform_rules_str) as m_transform_class: m_transform = mock.Mock() m_transform_class.return_value = m_transform m_transform.attach_mock(mock.Mock(), 'apply_all_rules') utc_now_str = 'socorro.processor.hybrid_processor.utc_now' with mock.patch(utc_now_str) as m_utc_now: m_utc_now.return_value = datetime(2012, 5, 4, 15, 11, tzinfo=UTC) started_timestamp = datetime(2012, 5, 4, 15, 10, tzinfo=UTC) raw_crash = canonical_standard_raw_crash leg_proc = HybridCrashProcessor(config, config.mock_quit_fn) processor_notes = [] # test 01 processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) assert 'exploitability' in processed_crash self.assertEqual(processed_crash, dict(cannonical_basic_processed_crash)) # test 02 processor_notes = [] raw_crash_missing_product = copy.deepcopy(raw_crash) del raw_crash_missing_product['ProductName'] processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_missing_product, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_missing_product = \ copy.copy(cannonical_basic_processed_crash) processed_crash_missing_product.product = None self.assertEqual(processed_crash, processed_crash_missing_product) self.assertTrue('WARNING: raw_crash missing ProductName' in processor_notes) self.assertEqual(len(processor_notes), 1) # test 03 processor_notes = [] raw_crash_missing_version = copy.deepcopy(raw_crash) del raw_crash_missing_version['Version'] processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_missing_version, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_missing_version = \ copy.copy(cannonical_basic_processed_crash) processed_crash_missing_version.version = None self.assertEqual(processed_crash, processed_crash_missing_version) self.assertTrue( 'WARNING: raw_crash missing Version' in processor_notes) self.assertEqual(len(processor_notes), 1) # test 04 processor_notes = [] raw_crash_with_hangid = copy.deepcopy(raw_crash) raw_crash_with_hangid.HangID = \ '30cb3212-b61d-4d1f-85ae-3bc4bcaa0504' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_with_hangid, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_with_hangid = \ copy.copy(cannonical_basic_processed_crash) processed_crash_with_hangid.hangid = \ raw_crash_with_hangid.HangID processed_crash_with_hangid.hang_type = -1 self.assertEqual(processed_crash, processed_crash_with_hangid) self.assertEqual(len(processor_notes), 0) # test 05 processor_notes = [] raw_crash_with_pluginhang = copy.deepcopy(raw_crash) raw_crash_with_pluginhang.PluginHang = '1' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_with_pluginhang, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_with_pluginhang = \ copy.copy(cannonical_basic_processed_crash) processed_crash_with_pluginhang.hangid = \ 'fake-3bc4bcaa-b61d-4d1f-85ae-30cb32120504' processed_crash_with_pluginhang.hang_type = -1 self.assertEqual(processed_crash, processed_crash_with_pluginhang) self.assertEqual(len(processor_notes), 0) # test 06 processor_notes = [] raw_crash_with_hang_only = copy.deepcopy(raw_crash) raw_crash_with_hang_only.Hang = 16 processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_with_hang_only, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_with_hang_only = \ copy.copy(cannonical_basic_processed_crash) processed_crash_with_hang_only.hang_type = 1 self.assertEqual(processed_crash, processed_crash_with_hang_only) self.assertEqual(len(processor_notes), 0) leg_proc._statistics.assert_has_calls([ mock.call.incr('restarts'), ], any_order=True) # test 07 processor_notes = [] raw_crash_with_hang_only = copy.deepcopy(raw_crash) raw_crash_with_hang_only.Hang = 'bad value' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_with_hang_only, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_with_hang_only = \ copy.copy(cannonical_basic_processed_crash) processed_crash_with_hang_only.hang_type = 0 self.assertEqual(processed_crash, processed_crash_with_hang_only) self.assertEqual(len(processor_notes), 0) leg_proc._statistics.assert_has_calls([ mock.call.incr('restarts'), ], any_order=True) # test 08 processor_notes = [] bad_raw_crash = copy.deepcopy(raw_crash) bad_raw_crash['SecondsSinceLastCrash'] = 'badness' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', bad_raw_crash, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) self.assertEqual(processed_crash.last_crash, None) self.assertTrue('non-integer value of "SecondsSinceLastCrash"' in processor_notes) # test 09 processor_notes = [] bad_raw_crash = copy.deepcopy(raw_crash) bad_raw_crash['CrashTime'] = 'badness' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', bad_raw_crash, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) self.assertEqual(processed_crash.crash_time, 0) self.assertTrue( 'non-integer value of "CrashTime"' in processor_notes) # test 10 processor_notes = [] bad_raw_crash = copy.deepcopy(raw_crash) bad_raw_crash['StartupTime'] = 'badness' bad_raw_crash['InstallTime'] = 'more badness' bad_raw_crash['CrashTime'] = 'even more badness' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', bad_raw_crash, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) self.assertEqual(processed_crash.install_age, 0) self.assertTrue( 'non-integer value of "StartupTime"' in processor_notes) self.assertTrue( 'non-integer value of "InstallTime"' in processor_notes) self.assertTrue( 'non-integer value of "CrashTime"' in processor_notes)
def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta): processor_notes = processor_meta.processor_notes processed_crash.submitted_timestamp = raw_crash.get( 'submitted_timestamp', dateFromOoid(raw_crash.uuid)) if isinstance(processed_crash.submitted_timestamp, basestring): processed_crash.submitted_timestamp = datetimeFromISOdateString( processed_crash.submitted_timestamp) processed_crash.date_processed = processed_crash.submitted_timestamp # defaultCrashTime: must have crashed before date processed submitted_timestamp_as_epoch = int( time.mktime(processed_crash.submitted_timestamp.timetuple())) try: timestampTime = int( raw_crash.get('timestamp', submitted_timestamp_as_epoch) ) # the old name for crash time except ValueError: timestampTime = 0 processor_notes.append('non-integer value of "timestamp"') try: crash_time = int( self._get_truncate_or_warn(raw_crash, 'CrashTime', processor_notes, timestampTime, 10)) except ValueError: crash_time = 0 processor_notes.append('non-integer value of "CrashTime" (%s)' % raw_crash.CrashTime) processed_crash.crash_time = crash_time if crash_time == submitted_timestamp_as_epoch: processor_notes.append("client_crash_date is unknown") # StartupTime: must have started up some time before crash try: startupTime = int(raw_crash.get('StartupTime', crash_time)) except ValueError: startupTime = 0 processor_notes.append('non-integer value of "StartupTime"') # InstallTime: must have installed some time before startup try: installTime = int(raw_crash.get('InstallTime', startupTime)) except ValueError: installTime = 0 processor_notes.append('non-integer value of "InstallTime"') processed_crash.client_crash_date = datetime.datetime.fromtimestamp( crash_time, UTC) processed_crash.install_age = crash_time - installTime processed_crash.uptime = max(0, crash_time - startupTime) try: last_crash = int(raw_crash.SecondsSinceLastCrash) except (KeyError, TypeError, ValueError): last_crash = None processor_notes.append( 'non-integer value of "SecondsSinceLastCrash"') if last_crash > maxint: last_crash = None processor_notes.append( '"SecondsSinceLastCrash" larger than MAXINT - set to NULL') processed_crash.last_crash = last_crash return True
def export(path, numberofdays=0): """Export crash reports from a PostgreSQL database. path - Directory where the dump file will be created. numberofdays - Number of days of crash reports to retrieve, before the most recent crash date. """ database = db.Database(config) connection = database.connection() cur = connection.cursor() crash_files = [] fields_list = ("client_crash_date", "date_processed", "uuid", "product", "version", "build", "signature", "url", "install_age", "last_crash", "uptime", "cpu_name", "cpu_info", "reason", "address", "os_name", "os_version", "email", "build_date", "user_id", "started_datetime", "completed_datetime", "success", "truncated", "processor_notes", "user_comments", "app_notes", "distributor", "distributor_version", "topmost_filenames", "addons_checked", "flash_version", "hangid", "process_type", "release_channel") # steps # 1. pull all distinct dates sql = """ SELECT DISTINCT to_char(date_processed, 'YYYY-MM-DD') as day FROM reports ORDER BY day DESC """ if numberofdays: sql = "%s LIMIT %s" % (sql, numberofdays) print 'Calculating dates... ' days = db.execute(cur, sql) days_list = [] for day in days: days_list.append(day[0]) #~ days_list = [ #~ '2012-03-04T00:00:00+00:00' #~ ] store_filename = 'dump.json' store_filename = os.path.normpath('%s/%s' % (path, store_filename)) store = open(store_filename, 'w') print 'Store file created: %s' % store_filename indexes_filename = 'indexes.txt' indexes_filename = os.path.normpath('%s/%s' % (path, indexes_filename)) indexes = open(indexes_filename, 'w') print 'Indexes file created: %s' % indexes_filename for day in days_list: date_from = dtu.datetimeFromISOdateString(day) date_to = date_from + datetime.timedelta(1) datestr = date_from.strftime('%y%m%d') es_index = 'socorro_%s' % datestr es_type = 'crash_reports' action_line = '{"index":{"_index":"%s","_type":"%s"}}\n' % ( es_index, es_type) indexes.write('%s\n' % es_index) # 2. for each date, pull all crashes of the day day_sql = " ".join(("SELECT %s" % ", ".join(fields_list), "FROM reports", "WHERE date_processed BETWEEN %s AND %s")) print 'Getting crash reports for day %s' % date_from.date() crashes_list = db.execute(cur, day_sql, (date_from, date_to)) for crash in crashes_list: # 3. for each crash report json_crash = dict(zip(fields_list, crash)) # stringify datetime fields for i in json_crash: if isinstance(json_crash[i], datetime.datetime): json_crash[i] = dtu.date_to_string(json_crash[i]) store.write(action_line) store.write('%s\n' % json.dumps(json_crash)) store.close() crash_files.append(store_filename) indexes.close() crash_files.append(indexes_filename) connection.close() return generate_dump(crash_files, path)
def test_create_basic_processed_crash_normal(self): config = setup_config_with_mocks() config.collect_addon = False config.collect_crash_process = False mocked_transform_rules_str = \ 'socorro.processor.hybrid_processor.TransformRuleSystem' with mock.patch(mocked_transform_rules_str) as m_transform_class: m_transform = mock.Mock() m_transform_class.return_value = m_transform m_transform.attach_mock(mock.Mock(), 'apply_all_rules') utc_now_str = 'socorro.processor.hybrid_processor.utc_now' with mock.patch(utc_now_str) as m_utc_now: m_utc_now.return_value = datetime(2012, 5, 4, 15, 11, tzinfo=UTC) started_timestamp = datetime(2012, 5, 4, 15, 10, tzinfo=UTC) raw_crash = canonical_standard_raw_crash leg_proc = HybridCrashProcessor(config, config.mock_quit_fn) processor_notes = [] # test 01 processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) assert 'exploitability' in processed_crash self.assertEqual( processed_crash, dict(cannonical_basic_processed_crash) ) # test 02 processor_notes = [] raw_crash_missing_product = copy.deepcopy(raw_crash) del raw_crash_missing_product['ProductName'] processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_missing_product, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_missing_product = \ copy.copy(cannonical_basic_processed_crash) processed_crash_missing_product.product = None self.assertEqual( processed_crash, processed_crash_missing_product ) self.assertTrue('WARNING: raw_crash missing ProductName' in processor_notes) self.assertEqual(len(processor_notes), 1) # test 03 processor_notes = [] raw_crash_missing_version = copy.deepcopy(raw_crash) del raw_crash_missing_version['Version'] processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_missing_version, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_missing_version = \ copy.copy(cannonical_basic_processed_crash) processed_crash_missing_version.version = None self.assertEqual( processed_crash, processed_crash_missing_version ) self.assertTrue('WARNING: raw_crash missing Version' in processor_notes) self.assertEqual(len(processor_notes), 1) # test 04 processor_notes = [] raw_crash_with_hangid = copy.deepcopy(raw_crash) raw_crash_with_hangid.HangID = \ '30cb3212-b61d-4d1f-85ae-3bc4bcaa0504' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_with_hangid, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_with_hangid = \ copy.copy(cannonical_basic_processed_crash) processed_crash_with_hangid.hangid = \ raw_crash_with_hangid.HangID processed_crash_with_hangid.hang_type = -1 self.assertEqual( processed_crash, processed_crash_with_hangid ) self.assertEqual(len(processor_notes), 0) # test 05 processor_notes = [] raw_crash_with_pluginhang = copy.deepcopy(raw_crash) raw_crash_with_pluginhang.PluginHang = '1' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_with_pluginhang, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_with_pluginhang = \ copy.copy(cannonical_basic_processed_crash) processed_crash_with_pluginhang.hangid = \ 'fake-3bc4bcaa-b61d-4d1f-85ae-30cb32120504' processed_crash_with_pluginhang.hang_type = -1 self.assertEqual( processed_crash, processed_crash_with_pluginhang ) self.assertEqual(len(processor_notes), 0) # test 06 processor_notes = [] raw_crash_with_hang_only = copy.deepcopy(raw_crash) raw_crash_with_hang_only.Hang = 16 processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_with_hang_only, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_with_hang_only = \ copy.copy(cannonical_basic_processed_crash) processed_crash_with_hang_only.hang_type = 1 self.assertEqual( processed_crash, processed_crash_with_hang_only ) self.assertEqual(len(processor_notes), 0) leg_proc._statistics.assert_has_calls( [ mock.call.incr('restarts'), ], any_order=True ) # test 07 processor_notes = [] raw_crash_with_hang_only = copy.deepcopy(raw_crash) raw_crash_with_hang_only.Hang = 'bad value' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_with_hang_only, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_with_hang_only = \ copy.copy(cannonical_basic_processed_crash) processed_crash_with_hang_only.hang_type = 0 self.assertEqual( processed_crash, processed_crash_with_hang_only ) self.assertEqual(len(processor_notes), 0) leg_proc._statistics.assert_has_calls( [ mock.call.incr('restarts'), ], any_order=True ) # test 08 processor_notes = [] bad_raw_crash = copy.deepcopy(raw_crash) bad_raw_crash['SecondsSinceLastCrash'] = 'badness' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', bad_raw_crash, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) self.assertEqual(processed_crash.last_crash, None) self.assertTrue( 'non-integer value of "SecondsSinceLastCrash"' in processor_notes ) # test 09 processor_notes = [] bad_raw_crash = copy.deepcopy(raw_crash) bad_raw_crash['CrashTime'] = 'badness' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', bad_raw_crash, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) self.assertEqual(processed_crash.crash_time, 0) self.assertTrue( 'non-integer value of "CrashTime"' in processor_notes ) # test 10 processor_notes = [] bad_raw_crash = copy.deepcopy(raw_crash) bad_raw_crash['StartupTime'] = 'badness' bad_raw_crash['InstallTime'] = 'more badness' bad_raw_crash['CrashTime'] = 'even more badness' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', bad_raw_crash, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) self.assertEqual(processed_crash.install_age, 0) self.assertTrue( 'non-integer value of "StartupTime"' in processor_notes ) self.assertTrue( 'non-integer value of "InstallTime"' in processor_notes ) self.assertTrue( 'non-integer value of "CrashTime"' in processor_notes )
def test_create_basic_processed_crash_normal(self): config = setup_config_with_mocks() config.collect_addon = False config.collect_crash_process = False mocked_transform_rules_str = \ 'socorro.processor.legacy_processor.TransformRuleSystem' with mock.patch(mocked_transform_rules_str) as m_transform_class: m_transform = mock.Mock() m_transform_class.return_value = m_transform m_transform.attach_mock(mock.Mock(), 'apply_all_rules') utc_now_str = 'socorro.processor.legacy_processor.utc_now' with mock.patch(utc_now_str) as m_utc_now: m_utc_now.return_value = datetime(2012, 5, 4, 15, 11, tzinfo=UTC) started_timestamp = datetime(2012, 5, 4, 15, 10, tzinfo=UTC) raw_crash = canonical_standard_raw_crash leg_proc = LegacyCrashProcessor(config, config.mock_quit_fn) processor_notes = [] # test 01 processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) assert 'exploitability' in processed_crash self.assertEqual( processed_crash, dict(cannonical_basic_processed_crash) ) # test 02 processor_notes = [] raw_crash_missing_product = copy.deepcopy(raw_crash) del raw_crash_missing_product['ProductName'] processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_missing_product, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_missing_product = \ copy.copy(cannonical_basic_processed_crash) processed_crash_missing_product.product = None self.assertEqual( processed_crash, processed_crash_missing_product ) self.assertTrue('WARNING: raw_crash missing ProductName' in processor_notes) self.assertEqual(len(processor_notes), 1) # test 03 processor_notes = [] raw_crash_missing_version = copy.deepcopy(raw_crash) del raw_crash_missing_version['Version'] processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_missing_version, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_missing_version = \ copy.copy(cannonical_basic_processed_crash) processed_crash_missing_version.version = None self.assertEqual( processed_crash, processed_crash_missing_version ) self.assertTrue('WARNING: raw_crash missing Version' in processor_notes) self.assertEqual(len(processor_notes), 1) # test 04 processor_notes = [] raw_crash_with_hangid = copy.deepcopy(raw_crash) raw_crash_with_hangid.HangID = \ '30cb3212-b61d-4d1f-85ae-3bc4bcaa0504' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_with_hangid, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_with_hangid = \ copy.copy(cannonical_basic_processed_crash) processed_crash_with_hangid.hangid = \ raw_crash_with_hangid.HangID processed_crash_with_hangid.hang_type = -1 self.assertEqual( processed_crash, processed_crash_with_hangid ) self.assertEqual(len(processor_notes), 0) # test 05 processor_notes = [] raw_crash_with_pluginhang = copy.deepcopy(raw_crash) raw_crash_with_pluginhang.PluginHang = '1' processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_with_pluginhang, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_with_pluginhang = \ copy.copy(cannonical_basic_processed_crash) processed_crash_with_pluginhang.hangid = \ 'fake-3bc4bcaa-b61d-4d1f-85ae-30cb32120504' processed_crash_with_pluginhang.hang_type = -1 self.assertEqual( processed_crash, processed_crash_with_pluginhang ) self.assertEqual(len(processor_notes), 0) # test 06 processor_notes = [] raw_crash_with_hang_only = copy.deepcopy(raw_crash) raw_crash_with_hang_only.Hang = 16 processed_crash = leg_proc._create_basic_processed_crash( '3bc4bcaa-b61d-4d1f-85ae-30cb32120504', raw_crash_with_hang_only, datetimeFromISOdateString(raw_crash.submitted_timestamp), started_timestamp, processor_notes, ) processed_crash_with_hang_only = \ copy.copy(cannonical_basic_processed_crash) processed_crash_with_hang_only.hang_type = 1 self.assertEqual( processed_crash, processed_crash_with_hang_only ) self.assertEqual(len(processor_notes), 0)
def get_list(self, **kwargs): """ List all crashes with a given signature and return them. Both `from_date` and `to_date` (and their aliases `from` and `to`) are required and can not be greater than 30 days apart. Optional arguments: see SearchCommon.get_parameters() """ # aliases if "from" in kwargs and "from_date" not in kwargs: kwargs["from_date"] = kwargs.get("from") if "to" in kwargs and "to_date" not in kwargs: kwargs["to_date"] = kwargs.get("to") if not kwargs.get('from_date'): raise MissingArgumentError('from_date') if not kwargs.get('to_date'): raise MissingArgumentError('to_date') from_date = datetimeutil.datetimeFromISOdateString(kwargs['from_date']) to_date = datetimeutil.datetimeFromISOdateString(kwargs['to_date']) span_days = (to_date - from_date).days if span_days > 30: raise BadArgumentError( 'Span between from_date and to_date can not be more than 30') # start with the default sort_order = {'key': 'date_processed', 'direction': 'DESC'} if 'sort' in kwargs: sort_order['key'] = kwargs.pop('sort') _recognized_sort_orders = ( 'date_processed', 'uptime', 'user_comments', 'uuid', 'uuid_text', 'product', 'version', 'build', 'signature', 'url', 'os_name', 'os_version', 'cpu_name', 'cpu_info', 'address', 'reason', 'last_crash', 'install_age', 'hangid', 'process_type', 'release_channel', 'install_time', 'duplicate_of', ) if sort_order['key'] not in _recognized_sort_orders: raise BadArgumentError( '%s is not a recognized sort order key' % sort_order['key']) sort_order['direction'] = 'ASC' if 'reverse' in kwargs: if kwargs.pop('reverse'): sort_order['direction'] = 'DESC' include_raw_crash = kwargs.get('include_raw_crash') or False params = search_common.get_parameters(kwargs) if not params["signature"]: raise MissingArgumentError('signature') params["terms"] = params["signature"] params["search_mode"] = "is_exactly" # Default mode falls back to starts_with for postgres if params["plugin_search_mode"] == "default": params["plugin_search_mode"] = "starts_with" # Limiting to a signature if params["terms"]: params["terms"] = self.prepare_terms(params["terms"], params["search_mode"]) # Searching for terms in plugins if params["report_process"] == "plugin" and params["plugin_terms"]: params["plugin_terms"] = " ".join(params["plugin_terms"]) params["plugin_terms"] = self.prepare_terms( params["plugin_terms"], params["plugin_search_mode"]) # Get information about the versions util_service = Util(config=self.context) params["versions_info"] = util_service.versions_info(**params) # Parsing the versions params["versions_string"] = params["versions"] (params["versions"], params["products"]) = self.parse_versions(params["versions"], params["products"]) if hasattr(self.context, 'webapi'): context = self.context.webapi else: # old middleware context = self.context # Changing the OS ids to OS names for i, elem in enumerate(params["os"]): for platform in context.platforms: if platform["id"][:3] == elem[:3]: params["os"][i] = platform["name"] # Creating the parameters for the sql query sql_params = {} # Preparing the different parts of the sql query sql_select = """ SELECT r.date_processed, r.uptime, r.user_comments, r.uuid::uuid, r.uuid as uuid_text, r.product, r.version, r.build, r.signature, r.url, r.os_name, r.os_version, r.cpu_name, r.cpu_info, r.address, r.reason, r.last_crash, r.install_age, r.hangid, r.process_type, r.release_channel, (r.client_crash_date - (r.install_age * INTERVAL '1 second')) AS install_time """ if include_raw_crash: pass else: sql_select += """ , rd.duplicate_of """ wrapped_select = """ WITH report_slice AS ( %s ), dupes AS ( SELECT report_slice.uuid, rd.duplicate_of FROM reports_duplicates rd JOIN report_slice ON report_slice.uuid_text = rd.uuid WHERE rd.date_processed BETWEEN %%(from_date)s AND %%(to_date)s ) SELECT rs.*, dupes.duplicate_of, rc.raw_crash FROM report_slice rs LEFT OUTER JOIN dupes USING (uuid) LEFT OUTER JOIN raw_crashes rc ON rs.uuid = rc.uuid AND rc.date_processed BETWEEN %%(from_date)s AND %%(to_date)s """ sql_from = self.build_reports_sql_from(params) if not include_raw_crash: sql_from = """%s LEFT OUTER JOIN reports_duplicates rd ON r.uuid = rd.uuid """ % sql_from sql_where, sql_params = self.build_reports_sql_where( params, sql_params, self.context) sql_order = """ ORDER BY %(key)s %(direction)s """ % sort_order sql_limit, sql_params = self.build_reports_sql_limit( params, sql_params) # Assembling the query if include_raw_crash: sql_query = "\n".join( ("/* socorro.external.postgresql.report.Report.list */", sql_select, sql_from, sql_where, sql_order, sql_limit)) else: sql_query = "\n".join( ("/* socorro.external.postgresql.report.Report.list */", sql_select, sql_from, sql_where, sql_order, sql_limit)) # Query for counting the results sql_count_query = "\n".join( ("/* socorro.external.postgresql.report.Report.list */", "SELECT count(*)", sql_from, sql_where)) # Querying the DB with self.get_connection() as connection: total = self.count( sql_count_query, sql_params, error_message="Failed to count crashes from reports.", connection=connection) # No need to call Postgres if we know there will be no results if total: if include_raw_crash: sql_query = wrapped_select % sql_query results = self.query( sql_query, sql_params, error_message="Failed to retrieve crashes from reports", connection=connection) else: results = [] # Transforming the results into what we want fields = ( "date_processed", "uptime", "user_comments", "uuid", "uuid", # the uuid::text one "product", "version", "build", "signature", "url", "os_name", "os_version", "cpu_name", "cpu_info", "address", "reason", "last_crash", "install_age", "hangid", "process_type", "release_channel", "install_time", "duplicate_of", ) if include_raw_crash: fields += ("raw_crash", ) crashes = [] for row in results: crash = dict(zip(fields, row)) if include_raw_crash and crash['raw_crash']: crash['raw_crash'] = json.loads(crash['raw_crash']) for i in crash: try: crash[i] = datetimeutil.date_to_string(crash[i]) except TypeError: pass crashes.append(crash) return {"hits": crashes, "total": total}
def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dump): """ This function is run only by a worker thread. Given a job, fetch a thread local database connection and the json document. Use these to create the record in the 'reports' table, then start the analysis of the dump file. input parameters: """ try: self.quit_check() crash_id = raw_crash.uuid processor_notes = [] processed_crash = DotDict() processed_crash.uuid = raw_crash.uuid processed_crash.success = False started_timestamp = self._log_job_start(crash_id) #self.config.logger.debug('about to apply rules') self.raw_crash_transform_rule_system.apply_all_rules(raw_crash, self) #self.config.logger.debug('done applying transform rules') try: submitted_timestamp = datetimeFromISOdateString( raw_crash.submitted_timestamp ) except KeyError: submitted_timestamp = dateFromOoid(crash_id) # formerly the call to 'insertReportIntoDatabase' processed_crash_update = self._create_basic_processed_crash( crash_id, raw_crash, submitted_timestamp, started_timestamp, processor_notes ) processed_crash.update(processed_crash_update) temp_dump_pathname = self._get_temp_dump_pathname( crash_id, raw_dump ) try: #logger.debug('about to doBreakpadStackDumpAnalysis') processed_crash_update_dict = \ self._do_breakpad_stack_dump_analysis( crash_id, temp_dump_pathname, processed_crash.hang_type, processed_crash.java_stack_trace, submitted_timestamp, processor_notes ) processed_crash.update(processed_crash_update_dict) finally: self._cleanup_temp_file(temp_dump_pathname) processed_crash.topmost_filenames = "|".join( processed_crash.get('topmost_filenames', []) ) try: processed_crash.Winsock_LSP = raw_crash.Winsock_LSP except KeyError: pass # if it's not in the original raw_crash, # it does get into the jsonz #except (KeyboardInterrupt, SystemExit): #self.config.logger.info("quit request detected") #raise except Exception, x: self.config.logger.warning( 'Error while processing %s: %s', crash_id, str(x), exc_info=True ) processor_notes.append(str(x))
def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta): processor_notes = processor_meta.processor_notes processed_crash.submitted_timestamp = raw_crash.get( 'submitted_timestamp', dateFromOoid(raw_crash.uuid) ) if isinstance(processed_crash.submitted_timestamp, basestring): processed_crash.submitted_timestamp = datetimeFromISOdateString( processed_crash.submitted_timestamp ) processed_crash.date_processed = processed_crash.submitted_timestamp # defaultCrashTime: must have crashed before date processed submitted_timestamp_as_epoch = int( time.mktime(processed_crash.submitted_timestamp.timetuple()) ) try: timestampTime = int( raw_crash.get('timestamp', submitted_timestamp_as_epoch) ) # the old name for crash time except ValueError: timestampTime = 0 processor_notes.append('non-integer value of "timestamp"') try: crash_time = int( self._get_truncate_or_warn( raw_crash, 'CrashTime', processor_notes, timestampTime, 10 ) ) except ValueError: crash_time = 0 processor_notes.append( 'non-integer value of "CrashTime" (%s)' % raw_crash.CrashTime ) processed_crash.crash_time = crash_time if crash_time == submitted_timestamp_as_epoch: processor_notes.append("client_crash_date is unknown") # StartupTime: must have started up some time before crash try: startupTime = int(raw_crash.get('StartupTime', crash_time)) except ValueError: startupTime = 0 processor_notes.append('non-integer value of "StartupTime"') # InstallTime: must have installed some time before startup try: installTime = int(raw_crash.get('InstallTime', startupTime)) except ValueError: installTime = 0 processor_notes.append('non-integer value of "InstallTime"') processed_crash.client_crash_date = datetime.datetime.fromtimestamp( crash_time, UTC ) processed_crash.install_age = crash_time - installTime processed_crash.uptime = max(0, crash_time - startupTime) try: last_crash = int(raw_crash.SecondsSinceLastCrash) except (KeyError, TypeError, ValueError): last_crash = None processor_notes.append( 'non-integer value of "SecondsSinceLastCrash"' ) if last_crash > maxint: last_crash = None processor_notes.append( '"SecondsSinceLastCrash" larger than MAXINT - set to NULL' ) processed_crash.last_crash = last_crash return True
def processJob (self, jobTuple): """ This function is run only by a worker thread. Given a job, fetch a thread local database connection and the json document. Use these to create the record in the 'reports' table, then start the analysis of the dump file. input parameters: jobTuple: a tuple containing up to three items: the jobId (the primary key from the jobs table), the jobUuid (a unique string with the json file basename minus the extension) and the priority (an integer) """ if self.quit: return Processor.quit threadName = threading.currentThread().getName() try: threadLocalDatabaseConnection, threadLocalCursor = self.databaseConnectionPool.connectionCursorPair() threadLocalCrashStorage = self.crashStorePool.crashStorage(threadName) except sdb.exceptions_eligible_for_retry: logger.critical("something's gone horribly wrong with the database connection") sutil.reportExceptionAndContinue(logger, loggingLevel=logging.CRITICAL) return Processor.criticalError except hbc.FatalException: logger.critical("something's gone horribly wrong with the HBase connection") sutil.reportExceptionAndContinue(logger, loggingLevel=logging.CRITICAL) return Processor.criticalError except Exception: self.quit = True sutil.reportExceptionAndContinue(logger, loggingLevel=logging.CRITICAL) return Processor.quit try: self.quitCheck() newReportRecordAsDict = {} processorErrorMessages = [] jobId, jobUuid, jobPriority = jobTuple logger.info("starting job: %s", jobUuid) startedDateTime = self.nowFunc() threadLocalCursor.execute("update jobs set starteddatetime = %s where id = %s", (startedDateTime, jobId)) threadLocalDatabaseConnection.commit() jsonDocument = threadLocalCrashStorage.get_meta(jobUuid) self.config.logger.debug('about to apply rules') self.json_transform_rule_system.apply_all_rules(jsonDocument, self) self.config.logger.debug('done applying transform rules') try: date_processed = sdt.datetimeFromISOdateString(jsonDocument["submitted_timestamp"]) except KeyError: date_processed = ooid.dateFromOoid(jobUuid) newReportRecordAsDict = self.insertReportIntoDatabase(threadLocalCursor, jobUuid, jsonDocument, date_processed, processorErrorMessages) if jobUuid in self.priority_job_set: processorErrorMessages.append('Priority Job') self.priority_job_set.remove(jobUuid) threadLocalDatabaseConnection.commit() reportId = newReportRecordAsDict["id"] newReportRecordAsDict['dump'] = '' newReportRecordAsDict["startedDateTime"] = startedDateTime try: newReportRecordAsDict["ReleaseChannel"] = jsonDocument["ReleaseChannel"] except KeyError: newReportRecordAsDict["ReleaseChannel"] = 'unknown' if self.config.collectAddon: #logger.debug("collecting Addons") addonsAsAListOfTuples = self.insertAdddonsIntoDatabase(threadLocalCursor, reportId, jsonDocument, date_processed, processorErrorMessages) newReportRecordAsDict["addons"] = addonsAsAListOfTuples if self.config.collectCrashProcess: #logger.debug("collecting Crash Process") crashProcessAsDict = self.insertCrashProcess(threadLocalCursor, reportId, jsonDocument, date_processed, processorErrorMessages) newReportRecordAsDict.update( crashProcessAsDict ) try: dumpfilePathname = threadLocalCrashStorage.dumpPathForUuid(jobUuid, self.config.temporaryFileSystemStoragePath) #logger.debug('about to doBreakpadStackDumpAnalysis') isHang = 'hangid' in newReportRecordAsDict and bool(newReportRecordAsDict['hangid']) # hangType values: -1 if old style hang with hangid and Hang not present # else hangType == jsonDocument.Hang hangType = int(jsonDocument.get("Hang", -1 if isHang else 0)) java_stack_trace = jsonDocument.setdefault('JavaStackTrace', None) additionalReportValuesAsDict = self.doBreakpadStackDumpAnalysis(reportId, jobUuid, dumpfilePathname, hangType, java_stack_trace, threadLocalCursor, date_processed, processorErrorMessages) newReportRecordAsDict.update(additionalReportValuesAsDict) finally: newReportRecordAsDict["completeddatetime"] = completedDateTime = self.nowFunc() threadLocalCrashStorage.cleanUpTempDumpStorage(jobUuid, self.config.temporaryFileSystemStoragePath) #logger.debug('finished a job - cleanup') #finished a job - cleanup threadLocalCursor.execute("update jobs set completeddatetime = %s, success = %s where id = %s", (completedDateTime, newReportRecordAsDict['success'], jobId)) # Bug 519703: Collect setting for topmost source filename(s), addon compatibility check override, flash version reportsSql = """ update reports set signature = %%s, processor_notes = %%s, started_datetime = timestamp with time zone %%s, completed_datetime = timestamp with time zone %%s, success = %%s, truncated = %%s, topmost_filenames = %%s, addons_checked = %%s, flash_version = %%s where id = %s and date_processed = timestamp with time zone '%s' """ % (reportId,date_processed) #logger.debug("newReportRecordAsDict %s, %s", newReportRecordAsDict['topmost_filenames'], newReportRecordAsDict['flash_version']) #topmost_filenames = "|".join(jsonDocument.get('topmost_filenames',[])) topmost_filenames = "|".join(newReportRecordAsDict.get('topmost_filenames',[])) addons_checked = None try: addons_checked_txt = jsonDocument['EMCheckCompatibility'].lower() addons_checked = False if addons_checked_txt == 'true': addons_checked = True except KeyError: pass # leaving it as None if not in the document try: newReportRecordAsDict['Winsock_LSP'] = jsonDocument['Winsock_LSP'] except KeyError: pass # if it's not in the original json, it does get into the jsonz #flash_version = jsonDocument.get('flash_version') flash_version = newReportRecordAsDict.get('flash_version') processor_notes = '; '.join(processorErrorMessages) newReportRecordAsDict['processor_notes'] = processor_notes infoTuple = (newReportRecordAsDict['signature'], processor_notes, startedDateTime, completedDateTime, newReportRecordAsDict["success"], newReportRecordAsDict["truncated"], topmost_filenames, addons_checked, flash_version) #logger.debug("Updated report %s (%s): %s", reportId, jobUuid, str(infoTuple)) threadLocalCursor.execute(reportsSql, infoTuple) threadLocalDatabaseConnection.commit() self.saveProcessedDumpJson(newReportRecordAsDict, threadLocalCrashStorage) self.submitOoidToElasticSearch(jobUuid) if newReportRecordAsDict["success"]: logger.info("succeeded and committed: %s", jobUuid) else: logger.info("failed but committed: %s", jobUuid) self.quitCheck() return Processor.ok except (KeyboardInterrupt, SystemExit): logger.info("quit request detected") self.quit = True return Processor.quit except DuplicateEntryException, x: logger.warning("duplicate entry: %s", jobUuid) threadLocalCursor.execute('delete from jobs where id = %s', (jobId,)) threadLocalDatabaseConnection.commit() return Processor.ok
def get_list(self, **kwargs): """ List all crashes with a given signature and return them. Both `from_date` and `to_date` (and their aliases `from` and `to`) are required and can not be greater than 30 days apart. Optional arguments: see SearchCommon.get_parameters() """ # aliases if "from" in kwargs and "from_date" not in kwargs: kwargs["from_date"] = kwargs.get("from") if "to" in kwargs and "to_date" not in kwargs: kwargs["to_date"] = kwargs.get("to") if not kwargs.get('from_date'): raise MissingArgumentError('from_date') if not kwargs.get('to_date'): raise MissingArgumentError('to_date') from_date = datetimeutil.datetimeFromISOdateString(kwargs['from_date']) to_date = datetimeutil.datetimeFromISOdateString(kwargs['to_date']) span_days = (to_date - from_date).days if span_days > 30: raise BadArgumentError( 'Span between from_date and to_date can not be more than 30' ) # start with the default sort_order = { 'key': 'date_processed', 'direction': 'DESC' } if 'sort' in kwargs: sort_order['key'] = kwargs.pop('sort') _recognized_sort_orders = ( 'date_processed', 'uptime', 'user_comments', 'uuid', 'uuid_text', 'product', 'version', 'build', 'signature', 'url', 'os_name', 'os_version', 'cpu_name', 'cpu_info', 'address', 'reason', 'last_crash', 'install_age', 'hangid', 'process_type', 'release_channel', 'install_time', 'duplicate_of', ) if sort_order['key'] not in _recognized_sort_orders: raise BadArgumentError( '%s is not a recognized sort order key' % sort_order['key'] ) sort_order['direction'] = 'ASC' if 'reverse' in kwargs: if kwargs.pop('reverse'): sort_order['direction'] = 'DESC' include_raw_crash = kwargs.get('include_raw_crash') or False params = search_common.get_parameters(kwargs) if not params["signature"]: raise MissingArgumentError('signature') params["terms"] = params["signature"] params["search_mode"] = "is_exactly" # Default mode falls back to starts_with for postgres if params["plugin_search_mode"] == "default": params["plugin_search_mode"] = "starts_with" # Limiting to a signature if params["terms"]: params["terms"] = self.prepare_terms(params["terms"], params["search_mode"]) # Searching for terms in plugins if params["report_process"] == "plugin" and params["plugin_terms"]: params["plugin_terms"] = " ".join(params["plugin_terms"]) params["plugin_terms"] = self.prepare_terms( params["plugin_terms"], params["plugin_search_mode"] ) # Get information about the versions util_service = Util(config=self.context) params["versions_info"] = util_service.versions_info(**params) # Parsing the versions params["versions_string"] = params["versions"] (params["versions"], params["products"]) = self.parse_versions( params["versions"], params["products"] ) if hasattr(self.context, 'webapi'): context = self.context.webapi else: # old middleware context = self.context # Changing the OS ids to OS names for i, elem in enumerate(params["os"]): for platform in context.platforms: if platform["id"][:3] == elem[:3]: params["os"][i] = platform["name"] # Creating the parameters for the sql query sql_params = { } # Preparing the different parts of the sql query sql_select = """ SELECT r.date_processed, r.uptime, r.user_comments, r.uuid::uuid, r.uuid as uuid_text, r.product, r.version, r.build, r.signature, r.url, r.os_name, r.os_version, r.cpu_name, r.cpu_info, r.address, r.reason, r.last_crash, r.install_age, r.hangid, r.process_type, r.release_channel, (r.client_crash_date - (r.install_age * INTERVAL '1 second')) AS install_time """ if include_raw_crash: pass else: sql_select += """ , rd.duplicate_of """ wrapped_select = """ WITH report_slice AS ( %s ), dupes AS ( SELECT report_slice.uuid, rd.duplicate_of FROM reports_duplicates rd JOIN report_slice ON report_slice.uuid_text = rd.uuid WHERE rd.date_processed BETWEEN %%(from_date)s AND %%(to_date)s ) SELECT rs.*, dupes.duplicate_of, rc.raw_crash FROM report_slice rs LEFT OUTER JOIN dupes USING (uuid) LEFT OUTER JOIN raw_crashes rc ON rs.uuid = rc.uuid AND rc.date_processed BETWEEN %%(from_date)s AND %%(to_date)s """ sql_from = self.build_reports_sql_from(params) if not include_raw_crash: sql_from = """%s LEFT OUTER JOIN reports_duplicates rd ON r.uuid = rd.uuid """ % sql_from sql_where, sql_params = self.build_reports_sql_where( params, sql_params, self.context ) sql_order = """ ORDER BY %(key)s %(direction)s """ % sort_order sql_limit, sql_params = self.build_reports_sql_limit( params, sql_params ) # Assembling the query if include_raw_crash: sql_query = "\n".join(( "/* socorro.external.postgresql.report.Report.list */", sql_select, sql_from, sql_where, sql_order, sql_limit) ) else: sql_query = "\n".join(( "/* socorro.external.postgresql.report.Report.list */", sql_select, sql_from, sql_where, sql_order, sql_limit) ) # Query for counting the results sql_count_query = "\n".join(( "/* socorro.external.postgresql.report.Report.list */", "SELECT count(*)", sql_from, sql_where) ) # Querying the DB with self.get_connection() as connection: total = self.count( sql_count_query, sql_params, error_message="Failed to count crashes from reports.", connection=connection ) # No need to call Postgres if we know there will be no results if total: if include_raw_crash: sql_query = wrapped_select % sql_query results = self.query( sql_query, sql_params, error_message="Failed to retrieve crashes from reports", connection=connection ) else: results = [] # Transforming the results into what we want fields = ( "date_processed", "uptime", "user_comments", "uuid", "uuid", # the uuid::text one "product", "version", "build", "signature", "url", "os_name", "os_version", "cpu_name", "cpu_info", "address", "reason", "last_crash", "install_age", "hangid", "process_type", "release_channel", "install_time", "duplicate_of", ) if include_raw_crash: fields += ("raw_crash",) crashes = [] for row in results: crash = dict(zip(fields, row)) for i in crash: try: crash[i] = datetimeutil.date_to_string(crash[i]) except TypeError: pass crashes.append(crash) return { "hits": crashes, "total": total }
def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dump): """ This function is run only by a worker thread. Given a job, fetch a thread local database connection and the json document. Use these to create the record in the 'reports' table, then start the analysis of the dump file. input parameters: """ try: self.quit_check() crash_id = raw_crash.uuid processor_notes = [] processed_crash = DotDict() processed_crash.uuid = raw_crash.uuid processed_crash.success = False started_timestamp = self._log_job_start(crash_id) #self.config.logger.debug('about to apply rules') self.raw_crash_transform_rule_system.apply_all_rules( raw_crash, self) #self.config.logger.debug('done applying transform rules') try: submitted_timestamp = datetimeFromISOdateString( raw_crash.submitted_timestamp) except KeyError: submitted_timestamp = dateFromOoid(crash_id) # formerly the call to 'insertReportIntoDatabase' processed_crash_update = self._create_basic_processed_crash( crash_id, raw_crash, submitted_timestamp, started_timestamp, processor_notes) processed_crash.update(processed_crash_update) temp_dump_pathname = self._get_temp_dump_pathname( crash_id, raw_dump) try: #logger.debug('about to doBreakpadStackDumpAnalysis') processed_crash_update_dict = \ self._do_breakpad_stack_dump_analysis( crash_id, temp_dump_pathname, processed_crash.hang_type, processed_crash.java_stack_trace, submitted_timestamp, processor_notes ) processed_crash.update(processed_crash_update_dict) finally: self._cleanup_temp_file(temp_dump_pathname) processed_crash.topmost_filenames = "|".join( processed_crash.get('topmost_filenames', [])) try: processed_crash.Winsock_LSP = raw_crash.Winsock_LSP except KeyError: pass # if it's not in the original raw_crash, # it does get into the jsonz #except (KeyboardInterrupt, SystemExit): #self.config.logger.info("quit request detected") #raise except Exception, x: self.config.logger.warning('Error while processing %s: %s', crash_id, str(x), exc_info=True) processor_notes.append(str(x))