def testCreateNewOoid(self): new_ooid = ooid.create_new_ooid() ndate = ooid.dateFromOoid(new_ooid) ndepth = ooid.depthFromOoid(new_ooid) assert self.nowstamp == ndate assert ooid.defaultDepth == ndepth new_ooid = ooid.create_new_ooid(timestamp=self.xmas05) ndate = ooid.dateFromOoid(new_ooid) ndepth = ooid.depthFromOoid(new_ooid) assert self.xmas05 == ndate assert ooid.defaultDepth == ndepth for d in range(1, 5): ooid0 = ooid.create_new_ooid(depth=d) ooid1 = ooid.create_new_ooid(timestamp=self.xmas05, depth=d) ndate0 = ooid.dateFromOoid(ooid0) ndepth0 = ooid.depthFromOoid(ooid0) ndate1 = ooid.dateFromOoid(ooid1) ndepth1 = ooid.depthFromOoid(ooid1) assert self.nowstamp == ndate0 assert self.xmas05 == ndate1 assert ndepth0 == ndepth1 assert d == ndepth0 assert ooid.depthFromOoid(self.badooid0) is None assert ooid.depthFromOoid(self.badooid1) is None
def testCreateNewOoid(self): ooid = oo.createNewOoid() ndate = oo.dateFromOoid(ooid) ndepth = oo.depthFromOoid(ooid) assert self.nowstamp == ndate, 'Expect date of %s, got %s' %(self.nowstamp,ndate) assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth) ooid = oo.createNewOoid(timestamp=self.xmas05) ndate = oo.dateFromOoid(ooid) ndepth = oo.depthFromOoid(ooid) assert self.xmas05 == ndate, 'Expect date of %s, got %s' %(self.xmas05,ndate) assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth) for d in range(1,5): ooid0 = oo.createNewOoid(depth=d) ooid1 = oo.createNewOoid(timestamp=self.xmas05,depth=d) ndate0 = oo.dateFromOoid(ooid0) ndepth0 = oo.depthFromOoid(ooid0) ndate1 = oo.dateFromOoid(ooid1) ndepth1 = oo.depthFromOoid(ooid1) assert self.nowstamp == ndate0, 'Expect date of %s, got %s' %(self.nowstamp,ndate0) assert self.xmas05 == ndate1, 'Expect date of %s, got %s' %(self.xmas05,ndate1) assert ndepth0 == ndepth1, 'Expect depth0(%d) == depth1(%d)' %(ndepth0,ndepth1) assert d == ndepth0, 'Expect depth %d, got %d' % (d,ndepth0) assert None == oo.depthFromOoid(self.badooid0) assert None == oo.depthFromOoid(self.badooid1)
def testGetDate(self): for ooid in self.yyyyoids: assert self.baseDate == oo.dateFromOoid( ooid), 'Expected %s got %s' % (self.baseDate, oo.dateFromOoid(ooid)) assert 4 == oo.depthFromOoid( ooid), 'Expected %d, got %d' % (4, oo.depthFromOoid(ooid)) assert None == oo.dateFromOoid(self.badooid0) assert None == oo.dateFromOoid(self.badooid1)
def build_keys(self, prefix, name_of_thing, id): """Use S3 pseudo-directories to make it easier to list/expire. For "raw_crash" things, the id is an ooid/crash_id. This uses the first three characters of the ooid for entropy and extracts the crash submission date from the last 6 charactes. Then it mooshes that all together into this structure:: {prefix}/v2/{name_of_thing}/{entropy}/{date}/{id} This makes it possible to list all the raw_crashes submitted on a given date. It also returns the keys that KeyBuilderBase builds so that we can fetch objects that were saved with older keys. """ keys = [] if name_of_thing == 'raw_crash': datestamp = dateFromOoid(id) if datestamp is not None: # We insert the first 3 chars of the ooid/crash_id providing # some entropy earlier in the key so that consecutive s3 # requests get distributed across multiple s3 partitions. first_chars = id[:3] date = datestamp.strftime('%Y%m%d') keys.append('%s/v2/%s/%s/%s/%s' % (prefix, name_of_thing, first_chars, date, id)) keys.extend( super(DatePrefixKeyBuilder, self).build_keys(prefix, name_of_thing, id)) return keys
def _get_datestamp(self, crashid): """Retrieves the datestamp from a crashid or raises an exception""" datestamp = dateFromOoid(crashid) if datestamp is None: # We should never hit this situation unless the crashid is a bad crashid raise CrashidMissingDatestamp('%s is missing datestamp' % crashid) return datestamp
def processJob (self, jobTuple): """ This function is run only by a worker thread. Given a job, fetch a thread local database connection and the json document. Use these to create the record in the 'reports' table, then start the analysis of the dump file. input parameters: jobTuple: a tuple containing up to three items: the jobId (the primary key from the jobs table), the jobUuid (a unique string with the json file basename minus the extension) and the priority (an integer) """ threadName = threading.currentThread().getName() try: threadLocalDatabaseConnection, threadLocalCursor = self.databaseConnectionPool.connectionCursorPair() except: self.quit = True socorro.lib.util.reportExceptionAndAbort(logger) # can't continue without a database connection try: processorErrorMessages = [] jobId, jobUuid, jobPriority = jobTuple logger.info("%s - starting job: %s, %s", threadName, jobId, jobUuid) startedDateTime = datetime.datetime.now() threadLocalCursor.execute("update jobs set starteddatetime = %s where id = %s", (startedDateTime, jobId)) threadLocalDatabaseConnection.commit() jobPathname = self.jsonPathForUuidInJsonDumpStorage(jobUuid) dumpfilePathname = self.dumpPathForUuidInJsonDumpStorage(jobUuid) jsonFile = open(jobPathname) try: jsonDocument = simplejson.load(jsonFile) finally: jsonFile.close() try: date_processed = sdt.datetimeFromISOdateString(jsonDocument["submitted_timestamp"]) except KeyError: date_processed = ooid.dateFromOoid(jobUuid) reportId = self.insertReportIntoDatabase(threadLocalCursor, jobUuid, jsonDocument, jobPathname, date_processed, processorErrorMessages) threadLocalDatabaseConnection.commit() truncated = self.doBreakpadStackDumpAnalysis(reportId, jobUuid, dumpfilePathname, threadLocalCursor, date_processed, processorErrorMessages) self.quitCheck() #finished a job - cleanup threadLocalCursor.execute("update jobs set completeddatetime = %s, success = True where id = %s", (datetime.datetime.now(), jobId)) threadLocalCursor.execute("update reports set started_datetime = timestamp without time zone '%s', completed_datetime = timestamp without time zone '%s', success = True, truncated = %s where id = %s and date_processed = timestamp without time zone '%s'" % (startedDateTime, datetime.datetime.now(), truncated, reportId, date_processed)) #self.updateRegistrationNoCommit(threadLocalCursor) threadLocalDatabaseConnection.commit() logger.info("%s - succeeded and committed: %s, %s", threadName, jobId, jobUuid) except (KeyboardInterrupt, SystemExit): logger.info("%s - quit request detected", threadName) self.quit = True try: logger.info("%s - abandoning job with rollback: %s, %s", threadName, jobId, jobUuid) threadLocalDatabaseConnection.rollback() threadLocalDatabaseConnection.close() except: pass except DuplicateEntryException, x: logger.warning("%s - duplicate entry: %s", threadName, jobUuid)
def build_keys(self, prefix, name_of_thing, id): """return a list of one key. The reason for not returning more than one is that this key builder class is going to be used for something new so it has no legacy.""" datestamp = dateFromOoid(id) if datestamp is None: # the id did not have a date component in it datestamp = datetime.datetime.utcnow() date = datestamp.strftime('%Y%m%d') keys = ['%s/v1/%s/%s/%s' % (prefix, name_of_thing, date, id)] return keys
def dailyPart(self, ooid, timestamp=None): """ return YYYYMMDD use the timestamp if any, else the ooid's last 6 chars if reasonable, else now() """ year,month,day = None,None,None if not timestamp: timestamp = socorro_ooid.dateFromOoid(ooid) if not timestamp: timestamp = utc_now() (year,month,day) = (timestamp.year,timestamp.month,timestamp.day) return "%4d%02d%02d"%(year,month,day)
def _get_base(self, crash_id): """this method overrides the base method to define the daily file system root directory name. While the default class is to use a YYYYMMDD form, this class substitutes a simple DD form. This is the mechanism of directory recycling as at the first day of a new month we return to the same directiory structures that were created on the first day of the previous month""" date = dateFromOoid(crash_id) if not date: date = utc_now() date_formatted = "%02d" % (date.day,) return [self.config.fs_root, date_formatted]
def _get_base(self, crash_id): """this method overrides the base method to define the daily file system root directory name. While the default class is to use a YYYYMMDD form, this class substitutes a simple DD form. This is the mechanism of directory recycling as at the first day of a new month we return to the same directiory structures that were created on the first day of the previous month""" date = dateFromOoid(crash_id) if not date: date = utc_now() date_formatted = "%02d" % (date.day, ) return [self.config.fs_root, date_formatted]
def build_keys(self, prefix, name_of_thing, id): """return a list of one key. The reason for not returning more than one is that this key builder class is going to be used for something new so it has no legacy.""" datestamp = dateFromOoid(id) if datestamp is None: # the id did not have a date component in it datestamp = datetime.datetime.utcnow() date = datestamp.strftime('%Y%m%d') keys = [ '%s/v1/%s/%s/%s' % ( prefix, name_of_thing, date, id ) ] return keys
def newEntry(self, ooid, timestamp=None, webheadName=None): """ Sets up the name and date storage directory branches for the given ooid. Creates any needed directories along the path to the appropriate storage location. Sets gid and mode if specified Creates one symbolic link in the date leaf directory with name ooid and referencing the name leaf directory returns (nameDir,dateDir) """ if not timestamp: timestamp = socorro_ooid.dateFromOoid(ooid) if not timestamp: timestamp = utc_now() if not self.osModule.path.isdir(self.root): um = self.osModule.umask(0) try: self.osModule.mkdir(self.root, self.dirPermissions) finally: self.osModule.umask(um) nameDir, nparts = self.makeNameDir(ooid, timestamp) dateDir, dparts = self.makeDateDir(timestamp, webheadName) # adjust the current subslot only when inserting a new entry if self.subSlotCount: k = dparts[-1].split('_')[0] curcount = self.currentSubSlots.setdefault(k, 0) self.currentSubSlots[k] = (curcount + 1) % self.subSlotCount parts = [ os.path.pardir, ] * (len(dparts) - 2) # lose root / dailypart parts.append(self.indexName) parts.extend(self.relativeNameParts(ooid)) relNameDir = os.sep.join(parts) try: self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid)) except OSError, x: if errno.ENOENT == x.errno: # maybe another thread cleaned this out from under us. Try again nameDir = self.makeNameDir(ooid) # might be overkill, # but cheap insurance dateDir = self.makeDateDir(timestamp) self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid)) elif errno.EEXIST == x.errno: self.osModule.unlink(os.path.join(dateDir, ooid)) self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid)) else: raise
def newEntry(self, ooid, timestamp=None, webheadName=None): """ Sets up the name and date storage directory branches for the given ooid. Creates any needed directories along the path to the appropriate storage location. Sets gid and mode if specified Creates one symbolic link in the date leaf directory with name ooid and referencing the name leaf directory returns (nameDir,dateDir) """ if not timestamp: timestamp = socorro_ooid.dateFromOoid(ooid) if not timestamp: timestamp = utc_now() if not self.osModule.path.isdir(self.root): um = self.osModule.umask(0) try: self.osModule.mkdir(self.root, self.dirPermissions) finally: self.osModule.umask(um) nameDir, nparts = self.makeNameDir(ooid, timestamp) dateDir, dparts = self.makeDateDir(timestamp, webheadName) # adjust the current subslot only when inserting a new entry if self.subSlotCount: k = dparts[-1].split('_')[0] curcount = self.currentSubSlots.setdefault(k, 0) self.currentSubSlots[k] = (curcount + 1) % self.subSlotCount parts = [os.path.pardir, ] * (len(dparts) - 2) # lose root / dailypart parts.append(self.indexName) parts.extend(self.relativeNameParts(ooid)) relNameDir = os.sep.join(parts) try: self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid)) except OSError, x: if errno.ENOENT == x.errno: # maybe another thread cleaned this out from under us. Try again nameDir = self.makeNameDir(ooid) # might be overkill, # but cheap insurance dateDir = self.makeDateDir(timestamp) self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid)) elif errno.EEXIST == x.errno: self.osModule.unlink(os.path.join(dateDir, ooid)) self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid)) else: raise
def lookupOoidInDatePath(self, date, ooid, webheadName=None): """Look for the date path holding a symbolic link named 'ooid', return datePath, dateParts on failure return None,[]""" if not date: date = socorro_ooid.dateFromOoid(ooid) if date: datePath, dateParts = self.datePath(date, webheadName) if self.osModule.path.exists(os.path.join(datePath, ooid)): return datePath, dateParts for d in self.osModule.listdir(self.root): # We don't know webhead if any, so avoid confusion by looking # everywhere for dir, dirs, files in os.walk( os.sep.join((self.root, d, self.dateName)) ): if ooid in dirs or ooid in files: # probably dirs dirPath = dir dirParts = dir.split(os.sep) return dirPath, dirParts return None, []
def build_keys(self, prefix, name_of_thing, id): """Use S3 pseudo-directories to make it easier to list/expire. For "raw_crash" things, the id is an ooid/crash_id. This uses the first three characters of the ooid for entropy and extracts the crash submission date from the last 6 charactes. Then it mooshes that all together into this structure:: {prefix}/v2/{name_of_thing}/{entropy}/{date}/{id} This makes it possible to list all the raw_crashes submitted on a given date. It also returns the keys that KeyBuilderBase builds so that we can fetch objects that were saved with older keys. """ keys = [] if name_of_thing == 'raw_crash': datestamp = dateFromOoid(id) if datestamp is not None: # We insert the first 3 chars of the ooid/crash_id providing # some entropy earlier in the key so that consecutive s3 # requests get distributed across multiple s3 partitions. first_chars = id[:3] date = datestamp.strftime('%Y%m%d') keys.append( '%s/v2/%s/%s/%s/%s' % ( prefix, name_of_thing, first_chars, date, id ) ) keys.extend( super(DatePrefixKeyBuilder, self).build_keys( prefix, name_of_thing, id ) ) return keys
def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dump): """ This function is run only by a worker thread. Given a job, fetch a thread local database connection and the json document. Use these to create the record in the 'reports' table, then start the analysis of the dump file. input parameters: """ try: self.quit_check() crash_id = raw_crash.uuid processor_notes = [] processed_crash = DotDict() processed_crash.uuid = raw_crash.uuid processed_crash.success = False started_timestamp = self._log_job_start(crash_id) #self.config.logger.debug('about to apply rules') self.raw_crash_transform_rule_system.apply_all_rules( raw_crash, self) #self.config.logger.debug('done applying transform rules') try: submitted_timestamp = datetimeFromISOdateString( raw_crash.submitted_timestamp) except KeyError: submitted_timestamp = dateFromOoid(crash_id) # formerly the call to 'insertReportIntoDatabase' processed_crash_update = self._create_basic_processed_crash( crash_id, raw_crash, submitted_timestamp, started_timestamp, processor_notes) processed_crash.update(processed_crash_update) temp_dump_pathname = self._get_temp_dump_pathname( crash_id, raw_dump) try: #logger.debug('about to doBreakpadStackDumpAnalysis') processed_crash_update_dict = \ self._do_breakpad_stack_dump_analysis( crash_id, temp_dump_pathname, processed_crash.hang_type, processed_crash.java_stack_trace, submitted_timestamp, processor_notes ) processed_crash.update(processed_crash_update_dict) finally: self._cleanup_temp_file(temp_dump_pathname) processed_crash.topmost_filenames = "|".join( processed_crash.get('topmost_filenames', [])) try: processed_crash.Winsock_LSP = raw_crash.Winsock_LSP except KeyError: pass # if it's not in the original raw_crash, # it does get into the jsonz #except (KeyboardInterrupt, SystemExit): #self.config.logger.info("quit request detected") #raise except Exception, x: self.config.logger.warning('Error while processing %s: %s', crash_id, str(x), exc_info=True) processor_notes.append(str(x))
def _get_base(self, crash_id): date = dateFromOoid(crash_id) if not date: date = utc_now() date_formatted = "%4d%02d%02d" % (date.year, date.month, date.day) return [self.config.fs_root, date_formatted]
def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta): processor_notes = processor_meta.processor_notes processed_crash.submitted_timestamp = raw_crash.get( 'submitted_timestamp', dateFromOoid(raw_crash.uuid)) if isinstance(processed_crash.submitted_timestamp, basestring): processed_crash.submitted_timestamp = datetime_from_isodate_string( processed_crash.submitted_timestamp) processed_crash.date_processed = processed_crash.submitted_timestamp # defaultCrashTime: must have crashed before date processed submitted_timestamp_as_epoch = int( time.mktime(processed_crash.submitted_timestamp.timetuple())) try: timestampTime = int( raw_crash.get('timestamp', submitted_timestamp_as_epoch) ) # the old name for crash time except ValueError: timestampTime = 0 processor_notes.append('non-integer value of "timestamp"') try: crash_time = int( self._get_truncate_or_warn(raw_crash, 'CrashTime', processor_notes, timestampTime, 10)) except ValueError: crash_time = 0 processor_notes.append('non-integer value of "CrashTime" (%s)' % raw_crash.CrashTime) processed_crash.crash_time = crash_time if crash_time == submitted_timestamp_as_epoch: processor_notes.append("client_crash_date is unknown") # StartupTime: must have started up some time before crash try: startupTime = int(raw_crash.get('StartupTime', crash_time)) except ValueError: startupTime = 0 processor_notes.append('non-integer value of "StartupTime"') # InstallTime: must have installed some time before startup try: installTime = int(raw_crash.get('InstallTime', startupTime)) except ValueError: installTime = 0 processor_notes.append('non-integer value of "InstallTime"') processed_crash.client_crash_date = datetime.datetime.fromtimestamp( crash_time, UTC) processed_crash.install_age = crash_time - installTime processed_crash.uptime = max(0, crash_time - startupTime) try: last_crash = int(raw_crash.SecondsSinceLastCrash) except (KeyError, TypeError, ValueError): last_crash = None processor_notes.append( 'non-integer value of "SecondsSinceLastCrash"') if last_crash > maxint: last_crash = None processor_notes.append( '"SecondsSinceLastCrash" larger than MAXINT - set to NULL') processed_crash.last_crash = last_crash return True
def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta): processor_notes = processor_meta.processor_notes processed_crash.submitted_timestamp = raw_crash.get( 'submitted_timestamp', dateFromOoid(raw_crash.uuid) ) if isinstance(processed_crash.submitted_timestamp, basestring): processed_crash.submitted_timestamp = datetime_from_isodate_string( processed_crash.submitted_timestamp ) processed_crash.date_processed = processed_crash.submitted_timestamp # defaultCrashTime: must have crashed before date processed submitted_timestamp_as_epoch = int( time.mktime(processed_crash.submitted_timestamp.timetuple()) ) try: timestampTime = int( raw_crash.get('timestamp', submitted_timestamp_as_epoch) ) # the old name for crash time except ValueError: timestampTime = 0 processor_notes.append('non-integer value of "timestamp"') try: crash_time = int( self._get_truncate_or_warn( raw_crash, 'CrashTime', processor_notes, timestampTime, 10 ) ) except ValueError: crash_time = 0 processor_notes.append( 'non-integer value of "CrashTime" (%s)' % raw_crash.CrashTime ) processed_crash.crash_time = crash_time if crash_time == submitted_timestamp_as_epoch: processor_notes.append("client_crash_date is unknown") # StartupTime: must have started up some time before crash try: startupTime = int(raw_crash.get('StartupTime', crash_time)) except ValueError: startupTime = 0 processor_notes.append('non-integer value of "StartupTime"') # InstallTime: must have installed some time before startup try: installTime = int(raw_crash.get('InstallTime', startupTime)) except ValueError: installTime = 0 processor_notes.append('non-integer value of "InstallTime"') processed_crash.client_crash_date = datetime.datetime.fromtimestamp( crash_time, UTC ) processed_crash.install_age = crash_time - installTime processed_crash.uptime = max(0, crash_time - startupTime) try: last_crash = int(raw_crash.SecondsSinceLastCrash) except (KeyError, TypeError, ValueError): last_crash = None processor_notes.append( 'non-integer value of "SecondsSinceLastCrash"' ) if last_crash > maxint: last_crash = None processor_notes.append( '"SecondsSinceLastCrash" larger than MAXINT - set to NULL' ) processed_crash.last_crash = last_crash return True
def testGetDate(self): for ooid in self.yyyyoids: assert self.baseDate == oo.dateFromOoid(ooid), 'Expected %s got %s' %(self.baseDate, oo.dateFromOoid(ooid)) assert 4 == oo.depthFromOoid(ooid), 'Expected %d, got %d' %(4, oo.depthFromOoid(ooid)) assert None == oo.dateFromOoid(self.badooid0) assert None == oo.dateFromOoid(self.badooid1)
def testGetDate(self): for this_ooid in self.yyyyoids: assert self.baseDate == ooid.dateFromOoid(this_ooid) assert 4 == ooid.depthFromOoid(this_ooid) assert ooid.dateFromOoid(self.badooid0) is None assert ooid.dateFromOoid(self.badooid1) is None
def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dump): """ This function is run only by a worker thread. Given a job, fetch a thread local database connection and the json document. Use these to create the record in the 'reports' table, then start the analysis of the dump file. input parameters: """ try: self.quit_check() crash_id = raw_crash.uuid processor_notes = [] processed_crash = DotDict() processed_crash.uuid = raw_crash.uuid processed_crash.success = False started_timestamp = self._log_job_start(crash_id) #self.config.logger.debug('about to apply rules') self.raw_crash_transform_rule_system.apply_all_rules(raw_crash, self) #self.config.logger.debug('done applying transform rules') try: submitted_timestamp = datetimeFromISOdateString( raw_crash.submitted_timestamp ) except KeyError: submitted_timestamp = dateFromOoid(crash_id) # formerly the call to 'insertReportIntoDatabase' processed_crash_update = self._create_basic_processed_crash( crash_id, raw_crash, submitted_timestamp, started_timestamp, processor_notes ) processed_crash.update(processed_crash_update) temp_dump_pathname = self._get_temp_dump_pathname( crash_id, raw_dump ) try: #logger.debug('about to doBreakpadStackDumpAnalysis') processed_crash_update_dict = \ self._do_breakpad_stack_dump_analysis( crash_id, temp_dump_pathname, processed_crash.hang_type, processed_crash.java_stack_trace, submitted_timestamp, processor_notes ) processed_crash.update(processed_crash_update_dict) finally: self._cleanup_temp_file(temp_dump_pathname) processed_crash.topmost_filenames = "|".join( processed_crash.get('topmost_filenames', []) ) try: processed_crash.Winsock_LSP = raw_crash.Winsock_LSP except KeyError: pass # if it's not in the original raw_crash, # it does get into the jsonz #except (KeyboardInterrupt, SystemExit): #self.config.logger.info("quit request detected") #raise except Exception, x: self.config.logger.warning( 'Error while processing %s: %s', crash_id, str(x), exc_info=True ) processor_notes.append(str(x))
def processJob (self, jobTuple): """ This function is run only by a worker thread. Given a job, fetch a thread local database connection and the json document. Use these to create the record in the 'reports' table, then start the analysis of the dump file. input parameters: jobTuple: a tuple containing up to three items: the jobId (the primary key from the jobs table), the jobUuid (a unique string with the json file basename minus the extension) and the priority (an integer) """ if self.quit: return Processor.quit threadName = threading.currentThread().getName() try: threadLocalDatabaseConnection, threadLocalCursor = self.databaseConnectionPool.connectionCursorPair() threadLocalCrashStorage = self.crashStorePool.crashStorage(threadName) except sdb.exceptions_eligible_for_retry: logger.critical("something's gone horribly wrong with the database connection") sutil.reportExceptionAndContinue(logger, loggingLevel=logging.CRITICAL) return Processor.criticalError except hbc.FatalException: logger.critical("something's gone horribly wrong with the HBase connection") sutil.reportExceptionAndContinue(logger, loggingLevel=logging.CRITICAL) return Processor.criticalError except Exception: self.quit = True sutil.reportExceptionAndContinue(logger, loggingLevel=logging.CRITICAL) return Processor.quit try: self.quitCheck() newReportRecordAsDict = {} processorErrorMessages = [] jobId, jobUuid, jobPriority = jobTuple logger.info("starting job: %s", jobUuid) startedDateTime = self.nowFunc() threadLocalCursor.execute("update jobs set starteddatetime = %s where id = %s", (startedDateTime, jobId)) threadLocalDatabaseConnection.commit() jsonDocument = threadLocalCrashStorage.get_meta(jobUuid) self.config.logger.debug('about to apply rules') self.json_transform_rule_system.apply_all_rules(jsonDocument, self) self.config.logger.debug('done applying transform rules') try: date_processed = sdt.datetimeFromISOdateString(jsonDocument["submitted_timestamp"]) except KeyError: date_processed = ooid.dateFromOoid(jobUuid) newReportRecordAsDict = self.insertReportIntoDatabase(threadLocalCursor, jobUuid, jsonDocument, date_processed, processorErrorMessages) if jobUuid in self.priority_job_set: processorErrorMessages.append('Priority Job') self.priority_job_set.remove(jobUuid) threadLocalDatabaseConnection.commit() reportId = newReportRecordAsDict["id"] newReportRecordAsDict['dump'] = '' newReportRecordAsDict["startedDateTime"] = startedDateTime try: newReportRecordAsDict["ReleaseChannel"] = jsonDocument["ReleaseChannel"] except KeyError: newReportRecordAsDict["ReleaseChannel"] = 'unknown' if self.config.collectAddon: #logger.debug("collecting Addons") addonsAsAListOfTuples = self.insertAdddonsIntoDatabase(threadLocalCursor, reportId, jsonDocument, date_processed, processorErrorMessages) newReportRecordAsDict["addons"] = addonsAsAListOfTuples if self.config.collectCrashProcess: #logger.debug("collecting Crash Process") crashProcessAsDict = self.insertCrashProcess(threadLocalCursor, reportId, jsonDocument, date_processed, processorErrorMessages) newReportRecordAsDict.update( crashProcessAsDict ) try: dumpfilePathname = threadLocalCrashStorage.dumpPathForUuid(jobUuid, self.config.temporaryFileSystemStoragePath) #logger.debug('about to doBreakpadStackDumpAnalysis') isHang = 'hangid' in newReportRecordAsDict and bool(newReportRecordAsDict['hangid']) # hangType values: -1 if old style hang with hangid and Hang not present # else hangType == jsonDocument.Hang hangType = int(jsonDocument.get("Hang", -1 if isHang else 0)) java_stack_trace = jsonDocument.setdefault('JavaStackTrace', None) additionalReportValuesAsDict = self.doBreakpadStackDumpAnalysis(reportId, jobUuid, dumpfilePathname, hangType, java_stack_trace, threadLocalCursor, date_processed, processorErrorMessages) newReportRecordAsDict.update(additionalReportValuesAsDict) finally: newReportRecordAsDict["completeddatetime"] = completedDateTime = self.nowFunc() threadLocalCrashStorage.cleanUpTempDumpStorage(jobUuid, self.config.temporaryFileSystemStoragePath) #logger.debug('finished a job - cleanup') #finished a job - cleanup threadLocalCursor.execute("update jobs set completeddatetime = %s, success = %s where id = %s", (completedDateTime, newReportRecordAsDict['success'], jobId)) # Bug 519703: Collect setting for topmost source filename(s), addon compatibility check override, flash version reportsSql = """ update reports set signature = %%s, processor_notes = %%s, started_datetime = timestamp with time zone %%s, completed_datetime = timestamp with time zone %%s, success = %%s, truncated = %%s, topmost_filenames = %%s, addons_checked = %%s, flash_version = %%s where id = %s and date_processed = timestamp with time zone '%s' """ % (reportId,date_processed) #logger.debug("newReportRecordAsDict %s, %s", newReportRecordAsDict['topmost_filenames'], newReportRecordAsDict['flash_version']) #topmost_filenames = "|".join(jsonDocument.get('topmost_filenames',[])) topmost_filenames = "|".join(newReportRecordAsDict.get('topmost_filenames',[])) addons_checked = None try: addons_checked_txt = jsonDocument['EMCheckCompatibility'].lower() addons_checked = False if addons_checked_txt == 'true': addons_checked = True except KeyError: pass # leaving it as None if not in the document try: newReportRecordAsDict['Winsock_LSP'] = jsonDocument['Winsock_LSP'] except KeyError: pass # if it's not in the original json, it does get into the jsonz #flash_version = jsonDocument.get('flash_version') flash_version = newReportRecordAsDict.get('flash_version') processor_notes = '; '.join(processorErrorMessages) newReportRecordAsDict['processor_notes'] = processor_notes infoTuple = (newReportRecordAsDict['signature'], processor_notes, startedDateTime, completedDateTime, newReportRecordAsDict["success"], newReportRecordAsDict["truncated"], topmost_filenames, addons_checked, flash_version) #logger.debug("Updated report %s (%s): %s", reportId, jobUuid, str(infoTuple)) threadLocalCursor.execute(reportsSql, infoTuple) threadLocalDatabaseConnection.commit() self.saveProcessedDumpJson(newReportRecordAsDict, threadLocalCrashStorage) self.submitOoidToElasticSearch(jobUuid) if newReportRecordAsDict["success"]: logger.info("succeeded and committed: %s", jobUuid) else: logger.info("failed but committed: %s", jobUuid) self.quitCheck() return Processor.ok except (KeyboardInterrupt, SystemExit): logger.info("quit request detected") self.quit = True return Processor.quit except DuplicateEntryException, x: logger.warning("duplicate entry: %s", jobUuid) threadLocalCursor.execute('delete from jobs where id = %s', (jobId,)) threadLocalDatabaseConnection.commit() return Processor.ok