def testCreateNewOoid(self):
new_ooid = ooid.create_new_ooid()
ndate = ooid.dateFromOoid(new_ooid)
ndepth = ooid.depthFromOoid(new_ooid)
assert self.nowstamp == ndate
assert ooid.defaultDepth == ndepth
new_ooid = ooid.create_new_ooid(timestamp=self.xmas05)
ndate = ooid.dateFromOoid(new_ooid)
ndepth = ooid.depthFromOoid(new_ooid)
assert self.xmas05 == ndate
assert ooid.defaultDepth == ndepth
for d in range(1, 5):
ooid0 = ooid.create_new_ooid(depth=d)
ooid1 = ooid.create_new_ooid(timestamp=self.xmas05, depth=d)
ndate0 = ooid.dateFromOoid(ooid0)
ndepth0 = ooid.depthFromOoid(ooid0)
ndate1 = ooid.dateFromOoid(ooid1)
ndepth1 = ooid.depthFromOoid(ooid1)
assert self.nowstamp == ndate0
assert self.xmas05 == ndate1
assert ndepth0 == ndepth1
assert d == ndepth0
assert ooid.depthFromOoid(self.badooid0) is None
assert ooid.depthFromOoid(self.badooid1) is None
def testCreateNewOoid(self):
new_ooid = ooid.create_new_ooid()
ndate = ooid.dateFromOoid(new_ooid)
ndepth = ooid.depthFromOoid(new_ooid)
assert self.nowstamp == ndate
assert ooid.defaultDepth == ndepth
new_ooid = ooid.create_new_ooid(timestamp=self.xmas05)
ndate = ooid.dateFromOoid(new_ooid)
ndepth = ooid.depthFromOoid(new_ooid)
assert self.xmas05 == ndate
assert ooid.defaultDepth == ndepth
for d in range(1, 5):
ooid0 = ooid.create_new_ooid(depth=d)
ooid1 = ooid.create_new_ooid(timestamp=self.xmas05, depth=d)
ndate0 = ooid.dateFromOoid(ooid0)
ndepth0 = ooid.depthFromOoid(ooid0)
ndate1 = ooid.dateFromOoid(ooid1)
ndepth1 = ooid.depthFromOoid(ooid1)
assert self.nowstamp == ndate0
assert self.xmas05 == ndate1
assert ndepth0 == ndepth1
assert d == ndepth0
assert ooid.depthFromOoid(self.badooid0) is None
assert ooid.depthFromOoid(self.badooid1) is None
def testCreateNewOoid(self):
ooid = oo.createNewOoid()
ndate = oo.dateFromOoid(ooid)
ndepth = oo.depthFromOoid(ooid)
assert self.nowstamp == ndate, 'Expect date of %s, got %s' %(self.nowstamp,ndate)
assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth)
ooid = oo.createNewOoid(timestamp=self.xmas05)
ndate = oo.dateFromOoid(ooid)
ndepth = oo.depthFromOoid(ooid)
assert self.xmas05 == ndate, 'Expect date of %s, got %s' %(self.xmas05,ndate)
assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth)
for d in range(1,5):
ooid0 = oo.createNewOoid(depth=d)
ooid1 = oo.createNewOoid(timestamp=self.xmas05,depth=d)
ndate0 = oo.dateFromOoid(ooid0)
ndepth0 = oo.depthFromOoid(ooid0)
ndate1 = oo.dateFromOoid(ooid1)
ndepth1 = oo.depthFromOoid(ooid1)
assert self.nowstamp == ndate0, 'Expect date of %s, got %s' %(self.nowstamp,ndate0)
assert self.xmas05 == ndate1, 'Expect date of %s, got %s' %(self.xmas05,ndate1)
assert ndepth0 == ndepth1, 'Expect depth0(%d) == depth1(%d)' %(ndepth0,ndepth1)
assert d == ndepth0, 'Expect depth %d, got %d' % (d,ndepth0)
assert None == oo.depthFromOoid(self.badooid0)
assert None == oo.depthFromOoid(self.badooid1)
def testCreateNewOoid(self):
ooid = oo.createNewOoid()
ndate = oo.dateFromOoid(ooid)
ndepth = oo.depthFromOoid(ooid)
assert self.nowstamp == ndate, 'Expect date of %s, got %s' %(self.nowstamp,ndate)
assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth)
ooid = oo.createNewOoid(timestamp=self.xmas05)
ndate = oo.dateFromOoid(ooid)
ndepth = oo.depthFromOoid(ooid)
assert self.xmas05 == ndate, 'Expect date of %s, got %s' %(self.xmas05,ndate)
assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth)
for d in range(1,5):
ooid0 = oo.createNewOoid(depth=d)
ooid1 = oo.createNewOoid(timestamp=self.xmas05,depth=d)
ndate0 = oo.dateFromOoid(ooid0)
ndepth0 = oo.depthFromOoid(ooid0)
ndate1 = oo.dateFromOoid(ooid1)
ndepth1 = oo.depthFromOoid(ooid1)
assert self.nowstamp == ndate0, 'Expect date of %s, got %s' %(self.nowstamp,ndate0)
assert self.xmas05 == ndate1, 'Expect date of %s, got %s' %(self.xmas05,ndate1)
assert ndepth0 == ndepth1, 'Expect depth0(%d) == depth1(%d)' %(ndepth0,ndepth1)
assert d == ndepth0, 'Expect depth %d, got %d' % (d,ndepth0)
assert None == oo.depthFromOoid(self.badooid0)
assert None == oo.depthFromOoid(self.badooid1)
def testGetDate(self):
for ooid in self.yyyyoids:
assert self.baseDate == oo.dateFromOoid(
ooid), 'Expected %s got %s' % (self.baseDate,
oo.dateFromOoid(ooid))
assert 4 == oo.depthFromOoid(
ooid), 'Expected %d, got %d' % (4, oo.depthFromOoid(ooid))
assert None == oo.dateFromOoid(self.badooid0)
assert None == oo.dateFromOoid(self.badooid1)
def build_keys(self, prefix, name_of_thing, id):
"""Use S3 pseudo-directories to make it easier to list/expire.
For "raw_crash" things, the id is an ooid/crash_id. This uses the
first three characters of the ooid for entropy and extracts the
crash submission date from the last 6 charactes. Then it mooshes
that all together into this structure::
{prefix}/v2/{name_of_thing}/{entropy}/{date}/{id}
This makes it possible to list all the raw_crashes submitted on a
given date.
It also returns the keys that KeyBuilderBase builds so that we can
fetch objects that were saved with older keys.
"""
keys = []
if name_of_thing == 'raw_crash':
datestamp = dateFromOoid(id)
if datestamp is not None:
# We insert the first 3 chars of the ooid/crash_id providing
# some entropy earlier in the key so that consecutive s3
# requests get distributed across multiple s3 partitions.
first_chars = id[:3]
date = datestamp.strftime('%Y%m%d')
keys.append('%s/v2/%s/%s/%s/%s' %
(prefix, name_of_thing, first_chars, date, id))
keys.extend(
super(DatePrefixKeyBuilder,
self).build_keys(prefix, name_of_thing, id))
return keys
def _get_datestamp(self, crashid):
"""Retrieves the datestamp from a crashid or raises an exception"""
datestamp = dateFromOoid(crashid)
if datestamp is None:
# We should never hit this situation unless the crashid is a bad crashid
raise CrashidMissingDatestamp('%s is missing datestamp' % crashid)
return datestamp
def _get_datestamp(self, crashid):
"""Retrieves the datestamp from a crashid or raises an exception"""
datestamp = dateFromOoid(crashid)
if datestamp is None:
# We should never hit this situation unless the crashid is a bad crashid
raise CrashidMissingDatestamp('%s is missing datestamp' % crashid)
return datestamp
def processJob (self, jobTuple):
""" This function is run only by a worker thread.
Given a job, fetch a thread local database connection and the json document. Use these
to create the record in the 'reports' table, then start the analysis of the dump file.
input parameters:
jobTuple: a tuple containing up to three items: the jobId (the primary key from the jobs table), the
jobUuid (a unique string with the json file basename minus the extension) and the priority (an integer)
"""
threadName = threading.currentThread().getName()
try:
threadLocalDatabaseConnection, threadLocalCursor = self.databaseConnectionPool.connectionCursorPair()
except:
self.quit = True
socorro.lib.util.reportExceptionAndAbort(logger) # can't continue without a database connection
try:
processorErrorMessages = []
jobId, jobUuid, jobPriority = jobTuple
logger.info("%s - starting job: %s, %s", threadName, jobId, jobUuid)
startedDateTime = datetime.datetime.now()
threadLocalCursor.execute("update jobs set starteddatetime = %s where id = %s", (startedDateTime, jobId))
threadLocalDatabaseConnection.commit()
jobPathname = self.jsonPathForUuidInJsonDumpStorage(jobUuid)
dumpfilePathname = self.dumpPathForUuidInJsonDumpStorage(jobUuid)
jsonFile = open(jobPathname)
try:
jsonDocument = simplejson.load(jsonFile)
finally:
jsonFile.close()
try:
date_processed = sdt.datetimeFromISOdateString(jsonDocument["submitted_timestamp"])
except KeyError:
date_processed = ooid.dateFromOoid(jobUuid)
reportId = self.insertReportIntoDatabase(threadLocalCursor, jobUuid, jsonDocument, jobPathname, date_processed, processorErrorMessages)
threadLocalDatabaseConnection.commit()
truncated = self.doBreakpadStackDumpAnalysis(reportId, jobUuid, dumpfilePathname, threadLocalCursor, date_processed, processorErrorMessages)
self.quitCheck()
#finished a job - cleanup
threadLocalCursor.execute("update jobs set completeddatetime = %s, success = True where id = %s", (datetime.datetime.now(), jobId))
threadLocalCursor.execute("update reports set started_datetime = timestamp without time zone '%s', completed_datetime = timestamp without time zone '%s', success = True, truncated = %s where id = %s and date_processed = timestamp without time zone '%s'" % (startedDateTime, datetime.datetime.now(), truncated, reportId, date_processed))
#self.updateRegistrationNoCommit(threadLocalCursor)
threadLocalDatabaseConnection.commit()
logger.info("%s - succeeded and committed: %s, %s", threadName, jobId, jobUuid)
except (KeyboardInterrupt, SystemExit):
logger.info("%s - quit request detected", threadName)
self.quit = True
try:
logger.info("%s - abandoning job with rollback: %s, %s", threadName, jobId, jobUuid)
threadLocalDatabaseConnection.rollback()
threadLocalDatabaseConnection.close()
except:
pass
except DuplicateEntryException, x:
logger.warning("%s - duplicate entry: %s", threadName, jobUuid)
def build_keys(self, prefix, name_of_thing, id):
"""return a list of one key. The reason for not returning more than
one is that this key builder class is going to be used for
something new so it has no legacy."""
datestamp = dateFromOoid(id)
if datestamp is None:
# the id did not have a date component in it
datestamp = datetime.datetime.utcnow()
date = datestamp.strftime('%Y%m%d')
keys = ['%s/v1/%s/%s/%s' % (prefix, name_of_thing, date, id)]
return keys
def dailyPart(self, ooid, timestamp=None):
"""
return YYYYMMDD
use the timestamp if any, else the ooid's last 6 chars if reasonable, else now()
"""
year,month,day = None,None,None
if not timestamp:
timestamp = socorro_ooid.dateFromOoid(ooid)
if not timestamp:
timestamp = utc_now()
(year,month,day) = (timestamp.year,timestamp.month,timestamp.day)
return "%4d%02d%02d"%(year,month,day)
def _get_base(self, crash_id):
"""this method overrides the base method to define the daily file
system root directory name. While the default class is to use a
YYYYMMDD form, this class substitutes a simple DD form. This is the
mechanism of directory recycling as at the first day of a new month
we return to the same directiory structures that were created on the
first day of the previous month"""
date = dateFromOoid(crash_id)
if not date:
date = utc_now()
date_formatted = "%02d" % (date.day,)
return [self.config.fs_root, date_formatted]
def _get_base(self, crash_id):
"""this method overrides the base method to define the daily file
system root directory name. While the default class is to use a
YYYYMMDD form, this class substitutes a simple DD form. This is the
mechanism of directory recycling as at the first day of a new month
we return to the same directiory structures that were created on the
first day of the previous month"""
date = dateFromOoid(crash_id)
if not date:
date = utc_now()
date_formatted = "%02d" % (date.day, )
return [self.config.fs_root, date_formatted]
def dailyPart(self, ooid, timestamp=None):
"""
return YYYYMMDD
use the timestamp if any, else the ooid's last 6 chars if reasonable, else now()
"""
year,month,day = None,None,None
if not timestamp:
timestamp = socorro_ooid.dateFromOoid(ooid)
if not timestamp:
timestamp = utc_now()
(year,month,day) = (timestamp.year,timestamp.month,timestamp.day)
return "%4d%02d%02d"%(year,month,day)
def build_keys(self, prefix, name_of_thing, id):
"""return a list of one key. The reason for not returning more than
one is that this key builder class is going to be used for
something new so it has no legacy."""
datestamp = dateFromOoid(id)
if datestamp is None:
# the id did not have a date component in it
datestamp = datetime.datetime.utcnow()
date = datestamp.strftime('%Y%m%d')
keys = [
'%s/v1/%s/%s/%s' % (
prefix, name_of_thing, date, id
)
]
return keys
def newEntry(self, ooid, timestamp=None, webheadName=None):
"""
Sets up the name and date storage directory branches for the given
ooid.
Creates any needed directories along the path to the appropriate
storage location. Sets gid and mode if specified
Creates one symbolic link in the date leaf directory with name ooid
and referencing the name leaf directory
returns (nameDir,dateDir)
"""
if not timestamp:
timestamp = socorro_ooid.dateFromOoid(ooid)
if not timestamp:
timestamp = utc_now()
if not self.osModule.path.isdir(self.root):
um = self.osModule.umask(0)
try:
self.osModule.mkdir(self.root, self.dirPermissions)
finally:
self.osModule.umask(um)
nameDir, nparts = self.makeNameDir(ooid, timestamp)
dateDir, dparts = self.makeDateDir(timestamp, webheadName)
# adjust the current subslot only when inserting a new entry
if self.subSlotCount:
k = dparts[-1].split('_')[0]
curcount = self.currentSubSlots.setdefault(k, 0)
self.currentSubSlots[k] = (curcount + 1) % self.subSlotCount
parts = [
os.path.pardir,
] * (len(dparts) - 2) # lose root / dailypart
parts.append(self.indexName)
parts.extend(self.relativeNameParts(ooid))
relNameDir = os.sep.join(parts)
try:
self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid))
except OSError, x:
if errno.ENOENT == x.errno:
# maybe another thread cleaned this out from under us. Try again
nameDir = self.makeNameDir(ooid) # might be overkill,
# but cheap insurance
dateDir = self.makeDateDir(timestamp)
self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid))
elif errno.EEXIST == x.errno:
self.osModule.unlink(os.path.join(dateDir, ooid))
self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid))
else:
raise
def newEntry(self, ooid, timestamp=None, webheadName=None):
"""
Sets up the name and date storage directory branches for the given
ooid.
Creates any needed directories along the path to the appropriate
storage location. Sets gid and mode if specified
Creates one symbolic link in the date leaf directory with name ooid
and referencing the name leaf directory
returns (nameDir,dateDir)
"""
if not timestamp:
timestamp = socorro_ooid.dateFromOoid(ooid)
if not timestamp:
timestamp = utc_now()
if not self.osModule.path.isdir(self.root):
um = self.osModule.umask(0)
try:
self.osModule.mkdir(self.root, self.dirPermissions)
finally:
self.osModule.umask(um)
nameDir, nparts = self.makeNameDir(ooid, timestamp)
dateDir, dparts = self.makeDateDir(timestamp, webheadName)
# adjust the current subslot only when inserting a new entry
if self.subSlotCount:
k = dparts[-1].split('_')[0]
curcount = self.currentSubSlots.setdefault(k, 0)
self.currentSubSlots[k] = (curcount + 1) % self.subSlotCount
parts = [os.path.pardir, ] * (len(dparts) - 2) # lose root / dailypart
parts.append(self.indexName)
parts.extend(self.relativeNameParts(ooid))
relNameDir = os.sep.join(parts)
try:
self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid))
except OSError, x:
if errno.ENOENT == x.errno:
# maybe another thread cleaned this out from under us. Try again
nameDir = self.makeNameDir(ooid) # might be overkill,
# but cheap insurance
dateDir = self.makeDateDir(timestamp)
self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid))
elif errno.EEXIST == x.errno:
self.osModule.unlink(os.path.join(dateDir, ooid))
self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid))
else:
raise
def lookupOoidInDatePath(self, date, ooid, webheadName=None):
"""Look for the date path holding a symbolic link named 'ooid', return
datePath, dateParts on failure return None,[]"""
if not date:
date = socorro_ooid.dateFromOoid(ooid)
if date:
datePath, dateParts = self.datePath(date, webheadName)
if self.osModule.path.exists(os.path.join(datePath, ooid)):
return datePath, dateParts
for d in self.osModule.listdir(self.root):
# We don't know webhead if any, so avoid confusion by looking
# everywhere
for dir, dirs, files in os.walk(
os.sep.join((self.root, d, self.dateName))
):
if ooid in dirs or ooid in files: # probably dirs
dirPath = dir
dirParts = dir.split(os.sep)
return dirPath, dirParts
return None, []
def build_keys(self, prefix, name_of_thing, id):
"""Use S3 pseudo-directories to make it easier to list/expire.
For "raw_crash" things, the id is an ooid/crash_id. This uses the
first three characters of the ooid for entropy and extracts the
crash submission date from the last 6 charactes. Then it mooshes
that all together into this structure::
{prefix}/v2/{name_of_thing}/{entropy}/{date}/{id}
This makes it possible to list all the raw_crashes submitted on a
given date.
It also returns the keys that KeyBuilderBase builds so that we can
fetch objects that were saved with older keys.
"""
keys = []
if name_of_thing == 'raw_crash':
datestamp = dateFromOoid(id)
if datestamp is not None:
# We insert the first 3 chars of the ooid/crash_id providing
# some entropy earlier in the key so that consecutive s3
# requests get distributed across multiple s3 partitions.
first_chars = id[:3]
date = datestamp.strftime('%Y%m%d')
keys.append(
'%s/v2/%s/%s/%s/%s' % (
prefix, name_of_thing, first_chars, date, id
)
)
keys.extend(
super(DatePrefixKeyBuilder, self).build_keys(
prefix, name_of_thing, id
)
)
return keys
def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dump):
""" This function is run only by a worker thread.
Given a job, fetch a thread local database connection and the json
document. Use these to create the record in the 'reports' table,
then start the analysis of the dump file.
input parameters:
"""
try:
self.quit_check()
crash_id = raw_crash.uuid
processor_notes = []
processed_crash = DotDict()
processed_crash.uuid = raw_crash.uuid
processed_crash.success = False
started_timestamp = self._log_job_start(crash_id)
#self.config.logger.debug('about to apply rules')
self.raw_crash_transform_rule_system.apply_all_rules(
raw_crash, self)
#self.config.logger.debug('done applying transform rules')
try:
submitted_timestamp = datetimeFromISOdateString(
raw_crash.submitted_timestamp)
except KeyError:
submitted_timestamp = dateFromOoid(crash_id)
# formerly the call to 'insertReportIntoDatabase'
processed_crash_update = self._create_basic_processed_crash(
crash_id, raw_crash, submitted_timestamp, started_timestamp,
processor_notes)
processed_crash.update(processed_crash_update)
temp_dump_pathname = self._get_temp_dump_pathname(
crash_id, raw_dump)
try:
#logger.debug('about to doBreakpadStackDumpAnalysis')
processed_crash_update_dict = \
self._do_breakpad_stack_dump_analysis(
crash_id,
temp_dump_pathname,
processed_crash.hang_type,
processed_crash.java_stack_trace,
submitted_timestamp,
processor_notes
)
processed_crash.update(processed_crash_update_dict)
finally:
self._cleanup_temp_file(temp_dump_pathname)
processed_crash.topmost_filenames = "|".join(
processed_crash.get('topmost_filenames', []))
try:
processed_crash.Winsock_LSP = raw_crash.Winsock_LSP
except KeyError:
pass # if it's not in the original raw_crash,
# it does get into the jsonz
#except (KeyboardInterrupt, SystemExit):
#self.config.logger.info("quit request detected")
#raise
except Exception, x:
self.config.logger.warning('Error while processing %s: %s',
crash_id,
str(x),
exc_info=True)
processor_notes.append(str(x))
def _get_base(self, crash_id):
date = dateFromOoid(crash_id)
if not date:
date = utc_now()
date_formatted = "%4d%02d%02d" % (date.year, date.month, date.day)
return [self.config.fs_root, date_formatted]
def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
processor_notes = processor_meta.processor_notes
processed_crash.submitted_timestamp = raw_crash.get(
'submitted_timestamp', dateFromOoid(raw_crash.uuid))
if isinstance(processed_crash.submitted_timestamp, basestring):
processed_crash.submitted_timestamp = datetime_from_isodate_string(
processed_crash.submitted_timestamp)
processed_crash.date_processed = processed_crash.submitted_timestamp
# defaultCrashTime: must have crashed before date processed
submitted_timestamp_as_epoch = int(
time.mktime(processed_crash.submitted_timestamp.timetuple()))
try:
timestampTime = int(
raw_crash.get('timestamp', submitted_timestamp_as_epoch)
) # the old name for crash time
except ValueError:
timestampTime = 0
processor_notes.append('non-integer value of "timestamp"')
try:
crash_time = int(
self._get_truncate_or_warn(raw_crash, 'CrashTime',
processor_notes, timestampTime, 10))
except ValueError:
crash_time = 0
processor_notes.append('non-integer value of "CrashTime" (%s)' %
raw_crash.CrashTime)
processed_crash.crash_time = crash_time
if crash_time == submitted_timestamp_as_epoch:
processor_notes.append("client_crash_date is unknown")
# StartupTime: must have started up some time before crash
try:
startupTime = int(raw_crash.get('StartupTime', crash_time))
except ValueError:
startupTime = 0
processor_notes.append('non-integer value of "StartupTime"')
# InstallTime: must have installed some time before startup
try:
installTime = int(raw_crash.get('InstallTime', startupTime))
except ValueError:
installTime = 0
processor_notes.append('non-integer value of "InstallTime"')
processed_crash.client_crash_date = datetime.datetime.fromtimestamp(
crash_time, UTC)
processed_crash.install_age = crash_time - installTime
processed_crash.uptime = max(0, crash_time - startupTime)
try:
last_crash = int(raw_crash.SecondsSinceLastCrash)
except (KeyError, TypeError, ValueError):
last_crash = None
processor_notes.append(
'non-integer value of "SecondsSinceLastCrash"')
if last_crash > maxint:
last_crash = None
processor_notes.append(
'"SecondsSinceLastCrash" larger than MAXINT - set to NULL')
processed_crash.last_crash = last_crash
return True
def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
processor_notes = processor_meta.processor_notes
processed_crash.submitted_timestamp = raw_crash.get(
'submitted_timestamp',
dateFromOoid(raw_crash.uuid)
)
if isinstance(processed_crash.submitted_timestamp, basestring):
processed_crash.submitted_timestamp = datetime_from_isodate_string(
processed_crash.submitted_timestamp
)
processed_crash.date_processed = processed_crash.submitted_timestamp
# defaultCrashTime: must have crashed before date processed
submitted_timestamp_as_epoch = int(
time.mktime(processed_crash.submitted_timestamp.timetuple())
)
try:
timestampTime = int(
raw_crash.get('timestamp', submitted_timestamp_as_epoch)
) # the old name for crash time
except ValueError:
timestampTime = 0
processor_notes.append('non-integer value of "timestamp"')
try:
crash_time = int(
self._get_truncate_or_warn(
raw_crash,
'CrashTime',
processor_notes,
timestampTime,
10
)
)
except ValueError:
crash_time = 0
processor_notes.append(
'non-integer value of "CrashTime" (%s)' % raw_crash.CrashTime
)
processed_crash.crash_time = crash_time
if crash_time == submitted_timestamp_as_epoch:
processor_notes.append("client_crash_date is unknown")
# StartupTime: must have started up some time before crash
try:
startupTime = int(raw_crash.get('StartupTime', crash_time))
except ValueError:
startupTime = 0
processor_notes.append('non-integer value of "StartupTime"')
# InstallTime: must have installed some time before startup
try:
installTime = int(raw_crash.get('InstallTime', startupTime))
except ValueError:
installTime = 0
processor_notes.append('non-integer value of "InstallTime"')
processed_crash.client_crash_date = datetime.datetime.fromtimestamp(
crash_time,
UTC
)
processed_crash.install_age = crash_time - installTime
processed_crash.uptime = max(0, crash_time - startupTime)
try:
last_crash = int(raw_crash.SecondsSinceLastCrash)
except (KeyError, TypeError, ValueError):
last_crash = None
processor_notes.append(
'non-integer value of "SecondsSinceLastCrash"'
)
if last_crash > maxint:
last_crash = None
processor_notes.append(
'"SecondsSinceLastCrash" larger than MAXINT - set to NULL'
)
processed_crash.last_crash = last_crash
return True
def _get_base(self, crash_id):
date = dateFromOoid(crash_id)
if not date:
date = utc_now()
date_formatted = "%4d%02d%02d" % (date.year, date.month, date.day)
return [self.config.fs_root, date_formatted]
def testGetDate(self):
for ooid in self.yyyyoids:
assert self.baseDate == oo.dateFromOoid(ooid), 'Expected %s got %s' %(self.baseDate, oo.dateFromOoid(ooid))
assert 4 == oo.depthFromOoid(ooid), 'Expected %d, got %d' %(4, oo.depthFromOoid(ooid))
assert None == oo.dateFromOoid(self.badooid0)
assert None == oo.dateFromOoid(self.badooid1)
def testGetDate(self):
for this_ooid in self.yyyyoids:
assert self.baseDate == ooid.dateFromOoid(this_ooid)
assert 4 == ooid.depthFromOoid(this_ooid)
assert ooid.dateFromOoid(self.badooid0) is None
assert ooid.dateFromOoid(self.badooid1) is None
def testGetDate(self):
for this_ooid in self.yyyyoids:
assert self.baseDate == ooid.dateFromOoid(this_ooid)
assert 4 == ooid.depthFromOoid(this_ooid)
assert ooid.dateFromOoid(self.badooid0) is None
assert ooid.dateFromOoid(self.badooid1) is None
def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dump):
""" This function is run only by a worker thread.
Given a job, fetch a thread local database connection and the json
document. Use these to create the record in the 'reports' table,
then start the analysis of the dump file.
input parameters:
"""
try:
self.quit_check()
crash_id = raw_crash.uuid
processor_notes = []
processed_crash = DotDict()
processed_crash.uuid = raw_crash.uuid
processed_crash.success = False
started_timestamp = self._log_job_start(crash_id)
#self.config.logger.debug('about to apply rules')
self.raw_crash_transform_rule_system.apply_all_rules(raw_crash,
self)
#self.config.logger.debug('done applying transform rules')
try:
submitted_timestamp = datetimeFromISOdateString(
raw_crash.submitted_timestamp
)
except KeyError:
submitted_timestamp = dateFromOoid(crash_id)
# formerly the call to 'insertReportIntoDatabase'
processed_crash_update = self._create_basic_processed_crash(
crash_id,
raw_crash,
submitted_timestamp,
started_timestamp,
processor_notes
)
processed_crash.update(processed_crash_update)
temp_dump_pathname = self._get_temp_dump_pathname(
crash_id,
raw_dump
)
try:
#logger.debug('about to doBreakpadStackDumpAnalysis')
processed_crash_update_dict = \
self._do_breakpad_stack_dump_analysis(
crash_id,
temp_dump_pathname,
processed_crash.hang_type,
processed_crash.java_stack_trace,
submitted_timestamp,
processor_notes
)
processed_crash.update(processed_crash_update_dict)
finally:
self._cleanup_temp_file(temp_dump_pathname)
processed_crash.topmost_filenames = "|".join(
processed_crash.get('topmost_filenames', [])
)
try:
processed_crash.Winsock_LSP = raw_crash.Winsock_LSP
except KeyError:
pass # if it's not in the original raw_crash,
# it does get into the jsonz
#except (KeyboardInterrupt, SystemExit):
#self.config.logger.info("quit request detected")
#raise
except Exception, x:
self.config.logger.warning(
'Error while processing %s: %s',
crash_id,
str(x),
exc_info=True
)
processor_notes.append(str(x))
def processJob (self, jobTuple):
""" This function is run only by a worker thread.
Given a job, fetch a thread local database connection and the json document. Use these
to create the record in the 'reports' table, then start the analysis of the dump file.
input parameters:
jobTuple: a tuple containing up to three items: the jobId (the primary key from the jobs table), the
jobUuid (a unique string with the json file basename minus the extension) and the priority (an integer)
"""
if self.quit:
return Processor.quit
threadName = threading.currentThread().getName()
try:
threadLocalDatabaseConnection, threadLocalCursor = self.databaseConnectionPool.connectionCursorPair()
threadLocalCrashStorage = self.crashStorePool.crashStorage(threadName)
except sdb.exceptions_eligible_for_retry:
logger.critical("something's gone horribly wrong with the database connection")
sutil.reportExceptionAndContinue(logger, loggingLevel=logging.CRITICAL)
return Processor.criticalError
except hbc.FatalException:
logger.critical("something's gone horribly wrong with the HBase connection")
sutil.reportExceptionAndContinue(logger, loggingLevel=logging.CRITICAL)
return Processor.criticalError
except Exception:
self.quit = True
sutil.reportExceptionAndContinue(logger, loggingLevel=logging.CRITICAL)
return Processor.quit
try:
self.quitCheck()
newReportRecordAsDict = {}
processorErrorMessages = []
jobId, jobUuid, jobPriority = jobTuple
logger.info("starting job: %s", jobUuid)
startedDateTime = self.nowFunc()
threadLocalCursor.execute("update jobs set starteddatetime = %s where id = %s", (startedDateTime, jobId))
threadLocalDatabaseConnection.commit()
jsonDocument = threadLocalCrashStorage.get_meta(jobUuid)
self.config.logger.debug('about to apply rules')
self.json_transform_rule_system.apply_all_rules(jsonDocument, self)
self.config.logger.debug('done applying transform rules')
try:
date_processed = sdt.datetimeFromISOdateString(jsonDocument["submitted_timestamp"])
except KeyError:
date_processed = ooid.dateFromOoid(jobUuid)
newReportRecordAsDict = self.insertReportIntoDatabase(threadLocalCursor, jobUuid, jsonDocument, date_processed, processorErrorMessages)
if jobUuid in self.priority_job_set:
processorErrorMessages.append('Priority Job')
self.priority_job_set.remove(jobUuid)
threadLocalDatabaseConnection.commit()
reportId = newReportRecordAsDict["id"]
newReportRecordAsDict['dump'] = ''
newReportRecordAsDict["startedDateTime"] = startedDateTime
try:
newReportRecordAsDict["ReleaseChannel"] = jsonDocument["ReleaseChannel"]
except KeyError:
newReportRecordAsDict["ReleaseChannel"] = 'unknown'
if self.config.collectAddon:
#logger.debug("collecting Addons")
addonsAsAListOfTuples = self.insertAdddonsIntoDatabase(threadLocalCursor, reportId, jsonDocument, date_processed, processorErrorMessages)
newReportRecordAsDict["addons"] = addonsAsAListOfTuples
if self.config.collectCrashProcess:
#logger.debug("collecting Crash Process")
crashProcessAsDict = self.insertCrashProcess(threadLocalCursor, reportId, jsonDocument, date_processed, processorErrorMessages)
newReportRecordAsDict.update( crashProcessAsDict )
try:
dumpfilePathname = threadLocalCrashStorage.dumpPathForUuid(jobUuid,
self.config.temporaryFileSystemStoragePath)
#logger.debug('about to doBreakpadStackDumpAnalysis')
isHang = 'hangid' in newReportRecordAsDict and bool(newReportRecordAsDict['hangid'])
# hangType values: -1 if old style hang with hangid and Hang not present
# else hangType == jsonDocument.Hang
hangType = int(jsonDocument.get("Hang", -1 if isHang else 0))
java_stack_trace = jsonDocument.setdefault('JavaStackTrace', None)
additionalReportValuesAsDict = self.doBreakpadStackDumpAnalysis(reportId, jobUuid, dumpfilePathname, hangType, java_stack_trace, threadLocalCursor, date_processed, processorErrorMessages)
newReportRecordAsDict.update(additionalReportValuesAsDict)
finally:
newReportRecordAsDict["completeddatetime"] = completedDateTime = self.nowFunc()
threadLocalCrashStorage.cleanUpTempDumpStorage(jobUuid, self.config.temporaryFileSystemStoragePath)
#logger.debug('finished a job - cleanup')
#finished a job - cleanup
threadLocalCursor.execute("update jobs set completeddatetime = %s, success = %s where id = %s", (completedDateTime, newReportRecordAsDict['success'], jobId))
# Bug 519703: Collect setting for topmost source filename(s), addon compatibility check override, flash version
reportsSql = """
update reports set
signature = %%s,
processor_notes = %%s,
started_datetime = timestamp with time zone %%s,
completed_datetime = timestamp with time zone %%s,
success = %%s,
truncated = %%s,
topmost_filenames = %%s,
addons_checked = %%s,
flash_version = %%s
where id = %s and date_processed = timestamp with time zone '%s'
""" % (reportId,date_processed)
#logger.debug("newReportRecordAsDict %s, %s", newReportRecordAsDict['topmost_filenames'], newReportRecordAsDict['flash_version'])
#topmost_filenames = "|".join(jsonDocument.get('topmost_filenames',[]))
topmost_filenames = "|".join(newReportRecordAsDict.get('topmost_filenames',[]))
addons_checked = None
try:
addons_checked_txt = jsonDocument['EMCheckCompatibility'].lower()
addons_checked = False
if addons_checked_txt == 'true':
addons_checked = True
except KeyError:
pass # leaving it as None if not in the document
try:
newReportRecordAsDict['Winsock_LSP'] = jsonDocument['Winsock_LSP']
except KeyError:
pass # if it's not in the original json, it does get into the jsonz
#flash_version = jsonDocument.get('flash_version')
flash_version = newReportRecordAsDict.get('flash_version')
processor_notes = '; '.join(processorErrorMessages)
newReportRecordAsDict['processor_notes'] = processor_notes
infoTuple = (newReportRecordAsDict['signature'], processor_notes, startedDateTime, completedDateTime, newReportRecordAsDict["success"], newReportRecordAsDict["truncated"], topmost_filenames, addons_checked, flash_version)
#logger.debug("Updated report %s (%s): %s", reportId, jobUuid, str(infoTuple))
threadLocalCursor.execute(reportsSql, infoTuple)
threadLocalDatabaseConnection.commit()
self.saveProcessedDumpJson(newReportRecordAsDict, threadLocalCrashStorage)
self.submitOoidToElasticSearch(jobUuid)
if newReportRecordAsDict["success"]:
logger.info("succeeded and committed: %s", jobUuid)
else:
logger.info("failed but committed: %s", jobUuid)
self.quitCheck()
return Processor.ok
except (KeyboardInterrupt, SystemExit):
logger.info("quit request detected")
self.quit = True
return Processor.quit
except DuplicateEntryException, x:
logger.warning("duplicate entry: %s", jobUuid)
threadLocalCursor.execute('delete from jobs where id = %s', (jobId,))
threadLocalDatabaseConnection.commit()
return Processor.ok
