def test_save_raw_and_processed(self):
config = self.get_standard_config()
db_sampling = DBCrashStorageWrapperNewCrashSource(config)
crash_id = '86b58ff2-9708-487d-bfc4-9dac32121214'
fake_raw_crash = SocorroDotDict({
"name":
"Gabi",
"submitted_timestamp":
"2012-12-14T00:00:00"
})
fake_dumps_as_files = FileDumpsMapping({
'upload_file_minidump':
'86b58ff2-9708-487d-bfc4-9dac32121214'
'.upload_file_minidump.TEMPORARY.dump'
})
fake_processed = SocorroDotDict({
"name":
"Gabi",
"submitted_timestamp":
"2012-12-14T00:00:00"
})
# the call to be tested
db_sampling.save_raw_and_processed(fake_raw_crash, fake_dumps_as_files,
fake_processed, crash_id)
# this is what should have happened
db_sampling._implementation.save_raw_and_processed \
.assert_called_once_with(
fake_raw_crash,
fake_dumps_as_files,
fake_processed,
crash_id
)
def test_save_processed(self):
config = self.get_standard_config()
db_sampling = DBCrashStorageWrapperNewCrashSource(config)
fake_processed = SocorroDotDict({
"name":
"Gabi",
"submitted_timestamp":
"2012-12-14T00:00:00"
})
# the call to be tested
db_sampling.save_processed(fake_processed)
# this is what should have happened
db_sampling._implementation.save_processed.assert_called_once_with(
fake_processed)
def test_get_processed(self):
config = self.get_standard_config()
db_sampling = DBCrashStorageWrapperNewCrashSource(config)
crash_id = '86b58ff2-9708-487d-bfc4-9dac32121214'
fake_processed = SocorroDotDict({
"name":
"Gabi",
"submitted_timestamp":
"2012-12-14T00:00:00"
})
mocked_get_processed = Mock(return_value=fake_processed)
db_sampling._implementation.get_processed = mocked_get_processed
# the call to be tested
processed = db_sampling.get_processed(crash_id)
# this is what should have happened
ok_(fake_processed is processed)
db_sampling._implementation.get_processed.assert_called_with(crash_id)
def _summary_for_a_product_version_pair(self, an_accumulator):
"""in the original code, the counter structures were walked and
manipulated to form the statistics. Once a stat was determined,
it was printed to stdout. Since we want to have various means of
outputting the data, instead of printing to stdout, this method
save the statistic in a "summary_structure" This structure will
later be walked for printing or output to some future storage scheme
The summary structure looks like this:
summary[product_version*]
.note - a list of comments by the algorithm
[os_name]
.count
.signatures[signame*]
.name
.count
.cores[number_of_cores]
.in_sig_count
.in_sig_ratio
.rounded_in_sig_ratio
.in_os_count
.in_os_ratio
.rounded_in_os_ratio
"""
pv_summary = {
'notes': [],
}
if (len(self.date_suffix) > 1):
message = ("crashes from more than one day %s" %
str(tuple(self.date_suffix.keys())))
pv_summary['notes'].append(message)
pv_summary['date_key'] = self.date_suffix.keys()[0]
MIN_CRASHES = self.config.min_crashes
osyses = an_accumulator.osyses
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# begin - minimally altered section from original code
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
infostr_re = re.compile("^(.*) with (\d+) cores$")
#----------------------------------------------------------------------
def cmp_infostr(x, y):
(familyx, coresx) = infostr_re.match(x).groups()
(familyy, coresy) = infostr_re.match(y).groups()
if familyx != familyy:
return cmp(familyx, familyy)
return cmp(int(coresx), int(coresy))
#----------------------------------------------------------------------
sorted_osyses = osyses.keys()
sorted_osyses.sort()
for osname in sorted_osyses:
osys = osyses[osname]
pv_summary[osname] = SocorroDotDict()
pv_summary[osname].count = osys['count']
pv_summary[osname].signatures = {}
sorted_signatures = [
sig for sig in osys["signatures"].items()
if sig[1]["count"] >= MIN_CRASHES
]
sorted_signatures.sort(key=lambda tuple: tuple[1]["count"],
reverse=True)
sorted_cores = osys["core_counts"].keys()
# strongly suspect that sorting is useless here
sorted_cores.sort(cmp=cmp_infostr)
for signame, sig in sorted_signatures:
pv_summary[osname].signatures[signame] = SocorroDotDict({
'name':
signame,
'count':
sig['count'],
'cores': {},
})
by_number_of_cores = \
pv_summary[osname].signatures[signame].cores
for cores in sorted_cores:
by_number_of_cores[cores] = SocorroDotDict()
in_sig_count = sig["core_counts"].get(cores, 0)
in_sig_ratio = float(in_sig_count) / sig["count"]
in_os_count = osys["core_counts"][cores]
in_os_ratio = float(in_os_count) / osys["count"]
rounded_in_sig_ratio = int(round(in_sig_ratio * 100))
rounded_in_os_ratio = int(round(in_os_ratio * 100))
by_number_of_cores[cores].in_sig_count = in_sig_count
by_number_of_cores[cores].in_sig_ratio = in_sig_ratio
by_number_of_cores[cores].rounded_in_sig_ratio = \
rounded_in_sig_ratio
by_number_of_cores[cores].in_os_count = in_os_count
by_number_of_cores[cores].in_os_ratio = in_os_ratio
by_number_of_cores[cores].rounded_in_os_ratio = \
rounded_in_os_ratio
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# end - minimally altered code section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
return pv_summary
def _summary_for_a_product_version_pair(self, a_pv_accumulator):
"""in the original code, the counter structures were walked and
manipulated to form the statistics. Once a stat was determined,
it was printed to stdout. Since we want to have various means of
outputting the data, instead of printing to stdout, this method
save the statistic in a "summary_structure" This structure will
later be walked for printing or output to some future storage scheme
The summary structure looks like this:
pv_summary
.date_key # a list of the last six UUID characters present
.notes # any notes added by the algorithm to tell of problems
.os_counters[os_name*]
.count
.signatures[a_signature*]
.count
.in_sig_ratio
.in_os_ratio
.in_os_count
.osys_count
.modules[a_module*] # may be addons
.in_sig_ratio
.in_os_ratio
.in_os_count
.osys_count
.verisons[a_version*] # may be addon versions
.sig_ver_ratio
.sig_ver_count
.sig_count
.os_ver_ratio
.os_ver_count
.osys_count
.version
"""
options = self.config
pv_summary = SocorroDotDict({
'notes': [],
})
if (len(self.date_suffix) > 1):
message = ("crashes from more than one day %s" %
str(tuple(self.date_suffix.keys())))
## self.config.logger.debug(message)
pv_summary.notes.append(message)
pv_summary.date_key = self.date_suffix.keys()[0]
pv_summary.os_counters = {}
MIN_CRASHES = self.config.min_crashes
counters_for_multiple_os = a_pv_accumulator.osyses
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# begin - refactored code section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
infostr_re = re.compile("^(.*) with (\d+) cores$") # unused?
for os_name in counters_for_multiple_os.keys():
counters_for_an_os = counters_for_multiple_os[os_name]
pv_summary.os_counters[os_name] = SocorroDotDict()
pv_summary.os_counters[os_name].count = counters_for_multiple_os[
os_name].count
pv_summary.os_counters[os_name].signatures = {}
filtered_signatures = [(signature, signature_counter) for (
signature,
signature_counter) in counters_for_an_os["signatures"].items()
if signature_counter.count >= MIN_CRASHES]
for a_signature, a_signtaure_counter in filtered_signatures:
pv_summary.os_counters[os_name].signatures[
a_signature] = SocorroDotDict()
pv_summary.os_counters[os_name].signatures[
a_signature].count = a_signtaure_counter.count
pv_summary.os_counters[os_name].signatures[
a_signature].modules = {}
modules_list = [
SocorroDotDict({
"libname":
module_name,
"in_sig_count":
a_module_counter.count,
"in_sig_ratio":
float(a_module_counter.count) /
a_signtaure_counter.count,
"in_sig_versions":
a_module_counter.versions,
"in_os_count":
counters_for_an_os.modules[module_name].count,
"in_os_ratio":
(float(counters_for_an_os.modules[module_name].count) /
counters_for_an_os.count),
"in_os_versions":
counters_for_an_os.modules[module_name].versions
}) for module_name, a_module_counter in
a_signtaure_counter.modules.iteritems()
]
modules_list = [
module for module in modules_list if module.in_sig_ratio -
module.in_os_ratio >= self.config.min_baseline_diff
]
modules_list.sort(key=lambda module: module.in_sig_ratio -
module.in_os_ratio,
reverse=True)
for module in modules_list:
module_name = module.libname
if options.addons:
info = addonids.info_for_id(module_name)
if info is not None:
module_name = (
module_name +
u" ({0}, {1})".format(info.name, info.url))
if options.show_versions and len(
module["in_os_versions"]) == 1:
onlyver = module.in_os_versions.keys()[0]
if os_name.startswith("Mac OS X"):
info = macdebugids.info_for_id(
module_name, onlyver)
if info is not None:
onlyver = onlyver + "; " + info
if (onlyver != ""):
module_name = module_name + " (" + onlyver + ")"
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name] = SocorroDotDict()
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].in_sig_count = (
module.in_sig_count)
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].in_sig_ratio = (int(
round(module["in_sig_ratio"] * 100)))
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].in_os_ratio = (int(
round(module.in_os_ratio * 100)))
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].in_os_count = (
module.in_os_count)
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].osys_count = (
counters_for_an_os.count)
if options.show_versions and len(
module.in_os_versions) != 1:
versions = module.in_os_versions.keys()
versions.sort()
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].versions = {}
for version in versions:
sig_ver_count = module.in_sig_versions.get(
version, 0)
os_ver_count = module.in_os_versions[version]
if os_name.startswith("Mac OS X"):
info = macdebugids.info_for_id(
module_name, version)
if info is not None:
version = version + " (" + info + ")"
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].versions[
version] = SocorroDotDict()
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].versions[
version].sig_ver_ratio = (int(
round(
float(sig_ver_count) /
a_signtaure_counter.count * 100)))
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].versions[
version].sig_ver_count = sig_ver_count
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].versions[
version].sig_count = a_signtaure_counter.count
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].versions[
version].os_ver_ratio = (int(
round(
float(os_ver_count) /
counters_for_an_os.count * 100)))
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].versions[
version].os_ver_count = os_ver_count
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].versions[
version].osys_count = counters_for_an_os.count
pv_summary.os_counters[os_name].signatures[
a_signature].modules[module_name].versions[
version].version = version
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# end - refactored code section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
return pv_summary
def _action(self, raw, dumps, crash, processor_meta):
self.date_suffix[crash['crash_id'][-6:]] += 1
if not "os_name" in crash:
# We have some bad crash reports.
return False
# give the names of the old algorithm's critical variables to their
# variables in the new system
try:
osyses = self.counters_for_all_producs_and_versions[(
crash["product"], crash["version"])].osyses
self.counters_for_all_producs_and_versions[(
crash["product"], crash["version"])].counter += 1
except (AttributeError, KeyError):
# why both types? crashes can be represented by either the Socorro
# or configman DotDict types which raise different exception on
# not finding a key.
osyses = {}
self.counters_for_all_producs_and_versions[(
crash["product"], crash["version"])].osyses = osyses
self.counters_for_all_producs_and_versions[(
crash["product"], crash["version"])].counter = 1
options = self.config
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# begin - refactored code section
# unlike the "core count correlation report", this code from the
# was refactored to help understand the structure of the counters
# so that a generic summary structure could be made. This allows
# for output of the summary information to somewhere other than
# stdout.
#
# the structure has been broken down into levels of regular dicts
# and SocorroDotDicts. The DotDicts have keys that are constant
# and no more are added when new crashes come in. The regular dicts
# are key with variable things that come in with crashes. In the
# structure below, keys of DotDicts are shown as constants like
# ".count" and ".modules". The keys of the dicts are shown as the
# name of a field with a * (to designate zero or more) inside square
# brackets.
#
# the counters structure looks like this:
# pv_counters[os_name*]
# .count
# .signatures[a_signature*]
# .count
# .modules[a_module*]
# .count
# .versions[a_version*] int
# .modules[a_module*]
# .count
# .versions[a_version*] int
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
os_name = crash["os_name"]
# The os_version field is way too specific on Linux, and we don't
# have much Linux data anyway.
if options.by_os_version and os_name != "Linux":
os_name = os_name + " " + crash["os_version"]
counters_for_an_os = osyses.setdefault(
os_name,
SocorroDotDict({
"count": 0,
"signatures": {},
"modules": {},
}))
a_signature = crash["signature"]
if self.contains_bare_address(a_signature):
if options.condense:
# Condense all signatures in a given DLL.
a_signature = self.remove_bare_address_from_signature(
a_signature)
if "reason" in crash and crash["reason"] is not None:
a_signature = a_signature + "|" + crash["reason"]
counters_for_a_signature = counters_for_an_os.signatures.setdefault(
a_signature,
SocorroDotDict({
"count": 0,
"modules": {}
}),
)
list_of_counters = [counters_for_an_os, counters_for_a_signature]
# increment both the os & signature counters
for a_counter in list_of_counters:
a_counter.count += 1
for libname, version in self.generate_modules_or_addons(crash):
# Increment the global count on osys and the per-signature count.
for a_counter in list_of_counters:
counters_for_modules = a_counter.modules.setdefault(
libname,
SocorroDotDict({
"count": 0,
"versions": defaultdict(int),
}))
counters_for_modules.count += 1
# Count versions of each module as well.
counters_for_modules.versions[version] += 1
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# end - refactored code section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
return True