def test_save_raw_and_processed(self): config = self.get_standard_config() db_sampling = DBCrashStorageWrapperNewCrashSource(config) crash_id = '86b58ff2-9708-487d-bfc4-9dac32121214' fake_raw_crash = SocorroDotDict({ "name": "Gabi", "submitted_timestamp": "2012-12-14T00:00:00" }) fake_dumps_as_files = FileDumpsMapping({ 'upload_file_minidump': '86b58ff2-9708-487d-bfc4-9dac32121214' '.upload_file_minidump.TEMPORARY.dump' }) fake_processed = SocorroDotDict({ "name": "Gabi", "submitted_timestamp": "2012-12-14T00:00:00" }) # the call to be tested db_sampling.save_raw_and_processed(fake_raw_crash, fake_dumps_as_files, fake_processed, crash_id) # this is what should have happened db_sampling._implementation.save_raw_and_processed \ .assert_called_once_with( fake_raw_crash, fake_dumps_as_files, fake_processed, crash_id )
def test_save_processed(self): config = self.get_standard_config() db_sampling = DBCrashStorageWrapperNewCrashSource(config) fake_processed = SocorroDotDict({ "name": "Gabi", "submitted_timestamp": "2012-12-14T00:00:00" }) # the call to be tested db_sampling.save_processed(fake_processed) # this is what should have happened db_sampling._implementation.save_processed.assert_called_once_with( fake_processed)
def test_get_processed(self): config = self.get_standard_config() db_sampling = DBCrashStorageWrapperNewCrashSource(config) crash_id = '86b58ff2-9708-487d-bfc4-9dac32121214' fake_processed = SocorroDotDict({ "name": "Gabi", "submitted_timestamp": "2012-12-14T00:00:00" }) mocked_get_processed = Mock(return_value=fake_processed) db_sampling._implementation.get_processed = mocked_get_processed # the call to be tested processed = db_sampling.get_processed(crash_id) # this is what should have happened ok_(fake_processed is processed) db_sampling._implementation.get_processed.assert_called_with(crash_id)
def _summary_for_a_product_version_pair(self, an_accumulator): """in the original code, the counter structures were walked and manipulated to form the statistics. Once a stat was determined, it was printed to stdout. Since we want to have various means of outputting the data, instead of printing to stdout, this method save the statistic in a "summary_structure" This structure will later be walked for printing or output to some future storage scheme The summary structure looks like this: summary[product_version*] .note - a list of comments by the algorithm [os_name] .count .signatures[signame*] .name .count .cores[number_of_cores] .in_sig_count .in_sig_ratio .rounded_in_sig_ratio .in_os_count .in_os_ratio .rounded_in_os_ratio """ pv_summary = { 'notes': [], } if (len(self.date_suffix) > 1): message = ("crashes from more than one day %s" % str(tuple(self.date_suffix.keys()))) pv_summary['notes'].append(message) pv_summary['date_key'] = self.date_suffix.keys()[0] MIN_CRASHES = self.config.min_crashes osyses = an_accumulator.osyses # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # begin - minimally altered section from original code # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - infostr_re = re.compile("^(.*) with (\d+) cores$") #---------------------------------------------------------------------- def cmp_infostr(x, y): (familyx, coresx) = infostr_re.match(x).groups() (familyy, coresy) = infostr_re.match(y).groups() if familyx != familyy: return cmp(familyx, familyy) return cmp(int(coresx), int(coresy)) #---------------------------------------------------------------------- sorted_osyses = osyses.keys() sorted_osyses.sort() for osname in sorted_osyses: osys = osyses[osname] pv_summary[osname] = SocorroDotDict() pv_summary[osname].count = osys['count'] pv_summary[osname].signatures = {} sorted_signatures = [ sig for sig in osys["signatures"].items() if sig[1]["count"] >= MIN_CRASHES ] sorted_signatures.sort(key=lambda tuple: tuple[1]["count"], reverse=True) sorted_cores = osys["core_counts"].keys() # strongly suspect that sorting is useless here sorted_cores.sort(cmp=cmp_infostr) for signame, sig in sorted_signatures: pv_summary[osname].signatures[signame] = SocorroDotDict({ 'name': signame, 'count': sig['count'], 'cores': {}, }) by_number_of_cores = \ pv_summary[osname].signatures[signame].cores for cores in sorted_cores: by_number_of_cores[cores] = SocorroDotDict() in_sig_count = sig["core_counts"].get(cores, 0) in_sig_ratio = float(in_sig_count) / sig["count"] in_os_count = osys["core_counts"][cores] in_os_ratio = float(in_os_count) / osys["count"] rounded_in_sig_ratio = int(round(in_sig_ratio * 100)) rounded_in_os_ratio = int(round(in_os_ratio * 100)) by_number_of_cores[cores].in_sig_count = in_sig_count by_number_of_cores[cores].in_sig_ratio = in_sig_ratio by_number_of_cores[cores].rounded_in_sig_ratio = \ rounded_in_sig_ratio by_number_of_cores[cores].in_os_count = in_os_count by_number_of_cores[cores].in_os_ratio = in_os_ratio by_number_of_cores[cores].rounded_in_os_ratio = \ rounded_in_os_ratio # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # end - minimally altered code section # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - return pv_summary
def _summary_for_a_product_version_pair(self, a_pv_accumulator): """in the original code, the counter structures were walked and manipulated to form the statistics. Once a stat was determined, it was printed to stdout. Since we want to have various means of outputting the data, instead of printing to stdout, this method save the statistic in a "summary_structure" This structure will later be walked for printing or output to some future storage scheme The summary structure looks like this: pv_summary .date_key # a list of the last six UUID characters present .notes # any notes added by the algorithm to tell of problems .os_counters[os_name*] .count .signatures[a_signature*] .count .in_sig_ratio .in_os_ratio .in_os_count .osys_count .modules[a_module*] # may be addons .in_sig_ratio .in_os_ratio .in_os_count .osys_count .verisons[a_version*] # may be addon versions .sig_ver_ratio .sig_ver_count .sig_count .os_ver_ratio .os_ver_count .osys_count .version """ options = self.config pv_summary = SocorroDotDict({ 'notes': [], }) if (len(self.date_suffix) > 1): message = ("crashes from more than one day %s" % str(tuple(self.date_suffix.keys()))) ## self.config.logger.debug(message) pv_summary.notes.append(message) pv_summary.date_key = self.date_suffix.keys()[0] pv_summary.os_counters = {} MIN_CRASHES = self.config.min_crashes counters_for_multiple_os = a_pv_accumulator.osyses # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # begin - refactored code section # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - infostr_re = re.compile("^(.*) with (\d+) cores$") # unused? for os_name in counters_for_multiple_os.keys(): counters_for_an_os = counters_for_multiple_os[os_name] pv_summary.os_counters[os_name] = SocorroDotDict() pv_summary.os_counters[os_name].count = counters_for_multiple_os[ os_name].count pv_summary.os_counters[os_name].signatures = {} filtered_signatures = [(signature, signature_counter) for ( signature, signature_counter) in counters_for_an_os["signatures"].items() if signature_counter.count >= MIN_CRASHES] for a_signature, a_signtaure_counter in filtered_signatures: pv_summary.os_counters[os_name].signatures[ a_signature] = SocorroDotDict() pv_summary.os_counters[os_name].signatures[ a_signature].count = a_signtaure_counter.count pv_summary.os_counters[os_name].signatures[ a_signature].modules = {} modules_list = [ SocorroDotDict({ "libname": module_name, "in_sig_count": a_module_counter.count, "in_sig_ratio": float(a_module_counter.count) / a_signtaure_counter.count, "in_sig_versions": a_module_counter.versions, "in_os_count": counters_for_an_os.modules[module_name].count, "in_os_ratio": (float(counters_for_an_os.modules[module_name].count) / counters_for_an_os.count), "in_os_versions": counters_for_an_os.modules[module_name].versions }) for module_name, a_module_counter in a_signtaure_counter.modules.iteritems() ] modules_list = [ module for module in modules_list if module.in_sig_ratio - module.in_os_ratio >= self.config.min_baseline_diff ] modules_list.sort(key=lambda module: module.in_sig_ratio - module.in_os_ratio, reverse=True) for module in modules_list: module_name = module.libname if options.addons: info = addonids.info_for_id(module_name) if info is not None: module_name = ( module_name + u" ({0}, {1})".format(info.name, info.url)) if options.show_versions and len( module["in_os_versions"]) == 1: onlyver = module.in_os_versions.keys()[0] if os_name.startswith("Mac OS X"): info = macdebugids.info_for_id( module_name, onlyver) if info is not None: onlyver = onlyver + "; " + info if (onlyver != ""): module_name = module_name + " (" + onlyver + ")" pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name] = SocorroDotDict() pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].in_sig_count = ( module.in_sig_count) pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].in_sig_ratio = (int( round(module["in_sig_ratio"] * 100))) pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].in_os_ratio = (int( round(module.in_os_ratio * 100))) pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].in_os_count = ( module.in_os_count) pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].osys_count = ( counters_for_an_os.count) if options.show_versions and len( module.in_os_versions) != 1: versions = module.in_os_versions.keys() versions.sort() pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].versions = {} for version in versions: sig_ver_count = module.in_sig_versions.get( version, 0) os_ver_count = module.in_os_versions[version] if os_name.startswith("Mac OS X"): info = macdebugids.info_for_id( module_name, version) if info is not None: version = version + " (" + info + ")" pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].versions[ version] = SocorroDotDict() pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].versions[ version].sig_ver_ratio = (int( round( float(sig_ver_count) / a_signtaure_counter.count * 100))) pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].versions[ version].sig_ver_count = sig_ver_count pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].versions[ version].sig_count = a_signtaure_counter.count pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].versions[ version].os_ver_ratio = (int( round( float(os_ver_count) / counters_for_an_os.count * 100))) pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].versions[ version].os_ver_count = os_ver_count pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].versions[ version].osys_count = counters_for_an_os.count pv_summary.os_counters[os_name].signatures[ a_signature].modules[module_name].versions[ version].version = version # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # end - refactored code section # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - return pv_summary
def _action(self, raw, dumps, crash, processor_meta): self.date_suffix[crash['crash_id'][-6:]] += 1 if not "os_name" in crash: # We have some bad crash reports. return False # give the names of the old algorithm's critical variables to their # variables in the new system try: osyses = self.counters_for_all_producs_and_versions[( crash["product"], crash["version"])].osyses self.counters_for_all_producs_and_versions[( crash["product"], crash["version"])].counter += 1 except (AttributeError, KeyError): # why both types? crashes can be represented by either the Socorro # or configman DotDict types which raise different exception on # not finding a key. osyses = {} self.counters_for_all_producs_and_versions[( crash["product"], crash["version"])].osyses = osyses self.counters_for_all_producs_and_versions[( crash["product"], crash["version"])].counter = 1 options = self.config # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # begin - refactored code section # unlike the "core count correlation report", this code from the # was refactored to help understand the structure of the counters # so that a generic summary structure could be made. This allows # for output of the summary information to somewhere other than # stdout. # # the structure has been broken down into levels of regular dicts # and SocorroDotDicts. The DotDicts have keys that are constant # and no more are added when new crashes come in. The regular dicts # are key with variable things that come in with crashes. In the # structure below, keys of DotDicts are shown as constants like # ".count" and ".modules". The keys of the dicts are shown as the # name of a field with a * (to designate zero or more) inside square # brackets. # # the counters structure looks like this: # pv_counters[os_name*] # .count # .signatures[a_signature*] # .count # .modules[a_module*] # .count # .versions[a_version*] int # .modules[a_module*] # .count # .versions[a_version*] int # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - os_name = crash["os_name"] # The os_version field is way too specific on Linux, and we don't # have much Linux data anyway. if options.by_os_version and os_name != "Linux": os_name = os_name + " " + crash["os_version"] counters_for_an_os = osyses.setdefault( os_name, SocorroDotDict({ "count": 0, "signatures": {}, "modules": {}, })) a_signature = crash["signature"] if self.contains_bare_address(a_signature): if options.condense: # Condense all signatures in a given DLL. a_signature = self.remove_bare_address_from_signature( a_signature) if "reason" in crash and crash["reason"] is not None: a_signature = a_signature + "|" + crash["reason"] counters_for_a_signature = counters_for_an_os.signatures.setdefault( a_signature, SocorroDotDict({ "count": 0, "modules": {} }), ) list_of_counters = [counters_for_an_os, counters_for_a_signature] # increment both the os & signature counters for a_counter in list_of_counters: a_counter.count += 1 for libname, version in self.generate_modules_or_addons(crash): # Increment the global count on osys and the per-signature count. for a_counter in list_of_counters: counters_for_modules = a_counter.modules.setdefault( libname, SocorroDotDict({ "count": 0, "versions": defaultdict(int), })) counters_for_modules.count += 1 # Count versions of each module as well. counters_for_modules.versions[version] += 1 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # end - refactored code section # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - return True