def update_all(date=None): date = magutils.get_date(date) last = Lastdate.get_last() if date is None: yesterday = utils.get_date('yesterday') if last != yesterday: date = yesterday if date: update_lastdate = True date = utils.get_date(date) logger.info('Update all for {}'.format(date)) for p in magutils.get_products(): chans = magutils.get_channels() for c in chans: data = crashes_bytype.get(p, c, date=date) logger.info('Data: {}::{}::{}::{}'.format(p, c, date, data)) if data: Bytype.put_data(p, c, date, data, commit=False) data = crashes_categories.get(p, c, date=date) Categories.put_data(p, c, data, commit=False) else: # no ADI or no crash data logger.info('No ADI or no crash data for {}::{}'.format( p, c)) update_lastdate = False if update_lastdate and \ magutils.get_date(last) < magutils.get_date(date): logger.info('Lastdate updated to: {}'.format(date)) Lastdate.put(date, commit=False) db.session.commit()
def get(channel, versions=None, product='Firefox', start_date=None, end_date='today', duration=30, platforms=None): if not isinstance(versions, list): if isinstance(versions, numbers.Number): versions = socorro.ProductVersions.get_active(vnumber=versions, product=product) else: versions = socorro.ProductVersions.get_active(product=product) versions = versions[channel.lower()] if start_date: _sdate = utils.get_date_ymd(start_date) _edate = utils.get_date_ymd(end_date) duration = (_edate - _sdate).days adi = socorro.ADI.get(version=versions, product=product, end_date=end_date, duration=duration, platforms=platforms) data = {} for d, n in adi.items(): data[d] = {'adi': n, 'browser': 0, 'content': 0, 'plugin': 0, 'browser_rate': 0, 'content_rate': 0, 'b+c_rate': 0, 'plugin_rate': 0} start_date = utils.get_date(end_date, duration) search_date = socorro.SuperSearch.get_search_date(start_date, end_date) socorro.SuperSearch(params={'product': product, 'version': versions, 'release_channel': channel, 'date': search_date, '_results_number': 0, '_facets_size': 2, # 2 is for a facet on plugin and on content '_histogram.date': ['process_type']}, handler=__super_search_handler, handlerdata=data).wait() return data
def get_crash_positions(limit, product, versions, channel, search_date='', end_date='today', verbose=False): def handler_ss(chan, json, data): if json['errors']: __warn('Error in getting ranks for channel %s: %s' % (chan, str(json['errors'])), verbose) signatures = {} for sgn in json['facets']['signature']: total = sgn['count'] signature = sgn['term'] content = 0 plugin = 0 gpu = 0 for pt in sgn['facets']['process_type']: N = pt['count'] ty = pt['term'] if ty == 'content': content += N elif ty == 'plugin': plugin += N elif ty == 'gpu': gpu += N else: __warn('Unknown process type: %s' % ty) browser = total - content - plugin - gpu signatures[signature] = {'browser': browser, 'content': content, 'plugin': plugin, 'gpu': gpu} # now we sort the data according to the crash volume types = ['browser', 'content', 'plugin', 'gpu'] rank = [sorted(signatures.items(), key=lambda t: (-t[1][typ], t[0])) for typ in types] rank = [{r[i][0]: i + 1 for i in range(len(r))} for r in rank] for s, v in signatures.items(): signatures[s] = {'browser': -1 if v['browser'] == 0 else rank[0][s], 'content': -1 if v['content'] == 0 else rank[1][s], 'plugin': -1 if v['plugin'] == 0 else rank[2][s], 'gpu': -1 if v['gpu'] == 0 else rank[2][s]} data[chan] = signatures queries = [] data = {} if not search_date: search_date = socorro.SuperSearch.get_search_date(utils.get_date(end_date, 7)) if limit == -1: limit = 10000 for chan in channel: data[chan] = {} queries.append(Query(socorro.SuperSearch.URL, {'product': product, 'version': versions[chan], 'release_channel': chan, 'date': search_date, '_aggs.signature': 'process_type', '_facets': 'signature', '_facets_size': limit, '_results_number': 0}, handler=functools.partial(handler_ss, chan), handlerdata=data)) return socorro.SuperSearch(queries=queries), data
def get_bz_params(self, date): tomorrow = lmdutils.get_date('tomorrow') start_date, _ = self.get_dates(date) fields = ['creator'] params = { 'include_fields': fields, 'resolution': '---', 'f1': 'longdesc', 'o1': 'regexp', 'v1': '([[:<:]]{}[[:>:]])|([[:<:]]{}?[[:>:]])|({}?[ \t]+{}[ \t]+{})[ \t]*:'.format( *map(utils.bz_ignore_case, ('str', 'steps', 'steps', 'to', 'reproduce')) ), 'f2': 'cf_has_str', 'o2': 'equals', 'v2': '---', 'n3': 1, 'f3': 'cf_has_str', 'o3': 'changedbefore', 'v3': tomorrow, 'f4': 'creation_ts', 'o4': 'greaterthan', 'v4': start_date, 'f5': 'keywords', 'o5': 'notsubstring', 'v5': 'testcase-wanted', } return params
def analyze_gfx_critical_errors(signature='', product='Firefox', channel=['all'], versions=[], start_date=''): if product.lower() == 'firefox': product = 'Firefox' if channel == [] or channel[0].lower() == 'all': channel = ['release', 'beta', 'nightly'] if product == 'Firefox': channel.append('esr') else: channel = [c.lower() for c in channel] if not versions: base_versions = libmozdata.versions.get(base=True) versions_by_channel = socorro.ProductVersions.get_info_from_major( base_versions, product=product) versions = [] for v1 in versions_by_channel.values(): for v2 in v1: versions.append(v2['version']) if not start_date: start_date = utils.get_date('today', 7) gfx_critical_errors = get_critical_errors() count = {} def handler(json, gfx_critical_error): count[gfx_critical_error] = json['total'] base_params = { 'product': product, 'release_channel': channel, 'version': versions, 'date': '>=' + start_date, '_results_number': 0, '_facets_size': 0, } if signature: base_params['signature'] = signature queries = [] for gfx_critical_error in gfx_critical_errors: params = base_params.copy() params['graphics_critical_error'] = '~' + gfx_critical_error queries.append( Query(socorro.SuperSearch.URL, params=params, handler=handler, handlerdata=gfx_critical_error)) socorro.SuperSearch(queries=queries).wait() return count
def get_search_date(search_start_date, start_date, end_date=utils.get_date('today')): if search_start_date: return socorro.SuperSearch.get_search_date(search_start_date, end_date) else: return socorro.SuperSearch.get_search_date( utils.get_date_str(start_date), end_date)
def update(date='today'): d = lmdutils.get_date(date) logger.info('Update data for {}: started.'.format(d)) data, bids, ratios, ranges, last_date = get(date=date) models.Signatures.put_data(data, bids, ratios) models.Signatures.clean(ranges) models.Lastdate.set(last_date) logger.info('Update data for {}: finished.'.format(d))
def analyze_gfx_critical_errors(signature='', product='Firefox', channel=['all'], versions=[], start_date=''): if product.lower() == 'firefox': product = 'Firefox' if channel == [] or channel[0].lower() == 'all': channel = ['release', 'beta', 'aurora', 'nightly'] if product == 'Firefox': channel.append('esr') else: channel = [c.lower() for c in channel] if not versions: base_versions = libmozdata.versions.get(base=True) versions_by_channel = socorro.ProductVersions.get_info_from_major(base_versions, product=product) versions = [] for v1 in versions_by_channel.values(): for v2 in v1: versions.append(v2['version']) if not start_date: start_date = utils.get_date('today', 7) gfx_critical_errors = get_critical_errors() count = {} def handler(json, gfx_critical_error): count[gfx_critical_error] = json['total'] base_params = { 'product': product, 'release_channel': channel, 'version': versions, 'date': '>=' + start_date, '_results_number': 0, '_facets_size': 0, } if signature: base_params['signature'] = signature queries = [] for gfx_critical_error in gfx_critical_errors: params = base_params.copy() params['graphics_critical_error'] = '~' + gfx_critical_error queries.append(Query(socorro.SuperSearch.URL, params=params, handler=handler, handlerdata=gfx_critical_error)) socorro.SuperSearch(queries=queries).wait() return count
def prepare(spikes, bugs_by_signature, date, versions, query, ndays): if spikes: affected_chans = set() results = OrderedDict() today = utils.get_date(date) params = sputils.get_params_for_link(date, query=query) for product in sputils.get_products(): if product in spikes: params['product'] = product data1 = spikes[product] results1 = OrderedDict() results[product] = results1 version_prod = versions[product] for chan in sputils.get_channels(): if chan in data1: affected_chans.add(chan) params['release_channel'] = chan if version_prod: params['version'] = version_prod[chan] results2 = OrderedDict() results1[chan] = results2 for stats in sorted( data1[chan], reverse=True, key=lambda d: (d['diff'], d['numbers'][-1], d['signature'])): sgn = stats['signature'] sgn = sputils.get_str(sgn) params['signature'] = sputils.get_esearch_sgn(sgn) url = socorro.SuperSearch.get_link(params) url += '#crash-reports' bugs = bugs_by_signature.get(sgn, {}) results3 = OrderedDict() results2[sgn] = results3 numbers = stats['numbers'] results3['numbers'] = sputils.make_numbers( today, numbers, ndays) results3['resolved'] = bugs.get('resolved', None) results3['unresolved'] = bugs.get( 'unresolved', None) results3['url'] = url affected_chans = list(sorted(affected_chans)) return results, affected_chans, today return None
def get_bz_params(self, date): tomorrow = lmdutils.get_date('tomorrow') start_date, _ = self.get_dates(date) fields = ['creator'] params = { 'include_fields': fields, 'resolution': '---', 'f1': 'longdesc', 'o1': 'regexp', 'v1': '([[:<:]]{}[[:>:]])|([[:<:]]{}?[[:>:]])|({}?[ \t]+{}[ \t]+{})[ \t]*:' .format(*map(utils.bz_ignore_case, ('str', 'steps', 'steps', 'to', 'reproduce'))), 'f2': 'cf_has_str', 'o2': 'equals', 'v2': '---', 'n3': 1, 'f3': 'cf_has_str', 'o3': 'changedbefore', 'v3': tomorrow, 'f4': 'creation_ts', 'o4': 'greaterthan', 'v4': start_date, 'f5': 'keywords', 'o5': 'notsubstring', 'v5': 'testcase-wanted', } return params
def get_bz_params(self, date): tomorrow = lmdutils.get_date("tomorrow") start_date, _ = self.get_dates(date) fields = ["creator"] params = { "include_fields": fields, "resolution": "---", "f1": "longdesc", "o1": "regexp", "v1": "([[:<:]]{}[[:>:]])|([[:<:]]{}?[[:>:]])|({}?[ \t]+{}[ \t]+{})[ \t]*:" .format(*map(utils.bz_ignore_case, ("str", "steps", "steps", "to", "reproduce"))), "f2": "cf_has_str", "o2": "equals", "v2": "---", "n3": 1, "f3": "cf_has_str", "o3": "changedbefore", "v3": tomorrow, "f4": "creation_ts", "o4": "greaterthan", "v4": start_date, "f5": "keywords", "o5": "notsubstring", "v5": "testcase-wanted", } return params
def send_email(self, date='today'): """Send the email""" if date: date = lmdutils.get_date(date) d = lmdutils.get_date_ymd(date) if isinstance(self, Nag): self.nag_date = d if not self.must_run(d): return if not self.has_enough_data(): logger.info('The tool {} hasn\'t enough data to run'.format( self.name())) return login_info = utils.get_login_info() title, body = self.get_email(date) if title: receivers = self.get_receivers() status = 'Success' try: mail.send( login_info['ldap_username'], receivers, title, body, html=True, login=login_info, dryrun=self.dryrun, ) except: # NOQA logger.exception('Tool {}'.format(self.name())) status = 'Failure' db.Email.add(self.name(), receivers, 'global', status) if isinstance(self, Nag): self.send_mails(title, dryrun=self.dryrun) else: name = self.name().upper() if date: logger.info('{}: No data for {}'.format(name, date)) else: logger.info('{}: No data'.format(name)) logger.info('Query: {}'.format(self.query_url))
def send_email(self, date="today"): """Send the email""" if date: date = lmdutils.get_date(date) d = lmdutils.get_date_ymd(date) if isinstance(self, Nag): self.nag_date = d if not self.must_run(d): return if not self.has_enough_data(): logger.info("The tool {} hasn't enough data to run".format( self.name())) return login_info = utils.get_login_info() title, body = self.get_email(date) if title: receivers = self.get_receivers() status = "Success" try: mail.send( login_info["ldap_username"], receivers, title, body, html=True, login=login_info, dryrun=self.dryrun, ) except Exception: logger.exception("Tool {}".format(self.name())) status = "Failure" db.Email.add(self.name(), receivers, "global", status) if isinstance(self, Nag): self.send_mails(title, dryrun=self.dryrun) else: name = self.name().upper() if date: logger.info("{}: No data for {}".format(name, date)) else: logger.info("{}: No data".format(name)) logger.info("Query: {}".format(self.query_url))
def send_email(self, date='today'): """Send the email""" if date: date = lmdutils.get_date(date) d = lmdutils.get_date_ymd(date) if isinstance(self, Nag): self.nag_date = d if not self.must_run(d): return if not self.has_enough_data(): logger.info('The tool {} hasn\'t enough data to run'.format(self.name())) return login_info = utils.get_login_info() title, body = self.get_email(date) if title: receivers = self.get_receivers() status = 'Success' try: mail.send( login_info['ldap_username'], receivers, title, body, html=True, login=login_info, dryrun=self.dryrun, ) except: # NOQA logger.exception('Tool {}'.format(self.name())) status = 'Failure' db.Email.add(self.name(), receivers, 'global', status) if isinstance(self, Nag): self.send_mails(title, dryrun=self.dryrun) else: name = self.name().upper() if date: logger.info('{}: No data for {}'.format(name, date)) else: logger.info('{}: No data'.format(name)) logger.info('Query: {}'.format(self.query_url))
def send(): path = utils.get_config("common", "log") try: n = os.path.getsize(path) if n != 0: login_info = utils.get_login_info() date = lmdutils.get_date("today") msg, files = get_msg(path) mail.send( login_info["ldap_username"], utils.get_config("common", "on-errors"), "[autonag] Something bad happened when running auto-nag the {}" .format(date), msg, html=False, login=login_info, dryrun=False, files=files, ) except Exception: pass
def send(): path = utils.get_config('common', 'log') try: n = os.path.getsize(path) if n != 0: login_info = utils.get_login_info() date = lmdutils.get_date('today') msg, files = get_msg(path) mail.send( login_info['ldap_username'], utils.get_config('common', 'on-errors'), '[autonag] Something bad happened when running auto-nag the {}'.format( date ), msg, html=False, login=login_info, dryrun=False, files=files, ) except Exception: pass
def send_email(self, date='today', dryrun=False): """Send the email""" if date: date = lmdutils.get_date(date) d = lmdutils.get_date_ymd(date) if isinstance(self, Nag): self.nag_date = d if not self.must_run(d): return if not self.has_enough_data(): logger.info('The tool {} hasn\'t enough data to run'.format( self.name())) return login_info = utils.get_login_info() title, body = self.get_email(login_info['bz_api_key'], date, dryrun) if title: mail.send( login_info['ldap_username'], utils.get_config(self.name(), 'receivers'), title, body, html=True, login=login_info, dryrun=dryrun, ) if isinstance(self, Nag): self.send_mails(title, dryrun=dryrun) else: name = self.name().upper() if date: logger.info('{}: No data for {}'.format(name, date)) else: logger.info('{}: No data'.format(name))
def generate_bug_report(sgn, info, status_flags_by_channel, base_versions, start_date_by_channel, end_date, check_for_fx=True): data = {} if not check_for_fx or info['firefox']: volumes = default_volumes.copy() data = {} for channel, volume in info['affected']: data[status_flags_by_channel[channel]] = 'affected' volumes[channel] = volume for channel, volume in info['leftovers']: volumes[channel] = volume # We begin with the crash volume comment = 'Crash volume for signature \'%s\':\n' % sgn table = [] for chan, volume in sorted(volumes.items(), key=lambda k: channel_order[k[0]]): version = base_versions[chan] start_date = start_date_by_channel[chan] plural = 'es' if volume != 1 else '' table.append(['- %s' % chan, '(version %d):' % version, '%d crash%s from %s.' % (volume, plural, utils.get_date(start_date))]) comment += __mk_volume_table(table, 'global') # Make the table for the trend table = [] empty = False N = -1 for chan, trend in sorted(info['trend'].items(), key=lambda k: channel_order[k[0]]): if len(trend) >= 1: # we remove data for this week del(trend[0]) if len(trend) >= 8: # keep only the last seven weeks trend = trend[:7] if not trend: empty = True break N = max(N, len(trend)) row = [str(n) for n in trend] row.insert(0, '- %s' % chan) table.append(row) if not empty: # we've trends monday, sunday = utils.get_monday_sunday(utils.get_date_ymd(end_date)) comment += '\n\nCrash volume on the last weeks (Week N is from %s to %s):\n' % (monday.strftime('%m-%d'), sunday.strftime('%m-%d')) headers = [''] for w in range(1, N + 1): headers.append('W. N-%d' % w) comment += __mk_volume_table(table, 'byweek', headers=headers) # Add affected platforms platforms = info['platforms'] if platforms: comment += '\n\nAffected platform' if len(platforms) >= 2: comment += 's' platforms = sorted(platforms, key=lambda k: platform_order[k]) comment += ': ' + ', '.join(platforms) ranks = info['rank'] if ranks: # check if we've ranks empty = True for types in ranks.values(): for v in types.values(): if v != -1: empty = False break if not empty: comment += '\n\nCrash rank on the last 7 days:\n' headers = ['', 'Browser', 'Content', 'Plugin'] table = [] def fmt_rank(s): return None if s == -1 else '#' + str(s) for chan, types in sorted(ranks.items(), key=lambda k: channel_order[k[0]]): table.append(['- %s' % chan, fmt_rank(types['browser']), fmt_rank(types['content']), fmt_rank(types['plugin'])]) comment += __mk_volume_table(table, 'rank', headers=headers) data['comment'] = {'body': comment} return data
def get(product='Firefox', limit=1000, verbose=False, search_start_date='', end_date=None, signatures=[], bug_ids=[], max_bugs=-1, base_versions=None, check_for_fx=True, check_bz_version=True, check_noisy=True): """Get crashes info Args: product (Optional[str]): the product limit (Optional[int]): the number of crashes to get from tcbs Returns: dict: contains all the info about how to update flags """ p = product.lower() if p == 'firefox': product = 'Firefox' elif p == 'fennecandroid': product = 'FennecAndroid' channel = ['release', 'beta', 'aurora', 'nightly'] if product == 'Firefox': channel.append('esr') start_date, min_date, versions_by_channel, start_date_by_channel, base_versions = get_versions_info(product, date=end_date, base_versions=base_versions) nv = Bugzilla.get_nightly_version() if check_bz_version and nv != base_versions['nightly']: __warn('Mismatch between nightly version from Bugzilla (%d) and Socorro (%d)' % (nv, base_versions['nightly']), verbose) return None if check_bz_version and (base_versions['aurora'] != nv - 1 or base_versions['beta'] != nv - 2 or base_versions['release'] != nv - 3): __warn('All versions are not up to date (Bugzilla nightly version is %d): %s' % (nv, base_versions), verbose) return None __warn('Versions: %s' % versions_by_channel, verbose) __warn('Start dates: %s' % start_date_by_channel, verbose) if not end_date: end_date = utils.get_date('today') search_date = get_search_date(search_start_date, start_date, end_date) signatures = get_signatures(limit, product, versions_by_channel, channel, search_date, signatures, bug_ids, verbose) # signatures == { 'foo::bar': {'affected_channels': [('release', 1234), ...], # 'bugs': None, # 'platforms': ['Windows'], # 'selected_bug': None}, ... } __warn('Collected signatures: %d' % len(signatures), verbose) # get the bugs for each signatures bugs_by_signature = socorro.Bugs.get_bugs(list(signatures.keys())) # if we've some bugs in bug_ids then we must remove the other ones for a given signature if bug_ids: bids = set(bug_ids) for s, bugids in bugs_by_signature.items(): inter = bids.intersection(bugids) if inter: bugs_by_signature[s] = inter __warn('Collected bugs in Socorro: Ok', verbose) bugs, bugs_count = reduce_set_of_bugs(bugs_by_signature) __warn('Remove duplicates: Ok', verbose) __warn('Bugs to analyze: %d' % bugs_count, verbose) # we filter the bugs to remove meaningless ones if not bug_ids: bugs = filter_bugs(bugs, product) status_flags = Bugzilla.get_status_flags(base_versions=base_versions) # we get the "better" bug where to update the info bugs_history_info = get_bugs_info(bugs, status_flags) patched_bugs = [] for bugid, hinfo in bugs_history_info.items(): if hinfo['patched']: patched_bugs.append(bugid) if patched_bugs: patch_info = dataanalysis.analyze_bugs(patched_bugs, min_date=min_date, base_versions=base_versions) else: patch_info = {} crashes_to_reopen = [] bugs.clear() for s, v in bugs_by_signature.items(): info = signatures[s] no_change = set() if v: bug_to_touch = get_last_bug(v, s, signatures[s], patch_info, bugs_history_info, min_date) if bug_to_touch: no_change = bugs_history_info[bug_to_touch]['no_change'] else: crashes_to_reopen.append(s) else: bug_to_touch = None info['selected_bug'] = bug_to_touch info['bugs'] = v info['no_change'] = no_change if bug_to_touch: bugs.add(bug_to_touch) __warn('Collected last bugs: %d' % len(bugs), verbose) # add bug info in signatures add_bug_info(signatures, list(bugs), status_flags, product, verbose) # analyze the signatures analysis = analyze(signatures, status_flags, base_versions) if max_bugs > 0: __analysis = {} count = 0 for signature, info in analysis.items(): if not check_for_fx or info['firefox']: __analysis[signature] = info count += 1 if count == max_bugs: analysis = __analysis break __warn('Analysis: Ok', verbose) positions_result, positions = get_crash_positions(-1, product, versions_by_channel, channel, search_date=search_date, verbose=verbose) # Now get the number of crashes for each signature trends = get_stats_for_past_weeks(product, channel, start_date_by_channel, versions_by_channel, analysis, search_start_date, end_date, check_for_fx=check_for_fx) if check_noisy: noisy = get_noisy(trends, analysis) __warn('Noisy signatures: %s' % [analysis[s] for s in noisy], verbose) else: noisy = set() __warn('Collected trends: Ok\n', verbose) positions_result.wait() # replace dictionary containing trends by a list empty_ranks = {'browser': -1, 'content': -1, 'plugin': -1, 'gpu': -1} for signature, i in trends.items(): if signature in noisy: del analysis[signature] else: signature_info = analysis[signature] ranks = signature_info['rank'] for chan, trend in i.items(): i[chan] = [trend[week] for week in sorted(trend.keys(), reverse=False)] ranks[chan] = positions[chan].get(signature, empty_ranks) signature_info['trend'] = i __prettywarn(analysis, verbose) return {'status_flags': status_flags, 'base_versions': base_versions, 'start_dates': start_date_by_channel, 'signatures': analysis, 'end_date': end_date}
def get_search_date(search_start_date, start_date, end_date=utils.get_date('today')): if search_start_date: return socorro.SuperSearch.get_search_date(search_start_date, end_date) else: return socorro.SuperSearch.get_search_date(utils.get_date_str(start_date), end_date)
def get(product='Firefox', limit=1000, verbose=False, search_start_date='', end_date=None, signatures=[], bug_ids=[], max_bugs=-1, base_versions=None, check_for_fx=True, check_bz_version=True, check_noisy=True): """Get crashes info Args: product (Optional[str]): the product limit (Optional[int]): the number of crashes to get from tcbs Returns: dict: contains all the info about how to update flags """ p = product.lower() if p == 'firefox': product = 'Firefox' elif p == 'fennecandroid': product = 'FennecAndroid' channel = ['release', 'beta', 'aurora', 'nightly'] if product == 'Firefox': channel.append('esr') start_date, min_date, versions_by_channel, start_date_by_channel, base_versions = get_versions_info( product, date=end_date, base_versions=base_versions) nv = Bugzilla.get_nightly_version() if check_bz_version and nv != base_versions['nightly']: __warn( 'Mismatch between nightly version from Bugzilla (%d) and Socorro (%d)' % (nv, base_versions['nightly']), verbose) return None if check_bz_version and (base_versions['aurora'] != nv - 1 or base_versions['beta'] != nv - 2 or base_versions['release'] != nv - 3): __warn( 'All versions are not up to date (Bugzilla nightly version is %d): %s' % (nv, base_versions), verbose) return None __warn('Versions: %s' % versions_by_channel, verbose) __warn('Start dates: %s' % start_date_by_channel, verbose) if not end_date: end_date = utils.get_date('today') search_date = get_search_date(search_start_date, start_date, end_date) signatures = get_signatures(limit, product, versions_by_channel, channel, search_date, signatures, bug_ids, verbose) # signatures == { 'foo::bar': {'affected_channels': [('release', 1234), ...], # 'bugs': None, # 'platforms': ['Windows'], # 'selected_bug': None}, ... } __warn('Collected signatures: %d' % len(signatures), verbose) # get the bugs for each signatures bugs_by_signature = socorro.Bugs.get_bugs(list(signatures.keys())) # if we've some bugs in bug_ids then we must remove the other ones for a given signature if bug_ids: bids = set(bug_ids) for s, bugids in bugs_by_signature.items(): inter = bids.intersection(bugids) if inter: bugs_by_signature[s] = inter __warn('Collected bugs in Socorro: Ok', verbose) bugs, bugs_count = reduce_set_of_bugs(bugs_by_signature) __warn('Remove duplicates: Ok', verbose) __warn('Bugs to analyze: %d' % bugs_count, verbose) # we filter the bugs to remove meaningless ones if not bug_ids: bugs = filter_bugs(bugs, product) status_flags = Bugzilla.get_status_flags(base_versions=base_versions) # we get the "better" bug where to update the info bugs_history_info = get_bugs_info(bugs, status_flags) patched_bugs = [] for bugid, hinfo in bugs_history_info.items(): if hinfo['patched']: patched_bugs.append(bugid) if patched_bugs: patch_info = dataanalysis.analyze_bugs(patched_bugs, min_date=min_date, base_versions=base_versions) else: patch_info = {} crashes_to_reopen = [] bugs.clear() for s, v in bugs_by_signature.items(): info = signatures[s] no_change = set() if v: bug_to_touch = get_last_bug(v, s, signatures[s], patch_info, bugs_history_info, min_date) if bug_to_touch: no_change = bugs_history_info[bug_to_touch]['no_change'] else: crashes_to_reopen.append(s) else: bug_to_touch = None info['selected_bug'] = bug_to_touch info['bugs'] = v info['no_change'] = no_change if bug_to_touch: bugs.add(bug_to_touch) __warn('Collected last bugs: %d' % len(bugs), verbose) # add bug info in signatures add_bug_info(signatures, list(bugs), status_flags, product, verbose) # analyze the signatures analysis = analyze(signatures, status_flags, base_versions) if max_bugs > 0: __analysis = {} count = 0 for signature, info in analysis.items(): if not check_for_fx or info['firefox']: __analysis[signature] = info count += 1 if count == max_bugs: analysis = __analysis break __warn('Analysis: Ok', verbose) positions_result, positions = get_crash_positions(-1, product, versions_by_channel, channel, search_date=search_date, verbose=verbose) # Now get the number of crashes for each signature trends = get_stats_for_past_weeks(product, channel, start_date_by_channel, versions_by_channel, analysis, search_start_date, end_date, check_for_fx=check_for_fx) if check_noisy: noisy = get_noisy(trends, analysis) __warn('Noisy signatures: %s' % [analysis[s] for s in noisy], verbose) else: noisy = set() __warn('Collected trends: Ok\n', verbose) positions_result.wait() # replace dictionary containing trends by a list empty_ranks = {'browser': -1, 'content': -1, 'plugin': -1, 'gpu': -1} for signature, i in trends.items(): if signature in noisy: del analysis[signature] else: signature_info = analysis[signature] ranks = signature_info['rank'] for chan, trend in i.items(): i[chan] = [ trend[week] for week in sorted(trend.keys(), reverse=False) ] ranks[chan] = positions[chan].get(signature, empty_ranks) signature_info['trend'] = i __prettywarn(analysis, verbose) return { 'status_flags': status_flags, 'base_versions': base_versions, 'start_dates': start_date_by_channel, 'signatures': analysis, 'end_date': end_date }
def get_sgns_info(sgns_by_chan, product='Firefox', date='today', query={}, versions=None): today = utils.get_date(date) tomorrow = utils.get_date(date, -1) data = {chan: {} for chan in sgns_by_chan.keys()} def handler(json, data): if json['errors'] or not json['facets']['signature']: return for facets in json['facets']['signature']: sgn = facets['term'] count = facets['count'] platforms = defaultdict(lambda: 0) startup = {True: 0, False: 0} data[sgn] = { 'count': count, 'platforms': platforms, 'startup': startup } facets = facets['facets'] for ppv in facets['platform_pretty_version']: platforms[ppv['term']] += ppv['count'] for sc in facets['startup_crash']: term = sc['term'] startup[term == 'T'] += sc['count'] base = { 'product': product, 'date': ['>=' + today, '<' + tomorrow], 'release_channel': '', 'version': '', 'signature': '', '_aggs.signature': ['platform_pretty_version', 'startup_crash'], '_results_number': 0, '_facets_size': 100 } base.update(query) searches = [] for chan, signatures in sgns_by_chan.items(): params = copy.copy(base) params['release_channel'] = chan if versions: params['version'] = versions[chan] for sgns in Connection.chunks(signatures, 10): p = copy.copy(params) p['signature'] = ['=' + s for s in sgns] searches.append( socorro.SuperSearch(params=p, handler=handler, handlerdata=data[chan])) for s in searches: s.wait() return data
def test_get_date(self): self.assertEqual(utils.get_date('1991/04/16'), '1991-04-16') self.assertEqual(utils.get_date('1991/04/16', 1), '1991-04-15')
def get_correct_date(date): date = get_date(date) if date: return utils.get_date_str(date) return utils.get_date('today')
def get(signature, matching_mode, module, addon, product='Firefox', channel=['all'], versions=[], start_date='', limit=0, check_bt=False, verbose=False, ratio=1.): if product.lower() == 'firefox': product = 'Firefox' if channel == [] or channel[0].lower() == 'all': channel = ['release', 'beta', 'aurora', 'nightly'] if product == 'Firefox': channel.append('esr') else: channel = [c.lower() for c in channel] if not versions: base_versions = libmozdata.versions.get(base=True) versions_by_channel = socorro.ProductVersions.get_info_from_major( base_versions, product=product) versions = [] for v1 in versions_by_channel.values(): for v2 in v1: versions.append(v2['version']) if not start_date: start_date = utils.get_date('today', 7) if limit <= 0: count = [] socorro.SuperSearch( params={ 'product': product, 'version': versions, 'signature': matching_mode + signature, 'release_channel': channel, 'date': '>=' + start_date, '_facets_size': 1, '_results_number': 0 }, handler=lambda json: count.append(json['total'])).wait() limit = count[0] if limit == 0: return {'limit': 0} def handler_ss(json, data): if json['errors']: print('Errors occured: %s' % json['errors']) if json['total']: for signature in json['facets']['signature']: for hit in signature['facets']['uuid']: data.append(hit['term']) uuids = [] socorro.SuperSearch(params={ 'product': product, 'version': versions, 'signature': matching_mode + signature, 'release_channel': channel, 'date': '>=' + start_date, '_aggs.signature': 'uuid', '_facets_size': limit, '_results_number': 0 }, handler=handler_ss, handlerdata=uuids).wait() uuids = utils.get_sample(uuids, ratio) limit = len(uuids) module = [m.lower() for m in module] addon = [a.lower() for a in addon] if verbose: count = [0] def handler_pc(json, data): if verbose: count[0] += 1 if count[0] % 100 == 0: print('Treated reports: %d' % count[0]) addon_version = '' if addon: for a in json.get('addons', []): addon_id = a[0].lower() if len(a) == 2 and addon_id in addon: versions = data['versions'] addon_version = a[1] versions[addon_id][addon_version] += 1 break if not addon_version: data['not_in_addon'].append(json['uuid']) if module: dll_version = '' json_dump = json['json_dump'] for m in json_dump.get('modules', []): filename = m['filename'].lower() if filename in module: versions = data['versions'] debug_ids = data['debug_ids'] dll_version = m['version'] debug_id = m['debug_id'] versions[filename][dll_version] += 1 debug_ids[filename][debug_id] += 1 # if addon_version and dll_version and (addon_version == dll_version): # data['match'].append(json['uuid']) if check_bt and 'crashing_thread' in json_dump: crashing_thread = json_dump['crashing_thread'] in_bt = False for frame in crashing_thread['frames']: if frame['module'].lower() in module: in_bt = True break if not in_bt: data['not_in_bt'].append(json['uuid']) info = { 'versions': defaultdict(lambda: defaultdict(int)), 'debug_ids': defaultdict(lambda: defaultdict(int)), 'limit': limit, 'not_in_bt': [], 'not_in_addon': [], 'match': [] } queries = [] print(str(len(uuids)) + ' reports will be analyzed.') for uuid in uuids: queries.append( Query(socorro.ProcessedCrash.URL, params={'crash_id': uuid}, handler=handler_pc, handlerdata=info)) socorro.ProcessedCrash(queries=queries).wait() return info
def get_crash_positions(limit, product, versions, channel, search_date='', end_date='today', verbose=False): def handler_ss(chan, json, data): if json['errors']: __warn( 'Error in getting ranks for channel %s: %s' % (chan, str(json['errors'])), verbose) signatures = {} for sgn in json['facets']['signature']: total = sgn['count'] signature = sgn['term'] content = 0 plugin = 0 gpu = 0 for pt in sgn['facets']['process_type']: N = pt['count'] ty = pt['term'] if ty == 'content': content += N elif ty == 'plugin': plugin += N elif ty == 'gpu': gpu += N else: __warn('Unknown process type: %s' % ty) browser = total - content - plugin - gpu signatures[signature] = { 'browser': browser, 'content': content, 'plugin': plugin, 'gpu': gpu } # now we sort the data according to the crash volume types = ['browser', 'content', 'plugin', 'gpu'] rank = [ sorted(signatures.items(), key=lambda t: (-t[1][typ], t[0])) for typ in types ] rank = [{r[i][0]: i + 1 for i in range(len(r))} for r in rank] for s, v in signatures.items(): signatures[s] = { 'browser': -1 if v['browser'] == 0 else rank[0][s], 'content': -1 if v['content'] == 0 else rank[1][s], 'plugin': -1 if v['plugin'] == 0 else rank[2][s], 'gpu': -1 if v['gpu'] == 0 else rank[2][s] } data[chan] = signatures queries = [] data = {} if not search_date: search_date = socorro.SuperSearch.get_search_date( utils.get_date(end_date, 7)) if limit == -1: limit = 10000 for chan in channel: data[chan] = {} queries.append( Query(socorro.SuperSearch.URL, { 'product': product, 'version': versions[chan], 'release_channel': chan, 'date': search_date, '_aggs.signature': 'process_type', '_facets': 'signature', '_facets_size': limit, '_results_number': 0 }, handler=functools.partial(handler_ss, chan), handlerdata=data)) return socorro.SuperSearch(queries=queries), data
def generate_bug_report(sgn, info, status_flags_by_channel, base_versions, start_date_by_channel, end_date, check_for_fx=True): data = {} if not check_for_fx or info['firefox']: volumes = default_volumes.copy() data = {} for channel, volume in info['affected']: data[status_flags_by_channel[channel]] = 'affected' volumes[channel] = volume for channel, volume in info['leftovers']: volumes[channel] = volume # We begin with the crash volume comment = 'Crash volume for signature \'%s\':\n' % sgn table = [] for chan, volume in sorted(volumes.items(), key=lambda k: channel_order[k[0]]): version = base_versions[chan] start_date = start_date_by_channel[chan] plural = 'es' if volume != 1 else '' table.append([ '- %s' % chan, '(version %d):' % version, '%d crash%s from %s.' % (volume, plural, utils.get_date(start_date)) ]) comment += __mk_volume_table(table, 'global') # Make the table for the trend table = [] empty = False N = -1 for chan, trend in sorted(info['trend'].items(), key=lambda k: channel_order[k[0]]): if len(trend) >= 1: # we remove data for this week del (trend[0]) if len(trend) >= 8: # keep only the last seven weeks trend = trend[:7] if not trend: empty = True break N = max(N, len(trend)) row = [str(n) for n in trend] row.insert(0, '- %s' % chan) table.append(row) if not empty: # we've trends monday, sunday = utils.get_monday_sunday( utils.get_date_ymd(end_date)) comment += '\n\nCrash volume on the last weeks (Week N is from %s to %s):\n' % ( monday.strftime('%m-%d'), sunday.strftime('%m-%d')) headers = [''] for w in range(1, N + 1): headers.append('W. N-%d' % w) comment += __mk_volume_table(table, 'byweek', headers=headers) # Add affected platforms platforms = info['platforms'] if platforms: comment += '\n\nAffected platform' if len(platforms) >= 2: comment += 's' platforms = sorted(platforms, key=lambda k: platform_order[k]) comment += ': ' + ', '.join(platforms) ranks = info['rank'] if ranks: # check if we've ranks empty = True for types in ranks.values(): for v in types.values(): if v != -1: empty = False break if not empty: comment += '\n\nCrash rank on the last 7 days:\n' headers = ['', 'Browser', 'Content', 'Plugin'] table = [] def fmt_rank(s): return None if s == -1 else '#' + str(s) for chan, types in sorted(ranks.items(), key=lambda k: channel_order[k[0]]): table.append([ '- %s' % chan, fmt_rank(types['browser']), fmt_rank(types['content']), fmt_rank(types['plugin']) ]) comment += __mk_volume_table(table, 'rank', headers=headers) data['comment'] = {'body': comment} return data
def get(signature, matching_mode, module, addon, product='Firefox', channel=['all'], versions=[], start_date='', limit=0, check_bt=False, verbose=False, ratio=1.): if product.lower() == 'firefox': product = 'Firefox' if channel == [] or channel[0].lower() == 'all': channel = ['release', 'beta', 'aurora', 'nightly'] if product == 'Firefox': channel.append('esr') else: channel = [c.lower() for c in channel] if not versions: base_versions = libmozdata.versions.get(base=True) versions_by_channel = socorro.ProductVersions.get_info_from_major(base_versions, product=product) versions = [] for v1 in versions_by_channel.values(): for v2 in v1: versions.append(v2['version']) if not start_date: start_date = utils.get_date('today', 7) if limit <= 0: count = [] socorro.SuperSearch(params={'product': product, 'version': versions, 'signature': matching_mode + signature, 'release_channel': channel, 'date': '>=' + start_date, '_facets_size': 1, '_results_number': 0}, handler=lambda json: count.append(json['total'])).wait() limit = count[0] if limit == 0: return {'limit': 0} def handler_ss(json, data): if json['errors']: print('Errors occured: %s' % json['errors']) if json['total']: for signature in json['facets']['signature']: for hit in signature['facets']['uuid']: data.append(hit['term']) uuids = [] socorro.SuperSearch(params={'product': product, 'version': versions, 'signature': matching_mode + signature, 'release_channel': channel, 'date': '>=' + start_date, '_aggs.signature': 'uuid', '_facets_size': limit, '_results_number': 0}, handler=handler_ss, handlerdata=uuids).wait() uuids = utils.get_sample(uuids, ratio) limit = len(uuids) module = [m.lower() for m in module] addon = [a.lower() for a in addon] if verbose: count = [0] def handler_pc(json, data): if verbose: count[0] += 1 if count[0] % 100 == 0: print('Treated reports: %d' % count[0]) addon_version = '' if addon: for a in json.get('addons', []): addon_id = a[0].lower() if len(a) == 2 and addon_id in addon: versions = data['versions'] addon_version = a[1] versions[addon_id][addon_version] += 1 break if not addon_version: data['not_in_addon'].append(json['uuid']) if module: dll_version = '' json_dump = json['json_dump'] for m in json_dump.get('modules', []): filename = m['filename'].lower() if filename in module: versions = data['versions'] debug_ids = data['debug_ids'] dll_version = m['version'] debug_id = m['debug_id'] versions[filename][dll_version] += 1 debug_ids[filename][debug_id] += 1 # if addon_version and dll_version and (addon_version == dll_version): # data['match'].append(json['uuid']) if check_bt and 'crashing_thread' in json_dump: crashing_thread = json_dump['crashing_thread'] in_bt = False for frame in crashing_thread['frames']: if frame['module'].lower() in module: in_bt = True break if not in_bt: data['not_in_bt'].append(json['uuid']) info = { 'versions': defaultdict(lambda: defaultdict(int)), 'debug_ids': defaultdict(lambda: defaultdict(int)), 'limit': limit, 'not_in_bt': [], 'not_in_addon': [], 'match': [] } queries = [] print(str(len(uuids)) + ' reports will be analyzed.') for uuid in uuids: queries.append(Query(socorro.ProcessedCrash.URL, params={'crash_id': uuid}, handler=handler_pc, handlerdata=info)) socorro.ProcessedCrash(queries=queries).wait() return info