def get_versions_info(product, date='today', base_versions=libmozdata.versions.get(base=True)): if not date: date = 'today' if base_versions is None: base_versions = libmozdata.versions.get(base=True) versions_by_channel = socorro.ProductVersions.get_info_from_major(base_versions, product=product, active=None) channel_by_version = {} vbc = {} start_date_by_channel = {} start_date = utils.get_date_ymd(date) min_date = start_date six_months_ago = utils.get_date_ymd(date) - relativedelta(weeks=26) for chan, versions in versions_by_channel.items(): start_date_by_channel[chan] = utils.get_date_ymd('tomorrow') l = [] vbc[chan] = l for v in versions: vers = v['version'] if not vers.endswith('b'): # 45.0b is not a true beta version l.append(vers) channel_by_version[vers] = chan d = utils.get_date_ymd(v['start_date']) if d < start_date: start_date = max(d, six_months_ago) if d < start_date_by_channel[chan]: start_date_by_channel[chan] = max(d, six_months_ago) if chan != 'esr' and d < min_date: min_date = d return start_date, min_date, vbc, start_date_by_channel, base_versions
def get(channel, versions=None, product='Firefox', start_date=None, end_date='today', duration=30, platforms=None): if not isinstance(versions, list): if isinstance(versions, numbers.Number): versions = socorro.ProductVersions.get_active(vnumber=versions, product=product) else: versions = socorro.ProductVersions.get_active(product=product) versions = versions[channel.lower()] if start_date: _sdate = utils.get_date_ymd(start_date) _edate = utils.get_date_ymd(end_date) duration = (_edate - _sdate).days adi = socorro.ADI.get(version=versions, product=product, end_date=end_date, duration=duration, platforms=platforms) data = {} for d, n in adi.items(): data[d] = {'adi': n, 'browser': 0, 'content': 0, 'plugin': 0, 'browser_rate': 0, 'content_rate': 0, 'b+c_rate': 0, 'plugin_rate': 0} start_date = utils.get_date(end_date, duration) search_date = socorro.SuperSearch.get_search_date(start_date, end_date) socorro.SuperSearch(params={'product': product, 'version': versions, 'release_channel': channel, 'date': search_date, '_results_number': 0, '_facets_size': 2, # 2 is for a facet on plugin and on content '_histogram.date': ['process_type']}, handler=__super_search_handler, handlerdata=data).wait() return data
def test_get(self): class MyConf(libmozdata.config.Config): def get(self, section, option, default=None, type=str): if section == 'StatusFlags' and option == 'ignored': return '\'OOM | small\', \'EMPTY: no crashing thread identified; ERROR_NO_MINIDUMP_HEADER\', \'F1398665248_____________________________\'' else: return default config.set_config(MyConf()) base_versions = {'nightly': 51, 'aurora': 50, 'beta': 49, 'release': 48, 'esr': 45} info = statusflags.get('Firefox', 2, end_date='2016-09-14', base_versions=base_versions, check_for_fx=False, check_bz_version=False, check_noisy=False, verbose=False) self.assertEqual(info['base_versions']['aurora'], 50) self.assertEqual(info['base_versions']['beta'], 49) self.assertEqual(info['base_versions']['esr'], 45) self.assertEqual(info['base_versions']['nightly'], 51) self.assertEqual(info['base_versions']['release'], 48) self.assertEqual(info['start_dates']['aurora'], utils.get_date_ymd('2016-08-01')) self.assertEqual(info['start_dates']['beta'], utils.get_date_ymd('2016-08-02')) self.assertEqual(info['start_dates']['esr'], utils.get_date_ymd('2016-03-16')) self.assertEqual(info['start_dates']['nightly'], utils.get_date_ymd('2016-08-01')) self.assertEqual(info['start_dates']['release'], utils.get_date_ymd('2016-07-25')) self.assertTrue('IPCError-browser | ShutDownKill' in info['signatures']) isgn = info['signatures']['IPCError-browser | ShutDownKill'] self.assertEqual(set(isgn['leftovers']), {('aurora', 114075), ('nightly', 65105), ('beta', 33208), ('release', 897), ('esr', 23)}) self.assertEqual(isgn['bugid'], 1216774) self.assertEqual(set(isgn['bugs']), {1238657, 1151237, 1216774, 1260551, 1177484, 1173134, 1168272, 1132053, 1133597, 1167902, 1200671, 1213092, 1200646, 1213096, 1223594, 1200685, 1279293, 1206729, 1177425, 1219672, 1205467, 1240542, 1266275, 1290280, 1259125, 1164155, 1150846, 1311297, 1316867, 1289405, 1311869}) self.assertEqual(set(isgn['platforms']), {'Windows', 'Mac OS X'}) self.assertFalse(isgn['private']) def getnumbers(b, c, p, g): return {'browser': b, 'content': c, 'plugin': p, 'gpu': g} self.assertEqual(isgn['rank']['aurora'], getnumbers(-1, 1, -1, -1)) self.assertEqual(isgn['rank']['beta'], getnumbers(-1, 1, -1, -1)) self.assertEqual(isgn['rank']['esr'], getnumbers(-1, 1, -1, -1)) self.assertEqual(isgn['rank']['nightly'], getnumbers(-1, 1, -1, -1)) self.assertEqual(isgn['rank']['release'], getnumbers(-1, 10, -1, -1)) self.assertTrue(isgn['resolved']) self.assertEqual(isgn['trend']['aurora'], [5776, 18706, 21015, 20699, 22058, 20100, 5721]) self.assertEqual(isgn['trend']['beta'], [206, 784, 817, 987, 1419, 18147, 10848]) self.assertEqual(isgn['trend']['esr'], [4, 5, 2, 5, 2, 2, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) self.assertEqual(isgn['trend']['nightly'], [3280, 10192, 10261, 10294, 10631, 11974, 8471]) self.assertEqual(isgn['trend']['release'], [101, 232, 169, 134, 117, 92, 49, 1]) data = statusflags.generate_bug_report('IPCError-browser | ShutDownKill', isgn, info['status_flags'], info['base_versions'], info['start_dates'], end_date='2016-09-14', check_for_fx=False) self.assertEqual(data['comment']['body'], 'Crash volume for signature \'IPCError-browser | ShutDownKill\':\n - nightly (version 51): 65105 crashes from 2016-08-01.\n - aurora (version 50): 114075 crashes from 2016-08-01.\n - beta (version 49): 33208 crashes from 2016-08-02.\n - release (version 48): 897 crashes from 2016-07-25.\n - esr (version 45): 23 crashes from 2016-03-16.\n\nCrash volume on the last weeks (Week N is from 09-12 to 09-18):\n W. N-1 W. N-2 W. N-3 W. N-4 W. N-5 W. N-6 W. N-7\n - nightly 10192 10261 10294 10631 11974 8471\n - aurora 18706 21015 20699 22058 20100 5721\n - beta 784 817 987 1419 18147 10848\n - release 232 169 134 117 92 49 1\n - esr 5 2 5 2 2 3 2\n\nAffected platforms: Windows, Mac OS X\n\nCrash rank on the last 7 days:\n Browser Content Plugin\n - nightly #1\n - aurora #1\n - beta #1\n - release #10\n - esr #1')
def get_data(self): if self.data is None: path = self.get_path() self.data = {} if os.path.exists(path): with open(path, 'r') as In: data = json.load(In) for bugid, date in data.items(): delta = lmdutils.get_date_ymd('today') - lmdutils.get_date_ymd( date ) if delta.days < self.max_days: self.data[int(bugid)] = date return self.data
def get_who_to_nag(self, date): fallbacks = {} date = lmdutils.get_date_ymd(date) days = utils.get_config('round-robin', 'days_to_nag', 7) next_date = date + relativedelta(days=days) for cal in self.all_calendars: persons = cal.get_persons(next_date) if persons and all(p is not None for _, p in persons): continue name = cal.get_team_name() fb = cal.get_fallback_mozmail() if fb not in fallbacks: fallbacks[fb] = {} if name not in fallbacks[fb]: fallbacks[fb][name] = {'nobody': False, 'persons': []} info = fallbacks[fb][name] if not persons: info['nobody'] = True else: people_names = [n for n, p in persons if p is None] if people_names: info['persons'] += people_names return fallbacks
def get_bz_params(self, date): date = lmdutils.get_date_ymd(date) - relativedelta(weeks=self.nweeks) reporters = self.get_config('reporter_exception', default=[]) reporters = ','.join(reporters) keywords = self.get_config('keyword_exception', default=[]) keywords = ','.join(keywords) fields = ['cf_crash_signature'] params = { 'include_fields': fields, 'resolution': '---', 'f1': 'cf_crash_signature', 'o1': 'isnotempty', 'f2': 'creation_ts', 'o2': 'lessthan', 'v2': date, 'f3': 'days_elapsed', 'o3': 'greaterthan', 'v3': self.nweeks * 7, } if reporters: params.update({'f4': 'reporter', 'o4': 'nowordssubstr', 'v4': reporters}) if keywords: params.update({'f5': 'keywords', 'o5': 'nowords', 'v5': keywords}) return params
def test_cache(self): cache = Cache('test_cache', 7) cache.set_dry_run(False) bugids = [123, 456, 789] cache.add(bugids) for bugid in bugids: assert bugid in cache assert str(bugid) in cache assert 101112 not in cache assert '101112' not in cache with open(cache.get_path(), 'r') as In: data = json.load(In) for bugid in ['123', '456']: date = data[bugid] date = lmdutils.get_date_ymd(date) - relativedelta(days=8) data[bugid] = lmdutils.get_date_str(date) with open(cache.get_path(), 'w') as Out: json.dump(data, Out) cache = Cache('test_cache', 7) cache.set_dry_run(False) assert 123 not in cache assert 456 not in cache assert 789 in cache
def get_stats(self, signatures, date): def handler(json, data): if json['errors']: raise SocorroError() del json['hits'] for facet in json['facets'].get('signature', {}): data.remove(facet['term']) date = lmdutils.get_date_ymd(date) - relativedelta(weeks=self.nweeks) search_date = SuperSearch.get_search_date(date) chunks, size = self.chunkify(signatures) base = { 'date': search_date, 'signature': '', '_result_number': 0, '_facets': 'signature', '_facets_size': size, } searches = [] for chunk in chunks: params = base.copy() params['signature'] = ['=' + x for x in chunk] searches.append( SuperSearch( params=params, handler=handler, handlerdata=signatures, raise_error=True, ) ) for s in searches: s.wait()
def analyze(channel, data, date='today', max_days=3, cache=None, verbose=False): results = {} if data: bt_info = get_bt(data, cache=cache, verbose=verbose) ts = utils.get_timestamp(date) date = utils.get_date_ymd(date) for sgn, info1 in bt_info.items(): res = [] info = walk_on_the_bt(channel, ts, max_days, info1, sgn=sgn, verbose=verbose) for bt, info2 in info1.items(): if not info2['processed']: l = [] haspatch = False for i in range(len(info2['files'])): f = info2['files'][i] if f: if not haspatch and info[f]['patches']: haspatch = True l.append((bt[i], info[f])) else: l.append((bt[i], {'filename': '', 'patches': []})) if haspatch: res.append({'count': info2['count'], 'uuids': info2['uuids'], 'bt': l}) if res: results[sgn] = res return results
def get_bz_params(self, date): self.date = lmdutils.get_date_ymd(date) self.tomorrow = self.date + relativedelta(days=1) bugs = utils.get_bugs_from_pushlog(self.date, self.tomorrow) assignee_blacklist = self.get_config('assignee_blacklist', default=[]) assignee_blacklist = ','.join(assignee_blacklist) fields = [ 'assigned_to', 'assigned_to_detail', 'status', 'resolution', 'priority', 'severity', 'keywords', 'cf_crash_signature', ] fields += [self.status_nightly, self.status_beta, self.status_release] fields += [self.tracking_nightly] params = { 'include_fields': fields, 'bug_id': ','.join(bugs), 'f1': 'assigned_to', 'o1': 'nowords', 'v1': assignee_blacklist, } return params
def get_bz_params(self, date): self.date = lmdutils.get_date_ymd(date) start_date = self.date - relativedelta(years=self.nyears) fields = ['flags', 'depends_on'] params = { 'include_fields': fields, 'resolution': '---', 'f1': 'attachment.ispatch', 'n2': 1, 'f2': 'attachments.isobsolete', 'f3': 'attachments.mimetype', 'o3': 'anywordssubstr', 'v3': 'text/x-phabricator-request,text/plain', 'f4': 'creation_ts', 'o4': 'greaterthan', 'v4': start_date, 'f5': 'days_elapsed', 'o5': 'greaterthaneq', 'v5': self.nweeks * 7, 'n6': 1, 'f6': 'longdesc', 'o6': 'casesubstring', 'v6': 'which didn\'t land and no activity in this bug for', } return params
def run(self): args = self.get_args_parser().parse_args() self.date = lmdutils.get_date_ymd(args.date) for nagger in self.naggers: nagger.send_nag_mail = False nagger.run() self.gather(args.dryrun)
def get_stats_for_past_weeks(product, channel, start_date_by_channel, versions_by_channel, analysis, search_start_date, end_date, check_for_fx=True): queries = [] trends = {} signatures_by_chan = {} default_trend_by_chan = {} ref_monday, _ = utils.get_monday_sunday(utils.get_date_ymd(end_date)) def get_past_week(date): monday, _ = utils.get_monday_sunday(date) return (ref_monday - monday).days // 7 for chan in channel: past_w = get_past_week(start_date_by_channel[chan]) default_trend_by_chan[chan] = {i: 0 for i in range(past_w + 1)} for signature, info in analysis.items(): if not check_for_fx or info['firefox']: data = {} trends[signature] = data # for chan, volume in info['affected']: for chan in channel: if chan in signatures_by_chan: signatures_by_chan[chan].append(signature) else: signatures_by_chan[chan] = [signature] data[chan] = default_trend_by_chan[chan].copy() def handler_ss(chan, json, data): sgns = [] for facets in json['facets']['histogram_date']: d = utils.get_date_ymd(facets['term']) w = get_past_week(d) s = facets['facets']['signature'] for signature in s: count = signature['count'] sgn = signature['term'] sgns.append(sgn) data[sgn][chan][w] += count for chan, signatures in signatures_by_chan.items(): if search_start_date: search_date = socorro.SuperSearch.get_search_date(search_start_date, end_date) else: search_date = socorro.SuperSearch.get_search_date(utils.get_date_str(start_date_by_channel[chan]), end_date) vers = versions_by_channel[chan] for sgns in Connection.chunks(signatures, 10): queries.append(Query(socorro.SuperSearch.URL, {'signature': ['=' + s for s in sgns], 'product': product, 'version': vers, 'release_channel': chan, 'date': search_date, '_histogram.date': 'signature', '_results_number': 0}, handler=functools.partial(handler_ss, chan), handlerdata=trends)) socorro.SuperSearch(queries=queries).wait() return trends
def get_dates(self, date): """Get the dates for the bugzilla query (changedafter and changedbefore fields)""" date = lmdutils.get_date_ymd(date) lookup = self.get_config('days_lookup', 7) start_date = date - relativedelta(days=lookup) end_date = date + relativedelta(days=1) return start_date, end_date
def check_dates(dryrun=False): next_date = utils.get_next_release_date() bad_date_nrd = bad_date_ro = None pat = re.compile(r'<p>(.*)</p>', re.DOTALL) url = 'https://wiki.mozilla.org/Template:NextReleaseDate' template_page = str(requests.get(url).text.encode('utf-8')) m = pat.search(template_page) date = dateutil.parser.parse(m.group(1).strip()) date = pytz.utc.localize(date) if date != next_date: # so two possibilities: # - Release services people just changed the release date # - something is wrong and we must nag now = lmdutils.get_date_ymd('today') cal = utils.get_release_calendar() must_nag = True for i, c in enumerate(cal): if ( now < c['release date'] and i + 1 < len(cal) and cal[i + 1]['release date'] == date ): # The date is just the one after the "normal" release date # so here probably someone just changed the date because # we're close the merge day must_nag = False break if must_nag: bad_date_nrd = date.strftime('%Y-%m-%d') owners = ro.get_owners() now = lmdutils.get_date_ymd('today') for o in owners[::-1]: date = o['release date'] if now < date: if date != next_date: bad_date_ro = date.strftime('%Y-%m-%d') break if bad_date_nrd or bad_date_ro: next_date = next_date.strftime('%Y-%m-%d') send_mail(next_date, bad_date_nrd, bad_date_ro, dryrun=dryrun)
def __init__(self, cal, fallback, team_name, people=None): super(JSONCalendar, self).__init__( cal['duty-start-dates'], fallback, team_name, people=people ) dates = sorted((lmdutils.get_date_ymd(d), d) for d in self.cal.keys()) self.set_team(list(self.cal[d] for _, d in dates), cal.get('triagers', {})) self.dates = [d for d, _ in dates] cycle = self.guess_cycle() self.dates.append(self.dates[-1] + relativedelta(days=cycle)) self.team.append(None)
def bug_handler(bug, data): last_change_date = utils.get_date_ymd(bug['last_change_time']) data[str(bug['id'])] = {'resolved': False, 'incomplete': False, 'fixed': False, 'patched': False, 'assigned': False, 'last_change': last_change_date, 'fixed_dates': [], 'no_change': set()}
def handler_ss_spikers(json, data): if not json['errors']: for facets in json['facets']['histogram_date']: date = utils.get_date_ymd(facets['term']) s = facets['facets']['signature'] d = {} data[date] = d for signature in s: count = signature['count'] sgn = signature['term'] d[sgn] = count
def get_persons(self, date): date = lmdutils.get_date_ymd(date) if date in self.cache: return self.cache[date] res = parse_events(self.cal, start=date, end=date) self.cache[date] = res = [ (p.summary, self.people.get_bzmail_from_name(p.summary)) for p in res ] return res
def handler_ss(chan, json, data): sgns = [] for facets in json['facets']['histogram_date']: d = utils.get_date_ymd(facets['term']) w = get_past_week(d) s = facets['facets']['signature'] for signature in s: count = signature['count'] sgn = signature['term'] sgns.append(sgn) data[sgn][chan][w] += count
def get_filename(date, output_dir): try: if date: date = utils.get_date_str(utils.get_date_ymd(date)) else: dates = getdates(output_dir) if dates['dates']: date = dates['dates'][-1] else: return None return os.path.join(output_dir, date + '.json') except: return None
def get_persons(self, date): date = lmdutils.get_date_ymd(date) date += relativedelta(seconds=1) date = date.replace(tzinfo=self.cal_tz) if date in self.cache: return self.cache[date] res = parse_events(self.cal, start=date, end=date) self.cache[date] = res = [ (p.summary, self.people.get_bzmail_from_name(p.summary)) for p in res ] return res
def test_get_partial(self): channel = ['release', 'beta', 'aurora', 'nightly', 'esr'] signature = 'js::GCMarker::processMarkStackTop' bugids = socorro.Bugs.get_bugs([signature]) base_versions = {'nightly': 51, 'aurora': 50, 'beta': 49, 'release': 48, 'esr': 45} self.assertEqual(set(bugids[signature]), {792226, 789892, 719114, 730283, 1257309, 941491, 745334, 772441, 952381}) start_date, min_date, versions_by_channel, start_date_by_channel, base_versions = statusflags.get_versions_info('Firefox', base_versions=base_versions) search_date = statusflags.get_search_date('', start_date, '2016-09-14') sgninfo = statusflags.get_signatures(100, 'Firefox', versions_by_channel, channel, search_date, [signature], [], False) status_flags = Bugzilla.get_status_flags(base_versions=base_versions) bugs_history_info = statusflags.get_bugs_info(bugids[signature], status_flags) last_bugs_info, _ = statusflags.get_last_bugs_info(bugids[signature], signature, sgninfo, [], bugs_history_info, min_date) self.assertEqual(last_bugs_info['resolved-fixed-patched'], ['', utils.get_guttenberg_death()]) self.assertEqual(last_bugs_info['resolved-fixed-unpatched'], ['', utils.get_guttenberg_death()]) self.assertEqual(last_bugs_info['resolved-unfixed'], ['952381', utils.get_date_ymd('2014-05-13 12:12:41')]) self.assertEqual(last_bugs_info['unresolved-assigned'], ['', utils.get_guttenberg_death()]) self.assertEqual(last_bugs_info['unresolved-unassigned'], ['719114', utils.get_date_ymd('2016-08-30 23:15:17')]) last_bug = statusflags.get_last_bug(bugids[signature], signature, sgninfo, [], bugs_history_info, min_date) self.assertEqual(last_bug, '719114')
def search_prev_merge(beta): tables = rc.get_all() # the first table is the future and the second is the recent past table = tables[1] central = table[0].index('Central') central = rc.get_versions(table[1][central])[0][0] # just check consistency assert beta == central merge = table[0].index('Merge Date') return lmdutils.get_date_ymd(table[1][merge])
def get_bz_params(self, date): date = lmdutils.get_date_ymd(date) start_date = date - relativedelta(months=self.nmonths) fields = ['assigned_to', 'triage_owner', 'flags'] params = { 'include_fields': fields, 'resolution': '---', 'f1': 'assignee_last_login', 'o1': 'lessthan', 'v1': start_date, } utils.get_empty_assignees(params, negation=True) return params
def get_bz_params(self, date): date = lmdutils.get_date_ymd(date) start_date = date - relativedelta(months=self.nmonths) days = (date - start_date).days self.components = utils.get_config('workflow', 'components') params = { 'component': utils.get_components(self.components), 'bug_type': 'defect', 'resolution': '---', 'f1': 'priority', 'o1': 'anywordssubstr', 'v1': ','.join(['P3', 'P4', 'P5']), 'f2': 'days_elapsed', 'o2': 'greaterthaneq', 'v2': days, } return params
def get_cycle_span(): global _CYCLE_SPAN if _CYCLE_SPAN is None: cal = get_release_calendar() now = lmdutils.get_date_ymd('today') cycle = None for i, c in enumerate(cal): if now < c['merge']: if i == 0: cycle = [search_prev_merge(c['beta']), c['merge']] else: cycle = [cal[i - 1]['merge'], c['merge']] break if cycle: _CYCLE_SPAN = '-'.join(x.strftime('%Y%m%d') for x in cycle) return _CYCLE_SPAN
def get_persons(self, date): date = lmdutils.get_date_ymd(date) if date in self.cache: return self.cache[date] i = bisect_left(self.dates, date) if i == len(self.dates): self.cache[date] = [] return [] if date == self.dates[i]: person = self.team[i] else: person = self.team[i - 1] if i != 0 else self.team[0] self.cache[date] = [person] return [person]
def send_email(self, date='today'): """Send the email""" if date: date = lmdutils.get_date(date) d = lmdutils.get_date_ymd(date) if isinstance(self, Nag): self.nag_date = d if not self.must_run(d): return if not self.has_enough_data(): logger.info('The tool {} hasn\'t enough data to run'.format(self.name())) return login_info = utils.get_login_info() title, body = self.get_email(date) if title: receivers = self.get_receivers() status = 'Success' try: mail.send( login_info['ldap_username'], receivers, title, body, html=True, login=login_info, dryrun=self.dryrun, ) except: # NOQA logger.exception('Tool {}'.format(self.name())) status = 'Failure' db.Email.add(self.name(), receivers, 'global', status) if isinstance(self, Nag): self.send_mails(title, dryrun=self.dryrun) else: name = self.name().upper() if date: logger.info('{}: No data for {}'.format(name, date)) else: logger.info('{}: No data'.format(name)) logger.info('Query: {}'.format(self.query_url))
def get_bz_params(self, date): date = lmdutils.get_date_ymd(date) start_date = date - relativedelta(months=self.nmonths) days = (date - start_date).days fields = ['triage_owner'] self.components = utils.get_config('workflow', 'components') params = { 'include_fields': fields, 'bug_type': 'defect', 'component': utils.get_components(self.components), 'resolution': '---', 'f1': 'priority', 'o1': 'equals', 'v1': 'P2', 'f2': 'days_elapsed', 'o2': 'greaterthaneq', 'v2': days, } return params
def mk_burndown(start, end, data): weeks = mk_weeks(start, end) periods = len(weeks) mk_weeks_stats(weeks, data) tomorrow = lmdutils.get_date_ymd("tomorrow") labels = [] totals = [] _totals = [] unresolved = [] forecasted = [] todo = None for n, week in enumerate(weeks): date = week["end"].strftime("%m-%d") labels.append(date) total = week["resolved"] + week["unresolved"] _totals.append(total) if week["start"] < tomorrow: totals.append(total) unresolved.append(week["unresolved"]) else: totals.append(None) unresolved.append(None) # diff from the prev week diff = 0 if len(_totals) == 1 else _totals[-1] - _totals[-2] last = total if len(forecasted) == 0 else forecasted[-1] + diff if diff != 0: # we need to readjust because new bugs appeared todo = ceil(last / (periods - n)) if todo is None: todo = ceil(total / periods) forecasted.append(max(0, last - todo)) return { "labels": labels, "totals": totals, "unresolved": unresolved, "forecasted": forecasted, "unresolved_link": "https://bugzilla.mozilla.org/buglist.cgi?list_id=14664631&o1=equals&v1=M4&f1=cf_fission_milestone&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED", "total_link": "https://bugzilla.mozilla.org/buglist.cgi?o1=equals&v1=M4&f1=cf_fission_milestone&list_id=14664661", }
def get_filter_query(fa_bids, fx_bids, channel): min_bid = fa_bids[0][0] if fa_bids else '30000101000000' min_bid = min(min_bid, fx_bids[0][0]) if fx_bids else min_bid date = max( utils.get_build_date(min_bid), lmdutils.get_date_ymd('today') - relativedelta(days=180), ) date = date.strftime('%Y-%m-%d') if channel == 'beta': channel = ['beta', 'aurora'] return { 'product': ['FennecAndroid', 'Firefox'], 'date': '>=' + date, 'build_id': '>=' + min_bid, 'release_channel': channel, '_aggs.product': 'build_id', '_results_number': 0, '_facets': 'release_channel', '_facets_size': 1000, }
def get_persons(self, date): date = lmdutils.get_date_ymd(date) if date in self.cache: return self.cache[date] if not self.dates: # no dates so only triagers return self.team i = bisect_left(self.dates, date) if i == len(self.dates): self.cache[date] = [] return [] if date == self.dates[i]: person = self.team[i] else: person = self.team[i - 1] if i != 0 else self.team[0] self.cache[date] = [person] return [person]
def get_bz_params(self, date): self.date = lmdutils.get_date_ymd(date) start_date = self.date - relativedelta(years=self.nyears) fields = ['flags'] params = { 'include_fields': fields, 'resolution': '---', 'f1': 'attachment.ispatch', 'n2': 1, 'f2': 'attachments.isobsolete', 'f3': 'attachments.mimetype', 'o3': 'anywordssubstr', 'v3': 'text/x-phabricator-request,text/plain', 'f4': 'creation_ts', 'o4': 'greaterthan', 'v4': start_date, 'f5': 'days_elapsed', 'o5': 'greaterthaneq', 'v5': self.nweeks * 7, } return params
def mk_burndown(start, end, data): weeks = mk_weeks(start, end) periods = len(weeks) mk_weeks_stats(weeks, data) tomorrow = lmdutils.get_date_ymd('tomorrow') labels = [] totals = [] _totals = [] unresolved = [] forecasted = [] todo = None for n, week in enumerate(weeks): date = week['end'].strftime('%m-%d') labels.append(date) total = week['resolved'] + week['unresolved'] _totals.append(total) if week['start'] < tomorrow: totals.append(total) unresolved.append(week['unresolved']) else: totals.append(None) unresolved.append(None) # diff from the prev week diff = 0 if len(_totals) == 1 else _totals[-1] - _totals[-2] last = total if len(forecasted) == 0 else forecasted[-1] + diff if diff != 0: # we need to readjust because new bugs appeared todo = ceil(last / (periods - n)) if todo is None: todo = ceil(total / periods) forecasted.append(max(0, last - todo)) return { 'labels': labels, 'totals': totals, 'unresolved': unresolved, 'forecasted': forecasted, }
def get_by_install_time(channels, product='Firefox', date='today', query={}): today = utils.get_date_ymd(date) tomorrow = today + relativedelta(days=1) six_months_ago = today - relativedelta(weeks=25) search_date = socorro.SuperSearch.get_search_date(six_months_ago, tomorrow) data = {chan: {} for chan in channels} def handler(json, data): if not json['facets']['histogram_date']: return for facets in json['facets']['histogram_date']: date = utils.get_date_ymd(facets['term']) ninstalls = facets['facets']['cardinality_install_time']['value'] data[date] = ninstalls params = { 'product': product, 'date': search_date, 'release_channel': '', '_histogram.date': '_cardinality.install_time', '_results_number': 10 } params.update(query) searches = [] for chan in channels: params = copy.deepcopy(params) params['release_channel'] = chan searches.append( socorro.SuperSearch(params=params, handler=handler, handlerdata=data[chan])) for s in searches: s.wait() return data
def get_who_to_nag(self, date): fallbacks = {} date = lmdutils.get_date_ymd(date) days = utils.get_config('round-robin', 'days_to_nag', 7) for pc, strategy in self.data.items(): last_date = strategy['dates'][-1] if (last_date - date ).days <= days and strategy['filename'] not in fallbacks: fallbacks[strategy['filename']] = strategy['fallback'] # create a dict: mozmail -> list of filenames to check res = {} for fn, fb in fallbacks.items(): if not self.is_mozilla(fb): raise BadFallback() mozmail = self.people.get_moz_mail(fb) if mozmail not in res: res[mozmail] = [] res[mozmail].append(fn) res = {fb: sorted(fn) for fb, fn in res.items()} return res
def get_params_for_link(date, query={}): today = utils.get_date_ymd(date) tomorrow = today + relativedelta(days=1) tomorrow = utils.get_date_str(tomorrow) today = utils.get_date_str(today) search_date = ['>=' + today, '<' + tomorrow] params = {'product': '', 'date': search_date, 'release_channel': '', 'signature': '', '_facets': ['url', 'user_comments', 'install_time', 'version', 'address', 'moz_crash_reason', 'reason', 'build_id', 'platform_pretty_version', 'signature', 'useragent_locale']} params.update(query) return params
def get_dates(bids): start_date = utils.get_date_ymd('tomorrow') end_date = utils.get_guttenberg_death() date_ranges = {} for i in bids.values(): for chan, j in i.items(): # TODO: handle the case where j is empty... md, Md = j[0][0], j[-1][0] if md < start_date: start_date = md if Md > end_date: end_date = Md if chan not in date_ranges: date_ranges[chan] = [md, Md] else: r = date_ranges[chan] if md < r[0]: r[0] = md if Md > r[1]: r[1] = Md return start_date, end_date, date_ranges
def send_email(self, date='today', dryrun=False): """Send the email""" if date: date = lmdutils.get_date(date) d = lmdutils.get_date_ymd(date) if isinstance(self, Nag): self.nag_date = d if not self.must_run(d): return if not self.has_enough_data(): logger.info('The tool {} hasn\'t enough data to run'.format( self.name())) return login_info = utils.get_login_info() title, body = self.get_email(login_info['bz_api_key'], date, dryrun) if title: mail.send( login_info['ldap_username'], utils.get_config(self.name(), 'receivers'), title, body, html=True, login=login_info, dryrun=dryrun, ) if isinstance(self, Nag): self.send_mails(title, dryrun=dryrun) else: name = self.name().upper() if date: logger.info('{}: No data for {}'.format(name, date)) else: logger.info('{}: No data'.format(name))
def get_hg(self, bugs): url = hgmozilla.Revision.get_url('nightly') queries = [] def handler_rev(json, data): push = json['pushdate'][0] push = datetime.datetime.utcfromtimestamp(push) push = lmdutils.as_utc(push) data['date'] = lmdutils.get_date_str(push) data['backedout'] = utils.is_backout(json) m = BUG_PAT.search(json['desc']) if not m or m.group(1) != data['bugid']: data['bugid'] = '' for info in bugs.values(): for rev, i in info['land'].items(): queries.append(Query(url, {'node': rev}, handler_rev, i)) if queries: hgmozilla.Revision(queries=queries).wait() # clean bug_torm = [] for bug, info in bugs.items(): torm = [] for rev, i in info['land'].items(): if not i['bugid'] or not (self.date <= lmdutils.get_date_ymd( i['date']) < self.tomorrow): torm.append(rev) for x in torm: del info['land'][x] if not info['land']: bug_torm.append(bug) for x in bug_torm: del bugs[x] self.get_hg_patches(bugs)
def create(date=None, extra={}, hgauthors={}): """Clear the current database (if one), create a new one and add everything we need""" models.clear() if not models.create(): return if not date: date = pytz.utc.localize(datetime.utcnow()) else: date = lmdutils.get_date_ymd(date) logger.info('Populate with java files: started.') try: java.populate_java_files() except Exception as e: logger.error(e, exc_info=True) return logger.info('Populate with java files: finished.') models.HGAuthor.put(hgauthors) start_date = date - relativedelta(days=config.get_ndays_of_data()) logger.info('Create data for {}: started.'.format(date)) for chan in config.get_channels(): update.put_filelog(chan, start_date=start_date, end_date=date) for prod in config.get_products(): update.update_builds(start_date + relativedelta(days=1), chan, prod) if isinstance(extra, six.string_types): extra = json.loads(extra) for build in extra.get('builds', []): update.put_build(*build) logger.info('Create data for {}: finished.'.format(date)) update.update_all()
def get_params_for_link(query={}): today = utils.get_date_ymd('today') last = today - relativedelta(days=config.get_limit()) last = utils.get_date_str(last) search_date = ['>=' + last] params = { 'product': '', 'date': search_date, 'release_channel': '', 'version': '', 'signature': '', '_facets': [ 'url', 'user_comments', 'install_time', 'version', 'address', 'moz_crash_reason', 'reason', 'build_id', 'platform_pretty_version', 'signature', 'useragent_locale' ] } params.update(query) return params
def get_bz_params(self, date): fields = ['triage_owner', 'flags'] comps = utils.get_config('workflow', 'components') params = { 'component': comps, 'include_fields': fields, 'resolution': '---', 'f1': 'priority', 'o1': 'equals', 'v1': '--', } self.date = lmdutils.get_date_ymd(date) if self.typ == 'first': params.update( { 'f2': 'flagtypes.name', 'o2': 'notequals', 'v2': 'needinfo?', 'f3': 'creation_ts', 'o3': 'lessthaneq', 'v3': self.date - relativedelta(days=self.lookup_first * 7), 'f4': 'creation_ts', 'o4': 'greaterthan', 'v4': self.date - relativedelta(days=self.lookup_second * 7), } ) else: params.update( { 'f2': 'creation_ts', 'o2': 'lessthaneq', 'v2': self.date - relativedelta(days=self.lookup_second * 7), } ) return params
def get_bz_params(self, date): fields = ["triage_owner", "flags"] params = { "include_fields": fields, "keywords": "intermittent-failure", "keywords_type": "nowords", "email2": "*****@*****.**", "emailreporter2": "1", "emailtype2": "notequals", "resolution": "---", "f21": "bug_type", "o21": "equals", "v21": "defect", "f22": "flagtypes.name", "o22": "notsubstring", "v22": "needinfo?", "f23": "bug_severity", "o23": "anyexact", "v23": "--, n/a", } self.date = lmdutils.get_date_ymd(date) first = f"-{self.lookup_first * 7}d" second = f"-{self.lookup_second * 7}d" if self.typ == "first": # TODO: change this when https://bugzilla.mozilla.org/1543984 will be fixed # Here we have to get bugs where product/component have been set (bug has been triaged) # between 4 and 2 weeks # If the product/component never changed after bug creation, we need to get them too # (second < p < first && c < first) || # (second < c < first && p < first) || # ((second < creation < first) && pc never changed) params.update({ "f2": "flagtypes.name", "o2": "notequals", "v2": "needinfo?", "j3": "OR", "f3": "OP", "j4": "AND", "f4": "OP", "n5": 1, # we use a negation here to be sure that no change after first "f5": "product", "o5": "changedafter", "v5": first, "f6": "product", # here the bug has changed "o6": "changedafter", "v6": second, "n7": 1, "f7": "component", "o7": "changedafter", "v7": first, "f8": "CP", "j9": "AND", "f9": "OP", "n10": 1, "f10": "component", "o10": "changedafter", "v10": first, "f11": "component", "o11": "changedafter", "v11": second, "n12": 1, "f12": "product", "o12": "changedafter", "v12": first, "f13": "CP", "j14": "AND", "f14": "OP", "f15": "creation_ts", "o15": "lessthaneq", "v15": first, "f16": "creation_ts", "o16": "greaterthan", "v16": second, "n17": 1, "f17": "product", "o17": "everchanged", "n18": 1, "f18": "component", "o18": "everchanged", "f19": "CP", "f20": "CP", }) else: params.update({ "j2": "OR", "f2": "OP", "j3": "AND", "f3": "OP", "f4": "product", "o4": "changedbefore", "v4": second, "n5": 1, "f5": "product", "o5": "changedafter", "v5": second, "f6": "CP", "j7": "AND", "f7": "OP", "f8": "component", "o8": "changedbefore", "v8": second, "n9": 1, "f9": "component", "o9": "changedafter", "v9": second, "f10": "CP", "j11": "AND", "f11": "OP", "f12": "creation_ts", "o12": "lessthaneq", "v12": second, "n13": 1, "f13": "product", "o13": "everchanged", "n14": 1, "f14": "component", "o14": "everchanged", "f15": "CP", "f16": "CP", }) return params
def monitor(emails=[], date='yesterday', path='', data=None, verbose=False, writejson=False): if not data: try: with open(path, 'r') as In: data = json.load(In) except IOError: data = {p: {c: {} for c in channels} for p in products} searches = [] start_date = utils.get_date_ymd(date) end_date = start_date + datetime.timedelta(days=1) search_date = socorro.SuperSearch.get_search_date(start_date, end_date) all_versions = {} versions_pc = defaultdict(lambda: defaultdict(lambda: [])) def handler_ss(date, product, json, data): if not json['errors']: for info in json['facets']['release_channel']: chan = info['term'] total = info['count'] if product == 'FennecAndroid': d = {'total': total} if chan in data: data[chan][date] = d else: data[chan] = {date: d} else: throttle = 10 if chan == 'release' else 1 total *= throttle d = { 'browser': 0, 'plugin': 0, 'content': 0, 'total': total } for pt in info['facets']['process_type']: term = pt['term'] count = pt['count'] if term in d: # term can be 'gpu' (very rare) d[term] += count * throttle d['browser'] = total - (d['plugin'] + d['content']) if chan in data: data[chan][date] = d else: data[chan] = {date: d} if date == 'today': dates = ['yesterday', 'today'] else: dates = [date] delay_by_channel = { 'release': config.get('MonitorStartupCrashes', 'delay_release', 12, type=int), 'beta': config.get('MonitorStartupCrashes', 'delay_beta', 4, type=int), 'aurora': config.get('MonitorStartupCrashes', 'delay_aurora', 9, type=int), 'nightly': config.get('MonitorStartupCrashes', 'delay_nightly', 9, type=int) } for data_date in dates: data_date = utils.get_date_ymd(data_date) next_data_date = data_date + datetime.timedelta(days=1) search_data_date = socorro.SuperSearch.get_search_date( data_date, next_data_date) for product in products: versions = socorro.ProductVersions.get_all_versions(product) all_versions[product] = [] for chan in channels: info = versions[chan] last_ver_major = max(info.keys()) _start_date = data_date - datetime.timedelta( weeks=delay_by_channel[chan]) for major in range(last_ver_major, last_ver_major - 4, -1): for v, d in info[major]['versions'].items(): if not v.endswith( 'b') and _start_date <= d <= data_date: all_versions[product].append(v) versions_pc[product][chan].append(v) searches.append( socorro.SuperSearch(params={ 'product': product, 'date': search_data_date, 'release_channel': channels, 'version': all_versions[product], 'uptime': '<60', '_results_number': 0, '_facets_size': 100, '_aggs.release_channel': 'process_type' }, handler=functools.partial( handler_ss, utils.get_date_str(data_date), product), handlerdata=data[product])) for s in searches: s.wait() if writejson and path: with open(path, 'w') as Out: json.dump(data, Out, sort_keys=True) new_start_date = start_date - datetime.timedelta(days=1) new_search_date = socorro.SuperSearch.get_search_date( new_start_date, end_date) def handler_ss_spikers(json, data): if not json['errors']: for facets in json['facets']['histogram_date']: date = utils.get_date_ymd(facets['term']) s = facets['facets']['signature'] d = {} data[date] = d for signature in s: count = signature['count'] sgn = signature['term'] d[sgn] = count spikers_info = defaultdict(lambda: defaultdict(lambda: dict())) searches = [] for product, i1 in data.items(): for chan, i2 in i1.items(): _data = [ float(i[1]['total']) for i in sorted(i2.items(), key=lambda p: utils.get_date_ymd(p[0])) if utils.get_date_ymd(i[0]) <= start_date ] # print(product, chan) issp = spikeanalysis.is_spiking_ma(_data, alpha=2.5, win=7, method='mean', plot=False) == 'up' # spikeanalysis.get_spikes_ma(_data, alpha=2.5, win=7, method='mean', plot=True) if issp: # spikeanalysis.is_spiking_ma(_data, alpha=2.5, win=7, method='mean', plot=True) searches.append( socorro.SuperSearch( params={ 'product': product, 'date': new_search_date, 'release_channel': chan, 'version': all_versions[product], 'uptime': '<60', '_results_number': 0, '_histogram.date': 'signature', '_facets_size': 100 }, handler=handler_ss_spikers, handlerdata=spikers_info[product][chan])) for s in searches: s.wait() if spikers_info: # So we've some spikes... need to send an email with all the info searches = [] interesting_sgns = get_most_signifiant_increases(spikers_info) bugs_by_signature = get_bugs(interesting_sgns) affected_chans = set() crash_data = {p: {} for p in spikers_info.keys()} spikes_number = 0 def handler_global(product, json, data): if not json['errors']: for info in json['facets']['release_channel']: chan = info['term'] throttle = 10 if product == 'FennecAndroid' and chan == 'release' else 1 total = info['count'] * throttle data[chan] = total for p, i1 in spikers_info.items(): searches.append( socorro.SuperSearch(params={ 'product': p, 'date': search_date, 'release_channel': list(i1.keys()), 'version': all_versions[p], '_results_number': 0, '_facets_size': 5, '_aggs.release_channel': 'signature' }, handler=functools.partial( handler_global, p), handlerdata=crash_data[p])) for c in i1.keys(): spikes_number += 1 affected_chans.add(c) url = socorro.SuperSearch.get_link({ 'product': p, 'date': search_date, 'release_channel': c, 'version': versions_pc[p][c], 'uptime': '<60' }) sgns_chan = interesting_sgns[p] sgns_stats = [(s, bugs_by_signature[s], t[2], t[1], t[3]) for s, t in sorted(sgns_chan[c].items(), key=lambda p: p[1][0], reverse=True)] sgns_chan[c] = (url, sgns_stats) for s in searches: s.wait() env = Environment(loader=FileSystemLoader('templates')) env.filters['inflect'] = inflect template = env.get_template('startup_crashes_email') _is = OrderedDict() for product in sorted(interesting_sgns.keys()): _is[product] = OrderedDict() for chan in sorted(interesting_sgns[product].keys()): _is[product][chan] = interesting_sgns[product][chan] interesting_sgns = _is body = template.render( spikes_number=spikes_number, spikes_number_word=inflect.engine().number_to_words(spikes_number), crash_data=crash_data, start_date=utils.get_date_str(new_start_date), end_date=utils.get_date_str(start_date), interesting_sgns=interesting_sgns) title = 'Spikes in startup crashes in %s' % ', '.join(affected_chans) if emails: gmail.send(emails, title, body, html=True) if verbose: print('Title: %s' % title) print('Body:') print(body) return {'title': title, 'body': body} return None
def get_sgns_data(channels, versions, signatures, extra, products, towait, date='today'): today = lmdutils.get_date_ymd(date) few_days_ago = today - relativedelta(days=config.get_limit()) search_date = socorro.SuperSearch.get_search_date(few_days_ago) nbase = [0, 0, 0, {}] data = {} unique = {} unique_prod = defaultdict(lambda: dict()) leftovers = {} allbids = {} for p, i in versions.items(): allbids[p] = allbids_p = {} for c, j in i.items(): allbids_p[c] = allbids_pc = [] for b, k in j.items(): allbids_pc.append(b) v, u, up = k if u: unique[b] = (p, c) elif up: unique_prod[p][b] = (p, c) else: leftovers[b] = (p, c) for product in products: data[product] = d1 = {} b1 = allbids[product] for chan in channels: d1[chan] = d2 = {} b2 = b1[chan] for signature in signatures: d2[signature] = b2 if not unique_prod: # if we've only unique buildids: only N queries for the N signatures towait.append( get_sgns_data_helper(data, signatures, unique, nbase, extra, search_date)) else: # if a buildid is unique then it's unique for its product too. # So we've only 2xN queries (2 == len(['Firefox', 'FennecAndroid'])) for b, x in unique.items(): p, c = x unique_prod[p][b] = (p, c) for prod, bids in unique_prod.items(): towait.append( get_sgns_data_helper(data, signatures, bids, nbase, extra, search_date, product=prod)) # handle the leftovers: normally they should be pretty rare # we've them when they're not unique within the same product (e.g. a nightly and a beta have the same buildid) for b, x in leftovers.items(): prod, chan = x towait.append( get_sgns_data_helper(data, signatures, leftovers, nbase, extra, search_date, product=prod, channel=chan)) return data
def test_version_dates(self): self.assertEqual(versions.getMajorDate(46), datetime.datetime(2016, 4, 26, 7, 0, tzinfo=tzutc())) self.assertEqual(versions.getMajorDate('46'), datetime.datetime(2016, 4, 26, 7, 0, tzinfo=tzutc())) self.assertEqual(versions.getMajorDate('46.0'), datetime.datetime(2016, 4, 26, 7, 0, tzinfo=tzutc())) self.assertEqual(versions.getMajorDate('46.0.1'), datetime.datetime(2016, 4, 26, 7, 0, tzinfo=tzutc())) self.assertEqual(versions.getMajorDate('1'), datetime.datetime(2004, 11, 9, 8, 0, tzinfo=tzutc())) self.assertEqual(versions.getMajorDate('1.0'), datetime.datetime(2004, 11, 9, 8, 0, tzinfo=tzutc())) self.assertEqual(versions.getMajorDate('1.5'), datetime.datetime(2005, 11, 29, 8, 0, tzinfo=tzutc())) self.assertEqual(versions.getMajorDate('14'), datetime.datetime(2012, 7, 17, 7, 0, tzinfo=tzutc())) self.assertEqual(versions.getMajorDate('14.0'), datetime.datetime(2012, 7, 17, 7, 0, tzinfo=tzutc())) self.assertEqual(versions.getMajorDate('14.0.1'), datetime.datetime(2012, 7, 17, 7, 0, tzinfo=tzutc())) self.assertEqual(versions.getMajorDate('33'), datetime.datetime(2014, 10, 14, 7, 0, tzinfo=tzutc())) self.assertEqual(versions.getMajorDate('33.0'), datetime.datetime(2014, 10, 14, 7, 0, tzinfo=tzutc())) self.assertEqual(versions.getMajorDate('33.1'), datetime.datetime(2014, 11, 10, 8, 0, tzinfo=tzutc())) self.assertEqual(versions.getMajorDate('46'), versions.getDate('46')) self.assertEqual(versions.getMajorDate('46.0'), versions.getDate('46.0')) self.assertNotEqual(versions.getMajorDate('48.0'), versions.getDate('48.0.1')) self.assertEqual(versions.getDate('48.0.1'), datetime.datetime(2016, 8, 18, 7, 0, tzinfo=tzutc())) self.assertEqual(versions.getDate('48.0.2'), datetime.datetime(2016, 8, 24, 7, 0, tzinfo=tzutc())) v = versions.get(base=True) if versions.getMajorDate(v['nightly']) is not None: self.assertTrue( versions.getMajorDate(v['release']) <= versions.getMajorDate( v['beta']) <= versions.getMajorDate(v['aurora']) <= versions.getMajorDate(v['nightly'])) else: self.assertTrue( versions.getMajorDate(v['release']) <= versions.getMajorDate( v['beta']) <= versions.getMajorDate(v['aurora'])) date = utils.get_date_ymd('2011-08-24T14:52:52Z') self.assertEqual(date - versions.getMajorDate('7'), datetime.timedelta(-34, 28372)) self.assertEqual( versions.getCloserMajorRelease(date), ('7.0', datetime.datetime(2011, 9, 27, 7, 0, tzinfo=tzutc()))) self.assertEqual( versions.getCloserMajorRelease(date, negative=True), ('6.0', datetime.datetime(2011, 8, 16, 7, 0, tzinfo=tzutc()))) self.assertEqual( versions.getCloserMajorRelease(date, negative=False), ('7.0', datetime.datetime(2011, 9, 27, 7, 0, tzinfo=tzutc()))) date = utils.get_date_ymd('2016-08-19') self.assertEqual( versions.getCloserRelease(date), ('48.0.2', datetime.datetime(2016, 8, 24, 7, 0, tzinfo=tzutc()))) self.assertEqual( versions.getCloserRelease(date, negative=True), ('48.0.1', datetime.datetime(2016, 8, 18, 7, 0, tzinfo=tzutc()))) self.assertEqual( versions.getCloserRelease(date, negative=False), ('48.0.2', datetime.datetime(2016, 8, 24, 7, 0, tzinfo=tzutc())))
if when_blocking is None: when_blocking = entry['when'] elif change['added'] == '+': tracked = True if when_tracked is None: when_tracked = entry['when'] tracking_value = 'blocking' if blocking else 'tracked' if tracked else 'never' if bug_data['cf_last_resolved'] is None: fixing_time = -1 fixing_time_since_blocking = -1 fixing_time_since_tracked = -1 else: fixing_time = int(( utils.get_date_ymd(bug_data['cf_last_resolved']) - utils.get_date_ymd(bug_data['creation_time'])).total_seconds()) if blocking: fixing_time_since_blocking = int( (utils.get_date_ymd(bug_data['cf_last_resolved']) - utils.get_date_ymd(when_blocking)).total_seconds()) else: fixing_time_since_blocking = -1 if tracked: fixing_time_since_tracked = int( (utils.get_date_ymd(bug_data['cf_last_resolved']) - utils.get_date_ymd(when_tracked)).total_seconds()) else: fixing_time_since_tracked = -1 csv_writer.writerow([
def get(channel, date, versions=None, product='Firefox', duration=1): """Get stability info Args: channel (str): the channel date (str): the final date versions (Optional[List[str]]): the versions to treat product (Optional[str]): the product duration (Optional[int]): the duration to retrieve the data Returns: dict: contains all the info relative to stability """ channel = channel.lower() cycle = duration <= 0 versions_info = socorro.ProductVersions.get_version_info(versions, channel=channel, product=product) versions = versions_info.keys() throttle = set(map(lambda p: p[1], versions_info.values())) diff_throttle = len(throttle) != 1 # normally the throttle is 10% for release and 100% for others channel if not diff_throttle: throttle = throttle.pop() platforms = socorro.Platforms.get_cached_all() end_date_dt = utils.get_date_ymd(date) if cycle: # we get all the start date for each versions and get the min start_date_dt = min( map(lambda p: utils.get_date_ymd(p[0]), versions_info.values())) duration = (end_date_dt - start_date_dt).days + 1 else: start_date_dt = end_date_dt - timedelta(duration - 1) start_date_str = utils.get_date_str(start_date_dt) end_date_str = utils.get_date_str(end_date_dt) # First, we get the ADI adi = socorro.ADI.get(version=versions, product=product, end_date=end_date_str, duration=duration, platforms=platforms) adi = [adi[key] for key in sorted(adi.keys(), reverse=False)] # Get the khours khours = Redash.get_khours(start_date_dt, end_date_dt, channel, versions, product) khours = [khours[key] for key in sorted(khours.keys(), reverse=False)] # Get the # of crashes (crash pings) crash_pings = Redash.get_number_of_crash(start_date_dt, end_date_dt, channel, versions, product) crashes = {} stats = {'m+c': 0., 'main': 0., 'content': 0., 'plugin': 0., 'all': 0.} for i in range(duration): d = end_date_dt - timedelta(i) crashes[d] = {} crashes[d]['socorro'] = { 'global': stats.copy(), 'startup': stats.copy() } crashes[d]['telemetry'] = crash_pings[d] base = { 'product': product, 'version': None, 'date': socorro.SuperSearch.get_search_date(start_date_str, end_date_str), 'release_channel': channel, '_results_number': 1, '_histogram.date': ['product', 'process_type'], '_facets_size': 3 } if diff_throttle: # in this case each version could have a different throttle so we need to compute stats for each version queries = [] for v, t in versions_info.items(): cparams = base.copy() cparams['version'] = v queries.append( Query(socorro.SuperSearch.URL, cparams, functools.partial(__crash_handler, t[1]), crashes)) cparams = copy.deepcopy(cparams) cparams['uptime'] = '<60' cparams['_histogram.date'].append('uptime') queries.append( Query(socorro.SuperSearch.URL, cparams, functools.partial(__crash_handler, t[1]), crashes)) else: base['version'] = versions queries = [] queries.append( Query(socorro.SuperSearch.URL, base, functools.partial(__crash_handler, throttle), crashes)) cparams = copy.deepcopy(base) cparams['uptime'] = '<60' cparams['_histogram.date'].append('uptime') queries.append( Query(socorro.SuperSearch.URL, cparams, functools.partial(__crash_handler, throttle), crashes)) socorro.SuperSearch(queries=queries).wait() crashes = [crashes[key] for key in sorted(crashes.keys(), reverse=False)] # Now we compute the rates and the averages stats = { 'm+c': [0., 0., 0., 0.], 'main': [0., 0., 0., 0.], 'content': [0., 0., 0., 0.], 'plugin': [0., 0., 0., 0.], 'all': [0., 0., 0., 0.] } averages = {} averages['socorro'] = {'global': stats, 'startup': copy.deepcopy(stats)} averages['telemetry'] = copy.deepcopy(stats) N = len(adi) # sum for i in range(N): crash_soc = crashes[i]['socorro'] for k1, v1 in averages['socorro'].items(): for k2, av in v1.items(): c = crash_soc[k1][k2] # the rate is computed for 100 adi x = utils.rate(100. * c, adi[i]) av[0] += x av[1] += x**2 y = utils.rate(c, khours[i]) av[2] += y av[3] += y**2 crash_soc[k1][k2] = (c, x, y) crash_tel = crashes[i]['telemetry'] for k1, av in averages['telemetry'].items(): c = crash_tel[k1] # the rate is computed for 100 adi x = utils.rate(100. * c, adi[i]) av[0] += x av[1] += x**2 y = utils.rate(c, khours[i]) av[2] += y av[3] += y**2 crash_tel[k1] = (c, x, y) N = float(N) averages_old = {'socorro': {}, 'telemetry': {}} averages_new = copy.deepcopy(averages_old) # mean & standard deviation av_new_soc = averages_new['socorro'] av_old_soc = averages_old['socorro'] for k1, v1 in averages['socorro'].items(): d1 = {} av_old_soc[k1] = d1 d2 = {} av_new_soc[k1] = d2 for k2, av in v1.items(): m = av[0] / N d1[k2] = (m, math.sqrt(av[1] / N - m**2)) m = av[2] / N d2[k2] = (m, math.sqrt(av[3] / N - m**2)) av_new_tel = averages_new['telemetry'] av_old_tel = averages_old['telemetry'] for k1, av in averages['telemetry'].items(): m = av[0] / N av_old_tel[k1] = (m, math.sqrt(av[1] / N - m**2)) m = av[2] / N av_new_tel[k1] = (m, math.sqrt(av[3] / N - m**2)) return { 'start_date': start_date_str, 'end_date': end_date_str, 'versions': versions, 'adi': adi, 'khours': khours, 'crashes': crashes, 'averages_old': averages_old, 'averages_new': averages_new }
def get_uuids(channel, product='Firefox', date='today', limit=10000, max_days=3, threshold=5): end_date = utils.get_date_ymd(date) start_date = end_date - timedelta(days=max_days + 1) search_date = socorro.SuperSearch.get_search_date(start_date, end_date) r = range(max_days + 1) default_trend = {start_date + timedelta(days=i): 0 for i in r} data = defaultdict(lambda: copy.deepcopy(default_trend)) def handler(json, data): if not json['errors']: for facets in json['facets']['histogram_date']: d = utils.get_date_ymd(facets['term']) s = facets['facets']['signature'] for signature in s: count = signature['count'] sgn = signature['term'] data[sgn][d] += count socorro.SuperSearch(params={ 'product': product, 'date': search_date, 'release_channel': channel, '_histogram.date': 'signature', '_facets_size': limit, '_results_number': 1 }, handler=handler, handlerdata=data).wait() new_signatures = get_new_signatures(data, threshold=threshold) if new_signatures: data = {} queries = [] def handler(json, data): if not json['errors']: for facets in json['facets']['proto_signature']: proto = facets['term'] count = facets['count'] facets = facets['facets'] signature = facets['signature'][0]['term'] first_uuid = facets['uuid'][0]['term'] data[proto] = { 'uuid': first_uuid, 'count': count, 'signature': signature } for sgns in Connection.chunks(new_signatures, 5): queries.append( Query(socorro.SuperSearch.URL, { 'product': product, 'date': search_date, 'signature': ['=' + s for s in sgns], 'release_channel': channel, '_aggs.proto_signature': ['uuid', 'signature'], '_facets_size': 1000, '_results_number': 0 }, handler=handler, handlerdata=data)) socorro.SuperSearch(queries=queries).wait() return data, search_date return {}, ''
def test_get(self): class MyConf(libmozdata.config.Config): def get(self, section, option, default=None, type=str): if section == 'StatusFlags' and option == 'ignored': return '\'OOM | small\', \'EMPTY: no crashing thread identified; ERROR_NO_MINIDUMP_HEADER\', \'F1398665248_____________________________\'' else: return default config.set_config(MyConf()) base_versions = { 'nightly': 51, 'aurora': 50, 'beta': 49, 'release': 48, 'esr': 45 } info = statusflags.get('Firefox', 2, end_date='2016-09-14', base_versions=base_versions, check_for_fx=False, check_bz_version=False, check_noisy=False, verbose=False) self.assertEqual(info['base_versions']['aurora'], 50) self.assertEqual(info['base_versions']['beta'], 49) self.assertEqual(info['base_versions']['esr'], 45) self.assertEqual(info['base_versions']['nightly'], 51) self.assertEqual(info['base_versions']['release'], 48) self.assertEqual(info['start_dates']['aurora'], utils.get_date_ymd('2016-08-01')) self.assertEqual(info['start_dates']['beta'], utils.get_date_ymd('2016-08-02')) self.assertEqual(info['start_dates']['esr'], utils.get_date_ymd('2016-03-16')) self.assertEqual(info['start_dates']['nightly'], utils.get_date_ymd('2016-08-01')) self.assertEqual(info['start_dates']['release'], utils.get_date_ymd('2016-07-25')) self.assertTrue( 'IPCError-browser | ShutDownKill' in info['signatures']) isgn = info['signatures']['IPCError-browser | ShutDownKill'] self.assertEqual( set(isgn['leftovers']), {('aurora', 114075), ('nightly', 65105), ('beta', 33208), ('release', 897), ('esr', 23)}) self.assertEqual(isgn['bugid'], 1216774) self.assertEqual( set(isgn['bugs']), { 1238657, 1151237, 1216774, 1260551, 1177484, 1173134, 1168272, 1132053, 1133597, 1167902, 1200671, 1213092, 1200646, 1213096, 1223594, 1200685, 1279293, 1206729, 1177425, 1219672, 1205467, 1240542, 1266275, 1290280, 1259125, 1164155, 1150846, 1311297, 1316867, 1289405, 1311869 }) self.assertEqual(set(isgn['platforms']), {'Windows', 'Mac OS X'}) self.assertFalse(isgn['private']) def getnumbers(b, c, p, g): return {'browser': b, 'content': c, 'plugin': p, 'gpu': g} self.assertEqual(isgn['rank']['aurora'], getnumbers(-1, 1, -1, -1)) self.assertEqual(isgn['rank']['beta'], getnumbers(-1, 1, -1, -1)) self.assertEqual(isgn['rank']['esr'], getnumbers(-1, 1, -1, -1)) self.assertEqual(isgn['rank']['nightly'], getnumbers(-1, 1, -1, -1)) self.assertEqual(isgn['rank']['release'], getnumbers(-1, 10, -1, -1)) self.assertTrue(isgn['resolved']) self.assertEqual(isgn['trend']['aurora'], [5776, 18706, 21015, 20699, 22058, 20100, 5721]) self.assertEqual(isgn['trend']['beta'], [206, 784, 817, 987, 1419, 18147, 10848]) self.assertEqual(isgn['trend']['esr'], [ 4, 5, 2, 5, 2, 2, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]) self.assertEqual(isgn['trend']['nightly'], [3280, 10192, 10261, 10294, 10631, 11974, 8471]) self.assertEqual(isgn['trend']['release'], [101, 232, 169, 134, 117, 92, 49, 1]) data = statusflags.generate_bug_report( 'IPCError-browser | ShutDownKill', isgn, info['status_flags'], info['base_versions'], info['start_dates'], end_date='2016-09-14', check_for_fx=False) self.assertEqual( data['comment']['body'], 'Crash volume for signature \'IPCError-browser | ShutDownKill\':\n - nightly (version 51): 65105 crashes from 2016-08-01.\n - aurora (version 50): 114075 crashes from 2016-08-01.\n - beta (version 49): 33208 crashes from 2016-08-02.\n - release (version 48): 897 crashes from 2016-07-25.\n - esr (version 45): 23 crashes from 2016-03-16.\n\nCrash volume on the last weeks (Week N is from 09-12 to 09-18):\n W. N-1 W. N-2 W. N-3 W. N-4 W. N-5 W. N-6 W. N-7\n - nightly 10192 10261 10294 10631 11974 8471\n - aurora 18706 21015 20699 22058 20100 5721\n - beta 784 817 987 1419 18147 10848\n - release 232 169 134 117 92 49 1\n - esr 5 2 5 2 2 3 2\n\nAffected platforms: Windows, Mac OS X\n\nCrash rank on the last 7 days:\n Browser Content Plugin\n - nightly #1\n - aurora #1\n - beta #1\n - release #10\n - esr #1' )
def get(channel, date, product='Firefox', duration=11, tc_limit=50, crash_type='all', startup=False): """Get crashes info Args: channel (str): the channel date (str): the final date product (Optional[str]): the product duration (Optional[int]): the duration to retrieve the data tc_limit (Optional[int]): the number of topcrashes to load crash_type (Optional[str]): 'all' (default) or 'browser' or 'content' or 'plugin' Returns: dict: contains all the info relative to the crashes """ channel = channel.lower() version = v[channel] sys.stdout.write('Getting version information from Socorro...') sys.stdout.flush() versions = versions_util.get_channel_versions(channel, product) sys.stdout.write(' ✔\n') sys.stdout.flush() if crash_type and isinstance(crash_type, six.string_types): crash_type = [crash_type] _date = utils.get_date_ymd(date) start_date = utils.get_date_str(_date - timedelta(duration - 1)) end_date = utils.get_date_str(_date) signatures = {} def signature_handler(json): for signature in json['facets']['signature']: signatures[signature['term']] = [signature['count'], 0, 0, 0, 0, 0] for platform in signature['facets']['platform']: if platform['term'] == 'Linux': signatures[signature['term']][3] = platform['count'] elif platform['term'] == 'Windows NT': signatures[signature['term']][1] = platform['count'] elif platform['term'] == 'Mac OS X': signatures[signature['term']][2] = platform['count'] for startup_crash in signature['facets']['startup_crash']: if startup_crash['term'] in ['1', 'T']: signatures[signature['term']][4] += startup_crash['count'] signatures[signature['term']][5] = signature['facets'][ 'cardinality_install_time']['value'] params = { 'product': product, 'version': versions, 'date': socorro.SuperSearch.get_search_date(start_date, end_date), 'release_channel': channel, '_aggs.signature': ['platform', '_cardinality.install_time', 'startup_crash'], '_results_number': 0, '_facets_size': tc_limit, } if startup: params['startup_crash'] = True sys.stdout.write('Getting top signatures from Socorro...') sys.stdout.flush() socorro.SuperSearch(params=params, handler=signature_handler).wait() sys.stdout.write(' ✔\n') sys.stdout.flush() bug_flags = [ 'resolution', 'id', 'last_change_time', 'cf_tracking_firefox' + str(version) ] for i in range(int(version), int(v['nightly']) + 1): bug_flags.append('cf_status_firefox' + str(i)) # TODO: too many requests... should be improved with chunks bugs = {} # TODO: Use regexp, when the Bugzilla bug that prevents them from working will be fixed. base = { 'j_top': 'OR', 'o1': 'substring', 'f1': 'cf_crash_signature', 'v1': None, 'o2': 'substring', 'f2': 'cf_crash_signature', 'v2': None, 'o3': 'substring', 'f3': 'cf_crash_signature', 'v3': None, 'o4': 'substring', 'f4': 'cf_crash_signature', 'v4': None, 'include_fields': bug_flags } queries = [] for sgn in signatures.keys(): cparams = base.copy() cparams['v1'] = '[@' + sgn + ']' cparams['v2'] = '[@ ' + sgn + ' ]' cparams['v3'] = '[@ ' + sgn + ']' cparams['v4'] = '[@' + sgn + ' ]' bugs[sgn] = [] queries.append( Query(Bugzilla.API_URL, cparams, __bug_handler, bugs[sgn])) res_bugs = Bugzilla(queries=queries) # we have stats by signature in self.signatures # for each signature get the number of crashes on the last X days # so get the signature trend trends = {} default_trend = {} for i in range(duration): default_trend[_date - timedelta(i)] = 0 base = { 'product': product, 'version': versions, 'signature': None, 'date': socorro.SuperSearch.get_search_date(start_date, end_date), 'release_channel': channel, '_results_number': 0, '_histogram.date': ['signature'], '_histogram_interval': 1 } queries = [] for sgns in Connection.chunks( list(map(lambda sgn: '=' + sgn, signatures.keys())), 10): sgn_group = [] for sgn in sgns: if sum(len(s) for s in sgn_group) >= 1000: cparams = base.copy() cparams['signature'] = sgn_group queries.append( Query(socorro.SuperSearch.URL, cparams, functools.partial(__trend_handler, default_trend), trends)) sgn_group = [] sgn_group.append(sgn) if len(sgn_group) > 0: cparams = base.copy() cparams['signature'] = sgn_group queries.append( Query(socorro.SuperSearch.URL, cparams, functools.partial(__trend_handler, default_trend), trends)) sys.stdout.write('Getting trends for top signatures from Socorro...') sys.stdout.flush() socorro.SuperSearch(queries=queries).wait() sys.stdout.write(' ✔\n') sys.stdout.flush() for sgn, trend in trends.items(): signatures[sgn] = (signatures[sgn], [ trend[key] for key in sorted(trend.keys(), reverse=True) ]) _signatures = {} # order self.signatures by crash count sorted_signatures = sorted(signatures.items(), key=lambda x: x[1][0][0], reverse=True) i = 1 for s in sorted_signatures: _signatures[s[0]] = i # top crash rank i += 1 sys.stdout.write( 'Getting bugs linked to the top signatures from Bugzilla...') sys.stdout.flush() while True: try: for r in res_bugs.results: r.result(timeout=2) break except TimeoutError: sys.stdout.write('.') sys.stdout.flush() sys.stdout.write(' ✔\n') sys.stdout.flush() # TODO: In the first query to get the bugs, also get dupe_of and avoid the first query # in follow_dup (so modify follow_dup to accept both a bug ID or a bug object). queries = [] for sgn in signatures.keys(): duplicate_ids = [ bug['id'] for bug in bugs[sgn] if bug['resolution'] == 'DUPLICATE' ] # Remove bugs resolved as DUPLICATE from the list of bugs associated to the signature. bugs[sgn] = [ bug for bug in bugs[sgn] if bug['id'] not in duplicate_ids ] # Find duplicates for bugs resolved as DUPLICATE. duplicates = { k: v for k, v in Bugzilla.follow_dup(duplicate_ids).items() if v is not None } duplicate_targets = [ bug_id for bug_id in duplicates.values() if int(bug_id) not in [bug['id'] for bug in bugs[sgn]] ] if len(duplicate_targets) == 0: continue # Get info about bugs that the DUPLICATE bugs have been duped to. params = { 'id': ','.join(duplicate_targets), 'include_fields': bug_flags, } queries.append( Query(Bugzilla.API_URL, params, __bug_handler, bugs[sgn])) sys.stdout.write( 'Resolving duplicate bugs to the bugs they\'ve been duplicated to...') sys.stdout.flush() Bugzilla(queries=queries).wait() sys.stdout.write(' ✔\n') sys.stdout.flush() for sgn, stats in signatures.items(): # stats is 2-uple: ([count, win_count, mac_count, linux_count, startup_count], trend) startup_percent = float(stats[0][4]) / float(stats[0][0]) _signatures[sgn] = { 'tc_rank': _signatures[sgn], 'crash_count': stats[0][0], 'estimated_user_count': stats[0][5], 'startup_percent': startup_percent, 'bugs': bugs[sgn] } return { 'start_date': start_date, 'end_date': end_date, 'versions': list(versions), 'signatures': _signatures, }
def amend_bzparams(self, params, bug_ids): """Amend the Bugzilla params""" if not self.all_include_fields(): if 'include_fields' in params: fields = params['include_fields'] if isinstance(fields, list): if 'id' not in fields: fields.append('id') elif isinstance(fields, six.string_types): if fields != 'id': params['include_fields'] = [fields, 'id'] else: params['include_fields'] = [fields, 'id'] else: params['include_fields'] = ['id'] params['include_fields'] += ['summary', 'groups'] if self.has_assignee( ) and 'assigned_to' not in params['include_fields']: params['include_fields'].append('assigned_to') if self.has_product_component(): if 'product' not in params['include_fields']: params['include_fields'].append('product') if 'component' not in params['include_fields']: params['include_fields'].append('component') if self.has_needinfo() and 'flags' not in params['include_fields']: params['include_fields'].append('flags') if bug_ids: params['bug_id'] = bug_ids if self.filter_no_nag_keyword(): n = utils.get_last_field_num(params) params.update({ 'f' + n: 'status_whiteboard', 'o' + n: 'notsubstring', 'v' + n: '[no-nag]', }) if self.ignore_meta(): n = utils.get_last_field_num(params) params.update({ 'f' + n: 'keywords', 'o' + n: 'nowords', 'v' + n: 'meta' }) # Limit the checkers to X years. Unlimited if max_years = -1 max_years = self.get_max_years() if max_years > 0: n = utils.get_last_field_num(params) today = lmdutils.get_date_ymd('today') few_years_ago = today - relativedelta(years=max_years) params.update({ 'f' + n: 'creation_ts', 'o' + n: 'greaterthan', 'v' + n: few_years_ago }) if self.has_default_products(): params['product'] = utils.get_config(self.name(), 'products') self.has_flags = 'flags' in params.get('include_fields', [])
def feed(self, rr=None): self.data = {} filenames = {} if rr is None: rr = {} for team, path in utils.get_config('round-robin', "teams", default={}).items(): with open('./auto_nag/scripts/configs/{}'.format(path), 'r') as In: rr[team] = json.load(In) filenames[team] = path # rr is dictionary: # - doc -> documentation # - triagers -> dictionary: Real Name -> {bzmail: bugzilla email, nick: bugzilla nickname} # - components -> dictionary: Product::Component -> strategy name # - strategies: dictionay: {duty en date -> Real Name} # Get all the strategies for each team for team, data in rr.items(): if 'doc' in data: del data['doc'] strategies = {} triagers = data['triagers'] fallback_bzmail = triagers['Fallback']['bzmail'] path = filenames.get(team, '') # collect strategies for pc, strategy in data['components'].items(): strategy_data = data[strategy] if strategy not in strategies: strategies[strategy] = strategy_data # rewrite strategy to have a sorted list of end dates for strat_name, strategy in strategies.items(): if 'doc' in strategy: del strategy['doc'] date_name = [] # end date and real name of the triager for date, name in strategy.items(): # collect the tuple (end_date, bzmail) date = lmdutils.get_date_ymd(date) bzmail = triagers[name]['bzmail'] date_name.append((date, bzmail)) # we sort the list to use bisection to find the triager date_name = sorted(date_name) strategies[strat_name] = { 'dates': [d for d, _ in date_name], 'mails': [m for _, m in date_name], 'fallback': fallback_bzmail, 'filename': path, } # finally self.data is a dictionary: # - Product::Component -> dictionary {dates: sorted list of end date # mails: list # fallback: who to nag when we've nobody # filename: the file containing strategy} for pc, strategy in data['components'].items(): self.data[pc] = strategies[strategy]
def test_get_date_ymd(self): self.assertIsNotNone(utils.get_date_ymd('today')) self.assertIsNotNone(utils.get_date_ymd('yesterday')) self.assertIsNotNone(utils.get_date_ymd('tomorrow')) self.assertTrue( utils.get_date_ymd('yesterday') < utils.get_date_ymd('today') < utils.get_date_ymd('tomorrow')) date = utils.as_utc( datetime.datetime.strptime('1991-04-16', '%Y-%m-%d')) self.assertEqual(utils.get_date_ymd('1991/04/16'), date) self.assertEqual(utils.get_date_ymd('1991-04-16'), date) self.assertEqual(utils.get_date_ymd('1991 04 16'), date) self.assertEqual(utils.get_date_ymd('04/16/1991'), date) self.assertEqual(utils.get_date_ymd('16/04/1991'), date) self.assertEqual(utils.get_date_ymd('1991-04-16 12:00:00'), utils.as_utc(datetime.datetime(1991, 4, 16, 12, 0))) with self.assertRaises(Exception): utils.get_date_ymd('') with self.assertRaises(Exception): utils.get_date_ymd('marco')
def get_sgns_data(channels, versions, signatures, products, date='today'): today = lmdutils.get_date_ymd(date) few_days_ago = today - relativedelta(days=config.get_limit()) search_date = socorro.SuperSearch.get_search_date(few_days_ago) nbase = [0, 0] data = {} bids, all_bids, all_versions, doubloons = get_all_buildids(versions) for product in products: data[product] = d1 = {} b1 = bids[product] for chan in channels: d1[chan] = d2 = {} b2 = b1[chan] for signature in signatures: d2[signature] = b2 limit = 80 def handler(sgn, json, data): if not json['facets']['build_id']: return for facets in json['facets']['build_id']: bid = facets['term'] prod, chan, ver = all_bids[str(bid)] _facets = facets['facets'] chans = set() for c in _facets['release_channel']: c = c['term'] if c == 'aurora': chans.add('beta') else: chans.add(c) if len(_facets['product']) != 1 or len(chans) != 1: bid = str(bid) doubloons[bid] = [(prod, chan, ver)] else: dpc = data[prod][chan] nums = dpc[sgn] bid = utils.get_build_date(bid) if isinstance(nums, list): dpc[sgn] = nums = {b: copy.copy(nbase) for b in dpc[sgn]} if bid in nums: n = nums[bid] n[RAW] = facets['count'] N = len(_facets['install_time']) if N == limit: N = _facets['cardinality_install_time']['value'] n[INSTALLS] = N base_params = { 'build_id': list(all_bids.keys()), 'signature': '', 'version': all_versions, 'date': search_date, '_aggs.build_id': [ 'install_time', '_cardinality.install_time', 'release_channel', 'product' ], '_results_number': 0, '_facets': 'signature', '_facets_size': limit } queries = [] for signature in signatures: params = copy.deepcopy(base_params) params['signature'] = '=' + signature hdler = functools.partial(handler, signature) queries.append( Query(socorro.SuperSearch.URL, params=params, handler=hdler, handlerdata=data)) socorro.SuperSearch(queries=queries).wait() get_sgns_for_doubloons(doubloons, signatures, search_date, data) res = defaultdict(lambda: defaultdict(lambda: dict())) for p, i in data.items(): for c, j in i.items(): for sgn, numbers in j.items(): if not isinstance(numbers, list): res[p][c][sgn] = numbers return res
def get_uuids_for_spiking_signatures(channel, cache=None, product='Firefox', date='today', limit=10000, max_days=3, threshold=5): psttz = pytz.timezone('US/Pacific') end_date = utils.get_date_ymd(date) # 2016-10-18 UTC end_date_moz = psttz.localize( datetime(end_date.year, end_date.month, end_date.day)) # 2016-10-18 PST end_buildid = utils.get_buildid_from_date(end_date_moz) # < 20161018000000 start_date_moz = end_date_moz - timedelta( days=max_days + 1) # 2016-10-14 PST (max_days == 3) start_buildid = utils.get_buildid_from_date( start_date_moz) # >= 20161014000000 search_buildid = ['>=' + start_buildid, '<' + end_buildid] start_date = utils.as_utc(start_date_moz) # 2016-10-14 07:00:00 UTC search_date = '>=' + utils.get_date_str(start_date) data = defaultdict(lambda: defaultdict(lambda: 0)) buildids = {} def handler(json, data): if not json['errors']: for facets in json['facets']['build_id']: date = utils.get_date_from_buildid( facets['term']).astimezone(psttz) buildids[date] = facets['count'] for s in facets['facets']['signature']: sgn = s['term'] count = s['count'] data[sgn][date] += count socorro.SuperSearch(params={ 'product': product, 'date': search_date, 'build_id': search_buildid, 'release_channel': channel, '_aggs.build_id': 'signature', '_facets_size': limit, '_results_number': 0 }, handler=handler, handlerdata=data).wait() _data = {} base = { start_date_moz + timedelta(days=i): { 'buildids': {}, 'total': 0 } for i in range(max_days + 1) } # from 2016-10-14 to 2016-10-17 PST for sgn, info in data.items(): d = copy.deepcopy(base) _data[sgn] = d for bid, count in info.items(): date = psttz.localize(datetime(bid.year, bid.month, bid.day)) d[date]['buildids'][bid] = count d[date]['total'] += count data = _data spiking_signatures = [] for sgn, info in data.items(): stats2 = [ i['total'] for _, i in sorted(info.items(), key=lambda p: p[0]) ] if all(i == 0 for i in stats2[:-1]) and stats2[-1] >= threshold: spiking_signatures.append(sgn) data = None if spiking_signatures: # sort the signatures to be sure to always have the same order for the test spiking_signatures = sorted(spiking_signatures) start_buildid = utils.get_buildid_from_date(end_date_moz - timedelta(days=1)) search_buildid = ['>=' + start_buildid, '<' + end_buildid] queries = [] data = defaultdict(lambda: list()) def handler(json, data): if not json['errors']: for facets in json['facets']['proto_signature']: proto = facets['term'] count = facets['count'] facets = facets['facets'] sgn = facets['signature'][0]['term'] first_uuid = facets['uuid'][0]['term'] uuids = {i['term'] for i in facets['uuid']} if cache: i = uuids.intersection(cache['uuids']) uuid = i.pop() if i else first_uuid else: uuid = first_uuid data[sgn].append({ 'proto': proto, 'uuid': uuid, 'count': count }) for sgns in Connection.chunks(spiking_signatures, 5): queries.append( Query(socorro.SuperSearch.URL, { 'product': product, 'date': search_date, 'build_id': search_buildid, 'signature': ['=' + s for s in sgns], 'release_channel': channel, '_aggs.proto_signature': ['uuid', 'signature'], '_facets_size': 10000, '_results_number': 0 }, handler=handler, handlerdata=data)) socorro.SuperSearch(queries=queries).wait() return data
type=int, help='the duration') parser.add_argument('-p', '--product', action='store', default='Firefox', help='the product') parser.add_argument('-v', '--versions', action='store', nargs='+', help='the Firefox versions') parser.add_argument( '--cycle', action='store_true', help='duration is computed to take into account all the cycle') args = parser.parse_args() if args.startdate: duration = (utils.get_date_ymd(args.enddate) - utils.get_date_ymd(args.startdate)).days + 1 else: duration = -1 if args.cycle else args.duration stats = get(args.channel, args.enddate, product=args.product, versions=args.versions, duration=duration) pprint(stats)
def is_merge_day(): next_merge = get_merge_day() today = lmdutils.get_date_ymd("today") return next_merge == today