def get_versions_info(product, date='today', base_versions=libmozdata.versions.get(base=True)):
if not date:
date = 'today'
if base_versions is None:
base_versions = libmozdata.versions.get(base=True)
versions_by_channel = socorro.ProductVersions.get_info_from_major(base_versions, product=product, active=None)
channel_by_version = {}
vbc = {}
start_date_by_channel = {}
start_date = utils.get_date_ymd(date)
min_date = start_date
six_months_ago = utils.get_date_ymd(date) - relativedelta(weeks=26)
for chan, versions in versions_by_channel.items():
start_date_by_channel[chan] = utils.get_date_ymd('tomorrow')
l = []
vbc[chan] = l
for v in versions:
vers = v['version']
if not vers.endswith('b'): # 45.0b is not a true beta version
l.append(vers)
channel_by_version[vers] = chan
d = utils.get_date_ymd(v['start_date'])
if d < start_date:
start_date = max(d, six_months_ago)
if d < start_date_by_channel[chan]:
start_date_by_channel[chan] = max(d, six_months_ago)
if chan != 'esr' and d < min_date:
min_date = d
return start_date, min_date, vbc, start_date_by_channel, base_versions
def get(channel, versions=None, product='Firefox', start_date=None, end_date='today', duration=30, platforms=None):
if not isinstance(versions, list):
if isinstance(versions, numbers.Number):
versions = socorro.ProductVersions.get_active(vnumber=versions, product=product)
else:
versions = socorro.ProductVersions.get_active(product=product)
versions = versions[channel.lower()]
if start_date:
_sdate = utils.get_date_ymd(start_date)
_edate = utils.get_date_ymd(end_date)
duration = (_edate - _sdate).days
adi = socorro.ADI.get(version=versions, product=product, end_date=end_date, duration=duration, platforms=platforms)
data = {}
for d, n in adi.items():
data[d] = {'adi': n, 'browser': 0, 'content': 0, 'plugin': 0, 'browser_rate': 0, 'content_rate': 0, 'b+c_rate': 0, 'plugin_rate': 0}
start_date = utils.get_date(end_date, duration)
search_date = socorro.SuperSearch.get_search_date(start_date, end_date)
socorro.SuperSearch(params={'product': product,
'version': versions,
'release_channel': channel,
'date': search_date,
'_results_number': 0,
'_facets_size': 2, # 2 is for a facet on plugin and on content
'_histogram.date': ['process_type']},
handler=__super_search_handler,
handlerdata=data).wait()
return data
def test_get(self):
class MyConf(libmozdata.config.Config):
def get(self, section, option, default=None, type=str):
if section == 'StatusFlags' and option == 'ignored':
return '\'OOM | small\', \'EMPTY: no crashing thread identified; ERROR_NO_MINIDUMP_HEADER\', \'F1398665248_____________________________\''
else:
return default
config.set_config(MyConf())
base_versions = {'nightly': 51, 'aurora': 50, 'beta': 49, 'release': 48, 'esr': 45}
info = statusflags.get('Firefox', 2, end_date='2016-09-14', base_versions=base_versions, check_for_fx=False, check_bz_version=False, check_noisy=False, verbose=False)
self.assertEqual(info['base_versions']['aurora'], 50)
self.assertEqual(info['base_versions']['beta'], 49)
self.assertEqual(info['base_versions']['esr'], 45)
self.assertEqual(info['base_versions']['nightly'], 51)
self.assertEqual(info['base_versions']['release'], 48)
self.assertEqual(info['start_dates']['aurora'], utils.get_date_ymd('2016-08-01'))
self.assertEqual(info['start_dates']['beta'], utils.get_date_ymd('2016-08-02'))
self.assertEqual(info['start_dates']['esr'], utils.get_date_ymd('2016-03-16'))
self.assertEqual(info['start_dates']['nightly'], utils.get_date_ymd('2016-08-01'))
self.assertEqual(info['start_dates']['release'], utils.get_date_ymd('2016-07-25'))
self.assertTrue('IPCError-browser | ShutDownKill' in info['signatures'])
isgn = info['signatures']['IPCError-browser | ShutDownKill']
self.assertEqual(set(isgn['leftovers']), {('aurora', 114075), ('nightly', 65105), ('beta', 33208), ('release', 897), ('esr', 23)})
self.assertEqual(isgn['bugid'], 1216774)
self.assertEqual(set(isgn['bugs']), {1238657, 1151237, 1216774, 1260551, 1177484, 1173134, 1168272, 1132053, 1133597, 1167902, 1200671, 1213092, 1200646, 1213096, 1223594, 1200685, 1279293, 1206729, 1177425, 1219672, 1205467, 1240542, 1266275, 1290280, 1259125, 1164155, 1150846, 1311297, 1316867, 1289405, 1311869})
self.assertEqual(set(isgn['platforms']), {'Windows', 'Mac OS X'})
self.assertFalse(isgn['private'])
def getnumbers(b, c, p, g):
return {'browser': b, 'content': c, 'plugin': p, 'gpu': g}
self.assertEqual(isgn['rank']['aurora'], getnumbers(-1, 1, -1, -1))
self.assertEqual(isgn['rank']['beta'], getnumbers(-1, 1, -1, -1))
self.assertEqual(isgn['rank']['esr'], getnumbers(-1, 1, -1, -1))
self.assertEqual(isgn['rank']['nightly'], getnumbers(-1, 1, -1, -1))
self.assertEqual(isgn['rank']['release'], getnumbers(-1, 10, -1, -1))
self.assertTrue(isgn['resolved'])
self.assertEqual(isgn['trend']['aurora'], [5776, 18706, 21015, 20699, 22058, 20100, 5721])
self.assertEqual(isgn['trend']['beta'], [206, 784, 817, 987, 1419, 18147, 10848])
self.assertEqual(isgn['trend']['esr'], [4, 5, 2, 5, 2, 2, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
self.assertEqual(isgn['trend']['nightly'], [3280, 10192, 10261, 10294, 10631, 11974, 8471])
self.assertEqual(isgn['trend']['release'], [101, 232, 169, 134, 117, 92, 49, 1])
data = statusflags.generate_bug_report('IPCError-browser | ShutDownKill', isgn, info['status_flags'], info['base_versions'], info['start_dates'], end_date='2016-09-14', check_for_fx=False)
self.assertEqual(data['comment']['body'], 'Crash volume for signature \'IPCError-browser | ShutDownKill\':\n - nightly (version 51): 65105 crashes from 2016-08-01.\n - aurora (version 50): 114075 crashes from 2016-08-01.\n - beta (version 49): 33208 crashes from 2016-08-02.\n - release (version 48): 897 crashes from 2016-07-25.\n - esr (version 45): 23 crashes from 2016-03-16.\n\nCrash volume on the last weeks (Week N is from 09-12 to 09-18):\n W. N-1 W. N-2 W. N-3 W. N-4 W. N-5 W. N-6 W. N-7\n - nightly 10192 10261 10294 10631 11974 8471\n - aurora 18706 21015 20699 22058 20100 5721\n - beta 784 817 987 1419 18147 10848\n - release 232 169 134 117 92 49 1\n - esr 5 2 5 2 2 3 2\n\nAffected platforms: Windows, Mac OS X\n\nCrash rank on the last 7 days:\n Browser Content Plugin\n - nightly #1\n - aurora #1\n - beta #1\n - release #10\n - esr #1')
def get_data(self):
if self.data is None:
path = self.get_path()
self.data = {}
if os.path.exists(path):
with open(path, 'r') as In:
data = json.load(In)
for bugid, date in data.items():
delta = lmdutils.get_date_ymd('today') - lmdutils.get_date_ymd(
date
)
if delta.days < self.max_days:
self.data[int(bugid)] = date
return self.data
def get_who_to_nag(self, date):
fallbacks = {}
date = lmdutils.get_date_ymd(date)
days = utils.get_config('round-robin', 'days_to_nag', 7)
next_date = date + relativedelta(days=days)
for cal in self.all_calendars:
persons = cal.get_persons(next_date)
if persons and all(p is not None for _, p in persons):
continue
name = cal.get_team_name()
fb = cal.get_fallback_mozmail()
if fb not in fallbacks:
fallbacks[fb] = {}
if name not in fallbacks[fb]:
fallbacks[fb][name] = {'nobody': False, 'persons': []}
info = fallbacks[fb][name]
if not persons:
info['nobody'] = True
else:
people_names = [n for n, p in persons if p is None]
if people_names:
info['persons'] += people_names
return fallbacks
def get_bz_params(self, date):
date = lmdutils.get_date_ymd(date) - relativedelta(weeks=self.nweeks)
reporters = self.get_config('reporter_exception', default=[])
reporters = ','.join(reporters)
keywords = self.get_config('keyword_exception', default=[])
keywords = ','.join(keywords)
fields = ['cf_crash_signature']
params = {
'include_fields': fields,
'resolution': '---',
'f1': 'cf_crash_signature',
'o1': 'isnotempty',
'f2': 'creation_ts',
'o2': 'lessthan',
'v2': date,
'f3': 'days_elapsed',
'o3': 'greaterthan',
'v3': self.nweeks * 7,
}
if reporters:
params.update({'f4': 'reporter', 'o4': 'nowordssubstr', 'v4': reporters})
if keywords:
params.update({'f5': 'keywords', 'o5': 'nowords', 'v5': keywords})
return params
def test_cache(self):
cache = Cache('test_cache', 7)
cache.set_dry_run(False)
bugids = [123, 456, 789]
cache.add(bugids)
for bugid in bugids:
assert bugid in cache
assert str(bugid) in cache
assert 101112 not in cache
assert '101112' not in cache
with open(cache.get_path(), 'r') as In:
data = json.load(In)
for bugid in ['123', '456']:
date = data[bugid]
date = lmdutils.get_date_ymd(date) - relativedelta(days=8)
data[bugid] = lmdutils.get_date_str(date)
with open(cache.get_path(), 'w') as Out:
json.dump(data, Out)
cache = Cache('test_cache', 7)
cache.set_dry_run(False)
assert 123 not in cache
assert 456 not in cache
assert 789 in cache
def get_stats(self, signatures, date):
def handler(json, data):
if json['errors']:
raise SocorroError()
del json['hits']
for facet in json['facets'].get('signature', {}):
data.remove(facet['term'])
date = lmdutils.get_date_ymd(date) - relativedelta(weeks=self.nweeks)
search_date = SuperSearch.get_search_date(date)
chunks, size = self.chunkify(signatures)
base = {
'date': search_date,
'signature': '',
'_result_number': 0,
'_facets': 'signature',
'_facets_size': size,
}
searches = []
for chunk in chunks:
params = base.copy()
params['signature'] = ['=' + x for x in chunk]
searches.append(
SuperSearch(
params=params,
handler=handler,
handlerdata=signatures,
raise_error=True,
)
)
for s in searches:
s.wait()
def analyze(channel, data, date='today', max_days=3, cache=None, verbose=False):
results = {}
if data:
bt_info = get_bt(data, cache=cache, verbose=verbose)
ts = utils.get_timestamp(date)
date = utils.get_date_ymd(date)
for sgn, info1 in bt_info.items():
res = []
info = walk_on_the_bt(channel, ts, max_days, info1, sgn=sgn, verbose=verbose)
for bt, info2 in info1.items():
if not info2['processed']:
l = []
haspatch = False
for i in range(len(info2['files'])):
f = info2['files'][i]
if f:
if not haspatch and info[f]['patches']:
haspatch = True
l.append((bt[i], info[f]))
else:
l.append((bt[i], {'filename': '', 'patches': []}))
if haspatch:
res.append({'count': info2['count'], 'uuids': info2['uuids'], 'bt': l})
if res:
results[sgn] = res
return results
def get_bz_params(self, date):
self.date = lmdutils.get_date_ymd(date)
self.tomorrow = self.date + relativedelta(days=1)
bugs = utils.get_bugs_from_pushlog(self.date, self.tomorrow)
assignee_blacklist = self.get_config('assignee_blacklist', default=[])
assignee_blacklist = ','.join(assignee_blacklist)
fields = [
'assigned_to',
'assigned_to_detail',
'status',
'resolution',
'priority',
'severity',
'keywords',
'cf_crash_signature',
]
fields += [self.status_nightly, self.status_beta, self.status_release]
fields += [self.tracking_nightly]
params = {
'include_fields': fields,
'bug_id': ','.join(bugs),
'f1': 'assigned_to',
'o1': 'nowords',
'v1': assignee_blacklist,
}
return params
def get_bz_params(self, date):
self.date = lmdutils.get_date_ymd(date)
start_date = self.date - relativedelta(years=self.nyears)
fields = ['flags', 'depends_on']
params = {
'include_fields': fields,
'resolution': '---',
'f1': 'attachment.ispatch',
'n2': 1,
'f2': 'attachments.isobsolete',
'f3': 'attachments.mimetype',
'o3': 'anywordssubstr',
'v3': 'text/x-phabricator-request,text/plain',
'f4': 'creation_ts',
'o4': 'greaterthan',
'v4': start_date,
'f5': 'days_elapsed',
'o5': 'greaterthaneq',
'v5': self.nweeks * 7,
'n6': 1,
'f6': 'longdesc',
'o6': 'casesubstring',
'v6': 'which didn\'t land and no activity in this bug for',
}
return params
def run(self):
args = self.get_args_parser().parse_args()
self.date = lmdutils.get_date_ymd(args.date)
for nagger in self.naggers:
nagger.send_nag_mail = False
nagger.run()
self.gather(args.dryrun)
def get_stats_for_past_weeks(product, channel, start_date_by_channel, versions_by_channel, analysis, search_start_date, end_date, check_for_fx=True):
queries = []
trends = {}
signatures_by_chan = {}
default_trend_by_chan = {}
ref_monday, _ = utils.get_monday_sunday(utils.get_date_ymd(end_date))
def get_past_week(date):
monday, _ = utils.get_monday_sunday(date)
return (ref_monday - monday).days // 7
for chan in channel:
past_w = get_past_week(start_date_by_channel[chan])
default_trend_by_chan[chan] = {i: 0 for i in range(past_w + 1)}
for signature, info in analysis.items():
if not check_for_fx or info['firefox']:
data = {}
trends[signature] = data
# for chan, volume in info['affected']:
for chan in channel:
if chan in signatures_by_chan:
signatures_by_chan[chan].append(signature)
else:
signatures_by_chan[chan] = [signature]
data[chan] = default_trend_by_chan[chan].copy()
def handler_ss(chan, json, data):
sgns = []
for facets in json['facets']['histogram_date']:
d = utils.get_date_ymd(facets['term'])
w = get_past_week(d)
s = facets['facets']['signature']
for signature in s:
count = signature['count']
sgn = signature['term']
sgns.append(sgn)
data[sgn][chan][w] += count
for chan, signatures in signatures_by_chan.items():
if search_start_date:
search_date = socorro.SuperSearch.get_search_date(search_start_date, end_date)
else:
search_date = socorro.SuperSearch.get_search_date(utils.get_date_str(start_date_by_channel[chan]), end_date)
vers = versions_by_channel[chan]
for sgns in Connection.chunks(signatures, 10):
queries.append(Query(socorro.SuperSearch.URL,
{'signature': ['=' + s for s in sgns],
'product': product,
'version': vers,
'release_channel': chan,
'date': search_date,
'_histogram.date': 'signature',
'_results_number': 0},
handler=functools.partial(handler_ss, chan), handlerdata=trends))
socorro.SuperSearch(queries=queries).wait()
return trends
def get_dates(self, date):
"""Get the dates for the bugzilla query (changedafter and changedbefore fields)"""
date = lmdutils.get_date_ymd(date)
lookup = self.get_config('days_lookup', 7)
start_date = date - relativedelta(days=lookup)
end_date = date + relativedelta(days=1)
return start_date, end_date
def check_dates(dryrun=False):
next_date = utils.get_next_release_date()
bad_date_nrd = bad_date_ro = None
pat = re.compile(r'<p>(.*)</p>', re.DOTALL)
url = 'https://wiki.mozilla.org/Template:NextReleaseDate'
template_page = str(requests.get(url).text.encode('utf-8'))
m = pat.search(template_page)
date = dateutil.parser.parse(m.group(1).strip())
date = pytz.utc.localize(date)
if date != next_date:
# so two possibilities:
# - Release services people just changed the release date
# - something is wrong and we must nag
now = lmdutils.get_date_ymd('today')
cal = utils.get_release_calendar()
must_nag = True
for i, c in enumerate(cal):
if (
now < c['release date']
and i + 1 < len(cal)
and cal[i + 1]['release date'] == date
):
# The date is just the one after the "normal" release date
# so here probably someone just changed the date because
# we're close the merge day
must_nag = False
break
if must_nag:
bad_date_nrd = date.strftime('%Y-%m-%d')
owners = ro.get_owners()
now = lmdutils.get_date_ymd('today')
for o in owners[::-1]:
date = o['release date']
if now < date:
if date != next_date:
bad_date_ro = date.strftime('%Y-%m-%d')
break
if bad_date_nrd or bad_date_ro:
next_date = next_date.strftime('%Y-%m-%d')
send_mail(next_date, bad_date_nrd, bad_date_ro, dryrun=dryrun)
def __init__(self, cal, fallback, team_name, people=None):
super(JSONCalendar, self).__init__(
cal['duty-start-dates'], fallback, team_name, people=people
)
dates = sorted((lmdutils.get_date_ymd(d), d) for d in self.cal.keys())
self.set_team(list(self.cal[d] for _, d in dates), cal.get('triagers', {}))
self.dates = [d for d, _ in dates]
cycle = self.guess_cycle()
self.dates.append(self.dates[-1] + relativedelta(days=cycle))
self.team.append(None)
def bug_handler(bug, data):
last_change_date = utils.get_date_ymd(bug['last_change_time'])
data[str(bug['id'])] = {'resolved': False,
'incomplete': False,
'fixed': False,
'patched': False,
'assigned': False,
'last_change': last_change_date,
'fixed_dates': [],
'no_change': set()}
def handler_ss_spikers(json, data):
if not json['errors']:
for facets in json['facets']['histogram_date']:
date = utils.get_date_ymd(facets['term'])
s = facets['facets']['signature']
d = {}
data[date] = d
for signature in s:
count = signature['count']
sgn = signature['term']
d[sgn] = count
def get_persons(self, date):
date = lmdutils.get_date_ymd(date)
if date in self.cache:
return self.cache[date]
res = parse_events(self.cal, start=date, end=date)
self.cache[date] = res = [
(p.summary, self.people.get_bzmail_from_name(p.summary)) for p in res
]
return res
def handler_ss(chan, json, data):
sgns = []
for facets in json['facets']['histogram_date']:
d = utils.get_date_ymd(facets['term'])
w = get_past_week(d)
s = facets['facets']['signature']
for signature in s:
count = signature['count']
sgn = signature['term']
sgns.append(sgn)
data[sgn][chan][w] += count
def get_filename(date, output_dir):
try:
if date:
date = utils.get_date_str(utils.get_date_ymd(date))
else:
dates = getdates(output_dir)
if dates['dates']:
date = dates['dates'][-1]
else:
return None
return os.path.join(output_dir, date + '.json')
except:
return None
def get_persons(self, date):
date = lmdutils.get_date_ymd(date)
date += relativedelta(seconds=1)
date = date.replace(tzinfo=self.cal_tz)
if date in self.cache:
return self.cache[date]
res = parse_events(self.cal, start=date, end=date)
self.cache[date] = res = [
(p.summary, self.people.get_bzmail_from_name(p.summary)) for p in res
]
return res
def test_get_partial(self):
channel = ['release', 'beta', 'aurora', 'nightly', 'esr']
signature = 'js::GCMarker::processMarkStackTop'
bugids = socorro.Bugs.get_bugs([signature])
base_versions = {'nightly': 51, 'aurora': 50, 'beta': 49, 'release': 48, 'esr': 45}
self.assertEqual(set(bugids[signature]), {792226, 789892, 719114, 730283, 1257309, 941491, 745334, 772441, 952381})
start_date, min_date, versions_by_channel, start_date_by_channel, base_versions = statusflags.get_versions_info('Firefox', base_versions=base_versions)
search_date = statusflags.get_search_date('', start_date, '2016-09-14')
sgninfo = statusflags.get_signatures(100, 'Firefox', versions_by_channel, channel, search_date, [signature], [], False)
status_flags = Bugzilla.get_status_flags(base_versions=base_versions)
bugs_history_info = statusflags.get_bugs_info(bugids[signature], status_flags)
last_bugs_info, _ = statusflags.get_last_bugs_info(bugids[signature], signature, sgninfo, [], bugs_history_info, min_date)
self.assertEqual(last_bugs_info['resolved-fixed-patched'], ['', utils.get_guttenberg_death()])
self.assertEqual(last_bugs_info['resolved-fixed-unpatched'], ['', utils.get_guttenberg_death()])
self.assertEqual(last_bugs_info['resolved-unfixed'], ['952381', utils.get_date_ymd('2014-05-13 12:12:41')])
self.assertEqual(last_bugs_info['unresolved-assigned'], ['', utils.get_guttenberg_death()])
self.assertEqual(last_bugs_info['unresolved-unassigned'], ['719114', utils.get_date_ymd('2016-08-30 23:15:17')])
last_bug = statusflags.get_last_bug(bugids[signature], signature, sgninfo, [], bugs_history_info, min_date)
self.assertEqual(last_bug, '719114')
def search_prev_merge(beta):
tables = rc.get_all()
# the first table is the future and the second is the recent past
table = tables[1]
central = table[0].index('Central')
central = rc.get_versions(table[1][central])[0][0]
# just check consistency
assert beta == central
merge = table[0].index('Merge Date')
return lmdutils.get_date_ymd(table[1][merge])
def get_bz_params(self, date):
date = lmdutils.get_date_ymd(date)
start_date = date - relativedelta(months=self.nmonths)
fields = ['assigned_to', 'triage_owner', 'flags']
params = {
'include_fields': fields,
'resolution': '---',
'f1': 'assignee_last_login',
'o1': 'lessthan',
'v1': start_date,
}
utils.get_empty_assignees(params, negation=True)
return params
def get_bz_params(self, date):
date = lmdutils.get_date_ymd(date)
start_date = date - relativedelta(months=self.nmonths)
days = (date - start_date).days
self.components = utils.get_config('workflow', 'components')
params = {
'component': utils.get_components(self.components),
'bug_type': 'defect',
'resolution': '---',
'f1': 'priority',
'o1': 'anywordssubstr',
'v1': ','.join(['P3', 'P4', 'P5']),
'f2': 'days_elapsed',
'o2': 'greaterthaneq',
'v2': days,
}
return params
def get_cycle_span():
global _CYCLE_SPAN
if _CYCLE_SPAN is None:
cal = get_release_calendar()
now = lmdutils.get_date_ymd('today')
cycle = None
for i, c in enumerate(cal):
if now < c['merge']:
if i == 0:
cycle = [search_prev_merge(c['beta']), c['merge']]
else:
cycle = [cal[i - 1]['merge'], c['merge']]
break
if cycle:
_CYCLE_SPAN = '-'.join(x.strftime('%Y%m%d') for x in cycle)
return _CYCLE_SPAN
def get_persons(self, date):
date = lmdutils.get_date_ymd(date)
if date in self.cache:
return self.cache[date]
i = bisect_left(self.dates, date)
if i == len(self.dates):
self.cache[date] = []
return []
if date == self.dates[i]:
person = self.team[i]
else:
person = self.team[i - 1] if i != 0 else self.team[0]
self.cache[date] = [person]
return [person]
def send_email(self, date='today'):
"""Send the email"""
if date:
date = lmdutils.get_date(date)
d = lmdutils.get_date_ymd(date)
if isinstance(self, Nag):
self.nag_date = d
if not self.must_run(d):
return
if not self.has_enough_data():
logger.info('The tool {} hasn\'t enough data to run'.format(self.name()))
return
login_info = utils.get_login_info()
title, body = self.get_email(date)
if title:
receivers = self.get_receivers()
status = 'Success'
try:
mail.send(
login_info['ldap_username'],
receivers,
title,
body,
html=True,
login=login_info,
dryrun=self.dryrun,
)
except: # NOQA
logger.exception('Tool {}'.format(self.name()))
status = 'Failure'
db.Email.add(self.name(), receivers, 'global', status)
if isinstance(self, Nag):
self.send_mails(title, dryrun=self.dryrun)
else:
name = self.name().upper()
if date:
logger.info('{}: No data for {}'.format(name, date))
else:
logger.info('{}: No data'.format(name))
logger.info('Query: {}'.format(self.query_url))
def get_bz_params(self, date):
date = lmdutils.get_date_ymd(date)
start_date = date - relativedelta(months=self.nmonths)
days = (date - start_date).days
fields = ['triage_owner']
self.components = utils.get_config('workflow', 'components')
params = {
'include_fields': fields,
'bug_type': 'defect',
'component': utils.get_components(self.components),
'resolution': '---',
'f1': 'priority',
'o1': 'equals',
'v1': 'P2',
'f2': 'days_elapsed',
'o2': 'greaterthaneq',
'v2': days,
}
return params
def mk_burndown(start, end, data):
weeks = mk_weeks(start, end)
periods = len(weeks)
mk_weeks_stats(weeks, data)
tomorrow = lmdutils.get_date_ymd("tomorrow")
labels = []
totals = []
_totals = []
unresolved = []
forecasted = []
todo = None
for n, week in enumerate(weeks):
date = week["end"].strftime("%m-%d")
labels.append(date)
total = week["resolved"] + week["unresolved"]
_totals.append(total)
if week["start"] < tomorrow:
totals.append(total)
unresolved.append(week["unresolved"])
else:
totals.append(None)
unresolved.append(None)
# diff from the prev week
diff = 0 if len(_totals) == 1 else _totals[-1] - _totals[-2]
last = total if len(forecasted) == 0 else forecasted[-1] + diff
if diff != 0:
# we need to readjust because new bugs appeared
todo = ceil(last / (periods - n))
if todo is None:
todo = ceil(total / periods)
forecasted.append(max(0, last - todo))
return {
"labels": labels,
"totals": totals,
"unresolved": unresolved,
"forecasted": forecasted,
"unresolved_link": "https://bugzilla.mozilla.org/buglist.cgi?list_id=14664631&o1=equals&v1=M4&f1=cf_fission_milestone&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED",
"total_link": "https://bugzilla.mozilla.org/buglist.cgi?o1=equals&v1=M4&f1=cf_fission_milestone&list_id=14664661",
}
def get_filter_query(fa_bids, fx_bids, channel):
min_bid = fa_bids[0][0] if fa_bids else '30000101000000'
min_bid = min(min_bid, fx_bids[0][0]) if fx_bids else min_bid
date = max(
utils.get_build_date(min_bid),
lmdutils.get_date_ymd('today') - relativedelta(days=180),
)
date = date.strftime('%Y-%m-%d')
if channel == 'beta':
channel = ['beta', 'aurora']
return {
'product': ['FennecAndroid', 'Firefox'],
'date': '>=' + date,
'build_id': '>=' + min_bid,
'release_channel': channel,
'_aggs.product': 'build_id',
'_results_number': 0,
'_facets': 'release_channel',
'_facets_size': 1000,
}
def get_persons(self, date):
date = lmdutils.get_date_ymd(date)
if date in self.cache:
return self.cache[date]
if not self.dates:
# no dates so only triagers
return self.team
i = bisect_left(self.dates, date)
if i == len(self.dates):
self.cache[date] = []
return []
if date == self.dates[i]:
person = self.team[i]
else:
person = self.team[i - 1] if i != 0 else self.team[0]
self.cache[date] = [person]
return [person]
def get_bz_params(self, date):
self.date = lmdutils.get_date_ymd(date)
start_date = self.date - relativedelta(years=self.nyears)
fields = ['flags']
params = {
'include_fields': fields,
'resolution': '---',
'f1': 'attachment.ispatch',
'n2': 1,
'f2': 'attachments.isobsolete',
'f3': 'attachments.mimetype',
'o3': 'anywordssubstr',
'v3': 'text/x-phabricator-request,text/plain',
'f4': 'creation_ts',
'o4': 'greaterthan',
'v4': start_date,
'f5': 'days_elapsed',
'o5': 'greaterthaneq',
'v5': self.nweeks * 7,
}
return params
def mk_burndown(start, end, data):
weeks = mk_weeks(start, end)
periods = len(weeks)
mk_weeks_stats(weeks, data)
tomorrow = lmdutils.get_date_ymd('tomorrow')
labels = []
totals = []
_totals = []
unresolved = []
forecasted = []
todo = None
for n, week in enumerate(weeks):
date = week['end'].strftime('%m-%d')
labels.append(date)
total = week['resolved'] + week['unresolved']
_totals.append(total)
if week['start'] < tomorrow:
totals.append(total)
unresolved.append(week['unresolved'])
else:
totals.append(None)
unresolved.append(None)
# diff from the prev week
diff = 0 if len(_totals) == 1 else _totals[-1] - _totals[-2]
last = total if len(forecasted) == 0 else forecasted[-1] + diff
if diff != 0:
# we need to readjust because new bugs appeared
todo = ceil(last / (periods - n))
if todo is None:
todo = ceil(total / periods)
forecasted.append(max(0, last - todo))
return {
'labels': labels,
'totals': totals,
'unresolved': unresolved,
'forecasted': forecasted,
}
def get_by_install_time(channels, product='Firefox', date='today', query={}):
today = utils.get_date_ymd(date)
tomorrow = today + relativedelta(days=1)
six_months_ago = today - relativedelta(weeks=25)
search_date = socorro.SuperSearch.get_search_date(six_months_ago, tomorrow)
data = {chan: {} for chan in channels}
def handler(json, data):
if not json['facets']['histogram_date']:
return
for facets in json['facets']['histogram_date']:
date = utils.get_date_ymd(facets['term'])
ninstalls = facets['facets']['cardinality_install_time']['value']
data[date] = ninstalls
params = {
'product': product,
'date': search_date,
'release_channel': '',
'_histogram.date': '_cardinality.install_time',
'_results_number': 10
}
params.update(query)
searches = []
for chan in channels:
params = copy.deepcopy(params)
params['release_channel'] = chan
searches.append(
socorro.SuperSearch(params=params,
handler=handler,
handlerdata=data[chan]))
for s in searches:
s.wait()
return data
def get_who_to_nag(self, date):
fallbacks = {}
date = lmdutils.get_date_ymd(date)
days = utils.get_config('round-robin', 'days_to_nag', 7)
for pc, strategy in self.data.items():
last_date = strategy['dates'][-1]
if (last_date - date
).days <= days and strategy['filename'] not in fallbacks:
fallbacks[strategy['filename']] = strategy['fallback']
# create a dict: mozmail -> list of filenames to check
res = {}
for fn, fb in fallbacks.items():
if not self.is_mozilla(fb):
raise BadFallback()
mozmail = self.people.get_moz_mail(fb)
if mozmail not in res:
res[mozmail] = []
res[mozmail].append(fn)
res = {fb: sorted(fn) for fb, fn in res.items()}
return res
def get_params_for_link(date, query={}):
today = utils.get_date_ymd(date)
tomorrow = today + relativedelta(days=1)
tomorrow = utils.get_date_str(tomorrow)
today = utils.get_date_str(today)
search_date = ['>=' + today, '<' + tomorrow]
params = {'product': '',
'date': search_date,
'release_channel': '',
'signature': '',
'_facets': ['url',
'user_comments',
'install_time',
'version',
'address',
'moz_crash_reason',
'reason',
'build_id',
'platform_pretty_version',
'signature',
'useragent_locale']}
params.update(query)
return params
def get_dates(bids):
start_date = utils.get_date_ymd('tomorrow')
end_date = utils.get_guttenberg_death()
date_ranges = {}
for i in bids.values():
for chan, j in i.items():
# TODO: handle the case where j is empty...
md, Md = j[0][0], j[-1][0]
if md < start_date:
start_date = md
if Md > end_date:
end_date = Md
if chan not in date_ranges:
date_ranges[chan] = [md, Md]
else:
r = date_ranges[chan]
if md < r[0]:
r[0] = md
if Md > r[1]:
r[1] = Md
return start_date, end_date, date_ranges
def send_email(self, date='today', dryrun=False):
"""Send the email"""
if date:
date = lmdutils.get_date(date)
d = lmdutils.get_date_ymd(date)
if isinstance(self, Nag):
self.nag_date = d
if not self.must_run(d):
return
if not self.has_enough_data():
logger.info('The tool {} hasn\'t enough data to run'.format(
self.name()))
return
login_info = utils.get_login_info()
title, body = self.get_email(login_info['bz_api_key'], date, dryrun)
if title:
mail.send(
login_info['ldap_username'],
utils.get_config(self.name(), 'receivers'),
title,
body,
html=True,
login=login_info,
dryrun=dryrun,
)
if isinstance(self, Nag):
self.send_mails(title, dryrun=dryrun)
else:
name = self.name().upper()
if date:
logger.info('{}: No data for {}'.format(name, date))
else:
logger.info('{}: No data'.format(name))
def get_hg(self, bugs):
url = hgmozilla.Revision.get_url('nightly')
queries = []
def handler_rev(json, data):
push = json['pushdate'][0]
push = datetime.datetime.utcfromtimestamp(push)
push = lmdutils.as_utc(push)
data['date'] = lmdutils.get_date_str(push)
data['backedout'] = utils.is_backout(json)
m = BUG_PAT.search(json['desc'])
if not m or m.group(1) != data['bugid']:
data['bugid'] = ''
for info in bugs.values():
for rev, i in info['land'].items():
queries.append(Query(url, {'node': rev}, handler_rev, i))
if queries:
hgmozilla.Revision(queries=queries).wait()
# clean
bug_torm = []
for bug, info in bugs.items():
torm = []
for rev, i in info['land'].items():
if not i['bugid'] or not (self.date <= lmdutils.get_date_ymd(
i['date']) < self.tomorrow):
torm.append(rev)
for x in torm:
del info['land'][x]
if not info['land']:
bug_torm.append(bug)
for x in bug_torm:
del bugs[x]
self.get_hg_patches(bugs)
def create(date=None, extra={}, hgauthors={}):
"""Clear the current database (if one), create a new one and add everything we need"""
models.clear()
if not models.create():
return
if not date:
date = pytz.utc.localize(datetime.utcnow())
else:
date = lmdutils.get_date_ymd(date)
logger.info('Populate with java files: started.')
try:
java.populate_java_files()
except Exception as e:
logger.error(e, exc_info=True)
return
logger.info('Populate with java files: finished.')
models.HGAuthor.put(hgauthors)
start_date = date - relativedelta(days=config.get_ndays_of_data())
logger.info('Create data for {}: started.'.format(date))
for chan in config.get_channels():
update.put_filelog(chan, start_date=start_date, end_date=date)
for prod in config.get_products():
update.update_builds(start_date + relativedelta(days=1), chan,
prod)
if isinstance(extra, six.string_types):
extra = json.loads(extra)
for build in extra.get('builds', []):
update.put_build(*build)
logger.info('Create data for {}: finished.'.format(date))
update.update_all()
def get_params_for_link(query={}):
today = utils.get_date_ymd('today')
last = today - relativedelta(days=config.get_limit())
last = utils.get_date_str(last)
search_date = ['>=' + last]
params = {
'product':
'',
'date':
search_date,
'release_channel':
'',
'version':
'',
'signature':
'',
'_facets': [
'url', 'user_comments', 'install_time', 'version', 'address',
'moz_crash_reason', 'reason', 'build_id',
'platform_pretty_version', 'signature', 'useragent_locale'
]
}
params.update(query)
return params
def get_bz_params(self, date):
fields = ['triage_owner', 'flags']
comps = utils.get_config('workflow', 'components')
params = {
'component': comps,
'include_fields': fields,
'resolution': '---',
'f1': 'priority',
'o1': 'equals',
'v1': '--',
}
self.date = lmdutils.get_date_ymd(date)
if self.typ == 'first':
params.update(
{
'f2': 'flagtypes.name',
'o2': 'notequals',
'v2': 'needinfo?',
'f3': 'creation_ts',
'o3': 'lessthaneq',
'v3': self.date - relativedelta(days=self.lookup_first * 7),
'f4': 'creation_ts',
'o4': 'greaterthan',
'v4': self.date - relativedelta(days=self.lookup_second * 7),
}
)
else:
params.update(
{
'f2': 'creation_ts',
'o2': 'lessthaneq',
'v2': self.date - relativedelta(days=self.lookup_second * 7),
}
)
return params
def get_bz_params(self, date):
fields = ["triage_owner", "flags"]
params = {
"include_fields": fields,
"keywords": "intermittent-failure",
"keywords_type": "nowords",
"email2": "*****@*****.**",
"emailreporter2": "1",
"emailtype2": "notequals",
"resolution": "---",
"f21": "bug_type",
"o21": "equals",
"v21": "defect",
"f22": "flagtypes.name",
"o22": "notsubstring",
"v22": "needinfo?",
"f23": "bug_severity",
"o23": "anyexact",
"v23": "--, n/a",
}
self.date = lmdutils.get_date_ymd(date)
first = f"-{self.lookup_first * 7}d"
second = f"-{self.lookup_second * 7}d"
if self.typ == "first":
# TODO: change this when https://bugzilla.mozilla.org/1543984 will be fixed
# Here we have to get bugs where product/component have been set (bug has been triaged)
# between 4 and 2 weeks
# If the product/component never changed after bug creation, we need to get them too
# (second < p < first && c < first) ||
# (second < c < first && p < first) ||
# ((second < creation < first) && pc never changed)
params.update({
"f2": "flagtypes.name",
"o2": "notequals",
"v2": "needinfo?",
"j3": "OR",
"f3": "OP",
"j4": "AND",
"f4": "OP",
"n5":
1, # we use a negation here to be sure that no change after first
"f5": "product",
"o5": "changedafter",
"v5": first,
"f6": "product", # here the bug has changed
"o6": "changedafter",
"v6": second,
"n7": 1,
"f7": "component",
"o7": "changedafter",
"v7": first,
"f8": "CP",
"j9": "AND",
"f9": "OP",
"n10": 1,
"f10": "component",
"o10": "changedafter",
"v10": first,
"f11": "component",
"o11": "changedafter",
"v11": second,
"n12": 1,
"f12": "product",
"o12": "changedafter",
"v12": first,
"f13": "CP",
"j14": "AND",
"f14": "OP",
"f15": "creation_ts",
"o15": "lessthaneq",
"v15": first,
"f16": "creation_ts",
"o16": "greaterthan",
"v16": second,
"n17": 1,
"f17": "product",
"o17": "everchanged",
"n18": 1,
"f18": "component",
"o18": "everchanged",
"f19": "CP",
"f20": "CP",
})
else:
params.update({
"j2": "OR",
"f2": "OP",
"j3": "AND",
"f3": "OP",
"f4": "product",
"o4": "changedbefore",
"v4": second,
"n5": 1,
"f5": "product",
"o5": "changedafter",
"v5": second,
"f6": "CP",
"j7": "AND",
"f7": "OP",
"f8": "component",
"o8": "changedbefore",
"v8": second,
"n9": 1,
"f9": "component",
"o9": "changedafter",
"v9": second,
"f10": "CP",
"j11": "AND",
"f11": "OP",
"f12": "creation_ts",
"o12": "lessthaneq",
"v12": second,
"n13": 1,
"f13": "product",
"o13": "everchanged",
"n14": 1,
"f14": "component",
"o14": "everchanged",
"f15": "CP",
"f16": "CP",
})
return params
def monitor(emails=[],
date='yesterday',
path='',
data=None,
verbose=False,
writejson=False):
if not data:
try:
with open(path, 'r') as In:
data = json.load(In)
except IOError:
data = {p: {c: {} for c in channels} for p in products}
searches = []
start_date = utils.get_date_ymd(date)
end_date = start_date + datetime.timedelta(days=1)
search_date = socorro.SuperSearch.get_search_date(start_date, end_date)
all_versions = {}
versions_pc = defaultdict(lambda: defaultdict(lambda: []))
def handler_ss(date, product, json, data):
if not json['errors']:
for info in json['facets']['release_channel']:
chan = info['term']
total = info['count']
if product == 'FennecAndroid':
d = {'total': total}
if chan in data:
data[chan][date] = d
else:
data[chan] = {date: d}
else:
throttle = 10 if chan == 'release' else 1
total *= throttle
d = {
'browser': 0,
'plugin': 0,
'content': 0,
'total': total
}
for pt in info['facets']['process_type']:
term = pt['term']
count = pt['count']
if term in d: # term can be 'gpu' (very rare)
d[term] += count * throttle
d['browser'] = total - (d['plugin'] + d['content'])
if chan in data:
data[chan][date] = d
else:
data[chan] = {date: d}
if date == 'today':
dates = ['yesterday', 'today']
else:
dates = [date]
delay_by_channel = {
'release':
config.get('MonitorStartupCrashes', 'delay_release', 12, type=int),
'beta':
config.get('MonitorStartupCrashes', 'delay_beta', 4, type=int),
'aurora':
config.get('MonitorStartupCrashes', 'delay_aurora', 9, type=int),
'nightly':
config.get('MonitorStartupCrashes', 'delay_nightly', 9, type=int)
}
for data_date in dates:
data_date = utils.get_date_ymd(data_date)
next_data_date = data_date + datetime.timedelta(days=1)
search_data_date = socorro.SuperSearch.get_search_date(
data_date, next_data_date)
for product in products:
versions = socorro.ProductVersions.get_all_versions(product)
all_versions[product] = []
for chan in channels:
info = versions[chan]
last_ver_major = max(info.keys())
_start_date = data_date - datetime.timedelta(
weeks=delay_by_channel[chan])
for major in range(last_ver_major, last_ver_major - 4, -1):
for v, d in info[major]['versions'].items():
if not v.endswith(
'b') and _start_date <= d <= data_date:
all_versions[product].append(v)
versions_pc[product][chan].append(v)
searches.append(
socorro.SuperSearch(params={
'product': product,
'date': search_data_date,
'release_channel': channels,
'version': all_versions[product],
'uptime': '<60',
'_results_number': 0,
'_facets_size': 100,
'_aggs.release_channel': 'process_type'
},
handler=functools.partial(
handler_ss,
utils.get_date_str(data_date),
product),
handlerdata=data[product]))
for s in searches:
s.wait()
if writejson and path:
with open(path, 'w') as Out:
json.dump(data, Out, sort_keys=True)
new_start_date = start_date - datetime.timedelta(days=1)
new_search_date = socorro.SuperSearch.get_search_date(
new_start_date, end_date)
def handler_ss_spikers(json, data):
if not json['errors']:
for facets in json['facets']['histogram_date']:
date = utils.get_date_ymd(facets['term'])
s = facets['facets']['signature']
d = {}
data[date] = d
for signature in s:
count = signature['count']
sgn = signature['term']
d[sgn] = count
spikers_info = defaultdict(lambda: defaultdict(lambda: dict()))
searches = []
for product, i1 in data.items():
for chan, i2 in i1.items():
_data = [
float(i[1]['total'])
for i in sorted(i2.items(),
key=lambda p: utils.get_date_ymd(p[0]))
if utils.get_date_ymd(i[0]) <= start_date
]
# print(product, chan)
issp = spikeanalysis.is_spiking_ma(_data,
alpha=2.5,
win=7,
method='mean',
plot=False) == 'up'
# spikeanalysis.get_spikes_ma(_data, alpha=2.5, win=7, method='mean', plot=True)
if issp:
# spikeanalysis.is_spiking_ma(_data, alpha=2.5, win=7, method='mean', plot=True)
searches.append(
socorro.SuperSearch(
params={
'product': product,
'date': new_search_date,
'release_channel': chan,
'version': all_versions[product],
'uptime': '<60',
'_results_number': 0,
'_histogram.date': 'signature',
'_facets_size': 100
},
handler=handler_ss_spikers,
handlerdata=spikers_info[product][chan]))
for s in searches:
s.wait()
if spikers_info:
# So we've some spikes... need to send an email with all the info
searches = []
interesting_sgns = get_most_signifiant_increases(spikers_info)
bugs_by_signature = get_bugs(interesting_sgns)
affected_chans = set()
crash_data = {p: {} for p in spikers_info.keys()}
spikes_number = 0
def handler_global(product, json, data):
if not json['errors']:
for info in json['facets']['release_channel']:
chan = info['term']
throttle = 10 if product == 'FennecAndroid' and chan == 'release' else 1
total = info['count'] * throttle
data[chan] = total
for p, i1 in spikers_info.items():
searches.append(
socorro.SuperSearch(params={
'product': p,
'date': search_date,
'release_channel': list(i1.keys()),
'version': all_versions[p],
'_results_number': 0,
'_facets_size': 5,
'_aggs.release_channel': 'signature'
},
handler=functools.partial(
handler_global, p),
handlerdata=crash_data[p]))
for c in i1.keys():
spikes_number += 1
affected_chans.add(c)
url = socorro.SuperSearch.get_link({
'product': p,
'date': search_date,
'release_channel': c,
'version': versions_pc[p][c],
'uptime': '<60'
})
sgns_chan = interesting_sgns[p]
sgns_stats = [(s, bugs_by_signature[s], t[2], t[1], t[3])
for s, t in sorted(sgns_chan[c].items(),
key=lambda p: p[1][0],
reverse=True)]
sgns_chan[c] = (url, sgns_stats)
for s in searches:
s.wait()
env = Environment(loader=FileSystemLoader('templates'))
env.filters['inflect'] = inflect
template = env.get_template('startup_crashes_email')
_is = OrderedDict()
for product in sorted(interesting_sgns.keys()):
_is[product] = OrderedDict()
for chan in sorted(interesting_sgns[product].keys()):
_is[product][chan] = interesting_sgns[product][chan]
interesting_sgns = _is
body = template.render(
spikes_number=spikes_number,
spikes_number_word=inflect.engine().number_to_words(spikes_number),
crash_data=crash_data,
start_date=utils.get_date_str(new_start_date),
end_date=utils.get_date_str(start_date),
interesting_sgns=interesting_sgns)
title = 'Spikes in startup crashes in %s' % ', '.join(affected_chans)
if emails:
gmail.send(emails, title, body, html=True)
if verbose:
print('Title: %s' % title)
print('Body:')
print(body)
return {'title': title, 'body': body}
return None
def get_sgns_data(channels,
versions,
signatures,
extra,
products,
towait,
date='today'):
today = lmdutils.get_date_ymd(date)
few_days_ago = today - relativedelta(days=config.get_limit())
search_date = socorro.SuperSearch.get_search_date(few_days_ago)
nbase = [0, 0, 0, {}]
data = {}
unique = {}
unique_prod = defaultdict(lambda: dict())
leftovers = {}
allbids = {}
for p, i in versions.items():
allbids[p] = allbids_p = {}
for c, j in i.items():
allbids_p[c] = allbids_pc = []
for b, k in j.items():
allbids_pc.append(b)
v, u, up = k
if u:
unique[b] = (p, c)
elif up:
unique_prod[p][b] = (p, c)
else:
leftovers[b] = (p, c)
for product in products:
data[product] = d1 = {}
b1 = allbids[product]
for chan in channels:
d1[chan] = d2 = {}
b2 = b1[chan]
for signature in signatures:
d2[signature] = b2
if not unique_prod:
# if we've only unique buildids: only N queries for the N signatures
towait.append(
get_sgns_data_helper(data, signatures, unique, nbase, extra,
search_date))
else:
# if a buildid is unique then it's unique for its product too.
# So we've only 2xN queries (2 == len(['Firefox', 'FennecAndroid']))
for b, x in unique.items():
p, c = x
unique_prod[p][b] = (p, c)
for prod, bids in unique_prod.items():
towait.append(
get_sgns_data_helper(data,
signatures,
bids,
nbase,
extra,
search_date,
product=prod))
# handle the leftovers: normally they should be pretty rare
# we've them when they're not unique within the same product (e.g. a nightly and a beta have the same buildid)
for b, x in leftovers.items():
prod, chan = x
towait.append(
get_sgns_data_helper(data,
signatures,
leftovers,
nbase,
extra,
search_date,
product=prod,
channel=chan))
return data
def test_version_dates(self):
self.assertEqual(versions.getMajorDate(46),
datetime.datetime(2016, 4, 26, 7, 0, tzinfo=tzutc()))
self.assertEqual(versions.getMajorDate('46'),
datetime.datetime(2016, 4, 26, 7, 0, tzinfo=tzutc()))
self.assertEqual(versions.getMajorDate('46.0'),
datetime.datetime(2016, 4, 26, 7, 0, tzinfo=tzutc()))
self.assertEqual(versions.getMajorDate('46.0.1'),
datetime.datetime(2016, 4, 26, 7, 0, tzinfo=tzutc()))
self.assertEqual(versions.getMajorDate('1'),
datetime.datetime(2004, 11, 9, 8, 0, tzinfo=tzutc()))
self.assertEqual(versions.getMajorDate('1.0'),
datetime.datetime(2004, 11, 9, 8, 0, tzinfo=tzutc()))
self.assertEqual(versions.getMajorDate('1.5'),
datetime.datetime(2005, 11, 29, 8, 0, tzinfo=tzutc()))
self.assertEqual(versions.getMajorDate('14'),
datetime.datetime(2012, 7, 17, 7, 0, tzinfo=tzutc()))
self.assertEqual(versions.getMajorDate('14.0'),
datetime.datetime(2012, 7, 17, 7, 0, tzinfo=tzutc()))
self.assertEqual(versions.getMajorDate('14.0.1'),
datetime.datetime(2012, 7, 17, 7, 0, tzinfo=tzutc()))
self.assertEqual(versions.getMajorDate('33'),
datetime.datetime(2014, 10, 14, 7, 0, tzinfo=tzutc()))
self.assertEqual(versions.getMajorDate('33.0'),
datetime.datetime(2014, 10, 14, 7, 0, tzinfo=tzutc()))
self.assertEqual(versions.getMajorDate('33.1'),
datetime.datetime(2014, 11, 10, 8, 0, tzinfo=tzutc()))
self.assertEqual(versions.getMajorDate('46'), versions.getDate('46'))
self.assertEqual(versions.getMajorDate('46.0'),
versions.getDate('46.0'))
self.assertNotEqual(versions.getMajorDate('48.0'),
versions.getDate('48.0.1'))
self.assertEqual(versions.getDate('48.0.1'),
datetime.datetime(2016, 8, 18, 7, 0, tzinfo=tzutc()))
self.assertEqual(versions.getDate('48.0.2'),
datetime.datetime(2016, 8, 24, 7, 0, tzinfo=tzutc()))
v = versions.get(base=True)
if versions.getMajorDate(v['nightly']) is not None:
self.assertTrue(
versions.getMajorDate(v['release']) <= versions.getMajorDate(
v['beta']) <= versions.getMajorDate(v['aurora']) <=
versions.getMajorDate(v['nightly']))
else:
self.assertTrue(
versions.getMajorDate(v['release']) <= versions.getMajorDate(
v['beta']) <= versions.getMajorDate(v['aurora']))
date = utils.get_date_ymd('2011-08-24T14:52:52Z')
self.assertEqual(date - versions.getMajorDate('7'),
datetime.timedelta(-34, 28372))
self.assertEqual(
versions.getCloserMajorRelease(date),
('7.0', datetime.datetime(2011, 9, 27, 7, 0, tzinfo=tzutc())))
self.assertEqual(
versions.getCloserMajorRelease(date, negative=True),
('6.0', datetime.datetime(2011, 8, 16, 7, 0, tzinfo=tzutc())))
self.assertEqual(
versions.getCloserMajorRelease(date, negative=False),
('7.0', datetime.datetime(2011, 9, 27, 7, 0, tzinfo=tzutc())))
date = utils.get_date_ymd('2016-08-19')
self.assertEqual(
versions.getCloserRelease(date),
('48.0.2', datetime.datetime(2016, 8, 24, 7, 0, tzinfo=tzutc())))
self.assertEqual(
versions.getCloserRelease(date, negative=True),
('48.0.1', datetime.datetime(2016, 8, 18, 7, 0, tzinfo=tzutc())))
self.assertEqual(
versions.getCloserRelease(date, negative=False),
('48.0.2', datetime.datetime(2016, 8, 24, 7, 0, tzinfo=tzutc())))
if when_blocking is None:
when_blocking = entry['when']
elif change['added'] == '+':
tracked = True
if when_tracked is None:
when_tracked = entry['when']
tracking_value = 'blocking' if blocking else 'tracked' if tracked else 'never'
if bug_data['cf_last_resolved'] is None:
fixing_time = -1
fixing_time_since_blocking = -1
fixing_time_since_tracked = -1
else:
fixing_time = int((
utils.get_date_ymd(bug_data['cf_last_resolved']) -
utils.get_date_ymd(bug_data['creation_time'])).total_seconds())
if blocking:
fixing_time_since_blocking = int(
(utils.get_date_ymd(bug_data['cf_last_resolved']) -
utils.get_date_ymd(when_blocking)).total_seconds())
else:
fixing_time_since_blocking = -1
if tracked:
fixing_time_since_tracked = int(
(utils.get_date_ymd(bug_data['cf_last_resolved']) -
utils.get_date_ymd(when_tracked)).total_seconds())
else:
fixing_time_since_tracked = -1
csv_writer.writerow([
def get(channel, date, versions=None, product='Firefox', duration=1):
"""Get stability info
Args:
channel (str): the channel
date (str): the final date
versions (Optional[List[str]]): the versions to treat
product (Optional[str]): the product
duration (Optional[int]): the duration to retrieve the data
Returns:
dict: contains all the info relative to stability
"""
channel = channel.lower()
cycle = duration <= 0
versions_info = socorro.ProductVersions.get_version_info(versions,
channel=channel,
product=product)
versions = versions_info.keys()
throttle = set(map(lambda p: p[1], versions_info.values()))
diff_throttle = len(throttle) != 1
# normally the throttle is 10% for release and 100% for others channel
if not diff_throttle:
throttle = throttle.pop()
platforms = socorro.Platforms.get_cached_all()
end_date_dt = utils.get_date_ymd(date)
if cycle:
# we get all the start date for each versions and get the min
start_date_dt = min(
map(lambda p: utils.get_date_ymd(p[0]), versions_info.values()))
duration = (end_date_dt - start_date_dt).days + 1
else:
start_date_dt = end_date_dt - timedelta(duration - 1)
start_date_str = utils.get_date_str(start_date_dt)
end_date_str = utils.get_date_str(end_date_dt)
# First, we get the ADI
adi = socorro.ADI.get(version=versions,
product=product,
end_date=end_date_str,
duration=duration,
platforms=platforms)
adi = [adi[key] for key in sorted(adi.keys(), reverse=False)]
# Get the khours
khours = Redash.get_khours(start_date_dt, end_date_dt, channel, versions,
product)
khours = [khours[key] for key in sorted(khours.keys(), reverse=False)]
# Get the # of crashes (crash pings)
crash_pings = Redash.get_number_of_crash(start_date_dt, end_date_dt,
channel, versions, product)
crashes = {}
stats = {'m+c': 0., 'main': 0., 'content': 0., 'plugin': 0., 'all': 0.}
for i in range(duration):
d = end_date_dt - timedelta(i)
crashes[d] = {}
crashes[d]['socorro'] = {
'global': stats.copy(),
'startup': stats.copy()
}
crashes[d]['telemetry'] = crash_pings[d]
base = {
'product': product,
'version': None,
'date': socorro.SuperSearch.get_search_date(start_date_str,
end_date_str),
'release_channel': channel,
'_results_number': 1,
'_histogram.date': ['product', 'process_type'],
'_facets_size': 3
}
if diff_throttle:
# in this case each version could have a different throttle so we need to compute stats for each version
queries = []
for v, t in versions_info.items():
cparams = base.copy()
cparams['version'] = v
queries.append(
Query(socorro.SuperSearch.URL, cparams,
functools.partial(__crash_handler, t[1]), crashes))
cparams = copy.deepcopy(cparams)
cparams['uptime'] = '<60'
cparams['_histogram.date'].append('uptime')
queries.append(
Query(socorro.SuperSearch.URL, cparams,
functools.partial(__crash_handler, t[1]), crashes))
else:
base['version'] = versions
queries = []
queries.append(
Query(socorro.SuperSearch.URL, base,
functools.partial(__crash_handler, throttle), crashes))
cparams = copy.deepcopy(base)
cparams['uptime'] = '<60'
cparams['_histogram.date'].append('uptime')
queries.append(
Query(socorro.SuperSearch.URL, cparams,
functools.partial(__crash_handler, throttle), crashes))
socorro.SuperSearch(queries=queries).wait()
crashes = [crashes[key] for key in sorted(crashes.keys(), reverse=False)]
# Now we compute the rates and the averages
stats = {
'm+c': [0., 0., 0., 0.],
'main': [0., 0., 0., 0.],
'content': [0., 0., 0., 0.],
'plugin': [0., 0., 0., 0.],
'all': [0., 0., 0., 0.]
}
averages = {}
averages['socorro'] = {'global': stats, 'startup': copy.deepcopy(stats)}
averages['telemetry'] = copy.deepcopy(stats)
N = len(adi)
# sum
for i in range(N):
crash_soc = crashes[i]['socorro']
for k1, v1 in averages['socorro'].items():
for k2, av in v1.items():
c = crash_soc[k1][k2]
# the rate is computed for 100 adi
x = utils.rate(100. * c, adi[i])
av[0] += x
av[1] += x**2
y = utils.rate(c, khours[i])
av[2] += y
av[3] += y**2
crash_soc[k1][k2] = (c, x, y)
crash_tel = crashes[i]['telemetry']
for k1, av in averages['telemetry'].items():
c = crash_tel[k1]
# the rate is computed for 100 adi
x = utils.rate(100. * c, adi[i])
av[0] += x
av[1] += x**2
y = utils.rate(c, khours[i])
av[2] += y
av[3] += y**2
crash_tel[k1] = (c, x, y)
N = float(N)
averages_old = {'socorro': {}, 'telemetry': {}}
averages_new = copy.deepcopy(averages_old)
# mean & standard deviation
av_new_soc = averages_new['socorro']
av_old_soc = averages_old['socorro']
for k1, v1 in averages['socorro'].items():
d1 = {}
av_old_soc[k1] = d1
d2 = {}
av_new_soc[k1] = d2
for k2, av in v1.items():
m = av[0] / N
d1[k2] = (m, math.sqrt(av[1] / N - m**2))
m = av[2] / N
d2[k2] = (m, math.sqrt(av[3] / N - m**2))
av_new_tel = averages_new['telemetry']
av_old_tel = averages_old['telemetry']
for k1, av in averages['telemetry'].items():
m = av[0] / N
av_old_tel[k1] = (m, math.sqrt(av[1] / N - m**2))
m = av[2] / N
av_new_tel[k1] = (m, math.sqrt(av[3] / N - m**2))
return {
'start_date': start_date_str,
'end_date': end_date_str,
'versions': versions,
'adi': adi,
'khours': khours,
'crashes': crashes,
'averages_old': averages_old,
'averages_new': averages_new
}
def get_uuids(channel,
product='Firefox',
date='today',
limit=10000,
max_days=3,
threshold=5):
end_date = utils.get_date_ymd(date)
start_date = end_date - timedelta(days=max_days + 1)
search_date = socorro.SuperSearch.get_search_date(start_date, end_date)
r = range(max_days + 1)
default_trend = {start_date + timedelta(days=i): 0 for i in r}
data = defaultdict(lambda: copy.deepcopy(default_trend))
def handler(json, data):
if not json['errors']:
for facets in json['facets']['histogram_date']:
d = utils.get_date_ymd(facets['term'])
s = facets['facets']['signature']
for signature in s:
count = signature['count']
sgn = signature['term']
data[sgn][d] += count
socorro.SuperSearch(params={
'product': product,
'date': search_date,
'release_channel': channel,
'_histogram.date': 'signature',
'_facets_size': limit,
'_results_number': 1
},
handler=handler,
handlerdata=data).wait()
new_signatures = get_new_signatures(data, threshold=threshold)
if new_signatures:
data = {}
queries = []
def handler(json, data):
if not json['errors']:
for facets in json['facets']['proto_signature']:
proto = facets['term']
count = facets['count']
facets = facets['facets']
signature = facets['signature'][0]['term']
first_uuid = facets['uuid'][0]['term']
data[proto] = {
'uuid': first_uuid,
'count': count,
'signature': signature
}
for sgns in Connection.chunks(new_signatures, 5):
queries.append(
Query(socorro.SuperSearch.URL, {
'product': product,
'date': search_date,
'signature': ['=' + s for s in sgns],
'release_channel': channel,
'_aggs.proto_signature': ['uuid', 'signature'],
'_facets_size': 1000,
'_results_number': 0
},
handler=handler,
handlerdata=data))
socorro.SuperSearch(queries=queries).wait()
return data, search_date
return {}, ''
def test_get(self):
class MyConf(libmozdata.config.Config):
def get(self, section, option, default=None, type=str):
if section == 'StatusFlags' and option == 'ignored':
return '\'OOM | small\', \'EMPTY: no crashing thread identified; ERROR_NO_MINIDUMP_HEADER\', \'F1398665248_____________________________\''
else:
return default
config.set_config(MyConf())
base_versions = {
'nightly': 51,
'aurora': 50,
'beta': 49,
'release': 48,
'esr': 45
}
info = statusflags.get('Firefox',
2,
end_date='2016-09-14',
base_versions=base_versions,
check_for_fx=False,
check_bz_version=False,
check_noisy=False,
verbose=False)
self.assertEqual(info['base_versions']['aurora'], 50)
self.assertEqual(info['base_versions']['beta'], 49)
self.assertEqual(info['base_versions']['esr'], 45)
self.assertEqual(info['base_versions']['nightly'], 51)
self.assertEqual(info['base_versions']['release'], 48)
self.assertEqual(info['start_dates']['aurora'],
utils.get_date_ymd('2016-08-01'))
self.assertEqual(info['start_dates']['beta'],
utils.get_date_ymd('2016-08-02'))
self.assertEqual(info['start_dates']['esr'],
utils.get_date_ymd('2016-03-16'))
self.assertEqual(info['start_dates']['nightly'],
utils.get_date_ymd('2016-08-01'))
self.assertEqual(info['start_dates']['release'],
utils.get_date_ymd('2016-07-25'))
self.assertTrue(
'IPCError-browser | ShutDownKill' in info['signatures'])
isgn = info['signatures']['IPCError-browser | ShutDownKill']
self.assertEqual(
set(isgn['leftovers']), {('aurora', 114075), ('nightly', 65105),
('beta', 33208), ('release', 897),
('esr', 23)})
self.assertEqual(isgn['bugid'], 1216774)
self.assertEqual(
set(isgn['bugs']), {
1238657, 1151237, 1216774, 1260551, 1177484, 1173134, 1168272,
1132053, 1133597, 1167902, 1200671, 1213092, 1200646, 1213096,
1223594, 1200685, 1279293, 1206729, 1177425, 1219672, 1205467,
1240542, 1266275, 1290280, 1259125, 1164155, 1150846, 1311297,
1316867, 1289405, 1311869
})
self.assertEqual(set(isgn['platforms']), {'Windows', 'Mac OS X'})
self.assertFalse(isgn['private'])
def getnumbers(b, c, p, g):
return {'browser': b, 'content': c, 'plugin': p, 'gpu': g}
self.assertEqual(isgn['rank']['aurora'], getnumbers(-1, 1, -1, -1))
self.assertEqual(isgn['rank']['beta'], getnumbers(-1, 1, -1, -1))
self.assertEqual(isgn['rank']['esr'], getnumbers(-1, 1, -1, -1))
self.assertEqual(isgn['rank']['nightly'], getnumbers(-1, 1, -1, -1))
self.assertEqual(isgn['rank']['release'], getnumbers(-1, 10, -1, -1))
self.assertTrue(isgn['resolved'])
self.assertEqual(isgn['trend']['aurora'],
[5776, 18706, 21015, 20699, 22058, 20100, 5721])
self.assertEqual(isgn['trend']['beta'],
[206, 784, 817, 987, 1419, 18147, 10848])
self.assertEqual(isgn['trend']['esr'], [
4, 5, 2, 5, 2, 2, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0
])
self.assertEqual(isgn['trend']['nightly'],
[3280, 10192, 10261, 10294, 10631, 11974, 8471])
self.assertEqual(isgn['trend']['release'],
[101, 232, 169, 134, 117, 92, 49, 1])
data = statusflags.generate_bug_report(
'IPCError-browser | ShutDownKill',
isgn,
info['status_flags'],
info['base_versions'],
info['start_dates'],
end_date='2016-09-14',
check_for_fx=False)
self.assertEqual(
data['comment']['body'],
'Crash volume for signature \'IPCError-browser | ShutDownKill\':\n - nightly (version 51): 65105 crashes from 2016-08-01.\n - aurora (version 50): 114075 crashes from 2016-08-01.\n - beta (version 49): 33208 crashes from 2016-08-02.\n - release (version 48): 897 crashes from 2016-07-25.\n - esr (version 45): 23 crashes from 2016-03-16.\n\nCrash volume on the last weeks (Week N is from 09-12 to 09-18):\n W. N-1 W. N-2 W. N-3 W. N-4 W. N-5 W. N-6 W. N-7\n - nightly 10192 10261 10294 10631 11974 8471\n - aurora 18706 21015 20699 22058 20100 5721\n - beta 784 817 987 1419 18147 10848\n - release 232 169 134 117 92 49 1\n - esr 5 2 5 2 2 3 2\n\nAffected platforms: Windows, Mac OS X\n\nCrash rank on the last 7 days:\n Browser Content Plugin\n - nightly #1\n - aurora #1\n - beta #1\n - release #10\n - esr #1'
)
def get(channel,
date,
product='Firefox',
duration=11,
tc_limit=50,
crash_type='all',
startup=False):
"""Get crashes info
Args:
channel (str): the channel
date (str): the final date
product (Optional[str]): the product
duration (Optional[int]): the duration to retrieve the data
tc_limit (Optional[int]): the number of topcrashes to load
crash_type (Optional[str]): 'all' (default) or 'browser' or 'content' or 'plugin'
Returns:
dict: contains all the info relative to the crashes
"""
channel = channel.lower()
version = v[channel]
sys.stdout.write('Getting version information from Socorro...')
sys.stdout.flush()
versions = versions_util.get_channel_versions(channel, product)
sys.stdout.write(' ✔\n')
sys.stdout.flush()
if crash_type and isinstance(crash_type, six.string_types):
crash_type = [crash_type]
_date = utils.get_date_ymd(date)
start_date = utils.get_date_str(_date - timedelta(duration - 1))
end_date = utils.get_date_str(_date)
signatures = {}
def signature_handler(json):
for signature in json['facets']['signature']:
signatures[signature['term']] = [signature['count'], 0, 0, 0, 0, 0]
for platform in signature['facets']['platform']:
if platform['term'] == 'Linux':
signatures[signature['term']][3] = platform['count']
elif platform['term'] == 'Windows NT':
signatures[signature['term']][1] = platform['count']
elif platform['term'] == 'Mac OS X':
signatures[signature['term']][2] = platform['count']
for startup_crash in signature['facets']['startup_crash']:
if startup_crash['term'] in ['1', 'T']:
signatures[signature['term']][4] += startup_crash['count']
signatures[signature['term']][5] = signature['facets'][
'cardinality_install_time']['value']
params = {
'product':
product,
'version':
versions,
'date':
socorro.SuperSearch.get_search_date(start_date, end_date),
'release_channel':
channel,
'_aggs.signature':
['platform', '_cardinality.install_time', 'startup_crash'],
'_results_number':
0,
'_facets_size':
tc_limit,
}
if startup:
params['startup_crash'] = True
sys.stdout.write('Getting top signatures from Socorro...')
sys.stdout.flush()
socorro.SuperSearch(params=params, handler=signature_handler).wait()
sys.stdout.write(' ✔\n')
sys.stdout.flush()
bug_flags = [
'resolution', 'id', 'last_change_time',
'cf_tracking_firefox' + str(version)
]
for i in range(int(version), int(v['nightly']) + 1):
bug_flags.append('cf_status_firefox' + str(i))
# TODO: too many requests... should be improved with chunks
bugs = {}
# TODO: Use regexp, when the Bugzilla bug that prevents them from working will be fixed.
base = {
'j_top': 'OR',
'o1': 'substring',
'f1': 'cf_crash_signature',
'v1': None,
'o2': 'substring',
'f2': 'cf_crash_signature',
'v2': None,
'o3': 'substring',
'f3': 'cf_crash_signature',
'v3': None,
'o4': 'substring',
'f4': 'cf_crash_signature',
'v4': None,
'include_fields': bug_flags
}
queries = []
for sgn in signatures.keys():
cparams = base.copy()
cparams['v1'] = '[@' + sgn + ']'
cparams['v2'] = '[@ ' + sgn + ' ]'
cparams['v3'] = '[@ ' + sgn + ']'
cparams['v4'] = '[@' + sgn + ' ]'
bugs[sgn] = []
queries.append(
Query(Bugzilla.API_URL, cparams, __bug_handler, bugs[sgn]))
res_bugs = Bugzilla(queries=queries)
# we have stats by signature in self.signatures
# for each signature get the number of crashes on the last X days
# so get the signature trend
trends = {}
default_trend = {}
for i in range(duration):
default_trend[_date - timedelta(i)] = 0
base = {
'product': product,
'version': versions,
'signature': None,
'date': socorro.SuperSearch.get_search_date(start_date, end_date),
'release_channel': channel,
'_results_number': 0,
'_histogram.date': ['signature'],
'_histogram_interval': 1
}
queries = []
for sgns in Connection.chunks(
list(map(lambda sgn: '=' + sgn, signatures.keys())), 10):
sgn_group = []
for sgn in sgns:
if sum(len(s) for s in sgn_group) >= 1000:
cparams = base.copy()
cparams['signature'] = sgn_group
queries.append(
Query(socorro.SuperSearch.URL, cparams,
functools.partial(__trend_handler, default_trend),
trends))
sgn_group = []
sgn_group.append(sgn)
if len(sgn_group) > 0:
cparams = base.copy()
cparams['signature'] = sgn_group
queries.append(
Query(socorro.SuperSearch.URL, cparams,
functools.partial(__trend_handler, default_trend),
trends))
sys.stdout.write('Getting trends for top signatures from Socorro...')
sys.stdout.flush()
socorro.SuperSearch(queries=queries).wait()
sys.stdout.write(' ✔\n')
sys.stdout.flush()
for sgn, trend in trends.items():
signatures[sgn] = (signatures[sgn], [
trend[key] for key in sorted(trend.keys(), reverse=True)
])
_signatures = {}
# order self.signatures by crash count
sorted_signatures = sorted(signatures.items(),
key=lambda x: x[1][0][0],
reverse=True)
i = 1
for s in sorted_signatures:
_signatures[s[0]] = i # top crash rank
i += 1
sys.stdout.write(
'Getting bugs linked to the top signatures from Bugzilla...')
sys.stdout.flush()
while True:
try:
for r in res_bugs.results:
r.result(timeout=2)
break
except TimeoutError:
sys.stdout.write('.')
sys.stdout.flush()
sys.stdout.write(' ✔\n')
sys.stdout.flush()
# TODO: In the first query to get the bugs, also get dupe_of and avoid the first query
# in follow_dup (so modify follow_dup to accept both a bug ID or a bug object).
queries = []
for sgn in signatures.keys():
duplicate_ids = [
bug['id'] for bug in bugs[sgn] if bug['resolution'] == 'DUPLICATE'
]
# Remove bugs resolved as DUPLICATE from the list of bugs associated to the signature.
bugs[sgn] = [
bug for bug in bugs[sgn] if bug['id'] not in duplicate_ids
]
# Find duplicates for bugs resolved as DUPLICATE.
duplicates = {
k: v
for k, v in Bugzilla.follow_dup(duplicate_ids).items()
if v is not None
}
duplicate_targets = [
bug_id for bug_id in duplicates.values()
if int(bug_id) not in [bug['id'] for bug in bugs[sgn]]
]
if len(duplicate_targets) == 0:
continue
# Get info about bugs that the DUPLICATE bugs have been duped to.
params = {
'id': ','.join(duplicate_targets),
'include_fields': bug_flags,
}
queries.append(
Query(Bugzilla.API_URL, params, __bug_handler, bugs[sgn]))
sys.stdout.write(
'Resolving duplicate bugs to the bugs they\'ve been duplicated to...')
sys.stdout.flush()
Bugzilla(queries=queries).wait()
sys.stdout.write(' ✔\n')
sys.stdout.flush()
for sgn, stats in signatures.items():
# stats is 2-uple: ([count, win_count, mac_count, linux_count, startup_count], trend)
startup_percent = float(stats[0][4]) / float(stats[0][0])
_signatures[sgn] = {
'tc_rank': _signatures[sgn],
'crash_count': stats[0][0],
'estimated_user_count': stats[0][5],
'startup_percent': startup_percent,
'bugs': bugs[sgn]
}
return {
'start_date': start_date,
'end_date': end_date,
'versions': list(versions),
'signatures': _signatures,
}
def amend_bzparams(self, params, bug_ids):
"""Amend the Bugzilla params"""
if not self.all_include_fields():
if 'include_fields' in params:
fields = params['include_fields']
if isinstance(fields, list):
if 'id' not in fields:
fields.append('id')
elif isinstance(fields, six.string_types):
if fields != 'id':
params['include_fields'] = [fields, 'id']
else:
params['include_fields'] = [fields, 'id']
else:
params['include_fields'] = ['id']
params['include_fields'] += ['summary', 'groups']
if self.has_assignee(
) and 'assigned_to' not in params['include_fields']:
params['include_fields'].append('assigned_to')
if self.has_product_component():
if 'product' not in params['include_fields']:
params['include_fields'].append('product')
if 'component' not in params['include_fields']:
params['include_fields'].append('component')
if self.has_needinfo() and 'flags' not in params['include_fields']:
params['include_fields'].append('flags')
if bug_ids:
params['bug_id'] = bug_ids
if self.filter_no_nag_keyword():
n = utils.get_last_field_num(params)
params.update({
'f' + n: 'status_whiteboard',
'o' + n: 'notsubstring',
'v' + n: '[no-nag]',
})
if self.ignore_meta():
n = utils.get_last_field_num(params)
params.update({
'f' + n: 'keywords',
'o' + n: 'nowords',
'v' + n: 'meta'
})
# Limit the checkers to X years. Unlimited if max_years = -1
max_years = self.get_max_years()
if max_years > 0:
n = utils.get_last_field_num(params)
today = lmdutils.get_date_ymd('today')
few_years_ago = today - relativedelta(years=max_years)
params.update({
'f' + n: 'creation_ts',
'o' + n: 'greaterthan',
'v' + n: few_years_ago
})
if self.has_default_products():
params['product'] = utils.get_config(self.name(), 'products')
self.has_flags = 'flags' in params.get('include_fields', [])
def feed(self, rr=None):
self.data = {}
filenames = {}
if rr is None:
rr = {}
for team, path in utils.get_config('round-robin',
"teams",
default={}).items():
with open('./auto_nag/scripts/configs/{}'.format(path),
'r') as In:
rr[team] = json.load(In)
filenames[team] = path
# rr is dictionary:
# - doc -> documentation
# - triagers -> dictionary: Real Name -> {bzmail: bugzilla email, nick: bugzilla nickname}
# - components -> dictionary: Product::Component -> strategy name
# - strategies: dictionay: {duty en date -> Real Name}
# Get all the strategies for each team
for team, data in rr.items():
if 'doc' in data:
del data['doc']
strategies = {}
triagers = data['triagers']
fallback_bzmail = triagers['Fallback']['bzmail']
path = filenames.get(team, '')
# collect strategies
for pc, strategy in data['components'].items():
strategy_data = data[strategy]
if strategy not in strategies:
strategies[strategy] = strategy_data
# rewrite strategy to have a sorted list of end dates
for strat_name, strategy in strategies.items():
if 'doc' in strategy:
del strategy['doc']
date_name = []
# end date and real name of the triager
for date, name in strategy.items():
# collect the tuple (end_date, bzmail)
date = lmdutils.get_date_ymd(date)
bzmail = triagers[name]['bzmail']
date_name.append((date, bzmail))
# we sort the list to use bisection to find the triager
date_name = sorted(date_name)
strategies[strat_name] = {
'dates': [d for d, _ in date_name],
'mails': [m for _, m in date_name],
'fallback': fallback_bzmail,
'filename': path,
}
# finally self.data is a dictionary:
# - Product::Component -> dictionary {dates: sorted list of end date
# mails: list
# fallback: who to nag when we've nobody
# filename: the file containing strategy}
for pc, strategy in data['components'].items():
self.data[pc] = strategies[strategy]
def test_get_date_ymd(self):
self.assertIsNotNone(utils.get_date_ymd('today'))
self.assertIsNotNone(utils.get_date_ymd('yesterday'))
self.assertIsNotNone(utils.get_date_ymd('tomorrow'))
self.assertTrue(
utils.get_date_ymd('yesterday') < utils.get_date_ymd('today') <
utils.get_date_ymd('tomorrow'))
date = utils.as_utc(
datetime.datetime.strptime('1991-04-16', '%Y-%m-%d'))
self.assertEqual(utils.get_date_ymd('1991/04/16'), date)
self.assertEqual(utils.get_date_ymd('1991-04-16'), date)
self.assertEqual(utils.get_date_ymd('1991 04 16'), date)
self.assertEqual(utils.get_date_ymd('04/16/1991'), date)
self.assertEqual(utils.get_date_ymd('16/04/1991'), date)
self.assertEqual(utils.get_date_ymd('1991-04-16 12:00:00'),
utils.as_utc(datetime.datetime(1991, 4, 16, 12, 0)))
with self.assertRaises(Exception):
utils.get_date_ymd('')
with self.assertRaises(Exception):
utils.get_date_ymd('marco')
def get_sgns_data(channels, versions, signatures, products, date='today'):
today = lmdutils.get_date_ymd(date)
few_days_ago = today - relativedelta(days=config.get_limit())
search_date = socorro.SuperSearch.get_search_date(few_days_ago)
nbase = [0, 0]
data = {}
bids, all_bids, all_versions, doubloons = get_all_buildids(versions)
for product in products:
data[product] = d1 = {}
b1 = bids[product]
for chan in channels:
d1[chan] = d2 = {}
b2 = b1[chan]
for signature in signatures:
d2[signature] = b2
limit = 80
def handler(sgn, json, data):
if not json['facets']['build_id']:
return
for facets in json['facets']['build_id']:
bid = facets['term']
prod, chan, ver = all_bids[str(bid)]
_facets = facets['facets']
chans = set()
for c in _facets['release_channel']:
c = c['term']
if c == 'aurora':
chans.add('beta')
else:
chans.add(c)
if len(_facets['product']) != 1 or len(chans) != 1:
bid = str(bid)
doubloons[bid] = [(prod, chan, ver)]
else:
dpc = data[prod][chan]
nums = dpc[sgn]
bid = utils.get_build_date(bid)
if isinstance(nums, list):
dpc[sgn] = nums = {b: copy.copy(nbase) for b in dpc[sgn]}
if bid in nums:
n = nums[bid]
n[RAW] = facets['count']
N = len(_facets['install_time'])
if N == limit:
N = _facets['cardinality_install_time']['value']
n[INSTALLS] = N
base_params = {
'build_id':
list(all_bids.keys()),
'signature':
'',
'version':
all_versions,
'date':
search_date,
'_aggs.build_id': [
'install_time', '_cardinality.install_time', 'release_channel',
'product'
],
'_results_number':
0,
'_facets':
'signature',
'_facets_size':
limit
}
queries = []
for signature in signatures:
params = copy.deepcopy(base_params)
params['signature'] = '=' + signature
hdler = functools.partial(handler, signature)
queries.append(
Query(socorro.SuperSearch.URL,
params=params,
handler=hdler,
handlerdata=data))
socorro.SuperSearch(queries=queries).wait()
get_sgns_for_doubloons(doubloons, signatures, search_date, data)
res = defaultdict(lambda: defaultdict(lambda: dict()))
for p, i in data.items():
for c, j in i.items():
for sgn, numbers in j.items():
if not isinstance(numbers, list):
res[p][c][sgn] = numbers
return res
def get_uuids_for_spiking_signatures(channel,
cache=None,
product='Firefox',
date='today',
limit=10000,
max_days=3,
threshold=5):
psttz = pytz.timezone('US/Pacific')
end_date = utils.get_date_ymd(date) # 2016-10-18 UTC
end_date_moz = psttz.localize(
datetime(end_date.year, end_date.month,
end_date.day)) # 2016-10-18 PST
end_buildid = utils.get_buildid_from_date(end_date_moz) # < 20161018000000
start_date_moz = end_date_moz - timedelta(
days=max_days + 1) # 2016-10-14 PST (max_days == 3)
start_buildid = utils.get_buildid_from_date(
start_date_moz) # >= 20161014000000
search_buildid = ['>=' + start_buildid, '<' + end_buildid]
start_date = utils.as_utc(start_date_moz) # 2016-10-14 07:00:00 UTC
search_date = '>=' + utils.get_date_str(start_date)
data = defaultdict(lambda: defaultdict(lambda: 0))
buildids = {}
def handler(json, data):
if not json['errors']:
for facets in json['facets']['build_id']:
date = utils.get_date_from_buildid(
facets['term']).astimezone(psttz)
buildids[date] = facets['count']
for s in facets['facets']['signature']:
sgn = s['term']
count = s['count']
data[sgn][date] += count
socorro.SuperSearch(params={
'product': product,
'date': search_date,
'build_id': search_buildid,
'release_channel': channel,
'_aggs.build_id': 'signature',
'_facets_size': limit,
'_results_number': 0
},
handler=handler,
handlerdata=data).wait()
_data = {}
base = {
start_date_moz + timedelta(days=i): {
'buildids': {},
'total': 0
}
for i in range(max_days + 1)
} # from 2016-10-14 to 2016-10-17 PST
for sgn, info in data.items():
d = copy.deepcopy(base)
_data[sgn] = d
for bid, count in info.items():
date = psttz.localize(datetime(bid.year, bid.month, bid.day))
d[date]['buildids'][bid] = count
d[date]['total'] += count
data = _data
spiking_signatures = []
for sgn, info in data.items():
stats2 = [
i['total'] for _, i in sorted(info.items(), key=lambda p: p[0])
]
if all(i == 0 for i in stats2[:-1]) and stats2[-1] >= threshold:
spiking_signatures.append(sgn)
data = None
if spiking_signatures:
# sort the signatures to be sure to always have the same order for the test
spiking_signatures = sorted(spiking_signatures)
start_buildid = utils.get_buildid_from_date(end_date_moz -
timedelta(days=1))
search_buildid = ['>=' + start_buildid, '<' + end_buildid]
queries = []
data = defaultdict(lambda: list())
def handler(json, data):
if not json['errors']:
for facets in json['facets']['proto_signature']:
proto = facets['term']
count = facets['count']
facets = facets['facets']
sgn = facets['signature'][0]['term']
first_uuid = facets['uuid'][0]['term']
uuids = {i['term'] for i in facets['uuid']}
if cache:
i = uuids.intersection(cache['uuids'])
uuid = i.pop() if i else first_uuid
else:
uuid = first_uuid
data[sgn].append({
'proto': proto,
'uuid': uuid,
'count': count
})
for sgns in Connection.chunks(spiking_signatures, 5):
queries.append(
Query(socorro.SuperSearch.URL, {
'product': product,
'date': search_date,
'build_id': search_buildid,
'signature': ['=' + s for s in sgns],
'release_channel': channel,
'_aggs.proto_signature': ['uuid', 'signature'],
'_facets_size': 10000,
'_results_number': 0
},
handler=handler,
handlerdata=data))
socorro.SuperSearch(queries=queries).wait()
return data
type=int,
help='the duration')
parser.add_argument('-p',
'--product',
action='store',
default='Firefox',
help='the product')
parser.add_argument('-v',
'--versions',
action='store',
nargs='+',
help='the Firefox versions')
parser.add_argument(
'--cycle',
action='store_true',
help='duration is computed to take into account all the cycle')
args = parser.parse_args()
if args.startdate:
duration = (utils.get_date_ymd(args.enddate) -
utils.get_date_ymd(args.startdate)).days + 1
else:
duration = -1 if args.cycle else args.duration
stats = get(args.channel,
args.enddate,
product=args.product,
versions=args.versions,
duration=duration)
pprint(stats)
def is_merge_day():
next_merge = get_merge_day()
today = lmdutils.get_date_ymd("today")
return next_merge == today
