def filter_bugs(bugids, product):
bad = []
exclude = []
if product == 'Firefox':
exclude.append('Firefox for Android')
exclude.append('Firefox for iOS')
def bug_handler(bug, data):
if bug.get('status', 'UNCONFIRMED') == 'UNCONFIRMED' or bug.get(
'product', '') in exclude:
return
signatures = bug.get('cf_crash_signature', None)
if signatures:
if '\r\n' not in signatures:
# we should have only one signature
data.add(bug['id'])
else:
if '[@' in signatures:
signatures = map(lambda s: s.strip(' \t\r\n'),
signatures.split('[@'))
signatures = map(lambda s: s[:-1].strip(' \t\r\n'),
filter(None, signatures))
if __is_same_signatures(signatures, [
functools.partial(__name, [
'@0x0',
'F1398665248_____________________________',
'unknown', 'OOM', 'hang', 'small', '_purecall',
'je_free', 'large'
]), __foo_at_address, __foo_args, __const
]):
data.add(bug['id'])
else:
bad.append(signatures)
else:
data.add(bug['id'])
data = set()
Bugzilla(bugids=bugids,
include_fields=['id', 'cf_crash_signature', 'status', 'product'],
bughandler=bug_handler,
bugdata=data).wait()
return data
def __get_signatures_from_bug_ids(bugids):
if not bugids:
return set()
def bug_handler(bug, data):
signatures = bug.get('cf_crash_signature', None)
if signatures:
signatures = map(lambda s: s.strip(' \t\r\n'),
signatures.split('[@'))
signatures = map(lambda s: s[:-1].strip(' \t\r\n'),
filter(None, signatures))
for s in filter(None, signatures):
data.add(s)
data = set()
Bugzilla(bugids=bugids,
include_fields=['cf_crash_signature'],
bughandler=bug_handler,
bugdata=data).wait()
return data
def get(channel, date, product='Firefox', duration=11, tc_limit=50, crash_type='all', startup=False):
"""Get crashes info
Args:
channel (str): the channel
date (str): the final date
product (Optional[str]): the product
duration (Optional[int]): the duration to retrieve the data
tc_limit (Optional[int]): the number of topcrashes to load
crash_type (Optional[str]): 'all' (default) or 'browser' or 'content' or 'plugin'
Returns:
dict: contains all the info relative to the crashes
"""
channel = channel.lower()
version = v[channel]
versions_info = socorro.ProductVersions.get_version_info(version, channel=channel, product=product)
versions = versions_info.keys()
platforms = socorro.Platforms.get_cached_all()
if crash_type and isinstance(crash_type, six.string_types):
crash_type = [crash_type]
throttle = set(map(lambda p: p[1], versions_info.values()))
if len(throttle) == 1:
throttle = throttle.pop()
else:
return
_date = utils.get_date_ymd(date)
start_date = utils.get_date_str(_date - timedelta(duration - 1))
end_date = utils.get_date_str(_date)
# First, we get the ADI
adi = socorro.ADI.get(version=versions, product=product, end_date=end_date, duration=duration, platforms=platforms)
adi = [adi[key] for key in sorted(adi.keys(), reverse=True)]
# get the khours
khours = Redash.get_khours(utils.get_date_ymd(start_date), utils.get_date_ymd(end_date), channel, versions, product)
khours = [khours[key] for key in sorted(khours.keys(), reverse=True)]
overall_crashes_by_day = []
signatures = {}
def signature_handler(json):
for signature in json['facets']['signature']:
signatures[signature['term']] = [signature['count'], 0, 0, 0, 0]
for platform in signature['facets']['platform']:
if platform['term'] == 'Linux':
signatures[signature['term']][3] = platform['count']
elif platform['term'] == 'Windows NT':
signatures[signature['term']][1] = platform['count']
elif platform['term'] == 'Mac OS X':
signatures[signature['term']][2] = platform['count']
for uptime in signature['facets']['uptime']:
if int(uptime['term']) < 60:
signatures[signature['term']][4] += uptime['count']
for facets in json['facets']['histogram_date']:
overall_crashes_by_day.insert(0, facets['count'])
params = {
'product': product,
'version': versions,
'date': socorro.SuperSearch.get_search_date(start_date, end_date),
'release_channel': channel,
'_aggs.signature': ['platform', 'uptime'],
'_results_number': 0,
'_facets_size': tc_limit,
'_histogram.date': ['product'],
'_histogram_interval': 1
}
if startup:
params['uptime'] = '<=60'
socorro.SuperSearch(params=params, handler=signature_handler).wait()
bug_flags = ['resolution', 'id', 'last_change_time', 'cf_tracking_firefox' + str(version)]
for i in range(int(version), int(v['nightly']) + 1):
bug_flags.append('cf_status_firefox' + str(i))
# TODO: too many requests... should be improved with chunks
bugs = {}
# TODO: Use regexp, when the Bugzilla bug that prevents them from working will be fixed.
base = {
'j_top': 'OR',
'o1': 'substring',
'f1': 'cf_crash_signature',
'v1': None,
'o2': 'substring',
'f2': 'cf_crash_signature',
'v2': None,
'o3': 'substring',
'f3': 'cf_crash_signature',
'v3': None,
'o4': 'substring',
'f4': 'cf_crash_signature',
'v4': None,
'include_fields': bug_flags
}
queries = []
for sgn in signatures.keys():
cparams = base.copy()
cparams['v1'] = '[@' + sgn + ']'
cparams['v2'] = '[@ ' + sgn + ' ]'
cparams['v3'] = '[@ ' + sgn + ']'
cparams['v4'] = '[@' + sgn + ' ]'
bugs[sgn] = []
queries.append(Query(Bugzilla.API_URL, cparams, __bug_handler, bugs[sgn]))
res_bugs = Bugzilla(queries=queries)
# we have stats by signature in self.signatures
# for each signature get the number of crashes on the last X days
# so get the signature trend
trends = {}
default_trend = {}
for i in range(duration):
default_trend[_date - timedelta(i)] = 0
base = {'product': product,
'version': versions,
'signature': None,
'date': socorro.SuperSearch.get_search_date(start_date, end_date),
'release_channel': channel,
'_results_number': 0,
'_histogram.date': ['signature'],
'_histogram_interval': 1}
queries = []
for sgns in Connection.chunks(list(map(lambda sgn: '=' + sgn, signatures.keys())), 10):
sgn_group = []
for sgn in sgns:
if sum(len(s) for s in sgn_group) >= 1000:
cparams = base.copy()
cparams['signature'] = sgn_group
queries.append(Query(socorro.SuperSearch.URL, cparams, functools.partial(__trend_handler, default_trend), trends))
sgn_group = []
sgn_group.append(sgn)
if len(sgn_group) > 0:
cparams = base.copy()
cparams['signature'] = sgn_group
queries.append(Query(socorro.SuperSearch.URL, cparams, functools.partial(__trend_handler, default_trend), trends))
socorro.SuperSearch(queries=queries).wait()
for sgn, trend in trends.items():
signatures[sgn] = (signatures[sgn], [trend[key] for key in sorted(trend.keys(), reverse=True)])
_signatures = {}
# order self.signatures by crash count
l = sorted(signatures.items(), key=lambda x: x[1][0][0], reverse=True)
i = 1
for s in l:
_signatures[s[0]] = i # top crash rank
i += 1
res_bugs.wait()
# TODO: In the first query to get the bugs, also get dupe_of and avoid the first query
# in follow_dup (so modify follow_dup to accept both a bug ID or a bug object).
queries = []
for sgn in signatures.keys():
duplicate_ids = [bug['id'] for bug in bugs[sgn] if bug['resolution'] == 'DUPLICATE']
# Remove bugs resolved as DUPLICATE from the list of bugs associated to the signature.
bugs[sgn] = [bug for bug in bugs[sgn] if bug['id'] not in duplicate_ids]
# Find duplicates for bugs resolved as DUPLICATE.
duplicates = {k: v for k, v in Bugzilla.follow_dup(duplicate_ids).items() if v is not None}
duplicate_targets = [bug_id for bug_id in duplicates.values() if int(bug_id) not in [bug['id'] for bug in bugs[sgn]]]
if len(duplicate_targets) == 0:
continue
# Get info about bugs that the DUPLICATE bugs have been duped to.
params = {
'id': ','.join(duplicate_targets),
'include_fields': bug_flags,
}
queries.append(Query(Bugzilla.API_URL, params, __bug_handler, bugs[sgn]))
Bugzilla(queries=queries).wait()
for sgn, stats in signatures.items():
# stats is 2-uple: ([count, win_count, mac_count, linux_count, startup_count], trend)
startup_percent = float(stats[0][4]) / float(stats[0][0])
_signatures[sgn] = {'tc_rank': _signatures[sgn],
'crash_count': stats[0][0],
'startup_percent': startup_percent,
'crash_by_day': stats[1],
'bugs': bugs[sgn]}
return {
'start_date': start_date,
'end_date': end_date,
'versions': list(versions),
'adi': adi,
'khours': khours,
'crash_by_day': overall_crashes_by_day,
'signatures': _signatures,
'throttle': float(throttle)
}
def test_bug_analysis(self):
info = patchanalysis.bug_analysis(547914)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 1)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 11)
self.assertEqual(info['changes_size'], 640)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 1)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 205)
self.assertEqual(info['code_churn_last_3_releases'], 36)
self.assertEqual(info['developer_familiarity_overall'], 13)
self.assertEqual(info['developer_familiarity_last_3_releases'], 1)
self.assertEqual(info['reviewer_familiarity_overall'], 0)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 0)
self.assertIn('*****@*****.**', info['users']['reviewers'])
self.assertIn('*****@*****.**', info['users']['reviewers'])
self.assertEqual(info['users']['assignee']['email'],
'*****@*****.**')
self.assertGreater(info['crashes'], 0)
bug = {}
def bughandler(found_bug, data):
bug.update(found_bug)
def commenthandler(found_bug, bugid, data):
bug['comments'] = found_bug['comments']
def historyhandler(found_bug, data):
bug['history'] = found_bug['history']
def attachmenthandler(attachments, bugid, data):
bug['attachments'] = attachments
Bugzilla('id=547914',
bughandler=bughandler,
commenthandler=commenthandler,
attachmenthandler=attachmenthandler,
historyhandler=historyhandler).get_data().wait()
info2 = patchanalysis.bug_analysis(bug)
self.assertEqual(info2, info)
info = patchanalysis.bug_analysis(647570)
self.assertEqual(info['backout_num'], 1)
self.assertEqual(info['blocks'], 3)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 40)
self.assertEqual(info['changes_size'], 488)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 3)
self.assertEqual(info['r-ed_patches'], 3)
self.assertEqual(info['code_churn_overall'], 184)
self.assertEqual(info['code_churn_last_3_releases'], 31)
self.assertEqual(info['developer_familiarity_overall'], 4)
self.assertEqual(info['developer_familiarity_last_3_releases'], 4)
self.assertEqual(info['reviewer_familiarity_overall'], 16)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 0)
self.assertGreater(info['crashes'], 0)
# Backed out once (when it was on inbound) with changesets from anther bug.
# Author of the patch uses a different email in Bugzilla and Mercurial.
# Reviewer's email doesn't start with his nick, but he's in CC list.
with warnings.catch_warnings(record=True) as w:
info = patchanalysis.bug_analysis(1271794)
self.assertWarnings(w, [
'Revision d0ab0d508a24 was not found.',
'Revision 9f4983dfd881 was not found.',
'Bug 1271794 doesn\'t have a uplift request date.'
])
self.assertEqual(info['backout_num'], 1)
self.assertEqual(info['blocks'], 1)
self.assertEqual(info['depends_on'], 2)
self.assertEqual(info['comments'], 24)
self.assertEqual(info['changes_size'], 76)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 3)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 249)
self.assertEqual(info['code_churn_last_3_releases'], 245)
self.assertEqual(info['developer_familiarity_overall'], 2)
self.assertEqual(info['developer_familiarity_last_3_releases'], 2)
self.assertEqual(info['reviewer_familiarity_overall'], 158)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 157)
self.assertGreaterEqual(info['crashes'], 0)
# Backed out from central and relanded on central.
# One of the reviewers email doesn't start with his nick and he isn't in CC list.
# The author of the patch changed his email on Bugzilla.
info = patchanalysis.bug_analysis(679352)
self.assertEqual(info['backout_num'], 1)
self.assertEqual(info['blocks'], 4)
self.assertEqual(info['depends_on'], 4)
self.assertEqual(info['comments'], 19)
self.assertEqual(info['changes_size'], 8836)
self.assertEqual(info['test_changes_size'], 410)
self.assertEqual(info['modules_num'], 5)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 1076)
self.assertEqual(info['code_churn_last_3_releases'], 183)
self.assertEqual(info['developer_familiarity_overall'], 10)
self.assertEqual(info['developer_familiarity_last_3_releases'], 10)
self.assertEqual(info['reviewer_familiarity_overall'], 57)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 3)
self.assertGreater(info['crashes'], 0)
# Changeset with multiple unrelated backouts (on fx-team).
# Landing comment with long revision (Entire hash instead of first 12 characters).
info = patchanalysis.bug_analysis(384458)
self.assertEqual(info['backout_num'], 1)
self.assertEqual(info['blocks'], 6)
self.assertEqual(info['depends_on'], 47)
self.assertEqual(info['comments'], 106)
self.assertEqual(info['changes_size'], 2752)
self.assertEqual(info['test_changes_size'], 462)
self.assertEqual(info['modules_num'], 11)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 8191)
self.assertEqual(info['code_churn_last_3_releases'], 801)
self.assertEqual(info['developer_familiarity_overall'], 162)
self.assertEqual(info['developer_familiarity_last_3_releases'], 51)
self.assertEqual(info['reviewer_familiarity_overall'], 2)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 0)
self.assertGreater(info['crashes'], 0)
# Custom backout (no reference to revision).
# Author has a different name on Bugzilla and Mercurial and they don't use the email on Mercurial.
with warnings.catch_warnings(record=True) as w:
info = patchanalysis.bug_analysis(1220307)
self.assertWarnings(w, [
'da10eecd0e76 looks like a backout, but we couldn\'t find which revision was backed out.',
'Revision da10eecd0e76 is related to another bug (1276850).',
'Bug 1220307 doesn\'t have a uplift request date.'
])
self.assertEqual(info['backout_num'], 2)
self.assertEqual(info['blocks'], 4)
self.assertEqual(info['depends_on'], 1)
self.assertEqual(info['comments'], 42)
self.assertEqual(info['changes_size'], 67)
self.assertEqual(info['test_changes_size'], 50)
self.assertEqual(info['modules_num'], 3)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 79)
self.assertEqual(info['code_churn_last_3_releases'], 33)
self.assertEqual(info['developer_familiarity_overall'], 5)
self.assertEqual(info['developer_familiarity_last_3_releases'], 5)
self.assertEqual(info['reviewer_familiarity_overall'], 2)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 1)
self.assertGreater(info['crashes'], 0)
with warnings.catch_warnings(record=True) as w:
info = patchanalysis.bug_analysis(1276850)
self.assertWarnings(w, [
'da10eecd0e76 looks like a backout, but we couldn\'t find which revision was backed out.',
'Author [email protected] is not in the list of authors on Bugzilla.',
'Bug 1276850 doesn\'t have a uplift request date.'
])
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 1)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 24)
self.assertEqual(info['changes_size'], 40)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 1)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 26)
self.assertEqual(info['code_churn_last_3_releases'], 21)
self.assertEqual(info['developer_familiarity_overall'], 0)
self.assertEqual(info['developer_familiarity_last_3_releases'], 0)
self.assertEqual(info['reviewer_familiarity_overall'], 26)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 21)
self.assertGreater(info['crashes'], 0)
# No landed patches.
# The author of the patch changed his email on Bugzilla, so past contributions
# are hard to find.
info = patchanalysis.bug_analysis(1007402)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 1)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 41)
self.assertEqual(info['changes_size'], 1035)
self.assertEqual(info['test_changes_size'], 445)
self.assertEqual(info['modules_num'], 6)
self.assertEqual(info['r-ed_patches'], 1)
self.assertEqual(info['code_churn_overall'], 2465)
self.assertEqual(info['code_churn_last_3_releases'], 316)
self.assertEqual(info['developer_familiarity_overall'], 4)
self.assertEqual(info['developer_familiarity_last_3_releases'], 4)
self.assertEqual(info['reviewer_familiarity_overall'], 266)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 15)
self.assertGreaterEqual(info['crashes'], 0)
# No link between Bugzilla account and Mercurial author.
# Reviewer uses different email on Bugzilla and Mercurial.
info = patchanalysis.bug_analysis(901821)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 1)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 11)
self.assertEqual(info['changes_size'], 18)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 1)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 1088)
self.assertEqual(info['code_churn_last_3_releases'], 152)
self.assertEqual(info['developer_familiarity_overall'], 115)
self.assertEqual(info['developer_familiarity_last_3_releases'], 23)
self.assertEqual(info['reviewer_familiarity_overall'], 0)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 0)
self.assertGreaterEqual(info['crashes'], 0)
# Reviewer has different emails on Bugzilla and Mercurial, and his short name is hard to find.
info = patchanalysis.bug_analysis(859425)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 0)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 8)
self.assertEqual(info['changes_size'], 31)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 1)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 79)
self.assertEqual(info['code_churn_last_3_releases'], 30)
self.assertEqual(info['developer_familiarity_overall'], 1)
self.assertEqual(info['developer_familiarity_last_3_releases'], 1)
self.assertEqual(info['reviewer_familiarity_overall'], 0)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 0)
self.assertGreaterEqual(info['crashes'], 0)
# r=bustage
info = patchanalysis.bug_analysis(701875)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 3)
self.assertEqual(info['depends_on'], 1)
self.assertEqual(info['comments'], 69)
self.assertEqual(info['changes_size'], 194)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 5)
self.assertEqual(info['r-ed_patches'], 1)
self.assertEqual(info['code_churn_overall'], 3770)
self.assertEqual(info['code_churn_last_3_releases'], 526)
self.assertEqual(info['developer_familiarity_overall'], 86)
self.assertEqual(info['developer_familiarity_last_3_releases'], 12)
self.assertEqual(info['reviewer_familiarity_overall'], 25)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 5)
self.assertGreaterEqual(info['crashes'], 0)
# Reviewer doesn't have his short name in his Bugzilla name.
with warnings.catch_warnings(record=True) as w:
info = patchanalysis.bug_analysis(853033)
self.assertWarnings(w, [
'Revision 8de609c5d378 is related to another bug (743252).',
'Reviewer jlebar could not be found.'
])
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 2)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 13)
self.assertEqual(info['changes_size'], 18)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 1)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 4)
self.assertEqual(info['code_churn_last_3_releases'], 1)
self.assertEqual(info['developer_familiarity_overall'], 1)
self.assertEqual(info['developer_familiarity_last_3_releases'], 1)
self.assertEqual(info['reviewer_familiarity_overall'], 0)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 0)
self.assertGreaterEqual(info['crashes'], 0)
# There are users in the CC list with empty real names.
info = patchanalysis.bug_analysis(699633)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 0)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 41)
self.assertEqual(info['changes_size'], 179)
self.assertEqual(info['test_changes_size'], 35)
self.assertEqual(info['modules_num'], 1)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 66)
self.assertEqual(info['code_churn_last_3_releases'], 66)
self.assertEqual(info['developer_familiarity_overall'], 28)
self.assertEqual(info['developer_familiarity_last_3_releases'], 28)
self.assertEqual(info['reviewer_familiarity_overall'], 0)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 0)
self.assertGreaterEqual(info['crashes'], 0)
# Reviewer with several IRC names.
info = patchanalysis.bug_analysis(914034)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 2)
self.assertEqual(info['depends_on'], 1)
self.assertEqual(info['comments'], 26)
self.assertEqual(info['changes_size'], 287)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 1)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 240)
self.assertEqual(info['code_churn_last_3_releases'], 27)
self.assertEqual(info['developer_familiarity_overall'], 7)
self.assertEqual(info['developer_familiarity_last_3_releases'], 7)
self.assertEqual(info['reviewer_familiarity_overall'], 3)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 2)
self.assertGreaterEqual(info['crashes'], 0)
# IRC handle in the domain of the email ([email protected]).
info = patchanalysis.bug_analysis(903475)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 1)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 71)
self.assertEqual(info['changes_size'], 18)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 1)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 13)
self.assertEqual(info['code_churn_last_3_releases'], 3)
self.assertEqual(info['developer_familiarity_overall'], 0)
self.assertEqual(info['developer_familiarity_last_3_releases'], 0)
self.assertEqual(info['reviewer_familiarity_overall'], 0)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 0)
self.assertGreaterEqual(info['crashes'], 0)
# Backout without the 'changeset' word.
info = patchanalysis.bug_analysis(829421)
self.assertEqual(info['backout_num'], 1)
self.assertEqual(info['blocks'], 0)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 22)
self.assertEqual(info['changes_size'], 21)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 1)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 110)
self.assertEqual(info['code_churn_last_3_releases'], 21)
self.assertEqual(info['developer_familiarity_overall'], 0)
self.assertEqual(info['developer_familiarity_last_3_releases'], 0)
self.assertEqual(info['reviewer_familiarity_overall'], 11)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 4)
self.assertGreaterEqual(info['crashes'], 0)
# IRC handle first character is lower case in Mercurial, upper case in Bugzilla.
info = patchanalysis.bug_analysis(799266)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 0)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 28)
self.assertEqual(info['changes_size'], 104)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 1)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 355)
self.assertEqual(info['code_churn_last_3_releases'], 37)
self.assertEqual(info['developer_familiarity_overall'], 36)
self.assertEqual(info['developer_familiarity_last_3_releases'], 5)
self.assertEqual(info['reviewer_familiarity_overall'], 1)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 0)
self.assertGreaterEqual(info['crashes'], 0)
# r=IRC_HANDLE_OF_THE_AUTHOR
info = patchanalysis.bug_analysis(721760)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 0)
self.assertEqual(info['depends_on'], 1)
self.assertEqual(info['comments'], 72)
self.assertEqual(info['changes_size'], 216)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 2)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 38)
self.assertEqual(info['code_churn_last_3_releases'], 25)
self.assertEqual(info['developer_familiarity_overall'], 28)
self.assertEqual(info['developer_familiarity_last_3_releases'], 17)
self.assertEqual(info['reviewer_familiarity_overall'], 13)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 6)
self.assertGreaterEqual(info['crashes'], 0)
# IRC handle is ':IRC_HANDLE.SURNAME' and reviewer is not a reviewer of the patch on Bugzilla.
info = patchanalysis.bug_analysis(1021265)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 3)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 111)
self.assertEqual(info['changes_size'], 173)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 5)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 1763)
self.assertEqual(info['code_churn_last_3_releases'], 150)
self.assertEqual(info['developer_familiarity_overall'], 66)
self.assertEqual(info['developer_familiarity_last_3_releases'], 17)
self.assertEqual(info['reviewer_familiarity_overall'], 325)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 36)
self.assertGreaterEqual(info['crashes'], 0)
# IRC handle is the beginning of the real name with a space after.
info = patchanalysis.bug_analysis(1029098)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 1)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 15)
self.assertEqual(info['changes_size'], 94)
self.assertEqual(info['test_changes_size'], 97)
self.assertEqual(info['modules_num'], 1)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 277)
self.assertEqual(info['code_churn_last_3_releases'], 15)
self.assertEqual(info['developer_familiarity_overall'], 81)
self.assertEqual(info['developer_familiarity_last_3_releases'], 8)
self.assertEqual(info['reviewer_familiarity_overall'], 9)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 0)
self.assertGreaterEqual(info['crashes'], 0)
# Typo in the reviewer name.
with warnings.catch_warnings(record=True) as w:
info = patchanalysis.bug_analysis(843733)
self.assertWarnings(w, ['Reviewer mjronseb could not be found.'])
# r=oops
info = patchanalysis.bug_analysis(843821)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 1)
self.assertEqual(info['depends_on'], 1)
self.assertEqual(info['comments'], 21)
self.assertEqual(info['changes_size'], 148)
self.assertEqual(info['test_changes_size'], 0)
self.assertEqual(info['modules_num'], 2)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 887)
self.assertEqual(info['code_churn_last_3_releases'], 149)
self.assertEqual(info['developer_familiarity_overall'], 131)
self.assertEqual(info['developer_familiarity_last_3_releases'], 19)
self.assertEqual(info['reviewer_familiarity_overall'], 7)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 7)
self.assertGreaterEqual(info['crashes'], 0)
# r=backout
info = patchanalysis.bug_analysis(679509)
self.assertEqual(info['backout_num'], 0)
self.assertEqual(info['blocks'], 0)
self.assertEqual(info['depends_on'], 0)
self.assertEqual(info['comments'], 97)
self.assertEqual(info['changes_size'], 347)
self.assertEqual(info['test_changes_size'], 108)
self.assertEqual(info['modules_num'], 5)
self.assertEqual(info['r-ed_patches'], 0)
self.assertEqual(info['code_churn_overall'], 1874)
self.assertEqual(info['code_churn_last_3_releases'], 334)
self.assertEqual(info['developer_familiarity_overall'], 116)
self.assertEqual(info['developer_familiarity_last_3_releases'], 43)
self.assertEqual(info['reviewer_familiarity_overall'], 53)
self.assertEqual(info['reviewer_familiarity_last_3_releases'], 44)
self.assertGreaterEqual(info['crashes'], 0)
# Bugzilla user is impossible to find from IRC handle.
with warnings.catch_warnings(record=True) as w:
info = patchanalysis.bug_analysis(700583)
self.assertWarnings(w, [
'Reviewer [email protected] is not in the list of reviewers on Bugzilla.',
'Bug 700583 doesn\'t have a uplift request date.'
])
# IRC handle is name+surname
info = patchanalysis.bug_analysis(701262)
# r=none
info = patchanalysis.bug_analysis(733614)
# Reviewer on Bugzilla is a different person than the reviewer in the Mercurial commit.
with warnings.catch_warnings(record=True) as w:
info = patchanalysis.bug_analysis(963621)
self.assertWarnings(w, ['Reviewer doublec could not be found.'])
# IRC handle is part of the name.
info = patchanalysis.bug_analysis(829646)
# Multiple backouts with a commit message of one line.
info = patchanalysis.bug_analysis(683280)
# IRC handle on Bugzilla is different than the one used in Mercurial.
info = patchanalysis.bug_analysis(680802)
# Weird situation: the mozilla-central commit referenced in the comments is from some other
# bug and the actual patch from the bug never landed on mozilla-central but directly on
# other channels.
try:
info = patchanalysis.bug_analysis(846986)
except Exception as e:
self.assertTrue(
str(e) in [
'Too many matching authors ([email protected], [email protected]) found for [email protected]',
'Too many matching authors ([email protected], [email protected]) found for [email protected]'
])
# A comment contains a non-existing revision.
with warnings.catch_warnings(record=True) as w:
info = patchanalysis.bug_analysis(1156913)
self.assertWarnings(w, ['Revision fa8854bd0029 doesn\'t exist.'])
# Author in mercurial doesn't use the same format as usual ("Full Name email" instead of "Full Name <email>").
info = patchanalysis.bug_analysis(1277522)
# Check uplift request
info = patchanalysis.bug_analysis(1230065)
self.assertIsNotNone(info['uplift_comment'])
self.assertEqual(len(info['uplift_comment']['text'].split('\n')), 11)
self.assertEqual(info['uplift_comment']['id'], 11222288)
self.assertIsNotNone(info['uplift_author'])
self.assertEqual(info['uplift_author']['email'],
'*****@*****.**')
def update_status_flags(info, update=False):
status_flags_by_channel = info['status_flags']
base_versions = info['base_versions']
channel_order = {
'nightly': 0,
'aurora': 1,
'beta': 2,
'release': 3,
'esr': 4
}
platform_order = {'Windows': 0, 'Mac OS X': 1, 'Linux': 2}
start_date_by_channel = info['start_dates']
for c, d in start_date_by_channel.items():
start_date_by_channel[c] = utils.get_date_str(d)
bugids = []
default_volumes = {c: 0 for c in channel_order.keys()}
for sgn, i in info['signatures'].items():
if i['firefox']:
volumes = default_volumes.copy()
data = {}
bugid = i['bugid']
bugids.append(str(bugid))
for channel, volume in i['affected']:
data[status_flags_by_channel[channel]] = 'affected'
volumes[channel] = volume
for channel, volume in i['leftovers']:
volumes[channel] = volume
if volumes:
comment = 'Crash volume for signature \'%s\':\n' % sgn
table = []
for p in sorted(volumes.items(),
key=lambda k: channel_order[k[0]]):
affected_chan = p[0]
affected_version = base_versions[p[0]]
start_date = start_date_by_channel[p[0]]
volume = p[1]
plural = 'es' if volume > 1 else ''
table.append([
'- %s' % affected_chan,
'(version %d):' % affected_version,
'%d crash%s from %s.' % (volume, plural, start_date)
])
comment += __mk_volume_table(table)
table = []
empty = False
N = -1
for chan, trend in sorted(i['trend'].items(),
key=lambda k: channel_order[k[0]]):
if len(trend) >= 1:
# we remove data for this week
del (trend[0])
if len(trend) >= 8: # keep only the last seven weeks
trend = trend[:7]
if not trend:
empty = True
break
N = max(N, len(trend))
row = [str(n) for n in trend]
row.insert(0, '- %s' % chan)
table.append(row)
if not empty:
comment += '\n\nCrash volume on the last weeks:\n'
headers = ['']
for w in range(1, N + 1):
headers.append('Week N-%d' % w)
comment += __mk_volume_table(table, headers=headers)
platforms = i['platforms']
if platforms:
comment += '\n\nAffected platform'
if len(platforms) >= 2:
comment += 's'
platforms = sorted(platforms,
key=lambda k: platform_order[k])
comment += ': ' + ', '.join(platforms)
print(comment)
data['comment'] = {'body': comment}
if update:
Bugzilla([str(bugid)]).put(data)
pprint((bugid, data))
else:
pprint((bugid, data))
if update:
links = '\n'.join(Bugzilla.get_links(bugids))
print(links)
def get(product='Firefox',
limit=1000,
verbose=False,
search_start_date='',
signatures=[],
bug_ids=[],
max_bugs=-1):
"""Get crashes info
Args:
product (Optional[str]): the product
limit (Optional[int]): the number of crashes to get from tcbs
Returns:
dict: contains all the info about how to update flags
"""
p = product.lower()
if p == 'firefox':
product = 'Firefox'
elif p == 'fennecandroid':
product = 'FennecAndroid'
channel = ['release', 'beta', 'aurora', 'nightly']
if product == 'Firefox':
channel.append('esr')
base_versions = clouseau.versions.get(base=True)
versions_by_channel = socorro.ProductVersions.get_info_from_major(
base_versions, product=product)
channel_by_version = {}
all_versions = []
start_date_by_channel = {}
start_date = utils.get_date_ymd('today')
for chan, versions in versions_by_channel.iteritems():
start_date_by_channel[chan] = utils.get_date_ymd('tomorrow')
for v in versions:
channel_by_version[v['version']] = chan
d = utils.get_date_ymd(v['start_date'])
all_versions.append(v['version'])
if d < start_date:
start_date = d
if d < start_date_by_channel[chan]:
start_date_by_channel[chan] = d
__warn('Versions: %s' % ', '.join(all_versions), verbose)
__warn('Start dates: %s' % start_date_by_channel, verbose)
end_date = utils.get_date('today')
if search_start_date:
search_date = socorro.SuperSearch.get_search_date(
search_start_date, end_date)
else:
search_date = socorro.SuperSearch.get_search_date(
utils.get_date_str(start_date), end_date)
signatures = __get_signatures(limit, product, all_versions, channel,
search_date, signatures, bug_ids, verbose)
__warn('Collected signatures: %d' % len(signatures), verbose)
# get the bugs for each signatures
bugs_by_signature = socorro.Bugs.get_bugs(signatures.keys())
# if we've some bugs in bug_ids then we must remove the other ones for a given signature
if bug_ids:
bids = set(bug_ids)
for s, bugids in bugs_by_signature.items():
inter = bids.intersection(bugids)
if inter:
bugs_by_signature[s] = inter
__warn('Collected bugs in Socorro: Ok', verbose)
# we remove dup bugs
# for example if we've {1,2,3,4,5} and if 2 is a dup of 5 then the set will be reduced to {1,3,4,5}
bugs = set()
for v in bugs_by_signature.values():
bugs = bugs.union(v)
dups = Bugzilla.follow_dup(bugs, only_final=False)
bugs_count = 0
bugs.clear()
for s, bugids in bugs_by_signature.items():
_bugids = set(bugids)
toremove = set()
for bugid in bugids:
chain = dups[str(bugid)]
if chain:
elems = []
for e in chain:
e = int(e)
if e in _bugids:
elems.append(e)
if elems:
elems[
-1] = bugid # we remove the final and put the initial
toremove = toremove.union(elems)
diff = _bugids - toremove
bugs_by_signature[s] = list(diff)
bugs_count += len(diff)
bugs = bugs.union(diff)
__warn('Remove duplicates: Ok', verbose)
__warn('Bugs to analyze: %d' % bugs_count, verbose)
# we filter the bugs to remove meaningless ones
if not bug_ids:
bugs = filter_bugs(bugs, product)
# we get the "better" bug where to update the info
bugs_history_info = __get_bugs_info(bugs)
crashes_to_reopen = []
bugs.clear()
tomorrow = utils.get_date_ymd('tomorrow')
for s, v in bugs_by_signature.items():
info = signatures[s]
if v:
min_date = tomorrow
for i in info['affected_channels']:
if i[0] != 'esr':
d = start_date_by_channel[i[0]]
if d < min_date:
min_date = d
bug_to_touch = get_last_bug(v, bugs_history_info, min_date)
if not bug_to_touch:
crashes_to_reopen.append(s)
else:
bug_to_touch = None
info['selected_bug'] = bug_to_touch
info['bugs'] = v
if bug_to_touch:
bugs.add(bug_to_touch)
__warn('Collected last bugs: %d' % len(bugs), verbose)
# get bug info
include_fields = ['status', 'id', 'cf_crash_signature']
status_flags = {}
for c, v in base_versions.iteritems():
v = str(v)
if c != 'esr':
f1 = 'cf_status_firefox' + v
else:
f1 = 'cf_status_firefox_esr' + v
include_fields.append(f1)
status_flags[c] = f1
bug_info = {}
def bug_handler(bug, data):
data[str(bug['id'])] = bug
Bugzilla(list(bugs),
include_fields=include_fields,
bughandler=bug_handler,
bugdata=bug_info).get_data().wait()
__warn('Collected bug info: Ok', verbose)
for info in signatures.values():
bug = info['selected_bug']
if bug:
if bug in bug_info:
info['selected_bug'] = bug_info[bug]
else:
info['selected_bug'] = 'private'
analysis = __analyze(signatures, status_flags)
if max_bugs > 0:
__analysis = {}
count = 0
for signature, info in analysis.items():
if info['firefox']:
__analysis[signature] = info
count += 1
if count == max_bugs:
analysis = __analysis
break
__warn('Analysis: Ok', verbose)
# Now get the number of crashes for each signature
queries = []
trends = {}
signatures_by_chan = {}
default_trend_by_chan = {}
today = utils.get_date_ymd('today')
ref_w = today.isocalendar()[1]
def get_past_week(date):
isodate = date.isocalendar()
w = isodate[1]
if w > ref_w:
return ref_w - w + 53
else:
return ref_w - w
for chan in channel:
past_w = get_past_week(start_date_by_channel[chan])
default_trend_by_chan[chan] = {i: 0 for i in range(past_w + 1)}
for signature, info in analysis.items():
if info['firefox']:
data = {}
trends[signature] = data
# for chan, volume in info['affected']:
for chan in channel:
if chan in signatures_by_chan:
signatures_by_chan[chan].append(signature)
else:
signatures_by_chan[chan] = [signature]
data[chan] = default_trend_by_chan[chan].copy()
def handler_ss(chan, json, data):
for facets in json['facets']['histogram_date']:
d = utils.get_date_ymd(facets['term'])
w = get_past_week(d)
s = facets['facets']['signature']
for signature in s:
count = signature['count']
sgn = signature['term']
data[sgn][chan][w] += count
for chan, signatures in signatures_by_chan.items():
if search_start_date:
search_date = socorro.SuperSearch.get_search_date(
search_start_date, end_date)
else:
search_date = socorro.SuperSearch.get_search_date(
utils.get_date_str(start_date_by_channel[chan]), end_date)
for sgns in Connection.chunks(signatures, 10):
queries.append(
Query(socorro.SuperSearch.URL, {
'signature': ['=' + s for s in sgns],
'product': product,
'version': all_versions,
'release_channel': chan,
'date': search_date,
'_histogram.date': 'signature',
'_histogram_interval': 1,
'_results_number': 0
},
handler=functools.partial(handler_ss, chan),
handlerdata=trends))
socorro.SuperSearch(queries=queries).wait()
__warn('Collected trends: Ok\n', verbose)
# replace dictionary containing trends by a list
for signature, i in trends.items():
for chan, trend in i.items():
i[chan] = [
trend[week] for week in sorted(trend.keys(), reverse=False)
]
analysis[signature]['trend'] = i
return {
'status_flags': status_flags,
'base_versions': base_versions,
'start_dates': start_date_by_channel,
'signatures': analysis
}
def __get_bugs_info(bugids):
def history_handler(_history, data):
bots = ['*****@*****.**', '*****@*****.**']
bugid = str(_history['id'])
history = _history['history']
if history:
last_change_date = utils.get_guttenberg_death()
has_patch = False
has_assignee = False
is_fixed = False
resolved = False
incomplete = False
for changes in history:
if changes['who'] not in bots:
last_change_date = utils.get_date_ymd(changes['when'])
for change in changes['changes']:
field_name = change.get('field_name', None)
if field_name == 'status':
if change.get('added', None) == 'RESOLVED':
resolved = True
elif change.get('removed', None) == 'RESOLVED':
resolved = False
elif field_name == 'resolution':
added = change.get('added', None)
removed = change.get('removed', None)
if added == 'FIXED':
is_fixed = True
elif added == 'INCOMPLETE':
incomplete = True
if removed == 'FIXED':
is_fixed = False
elif removed == 'INCOMPLETE':
incomplete = False
elif field_name == 'flagtypes.name':
if not has_patch and 'attachment_id' in change and 'added' in change:
added = change['added']
if added.startswith('review'):
has_patch = True
elif field_name == 'assigned_to':
has_assignee = change.get('added',
None) != '*****@*****.**'
data['bugs'][bugid] = {
'resolved':
resolved,
'incomplete':
incomplete,
'fixed':
is_fixed,
'patched':
has_patch,
'assigned':
has_assignee,
'last_change':
last_change_date.astimezone(pytz.utc).replace(tzinfo=None)
}
else:
data['no_history'].append(bugid)
data = {'no_history': [], 'bugs': {}}
Bugzilla(bugids=bugids, historyhandler=history_handler,
historydata=data).wait()
if data['no_history']:
def bug_handler(bug, data):
last_change_date = utils.get_date_ymd(bug['last_change_time'])
data[str(bug['id'])] = {
'resolved':
False,
'incomplete':
False,
'fixed':
False,
'patched':
False,
'assigned':
False,
'last_change':
last_change_date.astimezone(pytz.utc).replace(tzinfo=None)
}
Bugzilla(bugids=data['no_history'],
include_fields=['id', 'last_change_time'],
bughandler=bug_handler,
bugdata=data['bugs']).wait()
return data['bugs']
def get(channel,
date,
product='Firefox',
duration=11,
tc_limit=50,
crash_type='all',
startup=False):
"""Get crashes info
Args:
channel (str): the channel
date (str): the final date
product (Optional[str]): the product
duration (Optional[int]): the duration to retrieve the data
tc_limit (Optional[int]): the number of topcrashes to load
crash_type (Optional[str]): 'all' (default) or 'browser' or 'content' or 'plugin'
Returns:
dict: contains all the info relative to the crashes
"""
channel = channel.lower()
version = v[channel]
versions_info = socorro.ProductVersions.get_version_info(version,
channel=channel,
product=product)
versions = versions_info.keys()
platforms = socorro.Platforms.get_cached_all()
if crash_type and isinstance(crash_type, six.string_types):
crash_type = [crash_type]
throttle = set(map(lambda p: p[1], versions_info.values()))
if len(throttle) == 1:
throttle = throttle.pop()
else:
return
_date = utils.get_date_ymd(date)
start_date = utils.get_date_str(_date - timedelta(duration - 1))
end_date = utils.get_date_str(_date)
# First, we get the ADI
adi = socorro.ADI.get(version=versions,
product=product,
end_date=end_date,
duration=duration,
platforms=platforms)
adi = [adi[key] for key in sorted(adi.keys(), reverse=True)]
# get the khours
khours = Redash.get_khours(utils.get_date_ymd(start_date),
utils.get_date_ymd(end_date), channel, versions,
product)
khours = [khours[key] for key in sorted(khours.keys(), reverse=True)]
overall_crashes_by_day = []
signatures = {}
def signature_handler(json):
for signature in json['facets']['signature']:
signatures[signature['term']] = [signature['count'], 0, 0, 0, 0]
for platform in signature['facets']['platform']:
if platform['term'] == 'Linux':
signatures[signature['term']][3] = platform['count']
elif platform['term'] == 'Windows NT':
signatures[signature['term']][1] = platform['count']
elif platform['term'] == 'Mac OS X':
signatures[signature['term']][2] = platform['count']
for uptime in signature['facets']['uptime']:
if int(uptime['term']) < 60:
signatures[signature['term']][4] += uptime['count']
for facets in json['facets']['histogram_date']:
overall_crashes_by_day.insert(0, facets['count'])
params = {
'product': product,
'version': versions,
'date': socorro.SuperSearch.get_search_date(start_date, end_date),
'release_channel': channel,
'_aggs.signature': ['platform', 'uptime'],
'_results_number': 0,
'_facets_size': tc_limit,
'_histogram.date': ['product'],
'_histogram_interval': 1
}
if startup:
params['uptime'] = '<=60'
socorro.SuperSearch(params=params, handler=signature_handler).wait()
bug_flags = [
'resolution', 'id', 'last_change_time',
'cf_tracking_firefox' + str(version)
]
for i in range(int(version), int(v['nightly']) + 1):
bug_flags.append('cf_status_firefox' + str(i))
# TODO: too many requests... should be improved with chunks
bugs = {}
# TODO: Use regexp, when the Bugzilla bug that prevents them from working will be fixed.
base = {
'j_top': 'OR',
'o1': 'substring',
'f1': 'cf_crash_signature',
'v1': None,
'o2': 'substring',
'f2': 'cf_crash_signature',
'v2': None,
'o3': 'substring',
'f3': 'cf_crash_signature',
'v3': None,
'o4': 'substring',
'f4': 'cf_crash_signature',
'v4': None,
'include_fields': bug_flags
}
queries = []
for sgn in signatures.keys():
cparams = base.copy()
cparams['v1'] = '[@' + sgn + ']'
cparams['v2'] = '[@ ' + sgn + ' ]'
cparams['v3'] = '[@ ' + sgn + ']'
cparams['v4'] = '[@' + sgn + ' ]'
bugs[sgn] = []
queries.append(
Query(Bugzilla.API_URL, cparams, __bug_handler, bugs[sgn]))
res_bugs = Bugzilla(queries=queries)
# we have stats by signature in self.signatures
# for each signature get the number of crashes on the last X days
# so get the signature trend
trends = {}
default_trend = {}
for i in range(duration):
default_trend[_date - timedelta(i)] = 0
base = {
'product': product,
'version': versions,
'signature': None,
'date': socorro.SuperSearch.get_search_date(start_date, end_date),
'release_channel': channel,
'_results_number': 0,
'_histogram.date': ['signature'],
'_histogram_interval': 1
}
queries = []
for sgns in Connection.chunks(
list(map(lambda sgn: '=' + sgn, signatures.keys())), 10):
sgn_group = []
for sgn in sgns:
if sum(len(s) for s in sgn_group) >= 1000:
cparams = base.copy()
cparams['signature'] = sgn_group
queries.append(
Query(socorro.SuperSearch.URL, cparams,
functools.partial(__trend_handler, default_trend),
trends))
sgn_group = []
sgn_group.append(sgn)
if len(sgn_group) > 0:
cparams = base.copy()
cparams['signature'] = sgn_group
queries.append(
Query(socorro.SuperSearch.URL, cparams,
functools.partial(__trend_handler, default_trend),
trends))
socorro.SuperSearch(queries=queries).wait()
for sgn, trend in trends.items():
signatures[sgn] = (signatures[sgn], [
trend[key] for key in sorted(trend.keys(), reverse=True)
])
_signatures = {}
# order self.signatures by crash count
l = sorted(signatures.items(), key=lambda x: x[1][0][0], reverse=True)
i = 1
for s in l:
_signatures[s[0]] = i # top crash rank
i += 1
res_bugs.wait()
# TODO: In the first query to get the bugs, also get dupe_of and avoid the first query
# in follow_dup (so modify follow_dup to accept both a bug ID or a bug object).
queries = []
for sgn in signatures.keys():
duplicate_ids = [
bug['id'] for bug in bugs[sgn] if bug['resolution'] == 'DUPLICATE'
]
# Remove bugs resolved as DUPLICATE from the list of bugs associated to the signature.
bugs[sgn] = [
bug for bug in bugs[sgn] if bug['id'] not in duplicate_ids
]
# Find duplicates for bugs resolved as DUPLICATE.
duplicates = {
k: v
for k, v in Bugzilla.follow_dup(duplicate_ids).items()
if v is not None
}
duplicate_targets = [
bug_id for bug_id in duplicates.values()
if int(bug_id) not in [bug['id'] for bug in bugs[sgn]]
]
if len(duplicate_targets) == 0:
continue
# Get info about bugs that the DUPLICATE bugs have been duped to.
params = {
'id': ','.join(duplicate_targets),
'include_fields': bug_flags,
}
queries.append(
Query(Bugzilla.API_URL, params, __bug_handler, bugs[sgn]))
Bugzilla(queries=queries).wait()
for sgn, stats in signatures.items():
# stats is 2-uple: ([count, win_count, mac_count, linux_count, startup_count], trend)
startup_percent = float(stats[0][4]) / float(stats[0][0])
_signatures[sgn] = {
'tc_rank': _signatures[sgn],
'crash_count': stats[0][0],
'startup_percent': startup_percent,
'crash_by_day': stats[1],
'bugs': bugs[sgn]
}
return {
'start_date': start_date,
'end_date': end_date,
'versions': list(versions),
'adi': adi,
'khours': khours,
'crash_by_day': overall_crashes_by_day,
'signatures': _signatures,
'throttle': float(throttle)
}