def test_get_bugs_info(self):
status_flags = Bugzilla.get_status_flags()
bugs = ['701227', '701232']
info = statusflags.get_bugs_info(bugs, status_flags)
self.assertEqual(set(info.keys()), set(bugs))
info1 = info[bugs[0]]
self.assertEqual(info1['assigned'], True)
self.assertEqual(info1['fixed'], True)
self.assertEqual(info1['incomplete'], False)
self.assertEqual(info1['no_change'], set())
self.assertEqual(info1['patched'], True)
self.assertEqual(info1['resolved'], False)
self.assertEqual(info1['fixed_dates'][0],
utils.get_date_ymd('2011-11-11 10:26:06'))
self.assertEqual(info1['last_change'],
utils.get_date_ymd('2013-01-14 16:31:50'))
info2 = info[bugs[1]]
self.assertEqual(info2['assigned'], True)
self.assertEqual(info2['fixed'], True)
self.assertEqual(info2['incomplete'], False)
self.assertEqual(info2['no_change'], set())
self.assertEqual(info2['patched'], False)
self.assertEqual(info2['resolved'], True)
self.assertEqual(info2['fixed_dates'][0],
utils.get_date_ymd('2011-11-10 03:13:31'))
self.assertEqual(info2['last_change'],
utils.get_date_ymd('2015-03-12 15:17:16'))
bugs = ['844479']
fl = 'cf_status_firefox'
info = statusflags.get_bugs_info(
bugs, {
'nightly': fl + '23',
'aurora': fl + '22',
'beta': fl + '21',
'release': fl + '20'
})
self.assertEqual(set(info.keys()), set(bugs))
info3 = info[bugs[0]]
self.assertEqual(info3['assigned'], True)
self.assertEqual(info3['fixed'], True)
self.assertEqual(info3['incomplete'], False)
self.assertEqual(info3['no_change'], {'aurora', 'beta'})
self.assertEqual(info3['patched'], True)
self.assertEqual(info3['resolved'], True)
self.assertEqual(info3['fixed_dates'][0],
utils.get_date_ymd('2013-08-06 21:12:03'))
self.assertEqual(info3['last_change'],
utils.get_date_ymd('2013-08-20 19:26:07'))
def test_get_partial(self):
channel = ['release', 'beta', 'aurora', 'nightly', 'esr']
signature = 'js::GCMarker::processMarkStackTop'
bugids = socorro.Bugs.get_bugs([signature])
base_versions = {
'nightly': 51,
'aurora': 50,
'beta': 49,
'release': 48,
'esr': 45
}
self.assertEqual(
set(bugids[signature]), {
792226, 789892, 719114, 730283, 1257309, 941491, 745334,
772441, 952381
})
start_date, min_date, versions_by_channel, start_date_by_channel, base_versions = statusflags.get_versions_info(
'Firefox', base_versions=base_versions)
search_date = statusflags.get_search_date('', start_date, '2016-09-14')
sgninfo = statusflags.get_signatures(100, 'Firefox',
versions_by_channel, channel,
search_date, [signature], [],
False)
status_flags = Bugzilla.get_status_flags(base_versions=base_versions)
bugs_history_info = statusflags.get_bugs_info(bugids[signature],
status_flags)
last_bugs_info, _ = statusflags.get_last_bugs_info(
bugids[signature], signature, sgninfo, [], bugs_history_info,
min_date)
self.assertEqual(last_bugs_info['resolved-fixed-patched'],
['', utils.get_guttenberg_death()])
self.assertEqual(last_bugs_info['resolved-fixed-unpatched'],
['', utils.get_guttenberg_death()])
self.assertEqual(
last_bugs_info['resolved-unfixed'],
['952381', utils.get_date_ymd('2014-05-13 12:12:41')])
self.assertEqual(last_bugs_info['unresolved-assigned'],
['', utils.get_guttenberg_death()])
self.assertEqual(
last_bugs_info['unresolved-unassigned'],
['719114', utils.get_date_ymd('2016-08-30 23:15:17')])
last_bug = statusflags.get_last_bug(bugids[signature], signature,
sgninfo, [], bugs_history_info,
min_date)
self.assertEqual(last_bug, '719114')
def test_get_bugs_info(self):
status_flags = Bugzilla.get_status_flags()
bugs = ['701227', '701232']
info = statusflags.get_bugs_info(bugs, status_flags)
self.assertEqual(set(info.keys()), set(bugs))
info1 = info[bugs[0]]
self.assertEqual(info1['assigned'], True)
self.assertEqual(info1['fixed'], True)
self.assertEqual(info1['incomplete'], False)
self.assertEqual(info1['no_change'], set())
self.assertEqual(info1['patched'], True)
self.assertEqual(info1['resolved'], False)
self.assertEqual(info1['fixed_dates'][0], utils.get_date_ymd('2011-11-11 10:26:06'))
self.assertEqual(info1['last_change'], utils.get_date_ymd('2013-01-14 16:31:50'))
info2 = info[bugs[1]]
self.assertEqual(info2['assigned'], True)
self.assertEqual(info2['fixed'], True)
self.assertEqual(info2['incomplete'], False)
self.assertEqual(info2['no_change'], set())
self.assertEqual(info2['patched'], False)
self.assertEqual(info2['resolved'], True)
self.assertEqual(info2['fixed_dates'][0], utils.get_date_ymd('2011-11-10 03:13:31'))
self.assertEqual(info2['last_change'], utils.get_date_ymd('2015-03-12 15:17:16'))
bugs = ['844479']
fl = 'cf_status_firefox'
info = statusflags.get_bugs_info(bugs, {'nightly': fl + '23',
'aurora': fl + '22',
'beta': fl + '21',
'release': fl + '20'})
self.assertEqual(set(info.keys()), set(bugs))
info3 = info[bugs[0]]
self.assertEqual(info3['assigned'], True)
self.assertEqual(info3['fixed'], True)
self.assertEqual(info3['incomplete'], False)
self.assertEqual(info3['no_change'], {'aurora', 'beta'})
self.assertEqual(info3['patched'], True)
self.assertEqual(info3['resolved'], True)
self.assertEqual(info3['fixed_dates'][0], utils.get_date_ymd('2013-08-06 21:12:03'))
self.assertEqual(info3['last_change'], utils.get_date_ymd('2013-08-20 19:26:07'))
def test_get_partial(self):
channel = ['release', 'beta', 'aurora', 'nightly', 'esr']
signature = 'js::GCMarker::processMarkStackTop'
bugids = socorro.Bugs.get_bugs([signature])
base_versions = {'nightly': 51, 'aurora': 50, 'beta': 49, 'release': 48, 'esr': 45}
self.assertEqual(set(bugids[signature]), {792226, 789892, 719114, 730283, 1257309, 941491, 745334, 772441, 952381})
start_date, min_date, versions_by_channel, start_date_by_channel, base_versions = statusflags.get_versions_info('Firefox', base_versions=base_versions)
search_date = statusflags.get_search_date('', start_date, '2016-09-14')
sgninfo = statusflags.get_signatures(100, 'Firefox', versions_by_channel, channel, search_date, [signature], [], False)
status_flags = Bugzilla.get_status_flags(base_versions=base_versions)
bugs_history_info = statusflags.get_bugs_info(bugids[signature], status_flags)
last_bugs_info, _ = statusflags.get_last_bugs_info(bugids[signature], signature, sgninfo, [], bugs_history_info, min_date)
self.assertEqual(last_bugs_info['resolved-fixed-patched'], ['', utils.get_guttenberg_death()])
self.assertEqual(last_bugs_info['resolved-fixed-unpatched'], ['', utils.get_guttenberg_death()])
self.assertEqual(last_bugs_info['resolved-unfixed'], ['952381', utils.get_date_ymd('2014-05-13 12:12:41')])
self.assertEqual(last_bugs_info['unresolved-assigned'], ['', utils.get_guttenberg_death()])
self.assertEqual(last_bugs_info['unresolved-unassigned'], ['719114', utils.get_date_ymd('2016-08-30 23:15:17')])
last_bug = statusflags.get_last_bug(bugids[signature], signature, sgninfo, [], bugs_history_info, min_date)
self.assertEqual(last_bug, '719114')
def get(product='Firefox', limit=1000, verbose=False, search_start_date='', end_date=None, signatures=[], bug_ids=[], max_bugs=-1, base_versions=None, check_for_fx=True, check_bz_version=True, check_noisy=True):
"""Get crashes info
Args:
product (Optional[str]): the product
limit (Optional[int]): the number of crashes to get from tcbs
Returns:
dict: contains all the info about how to update flags
"""
p = product.lower()
if p == 'firefox':
product = 'Firefox'
elif p == 'fennecandroid':
product = 'FennecAndroid'
channel = ['release', 'beta', 'aurora', 'nightly']
if product == 'Firefox':
channel.append('esr')
start_date, min_date, versions_by_channel, start_date_by_channel, base_versions = get_versions_info(product, date=end_date, base_versions=base_versions)
nv = Bugzilla.get_nightly_version()
if check_bz_version and nv != base_versions['nightly']:
__warn('Mismatch between nightly version from Bugzilla (%d) and Socorro (%d)' % (nv, base_versions['nightly']), verbose)
return None
if check_bz_version and (base_versions['aurora'] != nv - 1 or base_versions['beta'] != nv - 2 or base_versions['release'] != nv - 3):
__warn('All versions are not up to date (Bugzilla nightly version is %d): %s' % (nv, base_versions), verbose)
return None
__warn('Versions: %s' % versions_by_channel, verbose)
__warn('Start dates: %s' % start_date_by_channel, verbose)
if not end_date:
end_date = utils.get_date('today')
search_date = get_search_date(search_start_date, start_date, end_date)
signatures = get_signatures(limit, product, versions_by_channel, channel, search_date, signatures, bug_ids, verbose)
# signatures == { 'foo::bar': {'affected_channels': [('release', 1234), ...],
# 'bugs': None,
# 'platforms': ['Windows'],
# 'selected_bug': None}, ... }
__warn('Collected signatures: %d' % len(signatures), verbose)
# get the bugs for each signatures
bugs_by_signature = socorro.Bugs.get_bugs(list(signatures.keys()))
# if we've some bugs in bug_ids then we must remove the other ones for a given signature
if bug_ids:
bids = set(bug_ids)
for s, bugids in bugs_by_signature.items():
inter = bids.intersection(bugids)
if inter:
bugs_by_signature[s] = inter
__warn('Collected bugs in Socorro: Ok', verbose)
bugs, bugs_count = reduce_set_of_bugs(bugs_by_signature)
__warn('Remove duplicates: Ok', verbose)
__warn('Bugs to analyze: %d' % bugs_count, verbose)
# we filter the bugs to remove meaningless ones
if not bug_ids:
bugs = filter_bugs(bugs, product)
status_flags = Bugzilla.get_status_flags(base_versions=base_versions)
# we get the "better" bug where to update the info
bugs_history_info = get_bugs_info(bugs, status_flags)
patched_bugs = []
for bugid, hinfo in bugs_history_info.items():
if hinfo['patched']:
patched_bugs.append(bugid)
if patched_bugs:
patch_info = dataanalysis.analyze_bugs(patched_bugs, min_date=min_date, base_versions=base_versions)
else:
patch_info = {}
crashes_to_reopen = []
bugs.clear()
for s, v in bugs_by_signature.items():
info = signatures[s]
no_change = set()
if v:
bug_to_touch = get_last_bug(v, s, signatures[s], patch_info, bugs_history_info, min_date)
if bug_to_touch:
no_change = bugs_history_info[bug_to_touch]['no_change']
else:
crashes_to_reopen.append(s)
else:
bug_to_touch = None
info['selected_bug'] = bug_to_touch
info['bugs'] = v
info['no_change'] = no_change
if bug_to_touch:
bugs.add(bug_to_touch)
__warn('Collected last bugs: %d' % len(bugs), verbose)
# add bug info in signatures
add_bug_info(signatures, list(bugs), status_flags, product, verbose)
# analyze the signatures
analysis = analyze(signatures, status_flags, base_versions)
if max_bugs > 0:
__analysis = {}
count = 0
for signature, info in analysis.items():
if not check_for_fx or info['firefox']:
__analysis[signature] = info
count += 1
if count == max_bugs:
analysis = __analysis
break
__warn('Analysis: Ok', verbose)
positions_result, positions = get_crash_positions(-1, product, versions_by_channel, channel, search_date=search_date, verbose=verbose)
# Now get the number of crashes for each signature
trends = get_stats_for_past_weeks(product, channel, start_date_by_channel, versions_by_channel, analysis, search_start_date, end_date, check_for_fx=check_for_fx)
if check_noisy:
noisy = get_noisy(trends, analysis)
__warn('Noisy signatures: %s' % [analysis[s] for s in noisy], verbose)
else:
noisy = set()
__warn('Collected trends: Ok\n', verbose)
positions_result.wait()
# replace dictionary containing trends by a list
empty_ranks = {'browser': -1, 'content': -1, 'plugin': -1, 'gpu': -1}
for signature, i in trends.items():
if signature in noisy:
del analysis[signature]
else:
signature_info = analysis[signature]
ranks = signature_info['rank']
for chan, trend in i.items():
i[chan] = [trend[week] for week in sorted(trend.keys(), reverse=False)]
ranks[chan] = positions[chan].get(signature, empty_ranks)
signature_info['trend'] = i
__prettywarn(analysis, verbose)
return {'status_flags': status_flags,
'base_versions': base_versions,
'start_dates': start_date_by_channel,
'signatures': analysis,
'end_date': end_date}
def get(product='Firefox',
limit=1000,
verbose=False,
search_start_date='',
end_date=None,
signatures=[],
bug_ids=[],
max_bugs=-1,
base_versions=None,
check_for_fx=True,
check_bz_version=True,
check_noisy=True):
"""Get crashes info
Args:
product (Optional[str]): the product
limit (Optional[int]): the number of crashes to get from tcbs
Returns:
dict: contains all the info about how to update flags
"""
p = product.lower()
if p == 'firefox':
product = 'Firefox'
elif p == 'fennecandroid':
product = 'FennecAndroid'
channel = ['release', 'beta', 'aurora', 'nightly']
if product == 'Firefox':
channel.append('esr')
start_date, min_date, versions_by_channel, start_date_by_channel, base_versions = get_versions_info(
product, date=end_date, base_versions=base_versions)
nv = Bugzilla.get_nightly_version()
if check_bz_version and nv != base_versions['nightly']:
__warn(
'Mismatch between nightly version from Bugzilla (%d) and Socorro (%d)'
% (nv, base_versions['nightly']), verbose)
return None
if check_bz_version and (base_versions['aurora'] != nv - 1
or base_versions['beta'] != nv - 2
or base_versions['release'] != nv - 3):
__warn(
'All versions are not up to date (Bugzilla nightly version is %d): %s'
% (nv, base_versions), verbose)
return None
__warn('Versions: %s' % versions_by_channel, verbose)
__warn('Start dates: %s' % start_date_by_channel, verbose)
if not end_date:
end_date = utils.get_date('today')
search_date = get_search_date(search_start_date, start_date, end_date)
signatures = get_signatures(limit, product, versions_by_channel, channel,
search_date, signatures, bug_ids, verbose)
# signatures == { 'foo::bar': {'affected_channels': [('release', 1234), ...],
# 'bugs': None,
# 'platforms': ['Windows'],
# 'selected_bug': None}, ... }
__warn('Collected signatures: %d' % len(signatures), verbose)
# get the bugs for each signatures
bugs_by_signature = socorro.Bugs.get_bugs(list(signatures.keys()))
# if we've some bugs in bug_ids then we must remove the other ones for a given signature
if bug_ids:
bids = set(bug_ids)
for s, bugids in bugs_by_signature.items():
inter = bids.intersection(bugids)
if inter:
bugs_by_signature[s] = inter
__warn('Collected bugs in Socorro: Ok', verbose)
bugs, bugs_count = reduce_set_of_bugs(bugs_by_signature)
__warn('Remove duplicates: Ok', verbose)
__warn('Bugs to analyze: %d' % bugs_count, verbose)
# we filter the bugs to remove meaningless ones
if not bug_ids:
bugs = filter_bugs(bugs, product)
status_flags = Bugzilla.get_status_flags(base_versions=base_versions)
# we get the "better" bug where to update the info
bugs_history_info = get_bugs_info(bugs, status_flags)
patched_bugs = []
for bugid, hinfo in bugs_history_info.items():
if hinfo['patched']:
patched_bugs.append(bugid)
if patched_bugs:
patch_info = dataanalysis.analyze_bugs(patched_bugs,
min_date=min_date,
base_versions=base_versions)
else:
patch_info = {}
crashes_to_reopen = []
bugs.clear()
for s, v in bugs_by_signature.items():
info = signatures[s]
no_change = set()
if v:
bug_to_touch = get_last_bug(v, s, signatures[s], patch_info,
bugs_history_info, min_date)
if bug_to_touch:
no_change = bugs_history_info[bug_to_touch]['no_change']
else:
crashes_to_reopen.append(s)
else:
bug_to_touch = None
info['selected_bug'] = bug_to_touch
info['bugs'] = v
info['no_change'] = no_change
if bug_to_touch:
bugs.add(bug_to_touch)
__warn('Collected last bugs: %d' % len(bugs), verbose)
# add bug info in signatures
add_bug_info(signatures, list(bugs), status_flags, product, verbose)
# analyze the signatures
analysis = analyze(signatures, status_flags, base_versions)
if max_bugs > 0:
__analysis = {}
count = 0
for signature, info in analysis.items():
if not check_for_fx or info['firefox']:
__analysis[signature] = info
count += 1
if count == max_bugs:
analysis = __analysis
break
__warn('Analysis: Ok', verbose)
positions_result, positions = get_crash_positions(-1,
product,
versions_by_channel,
channel,
search_date=search_date,
verbose=verbose)
# Now get the number of crashes for each signature
trends = get_stats_for_past_weeks(product,
channel,
start_date_by_channel,
versions_by_channel,
analysis,
search_start_date,
end_date,
check_for_fx=check_for_fx)
if check_noisy:
noisy = get_noisy(trends, analysis)
__warn('Noisy signatures: %s' % [analysis[s] for s in noisy], verbose)
else:
noisy = set()
__warn('Collected trends: Ok\n', verbose)
positions_result.wait()
# replace dictionary containing trends by a list
empty_ranks = {'browser': -1, 'content': -1, 'plugin': -1, 'gpu': -1}
for signature, i in trends.items():
if signature in noisy:
del analysis[signature]
else:
signature_info = analysis[signature]
ranks = signature_info['rank']
for chan, trend in i.items():
i[chan] = [
trend[week] for week in sorted(trend.keys(), reverse=False)
]
ranks[chan] = positions[chan].get(signature, empty_ranks)
signature_info['trend'] = i
__prettywarn(analysis, verbose)
return {
'status_flags': status_flags,
'base_versions': base_versions,
'start_dates': start_date_by_channel,
'signatures': analysis,
'end_date': end_date
}
