def update_all(date=None):
date = magutils.get_date(date)
last = Lastdate.get_last()
if date is None:
yesterday = utils.get_date('yesterday')
if last != yesterday:
date = yesterday
if date:
update_lastdate = True
date = utils.get_date(date)
logger.info('Update all for {}'.format(date))
for p in magutils.get_products():
chans = magutils.get_channels()
for c in chans:
data = crashes_bytype.get(p, c, date=date)
logger.info('Data: {}::{}::{}::{}'.format(p, c, date, data))
if data:
Bytype.put_data(p, c, date, data, commit=False)
data = crashes_categories.get(p, c, date=date)
Categories.put_data(p, c, data, commit=False)
else: # no ADI or no crash data
logger.info('No ADI or no crash data for {}::{}'.format(
p, c))
update_lastdate = False
if update_lastdate and \
magutils.get_date(last) < magutils.get_date(date):
logger.info('Lastdate updated to: {}'.format(date))
Lastdate.put(date, commit=False)
db.session.commit()
def get(channel, versions=None, product='Firefox', start_date=None, end_date='today', duration=30, platforms=None):
if not isinstance(versions, list):
if isinstance(versions, numbers.Number):
versions = socorro.ProductVersions.get_active(vnumber=versions, product=product)
else:
versions = socorro.ProductVersions.get_active(product=product)
versions = versions[channel.lower()]
if start_date:
_sdate = utils.get_date_ymd(start_date)
_edate = utils.get_date_ymd(end_date)
duration = (_edate - _sdate).days
adi = socorro.ADI.get(version=versions, product=product, end_date=end_date, duration=duration, platforms=platforms)
data = {}
for d, n in adi.items():
data[d] = {'adi': n, 'browser': 0, 'content': 0, 'plugin': 0, 'browser_rate': 0, 'content_rate': 0, 'b+c_rate': 0, 'plugin_rate': 0}
start_date = utils.get_date(end_date, duration)
search_date = socorro.SuperSearch.get_search_date(start_date, end_date)
socorro.SuperSearch(params={'product': product,
'version': versions,
'release_channel': channel,
'date': search_date,
'_results_number': 0,
'_facets_size': 2, # 2 is for a facet on plugin and on content
'_histogram.date': ['process_type']},
handler=__super_search_handler,
handlerdata=data).wait()
return data
def get_crash_positions(limit, product, versions, channel, search_date='', end_date='today', verbose=False):
def handler_ss(chan, json, data):
if json['errors']:
__warn('Error in getting ranks for channel %s: %s' % (chan, str(json['errors'])), verbose)
signatures = {}
for sgn in json['facets']['signature']:
total = sgn['count']
signature = sgn['term']
content = 0
plugin = 0
gpu = 0
for pt in sgn['facets']['process_type']:
N = pt['count']
ty = pt['term']
if ty == 'content':
content += N
elif ty == 'plugin':
plugin += N
elif ty == 'gpu':
gpu += N
else:
__warn('Unknown process type: %s' % ty)
browser = total - content - plugin - gpu
signatures[signature] = {'browser': browser, 'content': content, 'plugin': plugin, 'gpu': gpu}
# now we sort the data according to the crash volume
types = ['browser', 'content', 'plugin', 'gpu']
rank = [sorted(signatures.items(), key=lambda t: (-t[1][typ], t[0])) for typ in types]
rank = [{r[i][0]: i + 1 for i in range(len(r))} for r in rank]
for s, v in signatures.items():
signatures[s] = {'browser': -1 if v['browser'] == 0 else rank[0][s],
'content': -1 if v['content'] == 0 else rank[1][s],
'plugin': -1 if v['plugin'] == 0 else rank[2][s],
'gpu': -1 if v['gpu'] == 0 else rank[2][s]}
data[chan] = signatures
queries = []
data = {}
if not search_date:
search_date = socorro.SuperSearch.get_search_date(utils.get_date(end_date, 7))
if limit == -1:
limit = 10000
for chan in channel:
data[chan] = {}
queries.append(Query(socorro.SuperSearch.URL,
{'product': product,
'version': versions[chan],
'release_channel': chan,
'date': search_date,
'_aggs.signature': 'process_type',
'_facets': 'signature',
'_facets_size': limit,
'_results_number': 0},
handler=functools.partial(handler_ss, chan), handlerdata=data))
return socorro.SuperSearch(queries=queries), data
def get_bz_params(self, date):
tomorrow = lmdutils.get_date('tomorrow')
start_date, _ = self.get_dates(date)
fields = ['creator']
params = {
'include_fields': fields,
'resolution': '---',
'f1': 'longdesc',
'o1': 'regexp',
'v1': '([[:<:]]{}[[:>:]])|([[:<:]]{}?[[:>:]])|({}?[ \t]+{}[ \t]+{})[ \t]*:'.format(
*map(utils.bz_ignore_case, ('str', 'steps', 'steps', 'to', 'reproduce'))
),
'f2': 'cf_has_str',
'o2': 'equals',
'v2': '---',
'n3': 1,
'f3': 'cf_has_str',
'o3': 'changedbefore',
'v3': tomorrow,
'f4': 'creation_ts',
'o4': 'greaterthan',
'v4': start_date,
'f5': 'keywords',
'o5': 'notsubstring',
'v5': 'testcase-wanted',
}
return params
def analyze_gfx_critical_errors(signature='',
product='Firefox',
channel=['all'],
versions=[],
start_date=''):
if product.lower() == 'firefox':
product = 'Firefox'
if channel == [] or channel[0].lower() == 'all':
channel = ['release', 'beta', 'nightly']
if product == 'Firefox':
channel.append('esr')
else:
channel = [c.lower() for c in channel]
if not versions:
base_versions = libmozdata.versions.get(base=True)
versions_by_channel = socorro.ProductVersions.get_info_from_major(
base_versions, product=product)
versions = []
for v1 in versions_by_channel.values():
for v2 in v1:
versions.append(v2['version'])
if not start_date:
start_date = utils.get_date('today', 7)
gfx_critical_errors = get_critical_errors()
count = {}
def handler(json, gfx_critical_error):
count[gfx_critical_error] = json['total']
base_params = {
'product': product,
'release_channel': channel,
'version': versions,
'date': '>=' + start_date,
'_results_number': 0,
'_facets_size': 0,
}
if signature:
base_params['signature'] = signature
queries = []
for gfx_critical_error in gfx_critical_errors:
params = base_params.copy()
params['graphics_critical_error'] = '~' + gfx_critical_error
queries.append(
Query(socorro.SuperSearch.URL,
params=params,
handler=handler,
handlerdata=gfx_critical_error))
socorro.SuperSearch(queries=queries).wait()
return count
def get_search_date(search_start_date,
start_date,
end_date=utils.get_date('today')):
if search_start_date:
return socorro.SuperSearch.get_search_date(search_start_date, end_date)
else:
return socorro.SuperSearch.get_search_date(
utils.get_date_str(start_date), end_date)
def update(date='today'):
d = lmdutils.get_date(date)
logger.info('Update data for {}: started.'.format(d))
data, bids, ratios, ranges, last_date = get(date=date)
models.Signatures.put_data(data, bids, ratios)
models.Signatures.clean(ranges)
models.Lastdate.set(last_date)
logger.info('Update data for {}: finished.'.format(d))
def analyze_gfx_critical_errors(signature='', product='Firefox', channel=['all'], versions=[], start_date=''):
if product.lower() == 'firefox':
product = 'Firefox'
if channel == [] or channel[0].lower() == 'all':
channel = ['release', 'beta', 'aurora', 'nightly']
if product == 'Firefox':
channel.append('esr')
else:
channel = [c.lower() for c in channel]
if not versions:
base_versions = libmozdata.versions.get(base=True)
versions_by_channel = socorro.ProductVersions.get_info_from_major(base_versions, product=product)
versions = []
for v1 in versions_by_channel.values():
for v2 in v1:
versions.append(v2['version'])
if not start_date:
start_date = utils.get_date('today', 7)
gfx_critical_errors = get_critical_errors()
count = {}
def handler(json, gfx_critical_error):
count[gfx_critical_error] = json['total']
base_params = {
'product': product,
'release_channel': channel,
'version': versions,
'date': '>=' + start_date,
'_results_number': 0,
'_facets_size': 0,
}
if signature:
base_params['signature'] = signature
queries = []
for gfx_critical_error in gfx_critical_errors:
params = base_params.copy()
params['graphics_critical_error'] = '~' + gfx_critical_error
queries.append(Query(socorro.SuperSearch.URL, params=params, handler=handler, handlerdata=gfx_critical_error))
socorro.SuperSearch(queries=queries).wait()
return count
def prepare(spikes, bugs_by_signature, date, versions, query, ndays):
if spikes:
affected_chans = set()
results = OrderedDict()
today = utils.get_date(date)
params = sputils.get_params_for_link(date, query=query)
for product in sputils.get_products():
if product in spikes:
params['product'] = product
data1 = spikes[product]
results1 = OrderedDict()
results[product] = results1
version_prod = versions[product]
for chan in sputils.get_channels():
if chan in data1:
affected_chans.add(chan)
params['release_channel'] = chan
if version_prod:
params['version'] = version_prod[chan]
results2 = OrderedDict()
results1[chan] = results2
for stats in sorted(
data1[chan],
reverse=True,
key=lambda d:
(d['diff'], d['numbers'][-1], d['signature'])):
sgn = stats['signature']
sgn = sputils.get_str(sgn)
params['signature'] = sputils.get_esearch_sgn(sgn)
url = socorro.SuperSearch.get_link(params)
url += '#crash-reports'
bugs = bugs_by_signature.get(sgn, {})
results3 = OrderedDict()
results2[sgn] = results3
numbers = stats['numbers']
results3['numbers'] = sputils.make_numbers(
today, numbers, ndays)
results3['resolved'] = bugs.get('resolved', None)
results3['unresolved'] = bugs.get(
'unresolved', None)
results3['url'] = url
affected_chans = list(sorted(affected_chans))
return results, affected_chans, today
return None
def get_bz_params(self, date):
tomorrow = lmdutils.get_date('tomorrow')
start_date, _ = self.get_dates(date)
fields = ['creator']
params = {
'include_fields':
fields,
'resolution':
'---',
'f1':
'longdesc',
'o1':
'regexp',
'v1':
'([[:<:]]{}[[:>:]])|([[:<:]]{}?[[:>:]])|({}?[ \t]+{}[ \t]+{})[ \t]*:'
.format(*map(utils.bz_ignore_case, ('str', 'steps', 'steps', 'to',
'reproduce'))),
'f2':
'cf_has_str',
'o2':
'equals',
'v2':
'---',
'n3':
1,
'f3':
'cf_has_str',
'o3':
'changedbefore',
'v3':
tomorrow,
'f4':
'creation_ts',
'o4':
'greaterthan',
'v4':
start_date,
'f5':
'keywords',
'o5':
'notsubstring',
'v5':
'testcase-wanted',
}
return params
def get_bz_params(self, date):
tomorrow = lmdutils.get_date("tomorrow")
start_date, _ = self.get_dates(date)
fields = ["creator"]
params = {
"include_fields":
fields,
"resolution":
"---",
"f1":
"longdesc",
"o1":
"regexp",
"v1":
"([[:<:]]{}[[:>:]])|([[:<:]]{}?[[:>:]])|({}?[ \t]+{}[ \t]+{})[ \t]*:"
.format(*map(utils.bz_ignore_case, ("str", "steps", "steps", "to",
"reproduce"))),
"f2":
"cf_has_str",
"o2":
"equals",
"v2":
"---",
"n3":
1,
"f3":
"cf_has_str",
"o3":
"changedbefore",
"v3":
tomorrow,
"f4":
"creation_ts",
"o4":
"greaterthan",
"v4":
start_date,
"f5":
"keywords",
"o5":
"notsubstring",
"v5":
"testcase-wanted",
}
return params
def send_email(self, date='today'):
"""Send the email"""
if date:
date = lmdutils.get_date(date)
d = lmdutils.get_date_ymd(date)
if isinstance(self, Nag):
self.nag_date = d
if not self.must_run(d):
return
if not self.has_enough_data():
logger.info('The tool {} hasn\'t enough data to run'.format(
self.name()))
return
login_info = utils.get_login_info()
title, body = self.get_email(date)
if title:
receivers = self.get_receivers()
status = 'Success'
try:
mail.send(
login_info['ldap_username'],
receivers,
title,
body,
html=True,
login=login_info,
dryrun=self.dryrun,
)
except: # NOQA
logger.exception('Tool {}'.format(self.name()))
status = 'Failure'
db.Email.add(self.name(), receivers, 'global', status)
if isinstance(self, Nag):
self.send_mails(title, dryrun=self.dryrun)
else:
name = self.name().upper()
if date:
logger.info('{}: No data for {}'.format(name, date))
else:
logger.info('{}: No data'.format(name))
logger.info('Query: {}'.format(self.query_url))
def send_email(self, date="today"):
"""Send the email"""
if date:
date = lmdutils.get_date(date)
d = lmdutils.get_date_ymd(date)
if isinstance(self, Nag):
self.nag_date = d
if not self.must_run(d):
return
if not self.has_enough_data():
logger.info("The tool {} hasn't enough data to run".format(
self.name()))
return
login_info = utils.get_login_info()
title, body = self.get_email(date)
if title:
receivers = self.get_receivers()
status = "Success"
try:
mail.send(
login_info["ldap_username"],
receivers,
title,
body,
html=True,
login=login_info,
dryrun=self.dryrun,
)
except Exception:
logger.exception("Tool {}".format(self.name()))
status = "Failure"
db.Email.add(self.name(), receivers, "global", status)
if isinstance(self, Nag):
self.send_mails(title, dryrun=self.dryrun)
else:
name = self.name().upper()
if date:
logger.info("{}: No data for {}".format(name, date))
else:
logger.info("{}: No data".format(name))
logger.info("Query: {}".format(self.query_url))
def send_email(self, date='today'):
"""Send the email"""
if date:
date = lmdutils.get_date(date)
d = lmdutils.get_date_ymd(date)
if isinstance(self, Nag):
self.nag_date = d
if not self.must_run(d):
return
if not self.has_enough_data():
logger.info('The tool {} hasn\'t enough data to run'.format(self.name()))
return
login_info = utils.get_login_info()
title, body = self.get_email(date)
if title:
receivers = self.get_receivers()
status = 'Success'
try:
mail.send(
login_info['ldap_username'],
receivers,
title,
body,
html=True,
login=login_info,
dryrun=self.dryrun,
)
except: # NOQA
logger.exception('Tool {}'.format(self.name()))
status = 'Failure'
db.Email.add(self.name(), receivers, 'global', status)
if isinstance(self, Nag):
self.send_mails(title, dryrun=self.dryrun)
else:
name = self.name().upper()
if date:
logger.info('{}: No data for {}'.format(name, date))
else:
logger.info('{}: No data'.format(name))
logger.info('Query: {}'.format(self.query_url))
def send():
path = utils.get_config("common", "log")
try:
n = os.path.getsize(path)
if n != 0:
login_info = utils.get_login_info()
date = lmdutils.get_date("today")
msg, files = get_msg(path)
mail.send(
login_info["ldap_username"],
utils.get_config("common", "on-errors"),
"[autonag] Something bad happened when running auto-nag the {}"
.format(date),
msg,
html=False,
login=login_info,
dryrun=False,
files=files,
)
except Exception:
pass
def send():
path = utils.get_config('common', 'log')
try:
n = os.path.getsize(path)
if n != 0:
login_info = utils.get_login_info()
date = lmdutils.get_date('today')
msg, files = get_msg(path)
mail.send(
login_info['ldap_username'],
utils.get_config('common', 'on-errors'),
'[autonag] Something bad happened when running auto-nag the {}'.format(
date
),
msg,
html=False,
login=login_info,
dryrun=False,
files=files,
)
except Exception:
pass
def send_email(self, date='today', dryrun=False):
"""Send the email"""
if date:
date = lmdutils.get_date(date)
d = lmdutils.get_date_ymd(date)
if isinstance(self, Nag):
self.nag_date = d
if not self.must_run(d):
return
if not self.has_enough_data():
logger.info('The tool {} hasn\'t enough data to run'.format(
self.name()))
return
login_info = utils.get_login_info()
title, body = self.get_email(login_info['bz_api_key'], date, dryrun)
if title:
mail.send(
login_info['ldap_username'],
utils.get_config(self.name(), 'receivers'),
title,
body,
html=True,
login=login_info,
dryrun=dryrun,
)
if isinstance(self, Nag):
self.send_mails(title, dryrun=dryrun)
else:
name = self.name().upper()
if date:
logger.info('{}: No data for {}'.format(name, date))
else:
logger.info('{}: No data'.format(name))
def generate_bug_report(sgn, info, status_flags_by_channel, base_versions, start_date_by_channel, end_date, check_for_fx=True):
data = {}
if not check_for_fx or info['firefox']:
volumes = default_volumes.copy()
data = {}
for channel, volume in info['affected']:
data[status_flags_by_channel[channel]] = 'affected'
volumes[channel] = volume
for channel, volume in info['leftovers']:
volumes[channel] = volume
# We begin with the crash volume
comment = 'Crash volume for signature \'%s\':\n' % sgn
table = []
for chan, volume in sorted(volumes.items(), key=lambda k: channel_order[k[0]]):
version = base_versions[chan]
start_date = start_date_by_channel[chan]
plural = 'es' if volume != 1 else ''
table.append(['- %s' % chan,
'(version %d):' % version,
'%d crash%s from %s.' % (volume, plural, utils.get_date(start_date))])
comment += __mk_volume_table(table, 'global')
# Make the table for the trend
table = []
empty = False
N = -1
for chan, trend in sorted(info['trend'].items(), key=lambda k: channel_order[k[0]]):
if len(trend) >= 1:
# we remove data for this week
del(trend[0])
if len(trend) >= 8: # keep only the last seven weeks
trend = trend[:7]
if not trend:
empty = True
break
N = max(N, len(trend))
row = [str(n) for n in trend]
row.insert(0, '- %s' % chan)
table.append(row)
if not empty: # we've trends
monday, sunday = utils.get_monday_sunday(utils.get_date_ymd(end_date))
comment += '\n\nCrash volume on the last weeks (Week N is from %s to %s):\n' % (monday.strftime('%m-%d'), sunday.strftime('%m-%d'))
headers = ['']
for w in range(1, N + 1):
headers.append('W. N-%d' % w)
comment += __mk_volume_table(table, 'byweek', headers=headers)
# Add affected platforms
platforms = info['platforms']
if platforms:
comment += '\n\nAffected platform'
if len(platforms) >= 2:
comment += 's'
platforms = sorted(platforms, key=lambda k: platform_order[k])
comment += ': ' + ', '.join(platforms)
ranks = info['rank']
if ranks:
# check if we've ranks
empty = True
for types in ranks.values():
for v in types.values():
if v != -1:
empty = False
break
if not empty:
comment += '\n\nCrash rank on the last 7 days:\n'
headers = ['', 'Browser', 'Content', 'Plugin']
table = []
def fmt_rank(s):
return None if s == -1 else '#' + str(s)
for chan, types in sorted(ranks.items(), key=lambda k: channel_order[k[0]]):
table.append(['- %s' % chan,
fmt_rank(types['browser']),
fmt_rank(types['content']),
fmt_rank(types['plugin'])])
comment += __mk_volume_table(table, 'rank', headers=headers)
data['comment'] = {'body': comment}
return data
def get(product='Firefox', limit=1000, verbose=False, search_start_date='', end_date=None, signatures=[], bug_ids=[], max_bugs=-1, base_versions=None, check_for_fx=True, check_bz_version=True, check_noisy=True):
"""Get crashes info
Args:
product (Optional[str]): the product
limit (Optional[int]): the number of crashes to get from tcbs
Returns:
dict: contains all the info about how to update flags
"""
p = product.lower()
if p == 'firefox':
product = 'Firefox'
elif p == 'fennecandroid':
product = 'FennecAndroid'
channel = ['release', 'beta', 'aurora', 'nightly']
if product == 'Firefox':
channel.append('esr')
start_date, min_date, versions_by_channel, start_date_by_channel, base_versions = get_versions_info(product, date=end_date, base_versions=base_versions)
nv = Bugzilla.get_nightly_version()
if check_bz_version and nv != base_versions['nightly']:
__warn('Mismatch between nightly version from Bugzilla (%d) and Socorro (%d)' % (nv, base_versions['nightly']), verbose)
return None
if check_bz_version and (base_versions['aurora'] != nv - 1 or base_versions['beta'] != nv - 2 or base_versions['release'] != nv - 3):
__warn('All versions are not up to date (Bugzilla nightly version is %d): %s' % (nv, base_versions), verbose)
return None
__warn('Versions: %s' % versions_by_channel, verbose)
__warn('Start dates: %s' % start_date_by_channel, verbose)
if not end_date:
end_date = utils.get_date('today')
search_date = get_search_date(search_start_date, start_date, end_date)
signatures = get_signatures(limit, product, versions_by_channel, channel, search_date, signatures, bug_ids, verbose)
# signatures == { 'foo::bar': {'affected_channels': [('release', 1234), ...],
# 'bugs': None,
# 'platforms': ['Windows'],
# 'selected_bug': None}, ... }
__warn('Collected signatures: %d' % len(signatures), verbose)
# get the bugs for each signatures
bugs_by_signature = socorro.Bugs.get_bugs(list(signatures.keys()))
# if we've some bugs in bug_ids then we must remove the other ones for a given signature
if bug_ids:
bids = set(bug_ids)
for s, bugids in bugs_by_signature.items():
inter = bids.intersection(bugids)
if inter:
bugs_by_signature[s] = inter
__warn('Collected bugs in Socorro: Ok', verbose)
bugs, bugs_count = reduce_set_of_bugs(bugs_by_signature)
__warn('Remove duplicates: Ok', verbose)
__warn('Bugs to analyze: %d' % bugs_count, verbose)
# we filter the bugs to remove meaningless ones
if not bug_ids:
bugs = filter_bugs(bugs, product)
status_flags = Bugzilla.get_status_flags(base_versions=base_versions)
# we get the "better" bug where to update the info
bugs_history_info = get_bugs_info(bugs, status_flags)
patched_bugs = []
for bugid, hinfo in bugs_history_info.items():
if hinfo['patched']:
patched_bugs.append(bugid)
if patched_bugs:
patch_info = dataanalysis.analyze_bugs(patched_bugs, min_date=min_date, base_versions=base_versions)
else:
patch_info = {}
crashes_to_reopen = []
bugs.clear()
for s, v in bugs_by_signature.items():
info = signatures[s]
no_change = set()
if v:
bug_to_touch = get_last_bug(v, s, signatures[s], patch_info, bugs_history_info, min_date)
if bug_to_touch:
no_change = bugs_history_info[bug_to_touch]['no_change']
else:
crashes_to_reopen.append(s)
else:
bug_to_touch = None
info['selected_bug'] = bug_to_touch
info['bugs'] = v
info['no_change'] = no_change
if bug_to_touch:
bugs.add(bug_to_touch)
__warn('Collected last bugs: %d' % len(bugs), verbose)
# add bug info in signatures
add_bug_info(signatures, list(bugs), status_flags, product, verbose)
# analyze the signatures
analysis = analyze(signatures, status_flags, base_versions)
if max_bugs > 0:
__analysis = {}
count = 0
for signature, info in analysis.items():
if not check_for_fx or info['firefox']:
__analysis[signature] = info
count += 1
if count == max_bugs:
analysis = __analysis
break
__warn('Analysis: Ok', verbose)
positions_result, positions = get_crash_positions(-1, product, versions_by_channel, channel, search_date=search_date, verbose=verbose)
# Now get the number of crashes for each signature
trends = get_stats_for_past_weeks(product, channel, start_date_by_channel, versions_by_channel, analysis, search_start_date, end_date, check_for_fx=check_for_fx)
if check_noisy:
noisy = get_noisy(trends, analysis)
__warn('Noisy signatures: %s' % [analysis[s] for s in noisy], verbose)
else:
noisy = set()
__warn('Collected trends: Ok\n', verbose)
positions_result.wait()
# replace dictionary containing trends by a list
empty_ranks = {'browser': -1, 'content': -1, 'plugin': -1, 'gpu': -1}
for signature, i in trends.items():
if signature in noisy:
del analysis[signature]
else:
signature_info = analysis[signature]
ranks = signature_info['rank']
for chan, trend in i.items():
i[chan] = [trend[week] for week in sorted(trend.keys(), reverse=False)]
ranks[chan] = positions[chan].get(signature, empty_ranks)
signature_info['trend'] = i
__prettywarn(analysis, verbose)
return {'status_flags': status_flags,
'base_versions': base_versions,
'start_dates': start_date_by_channel,
'signatures': analysis,
'end_date': end_date}
def get_search_date(search_start_date, start_date, end_date=utils.get_date('today')):
if search_start_date:
return socorro.SuperSearch.get_search_date(search_start_date, end_date)
else:
return socorro.SuperSearch.get_search_date(utils.get_date_str(start_date), end_date)
def get(product='Firefox',
limit=1000,
verbose=False,
search_start_date='',
end_date=None,
signatures=[],
bug_ids=[],
max_bugs=-1,
base_versions=None,
check_for_fx=True,
check_bz_version=True,
check_noisy=True):
"""Get crashes info
Args:
product (Optional[str]): the product
limit (Optional[int]): the number of crashes to get from tcbs
Returns:
dict: contains all the info about how to update flags
"""
p = product.lower()
if p == 'firefox':
product = 'Firefox'
elif p == 'fennecandroid':
product = 'FennecAndroid'
channel = ['release', 'beta', 'aurora', 'nightly']
if product == 'Firefox':
channel.append('esr')
start_date, min_date, versions_by_channel, start_date_by_channel, base_versions = get_versions_info(
product, date=end_date, base_versions=base_versions)
nv = Bugzilla.get_nightly_version()
if check_bz_version and nv != base_versions['nightly']:
__warn(
'Mismatch between nightly version from Bugzilla (%d) and Socorro (%d)'
% (nv, base_versions['nightly']), verbose)
return None
if check_bz_version and (base_versions['aurora'] != nv - 1
or base_versions['beta'] != nv - 2
or base_versions['release'] != nv - 3):
__warn(
'All versions are not up to date (Bugzilla nightly version is %d): %s'
% (nv, base_versions), verbose)
return None
__warn('Versions: %s' % versions_by_channel, verbose)
__warn('Start dates: %s' % start_date_by_channel, verbose)
if not end_date:
end_date = utils.get_date('today')
search_date = get_search_date(search_start_date, start_date, end_date)
signatures = get_signatures(limit, product, versions_by_channel, channel,
search_date, signatures, bug_ids, verbose)
# signatures == { 'foo::bar': {'affected_channels': [('release', 1234), ...],
# 'bugs': None,
# 'platforms': ['Windows'],
# 'selected_bug': None}, ... }
__warn('Collected signatures: %d' % len(signatures), verbose)
# get the bugs for each signatures
bugs_by_signature = socorro.Bugs.get_bugs(list(signatures.keys()))
# if we've some bugs in bug_ids then we must remove the other ones for a given signature
if bug_ids:
bids = set(bug_ids)
for s, bugids in bugs_by_signature.items():
inter = bids.intersection(bugids)
if inter:
bugs_by_signature[s] = inter
__warn('Collected bugs in Socorro: Ok', verbose)
bugs, bugs_count = reduce_set_of_bugs(bugs_by_signature)
__warn('Remove duplicates: Ok', verbose)
__warn('Bugs to analyze: %d' % bugs_count, verbose)
# we filter the bugs to remove meaningless ones
if not bug_ids:
bugs = filter_bugs(bugs, product)
status_flags = Bugzilla.get_status_flags(base_versions=base_versions)
# we get the "better" bug where to update the info
bugs_history_info = get_bugs_info(bugs, status_flags)
patched_bugs = []
for bugid, hinfo in bugs_history_info.items():
if hinfo['patched']:
patched_bugs.append(bugid)
if patched_bugs:
patch_info = dataanalysis.analyze_bugs(patched_bugs,
min_date=min_date,
base_versions=base_versions)
else:
patch_info = {}
crashes_to_reopen = []
bugs.clear()
for s, v in bugs_by_signature.items():
info = signatures[s]
no_change = set()
if v:
bug_to_touch = get_last_bug(v, s, signatures[s], patch_info,
bugs_history_info, min_date)
if bug_to_touch:
no_change = bugs_history_info[bug_to_touch]['no_change']
else:
crashes_to_reopen.append(s)
else:
bug_to_touch = None
info['selected_bug'] = bug_to_touch
info['bugs'] = v
info['no_change'] = no_change
if bug_to_touch:
bugs.add(bug_to_touch)
__warn('Collected last bugs: %d' % len(bugs), verbose)
# add bug info in signatures
add_bug_info(signatures, list(bugs), status_flags, product, verbose)
# analyze the signatures
analysis = analyze(signatures, status_flags, base_versions)
if max_bugs > 0:
__analysis = {}
count = 0
for signature, info in analysis.items():
if not check_for_fx or info['firefox']:
__analysis[signature] = info
count += 1
if count == max_bugs:
analysis = __analysis
break
__warn('Analysis: Ok', verbose)
positions_result, positions = get_crash_positions(-1,
product,
versions_by_channel,
channel,
search_date=search_date,
verbose=verbose)
# Now get the number of crashes for each signature
trends = get_stats_for_past_weeks(product,
channel,
start_date_by_channel,
versions_by_channel,
analysis,
search_start_date,
end_date,
check_for_fx=check_for_fx)
if check_noisy:
noisy = get_noisy(trends, analysis)
__warn('Noisy signatures: %s' % [analysis[s] for s in noisy], verbose)
else:
noisy = set()
__warn('Collected trends: Ok\n', verbose)
positions_result.wait()
# replace dictionary containing trends by a list
empty_ranks = {'browser': -1, 'content': -1, 'plugin': -1, 'gpu': -1}
for signature, i in trends.items():
if signature in noisy:
del analysis[signature]
else:
signature_info = analysis[signature]
ranks = signature_info['rank']
for chan, trend in i.items():
i[chan] = [
trend[week] for week in sorted(trend.keys(), reverse=False)
]
ranks[chan] = positions[chan].get(signature, empty_ranks)
signature_info['trend'] = i
__prettywarn(analysis, verbose)
return {
'status_flags': status_flags,
'base_versions': base_versions,
'start_dates': start_date_by_channel,
'signatures': analysis,
'end_date': end_date
}
def get_sgns_info(sgns_by_chan,
product='Firefox',
date='today',
query={},
versions=None):
today = utils.get_date(date)
tomorrow = utils.get_date(date, -1)
data = {chan: {} for chan in sgns_by_chan.keys()}
def handler(json, data):
if json['errors'] or not json['facets']['signature']:
return
for facets in json['facets']['signature']:
sgn = facets['term']
count = facets['count']
platforms = defaultdict(lambda: 0)
startup = {True: 0, False: 0}
data[sgn] = {
'count': count,
'platforms': platforms,
'startup': startup
}
facets = facets['facets']
for ppv in facets['platform_pretty_version']:
platforms[ppv['term']] += ppv['count']
for sc in facets['startup_crash']:
term = sc['term']
startup[term == 'T'] += sc['count']
base = {
'product': product,
'date': ['>=' + today, '<' + tomorrow],
'release_channel': '',
'version': '',
'signature': '',
'_aggs.signature': ['platform_pretty_version', 'startup_crash'],
'_results_number': 0,
'_facets_size': 100
}
base.update(query)
searches = []
for chan, signatures in sgns_by_chan.items():
params = copy.copy(base)
params['release_channel'] = chan
if versions:
params['version'] = versions[chan]
for sgns in Connection.chunks(signatures, 10):
p = copy.copy(params)
p['signature'] = ['=' + s for s in sgns]
searches.append(
socorro.SuperSearch(params=p,
handler=handler,
handlerdata=data[chan]))
for s in searches:
s.wait()
return data
def test_get_date(self):
self.assertEqual(utils.get_date('1991/04/16'), '1991-04-16')
self.assertEqual(utils.get_date('1991/04/16', 1), '1991-04-15')
def get_correct_date(date):
date = get_date(date)
if date:
return utils.get_date_str(date)
return utils.get_date('today')
def get(signature,
matching_mode,
module,
addon,
product='Firefox',
channel=['all'],
versions=[],
start_date='',
limit=0,
check_bt=False,
verbose=False,
ratio=1.):
if product.lower() == 'firefox':
product = 'Firefox'
if channel == [] or channel[0].lower() == 'all':
channel = ['release', 'beta', 'aurora', 'nightly']
if product == 'Firefox':
channel.append('esr')
else:
channel = [c.lower() for c in channel]
if not versions:
base_versions = libmozdata.versions.get(base=True)
versions_by_channel = socorro.ProductVersions.get_info_from_major(
base_versions, product=product)
versions = []
for v1 in versions_by_channel.values():
for v2 in v1:
versions.append(v2['version'])
if not start_date:
start_date = utils.get_date('today', 7)
if limit <= 0:
count = []
socorro.SuperSearch(
params={
'product': product,
'version': versions,
'signature': matching_mode + signature,
'release_channel': channel,
'date': '>=' + start_date,
'_facets_size': 1,
'_results_number': 0
},
handler=lambda json: count.append(json['total'])).wait()
limit = count[0]
if limit == 0:
return {'limit': 0}
def handler_ss(json, data):
if json['errors']:
print('Errors occured: %s' % json['errors'])
if json['total']:
for signature in json['facets']['signature']:
for hit in signature['facets']['uuid']:
data.append(hit['term'])
uuids = []
socorro.SuperSearch(params={
'product': product,
'version': versions,
'signature': matching_mode + signature,
'release_channel': channel,
'date': '>=' + start_date,
'_aggs.signature': 'uuid',
'_facets_size': limit,
'_results_number': 0
},
handler=handler_ss,
handlerdata=uuids).wait()
uuids = utils.get_sample(uuids, ratio)
limit = len(uuids)
module = [m.lower() for m in module]
addon = [a.lower() for a in addon]
if verbose:
count = [0]
def handler_pc(json, data):
if verbose:
count[0] += 1
if count[0] % 100 == 0:
print('Treated reports: %d' % count[0])
addon_version = ''
if addon:
for a in json.get('addons', []):
addon_id = a[0].lower()
if len(a) == 2 and addon_id in addon:
versions = data['versions']
addon_version = a[1]
versions[addon_id][addon_version] += 1
break
if not addon_version:
data['not_in_addon'].append(json['uuid'])
if module:
dll_version = ''
json_dump = json['json_dump']
for m in json_dump.get('modules', []):
filename = m['filename'].lower()
if filename in module:
versions = data['versions']
debug_ids = data['debug_ids']
dll_version = m['version']
debug_id = m['debug_id']
versions[filename][dll_version] += 1
debug_ids[filename][debug_id] += 1
# if addon_version and dll_version and (addon_version == dll_version):
# data['match'].append(json['uuid'])
if check_bt and 'crashing_thread' in json_dump:
crashing_thread = json_dump['crashing_thread']
in_bt = False
for frame in crashing_thread['frames']:
if frame['module'].lower() in module:
in_bt = True
break
if not in_bt:
data['not_in_bt'].append(json['uuid'])
info = {
'versions': defaultdict(lambda: defaultdict(int)),
'debug_ids': defaultdict(lambda: defaultdict(int)),
'limit': limit,
'not_in_bt': [],
'not_in_addon': [],
'match': []
}
queries = []
print(str(len(uuids)) + ' reports will be analyzed.')
for uuid in uuids:
queries.append(
Query(socorro.ProcessedCrash.URL,
params={'crash_id': uuid},
handler=handler_pc,
handlerdata=info))
socorro.ProcessedCrash(queries=queries).wait()
return info
def get_crash_positions(limit,
product,
versions,
channel,
search_date='',
end_date='today',
verbose=False):
def handler_ss(chan, json, data):
if json['errors']:
__warn(
'Error in getting ranks for channel %s: %s' %
(chan, str(json['errors'])), verbose)
signatures = {}
for sgn in json['facets']['signature']:
total = sgn['count']
signature = sgn['term']
content = 0
plugin = 0
gpu = 0
for pt in sgn['facets']['process_type']:
N = pt['count']
ty = pt['term']
if ty == 'content':
content += N
elif ty == 'plugin':
plugin += N
elif ty == 'gpu':
gpu += N
else:
__warn('Unknown process type: %s' % ty)
browser = total - content - plugin - gpu
signatures[signature] = {
'browser': browser,
'content': content,
'plugin': plugin,
'gpu': gpu
}
# now we sort the data according to the crash volume
types = ['browser', 'content', 'plugin', 'gpu']
rank = [
sorted(signatures.items(), key=lambda t: (-t[1][typ], t[0]))
for typ in types
]
rank = [{r[i][0]: i + 1 for i in range(len(r))} for r in rank]
for s, v in signatures.items():
signatures[s] = {
'browser': -1 if v['browser'] == 0 else rank[0][s],
'content': -1 if v['content'] == 0 else rank[1][s],
'plugin': -1 if v['plugin'] == 0 else rank[2][s],
'gpu': -1 if v['gpu'] == 0 else rank[2][s]
}
data[chan] = signatures
queries = []
data = {}
if not search_date:
search_date = socorro.SuperSearch.get_search_date(
utils.get_date(end_date, 7))
if limit == -1:
limit = 10000
for chan in channel:
data[chan] = {}
queries.append(
Query(socorro.SuperSearch.URL, {
'product': product,
'version': versions[chan],
'release_channel': chan,
'date': search_date,
'_aggs.signature': 'process_type',
'_facets': 'signature',
'_facets_size': limit,
'_results_number': 0
},
handler=functools.partial(handler_ss, chan),
handlerdata=data))
return socorro.SuperSearch(queries=queries), data
def generate_bug_report(sgn,
info,
status_flags_by_channel,
base_versions,
start_date_by_channel,
end_date,
check_for_fx=True):
data = {}
if not check_for_fx or info['firefox']:
volumes = default_volumes.copy()
data = {}
for channel, volume in info['affected']:
data[status_flags_by_channel[channel]] = 'affected'
volumes[channel] = volume
for channel, volume in info['leftovers']:
volumes[channel] = volume
# We begin with the crash volume
comment = 'Crash volume for signature \'%s\':\n' % sgn
table = []
for chan, volume in sorted(volumes.items(),
key=lambda k: channel_order[k[0]]):
version = base_versions[chan]
start_date = start_date_by_channel[chan]
plural = 'es' if volume != 1 else ''
table.append([
'- %s' % chan,
'(version %d):' % version,
'%d crash%s from %s.' %
(volume, plural, utils.get_date(start_date))
])
comment += __mk_volume_table(table, 'global')
# Make the table for the trend
table = []
empty = False
N = -1
for chan, trend in sorted(info['trend'].items(),
key=lambda k: channel_order[k[0]]):
if len(trend) >= 1:
# we remove data for this week
del (trend[0])
if len(trend) >= 8: # keep only the last seven weeks
trend = trend[:7]
if not trend:
empty = True
break
N = max(N, len(trend))
row = [str(n) for n in trend]
row.insert(0, '- %s' % chan)
table.append(row)
if not empty: # we've trends
monday, sunday = utils.get_monday_sunday(
utils.get_date_ymd(end_date))
comment += '\n\nCrash volume on the last weeks (Week N is from %s to %s):\n' % (
monday.strftime('%m-%d'), sunday.strftime('%m-%d'))
headers = ['']
for w in range(1, N + 1):
headers.append('W. N-%d' % w)
comment += __mk_volume_table(table, 'byweek', headers=headers)
# Add affected platforms
platforms = info['platforms']
if platforms:
comment += '\n\nAffected platform'
if len(platforms) >= 2:
comment += 's'
platforms = sorted(platforms, key=lambda k: platform_order[k])
comment += ': ' + ', '.join(platforms)
ranks = info['rank']
if ranks:
# check if we've ranks
empty = True
for types in ranks.values():
for v in types.values():
if v != -1:
empty = False
break
if not empty:
comment += '\n\nCrash rank on the last 7 days:\n'
headers = ['', 'Browser', 'Content', 'Plugin']
table = []
def fmt_rank(s):
return None if s == -1 else '#' + str(s)
for chan, types in sorted(ranks.items(),
key=lambda k: channel_order[k[0]]):
table.append([
'- %s' % chan,
fmt_rank(types['browser']),
fmt_rank(types['content']),
fmt_rank(types['plugin'])
])
comment += __mk_volume_table(table, 'rank', headers=headers)
data['comment'] = {'body': comment}
return data
def get(signature, matching_mode, module, addon, product='Firefox', channel=['all'], versions=[], start_date='', limit=0, check_bt=False, verbose=False, ratio=1.):
if product.lower() == 'firefox':
product = 'Firefox'
if channel == [] or channel[0].lower() == 'all':
channel = ['release', 'beta', 'aurora', 'nightly']
if product == 'Firefox':
channel.append('esr')
else:
channel = [c.lower() for c in channel]
if not versions:
base_versions = libmozdata.versions.get(base=True)
versions_by_channel = socorro.ProductVersions.get_info_from_major(base_versions, product=product)
versions = []
for v1 in versions_by_channel.values():
for v2 in v1:
versions.append(v2['version'])
if not start_date:
start_date = utils.get_date('today', 7)
if limit <= 0:
count = []
socorro.SuperSearch(params={'product': product,
'version': versions,
'signature': matching_mode + signature,
'release_channel': channel,
'date': '>=' + start_date,
'_facets_size': 1,
'_results_number': 0},
handler=lambda json: count.append(json['total'])).wait()
limit = count[0]
if limit == 0:
return {'limit': 0}
def handler_ss(json, data):
if json['errors']:
print('Errors occured: %s' % json['errors'])
if json['total']:
for signature in json['facets']['signature']:
for hit in signature['facets']['uuid']:
data.append(hit['term'])
uuids = []
socorro.SuperSearch(params={'product': product,
'version': versions,
'signature': matching_mode + signature,
'release_channel': channel,
'date': '>=' + start_date,
'_aggs.signature': 'uuid',
'_facets_size': limit,
'_results_number': 0},
handler=handler_ss, handlerdata=uuids).wait()
uuids = utils.get_sample(uuids, ratio)
limit = len(uuids)
module = [m.lower() for m in module]
addon = [a.lower() for a in addon]
if verbose:
count = [0]
def handler_pc(json, data):
if verbose:
count[0] += 1
if count[0] % 100 == 0:
print('Treated reports: %d' % count[0])
addon_version = ''
if addon:
for a in json.get('addons', []):
addon_id = a[0].lower()
if len(a) == 2 and addon_id in addon:
versions = data['versions']
addon_version = a[1]
versions[addon_id][addon_version] += 1
break
if not addon_version:
data['not_in_addon'].append(json['uuid'])
if module:
dll_version = ''
json_dump = json['json_dump']
for m in json_dump.get('modules', []):
filename = m['filename'].lower()
if filename in module:
versions = data['versions']
debug_ids = data['debug_ids']
dll_version = m['version']
debug_id = m['debug_id']
versions[filename][dll_version] += 1
debug_ids[filename][debug_id] += 1
# if addon_version and dll_version and (addon_version == dll_version):
# data['match'].append(json['uuid'])
if check_bt and 'crashing_thread' in json_dump:
crashing_thread = json_dump['crashing_thread']
in_bt = False
for frame in crashing_thread['frames']:
if frame['module'].lower() in module:
in_bt = True
break
if not in_bt:
data['not_in_bt'].append(json['uuid'])
info = {
'versions': defaultdict(lambda: defaultdict(int)),
'debug_ids': defaultdict(lambda: defaultdict(int)),
'limit': limit,
'not_in_bt': [],
'not_in_addon': [],
'match': []
}
queries = []
print(str(len(uuids)) + ' reports will be analyzed.')
for uuid in uuids:
queries.append(Query(socorro.ProcessedCrash.URL, params={'crash_id': uuid}, handler=handler_pc, handlerdata=info))
socorro.ProcessedCrash(queries=queries).wait()
return info
