def test_related(self):
models.BugAssociation.objects.create(
bug_id='999999',
signature='OOM | small'
)
models.BugAssociation.objects.create(
bug_id='999999',
signature='OOM | medium'
)
models.BugAssociation.objects.create(
bug_id='1000000',
signature='OOM | large'
)
api = models.Bugs()
resp = api.get(signatures=['OOM | small'])
assert resp == {
'hits': [
{
'id': 999999,
'signature': 'OOM | medium'
},
{
'id': 999999,
'signature': 'OOM | small'
}
],
'total': 2
}
def test_related(self):
models.BugAssociation.objects.create(bug_id="999999",
signature="OOM | small")
models.BugAssociation.objects.create(bug_id="999999",
signature="OOM | medium")
models.BugAssociation.objects.create(bug_id="1000000",
signature="OOM | large")
api = models.Bugs()
resp = api.get(signatures=["OOM | small"])
assert resp == {
"hits": [
{
"id": 999999,
"signature": "OOM | medium"
},
{
"id": 999999,
"signature": "OOM | small"
},
],
"total":
2,
}
def test_get_one(self):
models.BugAssociation.objects.create(bug_id="999999", signature="OOM | small")
api = models.Bugs()
resp = api.get(signatures=["OOM | small"])
assert resp == {
"hits": [{"id": 999999, "signature": "OOM | small"}],
"total": 1,
}
def signature_bugzilla(request, params):
'''Return a list of associated bugs. '''
context = {}
signature = params['signature'][0]
context['signature'] = signature
bugs_api = models.Bugs()
context['bugs'] = bugs_api.get(signatures=[signature])['hits']
context['bugs'].sort(key=lambda x: x['id'], reverse=True)
return render(request, 'signature/signature_bugzilla.html', context)
def test_get_one(self):
models.BugAssociation.objects.create(bug_id='999999',
signature='OOM | small')
api = models.Bugs()
resp = api.get(signatures=['OOM | small'])
assert resp == {
'hits': [{
'id': 999999,
'signature': 'OOM | small'
}],
'total': 1
}
def topcrashers(request, days=None, possible_days=None, default_context=None):
context = default_context or {}
product = request.GET.get('product')
versions = request.GET.getlist('version')
crash_type = request.GET.get('process_type')
os_name = request.GET.get('platform')
result_count = request.GET.get('_facets_size')
tcbs_mode = request.GET.get('_tcbs_mode')
range_type = request.GET.get('_range_type')
range_type = 'build' if range_type == 'build' else 'report'
if not tcbs_mode or tcbs_mode not in ('realtime', 'byday'):
tcbs_mode = 'realtime'
if product not in context['active_versions']:
raise http.Http404('Unrecognized product')
context['product'] = product
if not versions:
# :(
# simulate what the nav.js does which is to take the latest version
# for this product.
for pv in context['active_versions'][product]:
if pv['is_featured']:
url = '%s&version=%s' % (request.build_absolute_uri(),
urlquote(pv['version']))
return redirect(url)
# See if all versions support builds. If not, refuse to show the "by build"
# range option in the UI.
versions_have_builds = True
for version in versions:
for pv in context['active_versions'][product]:
if pv['version'] == version and not pv['has_builds']:
versions_have_builds = False
break
context['versions_have_builds'] = versions_have_builds
# Used to pick a version in the dropdown menu.
context['version'] = versions[0]
if tcbs_mode == 'realtime':
end_date = timezone.now().replace(microsecond=0)
elif tcbs_mode == 'byday':
end_date = timezone.now().replace(hour=0,
minute=0,
second=0,
microsecond=0)
# settings.PROCESS_TYPES might contain tuple to indicate that some
# are actual labels.
process_types = []
for option in settings.PROCESS_TYPES:
if isinstance(option, (list, tuple)):
process_types.append(option[0])
else:
process_types.append(option)
if crash_type not in process_types:
crash_type = 'browser'
context['crash_type'] = crash_type
os_api = models.Platforms()
operating_systems = os_api.get()
if os_name not in (os_['name'] for os_ in operating_systems):
os_name = None
context['os_name'] = os_name
# set the result counts filter in the context to use in
# the template. This way we avoid hardcoding it twice and
# have it defined in one common location.
context['result_counts'] = settings.TCBS_RESULT_COUNTS
if result_count not in context['result_counts']:
result_count = settings.TCBS_RESULT_COUNTS[0]
context['result_count'] = result_count
context['query'] = {
'product': product,
'versions': versions,
'crash_type': crash_type,
'os_name': os_name,
'result_count': unicode(result_count),
'mode': tcbs_mode,
'range_type': range_type,
'end_date': end_date,
'start_date': end_date - datetime.timedelta(days=days),
}
api_results = get_topcrashers_results(
product=product,
version=versions,
platform=os_name,
process_type=crash_type,
date=[
'<' + end_date.isoformat(),
'>=' + context['query']['start_date'].isoformat()
],
_facets_size=result_count,
_range_type=range_type,
)
if api_results['total'] > 0:
tcbs = api_results['facets']['signature']
else:
tcbs = []
count_of_included_crashes = 0
signatures = []
for crash in tcbs[:int(result_count)]:
signatures.append(crash['signature'])
count_of_included_crashes += crash['count']
context['number_of_crashes'] = count_of_included_crashes
context['total_percentage'] = api_results['total'] and (
100.0 * count_of_included_crashes / api_results['total'])
# Get augmented bugs data.
bugs = defaultdict(list)
if signatures:
bugs_api = models.Bugs()
for b in bugs_api.get(signatures=signatures)['hits']:
bugs[b['signature']].append(b['id'])
# Get augmented signature data.
sig_date_data = {}
if signatures:
sig_api = models.SignatureFirstDate()
# SignatureFirstDate().get_dates() is an optimized version
# of SignatureFirstDate().get() that returns a dict of
# signature --> dates.
first_dates = sig_api.get_dates(signatures)
for sig, dates in first_dates.items():
sig_date_data[sig] = dates['first_date']
for crash in tcbs:
crash_counts = []
# Due to the inconsistencies of OS usage and naming of
# codes and props for operating systems the hacky bit below
# is required. Socorro and the world will be a better place
# once https://bugzilla.mozilla.org/show_bug.cgi?id=790642 lands.
for operating_system in operating_systems:
if operating_system['name'] == 'Unknown':
# not applicable in this context
continue
os_code = operating_system['code'][0:3].lower()
key = '%s_count' % os_code
crash_counts.append([crash[key], operating_system['name']])
crash['correlation_os'] = max(crash_counts)[1]
sig = crash['signature']
# Augment with bugs.
if sig in bugs:
if 'bugs' in crash:
crash['bugs'].extend(bugs[sig])
else:
crash['bugs'] = bugs[sig]
# Augment with first appearance dates.
if sig in sig_date_data:
crash['first_report'] = sig_date_data[sig]
if 'bugs' in crash:
crash['bugs'].sort(reverse=True)
context['tcbs'] = tcbs
context['days'] = days
context['report'] = 'topcrasher'
context['possible_days'] = possible_days
context['total_crashing_signatures'] = len(signatures)
context['total_number_of_crashes'] = api_results['total']
context['process_type_values'] = []
for option in settings.PROCESS_TYPES:
if option == 'all':
continue
if isinstance(option, (list, tuple)):
value, label = option
else:
value = option
label = option.capitalize()
context['process_type_values'].append((value, label))
context['platform_values'] = settings.DISPLAY_OS_NAMES
return render(request, 'topcrashers/topcrashers.html', context)
def topcrashers(request, days=None, possible_days=None, default_context=None):
context = default_context or {}
product = request.GET.get('product')
versions = request.GET.get('version')
crash_type = request.GET.get('process_type')
os_name = request.GET.get('platform')
result_count = request.GET.get('_facets_size')
tcbs_mode = request.GET.get('_tcbs_mode')
if not tcbs_mode or tcbs_mode not in ('realtime', 'byday'):
tcbs_mode = 'realtime'
if product not in context['releases']:
raise http.Http404('Unrecognized product')
context['product'] = product
if not versions:
# :(
# simulate what the nav.js does which is to take the latest version
# for this product.
for release in context['currentversions']:
if release['product'] == product and release['featured']:
url = '%s&version=%s' % (request.build_absolute_uri(),
urlquote(release['version']))
return redirect(url)
else:
versions = versions.split(';')
context['version'] = versions[0]
if tcbs_mode == 'realtime':
end_date = datetime.datetime.utcnow().replace(microsecond=0)
elif tcbs_mode == 'byday':
end_date = datetime.datetime.utcnow().replace(hour=0,
minute=0,
second=0,
microsecond=0)
if crash_type not in settings.PROCESS_TYPES:
crash_type = 'browser'
context['crash_type'] = crash_type
os_api = models.Platforms()
operating_systems = os_api.get()
if os_name not in (os_['name'] for os_ in operating_systems):
os_name = None
context['os_name'] = os_name
# set the result counts filter in the context to use in
# the template. This way we avoid hardcoding it twice and
# have it defined in one common location.
context['result_counts'] = settings.TCBS_RESULT_COUNTS
if result_count not in context['result_counts']:
result_count = settings.TCBS_RESULT_COUNTS[0]
context['result_count'] = result_count
context['query'] = {
'product': product,
'versions': versions[0],
'crash_type': crash_type,
'os_name': os_name,
'result_count': unicode(result_count),
'mode': tcbs_mode,
'end_date': end_date,
'start_date': end_date - datetime.timedelta(days=days),
}
api_results = get_topcrashers_results(
product=product,
version=context['version'],
platform=os_name,
process_type=crash_type,
date=[
'<' + end_date.isoformat(),
'>=' + context['query']['start_date'].isoformat()
],
_facets_size=result_count,
)
if api_results['total'] > 0:
tcbs = api_results['facets']['signature']
else:
tcbs = []
count_of_included_crashes = 0
signatures = []
for crash in tcbs[:int(result_count)]:
signatures.append(crash['signature'])
count_of_included_crashes += crash['count']
context['number_of_crashes'] = count_of_included_crashes
context['total_percentage'] = api_results['total'] and (
100.0 * count_of_included_crashes / api_results['total'])
# Get augmented bugs data.
bugs = defaultdict(list)
if signatures:
bugs_api = models.Bugs()
for b in bugs_api.get(signatures=signatures)['hits']:
bugs[b['signature']].append(b['id'])
# Get augmented signature data.
sig_date_data = {}
if signatures:
sig_api = models.SignatureFirstDate()
first_dates = sig_api.get(signatures=signatures)
for sig in first_dates['hits']:
sig_date_data[sig['signature']] = sig['first_date']
for crash in tcbs:
crash_counts = []
# Due to the inconsistencies of OS usage and naming of
# codes and props for operating systems the hacky bit below
# is required. Socorro and the world will be a better place
# once https://bugzilla.mozilla.org/show_bug.cgi?id=790642 lands.
for operating_system in operating_systems:
if operating_system['name'] == 'Unknown':
# not applicable in this context
continue
os_code = operating_system['code'][0:3].lower()
key = '%s_count' % os_code
crash_counts.append([crash[key], operating_system['name']])
crash['correlation_os'] = max(crash_counts)[1]
sig = crash['signature']
# Augment with bugs.
if sig in bugs:
if 'bugs' in crash:
crash['bugs'].extend(bugs[sig])
else:
crash['bugs'] = bugs[sig]
# Augment with first appearance dates.
if sig in sig_date_data:
crash['first_report'] = isodate.parse_datetime(sig_date_data[sig])
if 'bugs' in crash:
crash['bugs'].sort(reverse=True)
context['tcbs'] = tcbs
context['days'] = days
context['report'] = 'topcrasher'
context['possible_days'] = possible_days
context['total_crashing_signatures'] = len(signatures)
context['total_number_of_crashes'] = api_results['total']
context['process_type_values'] = (x for x in settings.PROCESS_TYPES
if x != 'all')
context['platform_values'] = settings.DISPLAY_OS_NAMES
return render(request, 'topcrashers/topcrashers.html', context)
def search_results(request):
'''Return the results of a search. '''
try:
params = get_params(request)
except ValidationError as e:
# There was an error in the form, let's return it.
return http.HttpResponseBadRequest(str(e))
context = {}
context['query'] = {'total': 0, 'total_count': 0, 'total_pages': 0}
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
context['params'] = current_query.copy()
if '_columns' in context['params']:
del context['params']['_columns']
if '_facets' in context['params']:
del context['params']['_facets']
context['sort'] = list(params['_sort'])
# Copy the list of columns so that they can differ.
context['columns'] = list(params['_columns'])
# The `uuid` field is a special case, it is always displayed in the first
# column of the table. Hence we do not want to show it again in the
# auto-generated list of columns, so we remove it from the list of
# columns to display.
if 'uuid' in context['columns']:
context['columns'].remove('uuid')
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
context['current_page'] = current_page
context['results_offset'] = results_per_page * (current_page - 1)
params['_results_number'] = results_per_page
params['_results_offset'] = context['results_offset']
context['current_url'] = '%s?%s' % (reverse('supersearch.search'),
urlencode_obj(current_query))
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as exception:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(exception))
if 'signature' in search_results['facets']:
# Bugs for each signature
signatures = [h['term'] for h in search_results['facets']['signature']]
if signatures:
bugs = defaultdict(list)
bugs_api = models.Bugs()
for b in bugs_api.get(signatures=signatures)['hits']:
bugs[b['signature']].append(b['id'])
for hit in search_results['facets']['signature']:
sig = hit['term']
if sig in bugs:
if 'bugs' in hit:
hit['bugs'].extend(bugs[sig])
else:
hit['bugs'] = bugs[sig]
# most recent bugs first
hit['bugs'].sort(reverse=True)
search_results['total_pages'] = int(
math.ceil(search_results['total'] / float(results_per_page)))
search_results['total_count'] = search_results['total']
context['query'] = search_results
return render(request, 'supersearch/search_results.html', context)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError, e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
if 'signature' in search_results['facets']:
# Bugs for each signature
signatures = [h['term'] for h in search_results['facets']['signature']]
if signatures:
bugs = defaultdict(list)
bugs_api = models.Bugs()
for b in bugs_api.get(signatures=signatures)['hits']:
bugs[b['signature']].append(b['id'])
for hit in search_results['facets']['signature']:
sig = hit['term']
if sig in bugs:
if 'bugs' in hit:
hit['bugs'].extend(bugs[sig])
else:
hit['bugs'] = bugs[sig]
search_results['total_pages'] = int(
math.ceil(search_results['total'] / float(results_per_page)))
search_results['total_count'] = search_results['total']
def search_results(request):
products = models.ProductsVersions().get()
versions = models.CurrentVersions().get()
platforms = models.Platforms().get()
form = forms.SearchForm(products, versions, platforms,
request.user.is_authenticated(), request.GET)
if not form.is_valid():
return http.HttpResponseBadRequest(str(form.errors))
params = {}
for key in form.cleaned_data:
if hasattr(form.fields[key], 'prefixed_value'):
value = form.fields[key].prefixed_value
else:
value = form.cleaned_data[key]
params[key] = value
data = {}
data['query'] = {'total': 0, 'total_count': 0, 'total_pages': 0}
allowed_fields = ALL_POSSIBLE_FIELDS
if request.user.is_authenticated():
allowed_fields += ADMIN_RESTRICTED_FIELDS
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
data['params'] = current_query.copy()
if '_columns' in data['params']:
del data['params']['_columns']
if '_facets' in params:
del data['params']['_facets']
params['_facets'] = request.GET.getlist('_facets') or DEFAULT_FACETS
data['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS
# Make sure only allowed fields are used
params['_facets'] = [x for x in params['_facets'] if x in allowed_fields]
data['columns'] = [x for x in data['columns'] if x in allowed_fields]
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
data['current_page'] = current_page
data['results_offset'] = results_per_page * (current_page - 1)
params['_results_number'] = results_per_page
params['_results_offset'] = data['results_offset']
data['current_url'] = '%s?%s' % (reverse('supersearch.search'),
current_query.urlencode())
api = SuperSearch()
search_results = api.get(**params)
if 'signature' in search_results['facets']:
# Bugs for each signature
signatures = [h['term'] for h in search_results['facets']['signature']]
if signatures:
bugs = defaultdict(list)
bugs_api = models.Bugs()
for b in bugs_api.get(signatures=signatures)['hits']:
bugs[b['signature']].append(b['id'])
for hit in search_results['facets']['signature']:
sig = hit['term']
if sig in bugs:
if 'bugs' in hit:
hit['bugs'].extend(bugs[sig])
else:
hit['bugs'] = bugs[sig]
search_results['total_pages'] = int(
math.ceil(search_results['total'] / float(results_per_page)))
search_results['total_count'] = search_results['total']
data['query'] = search_results
data['report_list_query_string'] = urllib.urlencode(
utils.sanitize_dict(get_report_list_parameters(params)), True)
return render(request, 'supersearch/search_results.html', data)
def search_results(request):
'''Return the results of a search. '''
try:
params = get_params(request)
except ValidationError as e:
# There was an error in the form, let's return it.
return http.HttpResponseBadRequest(str(e))
data = {}
data['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
allowed_fields = get_allowed_fields(request.user)
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
data['params'] = current_query.copy()
if '_columns' in data['params']:
del data['params']['_columns']
if '_facets' in data['params']:
del data['params']['_facets']
data['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS
# Make sure only allowed fields are used
data['columns'] = [
x for x in data['columns'] if x in allowed_fields
]
# Copy the list of columns so that they can differ.
params['_columns'] = list(data['columns'])
# The uuid is always displayed in the UI so we need to make sure it is
# always returned by the model.
if 'uuid' not in params['_columns']:
params['_columns'].append('uuid')
# The `uuid` field is a special case, it is always displayed in the first
# column of the table. Hence we do not want to show it again in the
# auto-generated list of columns, so we its name from the list of
# columns to display.
if 'uuid' in data['columns']:
data['columns'].remove('uuid')
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
data['current_page'] = current_page
data['results_offset'] = results_per_page * (current_page - 1)
params['_results_number'] = results_per_page
params['_results_offset'] = data['results_offset']
data['current_url'] = '%s?%s' % (
reverse('supersearch.search'),
current_query.urlencode()
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
if 'signature' in search_results['facets']:
# Bugs for each signature
signatures = [h['term'] for h in search_results['facets']['signature']]
if signatures:
bugs = defaultdict(list)
bugs_api = models.Bugs()
for b in bugs_api.get(signatures=signatures)['hits']:
bugs[b['signature']].append(b['id'])
for hit in search_results['facets']['signature']:
sig = hit['term']
if sig in bugs:
if 'bugs' in hit:
hit['bugs'].extend(bugs[sig])
else:
hit['bugs'] = bugs[sig]
search_results['total_pages'] = int(math.ceil(
search_results['total'] / float(results_per_page)))
search_results['total_count'] = search_results['total']
data['query'] = search_results
data['report_list_query_string'] = urllib.urlencode(
utils.sanitize_dict(
get_report_list_parameters(params)
),
True
)
return render(request, 'supersearch/search_results.html', data)
def exploitability_report(request, default_context=None):
context = default_context or {}
if not request.GET.get('product'):
url = reverse('exploitability:report')
url += '?' + urllib.urlencode({'product': settings.DEFAULT_PRODUCT})
return redirect(url)
form = ExploitabilityReportForm(
request.GET,
active_versions=context['active_versions'],
)
if not form.is_valid():
return http.HttpResponseBadRequest(str(form.errors))
product = form.cleaned_data['product']
version = form.cleaned_data['version']
api = SuperSearchUnredacted()
params = {
'product': product,
'version': version,
'_results_number': 0,
# This aggregates on crashes that do NOT contain these
# key words. For example, if a crash has
# {'exploitability': 'error: unable to analyze dump'}
# then it won't get included.
'exploitability': ['!error', '!interesting'],
'_aggs.signature': 'exploitability',
'_facets_size': settings.EXPLOITABILITY_BATCH_SIZE,
}
results = api.get(**params)
base_signature_report_dict = {
'product': product,
}
if version:
base_signature_report_dict['version'] = version
crashes = []
categories = ('high', 'none', 'low', 'medium', 'null')
for signature_facet in results['facets']['signature']:
# this 'signature_facet' will look something like this:
#
# {
# 'count': 1234,
# 'term': 'My | Signature',
# 'facets': {
# 'exploitability': [
# {'count': 1, 'term': 'high'},
# {'count': 23, 'term': 'medium'},
# {'count': 11, 'term': 'other'},
#
# And we only want to include those where:
#
# low or medium or high are greater than 0
#
exploitability = signature_facet['facets']['exploitability']
if not any(x['count'] for x in exploitability
if x['term'] in ('high', 'medium', 'low')):
continue
crash = {
'bugs': [],
'signature':
signature_facet['term'],
'high_count':
0,
'medium_count':
0,
'low_count':
0,
'none_count':
0,
'url':
(reverse('signature:signature_report') + '?' + urllib.urlencode(
dict(base_signature_report_dict,
signature=signature_facet['term']))),
}
for cluster in exploitability:
if cluster['term'] in categories:
crash['{}_count'.format(cluster['term'])] = (cluster['count'])
crash['med_or_high'] = (crash.get('high_count', 0) +
crash.get('medium_count', 0))
crashes.append(crash)
# Sort by the 'med_or_high' key first (descending),
# and by the signature second (ascending).
crashes.sort(key=lambda x: (-x['med_or_high'], x['signature']))
# now, let's go back and fill in the bugs
signatures = [x['signature'] for x in crashes]
if signatures:
api = models.Bugs()
bugs = defaultdict(list)
for b in api.get(signatures=signatures)['hits']:
bugs[b['signature']].append(b['id'])
for crash in crashes:
crash['bugs'] = bugs.get(crash['signature'], [])
context['crashes'] = crashes
context['product'] = product
context['version'] = version
context['report'] = 'exploitable'
return render(request, 'exploitability/report.html', context)
