def signature_correlations(request, params):
'''Return a list of correlations combos, to be populated by AJAX calls. '''
signature = params['signature'][0]
context = {}
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_facets'] = []
params['_aggs.product.version'] = 'platform'
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
all_combos = []
for product in search_results['facets']['product']:
for version in product['facets']['version']:
for platform in version['facets']['platform']:
all_combos.append({
'product': product['term'],
'version': version['term'],
'platform': platform['term'],
'count': platform['count'],
})
all_combos = sorted(all_combos, key=lambda x: x['count'])
context['correlation_combos'] = (
all_combos[:settings.MAX_CORRELATION_COMBOS_PER_SIGNATURE])
return render(request, 'signature/signature_correlations.html', context)
def signature_comments(request, params):
'''Return a list of non-empty comments. '''
signature = params['signature'][0]
context = {}
context['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
context['params'] = current_query.copy()
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
context['current_page'] = current_page
context['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['user_comments'] = '!__null__'
params['_columns'] = ['uuid', 'user_comments', 'date', 'useragent_locale']
params['_results_number'] = results_per_page
params['_results_offset'] = context['results_offset']
params['_facets'] = [] # We don't need no facets.
context['current_url'] = '%s?%s' % (
reverse('signature:signature_report'),
urlencode_obj(current_query)
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
search_results['total_pages'] = int(
math.ceil(
search_results['total'] / float(results_per_page)
)
)
search_results['total_count'] = search_results['total']
context['query'] = search_results
return render(request, 'signature/signature_comments.html', context)
def signature_summary(request, params):
'''Return a list of specific aggregations. '''
context = {}
params['signature'] = '=' + params['signature'][0]
params['_aggs.signature'] = [
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
]
params['_results_number'] = 0
params['_facets'] = [
'platform_pretty_version',
'cpu_name',
'process_type',
'flash_version',
]
params['_histogram.uptime'] = ['product']
params['_histogram_interval.uptime'] = 60
params['_aggs.adapter_vendor_id'] = ['adapter_device_id']
params['_aggs.android_cpu_abi.android_manufacturer.android_model'] = [
'android_version'
]
params['_aggs.product.version'] = ['_cardinality.install_time']
# If the user has permissions, show exploitability.
all_fields = SuperSearchFields().get()
if has_permissions(request.user,
all_fields['exploitability']['permissions_needed']):
params['_histogram.date'] = ['exploitability']
api = SuperSearchUnredacted()
# Now make the actual request with all expected parameters.
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
facets = search_results['facets']
_transform_uptime_summary(facets)
_transform_graphics_summary(facets)
_transform_mobile_summary(facets)
_transform_exploitability_summary(facets)
context['query'] = search_results
context['product_version_total'] = search_results['total']
if 'signature' in facets and len(facets['signature']) > 0:
context['signature_stats'] = SignatureStats(
search_results['facets']['signature'][0], search_results['total'])
return render(request, 'signature/signature_summary.html', context)
def signature_comments(request, params):
'''Return a list of non-empty comments. '''
signature = params['signature'][0]
context = {}
context['query'] = {'total': 0, 'total_count': 0, 'total_pages': 0}
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
context['params'] = current_query.copy()
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
context['current_page'] = current_page
context['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['user_comments'] = '!__null__'
params['_columns'] = ['uuid', 'user_comments', 'date', 'useragent_locale']
params['_sort'] = '-date'
params['_results_number'] = results_per_page
params['_results_offset'] = context['results_offset']
params['_facets'] = [] # We don't need no facets.
context['current_url'] = '%s?%s' % (reverse('signature:signature_report'),
urlencode_obj(current_query))
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
search_results['total_pages'] = int(
math.ceil(search_results['total'] / float(results_per_page)))
search_results['total_count'] = search_results['total']
context['query'] = search_results
return render(request, 'signature/signature_comments.html', context)
def signature_summary(request, params):
'''Return a list of specific aggregations. '''
context = {}
params['signature'] = '=' + params['signature'][0]
params['_results_number'] = 0
params['_facets'] = [
'platform_pretty_version',
'cpu_name',
'process_type',
'flash_version',
]
params['_histogram.uptime'] = ['product']
params['_histogram_interval.uptime'] = 60
params['_aggs.adapter_vendor_id'] = ['adapter_device_id']
params['_aggs.android_cpu_abi.android_manufacturer.android_model'] = [
'android_version'
]
params['_aggs.product.version'] = ['_cardinality.install_time']
# If the user has permissions, show exploitability.
all_fields = SuperSearchFields().get()
if has_permissions(
request.user, all_fields['exploitability']['permissions_needed']
):
params['_histogram.date'] = ['exploitability']
api = SuperSearchUnredacted()
# Now make the actual request with all expected parameters.
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
facets = search_results['facets']
_transform_uptime_summary(facets)
_transform_graphics_summary(facets)
_transform_mobile_summary(facets)
_transform_exploitability_summary(facets)
context['query'] = search_results
context['product_version_total'] = search_results['total']
return render(request, 'signature/signature_summary.html', context)
def signature_aggregation(request, params, aggregation):
'''Return the aggregation of a field. '''
signature = params['signature'][0]
context = {}
context['aggregation'] = aggregation
allowed_fields = get_allowed_fields(request.user)
# Make sure the field we want to aggregate on is allowed.
if aggregation not in allowed_fields:
return http.HttpResponseBadRequest(
'<ul><li>'
'You are not allowed to aggregate on the "%s" field'
'</li></ul>' % aggregation
)
current_query = request.GET.copy()
context['params'] = current_query.copy()
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_results_offset'] = 0
params['_facets'] = [aggregation]
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
context['aggregates'] = []
if aggregation in search_results['facets']:
context['aggregates'] = search_results['facets'][aggregation]
context['total_count'] = search_results['total']
return render(request, 'signature/signature_aggregation.html', context)
def signature_graphs(request, params, field):
'''Return a multi-line graph of crashes per day grouped by field. '''
signature = params['signature'][0]
context = {}
context['aggregation'] = field
allowed_fields = get_allowed_fields(request.user)
# Make sure the field we want to aggregate on is allowed.
if field not in allowed_fields:
return http.HttpResponseBadRequest(
'<ul><li>'
'You are not allowed to group by the "%s" field'
'</li></ul>' % field
)
current_query = request.GET.copy()
context['params'] = current_query.copy()
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_results_offset'] = 0
params['_histogram.date'] = [field]
params['_facets'] = [field]
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
context['aggregates'] = search_results['facets'].get('histogram_date', [])
context['term_counts'] = search_results['facets'].get(field, [])
return context
def signature_correlations(request, params):
'''Return a list of correlations combos, to be populated by AJAX calls. '''
signature = params['signature'][0]
context = {}
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_facets'] = []
params['_aggs.product.version'] = 'platform'
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
all_combos = []
for product in search_results['facets']['product']:
for version in product['facets']['version']:
for platform in version['facets']['platform']:
all_combos.append({
'product': product['term'],
'version': version['term'],
'platform': platform['term'],
'count': platform['count'],
})
all_combos = sorted(all_combos, key=lambda x: x['count'])
context['correlation_combos'] = (
all_combos[:settings.MAX_CORRELATION_COMBOS_PER_SIGNATURE]
)
return render(request, 'signature/signature_correlations.html', context)
def test_escaped(self):
html = render_exception('<hi>')
eq_(html, '<ul><li><hi></li></ul>')
def signature_reports(request, params):
'''Return the results of a search. '''
signature = params['signature'][0]
context = {}
context['query'] = {'total': 0, 'total_count': 0, 'total_pages': 0}
allowed_fields = get_allowed_fields(request.user)
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
context['params'] = current_query.copy()
if '_sort' in context['params']:
del context['params']['_sort']
if '_columns' in context['params']:
del context['params']['_columns']
context['sort'] = request.GET.getlist('_sort')
context['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS
# Make sure only allowed fields are used.
context['sort'] = [
x for x in context['sort'] if x in allowed_fields or (
x.startswith('-') and x[1:] in allowed_fields)
]
context['columns'] = [x for x in context['columns'] if x in allowed_fields]
params['_sort'] = context['sort']
# Copy the list of columns so that they can differ.
params['_columns'] = list(context['columns'])
# The uuid is always displayed in the UI so we need to make sure it is
# always returned by the model.
if 'uuid' not in params['_columns']:
params['_columns'].append('uuid')
# We require the cpu_info field to show a special marker on some AMD CPU
# related crash reports.
if 'cpu_info' not in params['_columns']:
params['_columns'].append('cpu_info')
# The `uuid` field is a special case, it is always displayed in the first
# column of the table. Hence we do not want to show it again in the
# auto-generated list of columns, so we its name from the list of
# columns to display.
if 'uuid' in context['columns']:
context['columns'].remove('uuid')
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
context['current_page'] = current_page
context['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['_results_number'] = results_per_page
params['_results_offset'] = context['results_offset']
params['_facets'] = [] # We don't need no facets.
context['current_url'] = '%s?%s' % (reverse('signature:signature_report'),
urlencode_obj(current_query))
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
search_results['total_pages'] = int(
math.ceil(search_results['total'] / float(results_per_page)))
search_results['total_count'] = search_results['total']
context['query'] = search_results
return render(request, 'signature/signature_reports.html', context)
def search_results(request):
'''Return the results of a search. '''
try:
params = get_params(request)
except ValidationError as e:
# There was an error in the form, let's return it.
return http.HttpResponseBadRequest(str(e))
context = {}
context['query'] = {'total': 0, 'total_count': 0, 'total_pages': 0}
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
context['params'] = current_query.copy()
if '_columns' in context['params']:
del context['params']['_columns']
if '_facets' in context['params']:
del context['params']['_facets']
context['sort'] = list(params['_sort'])
# Copy the list of columns so that they can differ.
context['columns'] = list(params['_columns'])
# The `uuid` field is a special case, it is always displayed in the first
# column of the table. Hence we do not want to show it again in the
# auto-generated list of columns, so we remove it from the list of
# columns to display.
if 'uuid' in context['columns']:
context['columns'].remove('uuid')
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
context['current_page'] = current_page
context['results_offset'] = results_per_page * (current_page - 1)
params['_results_number'] = results_per_page
params['_results_offset'] = context['results_offset']
context['current_url'] = '%s?%s' % (reverse('supersearch.search'),
urlencode_obj(current_query))
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as exception:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(exception))
if 'signature' in search_results['facets']:
# Bugs for each signature
signatures = [h['term'] for h in search_results['facets']['signature']]
if signatures:
bugs = defaultdict(list)
bugs_api = models.Bugs()
for b in bugs_api.get(signatures=signatures)['hits']:
bugs[b['signature']].append(b['id'])
for hit in search_results['facets']['signature']:
sig = hit['term']
if sig in bugs:
if 'bugs' in hit:
hit['bugs'].extend(bugs[sig])
else:
hit['bugs'] = bugs[sig]
# most recent bugs first
hit['bugs'].sort(reverse=True)
search_results['total_pages'] = int(
math.ceil(search_results['total'] / float(results_per_page)))
search_results['total_count'] = search_results['total']
context['query'] = search_results
return render(request, 'supersearch/search_results.html', context)
def search_results(request):
'''Return the results of a search. '''
try:
params = get_params(request)
except ValidationError as e:
# There was an error in the form, let's return it.
return http.HttpResponseBadRequest(str(e))
context = {}
context['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
context['params'] = current_query.copy()
if '_columns' in context['params']:
del context['params']['_columns']
if '_facets' in context['params']:
del context['params']['_facets']
context['sort'] = list(params['_sort'])
# Copy the list of columns so that they can differ.
context['columns'] = list(params['_columns'])
# The `uuid` field is a special case, it is always displayed in the first
# column of the table. Hence we do not want to show it again in the
# auto-generated list of columns, so we remove it from the list of
# columns to display.
if 'uuid' in context['columns']:
context['columns'].remove('uuid')
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
context['current_page'] = current_page
context['results_offset'] = results_per_page * (current_page - 1)
params['_results_number'] = results_per_page
params['_results_offset'] = context['results_offset']
context['current_url'] = '%s?%s' % (
reverse('supersearch.search'),
urlencode_obj(current_query)
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as exception:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(
render_exception(exception)
)
if 'signature' in search_results['facets']:
# Bugs for each signature
signatures = [h['term'] for h in search_results['facets']['signature']]
if signatures:
bugs = defaultdict(list)
bugs_api = models.Bugs()
for b in bugs_api.get(signatures=signatures)['hits']:
bugs[b['signature']].append(b['id'])
for hit in search_results['facets']['signature']:
sig = hit['term']
if sig in bugs:
if 'bugs' in hit:
hit['bugs'].extend(bugs[sig])
else:
hit['bugs'] = bugs[sig]
# most recent bugs first
hit['bugs'].sort(reverse=True)
search_results['total_pages'] = int(math.ceil(
search_results['total'] / float(results_per_page)))
search_results['total_count'] = search_results['total']
context['query'] = search_results
return render(request, 'supersearch/search_results.html', context)
def test_basic(self):
html = render_exception('hi!')
assert html == '<ul><li>hi!</li></ul>'
def test_to_string(self):
try:
raise NameError('<hack>')
except NameError as exc:
html = render_exception(exc)
assert html == '<ul><li><hack></li></ul>'
def test_escaped(self):
html = render_exception('<hi>')
assert html == '<ul><li><hi></li></ul>'
def test_basic(self):
html = render_exception('hi!')
eq_(html, '<ul><li>hi!</li></ul>')
def signature_reports(request, params):
'''Return the results of a search. '''
signature = params['signature'][0]
context = {}
context['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
allowed_fields = get_allowed_fields(request.user)
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
context['params'] = current_query.copy()
if '_sort' in context['params']:
del context['params']['_sort']
if '_columns' in context['params']:
del context['params']['_columns']
context['sort'] = request.GET.getlist('_sort')
context['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS
# Make sure only allowed fields are used.
context['sort'] = [
x for x in context['sort']
if x in allowed_fields or
(x.startswith('-') and x[1:] in allowed_fields)
]
context['columns'] = [
x for x in context['columns'] if x in allowed_fields
]
params['_sort'] = context['sort']
# Copy the list of columns so that they can differ.
params['_columns'] = list(context['columns'])
# The uuid is always displayed in the UI so we need to make sure it is
# always returned by the model.
if 'uuid' not in params['_columns']:
params['_columns'].append('uuid')
# We require the cpu_info field to show a special marker on some AMD CPU
# related crash reports.
if 'cpu_info' not in params['_columns']:
params['_columns'].append('cpu_info')
# The `uuid` field is a special case, it is always displayed in the first
# column of the table. Hence we do not want to show it again in the
# auto-generated list of columns, so we its name from the list of
# columns to display.
if 'uuid' in context['columns']:
context['columns'].remove('uuid')
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
context['current_page'] = current_page
context['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['_results_number'] = results_per_page
params['_results_offset'] = context['results_offset']
params['_facets'] = [] # We don't need no facets.
context['current_url'] = '%s?%s' % (
reverse('signature:signature_report'),
urlencode_obj(current_query)
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
search_results['total_pages'] = int(
math.ceil(
search_results['total'] / float(results_per_page)
)
)
search_results['total_count'] = search_results['total']
context['query'] = search_results
return render(request, 'signature/signature_reports.html', context)
