def signature_comments(request, params):
'''Return a list of non-empty comments. '''
signature = params['signature'][0]
context = {}
context['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
context['params'] = current_query.copy()
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
context['current_page'] = current_page
context['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['user_comments'] = '!__null__'
params['_columns'] = ['uuid', 'user_comments', 'date', 'useragent_locale']
params['_results_number'] = results_per_page
params['_results_offset'] = context['results_offset']
params['_facets'] = [] # We don't need no facets.
context['current_url'] = '%s?%s' % (
reverse('signature:signature_report'),
urlencode_obj(current_query)
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
search_results['total_pages'] = int(
math.ceil(
search_results['total'] / float(results_per_page)
)
)
search_results['total_count'] = search_results['total']
context['query'] = search_results
return render(request, 'signature/signature_comments.html', context)
def signature_comments(request, params):
'''Return a list of non-empty comments. '''
signature = params['signature'][0]
data = {}
data['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
data['params'] = current_query.copy()
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
data['current_page'] = current_page
data['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['user_comments'] = '!__null__'
params['_columns'] = ['uuid', 'user_comments', 'date', 'useragent_locale']
params['_results_number'] = results_per_page
params['_results_offset'] = data['results_offset']
params['_facets'] = [] # We don't need no facets.
data['current_url'] = '%s?%s' % (
reverse('signature:signature_report'),
current_query.urlencode()
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
search_results['total_pages'] = int(
math.ceil(
search_results['total'] / float(results_per_page)
)
)
search_results['total_count'] = search_results['total']
data['query'] = search_results
return render(request, 'signature/signature_comments.html', data)
def signature_correlations(request, params):
'''Return a list of correlations combos, to be populated by AJAX calls. '''
signature = params['signature'][0]
context = {}
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_facets'] = []
params['_aggs.product.version'] = 'platform'
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
all_combos = []
for product in search_results['facets']['product']:
for version in product['facets']['version']:
for platform in version['facets']['platform']:
all_combos.append({
'product': product['term'],
'version': version['term'],
'platform': platform['term'],
'count': platform['count'],
})
all_combos = sorted(all_combos, key=lambda x: x['count'])
context['correlation_combos'] = (
all_combos[:settings.MAX_CORRELATION_COMBOS_PER_SIGNATURE])
return render(request, 'signature/signature_correlations.html', context)
def profile(request, default_context=None):
context = default_context or {}
context['permissions'] = (
Permission.objects.filter(content_type__model='')
.order_by('name')
)
start_date = (
datetime.datetime.utcnow() - datetime.timedelta(weeks=4)
).isoformat()
api = SuperSearchUnredacted()
results = api.get(
email=request.user.email,
date='>%s' % start_date,
_columns=['date', 'uuid'],
_sort='-date',
)
context['crashes_list'] = [
dict(zip(('crash_id', 'date'), (x['uuid'], x['date'])))
for x in results['hits']
]
return render(request, 'profile/profile.html', context)
def profile(request, default_context=None):
context = default_context or {}
context['permissions'] = (
Permission.objects.filter(content_type__model='')
.order_by('name')
)
start_date = (
datetime.datetime.utcnow() - datetime.timedelta(weeks=4)
).isoformat()
api = SuperSearchUnredacted()
results = api.get(
email=request.user.email,
date='>%s' % start_date,
_columns=['date', 'uuid'],
)
context['crashes_list'] = [
dict(zip(('crash_id', 'date'), (x['uuid'], x['date'])))
for x in results['hits']
]
context['your_tokens'] = (
models.Token.objects
.filter(user=request.user)
.order_by('-created')
)
return render(request, 'profile/profile.html', context)
def signature_summary(request, params):
"""Return a list of specific aggregations"""
context = {}
params["signature"] = "=" + params["signature"][0]
params["_aggs.signature"] = [
"hang_type",
"process_type",
"startup_crash",
"dom_fission_enabled",
"_histogram.uptime",
]
params["_results_number"] = 0
params["_facets"] = [
"platform_pretty_version",
"cpu_arch",
"process_type",
"flash_version",
]
params["_histogram.uptime"] = ["product"]
params["_histogram_interval.uptime"] = 60
params["_aggs.adapter_vendor_id"] = ["adapter_device_id"]
params["_aggs.android_cpu_abi.android_manufacturer.android_model"] = [
"android_version"
]
params["_aggs.product.version"] = ["_cardinality.install_time"]
# If the user has permissions, show exploitability.
all_fields = SuperSearchFields().get()
if request.user.has_perms(
all_fields["exploitability"]["permissions_needed"]):
params["_histogram.date"] = ["exploitability"]
api = SuperSearchUnredacted()
# Now make the actual request with all expected parameters.
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
facets = search_results["facets"]
_transform_uptime_summary(facets)
_transform_graphics_summary(facets)
_transform_mobile_summary(facets)
_transform_exploitability_summary(facets)
context["query"] = search_results
context["product_version_total"] = search_results["total"]
if "signature" in facets and len(facets["signature"]) > 0:
context["signature_stats"] = SignatureStats(
search_results["facets"]["signature"][0], search_results["total"])
return render(request, "signature/signature_summary.html", context)
def signature_comments(request, params):
"""Return a list of non-empty comments."""
# Users can't see comments unless they have view_pii permissions.
if not request.user.has_perm("crashstats.view_pii"):
return http.HttpResponseForbidden()
signature = params["signature"][0]
context = {}
context["query"] = {"total": 0, "total_count": 0, "total_pages": 0}
current_query = request.GET.copy()
if "page" in current_query:
del current_query["page"]
context["params"] = current_query.copy()
try:
current_page = int(request.GET.get("page", 1))
except ValueError:
return http.HttpResponseBadRequest("Invalid page")
if current_page <= 0:
current_page = 1
results_per_page = 50
context["current_page"] = current_page
context["results_offset"] = results_per_page * (current_page - 1)
params["signature"] = "=" + signature
params["user_comments"] = "!__null__"
params["_columns"] = ["uuid", "user_comments", "date", "useragent_locale"]
params["_sort"] = "-date"
params["_results_number"] = results_per_page
params["_results_offset"] = context["results_offset"]
params["_facets"] = []
context["current_url"] = "%s?%s" % (
reverse("signature:signature_report"),
urlencode_obj(current_query),
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
search_results["total_pages"] = int(
math.ceil(search_results["total"] / float(results_per_page)))
search_results["total_count"] = search_results["total"]
context["query"] = search_results
return render(request, "signature/signature_comments.html", context)
def signature_summary(request, params):
"""Return a list of specific aggregations"""
context = {}
params['signature'] = '=' + params['signature'][0]
params['_aggs.signature'] = [
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
]
params['_results_number'] = 0
params['_facets'] = [
'platform_pretty_version',
'cpu_arch',
'process_type',
'flash_version',
]
params['_histogram.uptime'] = ['product']
params['_histogram_interval.uptime'] = 60
params['_aggs.adapter_vendor_id'] = ['adapter_device_id']
params['_aggs.android_cpu_abi.android_manufacturer.android_model'] = [
'android_version'
]
params['_aggs.product.version'] = ['_cardinality.install_time']
# If the user has permissions, show exploitability.
all_fields = SuperSearchFields().get()
if has_permissions(
request.user, all_fields['exploitability']['permissions_needed']
):
params['_histogram.date'] = ['exploitability']
api = SuperSearchUnredacted()
# Now make the actual request with all expected parameters.
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
facets = search_results['facets']
_transform_uptime_summary(facets)
_transform_graphics_summary(facets)
_transform_mobile_summary(facets)
_transform_exploitability_summary(facets)
context['query'] = search_results
context['product_version_total'] = search_results['total']
if 'signature' in facets and len(facets['signature']) > 0:
context['signature_stats'] = SignatureStats(search_results['facets']['signature'][0],
search_results['total'])
return render(request, 'signature/signature_summary.html', context)
def signature_summary(request, params):
'''Return a list of specific aggregations. '''
context = {}
params['signature'] = '=' + params['signature'][0]
params['_aggs.signature'] = [
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
]
params['_results_number'] = 0
params['_facets'] = [
'platform_pretty_version',
'cpu_name',
'process_type',
'flash_version',
]
params['_histogram.uptime'] = ['product']
params['_histogram_interval.uptime'] = 60
params['_aggs.adapter_vendor_id'] = ['adapter_device_id']
params['_aggs.android_cpu_abi.android_manufacturer.android_model'] = [
'android_version'
]
params['_aggs.product.version'] = ['_cardinality.install_time']
# If the user has permissions, show exploitability.
all_fields = SuperSearchFields().get()
if has_permissions(request.user,
all_fields['exploitability']['permissions_needed']):
params['_histogram.date'] = ['exploitability']
api = SuperSearchUnredacted()
# Now make the actual request with all expected parameters.
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
facets = search_results['facets']
_transform_uptime_summary(facets)
_transform_graphics_summary(facets)
_transform_mobile_summary(facets)
_transform_exploitability_summary(facets)
context['query'] = search_results
context['product_version_total'] = search_results['total']
if 'signature' in facets and len(facets['signature']) > 0:
context['signature_stats'] = SignatureStats(
search_results['facets']['signature'][0], search_results['total'])
return render(request, 'signature/signature_summary.html', context)
def signature_reports(request):
'''Return the results of a search. '''
params = get_validated_params(request)
if isinstance(params, http.HttpResponseBadRequest):
# There was an error in the form, let's return it.
return params
signature = params['signature'][0]
data = {}
data['query'] = {'total': 0, 'total_count': 0, 'total_pages': 0}
allowed_fields = get_allowed_fields(request.user)
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
data['params'] = current_query.copy()
if '_columns' in data['params']:
del data['params']['_columns']
data['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS
# Make sure only allowed fields are used
data['columns'] = [x for x in data['columns'] if x in allowed_fields]
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
data['current_page'] = current_page
data['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['_results_number'] = results_per_page
params['_results_offset'] = data['results_offset']
params['_facets'] = [] # We don't need no facets.
data['current_url'] = '%s?%s' % (reverse('signature:signature_report'),
current_query.urlencode())
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError, e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
def signature_comments(request, params):
"""Return a list of non-empty comments."""
# Users can't see comments unless they have view_pii permissions.
if not request.user.has_perm('crashstats.view_pii'):
return http.HttpResponseForbidden()
signature = params['signature'][0]
context = {}
context['query'] = {'total': 0, 'total_count': 0, 'total_pages': 0}
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
context['params'] = current_query.copy()
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
context['current_page'] = current_page
context['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['user_comments'] = '!__null__'
params['_columns'] = ['uuid', 'user_comments', 'date', 'useragent_locale']
params['_sort'] = '-date'
params['_results_number'] = results_per_page
params['_results_offset'] = context['results_offset']
params['_facets'] = []
context['current_url'] = '%s?%s' % (reverse('signature:signature_report'),
urlencode_obj(current_query))
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
search_results['total_pages'] = int(
math.ceil(search_results['total'] / float(results_per_page)))
search_results['total_count'] = search_results['total']
context['query'] = search_results
return render(request, 'signature/signature_comments.html', context)
def signature_comments(request):
'''Return a list of non-empty comments. '''
params = get_validated_params(request)
if isinstance(params, http.HttpResponseBadRequest):
# There was an error in the form, let's return it.
return params
signature = params['signature'][0]
data = {}
data['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
data['params'] = current_query.copy()
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
data['current_page'] = current_page
data['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['user_comments'] = '!__null__'
params['_columns'] = ['uuid', 'user_comments', 'date', 'useragent_locale']
params['_results_number'] = results_per_page
params['_results_offset'] = data['results_offset']
params['_facets'] = [] # We don't need no facets.
data['current_url'] = '%s?%s' % (
reverse('signature:signature_report'),
current_query.urlencode()
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError, e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
def profile(request, default_context=None):
context = default_context or {}
context["permissions"] = Permission.objects.filter(content_type__model="").order_by("name")
start_date = (datetime.datetime.utcnow() - datetime.timedelta(weeks=4)).isoformat()
api = SuperSearchUnredacted()
results = api.get(email=request.user.email, date=">%s" % start_date, _columns=["date", "uuid"], _sort="-date")
context["crashes_list"] = [dict(zip(("crash_id", "date"), (x["uuid"], x["date"]))) for x in results["hits"]]
return render(request, "profile/profile.html", context)
def signature_aggregation(request, aggregation):
'''Return the aggregation of a field. '''
params = get_params(request)
if isinstance(params, http.HttpResponseBadRequest):
# There was an error in the form, let's return it.
return params
if len(params['signature']) > 1:
return http.HttpResponseBadRequest(
'Invalid value for "signature" parameter, '
'only one value is accepted'
)
signature = params['signature'][0]
if not signature:
return http.HttpResponseBadRequest(
'"signature" parameter is mandatory'
)
data = {}
data['aggregation'] = aggregation
allowed_fields = get_allowed_fields(request.user)
# Make sure the field we want to aggregate on is allowed.
if aggregation not in allowed_fields:
return http.HttpResponseBadRequest(
'<ul><li>'
'You are not allowed to aggregate on the "%s" field'
'</li></ul>' % aggregation
)
current_query = request.GET.copy()
data['params'] = current_query.copy()
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_results_offset'] = 0
params['_facets'] = [aggregation]
data['current_url'] = '%s?%s' % (
reverse('signature:signature_report'),
current_query.urlencode()
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError, e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
def signature_aggregation(request, params, aggregation):
'''Return the aggregation of a field. '''
signature = params['signature'][0]
context = {}
context['aggregation'] = aggregation
allowed_fields = get_allowed_fields(request.user)
# Make sure the field we want to aggregate on is allowed.
if aggregation not in allowed_fields:
return http.HttpResponseBadRequest(
'<ul><li>'
'You are not allowed to aggregate on the "%s" field'
'</li></ul>' % aggregation
)
current_query = request.GET.copy()
context['params'] = current_query.copy()
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_results_offset'] = 0
params['_facets'] = [aggregation]
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
context['aggregates'] = []
if aggregation in search_results['facets']:
context['aggregates'] = search_results['facets'][aggregation]
context['total_count'] = search_results['total']
return render(request, 'signature/signature_aggregation.html', context)
def profile(request, default_context=None):
context = default_context or {}
context["permissions"] = Permission.objects.filter(content_type__model="").order_by(
"name"
)
start_date = (datetime.datetime.utcnow() - datetime.timedelta(weeks=4)).isoformat()
api = SuperSearchUnredacted()
results = api.get(
email=request.user.email,
date=">%s" % start_date,
_columns=["date", "uuid"],
_sort="-date",
)
context["crashes_list"] = [
dict(zip(("crash_id", "date"), (x["uuid"], x["date"]))) for x in results["hits"]
]
return render(request, "profile/profile.html", context)
def signature_aggregation(request, params, aggregation):
"""Return the aggregation of a field. """
signature = params["signature"][0]
context = {}
context["aggregation"] = aggregation
allowed_fields = get_allowed_fields(request.user)
# Make sure the field we want to aggregate on is allowed.
if aggregation not in allowed_fields:
return http.HttpResponseBadRequest(
"<ul><li>"
'You are not allowed to aggregate on the "%s" field'
"</li></ul>" % aggregation)
current_query = request.GET.copy()
context["params"] = current_query.copy()
params["signature"] = "=" + signature
params["_results_number"] = 0
params["_results_offset"] = 0
params["_facets"] = [aggregation]
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
context["aggregates"] = []
if aggregation in search_results["facets"]:
context["aggregates"] = search_results["facets"][aggregation]
context["total_count"] = search_results["total"]
return render(request, "signature/signature_aggregation.html", context)
def signature_graphs(request, params, field):
'''Return a multi-line graph of crashes per day grouped by field. '''
signature = params['signature'][0]
context = {}
context['aggregation'] = field
allowed_fields = get_allowed_fields(request.user)
# Make sure the field we want to aggregate on is allowed.
if field not in allowed_fields:
return http.HttpResponseBadRequest(
'<ul><li>'
'You are not allowed to group by the "%s" field'
'</li></ul>' % field
)
current_query = request.GET.copy()
context['params'] = current_query.copy()
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_results_offset'] = 0
params['_histogram.date'] = [field]
params['_facets'] = [field]
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
context['aggregates'] = search_results['facets'].get('histogram_date', [])
context['term_counts'] = search_results['facets'].get(field, [])
return context
def signature_graphs(request, field):
'''Return a multi-line graph of crashes per day grouped by field. '''
params = get_validated_params(request)
if isinstance(params, http.HttpResponseBadRequest):
# There was an error in the form, let's return it.
return params
signature = params['signature'][0]
data = {}
data['aggregation'] = field
allowed_fields = get_allowed_fields(request.user)
# Make sure the field we want to aggregate on is allowed.
if field not in allowed_fields:
return http.HttpResponseBadRequest(
'<ul><li>'
'You are not allowed to group by the "%s" field'
'</li></ul>' % field
)
current_query = request.GET.copy()
data['params'] = current_query.copy()
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_results_offset'] = 0
params['_histogram.date'] = [field]
params['_facets'] = [field]
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError, e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
def search_custom(request, default_context=None):
"""Return the basic search page, without any result"""
error = None
query = None
try:
params = get_params(request)
except ValidationError as e:
# There was an error in the form, but we want to do the default
# behavior and just display an error message.
error = str(e)
else:
# Get the JSON query that supersearch generates and show it.
params['_return_query'] = 'true'
api = SuperSearchUnredacted()
try:
query = api.get(**params)
except BadArgumentError as e:
error = e
schema = settings.ELASTICSEARCH_INDEX_SCHEMA
now = timezone.now()
possible_indices = []
for i in range(26):
index = (now - datetime.timedelta(weeks=i)).strftime(schema)
possible_indices.append({'id': index, 'text': index})
context = default_context
context['elasticsearch_indices'] = possible_indices
if query:
context['query'] = json.dumps(query['query'])
context['indices'] = ','.join(query['indices'])
context['error'] = error
return render(request, 'supersearch/search_custom.html', context)
def signature_graphs(request, params, field):
"""Return a multi-line graph of crashes per day grouped by field. """
signature = params["signature"][0]
context = {}
context["aggregation"] = field
allowed_fields = get_allowed_fields(request.user)
# Make sure the field we want to aggregate on is allowed.
if field not in allowed_fields:
return http.HttpResponseBadRequest(
"<ul><li>"
'You are not allowed to group by the "%s" field'
"</li></ul>" % field)
current_query = request.GET.copy()
context["params"] = current_query.copy()
params["signature"] = "=" + signature
params["_results_number"] = 0
params["_results_offset"] = 0
params["_histogram.date"] = [field]
params["_facets"] = [field]
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
context["aggregates"] = search_results["facets"].get("histogram_date", [])
context["term_counts"] = search_results["facets"].get(field, [])
return context
def search_custom(request, default_context=None):
"""Return the basic search page, without any result"""
error = None
query = None
try:
params = get_params(request)
except ValidationError as e:
# There was an error in the form, but we want to do the default
# behavior and just display an error message.
error = str(e)
else:
# Get the JSON query that supersearch generates and show it.
params["_return_query"] = "true"
api = SuperSearchUnredacted()
try:
query = api.get(**params)
except BadArgumentError as e:
error = e
schema = settings.ELASTICSEARCH_INDEX_SCHEMA
now = timezone.now()
possible_indices = []
for i in range(26):
index = (now - datetime.timedelta(weeks=i)).strftime(schema)
possible_indices.append({"id": index, "text": index})
context = default_context
context["elasticsearch_indices"] = possible_indices
if query:
context["query"] = json.dumps(query["query"])
context["indices"] = ",".join(query["indices"])
context["error"] = error
return render(request, "supersearch/search_custom.html", context)
def signature_graphs(request, field):
'''Return a multi-line graph of crashes per day grouped by field. '''
params = get_validated_params(request)
if isinstance(params, http.HttpResponseBadRequest):
# There was an error in the form, let's return it.
return params
signature = params['signature'][0]
data = {}
data['aggregation'] = field
allowed_fields = get_allowed_fields(request.user)
# Make sure the field we want to aggregate on is allowed.
if field not in allowed_fields:
return http.HttpResponseBadRequest(
'<ul><li>'
'You are not allowed to group by the "%s" field'
'</li></ul>' % field
)
current_query = request.GET.copy()
data['params'] = current_query.copy()
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_results_offset'] = 0
params['_histogram.date'] = [field]
params['_facets'] = [field]
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError, e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
def signature_correlations(request, params):
'''Return a list of correlations combos, to be populated by AJAX calls. '''
signature = params['signature'][0]
context = {}
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_facets'] = []
params['_aggs.product.version'] = 'platform'
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
all_combos = []
for product in search_results['facets']['product']:
for version in product['facets']['version']:
for platform in version['facets']['platform']:
all_combos.append({
'product': product['term'],
'version': version['term'],
'platform': platform['term'],
'count': platform['count'],
})
all_combos = sorted(all_combos, key=lambda x: x['count'])
context['correlation_combos'] = (
all_combos[:settings.MAX_CORRELATION_COMBOS_PER_SIGNATURE]
)
return render(request, 'signature/signature_correlations.html', context)
def search_results(request):
"""Return the results of a search"""
try:
params = get_params(request)
except ValidationError as e:
# There was an error in the form, let's return it.
return http.HttpResponseBadRequest(str(e))
context = {}
context["query"] = {"total": 0, "total_count": 0, "total_pages": 0}
current_query = request.GET.copy()
if "page" in current_query:
del current_query["page"]
context["params"] = current_query.copy()
if "_columns" in context["params"]:
del context["params"]["_columns"]
if "_facets" in context["params"]:
del context["params"]["_facets"]
context["sort"] = list(params["_sort"])
# Copy the list of columns so that they can differ.
context["columns"] = list(params["_columns"])
# The `uuid` field is a special case, it is always displayed in the first
# column of the table. Hence we do not want to show it again in the
# auto-generated list of columns, so we remove it from the list of
# columns to display.
if "uuid" in context["columns"]:
context["columns"].remove("uuid")
try:
current_page = int(request.GET.get("page", 1))
except ValueError:
return http.HttpResponseBadRequest("Invalid page")
if current_page <= 0:
current_page = 1
results_per_page = 50
context["current_page"] = current_page
context["results_offset"] = results_per_page * (current_page - 1)
params["_results_number"] = results_per_page
params["_results_offset"] = context["results_offset"]
context["current_url"] = "%s?%s" % (
reverse("supersearch:search"),
urlencode_obj(current_query),
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as exception:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(exception))
if "signature" in search_results["facets"]:
# Bugs for each signature
signatures = [h["term"] for h in search_results["facets"]["signature"]]
if signatures:
bugs = defaultdict(list)
qs = models.BugAssociation.objects.filter(signature__in=signatures).values(
"bug_id", "signature"
)
for item in qs:
bugs[item["signature"]].append(item["bug_id"])
for hit in search_results["facets"]["signature"]:
sig = hit["term"]
if sig in bugs:
if "bugs" in hit:
hit["bugs"].extend(bugs[sig])
else:
hit["bugs"] = bugs[sig]
# most recent bugs first
hit["bugs"].sort(reverse=True)
search_results["total_pages"] = int(
math.ceil(search_results["total"] / float(results_per_page))
)
search_results["total_count"] = search_results["total"]
context["query"] = search_results
return render(request, "supersearch/search_results.html", context)
def get_topcrashers_stats(**kwargs):
"""Return the results of a search. """
params = kwargs
range_type = params.pop("_range_type")
dates = get_date_boundaries(params)
params["_aggs.signature"] = [
"platform",
"is_garbage_collecting",
"hang_type",
"process_type",
"startup_crash",
"_histogram.uptime",
"_cardinality.install_time",
]
params["_histogram_interval.uptime"] = 60
# We don't care about no results, only facets.
params["_results_number"] = 0
if params.get("process_type") in ("any", "all"):
params["process_type"] = None
if range_type == "build":
params["build_id"] = [
">=" + datetime_to_build_id(dates[0]),
"<" + datetime_to_build_id(dates[1]),
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
signatures_stats = []
if search_results["total"] > 0:
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params["date"] = [
">=" + (dates[1] - delta).isoformat(),
"<" + dates[0].isoformat(),
]
params["_aggs.signature"] = ["platform"]
params["_facets_size"] *= 2
if range_type == "build":
params["date"][1] = "<" + dates[1].isoformat()
params["build_id"] = [
">=" + datetime_to_build_id(dates[1] - delta),
"<" + datetime_to_build_id(dates[0]),
]
previous_range_results = api.get(**params)
previous_signatures = get_comparison_signatures(previous_range_results)
for index, signature in enumerate(
search_results["facets"]["signature"]):
previous_signature = previous_signatures.get(signature["term"])
signatures_stats.append(
SignatureStats(
signature=signature,
num_total_crashes=search_results["total"],
rank=index,
platforms=models.Platform.objects.values(),
previous_signature=previous_signature,
))
return signatures_stats
def search_results(request):
"""Return the results of a search"""
try:
params = get_params(request)
except ValidationError as e:
# There was an error in the form, let's return it.
return http.HttpResponseBadRequest(str(e))
context = {}
context['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
context['params'] = current_query.copy()
if '_columns' in context['params']:
del context['params']['_columns']
if '_facets' in context['params']:
del context['params']['_facets']
context['sort'] = list(params['_sort'])
# Copy the list of columns so that they can differ.
context['columns'] = list(params['_columns'])
# The `uuid` field is a special case, it is always displayed in the first
# column of the table. Hence we do not want to show it again in the
# auto-generated list of columns, so we remove it from the list of
# columns to display.
if 'uuid' in context['columns']:
context['columns'].remove('uuid')
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
context['current_page'] = current_page
context['results_offset'] = results_per_page * (current_page - 1)
params['_results_number'] = results_per_page
params['_results_offset'] = context['results_offset']
context['current_url'] = '%s?%s' % (
reverse('supersearch:search'),
urlencode_obj(current_query)
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as exception:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(
render_exception(exception)
)
if 'signature' in search_results['facets']:
# Bugs for each signature
signatures = [h['term'] for h in search_results['facets']['signature']]
if signatures:
bugs = defaultdict(list)
qs = (
models.BugAssociation.objects
.filter(signature__in=signatures)
.values('bug_id', 'signature')
)
for item in qs:
bugs[item['signature']].append(item['bug_id'])
for hit in search_results['facets']['signature']:
sig = hit['term']
if sig in bugs:
if 'bugs' in hit:
hit['bugs'].extend(bugs[sig])
else:
hit['bugs'] = bugs[sig]
# most recent bugs first
hit['bugs'].sort(reverse=True)
search_results['total_pages'] = int(math.ceil(
search_results['total'] / float(results_per_page)))
search_results['total_count'] = search_results['total']
context['query'] = search_results
return render(request, 'supersearch/search_results.html', context)
def signature_reports(request, params):
'''Return the results of a search. '''
signature = params['signature'][0]
context = {}
context['query'] = {'total': 0, 'total_count': 0, 'total_pages': 0}
allowed_fields = get_allowed_fields(request.user)
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
context['params'] = current_query.copy()
if '_sort' in context['params']:
del context['params']['_sort']
if '_columns' in context['params']:
del context['params']['_columns']
context['sort'] = request.GET.getlist('_sort')
context['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS
# Make sure only allowed fields are used.
context['sort'] = [
x for x in context['sort'] if x in allowed_fields or (
x.startswith('-') and x[1:] in allowed_fields)
]
context['columns'] = [x for x in context['columns'] if x in allowed_fields]
params['_sort'] = context['sort']
# Copy the list of columns so that they can differ.
params['_columns'] = list(context['columns'])
# The uuid is always displayed in the UI so we need to make sure it is
# always returned by the model.
if 'uuid' not in params['_columns']:
params['_columns'].append('uuid')
# We require the cpu_info field to show a special marker on some AMD CPU
# related crash reports.
if 'cpu_info' not in params['_columns']:
params['_columns'].append('cpu_info')
# The `uuid` field is a special case, it is always displayed in the first
# column of the table. Hence we do not want to show it again in the
# auto-generated list of columns, so we its name from the list of
# columns to display.
if 'uuid' in context['columns']:
context['columns'].remove('uuid')
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
context['current_page'] = current_page
context['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['_results_number'] = results_per_page
params['_results_offset'] = context['results_offset']
params['_facets'] = [] # We don't need no facets.
context['current_url'] = '%s?%s' % (reverse('signature:signature_report'),
urlencode_obj(current_query))
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
search_results['total_pages'] = int(
math.ceil(search_results['total'] / float(results_per_page)))
search_results['total_count'] = search_results['total']
context['query'] = search_results
return render(request, 'signature/signature_reports.html', context)
def signature_summary(request, params):
'''Return a list of specific aggregations. '''
data = {}
params['signature'] = '=' + params['signature'][0]
params['_results_number'] = 0
params['_facets'] = [
'platform_pretty_version',
'cpu_name',
'process_type',
'flash_version',
]
params['_histogram.uptime'] = ['product']
params['_histogram_interval.uptime'] = 60
# If the user has permissions, show exploitability.
all_fields = SuperSearchFields().get()
if has_permissions(
request.user, all_fields['exploitability']['permissions_needed']
):
params['_histogram.date'] = ['exploitability']
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
facets = search_results['facets']
# Transform uptime data to be easier to consume.
# Keys are in minutes.
if 'histogram_uptime' in facets:
labels = {
0: '< 1 min',
1: '1-5 min',
5: '5-15 min',
15: '15-60 min',
60: '> 1 hour'
}
uptimes_count = dict((x, 0) for x in labels)
for uptime in facets['histogram_uptime']:
for uptime_minutes in sorted(uptimes_count.keys(), reverse=True):
uptime_seconds = uptime_minutes * 60
if uptime['term'] >= uptime_seconds:
uptimes_count[uptime_minutes] += uptime['count']
break
uptimes = [
{'term': labels.get(key), 'count': count}
for key, count in uptimes_count.items()
if count > 0
]
uptimes = sorted(uptimes, key=lambda x: x['count'], reverse=True)
data['uptimes'] = uptimes
# Transform exploitability facet.
if 'histogram_date' in facets:
exploitability_base = {
'none': 0,
'low': 0,
'medium': 0,
'high': 0,
}
for day in facets['histogram_date']:
exploitability = dict(exploitability_base)
for expl in day['facets']['exploitability']:
if expl['term'] in exploitability:
exploitability[expl['term']] = expl['count']
day['exploitability'] = exploitability
facets['histogram_date'] = sorted(
facets['histogram_date'],
key=lambda x: x['term'],
reverse=True
)
data['query'] = search_results
return render(request, 'signature/signature_summary.html', data)
def exploitability_report(request, default_context=None):
context = default_context or {}
if not request.GET.get('product'):
url = reverse('crashstats:exploitability_report')
url += '?' + urllib.urlencode({'product': settings.DEFAULT_PRODUCT})
return redirect(url)
form = forms.ExploitabilityReportForm(
request.GET,
active_versions=context['active_versions'],
)
if not form.is_valid():
return http.HttpResponseBadRequest(str(form.errors))
product = form.cleaned_data['product']
version = form.cleaned_data['version']
api = SuperSearchUnredacted()
params = {
'product': product,
'version': version,
'_results_number': 0,
# This aggregates on crashes that do NOT contain these
# key words. For example, if a crash has
# {'exploitability': 'error: unable to analyze dump'}
# then it won't get included.
'exploitability': ['!error', '!interesting'],
'_aggs.signature': 'exploitability',
'_facets_size': settings.EXPLOITABILITY_BATCH_SIZE,
}
results = api.get(**params)
base_signature_report_dict = {
'product': product,
}
if version:
base_signature_report_dict['version'] = version
crashes = []
categories = ('high', 'none', 'low', 'medium', 'null')
for signature_facet in results['facets']['signature']:
# this 'signature_facet' will look something like this:
#
# {
# 'count': 1234,
# 'term': 'My | Signature',
# 'facets': {
# 'exploitability': [
# {'count': 1, 'term': 'high'},
# {'count': 23, 'term': 'medium'},
# {'count': 11, 'term': 'other'},
#
# And we only want to include those where:
#
# low or medium or high are greater than 0
#
exploitability = signature_facet['facets']['exploitability']
if not any(x['count'] for x in exploitability
if x['term'] in ('high', 'medium', 'low')):
continue
crash = {
'bugs': [],
'signature':
signature_facet['term'],
'high_count':
0,
'medium_count':
0,
'low_count':
0,
'none_count':
0,
'url':
(reverse('signature:signature_report') + '?' + urllib.urlencode(
dict(base_signature_report_dict,
signature=signature_facet['term']))),
}
for cluster in exploitability:
if cluster['term'] in categories:
crash['{}_count'.format(cluster['term'])] = (cluster['count'])
crash['med_or_high'] = (crash.get('high_count', 0) +
crash.get('medium_count', 0))
crashes.append(crash)
# Sort by the 'med_or_high' key first (descending),
# and by the signature second (ascending).
crashes.sort(key=lambda x: (-x['med_or_high'], x['signature']))
# now, let's go back and fill in the bugs
signatures = [x['signature'] for x in crashes]
if signatures:
api = models.Bugs()
bugs = defaultdict(list)
for b in api.get(signatures=signatures)['hits']:
bugs[b['signature']].append(b['id'])
for crash in crashes:
crash['bugs'] = bugs.get(crash['signature'], [])
context['crashes'] = crashes
context['product'] = product
context['version'] = version
context['report'] = 'exploitable'
return render(request, 'crashstats/exploitability_report.html', context)
def get_topcrashers_results(**kwargs):
'''Return the results of a search. '''
results = []
params = kwargs
params['_aggs.signature'] = [
'platform',
'is_garbage_collecting',
'hang_type',
'process_type',
'_histogram.uptime',
]
params['_histogram_interval.uptime'] = 60
# We don't care about no results, only facets.
params['_results_number'] = 0
if params.get('process_type') in ('any', 'all'):
params['process_type'] = None
api = SuperSearchUnredacted()
search_results = api.get(**params)
if search_results['total'] > 0:
results = search_results['facets']['signature']
platforms = models.Platforms().get_all()['hits']
platform_codes = [
x['code'] for x in platforms if x['code'] != 'unknown'
]
for i, hit in enumerate(results):
hit['signature'] = hit['term']
hit['rank'] = i + 1
hit['percent'] = 100.0 * hit['count'] / search_results['total']
# Number of crash per platform.
for platform in platform_codes:
hit[platform + '_count'] = 0
sig_platforms = hit['facets']['platform']
for platform in sig_platforms:
code = platform['term'][:3].lower()
if code in platform_codes:
hit[code + '_count'] = platform['count']
# Number of crashes happening during garbage collection.
hit['is_gc_count'] = 0
sig_gc = hit['facets']['is_garbage_collecting']
for row in sig_gc:
if row['term'].lower() == 't':
hit['is_gc_count'] = row['count']
# Number of plugin crashes.
hit['plugin_count'] = 0
sig_process = hit['facets']['process_type']
for row in sig_process:
if row['term'].lower() == 'plugin':
hit['plugin_count'] = row['count']
# Number of hang crashes.
hit['hang_count'] = 0
sig_hang = hit['facets']['hang_type']
for row in sig_hang:
# Hangs have weird values in the database: a value of 1 or -1
# means it is a hang, a value of 0 or missing means it is not.
if row['term'] in (1, -1):
hit['hang_count'] += row['count']
# Number of startup crashes.
hit['startup_percent'] = 0
sig_startup = hit['facets']['histogram_uptime']
for row in sig_startup:
if row['term'] == 0:
ratio = 1.0 * row['count'] / hit['count']
hit['startup_crash'] = ratio > 0.5
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
dates = get_date_boundaries(params)
delta = (dates[1] - dates[0]) * 2
params['date'] = [
'>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat()
]
params['_aggs.signature'] = [
'platform',
]
previous_range_results = api.get(**params)
total = previous_range_results['total']
compare_signatures = {}
if total > 0 and 'signature' in previous_range_results['facets']:
signatures = previous_range_results['facets']['signature']
for i, hit in enumerate(signatures):
compare_signatures[hit['term']] = {
'count': hit['count'],
'rank': i + 1,
'percent': 100.0 * hit['count'] / total
}
for hit in results:
sig = compare_signatures.get(hit['term'])
if sig:
hit['diff'] = sig['percent'] - hit['percent']
hit['rank_diff'] = sig['rank'] - hit['rank']
hit['previous_percent'] = sig['percent']
else:
hit['diff'] = 'new'
hit['rank_diff'] = 0
hit['previous_percent'] = 0
return search_results
def exploitability_report(request, default_context=None):
context = default_context or {}
if not request.GET.get("product"):
url = reverse("exploitability:report")
url += "?" + urlencode({"product": settings.DEFAULT_PRODUCT})
return redirect(url)
form = ExploitabilityReportForm(
request.GET, active_versions=context["active_versions"]
)
if not form.is_valid():
return http.HttpResponseBadRequest(str(form.errors))
product = form.cleaned_data["product"]
version = form.cleaned_data["version"]
api = SuperSearchUnredacted()
params = {
"product": product,
"version": version,
"_results_number": 0,
# This aggregates on crashes that do NOT contain these
# key words. For example, if a crash has
# {'exploitability': 'error: unable to analyze dump'}
# then it won't get included.
"exploitability": ["!error", "!interesting"],
"_aggs.signature": "exploitability",
"_facets_size": settings.EXPLOITABILITY_BATCH_SIZE,
}
results = api.get(**params)
base_signature_report_dict = {"product": product}
if version:
base_signature_report_dict["version"] = version
crashes = []
categories = ("high", "none", "low", "medium", "null")
for signature_facet in results["facets"]["signature"]:
# this 'signature_facet' will look something like this:
#
# {
# 'count': 1234,
# 'term': 'My | Signature',
# 'facets': {
# 'exploitability': [
# {'count': 1, 'term': 'high'},
# {'count': 23, 'term': 'medium'},
# {'count': 11, 'term': 'other'},
#
# And we only want to include those where:
#
# low or medium or high are greater than 0
#
exploitability = signature_facet["facets"]["exploitability"]
if not any(
x["count"] for x in exploitability if x["term"] in ("high", "medium", "low")
):
continue
crash = {
"bugs": [],
"signature": signature_facet["term"],
"high_count": 0,
"medium_count": 0,
"low_count": 0,
"none_count": 0,
"url": (
reverse("signature:signature_report")
+ "?"
+ urlencode(
dict(base_signature_report_dict, signature=signature_facet["term"])
)
),
}
for cluster in exploitability:
if cluster["term"] in categories:
crash["{}_count".format(cluster["term"])] = cluster["count"]
crash["med_or_high"] = crash.get("high_count", 0) + crash.get("medium_count", 0)
crashes.append(crash)
# Sort by the 'med_or_high' key first (descending),
# and by the signature second (ascending).
crashes.sort(key=lambda x: (-x["med_or_high"], x["signature"]))
# now, let's go back and fill in the bugs
signatures = [x["signature"] for x in crashes]
if signatures:
qs = (
models.BugAssociation.objects.filter(signature__in=signatures)
.values("bug_id", "signature")
.order_by("-bug_id", "signature")
)
bugs = defaultdict(list)
for item in qs:
bugs[item["signature"]].append(item["bug_id"])
for crash in crashes:
crash["bugs"] = bugs.get(crash["signature"], [])
context["crashes"] = crashes
context["product"] = product
context["version"] = version
context["report"] = "exploitable"
return render(request, "exploitability/report.html", context)
def signature_summary(request, params):
'''Return a list of specific aggregations. '''
context = {}
params['signature'] = '=' + params['signature'][0]
params['_results_number'] = 0
params['_facets'] = [
'platform_pretty_version',
'cpu_name',
'process_type',
'flash_version',
]
params['_histogram.uptime'] = ['product']
params['_histogram_interval.uptime'] = 60
params['_aggs.adapter_vendor_id'] = ['adapter_device_id']
params['_aggs.android_cpu_abi.android_manufacturer.android_model'] = [
'android_version'
]
# If the user has permissions, show exploitability.
all_fields = SuperSearchFields().get()
if has_permissions(
request.user, all_fields['exploitability']['permissions_needed']
):
params['_histogram.date'] = ['exploitability']
api = SuperSearchUnredacted()
# Now make the actual request with all expected parameters.
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
facets = search_results['facets']
# We need to make a separate query so that we can show all versions and
# not just the one asked for.
params_copy = {
'signature': params['signature'],
'_aggs.product.version': ['_cardinality.install_time'],
}
try:
product_results = api.get(**params_copy)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery
# to pick it up.
return http.HttpResponseBadRequest(render_exception(e))
if 'product' in product_results['facets']:
facets['product'] = product_results['facets']['product']
else:
facets['product'] = []
context['product_version_total'] = product_results['total']
_transform_uptime_summary(facets)
_transform_graphics_summary(facets)
_transform_mobile_summary(facets)
_transform_exploitability_summary(facets)
context['query'] = search_results
return render(request, 'signature/signature_summary.html', context)
def exploitability_report(request, default_context=None):
context = default_context or {}
if not request.GET.get('product'):
url = reverse('exploitability:report')
url += '?' + urlencode({
'product': settings.DEFAULT_PRODUCT
})
return redirect(url)
form = ExploitabilityReportForm(
request.GET,
active_versions=context['active_versions'],
)
if not form.is_valid():
return http.HttpResponseBadRequest(str(form.errors))
product = form.cleaned_data['product']
version = form.cleaned_data['version']
api = SuperSearchUnredacted()
params = {
'product': product,
'version': version,
'_results_number': 0,
# This aggregates on crashes that do NOT contain these
# key words. For example, if a crash has
# {'exploitability': 'error: unable to analyze dump'}
# then it won't get included.
'exploitability': ['!error', '!interesting'],
'_aggs.signature': 'exploitability',
'_facets_size': settings.EXPLOITABILITY_BATCH_SIZE,
}
results = api.get(**params)
base_signature_report_dict = {
'product': product,
}
if version:
base_signature_report_dict['version'] = version
crashes = []
categories = ('high', 'none', 'low', 'medium', 'null')
for signature_facet in results['facets']['signature']:
# this 'signature_facet' will look something like this:
#
# {
# 'count': 1234,
# 'term': 'My | Signature',
# 'facets': {
# 'exploitability': [
# {'count': 1, 'term': 'high'},
# {'count': 23, 'term': 'medium'},
# {'count': 11, 'term': 'other'},
#
# And we only want to include those where:
#
# low or medium or high are greater than 0
#
exploitability = signature_facet['facets']['exploitability']
if not any(
x['count']
for x in exploitability
if x['term'] in ('high', 'medium', 'low')
):
continue
crash = {
'bugs': [],
'signature': signature_facet['term'],
'high_count': 0,
'medium_count': 0,
'low_count': 0,
'none_count': 0,
'url': (
reverse('signature:signature_report') + '?' +
urlencode(dict(
base_signature_report_dict,
signature=signature_facet['term']
))
),
}
for cluster in exploitability:
if cluster['term'] in categories:
crash['{}_count'.format(cluster['term'])] = (
cluster['count']
)
crash['med_or_high'] = (
crash.get('high_count', 0) +
crash.get('medium_count', 0)
)
crashes.append(crash)
# Sort by the 'med_or_high' key first (descending),
# and by the signature second (ascending).
crashes.sort(key=lambda x: (-x['med_or_high'], x['signature']))
# now, let's go back and fill in the bugs
signatures = [x['signature'] for x in crashes]
if signatures:
qs = (
models.BugAssociation.objects
.filter(signature__in=signatures)
.values('bug_id', 'signature')
.order_by('-bug_id', 'signature')
)
bugs = defaultdict(list)
for item in qs:
bugs[item['signature']].append(item['bug_id'])
for crash in crashes:
crash['bugs'] = bugs.get(crash['signature'], [])
context['crashes'] = crashes
context['product'] = product
context['version'] = version
context['report'] = 'exploitable'
return render(request, 'exploitability/report.html', context)
def get_topcrashers_results(**kwargs):
"""Return the results of a search. """
results = []
params = kwargs
range_type = params.pop('_range_type')
dates = get_date_boundaries(params)
params['_aggs.signature'] = [
'platform',
'is_garbage_collecting',
'hang_type',
'process_type',
'_histogram.uptime',
]
params['_histogram_interval.uptime'] = 60
# We don't care about no results, only facets.
params['_results_number'] = 0
if params.get('process_type') in ('any', 'all'):
params['process_type'] = None
if range_type == 'build':
params['build_id'] = [
'>=' + datetime_to_build_id(dates[0]),
'<' + datetime_to_build_id(dates[1])
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
if search_results['total'] > 0:
results = search_results['facets']['signature']
platforms = models.Platforms().get_all()['hits']
platform_codes = [
x['code'] for x in platforms if x['code'] != 'unknown'
]
for i, hit in enumerate(results):
hit['signature'] = hit['term']
hit['rank'] = i + 1
hit['percent'] = 100.0 * hit['count'] / search_results['total']
# Number of crash per platform.
for platform in platform_codes:
hit[platform + '_count'] = 0
sig_platforms = hit['facets']['platform']
for platform in sig_platforms:
code = platform['term'][:3].lower()
if code in platform_codes:
hit[code + '_count'] = platform['count']
# Number of crashes happening during garbage collection.
hit['is_gc_count'] = 0
sig_gc = hit['facets']['is_garbage_collecting']
for row in sig_gc:
if row['term'].lower() == 't':
hit['is_gc_count'] = row['count']
# Number of plugin crashes.
hit['plugin_count'] = 0
sig_process = hit['facets']['process_type']
for row in sig_process:
if row['term'].lower() == 'plugin':
hit['plugin_count'] = row['count']
# Number of hang crashes.
hit['hang_count'] = 0
sig_hang = hit['facets']['hang_type']
for row in sig_hang:
# Hangs have weird values in the database: a value of 1 or -1
# means it is a hang, a value of 0 or missing means it is not.
if row['term'] in (1, -1):
hit['hang_count'] += row['count']
# Number of startup crashes.
hit['startup_percent'] = 0
sig_startup = hit['facets']['histogram_uptime']
for row in sig_startup:
if row['term'] == 0:
ratio = 1.0 * row['count'] / hit['count']
hit['startup_crash'] = ratio > 0.5
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params['date'] = [
'>=' + (dates[1] - delta).isoformat(),
'<' + dates[0].isoformat()
]
params['_aggs.signature'] = [
'platform',
]
params['_facets_size'] *= 2
if range_type == 'build':
params['date'][1] = '<' + dates[1].isoformat()
params['build_id'] = [
'>=' + datetime_to_build_id(dates[1] - delta),
'<' + datetime_to_build_id(dates[0])
]
previous_range_results = api.get(**params)
total = previous_range_results['total']
compare_signatures = {}
if total > 0 and 'signature' in previous_range_results['facets']:
signatures = previous_range_results['facets']['signature']
for i, hit in enumerate(signatures):
compare_signatures[hit['term']] = {
'count': hit['count'],
'rank': i + 1,
'percent': 100.0 * hit['count'] / total
}
for hit in results:
sig = compare_signatures.get(hit['term'])
if sig:
hit['diff'] = sig['percent'] - hit['percent']
hit['rank_diff'] = sig['rank'] - hit['rank']
hit['previous_percent'] = sig['percent']
else:
hit['diff'] = 'new'
hit['rank_diff'] = 0
hit['previous_percent'] = 0
return search_results
def signature_reports(request):
'''Return the results of a search. '''
params = get_validated_params(request)
if isinstance(params, http.HttpResponseBadRequest):
# There was an error in the form, let's return it.
return params
signature = params['signature'][0]
data = {}
data['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
allowed_fields = get_allowed_fields(request.user)
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
data['params'] = current_query.copy()
if '_columns' in data['params']:
del data['params']['_columns']
data['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS
# Make sure only allowed fields are used
data['columns'] = [
x for x in data['columns'] if x in allowed_fields
]
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
data['current_page'] = current_page
data['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['_results_number'] = results_per_page
params['_results_offset'] = data['results_offset']
params['_facets'] = [] # We don't need no facets.
data['current_url'] = '%s?%s' % (
reverse('signature:signature_report'),
current_query.urlencode()
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError, e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
def _get_crashes_per_day_with_adu(
params, start_date, end_date, platforms, _date_range_type
):
api = SuperSearchUnredacted()
results = api.get(**params)
platforms_api = models.Platforms()
platforms = platforms_api.get()
# now get the ADI for these product versions
api = models.ADI()
adi_counts = api.get(
product=params['product'],
versions=params['version'],
start_date=start_date,
end_date=end_date,
platforms=[x['name'] for x in platforms if x.get('display')],
)
api = models.ProductBuildTypes()
product_build_types = api.get(product=params['product'])['hits']
# This `adi_counts` is a list of dicts that looks like this:
# {
# 'adi_count': 123,
# 'date': '2015-08-15',
# 'version': '40.0.2'
# 'build_type': 'beta',
# }
# We need to turn this around so that it's like this:
# {
# '40.0.2': {
# '2015-08-15': [123, 1.0],
# ...
# },
# ...
# }
# So it can easily be looked up how many counts there are per
# version per date.
#
# Note!! that the 1.0 in the example above is the throttle value
# for this build_type. We got that from the ProductBuildTypes API.
adi_by_version = {}
# If any of the versions end with a 'b' we want to collect
# and group them together under one.
# I.e. if we have adi count of '42.0b1':123 and '42.0b2':345
# then we want to combine that to just '42.0b':123+345
beta_versions = [x for x in params['version'] if x.endswith('b')]
def get_parent_version(ver):
try:
return [x for x in beta_versions if ver.startswith(x)][0]
except IndexError:
return None
def add_version_to_adi(version, count, date, throttle):
if version not in adi_by_version:
adi_by_version[version] = {}
try:
before = adi_by_version[version][date][0]
except KeyError:
before = 0
adi_by_version[version][date] = [count + before, throttle]
for group in adi_counts['hits']:
version = group['version']
# Make this a string so it can be paired with the facets 'term'
# key which is also a date in ISO format.
date = group['date'].isoformat()
build_type = group['build_type']
count = group['adi_count']
throttle = product_build_types[build_type]
# If this version was requested, add it to the data structure.
if version in params['version']:
add_version_to_adi(version, count, date, throttle)
# If this version is part of a beta, add it to the data structure.
parent_version = get_parent_version(version)
if parent_version is not None:
version = parent_version
add_version_to_adi(version, count, date, throttle)
# We might have queried for aggregates for version ['19.0a1', '18.0b']
# but SuperSearch will give us facets for versions:
# ['19.0a1', '18.0b1', '18.0b2', '18.0b3']
# The facets look something like this:
# {
# 'histogram_date': [
# {
# 'count': 1234,
# 'facets': [
# {'count': 201, 'term': '19.0a1'},
# {'count': 196, 'term': '18.0b1'},
# {'count': 309, 'term': '18.0b2'},
# {'count': 991, 'term': '18.0b3'},
# ],
# 'term': '2015-01-10T00:00:00'
# },
# ...
#
# 'version': [
# {'count': 45234, 'term': '19.0a1'},
# {'count': 39001, 'term': '18.0b1'},
# {'count': 56123, 'term': '18.0b2'},
# {'count': 90133, 'term': '18.0b3'},
#
# Our job is to rewrite that so it looks like this:
# {
# 'histogram_date': [
# {
# 'count': 1234,
# 'facets': [
# {'count': 201, 'term': '19.0a1'},
# {'count': 196+309+991, 'term': '18.0b'},
# ],
# 'term': '2015-01-10T00:00:00'
# },
# ...
#
# 'version': [
# {'count': 45234, 'term': '19.0a1'},
# {'count': 39001+56123+90133, 'term': '18.0b'},
#
histogram = results['facets']['histogram_date']
for date_cluster in histogram:
parent_totals = defaultdict(int)
for facet_cluster in list(date_cluster['facets']['version']):
version = facet_cluster['term']
parent_version = get_parent_version(version)
if parent_version is not None:
parent_totals[parent_version] += facet_cluster['count']
if version not in params['version']:
date_cluster['facets']['version'].remove(facet_cluster)
for version in parent_totals:
date_cluster['facets']['version'].append({
'count': parent_totals[version],
'term': version
})
parent_totals = defaultdict(int)
for facet_cluster in list(results['facets']['version']):
version = facet_cluster['term']
parent_version = get_parent_version(version)
if parent_version is not None:
parent_totals[parent_version] += facet_cluster['count']
if version not in params['version']:
results['facets']['version'].remove(facet_cluster)
for version in parent_totals:
results['facets']['version'].append({
'count': parent_totals[version],
'term': version,
})
graph_data = {}
graph_data = build_data_object_for_crashes_per_day_graph(
start_date.strftime('%Y-%m-%d'),
end_date.strftime('%Y-%m-%d'),
results['facets'],
adi_by_version,
_date_range_type
)
graph_data['product_versions'] = get_product_versions_for_crashes_per_day(
results['facets'],
params['product'],
)
return graph_data, results, adi_by_version
def _get_crashes_per_day_with_adu(params, start_date, end_date, platforms,
_date_range_type):
api = SuperSearchUnredacted()
results = api.get(**params)
platforms_api = models.Platforms()
platforms = platforms_api.get()
# now get the ADI for these product versions
api = models.ADI()
adi_counts = api.get(
product=params['product'],
versions=params['version'],
start_date=start_date,
end_date=end_date,
platforms=[x['name'] for x in platforms if x.get('display')],
)
api = models.ProductBuildTypes()
product_build_types = api.get(product=params['product'])['hits']
# This `adi_counts` is a list of dicts that looks like this:
# {
# 'adi_count': 123,
# 'date': '2015-08-15',
# 'version': '40.0.2'
# 'build_type': 'beta',
# }
# We need to turn this around so that it's like this:
# {
# '40.0.2': {
# '2015-08-15': [123, 1.0],
# ...
# },
# ...
# }
# So it can easily be looked up how many counts there are per
# version per date.
#
# Note!! that the 1.0 in the example above is the throttle value
# for this build_type. We got that from the ProductBuildTypes API.
adi_by_version = {}
# If any of the versions end with a 'b' we want to collect
# and group them together under one.
# I.e. if we have adi count of '42.0b1':123 and '42.0b2':345
# then we want to combine that to just '42.0b':123+345
beta_versions = [x for x in params['version'] if x.endswith('b')]
def get_parent_version(ver):
try:
return [x for x in beta_versions if ver.startswith(x)][0]
except IndexError:
return None
def add_version_to_adi(version, count, date, throttle):
if version not in adi_by_version:
adi_by_version[version] = {}
try:
before = adi_by_version[version][date][0]
except KeyError:
before = 0
adi_by_version[version][date] = [count + before, throttle]
for group in adi_counts['hits']:
version = group['version']
# Make this a string so it can be paired with the facets 'term'
# key which is also a date in ISO format.
date = group['date'].isoformat()
build_type = group['build_type']
count = group['adi_count']
throttle = product_build_types[build_type]
# If this version was requested, add it to the data structure.
if version in params['version']:
add_version_to_adi(version, count, date, throttle)
# If this version is part of a beta, add it to the data structure.
parent_version = get_parent_version(version)
if parent_version is not None:
version = parent_version
add_version_to_adi(version, count, date, throttle)
# We might have queried for aggregates for version ['19.0a1', '18.0b']
# but SuperSearch will give us facets for versions:
# ['19.0a1', '18.0b1', '18.0b2', '18.0b3']
# The facets look something like this:
# {
# 'histogram_date': [
# {
# 'count': 1234,
# 'facets': [
# {'count': 201, 'term': '19.0a1'},
# {'count': 196, 'term': '18.0b1'},
# {'count': 309, 'term': '18.0b2'},
# {'count': 991, 'term': '18.0b3'},
# ],
# 'term': '2015-01-10T00:00:00'
# },
# ...
#
# 'version': [
# {'count': 45234, 'term': '19.0a1'},
# {'count': 39001, 'term': '18.0b1'},
# {'count': 56123, 'term': '18.0b2'},
# {'count': 90133, 'term': '18.0b3'},
#
# Our job is to rewrite that so it looks like this:
# {
# 'histogram_date': [
# {
# 'count': 1234,
# 'facets': [
# {'count': 201, 'term': '19.0a1'},
# {'count': 196+309+991, 'term': '18.0b'},
# ],
# 'term': '2015-01-10T00:00:00'
# },
# ...
#
# 'version': [
# {'count': 45234, 'term': '19.0a1'},
# {'count': 39001+56123+90133, 'term': '18.0b'},
#
histogram = results['facets']['histogram_date']
for date_cluster in histogram:
parent_totals = defaultdict(int)
for facet_cluster in list(date_cluster['facets']['version']):
version = facet_cluster['term']
parent_version = get_parent_version(version)
if parent_version is not None:
parent_totals[parent_version] += facet_cluster['count']
if version not in params['version']:
date_cluster['facets']['version'].remove(facet_cluster)
for version in parent_totals:
date_cluster['facets']['version'].append({
'count':
parent_totals[version],
'term':
version
})
parent_totals = defaultdict(int)
for facet_cluster in list(results['facets']['version']):
version = facet_cluster['term']
parent_version = get_parent_version(version)
if parent_version is not None:
parent_totals[parent_version] += facet_cluster['count']
if version not in params['version']:
results['facets']['version'].remove(facet_cluster)
for version in parent_totals:
results['facets']['version'].append({
'count': parent_totals[version],
'term': version,
})
graph_data = {}
graph_data = build_data_object_for_crashes_per_day_graph(
start_date.strftime('%Y-%m-%d'), end_date.strftime('%Y-%m-%d'),
results['facets'], adi_by_version, _date_range_type)
graph_data['product_versions'] = get_product_versions_for_crashes_per_day(
results['facets'],
params['product'],
)
return graph_data, results, adi_by_version
def signature_reports(request, params):
'''Return the results of a search. '''
signature = params['signature'][0]
context = {}
context['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
allowed_fields = get_allowed_fields(request.user)
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
context['params'] = current_query.copy()
if '_sort' in context['params']:
del context['params']['_sort']
if '_columns' in context['params']:
del context['params']['_columns']
context['sort'] = request.GET.getlist('_sort')
context['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS
# Make sure only allowed fields are used.
context['sort'] = [
x for x in context['sort']
if x in allowed_fields or
(x.startswith('-') and x[1:] in allowed_fields)
]
context['columns'] = [
x for x in context['columns'] if x in allowed_fields
]
params['_sort'] = context['sort']
# Copy the list of columns so that they can differ.
params['_columns'] = list(context['columns'])
# The uuid is always displayed in the UI so we need to make sure it is
# always returned by the model.
if 'uuid' not in params['_columns']:
params['_columns'].append('uuid')
# We require the cpu_info field to show a special marker on some AMD CPU
# related crash reports.
if 'cpu_info' not in params['_columns']:
params['_columns'].append('cpu_info')
# The `uuid` field is a special case, it is always displayed in the first
# column of the table. Hence we do not want to show it again in the
# auto-generated list of columns, so we its name from the list of
# columns to display.
if 'uuid' in context['columns']:
context['columns'].remove('uuid')
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
context['current_page'] = current_page
context['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['_results_number'] = results_per_page
params['_results_offset'] = context['results_offset']
params['_facets'] = [] # We don't need no facets.
context['current_url'] = '%s?%s' % (
reverse('signature:signature_report'),
urlencode_obj(current_query)
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
search_results['total_pages'] = int(
math.ceil(
search_results['total'] / float(results_per_page)
)
)
search_results['total_count'] = search_results['total']
context['query'] = search_results
return render(request, 'signature/signature_reports.html', context)
def get_topcrashers_results(**kwargs):
"""Return the results of a search. """
results = []
params = kwargs
range_type = params.pop('_range_type')
dates = get_date_boundaries(params)
params['_aggs.signature'] = [
'platform',
'is_garbage_collecting',
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
'_cardinality.install_time',
]
params['_histogram_interval.uptime'] = 60
# We don't care about no results, only facets.
params['_results_number'] = 0
if params.get('process_type') in ('any', 'all'):
params['process_type'] = None
if range_type == 'build':
params['build_id'] = [
'>=' + datetime_to_build_id(dates[0]),
'<' + datetime_to_build_id(dates[1])
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
if search_results['total'] > 0:
results = search_results['facets']['signature']
platforms = models.Platforms().get_all()['hits']
platform_codes = [
x['code'] for x in platforms if x['code'] != 'unknown'
]
for i, hit in enumerate(results):
hit['signature'] = hit['term']
hit['rank'] = i + 1
hit['percent'] = 100.0 * hit['count'] / search_results['total']
# Number of crash per platform.
for platform in platform_codes:
hit[platform + '_count'] = 0
sig_platforms = hit['facets']['platform']
for platform in sig_platforms:
code = platform['term'][:3].lower()
if code in platform_codes:
hit[code + '_count'] = platform['count']
# Number of crashes happening during garbage collection.
hit['is_gc_count'] = 0
sig_gc = hit['facets']['is_garbage_collecting']
for row in sig_gc:
if row['term'].lower() == 't':
hit['is_gc_count'] = row['count']
# Number of plugin crashes.
hit['plugin_count'] = 0
sig_process = hit['facets']['process_type']
for row in sig_process:
if row['term'].lower() == 'plugin':
hit['plugin_count'] = row['count']
# Number of hang crashes.
hit['hang_count'] = 0
sig_hang = hit['facets']['hang_type']
for row in sig_hang:
# Hangs have weird values in the database: a value of 1 or -1
# means it is a hang, a value of 0 or missing means it is not.
if row['term'] in (1, -1):
hit['hang_count'] += row['count']
# Number of crashes happening during startup. This is defined by
# the client, as opposed to the next method which relies on
# the uptime of the client.
hit['startup_count'] = sum(
row['count'] for row in hit['facets']['startup_crash']
if row['term'] in ('T', '1')
)
# Is a startup crash if more than half of the crashes are happening
# in the first minute after launch.
hit['startup_crash'] = False
sig_uptime = hit['facets']['histogram_uptime']
for row in sig_uptime:
# Aggregation buckets use the lowest value of the bucket as
# term. So for everything between 0 and 60 excluded, the
# term will be `0`.
if row['term'] < 60:
ratio = 1.0 * row['count'] / hit['count']
hit['startup_crash'] = ratio > 0.5
# Number of distinct installations.
hit['installs_count'] = (
hit['facets']['cardinality_install_time']['value']
)
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params['date'] = [
'>=' + (dates[1] - delta).isoformat(),
'<' + dates[0].isoformat()
]
params['_aggs.signature'] = [
'platform',
]
params['_facets_size'] *= 2
if range_type == 'build':
params['date'][1] = '<' + dates[1].isoformat()
params['build_id'] = [
'>=' + datetime_to_build_id(dates[1] - delta),
'<' + datetime_to_build_id(dates[0])
]
previous_range_results = api.get(**params)
total = previous_range_results['total']
compare_signatures = {}
if total > 0 and 'signature' in previous_range_results['facets']:
signatures = previous_range_results['facets']['signature']
for i, hit in enumerate(signatures):
compare_signatures[hit['term']] = {
'count': hit['count'],
'rank': i + 1,
'percent': 100.0 * hit['count'] / total
}
for hit in results:
sig = compare_signatures.get(hit['term'])
if sig:
hit['diff'] = sig['percent'] - hit['percent']
hit['rank_diff'] = sig['rank'] - hit['rank']
hit['previous_percent'] = sig['percent']
else:
hit['diff'] = 'new'
hit['rank_diff'] = 0
hit['previous_percent'] = 0
return search_results
def signature_summary(request, params):
'''Return a list of specific aggregations. '''
data = {}
params['signature'] = '=' + params['signature'][0]
params['_results_number'] = 0
params['_facets'] = [
'platform_pretty_version',
'cpu_name',
'process_type',
'flash_version',
]
params['_histogram.uptime'] = ['product']
params['_histogram_interval.uptime'] = 60
params['_aggs.adapter_vendor_id'] = ['adapter_device_id']
params['_aggs.android_cpu_abi.android_manufacturer.android_model'] = [
'android_version'
]
# If the user has permissions, show exploitability.
all_fields = SuperSearchFields().get()
if has_permissions(
request.user, all_fields['exploitability']['permissions_needed']
):
params['_histogram.date'] = ['exploitability']
api = SuperSearchUnredacted()
# Now make the actual request with all expected parameters.
try:
search_results = api.get(**params)
except models.BadStatusCodeError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
facets = search_results['facets']
# We need to make a separate query so that we can show all versions and
# not just the one asked for.
params_copy = {
'signature': params['signature'],
'_aggs.product.version': ['_cardinality.install_time'],
}
try:
product_results = api.get(**params_copy)
except models.BadStatusCodeError as e:
# We need to return the error message in some HTML form for jQuery
# to pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
if 'product' in product_results['facets']:
facets['product'] = product_results['facets']['product']
else:
facets['product'] = []
data['product_version_total'] = product_results['total']
_transform_uptime_summary(facets)
_transform_graphics_summary(facets)
_transform_mobile_summary(facets)
_transform_exploitability_summary(facets)
data['query'] = search_results
return render(request, 'signature/signature_summary.html', data)
def get_topcrashers_results(**kwargs):
"""Return the results of a search. """
results = []
params = kwargs
params["_aggs.signature"] = ["platform", "is_garbage_collecting", "hang_type", "process_type", "_histogram.uptime"]
params["_histogram_interval.uptime"] = 60
# We don't care about no results, only facets.
params["_results_number"] = 0
if params.get("process_type") in ("any", "all"):
params["process_type"] = None
api = SuperSearchUnredacted()
search_results = api.get(**params)
if search_results["total"] > 0:
results = search_results["facets"]["signature"]
platforms = models.Platforms().get_all()["hits"]
platform_codes = [x["code"] for x in platforms if x["code"] != "unknown"]
for i, hit in enumerate(results):
hit["signature"] = hit["term"]
hit["rank"] = i + 1
hit["percent"] = 100.0 * hit["count"] / search_results["total"]
# Number of crash per platform.
for platform in platform_codes:
hit[platform + "_count"] = 0
sig_platforms = hit["facets"]["platform"]
for platform in sig_platforms:
code = platform["term"][:3].lower()
if code in platform_codes:
hit[code + "_count"] = platform["count"]
# Number of crashes happening during garbage collection.
hit["is_gc_count"] = 0
sig_gc = hit["facets"]["is_garbage_collecting"]
for row in sig_gc:
if row["term"].lower() == "t":
hit["is_gc_count"] = row["count"]
# Number of plugin crashes.
hit["plugin_count"] = 0
sig_process = hit["facets"]["process_type"]
for row in sig_process:
if row["term"].lower() == "plugin":
hit["plugin_count"] = row["count"]
# Number of hang crashes.
hit["hang_count"] = 0
sig_hang = hit["facets"]["hang_type"]
for row in sig_hang:
# Hangs have weird values in the database: a value of 1 or -1
# means it is a hang, a value of 0 or missing means it is not.
if row["term"] in (1, -1):
hit["hang_count"] += row["count"]
# Number of startup crashes.
hit["startup_percent"] = 0
sig_startup = hit["facets"]["histogram_uptime"]
for row in sig_startup:
if row["term"] == 0:
ratio = 1.0 * row["count"] / hit["count"]
hit["startup_crash"] = ratio > 0.5
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
dates = get_date_boundaries(params)
delta = (dates[1] - dates[0]) * 2
params["date"] = [">=" + (dates[1] - delta).isoformat(), "<" + dates[0].isoformat()]
params["_aggs.signature"] = ["platform"]
previous_range_results = api.get(**params)
total = previous_range_results["total"]
compare_signatures = {}
if total > 0 and "signature" in previous_range_results["facets"]:
signatures = previous_range_results["facets"]["signature"]
for i, hit in enumerate(signatures):
compare_signatures[hit["term"]] = {
"count": hit["count"],
"rank": i + 1,
"percent": 100.0 * hit["count"] / total,
}
for hit in results:
sig = compare_signatures.get(hit["term"])
if sig:
hit["diff"] = sig["percent"] - hit["percent"]
hit["rank_diff"] = sig["rank"] - hit["rank"]
hit["previous_percent"] = sig["percent"]
else:
hit["diff"] = "new"
hit["rank_diff"] = 0
hit["previous_percent"] = 0
return search_results
def signature_reports(request, params):
'''Return the results of a search. '''
signature = params['signature'][0]
data = {}
data['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
allowed_fields = get_allowed_fields(request.user)
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
data['params'] = current_query.copy()
if '_columns' in data['params']:
del data['params']['_columns']
data['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS
# Make sure only allowed fields are used
data['columns'] = [
x for x in data['columns'] if x in allowed_fields
]
# Copy the list of columns so that they can differ.
params['_columns'] = list(data['columns'])
# The uuid is always displayed in the UI so we need to make sure it is
# always returned by the model.
if 'uuid' not in params['_columns']:
params['_columns'].append('uuid')
# The `uuid` field is a special case, it is always displayed in the first
# column of the table. Hence we do not want to show it again in the
# auto-generated list of columns, so we its name from the list of
# columns to display.
if 'uuid' in data['columns']:
data['columns'].remove('uuid')
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
data['current_page'] = current_page
data['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['_results_number'] = results_per_page
params['_results_offset'] = data['results_offset']
params['_facets'] = [] # We don't need no facets.
data['current_url'] = '%s?%s' % (
reverse('signature:signature_report'),
current_query.urlencode()
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
search_results['total_pages'] = int(
math.ceil(
search_results['total'] / float(results_per_page)
)
)
search_results['total_count'] = search_results['total']
data['query'] = search_results
return render(request, 'signature/signature_reports.html', data)
def get_topcrashers_stats(**kwargs):
"""Return the results of a search. """
params = kwargs
range_type = params.pop('_range_type')
dates = get_date_boundaries(params)
params['_aggs.signature'] = [
'platform',
'is_garbage_collecting',
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
'_cardinality.install_time',
]
params['_histogram_interval.uptime'] = 60
# We don't care about no results, only facets.
params['_results_number'] = 0
if params.get('process_type') in ('any', 'all'):
params['process_type'] = None
if range_type == 'build':
params['build_id'] = [
'>=' + datetime_to_build_id(dates[0]),
'<' + datetime_to_build_id(dates[1])
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
signatures_stats = []
if search_results['total'] > 0:
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params['date'] = [
'>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat()
]
params['_aggs.signature'] = [
'platform',
]
params['_facets_size'] *= 2
if range_type == 'build':
params['date'][1] = '<' + dates[1].isoformat()
params['build_id'] = [
'>=' + datetime_to_build_id(dates[1] - delta),
'<' + datetime_to_build_id(dates[0])
]
previous_range_results = api.get(**params)
previous_signatures = get_comparison_signatures(previous_range_results)
for index, signature in enumerate(
search_results['facets']['signature']):
previous_signature = previous_signatures.get(signature['term'])
signatures_stats.append(
SignatureStats(
signature=signature,
num_total_crashes=search_results['total'],
rank=index,
platforms=models.Platforms().get_all()['hits'],
previous_signature=previous_signature,
))
return signatures_stats
def get_topcrashers_stats(**kwargs):
"""Return the results of a search. """
params = kwargs
range_type = params.pop('_range_type')
dates = get_date_boundaries(params)
params['_aggs.signature'] = [
'platform',
'is_garbage_collecting',
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
'_cardinality.install_time',
]
params['_histogram_interval.uptime'] = 60
# We don't care about no results, only facets.
params['_results_number'] = 0
if params.get('process_type') in ('any', 'all'):
params['process_type'] = None
if range_type == 'build':
params['build_id'] = [
'>=' + datetime_to_build_id(dates[0]),
'<' + datetime_to_build_id(dates[1])
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
signatures_stats = []
if search_results['total'] > 0:
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params['date'] = [
'>=' + (dates[1] - delta).isoformat(),
'<' + dates[0].isoformat()
]
params['_aggs.signature'] = [
'platform',
]
params['_facets_size'] *= 2
if range_type == 'build':
params['date'][1] = '<' + dates[1].isoformat()
params['build_id'] = [
'>=' + datetime_to_build_id(dates[1] - delta),
'<' + datetime_to_build_id(dates[0])
]
previous_range_results = api.get(**params)
previous_signatures = get_comparison_signatures(previous_range_results)
for index, signature in enumerate(search_results['facets']['signature']):
previous_signature = previous_signatures.get(signature['term'])
signatures_stats.append(SignatureStats(
signature=signature,
num_total_crashes=search_results['total'],
rank=index,
platforms=models.Platform.objects.values(),
previous_signature=previous_signature,
))
return signatures_stats
def get_topcrashers_results(**kwargs):
"""Return the results of a search. """
results = []
params = kwargs
range_type = params.pop('_range_type')
dates = get_date_boundaries(params)
params['_aggs.signature'] = [
'platform',
'is_garbage_collecting',
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
'_cardinality.install_time',
]
params['_histogram_interval.uptime'] = 60
# We don't care about no results, only facets.
params['_results_number'] = 0
if params.get('process_type') in ('any', 'all'):
params['process_type'] = None
if range_type == 'build':
params['build_id'] = [
'>=' + datetime_to_build_id(dates[0]),
'<' + datetime_to_build_id(dates[1])
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
if search_results['total'] > 0:
results = search_results['facets']['signature']
platforms = models.Platforms().get_all()['hits']
platform_codes = [
x['code'] for x in platforms if x['code'] != 'unknown'
]
for i, hit in enumerate(results):
hit['signature'] = hit['term']
hit['rank'] = i + 1
hit['percent'] = 100.0 * hit['count'] / search_results['total']
# Number of crash per platform.
for platform in platform_codes:
hit[platform + '_count'] = 0
sig_platforms = hit['facets']['platform']
for platform in sig_platforms:
code = platform['term'][:3].lower()
if code in platform_codes:
hit[code + '_count'] = platform['count']
# Number of crashes happening during garbage collection.
hit['is_gc_count'] = 0
sig_gc = hit['facets']['is_garbage_collecting']
for row in sig_gc:
if row['term'].lower() == 't':
hit['is_gc_count'] = row['count']
# Number of plugin crashes.
hit['plugin_count'] = 0
sig_process = hit['facets']['process_type']
for row in sig_process:
if row['term'].lower() == 'plugin':
hit['plugin_count'] = row['count']
# Number of hang crashes.
hit['hang_count'] = 0
sig_hang = hit['facets']['hang_type']
for row in sig_hang:
# Hangs have weird values in the database: a value of 1 or -1
# means it is a hang, a value of 0 or missing means it is not.
if row['term'] in (1, -1):
hit['hang_count'] += row['count']
# Number of crashes happening during startup. This is defined by
# the client, as opposed to the next method which relies on
# the uptime of the client.
hit['startup_count'] = sum(
row['count'] for row in hit['facets']['startup_crash']
if row['term'] in ('T', '1'))
# Is a startup crash if more than half of the crashes are happening
# in the first minute after launch.
hit['startup_crash'] = False
sig_uptime = hit['facets']['histogram_uptime']
for row in sig_uptime:
# Aggregation buckets use the lowest value of the bucket as
# term. So for everything between 0 and 60 excluded, the
# term will be `0`.
if row['term'] < 60:
ratio = 1.0 * row['count'] / hit['count']
hit['startup_crash'] = ratio > 0.5
# Number of distinct installations.
hit['installs_count'] = (
hit['facets']['cardinality_install_time']['value'])
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params['date'] = [
'>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat()
]
params['_aggs.signature'] = [
'platform',
]
params['_facets_size'] *= 2
if range_type == 'build':
params['date'][1] = '<' + dates[1].isoformat()
params['build_id'] = [
'>=' + datetime_to_build_id(dates[1] - delta),
'<' + datetime_to_build_id(dates[0])
]
previous_range_results = api.get(**params)
total = previous_range_results['total']
compare_signatures = {}
if total > 0 and 'signature' in previous_range_results['facets']:
signatures = previous_range_results['facets']['signature']
for i, hit in enumerate(signatures):
compare_signatures[hit['term']] = {
'count': hit['count'],
'rank': i + 1,
'percent': 100.0 * hit['count'] / total
}
for hit in results:
sig = compare_signatures.get(hit['term'])
if sig:
hit['diff'] = sig['percent'] - hit['percent']
hit['rank_diff'] = sig['rank'] - hit['rank']
hit['previous_percent'] = sig['percent']
else:
hit['diff'] = 'new'
hit['rank_diff'] = 0
hit['previous_percent'] = 0
return search_results
def signature_reports(request, params):
"""Return the results of a search. """
signature = params["signature"][0]
context = {}
context["query"] = {"total": 0, "total_count": 0, "total_pages": 0}
allowed_fields = get_allowed_fields(request.user)
current_query = request.GET.copy()
if "page" in current_query:
del current_query["page"]
context["params"] = current_query.copy()
if "_sort" in context["params"]:
del context["params"]["_sort"]
if "_columns" in context["params"]:
del context["params"]["_columns"]
context["sort"] = request.GET.getlist("_sort")
context["columns"] = request.GET.getlist("_columns") or DEFAULT_COLUMNS
# Make sure only allowed fields are used.
context["sort"] = [
x for x in context["sort"] if x in allowed_fields or (
x.startswith("-") and x[1:] in allowed_fields)
]
context["columns"] = [x for x in context["columns"] if x in allowed_fields]
params["_sort"] = context["sort"]
# Copy the list of columns so that they can differ.
params["_columns"] = list(context["columns"])
# The uuid is always displayed in the UI so we need to make sure it is
# always returned by the model.
if "uuid" not in params["_columns"]:
params["_columns"].append("uuid")
# We require the cpu_info field to show a special marker on some AMD CPU
# related crash reports.
if "cpu_info" not in params["_columns"]:
params["_columns"].append("cpu_info")
# The `uuid` field is a special case, it is always displayed in the first
# column of the table. Hence we do not want to show it again in the
# auto-generated list of columns, so we its name from the list of
# columns to display.
if "uuid" in context["columns"]:
context["columns"].remove("uuid")
try:
current_page = int(request.GET.get("page", 1))
except ValueError:
return http.HttpResponseBadRequest("Invalid page")
if current_page <= 0:
current_page = 1
results_per_page = 50
context["current_page"] = current_page
context["results_offset"] = results_per_page * (current_page - 1)
params["signature"] = "=" + signature
params["_results_number"] = results_per_page
params["_results_offset"] = context["results_offset"]
params["_facets"] = [] # We don't need no facets.
context["current_url"] = "%s?%s" % (
reverse("signature:signature_report"),
urlencode_obj(current_query),
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
search_results["total_pages"] = int(
math.ceil(search_results["total"] / float(results_per_page)))
search_results["total_count"] = search_results["total"]
context["query"] = search_results
return render(request, "signature/signature_reports.html", context)