def supersearch_field(request):
context = {}
field_name = request.GET.get('name')
if field_name:
all_fields = SuperSearchFields().get()
field_data = all_fields.get(field_name)
if not field_data:
return http.HttpResponseBadRequest(
'The field "%s" does not exist' % field_name)
else:
full_name = request.GET.get('full_name')
if full_name:
if '.' not in full_name:
name = full_name
namespace = None
else:
namespace, name = full_name.rsplit('.', 1)
field_data = {
'in_database_name': name,
'namespace': namespace,
}
else:
field_data = {}
context['field'] = field_data
perms = Permission.objects.filter(content_type__model='').order_by('name')
context['all_permissions'] = ['crashstats.' + x.codename for x in perms]
return render(request, 'manage/supersearch_field.html', context)
def supersearch_field(request):
context = {}
field_name = request.GET.get("name")
if field_name:
all_fields = SuperSearchFields().get()
field_data = all_fields.get(field_name)
if not field_data:
return http.HttpResponseBadRequest('The field "%s" does not exist' % field_name)
else:
full_name = request.GET.get("full_name")
if full_name:
if "." not in full_name:
name = full_name
namespace = None
else:
namespace, name = full_name.rsplit(".", 1)
field_data = {"in_database_name": name, "namespace": namespace}
else:
field_data = {}
context["field"] = field_data
perms = Permission.objects.filter(content_type__model="").order_by("name")
context["all_permissions"] = ["crashstats." + x.codename for x in perms]
return render(request, "manage/supersearch_field.html", context)
def get_from_es(namespace, baseline=None):
# @namespace is something like 'raw_crash' or 'processed_crash'
cache_key = 'api_supersearch_fields_%s' % namespace
fields = cache.get(cache_key)
if fields is None:
# This needs to be imported in runtime because otherwise you'll
# get a circular import.
from crashstats.supersearch.models import SuperSearchFields
all = SuperSearchFields().get()
fields = []
if baseline:
if isinstance(baseline, tuple):
baseline = list(baseline)
fields.extend(baseline)
for meta in all.itervalues():
if (meta['namespace'] == namespace
and not meta['permissions_needed']
and meta['is_returned']):
if meta['in_database_name'] not in fields:
fields.append(meta['in_database_name'])
fields = tuple(fields)
# Cache for 1 hour.
cache.set(cache_key, fields, 60 * 60)
return fields
def get_from_es(namespace, baseline=None):
# @namespace is something like 'raw_crash' or 'processed_crash'
cache_key = 'api_supersearch_fields_%s' % namespace
fields = cache.get(cache_key)
if fields is None:
# This needs to be imported in runtime because otherwise you'll
# get a circular import.
from crashstats.supersearch.models import SuperSearchFields
all = SuperSearchFields().get()
fields = []
if baseline:
if isinstance(baseline, tuple):
baseline = list(baseline)
fields.extend(baseline)
for meta in all.itervalues():
if (
meta['namespace'] == namespace and
not meta['permissions_needed'] and
meta['is_returned']
):
if meta['in_database_name'] not in fields:
fields.append(meta['in_database_name'])
fields = tuple(fields)
# Cache for 1 hour.
cache.set(cache_key, fields, 60 * 60)
return fields
def signature_report(request, default_context=None):
context = default_context
params = get_validated_params(request)
if isinstance(params, http.HttpResponseBadRequest):
# There was an error in the form, let's return it.
return params
signature = request.GET.get('signature')
if not signature:
return http.HttpResponseBadRequest(
'"signature" parameter is mandatory')
context['signature'] = signature
fields = sorted(x['name'] for x in SuperSearchFields().get().values()
if x['is_exposed'] and x['is_returned']
and has_permissions(request.user, x['permissions_needed'])
and x['name'] != 'signature' # exclude the signature field
)
context['fields'] = [{
'id': field,
'text': field.replace('_', ' ')
} for field in fields]
context['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS
context['report_list_query_string'] = urllib.urlencode(
utils.sanitize_dict(get_report_list_parameters(params)), True)
return render(request, 'signature/signature_report.html', context)
def supersearch_api(request, default_context=None):
context = default_context or {}
all_fields = SuperSearchFields().get().values()
all_fields = [x for x in all_fields if x["is_returned"]]
all_fields = sorted(all_fields, key=lambda x: x["name"].lower())
aggs_fields = list(all_fields)
# Those fields are hard-coded in `supersearch/models.py`.
aggs_fields.append({"name": "product.version", "is_exposed": False})
aggs_fields.append(
{
"name": "android_cpu_abi.android_manufacturer.android_model",
"is_exposed": False,
}
)
date_number_fields = [
x for x in all_fields if x["query_type"] in ("number", "date")
]
context["all_fields"] = all_fields
context["aggs_fields"] = aggs_fields
context["date_number_fields"] = date_number_fields
context["operators"] = OPERATORS_MAP
return render(request, "documentation/supersearch/api.html", context)
def supersearch_api(request, default_context=None):
context = default_context or {}
all_fields = SuperSearchFields().get().values()
all_fields = [x for x in all_fields if x['is_returned']]
all_fields = sorted(all_fields, key=lambda x: x['name'].lower())
aggs_fields = list(all_fields)
# Those fields are hard-coded in `supersearch/models.py`.
aggs_fields.append({'name': 'product.version', 'is_exposed': False})
aggs_fields.append({
'name': 'android_cpu_abi.android_manufacturer.android_model',
'is_exposed': False
})
date_number_fields = [
x for x in all_fields if x['query_type'] in ('number', 'date')
]
context['all_fields'] = all_fields
context['aggs_fields'] = aggs_fields
context['date_number_fields'] = date_number_fields
context['operators'] = OPERATORS_MAP
return render(request, 'documentation/supersearch/api.html', context)
def supersearch_fields(request):
context = {}
sorted_fields = sorted(
SuperSearchFields().get().values(),
key=lambda x: x['name'].lower()
)
context['fields'] = sorted_fields
return render(request, 'manage/supersearch_fields.html', context)
def signature_summary(request, params):
"""Return a list of specific aggregations"""
context = {}
params["signature"] = "=" + params["signature"][0]
params["_aggs.signature"] = [
"hang_type",
"process_type",
"startup_crash",
"dom_fission_enabled",
"_histogram.uptime",
]
params["_results_number"] = 0
params["_facets"] = [
"platform_pretty_version",
"cpu_arch",
"process_type",
"flash_version",
]
params["_histogram.uptime"] = ["product"]
params["_histogram_interval.uptime"] = 60
params["_aggs.adapter_vendor_id"] = ["adapter_device_id"]
params["_aggs.android_cpu_abi.android_manufacturer.android_model"] = [
"android_version"
]
params["_aggs.product.version"] = ["_cardinality.install_time"]
# If the user has permissions, show exploitability.
all_fields = SuperSearchFields().get()
if request.user.has_perms(
all_fields["exploitability"]["permissions_needed"]):
params["_histogram.date"] = ["exploitability"]
api = SuperSearchUnredacted()
# Now make the actual request with all expected parameters.
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
facets = search_results["facets"]
_transform_uptime_summary(facets)
_transform_graphics_summary(facets)
_transform_mobile_summary(facets)
_transform_exploitability_summary(facets)
context["query"] = search_results
context["product_version_total"] = search_results["total"]
if "signature" in facets and len(facets["signature"]) > 0:
context["signature_stats"] = SignatureStats(
search_results["facets"]["signature"][0], search_results["total"])
return render(request, "signature/signature_summary.html", context)
def signature_summary(request, params):
'''Return a list of specific aggregations. '''
context = {}
params['signature'] = '=' + params['signature'][0]
params['_aggs.signature'] = [
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
]
params['_results_number'] = 0
params['_facets'] = [
'platform_pretty_version',
'cpu_name',
'process_type',
'flash_version',
]
params['_histogram.uptime'] = ['product']
params['_histogram_interval.uptime'] = 60
params['_aggs.adapter_vendor_id'] = ['adapter_device_id']
params['_aggs.android_cpu_abi.android_manufacturer.android_model'] = [
'android_version'
]
params['_aggs.product.version'] = ['_cardinality.install_time']
# If the user has permissions, show exploitability.
all_fields = SuperSearchFields().get()
if has_permissions(request.user,
all_fields['exploitability']['permissions_needed']):
params['_histogram.date'] = ['exploitability']
api = SuperSearchUnredacted()
# Now make the actual request with all expected parameters.
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
facets = search_results['facets']
_transform_uptime_summary(facets)
_transform_graphics_summary(facets)
_transform_mobile_summary(facets)
_transform_exploitability_summary(facets)
context['query'] = search_results
context['product_version_total'] = search_results['total']
if 'signature' in facets and len(facets['signature']) > 0:
context['signature_stats'] = SignatureStats(
search_results['facets']['signature'][0], search_results['total'])
return render(request, 'signature/signature_summary.html', context)
def get_from_es(namespace, baseline=None):
# @namespace is something like 'raw_crash' or 'processed_crash'
fields = cache.get("api_supersearch_fields_%s" % namespace)
if fields is None:
# This needs to be imported in runtime because otherwise you'll
# get a circular import.
from crashstats.supersearch.models import SuperSearchFields
all = SuperSearchFields().get()
fields = []
if baseline:
if isinstance(baseline, tuple):
baseline = list(baseline)
fields.extend(baseline)
for meta in all.itervalues():
if meta["namespace"] == namespace and not meta["permissions_needed"] and meta["is_returned"]:
if meta["in_database_name"] not in fields:
fields.append(meta["in_database_name"])
fields = tuple(fields)
return fields
def supersearch_field_update(request):
field_data = _get_supersearch_field_data(request.POST)
if isinstance(field_data, basestring):
return http.HttpResponseBadRequest(field_data)
api = SuperSearchField()
api.put(field_data)
# Refresh the cache for the fields service.
SuperSearchFields().get(refresh_cache=True)
return redirect(reverse('manage:supersearch_fields'))
def get_supersearch_form(request):
platforms = list(models.Platform.objects.values_list("name", flat=True))
products = [product.name for product in productlib.get_products()]
# FIXME(willkg): this hardcodes always getting Firefox versions which
# seems unhelpful
product_versions = utils.get_versions_for_product("Firefox")
all_fields = SuperSearchFields().get()
form = forms.SearchForm(all_fields, products, product_versions, platforms,
request.user, request.GET)
return form
def supersearch_field_delete(request):
field_name = request.GET.get('name')
if not field_name:
return http.HttpResponseBadRequest('A "name" is needed')
api = SuperSearchField()
api.delete(name=field_name)
# Refresh the cache for the fields service.
SuperSearchFields().get(refresh_cache=True)
url = reverse('manage:supersearch_fields')
return redirect(url)
def supersearch_field(request):
context = {}
field_name = request.GET.get('name')
if field_name:
all_fields = SuperSearchFields().get()
field_data = all_fields.get(field_name)
if not field_data:
return http.HttpResponseBadRequest(
'The field "%s" does not exist' % field_name
)
else:
field_data = {}
context['field'] = field_data
perms = Permission.objects.filter(content_type__model='').order_by('name')
context['all_permissions'] = [
'crashstats.' + x.codename for x in perms
]
return render(request, 'manage/supersearch_field.html', context)
def supersearch_field_update(request):
field_data = _get_supersearch_field_data(request.POST)
if isinstance(field_data, basestring):
return http.HttpResponseBadRequest(field_data)
api = SuperSearchField()
api.update_field(**field_data)
SuperSearch.clear_implementations_cache()
log(request.user, 'supersearch_field.put', field_data)
# Refresh the cache for the fields service.
SuperSearchFields().get(refresh_cache=True)
return redirect(reverse('manage:supersearch_fields'))
def supersearch_field_delete(request):
field_name = request.GET.get('name')
if not field_name:
return http.HttpResponseBadRequest('A "name" is needed')
api = SuperSearchField()
api.delete_field(name=field_name)
SuperSearch.clear_implementations_cache()
log(request.user, 'supersearch_field.delete', {'name': field_name})
# Refresh the cache for the fields service.
SuperSearchFields().get(refresh_cache=True)
url = reverse('manage:supersearch_fields')
return redirect(url)
def supersearch_field_create(request):
field_data = _get_supersearch_field_data(request.POST)
if isinstance(field_data, basestring):
return http.HttpResponseBadRequest(field_data)
api = SuperSearchField()
api.post(field_data)
log(request.user, 'supersearch_field.post', field_data)
# Refresh the cache for the fields service.
SuperSearchFields().get(refresh_cache=True)
# The API is using cache to get all fields by a specific namespace
# for the whitelist lookup, clear that cache too.
cache.delete('api_supersearch_fields_%s' % field_data['namespace'])
return redirect(reverse('manage:supersearch_fields'))
def signature_report(request, params, default_context=None):
context = default_context
signature = request.GET.get('signature')
if not signature:
return http.HttpResponseBadRequest(
'"signature" parameter is mandatory'
)
context['signature'] = signature
fields = sorted(
x['name']
for x in SuperSearchFields().get().values()
if x['is_exposed'] and
x['is_returned'] and
has_permissions(request.user, x['permissions_needed']) and
x['name'] != 'signature' # exclude the signature field
)
context['fields'] = [
{'id': field, 'text': field.replace('_', ' ')} for field in fields
]
columns = request.GET.getlist('_columns')
columns = [x for x in columns if x in fields]
context['columns'] = columns or DEFAULT_COLUMNS
sort = request.GET.getlist('_sort')
sort = [x for x in sort if x in fields]
context['sort'] = sort or DEFAULT_SORT
context['channels'] = ','.join(settings.CHANNELS).split(',')
context['channel'] = settings.CHANNEL
# Compute dates to show them to the user.
start_date, end_date = get_date_boundaries(params)
context['query'] = {
'start_date': start_date,
'end_date': end_date,
}
return render(request, 'signature/signature_report.html', context)
def setUp(self):
super().setUp()
# Tests assume and require a non-persistent cache backend
assert "LocMemCache" in settings.CACHES["default"]["BACKEND"]
def mocked_supersearchfields(**params):
results = copy.deepcopy(FIELDS)
# to be realistic we want to introduce some dupes that have a
# different key but its `in_database_name` is one that is already
# in the hardcoded list (the baseline)
results["accessibility2"] = results["accessibility"]
return results
supersearchfields_mock_get = mock.Mock()
supersearchfields_mock_get.side_effect = mocked_supersearchfields
SuperSearchFields.get = supersearchfields_mock_get
# This will make sure the cache is pre-populated
SuperSearchFields().get()
def signature_report(request, default_context=None):
context = default_context
signature = request.GET.get('signature')
if not signature:
return http.HttpResponseBadRequest(
'"signature" parameter is mandatory')
context['signature'] = signature
fields = sorted(x['name'] for x in SuperSearchFields().get().values()
if x['is_exposed'] and x['is_returned']
and has_permissions(request.user, x['permissions_needed'])
and x['name'] != 'signature' # exclude the signature field
)
context['fields'] = [{
'id': field,
'text': field.replace('_', ' ')
} for field in fields]
return render(request, 'signature/signature_report.html', context)
def signature_report(request, params, default_context=None):
context = default_context
signature = request.GET.get("signature")
if not signature:
return http.HttpResponseBadRequest(
'"signature" parameter is mandatory')
context["signature"] = signature
fields = sorted(x["name"] for x in SuperSearchFields().get().values()
if x["is_exposed"] and x["is_returned"]
and request.user.has_perms(x["permissions_needed"])
and x["name"] != "signature" # exclude the signature field
)
context["fields"] = [{
"id": field,
"text": field.replace("_", " ")
} for field in fields]
columns = request.GET.getlist("_columns")
columns = [x for x in columns if x in fields]
context["columns"] = columns or DEFAULT_COLUMNS
sort = request.GET.getlist("_sort")
sort = [x for x in sort if x in fields]
context["sort"] = sort or DEFAULT_SORT
context["channels"] = ",".join(settings.CHANNELS).split(",")
context["channel"] = settings.CHANNEL
context["correlations_products"] = CORRELATIONS_PRODUCTS
# Compute dates to show them to the user.
start_date, end_date = get_date_boundaries(params)
context["query"] = {"start_date": start_date, "end_date": end_date}
return render(request, "signature/signature_report.html", context)
def signature_report(request, params, default_context=None):
context = default_context
signature = request.GET.get('signature')
if not signature:
return http.HttpResponseBadRequest(
'"signature" parameter is mandatory'
)
context['signature'] = signature
fields = sorted(
x['name']
for x in SuperSearchFields().get().values()
if x['is_exposed'] and
x['is_returned'] and
has_permissions(request.user, x['permissions_needed']) and
x['name'] != 'signature' # exclude the signature field
)
context['fields'] = [
{'id': field, 'text': field.replace('_', ' ')} for field in fields
]
context['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS
context['channels'] = ','.join(settings.CHANNELS).split(',')
context['channel'] = settings.CHANNEL
context['report_list_query_string'] = urllib.urlencode(
utils.sanitize_dict(
get_report_list_parameters(params)
),
True
)
return render(request, 'signature/signature_report.html', context)
def get_allowed_fields(user):
return tuple(
x["name"]
for x in SuperSearchFields().get().values()
if x["is_exposed"] and user.has_perms(x["permissions_needed"])
)
def report_index(request, crash_id, default_context=None):
valid_crash_id = utils.find_crash_id(crash_id)
if not valid_crash_id:
return http.HttpResponseBadRequest("Invalid crash ID")
# Sometimes, in Socorro we use a prefix on the crash ID. Usually it's
# 'bp-' but this is configurable.
# If you try to use this to reach the perma link for a crash, it should
# redirect to the report index with the correct crash ID.
if valid_crash_id != crash_id:
return redirect(
reverse("crashstats:report_index", args=(valid_crash_id, )))
context = default_context or {}
context["crash_id"] = crash_id
refresh_cache = request.GET.get("refresh") == "cache"
raw_api = models.RawCrash()
try:
context["raw"] = raw_api.get(crash_id=crash_id,
refresh_cache=refresh_cache)
except CrashIDNotFound:
# If the raw crash can't be found, we can't do much.
return render(request,
"crashstats/report_index_not_found.html",
context,
status=404)
utils.enhance_raw(context["raw"])
context["your_crash"] = (request.user.is_active and
context["raw"].get("Email") == request.user.email)
api = models.UnredactedCrash()
try:
context["report"] = api.get(crash_id=crash_id,
refresh_cache=refresh_cache)
except CrashIDNotFound:
# ...if we haven't already done so.
cache_key = "priority_job:{}".format(crash_id)
if not cache.get(cache_key):
priority_api = models.PriorityJob()
priority_api.post(crash_ids=[crash_id])
cache.set(cache_key, True, 60)
return render(request, "crashstats/report_index_pending.html", context)
if "json_dump" in context["report"]:
json_dump = context["report"]["json_dump"]
if "sensitive" in json_dump and not request.user.has_perm(
"crashstats.view_pii"):
del json_dump["sensitive"]
context["raw_stackwalker_output"] = json.dumps(json_dump,
sort_keys=True,
indent=4,
separators=(",", ": "))
utils.enhance_json_dump(json_dump, settings.VCS_MAPPINGS)
parsed_dump = json_dump
else:
context["raw_stackwalker_output"] = "No dump available"
parsed_dump = {}
# NOTE(willkg): pull cpu count from parsed_dump if it's not in report;
# remove in July 2019
if "cpu_count" not in context["report"]:
context["report"]["cpu_count"] = parsed_dump.get("system_info",
{}).get("cpu_count")
# NOTE(willkg): "cpu_name" is deprecated, but populate "cpu_arch" if
# cpu_arch is empty; remove in July 2019.
if "cpu_arch" not in context["report"]:
context["report"]["cpu_arch"] = context["report"]["cpu_name"]
context["crashing_thread"] = parsed_dump.get("crash_info",
{}).get("crashing_thread")
if context["report"]["signature"].startswith("shutdownhang"):
# For shutdownhang signatures, we want to use thread 0 as the
# crashing thread, because that's the thread that actually contains
# the useful data about what happened.
context["crashing_thread"] = 0
context["parsed_dump"] = parsed_dump
context["bug_product_map"] = settings.BUG_PRODUCT_MAP
context["bug_associations"] = list(
models.BugAssociation.objects.filter(
signature=context["report"]["signature"]).values(
"bug_id", "signature").order_by("-bug_id"))
context["raw_keys"] = []
if request.user.has_perm("crashstats.view_pii"):
# hold nothing back
context["raw_keys"] = context["raw"].keys()
else:
context["raw_keys"] = [
x for x in context["raw"] if x in models.RawCrash.API_ALLOWLIST()
]
# Sort keys case-insensitively
context["raw_keys"] = sorted(context["raw_keys"], key=lambda s: s.lower())
if request.user.has_perm("crashstats.view_rawdump"):
context["raw_dump_urls"] = [
reverse("crashstats:raw_data", args=(crash_id, "dmp")),
reverse("crashstats:raw_data", args=(crash_id, "json")),
]
if context["raw"].get("additional_minidumps"):
suffixes = [
x.strip()
for x in context["raw"]["additional_minidumps"].split(",")
if x.strip()
]
for suffix in suffixes:
name = "upload_file_minidump_%s" % (suffix, )
context["raw_dump_urls"].append(
reverse("crashstats:raw_data_named",
args=(crash_id, name, "dmp")))
if (context["raw"].get("ContainsMemoryReport")
and context["report"].get("memory_report")
and not context["report"].get("memory_report_error")):
context["raw_dump_urls"].append(
reverse(
"crashstats:raw_data_named",
args=(crash_id, "memory_report", "json.gz"),
))
# Add descriptions to all fields.
all_fields = SuperSearchFields().get()
descriptions = {}
for field in all_fields.values():
key = "{}.{}".format(field["namespace"], field["in_database_name"])
descriptions[key] = "{} Search: {}".format(
field.get("description", "").strip()
or "No description for this field.",
field["is_exposed"] and field["name"] or "N/A",
)
def make_raw_crash_key(key):
"""In the report_index.html template we need to create a key
that we can use to look up against the 'fields_desc' dict.
Because you can't do something like this in jinja::
{{ fields_desc.get(u'raw_crash.{}'.format(key), empty_desc) }}
we do it here in the function instead.
The trick is that the lookup key has to be a unicode object or
else you get UnicodeEncodeErrors in the template rendering.
"""
return "raw_crash.{}".format(key)
context["make_raw_crash_key"] = make_raw_crash_key
context["fields_desc"] = descriptions
context["empty_desc"] = "No description for this field. Search: unknown"
context["BUG_PRODUCT_MAP"] = settings.BUG_PRODUCT_MAP
# report.addons used to be a list of lists.
# In https://bugzilla.mozilla.org/show_bug.cgi?id=1250132
# we changed it from a list of lists to a list of strings, using
# a ':' to split the name and version.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=1250132#c7
# Considering legacy, let's tackle both.
# In late 2017, this code is going to be useless and can be removed.
if context["report"].get("addons") and isinstance(
context["report"]["addons"][0], (list, tuple)):
# This is the old legacy format. This crash hasn't been processed
# the new way.
context["report"]["addons"] = [
":".join(x) for x in context["report"]["addons"]
]
content = loader.render_to_string("crashstats/report_index.html", context,
request)
utf8_content = content.encode("utf-8", errors="backslashreplace")
return HttpResponse(utf8_content, charset="utf-8")
def report_index(request, crash_id, default_context=None):
valid_crash_id = utils.find_crash_id(crash_id)
if not valid_crash_id:
return http.HttpResponseBadRequest('Invalid crash ID')
# Sometimes, in Socorro we use a prefix on the crash ID. Usually it's
# 'bp-' but this is configurable.
# If you try to use this to reach the perma link for a crash, it should
# redirect to the report index with the correct crash ID.
if valid_crash_id != crash_id:
return redirect(reverse(
'crashstats:report_index',
args=(valid_crash_id,)
))
context = default_context or {}
context['crash_id'] = crash_id
raw_api = models.RawCrash()
try:
context['raw'] = raw_api.get(crash_id=crash_id)
except CrashIDNotFound:
# If the raw crash can't be found, we can't do much.
tmpl = 'crashstats/report_index_not_found.html'
return render(request, tmpl, context, status=404)
context['your_crash'] = (
request.user.is_active and
context['raw'].get('Email') == request.user.email
)
api = models.UnredactedCrash()
try:
context['report'] = api.get(crash_id=crash_id)
except CrashIDNotFound:
# ...if we haven't already done so.
cache_key = 'priority_job:{}'.format(crash_id)
if not cache.get(cache_key):
priority_api = models.Priorityjob()
priority_api.post(crash_ids=[crash_id])
cache.set(cache_key, True, 60)
tmpl = 'crashstats/report_index_pending.html'
return render(request, tmpl, context)
if 'json_dump' in context['report']:
json_dump = context['report']['json_dump']
if 'sensitive' in json_dump and \
not request.user.has_perm('crashstats.view_pii'):
del json_dump['sensitive']
context['raw_stackwalker_output'] = json.dumps(
json_dump,
sort_keys=True,
indent=4,
separators=(',', ': ')
)
utils.enhance_json_dump(json_dump, settings.VCS_MAPPINGS)
parsed_dump = json_dump
elif 'dump' in context['report']:
context['raw_stackwalker_output'] = context['report']['dump']
parsed_dump = utils.parse_dump(
context['report']['dump'],
settings.VCS_MAPPINGS
)
else:
context['raw_stackwalker_output'] = 'No dump available'
parsed_dump = {}
# If the parsed_dump lacks a `parsed_dump.crash_info.crashing_thread`
# we can't loop over the frames :(
crashing_thread = parsed_dump.get('crash_info', {}).get('crashing_thread')
if crashing_thread is None:
# the template does a big `{% if parsed_dump.threads %}`
parsed_dump['threads'] = None
else:
context['crashing_thread'] = crashing_thread
if context['report']['signature'].startswith('shutdownhang'):
# For shutdownhang signatures, we want to use thread 0 as the
# crashing thread, because that's the thread that actually contains
# the usefull data about the what happened.
context['crashing_thread'] = 0
context['parsed_dump'] = parsed_dump
context['bug_product_map'] = settings.BUG_PRODUCT_MAP
process_type = 'unknown'
if context['report']['process_type'] is None:
process_type = 'browser'
elif context['report']['process_type'] == 'plugin':
process_type = 'plugin'
elif context['report']['process_type'] == 'content':
process_type = 'content'
context['process_type'] = process_type
bugs_api = models.Bugs()
hits = bugs_api.get(signatures=[context['report']['signature']])['hits']
# bugs_api.get(signatures=...) will return all signatures associated
# with the bugs found, but we only want those with matching signature
context['bug_associations'] = [
x for x in hits
if x['signature'] == context['report']['signature']
]
context['bug_associations'].sort(
key=lambda x: x['id'],
reverse=True
)
context['raw_keys'] = []
if request.user.has_perm('crashstats.view_pii'):
# hold nothing back
context['raw_keys'] = context['raw'].keys()
else:
context['raw_keys'] = [
x for x in context['raw']
if x in models.RawCrash.API_WHITELIST
]
# Sort keys case-insensitively
context['raw_keys'].sort(key=lambda s: s.lower())
if request.user.has_perm('crashstats.view_rawdump'):
context['raw_dump_urls'] = [
reverse('crashstats:raw_data', args=(crash_id, 'dmp')),
reverse('crashstats:raw_data', args=(crash_id, 'json'))
]
if context['raw'].get('additional_minidumps'):
suffixes = [
x.strip()
for x in context['raw']['additional_minidumps'].split(',')
if x.strip()
]
for suffix in suffixes:
name = 'upload_file_minidump_%s' % (suffix,)
context['raw_dump_urls'].append(
reverse(
'crashstats:raw_data_named',
args=(crash_id, name, 'dmp')
)
)
if (
context['raw'].get('ContainsMemoryReport') and
context['report'].get('memory_report') and
not context['report'].get('memory_report_error')
):
context['raw_dump_urls'].append(
reverse(
'crashstats:raw_data_named',
args=(crash_id, 'memory_report', 'json.gz')
)
)
# Add descriptions to all fields.
all_fields = SuperSearchFields().get()
descriptions = {}
for field in all_fields.values():
key = '{}.{}'.format(field['namespace'], field['in_database_name'])
descriptions[key] = '{} Search: {}'.format(
field.get('description', '').strip() or
'No description for this field.',
field['is_exposed'] and field['name'] or 'N/A',
)
context['fields_desc'] = descriptions
context['empty_desc'] = 'No description for this field. Search: unknown'
context['BUG_PRODUCT_MAP'] = settings.BUG_PRODUCT_MAP
return render(request, 'crashstats/report_index.html', context)
def signature_summary(request, params):
'''Return a list of specific aggregations. '''
data = {}
params['signature'] = '=' + params['signature'][0]
params['_results_number'] = 0
params['_facets'] = [
'platform_pretty_version',
'cpu_name',
'process_type',
'flash_version',
]
params['_histogram.uptime'] = ['product']
params['_histogram_interval.uptime'] = 60
params['_aggs.adapter_vendor_id'] = ['adapter_device_id']
params['_aggs.android_cpu_abi.android_manufacturer.android_model'] = [
'android_version'
]
# If the user has permissions, show exploitability.
all_fields = SuperSearchFields().get()
if has_permissions(
request.user, all_fields['exploitability']['permissions_needed']
):
params['_histogram.date'] = ['exploitability']
api = SuperSearchUnredacted()
# Now make the actual request with all expected parameters.
try:
search_results = api.get(**params)
except models.BadStatusCodeError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
facets = search_results['facets']
# We need to make a separate query so that we can show all versions and
# not just the one asked for.
params_copy = {
'signature': params['signature'],
'_aggs.product.version': ['_cardinality.install_time'],
}
try:
product_results = api.get(**params_copy)
except models.BadStatusCodeError as e:
# We need to return the error message in some HTML form for jQuery
# to pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
if 'product' in product_results['facets']:
facets['product'] = product_results['facets']['product']
else:
facets['product'] = []
data['product_version_total'] = product_results['total']
_transform_uptime_summary(facets)
_transform_graphics_summary(facets)
_transform_mobile_summary(facets)
_transform_exploitability_summary(facets)
data['query'] = search_results
return render(request, 'signature/signature_summary.html', data)
def report_index(request, crash_id, default_context=None):
valid_crash_id = utils.find_crash_id(crash_id)
if not valid_crash_id:
return http.HttpResponseBadRequest('Invalid crash ID')
# Sometimes, in Socorro we use a prefix on the crash ID. Usually it's
# 'bp-' but this is configurable.
# If you try to use this to reach the perma link for a crash, it should
# redirect to the report index with the correct crash ID.
if valid_crash_id != crash_id:
return redirect(
reverse('crashstats:report_index', args=(valid_crash_id, )))
context = default_context or {}
context['crash_id'] = crash_id
raw_api = models.RawCrash()
try:
context['raw'] = raw_api.get(crash_id=crash_id)
except CrashIDNotFound:
# If the raw crash can't be found, we can't do much.
tmpl = 'crashstats/report_index_not_found.html'
return render(request, tmpl, context, status=404)
context['your_crash'] = (request.user.is_active and
context['raw'].get('Email') == request.user.email)
api = models.UnredactedCrash()
try:
context['report'] = api.get(crash_id=crash_id)
except CrashIDNotFound:
# ...if we haven't already done so.
cache_key = 'priority_job:{}'.format(crash_id)
if not cache.get(cache_key):
priority_api = models.Priorityjob()
priority_api.post(crash_ids=[crash_id])
cache.set(cache_key, True, 60)
tmpl = 'crashstats/report_index_pending.html'
return render(request, tmpl, context)
if 'json_dump' in context['report']:
json_dump = context['report']['json_dump']
if 'sensitive' in json_dump and \
not request.user.has_perm('crashstats.view_pii'):
del json_dump['sensitive']
context['raw_stackwalker_output'] = json.dumps(json_dump,
sort_keys=True,
indent=4,
separators=(',', ': '))
utils.enhance_json_dump(json_dump, settings.VCS_MAPPINGS)
parsed_dump = json_dump
elif 'dump' in context['report']:
context['raw_stackwalker_output'] = context['report']['dump']
parsed_dump = utils.parse_dump(context['report']['dump'],
settings.VCS_MAPPINGS)
else:
context['raw_stackwalker_output'] = 'No dump available'
parsed_dump = {}
# If the parsed_dump lacks a `parsed_dump.crash_info.crashing_thread`
# we can't loop over the frames :(
crashing_thread = parsed_dump.get('crash_info', {}).get('crashing_thread')
if crashing_thread is None:
# the template does a big `{% if parsed_dump.threads %}`
parsed_dump['threads'] = None
else:
context['crashing_thread'] = crashing_thread
if context['report']['signature'].startswith('shutdownhang'):
# For shutdownhang signatures, we want to use thread 0 as the
# crashing thread, because that's the thread that actually contains
# the usefull data about the what happened.
context['crashing_thread'] = 0
context['parsed_dump'] = parsed_dump
context['bug_product_map'] = settings.BUG_PRODUCT_MAP
process_type = 'unknown'
if context['report']['process_type'] is None:
process_type = 'browser'
elif context['report']['process_type'] == 'plugin':
process_type = 'plugin'
elif context['report']['process_type'] == 'content':
process_type = 'content'
context['process_type'] = process_type
bugs_api = models.Bugs()
hits = bugs_api.get(signatures=[context['report']['signature']])['hits']
# bugs_api.get(signatures=...) will return all signatures associated
# with the bugs found, but we only want those with matching signature
context['bug_associations'] = [
x for x in hits if x['signature'] == context['report']['signature']
]
context['bug_associations'].sort(key=lambda x: x['id'], reverse=True)
context['raw_keys'] = []
if request.user.has_perm('crashstats.view_pii'):
# hold nothing back
context['raw_keys'] = context['raw'].keys()
else:
context['raw_keys'] = [
x for x in context['raw'] if x in models.RawCrash.API_WHITELIST
]
# Sort keys case-insensitively
context['raw_keys'].sort(key=lambda s: s.lower())
if request.user.has_perm('crashstats.view_rawdump'):
context['raw_dump_urls'] = [
reverse('crashstats:raw_data', args=(crash_id, 'dmp')),
reverse('crashstats:raw_data', args=(crash_id, 'json'))
]
if context['raw'].get('additional_minidumps'):
suffixes = [
x.strip()
for x in context['raw']['additional_minidumps'].split(',')
if x.strip()
]
for suffix in suffixes:
name = 'upload_file_minidump_%s' % (suffix, )
context['raw_dump_urls'].append(
reverse('crashstats:raw_data_named',
args=(crash_id, name, 'dmp')))
if (context['raw'].get('ContainsMemoryReport')
and context['report'].get('memory_report')
and not context['report'].get('memory_report_error')):
context['raw_dump_urls'].append(
reverse('crashstats:raw_data_named',
args=(crash_id, 'memory_report', 'json.gz')))
# Add descriptions to all fields.
all_fields = SuperSearchFields().get()
descriptions = {}
for field in all_fields.values():
key = '{}.{}'.format(field['namespace'], field['in_database_name'])
descriptions[key] = '{} Search: {}'.format(
field.get('description', '').strip()
or 'No description for this field.',
field['is_exposed'] and field['name'] or 'N/A',
)
context['fields_desc'] = descriptions
context['empty_desc'] = 'No description for this field. Search: unknown'
context['BUG_PRODUCT_MAP'] = settings.BUG_PRODUCT_MAP
return render(request, 'crashstats/report_index.html', context)
def report_index(request, crash_id, default_context=None):
valid_crash_id = utils.find_crash_id(crash_id)
if not valid_crash_id:
return http.HttpResponseBadRequest('Invalid crash ID')
# Sometimes, in Socorro we use a prefix on the crash ID. Usually it's
# 'bp-' but this is configurable.
# If you try to use this to reach the perma link for a crash, it should
# redirect to the report index with the correct crash ID.
if valid_crash_id != crash_id:
return redirect(
reverse('crashstats:report_index', args=(valid_crash_id, )))
context = default_context or {}
context['crash_id'] = crash_id
refresh_cache = request.GET.get('refresh') == 'cache'
raw_api = models.RawCrash()
try:
context['raw'] = raw_api.get(
crash_id=crash_id,
refresh_cache=refresh_cache,
)
except CrashIDNotFound:
# If the raw crash can't be found, we can't do much.
tmpl = 'crashstats/report_index_not_found.html'
return render(request, tmpl, context, status=404)
context['your_crash'] = (request.user.is_active and
context['raw'].get('Email') == request.user.email)
api = models.UnredactedCrash()
try:
context['report'] = api.get(
crash_id=crash_id,
refresh_cache=refresh_cache,
)
except CrashIDNotFound:
# ...if we haven't already done so.
cache_key = 'priority_job:{}'.format(crash_id)
if not cache.get(cache_key):
priority_api = models.Priorityjob()
priority_api.post(crash_ids=[crash_id])
cache.set(cache_key, True, 60)
tmpl = 'crashstats/report_index_pending.html'
return render(request, tmpl, context)
if 'json_dump' in context['report']:
json_dump = context['report']['json_dump']
if 'sensitive' in json_dump and \
not request.user.has_perm('crashstats.view_pii'):
del json_dump['sensitive']
context['raw_stackwalker_output'] = json.dumps(json_dump,
sort_keys=True,
indent=4,
separators=(',', ': '))
utils.enhance_json_dump(json_dump, settings.VCS_MAPPINGS)
parsed_dump = json_dump
elif 'dump' in context['report']:
context['raw_stackwalker_output'] = context['report']['dump']
parsed_dump = utils.parse_dump(context['report']['dump'],
settings.VCS_MAPPINGS)
else:
context['raw_stackwalker_output'] = 'No dump available'
parsed_dump = {}
context['crashing_thread'] = parsed_dump.get('crash_info',
{}).get('crashing_thread')
if context['report']['signature'].startswith('shutdownhang'):
# For shutdownhang signatures, we want to use thread 0 as the
# crashing thread, because that's the thread that actually contains
# the usefull data about the what happened.
context['crashing_thread'] = 0
context['parsed_dump'] = parsed_dump
context['bug_product_map'] = settings.BUG_PRODUCT_MAP
process_type = 'unknown'
if context['report']['process_type'] is None:
process_type = 'browser'
elif context['report']['process_type'] == 'plugin':
process_type = 'plugin'
elif context['report']['process_type'] == 'content':
process_type = 'content'
context['process_type'] = process_type
bugs_api = models.Bugs()
hits = bugs_api.get(signatures=[context['report']['signature']])['hits']
# bugs_api.get(signatures=...) will return all signatures associated
# with the bugs found, but we only want those with matching signature
context['bug_associations'] = [
x for x in hits if x['signature'] == context['report']['signature']
]
context['bug_associations'].sort(key=lambda x: x['id'], reverse=True)
context['raw_keys'] = []
if request.user.has_perm('crashstats.view_pii'):
# hold nothing back
context['raw_keys'] = context['raw'].keys()
else:
context['raw_keys'] = [
x for x in context['raw'] if x in models.RawCrash.API_WHITELIST()
]
# Sort keys case-insensitively
context['raw_keys'].sort(key=lambda s: s.lower())
if request.user.has_perm('crashstats.view_rawdump'):
context['raw_dump_urls'] = [
reverse('crashstats:raw_data', args=(crash_id, 'dmp')),
reverse('crashstats:raw_data', args=(crash_id, 'json'))
]
if context['raw'].get('additional_minidumps'):
suffixes = [
x.strip()
for x in context['raw']['additional_minidumps'].split(',')
if x.strip()
]
for suffix in suffixes:
name = 'upload_file_minidump_%s' % (suffix, )
context['raw_dump_urls'].append(
reverse('crashstats:raw_data_named',
args=(crash_id, name, 'dmp')))
if (context['raw'].get('ContainsMemoryReport')
and context['report'].get('memory_report')
and not context['report'].get('memory_report_error')):
context['raw_dump_urls'].append(
reverse('crashstats:raw_data_named',
args=(crash_id, 'memory_report', 'json.gz')))
# Add descriptions to all fields.
all_fields = SuperSearchFields().get()
descriptions = {}
for field in all_fields.values():
key = '{}.{}'.format(field['namespace'], field['in_database_name'])
descriptions[key] = '{} Search: {}'.format(
field.get('description', '').strip()
or 'No description for this field.',
field['is_exposed'] and field['name'] or 'N/A',
)
def make_raw_crash_key(key):
"""In the report_index.html template we need to create a key
that we can use to look up against the 'fields_desc' dict.
Because you can't do something like this in jinja::
{{ fields_desc.get(u'raw_crash.{}'.format(key), empty_desc) }}
we do it here in the function instead.
The trick is that the lookup key has to be a unicode object or
else you get UnicodeEncodeErrors in the template rendering.
"""
return u'raw_crash.{}'.format(key)
context['make_raw_crash_key'] = make_raw_crash_key
context['fields_desc'] = descriptions
context['empty_desc'] = 'No description for this field. Search: unknown'
context['BUG_PRODUCT_MAP'] = settings.BUG_PRODUCT_MAP
# report.addons used to be a list of lists.
# In https://bugzilla.mozilla.org/show_bug.cgi?id=1250132
# we changed it from a list of lists to a list of strings, using
# a ':' to split the name and version.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=1250132#c7
# Considering legacy, let's tackle both.
# In late 2017, this code is going to be useless and can be removed.
if (context['report'].get('addons')
and isinstance(context['report']['addons'][0], (list, tuple))):
# This is the old legacy format. This crash hasn't been processed
# the new way.
context['report']['addons'] = [
':'.join(x) for x in context['report']['addons']
]
return render(request, 'crashstats/report_index.html', context)
def report_index(request, crash_id, default_context=None):
valid_crash_id = utils.find_crash_id(crash_id)
if not valid_crash_id:
return http.HttpResponseBadRequest('Invalid crash ID')
# Sometimes, in Socorro we use a prefix on the crash ID. Usually it's
# 'bp-' but this is configurable.
# If you try to use this to reach the perma link for a crash, it should
# redirect to the report index with the correct crash ID.
if valid_crash_id != crash_id:
return redirect(reverse('crashstats:report_index', args=(valid_crash_id,)))
context = default_context or {}
context['crash_id'] = crash_id
refresh_cache = request.GET.get('refresh') == 'cache'
raw_api = models.RawCrash()
try:
context['raw'] = raw_api.get(crash_id=crash_id, refresh_cache=refresh_cache)
except CrashIDNotFound:
# If the raw crash can't be found, we can't do much.
return render(request, 'crashstats/report_index_not_found.html', context, status=404)
utils.enhance_raw(context['raw'])
context['your_crash'] = (
request.user.is_active and
context['raw'].get('Email') == request.user.email
)
api = models.UnredactedCrash()
try:
context['report'] = api.get(crash_id=crash_id, refresh_cache=refresh_cache)
except CrashIDNotFound:
# ...if we haven't already done so.
cache_key = 'priority_job:{}'.format(crash_id)
if not cache.get(cache_key):
priority_api = models.PriorityJob()
priority_api.post(crash_ids=[crash_id])
cache.set(cache_key, True, 60)
return render(request, 'crashstats/report_index_pending.html', context)
if 'json_dump' in context['report']:
json_dump = context['report']['json_dump']
if 'sensitive' in json_dump and not request.user.has_perm('crashstats.view_pii'):
del json_dump['sensitive']
context['raw_stackwalker_output'] = json.dumps(
json_dump,
sort_keys=True,
indent=4,
separators=(',', ': ')
)
utils.enhance_json_dump(json_dump, settings.VCS_MAPPINGS)
parsed_dump = json_dump
else:
context['raw_stackwalker_output'] = 'No dump available'
parsed_dump = {}
# NOTE(willkg): pull cpu count from parsed_dump if it's not in report;
# remove in July 2019
if 'cpu_count' not in context['report']:
context['report']['cpu_count'] = parsed_dump.get('system_info', {}).get('cpu_count')
# NOTE(willkg): "cpu_name" is deprecated, but populate "cpu_arch" if
# cpu_arch is empty; remove in July 2019.
if 'cpu_arch' not in context['report']:
context['report']['cpu_arch'] = context['report']['cpu_name']
context['crashing_thread'] = parsed_dump.get('crash_info', {}).get('crashing_thread')
if context['report']['signature'].startswith('shutdownhang'):
# For shutdownhang signatures, we want to use thread 0 as the
# crashing thread, because that's the thread that actually contains
# the useful data about what happened.
context['crashing_thread'] = 0
context['parsed_dump'] = parsed_dump
context['bug_product_map'] = settings.BUG_PRODUCT_MAP
context['bug_associations'] = list(
models.BugAssociation.objects
.filter(signature=context['report']['signature'])
.values('bug_id', 'signature')
.order_by('-bug_id')
)
context['raw_keys'] = []
if request.user.has_perm('crashstats.view_pii'):
# hold nothing back
context['raw_keys'] = context['raw'].keys()
else:
context['raw_keys'] = [
x for x in context['raw']
if x in models.RawCrash.API_ALLOWLIST()
]
# Sort keys case-insensitively
context['raw_keys'] = sorted(context['raw_keys'], key=lambda s: s.lower())
if request.user.has_perm('crashstats.view_rawdump'):
context['raw_dump_urls'] = [
reverse('crashstats:raw_data', args=(crash_id, 'dmp')),
reverse('crashstats:raw_data', args=(crash_id, 'json'))
]
if context['raw'].get('additional_minidumps'):
suffixes = [
x.strip()
for x in context['raw']['additional_minidumps'].split(',')
if x.strip()
]
for suffix in suffixes:
name = 'upload_file_minidump_%s' % (suffix,)
context['raw_dump_urls'].append(
reverse('crashstats:raw_data_named', args=(crash_id, name, 'dmp'))
)
if (
context['raw'].get('ContainsMemoryReport') and
context['report'].get('memory_report') and
not context['report'].get('memory_report_error')
):
context['raw_dump_urls'].append(
reverse('crashstats:raw_data_named', args=(crash_id, 'memory_report', 'json.gz'))
)
# Add descriptions to all fields.
all_fields = SuperSearchFields().get()
descriptions = {}
for field in all_fields.values():
key = '{}.{}'.format(field['namespace'], field['in_database_name'])
descriptions[key] = '{} Search: {}'.format(
field.get('description', '').strip() or 'No description for this field.',
field['is_exposed'] and field['name'] or 'N/A',
)
def make_raw_crash_key(key):
"""In the report_index.html template we need to create a key
that we can use to look up against the 'fields_desc' dict.
Because you can't do something like this in jinja::
{{ fields_desc.get(u'raw_crash.{}'.format(key), empty_desc) }}
we do it here in the function instead.
The trick is that the lookup key has to be a unicode object or
else you get UnicodeEncodeErrors in the template rendering.
"""
return u'raw_crash.{}'.format(key)
context['make_raw_crash_key'] = make_raw_crash_key
context['fields_desc'] = descriptions
context['empty_desc'] = 'No description for this field. Search: unknown'
context['BUG_PRODUCT_MAP'] = settings.BUG_PRODUCT_MAP
# report.addons used to be a list of lists.
# In https://bugzilla.mozilla.org/show_bug.cgi?id=1250132
# we changed it from a list of lists to a list of strings, using
# a ':' to split the name and version.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=1250132#c7
# Considering legacy, let's tackle both.
# In late 2017, this code is going to be useless and can be removed.
if (
context['report'].get('addons') and
isinstance(context['report']['addons'][0], (list, tuple))
):
# This is the old legacy format. This crash hasn't been processed
# the new way.
context['report']['addons'] = [
':'.join(x) for x in context['report']['addons']
]
return render(request, 'crashstats/report_index.html', context)
def get_allowed_fields(user):
return tuple(
x['name']
for x in SuperSearchFields().get().values()
if x['is_exposed'] and has_permissions(user, x['permissions_needed'])
)
