def raw_data(request, crash_id, extension, name=None):
api = models.RawCrash()
if extension == "json":
format = "meta"
content_type = "application/json"
elif extension == "dmp":
format = "raw"
content_type = "application/octet-stream"
elif extension == "json.gz" and name == "memory_report":
# Note, if the name is 'memory_report' it will fetch a raw
# crash with name and the files in the memory_report bucket
# are already gzipped.
# This is important because it means we don't need to gzip
# the HttpResponse below.
format = "raw"
content_type = "application/octet-stream"
else:
raise NotImplementedError(extension)
data = api.get(crash_id=crash_id, format=format, name=name)
response = http.HttpResponse(content_type=content_type)
if extension == "json":
response.write(json.dumps(data))
else:
response.write(data)
return response
def crash_verify(request):
"""Verifies crash data in crash data destinations"""
crash_id = request.GET.get('crash_id', None)
if not crash_id or not is_crash_id_valid(crash_id):
return http.JsonResponse({'error': 'unknown crash id'}, status=400)
data = {'uuid': crash_id}
# Check S3 crash bucket for raw and processed crash data
raw_api = models.RawCrash()
try:
raw_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True)
has_raw_crash = True
except CrashIDNotFound:
has_raw_crash = False
data['s3_raw_crash'] = has_raw_crash
processed_api = models.ProcessedCrash()
try:
processed_api.get(crash_id=crash_id,
dont_cache=True,
refresh_cache=True)
has_processed_crash = True
except CrashIDNotFound:
has_processed_crash = False
data['s3_processed_crash'] = has_processed_crash
# Check Elasticsearch for crash data
supersearch_api = supersearch_models.SuperSearch()
params = {
'_columns': ['uuid'],
'_results_number': 1,
'uuid': crash_id,
'dont_cache': True,
'refresh_cache': True
}
results = supersearch_api.get(**params)
data['elasticsearch_crash'] = (results['total'] == 1
and results['hits'][0]['uuid'] == crash_id)
# Check S3 telemetry bucket for crash data
telemetry_api = models.TelemetryCrash()
try:
telemetry_api.get(crash_id=crash_id,
dont_cache=True,
refresh_cache=True)
has_telemetry_crash = True
except CrashIDNotFound:
has_telemetry_crash = False
data['s3_telemetry_crash'] = has_telemetry_crash
# NOTE(willkg): This doesn't check postgres because that's being phased
# out.
return http.HttpResponse(json.dumps(data), content_type='application/json')
def crash_verify(request):
"""Verifies crash data in crash data destinations"""
crash_id = request.GET.get("crash_id", None)
if not crash_id or not is_crash_id_valid(crash_id):
return http.JsonResponse({"error": "unknown crash id"}, status=400)
data = {"uuid": crash_id}
# Check S3 crash bucket for raw and processed crash data
raw_api = models.RawCrash()
try:
raw_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True)
has_raw_crash = True
except CrashIDNotFound:
has_raw_crash = False
data["s3_raw_crash"] = has_raw_crash
processed_api = models.ProcessedCrash()
try:
processed_api.get(crash_id=crash_id,
dont_cache=True,
refresh_cache=True)
has_processed_crash = True
except CrashIDNotFound:
has_processed_crash = False
data["s3_processed_crash"] = has_processed_crash
# Check Elasticsearch for crash data
supersearch_api = supersearch_models.SuperSearch()
params = {
"_columns": ["uuid"],
"_results_number": 1,
"uuid": crash_id,
"dont_cache": True,
"refresh_cache": True,
}
results = supersearch_api.get(**params)
data["elasticsearch_crash"] = (results["total"] == 1
and results["hits"][0]["uuid"] == crash_id)
# Check S3 telemetry bucket for crash data
telemetry_api = models.TelemetryCrash()
try:
telemetry_api.get(crash_id=crash_id,
dont_cache=True,
refresh_cache=True)
has_telemetry_crash = True
except CrashIDNotFound:
has_telemetry_crash = False
data["s3_telemetry_crash"] = has_telemetry_crash
return http.HttpResponse(json.dumps(data), content_type="application/json")
def report_index(request, crash_id, default_context=None):
valid_crash_id = utils.find_crash_id(crash_id)
if not valid_crash_id:
return http.HttpResponseBadRequest("Invalid crash ID")
# Sometimes, in Socorro we use a prefix on the crash ID. Usually it's
# 'bp-' but this is configurable.
# If you try to use this to reach the perma link for a crash, it should
# redirect to the report index with the correct crash ID.
if valid_crash_id != crash_id:
return redirect(
reverse("crashstats:report_index", args=(valid_crash_id, )))
context = default_context or {}
context["crash_id"] = crash_id
refresh_cache = request.GET.get("refresh") == "cache"
raw_api = models.RawCrash()
try:
context["raw"] = raw_api.get(crash_id=crash_id,
refresh_cache=refresh_cache)
except CrashIDNotFound:
# If the raw crash can't be found, we can't do much.
return render(request,
"crashstats/report_index_not_found.html",
context,
status=404)
utils.enhance_raw(context["raw"])
context["your_crash"] = (request.user.is_active and
context["raw"].get("Email") == request.user.email)
api = models.UnredactedCrash()
try:
context["report"] = api.get(crash_id=crash_id,
refresh_cache=refresh_cache)
except CrashIDNotFound:
# ...if we haven't already done so.
cache_key = "priority_job:{}".format(crash_id)
if not cache.get(cache_key):
priority_api = models.PriorityJob()
priority_api.post(crash_ids=[crash_id])
cache.set(cache_key, True, 60)
return render(request, "crashstats/report_index_pending.html", context)
if "json_dump" in context["report"]:
json_dump = context["report"]["json_dump"]
if "sensitive" in json_dump and not request.user.has_perm(
"crashstats.view_pii"):
del json_dump["sensitive"]
context["raw_stackwalker_output"] = json.dumps(json_dump,
sort_keys=True,
indent=4,
separators=(",", ": "))
utils.enhance_json_dump(json_dump, settings.VCS_MAPPINGS)
parsed_dump = json_dump
else:
context["raw_stackwalker_output"] = "No dump available"
parsed_dump = {}
# NOTE(willkg): pull cpu count from parsed_dump if it's not in report;
# remove in July 2019
if "cpu_count" not in context["report"]:
context["report"]["cpu_count"] = parsed_dump.get("system_info",
{}).get("cpu_count")
# NOTE(willkg): "cpu_name" is deprecated, but populate "cpu_arch" if
# cpu_arch is empty; remove in July 2019.
if "cpu_arch" not in context["report"]:
context["report"]["cpu_arch"] = context["report"]["cpu_name"]
context["crashing_thread"] = parsed_dump.get("crash_info",
{}).get("crashing_thread")
if context["report"]["signature"].startswith("shutdownhang"):
# For shutdownhang signatures, we want to use thread 0 as the
# crashing thread, because that's the thread that actually contains
# the useful data about what happened.
context["crashing_thread"] = 0
context["parsed_dump"] = parsed_dump
context["bug_product_map"] = settings.BUG_PRODUCT_MAP
context["bug_associations"] = list(
models.BugAssociation.objects.filter(
signature=context["report"]["signature"]).values(
"bug_id", "signature").order_by("-bug_id"))
context["raw_keys"] = []
if request.user.has_perm("crashstats.view_pii"):
# hold nothing back
context["raw_keys"] = context["raw"].keys()
else:
context["raw_keys"] = [
x for x in context["raw"] if x in models.RawCrash.API_ALLOWLIST()
]
# Sort keys case-insensitively
context["raw_keys"] = sorted(context["raw_keys"], key=lambda s: s.lower())
if request.user.has_perm("crashstats.view_rawdump"):
context["raw_dump_urls"] = [
reverse("crashstats:raw_data", args=(crash_id, "dmp")),
reverse("crashstats:raw_data", args=(crash_id, "json")),
]
if context["raw"].get("additional_minidumps"):
suffixes = [
x.strip()
for x in context["raw"]["additional_minidumps"].split(",")
if x.strip()
]
for suffix in suffixes:
name = "upload_file_minidump_%s" % (suffix, )
context["raw_dump_urls"].append(
reverse("crashstats:raw_data_named",
args=(crash_id, name, "dmp")))
if (context["raw"].get("ContainsMemoryReport")
and context["report"].get("memory_report")
and not context["report"].get("memory_report_error")):
context["raw_dump_urls"].append(
reverse(
"crashstats:raw_data_named",
args=(crash_id, "memory_report", "json.gz"),
))
# Add descriptions to all fields.
all_fields = SuperSearchFields().get()
descriptions = {}
for field in all_fields.values():
key = "{}.{}".format(field["namespace"], field["in_database_name"])
descriptions[key] = "{} Search: {}".format(
field.get("description", "").strip()
or "No description for this field.",
field["is_exposed"] and field["name"] or "N/A",
)
def make_raw_crash_key(key):
"""In the report_index.html template we need to create a key
that we can use to look up against the 'fields_desc' dict.
Because you can't do something like this in jinja::
{{ fields_desc.get(u'raw_crash.{}'.format(key), empty_desc) }}
we do it here in the function instead.
The trick is that the lookup key has to be a unicode object or
else you get UnicodeEncodeErrors in the template rendering.
"""
return "raw_crash.{}".format(key)
context["make_raw_crash_key"] = make_raw_crash_key
context["fields_desc"] = descriptions
context["empty_desc"] = "No description for this field. Search: unknown"
context["BUG_PRODUCT_MAP"] = settings.BUG_PRODUCT_MAP
# report.addons used to be a list of lists.
# In https://bugzilla.mozilla.org/show_bug.cgi?id=1250132
# we changed it from a list of lists to a list of strings, using
# a ':' to split the name and version.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=1250132#c7
# Considering legacy, let's tackle both.
# In late 2017, this code is going to be useless and can be removed.
if context["report"].get("addons") and isinstance(
context["report"]["addons"][0], (list, tuple)):
# This is the old legacy format. This crash hasn't been processed
# the new way.
context["report"]["addons"] = [
":".join(x) for x in context["report"]["addons"]
]
content = loader.render_to_string("crashstats/report_index.html", context,
request)
utf8_content = content.encode("utf-8", errors="backslashreplace")
return HttpResponse(utf8_content, charset="utf-8")
def report_index(request, crash_id, default_context=None):
valid_crash_id = utils.find_crash_id(crash_id)
if not valid_crash_id:
return http.HttpResponseBadRequest('Invalid crash ID')
# Sometimes, in Socorro we use a prefix on the crash ID. Usually it's
# 'bp-' but this is configurable.
# If you try to use this to reach the perma link for a crash, it should
# redirect to the report index with the correct crash ID.
if valid_crash_id != crash_id:
return redirect(
reverse('crashstats:report_index', args=(valid_crash_id, )))
context = default_context or {}
context['crash_id'] = crash_id
refresh_cache = request.GET.get('refresh') == 'cache'
raw_api = models.RawCrash()
try:
context['raw'] = raw_api.get(crash_id=crash_id,
refresh_cache=refresh_cache)
except CrashIDNotFound:
# If the raw crash can't be found, we can't do much.
return render(request,
'crashstats/report_index_not_found.html',
context,
status=404)
utils.enhance_raw(context['raw'])
context['your_crash'] = (request.user.is_active and
context['raw'].get('Email') == request.user.email)
api = models.UnredactedCrash()
try:
context['report'] = api.get(crash_id=crash_id,
refresh_cache=refresh_cache)
except CrashIDNotFound:
# ...if we haven't already done so.
cache_key = 'priority_job:{}'.format(crash_id)
if not cache.get(cache_key):
priority_api = models.Priorityjob()
priority_api.post(crash_ids=[crash_id])
cache.set(cache_key, True, 60)
return render(request, 'crashstats/report_index_pending.html', context)
if 'json_dump' in context['report']:
json_dump = context['report']['json_dump']
if 'sensitive' in json_dump and not request.user.has_perm(
'crashstats.view_pii'):
del json_dump['sensitive']
context['raw_stackwalker_output'] = json.dumps(json_dump,
sort_keys=True,
indent=4,
separators=(',', ': '))
utils.enhance_json_dump(json_dump, settings.VCS_MAPPINGS)
parsed_dump = json_dump
else:
context['raw_stackwalker_output'] = 'No dump available'
parsed_dump = {}
# NOTE(willkg): pull cpu count from parsed_dump if it's not in report;
# remove in July 2019
if 'cpu_count' not in context['report']:
context['report']['cpu_count'] = parsed_dump.get('system_info',
{}).get('cpu_count')
# NOTE(willkg): "cpu_name" is deprecated, but populate "cpu_arch" if
# cpu_arch is empty; remove in July 2019.
if 'cpu_arch' not in context['report']:
context['report']['cpu_arch'] = context['report']['cpu_name']
context['crashing_thread'] = parsed_dump.get('crash_info',
{}).get('crashing_thread')
if context['report']['signature'].startswith('shutdownhang'):
# For shutdownhang signatures, we want to use thread 0 as the
# crashing thread, because that's the thread that actually contains
# the useful data about what happened.
context['crashing_thread'] = 0
context['parsed_dump'] = parsed_dump
context['bug_product_map'] = settings.BUG_PRODUCT_MAP
context['bug_associations'] = list(
models.BugAssociation.objects.filter(
signature=context['report']['signature']).values(
'bug_id', 'signature').order_by('-bug_id'))
context['raw_keys'] = []
if request.user.has_perm('crashstats.view_pii'):
# hold nothing back
context['raw_keys'] = context['raw'].keys()
else:
context['raw_keys'] = [
x for x in context['raw'] if x in models.RawCrash.API_WHITELIST()
]
# Sort keys case-insensitively
context['raw_keys'] = sorted(context['raw_keys'], key=lambda s: s.lower())
if request.user.has_perm('crashstats.view_rawdump'):
context['raw_dump_urls'] = [
reverse('crashstats:raw_data', args=(crash_id, 'dmp')),
reverse('crashstats:raw_data', args=(crash_id, 'json'))
]
if context['raw'].get('additional_minidumps'):
suffixes = [
x.strip()
for x in context['raw']['additional_minidumps'].split(',')
if x.strip()
]
for suffix in suffixes:
name = 'upload_file_minidump_%s' % (suffix, )
context['raw_dump_urls'].append(
reverse('crashstats:raw_data_named',
args=(crash_id, name, 'dmp')))
if (context['raw'].get('ContainsMemoryReport')
and context['report'].get('memory_report')
and not context['report'].get('memory_report_error')):
context['raw_dump_urls'].append(
reverse('crashstats:raw_data_named',
args=(crash_id, 'memory_report', 'json.gz')))
# Add descriptions to all fields.
all_fields = SuperSearchFields().get()
descriptions = {}
for field in all_fields.values():
key = '{}.{}'.format(field['namespace'], field['in_database_name'])
descriptions[key] = '{} Search: {}'.format(
field.get('description', '').strip()
or 'No description for this field.',
field['is_exposed'] and field['name'] or 'N/A',
)
def make_raw_crash_key(key):
"""In the report_index.html template we need to create a key
that we can use to look up against the 'fields_desc' dict.
Because you can't do something like this in jinja::
{{ fields_desc.get(u'raw_crash.{}'.format(key), empty_desc) }}
we do it here in the function instead.
The trick is that the lookup key has to be a unicode object or
else you get UnicodeEncodeErrors in the template rendering.
"""
return u'raw_crash.{}'.format(key)
context['make_raw_crash_key'] = make_raw_crash_key
context['fields_desc'] = descriptions
context['empty_desc'] = 'No description for this field. Search: unknown'
context['BUG_PRODUCT_MAP'] = settings.BUG_PRODUCT_MAP
# report.addons used to be a list of lists.
# In https://bugzilla.mozilla.org/show_bug.cgi?id=1250132
# we changed it from a list of lists to a list of strings, using
# a ':' to split the name and version.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=1250132#c7
# Considering legacy, let's tackle both.
# In late 2017, this code is going to be useless and can be removed.
if (context['report'].get('addons')
and isinstance(context['report']['addons'][0], (list, tuple))):
# This is the old legacy format. This crash hasn't been processed
# the new way.
context['report']['addons'] = [
':'.join(x) for x in context['report']['addons']
]
return render(request, 'crashstats/report_index.html', context)
