def signature_report(request, params, default_context=None):
context = default_context
signature = request.GET.get('signature')
if not signature:
return http.HttpResponseBadRequest(
'"signature" parameter is mandatory'
)
context['signature'] = signature
fields = sorted(
x['name']
for x in SuperSearchFields().get().values()
if x['is_exposed'] and
x['is_returned'] and
has_permissions(request.user, x['permissions_needed']) and
x['name'] != 'signature' # exclude the signature field
)
context['fields'] = [
{'id': field, 'text': field.replace('_', ' ')} for field in fields
]
columns = request.GET.getlist('_columns')
columns = [x for x in columns if x in fields]
context['columns'] = columns or DEFAULT_COLUMNS
sort = request.GET.getlist('_sort')
sort = [x for x in sort if x in fields]
context['sort'] = sort or DEFAULT_SORT
context['channels'] = ','.join(settings.CHANNELS).split(',')
context['channel'] = settings.CHANNEL
# Compute dates to show them to the user.
start_date, end_date = get_date_boundaries(params)
context['query'] = {
'start_date': start_date,
'end_date': end_date,
}
return render(request, 'signature/signature_report.html', context)
def signature_report(request, params, default_context=None):
context = default_context
signature = request.GET.get("signature")
if not signature:
return http.HttpResponseBadRequest(
'"signature" parameter is mandatory')
context["signature"] = signature
fields = sorted(x["name"] for x in SuperSearchFields().get().values()
if x["is_exposed"] and x["is_returned"]
and request.user.has_perms(x["permissions_needed"])
and x["name"] != "signature" # exclude the signature field
)
context["fields"] = [{
"id": field,
"text": field.replace("_", " ")
} for field in fields]
columns = request.GET.getlist("_columns")
columns = [x for x in columns if x in fields]
context["columns"] = columns or DEFAULT_COLUMNS
sort = request.GET.getlist("_sort")
sort = [x for x in sort if x in fields]
context["sort"] = sort or DEFAULT_SORT
context["channels"] = ",".join(settings.CHANNELS).split(",")
context["channel"] = settings.CHANNEL
context["correlations_products"] = CORRELATIONS_PRODUCTS
# Compute dates to show them to the user.
start_date, end_date = get_date_boundaries(params)
context["query"] = {"start_date": start_date, "end_date": end_date}
return render(request, "signature/signature_report.html", context)
def get_topcrashers_results(**kwargs):
"""Return the results of a search. """
results = []
params = kwargs
range_type = params.pop('_range_type')
dates = get_date_boundaries(params)
params['_aggs.signature'] = [
'platform',
'is_garbage_collecting',
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
'_cardinality.install_time',
]
params['_histogram_interval.uptime'] = 60
# We don't care about no results, only facets.
params['_results_number'] = 0
if params.get('process_type') in ('any', 'all'):
params['process_type'] = None
if range_type == 'build':
params['build_id'] = [
'>=' + datetime_to_build_id(dates[0]),
'<' + datetime_to_build_id(dates[1])
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
if search_results['total'] > 0:
results = search_results['facets']['signature']
platforms = models.Platforms().get_all()['hits']
platform_codes = [
x['code'] for x in platforms if x['code'] != 'unknown'
]
for i, hit in enumerate(results):
hit['signature'] = hit['term']
hit['rank'] = i + 1
hit['percent'] = 100.0 * hit['count'] / search_results['total']
# Number of crash per platform.
for platform in platform_codes:
hit[platform + '_count'] = 0
sig_platforms = hit['facets']['platform']
for platform in sig_platforms:
code = platform['term'][:3].lower()
if code in platform_codes:
hit[code + '_count'] = platform['count']
# Number of crashes happening during garbage collection.
hit['is_gc_count'] = 0
sig_gc = hit['facets']['is_garbage_collecting']
for row in sig_gc:
if row['term'].lower() == 't':
hit['is_gc_count'] = row['count']
# Number of plugin crashes.
hit['plugin_count'] = 0
sig_process = hit['facets']['process_type']
for row in sig_process:
if row['term'].lower() == 'plugin':
hit['plugin_count'] = row['count']
# Number of hang crashes.
hit['hang_count'] = 0
sig_hang = hit['facets']['hang_type']
for row in sig_hang:
# Hangs have weird values in the database: a value of 1 or -1
# means it is a hang, a value of 0 or missing means it is not.
if row['term'] in (1, -1):
hit['hang_count'] += row['count']
# Number of crashes happening during startup. This is defined by
# the client, as opposed to the next method which relies on
# the uptime of the client.
hit['startup_count'] = sum(
row['count'] for row in hit['facets']['startup_crash']
if row['term'] in ('T', '1'))
# Is a startup crash if more than half of the crashes are happening
# in the first minute after launch.
hit['startup_crash'] = False
sig_uptime = hit['facets']['histogram_uptime']
for row in sig_uptime:
# Aggregation buckets use the lowest value of the bucket as
# term. So for everything between 0 and 60 excluded, the
# term will be `0`.
if row['term'] < 60:
ratio = 1.0 * row['count'] / hit['count']
hit['startup_crash'] = ratio > 0.5
# Number of distinct installations.
hit['installs_count'] = (
hit['facets']['cardinality_install_time']['value'])
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params['date'] = [
'>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat()
]
params['_aggs.signature'] = [
'platform',
]
params['_facets_size'] *= 2
if range_type == 'build':
params['date'][1] = '<' + dates[1].isoformat()
params['build_id'] = [
'>=' + datetime_to_build_id(dates[1] - delta),
'<' + datetime_to_build_id(dates[0])
]
previous_range_results = api.get(**params)
total = previous_range_results['total']
compare_signatures = {}
if total > 0 and 'signature' in previous_range_results['facets']:
signatures = previous_range_results['facets']['signature']
for i, hit in enumerate(signatures):
compare_signatures[hit['term']] = {
'count': hit['count'],
'rank': i + 1,
'percent': 100.0 * hit['count'] / total
}
for hit in results:
sig = compare_signatures.get(hit['term'])
if sig:
hit['diff'] = sig['percent'] - hit['percent']
hit['rank_diff'] = sig['rank'] - hit['rank']
hit['previous_percent'] = sig['percent']
else:
hit['diff'] = 'new'
hit['rank_diff'] = 0
hit['previous_percent'] = 0
return search_results
def get_topcrashers_results(**kwargs):
"""Return the results of a search. """
results = []
params = kwargs
range_type = params.pop('_range_type')
dates = get_date_boundaries(params)
params['_aggs.signature'] = [
'platform',
'is_garbage_collecting',
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
'_cardinality.install_time',
]
params['_histogram_interval.uptime'] = 60
# We don't care about no results, only facets.
params['_results_number'] = 0
if params.get('process_type') in ('any', 'all'):
params['process_type'] = None
if range_type == 'build':
params['build_id'] = [
'>=' + datetime_to_build_id(dates[0]),
'<' + datetime_to_build_id(dates[1])
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
if search_results['total'] > 0:
results = search_results['facets']['signature']
platforms = models.Platforms().get_all()['hits']
platform_codes = [
x['code'] for x in platforms if x['code'] != 'unknown'
]
for i, hit in enumerate(results):
hit['signature'] = hit['term']
hit['rank'] = i + 1
hit['percent'] = 100.0 * hit['count'] / search_results['total']
# Number of crash per platform.
for platform in platform_codes:
hit[platform + '_count'] = 0
sig_platforms = hit['facets']['platform']
for platform in sig_platforms:
code = platform['term'][:3].lower()
if code in platform_codes:
hit[code + '_count'] = platform['count']
# Number of crashes happening during garbage collection.
hit['is_gc_count'] = 0
sig_gc = hit['facets']['is_garbage_collecting']
for row in sig_gc:
if row['term'].lower() == 't':
hit['is_gc_count'] = row['count']
# Number of plugin crashes.
hit['plugin_count'] = 0
sig_process = hit['facets']['process_type']
for row in sig_process:
if row['term'].lower() == 'plugin':
hit['plugin_count'] = row['count']
# Number of hang crashes.
hit['hang_count'] = 0
sig_hang = hit['facets']['hang_type']
for row in sig_hang:
# Hangs have weird values in the database: a value of 1 or -1
# means it is a hang, a value of 0 or missing means it is not.
if row['term'] in (1, -1):
hit['hang_count'] += row['count']
# Number of crashes happening during startup. This is defined by
# the client, as opposed to the next method which relies on
# the uptime of the client.
hit['startup_count'] = sum(
row['count'] for row in hit['facets']['startup_crash']
if row['term'] in ('T', '1')
)
# Is a startup crash if more than half of the crashes are happening
# in the first minute after launch.
hit['startup_crash'] = False
sig_uptime = hit['facets']['histogram_uptime']
for row in sig_uptime:
# Aggregation buckets use the lowest value of the bucket as
# term. So for everything between 0 and 60 excluded, the
# term will be `0`.
if row['term'] < 60:
ratio = 1.0 * row['count'] / hit['count']
hit['startup_crash'] = ratio > 0.5
# Number of distinct installations.
hit['installs_count'] = (
hit['facets']['cardinality_install_time']['value']
)
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params['date'] = [
'>=' + (dates[1] - delta).isoformat(),
'<' + dates[0].isoformat()
]
params['_aggs.signature'] = [
'platform',
]
params['_facets_size'] *= 2
if range_type == 'build':
params['date'][1] = '<' + dates[1].isoformat()
params['build_id'] = [
'>=' + datetime_to_build_id(dates[1] - delta),
'<' + datetime_to_build_id(dates[0])
]
previous_range_results = api.get(**params)
total = previous_range_results['total']
compare_signatures = {}
if total > 0 and 'signature' in previous_range_results['facets']:
signatures = previous_range_results['facets']['signature']
for i, hit in enumerate(signatures):
compare_signatures[hit['term']] = {
'count': hit['count'],
'rank': i + 1,
'percent': 100.0 * hit['count'] / total
}
for hit in results:
sig = compare_signatures.get(hit['term'])
if sig:
hit['diff'] = sig['percent'] - hit['percent']
hit['rank_diff'] = sig['rank'] - hit['rank']
hit['previous_percent'] = sig['percent']
else:
hit['diff'] = 'new'
hit['rank_diff'] = 0
hit['previous_percent'] = 0
return search_results
def get_topcrashers_stats(**kwargs):
"""Return the results of a search. """
params = kwargs
range_type = params.pop("_range_type")
dates = get_date_boundaries(params)
params["_aggs.signature"] = [
"platform",
"is_garbage_collecting",
"hang_type",
"process_type",
"startup_crash",
"_histogram.uptime",
"_cardinality.install_time",
]
params["_histogram_interval.uptime"] = 60
# We don't care about no results, only facets.
params["_results_number"] = 0
if params.get("process_type") in ("any", "all"):
params["process_type"] = None
if range_type == "build":
params["build_id"] = [
">=" + datetime_to_build_id(dates[0]),
"<" + datetime_to_build_id(dates[1]),
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
signatures_stats = []
if search_results["total"] > 0:
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params["date"] = [
">=" + (dates[1] - delta).isoformat(),
"<" + dates[0].isoformat(),
]
params["_aggs.signature"] = ["platform"]
params["_facets_size"] *= 2
if range_type == "build":
params["date"][1] = "<" + dates[1].isoformat()
params["build_id"] = [
">=" + datetime_to_build_id(dates[1] - delta),
"<" + datetime_to_build_id(dates[0]),
]
previous_range_results = api.get(**params)
previous_signatures = get_comparison_signatures(previous_range_results)
for index, signature in enumerate(
search_results["facets"]["signature"]):
previous_signature = previous_signatures.get(signature["term"])
signatures_stats.append(
SignatureStats(
signature=signature,
num_total_crashes=search_results["total"],
rank=index,
platforms=models.Platform.objects.values(),
previous_signature=previous_signature,
))
return signatures_stats
def get_topcrashers_stats(**kwargs):
"""Return the results of a search. """
params = kwargs
range_type = params.pop('_range_type')
dates = get_date_boundaries(params)
params['_aggs.signature'] = [
'platform',
'is_garbage_collecting',
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
'_cardinality.install_time',
]
params['_histogram_interval.uptime'] = 60
# We don't care about no results, only facets.
params['_results_number'] = 0
if params.get('process_type') in ('any', 'all'):
params['process_type'] = None
if range_type == 'build':
params['build_id'] = [
'>=' + datetime_to_build_id(dates[0]),
'<' + datetime_to_build_id(dates[1])
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
signatures_stats = []
if search_results['total'] > 0:
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params['date'] = [
'>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat()
]
params['_aggs.signature'] = [
'platform',
]
params['_facets_size'] *= 2
if range_type == 'build':
params['date'][1] = '<' + dates[1].isoformat()
params['build_id'] = [
'>=' + datetime_to_build_id(dates[1] - delta),
'<' + datetime_to_build_id(dates[0])
]
previous_range_results = api.get(**params)
previous_signatures = get_comparison_signatures(previous_range_results)
for index, signature in enumerate(
search_results['facets']['signature']):
previous_signature = previous_signatures.get(signature['term'])
signatures_stats.append(
SignatureStats(
signature=signature,
num_total_crashes=search_results['total'],
rank=index,
platforms=models.Platforms().get_all()['hits'],
previous_signature=previous_signature,
))
return signatures_stats
def get_topcrashers_results(**kwargs):
"""Return the results of a search. """
results = []
params = kwargs
range_type = params.pop('_range_type')
dates = get_date_boundaries(params)
params['_aggs.signature'] = [
'platform',
'is_garbage_collecting',
'hang_type',
'process_type',
'_histogram.uptime',
]
params['_histogram_interval.uptime'] = 60
# We don't care about no results, only facets.
params['_results_number'] = 0
if params.get('process_type') in ('any', 'all'):
params['process_type'] = None
if range_type == 'build':
params['build_id'] = [
'>=' + datetime_to_build_id(dates[0]),
'<' + datetime_to_build_id(dates[1])
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
if search_results['total'] > 0:
results = search_results['facets']['signature']
platforms = models.Platforms().get_all()['hits']
platform_codes = [
x['code'] for x in platforms if x['code'] != 'unknown'
]
for i, hit in enumerate(results):
hit['signature'] = hit['term']
hit['rank'] = i + 1
hit['percent'] = 100.0 * hit['count'] / search_results['total']
# Number of crash per platform.
for platform in platform_codes:
hit[platform + '_count'] = 0
sig_platforms = hit['facets']['platform']
for platform in sig_platforms:
code = platform['term'][:3].lower()
if code in platform_codes:
hit[code + '_count'] = platform['count']
# Number of crashes happening during garbage collection.
hit['is_gc_count'] = 0
sig_gc = hit['facets']['is_garbage_collecting']
for row in sig_gc:
if row['term'].lower() == 't':
hit['is_gc_count'] = row['count']
# Number of plugin crashes.
hit['plugin_count'] = 0
sig_process = hit['facets']['process_type']
for row in sig_process:
if row['term'].lower() == 'plugin':
hit['plugin_count'] = row['count']
# Number of hang crashes.
hit['hang_count'] = 0
sig_hang = hit['facets']['hang_type']
for row in sig_hang:
# Hangs have weird values in the database: a value of 1 or -1
# means it is a hang, a value of 0 or missing means it is not.
if row['term'] in (1, -1):
hit['hang_count'] += row['count']
# Number of startup crashes.
hit['startup_percent'] = 0
sig_startup = hit['facets']['histogram_uptime']
for row in sig_startup:
if row['term'] == 0:
ratio = 1.0 * row['count'] / hit['count']
hit['startup_crash'] = ratio > 0.5
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params['date'] = [
'>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat()
]
params['_aggs.signature'] = [
'platform',
]
params['_facets_size'] *= 2
if range_type == 'build':
params['date'][1] = '<' + dates[1].isoformat()
params['build_id'] = [
'>=' + datetime_to_build_id(dates[1] - delta),
'<' + datetime_to_build_id(dates[0])
]
previous_range_results = api.get(**params)
total = previous_range_results['total']
compare_signatures = {}
if total > 0 and 'signature' in previous_range_results['facets']:
signatures = previous_range_results['facets']['signature']
for i, hit in enumerate(signatures):
compare_signatures[hit['term']] = {
'count': hit['count'],
'rank': i + 1,
'percent': 100.0 * hit['count'] / total
}
for hit in results:
sig = compare_signatures.get(hit['term'])
if sig:
hit['diff'] = sig['percent'] - hit['percent']
hit['rank_diff'] = sig['rank'] - hit['rank']
hit['previous_percent'] = sig['percent']
else:
hit['diff'] = 'new'
hit['rank_diff'] = 0
hit['previous_percent'] = 0
return search_results
def get_topcrashers_stats(**kwargs):
"""Return the results of a search. """
params = kwargs
range_type = params.pop('_range_type')
dates = get_date_boundaries(params)
params['_aggs.signature'] = [
'platform',
'is_garbage_collecting',
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
'_cardinality.install_time',
]
params['_histogram_interval.uptime'] = 60
# We don't care about no results, only facets.
params['_results_number'] = 0
if params.get('process_type') in ('any', 'all'):
params['process_type'] = None
if range_type == 'build':
params['build_id'] = [
'>=' + datetime_to_build_id(dates[0]),
'<' + datetime_to_build_id(dates[1])
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
signatures_stats = []
if search_results['total'] > 0:
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params['date'] = [
'>=' + (dates[1] - delta).isoformat(),
'<' + dates[0].isoformat()
]
params['_aggs.signature'] = [
'platform',
]
params['_facets_size'] *= 2
if range_type == 'build':
params['date'][1] = '<' + dates[1].isoformat()
params['build_id'] = [
'>=' + datetime_to_build_id(dates[1] - delta),
'<' + datetime_to_build_id(dates[0])
]
previous_range_results = api.get(**params)
previous_signatures = get_comparison_signatures(previous_range_results)
for index, signature in enumerate(search_results['facets']['signature']):
previous_signature = previous_signatures.get(signature['term'])
signatures_stats.append(SignatureStats(
signature=signature,
num_total_crashes=search_results['total'],
rank=index,
platforms=models.Platform.objects.values(),
previous_signature=previous_signature,
))
return signatures_stats
def get_topcrashers_results(**kwargs):
"""Return the results of a search. """
results = []
params = kwargs
range_type = params.pop('_range_type')
dates = get_date_boundaries(params)
params['_aggs.signature'] = [
'platform',
'is_garbage_collecting',
'hang_type',
'process_type',
'_histogram.uptime',
]
params['_histogram_interval.uptime'] = 60
# We don't care about no results, only facets.
params['_results_number'] = 0
if params.get('process_type') in ('any', 'all'):
params['process_type'] = None
if range_type == 'build':
params['build_id'] = [
'>=' + datetime_to_build_id(dates[0]),
'<' + datetime_to_build_id(dates[1])
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
if search_results['total'] > 0:
results = search_results['facets']['signature']
platforms = models.Platforms().get_all()['hits']
platform_codes = [
x['code'] for x in platforms if x['code'] != 'unknown'
]
for i, hit in enumerate(results):
hit['signature'] = hit['term']
hit['rank'] = i + 1
hit['percent'] = 100.0 * hit['count'] / search_results['total']
# Number of crash per platform.
for platform in platform_codes:
hit[platform + '_count'] = 0
sig_platforms = hit['facets']['platform']
for platform in sig_platforms:
code = platform['term'][:3].lower()
if code in platform_codes:
hit[code + '_count'] = platform['count']
# Number of crashes happening during garbage collection.
hit['is_gc_count'] = 0
sig_gc = hit['facets']['is_garbage_collecting']
for row in sig_gc:
if row['term'].lower() == 't':
hit['is_gc_count'] = row['count']
# Number of plugin crashes.
hit['plugin_count'] = 0
sig_process = hit['facets']['process_type']
for row in sig_process:
if row['term'].lower() == 'plugin':
hit['plugin_count'] = row['count']
# Number of hang crashes.
hit['hang_count'] = 0
sig_hang = hit['facets']['hang_type']
for row in sig_hang:
# Hangs have weird values in the database: a value of 1 or -1
# means it is a hang, a value of 0 or missing means it is not.
if row['term'] in (1, -1):
hit['hang_count'] += row['count']
# Number of startup crashes.
hit['startup_percent'] = 0
sig_startup = hit['facets']['histogram_uptime']
for row in sig_startup:
if row['term'] == 0:
ratio = 1.0 * row['count'] / hit['count']
hit['startup_crash'] = ratio > 0.5
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params['date'] = [
'>=' + (dates[1] - delta).isoformat(),
'<' + dates[0].isoformat()
]
params['_aggs.signature'] = [
'platform',
]
params['_facets_size'] *= 2
if range_type == 'build':
params['date'][1] = '<' + dates[1].isoformat()
params['build_id'] = [
'>=' + datetime_to_build_id(dates[1] - delta),
'<' + datetime_to_build_id(dates[0])
]
previous_range_results = api.get(**params)
total = previous_range_results['total']
compare_signatures = {}
if total > 0 and 'signature' in previous_range_results['facets']:
signatures = previous_range_results['facets']['signature']
for i, hit in enumerate(signatures):
compare_signatures[hit['term']] = {
'count': hit['count'],
'rank': i + 1,
'percent': 100.0 * hit['count'] / total
}
for hit in results:
sig = compare_signatures.get(hit['term'])
if sig:
hit['diff'] = sig['percent'] - hit['percent']
hit['rank_diff'] = sig['rank'] - hit['rank']
hit['previous_percent'] = sig['percent']
else:
hit['diff'] = 'new'
hit['rank_diff'] = 0
hit['previous_percent'] = 0
return search_results
