def signature_report(request, params, default_context=None): context = default_context signature = request.GET.get('signature') if not signature: return http.HttpResponseBadRequest( '"signature" parameter is mandatory' ) context['signature'] = signature fields = sorted( x['name'] for x in SuperSearchFields().get().values() if x['is_exposed'] and x['is_returned'] and has_permissions(request.user, x['permissions_needed']) and x['name'] != 'signature' # exclude the signature field ) context['fields'] = [ {'id': field, 'text': field.replace('_', ' ')} for field in fields ] columns = request.GET.getlist('_columns') columns = [x for x in columns if x in fields] context['columns'] = columns or DEFAULT_COLUMNS sort = request.GET.getlist('_sort') sort = [x for x in sort if x in fields] context['sort'] = sort or DEFAULT_SORT context['channels'] = ','.join(settings.CHANNELS).split(',') context['channel'] = settings.CHANNEL # Compute dates to show them to the user. start_date, end_date = get_date_boundaries(params) context['query'] = { 'start_date': start_date, 'end_date': end_date, } return render(request, 'signature/signature_report.html', context)
def signature_report(request, params, default_context=None): context = default_context signature = request.GET.get("signature") if not signature: return http.HttpResponseBadRequest( '"signature" parameter is mandatory') context["signature"] = signature fields = sorted(x["name"] for x in SuperSearchFields().get().values() if x["is_exposed"] and x["is_returned"] and request.user.has_perms(x["permissions_needed"]) and x["name"] != "signature" # exclude the signature field ) context["fields"] = [{ "id": field, "text": field.replace("_", " ") } for field in fields] columns = request.GET.getlist("_columns") columns = [x for x in columns if x in fields] context["columns"] = columns or DEFAULT_COLUMNS sort = request.GET.getlist("_sort") sort = [x for x in sort if x in fields] context["sort"] = sort or DEFAULT_SORT context["channels"] = ",".join(settings.CHANNELS).split(",") context["channel"] = settings.CHANNEL context["correlations_products"] = CORRELATIONS_PRODUCTS # Compute dates to show them to the user. start_date, end_date = get_date_boundaries(params) context["query"] = {"start_date": start_date, "end_date": end_date} return render(request, "signature/signature_report.html", context)
def get_topcrashers_results(**kwargs): """Return the results of a search. """ results = [] params = kwargs range_type = params.pop('_range_type') dates = get_date_boundaries(params) params['_aggs.signature'] = [ 'platform', 'is_garbage_collecting', 'hang_type', 'process_type', 'startup_crash', '_histogram.uptime', '_cardinality.install_time', ] params['_histogram_interval.uptime'] = 60 # We don't care about no results, only facets. params['_results_number'] = 0 if params.get('process_type') in ('any', 'all'): params['process_type'] = None if range_type == 'build': params['build_id'] = [ '>=' + datetime_to_build_id(dates[0]), '<' + datetime_to_build_id(dates[1]) ] api = SuperSearchUnredacted() search_results = api.get(**params) if search_results['total'] > 0: results = search_results['facets']['signature'] platforms = models.Platforms().get_all()['hits'] platform_codes = [ x['code'] for x in platforms if x['code'] != 'unknown' ] for i, hit in enumerate(results): hit['signature'] = hit['term'] hit['rank'] = i + 1 hit['percent'] = 100.0 * hit['count'] / search_results['total'] # Number of crash per platform. for platform in platform_codes: hit[platform + '_count'] = 0 sig_platforms = hit['facets']['platform'] for platform in sig_platforms: code = platform['term'][:3].lower() if code in platform_codes: hit[code + '_count'] = platform['count'] # Number of crashes happening during garbage collection. hit['is_gc_count'] = 0 sig_gc = hit['facets']['is_garbage_collecting'] for row in sig_gc: if row['term'].lower() == 't': hit['is_gc_count'] = row['count'] # Number of plugin crashes. hit['plugin_count'] = 0 sig_process = hit['facets']['process_type'] for row in sig_process: if row['term'].lower() == 'plugin': hit['plugin_count'] = row['count'] # Number of hang crashes. hit['hang_count'] = 0 sig_hang = hit['facets']['hang_type'] for row in sig_hang: # Hangs have weird values in the database: a value of 1 or -1 # means it is a hang, a value of 0 or missing means it is not. if row['term'] in (1, -1): hit['hang_count'] += row['count'] # Number of crashes happening during startup. This is defined by # the client, as opposed to the next method which relies on # the uptime of the client. hit['startup_count'] = sum( row['count'] for row in hit['facets']['startup_crash'] if row['term'] in ('T', '1')) # Is a startup crash if more than half of the crashes are happening # in the first minute after launch. hit['startup_crash'] = False sig_uptime = hit['facets']['histogram_uptime'] for row in sig_uptime: # Aggregation buckets use the lowest value of the bucket as # term. So for everything between 0 and 60 excluded, the # term will be `0`. if row['term'] < 60: ratio = 1.0 * row['count'] / hit['count'] hit['startup_crash'] = ratio > 0.5 # Number of distinct installations. hit['installs_count'] = ( hit['facets']['cardinality_install_time']['value']) # Run the same query but for the previous date range, so we can # compare the rankings and show rank changes. delta = (dates[1] - dates[0]) * 2 params['date'] = [ '>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat() ] params['_aggs.signature'] = [ 'platform', ] params['_facets_size'] *= 2 if range_type == 'build': params['date'][1] = '<' + dates[1].isoformat() params['build_id'] = [ '>=' + datetime_to_build_id(dates[1] - delta), '<' + datetime_to_build_id(dates[0]) ] previous_range_results = api.get(**params) total = previous_range_results['total'] compare_signatures = {} if total > 0 and 'signature' in previous_range_results['facets']: signatures = previous_range_results['facets']['signature'] for i, hit in enumerate(signatures): compare_signatures[hit['term']] = { 'count': hit['count'], 'rank': i + 1, 'percent': 100.0 * hit['count'] / total } for hit in results: sig = compare_signatures.get(hit['term']) if sig: hit['diff'] = sig['percent'] - hit['percent'] hit['rank_diff'] = sig['rank'] - hit['rank'] hit['previous_percent'] = sig['percent'] else: hit['diff'] = 'new' hit['rank_diff'] = 0 hit['previous_percent'] = 0 return search_results
def get_topcrashers_results(**kwargs): """Return the results of a search. """ results = [] params = kwargs range_type = params.pop('_range_type') dates = get_date_boundaries(params) params['_aggs.signature'] = [ 'platform', 'is_garbage_collecting', 'hang_type', 'process_type', 'startup_crash', '_histogram.uptime', '_cardinality.install_time', ] params['_histogram_interval.uptime'] = 60 # We don't care about no results, only facets. params['_results_number'] = 0 if params.get('process_type') in ('any', 'all'): params['process_type'] = None if range_type == 'build': params['build_id'] = [ '>=' + datetime_to_build_id(dates[0]), '<' + datetime_to_build_id(dates[1]) ] api = SuperSearchUnredacted() search_results = api.get(**params) if search_results['total'] > 0: results = search_results['facets']['signature'] platforms = models.Platforms().get_all()['hits'] platform_codes = [ x['code'] for x in platforms if x['code'] != 'unknown' ] for i, hit in enumerate(results): hit['signature'] = hit['term'] hit['rank'] = i + 1 hit['percent'] = 100.0 * hit['count'] / search_results['total'] # Number of crash per platform. for platform in platform_codes: hit[platform + '_count'] = 0 sig_platforms = hit['facets']['platform'] for platform in sig_platforms: code = platform['term'][:3].lower() if code in platform_codes: hit[code + '_count'] = platform['count'] # Number of crashes happening during garbage collection. hit['is_gc_count'] = 0 sig_gc = hit['facets']['is_garbage_collecting'] for row in sig_gc: if row['term'].lower() == 't': hit['is_gc_count'] = row['count'] # Number of plugin crashes. hit['plugin_count'] = 0 sig_process = hit['facets']['process_type'] for row in sig_process: if row['term'].lower() == 'plugin': hit['plugin_count'] = row['count'] # Number of hang crashes. hit['hang_count'] = 0 sig_hang = hit['facets']['hang_type'] for row in sig_hang: # Hangs have weird values in the database: a value of 1 or -1 # means it is a hang, a value of 0 or missing means it is not. if row['term'] in (1, -1): hit['hang_count'] += row['count'] # Number of crashes happening during startup. This is defined by # the client, as opposed to the next method which relies on # the uptime of the client. hit['startup_count'] = sum( row['count'] for row in hit['facets']['startup_crash'] if row['term'] in ('T', '1') ) # Is a startup crash if more than half of the crashes are happening # in the first minute after launch. hit['startup_crash'] = False sig_uptime = hit['facets']['histogram_uptime'] for row in sig_uptime: # Aggregation buckets use the lowest value of the bucket as # term. So for everything between 0 and 60 excluded, the # term will be `0`. if row['term'] < 60: ratio = 1.0 * row['count'] / hit['count'] hit['startup_crash'] = ratio > 0.5 # Number of distinct installations. hit['installs_count'] = ( hit['facets']['cardinality_install_time']['value'] ) # Run the same query but for the previous date range, so we can # compare the rankings and show rank changes. delta = (dates[1] - dates[0]) * 2 params['date'] = [ '>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat() ] params['_aggs.signature'] = [ 'platform', ] params['_facets_size'] *= 2 if range_type == 'build': params['date'][1] = '<' + dates[1].isoformat() params['build_id'] = [ '>=' + datetime_to_build_id(dates[1] - delta), '<' + datetime_to_build_id(dates[0]) ] previous_range_results = api.get(**params) total = previous_range_results['total'] compare_signatures = {} if total > 0 and 'signature' in previous_range_results['facets']: signatures = previous_range_results['facets']['signature'] for i, hit in enumerate(signatures): compare_signatures[hit['term']] = { 'count': hit['count'], 'rank': i + 1, 'percent': 100.0 * hit['count'] / total } for hit in results: sig = compare_signatures.get(hit['term']) if sig: hit['diff'] = sig['percent'] - hit['percent'] hit['rank_diff'] = sig['rank'] - hit['rank'] hit['previous_percent'] = sig['percent'] else: hit['diff'] = 'new' hit['rank_diff'] = 0 hit['previous_percent'] = 0 return search_results
def get_topcrashers_stats(**kwargs): """Return the results of a search. """ params = kwargs range_type = params.pop("_range_type") dates = get_date_boundaries(params) params["_aggs.signature"] = [ "platform", "is_garbage_collecting", "hang_type", "process_type", "startup_crash", "_histogram.uptime", "_cardinality.install_time", ] params["_histogram_interval.uptime"] = 60 # We don't care about no results, only facets. params["_results_number"] = 0 if params.get("process_type") in ("any", "all"): params["process_type"] = None if range_type == "build": params["build_id"] = [ ">=" + datetime_to_build_id(dates[0]), "<" + datetime_to_build_id(dates[1]), ] api = SuperSearchUnredacted() search_results = api.get(**params) signatures_stats = [] if search_results["total"] > 0: # Run the same query but for the previous date range, so we can # compare the rankings and show rank changes. delta = (dates[1] - dates[0]) * 2 params["date"] = [ ">=" + (dates[1] - delta).isoformat(), "<" + dates[0].isoformat(), ] params["_aggs.signature"] = ["platform"] params["_facets_size"] *= 2 if range_type == "build": params["date"][1] = "<" + dates[1].isoformat() params["build_id"] = [ ">=" + datetime_to_build_id(dates[1] - delta), "<" + datetime_to_build_id(dates[0]), ] previous_range_results = api.get(**params) previous_signatures = get_comparison_signatures(previous_range_results) for index, signature in enumerate( search_results["facets"]["signature"]): previous_signature = previous_signatures.get(signature["term"]) signatures_stats.append( SignatureStats( signature=signature, num_total_crashes=search_results["total"], rank=index, platforms=models.Platform.objects.values(), previous_signature=previous_signature, )) return signatures_stats
def get_topcrashers_stats(**kwargs): """Return the results of a search. """ params = kwargs range_type = params.pop('_range_type') dates = get_date_boundaries(params) params['_aggs.signature'] = [ 'platform', 'is_garbage_collecting', 'hang_type', 'process_type', 'startup_crash', '_histogram.uptime', '_cardinality.install_time', ] params['_histogram_interval.uptime'] = 60 # We don't care about no results, only facets. params['_results_number'] = 0 if params.get('process_type') in ('any', 'all'): params['process_type'] = None if range_type == 'build': params['build_id'] = [ '>=' + datetime_to_build_id(dates[0]), '<' + datetime_to_build_id(dates[1]) ] api = SuperSearchUnredacted() search_results = api.get(**params) signatures_stats = [] if search_results['total'] > 0: # Run the same query but for the previous date range, so we can # compare the rankings and show rank changes. delta = (dates[1] - dates[0]) * 2 params['date'] = [ '>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat() ] params['_aggs.signature'] = [ 'platform', ] params['_facets_size'] *= 2 if range_type == 'build': params['date'][1] = '<' + dates[1].isoformat() params['build_id'] = [ '>=' + datetime_to_build_id(dates[1] - delta), '<' + datetime_to_build_id(dates[0]) ] previous_range_results = api.get(**params) previous_signatures = get_comparison_signatures(previous_range_results) for index, signature in enumerate( search_results['facets']['signature']): previous_signature = previous_signatures.get(signature['term']) signatures_stats.append( SignatureStats( signature=signature, num_total_crashes=search_results['total'], rank=index, platforms=models.Platforms().get_all()['hits'], previous_signature=previous_signature, )) return signatures_stats
def get_topcrashers_results(**kwargs): """Return the results of a search. """ results = [] params = kwargs range_type = params.pop('_range_type') dates = get_date_boundaries(params) params['_aggs.signature'] = [ 'platform', 'is_garbage_collecting', 'hang_type', 'process_type', '_histogram.uptime', ] params['_histogram_interval.uptime'] = 60 # We don't care about no results, only facets. params['_results_number'] = 0 if params.get('process_type') in ('any', 'all'): params['process_type'] = None if range_type == 'build': params['build_id'] = [ '>=' + datetime_to_build_id(dates[0]), '<' + datetime_to_build_id(dates[1]) ] api = SuperSearchUnredacted() search_results = api.get(**params) if search_results['total'] > 0: results = search_results['facets']['signature'] platforms = models.Platforms().get_all()['hits'] platform_codes = [ x['code'] for x in platforms if x['code'] != 'unknown' ] for i, hit in enumerate(results): hit['signature'] = hit['term'] hit['rank'] = i + 1 hit['percent'] = 100.0 * hit['count'] / search_results['total'] # Number of crash per platform. for platform in platform_codes: hit[platform + '_count'] = 0 sig_platforms = hit['facets']['platform'] for platform in sig_platforms: code = platform['term'][:3].lower() if code in platform_codes: hit[code + '_count'] = platform['count'] # Number of crashes happening during garbage collection. hit['is_gc_count'] = 0 sig_gc = hit['facets']['is_garbage_collecting'] for row in sig_gc: if row['term'].lower() == 't': hit['is_gc_count'] = row['count'] # Number of plugin crashes. hit['plugin_count'] = 0 sig_process = hit['facets']['process_type'] for row in sig_process: if row['term'].lower() == 'plugin': hit['plugin_count'] = row['count'] # Number of hang crashes. hit['hang_count'] = 0 sig_hang = hit['facets']['hang_type'] for row in sig_hang: # Hangs have weird values in the database: a value of 1 or -1 # means it is a hang, a value of 0 or missing means it is not. if row['term'] in (1, -1): hit['hang_count'] += row['count'] # Number of startup crashes. hit['startup_percent'] = 0 sig_startup = hit['facets']['histogram_uptime'] for row in sig_startup: if row['term'] == 0: ratio = 1.0 * row['count'] / hit['count'] hit['startup_crash'] = ratio > 0.5 # Run the same query but for the previous date range, so we can # compare the rankings and show rank changes. delta = (dates[1] - dates[0]) * 2 params['date'] = [ '>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat() ] params['_aggs.signature'] = [ 'platform', ] params['_facets_size'] *= 2 if range_type == 'build': params['date'][1] = '<' + dates[1].isoformat() params['build_id'] = [ '>=' + datetime_to_build_id(dates[1] - delta), '<' + datetime_to_build_id(dates[0]) ] previous_range_results = api.get(**params) total = previous_range_results['total'] compare_signatures = {} if total > 0 and 'signature' in previous_range_results['facets']: signatures = previous_range_results['facets']['signature'] for i, hit in enumerate(signatures): compare_signatures[hit['term']] = { 'count': hit['count'], 'rank': i + 1, 'percent': 100.0 * hit['count'] / total } for hit in results: sig = compare_signatures.get(hit['term']) if sig: hit['diff'] = sig['percent'] - hit['percent'] hit['rank_diff'] = sig['rank'] - hit['rank'] hit['previous_percent'] = sig['percent'] else: hit['diff'] = 'new' hit['rank_diff'] = 0 hit['previous_percent'] = 0 return search_results
def get_topcrashers_stats(**kwargs): """Return the results of a search. """ params = kwargs range_type = params.pop('_range_type') dates = get_date_boundaries(params) params['_aggs.signature'] = [ 'platform', 'is_garbage_collecting', 'hang_type', 'process_type', 'startup_crash', '_histogram.uptime', '_cardinality.install_time', ] params['_histogram_interval.uptime'] = 60 # We don't care about no results, only facets. params['_results_number'] = 0 if params.get('process_type') in ('any', 'all'): params['process_type'] = None if range_type == 'build': params['build_id'] = [ '>=' + datetime_to_build_id(dates[0]), '<' + datetime_to_build_id(dates[1]) ] api = SuperSearchUnredacted() search_results = api.get(**params) signatures_stats = [] if search_results['total'] > 0: # Run the same query but for the previous date range, so we can # compare the rankings and show rank changes. delta = (dates[1] - dates[0]) * 2 params['date'] = [ '>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat() ] params['_aggs.signature'] = [ 'platform', ] params['_facets_size'] *= 2 if range_type == 'build': params['date'][1] = '<' + dates[1].isoformat() params['build_id'] = [ '>=' + datetime_to_build_id(dates[1] - delta), '<' + datetime_to_build_id(dates[0]) ] previous_range_results = api.get(**params) previous_signatures = get_comparison_signatures(previous_range_results) for index, signature in enumerate(search_results['facets']['signature']): previous_signature = previous_signatures.get(signature['term']) signatures_stats.append(SignatureStats( signature=signature, num_total_crashes=search_results['total'], rank=index, platforms=models.Platform.objects.values(), previous_signature=previous_signature, )) return signatures_stats