def home(request, product, default_context=None): context = default_context or {} form = forms.HomeForm(request.GET) if not form.is_valid(): return http.HttpResponseBadRequest(str(form.errors)) context['days'] = form.cleaned_data['days'] context['versions'] = form.cleaned_data['version'] if not context['versions']: context['versions'] = [ x['version'] for x in context['active_versions'][product] if x['is_featured'] ] # If there are no featured versions but there are active # versions, then fall back to use that instead. if not context['versions'] and context['active_versions'][product]: # But when we do that, we have to make a manual cut-off of # the number of versions to return. So make it max 4. context['versions'] = [ x['version'] for x in context['active_versions'][product] ][:settings.NUMBER_OF_FEATURED_VERSIONS] # Set selected version for the navigation bar. if len(context['versions']) == 1: context['version'] = context['versions'][0] platforms_api = models.Platforms() platforms = platforms_api.get() context['platforms'] = [x['name'] for x in platforms if x.get('display')] context['es_shards_per_index'] = settings.ES_SHARDS_PER_INDEX return render(request, 'home/home.html', context)
def test_platforms(self): api = models.Platforms() def mocked_get(**options): return { 'hits': [ { 'code': 'win', 'name': 'Windows' }, { 'code': 'unk', 'name': 'Unknown' } ], 'total': 2 } models.Platforms.implementation().get.side_effect = mocked_get r = api.get() assert len(r) == 2 assert 'Windows' in settings.DISPLAY_OS_NAMES assert r[0] == {'code': 'win', 'name': 'Windows', 'display': True} assert 'Unknown' not in settings.DISPLAY_OS_NAMES assert r[1] == {'code': 'unk', 'name': 'Unknown', 'display': False}
def home(request, product, default_context=None): context = default_context or {} form = forms.HomeForm(request.GET) if not form.is_valid(): return http.HttpResponseBadRequest(str(form.errors)) context['days'] = form.cleaned_data['days'] context['versions'] = form.cleaned_data['version'] if not context['versions']: context['versions'] = [ x['version'] for x in context['active_versions'][product] if x['is_featured'] ] # Set selected version for the navigation bar. if len(context['versions']) == 1: context['version'] = context['versions'][0] platforms_api = models.Platforms() platforms = platforms_api.get() context['platforms'] = [x['name'] for x in platforms if x.get('display')] context['es_shards_per_index'] = settings.ES_SHARDS_PER_INDEX return render(request, 'home/home.html', context)
def search_fields(request): products = models.ProductsVersions().get() versions = models.CurrentVersions().get() platforms = models.Platforms().get() form = forms.SearchForm(products, versions, platforms, request.user.is_authenticated(), request.GET) return form.get_fields_list()
def get_supersearch_form(request): platforms = models.Platforms().get() product_versions = models.ProductVersions().get(active=True)['hits'] all_fields = SuperSearchFields().get() form = forms.SearchForm(all_fields, product_versions, platforms, request.user, request.GET) return form
def search_fields(request): products = models.ProductsVersions().get() versions = models.CurrentVersions().get() platforms = models.Platforms().get() form = forms.SearchForm(products, versions, platforms, request.user.has_perm('crashstats.view_pii'), request.GET) return form.get_fields_list()
def get_supersearch_form(request): products = models.ProductsVersions().get() versions = models.CurrentVersions().get() platforms = models.Platforms().get() all_fields = SuperSearchFields().get() form = forms.SearchForm(all_fields, products, versions, platforms, request.user, request.GET) return form
def get_topcrashers_results(**kwargs): """Return the results of a search. """ results = [] params = kwargs range_type = params.pop('_range_type') dates = get_date_boundaries(params) params['_aggs.signature'] = [ 'platform', 'is_garbage_collecting', 'hang_type', 'process_type', 'startup_crash', '_histogram.uptime', '_cardinality.install_time', ] params['_histogram_interval.uptime'] = 60 # We don't care about no results, only facets. params['_results_number'] = 0 if params.get('process_type') in ('any', 'all'): params['process_type'] = None if range_type == 'build': params['build_id'] = [ '>=' + datetime_to_build_id(dates[0]), '<' + datetime_to_build_id(dates[1]) ] api = SuperSearchUnredacted() search_results = api.get(**params) if search_results['total'] > 0: results = search_results['facets']['signature'] platforms = models.Platforms().get_all()['hits'] platform_codes = [ x['code'] for x in platforms if x['code'] != 'unknown' ] for i, hit in enumerate(results): hit['signature'] = hit['term'] hit['rank'] = i + 1 hit['percent'] = 100.0 * hit['count'] / search_results['total'] # Number of crash per platform. for platform in platform_codes: hit[platform + '_count'] = 0 sig_platforms = hit['facets']['platform'] for platform in sig_platforms: code = platform['term'][:3].lower() if code in platform_codes: hit[code + '_count'] = platform['count'] # Number of crashes happening during garbage collection. hit['is_gc_count'] = 0 sig_gc = hit['facets']['is_garbage_collecting'] for row in sig_gc: if row['term'].lower() == 't': hit['is_gc_count'] = row['count'] # Number of plugin crashes. hit['plugin_count'] = 0 sig_process = hit['facets']['process_type'] for row in sig_process: if row['term'].lower() == 'plugin': hit['plugin_count'] = row['count'] # Number of hang crashes. hit['hang_count'] = 0 sig_hang = hit['facets']['hang_type'] for row in sig_hang: # Hangs have weird values in the database: a value of 1 or -1 # means it is a hang, a value of 0 or missing means it is not. if row['term'] in (1, -1): hit['hang_count'] += row['count'] # Number of crashes happening during startup. This is defined by # the client, as opposed to the next method which relies on # the uptime of the client. hit['startup_count'] = sum( row['count'] for row in hit['facets']['startup_crash'] if row['term'] in ('T', '1')) # Is a startup crash if more than half of the crashes are happening # in the first minute after launch. hit['startup_crash'] = False sig_uptime = hit['facets']['histogram_uptime'] for row in sig_uptime: # Aggregation buckets use the lowest value of the bucket as # term. So for everything between 0 and 60 excluded, the # term will be `0`. if row['term'] < 60: ratio = 1.0 * row['count'] / hit['count'] hit['startup_crash'] = ratio > 0.5 # Number of distinct installations. hit['installs_count'] = ( hit['facets']['cardinality_install_time']['value']) # Run the same query but for the previous date range, so we can # compare the rankings and show rank changes. delta = (dates[1] - dates[0]) * 2 params['date'] = [ '>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat() ] params['_aggs.signature'] = [ 'platform', ] params['_facets_size'] *= 2 if range_type == 'build': params['date'][1] = '<' + dates[1].isoformat() params['build_id'] = [ '>=' + datetime_to_build_id(dates[1] - delta), '<' + datetime_to_build_id(dates[0]) ] previous_range_results = api.get(**params) total = previous_range_results['total'] compare_signatures = {} if total > 0 and 'signature' in previous_range_results['facets']: signatures = previous_range_results['facets']['signature'] for i, hit in enumerate(signatures): compare_signatures[hit['term']] = { 'count': hit['count'], 'rank': i + 1, 'percent': 100.0 * hit['count'] / total } for hit in results: sig = compare_signatures.get(hit['term']) if sig: hit['diff'] = sig['percent'] - hit['percent'] hit['rank_diff'] = sig['rank'] - hit['rank'] hit['previous_percent'] = sig['percent'] else: hit['diff'] = 'new' hit['rank_diff'] = 0 hit['previous_percent'] = 0 return search_results
def topcrashers(request, days=None, possible_days=None, default_context=None): context = default_context or {} product = request.GET.get('product') versions = request.GET.getlist('version') crash_type = request.GET.get('process_type') os_name = request.GET.get('platform') result_count = request.GET.get('_facets_size') tcbs_mode = request.GET.get('_tcbs_mode') range_type = request.GET.get('_range_type') range_type = 'build' if range_type == 'build' else 'report' if not tcbs_mode or tcbs_mode not in ('realtime', 'byday'): tcbs_mode = 'realtime' if product not in context['active_versions']: raise http.Http404('Unrecognized product') context['product'] = product if not versions: # :( # simulate what the nav.js does which is to take the latest version # for this product. for pv in context['active_versions'][product]: if pv['is_featured']: url = '%s&version=%s' % (request.build_absolute_uri(), urlquote(pv['version'])) return redirect(url) # See if all versions support builds. If not, refuse to show the "by build" # range option in the UI. versions_have_builds = True for version in versions: for pv in context['active_versions'][product]: if pv['version'] == version and not pv['has_builds']: versions_have_builds = False break context['versions_have_builds'] = versions_have_builds # Used to pick a version in the dropdown menu. context['version'] = versions[0] if tcbs_mode == 'realtime': end_date = timezone.now().replace(microsecond=0) elif tcbs_mode == 'byday': end_date = timezone.now().replace(hour=0, minute=0, second=0, microsecond=0) # settings.PROCESS_TYPES might contain tuple to indicate that some # are actual labels. process_types = [] for option in settings.PROCESS_TYPES: if isinstance(option, (list, tuple)): process_types.append(option[0]) else: process_types.append(option) if crash_type not in process_types: crash_type = 'browser' context['crash_type'] = crash_type os_api = models.Platforms() operating_systems = os_api.get() if os_name not in (os_['name'] for os_ in operating_systems): os_name = None context['os_name'] = os_name # set the result counts filter in the context to use in # the template. This way we avoid hardcoding it twice and # have it defined in one common location. context['result_counts'] = settings.TCBS_RESULT_COUNTS if result_count not in context['result_counts']: result_count = settings.TCBS_RESULT_COUNTS[0] context['result_count'] = result_count context['query'] = { 'product': product, 'versions': versions, 'crash_type': crash_type, 'os_name': os_name, 'result_count': unicode(result_count), 'mode': tcbs_mode, 'range_type': range_type, 'end_date': end_date, 'start_date': end_date - datetime.timedelta(days=days), } api_results = get_topcrashers_results( product=product, version=versions, platform=os_name, process_type=crash_type, date=[ '<' + end_date.isoformat(), '>=' + context['query']['start_date'].isoformat() ], _facets_size=result_count, _range_type=range_type, ) if api_results['total'] > 0: tcbs = api_results['facets']['signature'] else: tcbs = [] count_of_included_crashes = 0 signatures = [] for crash in tcbs[:int(result_count)]: signatures.append(crash['signature']) count_of_included_crashes += crash['count'] context['number_of_crashes'] = count_of_included_crashes context['total_percentage'] = api_results['total'] and ( 100.0 * count_of_included_crashes / api_results['total']) # Get augmented bugs data. bugs = defaultdict(list) if signatures: bugs_api = models.Bugs() for b in bugs_api.get(signatures=signatures)['hits']: bugs[b['signature']].append(b['id']) # Get augmented signature data. sig_date_data = {} if signatures: sig_api = models.SignatureFirstDate() # SignatureFirstDate().get_dates() is an optimized version # of SignatureFirstDate().get() that returns a dict of # signature --> dates. first_dates = sig_api.get_dates(signatures) for sig, dates in first_dates.items(): sig_date_data[sig] = dates['first_date'] for crash in tcbs: crash_counts = [] # Due to the inconsistencies of OS usage and naming of # codes and props for operating systems the hacky bit below # is required. Socorro and the world will be a better place # once https://bugzilla.mozilla.org/show_bug.cgi?id=790642 lands. for operating_system in operating_systems: if operating_system['name'] == 'Unknown': # not applicable in this context continue os_code = operating_system['code'][0:3].lower() key = '%s_count' % os_code crash_counts.append([crash[key], operating_system['name']]) crash['correlation_os'] = max(crash_counts)[1] sig = crash['signature'] # Augment with bugs. if sig in bugs: if 'bugs' in crash: crash['bugs'].extend(bugs[sig]) else: crash['bugs'] = bugs[sig] # Augment with first appearance dates. if sig in sig_date_data: crash['first_report'] = sig_date_data[sig] if 'bugs' in crash: crash['bugs'].sort(reverse=True) context['tcbs'] = tcbs context['days'] = days context['report'] = 'topcrasher' context['possible_days'] = possible_days context['total_crashing_signatures'] = len(signatures) context['total_number_of_crashes'] = api_results['total'] context['process_type_values'] = [] for option in settings.PROCESS_TYPES: if option == 'all': continue if isinstance(option, (list, tuple)): value, label = option else: value = option label = option.capitalize() context['process_type_values'].append((value, label)) context['platform_values'] = settings.DISPLAY_OS_NAMES return render(request, 'topcrashers/topcrashers.html', context)
def get_topcrashers_results(**kwargs): '''Return the results of a search. ''' results = [] params = kwargs params['_aggs.signature'] = [ 'platform', 'is_garbage_collecting', 'hang_type', 'process_type', '_histogram.uptime', ] params['_histogram_interval.uptime'] = 60 # We don't care about no results, only facets. params['_results_number'] = 0 if params.get('process_type') in ('any', 'all'): params['process_type'] = None api = SuperSearchUnredacted() search_results = api.get(**params) if search_results['total'] > 0: results = search_results['facets']['signature'] platforms = models.Platforms().get_all()['hits'] platform_codes = [ x['code'] for x in platforms if x['code'] != 'unknown' ] for i, hit in enumerate(results): hit['signature'] = hit['term'] hit['rank'] = i + 1 hit['percent'] = 100.0 * hit['count'] / search_results['total'] # Number of crash per platform. for platform in platform_codes: hit[platform + '_count'] = 0 sig_platforms = hit['facets']['platform'] for platform in sig_platforms: code = platform['term'][:3].lower() if code in platform_codes: hit[code + '_count'] = platform['count'] # Number of crashes happening during garbage collection. hit['is_gc_count'] = 0 sig_gc = hit['facets']['is_garbage_collecting'] for row in sig_gc: if row['term'].lower() == 't': hit['is_gc_count'] = row['count'] # Number of plugin crashes. hit['plugin_count'] = 0 sig_process = hit['facets']['process_type'] for row in sig_process: if row['term'].lower() == 'plugin': hit['plugin_count'] = row['count'] # Number of hang crashes. hit['hang_count'] = 0 sig_hang = hit['facets']['hang_type'] for row in sig_hang: # Hangs have weird values in the database: a value of 1 or -1 # means it is a hang, a value of 0 or missing means it is not. if row['term'] in (1, -1): hit['hang_count'] += row['count'] # Number of startup crashes. hit['startup_percent'] = 0 sig_startup = hit['facets']['histogram_uptime'] for row in sig_startup: if row['term'] == 0: ratio = 1.0 * row['count'] / hit['count'] hit['startup_crash'] = ratio > 0.5 # Run the same query but for the previous date range, so we can # compare the rankings and show rank changes. dates = get_date_boundaries(params) delta = (dates[1] - dates[0]) * 2 params['date'] = [ '>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat() ] params['_aggs.signature'] = [ 'platform', ] previous_range_results = api.get(**params) total = previous_range_results['total'] compare_signatures = {} if total > 0 and 'signature' in previous_range_results['facets']: signatures = previous_range_results['facets']['signature'] for i, hit in enumerate(signatures): compare_signatures[hit['term']] = { 'count': hit['count'], 'rank': i + 1, 'percent': 100.0 * hit['count'] / total } for hit in results: sig = compare_signatures.get(hit['term']) if sig: hit['diff'] = sig['percent'] - hit['percent'] hit['rank_diff'] = sig['rank'] - hit['rank'] hit['previous_percent'] = sig['percent'] else: hit['diff'] = 'new' hit['rank_diff'] = 0 hit['previous_percent'] = 0 return search_results
def topcrashers(request, days=None, possible_days=None, default_context=None): context = default_context or {} product = request.GET.get('product') versions = request.GET.get('version') crash_type = request.GET.get('process_type') os_name = request.GET.get('platform') result_count = request.GET.get('_facets_size') tcbs_mode = request.GET.get('_tcbs_mode') if not tcbs_mode or tcbs_mode not in ('realtime', 'byday'): tcbs_mode = 'realtime' if product not in context['releases']: raise http.Http404('Unrecognized product') context['product'] = product if not versions: # :( # simulate what the nav.js does which is to take the latest version # for this product. for release in context['currentversions']: if release['product'] == product and release['featured']: url = '%s&version=%s' % (request.build_absolute_uri(), urlquote(release['version'])) return redirect(url) else: versions = versions.split(';') context['version'] = versions[0] if tcbs_mode == 'realtime': end_date = datetime.datetime.utcnow().replace(microsecond=0) elif tcbs_mode == 'byday': end_date = datetime.datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) if crash_type not in settings.PROCESS_TYPES: crash_type = 'browser' context['crash_type'] = crash_type os_api = models.Platforms() operating_systems = os_api.get() if os_name not in (os_['name'] for os_ in operating_systems): os_name = None context['os_name'] = os_name # set the result counts filter in the context to use in # the template. This way we avoid hardcoding it twice and # have it defined in one common location. context['result_counts'] = settings.TCBS_RESULT_COUNTS if result_count not in context['result_counts']: result_count = settings.TCBS_RESULT_COUNTS[0] context['result_count'] = result_count context['query'] = { 'product': product, 'versions': versions[0], 'crash_type': crash_type, 'os_name': os_name, 'result_count': unicode(result_count), 'mode': tcbs_mode, 'end_date': end_date, 'start_date': end_date - datetime.timedelta(days=days), } api_results = get_topcrashers_results( product=product, version=context['version'], platform=os_name, process_type=crash_type, date=[ '<' + end_date.isoformat(), '>=' + context['query']['start_date'].isoformat() ], _facets_size=result_count, ) if api_results['total'] > 0: tcbs = api_results['facets']['signature'] else: tcbs = [] count_of_included_crashes = 0 signatures = [] for crash in tcbs[:int(result_count)]: signatures.append(crash['signature']) count_of_included_crashes += crash['count'] context['number_of_crashes'] = count_of_included_crashes context['total_percentage'] = api_results['total'] and ( 100.0 * count_of_included_crashes / api_results['total']) # Get augmented bugs data. bugs = defaultdict(list) if signatures: bugs_api = models.Bugs() for b in bugs_api.get(signatures=signatures)['hits']: bugs[b['signature']].append(b['id']) # Get augmented signature data. sig_date_data = {} if signatures: sig_api = models.SignatureFirstDate() first_dates = sig_api.get(signatures=signatures) for sig in first_dates['hits']: sig_date_data[sig['signature']] = sig['first_date'] for crash in tcbs: crash_counts = [] # Due to the inconsistencies of OS usage and naming of # codes and props for operating systems the hacky bit below # is required. Socorro and the world will be a better place # once https://bugzilla.mozilla.org/show_bug.cgi?id=790642 lands. for operating_system in operating_systems: if operating_system['name'] == 'Unknown': # not applicable in this context continue os_code = operating_system['code'][0:3].lower() key = '%s_count' % os_code crash_counts.append([crash[key], operating_system['name']]) crash['correlation_os'] = max(crash_counts)[1] sig = crash['signature'] # Augment with bugs. if sig in bugs: if 'bugs' in crash: crash['bugs'].extend(bugs[sig]) else: crash['bugs'] = bugs[sig] # Augment with first appearance dates. if sig in sig_date_data: crash['first_report'] = isodate.parse_datetime(sig_date_data[sig]) if 'bugs' in crash: crash['bugs'].sort(reverse=True) context['tcbs'] = tcbs context['days'] = days context['report'] = 'topcrasher' context['possible_days'] = possible_days context['total_crashing_signatures'] = len(signatures) context['total_number_of_crashes'] = api_results['total'] context['process_type_values'] = (x for x in settings.PROCESS_TYPES if x != 'all') context['platform_values'] = settings.DISPLAY_OS_NAMES return render(request, 'topcrashers/topcrashers.html', context)
def get_topcrashers_stats(**kwargs): """Return the results of a search. """ params = kwargs range_type = params.pop('_range_type') dates = get_date_boundaries(params) params['_aggs.signature'] = [ 'platform', 'is_garbage_collecting', 'hang_type', 'process_type', 'startup_crash', '_histogram.uptime', '_cardinality.install_time', ] params['_histogram_interval.uptime'] = 60 # We don't care about no results, only facets. params['_results_number'] = 0 if params.get('process_type') in ('any', 'all'): params['process_type'] = None if range_type == 'build': params['build_id'] = [ '>=' + datetime_to_build_id(dates[0]), '<' + datetime_to_build_id(dates[1]) ] api = SuperSearchUnredacted() search_results = api.get(**params) signatures_stats = [] if search_results['total'] > 0: # Run the same query but for the previous date range, so we can # compare the rankings and show rank changes. delta = (dates[1] - dates[0]) * 2 params['date'] = [ '>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat() ] params['_aggs.signature'] = [ 'platform', ] params['_facets_size'] *= 2 if range_type == 'build': params['date'][1] = '<' + dates[1].isoformat() params['build_id'] = [ '>=' + datetime_to_build_id(dates[1] - delta), '<' + datetime_to_build_id(dates[0]) ] previous_range_results = api.get(**params) previous_signatures = get_comparison_signatures(previous_range_results) for index, signature in enumerate( search_results['facets']['signature']): previous_signature = previous_signatures.get(signature['term']) signatures_stats.append( SignatureStats( signature=signature, num_total_crashes=search_results['total'], rank=index, platforms=models.Platforms().get_all()['hits'], previous_signature=previous_signature, )) return signatures_stats
def search_results(request): products = models.ProductsVersions().get() versions = models.CurrentVersions().get() platforms = models.Platforms().get() form = forms.SearchForm(products, versions, platforms, request.user.is_authenticated(), request.GET) if not form.is_valid(): return http.HttpResponseBadRequest(str(form.errors)) params = {} for key in form.cleaned_data: if hasattr(form.fields[key], 'prefixed_value'): value = form.fields[key].prefixed_value else: value = form.cleaned_data[key] params[key] = value data = {} data['query'] = {'total': 0, 'total_count': 0, 'total_pages': 0} allowed_fields = ALL_POSSIBLE_FIELDS if request.user.is_authenticated(): allowed_fields += ADMIN_RESTRICTED_FIELDS current_query = request.GET.copy() if 'page' in current_query: del current_query['page'] data['params'] = current_query.copy() if '_columns' in data['params']: del data['params']['_columns'] if '_facets' in params: del data['params']['_facets'] params['_facets'] = request.GET.getlist('_facets') or DEFAULT_FACETS data['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS # Make sure only allowed fields are used params['_facets'] = [x for x in params['_facets'] if x in allowed_fields] data['columns'] = [x for x in data['columns'] if x in allowed_fields] try: current_page = int(request.GET.get('page', 1)) except ValueError: return http.HttpResponseBadRequest('Invalid page') if current_page <= 0: current_page = 1 results_per_page = 50 data['current_page'] = current_page data['results_offset'] = results_per_page * (current_page - 1) params['_results_number'] = results_per_page params['_results_offset'] = data['results_offset'] data['current_url'] = '%s?%s' % (reverse('supersearch.search'), current_query.urlencode()) api = SuperSearch() search_results = api.get(**params) if 'signature' in search_results['facets']: # Bugs for each signature signatures = [h['term'] for h in search_results['facets']['signature']] if signatures: bugs = defaultdict(list) bugs_api = models.Bugs() for b in bugs_api.get(signatures=signatures)['hits']: bugs[b['signature']].append(b['id']) for hit in search_results['facets']['signature']: sig = hit['term'] if sig in bugs: if 'bugs' in hit: hit['bugs'].extend(bugs[sig]) else: hit['bugs'] = bugs[sig] search_results['total_pages'] = int( math.ceil(search_results['total'] / float(results_per_page))) search_results['total_count'] = search_results['total'] data['query'] = search_results data['report_list_query_string'] = urllib.urlencode( utils.sanitize_dict(get_report_list_parameters(params)), True) return render(request, 'supersearch/search_results.html', data)