def test_related(self): models.BugAssociation.objects.create( bug_id='999999', signature='OOM | small' ) models.BugAssociation.objects.create( bug_id='999999', signature='OOM | medium' ) models.BugAssociation.objects.create( bug_id='1000000', signature='OOM | large' ) api = models.Bugs() resp = api.get(signatures=['OOM | small']) assert resp == { 'hits': [ { 'id': 999999, 'signature': 'OOM | medium' }, { 'id': 999999, 'signature': 'OOM | small' } ], 'total': 2 }
def test_related(self): models.BugAssociation.objects.create(bug_id="999999", signature="OOM | small") models.BugAssociation.objects.create(bug_id="999999", signature="OOM | medium") models.BugAssociation.objects.create(bug_id="1000000", signature="OOM | large") api = models.Bugs() resp = api.get(signatures=["OOM | small"]) assert resp == { "hits": [ { "id": 999999, "signature": "OOM | medium" }, { "id": 999999, "signature": "OOM | small" }, ], "total": 2, }
def test_get_one(self): models.BugAssociation.objects.create(bug_id="999999", signature="OOM | small") api = models.Bugs() resp = api.get(signatures=["OOM | small"]) assert resp == { "hits": [{"id": 999999, "signature": "OOM | small"}], "total": 1, }
def signature_bugzilla(request, params): '''Return a list of associated bugs. ''' context = {} signature = params['signature'][0] context['signature'] = signature bugs_api = models.Bugs() context['bugs'] = bugs_api.get(signatures=[signature])['hits'] context['bugs'].sort(key=lambda x: x['id'], reverse=True) return render(request, 'signature/signature_bugzilla.html', context)
def test_get_one(self): models.BugAssociation.objects.create(bug_id='999999', signature='OOM | small') api = models.Bugs() resp = api.get(signatures=['OOM | small']) assert resp == { 'hits': [{ 'id': 999999, 'signature': 'OOM | small' }], 'total': 1 }
def topcrashers(request, days=None, possible_days=None, default_context=None): context = default_context or {} product = request.GET.get('product') versions = request.GET.getlist('version') crash_type = request.GET.get('process_type') os_name = request.GET.get('platform') result_count = request.GET.get('_facets_size') tcbs_mode = request.GET.get('_tcbs_mode') range_type = request.GET.get('_range_type') range_type = 'build' if range_type == 'build' else 'report' if not tcbs_mode or tcbs_mode not in ('realtime', 'byday'): tcbs_mode = 'realtime' if product not in context['active_versions']: raise http.Http404('Unrecognized product') context['product'] = product if not versions: # :( # simulate what the nav.js does which is to take the latest version # for this product. for pv in context['active_versions'][product]: if pv['is_featured']: url = '%s&version=%s' % (request.build_absolute_uri(), urlquote(pv['version'])) return redirect(url) # See if all versions support builds. If not, refuse to show the "by build" # range option in the UI. versions_have_builds = True for version in versions: for pv in context['active_versions'][product]: if pv['version'] == version and not pv['has_builds']: versions_have_builds = False break context['versions_have_builds'] = versions_have_builds # Used to pick a version in the dropdown menu. context['version'] = versions[0] if tcbs_mode == 'realtime': end_date = timezone.now().replace(microsecond=0) elif tcbs_mode == 'byday': end_date = timezone.now().replace(hour=0, minute=0, second=0, microsecond=0) # settings.PROCESS_TYPES might contain tuple to indicate that some # are actual labels. process_types = [] for option in settings.PROCESS_TYPES: if isinstance(option, (list, tuple)): process_types.append(option[0]) else: process_types.append(option) if crash_type not in process_types: crash_type = 'browser' context['crash_type'] = crash_type os_api = models.Platforms() operating_systems = os_api.get() if os_name not in (os_['name'] for os_ in operating_systems): os_name = None context['os_name'] = os_name # set the result counts filter in the context to use in # the template. This way we avoid hardcoding it twice and # have it defined in one common location. context['result_counts'] = settings.TCBS_RESULT_COUNTS if result_count not in context['result_counts']: result_count = settings.TCBS_RESULT_COUNTS[0] context['result_count'] = result_count context['query'] = { 'product': product, 'versions': versions, 'crash_type': crash_type, 'os_name': os_name, 'result_count': unicode(result_count), 'mode': tcbs_mode, 'range_type': range_type, 'end_date': end_date, 'start_date': end_date - datetime.timedelta(days=days), } api_results = get_topcrashers_results( product=product, version=versions, platform=os_name, process_type=crash_type, date=[ '<' + end_date.isoformat(), '>=' + context['query']['start_date'].isoformat() ], _facets_size=result_count, _range_type=range_type, ) if api_results['total'] > 0: tcbs = api_results['facets']['signature'] else: tcbs = [] count_of_included_crashes = 0 signatures = [] for crash in tcbs[:int(result_count)]: signatures.append(crash['signature']) count_of_included_crashes += crash['count'] context['number_of_crashes'] = count_of_included_crashes context['total_percentage'] = api_results['total'] and ( 100.0 * count_of_included_crashes / api_results['total']) # Get augmented bugs data. bugs = defaultdict(list) if signatures: bugs_api = models.Bugs() for b in bugs_api.get(signatures=signatures)['hits']: bugs[b['signature']].append(b['id']) # Get augmented signature data. sig_date_data = {} if signatures: sig_api = models.SignatureFirstDate() # SignatureFirstDate().get_dates() is an optimized version # of SignatureFirstDate().get() that returns a dict of # signature --> dates. first_dates = sig_api.get_dates(signatures) for sig, dates in first_dates.items(): sig_date_data[sig] = dates['first_date'] for crash in tcbs: crash_counts = [] # Due to the inconsistencies of OS usage and naming of # codes and props for operating systems the hacky bit below # is required. Socorro and the world will be a better place # once https://bugzilla.mozilla.org/show_bug.cgi?id=790642 lands. for operating_system in operating_systems: if operating_system['name'] == 'Unknown': # not applicable in this context continue os_code = operating_system['code'][0:3].lower() key = '%s_count' % os_code crash_counts.append([crash[key], operating_system['name']]) crash['correlation_os'] = max(crash_counts)[1] sig = crash['signature'] # Augment with bugs. if sig in bugs: if 'bugs' in crash: crash['bugs'].extend(bugs[sig]) else: crash['bugs'] = bugs[sig] # Augment with first appearance dates. if sig in sig_date_data: crash['first_report'] = sig_date_data[sig] if 'bugs' in crash: crash['bugs'].sort(reverse=True) context['tcbs'] = tcbs context['days'] = days context['report'] = 'topcrasher' context['possible_days'] = possible_days context['total_crashing_signatures'] = len(signatures) context['total_number_of_crashes'] = api_results['total'] context['process_type_values'] = [] for option in settings.PROCESS_TYPES: if option == 'all': continue if isinstance(option, (list, tuple)): value, label = option else: value = option label = option.capitalize() context['process_type_values'].append((value, label)) context['platform_values'] = settings.DISPLAY_OS_NAMES return render(request, 'topcrashers/topcrashers.html', context)
def topcrashers(request, days=None, possible_days=None, default_context=None): context = default_context or {} product = request.GET.get('product') versions = request.GET.get('version') crash_type = request.GET.get('process_type') os_name = request.GET.get('platform') result_count = request.GET.get('_facets_size') tcbs_mode = request.GET.get('_tcbs_mode') if not tcbs_mode or tcbs_mode not in ('realtime', 'byday'): tcbs_mode = 'realtime' if product not in context['releases']: raise http.Http404('Unrecognized product') context['product'] = product if not versions: # :( # simulate what the nav.js does which is to take the latest version # for this product. for release in context['currentversions']: if release['product'] == product and release['featured']: url = '%s&version=%s' % (request.build_absolute_uri(), urlquote(release['version'])) return redirect(url) else: versions = versions.split(';') context['version'] = versions[0] if tcbs_mode == 'realtime': end_date = datetime.datetime.utcnow().replace(microsecond=0) elif tcbs_mode == 'byday': end_date = datetime.datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) if crash_type not in settings.PROCESS_TYPES: crash_type = 'browser' context['crash_type'] = crash_type os_api = models.Platforms() operating_systems = os_api.get() if os_name not in (os_['name'] for os_ in operating_systems): os_name = None context['os_name'] = os_name # set the result counts filter in the context to use in # the template. This way we avoid hardcoding it twice and # have it defined in one common location. context['result_counts'] = settings.TCBS_RESULT_COUNTS if result_count not in context['result_counts']: result_count = settings.TCBS_RESULT_COUNTS[0] context['result_count'] = result_count context['query'] = { 'product': product, 'versions': versions[0], 'crash_type': crash_type, 'os_name': os_name, 'result_count': unicode(result_count), 'mode': tcbs_mode, 'end_date': end_date, 'start_date': end_date - datetime.timedelta(days=days), } api_results = get_topcrashers_results( product=product, version=context['version'], platform=os_name, process_type=crash_type, date=[ '<' + end_date.isoformat(), '>=' + context['query']['start_date'].isoformat() ], _facets_size=result_count, ) if api_results['total'] > 0: tcbs = api_results['facets']['signature'] else: tcbs = [] count_of_included_crashes = 0 signatures = [] for crash in tcbs[:int(result_count)]: signatures.append(crash['signature']) count_of_included_crashes += crash['count'] context['number_of_crashes'] = count_of_included_crashes context['total_percentage'] = api_results['total'] and ( 100.0 * count_of_included_crashes / api_results['total']) # Get augmented bugs data. bugs = defaultdict(list) if signatures: bugs_api = models.Bugs() for b in bugs_api.get(signatures=signatures)['hits']: bugs[b['signature']].append(b['id']) # Get augmented signature data. sig_date_data = {} if signatures: sig_api = models.SignatureFirstDate() first_dates = sig_api.get(signatures=signatures) for sig in first_dates['hits']: sig_date_data[sig['signature']] = sig['first_date'] for crash in tcbs: crash_counts = [] # Due to the inconsistencies of OS usage and naming of # codes and props for operating systems the hacky bit below # is required. Socorro and the world will be a better place # once https://bugzilla.mozilla.org/show_bug.cgi?id=790642 lands. for operating_system in operating_systems: if operating_system['name'] == 'Unknown': # not applicable in this context continue os_code = operating_system['code'][0:3].lower() key = '%s_count' % os_code crash_counts.append([crash[key], operating_system['name']]) crash['correlation_os'] = max(crash_counts)[1] sig = crash['signature'] # Augment with bugs. if sig in bugs: if 'bugs' in crash: crash['bugs'].extend(bugs[sig]) else: crash['bugs'] = bugs[sig] # Augment with first appearance dates. if sig in sig_date_data: crash['first_report'] = isodate.parse_datetime(sig_date_data[sig]) if 'bugs' in crash: crash['bugs'].sort(reverse=True) context['tcbs'] = tcbs context['days'] = days context['report'] = 'topcrasher' context['possible_days'] = possible_days context['total_crashing_signatures'] = len(signatures) context['total_number_of_crashes'] = api_results['total'] context['process_type_values'] = (x for x in settings.PROCESS_TYPES if x != 'all') context['platform_values'] = settings.DISPLAY_OS_NAMES return render(request, 'topcrashers/topcrashers.html', context)
def search_results(request): '''Return the results of a search. ''' try: params = get_params(request) except ValidationError as e: # There was an error in the form, let's return it. return http.HttpResponseBadRequest(str(e)) context = {} context['query'] = {'total': 0, 'total_count': 0, 'total_pages': 0} current_query = request.GET.copy() if 'page' in current_query: del current_query['page'] context['params'] = current_query.copy() if '_columns' in context['params']: del context['params']['_columns'] if '_facets' in context['params']: del context['params']['_facets'] context['sort'] = list(params['_sort']) # Copy the list of columns so that they can differ. context['columns'] = list(params['_columns']) # The `uuid` field is a special case, it is always displayed in the first # column of the table. Hence we do not want to show it again in the # auto-generated list of columns, so we remove it from the list of # columns to display. if 'uuid' in context['columns']: context['columns'].remove('uuid') try: current_page = int(request.GET.get('page', 1)) except ValueError: return http.HttpResponseBadRequest('Invalid page') if current_page <= 0: current_page = 1 results_per_page = 50 context['current_page'] = current_page context['results_offset'] = results_per_page * (current_page - 1) params['_results_number'] = results_per_page params['_results_offset'] = context['results_offset'] context['current_url'] = '%s?%s' % (reverse('supersearch.search'), urlencode_obj(current_query)) api = SuperSearchUnredacted() try: search_results = api.get(**params) except BadArgumentError as exception: # We need to return the error message in some HTML form for jQuery to # pick it up. return http.HttpResponseBadRequest(render_exception(exception)) if 'signature' in search_results['facets']: # Bugs for each signature signatures = [h['term'] for h in search_results['facets']['signature']] if signatures: bugs = defaultdict(list) bugs_api = models.Bugs() for b in bugs_api.get(signatures=signatures)['hits']: bugs[b['signature']].append(b['id']) for hit in search_results['facets']['signature']: sig = hit['term'] if sig in bugs: if 'bugs' in hit: hit['bugs'].extend(bugs[sig]) else: hit['bugs'] = bugs[sig] # most recent bugs first hit['bugs'].sort(reverse=True) search_results['total_pages'] = int( math.ceil(search_results['total'] / float(results_per_page))) search_results['total_count'] = search_results['total'] context['query'] = search_results return render(request, 'supersearch/search_results.html', context)
api = SuperSearchUnredacted() try: search_results = api.get(**params) except models.BadStatusCodeError, e: # We need to return the error message in some HTML form for jQuery to # pick it up. return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e) if 'signature' in search_results['facets']: # Bugs for each signature signatures = [h['term'] for h in search_results['facets']['signature']] if signatures: bugs = defaultdict(list) bugs_api = models.Bugs() for b in bugs_api.get(signatures=signatures)['hits']: bugs[b['signature']].append(b['id']) for hit in search_results['facets']['signature']: sig = hit['term'] if sig in bugs: if 'bugs' in hit: hit['bugs'].extend(bugs[sig]) else: hit['bugs'] = bugs[sig] search_results['total_pages'] = int( math.ceil(search_results['total'] / float(results_per_page))) search_results['total_count'] = search_results['total']
def search_results(request): products = models.ProductsVersions().get() versions = models.CurrentVersions().get() platforms = models.Platforms().get() form = forms.SearchForm(products, versions, platforms, request.user.is_authenticated(), request.GET) if not form.is_valid(): return http.HttpResponseBadRequest(str(form.errors)) params = {} for key in form.cleaned_data: if hasattr(form.fields[key], 'prefixed_value'): value = form.fields[key].prefixed_value else: value = form.cleaned_data[key] params[key] = value data = {} data['query'] = {'total': 0, 'total_count': 0, 'total_pages': 0} allowed_fields = ALL_POSSIBLE_FIELDS if request.user.is_authenticated(): allowed_fields += ADMIN_RESTRICTED_FIELDS current_query = request.GET.copy() if 'page' in current_query: del current_query['page'] data['params'] = current_query.copy() if '_columns' in data['params']: del data['params']['_columns'] if '_facets' in params: del data['params']['_facets'] params['_facets'] = request.GET.getlist('_facets') or DEFAULT_FACETS data['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS # Make sure only allowed fields are used params['_facets'] = [x for x in params['_facets'] if x in allowed_fields] data['columns'] = [x for x in data['columns'] if x in allowed_fields] try: current_page = int(request.GET.get('page', 1)) except ValueError: return http.HttpResponseBadRequest('Invalid page') if current_page <= 0: current_page = 1 results_per_page = 50 data['current_page'] = current_page data['results_offset'] = results_per_page * (current_page - 1) params['_results_number'] = results_per_page params['_results_offset'] = data['results_offset'] data['current_url'] = '%s?%s' % (reverse('supersearch.search'), current_query.urlencode()) api = SuperSearch() search_results = api.get(**params) if 'signature' in search_results['facets']: # Bugs for each signature signatures = [h['term'] for h in search_results['facets']['signature']] if signatures: bugs = defaultdict(list) bugs_api = models.Bugs() for b in bugs_api.get(signatures=signatures)['hits']: bugs[b['signature']].append(b['id']) for hit in search_results['facets']['signature']: sig = hit['term'] if sig in bugs: if 'bugs' in hit: hit['bugs'].extend(bugs[sig]) else: hit['bugs'] = bugs[sig] search_results['total_pages'] = int( math.ceil(search_results['total'] / float(results_per_page))) search_results['total_count'] = search_results['total'] data['query'] = search_results data['report_list_query_string'] = urllib.urlencode( utils.sanitize_dict(get_report_list_parameters(params)), True) return render(request, 'supersearch/search_results.html', data)
def search_results(request): '''Return the results of a search. ''' try: params = get_params(request) except ValidationError as e: # There was an error in the form, let's return it. return http.HttpResponseBadRequest(str(e)) data = {} data['query'] = { 'total': 0, 'total_count': 0, 'total_pages': 0 } allowed_fields = get_allowed_fields(request.user) current_query = request.GET.copy() if 'page' in current_query: del current_query['page'] data['params'] = current_query.copy() if '_columns' in data['params']: del data['params']['_columns'] if '_facets' in data['params']: del data['params']['_facets'] data['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS # Make sure only allowed fields are used data['columns'] = [ x for x in data['columns'] if x in allowed_fields ] # Copy the list of columns so that they can differ. params['_columns'] = list(data['columns']) # The uuid is always displayed in the UI so we need to make sure it is # always returned by the model. if 'uuid' not in params['_columns']: params['_columns'].append('uuid') # The `uuid` field is a special case, it is always displayed in the first # column of the table. Hence we do not want to show it again in the # auto-generated list of columns, so we its name from the list of # columns to display. if 'uuid' in data['columns']: data['columns'].remove('uuid') try: current_page = int(request.GET.get('page', 1)) except ValueError: return http.HttpResponseBadRequest('Invalid page') if current_page <= 0: current_page = 1 results_per_page = 50 data['current_page'] = current_page data['results_offset'] = results_per_page * (current_page - 1) params['_results_number'] = results_per_page params['_results_offset'] = data['results_offset'] data['current_url'] = '%s?%s' % ( reverse('supersearch.search'), current_query.urlencode() ) api = SuperSearchUnredacted() try: search_results = api.get(**params) except models.BadStatusCodeError as e: # We need to return the error message in some HTML form for jQuery to # pick it up. return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e) if 'signature' in search_results['facets']: # Bugs for each signature signatures = [h['term'] for h in search_results['facets']['signature']] if signatures: bugs = defaultdict(list) bugs_api = models.Bugs() for b in bugs_api.get(signatures=signatures)['hits']: bugs[b['signature']].append(b['id']) for hit in search_results['facets']['signature']: sig = hit['term'] if sig in bugs: if 'bugs' in hit: hit['bugs'].extend(bugs[sig]) else: hit['bugs'] = bugs[sig] search_results['total_pages'] = int(math.ceil( search_results['total'] / float(results_per_page))) search_results['total_count'] = search_results['total'] data['query'] = search_results data['report_list_query_string'] = urllib.urlencode( utils.sanitize_dict( get_report_list_parameters(params) ), True ) return render(request, 'supersearch/search_results.html', data)
def exploitability_report(request, default_context=None): context = default_context or {} if not request.GET.get('product'): url = reverse('exploitability:report') url += '?' + urllib.urlencode({'product': settings.DEFAULT_PRODUCT}) return redirect(url) form = ExploitabilityReportForm( request.GET, active_versions=context['active_versions'], ) if not form.is_valid(): return http.HttpResponseBadRequest(str(form.errors)) product = form.cleaned_data['product'] version = form.cleaned_data['version'] api = SuperSearchUnredacted() params = { 'product': product, 'version': version, '_results_number': 0, # This aggregates on crashes that do NOT contain these # key words. For example, if a crash has # {'exploitability': 'error: unable to analyze dump'} # then it won't get included. 'exploitability': ['!error', '!interesting'], '_aggs.signature': 'exploitability', '_facets_size': settings.EXPLOITABILITY_BATCH_SIZE, } results = api.get(**params) base_signature_report_dict = { 'product': product, } if version: base_signature_report_dict['version'] = version crashes = [] categories = ('high', 'none', 'low', 'medium', 'null') for signature_facet in results['facets']['signature']: # this 'signature_facet' will look something like this: # # { # 'count': 1234, # 'term': 'My | Signature', # 'facets': { # 'exploitability': [ # {'count': 1, 'term': 'high'}, # {'count': 23, 'term': 'medium'}, # {'count': 11, 'term': 'other'}, # # And we only want to include those where: # # low or medium or high are greater than 0 # exploitability = signature_facet['facets']['exploitability'] if not any(x['count'] for x in exploitability if x['term'] in ('high', 'medium', 'low')): continue crash = { 'bugs': [], 'signature': signature_facet['term'], 'high_count': 0, 'medium_count': 0, 'low_count': 0, 'none_count': 0, 'url': (reverse('signature:signature_report') + '?' + urllib.urlencode( dict(base_signature_report_dict, signature=signature_facet['term']))), } for cluster in exploitability: if cluster['term'] in categories: crash['{}_count'.format(cluster['term'])] = (cluster['count']) crash['med_or_high'] = (crash.get('high_count', 0) + crash.get('medium_count', 0)) crashes.append(crash) # Sort by the 'med_or_high' key first (descending), # and by the signature second (ascending). crashes.sort(key=lambda x: (-x['med_or_high'], x['signature'])) # now, let's go back and fill in the bugs signatures = [x['signature'] for x in crashes] if signatures: api = models.Bugs() bugs = defaultdict(list) for b in api.get(signatures=signatures)['hits']: bugs[b['signature']].append(b['id']) for crash in crashes: crash['bugs'] = bugs.get(crash['signature'], []) context['crashes'] = crashes context['product'] = product context['version'] = version context['report'] = 'exploitable' return render(request, 'exploitability/report.html', context)