def profile(request, default_context=None):
context = default_context or {}
context['permissions'] = (
Permission.objects.filter(content_type__model='')
.order_by('name')
)
start_date = (
datetime.datetime.utcnow() - datetime.timedelta(weeks=4)
).isoformat()
api = SuperSearchUnredacted()
results = api.get(
email=request.user.email,
date='>%s' % start_date,
_columns=['date', 'uuid'],
)
context['crashes_list'] = [
dict(zip(('crash_id', 'date'), (x['uuid'], x['date'])))
for x in results['hits']
]
context['your_tokens'] = (
models.Token.objects
.filter(user=request.user)
.order_by('-created')
)
return render(request, 'profile/profile.html', context)
def signature_comments(request, params):
'''Return a list of non-empty comments. '''
signature = params['signature'][0]
context = {}
context['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
context['params'] = current_query.copy()
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
context['current_page'] = current_page
context['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['user_comments'] = '!__null__'
params['_columns'] = ['uuid', 'user_comments', 'date', 'useragent_locale']
params['_results_number'] = results_per_page
params['_results_offset'] = context['results_offset']
params['_facets'] = [] # We don't need no facets.
context['current_url'] = '%s?%s' % (
reverse('signature:signature_report'),
urlencode_obj(current_query)
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
search_results['total_pages'] = int(
math.ceil(
search_results['total'] / float(results_per_page)
)
)
search_results['total_count'] = search_results['total']
context['query'] = search_results
return render(request, 'signature/signature_comments.html', context)
def signature_summary(request, params):
"""Return a list of specific aggregations"""
context = {}
params['signature'] = '=' + params['signature'][0]
params['_aggs.signature'] = [
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
]
params['_results_number'] = 0
params['_facets'] = [
'platform_pretty_version',
'cpu_arch',
'process_type',
'flash_version',
]
params['_histogram.uptime'] = ['product']
params['_histogram_interval.uptime'] = 60
params['_aggs.adapter_vendor_id'] = ['adapter_device_id']
params['_aggs.android_cpu_abi.android_manufacturer.android_model'] = [
'android_version'
]
params['_aggs.product.version'] = ['_cardinality.install_time']
# If the user has permissions, show exploitability.
all_fields = SuperSearchFields().get()
if has_permissions(
request.user, all_fields['exploitability']['permissions_needed']
):
params['_histogram.date'] = ['exploitability']
api = SuperSearchUnredacted()
# Now make the actual request with all expected parameters.
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
facets = search_results['facets']
_transform_uptime_summary(facets)
_transform_graphics_summary(facets)
_transform_mobile_summary(facets)
_transform_exploitability_summary(facets)
context['query'] = search_results
context['product_version_total'] = search_results['total']
if 'signature' in facets and len(facets['signature']) > 0:
context['signature_stats'] = SignatureStats(search_results['facets']['signature'][0],
search_results['total'])
return render(request, 'signature/signature_summary.html', context)
def profile(request, default_context=None):
context = default_context or {}
context["permissions"] = Permission.objects.filter(content_type__model="").order_by("name")
start_date = (datetime.datetime.utcnow() - datetime.timedelta(weeks=4)).isoformat()
api = SuperSearchUnredacted()
results = api.get(email=request.user.email, date=">%s" % start_date, _columns=["date", "uuid"], _sort="-date")
context["crashes_list"] = [dict(zip(("crash_id", "date"), (x["uuid"], x["date"]))) for x in results["hits"]]
return render(request, "profile/profile.html", context)
def signature_comments(request):
'''Return a list of non-empty comments. '''
params = get_validated_params(request)
if isinstance(params, http.HttpResponseBadRequest):
# There was an error in the form, let's return it.
return params
signature = params['signature'][0]
data = {}
data['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
data['params'] = current_query.copy()
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
data['current_page'] = current_page
data['results_offset'] = results_per_page * (current_page - 1)
params['signature'] = '=' + signature
params['user_comments'] = '!__null__'
params['_columns'] = ['uuid', 'user_comments', 'date', 'useragent_locale']
params['_results_number'] = results_per_page
params['_results_offset'] = data['results_offset']
params['_facets'] = [] # We don't need no facets.
data['current_url'] = '%s?%s' % (
reverse('signature:signature_report'),
current_query.urlencode()
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError, e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
def signature_aggregation(request, aggregation):
'''Return the aggregation of a field. '''
params = get_params(request)
if isinstance(params, http.HttpResponseBadRequest):
# There was an error in the form, let's return it.
return params
if len(params['signature']) > 1:
return http.HttpResponseBadRequest(
'Invalid value for "signature" parameter, '
'only one value is accepted'
)
signature = params['signature'][0]
if not signature:
return http.HttpResponseBadRequest(
'"signature" parameter is mandatory'
)
data = {}
data['aggregation'] = aggregation
allowed_fields = get_allowed_fields(request.user)
# Make sure the field we want to aggregate on is allowed.
if aggregation not in allowed_fields:
return http.HttpResponseBadRequest(
'<ul><li>'
'You are not allowed to aggregate on the "%s" field'
'</li></ul>' % aggregation
)
current_query = request.GET.copy()
data['params'] = current_query.copy()
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_results_offset'] = 0
params['_facets'] = [aggregation]
data['current_url'] = '%s?%s' % (
reverse('signature:signature_report'),
current_query.urlencode()
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError, e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
def test_topcrasher_by_build(self):
def mocked_supersearch_get(**params):
ok_('build_id' in params)
return {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '19.0',
'_range_type': 'build',
})
eq_(response.status_code, 200)
# Test with a version that does not support builds.
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '18.0',
'_range_type': 'build',
})
eq_(response.status_code, 200)
ok_('versions do not support the by build date' in response.content)
ok_('Range Type:' not in response.content)
def test_topcrasher_without_any_signatures(self, rpost):
url = self.base_url + '?product=WaterWolf&version=19.0'
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
})
ok_(url in response['Location'])
rpost.side_effect = mocked_post_123
def mocked_supersearch_get(**params):
return {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '19.0',
})
eq_(response.status_code, 200)
def test_search_results_pagination(self):
"""Test that the pagination of results works as expected.
"""
def mocked_supersearch_get(**params):
assert '_columns' in params
# Make sure a negative page does not lead to negative offset value.
# But instead it is considered as the page 1 and thus is not added.
assert params.get('_results_offset') == 0
hits = []
for i in range(140):
hits.append({
"signature": "hang | nsASDOMWindowEnumerator::GetNext()",
"date": "2017-01-31T23:12:57",
"uuid": i,
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981
})
return {
"hits": self.only_certain_columns(hits, params['_columns']),
"facets": "",
"total": len(hits)
}
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
url = reverse('supersearch:search_results')
response = self.client.get(
url,
{
'signature': 'hang | nsASDOMWindowEnumerator::GetNext()',
'_columns': ['version'],
'_facets': ['platform'],
}
)
assert response.status_code == 200
assert '140' in smart_text(response.content)
# Check that the pagination URL contains all three expected parameters.
doc = pyquery.PyQuery(response.content)
next_page_url = str(doc('.pagination a').eq(0))
assert '_facets=platform' in next_page_url
assert '_columns=version' in next_page_url
assert 'page=2' in next_page_url
assert '#crash-reports' in next_page_url
# Verify white spaces are correctly encoded.
# Note we use `quote` and not `quote_plus`, so white spaces are
# turned into '%20' instead of '+'.
assert quote('hang | nsASDOMWindowEnumerator::GetNext()') in next_page_url
# Test that a negative page value does not break it.
response = self.client.get(url, {'page': '-1'})
assert response.status_code == 200
def test_SuperSearchUnredacted(self):
def mocked_supersearch_get(**params):
assert 'exploitability' in params
if 'product' in params:
assert params['product'] == ['WaterWolf', 'NightTrain']
return {
'hits': [
{
'signature': 'abcdef',
'product': 'WaterWolf',
'version': '1.0',
'email': '*****@*****.**',
'exploitability': 'high',
'url': 'http://embarassing.website.com',
'user_comments': 'hey I am [email protected]',
}
],
'facets': {
'signature': []
},
'total': 0
}
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
url = reverse('api:model_wrapper', args=('SuperSearchUnredacted',))
response = self.client.get(url, {'exploitability': 'high'})
assert response.status_code == 403
assert response['Content-Type'] == 'application/json'
error = json.loads(response.content)['error']
permission = Permission.objects.get(
codename='view_exploitability'
)
assert permission.name in error
# Log in to get permissions.
user = self._login()
self._add_permission(user, 'view_pii')
self._add_permission(user, 'view_exploitability')
response = self.client.get(url, {'exploitability': 'high'})
assert response.status_code == 200
res = json.loads(response.content)
assert res['hits']
assert res['facets']
# Verify forbidden fields are exposed.
assert 'email' in res['hits'][0]
assert 'exploitability' in res['hits'][0]
assert 'url' in res['hits'][0]
assert '*****@*****.**' in res['hits'][0]['user_comments']
# Verify values can be lists.
response = self.client.get(url, {
'exploitability': 'high',
'product': ['WaterWolf', 'NightTrain']
})
assert response.status_code == 200
def test_signature_reports_pagination(self):
"""Test that the pagination of results works as expected.
"""
def mocked_supersearch_get(**params):
assert '_columns' in params
# Make sure a negative page does not lead to negative offset value.
# But instead it is considered as the page 1 and thus is not added.
eq_(params.get('_results_offset'), 0)
hits = []
for i in range(140):
hits.append({
"signature": "nsASDOMWindowEnumerator::GetNext()",
"date": "2017-01-31T23:12:57",
"uuid": i,
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981
})
return {
"hits": self.only_certain_columns(hits, params['_columns']),
"facets": "",
"total": len(hits)
}
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
url = reverse('signature:signature_reports')
response = self.client.get(
url,
{
'signature': DUMB_SIGNATURE,
'product': ['WaterWolf'],
'_columns': ['platform']
}
)
eq_(response.status_code, 200)
ok_('140' in response.content)
# Check that the pagination URL contains all three expected parameters.
doc = pyquery.PyQuery(response.content)
next_page_url = str(doc('.pagination a').eq(0))
ok_('product=WaterWolf' in next_page_url)
ok_('_columns=platform' in next_page_url)
ok_('page=2' in next_page_url)
# Test that a negative page value does not break it.
response = self.client.get(url, {
'signature': DUMB_SIGNATURE,
'page': '-1',
})
eq_(response.status_code, 200)
def test_signature_aggregation(self):
def mocked_supersearch_get(**params):
assert 'signature' in params
assert params['signature'] == ['=' + DUMB_SIGNATURE]
assert '_facets' in params
if 'product' in params['_facets']:
return {
"hits": [],
"facets": {
"product": [
{
"term": "windows",
"count": 42,
},
{
"term": "linux",
"count": 1337,
},
{
"term": "mac",
"count": 3,
},
]
},
"total": 1382
}
# the default
return {
"hits": [],
"facets": {
"platform": []
},
"total": 0
}
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
# Test with no results.
url = reverse('signature:signature_aggregation', args=('platform',))
response = self.client.get(url, {'signature': DUMB_SIGNATURE})
assert response.status_code == 200
assert 'Product' not in smart_text(response.content)
assert 'No results were found' in smart_text(response.content)
# Test with results.
url = reverse('signature:signature_aggregation', args=('product',))
response = self.client.get(url, {'signature': DUMB_SIGNATURE})
assert response.status_code == 200
assert 'Product' in smart_text(response.content)
assert '1337' in smart_text(response.content)
assert 'linux' in smart_text(response.content)
assert str(int(1337 / 1382 * 100)) in smart_text(response.content)
assert 'windows' in smart_text(response.content)
assert 'mac' in smart_text(response.content)
def test_signature_comments_pagination(self):
"""Test that the pagination of comments works as expected"""
def mocked_supersearch_get(**params):
assert '_columns' in params
if params.get('_results_offset') != 0:
hits_range = range(100, 140)
else:
hits_range = range(100)
hits = []
for i in hits_range:
hits.append({
"date": "2017-01-31T23:12:57",
"uuid": i,
"user_comments": "hi",
})
return {
'hits': self.only_certain_columns(hits, params['_columns']),
'total': 140
}
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
user = self._login()
user.groups.add(self._create_group_with_permission('view_pii'))
assert user.has_perm('crashstats.view_pii')
url = reverse('signature:signature_comments')
response = self.client.get(
url,
{
'signature': DUMB_SIGNATURE,
'product': ['WaterWolf'],
}
)
assert response.status_code == 200
assert '140' in smart_text(response.content)
assert '99' in smart_text(response.content)
assert '139' not in smart_text(response.content)
# Check that the pagination URL contains all expected parameters.
doc = pyquery.PyQuery(response.content)
next_page_url = str(doc('.pagination a').eq(0))
assert 'product=WaterWolf' in next_page_url
assert 'page=2' in next_page_url
response = self.client.get(url, {
'signature': DUMB_SIGNATURE,
'page': '2',
})
assert response.status_code == 200
assert '140' in smart_text(response.content)
assert '99' not in smart_text(response.content)
assert '139' in smart_text(response.content)
def test_signature_comments_pagination(self):
"""Test that the pagination of comments works as expected. """
def mocked_supersearch_get(**params):
assert '_columns' in params
if params.get('_results_offset') != 0:
hits_range = range(100, 140)
else:
hits_range = range(100)
hits = []
for i in hits_range:
hits.append({
"date": "2017-01-31T23:12:57",
"uuid": i,
"user_comments": "hi",
})
return {
"hits": self.only_certain_columns(hits, params['_columns']),
"total": 140
}
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
url = reverse('signature:signature_comments')
response = self.client.get(
url,
{
'signature': DUMB_SIGNATURE,
'product': ['WaterWolf'],
}
)
eq_(response.status_code, 200)
ok_('140' in response.content)
ok_('99' in response.content)
ok_('139' not in response.content)
# Check that the pagination URL contains all expected parameters.
doc = pyquery.PyQuery(response.content)
next_page_url = str(doc('.pagination a').eq(0))
ok_('product=WaterWolf' in next_page_url)
ok_('page=2' in next_page_url)
response = self.client.get(url, {
'signature': DUMB_SIGNATURE,
'page': '2',
})
eq_(response.status_code, 200)
ok_('140' in response.content)
ok_('99' not in response.content)
ok_('139' in response.content)
def signature_aggregation(request, params, aggregation):
'''Return the aggregation of a field. '''
signature = params['signature'][0]
context = {}
context['aggregation'] = aggregation
allowed_fields = get_allowed_fields(request.user)
# Make sure the field we want to aggregate on is allowed.
if aggregation not in allowed_fields:
return http.HttpResponseBadRequest(
'<ul><li>'
'You are not allowed to aggregate on the "%s" field'
'</li></ul>' % aggregation
)
current_query = request.GET.copy()
context['params'] = current_query.copy()
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_results_offset'] = 0
params['_facets'] = [aggregation]
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
context['aggregates'] = []
if aggregation in search_results['facets']:
context['aggregates'] = search_results['facets'][aggregation]
context['total_count'] = search_results['total']
return render(request, 'signature/signature_aggregation.html', context)
def test_search_custom(self):
def mocked_supersearch_get(**params):
return None
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
self.create_custom_query_perm()
url = reverse('supersearch:search_custom')
response = self.client.get(url)
assert response.status_code == 200
assert 'Run a search to get some results' in smart_text(response.content)
def signature_graphs(request, params, field):
'''Return a multi-line graph of crashes per day grouped by field. '''
signature = params['signature'][0]
context = {}
context['aggregation'] = field
allowed_fields = get_allowed_fields(request.user)
# Make sure the field we want to aggregate on is allowed.
if field not in allowed_fields:
return http.HttpResponseBadRequest(
'<ul><li>'
'You are not allowed to group by the "%s" field'
'</li></ul>' % field
)
current_query = request.GET.copy()
context['params'] = current_query.copy()
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_results_offset'] = 0
params['_histogram.date'] = [field]
params['_facets'] = [field]
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
context['aggregates'] = search_results['facets'].get('histogram_date', [])
context['term_counts'] = search_results['facets'].get(field, [])
return context
def signature_graphs(request, field):
'''Return a multi-line graph of crashes per day grouped by field. '''
params = get_validated_params(request)
if isinstance(params, http.HttpResponseBadRequest):
# There was an error in the form, let's return it.
return params
signature = params['signature'][0]
data = {}
data['aggregation'] = field
allowed_fields = get_allowed_fields(request.user)
# Make sure the field we want to aggregate on is allowed.
if field not in allowed_fields:
return http.HttpResponseBadRequest(
'<ul><li>'
'You are not allowed to group by the "%s" field'
'</li></ul>' % field
)
current_query = request.GET.copy()
data['params'] = current_query.copy()
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_results_offset'] = 0
params['_histogram.date'] = [field]
params['_facets'] = [field]
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError, e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
def search_custom(request, default_context=None):
"""Return the basic search page, without any result"""
error = None
query = None
try:
params = get_params(request)
except ValidationError as e:
# There was an error in the form, but we want to do the default
# behavior and just display an error message.
error = str(e)
else:
# Get the JSON query that supersearch generates and show it.
params['_return_query'] = 'true'
api = SuperSearchUnredacted()
try:
query = api.get(**params)
except BadArgumentError as e:
error = e
schema = settings.ELASTICSEARCH_INDEX_SCHEMA
now = timezone.now()
possible_indices = []
for i in range(26):
index = (now - datetime.timedelta(weeks=i)).strftime(schema)
possible_indices.append({'id': index, 'text': index})
context = default_context
context['elasticsearch_indices'] = possible_indices
if query:
context['query'] = json.dumps(query['query'])
context['indices'] = ','.join(query['indices'])
context['error'] = error
return render(request, 'supersearch/search_custom.html', context)
def signature_correlations(request, params):
'''Return a list of correlations combos, to be populated by AJAX calls. '''
signature = params['signature'][0]
context = {}
params['signature'] = '=' + signature
params['_results_number'] = 0
params['_facets'] = []
params['_aggs.product.version'] = 'platform'
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except BadArgumentError as e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest(render_exception(e))
all_combos = []
for product in search_results['facets']['product']:
for version in product['facets']['version']:
for platform in version['facets']['platform']:
all_combos.append({
'product': product['term'],
'version': version['term'],
'platform': platform['term'],
'count': platform['count'],
})
all_combos = sorted(all_combos, key=lambda x: x['count'])
context['correlation_combos'] = (
all_combos[:settings.MAX_CORRELATION_COMBOS_PER_SIGNATURE]
)
return render(request, 'signature/signature_correlations.html', context)
def test_search_custom(self):
def mocked_supersearch_get(**params):
return None
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
self.create_custom_query_perm()
url = reverse('supersearch.search_custom')
response = self.client.get(url)
assert response.status_code == 200
assert 'Run a search to get some results' in response.content
def profile(request, default_context=None):
context = default_context or {}
context['permissions'] = (Permission.objects.filter(
content_type__model='').order_by('name'))
start_date = (datetime.datetime.utcnow() -
datetime.timedelta(weeks=4)).isoformat()
api = SuperSearchUnredacted()
results = api.get(
email=request.user.email,
date='>%s' % start_date,
_columns=['date', 'uuid'],
)
context['crashes_list'] = [
dict(zip(('crash_id', 'date'), (x['uuid'], x['date'])))
for x in results['hits']
]
context['your_tokens'] = (models.Token.objects.filter(
user=request.user).order_by('-created'))
return render(request, 'profile/profile.html', context)
def test_search_custom(self):
def mocked_supersearch_get(**params):
return None
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
self.create_custom_query_perm()
url = reverse('supersearch.search_custom')
response = self.client.get(url)
eq_(response.status_code, 200)
ok_('Run a search to get some results' in response.content)
def test_search_results_parameters(self, rpost):
def mocked_post(**options):
assert 'bugs' in options['url'], options['url']
return Response({
"hits": [],
"total": 0
})
rpost.side_effect = mocked_post
def mocked_supersearch_get(**params):
# Verify that all expected parameters are in the URL.
ok_('product' in params)
ok_('WaterWolf' in params['product'])
ok_('NightTrain' in params['product'])
ok_('address' in params)
ok_('0x0' in params['address'])
ok_('0xa' in params['address'])
ok_('reason' in params)
ok_('^hello' in params['reason'])
ok_('$thanks' in params['reason'])
ok_('java_stack_trace' in params)
ok_('Exception' in params['java_stack_trace'])
return {
"hits": [],
"facets": "",
"total": 0
}
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
url = reverse('supersearch.search_results')
response = self.client.get(
url, {
'product': ['WaterWolf', 'NightTrain'],
'address': ['0x0', '0xa'],
'reason': ['^hello', '$thanks'],
'java_stack_trace': 'Exception',
}
)
eq_(response.status_code, 200)
def test_product_sans_featured_version(self):
def mocked_supersearch_get(**params):
if '_columns' not in params:
params['_columns'] = []
# By default we range by date, so there should be no filter on
# the build id.
assert 'build_id' not in params
if 'hang_type' not in params['_aggs.signature']:
# Return results for the previous week.
results = {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
else:
# Return results for the current week.
results = {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
results['hits'] = self.only_certain_columns(
results['hits'],
params['_columns']
)
return results
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
self.set_product_versions(['20.0', '19.0', '18.0'])
# Redirects to the most recent featured version
response = self.client.get(self.base_url, {'product': 'SeaMonkey'})
assert response.status_code == 302
actual_url = self.base_url + '?product=SeaMonkey&version=20.0'
assert actual_url in response['Location']
# This version doesn't exist, but it still renders something and
# doesn't throw an error
response = self.client.get(self.base_url, {'product': 'SeaMonkey', 'version': '9.5'})
assert response.status_code == 200
def test_topcrashers_product_sans_featured_version(self):
def mocked_supersearch_get(**params):
if '_columns' not in params:
params['_columns'] = []
# By default we range by date, so there should be no filter on
# the build id.
ok_('build_id' not in params)
if 'hang_type' not in params['_aggs.signature']:
# Return results for the previous week.
results = {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
else:
# Return results for the current week.
results = {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
results['hits'] = self.only_certain_columns(
results['hits'],
params['_columns']
)
return results
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
response = self.client.get(self.base_url, {'product': 'SeaMonkey'})
eq_(response.status_code, 302)
actual_url = self.base_url + '?product=SeaMonkey&version=9.5'
ok_(actual_url in response['Location'])
response = self.client.get(self.base_url, {
'product': 'SeaMonkey',
'version': '9.5',
})
eq_(response.status_code, 200)
def test_product_sans_featured_version(self):
def mocked_supersearch_get(**params):
if '_columns' not in params:
params['_columns'] = []
# By default we range by date, so there should be no filter on
# the build id.
assert 'build_id' not in params
if 'hang_type' not in params['_aggs.signature']:
# Return results for the previous week.
results = {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
else:
# Return results for the current week.
results = {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
results['hits'] = self.only_certain_columns(
results['hits'],
params['_columns']
)
return results
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
self.set_product_versions(['20.0', '19.0', '18.0'])
# Redirects to the most recent featured version
response = self.client.get(self.base_url, {'product': 'SeaMonkey'})
assert response.status_code == 302
actual_url = self.base_url + '?product=SeaMonkey&version=20.0'
assert actual_url in response['Location']
# This version doesn't exist, but it still renders something and
# doesn't throw an error
response = self.client.get(self.base_url, {'product': 'SeaMonkey', 'version': '9.5'})
assert response.status_code == 200
def test_search_custom_permission(self):
def mocked_supersearch_get(**params):
return None
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
url = reverse("supersearch:search_custom")
response = self.client.get(url)
assert response.status_code == 302
self.create_custom_query_perm()
response = self.client.get(url)
assert response.status_code == 200
assert "Run a search to get some results" in smart_text(response.content)
def test_search_results_ratelimited(self):
def mocked_supersearch_get(**params):
return {"hits": [], "facets": [], "total": 0}
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
url = reverse("supersearch:search_results")
limit = int(re.findall(r"(\d+)", settings.RATELIMIT_SUPERSEARCH)[0])
params = {"product": "WaterWolf"}
# double to avoid https://bugzilla.mozilla.org/show_bug.cgi?id=1148470
for i in range(limit * 2):
self.client.get(url, params)
response = self.client.get(url, params, HTTP_X_REQUESTED_WITH="XMLHttpRequest")
assert response.status_code == 429
assert smart_text(response.content) == "Too Many Requests"
assert response["content-type"] == "text/plain"
def test_search_results_badargumenterror(self):
def mocked_supersearch_get(**params):
raise BadArgumentError('<script>xss')
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get)
url = reverse('supersearch.search_results')
params = {'product': 'WaterWolf'}
response = self.client.get(url,
params,
HTTP_X_REQUESTED_WITH='XMLHttpRequest')
eq_(response.status_code, 400)
eq_(response['content-type'], 'text/html; charset=utf-8')
ok_('<script>' not in response.content)
ok_('<script>' in response.content)
def test_search_results_badargumenterror(self):
def mocked_supersearch_get(**params):
raise BadArgumentError("<script>xss")
SuperSearchUnredacted.implementation(
).get.side_effect = mocked_supersearch_get
url = reverse("supersearch:search_results")
params = {"product": "WaterWolf"}
response = self.client.get(url,
params,
HTTP_X_REQUESTED_WITH="XMLHttpRequest")
assert response.status_code == 400
assert response["content-type"] == "text/html; charset=utf-8"
assert "<script>" not in smart_text(response.content)
assert "<script>" in smart_text(response.content)
def test_without_any_signatures(self):
def mocked_supersearch_get(**params):
return {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '19.0'
})
assert response.status_code == 200
def test_search_results_badargumenterror(self):
def mocked_supersearch_get(**params):
raise BadArgumentError('<script>xss')
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
url = reverse('supersearch.search_results')
params = {'product': 'WaterWolf'}
self.client.get(url, params)
response = self.client.get(url, params)
eq_(response.status_code, 400)
eq_(response['content-type'], 'text/html; charset=utf-8')
ok_('<script>' not in response.content)
ok_('<script>' in response.content)
def test_without_any_signatures(self):
def mocked_supersearch_get(**params):
return {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '19.0'
})
assert response.status_code == 200
def test_by_build(self):
def mocked_supersearch_get(**params):
assert "build_id" in params
return {"hits": [], "facets": {"signature": []}, "total": 0}
SuperSearchUnredacted.implementation(
).get.side_effect = mocked_supersearch_get
response = self.client.get(
self.base_url,
{
"product": "WaterWolf",
"version": "19.0",
"_range_type": "build"
},
)
assert response.status_code == 200
def test_topcrasher_without_any_signatures(self):
url = self.base_url + '?product=WaterWolf&version=19.0'
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
})
assert url in response['Location']
def mocked_supersearch_get(**params):
return {'hits': [], 'facets': {'signature': []}, 'total': 0}
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get)
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '19.0',
})
assert response.status_code == 200
def test_search_results_badargumenterror(self):
def mocked_supersearch_get(**params):
raise BadArgumentError('<script>xss')
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
url = reverse('supersearch:search_results')
params = {'product': 'WaterWolf'}
response = self.client.get(
url,
params,
HTTP_X_REQUESTED_WITH='XMLHttpRequest'
)
assert response.status_code == 400
assert response['content-type'] == 'text/html; charset=utf-8'
assert '<script>' not in smart_text(response.content)
assert '<script>' in smart_text(response.content)
def test_signature_summary_with_many_hexes(self):
def mocked_supersearch_get(**params):
assert 'signature' in params
assert params['signature'] == ['=' + DUMB_SIGNATURE]
adapters = [
{
'term': '0x{0:0>4}'.format(i),
'count': 1
}
for i in range(50)
]
vendors = [
{
'term': '0x{0:0>4}'.format(i),
'count': 50,
'facets': {
'adapter_device_id': adapters
}
}
for i in range(3)
]
res = {
'hits': [],
'total': 4,
'facets': {
'adapter_vendor_id': vendors,
}
}
return res
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
# Test with no results
url = reverse('signature:signature_summary')
response = self.client.get(url, {
'signature': DUMB_SIGNATURE,
'product': 'WaterWolf',
'version': '1.0',
})
assert response.status_code == 200
def test_signature_summary_with_many_hexes(self):
def mocked_supersearch_get(**params):
assert 'signature' in params
assert params['signature'] == ['=' + DUMB_SIGNATURE]
adapters = [
{
'term': '0x{0:0>4}'.format(i),
'count': 1
}
for i in range(50)
]
vendors = [
{
'term': '0x{0:0>4}'.format(i),
'count': 50,
'facets': {
'adapter_device_id': adapters
}
}
for i in range(3)
]
res = {
'hits': [],
'total': 4,
'facets': {
'adapter_vendor_id': vendors,
}
}
return res
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
# Test with no results
url = reverse('signature:signature_summary')
response = self.client.get(url, {
'signature': DUMB_SIGNATURE,
'product': 'WaterWolf',
'version': '1.0',
})
assert response.status_code == 200
def test_signature_aggregation(self):
def mocked_supersearch_get(**params):
assert "signature" in params
assert params["signature"] == ["=" + DUMB_SIGNATURE]
assert "_facets" in params
if "product" in params["_facets"]:
return {
"hits": [],
"facets": {
"product": [
{"term": "windows", "count": 42},
{"term": "linux", "count": 1337},
{"term": "mac", "count": 3},
]
},
"total": 1382,
}
# the default
return {"hits": [], "facets": {"platform": []}, "total": 0}
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
# Test with no results.
url = reverse("signature:signature_aggregation", args=("platform",))
response = self.client.get(url, {"signature": DUMB_SIGNATURE})
assert response.status_code == 200
assert "Product" not in smart_text(response.content)
assert "No results were found" in smart_text(response.content)
# Test with results.
url = reverse("signature:signature_aggregation", args=("product",))
response = self.client.get(url, {"signature": DUMB_SIGNATURE})
assert response.status_code == 200
assert "Product" in smart_text(response.content)
assert "1337" in smart_text(response.content)
assert "linux" in smart_text(response.content)
assert str(int(1337 / 1382 * 100)) in smart_text(response.content)
assert "windows" in smart_text(response.content)
assert "mac" in smart_text(response.content)
def test_by_build(self):
def mocked_supersearch_get(**params):
assert 'build_id' in params
return {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '19.0',
'_range_type': 'build',
})
assert response.status_code == 200
def test_search_results_ratelimited(self):
def mocked_supersearch_get(**params):
return {"hits": [], "facets": [], "total": 0}
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get)
url = reverse('supersearch.search_results')
limit = int(re.findall('(\d+)', settings.RATELIMIT_SUPERSEARCH)[0])
params = {'product': 'WaterWolf'}
# double to avoid https://bugzilla.mozilla.org/show_bug.cgi?id=1148470
for i in range(limit * 2):
self.client.get(url, params)
response = self.client.get(url,
params,
HTTP_X_REQUESTED_WITH='XMLHttpRequest')
eq_(response.status_code, 429)
eq_(response.content, 'Too Many Requests')
eq_(response['content-type'], 'text/plain')
def test_by_build(self):
def mocked_supersearch_get(**params):
assert 'build_id' in params
return {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '19.0',
'_range_type': 'build',
})
assert response.status_code == 200
def test_parameters(self):
def mocked_supersearch_get(**params):
# Verify that all expected parameters are in the URL.
assert 'product' in params
assert 'WaterWolf' in params['product']
assert 'NightTrain' in params['product']
assert 'address' in params
assert '0x0' in params['address']
assert '0xa' in params['address']
assert 'reason' in params
assert '^hello' in params['reason']
assert '$thanks' in params['reason']
assert 'java_stack_trace' in params
assert 'Exception' in params['java_stack_trace']
return {
"hits": [],
"facets": "",
"total": 0
}
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
url = reverse('signature:signature_reports')
response = self.client.get(
url, {
'signature': DUMB_SIGNATURE,
'product': ['WaterWolf', 'NightTrain'],
'address': ['0x0', '0xa'],
'reason': ['^hello', '$thanks'],
'java_stack_trace': 'Exception',
}
)
assert response.status_code == 200
def test_topcrasher_modes(self, rpost):
rpost.side_effect = mocked_post_123
def mocked_supersearch_get(**params):
return {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
now = datetime.datetime.utcnow()
today = now.replace(hour=0, minute=0, second=0, microsecond=0)
timestr = '%Y-%m-%d %H:%M:%S'
now = now.strftime(timestr)
today = today.strftime(timestr)
with freezegun.freeze_time(now, tz_offset=0):
# By default, it returns "real-time" data.
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '19.0',
})
eq_(response.status_code, 200)
ok_(now in response.content, now)
ok_(today not in response.content)
# Now test the "day time" data.
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '19.0',
'_tcbs_mode': 'byday',
})
eq_(response.status_code, 200)
ok_(today in response.content)
ok_(now not in response.content)
def test_search_custom_parameters(self):
self.create_custom_query_perm()
def mocked_supersearch_get(**params):
assert "_return_query" in params
assert "signature" in params
assert params["signature"] == ["nsA"]
return {
"query": {"query": None},
"indices": ["socorro200000", "socorro200001"],
}
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
url = reverse("supersearch:search_custom")
response = self.client.get(url, {"signature": "nsA"})
assert response.status_code == 200
assert "Run a search to get some results" in smart_text(response.content)
assert "{"query": null}" in smart_text(response.content)
assert "socorro200000" in smart_text(response.content)
assert "socorro200001" in smart_text(response.content)
def test_search_custom_parameters(self):
self.create_custom_query_perm()
def mocked_supersearch_get(**params):
assert '_return_query' in params
assert 'signature' in params
assert params['signature'] == ['nsA']
return {
"query": {"query": None},
"indices": ["socorro200000", "socorro200001"]
}
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
url = reverse('supersearch:search_custom')
response = self.client.get(url, {'signature': 'nsA'})
assert response.status_code == 200
assert 'Run a search to get some results' in smart_text(response.content)
assert '{"query": null}' in smart_text(response.content)
assert 'socorro200000' in smart_text(response.content)
assert 'socorro200001' in smart_text(response.content)
def test_signature_summary_with_many_hexes(self):
def mocked_supersearch_get(**params):
assert "signature" in params
assert params["signature"] == ["=" + DUMB_SIGNATURE]
adapters = [{
"term": "0x{0:0>4}".format(i),
"count": 1
} for i in range(50)]
vendors = [{
"term": "0x{0:0>4}".format(i),
"count": 50,
"facets": {
"adapter_device_id": adapters
},
} for i in range(3)]
res = {
"hits": [],
"total": 4,
"facets": {
"adapter_vendor_id": vendors
}
}
return res
SuperSearchUnredacted.implementation(
).get.side_effect = mocked_supersearch_get
# Test with no results
url = reverse("signature:signature_summary")
response = self.client.get(url, {
"signature": DUMB_SIGNATURE,
"product": "WaterWolf",
"version": "1.0"
})
assert response.status_code == 200
def check_crashids(entropy_chunk, date):
"""Checks crash ids for a given entropy and date."""
s3_context = get_s3_context()
bucket = s3_context.config.bucket_name
s3_client = s3_context.client
supersearch = SuperSearchUnredacted()
missing = []
for entropy in entropy_chunk:
raw_crash_key_prefix = RAW_CRASH_PREFIX_TEMPLATE % (entropy, date)
paginator = s3_client.get_paginator("list_objects_v2")
page_iterator = paginator.paginate(Bucket=bucket,
Prefix=raw_crash_key_prefix)
for page in page_iterator:
# NOTE(willkg): Keys look like /v2/raw_crash/ENTRPOY/DATE/CRASHID
crash_ids = [
item["Key"].split("/")[-1]
for item in page.get("Contents", [])
]
if not crash_ids:
continue
# Check S3 first
for crash_id in crash_ids:
if not is_in_s3(s3_client, bucket, crash_id):
missing.append(crash_id)
# Check Elasticsearch in batches
for crash_ids_batch in chunked(crash_ids, 100):
missing_in_es = check_elasticsearch(supersearch,
crash_ids_batch)
missing.extend(missing_in_es)
return list(set(missing))
def test_product_sans_featured_version(self):
def mocked_supersearch_get(**params):
if "_columns" not in params:
params["_columns"] = []
# By default we range by date, so there should be no filter on
# the build id.
assert "build_id" not in params
if "hang_type" not in params["_aggs.signature"]:
# Return results for the previous week.
results = {"hits": [], "facets": {"signature": []}, "total": 0}
else:
# Return results for the current week.
results = {"hits": [], "facets": {"signature": []}, "total": 0}
results["hits"] = self.only_certain_columns(
results["hits"], params["_columns"])
return results
SuperSearchUnredacted.implementation(
).get.side_effect = mocked_supersearch_get
self.set_product_versions(["20.0", "19.0", "18.0"])
# Redirects to the most recent featured version
response = self.client.get(self.base_url, {"product": "SeaMonkey"})
assert response.status_code == 302
actual_url = self.base_url + "?product=SeaMonkey&version=20.0"
assert actual_url in response["Location"]
# This version doesn't exist, but it still renders something and
# doesn't throw an error
response = self.client.get(self.base_url, {
"product": "SeaMonkey",
"version": "9.5"
})
assert response.status_code == 200
def test_modes(self):
def mocked_supersearch_get(**params):
return {
'hits': [],
'facets': {
'signature': []
},
'total': 0
}
SuperSearchUnredacted.implementation().get.side_effect = mocked_supersearch_get
now = datetime.datetime.utcnow().replace(microsecond=0)
today = now.replace(hour=0, minute=0, second=0)
with freezegun.freeze_time(now, tz_offset=0):
now = now.isoformat()
today = today.isoformat()
# By default, it returns "real-time" data.
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '19.0',
})
assert response.status_code == 200
assert now in smart_text(response.content)
assert today not in smart_text(response.content)
# Now test the "day time" data.
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '19.0',
'_tcbs_mode': 'byday',
})
assert response.status_code == 200
assert today in smart_text(response.content)
assert now not in smart_text(response.content)
def test_search_custom_parameters(self):
self.create_custom_query_perm()
def mocked_supersearch_get(**params):
ok_('_return_query' in params)
ok_('signature' in params)
eq_(params['signature'], ['nsA'])
return {
"query": {"query": None},
"indices": ["socorro200000", "socorro200001"]
}
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
url = reverse('supersearch.search_custom')
response = self.client.get(url, {'signature': 'nsA'})
eq_(response.status_code, 200)
ok_('Run a search to get some results' in response.content)
ok_('{"query": null}' in response.content)
ok_('socorro200000' in response.content)
ok_('socorro200001' in response.content)
def test_topcrasher_by_build(self):
def mocked_supersearch_get(**params):
assert 'build_id' in params
return {'hits': [], 'facets': {'signature': []}, 'total': 0}
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get)
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '19.0',
'_range_type': 'build',
})
assert response.status_code == 200
# Test with a version that does not support builds.
response = self.client.get(self.base_url, {
'product': 'WaterWolf',
'version': '18.0',
'_range_type': 'build',
})
assert response.status_code == 200
assert 'versions do not support the by build date' in response.content
assert 'Range Type:' not in response.content
def get_topcrashers_results(**kwargs):
"""Return the results of a search. """
results = []
params = kwargs
range_type = params.pop('_range_type')
dates = get_date_boundaries(params)
params['_aggs.signature'] = [
'platform',
'is_garbage_collecting',
'hang_type',
'process_type',
'startup_crash',
'_histogram.uptime',
'_cardinality.install_time',
]
params['_histogram_interval.uptime'] = 60
# We don't care about no results, only facets.
params['_results_number'] = 0
if params.get('process_type') in ('any', 'all'):
params['process_type'] = None
if range_type == 'build':
params['build_id'] = [
'>=' + datetime_to_build_id(dates[0]),
'<' + datetime_to_build_id(dates[1])
]
api = SuperSearchUnredacted()
search_results = api.get(**params)
if search_results['total'] > 0:
results = search_results['facets']['signature']
platforms = models.Platforms().get_all()['hits']
platform_codes = [
x['code'] for x in platforms if x['code'] != 'unknown'
]
for i, hit in enumerate(results):
hit['signature'] = hit['term']
hit['rank'] = i + 1
hit['percent'] = 100.0 * hit['count'] / search_results['total']
# Number of crash per platform.
for platform in platform_codes:
hit[platform + '_count'] = 0
sig_platforms = hit['facets']['platform']
for platform in sig_platforms:
code = platform['term'][:3].lower()
if code in platform_codes:
hit[code + '_count'] = platform['count']
# Number of crashes happening during garbage collection.
hit['is_gc_count'] = 0
sig_gc = hit['facets']['is_garbage_collecting']
for row in sig_gc:
if row['term'].lower() == 't':
hit['is_gc_count'] = row['count']
# Number of plugin crashes.
hit['plugin_count'] = 0
sig_process = hit['facets']['process_type']
for row in sig_process:
if row['term'].lower() == 'plugin':
hit['plugin_count'] = row['count']
# Number of hang crashes.
hit['hang_count'] = 0
sig_hang = hit['facets']['hang_type']
for row in sig_hang:
# Hangs have weird values in the database: a value of 1 or -1
# means it is a hang, a value of 0 or missing means it is not.
if row['term'] in (1, -1):
hit['hang_count'] += row['count']
# Number of crashes happening during startup. This is defined by
# the client, as opposed to the next method which relies on
# the uptime of the client.
hit['startup_count'] = sum(
row['count'] for row in hit['facets']['startup_crash']
if row['term'] in ('T', '1'))
# Is a startup crash if more than half of the crashes are happening
# in the first minute after launch.
hit['startup_crash'] = False
sig_uptime = hit['facets']['histogram_uptime']
for row in sig_uptime:
# Aggregation buckets use the lowest value of the bucket as
# term. So for everything between 0 and 60 excluded, the
# term will be `0`.
if row['term'] < 60:
ratio = 1.0 * row['count'] / hit['count']
hit['startup_crash'] = ratio > 0.5
# Number of distinct installations.
hit['installs_count'] = (
hit['facets']['cardinality_install_time']['value'])
# Run the same query but for the previous date range, so we can
# compare the rankings and show rank changes.
delta = (dates[1] - dates[0]) * 2
params['date'] = [
'>=' + (dates[1] - delta).isoformat(), '<' + dates[0].isoformat()
]
params['_aggs.signature'] = [
'platform',
]
params['_facets_size'] *= 2
if range_type == 'build':
params['date'][1] = '<' + dates[1].isoformat()
params['build_id'] = [
'>=' + datetime_to_build_id(dates[1] - delta),
'<' + datetime_to_build_id(dates[0])
]
previous_range_results = api.get(**params)
total = previous_range_results['total']
compare_signatures = {}
if total > 0 and 'signature' in previous_range_results['facets']:
signatures = previous_range_results['facets']['signature']
for i, hit in enumerate(signatures):
compare_signatures[hit['term']] = {
'count': hit['count'],
'rank': i + 1,
'percent': 100.0 * hit['count'] / total
}
for hit in results:
sig = compare_signatures.get(hit['term'])
if sig:
hit['diff'] = sig['percent'] - hit['percent']
hit['rank_diff'] = sig['rank'] - hit['rank']
hit['previous_percent'] = sig['percent']
else:
hit['diff'] = 'new'
hit['rank_diff'] = 0
hit['previous_percent'] = 0
return search_results
def test_search_results_pagination(self, rpost):
"""Test that the pagination of results works as expected.
"""
def mocked_post(**options):
assert 'bugs' in options['url'], options['url']
return Response("""
{"hits": [], "total": 0}
""")
rpost.side_effect = mocked_post
def mocked_supersearch_get(**params):
assert '_columns' in params
# Make sure a negative page does not lead to negative offset value.
# But instead it is considered as the page 1 and thus is not added.
eq_(params.get('_results_offset'), 0)
hits = []
for i in range(140):
hits.append({
"signature": "hang | nsASDOMWindowEnumerator::GetNext()",
"date": "2017-01-31T23:12:57",
"uuid": i,
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981
})
return {
"hits": self.only_certain_columns(hits, params['_columns']),
"facets": "",
"total": len(hits)
}
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
url = reverse('supersearch.search_results')
response = self.client.get(
url,
{
'signature': 'hang | nsASDOMWindowEnumerator::GetNext()',
'_columns': ['version'],
'_facets': ['platform'],
}
)
eq_(response.status_code, 200)
ok_('140' in response.content)
# Check that the pagination URL contains all three expected parameters.
doc = pyquery.PyQuery(response.content)
next_page_url = str(doc('.pagination a').eq(0))
ok_('_facets=platform' in next_page_url)
ok_('_columns=version' in next_page_url)
ok_('page=2' in next_page_url)
ok_('#crash-reports' in next_page_url)
# Verify white spaces are correctly encoded.
ok_(
# Note we use `quote` and not `quote_plus`, so white spaces are
# turned into '%20' instead of '+'.
urllib.quote('hang | nsASDOMWindowEnumerator::GetNext()')
in next_page_url
)
# Test that a negative page value does not break it.
response = self.client.get(url, {'page': '-1'})
eq_(response.status_code, 200)
def test_search_results_admin_mode(self, rpost):
"""Test that an admin can see more fields, and that a non-admin cannot.
"""
def mocked_post(**options):
assert 'bugs' in options['url'], options['url']
return Response({"hits": [], "total": 0})
rpost.side_effect = mocked_post
def mocked_supersearch_get(**params):
assert '_columns' in params
if '_facets' in params and 'url' in params['_facets']:
facets = {
"platform": [
{
"term": "Linux",
"count": 3
}
],
"url": [
{
"term": "http://example.org",
"count": 3
}
]
}
else:
facets = {
"platform": [
{
"term": "Linux",
"count": 3
}
]
}
results = {
"hits": [
{
"signature": "nsASDOMWindowEnumerator::GetNext()",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa1",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981,
"email": "*****@*****.**",
"url": "http://example.org",
"exploitability": "high"
},
{
"signature": "mySignatureIsCool",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa2",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981,
"email": "*****@*****.**",
"url": "http://example.org",
"exploitability": "low"
},
{
"signature": "mineIsCoolerThanYours",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa3",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": None,
"email": "*****@*****.**",
"url": "http://example.org",
"exploitability": "error"
}
],
"facets": facets,
"total": 3
}
results['hits'] = self.only_certain_columns(
results['hits'],
params['_columns']
)
return results
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
url = reverse('supersearch.search_results')
# Logged in user, can see the email field
user = self._login()
group = self._create_group_with_permission('view_pii')
user.groups.add(group)
response = self.client.get(
url,
{
'_columns': ['version', 'email', 'url', 'exploitability'],
'_facets': ['url', 'platform']
}
)
eq_(response.status_code, 200)
ok_('Email' in response.content)
ok_('*****@*****.**' in response.content)
ok_('Url facet' in response.content)
ok_('http://example.org' in response.content)
ok_('Version' in response.content)
ok_('1.0' in response.content)
# Without the correct permission the user cannot see exploitability.
ok_('Exploitability' not in response.content)
exp_group = self._create_group_with_permission('view_exploitability')
user.groups.add(exp_group)
response = self.client.get(
url,
{
'_columns': ['version', 'email', 'url', 'exploitability'],
'_facets': ['url', 'platform']
}
)
eq_(response.status_code, 200)
ok_('Email' in response.content)
ok_('Exploitability' in response.content)
ok_('high' in response.content)
# Logged out user, cannot see the email field
self._logout()
response = self.client.get(
url,
{
'_columns': ['version', 'email', 'url'],
'_facets': ['url', 'platform']
}
)
eq_(response.status_code, 200)
ok_('Email' not in response.content)
ok_('*****@*****.**' not in response.content)
ok_('Url facet' not in response.content)
ok_('http://example.org' not in response.content)
ok_('Version' in response.content)
ok_('1.0' in response.content)
def test_search_results(self, cpost):
def mocked_post(**options):
return {
"hits": [
{
"id": "123456",
"signature": u"nsASDOMWindowEnumerator::GetNext()"
}
],
"total": 1
}
cpost.side_effect = mocked_post
def mocked_supersearch_get(**params):
assert '_columns' in params
if 'product' in params and 'WaterWolf' in params['product']:
results = {
"hits": [
{
"signature": "nsASDOMWindowEnumerator::GetNext()",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa1",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981
},
{
"signature": "mySignatureIsCool",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa2",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981
},
{
"signature": "mineIsCoolerThanYours",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa3",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": None
},
{
"signature": "EMPTY",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa4",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": None
}
],
"facets": {
"signature": [
{
"term": "nsASDOMWindowEnumerator::GetNext()",
"count": 1
},
{
"term": "mySignatureIsCool",
"count": 1
},
{
"term": "mineIsCoolerThanYours",
"count": 1
},
{
"term": "EMPTY",
"count": 1
}
]
},
"total": 4
}
results['hits'] = self.only_certain_columns(
results['hits'],
params['_columns']
)
return results
elif 'product' in params and 'SeaMonkey' in params['product']:
results = {
"hits": [
{
"signature": "nsASDOMWindowEnumerator::GetNext()",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981
},
{
"signature": "mySignatureIsCool",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981
}
],
"facets": {
"build_id": [
{
"term": "888981",
"count": 2
}
]
},
"total": 2
}
results['hits'] = self.only_certain_columns(
results['hits'],
params['_columns']
)
return results
elif (
'signature' in params and
'~nsASDOMWindowEnumerator' in params['signature']
):
results = {
"hits": [
{
"signature": "nsASDOMWindowEnumerator::GetNext()",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 12345678
}
],
"facets": {
"signature": [
{
"term": "nsASDOMWindowEnumerator::GetNext()",
"count": 1
}
]
},
"total": 1
}
results['hits'] = self.only_certain_columns(
results['hits'],
params['_columns']
)
return results
else:
return {"hits": [], "facets": [], "total": 0}
SuperSearchUnredacted.implementation().get.side_effect = (
mocked_supersearch_get
)
url = reverse('supersearch.search_results')
response = self.client.get(
url,
{'product': 'WaterWolf'}
)
eq_(response.status_code, 200)
# Test results are existing
ok_('table id="reports-list"' in response.content)
ok_('nsASDOMWindowEnumerator::GetNext()' in response.content)
ok_('mySignatureIsCool' in response.content)
ok_('mineIsCoolerThanYours' in response.content)
ok_('EMPTY' in response.content)
ok_('aaaaaaaaaaaaa1' in response.content)
ok_('888981' in response.content)
ok_('Linux' in response.content)
ok_('2017-01-31 23:12:57' in response.content)
# Test facets are existing
ok_('table id="facets-list-' in response.content)
# Test bugs are existing
ok_('<th scope="col">Bugs</th>' in response.content)
ok_('123456' in response.content)
# Test links on terms are existing
ok_('product=%3DWaterWolf' in response.content)
# Test with empty results
response = self.client.get(url, {
'product': 'NightTrain',
'date': '2012-01-01'
})
eq_(response.status_code, 200)
ok_('table id="reports-list"' not in response.content)
ok_('No results were found' in response.content)
# Test with a signature param
response = self.client.get(
url,
{'signature': '~nsASDOMWindowEnumerator'}
)
eq_(response.status_code, 200)
ok_('table id="reports-list"' in response.content)
ok_('nsASDOMWindowEnumerator::GetNext()' in response.content)
ok_('123456' in response.content)
# Test with a different facet
response = self.client.get(
url,
{'_facets': 'build_id', 'product': 'SeaMonkey'}
)
eq_(response.status_code, 200)
ok_('table id="reports-list"' in response.content)
ok_('table id="facets-list-' in response.content)
ok_('888981' in response.content)
# Bugs should not be there, they appear only in the signature facet
ok_('<th>Bugs</th>' not in response.content)
ok_('123456' not in response.content)
# Test with a different columns list
response = self.client.get(
url,
{'_columns': ['build_id', 'platform'], 'product': 'WaterWolf'}
)
eq_(response.status_code, 200)
ok_('table id="reports-list"' in response.content)
ok_('table id="facets-list-' in response.content)
# The build and platform appear
ok_('888981' in response.content)
ok_('Linux' in response.content)
# The crash id is always shown
ok_('aaaaaaaaaaaaa1' in response.content)
# The version and date do not appear
ok_('1.0' not in response.content)
ok_('2017' not in response.content)
# Test missing parameters don't raise an exception.
response = self.client.get(
url,
{'product': 'WaterWolf', 'date': '', 'build_id': ''}
)
eq_(response.status_code, 200)
def test_search_results(self):
BugAssociation.objects.create(
bug_id=123456, signature="nsASDOMWindowEnumerator::GetNext()")
def mocked_supersearch_get(**params):
assert "_columns" in params
if "product" in params and "WaterWolf" in params["product"]:
results = {
"hits": [
{
"signature": "nsASDOMWindowEnumerator::GetNext()",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa1",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981,
},
{
"signature": "mySignatureIsCool",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa2",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981,
},
{
"signature": "mineIsCoolerThanYours",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa3",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": None,
},
{
"signature": "EMPTY",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa4",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": None,
},
],
"facets": {
"signature": [
{
"term": "nsASDOMWindowEnumerator::GetNext()",
"count": 1
},
{
"term": "mySignatureIsCool",
"count": 1
},
{
"term": "mineIsCoolerThanYours",
"count": 1
},
{
"term": "EMPTY",
"count": 1
},
]
},
"total":
4,
}
results["hits"] = self.only_certain_columns(
results["hits"], params["_columns"])
return results
elif "product" in params and "SeaMonkey" in params["product"]:
results = {
"hits": [
{
"signature": "nsASDOMWindowEnumerator::GetNext()",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981,
},
{
"signature": "mySignatureIsCool",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981,
},
],
"facets": {
"build_id": [{
"term": "888981",
"count": 2
}]
},
"total":
2,
}
results["hits"] = self.only_certain_columns(
results["hits"], params["_columns"])
return results
elif ("signature" in params
and "~nsASDOMWindowEnumerator" in params["signature"]):
results = {
"hits": [{
"signature": "nsASDOMWindowEnumerator::GetNext()",
"date": "2017-01-31T23:12:57",
"uuid": "aaaaaaaaaaaaa",
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 12345678,
}],
"facets": {
"signature": [{
"term": "nsASDOMWindowEnumerator::GetNext()",
"count": 1
}]
},
"total":
1,
}
results["hits"] = self.only_certain_columns(
results["hits"], params["_columns"])
return results
else:
return {"hits": [], "facets": [], "total": 0}
SuperSearchUnredacted.implementation(
).get.side_effect = mocked_supersearch_get
url = reverse("supersearch:search_results")
response = self.client.get(url, {"product": "WaterWolf"})
assert response.status_code == 200
# Test results are existing
assert 'table id="reports-list"' in smart_text(response.content)
assert "nsASDOMWindowEnumerator::GetNext()" in smart_text(
response.content)
assert "mySignatureIsCool" in smart_text(response.content)
assert "mineIsCoolerThanYours" in smart_text(response.content)
assert "EMPTY" in smart_text(response.content)
assert "aaaaaaaaaaaaa1" in smart_text(response.content)
assert "888981" in smart_text(response.content)
assert "Linux" in smart_text(response.content)
assert "2017-01-31 23:12:57" in smart_text(response.content)
# Test facets are existing
assert 'table id="facets-list-' in smart_text(response.content)
# Test bugs are existing
assert '<th scope="col">Bugs</th>' in smart_text(response.content)
assert "123456" in smart_text(response.content)
# Test links on terms are existing
assert "product=%3DWaterWolf" in smart_text(response.content)
# Test with empty results
response = self.client.get(url, {
"product": "NightTrain",
"date": "2012-01-01"
})
assert response.status_code == 200
assert 'table id="reports-list"' not in smart_text(response.content)
assert "No results were found" in smart_text(response.content)
# Test with a signature param
response = self.client.get(url,
{"signature": "~nsASDOMWindowEnumerator"})
assert response.status_code == 200
assert 'table id="reports-list"' in smart_text(response.content)
assert "nsASDOMWindowEnumerator::GetNext()" in smart_text(
response.content)
assert "123456" in smart_text(response.content)
# Test with a different facet
response = self.client.get(url, {
"_facets": "build_id",
"product": "SeaMonkey"
})
assert response.status_code == 200
assert 'table id="reports-list"' in smart_text(response.content)
assert 'table id="facets-list-' in smart_text(response.content)
assert "888981" in smart_text(response.content)
# Bugs should not be there, they appear only in the signature facet
assert "<th>Bugs</th>" not in smart_text(response.content)
assert "123456" not in smart_text(response.content)
# Test with a different columns list
response = self.client.get(url, {
"_columns": ["build_id", "platform"],
"product": "WaterWolf"
})
assert response.status_code == 200
assert 'table id="reports-list"' in smart_text(response.content)
assert 'table id="facets-list-' in smart_text(response.content)
# The build and platform appear
assert "888981" in smart_text(response.content)
assert "Linux" in smart_text(response.content)
# The crash id is always shown
assert "aaaaaaaaaaaaa1" in smart_text(response.content)
# The version and date do not appear
assert "1.0" not in smart_text(response.content)
assert "2017" not in smart_text(response.content)
# Test missing parameters don't raise an exception.
response = self.client.get(url, {
"product": "WaterWolf",
"date": "",
"build_id": ""
})
assert response.status_code == 200
def test_search_results_pagination(self):
"""Test that the pagination of results works as expected.
"""
def mocked_supersearch_get(**params):
assert "_columns" in params
# Make sure a negative page does not lead to negative offset value.
# But instead it is considered as the page 1 and thus is not added.
assert params.get("_results_offset") == 0
hits = []
for i in range(140):
hits.append({
"signature": "hang | nsASDOMWindowEnumerator::GetNext()",
"date": "2017-01-31T23:12:57",
"uuid": i,
"product": "WaterWolf",
"version": "1.0",
"platform": "Linux",
"build_id": 888981,
})
return {
"hits": self.only_certain_columns(hits, params["_columns"]),
"facets": "",
"total": len(hits),
}
SuperSearchUnredacted.implementation(
).get.side_effect = mocked_supersearch_get
url = reverse("supersearch:search_results")
response = self.client.get(
url,
{
"signature": "hang | nsASDOMWindowEnumerator::GetNext()",
"_columns": ["version"],
"_facets": ["platform"],
},
)
assert response.status_code == 200
assert "140" in smart_text(response.content)
# Check that the pagination URL contains all three expected parameters.
doc = pyquery.PyQuery(response.content)
next_page_url = str(doc(".pagination a").eq(0))
assert "_facets=platform" in next_page_url
assert "_columns=version" in next_page_url
assert "page=2" in next_page_url
assert "#crash-reports" in next_page_url
# Verify white spaces are correctly encoded.
# Note we use `quote` and not `quote_plus`, so white spaces are
# turned into '%20' instead of '+'.
assert quote(
"hang | nsASDOMWindowEnumerator::GetNext()") in next_page_url
# Test that a negative page value does not break it.
response = self.client.get(url, {"page": "-1"})
assert response.status_code == 200
