def test_index_chunk_task(self): responses = [response(save=True) for i in range(10)] # With live indexing, that'll create items in the index. Since # we want to test index_chunk_test, we need a clean index to # start with so we delete and recreate it. self.setup_indexes(empty=True) # Verify there's nothing in the index. eq_(len(ResponseMappingType.search()), 0) # Create the record and the chunk and then run it through # celery. batch_id = 'ou812' rec = record(batch_id=batch_id, save=True) chunk = (ResponseMappingType, [item.id for item in responses]) index_chunk_task.delay(get_index(), batch_id, rec.id, chunk) ResponseMappingType.refresh_index() # Verify everything is in the index now. eq_(len(ResponseMappingType.search()), 10) # Verify the record was marked succeeded. rec = Record.objects.get(pk=rec.id) eq_(rec.status, Record.STATUS_SUCCESS)
def response_view(request, responseid, template): response = get_object_or_404(Response, id=responseid) try: prod = Product.objects.get(db_name=response.product) if (not prod.on_dashboard and (not request.user.is_authenticated() or not request.user.has_perm( 'analytics.can_view_dashboard'))): # If this is a response for a hidden product and the user # isn't in the analytics group, then they can't see it. return HttpResponseForbidden() except Product.DoesNotExist: pass mlt = None records = None errors = [] if (request.user.is_authenticated() and request.user.has_perm('analytics.can_view_dashboard')): try: # Convert it to a list to force it to execute right now. mlt = ResponseMappingType.reshape( ResponseMappingType.morelikethis(response)) except ElasticsearchException as exc: errors.append('Failed to do morelikethis: %s' % exc) records = [ (u'Response records', Record.objects.records(response)), ] jobs = GengoJob.objects.filter( object_id=response.id, content_type=ContentType.objects.get_for_model(response) ) for job in jobs: records.append( (u'Gengo job record {0}'.format(job.id), job.records) ) # We don't pass the response directly to the template and instead # do some data tweaks here to make it more palatable for viewing. return render(request, template, { 'errors': errors, 'response': response, 'mlt': mlt, 'records': records, })
def test_purge(self): now = datetime.datetime.now() cutoff = now - datetime.timedelta(days=5) # Create 10 ResponseEmail objs--one for each day for the last # 10 days. for i in range(10): ResponseEmailFactory( opinion__created=(now - datetime.timedelta(days=i)) ) ResponseContextFactory( opinion__created=(now - datetime.timedelta(days=i)) ) # Since creating the objects and indexing them happens very # quickly in tests, we hit a race condition and the has_email # column ends up being false. So instead we just drop the # index and rebuild it. self.setup_indexes() # Make sure everything is in the db eq_(Response.objects.count(), 20) eq_(ResponseEmail.objects.count(), 10) eq_(ResponseContext.objects.count(), 10) # Make sure everything is in the index resp_s = ResponseMappingType.search() eq_(resp_s.count(), 20) eq_(resp_s.filter(has_email=True).count(), 10) # Now purge everything older than 5 days and make sure things # got removed that should have gotten removed cutoff = now - datetime.timedelta(days=5) purge_data(cutoff=cutoff) self.refresh() eq_(Response.objects.count(), 20) eq_(ResponseEmail.objects.count(), 5) eq_(ResponseEmail.objects.filter( opinion__created__gte=cutoff).count(), 5) eq_(ResponseContext.objects.count(), 5) eq_(ResponseContext.objects.filter( opinion__created__gte=cutoff).count(), 5) # Everything should still be in the index, but the number of # things with has_email=True should go down resp_s = ResponseMappingType.search() eq_(resp_s.count(), 20) eq_(resp_s.filter(has_email=True).count(), 5)
def response_view(request, responseid, template): response = get_object_or_404(Response, id=responseid) mlt = None records = None if (request.user.is_authenticated() and request.user.has_perm('analytics.can_view_dashboard')): mlt = ResponseMappingType.morelikethis(response) records = [ (u'Response records', Record.objects.records(response)), ] jobs = GengoJob.objects.filter( object_id=response.id, content_type=ContentType.objects.get_for_model(response) ) for job in jobs: records.append( (u'Gengo job record {0}'.format(job.id), job.records) ) # We don't pass the response directly to the template and instead # do some data tweaks here to make it more palatable for viewing. return render(request, template, { 'response': response, 'mlt': mlt, 'records': records, })
def timezone_view(request): """Admin view showing times and timezones in data.""" # Note: This is an admin page that gets used once in a blue moon. # As such, I'm taking some liberties (hand-indexing the response, # time.sleep, etc) that I would never take if it was used more # often or was viewable by users. If these two assumptions ever # change, then this should be rewritten. from elasticutils.contrib.django import get_es from fjord.feedback.models import Response, ResponseMappingType from fjord.feedback.tests import ResponseFactory from fjord.search.index import get_index server_time = datetime.now() # Create a new response. resp = ResponseFactory.create() resp_time = resp.created # Index the response by hand so we know it gets to Elasticsearch. Otherwise # it gets done by celery and we don't know how long that'll take. doc = ResponseMappingType.extract_document(resp.id) ResponseMappingType.index(doc, resp.id) # Fetch the response from the db. resp = Response.objects.get(id=resp.id) resp2_time = resp.created # Refresh and sleep 5 seconds as a hand-wavey way to make sure # that Elasticsearch has had time to refresh the index. get_es().indices.refresh(get_index()) time.sleep(5) es_time = ResponseMappingType.search().filter(id=resp.id)[0].created # Delete the test response we created. resp.delete() return render(request, 'admin/timezone_view.html', { 'server_time': server_time, 'resp_time': resp_time, 'resp2_time': resp2_time, 'es_time': es_time })
def analytics_duplicates(request): """Shows all duplicate descriptions over the last n days""" template = 'analytics/analyzer/duplicates.html' n = 14 responses = (ResponseMappingType.search() .filter(created__gte=datetime.now() - timedelta(days=n)) .values_dict('description', 'happy', 'created', 'locale', 'user_agent', 'id') .order_by('created').everything()) responses = ResponseMappingType.reshape(responses) total_count = len(responses) response_dupes = {} for resp in responses: response_dupes.setdefault(resp['description'], []).append(resp) response_dupes = [ (key, val) for key, val in response_dupes.items() if len(val) > 1 ] # convert the dict into a list of tuples sorted by the number of # responses per tuple largest number first response_dupes = sorted(response_dupes, key=lambda item: len(item[1]) * -1) # duplicate_count -> count # i.e. "how many responses had 2 duplicates?" summary_counts = defaultdict(int) for desc, responses in response_dupes: summary_counts[len(responses)] = summary_counts[len(responses)] + 1 summary_counts = sorted(summary_counts.items(), key=lambda item: item[0]) return render(request, template, { 'n': 14, 'response_dupes': response_dupes, 'render_time': datetime.now(), 'summary_counts': summary_counts, 'total_count': total_count, })
def analytics_duplicates(request): """Shows all duplicate descriptions over the last n days""" template = 'analytics/analyzer/duplicates.html' n = 14 responses = (ResponseMappingType.search().filter( created__gte=datetime.now() - timedelta(days=n)).values_dict( 'description', 'happy', 'created', 'locale', 'user_agent', 'id').order_by('created').everything()) responses = ResponseMappingType.reshape(responses) total_count = len(responses) response_dupes = {} for resp in responses: response_dupes.setdefault(resp['description'], []).append(resp) response_dupes = [(key, val) for key, val in response_dupes.items() if len(val) > 1] # convert the dict into a list of tuples sorted by the number of # responses per tuple largest number first response_dupes = sorted(response_dupes, key=lambda item: len(item[1]) * -1) # duplicate_count -> count # i.e. "how many responses had 2 duplicates?" summary_counts = defaultdict(int) for desc, responses in response_dupes: summary_counts[len(responses)] = summary_counts[len(responses)] + 1 summary_counts = sorted(summary_counts.items(), key=lambda item: item[0]) return render( request, template, { 'n': 14, 'response_dupes': response_dupes, 'render_time': datetime.now(), 'summary_counts': summary_counts, 'total_count': total_count, })
def analytics_hourly_histogram(request): """Shows an hourly histogram for the last 5 days of all responses""" template = 'analytics/analyzer/hourly_histogram.html' date_end = smart_date(request.GET.get('date_end', None), fallback=None) if date_end is None: date_end = date.today() date_start = date_end - timedelta(days=5) search = ResponseMappingType.search() filters = F(created__gte=date_start, created__lte=date_end) search.filter(filters) hourly_histogram = search.facet_raw( hourly={ 'date_histogram': { 'interval': 'hour', 'field': 'created' }, 'facet_filter': search._process_filters(filters.filters) }).facet_counts() hourly_data = dict( (p['time'], p['count']) for p in hourly_histogram['hourly']) hour = 60 * 60 * 1000.0 zero_fill(date_start, date_end, [hourly_data], spacing=hour) # FIXME: This is goofy. After zero_fill, we end up with a bunch of # trailing zeros for reasons I don't really understand, so instead # of fixing that, I'm just going to remove them here. hourly_data = sorted(hourly_data.items()) while hourly_data and hourly_data[-1][1] == 0: hourly_data.pop(-1) histogram = [ { 'label': 'Hourly', 'name': 'hourly', 'data': hourly_data }, ] return render(request, template, { 'histogram': histogram, 'start_date': date_start, 'end_date': date_end })
def response_view(request, responseid, template): response = get_object_or_404(Response, id=responseid) mlt = None records = None errors = [] if (request.user.is_authenticated() and request.user.has_perm('analytics.can_view_dashboard')): try: # Convert it to a list to force it to execute right now. mlt = ResponseMappingType.reshape( ResponseMappingType.morelikethis(response)) except ElasticsearchException as exc: errors.append('Failed to do morelikethis: %s' % exc) records = [ (u'Response records', Record.objects.records(response)), ] jobs = GengoJob.objects.filter( object_id=response.id, content_type=ContentType.objects.get_for_model(response) ) for job in jobs: records.append( (u'Gengo job record {0}'.format(job.id), job.records) ) # We don't pass the response directly to the template and instead # do some data tweaks here to make it more palatable for viewing. return render(request, template, { 'errors': errors, 'response': response, 'mlt': mlt, 'records': records, })
def response_view(request, responseid, template): response = get_object_or_404(Response, id=responseid) mlt = None if (request.user.is_authenticated() and request.user.has_perm('analytics.can_view_dashboard')): mlt = ResponseMappingType.morelikethis(response) # We don't pass the response directly to the template and instead # do some data tweaks here to make it more palatable for viewing. return render(request, template, { 'response': response, 'mlt': mlt, })
def analytics_search(request): template = 'analytics/analyzer/search.html' output_format = request.GET.get('format', None) page = smart_int(request.GET.get('page', 1), 1) # Note: If we add additional querystring fields, we need to add # them to generate_dashboard_url. search_happy = request.GET.get('happy', None) search_has_email = request.GET.get('has_email', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_country = request.GET.get('country', None) search_product = request.GET.get('product', None) search_domain = request.GET.get('domain', None) search_api = smart_int(request.GET.get('api', None), fallback=None) search_version = request.GET.get('version', None) search_query = request.GET.get('q', None) search_date_start = smart_date( request.GET.get('date_start', None), fallback=None) search_date_end = smart_date( request.GET.get('date_end', None), fallback=None) search_bigram = request.GET.get('bigram', None) search_source = request.GET.get('source', None) search_campaign = request.GET.get('campaign', None) search_organic = request.GET.get('organic', None) selected = request.GET.get('selected', None) filter_data = [] current_search = {'page': page} search = ResponseMappingType.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F(happy=search_happy) current_search['happy'] = int(search_happy) # If search has_email is '0' or '1', set it to False or True, # respectively. search_has_email = {'0': False, '1': True}.get(search_has_email, None) if search_has_email in [False, True]: f &= F(has_email=search_has_email) current_search['has_email'] = int(search_has_email) def unknown_to_empty(text): """Convert "Unknown" to "" to support old links""" return u'' if text.lower() == u'unknown' else text if search_platform is not None: f &= F(platform=unknown_to_empty(search_platform)) current_search['platform'] = search_platform if search_locale is not None: f &= F(locale=unknown_to_empty(search_locale)) current_search['locale'] = search_locale if search_product is not None: f &= F(product=unknown_to_empty(search_product)) current_search['product'] = search_product # Only show the version if there's a product. if search_version is not None: # Note: We only filter on version if we're filtering on # product. f &= F(version=unknown_to_empty(search_version)) current_search['version'] = search_version # Only show the country if the product is Firefox OS. if search_country is not None and search_product == 'Firefox OS': f &= F(country=unknown_to_empty(search_country)) current_search['country'] = search_country if search_domain is not None: f &= F(url_domain=unknown_to_empty(search_domain)) current_search['domain'] = search_domain if search_api is not None: f &= F(api=search_api) current_search['api'] = search_api if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = datetime.now() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') # Add one day, so that the search range includes the entire day. end = search_date_end + timedelta(days=1) # Note 'less than', not 'less than or equal', because of the added # day above. f &= F(created__lt=end) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F(created__gte=search_date_start) if search_query: current_search['q'] = search_query search = search.query(description__sqs=search_query) if search_bigram is not None: f &= F(description_bigrams=search_bigram) filter_data.append({ 'display': 'Bigram', 'name': 'bigram', 'options': [{ 'count': 'all', 'name': search_bigram, 'display': search_bigram, 'value': search_bigram, 'checked': True }] }) if search_source is not None: f &= F(source=search_source) current_search['source'] = search_source if search_campaign is not None: f &= F(campaign=search_campaign) current_search['campaign'] = search_campaign search_organic = {'0': False, '1': True}.get(search_organic, None) if search_organic in [False, True]: f &= F(organic=search_organic) current_search['organic'] = int(search_organic) search = search.filter(f).order_by('-created') # If they're asking for a CSV export, then send them to the export # screen. if output_format == 'csv': return _analytics_search_export(request, search) # Search results and pagination if page < 1: page = 1 page_count = 50 start = page_count * (page - 1) end = start + page_count search_count = search.count() search_results = search.values_list('id')[start:end] opinion_page_ids = [mem[0][0] for mem in search_results] # We convert what we get back from ES to what's in the db so we # can get all the information. opinion_page = Response.objects.filter(id__in=opinion_page_ids) # Navigation facet data # This loop does two things. First it maps 'T' -> True and 'F' -> # False. This is probably something EU should be doing for # us. Second, it restructures the data into a more convenient # form. counts = { 'happy': {}, 'has_email': {}, 'platform': {}, 'locale': {}, 'country': {}, 'product': {}, 'version': {}, 'url_domain': {}, 'api': {}, 'source': {}, 'campaign': {}, 'organic': {}, } facets = search.facet(*(counts.keys()), filtered=bool(search._process_filters(f.filters)), size=25) for param, terms in facets.facet_counts().items(): for term in terms: name = term['term'] if name == 'T': name = True elif name == 'F': name = False counts[param][name] = term['count'] def empty_to_unknown(text): return 'Unknown' if text == u'' else text filter_data.extend([ counts_to_options( counts['happy'].items(), name='happy', display='Sentiment', display_map={True: 'Happy', False: 'Sad'}, value_map={True: 1, False: 0}, checked=search_happy), counts_to_options( counts['has_email'].items(), name='has_email', display='Has email', display_map={True: 'Yes', False: 'No'}, value_map={True: 1, False: 0}, checked=search_has_email), counts_to_options( counts['product'].items(), name='product', display='Product', display_map=empty_to_unknown, checked=search_product) ]) # Only show the version if we're showing a specific # product. if search_product: filter_data.append( counts_to_options( counts['version'].items(), name='version', display='Version', display_map=empty_to_unknown, checked=search_version) ) # Only show the country if the product is Firefox OS. if search_product == 'Firefox OS': filter_data.append( counts_to_options( counts['country'].items(), name='country', display='Country', checked=search_country, display_map=country_name), ) filter_data.extend( [ counts_to_options( counts['platform'].items(), name='platform', display='Platform', display_map=empty_to_unknown, checked=search_platform), counts_to_options( counts['locale'].items(), name='locale', display='Locale', checked=search_locale, display_map=locale_name), counts_to_options( counts['url_domain'].items(), name='domain', display='Domain', checked=search_domain, display_map=empty_to_unknown), counts_to_options( counts['api'].items(), name='api', display='API version', checked=search_api, display_map=empty_to_unknown), counts_to_options( counts['organic'].items(), name='organic', display='Organic', display_map={True: 'Yes', False: 'No'}, value_map={True: 1, False: 0}, checked=search_organic), counts_to_options( counts['source'].items(), name='source', display='Source', checked=search_source, display_map=empty_to_unknown), counts_to_options( counts['campaign'].items(), name='campaign', display='Campaign', checked=search_campaign, display_map=empty_to_unknown), ] ) return render(request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, })
def analytics_occurrences(request): template = 'analytics/analyzer/occurrences.html' first_facet_bi = None first_params = {} first_facet_total = 0 second_facet_bi = None second_params = {} second_facet_total = 0 if 'product' in request.GET: form = OccurrencesComparisonForm(request.GET) if form.is_valid(): cleaned = form.cleaned_data # First item first_resp_s = (ResponseMappingType.search() .filter(product=cleaned['product']) .filter(locale__startswith='en')) first_params['product'] = cleaned['product'] if cleaned['first_version']: first_resp_s = first_resp_s.filter( version=cleaned['first_version']) first_params['version'] = cleaned['first_version'] if cleaned['first_start_date']: first_resp_s = first_resp_s.filter( created__gte=cleaned['first_start_date']) first_params['date_start'] = cleaned['first_start_date'] if cleaned['first_end_date']: first_resp_s = first_resp_s.filter( created__lte=cleaned['first_end_date']) first_params['date_end'] = cleaned['first_end_date'] if cleaned['first_search_term']: first_resp_s = first_resp_s.query( description__match=cleaned['first_search_term']) first_params['q'] = cleaned['first_search_term'] if ('date_start' not in first_params and 'date_end' not in first_params): # FIXME - If there's no start date, then we want # "everything" so we use a hard-coded 2013-01-01 date # here to hack that. # # Better way might be to change the dashboard to allow # for an "infinite" range, but there's no other use # case for that and the ranges are done in the ui--not # in the backend. first_params['date_start'] = '2013-01-01' # Have to do raw because we want a size > 10. first_resp_s = first_resp_s.facet('description_bigrams', size=30, filtered=True) first_resp_s = first_resp_s[0:0] first_facet_total = first_resp_s.count() first_facet = first_resp_s.facet_counts() first_facet_bi = first_facet['description_bigrams'] first_facet_bi = sorted( first_facet_bi, key=lambda item: -item['count']) if (cleaned['second_version'] or cleaned['second_search_term'] or cleaned['second_start_date']): second_resp_s = (ResponseMappingType.search() .filter(product=cleaned['product']) .filter(locale__startswith='en')) second_params['product'] = cleaned['product'] if cleaned['second_version']: second_resp_s = second_resp_s.filter( version=cleaned['second_version']) second_params['version'] = cleaned['second_version'] if cleaned['second_start_date']: second_resp_s = second_resp_s.filter( created__gte=cleaned['second_start_date']) second_params['date_start'] = cleaned['second_start_date'] if cleaned['second_end_date']: second_resp_s = second_resp_s.filter( created__lte=cleaned['second_end_date']) second_params['date_end'] = cleaned['second_end_date'] if form.cleaned_data['second_search_term']: second_resp_s = second_resp_s.query( description__match=cleaned['second_search_term']) second_params['q'] = cleaned['second_search_term'] if ('date_start' not in second_params and 'date_end' not in second_params): # FIXME - If there's no start date, then we want # "everything" so we use a hard-coded 2013-01-01 date # here to hack that. # # Better way might be to change the dashboard to allow # for an "infinite" range, but there's no other use # case for that and the ranges are done in the ui--not # in the backend. second_params['date_start'] = '2013-01-01' # Have to do raw because we want a size > 10. second_resp_s = second_resp_s.facet('description_bigrams', size=30, filtered=True) second_resp_s = second_resp_s[0:0] second_facet_total = second_resp_s.count() second_facet = second_resp_s.facet_counts() second_facet_bi = second_facet['description_bigrams'] second_facet_bi = sorted( second_facet_bi, key=lambda item: -item['count']) permalink = request.build_absolute_uri() else: permalink = '' form = OccurrencesComparisonForm() # FIXME - We have responses that have no product set. This ignores # those. That's probably the right thing to do for the Occurrences Report # but maybe not. products = [prod for prod in ResponseMappingType.get_products() if prod] return render(request, template, { 'permalink': permalink, 'form': form, 'products': products, 'first_facet_bi': first_facet_bi, 'first_params': first_params, 'first_facet_total': first_facet_total, 'first_normalization': round(first_facet_total * 1.0 / 1000, 3), 'second_facet_bi': second_facet_bi, 'second_params': second_params, 'second_facet_total': second_facet_total, 'second_normalization': round(second_facet_total * 1.0 / 1000, 3), 'render_time': datetime.now(), })
def dashboard(request): template = 'analytics/dashboard.html' output_format = request.GET.get('format', None) page = smart_int(request.GET.get('page', 1), 1) # Note: If we add additional querystring fields, we need to add # them to generate_dashboard_url. search_happy = request.GET.get('happy', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_product = request.GET.get('product', None) search_version = request.GET.get('version', None) search_query = request.GET.get('q', None) search_date_start = smart_date( request.GET.get('date_start', None), fallback=None) search_date_end = smart_date( request.GET.get('date_end', None), fallback=None) search_bigram = request.GET.get('bigram', None) selected = request.GET.get('selected', None) filter_data = [] current_search = {'page': page} search = ResponseMappingType.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F(happy=search_happy) current_search['happy'] = int(search_happy) def unknown_to_empty(text): """Convert "Unknown" to "" to support old links""" return u'' if text.lower() == u'unknown' else text if search_platform is not None: f &= F(platform=unknown_to_empty(search_platform)) current_search['platform'] = search_platform if search_locale is not None: f &= F(locale=unknown_to_empty(search_locale)) current_search['locale'] = search_locale visible_products = [ prod.encode('utf-8') for prod in Product.objects.public().values_list('db_name', flat=True) ] # This covers the "unknown" product which is also visible. visible_products.append('') if search_product in visible_products: f &= F(product=unknown_to_empty(search_product)) current_search['product'] = search_product if search_version is not None: # Note: We only filter on version if we're filtering on # product. f &= F(version=unknown_to_empty(search_version)) current_search['version'] = search_version else: f &= F(product__in=visible_products) if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = date.today() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) # If the start and end dates are inverted, switch them into proper # chronoligcal order search_date_start, search_date_end = sorted( [search_date_start, search_date_end]) # Restrict the frontpage dashboard to only show the last 6 months # of data six_months_ago = date.today() - timedelta(days=180) search_date_start = max(six_months_ago, search_date_start) search_date_end = max(search_date_start, search_date_end) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') f &= F(created__lte=search_date_end) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F(created__gte=search_date_start) if search_query: current_search['q'] = search_query search = search.query(description__sqs=search_query) if search_bigram is not None: f &= F(description_bigrams=search_bigram) filter_data.append({ 'display': _('Bigram'), 'name': 'bigram', 'options': [{ 'count': 'all', 'name': search_bigram, 'display': search_bigram, 'value': search_bigram, 'checked': True }] }) search = search.filter(f).order_by('-created') # If the user asked for a feed, give him/her a feed! if output_format == 'atom': return generate_atom_feed(request, search) elif output_format == 'json': return generate_json_feed(request, search) # Search results and pagination if page < 1: page = 1 page_count = 20 start = page_count * (page - 1) end = start + page_count search_count = search.count() opinion_page = search[start:end] # Navigation facet data facets = search.facet( 'happy', 'platform', 'locale', 'product', 'version', size=1000, filtered=bool(search._process_filters(f.filters))) # This loop does two things. First it maps 'T' -> True and 'F' -> # False. This is probably something EU should be doing for # us. Second, it restructures the data into a more convenient # form. counts = { 'happy': {}, 'platform': {}, 'locale': {}, 'product': {}, 'version': {} } happy_sad_filter = request.GET.get('happy', None) if happy_sad_filter: if happy_sad_filter == '1': counts['happy'] = {True: 0} elif happy_sad_filter == '0': counts['happy'] = {False: 0} if search_platform: counts['platform'] = {search_platform: 0} if search_locale: counts['locale'] = {search_locale: 0} if search_product: counts['product'] = {search_product: 0} if search_version: counts['version'] = {search_version: 0} for param, terms in facets.facet_counts().items(): for term in terms: name = term['term'] if name.upper() == 'T': name = True elif name.upper() == 'F': name = False counts[param][name] = term['count'] def empty_to_unknown(text): return _('Unknown') if text == u'' else text filter_data.extend([ counts_to_options( counts['happy'].items(), name='happy', display=_('Sentiment'), display_map={True: _('Happy'), False: _('Sad')}, value_map={True: 1, False: 0}, checked=search_happy), counts_to_options( counts['product'].items(), name='product', display=_('Product'), display_map=empty_to_unknown, checked=search_product) ]) # Only show the version if we're showing a specific # product. if search_product: filter_data.append( counts_to_options( counts['version'].items(), name='version', display=_('Version'), display_map=empty_to_unknown, checked=search_version) ) else: filter_data.append({ 'display': _('Version'), 'note': _('Select product to see version breakdown') }) filter_data.extend( [ counts_to_options( counts['platform'].items(), name='platform', display=_('Platform'), display_map=empty_to_unknown, checked=search_platform), counts_to_options( counts['locale'].items(), name='locale', display=_('Locale'), checked=search_locale, display_map=locale_name), ] ) # Histogram data happy_data = [] sad_data = [] happy_f = f & F(happy=True) sad_f = f & F(happy=False) histograms = search.facet_raw( happy={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': search._process_filters(happy_f.filters) }, sad={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': search._process_filters(sad_f.filters) }, ).facet_counts() # p['time'] is number of milliseconds since the epoch. Which is # convenient, because that is what the front end wants. happy_data = dict((p['time'], p['count']) for p in histograms['happy']) sad_data = dict((p['time'], p['count']) for p in histograms['sad']) zero_fill(search_date_start, search_date_end, [happy_data, sad_data]) histogram = [ {'label': _('Happy'), 'name': 'happy', 'data': sorted(happy_data.items())}, {'label': _('Sad'), 'name': 'sad', 'data': sorted(sad_data.items())}, ] return render(request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'histogram': histogram, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, 'atom_url': generate_dashboard_url(request), })
def test_purge(self): now = datetime.datetime.now() cutoff = now - datetime.timedelta(days=5) # Create 10 objs of each type--one for each day for the last # 10 days. for i in range(10): ResponseEmailFactory(opinion__created=(now - datetime.timedelta(days=i))) ResponseContextFactory( opinion__created=(now - datetime.timedelta(days=i))) ResponsePIFactory(opinion__created=(now - datetime.timedelta(days=i))) # Note that this creates 30 Response objects. # Since creating the objects and indexing them happens very # quickly in tests, we hit a race condition and the has_email # column ends up being false. So instead we just drop the # index and rebuild it. self.setup_indexes() # Make sure everything is in the db eq_(Response.objects.count(), 30) eq_(ResponseEmail.objects.count(), 10) eq_(ResponseContext.objects.count(), 10) eq_(ResponsePI.objects.count(), 10) # Make sure everything is in the index resp_s = ResponseMappingType.search() eq_(resp_s.count(), 30) eq_(resp_s.filter(has_email=True).count(), 10) # Now purge everything older than 5 days and make sure things # got removed that should have gotten removed. Also check if # there is a journal entry for the purge operation. cutoff = now - datetime.timedelta(days=5) purge_data(cutoff=cutoff) self.refresh() eq_(Response.objects.count(), 30) eq_(ResponseEmail.objects.count(), 5) eq_( ResponseEmail.objects.filter(opinion__created__gte=cutoff).count(), 5) eq_(ResponseContext.objects.count(), 5) eq_( ResponseContext.objects.filter( opinion__created__gte=cutoff).count(), 5) eq_(ResponsePI.objects.count(), 5) eq_(ResponsePI.objects.filter(opinion__created__gte=cutoff).count(), 5) eq_(1, Record.objects.filter(action='purge_data').count()) expected_msg = ('feedback_responseemail: 5, ' 'feedback_responsecontext: 5, ' 'feedback_responsepi: 5') eq_(expected_msg, Record.objects.get(action='purge_data').msg) # Everything should still be in the index, but the number of # things with has_email=True should go down resp_s = ResponseMappingType.search() eq_(resp_s.count(), 30) eq_(resp_s.filter(has_email=True).count(), 5)
def generate_totals_histogram(search_date_start, search_date_end, search_query, prod): # Note: Not localized because it's ultra-alpha. search_date_start = search_date_start - timedelta(days=1) search = ResponseMappingType.search() if search_query: search = search.query(description__sqs=search_query) f = F() f &= F(product=prod.db_name) f &= F(created__gte=search_date_start) f &= F(created__lt=search_date_end) happy_f = f & F(happy=True) totals_histogram = search.facet_raw( total={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': search._process_filters(f.filters) }, happy={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': search._process_filters(happy_f.filters) }, ).facet_counts() totals_data = dict((p['time'], p['count']) for p in totals_histogram['total']) zero_fill(search_date_start, search_date_end, [totals_data]) totals_data = sorted(totals_data.items()) happy_data = dict((p['time'], p['count']) for p in totals_histogram['happy']) zero_fill(search_date_start, search_date_end, [happy_data]) happy_data = sorted(happy_data.items()) up_deltas = [] down_deltas = [] for i, hap in enumerate(happy_data): if i == 0: continue yesterday = 0 today = 0 # Figure out yesterday and today as a percent to one # significant digit. if happy_data[i-1][1] and totals_data[i-1][1]: yesterday = ( int(happy_data[i-1][1] * 1.0 / totals_data[i-1][1] * 1000) / 10.0 ) if happy_data[i][1] and totals_data[i][1]: today = ( int(happy_data[i][1] * 1.0 / totals_data[i][1] * 1000) / 10.0 ) if (today - yesterday) >= 0: up_deltas.append((happy_data[i][0], today - yesterday)) else: down_deltas.append((happy_data[i][0], today - yesterday)) # Nix the first total because it's not in our date range totals_data = totals_data[1:] histogram = [ { 'name': 'zero', 'data': [(totals_data[0][0], 0), (totals_data[-1][0], 0)], 'yaxis': 2, 'lines': {'show': True, 'fill': False, 'lineWidth': 1, 'shadowSize': 0}, 'color': '#dddddd', }, { 'name': 'total', 'label': 'Total # responses', 'data': totals_data, 'yaxis': 1, 'lines': {'show': True, 'fill': False}, 'points': {'show': True}, 'color': '#3E72BF', }, { 'name': 'updeltas', 'label': 'Percent change in sentiment upwards', 'data': up_deltas, 'yaxis': 2, 'bars': {'show': True, 'lineWidth': 3}, 'points': {'show': True}, 'color': '#55E744', }, { 'name': 'downdeltas', 'label': 'Percent change in sentiment downwards', 'data': down_deltas, 'yaxis': 2, 'bars': {'show': True, 'lineWidth': 3}, 'points': {'show': True}, 'color': '#E73E3E', } ] return histogram
def product_dashboard_firefox(request, prod): # Note: Not localized because it's ultra-alpha. template = 'analytics/product_dashboard_firefox.html' current_search = {} search_query = request.GET.get('q', None) if search_query: current_search['q'] = search_query search_date_end = smart_date( request.GET.get('date_end', None), fallback=None) if search_date_end is None: search_date_end = date.today() current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') search_date_start = smart_date( request.GET.get('date_start', None), fallback=None) if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') histogram = generate_totals_histogram( search_date_start, search_date_end, search_query, prod) # FIXME: This is lame, but we need to make sure the item we're # looking at is the totals. assert histogram[1]['name'] == 'total' totals_sum = sum([p[1] for p in histogram[1]['data']]) search = ResponseMappingType.search() if search_query: search = search.query(description__sqs=search_query) base_f = F() base_f &= F(product=prod.db_name) base_f &= F(created__gte=search_date_start) base_f &= F(created__lt=search_date_end) search = search.filter(base_f) # Figure out the list of platforms and versions for this range. plats_and_vers = (search .facet('platform', 'version', size=50) .facet_counts()) # Figure out the "by platform" histogram platforms = [part['term'] for part in plats_and_vers['platform']] platform_facet = {} for plat in platforms: plat_f = base_f & F(platform=plat) platform_facet[plat if plat else 'unknown'] = { 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': search._process_filters(plat_f.filters) } platform_counts = search.facet_raw(**platform_facet).facet_counts() platforms_histogram = [] for key in platform_counts.keys(): data = dict((p['time'], p['count']) for p in platform_counts[key]) sum_counts = sum([p['count'] for p in platform_counts[key]]) if sum_counts < (totals_sum * 0.02): # Skip platforms where the number of responses is less than # 2% of the total. continue zero_fill(search_date_start, search_date_end, [data]) platforms_histogram.append({ 'name': key, 'label': key, 'data': sorted(data.items()), 'lines': {'show': True, 'fill': False}, 'points': {'show': True}, }) # Figure out the "by version" histogram versions = [part['term'] for part in plats_and_vers['version']] version_facet = {} for vers in versions: vers_f = base_f & F(version=vers) version_facet['v' + vers if vers else 'unknown'] = { 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': search._process_filters(vers_f.filters) } version_counts = search.facet_raw(**version_facet).facet_counts() versions_histogram = [] for key in version_counts.keys(): data = dict((p['time'], p['count']) for p in version_counts[key]) sum_counts = sum([p['count'] for p in version_counts[key]]) if sum_counts < (totals_sum * 0.02): # Skip versions where the number of responses is less than # 2% of the total. continue zero_fill(search_date_start, search_date_end, [data]) versions_histogram.append({ 'name': key, 'label': key, 'data': sorted(data.items()), 'lines': {'show': True, 'fill': False}, 'points': {'show': True}, }) return render(request, template, { 'start_date': search_date_start, 'end_date': search_date_end, 'current_search': current_search, 'platforms_histogram': platforms_histogram, 'versions_histogram': versions_histogram, 'histogram': histogram, 'product': prod })
def analytics_search(request): template = 'analytics/analyzer/search.html' output_format = request.GET.get('format', None) page = smart_int(request.GET.get('page', 1), 1) # Note: If we add additional querystring fields, we need to add # them to generate_dashboard_url. search_happy = request.GET.get('happy', None) search_has_email = request.GET.get('has_email', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_country = request.GET.get('country', None) search_product = request.GET.get('product', None) search_domain = request.GET.get('domain', None) search_api = smart_int(request.GET.get('api', None), fallback=None) search_version = request.GET.get('version', None) search_query = request.GET.get('q', None) search_date_start = smart_date(request.GET.get('date_start', None), fallback=None) search_date_end = smart_date(request.GET.get('date_end', None), fallback=None) search_bigram = request.GET.get('bigram', None) search_source = request.GET.get('source', None) search_campaign = request.GET.get('campaign', None) search_organic = request.GET.get('organic', None) selected = request.GET.get('selected', None) filter_data = [] current_search = {'page': page} search = ResponseMappingType.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F(happy=search_happy) current_search['happy'] = int(search_happy) # If search has_email is '0' or '1', set it to False or True, # respectively. search_has_email = {'0': False, '1': True}.get(search_has_email, None) if search_has_email in [False, True]: f &= F(has_email=search_has_email) current_search['has_email'] = int(search_has_email) def unknown_to_empty(text): """Convert "Unknown" to "" to support old links""" return u'' if text.lower() == u'unknown' else text if search_platform is not None: f &= F(platform=unknown_to_empty(search_platform)) current_search['platform'] = search_platform if search_locale is not None: f &= F(locale=unknown_to_empty(search_locale)) current_search['locale'] = search_locale if search_product is not None: f &= F(product=unknown_to_empty(search_product)) current_search['product'] = search_product # Only show the version if there's a product. if search_version is not None: # Note: We only filter on version if we're filtering on # product. f &= F(version=unknown_to_empty(search_version)) current_search['version'] = search_version # Only show the country if the product is Firefox OS. if search_country is not None and search_product == 'Firefox OS': f &= F(country=unknown_to_empty(search_country)) current_search['country'] = search_country if search_domain is not None: f &= F(url_domain=unknown_to_empty(search_domain)) current_search['domain'] = search_domain if search_api is not None: f &= F(api=search_api) current_search['api'] = search_api if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = datetime.now() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') # Add one day, so that the search range includes the entire day. end = search_date_end + timedelta(days=1) # Note 'less than', not 'less than or equal', because of the added # day above. f &= F(created__lt=end) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F(created__gte=search_date_start) if search_query: current_search['q'] = search_query search = search.query(description__sqs=search_query) if search_bigram is not None: f &= F(description_bigrams=search_bigram) filter_data.append({ 'display': 'Bigram', 'name': 'bigram', 'options': [{ 'count': 'all', 'name': search_bigram, 'display': search_bigram, 'value': search_bigram, 'checked': True }] }) if search_source is not None: f &= F(source=search_source) current_search['source'] = search_source if search_campaign is not None: f &= F(campaign=search_campaign) current_search['campaign'] = search_campaign search_organic = {'0': False, '1': True}.get(search_organic, None) if search_organic in [False, True]: f &= F(organic=search_organic) current_search['organic'] = int(search_organic) search = search.filter(f).order_by('-created') # If they're asking for a CSV export, then send them to the export # screen. if output_format == 'csv': return _analytics_search_export(request, search) # Search results and pagination if page < 1: page = 1 page_count = 50 start = page_count * (page - 1) end = start + page_count search_count = search.count() search_results = search.values_list('id')[start:end] opinion_page_ids = [mem[0][0] for mem in search_results] # We convert what we get back from ES to what's in the db so we # can get all the information. opinion_page = Response.objects.filter(id__in=opinion_page_ids) # Navigation facet data # This loop does two things. First it maps 'T' -> True and 'F' -> # False. This is probably something EU should be doing for # us. Second, it restructures the data into a more convenient # form. counts = { 'happy': {}, 'has_email': {}, 'platform': {}, 'locale': {}, 'country': {}, 'product': {}, 'version': {}, 'url_domain': {}, 'api': {}, 'source': {}, 'campaign': {}, 'organic': {}, } facets = search.facet(*(counts.keys()), size=1000, filtered=bool(search._process_filters(f.filters))) for param, terms in facets.facet_counts().items(): for term in terms: name = term['term'] if name == 'T': name = True elif name == 'F': name = False counts[param][name] = term['count'] def empty_to_unknown(text): return 'Unknown' if text == u'' else text filter_data.extend([ counts_to_options(counts['happy'].items(), name='happy', display='Sentiment', display_map={ True: 'Happy', False: 'Sad' }, value_map={ True: 1, False: 0 }, checked=search_happy), counts_to_options(counts['has_email'].items(), name='has_email', display='Has email', display_map={ True: 'Yes', False: 'No' }, value_map={ True: 1, False: 0 }, checked=search_has_email), counts_to_options(counts['product'].items(), name='product', display='Product', display_map=empty_to_unknown, checked=search_product) ]) # Only show the version if we're showing a specific # product. if search_product: filter_data.append( counts_to_options(counts['version'].items(), name='version', display='Version', display_map=empty_to_unknown, checked=search_version)) # Only show the country if the product is Firefox OS. if search_product == 'Firefox OS': filter_data.append( counts_to_options(counts['country'].items(), name='country', display='Country', checked=search_country, display_map=country_name), ) filter_data.extend([ counts_to_options(counts['platform'].items(), name='platform', display='Platform', display_map=empty_to_unknown, checked=search_platform), counts_to_options(counts['locale'].items(), name='locale', display='Locale', checked=search_locale, display_map=locale_name), counts_to_options(counts['url_domain'].items(), name='domain', display='Domain', checked=search_domain, display_map=empty_to_unknown), counts_to_options(counts['api'].items(), name='api', display='API version', checked=search_api, display_map=empty_to_unknown), counts_to_options(counts['organic'].items(), name='organic', display='Organic', display_map={ True: 'Yes', False: 'No' }, value_map={ True: 1, False: 0 }, checked=search_organic), counts_to_options(counts['source'].items(), name='source', display='Source', checked=search_source, display_map=empty_to_unknown), counts_to_options(counts['campaign'].items(), name='campaign', display='Campaign', checked=search_campaign, display_map=empty_to_unknown), ]) return render( request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, })
def test_purge(self): now = datetime.datetime.now() cutoff = now - datetime.timedelta(days=5) # Create 10 objs of each type--one for each day for the last # 10 days. for i in range(10): ResponseEmailFactory( opinion__created=(now - datetime.timedelta(days=i)) ) ResponseContextFactory( opinion__created=(now - datetime.timedelta(days=i)) ) ResponsePIFactory( opinion__created=(now - datetime.timedelta(days=i)) ) # Note that this creates 30 Response objects. # Since creating the objects and indexing them happens very # quickly in tests, we hit a race condition and the has_email # column ends up being false. So instead we just drop the # index and rebuild it. self.setup_indexes() # Make sure everything is in the db eq_(Response.objects.count(), 30) eq_(ResponseEmail.objects.count(), 10) eq_(ResponseContext.objects.count(), 10) eq_(ResponsePI.objects.count(), 10) # Make sure everything is in the index resp_s = ResponseMappingType.search() eq_(resp_s.count(), 30) eq_(resp_s.filter(has_email=True).count(), 10) # Now purge everything older than 5 days and make sure things # got removed that should have gotten removed. Also check if # there is a journal entry for the purge operation. cutoff = now - datetime.timedelta(days=5) purge_data(cutoff=cutoff) self.refresh() eq_(Response.objects.count(), 30) eq_(ResponseEmail.objects.count(), 5) eq_(ResponseEmail.objects.filter( opinion__created__gte=cutoff).count(), 5) eq_(ResponseContext.objects.count(), 5) eq_(ResponseContext.objects.filter( opinion__created__gte=cutoff).count(), 5) eq_(ResponsePI.objects.count(), 5) eq_(ResponsePI.objects.filter( opinion__created__gte=cutoff).count(), 5) eq_(1, Record.objects.filter(action='purge_data').count()) expected_msg = ('feedback_responseemail: 5, ' 'feedback_responsecontext: 5, ' 'feedback_responsepi: 5') eq_(expected_msg, Record.objects.get(action='purge_data').msg) # Everything should still be in the index, but the number of # things with has_email=True should go down resp_s = ResponseMappingType.search() eq_(resp_s.count(), 30) eq_(resp_s.filter(has_email=True).count(), 5)
def analytics_occurrences(request): template = 'analytics/analyzer/occurrences.html' first_facet_bi = None first_params = {} first_facet_total = 0 second_facet_bi = None second_params = {} second_facet_total = 0 if 'product' in request.GET: form = OccurrencesComparisonForm(request.GET) if form.is_valid(): cleaned = form.cleaned_data # First item first_resp_s = (ResponseMappingType.search().filter( product=cleaned['product']).filter(locale__startswith='en')) first_params['product'] = cleaned['product'] if cleaned['first_version']: first_resp_s = first_resp_s.filter( version=cleaned['first_version']) first_params['version'] = cleaned['first_version'] if cleaned['first_start_date']: first_resp_s = first_resp_s.filter( created__gte=cleaned['first_start_date']) first_params['date_start'] = cleaned['first_start_date'] if cleaned['first_end_date']: first_resp_s = first_resp_s.filter( created__lte=cleaned['first_end_date']) first_params['date_end'] = cleaned['first_end_date'] if cleaned['first_search_term']: first_resp_s = first_resp_s.query( description__match=cleaned['first_search_term']) first_params['q'] = cleaned['first_search_term'] if ('date_start' not in first_params and 'date_end' not in first_params): # FIXME - If there's no start date, then we want # "everything" so we use a hard-coded 2013-01-01 date # here to hack that. # # Better way might be to change the dashboard to allow # for an "infinite" range, but there's no other use # case for that and the ranges are done in the ui--not # in the backend. first_params['date_start'] = '2013-01-01' first_resp_s = first_resp_s.facet('description_bigrams', size=30, filtered=True) first_resp_s = first_resp_s[0:0] first_facet_total = first_resp_s.count() first_facet = first_resp_s.facet_counts() first_facet_bi = first_facet['description_bigrams'] first_facet_bi = sorted(first_facet_bi, key=lambda item: -item['count']) if (cleaned['second_version'] or cleaned['second_search_term'] or cleaned['second_start_date']): second_resp_s = (ResponseMappingType.search().filter( product=cleaned['product']).filter( locale__startswith='en')) second_params['product'] = cleaned['product'] if cleaned['second_version']: second_resp_s = second_resp_s.filter( version=cleaned['second_version']) second_params['version'] = cleaned['second_version'] if cleaned['second_start_date']: second_resp_s = second_resp_s.filter( created__gte=cleaned['second_start_date']) second_params['date_start'] = cleaned['second_start_date'] if cleaned['second_end_date']: second_resp_s = second_resp_s.filter( created__lte=cleaned['second_end_date']) second_params['date_end'] = cleaned['second_end_date'] if form.cleaned_data['second_search_term']: second_resp_s = second_resp_s.query( description__match=cleaned['second_search_term']) second_params['q'] = cleaned['second_search_term'] if ('date_start' not in second_params and 'date_end' not in second_params): # FIXME - If there's no start date, then we want # "everything" so we use a hard-coded 2013-01-01 date # here to hack that. # # Better way might be to change the dashboard to allow # for an "infinite" range, but there's no other use # case for that and the ranges are done in the ui--not # in the backend. second_params['date_start'] = '2013-01-01' # Have to do raw because we want a size > 10. second_resp_s = second_resp_s.facet('description_bigrams', size=30, filtered=True) second_resp_s = second_resp_s[0:0] second_facet_total = second_resp_s.count() second_facet = second_resp_s.facet_counts() second_facet_bi = second_facet['description_bigrams'] second_facet_bi = sorted(second_facet_bi, key=lambda item: -item['count']) permalink = request.build_absolute_uri() else: permalink = '' form = OccurrencesComparisonForm() # FIXME - We have responses that have no product set. This ignores # those. That's probably the right thing to do for the Occurrences Report # but maybe not. products = [prod for prod in ResponseMappingType.get_products() if prod] return render( request, template, { 'permalink': permalink, 'form': form, 'products': products, 'first_facet_bi': first_facet_bi, 'first_params': first_params, 'first_facet_total': first_facet_total, 'first_normalization': round(first_facet_total * 1.0 / 1000, 3), 'second_facet_bi': second_facet_bi, 'second_params': second_params, 'second_facet_total': second_facet_total, 'second_normalization': round(second_facet_total * 1.0 / 1000, 3), 'render_time': datetime.now(), })
def dashboard(request, template): page = smart_int(request.GET.get('page', 1), 1) search_happy = request.GET.get('happy', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_query = request.GET.get('q', None) search_date_start = smart_datetime(request.GET.get('date_start', None), fallback=None) search_date_end = smart_datetime(request.GET.get('date_end', None), fallback=None) selected = request.GET.get('selected', None) current_search = {'page': page} search = ResponseMappingType.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F(happy=search_happy) current_search['happy'] = int(search_happy) if search_platform: f &= F(platform=search_platform) current_search['platform'] = search_platform if search_locale: f &= F(locale=search_locale) current_search['locale'] = search_locale if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = datetime.now() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') # Add one day, so that the search range includes the entire day. end = search_date_end + timedelta(days=1) # Note 'less than', not 'less than or equal', because of the added # day above. f &= F(created__lt=end) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F(created__gte=search_date_start) if search_query: fields = ['text', 'text_phrase', 'fuzzy'] query = dict(('description__%s' % f, search_query) for f in fields) search = search.query(or_=query) current_search['q'] = search_query search = search.filter(f).order_by('-created') facets = search.facet('happy', 'platform', 'locale', filtered=bool(f.filters)) # This loop does two things. First it maps 'T' -> True and 'F' -> # False. This is probably something EU should be doing for # us. Second, it restructures the data into a more convenient # form. counts = {'happy': {}, 'platform': {}, 'locale': {}} for param, terms in facets.facet_counts().items(): for term in terms: name = term['term'] if name == 'T': name = True elif name == 'F': name = False counts[param][name] = term['count'] filter_data = [ counts_to_options(counts['happy'].items(), name='happy', display=_('Sentiment'), display_map={True: _('Happy'), False: _('Sad')}, value_map={True: 1, False: 0}, checked=search_happy), counts_to_options(counts['platform'].items(), name='platform', display=_('Platform'), checked=search_platform), counts_to_options(counts['locale'].items(), name='locale', display=_('Locale'), checked=search_locale, display_map=locale_name) ] # Histogram data happy_data = [] sad_data = [] histograms = search.facet_raw( happy={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': (f & F(happy=True)).filters }, sad={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': (f & F(happy=False)).filters }, ).facet_counts() # p['time'] is number of milliseconds since the epoch. Which is # convenient, because that is what the front end wants. happy_data = dict((p['time'], p['count']) for p in histograms['happy']) sad_data = dict((p['time'], p['count']) for p in histograms['sad']) _zero_fill(search_date_start, search_date_end, [happy_data, sad_data]) histogram = [ {'label': _('Happy'), 'name': 'happy', 'data': sorted(happy_data.items())}, {'label': _('Sad'), 'name': 'sad', 'data': sorted(sad_data.items())}, ] # Pagination if page < 1: page = 1 page_count = 20 start = page_count * (page - 1) end = start + page_count search_count = search.count() opinion_page = search[start:end] return render(request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'histogram': histogram, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, })
def dashboard(request, template): output_format = request.GET.get('format', None) page = smart_int(request.GET.get('page', 1), 1) # Note: If we add additional querystring fields, we need to add # them to generate_dashboard_url. search_happy = request.GET.get('happy', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_product = request.GET.get('product', None) search_version = request.GET.get('browser_version', None) search_query = request.GET.get('q', None) search_date_start = smart_datetime(request.GET.get('date_start', None), fallback=None) search_date_end = smart_datetime(request.GET.get('date_end', None), fallback=None) selected = request.GET.get('selected', None) current_search = {'page': page} search = ResponseMappingType.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F(happy=search_happy) current_search['happy'] = int(search_happy) if search_platform: f &= F(platform=search_platform) current_search['platform'] = search_platform if search_locale: f &= F(locale=search_locale) current_search['locale'] = search_locale if search_product: f &= F(product=search_product) current_search['product'] = search_product if search_version: # Note: We only filter on version if we're filtering on # product. f &= F(browser_version=search_version) current_search['browser_version'] = search_version if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = datetime.now() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') # Add one day, so that the search range includes the entire day. end = search_date_end + timedelta(days=1) # Note 'less than', not 'less than or equal', because of the added # day above. f &= F(created__lt=end) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F(created__gte=search_date_start) if search_query: fields = ['text', 'text_phrase', 'fuzzy'] query = dict(('description__%s' % f, search_query) for f in fields) search = search.query(or_=query) current_search['q'] = search_query search = search.filter(f).order_by('-created') # If the user asked for a feed, give him/her a feed! if output_format == 'atom': return generate_atom_feed(request, search) elif output_format == 'json': return generate_json_feed(request, search) # Search results and pagination if page < 1: page = 1 page_count = 20 start = page_count * (page - 1) end = start + page_count search_count = search.count() opinion_page = search[start:end] # Navigation facet data facets = search.facet( 'happy', 'platform', 'locale', 'product', 'browser_version', filtered=bool(f.filters)) # This loop does two things. First it maps 'T' -> True and 'F' -> # False. This is probably something EU should be doing for # us. Second, it restructures the data into a more convenient # form. counts = { 'happy': {}, 'platform': {}, 'locale': {}, 'product': {}, 'browser_version': {} } for param, terms in facets.facet_counts().items(): for term in terms: name = term['term'] if name == 'T': name = True elif name == 'F': name = False counts[param][name] = term['count'] filter_data = [ counts_to_options( counts['happy'].items(), name='happy', display=_('Sentiment'), display_map={True: _('Happy'), False: _('Sad')}, value_map={True: 1, False: 0}, checked=search_happy), counts_to_options( counts['product'].items(), name='product', display=_('Product'), checked=search_product) ] # Only show the browser_version if we're showing a specific # product. if search_product: filter_data.append( counts_to_options( counts['browser_version'].items(), name='browser_version', display=_('Version'), checked=search_version) ) filter_data.extend( [ counts_to_options( counts['platform'].items(), name='platform', display=_('Platform'), checked=search_platform), counts_to_options( counts['locale'].items(), name='locale', display=_('Locale'), checked=search_locale, display_map=locale_name), ] ) # Histogram data happy_data = [] sad_data = [] histograms = search.facet_raw( happy={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': (f & F(happy=True)).filters }, sad={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': (f & F(happy=False)).filters }, ).facet_counts() # p['time'] is number of milliseconds since the epoch. Which is # convenient, because that is what the front end wants. happy_data = dict((p['time'], p['count']) for p in histograms['happy']) sad_data = dict((p['time'], p['count']) for p in histograms['sad']) _zero_fill(search_date_start, search_date_end, [happy_data, sad_data]) histogram = [ {'label': _('Happy'), 'name': 'happy', 'data': sorted(happy_data.items())}, {'label': _('Sad'), 'name': 'sad', 'data': sorted(sad_data.items())}, ] return render(request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'histogram': histogram, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, 'atom_url': generate_dashboard_atom_url(request) })
def dashboard(request, template): output_format = request.GET.get('format', None) page = smart_int(request.GET.get('page', 1), 1) # Note: If we add additional querystring fields, we need to add # them to generate_dashboard_url. search_happy = request.GET.get('happy', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_product = request.GET.get('product', None) search_version = request.GET.get('browser_version', None) search_query = request.GET.get('q', None) search_date_start = smart_datetime(request.GET.get('date_start', None), fallback=None) search_date_end = smart_datetime(request.GET.get('date_end', None), fallback=None) selected = request.GET.get('selected', None) current_search = {'page': page} search = ResponseMappingType.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F(happy=search_happy) current_search['happy'] = int(search_happy) if search_platform: f &= F(platform=search_platform) current_search['platform'] = search_platform if search_locale: f &= F(locale=search_locale) current_search['locale'] = search_locale if search_product: f &= F(product=search_product) current_search['product'] = search_product if search_version: # Note: We only filter on version if we're filtering on # product. f &= F(browser_version=search_version) current_search['browser_version'] = search_version if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = datetime.now() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') # Add one day, so that the search range includes the entire day. end = search_date_end + timedelta(days=1) # Note 'less than', not 'less than or equal', because of the added # day above. f &= F(created__lt=end) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F(created__gte=search_date_start) if search_query: current_search['q'] = search_query es_query = generate_query_parsed('description', search_query) search = search.query_raw(es_query) search = search.filter(f).order_by('-created') # If the user asked for a feed, give him/her a feed! if output_format == 'atom': return generate_atom_feed(request, search) elif output_format == 'json': return generate_json_feed(request, search) # Search results and pagination if page < 1: page = 1 page_count = 20 start = page_count * (page - 1) end = start + page_count search_count = search.count() opinion_page = search[start:end] # Navigation facet data facets = search.facet( 'happy', 'platform', 'locale', 'product', 'browser_version', filtered=bool(search._process_filters(f.filters))) # This loop does two things. First it maps 'T' -> True and 'F' -> # False. This is probably something EU should be doing for # us. Second, it restructures the data into a more convenient # form. counts = { 'happy': {}, 'platform': {}, 'locale': {}, 'product': {}, 'browser_version': {} } for param, terms in facets.facet_counts().items(): for term in terms: name = term['term'] if name == 'T': name = True elif name == 'F': name = False counts[param][name] = term['count'] filter_data = [ counts_to_options( counts['happy'].items(), name='happy', display=_('Sentiment'), display_map={True: _('Happy'), False: _('Sad')}, value_map={True: 1, False: 0}, checked=search_happy), counts_to_options( counts['product'].items(), name='product', display=_('Product'), checked=search_product) ] # Only show the browser_version if we're showing a specific # product. if search_product: filter_data.append( counts_to_options( counts['browser_version'].items(), name='browser_version', display=_('Version'), checked=search_version) ) filter_data.extend( [ counts_to_options( counts['platform'].items(), name='platform', display=_('Platform'), checked=search_platform), counts_to_options( counts['locale'].items(), name='locale', display=_('Locale'), checked=search_locale, display_map=locale_name), ] ) # Histogram data happy_data = [] sad_data = [] happy_f = f & F(happy=True) sad_f = f & F(happy=False) histograms = search.facet_raw( happy={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': search._process_filters(happy_f.filters) }, sad={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': search._process_filters(sad_f.filters) }, ).facet_counts() # p['time'] is number of milliseconds since the epoch. Which is # convenient, because that is what the front end wants. happy_data = dict((p['time'], p['count']) for p in histograms['happy']) sad_data = dict((p['time'], p['count']) for p in histograms['sad']) _zero_fill(search_date_start, search_date_end, [happy_data, sad_data]) histogram = [ {'label': _('Happy'), 'name': 'happy', 'data': sorted(happy_data.items())}, {'label': _('Sad'), 'name': 'sad', 'data': sorted(sad_data.items())}, ] return render(request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'histogram': histogram, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, 'atom_url': generate_dashboard_atom_url(request) })