def thanks(request): feedback = None suggestions = None # FIXME: Hard-coded default product. product = u'Firefox' response_id = None # If the user is an analyzer/admin, then we let them specify # the response_id via the querystring. This makes debugging # the system easier. if ((request.user.is_authenticated() and request.user.has_perm('analytics.can_view_dashboard'))): response_id = smart_int(request.GET.get('response_id', None)) # If we don't have a response_id, then pull it from the # session where it was placed if the user had just left # feedback. if not response_id: response_id = request.session.get('response_id') if response_id: try: feedback = Response.objects.get(id=response_id) except Response.DoesNotExist: pass if feedback: product = feedback.product suggestions = get_suggestions(feedback, request) return render(request, 'feedback/thanks.html', { 'product': product, 'feedback': feedback, 'suggestions': suggestions })
def thanks_view(request): feedback = None suggestions = None response_id = None # If the user is an analyzer/admin, then we let them specify # the response_id via the querystring. This makes debugging # the system easier. if ((request.user.is_authenticated() and request.user.has_perm('analytics.can_view_dashboard'))): response_id = smart_int(request.GET.get('response_id', None)) # If we don't have a response_id, then pull it from the # session where it was placed if the user had just left # feedback. if not response_id: response_id = request.session.get('response_id') if response_id: try: feedback = Response.objects.get(id=response_id) except Response.DoesNotExist: pass if feedback: product = Product.objects.get(db_name=feedback.product) suggestions = get_suggestions(feedback, request) else: # If there's no feedback, then we just show the thanks page as if it # were for Firefox. This is a weird edge case that might happen if the # user has cookies disabled or something like that. product = Product.objects.get(db_name=u'Firefox') template = get_config(product.slug)['thanks_template'] return render(request, template, { 'product': product, 'feedback': feedback, 'suggestions': suggestions })
def get(self, request): flavorslugs = smart_str(request.GET.get('flavors', '')).split(',') max_count = smart_int(request.GET.get('max', None)) max_count = max_count or 100 max_count = min(max(1, max_count), 10000) if not flavorslugs: return self.rest_error( status=400, errors='You must specify flavors to retrieve alerts for.') flavors = [] for flavorslug in flavorslugs: try: flavor = AlertFlavor.objects.get(slug=flavorslug) except AlertFlavor.DoesNotExist: return self.rest_error( status=404, errors='Flavor "{}" does not exist.'.format(flavorslug)) self.check_object_permissions(request, flavor) if not flavor.enabled: return self.rest_error( status=400, errors='Flavor "{}" is disabled.'.format(flavorslug)) flavors.append(flavor) alerts = Alert.objects.filter(flavor__in=flavors).order_by('-created') alerts_ser = AlertSerializer(alerts[:max_count], many=True) return rest_framework.response.Response({ 'total': alerts.count(), 'count': len(alerts_ser.data), 'alerts': alerts_ser.data })
def test_empty_string(self): assert 0 == smart_int('')
def dashboard(request): template = 'analytics/dashboard.html' output_format = request.GET.get('format', None) page = smart_int(request.GET.get('page', 1), 1) # Note: If we add additional querystring fields, we need to add # them to generate_dashboard_url. search_happy = request.GET.get('happy', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_product = request.GET.get('product', None) search_version = request.GET.get('version', None) search_query = request.GET.get('q', None) search_date_start = smart_date(request.GET.get('date_start', None), fallback=None) search_date_end = smart_date(request.GET.get('date_end', None), fallback=None) selected = request.GET.get('selected', None) filter_data = [] current_search = {'page': page} search = ResponseDocType.docs.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F('term', happy=search_happy) current_search['happy'] = int(search_happy) def unknown_to_empty(text): """Convert "Unknown" to "" to support old links""" return u'' if text.lower() == u'unknown' else text if search_platform is not None: f &= F('term', platform=unknown_to_empty(search_platform)) current_search['platform'] = search_platform if search_locale is not None: f &= F('term', locale=unknown_to_empty(search_locale)) current_search['locale'] = search_locale visible_products = [ prod.encode('utf-8') for prod in Product.objects.on_dashboard().values_list('db_name', flat=True) ] # This covers the "unknown" product which is also visible. visible_products.append('') if search_product in visible_products: f &= F('term', product=unknown_to_empty(search_product)) current_search['product'] = search_product if search_version is not None: # Note: We only filter on version if we're filtering on # product. f &= F('term', version=unknown_to_empty(search_version)) current_search['version'] = search_version else: f &= F('terms', product=visible_products) if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = date.today() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) # If the start and end dates are inverted, switch them into proper # chronoligcal order search_date_start, search_date_end = sorted( [search_date_start, search_date_end]) # Restrict the frontpage dashboard to only show the last 6 months # of data six_months_ago = date.today() - timedelta(days=180) search_date_start = max(six_months_ago, search_date_start) search_date_end = max(search_date_start, search_date_end) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') f &= F('range', created={'lte': search_date_end}) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F('range', created={'gte': search_date_start}) if search_query: current_search['q'] = search_query search = search.query('simple_query_string', query=search_query, fields=['description']) search = search.filter(f).sort('-created') # If the user asked for a feed, give him/her a feed! if output_format == 'atom': return generate_atom_feed(request, list(search.execute())) elif output_format == 'json': return generate_json_feed(request, list(search.execute())) # Search results and pagination if page < 1: page = 1 page_count = 20 start = page_count * (page - 1) end = start + page_count search_count = search.count() opinion_page = search[start:end].execute() # Add navigation aggregations counts = { 'happy': {}, 'platform': {}, 'locale': {}, 'product': {}, 'version': {} } for name in counts.keys(): search.aggs.bucket(name, 'terms', field=name, size=1000) happy_sad_filter = request.GET.get('happy', None) if happy_sad_filter: if happy_sad_filter == '1': counts['happy'] = {True: 0} elif happy_sad_filter == '0': counts['happy'] = {False: 0} else: counts['happy'] = {True: 0, False: 0} if search_platform: counts['platform'] = {search_platform: 0} if search_locale: counts['locale'] = {search_locale: 0} if search_product: counts['product'] = {search_product: 0} if search_version: counts['version'] = {search_version: 0} results = search.execute() # Extract the value and doc_count for the various facets we do # faceted navigation on. for name in counts.keys(): buckets = getattr(results.aggregations, name)['buckets'] for bucket in buckets: key = bucket['key'] # Convert from 'T'/'F' to True/False if key in ['T', 'F']: key = (key == 'T') counts[name][key] = bucket['doc_count'] def empty_to_unknown(text): return _('Unknown') if text == u'' else text filter_data.extend([ counts_to_options(counts['happy'].items(), name='happy', display=_('Sentiment'), display_map={ True: _('Happy'), False: _('Sad') }, value_map={ True: 1, False: 0 }, checked=search_happy), counts_to_options(counts['product'].items(), name='product', display=_('Product'), display_map=empty_to_unknown, checked=search_product) ]) # Only show the version if we're showing a specific # product. if search_product: filter_data.append( counts_to_options(counts['version'].items(), name='version', display=_('Version'), display_map=empty_to_unknown, checked=search_version)) else: filter_data.append({ 'display': _('Version'), 'note': _('Select product to see version breakdown') }) filter_data.extend([ counts_to_options(counts['platform'].items(), name='platform', display=_('Platform'), display_map=empty_to_unknown, checked=search_platform), counts_to_options(counts['locale'].items(), name='locale', display=_('Locale'), checked=search_locale, display_map=locale_name), ]) return render( request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, 'atom_url': generate_dashboard_url(request), })
def test_int(self): assert 10 == smart_int(10)
def test_overflow(self): eq_(0, smart_int('1e309'))
def test_invalid_string(self): eq_(0, smart_int("invalid"))
def test_int(self): eq_(10, smart_int(10))
def test_empty_string(self): eq_(0, smart_int(''))
def test_overflow(self): eq_(0, smart_int("1e309"))
def dashboard(request): template = 'analytics/dashboard.html' output_format = request.GET.get('format', None) page = smart_int(request.GET.get('page', 1), 1) # Note: If we add additional querystring fields, we need to add # them to generate_dashboard_url. search_happy = request.GET.get('happy', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_product = request.GET.get('product', None) search_version = request.GET.get('version', None) search_query = request.GET.get('q', None) search_date_start = smart_date( request.GET.get('date_start', None), fallback=None) search_date_end = smart_date( request.GET.get('date_end', None), fallback=None) search_bigram = request.GET.get('bigram', None) selected = request.GET.get('selected', None) filter_data = [] current_search = {'page': page} search = ResponseMappingType.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F(happy=search_happy) current_search['happy'] = int(search_happy) def unknown_to_empty(text): """Convert "Unknown" to "" to support old links""" return u'' if text.lower() == u'unknown' else text if search_platform is not None: f &= F(platform=unknown_to_empty(search_platform)) current_search['platform'] = search_platform if search_locale is not None: f &= F(locale=unknown_to_empty(search_locale)) current_search['locale'] = search_locale visible_products = [ prod.encode('utf-8') for prod in Product.objects.public().values_list('db_name', flat=True) ] # This covers the "unknown" product which is also visible. visible_products.append('') if search_product in visible_products: f &= F(product=unknown_to_empty(search_product)) current_search['product'] = search_product if search_version is not None: # Note: We only filter on version if we're filtering on # product. f &= F(version=unknown_to_empty(search_version)) current_search['version'] = search_version else: f &= F(product__in=visible_products) if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = date.today() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) # If the start and end dates are inverted, switch them into proper # chronoligcal order search_date_start, search_date_end = sorted( [search_date_start, search_date_end]) # Restrict the frontpage dashboard to only show the last 6 months # of data six_months_ago = date.today() - timedelta(days=180) search_date_start = max(six_months_ago, search_date_start) search_date_end = max(search_date_start, search_date_end) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') f &= F(created__lte=search_date_end) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F(created__gte=search_date_start) if search_query: current_search['q'] = search_query search = search.query(description__sqs=search_query) if search_bigram is not None: f &= F(description_bigrams=search_bigram) filter_data.append({ 'display': _('Bigram'), 'name': 'bigram', 'options': [{ 'count': 'all', 'name': search_bigram, 'display': search_bigram, 'value': search_bigram, 'checked': True }] }) search = search.filter(f).order_by('-created') # If the user asked for a feed, give him/her a feed! if output_format == 'atom': return generate_atom_feed(request, search) elif output_format == 'json': return generate_json_feed(request, search) # Search results and pagination if page < 1: page = 1 page_count = 20 start = page_count * (page - 1) end = start + page_count search_count = search.count() opinion_page = search[start:end] # Navigation facet data facets = search.facet( 'happy', 'platform', 'locale', 'product', 'version', size=1000, filtered=bool(search._process_filters(f.filters))) # This loop does two things. First it maps 'T' -> True and 'F' -> # False. This is probably something EU should be doing for # us. Second, it restructures the data into a more convenient # form. counts = { 'happy': {}, 'platform': {}, 'locale': {}, 'product': {}, 'version': {} } happy_sad_filter = request.GET.get('happy', None) if happy_sad_filter: if happy_sad_filter == '1': counts['happy'] = {True: 0} elif happy_sad_filter == '0': counts['happy'] = {False: 0} if search_platform: counts['platform'] = {search_platform: 0} if search_locale: counts['locale'] = {search_locale: 0} if search_product: counts['product'] = {search_product: 0} if search_version: counts['version'] = {search_version: 0} for param, terms in facets.facet_counts().items(): for term in terms: name = term['term'] if name.upper() == 'T': name = True elif name.upper() == 'F': name = False counts[param][name] = term['count'] def empty_to_unknown(text): return _('Unknown') if text == u'' else text filter_data.extend([ counts_to_options( counts['happy'].items(), name='happy', display=_('Sentiment'), display_map={True: _('Happy'), False: _('Sad')}, value_map={True: 1, False: 0}, checked=search_happy), counts_to_options( counts['product'].items(), name='product', display=_('Product'), display_map=empty_to_unknown, checked=search_product) ]) # Only show the version if we're showing a specific # product. if search_product: filter_data.append( counts_to_options( counts['version'].items(), name='version', display=_('Version'), display_map=empty_to_unknown, checked=search_version) ) else: filter_data.append({ 'display': _('Version'), 'note': _('Select product to see version breakdown') }) filter_data.extend( [ counts_to_options( counts['platform'].items(), name='platform', display=_('Platform'), display_map=empty_to_unknown, checked=search_platform), counts_to_options( counts['locale'].items(), name='locale', display=_('Locale'), checked=search_locale, display_map=locale_name), ] ) # Histogram data happy_data = [] sad_data = [] happy_f = f & F(happy=True) sad_f = f & F(happy=False) histograms = search.facet_raw( happy={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': search._process_filters(happy_f.filters) }, sad={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': search._process_filters(sad_f.filters) }, ).facet_counts() # p['time'] is number of milliseconds since the epoch. Which is # convenient, because that is what the front end wants. happy_data = dict((p['time'], p['count']) for p in histograms['happy']) sad_data = dict((p['time'], p['count']) for p in histograms['sad']) zero_fill(search_date_start, search_date_end, [happy_data, sad_data]) histogram = [ {'label': _('Happy'), 'name': 'happy', 'data': sorted(happy_data.items())}, {'label': _('Sad'), 'name': 'sad', 'data': sorted(sad_data.items())}, ] return render(request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'histogram': histogram, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, 'atom_url': generate_dashboard_url(request), })
def test_wrong_type(self): assert 0 == smart_int(None) assert 10 == smart_int([], 10)
def test_overflow(self): assert 0 == smart_int('1e309')
def test_invalid_string(self): assert 0 == smart_int('invalid')
def test_sanity(self): assert 10 == smart_int('10') assert 10 == smart_int('10.5')
def get(self, request): """Returns JSON feed of first 10000 results This feels like a duplication of the front-page dashboard search logic, but it's separate which allows us to handle multiple values. """ search = models.ResponseMappingType.search() f = F() if 'happy' in request.GET: happy = {'0': False, '1': True}.get(request.GET['happy'], None) if happy is not None: f &= F(happy=happy) if 'platforms' in request.GET: platforms = request.GET['platforms'].split(',') if platforms: f &= F(platform__in=platforms) if 'locales' in request.GET: locales = request.GET['locales'].split(',') if locales: f &= F(locale__in=locales) if 'products' in request.GET: products = request.GET['products'].split(',') if products: f &= F(product__in=products) if 'versions' in request.GET: versions = request.GET['versions'].split(',') if versions: f &= F(version__in=versions) date_start = smart_date(request.GET.get('date_start', None)) date_end = smart_date(request.GET.get('date_end', None)) delta = smart_timedelta(request.GET.get('date_delta', None)) if delta is not None: if date_end is not None: date_start = date_end - delta elif date_start is not None: date_end = date_start + delta else: date_end = date.today() date_start = date_end - delta # We restrict public API access to the last 6 months. six_months_ago = date.today() - timedelta(days=180) if date_start: date_start = max(six_months_ago, date_start) f &= F(created__gte=date_start) if date_end: date_end = max(six_months_ago, date_end) f &= F(created__lte=date_end) search = search.filter(f) search_query = request.GET.get('q', None) if search_query is not None: search = search.query(description__sqs=search_query) # FIXME: Probably want to make this specifyable search = search.order_by('-created') # Explicitly include only publicly visible fields search = search.values_dict(*models.ResponseMappingType.public_fields()) maximum = smart_int(request.GET.get('max', None)) maximum = maximum or 1000 maximum = min(max(1, maximum), 10000) responses = models.ResponseMappingType.reshape(search[:maximum]) return rest_framework.response.Response({ 'count': len(responses), 'results': list(responses) })
def test_sanity(self): eq_(10, smart_int("10")) eq_(10, smart_int("10.5"))
def get(self, request): """Returns JSON feed of first 10000 results This feels like a duplication of the front-page dashboard search logic, but it's separate which allows us to handle multiple values. """ search = models.ResponseMappingType.search() f = F() if 'happy' in request.GET: happy = {'0': False, '1': True}.get(request.GET['happy'], None) if happy is not None: f &= F(happy=happy) if 'platforms' in request.GET: platforms = request.GET['platforms'].split(',') if platforms: f &= F(platform__in=platforms) if 'locales' in request.GET: locales = request.GET['locales'].split(',') if locales: f &= F(locale__in=locales) if 'products' in request.GET: products = request.GET['products'].split(',') if products: f &= F(product__in=products) if 'versions' in request.GET: versions = request.GET['versions'].split(',') if versions: f &= F(version__in=versions) date_start = smart_date(request.GET.get('date_start', None)) date_end = smart_date(request.GET.get('date_end', None)) delta = smart_timedelta(request.GET.get('date_delta', None)) if delta is not None: if date_end is not None: date_start = date_end - delta elif date_start is not None: date_end = date_start + delta else: date_end = date.today() date_start = date_end - delta # We restrict public API access to the last 6 months. six_months_ago = date.today() - timedelta(days=180) if date_start: date_start = max(six_months_ago, date_start) f &= F(created__gte=date_start) if date_end: date_end = max(six_months_ago, date_end) f &= F(created__lte=date_end) search = search.filter(f) search_query = request.GET.get('q', None) if search_query is not None: search = search.query(description__sqs=search_query) # FIXME: Probably want to make this specifyable search = search.order_by('-created') # Explicitly include only publicly visible fields search = search.values_dict(models.ResponseMappingType.public_fields()) maximum = smart_int(request.GET.get('max', None)) maximum = maximum or 1000 maximum = min(max(1, maximum), 10000) responses = models.ResponseMappingType.reshape(search[:maximum]) return rest_framework.response.Response({ 'count': len(responses), 'results': list(responses) })
def test_invalid_string(self): eq_(0, smart_int('invalid'))
def analytics_search(request): template = 'analytics/analyzer/search.html' output_format = request.GET.get('format', None) page = smart_int(request.GET.get('page', 1), 1) # Note: If we add additional querystring fields, we need to add # them to generate_dashboard_url. search_happy = request.GET.get('happy', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_product = request.GET.get('product', None) search_version = request.GET.get('version', None) search_query = request.GET.get('q', None) search_date_start = smart_date( request.GET.get('date_start', None), fallback=None) search_date_end = smart_date( request.GET.get('date_end', None), fallback=None) search_bigram = request.GET.get('bigram', None) selected = request.GET.get('selected', None) search_has_email = request.GET.get('has_email', None) search_country = request.GET.get('country', None) search_domain = request.GET.get('domain', None) search_api = smart_int(request.GET.get('api', None), fallback=None) search_source = request.GET.get('source', None) search_campaign = request.GET.get('campaign', None) search_organic = request.GET.get('organic', None) filter_data = [] current_search = {'page': page} search = ResponseDocType.docs.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F('term', happy=search_happy) current_search['happy'] = int(search_happy) # If search has_email is '0' or '1', set it to False or True, # respectively. search_has_email = {'0': False, '1': True}.get(search_has_email, None) if search_has_email in [False, True]: f &= F('term', has_email=search_has_email) current_search['has_email'] = int(search_has_email) def unknown_to_empty(text): """Convert "Unknown" to "" to support old links""" return u'' if text.lower() == u'unknown' else text if search_platform is not None: f &= F('term', platform=unknown_to_empty(search_platform)) current_search['platform'] = search_platform if search_locale is not None: f &= F('term', locale=unknown_to_empty(search_locale)) current_search['locale'] = search_locale if search_product is not None: f &= F('term', product=unknown_to_empty(search_product)) current_search['product'] = search_product # Only show the version if there's a product. if search_version is not None: # Note: We only filter on version if we're filtering on # product. f &= F('term', version=unknown_to_empty(search_version)) current_search['version'] = search_version # Only show the country if the product is Firefox OS. if search_country is not None and search_product == 'Firefox OS': f &= F('term', country=unknown_to_empty(search_country)) current_search['country'] = search_country if search_domain is not None: f &= F('term', url_domain=unknown_to_empty(search_domain)) current_search['domain'] = search_domain if search_api is not None: f &= F('term', api=search_api) current_search['api'] = search_api if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = date.today() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) # If the start and end dates are inverted, switch them into proper # chronological order search_date_start, search_date_end = sorted( [search_date_start, search_date_end]) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') f &= F('range', created={'lte': search_date_end}) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F('range', created={'gte': search_date_start}) if search_query: current_search['q'] = search_query search = search.query('simple_query_string', query=search_query, fields=['description']) if search_bigram is not None: f &= F('terms', description_bigrams=search_bigram) filter_data.append({ 'display': 'Bigram', 'name': 'bigram', 'options': [{ 'count': 'all', 'name': search_bigram, 'display': search_bigram, 'value': search_bigram, 'checked': True }] }) if search_source is not None: f &= F('term', source=search_source) current_search['source'] = search_source if search_campaign is not None: f &= F('term', campaign=search_campaign) current_search['campaign'] = search_campaign search_organic = {'0': False, '1': True}.get(search_organic, None) if search_organic in [False, True]: f &= F('term', organic=search_organic) current_search['organic'] = int(search_organic) search = search.filter(f).sort('-created') # If they're asking for a CSV export, then send them to the export # screen. if output_format == 'csv': return _analytics_search_export(request, search) original_search = search._clone() # Search results and pagination if page < 1: page = 1 page_count = 50 start = page_count * (page - 1) end = start + page_count search_count = search.count() search_results = search.fields('id')[start:end].execute() opinion_page_ids = [mem['id'][0] for mem in search_results] # We convert what we get back from ES to what's in the db so we # can get all the information. opinion_page = Response.objects.filter(id__in=opinion_page_ids) # Add navigation aggregations counts = { 'happy': {}, 'has_email': {}, 'platform': {}, 'locale': {}, 'country': {}, 'product': {}, 'version': {}, 'url_domain': {}, 'api': {}, 'source': {}, 'campaign': {}, 'organic': {}, } for name in counts.keys(): search.aggs.bucket(name, 'terms', field=name, size=1000) results = search.execute() # Extract the value and doc_count for the various facets we do # faceted navigation on. for name in counts.keys(): buckets = getattr(results.aggregations, name)['buckets'] for bucket in buckets: key = bucket['key'] # Convert from 'T'/'F' to True/False if key in ['T', 'F']: key = (key == 'T') counts[name][key] = bucket['doc_count'] def empty_to_unknown(text): return 'Unknown' if text == u'' else text filter_data.extend([ counts_to_options( counts['happy'].items(), name='happy', display='Sentiment', display_map={True: 'Happy', False: 'Sad'}, value_map={True: 1, False: 0}, checked=search_happy), counts_to_options( counts['has_email'].items(), name='has_email', display='Has email', display_map={True: 'Yes', False: 'No'}, value_map={True: 1, False: 0}, checked=search_has_email), counts_to_options( counts['product'].items(), name='product', display='Product', display_map=empty_to_unknown, checked=search_product) ]) # Only show the version if we're showing a specific # product. if search_product: filter_data.append( counts_to_options( counts['version'].items(), name='version', display='Version', display_map=empty_to_unknown, checked=search_version) ) # Only show the country if the product is Firefox OS. if search_product == 'Firefox OS': filter_data.append( counts_to_options( counts['country'].items(), name='country', display='Country', checked=search_country, display_map=country_name), ) filter_data.extend( [ counts_to_options( counts['platform'].items(), name='platform', display='Platform', display_map=empty_to_unknown, checked=search_platform), counts_to_options( counts['locale'].items(), name='locale', display='Locale', checked=search_locale, display_map=locale_name), counts_to_options( counts['url_domain'].items(), name='domain', display='Domain', checked=search_domain, display_map=empty_to_unknown), counts_to_options( counts['api'].items(), name='api', display='API version', checked=search_api, display_map=empty_to_unknown), counts_to_options( counts['organic'].items(), name='organic', display='Organic', display_map={True: 'Yes', False: 'No'}, value_map={True: 1, False: 0}, checked=search_organic), counts_to_options( counts['source'].items(), name='source', display='Source', checked=search_source, display_map=empty_to_unknown), counts_to_options( counts['campaign'].items(), name='campaign', display='Campaign', checked=search_campaign, display_map=empty_to_unknown), ] ) # Histogram data happy_data = [] sad_data = [] (original_search.aggs .bucket('histogram', 'date_histogram', field='created', interval='day') .bucket('per_sentiment', 'terms', field='happy') ) results = original_search.execute() buckets = results.aggregations['histogram']['buckets'] happy_data = {} sad_data = {} for bucket in buckets: # value -> count val_counts = dict( (item['key'], item['doc_count']) for item in bucket['per_sentiment']['buckets'] ) # key is ms since epoch here which is what the frontend wants, so # we can just leave it. happy_data[bucket['key']] = val_counts.get('T', 0) sad_data[bucket['key']] = val_counts.get('F', 0) zero_fill(search_date_start, search_date_end, [happy_data, sad_data]) histogram = [ {'label': 'Happy', 'name': 'happy', 'data': sorted(happy_data.items())}, {'label': 'Sad', 'name': 'sad', 'data': sorted(sad_data.items())}, ] return render(request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'histogram': histogram, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, })
def test_wrong_type(self): eq_(0, smart_int(None)) eq_(10, smart_int([], 10))
def analytics_flagged(request): """View showing responses with flags NOTE: This is not permanent and might go away depending on how the spicedham prototype works. """ template = 'analytics/analyzer/flags.html' # FIXME: Importing this here so all the changes are localized to # this function. If we decide to go forward with this, we should # unlocalize it. from django.contrib import messages from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.http import HttpResponseRedirect from fjord.flags.models import Flag if request.method == 'POST': flag_action = request.POST.get('flag') if flag_action: # We do some shenanigans here to make sure we're fetching # and operating on a flag_set that's not # cached. django-cache-machine doesn't invalidate m2m # queries correctly. :( resp = get_object_or_404( Response, pk=smart_int(request.POST['id'])) flag = get_object_or_404(Flag, name=flag_action) resp_flags = dict([(f.name, f) for f in resp.flag_set.no_cache().all()]) if flag.name in resp_flags: del resp_flags[flag.name] messages.success(request, 'removed %s flag from %d' % ( flag_action, resp.id)) else: resp_flags[flag.name] = flag messages.success(request, 'added %s flag from %d' % ( flag_action, resp.id)) resp.flag_set.clear() resp.flag_set.add(*resp_flags.values()) return HttpResponseRedirect(request.get_full_path()) resp_filter = smart_str(request.GET.get('filter')) # Only look at en-US locale responses since Monday September 8th, # 2014 we pushed the integration code out. response_list = (Response.uncached .filter(locale=u'en-US') .filter(created__gte='2014-09-08')) counts = { 'total': response_list.count(), 'abuse': response_list.filter(flag__name='abuse').count(), 'abuse-wrong': response_list.filter(flag__name='abuse-wrong').count(), 'false-positive': (response_list .filter(flag__name='abuse') .filter(flag__name='abuse-wrong').count()), } counts['false-negative'] = ( counts['abuse-wrong'] - counts['false-positive'] ) if resp_filter: response_list = response_list.filter(flag__name=resp_filter) paginator = Paginator(response_list, 50) page = request.GET.get('page') try: responses = paginator.page(page) except PageNotAnInteger: responses = paginator.page(1) except EmptyPage: responses = paginator.page(paginator.num_pages) return render(request, template, { 'counts': counts, 'responses': responses })
def dashboard(request): template = 'analytics/dashboard.html' output_format = request.GET.get('format', None) page = smart_int(request.GET.get('page', 1), 1) # Note: If we add additional querystring fields, we need to add # them to generate_dashboard_url. search_happy = request.GET.get('happy', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_product = request.GET.get('product', None) search_version = request.GET.get('version', None) search_query = request.GET.get('q', None) search_date_start = smart_date( request.GET.get('date_start', None), fallback=None) search_date_end = smart_date( request.GET.get('date_end', None), fallback=None) selected = request.GET.get('selected', None) filter_data = [] current_search = {'page': page} search = ResponseDocType.docs.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F('term', happy=search_happy) current_search['happy'] = int(search_happy) def unknown_to_empty(text): """Convert "Unknown" to "" to support old links""" return u'' if text.lower() == u'unknown' else text if search_platform is not None: f &= F('term', platform=unknown_to_empty(search_platform)) current_search['platform'] = search_platform if search_locale is not None: f &= F('term', locale=unknown_to_empty(search_locale)) current_search['locale'] = search_locale visible_products = [ prod.encode('utf-8') for prod in Product.objects.public().values_list('db_name', flat=True) ] # This covers the "unknown" product which is also visible. visible_products.append('') if search_product in visible_products: f &= F('term', product=unknown_to_empty(search_product)) current_search['product'] = search_product if search_version is not None: # Note: We only filter on version if we're filtering on # product. f &= F('term', version=unknown_to_empty(search_version)) current_search['version'] = search_version else: f &= F('terms', product=visible_products) if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = date.today() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) # If the start and end dates are inverted, switch them into proper # chronoligcal order search_date_start, search_date_end = sorted( [search_date_start, search_date_end]) # Restrict the frontpage dashboard to only show the last 6 months # of data six_months_ago = date.today() - timedelta(days=180) search_date_start = max(six_months_ago, search_date_start) search_date_end = max(search_date_start, search_date_end) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') f &= F('range', created={'lte': search_date_end}) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F('range', created={'gte': search_date_start}) if search_query: current_search['q'] = search_query search = search.query('simple_query_string', query=search_query, fields=['description']) search = search.filter(f).sort('-created') # If the user asked for a feed, give him/her a feed! if output_format == 'atom': return generate_atom_feed(request, list(search.execute())) elif output_format == 'json': return generate_json_feed(request, list(search.execute())) # Search results and pagination if page < 1: page = 1 page_count = 20 start = page_count * (page - 1) end = start + page_count search_count = search.count() opinion_page = search[start:end].execute() # Add navigation aggregations counts = { 'happy': {}, 'platform': {}, 'locale': {}, 'product': {}, 'version': {} } for name in counts.keys(): search.aggs.bucket(name, 'terms', field=name, size=1000) happy_sad_filter = request.GET.get('happy', None) if happy_sad_filter: if happy_sad_filter == '1': counts['happy'] = {True: 0} elif happy_sad_filter == '0': counts['happy'] = {False: 0} else: counts['happy'] = {True: 0, False: 0} if search_platform: counts['platform'] = {search_platform: 0} if search_locale: counts['locale'] = {search_locale: 0} if search_product: counts['product'] = {search_product: 0} if search_version: counts['version'] = {search_version: 0} results = search.execute() # Extract the value and doc_count for the various facets we do # faceted navigation on. for name in counts.keys(): buckets = getattr(results.aggregations, name)['buckets'] for bucket in buckets: key = bucket['key'] # Convert from 'T'/'F' to True/False if key in ['T', 'F']: key = (key == 'T') counts[name][key] = bucket['doc_count'] def empty_to_unknown(text): return _('Unknown') if text == u'' else text filter_data.extend([ counts_to_options( counts['happy'].items(), name='happy', display=_('Sentiment'), display_map={True: _('Happy'), False: _('Sad')}, value_map={True: 1, False: 0}, checked=search_happy), counts_to_options( counts['product'].items(), name='product', display=_('Product'), display_map=empty_to_unknown, checked=search_product) ]) # Only show the version if we're showing a specific # product. if search_product: filter_data.append( counts_to_options( counts['version'].items(), name='version', display=_('Version'), display_map=empty_to_unknown, checked=search_version) ) else: filter_data.append({ 'display': _('Version'), 'note': _('Select product to see version breakdown') }) filter_data.extend( [ counts_to_options( counts['platform'].items(), name='platform', display=_('Platform'), display_map=empty_to_unknown, checked=search_platform), counts_to_options( counts['locale'].items(), name='locale', display=_('Locale'), checked=search_locale, display_map=locale_name), ] ) return render(request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, 'atom_url': generate_dashboard_url(request), })
def get(self, request): """Returns JSON feed of first 10000 results This feels like a duplication of the front-page dashboard search logic, but it's separate which allows us to handle multiple values. """ search = models.ResponseDocType.docs.search() f = F() if 'id' in request.GET: id_list = request.GET['id'].split(',') id_list = [smart_int(id_, fallback=None) for id_ in id_list] id_list = [id_ for id_ in id_list if id_] f &= F('terms', id=id_list) else: if 'happy' in request.GET: happy = {'0': False, '1': True}.get(request.GET['happy'], None) if happy is not None: f &= F('term', happy=happy) if 'platforms' in request.GET: platforms = request.GET['platforms'].split(',') if platforms: f &= F('terms', platform=platforms) if 'locales' in request.GET: locales = request.GET['locales'].split(',') if locales: f &= F('terms', locale=locales) if 'products' in request.GET: products = request.GET['products'].split(',') if products: f &= F('terms', product=products) if 'versions' in request.GET: versions = request.GET['versions'].split(',') if versions: f &= F('terms', version=versions) date_start = smart_date(request.GET.get('date_start', None)) date_end = smart_date(request.GET.get('date_end', None)) delta = smart_timedelta(request.GET.get('date_delta', None)) if delta is not None: if date_end is not None: date_start = date_end - delta elif date_start is not None: date_end = date_start + delta else: date_end = date.today() date_start = date_end - delta # We restrict public API access to the last 6 months. six_months_ago = date.today() - timedelta(days=180) if date_start: date_start = max(six_months_ago, date_start) f &= F('range', created={'gte': date_start}) if date_end: date_end = max(six_months_ago, date_end) f &= F('range', created={'lte': date_end}) search_query = request.GET.get('q', None) if search_query is not None: search = search.query('simple_query_string', query=search_query, fields=['description']) # FIXME: Probably want to make this specifyable search = search.sort('-created') search = search.filter(f) maximum = smart_int(request.GET.get('max', None)) maximum = maximum or 1000 maximum = min(max(1, maximum), 10000) responses = list(search[:maximum].execute()) responses = models.ResponseDocType.docs.to_public(responses) return rest_framework.response.Response({ 'count': len(responses), 'results': responses })
def test_sanity(self): eq_(10, smart_int('10')) eq_(10, smart_int('10.5'))
def analytics_search(request): template = 'analytics/analyzer/search.html' output_format = request.GET.get('format', None) page = smart_int(request.GET.get('page', 1), 1) # Note: If we add additional querystring fields, we need to add # them to generate_dashboard_url. search_happy = request.GET.get('happy', None) search_has_email = request.GET.get('has_email', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_country = request.GET.get('country', None) search_product = request.GET.get('product', None) search_domain = request.GET.get('domain', None) search_api = smart_int(request.GET.get('api', None), fallback=None) search_version = request.GET.get('version', None) search_query = request.GET.get('q', None) search_date_start = smart_date(request.GET.get('date_start', None), fallback=None) search_date_end = smart_date(request.GET.get('date_end', None), fallback=None) search_bigram = request.GET.get('bigram', None) search_source = request.GET.get('source', None) search_campaign = request.GET.get('campaign', None) search_organic = request.GET.get('organic', None) selected = request.GET.get('selected', None) filter_data = [] current_search = {'page': page} search = ResponseMappingType.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F(happy=search_happy) current_search['happy'] = int(search_happy) # If search has_email is '0' or '1', set it to False or True, # respectively. search_has_email = {'0': False, '1': True}.get(search_has_email, None) if search_has_email in [False, True]: f &= F(has_email=search_has_email) current_search['has_email'] = int(search_has_email) def unknown_to_empty(text): """Convert "Unknown" to "" to support old links""" return u'' if text.lower() == u'unknown' else text if search_platform is not None: f &= F(platform=unknown_to_empty(search_platform)) current_search['platform'] = search_platform if search_locale is not None: f &= F(locale=unknown_to_empty(search_locale)) current_search['locale'] = search_locale if search_product is not None: f &= F(product=unknown_to_empty(search_product)) current_search['product'] = search_product # Only show the version if there's a product. if search_version is not None: # Note: We only filter on version if we're filtering on # product. f &= F(version=unknown_to_empty(search_version)) current_search['version'] = search_version # Only show the country if the product is Firefox OS. if search_country is not None and search_product == 'Firefox OS': f &= F(country=unknown_to_empty(search_country)) current_search['country'] = search_country if search_domain is not None: f &= F(url_domain=unknown_to_empty(search_domain)) current_search['domain'] = search_domain if search_api is not None: f &= F(api=search_api) current_search['api'] = search_api if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = datetime.now() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') # Add one day, so that the search range includes the entire day. end = search_date_end + timedelta(days=1) # Note 'less than', not 'less than or equal', because of the added # day above. f &= F(created__lt=end) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F(created__gte=search_date_start) if search_query: current_search['q'] = search_query search = search.query(description__sqs=search_query) if search_bigram is not None: f &= F(description_bigrams=search_bigram) filter_data.append({ 'display': 'Bigram', 'name': 'bigram', 'options': [{ 'count': 'all', 'name': search_bigram, 'display': search_bigram, 'value': search_bigram, 'checked': True }] }) if search_source is not None: f &= F(source=search_source) current_search['source'] = search_source if search_campaign is not None: f &= F(campaign=search_campaign) current_search['campaign'] = search_campaign search_organic = {'0': False, '1': True}.get(search_organic, None) if search_organic in [False, True]: f &= F(organic=search_organic) current_search['organic'] = int(search_organic) search = search.filter(f).order_by('-created') # If they're asking for a CSV export, then send them to the export # screen. if output_format == 'csv': return _analytics_search_export(request, search) # Search results and pagination if page < 1: page = 1 page_count = 50 start = page_count * (page - 1) end = start + page_count search_count = search.count() search_results = search.values_list('id')[start:end] opinion_page_ids = [mem[0][0] for mem in search_results] # We convert what we get back from ES to what's in the db so we # can get all the information. opinion_page = Response.objects.filter(id__in=opinion_page_ids) # Navigation facet data # This loop does two things. First it maps 'T' -> True and 'F' -> # False. This is probably something EU should be doing for # us. Second, it restructures the data into a more convenient # form. counts = { 'happy': {}, 'has_email': {}, 'platform': {}, 'locale': {}, 'country': {}, 'product': {}, 'version': {}, 'url_domain': {}, 'api': {}, 'source': {}, 'campaign': {}, 'organic': {}, } facets = search.facet(*(counts.keys()), size=1000, filtered=bool(search._process_filters(f.filters))) for param, terms in facets.facet_counts().items(): for term in terms: name = term['term'] if name == 'T': name = True elif name == 'F': name = False counts[param][name] = term['count'] def empty_to_unknown(text): return 'Unknown' if text == u'' else text filter_data.extend([ counts_to_options(counts['happy'].items(), name='happy', display='Sentiment', display_map={ True: 'Happy', False: 'Sad' }, value_map={ True: 1, False: 0 }, checked=search_happy), counts_to_options(counts['has_email'].items(), name='has_email', display='Has email', display_map={ True: 'Yes', False: 'No' }, value_map={ True: 1, False: 0 }, checked=search_has_email), counts_to_options(counts['product'].items(), name='product', display='Product', display_map=empty_to_unknown, checked=search_product) ]) # Only show the version if we're showing a specific # product. if search_product: filter_data.append( counts_to_options(counts['version'].items(), name='version', display='Version', display_map=empty_to_unknown, checked=search_version)) # Only show the country if the product is Firefox OS. if search_product == 'Firefox OS': filter_data.append( counts_to_options(counts['country'].items(), name='country', display='Country', checked=search_country, display_map=country_name), ) filter_data.extend([ counts_to_options(counts['platform'].items(), name='platform', display='Platform', display_map=empty_to_unknown, checked=search_platform), counts_to_options(counts['locale'].items(), name='locale', display='Locale', checked=search_locale, display_map=locale_name), counts_to_options(counts['url_domain'].items(), name='domain', display='Domain', checked=search_domain, display_map=empty_to_unknown), counts_to_options(counts['api'].items(), name='api', display='API version', checked=search_api, display_map=empty_to_unknown), counts_to_options(counts['organic'].items(), name='organic', display='Organic', display_map={ True: 'Yes', False: 'No' }, value_map={ True: 1, False: 0 }, checked=search_organic), counts_to_options(counts['source'].items(), name='source', display='Source', checked=search_source, display_map=empty_to_unknown), counts_to_options(counts['campaign'].items(), name='campaign', display='Campaign', checked=search_campaign, display_map=empty_to_unknown), ]) return render( request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, })
def analytics_search(request): template = 'analytics/analyzer/search.html' output_format = request.GET.get('format', None) page = smart_int(request.GET.get('page', 1), 1) # Note: If we add additional querystring fields, we need to add # them to generate_dashboard_url. search_happy = request.GET.get('happy', None) search_has_email = request.GET.get('has_email', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_country = request.GET.get('country', None) search_product = request.GET.get('product', None) search_domain = request.GET.get('domain', None) search_api = smart_int(request.GET.get('api', None), fallback=None) search_version = request.GET.get('version', None) search_query = request.GET.get('q', None) search_date_start = smart_date( request.GET.get('date_start', None), fallback=None) search_date_end = smart_date( request.GET.get('date_end', None), fallback=None) search_bigram = request.GET.get('bigram', None) search_source = request.GET.get('source', None) search_campaign = request.GET.get('campaign', None) search_organic = request.GET.get('organic', None) selected = request.GET.get('selected', None) filter_data = [] current_search = {'page': page} search = ResponseMappingType.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F(happy=search_happy) current_search['happy'] = int(search_happy) # If search has_email is '0' or '1', set it to False or True, # respectively. search_has_email = {'0': False, '1': True}.get(search_has_email, None) if search_has_email in [False, True]: f &= F(has_email=search_has_email) current_search['has_email'] = int(search_has_email) def unknown_to_empty(text): """Convert "Unknown" to "" to support old links""" return u'' if text.lower() == u'unknown' else text if search_platform is not None: f &= F(platform=unknown_to_empty(search_platform)) current_search['platform'] = search_platform if search_locale is not None: f &= F(locale=unknown_to_empty(search_locale)) current_search['locale'] = search_locale if search_product is not None: f &= F(product=unknown_to_empty(search_product)) current_search['product'] = search_product # Only show the version if there's a product. if search_version is not None: # Note: We only filter on version if we're filtering on # product. f &= F(version=unknown_to_empty(search_version)) current_search['version'] = search_version # Only show the country if the product is Firefox OS. if search_country is not None and search_product == 'Firefox OS': f &= F(country=unknown_to_empty(search_country)) current_search['country'] = search_country if search_domain is not None: f &= F(url_domain=unknown_to_empty(search_domain)) current_search['domain'] = search_domain if search_api is not None: f &= F(api=search_api) current_search['api'] = search_api if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = datetime.now() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') # Add one day, so that the search range includes the entire day. end = search_date_end + timedelta(days=1) # Note 'less than', not 'less than or equal', because of the added # day above. f &= F(created__lt=end) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F(created__gte=search_date_start) if search_query: current_search['q'] = search_query search = search.query(description__sqs=search_query) if search_bigram is not None: f &= F(description_bigrams=search_bigram) filter_data.append({ 'display': 'Bigram', 'name': 'bigram', 'options': [{ 'count': 'all', 'name': search_bigram, 'display': search_bigram, 'value': search_bigram, 'checked': True }] }) if search_source is not None: f &= F(source=search_source) current_search['source'] = search_source if search_campaign is not None: f &= F(campaign=search_campaign) current_search['campaign'] = search_campaign search_organic = {'0': False, '1': True}.get(search_organic, None) if search_organic in [False, True]: f &= F(organic=search_organic) current_search['organic'] = int(search_organic) search = search.filter(f).order_by('-created') # If they're asking for a CSV export, then send them to the export # screen. if output_format == 'csv': return _analytics_search_export(request, search) # Search results and pagination if page < 1: page = 1 page_count = 50 start = page_count * (page - 1) end = start + page_count search_count = search.count() search_results = search.values_list('id')[start:end] opinion_page_ids = [mem[0][0] for mem in search_results] # We convert what we get back from ES to what's in the db so we # can get all the information. opinion_page = Response.objects.filter(id__in=opinion_page_ids) # Navigation facet data # This loop does two things. First it maps 'T' -> True and 'F' -> # False. This is probably something EU should be doing for # us. Second, it restructures the data into a more convenient # form. counts = { 'happy': {}, 'has_email': {}, 'platform': {}, 'locale': {}, 'country': {}, 'product': {}, 'version': {}, 'url_domain': {}, 'api': {}, 'source': {}, 'campaign': {}, 'organic': {}, } facets = search.facet(*(counts.keys()), size=1000, filtered=bool(search._process_filters(f.filters))) for param, terms in facets.facet_counts().items(): for term in terms: name = term['term'] if name == 'T': name = True elif name == 'F': name = False counts[param][name] = term['count'] def empty_to_unknown(text): return 'Unknown' if text == u'' else text filter_data.extend([ counts_to_options( counts['happy'].items(), name='happy', display='Sentiment', display_map={True: 'Happy', False: 'Sad'}, value_map={True: 1, False: 0}, checked=search_happy), counts_to_options( counts['has_email'].items(), name='has_email', display='Has email', display_map={True: 'Yes', False: 'No'}, value_map={True: 1, False: 0}, checked=search_has_email), counts_to_options( counts['product'].items(), name='product', display='Product', display_map=empty_to_unknown, checked=search_product) ]) # Only show the version if we're showing a specific # product. if search_product: filter_data.append( counts_to_options( counts['version'].items(), name='version', display='Version', display_map=empty_to_unknown, checked=search_version) ) # Only show the country if the product is Firefox OS. if search_product == 'Firefox OS': filter_data.append( counts_to_options( counts['country'].items(), name='country', display='Country', checked=search_country, display_map=country_name), ) filter_data.extend( [ counts_to_options( counts['platform'].items(), name='platform', display='Platform', display_map=empty_to_unknown, checked=search_platform), counts_to_options( counts['locale'].items(), name='locale', display='Locale', checked=search_locale, display_map=locale_name), counts_to_options( counts['url_domain'].items(), name='domain', display='Domain', checked=search_domain, display_map=empty_to_unknown), counts_to_options( counts['api'].items(), name='api', display='API version', checked=search_api, display_map=empty_to_unknown), counts_to_options( counts['organic'].items(), name='organic', display='Organic', display_map={True: 'Yes', False: 'No'}, value_map={True: 1, False: 0}, checked=search_organic), counts_to_options( counts['source'].items(), name='source', display='Source', checked=search_source, display_map=empty_to_unknown), counts_to_options( counts['campaign'].items(), name='campaign', display='Campaign', checked=search_campaign, display_map=empty_to_unknown), ] ) return render(request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, })
def analytics_search(request): template = 'analytics/analyzer/search.html' output_format = request.GET.get('format', None) page = smart_int(request.GET.get('page', 1), 1) # Note: If we add additional querystring fields, we need to add # them to generate_dashboard_url. search_happy = request.GET.get('happy', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_product = request.GET.get('product', None) search_version = request.GET.get('version', None) search_query = request.GET.get('q', None) search_date_start = smart_date( request.GET.get('date_start', None), fallback=None) search_date_end = smart_date( request.GET.get('date_end', None), fallback=None) selected = request.GET.get('selected', None) search_has_email = request.GET.get('has_email', None) search_country = request.GET.get('country', None) search_domain = request.GET.get('domain', None) search_api = smart_int(request.GET.get('api', None), fallback=None) search_source = request.GET.get('source', None) search_campaign = request.GET.get('campaign', None) search_organic = request.GET.get('organic', None) filter_data = [] current_search = {'page': page} search = ResponseDocType.docs.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F('term', happy=search_happy) current_search['happy'] = int(search_happy) # If search has_email is '0' or '1', set it to False or True, # respectively. search_has_email = {'0': False, '1': True}.get(search_has_email, None) if search_has_email in [False, True]: f &= F('term', has_email=search_has_email) current_search['has_email'] = int(search_has_email) def unknown_to_empty(text): """Convert "Unknown" to "" to support old links""" return u'' if text.lower() == u'unknown' else text if search_platform is not None: f &= F('term', platform=unknown_to_empty(search_platform)) current_search['platform'] = search_platform if search_locale is not None: f &= F('term', locale=unknown_to_empty(search_locale)) current_search['locale'] = search_locale if search_product is not None: f &= F('term', product=unknown_to_empty(search_product)) current_search['product'] = search_product # Only show the version if there's a product. if search_version is not None: # Note: We only filter on version if we're filtering on # product. f &= F('term', version=unknown_to_empty(search_version)) current_search['version'] = search_version # Only show the country if the product is Firefox OS. if search_country is not None and search_product == 'Firefox OS': f &= F('term', country=unknown_to_empty(search_country)) current_search['country'] = search_country if search_domain is not None: f &= F('term', url_domain=unknown_to_empty(search_domain)) current_search['domain'] = search_domain if search_api is not None: f &= F('term', api=search_api) current_search['api'] = search_api if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = date.today() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) # If the start and end dates are inverted, switch them into proper # chronological order search_date_start, search_date_end = sorted( [search_date_start, search_date_end]) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') f &= F('range', created={'lte': search_date_end}) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F('range', created={'gte': search_date_start}) if search_query: current_search['q'] = search_query search = search.query('simple_query_string', query=search_query, fields=['description']) if search_source is not None: f &= F('term', source=search_source) current_search['source'] = search_source if search_campaign is not None: f &= F('term', campaign=search_campaign) current_search['campaign'] = search_campaign search_organic = {'0': False, '1': True}.get(search_organic, None) if search_organic in [False, True]: f &= F('term', organic=search_organic) current_search['organic'] = int(search_organic) search = search.filter(f).sort('-created') # If they're asking for a CSV export, then send them to the export # screen. if output_format == 'csv': return _analytics_search_export(request, search) original_search = search._clone() # Search results and pagination if page < 1: page = 1 page_count = 50 start = page_count * (page - 1) end = start + page_count search_count = search.count() search_results = search.fields('id')[start:end].execute() opinion_page_ids = [mem['id'][0] for mem in search_results] # We convert what we get back from ES to what's in the db so we # can get all the information. opinion_page = Response.objects.filter(id__in=opinion_page_ids) # Add navigation aggregations counts = { 'happy': {}, 'has_email': {}, 'platform': {}, 'locale': {}, 'country': {}, 'product': {}, 'version': {}, 'url_domain': {}, 'api': {}, 'source': {}, 'campaign': {}, 'organic': {}, } for name in counts.keys(): search.aggs.bucket(name, 'terms', field=name, size=1000) results = search.execute() # Extract the value and doc_count for the various facets we do # faceted navigation on. for name in counts.keys(): buckets = getattr(results.aggregations, name)['buckets'] for bucket in buckets: key = bucket['key'] # Convert from 'T'/'F' to True/False if key in ['T', 'F']: key = (key == 'T') counts[name][key] = bucket['doc_count'] def empty_to_unknown(text): return 'Unknown' if text == u'' else text filter_data.extend([ counts_to_options( counts['happy'].items(), name='happy', display='Sentiment', display_map={True: 'Happy', False: 'Sad'}, value_map={True: 1, False: 0}, checked=search_happy), counts_to_options( counts['has_email'].items(), name='has_email', display='Has email', display_map={True: 'Yes', False: 'No'}, value_map={True: 1, False: 0}, checked=search_has_email), counts_to_options( counts['product'].items(), name='product', display='Product', display_map=empty_to_unknown, checked=search_product) ]) # Only show the version if we're showing a specific # product. if search_product: filter_data.append( counts_to_options( counts['version'].items(), name='version', display='Version', display_map=empty_to_unknown, checked=search_version) ) # Only show the country if the product is Firefox OS. if search_product == 'Firefox OS': filter_data.append( counts_to_options( counts['country'].items(), name='country', display='Country', checked=search_country, display_map=country_name), ) filter_data.extend( [ counts_to_options( counts['platform'].items(), name='platform', display='Platform', display_map=empty_to_unknown, checked=search_platform), counts_to_options( counts['locale'].items(), name='locale', display='Locale', checked=search_locale, display_map=locale_name), counts_to_options( counts['url_domain'].items(), name='domain', display='Domain', checked=search_domain, display_map=empty_to_unknown), counts_to_options( counts['api'].items(), name='api', display='API version', checked=search_api, display_map=empty_to_unknown), counts_to_options( counts['organic'].items(), name='organic', display='Organic', display_map={True: 'Yes', False: 'No'}, value_map={True: 1, False: 0}, checked=search_organic), counts_to_options( counts['source'].items(), name='source', display='Source', checked=search_source, display_map=empty_to_unknown), counts_to_options( counts['campaign'].items(), name='campaign', display='Campaign', checked=search_campaign, display_map=empty_to_unknown), ] ) # Histogram data happy_data = [] sad_data = [] (original_search.aggs .bucket('histogram', 'date_histogram', field='created', interval='day') .bucket('per_sentiment', 'terms', field='happy')) results = original_search.execute() buckets = results.aggregations['histogram']['buckets'] happy_data = {} sad_data = {} for bucket in buckets: # value -> count val_counts = dict( (item['key'], item['doc_count']) for item in bucket['per_sentiment']['buckets'] ) # key is ms since epoch here which is what the frontend wants, so # we can just leave it. happy_data[bucket['key']] = val_counts.get('T', 0) sad_data[bucket['key']] = val_counts.get('F', 0) zero_fill(search_date_start, search_date_end, [happy_data, sad_data]) histogram = [ {'label': 'Happy', 'name': 'happy', 'data': sorted(happy_data.items())}, {'label': 'Sad', 'name': 'sad', 'data': sorted(sad_data.items())}, ] return render(request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'histogram': histogram, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, })