Esempio n. 1
0
def analytics_flagged(request):
    """View showing responses with flags

    NOTE: This is not permanent and might go away depending on how the
    spicedham prototype works.

    """
    template = 'analytics/analyzer/flags.html'

    # FIXME: Importing this here so all the changes are localized to
    # this function.  If we decide to go forward with this, we should
    # unlocalize it.

    from django.contrib import messages
    from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
    from django.http import HttpResponseRedirect

    from fjord.flags.models import Flag

    if request.method == 'POST':
        flag_action = request.POST.get('flag')
        if flag_action:
            # We do some shenanigans here to make sure we're fetching
            # and operating on a flag_set that's not
            # cached. django-cache-machine doesn't invalidate m2m
            # queries correctly. :(
            resp = get_object_or_404(
                Response, pk=smart_int(request.POST['id']))
            flag = get_object_or_404(Flag, name=flag_action)

            resp_flags = dict([(f.name, f)
                               for f in resp.flag_set.no_cache().all()])
            if flag.name in resp_flags:
                del resp_flags[flag.name]
                messages.success(request, 'removed %s flag from %d' % (
                    flag_action, resp.id))
            else:
                resp_flags[flag.name] = flag
                messages.success(request, 'added %s flag from %d' % (
                    flag_action, resp.id))

            resp.flag_set.clear()
            resp.flag_set.add(*resp_flags.values())
            return HttpResponseRedirect(request.get_full_path())

    resp_filter = smart_str(request.GET.get('filter'))

    # Only look at en-US locale responses since Monday September 8th,
    # 2014 we pushed the integration code out.
    response_list = (Response.uncached
                     .filter(locale=u'en-US')
                     .filter(created__gte='2014-09-08'))

    counts = {
        'total': response_list.count(),
        'abuse': response_list.filter(flag__name='abuse').count(),
        'abuse-wrong': response_list.filter(flag__name='abuse-wrong').count(),
        'false-positive': (response_list
                           .filter(flag__name='abuse')
                           .filter(flag__name='abuse-wrong').count()),
    }
    counts['false-negative'] = (
        counts['abuse-wrong'] - counts['false-positive']
    )

    if resp_filter:
        response_list = response_list.filter(flag__name=resp_filter)

    paginator = Paginator(response_list, 50)

    page = request.GET.get('page')
    try:
        responses = paginator.page(page)
    except PageNotAnInteger:
        responses = paginator.page(1)
    except EmptyPage:
        responses = paginator.page(paginator.num_pages)

    return render(request, template, {
        'counts': counts,
        'responses': responses
    })
Esempio n. 2
0
 def test_sanity(self):
     eq_(10, smart_int('10'))
     eq_(10, smart_int('10.5'))
Esempio n. 3
0
def dashboard(request):
    template = 'analytics/dashboard.html'

    output_format = request.GET.get('format', None)
    page = smart_int(request.GET.get('page', 1), 1)

    # Note: If we add additional querystring fields, we need to add
    # them to generate_dashboard_url.
    search_happy = request.GET.get('happy', None)
    search_platform = request.GET.get('platform', None)
    search_locale = request.GET.get('locale', None)
    search_product = request.GET.get('product', None)
    search_version = request.GET.get('version', None)
    search_query = request.GET.get('q', None)
    search_date_start = smart_date(
        request.GET.get('date_start', None), fallback=None)
    search_date_end = smart_date(
        request.GET.get('date_end', None), fallback=None)
    search_bigram = request.GET.get('bigram', None)
    selected = request.GET.get('selected', None)

    filter_data = []
    current_search = {'page': page}

    search = ResponseMappingType.search()
    f = F()
    # If search happy is '0' or '1', set it to False or True, respectively.
    search_happy = {'0': False, '1': True}.get(search_happy, None)
    if search_happy in [False, True]:
        f &= F(happy=search_happy)
        current_search['happy'] = int(search_happy)

    def unknown_to_empty(text):
        """Convert "Unknown" to "" to support old links"""
        return u'' if text.lower() == u'unknown' else text

    if search_platform is not None:
        f &= F(platform=unknown_to_empty(search_platform))
        current_search['platform'] = search_platform
    if search_locale is not None:
        f &= F(locale=unknown_to_empty(search_locale))
        current_search['locale'] = search_locale
    if search_product is not None:
        f &= F(product=unknown_to_empty(search_product))
        current_search['product'] = search_product

        if search_version is not None:
            # Note: We only filter on version if we're filtering on
            # product.
            f &= F(version=unknown_to_empty(search_version))
            current_search['version'] = search_version

    if search_date_start is None and search_date_end is None:
        selected = '7d'

    if search_date_end is None:
        search_date_end = datetime.now()
    if search_date_start is None:
        search_date_start = search_date_end - timedelta(days=7)

    current_search['date_end'] = search_date_end.strftime('%Y-%m-%d')
    # Add one day, so that the search range includes the entire day.
    end = search_date_end + timedelta(days=1)
    # Note 'less than', not 'less than or equal', because of the added
    # day above.
    f &= F(created__lt=end)

    current_search['date_start'] = search_date_start.strftime('%Y-%m-%d')
    f &= F(created__gte=search_date_start)

    if search_query:
        current_search['q'] = search_query
        es_query = generate_query_parsed('description', search_query)
        search = search.query_raw(es_query)

    if search_bigram is not None:
        f &= F(description_bigrams=search_bigram)
        filter_data.append({
            'display': _('Bigram'),
            'name': 'bigram',
            'options': [{
                'count': 'all',
                'name': search_bigram,
                'display': search_bigram,
                'value': search_bigram,
                'checked': True
            }]
        })

    search = search.filter(f).order_by('-created')

    # If the user asked for a feed, give him/her a feed!
    if output_format == 'atom':
        return generate_atom_feed(request, search)

    elif output_format == 'json':
        return generate_json_feed(request, search)

    # Search results and pagination
    if page < 1:
        page = 1
    page_count = 20
    start = page_count * (page - 1)
    end = start + page_count

    search_count = search.count()
    opinion_page = search[start:end]

    # Navigation facet data
    facets = search.facet(
        'happy', 'platform', 'locale', 'product', 'version',
        filtered=bool(search._process_filters(f.filters)))

    # This loop does two things. First it maps 'T' -> True and 'F' ->
    # False.  This is probably something EU should be doing for
    # us. Second, it restructures the data into a more convenient
    # form.
    counts = {
        'happy': {},
        'platform': {},
        'locale': {},
        'product': {},
        'version': {}
    }
    for param, terms in facets.facet_counts().items():
        for term in terms:
            name = term['term']
            if name == 'T':
                name = True
            elif name == 'F':
                name = False

            counts[param][name] = term['count']

    def empty_to_unknown(text):
        return _('Unknown') if text == u'' else text

    filter_data.extend([
        counts_to_options(
            counts['happy'].items(),
            name='happy',
            display=_('Sentiment'),
            display_map={True: _('Happy'), False: _('Sad')},
            value_map={True: 1, False: 0},
            checked=search_happy),
        counts_to_options(
            counts['product'].items(),
            name='product',
            display=_('Product'),
            display_map=empty_to_unknown,
            checked=search_product)
    ])
    # Only show the version if we're showing a specific
    # product.
    if search_product:
        filter_data.append(
            counts_to_options(
                counts['version'].items(),
                name='version',
                display=_('Version'),
                display_map=empty_to_unknown,
                checked=search_version)
        )

    filter_data.extend(
        [
            counts_to_options(
                counts['platform'].items(),
                name='platform',
                display=_('Platform'),
                display_map=empty_to_unknown,
                checked=search_platform),
            counts_to_options(
                counts['locale'].items(),
                name='locale',
                display=_('Locale'),
                checked=search_locale,
                display_map=locale_name),
        ]
    )

    # Histogram data
    happy_data = []
    sad_data = []

    happy_f = f & F(happy=True)
    sad_f = f & F(happy=False)
    histograms = search.facet_raw(
        happy={
            'date_histogram': {'interval': 'day', 'field': 'created'},
            'facet_filter': search._process_filters(happy_f.filters)
        },
        sad={
            'date_histogram': {'interval': 'day', 'field': 'created'},
            'facet_filter': search._process_filters(sad_f.filters)
        },
    ).facet_counts()

    # p['time'] is number of milliseconds since the epoch. Which is
    # convenient, because that is what the front end wants.
    happy_data = dict((p['time'], p['count']) for p in histograms['happy'])
    sad_data = dict((p['time'], p['count']) for p in histograms['sad'])

    zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
    histogram = [
        {'label': _('Happy'), 'name': 'happy',
         'data': sorted(happy_data.items())},
        {'label': _('Sad'), 'name': 'sad',
         'data': sorted(sad_data.items())},
    ]

    return render(request, template, {
        'opinions': opinion_page,
        'opinion_count': search_count,
        'filter_data': filter_data,
        'histogram': histogram,
        'page': page,
        'prev_page': page - 1 if start > 0 else None,
        'next_page': page + 1 if end < search_count else None,
        'current_search': current_search,
        'selected': selected,
        'atom_url': generate_dashboard_url(request),
    })
Esempio n. 4
0
def analytics_search(request):
    template = 'analytics/analyzer/search.html'

    output_format = request.GET.get('format', None)
    page = smart_int(request.GET.get('page', 1), 1)

    # Note: If we add additional querystring fields, we need to add
    # them to generate_dashboard_url.
    search_happy = request.GET.get('happy', None)
    search_has_email = request.GET.get('has_email', None)
    search_platform = request.GET.get('platform', None)
    search_locale = request.GET.get('locale', None)
    search_country = request.GET.get('country', None)
    search_product = request.GET.get('product', None)
    search_domain = request.GET.get('domain', None)
    search_api = smart_int(request.GET.get('api', None), fallback=None)
    search_version = request.GET.get('version', None)
    search_query = request.GET.get('q', None)
    search_date_start = smart_date(
        request.GET.get('date_start', None), fallback=None)
    search_date_end = smart_date(
        request.GET.get('date_end', None), fallback=None)
    search_bigram = request.GET.get('bigram', None)
    search_source = request.GET.get('source', None)
    search_campaign = request.GET.get('campaign', None)
    search_organic = request.GET.get('organic', None)
    selected = request.GET.get('selected', None)

    filter_data = []
    current_search = {'page': page}

    search = ResponseMappingType.search()
    f = F()
    # If search happy is '0' or '1', set it to False or True, respectively.
    search_happy = {'0': False, '1': True}.get(search_happy, None)
    if search_happy in [False, True]:
        f &= F(happy=search_happy)
        current_search['happy'] = int(search_happy)

    # If search has_email is '0' or '1', set it to False or True,
    # respectively.
    search_has_email = {'0': False, '1': True}.get(search_has_email, None)
    if search_has_email in [False, True]:
        f &= F(has_email=search_has_email)
        current_search['has_email'] = int(search_has_email)

    def unknown_to_empty(text):
        """Convert "Unknown" to "" to support old links"""
        return u'' if text.lower() == u'unknown' else text

    if search_platform is not None:
        f &= F(platform=unknown_to_empty(search_platform))
        current_search['platform'] = search_platform
    if search_locale is not None:
        f &= F(locale=unknown_to_empty(search_locale))
        current_search['locale'] = search_locale
    if search_product is not None:
        f &= F(product=unknown_to_empty(search_product))
        current_search['product'] = search_product

        # Only show the version if there's a product.
        if search_version is not None:
            # Note: We only filter on version if we're filtering on
            # product.
            f &= F(version=unknown_to_empty(search_version))
            current_search['version'] = search_version

        # Only show the country if the product is Firefox OS.
        if search_country is not None and search_product == 'Firefox OS':
            f &= F(country=unknown_to_empty(search_country))
            current_search['country'] = search_country
    if search_domain is not None:
        f &= F(url_domain=unknown_to_empty(search_domain))
        current_search['domain'] = search_domain
    if search_api is not None:
        f &= F(api=search_api)
        current_search['api'] = search_api

    if search_date_start is None and search_date_end is None:
        selected = '7d'

    if search_date_end is None:
        search_date_end = datetime.now()
    if search_date_start is None:
        search_date_start = search_date_end - timedelta(days=7)

    current_search['date_end'] = search_date_end.strftime('%Y-%m-%d')
    # Add one day, so that the search range includes the entire day.
    end = search_date_end + timedelta(days=1)
    # Note 'less than', not 'less than or equal', because of the added
    # day above.
    f &= F(created__lt=end)

    current_search['date_start'] = search_date_start.strftime('%Y-%m-%d')
    f &= F(created__gte=search_date_start)

    if search_query:
        current_search['q'] = search_query
        search = search.query(description__sqs=search_query)

    if search_bigram is not None:
        f &= F(description_bigrams=search_bigram)
        filter_data.append({
            'display': 'Bigram',
            'name': 'bigram',
            'options': [{
                'count': 'all',
                'name': search_bigram,
                'display': search_bigram,
                'value': search_bigram,
                'checked': True
            }]
        })

    if search_source is not None:
        f &= F(source=search_source)
        current_search['source'] = search_source
    if search_campaign is not None:
        f &= F(campaign=search_campaign)
        current_search['campaign'] = search_campaign
    search_organic = {'0': False, '1': True}.get(search_organic, None)
    if search_organic in [False, True]:
        f &= F(organic=search_organic)
        current_search['organic'] = int(search_organic)

    search = search.filter(f).order_by('-created')

    # If they're asking for a CSV export, then send them to the export
    # screen.
    if output_format == 'csv':
        return _analytics_search_export(request, search)

    # Search results and pagination
    if page < 1:
        page = 1
    page_count = 50
    start = page_count * (page - 1)
    end = start + page_count

    search_count = search.count()
    search_results = search.values_list('id')[start:end]
    opinion_page_ids = [mem[0][0] for mem in search_results]

    # We convert what we get back from ES to what's in the db so we
    # can get all the information.
    opinion_page = Response.objects.filter(id__in=opinion_page_ids)

    # Navigation facet data

    # This loop does two things. First it maps 'T' -> True and 'F' ->
    # False.  This is probably something EU should be doing for
    # us. Second, it restructures the data into a more convenient
    # form.
    counts = {
        'happy': {},
        'has_email': {},
        'platform': {},
        'locale': {},
        'country': {},
        'product': {},
        'version': {},
        'url_domain': {},
        'api': {},
        'source': {},
        'campaign': {},
        'organic': {},
    }
    facets = search.facet(*(counts.keys()),
                          filtered=bool(search._process_filters(f.filters)),
                          size=25)

    for param, terms in facets.facet_counts().items():
        for term in terms:
            name = term['term']
            if name == 'T':
                name = True
            elif name == 'F':
                name = False

            counts[param][name] = term['count']

    def empty_to_unknown(text):
        return 'Unknown' if text == u'' else text

    filter_data.extend([
        counts_to_options(
            counts['happy'].items(),
            name='happy',
            display='Sentiment',
            display_map={True: 'Happy', False: 'Sad'},
            value_map={True: 1, False: 0},
            checked=search_happy),
        counts_to_options(
            counts['has_email'].items(),
            name='has_email',
            display='Has email',
            display_map={True: 'Yes', False: 'No'},
            value_map={True: 1, False: 0},
            checked=search_has_email),
        counts_to_options(
            counts['product'].items(),
            name='product',
            display='Product',
            display_map=empty_to_unknown,
            checked=search_product)
    ])
    # Only show the version if we're showing a specific
    # product.
    if search_product:
        filter_data.append(
            counts_to_options(
                counts['version'].items(),
                name='version',
                display='Version',
                display_map=empty_to_unknown,
                checked=search_version)
        )
        # Only show the country if the product is Firefox OS.
        if search_product == 'Firefox OS':
            filter_data.append(
                counts_to_options(
                    counts['country'].items(),
                    name='country',
                    display='Country',
                    checked=search_country,
                    display_map=country_name),
            )

    filter_data.extend(
        [
            counts_to_options(
                counts['platform'].items(),
                name='platform',
                display='Platform',
                display_map=empty_to_unknown,
                checked=search_platform),
            counts_to_options(
                counts['locale'].items(),
                name='locale',
                display='Locale',
                checked=search_locale,
                display_map=locale_name),
            counts_to_options(
                counts['url_domain'].items(),
                name='domain',
                display='Domain',
                checked=search_domain,
                display_map=empty_to_unknown),
            counts_to_options(
                counts['api'].items(),
                name='api',
                display='API version',
                checked=search_api,
                display_map=empty_to_unknown),
            counts_to_options(
                counts['organic'].items(),
                name='organic',
                display='Organic',
                display_map={True: 'Yes', False: 'No'},
                value_map={True: 1, False: 0},
                checked=search_organic),
            counts_to_options(
                counts['source'].items(),
                name='source',
                display='Source',
                checked=search_source,
                display_map=empty_to_unknown),
            counts_to_options(
                counts['campaign'].items(),
                name='campaign',
                display='Campaign',
                checked=search_campaign,
                display_map=empty_to_unknown),
        ]
    )

    return render(request, template, {
        'opinions': opinion_page,
        'opinion_count': search_count,
        'filter_data': filter_data,
        'page': page,
        'prev_page': page - 1 if start > 0 else None,
        'next_page': page + 1 if end < search_count else None,
        'current_search': current_search,
        'selected': selected,
    })
Esempio n. 5
0
 def test_overflow(self):
     eq_(0, smart_int('1e309'))
Esempio n. 6
0
def dashboard(request):
    template = 'analytics/dashboard.html'

    output_format = request.GET.get('format', None)
    page = smart_int(request.GET.get('page', 1), 1)

    # Note: If we add additional querystring fields, we need to add
    # them to generate_dashboard_url.
    search_happy = request.GET.get('happy', None)
    search_platform = request.GET.get('platform', None)
    search_locale = request.GET.get('locale', None)
    search_product = request.GET.get('product', None)
    search_version = request.GET.get('version', None)
    search_query = request.GET.get('q', None)
    search_date_start = smart_date(request.GET.get('date_start', None),
                                   fallback=None)
    search_date_end = smart_date(request.GET.get('date_end', None),
                                 fallback=None)
    search_bigram = request.GET.get('bigram', None)
    selected = request.GET.get('selected', None)

    filter_data = []
    current_search = {'page': page}

    search = ResponseMappingType.search()
    f = F()
    # If search happy is '0' or '1', set it to False or True, respectively.
    search_happy = {'0': False, '1': True}.get(search_happy, None)
    if search_happy in [False, True]:
        f &= F(happy=search_happy)
        current_search['happy'] = int(search_happy)

    def unknown_to_empty(text):
        """Convert "Unknown" to "" to support old links"""
        return u'' if text.lower() == u'unknown' else text

    if search_platform is not None:
        f &= F(platform=unknown_to_empty(search_platform))
        current_search['platform'] = search_platform
    if search_locale is not None:
        f &= F(locale=unknown_to_empty(search_locale))
        current_search['locale'] = search_locale
    if search_product is not None:
        f &= F(product=unknown_to_empty(search_product))
        current_search['product'] = search_product

        if search_version is not None:
            # Note: We only filter on version if we're filtering on
            # product.
            f &= F(version=unknown_to_empty(search_version))
            current_search['version'] = search_version

    if search_date_start is None and search_date_end is None:
        selected = '7d'

    if search_date_end is None:
        search_date_end = date.today()
    if search_date_start is None:
        search_date_start = search_date_end - timedelta(days=7)

    current_search['date_end'] = search_date_end.strftime('%Y-%m-%d')
    f &= F(created__lte=search_date_end)

    current_search['date_start'] = search_date_start.strftime('%Y-%m-%d')
    f &= F(created__gte=search_date_start)

    if search_query:
        current_search['q'] = search_query
        search = search.query(description__sqs=search_query)

    if search_bigram is not None:
        f &= F(description_bigrams=search_bigram)
        filter_data.append({
            'display':
            _('Bigram'),
            'name':
            'bigram',
            'options': [{
                'count': 'all',
                'name': search_bigram,
                'display': search_bigram,
                'value': search_bigram,
                'checked': True
            }]
        })

    search = search.filter(f).order_by('-created')

    # If the user asked for a feed, give him/her a feed!
    if output_format == 'atom':
        return generate_atom_feed(request, search)

    elif output_format == 'json':
        return generate_json_feed(request, search)

    # Search results and pagination
    if page < 1:
        page = 1
    page_count = 20
    start = page_count * (page - 1)
    end = start + page_count

    search_count = search.count()
    opinion_page = search[start:end]

    # Navigation facet data
    facets = search.facet('happy',
                          'platform',
                          'locale',
                          'product',
                          'version',
                          filtered=bool(search._process_filters(f.filters)))

    # This loop does two things. First it maps 'T' -> True and 'F' ->
    # False.  This is probably something EU should be doing for
    # us. Second, it restructures the data into a more convenient
    # form.
    counts = {
        'happy': {},
        'platform': {},
        'locale': {},
        'product': {},
        'version': {}
    }
    for param, terms in facets.facet_counts().items():
        for term in terms:
            name = term['term']
            if name.upper() == 'T':
                name = True
            elif name.upper() == 'F':
                name = False

            counts[param][name] = term['count']

    def empty_to_unknown(text):
        return _('Unknown') if text == u'' else text

    filter_data.extend([
        counts_to_options(counts['happy'].items(),
                          name='happy',
                          display=_('Sentiment'),
                          display_map={
                              True: _('Happy'),
                              False: _('Sad')
                          },
                          value_map={
                              True: 1,
                              False: 0
                          },
                          checked=search_happy),
        counts_to_options(counts['product'].items(),
                          name='product',
                          display=_('Product'),
                          display_map=empty_to_unknown,
                          checked=search_product)
    ])
    # Only show the version if we're showing a specific
    # product.
    if search_product:
        filter_data.append(
            counts_to_options(counts['version'].items(),
                              name='version',
                              display=_('Version'),
                              display_map=empty_to_unknown,
                              checked=search_version))
    else:
        filter_data.append({
            'display': _('Version'),
            'note': _('Select product to see version facet')
        })

    filter_data.extend([
        counts_to_options(counts['platform'].items(),
                          name='platform',
                          display=_('Platform'),
                          display_map=empty_to_unknown,
                          checked=search_platform),
        counts_to_options(counts['locale'].items(),
                          name='locale',
                          display=_('Locale'),
                          checked=search_locale,
                          display_map=locale_name),
    ])

    # Histogram data
    happy_data = []
    sad_data = []

    happy_f = f & F(happy=True)
    sad_f = f & F(happy=False)
    histograms = search.facet_raw(
        happy={
            'date_histogram': {
                'interval': 'day',
                'field': 'created'
            },
            'facet_filter': search._process_filters(happy_f.filters)
        },
        sad={
            'date_histogram': {
                'interval': 'day',
                'field': 'created'
            },
            'facet_filter': search._process_filters(sad_f.filters)
        },
    ).facet_counts()

    # p['time'] is number of milliseconds since the epoch. Which is
    # convenient, because that is what the front end wants.
    happy_data = dict((p['time'], p['count']) for p in histograms['happy'])
    sad_data = dict((p['time'], p['count']) for p in histograms['sad'])

    zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
    histogram = [
        {
            'label': _('Happy'),
            'name': 'happy',
            'data': sorted(happy_data.items())
        },
        {
            'label': _('Sad'),
            'name': 'sad',
            'data': sorted(sad_data.items())
        },
    ]

    return render(
        request, template, {
            'opinions': opinion_page,
            'opinion_count': search_count,
            'filter_data': filter_data,
            'histogram': histogram,
            'page': page,
            'prev_page': page - 1 if start > 0 else None,
            'next_page': page + 1 if end < search_count else None,
            'current_search': current_search,
            'selected': selected,
            'atom_url': generate_dashboard_url(request),
        })
Esempio n. 7
0
 def test_empty_string(self):
     eq_(0, smart_int(''))
Esempio n. 8
0
 def test_empty_string(self):
     eq_(0, smart_int(''))
Esempio n. 9
0
 def test_int(self):
     eq_(10, smart_int(10))
Esempio n. 10
0
 def test_invalid_string(self):
     eq_(0, smart_int('invalid'))
Esempio n. 11
0
 def test_sanity(self):
     eq_(10, smart_int('10'))
     eq_(10, smart_int('10.5'))
Esempio n. 12
0
def dashboard(request, template):
    page = smart_int(request.GET.get("page", 1), 1)
    search_happy = request.GET.get("happy", None)
    search_platform = request.GET.get("platform", None)
    search_locale = request.GET.get("locale", None)
    search_query = request.GET.get("q", None)
    search_date_start = smart_datetime(request.GET.get("date_start", None), fallback=None)
    search_date_end = smart_datetime(request.GET.get("date_end", None), fallback=None)
    selected = request.GET.get("selected", None)

    current_search = {"page": page}

    search = SimpleIndex.search()
    f = F()
    # If search happy is '0' or '1', set it to False or True, respectively.
    search_happy = {"0": False, "1": True}.get(search_happy, None)
    if search_happy in [False, True]:
        f &= F(happy=search_happy)
        current_search["happy"] = int(search_happy)
    if search_platform:
        f &= F(platform=search_platform)
        current_search["platform"] = search_platform
    if search_locale:
        f &= F(locale=search_locale)
        current_search["locale"] = search_locale

    if search_date_start is None and search_date_end is None:
        selected = "7d"

    if search_date_end is None:
        search_date_end = datetime.now()
    if search_date_start is None:
        search_date_start = search_date_end - timedelta(days=7)

    current_search["date_end"] = search_date_end.strftime("%Y-%m-%d")
    # Add one day, so that the search range includes the entire day.
    end = search_date_end + timedelta(days=1)
    # Note 'less than', not 'less than or equal', because of the added
    # day above.
    f &= F(created__lt=end)

    current_search["date_start"] = search_date_start.strftime("%Y-%m-%d")
    f &= F(created__gte=search_date_start)

    if search_query:
        fields = ["text", "text_phrase", "fuzzy"]
        query = dict(("description__%s" % f, search_query) for f in fields)
        search = search.query(or_=query)
        current_search["q"] = search_query

    search = search.filter(f).order_by("-created")

    facets = search.facet("happy", "platform", "locale", filtered=bool(f.filters))

    # This loop does two things. First it maps 'T' -> True and 'F' ->
    # False.  This is probably something EU should be doing for
    # us. Second, it restructures the data into a more convenient
    # form.
    counts = {"happy": {}, "platform": {}, "locale": {}}
    for param, terms in facets.facet_counts().items():
        for term in terms:
            name = term["term"]
            if name == "T":
                name = True
            elif name == "F":
                name = False

            counts[param][name] = term["count"]

    filter_data = [
        counts_to_options(
            counts["happy"].items(),
            name="happy",
            display=_("Sentiment"),
            display_map={True: _("Happy"), False: _("Sad")},
            value_map={True: 1, False: 0},
            checked=search_happy,
        ),
        counts_to_options(counts["platform"].items(), name="platform", display=_("Platform"), checked=search_platform),
        counts_to_options(
            counts["locale"].items(), name="locale", display=_("Locale"), checked=search_locale, display_map=locale_name
        ),
    ]

    # Histogram data
    happy_data = []
    sad_data = []

    histograms = search.facet_raw(
        happy={"date_histogram": {"interval": "day", "field": "created"}, "facet_filter": (f & F(happy=True)).filters},
        sad={"date_histogram": {"interval": "day", "field": "created"}, "facet_filter": (f & F(happy=False)).filters},
    ).facet_counts()

    # p['time'] is number of milliseconds since the epoch. Which is
    # convenient, because that is what the front end wants.
    happy_data = dict((p["time"], p["count"]) for p in histograms["happy"])
    sad_data = dict((p["time"], p["count"]) for p in histograms["sad"])

    _zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
    histogram = [
        {"label": _("Happy"), "name": "happy", "data": sorted(happy_data.items())},
        {"label": _("Sad"), "name": "sad", "data": sorted(sad_data.items())},
    ]

    # Pagination
    if page < 1:
        page = 1
    page_count = 20
    start = page_count * (page - 1)
    end = start + page_count

    search_count = search.count()
    opinion_page = search[start:end]

    return render(
        request,
        template,
        {
            "opinions": opinion_page,
            "opinion_count": search_count,
            "filter_data": filter_data,
            "histogram": histogram,
            "page": page,
            "prev_page": page - 1 if start > 0 else None,
            "next_page": page + 1 if end < search_count else None,
            "current_search": current_search,
            "selected": selected,
        },
    )
Esempio n. 13
0
def dashboard(request, template):
    output_format = request.GET.get('format', None)
    page = smart_int(request.GET.get('page', 1), 1)

    # Note: If we add additional querystring fields, we need to add
    # them to generate_dashboard_url.
    search_happy = request.GET.get('happy', None)
    search_platform = request.GET.get('platform', None)
    search_locale = request.GET.get('locale', None)
    search_product = request.GET.get('product', None)
    search_version = request.GET.get('browser_version', None)
    search_query = request.GET.get('q', None)
    search_date_start = smart_datetime(request.GET.get('date_start', None),
                                       fallback=None)
    search_date_end = smart_datetime(request.GET.get('date_end', None),
                                     fallback=None)
    selected = request.GET.get('selected', None)

    current_search = {'page': page}

    search = ResponseMappingType.search()
    f = F()
    # If search happy is '0' or '1', set it to False or True, respectively.
    search_happy = {'0': False, '1': True}.get(search_happy, None)
    if search_happy in [False, True]:
        f &= F(happy=search_happy)
        current_search['happy'] = int(search_happy)
    if search_platform:
        f &= F(platform=search_platform)
        current_search['platform'] = search_platform
    if search_locale:
        f &= F(locale=search_locale)
        current_search['locale'] = search_locale
    if search_product:
        f &= F(product=search_product)
        current_search['product'] = search_product

        if search_version:
            # Note: We only filter on version if we're filtering on
            # product.
            f &= F(browser_version=search_version)
            current_search['browser_version'] = search_version

    if search_date_start is None and search_date_end is None:
        selected = '7d'

    if search_date_end is None:
        search_date_end = datetime.now()
    if search_date_start is None:
        search_date_start = search_date_end - timedelta(days=7)

    current_search['date_end'] = search_date_end.strftime('%Y-%m-%d')
    # Add one day, so that the search range includes the entire day.
    end = search_date_end + timedelta(days=1)
    # Note 'less than', not 'less than or equal', because of the added
    # day above.
    f &= F(created__lt=end)

    current_search['date_start'] = search_date_start.strftime('%Y-%m-%d')
    f &= F(created__gte=search_date_start)

    if search_query:
        fields = ['text', 'text_phrase', 'fuzzy']
        query = dict(('description__%s' % f, search_query) for f in fields)
        search = search.query(or_=query)
        current_search['q'] = search_query

    search = search.filter(f).order_by('-created')

    # If the user asked for a feed, give him/her a feed!
    if output_format == 'atom':
        return generate_atom_feed(request, search)

    elif output_format == 'json':
        return generate_json_feed(request, search)

    # Search results and pagination
    if page < 1:
        page = 1
    page_count = 20
    start = page_count * (page - 1)
    end = start + page_count

    search_count = search.count()
    opinion_page = search[start:end]

    # Navigation facet data
    facets = search.facet(
        'happy', 'platform', 'locale', 'product', 'browser_version',
        filtered=bool(f.filters))

    # This loop does two things. First it maps 'T' -> True and 'F' ->
    # False.  This is probably something EU should be doing for
    # us. Second, it restructures the data into a more convenient
    # form.
    counts = {
        'happy': {},
        'platform': {},
        'locale': {},
        'product': {},
        'browser_version': {}
    }
    for param, terms in facets.facet_counts().items():
        for term in terms:
            name = term['term']
            if name == 'T':
                name = True
            elif name == 'F':
                name = False

            counts[param][name] = term['count']

    filter_data = [
        counts_to_options(
            counts['happy'].items(),
            name='happy',
            display=_('Sentiment'),
            display_map={True: _('Happy'), False: _('Sad')},
            value_map={True: 1, False: 0},
            checked=search_happy),
        counts_to_options(
            counts['product'].items(),
            name='product',
            display=_('Product'),
            checked=search_product)
    ]
    # Only show the browser_version if we're showing a specific
    # product.
    if search_product:
        filter_data.append(
            counts_to_options(
                counts['browser_version'].items(),
                name='browser_version',
                display=_('Version'),
                checked=search_version)
        )

    filter_data.extend(
        [
            counts_to_options(
                counts['platform'].items(),
                name='platform',
                display=_('Platform'),
                checked=search_platform),
            counts_to_options(
                counts['locale'].items(),
                name='locale',
                display=_('Locale'),
                checked=search_locale,
                display_map=locale_name),
        ]
    )

    # Histogram data
    happy_data = []
    sad_data = []

    histograms = search.facet_raw(
        happy={
            'date_histogram': {'interval': 'day', 'field': 'created'},
            'facet_filter': (f & F(happy=True)).filters
        },
        sad={
            'date_histogram': {'interval': 'day', 'field': 'created'},
            'facet_filter': (f & F(happy=False)).filters
        },
    ).facet_counts()

    # p['time'] is number of milliseconds since the epoch. Which is
    # convenient, because that is what the front end wants.
    happy_data = dict((p['time'], p['count']) for p in histograms['happy'])
    sad_data = dict((p['time'], p['count']) for p in histograms['sad'])

    _zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
    histogram = [
        {'label': _('Happy'), 'name': 'happy',
         'data': sorted(happy_data.items())},
        {'label': _('Sad'), 'name': 'sad',
         'data': sorted(sad_data.items())},
    ]

    return render(request, template, {
        'opinions': opinion_page,
        'opinion_count': search_count,
        'filter_data': filter_data,
        'histogram': histogram,
        'page': page,
        'prev_page': page - 1 if start > 0 else None,
        'next_page': page + 1 if end < search_count else None,
        'current_search': current_search,
        'selected': selected,
        'atom_url': generate_dashboard_atom_url(request)
    })
Esempio n. 14
0
 def test_int(self):
     eq_(10, smart_int(10))
Esempio n. 15
0
 def test_wrong_type(self):
     eq_(0, smart_int(None))
     eq_(10, smart_int([], 10))
Esempio n. 16
0
 def test_invalid_string(self):
     eq_(0, smart_int('invalid'))
Esempio n. 17
0
 def test_overflow(self):
     eq_(0, smart_int('1e309'))
Esempio n. 18
0
 def test_wrong_type(self):
     eq_(0, smart_int(None))
     eq_(10, smart_int([], 10))
Esempio n. 19
0
def dashboard(request, template):
    page = smart_int(request.GET.get('page', 1), 1)
    search_happy = request.GET.get('happy', None)
    search_platform = request.GET.get('platform', None)
    search_locale = request.GET.get('locale', None)
    search_query = request.GET.get('q', None)
    search_date_start = smart_datetime(request.GET.get('date_start', None), fallback=None)
    search_date_end = smart_datetime(request.GET.get('date_end', None), fallback=None)
    selected = request.GET.get('selected', None)

    current_search = {'page': page}

    search = SimpleIndex.search()
    f = F()
    # If search happy is '0' or '1', set it to False or True, respectively.
    search_happy = {'0': False, '1': True}.get(search_happy, None)
    if search_happy in [False, True]:
        f &= F(happy=search_happy)
        current_search['happy'] = int(search_happy)
    if search_platform:
        f &= F(platform=search_platform)
        current_search['platform'] = search_platform
    if search_locale:
        f &= F(locale=search_locale)
        current_search['locale'] = search_locale

    if search_date_start is None and search_date_end is None:
        selected = '7d'

    if search_date_end is None:
        search_date_end = datetime.now()
    if search_date_start is None:
        search_date_start = search_date_end - timedelta(days=7)

    current_search['date_end'] = search_date_end.strftime('%Y-%m-%d')
    # Add one day, so that the search range includes the entire day.
    end = search_date_end + timedelta(days=1)
    # Note 'less than', not 'less than or equal', because of the added day above.
    f &= F(created__lt=end)

    current_search['date_start'] = search_date_start.strftime('%Y-%m-%d')
    f &= F(created__gte=search_date_start)

    if search_query:
        fields = ['text', 'text_phrase', 'fuzzy']
        query = dict(('description__%s' % f, search_query) for f in fields)
        search = search.query(or_=query)
        current_search['q'] = search_query

    search = search.filter(f).order_by('-created')

    facets = search.facet('happy', 'platform', 'locale',
        filtered=bool(f.filters))

    # This loop does two things. First it maps 'T' -> True and 'F' -> False.
    # This is probably something EU should be doing for us. Second, it
    # restructures the data into a more convenient form.
    counts = {'happy': {}, 'platform': {}, 'locale': {}}
    for param, terms in facets.facet_counts().items():
        for term in terms:
            name = term['term']
            if name == 'T':
                name = True
            elif name == 'F':
                name = False

            counts[param][name] = term['count']

    filter_data = [
        counts_to_options(counts['happy'].items(), name='happy',
            display=_('Sentiment'),
            display_map={True: _('Happy'), False: _('Sad')},
            value_map={True: 1, False: 0}, checked=search_happy),
        counts_to_options(counts['platform'].items(),
            name='platform', display=_('Platform'), checked=search_platform),
        counts_to_options(counts['locale'].items(),
            name='locale', display=_('Locale'), checked=search_locale,
            display_map=locale_name)
    ]

    # Histogram data
    happy_data = []
    sad_data = []

    histograms = search.facet_raw(
        happy={
            'date_histogram': {'interval': 'day', 'field': 'created'},
            'facet_filter': (f & F(happy=True)).filters
        },
        sad={
            'date_histogram': {'interval': 'day', 'field': 'created'},
            'facet_filter': (f & F(happy=False)).filters
        },
    ).facet_counts()

    # p['time'] is number of milliseconds since the epoch. Which is
    # convenient, because that is what the front end wants.
    happy_data = dict((p['time'], p['count']) for p in histograms['happy'])
    sad_data = dict((p['time'], p['count']) for p in histograms['sad'])

    _zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
    histogram = [
        {'label': _('Happy'), 'name': 'happy', 'data': sorted(happy_data.items())},
        {'label': _('Sad'), 'name': 'sad', 'data': sorted(sad_data.items())},
    ]

    # Pagination
    if page < 1:
        page = 1
    page_count = 20
    start = page_count * (page - 1)
    end = start + page_count

    search_count = search.count()
    opinion_page = search[start:end]

    return render(request, template, {
        'opinions': opinion_page,
        'opinion_count': search_count,
        'filter_data': filter_data,
        'histogram': histogram,
        'page': page,
        'prev_page': page - 1 if start > 0 else None,
        'next_page': page + 1 if end < search_count else None,
        'current_search': current_search,
        'selected': selected,
    })
Esempio n. 20
0
    def get(self, request):
        """Returns JSON feed of first 10000 results

        This feels like a duplication of the front-page dashboard search
        logic, but it's separate which allows us to handle multiple
        values.

        """
        search = models.ResponseMappingType.search()
        f = F()

        if "happy" in request.GET:
            happy = {"0": False, "1": True}.get(request.GET["happy"], None)
            if happy is not None:
                f &= F(happy=happy)

        if "platforms" in request.GET:
            platforms = request.GET["platforms"].split(",")
            if platforms:
                f &= F(platform__in=platforms)

        if "locales" in request.GET:
            locales = request.GET["locales"].split(",")
            if locales:
                f &= F(locale__in=locales)

        if "products" in request.GET:
            products = request.GET["products"].split(",")
            if products:
                f &= F(product__in=products)

                if "versions" in request.GET:
                    versions = request.GET["versions"].split(",")
                    if versions:
                        f &= F(version__in=versions)

        date_start = smart_date(request.GET.get("date_start", None))
        date_end = smart_date(request.GET.get("date_end", None))
        delta = smart_timedelta(request.GET.get("date_delta", None))

        if delta is not None:
            if date_end is not None:
                date_start = date_end - delta
            elif date_start is not None:
                date_end = date_start + delta
            else:
                date_end = date.today()
                date_start = date_end - delta

        # We restrict public API access to the last 6 months.
        six_months_ago = date.today() - timedelta(days=180)
        if date_start:
            date_start = max(six_months_ago, date_start)
            f &= F(created__gte=date_start)
        if date_end:
            date_end = max(six_months_ago, date_end)
            f &= F(created__lte=date_end)

        search = search.filter(f)

        search_query = request.GET.get("q", None)
        if search_query is not None:
            search = search.query(description__sqs=search_query)

        # FIXME: Probably want to make this specifyable
        search = search.order_by("-created")

        # Explicitly include only publicly visible fields
        search = search.values_dict(models.ResponseMappingType.public_fields())

        maximum = smart_int(request.GET.get("max", None))
        maximum = maximum or 1000
        maximum = min(max(1, maximum), 10000)

        responses = models.ResponseMappingType.reshape(search[:maximum])
        return rest_framework.response.Response({"count": len(responses), "results": list(responses)})
Esempio n. 21
0
def analytics_search(request):
    template = 'analytics/analyzer/search.html'

    output_format = request.GET.get('format', None)
    page = smart_int(request.GET.get('page', 1), 1)

    # Note: If we add additional querystring fields, we need to add
    # them to generate_dashboard_url.
    search_happy = request.GET.get('happy', None)
    search_has_email = request.GET.get('has_email', None)
    search_platform = request.GET.get('platform', None)
    search_locale = request.GET.get('locale', None)
    search_country = request.GET.get('country', None)
    search_product = request.GET.get('product', None)
    search_domain = request.GET.get('domain', None)
    search_api = smart_int(request.GET.get('api', None), fallback=None)
    search_version = request.GET.get('version', None)
    search_query = request.GET.get('q', None)
    search_date_start = smart_date(request.GET.get('date_start', None),
                                   fallback=None)
    search_date_end = smart_date(request.GET.get('date_end', None),
                                 fallback=None)
    search_bigram = request.GET.get('bigram', None)
    search_source = request.GET.get('source', None)
    search_campaign = request.GET.get('campaign', None)
    search_organic = request.GET.get('organic', None)
    selected = request.GET.get('selected', None)

    filter_data = []
    current_search = {'page': page}

    search = ResponseMappingType.search()
    f = F()
    # If search happy is '0' or '1', set it to False or True, respectively.
    search_happy = {'0': False, '1': True}.get(search_happy, None)
    if search_happy in [False, True]:
        f &= F(happy=search_happy)
        current_search['happy'] = int(search_happy)

    # If search has_email is '0' or '1', set it to False or True,
    # respectively.
    search_has_email = {'0': False, '1': True}.get(search_has_email, None)
    if search_has_email in [False, True]:
        f &= F(has_email=search_has_email)
        current_search['has_email'] = int(search_has_email)

    def unknown_to_empty(text):
        """Convert "Unknown" to "" to support old links"""
        return u'' if text.lower() == u'unknown' else text

    if search_platform is not None:
        f &= F(platform=unknown_to_empty(search_platform))
        current_search['platform'] = search_platform
    if search_locale is not None:
        f &= F(locale=unknown_to_empty(search_locale))
        current_search['locale'] = search_locale
    if search_product is not None:
        f &= F(product=unknown_to_empty(search_product))
        current_search['product'] = search_product

        # Only show the version if there's a product.
        if search_version is not None:
            # Note: We only filter on version if we're filtering on
            # product.
            f &= F(version=unknown_to_empty(search_version))
            current_search['version'] = search_version

        # Only show the country if the product is Firefox OS.
        if search_country is not None and search_product == 'Firefox OS':
            f &= F(country=unknown_to_empty(search_country))
            current_search['country'] = search_country
    if search_domain is not None:
        f &= F(url_domain=unknown_to_empty(search_domain))
        current_search['domain'] = search_domain
    if search_api is not None:
        f &= F(api=search_api)
        current_search['api'] = search_api

    if search_date_start is None and search_date_end is None:
        selected = '7d'

    if search_date_end is None:
        search_date_end = datetime.now()
    if search_date_start is None:
        search_date_start = search_date_end - timedelta(days=7)

    current_search['date_end'] = search_date_end.strftime('%Y-%m-%d')
    # Add one day, so that the search range includes the entire day.
    end = search_date_end + timedelta(days=1)
    # Note 'less than', not 'less than or equal', because of the added
    # day above.
    f &= F(created__lt=end)

    current_search['date_start'] = search_date_start.strftime('%Y-%m-%d')
    f &= F(created__gte=search_date_start)

    if search_query:
        current_search['q'] = search_query
        search = search.query(description__sqs=search_query)

    if search_bigram is not None:
        f &= F(description_bigrams=search_bigram)
        filter_data.append({
            'display':
            'Bigram',
            'name':
            'bigram',
            'options': [{
                'count': 'all',
                'name': search_bigram,
                'display': search_bigram,
                'value': search_bigram,
                'checked': True
            }]
        })

    if search_source is not None:
        f &= F(source=search_source)
        current_search['source'] = search_source
    if search_campaign is not None:
        f &= F(campaign=search_campaign)
        current_search['campaign'] = search_campaign
    search_organic = {'0': False, '1': True}.get(search_organic, None)
    if search_organic in [False, True]:
        f &= F(organic=search_organic)
        current_search['organic'] = int(search_organic)

    search = search.filter(f).order_by('-created')

    # If they're asking for a CSV export, then send them to the export
    # screen.
    if output_format == 'csv':
        return _analytics_search_export(request, search)

    # Search results and pagination
    if page < 1:
        page = 1
    page_count = 50
    start = page_count * (page - 1)
    end = start + page_count

    search_count = search.count()
    opinion_page_ids = [mem[0] for mem in search.values_list('id')[start:end]]

    # We convert what we get back from ES to what's in the db so we
    # can get all the information.
    opinion_page = Response.objects.filter(id__in=opinion_page_ids)

    # Navigation facet data

    # This loop does two things. First it maps 'T' -> True and 'F' ->
    # False.  This is probably something EU should be doing for
    # us. Second, it restructures the data into a more convenient
    # form.
    counts = {
        'happy': {},
        'has_email': {},
        'platform': {},
        'locale': {},
        'country': {},
        'product': {},
        'version': {},
        'url_domain': {},
        'api': {},
        'source': {},
        'campaign': {},
        'organic': {},
    }
    facets = search.facet(*(counts.keys()),
                          filtered=bool(search._process_filters(f.filters)),
                          size=25)

    for param, terms in facets.facet_counts().items():
        for term in terms:
            name = term['term']
            if name == 'T':
                name = True
            elif name == 'F':
                name = False

            counts[param][name] = term['count']

    def empty_to_unknown(text):
        return 'Unknown' if text == u'' else text

    filter_data.extend([
        counts_to_options(counts['happy'].items(),
                          name='happy',
                          display='Sentiment',
                          display_map={
                              True: 'Happy',
                              False: 'Sad'
                          },
                          value_map={
                              True: 1,
                              False: 0
                          },
                          checked=search_happy),
        counts_to_options(counts['has_email'].items(),
                          name='has_email',
                          display='Has email',
                          display_map={
                              True: 'Yes',
                              False: 'No'
                          },
                          value_map={
                              True: 1,
                              False: 0
                          },
                          checked=search_has_email),
        counts_to_options(counts['product'].items(),
                          name='product',
                          display='Product',
                          display_map=empty_to_unknown,
                          checked=search_product)
    ])
    # Only show the version if we're showing a specific
    # product.
    if search_product:
        filter_data.append(
            counts_to_options(counts['version'].items(),
                              name='version',
                              display='Version',
                              display_map=empty_to_unknown,
                              checked=search_version))
        # Only show the country if the product is Firefox OS.
        if search_product == 'Firefox OS':
            filter_data.append(
                counts_to_options(counts['country'].items(),
                                  name='country',
                                  display='Country',
                                  checked=search_country,
                                  display_map=country_name), )

    filter_data.extend([
        counts_to_options(counts['platform'].items(),
                          name='platform',
                          display='Platform',
                          display_map=empty_to_unknown,
                          checked=search_platform),
        counts_to_options(counts['locale'].items(),
                          name='locale',
                          display='Locale',
                          checked=search_locale,
                          display_map=locale_name),
        counts_to_options(counts['url_domain'].items(),
                          name='domain',
                          display='Domain',
                          checked=search_domain,
                          display_map=empty_to_unknown),
        counts_to_options(counts['api'].items(),
                          name='api',
                          display='API version',
                          checked=search_api,
                          display_map=empty_to_unknown),
        counts_to_options(counts['organic'].items(),
                          name='organic',
                          display='Organic',
                          display_map={
                              True: 'Yes',
                              False: 'No'
                          },
                          value_map={
                              True: 1,
                              False: 0
                          },
                          checked=search_organic),
        counts_to_options(counts['source'].items(),
                          name='source',
                          display='Source',
                          checked=search_source,
                          display_map=empty_to_unknown),
        counts_to_options(counts['campaign'].items(),
                          name='campaign',
                          display='Campaign',
                          checked=search_campaign,
                          display_map=empty_to_unknown),
    ])

    return render(
        request, template, {
            'opinions': opinion_page,
            'opinion_count': search_count,
            'filter_data': filter_data,
            'page': page,
            'prev_page': page - 1 if start > 0 else None,
            'next_page': page + 1 if end < search_count else None,
            'current_search': current_search,
            'selected': selected,
        })