def analytics_flagged(request):
"""View showing responses with flags
NOTE: This is not permanent and might go away depending on how the
spicedham prototype works.
"""
template = 'analytics/analyzer/flags.html'
# FIXME: Importing this here so all the changes are localized to
# this function. If we decide to go forward with this, we should
# unlocalize it.
from django.contrib import messages
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.http import HttpResponseRedirect
from fjord.flags.models import Flag
if request.method == 'POST':
flag_action = request.POST.get('flag')
if flag_action:
# We do some shenanigans here to make sure we're fetching
# and operating on a flag_set that's not
# cached. django-cache-machine doesn't invalidate m2m
# queries correctly. :(
resp = get_object_or_404(
Response, pk=smart_int(request.POST['id']))
flag = get_object_or_404(Flag, name=flag_action)
resp_flags = dict([(f.name, f)
for f in resp.flag_set.no_cache().all()])
if flag.name in resp_flags:
del resp_flags[flag.name]
messages.success(request, 'removed %s flag from %d' % (
flag_action, resp.id))
else:
resp_flags[flag.name] = flag
messages.success(request, 'added %s flag from %d' % (
flag_action, resp.id))
resp.flag_set.clear()
resp.flag_set.add(*resp_flags.values())
return HttpResponseRedirect(request.get_full_path())
resp_filter = smart_str(request.GET.get('filter'))
# Only look at en-US locale responses since Monday September 8th,
# 2014 we pushed the integration code out.
response_list = (Response.uncached
.filter(locale=u'en-US')
.filter(created__gte='2014-09-08'))
counts = {
'total': response_list.count(),
'abuse': response_list.filter(flag__name='abuse').count(),
'abuse-wrong': response_list.filter(flag__name='abuse-wrong').count(),
'false-positive': (response_list
.filter(flag__name='abuse')
.filter(flag__name='abuse-wrong').count()),
}
counts['false-negative'] = (
counts['abuse-wrong'] - counts['false-positive']
)
if resp_filter:
response_list = response_list.filter(flag__name=resp_filter)
paginator = Paginator(response_list, 50)
page = request.GET.get('page')
try:
responses = paginator.page(page)
except PageNotAnInteger:
responses = paginator.page(1)
except EmptyPage:
responses = paginator.page(paginator.num_pages)
return render(request, template, {
'counts': counts,
'responses': responses
})
def test_sanity(self):
eq_(10, smart_int('10'))
eq_(10, smart_int('10.5'))
def dashboard(request):
template = 'analytics/dashboard.html'
output_format = request.GET.get('format', None)
page = smart_int(request.GET.get('page', 1), 1)
# Note: If we add additional querystring fields, we need to add
# them to generate_dashboard_url.
search_happy = request.GET.get('happy', None)
search_platform = request.GET.get('platform', None)
search_locale = request.GET.get('locale', None)
search_product = request.GET.get('product', None)
search_version = request.GET.get('version', None)
search_query = request.GET.get('q', None)
search_date_start = smart_date(
request.GET.get('date_start', None), fallback=None)
search_date_end = smart_date(
request.GET.get('date_end', None), fallback=None)
search_bigram = request.GET.get('bigram', None)
selected = request.GET.get('selected', None)
filter_data = []
current_search = {'page': page}
search = ResponseMappingType.search()
f = F()
# If search happy is '0' or '1', set it to False or True, respectively.
search_happy = {'0': False, '1': True}.get(search_happy, None)
if search_happy in [False, True]:
f &= F(happy=search_happy)
current_search['happy'] = int(search_happy)
def unknown_to_empty(text):
"""Convert "Unknown" to "" to support old links"""
return u'' if text.lower() == u'unknown' else text
if search_platform is not None:
f &= F(platform=unknown_to_empty(search_platform))
current_search['platform'] = search_platform
if search_locale is not None:
f &= F(locale=unknown_to_empty(search_locale))
current_search['locale'] = search_locale
if search_product is not None:
f &= F(product=unknown_to_empty(search_product))
current_search['product'] = search_product
if search_version is not None:
# Note: We only filter on version if we're filtering on
# product.
f &= F(version=unknown_to_empty(search_version))
current_search['version'] = search_version
if search_date_start is None and search_date_end is None:
selected = '7d'
if search_date_end is None:
search_date_end = datetime.now()
if search_date_start is None:
search_date_start = search_date_end - timedelta(days=7)
current_search['date_end'] = search_date_end.strftime('%Y-%m-%d')
# Add one day, so that the search range includes the entire day.
end = search_date_end + timedelta(days=1)
# Note 'less than', not 'less than or equal', because of the added
# day above.
f &= F(created__lt=end)
current_search['date_start'] = search_date_start.strftime('%Y-%m-%d')
f &= F(created__gte=search_date_start)
if search_query:
current_search['q'] = search_query
es_query = generate_query_parsed('description', search_query)
search = search.query_raw(es_query)
if search_bigram is not None:
f &= F(description_bigrams=search_bigram)
filter_data.append({
'display': _('Bigram'),
'name': 'bigram',
'options': [{
'count': 'all',
'name': search_bigram,
'display': search_bigram,
'value': search_bigram,
'checked': True
}]
})
search = search.filter(f).order_by('-created')
# If the user asked for a feed, give him/her a feed!
if output_format == 'atom':
return generate_atom_feed(request, search)
elif output_format == 'json':
return generate_json_feed(request, search)
# Search results and pagination
if page < 1:
page = 1
page_count = 20
start = page_count * (page - 1)
end = start + page_count
search_count = search.count()
opinion_page = search[start:end]
# Navigation facet data
facets = search.facet(
'happy', 'platform', 'locale', 'product', 'version',
filtered=bool(search._process_filters(f.filters)))
# This loop does two things. First it maps 'T' -> True and 'F' ->
# False. This is probably something EU should be doing for
# us. Second, it restructures the data into a more convenient
# form.
counts = {
'happy': {},
'platform': {},
'locale': {},
'product': {},
'version': {}
}
for param, terms in facets.facet_counts().items():
for term in terms:
name = term['term']
if name == 'T':
name = True
elif name == 'F':
name = False
counts[param][name] = term['count']
def empty_to_unknown(text):
return _('Unknown') if text == u'' else text
filter_data.extend([
counts_to_options(
counts['happy'].items(),
name='happy',
display=_('Sentiment'),
display_map={True: _('Happy'), False: _('Sad')},
value_map={True: 1, False: 0},
checked=search_happy),
counts_to_options(
counts['product'].items(),
name='product',
display=_('Product'),
display_map=empty_to_unknown,
checked=search_product)
])
# Only show the version if we're showing a specific
# product.
if search_product:
filter_data.append(
counts_to_options(
counts['version'].items(),
name='version',
display=_('Version'),
display_map=empty_to_unknown,
checked=search_version)
)
filter_data.extend(
[
counts_to_options(
counts['platform'].items(),
name='platform',
display=_('Platform'),
display_map=empty_to_unknown,
checked=search_platform),
counts_to_options(
counts['locale'].items(),
name='locale',
display=_('Locale'),
checked=search_locale,
display_map=locale_name),
]
)
# Histogram data
happy_data = []
sad_data = []
happy_f = f & F(happy=True)
sad_f = f & F(happy=False)
histograms = search.facet_raw(
happy={
'date_histogram': {'interval': 'day', 'field': 'created'},
'facet_filter': search._process_filters(happy_f.filters)
},
sad={
'date_histogram': {'interval': 'day', 'field': 'created'},
'facet_filter': search._process_filters(sad_f.filters)
},
).facet_counts()
# p['time'] is number of milliseconds since the epoch. Which is
# convenient, because that is what the front end wants.
happy_data = dict((p['time'], p['count']) for p in histograms['happy'])
sad_data = dict((p['time'], p['count']) for p in histograms['sad'])
zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
histogram = [
{'label': _('Happy'), 'name': 'happy',
'data': sorted(happy_data.items())},
{'label': _('Sad'), 'name': 'sad',
'data': sorted(sad_data.items())},
]
return render(request, template, {
'opinions': opinion_page,
'opinion_count': search_count,
'filter_data': filter_data,
'histogram': histogram,
'page': page,
'prev_page': page - 1 if start > 0 else None,
'next_page': page + 1 if end < search_count else None,
'current_search': current_search,
'selected': selected,
'atom_url': generate_dashboard_url(request),
})
def analytics_search(request):
template = 'analytics/analyzer/search.html'
output_format = request.GET.get('format', None)
page = smart_int(request.GET.get('page', 1), 1)
# Note: If we add additional querystring fields, we need to add
# them to generate_dashboard_url.
search_happy = request.GET.get('happy', None)
search_has_email = request.GET.get('has_email', None)
search_platform = request.GET.get('platform', None)
search_locale = request.GET.get('locale', None)
search_country = request.GET.get('country', None)
search_product = request.GET.get('product', None)
search_domain = request.GET.get('domain', None)
search_api = smart_int(request.GET.get('api', None), fallback=None)
search_version = request.GET.get('version', None)
search_query = request.GET.get('q', None)
search_date_start = smart_date(
request.GET.get('date_start', None), fallback=None)
search_date_end = smart_date(
request.GET.get('date_end', None), fallback=None)
search_bigram = request.GET.get('bigram', None)
search_source = request.GET.get('source', None)
search_campaign = request.GET.get('campaign', None)
search_organic = request.GET.get('organic', None)
selected = request.GET.get('selected', None)
filter_data = []
current_search = {'page': page}
search = ResponseMappingType.search()
f = F()
# If search happy is '0' or '1', set it to False or True, respectively.
search_happy = {'0': False, '1': True}.get(search_happy, None)
if search_happy in [False, True]:
f &= F(happy=search_happy)
current_search['happy'] = int(search_happy)
# If search has_email is '0' or '1', set it to False or True,
# respectively.
search_has_email = {'0': False, '1': True}.get(search_has_email, None)
if search_has_email in [False, True]:
f &= F(has_email=search_has_email)
current_search['has_email'] = int(search_has_email)
def unknown_to_empty(text):
"""Convert "Unknown" to "" to support old links"""
return u'' if text.lower() == u'unknown' else text
if search_platform is not None:
f &= F(platform=unknown_to_empty(search_platform))
current_search['platform'] = search_platform
if search_locale is not None:
f &= F(locale=unknown_to_empty(search_locale))
current_search['locale'] = search_locale
if search_product is not None:
f &= F(product=unknown_to_empty(search_product))
current_search['product'] = search_product
# Only show the version if there's a product.
if search_version is not None:
# Note: We only filter on version if we're filtering on
# product.
f &= F(version=unknown_to_empty(search_version))
current_search['version'] = search_version
# Only show the country if the product is Firefox OS.
if search_country is not None and search_product == 'Firefox OS':
f &= F(country=unknown_to_empty(search_country))
current_search['country'] = search_country
if search_domain is not None:
f &= F(url_domain=unknown_to_empty(search_domain))
current_search['domain'] = search_domain
if search_api is not None:
f &= F(api=search_api)
current_search['api'] = search_api
if search_date_start is None and search_date_end is None:
selected = '7d'
if search_date_end is None:
search_date_end = datetime.now()
if search_date_start is None:
search_date_start = search_date_end - timedelta(days=7)
current_search['date_end'] = search_date_end.strftime('%Y-%m-%d')
# Add one day, so that the search range includes the entire day.
end = search_date_end + timedelta(days=1)
# Note 'less than', not 'less than or equal', because of the added
# day above.
f &= F(created__lt=end)
current_search['date_start'] = search_date_start.strftime('%Y-%m-%d')
f &= F(created__gte=search_date_start)
if search_query:
current_search['q'] = search_query
search = search.query(description__sqs=search_query)
if search_bigram is not None:
f &= F(description_bigrams=search_bigram)
filter_data.append({
'display': 'Bigram',
'name': 'bigram',
'options': [{
'count': 'all',
'name': search_bigram,
'display': search_bigram,
'value': search_bigram,
'checked': True
}]
})
if search_source is not None:
f &= F(source=search_source)
current_search['source'] = search_source
if search_campaign is not None:
f &= F(campaign=search_campaign)
current_search['campaign'] = search_campaign
search_organic = {'0': False, '1': True}.get(search_organic, None)
if search_organic in [False, True]:
f &= F(organic=search_organic)
current_search['organic'] = int(search_organic)
search = search.filter(f).order_by('-created')
# If they're asking for a CSV export, then send them to the export
# screen.
if output_format == 'csv':
return _analytics_search_export(request, search)
# Search results and pagination
if page < 1:
page = 1
page_count = 50
start = page_count * (page - 1)
end = start + page_count
search_count = search.count()
search_results = search.values_list('id')[start:end]
opinion_page_ids = [mem[0][0] for mem in search_results]
# We convert what we get back from ES to what's in the db so we
# can get all the information.
opinion_page = Response.objects.filter(id__in=opinion_page_ids)
# Navigation facet data
# This loop does two things. First it maps 'T' -> True and 'F' ->
# False. This is probably something EU should be doing for
# us. Second, it restructures the data into a more convenient
# form.
counts = {
'happy': {},
'has_email': {},
'platform': {},
'locale': {},
'country': {},
'product': {},
'version': {},
'url_domain': {},
'api': {},
'source': {},
'campaign': {},
'organic': {},
}
facets = search.facet(*(counts.keys()),
filtered=bool(search._process_filters(f.filters)),
size=25)
for param, terms in facets.facet_counts().items():
for term in terms:
name = term['term']
if name == 'T':
name = True
elif name == 'F':
name = False
counts[param][name] = term['count']
def empty_to_unknown(text):
return 'Unknown' if text == u'' else text
filter_data.extend([
counts_to_options(
counts['happy'].items(),
name='happy',
display='Sentiment',
display_map={True: 'Happy', False: 'Sad'},
value_map={True: 1, False: 0},
checked=search_happy),
counts_to_options(
counts['has_email'].items(),
name='has_email',
display='Has email',
display_map={True: 'Yes', False: 'No'},
value_map={True: 1, False: 0},
checked=search_has_email),
counts_to_options(
counts['product'].items(),
name='product',
display='Product',
display_map=empty_to_unknown,
checked=search_product)
])
# Only show the version if we're showing a specific
# product.
if search_product:
filter_data.append(
counts_to_options(
counts['version'].items(),
name='version',
display='Version',
display_map=empty_to_unknown,
checked=search_version)
)
# Only show the country if the product is Firefox OS.
if search_product == 'Firefox OS':
filter_data.append(
counts_to_options(
counts['country'].items(),
name='country',
display='Country',
checked=search_country,
display_map=country_name),
)
filter_data.extend(
[
counts_to_options(
counts['platform'].items(),
name='platform',
display='Platform',
display_map=empty_to_unknown,
checked=search_platform),
counts_to_options(
counts['locale'].items(),
name='locale',
display='Locale',
checked=search_locale,
display_map=locale_name),
counts_to_options(
counts['url_domain'].items(),
name='domain',
display='Domain',
checked=search_domain,
display_map=empty_to_unknown),
counts_to_options(
counts['api'].items(),
name='api',
display='API version',
checked=search_api,
display_map=empty_to_unknown),
counts_to_options(
counts['organic'].items(),
name='organic',
display='Organic',
display_map={True: 'Yes', False: 'No'},
value_map={True: 1, False: 0},
checked=search_organic),
counts_to_options(
counts['source'].items(),
name='source',
display='Source',
checked=search_source,
display_map=empty_to_unknown),
counts_to_options(
counts['campaign'].items(),
name='campaign',
display='Campaign',
checked=search_campaign,
display_map=empty_to_unknown),
]
)
return render(request, template, {
'opinions': opinion_page,
'opinion_count': search_count,
'filter_data': filter_data,
'page': page,
'prev_page': page - 1 if start > 0 else None,
'next_page': page + 1 if end < search_count else None,
'current_search': current_search,
'selected': selected,
})
def test_overflow(self):
eq_(0, smart_int('1e309'))
def dashboard(request):
template = 'analytics/dashboard.html'
output_format = request.GET.get('format', None)
page = smart_int(request.GET.get('page', 1), 1)
# Note: If we add additional querystring fields, we need to add
# them to generate_dashboard_url.
search_happy = request.GET.get('happy', None)
search_platform = request.GET.get('platform', None)
search_locale = request.GET.get('locale', None)
search_product = request.GET.get('product', None)
search_version = request.GET.get('version', None)
search_query = request.GET.get('q', None)
search_date_start = smart_date(request.GET.get('date_start', None),
fallback=None)
search_date_end = smart_date(request.GET.get('date_end', None),
fallback=None)
search_bigram = request.GET.get('bigram', None)
selected = request.GET.get('selected', None)
filter_data = []
current_search = {'page': page}
search = ResponseMappingType.search()
f = F()
# If search happy is '0' or '1', set it to False or True, respectively.
search_happy = {'0': False, '1': True}.get(search_happy, None)
if search_happy in [False, True]:
f &= F(happy=search_happy)
current_search['happy'] = int(search_happy)
def unknown_to_empty(text):
"""Convert "Unknown" to "" to support old links"""
return u'' if text.lower() == u'unknown' else text
if search_platform is not None:
f &= F(platform=unknown_to_empty(search_platform))
current_search['platform'] = search_platform
if search_locale is not None:
f &= F(locale=unknown_to_empty(search_locale))
current_search['locale'] = search_locale
if search_product is not None:
f &= F(product=unknown_to_empty(search_product))
current_search['product'] = search_product
if search_version is not None:
# Note: We only filter on version if we're filtering on
# product.
f &= F(version=unknown_to_empty(search_version))
current_search['version'] = search_version
if search_date_start is None and search_date_end is None:
selected = '7d'
if search_date_end is None:
search_date_end = date.today()
if search_date_start is None:
search_date_start = search_date_end - timedelta(days=7)
current_search['date_end'] = search_date_end.strftime('%Y-%m-%d')
f &= F(created__lte=search_date_end)
current_search['date_start'] = search_date_start.strftime('%Y-%m-%d')
f &= F(created__gte=search_date_start)
if search_query:
current_search['q'] = search_query
search = search.query(description__sqs=search_query)
if search_bigram is not None:
f &= F(description_bigrams=search_bigram)
filter_data.append({
'display':
_('Bigram'),
'name':
'bigram',
'options': [{
'count': 'all',
'name': search_bigram,
'display': search_bigram,
'value': search_bigram,
'checked': True
}]
})
search = search.filter(f).order_by('-created')
# If the user asked for a feed, give him/her a feed!
if output_format == 'atom':
return generate_atom_feed(request, search)
elif output_format == 'json':
return generate_json_feed(request, search)
# Search results and pagination
if page < 1:
page = 1
page_count = 20
start = page_count * (page - 1)
end = start + page_count
search_count = search.count()
opinion_page = search[start:end]
# Navigation facet data
facets = search.facet('happy',
'platform',
'locale',
'product',
'version',
filtered=bool(search._process_filters(f.filters)))
# This loop does two things. First it maps 'T' -> True and 'F' ->
# False. This is probably something EU should be doing for
# us. Second, it restructures the data into a more convenient
# form.
counts = {
'happy': {},
'platform': {},
'locale': {},
'product': {},
'version': {}
}
for param, terms in facets.facet_counts().items():
for term in terms:
name = term['term']
if name.upper() == 'T':
name = True
elif name.upper() == 'F':
name = False
counts[param][name] = term['count']
def empty_to_unknown(text):
return _('Unknown') if text == u'' else text
filter_data.extend([
counts_to_options(counts['happy'].items(),
name='happy',
display=_('Sentiment'),
display_map={
True: _('Happy'),
False: _('Sad')
},
value_map={
True: 1,
False: 0
},
checked=search_happy),
counts_to_options(counts['product'].items(),
name='product',
display=_('Product'),
display_map=empty_to_unknown,
checked=search_product)
])
# Only show the version if we're showing a specific
# product.
if search_product:
filter_data.append(
counts_to_options(counts['version'].items(),
name='version',
display=_('Version'),
display_map=empty_to_unknown,
checked=search_version))
else:
filter_data.append({
'display': _('Version'),
'note': _('Select product to see version facet')
})
filter_data.extend([
counts_to_options(counts['platform'].items(),
name='platform',
display=_('Platform'),
display_map=empty_to_unknown,
checked=search_platform),
counts_to_options(counts['locale'].items(),
name='locale',
display=_('Locale'),
checked=search_locale,
display_map=locale_name),
])
# Histogram data
happy_data = []
sad_data = []
happy_f = f & F(happy=True)
sad_f = f & F(happy=False)
histograms = search.facet_raw(
happy={
'date_histogram': {
'interval': 'day',
'field': 'created'
},
'facet_filter': search._process_filters(happy_f.filters)
},
sad={
'date_histogram': {
'interval': 'day',
'field': 'created'
},
'facet_filter': search._process_filters(sad_f.filters)
},
).facet_counts()
# p['time'] is number of milliseconds since the epoch. Which is
# convenient, because that is what the front end wants.
happy_data = dict((p['time'], p['count']) for p in histograms['happy'])
sad_data = dict((p['time'], p['count']) for p in histograms['sad'])
zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
histogram = [
{
'label': _('Happy'),
'name': 'happy',
'data': sorted(happy_data.items())
},
{
'label': _('Sad'),
'name': 'sad',
'data': sorted(sad_data.items())
},
]
return render(
request, template, {
'opinions': opinion_page,
'opinion_count': search_count,
'filter_data': filter_data,
'histogram': histogram,
'page': page,
'prev_page': page - 1 if start > 0 else None,
'next_page': page + 1 if end < search_count else None,
'current_search': current_search,
'selected': selected,
'atom_url': generate_dashboard_url(request),
})
def test_empty_string(self):
eq_(0, smart_int(''))
def test_empty_string(self):
eq_(0, smart_int(''))
def test_int(self):
eq_(10, smart_int(10))
def test_invalid_string(self):
eq_(0, smart_int('invalid'))
def test_sanity(self):
eq_(10, smart_int('10'))
eq_(10, smart_int('10.5'))
def dashboard(request, template):
page = smart_int(request.GET.get("page", 1), 1)
search_happy = request.GET.get("happy", None)
search_platform = request.GET.get("platform", None)
search_locale = request.GET.get("locale", None)
search_query = request.GET.get("q", None)
search_date_start = smart_datetime(request.GET.get("date_start", None), fallback=None)
search_date_end = smart_datetime(request.GET.get("date_end", None), fallback=None)
selected = request.GET.get("selected", None)
current_search = {"page": page}
search = SimpleIndex.search()
f = F()
# If search happy is '0' or '1', set it to False or True, respectively.
search_happy = {"0": False, "1": True}.get(search_happy, None)
if search_happy in [False, True]:
f &= F(happy=search_happy)
current_search["happy"] = int(search_happy)
if search_platform:
f &= F(platform=search_platform)
current_search["platform"] = search_platform
if search_locale:
f &= F(locale=search_locale)
current_search["locale"] = search_locale
if search_date_start is None and search_date_end is None:
selected = "7d"
if search_date_end is None:
search_date_end = datetime.now()
if search_date_start is None:
search_date_start = search_date_end - timedelta(days=7)
current_search["date_end"] = search_date_end.strftime("%Y-%m-%d")
# Add one day, so that the search range includes the entire day.
end = search_date_end + timedelta(days=1)
# Note 'less than', not 'less than or equal', because of the added
# day above.
f &= F(created__lt=end)
current_search["date_start"] = search_date_start.strftime("%Y-%m-%d")
f &= F(created__gte=search_date_start)
if search_query:
fields = ["text", "text_phrase", "fuzzy"]
query = dict(("description__%s" % f, search_query) for f in fields)
search = search.query(or_=query)
current_search["q"] = search_query
search = search.filter(f).order_by("-created")
facets = search.facet("happy", "platform", "locale", filtered=bool(f.filters))
# This loop does two things. First it maps 'T' -> True and 'F' ->
# False. This is probably something EU should be doing for
# us. Second, it restructures the data into a more convenient
# form.
counts = {"happy": {}, "platform": {}, "locale": {}}
for param, terms in facets.facet_counts().items():
for term in terms:
name = term["term"]
if name == "T":
name = True
elif name == "F":
name = False
counts[param][name] = term["count"]
filter_data = [
counts_to_options(
counts["happy"].items(),
name="happy",
display=_("Sentiment"),
display_map={True: _("Happy"), False: _("Sad")},
value_map={True: 1, False: 0},
checked=search_happy,
),
counts_to_options(counts["platform"].items(), name="platform", display=_("Platform"), checked=search_platform),
counts_to_options(
counts["locale"].items(), name="locale", display=_("Locale"), checked=search_locale, display_map=locale_name
),
]
# Histogram data
happy_data = []
sad_data = []
histograms = search.facet_raw(
happy={"date_histogram": {"interval": "day", "field": "created"}, "facet_filter": (f & F(happy=True)).filters},
sad={"date_histogram": {"interval": "day", "field": "created"}, "facet_filter": (f & F(happy=False)).filters},
).facet_counts()
# p['time'] is number of milliseconds since the epoch. Which is
# convenient, because that is what the front end wants.
happy_data = dict((p["time"], p["count"]) for p in histograms["happy"])
sad_data = dict((p["time"], p["count"]) for p in histograms["sad"])
_zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
histogram = [
{"label": _("Happy"), "name": "happy", "data": sorted(happy_data.items())},
{"label": _("Sad"), "name": "sad", "data": sorted(sad_data.items())},
]
# Pagination
if page < 1:
page = 1
page_count = 20
start = page_count * (page - 1)
end = start + page_count
search_count = search.count()
opinion_page = search[start:end]
return render(
request,
template,
{
"opinions": opinion_page,
"opinion_count": search_count,
"filter_data": filter_data,
"histogram": histogram,
"page": page,
"prev_page": page - 1 if start > 0 else None,
"next_page": page + 1 if end < search_count else None,
"current_search": current_search,
"selected": selected,
},
)
def dashboard(request, template):
output_format = request.GET.get('format', None)
page = smart_int(request.GET.get('page', 1), 1)
# Note: If we add additional querystring fields, we need to add
# them to generate_dashboard_url.
search_happy = request.GET.get('happy', None)
search_platform = request.GET.get('platform', None)
search_locale = request.GET.get('locale', None)
search_product = request.GET.get('product', None)
search_version = request.GET.get('browser_version', None)
search_query = request.GET.get('q', None)
search_date_start = smart_datetime(request.GET.get('date_start', None),
fallback=None)
search_date_end = smart_datetime(request.GET.get('date_end', None),
fallback=None)
selected = request.GET.get('selected', None)
current_search = {'page': page}
search = ResponseMappingType.search()
f = F()
# If search happy is '0' or '1', set it to False or True, respectively.
search_happy = {'0': False, '1': True}.get(search_happy, None)
if search_happy in [False, True]:
f &= F(happy=search_happy)
current_search['happy'] = int(search_happy)
if search_platform:
f &= F(platform=search_platform)
current_search['platform'] = search_platform
if search_locale:
f &= F(locale=search_locale)
current_search['locale'] = search_locale
if search_product:
f &= F(product=search_product)
current_search['product'] = search_product
if search_version:
# Note: We only filter on version if we're filtering on
# product.
f &= F(browser_version=search_version)
current_search['browser_version'] = search_version
if search_date_start is None and search_date_end is None:
selected = '7d'
if search_date_end is None:
search_date_end = datetime.now()
if search_date_start is None:
search_date_start = search_date_end - timedelta(days=7)
current_search['date_end'] = search_date_end.strftime('%Y-%m-%d')
# Add one day, so that the search range includes the entire day.
end = search_date_end + timedelta(days=1)
# Note 'less than', not 'less than or equal', because of the added
# day above.
f &= F(created__lt=end)
current_search['date_start'] = search_date_start.strftime('%Y-%m-%d')
f &= F(created__gte=search_date_start)
if search_query:
fields = ['text', 'text_phrase', 'fuzzy']
query = dict(('description__%s' % f, search_query) for f in fields)
search = search.query(or_=query)
current_search['q'] = search_query
search = search.filter(f).order_by('-created')
# If the user asked for a feed, give him/her a feed!
if output_format == 'atom':
return generate_atom_feed(request, search)
elif output_format == 'json':
return generate_json_feed(request, search)
# Search results and pagination
if page < 1:
page = 1
page_count = 20
start = page_count * (page - 1)
end = start + page_count
search_count = search.count()
opinion_page = search[start:end]
# Navigation facet data
facets = search.facet(
'happy', 'platform', 'locale', 'product', 'browser_version',
filtered=bool(f.filters))
# This loop does two things. First it maps 'T' -> True and 'F' ->
# False. This is probably something EU should be doing for
# us. Second, it restructures the data into a more convenient
# form.
counts = {
'happy': {},
'platform': {},
'locale': {},
'product': {},
'browser_version': {}
}
for param, terms in facets.facet_counts().items():
for term in terms:
name = term['term']
if name == 'T':
name = True
elif name == 'F':
name = False
counts[param][name] = term['count']
filter_data = [
counts_to_options(
counts['happy'].items(),
name='happy',
display=_('Sentiment'),
display_map={True: _('Happy'), False: _('Sad')},
value_map={True: 1, False: 0},
checked=search_happy),
counts_to_options(
counts['product'].items(),
name='product',
display=_('Product'),
checked=search_product)
]
# Only show the browser_version if we're showing a specific
# product.
if search_product:
filter_data.append(
counts_to_options(
counts['browser_version'].items(),
name='browser_version',
display=_('Version'),
checked=search_version)
)
filter_data.extend(
[
counts_to_options(
counts['platform'].items(),
name='platform',
display=_('Platform'),
checked=search_platform),
counts_to_options(
counts['locale'].items(),
name='locale',
display=_('Locale'),
checked=search_locale,
display_map=locale_name),
]
)
# Histogram data
happy_data = []
sad_data = []
histograms = search.facet_raw(
happy={
'date_histogram': {'interval': 'day', 'field': 'created'},
'facet_filter': (f & F(happy=True)).filters
},
sad={
'date_histogram': {'interval': 'day', 'field': 'created'},
'facet_filter': (f & F(happy=False)).filters
},
).facet_counts()
# p['time'] is number of milliseconds since the epoch. Which is
# convenient, because that is what the front end wants.
happy_data = dict((p['time'], p['count']) for p in histograms['happy'])
sad_data = dict((p['time'], p['count']) for p in histograms['sad'])
_zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
histogram = [
{'label': _('Happy'), 'name': 'happy',
'data': sorted(happy_data.items())},
{'label': _('Sad'), 'name': 'sad',
'data': sorted(sad_data.items())},
]
return render(request, template, {
'opinions': opinion_page,
'opinion_count': search_count,
'filter_data': filter_data,
'histogram': histogram,
'page': page,
'prev_page': page - 1 if start > 0 else None,
'next_page': page + 1 if end < search_count else None,
'current_search': current_search,
'selected': selected,
'atom_url': generate_dashboard_atom_url(request)
})
def test_int(self):
eq_(10, smart_int(10))
def test_wrong_type(self):
eq_(0, smart_int(None))
eq_(10, smart_int([], 10))
def test_invalid_string(self):
eq_(0, smart_int('invalid'))
def test_overflow(self):
eq_(0, smart_int('1e309'))
def test_wrong_type(self):
eq_(0, smart_int(None))
eq_(10, smart_int([], 10))
def dashboard(request, template):
page = smart_int(request.GET.get('page', 1), 1)
search_happy = request.GET.get('happy', None)
search_platform = request.GET.get('platform', None)
search_locale = request.GET.get('locale', None)
search_query = request.GET.get('q', None)
search_date_start = smart_datetime(request.GET.get('date_start', None), fallback=None)
search_date_end = smart_datetime(request.GET.get('date_end', None), fallback=None)
selected = request.GET.get('selected', None)
current_search = {'page': page}
search = SimpleIndex.search()
f = F()
# If search happy is '0' or '1', set it to False or True, respectively.
search_happy = {'0': False, '1': True}.get(search_happy, None)
if search_happy in [False, True]:
f &= F(happy=search_happy)
current_search['happy'] = int(search_happy)
if search_platform:
f &= F(platform=search_platform)
current_search['platform'] = search_platform
if search_locale:
f &= F(locale=search_locale)
current_search['locale'] = search_locale
if search_date_start is None and search_date_end is None:
selected = '7d'
if search_date_end is None:
search_date_end = datetime.now()
if search_date_start is None:
search_date_start = search_date_end - timedelta(days=7)
current_search['date_end'] = search_date_end.strftime('%Y-%m-%d')
# Add one day, so that the search range includes the entire day.
end = search_date_end + timedelta(days=1)
# Note 'less than', not 'less than or equal', because of the added day above.
f &= F(created__lt=end)
current_search['date_start'] = search_date_start.strftime('%Y-%m-%d')
f &= F(created__gte=search_date_start)
if search_query:
fields = ['text', 'text_phrase', 'fuzzy']
query = dict(('description__%s' % f, search_query) for f in fields)
search = search.query(or_=query)
current_search['q'] = search_query
search = search.filter(f).order_by('-created')
facets = search.facet('happy', 'platform', 'locale',
filtered=bool(f.filters))
# This loop does two things. First it maps 'T' -> True and 'F' -> False.
# This is probably something EU should be doing for us. Second, it
# restructures the data into a more convenient form.
counts = {'happy': {}, 'platform': {}, 'locale': {}}
for param, terms in facets.facet_counts().items():
for term in terms:
name = term['term']
if name == 'T':
name = True
elif name == 'F':
name = False
counts[param][name] = term['count']
filter_data = [
counts_to_options(counts['happy'].items(), name='happy',
display=_('Sentiment'),
display_map={True: _('Happy'), False: _('Sad')},
value_map={True: 1, False: 0}, checked=search_happy),
counts_to_options(counts['platform'].items(),
name='platform', display=_('Platform'), checked=search_platform),
counts_to_options(counts['locale'].items(),
name='locale', display=_('Locale'), checked=search_locale,
display_map=locale_name)
]
# Histogram data
happy_data = []
sad_data = []
histograms = search.facet_raw(
happy={
'date_histogram': {'interval': 'day', 'field': 'created'},
'facet_filter': (f & F(happy=True)).filters
},
sad={
'date_histogram': {'interval': 'day', 'field': 'created'},
'facet_filter': (f & F(happy=False)).filters
},
).facet_counts()
# p['time'] is number of milliseconds since the epoch. Which is
# convenient, because that is what the front end wants.
happy_data = dict((p['time'], p['count']) for p in histograms['happy'])
sad_data = dict((p['time'], p['count']) for p in histograms['sad'])
_zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
histogram = [
{'label': _('Happy'), 'name': 'happy', 'data': sorted(happy_data.items())},
{'label': _('Sad'), 'name': 'sad', 'data': sorted(sad_data.items())},
]
# Pagination
if page < 1:
page = 1
page_count = 20
start = page_count * (page - 1)
end = start + page_count
search_count = search.count()
opinion_page = search[start:end]
return render(request, template, {
'opinions': opinion_page,
'opinion_count': search_count,
'filter_data': filter_data,
'histogram': histogram,
'page': page,
'prev_page': page - 1 if start > 0 else None,
'next_page': page + 1 if end < search_count else None,
'current_search': current_search,
'selected': selected,
})
def get(self, request):
"""Returns JSON feed of first 10000 results
This feels like a duplication of the front-page dashboard search
logic, but it's separate which allows us to handle multiple
values.
"""
search = models.ResponseMappingType.search()
f = F()
if "happy" in request.GET:
happy = {"0": False, "1": True}.get(request.GET["happy"], None)
if happy is not None:
f &= F(happy=happy)
if "platforms" in request.GET:
platforms = request.GET["platforms"].split(",")
if platforms:
f &= F(platform__in=platforms)
if "locales" in request.GET:
locales = request.GET["locales"].split(",")
if locales:
f &= F(locale__in=locales)
if "products" in request.GET:
products = request.GET["products"].split(",")
if products:
f &= F(product__in=products)
if "versions" in request.GET:
versions = request.GET["versions"].split(",")
if versions:
f &= F(version__in=versions)
date_start = smart_date(request.GET.get("date_start", None))
date_end = smart_date(request.GET.get("date_end", None))
delta = smart_timedelta(request.GET.get("date_delta", None))
if delta is not None:
if date_end is not None:
date_start = date_end - delta
elif date_start is not None:
date_end = date_start + delta
else:
date_end = date.today()
date_start = date_end - delta
# We restrict public API access to the last 6 months.
six_months_ago = date.today() - timedelta(days=180)
if date_start:
date_start = max(six_months_ago, date_start)
f &= F(created__gte=date_start)
if date_end:
date_end = max(six_months_ago, date_end)
f &= F(created__lte=date_end)
search = search.filter(f)
search_query = request.GET.get("q", None)
if search_query is not None:
search = search.query(description__sqs=search_query)
# FIXME: Probably want to make this specifyable
search = search.order_by("-created")
# Explicitly include only publicly visible fields
search = search.values_dict(models.ResponseMappingType.public_fields())
maximum = smart_int(request.GET.get("max", None))
maximum = maximum or 1000
maximum = min(max(1, maximum), 10000)
responses = models.ResponseMappingType.reshape(search[:maximum])
return rest_framework.response.Response({"count": len(responses), "results": list(responses)})
def analytics_search(request):
template = 'analytics/analyzer/search.html'
output_format = request.GET.get('format', None)
page = smart_int(request.GET.get('page', 1), 1)
# Note: If we add additional querystring fields, we need to add
# them to generate_dashboard_url.
search_happy = request.GET.get('happy', None)
search_has_email = request.GET.get('has_email', None)
search_platform = request.GET.get('platform', None)
search_locale = request.GET.get('locale', None)
search_country = request.GET.get('country', None)
search_product = request.GET.get('product', None)
search_domain = request.GET.get('domain', None)
search_api = smart_int(request.GET.get('api', None), fallback=None)
search_version = request.GET.get('version', None)
search_query = request.GET.get('q', None)
search_date_start = smart_date(request.GET.get('date_start', None),
fallback=None)
search_date_end = smart_date(request.GET.get('date_end', None),
fallback=None)
search_bigram = request.GET.get('bigram', None)
search_source = request.GET.get('source', None)
search_campaign = request.GET.get('campaign', None)
search_organic = request.GET.get('organic', None)
selected = request.GET.get('selected', None)
filter_data = []
current_search = {'page': page}
search = ResponseMappingType.search()
f = F()
# If search happy is '0' or '1', set it to False or True, respectively.
search_happy = {'0': False, '1': True}.get(search_happy, None)
if search_happy in [False, True]:
f &= F(happy=search_happy)
current_search['happy'] = int(search_happy)
# If search has_email is '0' or '1', set it to False or True,
# respectively.
search_has_email = {'0': False, '1': True}.get(search_has_email, None)
if search_has_email in [False, True]:
f &= F(has_email=search_has_email)
current_search['has_email'] = int(search_has_email)
def unknown_to_empty(text):
"""Convert "Unknown" to "" to support old links"""
return u'' if text.lower() == u'unknown' else text
if search_platform is not None:
f &= F(platform=unknown_to_empty(search_platform))
current_search['platform'] = search_platform
if search_locale is not None:
f &= F(locale=unknown_to_empty(search_locale))
current_search['locale'] = search_locale
if search_product is not None:
f &= F(product=unknown_to_empty(search_product))
current_search['product'] = search_product
# Only show the version if there's a product.
if search_version is not None:
# Note: We only filter on version if we're filtering on
# product.
f &= F(version=unknown_to_empty(search_version))
current_search['version'] = search_version
# Only show the country if the product is Firefox OS.
if search_country is not None and search_product == 'Firefox OS':
f &= F(country=unknown_to_empty(search_country))
current_search['country'] = search_country
if search_domain is not None:
f &= F(url_domain=unknown_to_empty(search_domain))
current_search['domain'] = search_domain
if search_api is not None:
f &= F(api=search_api)
current_search['api'] = search_api
if search_date_start is None and search_date_end is None:
selected = '7d'
if search_date_end is None:
search_date_end = datetime.now()
if search_date_start is None:
search_date_start = search_date_end - timedelta(days=7)
current_search['date_end'] = search_date_end.strftime('%Y-%m-%d')
# Add one day, so that the search range includes the entire day.
end = search_date_end + timedelta(days=1)
# Note 'less than', not 'less than or equal', because of the added
# day above.
f &= F(created__lt=end)
current_search['date_start'] = search_date_start.strftime('%Y-%m-%d')
f &= F(created__gte=search_date_start)
if search_query:
current_search['q'] = search_query
search = search.query(description__sqs=search_query)
if search_bigram is not None:
f &= F(description_bigrams=search_bigram)
filter_data.append({
'display':
'Bigram',
'name':
'bigram',
'options': [{
'count': 'all',
'name': search_bigram,
'display': search_bigram,
'value': search_bigram,
'checked': True
}]
})
if search_source is not None:
f &= F(source=search_source)
current_search['source'] = search_source
if search_campaign is not None:
f &= F(campaign=search_campaign)
current_search['campaign'] = search_campaign
search_organic = {'0': False, '1': True}.get(search_organic, None)
if search_organic in [False, True]:
f &= F(organic=search_organic)
current_search['organic'] = int(search_organic)
search = search.filter(f).order_by('-created')
# If they're asking for a CSV export, then send them to the export
# screen.
if output_format == 'csv':
return _analytics_search_export(request, search)
# Search results and pagination
if page < 1:
page = 1
page_count = 50
start = page_count * (page - 1)
end = start + page_count
search_count = search.count()
opinion_page_ids = [mem[0] for mem in search.values_list('id')[start:end]]
# We convert what we get back from ES to what's in the db so we
# can get all the information.
opinion_page = Response.objects.filter(id__in=opinion_page_ids)
# Navigation facet data
# This loop does two things. First it maps 'T' -> True and 'F' ->
# False. This is probably something EU should be doing for
# us. Second, it restructures the data into a more convenient
# form.
counts = {
'happy': {},
'has_email': {},
'platform': {},
'locale': {},
'country': {},
'product': {},
'version': {},
'url_domain': {},
'api': {},
'source': {},
'campaign': {},
'organic': {},
}
facets = search.facet(*(counts.keys()),
filtered=bool(search._process_filters(f.filters)),
size=25)
for param, terms in facets.facet_counts().items():
for term in terms:
name = term['term']
if name == 'T':
name = True
elif name == 'F':
name = False
counts[param][name] = term['count']
def empty_to_unknown(text):
return 'Unknown' if text == u'' else text
filter_data.extend([
counts_to_options(counts['happy'].items(),
name='happy',
display='Sentiment',
display_map={
True: 'Happy',
False: 'Sad'
},
value_map={
True: 1,
False: 0
},
checked=search_happy),
counts_to_options(counts['has_email'].items(),
name='has_email',
display='Has email',
display_map={
True: 'Yes',
False: 'No'
},
value_map={
True: 1,
False: 0
},
checked=search_has_email),
counts_to_options(counts['product'].items(),
name='product',
display='Product',
display_map=empty_to_unknown,
checked=search_product)
])
# Only show the version if we're showing a specific
# product.
if search_product:
filter_data.append(
counts_to_options(counts['version'].items(),
name='version',
display='Version',
display_map=empty_to_unknown,
checked=search_version))
# Only show the country if the product is Firefox OS.
if search_product == 'Firefox OS':
filter_data.append(
counts_to_options(counts['country'].items(),
name='country',
display='Country',
checked=search_country,
display_map=country_name), )
filter_data.extend([
counts_to_options(counts['platform'].items(),
name='platform',
display='Platform',
display_map=empty_to_unknown,
checked=search_platform),
counts_to_options(counts['locale'].items(),
name='locale',
display='Locale',
checked=search_locale,
display_map=locale_name),
counts_to_options(counts['url_domain'].items(),
name='domain',
display='Domain',
checked=search_domain,
display_map=empty_to_unknown),
counts_to_options(counts['api'].items(),
name='api',
display='API version',
checked=search_api,
display_map=empty_to_unknown),
counts_to_options(counts['organic'].items(),
name='organic',
display='Organic',
display_map={
True: 'Yes',
False: 'No'
},
value_map={
True: 1,
False: 0
},
checked=search_organic),
counts_to_options(counts['source'].items(),
name='source',
display='Source',
checked=search_source,
display_map=empty_to_unknown),
counts_to_options(counts['campaign'].items(),
name='campaign',
display='Campaign',
checked=search_campaign,
display_map=empty_to_unknown),
])
return render(
request, template, {
'opinions': opinion_page,
'opinion_count': search_count,
'filter_data': filter_data,
'page': page,
'prev_page': page - 1 if start > 0 else None,
'next_page': page + 1 if end < search_count else None,
'current_search': current_search,
'selected': selected,
})