def test_index_chunk_task(self):
simple_items = [simple(save=True) for i in range(10)]
# With live indexing, that'll create items in the index. Since
# we want to test index_chunk_test, we need a clean index to
# start with so we delete and recreate it.
self.setup_indexes(empty=True)
self.refresh()
# Verify there's nothing in the index.
eq_(len(SimpleIndex.search()), 0)
# Create the record and the chunk and then run it through
# celery.
batch_id = 'ou812'
rec = record(batch_id=batch_id, save=True)
chunk = (SimpleIndex, [item.id for item in simple_items])
index_chunk_task.delay(get_index(), batch_id, rec.id, chunk)
# Verify everything is in the index now.
eq_(len(SimpleIndex.search()), 10)
# Verify the record was marked succeeded.
rec = Record.objects.get(pk=rec.id)
eq_(rec.status, Record.STATUS_SUCCESS)
def test_index_chunk_task(self):
simple_items = [simple(save=True) for i in range(10)]
# With live indexing, that'll create items in the index. Since
# we want to test index_chunk_test, we need a clean index to
# start with so we delete and recreate it.
self.setup_indexes(empty=True)
self.refresh()
# Verify there's nothing in the index.
eq_(len(SimpleIndex.search()), 0)
# Create the record and the chunk and then run it through
# celery.
batch_id = "ou812"
rec = record(batch_id=batch_id, save=True)
chunk = (SimpleIndex, [item.id for item in simple_items])
index_chunk_task.delay(get_index(), batch_id, rec.id, chunk)
# Verify everything is in the index now.
eq_(len(SimpleIndex.search()), 10)
# Verify the record was marked succeeded.
rec = Record.objects.get(pk=rec.id)
eq_(rec.status, Record.STATUS_SUCCESS)
def dashboard(request, template):
page = smart_int(request.GET.get('page', 1), 1)
search_happy = request.GET.get('happy', None)
search_platform = request.GET.get('platform', None)
search_locale = request.GET.get('locale', None)
search_query = request.GET.get('q', None)
search_date_start = smart_datetime(request.GET.get('date_start', None), fallback=None)
search_date_end = smart_datetime(request.GET.get('date_end', None), fallback=None)
selected = request.GET.get('selected', None)
current_search = {'page': page}
search = SimpleIndex.search()
f = F()
# If search happy is '0' or '1', set it to False or True, respectively.
search_happy = {'0': False, '1': True}.get(search_happy, None)
if search_happy in [False, True]:
f &= F(happy=search_happy)
current_search['happy'] = int(search_happy)
if search_platform:
f &= F(platform=search_platform)
current_search['platform'] = search_platform
if search_locale:
f &= F(locale=search_locale)
current_search['locale'] = search_locale
if search_date_start is None and search_date_end is None:
selected = '7d'
if search_date_end is None:
search_date_end = datetime.now()
if search_date_start is None:
search_date_start = search_date_end - timedelta(days=7)
current_search['date_end'] = search_date_end.strftime('%Y-%m-%d')
# Add one day, so that the search range includes the entire day.
end = search_date_end + timedelta(days=1)
# Note 'less than', not 'less than or equal', because of the added day above.
f &= F(created__lt=end)
current_search['date_start'] = search_date_start.strftime('%Y-%m-%d')
f &= F(created__gte=search_date_start)
if search_query:
fields = ['text', 'text_phrase', 'fuzzy']
query = dict(('description__%s' % f, search_query) for f in fields)
search = search.query(or_=query)
current_search['q'] = search_query
search = search.filter(f).order_by('-created')
facets = search.facet('happy', 'platform', 'locale',
filtered=bool(f.filters))
# This loop does two things. First it maps 'T' -> True and 'F' -> False.
# This is probably something EU should be doing for us. Second, it
# restructures the data into a more convenient form.
counts = {'happy': {}, 'platform': {}, 'locale': {}}
for param, terms in facets.facet_counts().items():
for term in terms:
name = term['term']
if name == 'T':
name = True
elif name == 'F':
name = False
counts[param][name] = term['count']
filter_data = [
counts_to_options(counts['happy'].items(), name='happy',
display=_('Sentiment'),
display_map={True: _('Happy'), False: _('Sad')},
value_map={True: 1, False: 0}, checked=search_happy),
counts_to_options(counts['platform'].items(),
name='platform', display=_('Platform'), checked=search_platform),
counts_to_options(counts['locale'].items(),
name='locale', display=_('Locale'), checked=search_locale,
display_map=locale_name)
]
# Histogram data
happy_data = []
sad_data = []
histograms = search.facet_raw(
happy={
'date_histogram': {'interval': 'day', 'field': 'created'},
'facet_filter': (f & F(happy=True)).filters
},
sad={
'date_histogram': {'interval': 'day', 'field': 'created'},
'facet_filter': (f & F(happy=False)).filters
},
).facet_counts()
# p['time'] is number of milliseconds since the epoch. Which is
# convenient, because that is what the front end wants.
happy_data = dict((p['time'], p['count']) for p in histograms['happy'])
sad_data = dict((p['time'], p['count']) for p in histograms['sad'])
_zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
histogram = [
{'label': _('Happy'), 'name': 'happy', 'data': sorted(happy_data.items())},
{'label': _('Sad'), 'name': 'sad', 'data': sorted(sad_data.items())},
]
# Pagination
if page < 1:
page = 1
page_count = 20
start = page_count * (page - 1)
end = start + page_count
search_count = search.count()
opinion_page = search[start:end]
return render(request, template, {
'opinions': opinion_page,
'opinion_count': search_count,
'filter_data': filter_data,
'histogram': histogram,
'page': page,
'prev_page': page - 1 if start > 0 else None,
'next_page': page + 1 if end < search_count else None,
'current_search': current_search,
'selected': selected,
})
def dashboard(request, template):
page = int(request.GET.get('page', 1))
search_happy = request.GET.get('happy', None)
search_platform = request.GET.get('platform', None)
search_locale = request.GET.get('locale', None)
current_search = {'page': page}
search = SimpleIndex.search()
f = F()
# If search happy is '0' or '1', set it to False or True, respectively.
search_happy = {'0': False, '1': True}.get(search_happy, None)
if search_happy in [False, True]:
f &= F(happy=search_happy)
current_search['happy'] = search_happy
if search_platform:
f &= F(platform=search_platform)
current_search['platform'] = search_platform
if search_locale:
f &= F(locale=search_locale)
current_search['locale'] = search_locale
search = search.filter(f).order_by('-created')
facets = search.facet('happy',
'platform',
'locale',
filtered=bool(f.filters))
# This loop does two things. First it maps 'T' -> True and 'F' -> False.
# This is probably something EU should be doing for us. Second, it
# restructures the data into a more convenient form.
counts = {'happy': {}, 'platform': {}, 'locale': {}}
try:
for param, terms in facets.facet_counts().items():
for term in terms:
name = term['term']
if name == 'T':
name = True
elif name == 'F':
name = False
counts[param][name] = term['count']
except (pyes.urllib3.TimeoutError, pyes.urllib3.MaxRetryError,
pyes.exceptions.IndexMissingException,
pyes.exceptions.ElasticSearchException):
# TODO: Fix this--we should log an error or show a message or
# something--anything except an HTTP 500 error on the front
# page.
pass
filter_data = [
counts_to_options(counts['happy'].items(),
name='happy',
display=_('Sentiment'),
display_map={
True: 'Happy',
False: 'Sad'
},
value_map={
True: 1,
False: 0
},
checked=search_happy),
counts_to_options(counts['platform'].items(),
name='platform',
display=_('Platform'),
checked=search_platform),
counts_to_options(counts['locale'].items(),
name='locale',
display=_('Locale'),
checked=search_locale,
display_map=locale_name)
]
# Histogram data
happy_data = []
sad_data = []
try:
histograms = search.facet_raw(
happy={
'date_histogram': {
'interval': 'day',
'field': 'created'
},
'facet_filter': (f & F(happy=True)).filters
},
sad={
'date_histogram': {
'interval': 'day',
'field': 'created'
},
'facet_filter': (f & F(happy=False)).filters
},
).facet_counts()
# p['time'] is number of milliseconds since the epoch. Which is
# convenient, because that is what the front end wants.
happy_data = [(p['time'], p['count']) for p in histograms['happy']]
sad_data = [(p['time'], int(p['count'])) for p in histograms['sad']]
except (pyes.urllib3.TimeoutError, pyes.urllib3.MaxRetryError,
pyes.exceptions.IndexMissingException,
pyes.exceptions.ElasticSearchException):
# TODO: Fix this--we should log an error or show a message or
# something--anything except an HTTP 500 error on the front
# page.
pass
histogram = [
{
'label': 'Happy',
'data': happy_data
},
{
'label': 'Sad',
'data': sad_data
},
]
# Pagination
if page < 1:
page = 1
page_count = 20
start = page_count * (page - 1)
end = start + page_count
try:
search_count = search.count()
opinion_page = search[start:end]
except (pyes.urllib3.TimeoutError, pyes.urllib3.MaxRetryError,
pyes.exceptions.IndexMissingException,
pyes.exceptions.ElasticSearchException):
# TODO: Fix this--we should log an error or show a message or
# something--anything except an HTTP 500 error on the front
# page.
search_count = 0
opinion_page = []
return render(
request, template, {
'opinions': opinion_page,
'opinion_count': search_count,
'filter_data': filter_data,
'histogram': histogram,
'page': page,
'prev_page': page - 1 if start > 0 else None,
'next_page': page + 1 if end < search_count else None,
'current_search': current_search,
})
def dashboard(request, template):
page = smart_int(request.GET.get("page", 1), 1)
search_happy = request.GET.get("happy", None)
search_platform = request.GET.get("platform", None)
search_locale = request.GET.get("locale", None)
search_query = request.GET.get("q", None)
search_date_start = smart_datetime(request.GET.get("date_start", None), fallback=None)
search_date_end = smart_datetime(request.GET.get("date_end", None), fallback=None)
selected = request.GET.get("selected", None)
current_search = {"page": page}
search = SimpleIndex.search()
f = F()
# If search happy is '0' or '1', set it to False or True, respectively.
search_happy = {"0": False, "1": True}.get(search_happy, None)
if search_happy in [False, True]:
f &= F(happy=search_happy)
current_search["happy"] = int(search_happy)
if search_platform:
f &= F(platform=search_platform)
current_search["platform"] = search_platform
if search_locale:
f &= F(locale=search_locale)
current_search["locale"] = search_locale
if search_date_start is None and search_date_end is None:
selected = "7d"
if search_date_end is None:
search_date_end = datetime.now()
if search_date_start is None:
search_date_start = search_date_end - timedelta(days=7)
current_search["date_end"] = search_date_end.strftime("%Y-%m-%d")
# Add one day, so that the search range includes the entire day.
end = search_date_end + timedelta(days=1)
# Note 'less than', not 'less than or equal', because of the added
# day above.
f &= F(created__lt=end)
current_search["date_start"] = search_date_start.strftime("%Y-%m-%d")
f &= F(created__gte=search_date_start)
if search_query:
fields = ["text", "text_phrase", "fuzzy"]
query = dict(("description__%s" % f, search_query) for f in fields)
search = search.query(or_=query)
current_search["q"] = search_query
search = search.filter(f).order_by("-created")
facets = search.facet("happy", "platform", "locale", filtered=bool(f.filters))
# This loop does two things. First it maps 'T' -> True and 'F' ->
# False. This is probably something EU should be doing for
# us. Second, it restructures the data into a more convenient
# form.
counts = {"happy": {}, "platform": {}, "locale": {}}
for param, terms in facets.facet_counts().items():
for term in terms:
name = term["term"]
if name == "T":
name = True
elif name == "F":
name = False
counts[param][name] = term["count"]
filter_data = [
counts_to_options(
counts["happy"].items(),
name="happy",
display=_("Sentiment"),
display_map={True: _("Happy"), False: _("Sad")},
value_map={True: 1, False: 0},
checked=search_happy,
),
counts_to_options(counts["platform"].items(), name="platform", display=_("Platform"), checked=search_platform),
counts_to_options(
counts["locale"].items(), name="locale", display=_("Locale"), checked=search_locale, display_map=locale_name
),
]
# Histogram data
happy_data = []
sad_data = []
histograms = search.facet_raw(
happy={"date_histogram": {"interval": "day", "field": "created"}, "facet_filter": (f & F(happy=True)).filters},
sad={"date_histogram": {"interval": "day", "field": "created"}, "facet_filter": (f & F(happy=False)).filters},
).facet_counts()
# p['time'] is number of milliseconds since the epoch. Which is
# convenient, because that is what the front end wants.
happy_data = dict((p["time"], p["count"]) for p in histograms["happy"])
sad_data = dict((p["time"], p["count"]) for p in histograms["sad"])
_zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
histogram = [
{"label": _("Happy"), "name": "happy", "data": sorted(happy_data.items())},
{"label": _("Sad"), "name": "sad", "data": sorted(sad_data.items())},
]
# Pagination
if page < 1:
page = 1
page_count = 20
start = page_count * (page - 1)
end = start + page_count
search_count = search.count()
opinion_page = search[start:end]
return render(
request,
template,
{
"opinions": opinion_page,
"opinion_count": search_count,
"filter_data": filter_data,
"histogram": histogram,
"page": page,
"prev_page": page - 1 if start > 0 else None,
"next_page": page + 1 if end < search_count else None,
"current_search": current_search,
"selected": selected,
},
)
def dashboard(request, template):
page = int(request.GET.get('page', 1))
search_happy = request.GET.get('happy', None)
search_platform = request.GET.get('platform', None)
search_locale = request.GET.get('locale', None)
current_search = {'page': page}
search = SimpleIndex.search()
f = F()
# If search happy is '0' or '1', set it to False or True, respectively.
search_happy = {'0': False, '1': True}.get(search_happy, None)
if search_happy in [False, True]:
f &= F(happy=search_happy)
current_search['happy'] = search_happy
if search_platform:
f &= F(platform=search_platform)
current_search['platform'] = search_platform
if search_locale:
f &= F(locale=search_locale)
current_search['locale'] = search_locale
search = search.filter(f).order_by('-created')
facets = search.facet('happy', 'platform', 'locale',
filtered=bool(f.filters))
# This loop does two things. First it maps 'T' -> True and 'F' -> False.
# This is probably something EU should be doing for us. Second, it
# restructures the data into a more convenient form.
counts = {'happy': {}, 'platform': {}, 'locale': {}}
try:
for param, terms in facets.facet_counts().items():
for term in terms:
name = term['term']
if name == 'T':
name = True
elif name == 'F':
name = False
counts[param][name] = term['count']
except (pyes.urllib3.TimeoutError,
pyes.urllib3.MaxRetryError,
pyes.exceptions.IndexMissingException,
pyes.exceptions.ElasticSearchException):
# TODO: Fix this--we should log an error or show a message or
# something--anything except an HTTP 500 error on the front
# page.
pass
filter_data = [
counts_to_options(counts['happy'].items(), name='happy',
display=_('Sentiment'), display_map={True: 'Happy', False: 'Sad'},
value_map={True: 1, False: 0}, checked=search_happy),
counts_to_options(counts['platform'].items(),
name='platform', display=_('Platform'), checked=search_platform),
counts_to_options(counts['locale'].items(),
name='locale', display=_('Locale'), checked=search_locale,
display_map=locale_name)
]
# Histogram data
happy_data = []
sad_data = []
try:
histograms = search.facet_raw(
happy={
'date_histogram': {'interval': 'day', 'field': 'created'},
'facet_filter': (f & F(happy=True)).filters
},
sad={
'date_histogram': {'interval': 'day', 'field': 'created'},
'facet_filter': (f & F(happy=False)).filters
},
).facet_counts()
# p['time'] is number of milliseconds since the epoch. Which is
# convenient, because that is what the front end wants.
happy_data = [(p['time'], p['count']) for p in histograms['happy']]
sad_data = [(p['time'], int(p['count'])) for p in histograms['sad']]
except (pyes.urllib3.TimeoutError,
pyes.urllib3.MaxRetryError,
pyes.exceptions.IndexMissingException,
pyes.exceptions.ElasticSearchException):
# TODO: Fix this--we should log an error or show a message or
# something--anything except an HTTP 500 error on the front
# page.
pass
histogram = [
{'label': 'Happy', 'data': happy_data},
{'label': 'Sad', 'data': sad_data},
]
# Pagination
if page < 1:
page = 1
page_count = 20
start = page_count * (page - 1)
end = start + page_count
try:
search_count = search.count()
opinion_page = search[start:end]
except (pyes.urllib3.TimeoutError,
pyes.urllib3.MaxRetryError,
pyes.exceptions.IndexMissingException,
pyes.exceptions.ElasticSearchException):
# TODO: Fix this--we should log an error or show a message or
# something--anything except an HTTP 500 error on the front
# page.
search_count = 0
opinion_page = []
return render(request, template, {
'opinions': opinion_page,
'opinion_count': search_count,
'filter_data': filter_data,
'histogram': histogram,
'page': page,
'prev_page': page - 1 if start > 0 else None,
'next_page': page + 1 if end < search_count else None,
'current_search': current_search,
})
