def test_index_chunk_task(self): simple_items = [simple(save=True) for i in range(10)] # With live indexing, that'll create items in the index. Since # we want to test index_chunk_test, we need a clean index to # start with so we delete and recreate it. self.setup_indexes(empty=True) self.refresh() # Verify there's nothing in the index. eq_(len(SimpleIndex.search()), 0) # Create the record and the chunk and then run it through # celery. batch_id = 'ou812' rec = record(batch_id=batch_id, save=True) chunk = (SimpleIndex, [item.id for item in simple_items]) index_chunk_task.delay(get_index(), batch_id, rec.id, chunk) # Verify everything is in the index now. eq_(len(SimpleIndex.search()), 10) # Verify the record was marked succeeded. rec = Record.objects.get(pk=rec.id) eq_(rec.status, Record.STATUS_SUCCESS)
def test_index_chunk_task(self): simple_items = [simple(save=True) for i in range(10)] # With live indexing, that'll create items in the index. Since # we want to test index_chunk_test, we need a clean index to # start with so we delete and recreate it. self.setup_indexes(empty=True) self.refresh() # Verify there's nothing in the index. eq_(len(SimpleIndex.search()), 0) # Create the record and the chunk and then run it through # celery. batch_id = "ou812" rec = record(batch_id=batch_id, save=True) chunk = (SimpleIndex, [item.id for item in simple_items]) index_chunk_task.delay(get_index(), batch_id, rec.id, chunk) # Verify everything is in the index now. eq_(len(SimpleIndex.search()), 10) # Verify the record was marked succeeded. rec = Record.objects.get(pk=rec.id) eq_(rec.status, Record.STATUS_SUCCESS)
def dashboard(request, template): page = smart_int(request.GET.get('page', 1), 1) search_happy = request.GET.get('happy', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) search_query = request.GET.get('q', None) search_date_start = smart_datetime(request.GET.get('date_start', None), fallback=None) search_date_end = smart_datetime(request.GET.get('date_end', None), fallback=None) selected = request.GET.get('selected', None) current_search = {'page': page} search = SimpleIndex.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F(happy=search_happy) current_search['happy'] = int(search_happy) if search_platform: f &= F(platform=search_platform) current_search['platform'] = search_platform if search_locale: f &= F(locale=search_locale) current_search['locale'] = search_locale if search_date_start is None and search_date_end is None: selected = '7d' if search_date_end is None: search_date_end = datetime.now() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) current_search['date_end'] = search_date_end.strftime('%Y-%m-%d') # Add one day, so that the search range includes the entire day. end = search_date_end + timedelta(days=1) # Note 'less than', not 'less than or equal', because of the added day above. f &= F(created__lt=end) current_search['date_start'] = search_date_start.strftime('%Y-%m-%d') f &= F(created__gte=search_date_start) if search_query: fields = ['text', 'text_phrase', 'fuzzy'] query = dict(('description__%s' % f, search_query) for f in fields) search = search.query(or_=query) current_search['q'] = search_query search = search.filter(f).order_by('-created') facets = search.facet('happy', 'platform', 'locale', filtered=bool(f.filters)) # This loop does two things. First it maps 'T' -> True and 'F' -> False. # This is probably something EU should be doing for us. Second, it # restructures the data into a more convenient form. counts = {'happy': {}, 'platform': {}, 'locale': {}} for param, terms in facets.facet_counts().items(): for term in terms: name = term['term'] if name == 'T': name = True elif name == 'F': name = False counts[param][name] = term['count'] filter_data = [ counts_to_options(counts['happy'].items(), name='happy', display=_('Sentiment'), display_map={True: _('Happy'), False: _('Sad')}, value_map={True: 1, False: 0}, checked=search_happy), counts_to_options(counts['platform'].items(), name='platform', display=_('Platform'), checked=search_platform), counts_to_options(counts['locale'].items(), name='locale', display=_('Locale'), checked=search_locale, display_map=locale_name) ] # Histogram data happy_data = [] sad_data = [] histograms = search.facet_raw( happy={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': (f & F(happy=True)).filters }, sad={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': (f & F(happy=False)).filters }, ).facet_counts() # p['time'] is number of milliseconds since the epoch. Which is # convenient, because that is what the front end wants. happy_data = dict((p['time'], p['count']) for p in histograms['happy']) sad_data = dict((p['time'], p['count']) for p in histograms['sad']) _zero_fill(search_date_start, search_date_end, [happy_data, sad_data]) histogram = [ {'label': _('Happy'), 'name': 'happy', 'data': sorted(happy_data.items())}, {'label': _('Sad'), 'name': 'sad', 'data': sorted(sad_data.items())}, ] # Pagination if page < 1: page = 1 page_count = 20 start = page_count * (page - 1) end = start + page_count search_count = search.count() opinion_page = search[start:end] return render(request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'histogram': histogram, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, 'selected': selected, })
def dashboard(request, template): page = int(request.GET.get('page', 1)) search_happy = request.GET.get('happy', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) current_search = {'page': page} search = SimpleIndex.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F(happy=search_happy) current_search['happy'] = search_happy if search_platform: f &= F(platform=search_platform) current_search['platform'] = search_platform if search_locale: f &= F(locale=search_locale) current_search['locale'] = search_locale search = search.filter(f).order_by('-created') facets = search.facet('happy', 'platform', 'locale', filtered=bool(f.filters)) # This loop does two things. First it maps 'T' -> True and 'F' -> False. # This is probably something EU should be doing for us. Second, it # restructures the data into a more convenient form. counts = {'happy': {}, 'platform': {}, 'locale': {}} try: for param, terms in facets.facet_counts().items(): for term in terms: name = term['term'] if name == 'T': name = True elif name == 'F': name = False counts[param][name] = term['count'] except (pyes.urllib3.TimeoutError, pyes.urllib3.MaxRetryError, pyes.exceptions.IndexMissingException, pyes.exceptions.ElasticSearchException): # TODO: Fix this--we should log an error or show a message or # something--anything except an HTTP 500 error on the front # page. pass filter_data = [ counts_to_options(counts['happy'].items(), name='happy', display=_('Sentiment'), display_map={ True: 'Happy', False: 'Sad' }, value_map={ True: 1, False: 0 }, checked=search_happy), counts_to_options(counts['platform'].items(), name='platform', display=_('Platform'), checked=search_platform), counts_to_options(counts['locale'].items(), name='locale', display=_('Locale'), checked=search_locale, display_map=locale_name) ] # Histogram data happy_data = [] sad_data = [] try: histograms = search.facet_raw( happy={ 'date_histogram': { 'interval': 'day', 'field': 'created' }, 'facet_filter': (f & F(happy=True)).filters }, sad={ 'date_histogram': { 'interval': 'day', 'field': 'created' }, 'facet_filter': (f & F(happy=False)).filters }, ).facet_counts() # p['time'] is number of milliseconds since the epoch. Which is # convenient, because that is what the front end wants. happy_data = [(p['time'], p['count']) for p in histograms['happy']] sad_data = [(p['time'], int(p['count'])) for p in histograms['sad']] except (pyes.urllib3.TimeoutError, pyes.urllib3.MaxRetryError, pyes.exceptions.IndexMissingException, pyes.exceptions.ElasticSearchException): # TODO: Fix this--we should log an error or show a message or # something--anything except an HTTP 500 error on the front # page. pass histogram = [ { 'label': 'Happy', 'data': happy_data }, { 'label': 'Sad', 'data': sad_data }, ] # Pagination if page < 1: page = 1 page_count = 20 start = page_count * (page - 1) end = start + page_count try: search_count = search.count() opinion_page = search[start:end] except (pyes.urllib3.TimeoutError, pyes.urllib3.MaxRetryError, pyes.exceptions.IndexMissingException, pyes.exceptions.ElasticSearchException): # TODO: Fix this--we should log an error or show a message or # something--anything except an HTTP 500 error on the front # page. search_count = 0 opinion_page = [] return render( request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'histogram': histogram, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, })
def dashboard(request, template): page = smart_int(request.GET.get("page", 1), 1) search_happy = request.GET.get("happy", None) search_platform = request.GET.get("platform", None) search_locale = request.GET.get("locale", None) search_query = request.GET.get("q", None) search_date_start = smart_datetime(request.GET.get("date_start", None), fallback=None) search_date_end = smart_datetime(request.GET.get("date_end", None), fallback=None) selected = request.GET.get("selected", None) current_search = {"page": page} search = SimpleIndex.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {"0": False, "1": True}.get(search_happy, None) if search_happy in [False, True]: f &= F(happy=search_happy) current_search["happy"] = int(search_happy) if search_platform: f &= F(platform=search_platform) current_search["platform"] = search_platform if search_locale: f &= F(locale=search_locale) current_search["locale"] = search_locale if search_date_start is None and search_date_end is None: selected = "7d" if search_date_end is None: search_date_end = datetime.now() if search_date_start is None: search_date_start = search_date_end - timedelta(days=7) current_search["date_end"] = search_date_end.strftime("%Y-%m-%d") # Add one day, so that the search range includes the entire day. end = search_date_end + timedelta(days=1) # Note 'less than', not 'less than or equal', because of the added # day above. f &= F(created__lt=end) current_search["date_start"] = search_date_start.strftime("%Y-%m-%d") f &= F(created__gte=search_date_start) if search_query: fields = ["text", "text_phrase", "fuzzy"] query = dict(("description__%s" % f, search_query) for f in fields) search = search.query(or_=query) current_search["q"] = search_query search = search.filter(f).order_by("-created") facets = search.facet("happy", "platform", "locale", filtered=bool(f.filters)) # This loop does two things. First it maps 'T' -> True and 'F' -> # False. This is probably something EU should be doing for # us. Second, it restructures the data into a more convenient # form. counts = {"happy": {}, "platform": {}, "locale": {}} for param, terms in facets.facet_counts().items(): for term in terms: name = term["term"] if name == "T": name = True elif name == "F": name = False counts[param][name] = term["count"] filter_data = [ counts_to_options( counts["happy"].items(), name="happy", display=_("Sentiment"), display_map={True: _("Happy"), False: _("Sad")}, value_map={True: 1, False: 0}, checked=search_happy, ), counts_to_options(counts["platform"].items(), name="platform", display=_("Platform"), checked=search_platform), counts_to_options( counts["locale"].items(), name="locale", display=_("Locale"), checked=search_locale, display_map=locale_name ), ] # Histogram data happy_data = [] sad_data = [] histograms = search.facet_raw( happy={"date_histogram": {"interval": "day", "field": "created"}, "facet_filter": (f & F(happy=True)).filters}, sad={"date_histogram": {"interval": "day", "field": "created"}, "facet_filter": (f & F(happy=False)).filters}, ).facet_counts() # p['time'] is number of milliseconds since the epoch. Which is # convenient, because that is what the front end wants. happy_data = dict((p["time"], p["count"]) for p in histograms["happy"]) sad_data = dict((p["time"], p["count"]) for p in histograms["sad"]) _zero_fill(search_date_start, search_date_end, [happy_data, sad_data]) histogram = [ {"label": _("Happy"), "name": "happy", "data": sorted(happy_data.items())}, {"label": _("Sad"), "name": "sad", "data": sorted(sad_data.items())}, ] # Pagination if page < 1: page = 1 page_count = 20 start = page_count * (page - 1) end = start + page_count search_count = search.count() opinion_page = search[start:end] return render( request, template, { "opinions": opinion_page, "opinion_count": search_count, "filter_data": filter_data, "histogram": histogram, "page": page, "prev_page": page - 1 if start > 0 else None, "next_page": page + 1 if end < search_count else None, "current_search": current_search, "selected": selected, }, )
def dashboard(request, template): page = int(request.GET.get('page', 1)) search_happy = request.GET.get('happy', None) search_platform = request.GET.get('platform', None) search_locale = request.GET.get('locale', None) current_search = {'page': page} search = SimpleIndex.search() f = F() # If search happy is '0' or '1', set it to False or True, respectively. search_happy = {'0': False, '1': True}.get(search_happy, None) if search_happy in [False, True]: f &= F(happy=search_happy) current_search['happy'] = search_happy if search_platform: f &= F(platform=search_platform) current_search['platform'] = search_platform if search_locale: f &= F(locale=search_locale) current_search['locale'] = search_locale search = search.filter(f).order_by('-created') facets = search.facet('happy', 'platform', 'locale', filtered=bool(f.filters)) # This loop does two things. First it maps 'T' -> True and 'F' -> False. # This is probably something EU should be doing for us. Second, it # restructures the data into a more convenient form. counts = {'happy': {}, 'platform': {}, 'locale': {}} try: for param, terms in facets.facet_counts().items(): for term in terms: name = term['term'] if name == 'T': name = True elif name == 'F': name = False counts[param][name] = term['count'] except (pyes.urllib3.TimeoutError, pyes.urllib3.MaxRetryError, pyes.exceptions.IndexMissingException, pyes.exceptions.ElasticSearchException): # TODO: Fix this--we should log an error or show a message or # something--anything except an HTTP 500 error on the front # page. pass filter_data = [ counts_to_options(counts['happy'].items(), name='happy', display=_('Sentiment'), display_map={True: 'Happy', False: 'Sad'}, value_map={True: 1, False: 0}, checked=search_happy), counts_to_options(counts['platform'].items(), name='platform', display=_('Platform'), checked=search_platform), counts_to_options(counts['locale'].items(), name='locale', display=_('Locale'), checked=search_locale, display_map=locale_name) ] # Histogram data happy_data = [] sad_data = [] try: histograms = search.facet_raw( happy={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': (f & F(happy=True)).filters }, sad={ 'date_histogram': {'interval': 'day', 'field': 'created'}, 'facet_filter': (f & F(happy=False)).filters }, ).facet_counts() # p['time'] is number of milliseconds since the epoch. Which is # convenient, because that is what the front end wants. happy_data = [(p['time'], p['count']) for p in histograms['happy']] sad_data = [(p['time'], int(p['count'])) for p in histograms['sad']] except (pyes.urllib3.TimeoutError, pyes.urllib3.MaxRetryError, pyes.exceptions.IndexMissingException, pyes.exceptions.ElasticSearchException): # TODO: Fix this--we should log an error or show a message or # something--anything except an HTTP 500 error on the front # page. pass histogram = [ {'label': 'Happy', 'data': happy_data}, {'label': 'Sad', 'data': sad_data}, ] # Pagination if page < 1: page = 1 page_count = 20 start = page_count * (page - 1) end = start + page_count try: search_count = search.count() opinion_page = search[start:end] except (pyes.urllib3.TimeoutError, pyes.urllib3.MaxRetryError, pyes.exceptions.IndexMissingException, pyes.exceptions.ElasticSearchException): # TODO: Fix this--we should log an error or show a message or # something--anything except an HTTP 500 error on the front # page. search_count = 0 opinion_page = [] return render(request, template, { 'opinions': opinion_page, 'opinion_count': search_count, 'filter_data': filter_data, 'histogram': histogram, 'page': page, 'prev_page': page - 1 if start > 0 else None, 'next_page': page + 1 if end < search_count else None, 'current_search': current_search, })