def auto_lock_old_questions(): """Locks all questions that were created over 180 days ago""" # Set up logging so it doesn't send Ricky email. logging.basicConfig(level=logging.ERROR) # Get a list of ids of questions we're going to go change. We need # a list of ids so that we can feed it to the update, but then # also know what we need to update in the index. days_180 = datetime.now() - timedelta(days=180) q_ids = list( Question.objects.filter(is_locked=False).filter( created__lte=days_180).values_list('id', flat=True)) if q_ids: log.info('Updating %d questions', len(q_ids)) sql = """ UPDATE questions_question SET is_locked = 1 WHERE id IN (%s) """ % ','.join(map(str, q_ids)) cursor = connection.cursor() cursor.execute(sql) transaction.commit_unless_managed() if settings.ES_LIVE_INDEXING: try: # So... the first time this runs, it'll handle 160K # questions or so which stresses everything. Thus we # do it in chunks because otherwise this won't work. # # After we've done this for the first time, we can nix # the chunking code. from search.utils import chunked for chunk in chunked(q_ids, 100): # Fetch all the documents we need to update. es_docs = get_documents(QuestionMappingType, chunk) log.info('Updating %d index documents', len(es_docs)) documents = [] # For each document, update the data and stick it # back in the index. for doc in es_docs: doc[u'question_is_locked'] = True doc[u'indexed_on'] = int(time.time()) documents.append(doc) QuestionMappingType.bulk_index(documents, id_field='document_id') except ES_EXCEPTIONS: # Something happened with ES, so let's push index # updating into an index_task which retries when it # fails because of ES issues. index_task.delay(QuestionMappingType, q_ids)
def test_added(self): search = QuestionMappingType.search() # Create a question--that adds one document to the index. q = question(title=u'Does this test work?', save=True) self.refresh() query = dict(('%s__text' % field, 'test') for field in QuestionMappingType.get_query_fields()) eq_(search.query(should=True, **query).count(), 1) # Create an answer for the question. It shouldn't be searchable # until the answer is saved. a = answer(content=u'There\'s only one way to find out!', question=q) self.refresh() query = dict(('%s__text' % field, 'only') for field in QuestionMappingType.get_query_fields()) eq_(search.query(should=True, **query).count(), 0) a.save() self.refresh() query = dict(('%s__text' % field, 'only') for field in QuestionMappingType.get_query_fields()) eq_(search.query(should=True, **query).count(), 1) # Make sure that there's only one question document in the # index--creating an answer should have updated the existing # one. eq_(search.count(), 1)
def test_cron_updates_counts(self): q = question(save=True) self.refresh() eq_(q.num_votes_past_week, 0) # NB: Need to call .values_dict() here and later otherwise we # get a Question object which has data from the database and # not the index. document = (QuestionMappingType.search() .values_dict('question_num_votes_past_week') .filter(id=q.id))[0] eq_(document['question_num_votes_past_week'], 0) vote = questionvote(question=q, anonymous_id='abc123') vote.save() q.num_votes_past_week = 0 q.save() update_weekly_votes() self.refresh() q = Question.objects.get(pk=q.pk) eq_(1, q.num_votes_past_week) document = (QuestionMappingType.search() .values_dict('question_num_votes_past_week') .filter(id=q.id))[0] eq_(document['question_num_votes_past_week'], 1)
def auto_lock_old_questions(): """Locks all questions that were created over 180 days ago""" # Set up logging so it doesn't send Ricky email. logging.basicConfig(level=logging.ERROR) # Get a list of ids of questions we're going to go change. We need # a list of ids so that we can feed it to the update, but then # also know what we need to update in the index. days_180 = datetime.now() - timedelta(days=180) q_ids = list(Question.objects.filter(is_locked=False) .filter(created__lte=days_180) .values_list('id', flat=True)) if q_ids: log.info('Updating %d questions', len(q_ids)) sql = """ UPDATE questions_question SET is_locked = 1 WHERE id IN (%s) """ % ','.join(map(str, q_ids)) cursor = connection.cursor() cursor.execute(sql) transaction.commit_unless_managed() if settings.ES_LIVE_INDEXING: try: # So... the first time this runs, it'll handle 160K # questions or so which stresses everything. Thus we # do it in chunks because otherwise this won't work. # # After we've done this for the first time, we can nix # the chunking code. from search.utils import chunked for chunk in chunked(q_ids, 100): # Fetch all the documents we need to update. es_docs = get_documents(QuestionMappingType, chunk) log.info('Updating %d index documents', len(es_docs)) documents = [] # For each document, update the data and stick it # back in the index. for doc in es_docs: doc[u'question_is_locked'] = True doc[u'indexed_on'] = int(time.time()) documents.append(doc) QuestionMappingType.bulk_index( documents, id_field='document_id') except ES_EXCEPTIONS: # Something happened with ES, so let's push index # updating into an index_task which retries when it # fails because of ES issues. index_task.delay(QuestionMappingType, q_ids)
def test_cron_updates_counts(self): q = question(save=True) self.refresh() eq_(q.num_votes_past_week, 0) # NB: Need to call .values_dict() here and later otherwise we # get a Question object which has data from the database and # not the index. document = (QuestionMappingType.search().values_dict( 'question_num_votes_past_week').filter(id=q.id))[0] eq_(document['question_num_votes_past_week'], 0) vote = questionvote(question=q, anonymous_id='abc123') vote.save() q.num_votes_past_week = 0 q.save() update_weekly_votes() self.refresh() q = Question.objects.get(pk=q.pk) eq_(1, q.num_votes_past_week) document = (QuestionMappingType.search().values_dict( 'question_num_votes_past_week').filter(id=q.id))[0] eq_(document['question_num_votes_past_week'], 1)
def update_question_vote_chunk(data): """Update num_votes_past_week for a number of questions.""" # First we recalculate num_votes_past_week in the db. log.info('Calculating past week votes for %s questions.' % len(data)) ids = ','.join(map(str, data)) sql = """ UPDATE questions_question q SET num_votes_past_week = ( SELECT COUNT(created) FROM questions_questionvote qv WHERE qv.question_id = q.id AND qv.created >= DATE(SUBDATE(NOW(), 7)) ) WHERE q.id IN (%s); """ % ids cursor = connection.cursor() cursor.execute(sql) transaction.commit_unless_managed() # Next we update our index with the changes we made directly in # the db. if data and settings.ES_LIVE_INDEXING: # Get the data we just updated from the database. sql = """ SELECT id, num_votes_past_week FROM questions_question WHERE id in (%s); """ % ids cursor = connection.cursor() cursor.execute(sql) # Since this returns (id, num_votes_past_week) tuples, we can # convert that directly to a dict. id_to_num = dict(cursor.fetchall()) try: # Fetch all the documents we need to update. from questions.models import QuestionMappingType from search import es_utils es_docs = es_utils.get_documents(QuestionMappingType, data) # For each document, update the data and stick it back in the # index. for doc in es_docs: # Note: Need to keep this in sync with # Question.extract_document. num = id_to_num[int(doc[u'id'])] doc[u'question_num_votes_past_week'] = num QuestionMappingType.index(doc, id_=doc['id']) except ES_EXCEPTIONS: # Something happened with ES, so let's push index updating # into an index_task which retries when it fails because # of ES issues. index_task.delay(QuestionMappingType, id_to_num.keys())
def test_questions_tags(self): """Make sure that adding tags to a Question causes it to refresh the index. """ tag = u'hiphop' eq_(QuestionMappingType.search().filter(question_tag=tag).count(), 0) q = question(save=True) self.refresh() eq_(QuestionMappingType.search().filter(question_tag=tag).count(), 0) q.tags.add(tag) self.refresh() eq_(QuestionMappingType.search().filter(question_tag=tag).count(), 1) q.tags.remove(tag) self.refresh() eq_(QuestionMappingType.search().filter(question_tag=tag).count(), 0)
def test_question_no_answers_deleted(self): search = QuestionMappingType.search() q = question(title=u'Does this work?', save=True) self.refresh() eq_(search.query(question_title__text='work').count(), 1) q.delete() self.refresh() eq_(search.query(question_title__text='work').count(), 0)
def test_question_products(self): """Make sure that adding products to a Question causes it to refresh the index. """ p = product(slug=u'desktop', save=True) eq_(QuestionMappingType.search().filter(product=p.slug).count(), 0) q = question(save=True) self.refresh() eq_(QuestionMappingType.search().filter(product=p.slug).count(), 0) q.products.add(p) self.refresh() eq_(QuestionMappingType.search().filter(product=p.slug).count(), 1) q.products.remove(p) self.refresh() # Make sure the question itself is still there and that we didn't # accidentally delete it through screwed up signal handling: eq_(QuestionMappingType.search().filter().count(), 1) eq_(QuestionMappingType.search().filter(product=p.slug).count(), 0)
def test_question_topics(self): """Make sure that adding topics to a Question causes it to refresh the index. """ t = topic(slug=u'hiphop', save=True) eq_(QuestionMappingType.search().filter(topic=t.slug).count(), 0) q = question(save=True) self.refresh() eq_(QuestionMappingType.search().filter(topic=t.slug).count(), 0) q.topics.add(t) self.refresh() eq_(QuestionMappingType.search().filter(topic=t.slug).count(), 1) q.topics.clear() self.refresh() # Make sure the question itself is still there and that we didn't # accidentally delete it through screwed up signal handling: eq_(QuestionMappingType.search().filter().count(), 1) eq_(QuestionMappingType.search().filter(topic=t.slug).count(), 0)
def suggestions(request): """A simple search view that returns OpenSearch suggestions.""" mimetype = 'application/x-suggestions+json' term = request.GET.get('q') if not term: return HttpResponseBadRequest(mimetype=mimetype) site = Site.objects.get_current() locale = locale_or_default(request.LANGUAGE_CODE) try: query = dict(('%s__text' % field, term) for field in DocumentMappingType.get_query_fields()) wiki_s = (DocumentMappingType.search() .filter(document_is_archived=False) .filter(document_locale=locale) .values_dict('document_title', 'url') .query(or_=query)[:5]) query = dict(('%s__text' % field, term) for field in QuestionMappingType.get_query_fields()) question_s = (QuestionMappingType.search() .filter(question_has_helpful=True) .values_dict('question_title', 'url') .query(or_=query)[:5]) results = list(chain(question_s, wiki_s)) except ES_EXCEPTIONS: # If we have ES problems, we just send back an empty result # set. results = [] urlize = lambda r: u'https://%s%s' % (site, r['url']) titleize = lambda r: (r['document_title'] if 'document_title' in r else r['question_title']) data = [term, [titleize(r) for r in results], [], [urlize(r) for r in results]] return HttpResponse(json.dumps(data), mimetype=mimetype)
def test_question_questionvote(self): search = QuestionMappingType.search() # Create a question and verify it doesn't show up in a # query for num_votes__gt=0. q = question(title=u'model makers will inherit the earth', save=True) self.refresh() eq_(search.filter(question_num_votes__gt=0).count(), 0) # Add a QuestionVote--it should show up now. questionvote(question=q, save=True) self.refresh() eq_(search.filter(question_num_votes__gt=0).count(), 1)
def test_case_insensitive_search(self): """Ensure the default searcher is case insensitive.""" answervote( answer=answer(question=question(title='lolrus', content='I am the lolrus.', save=True), save=True), helpful=True).save() self.refresh() result = QuestionMappingType.search().query( question_title__text='LOLRUS', question_content__text='LOLRUS') assert result.count() > 0
def test_question_one_answer_deleted(self): search = QuestionMappingType.search() q = question(title=u'are model makers the new pink?', save=True) a = answer(content=u'yes.', question=q, save=True) self.refresh() # Question and its answers are a single document--so the # index count should be only 1. eq_(search.query(question_title__text='pink').count(), 1) # After deleting the answer, the question document should # remain. a.delete() self.refresh() eq_(search.query(question_title__text='pink').count(), 1) # Delete the question and it should be removed from the # index. q.delete() self.refresh() eq_(search.query(question_title__text='pink').count(), 0)
def search(request, template=None): """ES-specific search view""" # JSON-specific variables is_json = (request.GET.get('format') == 'json') callback = request.GET.get('callback', '').strip() mimetype = 'application/x-javascript' if callback else 'application/json' # Search "Expires" header format expires_fmt = '%A, %d %B %Y %H:%M:%S GMT' # Check callback is valid if is_json and callback and not jsonp_is_valid(callback): return HttpResponse( json.dumps({'error': _('Invalid callback function.')}), mimetype=mimetype, status=400) language = locale_or_default( request.GET.get('language', request.LANGUAGE_CODE)) r = request.GET.copy() a = request.GET.get('a', '0') # Search default values try: category = (map(int, r.getlist('category')) or settings.SEARCH_DEFAULT_CATEGORIES) except ValueError: category = settings.SEARCH_DEFAULT_CATEGORIES r.setlist('category', category) # Basic form if a == '0': r['w'] = r.get('w', constants.WHERE_BASIC) # Advanced form if a == '2': r['language'] = language r['a'] = '1' # TODO: Rewrite so SearchForm is unbound initially and we can use # `initial` on the form fields. if 'include_archived' not in r: r['include_archived'] = False search_form = SearchForm(r) if not search_form.is_valid() or a == '2': if is_json: return HttpResponse( json.dumps({'error': _('Invalid search data.')}), mimetype=mimetype, status=400) t = template if request.MOBILE else 'search/form.html' search_ = render(request, t, { 'advanced': a, 'request': request, 'search_form': search_form}) search_['Cache-Control'] = 'max-age=%s' % \ (settings.SEARCH_CACHE_PERIOD * 60) search_['Expires'] = (datetime.utcnow() + timedelta( minutes=settings.SEARCH_CACHE_PERIOD)) \ .strftime(expires_fmt) return search_ cleaned = search_form.cleaned_data if request.MOBILE and cleaned['w'] == constants.WHERE_BASIC: cleaned['w'] = constants.WHERE_WIKI page = max(smart_int(request.GET.get('page')), 1) offset = (page - 1) * settings.SEARCH_RESULTS_PER_PAGE lang = language.lower() if settings.LANGUAGES.get(lang): lang_name = settings.LANGUAGES[lang] else: lang_name = '' # We use a regular S here because we want to search across # multiple doctypes. searcher = (UntypedS().es(urls=settings.ES_URLS) .indexes(es_utils.READ_INDEX)) wiki_f = F(model='wiki_document') question_f = F(model='questions_question') discussion_f = F(model='forums_thread') # Start - wiki filters if cleaned['w'] & constants.WHERE_WIKI: # Category filter if cleaned['category']: wiki_f &= F(document_category__in=cleaned['category']) # Locale filter wiki_f &= F(document_locale=language) # Product filter products = cleaned['product'] for p in products: wiki_f &= F(product=p) # Topics filter topics = cleaned['topics'] for t in topics: wiki_f &= F(topic=t) # Archived bit if a == '0' and not cleaned['include_archived']: # Default to NO for basic search: cleaned['include_archived'] = False if not cleaned['include_archived']: wiki_f &= F(document_is_archived=False) # End - wiki filters # Start - support questions filters if cleaned['w'] & constants.WHERE_SUPPORT: # Solved is set by default if using basic search if a == '0' and not cleaned['has_helpful']: cleaned['has_helpful'] = constants.TERNARY_YES # These filters are ternary, they can be either YES, NO, or OFF ternary_filters = ('is_locked', 'is_solved', 'has_answers', 'has_helpful') d = dict(('question_%s' % filter_name, _ternary_filter(cleaned[filter_name])) for filter_name in ternary_filters if cleaned[filter_name]) if d: question_f &= F(**d) if cleaned['asked_by']: question_f &= F(question_creator=cleaned['asked_by']) if cleaned['answered_by']: question_f &= F(question_answer_creator=cleaned['answered_by']) q_tags = [t.strip() for t in cleaned['q_tags'].split(',')] for t in q_tags: if t: question_f &= F(question_tag=t) # Product filter products = cleaned['product'] for p in products: question_f &= F(product=p) # Topics filter topics = cleaned['topics'] for t in topics: question_f &= F(topic=t) # End - support questions filters # Start - discussion forum filters if cleaned['w'] & constants.WHERE_DISCUSSION: if cleaned['author']: discussion_f &= F(post_author_ord=cleaned['author']) if cleaned['thread_type']: if constants.DISCUSSION_STICKY in cleaned['thread_type']: discussion_f &= F(post_is_sticky=1) if constants.DISCUSSION_LOCKED in cleaned['thread_type']: discussion_f &= F(post_is_locked=1) if cleaned['forum']: discussion_f &= F(post_forum_id__in=cleaned['forum']) # End - discussion forum filters # Created filter unix_now = int(time.time()) interval_filters = ( ('created', cleaned['created'], cleaned['created_date']), ('updated', cleaned['updated'], cleaned['updated_date'])) for filter_name, filter_option, filter_date in interval_filters: if filter_option == constants.INTERVAL_BEFORE: before = {filter_name + '__gte': 0, filter_name + '__lte': max(filter_date, 0)} discussion_f &= F(**before) question_f &= F(**before) elif filter_option == constants.INTERVAL_AFTER: after = {filter_name + '__gte': min(filter_date, unix_now), filter_name + '__lte': unix_now} discussion_f &= F(**after) question_f &= F(**after) # In basic search, we limit questions from the last # SEARCH_DEFAULT_MAX_QUESTION_AGE seconds. if a == '0': start_date = unix_now - settings.SEARCH_DEFAULT_MAX_QUESTION_AGE question_f &= F(created__gte=start_date) # Note: num_voted (with a d) is a different field than num_votes # (with an s). The former is a dropdown and the latter is an # integer value. if cleaned['num_voted'] == constants.INTERVAL_BEFORE: question_f &= F(question_num_votes__lte=max(cleaned['num_votes'], 0)) elif cleaned['num_voted'] == constants.INTERVAL_AFTER: question_f &= F(question_num_votes__gte=cleaned['num_votes']) # Done with all the filtery stuff--time to generate results # Combine all the filters and add to the searcher doctypes = [] final_filter = F() if cleaned['w'] & constants.WHERE_WIKI: doctypes.append(DocumentMappingType.get_mapping_type_name()) final_filter |= wiki_f if cleaned['w'] & constants.WHERE_SUPPORT: doctypes.append(QuestionMappingType.get_mapping_type_name()) final_filter |= question_f if cleaned['w'] & constants.WHERE_DISCUSSION: doctypes.append(ThreadMappingType.get_mapping_type_name()) final_filter |= discussion_f searcher = searcher.doctypes(*doctypes) searcher = searcher.filter(final_filter) if 'explain' in request.GET and request.GET['explain'] == '1': searcher = searcher.explain() documents = ComposedList() try: cleaned_q = cleaned['q'] # Set up the highlights # First 500 characters of content in one big fragment searcher = searcher.highlight( 'question_content', 'discussion_content', 'document_summary', pre_tags=['<b>'], post_tags=['</b>'], number_of_fragments=0, fragment_size=500) # Set up boosts searcher = searcher.boost( question_title=4.0, question_content=3.0, question_answer_content=3.0, post_title=2.0, post_content=1.0, document_title=6.0, document_content=1.0, document_keywords=8.0, document_summary=2.0, # Text phrases in document titles and content get an extra # boost. document_title__text_phrase=10.0, document_content__text_phrase=8.0) # Apply sortby for advanced search of questions if cleaned['w'] == constants.WHERE_SUPPORT: sortby = cleaned['sortby'] try: searcher = searcher.order_by( *constants.SORT_QUESTIONS[sortby]) except IndexError: # Skip index errors because they imply the user is # sending us sortby values that aren't valid. pass # Apply sortby for advanced search of kb documents if cleaned['w'] == constants.WHERE_WIKI: sortby = cleaned['sortby_documents'] try: searcher = searcher.order_by( *constants.SORT_DOCUMENTS[sortby]) except IndexError: # Skip index errors because they imply the user is # sending us sortby values that aren't valid. pass # Build the query if cleaned_q: query_fields = chain(*[cls.get_query_fields() for cls in get_mapping_types()]) query = {} # Create text and text_phrase queries for every field # we want to search. for field in query_fields: for query_type in ['text', 'text_phrase']: query['%s__%s' % (field, query_type)] = cleaned_q searcher = searcher.query(should=True, **query) num_results = min(searcher.count(), settings.SEARCH_MAX_RESULTS) # TODO - Can ditch the ComposedList here, but we need # something that paginate can use to figure out the paging. documents = ComposedList() documents.set_count(('results', searcher), num_results) results_per_page = settings.SEARCH_RESULTS_PER_PAGE pages = paginate(request, documents, results_per_page) # Facets product_facets = {} # If we know there aren't any results, let's cheat and in # doing that, not hit ES again. if num_results == 0: searcher = [] else: # Get the documents we want to show and add them to # docs_for_page documents = documents[offset:offset + results_per_page] if len(documents) == 0: # If the user requested a page that's beyond the # pagination, then documents is an empty list and # there are no results to show. searcher = [] else: bounds = documents[0][1] searcher = searcher.values_dict()[bounds[0]:bounds[1]] # If we are doing basic search, we show product facets. if a == '0': pfc = searcher.facet( 'product', filtered=True).facet_counts() product_facets = dict( [(p['term'], p['count']) for p in pfc['product']]) results = [] for i, doc in enumerate(searcher): rank = i + offset if doc['model'] == 'wiki_document': summary = _build_es_excerpt(doc) if not summary: summary = doc['document_summary'] result = { 'title': doc['document_title'], 'type': 'document'} elif doc['model'] == 'questions_question': summary = _build_es_excerpt(doc) if not summary: # We're excerpting only question_content, so if # the query matched question_title or # question_answer_content, then there won't be any # question_content excerpts. In that case, just # show the question--but only the first 500 # characters. summary = bleach.clean( doc['question_content'], strip=True)[:500] result = { 'title': doc['question_title'], 'type': 'question', 'is_solved': doc['question_is_solved'], 'num_answers': doc['question_num_answers'], 'num_votes': doc['question_num_votes'], 'num_votes_past_week': doc['question_num_votes_past_week']} else: summary = _build_es_excerpt(doc) result = { 'title': doc['post_title'], 'type': 'thread'} result['url'] = doc['url'] result['object'] = ObjectDict(doc) result['search_summary'] = summary result['rank'] = rank result['score'] = doc._score result['explanation'] = escape(format_explanation( doc._explanation)) results.append(result) except ES_EXCEPTIONS as exc: # Handle timeout and all those other transient errors with a # "Search Unavailable" rather than a Django error page. if is_json: return HttpResponse(json.dumps({'error': _('Search Unavailable')}), mimetype=mimetype, status=503) # Cheating here: Convert from 'Timeout()' to 'timeout' so # we have less code, but still have good stats. exc_bucket = repr(exc).lower().strip('()') statsd.incr('search.esunified.{0}'.format(exc_bucket)) import logging logging.exception(exc) t = 'search/mobile/down.html' if request.MOBILE else 'search/down.html' return render(request, t, {'q': cleaned['q']}, status=503) items = [(k, v) for k in search_form.fields for v in r.getlist(k) if v and k != 'a'] items.append(('a', '2')) if is_json: # Models are not json serializable. for r in results: del r['object'] data = {} data['results'] = results data['total'] = len(results) data['query'] = cleaned['q'] if not results: data['message'] = _('No pages matched the search criteria') json_data = json.dumps(data) if callback: json_data = callback + '(' + json_data + ');' return HttpResponse(json_data, mimetype=mimetype) fallback_results = None if num_results == 0: fallback_results = _fallback_results(language, cleaned['product']) results_ = render(request, template, { 'num_results': num_results, 'results': results, 'fallback_results': fallback_results, 'q': cleaned['q'], 'w': cleaned['w'], 'product': cleaned['product'], 'products': Product.objects.filter(visible=True), 'product_facets': product_facets, 'pages': pages, 'search_form': search_form, 'lang_name': lang_name, }) results_['Cache-Control'] = 'max-age=%s' % \ (settings.SEARCH_CACHE_PERIOD * 60) results_['Expires'] = (datetime.utcnow() + timedelta(minutes=settings.SEARCH_CACHE_PERIOD)) \ .strftime(expires_fmt) results_.set_cookie(settings.LAST_SEARCH_COOKIE, urlquote(cleaned['q']), max_age=3600, secure=False, httponly=False) return results_