def test_recent_helpful_votes(self):
"""Recent helpful votes are indexed properly."""
# Create a document and verify it doesn't show up in a
# query for recent_helpful_votes__gt=0.
r = revision(is_approved=True, save=True)
self.refresh()
eq_(DocumentMappingType.search().filter(
document_recent_helpful_votes__gt=0).count(), 0)
# Add an unhelpful vote, it still shouldn't show up.
helpful_vote(revision=r, helpful=False, save=True)
r.document.save() # Votes don't trigger a reindex.
self.refresh()
eq_(DocumentMappingType.search().filter(
document_recent_helpful_votes__gt=0).count(), 0)
# Add an helpful vote created 31 days ago, it still shouldn't show up.
created = datetime.now() - timedelta(days=31)
helpful_vote(revision=r, helpful=True, created=created, save=True)
r.document.save() # Votes don't trigger a reindex.
self.refresh()
eq_(DocumentMappingType.search().filter(
document_recent_helpful_votes__gt=0).count(), 0)
# Add an helpful vote created 29 days ago, it should show up now.
created = datetime.now() - timedelta(days=29)
helpful_vote(revision=r, helpful=True, created=created, save=True)
r.document.save() # Votes don't trigger a reindex.
self.refresh()
eq_(DocumentMappingType.search().filter(
document_recent_helpful_votes__gt=0).count(), 1)
def test_recent_helpful_votes(self):
"""Recent helpful votes are indexed properly."""
# Create a document and verify it doesn't show up in a
# query for recent_helpful_votes__gt=0.
r = revision(is_approved=True, save=True)
self.refresh()
eq_(
DocumentMappingType.search().filter(
document_recent_helpful_votes__gt=0).count(), 0)
# Add an unhelpful vote, it still shouldn't show up.
helpful_vote(revision=r, helpful=False, save=True)
r.document.save() # Votes don't trigger a reindex.
self.refresh()
eq_(
DocumentMappingType.search().filter(
document_recent_helpful_votes__gt=0).count(), 0)
# Add an helpful vote created 31 days ago, it still shouldn't show up.
created = datetime.now() - timedelta(days=31)
helpful_vote(revision=r, helpful=True, created=created, save=True)
r.document.save() # Votes don't trigger a reindex.
self.refresh()
eq_(
DocumentMappingType.search().filter(
document_recent_helpful_votes__gt=0).count(), 0)
# Add an helpful vote created 29 days ago, it should show up now.
created = datetime.now() - timedelta(days=29)
helpful_vote(revision=r, helpful=True, created=created, save=True)
r.document.save() # Votes don't trigger a reindex.
self.refresh()
eq_(
DocumentMappingType.search().filter(
document_recent_helpful_votes__gt=0).count(), 1)
def test_add_and_delete(self):
"""Adding a doc should add it to the search index; deleting should
delete it."""
doc = document(save=True)
revision(document=doc, is_approved=True, save=True)
self.refresh()
eq_(DocumentMappingType.search().count(), 1)
doc.delete()
self.refresh()
eq_(DocumentMappingType.search().count(), 0)
def test_add_and_delete(self):
"""Adding a doc should add it to the search index; deleting should
delete it."""
doc = document(save=True)
revision(document=doc, is_approved=True, save=True)
self.refresh()
eq_(DocumentMappingType.search().count(), 1)
doc.delete()
self.refresh()
eq_(DocumentMappingType.search().count(), 0)
def test_wiki_no_revisions(self):
"""Don't index documents without approved revisions"""
# Create a document with no revisions and make sure the
# document is not in the index.
doc = document(save=True)
self.refresh()
eq_(DocumentMappingType.search().count(), 0)
# Create a revision that's not approved and make sure the
# document is still not in the index.
revision(document=doc, is_approved=False, save=True)
self.refresh()
eq_(DocumentMappingType.search().count(), 0)
def test_wiki_no_revisions(self):
"""Don't index documents without approved revisions"""
# Create a document with no revisions and make sure the
# document is not in the index.
doc = document(save=True)
self.refresh()
eq_(DocumentMappingType.search().count(), 0)
# Create a revision that's not approved and make sure the
# document is still not in the index.
revision(document=doc, is_approved=False, save=True)
self.refresh()
eq_(DocumentMappingType.search().count(), 0)
def test_wiki_redirects(self):
"""Make sure we don't index redirects"""
# First create a revision that doesn't have a redirect and
# make sure it's in the index.
doc = DocumentFactory(title=u'wool hats')
RevisionFactory(document=doc, is_approved=True)
self.refresh()
eq_(DocumentMappingType.search().query(document_title__match='wool').count(), 1)
# Now create a revision that is a redirect and make sure the
# document is removed from the index.
RedirectRevisionFactory(document=doc)
self.refresh()
eq_(DocumentMappingType.search().query(document_title__match='wool').count(), 0)
def test_wiki_keywords(self):
"""Make sure updating keywords updates the index."""
# Create a document with a revision with no keywords. It
# shouldn't show up with a document_keywords term query for
# 'wool' since it has no keywords.
doc = DocumentFactory(title=u'wool hats')
RevisionFactory(document=doc, is_approved=True)
self.refresh()
eq_(DocumentMappingType.search().query(
document_keywords='wool').count(), 0)
RevisionFactory(document=doc, is_approved=True, keywords='wool')
self.refresh()
eq_(DocumentMappingType.search().query(document_keywords='wool').count(), 1)
def test_wiki_keywords(self):
"""Make sure updating keywords updates the index."""
# Create a document with a revision with no keywords. It
# shouldn't show up with a document_keywords term query for
# 'wool' since it has no keywords.
doc = DocumentFactory(title='wool hats')
RevisionFactory(document=doc, is_approved=True)
self.refresh()
eq_(
DocumentMappingType.search().query(
document_keywords='wool').count(), 0)
RevisionFactory(document=doc, is_approved=True, keywords='wool')
self.refresh()
eq_(
DocumentMappingType.search().query(
document_keywords='wool').count(), 1)
def test_wiki_redirects(self):
"""Make sure we don't index redirects"""
# First create a revision that doesn't have a redirect and
# make sure it's in the index.
doc = DocumentFactory(title='wool hats')
RevisionFactory(document=doc, is_approved=True)
self.refresh()
eq_(
DocumentMappingType.search().query(
document_title__match='wool').count(), 1)
# Now create a revision that is a redirect and make sure the
# document is removed from the index.
RedirectRevisionFactory(document=doc)
self.refresh()
eq_(
DocumentMappingType.search().query(
document_title__match='wool').count(), 0)
def test_wiki_redirects(self):
"""Make sure we don't index redirects"""
# First create a revision that doesn't have a redirect and
# make sure it's in the index.
doc = document(title=u'wool hats')
doc.save()
revision(document=doc, is_approved=True, save=True)
self.refresh()
eq_(DocumentMappingType.search().query(
document_title__match='wool').count(), 1)
# Now create a revision that is a redirect and make sure the
# document is removed from the index.
revision(document=doc, content=REDIRECT_CONTENT, is_approved=True,
save=True)
self.refresh()
eq_(DocumentMappingType.search().query(
document_title__match='wool').count(), 0)
def troubleshooting_view(request):
# Build a list of the most recently indexed 50 wiki documents.
last_50_indexed = list(_fix_results(DocumentMappingType.search().order_by("-indexed_on")[:50]))
last_50_reviewed = list(
Document.objects.filter(current_revision__is_approved=True).order_by("-current_revision__reviewed")[:50]
)
diff_list = diff_it_for_realz(last_50_indexed, last_50_reviewed)
return render(request, "admin/search_troubleshooting.html", {"title": "Index Troubleshooting", "diffs": diff_list})
def test_wiki_topics(self):
"""Make sure that adding topics to a Document causes it to
refresh the index.
"""
t = topic(slug=u'hiphop', save=True)
eq_(DocumentMappingType.search().filter(topic=t.slug).count(), 0)
doc = document(save=True)
revision(document=doc, is_approved=True, save=True)
self.refresh()
eq_(DocumentMappingType.search().filter(topic=t.slug).count(), 0)
doc.topics.add(t)
self.refresh()
eq_(DocumentMappingType.search().filter(topic=t.slug).count(), 1)
doc.topics.clear()
self.refresh()
# Make sure the document itself is still there and that we didn't
# accidentally delete it through screwed up signal handling:
eq_(DocumentMappingType.search().filter().count(), 1)
eq_(DocumentMappingType.search().filter(topic=t.slug).count(), 0)
def test_wiki_products(self):
"""Make sure that adding products to a Document causes it to
refresh the index.
"""
p = product(slug=u'desktop', save=True)
eq_(DocumentMappingType.search().filter(product=p.slug).count(), 0)
doc = document(save=True)
revision(document=doc, is_approved=True, save=True)
self.refresh()
eq_(DocumentMappingType.search().filter(product=p.slug).count(), 0)
doc.products.add(p)
self.refresh()
eq_(DocumentMappingType.search().filter(product=p.slug).count(), 1)
doc.products.remove(p)
self.refresh()
# Make sure the document itself is still there and that we didn't
# accidentally delete it through screwed up signal handling:
eq_(DocumentMappingType.search().filter().count(), 1)
eq_(DocumentMappingType.search().filter(product=p.slug).count(), 0)
def test_wiki_products(self):
"""Make sure that adding products to a Document causes it to
refresh the index.
"""
p = product(slug=u'desktop', save=True)
eq_(DocumentMappingType.search().filter(product=p.slug).count(), 0)
doc = document(save=True)
revision(document=doc, is_approved=True, save=True)
self.refresh()
eq_(DocumentMappingType.search().filter(product=p.slug).count(), 0)
doc.products.add(p)
self.refresh()
eq_(DocumentMappingType.search().filter(product=p.slug).count(), 1)
doc.products.remove(p)
self.refresh()
# Make sure the document itself is still there and that we didn't
# accidentally delete it through screwed up signal handling:
eq_(DocumentMappingType.search().filter().count(), 1)
eq_(DocumentMappingType.search().filter(product=p.slug).count(), 0)
def test_wiki_topics(self):
"""Make sure that adding topics to a Document causes it to
refresh the index.
"""
t = topic(slug=u'hiphop', save=True)
eq_(DocumentMappingType.search().filter(topic=t.slug).count(), 0)
doc = document(save=True)
revision(document=doc, is_approved=True, save=True)
self.refresh()
eq_(DocumentMappingType.search().filter(topic=t.slug).count(), 0)
doc.topics.add(t)
self.refresh()
eq_(DocumentMappingType.search().filter(topic=t.slug).count(), 1)
doc.topics.clear()
self.refresh()
# Make sure the document itself is still there and that we didn't
# accidentally delete it through screwed up signal handling:
eq_(DocumentMappingType.search().filter().count(), 1)
eq_(DocumentMappingType.search().filter(topic=t.slug).count(), 0)
def test_wiki_redirects(self):
"""Make sure we don't index redirects"""
# First create a revision that doesn't have a redirect and
# make sure it's in the index.
doc = document(title=u'wool hats')
doc.save()
revision(document=doc, is_approved=True, save=True)
self.refresh()
eq_(
DocumentMappingType.search().query(
document_title__match='wool').count(), 1)
# Now create a revision that is a redirect and make sure the
# document is removed from the index.
revision(document=doc,
content=REDIRECT_CONTENT,
is_approved=True,
save=True)
self.refresh()
eq_(
DocumentMappingType.search().query(
document_title__match='wool').count(), 0)
def opensearch_suggestions(request):
"""A simple search view that returns OpenSearch suggestions."""
content_type = "application/x-suggestions+json"
term = request.GET.get("q")
if not term:
return HttpResponseBadRequest(content_type=content_type)
locale = locale_or_default(request.LANGUAGE_CODE)
# FIXME: Rewrite this using the simple search search business
# logic. This currently returns templates (amongst other things)
# which is totally wrong.
try:
query = dict(("%s__match" % field, term) for field in DocumentMappingType.get_query_fields())
# Upgrade the query to an analyzer-aware one.
query = es_utils.es_query_with_analyzer(query, locale)
wiki_s = (
DocumentMappingType.search()
.filter(document_is_archived=False)
.filter(document_locale=locale)
.values_dict("document_title", "url")
.query(or_=query)[:5]
)
query = dict(("%s__match" % field, term) for field in QuestionMappingType.get_query_fields())
question_s = (
QuestionMappingType.search()
.filter(question_has_helpful=True)
.values_dict("question_title", "url")
.query(or_=query)[:5]
)
results = list(chain(question_s, wiki_s))
except ES_EXCEPTIONS:
# If we have ES problems, we just send back an empty result
# set.
results = []
def urlize(r):
return u"%s://%s%s" % ("https" if request.is_secure() else "http", request.get_host(), r["url"][0])
def titleize(r):
# NB: Elasticsearch returns an array of strings as the value,
# so we mimic that and then pull out the first (and only)
# string.
return r.get("document_title", r.get("question_title", [_("No title")]))[0]
data = [term, [titleize(r) for r in results], [], [urlize(r) for r in results]]
return HttpResponse(json.dumps(data), content_type=content_type)
def opensearch_suggestions(request):
"""A simple search view that returns OpenSearch suggestions."""
content_type = 'application/x-suggestions+json'
term = request.GET.get('q')
if not term:
return HttpResponseBadRequest(content_type=content_type)
locale = locale_or_default(request.LANGUAGE_CODE)
# FIXME: Rewrite this using the simple search search business
# logic. This currently returns templates (amongst other things)
# which is totally wrong.
try:
query = dict(('%s__match' % field, term)
for field in DocumentMappingType.get_query_fields())
# Upgrade the query to an analyzer-aware one.
query = es_utils.es_query_with_analyzer(query, locale)
wiki_s = (DocumentMappingType.search().filter(
document_is_archived=False).filter(
document_locale=locale).values_dict(
'document_title', 'url').query(or_=query)[:5])
query = dict(('%s__match' % field, term)
for field in QuestionMappingType.get_query_fields())
question_s = (QuestionMappingType.search().filter(
question_has_helpful=True).values_dict('question_title',
'url').query(or_=query)[:5])
results = list(chain(question_s, wiki_s))
except ES_EXCEPTIONS:
# If we have ES problems, we just send back an empty result
# set.
results = []
def urlize(r):
return u'%s://%s%s' % ('https' if request.is_secure() else 'http',
request.get_host(), r['url'][0])
def titleize(r):
# NB: Elasticsearch returns an array of strings as the value,
# so we mimic that and then pull out the first (and only)
# string.
return r.get('document_title', r.get('question_title',
[_('No title')]))[0]
data = [
term, [titleize(r) for r in results], [], [urlize(r) for r in results]
]
return HttpResponse(json.dumps(data), content_type=content_type)
def _es_documents_for(locale, topics=None, products=None):
"""ES implementation of documents_for."""
s = (DocumentMappingType.search()
.values_dict('id', 'document_title', 'url', 'document_parent_id',
'document_summary')
.filter(document_locale=locale, document_is_archived=False,
document_category__in=settings.IA_DEFAULT_CATEGORIES))
for topic in topics or []:
s = s.filter(topic=topic.slug)
for product in products or []:
s = s.filter(product=product.slug)
return list(s.order_by('-document_recent_helpful_votes')[:100])
def troubleshooting_view(request):
# Build a list of the most recently indexed 50 wiki documents.
last_50_indexed = list(
_fix_value_dicts(DocumentMappingType.search().values_dict().order_by(
'-indexed_on')[:50]))
last_50_reviewed = list(
Document.uncached.filter(current_revision__is_approved=True).order_by(
'-current_revision__reviewed')[:50])
diff_list = diff_it_for_realz(last_50_indexed, last_50_reviewed)
return render(request, 'admin/search_troubleshooting.html', {
'title': 'Index Troubleshooting',
'diffs': diff_list,
})
def suggestions(request):
"""A simple search view that returns OpenSearch suggestions."""
content_type = 'application/x-suggestions+json'
term = request.GET.get('q')
if not term:
return HttpResponseBadRequest(content_type=content_type)
site = Site.objects.get_current()
locale = locale_or_default(request.LANGUAGE_CODE)
try:
query = dict(('%s__match' % field, term)
for field in DocumentMappingType.get_query_fields())
# Upgrade the query to an analyzer-aware one.
query = es_utils.es_query_with_analyzer(query, locale)
wiki_s = (DocumentMappingType.search()
.filter(document_is_archived=False)
.filter(document_locale=locale)
.values_dict('document_title', 'url')
.query(or_=query)[:5])
query = dict(('%s__match' % field, term)
for field in QuestionMappingType.get_query_fields())
question_s = (QuestionMappingType.search()
.filter(question_has_helpful=True)
.values_dict('question_title', 'url')
.query(or_=query)[:5])
results = list(chain(question_s, wiki_s))
except ES_EXCEPTIONS:
# If we have ES problems, we just send back an empty result
# set.
results = []
def urlize(r):
return u'https://%s%s' % (site, r['url'])
def titleize(r):
return r.get('document_title', r.get('document_title'))
data = [term,
[titleize(r) for r in results],
[],
[urlize(r) for r in results]]
return HttpResponse(json.dumps(data), content_type=content_type)
def suggestions(request):
"""A simple search view that returns OpenSearch suggestions."""
content_type = 'application/x-suggestions+json'
term = request.GET.get('q')
if not term:
return HttpResponseBadRequest(content_type=content_type)
site = Site.objects.get_current()
locale = locale_or_default(request.LANGUAGE_CODE)
try:
query = dict(('{0!s}__match'.format(field), term)
for field in DocumentMappingType.get_query_fields())
# Upgrade the query to an analyzer-aware one.
query = es_utils.es_query_with_analyzer(query, locale)
wiki_s = (DocumentMappingType.search()
.filter(document_is_archived=False)
.filter(document_locale=locale)
.values_dict('document_title', 'url')
.query(or_=query)[:5])
query = dict(('{0!s}__match'.format(field), term)
for field in QuestionMappingType.get_query_fields())
question_s = (QuestionMappingType.search()
.filter(question_has_helpful=True)
.values_dict('question_title', 'url')
.query(or_=query)[:5])
results = list(chain(question_s, wiki_s))
except ES_EXCEPTIONS:
# If we have ES problems, we just send back an empty result
# set.
results = []
def urlize(r):
return u'https://{0!s}{1!s}'.format(site, r['url'])
def titleize(r):
return r.get('document_title', r.get('document_title'))
data = [term,
[titleize(r) for r in results],
[],
[urlize(r) for r in results]]
return HttpResponse(json.dumps(data), content_type=content_type)
def troubleshooting_view(request):
# Build a list of the most recently indexed 50 wiki documents.
last_50_indexed = list(_fix_results(DocumentMappingType.search()
.order_by('-indexed_on')[:50]))
last_50_reviewed = list(Document.uncached
.filter(current_revision__is_approved=True)
.order_by('-current_revision__reviewed')[:50])
diff_list = diff_it_for_realz(last_50_indexed, last_50_reviewed)
return render(
request,
'admin/search_troubleshooting.html',
{'title': 'Index Troubleshooting',
'diffs': diff_list,
})
def _es_documents_for(locale, topics=None, products=None):
"""ES implementation of documents_for."""
s = (DocumentMappingType.search().values_dict(
"id", "document_title", "url", "document_parent_id",
"document_summary").filter(
document_locale=locale,
document_is_archived=False,
document_category__in=settings.IA_DEFAULT_CATEGORIES,
))
for topic in topics or []:
s = s.filter(topic=topic.slug)
for product in products or []:
s = s.filter(product=product.slug)
results = s.order_by("document_display_order",
"-document_recent_helpful_votes")[:100]
results = DocumentMappingType.reshape(results)
return results
def suggestions(request):
"""A simple search view that returns OpenSearch suggestions."""
mimetype = 'application/x-suggestions+json'
term = request.GET.get('q')
if not term:
return HttpResponseBadRequest(mimetype=mimetype)
site = Site.objects.get_current()
locale = locale_or_default(request.LANGUAGE_CODE)
try:
query = dict(('%s__text' % field, term)
for field in DocumentMappingType.get_query_fields())
wiki_s = (DocumentMappingType.search()
.filter(document_is_archived=False)
.filter(document_locale=locale)
.values_dict('document_title', 'url')
.query(or_=query)[:5])
query = dict(('%s__text' % field, term)
for field in QuestionMappingType.get_query_fields())
question_s = (QuestionMappingType.search()
.filter(question_has_helpful=True)
.values_dict('question_title', 'url')
.query(or_=query)[:5])
results = list(chain(question_s, wiki_s))
except ES_EXCEPTIONS:
# If we have ES problems, we just send back an empty result
# set.
results = []
urlize = lambda r: u'https://%s%s' % (site, r['url'])
titleize = lambda r: (r['document_title'] if 'document_title' in r
else r['question_title'])
data = [term,
[titleize(r) for r in results],
[],
[urlize(r) for r in results]]
return HttpResponse(json.dumps(data), mimetype=mimetype)
