def elastic_search(request):
"""Use Elasticsearch for global search."""
user_input = UserInput(
query=request.GET.get('q'),
type=request.GET.get('type', 'project'),
project=request.GET.get('project'),
version=request.GET.get('version', LATEST),
taxonomy=request.GET.get('taxonomy'),
language=request.GET.get('language'),
)
results = ''
facets = {}
if user_input.query:
if user_input.type == 'project':
project_search = ProjectDocument.faceted_search(
query=user_input.query, language=user_input.language)
results = project_search.execute()
facets = results.facets
elif user_input.type == 'file':
kwargs = {}
if user_input.project:
projects_list = get_project_list_or_404(
project_slug=user_input.project, user=request.user)
project_slug_list = [project.slug for project in projects_list]
kwargs['projects_list'] = project_slug_list
if user_input.version:
kwargs['versions_list'] = user_input.version
page_search = PageDocument.faceted_search(query=user_input.query,
**kwargs)
results = page_search.execute()
facets = results.facets
if settings.DEBUG:
print(pprint(results))
print(pprint(facets))
if user_input.query:
user = ''
if request.user.is_authenticated():
user = request.user
log.info(
LOG_TEMPLATE.format(
user=user,
project=user_input.project or '',
type=user_input.type or '',
version=user_input.version or '',
language=user_input.language or '',
msg=user_input.query or '',
))
template_vars = user_input._asdict()
template_vars.update({'results': results, 'facets': facets})
return render(
request,
'search/elastic_search.html',
template_vars,
)
def test_facted_page_xss(self, client, project):
query = 'XSS'
page_search = PageDocument.faceted_search(query=query, user='')
results = page_search.execute()
expected = """
<h3><span>XSS</span> exploit</h3>
""".strip()
hits = results.hits.hits
assert len(hits) == 1 # there should be only one result
inner_hits = hits[0]['inner_hits']
domain_hits = inner_hits['domains']['hits']['hits']
assert len(
domain_hits) == 0 # there shouldn't be any results from domains
section_hits = inner_hits['sections']['hits']['hits']
assert len(section_hits) == 1
section_content_highlight = section_hits[0]['highlight'][
'sections.content']
assert len(section_content_highlight) == 1
assert expected in section_content_highlight[0]
def test_facted_page_xss(self, client, project):
query = 'XSS'
page_search = PageDocument.faceted_search(query=query, user='')
results = page_search.execute()
expected = """
<h3><em>XSS</em> exploit</h3>
""".strip()
assert results[0].meta.highlight.content[0][:len(expected)] == expected
def test_facted_page_xss(self, client, project):
query = 'XSS'
page_search = PageDocument.faceted_search(query=query, user='')
results = page_search.execute()
expected = """
<h3><em>XSS</em> exploit</h3>
""".strip()
assert results[0].meta.highlight.content[0][:len(expected)] == expected
def test_search_exact_match(self, client, project, case):
"""Check quoted query match exact phrase with case insensitively
Making a query with quoted text like ``"foo bar"`` should match
exactly ``foo bar`` or ``Foo Bar`` etc
"""
# `Github` word is present both in `kuma` and `pipeline` files
# But the phrase Github can is available only in kuma docs.
# So search with this phrase to check
query_text = r'"GitHub can"'
cased_query = getattr(query_text, case)
query = cased_query()
page_search = PageDocument.faceted_search(query=query, user='')
results = page_search.execute()
assert len(results) == 1
assert results[0]['project'] == 'kuma'
assert results[0]['path'] == 'documentation'
def test_search_exact_match(self, client, project, case):
"""Check quoted query match exact phrase with case insensitively
Making a query with quoted text like ``"foo bar"`` should match
exactly ``foo bar`` or ``Foo Bar`` etc
"""
# `Github` word is present both in `kuma` and `pipeline` files
# But the phrase Github can is available only in kuma docs.
# So search with this phrase to check
query_text = r'"GitHub can"'
cased_query = getattr(query_text, case)
query = cased_query()
page_search = PageDocument.faceted_search(query=query, user='')
results = page_search.execute()
assert len(results) == 1
assert results[0]['project'] == 'kuma'
assert results[0]['path'] == 'documentation'
def get_queryset(self):
"""
Return Elasticsearch DSL Search object instead of Django Queryset.
Django Queryset and elasticsearch-dsl ``Search`` object is similar pattern.
So for searching, its possible to return ``Search`` object instead of queryset.
The ``filter_backends`` and ``pagination_class`` is compatible with ``Search``
"""
# Validate all the required params are there
self.validate_query_params()
query = self.request.query_params.get('q', '')
kwargs = {'filter_by_user': False}
kwargs['projects_list'] = [p.slug for p in self.get_all_projects()]
kwargs['versions_list'] = self.request.query_params.get('version')
user = self.request.user
queryset = PageDocument.faceted_search(query=query,
user=user,
**kwargs)
return queryset
def test_search_combined_result(self, client, project):
"""Check search result are combined of both `AND` and `OR` operator
If query is `Foo Bar` then the result should be as following order:
- Where both `Foo Bar` is present
- Where `Foo` or `Bar` is present
"""
query = 'Official Support'
page_search = PageDocument.faceted_search(query=query, user='')
results = page_search.execute()
assert len(results) == 3
result_paths = [r.path for r in results]
# ``open-source-philosophy`` page has both ``Official Support`` words
# ``docker`` page has ``Support`` word
# ``installation`` page has ``Official`` word
expected_paths = ['open-source-philosophy', 'docker', 'installation']
assert result_paths == expected_paths
def test_search_combined_result(self, client, project):
"""Check search result are combined of both `AND` and `OR` operator
If query is `Foo Bar` then the result should be as following order:
- Where both `Foo Bar` is present
- Where `Foo` or `Bar` is present
"""
query = 'Official Support'
page_search = PageDocument.faceted_search(query=query, user='')
results = page_search.execute()
assert len(results) == 3
result_paths = [r.path for r in results]
# ``open-source-philosophy`` page has both ``Official Support`` words
# ``docker`` page has ``Support`` word
# ``installation`` page has ``Official`` word
expected_paths = ['open-source-philosophy', 'docker', 'installation']
assert result_paths == expected_paths
def test_search_combined_result(self, client, project):
"""Check search result are combined of both `AND` and `OR` operator
If query is `Foo Bar` then the result should be as following order:
- Where both `Foo Bar` is present
- Where `Foo` or `Bar` is present
"""
query = 'Elasticsearch Query'
page_search = PageDocument.faceted_search(query=query, user='')
results = page_search.execute()
assert len(results) == 3
result_paths = [r.path for r in results]
# ``guides/wipe-environment`` page has both ``Elasticsearch Query`` words
# ``docker`` page has ``Elasticsearch`` word
# ``installation`` page has ``Query`` word.
expected_paths = ['guides/wipe-environment', 'docker', 'installation']
assert result_paths == expected_paths
def elastic_project_search(request, project_slug):
"""Use elastic search to search in a project."""
queryset = Project.objects.protected(request.user)
project = get_object_or_404(queryset, slug=project_slug)
version_slug = request.GET.get('version', LATEST)
query = request.GET.get('q', None)
results = None
if query:
kwargs = {}
kwargs['projects_list'] = [project.slug]
kwargs['versions_list'] = version_slug
page_search = PageDocument.faceted_search(query=query,
user=request.user,
**kwargs)
results = page_search.execute()
log.debug('Search results: %s', pformat(results.to_dict()))
log.debug('Search facets: %s', pformat(results.facets.to_dict()))
log.info(
LOG_TEMPLATE.format(
user=request.user,
project=project or '',
type='inproject',
version=version_slug or '',
language='',
msg=query or '',
), )
return render(
request,
'search/elastic_project_search.html',
{
'project': project,
'query': query,
'results': results,
},
)
def elastic_search(request):
"""Use Elasticsearch for global search."""
user_input = UserInput(
query=request.GET.get('q'),
type=request.GET.get('type', 'project'),
project=request.GET.get('project'),
version=request.GET.get('version', LATEST),
taxonomy=request.GET.get('taxonomy'),
language=request.GET.get('language'),
)
results = ''
facets = {}
if user_input.query:
if user_input.type == 'project':
project_search = ProjectDocument.faceted_search(
query=user_input.query,
user=request.user,
language=user_input.language)
results = project_search.execute()
facets = results.facets
elif user_input.type == 'file':
kwargs = {}
if user_input.project:
kwargs['projects_list'] = [user_input.project]
if user_input.version:
kwargs['versions_list'] = [user_input.version]
page_search = PageDocument.faceted_search(query=user_input.query,
user=request.user,
**kwargs)
results = page_search.execute()
facets = results.facets
log.info(
LOG_TEMPLATE.format(
user=request.user,
project=user_input.project or '',
type=user_input.type or '',
version=user_input.version or '',
language=user_input.language or '',
msg=user_input.query or '',
), )
if results:
if user_input.type == 'file':
# Change results to turn newlines in highlight into periods
# https://github.com/rtfd/readthedocs.org/issues/5168
for result in results:
if hasattr(result.meta.highlight, 'content'):
result.meta.highlight.content = [
result.replace('\n', '. ')
for result in result.meta.highlight.content
]
log.debug('Search results: %s', pformat(results.to_dict()))
log.debug('Search facets: %s', pformat(results.facets.to_dict()))
template_vars = user_input._asdict()
template_vars.update({'results': results, 'facets': facets})
return render(
request,
'search/elastic_search.html',
template_vars,
)