def index_all():
""" Reindex all objects """
with virtualenv(), env.cd("mootiro_maps"):
setup_django()
from organization.models import Organization
from komoo_resource.models import Resource
from need.models import Need
from community.models import Community
from authentication.models import User
from komoo_project.models import Project
from search.utils import reset_index, create_mapping, refresh_index, index_object
model_list = [Organization, Resource, Need, Community, User, Project]
logging.info("Recreating index ... ")
reset_index()
logging.info("Create mappings ... ")
create_mapping()
logging.info("Indexing each object ... ")
for model in model_list:
for obj in model.objects.all():
index_object(obj)
logging.info("refreshing index ... ")
refresh_index()
logging.info("ES indexes rebuilt.")
def index_all():
""" Reindex all objects """
with virtualenv(), env.cd('mootiro_maps'):
setup_django()
from organization.models import Organization
from komoo_resource.models import Resource
from need.models import Need
from community.models import Community
from authentication.models import User
from komoo_project.models import Project
from search.utils import (reset_index, create_mapping, refresh_index,
index_object)
model_list = [Organization, Resource, Need, Community, User, Project]
logging.info('Recreating index ... ')
reset_index()
logging.info('Create mappings ... ')
create_mapping()
logging.info('Indexing each object ... ')
for model in model_list:
for obj in model.objects.all():
index_object(obj)
logging.info('refreshing index ... ')
refresh_index()
logging.info('ES indexes rebuilt.')
def test_strip_xml(self):
"""Indexed content_xml should have XML stripped."""
xml = "<tag>you're it</tag>"
self.resource.content_xml = xml
self.resource.save()
refresh_index()
self.assertTrue(self.count_results("you're it") == 1)
self.assertTrue(self.count_results("tag") == 0)
def test_strip_xml(self):
"""Indexed content_xml should have XML stripped."""
xml = "<tag>you're it</tag>"
self.resource.content_xml = xml
self.resource.save()
refresh_index()
self.assertTrue(self.count_results("you're it") == 1)
self.assertTrue(self.count_results("tag") == 0)
def tearDown(self):
"""Clean up Elasticsearch and static assets between tests."""
for static_asset in StaticAsset.objects.all():
default_storage.delete(static_asset.asset)
for key, _ in haystack.connections.connections_info.items():
haystack.connections.reload(key)
call_command('clear_index', interactive=False, verbosity=0)
clear_index()
refresh_index()
def update_index(**kwargs):
if 'instance' in kwargs:
doc = kwargs.get('instance')
else:
doc = kwargs.get('document')
# When a document is first created, the Document is saved
# before the Metadata and MetadataRevision are created.
# Then, the Document is saved again
# Thus, we MUST not index the document on the first save, since the
# metadata and revision does not exist yet
created = kwargs.pop('created', False)
if not created and doc.is_indexable:
index_document(doc.pk)
refresh_index()
def update_index(**kwargs):
if 'instance' in kwargs:
doc = kwargs.get('instance')
else:
doc = kwargs.get('document')
# When a document is first created, the Document is saved
# before the Metadata and MetadataRevision are created.
# Then, the Document is saved again
# Thus, we MUST not index the document on the first save, since the
# metadata and revision does not exist yet
created = kwargs.pop('created', False)
if not created and doc.is_indexable:
index_document(doc.pk)
refresh_index()
def test_index_vocabulary(self):
"""
Test that LearningResource indexes are updated when a
a term is added or removed.
"""
set_cache_timeout(0)
term = self.terms[0]
self.assertEqual(
self.count_faceted_results(self.vocabulary.id, term.id), 0)
self.resource.terms.add(term)
refresh_index()
self.assertEqual(
self.count_faceted_results(self.vocabulary.id, term.id), 1)
self.resource.terms.remove(term)
refresh_index()
self.assertEqual(
self.count_faceted_results(self.vocabulary.id, term.id), 0)
def test_index_vocabulary(self):
"""
Test that LearningResource indexes are updated when a
a term is added or removed.
"""
set_cache_timeout(0)
term = self.terms[0]
self.assertEqual(self.count_faceted_results(
self.vocabulary.id, term.id), 0)
self.resource.terms.add(term)
refresh_index()
self.assertEqual(self.count_faceted_results(
self.vocabulary.id, term.id), 1)
self.resource.terms.remove(term)
refresh_index()
self.assertEqual(self.count_faceted_results(
self.vocabulary.id, term.id), 0)
def setUp(self):
"""set up"""
super(LoreTestCase, self).setUp()
clear_index()
self.user = User.objects.create_user(
username=self.USERNAME, password=self.PASSWORD
)
add_repo_perm = Permission.objects.get(codename=self.ADD_REPO_PERM)
self.user.user_permissions.add(add_repo_perm)
# user without permission to add a repo
self.user_norepo = User.objects.create_user(
username=self.USERNAME_NO_REPO, password=self.PASSWORD
)
self.repo = create_repo(
name="test repo",
description="just a test",
user_id=self.user.id,
)
self.course = create_course(
org="test-org",
repo_id=self.repo.id,
course_number="infinity",
run="Febtober",
user_id=self.user.id,
)
self.resource = self.create_resource(
course=self.course,
parent=None,
resource_type="example",
title="silly example",
content_xml="<blah>blah</blah>",
mpath="/blah",
url_name="url_name1",
)
self.toy_resource_count = 18 # Resources in toy course.
self.toy_asset_count = 5 # Static assets in toy course.
self.client = Client()
self.login(username=self.USERNAME)
refresh_index()
def setUp(self):
"""set up"""
super(LoreTestCase, self).setUp()
recreate_index()
self.user = User.objects.create_user(username=self.USERNAME,
password=self.PASSWORD)
add_repo_perm = Permission.objects.get(codename=self.ADD_REPO_PERM)
self.user.user_permissions.add(add_repo_perm)
# user without permission to add a repo
self.user_norepo = User.objects.create_user(
username=self.USERNAME_NO_REPO, password=self.PASSWORD)
self.repo = create_repo(
name="test repo",
description="just a test",
user_id=self.user.id,
)
self.course = create_course(
org="test-org",
repo_id=self.repo.id,
course_number="infinity",
run="Febtober",
user_id=self.user.id,
)
self.resource = self.create_resource(
course=self.course,
parent=None,
resource_type="example",
title="silly example",
content_xml="<blah>blah</blah>",
mpath="/blah",
url_name="url_name1",
)
self.toy_resource_count = 18 # Resources in toy course.
self.toy_asset_count = 5 # Static assets in toy course.
self.client = Client()
self.login(username=self.USERNAME)
refresh_index()
def test_search_fields(self):
"""
The content_xml, title, and description should be searchable
by partial matches.
"""
new_resource = LearningResource.objects.create(
course=self.resource.course,
learning_resource_type=self.resource.learning_resource_type,
title="accordion music",
content_xml="<blah>foo</blah>",
materialized_path="/foo",
url_name="url_name2",
description="the quick brown fox",
description_path="silver",
)
new_resource.save() # to trigger indexing
refresh_index()
self.assertEqual(search_index("accordion").count(), 1)
self.assertEqual(search_index("fox brown").count(), 1)
self.assertEqual(search_index("foo").count(), 1)
def test_search_fields(self):
"""
The content_xml, title, and description should be searchable
by partial matches.
"""
new_resource = LearningResource.objects.create(
course=self.resource.course,
learning_resource_type=self.resource.learning_resource_type,
title="accordion music",
content_xml="<blah>foo</blah>",
materialized_path="/foo",
url_name="url_name2",
description="the quick brown fox",
description_path="silver",
)
new_resource.save() # to trigger indexing
refresh_index()
self.assertEqual(search_index("accordion").count(), 1)
self.assertEqual(search_index("fox brown").count(), 1)
self.assertEqual(search_index("foo").count(), 1)
def test_import(self):
"""
Indexing should occur on course import.
"""
results = search_index()
original_count = results.count()
import_course_from_file(self.course_zip, self.repo.id, self.user.id)
refresh_index()
results = search_index()
self.assertEqual(results.count(), original_count + 18)
self.assertEqual(results.page_count(), 2)
# Ensure the correct number of results are returned in each page.
self.assertEqual(len(results.get_page(1)), 10)
self.assertEqual(len(results.get_page(2)), 9)
self.assertEqual(len(results.get_page(3)), 0)
# Make sure the .all() function (a convenience function which
# returns a generator), returns all the records without explicit
# pagination.
count = 0
for _ in results.all():
count += 1
self.assertEqual(count, results.count())
def test_import(self):
"""
Indexing should occur on course import.
"""
results = search_index()
original_count = results.count()
import_course_from_file(self.course_zip, self.repo.id, self.user.id)
refresh_index()
results = search_index()
self.assertEqual(results.count(), original_count + 18)
self.assertEqual(results.page_count(), 2)
# Ensure the correct number of results are returned in each page.
self.assertEqual(len(results.get_page(1)), 10)
self.assertEqual(len(results.get_page(2)), 9)
self.assertEqual(len(results.get_page(3)), 0)
# Make sure the .all() function (a convenience function which
# returns a generator), returns all the records without explicit
# pagination.
count = 0
for _ in results.all():
count += 1
self.assertEqual(count, results.count())
def test_sorting(self):
"""Test sorting for search"""
# remove the default resource to control the environment
self.resource.delete()
# create some resources
res1 = self.create_resource(**dict(
resource_type="example",
title="silly example 1",
content_xml="<blah>blah 1</blah>",
mpath="/blah1",
xa_nr_views=1001,
xa_nr_attempts=99,
xa_avg_grade=9.9
))
res2 = self.create_resource(**dict(
resource_type="example",
title="silly example 2",
content_xml="<blah>blah 2</blah>",
mpath="/blah2",
xa_nr_views=1003,
xa_nr_attempts=98,
xa_avg_grade=6.8
))
res3 = self.create_resource(**dict(
resource_type="example",
title="silly example 3",
content_xml="<blah>blah 3</blah>",
mpath="/blah3",
xa_nr_views=1002,
xa_nr_attempts=101,
xa_avg_grade=7.3
))
res4 = self.create_resource(**dict(
resource_type="example",
title="silly example 4",
content_xml="<blah>blah 4</blah>",
mpath="/blah3",
xa_nr_views=1003,
xa_nr_attempts=101,
xa_avg_grade=9.9
))
index_resources([res1.id, res2.id, res3.id, res4.id])
refresh_index()
self.assertEqual(self.count_results(), 4)
# sorting by number of views
results = self.search(
query=None,
sorting="-{0}".format(LoreSortingFields.SORT_BY_NR_VIEWS[0])
)
# expected position res2, res4
top_res = results[0]
sec_res = results[1]
self.assertEqual(
top_res.id,
res2.id
)
self.assertEqual(
sec_res.id,
res4.id
)
# sorting by number of attempts
results = self.search(
query=None,
sorting="-{0}".format(LoreSortingFields.SORT_BY_NR_ATTEMPTS[0])
)
# expected position res3, res4
top_res = results[0]
sec_res = results[1]
self.assertEqual(
top_res.id,
res3.id
)
self.assertEqual(
sec_res.id,
res4.id
)
# sorting by average grade
results = self.search(
query=None,
sorting="-{0}".format(LoreSortingFields.SORT_BY_AVG_GRADE[0])
)
# expected position res1, res4
top_res = results[0]
sec_res = results[1]
self.assertEqual(
top_res.id,
res1.id
)
self.assertEqual(
sec_res.id,
res4.id
)
def remove_from_index(sender, instance, **kwargs):
unindex_document(instance.pk)
refresh_index()
def remove_from_index(sender, instance, **kwargs):
unindex_document(instance.pk)
refresh_index()
def test_sorting(self):
"""Test sorting for search"""
# remove the default resource to control the environment
self.resource.delete()
# create some resources
res1 = self.create_resource(**dict(resource_type="example",
title="silly example 1",
content_xml="<blah>blah 1</blah>",
mpath="/blah1",
xa_nr_views=1001,
xa_nr_attempts=99,
xa_avg_grade=9.9))
res2 = self.create_resource(**dict(resource_type="example",
title="silly example 2",
content_xml="<blah>blah 2</blah>",
mpath="/blah2",
xa_nr_views=1003,
xa_nr_attempts=98,
xa_avg_grade=6.8))
res3 = self.create_resource(**dict(resource_type="example",
title="silly example 3",
content_xml="<blah>blah 3</blah>",
mpath="/blah3",
xa_nr_views=1002,
xa_nr_attempts=101,
xa_avg_grade=7.3))
res4 = self.create_resource(**dict(resource_type="example",
title="silly example 4",
content_xml="<blah>blah 4</blah>",
mpath="/blah3",
xa_nr_views=1003,
xa_nr_attempts=101,
xa_avg_grade=9.9))
index_resources([res1.id, res2.id, res3.id, res4.id])
refresh_index()
self.assertEqual(self.count_results(), 4)
# sorting by number of views
results = self.search(query=None,
sorting="-{0}".format(
LoreSortingFields.SORT_BY_NR_VIEWS[0]))
# expected position res2, res4
top_res = results[0]
sec_res = results[1]
self.assertEqual(top_res.id, res2.id)
self.assertEqual(sec_res.id, res4.id)
# sorting by number of attempts
results = self.search(query=None,
sorting="-{0}".format(
LoreSortingFields.SORT_BY_NR_ATTEMPTS[0]))
# expected position res3, res4
top_res = results[0]
sec_res = results[1]
self.assertEqual(top_res.id, res3.id)
self.assertEqual(sec_res.id, res4.id)
# sorting by average grade
results = self.search(query=None,
sorting="-{0}".format(
LoreSortingFields.SORT_BY_AVG_GRADE[0]))
# expected position res1, res4
top_res = results[0]
sec_res = results[1]
self.assertEqual(top_res.id, res1.id)
self.assertEqual(sec_res.id, res4.id)