def index_all(): """ Reindex all objects """ with virtualenv(), env.cd("mootiro_maps"): setup_django() from organization.models import Organization from komoo_resource.models import Resource from need.models import Need from community.models import Community from authentication.models import User from komoo_project.models import Project from search.utils import reset_index, create_mapping, refresh_index, index_object model_list = [Organization, Resource, Need, Community, User, Project] logging.info("Recreating index ... ") reset_index() logging.info("Create mappings ... ") create_mapping() logging.info("Indexing each object ... ") for model in model_list: for obj in model.objects.all(): index_object(obj) logging.info("refreshing index ... ") refresh_index() logging.info("ES indexes rebuilt.")
def index_all(): """ Reindex all objects """ with virtualenv(), env.cd('mootiro_maps'): setup_django() from organization.models import Organization from komoo_resource.models import Resource from need.models import Need from community.models import Community from authentication.models import User from komoo_project.models import Project from search.utils import (reset_index, create_mapping, refresh_index, index_object) model_list = [Organization, Resource, Need, Community, User, Project] logging.info('Recreating index ... ') reset_index() logging.info('Create mappings ... ') create_mapping() logging.info('Indexing each object ... ') for model in model_list: for obj in model.objects.all(): index_object(obj) logging.info('refreshing index ... ') refresh_index() logging.info('ES indexes rebuilt.')
def test_strip_xml(self): """Indexed content_xml should have XML stripped.""" xml = "<tag>you're it</tag>" self.resource.content_xml = xml self.resource.save() refresh_index() self.assertTrue(self.count_results("you're it") == 1) self.assertTrue(self.count_results("tag") == 0)
def tearDown(self): """Clean up Elasticsearch and static assets between tests.""" for static_asset in StaticAsset.objects.all(): default_storage.delete(static_asset.asset) for key, _ in haystack.connections.connections_info.items(): haystack.connections.reload(key) call_command('clear_index', interactive=False, verbosity=0) clear_index() refresh_index()
def update_index(**kwargs): if 'instance' in kwargs: doc = kwargs.get('instance') else: doc = kwargs.get('document') # When a document is first created, the Document is saved # before the Metadata and MetadataRevision are created. # Then, the Document is saved again # Thus, we MUST not index the document on the first save, since the # metadata and revision does not exist yet created = kwargs.pop('created', False) if not created and doc.is_indexable: index_document(doc.pk) refresh_index()
def test_index_vocabulary(self): """ Test that LearningResource indexes are updated when a a term is added or removed. """ set_cache_timeout(0) term = self.terms[0] self.assertEqual( self.count_faceted_results(self.vocabulary.id, term.id), 0) self.resource.terms.add(term) refresh_index() self.assertEqual( self.count_faceted_results(self.vocabulary.id, term.id), 1) self.resource.terms.remove(term) refresh_index() self.assertEqual( self.count_faceted_results(self.vocabulary.id, term.id), 0)
def test_index_vocabulary(self): """ Test that LearningResource indexes are updated when a a term is added or removed. """ set_cache_timeout(0) term = self.terms[0] self.assertEqual(self.count_faceted_results( self.vocabulary.id, term.id), 0) self.resource.terms.add(term) refresh_index() self.assertEqual(self.count_faceted_results( self.vocabulary.id, term.id), 1) self.resource.terms.remove(term) refresh_index() self.assertEqual(self.count_faceted_results( self.vocabulary.id, term.id), 0)
def setUp(self): """set up""" super(LoreTestCase, self).setUp() clear_index() self.user = User.objects.create_user( username=self.USERNAME, password=self.PASSWORD ) add_repo_perm = Permission.objects.get(codename=self.ADD_REPO_PERM) self.user.user_permissions.add(add_repo_perm) # user without permission to add a repo self.user_norepo = User.objects.create_user( username=self.USERNAME_NO_REPO, password=self.PASSWORD ) self.repo = create_repo( name="test repo", description="just a test", user_id=self.user.id, ) self.course = create_course( org="test-org", repo_id=self.repo.id, course_number="infinity", run="Febtober", user_id=self.user.id, ) self.resource = self.create_resource( course=self.course, parent=None, resource_type="example", title="silly example", content_xml="<blah>blah</blah>", mpath="/blah", url_name="url_name1", ) self.toy_resource_count = 18 # Resources in toy course. self.toy_asset_count = 5 # Static assets in toy course. self.client = Client() self.login(username=self.USERNAME) refresh_index()
def setUp(self): """set up""" super(LoreTestCase, self).setUp() recreate_index() self.user = User.objects.create_user(username=self.USERNAME, password=self.PASSWORD) add_repo_perm = Permission.objects.get(codename=self.ADD_REPO_PERM) self.user.user_permissions.add(add_repo_perm) # user without permission to add a repo self.user_norepo = User.objects.create_user( username=self.USERNAME_NO_REPO, password=self.PASSWORD) self.repo = create_repo( name="test repo", description="just a test", user_id=self.user.id, ) self.course = create_course( org="test-org", repo_id=self.repo.id, course_number="infinity", run="Febtober", user_id=self.user.id, ) self.resource = self.create_resource( course=self.course, parent=None, resource_type="example", title="silly example", content_xml="<blah>blah</blah>", mpath="/blah", url_name="url_name1", ) self.toy_resource_count = 18 # Resources in toy course. self.toy_asset_count = 5 # Static assets in toy course. self.client = Client() self.login(username=self.USERNAME) refresh_index()
def test_search_fields(self): """ The content_xml, title, and description should be searchable by partial matches. """ new_resource = LearningResource.objects.create( course=self.resource.course, learning_resource_type=self.resource.learning_resource_type, title="accordion music", content_xml="<blah>foo</blah>", materialized_path="/foo", url_name="url_name2", description="the quick brown fox", description_path="silver", ) new_resource.save() # to trigger indexing refresh_index() self.assertEqual(search_index("accordion").count(), 1) self.assertEqual(search_index("fox brown").count(), 1) self.assertEqual(search_index("foo").count(), 1)
def test_import(self): """ Indexing should occur on course import. """ results = search_index() original_count = results.count() import_course_from_file(self.course_zip, self.repo.id, self.user.id) refresh_index() results = search_index() self.assertEqual(results.count(), original_count + 18) self.assertEqual(results.page_count(), 2) # Ensure the correct number of results are returned in each page. self.assertEqual(len(results.get_page(1)), 10) self.assertEqual(len(results.get_page(2)), 9) self.assertEqual(len(results.get_page(3)), 0) # Make sure the .all() function (a convenience function which # returns a generator), returns all the records without explicit # pagination. count = 0 for _ in results.all(): count += 1 self.assertEqual(count, results.count())
def test_sorting(self): """Test sorting for search""" # remove the default resource to control the environment self.resource.delete() # create some resources res1 = self.create_resource(**dict( resource_type="example", title="silly example 1", content_xml="<blah>blah 1</blah>", mpath="/blah1", xa_nr_views=1001, xa_nr_attempts=99, xa_avg_grade=9.9 )) res2 = self.create_resource(**dict( resource_type="example", title="silly example 2", content_xml="<blah>blah 2</blah>", mpath="/blah2", xa_nr_views=1003, xa_nr_attempts=98, xa_avg_grade=6.8 )) res3 = self.create_resource(**dict( resource_type="example", title="silly example 3", content_xml="<blah>blah 3</blah>", mpath="/blah3", xa_nr_views=1002, xa_nr_attempts=101, xa_avg_grade=7.3 )) res4 = self.create_resource(**dict( resource_type="example", title="silly example 4", content_xml="<blah>blah 4</blah>", mpath="/blah3", xa_nr_views=1003, xa_nr_attempts=101, xa_avg_grade=9.9 )) index_resources([res1.id, res2.id, res3.id, res4.id]) refresh_index() self.assertEqual(self.count_results(), 4) # sorting by number of views results = self.search( query=None, sorting="-{0}".format(LoreSortingFields.SORT_BY_NR_VIEWS[0]) ) # expected position res2, res4 top_res = results[0] sec_res = results[1] self.assertEqual( top_res.id, res2.id ) self.assertEqual( sec_res.id, res4.id ) # sorting by number of attempts results = self.search( query=None, sorting="-{0}".format(LoreSortingFields.SORT_BY_NR_ATTEMPTS[0]) ) # expected position res3, res4 top_res = results[0] sec_res = results[1] self.assertEqual( top_res.id, res3.id ) self.assertEqual( sec_res.id, res4.id ) # sorting by average grade results = self.search( query=None, sorting="-{0}".format(LoreSortingFields.SORT_BY_AVG_GRADE[0]) ) # expected position res1, res4 top_res = results[0] sec_res = results[1] self.assertEqual( top_res.id, res1.id ) self.assertEqual( sec_res.id, res4.id )
def remove_from_index(sender, instance, **kwargs): unindex_document(instance.pk) refresh_index()
def test_sorting(self): """Test sorting for search""" # remove the default resource to control the environment self.resource.delete() # create some resources res1 = self.create_resource(**dict(resource_type="example", title="silly example 1", content_xml="<blah>blah 1</blah>", mpath="/blah1", xa_nr_views=1001, xa_nr_attempts=99, xa_avg_grade=9.9)) res2 = self.create_resource(**dict(resource_type="example", title="silly example 2", content_xml="<blah>blah 2</blah>", mpath="/blah2", xa_nr_views=1003, xa_nr_attempts=98, xa_avg_grade=6.8)) res3 = self.create_resource(**dict(resource_type="example", title="silly example 3", content_xml="<blah>blah 3</blah>", mpath="/blah3", xa_nr_views=1002, xa_nr_attempts=101, xa_avg_grade=7.3)) res4 = self.create_resource(**dict(resource_type="example", title="silly example 4", content_xml="<blah>blah 4</blah>", mpath="/blah3", xa_nr_views=1003, xa_nr_attempts=101, xa_avg_grade=9.9)) index_resources([res1.id, res2.id, res3.id, res4.id]) refresh_index() self.assertEqual(self.count_results(), 4) # sorting by number of views results = self.search(query=None, sorting="-{0}".format( LoreSortingFields.SORT_BY_NR_VIEWS[0])) # expected position res2, res4 top_res = results[0] sec_res = results[1] self.assertEqual(top_res.id, res2.id) self.assertEqual(sec_res.id, res4.id) # sorting by number of attempts results = self.search(query=None, sorting="-{0}".format( LoreSortingFields.SORT_BY_NR_ATTEMPTS[0])) # expected position res3, res4 top_res = results[0] sec_res = results[1] self.assertEqual(top_res.id, res3.id) self.assertEqual(sec_res.id, res4.id) # sorting by average grade results = self.search(query=None, sorting="-{0}".format( LoreSortingFields.SORT_BY_AVG_GRADE[0])) # expected position res1, res4 top_res = results[0] sec_res = results[1] self.assertEqual(top_res.id, res1.id) self.assertEqual(sec_res.id, res4.id)