Beispiel #1
0
def index_all():
    """ Reindex all objects """
    with virtualenv(), env.cd("mootiro_maps"):
        setup_django()

        from organization.models import Organization
        from komoo_resource.models import Resource
        from need.models import Need
        from community.models import Community
        from authentication.models import User
        from komoo_project.models import Project
        from search.utils import reset_index, create_mapping, refresh_index, index_object

        model_list = [Organization, Resource, Need, Community, User, Project]

        logging.info("Recreating index ... ")
        reset_index()

        logging.info("Create mappings ... ")
        create_mapping()

        logging.info("Indexing each object ... ")
        for model in model_list:
            for obj in model.objects.all():
                index_object(obj)

        logging.info("refreshing index ... ")
        refresh_index()

        logging.info("ES indexes rebuilt.")
Beispiel #2
0
def index_all():
    """ Reindex all objects """
    with virtualenv(), env.cd('mootiro_maps'):
        setup_django()

        from organization.models import Organization
        from komoo_resource.models import Resource
        from need.models import Need
        from community.models import Community
        from authentication.models import User
        from komoo_project.models import Project
        from search.utils import (reset_index, create_mapping, refresh_index,
                                  index_object)

        model_list = [Organization, Resource, Need, Community, User, Project]

        logging.info('Recreating index ... ')
        reset_index()

        logging.info('Create mappings ... ')
        create_mapping()

        logging.info('Indexing each object ... ')
        for model in model_list:
            for obj in model.objects.all():
                index_object(obj)

        logging.info('refreshing index ... ')
        refresh_index()

        logging.info('ES indexes rebuilt.')
Beispiel #3
0
 def test_strip_xml(self):
     """Indexed content_xml should have XML stripped."""
     xml = "<tag>you're it</tag>"
     self.resource.content_xml = xml
     self.resource.save()
     refresh_index()
     self.assertTrue(self.count_results("you're it") == 1)
     self.assertTrue(self.count_results("tag") == 0)
Beispiel #4
0
 def test_strip_xml(self):
     """Indexed content_xml should have XML stripped."""
     xml = "<tag>you're it</tag>"
     self.resource.content_xml = xml
     self.resource.save()
     refresh_index()
     self.assertTrue(self.count_results("you're it") == 1)
     self.assertTrue(self.count_results("tag") == 0)
Beispiel #5
0
 def tearDown(self):
     """Clean up Elasticsearch and static assets between tests."""
     for static_asset in StaticAsset.objects.all():
         default_storage.delete(static_asset.asset)
     for key, _ in haystack.connections.connections_info.items():
         haystack.connections.reload(key)
     call_command('clear_index', interactive=False, verbosity=0)
     clear_index()
     refresh_index()
Beispiel #6
0
def update_index(**kwargs):
    if 'instance' in kwargs:
        doc = kwargs.get('instance')
    else:
        doc = kwargs.get('document')

    # When a document is first created, the Document is saved
    # before the Metadata and MetadataRevision are created.
    # Then, the Document is saved again
    # Thus, we MUST not index the document on the first save, since the
    # metadata and revision does not exist yet
    created = kwargs.pop('created', False)
    if not created and doc.is_indexable:
        index_document(doc.pk)
    refresh_index()
Beispiel #7
0
def update_index(**kwargs):
    if 'instance' in kwargs:
        doc = kwargs.get('instance')
    else:
        doc = kwargs.get('document')

    # When a document is first created, the Document is saved
    # before the Metadata and MetadataRevision are created.
    # Then, the Document is saved again
    # Thus, we MUST not index the document on the first save, since the
    # metadata and revision does not exist yet
    created = kwargs.pop('created', False)
    if not created and doc.is_indexable:
        index_document(doc.pk)
    refresh_index()
Beispiel #8
0
 def test_index_vocabulary(self):
     """
     Test that LearningResource indexes are updated when a
     a term is added or removed.
     """
     set_cache_timeout(0)
     term = self.terms[0]
     self.assertEqual(
         self.count_faceted_results(self.vocabulary.id, term.id), 0)
     self.resource.terms.add(term)
     refresh_index()
     self.assertEqual(
         self.count_faceted_results(self.vocabulary.id, term.id), 1)
     self.resource.terms.remove(term)
     refresh_index()
     self.assertEqual(
         self.count_faceted_results(self.vocabulary.id, term.id), 0)
Beispiel #9
0
 def test_index_vocabulary(self):
     """
     Test that LearningResource indexes are updated when a
     a term is added or removed.
     """
     set_cache_timeout(0)
     term = self.terms[0]
     self.assertEqual(self.count_faceted_results(
         self.vocabulary.id, term.id), 0)
     self.resource.terms.add(term)
     refresh_index()
     self.assertEqual(self.count_faceted_results(
         self.vocabulary.id, term.id), 1)
     self.resource.terms.remove(term)
     refresh_index()
     self.assertEqual(self.count_faceted_results(
         self.vocabulary.id, term.id), 0)
Beispiel #10
0
    def setUp(self):
        """set up"""
        super(LoreTestCase, self).setUp()
        clear_index()
        self.user = User.objects.create_user(
            username=self.USERNAME, password=self.PASSWORD
        )
        add_repo_perm = Permission.objects.get(codename=self.ADD_REPO_PERM)
        self.user.user_permissions.add(add_repo_perm)
        # user without permission to add a repo
        self.user_norepo = User.objects.create_user(
            username=self.USERNAME_NO_REPO, password=self.PASSWORD
        )

        self.repo = create_repo(
            name="test repo",
            description="just a test",
            user_id=self.user.id,
        )
        self.course = create_course(
            org="test-org",
            repo_id=self.repo.id,
            course_number="infinity",
            run="Febtober",
            user_id=self.user.id,
        )
        self.resource = self.create_resource(
            course=self.course,
            parent=None,
            resource_type="example",
            title="silly example",
            content_xml="<blah>blah</blah>",
            mpath="/blah",
            url_name="url_name1",
        )

        self.toy_resource_count = 18  # Resources in toy course.
        self.toy_asset_count = 5  # Static assets in toy course.
        self.client = Client()

        self.login(username=self.USERNAME)
        refresh_index()
Beispiel #11
0
    def setUp(self):
        """set up"""
        super(LoreTestCase, self).setUp()
        recreate_index()
        self.user = User.objects.create_user(username=self.USERNAME,
                                             password=self.PASSWORD)
        add_repo_perm = Permission.objects.get(codename=self.ADD_REPO_PERM)
        self.user.user_permissions.add(add_repo_perm)
        # user without permission to add a repo
        self.user_norepo = User.objects.create_user(
            username=self.USERNAME_NO_REPO, password=self.PASSWORD)

        self.repo = create_repo(
            name="test repo",
            description="just a test",
            user_id=self.user.id,
        )
        self.course = create_course(
            org="test-org",
            repo_id=self.repo.id,
            course_number="infinity",
            run="Febtober",
            user_id=self.user.id,
        )
        self.resource = self.create_resource(
            course=self.course,
            parent=None,
            resource_type="example",
            title="silly example",
            content_xml="<blah>blah</blah>",
            mpath="/blah",
            url_name="url_name1",
        )

        self.toy_resource_count = 18  # Resources in toy course.
        self.toy_asset_count = 5  # Static assets in toy course.
        self.client = Client()

        self.login(username=self.USERNAME)
        refresh_index()
Beispiel #12
0
    def test_search_fields(self):
        """
        The content_xml, title, and description should be searchable
        by partial matches.
        """
        new_resource = LearningResource.objects.create(
            course=self.resource.course,
            learning_resource_type=self.resource.learning_resource_type,
            title="accordion music",
            content_xml="<blah>foo</blah>",
            materialized_path="/foo",
            url_name="url_name2",
            description="the quick brown fox",
            description_path="silver",
        )

        new_resource.save()  # to trigger indexing
        refresh_index()

        self.assertEqual(search_index("accordion").count(), 1)
        self.assertEqual(search_index("fox brown").count(), 1)
        self.assertEqual(search_index("foo").count(), 1)
Beispiel #13
0
    def test_search_fields(self):
        """
        The content_xml, title, and description should be searchable
        by partial matches.
        """
        new_resource = LearningResource.objects.create(
            course=self.resource.course,
            learning_resource_type=self.resource.learning_resource_type,
            title="accordion music",
            content_xml="<blah>foo</blah>",
            materialized_path="/foo",
            url_name="url_name2",
            description="the quick brown fox",
            description_path="silver",
        )

        new_resource.save()  # to trigger indexing
        refresh_index()

        self.assertEqual(search_index("accordion").count(), 1)
        self.assertEqual(search_index("fox brown").count(), 1)
        self.assertEqual(search_index("foo").count(), 1)
Beispiel #14
0
    def test_import(self):
        """
        Indexing should occur on course import.
        """
        results = search_index()
        original_count = results.count()
        import_course_from_file(self.course_zip, self.repo.id, self.user.id)
        refresh_index()
        results = search_index()
        self.assertEqual(results.count(), original_count + 18)
        self.assertEqual(results.page_count(), 2)

        # Ensure the correct number of results are returned in each page.
        self.assertEqual(len(results.get_page(1)), 10)
        self.assertEqual(len(results.get_page(2)), 9)
        self.assertEqual(len(results.get_page(3)), 0)

        # Make sure the .all() function (a convenience function which
        # returns a generator), returns all the records without explicit
        # pagination.
        count = 0
        for _ in results.all():
            count += 1
        self.assertEqual(count, results.count())
Beispiel #15
0
    def test_import(self):
        """
        Indexing should occur on course import.
        """
        results = search_index()
        original_count = results.count()
        import_course_from_file(self.course_zip, self.repo.id, self.user.id)
        refresh_index()
        results = search_index()
        self.assertEqual(results.count(), original_count + 18)
        self.assertEqual(results.page_count(), 2)

        # Ensure the correct number of results are returned in each page.
        self.assertEqual(len(results.get_page(1)), 10)
        self.assertEqual(len(results.get_page(2)), 9)
        self.assertEqual(len(results.get_page(3)), 0)

        # Make sure the .all() function (a convenience function which
        # returns a generator), returns all the records without explicit
        # pagination.
        count = 0
        for _ in results.all():
            count += 1
        self.assertEqual(count, results.count())
Beispiel #16
0
 def test_sorting(self):
     """Test sorting for search"""
     # remove the default resource to control the environment
     self.resource.delete()
     # create some resources
     res1 = self.create_resource(**dict(
         resource_type="example",
         title="silly example 1",
         content_xml="<blah>blah 1</blah>",
         mpath="/blah1",
         xa_nr_views=1001,
         xa_nr_attempts=99,
         xa_avg_grade=9.9
     ))
     res2 = self.create_resource(**dict(
         resource_type="example",
         title="silly example 2",
         content_xml="<blah>blah 2</blah>",
         mpath="/blah2",
         xa_nr_views=1003,
         xa_nr_attempts=98,
         xa_avg_grade=6.8
     ))
     res3 = self.create_resource(**dict(
         resource_type="example",
         title="silly example 3",
         content_xml="<blah>blah 3</blah>",
         mpath="/blah3",
         xa_nr_views=1002,
         xa_nr_attempts=101,
         xa_avg_grade=7.3
     ))
     res4 = self.create_resource(**dict(
         resource_type="example",
         title="silly example 4",
         content_xml="<blah>blah 4</blah>",
         mpath="/blah3",
         xa_nr_views=1003,
         xa_nr_attempts=101,
         xa_avg_grade=9.9
     ))
     index_resources([res1.id, res2.id, res3.id, res4.id])
     refresh_index()
     self.assertEqual(self.count_results(), 4)
     # sorting by number of views
     results = self.search(
         query=None,
         sorting="-{0}".format(LoreSortingFields.SORT_BY_NR_VIEWS[0])
     )
     # expected position res2, res4
     top_res = results[0]
     sec_res = results[1]
     self.assertEqual(
         top_res.id,
         res2.id
     )
     self.assertEqual(
         sec_res.id,
         res4.id
     )
     # sorting by number of attempts
     results = self.search(
         query=None,
         sorting="-{0}".format(LoreSortingFields.SORT_BY_NR_ATTEMPTS[0])
     )
     # expected position res3, res4
     top_res = results[0]
     sec_res = results[1]
     self.assertEqual(
         top_res.id,
         res3.id
     )
     self.assertEqual(
         sec_res.id,
         res4.id
     )
     # sorting by average grade
     results = self.search(
         query=None,
         sorting="-{0}".format(LoreSortingFields.SORT_BY_AVG_GRADE[0])
     )
     # expected position res1, res4
     top_res = results[0]
     sec_res = results[1]
     self.assertEqual(
         top_res.id,
         res1.id
     )
     self.assertEqual(
         sec_res.id,
         res4.id
     )
Beispiel #17
0
def remove_from_index(sender, instance, **kwargs):
    unindex_document(instance.pk)
    refresh_index()
Beispiel #18
0
def remove_from_index(sender, instance, **kwargs):
    unindex_document(instance.pk)
    refresh_index()
Beispiel #19
0
 def test_sorting(self):
     """Test sorting for search"""
     # remove the default resource to control the environment
     self.resource.delete()
     # create some resources
     res1 = self.create_resource(**dict(resource_type="example",
                                        title="silly example 1",
                                        content_xml="<blah>blah 1</blah>",
                                        mpath="/blah1",
                                        xa_nr_views=1001,
                                        xa_nr_attempts=99,
                                        xa_avg_grade=9.9))
     res2 = self.create_resource(**dict(resource_type="example",
                                        title="silly example 2",
                                        content_xml="<blah>blah 2</blah>",
                                        mpath="/blah2",
                                        xa_nr_views=1003,
                                        xa_nr_attempts=98,
                                        xa_avg_grade=6.8))
     res3 = self.create_resource(**dict(resource_type="example",
                                        title="silly example 3",
                                        content_xml="<blah>blah 3</blah>",
                                        mpath="/blah3",
                                        xa_nr_views=1002,
                                        xa_nr_attempts=101,
                                        xa_avg_grade=7.3))
     res4 = self.create_resource(**dict(resource_type="example",
                                        title="silly example 4",
                                        content_xml="<blah>blah 4</blah>",
                                        mpath="/blah3",
                                        xa_nr_views=1003,
                                        xa_nr_attempts=101,
                                        xa_avg_grade=9.9))
     index_resources([res1.id, res2.id, res3.id, res4.id])
     refresh_index()
     self.assertEqual(self.count_results(), 4)
     # sorting by number of views
     results = self.search(query=None,
                           sorting="-{0}".format(
                               LoreSortingFields.SORT_BY_NR_VIEWS[0]))
     # expected position res2, res4
     top_res = results[0]
     sec_res = results[1]
     self.assertEqual(top_res.id, res2.id)
     self.assertEqual(sec_res.id, res4.id)
     # sorting by number of attempts
     results = self.search(query=None,
                           sorting="-{0}".format(
                               LoreSortingFields.SORT_BY_NR_ATTEMPTS[0]))
     # expected position res3, res4
     top_res = results[0]
     sec_res = results[1]
     self.assertEqual(top_res.id, res3.id)
     self.assertEqual(sec_res.id, res4.id)
     # sorting by average grade
     results = self.search(query=None,
                           sorting="-{0}".format(
                               LoreSortingFields.SORT_BY_AVG_GRADE[0]))
     # expected position res1, res4
     top_res = results[0]
     sec_res = results[1]
     self.assertEqual(top_res.id, res1.id)
     self.assertEqual(sec_res.id, res4.id)