def prepare_index(self, courses, organizations=None):
        """
        Not a test.
        This method is doing the heavy lifting for the tests in this class:
        preparing the Elasticsearch index so that individual tests just have to execute
        the query.
        """
        organizations = organizations or []
        self.create_filter_pages()
        # Delete any existing indices so we get a clean slate
        ES_INDICES_CLIENT.delete(index="_all")

        # Create an index for our organizations
        ES_INDICES_CLIENT.create(index="richie_organizations")
        ES_INDICES_CLIENT.close(index="richie_organizations")
        ES_INDICES_CLIENT.put_settings(body=ANALYSIS_SETTINGS,
                                       index="richie_organizations")
        ES_INDICES_CLIENT.open(index="richie_organizations")
        # Use the default organizations mapping from the Indexer
        ES_INDICES_CLIENT.put_mapping(body=OrganizationsIndexer.mapping,
                                      index="richie_organizations")

        # Set up empty indices for categories & persons. They need to exist to avoid errors
        # but we do not use results from them in our tests.
        ES_INDICES_CLIENT.create(index="richie_categories")
        ES_INDICES_CLIENT.create(index="richie_persons")

        # Create an index we'll use to test the ES features
        ES_INDICES_CLIENT.create(index="test_courses")
        ES_INDICES_CLIENT.close(index="test_courses")
        ES_INDICES_CLIENT.put_settings(body=ANALYSIS_SETTINGS,
                                       index="test_courses")
        ES_INDICES_CLIENT.open(index="test_courses")
        # Use the default courses mapping from the Indexer
        ES_INDICES_CLIENT.put_mapping(body=CoursesIndexer.mapping,
                                      index="test_courses")
        # Add the sorting script
        ES_CLIENT.put_script(id="score", body=CoursesIndexer.scripts["score"])
        ES_CLIENT.put_script(id="state_field",
                             body=CoursesIndexer.scripts["state_field"])

        # Prepare actions to insert our courses and organizations in their indices
        actions = [
            OrganizationsIndexer.get_es_document_for_organization(
                organization.public_extension)
            for organization in organizations
        ] + [{
            "_id": course["id"],
            "_index": "test_courses",
            "_op_type": "create",
            **course,
        } for course in courses]

        bulk_compat(actions=actions, chunk_size=500, client=ES_CLIENT)
        ES_INDICES_CLIENT.refresh()
Exemplo n.º 2
0
    def execute_query(self, courses, querystring="", **extra):
        """
        Not a test.
        Prepare the ElasticSearch index and execute the query in it.
        """

        # Delete any existing indices so we get a clean slate
        ES_INDICES_CLIENT.delete(index="_all")
        # Create an index we'll use to test the ES features
        ES_INDICES_CLIENT.create(index=COURSES_INDEX)

        # The index needs to be closed before we set an analyzer
        ES_INDICES_CLIENT.close(index=COURSES_INDEX)
        ES_INDICES_CLIENT.put_settings(body=ANALYSIS_SETTINGS, index=COURSES_INDEX)
        ES_INDICES_CLIENT.open(index=COURSES_INDEX)

        # Use the default courses mapping from the Indexer
        ES_INDICES_CLIENT.put_mapping(body=CoursesIndexer.mapping, index=COURSES_INDEX)
        # Add the sorting script
        ES_CLIENT.put_script(id="score", body=CoursesIndexer.scripts["score"])
        ES_CLIENT.put_script(
            id="state_field", body=CoursesIndexer.scripts["state_field"]
        )

        # Actually insert our courses in the index
        actions = [
            {
                "_id": course["id"],
                "_index": COURSES_INDEX,
                "_op_type": "create",
                "absolute_url": {"en": "en/url", "fr": "fr/url"},
                "categories": ["1", "2", "3"],
                "cover_image": {"en": "en/image", "fr": "fr/image"},
                "is_meta": False,
                "logo": {"en": "/en/some/img.png", "fr": "/fr/some/img.png"},
                "nb_children": 0,
                "organizations": ["11", "12", "13"],
                **course,
            }
            for course in courses
        ]
        bulk_compat(actions=actions, chunk_size=500, client=ES_CLIENT)
        ES_INDICES_CLIENT.refresh()

        results = self.client.get(
            f"/api/v1.0/courses/autocomplete/?{querystring:s}", **extra
        )
        self.assertEqual(results.status_code, 200)

        return json.loads(results.content)
Exemplo n.º 3
0
    def execute_query(self, kind, categories=None, querystring=""):
        """
        Not a test.
        This method is doing the heavy lifting for the tests in this class: create and fill the
        index with our categories so we can run our queries and check the results.
        It also executes the query and returns the result from the API.
        """
        # Delete any existing indices so we get a clean slate
        ES_INDICES_CLIENT.delete(index="_all")
        # Create an index we'll use to test the ES features
        ES_INDICES_CLIENT.create(index="test_categories")
        ES_INDICES_CLIENT.close(index="test_categories")
        ES_INDICES_CLIENT.put_settings(body=ANALYSIS_SETTINGS,
                                       index="test_categories")
        ES_INDICES_CLIENT.open(index="test_categories")

        # Use the default categories mapping from the Indexer
        ES_INDICES_CLIENT.put_mapping(body=CategoriesIndexer.mapping,
                                      index="test_categories")

        # Actually insert our categories in the index
        actions = [{
            "_id": category["id"],
            "_index": "test_categories",
            "_op_type": "create",
            "absolute_url": {
                "en": "en/url"
            },
            "description": {
                "en": "en/description"
            },
            "icon": {
                "en": "en/icon"
            },
            "is_meta": False,
            "logo": {
                "en": "en/logo"
            },
            "nb_children": 0,
            "path": category["id"],
            "title_raw": category["title"],
            **category,
        } for category in categories or CATEGORIES]
        bulk_compat(actions=actions, chunk_size=500, client=ES_CLIENT)
        ES_INDICES_CLIENT.refresh()

        response = self.client.get(f"/api/v1.0/{kind:s}/?{querystring:s}")
        self.assertEqual(response.status_code, 200)

        return json.loads(response.content)
Exemplo n.º 4
0
    def execute_query(self, querystring="", **extra):
        """
        Not a test.
        Prepare the ElasticSearch index and execute the query in it.
        """

        licences = [
            {"id": "1", "title": {"en": "CC-BY-SA"}},
            {"id": "2", "title": {"en": "CC-BY-NC"}},
            {"id": "3", "title": {"en": "All Rights Résërvés"}},
        ]

        # Delete any existing indices so we get a clean slate
        ES_INDICES_CLIENT.delete(index="_all")
        # Create an index we'll use to test the ES features
        ES_INDICES_CLIENT.create(index=LICENCES_INDEX)

        # The index needs to be closed before we set an analyzer
        ES_INDICES_CLIENT.close(index=LICENCES_INDEX)
        ES_INDICES_CLIENT.put_settings(body=ANALYSIS_SETTINGS, index=LICENCES_INDEX)
        ES_INDICES_CLIENT.open(index=LICENCES_INDEX)

        # Use the default licences mapping from the Indexer
        ES_INDICES_CLIENT.put_mapping(
            body=LicencesIndexer.mapping, index=LICENCES_INDEX
        )

        # Actually insert our licences in the index
        actions = [
            {
                "_id": licence["id"],
                "_index": LICENCES_INDEX,
                "_op_type": "create",
                "complete": {"en": slice_string_for_completion(licence["title"]["en"])},
                **licence,
            }
            for licence in licences
        ]
        bulk_compat(actions=actions, chunk_size=500, client=ES_CLIENT)
        ES_INDICES_CLIENT.refresh()

        response = self.client.get(
            f"/api/v1.0/licences/autocomplete/?{querystring:s}", **extra
        )
        self.assertEqual(response.status_code, 200)

        return licences, json.loads(response.content)
Exemplo n.º 5
0
    def execute_query(self, persons=None, querystring=""):
        """
        Not a test.
        This method is doing the heavy lifting for the tests in this class: create and fill the
        index with our persons so we can run our queries and check the results.
        It also executes the query and returns the result from the API.
        """
        # Delete any existing indices so we get a clean slate
        ES_INDICES_CLIENT.delete(index="_all")
        # Create an index we'll use to test the ES features
        ES_INDICES_CLIENT.create(index="test_persons")
        ES_INDICES_CLIENT.close(index="test_persons")
        ES_INDICES_CLIENT.put_settings(body=ANALYSIS_SETTINGS,
                                       index="test_persons")
        ES_INDICES_CLIENT.open(index="test_persons")

        # Use the default persons mapping from the Indexer
        ES_INDICES_CLIENT.put_mapping(body=PersonsIndexer.mapping,
                                      index="test_persons")

        # Actually insert our persons in the index
        actions = [{
            "_id": person["id"],
            "_index": "test_persons",
            "_op_type": "create",
            "absolute_url": {
                "en": "en/url"
            },
            "bio": {
                "en": "en/bio"
            },
            "portrait": {
                "en": "en/image"
            },
            "title_raw": person["title"],
            **person,
        } for person in persons or PERSONS]
        bulk_compat(actions=actions, chunk_size=500, client=ES_CLIENT)
        ES_INDICES_CLIENT.refresh()

        response = self.client.get(f"/api/v1.0/persons/?{querystring:s}")
        self.assertEqual(response.status_code, 200)

        return json.loads(response.content)
Exemplo n.º 6
0
    def execute_query(self, querystring="", **extra):
        """
        Not a test.
        Prepare the ElasticSearch index and execute the query in it.
        """

        persons = [
            {
                "complete": {
                    "en": slice_string_for_completion("Éponine Thénardier")
                },
                "id": "25",
                "title": {
                    "en": "Éponine Thénardier"
                },
            },
            {
                "complete": {
                    "en":
                    slice_string_for_completion("Monseigneur Bienvenu Myriel")
                },
                "id": "34",
                "title": {
                    "en": "Monseigneur Bienvenu Myriel"
                },
            },
            {
                "complete": {
                    "en": slice_string_for_completion("Fantine")
                },
                "id": "52",
                "title": {
                    "en": "Fantine"
                },
            },
        ]

        # Delete any existing indices so we get a clean slate
        ES_INDICES_CLIENT.delete(index="_all")
        # Create an index we'll use to test the ES features
        ES_INDICES_CLIENT.create(index=PERSONS_INDEX)

        # The index needs to be closed before we set an analyzer
        ES_INDICES_CLIENT.close(index=PERSONS_INDEX)
        ES_INDICES_CLIENT.put_settings(body=ANALYSIS_SETTINGS,
                                       index=PERSONS_INDEX)
        ES_INDICES_CLIENT.open(index=PERSONS_INDEX)

        # Use the default persons mapping from the Indexer
        ES_INDICES_CLIENT.put_mapping(body=PersonsIndexer.mapping,
                                      index=PERSONS_INDEX)

        # Actually insert our persons in the index
        actions = [{
            "_id": person["id"],
            "_index": PERSONS_INDEX,
            "_op_type": "create",
            "absolute_url": {
                "en": "url"
            },
            "logo": {
                "en": "/some/img.png"
            },
            **person,
        } for person in persons]
        bulk_compat(actions=actions, chunk_size=500, client=ES_CLIENT)
        ES_INDICES_CLIENT.refresh()

        response = self.client.get(
            f"/api/v1.0/persons/autocomplete/?{querystring:s}", **extra)
        self.assertEqual(response.status_code, 200)

        return persons, json.loads(response.content)
Exemplo n.º 7
0
    def test_indexable_filters_internationalization(self):
        """
        Indexable filters (such as categories and organizations by default) should have
        their names localized in the filter definitions in course search responses.
        """
        # Create the meta categories, each with a child category that should appear in facets
        subjects_meta = CategoryFactory(page_reverse_id="subjects",
                                        should_publish=True)
        subject = CategoryFactory(page_parent=subjects_meta.extended_object,
                                  should_publish=True)
        levels_meta = CategoryFactory(page_reverse_id="levels",
                                      should_publish=True)
        level = CategoryFactory(page_parent=levels_meta.extended_object,
                                should_publish=True)
        # Create 2 organizations that should appear in facets
        org_meta = OrganizationFactory(page_reverse_id="organizations",
                                       should_publish=True)
        org_1 = OrganizationFactory(
            page_parent=org_meta.extended_object,
            page_title="First organization",
            should_publish=True,
        )
        org_2 = OrganizationFactory(
            page_parent=org_meta.extended_object,
            page_title="Second organization",
            should_publish=True,
        )
        # Create a course linked to our categories and organizations
        CourseFactory(
            fill_categories=[subject, level],
            fill_organizations=[org_1, org_2],
            should_publish=True,
        )
        # Index our objects into ES
        bulk_compat(
            actions=[
                *ES_INDICES.categories.get_es_documents(),
                *ES_INDICES.organizations.get_es_documents(),
                *ES_INDICES.courses.get_es_documents(),
            ],
            chunk_size=500,
            client=ES_CLIENT,
        )
        ES_INDICES_CLIENT.refresh()

        response = self.client.get("/api/v1.0/courses/?scope=filters")
        self.assertEqual(response.status_code, 200)

        self.assertEqual(
            response.json()["filters"]["subjects"],
            {
                "base_path":
                "0001",
                "human_name":
                "Subjects",
                "is_autocompletable":
                True,
                "is_drilldown":
                False,
                "is_searchable":
                True,
                "name":
                "subjects",
                "position":
                2,
                "has_more_values":
                False,
                "values": [{
                    "count": 1,
                    "human_name": subject.extended_object.get_title(),
                    "key": subject.get_es_id(),
                }],
            },
        )
        self.assertEqual(
            response.json()["filters"]["levels"],
            {
                "base_path":
                "0002",
                "human_name":
                "Levels",
                "is_autocompletable":
                True,
                "is_drilldown":
                False,
                "is_searchable":
                True,
                "name":
                "levels",
                "position":
                3,
                "has_more_values":
                False,
                "values": [{
                    "count": 1,
                    "human_name": level.extended_object.get_title(),
                    "key": level.get_es_id(),
                }],
            },
        )
        self.assertEqual(
            response.json()["filters"]["organizations"],
            {
                "base_path":
                "0003",
                "human_name":
                "Organizations",
                "is_autocompletable":
                True,
                "is_drilldown":
                False,
                "is_searchable":
                True,
                "name":
                "organizations",
                "position":
                4,
                "has_more_values":
                False,
                "values": [
                    {
                        "count": 1,
                        "human_name": org_1.extended_object.get_title(),
                        "key": org_1.get_es_id(),
                    },
                    {
                        "count": 1,
                        "human_name": org_2.extended_object.get_title(),
                        "key": org_2.get_es_id(),
                    },
                ],
            },
        )
Exemplo n.º 8
0
    def test_index_manager_perform_create_index(self):
        """
        Perform all side-effects through the ES client and return the index name (incl. timestamp)
        """

        # Create an indexable from scratch that mimicks the expected shape of the dynamic
        # import in es_index
        class IndexableClass:
            """Indexable stub"""

            index_name = "richie_courses"
            mapping = {
                "properties": {
                    "code": {
                        "type": "keyword"
                    },
                    "name": {
                        "type": "text"
                    }
                }
            }

            # pylint: disable=no-self-use
            def get_es_documents(self, index, action="index"):
                """Stub method"""

                for i in range(0, 10):
                    yield {
                        "_id": i,
                        "_index": index,
                        "_op_type": action,
                        "code": f"course-{i:d}",
                        "name": f"Course Number {i:d}",
                    }

        indexable = IndexableClass()

        # Set a fake time to check the name of the index
        now = datetime(2016, 5, 4, 3, 12, 33, 123456, tzinfo=pytz.utc)

        # Make sure our index is empty before we call the function
        self.assertEqual(ES_INDICES_CLIENT.get_alias("*"), {})

        mock_logger = mock.Mock(spec=["info"])

        with mock.patch.object(timezone, "now", return_value=now):
            new_index = perform_create_index(indexable, mock_logger)
        ES_INDICES_CLIENT.refresh()

        self.assertEqual(new_index,
                         "richie_courses_2016-05-04-03h12m33.123456s")
        self.assertEqual(ES_CLIENT.count()["count"], 10)
        self.assertEqual(
            ES_INDICES_CLIENT.get_mapping(),
            {
                "richie_courses_2016-05-04-03h12m33.123456s": {
                    "mappings": {
                        "properties": {
                            "code": {
                                "type": "keyword"
                            },
                            "name": {
                                "type": "text"
                            },
                        }
                    }
                }
            },
        )
        mock_logger.info.assert_called()
    def execute_query(self, querystring="", **extra):
        """
        Not a test.
        Prepare the ElasticSearch index and execute the query in it.
        """

        categories = [
            {
                "complete": {
                    "en": slice_string_for_completion("Electric Birdwatching"),
                    "fr": slice_string_for_completion(
                        "Observation des oiseaux électriques"
                    ),
                },
                "id": "24",
                "kind": "subjects",
                "path": "001000",
                "title": {
                    "en": "Electric Birdwatching",
                    "fr": "Observation des oiseaux électriques",
                },
            },
            {
                "complete": {
                    "en": slice_string_for_completion("Ocean biking"),
                    "fr": slice_string_for_completion("Cyclisme océanique"),
                },
                "id": "33",
                "kind": "subjects",
                "path": "001001",
                "title": {"en": "Ocean biking", "fr": "Cyclisme océanique"},
            },
            {
                "complete": {
                    "en": slice_string_for_completion("Elegiac bikeshedding"),
                    "fr": slice_string_for_completion("Élégie de l'abri à vélos"),
                },
                "id": "51",
                "kind": "subjects",
                "path": "001002",
                "title": {
                    "en": "Elegiac bikeshedding",
                    "fr": "Élégie de l'abri à vélos",
                },
            },
            {
                "complete": {
                    "en": slice_string_for_completion("Electric Decoys"),
                    "fr": slice_string_for_completion("Leurres électriques"),
                },
                "id": "44",
                "kind": "not_subjects",
                "path": "001003",
                "title": {"en": "Electric Decoys", "fr": "Leurres électriques"},
            },
        ]

        # Delete any existing indices so we get a clean slate
        ES_INDICES_CLIENT.delete(index="_all")
        # Create an index we'll use to test the ES features
        ES_INDICES_CLIENT.create(index=CATEGORIES_INDEX)

        # The index needs to be closed before we set an analyzer
        ES_INDICES_CLIENT.close(index=CATEGORIES_INDEX)
        ES_INDICES_CLIENT.put_settings(body=ANALYSIS_SETTINGS, index=CATEGORIES_INDEX)
        ES_INDICES_CLIENT.open(index=CATEGORIES_INDEX)

        # Use the default categories mapping from the Indexer
        ES_INDICES_CLIENT.put_mapping(
            body=CategoriesIndexer.mapping, index=CATEGORIES_INDEX
        )

        # Actually insert our categories in the index
        actions = [
            {
                "_id": category["id"],
                "_index": CATEGORIES_INDEX,
                "_op_type": "create",
                "absolute_url": {"en": "en/url", "fr": "fr/url"},
                "cover_image": {"en": "en/image", "fr": "fr/image"},
                "is_meta": False,
                "logo": {"en": "en/some/img.png", "fr": "fr/some/img.png"},
                "nb_children": 0,
                **category,
            }
            for category in categories
        ]
        bulk_compat(actions=actions, chunk_size=500, client=ES_CLIENT)
        ES_INDICES_CLIENT.refresh()

        response = self.client.get(
            f"/api/v1.0/subjects/autocomplete/?{querystring:s}", **extra
        )
        self.assertEqual(response.status_code, 200)

        return categories, json.loads(response.content)