def test_loading_with_single_object(self):
        # type: () -> None
        """
        Test Loading functionality with single python object
        """
        loader = FSElasticsearchJSONLoader()
        loader.init(conf=Scoped.get_scoped_conf(conf=self.conf,
                                                scope=loader.get_scope()))

        data = TableESDocument(
            database='test_database',
            cluster='test_cluster',
            schema_name='test_schema',
            table_name='test_table',
            table_key='test_table_key',
            table_last_updated_epoch=123456789,
            table_description='test_description',
            column_names=['test_col1', 'test_col2'],
            column_descriptions=['test_comment1', 'test_comment2'],
            total_usage=10,
            unique_usage=5,
            tag_names=['test_tag1', 'test_tag2'])
        loader.load(data)
        loader.close()

        expected = [(
            '{"table_key": "test_table_key", "column_descriptions": ["test_comment1", "test_comment2"], '
            '"schema_name": "test_schema", "database": "test_database", "cluster": "test_cluster", '
            '"column_names": ["test_col1", "test_col2"], "table_name": "test_table", '
            '"table_last_updated_epoch": 123456789,'
            '"table_description": "test_description", "unique_usage": 5, "total_usage": 10, '
            '"tag_names": ["test_tag1", "test_tag2"]}')]

        self._check_results_helper(expected=expected)
    def test_loading_with_different_object(self):
        # type: () -> None
        """
        Test Loading functionality with a python Dict object
        """
        loader = FSElasticsearchJSONLoader()
        loader.init(conf=Scoped.get_scoped_conf(conf=self.conf,
                                                scope=loader.get_scope()))

        data = dict(database='test_database',
                    cluster='test_cluster',
                    schema_name='test_schema',
                    table_name='test_table',
                    table_key='test_table_key',
                    table_last_updated_epoch=123456789,
                    table_description='test_description',
                    column_names=['test_col1', 'test_col2'],
                    column_descriptions=['test_comment1', 'test_comment2'],
                    total_usage=10,
                    unique_usage=5,
                    tag_names=['test_tag1', 'test_tag2'])

        with self.assertRaises(Exception) as context:
            loader.load(data)  # type: ignore
        self.assertTrue(
            "Record not of type 'ElasticsearchDocument'!" in context.exception)

        loader.close()
Beispiel #3
0
    def test_empty_loading(self) -> None:
        """
        Test loading functionality with no data
        """
        loader = FSElasticsearchJSONLoader()
        loader.init(conf=Scoped.get_scoped_conf(conf=self.conf,
                                                scope=loader.get_scope()))

        loader.load(None)  # type: ignore
        loader.close()

        self._check_results_helper(expected=[])
    def test_loading_with_list_of_objects(self):
        # type: () -> None
        """
        Test Loading functionality with list of objects.
        Check to ensure all objects are added to file
        """
        loader = FSElasticsearchJSONLoader()
        loader.init(conf=Scoped.get_scoped_conf(conf=self.conf,
                                                scope=loader.get_scope()))

        data = [
            TableESDocument(
                database='test_database',
                cluster='test_cluster',
                schema='test_schema',
                name='test_table',
                key='test_table_key',
                last_updated_timestamp=123456789,
                description='test_description',
                column_names=['test_col1', 'test_col2'],
                column_descriptions=['test_comment1', 'test_comment2'],
                total_usage=10,
                unique_usage=5,
                tags=['test_tag1', 'test_tag2'],
                badges=['badge1'],
                schema_description='schema_description',
                programmatic_descriptions=['test'])
        ] * 5

        for d in data:
            loader.load(d)
        loader.close()

        expected = [(
            '{"key": "test_table_key", "column_descriptions": ["test_comment1", "test_comment2"], '
            '"schema": "test_schema", "database": "test_database", "cluster": "test_cluster", '
            '"column_names": ["test_col1", "test_col2"], "name": "test_table", '
            '"last_updated_timestamp": 123456789, "display_name": "test_schema.test_table", '
            '"description": "test_description", "unique_usage": 5, "total_usage": 10, '
            '"tags": ["test_tag1", "test_tag2"], "schema_description": "schema_description", '
            '"programmatic_descriptions":["test"], '
            '"badges": ["badge1"]}')] * 5

        self._check_results_helper(expected=expected)
Beispiel #5
0
    def test_loading_with_list_of_objects(self):
        # type: () -> None
        """
        Test Loading functionality with list of objects.
        Check to ensure all objects are added to file
        """
        loader = FSElasticsearchJSONLoader()
        loader.init(conf=Scoped.get_scoped_conf(conf=self.conf,
                                                scope=loader.get_scope()))

        data = [
            ElasticsearchDocument(
                elasticsearch_index='test_es_index',
                elasticsearch_type='test_es_type',
                database='test_database',
                cluster='test_cluster',
                schema_name='test_schema',
                table_name='test_table',
                table_key='test_table_key',
                table_last_updated_epoch=123456789,
                table_description='test_description',
                column_names=['test_col1', 'test_col2'],
                column_descriptions=['test_comment1', 'test_comment2'],
                total_usage=10,
                unique_usage=5,
                tag_names=['test_tag1', 'test_tag2'])
        ] * 5

        for d in data:
            loader.load(d)
        loader.close()

        expected = [
            '{"index": {"_type": "test_es_type", "_index": "test_es_index"}}',
            ('{"table_key": "test_table_key", "column_descriptions": ["test_comment1", "test_comment2"], '
             '"schema_name": "test_schema", "database": "test_database", "cluster": "test_cluster", '
             '"column_names": ["test_col1", "test_col2"], "table_name": "test_table", '
             '"table_last_updated_epoch": 123456789,'
             '"table_description": "test_description", "unique_usage": 5, "total_usage": 10, '
             '"tag_names": ["test_tag1", "test_tag2"]}')
        ] * 5

        self._check_results_helper(expected=expected)