def test_loading_with_single_object(self):
# type: () -> None
"""
Test Loading functionality with single python object
"""
loader = FSElasticsearchJSONLoader()
loader.init(conf=Scoped.get_scoped_conf(conf=self.conf,
scope=loader.get_scope()))
data = TableESDocument(
database='test_database',
cluster='test_cluster',
schema_name='test_schema',
table_name='test_table',
table_key='test_table_key',
table_last_updated_epoch=123456789,
table_description='test_description',
column_names=['test_col1', 'test_col2'],
column_descriptions=['test_comment1', 'test_comment2'],
total_usage=10,
unique_usage=5,
tag_names=['test_tag1', 'test_tag2'])
loader.load(data)
loader.close()
expected = [(
'{"table_key": "test_table_key", "column_descriptions": ["test_comment1", "test_comment2"], '
'"schema_name": "test_schema", "database": "test_database", "cluster": "test_cluster", '
'"column_names": ["test_col1", "test_col2"], "table_name": "test_table", '
'"table_last_updated_epoch": 123456789,'
'"table_description": "test_description", "unique_usage": 5, "total_usage": 10, '
'"tag_names": ["test_tag1", "test_tag2"]}')]
self._check_results_helper(expected=expected)
def test_loading_with_different_object(self):
# type: () -> None
"""
Test Loading functionality with a python Dict object
"""
loader = FSElasticsearchJSONLoader()
loader.init(conf=Scoped.get_scoped_conf(conf=self.conf,
scope=loader.get_scope()))
data = dict(database='test_database',
cluster='test_cluster',
schema_name='test_schema',
table_name='test_table',
table_key='test_table_key',
table_last_updated_epoch=123456789,
table_description='test_description',
column_names=['test_col1', 'test_col2'],
column_descriptions=['test_comment1', 'test_comment2'],
total_usage=10,
unique_usage=5,
tag_names=['test_tag1', 'test_tag2'])
with self.assertRaises(Exception) as context:
loader.load(data) # type: ignore
self.assertTrue(
"Record not of type 'ElasticsearchDocument'!" in context.exception)
loader.close()
def test_empty_loading(self) -> None:
"""
Test loading functionality with no data
"""
loader = FSElasticsearchJSONLoader()
loader.init(conf=Scoped.get_scoped_conf(conf=self.conf,
scope=loader.get_scope()))
loader.load(None) # type: ignore
loader.close()
self._check_results_helper(expected=[])
def test_loading_with_list_of_objects(self):
# type: () -> None
"""
Test Loading functionality with list of objects.
Check to ensure all objects are added to file
"""
loader = FSElasticsearchJSONLoader()
loader.init(conf=Scoped.get_scoped_conf(conf=self.conf,
scope=loader.get_scope()))
data = [
TableESDocument(
database='test_database',
cluster='test_cluster',
schema='test_schema',
name='test_table',
key='test_table_key',
last_updated_timestamp=123456789,
description='test_description',
column_names=['test_col1', 'test_col2'],
column_descriptions=['test_comment1', 'test_comment2'],
total_usage=10,
unique_usage=5,
tags=['test_tag1', 'test_tag2'],
badges=['badge1'],
schema_description='schema_description',
programmatic_descriptions=['test'])
] * 5
for d in data:
loader.load(d)
loader.close()
expected = [(
'{"key": "test_table_key", "column_descriptions": ["test_comment1", "test_comment2"], '
'"schema": "test_schema", "database": "test_database", "cluster": "test_cluster", '
'"column_names": ["test_col1", "test_col2"], "name": "test_table", '
'"last_updated_timestamp": 123456789, "display_name": "test_schema.test_table", '
'"description": "test_description", "unique_usage": 5, "total_usage": 10, '
'"tags": ["test_tag1", "test_tag2"], "schema_description": "schema_description", '
'"programmatic_descriptions":["test"], '
'"badges": ["badge1"]}')] * 5
self._check_results_helper(expected=expected)
def test_loading_with_list_of_objects(self):
# type: () -> None
"""
Test Loading functionality with list of objects.
Check to ensure all objects are added to file
"""
loader = FSElasticsearchJSONLoader()
loader.init(conf=Scoped.get_scoped_conf(conf=self.conf,
scope=loader.get_scope()))
data = [
ElasticsearchDocument(
elasticsearch_index='test_es_index',
elasticsearch_type='test_es_type',
database='test_database',
cluster='test_cluster',
schema_name='test_schema',
table_name='test_table',
table_key='test_table_key',
table_last_updated_epoch=123456789,
table_description='test_description',
column_names=['test_col1', 'test_col2'],
column_descriptions=['test_comment1', 'test_comment2'],
total_usage=10,
unique_usage=5,
tag_names=['test_tag1', 'test_tag2'])
] * 5
for d in data:
loader.load(d)
loader.close()
expected = [
'{"index": {"_type": "test_es_type", "_index": "test_es_index"}}',
('{"table_key": "test_table_key", "column_descriptions": ["test_comment1", "test_comment2"], '
'"schema_name": "test_schema", "database": "test_database", "cluster": "test_cluster", '
'"column_names": ["test_col1", "test_col2"], "table_name": "test_table", '
'"table_last_updated_epoch": 123456789,'
'"table_description": "test_description", "unique_usage": 5, "total_usage": 10, '
'"tag_names": ["test_tag1", "test_tag2"]}')
] * 5
self._check_results_helper(expected=expected)