def test_fetch_entity_classifications_with_duplicates_should_return(self):
returned_entity = utils.MockedObject()
cursor = utils.MockedObject()
cursor._data = {
'list': [{
'typeName': 'Log Data',
'entityGuid': '30cfc9fc-aec4-4017-b649-f1f92351a727',
'propagate': True
}, {
'typeName': 'ETL',
'entityGuid': 'e73656c9-da05-4db7-9b88-e3669c1a98eb',
'propagate': True
}, {
'typeName': 'ETL',
'entityGuid': '30cfc9fc-aec4-4017-b649-f1f92351a727',
'propagate': True
}]
}
returned_entity.classifications = iter([cursor])
self.__atlas_client.entity_guid.return_value = returned_entity
response = self.__atlas_facade.fetch_entity_classifications(
'30cfc9fc-aec4-4017-b649-f1f92351a727')
self.assertEqual(2, len(response))
self.assertEqual('30cfc9fc-aec4-4017-b649-f1f92351a727',
response[1]['entityGuid'])
def test_search_entities_from_entity_type_paged_should_return(self):
search_result = utils.MockedObject()
table_1 = utils.MockedObject()
table_1._data = {
'typeName': 'Table',
'attributes': {
'owner': 'Joe',
'createTime': 1589983835754,
'qualifiedName': 'sales_fact',
'name': 'sales_fact',
'description': 'sales fact table'
},
'guid': '2286a6bd-4b06-4f7c-a666-920d774b040e',
'status': 'ACTIVE',
'displayText': 'sales_fact',
'classificationNames': ['Fact']
}
search_result.entities = [table_1]
search_results = [search_result]
search_result_2 = utils.MockedObject()
table_2 = utils.MockedObject()
table_2._data = {
'typeName': 'Table',
'attributes': {
'owner': 'Tim ETL',
'createTime': 1589983847754,
'qualifiedName': 'log_fact_daily_mv',
'name': 'log_fact_daily_mv',
'description': 'log fact daily materialized '
'view'
},
'guid': '30cfc9fc-aec4-4017-b649-f1f92351a727',
'status': 'ACTIVE',
'displayText': 'log_fact_daily_mv',
'classificationNames': ['Log Data']
}
search_result_2.entities = [table_2]
search_results_2 = [search_result_2]
self.__atlas_client.search_dsl.side_effect = [
search_results, search_results_2, []
]
response = self.__atlas_facade.search_entities_from_entity_type(
'Table')
self.assertDictEqual(table_1._data, response[0]._data)
self.assertDictEqual(table_2._data, response[1]._data)
self.assertEqual(2, len(response))
def test_scrape_should_return_metadata(self, enrich_entity_classifications,
enrich_entity_relationships):
# Step 1 - create the lazy object returned by Apache Atlas facade
typedef = utils.MockedObject()
typedefs = [typedef]
self.__apache_atlas_facade.get_typedefs.return_value = typedefs
# Step 2 - create the return for Atlas classifications
classifications_defs = self.__make_classification_object()
typedef.classificationDefs = classifications_defs
# Step 3 - create the return for Atlas Enum rypes
typedef.enumDefs = self.__make_enum_types_object()
# Step 4 - create the return for Atlas Entity types
entity_types = self.__make_entity_type_object()
typedef.entityDefs = entity_types
# Following steps are executed for each entity type.
# Step 5 - create the return for search results
search_results = self.__make_search_results_object()
self.__apache_atlas_facade.\
search_entities_from_entity_type.return_value = search_results
# Step 6 - create the return for fetched entities
fetched_entities_dict = \
utils.Utils.convert_json_to_object(self.__MODULE_PATH,
'fetched_entities_dict.json')
self.__apache_atlas_facade.fetch_entities.\
return_value = fetched_entities_dict
# Step 7 - create the return for entities classifications
entity_classifications = \
utils.Utils.convert_json_to_object(self.__MODULE_PATH,
'entity_classifications.json')
self.__apache_atlas_facade.\
fetch_entity_classifications.return_value = entity_classifications
metadata, _ = self.__scrape.get_metadata()
types_count = 51
self.assertEqual(3, len(metadata))
self.assertEqual(types_count, len(metadata['entity_types']))
self.assertEqual(8, len(metadata['entity_types']['Table']['entities']))
expected_table_metadata = utils.Utils.convert_json_to_object(
self.__MODULE_PATH, 'expected_table_metadata.json')
self.assertDictEqual(expected_table_metadata,
metadata['entity_types']['Table'])
enrich_entity_relationships.assert_called_once()
self.__apache_atlas_facade.get_typedefs.assert_called_once()
self.assertEqual(
types_count, self.__apache_atlas_facade.
search_entities_from_entity_type.call_count)
self.assertEqual(types_count,
self.__apache_atlas_facade.fetch_entities.call_count)
self.assertEqual(types_count, enrich_entity_classifications.call_count)
def __make_enum_types_object(cls):
enum_types_obj = []
enum_type_obj = utils.MockedObject()
enum_type_obj.name = 'my_enum_type'
enum_type_obj._data = {}
enum_type_obj.guid = '123'
enum_types_obj.append(enum_type_obj)
return enum_types_obj
def __make_search_results_object(self):
search_results = \
utils.Utils.convert_json_to_object(self.__MODULE_PATH,
'search_results.json')
search_results_obj = []
for search_result in search_results:
search_result_obj = utils.MockedObject()
search_result_obj.guid = search_result['guid']
search_result_obj._data = search_result['data']
search_results_obj.append(search_result_obj)
return search_results_obj
def test_fetch_entity_classifications_error_should_not_raise_it(self):
returned_entity = utils.MockedObject()
cursor = utils.MockedObject()
cursor._data = {
'list': [{
'typeName': 'Log Data',
'entityGuid': '30cfc9fc-aec4-4017-b649-f1f92351a727',
'propagate': True
}, {
'typeName': 'ETL',
'entityGuid': 'e73656c9-da05-4db7-9b88-e3669c1a98eb',
'propagate': True
}]
}
returned_entity.classifications = iter([])
self.__atlas_client.entity_guid.return_value = returned_entity
response = self.__atlas_facade.fetch_entity_classifications(
['30cfc9fc-aec4-4017-b649-f1f92351a727'])
self.assertIsNone(response)
def __make_entity_type_object(self):
entity_types = \
utils.Utils.convert_json_to_object(self.__MODULE_PATH,
'entity_types.json')
entity_types_obj = []
for entity_type in entity_types:
entity_type_obj = utils.MockedObject()
entity_type_obj.name = entity_type['name']
entity_type_obj.superTypes = entity_type['superTypes']
entity_type_obj._data = entity_type['data']
entity_types_obj.append(entity_type_obj)
return entity_types_obj
def __make_classification_object(self):
classifications = \
utils.Utils.convert_json_to_object(self.__MODULE_PATH,
'classifications.json')
classifications_obj = []
for classification in classifications:
classification_obj = utils.MockedObject()
classification_obj.name = classification['name']
classification_obj.guid = classification['guid']
classification_obj._data = classification['data']
classifications_obj.append(classification_obj)
return classifications_obj
def __make_delete_classification_events_object(self, event_name):
event_msgs = \
utils.Utils.convert_json_to_object(self.__MODULE_PATH,
'{}.json'.format(event_name))
events_obj = []
for event_msg in event_msgs:
event_obj = utils.MockedObject()
event_obj.topic = event_msg['topic']
event_obj.partition = event_msg['partition']
event_obj.offset = event_msg['offset']
event_obj.key = event_msg['key']
event_obj.value = json.dumps(event_msg['value']).encode('utf-8')
events_obj.append(event_obj)
return events_obj
def __create_scrape_event_scenario(self, event_name):
# Step 1 - Set up event message
event_msgs = self.__make_delete_classification_events_object(
event_name)
self.__apache_atlas_event_facade.create_event_consumer.return_value =\
event_msgs
# Step 2 - create the lazy object returned by Apache Atlas facade
typedef = utils.MockedObject()
typedefs = [typedef]
self.__apache_atlas_facade.get_typedefs.return_value = typedefs
# Step 3 - create the return for Atlas classifications
classifications_defs = self.__make_classification_object()
typedef.classificationDefs = classifications_defs
# Step 4 - create the return for Atlas Enum rypes
typedef.enumDefs = self.__make_enum_types_object()
# Step 5 - create the return for Atlas Entity types
entity_types = self.__make_entity_type_object()
typedef.entityDefs = entity_types
# Following steps are executed for each entity type.
# Step 6 - create the return for search results
search_results = self.__make_search_results_object()
self.__apache_atlas_facade.search_entities_from_entity_type.\
return_value = search_results
# Step 7 - create the return for fetched entities
fetched_entities_dict =\
utils.Utils.convert_json_to_object(self.__MODULE_PATH,
'fetched_entities_dict.json')
self.__apache_atlas_facade.fetch_entities.return_value =\
fetched_entities_dict
# Step 8 - create the return for entities classifications
entity_classifications =\
utils.Utils.convert_json_to_object(self.__MODULE_PATH,
'entity_classifications.json')
self.__apache_atlas_facade.fetch_entity_classifications.return_value =\
entity_classifications
metadata, _ = self.__scrape.get_metadata()
return metadata
def test_get_admin_metrics_should_succeed(self):
metric = utils.MockedObject()
metric.entity = {
'entityActive': {
'AtlasGlossary': 2,
'StorageDesc': 1,
'Table': 8,
'Column': 17,
'AtlasGlossaryTerm': 44,
'AtlasGlossaryCategory': 23,
'View': 2,
'DB': 3,
'LoadProcess': 3
},
'entityDeleted': {
'hbase_table': 2
}
}
metric.general = {
'collectionTime': 1590773837477,
'entityCount': 105,
'tagCount': 13,
'typeUnusedCount': 47,
'typeCount': 138
}
self.__atlas_client.admin_metrics = [metric]
response = self.__atlas_facade.get_admin_metrics()[0]
self.assertDictEqual(metric.entity['entityActive'],
response['entityActive'])
self.assertDictEqual(metric.entity['entityDeleted'],
response['entityDeleted'])
self.assertDictEqual(metric.general, response['generalMetrics'])
def test_fetch_entities_should_return(self):
table_1 = utils.MockedObject()
guid = '2286a6bd-4b06-4f7c-a666-920d774b040e'
table_1.guid = guid
table_1._data = {
'typeName': 'Table',
'attributes': {
'owner':
'Joe',
'temporary':
False,
'lastAccessTime':
1589983835754,
'replicatedTo':
None,
'replicatedFrom':
None,
'qualifiedName':
'sales_fact',
'columns': [{
'guid': 'c53f1ab7-970f-4199-bc9a-899291c93622',
'typeName': 'Column'
}, {
'guid': 'e5188d7b-916e-4420-b02e-8eac75b17a68',
'typeName': 'Column'
}, {
'guid': '84d83e16-e767-46f9-8165-db7e3a7a3762',
'typeName': 'Column'
}, {
'guid': '909805b1-59f0-44db-b0c8-b6b0c384d2c5',
'typeName': 'Column'
}],
'description':
'sales fact table',
'viewExpandedText':
None,
'sd': {
'guid': 'a0989a5b-19aa-4ac4-92db-e2449554c799',
'typeName': 'StorageDesc'
},
'tableType':
'Managed',
'createTime':
1589983835754,
'name':
'sales_fact',
'additionalProperties':
None,
'db': {
'guid': 'c0ebcb5e-21f8-424d-bf11-f39bb943ae2c',
'typeName': 'DB'
},
'retention':
1589983835754,
'viewOriginalText':
None
},
'guid': guid,
'status': 'ACTIVE',
'createdBy': 'admin',
'updatedBy': 'admin',
'createTime': 1589983835765,
'updateTime': 1589983850688,
'version': 0,
'relationshipAttributes': {
'schema': [],
'inputToProcesses': [{
'guid': '2aa15a65-ae8e-4e48-9b77-2ecb049f82b4',
'typeName': 'LoadProcess',
'displayText': 'loadSalesDaily',
'relationshipGuid': '95d240a4-009b-48c8-b1d3-66703720f60a',
'relationshipStatus': 'ACTIVE',
'relationshipAttributes': {
'typeName': 'dataset_process_inputs'
}
}],
'relatedFromObjectAnnotations': [],
'resourceListAnchors': [],
'relatedToObjectAnnotations':
None,
'supportingResources': [],
'meanings': [],
'outputFromProcesses': []
}
}
collection_item = utils.MockedObject()
entities = [table_1]
collection_item.entities = lambda: entities
bulk_collection = [collection_item]
self.__atlas_client.entity_bulk.return_value = bulk_collection
response = self.__atlas_facade.fetch_entities([guid])
self.assertEqual(1, len(response))
self.assertEqual(table_1.guid, response[guid]['guid'])
self.assertDictEqual(table_1._data, response[guid]['data'])
