def test_fetch_entity_classifications_with_duplicates_should_return(self): returned_entity = utils.MockedObject() cursor = utils.MockedObject() cursor._data = { 'list': [{ 'typeName': 'Log Data', 'entityGuid': '30cfc9fc-aec4-4017-b649-f1f92351a727', 'propagate': True }, { 'typeName': 'ETL', 'entityGuid': 'e73656c9-da05-4db7-9b88-e3669c1a98eb', 'propagate': True }, { 'typeName': 'ETL', 'entityGuid': '30cfc9fc-aec4-4017-b649-f1f92351a727', 'propagate': True }] } returned_entity.classifications = iter([cursor]) self.__atlas_client.entity_guid.return_value = returned_entity response = self.__atlas_facade.fetch_entity_classifications( '30cfc9fc-aec4-4017-b649-f1f92351a727') self.assertEqual(2, len(response)) self.assertEqual('30cfc9fc-aec4-4017-b649-f1f92351a727', response[1]['entityGuid'])
def test_search_entities_from_entity_type_paged_should_return(self): search_result = utils.MockedObject() table_1 = utils.MockedObject() table_1._data = { 'typeName': 'Table', 'attributes': { 'owner': 'Joe', 'createTime': 1589983835754, 'qualifiedName': 'sales_fact', 'name': 'sales_fact', 'description': 'sales fact table' }, 'guid': '2286a6bd-4b06-4f7c-a666-920d774b040e', 'status': 'ACTIVE', 'displayText': 'sales_fact', 'classificationNames': ['Fact'] } search_result.entities = [table_1] search_results = [search_result] search_result_2 = utils.MockedObject() table_2 = utils.MockedObject() table_2._data = { 'typeName': 'Table', 'attributes': { 'owner': 'Tim ETL', 'createTime': 1589983847754, 'qualifiedName': 'log_fact_daily_mv', 'name': 'log_fact_daily_mv', 'description': 'log fact daily materialized ' 'view' }, 'guid': '30cfc9fc-aec4-4017-b649-f1f92351a727', 'status': 'ACTIVE', 'displayText': 'log_fact_daily_mv', 'classificationNames': ['Log Data'] } search_result_2.entities = [table_2] search_results_2 = [search_result_2] self.__atlas_client.search_dsl.side_effect = [ search_results, search_results_2, [] ] response = self.__atlas_facade.search_entities_from_entity_type( 'Table') self.assertDictEqual(table_1._data, response[0]._data) self.assertDictEqual(table_2._data, response[1]._data) self.assertEqual(2, len(response))
def test_scrape_should_return_metadata(self, enrich_entity_classifications, enrich_entity_relationships): # Step 1 - create the lazy object returned by Apache Atlas facade typedef = utils.MockedObject() typedefs = [typedef] self.__apache_atlas_facade.get_typedefs.return_value = typedefs # Step 2 - create the return for Atlas classifications classifications_defs = self.__make_classification_object() typedef.classificationDefs = classifications_defs # Step 3 - create the return for Atlas Enum rypes typedef.enumDefs = self.__make_enum_types_object() # Step 4 - create the return for Atlas Entity types entity_types = self.__make_entity_type_object() typedef.entityDefs = entity_types # Following steps are executed for each entity type. # Step 5 - create the return for search results search_results = self.__make_search_results_object() self.__apache_atlas_facade.\ search_entities_from_entity_type.return_value = search_results # Step 6 - create the return for fetched entities fetched_entities_dict = \ utils.Utils.convert_json_to_object(self.__MODULE_PATH, 'fetched_entities_dict.json') self.__apache_atlas_facade.fetch_entities.\ return_value = fetched_entities_dict # Step 7 - create the return for entities classifications entity_classifications = \ utils.Utils.convert_json_to_object(self.__MODULE_PATH, 'entity_classifications.json') self.__apache_atlas_facade.\ fetch_entity_classifications.return_value = entity_classifications metadata, _ = self.__scrape.get_metadata() types_count = 51 self.assertEqual(3, len(metadata)) self.assertEqual(types_count, len(metadata['entity_types'])) self.assertEqual(8, len(metadata['entity_types']['Table']['entities'])) expected_table_metadata = utils.Utils.convert_json_to_object( self.__MODULE_PATH, 'expected_table_metadata.json') self.assertDictEqual(expected_table_metadata, metadata['entity_types']['Table']) enrich_entity_relationships.assert_called_once() self.__apache_atlas_facade.get_typedefs.assert_called_once() self.assertEqual( types_count, self.__apache_atlas_facade. search_entities_from_entity_type.call_count) self.assertEqual(types_count, self.__apache_atlas_facade.fetch_entities.call_count) self.assertEqual(types_count, enrich_entity_classifications.call_count)
def __make_enum_types_object(cls): enum_types_obj = [] enum_type_obj = utils.MockedObject() enum_type_obj.name = 'my_enum_type' enum_type_obj._data = {} enum_type_obj.guid = '123' enum_types_obj.append(enum_type_obj) return enum_types_obj
def __make_search_results_object(self): search_results = \ utils.Utils.convert_json_to_object(self.__MODULE_PATH, 'search_results.json') search_results_obj = [] for search_result in search_results: search_result_obj = utils.MockedObject() search_result_obj.guid = search_result['guid'] search_result_obj._data = search_result['data'] search_results_obj.append(search_result_obj) return search_results_obj
def test_fetch_entity_classifications_error_should_not_raise_it(self): returned_entity = utils.MockedObject() cursor = utils.MockedObject() cursor._data = { 'list': [{ 'typeName': 'Log Data', 'entityGuid': '30cfc9fc-aec4-4017-b649-f1f92351a727', 'propagate': True }, { 'typeName': 'ETL', 'entityGuid': 'e73656c9-da05-4db7-9b88-e3669c1a98eb', 'propagate': True }] } returned_entity.classifications = iter([]) self.__atlas_client.entity_guid.return_value = returned_entity response = self.__atlas_facade.fetch_entity_classifications( ['30cfc9fc-aec4-4017-b649-f1f92351a727']) self.assertIsNone(response)
def __make_entity_type_object(self): entity_types = \ utils.Utils.convert_json_to_object(self.__MODULE_PATH, 'entity_types.json') entity_types_obj = [] for entity_type in entity_types: entity_type_obj = utils.MockedObject() entity_type_obj.name = entity_type['name'] entity_type_obj.superTypes = entity_type['superTypes'] entity_type_obj._data = entity_type['data'] entity_types_obj.append(entity_type_obj) return entity_types_obj
def __make_classification_object(self): classifications = \ utils.Utils.convert_json_to_object(self.__MODULE_PATH, 'classifications.json') classifications_obj = [] for classification in classifications: classification_obj = utils.MockedObject() classification_obj.name = classification['name'] classification_obj.guid = classification['guid'] classification_obj._data = classification['data'] classifications_obj.append(classification_obj) return classifications_obj
def __make_delete_classification_events_object(self, event_name): event_msgs = \ utils.Utils.convert_json_to_object(self.__MODULE_PATH, '{}.json'.format(event_name)) events_obj = [] for event_msg in event_msgs: event_obj = utils.MockedObject() event_obj.topic = event_msg['topic'] event_obj.partition = event_msg['partition'] event_obj.offset = event_msg['offset'] event_obj.key = event_msg['key'] event_obj.value = json.dumps(event_msg['value']).encode('utf-8') events_obj.append(event_obj) return events_obj
def __create_scrape_event_scenario(self, event_name): # Step 1 - Set up event message event_msgs = self.__make_delete_classification_events_object( event_name) self.__apache_atlas_event_facade.create_event_consumer.return_value =\ event_msgs # Step 2 - create the lazy object returned by Apache Atlas facade typedef = utils.MockedObject() typedefs = [typedef] self.__apache_atlas_facade.get_typedefs.return_value = typedefs # Step 3 - create the return for Atlas classifications classifications_defs = self.__make_classification_object() typedef.classificationDefs = classifications_defs # Step 4 - create the return for Atlas Enum rypes typedef.enumDefs = self.__make_enum_types_object() # Step 5 - create the return for Atlas Entity types entity_types = self.__make_entity_type_object() typedef.entityDefs = entity_types # Following steps are executed for each entity type. # Step 6 - create the return for search results search_results = self.__make_search_results_object() self.__apache_atlas_facade.search_entities_from_entity_type.\ return_value = search_results # Step 7 - create the return for fetched entities fetched_entities_dict =\ utils.Utils.convert_json_to_object(self.__MODULE_PATH, 'fetched_entities_dict.json') self.__apache_atlas_facade.fetch_entities.return_value =\ fetched_entities_dict # Step 8 - create the return for entities classifications entity_classifications =\ utils.Utils.convert_json_to_object(self.__MODULE_PATH, 'entity_classifications.json') self.__apache_atlas_facade.fetch_entity_classifications.return_value =\ entity_classifications metadata, _ = self.__scrape.get_metadata() return metadata
def test_get_admin_metrics_should_succeed(self): metric = utils.MockedObject() metric.entity = { 'entityActive': { 'AtlasGlossary': 2, 'StorageDesc': 1, 'Table': 8, 'Column': 17, 'AtlasGlossaryTerm': 44, 'AtlasGlossaryCategory': 23, 'View': 2, 'DB': 3, 'LoadProcess': 3 }, 'entityDeleted': { 'hbase_table': 2 } } metric.general = { 'collectionTime': 1590773837477, 'entityCount': 105, 'tagCount': 13, 'typeUnusedCount': 47, 'typeCount': 138 } self.__atlas_client.admin_metrics = [metric] response = self.__atlas_facade.get_admin_metrics()[0] self.assertDictEqual(metric.entity['entityActive'], response['entityActive']) self.assertDictEqual(metric.entity['entityDeleted'], response['entityDeleted']) self.assertDictEqual(metric.general, response['generalMetrics'])
def test_fetch_entities_should_return(self): table_1 = utils.MockedObject() guid = '2286a6bd-4b06-4f7c-a666-920d774b040e' table_1.guid = guid table_1._data = { 'typeName': 'Table', 'attributes': { 'owner': 'Joe', 'temporary': False, 'lastAccessTime': 1589983835754, 'replicatedTo': None, 'replicatedFrom': None, 'qualifiedName': 'sales_fact', 'columns': [{ 'guid': 'c53f1ab7-970f-4199-bc9a-899291c93622', 'typeName': 'Column' }, { 'guid': 'e5188d7b-916e-4420-b02e-8eac75b17a68', 'typeName': 'Column' }, { 'guid': '84d83e16-e767-46f9-8165-db7e3a7a3762', 'typeName': 'Column' }, { 'guid': '909805b1-59f0-44db-b0c8-b6b0c384d2c5', 'typeName': 'Column' }], 'description': 'sales fact table', 'viewExpandedText': None, 'sd': { 'guid': 'a0989a5b-19aa-4ac4-92db-e2449554c799', 'typeName': 'StorageDesc' }, 'tableType': 'Managed', 'createTime': 1589983835754, 'name': 'sales_fact', 'additionalProperties': None, 'db': { 'guid': 'c0ebcb5e-21f8-424d-bf11-f39bb943ae2c', 'typeName': 'DB' }, 'retention': 1589983835754, 'viewOriginalText': None }, 'guid': guid, 'status': 'ACTIVE', 'createdBy': 'admin', 'updatedBy': 'admin', 'createTime': 1589983835765, 'updateTime': 1589983850688, 'version': 0, 'relationshipAttributes': { 'schema': [], 'inputToProcesses': [{ 'guid': '2aa15a65-ae8e-4e48-9b77-2ecb049f82b4', 'typeName': 'LoadProcess', 'displayText': 'loadSalesDaily', 'relationshipGuid': '95d240a4-009b-48c8-b1d3-66703720f60a', 'relationshipStatus': 'ACTIVE', 'relationshipAttributes': { 'typeName': 'dataset_process_inputs' } }], 'relatedFromObjectAnnotations': [], 'resourceListAnchors': [], 'relatedToObjectAnnotations': None, 'supportingResources': [], 'meanings': [], 'outputFromProcesses': [] } } collection_item = utils.MockedObject() entities = [table_1] collection_item.entities = lambda: entities bulk_collection = [collection_item] self.__atlas_client.entity_bulk.return_value = bulk_collection response = self.__atlas_facade.fetch_entities([guid]) self.assertEqual(1, len(response)) self.assertEqual(table_1.guid, response[guid]['guid']) self.assertDictEqual(table_1._data, response[guid]['data'])