def __make_assembled_entry_for_query(self, assembled_query_metadata, tag_template): entry_id, entry = \ self.__datacatalog_entry_factory.make_entry_for_query( assembled_query_metadata.query) tags = [] if tag_template: tags.append( self.__datacatalog_tag_factory.make_tag_for_query( tag_template, assembled_query_metadata)) return prepare.AssembledEntryData(entry_id, entry, tags)
def test_fulfill_tag_fields_should_resolve_visualization_app_mapping(self): app_id = 'test-app' app_entry = self.__make_fake_entry(app_id, 'app') app_tag = self.__make_fake_tag(string_fields=(('id', app_id),)) visualization_id = 'test-measure' visualization_entry = self.__make_fake_entry(visualization_id, 'visualization') string_fields = ('id', visualization_id), ('app_id', app_id) visualization_tag = self.__make_fake_tag(string_fields=string_fields) app_assembled_entry = commons_prepare.AssembledEntryData( app_id, app_entry, [app_tag]) measure_assembled_entry = commons_prepare.AssembledEntryData( visualization_id, visualization_entry, [visualization_tag]) prepare.EntryRelationshipMapper().fulfill_tag_fields( [app_assembled_entry, measure_assembled_entry]) self.assertEqual( 'https://console.cloud.google.com/datacatalog/' 'fake_entries/test-app', visualization_tag.fields['app_entry'].string_value)
def test_fulfill_tag_fields_should_resolve_element_look_mapping(self): look_id = 123 look_entry = self.__make_fake_entry(look_id, 'look') look_tag = self.__make_fake_tag(double_fields=(('id', look_id), )) element_id = 'test_element' element_entry = self.__make_fake_entry(element_id, 'dashboard_element') double_fields = ('look_id', look_id), element_tag = self.__make_fake_tag(string_fields=(('id', element_id), ), double_fields=double_fields) look_assembled_entry = commons_prepare.AssembledEntryData( look_id, look_entry, [look_tag]) element_assembled_entry = commons_prepare.AssembledEntryData( element_id, element_entry, [element_tag]) prepare.EntryRelationshipMapper().fulfill_tag_fields( [look_assembled_entry, element_assembled_entry]) self.assertEqual( f'https://console.cloud.google.com/datacatalog/{look_entry.name}', element_tag.fields['look_entry'].string_value)
def test_fulfill_tag_fields_should_resolve_sheet_workbook_mapping(self): workbook_luid = 'test_workbook' workbook_entry = self.__make_fake_entry(workbook_luid, 'workbook') workbook_tag = self.__make_fake_tag(string_fields=(('luid', workbook_luid), )) sheet_luid = 'test_dashboard' sheet_entry = self.__make_fake_entry(sheet_luid, 'sheet') string_fields = ('luid', sheet_luid), ('workbook_luid', workbook_luid) sheet_tag = self.__make_fake_tag(string_fields=string_fields) workbook_assembled_entry = commons_prepare.AssembledEntryData( workbook_luid, workbook_entry, [workbook_tag]) sheet_assembled_entry = commons_prepare.AssembledEntryData( sheet_luid, sheet_entry, [sheet_tag]) prepare.EntryRelationshipMapper().fulfill_tag_fields( [workbook_assembled_entry, sheet_assembled_entry]) self.assertEqual( f'https://console.cloud.google.com/datacatalog/' f'{workbook_entry.name}', sheet_tag.fields['workbook_entry'].string_value)
def test_fulfill_tag_fields_should_resolve_element_dashboard_mapping(self): dashboard_id = 'test_dashboard' dashboard_entry = self.__make_fake_entry(dashboard_id, 'dashboard') string_fields = ('id', dashboard_id), dashboard_tag = self.__make_fake_tag(string_fields=string_fields) element_id = 'test_element' element_entry = self.__make_fake_entry(element_id, 'dashboard_element') string_fields = ('id', element_id), ('dashboard_id', dashboard_id) element_tag = self.__make_fake_tag(string_fields=string_fields) dashboard_assembled_entry = commons_prepare.AssembledEntryData( dashboard_id, dashboard_entry, [dashboard_tag]) element_assembled_entry = commons_prepare.AssembledEntryData( element_id, element_entry, [element_tag]) prepare.EntryRelationshipMapper().fulfill_tag_fields( [dashboard_assembled_entry, element_assembled_entry]) self.assertEqual( f'https://console.cloud.google.com/datacatalog/' f'{dashboard_entry.name}', element_tag.fields['dashboard_entry'].string_value)
def test_fulfill_tag_fields_should_resolve_element_query_mapping(self): query_id = 1080 query_entry = self.__make_fake_entry(query_id, 'query') query_tag = self.__make_fake_tag(double_fields=(('id', query_id),)) element_id = 'test_element' element_entry = self.__make_fake_entry(element_id, 'dashboard_element') double_fields = ('query_id', query_id), element_tag = self.__make_fake_tag(string_fields=(('id', element_id),), double_fields=double_fields) query_assembled_entry = commons_prepare.AssembledEntryData( query_id, query_entry, [query_tag]) element_assembled_entry = commons_prepare.AssembledEntryData( element_id, element_entry, [element_tag]) prepare.EntryRelationshipMapper().fulfill_tag_fields( [query_assembled_entry, element_assembled_entry]) self.assertEqual( f'https://console.cloud.google.com/datacatalog/' f'{query_entry.name}', element_tag.fields['query_entry'].string_value)
def test_fulfill_tag_fields_should_resolve_look_folder_mapping(self): folder_id = 'test_folder' folder_entry = self.__make_fake_entry(folder_id, 'folder') folder_tag = self.__make_fake_tag(string_fields=(('id', folder_id),)) look_id = 123 look_entry = self.__make_fake_entry(look_id, 'look') string_fields = ('folder_id', folder_id), look_tag = self.__make_fake_tag(string_fields=string_fields, double_fields=(('id', look_id),)) folder_assembled_entry = commons_prepare.AssembledEntryData( folder_id, folder_entry, [folder_tag]) look_assembled_entry = commons_prepare.AssembledEntryData( look_id, look_entry, [look_tag]) prepare.EntryRelationshipMapper().fulfill_tag_fields( [folder_assembled_entry, look_assembled_entry]) self.assertEqual( f'https://console.cloud.google.com/datacatalog/' f'{folder_entry.name}', look_tag.fields['folder_entry'].string_value)
def __make_entry_for_sheet(self, sheet_metadata, workbook_metadata, tag_template): entry_id, entry = \ self.__datacatalog_entry_factory.make_entry_for_sheet( sheet_metadata, workbook_metadata) tags = [] if tag_template: tags.append( datacatalog_tag_factory.DataCatalogTagFactory. make_tag_for_sheet(tag_template, sheet_metadata, workbook_metadata)) return prepare.AssembledEntryData(entry_id, entry, tags)
def test_fulfill_tag_fields_should_resolve_parent_folder_mapping(self): parent_folder_id = 'parent-folder' parent_folder_entry = self.__make_fake_entry(parent_folder_id, 'Folder') parent_folder_tag = self.__make_fake_tag( string_fields=(('id', parent_folder_id), )) folder_id = 'test-folder' folder_entry = self.__make_fake_entry(folder_id, 'Folder') string_fields = ('id', folder_id), ('parent_id', parent_folder_id) folder_tag = self.__make_fake_tag(string_fields=string_fields) parent_folder_assembled_entry = commons_prepare.AssembledEntryData( parent_folder_id, parent_folder_entry, [parent_folder_tag]) folder_assembled_entry = commons_prepare.AssembledEntryData( folder_id, folder_entry, [folder_tag]) prepare.EntryRelationshipMapper().fulfill_tag_fields( [parent_folder_assembled_entry, folder_assembled_entry]) self.assertEqual( f'https://console.cloud.google.com/datacatalog/' f'{parent_folder_entry.name}', folder_tag.fields['parent_folder_entry'].string_value)
def test_resolve_folder_mappings_should_skip_non_folder_entries( self, mock_map_related_entry): entry_id = 'test' entry = self.__make_fake_entry(entry_id, 'not-a-folder') tag = self.__make_fake_tag(string_fields=(('id', entry_id), )) assembled_entry = commons_prepare.AssembledEntryData( entry_id, entry, [tag]) prepare.EntryRelationshipMapper()\ ._EntryRelationshipMapper__resolve_folder_mappings( [assembled_entry], {}) mock_map_related_entry.assert_not_called()
def make_assembled_entry_for_custom_property_def( self, custom_property_def_metadata, tag_template): entry_id, entry = self.__datacatalog_entry_factory\ .make_entry_for_custom_property_definition( custom_property_def_metadata) tags = [] if tag_template: tags.append( self.__datacatalog_tag_factory. make_tag_for_custom_property_definition( tag_template, custom_property_def_metadata)) return prepare.AssembledEntryData(entry_id, entry, tags)
def __make_assembled_entry_for_dashboard_element(self, element, dashboard, tag_template): entry_id, entry = self.__datacatalog_entry_factory.\ make_entry_for_dashboard_element(element) if not (entry_id and entry): return tags = [] if tag_template: tags.append( self.__datacatalog_tag_factory.make_tag_for_dashboard_element( tag_template, element, dashboard)) return prepare.AssembledEntryData(entry_id, entry, tags)
def __make_assembled_entry_for_sheet(self, sheet_metadata, tag_templates_dict): entry_id, entry = \ self.__datacatalog_entry_factory.make_entry_for_sheet( sheet_metadata) tag_template = tag_templates_dict.get(constants.TAG_TEMPLATE_ID_SHEET) tags = [] if tag_template: tags.append( self.__datacatalog_tag_factory.make_tag_for_sheet( tag_template, sheet_metadata)) return prepare.AssembledEntryData(entry_id, entry, tags)
def __make_entries_for_table_container(self, table_container): entry_id, entry = \ self.__datacatalog_entry_factory.make_entries_for_table_container( table_container) tags = [] if self.__tag_templates_dict: tag_template_id = '{}_{}_metadata'.format( self.__entry_group_id, self.__metadata_definition['table_container_def']['type']) tag_template = self.__tag_templates_dict[tag_template_id] tags.append( self.__datacatalog_tag_factory. make_tag_for_table_container_metadata(tag_template, table_container)) return prepare.AssembledEntryData(entry_id, entry, tags)
def create_assembled_entry_user_defined_type(cls, entry_id, entry_type, system, display_name, name, description, linked_resource, create_time, update_time, columns=None): entry = cls.create_entry_user_defined_type( entry_type, system, display_name, name, description, linked_resource, create_time, update_time, columns) return prepare.AssembledEntryData(entry_id, entry, None)
def __make_assembled_entry_for_dimension(self, dimension_metadata, tag_templates_dict): entry_id, entry = \ self.__datacatalog_entry_factory.make_entry_for_dimension( dimension_metadata) tag_template = tag_templates_dict.get( constants.TAG_TEMPLATE_ID_DIMENSION) tags = [] if tag_template: tags.append( self.__datacatalog_tag_factory.make_tag_for_dimension( tag_template, dimension_metadata)) return prepare.AssembledEntryData(entry_id, entry, tags)
def __make_assembled_entry_for_visualization(self, visualization_metadata, tag_templates_dict): entry_id, entry = \ self.__datacatalog_entry_factory.make_entry_for_visualization( visualization_metadata) tag_template = tag_templates_dict.get( constants.TAG_TEMPLATE_ID_VISUALIZATION) tags = [] if tag_template: tags.append( self.__datacatalog_tag_factory.make_tag_for_visualization( tag_template, visualization_metadata)) return prepare.AssembledEntryData(entry_id, entry, tags)
def __make_assembled_entry_for_app(self, app_metadata, tag_templates_dict): entry_id, entry = \ self.__datacatalog_entry_factory.make_entry_for_app(app_metadata) app_tag_template = tag_templates_dict.get( constants.TAG_TEMPLATE_ID_APP) tags = [] if app_tag_template: tags.append( self.__datacatalog_tag_factory.make_tag_for_app( app_tag_template, app_metadata)) tags.extend( self.__datacatalog_tag_factory.make_tags_for_custom_properties( tag_templates_dict, app_metadata.get('customProperties'))) return prepare.AssembledEntryData(entry_id, entry, tags)
def __make_assembled_entry_for_folder( self, folder_metadata: Dict[str, Any], tag_templates: Dict[str, TagTemplate]) -> AssembledEntryData: entry_id, entry = \ self.__datacatalog_entry_factory.make_entry_for_folder( folder_metadata) tags = [] tag_template = tag_templates.get(constants.TAG_TEMPLATE_ID_FOLDER) if tag_template: tags.append( self.__datacatalog_tag_factory.make_tag_for_folder( tag_template, folder_metadata)) return prepare.AssembledEntryData(entry_id=entry_id, entry=entry, tags=tags)
def __make_entry_for_tables(self, tables, table_container_name): entries = [] for table_dict in tables: entry_id, entry = self.__datacatalog_entry_factory. \ make_entry_for_tables(table_dict, table_container_name) tags = [] if self.__tag_templates_dict: tag_template_id = '{}_{}_metadata'.format( self.__entry_group_id, self.__metadata_definition['table_def']['type']) tag_template = self.__tag_templates_dict[tag_template_id] tags.append( self.__datacatalog_tag_factory.make_tag_for_table_metadata( tag_template, table_dict, table_container_name)) entries.append(prepare.AssembledEntryData(entry_id, entry, tags)) return entries
def __make_assembled_entry_for_entity(self, entity, entity_types_dict, classifications, enum_types_dict): entry_id, entry = self.__datacatalog_entry_factory. \ make_entry_for_entity(entity) tags = [ self.__datacatalog_tag_factory.make_tag_for_entity( entity, entity_types_dict, enum_types_dict) ] entry_classifications = entity.get('classifications') if entry_classifications: for classification in entry_classifications: tags.append( self.__datacatalog_tag_factory.make_tag_for_classification( classification, classifications, enum_types_dict)) self.__make_tags_for_columns(classifications, entity, tags, enum_types_dict) return prepare.AssembledEntryData(entry_id, entry, tags)
def test_run_custom_property_def_should_traverse_main_workflow_steps( self, mock_mapper, mock_cleaner, mock_ingestor): attrs = self.__synchronizer.__dict__ scraper = attrs['_MetadataSynchronizer__metadata_scraper'] assembled_entry_factory = attrs[ '_MetadataSynchronizer__assembled_entry_factory'] scraper.scrape_all_custom_property_definitions.return_value = [{ 'id': 'test-def', }] assembled_entry_factory.make_assembled_entry_for_custom_property_def\ .return_value = prepare.AssembledEntryData( 'test-def', self.__make_fake_entry('custom_property_definition'), []) self.__synchronizer.run() expected_make_assembled_entries_call_arg = { 'id': 'test-def', } actual_call_args = assembled_entry_factory\ .make_assembled_entry_for_custom_property_def.call_args[0] self.assertEqual(expected_make_assembled_entries_call_arg, actual_call_args[0]) mapper = mock_mapper.return_value mapper.fulfill_tag_fields.assert_called_once() cleaner = mock_cleaner.return_value cleaner.delete_obsolete_metadata.assert_called_once() ingestor = mock_ingestor.return_value ingestor.ingest_metadata.assert_called_once()
def test_run_should_process_scrape_prepare_ingest_workflow( self, mock_scraper, mock_assembled_entry_factory, mock_mapper, mock_cleaner, mock_ingestor): assembled_entry_factory = mock_assembled_entry_factory.return_value assembled_entry_factory.make_entries_for_workbooks.return_value = \ [(prepare.AssembledEntryData('test-entry-id-1', {}, []))] workbooks_synchronizer.WorkbooksSynchronizer( tableau_server_address='test-server', tableau_api_version='test-api-version', tableau_username='******', tableau_password='******', tableau_site='test-site', datacatalog_project_id='test-project-id', datacatalog_location_id='test-location-id').run() scraper = mock_scraper.return_value scraper.scrape_workbooks.assert_called_once() assembled_entry_factory.make_entries_for_workbooks.assert_called_once() mock_mapper.return_value.fulfill_tag_fields.assert_called_once() mock_cleaner.return_value.delete_obsolete_metadata.assert_called_once() mock_ingestor.return_value.ingest_metadata.assert_called_once()
def __make_assembled_entry_for_dashboard( self, dashboard_metadata: Dict[str, Any], tag_templates: Dict[str, TagTemplate]) -> AssembledEntryData: entry_id, entry = \ self.__datacatalog_entry_factory.make_entry_for_dashboard( dashboard_metadata) tags = [] dashboard_tag_template = tag_templates.get( constants.TAG_TEMPLATE_ID_DASHBOARD) if dashboard_tag_template: tags.append( self.__datacatalog_tag_factory.make_tag_for_dashboard( dashboard_tag_template, dashboard_metadata)) jaql_tag_template = tag_templates.get(constants.TAG_TEMPLATE_ID_JAQL) if jaql_tag_template: tags.extend( self.__datacatalog_tag_factory.make_tags_for_dashboard_filters( jaql_tag_template, dashboard_metadata)) return prepare.AssembledEntryData(entry_id=entry_id, entry=entry, tags=tags)
def test_run_not_published_app_should_properly_ask_assembled_entries(self): attrs = self.__synchronizer.__dict__ scraper = attrs['_MetadataSynchronizer__metadata_scraper'] assembled_entry_factory = attrs[ '_MetadataSynchronizer__assembled_entry_factory'] scraper.scrape_all_streams.return_value = [self.__make_fake_stream()] scraper.scrape_all_apps.return_value = [self.__make_fake_wip_app()] assembled_entry_factory.make_assembled_entries_for_stream\ .return_value = [ prepare.AssembledEntryData( 'test-stream', self.__make_fake_entry('stream'), []) ] self.__synchronizer.run() expected_make_assembled_entries_call_arg = { 'id': 'test-stream', } actual_call_args = assembled_entry_factory\ .make_assembled_entries_for_stream.call_args[0] self.assertEqual(expected_make_assembled_entries_call_arg, actual_call_args[0])
def __make_assembled_entry(self, group_id, data): entry_id, entry = self.__entry_factory.make_entry_from_dict( group_id, data) return prepare.AssembledEntryData(entry_id, entry)
def _make_assembled_entries(self, source_system_metadata, tag_templates_dict): return [(prepare.AssembledEntryData('test-entry-id-1', {}, []), [])]
def __make_entries_for_database(self, database_dict): entry_id, entry = self.\ __datacatalog_entry_factory.make_entries_for_database( database_dict) return prepare.AssembledEntryData(entry_id, entry)