def __make_assembled_entry_for_query(self, assembled_query_metadata,
tag_template):
entry_id, entry = \
self.__datacatalog_entry_factory.make_entry_for_query(
assembled_query_metadata.query)
tags = []
if tag_template:
tags.append(
self.__datacatalog_tag_factory.make_tag_for_query(
tag_template, assembled_query_metadata))
return prepare.AssembledEntryData(entry_id, entry, tags)
def test_fulfill_tag_fields_should_resolve_visualization_app_mapping(self):
app_id = 'test-app'
app_entry = self.__make_fake_entry(app_id, 'app')
app_tag = self.__make_fake_tag(string_fields=(('id', app_id),))
visualization_id = 'test-measure'
visualization_entry = self.__make_fake_entry(visualization_id,
'visualization')
string_fields = ('id', visualization_id), ('app_id', app_id)
visualization_tag = self.__make_fake_tag(string_fields=string_fields)
app_assembled_entry = commons_prepare.AssembledEntryData(
app_id, app_entry, [app_tag])
measure_assembled_entry = commons_prepare.AssembledEntryData(
visualization_id, visualization_entry, [visualization_tag])
prepare.EntryRelationshipMapper().fulfill_tag_fields(
[app_assembled_entry, measure_assembled_entry])
self.assertEqual(
'https://console.cloud.google.com/datacatalog/'
'fake_entries/test-app',
visualization_tag.fields['app_entry'].string_value)
def test_fulfill_tag_fields_should_resolve_element_look_mapping(self):
look_id = 123
look_entry = self.__make_fake_entry(look_id, 'look')
look_tag = self.__make_fake_tag(double_fields=(('id', look_id), ))
element_id = 'test_element'
element_entry = self.__make_fake_entry(element_id, 'dashboard_element')
double_fields = ('look_id', look_id),
element_tag = self.__make_fake_tag(string_fields=(('id',
element_id), ),
double_fields=double_fields)
look_assembled_entry = commons_prepare.AssembledEntryData(
look_id, look_entry, [look_tag])
element_assembled_entry = commons_prepare.AssembledEntryData(
element_id, element_entry, [element_tag])
prepare.EntryRelationshipMapper().fulfill_tag_fields(
[look_assembled_entry, element_assembled_entry])
self.assertEqual(
f'https://console.cloud.google.com/datacatalog/{look_entry.name}',
element_tag.fields['look_entry'].string_value)
def test_fulfill_tag_fields_should_resolve_sheet_workbook_mapping(self):
workbook_luid = 'test_workbook'
workbook_entry = self.__make_fake_entry(workbook_luid, 'workbook')
workbook_tag = self.__make_fake_tag(string_fields=(('luid',
workbook_luid), ))
sheet_luid = 'test_dashboard'
sheet_entry = self.__make_fake_entry(sheet_luid, 'sheet')
string_fields = ('luid', sheet_luid), ('workbook_luid', workbook_luid)
sheet_tag = self.__make_fake_tag(string_fields=string_fields)
workbook_assembled_entry = commons_prepare.AssembledEntryData(
workbook_luid, workbook_entry, [workbook_tag])
sheet_assembled_entry = commons_prepare.AssembledEntryData(
sheet_luid, sheet_entry, [sheet_tag])
prepare.EntryRelationshipMapper().fulfill_tag_fields(
[workbook_assembled_entry, sheet_assembled_entry])
self.assertEqual(
f'https://console.cloud.google.com/datacatalog/'
f'{workbook_entry.name}',
sheet_tag.fields['workbook_entry'].string_value)
def test_fulfill_tag_fields_should_resolve_element_dashboard_mapping(self):
dashboard_id = 'test_dashboard'
dashboard_entry = self.__make_fake_entry(dashboard_id, 'dashboard')
string_fields = ('id', dashboard_id),
dashboard_tag = self.__make_fake_tag(string_fields=string_fields)
element_id = 'test_element'
element_entry = self.__make_fake_entry(element_id, 'dashboard_element')
string_fields = ('id', element_id), ('dashboard_id', dashboard_id)
element_tag = self.__make_fake_tag(string_fields=string_fields)
dashboard_assembled_entry = commons_prepare.AssembledEntryData(
dashboard_id, dashboard_entry, [dashboard_tag])
element_assembled_entry = commons_prepare.AssembledEntryData(
element_id, element_entry, [element_tag])
prepare.EntryRelationshipMapper().fulfill_tag_fields(
[dashboard_assembled_entry, element_assembled_entry])
self.assertEqual(
f'https://console.cloud.google.com/datacatalog/'
f'{dashboard_entry.name}',
element_tag.fields['dashboard_entry'].string_value)
def test_fulfill_tag_fields_should_resolve_element_query_mapping(self):
query_id = 1080
query_entry = self.__make_fake_entry(query_id, 'query')
query_tag = self.__make_fake_tag(double_fields=(('id', query_id),))
element_id = 'test_element'
element_entry = self.__make_fake_entry(element_id, 'dashboard_element')
double_fields = ('query_id', query_id),
element_tag = self.__make_fake_tag(string_fields=(('id', element_id),),
double_fields=double_fields)
query_assembled_entry = commons_prepare.AssembledEntryData(
query_id, query_entry, [query_tag])
element_assembled_entry = commons_prepare.AssembledEntryData(
element_id, element_entry, [element_tag])
prepare.EntryRelationshipMapper().fulfill_tag_fields(
[query_assembled_entry, element_assembled_entry])
self.assertEqual(
f'https://console.cloud.google.com/datacatalog/'
f'{query_entry.name}',
element_tag.fields['query_entry'].string_value)
def test_fulfill_tag_fields_should_resolve_look_folder_mapping(self):
folder_id = 'test_folder'
folder_entry = self.__make_fake_entry(folder_id, 'folder')
folder_tag = self.__make_fake_tag(string_fields=(('id', folder_id),))
look_id = 123
look_entry = self.__make_fake_entry(look_id, 'look')
string_fields = ('folder_id', folder_id),
look_tag = self.__make_fake_tag(string_fields=string_fields,
double_fields=(('id', look_id),))
folder_assembled_entry = commons_prepare.AssembledEntryData(
folder_id, folder_entry, [folder_tag])
look_assembled_entry = commons_prepare.AssembledEntryData(
look_id, look_entry, [look_tag])
prepare.EntryRelationshipMapper().fulfill_tag_fields(
[folder_assembled_entry, look_assembled_entry])
self.assertEqual(
f'https://console.cloud.google.com/datacatalog/'
f'{folder_entry.name}',
look_tag.fields['folder_entry'].string_value)
def __make_entry_for_sheet(self, sheet_metadata, workbook_metadata,
tag_template):
entry_id, entry = \
self.__datacatalog_entry_factory.make_entry_for_sheet(
sheet_metadata,
workbook_metadata)
tags = []
if tag_template:
tags.append(
datacatalog_tag_factory.DataCatalogTagFactory.
make_tag_for_sheet(tag_template, sheet_metadata,
workbook_metadata))
return prepare.AssembledEntryData(entry_id, entry, tags)
def test_fulfill_tag_fields_should_resolve_parent_folder_mapping(self):
parent_folder_id = 'parent-folder'
parent_folder_entry = self.__make_fake_entry(parent_folder_id,
'Folder')
parent_folder_tag = self.__make_fake_tag(
string_fields=(('id', parent_folder_id), ))
folder_id = 'test-folder'
folder_entry = self.__make_fake_entry(folder_id, 'Folder')
string_fields = ('id', folder_id), ('parent_id', parent_folder_id)
folder_tag = self.__make_fake_tag(string_fields=string_fields)
parent_folder_assembled_entry = commons_prepare.AssembledEntryData(
parent_folder_id, parent_folder_entry, [parent_folder_tag])
folder_assembled_entry = commons_prepare.AssembledEntryData(
folder_id, folder_entry, [folder_tag])
prepare.EntryRelationshipMapper().fulfill_tag_fields(
[parent_folder_assembled_entry, folder_assembled_entry])
self.assertEqual(
f'https://console.cloud.google.com/datacatalog/'
f'{parent_folder_entry.name}',
folder_tag.fields['parent_folder_entry'].string_value)
def test_resolve_folder_mappings_should_skip_non_folder_entries(
self, mock_map_related_entry):
entry_id = 'test'
entry = self.__make_fake_entry(entry_id, 'not-a-folder')
tag = self.__make_fake_tag(string_fields=(('id', entry_id), ))
assembled_entry = commons_prepare.AssembledEntryData(
entry_id, entry, [tag])
prepare.EntryRelationshipMapper()\
._EntryRelationshipMapper__resolve_folder_mappings(
[assembled_entry], {})
mock_map_related_entry.assert_not_called()
def make_assembled_entry_for_custom_property_def(
self, custom_property_def_metadata, tag_template):
entry_id, entry = self.__datacatalog_entry_factory\
.make_entry_for_custom_property_definition(
custom_property_def_metadata)
tags = []
if tag_template:
tags.append(
self.__datacatalog_tag_factory.
make_tag_for_custom_property_definition(
tag_template, custom_property_def_metadata))
return prepare.AssembledEntryData(entry_id, entry, tags)
def __make_assembled_entry_for_dashboard_element(self, element, dashboard,
tag_template):
entry_id, entry = self.__datacatalog_entry_factory.\
make_entry_for_dashboard_element(element)
if not (entry_id and entry):
return
tags = []
if tag_template:
tags.append(
self.__datacatalog_tag_factory.make_tag_for_dashboard_element(
tag_template, element, dashboard))
return prepare.AssembledEntryData(entry_id, entry, tags)
def __make_assembled_entry_for_sheet(self, sheet_metadata,
tag_templates_dict):
entry_id, entry = \
self.__datacatalog_entry_factory.make_entry_for_sheet(
sheet_metadata)
tag_template = tag_templates_dict.get(constants.TAG_TEMPLATE_ID_SHEET)
tags = []
if tag_template:
tags.append(
self.__datacatalog_tag_factory.make_tag_for_sheet(
tag_template, sheet_metadata))
return prepare.AssembledEntryData(entry_id, entry, tags)
def __make_entries_for_table_container(self, table_container):
entry_id, entry = \
self.__datacatalog_entry_factory.make_entries_for_table_container(
table_container)
tags = []
if self.__tag_templates_dict:
tag_template_id = '{}_{}_metadata'.format(
self.__entry_group_id,
self.__metadata_definition['table_container_def']['type'])
tag_template = self.__tag_templates_dict[tag_template_id]
tags.append(
self.__datacatalog_tag_factory.
make_tag_for_table_container_metadata(tag_template,
table_container))
return prepare.AssembledEntryData(entry_id, entry, tags)
def create_assembled_entry_user_defined_type(cls,
entry_id,
entry_type,
system,
display_name,
name,
description,
linked_resource,
create_time,
update_time,
columns=None):
entry = cls.create_entry_user_defined_type(
entry_type, system, display_name, name, description,
linked_resource, create_time, update_time, columns)
return prepare.AssembledEntryData(entry_id, entry, None)
def __make_assembled_entry_for_dimension(self, dimension_metadata,
tag_templates_dict):
entry_id, entry = \
self.__datacatalog_entry_factory.make_entry_for_dimension(
dimension_metadata)
tag_template = tag_templates_dict.get(
constants.TAG_TEMPLATE_ID_DIMENSION)
tags = []
if tag_template:
tags.append(
self.__datacatalog_tag_factory.make_tag_for_dimension(
tag_template, dimension_metadata))
return prepare.AssembledEntryData(entry_id, entry, tags)
def __make_assembled_entry_for_visualization(self, visualization_metadata,
tag_templates_dict):
entry_id, entry = \
self.__datacatalog_entry_factory.make_entry_for_visualization(
visualization_metadata)
tag_template = tag_templates_dict.get(
constants.TAG_TEMPLATE_ID_VISUALIZATION)
tags = []
if tag_template:
tags.append(
self.__datacatalog_tag_factory.make_tag_for_visualization(
tag_template, visualization_metadata))
return prepare.AssembledEntryData(entry_id, entry, tags)
def __make_assembled_entry_for_app(self, app_metadata, tag_templates_dict):
entry_id, entry = \
self.__datacatalog_entry_factory.make_entry_for_app(app_metadata)
app_tag_template = tag_templates_dict.get(
constants.TAG_TEMPLATE_ID_APP)
tags = []
if app_tag_template:
tags.append(
self.__datacatalog_tag_factory.make_tag_for_app(
app_tag_template, app_metadata))
tags.extend(
self.__datacatalog_tag_factory.make_tags_for_custom_properties(
tag_templates_dict, app_metadata.get('customProperties')))
return prepare.AssembledEntryData(entry_id, entry, tags)
def __make_assembled_entry_for_folder(
self, folder_metadata: Dict[str, Any],
tag_templates: Dict[str, TagTemplate]) -> AssembledEntryData:
entry_id, entry = \
self.__datacatalog_entry_factory.make_entry_for_folder(
folder_metadata)
tags = []
tag_template = tag_templates.get(constants.TAG_TEMPLATE_ID_FOLDER)
if tag_template:
tags.append(
self.__datacatalog_tag_factory.make_tag_for_folder(
tag_template, folder_metadata))
return prepare.AssembledEntryData(entry_id=entry_id,
entry=entry,
tags=tags)
def __make_entry_for_tables(self, tables, table_container_name):
entries = []
for table_dict in tables:
entry_id, entry = self.__datacatalog_entry_factory. \
make_entry_for_tables(table_dict, table_container_name)
tags = []
if self.__tag_templates_dict:
tag_template_id = '{}_{}_metadata'.format(
self.__entry_group_id,
self.__metadata_definition['table_def']['type'])
tag_template = self.__tag_templates_dict[tag_template_id]
tags.append(
self.__datacatalog_tag_factory.make_tag_for_table_metadata(
tag_template, table_dict, table_container_name))
entries.append(prepare.AssembledEntryData(entry_id, entry, tags))
return entries
def __make_assembled_entry_for_entity(self, entity, entity_types_dict,
classifications, enum_types_dict):
entry_id, entry = self.__datacatalog_entry_factory. \
make_entry_for_entity(entity)
tags = [
self.__datacatalog_tag_factory.make_tag_for_entity(
entity, entity_types_dict, enum_types_dict)
]
entry_classifications = entity.get('classifications')
if entry_classifications:
for classification in entry_classifications:
tags.append(
self.__datacatalog_tag_factory.make_tag_for_classification(
classification, classifications, enum_types_dict))
self.__make_tags_for_columns(classifications, entity, tags,
enum_types_dict)
return prepare.AssembledEntryData(entry_id, entry, tags)
def test_run_custom_property_def_should_traverse_main_workflow_steps(
self, mock_mapper, mock_cleaner, mock_ingestor):
attrs = self.__synchronizer.__dict__
scraper = attrs['_MetadataSynchronizer__metadata_scraper']
assembled_entry_factory = attrs[
'_MetadataSynchronizer__assembled_entry_factory']
scraper.scrape_all_custom_property_definitions.return_value = [{
'id':
'test-def',
}]
assembled_entry_factory.make_assembled_entry_for_custom_property_def\
.return_value = prepare.AssembledEntryData(
'test-def',
self.__make_fake_entry('custom_property_definition'),
[])
self.__synchronizer.run()
expected_make_assembled_entries_call_arg = {
'id': 'test-def',
}
actual_call_args = assembled_entry_factory\
.make_assembled_entry_for_custom_property_def.call_args[0]
self.assertEqual(expected_make_assembled_entries_call_arg,
actual_call_args[0])
mapper = mock_mapper.return_value
mapper.fulfill_tag_fields.assert_called_once()
cleaner = mock_cleaner.return_value
cleaner.delete_obsolete_metadata.assert_called_once()
ingestor = mock_ingestor.return_value
ingestor.ingest_metadata.assert_called_once()
def test_run_should_process_scrape_prepare_ingest_workflow(
self, mock_scraper, mock_assembled_entry_factory, mock_mapper,
mock_cleaner, mock_ingestor):
assembled_entry_factory = mock_assembled_entry_factory.return_value
assembled_entry_factory.make_entries_for_workbooks.return_value = \
[(prepare.AssembledEntryData('test-entry-id-1', {}, []))]
workbooks_synchronizer.WorkbooksSynchronizer(
tableau_server_address='test-server',
tableau_api_version='test-api-version',
tableau_username='******',
tableau_password='******',
tableau_site='test-site',
datacatalog_project_id='test-project-id',
datacatalog_location_id='test-location-id').run()
scraper = mock_scraper.return_value
scraper.scrape_workbooks.assert_called_once()
assembled_entry_factory.make_entries_for_workbooks.assert_called_once()
mock_mapper.return_value.fulfill_tag_fields.assert_called_once()
mock_cleaner.return_value.delete_obsolete_metadata.assert_called_once()
mock_ingestor.return_value.ingest_metadata.assert_called_once()
def __make_assembled_entry_for_dashboard(
self, dashboard_metadata: Dict[str, Any],
tag_templates: Dict[str, TagTemplate]) -> AssembledEntryData:
entry_id, entry = \
self.__datacatalog_entry_factory.make_entry_for_dashboard(
dashboard_metadata)
tags = []
dashboard_tag_template = tag_templates.get(
constants.TAG_TEMPLATE_ID_DASHBOARD)
if dashboard_tag_template:
tags.append(
self.__datacatalog_tag_factory.make_tag_for_dashboard(
dashboard_tag_template, dashboard_metadata))
jaql_tag_template = tag_templates.get(constants.TAG_TEMPLATE_ID_JAQL)
if jaql_tag_template:
tags.extend(
self.__datacatalog_tag_factory.make_tags_for_dashboard_filters(
jaql_tag_template, dashboard_metadata))
return prepare.AssembledEntryData(entry_id=entry_id,
entry=entry,
tags=tags)
def test_run_not_published_app_should_properly_ask_assembled_entries(self):
attrs = self.__synchronizer.__dict__
scraper = attrs['_MetadataSynchronizer__metadata_scraper']
assembled_entry_factory = attrs[
'_MetadataSynchronizer__assembled_entry_factory']
scraper.scrape_all_streams.return_value = [self.__make_fake_stream()]
scraper.scrape_all_apps.return_value = [self.__make_fake_wip_app()]
assembled_entry_factory.make_assembled_entries_for_stream\
.return_value = [
prepare.AssembledEntryData(
'test-stream', self.__make_fake_entry('stream'), [])
]
self.__synchronizer.run()
expected_make_assembled_entries_call_arg = {
'id': 'test-stream',
}
actual_call_args = assembled_entry_factory\
.make_assembled_entries_for_stream.call_args[0]
self.assertEqual(expected_make_assembled_entries_call_arg,
actual_call_args[0])
def __make_assembled_entry(self, group_id, data):
entry_id, entry = self.__entry_factory.make_entry_from_dict(
group_id, data)
return prepare.AssembledEntryData(entry_id, entry)
def _make_assembled_entries(self, source_system_metadata,
tag_templates_dict):
return [(prepare.AssembledEntryData('test-entry-id-1', {}, []), [])]
def __make_entries_for_database(self, database_dict):
entry_id, entry = self.\
__datacatalog_entry_factory.make_entries_for_database(
database_dict)
return prepare.AssembledEntryData(entry_id, entry)
