def test_scrape_looks_from_folder_should_return_list(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
look_data = {
'id': 123,
'title': 'A B C',
}
sdk.search_looks.return_value = [
serialize.deserialize31(data=json.dumps(look_data),
structure=models.Look)
]
folder_data = {
'id': 'folder-id',
'name': 'Test folder',
'parent_id': '',
}
looks = self.__scraper.scrape_looks_from_folder(
serialize.deserialize31(data=json.dumps(folder_data),
structure=models.Folder))
self.assertEqual(1, len(looks))
self.assertEqual(123, looks[0].id)
sdk.search_looks.assert_called_once()
sdk.search_looks.assert_called_with(
space_id='folder-id',
fields='id,title,created_at,updated_at,description,space,public,'
'user_id,last_updater_id,query_id,url,short_url,public_url,'
'excel_file_url,google_spreadsheet_formula,view_count,'
'favorite_count,last_accessed_at,last_viewed_at,deleted,'
'deleter_id')
def test_scrape_dashboards_from_folder_should_return_list(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
dashboard_data = {
'id': 'dashboard-id',
'title': 'A B C',
}
sdk.search_dashboards.return_value = [
serialize.deserialize31(data=json.dumps(dashboard_data),
structure=models.Dashboard)
]
folder_data = {
'id': 'folder-id',
'name': 'Test folder',
'parent_id': '',
}
dashboards = self.__scraper.scrape_dashboards_from_folder(
serialize.deserialize31(data=json.dumps(folder_data),
structure=models.Folder))
self.assertEqual(1, len(dashboards))
self.assertEqual('dashboard-id', dashboards[0].id)
sdk.search_dashboards.assert_called_once()
sdk.search_dashboards.assert_called_with(
space_id='folder-id',
fields='id,title,created_at,description,space,hidden,user_id,'
'view_count,favorite_count,last_accessed_at,last_viewed_at,'
'deleted,deleted_at,deleter_id,dashboard_elements')
def test_scrape_child_folders_should_return_list(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
folder_data = {
'id': 'folder-id',
'name': 'Folder name',
'parent_id': '',
}
sdk.folder_children.return_value = [
serialize.deserialize31(data=json.dumps(folder_data),
structure=models.Folder)
]
parent_data = {
'id': 'parent-folder-id',
'name': 'Parent folder name',
'parent_id': '',
}
folders = self.__scraper.scrape_child_folders(
serialize.deserialize31(data=json.dumps(parent_data),
structure=models.Folder))
self.assertEqual(1, len(folders))
self.assertEqual('folder-id', folders[0].id)
sdk.folder_children.assert_called_once()
sdk.folder_children.assert_called_with(
folder_id='parent-folder-id',
fields='id,name,parent_id,child_count,creator_id')
def test_make_entry_for_dashboard_should_set_all_available_fields(self):
dashboard_data = {
'id': 'a123-b456',
'title': 'Test Name',
'created_at': '2019-09-12T16:30:00.000Z',
}
entry_id, entry = self.__factory.make_entry_for_dashboard(
serialize.deserialize31(data=json.dumps(dashboard_data),
structure=models.Dashboard))
self.assertEqual('lkr_test_server_com_db_a123_b456', entry_id)
self.assertEqual(
'projects/test-project/locations/test-location/'
'entryGroups/test-entry-group/entries/'
'lkr_test_server_com_db_a123_b456', entry.name)
self.assertEqual('test-system', entry.user_specified_system)
self.assertEqual('dashboard', entry.user_specified_type)
self.assertEqual('Test Name', entry.display_name)
self.assertEqual('https://test.server.com/dashboards/a123-b456',
entry.linked_resource)
created_datetime = self.__parse_datetime('2019-09-12T16:30:00+0000')
self.assertEqual(
created_datetime.timestamp(),
entry.source_system_timestamps.create_time.timestamp())
self.assertEqual(
(created_datetime + timedelta(seconds=10)).timestamp(),
entry.source_system_timestamps.update_time.timestamp())
def test_make_entry_for_look_should_set_all_available_fields(self):
look_data = {
'id': 123,
'title': 'Test Name',
'created_at': '2019-09-12T16:30:00.000Z',
'updated_at': '2019-09-12T17:35:00.000Z',
}
entry_id, entry = self.__factory.make_entry_for_look(
serialize.deserialize31(data=json.dumps(look_data),
structure=models.Look))
self.assertEqual('lkr_test_server_com_lk_123', entry_id)
self.assertEqual(
'projects/test-project/locations/test-location/'
'entryGroups/test-entry-group/entries/'
'lkr_test_server_com_lk_123', entry.name)
self.assertEqual('test-system', entry.user_specified_system)
self.assertEqual('look', entry.user_specified_type)
self.assertEqual('Test Name', entry.display_name)
self.assertEqual('https://test.server.com/looks/123',
entry.linked_resource)
created_datetime = self.__parse_datetime('2019-09-12T16:30:00+0000')
self.assertEqual(
created_datetime.timestamp(),
entry.source_system_timestamps.create_time.timestamp())
updated_datetime = self.__parse_datetime('2019-09-12T17:35:10+0000')
self.assertEqual(
updated_datetime.timestamp(),
entry.source_system_timestamps.update_time.timestamp())
def test_make_tag_for_folder_should_set_all_available_fields(self):
tag_template = \
self.__tag_template_factory.make_tag_template_for_folder()
folder_data = {
'id': 'a123-b456',
'name': 'Test name',
'parent_id': 'test-parent',
'child_count': 10,
'dashboards': [{}, {}],
'looks': [{}],
}
tag = self.__factory.make_tag_for_folder(
tag_template,
serialize.deserialize31(data=json.dumps(folder_data),
structure=models.Folder))
self.assertEqual(tag_template.name, tag.template)
self.assertEqual('a123-b456', tag.fields['id'].string_value)
self.assertEqual('Test name', tag.fields['name'].string_value)
self.assertEqual('test-parent', tag.fields['parent_id'].string_value)
self.assertEqual(True, tag.fields['has_children'].bool_value)
self.assertEqual(10, tag.fields['children_count'].double_value)
self.assertEqual(True, tag.fields['has_dashboards'].bool_value)
self.assertEqual(2, tag.fields['dashboards_count'].double_value)
self.assertEqual(True, tag.fields['has_looks'].bool_value)
self.assertEqual(1, tag.fields['looks_count'].double_value)
self.assertEqual('https://test.server.com',
tag.fields['instance_url'].string_value)
def test_make_tag_for_query_no_model_connection_should_succeed(self):
tag_template = \
self.__tag_template_factory.make_tag_template_for_query()
query_data = {
'id': 5,
'runtime': 0.5,
'client_id': 'test-client-id',
'query_timezone': 'America/Sao_Paulo',
'model': 'test-model',
'view': 'test-view',
}
# yapf: disable
assembled_metadata = entities.AssembledQueryMetadata(
serialize.deserialize31(
data=json.dumps(query_data),
structure=models.Query),
'select *',
None,
None)
# yapf: enable
tag = self.__factory.make_tag_for_query(tag_template,
assembled_metadata)
self.assertEqual(5, tag.fields['id'].double_value)
self.assertFalse('lookml_project' in tag.fields)
self.assertFalse('host' in tag.fields)
def __make_fake_look(cls, folder):
look_data = {
'id': 123,
'space': json.loads(serialize.serialize(folder)),
}
return serialize.deserialize31(data=json.dumps(look_data),
structure=models.Look)
def test_make_tag_for_dashboard_should_set_all_available_fields(self):
tag_template = \
self.__tag_template_factory.make_tag_template_for_dashboard()
dashboard_data = {
'id': 'a123-b456',
'description': 'Test description',
'space': {
'id': 'test-folder',
'name': 'Test folder',
'parent_id': '',
},
'hidden': True,
'user_id': 10,
'view_count': 200,
'favorite_count': 3,
'last_accessed_at': '2019-09-12T16:30:00.000Z',
'last_viewed_at': '2019-09-12T16:35:00.000Z',
'deleted': True,
'deleted_at': '2019-09-12T16:40:00.000Z',
'deleter_id': 12,
}
tag = self.__factory.make_tag_for_dashboard(
tag_template,
serialize.deserialize31(data=json.dumps(dashboard_data),
structure=models.Dashboard))
self.assertEqual(tag_template.name, tag.template)
self.assertEqual('a123-b456', tag.fields['id'].string_value)
self.assertEqual('Test description',
tag.fields['description'].string_value)
self.assertEqual('test-folder', tag.fields['folder_id'].string_value)
self.assertEqual('Test folder', tag.fields['folder_name'].string_value)
self.assertEqual(True, tag.fields['is_hidden'].bool_value)
self.assertEqual(10, tag.fields['user_id'].double_value)
self.assertEqual(200, tag.fields['view_count'].double_value)
self.assertEqual(3, tag.fields['favorite_count'].double_value)
last_accessed_at = self.__parse_datetime('2019-09-12T16:30:00+0000')
self.assertEqual(
last_accessed_at.timestamp(),
tag.fields['last_accessed_at'].timestamp_value.timestamp())
last_viewed_at = self.__parse_datetime('2019-09-12T16:35:00+0000')
self.assertEqual(
last_viewed_at.timestamp(),
tag.fields['last_viewed_at'].timestamp_value.timestamp())
self.assertEqual(True, tag.fields['is_deleted'].bool_value)
deleted_at = self.__parse_datetime('2019-09-12T16:40:00+0000')
self.assertEqual(deleted_at.timestamp(),
tag.fields['deleted_at'].timestamp_value.timestamp())
self.assertEqual(12, tag.fields['deleter_id'].double_value)
self.assertEqual('https://test.server.com',
tag.fields['instance_url'].string_value)
def __make_fake_query(cls):
query_data = {
'id': 837,
'model': '',
'view': '',
}
return serialize.deserialize31(data=json.dumps(query_data),
structure=models.Query)
def __make_fake_dashboard(cls, folder):
dashboard_data = {
'id': 'test_dashboard',
'space': json.loads(serialize.serialize(folder)),
'dashboard_elements': [],
}
return serialize.deserialize31(data=json.dumps(dashboard_data),
structure=models.Dashboard)
def test_make_entry_for_dashboard_tile_should_skip_empty_titles(self):
dashboard_element_data = {
'id': '196',
}
entry_id, entry = self.__factory.make_entry_for_dashboard_element(
serialize.deserialize31(data=json.dumps(dashboard_element_data),
structure=models.DashboardElement))
self.assertIsNone(entry_id)
self.assertIsNone(entry)
def test_make_entry_for_dashboard_tile_should_handle_title_text(self):
dashboard_element_data = {
'id': '196',
'title': '',
'title_text': 'Test Name',
}
entry_id, entry = self.__factory.make_entry_for_dashboard_element(
serialize.deserialize31(data=json.dumps(dashboard_element_data),
structure=models.DashboardElement))
self.assertEqual('Test Name', entry.display_name)
def __make_fake_folder(cls, parent=None):
parent_data = json.loads(
serialize.serialize(parent)) if parent else None
folder_data = {
'id': 'test_child_folder' if parent else 'test_folder',
'name': 'Test child folder' if parent else 'Test folder',
'parent_id': parent_data['id'] if parent_data else '',
'child_count': 1 if parent else 0,
}
return serialize.deserialize31(data=json.dumps(folder_data),
structure=models.Folder)
def test_make_tag_for_dashboard_tile_should_set_all_available_fields(self):
tag_template = self.__tag_template_factory\
.make_tag_template_for_dashboard_element()
dashboard_element_data = {
'id': '196',
'type': 'vis',
'dashboard_id': 'a123-b456',
'look_id': '25',
'look': {
'title': 'Test look',
},
'lookml_link_id': 'Test link ID',
'query_id': 837,
}
dashboard_data = {
'title': 'Test dashboard',
}
tag = self.__factory.make_tag_for_dashboard_element(
tag_template,
serialize.deserialize31(data=json.dumps(dashboard_element_data),
structure=models.DashboardElement),
serialize.deserialize31(data=json.dumps(dashboard_data),
structure=models.Dashboard))
self.assertEqual(tag_template.name, tag.template)
self.assertEqual('196', tag.fields['id'].string_value)
self.assertEqual('vis', tag.fields['type'].string_value)
self.assertEqual('a123-b456', tag.fields['dashboard_id'].string_value)
self.assertEqual('Test dashboard',
tag.fields['dashboard_title'].string_value)
self.assertEqual('25', tag.fields['look_id'].string_value)
self.assertEqual('Test look', tag.fields['look_title'].string_value)
self.assertEqual('Test link ID',
tag.fields['lookml_link_id'].string_value)
self.assertEqual(837, tag.fields['query_id'].double_value)
self.assertEqual('https://test.server.com',
tag.fields['instance_url'].string_value)
def test_make_entry_for_dashboard_no_created_at_should_succeed(self):
dashboard_data = {
'id': 'a123-b456',
'title': 'Test Name',
}
entry_id, entry = self.__factory.make_entry_for_dashboard(
serialize.deserialize31(data=json.dumps(dashboard_data),
structure=models.Dashboard))
self.assertIsNone(entry.source_system_timestamps.create_time)
self.assertIsNone(entry.source_system_timestamps.update_time)
def __make_fake_folder(cls, dashboard=None, look=None):
dashboard_data = json.loads(
serialize.serialize(dashboard)) if dashboard else None
look_data = json.loads(serialize.serialize(look)) if look else None
folder_data = {
'id': 'test_folder',
'name': 'Test folder',
'parent_id': '',
'dashboards': [dashboard_data] if dashboard_data else None,
'looks': [look_data] if look_data else None,
}
return serialize.deserialize31(data=json.dumps(folder_data),
structure=models.Folder)
def test_scrape_look_should_return_object_on_success(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
look_data = {
'id': 123,
}
sdk.look.return_value = serialize.deserialize31(
data=json.dumps(look_data), structure=models.Look)
look = self.__scraper.scrape_look(123)
self.assertEqual(123, look.id)
sdk.look.assert_called_once()
sdk.look.assert_called_with(look_id=123)
def test_scrape_dashboard_should_return_object_on_success(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
dashboard_data = {
'id': 'dashboard-id',
}
sdk.dashboard.return_value = serialize.deserialize31(
data=json.dumps(dashboard_data), structure=models.Dashboard)
dashboard = self.__scraper.scrape_dashboard('dashboard-id')
self.assertEqual('dashboard-id', dashboard.id)
sdk.dashboard.assert_called_once()
sdk.dashboard.assert_called_with(dashboard_id='dashboard-id')
def test_scrape_connection_should_return_object(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
connection_data = {
'name': 'test-connection',
}
sdk.connection.return_value = serialize.deserialize31(
data=json.dumps(connection_data), structure=models.DBConnection)
connection = self.__scraper.scrape_connection('test-connection')
self.assertEqual('test-connection', connection.name)
sdk.connection.assert_called_once()
sdk.connection.assert_called_with(connection_name='test-connection')
def test_scrape_query_should_return_object(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
query_data = {
'id': 123,
'model': '',
'view': '',
}
sdk.query.return_value = serialize.deserialize31(
data=json.dumps(query_data), structure=models.Query)
query = self.__scraper.scrape_query(123)
self.assertEqual(123, query.id)
sdk.query.assert_called_once()
sdk.query.assert_called_with(query_id=123)
def test_make_tag_empty_string_value_should_skip_field(self):
tag_template = \
self.__tag_template_factory.make_tag_template_for_folder()
folder_data = {
'id': 'a123-b456',
'name': 'Test name',
'parent_id': '',
'child_count': 10,
}
tag = self.__factory.make_tag_for_folder(
tag_template,
serialize.deserialize31(data=json.dumps(folder_data),
structure=models.Folder))
self.assertFalse('parent_id' in tag.fields)
def test_make_entry_for_look_no_updated_at_should_succeed(self):
look_data = {
'id': 123,
'title': 'Test Name',
'created_at': '2019-09-12T16:30:00.000Z',
}
entry_id, entry = self.__factory.make_entry_for_look(
serialize.deserialize31(data=json.dumps(look_data),
structure=models.Look))
created_datetime = self.__parse_datetime('2019-09-12T16:30:00+0000')
self.assertEqual(
created_datetime.timestamp(),
entry.source_system_timestamps.create_time.timestamp())
self.assertEqual(
(created_datetime + timedelta(seconds=10)).timestamp(),
entry.source_system_timestamps.update_time.timestamp())
def test_make_assembled_entries_list_should_process_dashboard_tiles(self):
entry_factory = self.__entry_factory
entry_factory.make_entry_for_folder = self.__mock_make_entry
entry_factory.make_entry_for_dashboard = self.__mock_make_entry
entry_factory.make_entry_for_dashboard_element = self.__mock_make_entry
tag_factory = self.__tag_factory
tag_factory.make_tag_for_dashboard_element = \
self.__mock_make_tag_parent_dep
dashboard_data = {
'id': 'test_dashboard',
'dashboard_elements': [{
'id': 194,
}],
}
dashboard = serialize.deserialize31(data=json.dumps(dashboard_data),
structure=models.Dashboard)
folder = self.__make_fake_folder()
folder.dashboards = [dashboard]
tag_templates_dict = {
'looker_dashboard_element_metadata': {
'name': 'tagTemplates/looker_dashboard_element_metadata',
}
}
assembled_entries = \
self.__assembled_data_factory.make_assembled_entries_list(
[folder], [], tag_templates_dict)
self.assertEqual(3, len(assembled_entries))
# The first entry refers to the folder and the second to the dashboard.
element_assembled_entry = assembled_entries[2]
self.assertEqual('194', element_assembled_entry.entry_id)
self.assertEqual('fake_entries/194',
element_assembled_entry.entry.name)
tags = element_assembled_entry.tags
self.assertEqual(1, len(tags))
self.assertEqual('tagTemplates/looker_dashboard_element_metadata',
tags[0].template)
def test_make_tag_for_look_should_set_url_field_if_available(self):
tag_template = self.__tag_template_factory.make_tag_template_for_look()
look_data = {
'space': {
'id': 'test-folder',
'name': 'Test folder',
'parent_id': '',
},
'url': 'test/url',
}
tag = self.__factory.make_tag_for_look(
tag_template,
serialize.deserialize31(data=json.dumps(look_data),
structure=models.LookWithQuery))
self.assertEqual('test/url', tag.fields['url'].string_value)
def test_make_entry_for_dashboard_tile_should_set_fields(self):
dashboard_element_data = {
'id': '196',
'title': 'Test Name',
}
entry_id, entry = self.__factory.make_entry_for_dashboard_element(
serialize.deserialize31(data=json.dumps(dashboard_element_data),
structure=models.DashboardElement))
self.assertEqual('lkr_test_server_com_de_196', entry_id)
self.assertEqual(
'projects/test-project/locations/test-location/'
'entryGroups/test-entry-group/entries/'
'lkr_test_server_com_de_196', entry.name)
self.assertEqual('test-system', entry.user_specified_system)
self.assertEqual('dashboard_element', entry.user_specified_type)
self.assertEqual('Test Name', entry.display_name)
def test_scrape_folder_should_return_object(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
folder_data = {
'id': 'folder-id',
'name': 'Test folder',
'parent_id': '',
}
sdk.folder.return_value = serialize.deserialize31(
data=json.dumps(folder_data), structure=models.Folder)
folder = self.__scraper.scrape_folder('folder-id')
self.assertEqual('folder-id', folder.id)
sdk.folder.assert_called_once()
sdk.folder.assert_called_with(
folder_id='folder-id',
fields='id,name,parent_id,child_count,creator_id,dashboards,looks')
def test_scrape_model_explore_should_return_object_on_success(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
model_explore_data = {
'project_name': 'test-project',
'connection_name': 'test-connection',
}
sdk.lookml_model_explore.return_value = serialize.deserialize31(
data=json.dumps(model_explore_data),
structure=models.LookmlModelExplore)
model = self.__scraper.scrape_lookml_model_explore(
'test-model', 'test-view')
self.assertEqual('test-project', model.project_name)
sdk.lookml_model_explore.assert_called_once()
sdk.lookml_model_explore.assert_called_with(
lookml_model_name='test-model', explore_name='test-view')
def test_make_tag_for_dashboard_tile_result_maker_should_succeed(self):
tag_template = self.__tag_template_factory\
.make_tag_template_for_dashboard_element()
dashboard_element_data = {
'id': '196',
'type': 'vis',
'dashboard_id': 'a123-b456',
'result_maker': {
'query_id': 837,
},
}
tag = self.__factory.make_tag_for_dashboard_element(
tag_template,
serialize.deserialize31(data=json.dumps(dashboard_element_data),
structure=models.DashboardElement), None)
self.assertEqual(837, tag.fields['query_id'].double_value)
def test_make_entry_for_folder_should_set_all_available_fields(self):
folder_data = {
'id': 'a123-b456',
'name': 'Test Name',
'parent_id': '',
}
entry_id, entry = self.__factory.make_entry_for_folder(
serialize.deserialize31(data=json.dumps(folder_data),
structure=models.Folder))
self.assertEqual('lkr_test_server_com_fd_a123_b456', entry_id)
self.assertEqual(
'projects/test-project/locations/test-location/'
'entryGroups/test-entry-group/entries/'
'lkr_test_server_com_fd_a123_b456', entry.name)
self.assertEqual('test-system', entry.user_specified_system)
self.assertEqual('folder', entry.user_specified_type)
self.assertEqual('Test Name', entry.display_name)
self.assertEqual('https://test.server.com/folders/a123-b456',
entry.linked_resource)
