def test_scrape_dashboards_from_folder_should_return_list(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
dashboard_data = {
'id': 'dashboard-id',
'title': 'A B C',
}
sdk.search_dashboards.return_value = [
serialize.deserialize(json.dumps(dashboard_data), models.Dashboard)
]
folder_data = {
'id': 'folder-id',
'name': 'Test folder',
'parent_id': '',
}
dashboards = self.__scraper.scrape_dashboards_from_folder(
serialize.deserialize(json.dumps(folder_data), models.Folder))
self.assertEqual(1, len(dashboards))
self.assertEqual('dashboard-id', dashboards[0].id)
sdk.search_dashboards.assert_called_once()
sdk.search_dashboards.assert_called_with(
space_id='folder-id',
fields='id,title,created_at,description,space,hidden,user_id,'
'view_count,favorite_count,last_accessed_at,last_viewed_at,'
'deleted,deleted_at,deleter_id,dashboard_elements')
def test_scrape_looks_from_folder_should_return_list(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
look_data = {
'id': 123,
'title': 'A B C',
}
sdk.search_looks.return_value = [
serialize.deserialize(json.dumps(look_data), models.Look)
]
folder_data = {
'id': 'folder-id',
'name': 'Test folder',
'parent_id': '',
}
looks = self.__scraper.scrape_looks_from_folder(
serialize.deserialize(json.dumps(folder_data), models.Folder))
self.assertEqual(1, len(looks))
self.assertEqual(123, looks[0].id)
sdk.search_looks.assert_called_once()
sdk.search_looks.assert_called_with(
space_id='folder-id',
fields='id,title,created_at,updated_at,description,space,public,'
'user_id,last_updater_id,query_id,url,short_url,public_url,'
'excel_file_url,google_spreadsheet_formula,view_count,'
'favorite_count,last_accessed_at,last_viewed_at,deleted,'
'deleter_id')
def test_scrape_child_folders_should_return_list(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
folder_data = {
'id': 'folder-id',
'name': 'Folder name',
'parent_id': '',
}
sdk.folder_children.return_value = [
serialize.deserialize(json.dumps(folder_data), models.Folder)
]
parent_data = {
'id': 'parent-folder-id',
'name': 'Parent folder name',
'parent_id': '',
}
folders = self.__scraper.scrape_child_folders(
serialize.deserialize(json.dumps(parent_data), models.Folder))
self.assertEqual(1, len(folders))
self.assertEqual('folder-id', folders[0].id)
sdk.folder_children.assert_called_once()
sdk.folder_children.assert_called_with(
folder_id='parent-folder-id',
fields='id,name,parent_id,child_count,creator_id')
def test_deserialize_with_null():
data = copy.deepcopy(MODEL_DATA)
# json.dumps sets None to null
data["id"] = None
data["opt_enum1"] = None
data["opt_model_no_refs1"] = None
model = sr.deserialize(data=json.dumps(data),
structure=Model,
converter=converter)
assert model == Model(
enum1=Enum1.entry1,
model_no_refs1=ModelNoRefs1(name1="model_no_refs1_name"),
enum2=Enum2.entry2,
model_no_refs2=ModelNoRefs2(name2="model_no_refs2_name"),
list_enum1=[Enum1.entry1],
list_model_no_refs1=[ModelNoRefs1(name1="model_no_refs1_name")],
opt_enum1=None,
opt_model_no_refs1=None,
list_opt_model_no_refs1=[ModelNoRefs1(name1="model_no_refs1_name")],
id=None,
name="my-name",
datetime_field=DATETIME_VALUE,
class_="model-name",
finally_=[1, 2, 3],
)
def __make_fake_query(cls):
query_data = {
'id': 837,
'model': '',
'view': '',
}
return serialize.deserialize(json.dumps(query_data), models.Query)
def test_make_tag_for_folder_should_set_all_available_fields(self):
tag_template = \
self.__tag_template_factory.make_tag_template_for_folder()
folder_data = {
'id': 'a123-b456',
'name': 'Test name',
'parent_id': 'test-parent',
'child_count': 10,
'dashboards': [{}, {}],
'looks': [{}],
}
tag = self.__factory.make_tag_for_folder(
tag_template,
serialize.deserialize(json.dumps(folder_data), models.Folder))
self.assertEqual(tag_template.name, tag.template)
self.assertEqual('a123-b456', tag.fields['id'].string_value)
self.assertEqual('Test name', tag.fields['name'].string_value)
self.assertEqual('test-parent', tag.fields['parent_id'].string_value)
self.assertEqual(True, tag.fields['has_children'].bool_value)
self.assertEqual(10, tag.fields['children_count'].double_value)
self.assertEqual(True, tag.fields['has_dashboards'].bool_value)
self.assertEqual(2, tag.fields['dashboards_count'].double_value)
self.assertEqual(True, tag.fields['has_looks'].bool_value)
self.assertEqual(1, tag.fields['looks_count'].double_value)
self.assertEqual('https://test.server.com',
tag.fields['instance_url'].string_value)
def test_deserialize_single():
"""Deserialize functionality
Should handle python reserved keywords as well as attempting to
convert field values to proper type.
"""
# check that type conversion happens
data = copy.deepcopy(MODEL_DATA)
data["finally"][0]["id"] = "1"
model = sr.deserialize(data=json.dumps(data),
structure=Model,
converter=converter)
assert isinstance(model, Model)
assert isinstance(model.id, int)
assert model.id == 1
assert isinstance(model.name, str)
assert model.name == "my-name"
assert isinstance(model.class_, str)
assert model.class_ == "model-name"
assert isinstance(model.finally_, list)
assert len(model.finally_) == 2
for child in model.finally_:
assert isinstance(child, ChildModel)
assert isinstance(child.id, int)
def test_make_entry_for_dashboard_should_set_all_available_fields(self):
dashboard_data = {
'id': 'a123-b456',
'title': 'Test Name',
'created_at': '2019-09-12T16:30:00.000Z',
}
entry_id, entry = self.__factory.make_entry_for_dashboard(
serialize.deserialize(json.dumps(dashboard_data),
models.Dashboard))
self.assertEqual('lkr_test_server_com_db_a123_b456', entry_id)
self.assertEqual(
'projects/test-project/locations/test-location/'
'entryGroups/test-entry-group/entries/'
'lkr_test_server_com_db_a123_b456', entry.name)
self.assertEqual('test-system', entry.user_specified_system)
self.assertEqual('dashboard', entry.user_specified_type)
self.assertEqual('Test Name', entry.display_name)
self.assertEqual('https://test.server.com/dashboards/a123-b456',
entry.linked_resource)
created_datetime = self.__parse_datetime('2019-09-12T16:30:00+0000')
self.assertEqual(self.__get_datetime_secs(created_datetime),
entry.source_system_timestamps.create_time.seconds)
self.assertEqual(
self.__get_datetime_secs(created_datetime) + 10,
entry.source_system_timestamps.update_time.seconds)
def test_make_entry_for_look_should_set_all_available_fields(self):
look_data = {
'id': 123,
'title': 'Test Name',
'created_at': '2019-09-12T16:30:00.000Z',
'updated_at': '2019-09-12T17:35:00.000Z',
}
entry_id, entry = self.__factory.make_entry_for_look(
serialize.deserialize(json.dumps(look_data), models.Look))
self.assertEqual('lkr_test_server_com_lk_123', entry_id)
self.assertEqual(
'projects/test-project/locations/test-location/'
'entryGroups/test-entry-group/entries/'
'lkr_test_server_com_lk_123', entry.name)
self.assertEqual('test-system', entry.user_specified_system)
self.assertEqual('look', entry.user_specified_type)
self.assertEqual('Test Name', entry.display_name)
self.assertEqual('https://test.server.com/looks/123',
entry.linked_resource)
created_datetime = self.__parse_datetime('2019-09-12T16:30:00+0000')
self.assertEqual(self.__get_datetime_secs(created_datetime),
entry.source_system_timestamps.create_time.seconds)
updated_datetime = self.__parse_datetime('2019-09-12T17:35:10+0000')
self.assertEqual(self.__get_datetime_secs(updated_datetime),
entry.source_system_timestamps.update_time.seconds)
def test_deserialize_list():
# check that type conversion happens
data = [MODEL_DATA]
models = sr.deserialize(data=json.dumps(data),
structure=Sequence[Model],
converter=converter)
assert models == [
Model(
enum1=Enum1.entry1,
model_no_refs1=ModelNoRefs1(name1="model_no_refs1_name"),
enum2=Enum2.entry2,
model_no_refs2=ModelNoRefs2(name2="model_no_refs2_name"),
list_enum1=[Enum1.entry1],
list_model_no_refs1=[ModelNoRefs1(name1="model_no_refs1_name")],
opt_enum1=Enum1.entry1,
opt_model_no_refs1=ModelNoRefs1(name1="model_no_refs1_name"),
list_opt_model_no_refs1=[
ModelNoRefs1(name1="model_no_refs1_name")
],
id=1,
name="my-name",
datetime_field=DATETIME_VALUE,
class_="model-name",
finally_=[1, 2, 3],
),
]
def test_deserialize_single() -> None:
"""Deserialize functionality
Should handle python reserved keywords as well as attempting to
convert field values to proper type.
"""
# check that type conversion happens, str -> int and int -> str in this case
data = copy.deepcopy(MODEL_DATA)
data["id"] = "1"
data["name"] = 25
d = json.dumps(data)
model = sr.deserialize(data=d, structure=Model, converter=converter)
assert model == Model(
enum1=Enum1.entry1,
model_no_refs1=ModelNoRefs1(name1="model_no_refs1_name"),
enum2=Enum2.entry2,
model_no_refs2=ModelNoRefs2(name2="model_no_refs2_name"),
list_enum1=[Enum1.entry1],
list_model_no_refs1=[ModelNoRefs1(name1="model_no_refs1_name")],
opt_enum1=Enum1.entry1,
opt_model_no_refs1=ModelNoRefs1(name1="model_no_refs1_name"),
list_opt_model_no_refs1=[ModelNoRefs1(name1="model_no_refs1_name")],
id=1,
name="25",
datetime_field=DATETIME_VALUE,
class_="model-name",
finally_=[1, 2, 3],
)
def __make_fake_dashboard(cls, folder):
dashboard_data = {
'id': 'test_dashboard',
'space': json.loads(serialize.serialize(folder)),
'dashboard_elements': [],
}
return serialize.deserialize(json.dumps(dashboard_data),
models.Dashboard)
def test_make_tag_for_dashboard_should_set_all_available_fields(self):
tag_template = \
self.__tag_template_factory.make_tag_template_for_dashboard()
dashboard_data = {
'id': 'a123-b456',
'description': 'Test description',
'space': {
'id': 'test-folder',
'name': 'Test folder',
'parent_id': '',
},
'hidden': True,
'user_id': 10,
'view_count': 200,
'favorite_count': 3,
'last_accessed_at': '2019-09-12T16:30:00.000Z',
'last_viewed_at': '2019-09-12T16:35:00.000Z',
'deleted': True,
'deleted_at': '2019-09-12T16:40:00.000Z',
'deleter_id': 12,
}
tag = self.__factory.make_tag_for_dashboard(
tag_template,
serialize.deserialize(json.dumps(dashboard_data),
models.Dashboard))
self.assertEqual(tag_template.name, tag.template)
self.assertEqual('a123-b456', tag.fields['id'].string_value)
self.assertEqual('Test description',
tag.fields['description'].string_value)
self.assertEqual('test-folder', tag.fields['folder_id'].string_value)
self.assertEqual('Test folder', tag.fields['folder_name'].string_value)
self.assertEqual(True, tag.fields['is_hidden'].bool_value)
self.assertEqual(10, tag.fields['user_id'].double_value)
self.assertEqual(200, tag.fields['view_count'].double_value)
self.assertEqual(3, tag.fields['favorite_count'].double_value)
last_accessed_at = self.__parse_datetime('2019-09-12T16:30:00+0000')
self.assertEqual(
self.__get_datetime_secs(last_accessed_at),
tag.fields['last_accessed_at'].timestamp_value.seconds)
last_viewed_at = self.__parse_datetime('2019-09-12T16:35:00+0000')
self.assertEqual(self.__get_datetime_secs(last_viewed_at),
tag.fields['last_viewed_at'].timestamp_value.seconds)
self.assertEqual(True, tag.fields['is_deleted'].bool_value)
deleted_at = self.__parse_datetime('2019-09-12T16:40:00+0000')
self.assertEqual(self.__get_datetime_secs(deleted_at),
tag.fields['deleted_at'].timestamp_value.seconds)
self.assertEqual(12, tag.fields['deleter_id'].double_value)
self.assertEqual('https://test.server.com',
tag.fields['instance_url'].string_value)
def __make_fake_folder(cls, parent=None):
parent_data = json.loads(
serialize.serialize(parent)) if parent else None
folder_data = {
'id': 'test_child_folder' if parent else 'test_folder',
'name': 'Test child folder' if parent else 'Test folder',
'parent_id': parent_data['id'] if parent_data else '',
'child_count': 1 if parent else 0,
}
return serialize.deserialize(json.dumps(folder_data), models.Folder)
def test_make_entry_for_dashboard_tile_should_skip_empty_titles(self):
dashboard_element_data = {
'id': '196',
}
entry_id, entry = self.__factory.make_entry_for_dashboard_element(
serialize.deserialize(json.dumps(dashboard_element_data),
models.DashboardElement))
self.assertIsNone(entry_id)
self.assertIsNone(entry)
def test_make_entry_for_dashboard_tile_should_handle_title_text(self):
dashboard_element_data = {
'id': '196',
'title': '',
'title_text': 'Test Name',
}
entry_id, entry = self.__factory.make_entry_for_dashboard_element(
serialize.deserialize(json.dumps(dashboard_element_data),
models.DashboardElement))
self.assertEqual('Test Name', entry.display_name)
def test_make_tag_for_dashboard_tile_should_set_all_available_fields(self):
tag_template = self.__tag_template_factory\
.make_tag_template_for_dashboard_element()
dashboard_element_data = {
'id': '196',
'type': 'vis',
'dashboard_id': 'a123-b456',
'look_id': '25',
'look': {
'title': 'Test look',
},
'lookml_link_id': 'Test link ID',
'query_id': 837,
}
dashboard_data = {
'title': 'Test dashboard',
}
tag = self.__factory.make_tag_for_dashboard_element(
tag_template,
serialize.deserialize(json.dumps(dashboard_element_data),
models.DashboardElement),
serialize.deserialize(json.dumps(dashboard_data),
models.Dashboard))
self.assertEqual(tag_template.name, tag.template)
self.assertEqual('196', tag.fields['id'].string_value)
self.assertEqual('vis', tag.fields['type'].string_value)
self.assertEqual('a123-b456', tag.fields['dashboard_id'].string_value)
self.assertEqual('Test dashboard',
tag.fields['dashboard_title'].string_value)
self.assertEqual('25', tag.fields['look_id'].string_value)
self.assertEqual('Test look', tag.fields['look_title'].string_value)
self.assertEqual('Test link ID',
tag.fields['lookml_link_id'].string_value)
self.assertEqual(837, tag.fields['query_id'].double_value)
self.assertEqual('https://test.server.com',
tag.fields['instance_url'].string_value)
def __make_fake_folder(cls, dashboard=None, look=None):
dashboard_data = json.loads(
serialize.serialize(dashboard)) if dashboard else None
look_data = json.loads(serialize.serialize(look)) if look else None
folder_data = {
'id': 'test_folder',
'name': 'Test folder',
'parent_id': '',
'dashboards': [dashboard_data] if dashboard_data else None,
'looks': [look_data] if look_data else None,
}
return serialize.deserialize(json.dumps(folder_data), models.Folder)
def test_make_entry_for_dashboard_no_created_at_should_succeed(self):
dashboard_data = {
'id': 'a123-b456',
'title': 'Test Name',
}
entry_id, entry = self.__factory.make_entry_for_dashboard(
serialize.deserialize(json.dumps(dashboard_data),
models.Dashboard))
self.assertEqual(0, entry.source_system_timestamps.create_time.seconds)
self.assertEqual(0, entry.source_system_timestamps.update_time.seconds)
def test_deserialize_list():
# check that type conversion happens
data = [MODEL_DATA, MODEL_DATA]
models = sr.deserialize(json.dumps(data), Sequence[Model])
assert isinstance(models, list)
assert len(models) == 2
for model in models:
assert isinstance(model.finally_, list)
assert len(model.finally_) == 2
for child in model.finally_:
assert isinstance(child, ChildModel)
assert isinstance(child.id, int)
def test_deserialize_partial():
data = copy.deepcopy(MODEL_DATA)
del data["id"]
del data["finally"][0]["id"]
model = sr.deserialize(json.dumps(data), Model)
assert isinstance(model, Model)
assert model.id is None
assert isinstance(model.name, str)
assert model.name == "my-name"
assert isinstance(model.finally_, list)
assert len(model.finally_) == 2
assert model.finally_[0].id is None
assert model.finally_[1].id == 2
def test_scrape_look_should_return_object_on_success(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
look_data = {
'id': 123,
}
sdk.look.return_value = serialize.deserialize(json.dumps(look_data),
models.Look)
look = self.__scraper.scrape_look(123)
self.assertEqual(123, look.id)
sdk.look.assert_called_once()
sdk.look.assert_called_with(look_id=123)
def test_scrape_dashboard_should_return_object_on_success(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
dashboard_data = {
'id': 'dashboard-id',
}
sdk.dashboard.return_value = serialize.deserialize(
json.dumps(dashboard_data), models.Dashboard)
dashboard = self.__scraper.scrape_dashboard('dashboard-id')
self.assertEqual('dashboard-id', dashboard.id)
sdk.dashboard.assert_called_once()
sdk.dashboard.assert_called_with(dashboard_id='dashboard-id')
def test_scrape_connection_should_return_object(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
connection_data = {
'name': 'test-connection',
}
sdk.connection.return_value = serialize.deserialize(
json.dumps(connection_data), models.DBConnection)
connection = self.__scraper.scrape_connection('test-connection')
self.assertEqual('test-connection', connection.name)
sdk.connection.assert_called_once()
sdk.connection.assert_called_with(connection_name='test-connection')
def test_make_entry_for_look_no_updated_at_should_succeed(self):
look_data = {
'id': 123,
'title': 'Test Name',
'created_at': '2019-09-12T16:30:00.000Z',
}
entry_id, entry = self.__factory.make_entry_for_look(
serialize.deserialize(json.dumps(look_data), models.Look))
created_datetime = self.__parse_datetime('2019-09-12T16:30:00+0000')
self.assertEqual(self.__get_datetime_secs(created_datetime),
entry.source_system_timestamps.create_time.seconds)
self.assertEqual(
int(created_datetime.timestamp()) + 10,
entry.source_system_timestamps.update_time.seconds)
def test_deserialize_with_null():
data = copy.deepcopy(MODEL_DATA)
# json.dumps sets these to null
data["id"] = None
data["finally"][0]["id"] = None
model = sr.deserialize(data=json.dumps(data), structure=Model, converter=converter)
assert isinstance(model, Model)
assert model.id is None
assert isinstance(model.name, str)
assert model.name == "my-name"
assert isinstance(model.finally_, list)
assert len(model.finally_) == 2
assert model.finally_[0].id is None
assert model.finally_[1].id == 2
def test_make_tag_empty_string_value_should_skip_field(self):
tag_template = \
self.__tag_template_factory.make_tag_template_for_folder()
folder_data = {
'id': 'a123-b456',
'name': 'Test name',
'parent_id': '',
'child_count': 10,
}
tag = self.__factory.make_tag_for_folder(
tag_template,
serialize.deserialize(json.dumps(folder_data), models.Folder))
self.assertFalse('parent_id' in tag.fields)
def test_scrape_query_should_return_object(self):
sdk = self.__scraper.__dict__['_MetadataScraper__sdk']
query_data = {
'id': 123,
'model': '',
'view': '',
}
sdk.query.return_value = serialize.deserialize(json.dumps(query_data),
models.Query)
query = self.__scraper.scrape_query(123)
self.assertEqual(123, query.id)
sdk.query.assert_called_once()
sdk.query.assert_called_with(query_id=123)
def test_make_tag_for_look_should_set_url_field_if_available(self):
tag_template = self.__tag_template_factory.make_tag_template_for_look()
look_data = {
'space': {
'id': 'test-folder',
'name': 'Test folder',
'parent_id': '',
},
'url': 'test/url',
}
tag = self.__factory.make_tag_for_look(
tag_template,
serialize.deserialize(json.dumps(look_data), models.LookWithQuery))
self.assertEqual('test/url', tag.fields['url'].string_value)
def test_make_assembled_entries_list_should_process_dashboard_tiles(self):
entry_factory = self.__entry_factory
entry_factory.make_entry_for_folder = self.__mock_make_entry
entry_factory.make_entry_for_dashboard = self.__mock_make_entry
entry_factory.make_entry_for_dashboard_element = self.__mock_make_entry
tag_factory = self.__tag_factory
tag_factory.make_tag_for_dashboard_element = \
self.__mock_make_tag_parent_dep
dashboard_data = {
'id': 'test_dashboard',
'dashboard_elements': [{
'id': 194,
}],
}
dashboard = serialize.deserialize(json.dumps(dashboard_data),
models.Dashboard)
folder = self.__make_fake_folder()
folder.dashboards = [dashboard]
tag_templates_dict = {
'looker_dashboard_element_metadata': {
'name': 'tagTemplates/looker_dashboard_element_metadata',
}
}
assembled_entries = \
self.__assembled_data_factory.make_assembled_entries_list(
[folder], [], tag_templates_dict)
self.assertEqual(3, len(assembled_entries))
# The first entry refers to the folder and the second to the dashboard.
element_assembled_entry = assembled_entries[2]
self.assertEqual('194', element_assembled_entry.entry_id)
self.assertEqual('fake_entries/194',
element_assembled_entry.entry.name)
tags = element_assembled_entry.tags
self.assertEqual(1, len(tags))
self.assertEqual('tagTemplates/looker_dashboard_element_metadata',
tags[0].template)