def test_get_popular_tables(self) -> None:
# Test cache hit
with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jProxy, '_execute_cypher_query') as mock_execute:
mock_execute.return_value = [{'table_key': 'foo'}, {'table_key': 'bar'}]
neo4j_proxy = Neo4jProxy(host='DOES_NOT_MATTER', port=0000)
self.assertEqual(neo4j_proxy._get_popular_tables_uris(2), ['foo', 'bar'])
self.assertEqual(neo4j_proxy._get_popular_tables_uris(2), ['foo', 'bar'])
self.assertEqual(neo4j_proxy._get_popular_tables_uris(2), ['foo', 'bar'])
self.assertEquals(mock_execute.call_count, 1)
with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jProxy, '_execute_cypher_query') as mock_execute:
mock_execute.return_value = [
{'database_name': 'db', 'cluster_name': 'clstr', 'schema_name': 'sch', 'table_name': 'foo',
'table_description': 'test description'},
{'database_name': 'db', 'cluster_name': 'clstr', 'schema_name': 'sch', 'table_name': 'bar'}
]
neo4j_proxy = Neo4jProxy(host='DOES_NOT_MATTER', port=0000)
actual = neo4j_proxy.get_popular_tables(num_entries=2)
expected = [
PopularTable(database='db', cluster='clstr', schema='sch', name='foo', description='test description'),
PopularTable(database='db', cluster='clstr', schema='sch', name='bar'),
]
self.assertEqual(actual.__repr__(), expected.__repr__())
def test_get_popular_tables(self) -> None:
ent1 = self.to_class(self.entity1)
ent2 = self.to_class(self.entity2)
table_collection = MagicMock()
table_collection.entities = [ent1, ent2]
result = MagicMock(return_value=table_collection)
with patch.object(self.proxy._driver.search_basic, 'create', result):
response = self.proxy.get_popular_tables(num_entries=2)
ent1_attrs = cast(dict, self.entity1['attributes'])
ent2_attrs = cast(dict, self.entity2['attributes'])
expected = [
PopularTable(database=self.entity_type,
cluster=self.cluster,
schema=self.db,
name=ent1_attrs['name'],
description=ent1_attrs['description']),
PopularTable(database=self.entity_type,
cluster=self.cluster,
schema=self.db,
name=ent2_attrs['name'],
description=ent1_attrs['description']),
]
self.assertEqual(expected.__repr__(), response.__repr__())
def test_get_popular_tables_without_db(self) -> None:
meta1: Dict = copy.deepcopy(self.metadata1)
meta2: Dict = copy.deepcopy(self.metadata2)
meta1['attributes']['table'] = self.entity1
meta2['attributes']['table'] = self.entity2
metadata1 = self.to_class(meta1)
metadata2 = self.to_class(meta2)
metadata_collection = MagicMock()
metadata_collection.entities = [metadata1, metadata2]
result = MagicMock(return_value=metadata_collection)
with patch.object(self.proxy._driver.search_basic, 'create', result):
entity1: Dict = copy.deepcopy(self.entity1)
entity2: Dict = copy.deepcopy(self.entity2)
for entity in [entity1, entity2]:
entity['attributes']['qualifiedName'] = entity['attributes']['name']
entities_collection = MagicMock()
entities_collection.entities = [self.to_class(entity1), self.to_class(entity2)]
# Invalidate the cache to test the cache functionality
popular_query_params = {'typeName': 'table_metadata',
'sortBy': 'popularityScore',
'sortOrder': 'DESCENDING',
'excludeDeletedEntities': True,
'limit': 2,
'attributes': ['table']}
self.proxy._CACHE.region_invalidate(self.proxy._get_metadata_entities,
None, '_get_metadata_entities',
popular_query_params)
self.proxy._driver.entity_bulk = MagicMock(return_value=[entities_collection])
response = self.proxy.get_popular_tables(num_entries=2)
# Call multiple times for cache test.
self.proxy.get_popular_tables(num_entries=2)
self.proxy.get_popular_tables(num_entries=2)
self.proxy.get_popular_tables(num_entries=2)
self.proxy.get_popular_tables(num_entries=2)
self.assertEqual(1, self.proxy._driver.entity_bulk.call_count)
ent1_attrs = cast(dict, self.entity1['attributes'])
ent2_attrs = cast(dict, self.entity2['attributes'])
expected = [
PopularTable(database=self.entity_type, cluster='default', schema='default',
name=ent1_attrs['name'], description=ent1_attrs['description']),
PopularTable(database=self.entity_type, cluster='default', schema='default',
name=ent2_attrs['name'], description=ent1_attrs['description']),
]
self.assertEqual(expected.__repr__(), response.__repr__())
def test_get_popular_tables(self) -> None:
meta1: Dict = copy.deepcopy(self.metadata1)
meta2: Dict = copy.deepcopy(self.metadata2)
meta1['attributes']['table'] = self.entity1
meta2['attributes']['table'] = self.entity2
metadata1 = self.to_class(meta1)
metadata2 = self.to_class(meta2)
metadata_collection = MagicMock()
metadata_collection.entities = [metadata1, metadata2]
result = MagicMock(return_value=metadata_collection)
with patch.object(self.proxy._driver.search_basic, 'create', result):
entities_collection = MagicMock()
entities_collection.entities = [
self.to_class(self.entity1),
self.to_class(self.entity2)
]
self.proxy._driver.entity_bulk = MagicMock(
return_value=[entities_collection])
response = self.proxy.get_popular_tables(num_entries=2)
# Call multiple times for cache test.
self.proxy.get_popular_tables(num_entries=2)
self.proxy.get_popular_tables(num_entries=2)
self.proxy.get_popular_tables(num_entries=2)
self.proxy.get_popular_tables(num_entries=2)
self.assertEqual(self.proxy._driver.entity_bulk.call_count, 1)
ent1_attrs = cast(dict, self.entity1['attributes'])
ent2_attrs = cast(dict, self.entity2['attributes'])
expected = [
PopularTable(database=self.entity_type,
cluster=self.cluster,
schema=self.db,
name=ent1_attrs['name'],
description=ent1_attrs['description']),
PopularTable(database=self.entity_type,
cluster=self.cluster,
schema=self.db,
name=ent2_attrs['name'],
description=ent1_attrs['description']),
]
self.assertEqual(expected.__repr__(), response.__repr__())
def test_get_table_by_user_relation_own(self) -> None:
unique_attr_response = MagicMock()
unique_attr_response.entity = Data.user_entity_2
self.proxy._driver.entity_unique_attribute = MagicMock(
return_value=unique_attr_response)
entity_bulk_result = MagicMock()
entity_bulk_result.entities = [DottedDict(self.entity1)]
self.proxy._driver.entity_bulk = MagicMock(
return_value=[entity_bulk_result])
res = self.proxy.get_table_by_user_relation(
user_email='test_user_id', relation_type=UserResourceRel.own)
self.assertEqual(len(res.get("table")), 1) # type: ignore
ent1_attrs = cast(dict, self.entity1['attributes'])
expected = [
PopularTable(database=self.entity_type,
cluster=self.cluster,
schema=self.db,
name=ent1_attrs['name'],
description=ent1_attrs['description'])
]
self.assertEqual({'table': expected}, res)
def test_get_resources_owned_by_user_success(self) -> None:
unique_attr_response = MagicMock()
unique_attr_response.entity = Data.user_entity_2
self.proxy._driver.entity_unique_attribute = MagicMock(
return_value=unique_attr_response)
entity_bulk_result = MagicMock()
entity_bulk_result.entities = [DottedDict(self.entity1)]
self.proxy._driver.entity_bulk = MagicMock(
return_value=[entity_bulk_result])
res = self.proxy._get_resources_owned_by_user(
user_id='test_user_2', resource_type=ResourceType.Table.name)
self.assertEqual(len(res), 1)
ent1_attrs = cast(dict, self.entity1['attributes'])
expected = [
PopularTable(database=self.entity_type,
cluster=self.cluster,
schema=self.db,
name=ent1_attrs['name'],
description=ent1_attrs['description'])
]
self.assertEqual(expected, res)
def get_frequently_used_tables(self, *, user_email: str) -> Dict[str, Any]:
"""
Retrieves all Table the resources per user on READ relation.
:param user_email: the email of the user
:return:
"""
query = textwrap.dedent("""
MATCH (user:User {key: $query_key})-[r:READ]->(tbl:Table)
WHERE EXISTS(r.published_tag) AND r.published_tag IS NOT NULL
WITH user, r, tbl ORDER BY r.published_tag DESC, r.read_count DESC LIMIT 50
MATCH (tbl:Table)<-[:TABLE]-(schema:Schema)<-[:SCHEMA]-(clstr:Cluster)<-[:CLUSTER]-(db:Database)
OPTIONAL MATCH (tbl)-[:DESCRIPTION]->(tbl_dscrpt:Description)
RETURN db, clstr, schema, tbl, tbl_dscrpt
""")
table_records = self._execute_cypher_query(statement=query, param_dict={'query_key': user_email})
if not table_records:
raise NotFoundException('User {user_id} does not READ any resources'.format(user_id=user_email))
results = []
for record in table_records:
results.append(PopularTable(
database=record['db']['name'],
cluster=record['clstr']['name'],
schema=record['schema']['name'],
name=record['tbl']['name'],
description=self._safe_get(record, 'tbl_dscrpt', 'description')))
return {'table': results}
def get_table_by_user_relation(self, *, user_email: str, relation_type: UserResourceRel) \
-> Dict[str, List[PopularTable]]:
"""
Retrive all follow the Table per user based on the relation.
:param user_email: the email of the user
:param relation_type: the relation between the user and the resource
:return:
"""
rel_clause: str = self._get_user_resource_relationship_clause(relation_type=relation_type,
id='',
resource_type=ResourceType.Table,
user_key=user_email)
query = textwrap.dedent(f"""
MATCH {rel_clause}<-[:TABLE]-(schema:Schema)<-[:SCHEMA]-(clstr:Cluster)<-[:CLUSTER]-(db:Database)
WITH db, clstr, schema, resource
OPTIONAL MATCH (resource)-[:DESCRIPTION]->(tbl_dscrpt:Description)
RETURN db, clstr, schema, resource, tbl_dscrpt""")
table_records = self._execute_cypher_query(statement=query, param_dict={'user_key': user_email})
if not table_records:
raise NotFoundException('User {user_id} does not {relation} any resources'.format(user_id=user_email,
relation=relation_type))
results = []
for record in table_records:
results.append(PopularTable(
database=record['db']['name'],
cluster=record['clstr']['name'],
schema=record['schema']['name'],
name=record['resource']['name'],
description=self._safe_get(record, 'tbl_dscrpt', 'description')))
return {ResourceType.Table.name.lower(): results}
def get_popular_tables(self, *, num_entries: int) -> List[PopularTable]:
"""
Retrieve popular tables. As popular table computation requires full scan of table and user relationship,
it will utilize cached method _get_popular_tables_uris.
:param num_entries:
:return: Iterable of PopularTable
"""
table_uris = self._get_popular_tables_uris(num_entries)
if not table_uris:
return []
query = textwrap.dedent("""
MATCH (db:Database)-[:CLUSTER]->(clstr:Cluster)-[:SCHEMA]->(schema:Schema)-[:TABLE]->(tbl:Table)
WHERE tbl.key IN $table_uris
WITH db.name as database_name, clstr.name as cluster_name, schema.name as schema_name, tbl
OPTIONAL MATCH (tbl)-[:DESCRIPTION]->(dscrpt:Description)
RETURN database_name, cluster_name, schema_name, tbl.name as table_name,
dscrpt.description as table_description;
""")
records = self._execute_cypher_query(statement=query,
param_dict={'table_uris': table_uris})
popular_tables = []
for record in records:
popular_table = PopularTable(database=record['database_name'],
cluster=record['cluster_name'],
schema=record['schema_name'],
name=record['table_name'],
description=self._safe_get(record, 'table_description'))
popular_tables.append(popular_table)
return popular_tables
def get_popular_tables(self, *, num_entries: int) -> List[PopularTable]:
"""
:param num_entries: Number of popular tables to fetch
:return: A List of popular tables instances
"""
popular_tables = list()
popular_query_params = {
'typeName': 'Table',
'sortBy': 'popularityScore',
'sortOrder': 'DESCENDING',
'excludeDeletedEntities': True,
'limit': num_entries
}
search_results = self._driver.search_basic.create(
data=popular_query_params)
for table in search_results.entities:
table_attrs = table.attributes
table_qn = parse_table_qualified_name(
qualified_name=table_attrs.get(self.QN_KEY))
table_name = table_qn.get("table_name") or table_attrs.get('name')
db_name = table_qn.get("db_name", '')
db_cluster = table_qn.get("cluster_name", '')
popular_table = PopularTable(
database=table.typeName,
cluster=db_cluster,
schema=db_name,
name=table_name,
description=table_attrs.get('description')
or table_attrs.get('comment'))
popular_tables.append(popular_table)
return popular_tables
def _serialize_popular_tables(self, entities: list) -> List[PopularTable]:
"""
Gets a list of entities and serialize the popular tables.
:param entities: List of entities from atlas client
:return: a list of PopularTable objects
"""
popular_tables = list()
for table in entities:
table_attrs = table.attributes
table_qn = parse_table_qualified_name(
qualified_name=table_attrs.get(self.QN_KEY)
)
table_name = table_qn.get("table_name") or table_attrs.get('name')
db_name = table_qn.get("db_name", '')
db_cluster = table_qn.get("cluster_name", '')
popular_table = PopularTable(
database=table.typeName,
cluster=db_cluster,
schema=db_name,
name=table_name,
description=table_attrs.get('description') or table_attrs.get('comment'))
popular_tables.append(popular_table)
return popular_tables
def test_get_table_by_user_relation(self) -> None:
bookmark1 = copy.deepcopy(self.bookmark_entity1)
bookmark1 = self.to_class(bookmark1)
bookmark_collection = MagicMock()
bookmark_collection.entities = [bookmark1]
self.proxy._driver.search_basic.create = MagicMock(return_value=bookmark_collection)
res = self.proxy.get_table_by_user_relation(user_email='test_user_id',
relation_type=UserResourceRel.follow)
expected = [PopularTable(database=Data.entity_type, cluster=Data.cluster, schema=Data.db,
name=Data.name, description=None)]
self.assertEqual(res, {'table': expected})
def test_get_frequently_used_tables(self) -> None:
entity_unique_attribute_result = MagicMock()
entity_unique_attribute_result.entity = DottedDict(self.user_entity_2)
self.proxy._driver.entity_unique_attribute = MagicMock(return_value=entity_unique_attribute_result)
entity_bulk_result = MagicMock()
entity_bulk_result.entities = [DottedDict(self.reader_entity_1)]
self.proxy._driver.entity_bulk = MagicMock(return_value=[entity_bulk_result])
expected = {'table': [PopularTable(cluster=self.cluster,
name='Table1',
schema=self.db,
database=self.entity_type)]}
res = self.proxy.get_frequently_used_tables(user_email='dummy')
self.assertEqual(expected, res)
def test_get_table_by_user_relation(self, mock_rds_client: Any) -> None:
table = RDSTable(name='test_table')
table_description = RDSTableDescription(description='test_description')
schema = RDSSchema(name='test_schema')
cluster = RDSCluster(name='test_cluster')
database = RDSDatabase(name='test_database')
cluster.database = database
schema.cluster = cluster
table.schema = schema
table.description = table_description
mock_client = MagicMock()
mock_rds_client.return_value = mock_client
mock_create_session = MagicMock()
mock_client.create_session.return_value = mock_create_session
mock_session = MagicMock()
mock_create_session.__enter__.return_value = mock_session
mock_session_query = MagicMock()
mock_session.query.return_value = mock_session_query
mock_session_query_filter = MagicMock()
mock_session_query.filter.return_value = mock_session_query_filter
mock_session_query_filter_options = MagicMock()
mock_session_query_filter.options.return_value = mock_session_query_filter_options
mock_session_query_filter_options.all.return_value = [table]
expected = {
'table': [
PopularTable(database='test_database',
cluster='test_cluster',
schema='test_schema',
name='test_table',
description='test_description')
]
}
proxy = MySQLProxy()
actual = proxy.get_table_by_user_relation(
user_email='test_user', relation_type=UserResourceRel.follow)
self.assertEqual(len(actual['table']), 1)
self.assertEqual(expected, actual)
def _get_resources_followed_by_user(self, user_id: str, resource_type: str) \
-> List[Union[PopularTable, DashboardSummary]]:
"""
ToDo (Verdan): Dashboard still needs to be implemented.
Helper function to get the resource, table, dashboard etc followed by a user.
:param user_id: User ID of a user
:param resource_type: Type of a resource that returns, could be table, dashboard etc.
:return: A list of PopularTable, DashboardSummary or any other resource.
"""
params = {
'typeName': self.BOOKMARK_TYPE,
'offset': '0',
'limit': '1000',
'excludeDeletedEntities': True,
'entityFilters': {
'condition': 'AND',
'criterion': [
{
'attributeName': self.QN_KEY,
'operator': 'contains',
'attributeValue': f'.{user_id}.bookmark'
},
{
'attributeName': self.BOOKMARK_ACTIVE_KEY,
'operator': 'eq',
'attributeValue': 'true'
}
]
},
'attributes': ['count', self.QN_KEY, self.ENTITY_URI_KEY]
}
# Fetches the bookmark entities based on filters
search_results = self.client.discovery.faceted_search(search_parameters=params)
resources = []
for record in search_results.entities:
table_info = self._extract_info_from_uri(table_uri=record.attributes[self.ENTITY_URI_KEY])
res = self._parse_bookmark_qn(record.attributes[self.QN_KEY])
resources.append(PopularTable(
database=table_info['entity'],
cluster=res['cluster'],
schema=res['db'],
name=res['table']))
return resources
def get_frequently_used_tables(
self, *, user_email: str) -> Dict[str, List[PopularTable]]:
user = self._driver.entity_unique_attribute(
self.USER_TYPE, qualifiedName=user_email).entity
readers_guids = []
for user_reads in user['relationshipAttributes'].get('entityReads'):
entity_status = user_reads['entityStatus']
relationship_status = user_reads['relationshipStatus']
if entity_status == 'ACTIVE' and relationship_status == 'ACTIVE':
readers_guids.append(user_reads['guid'])
readers = extract_entities(
self._driver.entity_bulk(guid=readers_guids,
ignoreRelationships=True))
_results = {}
for reader in readers:
entity_uri = reader.attributes.get(self.ENTITY_URI_KEY)
count = reader.attributes.get('count')
if count:
details = self._extract_info_from_uri(table_uri=entity_uri)
_results[count] = dict(cluster=details.get('cluster'),
name=details.get('name'),
schema=details.get('db'),
database=details.get('entity'))
sorted_counts = sorted(_results.keys())
results = []
for count in sorted_counts:
data: dict = _results.get(count, dict())
table = PopularTable(**data)
results.append(table)
return {'table': results}
def test_get_popular_tables(self) -> None:
application = Fixtures.next_application()
self.get_proxy().put_app(data=application)
# Add 10 tables
tables: List[Table] = [
Fixtures.next_table(application=application) for _ in range(10)
]
self.get_proxy().post_tables(tables=tables)
user = Fixtures.next_user()
self.get_proxy().put_user(data=user)
# add reads to 6 of them, expecting that only the top five will be "popular"
expected_popular_tables = []
reads = 0
for i in range(6):
table_name: str = checkNotNone(tables[i].name)
table_uri: str = checkNotNone(tables[i].key)
self.get_proxy().add_read_count(table_uri=table_uri,
user_id=f'{user.user_id}',
read_count=reads)
if reads > 0:
expected_popular_tables.append(table_name)
reads += 1000
# ensure popular tables returns those 5 we added
actual_popular_tables = self.get_proxy().get_popular_tables(
num_entries=5)
self.assertEqual(len(actual_popular_tables), 5)
popular_tables = []
for table in tables:
if table.name in expected_popular_tables:
popular_tables.append(
PopularTable(database=table.database,
cluster=table.cluster,
schema=table.schema,
name=table.name,
description=table.description))
self.assertEqual(sorted(actual_popular_tables), sorted(popular_tables))
def get_frequently_used_tables(
self, *, user_email: str) -> Dict[str, List[PopularTable]]:
user = self.client.entity.get_entity_by_attribute(
type_name=self.USER_TYPE,
uniq_attributes=[(self.QN_KEY, user_email)]).entity
readers_guids = []
for user_reads in user['relationshipAttributes'].get('entityReads'):
entity_status = user_reads['entityStatus']
relationship_status = user_reads['relationshipStatus']
if entity_status == Status.ACTIVE and relationship_status == Status.ACTIVE:
readers_guids.append(user_reads['guid'])
readers = self.client.entity.get_entities_by_guids(
guids=list(readers_guids), ignore_relationships=True)
_results = {}
for reader in readers.entities or list():
entity_uri = reader.attributes.get(self.ENTITY_URI_KEY)
count = reader.attributes.get('count')
if count:
details = self._extract_info_from_uri(table_uri=entity_uri)
_results[count] = dict(cluster=details.get('cluster'),
name=details.get('name'),
schema=details.get('db'),
database=details.get('entity'))
sorted_counts = sorted(_results.keys())
results = []
for count in sorted_counts:
data: dict = _results.get(count, dict())
table = PopularTable(**data)
results.append(table)
return {'table': results}
def get_table_by_user_relation(
self, *, user_email: str,
relation_type: UserResourceRel) -> Dict[str, Any]:
params = {
'typeName': self.BOOKMARK_TYPE,
'offset': '0',
'limit': '1000',
'excludeDeletedEntities': True,
'entityFilters': {
'condition':
'AND',
'criterion': [{
'attributeName': self.QN_KEY,
'operator': 'contains',
'attributeValue': f'.{user_email}.bookmark'
}, {
'attributeName': self.BOOKMARK_ACTIVE_KEY,
'operator': 'eq',
'attributeValue': 'true'
}]
},
'attributes': ['count', self.QN_KEY, self.ENTITY_URI_KEY]
}
# Fetches the bookmark entities based on filters
search_results = self._driver.search_basic.create(data=params)
results = []
for record in search_results.entities:
table_info = self._extract_info_from_uri(
table_uri=record.attributes[self.ENTITY_URI_KEY])
res = self._parse_bookmark_qn(record.attributes[self.QN_KEY])
results.append(
PopularTable(database=table_info['entity'],
cluster=res['cluster'],
schema=res['db'],
name=res['table']))
return {'table': results}
def test_get(self) -> None:
self.mock_client.get_table_by_user_relation.return_value = {
'table': [
PopularTable(database='d1',
cluster='c1',
schema='s1bbc',
name='n1_test_a_table'),
PopularTable(database='d1',
cluster='c2',
schema='s1abc',
name='n1_test_b_table'),
PopularTable(database='d1',
cluster='c3',
schema='s1abc',
name='n1_test_a_table'),
]
}
self.mock_client.get_dashboard_by_user_relation.return_value = {
'dashboard': [
DashboardSummary(uri='foobar_dashboard_3',
cluster='cluster',
group_name='dashboard_group_b',
group_url='http://foo.bar/group',
product='foobar',
name='dashboard_b',
url='http://foo.bar/dashboard_b'),
DashboardSummary(uri='foobar_dashboard_2',
cluster='cluster',
group_name='dashboard_group_a',
group_url='http://foo.bar/group',
product='foobar',
name='dashboard_a',
url='http://foo.bar/dashboard_a'),
DashboardSummary(uri='foobar_dashboard_1',
cluster='cluster',
group_name='dashboard_group_a',
group_url='http://foo.bar/group',
product='foobar',
name='dashboard_c',
url='http://foo.bar/dashboard_c'),
]
}
response = self.api.get(user_id='username')
self.assertEqual(list(response)[1], HTTPStatus.OK)
self.mock_client.get_table_by_user_relation.assert_called_once()
# test results are sorted
assert list(response)[0].get('table') == [
{
'cluster': 'c3',
'database': 'd1',
'description': None,
'name': 'n1_test_a_table',
'schema': 's1abc'
},
{
'cluster': 'c2',
'database': 'd1',
'description': None,
'name': 'n1_test_b_table',
'schema': 's1abc'
},
{
'cluster': 'c1',
'database': 'd1',
'description': None,
'name': 'n1_test_a_table',
'schema': 's1bbc'
},
]
assert list(response)[0].get('dashboard') == [
{
'chart_names': [],
'cluster': 'cluster',
'description': None,
'group_name': 'dashboard_group_a',
'group_url': 'http://foo.bar/group',
'last_successful_run_timestamp': None,
'name': 'dashboard_a',
'product': 'foobar',
'uri': 'foobar_dashboard_2',
'url': 'http://foo.bar/dashboard_a'
},
{
'chart_names': [],
'cluster': 'cluster',
'description': None,
'group_name': 'dashboard_group_a',
'group_url': 'http://foo.bar/group',
'last_successful_run_timestamp': None,
'name': 'dashboard_c',
'product': 'foobar',
'uri': 'foobar_dashboard_1',
'url': 'http://foo.bar/dashboard_c'
},
{
'chart_names': [],
'cluster': 'cluster',
'description': None,
'group_name': 'dashboard_group_b',
'group_url': 'http://foo.bar/group',
'last_successful_run_timestamp': None,
'name': 'dashboard_b',
'product': 'foobar',
'uri': 'foobar_dashboard_3',
'url': 'http://foo.bar/dashboard_b'
},
]
def test_get_dashboard(self) -> None:
self.proxy.client.entity.get_entity_by_attribute = MagicMock(
return_value=self.dashboard_data) # type: ignore
self.proxy._get_dashboard_group = MagicMock(
return_value=self.dashboard_group_data) # type: ignore
self.proxy.client.entity.get_entities_by_guids = MagicMock(
return_value=DottedDict({'entities': [DottedDict(self.entity1)]}))
expected = DashboardDetail(
uri='superset_dashboard://datalab.prod/1',
cluster='datalab',
group_name='prod superset',
group_url='https://superset.prod',
product='superset',
name='Prod Usage',
url='https://prod.superset/dashboards/1',
description='Robs famous dashboard',
created_timestamp=1619517099,
updated_timestamp=1619626531,
last_successful_run_timestamp=1619517099,
last_run_timestamp=1619517150,
last_run_state='failed',
owners=[
User(user_id='lisa_salinas',
email='lisa_salinas',
first_name=None,
last_name=None,
full_name=None,
display_name=None,
is_active=True,
github_username=None,
team_name=None,
slack_id=None,
employee_type=None,
manager_fullname=None,
manager_email=None,
manager_id=None,
role_name=None,
profile_url=None,
other_key_values={})
],
frequent_users=[],
chart_names=['Count Users by Time', 'Total Count'],
query_names=['User Count By Time', 'Total Count'],
queries=[
DashboardQuery(
name='User Count By Time',
url='https://prod.superset/dashboards/1/query/1',
query_text='SELECT date, COUNT(1) FROM db.table GROUP BY 1'
),
DashboardQuery(
name='Total Count',
url='https://prod.superset/dashboards/1/query/2',
query_text='SELECT COUNT(1) FROM db.table')
],
tables=[
PopularTable(database='hive_table',
cluster='TEST_CLUSTER',
schema='TEST_DB',
name='Table1',
description='Dummy Description')
],
tags=[],
badges=[],
recent_view_count=0)
result = self.proxy.get_dashboard(
id='superset_dashboard://datalab.prod/1')
self.assertEqual(expected, result)
def test_get_dashboard(self) -> None:
with patch.object(GraphDatabase, 'driver'), patch.object(
Neo4jProxy, '_execute_cypher_query') as mock_execute:
mock_execute.return_value.single.side_effect = [{
'cluster_name':
'cluster_name',
'uri':
'foo_dashboard://gold.bar/dashboard_id',
'url':
'http://www.foo.bar/dashboard_id',
'product':
'foobar',
'name':
'dashboard name',
'created_timestamp':
123456789,
'description':
'description',
'group_name':
'group_name',
'group_url':
'http://www.group_url.com',
'last_successful_run_timestamp':
9876543210,
'last_run_timestamp':
987654321,
'last_run_state':
'good_state',
'updated_timestamp':
123456654321,
'recent_view_count':
100,
'owners': [{
'employee_type': 'teamMember',
'full_name': 'test_full_name',
'is_active': 'True',
'github_username': '******',
'slack_id': 'test_id',
'last_name': 'test_last_name',
'first_name': 'test_first_name',
'team_name': 'test_team',
'email': 'test_email',
}, {
'employee_type': 'teamMember',
'full_name': 'test_full_name2',
'is_active': 'True',
'github_username': '******',
'slack_id': 'test_id2',
'last_name': 'test_last_name2',
'first_name': 'test_first_name2',
'team_name': 'test_team2',
'email': 'test_email2',
}],
'tags': [{
'key': 'tag_key1',
'tag_type': 'tag_type1'
}, {
'key': 'tag_key2',
'tag_type': 'tag_type2'
}],
'charts': [{
'name': 'chart1'
}, {
'name': 'chart2'
}],
'queries': [{
'name': 'query1'
}, {
'name': 'query2'
}],
'tables': [{
'database': 'db1',
'name': 'table1',
'description': 'table description 1',
'cluster': 'cluster1',
'schema': 'schema1'
}, {
'database': 'db2',
'name': 'table2',
'description': None,
'cluster': 'cluster2',
'schema': 'schema2'
}]
}, {
'cluster_name':
'cluster_name',
'uri':
'foo_dashboard://gold.bar/dashboard_id',
'url':
'http://www.foo.bar/dashboard_id',
'product':
'foobar',
'name':
'dashboard name',
'created_timestamp':
123456789,
'description':
None,
'group_name':
'group_name',
'group_url':
'http://www.group_url.com',
'last_run_timestamp':
None,
'last_run_state':
None,
'updated_timestamp':
None,
'recent_view_count':
0,
'owners': [],
'tags': [],
'charts': [],
'queries': [],
'tables': []
}]
neo4j_proxy = Neo4jProxy(host='DOES_NOT_MATTER', port=0000)
dashboard = neo4j_proxy.get_dashboard(id='dashboard_id')
expected = DashboardDetail(
uri='foo_dashboard://gold.bar/dashboard_id',
cluster='cluster_name',
group_name='group_name',
group_url='http://www.group_url.com',
product='foobar',
name='dashboard name',
url='http://www.foo.bar/dashboard_id',
description='description',
created_timestamp=123456789,
last_successful_run_timestamp=9876543210,
updated_timestamp=123456654321,
last_run_timestamp=987654321,
last_run_state='good_state',
owners=[
User(email='test_email',
first_name='test_first_name',
last_name='test_last_name',
full_name='test_full_name',
is_active='True',
github_username='******',
team_name='test_team',
slack_id='test_id',
employee_type='teamMember',
manager_fullname=''),
User(email='test_email2',
first_name='test_first_name2',
last_name='test_last_name2',
full_name='test_full_name2',
is_active='True',
github_username='******',
team_name='test_team2',
slack_id='test_id2',
employee_type='teamMember',
manager_fullname='')
],
frequent_users=[],
chart_names=['chart1', 'chart2'],
query_names=['query1', 'query2'],
tables=[
PopularTable(database='db1',
name='table1',
description='table description 1',
cluster='cluster1',
schema='schema1'),
PopularTable(database='db2',
name='table2',
cluster='cluster2',
schema='schema2'),
],
tags=[
Tag(tag_type='tag_type1', tag_name='tag_key1'),
Tag(tag_type='tag_type2', tag_name='tag_key2')
],
recent_view_count=100)
self.assertEqual(expected, dashboard)
dashboard2 = neo4j_proxy.get_dashboard(id='dashboard_id')
expected2 = DashboardDetail(
uri='foo_dashboard://gold.bar/dashboard_id',
cluster='cluster_name',
group_name='group_name',
group_url='http://www.group_url.com',
product='foobar',
name='dashboard name',
url='http://www.foo.bar/dashboard_id',
description=None,
created_timestamp=123456789,
updated_timestamp=None,
last_run_timestamp=None,
last_run_state=None,
owners=[],
frequent_users=[],
chart_names=[],
query_names=[],
tables=[],
tags=[],
last_successful_run_timestamp=None,
recent_view_count=0)
self.assertEqual(expected2, dashboard2)
def test_get_dashboard(self, mock_rds_client: Any) -> None:
# dashboard_metadata
dashboard = RDSDashboard(
rk='foo_dashboard://gold.bar/dashboard_id',
name='dashboard name',
dashboard_url='http://www.foo.bar/dashboard_id',
created_timestamp=123456789)
dashboard_group = RDSDashboardGroup(
name='group_name', dashboard_group_url='http://www.group_url.com')
dashboard_group.cluster = RDSCluster(name='cluster_name')
dashboard.group = dashboard_group
dashboard.description = RDSDashboardDescription(
description='description')
dashboard.execution = [
RDSDashboardExecution(rk='dashboard_last_successful_execution',
timestamp=9876543210),
RDSDashboardExecution(rk='dashboard_last_execution',
timestamp=987654321,
state='good_state')
]
dashboard.timestamp = RDSDashboardTimestamp(timestamp=123456654321)
dashboard.tags = [
RDSTag(rk='tag_key1', tag_type='default'),
RDSTag(rk='tag_key2', tag_type='default')
]
dashboard.badges = [RDSBadge(rk='golden', category='table_status')]
dashboard.owners = [
RDSUser(email='test_email',
first_name='test_first_name',
last_name='test_last_name',
full_name='test_full_name',
is_active=True,
github_username='******',
team_name='test_team',
slack_id='test_id',
employee_type='teamMember'),
RDSUser(email='test_email2',
first_name='test_first_name2',
last_name='test_last_name2',
full_name='test_full_name2',
is_active=True,
github_username='******',
team_name='test_team2',
slack_id='test_id2',
employee_type='teamMember')
]
dashboard.usage = [RDSDashboardUsage(read_count=100)]
mock_client = MagicMock()
mock_rds_client.return_value = mock_client
mock_create_session = MagicMock()
mock_client.create_session.return_value = mock_create_session
mock_session = MagicMock()
mock_create_session.__enter__.return_value = mock_session
mock_session_query = MagicMock()
mock_session.query.return_value = mock_session_query
mock_session_query_filter = MagicMock()
mock_session_query.filter.return_value = mock_session_query_filter
mock_session_query_filter.first.return_value = dashboard
# queries
query1 = RDSDashboardQuery(name='query1')
query2 = RDSDashboardQuery(name='query2',
url='http://foo.bar/query',
query_text='SELECT * FROM foo.bar')
query1.charts = [RDSDashboardChart(name='chart1')]
query2.charts = [RDSDashboardChart(name='chart2')]
queries = [query1, query2]
# tables
database1 = RDSDatabase(name='db1')
database2 = RDSDatabase(name='db2')
cluster1 = RDSCluster(name='cluster1')
cluster2 = RDSCluster(name='cluster2')
schema1 = RDSSchema(name='schema1')
schema2 = RDSSchema(name='schema2')
table1 = RDSTable(name='table1')
table2 = RDSTable(name='table2')
description1 = RDSTableDescription(description='table description 1')
schema1.cluster = cluster1
cluster1.database = database1
schema2.cluster = cluster2
cluster2.database = database2
table1.schema = schema1
table2.schema = schema2
table1.description = description1
tables = [table1, table2]
mock_session_query_filter_options = MagicMock()
mock_session_query_filter.options.return_value = mock_session_query_filter_options
mock_session_query_filter_options.all.side_effect = [queries, tables]
expected = DashboardDetail(
uri='foo_dashboard://gold.bar/dashboard_id',
cluster='cluster_name',
group_name='group_name',
group_url='http://www.group_url.com',
product='foo',
name='dashboard name',
url='http://www.foo.bar/dashboard_id',
description='description',
created_timestamp=123456789,
last_successful_run_timestamp=9876543210,
updated_timestamp=123456654321,
last_run_timestamp=987654321,
last_run_state='good_state',
owners=[
User(email='test_email',
first_name='test_first_name',
last_name='test_last_name',
full_name='test_full_name',
is_active=True,
github_username='******',
team_name='test_team',
slack_id='test_id',
employee_type='teamMember',
manager_fullname=''),
User(email='test_email2',
first_name='test_first_name2',
last_name='test_last_name2',
full_name='test_full_name2',
is_active=True,
github_username='******',
team_name='test_team2',
slack_id='test_id2',
employee_type='teamMember',
manager_fullname='')
],
frequent_users=[],
chart_names=['chart1', 'chart2'],
query_names=['query1', 'query2'],
queries=[
DashboardQuery(name='query1'),
DashboardQuery(name='query2',
url='http://foo.bar/query',
query_text='SELECT * FROM foo.bar')
],
tables=[
PopularTable(database='db1',
name='table1',
description='table description 1',
cluster='cluster1',
schema='schema1'),
PopularTable(database='db2',
name='table2',
cluster='cluster2',
schema='schema2'),
],
tags=[
Tag(tag_type='default', tag_name='tag_key1'),
Tag(tag_type='default', tag_name='tag_key2')
],
badges=[Badge(badge_name='golden', category='table_status')],
recent_view_count=100)
proxy = MySQLProxy()
actual = proxy.get_dashboard(id='dashboard_id')
self.assertEqual(expected, actual)
