def test_get_popular_tables(self) -> None: # Test cache hit with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jProxy, '_execute_cypher_query') as mock_execute: mock_execute.return_value = [{'table_key': 'foo'}, {'table_key': 'bar'}] neo4j_proxy = Neo4jProxy(host='DOES_NOT_MATTER', port=0000) self.assertEqual(neo4j_proxy._get_popular_tables_uris(2), ['foo', 'bar']) self.assertEqual(neo4j_proxy._get_popular_tables_uris(2), ['foo', 'bar']) self.assertEqual(neo4j_proxy._get_popular_tables_uris(2), ['foo', 'bar']) self.assertEquals(mock_execute.call_count, 1) with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jProxy, '_execute_cypher_query') as mock_execute: mock_execute.return_value = [ {'database_name': 'db', 'cluster_name': 'clstr', 'schema_name': 'sch', 'table_name': 'foo', 'table_description': 'test description'}, {'database_name': 'db', 'cluster_name': 'clstr', 'schema_name': 'sch', 'table_name': 'bar'} ] neo4j_proxy = Neo4jProxy(host='DOES_NOT_MATTER', port=0000) actual = neo4j_proxy.get_popular_tables(num_entries=2) expected = [ PopularTable(database='db', cluster='clstr', schema='sch', name='foo', description='test description'), PopularTable(database='db', cluster='clstr', schema='sch', name='bar'), ] self.assertEqual(actual.__repr__(), expected.__repr__())
def test_get_popular_tables(self) -> None: ent1 = self.to_class(self.entity1) ent2 = self.to_class(self.entity2) table_collection = MagicMock() table_collection.entities = [ent1, ent2] result = MagicMock(return_value=table_collection) with patch.object(self.proxy._driver.search_basic, 'create', result): response = self.proxy.get_popular_tables(num_entries=2) ent1_attrs = cast(dict, self.entity1['attributes']) ent2_attrs = cast(dict, self.entity2['attributes']) expected = [ PopularTable(database=self.entity_type, cluster=self.cluster, schema=self.db, name=ent1_attrs['name'], description=ent1_attrs['description']), PopularTable(database=self.entity_type, cluster=self.cluster, schema=self.db, name=ent2_attrs['name'], description=ent1_attrs['description']), ] self.assertEqual(expected.__repr__(), response.__repr__())
def test_get_popular_tables_without_db(self) -> None: meta1: Dict = copy.deepcopy(self.metadata1) meta2: Dict = copy.deepcopy(self.metadata2) meta1['attributes']['table'] = self.entity1 meta2['attributes']['table'] = self.entity2 metadata1 = self.to_class(meta1) metadata2 = self.to_class(meta2) metadata_collection = MagicMock() metadata_collection.entities = [metadata1, metadata2] result = MagicMock(return_value=metadata_collection) with patch.object(self.proxy._driver.search_basic, 'create', result): entity1: Dict = copy.deepcopy(self.entity1) entity2: Dict = copy.deepcopy(self.entity2) for entity in [entity1, entity2]: entity['attributes']['qualifiedName'] = entity['attributes']['name'] entities_collection = MagicMock() entities_collection.entities = [self.to_class(entity1), self.to_class(entity2)] # Invalidate the cache to test the cache functionality popular_query_params = {'typeName': 'table_metadata', 'sortBy': 'popularityScore', 'sortOrder': 'DESCENDING', 'excludeDeletedEntities': True, 'limit': 2, 'attributes': ['table']} self.proxy._CACHE.region_invalidate(self.proxy._get_metadata_entities, None, '_get_metadata_entities', popular_query_params) self.proxy._driver.entity_bulk = MagicMock(return_value=[entities_collection]) response = self.proxy.get_popular_tables(num_entries=2) # Call multiple times for cache test. self.proxy.get_popular_tables(num_entries=2) self.proxy.get_popular_tables(num_entries=2) self.proxy.get_popular_tables(num_entries=2) self.proxy.get_popular_tables(num_entries=2) self.assertEqual(1, self.proxy._driver.entity_bulk.call_count) ent1_attrs = cast(dict, self.entity1['attributes']) ent2_attrs = cast(dict, self.entity2['attributes']) expected = [ PopularTable(database=self.entity_type, cluster='default', schema='default', name=ent1_attrs['name'], description=ent1_attrs['description']), PopularTable(database=self.entity_type, cluster='default', schema='default', name=ent2_attrs['name'], description=ent1_attrs['description']), ] self.assertEqual(expected.__repr__(), response.__repr__())
def test_get_popular_tables(self) -> None: meta1: Dict = copy.deepcopy(self.metadata1) meta2: Dict = copy.deepcopy(self.metadata2) meta1['attributes']['table'] = self.entity1 meta2['attributes']['table'] = self.entity2 metadata1 = self.to_class(meta1) metadata2 = self.to_class(meta2) metadata_collection = MagicMock() metadata_collection.entities = [metadata1, metadata2] result = MagicMock(return_value=metadata_collection) with patch.object(self.proxy._driver.search_basic, 'create', result): entities_collection = MagicMock() entities_collection.entities = [ self.to_class(self.entity1), self.to_class(self.entity2) ] self.proxy._driver.entity_bulk = MagicMock( return_value=[entities_collection]) response = self.proxy.get_popular_tables(num_entries=2) # Call multiple times for cache test. self.proxy.get_popular_tables(num_entries=2) self.proxy.get_popular_tables(num_entries=2) self.proxy.get_popular_tables(num_entries=2) self.proxy.get_popular_tables(num_entries=2) self.assertEqual(self.proxy._driver.entity_bulk.call_count, 1) ent1_attrs = cast(dict, self.entity1['attributes']) ent2_attrs = cast(dict, self.entity2['attributes']) expected = [ PopularTable(database=self.entity_type, cluster=self.cluster, schema=self.db, name=ent1_attrs['name'], description=ent1_attrs['description']), PopularTable(database=self.entity_type, cluster=self.cluster, schema=self.db, name=ent2_attrs['name'], description=ent1_attrs['description']), ] self.assertEqual(expected.__repr__(), response.__repr__())
def test_get_table_by_user_relation_own(self) -> None: unique_attr_response = MagicMock() unique_attr_response.entity = Data.user_entity_2 self.proxy._driver.entity_unique_attribute = MagicMock( return_value=unique_attr_response) entity_bulk_result = MagicMock() entity_bulk_result.entities = [DottedDict(self.entity1)] self.proxy._driver.entity_bulk = MagicMock( return_value=[entity_bulk_result]) res = self.proxy.get_table_by_user_relation( user_email='test_user_id', relation_type=UserResourceRel.own) self.assertEqual(len(res.get("table")), 1) # type: ignore ent1_attrs = cast(dict, self.entity1['attributes']) expected = [ PopularTable(database=self.entity_type, cluster=self.cluster, schema=self.db, name=ent1_attrs['name'], description=ent1_attrs['description']) ] self.assertEqual({'table': expected}, res)
def test_get_resources_owned_by_user_success(self) -> None: unique_attr_response = MagicMock() unique_attr_response.entity = Data.user_entity_2 self.proxy._driver.entity_unique_attribute = MagicMock( return_value=unique_attr_response) entity_bulk_result = MagicMock() entity_bulk_result.entities = [DottedDict(self.entity1)] self.proxy._driver.entity_bulk = MagicMock( return_value=[entity_bulk_result]) res = self.proxy._get_resources_owned_by_user( user_id='test_user_2', resource_type=ResourceType.Table.name) self.assertEqual(len(res), 1) ent1_attrs = cast(dict, self.entity1['attributes']) expected = [ PopularTable(database=self.entity_type, cluster=self.cluster, schema=self.db, name=ent1_attrs['name'], description=ent1_attrs['description']) ] self.assertEqual(expected, res)
def get_frequently_used_tables(self, *, user_email: str) -> Dict[str, Any]: """ Retrieves all Table the resources per user on READ relation. :param user_email: the email of the user :return: """ query = textwrap.dedent(""" MATCH (user:User {key: $query_key})-[r:READ]->(tbl:Table) WHERE EXISTS(r.published_tag) AND r.published_tag IS NOT NULL WITH user, r, tbl ORDER BY r.published_tag DESC, r.read_count DESC LIMIT 50 MATCH (tbl:Table)<-[:TABLE]-(schema:Schema)<-[:SCHEMA]-(clstr:Cluster)<-[:CLUSTER]-(db:Database) OPTIONAL MATCH (tbl)-[:DESCRIPTION]->(tbl_dscrpt:Description) RETURN db, clstr, schema, tbl, tbl_dscrpt """) table_records = self._execute_cypher_query(statement=query, param_dict={'query_key': user_email}) if not table_records: raise NotFoundException('User {user_id} does not READ any resources'.format(user_id=user_email)) results = [] for record in table_records: results.append(PopularTable( database=record['db']['name'], cluster=record['clstr']['name'], schema=record['schema']['name'], name=record['tbl']['name'], description=self._safe_get(record, 'tbl_dscrpt', 'description'))) return {'table': results}
def get_table_by_user_relation(self, *, user_email: str, relation_type: UserResourceRel) \ -> Dict[str, List[PopularTable]]: """ Retrive all follow the Table per user based on the relation. :param user_email: the email of the user :param relation_type: the relation between the user and the resource :return: """ rel_clause: str = self._get_user_resource_relationship_clause(relation_type=relation_type, id='', resource_type=ResourceType.Table, user_key=user_email) query = textwrap.dedent(f""" MATCH {rel_clause}<-[:TABLE]-(schema:Schema)<-[:SCHEMA]-(clstr:Cluster)<-[:CLUSTER]-(db:Database) WITH db, clstr, schema, resource OPTIONAL MATCH (resource)-[:DESCRIPTION]->(tbl_dscrpt:Description) RETURN db, clstr, schema, resource, tbl_dscrpt""") table_records = self._execute_cypher_query(statement=query, param_dict={'user_key': user_email}) if not table_records: raise NotFoundException('User {user_id} does not {relation} any resources'.format(user_id=user_email, relation=relation_type)) results = [] for record in table_records: results.append(PopularTable( database=record['db']['name'], cluster=record['clstr']['name'], schema=record['schema']['name'], name=record['resource']['name'], description=self._safe_get(record, 'tbl_dscrpt', 'description'))) return {ResourceType.Table.name.lower(): results}
def get_popular_tables(self, *, num_entries: int) -> List[PopularTable]: """ Retrieve popular tables. As popular table computation requires full scan of table and user relationship, it will utilize cached method _get_popular_tables_uris. :param num_entries: :return: Iterable of PopularTable """ table_uris = self._get_popular_tables_uris(num_entries) if not table_uris: return [] query = textwrap.dedent(""" MATCH (db:Database)-[:CLUSTER]->(clstr:Cluster)-[:SCHEMA]->(schema:Schema)-[:TABLE]->(tbl:Table) WHERE tbl.key IN $table_uris WITH db.name as database_name, clstr.name as cluster_name, schema.name as schema_name, tbl OPTIONAL MATCH (tbl)-[:DESCRIPTION]->(dscrpt:Description) RETURN database_name, cluster_name, schema_name, tbl.name as table_name, dscrpt.description as table_description; """) records = self._execute_cypher_query(statement=query, param_dict={'table_uris': table_uris}) popular_tables = [] for record in records: popular_table = PopularTable(database=record['database_name'], cluster=record['cluster_name'], schema=record['schema_name'], name=record['table_name'], description=self._safe_get(record, 'table_description')) popular_tables.append(popular_table) return popular_tables
def get_popular_tables(self, *, num_entries: int) -> List[PopularTable]: """ :param num_entries: Number of popular tables to fetch :return: A List of popular tables instances """ popular_tables = list() popular_query_params = { 'typeName': 'Table', 'sortBy': 'popularityScore', 'sortOrder': 'DESCENDING', 'excludeDeletedEntities': True, 'limit': num_entries } search_results = self._driver.search_basic.create( data=popular_query_params) for table in search_results.entities: table_attrs = table.attributes table_qn = parse_table_qualified_name( qualified_name=table_attrs.get(self.QN_KEY)) table_name = table_qn.get("table_name") or table_attrs.get('name') db_name = table_qn.get("db_name", '') db_cluster = table_qn.get("cluster_name", '') popular_table = PopularTable( database=table.typeName, cluster=db_cluster, schema=db_name, name=table_name, description=table_attrs.get('description') or table_attrs.get('comment')) popular_tables.append(popular_table) return popular_tables
def _serialize_popular_tables(self, entities: list) -> List[PopularTable]: """ Gets a list of entities and serialize the popular tables. :param entities: List of entities from atlas client :return: a list of PopularTable objects """ popular_tables = list() for table in entities: table_attrs = table.attributes table_qn = parse_table_qualified_name( qualified_name=table_attrs.get(self.QN_KEY) ) table_name = table_qn.get("table_name") or table_attrs.get('name') db_name = table_qn.get("db_name", '') db_cluster = table_qn.get("cluster_name", '') popular_table = PopularTable( database=table.typeName, cluster=db_cluster, schema=db_name, name=table_name, description=table_attrs.get('description') or table_attrs.get('comment')) popular_tables.append(popular_table) return popular_tables
def test_get_table_by_user_relation(self) -> None: bookmark1 = copy.deepcopy(self.bookmark_entity1) bookmark1 = self.to_class(bookmark1) bookmark_collection = MagicMock() bookmark_collection.entities = [bookmark1] self.proxy._driver.search_basic.create = MagicMock(return_value=bookmark_collection) res = self.proxy.get_table_by_user_relation(user_email='test_user_id', relation_type=UserResourceRel.follow) expected = [PopularTable(database=Data.entity_type, cluster=Data.cluster, schema=Data.db, name=Data.name, description=None)] self.assertEqual(res, {'table': expected})
def test_get_frequently_used_tables(self) -> None: entity_unique_attribute_result = MagicMock() entity_unique_attribute_result.entity = DottedDict(self.user_entity_2) self.proxy._driver.entity_unique_attribute = MagicMock(return_value=entity_unique_attribute_result) entity_bulk_result = MagicMock() entity_bulk_result.entities = [DottedDict(self.reader_entity_1)] self.proxy._driver.entity_bulk = MagicMock(return_value=[entity_bulk_result]) expected = {'table': [PopularTable(cluster=self.cluster, name='Table1', schema=self.db, database=self.entity_type)]} res = self.proxy.get_frequently_used_tables(user_email='dummy') self.assertEqual(expected, res)
def test_get_table_by_user_relation(self, mock_rds_client: Any) -> None: table = RDSTable(name='test_table') table_description = RDSTableDescription(description='test_description') schema = RDSSchema(name='test_schema') cluster = RDSCluster(name='test_cluster') database = RDSDatabase(name='test_database') cluster.database = database schema.cluster = cluster table.schema = schema table.description = table_description mock_client = MagicMock() mock_rds_client.return_value = mock_client mock_create_session = MagicMock() mock_client.create_session.return_value = mock_create_session mock_session = MagicMock() mock_create_session.__enter__.return_value = mock_session mock_session_query = MagicMock() mock_session.query.return_value = mock_session_query mock_session_query_filter = MagicMock() mock_session_query.filter.return_value = mock_session_query_filter mock_session_query_filter_options = MagicMock() mock_session_query_filter.options.return_value = mock_session_query_filter_options mock_session_query_filter_options.all.return_value = [table] expected = { 'table': [ PopularTable(database='test_database', cluster='test_cluster', schema='test_schema', name='test_table', description='test_description') ] } proxy = MySQLProxy() actual = proxy.get_table_by_user_relation( user_email='test_user', relation_type=UserResourceRel.follow) self.assertEqual(len(actual['table']), 1) self.assertEqual(expected, actual)
def _get_resources_followed_by_user(self, user_id: str, resource_type: str) \ -> List[Union[PopularTable, DashboardSummary]]: """ ToDo (Verdan): Dashboard still needs to be implemented. Helper function to get the resource, table, dashboard etc followed by a user. :param user_id: User ID of a user :param resource_type: Type of a resource that returns, could be table, dashboard etc. :return: A list of PopularTable, DashboardSummary or any other resource. """ params = { 'typeName': self.BOOKMARK_TYPE, 'offset': '0', 'limit': '1000', 'excludeDeletedEntities': True, 'entityFilters': { 'condition': 'AND', 'criterion': [ { 'attributeName': self.QN_KEY, 'operator': 'contains', 'attributeValue': f'.{user_id}.bookmark' }, { 'attributeName': self.BOOKMARK_ACTIVE_KEY, 'operator': 'eq', 'attributeValue': 'true' } ] }, 'attributes': ['count', self.QN_KEY, self.ENTITY_URI_KEY] } # Fetches the bookmark entities based on filters search_results = self.client.discovery.faceted_search(search_parameters=params) resources = [] for record in search_results.entities: table_info = self._extract_info_from_uri(table_uri=record.attributes[self.ENTITY_URI_KEY]) res = self._parse_bookmark_qn(record.attributes[self.QN_KEY]) resources.append(PopularTable( database=table_info['entity'], cluster=res['cluster'], schema=res['db'], name=res['table'])) return resources
def get_frequently_used_tables( self, *, user_email: str) -> Dict[str, List[PopularTable]]: user = self._driver.entity_unique_attribute( self.USER_TYPE, qualifiedName=user_email).entity readers_guids = [] for user_reads in user['relationshipAttributes'].get('entityReads'): entity_status = user_reads['entityStatus'] relationship_status = user_reads['relationshipStatus'] if entity_status == 'ACTIVE' and relationship_status == 'ACTIVE': readers_guids.append(user_reads['guid']) readers = extract_entities( self._driver.entity_bulk(guid=readers_guids, ignoreRelationships=True)) _results = {} for reader in readers: entity_uri = reader.attributes.get(self.ENTITY_URI_KEY) count = reader.attributes.get('count') if count: details = self._extract_info_from_uri(table_uri=entity_uri) _results[count] = dict(cluster=details.get('cluster'), name=details.get('name'), schema=details.get('db'), database=details.get('entity')) sorted_counts = sorted(_results.keys()) results = [] for count in sorted_counts: data: dict = _results.get(count, dict()) table = PopularTable(**data) results.append(table) return {'table': results}
def test_get_popular_tables(self) -> None: application = Fixtures.next_application() self.get_proxy().put_app(data=application) # Add 10 tables tables: List[Table] = [ Fixtures.next_table(application=application) for _ in range(10) ] self.get_proxy().post_tables(tables=tables) user = Fixtures.next_user() self.get_proxy().put_user(data=user) # add reads to 6 of them, expecting that only the top five will be "popular" expected_popular_tables = [] reads = 0 for i in range(6): table_name: str = checkNotNone(tables[i].name) table_uri: str = checkNotNone(tables[i].key) self.get_proxy().add_read_count(table_uri=table_uri, user_id=f'{user.user_id}', read_count=reads) if reads > 0: expected_popular_tables.append(table_name) reads += 1000 # ensure popular tables returns those 5 we added actual_popular_tables = self.get_proxy().get_popular_tables( num_entries=5) self.assertEqual(len(actual_popular_tables), 5) popular_tables = [] for table in tables: if table.name in expected_popular_tables: popular_tables.append( PopularTable(database=table.database, cluster=table.cluster, schema=table.schema, name=table.name, description=table.description)) self.assertEqual(sorted(actual_popular_tables), sorted(popular_tables))
def get_frequently_used_tables( self, *, user_email: str) -> Dict[str, List[PopularTable]]: user = self.client.entity.get_entity_by_attribute( type_name=self.USER_TYPE, uniq_attributes=[(self.QN_KEY, user_email)]).entity readers_guids = [] for user_reads in user['relationshipAttributes'].get('entityReads'): entity_status = user_reads['entityStatus'] relationship_status = user_reads['relationshipStatus'] if entity_status == Status.ACTIVE and relationship_status == Status.ACTIVE: readers_guids.append(user_reads['guid']) readers = self.client.entity.get_entities_by_guids( guids=list(readers_guids), ignore_relationships=True) _results = {} for reader in readers.entities or list(): entity_uri = reader.attributes.get(self.ENTITY_URI_KEY) count = reader.attributes.get('count') if count: details = self._extract_info_from_uri(table_uri=entity_uri) _results[count] = dict(cluster=details.get('cluster'), name=details.get('name'), schema=details.get('db'), database=details.get('entity')) sorted_counts = sorted(_results.keys()) results = [] for count in sorted_counts: data: dict = _results.get(count, dict()) table = PopularTable(**data) results.append(table) return {'table': results}
def get_table_by_user_relation( self, *, user_email: str, relation_type: UserResourceRel) -> Dict[str, Any]: params = { 'typeName': self.BOOKMARK_TYPE, 'offset': '0', 'limit': '1000', 'excludeDeletedEntities': True, 'entityFilters': { 'condition': 'AND', 'criterion': [{ 'attributeName': self.QN_KEY, 'operator': 'contains', 'attributeValue': f'.{user_email}.bookmark' }, { 'attributeName': self.BOOKMARK_ACTIVE_KEY, 'operator': 'eq', 'attributeValue': 'true' }] }, 'attributes': ['count', self.QN_KEY, self.ENTITY_URI_KEY] } # Fetches the bookmark entities based on filters search_results = self._driver.search_basic.create(data=params) results = [] for record in search_results.entities: table_info = self._extract_info_from_uri( table_uri=record.attributes[self.ENTITY_URI_KEY]) res = self._parse_bookmark_qn(record.attributes[self.QN_KEY]) results.append( PopularTable(database=table_info['entity'], cluster=res['cluster'], schema=res['db'], name=res['table'])) return {'table': results}
def test_get(self) -> None: self.mock_client.get_table_by_user_relation.return_value = { 'table': [ PopularTable(database='d1', cluster='c1', schema='s1bbc', name='n1_test_a_table'), PopularTable(database='d1', cluster='c2', schema='s1abc', name='n1_test_b_table'), PopularTable(database='d1', cluster='c3', schema='s1abc', name='n1_test_a_table'), ] } self.mock_client.get_dashboard_by_user_relation.return_value = { 'dashboard': [ DashboardSummary(uri='foobar_dashboard_3', cluster='cluster', group_name='dashboard_group_b', group_url='http://foo.bar/group', product='foobar', name='dashboard_b', url='http://foo.bar/dashboard_b'), DashboardSummary(uri='foobar_dashboard_2', cluster='cluster', group_name='dashboard_group_a', group_url='http://foo.bar/group', product='foobar', name='dashboard_a', url='http://foo.bar/dashboard_a'), DashboardSummary(uri='foobar_dashboard_1', cluster='cluster', group_name='dashboard_group_a', group_url='http://foo.bar/group', product='foobar', name='dashboard_c', url='http://foo.bar/dashboard_c'), ] } response = self.api.get(user_id='username') self.assertEqual(list(response)[1], HTTPStatus.OK) self.mock_client.get_table_by_user_relation.assert_called_once() # test results are sorted assert list(response)[0].get('table') == [ { 'cluster': 'c3', 'database': 'd1', 'description': None, 'name': 'n1_test_a_table', 'schema': 's1abc' }, { 'cluster': 'c2', 'database': 'd1', 'description': None, 'name': 'n1_test_b_table', 'schema': 's1abc' }, { 'cluster': 'c1', 'database': 'd1', 'description': None, 'name': 'n1_test_a_table', 'schema': 's1bbc' }, ] assert list(response)[0].get('dashboard') == [ { 'chart_names': [], 'cluster': 'cluster', 'description': None, 'group_name': 'dashboard_group_a', 'group_url': 'http://foo.bar/group', 'last_successful_run_timestamp': None, 'name': 'dashboard_a', 'product': 'foobar', 'uri': 'foobar_dashboard_2', 'url': 'http://foo.bar/dashboard_a' }, { 'chart_names': [], 'cluster': 'cluster', 'description': None, 'group_name': 'dashboard_group_a', 'group_url': 'http://foo.bar/group', 'last_successful_run_timestamp': None, 'name': 'dashboard_c', 'product': 'foobar', 'uri': 'foobar_dashboard_1', 'url': 'http://foo.bar/dashboard_c' }, { 'chart_names': [], 'cluster': 'cluster', 'description': None, 'group_name': 'dashboard_group_b', 'group_url': 'http://foo.bar/group', 'last_successful_run_timestamp': None, 'name': 'dashboard_b', 'product': 'foobar', 'uri': 'foobar_dashboard_3', 'url': 'http://foo.bar/dashboard_b' }, ]
def test_get_dashboard(self) -> None: self.proxy.client.entity.get_entity_by_attribute = MagicMock( return_value=self.dashboard_data) # type: ignore self.proxy._get_dashboard_group = MagicMock( return_value=self.dashboard_group_data) # type: ignore self.proxy.client.entity.get_entities_by_guids = MagicMock( return_value=DottedDict({'entities': [DottedDict(self.entity1)]})) expected = DashboardDetail( uri='superset_dashboard://datalab.prod/1', cluster='datalab', group_name='prod superset', group_url='https://superset.prod', product='superset', name='Prod Usage', url='https://prod.superset/dashboards/1', description='Robs famous dashboard', created_timestamp=1619517099, updated_timestamp=1619626531, last_successful_run_timestamp=1619517099, last_run_timestamp=1619517150, last_run_state='failed', owners=[ User(user_id='lisa_salinas', email='lisa_salinas', first_name=None, last_name=None, full_name=None, display_name=None, is_active=True, github_username=None, team_name=None, slack_id=None, employee_type=None, manager_fullname=None, manager_email=None, manager_id=None, role_name=None, profile_url=None, other_key_values={}) ], frequent_users=[], chart_names=['Count Users by Time', 'Total Count'], query_names=['User Count By Time', 'Total Count'], queries=[ DashboardQuery( name='User Count By Time', url='https://prod.superset/dashboards/1/query/1', query_text='SELECT date, COUNT(1) FROM db.table GROUP BY 1' ), DashboardQuery( name='Total Count', url='https://prod.superset/dashboards/1/query/2', query_text='SELECT COUNT(1) FROM db.table') ], tables=[ PopularTable(database='hive_table', cluster='TEST_CLUSTER', schema='TEST_DB', name='Table1', description='Dummy Description') ], tags=[], badges=[], recent_view_count=0) result = self.proxy.get_dashboard( id='superset_dashboard://datalab.prod/1') self.assertEqual(expected, result)
def test_get_dashboard(self) -> None: with patch.object(GraphDatabase, 'driver'), patch.object( Neo4jProxy, '_execute_cypher_query') as mock_execute: mock_execute.return_value.single.side_effect = [{ 'cluster_name': 'cluster_name', 'uri': 'foo_dashboard://gold.bar/dashboard_id', 'url': 'http://www.foo.bar/dashboard_id', 'product': 'foobar', 'name': 'dashboard name', 'created_timestamp': 123456789, 'description': 'description', 'group_name': 'group_name', 'group_url': 'http://www.group_url.com', 'last_successful_run_timestamp': 9876543210, 'last_run_timestamp': 987654321, 'last_run_state': 'good_state', 'updated_timestamp': 123456654321, 'recent_view_count': 100, 'owners': [{ 'employee_type': 'teamMember', 'full_name': 'test_full_name', 'is_active': 'True', 'github_username': '******', 'slack_id': 'test_id', 'last_name': 'test_last_name', 'first_name': 'test_first_name', 'team_name': 'test_team', 'email': 'test_email', }, { 'employee_type': 'teamMember', 'full_name': 'test_full_name2', 'is_active': 'True', 'github_username': '******', 'slack_id': 'test_id2', 'last_name': 'test_last_name2', 'first_name': 'test_first_name2', 'team_name': 'test_team2', 'email': 'test_email2', }], 'tags': [{ 'key': 'tag_key1', 'tag_type': 'tag_type1' }, { 'key': 'tag_key2', 'tag_type': 'tag_type2' }], 'charts': [{ 'name': 'chart1' }, { 'name': 'chart2' }], 'queries': [{ 'name': 'query1' }, { 'name': 'query2' }], 'tables': [{ 'database': 'db1', 'name': 'table1', 'description': 'table description 1', 'cluster': 'cluster1', 'schema': 'schema1' }, { 'database': 'db2', 'name': 'table2', 'description': None, 'cluster': 'cluster2', 'schema': 'schema2' }] }, { 'cluster_name': 'cluster_name', 'uri': 'foo_dashboard://gold.bar/dashboard_id', 'url': 'http://www.foo.bar/dashboard_id', 'product': 'foobar', 'name': 'dashboard name', 'created_timestamp': 123456789, 'description': None, 'group_name': 'group_name', 'group_url': 'http://www.group_url.com', 'last_run_timestamp': None, 'last_run_state': None, 'updated_timestamp': None, 'recent_view_count': 0, 'owners': [], 'tags': [], 'charts': [], 'queries': [], 'tables': [] }] neo4j_proxy = Neo4jProxy(host='DOES_NOT_MATTER', port=0000) dashboard = neo4j_proxy.get_dashboard(id='dashboard_id') expected = DashboardDetail( uri='foo_dashboard://gold.bar/dashboard_id', cluster='cluster_name', group_name='group_name', group_url='http://www.group_url.com', product='foobar', name='dashboard name', url='http://www.foo.bar/dashboard_id', description='description', created_timestamp=123456789, last_successful_run_timestamp=9876543210, updated_timestamp=123456654321, last_run_timestamp=987654321, last_run_state='good_state', owners=[ User(email='test_email', first_name='test_first_name', last_name='test_last_name', full_name='test_full_name', is_active='True', github_username='******', team_name='test_team', slack_id='test_id', employee_type='teamMember', manager_fullname=''), User(email='test_email2', first_name='test_first_name2', last_name='test_last_name2', full_name='test_full_name2', is_active='True', github_username='******', team_name='test_team2', slack_id='test_id2', employee_type='teamMember', manager_fullname='') ], frequent_users=[], chart_names=['chart1', 'chart2'], query_names=['query1', 'query2'], tables=[ PopularTable(database='db1', name='table1', description='table description 1', cluster='cluster1', schema='schema1'), PopularTable(database='db2', name='table2', cluster='cluster2', schema='schema2'), ], tags=[ Tag(tag_type='tag_type1', tag_name='tag_key1'), Tag(tag_type='tag_type2', tag_name='tag_key2') ], recent_view_count=100) self.assertEqual(expected, dashboard) dashboard2 = neo4j_proxy.get_dashboard(id='dashboard_id') expected2 = DashboardDetail( uri='foo_dashboard://gold.bar/dashboard_id', cluster='cluster_name', group_name='group_name', group_url='http://www.group_url.com', product='foobar', name='dashboard name', url='http://www.foo.bar/dashboard_id', description=None, created_timestamp=123456789, updated_timestamp=None, last_run_timestamp=None, last_run_state=None, owners=[], frequent_users=[], chart_names=[], query_names=[], tables=[], tags=[], last_successful_run_timestamp=None, recent_view_count=0) self.assertEqual(expected2, dashboard2)
def test_get_dashboard(self, mock_rds_client: Any) -> None: # dashboard_metadata dashboard = RDSDashboard( rk='foo_dashboard://gold.bar/dashboard_id', name='dashboard name', dashboard_url='http://www.foo.bar/dashboard_id', created_timestamp=123456789) dashboard_group = RDSDashboardGroup( name='group_name', dashboard_group_url='http://www.group_url.com') dashboard_group.cluster = RDSCluster(name='cluster_name') dashboard.group = dashboard_group dashboard.description = RDSDashboardDescription( description='description') dashboard.execution = [ RDSDashboardExecution(rk='dashboard_last_successful_execution', timestamp=9876543210), RDSDashboardExecution(rk='dashboard_last_execution', timestamp=987654321, state='good_state') ] dashboard.timestamp = RDSDashboardTimestamp(timestamp=123456654321) dashboard.tags = [ RDSTag(rk='tag_key1', tag_type='default'), RDSTag(rk='tag_key2', tag_type='default') ] dashboard.badges = [RDSBadge(rk='golden', category='table_status')] dashboard.owners = [ RDSUser(email='test_email', first_name='test_first_name', last_name='test_last_name', full_name='test_full_name', is_active=True, github_username='******', team_name='test_team', slack_id='test_id', employee_type='teamMember'), RDSUser(email='test_email2', first_name='test_first_name2', last_name='test_last_name2', full_name='test_full_name2', is_active=True, github_username='******', team_name='test_team2', slack_id='test_id2', employee_type='teamMember') ] dashboard.usage = [RDSDashboardUsage(read_count=100)] mock_client = MagicMock() mock_rds_client.return_value = mock_client mock_create_session = MagicMock() mock_client.create_session.return_value = mock_create_session mock_session = MagicMock() mock_create_session.__enter__.return_value = mock_session mock_session_query = MagicMock() mock_session.query.return_value = mock_session_query mock_session_query_filter = MagicMock() mock_session_query.filter.return_value = mock_session_query_filter mock_session_query_filter.first.return_value = dashboard # queries query1 = RDSDashboardQuery(name='query1') query2 = RDSDashboardQuery(name='query2', url='http://foo.bar/query', query_text='SELECT * FROM foo.bar') query1.charts = [RDSDashboardChart(name='chart1')] query2.charts = [RDSDashboardChart(name='chart2')] queries = [query1, query2] # tables database1 = RDSDatabase(name='db1') database2 = RDSDatabase(name='db2') cluster1 = RDSCluster(name='cluster1') cluster2 = RDSCluster(name='cluster2') schema1 = RDSSchema(name='schema1') schema2 = RDSSchema(name='schema2') table1 = RDSTable(name='table1') table2 = RDSTable(name='table2') description1 = RDSTableDescription(description='table description 1') schema1.cluster = cluster1 cluster1.database = database1 schema2.cluster = cluster2 cluster2.database = database2 table1.schema = schema1 table2.schema = schema2 table1.description = description1 tables = [table1, table2] mock_session_query_filter_options = MagicMock() mock_session_query_filter.options.return_value = mock_session_query_filter_options mock_session_query_filter_options.all.side_effect = [queries, tables] expected = DashboardDetail( uri='foo_dashboard://gold.bar/dashboard_id', cluster='cluster_name', group_name='group_name', group_url='http://www.group_url.com', product='foo', name='dashboard name', url='http://www.foo.bar/dashboard_id', description='description', created_timestamp=123456789, last_successful_run_timestamp=9876543210, updated_timestamp=123456654321, last_run_timestamp=987654321, last_run_state='good_state', owners=[ User(email='test_email', first_name='test_first_name', last_name='test_last_name', full_name='test_full_name', is_active=True, github_username='******', team_name='test_team', slack_id='test_id', employee_type='teamMember', manager_fullname=''), User(email='test_email2', first_name='test_first_name2', last_name='test_last_name2', full_name='test_full_name2', is_active=True, github_username='******', team_name='test_team2', slack_id='test_id2', employee_type='teamMember', manager_fullname='') ], frequent_users=[], chart_names=['chart1', 'chart2'], query_names=['query1', 'query2'], queries=[ DashboardQuery(name='query1'), DashboardQuery(name='query2', url='http://foo.bar/query', query_text='SELECT * FROM foo.bar') ], tables=[ PopularTable(database='db1', name='table1', description='table description 1', cluster='cluster1', schema='schema1'), PopularTable(database='db2', name='table2', cluster='cluster2', schema='schema2'), ], tags=[ Tag(tag_type='default', tag_name='tag_key1'), Tag(tag_type='default', tag_name='tag_key2') ], badges=[Badge(badge_name='golden', category='table_status')], recent_view_count=100) proxy = MySQLProxy() actual = proxy.get_dashboard(id='dashboard_id') self.assertEqual(expected, actual)