def test_get_table_view_only(self) -> None: col_usage_return_value = copy.deepcopy(self.col_usage_return_value) for col in col_usage_return_value: col['tbl']['is_view'] = True with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jProxy, '_execute_cypher_query') as mock_execute: mock_execute.side_effect = [col_usage_return_value, [], self.table_level_return_value] neo4j_proxy = Neo4jProxy(host='DOES_NOT_MATTER', port=0000) table = neo4j_proxy.get_table(table_uri='dummy_uri') expected = Table(database='hive', cluster='gold', schema='foo_schema', name='foo_table', tags=[Tag(tag_name='test', tag_type='default')], badges=[Tag(tag_name='golden', tag_type='badge')], table_readers=[], description='foo description', watermarks=[Watermark(watermark_type='high_watermark', partition_key='ds', partition_value='fake_value', create_time='fake_time'), Watermark(watermark_type='low_watermark', partition_key='ds', partition_value='fake_value', create_time='fake_time')], columns=[Column(name='bar_id_1', description='bar col description', col_type='varchar', sort_order=0, stats=[Statistics(start_epoch=1, end_epoch=1, stat_type='avg', stat_val='1')]), Column(name='bar_id_2', description='bar col2 description', col_type='bigint', sort_order=1, stats=[Statistics(start_epoch=2, end_epoch=2, stat_type='avg', stat_val='2')])], owners=[User(email='*****@*****.**')], table_writer=Application(application_url=self.table_writer['application_url'], description=self.table_writer['description'], name=self.table_writer['name'], id=self.table_writer['id']), last_updated_timestamp=1, source=Source(source='/source_file_loc', source_type='github'), is_view=True, programmatic_descriptions=[ ProgrammaticDescription(source='quality_report', text='Test Test'), ProgrammaticDescription(source='s3_crawler', text='Test Test Test') ]) self.assertEqual(str(expected), str(table))
def test_get_table(self) -> None: self._mock_get_table_entity() response = self.proxy.get_table(table_uri=self.table_uri) classif_name = self.classification_entity['classifications'][0]['typeName'] ent_attrs = cast(dict, self.entity1['attributes']) col_attrs = cast(dict, self.test_column['attributes']) col_metadata_attrs = cast(dict, self.column_metadata_entity['attributes']) exp_col_stats = list() for stats in col_metadata_attrs['statistics']: exp_col_stats.append( Statistics( stat_type=stats['attributes']['stat_name'], stat_val=stats['attributes']['stat_val'], start_epoch=stats['attributes']['start_epoch'], end_epoch=stats['attributes']['end_epoch'], ) ) exp_col = Column(name=col_attrs['name'], description='column description', col_type='Managed', sort_order=col_attrs['position'], stats=exp_col_stats) expected = Table(database=self.entity_type, cluster=self.cluster, schema=self.db, name=ent_attrs['name'], tags=[Tag(tag_name=classif_name, tag_type="default")], description=ent_attrs['description'], owners=[User(email=ent_attrs['owner'])], columns=[exp_col], last_updated_timestamp=cast(int, self.entity1['updateTime'])) self.assertEqual(str(expected), str(response))
def get_dashboard( self, id: str, ) -> DashboardDetailEntity: get_dashboard_detail_query = textwrap.dedent(u""" MATCH (d:Dashboard {key: $query_key})-[:DASHBOARD_OF]->(dg:Dashboardgroup)-[:DASHBOARD_GROUP_OF]->(c:Cluster) OPTIONAL MATCH (d)-[:DESCRIPTION]->(description:Description) OPTIONAL MATCH (d)-[:EXECUTED]->(last_exec:Execution) WHERE split(last_exec.key, '/')[5] = '_last_execution' OPTIONAL MATCH (d)-[:LAST_UPDATED_AT]->(t:Timestamp) OPTIONAL MATCH (d)-[:OWNER]->(owner:User) OPTIONAL MATCH (d)-[:TAG]->(tag:Tag) RETURN c.name as cluster_name, d.key as uri, d.dashboard_url as url, d.name as name, toInteger(d.created_timestamp) as created_timestamp, description.description as description, dg.name as group_name, dg.dashboard_group_url as group_url, toInteger(last_exec.timestamp) as last_run_timestamp, last_exec.state as last_run_state, toInteger(t.timestamp) as updated_timestamp, collect(owner) as owners, collect(tag) as tags; """) record = self._execute_cypher_query( statement=get_dashboard_detail_query, param_dict={ 'query_key': id }).single() if not record: raise NotFoundException( 'No dashboard exist with URI: {}'.format(id)) owners = [ self._build_user_from_record(record=owner) for owner in record['owners'] ] tags = [ Tag(tag_type=tag['tag_type'], tag_name=tag['key']) for tag in record['tags'] ] return DashboardDetailEntity( uri=record['uri'], cluster=record['cluster_name'], url=record['url'], name=record['name'], created_timestamp=record['created_timestamp'], description=self._safe_get(record, 'description'), group_name=self._safe_get(record, 'group_name'), group_url=self._safe_get(record, 'group_url'), last_run_timestamp=self._safe_get(record, 'last_run_timestamp'), last_run_state=self._safe_get(record, 'last_run_state'), updated_timestamp=self._safe_get(record, 'updated_timestamp'), owners=owners, tags=tags)
def _get_table(self, custom_stats_format: bool = False) -> None: if custom_stats_format: test_exp_col = self.test_exp_col_stats_formatted else: test_exp_col = self.test_exp_col_stats_raw ent_attrs = cast(dict, self.entity1['attributes']) self._mock_get_table_entity() self._create_mocked_report_entities_collection() self.proxy._get_owners = MagicMock( return_value=[User(email=ent_attrs['owner'])]) # type: ignore self.proxy._driver.entity_bulk = MagicMock( return_value=self.report_entity_collection) response = self.proxy.get_table(table_uri=self.table_uri) classif_name = self.classification_entity['classifications'][0][ 'typeName'] col_attrs = cast(dict, self.test_column['attributes']) exp_col_stats = list() for stats in test_exp_col: exp_col_stats.append( Stat( stat_type=stats['attributes']['stat_name'], stat_val=stats['attributes']['stat_val'], start_epoch=stats['attributes']['start_epoch'], end_epoch=stats['attributes']['end_epoch'], )) exp_col = Column(name=col_attrs['name'], description='column description', col_type='Managed', sort_order=col_attrs['position'], stats=exp_col_stats) expected = Table( database=self.entity_type, cluster=self.cluster, schema=self.db, name=ent_attrs['name'], tags=[Tag(tag_name=classif_name, tag_type="default")], description=ent_attrs['description'], owners=[User(email=ent_attrs['owner'])], resource_reports=[ ResourceReport(name='test_report', url='http://test'), ResourceReport(name='test_report3', url='http://test3') ], last_updated_timestamp=int(str(self.entity1['updateTime'])[:10]), columns=[exp_col] * self.active_columns, programmatic_descriptions=[ ProgrammaticDescription(source='test parameter key a', text='testParameterValueA'), ProgrammaticDescription(source='test parameter key b', text='testParameterValueB') ], is_view=False) self.assertEqual(str(expected), str(response))
def get_table(self, *, table_uri: str) -> Table: """ Gathers all the information needed for the Table Detail Page. :param table_uri: :return: A Table object with all the information available or gathered from different entities. """ entity = self._get_table_entity(table_uri=table_uri) table_details = entity.entity try: attrs = table_details[self.ATTRS_KEY] programmatic_descriptions = self._get_programmatic_descriptions( attrs.get('parameters')) table_qn = parse_table_qualified_name( qualified_name=attrs.get(self.QN_KEY)) tags = [] # Using or in case, if the key 'classifications' is there with a None for classification in table_details.get( "classifications") or list(): tags.append( Tag(tag_name=classification.get('typeName'), tag_type="default")) columns = self._serialize_columns(entity=entity) reports_guids = [ report.get("guid") for report in attrs.get("reports") or list() ] table = Table( database=table_details.get('typeName'), cluster=table_qn.get('cluster_name', ''), schema=table_qn.get('db_name', ''), name=attrs.get('name') or table_qn.get("table_name", ''), tags=tags, description=attrs.get('description') or attrs.get('comment'), owners=[User(email=attrs.get('owner'))], resource_reports=self._get_reports(guids=reports_guids), columns=columns, table_readers=self._get_readers(attrs.get(self.QN_KEY)), last_updated_timestamp=self._parse_date( table_details.get('updateTime')), programmatic_descriptions=programmatic_descriptions) return table except KeyError as ex: LOGGER.exception( 'Error while accessing table information. {}'.format(str(ex))) raise BadRequest( 'Some of the required attributes ' 'are missing in : ( {table_uri} )'.format(table_uri=table_uri))
def get_table(self, *, table_uri: str) -> Table: """ Gathers all the information needed for the Table Detail Page. :param table_uri: :return: A Table object with all the information available or gathered from different entities. """ entity = self._get_table_entity(table_uri=table_uri) table_details = entity.entity try: attrs = table_details[self.ATTRS_KEY] programmatic_descriptions = self._get_programmatic_descriptions(attrs.get('parameters', dict())) table_qn = parse_table_qualified_name( qualified_name=attrs.get(self.QN_KEY) ) badges = [] # Using or in case, if the key 'classifications' is there with a None for classification in table_details.get('classifications') or list(): badges.append( Badge( badge_name=classification.get('typeName'), category="default" ) ) tags = [] for term in table_details.get(self.REL_ATTRS_KEY).get("meanings") or list(): if term.get('entityStatus') == Status.ACTIVE and \ term.get('relationshipStatus') == Status.ACTIVE: tags.append( Tag( tag_name=term.get("displayText"), tag_type="default" ) ) columns = self._serialize_columns(entity=entity) reports_guids = [report.get("guid") for report in attrs.get("reports") or list()] table_type = attrs.get('tableType') or 'table' is_view = 'view' in table_type.lower() readers = self._get_readers(table_details) table = Table( database=table_details.get('typeName'), cluster=table_qn.get('cluster_name', ''), schema=table_qn.get('db_name', ''), name=attrs.get('name') or table_qn.get("table_name", ''), badges=badges, tags=tags, description=attrs.get('description') or attrs.get('comment'), owners=self._get_owners( table_details[self.REL_ATTRS_KEY].get('ownedBy', []), attrs.get('owner')), resource_reports=self._get_reports(guids=reports_guids), columns=columns, is_view=is_view, table_readers=readers, last_updated_timestamp=self._parse_date(table_details.get('updateTime')), programmatic_descriptions=programmatic_descriptions, watermarks=self._get_table_watermarks(table_details)) return table except KeyError as ex: LOGGER.exception('Error while accessing table information. {}' .format(str(ex))) raise BadRequest('Some of the required attributes ' 'are missing in : ( {table_uri} )' .format(table_uri=table_uri))
def test_get_dashboard(self) -> None: with patch.object(GraphDatabase, 'driver'), patch.object( Neo4jProxy, '_execute_cypher_query') as mock_execute: mock_execute.return_value.single.side_effect = [{ 'cluster_name': 'cluster_name', 'uri': 'foo_dashboard://gold.bar/dashboard_id', 'url': 'http://www.foo.bar/dashboard_id', 'product': 'foobar', 'name': 'dashboard name', 'created_timestamp': 123456789, 'description': 'description', 'group_name': 'group_name', 'group_url': 'http://www.group_url.com', 'last_successful_run_timestamp': 9876543210, 'last_run_timestamp': 987654321, 'last_run_state': 'good_state', 'updated_timestamp': 123456654321, 'recent_view_count': 100, 'owners': [{ 'employee_type': 'teamMember', 'full_name': 'test_full_name', 'is_active': 'True', 'github_username': '******', 'slack_id': 'test_id', 'last_name': 'test_last_name', 'first_name': 'test_first_name', 'team_name': 'test_team', 'email': 'test_email', }, { 'employee_type': 'teamMember', 'full_name': 'test_full_name2', 'is_active': 'True', 'github_username': '******', 'slack_id': 'test_id2', 'last_name': 'test_last_name2', 'first_name': 'test_first_name2', 'team_name': 'test_team2', 'email': 'test_email2', }], 'tags': [{ 'key': 'tag_key1', 'tag_type': 'tag_type1' }, { 'key': 'tag_key2', 'tag_type': 'tag_type2' }], 'charts': [{ 'name': 'chart1' }, { 'name': 'chart2' }], 'queries': [{ 'name': 'query1' }, { 'name': 'query2' }], 'tables': [{ 'database': 'db1', 'name': 'table1', 'description': 'table description 1', 'cluster': 'cluster1', 'schema': 'schema1' }, { 'database': 'db2', 'name': 'table2', 'description': None, 'cluster': 'cluster2', 'schema': 'schema2' }] }, { 'cluster_name': 'cluster_name', 'uri': 'foo_dashboard://gold.bar/dashboard_id', 'url': 'http://www.foo.bar/dashboard_id', 'product': 'foobar', 'name': 'dashboard name', 'created_timestamp': 123456789, 'description': None, 'group_name': 'group_name', 'group_url': 'http://www.group_url.com', 'last_run_timestamp': None, 'last_run_state': None, 'updated_timestamp': None, 'recent_view_count': 0, 'owners': [], 'tags': [], 'charts': [], 'queries': [], 'tables': [] }] neo4j_proxy = Neo4jProxy(host='DOES_NOT_MATTER', port=0000) dashboard = neo4j_proxy.get_dashboard(id='dashboard_id') expected = DashboardDetail( uri='foo_dashboard://gold.bar/dashboard_id', cluster='cluster_name', group_name='group_name', group_url='http://www.group_url.com', product='foobar', name='dashboard name', url='http://www.foo.bar/dashboard_id', description='description', created_timestamp=123456789, last_successful_run_timestamp=9876543210, updated_timestamp=123456654321, last_run_timestamp=987654321, last_run_state='good_state', owners=[ User(email='test_email', first_name='test_first_name', last_name='test_last_name', full_name='test_full_name', is_active='True', github_username='******', team_name='test_team', slack_id='test_id', employee_type='teamMember', manager_fullname=''), User(email='test_email2', first_name='test_first_name2', last_name='test_last_name2', full_name='test_full_name2', is_active='True', github_username='******', team_name='test_team2', slack_id='test_id2', employee_type='teamMember', manager_fullname='') ], frequent_users=[], chart_names=['chart1', 'chart2'], query_names=['query1', 'query2'], tables=[ PopularTable(database='db1', name='table1', description='table description 1', cluster='cluster1', schema='schema1'), PopularTable(database='db2', name='table2', cluster='cluster2', schema='schema2'), ], tags=[ Tag(tag_type='tag_type1', tag_name='tag_key1'), Tag(tag_type='tag_type2', tag_name='tag_key2') ], recent_view_count=100) self.assertEqual(expected, dashboard) dashboard2 = neo4j_proxy.get_dashboard(id='dashboard_id') expected2 = DashboardDetail( uri='foo_dashboard://gold.bar/dashboard_id', cluster='cluster_name', group_name='group_name', group_url='http://www.group_url.com', product='foobar', name='dashboard name', url='http://www.foo.bar/dashboard_id', description=None, created_timestamp=123456789, updated_timestamp=None, last_run_timestamp=None, last_run_state=None, owners=[], frequent_users=[], chart_names=[], query_names=[], tables=[], tags=[], last_successful_run_timestamp=None, recent_view_count=0) self.assertEqual(expected2, dashboard2)
def test_get_dashboard(self, mock_rds_client: Any) -> None: # dashboard_metadata dashboard = RDSDashboard( rk='foo_dashboard://gold.bar/dashboard_id', name='dashboard name', dashboard_url='http://www.foo.bar/dashboard_id', created_timestamp=123456789) dashboard_group = RDSDashboardGroup( name='group_name', dashboard_group_url='http://www.group_url.com') dashboard_group.cluster = RDSCluster(name='cluster_name') dashboard.group = dashboard_group dashboard.description = RDSDashboardDescription( description='description') dashboard.execution = [ RDSDashboardExecution(rk='dashboard_last_successful_execution', timestamp=9876543210), RDSDashboardExecution(rk='dashboard_last_execution', timestamp=987654321, state='good_state') ] dashboard.timestamp = RDSDashboardTimestamp(timestamp=123456654321) dashboard.tags = [ RDSTag(rk='tag_key1', tag_type='default'), RDSTag(rk='tag_key2', tag_type='default') ] dashboard.badges = [RDSBadge(rk='golden', category='table_status')] dashboard.owners = [ RDSUser(email='test_email', first_name='test_first_name', last_name='test_last_name', full_name='test_full_name', is_active=True, github_username='******', team_name='test_team', slack_id='test_id', employee_type='teamMember'), RDSUser(email='test_email2', first_name='test_first_name2', last_name='test_last_name2', full_name='test_full_name2', is_active=True, github_username='******', team_name='test_team2', slack_id='test_id2', employee_type='teamMember') ] dashboard.usage = [RDSDashboardUsage(read_count=100)] mock_client = MagicMock() mock_rds_client.return_value = mock_client mock_create_session = MagicMock() mock_client.create_session.return_value = mock_create_session mock_session = MagicMock() mock_create_session.__enter__.return_value = mock_session mock_session_query = MagicMock() mock_session.query.return_value = mock_session_query mock_session_query_filter = MagicMock() mock_session_query.filter.return_value = mock_session_query_filter mock_session_query_filter.first.return_value = dashboard # queries query1 = RDSDashboardQuery(name='query1') query2 = RDSDashboardQuery(name='query2', url='http://foo.bar/query', query_text='SELECT * FROM foo.bar') query1.charts = [RDSDashboardChart(name='chart1')] query2.charts = [RDSDashboardChart(name='chart2')] queries = [query1, query2] # tables database1 = RDSDatabase(name='db1') database2 = RDSDatabase(name='db2') cluster1 = RDSCluster(name='cluster1') cluster2 = RDSCluster(name='cluster2') schema1 = RDSSchema(name='schema1') schema2 = RDSSchema(name='schema2') table1 = RDSTable(name='table1') table2 = RDSTable(name='table2') description1 = RDSTableDescription(description='table description 1') schema1.cluster = cluster1 cluster1.database = database1 schema2.cluster = cluster2 cluster2.database = database2 table1.schema = schema1 table2.schema = schema2 table1.description = description1 tables = [table1, table2] mock_session_query_filter_options = MagicMock() mock_session_query_filter.options.return_value = mock_session_query_filter_options mock_session_query_filter_options.all.side_effect = [queries, tables] expected = DashboardDetail( uri='foo_dashboard://gold.bar/dashboard_id', cluster='cluster_name', group_name='group_name', group_url='http://www.group_url.com', product='foo', name='dashboard name', url='http://www.foo.bar/dashboard_id', description='description', created_timestamp=123456789, last_successful_run_timestamp=9876543210, updated_timestamp=123456654321, last_run_timestamp=987654321, last_run_state='good_state', owners=[ User(email='test_email', first_name='test_first_name', last_name='test_last_name', full_name='test_full_name', is_active=True, github_username='******', team_name='test_team', slack_id='test_id', employee_type='teamMember', manager_fullname=''), User(email='test_email2', first_name='test_first_name2', last_name='test_last_name2', full_name='test_full_name2', is_active=True, github_username='******', team_name='test_team2', slack_id='test_id2', employee_type='teamMember', manager_fullname='') ], frequent_users=[], chart_names=['chart1', 'chart2'], query_names=['query1', 'query2'], queries=[ DashboardQuery(name='query1'), DashboardQuery(name='query2', url='http://foo.bar/query', query_text='SELECT * FROM foo.bar') ], tables=[ PopularTable(database='db1', name='table1', description='table description 1', cluster='cluster1', schema='schema1'), PopularTable(database='db2', name='table2', cluster='cluster2', schema='schema2'), ], tags=[ Tag(tag_type='default', tag_name='tag_key1'), Tag(tag_type='default', tag_name='tag_key2') ], badges=[Badge(badge_name='golden', category='table_status')], recent_view_count=100) proxy = MySQLProxy() actual = proxy.get_dashboard(id='dashboard_id') self.assertEqual(expected, actual)
def test_get_table(self, mock_rds_client: Any) -> None: database = RDSDatabase(name='hive') cluster = RDSCluster(name='gold') schema = RDSSchema(name='foo_schema') schema.cluster = cluster cluster.database = database table = RDSTable(name='foo_table') table.schema = schema table.description = RDSTableDescription(description='foo description') col1 = RDSColumn(name='bar_id_1', type='varchar', sort_order=0) col1.description = RDSColumnDescription( description='bar col description') col1.stats = [ RDSColumnStat(stat_type='avg', start_epoch='1', end_epoch='1', stat_val='1') ] col2 = RDSColumn(name='bar_id_2', type='bigint', sort_order=1) col2.description = RDSColumnDescription( description='bar col2 description') col2.stats = [ RDSColumnStat(stat_type='avg', start_epoch='2', end_epoch='2', stat_val='2') ] col2.badges = [RDSBadge(rk='primary key', category='column')] columns = [col1, col2] table.watermarks = [ RDSTableWatermark( rk='hive://gold.test_schema/test_table/high_watermark/', partition_key='ds', partition_value='fake_value', create_time='fake_time'), RDSTableWatermark( rk='hive://gold.test_schema/test_table/low_watermark/', partition_key='ds', partition_value='fake_value', create_time='fake_time') ] table.application = RDSApplication( application_url='airflow_host/admin/airflow/tree?dag_id=test_table', description='DAG generating a table', name='Airflow', id='dag/task_id') table.timestamp = RDSTableTimestamp(last_updated_timestamp=1) table.owners = [ RDSUser(rk='*****@*****.**', email='*****@*****.**') ] table.tags = [RDSTag(rk='test', tag_type='default')] table.badges = [RDSBadge(rk='golden', category='table_status')] table.source = RDSTableSource(rk='some key', source_type='github', source='/source_file_loc') table.programmatic_descriptions = [ RDSTableProgrammaticDescription(description_source='s3_crawler', description='Test Test Test'), RDSTableProgrammaticDescription( description_source='quality_report', description='Test Test') ] readers = [RDSTableUsage(user_rk='*****@*****.**', read_count=5)] mock_client = MagicMock() mock_rds_client.return_value = mock_client mock_create_session = MagicMock() mock_client.create_session.return_value = mock_create_session mock_session = MagicMock() mock_create_session.__enter__.return_value = mock_session mock_session_query = MagicMock() mock_session.query.return_value = mock_session_query mock_session_query_filter = MagicMock() mock_session_query.filter.return_value = mock_session_query_filter mock_session_query_filter.first.return_value = table mock_session_query_filter_orderby = MagicMock() mock_session_query_filter.order_by.return_value = mock_session_query_filter_orderby mock_session_query_filter_orderby_limit = MagicMock() mock_session_query_filter_orderby.limit.return_value = mock_session_query_filter_orderby_limit mock_session_query_filter_orderby_limit.all.return_value = readers mock_session_query_filter_options = MagicMock() mock_session_query_filter.options.return_value = mock_session_query_filter_options mock_session_query_filter_options.all.return_value = columns proxy = MySQLProxy() actual_table = proxy.get_table(table_uri='dummy_uri') expected = Table( database='hive', cluster='gold', schema='foo_schema', name='foo_table', tags=[Tag(tag_name='test', tag_type='default')], badges=[Badge(badge_name='golden', category='table_status')], table_readers=[ Reader(user=User(email='*****@*****.**'), read_count=5) ], description='foo description', watermarks=[ Watermark(watermark_type='high_watermark', partition_key='ds', partition_value='fake_value', create_time='fake_time'), Watermark(watermark_type='low_watermark', partition_key='ds', partition_value='fake_value', create_time='fake_time') ], columns=[ Column(name='bar_id_1', description='bar col description', col_type='varchar', sort_order=0, stats=[ Stat(start_epoch=1, end_epoch=1, stat_type='avg', stat_val='1') ], badges=[]), Column(name='bar_id_2', description='bar col2 description', col_type='bigint', sort_order=1, stats=[ Stat(start_epoch=2, end_epoch=2, stat_type='avg', stat_val='2') ], badges=[ Badge(badge_name='primary key', category='column') ]) ], owners=[User(email='*****@*****.**')], table_writer=Application( application_url= 'airflow_host/admin/airflow/tree?dag_id=test_table', description='DAG generating a table', name='Airflow', id='dag/task_id'), last_updated_timestamp=1, source=Source(source='/source_file_loc', source_type='github'), is_view=False, programmatic_descriptions=[ ProgrammaticDescription(source='quality_report', text='Test Test'), ProgrammaticDescription(source='s3_crawler', text='Test Test Test') ]) self.assertEqual(str(expected), str(actual_table))
def _exec_table_query(self, table_uri: str) -> Tuple: """ Queries one Cypher record with watermark list, Application, ,timestamp, owner records and tag records. """ # Return Value: (Watermark Results, Table Writer, Last Updated Timestamp, owner records, tag records) table_level_query = textwrap.dedent("""\ MATCH (tbl:Table {key: $tbl_key}) OPTIONAL MATCH (wmk:Watermark)-[:BELONG_TO_TABLE]->(tbl) OPTIONAL MATCH (application:Application)-[:GENERATES]->(tbl) OPTIONAL MATCH (tbl)-[:LAST_UPDATED_AT]->(t:Timestamp) OPTIONAL MATCH (owner:User)<-[:OWNER]-(tbl) OPTIONAL MATCH (tbl)-[:TAGGED_BY]->(tag:Tag{tag_type: $tag_normal_type}) OPTIONAL MATCH (tbl)-[:TAGGED_BY]->(badge:Tag{tag_type: $tag_badge_type}) OPTIONAL MATCH (tbl)-[:SOURCE]->(src:Source) OPTIONAL MATCH (tbl)-[:DESCRIPTION]->(prog_descriptions:Programmatic_Description) RETURN collect(distinct wmk) as wmk_records, application, t.last_updated_timestamp as last_updated_timestamp, collect(distinct owner) as owner_records, collect(distinct tag) as tag_records, collect(distinct badge) as badge_records, src, collect(distinct prog_descriptions) as prog_descriptions """) table_records = self._execute_cypher_query(statement=table_level_query, param_dict={'tbl_key': table_uri, 'tag_normal_type': 'default', 'tag_badge_type': 'badge'}) table_records = table_records.single() wmk_results = [] table_writer = None wmk_records = table_records['wmk_records'] for record in wmk_records: if record['key'] is not None: watermark_type = record['key'].split('/')[-2] wmk_result = Watermark(watermark_type=watermark_type, partition_key=record['partition_key'], partition_value=record['partition_value'], create_time=record['create_time']) wmk_results.append(wmk_result) tags = [] if table_records.get('tag_records'): tag_records = table_records['tag_records'] for record in tag_records: tag_result = Tag(tag_name=record['key'], tag_type=record['tag_type']) tags.append(tag_result) badges = [] if table_records.get('badge_records'): badge_records = table_records['badge_records'] for record in badge_records: badge_result = Tag(tag_name=record['key'], tag_type=record['tag_type']) badges.append(badge_result) application_record = table_records['application'] if application_record is not None: table_writer = Application( application_url=application_record['application_url'], description=application_record['description'], name=application_record['name'], id=application_record.get('id', '') ) timestamp_value = table_records['last_updated_timestamp'] owner_record = [] for owner in table_records.get('owner_records', []): owner_record.append(User(email=owner['email'])) src = None if table_records['src']: src = Source(source_type=table_records['src']['source_type'], source=table_records['src']['source']) prog_descriptions = self._extract_programmatic_descriptions_from_query( table_records.get('prog_descriptions', []) ) return wmk_results, table_writer, timestamp_value, owner_record, tags, src, badges, prog_descriptions