def _build_restapi_query(self): """ Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports API, and queries API) joining together. :return: A RestApiQuery that provides Mode Dashboard execution (run) """ # type: () -> RestApiQuery spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf) # Reports # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' json_path = '(_embedded.reports[*].token)' field_names = ['dashboard_id'] reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True) queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries' json_path = '_embedded.queries[*].[token,name,raw_query]' field_names = ['query_id', 'query_name', 'query_text'] query_names_query = RestApiQuery(query_to_join=reports_query, url=queries_url_template, params=params, json_path=json_path, field_names=field_names, skip_no_result=True) return query_names_query
def _build_restapi_query(self) -> RestApiQuery: """ Build REST API Query. To get Mode Dashboard metadata, it needs to call two APIs (spaces API and reports API) joining together. :return: A RestApiQuery that provides Mode Dashboard metadata """ # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace reports_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf) # Reports # JSONPATH expression. it goes into array which is located in _embedded.reports and then extracts token, name, # and description json_path = '_embedded.reports[*].[token,name,description,created_at]' field_names = [ 'dashboard_id', 'dashboard_name', 'description', 'created_timestamp' ] reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params, json_path=json_path, field_names=field_names, skip_no_result=True) return reports_query
def _build_restapi_query(self) -> ModePaginatedRestApiQuery: """ Build REST API Query to get Mode Dashboard queries :return: A RestApiQuery that provides Mode Dashboard execution (run) """ seed_query = ModeDashboardUtils.get_seed_query(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf, discover_auth=True) # Queries # https://mode.com/developer/discovery-api/analytics/queries/ url = 'https://app.mode.com/batch/{organization}/queries' json_path = 'queries[*].[report_token, space_token, token, name, raw_query]' field_names = [ 'dashboard_id', 'dashboard_group_id', 'query_id', 'query_name', 'query_text' ] max_record_size = 1000 pagination_json_path = 'queries[*]' query_names_query = ModePaginatedRestApiQuery( query_to_join=seed_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, max_record_size=max_record_size, pagination_json_path=pagination_json_path) return query_names_query
def _build_restapi_query(self) -> ModePaginatedRestApiQuery: """ Build REST API Query to get Mode Dashboard last successful execution metadata. :return: A RestApiQuery that provides Mode Dashboard last successful execution (run) """ seed_query = ModeDashboardUtils.get_seed_query(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf, discover_auth=True) # Reports # https://mode.com/developer/discovery-api/analytics/reports/ url = 'https://app.mode.com/batch/{organization}/reports' json_path = 'reports[*].[token, space_token, last_successfully_run_at]' field_names = [ 'dashboard_id', 'dashboard_group_id', 'execution_timestamp' ] max_record_size = 1000 pagination_json_path = 'reports[*]' last_successful_run_query = ModePaginatedRestApiQuery( query_to_join=seed_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, max_record_size=max_record_size, pagination_json_path=pagination_json_path) return last_successful_run_query
def _build_restapi_query(self): """ Build REST API Query. To get Mode Dashboard last successful execution, it needs to call two APIs (spaces API, and reports API) joining together. :return: A RestApiQuery that provides Mode Dashboard last successful execution (run) """ # type: () -> RestApiQuery spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf) # Reports # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' json_path = '_embedded.reports[*].[token,last_successfully_run_at]' field_names = ['dashboard_id', 'execution_timestamp'] last_successful_run_query = ModePaginatedRestApiQuery( query_to_join=spaces_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True) return last_successful_run_query
def _build_restapi_query(self) -> RestApiQuery: """ Build a paginated REST API based on Mode discovery API :return: """ params = ModeDashboardUtils.get_auth_params(conf=self._conf, discover_auth=True) seed_record = [{ 'organization': self._conf.get_string(ORGANIZATION), 'is_active': None, 'updated_at': None, 'do_not_update_empty_attribute': True, }] seed_query = RestApiQuerySeed(seed_record=seed_record) chart_url_template = 'http://app.mode.com/batch/{organization}/charts' if self._conf.get_bool( ModeDashboardChartsBatchExtractor.INCLUDE_ALL_SPACE, default=False): chart_url_template += '?include_spaces=all' json_path = '(charts[*].[space_token,report_token,query_token,token,chart_title,chart_type])' field_names = [ 'dashboard_group_id', 'dashboard_id', 'query_id', 'chart_id', 'chart_name', 'chart_type' ] chart_batch_query = ModePaginatedRestApiQuery( query_to_join=seed_query, url=chart_url_template, params=params, json_path=json_path, pagination_json_path=json_path, field_names=field_names, skip_no_result=True) return chart_batch_query
def _build_restapi_query(self) -> ModePaginatedRestApiQuery: """ Build REST API Query. To get Mode Dashboard usage, it needs to call three discovery APIs ( spaces API, reports API and report stats API). :return: A RestApiQuery that provides Mode Dashboard metadata """ seed_query = ModeDashboardUtils.get_seed_query(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf, discover_auth=True) # Reports # https://mode.com/developer/discovery-api/analytics/reports/ reports_url = 'https://app.mode.com/batch/{organization}/reports' reports_json_path = 'reports[*].[token, space_token]' reports_field_names = ['dashboard_id', 'dashboard_group_id'] reports_max_record_size = 1000 reports_pagination_json_path = 'reports[*]' spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) spaces_query_merger = QueryMerger(query_to_merge=spaces_query, merge_key='dashboard_group_id') reports_query = ModePaginatedRestApiQuery(query_to_join=seed_query, url=reports_url, params=params, json_path=reports_json_path, field_names=reports_field_names, skip_no_result=True, max_record_size=reports_max_record_size, pagination_json_path=reports_pagination_json_path, query_merger=spaces_query_merger) # https://mode.com/developer/discovery-api/analytics/report-stats/ stats_url = 'https://app.mode.com/batch/{organization}/report_stats' stats_json_path = 'report_stats[*].[report_token, view_count]' stats_field_names = ['dashboard_id', 'accumulated_view_count'] stats_max_record_size = 1000 stats_pagination_json_path = 'report_stats[*]' reports_query_merger = QueryMerger(query_to_merge=reports_query, merge_key='dashboard_id') report_stats_query = ModePaginatedRestApiQuery(query_to_join=seed_query, url=stats_url, params=params, json_path=stats_json_path, field_names=stats_field_names, skip_no_result=True, max_record_size=stats_max_record_size, pagination_json_path=stats_pagination_json_path, query_merger=reports_query_merger) return report_stats_query
def _build_restapi_query(self): """ Build REST API Query. To get Mode Dashboard owner, it needs to call three APIs (spaces API, reports API, and user API) joining together. :return: A RestApiQuery that provides Mode Dashboard owner """ # type: () -> RestApiQuery # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace report_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' # https://mode.com/developer/api-reference/management/users/ creator_url_template = 'https://app.mode.com{creator_resource_path}' spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf) # Reports json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.creator.href)' field_names = ['dashboard_id', 'creator_resource_path'] creator_resource_path_query = ModePaginatedRestApiQuery( query_to_join=spaces_query, url=report_url_template, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, json_path_contains_or=True) json_path = 'email' field_names = ['email'] failure_handler = HttpFailureSkipOnStatus(status_codes_to_skip={404}) owner_email_query = RestApiQuery( query_to_join=creator_resource_path_query, url=creator_url_template, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, can_skip_failure=failure_handler.can_skip_failure) return owner_email_query
def get_spaces_query_api(conf: ConfigTree) -> BaseRestApiQuery: """ Provides RestApiQuerySeed where it will provides iterator of dictionaries as records where dictionary keys are organization, dashboard_group_id, dashboard_group and dashboard_group_description :param conf: :return: """ # https://mode.com/developer/discovery-api/analytics/spaces spaces_url_template = 'https://app.mode.com/batch/{organization}/spaces' # Seed query record for next query api to join with seed_record = [{'organization': conf.get_string(ORGANIZATION)}] seed_query = RestApiQuerySeed(seed_record=seed_record) # mode_bearer_token must be provided in the conf # the token is required to access discovery endpoint # https://mode.com/developer/discovery-api/introduction/ params = ModeDashboardUtils.get_auth_params(conf=conf, discover_auth=True) json_path = 'spaces[*].[token,name,description]' field_names = [ 'dashboard_group_id', 'dashboard_group', 'dashboard_group_description' ] # based on https://mode.com/developer/discovery-api/analytics/spaces/#listSpacesForAccount pagination_json_path = 'spaces[*]' max_per_page = 1000 spaces_query = ModePaginatedRestApiQuery( pagination_json_path=pagination_json_path, max_record_size=max_per_page, query_to_join=seed_query, url=spaces_url_template, params=params, json_path=json_path, field_names=field_names) return spaces_query
def _build_restapi_query(self): """ Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports API, and run API) joining together. :return: A RestApiQuery that provides Mode Dashboard execution (run) """ # type: () -> RestApiQuery spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf) # Reports # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.last_run.href)' field_names = ['dashboard_id', 'last_run_resource_path'] last_run_resource_path_query = ModePaginatedRestApiQuery( query_to_join=spaces_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, json_path_contains_or=True) # https://mode.com/developer/api-reference/analytics/report-runs/#getReportRun url = 'https://app.mode.com{last_run_resource_path}' json_path = '[state,completed_at]' field_names = ['execution_state', 'execution_timestamp'] last_run_state_query = RestApiQuery( query_to_join=last_run_resource_path_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True) return last_run_state_query
def _build_restapi_query(self) -> ModePaginatedRestApiQuery: """ Build REST API Query to get Mode Dashboard metadata :return: A RestApiQuery that provides Mode Dashboard metadata """ seed_query = ModeDashboardUtils.get_seed_query(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf, discover_auth=True) # Reports # https://mode.com/developer/discovery-api/analytics/reports/ url = 'https://app.mode.com/batch/{organization}/reports' json_path = 'reports[*].[token, name, description, created_at, space_token]' field_names = [ 'dashboard_id', 'dashboard_name', 'description', 'created_timestamp', 'dashboard_group_id' ] max_record_size = 1000 pagination_json_path = 'reports[*]' spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) query_merger = QueryMerger(query_to_merge=spaces_query, merge_key='dashboard_group_id') reports_query = ModePaginatedRestApiQuery( query_to_join=seed_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, max_record_size=max_record_size, pagination_json_path=pagination_json_path, query_merger=query_merger) return reports_query
def test_no_pagination(self) -> None: seed_record = [{'foo1': 'bar1'}, {'foo2': 'bar2'}, {'foo3': 'bar3'}] seed_query = RestApiQuerySeed(seed_record=seed_record) with patch('databuilder.rest_api.rest_api_query.requests.get' ) as mock_get: json_path = 'foo[*].name' field_names = ['name_field'] mock_get.return_value.json.side_effect = [ # need to duplicate for json() is called twice { 'foo': [{ 'name': 'v1' }, { 'name': 'v2' }] }, { 'foo': [{ 'name': 'v1' }, { 'name': 'v2' }] }, { 'foo': [{ 'name': 'v3' }] }, { 'foo': [{ 'name': 'v3' }] }, { 'foo': [{ 'name': 'v4' }, { 'name': 'v5' }] }, { 'foo': [{ 'name': 'v4' }, { 'name': 'v5' }] }, ] query = ModePaginatedRestApiQuery(query_to_join=seed_query, url='foobar', params={}, json_path=json_path, field_names=field_names, pagination_json_path='foo[*]', max_record_size=3) expected_list = [{ 'name_field': 'v1', 'foo1': 'bar1' }, { 'name_field': 'v2', 'foo1': 'bar1' }, { 'name_field': 'v3', 'foo2': 'bar2' }, { 'name_field': 'v4', 'foo3': 'bar3' }, { 'name_field': 'v5', 'foo3': 'bar3' }] for actual in query.execute(): self.assertDictEqual(actual, expected_list.pop(0)) self.assertEqual(mock_get.call_count, 3) calls = [call('foobar?page=1')] mock_get.assert_has_calls(calls, any_order=True)