コード例 #1
0
    def test_ensure_record_get_updated(self) -> None:
        query_to_merge_seed_record = [
            {
                'organization': 'amundsen',
                'dashboard_id': 'd1'
            },
            {
                'organization': 'amundsen-databuilder',
                'dashboard_id': 'd2'
            },
            {
                'organization': 'amundsen-dashboard',
                'dashboard_id': 'd3'
            },
        ]
        query_to_merge = RestApiQuerySeed(
            seed_record=query_to_merge_seed_record)
        query_merger = QueryMerger(query_to_merge=query_to_merge,
                                   merge_key='dashboard_id')

        with patch('databuilder.rest_api.rest_api_query.requests.get'
                   ) as mock_get:
            mock_get.return_value.json.side_effect = [
                {
                    'foo': {
                        'name': 'john'
                    }
                },
                {
                    'foo': {
                        'name': 'doe'
                    }
                },
            ]
            query = RestApiQuery(query_to_join=self.query_to_join,
                                 url=self.url,
                                 params={},
                                 json_path=self.json_path,
                                 field_names=self.field_names,
                                 query_merger=query_merger)
            results = list(query.execute())
            self.assertEqual(len(results), 2)
            self.assertDictEqual(
                {
                    'dashboard_id': 'd1',
                    'foo1': 'bar1',
                    'name_field': 'john',
                    'organization': 'amundsen'
                },
                results[0],
            )
            self.assertDictEqual(
                {
                    'dashboard_id': 'd3',
                    'foo2': 'bar2',
                    'name_field': 'doe',
                    'organization': 'amundsen-dashboard'
                },
                results[1],
            )
コード例 #2
0
    def _build_restapi_query(self) -> RestApiQuery:
        """
        Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports
        API, and run API) joining together.
        :return: A RestApiQuery that provides Mode Dashboard execution (run)
        """

        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
        params = ModeDashboardUtils.get_auth_params(conf=self._conf)

        # Reports
        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
        report_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
        json_path = '(_embedded.reports[*].token)'
        field_names = ['dashboard_id']
        reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params,
                                                  json_path=json_path, field_names=field_names, skip_no_result=True)

        queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries'
        json_path = '_embedded.queries[*].[token,name]'
        field_names = ['query_id', 'query_name']
        query_names_query = RestApiQuery(query_to_join=reports_query, url=queries_url_template, params=params,
                                         json_path=json_path, field_names=field_names, skip_no_result=True)

        charts_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries/{query_id}/charts'
        json_path = '(_embedded.charts[*].token) | (_embedded.charts[*]._links.report_viz_web.href)'
        field_names = ['chart_id', 'chart_url']
        chart_names_query = RestApiQuery(query_to_join=query_names_query, url=charts_url_template, params=params,
                                         json_path=json_path, field_names=field_names, skip_no_result=True,
                                         json_path_contains_or=True)

        return chart_names_query
コード例 #3
0
    def _build_restapi_query(self):
        """
        Build REST API Query. To get Mode Dashboard owner, it needs to call three APIs (spaces API, reports
        API, and user API) joining together.
        :return: A RestApiQuery that provides Mode Dashboard owner
        """
        # type: () -> RestApiQuery

        # Seed query record for next query api to join with
        seed_record = [{
            'organization': self._conf.get_string(ORGANIZATION),
            'is_active': None,
            'updated_at': None,
            'do_not_update_empty_attribute': True,
        }]
        seed_query = RestApiQuerySeed(seed_record=seed_record)

        # memberships
        # https://mode.com/developer/api-reference/management/organization-memberships/#listMemberships
        memberships_url_template = 'https://app.mode.com/api/{organization}/memberships'
        params = {
            'auth':
            HTTPBasicAuth(self._conf.get_string(MODE_ACCESS_TOKEN),
                          self._conf.get_string(MODE_PASSWORD_TOKEN))
        }

        json_path = '(_embedded.memberships[*].member_username) | (_embedded.memberships[*]._links.user.href)'
        field_names = ['mode_user_id', 'mode_user_resource_path']
        mode_user_ids_query = RestApiQuery(query_to_join=seed_query,
                                           url=memberships_url_template,
                                           params=params,
                                           json_path=json_path,
                                           field_names=field_names,
                                           skip_no_result=True,
                                           json_path_contains_or=True)

        # https://mode.com/developer/api-reference/management/users/
        user_url_template = 'https://app.mode.com{mode_user_resource_path}'

        json_path = 'email'
        field_names = ['email']
        failure_handler = HttpFailureSkipOnStatus(status_codes_to_skip={404})
        mode_user_email_query = RestApiQuery(
            query_to_join=mode_user_ids_query,
            url=user_url_template,
            params=params,
            json_path=json_path,
            field_names=field_names,
            skip_no_result=True,
            can_skip_failure=failure_handler.can_skip_failure)

        return mode_user_email_query
コード例 #4
0
    def test_compute_subresult_single_field(self) -> None:
        sub_records = RestApiQuery._compute_sub_records(result_list=['1', '2', '3'], field_names=['foo'])

        expected_records = [
            ['1'], ['2'], ['3']
        ]

        assert expected_records == sub_records

        sub_records = RestApiQuery._compute_sub_records(result_list=['1', '2', '3'], field_names=['foo'],
                                                        json_path_contains_or=True)

        assert expected_records == sub_records
コード例 #5
0
    def test_compute_subresult_multiple_fields_json_path_and_expression(self):
        sub_records = RestApiQuery._compute_sub_records(
            result_list=['1', 'a', '2', 'b', '3', 'c'],
            field_names=['foo', 'bar'])

        expected_records = [['1', 'a'], ['2', 'b'], ['3', 'c']]

        assert expected_records == sub_records

        sub_records = RestApiQuery._compute_sub_records(
            result_list=['1', 'a', 'x', '2', 'b', 'y', '3', 'c', 'z'],
            field_names=['foo', 'bar', 'baz'])

        expected_records = [['1', 'a', 'x'], ['2', 'b', 'y'], ['3', 'c', 'z']]

        assert expected_records == sub_records
コード例 #6
0
    def get_spaces_query_api(conf: ConfigTree) -> BaseRestApiQuery:
        """
        Provides RestApiQuerySeed where it will provides iterator of dictionaries as records where dictionary keys are
         organization, dashboard_group_id, dashboard_group and dashboard_group_description
        :param conf:
        :return:
        """

        # https://mode.com/developer/api-reference/management/spaces/#listSpaces
        spaces_url_template = 'https://app.mode.com/api/{organization}/spaces?filter=all'

        # Seed query record for next query api to join with
        seed_record = [{'organization': conf.get_string(ORGANIZATION)}]
        seed_query = RestApiQuerySeed(seed_record=seed_record)

        # Spaces
        params = {
            'auth':
            HTTPBasicAuth(conf.get_string(MODE_ACCESS_TOKEN),
                          conf.get_string(MODE_PASSWORD_TOKEN))
        }

        json_path = '_embedded.spaces[*].[token,name,description]'
        field_names = [
            'dashboard_group_id', 'dashboard_group',
            'dashboard_group_description'
        ]
        spaces_query = RestApiQuery(query_to_join=seed_query,
                                    url=spaces_url_template,
                                    params=params,
                                    json_path=json_path,
                                    field_names=field_names)

        return spaces_query
コード例 #7
0
    def _build_restapi_query(self) -> RestApiQuery:

        dashes_query = RedashPaginatedRestApiQuery(
            query_to_join=EmptyRestApiQuerySeed(),
            url=f'{self._api_base_url}/dashboards',
            params=self._get_default_api_query_params(),
            json_path=
            'results[*].[id,name,slug,created_at,updated_at,is_archived,is_draft,user]',
            field_names=[
                'dashboard_id', 'dashboard_name', 'slug', 'created_timestamp',
                'last_modified_timestamp', 'is_archived', 'is_draft', 'user'
            ],
            skip_no_result=True)

        if self._redash_version >= 9:
            dashboard_url = f'{self._api_base_url}/dashboards/{{dashboard_id}}'
        else:
            dashboard_url = f'{self._api_base_url}/dashboards/{{slug}}'

        return RestApiQuery(query_to_join=dashes_query,
                            url=dashboard_url,
                            params=self._get_default_api_query_params(),
                            json_path='widgets',
                            field_names=['widgets'],
                            skip_no_result=True)
コード例 #8
0
    def _build_restapi_query(self) -> RestApiQuery:
        databricks_sql_dashboard_query = DatabricksSQLPaginatedRestApiQuery(
            query_to_join=EmptyRestApiQuerySeed(),
            url=self._databricks_sql_dashboards_api_base,
            params={"headers": self._get_databrick_request_headers()},
            json_path="results[*].[id,name,tags,updated_at,created_at,user]",
            field_names=[
                "dashboard_id",
                "dashboard_name",
                "tags",
                "last_modified_timestamp",
                "created_timestamp",
                "user",
            ],
            skip_no_results=True,
        )

        return RestApiQuery(
            query_to_join=databricks_sql_dashboard_query,
            url=f"{self._databricks_sql_dashboards_api_base}/{{dashboard_id}}",
            params={"headers": self._get_databrick_request_headers()},
            json_path="widgets",
            field_names=["widgets"],
            skip_no_result=True,
        )
コード例 #9
0
    def test_exception_rasied_with_duplicate_merge_key(self) -> None:
        """
         Two records in query_to_merge results have {'dashboard_id': 'd2'},
         exception should be raised
        """
        query_to_merge_seed_record = [
            {
                'organization': 'amundsen',
                'dashboard_id': 'd1'
            },
            {
                'organization': 'amundsen-databuilder',
                'dashboard_id': 'd2'
            },
            {
                'organization': 'amundsen-dashboard',
                'dashboard_id': 'd2'
            },
        ]
        query_to_merge = RestApiQuerySeed(
            seed_record=query_to_merge_seed_record)
        query_merger = QueryMerger(query_to_merge=query_to_merge,
                                   merge_key='dashboard_id')

        with patch('databuilder.rest_api.rest_api_query.requests.get'
                   ) as mock_get:
            mock_get.return_value.json.side_effect = [
                {
                    'foo': {
                        'name': 'john'
                    }
                },
                {
                    'foo': {
                        'name': 'doe'
                    }
                },
            ]
            query = RestApiQuery(query_to_join=self.query_to_join,
                                 url=self.url,
                                 params={},
                                 json_path=self.json_path,
                                 field_names=self.field_names,
                                 query_merger=query_merger)
            self.assertRaises(Exception, query.execute())
コード例 #10
0
    def test_compute_subresult_multiple_fields_json_path_or_expression(self):
        sub_records = RestApiQuery._compute_sub_records(
            result_list=['1', '2', '3', 'a', 'b', 'c'],
            field_names=['foo', 'bar'],
            json_path_contains_or=True)

        expected_records = [['1', 'a'], ['2', 'b'], ['3', 'c']]

        self.assertEqual(expected_records, sub_records)

        sub_records = RestApiQuery._compute_sub_records(
            result_list=['1', '2', '3', 'a', 'b', 'c', 'x', 'y', 'z'],
            field_names=['foo', 'bar', 'baz'],
            json_path_contains_or=True)

        expected_records = [['1', 'a', 'x'], ['2', 'b', 'y'], ['3', 'c', 'z']]

        self.assertEqual(expected_records, sub_records)
コード例 #11
0
    def test_rest_api_query_multiple_fields(self):

        seed_record = [{'foo1': 'bar1'}, {'foo2': 'bar2'}]
        seed_query = RestApiQuerySeed(seed_record=seed_record)

        with patch('databuilder.rest_api.rest_api_query.requests.get'
                   ) as mock_get:
            json_path = 'foo.[name,hobby]'
            field_names = ['name_field', 'hobby']

            mock_get.return_value.json.side_effect = [
                {
                    'foo': {
                        'name': 'john',
                        'hobby': 'skiing'
                    }
                },
                {
                    'foo': {
                        'name': 'doe',
                        'hobby': 'snowboarding'
                    }
                },
            ]
            query = RestApiQuery(query_to_join=seed_query,
                                 url='foobar',
                                 params={},
                                 json_path=json_path,
                                 field_names=field_names)

            expected = [{
                'name_field': 'john',
                'hobby': 'skiing',
                'foo1': 'bar1'
            }, {
                'name_field': 'doe',
                'hobby': 'snowboarding',
                'foo2': 'bar2'
            }]

            for actual in query.execute():
                self.assertDictEqual(expected.pop(0), actual)
コード例 #12
0
    def _build_restapi_query(self):
        """
        Build REST API Query. To get Mode Dashboard owner, it needs to call three APIs (spaces API, reports
        API, and user API) joining together.
        :return: A RestApiQuery that provides Mode Dashboard owner
        """
        # type: () -> RestApiQuery

        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
        report_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'

        # https://mode.com/developer/api-reference/management/users/
        creator_url_template = 'https://app.mode.com{creator_resource_path}'

        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
        params = ModeDashboardUtils.get_auth_params(conf=self._conf)

        # Reports
        json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.creator.href)'
        field_names = ['dashboard_id', 'creator_resource_path']
        creator_resource_path_query = RestApiQuery(query_to_join=spaces_query,
                                                   url=report_url_template,
                                                   params=params,
                                                   json_path=json_path,
                                                   field_names=field_names,
                                                   skip_no_result=True,
                                                   json_path_contains_or=True)

        json_path = 'email'
        field_names = ['email']
        failure_handler = HttpFailureSkipOnStatus(status_codes_to_skip={404})
        owner_email_query = RestApiQuery(
            query_to_join=creator_resource_path_query,
            url=creator_url_template,
            params=params,
            json_path=json_path,
            field_names=field_names,
            skip_no_result=True,
            can_skip_failure=failure_handler.can_skip_failure)

        return owner_email_query
    def _build_restapi_query(self):
        """
        Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports
        API, and run API) joining together.
        :return: A RestApiQuery that provides Mode Dashboard execution (run)
        """
        # type: () -> RestApiQuery

        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
        params = ModeDashboardUtils.get_auth_params(conf=self._conf)

        # Reports
        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
        url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
        json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.last_run.href)'
        field_names = ['dashboard_id', 'last_run_resource_path']
        last_run_resource_path_query = RestApiQuery(query_to_join=spaces_query,
                                                    url=url,
                                                    params=params,
                                                    json_path=json_path,
                                                    field_names=field_names,
                                                    skip_no_result=True,
                                                    json_path_contains_or=True)

        # https://mode.com/developer/api-reference/analytics/report-runs/#getReportRun
        url = 'https://app.mode.com{last_run_resource_path}'
        json_path = '[state,completed_at]'
        field_names = ['execution_state', 'execution_timestamp']
        last_run_state_query = RestApiQuery(
            query_to_join=last_run_resource_path_query,
            url=url,
            params=params,
            json_path=json_path,
            field_names=field_names,
            skip_no_result=True)

        return last_run_state_query
コード例 #14
0
    def _build_restapi_query(self):
        """
        Build REST API Query. To get Mode Dashboard usage, it needs to call two APIs (spaces API and reports
        API) joining together.
        :return: A RestApiQuery that provides Mode Dashboard metadata
        """
        # type: () -> RestApiQuery

        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
        reports_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'

        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
        params = ModeDashboardUtils.get_auth_params(conf=self._conf)

        # Reports
        # JSONPATH expression. it goes into array which is located in _embedded.reports and then extracts token,
        # and view_count
        json_path = '_embedded.reports[*].[token,view_count]'
        field_names = ['dashboard_id', 'accumulated_view_count']
        reports_query = RestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params,
                                     json_path=json_path, field_names=field_names, skip_no_result=True)
        return reports_query
コード例 #15
0
    def _build_restapi_query(self):
        # type: () -> RestApiQuery

        dashes_query = RedashPaginatedRestApiQuery(
            query_to_join=EmptyRestApiQuerySeed(),
            url='{redash_api}/dashboards'.format(redash_api=self._api_base_url),
            params=self._get_default_api_query_params(),
            json_path='results[*].[id,name,slug,created_at,updated_at,is_archived,is_draft,user]',
            field_names=[
                'dashboard_id', 'dashboard_name', 'slug', 'created_timestamp',
                'last_modified_timestamp', 'is_archived', 'is_draft', 'user'
            ],
            skip_no_result=True
        )

        return RestApiQuery(
            query_to_join=dashes_query,
            url='{redash_api}/dashboards/{{slug}}'.format(redash_api=self._api_base_url),
            params=self._get_default_api_query_params(),
            json_path='widgets',
            field_names=['widgets'],
            skip_no_result=True
        )
    def _build_restapi_query(self):
        """
        Build REST API Query. To get Mode Dashboard last modified timestamp, it needs to call two APIs (spaces API,
        and reports API) joining together.
        :return: A RestApiQuery that provides Mode Dashboard last successful execution (run)
        """
        # type: () -> RestApiQuery

        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
        params = ModeDashboardUtils.get_auth_params(conf=self._conf)

        # Reports
        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
        url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
        json_path = '_embedded.reports[*].[token,edited_at]'
        field_names = ['dashboard_id', 'last_modified_timestamp']
        last_modified_query = RestApiQuery(query_to_join=spaces_query,
                                           url=url,
                                           params=params,
                                           json_path=json_path,
                                           field_names=field_names,
                                           skip_no_result=True)

        return last_modified_query