def test_rest_api_query_seed(self): rest_api_query = RestApiQuerySeed(seed_record=[{ 'foo': 'bar' }, { 'john': 'doe' }]) result = [v for v in rest_api_query.execute()] expected = [{'foo': 'bar'}, {'john': 'doe'}] self.assertListEqual(expected, result)
def get_spaces_query_api(conf: ConfigTree) -> BaseRestApiQuery: """ Provides RestApiQuerySeed where it will provides iterator of dictionaries as records where dictionary keys are organization, dashboard_group_id, dashboard_group and dashboard_group_description :param conf: :return: """ # https://mode.com/developer/api-reference/management/spaces/#listSpaces spaces_url_template = 'https://app.mode.com/api/{organization}/spaces?filter=all' # Seed query record for next query api to join with seed_record = [{'organization': conf.get_string(ORGANIZATION)}] seed_query = RestApiQuerySeed(seed_record=seed_record) # Spaces params = { 'auth': HTTPBasicAuth(conf.get_string(MODE_ACCESS_TOKEN), conf.get_string(MODE_PASSWORD_TOKEN)) } json_path = '_embedded.spaces[*].[token,name,description]' field_names = [ 'dashboard_group_id', 'dashboard_group', 'dashboard_group_description' ] spaces_query = RestApiQuery(query_to_join=seed_query, url=spaces_url_template, params=params, json_path=json_path, field_names=field_names) return spaces_query
def _build_restapi_query(self) -> RestApiQuery: """ Build a paginated REST API based on Mode discovery API :return: """ params = ModeDashboardUtils.get_auth_params(conf=self._conf, discover_auth=True) seed_record = [{ 'organization': self._conf.get_string(ORGANIZATION), 'is_active': None, 'updated_at': None, 'do_not_update_empty_attribute': True, }] seed_query = RestApiQuerySeed(seed_record=seed_record) chart_url_template = 'http://app.mode.com/batch/{organization}/charts' if self._conf.get_bool( ModeDashboardChartsBatchExtractor.INCLUDE_ALL_SPACE, default=False): chart_url_template += '?include_spaces=all' json_path = '(charts[*].[space_token,report_token,query_token,token,chart_title,chart_type])' field_names = [ 'dashboard_group_id', 'dashboard_id', 'query_id', 'chart_id', 'chart_name', 'chart_type' ] chart_batch_query = ModePaginatedRestApiQuery( query_to_join=seed_query, url=chart_url_template, params=params, json_path=json_path, pagination_json_path=json_path, field_names=field_names, skip_no_result=True) return chart_batch_query
def test_ensure_record_get_updated(self) -> None: query_to_merge_seed_record = [ { 'organization': 'amundsen', 'dashboard_id': 'd1' }, { 'organization': 'amundsen-databuilder', 'dashboard_id': 'd2' }, { 'organization': 'amundsen-dashboard', 'dashboard_id': 'd3' }, ] query_to_merge = RestApiQuerySeed( seed_record=query_to_merge_seed_record) query_merger = QueryMerger(query_to_merge=query_to_merge, merge_key='dashboard_id') with patch('databuilder.rest_api.rest_api_query.requests.get' ) as mock_get: mock_get.return_value.json.side_effect = [ { 'foo': { 'name': 'john' } }, { 'foo': { 'name': 'doe' } }, ] query = RestApiQuery(query_to_join=self.query_to_join, url=self.url, params={}, json_path=self.json_path, field_names=self.field_names, query_merger=query_merger) results = list(query.execute()) self.assertEqual(len(results), 2) self.assertDictEqual( { 'dashboard_id': 'd1', 'foo1': 'bar1', 'name_field': 'john', 'organization': 'amundsen' }, results[0], ) self.assertDictEqual( { 'dashboard_id': 'd3', 'foo2': 'bar2', 'name_field': 'doe', 'organization': 'amundsen-dashboard' }, results[1], )
def _build_restapi_query(self): """ Build REST API Query. To get Mode Dashboard owner, it needs to call three APIs (spaces API, reports API, and user API) joining together. :return: A RestApiQuery that provides Mode Dashboard owner """ # type: () -> RestApiQuery # Seed query record for next query api to join with seed_record = [{ 'organization': self._conf.get_string(ORGANIZATION), 'is_active': None, 'updated_at': None, 'do_not_update_empty_attribute': True, }] seed_query = RestApiQuerySeed(seed_record=seed_record) # memberships # https://mode.com/developer/api-reference/management/organization-memberships/#listMemberships memberships_url_template = 'https://app.mode.com/api/{organization}/memberships' params = { 'auth': HTTPBasicAuth(self._conf.get_string(MODE_ACCESS_TOKEN), self._conf.get_string(MODE_PASSWORD_TOKEN)) } json_path = '(_embedded.memberships[*].member_username) | (_embedded.memberships[*]._links.user.href)' field_names = ['mode_user_id', 'mode_user_resource_path'] mode_user_ids_query = RestApiQuery(query_to_join=seed_query, url=memberships_url_template, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, json_path_contains_or=True) # https://mode.com/developer/api-reference/management/users/ user_url_template = 'https://app.mode.com{mode_user_resource_path}' json_path = 'email' field_names = ['email'] failure_handler = HttpFailureSkipOnStatus(status_codes_to_skip={404}) mode_user_email_query = RestApiQuery( query_to_join=mode_user_ids_query, url=user_url_template, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, can_skip_failure=failure_handler.can_skip_failure) return mode_user_email_query
def setUp(self) -> None: query_to_join_seed_record = [{ 'foo1': 'bar1', 'dashboard_id': 'd1' }, { 'foo2': 'bar2', 'dashboard_id': 'd3' }] self.query_to_join = RestApiQuerySeed( seed_record=query_to_join_seed_record) self.json_path = 'foo.name' self.field_names = ['name_field'] self.url = 'foobar'
def test_static_data(self) -> None: conf = ConfigFactory.from_dict( { REST_API_QUERY: RestApiQuerySeed(seed_record=[{'foo': 'bar'}]), STATIC_RECORD_DICT: {'john': 'doe'} } ) extractor = RestAPIExtractor() extractor.init(conf=conf) record = extractor.extract() expected = {'foo': 'bar', 'john': 'doe'} self.assertDictEqual(expected, record)
def test_exception_rasied_with_duplicate_merge_key(self) -> None: """ Two records in query_to_merge results have {'dashboard_id': 'd2'}, exception should be raised """ query_to_merge_seed_record = [ { 'organization': 'amundsen', 'dashboard_id': 'd1' }, { 'organization': 'amundsen-databuilder', 'dashboard_id': 'd2' }, { 'organization': 'amundsen-dashboard', 'dashboard_id': 'd2' }, ] query_to_merge = RestApiQuerySeed( seed_record=query_to_merge_seed_record) query_merger = QueryMerger(query_to_merge=query_to_merge, merge_key='dashboard_id') with patch('databuilder.rest_api.rest_api_query.requests.get' ) as mock_get: mock_get.return_value.json.side_effect = [ { 'foo': { 'name': 'john' } }, { 'foo': { 'name': 'doe' } }, ] query = RestApiQuery(query_to_join=self.query_to_join, url=self.url, params={}, json_path=self.json_path, field_names=self.field_names, query_merger=query_merger) self.assertRaises(Exception, query.execute())
def test_model_construction(self) -> None: conf = ConfigFactory.from_dict( { REST_API_QUERY: RestApiQuerySeed( seed_record=[{'dashboard_group': 'foo', 'dashboard_name': 'bar', 'description': 'john', 'dashboard_group_description': 'doe'}]), MODEL_CLASS: 'databuilder.models.dashboard.dashboard_metadata.DashboardMetadata', } ) extractor = RestAPIExtractor() extractor.init(conf=conf) record = extractor.extract() expected = DashboardMetadata(dashboard_group='foo', dashboard_name='bar', description='john', dashboard_group_description='doe') self.assertEqual(expected.__repr__(), record.__repr__())
def test_rest_api_query_multiple_fields(self): seed_record = [{'foo1': 'bar1'}, {'foo2': 'bar2'}] seed_query = RestApiQuerySeed(seed_record=seed_record) with patch('databuilder.rest_api.rest_api_query.requests.get' ) as mock_get: json_path = 'foo.[name,hobby]' field_names = ['name_field', 'hobby'] mock_get.return_value.json.side_effect = [ { 'foo': { 'name': 'john', 'hobby': 'skiing' } }, { 'foo': { 'name': 'doe', 'hobby': 'snowboarding' } }, ] query = RestApiQuery(query_to_join=seed_query, url='foobar', params={}, json_path=json_path, field_names=field_names) expected = [{ 'name_field': 'john', 'hobby': 'skiing', 'foo1': 'bar1' }, { 'name_field': 'doe', 'hobby': 'snowboarding', 'foo2': 'bar2' }] for actual in query.execute(): self.assertDictEqual(expected.pop(0), actual)
def get_spaces_query_api(conf: ConfigTree) -> BaseRestApiQuery: """ Provides RestApiQuerySeed where it will provides iterator of dictionaries as records where dictionary keys are organization, dashboard_group_id, dashboard_group and dashboard_group_description :param conf: :return: """ # https://mode.com/developer/discovery-api/analytics/spaces spaces_url_template = 'https://app.mode.com/batch/{organization}/spaces' # Seed query record for next query api to join with seed_record = [{'organization': conf.get_string(ORGANIZATION)}] seed_query = RestApiQuerySeed(seed_record=seed_record) # mode_bearer_token must be provided in the conf # the token is required to access discovery endpoint # https://mode.com/developer/discovery-api/introduction/ params = ModeDashboardUtils.get_auth_params(conf=conf, discover_auth=True) json_path = 'spaces[*].[token,name,description]' field_names = [ 'dashboard_group_id', 'dashboard_group', 'dashboard_group_description' ] # based on https://mode.com/developer/discovery-api/analytics/spaces/#listSpacesForAccount pagination_json_path = 'spaces[*]' max_per_page = 1000 spaces_query = ModePaginatedRestApiQuery( pagination_json_path=pagination_json_path, max_record_size=max_per_page, query_to_join=seed_query, url=spaces_url_template, params=params, json_path=json_path, field_names=field_names) return spaces_query
def test_no_pagination(self) -> None: seed_record = [{'foo1': 'bar1'}, {'foo2': 'bar2'}, {'foo3': 'bar3'}] seed_query = RestApiQuerySeed(seed_record=seed_record) with patch('databuilder.rest_api.rest_api_query.requests.get' ) as mock_get: json_path = 'foo[*].name' field_names = ['name_field'] mock_get.return_value.json.side_effect = [ # need to duplicate for json() is called twice { 'foo': [{ 'name': 'v1' }, { 'name': 'v2' }] }, { 'foo': [{ 'name': 'v1' }, { 'name': 'v2' }] }, { 'foo': [{ 'name': 'v3' }] }, { 'foo': [{ 'name': 'v3' }] }, { 'foo': [{ 'name': 'v4' }, { 'name': 'v5' }] }, { 'foo': [{ 'name': 'v4' }, { 'name': 'v5' }] }, ] query = ModePaginatedRestApiQuery(query_to_join=seed_query, url='foobar', params={}, json_path=json_path, field_names=field_names, pagination_json_path='foo[*]', max_record_size=3) expected_list = [{ 'name_field': 'v1', 'foo1': 'bar1' }, { 'name_field': 'v2', 'foo1': 'bar1' }, { 'name_field': 'v3', 'foo2': 'bar2' }, { 'name_field': 'v4', 'foo3': 'bar3' }, { 'name_field': 'v5', 'foo3': 'bar3' }] for actual in query.execute(): self.assertDictEqual(actual, expected_list.pop(0)) self.assertEqual(mock_get.call_count, 3) calls = [call('foobar?page=1')] mock_get.assert_has_calls(calls, any_order=True)
def get_seed_query(conf: ConfigTree) -> BaseRestApiQuery: # Seed query record for next query api to join with seed_record = [{'organization': conf.get_string(ORGANIZATION)}] seed_query = RestApiQuerySeed(seed_record=seed_record) return seed_query