def test_rest_api_query_seed(self):
rest_api_query = RestApiQuerySeed(seed_record=[{
'foo': 'bar'
}, {
'john': 'doe'
}])
result = [v for v in rest_api_query.execute()]
expected = [{'foo': 'bar'}, {'john': 'doe'}]
self.assertListEqual(expected, result)
def get_spaces_query_api(conf: ConfigTree) -> BaseRestApiQuery:
"""
Provides RestApiQuerySeed where it will provides iterator of dictionaries as records where dictionary keys are
organization, dashboard_group_id, dashboard_group and dashboard_group_description
:param conf:
:return:
"""
# https://mode.com/developer/api-reference/management/spaces/#listSpaces
spaces_url_template = 'https://app.mode.com/api/{organization}/spaces?filter=all'
# Seed query record for next query api to join with
seed_record = [{'organization': conf.get_string(ORGANIZATION)}]
seed_query = RestApiQuerySeed(seed_record=seed_record)
# Spaces
params = {
'auth':
HTTPBasicAuth(conf.get_string(MODE_ACCESS_TOKEN),
conf.get_string(MODE_PASSWORD_TOKEN))
}
json_path = '_embedded.spaces[*].[token,name,description]'
field_names = [
'dashboard_group_id', 'dashboard_group',
'dashboard_group_description'
]
spaces_query = RestApiQuery(query_to_join=seed_query,
url=spaces_url_template,
params=params,
json_path=json_path,
field_names=field_names)
return spaces_query
def _build_restapi_query(self) -> RestApiQuery:
"""
Build a paginated REST API based on Mode discovery API
:return:
"""
params = ModeDashboardUtils.get_auth_params(conf=self._conf,
discover_auth=True)
seed_record = [{
'organization': self._conf.get_string(ORGANIZATION),
'is_active': None,
'updated_at': None,
'do_not_update_empty_attribute': True,
}]
seed_query = RestApiQuerySeed(seed_record=seed_record)
chart_url_template = 'http://app.mode.com/batch/{organization}/charts'
if self._conf.get_bool(
ModeDashboardChartsBatchExtractor.INCLUDE_ALL_SPACE,
default=False):
chart_url_template += '?include_spaces=all'
json_path = '(charts[*].[space_token,report_token,query_token,token,chart_title,chart_type])'
field_names = [
'dashboard_group_id', 'dashboard_id', 'query_id', 'chart_id',
'chart_name', 'chart_type'
]
chart_batch_query = ModePaginatedRestApiQuery(
query_to_join=seed_query,
url=chart_url_template,
params=params,
json_path=json_path,
pagination_json_path=json_path,
field_names=field_names,
skip_no_result=True)
return chart_batch_query
def test_ensure_record_get_updated(self) -> None:
query_to_merge_seed_record = [
{
'organization': 'amundsen',
'dashboard_id': 'd1'
},
{
'organization': 'amundsen-databuilder',
'dashboard_id': 'd2'
},
{
'organization': 'amundsen-dashboard',
'dashboard_id': 'd3'
},
]
query_to_merge = RestApiQuerySeed(
seed_record=query_to_merge_seed_record)
query_merger = QueryMerger(query_to_merge=query_to_merge,
merge_key='dashboard_id')
with patch('databuilder.rest_api.rest_api_query.requests.get'
) as mock_get:
mock_get.return_value.json.side_effect = [
{
'foo': {
'name': 'john'
}
},
{
'foo': {
'name': 'doe'
}
},
]
query = RestApiQuery(query_to_join=self.query_to_join,
url=self.url,
params={},
json_path=self.json_path,
field_names=self.field_names,
query_merger=query_merger)
results = list(query.execute())
self.assertEqual(len(results), 2)
self.assertDictEqual(
{
'dashboard_id': 'd1',
'foo1': 'bar1',
'name_field': 'john',
'organization': 'amundsen'
},
results[0],
)
self.assertDictEqual(
{
'dashboard_id': 'd3',
'foo2': 'bar2',
'name_field': 'doe',
'organization': 'amundsen-dashboard'
},
results[1],
)
def _build_restapi_query(self):
"""
Build REST API Query. To get Mode Dashboard owner, it needs to call three APIs (spaces API, reports
API, and user API) joining together.
:return: A RestApiQuery that provides Mode Dashboard owner
"""
# type: () -> RestApiQuery
# Seed query record for next query api to join with
seed_record = [{
'organization': self._conf.get_string(ORGANIZATION),
'is_active': None,
'updated_at': None,
'do_not_update_empty_attribute': True,
}]
seed_query = RestApiQuerySeed(seed_record=seed_record)
# memberships
# https://mode.com/developer/api-reference/management/organization-memberships/#listMemberships
memberships_url_template = 'https://app.mode.com/api/{organization}/memberships'
params = {
'auth':
HTTPBasicAuth(self._conf.get_string(MODE_ACCESS_TOKEN),
self._conf.get_string(MODE_PASSWORD_TOKEN))
}
json_path = '(_embedded.memberships[*].member_username) | (_embedded.memberships[*]._links.user.href)'
field_names = ['mode_user_id', 'mode_user_resource_path']
mode_user_ids_query = RestApiQuery(query_to_join=seed_query,
url=memberships_url_template,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True,
json_path_contains_or=True)
# https://mode.com/developer/api-reference/management/users/
user_url_template = 'https://app.mode.com{mode_user_resource_path}'
json_path = 'email'
field_names = ['email']
failure_handler = HttpFailureSkipOnStatus(status_codes_to_skip={404})
mode_user_email_query = RestApiQuery(
query_to_join=mode_user_ids_query,
url=user_url_template,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True,
can_skip_failure=failure_handler.can_skip_failure)
return mode_user_email_query
def setUp(self) -> None:
query_to_join_seed_record = [{
'foo1': 'bar1',
'dashboard_id': 'd1'
}, {
'foo2': 'bar2',
'dashboard_id': 'd3'
}]
self.query_to_join = RestApiQuerySeed(
seed_record=query_to_join_seed_record)
self.json_path = 'foo.name'
self.field_names = ['name_field']
self.url = 'foobar'
def test_static_data(self) -> None:
conf = ConfigFactory.from_dict(
{
REST_API_QUERY: RestApiQuerySeed(seed_record=[{'foo': 'bar'}]),
STATIC_RECORD_DICT: {'john': 'doe'}
}
)
extractor = RestAPIExtractor()
extractor.init(conf=conf)
record = extractor.extract()
expected = {'foo': 'bar', 'john': 'doe'}
self.assertDictEqual(expected, record)
def test_exception_rasied_with_duplicate_merge_key(self) -> None:
"""
Two records in query_to_merge results have {'dashboard_id': 'd2'},
exception should be raised
"""
query_to_merge_seed_record = [
{
'organization': 'amundsen',
'dashboard_id': 'd1'
},
{
'organization': 'amundsen-databuilder',
'dashboard_id': 'd2'
},
{
'organization': 'amundsen-dashboard',
'dashboard_id': 'd2'
},
]
query_to_merge = RestApiQuerySeed(
seed_record=query_to_merge_seed_record)
query_merger = QueryMerger(query_to_merge=query_to_merge,
merge_key='dashboard_id')
with patch('databuilder.rest_api.rest_api_query.requests.get'
) as mock_get:
mock_get.return_value.json.side_effect = [
{
'foo': {
'name': 'john'
}
},
{
'foo': {
'name': 'doe'
}
},
]
query = RestApiQuery(query_to_join=self.query_to_join,
url=self.url,
params={},
json_path=self.json_path,
field_names=self.field_names,
query_merger=query_merger)
self.assertRaises(Exception, query.execute())
def test_model_construction(self) -> None:
conf = ConfigFactory.from_dict(
{
REST_API_QUERY: RestApiQuerySeed(
seed_record=[{'dashboard_group': 'foo',
'dashboard_name': 'bar',
'description': 'john',
'dashboard_group_description': 'doe'}]),
MODEL_CLASS: 'databuilder.models.dashboard.dashboard_metadata.DashboardMetadata',
}
)
extractor = RestAPIExtractor()
extractor.init(conf=conf)
record = extractor.extract()
expected = DashboardMetadata(dashboard_group='foo', dashboard_name='bar', description='john',
dashboard_group_description='doe')
self.assertEqual(expected.__repr__(), record.__repr__())
def test_rest_api_query_multiple_fields(self):
seed_record = [{'foo1': 'bar1'}, {'foo2': 'bar2'}]
seed_query = RestApiQuerySeed(seed_record=seed_record)
with patch('databuilder.rest_api.rest_api_query.requests.get'
) as mock_get:
json_path = 'foo.[name,hobby]'
field_names = ['name_field', 'hobby']
mock_get.return_value.json.side_effect = [
{
'foo': {
'name': 'john',
'hobby': 'skiing'
}
},
{
'foo': {
'name': 'doe',
'hobby': 'snowboarding'
}
},
]
query = RestApiQuery(query_to_join=seed_query,
url='foobar',
params={},
json_path=json_path,
field_names=field_names)
expected = [{
'name_field': 'john',
'hobby': 'skiing',
'foo1': 'bar1'
}, {
'name_field': 'doe',
'hobby': 'snowboarding',
'foo2': 'bar2'
}]
for actual in query.execute():
self.assertDictEqual(expected.pop(0), actual)
def get_spaces_query_api(conf: ConfigTree) -> BaseRestApiQuery:
"""
Provides RestApiQuerySeed where it will provides iterator of dictionaries as records where dictionary keys are
organization, dashboard_group_id, dashboard_group and dashboard_group_description
:param conf:
:return:
"""
# https://mode.com/developer/discovery-api/analytics/spaces
spaces_url_template = 'https://app.mode.com/batch/{organization}/spaces'
# Seed query record for next query api to join with
seed_record = [{'organization': conf.get_string(ORGANIZATION)}]
seed_query = RestApiQuerySeed(seed_record=seed_record)
# mode_bearer_token must be provided in the conf
# the token is required to access discovery endpoint
# https://mode.com/developer/discovery-api/introduction/
params = ModeDashboardUtils.get_auth_params(conf=conf,
discover_auth=True)
json_path = 'spaces[*].[token,name,description]'
field_names = [
'dashboard_group_id', 'dashboard_group',
'dashboard_group_description'
]
# based on https://mode.com/developer/discovery-api/analytics/spaces/#listSpacesForAccount
pagination_json_path = 'spaces[*]'
max_per_page = 1000
spaces_query = ModePaginatedRestApiQuery(
pagination_json_path=pagination_json_path,
max_record_size=max_per_page,
query_to_join=seed_query,
url=spaces_url_template,
params=params,
json_path=json_path,
field_names=field_names)
return spaces_query
def test_no_pagination(self) -> None:
seed_record = [{'foo1': 'bar1'}, {'foo2': 'bar2'}, {'foo3': 'bar3'}]
seed_query = RestApiQuerySeed(seed_record=seed_record)
with patch('databuilder.rest_api.rest_api_query.requests.get'
) as mock_get:
json_path = 'foo[*].name'
field_names = ['name_field']
mock_get.return_value.json.side_effect = [ # need to duplicate for json() is called twice
{
'foo': [{
'name': 'v1'
}, {
'name': 'v2'
}]
},
{
'foo': [{
'name': 'v1'
}, {
'name': 'v2'
}]
},
{
'foo': [{
'name': 'v3'
}]
},
{
'foo': [{
'name': 'v3'
}]
},
{
'foo': [{
'name': 'v4'
}, {
'name': 'v5'
}]
},
{
'foo': [{
'name': 'v4'
}, {
'name': 'v5'
}]
},
]
query = ModePaginatedRestApiQuery(query_to_join=seed_query,
url='foobar',
params={},
json_path=json_path,
field_names=field_names,
pagination_json_path='foo[*]',
max_record_size=3)
expected_list = [{
'name_field': 'v1',
'foo1': 'bar1'
}, {
'name_field': 'v2',
'foo1': 'bar1'
}, {
'name_field': 'v3',
'foo2': 'bar2'
}, {
'name_field': 'v4',
'foo3': 'bar3'
}, {
'name_field': 'v5',
'foo3': 'bar3'
}]
for actual in query.execute():
self.assertDictEqual(actual, expected_list.pop(0))
self.assertEqual(mock_get.call_count, 3)
calls = [call('foobar?page=1')]
mock_get.assert_has_calls(calls, any_order=True)
def get_seed_query(conf: ConfigTree) -> BaseRestApiQuery:
# Seed query record for next query api to join with
seed_record = [{'organization': conf.get_string(ORGANIZATION)}]
seed_query = RestApiQuerySeed(seed_record=seed_record)
return seed_query
