def test_ensure_record_get_updated(self) -> None:
query_to_merge_seed_record = [
{
'organization': 'amundsen',
'dashboard_id': 'd1'
},
{
'organization': 'amundsen-databuilder',
'dashboard_id': 'd2'
},
{
'organization': 'amundsen-dashboard',
'dashboard_id': 'd3'
},
]
query_to_merge = RestApiQuerySeed(
seed_record=query_to_merge_seed_record)
query_merger = QueryMerger(query_to_merge=query_to_merge,
merge_key='dashboard_id')
with patch('databuilder.rest_api.rest_api_query.requests.get'
) as mock_get:
mock_get.return_value.json.side_effect = [
{
'foo': {
'name': 'john'
}
},
{
'foo': {
'name': 'doe'
}
},
]
query = RestApiQuery(query_to_join=self.query_to_join,
url=self.url,
params={},
json_path=self.json_path,
field_names=self.field_names,
query_merger=query_merger)
results = list(query.execute())
self.assertEqual(len(results), 2)
self.assertDictEqual(
{
'dashboard_id': 'd1',
'foo1': 'bar1',
'name_field': 'john',
'organization': 'amundsen'
},
results[0],
)
self.assertDictEqual(
{
'dashboard_id': 'd3',
'foo2': 'bar2',
'name_field': 'doe',
'organization': 'amundsen-dashboard'
},
results[1],
)
def _build_restapi_query(self) -> RestApiQuery:
"""
Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports
API, and run API) joining together.
:return: A RestApiQuery that provides Mode Dashboard execution (run)
"""
spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf)
# Reports
# https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
report_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
json_path = '(_embedded.reports[*].token)'
field_names = ['dashboard_id']
reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params,
json_path=json_path, field_names=field_names, skip_no_result=True)
queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries'
json_path = '_embedded.queries[*].[token,name]'
field_names = ['query_id', 'query_name']
query_names_query = RestApiQuery(query_to_join=reports_query, url=queries_url_template, params=params,
json_path=json_path, field_names=field_names, skip_no_result=True)
charts_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries/{query_id}/charts'
json_path = '(_embedded.charts[*].token) | (_embedded.charts[*]._links.report_viz_web.href)'
field_names = ['chart_id', 'chart_url']
chart_names_query = RestApiQuery(query_to_join=query_names_query, url=charts_url_template, params=params,
json_path=json_path, field_names=field_names, skip_no_result=True,
json_path_contains_or=True)
return chart_names_query
def _build_restapi_query(self):
"""
Build REST API Query. To get Mode Dashboard owner, it needs to call three APIs (spaces API, reports
API, and user API) joining together.
:return: A RestApiQuery that provides Mode Dashboard owner
"""
# type: () -> RestApiQuery
# Seed query record for next query api to join with
seed_record = [{
'organization': self._conf.get_string(ORGANIZATION),
'is_active': None,
'updated_at': None,
'do_not_update_empty_attribute': True,
}]
seed_query = RestApiQuerySeed(seed_record=seed_record)
# memberships
# https://mode.com/developer/api-reference/management/organization-memberships/#listMemberships
memberships_url_template = 'https://app.mode.com/api/{organization}/memberships'
params = {
'auth':
HTTPBasicAuth(self._conf.get_string(MODE_ACCESS_TOKEN),
self._conf.get_string(MODE_PASSWORD_TOKEN))
}
json_path = '(_embedded.memberships[*].member_username) | (_embedded.memberships[*]._links.user.href)'
field_names = ['mode_user_id', 'mode_user_resource_path']
mode_user_ids_query = RestApiQuery(query_to_join=seed_query,
url=memberships_url_template,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True,
json_path_contains_or=True)
# https://mode.com/developer/api-reference/management/users/
user_url_template = 'https://app.mode.com{mode_user_resource_path}'
json_path = 'email'
field_names = ['email']
failure_handler = HttpFailureSkipOnStatus(status_codes_to_skip={404})
mode_user_email_query = RestApiQuery(
query_to_join=mode_user_ids_query,
url=user_url_template,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True,
can_skip_failure=failure_handler.can_skip_failure)
return mode_user_email_query
def test_compute_subresult_single_field(self) -> None:
sub_records = RestApiQuery._compute_sub_records(result_list=['1', '2', '3'], field_names=['foo'])
expected_records = [
['1'], ['2'], ['3']
]
assert expected_records == sub_records
sub_records = RestApiQuery._compute_sub_records(result_list=['1', '2', '3'], field_names=['foo'],
json_path_contains_or=True)
assert expected_records == sub_records
def test_compute_subresult_multiple_fields_json_path_and_expression(self):
sub_records = RestApiQuery._compute_sub_records(
result_list=['1', 'a', '2', 'b', '3', 'c'],
field_names=['foo', 'bar'])
expected_records = [['1', 'a'], ['2', 'b'], ['3', 'c']]
assert expected_records == sub_records
sub_records = RestApiQuery._compute_sub_records(
result_list=['1', 'a', 'x', '2', 'b', 'y', '3', 'c', 'z'],
field_names=['foo', 'bar', 'baz'])
expected_records = [['1', 'a', 'x'], ['2', 'b', 'y'], ['3', 'c', 'z']]
assert expected_records == sub_records
def get_spaces_query_api(conf: ConfigTree) -> BaseRestApiQuery:
"""
Provides RestApiQuerySeed where it will provides iterator of dictionaries as records where dictionary keys are
organization, dashboard_group_id, dashboard_group and dashboard_group_description
:param conf:
:return:
"""
# https://mode.com/developer/api-reference/management/spaces/#listSpaces
spaces_url_template = 'https://app.mode.com/api/{organization}/spaces?filter=all'
# Seed query record for next query api to join with
seed_record = [{'organization': conf.get_string(ORGANIZATION)}]
seed_query = RestApiQuerySeed(seed_record=seed_record)
# Spaces
params = {
'auth':
HTTPBasicAuth(conf.get_string(MODE_ACCESS_TOKEN),
conf.get_string(MODE_PASSWORD_TOKEN))
}
json_path = '_embedded.spaces[*].[token,name,description]'
field_names = [
'dashboard_group_id', 'dashboard_group',
'dashboard_group_description'
]
spaces_query = RestApiQuery(query_to_join=seed_query,
url=spaces_url_template,
params=params,
json_path=json_path,
field_names=field_names)
return spaces_query
def _build_restapi_query(self) -> RestApiQuery:
dashes_query = RedashPaginatedRestApiQuery(
query_to_join=EmptyRestApiQuerySeed(),
url=f'{self._api_base_url}/dashboards',
params=self._get_default_api_query_params(),
json_path=
'results[*].[id,name,slug,created_at,updated_at,is_archived,is_draft,user]',
field_names=[
'dashboard_id', 'dashboard_name', 'slug', 'created_timestamp',
'last_modified_timestamp', 'is_archived', 'is_draft', 'user'
],
skip_no_result=True)
if self._redash_version >= 9:
dashboard_url = f'{self._api_base_url}/dashboards/{{dashboard_id}}'
else:
dashboard_url = f'{self._api_base_url}/dashboards/{{slug}}'
return RestApiQuery(query_to_join=dashes_query,
url=dashboard_url,
params=self._get_default_api_query_params(),
json_path='widgets',
field_names=['widgets'],
skip_no_result=True)
def _build_restapi_query(self) -> RestApiQuery:
databricks_sql_dashboard_query = DatabricksSQLPaginatedRestApiQuery(
query_to_join=EmptyRestApiQuerySeed(),
url=self._databricks_sql_dashboards_api_base,
params={"headers": self._get_databrick_request_headers()},
json_path="results[*].[id,name,tags,updated_at,created_at,user]",
field_names=[
"dashboard_id",
"dashboard_name",
"tags",
"last_modified_timestamp",
"created_timestamp",
"user",
],
skip_no_results=True,
)
return RestApiQuery(
query_to_join=databricks_sql_dashboard_query,
url=f"{self._databricks_sql_dashboards_api_base}/{{dashboard_id}}",
params={"headers": self._get_databrick_request_headers()},
json_path="widgets",
field_names=["widgets"],
skip_no_result=True,
)
def test_exception_rasied_with_duplicate_merge_key(self) -> None:
"""
Two records in query_to_merge results have {'dashboard_id': 'd2'},
exception should be raised
"""
query_to_merge_seed_record = [
{
'organization': 'amundsen',
'dashboard_id': 'd1'
},
{
'organization': 'amundsen-databuilder',
'dashboard_id': 'd2'
},
{
'organization': 'amundsen-dashboard',
'dashboard_id': 'd2'
},
]
query_to_merge = RestApiQuerySeed(
seed_record=query_to_merge_seed_record)
query_merger = QueryMerger(query_to_merge=query_to_merge,
merge_key='dashboard_id')
with patch('databuilder.rest_api.rest_api_query.requests.get'
) as mock_get:
mock_get.return_value.json.side_effect = [
{
'foo': {
'name': 'john'
}
},
{
'foo': {
'name': 'doe'
}
},
]
query = RestApiQuery(query_to_join=self.query_to_join,
url=self.url,
params={},
json_path=self.json_path,
field_names=self.field_names,
query_merger=query_merger)
self.assertRaises(Exception, query.execute())
def test_compute_subresult_multiple_fields_json_path_or_expression(self):
sub_records = RestApiQuery._compute_sub_records(
result_list=['1', '2', '3', 'a', 'b', 'c'],
field_names=['foo', 'bar'],
json_path_contains_or=True)
expected_records = [['1', 'a'], ['2', 'b'], ['3', 'c']]
self.assertEqual(expected_records, sub_records)
sub_records = RestApiQuery._compute_sub_records(
result_list=['1', '2', '3', 'a', 'b', 'c', 'x', 'y', 'z'],
field_names=['foo', 'bar', 'baz'],
json_path_contains_or=True)
expected_records = [['1', 'a', 'x'], ['2', 'b', 'y'], ['3', 'c', 'z']]
self.assertEqual(expected_records, sub_records)
def test_rest_api_query_multiple_fields(self):
seed_record = [{'foo1': 'bar1'}, {'foo2': 'bar2'}]
seed_query = RestApiQuerySeed(seed_record=seed_record)
with patch('databuilder.rest_api.rest_api_query.requests.get'
) as mock_get:
json_path = 'foo.[name,hobby]'
field_names = ['name_field', 'hobby']
mock_get.return_value.json.side_effect = [
{
'foo': {
'name': 'john',
'hobby': 'skiing'
}
},
{
'foo': {
'name': 'doe',
'hobby': 'snowboarding'
}
},
]
query = RestApiQuery(query_to_join=seed_query,
url='foobar',
params={},
json_path=json_path,
field_names=field_names)
expected = [{
'name_field': 'john',
'hobby': 'skiing',
'foo1': 'bar1'
}, {
'name_field': 'doe',
'hobby': 'snowboarding',
'foo2': 'bar2'
}]
for actual in query.execute():
self.assertDictEqual(expected.pop(0), actual)
def _build_restapi_query(self):
"""
Build REST API Query. To get Mode Dashboard owner, it needs to call three APIs (spaces API, reports
API, and user API) joining together.
:return: A RestApiQuery that provides Mode Dashboard owner
"""
# type: () -> RestApiQuery
# https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
report_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
# https://mode.com/developer/api-reference/management/users/
creator_url_template = 'https://app.mode.com{creator_resource_path}'
spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf)
# Reports
json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.creator.href)'
field_names = ['dashboard_id', 'creator_resource_path']
creator_resource_path_query = RestApiQuery(query_to_join=spaces_query,
url=report_url_template,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True,
json_path_contains_or=True)
json_path = 'email'
field_names = ['email']
failure_handler = HttpFailureSkipOnStatus(status_codes_to_skip={404})
owner_email_query = RestApiQuery(
query_to_join=creator_resource_path_query,
url=creator_url_template,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True,
can_skip_failure=failure_handler.can_skip_failure)
return owner_email_query
def _build_restapi_query(self):
"""
Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports
API, and run API) joining together.
:return: A RestApiQuery that provides Mode Dashboard execution (run)
"""
# type: () -> RestApiQuery
spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf)
# Reports
# https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.last_run.href)'
field_names = ['dashboard_id', 'last_run_resource_path']
last_run_resource_path_query = RestApiQuery(query_to_join=spaces_query,
url=url,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True,
json_path_contains_or=True)
# https://mode.com/developer/api-reference/analytics/report-runs/#getReportRun
url = 'https://app.mode.com{last_run_resource_path}'
json_path = '[state,completed_at]'
field_names = ['execution_state', 'execution_timestamp']
last_run_state_query = RestApiQuery(
query_to_join=last_run_resource_path_query,
url=url,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True)
return last_run_state_query
def _build_restapi_query(self):
"""
Build REST API Query. To get Mode Dashboard usage, it needs to call two APIs (spaces API and reports
API) joining together.
:return: A RestApiQuery that provides Mode Dashboard metadata
"""
# type: () -> RestApiQuery
# https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
reports_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf)
# Reports
# JSONPATH expression. it goes into array which is located in _embedded.reports and then extracts token,
# and view_count
json_path = '_embedded.reports[*].[token,view_count]'
field_names = ['dashboard_id', 'accumulated_view_count']
reports_query = RestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params,
json_path=json_path, field_names=field_names, skip_no_result=True)
return reports_query
def _build_restapi_query(self):
# type: () -> RestApiQuery
dashes_query = RedashPaginatedRestApiQuery(
query_to_join=EmptyRestApiQuerySeed(),
url='{redash_api}/dashboards'.format(redash_api=self._api_base_url),
params=self._get_default_api_query_params(),
json_path='results[*].[id,name,slug,created_at,updated_at,is_archived,is_draft,user]',
field_names=[
'dashboard_id', 'dashboard_name', 'slug', 'created_timestamp',
'last_modified_timestamp', 'is_archived', 'is_draft', 'user'
],
skip_no_result=True
)
return RestApiQuery(
query_to_join=dashes_query,
url='{redash_api}/dashboards/{{slug}}'.format(redash_api=self._api_base_url),
params=self._get_default_api_query_params(),
json_path='widgets',
field_names=['widgets'],
skip_no_result=True
)
def _build_restapi_query(self):
"""
Build REST API Query. To get Mode Dashboard last modified timestamp, it needs to call two APIs (spaces API,
and reports API) joining together.
:return: A RestApiQuery that provides Mode Dashboard last successful execution (run)
"""
# type: () -> RestApiQuery
spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf)
# Reports
# https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
json_path = '_embedded.reports[*].[token,edited_at]'
field_names = ['dashboard_id', 'last_modified_timestamp']
last_modified_query = RestApiQuery(query_to_join=spaces_query,
url=url,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True)
return last_modified_query