def _build_restapi_query(self):
"""
Build REST API Query. To get Mode Dashboard last successful execution, it needs to call two APIs (spaces API,
and reports API) joining together.
:return: A RestApiQuery that provides Mode Dashboard last successful execution (run)
"""
# type: () -> RestApiQuery
spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf)
# Reports
# https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
json_path = '_embedded.reports[*].[token,last_successfully_run_at]'
field_names = ['dashboard_id', 'execution_timestamp']
last_successful_run_query = ModePaginatedRestApiQuery(
query_to_join=spaces_query,
url=url,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True)
return last_successful_run_query
def _build_restapi_query(self) -> RestApiQuery:
"""
Build REST API Query. To get Mode Dashboard metadata, it needs to call two APIs (spaces API and reports
API) joining together.
:return: A RestApiQuery that provides Mode Dashboard metadata
"""
# https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
reports_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf)
# Reports
# JSONPATH expression. it goes into array which is located in _embedded.reports and then extracts token, name,
# and description
json_path = '_embedded.reports[*].[token,name,description,created_at]'
field_names = [
'dashboard_id', 'dashboard_name', 'description',
'created_timestamp'
]
reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query,
url=reports_url_template,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True)
return reports_query
def _build_restapi_query(self) -> ModePaginatedRestApiQuery:
"""
Build REST API Query to get Mode Dashboard last successful execution metadata.
:return: A RestApiQuery that provides Mode Dashboard last successful execution (run)
"""
seed_query = ModeDashboardUtils.get_seed_query(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf,
discover_auth=True)
# Reports
# https://mode.com/developer/discovery-api/analytics/reports/
url = 'https://app.mode.com/batch/{organization}/reports'
json_path = 'reports[*].[token, space_token, last_successfully_run_at]'
field_names = [
'dashboard_id', 'dashboard_group_id', 'execution_timestamp'
]
max_record_size = 1000
pagination_json_path = 'reports[*]'
last_successful_run_query = ModePaginatedRestApiQuery(
query_to_join=seed_query,
url=url,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True,
max_record_size=max_record_size,
pagination_json_path=pagination_json_path)
return last_successful_run_query
def _build_restapi_query(self):
"""
Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports
API, and queries API) joining together.
:return: A RestApiQuery that provides Mode Dashboard execution (run)
"""
# type: () -> RestApiQuery
spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf)
# Reports
# https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
json_path = '(_embedded.reports[*].token)'
field_names = ['dashboard_id']
reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params,
json_path=json_path, field_names=field_names, skip_no_result=True)
queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries'
json_path = '_embedded.queries[*].[token,name,raw_query]'
field_names = ['query_id', 'query_name', 'query_text']
query_names_query = RestApiQuery(query_to_join=reports_query, url=queries_url_template, params=params,
json_path=json_path, field_names=field_names, skip_no_result=True)
return query_names_query
def _build_restapi_query(self) -> ModePaginatedRestApiQuery:
"""
Build REST API Query to get Mode Dashboard queries
:return: A RestApiQuery that provides Mode Dashboard execution (run)
"""
seed_query = ModeDashboardUtils.get_seed_query(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf,
discover_auth=True)
# Queries
# https://mode.com/developer/discovery-api/analytics/queries/
url = 'https://app.mode.com/batch/{organization}/queries'
json_path = 'queries[*].[report_token, space_token, token, name, raw_query]'
field_names = [
'dashboard_id', 'dashboard_group_id', 'query_id', 'query_name',
'query_text'
]
max_record_size = 1000
pagination_json_path = 'queries[*]'
query_names_query = ModePaginatedRestApiQuery(
query_to_join=seed_query,
url=url,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True,
max_record_size=max_record_size,
pagination_json_path=pagination_json_path)
return query_names_query
def init(self, conf: ConfigTree) -> None:
self._conf = conf
restapi_query = self._build_restapi_query()
self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
restapi_query=restapi_query, conf=self._conf)
# Remove all unnecessary fields because User model accepts all attributes and push it to Neo4j.
transformers: List[Transformer] = []
remove_fields_transformer = RemoveFieldTransformer()
remove_fields_transformer.init(conf=Scoped.get_scoped_conf(
self._conf, remove_fields_transformer.get_scope()).with_fallback(
ConfigFactory.from_dict({
FIELD_NAMES:
['organization', 'mode_user_resource_path', 'product']
})))
transformers.append(remove_fields_transformer)
dict_to_model_transformer = DictToModel()
dict_to_model_transformer.init(conf=Scoped.get_scoped_conf(
self._conf, dict_to_model_transformer.get_scope()).with_fallback(
ConfigFactory.from_dict(
{MODEL_CLASS: 'databuilder.models.user.User'})))
transformers.append(dict_to_model_transformer)
self._transformer = ChainedTransformer(transformers=transformers)
def init(self, conf: ConfigTree) -> None:
self._conf = conf
restapi_query = self._build_restapi_query()
self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
restapi_query=restapi_query, conf=self._conf)
# Payload from RestApiQuery has timestamp which is ISO8601. Here we are using TimestampStringToEpoch to
# transform into epoch and then using DictToModel to convert Dictionary to Model
transformers: List[Transformer] = []
timestamp_str_to_epoch_transformer = TimestampStringToEpoch()
timestamp_str_to_epoch_transformer.init(conf=Scoped.get_scoped_conf(
self._conf,
timestamp_str_to_epoch_transformer.get_scope()).with_fallback(
ConfigFactory.from_dict({
FIELD_NAME: 'execution_timestamp',
})))
transformers.append(timestamp_str_to_epoch_transformer)
dict_to_model_transformer = DictToModel()
dict_to_model_transformer.init(conf=Scoped.get_scoped_conf(
self._conf, dict_to_model_transformer.get_scope()
).with_fallback(
ConfigFactory.from_dict({
MODEL_CLASS:
'databuilder.models.dashboard.dashboard_execution.DashboardExecution'
})))
transformers.append(dict_to_model_transformer)
self._transformer = ChainedTransformer(transformers=transformers)
def init(self, conf):
# type: (ConfigTree) -> None
self._conf = conf
restapi_query = self._build_restapi_query()
self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
restapi_query=restapi_query, conf=self._conf)
# Constructing URL using resource path via TemplateVariableSubstitutionTransformer
transformers = []
chart_url_transformer = TemplateVariableSubstitutionTransformer()
chart_url_transformer.init(conf=Scoped.get_scoped_conf(
self._conf, chart_url_transformer.get_scope()).with_fallback(
ConfigFactory.from_dict(
{
FIELD_NAME: 'chart_url',
TEMPLATE: 'https://app.mode.com{chart_url}'
})))
transformers.append(chart_url_transformer)
dict_to_model_transformer = DictToModel()
dict_to_model_transformer.init(conf=Scoped.get_scoped_conf(
self._conf, dict_to_model_transformer.get_scope()
).with_fallback(
ConfigFactory.from_dict({
MODEL_CLASS:
'databuilder.models.dashboard.dashboard_chart.DashboardChart'
})))
transformers.append(dict_to_model_transformer)
self._transformer = ChainedTransformer(transformers=transformers)
def init(self, conf: ConfigTree) -> None:
self._conf = conf
self.dashboard_group_ids_to_skip = self._conf.get_list(
DASHBOARD_GROUP_IDS_TO_SKIP, [])
restapi_query = self._build_restapi_query()
self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
restapi_query=restapi_query, conf=self._conf)
# Payload from RestApiQuery has timestamp which is ISO8601. Here we are using TimestampStringToEpoch to
# transform into epoch and then using DictToModel to convert Dictionary to Model
transformers: List[Transformer] = []
timestamp_str_to_epoch_transformer = TimestampStringToEpoch()
timestamp_str_to_epoch_transformer.init(conf=Scoped.get_scoped_conf(
self._conf,
timestamp_str_to_epoch_transformer.get_scope()).with_fallback(
ConfigFactory.from_dict({
FIELD_NAME: 'created_timestamp',
})))
transformers.append(timestamp_str_to_epoch_transformer)
dashboard_group_url_transformer = TemplateVariableSubstitutionTransformer(
)
dashboard_group_url_transformer.init(conf=Scoped.get_scoped_conf(
self._conf, dashboard_group_url_transformer.get_scope()
).with_fallback(
ConfigFactory.from_dict({
VAR_FIELD_NAME:
'dashboard_group_url',
TEMPLATE:
'https://app.mode.com/{organization}/spaces/{dashboard_group_id}'
})))
transformers.append(dashboard_group_url_transformer)
dashboard_url_transformer = TemplateVariableSubstitutionTransformer()
dashboard_url_transformer.init(conf=Scoped.get_scoped_conf(
self._conf, dashboard_url_transformer.get_scope()
).with_fallback(
ConfigFactory.from_dict({
VAR_FIELD_NAME:
'dashboard_url',
TEMPLATE:
'https://app.mode.com/{organization}/reports/{dashboard_id}'
})))
transformers.append(dashboard_url_transformer)
dict_to_model_transformer = DictToModel()
dict_to_model_transformer.init(conf=Scoped.get_scoped_conf(
self._conf, dict_to_model_transformer.get_scope()
).with_fallback(
ConfigFactory.from_dict({
MODEL_CLASS:
'databuilder.models.dashboard.dashboard_metadata.DashboardMetadata'
})))
transformers.append(dict_to_model_transformer)
self._transformer = ChainedTransformer(transformers=transformers)
def _build_restapi_query(self) -> RestApiQuery:
"""
Build a paginated REST API based on Mode discovery API
:return:
"""
params = ModeDashboardUtils.get_auth_params(conf=self._conf,
discover_auth=True)
seed_record = [{
'organization': self._conf.get_string(ORGANIZATION),
'is_active': None,
'updated_at': None,
'do_not_update_empty_attribute': True,
}]
seed_query = RestApiQuerySeed(seed_record=seed_record)
chart_url_template = 'http://app.mode.com/batch/{organization}/charts'
if self._conf.get_bool(
ModeDashboardChartsBatchExtractor.INCLUDE_ALL_SPACE,
default=False):
chart_url_template += '?include_spaces=all'
json_path = '(charts[*].[space_token,report_token,query_token,token,chart_title,chart_type])'
field_names = [
'dashboard_group_id', 'dashboard_id', 'query_id', 'chart_id',
'chart_name', 'chart_type'
]
chart_batch_query = ModePaginatedRestApiQuery(
query_to_join=seed_query,
url=chart_url_template,
params=params,
json_path=json_path,
pagination_json_path=json_path,
field_names=field_names,
skip_no_result=True)
return chart_batch_query
def init(self, conf):
# type: (ConfigTree) -> None
self._conf = conf
restapi_query = self._build_restapi_query()
self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(restapi_query=restapi_query,
conf=self._conf)
def _build_restapi_query(self):
"""
Build REST API Query. To get Mode Dashboard owner, it needs to call three APIs (spaces API, reports
API, and user API) joining together.
:return: A RestApiQuery that provides Mode Dashboard owner
"""
# type: () -> RestApiQuery
# https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
report_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
# https://mode.com/developer/api-reference/management/users/
creator_url_template = 'https://app.mode.com{creator_resource_path}'
spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf)
# Reports
json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.creator.href)'
field_names = ['dashboard_id', 'creator_resource_path']
creator_resource_path_query = ModePaginatedRestApiQuery(
query_to_join=spaces_query,
url=report_url_template,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True,
json_path_contains_or=True)
json_path = 'email'
field_names = ['email']
failure_handler = HttpFailureSkipOnStatus(status_codes_to_skip={404})
owner_email_query = RestApiQuery(
query_to_join=creator_resource_path_query,
url=creator_url_template,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True,
can_skip_failure=failure_handler.can_skip_failure)
return owner_email_query
def init(self, conf: ConfigTree) -> None:
self._conf = conf
restapi_query = self._build_restapi_query()
self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
restapi_query=restapi_query,
conf=self._conf.with_fallback(
ConfigFactory.from_dict({
MODEL_CLASS:
'databuilder.models.dashboard.dashboard_owner.DashboardOwner',
})))
def _build_restapi_query(self) -> ModePaginatedRestApiQuery:
"""
Build REST API Query. To get Mode Dashboard usage, it needs to call three discovery APIs (
spaces API, reports API and report stats API).
:return: A RestApiQuery that provides Mode Dashboard metadata
"""
seed_query = ModeDashboardUtils.get_seed_query(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf, discover_auth=True)
# Reports
# https://mode.com/developer/discovery-api/analytics/reports/
reports_url = 'https://app.mode.com/batch/{organization}/reports'
reports_json_path = 'reports[*].[token, space_token]'
reports_field_names = ['dashboard_id', 'dashboard_group_id']
reports_max_record_size = 1000
reports_pagination_json_path = 'reports[*]'
spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
spaces_query_merger = QueryMerger(query_to_merge=spaces_query, merge_key='dashboard_group_id')
reports_query = ModePaginatedRestApiQuery(query_to_join=seed_query, url=reports_url, params=params,
json_path=reports_json_path, field_names=reports_field_names,
skip_no_result=True, max_record_size=reports_max_record_size,
pagination_json_path=reports_pagination_json_path,
query_merger=spaces_query_merger)
# https://mode.com/developer/discovery-api/analytics/report-stats/
stats_url = 'https://app.mode.com/batch/{organization}/report_stats'
stats_json_path = 'report_stats[*].[report_token, view_count]'
stats_field_names = ['dashboard_id', 'accumulated_view_count']
stats_max_record_size = 1000
stats_pagination_json_path = 'report_stats[*]'
reports_query_merger = QueryMerger(query_to_merge=reports_query, merge_key='dashboard_id')
report_stats_query = ModePaginatedRestApiQuery(query_to_join=seed_query, url=stats_url, params=params,
json_path=stats_json_path, field_names=stats_field_names,
skip_no_result=True, max_record_size=stats_max_record_size,
pagination_json_path=stats_pagination_json_path,
query_merger=reports_query_merger)
return report_stats_query
def _build_restapi_query(self):
"""
Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports
API, and run API) joining together.
:return: A RestApiQuery that provides Mode Dashboard execution (run)
"""
# type: () -> RestApiQuery
spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf)
# Reports
# https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.last_run.href)'
field_names = ['dashboard_id', 'last_run_resource_path']
last_run_resource_path_query = ModePaginatedRestApiQuery(
query_to_join=spaces_query,
url=url,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True,
json_path_contains_or=True)
# https://mode.com/developer/api-reference/analytics/report-runs/#getReportRun
url = 'https://app.mode.com{last_run_resource_path}'
json_path = '[state,completed_at]'
field_names = ['execution_state', 'execution_timestamp']
last_run_state_query = RestApiQuery(
query_to_join=last_run_resource_path_query,
url=url,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True)
return last_run_state_query
def init(self, conf: ConfigTree) -> None:
self._conf = conf
restapi_query = self._build_restapi_query()
self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
restapi_query=restapi_query,
conf=self._conf
)
dict_to_model_transformer = DictToModel()
dict_to_model_transformer.init(
conf=Scoped.get_scoped_conf(self._conf, dict_to_model_transformer.get_scope()).with_fallback(
ConfigFactory.from_dict(
{MODEL_CLASS: 'databuilder.models.dashboard.dashboard_chart.DashboardChart'})))
self._transformer = dict_to_model_transformer
def _build_restapi_query(self) -> ModePaginatedRestApiQuery:
"""
Build REST API Query to get Mode Dashboard metadata
:return: A RestApiQuery that provides Mode Dashboard metadata
"""
seed_query = ModeDashboardUtils.get_seed_query(conf=self._conf)
params = ModeDashboardUtils.get_auth_params(conf=self._conf,
discover_auth=True)
# Reports
# https://mode.com/developer/discovery-api/analytics/reports/
url = 'https://app.mode.com/batch/{organization}/reports'
json_path = 'reports[*].[token, name, description, created_at, space_token]'
field_names = [
'dashboard_id', 'dashboard_name', 'description',
'created_timestamp', 'dashboard_group_id'
]
max_record_size = 1000
pagination_json_path = 'reports[*]'
spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
query_merger = QueryMerger(query_to_merge=spaces_query,
merge_key='dashboard_group_id')
reports_query = ModePaginatedRestApiQuery(
query_to_join=seed_query,
url=url,
params=params,
json_path=json_path,
field_names=field_names,
skip_no_result=True,
max_record_size=max_record_size,
pagination_json_path=pagination_json_path,
query_merger=query_merger)
return reports_query
def init(self, conf):
# type: (ConfigTree) -> None
self._conf = conf
restapi_query = self._build_restapi_query()
self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
restapi_query=restapi_query, conf=self._conf)
# Constructing URL using several ID via TemplateVariableSubstitutionTransformer
transformers = []
variable_substitution_transformer = TemplateVariableSubstitutionTransformer(
)
variable_substitution_transformer.init(conf=Scoped.get_scoped_conf(
self._conf,
variable_substitution_transformer.get_scope()).with_fallback(
ConfigFactory.from_dict({
FIELD_NAME:
'url',
TEMPLATE:
'https://app.mode.com/{organization}'
'/reports/{dashboard_id}/queries/{query_id}'
})))
transformers.append(variable_substitution_transformer)
# Escape backslash as it breaks Cypher statement.
replace_transformer = RegexStrReplaceTransformer()
replace_transformer.init(conf=Scoped.get_scoped_conf(
self._conf, replace_transformer.get_scope()).with_fallback(
ConfigFactory.from_dict({
REGEX_REPLACE_TUPLE_LIST: [('\\', '\\\\')],
ATTRIBUTE_NAME:
'query_text'
})))
transformers.append(replace_transformer)
dict_to_model_transformer = DictToModel()
dict_to_model_transformer.init(conf=Scoped.get_scoped_conf(
self._conf, dict_to_model_transformer.get_scope()
).with_fallback(
ConfigFactory.from_dict({
MODEL_CLASS:
'databuilder.models.dashboard.dashboard_query.DashboardQuery'
})))
transformers.append(dict_to_model_transformer)
self._transformer = ChainedTransformer(transformers=transformers)
