def _build_restapi_query(self): """ Build REST API Query. To get Mode Dashboard last successful execution, it needs to call two APIs (spaces API, and reports API) joining together. :return: A RestApiQuery that provides Mode Dashboard last successful execution (run) """ # type: () -> RestApiQuery spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf) # Reports # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' json_path = '_embedded.reports[*].[token,last_successfully_run_at]' field_names = ['dashboard_id', 'execution_timestamp'] last_successful_run_query = ModePaginatedRestApiQuery( query_to_join=spaces_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True) return last_successful_run_query
def _build_restapi_query(self) -> RestApiQuery: """ Build REST API Query. To get Mode Dashboard metadata, it needs to call two APIs (spaces API and reports API) joining together. :return: A RestApiQuery that provides Mode Dashboard metadata """ # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace reports_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf) # Reports # JSONPATH expression. it goes into array which is located in _embedded.reports and then extracts token, name, # and description json_path = '_embedded.reports[*].[token,name,description,created_at]' field_names = [ 'dashboard_id', 'dashboard_name', 'description', 'created_timestamp' ] reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params, json_path=json_path, field_names=field_names, skip_no_result=True) return reports_query
def _build_restapi_query(self) -> ModePaginatedRestApiQuery: """ Build REST API Query to get Mode Dashboard last successful execution metadata. :return: A RestApiQuery that provides Mode Dashboard last successful execution (run) """ seed_query = ModeDashboardUtils.get_seed_query(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf, discover_auth=True) # Reports # https://mode.com/developer/discovery-api/analytics/reports/ url = 'https://app.mode.com/batch/{organization}/reports' json_path = 'reports[*].[token, space_token, last_successfully_run_at]' field_names = [ 'dashboard_id', 'dashboard_group_id', 'execution_timestamp' ] max_record_size = 1000 pagination_json_path = 'reports[*]' last_successful_run_query = ModePaginatedRestApiQuery( query_to_join=seed_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, max_record_size=max_record_size, pagination_json_path=pagination_json_path) return last_successful_run_query
def _build_restapi_query(self): """ Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports API, and queries API) joining together. :return: A RestApiQuery that provides Mode Dashboard execution (run) """ # type: () -> RestApiQuery spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf) # Reports # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' json_path = '(_embedded.reports[*].token)' field_names = ['dashboard_id'] reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True) queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries' json_path = '_embedded.queries[*].[token,name,raw_query]' field_names = ['query_id', 'query_name', 'query_text'] query_names_query = RestApiQuery(query_to_join=reports_query, url=queries_url_template, params=params, json_path=json_path, field_names=field_names, skip_no_result=True) return query_names_query
def _build_restapi_query(self) -> ModePaginatedRestApiQuery: """ Build REST API Query to get Mode Dashboard queries :return: A RestApiQuery that provides Mode Dashboard execution (run) """ seed_query = ModeDashboardUtils.get_seed_query(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf, discover_auth=True) # Queries # https://mode.com/developer/discovery-api/analytics/queries/ url = 'https://app.mode.com/batch/{organization}/queries' json_path = 'queries[*].[report_token, space_token, token, name, raw_query]' field_names = [ 'dashboard_id', 'dashboard_group_id', 'query_id', 'query_name', 'query_text' ] max_record_size = 1000 pagination_json_path = 'queries[*]' query_names_query = ModePaginatedRestApiQuery( query_to_join=seed_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, max_record_size=max_record_size, pagination_json_path=pagination_json_path) return query_names_query
def init(self, conf: ConfigTree) -> None: self._conf = conf restapi_query = self._build_restapi_query() self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor( restapi_query=restapi_query, conf=self._conf) # Remove all unnecessary fields because User model accepts all attributes and push it to Neo4j. transformers: List[Transformer] = [] remove_fields_transformer = RemoveFieldTransformer() remove_fields_transformer.init(conf=Scoped.get_scoped_conf( self._conf, remove_fields_transformer.get_scope()).with_fallback( ConfigFactory.from_dict({ FIELD_NAMES: ['organization', 'mode_user_resource_path', 'product'] }))) transformers.append(remove_fields_transformer) dict_to_model_transformer = DictToModel() dict_to_model_transformer.init(conf=Scoped.get_scoped_conf( self._conf, dict_to_model_transformer.get_scope()).with_fallback( ConfigFactory.from_dict( {MODEL_CLASS: 'databuilder.models.user.User'}))) transformers.append(dict_to_model_transformer) self._transformer = ChainedTransformer(transformers=transformers)
def init(self, conf: ConfigTree) -> None: self._conf = conf restapi_query = self._build_restapi_query() self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor( restapi_query=restapi_query, conf=self._conf) # Payload from RestApiQuery has timestamp which is ISO8601. Here we are using TimestampStringToEpoch to # transform into epoch and then using DictToModel to convert Dictionary to Model transformers: List[Transformer] = [] timestamp_str_to_epoch_transformer = TimestampStringToEpoch() timestamp_str_to_epoch_transformer.init(conf=Scoped.get_scoped_conf( self._conf, timestamp_str_to_epoch_transformer.get_scope()).with_fallback( ConfigFactory.from_dict({ FIELD_NAME: 'execution_timestamp', }))) transformers.append(timestamp_str_to_epoch_transformer) dict_to_model_transformer = DictToModel() dict_to_model_transformer.init(conf=Scoped.get_scoped_conf( self._conf, dict_to_model_transformer.get_scope() ).with_fallback( ConfigFactory.from_dict({ MODEL_CLASS: 'databuilder.models.dashboard.dashboard_execution.DashboardExecution' }))) transformers.append(dict_to_model_transformer) self._transformer = ChainedTransformer(transformers=transformers)
def init(self, conf): # type: (ConfigTree) -> None self._conf = conf restapi_query = self._build_restapi_query() self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor( restapi_query=restapi_query, conf=self._conf) # Constructing URL using resource path via TemplateVariableSubstitutionTransformer transformers = [] chart_url_transformer = TemplateVariableSubstitutionTransformer() chart_url_transformer.init(conf=Scoped.get_scoped_conf( self._conf, chart_url_transformer.get_scope()).with_fallback( ConfigFactory.from_dict( { FIELD_NAME: 'chart_url', TEMPLATE: 'https://app.mode.com{chart_url}' }))) transformers.append(chart_url_transformer) dict_to_model_transformer = DictToModel() dict_to_model_transformer.init(conf=Scoped.get_scoped_conf( self._conf, dict_to_model_transformer.get_scope() ).with_fallback( ConfigFactory.from_dict({ MODEL_CLASS: 'databuilder.models.dashboard.dashboard_chart.DashboardChart' }))) transformers.append(dict_to_model_transformer) self._transformer = ChainedTransformer(transformers=transformers)
def init(self, conf: ConfigTree) -> None: self._conf = conf self.dashboard_group_ids_to_skip = self._conf.get_list( DASHBOARD_GROUP_IDS_TO_SKIP, []) restapi_query = self._build_restapi_query() self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor( restapi_query=restapi_query, conf=self._conf) # Payload from RestApiQuery has timestamp which is ISO8601. Here we are using TimestampStringToEpoch to # transform into epoch and then using DictToModel to convert Dictionary to Model transformers: List[Transformer] = [] timestamp_str_to_epoch_transformer = TimestampStringToEpoch() timestamp_str_to_epoch_transformer.init(conf=Scoped.get_scoped_conf( self._conf, timestamp_str_to_epoch_transformer.get_scope()).with_fallback( ConfigFactory.from_dict({ FIELD_NAME: 'created_timestamp', }))) transformers.append(timestamp_str_to_epoch_transformer) dashboard_group_url_transformer = TemplateVariableSubstitutionTransformer( ) dashboard_group_url_transformer.init(conf=Scoped.get_scoped_conf( self._conf, dashboard_group_url_transformer.get_scope() ).with_fallback( ConfigFactory.from_dict({ VAR_FIELD_NAME: 'dashboard_group_url', TEMPLATE: 'https://app.mode.com/{organization}/spaces/{dashboard_group_id}' }))) transformers.append(dashboard_group_url_transformer) dashboard_url_transformer = TemplateVariableSubstitutionTransformer() dashboard_url_transformer.init(conf=Scoped.get_scoped_conf( self._conf, dashboard_url_transformer.get_scope() ).with_fallback( ConfigFactory.from_dict({ VAR_FIELD_NAME: 'dashboard_url', TEMPLATE: 'https://app.mode.com/{organization}/reports/{dashboard_id}' }))) transformers.append(dashboard_url_transformer) dict_to_model_transformer = DictToModel() dict_to_model_transformer.init(conf=Scoped.get_scoped_conf( self._conf, dict_to_model_transformer.get_scope() ).with_fallback( ConfigFactory.from_dict({ MODEL_CLASS: 'databuilder.models.dashboard.dashboard_metadata.DashboardMetadata' }))) transformers.append(dict_to_model_transformer) self._transformer = ChainedTransformer(transformers=transformers)
def _build_restapi_query(self) -> RestApiQuery: """ Build a paginated REST API based on Mode discovery API :return: """ params = ModeDashboardUtils.get_auth_params(conf=self._conf, discover_auth=True) seed_record = [{ 'organization': self._conf.get_string(ORGANIZATION), 'is_active': None, 'updated_at': None, 'do_not_update_empty_attribute': True, }] seed_query = RestApiQuerySeed(seed_record=seed_record) chart_url_template = 'http://app.mode.com/batch/{organization}/charts' if self._conf.get_bool( ModeDashboardChartsBatchExtractor.INCLUDE_ALL_SPACE, default=False): chart_url_template += '?include_spaces=all' json_path = '(charts[*].[space_token,report_token,query_token,token,chart_title,chart_type])' field_names = [ 'dashboard_group_id', 'dashboard_id', 'query_id', 'chart_id', 'chart_name', 'chart_type' ] chart_batch_query = ModePaginatedRestApiQuery( query_to_join=seed_query, url=chart_url_template, params=params, json_path=json_path, pagination_json_path=json_path, field_names=field_names, skip_no_result=True) return chart_batch_query
def init(self, conf): # type: (ConfigTree) -> None self._conf = conf restapi_query = self._build_restapi_query() self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(restapi_query=restapi_query, conf=self._conf)
def _build_restapi_query(self): """ Build REST API Query. To get Mode Dashboard owner, it needs to call three APIs (spaces API, reports API, and user API) joining together. :return: A RestApiQuery that provides Mode Dashboard owner """ # type: () -> RestApiQuery # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace report_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' # https://mode.com/developer/api-reference/management/users/ creator_url_template = 'https://app.mode.com{creator_resource_path}' spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf) # Reports json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.creator.href)' field_names = ['dashboard_id', 'creator_resource_path'] creator_resource_path_query = ModePaginatedRestApiQuery( query_to_join=spaces_query, url=report_url_template, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, json_path_contains_or=True) json_path = 'email' field_names = ['email'] failure_handler = HttpFailureSkipOnStatus(status_codes_to_skip={404}) owner_email_query = RestApiQuery( query_to_join=creator_resource_path_query, url=creator_url_template, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, can_skip_failure=failure_handler.can_skip_failure) return owner_email_query
def init(self, conf: ConfigTree) -> None: self._conf = conf restapi_query = self._build_restapi_query() self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor( restapi_query=restapi_query, conf=self._conf.with_fallback( ConfigFactory.from_dict({ MODEL_CLASS: 'databuilder.models.dashboard.dashboard_owner.DashboardOwner', })))
def _build_restapi_query(self) -> ModePaginatedRestApiQuery: """ Build REST API Query. To get Mode Dashboard usage, it needs to call three discovery APIs ( spaces API, reports API and report stats API). :return: A RestApiQuery that provides Mode Dashboard metadata """ seed_query = ModeDashboardUtils.get_seed_query(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf, discover_auth=True) # Reports # https://mode.com/developer/discovery-api/analytics/reports/ reports_url = 'https://app.mode.com/batch/{organization}/reports' reports_json_path = 'reports[*].[token, space_token]' reports_field_names = ['dashboard_id', 'dashboard_group_id'] reports_max_record_size = 1000 reports_pagination_json_path = 'reports[*]' spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) spaces_query_merger = QueryMerger(query_to_merge=spaces_query, merge_key='dashboard_group_id') reports_query = ModePaginatedRestApiQuery(query_to_join=seed_query, url=reports_url, params=params, json_path=reports_json_path, field_names=reports_field_names, skip_no_result=True, max_record_size=reports_max_record_size, pagination_json_path=reports_pagination_json_path, query_merger=spaces_query_merger) # https://mode.com/developer/discovery-api/analytics/report-stats/ stats_url = 'https://app.mode.com/batch/{organization}/report_stats' stats_json_path = 'report_stats[*].[report_token, view_count]' stats_field_names = ['dashboard_id', 'accumulated_view_count'] stats_max_record_size = 1000 stats_pagination_json_path = 'report_stats[*]' reports_query_merger = QueryMerger(query_to_merge=reports_query, merge_key='dashboard_id') report_stats_query = ModePaginatedRestApiQuery(query_to_join=seed_query, url=stats_url, params=params, json_path=stats_json_path, field_names=stats_field_names, skip_no_result=True, max_record_size=stats_max_record_size, pagination_json_path=stats_pagination_json_path, query_merger=reports_query_merger) return report_stats_query
def _build_restapi_query(self): """ Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports API, and run API) joining together. :return: A RestApiQuery that provides Mode Dashboard execution (run) """ # type: () -> RestApiQuery spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf) # Reports # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.last_run.href)' field_names = ['dashboard_id', 'last_run_resource_path'] last_run_resource_path_query = ModePaginatedRestApiQuery( query_to_join=spaces_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, json_path_contains_or=True) # https://mode.com/developer/api-reference/analytics/report-runs/#getReportRun url = 'https://app.mode.com{last_run_resource_path}' json_path = '[state,completed_at]' field_names = ['execution_state', 'execution_timestamp'] last_run_state_query = RestApiQuery( query_to_join=last_run_resource_path_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True) return last_run_state_query
def init(self, conf: ConfigTree) -> None: self._conf = conf restapi_query = self._build_restapi_query() self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor( restapi_query=restapi_query, conf=self._conf ) dict_to_model_transformer = DictToModel() dict_to_model_transformer.init( conf=Scoped.get_scoped_conf(self._conf, dict_to_model_transformer.get_scope()).with_fallback( ConfigFactory.from_dict( {MODEL_CLASS: 'databuilder.models.dashboard.dashboard_chart.DashboardChart'}))) self._transformer = dict_to_model_transformer
def _build_restapi_query(self) -> ModePaginatedRestApiQuery: """ Build REST API Query to get Mode Dashboard metadata :return: A RestApiQuery that provides Mode Dashboard metadata """ seed_query = ModeDashboardUtils.get_seed_query(conf=self._conf) params = ModeDashboardUtils.get_auth_params(conf=self._conf, discover_auth=True) # Reports # https://mode.com/developer/discovery-api/analytics/reports/ url = 'https://app.mode.com/batch/{organization}/reports' json_path = 'reports[*].[token, name, description, created_at, space_token]' field_names = [ 'dashboard_id', 'dashboard_name', 'description', 'created_timestamp', 'dashboard_group_id' ] max_record_size = 1000 pagination_json_path = 'reports[*]' spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf) query_merger = QueryMerger(query_to_merge=spaces_query, merge_key='dashboard_group_id') reports_query = ModePaginatedRestApiQuery( query_to_join=seed_query, url=url, params=params, json_path=json_path, field_names=field_names, skip_no_result=True, max_record_size=max_record_size, pagination_json_path=pagination_json_path, query_merger=query_merger) return reports_query
def init(self, conf): # type: (ConfigTree) -> None self._conf = conf restapi_query = self._build_restapi_query() self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor( restapi_query=restapi_query, conf=self._conf) # Constructing URL using several ID via TemplateVariableSubstitutionTransformer transformers = [] variable_substitution_transformer = TemplateVariableSubstitutionTransformer( ) variable_substitution_transformer.init(conf=Scoped.get_scoped_conf( self._conf, variable_substitution_transformer.get_scope()).with_fallback( ConfigFactory.from_dict({ FIELD_NAME: 'url', TEMPLATE: 'https://app.mode.com/{organization}' '/reports/{dashboard_id}/queries/{query_id}' }))) transformers.append(variable_substitution_transformer) # Escape backslash as it breaks Cypher statement. replace_transformer = RegexStrReplaceTransformer() replace_transformer.init(conf=Scoped.get_scoped_conf( self._conf, replace_transformer.get_scope()).with_fallback( ConfigFactory.from_dict({ REGEX_REPLACE_TUPLE_LIST: [('\\', '\\\\')], ATTRIBUTE_NAME: 'query_text' }))) transformers.append(replace_transformer) dict_to_model_transformer = DictToModel() dict_to_model_transformer.init(conf=Scoped.get_scoped_conf( self._conf, dict_to_model_transformer.get_scope() ).with_fallback( ConfigFactory.from_dict({ MODEL_CLASS: 'databuilder.models.dashboard.dashboard_query.DashboardQuery' }))) transformers.append(dict_to_model_transformer) self._transformer = ChainedTransformer(transformers=transformers)