def init(self, conf: ConfigTree) -> None: self._conf = conf self.query = """query { workbooks { id name projectName updatedAt } }""" self._extractor = self._build_extractor() transformers: List[Transformer] = [] timestamp_str_to_epoch_transformer = TimestampStringToEpoch() timestamp_str_to_epoch_transformer.init(conf=Scoped.get_scoped_conf( self._conf, timestamp_str_to_epoch_transformer.get_scope()).with_fallback( ConfigFactory.from_dict({ FIELD_NAME: 'last_modified_timestamp', }))) transformers.append(timestamp_str_to_epoch_transformer) dict_to_model_transformer = DictToModel() dict_to_model_transformer.init(conf=Scoped.get_scoped_conf( self._conf, dict_to_model_transformer.get_scope() ).with_fallback( ConfigFactory.from_dict({ MODEL_CLASS: 'databuilder.models.dashboard.dashboard_last_modified.DashboardLastModifiedTimestamp' }))) transformers.append(dict_to_model_transformer) self._transformer = ChainedTransformer(transformers=transformers)
def init(self, conf: ConfigTree) -> None: self._conf = conf restapi_query = self._build_restapi_query() self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor( restapi_query=restapi_query, conf=self._conf) # Payload from RestApiQuery has timestamp which is ISO8601. Here we are using TimestampStringToEpoch to # transform into epoch and then using DictToModel to convert Dictionary to Model transformers: List[Transformer] = [] timestamp_str_to_epoch_transformer = TimestampStringToEpoch() timestamp_str_to_epoch_transformer.init(conf=Scoped.get_scoped_conf( self._conf, timestamp_str_to_epoch_transformer.get_scope()).with_fallback( ConfigFactory.from_dict({ FIELD_NAME: 'execution_timestamp', }))) transformers.append(timestamp_str_to_epoch_transformer) dict_to_model_transformer = DictToModel() dict_to_model_transformer.init(conf=Scoped.get_scoped_conf( self._conf, dict_to_model_transformer.get_scope() ).with_fallback( ConfigFactory.from_dict({ MODEL_CLASS: 'databuilder.models.dashboard.dashboard_execution.DashboardExecution' }))) transformers.append(dict_to_model_transformer) self._transformer = ChainedTransformer(transformers=transformers)
def init(self, conf: ConfigTree) -> None: self._conf = conf self.dashboard_group_ids_to_skip = self._conf.get_list( DASHBOARD_GROUP_IDS_TO_SKIP, []) restapi_query = self._build_restapi_query() self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor( restapi_query=restapi_query, conf=self._conf) # Payload from RestApiQuery has timestamp which is ISO8601. Here we are using TimestampStringToEpoch to # transform into epoch and then using DictToModel to convert Dictionary to Model transformers: List[Transformer] = [] timestamp_str_to_epoch_transformer = TimestampStringToEpoch() timestamp_str_to_epoch_transformer.init(conf=Scoped.get_scoped_conf( self._conf, timestamp_str_to_epoch_transformer.get_scope()).with_fallback( ConfigFactory.from_dict({ FIELD_NAME: 'created_timestamp', }))) transformers.append(timestamp_str_to_epoch_transformer) dashboard_group_url_transformer = TemplateVariableSubstitutionTransformer( ) dashboard_group_url_transformer.init(conf=Scoped.get_scoped_conf( self._conf, dashboard_group_url_transformer.get_scope() ).with_fallback( ConfigFactory.from_dict({ VAR_FIELD_NAME: 'dashboard_group_url', TEMPLATE: 'https://app.mode.com/{organization}/spaces/{dashboard_group_id}' }))) transformers.append(dashboard_group_url_transformer) dashboard_url_transformer = TemplateVariableSubstitutionTransformer() dashboard_url_transformer.init(conf=Scoped.get_scoped_conf( self._conf, dashboard_url_transformer.get_scope() ).with_fallback( ConfigFactory.from_dict({ VAR_FIELD_NAME: 'dashboard_url', TEMPLATE: 'https://app.mode.com/{organization}/reports/{dashboard_id}' }))) transformers.append(dashboard_url_transformer) dict_to_model_transformer = DictToModel() dict_to_model_transformer.init(conf=Scoped.get_scoped_conf( self._conf, dict_to_model_transformer.get_scope() ).with_fallback( ConfigFactory.from_dict({ MODEL_CLASS: 'databuilder.models.dashboard.dashboard_metadata.DashboardMetadata' }))) transformers.append(dict_to_model_transformer) self._transformer = ChainedTransformer(transformers=transformers)
def test_invalid_timestamp(self) -> None: transformer = TimestampStringToEpoch() config = ConfigFactory.from_dict({ FIELD_NAME: 'foo', }) transformer.init(conf=config) actual = transformer.transform({'foo': '165de33266d4'}) self.assertEqual(actual['foo'], 0)
def test_conversion(self) -> None: transformer = TimestampStringToEpoch() config = ConfigFactory.from_dict({ FIELD_NAME: 'foo', }) transformer.init(conf=config) actual = transformer.transform({'foo': '2020-02-19T19:52:33.1Z'}) self.assertDictEqual({'foo': 1582141953}, actual)
def test_conversion_with_format(self) -> None: transformer = TimestampStringToEpoch() config = ConfigFactory.from_dict({ FIELD_NAME: 'foo', TIMESTAMP_FORMAT: '%Y-%m-%dT%H:%M:%SZ' }) transformer.init(conf=config) actual = transformer.transform({'foo': '2020-02-19T19:52:33Z'}) self.assertDictEqual({'foo': 1582141953}, actual)
def init(self, conf: ConfigTree) -> None: conf = conf.with_fallback( ConfigFactory.from_dict({ STATIC_RECORD_DICT: { 'product': 'mode' }, '{}.{}'.format(DictToModel().get_scope(), MODEL_CLASS): 'databuilder.models.dashboard.dashboard_last_modified.DashboardLastModifiedTimestamp', '{}.{}'.format(TimestampStringToEpoch().get_scope(), FIELD_NAME): 'last_modified_timestamp' })) super(ModeDashboardLastModifiedTimestampExtractor, self).init(conf)
def _build_transformer(self) -> ChainedTransformer: transformers = [] # transform timestamps from ISO to unix epoch ts_transformer_1 = TimestampStringToEpoch() ts_transformer_1.init( ConfigFactory.from_dict({TS_FIELD_NAME: "created_timestamp"})) transformers.append(ts_transformer_1) ts_transformer_2 = TimestampStringToEpoch() ts_transformer_2.init( ConfigFactory.from_dict({TS_FIELD_NAME: "last_modified_timestamp"})) transformers.append(ts_transformer_2) return ChainedTransformer(transformers=transformers)