def get_key(self) -> DataContextKey: """ Generates the appropriate Store key to retrieve/store configs. """ key: ConfigurationIdentifier = ConfigurationIdentifier( configuration_key=DataContextVariableSchema.ALL_VARIABLES) return key
def delete_profiler( profiler_store: ProfilerStore, name: Optional[str] = None, ge_cloud_id: Optional[str] = None, ) -> None: assert bool(name) ^ bool( ge_cloud_id ), "Must provide either name or ge_cloud_id (but not both)" key: Union[GeCloudIdentifier, ConfigurationIdentifier] if ge_cloud_id: key = GeCloudIdentifier(resource_type="contract", ge_cloud_id=ge_cloud_id) else: key = ConfigurationIdentifier(configuration_key=name) try: profiler_store.remove_key(key=key) except (ge_exceptions.InvalidKeyError, KeyError) as exc_ik: id_ = (key.configuration_key if isinstance( key, ConfigurationIdentifier) else key) raise ge_exceptions.ProfilerNotFoundError( message= f'Non-existent Profiler configuration named "{id_}".\n\nDetails: {exc_ik}' )
def add_profiler( config: RuleBasedProfilerConfig, data_context: "DataContext", # noqa: F821 profiler_store: ProfilerStore, ge_cloud_id: Optional[str] = None, ) -> "RuleBasedProfiler": if not RuleBasedProfiler._check_validity_of_batch_requests_in_config( config=config): raise ge_exceptions.InvalidConfigError( f'batch_data found in batch_request cannot be saved to ProfilerStore "{profiler_store.store_name}"' ) # Chetan - 20220204 - DataContext to be removed once it can be decoupled from RBP new_profiler: "RuleBasedProfiler" = instantiate_class_from_config( config=config.to_json_dict(), runtime_environment={ "data_context": data_context, }, config_defaults={ "module_name": "great_expectations.rule_based_profiler", "class_name": "RuleBasedProfiler", }, ) key: Union[GeCloudIdentifier, ConfigurationIdentifier] if ge_cloud_id: key = GeCloudIdentifier(resource_type="contract", ge_cloud_id=ge_cloud_id) else: key = ConfigurationIdentifier(configuration_key=config.name, ) profiler_store.set(key=key, value=config) return new_profiler
def test_list_profilers(mock_profiler_store: mock.MagicMock): store = mock_profiler_store() keys = ["a", "b", "c"] store.list_keys.return_value = [ ConfigurationIdentifier(char) for char in keys ] res = RuleBasedProfiler.list_profilers(store, ge_cloud_mode=False) assert res == keys assert store.list_keys.called
def delete_checkpoint_config_from_store_backend( store_name: str, store_backend: Union[StoreBackend, dict], checkpoint_name: str, ): config_store: CheckpointStore = build_checkpoint_store_using_store_backend( store_name=store_name, store_backend=store_backend, ) key: ConfigurationIdentifier = ConfigurationIdentifier( configuration_key=checkpoint_name, ) config_store.remove_key(key=key)
def test_delete_profiler(populated_profiler_store: ProfilerStore, ): with mock.patch( "great_expectations.data_context.store.profiler_store.ProfilerStore.remove_key", ) as mock_remove_key: RuleBasedProfiler.delete_profiler( profiler_store=populated_profiler_store, name="my_profiler", ge_cloud_id=None, ) assert mock_remove_key.call_count == 1 assert mock_remove_key.call_args == mock.call( key=ConfigurationIdentifier("my_profiler"))
def save_checkpoint_config_to_store_backend( store_name: str, store_backend: Union[StoreBackend, dict], checkpoint_name: str, checkpoint_configuration: CheckpointConfig, ): config_store: CheckpointStore = build_checkpoint_store_using_store_backend( store_name=store_name, store_backend=store_backend, overwrite_existing=True, ) key: ConfigurationIdentifier = ConfigurationIdentifier( configuration_key=checkpoint_name, ) config_store.set(key=key, value=checkpoint_configuration)
def determine_key( name: Optional[str], ge_cloud_id: Optional[str] ) -> Union[GeCloudIdentifier, ConfigurationIdentifier]: assert bool(name) ^ bool( ge_cloud_id), "Must provide either name or ge_cloud_id." key: Union[GeCloudIdentifier, ConfigurationIdentifier] if ge_cloud_id: key = GeCloudIdentifier(resource_type=GeCloudRESTResource.CONTRACT, ge_cloud_id=ge_cloud_id) else: key = ConfigurationIdentifier(configuration_key=name) return key
def delete_config_from_store_backend( class_name: str, module_name: str, store_name: str, store_backend: Union[StoreBackend, dict], configuration_key: str, ): config_store: ConfigurationStore = build_configuration_store( class_name=class_name, module_name=module_name, store_name=store_name, store_backend=store_backend, overwrite_existing=True, ) key: ConfigurationIdentifier = ConfigurationIdentifier( configuration_key=configuration_key, ) config_store.remove_key(key=key)
def load_config_from_store_backend( class_name: str, module_name: str, store_name: str, store_backend: Union[StoreBackend, dict], configuration_key: str, ) -> BaseYamlConfig: config_store: ConfigurationStore = build_configuration_store( class_name=class_name, module_name=module_name, store_name=store_name, store_backend=store_backend, overwrite_existing=False, ) key: ConfigurationIdentifier = ConfigurationIdentifier( configuration_key=configuration_key, ) return config_store.get(key=key)
def load_checkpoint_config_from_store_backend( store_name: str, store_backend: Union[StoreBackend, dict], checkpoint_name: str, ) -> CheckpointConfig: config_store: CheckpointStore = build_checkpoint_store_using_store_backend( store_name=store_name, store_backend=store_backend, ) key: ConfigurationIdentifier = ConfigurationIdentifier( configuration_key=checkpoint_name, ) try: return config_store.get(key=key) except ge_exceptions.InvalidBaseYamlConfigError as exc: logger.error(exc.messages) raise ge_exceptions.InvalidCheckpointConfigError( "Error while processing DataContextConfig.", exc)
def save_config_to_store_backend( class_name: str, module_name: str, store_name: str, store_backend: Union[StoreBackend, dict], configuration_key: str, configuration: BaseYamlConfig, ): config_store: ConfigurationStore = build_configuration_store( class_name=class_name, module_name=module_name, store_name=store_name, store_backend=store_backend, overwrite_existing=True, ) key: ConfigurationIdentifier = ConfigurationIdentifier( configuration_key=configuration_key, ) config_store.set(key=key, value=configuration)
def get_profiler( data_context: "DataContext", # noqa: F821 profiler_store: ProfilerStore, name: Optional[str] = None, ge_cloud_id: Optional[str] = None, ) -> "RuleBasedProfiler": assert bool(name) ^ bool( ge_cloud_id ), "Must provide either name or ge_cloud_id (but not both)" key: Union[GeCloudIdentifier, ConfigurationIdentifier] if ge_cloud_id: key = GeCloudIdentifier(resource_type="contract", ge_cloud_id=ge_cloud_id) else: key = ConfigurationIdentifier(configuration_key=name, ) try: profiler_config: RuleBasedProfilerConfig = profiler_store.get( key=key) except ge_exceptions.InvalidKeyError as exc_ik: id_ = (key.configuration_key if isinstance( key, ConfigurationIdentifier) else key) raise ge_exceptions.ProfilerNotFoundError( message= f'Non-existent Profiler configuration named "{id_}".\n\nDetails: {exc_ik}' ) config = profiler_config.to_json_dict() if name: config.update({"name": name}) config = filter_properties_dict(properties=config, clean_falsy=True) profiler = instantiate_class_from_config( config=config, runtime_environment={ "data_context": data_context, }, config_defaults={ "module_name": "great_expectations.rule_based_profiler", "class_name": "RuleBasedProfiler", }, ) return profiler
def delete_checkpoint( checkpoint_store: CheckpointStore, name: Optional[str] = None, ge_cloud_id: Optional[str] = None, ): assert bool(name) ^ bool(ge_cloud_id), "Must provide either name or ge_cloud_id." if ge_cloud_id: key: GeCloudIdentifier = GeCloudIdentifier( resource_type="contract", ge_cloud_id=ge_cloud_id ) else: key: ConfigurationIdentifier = ConfigurationIdentifier(configuration_key=name) try: checkpoint_store.remove_key(key=key) except ge_exceptions.InvalidKeyError as exc_ik: raise ge_exceptions.CheckpointNotFoundError( message=f'Non-existent Checkpoint configuration named "{key.configuration_key}".\n\nDetails: {exc_ik}' )
def add_checkpoint( data_context: "DataContext", # noqa: F821 checkpoint_store: CheckpointStore, checkpoint_store_name: str, ge_cloud_mode: bool, name: str, config_version: Optional[Union[int, float]] = None, template_name: Optional[str] = None, module_name: Optional[str] = None, class_name: Optional[str] = None, run_name_template: Optional[str] = None, expectation_suite_name: Optional[str] = None, batch_request: Optional[dict] = None, action_list: Optional[List[dict]] = None, evaluation_parameters: Optional[dict] = None, runtime_configuration: Optional[dict] = None, validations: Optional[List[dict]] = None, profilers: Optional[List[dict]] = None, # Next two fields are for LegacyCheckpoint configuration validation_operator_name: Optional[str] = None, batches: Optional[List[dict]] = None, # the following four arguments are used by SimpleCheckpoint site_names: Optional[Union[str, List[str]]] = None, slack_webhook: Optional[str] = None, notify_on: Optional[str] = None, notify_with: Optional[Union[str, List[str]]] = None, ge_cloud_id: Optional[str] = None, expectation_suite_ge_cloud_id: Optional[str] = None, ) -> Union[Checkpoint, LegacyCheckpoint]: checkpoint_config: Union[CheckpointConfig, dict] # These checks protect against typed objects (BatchRequest and/or RuntimeBatchRequest) encountered in arguments. batch_request = get_batch_request_as_dict(batch_request=batch_request) validations = get_validations_with_batch_request_as_dict(validations=validations) # DataFrames shouldn't be saved to CheckpointStore if batch_request_contains_batch_data(batch_request=batch_request): raise ge_exceptions.InvalidConfigError( f'batch_data found in batch_request cannot be saved to CheckpointStore "{checkpoint_store_name}"' ) if batch_request_in_validations_contains_batch_data(validations=validations): raise ge_exceptions.InvalidConfigError( f'batch_data found in validations cannot be saved to CheckpointStore "{checkpoint_store_name}"' ) checkpoint_config = { "name": name, "config_version": config_version, "template_name": template_name, "module_name": module_name, "class_name": class_name, "run_name_template": run_name_template, "expectation_suite_name": expectation_suite_name, "batch_request": batch_request, "action_list": action_list, "evaluation_parameters": evaluation_parameters, "runtime_configuration": runtime_configuration, "validations": validations, "profilers": profilers, # Next two fields are for LegacyCheckpoint configuration "validation_operator_name": validation_operator_name, "batches": batches, # the following four keys are used by SimpleCheckpoint "site_names": site_names, "slack_webhook": slack_webhook, "notify_on": notify_on, "notify_with": notify_with, "ge_cloud_id": ge_cloud_id, "expectation_suite_ge_cloud_id": expectation_suite_ge_cloud_id, } checkpoint_config = deep_filter_properties_iterable( properties=checkpoint_config, clean_falsy=True, ) new_checkpoint: Union[ Checkpoint, SimpleCheckpoint, LegacyCheckpoint ] = instantiate_class_from_config( config=checkpoint_config, runtime_environment={ "data_context": data_context, }, config_defaults={ "module_name": "great_expectations.checkpoint", }, ) if ge_cloud_mode: key: GeCloudIdentifier = GeCloudIdentifier( resource_type="contract", ge_cloud_id=ge_cloud_id ) else: key: ConfigurationIdentifier = ConfigurationIdentifier( configuration_key=name, ) checkpoint_config = new_checkpoint.get_config() checkpoint_ref = checkpoint_store.set(key=key, value=checkpoint_config) if isinstance(checkpoint_ref, GeCloudIdAwareRef): ge_cloud_id = checkpoint_ref.ge_cloud_id new_checkpoint.ge_cloud_id = uuid.UUID(ge_cloud_id) return new_checkpoint
def get_checkpoint( data_context: "DataContext", # noqa: F821 checkpoint_store: CheckpointStore, name: Optional[str] = None, ge_cloud_id: Optional[str] = None, ) -> Union[Checkpoint, LegacyCheckpoint]: if ge_cloud_id: key: GeCloudIdentifier = GeCloudIdentifier( resource_type="contract", ge_cloud_id=ge_cloud_id ) else: key: ConfigurationIdentifier = ConfigurationIdentifier( configuration_key=name, ) try: checkpoint_config: CheckpointConfig = checkpoint_store.get(key=key) except ge_exceptions.InvalidKeyError as exc_ik: raise ge_exceptions.CheckpointNotFoundError( message=f'Non-existent Checkpoint configuration named "{key.configuration_key}".\n\nDetails: {exc_ik}' ) except ValidationError as exc_ve: raise ge_exceptions.InvalidCheckpointConfigError( message="Invalid Checkpoint configuration", validation_error=exc_ve ) if checkpoint_config.config_version is None: if not ( "batches" in checkpoint_config.to_json_dict() and ( len(checkpoint_config.to_json_dict()["batches"]) == 0 or {"batch_kwargs", "expectation_suite_names",}.issubset( set( list( itertools.chain.from_iterable( [ item.keys() for item in checkpoint_config.to_json_dict()[ "batches" ] ] ) ) ) ) ) ): raise ge_exceptions.CheckpointError( message="Attempt to instantiate LegacyCheckpoint with insufficient and/or incorrect arguments." ) config: dict = checkpoint_config.to_json_dict() if name: config.update({"name": name}) config = filter_properties_dict(properties=config, clean_falsy=True) checkpoint: Union[Checkpoint, LegacyCheckpoint] = instantiate_class_from_config( config=config, runtime_environment={ "data_context": data_context, }, config_defaults={ "module_name": "great_expectations.checkpoint", }, ) return checkpoint
def test_profiler_store_integration( empty_data_context: DataContext, profiler_store_name: str, profiler_name: str, profiler_config_with_placeholder_args: RuleBasedProfilerConfig, ): base_directory: str = str( Path(empty_data_context.root_directory) / "profilers") profiler_store: ProfilerStore = build_profiler_store_using_filesystem( store_name=profiler_store_name, base_directory=base_directory, overwrite_existing=True, ) dir_tree: str dir_tree = gen_directory_tree_str(startpath=base_directory) assert (dir_tree == """profilers/ .ge_store_backend_id """) key: ConfigurationIdentifier = ConfigurationIdentifier( configuration_key=profiler_name) profiler_store.set(key=key, value=profiler_config_with_placeholder_args) dir_tree = gen_directory_tree_str(startpath=base_directory) assert (dir_tree == """profilers/ .ge_store_backend_id my_first_profiler.yml """) assert len(profiler_store.list_keys()) == 1 profiler_store.remove_key(key=key) assert len(profiler_store.list_keys()) == 0 data: dict = profiler_store.self_check() self_check_report: dict = convert_to_json_serializable(data=data) # Drop dynamic value to ensure appropriate assert self_check_report["config"]["store_backend"].pop("base_directory") assert self_check_report == { "config": { "class_name": "ProfilerStore", "module_name": "great_expectations.data_context.store.profiler_store", "overwrite_existing": True, "store_backend": { "class_name": "TupleFilesystemStoreBackend", "filepath_suffix": ".yml", "fixed_length_key": False, "module_name": "great_expectations.data_context.store.tuple_store_backend", "platform_specific_separator": True, "suppress_store_backend_id": False, }, "store_name": "profiler_store", }, "keys": [], "len_keys": 0, }
def test_checkpoint_store(empty_data_context): store_name: str = "checkpoint_store" base_directory: str = str( Path(empty_data_context.root_directory) / "checkpoints") checkpoint_store: CheckpointStore = build_checkpoint_store_using_filesystem( store_name=store_name, base_directory=base_directory, overwrite_existing=True, ) assert len(checkpoint_store.list_keys()) == 0 with pytest.raises(TypeError): checkpoint_store.set(key="my_first_checkpoint", value="this is not a checkpoint") assert len(checkpoint_store.list_keys()) == 0 checkpoint_name_0: str = "my_checkpoint_0" run_name_template_0: str = "%Y-%M-my-run-template-$VAR" validations_0: Union[List, Dict] = [{ "batch_request": { "datasource_name": "my_pandas_datasource", "data_connector_name": "my_runtime_data_connector", "data_asset_name": "my_website_logs", }, "action_list": [ { "name": "store_validation_result", "action": { "class_name": "StoreValidationResultAction", }, }, { "name": "store_evaluation_params", "action": { "class_name": "StoreEvaluationParametersAction", }, }, { "name": "update_data_docs", "action": { "class_name": "UpdateDataDocsAction", }, }, ], }] expectation_suite_name_0: str = "my.test.expectation_suite.name" evaluation_parameters_0: dict = { "environment": "$GE_ENVIRONMENT", "tolerance": 1.0e-2, "aux_param_0": "$MY_PARAM", "aux_param_1": "1 + $MY_PARAM", } runtime_configuration_0: dict = { "result_format": { "result_format": "BASIC", "partial_unexpected_count": 20, }, } my_checkpoint_config_0: CheckpointConfig = CheckpointConfig( name=checkpoint_name_0, run_name_template=run_name_template_0, expectation_suite_name=expectation_suite_name_0, evaluation_parameters=evaluation_parameters_0, runtime_configuration=runtime_configuration_0, validations=validations_0, ) key_0: ConfigurationIdentifier = ConfigurationIdentifier( configuration_key=checkpoint_name_0, ) checkpoint_store.set(key=key_0, value=my_checkpoint_config_0) assert len(checkpoint_store.list_keys()) == 1 assert filter_properties_dict(properties=checkpoint_store.get( key=key_0).to_json_dict(), ) == filter_properties_dict( properties=my_checkpoint_config_0.to_json_dict(), ) dir_tree: str = gen_directory_tree_str(startpath=base_directory) assert (dir_tree == """checkpoints/ .ge_store_backend_id my_checkpoint_0.yml """) checkpoint_name_1: str = "my_checkpoint_1" run_name_template_1: str = "%Y-%M-my-run-template-$VAR" validations_1: Union[List, Dict] = [{ "action_list": [ { "name": "store_validation_result", "action": { "class_name": "StoreValidationResultAction", }, }, { "name": "store_evaluation_params", "action": { "class_name": "StoreEvaluationParametersAction", }, }, { "name": "update_data_docs", "action": { "class_name": "UpdateDataDocsAction", }, }, ] }] expectation_suite_name_1: str = "my.test.expectation_suite.name" batch_request_1: dict = { "datasource_name": "my_pandas_datasource", "data_connector_name": "my_runtime_data_connector", "data_asset_name": "my_website_logs", } evaluation_parameters_1: dict = { "environment": "$GE_ENVIRONMENT", "tolerance": 1.0e-2, "aux_param_0": "$MY_PARAM", "aux_param_1": "1 + $MY_PARAM", } runtime_configuration_1: dict = { "result_format": { "result_format": "BASIC", "partial_unexpected_count": 20, }, } my_checkpoint_config_1: CheckpointConfig = CheckpointConfig( name=checkpoint_name_1, run_name_template=run_name_template_1, expectation_suite_name=expectation_suite_name_1, batch_request=batch_request_1, evaluation_parameters=evaluation_parameters_1, runtime_configuration=runtime_configuration_1, validations=validations_1, ) key_1: ConfigurationIdentifier = ConfigurationIdentifier( configuration_key=checkpoint_name_1, ) checkpoint_store.set(key=key_1, value=my_checkpoint_config_1) assert len(checkpoint_store.list_keys()) == 2 assert filter_properties_dict(properties=checkpoint_store.get( key=key_1).to_json_dict(), ) == filter_properties_dict( properties=my_checkpoint_config_1.to_json_dict(), ) dir_tree: str = gen_directory_tree_str(startpath=base_directory) assert (dir_tree == """checkpoints/ .ge_store_backend_id my_checkpoint_0.yml my_checkpoint_1.yml """) self_check_report: dict = convert_to_json_serializable( data=checkpoint_store.self_check()) assert self_check_report == { "keys": ["my_checkpoint_0", "my_checkpoint_1"], "len_keys": 2, "config": { "store_name": "checkpoint_store", "class_name": "CheckpointStore", "module_name": "great_expectations.data_context.store.checkpoint_store", "overwrite_existing": True, "store_backend": { "base_directory": f"{empty_data_context.root_directory}/checkpoints", "platform_specific_separator": True, "fixed_length_key": False, "suppress_store_backend_id": False, "module_name": "great_expectations.data_context.store.tuple_store_backend", "class_name": "TupleFilesystemStoreBackend", "filepath_template": "{0}.yml", }, }, } checkpoint_store.remove_key(key=key_0) checkpoint_store.remove_key(key=key_1) assert len(checkpoint_store.list_keys()) == 0