def test_substitute_env_var_in_config_variable_file(monkeypatch): monkeypatch.setenv("FOO", "val_of_arg_0") config_variables_dict = { "arg0": "${FOO}", "arg2": { "v1": 2 }, "replace_me": "wrong" } assert (substitute_config_variable( "abc${arg0}", config_variables_dict) == "abcval_of_arg_0") monkeypatch.delenv("FOO") with pytest.raises(MissingConfigVariableError): substitute_config_variable("abc${arg0}", config_variables_dict) with open( file_relative_path( __file__, "../test_fixtures/great_expectations_basic_with_variables.yml", )) as f: config = yaml.load(f) monkeypatch.setenv("replace_me", "correct") # this is how dict is created in data_context.get_config_with_variables_substituted, for env var override config_variables_dict = { **config_variables_dict, **dict(os.environ), } config = substitute_all_config_variables(config, config_variables_dict) assert ( config["datasources"]["mydatasource"]["batch_kwargs_generators"] ["mygenerator"]["reader_options"]["test_variable_sub1"] == "correct")
def _substitute_config_variables( self, config: CheckpointConfig) -> CheckpointConfig: substituted_config_variables = substitute_all_config_variables( self.data_context.config_variables, dict(os.environ), self.data_context.DOLLAR_SIGN_ESCAPE_STRING, ) substitutions = { **substituted_config_variables, **dict(os.environ), **self.data_context.runtime_environment, } return CheckpointConfig(**substitute_all_config_variables( config, substitutions, self.data_context.DOLLAR_SIGN_ESCAPE_STRING))
def _substitute_config_variables(self, config: dict) -> dict: substituted_config_variables = substitute_all_config_variables( self.data_context.config_variables, dict(os.environ), self.data_context.DOLLAR_SIGN_ESCAPE_STRING, ) substitutions = { **substituted_config_variables, **dict(os.environ), **self.data_context.runtime_environment, } return substitute_all_config_variables( data=config, replace_variables_dict=substitutions, dollar_sign_escape_string=self.data_context. DOLLAR_SIGN_ESCAPE_STRING, )
def get_config_with_variables_substituted( self, config: Optional[DataContextConfig] = None) -> DataContextConfig: """ Substitute vars in config of form ${var} or $(var) with values found in the following places, in order of precedence: ge_cloud_config (for Data Contexts in GE Cloud mode), runtime_environment, environment variables, config_variables, or ge_cloud_config_variable_defaults (allows certain variables to be optional in GE Cloud mode). """ if not config: config = self.config substitutions: dict = self._determine_substitutions() ge_cloud_config_variable_defaults = { "plugins_directory": self._normalize_absolute_or_relative_path( path=DataContextConfigDefaults.DEFAULT_PLUGINS_DIRECTORY.value ), "usage_statistics_url": DEFAULT_USAGE_STATISTICS_URL, } for config_variable, value in ge_cloud_config_variable_defaults.items( ): if substitutions.get(config_variable) is None: logger.info( f'Config variable "{config_variable}" was not found in environment or global config (' f'{self.GLOBAL_CONFIG_PATHS}). Using default value "{value}" instead. If you would ' f"like to " f"use a different value, please specify it in an environment variable or in a " f"great_expectations.conf file located at one of the above paths, in a section named " f'"ge_cloud_config".') substitutions[config_variable] = value return DataContextConfig(**substitute_all_config_variables( config, substitutions, self.DOLLAR_SIGN_ESCAPE_STRING))
def test_yaml_config( self, yaml_config: str, name=None, pretty_print=True, return_mode="instantiated_class", shorten_tracebacks=False, ): """ Convenience method for testing yaml configs test_yaml_config is a convenience method for configuring the moving parts of a Great Expectations deployment. It allows you to quickly test out configs for system components, especially Datasources, Checkpoints, and Stores. For many deployments of Great Expectations, these components (plus Expectations) are the only ones you'll need. test_yaml_config is mainly intended for use within notebooks and tests. Parameters ---------- yaml_config : str A string containing the yaml config to be tested name: str (Optional) A string containing the name of the component to instantiate pretty_print : bool Determines whether to print human-readable output return_mode : str Determines what type of object test_yaml_config will return Valid modes are "instantiated_class" and "report_object" shorten_tracebacks : bool If true, catch any errors during instantiation and print only the last element of the traceback stack. This can be helpful for rapid iteration on configs in a notebook, because it can remove the need to scroll up and down a lot. Returns ------- The instantiated component (e.g. a Datasource) OR a json object containing metadata from the component's self_check method The returned object is determined by return_mode. """ if pretty_print: print("Attempting to instantiate class from config...") if not return_mode in ["instantiated_class", "report_object"]: raise ValueError(f"Unknown return_mode: {return_mode}.") substituted_config_variables = substitute_all_config_variables( self.config_variables, dict(os.environ), ) substitutions = { **substituted_config_variables, **dict(os.environ), **self.runtime_environment, } config_str_with_substituted_variables = substitute_all_config_variables( yaml_config, substitutions, ) config = yaml.load(config_str_with_substituted_variables) if "class_name" in config: class_name = config["class_name"] else: class_name = None try: if class_name in [ "ExpectationsStore", "ValidationsStore", "HtmlSiteStore", "EvaluationParameterStore", "MetricStore", "SqlAlchemyQueryStore", ]: print( f"\tInstantiating as a Store, since class_name is {class_name}" ) instantiated_class = self._build_store_from_config( "my_temp_store", config) elif class_name in [ "Datasource", "SimpleSqlalchemyDatasource", ]: print( f"\tInstantiating as a Datasource, since class_name is {class_name}" ) datasource_name = name or "my_temp_datasource" instantiated_class = self._build_datasource_from_config( datasource_name, config, ) else: print( "\tNo matching class found. Attempting to instantiate class from the raw config..." ) instantiated_class = instantiate_class_from_config( config, runtime_environment={}, config_defaults={}) if pretty_print: print( f"\tSuccessfully instantiated {instantiated_class.__class__.__name__}" ) print() report_object = instantiated_class.self_check(pretty_print) if return_mode == "instantiated_class": return instantiated_class elif return_mode == "report_object": return report_object except Exception as e: if shorten_tracebacks: traceback.print_exc(limit=1) else: raise (e)
def _get(self, attr: DataContextVariableSchema) -> Any: key: str = attr.value val: Any = self.config[key] substituted_val: Any = substitute_all_config_variables( val, self.substitutions) return substituted_val