def test_substitute_env_var_in_config_variable_file(monkeypatch):
monkeypatch.setenv("FOO", "val_of_arg_0")
config_variables_dict = {
"arg0": "${FOO}",
"arg2": {
"v1": 2
},
"replace_me": "wrong"
}
assert (substitute_config_variable(
"abc${arg0}", config_variables_dict) == "abcval_of_arg_0")
monkeypatch.delenv("FOO")
with pytest.raises(MissingConfigVariableError):
substitute_config_variable("abc${arg0}", config_variables_dict)
with open(
file_relative_path(
__file__,
"../test_fixtures/great_expectations_basic_with_variables.yml",
)) as f:
config = yaml.load(f)
monkeypatch.setenv("replace_me", "correct")
# this is how dict is created in data_context.get_config_with_variables_substituted, for env var override
config_variables_dict = {
**config_variables_dict,
**dict(os.environ),
}
config = substitute_all_config_variables(config, config_variables_dict)
assert (
config["datasources"]["mydatasource"]["batch_kwargs_generators"]
["mygenerator"]["reader_options"]["test_variable_sub1"] == "correct")
def _substitute_config_variables(
self, config: CheckpointConfig) -> CheckpointConfig:
substituted_config_variables = substitute_all_config_variables(
self.data_context.config_variables,
dict(os.environ),
self.data_context.DOLLAR_SIGN_ESCAPE_STRING,
)
substitutions = {
**substituted_config_variables,
**dict(os.environ),
**self.data_context.runtime_environment,
}
return CheckpointConfig(**substitute_all_config_variables(
config, substitutions,
self.data_context.DOLLAR_SIGN_ESCAPE_STRING))
def _substitute_config_variables(self, config: dict) -> dict:
substituted_config_variables = substitute_all_config_variables(
self.data_context.config_variables,
dict(os.environ),
self.data_context.DOLLAR_SIGN_ESCAPE_STRING,
)
substitutions = {
**substituted_config_variables,
**dict(os.environ),
**self.data_context.runtime_environment,
}
return substitute_all_config_variables(
data=config,
replace_variables_dict=substitutions,
dollar_sign_escape_string=self.data_context.
DOLLAR_SIGN_ESCAPE_STRING,
)
def get_config_with_variables_substituted(
self,
config: Optional[DataContextConfig] = None) -> DataContextConfig:
"""
Substitute vars in config of form ${var} or $(var) with values found in the following places,
in order of precedence: ge_cloud_config (for Data Contexts in GE Cloud mode), runtime_environment,
environment variables, config_variables, or ge_cloud_config_variable_defaults (allows certain variables to
be optional in GE Cloud mode).
"""
if not config:
config = self.config
substitutions: dict = self._determine_substitutions()
ge_cloud_config_variable_defaults = {
"plugins_directory":
self._normalize_absolute_or_relative_path(
path=DataContextConfigDefaults.DEFAULT_PLUGINS_DIRECTORY.value
),
"usage_statistics_url":
DEFAULT_USAGE_STATISTICS_URL,
}
for config_variable, value in ge_cloud_config_variable_defaults.items(
):
if substitutions.get(config_variable) is None:
logger.info(
f'Config variable "{config_variable}" was not found in environment or global config ('
f'{self.GLOBAL_CONFIG_PATHS}). Using default value "{value}" instead. If you would '
f"like to "
f"use a different value, please specify it in an environment variable or in a "
f"great_expectations.conf file located at one of the above paths, in a section named "
f'"ge_cloud_config".')
substitutions[config_variable] = value
return DataContextConfig(**substitute_all_config_variables(
config, substitutions, self.DOLLAR_SIGN_ESCAPE_STRING))
def test_yaml_config(
self,
yaml_config: str,
name=None,
pretty_print=True,
return_mode="instantiated_class",
shorten_tracebacks=False,
):
""" Convenience method for testing yaml configs
test_yaml_config is a convenience method for configuring the moving
parts of a Great Expectations deployment. It allows you to quickly
test out configs for system components, especially Datasources,
Checkpoints, and Stores.
For many deployments of Great Expectations, these components (plus
Expectations) are the only ones you'll need.
test_yaml_config is mainly intended for use within notebooks and tests.
Parameters
----------
yaml_config : str
A string containing the yaml config to be tested
name: str
(Optional) A string containing the name of the component to instantiate
pretty_print : bool
Determines whether to print human-readable output
return_mode : str
Determines what type of object test_yaml_config will return
Valid modes are "instantiated_class" and "report_object"
shorten_tracebacks : bool
If true, catch any errors during instantiation and print only the
last element of the traceback stack. This can be helpful for
rapid iteration on configs in a notebook, because it can remove
the need to scroll up and down a lot.
Returns
-------
The instantiated component (e.g. a Datasource)
OR
a json object containing metadata from the component's self_check method
The returned object is determined by return_mode.
"""
if pretty_print:
print("Attempting to instantiate class from config...")
if not return_mode in ["instantiated_class", "report_object"]:
raise ValueError(f"Unknown return_mode: {return_mode}.")
substituted_config_variables = substitute_all_config_variables(
self.config_variables,
dict(os.environ),
)
substitutions = {
**substituted_config_variables,
**dict(os.environ),
**self.runtime_environment,
}
config_str_with_substituted_variables = substitute_all_config_variables(
yaml_config,
substitutions,
)
config = yaml.load(config_str_with_substituted_variables)
if "class_name" in config:
class_name = config["class_name"]
else:
class_name = None
try:
if class_name in [
"ExpectationsStore",
"ValidationsStore",
"HtmlSiteStore",
"EvaluationParameterStore",
"MetricStore",
"SqlAlchemyQueryStore",
]:
print(
f"\tInstantiating as a Store, since class_name is {class_name}"
)
instantiated_class = self._build_store_from_config(
"my_temp_store", config)
elif class_name in [
"Datasource",
"SimpleSqlalchemyDatasource",
]:
print(
f"\tInstantiating as a Datasource, since class_name is {class_name}"
)
datasource_name = name or "my_temp_datasource"
instantiated_class = self._build_datasource_from_config(
datasource_name,
config,
)
else:
print(
"\tNo matching class found. Attempting to instantiate class from the raw config..."
)
instantiated_class = instantiate_class_from_config(
config, runtime_environment={}, config_defaults={})
if pretty_print:
print(
f"\tSuccessfully instantiated {instantiated_class.__class__.__name__}"
)
print()
report_object = instantiated_class.self_check(pretty_print)
if return_mode == "instantiated_class":
return instantiated_class
elif return_mode == "report_object":
return report_object
except Exception as e:
if shorten_tracebacks:
traceback.print_exc(limit=1)
else:
raise (e)
def _get(self, attr: DataContextVariableSchema) -> Any:
key: str = attr.value
val: Any = self.config[key]
substituted_val: Any = substitute_all_config_variables(
val, self.substitutions)
return substituted_val