def test_runtime_environment_are_used_preferentially(tmp_path_factory):
value_from_environment = "from_environment"
os.environ["replace_me"] = value_from_environment
value_from_runtime_override = "runtime_var"
runtime_environment = {"replace_me": value_from_runtime_override}
project_path = str(tmp_path_factory.mktemp("data_context"))
context_path = os.path.join(project_path, "great_expectations")
asset_config_path = os.path.join(context_path, "expectations")
create_data_context_files(context_path, asset_config_path, True)
data_context = ge.data_context.DataContext(
context_path, runtime_environment=runtime_environment
)
config = data_context.get_config_with_variables_substituted()
try:
assert (
config.datasources["mydatasource"]["batch_kwargs_generators"][
"mygenerator"
]["reader_options"]["test_variable_sub1"]
== value_from_runtime_override
)
assert (
config.datasources["mydatasource"]["batch_kwargs_generators"][
"mygenerator"
]["reader_options"]["test_variable_sub2"]
== value_from_runtime_override
)
except Exception:
raise
finally:
del os.environ["replace_me"]
def test_substituted_config_variables_not_written_to_file(tmp_path_factory):
# this test uses a great_expectations.yml with almost all values replaced
# with substitution variables
project_path = str(tmp_path_factory.mktemp("data_context"))
context_path = os.path.join(project_path, "great_expectations")
asset_config_path = os.path.join(context_path, "expectations")
create_data_context_files(
context_path,
asset_config_path,
ge_config_fixture_filename=
"great_expectations_basic_with_exhaustive_variables.yml",
config_variables_fixture_filename="config_variables_exhaustive.yml",
)
# load ge config fixture for expected
path_to_yml = (
"../test_fixtures/great_expectations_basic_with_exhaustive_variables.yml"
)
path_to_yml = file_relative_path(__file__, path_to_yml)
with open(path_to_yml) as data:
config_dict = yaml.load(data)
expected_config = DataContextConfig.from_commented_map(config_dict)
expected_config_dict = dataContextConfigSchema.dump(expected_config)
expected_config_dict.pop("anonymous_usage_statistics")
# instantiate data_context twice to go through cycle of loading config from file then saving
context = ge.data_context.DataContext(context_path)
context._save_project_config()
context_config_dict = dataContextConfigSchema.dump(
ge.data_context.DataContext(context_path)._project_config)
context_config_dict.pop("anonymous_usage_statistics")
assert context_config_dict == expected_config_dict
def test_runtime_environment_are_used_preferentially(tmp_path_factory, monkeypatch):
monkeypatch.setenv("FOO", "BAR")
monkeypatch.setenv("REPLACE_ME_ESCAPED_ENV", r"ive_been_\$replaced")
value_from_environment = "from_environment"
os.environ["replace_me"] = value_from_environment
value_from_runtime_override = "runtime_var"
runtime_environment = {"replace_me": value_from_runtime_override}
project_path = str(tmp_path_factory.mktemp("data_context"))
context_path = os.path.join(project_path, "great_expectations")
asset_config_path = os.path.join(context_path, "expectations")
create_data_context_files(
context_path,
asset_config_path,
ge_config_fixture_filename="great_expectations_basic_with_variables.yml",
config_variables_fixture_filename="config_variables.yml",
)
data_context = ge.data_context.DataContext(
context_path, runtime_environment=runtime_environment
)
config = data_context.get_config_with_variables_substituted()
try:
assert (
config.datasources["mydatasource"]["batch_kwargs_generators"][
"mygenerator"
]["reader_options"]["test_variable_sub1"]
== value_from_runtime_override
)
assert (
config.datasources["mydatasource"]["batch_kwargs_generators"][
"mygenerator"
]["reader_options"]["test_variable_sub2"]
== value_from_runtime_override
)
except Exception:
raise
finally:
del os.environ["replace_me"]
