def _upgrade_configuration_automatically(self):
if not self.upgrade_log["skipped_checkpoint_store_upgrade"]:
config_commented_map: CommentedMap = (
self.data_context.get_config().commented_map)
for key, config in self.upgrade_checklist["automatic"][
"stores"].items():
config_commented_map["stores"][key] = config
for key, value in self.upgrade_checklist["automatic"][
"store_names"].items():
config_commented_map[key] = value
data_context_config: DataContextConfig = (
DataContextConfig.from_commented_map(
commented_map=config_commented_map))
self.data_context.set_config(project_config=data_context_config)
self.data_context._save_project_config()
checkpoint_log_entry = {
"stores": {
DataContextConfigDefaults.DEFAULT_CHECKPOINT_STORE_NAME.value:
data_context_config.stores[
DataContextConfigDefaults.
DEFAULT_CHECKPOINT_STORE_NAME.value],
},
"checkpoint_store_name":
data_context_config.checkpoint_store_name,
}
self.upgrade_log["added_checkpoint_store"].update(
checkpoint_log_entry)
def test_substituted_config_variables_not_written_to_file(tmp_path_factory):
# this test uses a great_expectations.yml with almost all values replaced
# with substitution variables
project_path = str(tmp_path_factory.mktemp("data_context"))
context_path = os.path.join(project_path, "great_expectations")
asset_config_path = os.path.join(context_path, "expectations")
create_data_context_files(
context_path,
asset_config_path,
ge_config_fixture_filename=
"great_expectations_basic_with_exhaustive_variables.yml",
config_variables_fixture_filename="config_variables_exhaustive.yml",
)
# load ge config fixture for expected
path_to_yml = (
"../test_fixtures/great_expectations_basic_with_exhaustive_variables.yml"
)
path_to_yml = file_relative_path(__file__, path_to_yml)
with open(path_to_yml) as data:
config_dict = yaml.load(data)
expected_config = DataContextConfig.from_commented_map(config_dict)
expected_config_dict = dataContextConfigSchema.dump(expected_config)
expected_config_dict.pop("anonymous_usage_statistics")
# instantiate data_context twice to go through cycle of loading config from file then saving
context = ge.data_context.DataContext(context_path)
context._save_project_config()
context_config_dict = dataContextConfigSchema.dump(
ge.data_context.DataContext(context_path)._project_config)
context_config_dict.pop("anonymous_usage_statistics")
assert context_config_dict == expected_config_dict
def _upgrade_configuration(self):
if self.upgrade_log["skipped_upgrade"]:
return
config_commented_map: CommentedMap = (
self.data_context.get_config().commented_map)
for name, value in self.upgrade_checklist.items():
if isinstance(value, dict):
for key, config in value.items():
config_commented_map[name][key] = config
else:
config_commented_map[name] = value
data_context_config: DataContextConfig = DataContextConfig.from_commented_map(
commented_map=config_commented_map)
self.data_context.set_config(project_config=data_context_config)
# noinspection PyProtectedMember
self.data_context._save_project_config()
self._update_upgrade_log()
def _set(self, key: Tuple[str, ...], value: Any, **kwargs: dict) -> None:
(
resource_type,
resource_name,
) = InlineStoreBackend._determine_resource_type_and_name_from_key(key)
project_config: DataContextConfig = self._data_context.config
if resource_type is DataContextVariableSchema.ALL_VARIABLES:
config_commented_map_from_yaml = yaml.load(value)
value = DataContextConfig.from_commented_map(
commented_map=config_commented_map_from_yaml)
self._data_context.set_config(value)
elif resource_name is not None:
project_config[resource_type][resource_name] = value
else:
project_config[resource_type] = value
self._save_changes()
def _load_project_config(self):
"""
Reads the project configuration from the project configuration file.
The file may contain ${SOME_VARIABLE} variables - see self.project_config_with_variables_substituted
for how these are substituted.
For Data Contexts in GE Cloud mode, a user-specific template is retrieved from the Cloud API
- see self._retrieve_data_context_config_from_ge_cloud for more details.
:return: the configuration object read from the file or template
"""
if self.ge_cloud_mode:
config = self._retrieve_data_context_config_from_ge_cloud()
return config
path_to_yml = os.path.join(self._context_root_directory, self.GE_YML)
try:
with open(path_to_yml) as data:
config_commented_map_from_yaml = yaml.load(data)
except YAMLError as err:
raise ge_exceptions.InvalidConfigurationYamlError(
"Your configuration file is not a valid yml file likely due to a yml syntax error:\n\n{}".format(
err
)
)
except DuplicateKeyError:
raise ge_exceptions.InvalidConfigurationYamlError(
"Error: duplicate key found in project YAML file."
)
except OSError:
raise ge_exceptions.ConfigNotFoundError()
try:
return DataContextConfig.from_commented_map(
commented_map=config_commented_map_from_yaml
)
except ge_exceptions.InvalidDataContextConfigError:
# Just to be explicit about what we intended to catch
raise
