def _upgrade_configuration_automatically(self): if not self.upgrade_log["skipped_checkpoint_store_upgrade"]: config_commented_map: CommentedMap = ( self.data_context.get_config().commented_map) for key, config in self.upgrade_checklist["automatic"][ "stores"].items(): config_commented_map["stores"][key] = config for key, value in self.upgrade_checklist["automatic"][ "store_names"].items(): config_commented_map[key] = value data_context_config: DataContextConfig = ( DataContextConfig.from_commented_map( commented_map=config_commented_map)) self.data_context.set_config(project_config=data_context_config) self.data_context._save_project_config() checkpoint_log_entry = { "stores": { DataContextConfigDefaults.DEFAULT_CHECKPOINT_STORE_NAME.value: data_context_config.stores[ DataContextConfigDefaults. DEFAULT_CHECKPOINT_STORE_NAME.value], }, "checkpoint_store_name": data_context_config.checkpoint_store_name, } self.upgrade_log["added_checkpoint_store"].update( checkpoint_log_entry)
def test_substituted_config_variables_not_written_to_file(tmp_path_factory): # this test uses a great_expectations.yml with almost all values replaced # with substitution variables project_path = str(tmp_path_factory.mktemp("data_context")) context_path = os.path.join(project_path, "great_expectations") asset_config_path = os.path.join(context_path, "expectations") create_data_context_files( context_path, asset_config_path, ge_config_fixture_filename= "great_expectations_basic_with_exhaustive_variables.yml", config_variables_fixture_filename="config_variables_exhaustive.yml", ) # load ge config fixture for expected path_to_yml = ( "../test_fixtures/great_expectations_basic_with_exhaustive_variables.yml" ) path_to_yml = file_relative_path(__file__, path_to_yml) with open(path_to_yml) as data: config_dict = yaml.load(data) expected_config = DataContextConfig.from_commented_map(config_dict) expected_config_dict = dataContextConfigSchema.dump(expected_config) expected_config_dict.pop("anonymous_usage_statistics") # instantiate data_context twice to go through cycle of loading config from file then saving context = ge.data_context.DataContext(context_path) context._save_project_config() context_config_dict = dataContextConfigSchema.dump( ge.data_context.DataContext(context_path)._project_config) context_config_dict.pop("anonymous_usage_statistics") assert context_config_dict == expected_config_dict
def _upgrade_configuration(self): if self.upgrade_log["skipped_upgrade"]: return config_commented_map: CommentedMap = ( self.data_context.get_config().commented_map) for name, value in self.upgrade_checklist.items(): if isinstance(value, dict): for key, config in value.items(): config_commented_map[name][key] = config else: config_commented_map[name] = value data_context_config: DataContextConfig = DataContextConfig.from_commented_map( commented_map=config_commented_map) self.data_context.set_config(project_config=data_context_config) # noinspection PyProtectedMember self.data_context._save_project_config() self._update_upgrade_log()
def _set(self, key: Tuple[str, ...], value: Any, **kwargs: dict) -> None: ( resource_type, resource_name, ) = InlineStoreBackend._determine_resource_type_and_name_from_key(key) project_config: DataContextConfig = self._data_context.config if resource_type is DataContextVariableSchema.ALL_VARIABLES: config_commented_map_from_yaml = yaml.load(value) value = DataContextConfig.from_commented_map( commented_map=config_commented_map_from_yaml) self._data_context.set_config(value) elif resource_name is not None: project_config[resource_type][resource_name] = value else: project_config[resource_type] = value self._save_changes()
def _load_project_config(self): """ Reads the project configuration from the project configuration file. The file may contain ${SOME_VARIABLE} variables - see self.project_config_with_variables_substituted for how these are substituted. For Data Contexts in GE Cloud mode, a user-specific template is retrieved from the Cloud API - see self._retrieve_data_context_config_from_ge_cloud for more details. :return: the configuration object read from the file or template """ if self.ge_cloud_mode: config = self._retrieve_data_context_config_from_ge_cloud() return config path_to_yml = os.path.join(self._context_root_directory, self.GE_YML) try: with open(path_to_yml) as data: config_commented_map_from_yaml = yaml.load(data) except YAMLError as err: raise ge_exceptions.InvalidConfigurationYamlError( "Your configuration file is not a valid yml file likely due to a yml syntax error:\n\n{}".format( err ) ) except DuplicateKeyError: raise ge_exceptions.InvalidConfigurationYamlError( "Error: duplicate key found in project YAML file." ) except OSError: raise ge_exceptions.ConfigNotFoundError() try: return DataContextConfig.from_commented_map( commented_map=config_commented_map_from_yaml ) except ge_exceptions.InvalidDataContextConfigError: # Just to be explicit about what we intended to catch raise