def __init__(self, assimilation_configs=None, output_configs=None):
self.assimilation_configs = self.validate_assimilation_configs(
assimilation_configs, FilteringProcess._def_assimilation_configs)
# extract assimilation configurations for easier access
self.filter = self.assimilation_configs['filter']
try:
self.model = self.filter.filter_configs['model']
except (KeyError, NameError, AttributeError):
self.model = self.filter.model
self.da_checkpoints = self.assimilation_configs['da_checkpoints']
self.obs_checkpoints = self.assimilation_configs['obs_checkpoints']
self.synchronous = self.assimilation_configs['synchronous']
self.ref_initial_time = self.assimilation_configs['ref_initial_time']
self.ref_initial_condition = self.assimilation_configs[
'ref_initial_condition']
self.forecast_first = self.assimilation_configs['forecast_first']
#
self._callback = self.assimilation_configs['callback']
self._callback_args = self.assimilation_configs['callback_args']
# extract output configurations for easier access
self.output_configs = self.validate_output_configs(
output_configs, FilteringProcess._def_output_configs)
self.scr_output = self.output_configs['scr_output']
self.scr_output_iter = self.output_configs['scr_output_iter']
self.file_output = self.output_configs['file_output']
self.file_output_iter = self.output_configs['file_output_iter']
self.file_output_dir = self.output_configs['file_output_dir']
self.file_output_variables = self.output_configs[
'file_output_variables']
self.random_seed = self.assimilation_configs['random_seed']
# Make sure the directory is created and cleaned up at this point...
self.set_filtering_output_dir(self.file_output_dir,
rel_to_root_dir=True)
#
model_conf = self.model.model_configs.copy()
filter_conf = self.filter.filter_configs.copy()
filter_conf.update({'model': None})
assim_conf = self.assimilation_configs.copy()
assim_conf.update({'filter': None})
file_output_dir = self.output_configs['file_output_dir']
utility.write_dicts_to_config_file(
'setup.pickle', file_output_dir,
[model_conf, filter_conf, assim_conf],
['Model Configs', 'Filter Configs', 'Assimilation Configs'], True)
def save_cycle_results(self,
output_dir=None,
cleanup_out_dir=False,
save_err_covars=False):
"""
Save filtering results from the current cycle to file(s).
A check on the correspondidng options in the configurations dictionary is made to make sure
saving is requested.
Args:
out_dir (default None): directory to put results in.
The output_dir is created (with all necessary parent paths) if it is not on disc.
The directory is relative to DATeS root directory.
cleanup_out_dir (default None): bool,
Takes effect if the output directory is not empty. True: remove directory contents.
"""
# Retrieve output configurations
output_configs = self.output_configs
file_output = output_configs['file_output']
if not file_output:
raise ValueError(
"The output flag is turned of. The method 'save_cycle_results' is called though!"
)
# We are good to go! --> Start preparing directories (if necessary) then save results...
if output_dir is not None:
file_output_directory = output_dir
else:
file_output_directory = output_configs['file_output_dir']
# clean-up output directory; this is set to true only if the filter is called once, otherwise filtering_process should handle it.
if cleanup_out_dir:
parent_path, out_dir = os.path.split(file_output_directory)
utility.cleanup_directory(directory_name=out_dir,
parent_path=parent_path)
# check the output sub-directories...
filter_statistics_dir = os.path.join(
file_output_directory, output_configs['filter_statistics_dir'])
model_states_dir = os.path.join(file_output_directory,
output_configs['model_states_dir'])
observations_dir = os.path.join(file_output_directory,
output_configs['observations_dir'])
file_output_variables = output_configs[
'file_output_variables'] # I think it's better to remove it from the filter base...
if not os.path.isdir(filter_statistics_dir):
os.makedirs(filter_statistics_dir)
if not os.path.isdir(model_states_dir):
os.makedirs(model_states_dir)
if not os.path.isdir(observations_dir):
os.makedirs(observations_dir)
# check if results are to be saved to separate files or appended on existing files.
# This may be overridden if not adequate for some output (such as model states), we will see!
file_output_separate_files = output_configs[
'file_output_separate_files']
# This is useful for saving filter statistics but not model states or observations as models should handle both
file_output_file_format = output_configs[
'file_output_file_format'].lower()
file_output_file_name_prefix = output_configs[
'file_output_file_name_prefix'] # this is useless!
# SAVING MODEL STATES (Either Moments Only or Full Ensembles)
# write cycle configurations:
model_conf = self.model.get_model_configs()
utility.write_dicts_to_config_file('setup.dat', file_output_directory,
model_conf, 'Model Configs')
# get a proper name for the folder (cycle_*) under the model_states_dir path
suffix = 0
cycle_prefix = 'cycle_'
while True:
cycle_dir = cycle_prefix + str(suffix)
cycle_states_out_dir = os.path.join(
model_states_dir, cycle_dir
) # full path where states will be saved for the current cycle
if not os.path.isdir(cycle_states_out_dir):
cycle_observations_out_dir = os.path.join(
observations_dir, cycle_dir)
if os.path.isdir(cycle_observations_out_dir):
raise IOError(
"There is inconsistency problem. Naming mismatch in cycles folders for states and observations!"
)
os.makedirs(cycle_states_out_dir)
os.makedirs(cycle_observations_out_dir)
break
else:
suffix += 1
# Now we have all directories cleaned-up and ready for outputting.
output_dir_structure_file = os.path.join(file_output_directory,
'output_dir_structure.txt')
if not os.path.isfile(output_dir_structure_file):
# First, we need to save the output paths info to a file to be used later by results' reader
# print('writing output directory structure to config file \n \t%s \n' % output_dir_structure_file)
out_dir_tree_structure = dict(
file_output_separate_files=file_output_separate_files,
file_output_directory=file_output_directory,
model_states_dir=model_states_dir,
observations_dir=observations_dir,
filter_statistics_dir=filter_statistics_dir,
cycle_prefix=cycle_prefix)
utility.write_dicts_to_config_file(
file_name='output_dir_structure.txt',
out_dir=file_output_directory,
dicts=out_dir_tree_structure,
sections_headers='out_dir_tree_structure')
# 3- save reference, forecast, and analysis states; use model to write states to file(s)
# i- save reference state
reference_state = self.filter_configs['reference_state'].copy()
self.model.write_state(state=reference_state,
directory=cycle_states_out_dir,
file_name='reference_state')
# ii- save forecast state
forecast_state = self.filter_configs['forecast_state']
self.model.write_state(state=forecast_state,
directory=cycle_states_out_dir,
file_name='forecast_state')
# iii- save analysis state
analysis_state = self.filter_configs['analysis_state'].copy()
self.model.write_state(state=analysis_state,
directory=cycle_states_out_dir,
file_name='analysis_state')
# 4- Save observations to files; use model to write observations to file(s)
observation = self.filter_configs['observation']
file_name = utility.try_file_name(directory=cycle_observations_out_dir,
file_prefix='observation')
self.model.write_observation(observation=observation,
directory=cycle_observations_out_dir,
file_name=file_name,
append=False)
# 4- Save filter configurations and statistics to file,
# i- Output the configurations dictionaries:
assim_cycle_configs_file_name = 'assim_cycle_configs'
if file_output_file_format in ['txt', 'ascii', 'mat']:
# Output filter and model configurations; this goes under state directory
assim_cycle_configs_file_name += '.txt'
utility.write_dicts_to_config_file(
assim_cycle_configs_file_name, cycle_states_out_dir,
[self.filter_configs, self.output_configs],
['Filter Configs', 'Output Configs'])
elif file_output_file_format in ['pickle']:
#
# Output filter and model configurations; this goes under state directory
assim_cycle_configs_file_name += '.pickle'
assim_cycle_configs = dict(filter_configs=self.filter_configs,
output_configs=self.output_configs)
pickle.dump(
assim_cycle_configs,
open(
os.path.join(cycle_states_out_dir,
assim_cycle_configs_file_name)))
else:
raise ValueError(
"Unsupported output format for configurations dictionaries: '%s' !"
% file_output_file_format)
#
# ii Output the RMSE results; it's meaningless to create a new file for each cycle:
rmse_file_name = 'rmse.txt' # RMSE are always saved in text files
rmse_file_path = os.path.join(filter_statistics_dir, rmse_file_name)
# Create a header for the file if it is newely created
if not os.path.isfile(rmse_file_path):
# rmse file does not exist. create file and add header.
header = "RMSE Results: DA: '%s' \n %s \t %s \n" % (
self._filter_name,
'Forecast-RMSE'.rjust(20),
'Analysis-RMSE'.rjust(20),
)
# dump the header to the file
with open(rmse_file_path, mode='w') as file_handler:
file_handler.write(header)
else:
# rmse file does exist. Header should be already there!
pass
# rmse file exists --> Append rmse results to the file.
forecast_time = self.filter_configs['forecast_time']
analysis_time = self.filter_configs['analysis_time']
observation_time = self.filter_configs['observation_time']
#
forecast_rmse = self.output_configs['filter_statistics'][
'forecast_rmse']
analysis_rmse = self.output_configs['filter_statistics'][
'analysis_rmse']
output_line = u" {0:20.14e} \t {1:20.14e} \t {2:20.14e} \t {3:20.14e} \t {4:20.14e} \n".format(
observation_time, forecast_time, analysis_time, forecast_rmse,
analysis_rmse)
#
# now write the rmse results to file
with open(rmse_file_path, mode='a') as file_handler:
file_handler.write(output_line)
#
# save error covariance matrices if requested; these will go in the state output directory
if save_err_covars:
Pf = self.filter_configs['forecast_error_covariance']
Pa = self.filter_configs['forecast_error_covariance']
print(
"Saving covariance matrices is not supported yet. CDF will be considered soon!"
)
raise NotImplementedError()
else:
pass
def save_cycle_results(self, output_dir=None, cleanup_out_dir=False):
"""
Save filtering results from the current cycle to file(s).
Check the output directory first. If the directory does not exist, create it.
Args:
output_dir: full path of the directory to save results in
Returns:
None
"""
model = self.filter_configs['model']
# Retrieve output configurations
output_configs = self.output_configs
file_output = output_configs['file_output']
if not file_output:
raise ValueError(
"The output flag is turned of. The method 'save_cycle_results' is called though!"
)
# We are good to go! --> Start preparing directories (if necessary) then save results...
if output_dir is not None:
file_output_directory = output_dir
else:
file_output_directory = output_configs['file_output_dir']
# clean-up output directory; this is set to true only if the filter is called once, otherwise filtering_process should handle it.
if cleanup_out_dir:
parent_path, out_dir = os.path.split(file_output_directory)
utility.cleanup_directory(directory_name=out_dir,
parent_path=parent_path)
# check the output sub-directories...
filter_statistics_dir = os.path.join(
file_output_directory, output_configs['filter_statistics_dir'])
model_states_dir = os.path.join(file_output_directory,
output_configs['model_states_dir'])
observations_dir = os.path.join(file_output_directory,
output_configs['observations_dir'])
file_output_variables = output_configs[
'file_output_variables'] # I think it's better to remove it from the filter base...
if not os.path.isdir(filter_statistics_dir):
os.makedirs(filter_statistics_dir)
if not os.path.isdir(model_states_dir):
os.makedirs(model_states_dir)
if not os.path.isdir(observations_dir):
os.makedirs(observations_dir)
# check if results are to be saved to separate files or appended on existing files.
# This may be overridden if not adequate for some output (such as model states), we will see!
file_output_separate_files = output_configs[
'file_output_separate_files']
# This is useful for saving filter statistics but not model states or observations as models should handle both
file_output_file_format = output_configs[
'file_output_file_format'].lower()
file_output_file_name_prefix = output_configs[
'file_output_file_name_prefix'] # this is useless!
# SAVING MODEL STATES (Either Moments Only or Full Ensembles)
# write cycle configurations:
model_conf = model.get_model_configs()
utility.write_dicts_to_config_file('setup.dat', file_output_directory,
model_conf, 'Model Configs')
# get a proper name for the folder (cycle_*) under the model_states_dir path
suffix = 0
cycle_prefix = 'cycle_'
while True:
cycle_dir = cycle_prefix + str(suffix)
cycle_states_out_dir = os.path.join(
model_states_dir, cycle_dir
) # full path where states will be saved for the current cycle
if not os.path.isdir(cycle_states_out_dir):
cycle_observations_out_dir = os.path.join(
observations_dir, cycle_dir)
if os.path.isdir(cycle_observations_out_dir):
raise IOError(
"There is inconsistency problem. Naming mismatch in cycles folders for states and observations!"
)
os.makedirs(cycle_states_out_dir)
os.makedirs(cycle_observations_out_dir)
break
else:
suffix += 1
# Now we have all directories cleaned-up and ready for outputting.
output_dir_structure_file = os.path.join(file_output_directory,
'output_dir_structure.txt')
if not os.path.isfile(output_dir_structure_file):
# First, we need to save the output paths info to a file to be used later by results' reader
# print('writing output directory structure to config file \n \t%s \n' % output_dir_structure_file)
out_dir_tree_structure = dict(
file_output_separate_files=file_output_separate_files,
file_output_directory=file_output_directory,
model_states_dir=model_states_dir,
observations_dir=observations_dir,
filter_statistics_dir=filter_statistics_dir,
cycle_prefix=cycle_prefix)
utility.write_dicts_to_config_file(
file_name='output_dir_structure.txt',
out_dir=file_output_directory,
dicts=out_dir_tree_structure,
sections_headers='out_dir_tree_structure')
# save states
file_output_moment_only = output_configs['file_output_moment_only']
if file_output_moment_only:
file_output_moment_name = output_configs[
'file_output_moment_name'].lower()
if file_output_moment_name in ['mean', 'average']:
# start outputting ensemble means... (both forecast and analysis of course).
# save forecast mean
forecast_state = self.filter_configs['forecast_state']
model.write_state(state=forecast_state,
directory=cycle_states_out_dir,
file_name='forecast_mean')
# save analysis mean
analysis_state = self.filter_configs['analysis_state']
model.write_state(state=analysis_state,
directory=cycle_states_out_dir,
file_name='analysis_mean')
else:
raise ValueError("Unsupported ensemble moment: '%s' !" %
(file_output_moment_name))
else:
# start outputting the whole ensemble members (both forecast and analysis ensembles of course).
# check if all ensembles are to be saved or just one of the supported ensemble moments
for ens_ind in xrange(self.sample_size):
if file_output_separate_files:
# print('saving ensemble member to separate files: %d' % ens_ind)
forecast_ensemble_member = self.filter_configs[
'forecast_ensemble'][ens_ind]
model.write_state(state=forecast_ensemble_member,
directory=cycle_states_out_dir,
file_name='forecast_ensemble_' +
str(ens_ind),
append=False)
#
analysis_ensemble_member = self.filter_configs[
'analysis_ensemble'][ens_ind]
model.write_state(state=analysis_ensemble_member,
directory=cycle_states_out_dir,
file_name='analysis_ensemble_' +
str(ens_ind),
append=False)
else:
# print('saving ensemble member to same file with resizing: %d' % ens_ind)
# save results to different files. For moments
forecast_ensemble_member = self.filter_configs[
'forecast_ensemble'][ens_ind]
model.write_state(state=forecast_ensemble_member.copy(),
directory=cycle_states_out_dir,
file_name='forecast_ensemble',
append=True)
#
analysis_ensemble_member = self.filter_configs[
'analysis_ensemble'][ens_ind]
model.write_state(state=analysis_ensemble_member.copy(),
directory=cycle_states_out_dir,
file_name='analysis_ensemble',
append=True)
# save reference state
reference_state = self.filter_configs['reference_state']
model.write_state(state=reference_state,
directory=cycle_states_out_dir,
file_name='reference_state')
#
# Save observation to file; use model to write observations to file(s)
# save analysis mean
observation = self.filter_configs['observation']
model.write_observation(observation=observation,
directory=cycle_observations_out_dir,
file_name='observation',
append=False)
# Save filter statistics to file
# 1- Output filter RMSEs: RMSEs are saved to the same file. It's meaningless to create a new file for each cycle
rmse_file_name = 'rmse'
if file_output_file_format in ['txt', 'ascii']:
rmse_file_name += '.dat'
rmse_file_path = os.path.join(filter_statistics_dir,
rmse_file_name)
if not os.path.isfile(rmse_file_path):
# rmse file does not exist. create file and add header.
filter_name = self.filter_configs['filter_name']
header = "RMSE Results: Filter: '%s' \n %s \t %s \t %s \t %s \t %s \n" % (
filter_name,
'Observation-Time'.rjust(20),
'Forecast-Time'.rjust(20),
'Analysis-Time'.rjust(20),
'Forecast-RMSE'.rjust(20),
'Analysis-RMSE'.rjust(20),
)
# get the initial RMSE and add it if forecast is done first...
if self.filter_configs['forecast_first']:
initial_time = self.filter_configs['timespan'][0]
initial_rmse = self.output_configs['filter_statistics'][
'initial_rmse']
header += " %20s \t %20.14e \t %20.14e \t %20.14e \t %20.14e \n" % (
'0000000', initial_time, initial_time, initial_rmse,
initial_rmse)
# dump the header to the file
with open(rmse_file_path, mode='w') as file_handler:
file_handler.write(header)
else:
pass
# rmse file exists --> Append rmse results to the file.
forecast_time = self.filter_configs['forecast_time']
analysis_time = self.filter_configs['analysis_time']
observation_time = self.filter_configs['observation_time']
#
forecast_rmse = self.output_configs['filter_statistics'][
'forecast_rmse']
analysis_rmse = self.output_configs['filter_statistics'][
'analysis_rmse']
output_line = u" {0:20.14e} \t {1:20.14e} \t {2:20.14e} \t {3:20.14e} \t {4:20.14e} \n".format(
observation_time, forecast_time, analysis_time, forecast_rmse,
analysis_rmse)
#
with open(rmse_file_path, mode='a') as file_handler:
file_handler.write(output_line)
# save filter and model configurations (a copy under observation directory and another under state directory)...
filter_configs = self.filter_configs
filter_conf = dict(
filter_name=filter_configs['filter_name'],
ensemble_size=filter_configs['ensemble_size'],
apply_preprocessing=filter_configs['apply_preprocessing'],
apply_postprocessing=filter_configs['apply_postprocessing'],
timespan=filter_configs['timespan'],
analysis_time=filter_configs['analysis_time'],
observation_time=filter_configs['observation_time'],
forecast_time=filter_configs['forecast_time'],
forecast_first=filter_configs['forecast_first'],
resampling_weights=filter_configs['normalized_weights'],
resampling_indexes=filter_configs['resampling_indexes'],
particles_likelihood=filter_configs['particles_likelihood'],
effective_sample_size=filter_configs['effective_sample_size'],
)
io_conf = output_configs
#
utility.write_dicts_to_config_file(
'setup.dat', cycle_observations_out_dir,
[filter_conf, io_conf], ['Filter Configs', 'Output Configs'])
utility.write_dicts_to_config_file(
'setup.dat', cycle_states_out_dir, [filter_conf, io_conf],
['Filter Configs', 'Output Configs'])
else:
print("Unsupported output format: '%s' !" %
file_output_file_format)
raise ValueError()
def save_cycle_results(self,
output_dir=None,
cleanup_out_dir=False,
save_err_covars=False):
"""
Save filtering results from the current cycle to file(s).
A check on the correspondidng options in the configurations dictionary is made to make sure
saving is requested.
Args:
out_dir (default None): directory to put results in.
The output_dir is created (with all necessary parent paths) if it is not on disc.
The directory is relative to DATeS root directory.
cleanup_out_dir (default None): bool,
Takes effect if the output directory is not empty. True: remove directory contents.
Returns:
None
"""
#
# The first code block that prepares the output directory can be moved to parent class later...
# Retrieve output configurations
output_configs = self.output_configs
file_output = output_configs['file_output']
if not file_output:
raise ValueError(
"The output flag is turned of. The method 'save_cycle_results' is called though!"
)
# We are good to go! --> Start preparing directories (if necessary) then save results...
if output_dir is not None:
file_output_directory = output_dir
else:
file_output_directory = output_configs['file_output_dir']
# clean-up output directory; this is set to true only if the filter is called once, otherwise filtering_process should handle it.
if cleanup_out_dir:
parent_path, out_dir = os.path.split(file_output_directory)
utility.cleanup_directory(directory_name=out_dir,
parent_path=parent_path)
# check the output sub-directories...
filter_statistics_dir = os.path.join(
file_output_directory, output_configs['filter_statistics_dir'])
model_states_dir = os.path.join(file_output_directory,
output_configs['model_states_dir'])
observations_dir = os.path.join(file_output_directory,
output_configs['observations_dir'])
file_output_variables = output_configs[
'file_output_variables'] # I think it's better to remove it from the filter base...
if not os.path.isdir(filter_statistics_dir):
os.makedirs(filter_statistics_dir)
if not os.path.isdir(model_states_dir):
os.makedirs(model_states_dir)
if not os.path.isdir(observations_dir):
os.makedirs(observations_dir)
# check if results are to be saved to separate files or appended on existing files.
# This may be overridden if not adequate for some output (such as model states), we will see!
file_output_separate_files = output_configs[
'file_output_separate_files']
# This is useful for saving filter statistics but not model states or observations as models should handle both
file_output_file_name_prefix = output_configs[
'file_output_file_name_prefix'] # this is useless!
# Format of the ouput files
file_output_file_format = output_configs[
'file_output_file_format'].lower()
if file_output_file_format not in ['mat', 'pickle', 'txt', 'ascii']:
print("The file format ['%s'] is not supported!" %
file_output_file_format)
raise ValueError()
# Retrieve filter and ouput configurations needed to be saved
filter_configs = self.filter_configs # we don't need to save all configs
filter_conf = dict(
filter_name=filter_configs['filter_name'],
prior_distribution=filter_configs['prior_distribution'],
gmm_prior_settings=filter_configs['gmm_prior_settings'],
ensemble_size=filter_configs['ensemble_size'],
apply_preprocessing=filter_configs['apply_preprocessing'],
apply_postprocessing=filter_configs['apply_postprocessing'],
timespan=filter_configs['timespan'],
analysis_time=filter_configs['analysis_time'],
observation_time=filter_configs['observation_time'],
forecast_time=filter_configs['forecast_time'],
forecast_first=filter_configs['forecast_first'])
io_conf = output_configs
# Start writing cycle settings, and reuslts:
# 1- write model configurations configurations:
model_conf = self.model.get_model_configs()
if file_output_file_format == 'pickle':
pickle.dump(
model_conf,
open(
os.path.join(file_output_directory,
'model_configs.pickle')))
elif file_output_file_format in ['txt', 'ascii',
'mat']: # 'mat' here has no effect.
utility.write_dicts_to_config_file('model_configs.txt',
file_output_directory,
model_conf, 'Model Configs')
# 2- get a proper name for the folder (cycle_*) under the model_states_dir path
suffix = 0
while True:
cycle_dir = 'cycle_' + str(suffix)
cycle_states_out_dir = os.path.join(
model_states_dir, cycle_dir
) # full path where states will be saved for the current cycle
if not os.path.isdir(cycle_states_out_dir):
cycle_observations_out_dir = os.path.join(
observations_dir, cycle_dir)
if os.path.isdir(cycle_observations_out_dir):
raise IOError(
"There is inconsistency problem. Naming mismatch in cycles folders for states and observations!"
)
os.makedirs(cycle_states_out_dir)
os.makedirs(cycle_observations_out_dir)
break
else:
suffix += 1
# 3- save reference, forecast, and analysis states; use model to write states to file(s)
# i- save reference state
reference_state = self.filter_configs['reference_state'].copy()
self.model.write_state(state=reference_state,
directory=cycle_states_out_dir,
file_name='reference_state')
# ii- save forecast state
forecast_state = self.filter_configs['forecast_state']
self.model.write_state(state=forecast_state,
directory=cycle_states_out_dir,
file_name='forecast_mean')
# iii- save analysis state
analysis_state = self.filter_configs['analysis_state']
self.model.write_state(state=analysis_state,
directory=cycle_states_out_dir,
file_name='analysis_mean')
# 4- Save observation to file; use model to write observations to file(s)
observation = self.filter_configs['observation'].copy()
self.model.write_observation(observation=observation,
directory=cycle_observations_out_dir,
file_name='observation',
append=False)
# 4- Save filter configurations and statistics to file,
# i- Output the configurations dictionaries:
assim_cycle_configs_file_name = 'assim_cycle_configs'
if file_output_file_format in ['txt', 'ascii', 'mat']:
# Output filter and model configurations; this goes under state directory
assim_cycle_configs_file_name += '.txt'
utility.write_dicts_to_config_file(
assim_cycle_configs_file_name, cycle_states_out_dir,
[filter_conf, io_conf], ['Filter Configs', 'Output Configs'])
elif file_output_file_format in ['pickle']:
#
# Output filter and model configurations; this goes under state directory
assim_cycle_configs_file_name += '.pickle'
assim_cycle_configs = dict(filter_configs=filter_conf,
output_configs=io_conf)
pickle.dump(
assim_cycle_configs,
open(
os.path.join(cycle_states_out_dir,
assim_cycle_configs_file_name)))
else:
raise ValueError(
"Unsupported output format for configurations dictionaries: '%s' !"
% file_output_file_format)
#
# ii Output the RMSE results; it's meaningless to create a new file for each cycle:
rmse_file_name = 'rmse.txt' # RMSE are always saved in text files
rmse_file_path = os.path.join(filter_statistics_dir, rmse_file_name)
# Create a header for the file if it is newely created
if not os.path.isfile(rmse_file_path):
# rmse file does not exist. create file and add header.
header = "RMSE Results: Filter: '%s' \n %s \t %s \t %s \t %s \t %s \n" % (
self._filter_name,
'Observation-Time'.rjust(20),
'Forecast-Time'.rjust(20),
'Analysis-Time'.rjust(20),
'Forecast-RMSE'.rjust(20),
'Analysis-RMSE'.rjust(20),
)
# get the initial RMSE and add it if forecast is done first...
if self.filter_configs['forecast_first']:
initial_time = self.filter_configs['timespan'][0]
initial_rmse = self.output_configs['filter_statistics'][
'initial_rmse']
header += " %20s \t %20.14e \t %20.14e \t %20.14e \t %20.14e \n" % (
'0000000', initial_time, initial_time, initial_rmse,
initial_rmse)
# dump the header to the file
with open(rmse_file_path, mode='w') as file_handler:
file_handler.write(header)
else:
# rmse file does exist. Header should be already there!
pass
# Now rmse results file exists --> Append rmse results to the file:
forecast_time = self.filter_configs['forecast_time']
analysis_time = self.filter_configs['analysis_time']
observation_time = self.filter_configs['observation_time']
#
forecast_rmse = self.output_configs['filter_statistics'][
'forecast_rmse']
analysis_rmse = self.output_configs['filter_statistics'][
'analysis_rmse']
output_line = u" {0:20.14e} \t {1:20.14e} \t {2:20.14e} \t {3:20.14e} \t {4:20.14e} \n".format(
observation_time, forecast_time, analysis_time, forecast_rmse,
analysis_rmse)
# now write the rmse results to file
with open(rmse_file_path, mode='a') as file_handler:
file_handler.write(output_line)
#
# save error covariance matrices if requested; these will go in the state output directory
if save_err_covars:
Pf = self.filter_configs['forecast_error_covariance']
Pa = self.filter_configs['forecast_error_covariance']
print(
"Saving covariance matrices is not supported yet. CDF will be considered soon!"
)
raise NotImplementedError()
else:
pass
for ensemble_size in ensemble_size_pool:
adaptive_localization = not adaptive_inflation
postfix = "EnsembleSize_%d" % ensemble_size
# Adjust settings file
if adaptive_inflation:
# get stuff from oed_adaptive_inflation
default_configs = __adaptive_infl_configs
exp_tag = "Adaptive_Inflation"
else:
# get stuff from oed_adaptive_localization
default_configs = __adaptive_loc_configs
exp_tag = "Adaptive_Localization"
default_configs.update({'ensemble_size': ensemble_size})
utility.write_dicts_to_config_file(settings_filename, this_dir,
default_configs,
'filter settings')
# Print some header
sep = "%sEnKF Experiment%s" % ('=' * 25, '=' * 25)
exp_no += 1
print(sep)
print("Experiment Number [%d] out of [%d]" %
(exp_no, num_experiments))
print(exp_tag)
print("Ensemble Size: %d" % ensemble_size)
print("%s\n" % ('=' * len(sep)))
# Prepare output path, and start assimilation
results_dir = os.path.join(__BASE_RESULTS_DIR,
"%s/%s" % (exp_tag, postfix))
def start_filtering(results_dir=None, overwrite=True, create_plots=True):
"""
"""
# Experiment Settings:
# ============================================================
# Timesetup
experiment_tspan = np.arange(0, 100.001, 0.1)
# Model settings:
num_Xs = 40
num_Ys = 32
F = 8.0
observation_size = num_Xs
# Filter settings:
ensemble_size = 25
#
# ============================================================
# Create a model object for the truth
try:
_, cpld_ref_IC, cpld_ref_trajectory, cpld_init_ensemble, cpld_observations = get_model_info(
experiment_tspan, ensemble_size)
except ValueError:
coupled_model_configs = dict(
num_prognostic_variables=[
num_Xs, num_Ys
], # [X, Y's per each X] each of the 8 is coupled to all 32
force=F, # forcing term: F
subgrid_varibles_parameters=[1.0, 10.0, 10.0], # (h, c, b)
# create_background_errors_correlations=True,
observation_opertor_type='linear-coarse',
observation_noise_level=0.05,
background_noise_level=0.08)
coupled_model = Coupled_Lorenz(coupled_model_configs)
# Get Repo Info:
_, cpld_ref_IC, cpld_ref_trajectory, cpld_init_ensemble, cpld_observations = get_model_info(
experiment_tspan, ensemble_size, coupled_model)
del coupled_model, coupled_model_configs
#
# Create a forecast model: i.e. use the reduced version Lorenz-96 with 40 variables
model_configs = {
'create_background_errors_correlations': True,
'num_prognostic_variables': num_Xs,
'observation_error_variances': 0.05,
# 'observation_noise_level':0.05,
'observation_vector_size':
observation_size, # observe everything first
'background_noise_level': 0.08
}
model = Lorenz96(model_configs)
model_name = model._model_name
# return is in NumPy format
# convert entities to model-based formats
state_size = model.state_size()
obs_size = model.observation_size()
ref_IC = model.state_vector(cpld_ref_IC[:state_size].copy())
ref_trajectory = []
observations = []
for i in xrange(len(experiment_tspan)):
state = model.state_vector(cpld_ref_trajectory[:state_size, i])
ref_trajectory.append(state)
obs = model.state_vector(cpld_observations[:state_size, i])
observations.append(model.evaluate_theoretical_observation(obs))
# Create initial ensemble...
init_ensemble = model.create_initial_ensemble(ensemble_size=ensemble_size)
# init_ensemble = utility.inflate_ensemble(init_ensemble, 4, in_place=True)
print(
"Lorenz model and corresponding observations created. Starting the Assimilation section"
)
#
# ======================================================================================== #
# Inititalize the filter object #
# ======================================================================================== #
# Filter Configs
# read settings from input file
settings_filename = __FILTER_CONFIGS_FILENAME
default_configs = __DEF_FILTER_CONFIGS
if os.path.isfile(settings_filename):
_, parser = utility.read_configs(settings_filename)
section_name = 'filter settings'
if not parser.has_section(section_name):
# No configurations found: set defaults
print(
"Configurations file found, with nothing in it! Setting to defaults"
)
this_dir = os.path.abspath(os.path.dirname(__file__))
utility.write_dicts_to_config_file(settings_filename, this_dir,
default_configs,
'filter settings')
fetched = False
else:
adaptive_inflation = parser.getboolean(section_name,
'adaptive_inflation')
inflation_bounds = eval(
parser.get(section_name, 'inflation_bounds'))
inflation_design_penalty = parser.getfloat(
section_name, 'inflation_design_penalty')
inflation_factor = parser.getfloat(section_name,
'inflation_factor')
forecast_inflation_factor = parser.getfloat(
section_name, 'forecast_inflation_factor')
#
adaptive_localization = parser.getboolean(section_name,
'adaptive_localization')
localization_function = parser.get(section_name,
'localization_function')
localization_radius = parser.getfloat(section_name,
'localization_radius')
localization_design_penalty = parser.getfloat(
section_name, 'localization_design_penalty')
localization_bounds = eval(
parser.get(section_name, 'localization_bounds'))
loc_direct_approach = parser.getint(section_name,
'loc_direct_approach')
localization_space = parser.get(
section_name, 'localization_space').upper().strip()
#
regularization_norm = parser.get(
section_name, 'regularization_norm').lower().strip()
moving_average_radius = parser.getint(section_name,
'moving_average_radius')
ensemble_size = parser.getint(section_name, 'ensemble_size')
#
fetched = True
else:
print("Couldn't find configs file: %s" % settings_filename)
print("Added the default values to this config file for later use...")
this_dir = os.path.abspath(os.path.dirname(__file__))
utility.write_dicts_to_config_file(settings_filename, this_dir,
default_configs, 'filter settings')
fetched = False
if not fetched:
print("Gettings things from default dict")
for k in default_configs:
exec("%s = default_configs['%s']" % (k, k))
#
#
# Both are now implemented in Adaptive OED-EnKF ; we will test both
if adaptive_inflation and adaptive_localization:
forecast_inflation_factor = inflation_factor = 1.0
if results_dir is None:
results_dir = __BASE_RESULTS_DIR + '_ADAPTIVE_INFL_LOC'
results_dir = os.path.join(
results_dir, 'InflPenalty_%f' % (inflation_design_penalty))
#
elif adaptive_inflation:
forecast_inflation_factor = inflation_factor = 1.0
if results_dir is None:
results_dir = __BASE_RESULTS_DIR + '_ADAPTIVE_INFL'
results_dir = os.path.join(
results_dir, 'LocRad_%f_InflPenalty_%f' %
(localization_radius, inflation_design_penalty))
#
elif adaptive_localization:
if results_dir is None:
results_dir = __BASE_RESULTS_DIR + '_ADAPTIVE_LOC'
results_dir = os.path.join(
results_dir, 'InflFac_%f_LocPenalty_%f' %
(forecast_inflation_factor, localization_design_penalty))
else:
results_dir = __BASE_RESULTS_DIR + '_NonAdaptive'
inflation_factor = forecast_inflation_factor
results_dir = os.path.join(
results_dir,
'InflFac_%f_LocRad_%f' % (inflation_factor, localization_radius))
#
if os.path.isdir(results_dir):
if overwrite:
pass
else:
return None
#
enkf_filter_configs = dict(
model=model,
analysis_ensemble=init_ensemble,
forecast_ensemble=None,
ensemble_size=ensemble_size,
#
adaptive_inflation=adaptive_inflation,
forecast_inflation_factor=forecast_inflation_factor,
inflation_design_penalty=
inflation_design_penalty, # penalty of the regularization parameter
localization_design_penalty=
localization_design_penalty, # penalty of the regularization parameter
inflation_factor=inflation_factor,
inflation_factor_bounds=inflation_bounds,
adaptive_localization=adaptive_localization,
localize_covariances=True,
localization_radii_bounds=localization_bounds,
localization_method='covariance_filtering',
localization_radius=localization_radius,
localization_function=localization_function,
loc_direct_approach=loc_direct_approach,
localization_space=localization_space,
regularization_norm=regularization_norm,
moving_average_radius=moving_average_radius,
)
#
if adaptive_inflation and adaptive_localization:
from adaptive_EnKF import EnKF_OED_Adaptive
elif adaptive_inflation:
from adaptive_EnKF import EnKF_OED_Inflation as EnKF_OED_Adaptive
elif adaptive_localization:
from adaptive_EnKF import EnKF_OED_Localization as EnKF_OED_Adaptive
else:
from EnKF import DEnKF as EnKF_OED_Adaptive
print(
"neither adapgtive inflation nor adaptive localization are revoked!"
)
# raise ValueError
#
filter_obj = EnKF_OED_Adaptive(filter_configs=enkf_filter_configs,
output_configs=dict(
file_output_moment_only=False,
verbose=False))
#
# ======================================================================================== #
#
#
# ======================================================================================== #
# Inititalize the sequential DA process #
# ======================================================================================== #
# Create the processing object:
# -------------------------------------
#
# create observations' and assimilation checkpoints:
obs_checkpoints = experiment_tspan
da_checkpoints = obs_checkpoints
#
# Here this is a filtering_process object;
from filtering_process import FilteringProcess
assimilation_configs = dict(
filter=filter_obj,
obs_checkpoints=obs_checkpoints,
# da_checkpoints=da_checkpoints,
forecast_first=True,
ref_initial_condition=ref_IC,
ref_initial_time=experiment_tspan[0],
random_seed=2345)
assim_output_configs = dict(scr_output=True,
scr_output_iter=1,
file_output_dir=results_dir,
file_output=True,
file_output_iter=1)
experiment = FilteringProcess(assimilation_configs=assimilation_configs,
output_configs=assim_output_configs)
# Save reference Trajectory:
results_dir = os.path.abspath(results_dir)
np.save(os.path.join(results_dir, 'Reference_Trajectory.npy'),
utility.ensemble_to_np_array(ref_trajectory, state_as_col=True))
np.save(os.path.join(results_dir, 'Initial_Ensemble.npy'),
utility.ensemble_to_np_array(init_ensemble, state_as_col=True))
np.save(os.path.join(results_dir, 'Observations.npy'),
utility.ensemble_to_np_array(observations, state_as_col=True))
# Run the sequential filtering process:
# -------------------------------------
experiment.recursive_assimilation_process(observations, obs_checkpoints,
da_checkpoints)
#
# ======================================================================================== #
#
if create_plots:
print("Creating Plots")
cmd = "python filtering_results_reader_coupledLorenz.py -f %s -r True -o True" % os.path.join(
results_dir, 'output_dir_structure.txt')
os.system(cmd)