def test_multi_checkpoints(self, tmp_path):
"""Test using checkpoints multiple times for a given model run."""
# define a yaml for the base model run
file_name = 'base_run.yaml'
base_p, base_f = utilities.create_temporary_file(tmp_path, file_name)
utilities.write_parameter_to_file(base_f, 'save_eta_grids', True)
utilities.write_parameter_to_file(base_f, 'save_checkpoint', True)
utilities.write_parameter_to_file(base_f, 'out_dir', tmp_path / 'test')
base_f.close()
baseModel = DeltaModel(input_file=base_p)
# run base for 2 timesteps
for _ in range(0, 50):
baseModel.update()
baseModel.finalize()
# try defining a new model but plan to load checkpoint from baseModel
file_name = 'base_run.yaml'
base_p, base_f = utilities.create_temporary_file(tmp_path, file_name)
utilities.write_parameter_to_file(base_f, 'save_eta_grids', True)
utilities.write_parameter_to_file(base_f, 'save_checkpoint', True)
utilities.write_parameter_to_file(base_f, 'resume_checkpoint', True)
utilities.write_parameter_to_file(base_f, 'out_dir', tmp_path / 'test')
base_f.close()
resumeModel = DeltaModel(input_file=base_p)
assert resumeModel.time <= baseModel.time
# advance it more steps
for _ in range(0, 25):
resumeModel.update()
resumeModel.finalize()
# create another resume model
resumeModel02 = DeltaModel(input_file=base_p)
assert resumeModel02.time <= resumeModel.time # should be same
# step it some more
nt_resume02 = 0
while (resumeModel02._save_time_since_data != 0) or (nt_resume02 < 50):
resumeModel02.update()
nt_resume02 += 1
# assert that output netCDF4 exists
exp_path_nc = os.path.join(tmp_path / 'test', 'pyDeltaRCM_output.nc')
assert os.path.isfile(exp_path_nc)
# load it into memory and check values in the netCDF4
output = Dataset(exp_path_nc, 'r', allow_pickle=True)
out_vars = output.variables.keys()
# check that expected variables are in the file
assert 'x' in out_vars
assert 'y' in out_vars
assert 'time' in out_vars
assert 'eta' in out_vars
# check attributes of variables
assert output['time'][0].tolist() == 0.0
assert output['time'][-1].tolist() == resumeModel02.time
def test_checkpoint_nc(self, tmp_path):
"""Test the netCDF that is written to by the checkpointing."""
# define a yaml for the base model run
file_name = 'base_run.yaml'
base_p, base_f = utilities.create_temporary_file(tmp_path, file_name)
utilities.write_parameter_to_file(base_f, 'out_dir', tmp_path / 'test')
utilities.write_parameter_to_file(base_f, 'save_eta_grids', True)
utilities.write_parameter_to_file(base_f, 'save_depth_grids', True)
utilities.write_parameter_to_file(base_f, 'save_discharge_grids', True)
utilities.write_parameter_to_file(base_f, 'save_sandfrac_grids', True)
utilities.write_parameter_to_file(base_f, 'save_checkpoint', True)
base_f.close()
baseModel = DeltaModel(input_file=base_p)
# run for some base number of steps
nt_base = 50
for _ in range(0, 50):
baseModel.update()
# force the model run to end immmediately after exporting a checkpoint
nt_var = 0
while (baseModel._save_time_since_checkpoint != 0):
baseModel.update()
nt_var += 1
# then finalize
baseModel.finalize()
# check that the time makes sense
assert baseModel.time == baseModel._dt * (nt_base + nt_var)
# try defining a new model but plan to load checkpoint from baseModel
file_name = 'base_run.yaml'
base_p, base_f = utilities.create_temporary_file(tmp_path, file_name)
utilities.write_parameter_to_file(base_f, 'out_dir', tmp_path / 'test')
utilities.write_parameter_to_file(base_f, 'save_eta_grids', True)
utilities.write_parameter_to_file(base_f, 'save_depth_grids', True)
utilities.write_parameter_to_file(base_f, 'save_discharge_grids', True)
utilities.write_parameter_to_file(base_f, 'save_sandfrac_grids', True)
utilities.write_parameter_to_file(base_f, 'save_checkpoint', False)
utilities.write_parameter_to_file(base_f, 'resume_checkpoint', True)
base_f.close()
resumeModel = DeltaModel(input_file=base_p)
assert resumeModel.time == baseModel.time # same when resumed
# advance it until output_data has been called again
nt_resume = 0
while (resumeModel._save_time_since_data != 0) or (nt_resume < 50):
resumeModel.update()
nt_resume += 1
resumeModel.finalize()
assert nt_resume > 0
assert resumeModel.time > baseModel.time
# assert that output netCDF4 exists
exp_path_nc = os.path.join(tmp_path / 'test', 'pyDeltaRCM_output.nc')
assert os.path.isfile(exp_path_nc)
# load it into memory and check values in the netCDF4
output = Dataset(exp_path_nc, 'r', allow_pickle=True)
out_vars = output.variables.keys()
# check that expected variables are in the file
assert 'x' in out_vars
assert 'y' in out_vars
assert 'time' in out_vars
assert 'eta' in out_vars
assert 'depth' in out_vars
assert 'discharge' in out_vars
assert 'sandfrac' in out_vars
# check attributes of variables
assert output['time'][0].tolist() == 0.0
assert output['time'][-1] == resumeModel.time
assert output['time'][-1].tolist() == resumeModel._dt * \
(nt_base + nt_var + nt_resume)
assert output['eta'][0].shape == resumeModel.eta.shape
assert output['eta'][-1].shape == resumeModel.eta.shape
assert output['depth'][-1].shape == resumeModel.eta.shape
assert output['discharge'][-1].shape == resumeModel.eta.shape
assert output['sandfrac'][-1].shape == resumeModel.eta.shape
# check the metadata
assert output['meta']['L0'][:] == resumeModel.L0
assert output['meta']['N0'][:] == resumeModel.N0
assert output['meta']['CTR'][:] == resumeModel.CTR
assert output['meta']['dx'][:] == resumeModel.dx
assert output['meta']['h0'][:] == resumeModel.h0
assert np.all(output['meta']['cell_type'][:] == resumeModel.cell_type)
assert output['meta']['H_SL'][-1].data == resumeModel.H_SL
assert output['meta']['f_bedload'][-1].data == resumeModel.f_bedload
C0_from_file = float(output['meta']['C0_percent'][-1].data)
assert pytest.approx(C0_from_file) == resumeModel.C0_percent
assert output['meta']['u0'][-1].data == resumeModel.u0
# checkpoint interval aligns w/ timestep dt so these should match
assert output['time'][-1].tolist() == resumeModel.time
def test_load_ckpt_wo_netcdf_parallel_spinup_to_matrix(self, tmp_path):
"""
Test that multiple matrix runs can be resumed from a single checkpoint
file, and take advantage of the preprocessor parallel infrastructure.
"""
# define a yaml with an output and checkpoint
p = utilities.yaml_from_dict(tmp_path, 'input.yaml',
{'save_checkpoint': True,
'save_eta_grids': True})
baseModel = DeltaModel(input_file=p)
# run base for 2 timesteps
for _ in range(0, 50):
baseModel.update()
baseModel.finalize()
# get the number of times the data has been output in the base file
bmsi = baseModel._save_iter
assert bmsi > 0
# check that files exist, and then delete nc
assert os.path.isfile(os.path.join(
baseModel.prefix, 'pyDeltaRCM_output.nc'))
assert os.path.isfile(os.path.join(
baseModel.prefix, 'checkpoint.npz'))
# open the file and check dimensions
exp_nc_path = os.path.join(baseModel.prefix, 'pyDeltaRCM_output.nc')
output = Dataset(exp_nc_path, 'r', allow_pickle=True)
out_vars = output.variables.keys()
# check that expected variables are in the file
assert 'x' in out_vars
assert 'y' in out_vars
assert 'time' in out_vars
assert 'eta' in out_vars
# check attributes of variables
assert output['time'].shape[0] == bmsi
########################
# set up a matrix of runs
resume_dict = {'save_checkpoint': False,
'resume_checkpoint': True,
'save_eta_grids': True,
'out_dir': os.path.join(tmp_path, 'matrix'),
'parallel': 4}
_matrix = {'f_bedload': [0.1, 0.2, 0.5, 1]}
resume_dict['matrix'] = _matrix
# let the preprocessor write the initial matrix and
# create a new output netcdf file
pp = preprocessor.Preprocessor(
resume_dict,
timesteps=25) # 2000
# now copy the checkpoint
for j, j_file in enumerate(pp.file_list):
# get folder
j_folder = j_file.parent
# copy the spinup checkpoint to each of the folders
shutil.copy(
src=os.path.join(baseModel.prefix, 'checkpoint.npz'),
dst=os.path.join(tmp_path, 'matrix', j_folder))
pp.run_jobs()
# first job save dimension
fjsi = pp.job_list[0].deltamodel._save_iter
# proceed with assertions
for j, j_job in enumerate(pp.job_list):
exp_nc_path = os.path.join(
tmp_path, 'matrix', j_folder, 'pyDeltaRCM_output.nc')
assert os.path.isfile(exp_nc_path)
# check all jobs have same dimensionality
jsi = j_job.deltamodel._save_iter
assert jsi == fjsi
# open the file and check dimensions
output = Dataset(exp_nc_path, 'r', allow_pickle=True)
out_vars = output.variables.keys()
# check that expected variables are in the file
assert 'x' in out_vars
assert 'y' in out_vars
assert 'time' in out_vars
assert 'eta' in out_vars
# check attributes of variables
# this is the critical check, that the dimension of the second
# netcdf is not expanded to begin at _save_ter from the initial
# basemodel
assert output['time'].shape[0] <= (bmsi // 2)
def test_simple_checkpoint(self, tmp_path):
"""Test checkpoint vs a base run.
Also, checks resumed model against another checkpoint run.
"""
# define a yaml for the longer model run
file_name = 'base_run.yaml'
base_p, base_f = utilities.create_temporary_file(tmp_path, file_name)
utilities.write_parameter_to_file(base_f, 'out_dir', tmp_path / 'test')
utilities.write_parameter_to_file(base_f, 'save_checkpoint', True)
base_f.close()
longModel = DeltaModel(input_file=base_p)
# run for some number of updates
for _ in range(0, 50):
longModel.update()
longModel.finalize()
# try defining a new model but plan to load checkpoint from longModel
file_name = 'base_run.yaml'
base_p, base_f = utilities.create_temporary_file(tmp_path, file_name)
utilities.write_parameter_to_file(base_f, 'out_dir', tmp_path / 'test')
utilities.write_parameter_to_file(base_f, 'resume_checkpoint', True)
base_f.close()
resumeModel = DeltaModel(input_file=base_p)
# advance the resumed model until it catch up to longModel
assert resumeModel.time < longModel.time
while resumeModel._time < longModel._time:
resumeModel.update()
resumeModel.finalize()
# the longModel and resumeModel should match
assert longModel.time == resumeModel.time
assert np.all(longModel.eta == resumeModel.eta)
assert np.all(longModel.uw == resumeModel.uw)
assert np.all(longModel.ux == resumeModel.ux)
assert np.all(longModel.uy == resumeModel.uy)
assert np.all(longModel.depth == resumeModel.depth)
assert np.all(longModel.stage == resumeModel.stage)
assert np.all(longModel.sand_frac == resumeModel.sand_frac)
assert np.all(longModel.active_layer == resumeModel.active_layer)
# define another model that loads the checkpoint
file_name = 'base_run.yaml'
base_p, base_f = utilities.create_temporary_file(tmp_path, file_name)
utilities.write_parameter_to_file(base_f, 'out_dir', tmp_path / 'test')
utilities.write_parameter_to_file(base_f, 'resume_checkpoint', True)
base_f.close()
resumeModel2 = DeltaModel(input_file=base_p)
# advance the resumed model until it catch up to longModel
while resumeModel2._time < resumeModel._time:
resumeModel2.update()
resumeModel2.finalize()
# the two models that resumed from the checkpoint should be the same
assert resumeModel2.time == resumeModel.time
assert np.all(resumeModel2.uw == resumeModel.uw)
assert np.all(resumeModel2.ux == resumeModel.ux)
assert np.all(resumeModel2.uy == resumeModel.uy)
assert np.all(resumeModel2.depth == resumeModel.depth)
assert np.all(resumeModel2.stage == resumeModel.stage)
assert np.all(resumeModel2.sand_frac == resumeModel.sand_frac)
assert np.all(resumeModel2.active_layer == resumeModel.active_layer)
def test_checkpoint_diff_dt(self, tmp_path):
"""Test when checkpoint_dt does not match dt or save_dt."""
# define a yaml for the base model run
file_name = 'base_run.yaml'
base_p, base_f = utilities.create_temporary_file(tmp_path, file_name)
utilities.write_parameter_to_file(base_f, 'save_eta_grids', True)
utilities.write_parameter_to_file(base_f, 'save_depth_grids', True)
utilities.write_parameter_to_file(base_f, 'save_discharge_grids', True)
utilities.write_parameter_to_file(base_f, 'save_checkpoint', True)
utilities.write_parameter_to_file(base_f, 'out_dir', tmp_path / 'test')
base_f.close()
baseModel = DeltaModel(input_file=base_p)
# modify the checkpoint dt to be different than save_dt
baseModel._checkpoint_dt = (baseModel.save_dt * 0.65)
for _ in range(0, 50):
baseModel.update()
baseModel.finalize()
assert baseModel.time == baseModel._dt * 50
baseModelSavedTime = (baseModel.time -
baseModel._save_time_since_checkpoint)
assert baseModelSavedTime > 0
# try defining a new model but plan to load checkpoint from baseModel
file_name = 'base_run.yaml'
base_p, base_f = utilities.create_temporary_file(tmp_path, file_name)
utilities.write_parameter_to_file(base_f, 'save_eta_grids', True)
utilities.write_parameter_to_file(base_f, 'save_depth_grids', True)
utilities.write_parameter_to_file(base_f, 'save_discharge_grids', True)
utilities.write_parameter_to_file(base_f, 'save_checkpoint', False)
utilities.write_parameter_to_file(base_f, 'resume_checkpoint', True)
utilities.write_parameter_to_file(base_f, 'out_dir', tmp_path / 'test')
base_f.close()
resumeModel = DeltaModel(input_file=base_p)
assert resumeModel.time == baseModelSavedTime
# advance until some steps and just saved
nt_resume = 0
while (resumeModel._save_time_since_data != 0) or (nt_resume < 50):
resumeModel.update()
nt_resume += 1
resumeModel.finalize()
# assert that output netCDF4 exists
exp_path_nc = os.path.join(tmp_path / 'test', 'pyDeltaRCM_output.nc')
assert os.path.isfile(exp_path_nc)
# load it into memory and check values in the netCDF4
output = Dataset(exp_path_nc, 'r', allow_pickle=True)
out_vars = output.variables.keys()
# check that expected variables are in the file
assert 'x' in out_vars
assert 'y' in out_vars
assert 'time' in out_vars
assert 'eta' in out_vars
assert 'depth' in out_vars
assert 'discharge' in out_vars
# check attributes of variables
assert output['time'][0].tolist() == 0.0
assert output['time'][-1].tolist() == resumeModel.time
