def test_cycle_addsimulation_translate(
restart_dirs,
expected,
init_times,
simulation_compiled,
job_restart,
tmpdir
):
os.chdir(tmpdir)
sim = simulation
cy1 = CycleSimulation(
init_times=init_times,
restart_dirs=restart_dirs
)
sim_compiled = simulation_compiled
cy1.add(sim_compiled)
cy1.add(job_restart)
# translation happens on compose.
try:
os.mkdir(tmpdir / 'compose')
os.chdir(tmpdir / 'compose')
# This may be use in some of the tests
pathlib.Path('../dummy_extant_dir').touch()
cy1.compose(rm_casts_from_memory=False)
lsm_keys = ['noahlsm_offline', 'restart_filename_requested']
hyd_keys = ['hydro_nlist', 'restart_file']
result = {
'lsm': [cast.base_hrldas_namelist[lsm_keys[0]][lsm_keys[1]] for cast in cy1.casts],
'hyd': [cast.base_hydro_namelist[hyd_keys[0]][hyd_keys[1]] for cast in cy1.casts]
}
except Exception as e:
result = [str(e)] # Cludgy but it works
if isinstance(expected, dict):
for key, v_list in expected.items():
for ii, path in enumerate(v_list):
expected[key][ii] = sub_tmpdir(path, tmpdir)
assert result == expected
def test_cycle_ensemble_parallel_compose(ensemble, job_restart, scheduler,
tmpdir, init_times,
restart_dirs_ensemble):
ens = ensemble
cy = CycleSimulation(init_times=init_times,
restart_dirs=restart_dirs_ensemble,
ncores=2)
cy.add(job_restart)
# Adding the scheduler ruins the run in CI.
# cy.add(scheduler)
# Make a copy where we keep the casts in memory for checking.
cy_check_casts = copy.deepcopy(cy)
cy_ens_compose = copy.deepcopy(cy)
with pytest.raises(Exception) as e_info:
cy.compose()
cy_ens_compose.add(ens)
compose_dir = pathlib.Path(tmpdir).joinpath('cycle_ensemble_compose')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
pathlib.Path('../dummy_extant_dir').touch()
cy_ens_compose.compose()
cy_run_success = cy_ens_compose.run()
assert cy_run_success == 0
cy.pickle(
str(
pathlib.Path(tmpdir) /
'cycle_ensemble_compose/WrfHydroCycleEns.pkl'))
# Is this pickle used?
# The cycle-in-memory version for checking the casts.
cy_check_casts.add(ens)
compose_dir = pathlib.Path(tmpdir).joinpath('cycle_compose_check_casts')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
pathlib.Path('../dummy_extant_dir').touch()
cy_check_casts.compose(rm_casts_from_memory=False,
rm_members_from_memory=False)
# The job gets heavily modified on compose.
answer = {
'_entry_cmd': 'bogus entry cmd',
'_exe_cmd': './wrf_hydro.exe',
'_exit_cmd': 'bogus exit cmd',
'_hrldas_namelist': {
'noahlsm_offline': {
'btr_option': 1,
'canopy_stomatal_resistance_option': 1,
'hrldas_setup_file': './NWM/DOMAIN/wrfinput_d01.nc',
'indir': './FORCING',
'output_timestep': 86400,
'restart_filename_requested':
'./NWM/RESTART/RESTART.2011082600_DOMAIN1',
'restart_frequency_hours': 24
},
'wrf_hydro_offline': {
'forc_typ': 1
}
},
'_hrldas_times': {
'noahlsm_offline': {
'khour': 282480,
'restart_frequency_hours': 24,
'output_timestep': 86400,
'restart_filename_requested':
'NWM/RESTART/RESTART.2013101300_DOMAIN1',
'start_day': 12,
'start_hour': 00,
'start_min': 00,
'start_month': 12,
'start_year': 2012
}
},
'_hydro_namelist': {
'hydro_nlist': {
'aggfactrt': 4,
'channel_option': 2,
'chanobs_domain': 0,
'chanrtswcrt': 1,
'chrtout_domain': 1,
'geo_static_flnm': './NWM/DOMAIN/geo_em.d01.nc',
'restart_file':
'./NWM/RESTART/HYDRO_RST.2011-08-26_00:00_DOMAIN1',
'udmp_opt': 1,
'rst_dt': 1440,
'out_dt': 1440
},
'nudging_nlist': {
'maxagepairsbiaspersist':
3,
'minnumpairsbiaspersist':
1,
'nudginglastobsfile':
'./NWM/RESTART/nudgingLastObs.2011-08-26_00:00:00.nc'
}
},
'_hydro_times': {
'hydro_nlist': {
'out_dt': 1440,
'rst_dt': 1440,
'restart_file':
'NWM/RESTART/HYDRO_RST.2013-10-13_00:00_DOMAIN1'
},
'nudging_nlist': {
'nudginglastobsfile':
'NWM/RESTART/nudgingLastObs.2013-10-13_00:00:00.nc'
}
},
'_job_end_time': None,
'_job_start_time': None,
'_job_submission_time': None,
'_model_end_time': pandas.Timestamp('2045-03-04 00:00:00'),
'_model_start_time': pandas.Timestamp('2012-12-12 00:00:00'),
'exit_status': None,
'job_id': 'test_job_1',
'restart_freq_hr_hydro': None,
'restart_freq_hr_hrldas': None,
'output_freq_hr_hydro': None,
'output_freq_hr_hrldas': None,
'restart': True,
'restart_dir': None,
'_restart_dir_hydro': None,
'_restart_dir_hrldas': None,
'restart_file_time': '2013-10-13',
'_restart_file_time_hydro': pandas.Timestamp('2013-10-13 00:00:00'),
'_restart_file_time_hrldas': pandas.Timestamp('2013-10-13 00:00:00')
}
# These answer patches respond to the variety of things in restart_dirs_ensemble
dum_ext = str(tmpdir) + '/dummy_extant_dir/'
answer_patches = {
'cast_index': [0, 1, 2],
'start_time_patch': [
pandas.Timestamp('2012-12-12 00:00:00'),
pandas.Timestamp('2012-12-15 00:00:00'),
pandas.Timestamp('2012-12-18 00:00:00')
],
'end_time_patch': [
pandas.Timestamp('2045-03-04 00:00:00'),
pandas.Timestamp('2045-03-07 00:00:00'),
pandas.Timestamp('2045-03-10 00:00:00')
],
# These "time patches" reveal the awkwardness of that construct.
'lsm_times_patch': [
'NWM/RESTART/RESTART.2013101300_DOMAIN1',
'../../cast_2012121200/member_000/RESTART.2013101300_DOMAIN1',
dum_ext + 'RESTART.2013101300_DOMAIN1'
],
'hydro_times_patch': [
'NWM/RESTART/HYDRO_RST.2013-10-13_00:00_DOMAIN1',
'../../cast_2012121200/member_000/HYDRO_RST.2013-10-13_00:00_DOMAIN1',
dum_ext + 'HYDRO_RST.2013-10-13_00:00_DOMAIN1'
],
'ndg_times_patch': [
'NWM/RESTART/nudgingLastObs.2013-10-13_00:00:00.nc',
'../../cast_2012121200/member_000/nudgingLastObs.2013-10-13_00:00:00.nc',
dum_ext + 'nudgingLastObs.2013-10-13_00:00:00.nc'
],
# These namelist patches are consistent with the model times except in the
# first "do nothing" case which leaves the start time != restart file time
'lsm_nlst_patch': [
'./NWM/RESTART/RESTART.2011082600_DOMAIN1',
'../../cast_2012121200/member_000/RESTART.2012121500_DOMAIN1',
dum_ext + 'RESTART.2012121800_DOMAIN1'
],
'hydro_nlst_patch': [
'./NWM/RESTART/HYDRO_RST.2011-08-26_00:00_DOMAIN1',
'../../cast_2012121200/member_000/HYDRO_RST.2012-12-15_00:00_DOMAIN1',
dum_ext + 'HYDRO_RST.2012-12-18_00:00_DOMAIN1'
],
'ndg_nlst_patch': [
'./NWM/RESTART/nudgingLastObs.2011-08-26_00:00:00.nc',
'../../cast_2012121200/member_000/nudgingLastObs.2012-12-15_00:00:00.nc',
dum_ext + 'nudgingLastObs.2012-12-18_00:00:00.nc'
]
}
# Check a cycle where the compse retains the casts (otherwise nothing in memory).
# This fails:
# deepdiff.DeepDiff(answer, cy.casts[0].jobs[0].__dict__)
# Instead, iterate on keys to "declass":
# Just check the first ensemble cast.
def sub_member(the_string, replace_num, find_num=0):
replace = "member_{:03d}".format(replace_num)
find = "member_{:03d}".format(find_num)
return the_string.replace(find, replace)
for ii in answer_patches['cast_index']:
cc = cy_check_casts.casts[ii]
for mm, member in enumerate(cc.members):
answer['_model_start_time'] = answer_patches['start_time_patch'][
ii]
answer['_model_end_time'] = answer_patches['end_time_patch'][ii]
keys = ['noahlsm_offline', 'restart_filename_requested']
answer['_hrldas_namelist'][keys[0]][keys[1]] = \
sub_member(answer_patches['lsm_nlst_patch'][ii], mm)
answer['_hrldas_times'][keys[0]][keys[1]] = \
sub_member(answer_patches['lsm_times_patch'][ii], mm)
keys = ['_hydro_namelist', 'hydro_nlist', 'restart_file']
answer[keys[0]][keys[1]][keys[2]] = \
sub_member(answer_patches['hydro_nlst_patch'][ii], mm)
keys = ['_hydro_namelist', 'nudging_nlist', 'nudginglastobsfile']
answer[keys[0]][keys[1]][keys[2]] = \
sub_member(answer_patches['ndg_nlst_patch'][ii], mm)
keys = ['_hydro_times', 'hydro_nlist', 'restart_file']
answer[keys[0]][keys[1]][keys[2]] =\
sub_member(answer_patches['hydro_times_patch'][ii], mm)
keys = ['_hydro_times', 'nudging_nlist', 'nudginglastobsfile']
answer[keys[0]][keys[1]][keys[2]] = \
sub_member(answer_patches['ndg_times_patch'][ii], mm)
# hrldas times
fmt_keys = {
'%Y': 'start_year',
'%m': 'start_month',
'%d': 'start_day',
'%H': 'start_hour'
}
the_mutable = answer['_hrldas_times']['noahlsm_offline']
for fmt, key in fmt_keys.items():
the_mutable[key] = int(
answer['_model_start_time'].strftime(fmt))
# Actually check
for kk in member.jobs[0].__dict__.keys():
assert member.jobs[0].__dict__[kk] == answer[kk]
# Check the scheduler too
# assert cy_check_casts.casts[0].scheduler.__dict__ == scheduler.__dict__
# For the cycle where the compse removes the casts...
# Check that the casts are all now simply pathlib objects
assert all([type(mm) is str for mm in cy.casts])
# the tmpdir gets nuked after the test... ?
# Test the cast pickle size in terms of load speed.
# Note that the deletion of the model, domain, and output objects are
# done for the casts regardless of not removing the casts
# from memory (currently).
os.chdir(
str(pathlib.Path(tmpdir) / 'cycle_ensemble_compose/cast_2012121200'))
time_taken = timeit.timeit(
setup='import pickle',
stmt='pickle.load(open("WrfHydroEns.pkl","rb"))',
number=10000)
# If your system is busy, this could take longer... and spuriously fail the test.
# Notes(JLM): coverage makes this slow
assert time_taken < 1.5
# Test the cycle pickle size in terms of load speed.
os.chdir(str(pathlib.Path(tmpdir) / 'cycle_ensemble_compose/'))
time_taken = timeit.timeit(
setup='import pickle',
stmt='pickle.load(open("WrfHydroCycleEns.pkl","rb"))',
number=10000)
# If your system is busy, this could take longer...
# Notes(JLM): coveage makes this slow
assert time_taken < 1.5
def test_cycle_ensemble_compose(ensemble, job_restart, scheduler, tmpdir,
init_times, restart_dirs_ensemble):
# These might be parametizable.
# Length zero cycle compose.
cy = CycleSimulation(init_times=[], restart_dirs=[], ncores=1)
cy.add(job_restart)
cy.add(ensemble)
compose_dir = pathlib.Path(tmpdir).joinpath('cycle_sim0_compose')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
with pytest.raises(Exception) as e_info:
cy.compose()
assert str(e_info.value) == "There are no casts (init_times) to compose."
# Inconsistent forcing dir length and ensemble length
cy = CycleSimulation(init_times=init_times,
restart_dirs=restart_dirs_ensemble,
ncores=1,
forcing_dirs=[['.', '.', '../dummy_extant_dir'],
['.', -72, '../dummy_extant_dir'],
['.', -72, '../dummy_extant_dir']])
cy.add(job_restart)
with pytest.raises(Exception) as e_info:
cy.add(ensemble)
assert str(e_info.value) == \
"Ensemble to add has inconsistent length with existing cycle forcing_dirs"
# Inconsistent forcing dir length and ensemble length
cy = CycleSimulation(init_times=init_times,
restart_dirs=[[rr[0]]
for rr in restart_dirs_ensemble],
ncores=1,
forcing_dirs=[['.', '../dummy_extant_dir'],
[-72, '../dummy_extant_dir'],
[-72, '../dummy_extant_dir']])
cy.add(job_restart)
with pytest.raises(Exception) as e_info:
cy.add(ensemble)
assert str(e_info.value) == \
"Ensemble to add has inconsistent length with existing cycle restart_dirs"
# Valid force dir exercise.
cy = CycleSimulation(init_times=init_times,
restart_dirs=restart_dirs_ensemble,
ncores=1,
forcing_dirs=[['.', '../dummy_extant_dir'],
[-72, '../dummy_extant_dir'],
[-72, '../dummy_extant_dir']])
cy.add(job_restart)
cy.add(ensemble)
compose_dir = pathlib.Path(tmpdir).joinpath('cycle_forc_dir_compose')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
compose_dir.joinpath('../dummy_extant_dir').touch()
cy.compose()
# In valid force dir exercise.
cy = CycleSimulation(init_times=init_times,
restart_dirs=restart_dirs_ensemble,
ncores=1,
forcing_dirs=[[72, '../dummy_extant_dir'],
[72, '../dummy_extant_dir'],
[72, '../dummy_extant_dir']])
cy.add(job_restart)
cy.add(ensemble)
compose_dir = pathlib.Path(tmpdir).joinpath(
'cycle_forc_dir_fail_1_compose')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
compose_dir.joinpath('../dummy_extant_dir').touch()
with pytest.raises(Exception) as e_info:
cy.compose()
assert str(
e_info.value
) == 'Only non-negative integers can be used to specify forcing_dirs'
# In valid force dir exercise.
cy = CycleSimulation(init_times=init_times,
restart_dirs=restart_dirs_ensemble,
ncores=1,
forcing_dirs=[['.', 'dummy_non-extant_dir'],
['.', 'dummy_non-extant_dir'],
['.', 'dummy_non-extant_dir']])
cy.add(job_restart)
cy.add(ensemble)
compose_dir = pathlib.Path(tmpdir).joinpath(
'cycle_forc_dir_fail_2_compose')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
compose_dir.joinpath('../dummy_extant_dir').touch()
with pytest.raises(Exception) as e_info:
cy.compose()
assert str(
e_info.value) == 'No such forcing directory: dummy_non-extant_dir'
def test_cycle_parallel_compose(simulation_compiled, job_restart, scheduler,
tmpdir, init_times, restart_dirs):
""" A more comprehensive test of the object composed."""
# A compiled simulation passed. Successfull compose in parallel.
cy = CycleSimulation(init_times=init_times,
restart_dirs=restart_dirs,
ncores=2)
cy.add(job_restart)
cy.add(simulation_compiled)
# Make a copy where we keep the casts in memory for checking.
cy_check_casts = copy.deepcopy(cy)
compose_dir = pathlib.Path(tmpdir).joinpath('cycle_compose')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
cy.compose()
cy_run_success = cy.run()
assert cy_run_success == 0
cy.pickle(str(pathlib.Path(tmpdir) / 'cycle_compose/WrfHydroCycleSim.pkl'))
# The cycle-in-memory version for checking the casts.
compose_dir = pathlib.Path(tmpdir).joinpath('cycle_compose_check_casts')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
cy_check_casts.compose(rm_casts_from_memory=False)
# The job gets heavily modified on compose.
answer = {
'_entry_cmd': 'bogus entry cmd',
'_exe_cmd': './wrf_hydro.exe',
'_exit_cmd': 'bogus exit cmd',
'_hrldas_namelist': {
'noahlsm_offline': {
'btr_option': 1,
'canopy_stomatal_resistance_option': 1,
'hrldas_setup_file': './NWM/DOMAIN/wrfinput_d01.nc',
'indir': './FORCING',
'output_timestep': 86400,
'restart_filename_requested':
'./NWM/RESTART/RESTART.2011082600_DOMAIN1',
'restart_frequency_hours': 24
},
'wrf_hydro_offline': {
'forc_typ': 1
}
},
'_hrldas_times': {
'noahlsm_offline': {
'khour': 282480,
'restart_frequency_hours': 24,
'output_timestep': 86400,
'restart_filename_requested':
'NWM/RESTART/RESTART.2013101300_DOMAIN1',
'start_day': 12,
'start_hour': 00,
'start_min': 00,
'start_month': 12,
'start_year': 2012
}
},
'_hydro_namelist': {
'hydro_nlist': {
'aggfactrt': 4,
'channel_option': 2,
'chanobs_domain': 0,
'chanrtswcrt': 1,
'chrtout_domain': 1,
'geo_static_flnm': './NWM/DOMAIN/geo_em.d01.nc',
'restart_file':
'./NWM/RESTART/HYDRO_RST.2011-08-26_00:00_DOMAIN1',
'udmp_opt': 1,
'rst_dt': 1440,
'out_dt': 1440
},
'nudging_nlist': {
'maxagepairsbiaspersist':
3,
'minnumpairsbiaspersist':
1,
'nudginglastobsfile':
'./NWM/RESTART/nudgingLastObs.2011-08-26_00:00:00.nc'
}
},
'_hydro_times': {
'hydro_nlist': {
'out_dt': 1440,
'rst_dt': 1440,
'restart_file':
'NWM/RESTART/HYDRO_RST.2013-10-13_00:00_DOMAIN1'
},
'nudging_nlist': {
'nudginglastobsfile':
'NWM/RESTART/nudgingLastObs.2013-10-13_00:00:00.nc'
}
},
'_job_end_time': None,
'_job_start_time': None,
'_job_submission_time': None,
'_model_end_time': pandas.Timestamp('2045-03-04 00:00:00'),
'_model_start_time': pandas.Timestamp('2012-12-12 00:00:00'),
'exit_status': None,
'job_id': 'test_job_1',
'restart_freq_hr_hydro': None,
'restart_freq_hr_hrldas': None,
'output_freq_hr_hydro': None,
'output_freq_hr_hrldas': None,
'restart': True,
'restart_dir': None,
'_restart_dir_hydro': None,
'_restart_dir_hrldas': None,
'restart_file_time': '2013-10-13',
'_restart_file_time_hydro': pandas.Timestamp('2013-10-13 00:00:00'),
'_restart_file_time_hrldas': pandas.Timestamp('2013-10-13 00:00:00')
}
# For the cycle where the compse retains the casts...
# This fails:
# deepdiff.DeepDiff(answer, cy.casts[0].jobs[0].__dict__)
# Instead, iterate on keys to "declass":
for kk in cy_check_casts.casts[0].jobs[0].__dict__.keys():
assert cy_check_casts.casts[0].jobs[0].__dict__[kk] == answer[kk]
# Check the scheduler too
# assert cy_check_casts.casts[0].scheduler.__dict__ == scheduler.__dict__
# For the cycle where the compse removes the casts...
# Check that the casts are all now simply pathlib objects
assert all([type(mm) is str for mm in cy.casts])
# the tmpdir gets nuked after the test... ?
# Test the cast pickle size in terms of load speed.
# Note that the deletion of the model, domain, and output objects are
# done for the casts regardless of not removing the casts
# from memory (currently).
os.chdir(str(pathlib.Path(tmpdir) / 'cycle_compose/cast_2012121200'))
time_taken = timeit.timeit(
setup='import pickle',
stmt='pickle.load(open("WrfHydroSim.pkl","rb"))',
number=10000)
# If your system is busy, this could take longer... and spuriously fail the test.
# Notes(JLM): coverage is the limiting factor
assert time_taken < 1.5
# Test the cycle pickle size in terms of load speed.
os.chdir(str(pathlib.Path(tmpdir) / 'cycle_compose/'))
time_taken = timeit.timeit(
setup='import pickle',
stmt='pickle.load(open("WrfHydroCycleSim.pkl","rb"))',
number=10000)
# If your system is busy, this could take longer...
# Notes(JLM): coverage is the limiting factor.
assert time_taken < 1.2
def test_cycle_compose(simulation, simulation_compiled, job_restart, scheduler,
tmpdir, init_times, restart_dirs):
# These might be parametizable.
# Compose without adding a simulation.
cy = CycleSimulation(init_times=init_times,
restart_dirs=restart_dirs,
ncores=2)
cy.add(job_restart)
# Adding the scheduler ruins the run in CI.
# cy.add(scheduler)
with pytest.raises(Exception) as e_info:
cy.compose()
assert str(e_info.value) == \
'Unable to compose, current working directory is not empty. \n' + \
'Change working directory to an empty directory with os.chdir()'
compose_dir = pathlib.Path(tmpdir).joinpath('cycle_no_sim_compose')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
with pytest.raises(Exception) as e_info:
cy.compose()
assert str(e_info.value
) == 'The cycle does not contain a _simulation or an _ensemble.'
# Length zero cycle compose.
cy = CycleSimulation(init_times=[], restart_dirs=[], ncores=1)
cy.add(job_restart)
cy.add(simulation_compiled)
compose_dir = pathlib.Path(tmpdir).joinpath('cycle_sim0_compose')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
with pytest.raises(Exception) as e_info:
cy.compose()
assert str(e_info.value) == "There are no casts (init_times) to compose."
# This simultion is not compiled. It compiles and composes successfully.
cy = CycleSimulation(init_times=init_times,
restart_dirs=restart_dirs,
ncores=1)
cy.add(job_restart)
cy.add(simulation)
compose_dir = pathlib.Path(tmpdir).joinpath('cycle_uncompiled_compose')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
cy.compose()
# Valid force dir exercise.
cy = CycleSimulation(init_times=init_times,
restart_dirs=restart_dirs,
ncores=1,
forcing_dirs=['.', -72, '../dummy_extant_dir'])
cy.add(job_restart)
cy.add(simulation)
compose_dir = pathlib.Path(tmpdir).joinpath('cycle_forc_dir_compose')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
compose_dir.joinpath('../dummy_extant_dir').touch()
cy.compose()
# In valid force dir exercise.
cy = CycleSimulation(init_times=init_times,
restart_dirs=restart_dirs,
ncores=1,
forcing_dirs=['.', 72, '../dummy_extant_dir'])
cy.add(job_restart)
cy.add(simulation)
compose_dir = pathlib.Path(tmpdir).joinpath(
'cycle_forc_dir_fail_2_compose')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
with pytest.raises(Exception) as e_info:
cy.compose()
assert str(
e_info.value
) == 'Only non-negative integers can be used to specify forcing_dirs'
# In valid force dir exercise.
cy = CycleSimulation(init_times=init_times,
restart_dirs=restart_dirs,
ncores=1,
forcing_dirs=['.', 'dummy_non-extant_dir', -72])
cy.add(job_restart)
cy.add(simulation)
compose_dir = pathlib.Path(tmpdir).joinpath('cycle_forc_dir_fail_compose')
os.mkdir(str(compose_dir))
os.chdir(str(compose_dir))
with pytest.raises(Exception) as e_info:
cy.compose()
assert str(
e_info.value) == 'No such forcing directory: dummy_non-extant_dir'