def test_cycle_length(simulation_compiled, init_times): sim = simulation_compiled cy1 = CycleSimulation(init_times=init_times, restart_dirs=['.'] * len(init_times)) cy1.add(sim) assert len(cy1) == len(init_times) assert cy1.N == len(init_times)
def test_add_forcing_dirs(forcing_dirs, expected, restart_dirs, init_times): try: cycle = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs, forcing_dirs=forcing_dirs) result = cycle._forcing_dirs except Exception as e: result = [str(e)] # Cludgy but it works assert result == expected
def test_cycle_init(init_times, restart_dirs): cycle = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs) assert type(cycle) is CycleSimulation # Not sure why this dosent vectorize well. atts = [ '_addforcingdirs', '_addinittimes', '_addjob', '_addrestartdirs', '_addscheduler', '_addsimulation', '_forcing_dirs', '_init_times', '_job', '_restart_dirs', '_scheduler', 'add', 'casts', 'compose', 'ncores', 'pickle', 'rm_casts', 'run' ] for kk in cycle.__dict__.keys(): assert kk in atts
def test_cycle_run_parallel_teams(simulation_compiled, job_restart, scheduler, tmpdir, capfd, init_times, restart_dirs): sim = simulation_compiled tmp = pathlib.Path(tmpdir) forcing_dirs = [ tmp / letter for letter in list(string.ascii_lowercase)[0:len(init_times)] ] for dir in forcing_dirs: dir.mkdir() cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs, forcing_dirs=forcing_dirs) cy.add(sim) job_restart._entry_cmd = 'echo mpirun entry_cmd > entry_cmd.output' job_restart._exit_cmd = 'echo mpirun exit_cmd > exit_cmd.output' cy.add(job_restart) # Parallel teams test - test casts both in memory and not cy_teams_to_test = { 'cy_teams_not_memory': copy.deepcopy(cy), 'cy_teams_memory': copy.deepcopy(cy) } for key, cy_teams in cy_teams_to_test.items(): cy_dir = pathlib.Path(tmpdir).joinpath(key) os.chdir(tmpdir) os.mkdir(str(cy_dir)) os.chdir(str(cy_dir)) if key == 'cy_teams_not_memory': cy_teams.compose() else: cy_teams.compose(rm_casts_from_memory=False) cy_teams_run_success = cy_teams.run( teams=True, teams_exe_cmd= ' ./wrf_hydro.exe mpirun --host {hostname} -np {nproc} {cmd}', teams_exe_cmd_nproc=2, teams_node_file={'pbs': node_file}) assert cy_teams_run_success == 0, \ "Some parallel team cycle casts did not run successfully." assert cy_dir.joinpath("WrfHydroCycle.pkl").exists() # Check for command correctness in output files. file_check = { ('cast_2012121200/entry_cmd.output', 'cast_2012121500/entry_cmd.output', 'cast_2012121800/entry_cmd.output'): 'mpirun entry_cmd\n', ('cast_2012121200/exit_cmd.output', 'cast_2012121500/exit_cmd.output', 'cast_2012121800/exit_cmd.output'): 'mpirun exit_cmd\n', ('cast_2012121200/job_test_job_1/diag_hydro.00000', 'cast_2012121800/job_test_job_1/diag_hydro.00000'): 'mpirun --host r10i1n1,r10i1n1 -np 2 ./wrf_hydro.exe\n', ('cast_2012121500/job_test_job_1/diag_hydro.00000', ): 'mpirun --host r10i1n2,r10i1n2 -np 2 ./wrf_hydro.exe\n' } for tup, ans in file_check.items(): for file in tup: check_first_line(file, ans)
def test_cycle_run_parallel_casts_in_memory(simulation_compiled, job_restart, scheduler, tmpdir, capfd, init_times, restart_dirs): sim = simulation_compiled tmp = pathlib.Path(tmpdir) forcing_dirs = [ tmp / letter for letter in list(string.ascii_lowercase)[0:len(init_times)] ] for dir in forcing_dirs: dir.mkdir() cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs, forcing_dirs=forcing_dirs) cy.add(sim) cy.add(job_restart) # Parallel test with ensemble in memory cy_parallel = copy.deepcopy(cy) cy_dir = pathlib.Path(tmpdir).joinpath('cy_parallel_run_cy_in_memory') os.chdir(tmpdir) os.mkdir(str(cy_dir)) os.chdir(str(cy_dir)) cy_parallel.compose(rm_casts_from_memory=False) cy_run_mem_success = cy_parallel.run(n_concurrent=2) assert cy_run_mem_success == 0, \ "Some parallel cycle casts in memory did not run successfully." assert cy_dir.joinpath("WrfHydroCycle.pkl").exists()
def test_cycle_serial(simulation_compiled, job_restart, scheduler, tmpdir, capfd, init_times, restart_dirs): sim = simulation_compiled tmp = pathlib.Path(tmpdir) forcing_dirs = [ tmp / letter for letter in list(string.ascii_lowercase)[0:len(init_times)] ] for dir in forcing_dirs: dir.mkdir() cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs, forcing_dirs=forcing_dirs) cy.add(sim) cy.add(job_restart) # Serial test cy_serial = copy.deepcopy(cy) cy_dir = pathlib.Path(tmpdir).joinpath('cycle_serial_run') os.chdir(tmpdir) os.mkdir(str(cy_dir)) os.chdir(str(cy_dir)) cy_serial.compose(rm_casts_from_memory=False) serial_run_success = cy_serial.run() assert serial_run_success == 0, \ "Some serial cycle casts did not run successfully." assert cy_dir.joinpath("WrfHydroCycle.pkl").exists()
def test_cycle_self_dependent_run( simulation_compiled, job_restart, scheduler, tmpdir, capfd, init_times ): sim = simulation_compiled cy = CycleSimulation( init_times=init_times, restart_dirs=['.', '-3', '-3'] ) cy.add(job_restart) cy.add(sim) # Serial test cy_serial = copy.deepcopy(cy) cy_dir = pathlib.Path(tmpdir).joinpath('cycle_serial_run') os.chdir(tmpdir) os.mkdir(str(cy_dir)) os.chdir(str(cy_dir)) cy_serial.compose(rm_casts_from_memory=False) serial_run_success = cy_serial.run() assert serial_run_success, \ "Some serial cycle casts did not run successfully."
def test_cycle_ensemble_run(ensemble, job_restart, scheduler, tmpdir, capfd, init_times, restart_dirs_ensemble): # Valid force dir exercise. cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs_ensemble, ncores=1, forcing_dirs=[['.', '../dummy_extant_dir'], [-72, '../dummy_extant_dir'], [-72, '../dummy_extant_dir']]) job_restart._entry_cmd = 'echo mpirun entry_cmd > entry_cmd.output' job_restart._exit_cmd = 'echo mpirun exit_cmd > exit_cmd.output' cy.add(job_restart) cy.add(ensemble) # Parallel teams test - test casts both in memory and not cy_teams_to_test = { 'cy_teams_not_memory': copy.deepcopy(cy), 'cy_teams_memory': copy.deepcopy(cy) } for key, cy_teams in cy_teams_to_test.items(): cy_dir = pathlib.Path(tmpdir).joinpath(key) os.chdir(tmpdir) os.mkdir(str(cy_dir)) os.chdir(str(cy_dir)) cy_dir.joinpath('../dummy_extant_dir').touch() if key == 'cy_teams_not_memory': cy_teams.compose() else: cy_teams.compose(rm_casts_from_memory=False) cy_teams_run_success = cy_teams.run( teams=True, teams_exe_cmd=( ' ./wrf_hydro.exe mpirun --host {nodelist} -np {nproc} {cmd}'), teams_exe_cmd_nproc=2, teams_node_file=node_file) assert cy_teams_run_success == 0, \ "Some parallel team cycle casts did not run successfully." assert cy_dir.joinpath("WrfHydroCycle.pkl").exists() # Check for command correctness in output files. file_check = { ('cast_2012121200/member_000/entry_cmd.output', 'cast_2012121200/member_001/entry_cmd.output', 'cast_2012121500/member_000/entry_cmd.output', 'cast_2012121500/member_001/entry_cmd.output', 'cast_2012121800/member_000/entry_cmd.output', 'cast_2012121800/member_001/entry_cmd.output'): 'mpirun entry_cmd\n', ('cast_2012121200/member_000/exit_cmd.output', 'cast_2012121200/member_001/exit_cmd.output', 'cast_2012121500/member_000/exit_cmd.output', 'cast_2012121500/member_001/exit_cmd.output', 'cast_2012121800/member_000/exit_cmd.output', 'cast_2012121800/member_001/exit_cmd.output'): 'mpirun exit_cmd\n', ('cast_2012121200/member_000/job_test_job_1/diag_hydro.00000', 'cast_2012121200/member_001/job_test_job_1/diag_hydro.00000'): 'mpirun --host r10i1n1.ib0.cheyenne.ucar.edu,r10i1n1.ib0.cheyenne.ucar.edu -np 2 ./wrf_hydro.exe\n', ('cast_2012121500/member_000/job_test_job_1/diag_hydro.00000', 'cast_2012121500/member_001/job_test_job_1/diag_hydro.00000'): 'mpirun --host r10i1n2.ib0.cheyenne.ucar.edu,r10i1n2.ib0.cheyenne.ucar.edu -np 2 ./wrf_hydro.exe\n', ('cast_2012121800/member_000/job_test_job_1/diag_hydro.00000', 'cast_2012121800/member_001/job_test_job_1/diag_hydro.00000'): 'mpirun --host r10i1n3.ib0.cheyenne.ucar.edu,r10i1n3.ib0.cheyenne.ucar.edu -np 2 ./wrf_hydro.exe\n', } for tup, ans in file_check.items(): for file in tup: check_first_line(file, ans)
def test_cycle_addjob(job_restart, init_times, restart_dirs): cy1 = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs) cy1.add(job_restart) assert deepdiff.DeepDiff(cy1._job, job_restart) == {} job_restart.job_id = 'a_different_id' cy1.add(job_restart) assert deepdiff.DeepDiff(cy1._job, job_restart) == {}
def test_cycle_addscheduler(scheduler, init_times, restart_dirs): cy1 = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs) cy1.add(scheduler) assert deepdiff.DeepDiff(cy1._scheduler, scheduler) == {} sched2 = copy.deepcopy(scheduler) sched2.nnodes = 99 cy1.add(sched2) assert deepdiff.DeepDiff(cy1._scheduler, sched2) == {}
def test_cycle_addsimulation_translate( restart_dirs, expected, init_times, simulation_compiled, job_restart, tmpdir ): os.chdir(tmpdir) sim = simulation cy1 = CycleSimulation( init_times=init_times, restart_dirs=restart_dirs ) sim_compiled = simulation_compiled cy1.add(sim_compiled) cy1.add(job_restart) # translation happens on compose. try: os.mkdir(tmpdir / 'compose') os.chdir(tmpdir / 'compose') # This may be use in some of the tests pathlib.Path('../dummy_extant_dir').touch() cy1.compose(rm_casts_from_memory=False) lsm_keys = ['noahlsm_offline', 'restart_filename_requested'] hyd_keys = ['hydro_nlist', 'restart_file'] result = { 'lsm': [cast.base_hrldas_namelist[lsm_keys[0]][lsm_keys[1]] for cast in cy1.casts], 'hyd': [cast.base_hydro_namelist[hyd_keys[0]][hyd_keys[1]] for cast in cy1.casts] } except Exception as e: result = [str(e)] # Cludgy but it works if isinstance(expected, dict): for key, v_list in expected.items(): for ii, path in enumerate(v_list): expected[key][ii] = sub_tmpdir(path, tmpdir) assert result == expected
def test_cycle_ensemble_parallel_compose(ensemble, job_restart, scheduler, tmpdir, init_times, restart_dirs_ensemble): ens = ensemble cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs_ensemble, ncores=2) cy.add(job_restart) # Adding the scheduler ruins the run in CI. # cy.add(scheduler) # Make a copy where we keep the casts in memory for checking. cy_check_casts = copy.deepcopy(cy) cy_ens_compose = copy.deepcopy(cy) with pytest.raises(Exception) as e_info: cy.compose() cy_ens_compose.add(ens) compose_dir = pathlib.Path(tmpdir).joinpath('cycle_ensemble_compose') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) pathlib.Path('../dummy_extant_dir').touch() cy_ens_compose.compose() cy_run_success = cy_ens_compose.run() assert cy_run_success == 0 cy.pickle( str( pathlib.Path(tmpdir) / 'cycle_ensemble_compose/WrfHydroCycleEns.pkl')) # Is this pickle used? # The cycle-in-memory version for checking the casts. cy_check_casts.add(ens) compose_dir = pathlib.Path(tmpdir).joinpath('cycle_compose_check_casts') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) pathlib.Path('../dummy_extant_dir').touch() cy_check_casts.compose(rm_casts_from_memory=False, rm_members_from_memory=False) # The job gets heavily modified on compose. answer = { '_entry_cmd': 'bogus entry cmd', '_exe_cmd': './wrf_hydro.exe', '_exit_cmd': 'bogus exit cmd', '_hrldas_namelist': { 'noahlsm_offline': { 'btr_option': 1, 'canopy_stomatal_resistance_option': 1, 'hrldas_setup_file': './NWM/DOMAIN/wrfinput_d01.nc', 'indir': './FORCING', 'output_timestep': 86400, 'restart_filename_requested': './NWM/RESTART/RESTART.2011082600_DOMAIN1', 'restart_frequency_hours': 24 }, 'wrf_hydro_offline': { 'forc_typ': 1 } }, '_hrldas_times': { 'noahlsm_offline': { 'khour': 282480, 'restart_frequency_hours': 24, 'output_timestep': 86400, 'restart_filename_requested': 'NWM/RESTART/RESTART.2013101300_DOMAIN1', 'start_day': 12, 'start_hour': 00, 'start_min': 00, 'start_month': 12, 'start_year': 2012 } }, '_hydro_namelist': { 'hydro_nlist': { 'aggfactrt': 4, 'channel_option': 2, 'chanobs_domain': 0, 'chanrtswcrt': 1, 'chrtout_domain': 1, 'geo_static_flnm': './NWM/DOMAIN/geo_em.d01.nc', 'restart_file': './NWM/RESTART/HYDRO_RST.2011-08-26_00:00_DOMAIN1', 'udmp_opt': 1, 'rst_dt': 1440, 'out_dt': 1440 }, 'nudging_nlist': { 'maxagepairsbiaspersist': 3, 'minnumpairsbiaspersist': 1, 'nudginglastobsfile': './NWM/RESTART/nudgingLastObs.2011-08-26_00:00:00.nc' } }, '_hydro_times': { 'hydro_nlist': { 'out_dt': 1440, 'rst_dt': 1440, 'restart_file': 'NWM/RESTART/HYDRO_RST.2013-10-13_00:00_DOMAIN1' }, 'nudging_nlist': { 'nudginglastobsfile': 'NWM/RESTART/nudgingLastObs.2013-10-13_00:00:00.nc' } }, '_job_end_time': None, '_job_start_time': None, '_job_submission_time': None, '_model_end_time': pandas.Timestamp('2045-03-04 00:00:00'), '_model_start_time': pandas.Timestamp('2012-12-12 00:00:00'), 'exit_status': None, 'job_id': 'test_job_1', 'restart_freq_hr_hydro': None, 'restart_freq_hr_hrldas': None, 'output_freq_hr_hydro': None, 'output_freq_hr_hrldas': None, 'restart': True, 'restart_dir': None, '_restart_dir_hydro': None, '_restart_dir_hrldas': None, 'restart_file_time': '2013-10-13', '_restart_file_time_hydro': pandas.Timestamp('2013-10-13 00:00:00'), '_restart_file_time_hrldas': pandas.Timestamp('2013-10-13 00:00:00') } # These answer patches respond to the variety of things in restart_dirs_ensemble dum_ext = str(tmpdir) + '/dummy_extant_dir/' answer_patches = { 'cast_index': [0, 1, 2], 'start_time_patch': [ pandas.Timestamp('2012-12-12 00:00:00'), pandas.Timestamp('2012-12-15 00:00:00'), pandas.Timestamp('2012-12-18 00:00:00') ], 'end_time_patch': [ pandas.Timestamp('2045-03-04 00:00:00'), pandas.Timestamp('2045-03-07 00:00:00'), pandas.Timestamp('2045-03-10 00:00:00') ], # These "time patches" reveal the awkwardness of that construct. 'lsm_times_patch': [ 'NWM/RESTART/RESTART.2013101300_DOMAIN1', '../../cast_2012121200/member_000/RESTART.2013101300_DOMAIN1', dum_ext + 'RESTART.2013101300_DOMAIN1' ], 'hydro_times_patch': [ 'NWM/RESTART/HYDRO_RST.2013-10-13_00:00_DOMAIN1', '../../cast_2012121200/member_000/HYDRO_RST.2013-10-13_00:00_DOMAIN1', dum_ext + 'HYDRO_RST.2013-10-13_00:00_DOMAIN1' ], 'ndg_times_patch': [ 'NWM/RESTART/nudgingLastObs.2013-10-13_00:00:00.nc', '../../cast_2012121200/member_000/nudgingLastObs.2013-10-13_00:00:00.nc', dum_ext + 'nudgingLastObs.2013-10-13_00:00:00.nc' ], # These namelist patches are consistent with the model times except in the # first "do nothing" case which leaves the start time != restart file time 'lsm_nlst_patch': [ './NWM/RESTART/RESTART.2011082600_DOMAIN1', '../../cast_2012121200/member_000/RESTART.2012121500_DOMAIN1', dum_ext + 'RESTART.2012121800_DOMAIN1' ], 'hydro_nlst_patch': [ './NWM/RESTART/HYDRO_RST.2011-08-26_00:00_DOMAIN1', '../../cast_2012121200/member_000/HYDRO_RST.2012-12-15_00:00_DOMAIN1', dum_ext + 'HYDRO_RST.2012-12-18_00:00_DOMAIN1' ], 'ndg_nlst_patch': [ './NWM/RESTART/nudgingLastObs.2011-08-26_00:00:00.nc', '../../cast_2012121200/member_000/nudgingLastObs.2012-12-15_00:00:00.nc', dum_ext + 'nudgingLastObs.2012-12-18_00:00:00.nc' ] } # Check a cycle where the compse retains the casts (otherwise nothing in memory). # This fails: # deepdiff.DeepDiff(answer, cy.casts[0].jobs[0].__dict__) # Instead, iterate on keys to "declass": # Just check the first ensemble cast. def sub_member(the_string, replace_num, find_num=0): replace = "member_{:03d}".format(replace_num) find = "member_{:03d}".format(find_num) return the_string.replace(find, replace) for ii in answer_patches['cast_index']: cc = cy_check_casts.casts[ii] for mm, member in enumerate(cc.members): answer['_model_start_time'] = answer_patches['start_time_patch'][ ii] answer['_model_end_time'] = answer_patches['end_time_patch'][ii] keys = ['noahlsm_offline', 'restart_filename_requested'] answer['_hrldas_namelist'][keys[0]][keys[1]] = \ sub_member(answer_patches['lsm_nlst_patch'][ii], mm) answer['_hrldas_times'][keys[0]][keys[1]] = \ sub_member(answer_patches['lsm_times_patch'][ii], mm) keys = ['_hydro_namelist', 'hydro_nlist', 'restart_file'] answer[keys[0]][keys[1]][keys[2]] = \ sub_member(answer_patches['hydro_nlst_patch'][ii], mm) keys = ['_hydro_namelist', 'nudging_nlist', 'nudginglastobsfile'] answer[keys[0]][keys[1]][keys[2]] = \ sub_member(answer_patches['ndg_nlst_patch'][ii], mm) keys = ['_hydro_times', 'hydro_nlist', 'restart_file'] answer[keys[0]][keys[1]][keys[2]] =\ sub_member(answer_patches['hydro_times_patch'][ii], mm) keys = ['_hydro_times', 'nudging_nlist', 'nudginglastobsfile'] answer[keys[0]][keys[1]][keys[2]] = \ sub_member(answer_patches['ndg_times_patch'][ii], mm) # hrldas times fmt_keys = { '%Y': 'start_year', '%m': 'start_month', '%d': 'start_day', '%H': 'start_hour' } the_mutable = answer['_hrldas_times']['noahlsm_offline'] for fmt, key in fmt_keys.items(): the_mutable[key] = int( answer['_model_start_time'].strftime(fmt)) # Actually check for kk in member.jobs[0].__dict__.keys(): assert member.jobs[0].__dict__[kk] == answer[kk] # Check the scheduler too # assert cy_check_casts.casts[0].scheduler.__dict__ == scheduler.__dict__ # For the cycle where the compse removes the casts... # Check that the casts are all now simply pathlib objects assert all([type(mm) is str for mm in cy.casts]) # the tmpdir gets nuked after the test... ? # Test the cast pickle size in terms of load speed. # Note that the deletion of the model, domain, and output objects are # done for the casts regardless of not removing the casts # from memory (currently). os.chdir( str(pathlib.Path(tmpdir) / 'cycle_ensemble_compose/cast_2012121200')) time_taken = timeit.timeit( setup='import pickle', stmt='pickle.load(open("WrfHydroEns.pkl","rb"))', number=10000) # If your system is busy, this could take longer... and spuriously fail the test. # Notes(JLM): coverage makes this slow assert time_taken < 1.5 # Test the cycle pickle size in terms of load speed. os.chdir(str(pathlib.Path(tmpdir) / 'cycle_ensemble_compose/')) time_taken = timeit.timeit( setup='import pickle', stmt='pickle.load(open("WrfHydroCycleEns.pkl","rb"))', number=10000) # If your system is busy, this could take longer... # Notes(JLM): coveage makes this slow assert time_taken < 1.5
def test_cycle_parallel_compose(simulation_compiled, job_restart, scheduler, tmpdir, init_times, restart_dirs): """ A more comprehensive test of the object composed.""" # A compiled simulation passed. Successfull compose in parallel. cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs, ncores=2) cy.add(job_restart) cy.add(simulation_compiled) # Make a copy where we keep the casts in memory for checking. cy_check_casts = copy.deepcopy(cy) compose_dir = pathlib.Path(tmpdir).joinpath('cycle_compose') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) cy.compose() cy_run_success = cy.run() assert cy_run_success == 0 cy.pickle(str(pathlib.Path(tmpdir) / 'cycle_compose/WrfHydroCycleSim.pkl')) # The cycle-in-memory version for checking the casts. compose_dir = pathlib.Path(tmpdir).joinpath('cycle_compose_check_casts') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) cy_check_casts.compose(rm_casts_from_memory=False) # The job gets heavily modified on compose. answer = { '_entry_cmd': 'bogus entry cmd', '_exe_cmd': './wrf_hydro.exe', '_exit_cmd': 'bogus exit cmd', '_hrldas_namelist': { 'noahlsm_offline': { 'btr_option': 1, 'canopy_stomatal_resistance_option': 1, 'hrldas_setup_file': './NWM/DOMAIN/wrfinput_d01.nc', 'indir': './FORCING', 'output_timestep': 86400, 'restart_filename_requested': './NWM/RESTART/RESTART.2011082600_DOMAIN1', 'restart_frequency_hours': 24 }, 'wrf_hydro_offline': { 'forc_typ': 1 } }, '_hrldas_times': { 'noahlsm_offline': { 'khour': 282480, 'restart_frequency_hours': 24, 'output_timestep': 86400, 'restart_filename_requested': 'NWM/RESTART/RESTART.2013101300_DOMAIN1', 'start_day': 12, 'start_hour': 00, 'start_min': 00, 'start_month': 12, 'start_year': 2012 } }, '_hydro_namelist': { 'hydro_nlist': { 'aggfactrt': 4, 'channel_option': 2, 'chanobs_domain': 0, 'chanrtswcrt': 1, 'chrtout_domain': 1, 'geo_static_flnm': './NWM/DOMAIN/geo_em.d01.nc', 'restart_file': './NWM/RESTART/HYDRO_RST.2011-08-26_00:00_DOMAIN1', 'udmp_opt': 1, 'rst_dt': 1440, 'out_dt': 1440 }, 'nudging_nlist': { 'maxagepairsbiaspersist': 3, 'minnumpairsbiaspersist': 1, 'nudginglastobsfile': './NWM/RESTART/nudgingLastObs.2011-08-26_00:00:00.nc' } }, '_hydro_times': { 'hydro_nlist': { 'out_dt': 1440, 'rst_dt': 1440, 'restart_file': 'NWM/RESTART/HYDRO_RST.2013-10-13_00:00_DOMAIN1' }, 'nudging_nlist': { 'nudginglastobsfile': 'NWM/RESTART/nudgingLastObs.2013-10-13_00:00:00.nc' } }, '_job_end_time': None, '_job_start_time': None, '_job_submission_time': None, '_model_end_time': pandas.Timestamp('2045-03-04 00:00:00'), '_model_start_time': pandas.Timestamp('2012-12-12 00:00:00'), 'exit_status': None, 'job_id': 'test_job_1', 'restart_freq_hr_hydro': None, 'restart_freq_hr_hrldas': None, 'output_freq_hr_hydro': None, 'output_freq_hr_hrldas': None, 'restart': True, 'restart_dir': None, '_restart_dir_hydro': None, '_restart_dir_hrldas': None, 'restart_file_time': '2013-10-13', '_restart_file_time_hydro': pandas.Timestamp('2013-10-13 00:00:00'), '_restart_file_time_hrldas': pandas.Timestamp('2013-10-13 00:00:00') } # For the cycle where the compse retains the casts... # This fails: # deepdiff.DeepDiff(answer, cy.casts[0].jobs[0].__dict__) # Instead, iterate on keys to "declass": for kk in cy_check_casts.casts[0].jobs[0].__dict__.keys(): assert cy_check_casts.casts[0].jobs[0].__dict__[kk] == answer[kk] # Check the scheduler too # assert cy_check_casts.casts[0].scheduler.__dict__ == scheduler.__dict__ # For the cycle where the compse removes the casts... # Check that the casts are all now simply pathlib objects assert all([type(mm) is str for mm in cy.casts]) # the tmpdir gets nuked after the test... ? # Test the cast pickle size in terms of load speed. # Note that the deletion of the model, domain, and output objects are # done for the casts regardless of not removing the casts # from memory (currently). os.chdir(str(pathlib.Path(tmpdir) / 'cycle_compose/cast_2012121200')) time_taken = timeit.timeit( setup='import pickle', stmt='pickle.load(open("WrfHydroSim.pkl","rb"))', number=10000) # If your system is busy, this could take longer... and spuriously fail the test. # Notes(JLM): coverage is the limiting factor assert time_taken < 1.5 # Test the cycle pickle size in terms of load speed. os.chdir(str(pathlib.Path(tmpdir) / 'cycle_compose/')) time_taken = timeit.timeit( setup='import pickle', stmt='pickle.load(open("WrfHydroCycleSim.pkl","rb"))', number=10000) # If your system is busy, this could take longer... # Notes(JLM): coverage is the limiting factor. assert time_taken < 1.2
def test_cycle_ensemble_compose(ensemble, job_restart, scheduler, tmpdir, init_times, restart_dirs_ensemble): # These might be parametizable. # Length zero cycle compose. cy = CycleSimulation(init_times=[], restart_dirs=[], ncores=1) cy.add(job_restart) cy.add(ensemble) compose_dir = pathlib.Path(tmpdir).joinpath('cycle_sim0_compose') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) with pytest.raises(Exception) as e_info: cy.compose() assert str(e_info.value) == "There are no casts (init_times) to compose." # Inconsistent forcing dir length and ensemble length cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs_ensemble, ncores=1, forcing_dirs=[['.', '.', '../dummy_extant_dir'], ['.', -72, '../dummy_extant_dir'], ['.', -72, '../dummy_extant_dir']]) cy.add(job_restart) with pytest.raises(Exception) as e_info: cy.add(ensemble) assert str(e_info.value) == \ "Ensemble to add has inconsistent length with existing cycle forcing_dirs" # Inconsistent forcing dir length and ensemble length cy = CycleSimulation(init_times=init_times, restart_dirs=[[rr[0]] for rr in restart_dirs_ensemble], ncores=1, forcing_dirs=[['.', '../dummy_extant_dir'], [-72, '../dummy_extant_dir'], [-72, '../dummy_extant_dir']]) cy.add(job_restart) with pytest.raises(Exception) as e_info: cy.add(ensemble) assert str(e_info.value) == \ "Ensemble to add has inconsistent length with existing cycle restart_dirs" # Valid force dir exercise. cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs_ensemble, ncores=1, forcing_dirs=[['.', '../dummy_extant_dir'], [-72, '../dummy_extant_dir'], [-72, '../dummy_extant_dir']]) cy.add(job_restart) cy.add(ensemble) compose_dir = pathlib.Path(tmpdir).joinpath('cycle_forc_dir_compose') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) compose_dir.joinpath('../dummy_extant_dir').touch() cy.compose() # In valid force dir exercise. cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs_ensemble, ncores=1, forcing_dirs=[[72, '../dummy_extant_dir'], [72, '../dummy_extant_dir'], [72, '../dummy_extant_dir']]) cy.add(job_restart) cy.add(ensemble) compose_dir = pathlib.Path(tmpdir).joinpath( 'cycle_forc_dir_fail_1_compose') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) compose_dir.joinpath('../dummy_extant_dir').touch() with pytest.raises(Exception) as e_info: cy.compose() assert str( e_info.value ) == 'Only non-negative integers can be used to specify forcing_dirs' # In valid force dir exercise. cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs_ensemble, ncores=1, forcing_dirs=[['.', 'dummy_non-extant_dir'], ['.', 'dummy_non-extant_dir'], ['.', 'dummy_non-extant_dir']]) cy.add(job_restart) cy.add(ensemble) compose_dir = pathlib.Path(tmpdir).joinpath( 'cycle_forc_dir_fail_2_compose') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) compose_dir.joinpath('../dummy_extant_dir').touch() with pytest.raises(Exception) as e_info: cy.compose() assert str( e_info.value) == 'No such forcing directory: dummy_non-extant_dir'
def test_cycle_compose(simulation, simulation_compiled, job_restart, scheduler, tmpdir, init_times, restart_dirs): # These might be parametizable. # Compose without adding a simulation. cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs, ncores=2) cy.add(job_restart) # Adding the scheduler ruins the run in CI. # cy.add(scheduler) with pytest.raises(Exception) as e_info: cy.compose() assert str(e_info.value) == \ 'Unable to compose, current working directory is not empty. \n' + \ 'Change working directory to an empty directory with os.chdir()' compose_dir = pathlib.Path(tmpdir).joinpath('cycle_no_sim_compose') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) with pytest.raises(Exception) as e_info: cy.compose() assert str(e_info.value ) == 'The cycle does not contain a _simulation or an _ensemble.' # Length zero cycle compose. cy = CycleSimulation(init_times=[], restart_dirs=[], ncores=1) cy.add(job_restart) cy.add(simulation_compiled) compose_dir = pathlib.Path(tmpdir).joinpath('cycle_sim0_compose') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) with pytest.raises(Exception) as e_info: cy.compose() assert str(e_info.value) == "There are no casts (init_times) to compose." # This simultion is not compiled. It compiles and composes successfully. cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs, ncores=1) cy.add(job_restart) cy.add(simulation) compose_dir = pathlib.Path(tmpdir).joinpath('cycle_uncompiled_compose') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) cy.compose() # Valid force dir exercise. cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs, ncores=1, forcing_dirs=['.', -72, '../dummy_extant_dir']) cy.add(job_restart) cy.add(simulation) compose_dir = pathlib.Path(tmpdir).joinpath('cycle_forc_dir_compose') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) compose_dir.joinpath('../dummy_extant_dir').touch() cy.compose() # In valid force dir exercise. cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs, ncores=1, forcing_dirs=['.', 72, '../dummy_extant_dir']) cy.add(job_restart) cy.add(simulation) compose_dir = pathlib.Path(tmpdir).joinpath( 'cycle_forc_dir_fail_2_compose') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) with pytest.raises(Exception) as e_info: cy.compose() assert str( e_info.value ) == 'Only non-negative integers can be used to specify forcing_dirs' # In valid force dir exercise. cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs, ncores=1, forcing_dirs=['.', 'dummy_non-extant_dir', -72]) cy.add(job_restart) cy.add(simulation) compose_dir = pathlib.Path(tmpdir).joinpath('cycle_forc_dir_fail_compose') os.mkdir(str(compose_dir)) os.chdir(str(compose_dir)) with pytest.raises(Exception) as e_info: cy.compose() assert str( e_info.value) == 'No such forcing directory: dummy_non-extant_dir'
def test_cycle_addsimulation( simulation, job_restart, scheduler, simulation_compiled, init_times ): sim = simulation cy1 = CycleSimulation( init_times=init_times, restart_dirs=['.'] * len(init_times) ) # This sim does not have the required pre-compiled model with pytest.raises(Exception) as e_info: cy1.add(sim) sim_compiled = simulation_compiled # cant add a list, even if pre-compiled with pytest.raises(Exception) as e_info: cy1.add([sim_compiled]) cy1.add(sim_compiled) # add a sim with job and make sure it is deleted. sim_compiled.add(job_restart) sim_compiled.add(scheduler) cy2 = CycleSimulation( init_times=init_times, restart_dirs=['.'] * len(init_times) ) cy2.add(sim_compiled) assert all([len(cc.jobs) == 0 for cc in cy2.casts]) assert all([cc.scheduler is None for cc in cy2.casts])
def test_cycle_run_parallel_teams(simulation_compiled, job_restart, scheduler, tmpdir, capfd, init_times, restart_dirs): sim = simulation_compiled tmp = pathlib.Path(tmpdir) forcing_dirs = [ tmp / letter for letter in list(string.ascii_lowercase)[0:len(init_times)] ] for dir in forcing_dirs: dir.mkdir() cy = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs, forcing_dirs=forcing_dirs) cy.add(sim) job_restart._entry_cmd = 'echo mpirun entry_cmd > entry_cmd.output' job_restart._exit_cmd = 'echo mpirun exit_cmd > exit_cmd.output' cy.add(job_restart) # Parallel teams test - test casts both in memory and not cy_teams_to_test = { 'cy_teams_not_memory': copy.deepcopy(cy), 'cy_teams_memory': copy.deepcopy(cy) } for key, cy_teams in cy_teams_to_test.items(): cy_dir = pathlib.Path(tmpdir).joinpath(key) os.chdir(tmpdir) os.mkdir(str(cy_dir)) os.chdir(str(cy_dir)) cy_teams_cp = copy.deepcopy(cy_teams) if key == 'cy_teams_not_memory': cy_teams_cp.compose() else: cy_teams_cp.compose(rm_casts_from_memory=False) with pytest.raises(Exception) as e_info: cy_teams_run_fail_xnode = cy_teams_cp.run( teams=True, teams_exe_cmd= ' ./wrf_hydro.exe mpirun --host {nodelist} -np {nproc} {cmd}', teams_exe_cmd_nproc=3, teams_node_file=node_file) the_error = ( "ValueError('teams_exe_cmd_nproc > number of cores/node: " "teams does not currently function in this capacity.',)") assert repr(e_info._excinfo[1] ) == the_error, 'Teams is not failing on xnode request' os.chdir(tmpdir) shutil.rmtree(str(cy_dir)) os.mkdir(str(cy_dir)) os.chdir(str(cy_dir)) if key == 'cy_teams_not_memory': cy_teams.compose() else: cy_teams.compose(rm_casts_from_memory=False) cy_teams_run_success = cy_teams.run( teams=True, teams_exe_cmd= ' ./wrf_hydro.exe mpirun --host {nodelist} -np {nproc} {cmd}', teams_exe_cmd_nproc=2, teams_node_file=node_file) assert cy_teams_run_success == 0, \ "Some parallel team cycle casts did not run successfully." assert cy_dir.joinpath("WrfHydroCycle.pkl").exists() # Check for command correctness in output files. file_check = { ('cast_2012121200/entry_cmd.output', 'cast_2012121500/entry_cmd.output', 'cast_2012121800/entry_cmd.output'): 'mpirun entry_cmd\n', ('cast_2012121200/exit_cmd.output', 'cast_2012121500/exit_cmd.output', 'cast_2012121800/exit_cmd.output'): 'mpirun exit_cmd\n', ('cast_2012121200/job_test_job_1/diag_hydro.00000', ): 'mpirun --host r10i1n1.ib0.cheyenne.ucar.edu,r10i1n1.ib0.cheyenne.ucar.edu -np 2 ./wrf_hydro.exe\n', ('cast_2012121500/job_test_job_1/diag_hydro.00000', ): 'mpirun --host r10i1n2.ib0.cheyenne.ucar.edu,r10i1n2.ib0.cheyenne.ucar.edu -np 2 ./wrf_hydro.exe\n', ('cast_2012121800/job_test_job_1/diag_hydro.00000', ): 'mpirun --host r10i1n3.ib0.cheyenne.ucar.edu,r10i1n3.ib0.cheyenne.ucar.edu -np 2 ./wrf_hydro.exe\n', } for tup, ans in file_check.items(): for file in tup: check_first_line(file, ans)
def test_cycle_addensemble(ensemble, job_restart, scheduler, init_times, restart_dirs, restart_dirs_ensemble): # The ensemble necessarily has a compiled model (unlike a Simulation). # That is a separate test. cy1 = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs) with pytest.raises(Exception) as e_info: cy1.add(ensemble) assert str(e_info.value) == \ 'An ensemble cycle simulation requires the restart_dirs to be a list of lists.' cy1 = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs_ensemble) cy1.add(ensemble) assert isinstance(cy1._ensemble, EnsembleSimulation) # add an ens with a job and make sure it is deleted. ens = ensemble ens.add(job_restart) ens.add(scheduler) cy2 = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs_ensemble) cy2.add(ensemble) assert cy2._ensemble.jobs == [] assert cy2._ensemble.scheduler is None
def test_cycle_addsimulation(simulation, job_restart, scheduler, simulation_compiled, init_times, restart_dirs): sim = 'not a simulation object' cy1 = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs) with pytest.raises(Exception) as e_info: cy1.add(sim) assert str(e_info.value ) == 'Object is not of a type expected for a CycleSimulation.' sim = simulation cy1 = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs) # This sim does not have the required pre-compiled model # with pytest.raises(Exception) as e_info: cy1.add(sim) # assert str(e_info.value) == \ # 'Only Simulations with compiled model objects can be added to an ensemble simulation.' sim_compiled = simulation_compiled # cant add a list, even if pre-compiled with pytest.raises(Exception) as e_info: cy1.add([sim_compiled]) assert str(e_info.value ) == 'Object is not of a type expected for a CycleSimulation.' cy1.add(sim_compiled) assert isinstance(cy1._simulation, Simulation) # add a sim with job and make sure it is deleted. sim_compiled.add(job_restart) sim_compiled.add(scheduler) cy2 = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs) cy2.add(sim_compiled) assert cy2._simulation.jobs == [] assert cy2._simulation.scheduler is None