def test_run_candidate_nwm_output_sim(candidate_sim, candidate_nwm_output_sim, output_dir, ncores): if candidate_nwm_output_sim.model.model_config.lower().find('nwm') < 0: pytest.skip('NWM Output test only applicable to nwm configs') print( "\nQuestion: The candidate nwm ouput configuration runs successfully?\n", end='') print('\n') # Set run directory and change working directory to run dir for simulation run_dir = output_dir / 'nwm_output_candidate' if run_dir.exists(): pytest.skip( 'Candidate nwm output run exists, skipping nwm candidate output run.' ) run_dir.mkdir(parents=True) os.chdir(str(run_dir)) candidate_nwm_output_sim_copy = copy.deepcopy(candidate_nwm_output_sim) # Dont recompile the model, just use the candidate's model. candidate_nwm_output_sim_copy.model = copy.deepcopy(candidate_sim.model) # Job exe_command = 'mpirun -np {0} ./wrf_hydro.exe'.format(str(ncores)) job = wrfhydropy.Job(job_id='run_candidate', exe_cmd=exe_command, restart_freq_hr=1, output_freq_hr=1) candidate_nwm_output_sim_copy.add(job) # start_time, end_time = candidate_nwm_output_sim_copy.jobs[0]._solve_model_start_end_times() # candidate_nwm_output_sim_copy.jobs[0].model_start_time = start_time # candidate_nwm_output_sim_copy.jobs[0].model_end_time = start_time + dt.timedelta(hours=24) # candidate_nwm_output_sim_copy.jobs[0].restart_freq_hr = 1 # Run, catch warnings related to missing start and end job times with warnings.catch_warnings(): warnings.simplefilter("ignore") candidate_nwm_output_sim_copy.compose() print('\nwaiting for job to complete...', end='') candidate_nwm_output_sim_copy.run() # Wait to collect until job has finished. All test runs are performed on a single job with # job_id='test_job' wait_job(candidate_nwm_output_sim_copy) candidate_nwm_output_sim_copy.collect() candidate_nwm_output_sim_copy.pickle( run_dir.joinpath('WrfHydroSim_collected.pkl')) # Check job run statuses for job in candidate_nwm_output_sim.jobs: assert job.exit_status == 0, \ "Candidate run exited with non-zero status"
def test_run_candidate(candidate_sim, output_dir, ncores, exe_cmd): print("\nQuestion: The candidate runs successfully?\n", end='') print('\n') candidate_sim_copy = copy.deepcopy(candidate_sim) # Set run directory and change working directory to run dir for simulation run_dir = output_dir / 'run_candidate' if run_dir.exists(): pytest.skip('Candidate run dir exists, skipping candidate run test') run_dir.mkdir(parents=True) os.chdir(str(run_dir)) # Job exe_command = exe_cmd.format(str(ncores)) out_dt = 1 if 'channel' in candidate_sim.model.model_config else 24 job = wrfhydropy.Job( job_id='run_candidate', exe_cmd=exe_command, restart_freq_hr=24, output_freq_hr=out_dt ) candidate_sim_copy.add(job) # Run, catch warnings related to missing start and end job times with warnings.catch_warnings(): warnings.simplefilter("ignore") candidate_sim_copy.compose() print('\nwaiting for job to complete...', end='') candidate_sim_copy.run() # Wait to collect until job has finished. All test runs are performed on a single job with # job_id='test_job' wait_job(candidate_sim_copy) candidate_sim_copy.collect() candidate_sim_copy.pickle(run_dir.joinpath('WrfHydroSim_collected.pkl')) # Check job run statuses for job in candidate_sim_copy.jobs: assert job.exit_status == 0, \ "Candidate code run exited with non-zero status"
def test_run_reference(reference_sim, output_dir, ncores): print("\nQuestion: The reference runs successfully?\n", end='') print('\n') reference_sim_copy = copy.deepcopy(reference_sim) # Set run directory and change working directory to run dir for simulation run_dir = output_dir / 'run_reference' run_dir.mkdir(parents=True) os.chdir(str(run_dir)) # Job exe_command = 'mpirun -np {0} ./wrf_hydro.exe'.format(str(ncores)) job = wrfhydropy.Job(job_id='run_reference', exe_cmd=exe_command, restart_freq_hr=24, output_freq_hr=24) reference_sim_copy.add(job) # Run, catch warnings related to missing start and end job times with warnings.catch_warnings(): warnings.simplefilter("ignore") reference_sim_copy.compose() print('\nwaiting for job to complete...', end='') reference_sim_copy.run() # Wait to collect until job has finished. All test runs are performed on a single job with # job_id='test_job' wait_job(reference_sim_copy) reference_sim_copy.collect() reference_sim_copy.pickle(run_dir.joinpath('WrfHydroSim_collected.pkl')) # Check job run statuses for job in reference_sim_copy.jobs: assert job.exit_status == 0, \ "Reference code run exited with non-zero status"
def test_perfrestart_candidate(output_dir): print( "\nQuestion: The candidate outputs from a restart run match the outputs from standard " "run?\n", end='') print('\n') candidate_sim_file = output_dir / 'run_candidate' / 'WrfHydroSim.pkl' candidate_collected_file = output_dir / 'run_candidate' / 'WrfHydroSim_collected.pkl' if candidate_collected_file.is_file() is False: pytest.skip('Candidate run object not found, skipping test.') # Load initial run model object and copy candidate_sim = pickle.load(candidate_sim_file.open(mode="rb")) candidate_sim_expected = pickle.load( candidate_collected_file.open(mode="rb")) candidate_sim_restart = copy.deepcopy(candidate_sim) # Set run directory run_dir = output_dir.joinpath('restart_candidate') run_dir.mkdir(parents=True) os.chdir(str(run_dir)) # Get a new start time halfway along the run, make sure the restart frequency accomodates restart_job = candidate_sim_restart.jobs[0] duration = restart_job.model_end_time - restart_job.model_start_time delay_restart_hr = int((duration.total_seconds() / 3600) / 2) assert delay_restart_hr % candidate_sim_restart.jobs[0].restart_freq_hr == 0, \ "The restart delay is not a multiple of the restart frequency." restart_job.model_start_time = \ restart_job.model_start_time + dt.timedelta(hours=delay_restart_hr) # Get restart files from previous run and symlink into restart sim dir # (Remember that we are in the run/sim dir) # Hydro: Use actual time listed in meta data, not filename or positional list index for restart_file in candidate_sim_expected.output.restart_hydro: restart_time = restart_file.open().Restart_Time restart_time = pd.to_datetime(restart_time, format='%Y-%m-%d_%H:%M:%S') if restart_time == restart_job.model_start_time: candidate_hydro_restart_file = pathlib.Path(restart_file.name) candidate_hydro_restart_file.symlink_to(restart_file) # LSM: Use actual time listed in meta data, not filename or positional list index for restart_file in candidate_sim_expected.output.restart_lsm: restart_time = restart_file.open().Times[0] restart_time = restart_time.astype(str).item(0) restart_time = pd.to_datetime(restart_time, format='%Y-%m-%d_%H:%M:%S') if restart_time == restart_job.model_start_time: candidate_lsm_restart_file = pathlib.Path(restart_file.name) candidate_lsm_restart_file.symlink_to(restart_file) # Nudging: Use actual time listed in meta data, not filename or positional list index if candidate_sim_expected.output.restart_nudging is not None: for restart_file in candidate_sim_expected.output.restart_nudging: restart_time = restart_file.open().modelTimeAtOutput restart_time = pd.to_datetime(restart_time, format='%Y-%m-%d_%H:%M:%S') if restart_time == restart_job.model_start_time: candidate_nudging_restart_file = pathlib.Path( restart_file.name) candidate_nudging_restart_file.symlink_to(restart_file) # Compose and run # catch warnings related to missing start and end job times with warnings.catch_warnings(): warnings.simplefilter("ignore") candidate_sim_restart.compose(force=True) print('\nwaiting for job to complete...', end='') candidate_sim_restart.run() # Wait to collect until job has finished. All test runs are performed on a single job with wait_job(candidate_sim_restart) candidate_sim_restart.collect() candidate_sim_restart.pickle(run_dir.joinpath('WrfHydroSim_collected.pkl')) # Check outputs with warnings.catch_warnings(): warnings.simplefilter("ignore") diffs = wrfhydropy.outputdiffs.OutputDataDiffs( candidate_sim_restart.output, candidate_sim_expected.output, exclude_vars=EXCLUDE_VARS) # Assert all diff values are 0 and print diff stats if not has_diffs = any(value != 0 for value in diffs.diff_counts.values()) if has_diffs: print_diffs(diffs) assert has_diffs is False, \ 'Outputs for candidate run do not match outputs from candidate restart run'
def test_ncores_candidate(output_dir): print( "\nQuestion: The candidate outputs from a ncores run match outputs from" " ncores-1 run?\n", end='') print('\n') candidate_sim_file = output_dir / 'run_candidate' / 'WrfHydroSim.pkl' candidate_collected_file = output_dir / 'run_candidate' / 'WrfHydroSim_collected.pkl' if candidate_collected_file.is_file() is False: pytest.skip('Candidate run object not found, skipping test.') # Load initial sim object, collect sim_object and copy for makign new sims candidate_sim = pickle.load(candidate_sim_file.open(mode="rb")) candidate_sim_expected = pickle.load( candidate_collected_file.open(mode="rb")) candidate_sim_ncores = copy.deepcopy(candidate_sim) # Set run directory run_dir = output_dir.joinpath('ncores_candidate') run_dir.mkdir(parents=True) os.chdir(str(run_dir)) # Make a new job based on the old job but with a new job ID old_job = candidate_sim.jobs[0] new_job = wrfhydropy.Job(job_id='ncores_candidate', exe_cmd=old_job._exe_cmd, restart_freq_hr=24, output_freq_hr=24) # Remove old job and add new job candidate_sim_ncores.jobs.pop(0) candidate_sim_ncores.add(new_job) # Edit the sim object number of cores if candidate_sim_ncores.scheduler is not None: candidate_sim_ncores.scheduler.nproc = candidate_sim_ncores.scheduler.nproc - 1 else: orig_exe_cmd = candidate_sim_ncores.jobs[0]._exe_cmd orig_exe_cmd = orig_exe_cmd.replace('-np 2', '-np 1') # Recompose into new directory and run # catch warnings related to missing start and end job times with warnings.catch_warnings(): warnings.simplefilter("ignore") candidate_sim_ncores.compose(force=True) print('\nwaiting for job to complete...', end='') candidate_sim_ncores.run() # Wait to collect until job has finished. All test runs are performed on a single job with # job_id='test_job' wait_job(candidate_sim_ncores) candidate_sim_ncores.collect() candidate_sim_ncores.pickle(run_dir.joinpath('WrfHydroSim_collected.pkl')) # Check outputs with warnings.catch_warnings(): warnings.simplefilter("ignore") diffs = wrfhydropy.outputdiffs.OutputDataDiffs( candidate_sim_ncores.output, candidate_sim_expected.output, exclude_vars=EXCLUDE_VARS) # Assert all diff values are 0 and print diff stats if not has_diffs = any(value != 0 for value in diffs.diff_counts.values()) if has_diffs: print_diffs(diffs) assert has_diffs is False, \ 'Outputs for candidate run with ncores do not match outputs with ncores-1'
def test_run_candidate_channel_only(candidate_sim, candidate_channel_only_sim, output_dir, ncores): if candidate_sim.model.model_config.lower().find('nwm') < 0: pytest.skip('Channel-only test only applicable to nwm_ana config') print("\nQuestion: The candidate channel-only mode runs successfully?\n", end='') print('\n') candidate_sim_copy = copy.deepcopy(candidate_sim) candidate_sim_copy.base_hydro_namelist['hydro_nlist'][ 'output_channelbucket_influx'] = 2 candidate_channel_only_sim_copy = copy.deepcopy(candidate_channel_only_sim) candidate_channel_only_sim_copy. \ base_hydro_namelist['hydro_nlist']['output_channelbucket_influx'] = 2 ################## # re-run candidate at shorter duration since requires hourly outputs # Set run directory and change working directory to run dir for simulation run_dir = output_dir / 'candidate_run_output_for_channel_only' run_dir.mkdir(parents=True) os.chdir(str(run_dir)) # Job exe_command = 'mpirun -np {0} ./wrf_hydro.exe'.format(str(ncores)) job = wrfhydropy.Job(job_id='run_candidate', exe_cmd=exe_command, restart_freq_hr=24, output_freq_hr=1) candidate_sim_copy.add(job) start_time, end_time = candidate_sim_copy.jobs[ 0]._solve_model_start_end_times() candidate_sim_copy.jobs[0].model_start_time = start_time candidate_sim_copy.jobs[0].model_end_time = start_time + dt.timedelta( hours=24) candidate_sim_copy.jobs[0].restart_freq_hr = 6 # Run, catch warnings related to missing start and end job times with warnings.catch_warnings(): warnings.simplefilter("ignore") candidate_sim_copy.compose() print('\nwaiting for job to complete...', end='') candidate_sim_copy.run() # Wait to collect until job has finished. All test runs are performed on a single job with # job_id='test_job' wait_job(candidate_sim_copy) candidate_sim_copy.collect() candidate_sim_copy.pickle(run_dir.joinpath('WrfHydroSim_collected.pkl')) # Check job run statuses for job in candidate_sim.jobs: assert job.exit_status == 0, \ "Candidate code run exited with non-zero status" ######################### # Run channel only # Dont recompile the model, just use the candidate's model. candidate_channel_only_sim_copy.model = copy.deepcopy(candidate_sim.model) # Set run directory and go for execution. run_dir = output_dir / 'run_candidate_channel_only' run_dir.mkdir(parents=True) os.chdir(str(run_dir)) # Set the forcing directory candidate_channel_only_sim_copy.base_hrldas_namelist['noahlsm_offline']['indir'] = \ str(output_dir / 'candidate_run_output_for_channel_only') # Job exe_command = 'mpirun -np {0} ./wrf_hydro.exe'.format(str(ncores)) job = wrfhydropy.Job(job_id='run_candidate_channel_only', exe_cmd=exe_command, restart_freq_hr=24, output_freq_hr=1) candidate_channel_only_sim_copy.add(job) start_time, end_time = candidate_channel_only_sim_copy.jobs[ 0]._solve_model_start_end_times() candidate_channel_only_sim_copy.jobs[0].model_start_time = start_time candidate_channel_only_sim_copy.jobs[ 0].model_end_time = start_time + dt.timedelta(hours=24) candidate_channel_only_sim_copy.jobs[0].restart_freq_hr = 6 # Run with warnings.catch_warnings(): warnings.simplefilter("ignore") candidate_channel_only_sim_copy.compose() print('\nwaiting for job to complete...', end='') candidate_channel_only_sim_copy.run() # Wait to collect until job has finished. All test runs are performed on a single job with # job_id='test_job' wait_job(candidate_channel_only_sim_copy) candidate_channel_only_sim_copy.collect() candidate_channel_only_sim_copy.pickle( run_dir.joinpath('WrfHydroSim_collected.pkl')) # Check job run statuses for job in candidate_channel_only_sim_copy.jobs: assert job.exit_status == 0, \ "Candidate channel-only code run exited with non-zero status"
def test_perfrestart_candidate_channel_only(output_dir, xrcmp_n_cores): candidate_channel_only_sim_file = \ output_dir / 'channel_only_candidate_run' / 'WrfHydroSim.pkl' candidate_channel_only_collected_file = \ output_dir / 'channel_only_candidate_run' / 'WrfHydroSim_collected.pkl' if candidate_channel_only_collected_file.is_file() is False: pytest.skip( 'candidate_channel_only run object not found, skipping test') print( "\nQuestion: The candidate_channel_only outputs from a restart run match the outputs " "from standard run?\n", end='') print('\n') candidate_channel_only_sim_expected = \ pickle.load(candidate_channel_only_collected_file.open(mode="rb")) run_dir = output_dir / 'channel_only_candidate_restart' if not run_dir.exists(): run_dir.mkdir(parents=True) os.chdir(str(run_dir)) candidate_channel_only_sim = \ pickle.load(candidate_channel_only_sim_file.open(mode="rb")) candidate_channel_only_sim_restart = copy.deepcopy( candidate_channel_only_sim) # Get a new start time halfway along the run, make sure the restart frequency accomodates restart_job = candidate_channel_only_sim_restart.jobs[0] duration = restart_job.model_end_time - restart_job.model_start_time delay_restart_hr = int((duration.total_seconds() / 3600) / 2) # Want matching restart frequencies for this test... assert \ candidate_channel_only_sim.jobs[0].restart_freq_hr_hydro == \ candidate_channel_only_sim.jobs[0].restart_freq_hr_hrldas, \ "Hydro and HRLDAS components do not have the same restart frequencies." assert delay_restart_hr % candidate_channel_only_sim.jobs[0].restart_freq_hr_hydro == 0, \ "The restart delay is not a multiple of the restart frequency." restart_job.model_start_time = \ restart_job.model_start_time + dt.timedelta(hours=delay_restart_hr) # Get restart files from previous run and symlink into restart sim dir # Hydro: Use actual time listed in meta data not filename or positional list index for restart_file in candidate_channel_only_sim_expected.output.restart_hydro: if isinstance(restart_file, pathlib.Path): restart_time = xr.open_dataset(restart_file).Restart_Time else: restart_time = restart_file.open( ).Restart_Time # backwards compatible wrfhydropy restart_time = pd.to_datetime(restart_time, format='%Y-%m-%d_%H:%M:%S') if restart_time == restart_job.model_start_time: candidate_hydro_restart_file = pathlib.Path(restart_file.name) candidate_hydro_restart_file.symlink_to(restart_file) key1 = 'hydro_nlist' key2 = 'restart_file' restart_job._hydro_namelist[key1][ key2] = candidate_hydro_restart_file # Nudging: Use actual time listed in meta data not filename or positional list index if candidate_channel_only_sim_expected.output.restart_nudging is not None: for restart_file in candidate_channel_only_sim_expected.output.restart_nudging: if isinstance(restart_file, pathlib.Path): restart_time = xr.open_dataset( restart_file).modelTimeAtOutput else: restart_time = restart_file.open( ).modelTimeAtOutput # backwards compatible wrfhydropy restart_time = pd.to_datetime(restart_time, format='%Y-%m-%d_%H:%M:%S') if restart_time == restart_job.model_start_time: candidate_nudging_restart_file = pathlib.Path( restart_file.name) candidate_nudging_restart_file.symlink_to(restart_file) key1 = 'nudging_nlist' key2 = 'nudginglastobsfile' restart_job._hydro_namelist[key1][ key2] = candidate_nudging_restart_file # Compose and run # Catch warnings related to missing start and end job times with warnings.catch_warnings(): warnings.simplefilter("ignore") candidate_channel_only_sim_restart.compose(force=True) print('\nwaiting for job to complete...', end='') candidate_channel_only_sim_restart.run() wait_job(candidate_channel_only_sim_restart) candidate_channel_only_sim_restart.collect() candidate_channel_only_sim_restart.pickle( run_dir.joinpath('WrfHydroSim_collected.pkl')) else: candidate_channel_only_sim_restart = pickle.load( open(run_dir.joinpath('WrfHydroSim_collected.pkl'), 'rb')) for job in candidate_channel_only_sim_restart.jobs: assert job.exit_status == 0, \ "Candidate channel-only ncores run exited with non-zero status" # Check outputs with warnings.catch_warnings(): warnings.simplefilter("ignore") diffs = wrfhydropy.outputdiffs.OutputDataDiffs( candidate_channel_only_sim_restart.output, candidate_channel_only_sim_expected.output, exclude_vars=EXCLUDE_VARS, xrcmp_n_cores=xrcmp_n_cores) # Assert all diff values are 0 and print diff stats if not has_diffs = any(value != 0 for value in diffs.diff_counts.values()) if has_diffs: print_diffs(diffs) assert has_diffs is False, \ 'Outputs for candidate run do not match outputs from candidate restart run'
def test_ncores_candidate_channel_only(output_dir, ncores, exe_cmd, xrcmp_n_cores): candidate_channel_only_sim_file = \ output_dir / 'channel_only_candidate_run' / 'WrfHydroSim.pkl' candidate_channel_only_collected_file = \ output_dir / 'channel_only_candidate_run' / 'WrfHydroSim_collected.pkl' if candidate_channel_only_collected_file.is_file() is False: pytest.skip( 'candidate_channel_only collected run object not found, skipping test.' ) print( "\nQuestion: The candidate_channel-only output files from an ncores runmatch those " "from an ncores-1 run?\n", end='') print('\n') candidate_channel_only_sim_expected = pickle.load( candidate_channel_only_collected_file.open("rb")) run_dir = output_dir / 'channel_only_candidate_ncores' if not run_dir.exists(): run_dir.mkdir(parents=True) os.chdir(str(run_dir)) candidate_channel_only_sim = \ pickle.load(candidate_channel_only_sim_file.open("rb")) candidate_channel_only_sim_ncores = copy.deepcopy( candidate_channel_only_sim) candidate_channel_only_sim_ncores. \ base_hydro_namelist['hydro_nlist']['output_channelbucket_influx'] = 2 old_job = candidate_channel_only_sim.jobs[0] new_job = wrfhydropy.Job(job_id='ncores_candidate', model_start_time=old_job._model_start_time, model_end_time=old_job._model_end_time, exe_cmd=old_job._exe_cmd, restart_freq_hr=6, output_freq_hr=1) # Remove old job and add new job candidate_channel_only_sim_ncores.jobs.pop(0) candidate_channel_only_sim_ncores.add(new_job) # Edit the sim object number of cores if candidate_channel_only_sim_ncores.scheduler is not None: candidate_channel_only_sim_ncores.scheduler.nproc = \ candidate_channel_only_sim_ncores.scheduler.nproc - 1 else: candidate_channel_only_sim_ncores.jobs[ 0]._exe_cmd = exe_cmd.format(str(int(ncores) - 1)) # Recompose into new directory and run # catch warnings related to missing start and end job times with warnings.catch_warnings(): warnings.simplefilter("ignore") candidate_channel_only_sim_ncores.compose(force=True) print('\nwaiting for job to complete...', end='') candidate_channel_only_sim_ncores.run() wait_job(candidate_channel_only_sim_ncores) candidate_channel_only_sim_ncores.collect() candidate_channel_only_sim_ncores.pickle( run_dir.joinpath('WrfHydroSim_collected.pkl')) else: print('Candidate channel-only n_cores run dir exists, ' 'skipping n_cores candidate channel-only run...') candidate_channel_only_sim_ncores = pickle.load( open(run_dir.joinpath('WrfHydroSim_collected.pkl'), 'rb')) for job in candidate_channel_only_sim_ncores.jobs: assert job.exit_status == 0, \ "Candidate channel-only ncores run exited with non-zero status" # Check outputs with warnings.catch_warnings(): warnings.simplefilter("ignore") diffs = wrfhydropy.outputdiffs.OutputDataDiffs( candidate_channel_only_sim_ncores.output, candidate_channel_only_sim_expected.output, exclude_vars=EXCLUDE_VARS, xrcmp_n_cores=xrcmp_n_cores) # Assert all diff values are 0 and print diff stats if not has_diffs = any(value != 0 for value in diffs.diff_counts.values()) if has_diffs: print_diffs(diffs) assert has_diffs is False, \ 'Outputs for candidate_channel_only run with ncores do not match outputs with ncores-1'
def test_perfrestart_candidate(output_dir, xrcmp_n_cores): print("\nQuestion: The candidate outputs from a restart run match the outputs from standard " "run?\n", end='') print('\n') candidate_sim_file = output_dir / 'run_candidate' / 'WrfHydroSim.pkl' candidate_collected_file = output_dir / 'run_candidate' / 'WrfHydroSim_collected.pkl' if candidate_collected_file.is_file() is False: pytest.skip('Candidate run object not found, skipping test.') # Load initial run model object and copy candidate_sim = pickle.load(candidate_sim_file.open(mode="rb")) candidate_sim_expected = pickle.load(candidate_collected_file.open(mode="rb")) # Set run directory run_dir = output_dir.joinpath('restart_candidate') if not run_dir.exists(): candidate_sim_restart = copy.deepcopy(candidate_sim) run_dir.mkdir(parents=True) os.chdir(str(run_dir)) # Get a new start time halfway along the run, make sure the restart frequency accomodates restart_job = candidate_sim_restart.jobs[0] duration = restart_job.model_end_time - restart_job.model_start_time delay_restart_hr = int((duration.total_seconds() / 3600)/2) # Want matching restart frequencies for this test... assert \ candidate_sim_restart.jobs[0].restart_freq_hr_hydro == \ candidate_sim_restart.jobs[0].restart_freq_hr_hrldas, \ "Hydro and HRLDAS components do not have the same restart frequencies." assert delay_restart_hr % candidate_sim_restart.jobs[0].restart_freq_hr_hydro == 0, \ "The restart delay is not a multiple of the hydro restart frequency." restart_job.model_start_time = \ restart_job.model_start_time + dt.timedelta(hours=delay_restart_hr) # Get restart files from previous run and symlink into restart sim dir # (Remember that we are in the run/sim dir) # Hydro: Use actual time listed in meta data, not filename or positional list index # JLM: seems like these loops can be replaced with a pathlib.Path.glob(), the loop is confusing. for restart_file in candidate_sim_expected.output.restart_hydro: restart_time = restart_file.open().Restart_Time restart_time = pd.to_datetime(restart_time, format='%Y-%m-%d_%H:%M:%S') if restart_time == restart_job.model_start_time: candidate_hydro_restart_file = pathlib.Path(restart_file.name) candidate_hydro_restart_file.symlink_to(restart_file) key1 = 'hydro_nlist' key2 = 'restart_file' restart_job._hydro_namelist[key1][key2] = str(candidate_hydro_restart_file) # LSM: Use actual time listed in meta data, not filename or positional list index for restart_file in candidate_sim_expected.output.restart_lsm: restart_time = restart_file.open().Times[0] restart_time = restart_time.astype(str).item(0) restart_time = pd.to_datetime(restart_time, format='%Y-%m-%d_%H:%M:%S') if restart_time == restart_job.model_start_time: candidate_lsm_restart_file = pathlib.Path(restart_file.name) candidate_lsm_restart_file.symlink_to(restart_file) key1 = 'noahlsm_offline' key2 = 'restart_filename_requested' restart_job._hrldas_namelist[key1][key2] = str(candidate_lsm_restart_file) # Nudging: Use actual time listed in meta data, not filename or positional list index if candidate_sim_expected.output.restart_nudging is not None: for restart_file in candidate_sim_expected.output.restart_nudging: restart_time = restart_file.open().modelTimeAtOutput restart_time = pd.to_datetime(restart_time, format='%Y-%m-%d_%H:%M:%S') if restart_time == restart_job.model_start_time: candidate_nudging_restart_file = pathlib.Path(restart_file.name) candidate_nudging_restart_file.symlink_to(restart_file) key1 = 'nudging_nlist' key2 = 'nudginglastobsfile' restart_job._hydro_namelist[key1][key2] = str(candidate_nudging_restart_file) # Compose and run # catch warnings related to missing start and end job times with warnings.catch_warnings(): warnings.simplefilter("ignore") candidate_sim_restart.compose(force=True) print('\nwaiting for job to complete...', end='') candidate_sim_restart.run() # Wait to collect until job has finished. All test runs are performed on a single job with wait_job(candidate_sim_restart) candidate_sim_restart.collect() candidate_sim_restart.pickle(run_dir.joinpath('WrfHydroSim_collected.pkl')) # Check job run statuses for job in candidate_sim_restart.jobs: assert job.exit_status == 0, \ "Candidate restart run exited with non-zero status" else: print('Candidate channel-only perfect restart run dir exists, ' 'skipping perfect restart candidate channel-only run...') candidate_sim_restart = pickle.load( open(run_dir.joinpath('WrfHydroSim_collected.pkl'), 'rb')) # Check outputs with warnings.catch_warnings(): warnings.simplefilter("ignore") diffs = wrfhydropy.outputdiffs.OutputDataDiffs( candidate_sim_restart.output, candidate_sim_expected.output, exclude_vars=EXCLUDE_VARS, xrcmp_n_cores=xrcmp_n_cores ) # Assert all diff values are 0 and print diff stats if not has_diffs = any(value != 0 for value in diffs.diff_counts.values()) if has_diffs: print_diffs(diffs) assert has_diffs is False, \ 'Outputs for candidate run do not match outputs from candidate restart run'
def test_ncores_candidate_channel_only(output_dir): candidate_channel_only_sim_file = \ output_dir / 'channel_only_candidate_run' / 'WrfHydroSim.pkl' candidate_channel_only_collected_file = \ output_dir / 'channel_only_candidate_run' / 'WrfHydroSim_collected.pkl' if candidate_channel_only_collected_file.is_file() is False: pytest.skip( 'candidate_channel_only collected run object not found, skipping test.' ) print( "\nQuestion: The candidate_channel-only output files from an ncores runmatch those " "from an ncores-1 run?\n", end='') print('\n') candidate_channel_only_sim = \ pickle.load(candidate_channel_only_sim_file.open("rb")) candidate_channel_only_sim_expected = \ pickle.load(candidate_channel_only_collected_file.open("rb")) candidate_channel_only_sim_ncores = copy.deepcopy( candidate_channel_only_sim) candidate_channel_only_sim_ncores. \ base_hydro_namelist['hydro_nlist']['output_channelbucket_influx'] = 2 run_dir = output_dir / 'channel_only_candidate_ncores' run_dir.mkdir(parents=True) os.chdir(str(run_dir)) old_job = candidate_channel_only_sim.jobs[0] new_job = wrfhydropy.Job(job_id='ncores_candidate', exe_cmd=old_job._exe_cmd, restart_freq_hr=6, output_freq_hr=1) # Remove old job and add new job candidate_channel_only_sim_ncores.jobs.pop(0) candidate_channel_only_sim_ncores.add(new_job) # Edit the sim object number of cores if candidate_channel_only_sim_ncores.scheduler is not None: candidate_channel_only_sim_ncores.scheduler.nproc = \ candidate_channel_only_sim_ncores.scheduler.nproc - 1 else: orig_exe_cmd = candidate_channel_only_sim_ncores.jobs[0]._exe_cmd orig_exe_cmd = orig_exe_cmd.replace('-np 2', '-np 1') # Recompose into new directory and run # catch warnings related to missing start and end job times with warnings.catch_warnings(): warnings.simplefilter("ignore") candidate_channel_only_sim_ncores.compose(force=True) print('\nwaiting for job to complete...', end='') candidate_channel_only_sim_ncores.run() # Wait to collect until job has finished. All test runs are performed on a single job with # job_id='test_job' wait_job(candidate_channel_only_sim_ncores) candidate_channel_only_sim_ncores.collect() candidate_channel_only_sim_ncores.pickle( run_dir.joinpath('WrfHydroSim_collected.pkl')) # Check outputs with warnings.catch_warnings(): warnings.simplefilter("ignore") diffs = wrfhydropy.outputdiffs.OutputDataDiffs( candidate_channel_only_sim_ncores.output, candidate_channel_only_sim_expected.output, exclude_vars=EXCLUDE_VARS) # Assert all diff values are 0 and print diff stats if not has_diffs = any(value != 0 for value in diffs.diff_counts.values()) if has_diffs: print_diffs(diffs) assert has_diffs is False, \ 'Outputs for candidate_channel_only run with ncores do not match outputs with ncores-1'