def test_ens_parallel_run(simulation_compiled, job, scheduler, tmpdir, capfd):
sim = simulation_compiled
ens = EnsembleSimulation()
ens.add(job)
ens.add([sim, sim])
# Serial test
ens_serial = copy.deepcopy(ens)
ens_dir = pathlib.Path(tmpdir).joinpath('ens_serial_run')
os.chdir(tmpdir)
os.mkdir(str(ens_dir))
os.chdir(str(ens_dir))
ens_serial.compose(rm_members_from_memory=False)
serial_run_success = ens_serial.run()
assert serial_run_success == 0, \
"Some serial ensemble members did not run successfully."
assert get_ens_dotfile_end_datetime(ens_dir) == datetime.datetime(
2017, 1, 4, 0, 0)
assert ens_dir.joinpath("WrfHydroEns.pkl").exists()
# Parallel test
ens_parallel = copy.deepcopy(ens)
ens_dir = pathlib.Path(tmpdir).joinpath('ens_parallel_run')
os.chdir(tmpdir)
os.mkdir(str(ens_dir))
os.chdir(str(ens_dir))
ens_parallel.compose()
ens_run_success = ens_parallel.run(n_concurrent=2)
assert ens_run_success == 0, \
"Some parallel ensemble members did not run successfully."
assert get_ens_dotfile_end_datetime(ens_dir) == datetime.datetime(
2017, 1, 4, 0, 0)
assert ens_dir.joinpath("WrfHydroEns.pkl").exists()
# Parallel test with ensemble in memory
ens_parallel = copy.deepcopy(ens)
ens_dir = pathlib.Path(tmpdir).joinpath('ens_parallel_run_ens_in_memory')
os.chdir(tmpdir)
os.mkdir(str(ens_dir))
os.chdir(str(ens_dir))
ens_parallel.compose(rm_members_from_memory=False)
ens_run_mem_success = ens_parallel.run(n_concurrent=2)
assert ens_run_mem_success == 0, \
"Some parallel ensemble members in memory did not run successfully."
assert get_ens_dotfile_end_datetime(ens_dir) == datetime.datetime(
2017, 1, 4, 0, 0)
assert ens_dir.joinpath("WrfHydroEns.pkl").exists()
def get_ensemble_time(run_dir: str, with_previous: int = 0):
current_time = get_ens_dotfile_end_datetime(run_dir)
if with_previous != 0:
previous_time = current_time - datetime.timedelta(hours=with_previous)
return previous_time, current_time
else:
return current_time
parser = argparse.ArgumentParser(
description='Get the time of a HydroDartRun Ensemble')
parser.add_argument(
'--run_dir',
required=False,
metavar='/abs/path/to/run/directory',
help=
'Path to the experiment directory. (Default is director where this script or a '
+ 'symlink (not resolved) to it lives).',
default=os.path.dirname(os.path.abspath(__file__)))
args = parser.parse_args()
run_dir = pathlib.PosixPath(args.run_dir)
valid_time = get_ens_dotfile_end_datetime(run_dir)
# Read the YAML config file.
experiment_dir = run_dir / 'experiment_dir'
config_file = sorted(experiment_dir.glob("*.original.*.yaml"))[0]
with open(config_file) as ff:
config = yaml.safe_load(ff)
timedelta = datetime.timedelta
config_run_time = config['run_experiment']['time']
assim_window_hr = int(
config_run_time['assim_window']['assim_window_size_hours'])
assim_half_window_sec = timedelta(seconds=(60 * 60 / 2) * assim_window_hr)
start_window = valid_time - assim_half_window_sec + timedelta(seconds=1)
end_window = valid_time + assim_half_window_sec
def advance_ensemble(
self,
model_start_time: datetime=None,
model_end_time: datetime=None,
job_entry_cmd: str=None,
job_exit_cmd: str=None,
afterok: str=None,
afterany: str=None,
hold: bool=False
):
# Setup job and scheduler.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Create a default job
the_job = Job(
nproc=self.config['run_experiment']['wrf_hydro_ens_advance']['nproc'],
entry_cmd=job_entry_cmd,
exit_cmd=job_exit_cmd
)
# TODO(JLM): add the entry and exit to the job
# Create a dummy scheduler
# This could be done with **{config['run_experiment']['wrf_hydro_ens_advance']['job_name']}
if self.config['run_experiment']['wrf_hydro_ens_advance']['with_filter']:
the_sched = None
ppn_max = self.config['run_experiment']['dart']['scheduler']['ppn_max']
nproc_model = \
self.config['run_experiment']['wrf_hydro_ens_advance']['nproc']
# Leave a processor for the OS.
n_mem_simul = math.floor((int(ppn_max) - 1) / int(nproc_model))
print("n_mem_simul: ", n_mem_simul)
else:
the_sched = Scheduler(
job_name=self.config['run_experiment']['wrf_hydro_ens_advance']['job_name'],
account=self.config['run_experiment']['wrf_hydro_ens_advance']['account'],
nproc=self.config['run_experiment']['wrf_hydro_ens_advance']['nproc'],
nnodes=self.config['run_experiment']['wrf_hydro_ens_advance']['nnodes'],
walltime=self.config['run_experiment']['wrf_hydro_ens_advance']['walltime'],
queue=self.config['run_experiment']['wrf_hydro_ens_advance']['queue'],
email_who=self.config['run_experiment']['wrf_hydro_ens_advance']['email_who'],
email_when=self.config['run_experiment']['wrf_hydro_ens_advance']['email_when'],
afterok=afterok
)
# TODO(JLM): add afterany
the_job.scheduler = the_sched
ens_sim = pickle.load(open(self.wrf_hydro_ens_sim_pkl, 'rb'))
# TODO (JLM): this is in place of the "sweeper" job or part of the submit script.
ens_sim.collect_ensemble_runs(n_mem_simultaneous=n_mem_simul)
if model_start_time is None:
model_start_time = get_ens_dotfile_end_datetime(self.run_dir)
if model_end_time is None:
model_end_time = \
model_start_time + \
timedelta(hours=self.config['run_experiment']['time']['advance_model_hours'])
the_job.model_start_time = model_start_time
the_job.model_end_time = model_end_time
ens_sim.add(the_job)
ens_sim.run(
hold=hold,
n_mem_simultaneous=n_mem_simul
)
self.pickle()
