示例#1
0
def test_cycle_addscheduler(
    simulation,
    scheduler,
    init_times
):
    cy1 = CycleSimulation(init_times=init_times, restart_dirs=['.'] * len(init_times))
    cy1.add(scheduler)
    assert deepdiff.DeepDiff(cy1._scheduler, scheduler) == {}

    sched2 = copy.deepcopy(scheduler)
    sched2.nnodes = 99
    cy1.add(sched2)
    assert deepdiff.DeepDiff(cy1._scheduler, sched2) == {}
示例#2
0
def test_cycle_addsimulation(simulation, job_restart, scheduler,
                             simulation_compiled, init_times):
    sim = simulation
    cy1 = CycleSimulation(init_times=init_times,
                          restart_dirs=['.'] * len(init_times))

    # This sim does not have the required pre-compiled model
    with pytest.raises(Exception) as e_info:
        cy1.add(sim)

    sim_compiled = simulation_compiled

    # cant add a list, even if pre-compiled
    with pytest.raises(Exception) as e_info:
        cy1.add([sim_compiled])

    cy1.add(sim_compiled)

    # add a sim with job and make sure it is deleted.
    sim_compiled.add(job_restart)
    sim_compiled.add(scheduler)
    cy2 = CycleSimulation(init_times=init_times,
                          restart_dirs=['.'] * len(init_times))
    cy2.add(sim_compiled)

    assert all([len(cc.jobs) == 0 for cc in cy2.casts])
    assert all([cc.scheduler is None for cc in cy2.casts])
示例#3
0
def test_cycle_addsimulation(
    simulation,
    job_restart,
    scheduler,
    simulation_compiled,
    init_times,
    restart_dirs
):
    sim = 'not a simulation object'
    cy1 = CycleSimulation(
        init_times=init_times,
        restart_dirs=restart_dirs
    )
    with pytest.raises(Exception) as e_info:
        cy1.add(sim)
    assert str(e_info.value) == 'Object is not of a type expected for a CycleSimulation.'

    sim = simulation
    cy1 = CycleSimulation(
        init_times=init_times,
        restart_dirs=restart_dirs
    )
    # This sim does not have the required pre-compiled model
    # with pytest.raises(Exception) as e_info:
    cy1.add(sim)
    # assert str(e_info.value) == \
    #    'Only Simulations with compiled model objects can be added to an ensemble simulation.'

    sim_compiled = simulation_compiled

    # cant add a list, even if pre-compiled
    with pytest.raises(Exception) as e_info:
        cy1.add([sim_compiled])
    assert str(e_info.value) == 'Object is not of a type expected for a CycleSimulation.'

    cy1.add(sim_compiled)
    assert isinstance(cy1._simulation, Simulation)

    # add a sim with job and make sure it is deleted.
    sim_compiled.add(job_restart)
    sim_compiled.add(scheduler)
    cy2 = CycleSimulation(
        init_times=init_times,
        restart_dirs=restart_dirs
    )
    cy2.add(sim_compiled)
    assert cy2._simulation.jobs == []
    assert cy2._simulation.scheduler is None
示例#4
0
def test_cycle_self_dependent_run(simulation_compiled, job_restart, scheduler,
                                  tmpdir, capfd, init_times):

    sim = simulation_compiled
    cy = CycleSimulation(init_times=init_times, restart_dirs=['.', '-3', '-3'])
    cy.add(job_restart)
    cy.add(sim)

    # Serial test
    cy_serial = copy.deepcopy(cy)
    cy_dir = pathlib.Path(tmpdir).joinpath('cycle_serial_run')
    os.chdir(tmpdir)
    os.mkdir(str(cy_dir))
    os.chdir(str(cy_dir))
    cy_serial.compose(rm_casts_from_memory=False)
    serial_run_success = cy_serial.run()
    assert serial_run_success == 0, \
        "Some serial cycle casts did not run successfully."
示例#5
0
def test_cycle_addsimulation_translate(
    restart_dirs,
    expected,
    init_times,
    simulation_compiled,
    job_restart,
    tmpdir
):
    os.chdir(tmpdir)
    sim = simulation
    cy1 = CycleSimulation(
        init_times=init_times,
        restart_dirs=restart_dirs
    )

    sim_compiled = simulation_compiled
    cy1.add(sim_compiled)
    cy1.add(job_restart)

    # translation happens on compose.
    try:
        os.mkdir(tmpdir / 'compose')
        os.chdir(tmpdir / 'compose')
        # This may be use in some of the tests
        pathlib.Path('../dummy_extant_dir').touch()
        cy1.compose(rm_casts_from_memory=False)
        lsm_keys = ['noahlsm_offline', 'restart_filename_requested']
        hyd_keys = ['hydro_nlist', 'restart_file']
        result = {
            'lsm': [cast.base_hrldas_namelist[lsm_keys[0]][lsm_keys[1]] for cast in cy1.casts],
            'hyd': [cast.base_hydro_namelist[hyd_keys[0]][hyd_keys[1]] for cast in cy1.casts]
        }
    except Exception as e:
        result = [str(e)]  # Cludgy but it works

    if isinstance(expected, dict):
        for key, v_list in expected.items():
            for ii, path in enumerate(v_list):
                expected[key][ii] = sub_tmpdir(path, tmpdir)

    assert result == expected
示例#6
0
def test_cycle_addensemble(ensemble, job_restart, scheduler, init_times,
                           restart_dirs, restart_dirs_ensemble):
    # The ensemble necessarily has a compiled model (unlike a Simulation).
    # That is a separate test.

    cy1 = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs)
    with pytest.raises(Exception) as e_info:
        cy1.add(ensemble)
    assert str(e_info.value) == \
        'An ensemble cycle simulation requires the restart_dirs to be a list of lists.'

    cy1 = CycleSimulation(init_times=init_times,
                          restart_dirs=restart_dirs_ensemble)
    cy1.add(ensemble)
    assert isinstance(cy1._ensemble, EnsembleSimulation)

    # add an ens with a job and make sure it is deleted.
    ens = ensemble
    ens.add(job_restart)
    ens.add(scheduler)
    cy2 = CycleSimulation(init_times=init_times,
                          restart_dirs=restart_dirs_ensemble)
    cy2.add(ensemble)
    assert cy2._ensemble.jobs == []
    assert cy2._ensemble.scheduler is None
示例#7
0
def test_cycle_ensemble_parallel_compose(ensemble, job_restart, scheduler,
                                         tmpdir, init_times,
                                         restart_dirs_ensemble):
    ens = ensemble
    cy = CycleSimulation(init_times=init_times,
                         restart_dirs=restart_dirs_ensemble,
                         ncores=2)
    cy.add(job_restart)
    # Adding the scheduler ruins the run in CI.
    # cy.add(scheduler)

    # Make a copy where we keep the casts in memory for checking.
    cy_check_casts = copy.deepcopy(cy)
    cy_ens_compose = copy.deepcopy(cy)

    with pytest.raises(Exception) as e_info:
        cy.compose()

    cy_ens_compose.add(ens)
    compose_dir = pathlib.Path(tmpdir).joinpath('cycle_ensemble_compose')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    pathlib.Path('../dummy_extant_dir').touch()
    cy_ens_compose.compose()

    cy_run_success = cy_ens_compose.run()
    assert cy_run_success == 0
    cy.pickle(
        str(
            pathlib.Path(tmpdir) /
            'cycle_ensemble_compose/WrfHydroCycleEns.pkl'))
    # Is this pickle used?

    # The cycle-in-memory version for checking the casts.
    cy_check_casts.add(ens)
    compose_dir = pathlib.Path(tmpdir).joinpath('cycle_compose_check_casts')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    pathlib.Path('../dummy_extant_dir').touch()
    cy_check_casts.compose(rm_casts_from_memory=False,
                           rm_members_from_memory=False)

    # The job gets heavily modified on compose.
    answer = {
        '_entry_cmd': 'bogus entry cmd',
        '_exe_cmd': './wrf_hydro.exe',
        '_exit_cmd': 'bogus exit cmd',
        '_hrldas_namelist': {
            'noahlsm_offline': {
                'btr_option': 1,
                'canopy_stomatal_resistance_option': 1,
                'hrldas_setup_file': './NWM/DOMAIN/wrfinput_d01.nc',
                'indir': './FORCING',
                'output_timestep': 86400,
                'restart_filename_requested':
                './NWM/RESTART/RESTART.2011082600_DOMAIN1',
                'restart_frequency_hours': 24
            },
            'wrf_hydro_offline': {
                'forc_typ': 1
            }
        },
        '_hrldas_times': {
            'noahlsm_offline': {
                'khour': 282480,
                'restart_frequency_hours': 24,
                'output_timestep': 86400,
                'restart_filename_requested':
                'NWM/RESTART/RESTART.2013101300_DOMAIN1',
                'start_day': 12,
                'start_hour': 00,
                'start_min': 00,
                'start_month': 12,
                'start_year': 2012
            }
        },
        '_hydro_namelist': {
            'hydro_nlist': {
                'aggfactrt': 4,
                'channel_option': 2,
                'chanobs_domain': 0,
                'chanrtswcrt': 1,
                'chrtout_domain': 1,
                'geo_static_flnm': './NWM/DOMAIN/geo_em.d01.nc',
                'restart_file':
                './NWM/RESTART/HYDRO_RST.2011-08-26_00:00_DOMAIN1',
                'udmp_opt': 1,
                'rst_dt': 1440,
                'out_dt': 1440
            },
            'nudging_nlist': {
                'maxagepairsbiaspersist':
                3,
                'minnumpairsbiaspersist':
                1,
                'nudginglastobsfile':
                './NWM/RESTART/nudgingLastObs.2011-08-26_00:00:00.nc'
            }
        },
        '_hydro_times': {
            'hydro_nlist': {
                'out_dt': 1440,
                'rst_dt': 1440,
                'restart_file':
                'NWM/RESTART/HYDRO_RST.2013-10-13_00:00_DOMAIN1'
            },
            'nudging_nlist': {
                'nudginglastobsfile':
                'NWM/RESTART/nudgingLastObs.2013-10-13_00:00:00.nc'
            }
        },
        '_job_end_time': None,
        '_job_start_time': None,
        '_job_submission_time': None,
        '_model_end_time': pandas.Timestamp('2045-03-04 00:00:00'),
        '_model_start_time': pandas.Timestamp('2012-12-12 00:00:00'),
        'exit_status': None,
        'job_id': 'test_job_1',
        'restart_freq_hr_hydro': None,
        'restart_freq_hr_hrldas': None,
        'output_freq_hr_hydro': None,
        'output_freq_hr_hrldas': None,
        'restart': True,
        'restart_dir': None,
        '_restart_dir_hydro': None,
        '_restart_dir_hrldas': None,
        'restart_file_time': '2013-10-13',
        '_restart_file_time_hydro': pandas.Timestamp('2013-10-13 00:00:00'),
        '_restart_file_time_hrldas': pandas.Timestamp('2013-10-13 00:00:00')
    }

    # These answer patches respond to the variety of things in restart_dirs_ensemble
    dum_ext = str(tmpdir) + '/dummy_extant_dir/'
    answer_patches = {
        'cast_index': [0, 1, 2],
        'start_time_patch': [
            pandas.Timestamp('2012-12-12 00:00:00'),
            pandas.Timestamp('2012-12-15 00:00:00'),
            pandas.Timestamp('2012-12-18 00:00:00')
        ],
        'end_time_patch': [
            pandas.Timestamp('2045-03-04 00:00:00'),
            pandas.Timestamp('2045-03-07 00:00:00'),
            pandas.Timestamp('2045-03-10 00:00:00')
        ],

        # These "time patches" reveal the awkwardness of that construct.
        'lsm_times_patch': [
            'NWM/RESTART/RESTART.2013101300_DOMAIN1',
            '../../cast_2012121200/member_000/RESTART.2013101300_DOMAIN1',
            dum_ext + 'RESTART.2013101300_DOMAIN1'
        ],
        'hydro_times_patch': [
            'NWM/RESTART/HYDRO_RST.2013-10-13_00:00_DOMAIN1',
            '../../cast_2012121200/member_000/HYDRO_RST.2013-10-13_00:00_DOMAIN1',
            dum_ext + 'HYDRO_RST.2013-10-13_00:00_DOMAIN1'
        ],
        'ndg_times_patch': [
            'NWM/RESTART/nudgingLastObs.2013-10-13_00:00:00.nc',
            '../../cast_2012121200/member_000/nudgingLastObs.2013-10-13_00:00:00.nc',
            dum_ext + 'nudgingLastObs.2013-10-13_00:00:00.nc'
        ],

        # These namelist patches are consistent with the model times except in the
        # first "do nothing" case which leaves the start time != restart file time
        'lsm_nlst_patch': [
            './NWM/RESTART/RESTART.2011082600_DOMAIN1',
            '../../cast_2012121200/member_000/RESTART.2012121500_DOMAIN1',
            dum_ext + 'RESTART.2012121800_DOMAIN1'
        ],
        'hydro_nlst_patch': [
            './NWM/RESTART/HYDRO_RST.2011-08-26_00:00_DOMAIN1',
            '../../cast_2012121200/member_000/HYDRO_RST.2012-12-15_00:00_DOMAIN1',
            dum_ext + 'HYDRO_RST.2012-12-18_00:00_DOMAIN1'
        ],
        'ndg_nlst_patch': [
            './NWM/RESTART/nudgingLastObs.2011-08-26_00:00:00.nc',
            '../../cast_2012121200/member_000/nudgingLastObs.2012-12-15_00:00:00.nc',
            dum_ext + 'nudgingLastObs.2012-12-18_00:00:00.nc'
        ]
    }

    # Check a cycle where the compse retains the casts (otherwise nothing in memory).
    # This fails:
    # deepdiff.DeepDiff(answer, cy.casts[0].jobs[0].__dict__)
    # Instead, iterate on keys to "declass":
    # Just check the first ensemble cast.
    def sub_member(the_string, replace_num, find_num=0):
        replace = "member_{:03d}".format(replace_num)
        find = "member_{:03d}".format(find_num)
        return the_string.replace(find, replace)

    for ii in answer_patches['cast_index']:
        cc = cy_check_casts.casts[ii]

        for mm, member in enumerate(cc.members):

            answer['_model_start_time'] = answer_patches['start_time_patch'][
                ii]
            answer['_model_end_time'] = answer_patches['end_time_patch'][ii]

            keys = ['noahlsm_offline', 'restart_filename_requested']
            answer['_hrldas_namelist'][keys[0]][keys[1]] = \
                sub_member(answer_patches['lsm_nlst_patch'][ii], mm)
            answer['_hrldas_times'][keys[0]][keys[1]] = \
                sub_member(answer_patches['lsm_times_patch'][ii], mm)

            keys = ['_hydro_namelist', 'hydro_nlist', 'restart_file']
            answer[keys[0]][keys[1]][keys[2]] = \
                sub_member(answer_patches['hydro_nlst_patch'][ii], mm)

            keys = ['_hydro_namelist', 'nudging_nlist', 'nudginglastobsfile']
            answer[keys[0]][keys[1]][keys[2]] = \
                sub_member(answer_patches['ndg_nlst_patch'][ii], mm)

            keys = ['_hydro_times', 'hydro_nlist', 'restart_file']
            answer[keys[0]][keys[1]][keys[2]] =\
                sub_member(answer_patches['hydro_times_patch'][ii], mm)

            keys = ['_hydro_times', 'nudging_nlist', 'nudginglastobsfile']
            answer[keys[0]][keys[1]][keys[2]] = \
                sub_member(answer_patches['ndg_times_patch'][ii], mm)

            # hrldas times
            fmt_keys = {
                '%Y': 'start_year',
                '%m': 'start_month',
                '%d': 'start_day',
                '%H': 'start_hour'
            }
            the_mutable = answer['_hrldas_times']['noahlsm_offline']
            for fmt, key in fmt_keys.items():
                the_mutable[key] = int(
                    answer['_model_start_time'].strftime(fmt))

            # Actually check
            for kk in member.jobs[0].__dict__.keys():
                assert member.jobs[0].__dict__[kk] == answer[kk]

    # Check the scheduler too
    # assert cy_check_casts.casts[0].scheduler.__dict__ == scheduler.__dict__

    # For the cycle where the compse removes the casts...
    # Check that the casts are all now simply pathlib objects
    assert all([type(mm) is str for mm in cy.casts])

    # the tmpdir gets nuked after the test... ?
    # Test the cast pickle size in terms of load speed.
    # Note that the deletion of the model, domain, and output objects are
    # done for the casts regardless of not removing the casts
    # from memory (currently).
    os.chdir(
        str(pathlib.Path(tmpdir) / 'cycle_ensemble_compose/cast_2012121200'))
    time_taken = timeit.timeit(
        setup='import pickle',
        stmt='pickle.load(open("WrfHydroEns.pkl","rb"))',
        number=10000)
    # If your system is busy, this could take longer... and spuriously fail the test.
    # Notes(JLM): coverage makes this slow
    assert time_taken < 1.5

    # Test the cycle pickle size in terms of load speed.
    os.chdir(str(pathlib.Path(tmpdir) / 'cycle_ensemble_compose/'))
    time_taken = timeit.timeit(
        setup='import pickle',
        stmt='pickle.load(open("WrfHydroCycleEns.pkl","rb"))',
        number=10000)
    # If your system is busy, this could take longer...
    # Notes(JLM): coveage makes this slow
    assert time_taken < 1.5
示例#8
0
def test_cycle_ensemble_compose(ensemble, job_restart, scheduler, tmpdir,
                                init_times, restart_dirs_ensemble):
    # These might be parametizable.

    # Length zero cycle compose.
    cy = CycleSimulation(init_times=[], restart_dirs=[], ncores=1)
    cy.add(job_restart)
    cy.add(ensemble)
    compose_dir = pathlib.Path(tmpdir).joinpath('cycle_sim0_compose')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    with pytest.raises(Exception) as e_info:
        cy.compose()
    assert str(e_info.value) == "There are no casts (init_times) to compose."

    # Inconsistent forcing dir length and ensemble length
    cy = CycleSimulation(init_times=init_times,
                         restart_dirs=restart_dirs_ensemble,
                         ncores=1,
                         forcing_dirs=[['.', '.', '../dummy_extant_dir'],
                                       ['.', -72, '../dummy_extant_dir'],
                                       ['.', -72, '../dummy_extant_dir']])
    cy.add(job_restart)
    with pytest.raises(Exception) as e_info:
        cy.add(ensemble)
    assert str(e_info.value) == \
        "Ensemble to add has inconsistent length with existing cycle forcing_dirs"

    # Inconsistent forcing dir length and ensemble length
    cy = CycleSimulation(init_times=init_times,
                         restart_dirs=[[rr[0]]
                                       for rr in restart_dirs_ensemble],
                         ncores=1,
                         forcing_dirs=[['.', '../dummy_extant_dir'],
                                       [-72, '../dummy_extant_dir'],
                                       [-72, '../dummy_extant_dir']])
    cy.add(job_restart)
    with pytest.raises(Exception) as e_info:
        cy.add(ensemble)
    assert str(e_info.value) == \
        "Ensemble to add has inconsistent length with existing cycle restart_dirs"

    # Valid force dir exercise.
    cy = CycleSimulation(init_times=init_times,
                         restart_dirs=restart_dirs_ensemble,
                         ncores=1,
                         forcing_dirs=[['.', '../dummy_extant_dir'],
                                       [-72, '../dummy_extant_dir'],
                                       [-72, '../dummy_extant_dir']])
    cy.add(job_restart)
    cy.add(ensemble)
    compose_dir = pathlib.Path(tmpdir).joinpath('cycle_forc_dir_compose')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    compose_dir.joinpath('../dummy_extant_dir').touch()
    cy.compose()

    # In valid force dir exercise.
    cy = CycleSimulation(init_times=init_times,
                         restart_dirs=restart_dirs_ensemble,
                         ncores=1,
                         forcing_dirs=[[72, '../dummy_extant_dir'],
                                       [72, '../dummy_extant_dir'],
                                       [72, '../dummy_extant_dir']])
    cy.add(job_restart)
    cy.add(ensemble)
    compose_dir = pathlib.Path(tmpdir).joinpath(
        'cycle_forc_dir_fail_1_compose')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    compose_dir.joinpath('../dummy_extant_dir').touch()
    with pytest.raises(Exception) as e_info:
        cy.compose()
    assert str(
        e_info.value
    ) == 'Only non-negative integers can be used to specify forcing_dirs'

    # In valid force dir exercise.
    cy = CycleSimulation(init_times=init_times,
                         restart_dirs=restart_dirs_ensemble,
                         ncores=1,
                         forcing_dirs=[['.', 'dummy_non-extant_dir'],
                                       ['.', 'dummy_non-extant_dir'],
                                       ['.', 'dummy_non-extant_dir']])
    cy.add(job_restart)
    cy.add(ensemble)
    compose_dir = pathlib.Path(tmpdir).joinpath(
        'cycle_forc_dir_fail_2_compose')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    compose_dir.joinpath('../dummy_extant_dir').touch()
    with pytest.raises(Exception) as e_info:
        cy.compose()
    assert str(
        e_info.value) == 'No such forcing directory: dummy_non-extant_dir'
示例#9
0
def test_cycle_parallel_compose(simulation_compiled, job_restart, scheduler,
                                tmpdir, init_times, restart_dirs):
    """ A more comprehensive test of the object composed."""
    # A compiled simulation passed. Successfull compose in parallel.
    cy = CycleSimulation(init_times=init_times,
                         restart_dirs=restart_dirs,
                         ncores=2)
    cy.add(job_restart)
    cy.add(simulation_compiled)

    # Make a copy where we keep the casts in memory for checking.
    cy_check_casts = copy.deepcopy(cy)

    compose_dir = pathlib.Path(tmpdir).joinpath('cycle_compose')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    cy.compose()

    cy_run_success = cy.run()
    assert cy_run_success == 0
    cy.pickle(str(pathlib.Path(tmpdir) / 'cycle_compose/WrfHydroCycleSim.pkl'))

    # The cycle-in-memory version for checking the casts.
    compose_dir = pathlib.Path(tmpdir).joinpath('cycle_compose_check_casts')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    cy_check_casts.compose(rm_casts_from_memory=False)

    # The job gets heavily modified on compose.
    answer = {
        '_entry_cmd': 'bogus entry cmd',
        '_exe_cmd': './wrf_hydro.exe',
        '_exit_cmd': 'bogus exit cmd',
        '_hrldas_namelist': {
            'noahlsm_offline': {
                'btr_option': 1,
                'canopy_stomatal_resistance_option': 1,
                'hrldas_setup_file': './NWM/DOMAIN/wrfinput_d01.nc',
                'indir': './FORCING',
                'output_timestep': 86400,
                'restart_filename_requested':
                './NWM/RESTART/RESTART.2011082600_DOMAIN1',
                'restart_frequency_hours': 24
            },
            'wrf_hydro_offline': {
                'forc_typ': 1
            }
        },
        '_hrldas_times': {
            'noahlsm_offline': {
                'khour': 282480,
                'restart_frequency_hours': 24,
                'output_timestep': 86400,
                'restart_filename_requested':
                'NWM/RESTART/RESTART.2013101300_DOMAIN1',
                'start_day': 12,
                'start_hour': 00,
                'start_min': 00,
                'start_month': 12,
                'start_year': 2012
            }
        },
        '_hydro_namelist': {
            'hydro_nlist': {
                'aggfactrt': 4,
                'channel_option': 2,
                'chanobs_domain': 0,
                'chanrtswcrt': 1,
                'chrtout_domain': 1,
                'geo_static_flnm': './NWM/DOMAIN/geo_em.d01.nc',
                'restart_file':
                './NWM/RESTART/HYDRO_RST.2011-08-26_00:00_DOMAIN1',
                'udmp_opt': 1,
                'rst_dt': 1440,
                'out_dt': 1440
            },
            'nudging_nlist': {
                'maxagepairsbiaspersist':
                3,
                'minnumpairsbiaspersist':
                1,
                'nudginglastobsfile':
                './NWM/RESTART/nudgingLastObs.2011-08-26_00:00:00.nc'
            }
        },
        '_hydro_times': {
            'hydro_nlist': {
                'out_dt': 1440,
                'rst_dt': 1440,
                'restart_file':
                'NWM/RESTART/HYDRO_RST.2013-10-13_00:00_DOMAIN1'
            },
            'nudging_nlist': {
                'nudginglastobsfile':
                'NWM/RESTART/nudgingLastObs.2013-10-13_00:00:00.nc'
            }
        },
        '_job_end_time': None,
        '_job_start_time': None,
        '_job_submission_time': None,
        '_model_end_time': pandas.Timestamp('2045-03-04 00:00:00'),
        '_model_start_time': pandas.Timestamp('2012-12-12 00:00:00'),
        'exit_status': None,
        'job_id': 'test_job_1',
        'restart_freq_hr_hydro': None,
        'restart_freq_hr_hrldas': None,
        'output_freq_hr_hydro': None,
        'output_freq_hr_hrldas': None,
        'restart': True,
        'restart_dir': None,
        '_restart_dir_hydro': None,
        '_restart_dir_hrldas': None,
        'restart_file_time': '2013-10-13',
        '_restart_file_time_hydro': pandas.Timestamp('2013-10-13 00:00:00'),
        '_restart_file_time_hrldas': pandas.Timestamp('2013-10-13 00:00:00')
    }

    # For the cycle where the compse retains the casts...

    # This fails:
    # deepdiff.DeepDiff(answer, cy.casts[0].jobs[0].__dict__)
    # Instead, iterate on keys to "declass":
    for kk in cy_check_casts.casts[0].jobs[0].__dict__.keys():
        assert cy_check_casts.casts[0].jobs[0].__dict__[kk] == answer[kk]
    # Check the scheduler too
    # assert cy_check_casts.casts[0].scheduler.__dict__ == scheduler.__dict__

    # For the cycle where the compse removes the casts...

    # Check that the casts are all now simply pathlib objects
    assert all([type(mm) is str for mm in cy.casts])

    # the tmpdir gets nuked after the test... ?
    # Test the cast pickle size in terms of load speed.
    # Note that the deletion of the model, domain, and output objects are
    # done for the casts regardless of not removing the casts
    # from memory (currently).
    os.chdir(str(pathlib.Path(tmpdir) / 'cycle_compose/cast_2012121200'))
    time_taken = timeit.timeit(
        setup='import pickle',
        stmt='pickle.load(open("WrfHydroSim.pkl","rb"))',
        number=10000)
    # If your system is busy, this could take longer... and spuriously fail the test.
    # Notes(JLM): coverage is the limiting factor
    assert time_taken < 1.5

    # Test the cycle pickle size in terms of load speed.
    os.chdir(str(pathlib.Path(tmpdir) / 'cycle_compose/'))
    time_taken = timeit.timeit(
        setup='import pickle',
        stmt='pickle.load(open("WrfHydroCycleSim.pkl","rb"))',
        number=10000)
    # If your system is busy, this could take longer...
    # Notes(JLM): coverage is the limiting factor.
    assert time_taken < 1.2
示例#10
0
def test_cycle_compose(simulation, simulation_compiled, job_restart, scheduler,
                       tmpdir, init_times, restart_dirs):
    # These might be parametizable.

    # Compose without adding a simulation.
    cy = CycleSimulation(init_times=init_times,
                         restart_dirs=restart_dirs,
                         ncores=2)
    cy.add(job_restart)
    # Adding the scheduler ruins the run in CI.
    # cy.add(scheduler)
    with pytest.raises(Exception) as e_info:
        cy.compose()
    assert str(e_info.value) == \
        'Unable to compose, current working directory is not empty. \n' + \
        'Change working directory to an empty directory with os.chdir()'

    compose_dir = pathlib.Path(tmpdir).joinpath('cycle_no_sim_compose')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    with pytest.raises(Exception) as e_info:
        cy.compose()
    assert str(e_info.value
               ) == 'The cycle does not contain a _simulation or an _ensemble.'

    # Length zero cycle compose.
    cy = CycleSimulation(init_times=[], restart_dirs=[], ncores=1)
    cy.add(job_restart)
    cy.add(simulation_compiled)
    compose_dir = pathlib.Path(tmpdir).joinpath('cycle_sim0_compose')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    with pytest.raises(Exception) as e_info:
        cy.compose()
    assert str(e_info.value) == "There are no casts (init_times) to compose."

    # This simultion is not compiled. It compiles and composes successfully.
    cy = CycleSimulation(init_times=init_times,
                         restart_dirs=restart_dirs,
                         ncores=1)
    cy.add(job_restart)
    cy.add(simulation)
    compose_dir = pathlib.Path(tmpdir).joinpath('cycle_uncompiled_compose')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    cy.compose()

    # Valid force dir exercise.
    cy = CycleSimulation(init_times=init_times,
                         restart_dirs=restart_dirs,
                         ncores=1,
                         forcing_dirs=['.', -72, '../dummy_extant_dir'])
    cy.add(job_restart)
    cy.add(simulation)
    compose_dir = pathlib.Path(tmpdir).joinpath('cycle_forc_dir_compose')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    compose_dir.joinpath('../dummy_extant_dir').touch()
    cy.compose()

    # In valid force dir exercise.
    cy = CycleSimulation(init_times=init_times,
                         restart_dirs=restart_dirs,
                         ncores=1,
                         forcing_dirs=['.', 72, '../dummy_extant_dir'])
    cy.add(job_restart)
    cy.add(simulation)
    compose_dir = pathlib.Path(tmpdir).joinpath(
        'cycle_forc_dir_fail_2_compose')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    with pytest.raises(Exception) as e_info:
        cy.compose()
    assert str(
        e_info.value
    ) == 'Only non-negative integers can be used to specify forcing_dirs'

    # In valid force dir exercise.
    cy = CycleSimulation(init_times=init_times,
                         restart_dirs=restart_dirs,
                         ncores=1,
                         forcing_dirs=['.', 'dummy_non-extant_dir', -72])
    cy.add(job_restart)
    cy.add(simulation)
    compose_dir = pathlib.Path(tmpdir).joinpath('cycle_forc_dir_fail_compose')
    os.mkdir(str(compose_dir))
    os.chdir(str(compose_dir))
    with pytest.raises(Exception) as e_info:
        cy.compose()
    assert str(
        e_info.value) == 'No such forcing directory: dummy_non-extant_dir'
示例#11
0
def test_cycle_length(simulation_compiled, init_times, restart_dirs):
    sim = simulation_compiled
    cy1 = CycleSimulation(init_times=init_times, restart_dirs=restart_dirs)
    cy1.add(sim)
    assert len(cy1) == len(init_times)
示例#12
0
def test_cycle_run_parallel_teams(simulation_compiled, job_restart, scheduler,
                                  tmpdir, capfd, init_times, restart_dirs):
    sim = simulation_compiled
    tmp = pathlib.Path(tmpdir)
    forcing_dirs = [
        tmp / letter
        for letter in list(string.ascii_lowercase)[0:len(init_times)]
    ]
    for dir in forcing_dirs:
        dir.mkdir()
    cy = CycleSimulation(init_times=init_times,
                         restart_dirs=restart_dirs,
                         forcing_dirs=forcing_dirs)
    cy.add(sim)
    job_restart._entry_cmd = 'echo mpirun entry_cmd > entry_cmd.output'
    job_restart._exit_cmd = 'echo mpirun exit_cmd > exit_cmd.output'
    cy.add(job_restart)

    # Parallel teams test - test casts both in memory and not
    cy_teams_to_test = {
        'cy_teams_not_memory': copy.deepcopy(cy),
        'cy_teams_memory': copy.deepcopy(cy)
    }

    for key, cy_teams in cy_teams_to_test.items():

        cy_dir = pathlib.Path(tmpdir).joinpath(key)
        os.chdir(tmpdir)
        os.mkdir(str(cy_dir))
        os.chdir(str(cy_dir))

        cy_teams_cp = copy.deepcopy(cy_teams)
        if key == 'cy_teams_not_memory':
            cy_teams_cp.compose()
        else:
            cy_teams_cp.compose(rm_casts_from_memory=False)

        with pytest.raises(Exception) as e_info:
            cy_teams_run_fail_xnode = cy_teams_cp.run(
                teams=True,
                teams_exe_cmd=
                ' ./wrf_hydro.exe mpirun --host {nodelist} -np {nproc} {cmd}',
                teams_exe_cmd_nproc=3,
                teams_node_file=node_file)
            the_error = (
                "ValueError('teams_exe_cmd_nproc > number of cores/node: "
                "teams does not currently function in this capacity.',)")
            assert repr(e_info._excinfo[1]
                        ) == the_error, 'Teams is not failing on xnode request'

        os.chdir(tmpdir)
        shutil.rmtree(str(cy_dir))
        os.mkdir(str(cy_dir))
        os.chdir(str(cy_dir))
        if key == 'cy_teams_not_memory':
            cy_teams.compose()
        else:
            cy_teams.compose(rm_casts_from_memory=False)

        cy_teams_run_success = cy_teams.run(
            teams=True,
            teams_exe_cmd=
            ' ./wrf_hydro.exe mpirun --host {nodelist} -np {nproc} {cmd}',
            teams_exe_cmd_nproc=2,
            teams_node_file=node_file)
        assert cy_teams_run_success == 0, \
            "Some parallel team cycle casts did not run successfully."
        assert cy_dir.joinpath("WrfHydroCycle.pkl").exists()

        # Check for command correctness in output files.
        file_check = {
            ('cast_2012121200/entry_cmd.output', 'cast_2012121500/entry_cmd.output', 'cast_2012121800/entry_cmd.output'):
            'mpirun entry_cmd\n',
            ('cast_2012121200/exit_cmd.output', 'cast_2012121500/exit_cmd.output', 'cast_2012121800/exit_cmd.output'):
            'mpirun exit_cmd\n',
            ('cast_2012121200/job_test_job_1/diag_hydro.00000', ):
            'mpirun --host r10i1n1.ib0.cheyenne.ucar.edu,r10i1n1.ib0.cheyenne.ucar.edu -np 2 ./wrf_hydro.exe\n',
            ('cast_2012121500/job_test_job_1/diag_hydro.00000', ):
            'mpirun --host r10i1n2.ib0.cheyenne.ucar.edu,r10i1n2.ib0.cheyenne.ucar.edu -np 2 ./wrf_hydro.exe\n',
            ('cast_2012121800/job_test_job_1/diag_hydro.00000', ):
            'mpirun --host r10i1n3.ib0.cheyenne.ucar.edu,r10i1n3.ib0.cheyenne.ucar.edu -np 2 ./wrf_hydro.exe\n',
        }
        for tup, ans in file_check.items():
            for file in tup:
                check_first_line(file, ans)
示例#13
0
def test_cycle_ensemble_run(
    ensemble,
    job_restart,
    scheduler,
    tmpdir,
    capfd,
    init_times,
    restart_dirs_ensemble
):
    # Valid force dir exercise.
    cy = CycleSimulation(
        init_times=init_times,
        restart_dirs=restart_dirs_ensemble,
        ncores=1,
        forcing_dirs=[
            ['.', '../dummy_extant_dir'],
            [-72, '../dummy_extant_dir'],
            [-72, '../dummy_extant_dir']
        ]
    )
    job_restart._entry_cmd = 'echo mpirun entry_cmd > entry_cmd.output'
    job_restart._exit_cmd = 'echo mpirun exit_cmd > exit_cmd.output'
    cy.add(job_restart)
    cy.add(ensemble)

    # Parallel teams test - test casts both in memory and not
    cy_teams_to_test = {
        'cy_teams_not_memory': copy.deepcopy(cy),
        'cy_teams_memory': copy.deepcopy(cy)
    }

    for key, cy_teams in cy_teams_to_test.items():

        cy_dir = pathlib.Path(tmpdir).joinpath(key)
        os.chdir(tmpdir)
        os.mkdir(str(cy_dir))
        os.chdir(str(cy_dir))
        cy_dir.joinpath('../dummy_extant_dir').touch()

        if key == 'cy_teams_not_memory':
            cy_teams.compose()
        else:
            cy_teams.compose(rm_casts_from_memory=False)

        cy_teams_run_success = cy_teams.run(
            teams=True,
            teams_exe_cmd=(
                ' ./wrf_hydro.exe mpirun --host {hostname} -np {nproc} {cmd}'),
            teams_exe_cmd_nproc=2,
            teams_node_file={'pbs': node_file}
        )
        assert cy_teams_run_success == 0, \
            "Some parallel team cycle casts did not run successfully."
        assert cy_dir.joinpath("WrfHydroCycle.pkl").exists()

        # Check for command correctness in output files.
        file_check = {
            ('cast_2012121200/member_000/entry_cmd.output',
             'cast_2012121200/member_001/entry_cmd.output',
             'cast_2012121500/member_000/entry_cmd.output',
             'cast_2012121500/member_001/entry_cmd.output',
             'cast_2012121800/member_000/entry_cmd.output',
             'cast_2012121800/member_001/entry_cmd.output'):
            'mpirun entry_cmd\n',

            ('cast_2012121200/member_000/exit_cmd.output',
             'cast_2012121200/member_001/exit_cmd.output',
             'cast_2012121500/member_000/exit_cmd.output',
             'cast_2012121500/member_001/exit_cmd.output',
             'cast_2012121800/member_000/exit_cmd.output',
             'cast_2012121800/member_001/exit_cmd.output'):
            'mpirun exit_cmd\n',

            ('cast_2012121200/member_000/job_test_job_1/diag_hydro.00000',
             'cast_2012121200/member_001/job_test_job_1/diag_hydro.00000',
             'cast_2012121800/member_000/job_test_job_1/diag_hydro.00000',
             'cast_2012121800/member_001/job_test_job_1/diag_hydro.00000'):
            'mpirun --host r10i1n1,r10i1n1 -np 2 ./wrf_hydro.exe\n',

            ('cast_2012121500/member_000/job_test_job_1/diag_hydro.00000',
             'cast_2012121500/member_001/job_test_job_1/diag_hydro.00000'):
            'mpirun --host r10i1n2,r10i1n2 -np 2 ./wrf_hydro.exe\n'

        }
        for tup, ans in file_check.items():
            for file in tup:
                check_first_line(file, ans)