def regrid_2d_prepare(environ, **kwargs):
input_file = StringIO()
get(fmt('{regrid_2d_namelist[file]}', environ), input_file)
data = nml_decode(input_file.getvalue())
input_file.close()
output = StringIO()
try:
tkeys = set(
environ['regrid_2d_namelist']['vars'].keys()) & set(data.keys())
except KeyError:
pass
else:
for k in tkeys:
keys = (set(environ['regrid_2d_namelist']['vars'][k].keys())
& set(data[k].keys()))
data[k].update([(ke, environ['regrid_2d_namelist']['vars'][k][ke])
for ke in keys])
src_file = data['regrid_2d_nml']['src_file']
run(fmt('cp %s {regrid_2d_workdir}/src_file.nc' % src_file, environ))
data['regrid_2d_nml']['src_file'] = 'src_file.nc'
output.write(yaml2nml(data))
put(output, fmt('{regrid_2d_workdir}/input.nml', environ))
output.close()
def archive(environ, **kwargs):
full_path, cname = hsm_full_path(environ)
run('mkdir -p %s/ocean/%s' % (full_path, cname))
# TODO: copy OGCM output ({workdir}/dataout)
run('mkdir -p %s/output' % full_path)
with cd(fmt('{workdir}', environ)):
with settings(warn_only=True):
out = run(
fmt(
'ls -1 *fms.out *logfile.*.out input.nml '
'set_g4c_model*out.txt *_table *diag_integral.out '
'set_g4c_pos_m4g4*out.txt *time_stamp.out 2> /dev/null',
environ))
files = clear_output(out).splitlines()
for f in files:
if exists(f):
if f not in ('data_table', 'diag_table', 'field_table',
'input.nml'):
run(fmt('gzip %s' % f, environ))
f = f + '.gz'
run(fmt('mv %s %s/output/' % (f, full_path), environ))
run('mkdir -p %s/restart' % full_path)
with cd(fmt('{workdir}/RESTART', environ)):
# TODO: check date in coupler.res!
# !Temporary solution! It fails since there is no land*
#run(fmt('tar czvf {finish}.tar.gz coupler* ice* land* ocean*', environ))
run(fmt('tar czvf {finish}.tar.gz coupler* ice* ocean*', environ))
run(fmt('mv {finish}.tar.gz %s/restart/' % full_path, environ))
with cd(fmt('{workdir}', environ)):
run(fmt('tar czvf INPUT.tar.gz INPUT/ --exclude="*.res*"', environ))
run(fmt('mv INPUT.tar.gz %s/restart/' % full_path, environ))
def run_post(environ, **kwargs):
''' Submits ocean post-processing
Used vars:
JobID_model
expdir
workdir
platform
Depends on:
None
'''
print(fc.yellow('Submitting ocean post-processing'))
opts = ''
if environ['JobID_model']:
opts = '-W depend=afterok:{JobID_model}'
with cd(fmt('{expdir}/runscripts', environ)):
out = run(
fmt('qsub %s {workdir}/set_g4c_pos_m4g4.{platform}' % opts,
environ))
environ['JobID_pos_ocean'] = out.split('\n')[-1]
if environ.get('run_drifters_pos', False):
out = run(
fmt('qsub %s {workdir}/run_pos_drifters.{platform}' % opts,
environ))
environ['JobID_pos_ocean'] = out.split('\n')[-1]
def regrid_2d_prepare(environ, **kwargs):
input_file = StringIO()
get(fmt('{regrid_2d_namelist[file]}', environ), input_file)
data = nml_decode(input_file.getvalue())
input_file.close()
output = StringIO()
try:
tkeys = set(
environ['regrid_2d_namelist']['vars'].keys()) & set(data.keys())
except KeyError:
pass
else:
for k in tkeys:
keys = (set(environ['regrid_2d_namelist']['vars'][k].keys())
& set(data[k].keys()))
data[k].update([(ke, environ['regrid_2d_namelist']['vars'][k][ke])
for ke in keys])
src_file = data['regrid_2d_nml']['src_file']
run(fmt('cp %s {regrid_2d_workdir}/src_file.nc' % src_file, environ))
data['regrid_2d_nml']['src_file'] = 'src_file.nc'
output.write(yaml2nml(data))
put(output, fmt('{regrid_2d_workdir}/input.nml', environ))
output.close()
def archive(environ, **kwargs):
full_path, cname = hsm_full_path(environ)
run('mkdir -p %s/ocean/%s' % (full_path, cname))
# TODO: copy OGCM output ({workdir}/dataout)
run('mkdir -p %s/output' % full_path)
with cd(fmt('{workdir}', environ)):
with settings(warn_only=True):
out = run(fmt('ls -1 *fms.out *logfile.*.out input.nml '
'set_g4c_model*out.txt *_table *diag_integral.out '
'set_g4c_pos_m4g4*out.txt *time_stamp.out 2> /dev/null', environ))
files = clear_output(out).splitlines()
for f in files:
if exists(f):
if f not in ('data_table', 'diag_table', 'field_table', 'input.nml'):
run(fmt('gzip %s' % f, environ))
f = f + '.gz'
run(fmt('mv %s %s/output/' % (f, full_path), environ))
run('mkdir -p %s/restart' % full_path)
with cd(fmt('{workdir}/RESTART', environ)):
# TODO: check date in coupler.res!
# !Temporary solution! It fails since there is no land*
#run(fmt('tar czvf {finish}.tar.gz coupler* ice* land* ocean*', environ))
run(fmt('tar czvf {finish}.tar.gz coupler* ice* ocean*', environ))
run(fmt('mv {finish}.tar.gz %s/restart/' % full_path, environ))
with cd(fmt('{workdir}', environ)):
run(fmt('tar czvf INPUT.tar.gz INPUT/ --exclude="*.res*"', environ))
run(fmt('mv INPUT.tar.gz %s/restart/' % full_path, environ))
def run_post(environ, **kwargs):
''' Submits ocean post-processing
Used vars:
JobID_model
expdir
workdir
platform
Depends on:
None
'''
print(fc.yellow('Submitting ocean post-processing'))
opts = ''
if environ['JobID_model']:
opts = '-W depend=afterok:{JobID_model}'
with cd(fmt('{expdir}/runscripts', environ)):
out = run(fmt(
'qsub %s {workdir}/set_g4c_pos_m4g4.{platform}' % opts, environ))
environ['JobID_pos_ocean'] = out.split('\n')[-1]
if environ.get('run_drifters_pos', False):
out = run(fmt('qsub %s {workdir}/run_pos_drifters.{platform}' %
opts, environ))
environ['JobID_pos_ocean'] = out.split('\n')[-1]
def prepare_restart(environ, **kwargs):
'''Prepare restart for new run'''
with settings(warn_only=True):
out = run(fmt('ls {workdir}/model/dataout/TQ{TRC:04d}L{LV:03d}/*{start}{restart}F.unf*outatt*', environ))
if out.failed:
with cd(fmt('{workdir}/model/dataout/TQ{TRC:04}L{LV:03}', environ)):
full_path, cname = hsm_full_path(environ)
run(fmt('tar xf %s/restart/GFCTNMC{start}{restart}F.unf.TQ{TRC:04}L{LV:03}.tar.gz' % full_path, environ))
def regrid_2d(environ, **kwargs):
regrid_2d_prepare(environ)
with shell_env(environ, keys=['mom4_pre_npes', 'mom4_pre_walltime',
'executable_regrid_2d', 'regrid_2d_workdir',
'regrid_2d_src_file', 'account', 'platform']):
with prefix(fmt('source {envconf}', environ)):
with cd(fmt('{expdir}/runscripts/mom4_pre', environ)):
out = run(fmt('/usr/bin/tcsh regrid_2d_run.csh', environ))
def compile_post(environ, **kwargs):
with shell_env(environ, keys=['root', 'platform']):
with prefix(fmt('source {envconf}', environ)):
with cd(environ['comb_exe']):
run(fmt('make -f {comb_src}/Make_combine', environ))
run(
fmt('cp {root}/src/shared/drifters/drifters_combine {comb_exe}/',
environ))
def regrid_2d(environ, **kwargs):
regrid_2d_prepare(environ)
with shell_env(environ, keys=['mom4_pre_npes', 'mom4_pre_walltime',
'executable_regrid_2d', 'regrid_2d_workdir',
'regrid_2d_src_file', 'account', 'platform']):
with prefix(fmt('source {envconf}', environ)):
with cd(fmt('{expdir}/runscripts/mom4_pre', environ)):
out = run(fmt('/usr/bin/tcsh regrid_2d_run.csh', environ))
def generate_grid(environ, **kwargs):
run(fmt('cp {topog_file} {gengrid_workdir}/topog_file.nc', environ))
with shell_env(
environ, keys=['mom4_pre_npes', 'mom4_pre_walltime', 'RUNTM',
'executable_gengrid', 'gengrid_workdir',
'account', 'topog_file', 'platform']):
with prefix(fmt('source {envconf}', environ)):
with cd(fmt('{expdir}/runscripts/mom4_pre', environ)):
out = run(fmt('/usr/bin/tcsh ocean_grid_run.csh', environ))
def generate_grid(environ, **kwargs):
run(fmt('cp {topog_file} {gengrid_workdir}/topog_file.nc', environ))
with shell_env(
environ, keys=['mom_pre_npes', 'mom_pre_walltime', 'RUNTM',
'executable_gengrid', 'gengrid_workdir',
'account', 'topog_file', 'platform', 'queue']):
with prefix(fmt('source {envconf}', environ)):
with cd(fmt('{expdir}/runscripts/preprocessing', environ)):
out = run(fmt('/usr/bin/tcsh ocean_grid_run.csh', environ))
def kill_experiment(environ, **kwargs):
print(fc.yellow('Killing experiment'))
statuses = _get_status(environ)
if statuses:
for status in statuses.values():
s = status['Jobname']
if (s in fmt('M_{name}', environ) or s in fmt('C_{name}', environ)
or s in fmt('P_{name}', environ)):
run('qdel %s' % status['ID'].split('-')[0])
def kill_experiment(environ, **kwargs):
print(fc.yellow('Killing experiment'))
statuses = _get_status(environ)
if statuses:
for status in statuses.values():
s = status['Jobname']
if (s in fmt('M_{name}', environ) or
s in fmt('C_{name}', environ) or
s in fmt('P_{name}', environ)):
run('qdel %s' % status['ID'].split('-')[0])
def regrid_3d(environ, **kwargs):
run(fmt(
'cp {regrid_3d_src_file} {regrid_3d_workdir}/src_file.nc', environ))
with shell_env(environ, keys=['mom4_pre_npes', 'mom4_pre_walltime',
'executable_regrid_3d', 'regrid_3d_workdir',
'regrid_3d_dest_grid', 'regrid_3d_output_filename',
'account', 'platform']):
with prefix(fmt('source {envconf}', environ)):
with cd(fmt('{expdir}/runscripts/mom4_pre', environ)):
out = run(fmt('/usr/bin/tcsh regrid_3d_run.csh', environ))
def regrid_3d(environ, **kwargs):
run(fmt(
'cp {regrid_3d_src_file} {regrid_3d_workdir}/src_file.nc', environ))
with shell_env(environ, keys=['mom4_pre_npes', 'mom4_pre_walltime',
'executable_regrid_3d', 'regrid_3d_workdir',
'regrid_3d_dest_grid', 'regrid_3d_output_filename',
'account', 'platform']):
with prefix(fmt('source {envconf}', environ)):
with cd(fmt('{expdir}/runscripts/mom4_pre', environ)):
out = run(fmt('/usr/bin/tcsh regrid_3d_run.csh', environ))
def compile_pre(environ, **kwargs):
with prefix(fmt('source {envconf}', environ)):
if environ.get('gengrid_run_this_module', False):
with shell_env(environ,
keys=[
'root', 'platform', 'mkmf_template',
'executable_gengrid'
]):
with cd(fmt('{execdir}/gengrid', environ)):
run(fmt('/usr/bin/tcsh {gengrid_makeconf}', environ))
if environ.get('regrid_3d_run_this_module', False):
with shell_env(
environ,
keys=['root', 'mkmf_template', 'executable_regrid_3d']):
with cd(fmt('{execdir}/regrid_3d', environ)):
run(fmt('/usr/bin/tcsh {regrid_3d_makeconf}', environ))
if environ.get('regrid_2d_run_this_module', False):
with shell_env(
environ,
keys=['root', 'mkmf_template', 'executable_regrid_2d']):
with cd(fmt('{execdir}/regrid_2d', environ)):
run(fmt('/usr/bin/tcsh {regrid_2d_makeconf}', environ))
if environ.get('make_xgrids_run_this_module', False):
with prefix(fmt('source {make_xgrids_envconf}', environ)):
#run(fmt('cc -g -V -O -o {executable_make_xgrids} {make_xgrids_src} -I $NETCDF_DIR/include -L $NETCDF_DIR/lib -lnetcdf -lm -Duse_LARGEFILE -Duse_netCDF -DLARGE_FILE -Duse_libMPI', environ))
fix_MAXLOCAL_make_xgrids(environ)
run(
fmt(
'cc -g -V -O -o {executable_make_xgrids} {make_xgrids_src} -I $NETCDF_DIR/include -L $NETCDF_DIR/lib -lnetcdf -lm -Duse_LARGEFILE -Duse_netCDF -DLARGE_FILE',
environ))
def make_xgrids(environ, **kwargs):
with prefix(fmt('source {envconf}', environ)):
with cd(fmt('{workdir}/gengrid', environ)):
with settings(warn_only=True):
out = run(
fmt(
'{executable_make_xgrids} -o ocean_grid.nc -a {atmos_gridx},{atmos_gridy}',
environ))
# TODO: need to check why it returns 41 even if program ended right.
# An appropriate return code is missing in make_xgrids.c ...
if out.return_code == 41:
run(fmt('cp ocean_grid.nc grid_spec_UNION.nc', environ))
if exists(fmt('{workdir}/gengrid/ocean_grid?.nc', environ)):
run(
fmt(
'for file in ocean_grid?.nc; do ncks -A $file grid_spec_UNION.nc; done',
environ))
run(fmt('ncks -A grid_spec.nc grid_spec_UNION.nc', environ))
if exists(fmt('{workdir}/gengrid/grid_spec?.nc', environ)):
run(
fmt(
'for file in grid_spec?.nc; do ncks -A $file grid_spec_UNION.nc; done',
environ))
else:
sys.exit(1)
def prepare_workdir(environ, **kwargs):
'''Prepare output dir
Used vars:
workdir
workdir_template
'''
print(fc.yellow('Preparing workdir'))
run(fmt('mkdir -p {workdir}', environ))
run(fmt('rsync -rtL --progress {workdir_template}/* {workdir}', environ))
run(fmt('touch {workdir}/time_stamp.restart', environ))
def prepare_workdir(environ, **kwargs):
'''Prepare output dir
Used vars:
workdir
workdir_template
'''
print(fc.yellow('Preparing workdir'))
run(fmt('mkdir -p {workdir}', environ))
run(fmt('rsync -rtL --progress {workdir_template}/* {workdir}', environ))
run(fmt('touch {workdir}/time_stamp.restart', environ))
def prepare_inputs(environ, **kwargs):
#TODO: copy data to pre/datain (look at oper experiment)
with cd(fmt('pre_atmos/scripts', environ)):
fix_atmos_runpre(environ)
envvars = {
'dirhome': fmt('{pre_atmos}', environ),
'dirdata': fmt('{rootexp}/AGCM-1.0', environ),
'direxe': fmt('{execdir}', environ)
}
with shell_env(envvars):
run(fmt('bash/runAll.bash', environ))
def prepare_inputs(environ, **kwargs):
#TODO: copy data to pre/datain (look at oper experiment)
with cd(fmt('pre_atmos/scripts', environ)):
fix_atmos_runpre(environ)
envvars = {
'dirhome': fmt('{pre_atmos}', environ),
'dirdata': fmt('{rootexp}/AGCM-1.0', environ),
'direxe': fmt('{execdir}', environ)
}
with shell_env(envvars):
run(fmt('bash/runAll.bash', environ))
def compile_post(environ, **kwargs):
with shell_env(environ, keys=['root', 'platform', 'code_dir', 'mppnccombine', 'name']):
with prefix(fmt('source {envconf}', environ)):
with cd(environ['comb_exe']):
#run(fmt('make -f {comb_src}/Make_combine', environ))
run(fmt('cc -V -O -o {mppnccombine} -I/usr/local/include -L/usr/local/lib {code_dir}/postprocessing/mppnccombine/mppnccombine.c -lnetcdf', environ))
with shell_env(environ, keys=['workdir', 'platform', 'npes', 'mppnccombine']):
with prefix(fmt('source {envconf}', environ)):
with cd(environ['comb_exe']):
#run(fmt('make -f {comb_src}/Make_combine', environ))
run(fmt('{expdir}/runscripts/set_mom5_pos.cray run {npes} {name}', environ))
def run_model(environ, **kwargs):
''' Submits coupled model
Used vars:
workdir
platform
walltime
datatable
diagtable
fieldtable
executable
execdir
TRC
LV
rootexp
mppnccombine
comb_exe
envconf
expdir
mode
start
restart
finish
npes
name
JobID_model
Depends on:
None
'''
print(fc.yellow('Submitting coupled model'))
trunc = "%04d" % environ['TRC']
lev = "%03d" % environ['LV']
env_vars = environ.copy()
env_vars.update({'TRUNC': trunc, 'LEV': lev})
keys = [
'workdir', 'platform', 'walltime', 'datatable', 'diagtable',
'fieldtable', 'executable', 'execdir', 'TRUNC', 'LEV', 'LV', 'rootexp',
'mppnccombine', 'comb_exe', 'account', 'DHEXT'
]
with shell_env(env_vars, keys=keys):
with prefix(fmt('source {envconf}', environ)):
with cd(fmt('{expdir}/runscripts', environ)):
output = run(
fmt(
'. run_g4c_model.cray {mode} {start} '
'{restart} {finish} {npes} {name}', environ))
environ['JobID_model'] = re.search(".*JobIDmodel:\s*(.*)\s*",
output).groups()[0]
def fix_atmos_runpre(environ):
'''Stupid pre runscripts...'''
run(
fmt(
"sed -i.bak -r -e 's/^export DATA=.*$/export DATA={start}/g' bash/runAll.bash",
environ))
run(
fmt(
"sed -i.bak -r -e 's|^export dirhome|#export dirhome|g' bash/runAll.bash",
environ))
run(
fmt(
"sed -i.bak -r -e 's|^export dirdata|#export dirdata|g' bash/runAll.bash",
environ))
def prepare_restart(environ, **kwargs):
'''Prepare restart for new run'''
with settings(warn_only=True):
out = run(
fmt(
'ls {workdir}/model/dataout/TQ{TRC:04d}L{LV:03d}/*{start}{restart}F.unf*outatt*',
environ))
if out.failed:
with cd(fmt('{workdir}/model/dataout/TQ{TRC:04}L{LV:03}', environ)):
full_path, cname = hsm_full_path(environ)
run(
fmt(
'tar xf %s/restart/GFCTNMC{start}{restart}F.unf.TQ{TRC:04}L{LV:03}.tar.gz'
% full_path, environ))
def prepare(environ, **kwargs):
'''Create all directories and put files in the right places.
Depends on:
prepare_expdir
link_agcm_inputs
prepare_workdir
'''
print(fc.yellow('Preparing expdir'))
frun(fmt('mkdir -p {expdir}', environ))
frun(fmt('mkdir -p {execdir}', environ))
environ['model'].prepare(environ)
frun(fmt('rsync -rtL --progress {expfiles}/exp/{name}/* {expdir}', environ))
def prepare(environ, **kwargs):
'''Create all directories and put files in the right places.
Depends on:
prepare_expdir
link_agcm_inputs
prepare_workdir
'''
print(fc.yellow('Preparing expdir'))
frun(fmt('mkdir -p {expdir}', environ))
frun(fmt('mkdir -p {execdir}', environ))
environ['model'].prepare(environ)
frun(fmt('rsync -rtL --progress {expfiles}/exp/{name}/* {expdir}', environ))
def prepare_workdir(environ, **kwargs):
'''Prepare output dir
Used vars:
workdir
workdir_template
'''
print(fc.yellow('Preparing workdir'))
run(fmt('mkdir -p {workdir}', environ))
if environ['workdir_template'][-7:] == '.tar.gz':
run(fmt('tar xzvf {workdir_template} -C {workdir} --strip-components 1', environ))
else:
run(fmt('rsync -rtL --progress {workdir_template}/* {workdir}', environ))
run(fmt('touch {workdir}/time_stamp.restart', environ))
def link_agcm_inputs(environ, **kwargs):
'''Copy AGCM inputs for model run and post-processing to the right place
Used vars:
rootexp
agcm_pos_inputs
agcm_model_inputs
'''
for comp in ['model', 'pos']:
print(fc.yellow(fmt("Linking AGCM %s input data" % comp, environ)))
if not exists(fmt('{rootexp}/AGCM-1.0/%s/datain' % comp, environ)):
run(fmt('mkdir -p {rootexp}/AGCM-1.0/%s/datain' % comp, environ))
run(fmt('cp -R {agcm_%s_inputs}/* '
'{rootexp}/AGCM-1.0/%s/datain' % (comp, comp), environ))
def archive(environ, **kwargs):
full_path, cname = hsm_full_path(environ)
run('mkdir -p %s/atmos/%s' % (full_path, cname))
# TODO: copy AGCM output ({workdir}/pos/dataout)
run('mkdir -p %s/output' % full_path)
with cd(fmt('{workdir}', environ)):
with settings(warn_only=True):
out = run(fmt('ls -1 MODELIN Out.MPI.* set_post*out.txt '
'set_g4c_posgrib*out.txt POSTIN-GRIB '
'set_g4c_poseta*out.txt 2>/dev/null', environ))
files = clear_output(out).splitlines()
for f in files:
if exists(f):
if f not in ('MODELIN', 'POSTIN-GRIB'):
run(fmt('gzip %s' % f, environ))
f = f + '.gz'
run(fmt('mv %s %s/output/' % (f, full_path), environ))
run('mkdir -p %s/restart' % full_path)
with cd(fmt('{workdir}/model/dataout/TQ{TRC:04}L{LV:03}', environ)):
run(fmt('tar czvf GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.tar.gz '
'GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.*P???', environ))
run(fmt('mv GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.tar.gz '
'%s/restart/' % full_path, environ))
run(fmt('rm GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.*P???',
environ))
def prepare_restart(environ, **kwargs):
'''Prepare restart for new run'''
# TODO: check if it starts from zero (ocean forced)
if 'cold' in environ['mode']:
cmp_date = str(environ['start'])
else:
cmp_date = str(environ['restart'])
with settings(warn_only=True):
with cd(fmt('{workdir}/INPUT', environ)):
full_path, cname = hsm_full_path(environ)
run(fmt('tar xf %s/restart/%s.tar.gz' % (full_path, cmp_date), environ))
def prepare_restart(environ, **kwargs):
'''Prepare restart for new run'''
# TODO: check if it starts from zero (ocean forced)
if 'cold' in environ['mode']:
cmp_date = str(environ['start'])
else:
cmp_date = str(environ['restart'])
with settings(warn_only=True):
with cd(fmt('{workdir}/INPUT', environ)):
full_path, cname = hsm_full_path(environ)
run(fmt('tar xf %s/restart/%s.tar.gz' % (full_path, cmp_date), environ))
def check_restart(environ, **kwargs):
if 'warm' in environ['mode']:
prepare_restart(environ)
run(
fmt(
'ls {workdir}/model/dataout/TQ{TRC:04d}L{LV:03d}/*{start}{restart}F.unf*outatt*',
environ))
def link_agcm_inputs(environ, **kwargs):
'''Copy AGCM inputs for model run and post-processing to the right place
Used vars:
rootexp
agcm_pos_inputs
agcm_model_inputs
'''
for comp in ['model', 'pos']:
print(fc.yellow(fmt("Linking AGCM %s input data" % comp, environ)))
if not exists(fmt('{rootexp}/AGCM-1.0/%s/datain' % comp, environ)):
run(fmt('mkdir -p {rootexp}/AGCM-1.0/%s/datain' % comp, environ))
run(
fmt(
'cp -R {agcm_%s_inputs}/* '
'{rootexp}/AGCM-1.0/%s/datain' % (comp, comp), environ))
def prepare_expdir(environ, **kwargs):
run(fmt('mkdir -p {comb_exe}', environ))
if environ.get('gengrid_run_this_module', False):
run(fmt('mkdir -p {execdir}/gengrid', environ))
run(fmt('mkdir -p {gengrid_workdir}', environ))
if environ.get('make_xgrids_run_this_module', False):
run(fmt('mkdir -p {execdir}/make_xgrids', environ))
run(fmt('mkdir -p {make_xgrids_workdir}', environ))
if environ.get('regrid_3d_run_this_module', False):
run(fmt('mkdir -p {execdir}/regrid_3d', environ))
run(fmt('mkdir -p {regrid_3d_workdir}', environ))
if environ.get('regrid_2d_run_this_module', False):
run(fmt('mkdir -p {execdir}/regrid_2d', environ))
run(fmt('mkdir -p {regrid_2d_workdir}', environ))
# Need to check if input.nml->ocean_drifters_nml->use_this_module is True
run(fmt('mkdir -p {workdir}/DRIFTERS', environ))
def prepare_expdir(environ, **kwargs):
run(fmt('mkdir -p {comb_exe}', environ))
if environ.get('gengrid_run_this_module', False):
run(fmt('mkdir -p {execdir}/gengrid', environ))
run(fmt('mkdir -p {gengrid_workdir}', environ))
if environ.get('make_xgrids_run_this_module', False):
run(fmt('mkdir -p {execdir}/make_xgrids', environ))
run(fmt('mkdir -p {make_xgrids_workdir}', environ))
if environ.get('regrid_3d_run_this_module', False):
run(fmt('mkdir -p {execdir}/regrid_3d', environ))
run(fmt('mkdir -p {regrid_3d_workdir}', environ))
if environ.get('regrid_2d_run_this_module', False):
run(fmt('mkdir -p {execdir}/regrid_2d', environ))
run(fmt('mkdir -p {regrid_2d_workdir}', environ))
# Need to check if input.nml->ocean_drifters_nml->use_this_module is True
run(fmt('mkdir -p {workdir}/DRIFTERS', environ))
def verify_run(environ, **kwargs):
# TODO: verify if was a natural end of run
if 'cold' in environ['mode']:
cmp_date = str(environ['start'])
else:
cmp_date = str(environ['restart'])
run(fmt('grep "Total runtime" {workdir}/%s.fms.out' % cmp_date[:8], environ))
def run_model(environ, **kwargs):
''' Submits coupled model
Used vars:
workdir
platform
walltime
datatable
diagtable
fieldtable
executable
execdir
TRC
LV
rootexp
mppnccombine
comb_exe
envconf
expdir
mode
start
restart
finish
npes
name
JobID_model
Depends on:
None
'''
print(fc.yellow('Submitting coupled model'))
trunc = "%04d" % environ['TRC']
lev = "%03d" % environ['LV']
env_vars = environ.copy()
env_vars.update({'TRUNC': trunc, 'LEV': lev})
keys = ['workdir', 'platform', 'walltime', 'datatable', 'diagtable',
'fieldtable', 'executable', 'execdir', 'TRUNC', 'LEV', 'LV',
'rootexp', 'mppnccombine', 'comb_exe', 'account', 'DHEXT']
with shell_env(env_vars, keys=keys):
with prefix(fmt('source {envconf}', environ)):
with cd(fmt('{expdir}/runscripts', environ)):
output = run(fmt('. run_g4c_model.cray {mode} {start} '
'{restart} {finish} {npes} {name}', environ))
environ['JobID_model'] = re.search(".*JobIDmodel:\s*(.*)\s*", output).groups()[0]
def prepare_expdir(environ, **kwargs):
run(fmt('mkdir -p {comb_exe}', environ))
if environ.get('gengrid_run_this_module', False):
run(fmt('mkdir -p {execdir}/gengrid', environ))
run(fmt('mkdir -p {gengrid_workdir}', environ))
if environ.get('make_xgrids_run_this_module', False):
run(fmt('mkdir -p {execdir}/gengrid', environ))
run(fmt('mkdir -p {make_xgrids_workdir}', environ))
if environ.get('regrid_3d_run_this_module', False):
run(fmt('mkdir -p {execdir}/regrid_3d', environ))
run(fmt('mkdir -p {regrid_3d_workdir}', environ))
if environ.get('regrid_2d_run_this_module', False):
run(fmt('mkdir -p {execdir}/regrid_2d', environ))
run(fmt('mkdir -p {regrid_2d_workdir}', environ))
def instrument_code(environ, **kwargs):
'''Instrument code using Cray's perftools.
Used vars:
executable
expdir
Depends on:
compile_model
'''
print(fc.yellow('Rebuilding executable with instrumentation'))
with prefix('module load perftools'):
compile_model(environ)
if exists(fmt('{executable}+apa', environ)):
run(fmt('rm {executable}+apa', environ))
run(fmt('pat_build -O {expdir}/instrument_coupler.apa '
'-o {executable}+apa {executable}', environ))
environ['executable'] = fmt('{executable}+apa', environ)
def run_model(environ, **kwargs):
''' Submits atmos model
Used vars:
rootexp
workdir
executable
walltime
execdir
platform
TRC
LV
envconf
expdir
start
restart
finish
npes
name
JobID_model
Depends on:
None
'''
print(fc.yellow('Submitting atmos model'))
trunc = "%04d" % environ['TRC']
lev = "%03d" % environ['LV']
env_vars = environ.copy()
env_vars.update({'TRUNC': trunc, 'LEV': lev})
keys = [
'rootexp', 'workdir', 'TRUNC', 'LEV', 'executable', 'walltime',
'execdir', 'platform', 'LV'
]
with shell_env(env_vars, keys=keys):
with prefix(fmt('source {envconf}', environ)):
with cd(fmt('{expdir}/runscripts', environ)):
output = run(
fmt(
'. run_atmos_model.cray run {start} {restart} '
'{finish} {npes} {name}', environ))
job_id = re.search(".*JobIDmodel:\s*(.*)\s*", output).groups()[0]
environ['JobID_model'] = job_id
def make_xgrids(environ, **kwargs):
with prefix(fmt('source {envconf}', environ)):
with cd(fmt('{workdir}/gengrid', environ)):
with settings(warn_only=True):
out = run(fmt('{executable_make_xgrids} -o ocean_grid.nc -a {atmos_gridx},{atmos_gridy}', environ))
# TODO: need to check why it returns 41 even if program ended right.
# An appropriate return code is missing in make_xgrids.c ...
if out.return_code == 41:
run(fmt('cp ocean_grid.nc grid_spec_UNION.nc', environ))
if exists(fmt('{workdir}/gengrid/ocean_grid?.nc', environ)):
run(fmt('for file in ocean_grid?.nc; do ncks -A $file grid_spec_UNION.nc; done', environ))
run(fmt('ncks -A grid_spec.nc grid_spec_UNION.nc', environ))
if exists(fmt('{workdir}/gengrid/grid_spec?.nc', environ)):
run(fmt('for file in grid_spec?.nc; do ncks -A $file grid_spec_UNION.nc; done', environ))
else:
sys.exit(1)
def run_post(environ, **kwargs):
''' Submits atmos post-processing
Used vars:
JobID_model
expdir
workdir
platform
Depends on:
None
'''
print(fc.yellow('Submitting atmos post-processing'))
opts = ''
if environ['JobID_model']:
opts = '-W depend=afterok:{JobID_model}'
with cd(fmt('{expdir}/runscripts', environ)):
out = run(fmt('qsub %s {workdir}/set_g4c_posgrib.cray' % opts,
environ))
environ['JobID_pos_atmos'] = out.split('\n')[-1]
def run_post(environ, **kwargs):
''' Submits atmos post-processing
Used vars:
JobID_model
expdir
workdir
platform
Depends on:
None
'''
print(fc.yellow('Submitting atmos post-processing'))
opts = ''
if environ['JobID_model']:
opts = '-W depend=afterok:{JobID_model}'
with cd(fmt('{expdir}/runscripts', environ)):
out = run(
fmt('qsub %s {workdir}/set_g4c_posgrib.cray' % opts, environ))
environ['JobID_pos_atmos'] = out.split('\n')[-1]
def check_restart(environ, **kwargs):
prepare_restart(environ)
if exists(fmt('{workdir}/INPUT/coupler.res', environ)):
res_date, cmp_date = get_coupler_dates(environ)
if res_date != cmp_date:
print(fc.red('ERROR'))
sys.exit(1)
else:
# TODO: check if it starts from zero (ocean forced)
sys.exit(1)
def instrument_code(environ, **kwargs):
'''Instrument code using Cray's perftools.
Used vars:
executable
expdir
Depends on:
compile_model
'''
print(fc.yellow('Rebuilding executable with instrumentation'))
with prefix('module load perftools'):
compile_model(environ)
if exists(fmt('{executable}+apa', environ)):
run(fmt('rm {executable}+apa', environ))
run(
fmt(
'pat_build -O {expdir}/instrument_coupler.apa '
'-o {executable}+apa {executable}', environ))
environ['executable'] = fmt('{executable}+apa', environ)
def check_restart(environ, **kwargs):
prepare_restart(environ)
if exists(fmt('{workdir}/INPUT/coupler.res', environ)):
res_date, cmp_date = get_coupler_dates(environ)
if res_date != cmp_date:
print(fc.red('ERROR'))
sys.exit(1)
else:
# TODO: check if it starts from zero (ocean forced)
sys.exit(1)
def get_coupler_dates(environ):
res_time = run(fmt('tail -1 {workdir}/INPUT/coupler.res', environ))
date_comp = [i for i in res_time.split('\n')[-1].split(' ') if i][:4]
res_date = int("".join(
date_comp[0:1] + ["%02d" % int(i) for i in date_comp[1:4]]))
if 'cold' in environ['mode']:
cmp_date = int(environ['start'])
else:
cmp_date = int(environ['restart'])
return res_date, cmp_date
def get_coupler_dates(environ):
res_time = run(fmt('tail -1 {workdir}/INPUT/coupler.res', environ))
date_comp = [i for i in res_time.split('\n')[-1].split(' ') if i][:4]
res_date = int("".join(
date_comp[0:1] + ["%02d" % int(i) for i in date_comp[1:4]]))
if 'cold' in environ['mode']:
cmp_date = int(environ['start'])
else:
cmp_date = int(environ['restart'])
return res_date, cmp_date
def run_model(environ, **kwargs):
''' Submits atmos model
Used vars:
rootexp
workdir
executable
walltime
execdir
platform
TRC
LV
envconf
expdir
start
restart
finish
npes
name
JobID_model
Depends on:
None
'''
print(fc.yellow('Submitting atmos model'))
trunc = "%04d" % environ['TRC']
lev = "%03d" % environ['LV']
env_vars = environ.copy()
env_vars.update({'TRUNC': trunc, 'LEV': lev})
keys = ['rootexp', 'workdir', 'TRUNC', 'LEV', 'executable', 'walltime',
'execdir', 'platform', 'LV']
with shell_env(env_vars, keys=keys):
with prefix(fmt('source {envconf}', environ)):
with cd(fmt('{expdir}/runscripts', environ)):
output = run(fmt('. run_atmos_model.cray run {start} {restart} '
'{finish} {npes} {name}', environ))
job_id = re.search(".*JobIDmodel:\s*(.*)\s*", output).groups()[0]
environ['JobID_model'] = job_id
def compile_pre(environ, **kwargs):
with prefix(fmt('source {envconf}', environ)):
if environ.get('gengrid_run_this_module', False):
with shell_env(environ, keys=['root', 'platform', 'mkmf_template', 'executable_gengrid']):
with cd(fmt('{execdir}/gengrid', environ)):
run(fmt('/usr/bin/tcsh {gengrid_makeconf}', environ))
if environ.get('regrid_3d_run_this_module', False):
with shell_env(environ, keys=['root', 'mkmf_template', 'executable_regrid_3d']):
with cd(fmt('{execdir}/regrid_3d', environ)):
run(fmt('/usr/bin/tcsh {regrid_3d_makeconf}', environ))
if environ.get('regrid_2d_run_this_module', False):
with shell_env(environ, keys=['root', 'mkmf_template', 'executable_regrid_2d']):
with cd(fmt('{execdir}/regrid_2d', environ)):
run(fmt('/usr/bin/tcsh {regrid_2d_makeconf}', environ))
if environ.get('make_xgrids_run_this_module', False):
with prefix(fmt('source {make_xgrids_envconf}', environ)):
#run(fmt('cc -g -V -O -o {executable_make_xgrids} {make_xgrids_src} -I $NETCDF_DIR/include -L $NETCDF_DIR/lib -lnetcdf -lm -Duse_LARGEFILE -Duse_netCDF -DLARGE_FILE -Duse_libMPI', environ))
fix_MAXLOCAL_make_xgrids(environ)
run(fmt('cc -g -V -O -o {executable_make_xgrids} {make_xgrids_src} -I $NETCDF_DIR/include -L $NETCDF_DIR/lib -lnetcdf -lm -Duse_LARGEFILE -Duse_netCDF -DLARGE_FILE', environ))
def check_restart(environ, **kwargs):
prepare_restart(environ)
if exists(fmt('{workdir}/INPUT/coupler.res', environ)):
res_date, cmp_date = get_coupler_dates(environ)
if res_date != cmp_date:
if 'cold' in environ['mode']:
print(fc.red("ERROR: requested to cold start model in '"+str(environ['start'])+"' but it is already partially run up to '"+str(res_date)+"'."))
else:
print(fc.red("ERROR: requested warm start model in '"+str(environ['restart'])+"' but it is already partially run up to '"+str(res_date)+"'. Did you mean to change namelist.yaml value to this?"))
sys.exit(1)
else:
# TODO: check if it starts from zero (ocean forced)
sys.exit(1)
def check_status(environ, status):
if status['ID'] in environ.get('JobID_pos_atmos', ""):
print(fc.yellow('Atmos post-processing: %s' % JOB_STATES[status['S']]))
elif status['ID'] in environ.get('JobID_model', ""):
if status['S'] == 'R':
with hide('running', 'stdout'):
fcts = run(fmt('find {workdir}/model/dataout/ -iname "*.fct.*" | sort', environ))
date = re.search('.*(\d{10})F.*', fcts.splitlines()[-1])
if date:
current = datetime.strptime(date.groups()[0], "%Y%m%d%H")
print_ETA(environ, status, current)
else:
print(fc.yellow('Preparing!'))
else:
print(fc.yellow('Model: %s' % JOB_STATES[status['S']]))
def hsm_full_path(environ):
start = str(environ["start"])
d = DATE_SLICES
path = "ic%02s/ic%04s/%02s" % (start[d["m"]], start[d["y"]], start[d["d"]])
if environ["type"] == "atmos":
cname = "AGCM"
elif environ["type"] == "mom4p1_falsecoupled":
cname = "OGCM"
elif environ["type"] == "coupled":
cname = "CGCM"
else:
cname = "GENERIC"
full_path = fmt("{hsm}/{name}/dataout/%s" % path, environ)
return full_path, cname
def compile_pre(environ, **kwargs):
with prefix(fmt('source {envconf}', environ)):
if environ.get('gengrid_run_this_module', False):
with shell_env(environ, keys=['root', 'platform', 'mkmfTemplate', 'executable_gengrid']):
with cd(fmt('{execdir}/gengrid', environ)):
run(fmt('/usr/bin/tcsh {gengrid_makeconf}', environ))
if environ.get('regrid_3d_run_this_module', False):
with shell_env(environ, keys=['root', 'mkmfTemplate', 'executable_regrid_3d']):
with cd(fmt('{execdir}/regrid_3d', environ)):
run(fmt('/usr/bin/tcsh {regrid_3d_makeconf}', environ))
if environ.get('regrid_2d_run_this_module', False):
with shell_env(environ, keys=['root', 'mkmfTemplate', 'executable_regrid_2d']):
with cd(fmt('{execdir}/regrid_2d', environ)):
run(fmt('/usr/bin/tcsh {regrid_2d_makeconf}', environ))
# Gui 20140219
# I know it's a one line code deal, but is it the best way to do to
# compile the xgrids here? To keep consistency, shouldn't it be
# at make_xgrid_compile.csh?
if environ.get('make_xgrids_run_this_module', False):
with prefix(fmt('source {make_xgrids_envconf}', environ)):
#run(fmt('cc -g -V -O -o {executable_make_xgrids} {make_xgrids_src} -I $NETCDF_DIR/include -L $NETCDF_DIR/lib -lnetcdf -lm -Duse_LARGEFILE -Duse_netCDF -DLARGE_FILE -Duse_libMPI', environ))
fix_MAXLOCAL_make_xgrids(environ)
run(fmt('cc -g -V -O -o {executable_make_xgrids} {make_xgrids_src} -I $NETCDF_DIR/include -L $NETCDF_DIR/lib -lnetcdf -lm -Duse_LARGEFILE -Duse_netCDF -DLARGE_FILE', environ))
def hsm_full_path(environ):
start = str(environ['start'])
d = DATE_SLICES
path = 'ic%02s/ic%04s/%02s' % (start[d['m']], start[d['y']], start[d['d']])
if environ['type'] == 'atmos':
cname = 'AGCM'
elif environ['type'] == 'mom4p1_falsecoupled':
cname = 'OGCM'
elif environ['type'] == 'coupled':
cname = 'CGCM'
elif environ['type'] in ['MOM_solo', 'MOM_SIS', 'CM2M', 'ESM2M', 'ICCM', 'EBM']:
cname = 'OGCM'
else:
cname = 'GENERIC'
full_path = fmt('{hsm}/{name}/dataout/%s' % path, environ)
return full_path, cname
def _get_status(environ):
''' Retrieve PBS status from remote side '''
with settings(warn_only=True):
with hide('running', 'stdout'):
job_ids = " ".join([environ[k] for k in environ.keys()
if "JobID" in k])
data = run(fmt("qstat -a %s" % job_ids, environ))
statuses = {}
if data.succeeded:
header = None
for line in data.split('\n'):
if header:
info = line.split()
try:
statuses[info[0]] = dict(zip(header.split()[1:], info))
except IndexError:
pass
elif line.startswith('Job ID'):
header = line
return statuses
def check_status(environ, status):
if status['ID'] in environ.get('JobID_pos_ocean', ""):
print(fc.yellow('Ocean post-processing: %s' % JOB_STATES[status['S']]))
elif status['ID'] in environ.get('JobID_model', ""):
fmsfile = fmt("{workdir}/fms.out", environ)
if status['S'] == 'R':
if not exists(fmsfile):
print(fc.yellow('Preparing!'))
else:
with hide('running', 'stdout'):
line = run('tac %s | grep -m1 yyyy' % fmsfile).splitlines()[-1]
current = re.search('(\d{4})/(\s*\d{1,2})/(\s*\d{1,2})\s(\s*'
'\d{1,2}):(\s*\d{1,2}):(\s*\d{1,2})', line)
if current:
current = datetime(*[int(i) for i in current.groups()])
print_ETA(environ, status, current)
else:
print(fc.yellow('Preparing!'))
else:
print(fc.yellow('Model: %s' % JOB_STATES[status['S']]))
def check_code(environ, **kwargs):
''' Checks out code in the remote side.
Used vars:
clean_checkout
code_dir
code_repo
code_dir
code_branch
revision
executable
'''
print(fc.yellow("Checking code"))
changed = False
if environ['clean_checkout']:
run(fmt('rm -rf {code_dir}', environ))
changed = True
if not exists(environ['code_dir']):
print(fc.yellow("Creating new repository"))
run(fmt('mkdir -p {code_dir}', environ))
run(fmt('hg clone {code_repo} {code_dir}', environ))
changed = True
with cd(environ['code_dir']):
print(fc.yellow("Updating existing repository"))
# First check if there is any change in repository, or
# if requesting a different branch/revision
with settings(warn_only=True):
res = run(fmt('hg incoming -b {code_branch}', environ))
if res.return_code == 0: # New changes!
run('hg pull')
changed = True
rev = environ.get('revision', None)
curr_rev = run('hg id -i').strip('+')
run(fmt('hg update {code_branch} --clean', environ))
if rev and rev != 'last' and rev != curr_rev:
run(fmt('hg update -r{revision}', environ))
changed = True
# Need to check if executables exists!
if not exists(environ['executable']):
changed = True
return changed
