def regrid_2d_prepare(environ, **kwargs): input_file = StringIO() get(fmt('{regrid_2d_namelist[file]}', environ), input_file) data = nml_decode(input_file.getvalue()) input_file.close() output = StringIO() try: tkeys = set( environ['regrid_2d_namelist']['vars'].keys()) & set(data.keys()) except KeyError: pass else: for k in tkeys: keys = (set(environ['regrid_2d_namelist']['vars'][k].keys()) & set(data[k].keys())) data[k].update([(ke, environ['regrid_2d_namelist']['vars'][k][ke]) for ke in keys]) src_file = data['regrid_2d_nml']['src_file'] run(fmt('cp %s {regrid_2d_workdir}/src_file.nc' % src_file, environ)) data['regrid_2d_nml']['src_file'] = 'src_file.nc' output.write(yaml2nml(data)) put(output, fmt('{regrid_2d_workdir}/input.nml', environ)) output.close()
def archive(environ, **kwargs): full_path, cname = hsm_full_path(environ) run('mkdir -p %s/ocean/%s' % (full_path, cname)) # TODO: copy OGCM output ({workdir}/dataout) run('mkdir -p %s/output' % full_path) with cd(fmt('{workdir}', environ)): with settings(warn_only=True): out = run( fmt( 'ls -1 *fms.out *logfile.*.out input.nml ' 'set_g4c_model*out.txt *_table *diag_integral.out ' 'set_g4c_pos_m4g4*out.txt *time_stamp.out 2> /dev/null', environ)) files = clear_output(out).splitlines() for f in files: if exists(f): if f not in ('data_table', 'diag_table', 'field_table', 'input.nml'): run(fmt('gzip %s' % f, environ)) f = f + '.gz' run(fmt('mv %s %s/output/' % (f, full_path), environ)) run('mkdir -p %s/restart' % full_path) with cd(fmt('{workdir}/RESTART', environ)): # TODO: check date in coupler.res! # !Temporary solution! It fails since there is no land* #run(fmt('tar czvf {finish}.tar.gz coupler* ice* land* ocean*', environ)) run(fmt('tar czvf {finish}.tar.gz coupler* ice* ocean*', environ)) run(fmt('mv {finish}.tar.gz %s/restart/' % full_path, environ)) with cd(fmt('{workdir}', environ)): run(fmt('tar czvf INPUT.tar.gz INPUT/ --exclude="*.res*"', environ)) run(fmt('mv INPUT.tar.gz %s/restart/' % full_path, environ))
def run_post(environ, **kwargs): ''' Submits ocean post-processing Used vars: JobID_model expdir workdir platform Depends on: None ''' print(fc.yellow('Submitting ocean post-processing')) opts = '' if environ['JobID_model']: opts = '-W depend=afterok:{JobID_model}' with cd(fmt('{expdir}/runscripts', environ)): out = run( fmt('qsub %s {workdir}/set_g4c_pos_m4g4.{platform}' % opts, environ)) environ['JobID_pos_ocean'] = out.split('\n')[-1] if environ.get('run_drifters_pos', False): out = run( fmt('qsub %s {workdir}/run_pos_drifters.{platform}' % opts, environ)) environ['JobID_pos_ocean'] = out.split('\n')[-1]
def archive(environ, **kwargs): full_path, cname = hsm_full_path(environ) run('mkdir -p %s/ocean/%s' % (full_path, cname)) # TODO: copy OGCM output ({workdir}/dataout) run('mkdir -p %s/output' % full_path) with cd(fmt('{workdir}', environ)): with settings(warn_only=True): out = run(fmt('ls -1 *fms.out *logfile.*.out input.nml ' 'set_g4c_model*out.txt *_table *diag_integral.out ' 'set_g4c_pos_m4g4*out.txt *time_stamp.out 2> /dev/null', environ)) files = clear_output(out).splitlines() for f in files: if exists(f): if f not in ('data_table', 'diag_table', 'field_table', 'input.nml'): run(fmt('gzip %s' % f, environ)) f = f + '.gz' run(fmt('mv %s %s/output/' % (f, full_path), environ)) run('mkdir -p %s/restart' % full_path) with cd(fmt('{workdir}/RESTART', environ)): # TODO: check date in coupler.res! # !Temporary solution! It fails since there is no land* #run(fmt('tar czvf {finish}.tar.gz coupler* ice* land* ocean*', environ)) run(fmt('tar czvf {finish}.tar.gz coupler* ice* ocean*', environ)) run(fmt('mv {finish}.tar.gz %s/restart/' % full_path, environ)) with cd(fmt('{workdir}', environ)): run(fmt('tar czvf INPUT.tar.gz INPUT/ --exclude="*.res*"', environ)) run(fmt('mv INPUT.tar.gz %s/restart/' % full_path, environ))
def run_post(environ, **kwargs): ''' Submits ocean post-processing Used vars: JobID_model expdir workdir platform Depends on: None ''' print(fc.yellow('Submitting ocean post-processing')) opts = '' if environ['JobID_model']: opts = '-W depend=afterok:{JobID_model}' with cd(fmt('{expdir}/runscripts', environ)): out = run(fmt( 'qsub %s {workdir}/set_g4c_pos_m4g4.{platform}' % opts, environ)) environ['JobID_pos_ocean'] = out.split('\n')[-1] if environ.get('run_drifters_pos', False): out = run(fmt('qsub %s {workdir}/run_pos_drifters.{platform}' % opts, environ)) environ['JobID_pos_ocean'] = out.split('\n')[-1]
def prepare_restart(environ, **kwargs): '''Prepare restart for new run''' with settings(warn_only=True): out = run(fmt('ls {workdir}/model/dataout/TQ{TRC:04d}L{LV:03d}/*{start}{restart}F.unf*outatt*', environ)) if out.failed: with cd(fmt('{workdir}/model/dataout/TQ{TRC:04}L{LV:03}', environ)): full_path, cname = hsm_full_path(environ) run(fmt('tar xf %s/restart/GFCTNMC{start}{restart}F.unf.TQ{TRC:04}L{LV:03}.tar.gz' % full_path, environ))
def regrid_2d(environ, **kwargs): regrid_2d_prepare(environ) with shell_env(environ, keys=['mom4_pre_npes', 'mom4_pre_walltime', 'executable_regrid_2d', 'regrid_2d_workdir', 'regrid_2d_src_file', 'account', 'platform']): with prefix(fmt('source {envconf}', environ)): with cd(fmt('{expdir}/runscripts/mom4_pre', environ)): out = run(fmt('/usr/bin/tcsh regrid_2d_run.csh', environ))
def compile_post(environ, **kwargs): with shell_env(environ, keys=['root', 'platform']): with prefix(fmt('source {envconf}', environ)): with cd(environ['comb_exe']): run(fmt('make -f {comb_src}/Make_combine', environ)) run( fmt('cp {root}/src/shared/drifters/drifters_combine {comb_exe}/', environ))
def generate_grid(environ, **kwargs): run(fmt('cp {topog_file} {gengrid_workdir}/topog_file.nc', environ)) with shell_env( environ, keys=['mom4_pre_npes', 'mom4_pre_walltime', 'RUNTM', 'executable_gengrid', 'gengrid_workdir', 'account', 'topog_file', 'platform']): with prefix(fmt('source {envconf}', environ)): with cd(fmt('{expdir}/runscripts/mom4_pre', environ)): out = run(fmt('/usr/bin/tcsh ocean_grid_run.csh', environ))
def generate_grid(environ, **kwargs): run(fmt('cp {topog_file} {gengrid_workdir}/topog_file.nc', environ)) with shell_env( environ, keys=['mom_pre_npes', 'mom_pre_walltime', 'RUNTM', 'executable_gengrid', 'gengrid_workdir', 'account', 'topog_file', 'platform', 'queue']): with prefix(fmt('source {envconf}', environ)): with cd(fmt('{expdir}/runscripts/preprocessing', environ)): out = run(fmt('/usr/bin/tcsh ocean_grid_run.csh', environ))
def kill_experiment(environ, **kwargs): print(fc.yellow('Killing experiment')) statuses = _get_status(environ) if statuses: for status in statuses.values(): s = status['Jobname'] if (s in fmt('M_{name}', environ) or s in fmt('C_{name}', environ) or s in fmt('P_{name}', environ)): run('qdel %s' % status['ID'].split('-')[0])
def regrid_3d(environ, **kwargs): run(fmt( 'cp {regrid_3d_src_file} {regrid_3d_workdir}/src_file.nc', environ)) with shell_env(environ, keys=['mom4_pre_npes', 'mom4_pre_walltime', 'executable_regrid_3d', 'regrid_3d_workdir', 'regrid_3d_dest_grid', 'regrid_3d_output_filename', 'account', 'platform']): with prefix(fmt('source {envconf}', environ)): with cd(fmt('{expdir}/runscripts/mom4_pre', environ)): out = run(fmt('/usr/bin/tcsh regrid_3d_run.csh', environ))
def compile_pre(environ, **kwargs): with prefix(fmt('source {envconf}', environ)): if environ.get('gengrid_run_this_module', False): with shell_env(environ, keys=[ 'root', 'platform', 'mkmf_template', 'executable_gengrid' ]): with cd(fmt('{execdir}/gengrid', environ)): run(fmt('/usr/bin/tcsh {gengrid_makeconf}', environ)) if environ.get('regrid_3d_run_this_module', False): with shell_env( environ, keys=['root', 'mkmf_template', 'executable_regrid_3d']): with cd(fmt('{execdir}/regrid_3d', environ)): run(fmt('/usr/bin/tcsh {regrid_3d_makeconf}', environ)) if environ.get('regrid_2d_run_this_module', False): with shell_env( environ, keys=['root', 'mkmf_template', 'executable_regrid_2d']): with cd(fmt('{execdir}/regrid_2d', environ)): run(fmt('/usr/bin/tcsh {regrid_2d_makeconf}', environ)) if environ.get('make_xgrids_run_this_module', False): with prefix(fmt('source {make_xgrids_envconf}', environ)): #run(fmt('cc -g -V -O -o {executable_make_xgrids} {make_xgrids_src} -I $NETCDF_DIR/include -L $NETCDF_DIR/lib -lnetcdf -lm -Duse_LARGEFILE -Duse_netCDF -DLARGE_FILE -Duse_libMPI', environ)) fix_MAXLOCAL_make_xgrids(environ) run( fmt( 'cc -g -V -O -o {executable_make_xgrids} {make_xgrids_src} -I $NETCDF_DIR/include -L $NETCDF_DIR/lib -lnetcdf -lm -Duse_LARGEFILE -Duse_netCDF -DLARGE_FILE', environ))
def make_xgrids(environ, **kwargs): with prefix(fmt('source {envconf}', environ)): with cd(fmt('{workdir}/gengrid', environ)): with settings(warn_only=True): out = run( fmt( '{executable_make_xgrids} -o ocean_grid.nc -a {atmos_gridx},{atmos_gridy}', environ)) # TODO: need to check why it returns 41 even if program ended right. # An appropriate return code is missing in make_xgrids.c ... if out.return_code == 41: run(fmt('cp ocean_grid.nc grid_spec_UNION.nc', environ)) if exists(fmt('{workdir}/gengrid/ocean_grid?.nc', environ)): run( fmt( 'for file in ocean_grid?.nc; do ncks -A $file grid_spec_UNION.nc; done', environ)) run(fmt('ncks -A grid_spec.nc grid_spec_UNION.nc', environ)) if exists(fmt('{workdir}/gengrid/grid_spec?.nc', environ)): run( fmt( 'for file in grid_spec?.nc; do ncks -A $file grid_spec_UNION.nc; done', environ)) else: sys.exit(1)
def prepare_workdir(environ, **kwargs): '''Prepare output dir Used vars: workdir workdir_template ''' print(fc.yellow('Preparing workdir')) run(fmt('mkdir -p {workdir}', environ)) run(fmt('rsync -rtL --progress {workdir_template}/* {workdir}', environ)) run(fmt('touch {workdir}/time_stamp.restart', environ))
def prepare_inputs(environ, **kwargs): #TODO: copy data to pre/datain (look at oper experiment) with cd(fmt('pre_atmos/scripts', environ)): fix_atmos_runpre(environ) envvars = { 'dirhome': fmt('{pre_atmos}', environ), 'dirdata': fmt('{rootexp}/AGCM-1.0', environ), 'direxe': fmt('{execdir}', environ) } with shell_env(envvars): run(fmt('bash/runAll.bash', environ))
def compile_post(environ, **kwargs): with shell_env(environ, keys=['root', 'platform', 'code_dir', 'mppnccombine', 'name']): with prefix(fmt('source {envconf}', environ)): with cd(environ['comb_exe']): #run(fmt('make -f {comb_src}/Make_combine', environ)) run(fmt('cc -V -O -o {mppnccombine} -I/usr/local/include -L/usr/local/lib {code_dir}/postprocessing/mppnccombine/mppnccombine.c -lnetcdf', environ)) with shell_env(environ, keys=['workdir', 'platform', 'npes', 'mppnccombine']): with prefix(fmt('source {envconf}', environ)): with cd(environ['comb_exe']): #run(fmt('make -f {comb_src}/Make_combine', environ)) run(fmt('{expdir}/runscripts/set_mom5_pos.cray run {npes} {name}', environ))
def run_model(environ, **kwargs): ''' Submits coupled model Used vars: workdir platform walltime datatable diagtable fieldtable executable execdir TRC LV rootexp mppnccombine comb_exe envconf expdir mode start restart finish npes name JobID_model Depends on: None ''' print(fc.yellow('Submitting coupled model')) trunc = "%04d" % environ['TRC'] lev = "%03d" % environ['LV'] env_vars = environ.copy() env_vars.update({'TRUNC': trunc, 'LEV': lev}) keys = [ 'workdir', 'platform', 'walltime', 'datatable', 'diagtable', 'fieldtable', 'executable', 'execdir', 'TRUNC', 'LEV', 'LV', 'rootexp', 'mppnccombine', 'comb_exe', 'account', 'DHEXT' ] with shell_env(env_vars, keys=keys): with prefix(fmt('source {envconf}', environ)): with cd(fmt('{expdir}/runscripts', environ)): output = run( fmt( '. run_g4c_model.cray {mode} {start} ' '{restart} {finish} {npes} {name}', environ)) environ['JobID_model'] = re.search(".*JobIDmodel:\s*(.*)\s*", output).groups()[0]
def fix_atmos_runpre(environ): '''Stupid pre runscripts...''' run( fmt( "sed -i.bak -r -e 's/^export DATA=.*$/export DATA={start}/g' bash/runAll.bash", environ)) run( fmt( "sed -i.bak -r -e 's|^export dirhome|#export dirhome|g' bash/runAll.bash", environ)) run( fmt( "sed -i.bak -r -e 's|^export dirdata|#export dirdata|g' bash/runAll.bash", environ))
def prepare_restart(environ, **kwargs): '''Prepare restart for new run''' with settings(warn_only=True): out = run( fmt( 'ls {workdir}/model/dataout/TQ{TRC:04d}L{LV:03d}/*{start}{restart}F.unf*outatt*', environ)) if out.failed: with cd(fmt('{workdir}/model/dataout/TQ{TRC:04}L{LV:03}', environ)): full_path, cname = hsm_full_path(environ) run( fmt( 'tar xf %s/restart/GFCTNMC{start}{restart}F.unf.TQ{TRC:04}L{LV:03}.tar.gz' % full_path, environ))
def prepare(environ, **kwargs): '''Create all directories and put files in the right places. Depends on: prepare_expdir link_agcm_inputs prepare_workdir ''' print(fc.yellow('Preparing expdir')) frun(fmt('mkdir -p {expdir}', environ)) frun(fmt('mkdir -p {execdir}', environ)) environ['model'].prepare(environ) frun(fmt('rsync -rtL --progress {expfiles}/exp/{name}/* {expdir}', environ))
def prepare_workdir(environ, **kwargs): '''Prepare output dir Used vars: workdir workdir_template ''' print(fc.yellow('Preparing workdir')) run(fmt('mkdir -p {workdir}', environ)) if environ['workdir_template'][-7:] == '.tar.gz': run(fmt('tar xzvf {workdir_template} -C {workdir} --strip-components 1', environ)) else: run(fmt('rsync -rtL --progress {workdir_template}/* {workdir}', environ)) run(fmt('touch {workdir}/time_stamp.restart', environ))
def link_agcm_inputs(environ, **kwargs): '''Copy AGCM inputs for model run and post-processing to the right place Used vars: rootexp agcm_pos_inputs agcm_model_inputs ''' for comp in ['model', 'pos']: print(fc.yellow(fmt("Linking AGCM %s input data" % comp, environ))) if not exists(fmt('{rootexp}/AGCM-1.0/%s/datain' % comp, environ)): run(fmt('mkdir -p {rootexp}/AGCM-1.0/%s/datain' % comp, environ)) run(fmt('cp -R {agcm_%s_inputs}/* ' '{rootexp}/AGCM-1.0/%s/datain' % (comp, comp), environ))
def archive(environ, **kwargs): full_path, cname = hsm_full_path(environ) run('mkdir -p %s/atmos/%s' % (full_path, cname)) # TODO: copy AGCM output ({workdir}/pos/dataout) run('mkdir -p %s/output' % full_path) with cd(fmt('{workdir}', environ)): with settings(warn_only=True): out = run(fmt('ls -1 MODELIN Out.MPI.* set_post*out.txt ' 'set_g4c_posgrib*out.txt POSTIN-GRIB ' 'set_g4c_poseta*out.txt 2>/dev/null', environ)) files = clear_output(out).splitlines() for f in files: if exists(f): if f not in ('MODELIN', 'POSTIN-GRIB'): run(fmt('gzip %s' % f, environ)) f = f + '.gz' run(fmt('mv %s %s/output/' % (f, full_path), environ)) run('mkdir -p %s/restart' % full_path) with cd(fmt('{workdir}/model/dataout/TQ{TRC:04}L{LV:03}', environ)): run(fmt('tar czvf GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.tar.gz ' 'GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.*P???', environ)) run(fmt('mv GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.tar.gz ' '%s/restart/' % full_path, environ)) run(fmt('rm GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.*P???', environ))
def prepare_restart(environ, **kwargs): '''Prepare restart for new run''' # TODO: check if it starts from zero (ocean forced) if 'cold' in environ['mode']: cmp_date = str(environ['start']) else: cmp_date = str(environ['restart']) with settings(warn_only=True): with cd(fmt('{workdir}/INPUT', environ)): full_path, cname = hsm_full_path(environ) run(fmt('tar xf %s/restart/%s.tar.gz' % (full_path, cmp_date), environ))
def check_restart(environ, **kwargs): if 'warm' in environ['mode']: prepare_restart(environ) run( fmt( 'ls {workdir}/model/dataout/TQ{TRC:04d}L{LV:03d}/*{start}{restart}F.unf*outatt*', environ))
def link_agcm_inputs(environ, **kwargs): '''Copy AGCM inputs for model run and post-processing to the right place Used vars: rootexp agcm_pos_inputs agcm_model_inputs ''' for comp in ['model', 'pos']: print(fc.yellow(fmt("Linking AGCM %s input data" % comp, environ))) if not exists(fmt('{rootexp}/AGCM-1.0/%s/datain' % comp, environ)): run(fmt('mkdir -p {rootexp}/AGCM-1.0/%s/datain' % comp, environ)) run( fmt( 'cp -R {agcm_%s_inputs}/* ' '{rootexp}/AGCM-1.0/%s/datain' % (comp, comp), environ))
def prepare_expdir(environ, **kwargs): run(fmt('mkdir -p {comb_exe}', environ)) if environ.get('gengrid_run_this_module', False): run(fmt('mkdir -p {execdir}/gengrid', environ)) run(fmt('mkdir -p {gengrid_workdir}', environ)) if environ.get('make_xgrids_run_this_module', False): run(fmt('mkdir -p {execdir}/make_xgrids', environ)) run(fmt('mkdir -p {make_xgrids_workdir}', environ)) if environ.get('regrid_3d_run_this_module', False): run(fmt('mkdir -p {execdir}/regrid_3d', environ)) run(fmt('mkdir -p {regrid_3d_workdir}', environ)) if environ.get('regrid_2d_run_this_module', False): run(fmt('mkdir -p {execdir}/regrid_2d', environ)) run(fmt('mkdir -p {regrid_2d_workdir}', environ)) # Need to check if input.nml->ocean_drifters_nml->use_this_module is True run(fmt('mkdir -p {workdir}/DRIFTERS', environ))
def verify_run(environ, **kwargs): # TODO: verify if was a natural end of run if 'cold' in environ['mode']: cmp_date = str(environ['start']) else: cmp_date = str(environ['restart']) run(fmt('grep "Total runtime" {workdir}/%s.fms.out' % cmp_date[:8], environ))
def run_model(environ, **kwargs): ''' Submits coupled model Used vars: workdir platform walltime datatable diagtable fieldtable executable execdir TRC LV rootexp mppnccombine comb_exe envconf expdir mode start restart finish npes name JobID_model Depends on: None ''' print(fc.yellow('Submitting coupled model')) trunc = "%04d" % environ['TRC'] lev = "%03d" % environ['LV'] env_vars = environ.copy() env_vars.update({'TRUNC': trunc, 'LEV': lev}) keys = ['workdir', 'platform', 'walltime', 'datatable', 'diagtable', 'fieldtable', 'executable', 'execdir', 'TRUNC', 'LEV', 'LV', 'rootexp', 'mppnccombine', 'comb_exe', 'account', 'DHEXT'] with shell_env(env_vars, keys=keys): with prefix(fmt('source {envconf}', environ)): with cd(fmt('{expdir}/runscripts', environ)): output = run(fmt('. run_g4c_model.cray {mode} {start} ' '{restart} {finish} {npes} {name}', environ)) environ['JobID_model'] = re.search(".*JobIDmodel:\s*(.*)\s*", output).groups()[0]
def prepare_expdir(environ, **kwargs): run(fmt('mkdir -p {comb_exe}', environ)) if environ.get('gengrid_run_this_module', False): run(fmt('mkdir -p {execdir}/gengrid', environ)) run(fmt('mkdir -p {gengrid_workdir}', environ)) if environ.get('make_xgrids_run_this_module', False): run(fmt('mkdir -p {execdir}/gengrid', environ)) run(fmt('mkdir -p {make_xgrids_workdir}', environ)) if environ.get('regrid_3d_run_this_module', False): run(fmt('mkdir -p {execdir}/regrid_3d', environ)) run(fmt('mkdir -p {regrid_3d_workdir}', environ)) if environ.get('regrid_2d_run_this_module', False): run(fmt('mkdir -p {execdir}/regrid_2d', environ)) run(fmt('mkdir -p {regrid_2d_workdir}', environ))
def instrument_code(environ, **kwargs): '''Instrument code using Cray's perftools. Used vars: executable expdir Depends on: compile_model ''' print(fc.yellow('Rebuilding executable with instrumentation')) with prefix('module load perftools'): compile_model(environ) if exists(fmt('{executable}+apa', environ)): run(fmt('rm {executable}+apa', environ)) run(fmt('pat_build -O {expdir}/instrument_coupler.apa ' '-o {executable}+apa {executable}', environ)) environ['executable'] = fmt('{executable}+apa', environ)
def run_model(environ, **kwargs): ''' Submits atmos model Used vars: rootexp workdir executable walltime execdir platform TRC LV envconf expdir start restart finish npes name JobID_model Depends on: None ''' print(fc.yellow('Submitting atmos model')) trunc = "%04d" % environ['TRC'] lev = "%03d" % environ['LV'] env_vars = environ.copy() env_vars.update({'TRUNC': trunc, 'LEV': lev}) keys = [ 'rootexp', 'workdir', 'TRUNC', 'LEV', 'executable', 'walltime', 'execdir', 'platform', 'LV' ] with shell_env(env_vars, keys=keys): with prefix(fmt('source {envconf}', environ)): with cd(fmt('{expdir}/runscripts', environ)): output = run( fmt( '. run_atmos_model.cray run {start} {restart} ' '{finish} {npes} {name}', environ)) job_id = re.search(".*JobIDmodel:\s*(.*)\s*", output).groups()[0] environ['JobID_model'] = job_id
def make_xgrids(environ, **kwargs): with prefix(fmt('source {envconf}', environ)): with cd(fmt('{workdir}/gengrid', environ)): with settings(warn_only=True): out = run(fmt('{executable_make_xgrids} -o ocean_grid.nc -a {atmos_gridx},{atmos_gridy}', environ)) # TODO: need to check why it returns 41 even if program ended right. # An appropriate return code is missing in make_xgrids.c ... if out.return_code == 41: run(fmt('cp ocean_grid.nc grid_spec_UNION.nc', environ)) if exists(fmt('{workdir}/gengrid/ocean_grid?.nc', environ)): run(fmt('for file in ocean_grid?.nc; do ncks -A $file grid_spec_UNION.nc; done', environ)) run(fmt('ncks -A grid_spec.nc grid_spec_UNION.nc', environ)) if exists(fmt('{workdir}/gengrid/grid_spec?.nc', environ)): run(fmt('for file in grid_spec?.nc; do ncks -A $file grid_spec_UNION.nc; done', environ)) else: sys.exit(1)
def run_post(environ, **kwargs): ''' Submits atmos post-processing Used vars: JobID_model expdir workdir platform Depends on: None ''' print(fc.yellow('Submitting atmos post-processing')) opts = '' if environ['JobID_model']: opts = '-W depend=afterok:{JobID_model}' with cd(fmt('{expdir}/runscripts', environ)): out = run(fmt('qsub %s {workdir}/set_g4c_posgrib.cray' % opts, environ)) environ['JobID_pos_atmos'] = out.split('\n')[-1]
def run_post(environ, **kwargs): ''' Submits atmos post-processing Used vars: JobID_model expdir workdir platform Depends on: None ''' print(fc.yellow('Submitting atmos post-processing')) opts = '' if environ['JobID_model']: opts = '-W depend=afterok:{JobID_model}' with cd(fmt('{expdir}/runscripts', environ)): out = run( fmt('qsub %s {workdir}/set_g4c_posgrib.cray' % opts, environ)) environ['JobID_pos_atmos'] = out.split('\n')[-1]
def check_restart(environ, **kwargs): prepare_restart(environ) if exists(fmt('{workdir}/INPUT/coupler.res', environ)): res_date, cmp_date = get_coupler_dates(environ) if res_date != cmp_date: print(fc.red('ERROR')) sys.exit(1) else: # TODO: check if it starts from zero (ocean forced) sys.exit(1)
def instrument_code(environ, **kwargs): '''Instrument code using Cray's perftools. Used vars: executable expdir Depends on: compile_model ''' print(fc.yellow('Rebuilding executable with instrumentation')) with prefix('module load perftools'): compile_model(environ) if exists(fmt('{executable}+apa', environ)): run(fmt('rm {executable}+apa', environ)) run( fmt( 'pat_build -O {expdir}/instrument_coupler.apa ' '-o {executable}+apa {executable}', environ)) environ['executable'] = fmt('{executable}+apa', environ)
def get_coupler_dates(environ): res_time = run(fmt('tail -1 {workdir}/INPUT/coupler.res', environ)) date_comp = [i for i in res_time.split('\n')[-1].split(' ') if i][:4] res_date = int("".join( date_comp[0:1] + ["%02d" % int(i) for i in date_comp[1:4]])) if 'cold' in environ['mode']: cmp_date = int(environ['start']) else: cmp_date = int(environ['restart']) return res_date, cmp_date
def run_model(environ, **kwargs): ''' Submits atmos model Used vars: rootexp workdir executable walltime execdir platform TRC LV envconf expdir start restart finish npes name JobID_model Depends on: None ''' print(fc.yellow('Submitting atmos model')) trunc = "%04d" % environ['TRC'] lev = "%03d" % environ['LV'] env_vars = environ.copy() env_vars.update({'TRUNC': trunc, 'LEV': lev}) keys = ['rootexp', 'workdir', 'TRUNC', 'LEV', 'executable', 'walltime', 'execdir', 'platform', 'LV'] with shell_env(env_vars, keys=keys): with prefix(fmt('source {envconf}', environ)): with cd(fmt('{expdir}/runscripts', environ)): output = run(fmt('. run_atmos_model.cray run {start} {restart} ' '{finish} {npes} {name}', environ)) job_id = re.search(".*JobIDmodel:\s*(.*)\s*", output).groups()[0] environ['JobID_model'] = job_id
def compile_pre(environ, **kwargs): with prefix(fmt('source {envconf}', environ)): if environ.get('gengrid_run_this_module', False): with shell_env(environ, keys=['root', 'platform', 'mkmf_template', 'executable_gengrid']): with cd(fmt('{execdir}/gengrid', environ)): run(fmt('/usr/bin/tcsh {gengrid_makeconf}', environ)) if environ.get('regrid_3d_run_this_module', False): with shell_env(environ, keys=['root', 'mkmf_template', 'executable_regrid_3d']): with cd(fmt('{execdir}/regrid_3d', environ)): run(fmt('/usr/bin/tcsh {regrid_3d_makeconf}', environ)) if environ.get('regrid_2d_run_this_module', False): with shell_env(environ, keys=['root', 'mkmf_template', 'executable_regrid_2d']): with cd(fmt('{execdir}/regrid_2d', environ)): run(fmt('/usr/bin/tcsh {regrid_2d_makeconf}', environ)) if environ.get('make_xgrids_run_this_module', False): with prefix(fmt('source {make_xgrids_envconf}', environ)): #run(fmt('cc -g -V -O -o {executable_make_xgrids} {make_xgrids_src} -I $NETCDF_DIR/include -L $NETCDF_DIR/lib -lnetcdf -lm -Duse_LARGEFILE -Duse_netCDF -DLARGE_FILE -Duse_libMPI', environ)) fix_MAXLOCAL_make_xgrids(environ) run(fmt('cc -g -V -O -o {executable_make_xgrids} {make_xgrids_src} -I $NETCDF_DIR/include -L $NETCDF_DIR/lib -lnetcdf -lm -Duse_LARGEFILE -Duse_netCDF -DLARGE_FILE', environ))
def check_restart(environ, **kwargs): prepare_restart(environ) if exists(fmt('{workdir}/INPUT/coupler.res', environ)): res_date, cmp_date = get_coupler_dates(environ) if res_date != cmp_date: if 'cold' in environ['mode']: print(fc.red("ERROR: requested to cold start model in '"+str(environ['start'])+"' but it is already partially run up to '"+str(res_date)+"'.")) else: print(fc.red("ERROR: requested warm start model in '"+str(environ['restart'])+"' but it is already partially run up to '"+str(res_date)+"'. Did you mean to change namelist.yaml value to this?")) sys.exit(1) else: # TODO: check if it starts from zero (ocean forced) sys.exit(1)
def check_status(environ, status): if status['ID'] in environ.get('JobID_pos_atmos', ""): print(fc.yellow('Atmos post-processing: %s' % JOB_STATES[status['S']])) elif status['ID'] in environ.get('JobID_model', ""): if status['S'] == 'R': with hide('running', 'stdout'): fcts = run(fmt('find {workdir}/model/dataout/ -iname "*.fct.*" | sort', environ)) date = re.search('.*(\d{10})F.*', fcts.splitlines()[-1]) if date: current = datetime.strptime(date.groups()[0], "%Y%m%d%H") print_ETA(environ, status, current) else: print(fc.yellow('Preparing!')) else: print(fc.yellow('Model: %s' % JOB_STATES[status['S']]))
def hsm_full_path(environ): start = str(environ["start"]) d = DATE_SLICES path = "ic%02s/ic%04s/%02s" % (start[d["m"]], start[d["y"]], start[d["d"]]) if environ["type"] == "atmos": cname = "AGCM" elif environ["type"] == "mom4p1_falsecoupled": cname = "OGCM" elif environ["type"] == "coupled": cname = "CGCM" else: cname = "GENERIC" full_path = fmt("{hsm}/{name}/dataout/%s" % path, environ) return full_path, cname
def compile_pre(environ, **kwargs): with prefix(fmt('source {envconf}', environ)): if environ.get('gengrid_run_this_module', False): with shell_env(environ, keys=['root', 'platform', 'mkmfTemplate', 'executable_gengrid']): with cd(fmt('{execdir}/gengrid', environ)): run(fmt('/usr/bin/tcsh {gengrid_makeconf}', environ)) if environ.get('regrid_3d_run_this_module', False): with shell_env(environ, keys=['root', 'mkmfTemplate', 'executable_regrid_3d']): with cd(fmt('{execdir}/regrid_3d', environ)): run(fmt('/usr/bin/tcsh {regrid_3d_makeconf}', environ)) if environ.get('regrid_2d_run_this_module', False): with shell_env(environ, keys=['root', 'mkmfTemplate', 'executable_regrid_2d']): with cd(fmt('{execdir}/regrid_2d', environ)): run(fmt('/usr/bin/tcsh {regrid_2d_makeconf}', environ)) # Gui 20140219 # I know it's a one line code deal, but is it the best way to do to # compile the xgrids here? To keep consistency, shouldn't it be # at make_xgrid_compile.csh? if environ.get('make_xgrids_run_this_module', False): with prefix(fmt('source {make_xgrids_envconf}', environ)): #run(fmt('cc -g -V -O -o {executable_make_xgrids} {make_xgrids_src} -I $NETCDF_DIR/include -L $NETCDF_DIR/lib -lnetcdf -lm -Duse_LARGEFILE -Duse_netCDF -DLARGE_FILE -Duse_libMPI', environ)) fix_MAXLOCAL_make_xgrids(environ) run(fmt('cc -g -V -O -o {executable_make_xgrids} {make_xgrids_src} -I $NETCDF_DIR/include -L $NETCDF_DIR/lib -lnetcdf -lm -Duse_LARGEFILE -Duse_netCDF -DLARGE_FILE', environ))
def hsm_full_path(environ): start = str(environ['start']) d = DATE_SLICES path = 'ic%02s/ic%04s/%02s' % (start[d['m']], start[d['y']], start[d['d']]) if environ['type'] == 'atmos': cname = 'AGCM' elif environ['type'] == 'mom4p1_falsecoupled': cname = 'OGCM' elif environ['type'] == 'coupled': cname = 'CGCM' elif environ['type'] in ['MOM_solo', 'MOM_SIS', 'CM2M', 'ESM2M', 'ICCM', 'EBM']: cname = 'OGCM' else: cname = 'GENERIC' full_path = fmt('{hsm}/{name}/dataout/%s' % path, environ) return full_path, cname
def _get_status(environ): ''' Retrieve PBS status from remote side ''' with settings(warn_only=True): with hide('running', 'stdout'): job_ids = " ".join([environ[k] for k in environ.keys() if "JobID" in k]) data = run(fmt("qstat -a %s" % job_ids, environ)) statuses = {} if data.succeeded: header = None for line in data.split('\n'): if header: info = line.split() try: statuses[info[0]] = dict(zip(header.split()[1:], info)) except IndexError: pass elif line.startswith('Job ID'): header = line return statuses
def check_status(environ, status): if status['ID'] in environ.get('JobID_pos_ocean', ""): print(fc.yellow('Ocean post-processing: %s' % JOB_STATES[status['S']])) elif status['ID'] in environ.get('JobID_model', ""): fmsfile = fmt("{workdir}/fms.out", environ) if status['S'] == 'R': if not exists(fmsfile): print(fc.yellow('Preparing!')) else: with hide('running', 'stdout'): line = run('tac %s | grep -m1 yyyy' % fmsfile).splitlines()[-1] current = re.search('(\d{4})/(\s*\d{1,2})/(\s*\d{1,2})\s(\s*' '\d{1,2}):(\s*\d{1,2}):(\s*\d{1,2})', line) if current: current = datetime(*[int(i) for i in current.groups()]) print_ETA(environ, status, current) else: print(fc.yellow('Preparing!')) else: print(fc.yellow('Model: %s' % JOB_STATES[status['S']]))
def check_code(environ, **kwargs): ''' Checks out code in the remote side. Used vars: clean_checkout code_dir code_repo code_dir code_branch revision executable ''' print(fc.yellow("Checking code")) changed = False if environ['clean_checkout']: run(fmt('rm -rf {code_dir}', environ)) changed = True if not exists(environ['code_dir']): print(fc.yellow("Creating new repository")) run(fmt('mkdir -p {code_dir}', environ)) run(fmt('hg clone {code_repo} {code_dir}', environ)) changed = True with cd(environ['code_dir']): print(fc.yellow("Updating existing repository")) # First check if there is any change in repository, or # if requesting a different branch/revision with settings(warn_only=True): res = run(fmt('hg incoming -b {code_branch}', environ)) if res.return_code == 0: # New changes! run('hg pull') changed = True rev = environ.get('revision', None) curr_rev = run('hg id -i').strip('+') run(fmt('hg update {code_branch} --clean', environ)) if rev and rev != 'last' and rev != curr_rev: run(fmt('hg update -r{revision}', environ)) changed = True # Need to check if executables exists! if not exists(environ['executable']): changed = True return changed