def archive(environ, **kwargs): full_path, cname = hsm_full_path(environ) run('mkdir -p %s/ocean/%s' % (full_path, cname)) # TODO: copy OGCM output ({workdir}/dataout) run('mkdir -p %s/output' % full_path) with cd(fmt('{workdir}', environ)): with settings(warn_only=True): out = run(fmt('ls -1 *fms.out *logfile.*.out input.nml ' 'set_g4c_model*out.txt *_table *diag_integral.out ' 'set_g4c_pos_m4g4*out.txt *time_stamp.out 2> /dev/null', environ)) files = clear_output(out).splitlines() for f in files: if exists(f): if f not in ('data_table', 'diag_table', 'field_table', 'input.nml'): run(fmt('gzip %s' % f, environ)) f = f + '.gz' run(fmt('mv %s %s/output/' % (f, full_path), environ)) run('mkdir -p %s/restart' % full_path) with cd(fmt('{workdir}/RESTART', environ)): # TODO: check date in coupler.res! # !Temporary solution! It fails since there is no land* #run(fmt('tar czvf {finish}.tar.gz coupler* ice* land* ocean*', environ)) run(fmt('tar czvf {finish}.tar.gz coupler* ice* ocean*', environ)) run(fmt('mv {finish}.tar.gz %s/restart/' % full_path, environ)) with cd(fmt('{workdir}', environ)): run(fmt('tar czvf INPUT.tar.gz INPUT/ --exclude="*.res*"', environ)) run(fmt('mv INPUT.tar.gz %s/restart/' % full_path, environ))
def archive(environ, **kwargs): full_path, cname = hsm_full_path(environ) run('mkdir -p %s/atmos/%s' % (full_path, cname)) # TODO: copy AGCM output ({workdir}/pos/dataout) run('mkdir -p %s/output' % full_path) with cd(fmt('{workdir}', environ)): with settings(warn_only=True): out = run(fmt('ls -1 MODELIN Out.MPI.* set_post*out.txt ' 'set_g4c_posgrib*out.txt POSTIN-GRIB ' 'set_g4c_poseta*out.txt 2>/dev/null', environ)) files = clear_output(out).splitlines() for f in files: if exists(f): if f not in ('MODELIN', 'POSTIN-GRIB'): run(fmt('gzip %s' % f, environ)) f = f + '.gz' run(fmt('mv %s %s/output/' % (f, full_path), environ)) run('mkdir -p %s/restart' % full_path) with cd(fmt('{workdir}/model/dataout/TQ{TRC:04}L{LV:03}', environ)): run(fmt('tar czvf GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.tar.gz ' 'GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.*P???', environ)) run(fmt('mv GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.tar.gz ' '%s/restart/' % full_path, environ)) run(fmt('rm GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.*P???', environ))
def archive(environ, **kwargs): full_path, cname = hsm_full_path(environ) run('mkdir -p %s/ocean/%s' % (full_path, cname)) outfiles = run(fmt('ls -1 {workdir}/dataout/*nc', environ)) for f in outfiles.splitlines(): run(fmt('mv %s %s/ocean/%s/' % (f, full_path, cname), environ)) run('mkdir -p %s/output' % full_path) with cd(fmt('{workdir}', environ)): with settings(warn_only=True): out = run(fmt('ls -1 *fms.out *logfile.*.out input.nml ' 'set_g4c_model*out.txt *_table *diag_integral.out ' 'set_g4c_pos_m4g4*out.txt *time_stamp.out 2> /dev/null', environ)) files = clear_output(out).splitlines() for f in files: if exists(f): if f not in ('data_table', 'diag_table', 'field_table', 'input.nml'): run(fmt('gzip %s' % f, environ)) f = f + '.gz' run(fmt('mv %s %s/output/' % (f, full_path), environ)) run('mkdir -p %s/restart' % full_path) with cd(fmt('{workdir}/RESTART', environ)): # TODO: check date in coupler.res! # Gui 20131029, tar failled in a case where no land* file were available. I should think a more robust way to do this. #run(fmt('tar czvf {finish}.tar.gz coupler* ice* land* ocean*', environ)) run(fmt('tar czvf {finish}.tar.gz *.res *res.nc', environ)) run(fmt('mv {finish}.tar.gz %s/restart/' % full_path, environ)) with cd(fmt('{workdir}', environ)): run(fmt('tar czvf INPUT.tar.gz INPUT/ --exclude="*.res*"', environ)) run(fmt('mv INPUT.tar.gz %s/restart/' % full_path, environ))
def archive(environ, **kwargs): full_path, cname = hsm_full_path(environ) run('mkdir -p %s/ocean/%s' % (full_path, cname)) # TODO: copy OGCM output ({workdir}/dataout) run('mkdir -p %s/output' % full_path) with cd(fmt('{workdir}', environ)): with settings(warn_only=True): out = run( fmt( 'ls -1 *fms.out *logfile.*.out input.nml ' 'set_g4c_model*out.txt *_table *diag_integral.out ' 'set_g4c_pos_m4g4*out.txt *time_stamp.out 2> /dev/null', environ)) files = clear_output(out).splitlines() for f in files: if exists(f): if f not in ('data_table', 'diag_table', 'field_table', 'input.nml'): run(fmt('gzip %s' % f, environ)) f = f + '.gz' run(fmt('mv %s %s/output/' % (f, full_path), environ)) run('mkdir -p %s/restart' % full_path) with cd(fmt('{workdir}/RESTART', environ)): # TODO: check date in coupler.res! # !Temporary solution! It fails since there is no land* #run(fmt('tar czvf {finish}.tar.gz coupler* ice* land* ocean*', environ)) run(fmt('tar czvf {finish}.tar.gz coupler* ice* ocean*', environ)) run(fmt('mv {finish}.tar.gz %s/restart/' % full_path, environ)) with cd(fmt('{workdir}', environ)): run(fmt('tar czvf INPUT.tar.gz INPUT/ --exclude="*.res*"', environ)) run(fmt('mv INPUT.tar.gz %s/restart/' % full_path, environ))
def prepare_restart(environ, **kwargs): '''Prepare restart for new run''' with settings(warn_only=True): out = run(fmt('ls {workdir}/model/dataout/TQ{TRC:04d}L{LV:03d}/*{start}{restart}F.unf*outatt*', environ)) if out.failed: with cd(fmt('{workdir}/model/dataout/TQ{TRC:04}L{LV:03}', environ)): full_path, cname = hsm_full_path(environ) run(fmt('tar xf %s/restart/GFCTNMC{start}{restart}F.unf.TQ{TRC:04}L{LV:03}.tar.gz' % full_path, environ))
def prepare_restart(environ, **kwargs): '''Prepare restart for new run''' # TODO: check if it starts from zero (ocean forced) if 'cold' in environ['mode']: cmp_date = str(environ['start']) else: cmp_date = str(environ['restart']) with settings(warn_only=True): with cd(fmt('{workdir}/INPUT', environ)): full_path, cname = hsm_full_path(environ) run(fmt('tar xf %s/restart/%s.tar.gz' % (full_path, cmp_date), environ))
def prepare_restart(environ, **kwargs): '''Prepare restart for new run''' with settings(warn_only=True): out = run( fmt( 'ls {workdir}/model/dataout/TQ{TRC:04d}L{LV:03d}/*{start}{restart}F.unf*outatt*', environ)) if out.failed: with cd(fmt('{workdir}/model/dataout/TQ{TRC:04}L{LV:03}', environ)): full_path, cname = hsm_full_path(environ) run( fmt( 'tar xf %s/restart/GFCTNMC{start}{restart}F.unf.TQ{TRC:04}L{LV:03}.tar.gz' % full_path, environ))
def test_hsm_full_path(): environ = { 'start': 2008012200, 'type': 'atmos', 'hsm': '/archive', 'name': 'base' } canonical_names = (('atmos', 'AGCM'), ('coupled', 'CGCM'), ('mom4p1_falsecoupled', 'OGCM')) for t, n in canonical_names: environ['type'] = t full_path, cname = utils.hsm_full_path(environ) assert full_path == "/archive/base/dataout/ic01/ic2008/22" assert cname == n
def archive(environ, **kwargs): full_path, cname = hsm_full_path(environ) run('mkdir -p %s/atmos/%s' % (full_path, cname)) # TODO: copy AGCM output ({workdir}/pos/dataout) run('mkdir -p %s/output' % full_path) with cd(fmt('{workdir}', environ)): with settings(warn_only=True): out = run( fmt( 'ls -1 MODELIN Out.MPI.* set_post*out.txt ' 'set_g4c_posgrib*out.txt POSTIN-GRIB ' 'set_g4c_poseta*out.txt 2>/dev/null', environ)) files = clear_output(out).splitlines() for f in files: if exists(f): if f not in ('MODELIN', 'POSTIN-GRIB'): run(fmt('gzip %s' % f, environ)) f = f + '.gz' run(fmt('mv %s %s/output/' % (f, full_path), environ)) run('mkdir -p %s/restart' % full_path) with cd(fmt('{workdir}/model/dataout/TQ{TRC:04}L{LV:03}', environ)): run( fmt( 'tar czvf GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.tar.gz ' 'GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.*P???', environ)) run( fmt( 'mv GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.tar.gz ' '%s/restart/' % full_path, environ)) run( fmt('rm GFCTNMC{start}{finish}F.unf.TQ{TRC:04}L{LV:03}.*P???', environ))