def gen_sub(date,FA,cf):
err=''
isFatal=False
info=opt.run_sub_info(cf,date=date,FA=FA)
# roms (not agrif) run cmd in serial mode is different from agrif:
model = opt.get_conf(cf,'MODEL','name',type=str)[0].lower()
mpi = opt.get_conf(cf,'FLAGS','mpi')[0]
if model=='roms' and not mpi:
tmp=info['run_cmd'].split()
tmp[0]=tmp[0]+' <'
info['run_cmd']=' '.join(tmp)
if not os.path.isfile(info['sub0']):
err='ERROR: standard sub file not found: %s' % info['sub0']
isFatal=True
else:
s=open(info['sub0']).read()
for k in info.keys(): s=s.replace('%'+k.upper()+'%',info[k])
f=open(info['sub'],'w')
f.write(s)
f.close()
return err, isFatal, info['sub']
def zip_out_monthly(cf, ccf):
'''Compress monthly model stdout files until previous month - minmonths
See clean.conf
'''
quiet = opt.get_conf(ccf, 'FLAGS')[0]['quiet']
conf, err = opt.get_conf(ccf, 'OUT')
if not conf['run']: return
dates = gen_mdates(cf, prev_month=-conf['minmonths'])
# start log:
flog = log_init(cf, ccf)
if not quiet: print >> flog, 'zip_out_monthly'.upper() + ' :'
for date1, date2 in dates:
y, m = dateu.parse_date(date1)[:-1]
month = dateu.month_names(m).lower()
label = '%s_%d' % (month, y)
for FA in 'fa':
zip_rout(cf,
date1,
date2,
FA,
conf['clean'],
quiet,
label,
output=flog)
# end log:
if not quiet: print >> flog, '' # empty line
log_end(flog)
def clean_ini_monthly(cf,ccf):
'''
Clean model ini link files until end of n prev months
Config is done in the configuration file <clean.conf>
Create text file with the current ini links
If the ini file is a link is can be unlinked; if it is a file it
is not removed !
'''
quiet=opt.get_conf(ccf,'FLAGS')[0]['quiet']
conf,err=opt.get_conf(ccf,'INI')
clean=conf['clean']
if not conf['run']: return
# start log:
flog=log_init(cf,ccf)
if not quiet: print >>flog, 'clean_ini_monthly'.upper()+' :'
dates=gen_mdates(cf,prev_month=-conf['minmonths'])
for startMonth,endMonth in dates:
clean_ini(cf,startMonth,endMonth,FA='a',nest=0,clean=clean,quiet=quiet,output=flog)
# end log:
if not quiet: print >>flog, '' # empty line
log_end(flog)
def clean_frc_monthly(cf,ccf):
'''
Clean model input atm forcing files until end of n prev months
Config is done in the configuration file <clean.conf>
'''
quiet=opt.get_conf(ccf,'FLAGS')[0]['quiet']
Types='BLK','FRC'
# start log:
flog=log_init(cf,ccf)
if not quiet: print >>flog, 'clean_frc_monthly'.upper()+' :'
for Type in Types:
if not quiet: print >>flog,Type
conf,err=opt.get_conf(ccf,Type)
if not conf['run']: continue
dates=gen_mdates(cf,prev_month=-conf['minmonths'])
date1=dates[0][0]
date2=dates[-1][1]
for FA in ('fa'):
if FA=='a':
clean = conf['aclean']
elif FA=='f':
clean = conf['fclean']
clean_frc(cf,date1,date2,Type.lower(),FA,nest=0,clean=clean,quiet=quiet,output=flog)
# end log:
if not quiet: print >>flog, '' # empty line
log_end(flog)
def clean_ini_monthly(cf, ccf):
'''
Clean model ini link files until end of n prev months
Config is done in the configuration file <clean.conf>
Create text file with the current ini links
If the ini file is a link is can be unlinked; if it is a file it
is not removed !
'''
quiet = opt.get_conf(ccf, 'FLAGS')[0]['quiet']
conf, err = opt.get_conf(ccf, 'INI')
clean = conf['clean']
if not conf['run']: return
# start log:
flog = log_init(cf, ccf)
if not quiet: print >> flog, 'clean_ini_monthly'.upper() + ' :'
dates = gen_mdates(cf, prev_month=-conf['minmonths'])
for startMonth, endMonth in dates:
clean_ini(cf,
startMonth,
endMonth,
FA='a',
nest=0,
clean=clean,
quiet=quiet,
output=flog)
# end log:
if not quiet: print >> flog, '' # empty line
log_end(flog)
def clean_nc_files_monthly(cf, ccf):
'''
Clean model output NetCDF files until end of n prev months
Config is done in the configuration file <clean.conf>
'''
quiet = opt.get_conf(ccf, 'FLAGS')[0]['quiet']
Types = 'HIS', 'AVG', 'RST', 'FLT'
# start log:
flog = log_init(cf, ccf)
if not quiet: print >> flog, 'clean_nc_files_monthly'.upper() + ' :'
for Type in Types:
if not quiet: print >> flog, Type
conf, err = opt.get_conf(ccf, Type)
if not conf['run']: continue
dates = gen_mdates(cf, prev_month=-conf['minmonths'])
date1 = dates[0][0]
date2 = dates[-1][1]
for FA in ('fa'):
if FA == 'a':
clean = conf['aclean']
mdkeep = conf['amdkeep']
wkeep = conf['awkeep']
elif FA == 'f':
clean = conf['fclean']
mdkeep = conf['fmdkeep']
wkeep = conf['fwkeep']
clean_nc_files(cf,
date1,
date2,
Type.lower(),
FA,
nest=0,
mdkeep=mdkeep,
wkeep=wkeep,
clean=clean,
quiet=quiet,
output=flog)
# end log:
if not quiet: print >> flog, '' # empty line
log_end(flog)
def gen_ini(date,FA='a',nest=0,cf=CONF):
date=dateu.parse_date(date)
dateRst=dateu.next_date(date,-1)
rst=opt.nameof('out','rst',date=dateRst,FA=FA,nest=nest,cf=cf)
ini=opt.nameof('in', 'ini',date=date,FA=FA,nest=nest,cf=cf)
err=False
isFatal=False
if os.path.isfile(ini):
err='INI file already exists: ....%s' % ini[-30:]
isFatal=False
elif not os.path.isfile(rst):
err='RST file not found: %s' % rst
isFatal=True
else:
y,m,d=date.year,date.month,date.day
model=opt.get_conf(cf,'MODEL','name',type=str)[nest]
if model.lower()=='roms-agrif' and (m,d)==(1,1):
# copy and change time to 0:
err=opt.restart_ini(rst,ini)
if err: isFatal=True
else:
# do not copy, create link:
try:
os.symlink(rst,ini)
except OSError, e:
err=e.strerror
isFatal=True
def gen_atmfrc(date,FA='a',nest=0,cf=CONF,quiet=True):
date=dateu.parse_date(date)
err=''
isFatal=False
fname=opt.nameof('in','blk',date=date,FA=FA,nest=nest,cf=cf)
if os.path.isfile(fname):
err='BLK file already exists'
isFatal=False
else:
grd=opt.nameof('in','grd',cf=cf)
atmPath = opt.pathof(cf,'external','atm')
atmData=opt.atm_info(cf)['data']
nforec=opt.n_pred(cf)
if atmData=='wrf':
cycle=366-29+dateu.mndays(date.year,date.month)
err=bfrc.make_blk_wrf(fname,grd,date,FA,atmPath,quiet=quiet,cycle=cycle)
elif atmData=='gfs':
from okean.roms.inputs import surface
if FA=='a': nforec=0
model=opt.get_conf(cf,'MODEL','name',type=str)[nest].lower()
surface.make_blk_gfs(atmPath,grd,fname,date,nforec=nforec,model=model,quiet=quiet)
try:
if err:
err='ERROR creating bulk file ('+err+')'
isFatal=True
except OSError, e:
err='ERROR creating bulk file ('+str(e)+')'
isFatal=True
except:
def op_plt_wind(conf, plconf, date, FA='a', nest=0, **kargs):
closefig = kargs.get('closefig', True)
clearfig = kargs.get('clearfig', True)
save = kargs.get('save', True)
date = dateu.parse_date(date)
Figure, err = opt.get_plconf(plconf, 'FIGURE')
Err = []
Out = []
pltpath = opt.nameof('out', 'plots', cf=conf)
Data, err = opt.get_plconf(plconf, 'WIND')
ifig = -1
for tag in Figure['tag']:
ifig += 1
args = {}
args['ifig'] = ifig
tind = Data['time'][ifig]
if tind == 'dailyMean':
if FA == 'a': ndays = 1
elif FA == 'f': ndays = int(opt.get_conf(conf, 'PRED', 'npred')[0])
for day in range(ndays): # loop over time:
args['day'] = day
err, fig, info = plt_wind(conf, plconf, date, FA, nest, **args)
Err += [err]
if not err:
# save:
if save:
if tag: Tag = '_' + tag
else: Tag = tag
savename = 'wind_%s_%s_%d_n%d%s' % (
date.strftime('%Y%m%d'), FA, day, nest, Tag)
Y, M, D = date.year, date.month, date.day
Ydest = os.path.join(pltpath, '%s' % Y)
if not os.path.isdir(Ydest): os.mkdir(Ydest)
MDdest = os.path.join(Ydest, '%02d_%02d' % (M, D))
if not os.path.isdir(MDdest): os.mkdir(MDdest)
savename = os.path.join(
MDdest, savename + '.' + Figure['extension'][ifig])
Out += [savename]
pl.savefig(savename, dpi=pl.gcf().dpi)
if clearfig: fig.clear()
if closefig: pl.close(fig)
return Err, Out
def op_plt_wind(conf,plconf,date,FA='a',nest=0,**kargs):
closefig = kargs.get('closefig',True)
clearfig = kargs.get('clearfig',True)
save = kargs.get('save',True)
date=dateu.parse_date(date)
Figure,err = opt.get_plconf(plconf,'FIGURE')
Err=[]
Out=[]
pltpath = opt.nameof('out','plots',cf=conf)
Data,err = opt.get_plconf(plconf,'WIND')
ifig=-1
for tag in Figure['tag']:
ifig+=1
args={}
args['ifig']=ifig
tind = Data['time'][ifig]
if tind == 'dailyMean':
if FA=='a': ndays=1
elif FA=='f': ndays=int(opt.get_conf(conf,'PRED','npred')[0])
for day in range(ndays): # loop over time:
args['day']=day
err,fig,info=plt_wind(conf,plconf,date,FA,nest,**args)
Err+=[err]
if not err:
# save:
if save:
if tag: Tag='_'+tag
else: Tag=tag
savename='wind_%s_%s_%d_n%d%s' % (date.strftime('%Y%m%d'),FA,day,nest,Tag)
Y,M,D=date.year,date.month,date.day
Ydest = os.path.join(pltpath,'%s' % Y )
if not os.path.isdir(Ydest): os.mkdir(Ydest)
MDdest = os.path.join(Ydest,'%02d_%02d' % (M,D) )
if not os.path.isdir(MDdest): os.mkdir(MDdest)
savename=os.path.join(MDdest,savename+'.'+Figure['extension'][ifig])
Out+=[savename]
pl.savefig(savename,dpi=pl.gcf().dpi)
if clearfig: fig.clear()
if closefig: pl.close(fig)
return Err,Out
def log_name(cf,ccf):
today=dateu.currday(format='short',sep='')
logconf,err = opt.get_conf(ccf,'LOG',type=str)
name = logconf['logname']
create = eval(logconf['create'])
if create:
p=opt.pathof(cf,'logpath')
name=name.replace('#TODAY#',today)
return os.path.join(p,name)
else:
return False
def zip_plots_monthly(cf,ccf):
'''Compress monthly model stdout files until previous month - minmonths
See clean.conf
'''
quiet=opt.get_conf(ccf,'FLAGS')[0]['quiet']
conf,err=opt.get_conf(ccf,'PLOTS')
if not conf['run']: return
dates=gen_mdates(cf,prev_month=-conf['minmonths'])
# start log:
flog=log_init(cf,ccf)
if not quiet: print >>flog, 'zip_plots_monthly'.upper()+' :'
for startMonth,endMonths in dates:
y,m,d=dateu.parse_date(startMonth)
zip_plots(cf,y,m,conf['clean'],quiet,overwrite=False,output=flog)
# end log:
if not quiet: print >>flog, '' # empty line
log_end(flog)
def zip_plots_monthly(cf, ccf):
'''Compress monthly model stdout files until previous month - minmonths
See clean.conf
'''
quiet = opt.get_conf(ccf, 'FLAGS')[0]['quiet']
conf, err = opt.get_conf(ccf, 'PLOTS')
if not conf['run']: return
dates = gen_mdates(cf, prev_month=-conf['minmonths'])
# start log:
flog = log_init(cf, ccf)
if not quiet: print >> flog, 'zip_plots_monthly'.upper() + ' :'
for startMonth, endMonths in dates:
y, m, d = dateu.parse_date(startMonth)
zip_plots(cf, y, m, conf['clean'], quiet, overwrite=False, output=flog)
# end log:
if not quiet: print >> flog, '' # empty line
log_end(flog)
def log_name(cf, ccf):
today = dateu.currday(format='short', sep='')
logconf, err = opt.get_conf(ccf, 'LOG', type=str)
name = logconf['logname']
create = eval(logconf['create'])
if create:
p = opt.pathof(cf, 'logpath')
name = name.replace('#TODAY#', today)
return os.path.join(p, name)
else:
return False
def zip_out_monthly(cf,ccf):
'''Compress monthly model stdout files until previous month - minmonths
See clean.conf
'''
quiet=opt.get_conf(ccf,'FLAGS')[0]['quiet']
conf,err=opt.get_conf(ccf,'OUT')
if not conf['run']: return
dates=gen_mdates(cf,prev_month=-conf['minmonths'])
# start log:
flog=log_init(cf,ccf)
if not quiet: print >>flog, 'zip_out_monthly'.upper()+' :'
for date1,date2 in dates:
y,m=dateu.parse_date(date1)[:-1]
month=dateu.month_names(m).lower()
label='%s_%d' % (month,y)
for FA in 'fa':
zip_rout(cf,date1,date2,FA,conf['clean'],quiet,label,output=flog)
# end log:
if not quiet: print >>flog, '' # empty line
log_end(flog)
def op_plt_hslice(conf,plconf,date,FA,nest=0,**kargs):
closefig = kargs.get('closefig',True)
clearfig = kargs.get('clearfig',True)
save = kargs.get('save',True)
Varname = kargs.get('vname',False)
Depth = kargs.get('depth',False)
date=dateu.parse_date(date)
Err=[]
Out=[]
pltpath = opt.nameof('out','plots',cf=conf)
Figure,err = opt.get_plconf(plconf,'FIGURE')
Hslices,err = opt.get_plconf(plconf,'HSLICES')
Title,err=opt.get_plconf(plconf,'AXES','title')
def iterv(v):
try: len(v)
except: v=[v]
return v
for i in range(len(Figure.values()[0])): # loop figures
Varnames = Hslices['varnames'][i].split(',')
Varnames=[s.strip() for s in Varnames]
depths = Hslices['depths'][i]
addcurr = Hslices['addcurr'][i]
# clims = Hslices['clim'][i]
depths = iterv(depths)
addcurr = iterv(addcurr)
# clims = iterv(clims)
kargs={}
kargs['ifig']=i
kargs['type']=Hslices['input'][i]
#kargs['time']=Inputs['time'][i]
TIMES=Hslices['time'][i]
kargs['dcurr']=Hslices['dcurr'][i]
kargs['scurr']=Hslices['scurr'][i]
kargs['lcurr']=Hslices['lcurr'][i]
if Varname: varnames=[Varname]
else: varnames=Varnames
for var in varnames: # loop variables
var=var.strip()
nv=Varnames.index(var)
kargs['cmap']=Hslices['cmap'][i].split(',')[nv].strip()
kargs['norm']=Hslices['norm'][i].split(',')[nv].strip()
kargs['var']=var
vdepths=depths[nv]
try: iter(vdepths)
except: vdepths=[vdepths]
if Depth: vdepths=[Depth]
nz=-1
for z in vdepths: # loop depths
nz+=1
if z in ('s','surf','surface') or z>=0:
kargs['ind'] = z
kargs['slice'] = 'k'
else:
kargs['ind'] = z
kargs['slice'] = 'z'
cl=opt.get_clims(date,var,z,plconf)[i]
kargs['clim']=cl
for addc in addcurr: # add/no currents
kargs['currents'] = addc
# loop over time:
if TIMES=='all':
if FA=='a': times=[-1]
else: times=range(int(opt.get_conf(conf,'PRED','npred')[0]))
else: times=[TIMES]
for it in times:
kargs['time']=it
e,fig,info=plt_hslice(conf,plconf,date,FA,nest,**kargs)
Err+=[e]
if not e:
if var: svar=var
else: svar=''
if addc: scurr='curr'
else: scurr=''
if info['hasz']: sslice=kargs['slice']+'_'+str(kargs['ind'])
else: sslice=''
if date.hour>0:
sdate=date.strftime('%Y%m%d_%H')
else:
sdate=date.strftime('%Y%m%d')
savename='%s_%s_n%d_%s_%d_%s_%s_%s_%s' % (sdate,FA,nest,kargs['type'],kargs['time'],svar,scurr,sslice,Figure['tag'][i])
if Title[i]:
simpleTitle=1
rdate=date.strftime('%d-%m-%Y')
if sslice: sslice=sslice[2:]
title='%s#%s#n%d#%s#%d#%s#%s#%s' % (rdate,FA,nest,kargs['type'],kargs['time'],svar,scurr,sslice)
if simpleTitle: # simpler version of title:
if FA=='f': # forecast date:
rdate=dateu.next_date(date,kargs['time'])
rdate=date.strftime('%d-%m-%Y')
title = '%s#%s#%s#%s' % (rdate,svar,scurr,sslice)
if FA=='f': title=title+' (forec)'
while 1:
if title.find('##')>-1: title=title.replace('##','#')
else: break
title=title.replace('#',' ')
pl.title(title)
while 1:
if savename.find('__')>-1: savename=savename.replace('__','_')
else: break
savename=savename.strip('_')
if save:
Y,M,D=date.year,date.month,date.day,
Ydest = os.path.join(pltpath,'%s' % Y )
if not os.path.isdir(Ydest): os.mkdir(Ydest)
MDdest = os.path.join(Ydest,'%02d_%02d' % (M,D) )
if not os.path.isdir(MDdest): os.mkdir(MDdest)
savename=os.path.join(MDdest,savename+'.'+Figure['extension'][i])
Out+=[savename]
pl.savefig(savename,dpi=pl.gcf().dpi)
#pl.savefig(savename,dpi=300)
if clearfig: fig.clear()
if closefig: pl.close(fig)
return Err,Out
def op_plt_hslice(conf, plconf, date, FA, nest=0, **kargs):
closefig = kargs.get('closefig', True)
clearfig = kargs.get('clearfig', True)
save = kargs.get('save', True)
Varname = kargs.get('vname', False)
Depth = kargs.get('depth', False)
date = dateu.parse_date(date)
Err = []
Out = []
pltpath = opt.nameof('out', 'plots', cf=conf)
Figure, err = opt.get_plconf(plconf, 'FIGURE')
Hslices, err = opt.get_plconf(plconf, 'HSLICES')
Title, err = opt.get_plconf(plconf, 'AXES', 'title')
def iterv(v):
try:
len(v)
except:
v = [v]
return v
for i in range(len(Figure.values()[0])): # loop figures
Varnames = Hslices['varnames'][i].split(',')
Varnames = [s.strip() for s in Varnames]
depths = Hslices['depths'][i]
addcurr = Hslices['addcurr'][i]
# clims = Hslices['clim'][i]
depths = iterv(depths)
addcurr = iterv(addcurr)
# clims = iterv(clims)
kargs = {}
kargs['ifig'] = i
kargs['type'] = Hslices['input'][i]
#kargs['time']=Inputs['time'][i]
TIMES = Hslices['time'][i]
kargs['dcurr'] = Hslices['dcurr'][i]
kargs['scurr'] = Hslices['scurr'][i]
kargs['lcurr'] = Hslices['lcurr'][i]
if Varname: varnames = [Varname]
else: varnames = Varnames
for var in varnames: # loop variables
var = var.strip()
nv = Varnames.index(var)
kargs['cmap'] = Hslices['cmap'][i].split(',')[nv].strip()
kargs['norm'] = Hslices['norm'][i].split(',')[nv].strip()
kargs['var'] = var
vdepths = depths[nv]
try:
iter(vdepths)
except:
vdepths = [vdepths]
if Depth: vdepths = [Depth]
nz = -1
for z in vdepths: # loop depths
nz += 1
if z in ('s', 'surf', 'surface') or z >= 0:
kargs['ind'] = z
kargs['slice'] = 'k'
else:
kargs['ind'] = z
kargs['slice'] = 'z'
cl = opt.get_clims(date, var, z, plconf)[i]
kargs['clim'] = cl
for addc in addcurr: # add/no currents
kargs['currents'] = addc
# loop over time:
if TIMES == 'all':
if FA == 'a': times = [-1]
else:
times = range(
int(opt.get_conf(conf, 'PRED', 'npred')[0]))
else:
times = [TIMES]
for it in times:
kargs['time'] = it
e, fig, info = plt_hslice(conf, plconf, date, FA, nest,
**kargs)
Err += [e]
if not e:
if var: svar = var
else: svar = ''
if addc: scurr = 'curr'
else: scurr = ''
if info['hasz']:
sslice = kargs['slice'] + '_' + str(
kargs['ind'])
else:
sslice = ''
if date.hour > 0:
sdate = date.strftime('%Y%m%d_%H')
else:
sdate = date.strftime('%Y%m%d')
savename = '%s_%s_n%d_%s_%d_%s_%s_%s_%s' % (
sdate, FA, nest, kargs['type'], kargs['time'],
svar, scurr, sslice, Figure['tag'][i])
if Title[i]:
simpleTitle = 1
rdate = date.strftime('%d-%m-%Y')
if sslice: sslice = sslice[2:]
title = '%s#%s#n%d#%s#%d#%s#%s#%s' % (
rdate, FA, nest, kargs['type'],
kargs['time'], svar, scurr, sslice)
if simpleTitle: # simpler version of title:
if FA == 'f': # forecast date:
rdate = dateu.next_date(
date, kargs['time'])
rdate = date.strftime('%d-%m-%Y')
title = '%s#%s#%s#%s' % (rdate, svar,
scurr, sslice)
if FA == 'f': title = title + ' (forec)'
while 1:
if title.find('##') > -1:
title = title.replace('##', '#')
else:
break
title = title.replace('#', ' ')
pl.title(title)
while 1:
if savename.find('__') > -1:
savename = savename.replace('__', '_')
else:
break
savename = savename.strip('_')
if save:
Y, M, D = date.year, date.month, date.day,
Ydest = os.path.join(pltpath, '%s' % Y)
if not os.path.isdir(Ydest): os.mkdir(Ydest)
MDdest = os.path.join(Ydest,
'%02d_%02d' % (M, D))
if not os.path.isdir(MDdest): os.mkdir(MDdest)
savename = os.path.join(
MDdest,
savename + '.' + Figure['extension'][i])
Out += [savename]
pl.savefig(savename, dpi=pl.gcf().dpi)
#pl.savefig(savename,dpi=300)
if clearfig: fig.clear()
if closefig: pl.close(fig)
return Err, Out