def check_fname_consistency(finput, validD, msg=None):
"""
Check file name consistency
Parameters:
----------
finput : class(e2oU.fname) :
validD : dictionary of valid keys
msg : message (optional)
Returns:
-------
msg is changed
"""
if msg is None:
msg = e2oU.init_msg()
emsg = 0
for att in ['cid', 'cver', 'cdomain', 'cfreq', 'cvar', 'ystart', 'yend']:
if not getattr(finput, att) in validD[att]:
msg['Emsg'].append(finput.fname + ': attribute:' + att +
' with value:' + str(getattr(finput, att)) +
' is not in the default list')
emsg = emsg + 1
if emsg == 0:
msg['Smsg'].append(finput.fname + ' file name consistency check OK')
def check_fname_consistency(finput, validD, msg=None):
"""
Check file name consistency
Parameters:
----------
finput : class(e2oU.fname) :
validD : dictionary of valid keys
msg : message (optional)
Returns:
-------
msg is changed
"""
if msg is None:
msg = e2oU.init_msg()
emsg = 0
for att in ["cid", "cver", "cdomain", "cfreq", "cvar", "ystart", "yend"]:
if not getattr(finput, att) in validD[att]:
msg["Emsg"].append(
finput.fname
+ ": attribute:"
+ att
+ " with value:"
+ str(getattr(finput, att))
+ " is not in the default list"
)
emsg = emsg + 1
if emsg == 0:
msg["Smsg"].append(finput.fname + " file name consistency check OK")
def check_fname_consistency(finput,validD,msg=None):
"""
Check file name consistency
Parameters:
----------
finput : class(e2oU.fname) :
validD : dictionary of valid keys
msg : message (optional)
Returns:
-------
msg is changed
"""
if msg is None:
msg = e2oU.init_msg()
emsg=0
for att in ['cid','cver','cdomain','cfreq','cvar','ystart','yend']:
if not getattr(finput,att) in validD[att] :
msg['Emsg'].append(finput.fname+': attribute:'+att +' with value:'+str(getattr(finput,att))+' is not in the default list' )
emsg=emsg+1
if emsg == 0 :
msg['Smsg'].append(finput.fname+' file name consistency check OK')
def check_eb(cf, ystart, yend, cdomain, cid, cver, msg=None):
"""
Energy balance check
Parameters:
----------
Returns:
-------
"""
rystart = ystart
ryend = yend
vLAT, vLON = e2oU.default_latlon(cf.cdomain)
nlat = len(vLAT)
nlon = len(vLON)
grid_area = e2oU.load_grid_area(fgarea)
cvarsEB = ["SWnet", "LWnet", "Qh", "Qle", "Qsm", "NET"]
if msg is None:
msg = e2oU.init_msg()
venB = {}
globM = {} ## global mean values for information only !
for cvar in cvarsEB:
cf = cf.attr2fpath(cfreq="mon", cvar=cvar, cdomain=cdomain, cid=cid, cver=cver)
print "EB, loading:", cvar
if cvar == "NET":
venB[cvar] = np.zeros((nlat, nlon))
for cc in cvarsEB[:-1]:
venB[cvar] = venB[cvar] + venB[cc]
else:
xdata, xtime = e2oU.load_nc_var(
cf.fpath, cf.cvar, dstart=dt.datetime(rystart, 1, 1), dend=dt.datetime(ryend, 12, 31)
)
if xdata is None:
venB[cvar] = np.zeros((nlat, nlon))
msg["Wmsg"].append("EB: Could not find variable: '%s', setting to zero!'" % (cvar))
else:
venB[cvar] = np.mean(xdata, 0)
if cvar == "Qsm":
venB[cvar] = venB[cvar] * 3.34e5 * -1.0
globM[cvar] = compute_area_mean(venB[cvar], grid_area)
for cvar in cvarsEB:
msg["Dmsg"].append(
"EB: Global mean of %s %f (W m-2) with %s/%s %f"
% (cvar, globM[cvar], cvar, cvarsEB[0], globM[cvar] / globM[cvarsEB[0]])
)
return msg
def check_variable_consistency(finput, msg=None):
"""
Check variables consistency (meta data only
Parameters:
----------
finput : class(e2oU.fname) :
msg : message (optional)
Returns:
-------
message
"""
if msg is None:
msg = e2oU.init_msg()
nc = Dataset(finput.fpath, 'r')
## general check
for cvtime in nc.variables.keys():
if cvtime in ['time', 'time_counter']: break
emsg = 0
for cvar in ['lat', 'lon', cvtime, finput.cvar]:
for att in ['long_name', 'units', '_FillValue', 'comment']:
if att == '_FillValue' and cvar != finput.cvar: continue
if att == 'comment' and cvar not in ['SurfMoist', 'RootMoist']:
continue
if finput.cfreq == "fix" and cvar == "time": continue
try:
getattr(nc.variables[cvar], att)
except:
msg['Emsg'].append(
finput.fname +
': attribute "%s" of variable "%s" not present' %
(att, cvar))
emsg = emsg + 1
if emsg == 0:
msg['Smsg'].append(finput.fname +
' variables attributes consistency check OK')
nc.close()
def check_variable_consistency(finput, msg=None):
"""
Check variables consistency (meta data only
Parameters:
----------
finput : class(e2oU.fname) :
msg : message (optional)
Returns:
-------
message
"""
if msg is None:
msg = e2oU.init_msg()
nc = Dataset(finput.fpath, "r")
## general check
emsg = 0
for cvar in ["lat", "lon", "time", finput.cvar]:
for att in ["long_name", "units", "_FillValue", "comment"]:
if att == "_FillValue" and cvar != finput.cvar:
continue
if att == "comment" and cvar not in ["SurfMoist", "RootMoist"]:
continue
if finput.cfreq == "fix" and cvar == "time":
continue
try:
getattr(nc.variables[cvar], att)
except:
msg["Emsg"].append(finput.fname + ': attribute "%s" of variable "%s" not present' % (att, cvar))
emsg = emsg + 1
if emsg == 0:
msg["Smsg"].append(finput.fname + " variables attributes consistency check OK")
nc.close()
def check_variable_consistency(finput,msg=None):
"""
Check variables consistency (meta data only
Parameters:
----------
finput : class(e2oU.fname) :
msg : message (optional)
Returns:
-------
message
"""
if msg is None:
msg = e2oU.init_msg()
nc = Dataset(finput.fpath,'r')
## general check
for cvtime in nc.variables.keys():
if cvtime in ['time','time_counter']: break
emsg=0
for cvar in ['lat','lon',cvtime,finput.cvar]:
for att in ['long_name','units','_FillValue','comment']:
if att == '_FillValue' and cvar != finput.cvar : continue
if att == 'comment' and cvar not in ['SurfMoist','RootMoist'] : continue
if finput.cfreq == "fix" and cvar == "time": continue
try:
getattr(nc.variables[cvar],att)
except:
msg['Emsg'].append(finput.fname+': attribute "%s" of variable "%s" not present'%(att,cvar) )
emsg=emsg+1
if emsg == 0 :
msg['Smsg'].append(finput.fname+' variables attributes consistency check OK')
nc.close()
def check_wb(cf, ystart, yend, cdomain, cid, cver, msg=None):
"""
Water balance check
Parameters:
----------
Returns:
-------
"""
rystart = ystart
ryend = yend
dstart = dt.datetime(cf.ystart, 1, 1).toordinal()
dend = dt.datetime(cf.yend, 12, 31).toordinal()
xTIME = num2date(np.arange(0, dend - dstart + 1, 1), "days since %4i-01-01 00:00:00" % cf.ystart)
tinD = [
np.nonzero(xTIME == dt.datetime(rystart, 1, 1))[0][0],
np.nonzero(xTIME == dt.datetime(ryend, 12, 31))[0][0],
]
ndays = tinD[1] - tinD[0] + 1
# print tinD
# print ndays
vLAT, vLON = e2oU.default_latlon(cf.cdomain)
nlat = len(vLAT)
nlon = len(vLON)
grid_area = e2oU.load_grid_area(fgarea)
cvarsWB = ["Precip", "Runoff", "Evap", "Stor", "NET"]
# cvarsWB=['Precip','Runoff','Evap','NET']
if msg is None:
msg = e2oU.init_msg()
venB = {}
globM = {} ## global mean values for information only !
for cvar in cvarsWB:
print "WB, loading:", cvar
if cvar == "NET":
venB[cvar] = np.zeros((nlat, nlon))
for cc in cvarsWB[:-1]:
venB[cvar] = venB[cvar] + venB[cc]
elif cvar == "Stor":
venB[cvar] = np.zeros((nlat, nlon))
for svar in ["TotMoist", "SWE", "CanopInt"]:
cf = cf.attr2fpath(cfreq="day", cvar=svar, cdomain=cdomain, cid=cid, cver=cver)
xdata, xtime = e2oU.load_nc_var(cf.fpath, cf.cvar, tinD=tinD)
if xdata is None:
xdata = np.zeros((2, nlat, nlon))
msg["Wmsg"].append("WB: Could not find variable: '%s', setting to zero!'" % (svar))
venB[cvar] = venB[cvar] + -1 * (xdata[1, :, :] - xdata[0, :, :]) / (ndays)
else:
cf = cf.attr2fpath(cfreq="mon", cvar=cvar, cdomain=cdomain, cid=cid, cver=cver)
xdata, xtime = e2oU.load_nc_var(
cf.fpath,
cf.cvar,
dstart=dt.datetime(rystart, 1, 1, 0, 0, 0),
dend=dt.datetime(ryend, 12, 31, 23, 59, 59),
)
# if cvar == 'Precip': print xtime
if xdata is None:
venB[cvar] = np.zeros((nlat, nlon))
msg["Wmsg"].append("WB: Could not find variable: '%s', setting to zero!'" % (cvar))
else:
venB[cvar] = np.mean(xdata, 0)
venB[cvar] = venB[cvar] * 86400.0
globM[cvar] = compute_area_mean(venB[cvar], grid_area)
for cvar in cvarsWB:
msg["Dmsg"].append(
"WB: Global mean of %s %f (mm day-1) with %s/%s %f"
% (cvar, globM[cvar], cvar, cvarsWB[0], globM[cvar] / globM[cvarsWB[0]])
)
if cvar == "NET":
msg["Dmsg"].append(
"WB:"
+ " variable %s with gpmin %e, gpmax %e fldmean %e #gp>thr %i"
% (
cvar,
np.min(venB[cvar]),
np.max(venB[cvar]),
globM[cvar],
np.sum(np.abs(venB[cvar]) > 5e-6 * 86400.0),
)
)
if np.sum(np.abs(venB[cvar]) > 5e-6 * 86400.0) > 0:
msg["Wmsg"].append(
"WB:"
+ " variable %s with gpmin %e, gpmax %e fldmean %e #gp>thr %i"
% (
cvar,
np.min(venB[cvar]),
np.max(venB[cvar]),
globM[cvar],
np.sum(np.abs(venB[cvar]) > 5e-6 * 86400.0),
)
)
# plt.figure()
# plt.pcolormesh(vLON,vLAT,venB[cvar]);plt.colorbar() # for plotting
# plt.title(cvar)
# if (cvar == "NET"):
# plt.figure()
# xx = venB[cvar]
# xx[np.abs(venB[cvar])<5e-6*86400.] = 0
# plt.pcolormesh(vLON,vLAT,venB[cvar]);plt.colorbar() # for plotting
# plt.title(cvar+' 0 == ok')
# plt.show()
return msg
def check_file_coords(finput, msg=None):
"""
Check file coordinates
Parameters:
----------
finput : class(e2oU.fname) :
msg : message (optional)
Returns:
-------
message
"""
if msg is None:
msg = e2oU.init_msg()
emsg = 0
nc = Dataset(finput.fpath, "r")
vLAT, vLON = e2oU.default_latlon(finput.cdomain)
if finput.cfreq == "mon":
nyears = finput.yend - finput.ystart + 1
vTIME = num2date(np.arange(5, 365.25 * nyears, 365.25 / 12), "days since %4i-01-01 00:00:00" % (finput.ystart))
elif finput.cfreq == "day":
dstart = dt.datetime(finput.ystart, 1, 1).toordinal()
dend = dt.datetime(finput.yend, 12, 31).toordinal()
vTIME = num2date(np.arange(0, dend - dstart + 1, 1), "days since %4i-01-01 00:00:00" % finput.ystart)
elif finput.cfreq == "fix":
pass
else:
print "This frequency test is not implemented yet", finput.cfreq
sys.exit(-1)
if finput.cfreq != "fix":
vYR, vMON, vDAY = date2yrmonday(vTIME)
fTIME = num2date(nc.variables["time"][:], nc.variables["time"].units)
fYR, fMON, fDAY = date2yrmonday(fTIME)
fLAT = nc.variables["lat"][:]
fLON = nc.variables["lon"][:]
try:
ddlon = np.abs(np.sum(vLON - fLON))
ddlat = np.abs(np.sum(vLAT - fLAT))
if ddlon > np.finfo(np.float32).eps or ddlat > np.finfo(np.float32).eps:
msg["Emsg"].append(finput.fname + " Lat or Lon arrays in file is not correct")
emsg = emsg + 1
except:
msg["Emsg"].append(finput.fname + " Some problem checking lat/lon arrays: check standard output")
traceback.print_exc()
emsg = emsg + 1
if finput.cfreq != "fix":
try:
ddyear = np.abs(np.sum(vYR - fYR))
ddmon = np.abs(np.sum(vMON - fMON))
ddday = np.abs(np.sum(vDAY - fDAY))
if ddyear > np.finfo(np.float32).eps or ddmon > np.finfo(np.float32).eps:
msg["Emsg"].append(finput.fname + " time year/mon arrays in file is not correct")
emsg = emsg + 1
if ddday > np.finfo(np.float32).eps and finput.cfreq == "day":
msg["Emsg"].append(finput.fname + " time day arrays in file is not correct")
emsg = emsg + 1
except:
msg["Emsg"].append(finput.fname + " Some problem checking time arrays: check standard output")
traceback.print_exc()
emsg = emsg + 1
if emsg == 0:
msg["Smsg"].append(finput.fname + " file coords check OK")
nc.close()
args = read_args()
fbase = args.fbase #'/scratch/rd/need/tmp/e2obs/g76h/' # folder location of the netcdf files
fgarea = args.fgarea #'/scratch/rd/need/tmp/e2obs/g57n/garea.nc' # location of the garea.nc file
## defaults
ystart = args.ystart # 1979 start year
yend = args.yend # 2012 end year
cdomain = args.cdomain # "glob30" simulations domain
cid = args.cid # "ecmwf" institution id
cver = args.cver # "wrr1" simulations version id
print args
##=======================================================
## 1. File consistency checks: we loop on all possible variables:
msg = e2oU.init_msg() # intialize message dictionary
cf = e2oU.fname() # initialize file name class
# loop on all possible variables / frequencies
for cvar in e2oU.validD["cvar"]:
for cfreq in ["day", "mon", "fix"]:
if cfreq == "fix" and cvar not in e2oU.validD["cvar_fix"]:
continue
if cvar in e2oU.validD["cvar_fix"] and cfreq != "fix":
continue
cf = cf.attr2fpath(
base=fbase, cfreq=cfreq, cvar=cvar, cdomain=cdomain, ystart=ystart, yend=yend, cid=cid, cver=cver
)
print "checking:", cf.fpath
# 1.1 :check if file can be opened !
def check_wb(cf,ystart,yend,cdomain,cid,cver,msg=None):
"""
Water balance check
Parameters:
----------
Returns:
-------
"""
rystart=ystart
ryend=yend
dstart=dt.datetime(cf.ystart,1,1).toordinal()
dend=dt.datetime(cf.yend,12,31).toordinal()
xTIME = num2date(np.arange(0,dend-dstart+1,1),"days since %4i-01-01 00:00:00"%cf.ystart)
tinD=[np.nonzero(xTIME==dt.datetime(rystart,1,1))[0][0],np.nonzero(xTIME==dt.datetime(ryend,12,31))[0][0]]
ndays=tinD[1]-tinD[0]+1
#print tinD
#print ndays
vLAT,vLON = e2oU.default_latlon(cf.cdomain)
nlat = len(vLAT)
nlon = len(vLON)
grid_area = e2oU.load_grid_area(fgarea)
cvarsWB=['Precip','Runoff','Evap','Stor','NET']
if msg is None:
msg=e2oU.init_msg()
venB={}
globM={} ## global mean values for information only !
for cvar in cvarsWB:
print 'WB, loading:',cvar
if cvar == "NET":
venB[cvar] = np.zeros((nlat,nlon))
for cc in cvarsWB[:-1]:
venB[cvar] = venB[cvar] + venB[cc]
elif cvar == "Stor":
venB[cvar] = np.zeros((nlat,nlon))
for svar in ['TotMoist','SWE','CanopInt','SurfStor']:
print 'WB, loading:',svar
cf = cf.attr2fpath(cfreq='day',cvar=svar,cdomain=cdomain,cid=cid,cver=cver)
xdata,xtime = e2oU.load_nc_var(cf.fpath,cf.cvar,tinD=tinD)
if xdata is None:
xdata = np.zeros((2,nlat,nlon))
msg['Wmsg'].append("WB: Could not find variable: '%s', setting to zero!'"%(svar))
venB[cvar] = venB[cvar] + -1*(xdata[1,:,:] - xdata[0,:,:])/(ndays)
else:
cf = cf.attr2fpath(cfreq='mon',cvar=cvar,cdomain=cdomain,cid=cid,cver=cver)
xdata,xtime = e2oU.load_nc_var(cf.fpath,cf.cvar,dstart=dt.datetime(rystart,1,1,0,0,0),dend=dt.datetime(ryend,12,31,23,59,59))
#if cvar == 'Precip': print xtime
if xdata is None:
venB[cvar] = np.zeros((nlat,nlon))
msg['Wmsg'].append("WB: Could not find variable: '%s', setting to zero!'"%(cvar))
else:
venB[cvar] = np.mean(xdata,0)
venB[cvar] = venB[cvar]*86400.
globM[cvar] = compute_area_mean(venB[cvar],grid_area)
for cvar in cvarsWB:
msg['Dmsg'].append("WB: Global mean of %s %f (mm day-1) with %s/%s %f"%
(cvar,globM[cvar],cvar,cvarsWB[0],globM[cvar]/globM[cvarsWB[0]]))
if cvar == "NET" :
msg['Dmsg'].append('WB:'+" variable %s with gpmin %e, gpmax %e fldmean %e #gp>thr %i"%
(cvar,np.min(venB[cvar]),np.max(venB[cvar]),globM[cvar],np.sum(np.abs(venB[cvar])>5e-6*86400.)))
if np.sum(np.abs(venB[cvar])>5e-6*86400.) > 0:
msg['Wmsg'].append('WB:'+" variable %s with gpmin %e, gpmax %e fldmean %e #gp>thr %i"%
(cvar,np.min(venB[cvar]),np.max(venB[cvar]),globM[cvar],np.sum(np.abs(venB[cvar])>5e-6*86400.)))
if LPLOT:
#produce map with WB residuals
#requires plot_utils from pyutils
if (cvar == "NET"):
from pyutils import plot_utils as pu
import matplotlib.pyplot as plt
opts={}
opts['Clevels']=np.arange(-2,2.5,0.5)*86400*5e-6
opts['cmap']=plt.cm.get_cmap('RdBu')
fig=pu.plot_map(vLON,vLAT,venB[cvar],titleC='WB residual',titleL=cf.cid,contourf=False,
titleR="%i #gp"%np.sum(np.abs(venB[cvar])>5e-6*86400.),Clabel='[mm/day]',**opts)
fout='map_wb_res_%s_%s_%s_%i_%i.png'%(cf.cid,cf.cver,cf.cdomain,cf.ystart,cf.yend)
print "Saving:",fout
fig[0].savefig(fout,bbox_inches="tight",dpi=200)
plt.close(fig[0])
return msg
def check_eb(cf,ystart,yend,cdomain,cid,cver,msg=None):
"""
Energy balance check
Parameters:
----------
Returns:
-------
"""
rystart=ystart
ryend=yend
vLAT,vLON = e2oU.default_latlon(cf.cdomain)
nlat = len(vLAT)
nlon = len(vLON)
grid_area = e2oU.load_grid_area(fgarea)
cvarsEB=['SWnet','LWnet','Qh','Qle','Qsm','NET']
if msg is None:
msg=e2oU.init_msg()
venB={}
globM={} ## global mean values for information only !
for cvar in cvarsEB:
cf = cf.attr2fpath(cfreq='mon',cvar=cvar,cdomain=cdomain,cid=cid,cver=cver)
print 'EB, loading:',cvar
if cvar == "NET":
venB[cvar] = np.zeros((nlat,nlon))
for cc in cvarsEB[:-1]:
venB[cvar] = venB[cvar] + venB[cc]
else:
xdata,xtime = e2oU.load_nc_var(cf.fpath,cf.cvar,dstart=dt.datetime(rystart,1,1),dend=dt.datetime(ryend,12,31))
if xdata is None:
venB[cvar] = np.zeros((nlat,nlon))
msg['Wmsg'].append("EB: Could not find variable: '%s', setting to zero!'"%(cvar))
else:
venB[cvar] = np.mean(xdata,0)
if cvar == "Qsm":
venB[cvar] = venB[cvar]*3.34e5*-1.
globM[cvar] = compute_area_mean(venB[cvar],grid_area)
for cvar in cvarsEB:
msg['Dmsg'].append("EB: Global mean of %s %f (W m-2) with %s/%s %f"%
(cvar,globM[cvar],cvar,cvarsEB[0],globM[cvar]/globM[cvarsEB[0]]))
if LPLOT:
#produce map with EB residuals
#requires plot_utils from pyutils # ask Edutra
from pyutils import plot_utils as pu
import matplotlib.pyplot as plt
opts={}
opts['Clevels']=np.linspace(-2,2,10)
opts['cmap']=plt.cm.get_cmap('RdBu')
fig=pu.plot_map(vLON,vLAT,venB['NET'],titleC='EB residual',titleL=cf.cid,contourf=False,
Clabel='[W m-2]',**opts)
fout='map_eb_res_%s_%s_%s_%i_%i.png'%(cf.cid,cf.cver,cf.cdomain,cf.ystart,cf.yend)
print "Saving:",fout
fig[0].savefig(fout,bbox_inches="tight",dpi=200)
plt.close(fig[0])
return msg
def check_wb(cf, ystart, yend, cdomain, cid, cver, msg=None):
"""
Water balance check
Parameters:
----------
Returns:
-------
"""
rystart = ystart
ryend = yend
dstart = dt.datetime(cf.ystart, 1, 1).toordinal()
dend = dt.datetime(cf.yend, 12, 31).toordinal()
xTIME = num2date(np.arange(0, dend - dstart + 1, 1),
"days since %4i-01-01 00:00:00" % cf.ystart)
tinD = [
np.nonzero(xTIME == dt.datetime(rystart, 1, 1))[0][0],
np.nonzero(xTIME == dt.datetime(ryend, 12, 31))[0][0]
]
ndays = tinD[1] - tinD[0] + 1
#print tinD
#print ndays
vLAT, vLON = e2oU.default_latlon(cf.cdomain)
nlat = len(vLAT)
nlon = len(vLON)
grid_area = e2oU.load_grid_area(fgarea)
cvarsWB = ['Precip', 'Runoff', 'Evap', 'Stor', 'NET']
if msg is None:
msg = e2oU.init_msg()
venB = {}
globM = {} ## global mean values for information only !
for cvar in cvarsWB:
print 'WB, loading:', cvar
if cvar == "NET":
venB[cvar] = np.zeros((nlat, nlon))
for cc in cvarsWB[:-1]:
venB[cvar] = venB[cvar] + venB[cc]
elif cvar == "Stor":
venB[cvar] = np.zeros((nlat, nlon))
for svar in ['TotMoist', 'SWE', 'CanopInt', 'SurfStor']:
print 'WB, loading:', svar
cf = cf.attr2fpath(cfreq='day',
cvar=svar,
cdomain=cdomain,
cid=cid,
cver=cver)
xdata, xtime = e2oU.load_nc_var(cf.fpath, cf.cvar, tinD=tinD)
if xdata is None:
xdata = np.zeros((2, nlat, nlon))
msg['Wmsg'].append(
"WB: Could not find variable: '%s', setting to zero!'"
% (svar))
venB[cvar] = venB[cvar] + -1 * (xdata[1, :, :] -
xdata[0, :, :]) / (ndays)
else:
cf = cf.attr2fpath(cfreq='mon',
cvar=cvar,
cdomain=cdomain,
cid=cid,
cver=cver)
xdata, xtime = e2oU.load_nc_var(
cf.fpath,
cf.cvar,
dstart=dt.datetime(rystart, 1, 1, 0, 0, 0),
dend=dt.datetime(ryend, 12, 31, 23, 59, 59))
#if cvar == 'Precip': print xtime
if xdata is None:
venB[cvar] = np.zeros((nlat, nlon))
msg['Wmsg'].append(
"WB: Could not find variable: '%s', setting to zero!'" %
(cvar))
else:
venB[cvar] = np.mean(xdata, 0)
venB[cvar] = venB[cvar] * 86400.
globM[cvar] = compute_area_mean(venB[cvar], grid_area)
for cvar in cvarsWB:
msg['Dmsg'].append(
"WB: Global mean of %s %f (mm day-1) with %s/%s %f" %
(cvar, globM[cvar], cvar, cvarsWB[0],
globM[cvar] / globM[cvarsWB[0]]))
if cvar == "NET":
msg['Dmsg'].append(
'WB:' +
" variable %s with gpmin %e, gpmax %e fldmean %e #gp>thr %i" %
(cvar, np.min(venB[cvar]), np.max(venB[cvar]), globM[cvar],
np.sum(np.abs(venB[cvar]) > 5e-6 * 86400.)))
if np.sum(np.abs(venB[cvar]) > 5e-6 * 86400.) > 0:
msg['Wmsg'].append(
'WB:' +
" variable %s with gpmin %e, gpmax %e fldmean %e #gp>thr %i"
%
(cvar, np.min(venB[cvar]), np.max(venB[cvar]), globM[cvar],
np.sum(np.abs(venB[cvar]) > 5e-6 * 86400.)))
if LPLOT:
#produce map with WB residuals
#requires plot_utils from pyutils
if (cvar == "NET"):
from pyutils import plot_utils as pu
import matplotlib.pyplot as plt
opts = {}
opts['Clevels'] = np.arange(-2, 2.5, 0.5) * 86400 * 5e-6
opts['cmap'] = plt.cm.get_cmap('RdBu')
fig = pu.plot_map(vLON,
vLAT,
venB[cvar],
titleC='WB residual',
titleL=cf.cid,
contourf=False,
titleR="%i #gp" %
np.sum(np.abs(venB[cvar]) > 5e-6 * 86400.),
Clabel='[mm/day]',
**opts)
fout = 'map_wb_res_%s_%s_%s_%i_%i.png' % (
cf.cid, cf.cver, cf.cdomain, cf.ystart, cf.yend)
print "Saving:", fout
fig[0].savefig(fout, bbox_inches="tight", dpi=200)
plt.close(fig[0])
return msg
def check_eb(cf, ystart, yend, cdomain, cid, cver, msg=None):
"""
Energy balance check
Parameters:
----------
Returns:
-------
"""
rystart = ystart
ryend = yend
vLAT, vLON = e2oU.default_latlon(cf.cdomain)
nlat = len(vLAT)
nlon = len(vLON)
grid_area = e2oU.load_grid_area(fgarea)
cvarsEB = ['SWnet', 'LWnet', 'Qh', 'Qle', 'Qsm', 'NET']
if msg is None:
msg = e2oU.init_msg()
venB = {}
globM = {} ## global mean values for information only !
for cvar in cvarsEB:
cf = cf.attr2fpath(cfreq='mon',
cvar=cvar,
cdomain=cdomain,
cid=cid,
cver=cver)
print 'EB, loading:', cvar
if cvar == "NET":
venB[cvar] = np.zeros((nlat, nlon))
for cc in cvarsEB[:-1]:
venB[cvar] = venB[cvar] + venB[cc]
else:
xdata, xtime = e2oU.load_nc_var(cf.fpath,
cf.cvar,
dstart=dt.datetime(rystart, 1, 1),
dend=dt.datetime(ryend, 12, 31))
if xdata is None:
venB[cvar] = np.zeros((nlat, nlon))
msg['Wmsg'].append(
"EB: Could not find variable: '%s', setting to zero!'" %
(cvar))
else:
venB[cvar] = np.mean(xdata, 0)
if cvar == "Qsm":
venB[cvar] = venB[cvar] * 3.34e5 * -1.
globM[cvar] = compute_area_mean(venB[cvar], grid_area)
for cvar in cvarsEB:
msg['Dmsg'].append("EB: Global mean of %s %f (W m-2) with %s/%s %f" %
(cvar, globM[cvar], cvar, cvarsEB[0],
globM[cvar] / globM[cvarsEB[0]]))
if LPLOT:
#produce map with EB residuals
#requires plot_utils from pyutils # ask Edutra
from pyutils import plot_utils as pu
import matplotlib.pyplot as plt
opts = {}
opts['Clevels'] = np.linspace(-2, 2, 10)
opts['cmap'] = plt.cm.get_cmap('RdBu')
fig = pu.plot_map(vLON,
vLAT,
venB['NET'],
titleC='EB residual',
titleL=cf.cid,
contourf=False,
Clabel='[W m-2]',
**opts)
fout = 'map_eb_res_%s_%s_%s_%i_%i.png' % (cf.cid, cf.cver, cf.cdomain,
cf.ystart, cf.yend)
print "Saving:", fout
fig[0].savefig(fout, bbox_inches="tight", dpi=200)
plt.close(fig[0])
return msg
def check_file_coords(finput, msg=None):
"""
Check file coordinates
Parameters:
----------
finput : class(e2oU.fname) :
msg : message (optional)
Returns:
-------
message
"""
if msg is None:
msg = e2oU.init_msg()
emsg = 0
nc = Dataset(finput.fpath, 'r')
vLAT, vLON = e2oU.default_latlon(finput.cdomain)
if finput.cfreq == "mon":
nyears = finput.yend - finput.ystart + 1
vTIME = num2date(np.arange(5, 365.25 * nyears, 365.25 / 12),
"days since %4i-01-01 00:00:00" % (finput.ystart))
elif finput.cfreq == "day":
dstart = dt.datetime(finput.ystart, 1, 1).toordinal()
dend = dt.datetime(finput.yend, 12, 31).toordinal()
vTIME = num2date(np.arange(0, dend - dstart + 1, 1),
"days since %4i-01-01 00:00:00" % finput.ystart)
elif finput.cfreq == "fix":
pass
else:
print "This frequency test is not implemented yet", finput.cfreq
sys.exit(-1)
for cvtime in nc.variables.keys():
if cvtime in ['time', 'time_counter']: break
if finput.cfreq != "fix":
vYR, vMON, vDAY = date2yrmonday(vTIME)
if tcheck:
fTIME = num2date(nc.variables[cvtime][:],
nc.variables[cvtime].units)
else:
fTIME = vTIME
fTIME[0] = num2date(nc.variables[cvtime][0],
nc.variables[cvtime].units)
fTIME[-1] = num2date(nc.variables[cvtime][-1],
nc.variables[cvtime].units)
fYR, fMON, fDAY = date2yrmonday(fTIME)
fLAT = nc.variables['lat'][:]
fLON = nc.variables['lon'][:]
try:
ddlon = np.abs(np.sum(vLON - fLON))
ddlat = np.abs(np.sum(vLAT - fLAT))
if ddlon > np.finfo(np.float32).eps or ddlat > np.finfo(
np.float32).eps:
msg['Emsg'].append(finput.fname +
' Lat or Lon arrays in file is not correct')
emsg = emsg + 1
except:
msg['Emsg'].append(
finput.fname +
' Some problem checking lat/lon arrays: check standard output')
traceback.print_exc()
emsg = emsg + 1
if finput.cfreq != "fix":
try:
ddyear = np.abs(np.sum(vYR - fYR))
ddmon = np.abs(np.sum(vMON - fMON))
ddday = np.abs(np.sum(vDAY - fDAY))
if ddyear > np.finfo(np.float32).eps or ddmon > np.finfo(
np.float32).eps:
msg['Emsg'].append(
finput.fname +
' time year/mon arrays in file is not correct')
emsg = emsg + 1
if ddday > np.finfo(np.float32).eps and finput.cfreq == "day":
msg['Emsg'].append(finput.fname +
' time day arrays in file is not correct')
emsg = emsg + 1
except:
msg['Emsg'].append(
finput.fname +
' Some problem checking time arrays: check standard output')
traceback.print_exc()
emsg = emsg + 1
if emsg == 0:
msg['Smsg'].append(finput.fname + ' file coords check OK')
nc.close()
## defaults
ystart = args.ystart #1979 start year
yend = args.yend #2012 end year
cdomain = args.cdomain #"glob30" simulations domain
cid = args.cid #"ecmwf" institution id
cver = args.cver #"wrr1" simulations version id
tcheck = args.tcheck
LPLOT = args.LPLOT
LSPLIT = args.LSPLIT
CHECK_WATER_ENERGY = args.CHECK_WATER_ENERGY
print args
##=======================================================
## 1. File consistency checks: we loop on all possible variables:
msg = e2oU.init_msg() # intialize message dictionary
cf = e2oU.fname() # initialize file name class
# loop on all possible variables / frequencies
for cvar in e2oU.validD['cvar']:
for cfreq in ['day', 'mon', 'fix']:
if cfreq == 'fix' and cvar not in e2oU.validD['cvar_fix']:
continue
if cvar in e2oU.validD['cvar_fix'] and cfreq != 'fix':
continue
if LSPLIT and (cfreq == 'day'):
ddyears = zip([1980, 1990, 2000], [1989, 1999, 2014])
else:
ddyears = zip([
ystart,
