def arg_parse(varname=None,domain=None,input_file=None,list_var_names=False):
wrf_dom = 3
if input_file is not None:
ncfile = Nio.open_file(input_file,format='grib')
else:
ncfile = Nio.open_file(find_inputfile(wrf_dom),format='')
if list_var_names:
print_var_summary(ncfile)
if len(varname) != 0:
for var in varname:
print_var_summary(ncfile,varname=var)
return
def split(self, num):
if type(num) is not int:
raise TypeError, "number to split must be an integer"
if self._open_file is False:
if self.trange is not None:
time_len = self.trange[1]-self.trange[0]
tstart = self.trange[0]
else:
f = Nio.open_file(self.pathname)
time_len = f.dimensions['time']
f.close()
tstart = 0
chunk_size = time_len//num
remainder = time_len%num
tranges = [(tstart+i*chunk_size+remainder*i//num, \
tstart+(i+1)*chunk_size+remainder*(i+1)//num) \
for i in range(num)]
return [RectGrid(self.pathname, self.varname, trange=it) \
for it in tranges]
else:
raise RuntimeError, "RectGrid must not be initialized before running split()"
def do_setup(filename):
if os.path.exists(filename): os.remove(filename)
f = Nio.open_file(filename, 'c')
(nx, ny, nz, nt,ns) = (21, 21, 12, 10,1)
(dx, dy, dz, dt) = (1000., 1000., 400., 3600.)
f.create_dimension('xc', nx)
f.create_dimension('yc', ny)
f.create_dimension('zc', nz)
f.create_dimension('time', nt)
f.create_dimension('single', ns)
f.Conventions = 'CF-1.0'
f.source = 'ARPS'
var = f.create_variable('xc', 'f', ('xc',))
setattr(var, 'axis', 'X')
var = f.create_variable('yc', 'f', ('yc',))
setattr(var, 'axis', 'Y')
var = f.create_variable('zc', 'f', ('zc',))
setattr(var, 'axis', 'Z')
var = f.create_variable('time', 'f', ('time',))
setattr(var, 'axis', 'T')
setattr(var, 'units', 'seconds since 2007-03-21 06:00:00')
var = f.create_variable('PT', 'f', ('time', 'zc', 'yc', 'xc'))
var = f.create_variable('PTS', 'f', ('single','time', 'zc', 'yc', 'xc'))
var = f.create_variable('ZP', 'f', ('zc', 'yc', 'xc'))
var = f.create_variable('TOPO', 'f', ('yc', 'xc'))
var = f.create_variable('lon', 'f', ('yc','xc'))
var = f.create_variable('lat', 'f', ('yc','xc'))
xc = N.arange(nx, dtype='float32')*dx
yc = N.arange(ny, dtype='float32')*dy
zc = N.arange(nz, dtype='float32')*dz
f.variables['xc'][:] = xc
f.variables['yc'][:] = yc
f.variables['zc'][:] = zc
f.variables['time'][:] = N.arange(nt, dtype='float32')*dt
a = N.arange(nt*nz*ny*nx,dtype = 'float32')
#a = N.array(N.random.randn(nt,nz,ny,nx), dtype='float32')
a = a.reshape(nt,nz,ny,nx)
#print a.shape
mask = N.zeros(a.shape,N.bool_)
mask[:,3,:,:] = 1
# tests adding a fill value
am = ma.array(a,mask=mask)
f.variables['PT'][:] = am[:]
f.variables['PTS'][:] = am[:]
#if verbose: print f.variables['PT']
H = 5000.
topo = 1000*N.cos(2*N.pi*(xc-10000.)/20000.)+1000.
zp = zc[:,N.newaxis]*(1-topo[N.newaxis,:]/H) + topo[N.newaxis,:]
topof = N.zeros((ny, nx), dtype='float32')
topof[:,:] = topo[N.newaxis,:]
zpf = N.zeros((nz,ny,nx), dtype='float32')
zpf[:] = zp[:,N.newaxis,:]
f.variables['ZP'][:] = zpf
f.variables['TOPO'][:] = topof
f.variables['lon'][:] = N.cos(0.1)*xc[N.newaxis,:] - N.sin(0.1)*yc[:,N.newaxis]
f.variables['lat'][:] = N.sin(0.1)*xc[N.newaxis,:] + N.cos(0.1)*yc[:,N.newaxis]
f.close()
def setup_nc_file(filename,lat,lon,nt,ny,nx):
"""setup a netcdf file for output"""
ncfile=Nio.open_file(filename,mode="w",format="nc")
ncfile.create_dimension("time",nt)
ncfile.create_dimension("lat",ny)
ncfile.create_dimension("lon",nx)
timev=ncfile.create_variable("time","f",("time",))
times=np.arange(nt)/48.0
timev[:]=times.astype("f")
timev.__setattr__("longname","Time")
timev.__setattr__("units","Days from "+start_date)
latv=ncfile.create_variable("lat","f",("lat","lon"))
latv[:]=lat.astype("f")
latv.__setattr__("longname","Latitude")
latv.__setattr__("units","degrees")
lonv=ncfile.create_variable("lon","f",("lat","lon"))
lonv[:]=lon.astype("f")
lonv.__setattr__("longname","Longitude")
lonv.__setattr__("units","degrees")
return ncfile
def do_setup_nocrd(filename):
if os.path.exists(filename): os.remove(filename)
f = Nio.open_file(filename, 'c')
(nx, ny, nz, nt) = (20, 25, 5, 10)
(dx, dy, dz, dt) = (1000., 1000., 800., 3600.)
f.create_dimension('xc', nx)
f.create_dimension('yc', ny)
f.create_dimension('zc', nz)
f.create_dimension('time', nt)
f.Conventions = 'CF-1.0'
f.source = 'ARPS'
var = f.create_variable('time', 'f', ('time',))
setattr(var, 'axis', 'T')
setattr(var, 'units', 'seconds since 2007-03-21 06:00:00')
var = f.create_variable('PT', 'f', ('time', 'zc', 'yc', 'xc'))
var = f.create_variable('ZP', 'f', ('zc', 'yc', 'xc'))
xc = N.arange(nx, dtype='float32')*dx
yc = N.arange(ny, dtype='float32')*dy
f.variables['time'][:] = N.arange(nt, dtype='float32')*dt
#a = N.array(N.random.randn(nt,nz,ny,nx), dtype='float32')
a = N.arange(nt*nz*ny*nx,dtype = 'float32')
a = a.reshape(nt,nz,ny,nx)
f.variables['PT'][:] = a
a = N.zeros((nz,ny,nx))
a[:] = N.arange(nz)[:,N.newaxis,N.newaxis]
f.variables['ZP'][:] = N.array(a, dtype='float32')
f.close()
def main():
base_path = "/caps2/tsupinie/1kmf-control/"
temp = goshen_1km_temporal(start=14400, end=14400)
grid = goshen_1km_grid()
n_ens_members = 40
np.seterr(all='ignore')
ens = loadEnsemble(base_path, [ 11 ], temp.getTimes(), ([ 'pt', 'p' ], computeDensity))
ens = ens[0, 0]
zs = decompressVariable(nio.open_file("%s/ena001.hdfgrdbas" % base_path, mode='r', format='hdf').variables['zp'])
xs, ys = grid.getXY()
xs = xs[np.newaxis, ...].repeat(zs.shape[0], axis=0)
ys = ys[np.newaxis, ...].repeat(zs.shape[0], axis=0)
eff_buoy = effectiveBuoyancy(ens, (zs, ys, xs), plane={'z':10})
print eff_buoy
pylab.figure()
pylab.contourf(xs[0], ys[0], eff_buoy[0], cmap=matplotlib.cm.get_cmap('RdBu_r'))
pylab.colorbar()
grid.drawPolitical()
pylab.suptitle("Effective Buoyancy")
pylab.savefig("eff_buoy.png")
pylab.close()
return
def main():
files = sorted(glob.glob("3kmgoshen.hdf[0123456789]?????"))
trajectory_points = [ [ (1, 0, 0) ],
[ (10, 0, 0) ],
[ (20, 0, 0) ],
[ (30, 0, 0) ],
[ (40, 0, 0) ] ]
grid_spacing = 1000
time_spacing = 300
for file_name in files:
hdf = nio.open_file(file_name, mode='r', format='hdf')
u_grid = hdf.variables['u'][:]
v_grid = hdf.variables['v'][:]
hdf.close()
for trajectory in trajectory_points:
last_z, last_x, last_y = trajectory[-1]
point_u = interpolate(u_grid, last_x, last_y, last_z)
point_v = interpolate(v_grid, last_x, last_y, last_z)
new_x = last_x + time_spacing * point_u / grid_spacing
new_y = last_y + time_spacing * point_v / grid_spacing
if new_x > u_grid.shape[1] - 1 or new_x < 0 or new_y > u_grid.shape[2] - 1 or new_y < 0:
print "Parcel out of bounds ..."
else:
trajectory.append((last_z, new_x, new_y))
print trajectory_points
return
def makeNioFile(self, variableName):
filename = '/tmp/good_%s.nc' % variableName
f = nio.open_file(filename, 'w')
f.create_dimension('test_dimension', 1)
f.create_variable(variableName,'l',('test_dimension',))
f.close()
return filename
def write_2d_output(fname,lat,lon,p):
Ny,Nx=p.shape
ncf=Nio.open_file(fname,mode="w",format="nc")
ncf.create_dimension("lat",Ny)
ncf.create_dimension("lon",Nx)
varname="XLAT"
ncf.create_variable(varname,'f',('lat','lon'))
ncf.variables[varname][:]=lat.astype('f')
ncf.variables[varname].units="degrees"
ncf.variables[varname].description="Latitude"
varname="XLONG"
ncf.create_variable(varname,'f',('lat','lon'))
ncf.variables[varname][:]=lon.astype('f')
ncf.variables[varname].units="degrees"
ncf.variables[varname].description="Longitude"
varname="precipitation"
ncf.create_variable(varname,'f',('lat','lon'))
ncf.variables[varname][:]=p.astype('f')
ncf.variables[varname].units="kg/m^2/s"
ncf.variables[varname].description="precipitation rate"
ncf.close()
def read_nc(filename,var="data",proj=None,returnNCvar=False):
'''read a netCDF file and return the specified variable
output is a structure :
data:raw data as an array
proj:string representation of the projection information
atts:data attribute dictionary (if any)
if (returnNCvar==True) then the Nio file is note closed and the Nio
representation of the variable is returned instead of being read into
memory immediately.
'''
d=Nio.open_file(filename, mode='r',format="nc")
outputdata=None
if var != None:
data=d.variables[var]
attributes=d.variables[var].__dict__
if returnNCvar:
outputdata=data
else:
outputdata=data[:]
outputproj=None
if proj!=None:
projection=d.variables[proj]
outputproj=str(projection)
if returnNCvar:
return Bunch(data=outputdata,proj=outputproj,ncfile=d,atts=attributes)
d.close()
return Bunch(data=outputdata,proj=outputproj,atts=attributes)
def findHeights(grdbas, bounds):
hdf = nio.open_file(grdbas, mode='r', format='hdf')
bounds_x, bounds_y = bounds
column_x = (bounds_x.start + bounds_x.stop) / 2
column_y = (bounds_y.start + bounds_y.stop) / 2
return hdf.variables['zp'][:, column_y, column_x]
def do_setup(filename):
if os.path.exists(filename):
os.remove(filename)
f = Nio.open_file(filename, "c")
(nx, ny, nz, nt) = (21, 21, 12, 10)
(dx, dy, dz, dt) = (1000.0, 1000.0, 400.0, 3600.0)
f.create_dimension("xc", nx)
f.create_dimension("yc", ny)
f.create_dimension("zc", nz)
f.create_dimension("time", nt)
f.Conventions = "CF-1.0"
f.source = "ARPS"
var = f.create_variable("xc", "f", ("xc",))
setattr(var, "axis", "X")
var = f.create_variable("yc", "f", ("yc",))
setattr(var, "axis", "Y")
var = f.create_variable("zc", "f", ("zc",))
setattr(var, "axis", "Z")
var = f.create_variable("time", "f", ("time",))
setattr(var, "axis", "T")
setattr(var, "units", "seconds since 2007-03-21 06:00:00")
var = f.create_variable("PT", "f", ("time", "zc", "yc", "xc"))
var = f.create_variable("ZP", "f", ("zc", "yc", "xc"))
var = f.create_variable("TOPO", "f", ("yc", "xc"))
var = f.create_variable("lon", "f", ("yc", "xc"))
var = f.create_variable("lat", "f", ("yc", "xc"))
xc = N.arange(nx, dtype="float32") * dx
yc = N.arange(ny, dtype="float32") * dy
zc = N.arange(nz, dtype="float32") * dz
f.variables["xc"][:] = xc
f.variables["yc"][:] = yc
f.variables["zc"][:] = zc
f.variables["time"][:] = N.arange(nt, dtype="float32") * dt
a = N.arange(nt * nz * ny * nx, dtype="float32")
# a = N.array(N.random.randn(nt,nz,ny,nx), dtype='float32')
a = a.reshape(nt, nz, ny, nx)
print a.shape
mask = N.zeros(a.shape, N.bool_)
mask[:, 3, :, :] = 1
# tests adding a fill value
am = ma.array(a, mask=mask)
f.variables["PT"][:] = am[:]
# if verbose: print f.variables['PT']
H = 5000.0
topo = 1000 * N.cos(2 * N.pi * (xc - 10000.0) / 20000.0) + 1000.0
zp = zc[:, N.newaxis] * (1 - topo[N.newaxis, :] / H) + topo[N.newaxis, :]
topof = N.zeros((ny, nx), dtype="float32")
topof[:, :] = topo[N.newaxis, :]
zpf = N.zeros((nz, ny, nx), dtype="float32")
zpf[:] = zp[:, N.newaxis, :]
f.variables["ZP"][:] = zpf
f.variables["TOPO"][:] = topof
f.variables["lon"][:] = N.cos(0.1) * xc[N.newaxis, :] - N.sin(0.1) * yc[:, N.newaxis]
f.variables["lat"][:] = N.sin(0.1) * xc[N.newaxis, :] + N.cos(0.1) * yc[:, N.newaxis]
f.close()
def merge(ts):
"""
Process an hour's worth of stage4 data into the hourly RE
"""
# Load up the 12z 24h total, this is what we base our deltas on
fp = "/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.24h.grib" % (
ts.strftime("%Y/%m/%d"), ts.strftime("%Y%m%d%H") )
grib = Nio.open_file(fp, 'r')
# Rough subsample, since the whole enchillata is too much
lats = numpy.ravel( grib.variables["g5_lat_0"][200:-100:5,300:900:5] )
lons = numpy.ravel( grib.variables["g5_lon_1"][200:-100:5,300:900:5] )
vals = numpy.ravel( grib.variables["A_PCP_GDS5_SFC_acc24h"][200:-100:5,300:900:5] )
res = Ngl.natgrid(lons, lats, vals, iemre.XAXIS, iemre.YAXIS)
stage4 = res.transpose()
# Prevent Large numbers, negative numbers
stage4 = numpy.where( stage4 < 10000., stage4, 0.)
stage4 = numpy.where( stage4 < 0., 0., stage4)
# Open up our RE file
nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,),'a')
ts0 = ts + mx.DateTime.RelativeDateTime(days=-1)
jan1 = mx.DateTime.DateTime(ts.year, 1, 1, 0, 0)
offset0 = int(( ts0 - jan1).hours)
offset1 = int(( ts - jan1).hours)
if offset0 < 0:
offset0 = 0
iemre2 = numpy.sum(nc.variables["p01m"][offset0:offset1,:,:], axis=0)
iemre2 = numpy.where( iemre2 > 0., iemre2, 0.00024)
iemre2 = numpy.where( iemre2 < 10000., iemre2, 0.00024)
print "Stage IV 24h [Avg %5.2f Max %5.2f] IEMRE Hourly [Avg %5.2f Max: %5.2f]" % (
numpy.average(stage4), numpy.max(stage4),
numpy.average(iemre2), numpy.max(iemre2) )
multiplier = stage4 / iemre2
print "Multiplier MIN: %5.2f AVG: %5.2f MAX: %5.2f" % (
numpy.min(multiplier), numpy.average(multiplier),numpy.max(multiplier))
for offset in range(offset0, offset1):
data = nc.variables["p01m"][offset,:,:]
# Keep data within reason
data = numpy.where( data > 10000., 0., data)
adjust = numpy.where( data > 0, data, 0.00001) * multiplier
adjust = numpy.where( adjust > 250.0, 0, adjust)
nc.variables["p01m"][offset,:,:] = numpy.where( adjust < 0.01, 0, adjust)
ts = jan1 + mx.DateTime.RelativeDateTime(hours=offset)
print "%s IEMRE %5.2f %5.2f Adjusted %5.2f %5.2f" % (ts.strftime("%Y-%m-%d %H"),
numpy.average(data), numpy.max(data),
numpy.average(nc.variables["p01m"][offset]),
numpy.max(nc.variables["p01m"][offset]))
nc.sync()
iemre2 = numpy.sum(nc.variables["p01m"][offset0:offset1,:,:], axis=0)
print "Stage IV 24h [Avg %5.2f Max %5.2f] IEMRE Hourly [Avg %5.2f Max: %5.2f]" % (
numpy.average(stage4), numpy.max(stage4),
numpy.average(iemre2), numpy.max(iemre2) )
nc.close()
def main():
files = glob("1kmgoshen/1kmgoshen.hdf0*")
hdf_grdbas = nio.open_file("1kmgoshen/1kmgoshen.hdfgrdbas", mode='r', format='hdf')
# topo, topo_lats, topo_lons = load_topo("e10g", (6000, 10800), ((0., 50.), (-180., -90.)), ((36., 46.), (-114., -101.)))
for file in files:
time_sec = file[-6:]
hdf_data = nio.open_file(file, mode='r', format='hdf')
hydrostatic, mass_cons, thermal_wind_u, thermal_wind_v = computeBalances(hdf_data, hdf_grdbas)
plot_map(hydrostatic[1], (1000, 1000), "xy", r"Hydrostatic Imbalance (Pa m$^{-1}$)", "hydrostatic_t%s.png" % time_sec) #, topo=(topo, topo_lats, topo_lons))
plot_map(mass_cons[1], (1000, 1000), "xy", r"Mass Conservation Imbalance (m s$^{-2}$)", "mass_cons_t%s.png" % time_sec) #, topo=(topo, topo_lats, topo_lons))
plot_map(thermal_wind_u[1], (1000, 1000), "xy", r"Thermal Wind $u$ Imbalance (m s$^{-2}$)", "thermal_wind_u_t%s.png" % time_sec) #, topo=(topo, topo_lats, topo_lons))
plot_map(thermal_wind_v[1], (1000, 1000), "xy", r"Thermal Wind $v$ Imbalance (m s$^{-2}$)", "thermal_wind_v_t%s.png" % time_sec) #, topo=(topo, topo_lats, topo_lons))
hdf_data.close()
hdf_grdbas.close()
return
def main():
print "Testing"
import Nio as nio
data = nio.open_file('/home/wrfuser/hootpy/data/wrfout_d01_PLEV.nc')
# print mslp(data.variables['PSFC'][:],data.variables['HGT'][:],data.variables['T2'][:],data.variables['Q2'][:])
dewp = dewpoint(data.variables['T'][0,:],data.variables['QVAPOR'][0,:],data.variables['P'][:]*100)
print dewp.max()
print dewp.min()
print dewp.mean()
def save_graham_data_to_netcdf(
netcdf_file_path, resolution_min=5, shape=(4320, 2160), lower_left_point=GeoPoint(-180.0, -90.0)
):
opt = "c"
if os.path.isfile(netcdf_file_path):
os.remove(netcdf_file_path)
file = Nio.open_file(netcdf_file_path, opt)
save_graham_data_to_obj(file, resolution_min=resolution_min, the_shape=shape, lower_left_point=lower_left_point)
file.close()
def getAxes(base_path, agl=True, z_coord_type=""):
grdbas_file = _buildEnsGrdbas(0)
hdf_grdbas = nio.open_file("%s/%s" % (base_path, grdbas_file), mode='r', format='hdf')
axes = dict([ (ax[:1], decompressVariable(hdf_grdbas.variables[ax])) for ax in ['x', 'y', "zp%s" % z_coord_type] ])
if agl:
axes['z'], axes['z_MSL'] = _makeZCoordsAGL(axes['z'])
else:
axes['z_AGL'], axes['z'] = _makeZCoordsAGL(axes['z'])
return axes
def main():
_epoch_time = datetime(1970, 1, 1, 0, 0, 0)
_initial_time = datetime(2009, 6, 5, 18, 0, 0) - _epoch_time
_initial_time = (_initial_time.microseconds + (_initial_time.seconds + _initial_time.days * 24 * 3600) * 1e6) / 1e6
_target_times = [ 1800, 3600, 5400, 7200, 9000, 10800, 11100, 11400, 11700, 12000, 12300, 12600, 12900, 13200, 13500, 13800, 14100, 14400,
14700, 15000, 15300, 15600, 15900, 16200, 16500, 16800, 17100, 17400, 17700, 18000 ]
inflow_wd_lbound, inflow_wd_ubound = (100, 240)
# bounds = (0, slice(90, 210), slice(40, 160))
# bounds = (0, slice(100, 180), slice(90, 170))
bounds = (0, slice(115, 140), slice(120, 145))
rev_bounds = [ 0 ]
rev_bounds.extend(bounds[2:0:-1])
rev_bounds = tuple(rev_bounds)
refl_base = "hdf/KCYS/1km/goshen.hdfrefl2d"
refl_times = np.array([ int(f[-6:]) for f in glob.glob("%s??????" % refl_base) ])
refl_keep_times = []
refl_data = {}
for tt in _target_times:
idx = np.argmin(np.abs(refl_times - tt))
if refl_times[idx] > tt and idx > 0:
idx -= 1
file_name = "%s%06d" % (refl_base, refl_times[idx])
hdf = nio.open_file(file_name, mode='r', format='hdf')
refl_keep_times.append(refl_times[idx])
refl_data[refl_times[idx]] = hdf.variables['refl2d'][rev_bounds]
_proj = setupMapProjection(goshen_1km_proj, goshen_1km_gs, bounds=bounds[1:])
# _proj['resolution'] = 'h'
map = Basemap(**_proj)
ttu_sticknet_obs = cPickle.load(open("ttu_sticknet.pkl", 'r'))
psu_straka_obs = cPickle.load(open("psu_straka_mesonet.pkl", 'r'))
all_obs = loadObs(['ttu_sticknet.pkl', 'psu_straka_mesonet.pkl'], [ _epoch_time + timedelta(seconds=(_initial_time + t)) for t in _target_times ], map, (goshen_1km_proj['width'], goshen_1km_proj['height']), round_time=True)
print all_obs
# partitioned_obs = gatherObservations(all_obs, [ _initial_time + t for t in _target_times ])
for time, refl_time in zip([ _initial_time + t for t in _target_times], refl_keep_times):
time_str = (_epoch_time + timedelta(seconds=time)).strftime("%d %B %Y %H%M UTC")
plot_obs = all_obs[np.where(all_obs['time'] == time)]
inflow_idxs = np.where((plot_obs['wind_dir'] >= inflow_wd_lbound) & (plot_obs['wind_dir'] <= inflow_wd_ubound))[0]
outflow_idxs = np.array([ idx for idx in range(plot_obs['id'].shape[0]) if idx not in inflow_idxs ])
title = "All MM observations at %s" % time_str
file_name = "mm_obs_%06d.png" % (time - _initial_time)
plotObservations(plot_obs, map, title, file_name, refl=refl_data[refl_time])
return
def make5VariableNioFile(self):
filename = '/tmp/5_variables.nc'
f = nio.open_file(filename, 'w')
f.create_dimension('dimension_one', 1)
f.create_variable('one', 'l', ('dimension_one',))
f.create_variable('two', 'l', ('dimension_one',))
f.create_variable('three', 'l', ('dimension_one',))
f.create_variable('four', 'l', ('dimension_one',))
f.create_variable('five', 'l', ('dimension_one',))
f.close()
return filename
def load_reflectivity_vars(file_name, vars, ens_member):
hdf = nio.open_file(file_name, mode='r', format='hdf')
vars['pt'][ens_member] = hdf.variables['pt'][12]
vars['p'][ens_member] = hdf.variables['p'][12]
vars['qr'][ens_member] = np.maximum(hdf.variables['qr'][12], np.zeros(hdf.variables['qr'][12].shape))
vars['qs'][ens_member] = np.maximum(hdf.variables['qs'][12], np.zeros(hdf.variables['qs'][12].shape))
vars['qh'][ens_member] = np.maximum(hdf.variables['qh'][12], np.zeros(hdf.variables['qh'][12].shape))
hdf.close()
return
def loadGrdbas(grdbas_file, agl):
grdbas = nio.open_file(grdbas_file, mode='r', format='hdf')
z_coords = decompressVariable(grdbas.variables['zp'])
x_coords = decompressVariable(grdbas.variables['x'])
y_coords = decompressVariable(grdbas.variables['y'])
if agl:
z_coords = _makeZCoordsAGL(z_coords)
return z_coords, y_coords, x_coords
def load_data(self):
"""
Loads data from MRMS GRIB2 files and handles compression duties if files are compressed.
"""
data = []
loaded_dates = []
loaded_indices = []
for t, timestamp in enumerate(self.all_dates):
date_str = timestamp.date().strftime("%Y%m%d")
full_path = self.path_start + date_str + "/"
if self.variable in os.listdir(full_path):
full_path += self.variable + "/"
data_files = sorted(os.listdir(full_path))
file_dates = pd.to_datetime([d.split("_")[-1][0:13] for d in data_files])
if timestamp in file_dates:
data_file = data_files[np.where(timestamp==file_dates)[0][0]]
print(full_path + data_file)
if data_file[-2:] == "gz":
subprocess.call(["gunzip", full_path + data_file])
file_obj = Nio.open_file(full_path + data_file[:-3])
else:
file_obj = Nio.open_file(full_path + data_file)
var_name = sorted(file_obj.variables.keys())[0]
data.append(file_obj.variables[var_name][:])
if self.lon is None:
self.lon = file_obj.variables["lon_0"][:]
# Translates longitude values from 0:360 to -180:180
if np.count_nonzero(self.lon > 180) > 0:
self.lon -= 360
self.lat = file_obj.variables["lat_0"][:]
file_obj.close()
if data_file[-2:] == "gz":
subprocess.call(["gzip", full_path + data_file[:-3]])
else:
subprocess.call(["gzip", full_path + data_file])
loaded_dates.append(timestamp)
loaded_indices.append(t)
if len(loaded_dates) > 0:
self.loaded_dates = pd.DatetimeIndex(loaded_dates)
self.data = np.ones((self.all_dates.shape[0], data[0].shape[0], data[0].shape[1])) * -9999
self.data[loaded_indices] = np.array(data)
def __init__(self, filename,nx,ny,nz=None,var=None,dtype='f'):
history = 'Created : ' + time.ctime() + '\nby:'+os.environ['USER']+" using NC_writer Class"
self.NCfile=Nio.open_file(filename,mode='w',format="nc",history=history)
self.NCfile.create_dimension('time', 0)
self.NCfile.create_dimension('lat', ny)
self.ny=ny
self.NCfile.create_dimension('lon', nx)
self.nx=nx
if nz:
self.NCfile.create_dimension('level',nz)
self.nz=nz
self.NCfile.create_variable('time','l',('time',))
if var: self.addVar(var,dtype=dtype)
def doit(ts):
"""
Generate hourly plot of stage4 data
"""
gmtnow = mx.DateTime.gmt()
routes = "a"
if (gmtnow - ts).hours < 2:
routes = "ac"
fp = "/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib" % (
ts.strftime("%Y/%m/%d"), ts.strftime("%Y%m%d%H") )
if not os.path.isfile(fp):
print 'Missing stage4 %s' % (fp,)
return
grib = Nio.open_file(fp)
lats = grib.variables["g5_lat_0"][:]
lons = grib.variables["g5_lon_1"][:]
vals = grib.variables["A_PCP_GDS5_SFC_acc1h"][:] / 25.4
cfg = {
'wkColorMap': 'BlAqGrYeOrRe',
'nglSpreadColorStart': -1,
'nglSpreadColorEnd' : 2,
'_MaskZero' : True,
'lbTitleString' : "[inch]",
'_valid' : 'Hour Ending %s' % (ts.localtime().strftime("%d %B %Y %I %p %Z"),),
'_title' : "StageIV 1 Hour Precipitation [inch]",
}
tmpfp = iemplot.simple_grid_fill(lons, lats, vals, cfg)
pqstr = "plot %s %s00 iowa_stage4_1h.png iowa_stage4_1h_%s.png png" % (
routes, ts.strftime("%Y%m%d%H"), ts.strftime("%H"))
iemplot.postprocess(tmpfp, pqstr)
# Plot Midwest
cfg = {
'wkColorMap': 'BlAqGrYeOrRe',
'nglSpreadColorStart': -1,
'nglSpreadColorEnd' : 2,
'_MaskZero' : True,
'_midwest' : True,
'lbTitleString' : "[inch]",
'_valid' : 'Hour Ending %s' % (ts.localtime().strftime("%d %B %Y %I %p %Z"),),
'_title' : "StageIV 1 Hour Precipitation [inch]",
}
tmpfp = iemplot.simple_grid_fill(lons, lats, vals, cfg)
pqstr = "plot %s %s00 midwest_stage4_1h.png midwest_stage4_1h_%s.png png" % (
routes, ts.strftime("%Y%m%d%H"),ts.strftime("%H") )
iemplot.postprocess(tmpfp, pqstr)
def test_nc4_NCFile_create_variable_ndim(self):
iobackend.set_backend('netCDF4')
ncf = iobackend.NCFile(self.ncfaname, mode='a')
v2 = ncf.create_variable('v2', np.dtype('f'), ('t', 'x'))
v2[:] = self.v2
ncf.close()
ncfr = Nio.open_file(self.ncfaname)
actual = ncfr.variables['v2'][:]
expected = self.v2
ncfr.close()
print_test_msg('NCFile.create_variable()',
actual=actual, expected=expected)
npt.assert_array_equal(
actual, expected, 'NCFile 2d-variable incorrect')
def do_simulated_plot(file_name, valid_time, sec_string, obs=None, date_tag=True):
path_name, file_base = os.path.split(file_name)
ena_string = file_base.split(".")[0]
if date_tag:
tag = path_name.split("/")[-1].split("-")[-1]
id_string = "-%s" % tag
else:
id_string = ""
hdf = nio.open_file(file_name, mode='r', format='hdf')
vars = {}
for var in ['pt', 'p', 'qr', 'qs', 'qh', 'w']:
vars[var] = hdf.variables[var][12]
if vars[var].min() == -32768 or vars[var].max() == 32767:
dindex = (12, slice(None), slice(None))
vars[var] = decompressVariable(hdf.variables[var], dindex=dindex)
if var in ['qr', 'qs', 'qh']:
vars[var] = np.maximum(vars[var], np.zeros(vars[var].shape))
reflectivity = computeReflectivity(**vars)
reflectivity_thresh = np.maximum(reflectivity, np.zeros(reflectivity.shape))
w_levels = range(-20, 0, 2)
w_levels.extend(range(2, 22, 2))
refl_title = "Base Reflectivity Valid %s" % valid_time.strftime("%d %b %Y %H%M UTC")
refl_img_file_name = "%s%s.bref.%s.png" % (ena_string, id_string, sec_string)
ptprt_title = r"$\theta$ Valid %s" % valid_time.strftime("%d %b %Y %H%M UTC")
ptprt_img_file_name = "%s%s.pt.%s.png" % (ena_string, id_string, sec_string)
# vars = {}
for var in ['pt', 'u', 'v']:
vars[var] = hdf.variables[var][2]
if vars[var].min() == -32768 or vars[var].max() == 32767:
dindex = (2, slice(None), slice(None))
vars[var] = decompressVariable(hdf.variables[var], dindex=dindex)
# print pt.min(), pt.max()
bounds = (slice(100, 130), slice(100, 130))
plot_map(vars['pt'], 1000, ptprt_title, ptprt_img_file_name, color_bar='pt', vectors=(vars['u'], vars['v']), obs=obs)
plot_map(reflectivity, 1000, refl_title, refl_img_file_name, color_bar='refl', aux_field=(w_levels, vars['w']))
return
def main():
base_path = "/caps1/tsupinie/1km-control-no-ua"
files = glob.glob("%s/ena???.hdf0*" % base_path)
for file in files:
hdf = nio.open_file(file, mode='r', format='hdf')
for var in ['u', 'v', 'w', 'pt', 'p', 'qv']:
if var not in hdf.variables:
print "%s incomplete ... " % file
break
hdf.close()
return
def _init(self):
if self._open_file is False:
self._open_file = True
f = Nio.open_file(self.pathname)
# Dimension of var is time, lat, lon
self._var = f.variables[self.varname]
self._time = f.variables['time']
self._lat = f.variables['lat']
self._lon = f.variables['lon']
self.time = None
self.lat = None
self.lon = None
def calculate_maxnormens(opts_dict,var_list):
ifiles=[]
Maxnormens={}
threshold=1e-12
# input file directory
inputdir=opts_dict['indir']
# the timeslice that we want to process
tstart=opts_dict['tslice']
# open all files
for frun_file in os.listdir(inputdir):
if (os.path.isfile(inputdir+frun_file)):
ifiles.append(Nio.open_file(inputdir+frun_file,"r"))
else:
print "COULD NOT LOCATE FILE "+inputdir+frun_file+" EXISTING"
sys.exit()
comparision={}
# loop through each variable
for k in var_list:
output=[]
# read all data of variable k from all files
for f in ifiles:
v=f.variables
output.append(v[k][tstart])
max_val=0
# open an output file
outmaxnormens=k+"_ens_maxnorm.txt"
fout=open(outmaxnormens,"w")
Maxnormens[k]=[]
# calculate E(i=0:n)(maxnormens[i][x])=max(comparision[i]-E(x=0:n)(output[x]))
for n in range(len(ifiles)):
Maxnormens[k].append(0)
comparision[k]=ifiles[n].variables[k][tstart]
for m in range(len(ifiles)):
max_val=np.max(np.abs(comparision[k]-output[m]))
if Maxnormens[k][n] < max_val:
Maxnormens[k][n]=max_val
range_max=np.max((comparision[k]))
range_min=np.min((comparision[k]))
if range_max-range_min < threshold:
Maxnormens[k][n]=0.
else:
Maxnormens[k][n]=Maxnormens[k][n]/(range_max-range_min)
fout.write(str(Maxnormens[k][n])+'\n')
strtmp = k + ' : ' + 'ensmax min max' + ' : ' + '{0:9.2e}'.format(min(Maxnormens[k]))+' '+'{0:9.2e}'.format(max(Maxnormens[k]))
print strtmp
fout.close()
def main():
files = [ 'hdf/KCYS/3km/manual/goshen.hdfrefl2d010691', 'hdf/KRIW/3km/goshen.hdfrefl2d010785', 'hdf/KFTG/3km/goshen.hdfrefl2d010638' ]
radar_data = [ nio.open_file(f, format='hdf', mode='r').variables['refl2d'][0] for f in files ]
radars = { 'KCYS':(41.15194, -104.80611), 'KFTG':(39.78667, -104.54583), 'KRIW':(43.06611, -108.47722) }
composite = makeComposite(radar_data)
proj = setupMapProjection(goshen_3km_proj, goshen_3km_gs)
map = Basemap(**proj)
makePlot(composite, goshen_3km_gs, map, "Radar Obs composite (2100 UTC)", "radar_composite_010800.png", radars=radars)
# for fn, rd in zip(files, radar_data):
# rid = fn[4:8]
# makePlot(rd, goshen_3km_gs, map, "Radar Obs composite component (%s, 2100 UTC)" % rid, "radar_composite_comp_%s_010800.png" % rid)
return
import Ngl, Nio
import sys
import numpy as np
nc_file = sys.argv[1]
grid_file = sys.argv[2]
output_dir = sys.argv[3]
f = Nio.open_file(nc_file, "r")
g = Nio.open_file(grid_file, "r")
lon = g.variables["xc"][1, :] #-- read clon
lat = g.variables["yc"][:, 1] #-- read clat
print(lon)
print(lat)
print("Getting Data")
data = f.variables["SSTAYYC"][11, :, :]
print(data.shape)
print("DONE")
data[:] = 0
#---Start the graphics
print("Start plotting...")
wks_type = "png"
wks = Ngl.open_wks(wks_type, "%s/SSTAYYC" % (output_dir,))
#---Read in desired color map so we can subset it later
# Notes: The data for this example can be downloaded from
# http://www.ncl.ucar.edu/Document/Manuals/NCL_User_Guide/Data/
#
'''
Transition Guide PyNGL Example: TRANS_streamline.py
- Read netCDF data
- Drawing a streamline plot
18-09-04 kmf
'''
from __future__ import print_function
import Ngl, Nio
#-- open a file and read variables
f = Nio.open_file("../read_data/rectilinear_grid_2D.nc", "r")
u = f.variables["u10"]
v = f.variables["v10"]
ua = f.variables["u10"][0, :, :]
va = f.variables["v10"][0, :, :]
lat = f.variables["lat"]
lon = f.variables["lon"]
nlon = len(lon)
nlat = len(lat)
#-- open a workstation
wks = Ngl.open_wks("png", "plot_TRANS_streamline_py")
#-- resource settings
def openWRF(main_directory, file_name):
try:
wrf = Nio.open_file(main_directory + file_name, format='nc')
return wrf
except:
print('Failure when opening WRF file')
def test_types(self):
#
# Specify a global history attribute and open a NetCDF file
# for writing.
#
hatt = "Created " + time.ctime(time.time()) + " by " + getUserName()
f = Nio.open_file(self.filename, "w", None, hatt)
#
# Create some global attributes.
#
f.title = "Nio test NetCDF file"
f.series = [1, 2, 3, 4, 5, 6]
f.version = 45
file_attributes.update({'history': hatt})
#
# Create some dimensions.
#
f.create_dimension("array", 3)
#f.create_dimension("strlen", 6)
f.create_dimension("strlen", 10)
f.create_dimension("dim1", 2)
f.create_dimension("dim2", 1)
f.create_dimension("dim3", 4)
#
# Create some variables.
#
#print("creating and assigning scalar double")
v1 = f.create_variable("v1", 'd', ())
v1.assign_value(42.0)
#print("creating and assigning scalar float")
v2 = f.create_variable("v2", 'f', ())
v2.assign_value(52.0)
#print("creating and assigning scalar integer")
v3 = f.create_variable("v3", 'i', ())
v3.assign_value(42)
#print("creating and assigning scalar long")
v4 = f.create_variable("v4", 'l', ())
v4.assign_value(42)
#print("creating and assigning scalar short")
v5 = f.create_variable("v5", 'h', ())
v5.assign_value(42)
#print("creating and assigning scalar byte")
v6 = f.create_variable("v6", 'b', ())
v6.assign_value(42)
#print("creating and assigning scalar char")
v7 = f.create_variable("v7", 'S1', ())
v7.assign_value('x')
#print("creating and assigning array double")
v11 = f.create_variable("v11", 'd', ('array', ))
v11.assign_value([42.0, 43.0, 44.0])
#print("creating and assigning array float")
v22 = f.create_variable("v22", 'f', ('array', ))
v22.assign_value([52.0, 53.0, 54.0])
#print("creating and assigning array integer")
v33 = f.create_variable("v33", 'i', ('array', ))
v33.assign_value([42, 43, 44])
#print("creating and assigning array long")
v44 = f.create_variable("v44", 'l', ('array', ))
a = np.array([42, 43, 44], 'l')
v44.assign_value(a)
#print("creating and assigning array short")
v55 = f.create_variable("v55", 'h', ('array', ))
v55.assign_value([42, 43, 44])
#print("creating and assigning array byte")
v66 = f.create_variable("v66", 'b', ('array', ))
v66.assign_value([42, 43, 44])
#print("creating and assigning array char")
v77 = f.create_variable("v77", 'S1', ('array', 'strlen'))
v77.assign_value(['bcdef', 'uvwxyz', 'ijklmnopqr'])
#v77.assign_value(['ab','uv','ij'])
#v77.assign_value(['a','u','i'])
#v77[1] = v77[1,::-1]
#print(v77[:])
v_single = f.create_variable("v_single", 'f', ("dim1", "dim2", "dim3"))
#print(v_single)
# type mismatch (double created then assigned to float variable)
a = np.array([1.0, 2, 3, 4, 5, 6, 7, 8], dtype=np.float64)
a.shape = (2, 1, 4)
#print(a)
with nt.assert_raises(Nio.NIOError):
v_single.assign_value(a)
# now do it right
a = np.array([1.0, 2, 3, 4, 5, 6, 7, 8], dtype=np.float32)
a.shape = (2, 1, 4)
#print(a)
v_single.assign_value(a)
#print(v_single[:])
v_single[1, 0, 2] = 11.0
v_single[:, 0, 2] = [11.0, 12.0]
var_names = list(f.variables.keys())
nt.assert_equal(set(var_names), file_variables)
for var in var_names:
v = f.variables[var]
nt.assert_equal(v.dimensions, var_dimensions[var])
nt.assert_equal(v.attributes, {})
nt.assert_equal(v.get_value(), var_values[var])
f.close()
#
# Read the file we just created.
#
f = Nio.open_file(self.filename, "r")
nt.assert_equal(f.attributes, file_attributes)
nt.assert_equal(f.dimensions, file_dimensions)
nt.assert_equal(set(f.variables.keys()), file_variables)
for var in var_names:
v = f.variables[var]
nt.assert_equal(v.dimensions, var_dimensions[var])
nt.assert_equal(v.attributes, {})
nt.assert_equal(v.get_value(), var_values[var])
f.close()
#
# To use the ScientificPython module to read in the netCDF file,
# comment out the above "import" command, and uncomment the
# import line below.
#
# from Scientific.IO.NetCDF import NetCDFFile
#
# Import Ngl support functions.
#
import Ngl
#
# Open three netCDF files and get variables.
#
data_dir = Ngl.pynglpath("data")
cdf_file1 = Nio.open_file(os.path.join(data_dir, "cdf", "941110_P.cdf"), "r")
cdf_file2 = Nio.open_file(os.path.join(data_dir, "cdf", "sstdata_netcdf.nc"),
"r")
cdf_file3 = Nio.open_file(os.path.join(data_dir, "cdf", "Pstorm.cdf"), "r")
#
# This is the ScientificPython method for opening netCDF files.
#
# cdf_file1 = NetCDFFile(os.path.join(data_dir,"cdf","941110_P.cdf"),"r")
# cdf_file2 = NetCDFFile(os.path.join(data_dir,"cdf","sstdata_netcdf.nc"),"r")
# cdf_file3 = NetCDFFile(os.path.join(data_dir,"cdf","Pstorm.cdf"),"r")
psl = cdf_file1.variables["Psl"][:]
sst = cdf_file2.variables["sst"]
pf = cdf_file3.variables["p"]
break
#print stationlist
allpath = '/Users/yetao.lu/Documents/testdata'
# 遍历文件
inputfile = ''
inputfile2 = ''
#filesdict={}
for rootpath, dirs, files in os.walk(allpath):
for file in files:
if file[:3] == 'sfc' and file[-5:] == '.grib' and (string.find(
file, '2014') == -1):
inputfile = os.path.join(rootpath, file)
inputfile2 = inputfile.replace('sfc', 'pl')
#filesdict[inputfile]=inputfile2
#print inputfile,inputfile2
sfcfile = Nio.open_file(inputfile, 'r')
plfile = Nio.open_file(inputfile2, 'r')
#参数0是指第0个时次的预报
GetStationsAndOnetimesFromEC(0, sfc_varinames, sfcfile,
pl_varinames, plfile, inputfile)
csvfile1 = '/Users/yetao.lu/Desktop/mos/data.csv'
filewrite = open(csvfile1, 'w')
#print len(alllist)
for i in range(len(stationsVlist)):
for j in range(len(stationsVlist[i])):
if j == len(stationsVlist[i]) - 1:
filewrite.write(str((stationsVlist[i])[j]))
else:
filewrite.write(str((stationsVlist[i])[j]) + ',')
filewrite.write('\n')
# file = Nio.open_file(inputfile, 'r')
if o == "-e":
export_flag = 1
if o == "-r":
restart_time = int(a)
# Skip is the length between outputs
# skip = 1
if dom == 'd02':
skip = 1
DP_CLEVS = range(55, 90, 5)
else:
skip = 3
DP_CLEVS = range(55, 90, 5)
filename = '../wrfout_' + dom + '_PLEV.nc'
nc = netcdf.open_file(filename)
# Grab three variables for now
#temps_ua = nc.variables['TT']
temps_ua = nc.variables['T']
#temps_base_ua = nc.variables['TB']
temps_base_ua = 300
u_wind_ms_ua = nc.variables['UU'][:, :, :, :-1]
v_wind_ms_ua = nc.variables['VV'][:, :, :-1, :]
w_wind_ms_ua = nc.variables['W']
#rhum_ua = nc.variables['RH']
qvap = nc.variables['QVAPOR']
#ght = nc.variables['GHT']
phb = nc.variables['PHB']
ph = nc.variables['PH']
w_wind_ua = nc.variables['W']
import Ngl, Nio
import math
import numpy as np
import json
import pandas as pd
import datetime
import time
st = time.time()
fname = "/home/alley/work/Dong/mongo/seasonal_analysis/data/data/1981-2019_gh_500.grib"
f = Nio.open_file(fname, 'r')
# print(f.variables)
lat = np.array(f.variables['g0_lat_1'])
lon = np.array(f.variables['g0_lon_2'])
Time = list(np.array(f.variables['initial_time0_encoded']))
gh = np.array(f.variables['Z_GDS0_ISBL_S123'])
dims = (gh.shape)
print(dims)
n = 0
with open(
'/home/alley/work/Dong/mongo/seasonal_analysis/data/data/1981-2019_gh_500_by_only_latlon.json',
'w',
encoding='utf-8') as fout:
for lat_i in range(len(lat)):
for lon_i in range(len(lon)):
tmp = (gh[:, lat_i, lon_i])
tmp = (np.array(tmp).reshape(int(dims[0] / 12),
12)).tolist() # 维度（年数，月数）
lat_ = float(lat[lat_i])
lon_ = float(lon[lon_i])
dic = {
- Reading netCDF file
- Converting data from Kelvin to degC
- Writing ASCII data to new file
2018-08-30 kmf
"""
from __future__ import print_function
import os, sys
import numpy as np
import Ngl, Nio
#-- data file name
fname = "../read_data/rectilinear_grid_3D.nc"
#-- open file
f = Nio.open_file(fname, "r")
#-- read variable, first time step, first level
var = f.variables["t"][0, 0, :, :]
#-- convert var from Kelvin to degC while retaining the missing values
var = var - 273.15
print(var)
# -- write var to an ASCII file
os.system("/bin/rm -f data_py.asc") #-- delete file
sys.stdout = open("data_py.asc", "w") #-- redirect stdout to file
for i in range(0, 10):
for j in range(0, 10):
print "%10.6f" % (var[i, j]) #-- write to file
# Description:
# This example shows how to draw partially opaque text on a plot
#
# Effects illustrated:
# o Using the new txFontOpacityF resource
# o Subscripting a variable using coordinate values
#
# Output:
# A single visualization with two XY curves
#
from __future__ import print_function
import numpy,os
import Ngl,Nio
dirc = Ngl.pynglpath("data")
f = Nio.open_file(os.path.join(dirc,"cdf","uv300.nc"))
u = f.variables["U"][0,:,8]
lat = f.variables["lat"][:]
#---Start the graphics section
wks_type = "png"
wks = Ngl.open_wks (wks_type,"newcolor2") # Open "newcolor2.png" for graphics
#---Set some resources.
res = Ngl.Resources()
res.nglDraw = False
res.nglFrame = False
res.xyLineThicknessF = 4.0
plot = Ngl.xy (wks,lat,u,res) # Create plot
import Ngl
import Nio
import numpy
import os
dirc=Ngl.pynglpath("data")
file=Nio.open_file(os.path.join(dirc,"cdf","pop.nc"))
wks_type="png"
wks=Ngl.open_wks(wks_type,"map2")
urot=file.variables["urot"]
t=file.variables["t"]
lat2d=file.variables["lat2d"]
lon2d=file.variables["lon2d"]
u=Ngl.add_cyclic(urot[:])
temp=Ngl.add_cyclic(t[0:])
lon=Ngl.add_cyclic(lon2d[0:])
lat=Ngl.add_cyclic(lat2d[0:])
resource=Ngl.Resources()
resource.vfXArray=lon
resource.vfYArray=lat
resource.mpProjection="Stereographic"
resource.mpFillOn=True
resource.mpInlandWaterFillColor="SkyBlue"
resource.mpProjection = "Stereographic"
#
# Import Nio for reading netCDF files.
#
import Nio
#
# Import Ngl support functions.
#
import Ngl
#
# Open the netCDF file containing the climate divisions polygons.
#
dirc = Ngl.pynglpath("data")
ncdf = Nio.open_file(os.path.join(dirc, "cdf", "climdiv_polygons.nc"))
#
# State names for the contiguous U.S. states.
#
statenames = ["AL","AR","AZ","CA","CO","CT","DE","FL","GA","IA","ID","IL", \
"IN","KS","KY","LA","MA","MD","ME","MI","MN","MO","MS","MT", \
"NC","ND","NE","NH","NJ","NM","NV","NY","OH","OK","OR","PA", \
"RI","SC","SD","TN","TX","UT","VA","VT","WA","WI","WV","WY"]
#
# Climate divisions in each state.
#
ncds = [8,9,7,7,5,3,2,6,9,9,10,9,9,9,4,9,3,8,3,10,9,6,10,7, \
8,9,8,2,3,8,4,10,10,9,9,10,1,7,9,4,10,7,7,3,10,9,6,10]
def modelprocess(ll):
sys.stdout=open(os.path.join(outpath,'ml'+str(ll)+'.out'),'w')
# 站点列表数据
stationlist = []
csvfile = '/Users/yetao.lu/Desktop/mos/t_p_station_cod.csv'
fileread = open(csvfile, 'r')
firstline = fileread.readline()
while True:
line = fileread.readline()
perlist = line.split(',')
if len(perlist) > 4:
stationlist.append(perlist)
if not line or line == '':
break
allpath = '/Users/yetao.lu/Documents/testdata'
sfc_varinames = ['2T_GDS0_SFC', '2D_GDS0_SFC', '10U_GDS0_SFC','10V_GDS0_SFC','TCC_GDS0_SFC', 'LCC_GDS0_SFC']
pl_varinames = ['R_GDS0_ISBL']
print 'a'
# 遍历文件
dict={}
stationsVlist = []
trainlebellist=[]
def GetOnetimeFromEC(n, sfc_varinames, sfc_file, indexlat, indexlon,
pl_varinames, pl_file):
vstring = []
levelArray = pl_file.variables['lv_ISBL1']
for i in range(len(sfc_varinames)):
variArray = sfc_file.variables[sfc_varinames[i]]
latlonArray = variArray[n]
vstring.append(latlonArray[indexlat][indexlon])
vstring.append(latlonArray[indexlat][indexlon + 1])
vstring.append(latlonArray[indexlat + 1][indexlon + 1])
vstring.append(latlonArray[indexlat + 1][indexlon])
vstring.append(latlonArray[indexlat - 1][indexlon - 1])
vstring.append(latlonArray[indexlat - 1][indexlon])
vstring.append(latlonArray[indexlat - 1][indexlon + 1])
vstring.append(latlonArray[indexlat - 1][indexlon + 2])
vstring.append(latlonArray[indexlat][indexlon + 2])
vstring.append(latlonArray[indexlat + 1][indexlon + 2])
vstring.append(latlonArray[indexlat + 2][indexlon + 2])
vstring.append(latlonArray[indexlat + 2][indexlon + 1])
vstring.append(latlonArray[indexlat + 2][indexlon])
vstring.append(latlonArray[indexlat + 2][indexlon - 1])
vstring.append(latlonArray[indexlat + 1][indexlon - 1])
vstring.append(latlonArray[indexlat][indexlon - 1])
for p in range(len(pl_varinames)):
pl_variArray = pl_file.variables[pl_varinames[p]]
phaArray = pl_variArray[n]
for k in range(len(phaArray)):
llArray = phaArray[k]
pha = levelArray[k]
# print pha
if pha == 500 or pha == 850:
# vstring.append(str(pha)+'hpa')
vstring.append(llArray[indexlat][indexlon])
vstring.append(llArray[indexlat][indexlon + 1])
vstring.append(llArray[indexlat + 1][indexlon + 1])
vstring.append(llArray[indexlat + 1][indexlon])
vstring.append(llArray[indexlat - 1][indexlon - 1])
vstring.append(llArray[indexlat - 1][indexlon])
vstring.append(llArray[indexlat - 1][indexlon + 1])
vstring.append(llArray[indexlat - 1][indexlon + 2])
vstring.append(llArray[indexlat][indexlon + 2])
vstring.append(llArray[indexlat + 1][indexlon + 2])
vstring.append(llArray[indexlat + 2][indexlon + 2])
vstring.append(llArray[indexlat + 2][indexlon + 1])
vstring.append(llArray[indexlat + 2][indexlon])
vstring.append(llArray[indexlat + 2][indexlon - 1])
vstring.append(llArray[indexlat + 1][indexlon - 1])
vstring.append(llArray[indexlat][indexlon - 1])
return vstring
def GetStationsAndOnetimesFromEC(i, sfc_varinames, sfc_file, pl_varinames,
pl_file, inputfile):
# print 'stationlist:',len(stationlist)
for j in range(len(stationlist)):
# 根据文件名来建立索引获取实况数据气温的值
strarray = inputfile.split('_')
odatetime = datetime.datetime.strptime(
(strarray[1] + strarray[2][:2]),
'%Y%m%d%H')
fdatetime = odatetime + datetime.timedelta(hours=i * 3)
fdateStr = datetime.datetime.strftime(fdatetime, '%Y%m%d%H%M%S')
# 这里只能是起报时间加预报时效，不能用预报时间。因预报时间有重复
dictid = stationlist[j][0] + '_' + strarray[1] + strarray[2][
:2] + '_' + str(i)
# 根据站号+实况数据的索引来获取实况数据
kid = stationlist[j][0] + '_' + fdateStr
trainlebel = stationdict.get(kid)
# 判断该实况数据是否是有效值（不为99999或者None）,若有效再计算16个格点值，将其一起添加到训练样本
if trainlebel <> None and trainlebel <> 999999:
latitude = float(stationlist[j][4])
longitude = float(stationlist[j][5])
# # #首先计算经纬度对应格点的索引，
indexlat = int((60 - latitude) / 0.125)
indexlon = int((longitude - 60) / 0.125)
# print latitude,longitude,indexlat,indexlon
# 则依次取周边16个点的索引为[indexlat,indexlon+1][indexlat+1,indexlon+1][indexlat+1,indexlon]...(顺时针)
perstationlist = GetOnetimeFromEC(i, sfc_varinames, sfc_file,
indexlat, indexlon,
pl_varinames, pl_file)
# print dictid,perstalist,kid,trainlebel
dict[dictid] = perstationlist
stationsVlist.append(perstationlist)
trainlebellist.append(trainlebel)
# print 'stationsVlist',len(stationsVlist),'trainlebellist',len(trainlebellist),'stationdict',len(stationdict)
#遍历所有的文件，
for rootpath, dirs, files in os.walk(allpath):
for file in files:
if file[:3] == 'sfc' and file[-5:] == '.grib' and (string.find(file,'2014')==-1):
inputfile = os.path.join(rootpath, file)
inputfile2=inputfile.replace('sfc','pl')
sfcfile=Nio.open_file(inputfile,'r')
plfile=Nio.open_file(inputfile2,'r')
#参数0是指第0个时次的预报,这里只是一个文件的2000个站的列表。
GetStationsAndOnetimesFromEC(ll,sfc_varinames,sfcfile,pl_varinames,plfile,inputfile)
stationArray=numpy.array(stationsVlist)
trainlebelArray=numpy.array(trainlebellist)
a_train,a_test=train_test_split(stationArray,test_size=0.33,random_state=7)
#print len(a_train),len(a_test),len(a_train)+len(a_test)
#数据训练前进行标准化
x_scaled=preprocessing.scale(stationArray)
stationArray=x_scaled
#xgboost
x_train,x_test,y_train,y_test=train_test_split(stationArray,trainlebelArray,test_size=0.33,random_state=7)
xgbtrain=xgboost.DMatrix(x_train,label=y_train)
xgbtest=xgboost.DMatrix(x_test,label=y_test)
#xgbtrain.save_binary('train.buffer')
#print len(x_train),len(x_test),len(y_train),len(y_test)
#print xgbtest
#训练和验证的错误率
watchlist=[(xgbtrain,'xgbtrain'),(xgbtest,'xgbeval')]
params={
'booster':'gbtree',
'objective': 'reg:linear', #线性回归
'gamma':0.2, # 用于控制是否后剪枝的参数,越大越保守，一般0.1、0.2这样子。
'max_depth':12, # 构建树的深度，越大越容易过拟合
'lambda':2, # 控制模型复杂度的权重值的L2正则化项参数，参数越大，模型越不容易过拟合。
'subsample':0.7, # 随机采样训练样本
'colsample_bytree':0.7, # 生成树时进行的列采样
'min_child_weight':3,
# 这个参数默认是 1，是每个叶子里面 h 的和至少是多少，对正负样本不均衡时的 0-1 分类而言
#，假设 h 在 0.01 附近，min_child_weight 为 1 意味着叶子节点中最少需要包含 100 个样本。
#这个参数非常影响结果，控制叶子节点中二阶导的和的最小值，该参数值越小，越容易 overfitting。
'silent':0 ,#设置成1则没有运行信息输出，最好是设置为0.
'eta': 0.01, # 如同学习率
'seed':1000,
'nthread':3,# cpu 线程数
#'eval_metric': 'auc'
'scale_pos_weight':1
}
plst=list(params.items())
num_rounds=20000
#early_stopping_rounds当设置的迭代次数较大时,early_stopping_rounds 可在一定的迭代次数内准确率没有提升就停止训练
model=xgboost.train(plst,xgbtrain,num_rounds,watchlist,early_stopping_rounds=800)
#print model,watchlist
preds=model.predict(xgbtest,ntree_limit=model.best_iteration)
# 将预测结果写入文件，方式有很多，自己顺手能实现即可
# numpy.savetxt('submission_xgb_MultiSoftmax.csv',numpy.c_[range(1,len(test)+1),preds],
# delimiter=',',header='ImageId,Label',comments='',fmt='%d')
#print preds
#print y_test.dtype,preds.dtype
y_test=y_test.astype('float32')
mse=mean_squared_error(y_test,preds)
rmse=math.sqrt(mse)
mae=mean_absolute_error(y_test,preds,multioutput='uniform_average')
print("训练后MSE: %.4f" % mse)
print("训练后RMSE: %.4f" % rmse)
print("训练后MAE: %.4f" % mae)
#气温差2度的准确率
n=0
for x,y in zip(y_test,preds):
if abs(x-y)<2:
n=n+1
accuracy=float(n)/float(len(y_test))
print ("训练后2度的accuracy: %.4f" % accuracy)
n=0
for x,y in zip(y_test,preds):
if abs(x-y)<3:
n=n+1
accuracy=float(n)/float(len(y_test))
print ("训练后3度的accuracy: %.4f" % accuracy)
#和EC中原始数据对比获取均方误差
y_origin=a_test[:,0]
#print y_origin
y_origin=y_origin-273.15
#print y_origin
mse0=mean_squared_error(y_test,y_origin)
rmse0=math.sqrt(mse0)
mae0=mean_absolute_error(y_test,y_origin,multioutput='uniform_average')
print("训练前MSE: %.4f" % mse0)
print("训练前RMSE: %.4f" % rmse0)
print("训练前MAE: %.4f" % mae0)
n=0
for x,y in zip(y_test,y_origin):
if abs(x-y)<2:
n=n+1
accuracy=float(n)/float(len(y_test))
print ("训练前2度的accuracy: %.4f" % accuracy)
n=0
for x,y in zip(y_test,y_origin):
if abs(x-y)<3:
n=n+1
accuracy=float(n)/float(len(y_test))
print ("训练前3度的accuracy: %.4f" % accuracy)
model.save_model(os.path.join(outpath,'ectemp'+str(ll)+'.model'))
testfile=os.path.join(outpath,'test'+str(ll)+'.csv')
predsfile=os.path.join(outpath,'preds'+str(ll)+'.csv')
originfile=os.path.join(outpath,'origin'+str(ll)+'.csv')
testfw=open(testfile,'w')
predsfw=open(predsfile,'w')
originfw=open(originfile,'w')
for u in y_test:
testfw.write(str(u)+',')
testfw.close()
for o in preds:
predsfw.write(str(o)+',')
predsfw.close()
for q in y_origin:
originfw.write(str(q)+',')
originfw.close()
del stationArray
del trainlebelArray
del a_test
del a_train
del x_test
del x_train
del y_origin
del y_test
del y_train
del xgbtest
del xgbtrain
del x_scaled
del plfile
del sfcfile
from __future__ import print_function
import numpy as np
import Nio, Ngl, os, sys
from wrf import getvar, latlon_coords, to_np
# Read data
filename = "wrfout_d03_2012-04-22_23_00_00"
if(not os.path.exists(filename)):
print("You do not have the necessary '{}' file to run this example.".format(filename))
print("You need to supply your own WRF output file")
print("WRF output files usually have names like '{}'".format(filename))
sys.exit()
# Read some WRF data
a = Nio.open_file(filename+".nc") # Must add ".nc" suffix for Nio.open_file
ua = getvar(a,"ua")
va = getvar(a,"va")
# First timestep, lowest (bottommost) level, every 5th lat/lon
nl = 0
nt = 0
nstep = 5 # a stride to cull some of the streamlines
u = ua[nl,::nstep,::nstep]
v = va[nl,::nstep,::nstep]
spd = np.sqrt(u**2+v**2)
# Get the latitude and longitude points
lat, lon = latlon_coords(ua)
lat = to_np(lat)
lon = to_np(lon)
#
# http://mailman.ucar.edu/mailman/listinfo/pyngl-talk
#
from __future__ import print_function
import numpy, Nio, Ngl, sys, os
#---Read data
filename = "b.e12.B1850C5CN.ne30_g16.init.ch.027.cam.h0.0001-01.nc"
if (not os.path.exists(filename)):
print(
"You do not have the necessary '{}' file to run this example.".format(
filename))
print("See the comments at the top of this script for more information.")
sys.exit()
a = Nio.open_file(filename)
vname = "TS"
data = a.variables[vname]
lat = a.variables["lat"][:] # 1D array (48602 cells)
lon = a.variables["lon"][:] # ditto
ncells = data.shape[1]
print("There are {} cells in the {} variable".format(ncells, vname))
wks_type = "png"
wks = Ngl.open_wks(wks_type, "camse1")
#---Set some plot options
res = Ngl.Resources()
# Contour options
#target.write(header1)
#target.write('\n')
#target.write(header2)
#target.write('\n')
#print var[0,0]
#target.write(var[:,:])
#target.write('\n')
#target.close()
for i in range(1, len(sys.argv)):
infilename = sys.argv[i]
print infilename
#file = Nio.open_file(os.path.join(Ngl.pynglpath("data"),"grb",infilename),"r")
infile = Nio.open_file(infilename, "r")
names = infile.variables.keys() # Get the variable names
print "\nVariable names:" # and print them out.
print names
for j in range(0, len(names)):
if names[j] == 'lat_0' or names[j] == 'lon_0':
continue
#
# For variable in names[1], retrieve and print all attributes
# and their values.
#
print "\nThe attributes and their values for variable " + names[j] + ":"
for attrib in infile.variables[names[j]].attributes.keys():
def modelprocess(stationdict, stationlist, ll, allpath, dempath, forehours):
max_varinames = ['MX2T6_GDS0_SFC_1']
max_varinames001 = ['MX2T6_GDS0_SFC']
min_varinames = ['MN2T6_GDS0_SFC_1']
min_varinames001 = ['MN2T6_GDS0_SFC']
sfc_varinames1 = [
'2T_GDS0_SFC', '2D_GDS0_SFC', '10U_GDS0_SFC', '10V_GDS0_SFC',
'TCC_GDS0_SFC', 'LCC_GDS0_SFC', 'Z_GDS0_SFC'
]
pl_varinames = ['R_GDS0_ISBL']
# print max_varinames
dict01 = {}
# 遍历文件
maxecvaluelist = []
minecvaluelist = []
maxtemplist = []
mintemplist = []
# 遍历所有的文件，
for rootpath, dirs, files in os.walk(allpath):
for file in files:
# print file
if file[:3] == 'sfc' and file[-5:] == '.grib' and (string.find(
file, '2017') == -1):
inputfile = os.path.join(rootpath, file)
inputfile2 = inputfile.replace('sfc', 'pl')
inputfile3 = inputfile.replace('sfc', '6h')
inputfile4 = inputfile.replace('sfc', '3h')
sfcfile = Nio.open_file(inputfile, 'r')
plfile = Nio.open_file(inputfile2, 'r')
h6file = Nio.open_file(inputfile3, 'r')
h3file = Nio.open_file(inputfile4, 'r')
# 参数0是指第0个时次的预报,这里只是一个文件的2000个站的列表。
GetStationsAndOnetimesFromEC(
ll, max_varinames, max_varinames001, min_varinames,
min_varinames001, h6file, h3file, sfc_varinames1, sfcfile,
pl_varinames, plfile, inputfile, maxecvaluelist,
minecvaluelist, stationdict, stationlist, dict01,
maxtemplist, mintemplist, dempath)
#ecvaluelist = numpy.array(ecvaluelist)
maxtemplist = numpy.array(maxtemplist)
mintemplist = numpy.array(mintemplist)
# class_train, class_test = train_test_split(ecvaluelist, test_size=0.33,random_state=7)
# 数据训练前进行标准化
#ecvaluelist = ecvaluelist.astype('float32')
maxtemplist = maxtemplist.astype('float32')
mintemplist = mintemplist.astype('float32')
# print ecvaluelist.shape, maxtemplist.shape
# x_scaled = preprocessing.scale(ecvaluelist)
# ecvaluelist = x_scaled
# max、min分开
#ecvaluelist = numpy.array(ecvaluelist)
maxecvaluelist = ecvaluelist
# print minecvaluelist.shape
# 为了统计准确率，提前分割数据集:这里只是为了计算训练前的准确率，训练后的准确率计算用不到这个分割；
max_train, max_test, min_train, min_test = train_test_split(maxtemplist,
mintemplist,
test_size=0.33,
random_state=7)
# 数据集标准化
x_scaler = preprocessing.StandardScaler().fit(maxecvaluelist)
max_scaler = x_scaler.transform(maxecvaluelist)
maxecvaluelist = max_scaler
maxscaler_file = os.path.join(outpath,
'dem_maxscale' + str(forehours) + '.save')
joblib.dump(x_scaler, maxscaler_file)
# y_scaler = preprocessing.StandardScaler().fit(minecvaluelist)
# min_scaler = y_scaler.transform(minecvaluelist)
# minecvaluelist = min_scaler
# minscaler_file = os.path.join(outpath, 'dem_minscale' + str(forehours) + '.save')
# joblib.dump(y_scaler, minscaler_file)
# xgboost，训练集和预测集分割：这里已经是标准化完的矩阵；
x_train, x_test, y_train, y_test = train_test_split(maxecvaluelist,
maxtemplist,
test_size=0.33,
random_state=7)
# u_train, u_test, v_train, v_test = train_test_split(minecvaluelist,
# mintemplist,
# test_size=0.33,
# random_state=7)
xgbtrain = xgboost.DMatrix(x_train, label=y_train)
xgbtest = xgboost.DMatrix(x_test, label=y_test)
# xgbtrain01 = xgboost.DMatrix(u_train, label=v_train)
# xgbtest01 = xgboost.DMatrix(u_test, label=v_test)
# xgbtrain.save_binary('train.buffer')
# 特征选址
# print x_train.shape, x_test.shape
ff, pp = f_regression(x_train, y_train)
# print ff, pp
# 训练和验证的错误率
watchlist = [(xgbtrain, 'xgbtrain'), (xgbtest, 'xgbeval')]
watchlist01 = [(xgbtrain01, 'xgbtrain01'), (xgbtest01, 'xgbeval01')]
params = {
'booster': 'gbtree',
'objective': 'reg:linear', # 线性回归
'gamma': 0.2, # 用于控制是否后剪枝的参数,越大越保守，一般0.1、0.2这样子。
'max_depth': 12, # 构建树的深度，越大越容易过拟合
'lambda': 2, # 控制模型复杂度的权重值的L2正则化项参数，参数越大，模型越不容易过拟合。
'subsample': 0.7, # 随机采样训练样本
'colsample_bytree': 0.7, # 生成树时进行的列采样
'min_child_weight': 3,
# 这个参数默认是 1，是每个叶子里面 h 的和至少是多少，对正负样本不均衡时的 0-1 分类而言
# ，假设 h 在 0.01 附近，min_child_weight 为 1 意味着叶子节点中最少需要包含 100 个样本。
# 这个参数非常影响结果，控制叶子节点中二阶导的和的最小值，该参数值越小，越容易 overfitting。
'silent': 0, # 设置成1则没有运行信息输出，最好是设置为0.
'eta': 0.1, # 如同学习率
'seed': 1000,
# 'nthread': 3, # cpu 线程数
# 'eval_metric': 'auc'
'scale_pos_weight': 1
}
params01 = {
'booster': 'gbtree',
'objective': 'reg:linear', # 线性回归
'gamma': 0.2, # 用于控制是否后剪枝的参数,越大越保守，一般0.1、0.2这样子。
'max_depth': 12, # 构建树的深度，越大越容易过拟合
'lambda': 2, # 控制模型复杂度的权重值的L2正则化项参数，参数越大，模型越不容易过拟合。
'subsample': 0.7, # 随机采样训练样本
'colsample_bytree': 0.7, # 生成树时进行的列采样
'min_child_weight': 3,
# 这个参数默认是 1，是每个叶子里面 h 的和至少是多少，对正负样本不均衡时的 0-1 分类而言
# ，假设 h 在 0.01 附近，min_child_weight 为 1 意味着叶子节点中最少需要包含 100 个样本。
# 这个参数非常影响结果，控制叶子节点中二阶导的和的最小值，该参数值越小，越容易 overfitting。
'silent': 0, # 设置成1则没有运行信息输出，最好是设置为0.
'eta': 0.1, # 如同学习率
'seed': 1000,
# 'nthread': 3, # cpu 线程数
# 'eval_metric': 'auc'
'scale_pos_weight': 1
}
plst = list(params.items())
plst01 = list(params01.items())
num_rounds = 99999
# early_stopping_rounds当设置的迭代次数较大时,early_stopping_rounds 可在一定的迭代次数内准确率没有提升就停止训练
model = xgboost.train(plst,
xgbtrain,
num_rounds,
watchlist,
early_stopping_rounds=500)
model01 = xgboost.train(plst01,
xgbtrain01,
num_rounds,
watchlist01,
early_stopping_rounds=500)
# print model,watchlist
preds = model.predict(xgbtest, ntree_limit=model.best_iteration)
#preds01 = model01.predict(xgbtest01, ntree_limit=model.best_iteration)
model.save_model(
os.path.join(outpath, 'dem_maxtemp' + str(forehours) + '.model'))
#model01.save_model(os.path.join(outpath, 'dem_mintemp' + str(forehours) + '.model'))
# print preds, preds01
# 训练后高温的RMSE，MAE
mse = mean_squared_error(y_test, preds)
rmse = math.sqrt(mse)
mae = mean_absolute_error(y_test, preds, multioutput='uniform_average')
bias = numpy.mean(y_test - preds)
print("训练后最高气温MSE: %.4f" % mse)
print("训练后最高气温RMSE: %.4f" % rmse)
print("训练后最高气温MAE: %.4f" % mae)
print("训练后最高气温bias: %.4f" % bias)
# 训练后最高气温2度以内的准确率
n = 0
for x, y in zip(y_test, preds):
if abs(x - y) < 2:
n = n + 1
accuracy2_after = float(n) / float(len(y_test))
print("训练后最高气温2度的accuracy: %.4f" % accuracy2_after)
n = 0
for x, y in zip(y_test, preds):
if abs(x - y) < 3:
n = n + 1
accuracy3_after = float(n) / float(len(y_test))
print("训练后最高气温3度的accuracy: %.4f" % accuracy3_after)
# 训练前高温的RMSE,MAE
class_test = max_test.astype('float64')
max_ec = (class_test[:, 0]) - 273.15
mse0 = mean_squared_error(max_ec, y_test)
rmse0 = math.sqrt(mse0)
mae0 = mean_absolute_error(max_ec, y_test, multioutput='uniform_average')
bias0 = numpy.mean(y_test - max_ec)
print("训练前最高气温MSE: %.4f" % mse0)
print("训练前最高气温RMSE: %.4f" % rmse0)
print("训练前最高气温MAE: %.4f" % mae0)
print("训练前最高气温BIAS: %.4f" % bias0)
n = 0
for x, y in zip(y_test, max_ec):
if abs(x - y) < 2:
n = n + 1
accuracy2_before = float(n) / float(len(y_test))
print("训练前最高气温2度的accuracy: %.4f" % accuracy2_before)
n = 0
for x, y in zip(y_test, max_ec):
if abs(x - y) < 3:
n = n + 1
accuracy3_before = float(n) / float(len(y_test))
print("训练前最高气温3度的accuracy: %.4f" % accuracy3_before)
# 训练后低温的RMSE|MAE
min_mse = mean_squared_error(v_test, preds01)
min_rmse = math.sqrt(min_mse)
min_mae = mean_absolute_error(v_test,
preds01,
multioutput='uniform_average')
min_bias = numpy.mean(v_test - preds01)
print("训练后最低气温MSE: %.4f" % min_mse)
print("训练后最低气温RMSE: %.4f" % min_rmse)
print("训练后最低气温MAE: %.4f" % min_mae)
print("训练后最低气温BIAS: %.4f" % min_bias)
# 训练后最低气温2度以内的准确率
n = 0
for x, y in zip(v_test, preds01):
if abs(x - y) < 2:
n = n + 1
min_accuracy2_after = float(n) / float(len(v_test))
print("训练后最低气温2度的accuracy: %.4f" % min_accuracy2_after)
n = 0
for x, y in zip(v_test, preds01):
if abs(x - y) < 3:
n = n + 1
min_accuracy3_after = float(n) / float(len(v_test))
print("训练后最低气温3度的accuracy: %.4f" % min_accuracy3_after)
# 训练前低温的RMSE,MAE
min_ec = min_test[:, 0] - 273.15
min_mse0 = mean_squared_error(min_ec, v_test)
min_rmse0 = math.sqrt(min_mse0)
min_mae0 = mean_absolute_error(min_ec,
v_test,
multioutput='uniform_average')
min_bias0 = numpy.mean(v_test - min_ec)
print("训练前最低气温MSE: %.4f" % min_mse0)
print("训练前最低气温RMSE: %.4f" % min_rmse0)
print("训练前最低气温MAE: %.4f" % min_mae0)
print("训练前最低气温BIAS: %.4f" % min_bias0)
n = 0
for x, y in zip(v_test, min_ec):
if abs(x - y) < 2:
n = n + 1
min_accuracy2_before = float(n) / float(len(v_test))
print("训练前最低气温2度的accuracy: %.4f" % min_accuracy2_before)
n = 0
for x, y in zip(v_test, min_ec):
if abs(x - y) < 3:
n = n + 1
min_accuracy3_before = float(n) / float(len(v_test))
print("训练前最低气温3度的accuracy: %.4f" % min_accuracy3_before)
print str(rmse) + ',' + str(mae) + ',' + str(accuracy2_after) + ',' + str(
accuracy3_after) + ',' + str(rmse0) + ',' + str(mae0) + ',' + str(
accuracy2_before) + ',' + str(accuracy3_before) + ',' + str(
bias) + ',' + str(bias0) + ',' + str(min_rmse) + ',' + str(
min_mae) + ',' + str(min_accuracy2_after) + ',' + str(
min_accuracy3_after) + ',' + str(
min_rmse0) + ',' + str(min_mae0) + ',' + str(
min_accuracy2_before) + ',' + str(
min_accuracy3_before) + ',' + str(
min_bias) + ',' + str(min_bias0)
maxfile = os.path.join(outpath, 'demmaxt' + str(ll) + '.csv')
maxtfw = open(maxfile, 'w')
for pp in range(len(y_test)):
maxtfw.write(
str(max_ec[pp]) + ',' + str(preds[pp]) + ',' + str(y_test[pp]))
maxtfw.write('\n')
maxtfw.close()
# 输出最低气温
minfile = os.path.join(outpath, 'demmint' + str(ll) + '.csv')
mintfw = open(minfile, 'w')
for qq in range(len(v_test)):
mintfw.write(
str(min_ec[qq]) + ',' + str(preds01[qq]) + ',' + str(v_test[qq]))
mintfw.write('\n')
mintfw.close()
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--exp', dest='exp_name', required=True)
ap.add_argument('--threshold', dest='threshold', type=int, default=20)
args = ap.parse_args()
bounds = (slice(100, 180), slice(90, 170))
radar_elev, radar_lat, radar_lon = 1883, 41.151944, -104.806111
proj = setupMapProjection(goshen_1km_proj, goshen_1km_gs)
threshold = args.threshold
exp_name = args.exp_name
img_dir = "images-%s/ets_%ddBZ" % (exp_name, threshold)
map = Basemap(**proj)
radar_x, radar_y = map(radar_lon, radar_lat)
obs_base = "hdf/KCYS/1km/goshen.hdfrefl2d"
obs_times = np.array([int(f[-6:]) for f in glob.glob("%s*" % obs_base)])
fcst_files = glob.glob(
"/caps1/tsupinie/1km-control-%s/ena???.hdf014[47]00" % exp_name)
fcst_files.extend(
glob.glob("/caps1/tsupinie/1km-control-%s/ena???.hdf01[5678]?00" %
exp_name))
ens_refl, ens_members, ens_times = loadAndInterpolateEnsemble(
fcst_files, ['pt', 'p', 'qr', 'qs', 'qh'],
computeReflectivity,
"/caps1/tsupinie/1km-control-20120712/ena001.hdfgrdbas", {
'z_base': radar_elev,
'y_base': radar_y,
'x_base': radar_x,
'elev_angle': 0.5
},
agl=False,
wrap=True) #, aggregator=lambda x: np.mean(x, axis=0))
# ens_refl, ens_members, ens_times = loadAndInterpolateEnsemble(fcst_files, ['pt', 'p', 'qr', 'qs', 'qh'], computeReflectivity, "/caps1/tsupinie/1km-control-20120712/ena001.hdfgrdbas",
# {'z_base':radar_elev, 'y_base':radar_y, 'x_base':radar_x, 'elev_angle':0.5}, agl=False, wrap=True)
# ens_refl_mean = ens_refl.mean(axis=0)
refl_ens_mean = probMatchMean(ens_refl)
bounds_rev = [slice(None), slice(None)]
bounds_rev.extend(bounds[::-1])
bounds_rev = tuple(bounds_rev)
# refl_ens_mean = refl_ens_mean[bounds_rev[1:]]
# ens_refl = ens_refl[bounds_rev]
all_ets = np.empty((len(ens_members), len(ens_times)), dtype=np.float32)
all_ets_mean = np.empty((len(ens_times), ), dtype=np.float32)
all_confusion = np.empty(ens_refl.shape, dtype=np.int32)
all_confusion_mean = np.empty(refl_ens_mean.shape, dtype=np.int32)
for wdt, time in enumerate(ens_times):
idx = np.argmin(np.abs(obs_times - time))
if obs_times[idx] > time and idx > 0:
idx -= 1
bounds_obs = [0]
bounds_obs.extend(bounds[::-1])
bounds_obs = tuple(bounds_obs)
obs_file_name = "%s%06d" % (obs_base, obs_times[idx])
obs_hdf = nio.open_file(obs_file_name, mode='r', format='hdf')
obs_refl = obs_hdf.variables['refl2d'][0] #[bounds_obs]
all_ets_mean[wdt], all_confusion_mean[wdt] = ETS(
refl_ens_mean[wdt], obs_refl, threshold)
gs_x, gs_y = goshen_1km_gs
for lde, member in enumerate(ens_members):
all_ets[lde,
wdt], all_confusion[lde,
wdt] = ETS(ens_refl[lde, wdt],
obs_refl, threshold)
# nx, ny = ens_refl[lde, wdt].shape
# xs, ys = np.meshgrid( gs_x * np.arange(nx), gs_y * np.arange(ny) )
# pylab.clf()
# pylab.contourf(xs, ys, ens_refl[lde, wdt], levels=np.arange(10, 80, 10))
# pylab.colorbar()
# pylab.savefig("sweep_interp_%s_%06d.png" % (member, time))
nx, ny = refl_ens_mean[wdt].shape
xs, ys = np.meshgrid(gs_x * np.arange(nx), gs_y * np.arange(ny))
pylab.clf()
pylab.contourf(xs,
ys,
refl_ens_mean[wdt],
levels=np.arange(10, 80, 10))
pylab.colorbar()
pylab.savefig("%s/sweep_interp_mean_%06d.png" % (img_dir, time))
cPickle.dump(all_ets_mean, open("%s_%ddBZ.pkl" % (exp_name, threshold),
'w'), -1)
time_mean_ets = all_ets.mean(axis=1)
sort_mean_idxs = np.argsort(time_mean_ets)
pylab.clf()
for lde, member in enumerate(ens_members):
print sort_mean_idxs[lde] + 1, time_mean_ets[sort_mean_idxs[lde]]
pylab.plot(ens_times, all_ets[lde], 'r-', lw=0.75)
pylab.plot(ens_times, all_ets_mean, 'k-', lw=1.5)
y_lb, y_ub = pylab.ylim()
pylab.plot([14400, 14400], [y_lb, y_ub], 'k--', lw=0.5)
pylab.ylim([y_lb, y_ub])
pylab.xlim([10800, 18000])
pylab.xlabel("Time (s)")
pylab.ylabel("ETS")
pylab.savefig("%s/ets_swath_mm.png" % img_dir)
pylab.close()
for wdt, time in enumerate(ens_times):
fudge = 16
if threshold == 20:
fudge = 32
plotConfusion(
all_confusion_mean[wdt],
map,
goshen_1km_gs,
"Confusion for Reflectivity of the Ensemble Mean at time %06d" %
time,
"%s/confusion_mean_%06d.png" % (img_dir, time),
inset=flux_boxes[exp_name][wdt],
fudge=fudge)
# for lde, member in enumerate(ens_members):
# plotConfusion(all_confusion[lde, wdt], map, goshen_1km_gs, "Confusion for Reflectivity of Member %s at time %06d" % (member, time), "%s/confusion_ena%s_zoom_%06d.png" % (img_dir, member, time))
gc.collect()
return
if latbl == lattr or lonbl == lontr:
sys.exit('lat and lon values must be different')
else:
if latbl < lattr:
latbl, lattr = lattr, latbl
if lonbl > lontr:
lonbl, lontr = lontr, lonbl
# read in analysis files
a_fili = "analysis_gfs_4_%s_%s00_000.nc" % (init_dt[:8], init_dt[8:10])
# read pressure levels from analysis file
analysis = nio.open_file(diri + a_fili)
level_dim = analysis.variables["UGRD_P0_L100_GLL0"].dimensions[0]
levs_p1 = analysis.variables[level_dim]
levs_p = ['{:.0f}'.format(x) for x in levs_p1[:] / 100.0]
del levs_p1
# identify level index
lev1_index = levs_p.index(lev1)
lev2_index = levs_p.index(lev2)
# read in lat
lat1 = analysis.variables["lat_0"]
parser = argparse.ArgumentParser()
parser.add_argument('--data-file-SSTAYYC', dest='SSTAYYC_file')
parser.add_argument('--data-file-SSTAVAR', dest='SSTAVAR_file')
parser.add_argument('--domain-file', dest='domain_file')
parser.add_argument('--output-dir', dest='output_dir')
parser.add_argument('--casename', dest='casename')
parser.add_argument('--selected-month', type=int)
args = parser.parse_args()
print(str(args))
selected_month = args.selected_month
g = Nio.open_file(args.domain_file, "r")
lon = g.variables["xc"][1, :] #-- read clon
lat = g.variables["yc"][:, 1] #-- read clat
lon = ext_axis(lon)
print(lon)
f = Nio.open_file(args.SSTAYYC_file, "r")
data = f.variables["SSTAYYC"][selected_month-1, :, :]
missing_value = f.variables["SSTAYYC"]._FillValue[0]
data[np.isnan(data)] = missing_value
data = ext(data)
f.close()
import Ngl, Nio
import numpy as np
p0mb = 101325.
p0mb1 = 1013.25
interp = 1
extrap = False
pnew = [800., 750.]
lev = 0
f = Nio.open_file("/home/yumeng/realthesis/LowRes/LowRes_200610.01_tracer.nc")
# interpolation from sigma coordinate to pressure coordinate
# get parameters and variables for interpolation
hyam = f.variables["hyam"][:] / p0mb
hybm = f.variables["hybm"][:]
PS = f.variables["aps"][:, :, :]
f = Nio.open_file("/home/yumeng/realthesis/Uniform/AMRDUST.nc")
DU_CI = f.variables["CI"][:, :, :, :] * 1e6
DU_AI = f.variables["AI"][:, :, :, :] * 1e6
lon_Uni = f.variables["lon"][:]
lat_Uni = f.variables["lat"][:]
# start the interpolation
NewTracer_CI_Uni = Ngl.vinth2p(DU_CI[:, :, :, :], hyam, hybm, pnew,
PS[:, :, :], interp, p0mb1, 1, extrap)
NewTracer_AI_Uni = Ngl.vinth2p(DU_AI[:, :, :, :], hyam, hybm, pnew,
PS[:, :, :], interp, p0mb1, 1, extrap)
NewTracer_AI_Uni = np.ma.masked_where(NewTracer_AI_Uni == 1.e30,
NewTracer_AI_Uni)
print(np.max(NewTracer_AI_Uni))
# day 10 to 20
import Nio
#
# Open a netCDF file containing the geodesic grid and data on that grid.
#
# This grid came to us via Dave Randall, Todd Ringler, and Ross Heikes of
# CSU. The data for this mesh were originally downloaded from:
#
# http://kiwi.atmos.colostate.edu:16080/BUGS/projects/geodesic/
#
# The above URL doesn't seem to be active anymore. Here's a new URL:
#
# http://kiwi.atmos.colostate.edu/BUGS/geodesic/interpolate.html
#
dirc = Ngl.pynglpath("data")
cfile = Nio.open_file(dirc + "/cdf/hswm_d000000p000.g2.nc", "r")
#
# Read the grid centers and the kinetic energy into local variables.
#
r2d = 57.2957795 # radians to degrees
x = cfile.variables["grid_center_lon"][:] * r2d
y = cfile.variables["grid_center_lat"][:] * r2d
cx = cfile.variables["grid_corner_lon"][:] * r2d
cy = cfile.variables["grid_corner_lat"][:] * r2d
ke = cfile.variables["kinetic_energy"][2, :]
#
# Open a workstation.
#
wks_type = "png"
infile = os.path.join(
indir, '%s.%s.clim.%04d-%04d.nc' % (case, var, yrstart, yrend))
else:
infile = os.path.join(
indir, case,
'%s.%s.clim.%04d-%04d.nc' % (case, var, yrstart, yrend))
if os.path.isfile(infile): # is file/variable available?
for kz in range(len(dpthsin)):
zdpth = dpthsin[kz]
#----------------------------------------------------------------------
# read model data
print('reading %s' % (os.path.basename(infile)))
fpmod = Nio.open_file(infile, 'r')
zt = fpmod.variables['z_t'][:] / 100. # cm -> m
zw = fpmod.variables['z_w'][:] / 100. # cm -> m
dz = fpmod.variables['dz'][:] / 100. # cm -> m
area = fpmod.variables['TAREA'][:] * 1.e-4 # cm^2 -> m^2
variables[var]['fillvalue'] = fpmod.variables[var]._FillValue[
0]
# read specified depth
zind = N.searchsorted(zt, zdpth).item()
dpthmod = int(zt[zind])
data['mod'][var] = fpmod.variables[var][:, zind, :, :]
fpmod.close() # close model file
# compute temporal mean
def main(argv):
# Get command line stuff and store in a dictionary
s = 'verbose sumfile= indir= timeslice= nPC= sigMul= minPCFail= minRunFail= numRunFile= printVarTest'
optkeys = s.split()
try:
opts, args = getopt.getopt(argv, "h", optkeys)
except getopt.GetoptError:
pyEnsLib.CECT_usage()
sys.exit(2)
# Set the default value for options
opts_dict = {}
opts_dict['timeslice'] = 1
opts_dict['nPC'] = 50
opts_dict['sigMul'] = 2
opts_dict['verbose'] = False
opts_dict['minPCFail'] = 3
opts_dict['minRunFail'] = 2
opts_dict['numRunFile'] = 3
opts_dict['printVarTest'] = False
# Call utility library getopt_parseconfig to parse the option keys
# and save to the dictionary
caller = 'CECT'
gmonly = False
opts_dict = pyEnsLib.getopt_parseconfig(opts, optkeys, caller, opts_dict)
# Print out timestamp, input ensemble file and new run directory
dt = datetime.now()
verbose = opts_dict['verbose']
print '--------pyCECT--------'
print ' '
print dt.strftime("%A, %d. %B %Y %I:%M%p")
print ' '
print 'Ensemble summary file = ' + opts_dict['sumfile']
print ' '
print 'Cam output directory = ' + opts_dict['indir']
print ' '
print ' '
# Open all input files
ifiles = []
in_files_temp = os.listdir(opts_dict['indir'])
in_files = sorted(in_files_temp)
in_files_random = pyEnsLib.Random_pickup(in_files, opts_dict)
for frun_file in in_files_random:
if (os.path.isfile(opts_dict['indir'] + '/' + frun_file)):
ifiles.append(
Nio.open_file(opts_dict['indir'] + '/' + frun_file, "r"))
else:
print "COULD NOT LOCATE FILE " + opts_dict[
'indir'] + frun_file + " EXISTING"
sys.exit()
# Read all variables from the ensemble summary file
ens_var_name, ens_avg, ens_stddev, ens_rmsz, ens_gm, num_3d, mu_gm, sigma_gm, loadings_gm, sigma_scores_gm = pyEnsLib.read_ensemble_summary(
opts_dict['sumfile'])
if len(ens_rmsz) == 0:
gmonly = True
# Add ensemble rmsz and global mean to the dictionary "variables"
variables = {}
if not gmonly:
for k, v in ens_rmsz.iteritems():
pyEnsLib.addvariables(variables, k, 'zscoreRange', v)
for k, v in ens_gm.iteritems():
pyEnsLib.addvariables(variables, k, 'gmRange', v)
# Get 3d variable name list and 2d variable name list seperately
var_name3d = []
var_name2d = []
for vcount, v in enumerate(ens_var_name):
if vcount < num_3d:
var_name3d.append(v)
else:
var_name2d.append(v)
# Get ncol and nlev value
npts3d, npts2d, is_SE = pyEnsLib.get_ncol_nlev(ifiles[0])
# Compare the new run and the ensemble summary file to get rmsz score
results = {}
countzscore = np.zeros(len(ifiles), dtype=np.int32)
countgm = np.zeros(len(ifiles), dtype=np.int32)
if not gmonly:
for fcount, fid in enumerate(ifiles):
otimeSeries = fid.variables
for var_name in ens_var_name:
orig = otimeSeries[var_name]
Zscore, has_zscore = pyEnsLib.calculate_raw_score(
var_name, orig[opts_dict['timeslice']], npts3d, npts2d,
ens_avg, ens_stddev, is_SE)
if has_zscore:
# Add the new run rmsz zscore to the dictionary "results"
pyEnsLib.addresults(results, 'zscore', Zscore, var_name,
'f' + str(fcount))
# Evaluate the new run rmsz score if is in the range of the ensemble summary rmsz zscore range
for fcount, fid in enumerate(ifiles):
countzscore[fcount] = pyEnsLib.evaluatestatus(
'zscore', 'zscoreRange', variables, 'ens', results,
'f' + str(fcount))
# Calculate the new run global mean
mean3d, mean2d = pyEnsLib.generate_global_mean_for_summary(
ifiles, var_name3d, var_name2d, opts_dict['timeslice'], is_SE, verbose)
means = np.concatenate((mean3d, mean2d), axis=0)
# Add the new run global mean to the dictionary "results"
for i in range(means.shape[1]):
for j in range(means.shape[0]):
pyEnsLib.addresults(results, 'means', means[j][i], ens_var_name[j],
'f' + str(i))
# Evaluate the new run global mean if it is in the range of the ensemble summary global mean range
for fcount, fid in enumerate(ifiles):
countgm[fcount] = pyEnsLib.evaluatestatus('means', 'gmRange',
variables, 'gm', results,
'f' + str(fcount))
# Calculate the PCA scores of the new run
new_scores = pyEnsLib.standardized(means, mu_gm, sigma_gm, loadings_gm)
pyEnsLib.comparePCAscores(ifiles, new_scores, sigma_scores_gm, opts_dict)
# Print out
if opts_dict['printVarTest']:
print '*********************************************** '
print 'Variable-based testing (for reference only - not used to determine pass/fail)'
print '*********************************************** '
for fcount, fid in enumerate(ifiles):
print ' '
print 'Run ' + str(fcount + 1) + ":"
print ' '
if not gmonly:
print '***' + str(countzscore[fcount]), " of " + str(
len(ens_var_name)
) + ' variables are outside of ensemble RMSZ distribution***'
pyEnsLib.printsummary(results, 'ens', 'zscore', 'zscoreRange',
(fcount), variables, 'RMSZ')
print ' '
print '***' + str(countgm[fcount]), " of " + str(
len(ens_var_name)
) + ' variables are outside of ensemble global mean distribution***'
pyEnsLib.printsummary(results, 'gm', 'means', 'gmRange', fcount,
variables, 'global mean')
print ' '
print '----------------------------------------------------------------------------'
import numpy as np
# Test if file exists
filename = "3B-MO.MS.MRG.3IMERG.20140701-S000000-E235959.07.V03D.HDF5"
if (not os.path.exists(filename)):
print("You do not have the necessary '{}' HDF5 file to run this example.".
format(filename))
print(
"You need to supply your own HDF5 data or download the file from http://www.ncl.ucar.edu/Applications/Data/"
)
sys.exit()
# Be sure to read this file using the advanced file structure.
opt = Nio.options()
opt.FileStructure = 'advanced'
f = Nio.open_file(filename, "r", options=opt)
# Open group "Grid" which will now look like a regular NioFile
g = f.groups['Grid']
# Read data from this group
precip = g.variables['precipitation']
lat = g.variables['lat'][:]
lon = g.variables['lon'][:]
# Print the metadata of precip, and min/max values
print(precip)
print("min/max = {:g} / {:g}".format(precip[:].min(), precip[:].max()))
wks_type = "png"
wks = Ngl.open_wks(wks_type, "mask1")
import Ngl
import os.path
#This script takes Grib Files from NAM simulations and converts them
#Into .dat format for Newmanv3.1
t=0
Dates = ["20160609","20160610","20160611"]
Initials = ["0000","0600","1200","1800"]
Hours = ["000","001","002","003","004","005"]
for day in Dates:
for init in Initials:
for hr in Hours:
filename = "nam_218_"+day+"_"+init+"_"+hr+".grb2"
print filename
file = Nio.open_file(filename,"r")
names = file.variables.keys()
# level 30 is the 800 millibar level
uvar = file.variables["UGRD_P0_L100_GLC0"][:,:,:]
uvar = numpy.squeeze(uvar[30,:,:])
dim = numpy.shape(uvar)
vvar = file.variables["VGRD_P0_L100_GLC0"][:,:,:]
vvar = numpy.squeeze(vvar[30,:,:])
#hardcoded origin to save time
iorg = 195
jorg = 155
#choose -102 +103, want 2000km domain, with 500km bufferzone
#(total velocity field defined on 2500km^2) centered at (36.7964,-120.822),
res.cnLevels = numpy.arange(-12, 42, 2)
res.cnFillOn = True
res.cnFillPalette = "BlueYellowRed"
res.cnLinesOn = False
res.cnLineLabelsOn = False
res.cnInfoLabelOn = False
res.lbOrientation = "Horizontal"
return (res)
# Read in zonal winds
f = Nio.open_file("$NCARG_ROOT/lib/ncarg/data/cdf/uv300.nc", "r")
u = f.variables["U"]
lat = f.variables["lat"]
lon = f.variables["lon"]
# Start the graphics
wks_type = "png"
wks = Ngl.open_wks(wks_type, "./output/ngl_report/newcolor6")
res = set_common_resources()
# Set resources for contour/map plot
bres = set_common_resources()
bres.mpFillOn = False
bres.tiMainString = "Use transparency to emphasize a particular area"
def itp_dis():
if state.get() == 'normal':
if fnmatch.fnmatch(ent_agr_z.get().replace(' ', '').replace('.',''), '*[!0-9]*') or fnmatch.fnmatch(ent_ind_z.get().replace(' ', '').replace('.',''), '*[!0-9]*') or fnmatch.fnmatch(ent_pow_z.get().replace(' ', '').replace('.',''), '*[!0-9]*') or fnmatch.fnmatch(ent_res_z.get().replace(' ', '').replace('.',''), '*[!0-9]*') or fnmatch.fnmatch(ent_tra_z.get().replace(' ', '').replace('.',''), '*[!0-9]*') or fnmatch.fnmatch(ent_agr_t.get().replace(' ', '').replace('.',''), '*[!0-9]*') or fnmatch.fnmatch(ent_ind_t.get().replace(' ', '').replace('.',''), '*[!0-9]*') or fnmatch.fnmatch(ent_pow_t.get().replace(' ', '').replace('.',''), '*[!0-9]*') or fnmatch.fnmatch(ent_res_t.get().replace(' ', '').replace('.',''), '*[!0-9]*') or fnmatch.fnmatch(ent_tra_t.get().replace(' ', '').replace('.',''), '*[!0-9]*'):
tk.messagebox.showerror(title='error',message='input format must be 0.23 0.34 ...')
elif not len(ent_agr_z.get().strip().split(' '))==len(ent_ind_z.get().strip().split(' '))==len(ent_pow_z.get().strip().split(' '))==len(ent_res_z.get().strip().split(' '))==len(ent_tra_z.get().strip().split(' ')):
tk.messagebox.showerror(title='error',message='number of z factor must be equal')
elif not len(ent_agr_t.get().strip().split(' '))==len(ent_ind_t.get().strip().split(' '))==len(ent_pow_t.get().strip().split(' '))==len(ent_res_t.get().strip().split(' '))==len(ent_tra_t.get().strip().split(' '))==24:
tk.messagebox.showerror(title='error',message='number of t factor must be 24')
else:
agr_z=[float(i) for i in ent_agr_z.get().strip().split(' ')]
ind_z=[float(i) for i in ent_ind_z.get().strip().split(' ')]
pow_z=[float(i) for i in ent_pow_z.get().strip().split(' ')]
res_z=[float(i) for i in ent_res_z.get().strip().split(' ')]
tra_z=[float(i) for i in ent_tra_z.get().strip().split(' ')]
sec_z=[agr_z,ind_z,pow_z,res_z,tra_z,]
agr_t=[float(i) for i in ent_agr_t.get().strip().split(' ')]
ind_t=[float(i) for i in ent_ind_t.get().strip().split(' ')]
pow_t=[float(i) for i in ent_pow_t.get().strip().split(' ')]
res_t=[float(i) for i in ent_res_t.get().strip().split(' ')]
tra_t=[float(i) for i in ent_tra_t.get().strip().split(' ')]
sec_t=[agr_t,ind_t,pow_t,res_t,tra_t,]
f_inp=Nio.open_file(ent_inp.get(),format='nc')
lon_inp=f_inp.variables['XLONG'][0,:]
lat_inp=f_inp.variables['XLAT'][0,:]
time_inp=f_inp.variables['Times'][:][0]
time_inp=''.join([i.decode('utf-8') for i in time_inp]).split('_')[0]
f_inp.close()
#put all the distributed meic species into meic_spec_emis:
meic_spec_emis=[]
#inorganic gas: ton/(grid.month) to mole/(km2.h)
for spec,M in zip(['CO','CO2','NH3','NOx','SO2',],[28,44,17,46,64]):
f_post=Nio.open_file(ent_dir.get()+'/merged/'+spec+'.nc')
lon=f_post.variables['lon'][:]
lat=f_post.variables['lat'][:]
section=[(f_post.variables[sec][:,:]*1e6)/(ll_area(lat,0.25)*30*24*M) for sec in ['act','idt','pwr','rdt','tpt',]]
f_post.close()
sections=[meic2wrf(lon_inp,lat_inp,lon,lat,emis,) for emis in section]
c=[sec2zt(i,j,k) for i,j,k in zip(sections,sec_z,sec_t)]
c=sum(c)
meic_spec_emis.append(c)
#organic gas: million_mole/(grid.month) to mole/(km2.h)
for spec in ['ALD','CSL','ETH','GLY','HC3','HC5','HC8','HCHO','ISO','KET','MACR','MGLY','MVK','NR','NVOL',
'OL2','OLI','OLT','ORA1','ORA2','TOL','XYL',]:
f_post=Nio.open_file(ent_dir.get()+'/merged/'+spec+'.nc')
#lon=f_post.variables['lon'][:]
#lat=f_post.variables['lat'][:]
section=[(f_post.variables[sec][:,:]*1e6)/(ll_area(lat,0.25)*30*24) for sec in ['act','idt','pwr','rdt','tpt',]]
f_post.close()
sections=[meic2wrf(lon_inp,lat_inp,lon,lat,emis,) for emis in section]
c=[sec2zt(i,j,k) for i,j,k in zip(sections,sec_z,sec_t)]
c=sum(c)
meic_spec_emis.append(c)
#aerosol: ton/(grid.month) to ug/(m2.s)
for spec in ['BC','OC','PM2.5','PMcoarse',]:
f_post=Nio.open_file(ent_dir.get()+'/merged/'+spec+'.nc')
#lon=f_post.variables['lon'][:]
#lat=f_post.variables['lat'][:]
section=[(f_post.variables[sec][:,:]*1e6)/(ll_area(lat,0.25)*30*24*3600) for sec in ['act','idt','pwr','rdt','tpt',]]
f_post.close()
sections=[meic2wrf(lon_inp,lat_inp,lon,lat,emis,) for emis in section]
c=[sec2zt(i,j,k) for i,j,k in zip(sections,sec_z,sec_t)]
c=sum(c)
meic_spec_emis.append(c)
#meic emission to RADM2 chemistry scheme:
wrf_spec_emis=[np.zeros(meic_spec_emis[0][:].shape, dtype='float32')]*31
wrf_spec_emis[0]=meic_spec_emis[0] #wrf: CO
wrf_spec_emis[1]=meic_spec_emis[2] #wrf: NH3
wrf_spec_emis[2]=meic_spec_emis[3]*0.9 #wrf: NO
wrf_spec_emis[3]=meic_spec_emis[3]*0.1 #wrf: NO2
wrf_spec_emis[4]=meic_spec_emis[4]*0.9 #wrf: SO2
wrf_spec_emis[5]=meic_spec_emis[5] #wrf: ALD
wrf_spec_emis[6]=meic_spec_emis[6] #wrf: CSL
wrf_spec_emis[7]=meic_spec_emis[7] #wrf: ETH
wrf_spec_emis[8]=meic_spec_emis[9] #wrf: HC3
wrf_spec_emis[9]=meic_spec_emis[10] #wrf: HC5
wrf_spec_emis[10]=meic_spec_emis[11] #wrf: HC8
wrf_spec_emis[11]=meic_spec_emis[12] #wrf: HCHO
wrf_spec_emis[12]=meic_spec_emis[13] #wrf: ISO
wrf_spec_emis[13]=meic_spec_emis[14] #wrf: KET
wrf_spec_emis[14]=meic_spec_emis[20]*1.1 #wrf: OL2
wrf_spec_emis[15]=meic_spec_emis[21]*1.1 #wrf: OLI
wrf_spec_emis[16]=meic_spec_emis[22]*1.1 #wrf: OLT
wrf_spec_emis[17]=meic_spec_emis[24] #wrf: ORA2
wrf_spec_emis[18]=meic_spec_emis[25]*1.1 #wrf: TOL
wrf_spec_emis[19]=meic_spec_emis[26]*1.1 #wrf: XYL
wrf_spec_emis[20]=meic_spec_emis[27]*0.2 #wrf: ECi
wrf_spec_emis[21]=meic_spec_emis[27]*0.8 #wrf: ECj
wrf_spec_emis[22]=meic_spec_emis[28]*0.2 #wrf: ORGi
wrf_spec_emis[23]=meic_spec_emis[28]*0.8 #wrf: ORGj
wrf_spec_emis[24]=meic_spec_emis[29]-meic_spec_emis[28]-meic_spec_emis[27]*0.2 #wrf: PM25i
wrf_spec_emis[25]=meic_spec_emis[29]-meic_spec_emis[28]-meic_spec_emis[27]*0.8 #wrf: PM25j
wrf_spec_emis[26]=meic_spec_emis[30]*0.8 #wrf: PM10
wrf_spec_emis[27]=np.zeros(meic_spec_emis[0][:].shape, dtype='float32') #wrf: SO4i
wrf_spec_emis[28]=np.zeros(meic_spec_emis[0][:].shape, dtype='float32') #wrf: SO4j
wrf_spec_emis[29]=np.zeros(meic_spec_emis[0][:].shape, dtype='float32') #wrf: NO3i
wrf_spec_emis[30]=np.zeros(meic_spec_emis[0][:].shape, dtype='float32') #wrf: NO3j
#generate wrfchemi_00z_d01 anthropogenic emission data for wrf-chem model run:
if os.path.exists(ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1]):
os.remove(ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1])
f_chem=Nio.open_file(ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1],'c',format='nc')
f_chem.create_dimension('Time',None)
f_chem.create_dimension('emissions_zdim',wrf_spec_emis[0].shape[1])
f_chem.create_dimension('south_north',wrf_spec_emis[0].shape[2])
f_chem.create_dimension('west_east',wrf_spec_emis[0].shape[3])
f_chem.create_dimension('DateStrLen',19)
f_chem.create_variable('Times','S1',('Time','DateStrLen'),)
for i,time in enumerate([time_inp+'_00:00:00',time_inp+'_01:00:00',time_inp+'_02:00:00',time_inp+'_03:00:00',time_inp+'_04:00:00',
time_inp+'_05:00:00',time_inp+'_06:00:00',time_inp+'_07:00:00',time_inp+'_08:00:00',time_inp+'_09:00:00',time_inp+'_10:00:00',
time_inp+'_11:00:00',]):
f_chem.variables['Times'][i]=list(time) #split the string to char
for ll, LL in zip([lon_inp, lat_inp],['XLONG', 'XLAT']):
f_chem.create_variable(LL, 'f', ('south_north', 'west_east',),)
f_chem.variables[LL][:]=ll
radm_gas=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL',]
radm_aerosol=['E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I','E_SO4J','E_NO3I','E_NO3J',]
for gas in radm_gas:
f_chem.create_variable(gas,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[gas].FieldType = np.int16(104)
f_chem.variables[gas].MemoryOrder = 'XYZ'
f_chem.variables[gas].description = 'EMISSIONS'
f_chem.variables[gas].units = 'mol km^-2 hr^-1'
f_chem.variables[gas].stagger = 'Z'
#f_chem.variables[gas].ordinates = 'XLONG XLAT'
for aerosol in radm_aerosol:
f_chem.create_variable(aerosol,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[aerosol].FieldType = np.int16(104)
f_chem.variables[aerosol].MemoryOrder = 'XYZ'
f_chem.variables[aerosol].description = 'EMISSIONS'
f_chem.variables[aerosol].units = 'ug/m3 m/s'
f_chem.variables[aerosol].stagger = 'Z'
#f_chem.variables[aerosol].ordinates = 'XLONG XLAT'
radm_spec=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL','E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I',
'E_SO4J','E_NO3I','E_NO3J',]
for i,spec in enumerate(radm_spec):
f_chem.variables[spec][:] = wrf_spec_emis[i][0:12,:,:,:] #dimension need to be matched with the variable defination
f_chem.close()
os.rename(ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1]+'.nc', ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1])
else:
f_chem=Nio.open_file(ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1],'c',format='nc')
f_chem.create_dimension('Time',None)
f_chem.create_dimension('emissions_zdim',wrf_spec_emis[0].shape[1])
f_chem.create_dimension('south_north',wrf_spec_emis[0].shape[2])
f_chem.create_dimension('west_east',wrf_spec_emis[0].shape[3])
f_chem.create_dimension('DateStrLen',19)
f_chem.create_variable('Times','S1',('Time','DateStrLen'),)
for i,time in enumerate([time_inp+'_00:00:00',time_inp+'_01:00:00',time_inp+'_02:00:00',time_inp+'_03:00:00',time_inp+'_04:00:00',
time_inp+'_05:00:00',time_inp+'_06:00:00',time_inp+'_07:00:00',time_inp+'_08:00:00',time_inp+'_09:00:00',time_inp+'_10:00:00',
time_inp+'_11:00:00',]):
f_chem.variables['Times'][i]=list(time) #split the string to char
for ll, LL in zip([lon_inp, lat_inp],['XLONG', 'XLAT']):
f_chem.create_variable(LL, 'f', ('south_north', 'west_east',),)
f_chem.variables[LL][:]=ll
radm_gas=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL',]
radm_aerosol=['E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I','E_SO4J','E_NO3I','E_NO3J',]
for gas in radm_gas:
f_chem.create_variable(gas,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[gas].FieldType = np.int16(104)
f_chem.variables[gas].MemoryOrder = 'XYZ'
f_chem.variables[gas].description = 'EMISSIONS'
f_chem.variables[gas].units = 'mol km^-2 hr^-1'
f_chem.variables[gas].stagger = 'Z'
#f_chem.variables[gas].ordinates = 'XLONG XLAT'
for aerosol in radm_aerosol:
f_chem.create_variable(aerosol,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[aerosol].FieldType = np.int16(104)
f_chem.variables[aerosol].MemoryOrder = 'XYZ'
f_chem.variables[aerosol].description = 'EMISSIONS'
f_chem.variables[aerosol].units = 'ug/m3 m/s'
f_chem.variables[aerosol].stagger = 'Z'
#f_chem.variables[aerosol].ordinates = 'XLONG XLAT'
radm_spec=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL','E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I',
'E_SO4J','E_NO3I','E_NO3J',]
for i,spec in enumerate(radm_spec):
f_chem.variables[spec][:] = wrf_spec_emis[i][0:12,:,:,:] #dimension need to be matched with the variable defination
f_chem.close()
os.rename(ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1]+'.nc', ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1])
#generate wrfchemi_12z_d01 anthropogenic emission data for wrf-chem model run:
if os.path.exists(ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1]):
os.remove(ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1])
f_chem=Nio.open_file(ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1],'c',format='nc')
f_chem.create_dimension('Time',None)
f_chem.create_dimension('emissions_zdim',wrf_spec_emis[0].shape[1])
f_chem.create_dimension('south_north',wrf_spec_emis[0].shape[2])
f_chem.create_dimension('west_east',wrf_spec_emis[0].shape[3])
f_chem.create_dimension('DateStrLen',19)
f_chem.create_variable('Times','S1',('Time','DateStrLen'),)
for i,time in enumerate([time_inp+'_12:00:00',time_inp+'_13:00:00',time_inp+'_14:00:00',time_inp+'_15:00:00',time_inp+'_16:00:00',
time_inp+'_17:00:00',time_inp+'_18:00:00',time_inp+'_19:00:00',time_inp+'_20:00:00',time_inp+'_21:00:00',time_inp+'_22:00:00',
time_inp+'_23:00:00',]):
f_chem.variables['Times'][i]=list(time)
for ll, LL in zip([lon_inp, lat_inp],['XLONG', 'XLAT']):
f_chem.create_variable(LL, 'f', ('south_north', 'west_east',),)
f_chem.variables[LL][:]=ll
radm_gas=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL',]
radm_aerosol=['E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I','E_SO4J','E_NO3I','E_NO3J',]
for gas in radm_gas:
f_chem.create_variable(gas,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[gas].FieldType = np.int16(104)
f_chem.variables[gas].MemoryOrder = 'XYZ'
f_chem.variables[gas].description = 'EMISSIONS'
f_chem.variables[gas].units = 'mol km^-2 hr^-1'
f_chem.variables[gas].stagger = 'Z'
#f_chem.variables[gas].ordinates = 'XLONG XLAT'
for aerosol in radm_aerosol:
f_chem.create_variable(aerosol,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[aerosol].FieldType = np.int16(104)
f_chem.variables[aerosol].MemoryOrder = 'XYZ'
f_chem.variables[aerosol].description = 'EMISSIONS'
f_chem.variables[aerosol].units = 'ug/m3 m/s'
f_chem.variables[aerosol].stagger = 'Z'
#f_chem.variables[aerosol].ordinates = 'XLONG XLAT'
radm_spec=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL','E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I',
'E_SO4J','E_NO3I','E_NO3J',]
for i,spec in enumerate(radm_spec):
f_chem.variables[spec][:] = wrf_spec_emis[i][12:24,:,:,:] #dimension need to be matched with the variable defination
f_chem.close()
os.rename(ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1]+'.nc', ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1])
else:
f_chem=Nio.open_file(ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1],'c',format='nc')
f_chem.create_dimension('Time',None)
f_chem.create_dimension('emissions_zdim',wrf_spec_emis[0].shape[1])
f_chem.create_dimension('south_north',wrf_spec_emis[0].shape[2])
f_chem.create_dimension('west_east',wrf_spec_emis[0].shape[3])
f_chem.create_dimension('DateStrLen',19)
f_chem.create_variable('Times','S1',('Time','DateStrLen'),)
for i,time in enumerate([time_inp+'_12:00:00',time_inp+'_13:00:00',time_inp+'_14:00:00',time_inp+'_15:00:00',time_inp+'_16:00:00',
time_inp+'_17:00:00',time_inp+'_18:00:00',time_inp+'_19:00:00',time_inp+'_20:00:00',time_inp+'_21:00:00',time_inp+'_22:00:00',
time_inp+'_23:00:00',]):
f_chem.variables['Times'][i]=list(time)
for ll, LL in zip([lon_inp, lat_inp],['XLONG', 'XLAT']):
f_chem.create_variable(LL, 'f', ('south_north', 'west_east',),)
f_chem.variables[LL][:]=ll
radm_gas=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL',]
radm_aerosol=['E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I','E_SO4J','E_NO3I','E_NO3J',]
for gas in radm_gas:
f_chem.create_variable(gas,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[gas].FieldType = np.int16(104)
f_chem.variables[gas].MemoryOrder = 'XYZ'
f_chem.variables[gas].description = 'EMISSIONS'
f_chem.variables[gas].units = 'mol km^-2 hr^-1'
f_chem.variables[gas].stagger = 'Z'
#f_chem.variables[gas].ordinates = 'XLONG XLAT'
for aerosol in radm_aerosol:
f_chem.create_variable(aerosol,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[aerosol].FieldType = np.int16(104)
f_chem.variables[aerosol].MemoryOrder = 'XYZ'
f_chem.variables[aerosol].description = 'EMISSIONS'
f_chem.variables[aerosol].units = 'ug/m3 m/s'
f_chem.variables[aerosol].stagger = 'Z'
#f_chem.variables[aerosol].ordinates = 'XLONG XLAT'
radm_spec=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL','E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I',
'E_SO4J','E_NO3I','E_NO3J',]
for i,spec in enumerate(radm_spec):
f_chem.variables[spec][:] = wrf_spec_emis[i][12:24,:,:,:] #dimension need to be matched with the variable defination
f_chem.close()
os.rename(ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1]+'.nc', ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1])
elif state.get() =='disable':
f_inp=Nio.open_file(ent_inp.get(),format='nc')
lon_inp=f_inp.variables['XLONG'][0,:]
lat_inp=f_inp.variables['XLAT'][0,:]
time_inp=f_inp.variables['Times'][:][0]
time_inp=''.join([i.decode('utf-8') for i in time_inp]).split('_')[0]
f_inp.close()
#put all the distributed meic species into meic_spec_emis:
meic_spec_emis=[]
#inorganic gas: ton/(grid.month) to mole/(km2.h)
for spec,M in zip(['CO','CO2','NH3','NOx','SO2',],[28,44,17,46,64]):
f_post=Nio.open_file(ent_dir.get()+'/merged/'+spec+'.nc')
lon=f_post.variables['lon'][:]
lat=f_post.variables['lat'][:]
section=[(f_post.variables[sec][:,:]*1e6)/(ll_area(lat,0.25)*30*24*M) for sec in ['act','idt','pwr','rdt','tpt',]]
f_post.close()
sections=[meic2wrf(lon_inp,lat_inp,lon,lat,emis,) for emis in section]
c=[sec2zt(i,j,k) for i,j,k in zip(sections,sec_z_d,sec_t_d)]
c=sum(c)
meic_spec_emis.append(c)
#organic gas: million_mole/(grid.month) to mole/(km2.h)
for spec in ['ALD','CSL','ETH','GLY','HC3','HC5','HC8','HCHO','ISO','KET','MACR','MGLY','MVK','NR','NVOL',
'OL2','OLI','OLT','ORA1','ORA2','TOL','XYL',]:
f_post=Nio.open_file(ent_dir.get()+'/merged/'+spec+'.nc')
#lon=f_post.variables['lon'][:]
#lat=f_post.variables['lat'][:]
section=[(f_post.variables[sec][:,:]*1e6)/(ll_area(lat,0.25)*30*24) for sec in ['act','idt','pwr','rdt','tpt',]]
f_post.close()
sections=[meic2wrf(lon_inp,lat_inp,lon,lat,emis,) for emis in section]
c=[sec2zt(i,j,k) for i,j,k in zip(sections,sec_z_d,sec_t_d)]
c=sum(c)
meic_spec_emis.append(c)
#aerosol: ton/(grid.month) to ug/(m2.s)
for spec in ['BC','OC','PM2.5','PMcoarse',]:
f_post=Nio.open_file(ent_dir.get()+'/merged/'+spec+'.nc')
#lon=f_post.variables['lon'][:]
#lat=f_post.variables['lat'][:]
section=[(f_post.variables[sec][:,:]*1e6)/(ll_area(lat,0.25)*30*24*3600) for sec in ['act','idt','pwr','rdt','tpt',]]
f_post.close()
sections=[meic2wrf(lon_inp,lat_inp,lon,lat,emis,) for emis in section]
c=[sec2zt(i,j,k) for i,j,k in zip(sections,sec_z_d,sec_t_d)]
c=sum(c)
meic_spec_emis.append(c)
#meic emission to RADM2 chemistry scheme:
wrf_spec_emis=[np.zeros(meic_spec_emis[0][:].shape, dtype='float32')]*31
wrf_spec_emis[0]=meic_spec_emis[0] #wrf: CO
wrf_spec_emis[1]=meic_spec_emis[2] #wrf: NH3
wrf_spec_emis[2]=meic_spec_emis[3]*0.9 #wrf: NO
wrf_spec_emis[3]=meic_spec_emis[3]*0.1 #wrf: NO2
wrf_spec_emis[4]=meic_spec_emis[4]*0.9 #wrf: SO2
wrf_spec_emis[5]=meic_spec_emis[5] #wrf: ALD
wrf_spec_emis[6]=meic_spec_emis[6] #wrf: CSL
wrf_spec_emis[7]=meic_spec_emis[7] #wrf: ETH
wrf_spec_emis[8]=meic_spec_emis[9] #wrf: HC3
wrf_spec_emis[9]=meic_spec_emis[10] #wrf: HC5
wrf_spec_emis[10]=meic_spec_emis[11] #wrf: HC8
wrf_spec_emis[11]=meic_spec_emis[12] #wrf: HCHO
wrf_spec_emis[12]=meic_spec_emis[13] #wrf: ISO
wrf_spec_emis[13]=meic_spec_emis[14] #wrf: KET
wrf_spec_emis[14]=meic_spec_emis[20]*1.1 #wrf: OL2
wrf_spec_emis[15]=meic_spec_emis[21]*1.1 #wrf: OLI
wrf_spec_emis[16]=meic_spec_emis[22]*1.1 #wrf: OLT
wrf_spec_emis[17]=meic_spec_emis[24] #wrf: ORA2
wrf_spec_emis[18]=meic_spec_emis[25]*1.1 #wrf: TOL
wrf_spec_emis[19]=meic_spec_emis[26]*1.1 #wrf: XYL
wrf_spec_emis[20]=meic_spec_emis[27]*0.2 #wrf: ECi
wrf_spec_emis[21]=meic_spec_emis[27]*0.8 #wrf: ECj
wrf_spec_emis[22]=meic_spec_emis[28]*0.2 #wrf: ORGi
wrf_spec_emis[23]=meic_spec_emis[28]*0.8 #wrf: ORGj
wrf_spec_emis[24]=meic_spec_emis[29]-meic_spec_emis[28]-meic_spec_emis[27]*0.2 #wrf: PM25i
wrf_spec_emis[25]=meic_spec_emis[29]-meic_spec_emis[28]-meic_spec_emis[27]*0.8 #wrf: PM25j
wrf_spec_emis[26]=meic_spec_emis[30]*0.8 #wrf: PM10
wrf_spec_emis[27]=np.zeros(meic_spec_emis[0][:].shape, dtype='float32') #wrf: SO4i
wrf_spec_emis[28]=np.zeros(meic_spec_emis[0][:].shape, dtype='float32') #wrf: SO4j
wrf_spec_emis[29]=np.zeros(meic_spec_emis[0][:].shape, dtype='float32') #wrf: NO3i
wrf_spec_emis[30]=np.zeros(meic_spec_emis[0][:].shape, dtype='float32') #wrf: NO3j
#generate wrfchemi_00z_d01 anthropogenic emission data for wrf-chem model run:
if os.path.exists(ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1]):
os.remove(ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1])
f_chem=Nio.open_file(ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1],'c',format='nc')
f_chem.create_dimension('Time',None)
f_chem.create_dimension('emissions_zdim',wrf_spec_emis[0].shape[1])
f_chem.create_dimension('south_north',wrf_spec_emis[0].shape[2])
f_chem.create_dimension('west_east',wrf_spec_emis[0].shape[3])
f_chem.create_dimension('DateStrLen',19)
f_chem.create_variable('Times','S1',('Time','DateStrLen'),)
for i,time in enumerate([time_inp+'_00:00:00',time_inp+'_01:00:00',time_inp+'_02:00:00',time_inp+'_03:00:00',time_inp+'_04:00:00',
time_inp+'_05:00:00',time_inp+'_06:00:00',time_inp+'_07:00:00',time_inp+'_08:00:00',time_inp+'_09:00:00',time_inp+'_10:00:00',
time_inp+'_11:00:00',]):
f_chem.variables['Times'][i]=list(time) #split the string to char
for ll, LL in zip([lon_inp, lat_inp],['XLONG', 'XLAT']):
f_chem.create_variable(LL, 'f', ('south_north', 'west_east',),)
f_chem.variables[LL][:]=ll
radm_gas=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL',]
radm_aerosol=['E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I','E_SO4J','E_NO3I','E_NO3J',]
for gas in radm_gas:
f_chem.create_variable(gas,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[gas].FieldType = np.int16(104)
f_chem.variables[gas].MemoryOrder = 'XYZ'
f_chem.variables[gas].description = 'EMISSIONS'
f_chem.variables[gas].units = 'mol km^-2 hr^-1'
f_chem.variables[gas].stagger = 'Z'
#f_chem.variables[gas].ordinates = 'XLONG XLAT'
for aerosol in radm_aerosol:
f_chem.create_variable(aerosol,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[aerosol].FieldType = np.int16(104)
f_chem.variables[aerosol].MemoryOrder = 'XYZ'
f_chem.variables[aerosol].description = 'EMISSIONS'
f_chem.variables[aerosol].units = 'ug/m3 m/s'
f_chem.variables[aerosol].stagger = 'Z'
#f_chem.variables[aerosol].ordinates = 'XLONG XLAT'
radm_spec=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL','E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I',
'E_SO4J','E_NO3I','E_NO3J',]
for i,spec in enumerate(radm_spec):
f_chem.variables[spec][:] = wrf_spec_emis[i][0:12,:,:,:] #dimension need to be matched with the variable defination
f_chem.close()
os.rename(ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1]+'.nc', ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1])
else:
f_chem=Nio.open_file(ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1],'c',format='nc')
f_chem.create_dimension('Time',None)
f_chem.create_dimension('emissions_zdim',wrf_spec_emis[0].shape[1])
f_chem.create_dimension('south_north',wrf_spec_emis[0].shape[2])
f_chem.create_dimension('west_east',wrf_spec_emis[0].shape[3])
f_chem.create_dimension('DateStrLen',19)
f_chem.create_variable('Times','S1',('Time','DateStrLen'),)
for i,time in enumerate([time_inp+'_00:00:00',time_inp+'_01:00:00',time_inp+'_02:00:00',time_inp+'_03:00:00',time_inp+'_04:00:00',
time_inp+'_05:00:00',time_inp+'_06:00:00',time_inp+'_07:00:00',time_inp+'_08:00:00',time_inp+'_09:00:00',time_inp+'_10:00:00',
time_inp+'_11:00:00',]):
f_chem.variables['Times'][i]=list(time) #split the string to char
for ll, LL in zip([lon_inp, lat_inp],['XLONG', 'XLAT']):
f_chem.create_variable(LL, 'f', ('south_north', 'west_east',),)
f_chem.variables[LL][:]=ll
radm_gas=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL',]
radm_aerosol=['E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I','E_SO4J','E_NO3I','E_NO3J',]
for gas in radm_gas:
f_chem.create_variable(gas,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[gas].FieldType = np.int16(104)
f_chem.variables[gas].MemoryOrder = 'XYZ'
f_chem.variables[gas].description = 'EMISSIONS'
f_chem.variables[gas].units = 'mol km^-2 hr^-1'
f_chem.variables[gas].stagger = 'Z'
#f_chem.variables[gas].ordinates = 'XLONG XLAT'
for aerosol in radm_aerosol:
f_chem.create_variable(aerosol,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[aerosol].FieldType = np.int16(104)
f_chem.variables[aerosol].MemoryOrder = 'XYZ'
f_chem.variables[aerosol].description = 'EMISSIONS'
f_chem.variables[aerosol].units = 'ug/m3 m/s'
f_chem.variables[aerosol].stagger = 'Z'
#f_chem.variables[aerosol].ordinates = 'XLONG XLAT'
radm_spec=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL','E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I',
'E_SO4J','E_NO3I','E_NO3J',]
for i,spec in enumerate(radm_spec):
f_chem.variables[spec][:] = wrf_spec_emis[i][0:12,:,:,:] #dimension need to be matched with the variable defination
f_chem.close()
os.rename(ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1]+'.nc', ent_dir.get()+'/merged/'+'wrfchemi_00z_'+ent_inp.get().split('_')[-1])
#generate wrfchemi_12z_d01 anthropogenic emission data for wrf-chem model run:
if os.path.exists(ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1]):
os.remove(ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1])
f_chem=Nio.open_file(ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1],'c',format='nc')
f_chem.create_dimension('Time',None)
f_chem.create_dimension('emissions_zdim',wrf_spec_emis[0].shape[1])
f_chem.create_dimension('south_north',wrf_spec_emis[0].shape[2])
f_chem.create_dimension('west_east',wrf_spec_emis[0].shape[3])
f_chem.create_dimension('DateStrLen',19)
f_chem.create_variable('Times','S1',('Time','DateStrLen'),)
for i,time in enumerate([time_inp+'_12:00:00',time_inp+'_13:00:00',time_inp+'_14:00:00',time_inp+'_15:00:00',time_inp+'_16:00:00',
time_inp+'_17:00:00',time_inp+'_18:00:00',time_inp+'_19:00:00',time_inp+'_20:00:00',time_inp+'_21:00:00',time_inp+'_22:00:00',
time_inp+'_23:00:00',]):
f_chem.variables['Times'][i]=list(time)
for ll, LL in zip([lon_inp, lat_inp],['XLONG', 'XLAT']):
f_chem.create_variable(LL, 'f', ('south_north', 'west_east',),)
f_chem.variables[LL][:]=ll
radm_gas=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL',]
radm_aerosol=['E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I','E_SO4J','E_NO3I','E_NO3J',]
for gas in radm_gas:
f_chem.create_variable(gas,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[gas].FieldType = np.int16(104)
f_chem.variables[gas].MemoryOrder = 'XYZ'
f_chem.variables[gas].description = 'EMISSIONS'
f_chem.variables[gas].units = 'mol km^-2 hr^-1'
f_chem.variables[gas].stagger = 'Z'
#f_chem.variables[gas].ordinates = 'XLONG XLAT'
for aerosol in radm_aerosol:
f_chem.create_variable(aerosol,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[aerosol].FieldType = np.int16(104)
f_chem.variables[aerosol].MemoryOrder = 'XYZ'
f_chem.variables[aerosol].description = 'EMISSIONS'
f_chem.variables[aerosol].units = 'ug/m3 m/s'
f_chem.variables[aerosol].stagger = 'Z'
#f_chem.variables[aerosol].ordinates = 'XLONG XLAT'
radm_spec=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL','E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I',
'E_SO4J','E_NO3I','E_NO3J',]
for i,spec in enumerate(radm_spec):
f_chem.variables[spec][:] = wrf_spec_emis[i][12:24,:,:,:] #dimension need to be matched with the variable defination
f_chem.close()
os.rename(ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1]+'.nc', ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1])
else:
f_chem=Nio.open_file(ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1],'c',format='nc')
f_chem.create_dimension('Time',None)
f_chem.create_dimension('emissions_zdim',wrf_spec_emis[0].shape[1])
f_chem.create_dimension('south_north',wrf_spec_emis[0].shape[2])
f_chem.create_dimension('west_east',wrf_spec_emis[0].shape[3])
f_chem.create_dimension('DateStrLen',19)
f_chem.create_variable('Times','S1',('Time','DateStrLen'),)
for i,time in enumerate([time_inp+'_12:00:00',time_inp+'_13:00:00',time_inp+'_14:00:00',time_inp+'_15:00:00',time_inp+'_16:00:00',
time_inp+'_17:00:00',time_inp+'_18:00:00',time_inp+'_19:00:00',time_inp+'_20:00:00',time_inp+'_21:00:00',time_inp+'_22:00:00',
time_inp+'_23:00:00',]):
f_chem.variables['Times'][i]=list(time)
for ll, LL in zip([lon_inp, lat_inp],['XLONG', 'XLAT']):
f_chem.create_variable(LL, 'f', ('south_north', 'west_east',),)
f_chem.variables[LL][:]=ll
radm_gas=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL',]
radm_aerosol=['E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I','E_SO4J','E_NO3I','E_NO3J',]
for gas in radm_gas:
f_chem.create_variable(gas,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[gas].FieldType = np.int16(104)
f_chem.variables[gas].MemoryOrder = 'XYZ'
f_chem.variables[gas].description = 'EMISSIONS'
f_chem.variables[gas].units = 'mol km^-2 hr^-1'
f_chem.variables[gas].stagger = 'Z'
#f_chem.variables[gas].ordinates = 'XLONG XLAT'
for aerosol in radm_aerosol:
f_chem.create_variable(aerosol,'f',('Time','emissions_zdim','south_north','west_east',))
f_chem.variables[aerosol].FieldType = np.int16(104)
f_chem.variables[aerosol].MemoryOrder = 'XYZ'
f_chem.variables[aerosol].description = 'EMISSIONS'
f_chem.variables[aerosol].units = 'ug/m3 m/s'
f_chem.variables[aerosol].stagger = 'Z'
#f_chem.variables[aerosol].ordinates = 'XLONG XLAT'
radm_spec=['E_CO','E_NH3','E_NO','E_NO2','E_SO2','E_ALD','E_CSL','E_ETH','E_HC3','E_HC5','E_HC8','E_HCHO','E_ISO','E_KET',
'E_OL2','E_OLI','E_OLT','E_ORA2','E_TOL','E_XYL','E_ECI','E_ECJ','E_ORGI','E_ORGJ','E_PM25I','E_PM25J','E_PM_10','E_SO4I',
'E_SO4J','E_NO3I','E_NO3J',]
for i,spec in enumerate(radm_spec):
f_chem.variables[spec][:] = wrf_spec_emis[i][12:24,:,:,:] #dimension need to be matched with the variable defination
f_chem.close()
os.rename(ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1]+'.nc', ent_dir.get()+'/merged/'+'wrfchemi_12z_'+ent_inp.get().split('_')[-1])
- Reading netCDF file
- Converting data from Kelvin to degC
- Writing data to new netCDF file
2018-08-28 kmf
"""
import os
import numpy as np
import Ngl, Nio
#-- data file name
fname = "rectilinear_grid_3D.nc"
#-- open file
f = Nio.open_file(fname, "r")
#-- read temperature, time, latitude and longitude arrays
var = f.variables["t"]
time = f.variables["time"]
lat = f.variables["lat"]
lon = f.variables["lon"]
#-- convert data from units Kelvin to degC
varC = var[:, 0, :, :] #-- copy variable at level=0; retain metadata
varC = varC - 273.15 #-- convert to degC
#-- open new netCDF file
os.system("rm -rf t_degC_py_short.nc") #-- delete file if it exists
outf = Nio.open_file("t_degC_py_short.nc", "c") #-- open new netCDF file
def main(argv):
# Get command line stuff and store in a dictionary
s = 'tag= compset= esize= tslice= res= sumfile= indir= sumfiledir= mach= verbose jsonfile= mpi_enable maxnorm gmonly popens cumul regx= startMon= endMon= fIndex='
optkeys = s.split()
try:
opts, args = getopt.getopt(argv, "h", optkeys)
except getopt.GetoptError:
pyEnsLib.EnsSum_usage()
sys.exit(2)
# Put command line options in a dictionary - also set defaults
opts_dict = {}
# Defaults
opts_dict['tag'] = 'cesm2_0_beta10'
opts_dict['compset'] = 'F2000climo'
opts_dict['mach'] = 'cheyenne'
opts_dict['esize'] = 350
opts_dict['tslice'] = 1
opts_dict['res'] = 'f19_f19'
opts_dict['sumfile'] = 'ens.summary.nc'
opts_dict['indir'] = './'
opts_dict['sumfiledir'] = './'
opts_dict['jsonfile'] = 'exclude_empty.json'
opts_dict['verbose'] = False
opts_dict['mpi_enable'] = True
opts_dict['maxnorm'] = False
opts_dict['gmonly'] = True
opts_dict['popens'] = False
opts_dict['cumul'] = False
opts_dict['regx'] = 'test'
opts_dict['startMon'] = 1
opts_dict['endMon'] = 1
opts_dict['fIndex'] = 151
# This creates the dictionary of input arguments
opts_dict = pyEnsLib.getopt_parseconfig(opts, optkeys, 'ES', opts_dict)
verbose = opts_dict['verbose']
st = opts_dict['esize']
esize = int(st)
if opts_dict['popens'] == True:
print "Error: Please use pyEnsSumPop.py for a POP ensemble (not --popens)."
sys.exit()
if not (opts_dict['tag'] and opts_dict['compset'] and opts_dict['mach']
or opts_dict['res']):
print 'Please specify --tag, --compset, --mach and --res options'
sys.exit()
# Now find file names in indir
input_dir = opts_dict['indir']
# The var list that will be excluded
ex_varlist = []
inc_varlist = []
# Create a mpi simplecomm object
if opts_dict['mpi_enable']:
me = simplecomm.create_comm()
else:
me = simplecomm.create_comm(not opts_dict['mpi_enable'])
if me.get_rank() == 0:
print 'Running pyEnsSum!'
if me.get_rank() == 0 and (verbose == True):
print opts_dict
print 'Ensemble size for summary = ', esize
exclude = False
if me.get_rank() == 0:
if opts_dict['jsonfile']:
inc_varlist = []
# Read in the excluded or included var list
ex_varlist, exclude = pyEnsLib.read_jsonlist(
opts_dict['jsonfile'], 'ES')
if exclude == False:
inc_varlist = ex_varlist
ex_varlist = []
# Read in the included var list
#inc_varlist=pyEnsLib.read_jsonlist(opts_dict['jsonfile'],'ES')
# Broadcast the excluded var list to each processor
#if opts_dict['mpi_enable']:
# ex_varlist=me.partition(ex_varlist,func=Duplicate(),involved=True)
# Broadcast the excluded var list to each processor
if opts_dict['mpi_enable']:
exclude = me.partition(exclude, func=Duplicate(), involved=True)
if exclude:
ex_varlist = me.partition(ex_varlist,
func=Duplicate(),
involved=True)
else:
inc_varlist = me.partition(inc_varlist,
func=Duplicate(),
involved=True)
in_files = []
if (os.path.exists(input_dir)):
# Get the list of files
in_files_temp = os.listdir(input_dir)
in_files = sorted(in_files_temp)
# Make sure we have enough
num_files = len(in_files)
if me.get_rank() == 0 and (verbose == True):
print 'Number of files in input directory = ', num_files
if (num_files < esize):
if me.get_rank() == 0 and (verbose == True):
print 'Number of files in input directory (',num_files,\
') is less than specified ensemble size of ', esize
sys.exit(2)
if (num_files > esize):
if me.get_rank() == 0 and (verbose == True):
print 'NOTE: Number of files in ', input_dir, \
'is greater than specified ensemble size of ', esize ,\
'\nwill just use the first ', esize, 'files'
else:
if me.get_rank() == 0:
print 'Input directory: ', input_dir, ' not found'
sys.exit(2)
if opts_dict['cumul']:
if opts_dict['regx']:
in_files_list = get_cumul_filelist(opts_dict, opts_dict['indir'],
opts_dict['regx'])
in_files = me.partition(in_files_list,
func=EqualLength(),
involved=True)
if me.get_rank() == 0 and (verbose == True):
print 'in_files=', in_files
# Open the files in the input directory
o_files = []
if me.get_rank() == 0 and opts_dict['verbose']:
print 'Input files are: '
print "\n".join(in_files)
#for i in in_files:
# print "in_files =",i
for onefile in in_files[0:esize]:
if (os.path.isfile(input_dir + '/' + onefile)):
o_files.append(Nio.open_file(input_dir + '/' + onefile, "r"))
else:
if me.get_rank() == 0:
print "COULD NOT LOCATE FILE ", input_dir + '/' + onefile, "! EXITING...."
sys.exit()
# Store dimensions of the input fields
if me.get_rank() == 0 and (verbose == True):
print "Getting spatial dimensions"
nlev = -1
nilev = -1
ncol = -1
nlat = -1
nlon = -1
lonkey = ''
latkey = ''
# Look at first file and get dims
input_dims = o_files[0].dimensions
ndims = len(input_dims)
for key in input_dims:
if key == "lev":
nlev = input_dims["lev"]
elif key == "ilev":
nilev = input_dims["ilev"]
elif key == "ncol":
ncol = input_dims["ncol"]
elif (key == "nlon") or (key == "lon"):
nlon = input_dims[key]
lonkey = key
elif (key == "nlat") or (key == "lat"):
nlat = input_dims[key]
latkey = key
if (nlev == -1):
if me.get_rank() == 0:
print "COULD NOT LOCATE valid dimension lev => EXITING...."
sys.exit()
if ((ncol == -1) and ((nlat == -1) or (nlon == -1))):
if me.get_rank() == 0:
print "Need either lat/lon or ncol => EXITING...."
sys.exit()
# Check if this is SE or FV data
if (ncol != -1):
is_SE = True
else:
is_SE = False
# Make sure all files have the same dimensions
if me.get_rank() == 0 and (verbose == True):
print "Checking dimensions across files...."
print 'lev = ', nlev
if (is_SE == True):
print 'ncol = ', ncol
else:
print 'nlat = ', nlat
print 'nlon = ', nlon
for count, this_file in enumerate(o_files):
input_dims = this_file.dimensions
if (is_SE == True):
if (nlev != int(input_dims["lev"])
or (ncol != int(input_dims["ncol"]))):
if me.get_rank() == 0:
print "Dimension mismatch between ", in_files[
0], 'and', in_files[0], '!!!'
sys.exit()
else:
if ( nlev != int(input_dims["lev"]) or ( nlat != int(input_dims[latkey]))\
or ( nlon != int(input_dims[lonkey]))):
if me.get_rank() == 0:
print "Dimension mismatch between ", in_files[
0], 'and', in_files[0], '!!!'
sys.exit()
# Get 2d vars, 3d vars and all vars (For now include all variables)
vars_dict_all = o_files[0].variables
# Remove the excluded variables (specified in json file) from variable dictionary
#print len(vars_dict_all)
if exclude:
vars_dict = vars_dict_all
for i in ex_varlist:
if i in vars_dict:
del vars_dict[i]
#Given an included var list, remove all float var that are not on the list
else:
vars_dict = vars_dict_all.copy()
for k, v in vars_dict_all.iteritems():
if (k not in inc_varlist) and (vars_dict_all[k].typecode() == 'f'):
#print vars_dict_all[k].typecode()
#print k
del vars_dict[k]
num_vars = len(vars_dict)
#print num_vars
#if me.get_rank() == 0:
# for k,v in vars_dict.iteritems():
# print 'vars_dict',k,vars_dict[k].typecode()
str_size = 0
d2_var_names = []
d3_var_names = []
num_2d = 0
num_3d = 0
# Which are 2d, which are 3d and max str_size
for k, v in vars_dict.iteritems():
var = k
vd = v.dimensions # all the variable's dimensions (names)
vr = v.rank # num dimension
vs = v.shape # dim values
is_2d = False
is_3d = False
if (is_SE == True): # (time, lev, ncol) or (time, ncol)
if ((vr == 2) and (vs[1] == ncol)):
is_2d = True
num_2d += 1
elif ((vr == 3) and (vs[2] == ncol and vs[1] == nlev)):
is_3d = True
num_3d += 1
else: # (time, lev, nlon, nlon) or (time, nlat, nlon)
if ((vr == 3) and (vs[1] == nlat and vs[2] == nlon)):
is_2d = True
num_2d += 1
elif ((vr == 4) and (vs[2] == nlat and vs[3] == nlon and
(vs[1] == nlev or vs[1] == nilev))):
is_3d = True
num_3d += 1
if (is_3d == True):
str_size = max(str_size, len(k))
d3_var_names.append(k)
elif (is_2d == True):
str_size = max(str_size, len(k))
d2_var_names.append(k)
#else:
# print 'var=',k
if me.get_rank() == 0 and (verbose == True):
print 'Number of variables found: ', num_3d + num_2d
print '3D variables: ' + str(num_3d) + ', 2D variables: ' + str(num_2d)
# Now sort these and combine (this sorts caps first, then lower case -
# which is what we want)
d2_var_names.sort()
d3_var_names.sort()
if esize < num_2d + num_3d:
if me.get_rank() == 0:
print "************************************************************************************************************************************"
print " Error: the total number of 3D and 2D variables " + str(
num_2d + num_3d
) + " is larger than the number of ensemble files " + str(esize)
print " Cannot generate ensemble summary file, please remove more variables from your included variable list,"
print " or add more varaibles in your excluded variable list!!!"
print "************************************************************************************************************************************"
sys.exit()
# All vars is 3d vars first (sorted), the 2d vars
all_var_names = list(d3_var_names)
all_var_names += d2_var_names
n_all_var_names = len(all_var_names)
#if me.get_rank() == 0 and (verbose == True):
# print 'num vars = ', n_all_var_names, '(3d = ', num_3d, ' and 2d = ', num_2d, ")"
# Create new summary ensemble file
this_sumfile = opts_dict["sumfile"]
if me.get_rank() == 0 and (verbose == True):
print "Creating ", this_sumfile, " ..."
if (me.get_rank() == 0):
if os.path.exists(this_sumfile):
os.unlink(this_sumfile)
opt = Nio.options()
opt.PreFill = False
opt.Format = 'NetCDF4Classic'
nc_sumfile = Nio.open_file(this_sumfile, 'w', options=opt)
# Set dimensions
if me.get_rank() == 0 and (verbose == True):
print "Setting dimensions ....."
if (is_SE == True):
nc_sumfile.create_dimension('ncol', ncol)
else:
nc_sumfile.create_dimension('nlat', nlat)
nc_sumfile.create_dimension('nlon', nlon)
nc_sumfile.create_dimension('nlev', nlev)
nc_sumfile.create_dimension('ens_size', esize)
nc_sumfile.create_dimension('nvars', num_3d + num_2d)
nc_sumfile.create_dimension('nvars3d', num_3d)
nc_sumfile.create_dimension('nvars2d', num_2d)
nc_sumfile.create_dimension('str_size', str_size)
# Set global attributes
now = time.strftime("%c")
if me.get_rank() == 0 and (verbose == True):
print "Setting global attributes ....."
setattr(nc_sumfile, 'creation_date', now)
setattr(nc_sumfile, 'title', 'CAM verification ensemble summary file')
setattr(nc_sumfile, 'tag', opts_dict["tag"])
setattr(nc_sumfile, 'compset', opts_dict["compset"])
setattr(nc_sumfile, 'resolution', opts_dict["res"])
setattr(nc_sumfile, 'machine', opts_dict["mach"])
# Create variables
if me.get_rank() == 0 and (verbose == True):
print "Creating variables ....."
v_lev = nc_sumfile.create_variable("lev", 'f', ('nlev', ))
v_vars = nc_sumfile.create_variable("vars", 'S1',
('nvars', 'str_size'))
v_var3d = nc_sumfile.create_variable("var3d", 'S1',
('nvars3d', 'str_size'))
v_var2d = nc_sumfile.create_variable("var2d", 'S1',
('nvars2d', 'str_size'))
if not opts_dict['gmonly']:
if (is_SE == True):
v_ens_avg3d = nc_sumfile.create_variable(
"ens_avg3d", 'f', ('nvars3d', 'nlev', 'ncol'))
v_ens_stddev3d = nc_sumfile.create_variable(
"ens_stddev3d", 'f', ('nvars3d', 'nlev', 'ncol'))
v_ens_avg2d = nc_sumfile.create_variable(
"ens_avg2d", 'f', ('nvars2d', 'ncol'))
v_ens_stddev2d = nc_sumfile.create_variable(
"ens_stddev2d", 'f', ('nvars2d', 'ncol'))
else:
v_ens_avg3d = nc_sumfile.create_variable(
"ens_avg3d", 'f', ('nvars3d', 'nlev', 'nlat', 'nlon'))
v_ens_stddev3d = nc_sumfile.create_variable(
"ens_stddev3d", 'f', ('nvars3d', 'nlev', 'nlat', 'nlon'))
v_ens_avg2d = nc_sumfile.create_variable(
"ens_avg2d", 'f', ('nvars2d', 'nlat', 'nlon'))
v_ens_stddev2d = nc_sumfile.create_variable(
"ens_stddev2d", 'f', ('nvars2d', 'nlat', 'nlon'))
v_RMSZ = nc_sumfile.create_variable("RMSZ", 'f',
('nvars', 'ens_size'))
v_gm = nc_sumfile.create_variable("global_mean", 'f',
('nvars', 'ens_size'))
v_standardized_gm = nc_sumfile.create_variable("standardized_gm", 'f',
('nvars', 'ens_size'))
v_loadings_gm = nc_sumfile.create_variable('loadings_gm', 'f',
('nvars', 'nvars'))
v_mu_gm = nc_sumfile.create_variable('mu_gm', 'f', ('nvars', ))
v_sigma_gm = nc_sumfile.create_variable('sigma_gm', 'f', ('nvars', ))
v_sigma_scores_gm = nc_sumfile.create_variable('sigma_scores_gm', 'f',
('nvars', ))
# Assign vars, var3d and var2d
if me.get_rank() == 0 and (verbose == True):
print "Assigning vars, var3d, and var2d ....."
eq_all_var_names = []
eq_d3_var_names = []
eq_d2_var_names = []
l_eq = len(all_var_names)
for i in range(l_eq):
tt = list(all_var_names[i])
l_tt = len(tt)
if (l_tt < str_size):
extra = list(' ') * (str_size - l_tt)
tt.extend(extra)
eq_all_var_names.append(tt)
l_eq = len(d3_var_names)
for i in range(l_eq):
tt = list(d3_var_names[i])
l_tt = len(tt)
if (l_tt < str_size):
extra = list(' ') * (str_size - l_tt)
tt.extend(extra)
eq_d3_var_names.append(tt)
l_eq = len(d2_var_names)
for i in range(l_eq):
tt = list(d2_var_names[i])
l_tt = len(tt)
if (l_tt < str_size):
extra = list(' ') * (str_size - l_tt)
tt.extend(extra)
eq_d2_var_names.append(tt)
v_vars[:] = eq_all_var_names[:]
v_var3d[:] = eq_d3_var_names[:]
v_var2d[:] = eq_d2_var_names[:]
# Time-invarient metadata
if me.get_rank() == 0 and (verbose == True):
print "Assigning time invariant metadata ....."
lev_data = vars_dict["lev"]
v_lev = lev_data
# Form ensembles, each missing one member; compute RMSZs and global means
#for each variable, we also do max norm also (currently done in pyStats)
tslice = opts_dict['tslice']
if not opts_dict['cumul']:
# Partition the var list
var3_list_loc = me.partition(d3_var_names,
func=EqualStride(),
involved=True)
var2_list_loc = me.partition(d2_var_names,
func=EqualStride(),
involved=True)
else:
var3_list_loc = d3_var_names
var2_list_loc = d2_var_names
# Calculate global means #
if me.get_rank() == 0 and (verbose == True):
print "Calculating global means ....."
if not opts_dict['cumul']:
gm3d, gm2d, var_list = pyEnsLib.generate_global_mean_for_summary(
o_files, var3_list_loc, var2_list_loc, is_SE, False, opts_dict)
if me.get_rank() == 0 and (verbose == True):
print "Finish calculating global means ....."
# Calculate RMSZ scores
if (not opts_dict['gmonly']) | (opts_dict['cumul']):
if me.get_rank() == 0 and (verbose == True):
print "Calculating RMSZ scores ....."
zscore3d, zscore2d, ens_avg3d, ens_stddev3d, ens_avg2d, ens_stddev2d, temp1, temp2 = pyEnsLib.calc_rmsz(
o_files, var3_list_loc, var2_list_loc, is_SE, opts_dict)
# Calculate max norm ensemble
if opts_dict['maxnorm']:
if me.get_rank() == 0 and (verbose == True):
print "Calculating max norm of ensembles ....."
pyEnsLib.calculate_maxnormens(opts_dict, var3_list_loc)
pyEnsLib.calculate_maxnormens(opts_dict, var2_list_loc)
if opts_dict['mpi_enable']:
if not opts_dict['cumul']:
# Gather the 3d variable results from all processors to the master processor
slice_index = get_stride_list(len(d3_var_names), me)
# Gather global means 3d results
gm3d = gather_npArray(gm3d, me, slice_index,
(len(d3_var_names), len(o_files)))
if not opts_dict['gmonly']:
# Gather zscore3d results
zscore3d = gather_npArray(zscore3d, me, slice_index,
(len(d3_var_names), len(o_files)))
# Gather ens_avg3d and ens_stddev3d results
shape_tuple3d = get_shape(ens_avg3d.shape, len(d3_var_names),
me.get_rank())
ens_avg3d = gather_npArray(ens_avg3d, me, slice_index,
shape_tuple3d)
ens_stddev3d = gather_npArray(ens_stddev3d, me, slice_index,
shape_tuple3d)
# Gather 2d variable results from all processors to the master processor
slice_index = get_stride_list(len(d2_var_names), me)
# Gather global means 2d results
gm2d = gather_npArray(gm2d, me, slice_index,
(len(d2_var_names), len(o_files)))
var_list = gather_list(var_list, me)
if not opts_dict['gmonly']:
# Gather zscore2d results
zscore2d = gather_npArray(zscore2d, me, slice_index,
(len(d2_var_names), len(o_files)))
# Gather ens_avg3d and ens_stddev2d results
shape_tuple2d = get_shape(ens_avg2d.shape, len(d2_var_names),
me.get_rank())
ens_avg2d = gather_npArray(ens_avg2d, me, slice_index,
shape_tuple2d)
ens_stddev2d = gather_npArray(ens_stddev2d, me, slice_index,
shape_tuple2d)
else:
gmall = np.concatenate((temp1, temp2), axis=0)
gmall = pyEnsLib.gather_npArray_pop(
gmall, me,
(me.get_size(), len(d3_var_names) + len(d2_var_names)))
# Assign to file:
if me.get_rank() == 0:
if not opts_dict['cumul']:
gmall = np.concatenate((gm3d, gm2d), axis=0)
if not opts_dict['gmonly']:
Zscoreall = np.concatenate((zscore3d, zscore2d), axis=0)
v_RMSZ[:, :] = Zscoreall[:, :]
if not opts_dict['gmonly']:
if (is_SE == True):
v_ens_avg3d[:, :, :] = ens_avg3d[:, :, :]
v_ens_stddev3d[:, :, :] = ens_stddev3d[:, :, :]
v_ens_avg2d[:, :] = ens_avg2d[:, :]
v_ens_stddev2d[:, :] = ens_stddev2d[:, :]
else:
v_ens_avg3d[:, :, :, :] = ens_avg3d[:, :, :, :]
v_ens_stddev3d[:, :, :, :] = ens_stddev3d[:, :, :, :]
v_ens_avg2d[:, :, :] = ens_avg2d[:, :, :]
v_ens_stddev2d[:, :, :] = ens_stddev2d[:, :, :]
else:
gmall_temp = np.transpose(gmall[:, :])
gmall = gmall_temp
mu_gm, sigma_gm, standardized_global_mean, loadings_gm, scores_gm = pyEnsLib.pre_PCA(
gmall, all_var_names, var_list, me)
v_gm[:, :] = gmall[:, :]
v_standardized_gm[:, :] = standardized_global_mean[:, :]
v_mu_gm[:] = mu_gm[:]
v_sigma_gm[:] = sigma_gm[:].astype(np.float32)
v_loadings_gm[:, :] = loadings_gm[:, :]
v_sigma_scores_gm[:] = scores_gm[:]
if me.get_rank() == 0:
print "All Done"
