def __init__(self, filesearch):
"""docstring for init"""
self.files=glob.glob(filesearch)
self.files.sort()
if len(self.files)==0:
raise ValueError("\nERROR: no files match search string:"+filesearch+"\n")
if verbose: print("Reading:{}".format(self.files[self.curfile]))
self.tdata = mygis.read_nc(self.files[self.curfile], self.t_var).data-273.15
self.rrdata = mygis.read_nc(self.files[self.curfile],self.rr_var).data
self.acdata = mygis.read_nc(self.files[self.curfile],self.ac_var).data
self.winddata = mygis.read_nc(self.files[self.curfile],self.u_var).data**2
self.winddata+= mygis.read_nc(self.files[self.curfile],self.v_var).data**2
self.winddata=np.sqrt(self.winddata)
self.husdata = mygis.read_nc(self.files[self.curfile],self.hus_var).data
self.swdata = mygis.read_nc(self.files[self.curfile],self.sw_var).data
self.lwdata = mygis.read_nc(self.files[self.curfile],self.lw_var).data
self.acsnowdata = mygis.read_nc(self.files[self.curfile],self.snow_var).data
self.snowdata=np.zeros(self.acsnowdata.shape)
self.snowdata[0]=self.acsnowdata[0]
self.snowdata[1:]=np.diff(self.acsnowdata,axis=0)
for i in range(self.snowdata.shape[0]):
if self.snowdata[i].min()<0:
self.snowdata[i]=self.acsnowdata[i]
self.lastsnow=self.acsnowdata[-1]
self.mask = mygis.read_nc(mask_file,"LANDMASK").data[0]==0
def load_wrf_data(stat,time):
"""Load WRF data and calculate the relevant statistic"""
output_wrf_file=stat+"_"+time+".nc"
try:
current=mygis.read_nc(wrfloc+"current_"+output_wrf_file).data
future=mygis.read_nc(wrfloc+"future_"+output_wrf_file).data
print("WRF data loaded from pre-calculated summary files")
except:
print(" Reading raw data...")
if time=="annual":
current=np.concatenate(mygis.read_files(wrfloc+"daily_NARR*"))
future=np.concatenate(mygis.read_files(wrfloc+"daily_PGW*"))
else:
month=time[-2:]
current=np.concatenate(mygis.read_files(wrfloc+"daily_NARR*"+month+".nc"))
future=np.concatenate(mygis.read_files(wrfloc+"daily_PGW*"+month+".nc"))
print(" Generating statistics")
if stat=="MAP":
ndays=calc_ndays(time)
current=current.mean(axis=0)*ndays
future=future.mean(axis=0)*ndays
elif stat=="wetfrac":
current=calc_wetfrac(current)
future=calc_wetfrac(future)
else:
raise KeyError("stat not created for WRF:"+stat)
print(" Writing stat for future reference")
mygis.write(wrfloc+"current_"+output_wrf_file,current)
mygis.write(wrfloc+"future_"+output_wrf_file,future)
# if stat=="MAP":
# current[current<1e-4]=1e-4
return (future-current),current
def load_hi_res_dem():
"""docstring for load_hi_res_dem"""
demf=mygis.read_nc(dem_file,"elev_m",returnNCvar=True)
latf=mygis.read_nc(dem_file,"lat",returnNCvar=True)
lonf=mygis.read_nc(dem_file,"lon",returnNCvar=True)
if DEBUG:
x0=2500;x1=4000;y0=12000;y1=10500
# x0=0;x1=None;y0=-1;y1=0
dem=demf.data[y0:y1:-1,x0:x1]
lat=latf.data[y0:y1:-1]
lon=lonf.data[x0:x1]
else:
dem=demf.data[::-1,:]
lat=latf.data[::-1]
lon=lonf.data[:]
demf.ncfile.close()
latf.ncfile.close()
lonf.ncfile.close()
dx=lon[1]-lon[0]
dy=lat[1]-lat[0]
nx=len(lon)
ny=len(lat)
lon,lat=np.meshgrid(lon,lat)
return Bunch(data=dem,lat=lat,lon=lon,
startx=lon[0,0]-dx/2,starty=lat[0,0]-dy/2,
dx=dx,dy=dy,nx=nx,ny=ny)
def load_cesm_means(wind_option):
"""docstring for load_cesm_means"""
cesmd="means/"
varlist=["p","rh","theta","u","v"]
if wind_option=="nowind":
varlist=["p","rh","theta"]
outputdata=[]
month_mid_point_doy=(start_day_per_month[1:]+np.array(start_day_per_month[:-1]))*0.5
for month in range(1,13):
curoutput=Bunch(doy=month_mid_point_doy[month-1])
try:
for v in varlist:
curoutput[v]=mygis.read_nc(cesmd+"month{0:02}_mean_{1}.nc".format(month,v),v).data
curoutput["z"]=mygis.read_nc(cesmd+"annual_mean_z.nc","z").data
except:
for v in varlist:
curoutput[v]=mygis.read_nc("month{0:02}_mean_{1}.nc".format(month,v),v).data
curoutput["z"]=mygis.read_nc("annual_mean_z.nc","z").data
if curoutput["p"][0,0,0]<1300:
curoutput["p"][0,0,0]*=100.0 # convert hPa to Pa
if (curoutput["z"][-1,0,0]<10000) and curoutput["p"][-1,0,0]<10000: # can't have 100hPa at 10km height
print("WARNING! Assuming that cesm z has been erroneously divided by 9.8")
curoutput["z"]*=9.8
outputdata.append(curoutput)
return outputdata
def load_erai_means(wind_option):
"""docstring for load_erai_means"""
eraid = "erai/"
varlist = ["p", "rh", "ta", "ua", "va", "z"]
if (wind_option == "nowind"):
varlist = ["p", "rh", "ta", "z"]
outputdata = []
month_mid_point_doy = (start_day_per_month[1:] +
np.array(start_day_per_month[:-1])) * 0.5
for month in range(1, 13):
curoutput = Bunch(doy=month_mid_point_doy[month - 1])
for v in varlist:
if v == "ta":
ta = mygis.read_nc(
eraid +
"regridded_ERAi_to_cesm_month{0:02}.nc".format(month),
v).data
else:
curoutput[v] = mygis.read_nc(
eraid +
"regridded_ERAi_to_cesm_month{0:02}.nc".format(month),
v).data
curoutput["theta"] = ta / units.exner(curoutput.p)
# erai is probably "upside down" so reverse the vertical dimension of all variables
if curoutput.p[0, 0, 0] < curoutput.p[-1, 0, 0]:
for v in curoutput.keys():
if v != "doy":
curoutput[v] = curoutput[v][::-1, :, :]
outputdata.append(curoutput)
return outputdata
def load_elev_comparison(swefile="SWE_Daily0600UTC_WesternUS_2010.dat",outputfile="snodas_by_elev.png",domainname="Front Range +",domain=None,dz=100):
import wsc.compare2lidar as c2l
demfile="snodas_dem.nc"
forestfile="snodas_forest.nc"
forest=[1]
bare=[0]
mayday=120
if domain is None:
minx=1800; miny=600; maxx=None;maxy=1100
else:
miny=domain[0]; maxy=domain[1]; minx=domain[2]; maxx=domain[3]
print("Loading data")
vegclass=myio.read_nc(forestfile).data[miny:maxy,minx:maxx]
forested=np.where(vegclass==forest[0])
exposed=np.where(vegclass==bare[0])
mask=np.zeros(vegclass.shape)
mask[forested]=1
mask[exposed]=2
dem=myio.read_nc(demfile).data[miny:maxy,minx:maxx]
snodas=load(swefile)
snow=snodas.data[mayday,miny:maxy,minx:maxx]
print("Binning")
banded=c2l.bin_by_elevation(snow,dem,mask,dz=dz)
print("Plotting")
c2l.plot_elevation_bands(banded,outputfile,title="SNODAS SWE over "+domainname)
def load_atm(time,info):
"""Load atmospheric variable from a GRIB file"""
uvfile,scfile=find_atm_file(time,info)
uvnc_file=grib2nc(uvfile,atmuvlist,info.nc_file_dir)
scnc_file=grib2nc(scfile,atmvarlist,info.nc_file_dir)
outputdata=Bunch()
for s,v in zip(icar_uv_var,atmuvlist):
nc_data=mygis.read_nc(uvnc_file,v,returnNCvar=True)
if len(nc_data.data.shape)==3:
outputdata[s]=nc_data.data[:,info.ymin:info.ymax,info.xmin:info.xmax]
else:
outputdata[s]=nc_data.data[info.ymin:info.ymax,info.xmin:info.xmax]
nc_data.ncfile.close()
for s,v in zip(icar_atm_var,atmvarlist):
nc_data = mygis.read_nc(scnc_file,v,returnNCvar=True)
if len(nc_data.data.shape)==3:
outputdata[s] = nc_data.data[:,info.ymin:info.ymax,info.xmin:info.xmax]
elif len(nc_data.data.shape)==2:
outputdata[s] = nc_data.data[info.ymin:info.ymax,info.xmin:info.xmax]
elif len(nc_data.data.shape)==1:
outputdata[s] = nc_data.data[:]
else:
try:
outputdata[s] = nc_data.data[:]
except:
outputdata[s] = nc_data.data.get_value()
nc_data.ncfile.close()
return outputdata
def main (filename, outputfile):
icar_data=ICAR_Reader(filename)
if verbose:print(icar_data.files[0],icar_data.files[-1])
raindata=[]
tmindata=[]
tmaxdata=[]
tavedata=[]
if verbose:print("Looping through data")
for data in icar_data:
raindata.append(data[0])
tmindata.append(data[1])
tmaxdata.append(data[2])
tavedata.append(data[3])
latvar.data=mygis.read_nc(icar_data.files[0],"lat").data
lonvar.data=mygis.read_nc(icar_data.files[0],"lon").data
dates_are_mjd = (icar_data.files[0].split("/")[1] == "erai")
if verbose:print("using modified julian day dates="+str(dates_are_mjd))
year = int(icar_data.files[0].split("/")[-1].split("_")[1])
if dates_are_mjd:
timevar.attributes = era_time_atts
start_date = date_fun.date2mjd(year,01,01,00,00)
else:
start_date = (year-1800) * 365
timevar.data = np.arange(start_date,start_date+len(raindata))
if verbose:print(year, timevar.data[0], len(raindata))
if verbose:print("Writing data")
# write_file(outputfile+"_rain.nc",raindata, varname="precipitation_amount", varatts=prec_atts)
# write_file(outputfile+"_tmin.nc",tmindata, varname="daily_minimum_temperature", varatts=tmin_atts)
# write_file(outputfile+"_tmax.nc",tmaxdata, varname="daily_maximum_temperature", varatts=tmax_atts)
write_file(outputfile+"_tave",tavedata, varname="daily_average_temperature", varatts=tave_atts)
def load_tskin(filename,tsvarname, maskvarname):
"""load skin temp data from wrf output file: filename
tskin uses it's own procedure so that we can average over the past 10 days for land points
ideally it will create a higher resolution output based on a high-resolution land-sea mask
"""
if verbose:print("Loading :"+maskvarname)
ncmask=mygis.read_nc(filename,maskvarname,returnNCvar=True)
mask=ncmask.data[0]
ncmask.ncfile.close()
land=np.where(mask==1)
if verbose:print("Loading :"+tsvarname)
ncts=mygis.read_nc(filename,tsvarname,returnNCvar=True)
atts=mygis.read_atts(filename,tsvarname)
dims=ncts.data.dimensions
if verbose:print(" "+str(dims))
if verbose:print(" "+str(atts))
data=ncts.data[:]
ncts.ncfile.close()
for i in range(data.shape[0]-1,0,-1):
start=max(i-(10*steps_per_day),0)
data[i][land]=data[start:i+1].mean(axis=0)[land]
return Bunch(name=tsvarname, data=data, attributes=atts, dims=dims, dtype='f')
def nextNN(self):
curfile=self._curfile
if curfile>=len(self._filenames):
raise StopIteration
minx=self.x.min()
maxx=self.x.max()+1
miny=self.y.min()
maxy=self.y.max()+1
curx=self.x-minx
cury=self.y-miny
data=list()
for thisvar in self._vars:
ncfile=mygis.read_nc(self._filenames[curfile],var=thisvar,returnNCvar=True)
if self.npos==1:
data.append(ncfile.data[miny:maxy,minx:maxx][cury,curx])
else:
data.append(ncfile.data[self.posinfile,miny:maxy,minx:maxx][cury,curx])
ncfile.ncfile.close()
self.posinfile+=1
if (self.npos==1) or (self.posinfile>=self.npos):
self._curfile+=1
self.posinfile=0
if self.npos>1:
if self._curfile<len(self._filenames):
d=mygis.read_nc(self._filenames[self._curfile],var=self._vars[0],returnNCvar=True)
ntimes=d.data.shape[0]
d.ncfile.close()
self.npos=ntimes
return data
def main(start_date="19900101"):
"""convert a file (or files) from downscaling code to maps"""
files=find_files(start_date)
files.sort()
geo=load_geo(global_basefile)
times=mygis.read_nc(files[0],"time").data
ntimes=times.size
tmp=mygis.read_nc(files[0],"coefficient",returnNCvar=True)
output_data=np.zeros((tmp.data.shape[1],ntimes,geo.lon.shape[0],geo.lon.shape[1]))
tmp.ncfile.close()
print(output_data.shape)
for f in files:
print(f)
data=mygis.read_nc(f,"coefficient").data
locations=get_xy(f,geo)
for i in range(len(locations.x)):
output_data[:,:,locations.y[i],locations.x[i]]=data[0,:,:,i]
print("Writing output file")
mygis.write(global_output_file,output_data,varname="coefficient",dtype="d",dims=("variable","time","latitude","longitude"),
extravars=[ Bunch(data=times,name="time",dims=("time",),dtype="d",
attributes=Bunch(units="seconds since 1970-01-01 00:00:00.0 0:00")),
Bunch(data=geo.lat,name="latitude",dims=("latitude","longitude"),dtype="f",
attributes=Bunch(units="degrees")),
Bunch(data=geo.lon,name="longitude",dims=("latitude","longitude"),dtype="f",
attributes=Bunch(units="degrees"))])
def load_geo(filename):
"""load a 2d lat and lon grid from filename"""
latnames=["lat","latitude","XLAT"]
lonnames=["lon","longitude","XLONG"]
lat=None
lon=None
for l in latnames:
if lat==None:
try:
lat=mygis.read_nc(filename,l).data
except:
lat=None
for l in lonnames:
if lon==None:
try:
lon=mygis.read_nc(filename,l).data
except:
lon=None
ymin=np.where(lat>global_bounds.lat.min)[0][0]
ymax=np.where(lat>global_bounds.lat.max)[0][0]
lat=lat[ymin:ymax]
xmin=np.where(lon>global_bounds.lon.min)[0][0]
xmax=np.where(lon>global_bounds.lon.max)[0][0]
lon=lon[xmin:xmax]
if len(lon.shape)==1:
lon,lat=np.meshgrid(lon,lat)
return Bunch(lon=lon,lat=lat)
def load_base_data(swefile="SWE_daily.nc",info="4km_wrf_output.nc",res=4,year=5):
# wrf.load_base_data(swefile="wrfout_d01_2008-05-01_00:00:00",res=2)
if res==2:
forest=[11,12,13,14,18]
exposed=[1,2,3,4,5,7,8,9,10,16,17,19,20,22,23,24,25,26,27,15,21] #warning 15 and 21 sound like they should be forest, but on the map they are in open areas
mayday=0
info=swefile
dz=100
else:
forest=[1,5]
exposed=[7,10]
mayday=212
for i in range(year):
mayday+=365
mayday+=np.floor(year/4)
print("Loading data")
vegclass=myio.read_nc(info,"IVGTYP").data[0,...]
mask=np.zeros(vegclass.shape)
for f in forest:
forested=np.where(vegclass==f)
mask[forested]=1
for e in exposed:
exposed=np.where(vegclass==e)
mask[exposed]=2
lat=myio.read_nc(info,"XLAT").data[0,...]
lon=myio.read_nc(info,"XLONG").data[0,...]
dem=myio.read_nc(info,"HGT").data[0,...]
all_snow=myio.read_nc(swefile,"SNOW").data/1000.0
snow=all_snow[mayday,:,:]
data_by_years=[all_snow[baseday:baseday+365,:,:] for baseday in range(0,all_snow.shape[0]-20,365)]
return Bunch(data=snow, lat=lat,lon=lon, dem=dem, lc=mask, yearly=data_by_years)
def main():
"""docstring for main"""
plt.figure(figsize=(25,25))
months=np.arange(1,13)
for i in range(1,30):
print(i)
currentdata=(mygis.read_nc(current_precc_file.format(i+1),"PRECC",returnNCvar=True).data[current_start_point:current_end_point,130:145,200:208].mean(axis=1).mean(axis=1)
+mygis.read_nc(current_precl_file.format(i+1),"PRECL",returnNCvar=True).data[current_start_point:current_end_point,130:145,200:208].mean(axis=1).mean(axis=1))
futuredata =(mygis.read_nc(future_precc_file.format(i+1), "PRECC",returnNCvar=True).data[future_start_point:future_end_point, 130:145,200:208].mean(axis=1).mean(axis=1)
+mygis.read_nc(future_precl_file.format(i+1), "PRECL",returnNCvar=True).data[future_start_point:future_end_point, 130:145,200:208].mean(axis=1).mean(axis=1))
cur=np.reshape(currentdata,(10,12)).mean(axis=0)
fut=np.reshape(futuredata,(10,12)).mean(axis=0)
delta=1000.*86400.*(fut-cur)
plt.subplot(6,5,i+1)
# plt.plot(months,delta,label="{:03}".format(i+1),color="black")
for j in range(12):
if delta[j]>0:
plt.fill_between([months[j]-0.4,months[j]+0.4],[delta[j],delta[j]],color="blue")
else:
plt.fill_between([months[j]-0.4,months[j]+0.4],[delta[j],delta[j]],color="red")
plt.plot([0,13],[0,0],":",color="black")
plt.ylim(-0.5,0.5)
plt.xlim(0.1,12.9)
plt.title("Ens {:03}".format(i+1))
# plt.legend()
plt.tight_layout
plt.savefig("all_diffs.png")
def write_monthly_pgw(current,future,geofile):
"""write out a file containing the ratios future/current for each month and lat/lon data"""
print("writing PGW results")
if esgvarname=="pr":
output_data=future/current
outputfilename="PGW_ratio_file.nc"
data_atts=Bunch(long_name="Ratio between future:current monthly precipitaton", units="mm/mm")
else:
output_data=future-current
outputfilename="PGW_difference_file.nc"
data_atts=Bunch(long_name="Difference between future-current monthly Temperature", units="K")
lat=io.read_nc(geofile,"lat").data
lon=io.read_nc(geofile,"lon").data
time=np.arange(12)
lat_atts =Bunch(long_name="latitude", units="degrees")
lon_atts =Bunch(long_name="longitude", units="degrees")
time_atts=Bunch(long_name="Month of the year",units="month",description="0=January,11=December,etc.")
datadims=("time","lat","lon")
evars=[ Bunch(data=lat, name="lat", dtype="f",attributes=lat_atts,dims=("lat",)),
Bunch(data=lon, name="lon", dtype="f",attributes=lon_atts,dims=("lon",)),
Bunch(data=time,name="time",dtype="f",attributes=time_atts,dims=("time",)),
]
io.write(outputfilename,output_data,dtype="f",varname="data",dims=datadims,attributes=data_atts,extravars=evars)
def load_atm(time, info):
"""Load atmospheric variable from a GRIB file"""
uvfile, scfile = find_atm_file(time, info)
uvnc_file = grib2nc(uvfile, atmuvlist, info.nc_file_dir)
scnc_file = grib2nc(scfile, atmvarlist, info.nc_file_dir)
outputdata = Bunch()
for s, v in zip(icar_uv_var, atmuvlist):
nc_data = mygis.read_nc(uvnc_file, v, returnNCvar=True)
if len(nc_data.data.shape) == 3:
outputdata[s] = nc_data.data[:, info.ymin : info.ymax, info.xmin : info.xmax]
else:
outputdata[s] = nc_data.data[info.ymin : info.ymax, info.xmin : info.xmax]
nc_data.ncfile.close()
for s, v in zip(icar_atm_var, atmvarlist):
nc_data = mygis.read_nc(scnc_file, v, returnNCvar=True)
if len(nc_data.data.shape) == 3:
outputdata[s] = nc_data.data[:, info.ymin : info.ymax, info.xmin : info.xmax]
elif len(nc_data.data.shape) == 2:
outputdata[s] = nc_data.data[info.ymin : info.ymax, info.xmin : info.xmax]
elif len(nc_data.data.shape) == 1:
outputdata[s] = nc_data.data[:]
else:
outputdata[s] = nc_data.data.get_value()
nc_data.ncfile.close()
return outputdata
def set_bounds(info):
atm_file = info.atmdir + info.atmfile
erai_file = atm_file.replace("_Y_", "2000").replace("_M_", "01").replace(
"_D_", "01").replace("_h_", "00")
varlist = ["g4_lat_0", "g4_lon_1"]
output_dir = info.nc_file_dir
try:
os.mkdir(output_dir)
except:
pass
ncfile = io.grib2nc(erai_file, varlist, output_dir)
lat = mygis.read_nc(ncfile, varlist[0]).data
lon = mygis.read_nc(ncfile, varlist[1]).data - 360
# print(lon, info.lon[0])
lon[lon < -180] += 360
info.xmin = np.where(lon >= info.lon[0])[0][0]
info.xmax = np.where(lon <= info.lon[1])[0][-1] + 1
#note lat is inverted from "expected"
info.ymin = np.where(lat <= info.lat[1])[0][0]
info.ymax = np.where(lat >= info.lat[0])[0][-1] + 1
lon, lat = np.meshgrid(lon[info.xmin:info.xmax],
lat[info.ymin:info.ymax][::-1])
info.lat_data = lat
info.lon_data = lon
def load_monthly_ar(filesearch,geolut):
"""load monthly means for a given filesearch"""
files=[]
for search in filesearch:
files.extend(glob.glob(search))
files.sort()
ncdata=io.read_nc(files[0],esgvarname,returnNCvar=True)
outputdata=np.zeros((12,ncdata.data.shape[1],ncdata.data.shape[2]))
ncdata.ncfile.close()
for f in files:
data=io.read_nc(f,esgvarname).data
startpoint=0
for i in range(12):
endpoint=startpoint+month_lengths[i]
outputdata[i,...]+=data[startpoint:endpoint,:,:].mean(axis=0)
startpoint=endpoint
outputdata/=len(files)
data=np.zeros((12,geolut.shape[0],geolut.shape[1]))
for i in range(4):
y=geolut[:,:,i,0].astype('i')
x=geolut[:,:,i,1].astype('i')
data+=np.float32(outputdata[:,y,x]*geolut[np.newaxis,:,:,i,2])
return data
def load_tskin(filename, tsvarname, maskvarname):
"""load skin temp data from wrf output file: filename
tskin uses it's own procedure so that we can average over the past 10 days for land points
ideally it will create a higher resolution output based on a high-resolution land-sea mask
"""
if verbose: print("Loading :" + maskvarname)
ncmask = mygis.read_nc(filename, maskvarname, returnNCvar=True)
mask = ncmask.data[0]
ncmask.ncfile.close()
land = np.where(mask == 1)
if verbose: print("Loading :" + tsvarname)
ncts = mygis.read_nc(filename, tsvarname, returnNCvar=True)
atts = mygis.read_atts(filename, tsvarname)
dims = ncts.data.dimensions
if verbose: print(" " + str(dims))
if verbose: print(" " + str(atts))
data = ncts.data[:]
ncts.ncfile.close()
for i in range(data.shape[0] - 1, 0, -1):
start = max(i - (10 * steps_per_day), 0)
data[i][land] = data[start:i + 1].mean(axis=0)[land]
return Bunch(name=tsvarname,
data=data,
attributes=atts,
dims=dims,
dtype='f')
def load_erai_means(wind_option):
"""docstring for load_erai_means"""
eraid="erai/"
varlist=["p","rh","ta","ua","va","z"]
if (wind_option=="nowind"):
varlist=["p","rh","ta","z"]
outputdata=[]
month_mid_point_doy=(start_day_per_month[1:]+np.array(start_day_per_month[:-1]))*0.5
for month in range(1,13):
curoutput=Bunch(doy=month_mid_point_doy[month-1])
for v in varlist:
if v=="ta":
ta=mygis.read_nc(eraid+"regridded_ERAi_to_cesm_month{0:02}.nc".format(month),v).data
else:
curoutput[v]=mygis.read_nc(eraid+"regridded_ERAi_to_cesm_month{0:02}.nc".format(month),v).data
curoutput["theta"] = ta / units.exner(curoutput.p)
# erai is probably "upside down" so reverse the vertical dimension of all variables
if curoutput.p[0,0,0]<curoutput.p[-1,0,0]:
for v in curoutput.keys():
if v!="doy":
curoutput[v]=curoutput[v][::-1,:,:]
outputdata.append(curoutput)
return outputdata
def vcoord(filename):
"""compute the vertical coordinate in space and time for a given file"""
na=np.newaxis
ap= mygis.read_nc(filename,"ap").data[na,:,na,na]
b = mygis.read_nc(filename,"b").data[na,:,na,na]
ps= mygis.read_nc(filename,"ps").data[:,na,:,:]
p= ap+b*ps
return p
def vcoord(filename):
"""compute the vertical coordinate in space and time for a given file"""
na = np.newaxis
a = mygis.read_nc(filename, "lev").data[na, :, na, na]
b = mygis.read_nc(filename, "b").data[na, :, na, na]
orog = mygis.read_nc(filename, "orog").data
z = a + b * orog
return z
def vcoord(filename):
"""compute the vertical coordinate in space and time for a given file"""
na=np.newaxis
ptop = mygis.read_nc(filename,"ptop").data
sigma = mygis.read_nc(filename,"lev").data[na,:,na,na]
ps= mygis.read_nc(filename,"ps").data[:,na,:,:]
p= ptop+sigma*(ps-ptop)
return p
def vcoord(filename):
"""compute the vertical coordinate in space and time for a given file"""
na=np.newaxis
a = mygis.read_nc(filename,"lev").data[na,:,na,na]
b = mygis.read_nc(filename,"b").data[na,:,na,na]
orog= mygis.read_nc(filename,"orog").data
z= a+b*orog
return z
def vcoord(filename):
"""compute the vertical coordinate in space and time for a given file"""
na = np.newaxis
ptop = mygis.read_nc(filename, "ptop").data
sigma = mygis.read_nc(filename, "lev").data[na, :, na, na]
ps = mygis.read_nc(filename, "ps").data[:, na, :, :]
p = ptop + sigma * (ps - ptop)
return p
def vcoord(filename):
"""compute the vertical coordinate in space and time for a given file"""
na = np.newaxis
ap = mygis.read_nc(filename, "ap").data[na, :, na, na]
b = mygis.read_nc(filename, "b").data[na, :, na, na]
ps = mygis.read_nc(filename, "ps").data[:, na, :, :]
p = ap + b * ps
return p
def main():
"""convert probability coefficients to precip amounts"""
print("Reading Time data")
timeseconds=mygis.read_nc(probfile,"time").data
time=[base_date+datetime.timedelta(i/86400.0) for i in timeseconds]
print("Reading Spatial data")
lat=mygis.read_nc(probfile,"latitude").data
lon=mygis.read_nc(probfile,"longitude").data
print("Reading Probability regressions")
prob=mygis.read_nc(probfile,"coefficient").data
print("Reading Precip regressions")
prcp=mygis.read_nc(prcpfile,"coefficient").data
nx=prob.shape[-1]
ny=prob.shape[-2]
nt=prob.shape[-3]
nv=prob.shape[0]
nt=len(time)
# print(nv)
# print(file_variable,variable_name)
# nt=30
print(time[0],time[-1],len(time),nt)
output_data=np.zeros((nt,ny,nx))
output_data2=np.zeros((nt,ny,nx))
output_data3=np.zeros((nt,ny,nx))
output_data4=np.zeros((nt,ny,nx))
print("starting")
nvars=(nv-3)/2 # minus 3 for constant, output, residual columns
# /2 because we get both the coefficient and the value used for each
for i in range(nt):
print(" ",i," / ",nt,end="\r")
sys.stdout.flush()
# gefs_data=load_gefs(time[i],geo_file=probfile)
rand_data=load_rand(i)
curprec=prcp[nvars+1,i,...]
curprob=prob[nvars+1,i,...]
# for v in range(1,nv):
# # gefs_data[v-1]=gefs_data[v-1]*gains[v-1]+offsets[v-1]
# curprec+=gefs_data[v-1]*prcp[v,i,...]
# curprob+=gefs_data[v-1]*prob[v,i,...]
# curprec[norm.cdf(rand_data)<curprob]=0
# output_data[i,...]=curprec
# output_data2[i,...]=curprob
curprob/=200.0
curprob[curprob<norm.cdf(rand_data)]=0
output_data3[i,...]=curprob
# anywhere the probability of precip is less than the random number, set precip to 0
curprec[curprob<norm.cdf(rand_data)]=0
# add a random component to the precipitation rescaled by the residuals
curprec+=prcp[nvars+2,i,...] * rand_data
curprec[curprec<0]=0
output_data4[i,...]=curprec**(1/3.0)
# mygis.write(output_file,output_data)
# mygis.write(output_file+"prob",output_data2)
mygis.write(output_file+"prob_thresh",output_data3)
mygis.write(output_file+"prec_thresh",output_data4)
def main (filesearch="*.nc",copy_from_file="master.nc",vars2copy=["lat","lon"],
ranges=None,inputvars=None,verbose=False,output_prefix="added_"):
if ranges==None:
ranges=[]
for f in vars2copy:
ranges.append([])
files=glob.glob(filesearch)
files=np.sort(files)
# this isn't the most general way to do this, but I'm not likely to need an arbitrary number of dimensions
if inputvars==None:
inputvars=[]
for v,minmax in zip(vars2copy,ranges):
# if it is only a one dimensional array
if len(minmax)==2:
xmin=minmax[0]
xmax=minmax[1]
print(v,xmin,xmax)
inputvars.append(swim_io.read_nc(copy_from_file,var=v).data[xmin:xmax])
# if it is a two dimensional array
elif len(minmax)==4:
xmin=minmax[0]
xmax=minmax[1]
ymin=minmax[2]
ymax=minmax[3]
inputvars.append(swim_io.read_nc(copy_from_file,var=v).data[ymin:ymax,xmin:xmax])
# three dimensions
elif len(minmax)==6:
xmin=minmax[0]
xmax=minmax[1]
ymin=minmax[2]
ymax=minmax[3]
tmin=minmax[4]
tmax=minmax[5]
inputvars.append(swim_io.read_nc(copy_from_file,var=v).data[tmin:tmax,ymin:ymax,xmin:xmax])
# four dimensions (e.g. x,y,z,time)
elif len(minmax)==8:
xmin=minmax[0]
xmax=minmax[1]
ymin=minmax[2]
ymax=minmax[3]
zmin=minmax[4]
zmax=minmax[5]
tmin=minmax[6]
tmax=minmax[7]
inputvars.append(swim_io.read_nc(copy_from_file,var=v).data[tmin:tmax,zmin:zmax,ymin:ymax,xmin:xmax])
else: #for now this defaults to all data if minmax=[] (or otherwise)
inputvars.append(swim_io.read_nc(copy_from_file,var=v).data)
else:
inputvars=[]
for f in files:
if verbose:
print(f)
add_vars_to_file(f,inputvars,vars2copy,output_prefix)
def load(snowoff="compiled_lidar.nc",snowon="snow-on-dem.nc"):
"""Load data from two netCDF files,
convert data to snow depth, elevation, and veg height"""
bare_data=io.read_nc(snowoff,"data").data
dem=filter_dem(bare_data[0,...])
veg=calc_veg_height(bare_data[1,...],dem)
snow_data=io.read_nc(snowon,"Band1").data
snow=calc_snow_depth(snow_data,dem)
return Bunch(snow=snow,dem=dem,veg=veg,raw=[bare_data,snow_data])
def load(filename,startyear=2004,startdate=None,fill=True,extractday=None):
"""Load a SNODAS SWE file
Assumes a flat binary file as described above, but calculates the number of days present
Returns the data along with lat, lon, and date (based on startdate or startyear)
"""
if filename is not None:
d=np.fromfile(filename,np.float32)
else:
d=None
# find the directory in which files will be
snodas_dir="/".join(filename.split("/")[:-1])
if len(snodas_dir)<1:
snodas_dir="."
startlon=-122.371249999998
nlon=2191
dlon=0.00833333333333333
lon=np.array([startlon+dlon*i for i in range(nlon)])
startlat=32.9962495168051
nlat=1291
dlat=dlon
lat=np.array([startlat+dlat*i for i in range(nlat)])
demfile=snodas_dir+"/snodas_dem.nc"
forestfile=snodas_dir+"/snodas_forest.nc"
forest=[1]
bare=[0]
vegclass=myio.read_nc(forestfile).data
forested=np.where(vegclass==forest[0])
exposed=np.where(vegclass==bare[0])
mask=np.zeros(vegclass.shape)
mask[forested]=1
mask[exposed]=2
dem=myio.read_nc(demfile).data
if startdate is None:startdate=datetime.datetime(startyear,1,1,0)
if d is None:
ntimes=365
else:
ntimes=ntimes=d.size/nlon/nlat
dates=[startdate+datetime.timedelta(i) for i in range(int(ntimes))]
if d is not None:
d=d.reshape((ntimes,nlat,nlon))
if fill:
fill_missing(d)
if extractday is not None:
d=d[extractday,...]
dates=np.array([dates[extractday]])
return Bunch(data=d,lat=lat,lon=lon,dates=dates,dem=dem,lc=mask)
def load_ncep():
"""load real ncep data (by file)"""
files=glob.glob(ncep_files)
files.sort()
data=[]
times=[]
for f in files:
data.append(mygis.read_nc(f,ncep_var).data)
times.append(mygis.read_nc(f,"time").data)
geo=mygis.read_geo(files[0])
return Bunch(data=data,geo=geo,files=files,time=times)
def get_xy(filename,geo):
"""return x,y coordinates in geo for all points in the file"""
lat=mygis.read_nc(filename,"station_latitude").data
lon=mygis.read_nc(filename,"station_longitude").data
xy=Bunch(x=np.zeros(len(lon)),y=np.zeros(len(lat)))
for i in range(lat.size):
dists=(lat[i]-geo.lat)**2+(lon[i]-geo.lon)**2
y,x=np.unravel_index(np.argmin(dists),dists.shape)
xy.x[i]=x
xy.y[i]=y
return xy
def geo(filename, subset=False):
X_utm=mygis.read_nc(filename,"X").data
Y_utm=mygis.read_nc(filename,"Y").data
X_utm,Y_utm=np.meshgrid(X_utm, Y_utm)
projection = pyproj.Proj(proj='utm',zone=33,ellps='WGS84')
lons, lats = projection(X_utm,Y_utm,inverse=True)
if subset:
lons=lons[1549:800:-1,:600]
lats=lats[1549:800:-1,:600]
return Bunch(lon=lons,lat=lats)
def load_noahmp(filename, startyear=2000,startdate=None):
"""Load WRF-Noahmp SWE data from a file
"""
wrf_dir="/".join(filename.split("/")[:-1])
geo_file=filename
data=myio.read_nc(filename,"SNEQV").data[8*2::48,:,:]
lat=myio.read_nc(geo_file,"lat").data
lon=myio.read_nc(geo_file,"lon").data
if startdate==None:startdate=datetime.datetime(startyear,10,1,0)
ntimes=data.shape[0]
dates=np.array([startdate+datetime.timedelta(i) for i in range(ntimes)])
return Bunch(data=data,lat=lat,lon=lon,dates=dates)
def load_data(filename, varname):
"""loads a variable:varname from the netcdf file:filename
if varname is actually a number, then just return that number in the data structure
"""
if verbose:print("Loading:"+str(varname))
if isinstance(varname, numbers.Number):
if verbose:print(" "+str(varname)+" is a number!")
return Bunch(name="n"+str(varname), data=varname, attributes="a number", dims=(), dtype='f')
# load the data as a netcdf4 data structure first
d=mygis.read_nc(filename,varname,returnNCvar=True)
# copy the dimensions of the variable
dims=d.data.dimensions
# read the attributes of the variable
atts=mygis.read_atts(filename,varname)
if verbose:print(" "+str(dims))
if verbose:print(" "+str(atts))
# now read the actual data
data=d.data[:]
d.ncfile.close()
return Bunch(name=varname,data=data,dims=dims,attributes=atts, dtype='f')
def nextBilin(self):
curfile=self._curfile
if curfile>=len(self._filenames):
raise StopIteration
geoLUT=self._geoLUT
x=geoLUT[:,:,:,1]
y=geoLUT[:,:,:,0]
w=geoLUT[:,:,:,2]
minx=x.min().astype('i')
maxx=x.max().astype('i')+1
miny=y.min().astype('i')
maxy=y.max().astype('i')+1
curx=(x-minx).astype('i')
cury=(y-miny).astype('i')
N=curx[:,:,0].shape
data=list()
for thisvar in self._vars:
thisdata=np.zeros(N)
ncfile=mygis.read_nc(self._filenames[curfile],var=thisvar,returnNCvar=True)
if self.npos==1:
curdata=ncfile.data[miny:maxy,minx:maxx]
else:
curdata=ncfile.data[self.posinfile,miny:maxy,minx:maxx]
self.posinfile+=1
for i in range(4):thisdata+=curdata[cury[:,:,i],curx[:,:,i]]*w[:,:,i]
data.append(thisdata)
ncfile.ncfile.close()
# datestr=self._filenames[curfile].split('.')[1]
if (self.npos==1) or (self.posinfile>=self.npos):
self._curfile+=1
self.posinfile=0
return data
def load_sfc(time,info):
"""load surface forcing from a grib file (or netcdf file if it has been converted previously)"""
inputfile,offset=find_sfc_file(time,info)
try:
nc_file=sfc_ncfiles[inputfile]
except KeyError:
nc_file=grib2nc(inputfile,sfcvarlist,info.nc_file_dir)
sfc_ncfiles[inputfile]=nc_file
outputdata=Bunch()
for s,v in zip(icar_sfc_var,sfcvarlist):
nc_data=mygis.read_nc(nc_file,v,returnNCvar=True)
input_data=nc_data.data[:,info.ymin:info.ymax,info.xmin:info.xmax]
if ((s=="latent_heat") or (s=="sensible_heat") or (s=="sw") or (s=="lw")):
# if offset>=3:
# input_data[offset,...]-=input_data[offset-3,...]
# input_data[offset,...]/3.0
if offset>=2:
input_data[offset,...]-=input_data[offset-2,...]
input_data[offset,...]/2.0
elif offset>=1:
input_data[offset,...]-=input_data[offset-1,...]
outputdata[s]=input_data[int(offset),:,:]
nc_data.ncfile.close()
return outputdata
def load_monthly_wrf_tair(filesearch):
"""docstring for load_monthly_wrf"""
files=glob.glob(filesearch)[0] #air T is all in one file
data=io.read_nc(files,"T2").data[24:(-31*24)]
output_data=np.zeros((12,data.shape[1],data.shape[2]))
time0=datetime.datetime(2000,10,1,0,0,0)
dt=datetime.timedelta(1/24.0)
times=[time0+dt*i for i in range(data.shape[0])]
newdelta=datetime.timedelta(0)
# no leap years in this dataset, so bump up the timeoffset periodically
for i in range(len(times)):
if (times[i].month==2) and (times[i].day==29):
newdelta+=datetime.timedelta(1)
times[i]+=newdelta
for i in range(data.shape[0]):
curmonth=times[i].month-1
output_data[curmonth,...]+=data[i,...]
for i in range(12):
output_data[i,...]/=month_lengths[i]*24
return output_data
def load_data(filename, varname):
"""loads a variable:varname from the netcdf file:filename
if varname is actually a number, then just return that number in the data structure
"""
if verbose: print("Loading:" + str(varname))
if isinstance(varname, numbers.Number):
if verbose: print(" " + str(varname) + " is a number!")
return Bunch(name="n" + str(varname),
data=varname,
attributes="a number",
dims=(),
dtype='f')
# load the data as a netcdf4 data structure first
d = mygis.read_nc(filename, varname, returnNCvar=True)
# copy the dimensions of the variable
dims = d.data.dimensions
# read the attributes of the variable
atts = mygis.read_atts(filename, varname)
if verbose: print(" " + str(dims))
if verbose: print(" " + str(atts))
# now read the actual data
data = d.data[:]
d.ncfile.close()
return Bunch(name=varname,
data=data,
dims=dims,
attributes=atts,
dtype='f')
def load_icar(filename,h,preciponly=False):
"""docstring for load_wrf"""
# run_num=filename.split("out")[-1]
# last_file=filename.replace(run_num,("{0:0"+str(len(run_num))+"}").format(int(run_num)-1))
if type(filename)==list:
output=[]
for f in filename:
output.append(load_icar(f,h,preciponly=preciponly))
return output
precip=mygis.read_nc(filename,"rain").data
precip=precip[11,1]-precip[10,1]
if preciponly:
return Bunch(precip=precip)
u=mygis.read_nc(filename,"u").data[0,1,...]
# dudx=-np.diff(u,axis=1)
dudx=u[:,:-1]-u[:,1:]
# w=mygis.read_nc(filename,"w").data[1,...]
z=dudx*0
z[0,:]=h+dz_levels[0]/2
nz=z.shape[0]
for i in range(1,nz):
z[i,:]=z[i-1,:]+(dz_levels[i-1]+dz_levels[i])/2
# print(z[:,0])
dz=np.diff(h)
# w1=u[:,1:-1]*np.sin(np.arctan(dz[np.newaxis,:]/dx))
# w1=u[:,1:-1]*dz[np.newaxis,:]/np.sqrt(dz[np.newaxis,:]**2 + dx**2)
w1=u[:,1:-1]*dz[np.newaxis,:]/dx
w1=(w1[:,1:]+w1[:,:-1])/2.0 # * dz_levels[:,np.newaxis]/dx # *10.0
w2=dudx*dz_levels[:,np.newaxis]/dx
for i in range(1,nz):
w2[i,:]+=w2[i-1,:]
w2=w2[:,1:-1]
w=z*0
w[:,1:-1]=w1+w2
# w[:,1:-1]=w2
return Bunch(w=w,z=z,hgt=h,u=u,precip=precip)
def apply_bias_correction(wind_option):
"""docstring for apply_bias_correction"""
varlist = ["theta", "p", "u", "v", "rh"]
if wind_option == "nowind": print("Skipping U and V bias correction.")
biases = load_biases(wind_option)
current_files = glob.glob("cesm_*.nc")
current_files.sort()
for f in current_files:
print("Bias Correcting : " + f)
output_data = mygis.Dataset(f, mode="a")
for v in varlist:
print(" Variable : " + v)
try:
d = mygis.read_nc("vinterpolated/" + v + "_" + f, v).data
except:
d = mygis.read_nc(v + "_" + f, v).data
if (wind_option == "nowind") and ((v == "u") or (v == "v")):
pass
if (wind_option == "meanwind") and ((v == "u") or (v == "v")):
for i in range(d.shape[0]):
windbias = biases[int(np.round(i /
float(times_per_day)))][v]
for j in range(d.shape[1]):
d[i, j, ...] -= np.mean(windbias[j, ...])
else:
for i in range(d.shape[0]):
d[i, ...] -= biases[int(np.round(i /
float(times_per_day)))][v]
if v == "rh":
d[d < 1e-10] = 1e-10
d[d > 100] = 100
rh = d
else:
output_data.variables[v][:] = d
p = output_data.variables["p"][:] / 100.0 # convert to mb
t = output_data.variables["theta"][:] * units.exner(p)
output_data.variables["qv"][:] = units.rh2sh(t, p, rh)
output_data.variables["z"] = biases[0]["z"][np.newaxis, :, :, :]
output_data.close()
def load_icar(filenames):
filenames=glob.glob(filenames)
filenames.sort()
data=[]
r0=mygis.read_nc(filenames[0],"rain").data[1,:]*0
for f in filenames:
r=mygis.read_nc(f,"rain").data[1,:]
outputdata=Bunch(qv= mygis.read_nc(f,"qv").data[1,:,:],
qc= mygis.read_nc(f,"qc").data[1,:,:]
+mygis.read_nc(f,"qi").data[1,:,:]
+mygis.read_nc(f,"qs").data[1,:,:]
+mygis.read_nc(f,"qr").data[1,:,:],
t = mygis.read_nc(f,"th").data[1,:,:],
p = mygis.read_nc(f,"p").data[1,:,:],
r = r - r0,
u = mygis.read_nc(f,"u").data[1,:,:])
r0=r
data.append(outputdata)
return data
def load_last_date(filename):
"""docstring for load_last_date"""
time_data=mygis.read_nc(filename,"time").data
last_hour=len(time_data)
year=filename.split("_")[-4]
month=filename.split("_")[-3]
day=filename.split("_")[-2]
hour=last_hour-1 # look back one time step incase it hadn't finished writing the last timestep
return "{0}, {1}, {2}, {3}, 0, 0".format(year,month,day,hour),hour
def load_sfc(time, info):
"""load surface forcing from a grib file (or netcdf file if it has been converted previously)"""
global last_precip
inputfile, offset = find_sfc_file(time, info)
if last_precip > offset: last_precip = -1
try:
nc_file = sfc_ncfiles[inputfile]
except KeyError:
nc_file = grib2nc(inputfile, sfcvarlist, info.nc_file_dir)
sfc_ncfiles[inputfile] = nc_file
outputdata = Bunch()
for s, v in zip(gard_sfc_var, sfcvarlist):
# print(nc_file)
nc_data = mygis.read_nc(nc_file, v, returnNCvar=True)
input_data = nc_data.data[:, info.ymin:info.ymax, info.xmin:info.xmax]
if ((s == "latent_heat") or (s == "sensible_heat") or (s == "sw")
or (s == "lw")):
# if offset>=3:
# input_data[offset,...]-=input_data[offset-3,...]
# input_data[offset,...]/3.0
# if offset>=2:
# input_data[offset,...] -= input_data[offset-2,...]
# input_data[offset,...] /= 2
# elif offset==1:
if offset > 0:
input_data[offset, ...] -= input_data[offset - 1, ...]
if (v == "2T_GDS4_SFC"):
outputdata["tmax"] = input_data[int(offset):int(offset +
2), :, :].max(
axis=0)
outputdata["tmin"] = input_data[int(offset):int(offset +
2), :, :].min(
axis=0)
if ((s == "precip_conv") or (s == "precip_total")):
offset += 1 # needs to accumulate over the entire time period, assumes two time steps... :(
if last_precip != -1:
input_data[offset, ...] -= input_data[last_precip, ...]
outputdata[s] = input_data[int(offset), :, :]
if ((s == "precip_conv") or (s == "precip_total")):
offset -= 1 # needs to be reset for other variables
nc_data.ncfile.close()
last_precip = offset + 1
return outputdata
def set_bounds(info):
atm_file = info.atmdir + info.atmfile
erai_file = atm_file.replace("_Y_", "2000").replace("_M_", "01").replace(
"_D_", "01").replace("_h_", "00")
varlist = ["g4_lat_0", "g4_lon_1"]
output_dir = info.nc_file_dir
try:
os.mkdir(output_dir)
except:
pass
ncfile = io.grib2nc(erai_file, varlist, output_dir)
lat = mygis.read_nc(ncfile, varlist[0]).data
lon = mygis.read_nc(ncfile, varlist[1]).data - 360
# print(lon, info.lon[0])
lon[lon < -180] += 360
info.xmin = max(0, np.argmin(np.abs(lon - info.lon[0])) - 1)
info.xmax = min(lon.size - 1, np.argmin(np.abs(lon - info.lon[1])) + 1)
if (info.xmax < info.xmin):
print("ERROR: attempting to wrap around the ERAi boundary lon=" +
str(lon[0]) + str(lon[-1]))
print(" Requested East lon = " + str(info.lon[0]))
print(" Requested West lon = " + str(info.lon[1]))
print("Requires Custom Solution!")
raise IndexError
# info.xmin = max(0,np.where(lon >= info.lon[0])[0][0]-1)
# info.xmax = min(lon.size-1, np.where(lon[info.xmin:] >= info.lon[1])[0][0] + info.xmin + 1)
#note lat is inverted from "expected"
info.ymin = np.where(lat <= info.lat[1])[0][0]
info.ymax = np.where(lat >= info.lat[0])[0][-1] + 1
lon, lat = np.meshgrid(lon[info.xmin:info.xmax],
lat[info.ymin:info.ymax][::-1])
info.lat_data = lat
info.lon_data = lon
# important to remove this file so that if this temporary date overlaps the actual time period we want to run,
# later steps won't break when they try to use this temporary file by mistake
os.remove(ncfile)
def load_last_date(filename, prefix):
"""docstring for load_last_date"""
time_data=mygis.read_nc(filename,"time").data
last_hour=len(time_data)
date_string = filename.replace(prefix,"")
year = date_string.split("_")[-4]
month = date_string.split("_")[-3]
day = date_string.split("_")[-2]
hour = last_hour - 2 # look back two time steps in case the last time steps were corrupted
# (not sure why more than 1 is / can be, but it happened)
return "{0}, {1}, {2}, {3}, 0, 0".format(year,month,day,hour),hour
def load_cesm_means(wind_option):
"""docstring for load_cesm_means"""
cesmd = "means/"
varlist = ["p", "rh", "theta", "u", "v"]
if wind_option == "nowind":
varlist = ["p", "rh", "theta"]
outputdata = []
month_mid_point_doy = (start_day_per_month[1:] +
np.array(start_day_per_month[:-1])) * 0.5
for month in range(1, 13):
curoutput = Bunch(doy=month_mid_point_doy[month - 1])
try:
for v in varlist:
curoutput[v] = mygis.read_nc(
cesmd + "month{0:02}_mean_{1}.nc".format(month, v), v).data
curoutput["z"] = mygis.read_nc(cesmd + "annual_mean_z.nc",
"z").data
except:
for v in varlist:
curoutput[v] = mygis.read_nc(
"month{0:02}_mean_{1}.nc".format(month, v), v).data
curoutput["z"] = mygis.read_nc("annual_mean_z.nc", "z").data
if curoutput["p"][0, 0, 0] < 1300:
curoutput["p"][0, 0, 0] *= 100.0 # convert hPa to Pa
if (curoutput["z"][-1, 0, 0] < 10000) and curoutput["p"][
-1, 0, 0] < 10000: # can't have 100hPa at 10km height
print(
"WARNING! Assuming that cesm z has been erroneously divided by 9.8"
)
curoutput["z"] *= 9.8
outputdata.append(curoutput)
return outputdata
def vinterp_q(file_name=None):
"""interpolate all variables to a common z level"""
print("Interpolating quantities")
if file_name != None:
file_search = file_name
qvarlist = ["qv", "theta", "p", "u", "v"]
qdata = Bunch()
print("Loading data")
t0 = time.time()
for varname in qvarlist:
print(varname)
qdata[varname] = mygis.read_files(file_search, varname, axis=0)
print("z")
qdata["z"] = mygis.read_files(file_search, "z", axis=0)
mean_z = mygis.read_nc("annual_mean_z.nc", "z").data
print(" Reading data took {0:6.2f} seconds".format((time.time() - t0)))
t0 = time.time()
temperature = qdata.theta * units.exner(qdata.p)
print(" Temperature conversion took {0:6.2f} seconds".format(
(time.time() - t0)))
t0 = time.time()
print("Computing RH")
qdata["rh"] = compute_rh(qdata, temperature)
qvarlist.append("rh")
print(" RH calculation took {0:6.2f} seconds".format((time.time() - t0)))
print("Interpolating")
qout = Bunch()
for varname in qvarlist:
qout[varname] = np.zeros(qdata[varname].shape)
t0 = time.time()
nsteps = qdata.z.shape[0]
for i in range(nsteps):
vertical_interp.interp_multivar(qdata,
qout,
i,
qdata.z[i],
mean_z,
vartype=varname,
inputt=temperature[i],
inputp=qdata.p[i])
print(" Interpolation took {0:6.2f} seconds per timestep".format(
(time.time() - t0) / nsteps))
print("Writing data")
write_interpolated_6hrly(qout, mean_z, file_name)
def load_icar(filename,preciponly=False):
"""docstring for load_wrf"""
precip=mygis.read_nc(filename,"rain").data
if preciponly:
return Bunch(precip=precip)
u=mygis.read_nc(filename,"u").data
v=mygis.read_nc(filename,"v").data
w=mygis.read_nc(filename,"w").data
z=mygis.read_nc(filename,"z").data
dz=np.zeros(z.shape)
dx=4000.0
dz[:,0,:]=z[:,0,:]*2
for i in range(1,z.shape[1]):
dz[:,i,:] = 2*(z[:,i,:]-z[:,i-1,:]) - dz[:,i-1,:]
w[:,:,i,:] *= dz[np.newaxis,:,i,:] / dx
dzdx=np.diff(z,axis=2)
dzdy=np.diff(z,axis=0)
w_u=u[:,:,:,:-2]*dzdx[np.newaxis,:,:,:]/dx
w_u=(w_u[:,:,:,1:]+w_u[:,:,:,:-1])/2.0
w_v=v[:,:-2,:,:]*dzdy[np.newaxis,:,:,:]/dx
w_v=(w_v[:,1:,:,:]+w_v[:,:-1,:,:])/2.0
w[:,1:-1,:,1:-1]+=w_u[:,1:-1,:,:]+w_v[:,:,:,1:-1]
qv=mygis.read_nc(filename,"qv").data
t=mygis.read_nc(filename,"th").data
p=mygis.read_nc(filename,"p").data
w=w[:,4:,:,1:]
v=v[:,4:,:,1:]
u=u[:,4:,:,1:]
z=z[4:,:,1:]
precip=precip[:,4:,1:]
t=t[:,4:,:,1:]
qv=qv[:,4:,:,1:]
p=p[:,4:,:,1:]
return Bunch(z=z,w=w,u=u,v=v,qv=qv,t=t,p=p,precip=precip)
def main(filesearch, outputfile):
files = glob.glob(filesearch)
files.sort()
d0 = mygis.read_nc(files[0], "rain").data
outputdata = np.zeros((len(files), d0.shape[1], d0.shape[2]))
if verbose: print("Outputdata.shape : " + str(outputdata.shape))
last_data = d0[0] * 0
lost_rain = d0[0] * 0
for i, f in enumerate(files):
if verbose: print(f)
d = mygis.read_nc(f, "rain").data + lost_rain
if np.max(d[-1, 5:-5, 5:-5]) < np.max(last_data[5:-5, 5:-5]):
if verbose: print("Updating lost_rain")
d -= lost_rain
lost_rain = update_data(d, last_data)
outputdata[i] = d[-1] - last_data
last_data = d[-1]
if verbose: print("Writing output file")
mygis.write(outputfile, outputdata, varname="pr")
def load_wrf(filename, preciponly=False):
"""docstring for load_wrf"""
time=15
precip=mygis.read_nc(filename,"RAINNC").data
precip=precip[time+25,1,:] - precip[time-5,1,:]
precip/=15.0 # 30 outputsteps = 15 hours
if preciponly:
return Bunch(precip=precip,hgt=None)
w=mygis.read_nc(filename,"W").data[time,:,1,:]
u=mygis.read_nc(filename,"U").data[time,:,1,:]
z=mygis.read_nc(filename,"PH").data[time,:,1,:]
z+=mygis.read_nc(filename,"PHB").data[time,:,1,:]
z/=9.81
hgt=mygis.read_nc(filename,"HGT").data[time,1,:]
return Bunch(w=w,z=z,hgt=hgt,u=u,precip=precip)
def main(filename, vars_to_map, outputfile, cmin, cmax):
# reads latitude and longitude information
if verbose: print("Reading lat/lon info")
geo = mygis.read_geo(filename)
map_bounds = [geo.lat.min(), geo.lat.max(), geo.lon.min(), geo.lon.max()]
# use a subset of the reversed rainbow colormap (if not specified it will use "jet")
cmap = custom_cmap.subset(cmap=plt.cm.rainbow_r, clim=(50, 255))
# alternatively a list of colormaps could be supplied so that each variable gets its own colormap
if verbose: print("Looping over variables : " + str(vars_to_map))
# optionally set the figure size with e.g. plt.figure(figsize=)
for v in vars_to_map:
# optionally put multiple plots on one figure with e.g. plt.subplot(nrows, ncols, current_plot_number)
if verbose:
print(' Reading variable: {} from file: {}'.format(v, filename))
# read the last time entry [-1] in the file
data = mygis.read_nc(filename, v).data[-1]
if len(data.shape) > 2:
# if this is a 3D variable, just pull the first level
data = data[0]
# set up the output filename
if len(vars_to_map) > 1:
# create a new output file by appending the variable name before the file extension
ofile = outputfile[:-4] + "_" + v + outputfile[-4:]
else:
ofile = outputfile
# to make a simple image with a colorbar this is all you need:
# if verbose: print(" Plotting data")
# plt.imshow(data, cmap=cmap)
# plt.colorbar()
# plt.clim(cmin, cmax) # to set the limits on the colorscale
# plt.title(v)
# To make a map instead use this code.
# Note that a lot of these keywords are optional, I've included them here
# to give a sense of what you can do with this routine :
if verbose: print(" Mapping data")
m = map_vis.vis(data,
geo=map_bounds,
title=v,
cmap=cmap,
colorbar=True,
latstep=2.0,
lonstep=5.0,
m=None,
lat=geo.lat,
lon=geo.lon,
clim=(cmin, cmax),
reproject=True,
width=None,
height=None,
cbar_label=None,
latlabels=[1, 0, 0, 0],
lonlabels=[0, 0, 0, 1],
statewidth=1.5,
countrywidth=1.5,
countrycolor="black",
coastwidth=1.5,
riverwidth=0.5,
rivercolor="blue")
# Stuffs more on a page, but can have a tendency to push labels off the page if the figure is not wide/tall enough
# plt.tight_layout()
if verbose: print(" Writing output file:" + ofile)
plt.savefig(ofile)
# in case we are looping over variables, we need to clear the figure for the next plot
plt.clf()
def main(inputfile, sounding_file=None):
filename = "icar_" + inputfile
print(filename)
yaxis = 2
yaxis2d = 1
u = mygis.read_nc(inputfile, "U").data.repeat(2, axis=yaxis)
# w =mygis.read_nc(inputfile,"W").data.repeat(2,axis=yaxis)
v = mygis.read_nc(inputfile, "V").data.repeat(
2, axis=yaxis
)[:, :, :
-1, :] # v has one extra cell in y, so when doubling ydim we have to remove gridcell
qv = mygis.read_nc(inputfile, "QVAPOR").data.repeat(2, axis=yaxis)
qc = mygis.read_nc(inputfile, "QCLOUD").data.repeat(2, axis=yaxis)
qi = mygis.read_nc(inputfile, "QICE").data.repeat(2, axis=yaxis)
th = mygis.read_nc(inputfile, "T").data.repeat(2, axis=yaxis)
pb = mygis.read_nc(inputfile, "PB").data.repeat(2, axis=yaxis)
p = mygis.read_nc(inputfile, "P").data.repeat(2, axis=yaxis) + pb
phb = mygis.read_nc(inputfile, "PHB").data.repeat(2, axis=yaxis)
ph = mygis.read_nc(inputfile, "PH").data.repeat(2, axis=yaxis) + phb
hgt = mygis.read_nc(inputfile, "HGT").data.repeat(2, axis=yaxis2d)
land = mygis.read_nc(inputfile, "XLAND").data.repeat(2, axis=yaxis2d)
nt, nz, ny, nx = qv.shape
print(nx, ny, nz)
dims = np.array(qv.shape)
z = (ph) / g
dz = np.diff(z, axis=1)
# dz shape = (time,nz-1,ny,nx)
# ph/phb are defined between model levels, we want z in the middle of each model level
# e.g. z[0]=0, but wrfz[0]=dz[0]/2 where dz[0]=z[1]-z[0]
wrfz = (z[:, :-1, :, :] + z[:, 1:, :, :]) / 2
# p=units.z2p(wrfz,100000)
# wrfz=np.zeros(dz.shape)
# wrfz[:,0,...]=dz[:,0,...]/2+hgt
# for i in range(1,nz):
# wrfz[:,i,:,:]=(dz[:,i,:,:]+dz[:,i-1,:,:])/2+wrfz[:,i-1,:,:]
mean_dz = dz[0, ...].mean(axis=1).mean(axis=1)
print("MEAN LEVELS:")
print("dz_levels=[")
for i in range(0, nz, 10):
curlist = [str(cur) for cur in mean_dz[i:i + 10]]
print(",".join(curlist) + ",")
if i + 10 < mean_dz.shape[0]:
curlist = [str(cur) for cur in mean_dz[i + 10:]]
print(",".join(curlist) + "]")
else:
print("]")
print("FIRST LEVELS:")
print("dz_levels=[")
for i in range(0, nz, 10):
curlist = [str(cur) for cur in dz[0, i:i + 10, 0, 0]]
print(",".join(curlist) + ",")
if i + 10 < dz.shape[1]:
curlist = [str(cur) for cur in dz[0, i + 10:, 0, 0]]
print(",".join(curlist) + "]")
else:
print("]")
# dz=np.zeros(dz.shape)+mean_dz[np.newaxis,:,np.newaxis,np.newaxis]
dz = np.zeros(dz.shape) + dz[:, :, :, np.newaxis, 0]
z = np.zeros(dz.shape)
z[:, 0, ...] = dz[:, 0, ...] / 2 + hgt
for i in range(1, nz):
z[:,
i, :, :] = (dz[:, i, :, :] + dz[:, i - 1, :, :]) / 2 + z[:,
i - 1, :, :]
# adjust_p(p,wrfz,z)
dx = mygis.read_attr(inputfile, "DX")
if type(dx) == np.ndarray:
dx = dx[0]
dlon = dx / 111.1
dlat = dx / 111.1
lonmin = -110.0
lonmax = lonmin + nx * dlon
latmin = 40.0
latmax = latmin + ny * dlat
udims = copy(dims)
udims[-1] += 1
vdims = copy(dims)
vdims[-2] += 1
lon = np.arange(lonmin, lonmax, dlon)[:nx]
lat = np.arange(latmin, latmax, dlat)[:ny]
lon, lat = np.meshgrid(lon, lat)
ulon = np.arange(lonmin - dlon / 2, lonmax + dlon / 2, dlon)[:nx + 1]
ulat = np.arange(latmin, latmax, dlat)[:ny]
ulon, ulat = np.meshgrid(ulon, ulat)
vlon = np.arange(lonmin, lonmax, dlon)[:nx]
vlat = np.arange(latmin - dlat / 2, latmax + dlat / 2, dlat)[:ny + 1]
vlon, vlat = np.meshgrid(vlon, vlat)
lat = lat.reshape((1, ny, nx))
lon = lon.reshape((1, ny, nx))
hgt = hgt.reshape((1, ny, nx))
d3dname = ("t", "z", "y", "x")
ud3dname = ("t", "z", "y", "xu")
ud2dname = ("t", "y", "xu")
vd3dname = ("t", "z", "yv", "x")
vd2dname = ("t", "yv", "x")
d2dname = ("t", "y", "x")
othervars = [
Bunch(data=v,
name="V",
dims=vd3dname,
dtype="f",
attributes=dict(units="m/s",
description="Horizontal (y) wind speed")),
# Bunch(data=w, name="W", dims=d3dname, dtype="f",attributes=dict(units="m/s", description="Vertical wind speed")),
Bunch(data=qv,
name="QVAPOR",
dims=d3dname,
dtype="f",
attributes=dict(units="kg/kg",
description="Water vapor mixing ratio")),
Bunch(data=qc,
name="QCLOUD",
dims=d3dname,
dtype="f",
attributes=dict(units="kg/kg",
description="Cloud water mixing ratio")),
Bunch(data=qi,
name="QICE",
dims=d3dname,
dtype="f",
attributes=dict(units="kg/kg",
description="Cloud ice mixing ratio")),
Bunch(data=p * 0,
name="P",
dims=d3dname,
dtype="f",
attributes=dict(units="Pa",
description="Pressure (perturbation)")),
Bunch(data=p,
name="PB",
dims=d3dname,
dtype="f",
attributes=dict(units="Pa", description="Pressure (base)")),
Bunch(data=th,
name="T",
dims=d3dname,
dtype="f",
attributes=dict(units="K", description="Potential temperature")),
Bunch(data=dz,
name="dz",
dims=d3dname,
dtype="f",
attributes=dict(units="m", description="Layer thickness")),
Bunch(data=z,
name="Z",
dims=d3dname,
dtype="f",
attributes=dict(units="m",
description="Layer Height AGL (also ASL here)")),
Bunch(
data=wrfz,
name="WRFZ",
dims=d3dname,
dtype="f",
attributes=dict(
units="m",
description="Layer Height AGL (also ASL here) in the WRF input"
)),
# Bunch(data=z*0, name="PH", dims=d3dname, dtype="f",attributes=dict(units="m2/s2",description="Geopotential Height ASL (perturbation)")),
# Bunch(data=z*g, name="PHB", dims=d3dname, dtype="f",attributes=dict(units="m2/s2",description="Geopotential Height ASL (base)")),
Bunch(data=lat,
name="XLAT",
dims=d2dname,
dtype="f",
attributes=dict(units="deg", description="Latitude")),
Bunch(data=lon,
name="XLONG",
dims=d2dname,
dtype="f",
attributes=dict(units="deg", description="Longitude")),
Bunch(data=ulat,
name="XLAT_U",
dims=ud2dname,
dtype="f",
attributes=dict(units="deg",
description="Latitude on U stagger")),
Bunch(data=ulon,
name="XLONG_U",
dims=ud2dname,
dtype="f",
attributes=dict(units="deg",
description="Longitude on U stagger")),
Bunch(data=vlat,
name="XLAT_V",
dims=vd2dname,
dtype="f",
attributes=dict(units="deg",
description="Latitude on V stagger")),
Bunch(data=vlon,
name="XLONG_V",
dims=vd2dname,
dtype="f",
attributes=dict(units="deg",
description="Longitude on V stagger")),
Bunch(data=hgt,
name="HGT",
dims=d2dname,
dtype="f",
attributes=dict(units="m", description="Terrain Elevation")),
Bunch(data=land,
name="XLAND",
dims=d2dname,
dtype="f",
attributes=dict(units="",
description="Land Mask [1=land,2=water]"))
]
fileexists = glob.glob(filename) or glob.glob(filename + ".nc")
if fileexists:
print("Removing : " + fileexists[0])
os.remove(fileexists[0])
mygis.write(filename,
u,
varname="U",
dims=ud3dname,
dtype="f",
attributes=dict(units="m/s",
description="Horizontal (x) wind speed"),
extravars=othervars)
def load_wrf(filename):
"""docstring for load_wrf"""
data=[]
datelist=[]
qvdata=mygis.read_nc(filename,"QVAPOR").data
qcdata=mygis.read_nc(filename,"QCLOUD").data \
+ mygis.read_nc(filename,"QICE").data \
+ mygis.read_nc(filename,"QSNOW").data \
+ mygis.read_nc(filename,"QRAIN").data
tdata=mygis.read_nc(filename,"T").data + 300
pdata=mygis.read_nc(filename,"P").data \
+ mygis.read_nc(filename,"PB").data
rdata=mygis.read_nc(filename,"RAINNC").data
udata=mygis.read_nc(filename,"U").data
dates=mygis.read_nc(filename,"Times").data
r0=np.zeros(rdata[0,1,:].shape)
for i in range(0,qvdata.shape[0],2):
outputdata=Bunch(qv=qvdata[i,:,1,:],
qc=qcdata[i,:,1,:],
t = tdata[i,:,1,:],
p = pdata[i,:,1,:],
r = rdata[i,1,:] - r0,
u = udata[i,:,1,:])
r0=rdata[i,1,:]
datelist.append("".join(dates[i]))
data.append(outputdata)
return data,datelist
def load_wrf(filename, preciponly=False):
"""docstring for load_wrf"""
precip=mygis.read_nc(filename,"RAINNC").data
try:
precip+=mygis.read_nc(filename,"I_RAINNC").data*100 # try to add bucket data
except KeyError:
pass
if preciponly:
return Bunch(precip=precip,hgt=None)
t=mygis.read_nc(filename,"T").data+300
qv=mygis.read_nc(filename,"QVAPOR").data
p=mygis.read_nc(filename,"P").data
p+=mygis.read_nc(filename,"PB").data
u=mygis.read_nc(filename,"U").data
v=mygis.read_nc(filename,"V").data
w=mygis.read_nc(filename,"W").data
z=mygis.read_nc(filename,"PH").data
z+=mygis.read_nc(filename,"PHB").data
z/=9.8
hgt=mygis.read_nc(filename,"HGT").data
hgt=hgt[0,:-6,:-4]
p=p[:,:,:-6,:-4]
t=t[:,:,:-6,:-4]
qv=qv[:,:,:-6,:-4]
u=u[:,:,:-6,:-4]
v=v[:,:,:-6,:-4]
w=w[:,:,:-6,:-4]
precip=precip[:,:-6,:-4]
return Bunch(w=w,z=z,hgt=hgt,u=u,v=v,precip=precip,p=p,qv=qv,t=t)
