def load_wrf_data(stat,time):
"""Load WRF data and calculate the relevant statistic"""
output_wrf_file=stat+"_"+time+".nc"
try:
current=mygis.read_nc(wrfloc+"current_"+output_wrf_file).data
future=mygis.read_nc(wrfloc+"future_"+output_wrf_file).data
print("WRF data loaded from pre-calculated summary files")
except:
print(" Reading raw data...")
if time=="annual":
current=np.concatenate(mygis.read_files(wrfloc+"daily_NARR*"))
future=np.concatenate(mygis.read_files(wrfloc+"daily_PGW*"))
else:
month=time[-2:]
current=np.concatenate(mygis.read_files(wrfloc+"daily_NARR*"+month+".nc"))
future=np.concatenate(mygis.read_files(wrfloc+"daily_PGW*"+month+".nc"))
print(" Generating statistics")
if stat=="MAP":
ndays=calc_ndays(time)
current=current.mean(axis=0)*ndays
future=future.mean(axis=0)*ndays
elif stat=="wetfrac":
current=calc_wetfrac(current)
future=calc_wetfrac(future)
else:
raise KeyError("stat not created for WRF:"+stat)
print(" Writing stat for future reference")
mygis.write(wrfloc+"current_"+output_wrf_file,current)
mygis.write(wrfloc+"future_"+output_wrf_file,future)
# if stat=="MAP":
# current[current<1e-4]=1e-4
return (future-current),current
def vinterp_q(file_name=None):
"""interpolate all variables to a common z level"""
print("Interpolating quantities")
if file_name != None:
file_search = file_name
qvarlist = ["qv", "theta", "p", "u", "v"]
qdata = Bunch()
print("Loading data")
t0 = time.time()
for varname in qvarlist:
print(varname)
qdata[varname] = mygis.read_files(file_search, varname, axis=0)
print("z")
qdata["z"] = mygis.read_files(file_search, "z", axis=0)
mean_z = mygis.read_nc("annual_mean_z.nc", "z").data
print(" Reading data took {0:6.2f} seconds".format((time.time() - t0)))
t0 = time.time()
temperature = qdata.theta * units.exner(qdata.p)
print(" Temperature conversion took {0:6.2f} seconds".format(
(time.time() - t0)))
t0 = time.time()
print("Computing RH")
qdata["rh"] = compute_rh(qdata, temperature)
qvarlist.append("rh")
print(" RH calculation took {0:6.2f} seconds".format((time.time() - t0)))
print("Interpolating")
qout = Bunch()
for varname in qvarlist:
qout[varname] = np.zeros(qdata[varname].shape)
t0 = time.time()
nsteps = qdata.z.shape[0]
for i in range(nsteps):
vertical_interp.interp_multivar(qdata,
qout,
i,
qdata.z[i],
mean_z,
vartype=varname,
inputt=temperature[i],
inputp=qdata.p[i])
print(" Interpolation took {0:6.2f} seconds per timestep".format(
(time.time() - t0) / nsteps))
print("Writing data")
write_interpolated_6hrly(qout, mean_z, file_name)
def load_wrf_march_data(varname):
"""docstring for load_wrf_march_data"""
# current_data=mygis.read_files("NARR*_"+varname+"_*2004*.nc",varname)
# future_data=mygis.read_files("PGW*_"+varname+"_*2004*.nc",varname)
current_data=mygis.read_files("NARR*_"+varname+"_*.nc",varname)
future_data=mygis.read_files("PGW*_"+varname+"_*.nc",varname)
if varname=="SNOW":
return summarize_snow(current_data,future_data)
if varname=="RAINNC":
return summarize_precip(current_data,future_data)
if varname=="T2":
return summarize_temp(current_data,future_data)
def compute_mean_z():
"""docstring for compute_mean_z"""
print("Computing mean levels")
print("loading files:")
print(glob.glob(file_search))
z=mygis.read_files(file_search,"z",axis=0)
mygis.write("annual_mean_z.nc",z.mean(axis=0),varname="z")
for month in range(12):
print("Month "+str(month+1))
curz=z[0]*0
nyears=0
month_start=start_day_per_month[month]
start_point=month_start*times_per_day
month_end=start_day_per_month[month+1]
end_point=month_end*times_per_day
while start_point<z.shape[0]:
curz+=z[start_point:end_point].mean(axis=0)
nyears+=1
start_point+=times_per_day*days_per_year
end_point+=times_per_day*days_per_year
print(nyears)
curz/=nyears
mygis.write(zoutputfile.format(month+1),curz,varname="z")
def compute_mean_q(wind_option):
print("Computing Monthly Mean fields")
qvarlist=["qv","theta","p","u","v","rh"]
if (wind_option == "nowind"):
qvarlist=["qv","theta","p","rh"]
full_data=[]
for varname in qvarlist:
print(varname)
data=mygis.read_files(varname+"_cesm_*_*.nc",varname,axis=0)
for month in range(12):
print("Month "+str(month+1))
meanq=np.zeros(data.shape[1:])
nyears=0
month_start=start_day_per_month[month]
start_point=month_start*times_per_day
month_end=start_day_per_month[month+1]
end_point=month_end*times_per_day
while start_point<data.shape[0]:
print(start_point,end_point)
meanq+=data[start_point:end_point,...].mean(axis=0)
nyears+=1
start_point+=times_per_day*days_per_year
end_point+=times_per_day*days_per_year
if nyears>0:
meanq/=nyears
print(nyears)
mygis.write(qoutputfile.format(month+1,varname),meanq,varname=varname)
def temp_stats(dir_name, fulldomain=False):
if verbose:
files=glob.glob(dir_name+temp_file_search)
files.sort()
print(files[0]+" "+files[-1])
data=mygis.read_files(dir_name+temp_file_search,"mean_temperature",axis=0, verbose=verbose)
if not fulldomain:
data=data[:,1549:800:-1,:600]
if verbose: print("mean annual temperature")
mygis.write(dir_name+"annual_mean_temp.nc",stats.mean(data))
ndays=data.shape[0]
dates=compute_dates(ndays)
curmonth=np.empty(ndays,dtype=bool)
for month in range(12):
for i in range(ndays):
curmonth[i] = (dates[i].month == (month+1))
if verbose: print("month{:02}".format(month+1))
if verbose: print(" mean")
mygis.write(dir_name+"month{:02}_mean_temp.nc".format(month+1),stats.mean(data[curmonth,:,:]))
return data
def compute_mean_z():
"""docstring for compute_mean_z"""
print("Computing mean levels")
print("loading files:")
print(glob.glob(file_search))
z = mygis.read_files(file_search, "z", axis=0)
mygis.write("annual_mean_z.nc", z.mean(axis=0), varname="z")
for month in range(12):
print("Month " + str(month + 1))
curz = z[0] * 0
nyears = 0
month_start = start_day_per_month[month]
start_point = month_start * times_per_day
month_end = start_day_per_month[month + 1]
end_point = month_end * times_per_day
while start_point < z.shape[0]:
curz += z[start_point:end_point].mean(axis=0)
nyears += 1
start_point += times_per_day * days_per_year
end_point += times_per_day * days_per_year
print(nyears)
curz /= nyears
mygis.write(zoutputfile.format(month + 1), curz, varname="z")
def compute_mean_q(wind_option):
print("Computing Monthly Mean fields")
qvarlist = ["qv", "theta", "p", "u", "v", "rh"]
if (wind_option == "nowind"):
qvarlist = ["qv", "theta", "p", "rh"]
full_data = []
for varname in qvarlist:
print(varname)
data = mygis.read_files(varname + "_cesm_*_*.nc", varname, axis=0)
for month in range(12):
print("Month " + str(month + 1))
meanq = np.zeros(data.shape[1:])
nyears = 0
month_start = start_day_per_month[month]
start_point = month_start * times_per_day
month_end = start_day_per_month[month + 1]
end_point = month_end * times_per_day
while start_point < data.shape[0]:
print(start_point, end_point)
meanq += data[start_point:end_point, ...].mean(axis=0)
nyears += 1
start_point += times_per_day * days_per_year
end_point += times_per_day * days_per_year
if nyears > 0:
meanq /= nyears
print(nyears)
mygis.write(qoutputfile.format(month + 1, varname),
meanq,
varname=varname)
def main (filesearch, varname, scale, current, future, nyears, geofile):
# TODO: Do something more interesting here...
if verbose:
print("Verbose output turned on")
print("Arguments: ")
print(filesearch, varname, scale, current, future, nyears, geofile)
try:
if verbose: print("Reading time data")
times=mygis.read_files(filesearch, "time",axis=0)
except:
pass
if verbose: print("Reading primary data")
data=mygis.read_files(filesearch,varname,axis=0)
data[data<0]=0
if verbose: print("Reading geographic data")
geo=mygis.read_geo("domain.nc")
current_year = int(current[:4])
future_year = int(future[:4])
if verbose: print("Computing current and future means")
current = scale * data[(current_year-base_year)*365:(current_year-base_year+nyears)*365].mean(axis=0) #*0.9
future = scale * data[( future_year-base_year)*365:( future_year-base_year+nyears)*365].mean(axis=0)
if verbose: print("Reading map geographic data")
mapgeo=mygis.read_geo(geofile)
geobounds=[mapgeo.lat.min(), mapgeo.lat.max(), mapgeo.lon.min(), mapgeo.lon.max()]
if verbose:
print(future.min(), future.mean(), future.max())
print(current.min(),current.mean(),current.max())
delta=future-current
print(delta.min(),delta.mean(),delta.max())
if verbose: print("Plotting map")
map_vis.vis((future - current), geo=geobounds, lat=geo.lat, lon=geo.lon, reproject=True,
cmap=plt.cm.seismic_r, clim=(-300,300), latstep=2, lonstep=5)
plt.savefig("Change_Map.png")
def vinterp_q(file_name=None):
"""interpolate all variables to a common z level"""
print("Interpolating quantities")
if file_name!=None:
file_search=file_name
qvarlist=["qv","theta","p","u","v"]
qdata=Bunch()
print("Loading data")
t0=time.time()
for varname in qvarlist:
print(varname)
qdata[varname]=mygis.read_files(file_search,varname,axis=0)
print("z")
qdata["z"]=mygis.read_files(file_search,"z",axis=0)
mean_z=mygis.read_nc("annual_mean_z.nc","z").data
print(" Reading data took {0:6.2f} seconds".format((time.time()-t0)))
t0=time.time()
temperature=qdata.theta*units.exner(qdata.p)
print(" Temperature conversion took {0:6.2f} seconds".format((time.time()-t0)))
t0=time.time()
print("Computing RH")
qdata["rh"]=compute_rh(qdata,temperature)
qvarlist.append("rh")
print(" RH calculation took {0:6.2f} seconds".format((time.time()-t0)))
print("Interpolating")
qout=Bunch()
for varname in qvarlist:
qout[varname]=np.zeros(qdata[varname].shape)
t0=time.time()
nsteps=qdata.z.shape[0]
for i in range(nsteps):
vertical_interp.interp_multivar(qdata,qout,i, qdata.z[i], mean_z,vartype=varname,
inputt=temperature[i],inputp=qdata.p[i])
print(" Interpolation took {0:6.2f} seconds per timestep".format((time.time()-t0)/nsteps))
print("Writing data")
write_interpolated_6hrly(qout,mean_z,file_name)
def update_files_for_year(year,res,variable,model,mask):
files=glob.glob(search_dir+model+"/"+variable+"/BCSAR*"+res+"*"+str(year)+"_*")
files.sort()
time_info.data=time_gen(year,model)
data=np.concatenate(myio.read_files(files,variable))
for i in range(data.shape[0]):
data[i,...][mask]=FILL_VALUE
info=data_info[variable]
newfilename=files[0].replace(".nc","")[:-3]+".nc"
print(newfilename)
extra_vars=[lat_info,lon_info,time_info]
myio.write(newfilename,data,varname=info.name,dtype=info.dtype,dims=info.dims,
attributes=info.attributes,extravars=extra_vars)
def load_data(filesearch="*pr.19*.nc",variable="pr",n=None):
files=glob.glob(filesearch)
files.sort()
geosubset=[34,43.9,360-115,360-100]
geosubset=[25,52.7,360-124.7,360-67] #CONUS
subset=read_geosubset(files[0],geosubset)
# print(files[0],subset)
# load in ALL the data (note this is likely to be 4-17GB)
# on data vis clusters (128GB RAM) this is fine,
# on personal machines (4-8GB RAM) it's tricky or a a non-starter
d=[]
print("Loading "+str(len(files))+" files.")
d=mygis.read_files(filesearch,variable,axis=0)
d=d[:,subset[0]:subset[1],subset[2]:subset[3]]
return d
def rain_stats(dir_name, fulldomain=False):
if verbose:
files=glob.glob(dir_name+rain_file_search)
files.sort()
print(files[0]+" "+files[-1])
data=mygis.read_files(dir_name+rain_file_search,"precipitation_amount",axis=0, verbose=verbose)
if not fulldomain:
data=data[:,1549:800:-1,:600]
if verbose: print("Computing wet day fraction")
mygis.write(dir_name+"annual_wetfrac.nc",stats.wetfrac(data))
if verbose: print("99th percentile")
mygis.write(dir_name+"annual_p99.nc",stats.percentile(data, percentile=0.99))
if verbose: print("mean annual precip")
mygis.write(dir_name+"annual_mean.nc",stats.mean(data)*days_per_year)
ndays=data.shape[0]
dates=compute_dates(ndays)
curmonth=np.empty(ndays,dtype=bool)
for month in range(12):
for i in range(ndays):
curmonth[i] = (dates[i].month == (month+1))
if verbose: print("month{:02}".format(month+1))
if verbose: print(" wet day fraction")
mygis.write(dir_name+"month{:02}_wetfrac.nc".format(month+1),stats.wetfrac(data[curmonth,:,:]))
if verbose: print(" 99th percentile")
mygis.write(dir_name+"month{:02}_p99.nc".format(month+1),stats.percentile(data[curmonth,:,:], percentile=0.99))
if verbose: print(" mean precip")
mygis.write(dir_name+"month{:02}_mean_precip.nc".format(month+1),stats.mean(data[curmonth,:,:])*(month_end[month]-month_start[month]))
return data
def tmin_stats(dir_name):
if verbose:
files=glob.glob(dir_name+tmin_file_search)
files.sort()
print(files[0]+" "+files[-1])
data=mygis.read_files(dir_name+tmin_file_search,"daily_minimum_temperature",axis=0, verbose=verbose)
if verbose: print("mean annual tmin")
mygis.write(dir_name+"annual_mean_tmin.nc",stats.mean(data))
ndays=data.shape[0]
dates=compute_dates(ndays)
curmonth=np.empty(ndays,dtype=bool)
for month in range(12):
for i in range(ndays):
curmonth[i] = (dates[i].month == (month+1))
if verbose: print("month{:02}".format(month+1))
if verbose: print(" mean")
mygis.write(dir_name+"month{:02}_mean_tmin.nc".format(month+1),stats.mean(data[curmonth,:,:]))
return data
def plot_runoff_delta(varname="runoff",extra_var=None,aggregation=np.sum,monthly_total=True,multiplier=86400.0,method="",calendar="noleap"):
"""docstring for plot_runoff_delta"""
print("Loading data :"+varname)
# read in runoff from all of the netcdf files in the current directory and concatenate them along axis 0 (time)
if varname=="tasmax":extra="tmax"
if varname=="tasmin":extra="tmin"
files=glob.glob(method+"_*"+extra+"*[1-2]*.nc")
files.sort()
first_year=int(files[0].split(".")[-2][-4:]) # assumes file name ends in yyyy.nc
print(first_year)
data = mygis.read_files(method+"_*"+extra+"*[1-2]*.nc",varname,axis=0)*multiplier # mm/s * 86400 s/day = mm/day)
if (extra_var!=None):
data += mygis.read_files(method+"_[1-2]*.nc",extra_var,axis=0)*multiplier # mm/s * 86400 s/day = mm/day)
data=data[:,60:150,80:200]
# create a list of datetime objects to match the netcdf data
if calendar=="noleap":
# to fake a 365 day calendar
times=[dt.datetime(first_year,10,1,0,0)+dt.timedelta(np.floor(i/365.0*365.25)) for i in range(data.shape[0])]
else:
times=[dt.datetime(first_year,1,1,0,0)+dt.timedelta(i) for i in range(data.shape[0])]
# summary data (12 months of no data to begin with)
n_cur = [0]*12
curdata=[None]*12
n_fut = [0]*12
futdata=[None]*12
print("Computing Monthly Means")
for month in range(1,13):
print(" Month:{0:02}".format(month),end="\r")
sys.stdout.flush()
for i,t in enumerate(times):
# if this time step is the month we are analyzing:
if t.month==month:
# if this time step is in the current time period:
if (t>=cur_start_time) and (t<cur_end_time):
if (curdata[month-1]==None):
curdata[month-1] = data[i,:,:]
n_cur[month-1] = 1
else:
curdata[month-1] += data[i,:,:]
n_cur[month-1] += 1
# if this time step is in the future time period:
if (t>=fut_start_time) and (t<fut_end_time):
if (futdata[month-1]==None):
futdata[month-1] = data[i,:,:]
n_fut[month-1] = 1
else:
futdata[month-1] += data[i,:,:]
n_fut[month-1] += 1
print("Plotting ")
monthly_deltas=np.zeros(12)
for i in range(12):
print(" Month:{0:02} n={1},{2} ".format(i+1,n_cur[i],n_fut[i]),end="\r")
sys.stdout.flush()
# divide by n to get daily mean and multiply by days/month to get monthly totals
curdata[i] /= n_cur[i]
futdata[i] /= n_fut[i]
if monthly_total:
curdata[i]*=month_lengths[i]
futdata[i]*=month_lengths[i]
# make a map of the current monthly changes
plt.clf()
if varname[0].lower()=="t":
cmap=plt.cm.seismic
max_range=5
else:
cmap=plt.cm.seismic_r
max_range=np.max(np.abs(futdata[i] - curdata[i]))
plt.imshow(futdata[i] - curdata[i],cmap=cmap)
plt.clim(-max_range,max_range)
plt.colorbar()
plt.title(varname+"_"+method+" delta month{0:02}".format(i+1))
plt.savefig(output_dir+varname+"_"+method+"_delta_month{0:02}.png".format(i+1))
# compute the sum of all runoff changes in the basin (the mask isn't necessary, was intended to mask out points outside the basin)
# mask=np.where(futdata[i]!=0)
monthly_deltas[i]=aggregation((futdata[i] - curdata[i])) #[mask])
# make a plot of the basin total changes
plt.clf()
plt.plot(monthly_deltas)
# print(monthly_deltas)
plt.plot([0,12],[0,0],color="black")
plt.xticks(range(12),xlabels)
plt.savefig(output_dir+varname+"_"+method+"_monthly_change.png")
print("Finished ")
print("-"*20)