def load_data(filename, varname):
"""loads a variable:varname from the netcdf file:filename
if varname is actually a number, then just return that number in the data structure
"""
if verbose: print("Loading:" + str(varname))
if isinstance(varname, numbers.Number):
if verbose: print(" " + str(varname) + " is a number!")
return Bunch(name="n" + str(varname),
data=varname,
attributes="a number",
dims=(),
dtype='f')
# load the data as a netcdf4 data structure first
d = mygis.read_nc(filename, varname, returnNCvar=True)
# copy the dimensions of the variable
dims = d.data.dimensions
# read the attributes of the variable
atts = mygis.read_atts(filename, varname)
if verbose: print(" " + str(dims))
if verbose: print(" " + str(atts))
# now read the actual data
data = d.data[:]
d.ncfile.close()
return Bunch(name=varname,
data=data,
dims=dims,
attributes=atts,
dtype='f')
def load_data(filename, varname):
"""loads a variable:varname from the netcdf file:filename
if varname is actually a number, then just return that number in the data structure
"""
if verbose:print("Loading:"+str(varname))
if isinstance(varname, numbers.Number):
if verbose:print(" "+str(varname)+" is a number!")
return Bunch(name="n"+str(varname), data=varname, attributes="a number", dims=(), dtype='f')
# load the data as a netcdf4 data structure first
d=mygis.read_nc(filename,varname,returnNCvar=True)
# copy the dimensions of the variable
dims=d.data.dimensions
# read the attributes of the variable
atts=mygis.read_atts(filename,varname)
if verbose:print(" "+str(dims))
if verbose:print(" "+str(atts))
# now read the actual data
data=d.data[:]
d.ncfile.close()
return Bunch(name=varname,data=data,dims=dims,attributes=atts, dtype='f')
def load_tskin(filename,tsvarname, maskvarname):
"""load skin temp data from wrf output file: filename
tskin uses it's own procedure so that we can average over the past 10 days for land points
ideally it will create a higher resolution output based on a high-resolution land-sea mask
"""
if verbose:print("Loading :"+maskvarname)
ncmask=mygis.read_nc(filename,maskvarname,returnNCvar=True)
mask=ncmask.data[0]
ncmask.ncfile.close()
land=np.where(mask==1)
if verbose:print("Loading :"+tsvarname)
ncts=mygis.read_nc(filename,tsvarname,returnNCvar=True)
atts=mygis.read_atts(filename,tsvarname)
dims=ncts.data.dimensions
if verbose:print(" "+str(dims))
if verbose:print(" "+str(atts))
data=ncts.data[:]
ncts.ncfile.close()
for i in range(data.shape[0]-1,0,-1):
start=max(i-(10*steps_per_day),0)
data[i][land]=data[start:i+1].mean(axis=0)[land]
return Bunch(name=tsvarname, data=data, attributes=atts, dims=dims, dtype='f')
def load_tskin(filename, tsvarname, maskvarname):
"""load skin temp data from wrf output file: filename
tskin uses it's own procedure so that we can average over the past 10 days for land points
ideally it will create a higher resolution output based on a high-resolution land-sea mask
"""
if verbose: print("Loading :" + maskvarname)
ncmask = mygis.read_nc(filename, maskvarname, returnNCvar=True)
mask = ncmask.data[0]
ncmask.ncfile.close()
land = np.where(mask == 1)
if verbose: print("Loading :" + tsvarname)
ncts = mygis.read_nc(filename, tsvarname, returnNCvar=True)
atts = mygis.read_atts(filename, tsvarname)
dims = ncts.data.dimensions
if verbose: print(" " + str(dims))
if verbose: print(" " + str(atts))
data = ncts.data[:]
ncts.ncfile.close()
for i in range(data.shape[0] - 1, 0, -1):
start = max(i - (10 * steps_per_day), 0)
data[i][land] = data[start:i + 1].mean(axis=0)[land]
return Bunch(name=tsvarname,
data=data,
attributes=atts,
dims=dims,
dtype='f')
def read_times(filename):
atts = mygis.read_atts(filename,"time")
calendar = atts.calendar
start_date = atts.units.split(" since ")[1]
times = mygis.read_nc(filename,"time").data
return parse_times(times, start_date, calendar)
def load_wrf_data(filename):
"""Load required data form the WRF output file : filename"""
base_data=load_vars(filename,wrfvars)
skin_t=load_tskin(filename,tsvar,landmask)
base_data.append(skin_t)
atts=mygis.read_atts(filename,global_atts=True)
return Bunch(data=base_data,global_atts=atts)
def load_wrf_data(filename):
"""Load required data form the WRF output file : filename"""
base_data = load_vars(filename, wrfvars)
skin_t = load_tskin(filename, tsvar, landmask)
base_data.append(skin_t)
atts = mygis.read_atts(filename, global_atts=True)
return Bunch(data=base_data, global_atts=atts)
def main(model, scenario, varname):
files=glob.glob(model+"/"+scenario+"/day/atmos/day/r1i1p1/latest/"+varname+"/*.nc")
files.sort()
atts = mygis.read_atts(files[0],varname)
gatts = mygis.read_atts(files[0],global_atts=True)
geo = mygis.read_geo(files[0])
lat = Bunch(data=geo.lat,name="lat",dtype='f',attributes=geo.latatts,dims=('lat','lon'))
lon = Bunch(data=geo.lon,name="lon",dtype='f',attributes=geo.lonatts,dims=('lat','lon'))
datadims=('time','lat','lon')
start_year = start_years[scenario]
end_year = end_years[scenario]
print("Getting time data")
years, months = read_times(files[0])
print(files[0].split("/")[-1])
print(years[0],years[-15:])
print(months[0],months[-15:])
print("Reading {} data".format(varname))
data = mygis.read_nc(files[0],varname,returnNCvar=True)
output = np.zeros((13,data.data.shape[1],data.data.shape[2]))
data.ncfile.close()
n = np.zeros(13)
for f in files:
print(" "+f)
years, months = read_times(f)
data = mygis.read_nc(f,varname,returnNCvar=True)
for i in range(data.data.shape[0]):
if (years[i]>start_year) & (years[i]<=end_year):
output[months[i]-1] += data.data[i]
n[months[i]-1]+=1
data.ncfile.close()
output[-1]=output[:-1].sum(axis=0)
n[-1] = n[:-1].sum(axis=0)
if varname=="pr":
n[-1]/=365.0
n[:-1] /= month_lengths[1:]
n/=86400.0
print("writing")
results = output/n[:,np.newaxis,np.newaxis]
mygis.write(outputfile.format(model, scenario,varname),results, dims=datadims, extravars=[lat,lon],
varname=varname, attributes=atts, global_attributes=gatts)
def main(f1=None, f2=None, zvar=None, zbasevar=None,skip_z0=False,
zrefvar=None, zrefbasevar=None, timevar=None,
output_dir="./", inputvar=None, outputvar=None,
regrid_vertical=True, regrid_horizontal=True):
"""docstring for main"""
if f1!=None:
inputfile=f1
else:
inputfile="tos_day_ACCESS1-3_historical_r1i1p1_19500101-19591231.nc"
if f2!=None:
reference=f2
else:
reference="../../completed/historical/access/subset/ta_6hrLev_ACCESS1-3_historical_r1i1p1_1950010106-1951010100.nc"
# find all files if more than one was specified.
files=glob.glob(inputfile)
files.sort()
if regrid_vertical and regrid_horizontal:
if zvar!=None:
geographic_vars.append(zvar)
if zbasevar!=None:
geographic_vars.append(zbasevar)
if timevar!=None:
geographic_vars.append(timevar)
print("Reading geo data")
geo1 = mygis.read_geo(files[0])
geo2 = mygis.read_geo(reference)
while len(geo1.lat.shape)>2:
geo1.lat=geo1.lat[0]
geo1.lon=geo1.lon[0]
while len(geo2.lat.shape)>2:
geo2.lat=geo2.lat[0]
geo2.lon=geo2.lon[0]
if geo1.latatts is None:
latatts=Bunch(standard_name="latitude",long_name="latitude coordinate",units="degrees_north")
else:
latatts=geo1.latatts
if geo1.lonatts is None:
lonatts=Bunch(standard_name="longitude",long_name="longitude coordinate",units="degrees_east")
else:
lonatts=geo1.lonatts
extra_vars=[Bunch(data=geo2.lat,name="lat",dims=("lat","lon"),dtype="f",attributes=latatts),
Bunch(data=geo2.lon,name="lon",dims=("lat","lon"),dtype="f",attributes=lonatts)]
if timevar:
timeatts=Bunch(standard_name="time")
dummy_time_var=Bunch(data=None,name="time",dims=("time",),dtype="d",attributes=timeatts)
extra_vars.append(dummy_time_var)
else:
timeatts=None
missing_value=0
# find the variable names that we need to regrid
# assume we are regridding everything that is not a "geographic" variable (e.g. lat, lon, time)
if inputvar is None:
fdata = mygis.Dataset(files[0])
varlist = []
for v in fdata.variables:
if (not (str(v) in geographic_vars)):
varlist.append(str(v))
fdata.close()
else:
varlist=[inputvar]
if outputvar is None:
outputvar=varlist
print(varlist)
print(latatts)
print(lonatts)
if timevar:print(timeatts)
# ncdata=mygis.read_nc(files[0],inputvar,returnNCvar=True)
if regrid_horizontal:
print(geo1.lat.shape, geo2.lat.shape)
print(geo1.lon.shape, geo2.lon.shape)
geoLUT = load_geoLUT(lat1=geo1.lat,lon1=geo1.lon,lat2=geo2.lat,lon2=geo2.lon, winhalfsize=5)#,
# mask=(ncdata.data[0,...]==missing_value),winhalfsize=7)
print("geolut shape", geoLUT.shape)
# ncdata.ncfile.close()
if zrefvar:
zout=mygis.read_nc(reference,zrefvar).data
if zrefbasevar!=None:
zout+=mygis.read_nc(reference,zrefbasevar).data
for f in files:
regridded_z=None
if zvar:
zin=mygis.read_nc(f,zvar).data
if zbasevar!=None:
zin+=mygis.read_nc(f,zbasevar).data
# if zvar=="GHT":
# zin/=9.81
if skip_z0:
zin=zin[:,1:]
# extra_vars = copy.deepcopy(master_vars)
global_atts= mygis.read_atts(f,global_atts=True)
if timevar:
time_atts=mygis.read_atts(f,varname=timevar)
extra_vars[2].data=mygis.read_nc(f,timevar).data
extra_vars[2].attributes=time_atts
for i,v in enumerate(varlist):
print("Reading data: "+f+" for variable:"+v)
data=mygis.read_nc(f,v).data
# the first model layer in erai is on the surface, but the second layer is at 1000mb (often below the surface)
# this lets you skip the surface layer and just interpolate over the pressure levels.
if skip_z0:
if len(data.shape)==4:
data=data[:,1:]
data_atts=mygis.read_atts(f,varname=v)
if len(data.shape)==4:
dims=("time","lev","lat","lon")
elif len(data.shape)==3:
dims=("time","lat","lon")
elif len(data.shape)==2:
dims=("lat","lon")
else:
print("Error with variable "+v)
print("can't geointerpolate 1D variable, skipping")
dims=mygis.read_dims(f,varname=v)
print(mygis.read_dims(f,varname=v))
print(dims)
print("Regridding")
if (regrid_horizontal and (len(data.shape)>1)):
print(data.shape)
output=regrid(data,geoLUT=geoLUT)
print(output.shape)
else:
output=data
if zvar:
if regrid_vertical:
if regridded_z is None:
if regrid_horizontal:
regridded_z = regrid(zin,geoLUT=geoLUT)
else:
regridded_z = zin
if len(output.shape)==4:
nz = zout.shape[1]
for t in range(output.shape[0]):
output[t,:nz] = vinterp(output[t],inputz=regridded_z[t],outputz=zout[t*4+2])
output=output[:,:nz,:,:]
if v!=varlist[-1]:
extra_vars.append(Bunch(data=output,name=v,dims=dims,dtype="f",attributes=data_atts))
else:
print("Writing data")
outputfile=f.split("/")[-1]
outputfile=output_dir+"regrid_"+outputfile#.replace(v,outputvar[i])
print(outputvar[i], dims, data_atts)
for e in extra_vars:
print(e.name)
if not (e.data is None):
print(e.dims, e.data.shape)
if outputfile[2:]!=inputfile:
mygis.write(outputfile,output,dims=dims,varname=outputvar[i],attributes=data_atts,
global_attributes=global_atts,extravars=extra_vars)#, format="NETCDF4")
else:
print("Error, outputfile==inputfile")