def read_wrf(filename,var="pr"):
"""read a variable from a net cdf file"""
f=nio.open_file(filename,mode="r")
# f=netCDF4.Dataset(filename,mode="r")
data=f.variables[var][:]
f.close()
return data
def sonic(filename, L5_fix=True):
'''
Read in RMY Sonic Anemometer Data from the Lake Thunderbird Micronet year >=2007
--includes fix for level 5 alignment as default (controlled with keyword
L5_fix (L5_fix=True default)
'''
from metobs.generic import horizontal_align_fix as haf
#
# Determine file type
#
if ((os.path.splitext(os.path.basename(filename))[1] == '.dat') |
(os.path.splitext(os.path.basename(filename))[1] == '.txt') |
(os.path.splitext(os.path.basename(filename))[1] == '.gz')):
#
# Open File
#
try:
date = np.loadtxt(filename,
delimiter=',',
skiprows=2,
usecols=np.arange(0, 1),
dtype=datetime.datetime,
converters={0: pull_date})
u1,v1,w1,T1,u2,v2,w2,T2,u3,v3,w3,T3,u4,v4,w4,T4,u5,v5,w5,T5= \
np.loadtxt(filename, usecols=np.arange(2,22), comments='%', delimiter=',',skiprows=2,unpack=True)
u5, v5 = haf(u5, v5, -15.3)
u = np.r_[u1, u2, u3, u4, u5]
v = np.r_[v1, v2, v3, v4, v5]
w = np.r_[w1, w2, w3, w4, w5]
T = np.r_[T1, T2, T3, T4, T5]
dt = np.dtype([('u', np.float), ('v', np.float), ('w', np.float),
('T', np.float)])
data = np.array(zip(u, v, w, T), dtype=dt).reshape(5, -1)
except IOError:
print '%s does not exist\n' % filename
raise
return date, data
elif os.path.splitext(os.path.basename(filename))[1] == '.nc':
import nio
f = nio.open_file(filename, mode='r')
u = f.variables['u'][:]
v = f.variables['v'][:]
w = f.variables['w'][:]
T = f.variables['T'][:]
f.close()
data = np.rec.fromarrays([u, v, w, T], names='u,v,w,T')
return data
def sonic_2005(filename):
'''
Read in RMY Sonic Anemometer Data from the Lake Thunderbird Micronet Spring 2005
'''
#
# Determine file type
#
if os.path.splitext(os.path.basename(filename))[1] == '.txt':
#
# Open File
#
try:
date = np.loadtxt(filename,
delimiter=',',
skiprows=2,
usecols=np.arange(0, 1),
dtype=datetime.datetime,
converters={0: pull_date})
u1,u2,u3,u4,u5,v1,v2,v3,v4,v5,w1,w2,w3,w4,w5,T1,T2,T3,T4,T5,voltage= \
np.loadtxt(filename, usecols=np.arange(1,22), comments='%', delimiter=',',unpack=True)
u = np.r_[u1, u2, u3, u4, u5]
v = np.r_[v1, v2, v3, v4, v5]
w = np.r_[w1, w2, w3, w4, w5]
T = np.r_[T1, T2, T3, T4, T5]
dt = np.dtype([('u', np.float), ('v', np.float), ('w', np.float),
('T', np.float)])
data = np.array(zip(u, v, w, T), dtype=dt).reshape(5, -1)
dt = np.dtype([('date', object), ('voltage', np.float)])
ext = np.array(zip(date, voltage), dtype=dt)
except IOError:
print '%s does not exist\n' % filename
raise
return data, ext
elif os.path.splitext(os.path.basename(filename))[1] == '.nc':
import nio
f = nio.open_file(filename, mode='r')
u = f.variables['u'][:]
v = f.variables['v'][:]
w = f.variables['w'][:]
T = f.variables['T'][:]
f.close()
data = np.rec.fromarrays([u, v, w, T], names='u,v,w,T')
return data
def sonic(filename, L5_fix=True):
'''
Read in RMY Sonic Anemometer Data from the Lake Thunderbird Micronet year >=2007
--includes fix for level 5 alignment as default (controlled with keyword
L5_fix (L5_fix=True default)
'''
from metobs.generic import horizontal_align_fix as haf
#
# Determine file type
#
if ((os.path.splitext(os.path.basename(filename))[1]=='.dat')|(os.path.splitext(os.path.basename(filename))[1]=='.txt')|(os.path.splitext(os.path.basename(filename))[1]=='.gz')):
#
# Open File
#
try:
date = np.loadtxt(filename,delimiter=',',skiprows=2,usecols=np.arange(0,1),dtype=datetime.datetime,converters={0:pull_date})
u1,v1,w1,T1,u2,v2,w2,T2,u3,v3,w3,T3,u4,v4,w4,T4,u5,v5,w5,T5= \
np.loadtxt(filename, usecols=np.arange(2,22), comments='%', delimiter=',',skiprows=2,unpack=True)
u5,v5=haf(u5,v5,-15.3)
u=np.r_[u1,u2,u3,u4,u5]
v=np.r_[v1,v2,v3,v4,v5]
w=np.r_[w1,w2,w3,w4,w5]
T=np.r_[T1,T2,T3,T4,T5]
dt = np.dtype([('u',np.float),('v',np.float),('w',np.float),
('T',np.float)])
data=np.array(zip(u,v,w,T), dtype=dt).reshape(5,-1)
except IOError:
print '%s does not exist\n'%filename
raise
return date,data
elif os.path.splitext(os.path.basename(filename))[1]=='.nc':
import nio
f=nio.open_file(filename,mode='r')
u=f.variables['u'][:]
v=f.variables['v'][:]
w=f.variables['w'][:]
T=f.variables['T'][:]
f.close()
data = np.rec.fromarrays([u,v,w,T],names='u,v,w,T')
return data
def sonic_2005(filename):
'''
Read in RMY Sonic Anemometer Data from the Lake Thunderbird Micronet Spring 2005
'''
#
# Determine file type
#
if os.path.splitext(os.path.basename(filename))[1]=='.txt':
#
# Open File
#
try:
date = np.loadtxt(filename,delimiter=',',skiprows=2,usecols=np.arange(0,1),dtype=datetime.datetime,converters={0:pull_date})
u1,u2,u3,u4,u5,v1,v2,v3,v4,v5,w1,w2,w3,w4,w5,T1,T2,T3,T4,T5,voltage= \
np.loadtxt(filename, usecols=np.arange(1,22), comments='%', delimiter=',',unpack=True)
u=np.r_[u1,u2,u3,u4,u5]
v=np.r_[v1,v2,v3,v4,v5]
w=np.r_[w1,w2,w3,w4,w5]
T=np.r_[T1,T2,T3,T4,T5]
dt = np.dtype([('u',np.float),('v',np.float),('w',np.float),
('T',np.float)])
data=np.array(zip(u,v,w,T), dtype=dt).reshape(5,-1)
dt = np.dtype([('date',object),('voltage',np.float)])
ext = np.array(zip(date,voltage), dtype=dt)
except IOError:
print '%s does not exist\n'%filename
raise
return data,ext
elif os.path.splitext(os.path.basename(filename))[1]=='.nc':
import nio
f=nio.open_file(filename,mode='r')
u=f.variables['u'][:]
v=f.variables['v'][:]
w=f.variables['w'][:]
T=f.variables['T'][:]
f.close()
data = np.rec.fromarrays([u,v,w,T],names='u,v,w,T')
return data
def read_base(filename):
"""Read all variable names attibutes dimensions..."""
f=nio.open_file(filename,mode="r")
# f=netCDF4.Dataset(filename,mode="r")
output=Bunch()
variables=Bunch()
for k in f.variables.keys():
curvar=f.variables[k]
data=Bunch()
if k=="Times":
data.data=curvar[:]
data.atts=copy.deepcopy(curvar.__dict__)
data.dims=copy.deepcopy(curvar.dimensions)
data.dtype=curvar.typecode()
variables[k]=data
output.variables=variables
output.global_attributes=copy.deepcopy(f.__dict__)
output.dimensions=copy.deepcopy(f.dimensions)
f.close()
return output
if os.path.isfile(fn_monthOut)==False:
fileNames.append(fn)
print fn_monthOut
# process the remaining input files.
startTime = datetime.now()
for fn in fileNames:
try:
print 'Starting averaging of', fn
pathSplit=fn.split('/')
fnSplit=pathSplit[-1].split('.')
# open files, get aice_d, and time and put into variables with the same name
if fnSplit[2]==bgKey: # if the control run
fnTemp=str.join(".", fnSplit[:-1])+'.timeseries.nc'
f=nio.open_file(fn, 'r')
aice_d=f.variables['aice_d'][:,:,:]
time=f.variables['time'][:]
if fnSplit[2]==runParts23keyRCP45:
# now instead you need to open THREE FILES:
# first open the first one
f=nio.open_file(fn, 'r')
aice_d=f.variables['aice_d'][:,:,:]
time=f.variables['time'][:]
# now find the other two and add them to the end
pathConstruct2=pathIn45+'*'+runParts23keyRCP45+'*'+fnSplit[4]+'*'+fnSplit[7]+'*nc'
findFiles=np.sort(glob.glob(pathConstruct2))
def sonic(filename):
'''
Create .nc files from RMY Sonic Anemometer ASCII Data from the Lake Thunderbird Micronet
'''
#
# Open ASCII File
#
try:
date = N.loadtxt(filename, usecols=[0], dtype=str, comments='%', delimiter=',')
u1,u2,u3,u4,u5,v1,v2,v3,v4,v5,w1,w2,w3,w4,w5,T1,T2,T3,T4,T5,voltage= \
N.loadtxt(filename, usecols=N.arange(1,22), comments='%', delimiter=',',unpack=True)
u=N.r_[u1,u2,u3,u4,u5]
v=N.r_[v1,v2,v3,v4,v5]
w=N.r_[w1,w2,w3,w4,w5]
T=N.r_[T1,T2,T3,T4,T5]
dt = N.dtype([('u',N.float),('v',N.float),('w',N.float),
('T',N.float)])
data=N.array(zip(u,v,w,T), dtype=dt).reshape(5,-1)
dt = N.dtype([('date',object),('voltage',N.float)])
ext = N.array(zip(date,voltage), dtype=dt)
netcdf_name = os.path.splitext(os.path.basename(filename))[0]+'.nc'
netcdf_loc = '/micronet/python/data/ltmsonic/netcdf'
netcdf_fpath = os.path.join(netcdf_loc,netcdf_name)
#
# Open netCDF file and let the fun begin!
#
f = nio.open_file(netcdf_fpath, mode='c')
#
# Set dimensions
#
f.create_dimension('station',5)
f.create_dimension('record_length',18000)
#
# Create Variables
#
# sdi_var=f.create_variable('station_id','i',('station',))
# lat_var=f.create_variable('latitude','f',('station',))
# long_var=f.create_variable('longitude','f',('station',))
# alt_var=f.create_variable('altitude','f',('station',))
# time_var=f.create_variable('time','i',('station','record_length',))
u_var=f.create_variable('u','f',('station','record_length'))
v_var=f.create_variable('v','f',('station','record_length'))
w_var=f.create_variable('w','f',('station','record_length'))
T_var=f.create_variable('T','f',('station','record_length'))
# bat_v_var=f.create_variable('bat_v','f',('record_length',))
#
# Assign attributes to variables
#
# f.variables['station_id'].long_name = 'tower level'
# f.variables['latitude'].units = 'degrees_north'
# f.variables['latitude'].long_name = 'latitude degrees_north'
# f.variables['longitude'].units = 'degrees_east'
# f.variables['longitude'].long_name = 'longitude degrees_east'
# f.variables['altitude'].units = 'meters'
# f.variables['altitude'].positive = 'up'
# f.variables['altitude'].long_name = 'height above ground level'
# f.variables['time'].units = 'seconds since 1970-01-01 00:00:00.000 UTC'
# f.variables['time'].long_nameunits = 'seconds since 1970-01-01 00:00:00.000 UTC'
f.variables['u'].units = 'm/s'
f.variables['u'].long_name = 'East-west component of velocity'
# f.variables['u'].missing_value = '-999'
f.variables['v'].units = 'm/s'
f.variables['v'].long_name = 'North-south component of velocity'
# f.variables['v'].missing_value = '-999'
f.variables['w'].units = 'm/s'
f.variables['w'].long_name = 'vertical component of velocity (parallel to gravity)'
# f.variables['w'].missing_value = '-999'
f.variables['T'].units = 'Kelvin'
f.variables['T'].long_name = 'Temperature in Kelvin'
# f.variables['T'].missing_value = '-999'
#
# Assign values to variables
#
# pnt=post_num
# latt=lat_array
# lont=lon_array
# altt=N.array([1.5,3.0,6.0,10.0,15.0])
# outtimet=outtime_array
# for i in range(0,287):
# post_num=N.c_[post_num,pnt]
# lat_array=N.c_[lat_array,latt]
# lon_array=N.c_[lon_array,lont]
# alt_array=N.c_[alt_array,altt]
# outtime_array=ext['date'][0]
# for i in range(1,5):
# outtime_array=N.c_[outtime_array,ext['date'][i]]
# sdi_var.assign_value(N.array([1,2,3,4,5]))
# lat_var.assign_value(lat_array)
# long_var.assign_value(lon_array)
# alt_var.assign_value(alt_array)
# time_var.assign_value(outtime_array)
u_var.assign_value(data['u'].astype('float32'))
v_var.assign_value(data['v'].astype('float32'))
w_var.assign_value(data['w'].astype('float32'))
T_var.assign_value(data['T'].astype('float32'))
# bat_v_var.assign_value(ext['voltage'])
#
# Set global attributes
#
setattr(f,'title','Lake Thunderbird Micronet sonic anemometer time series')
# setattr(f,'Conventions','Unidata Observation Dataset v1.0')
# setattr(f,'description','Lake Thunderbird Micronet Temperature/RH Daily Time Series')
# setattr(f,'time_coordinate','time')
# setattr(f,'cdm_datatype','Station')
# setattr(f,'stationDimension','station')
# setattr(f,'station_id_variable','station_id')
# setattr(f,'latitude_coortinate','latitude')
# setattr(f,'longitude_coortinate','longitude')
# setattr(f,'altitude_coortinate','altitude')
# setattr(f,'geospatial_lat_max',N.array2string(lat_array.max()))
# setattr(f,'geospatial_lat_min',N.array2string(lat_array.min()))
# setattr(f,'geospatial_lon_max',N.array2string(lon_array.max()))
# setattr(f,'geospatial_lon_min',N.array2string(lon_array.min()))
# setattr(f,'time_coverage_start',str(int(outtime[0]))+" seconds since 1970-01-01 00:00:00.000 UTC")
# setattr(f,'time_coverage_end',str(int(outtime[-1]))+" seconds since 1970-01-01 00:00:00.000 UTC")
# setattr(f,'observationDimension','record_length')
#
# Close netCDF file
#
f.close()
except IOError:
print '%s does not exist\n'%filename
raise
#totalTime.days
timestep=np.zeros(numFiles)
sicAll=np.zeros((numFiles, 448,304))
timeBounds=np.zeros((numFiles,2))
itters=[]
mD=[]
i=0
startTime = datetime.now()
dates=[]
#path=u'/Users/katherinebarnhart/Desktop/RESEARCH/seaIcePPF/b*timeseries.nc'
dirList=glob.glob(pathIn+'b*nh*.nc')
f=nio.open_file(dirList[0], 'r')
fillVal=f.variables['aice_d'].__dict__['_FillValue']
f.close()
del f
for y in yrs:
path=folderpath+str(y)
os.chdir(path)
filenames = sorted(glob.glob('nt*.bin'))
if (len(filenames)>367) or (len(filenames)==263):
datesSort=[fn[3:11] for fn in filenames]
udates=sorted(set(datesSort))
newFiles=[]
for ud in udates:
tempFiles = sorted(glob.glob('nt*'+ud+'*.bin'))
def openGFS(main_directory, file_name):
try:
gfs = nio.open_file(main_directory + file_name, 'r')
return gfs
except:
print 'Something went wrong'
midSIF=250 # day of year for "middle of SIF season" 215~= August 1st midIce=70 fn='/Volumes/Pitcairn/seaicePPF/northernHemisphere/analysisOutput/satelliteObs.timeseries.nc' daysPerMonth=[31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] cumDaysInYear=np.cumsum(daysPerMonth) daysPerMonthLeap=[31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] cumDaysInYearLeap=np.cumsum(daysPerMonthLeap) startTime = datetime.now() print 'Starting averaging of', fn fn_monthOut=fn[:-2]+'monthlyAvg.nc' # f=nio.open_file(fn, 'r') numyears=37 t=f.variables['time'][:] reftime=datetime(1979, 1, 1, 0, 0, 0) ni=f.dimensions['ni'] nj=f.dimensions['nj'] # for each cell do the following: # 1. Create monthly averages of SIC #** add montjly sea ice area estimate nummonths=numyears*12 # do the montly averaging
pickle.dump(running, open(pickleFile, "wb"))
print "Starting Analysis of", fn
fn_analysisOut = fn[:-24] + "Analysis.nc"
pathSplit = fn.split("/")
fnSplit = pathSplit[-1].split(".")
# open files, get aice_d, and time and put into variables with the same name
if fnSplit[2] == bgKey: # if the control run
# construct original file.
fn_inFind = pathIn + "*" + fnSplit[2] + "*" + fnSplit[4] + "*" + fnSplit[7] + "*" + fnSplit[-4] + "*nc"
fn_in = glob.glob(fn_inFind)[0]
f = nio.open_file(fn_in, "r")
aice_d = f.variables["aice_d"][:, :, :]
time = f.variables["time"][:]
numyears = time.size / 365
next_yr = str(int(fnSplit[-4][-8:-4]) + 1).zfill(4)
fn_in_next_Find = pathIn + "*" + fnSplit[2] + "*" + fnSplit[4] + "*" + fnSplit[7] + "*" + next_yr + "*nc"
nextFN = glob.glob(fn_in_next_Find)
if len(nextFN) > 0:
fnext = nio.open_file(nextFN[0], "r")
aice_d = np.append(aice_d, fnext.variables["aice_d"][:365, :, :], 0)
time = np.append(time, fnext.variables["time"][:365], 0)
def sonic(filename):
"""
Create .nc files from RMY Sonic Anemometer ASCII Data from the Lake Thunderbird Micronet
"""
#
# Open ASCII File
#
try:
date = N.loadtxt(filename, usecols=[0], dtype=str, comments="%", delimiter=",")
u1, u2, u3, u4, u5, v1, v2, v3, v4, v5, w1, w2, w3, w4, w5, T1, T2, T3, T4, T5, voltage = N.loadtxt(
filename, usecols=N.arange(1, 22), comments="%", delimiter=",", unpack=True
)
u = N.r_[u1, u2, u3, u4, u5]
v = N.r_[v1, v2, v3, v4, v5]
w = N.r_[w1, w2, w3, w4, w5]
T = N.r_[T1, T2, T3, T4, T5]
dt = N.dtype([("u", N.float), ("v", N.float), ("w", N.float), ("T", N.float)])
data = N.array(zip(u, v, w, T), dtype=dt).reshape(5, -1)
dt = N.dtype([("date", object), ("voltage", N.float)])
ext = N.array(zip(date, voltage), dtype=dt)
netcdf_name = os.path.splitext(os.path.basename(filename))[0] + ".nc"
netcdf_loc = "/micronet/python/data/ltmsonic/netcdf"
netcdf_fpath = os.path.join(netcdf_loc, netcdf_name)
#
# Open netCDF file and let the fun begin!
#
f = nio.open_file(netcdf_fpath, mode="c")
#
# Set dimensions
#
f.create_dimension("station", 5)
f.create_dimension("record_length", 18000)
#
# Create Variables
#
# sdi_var=f.create_variable('station_id','i',('station',))
# lat_var=f.create_variable('latitude','f',('station',))
# long_var=f.create_variable('longitude','f',('station',))
# alt_var=f.create_variable('altitude','f',('station',))
# time_var=f.create_variable('time','i',('station','record_length',))
u_var = f.create_variable("u", "f", ("station", "record_length"))
v_var = f.create_variable("v", "f", ("station", "record_length"))
w_var = f.create_variable("w", "f", ("station", "record_length"))
T_var = f.create_variable("T", "f", ("station", "record_length"))
# bat_v_var=f.create_variable('bat_v','f',('record_length',))
#
# Assign attributes to variables
#
# f.variables['station_id'].long_name = 'tower level'
# f.variables['latitude'].units = 'degrees_north'
# f.variables['latitude'].long_name = 'latitude degrees_north'
# f.variables['longitude'].units = 'degrees_east'
# f.variables['longitude'].long_name = 'longitude degrees_east'
# f.variables['altitude'].units = 'meters'
# f.variables['altitude'].positive = 'up'
# f.variables['altitude'].long_name = 'height above ground level'
# f.variables['time'].units = 'seconds since 1970-01-01 00:00:00.000 UTC'
# f.variables['time'].long_nameunits = 'seconds since 1970-01-01 00:00:00.000 UTC'
f.variables["u"].units = "m/s"
f.variables["u"].long_name = "East-west component of velocity"
# f.variables['u'].missing_value = '-999'
f.variables["v"].units = "m/s"
f.variables["v"].long_name = "North-south component of velocity"
# f.variables['v'].missing_value = '-999'
f.variables["w"].units = "m/s"
f.variables["w"].long_name = "vertical component of velocity (parallel to gravity)"
# f.variables['w'].missing_value = '-999'
f.variables["T"].units = "Kelvin"
f.variables["T"].long_name = "Temperature in Kelvin"
# f.variables['T'].missing_value = '-999'
#
# Assign values to variables
#
# pnt=post_num
# latt=lat_array
# lont=lon_array
# altt=N.array([1.5,3.0,6.0,10.0,15.0])
# outtimet=outtime_array
# for i in range(0,287):
# post_num=N.c_[post_num,pnt]
# lat_array=N.c_[lat_array,latt]
# lon_array=N.c_[lon_array,lont]
# alt_array=N.c_[alt_array,altt]
# outtime_array=ext['date'][0]
# for i in range(1,5):
# outtime_array=N.c_[outtime_array,ext['date'][i]]
# sdi_var.assign_value(N.array([1,2,3,4,5]))
# lat_var.assign_value(lat_array)
# long_var.assign_value(lon_array)
# alt_var.assign_value(alt_array)
# time_var.assign_value(outtime_array)
u_var.assign_value(data["u"].astype("float32"))
v_var.assign_value(data["v"].astype("float32"))
w_var.assign_value(data["w"].astype("float32"))
T_var.assign_value(data["T"].astype("float32"))
# bat_v_var.assign_value(ext['voltage'])
#
# Set global attributes
#
setattr(f, "title", "Lake Thunderbird Micronet sonic anemometer time series")
# setattr(f,'Conventions','Unidata Observation Dataset v1.0')
# setattr(f,'description','Lake Thunderbird Micronet Temperature/RH Daily Time Series')
# setattr(f,'time_coordinate','time')
# setattr(f,'cdm_datatype','Station')
# setattr(f,'stationDimension','station')
# setattr(f,'station_id_variable','station_id')
# setattr(f,'latitude_coortinate','latitude')
# setattr(f,'longitude_coortinate','longitude')
# setattr(f,'altitude_coortinate','altitude')
# setattr(f,'geospatial_lat_max',N.array2string(lat_array.max()))
# setattr(f,'geospatial_lat_min',N.array2string(lat_array.min()))
# setattr(f,'geospatial_lon_max',N.array2string(lon_array.max()))
# setattr(f,'geospatial_lon_min',N.array2string(lon_array.min()))
# setattr(f,'time_coverage_start',str(int(outtime[0]))+" seconds since 1970-01-01 00:00:00.000 UTC")
# setattr(f,'time_coverage_end',str(int(outtime[-1]))+" seconds since 1970-01-01 00:00:00.000 UTC")
# setattr(f,'observationDimension','record_length')
#
# Close netCDF file
#
f.close()
except IOError:
print "%s does not exist\n" % filename
raise
def write_data(filename,data,base,geo,inputtimes=None,fillvalue=-9999):
"""Write data to filename with attributes from base and lat-lon from geo"""
of=nio.open_file(filename,mode='w',format="nc")
# of=netCDF4.Dataset(filename,mode='w',format="nc")
of.title=meta.title
of.creator=meta.creator
of.creation_date=time.ctime()
if base.global_attributes.has_key("history"):
past_history=" last{"+base.global_attributes["history"]+"}"
else:
past_history=""
of.history="Created : "+time.ctime()+" with wrf_agg2obs.py "+past_history
of.source=meta.source
of.Conventions=meta.conventions
dimnames=dict(Time="Time",DateStrLen = "DateStrLen", west_east = "lon", south_north = "lat")
longnames=dict(Q2="Specific Humidity",T2="Temperature",PSFC="Pressure",U10="Wind Speed (east-west)",
V10="Wind Speed (North South)",RAINNC="Precipitation",
SWDOWN="Downward Shortwave Radiation",GLW="Downward Longwave Radiation")
for dimkey in base.dimensions.keys():
if dimkey=="Time":
of.create_dimension("Time",0)
elif dimkey=="soil_layers_stag":
pass
elif dimkey=="DateStrLen":
pass
elif dimkey=="south_north":
of.create_dimension(dimnames[dimkey],geo.lat.shape[0])
elif dimkey=="west_east":
of.create_dimension(dimnames[dimkey],geo.lat.shape[1])
else:
of.create_dimension(dimnames[dimkey],base.dimensions[dimkey])
outvar=of.create_variable("Times","d",("Time",))
outvar[:inputtimes.size]=inputtimes
outvar=of.create_variable("lat","f",("lat","lon"))
outvar.units="degrees"
outvar.longname="latitude"
outvar[:]=geo.lat.astype(np.float32)
outvar=of.create_variable("lon","f",("lat","lon"))
outvar[:]=geo.lon.astype(np.float32)
outvar.units="degrees"
outvar.longname="longitude"
for var in var_list:
curvar=base.variables[var]
curdims=tuple([dimnames[thisdim] for thisdim in curvar.dims])
outvar=of.create_variable(var,curvar.dtype,curdims)
for k in curvar.atts.keys():
setattr(outvar,k,curvar.atts[k])
if (var=="RAINNC"):
outvar.units="kg/m^2/s"
outvar.description="Hourly Precipitation"
outvar.longname=longnames[var]
outvar._FillValue=np.float32(fillvalue)
for i in range(data[var].shape[0]):
outvar[i,:,:]=data[var][i,:,:].astype(curvar.dtype)
of.close()
del las1850
for ittR in range(len(rcpkey)):
fnAnalysis = '/Volumes/Pitcairn/seaicePPF/northernHemisphere/analysisOutput/allModels.Summary.' + nsk + '.' + rcpName[
ittR] + '.nc'
print fnAnalysis
dirList = glob.glob(path + '*' + rcpName[ittR] + '*' + nsk +
'*Analysis.nc')
analysisFiles = []
for fn in dirList:
analysisFiles.append(Dataset(fn, 'r'))
#print fn, (datetime.now()-startTime)
key = 'numSIF'
f = nio.open_file(dirList[0], 'r')
landMask = f.variables[key][0, :, :] > 1200
fillVal = f.variables[key].__dict__['_FillValue']
ni = f.dimensions['ni']
nj = f.dimensions['nj']
nm = len(dirList)
numModels = len(dirList)
lastYear = int(np.floor(f.variables['time'][:].max() / 365))
numYears = lastYear - 1850
if numYears == 249:
numYears = 250
lastYear = lastYear - 1
del f
numSIF = np.nan * np.ones((numModels, numYears, nj, ni))
if os.path.isfile(fn_monthOut) == False:
fileNames.append(fn)
print fn_monthOut
# process the remaining input files.
startTime = datetime.now()
for fn in fileNames:
try:
print 'Starting averaging of', fn
pathSplit = fn.split('/')
fnSplit = pathSplit[-1].split('.')
# open files, get aice_d, and time and put into variables with the same name
if fnSplit[2] == bgKey: # if the control run
fnTemp = str.join(".", fnSplit[:-1]) + '.timeseries.nc'
f = nio.open_file(fn, 'r')
aice_d = f.variables['aice_d'][:, :, :]
time = f.variables['time'][:]
if fnSplit[2] == runPart1key:
fnTemp = str.join(".", fnSplit[:-2]) + '.timeseries.nc'
# now instead you need to open THREE FILES:
# first open the first one
f = nio.open_file(fn, 'r')
aice_d = f.variables['aice_d'][:, :, :]
time = f.variables['time'][:]
# now find the other two and add them to the end
pathConstruct2 = str.join(
'/', pathSplit[:-1]) + '/*' + runParts23key + '*' + fnSplit[
4] + '*' + fnSplit[7] + '*nc'
print 'Starting Analysis of', fn
fn_analysisOut = fn[:-24] + 'Analysis.nc'
pathSplit = fn.split('/')
fnSplit = pathSplit[-1].split('.')
# open files, get aice_d, and time and put into variables with the same name
if fnSplit[2] == bgKey: # if the control run
# construct original file.
fn_inFind = pathIn + '*' + fnSplit[2] + '*' + fnSplit[
4] + '*' + fnSplit[7] + '*' + fnSplit[-4] + '*nc'
fn_in = glob.glob(fn_inFind)[0]
f = nio.open_file(fn_in, 'r')
aice_d = f.variables['aice_d'][:, :, :]
time = f.variables['time'][:]
numyears = time.size / 365
next_yr = str(int(fnSplit[-4][-8:-4]) + 1).zfill(4)
fn_in_next_Find = pathIn + '*' + fnSplit[2] + '*' + fnSplit[
4] + '*' + fnSplit[7] + '*' + next_yr + '*nc'
nextFN = glob.glob(fn_in_next_Find)
if len(nextFN) > 0:
fnext = nio.open_file(nextFN[0], 'r')
aice_d = np.append(aice_d,
dataset['var_diff'] = {'data': var_diff}
dataset['var_diff'][
'longName'] = 'Difference of model mean and observed 1979-2014 detrended variance'
dataset['var_diff']['units'] = 'Days'
dataset['slope_diff'] = {'data': slope_diff}
dataset['slope_diff'][
'longName'] = 'Difference of model mean and observed 1979-2014 trend'
dataset['slope_diff']['units'] = 'Days per year'
# save as a netcdf
import nio
f = nio.open_file(obs, 'r')
fnAnalysis = '/Volumes/Pitcairn/seaicePPF/northernHemisphere/analysisOutput/obsModelComparison.nc'
fAn = Dataset(fnAnalysis, 'w', format='NETCDF4')
# create all the dimentions, set time to unlimited
for k in f.dimensions.keys():
if f.unlimited(k) == True:
fAn.createDimension(k, None)
else:
fAn.createDimension(k, f.dimensions[k])
fAn.createDimension('nm', numModels)
# use the netCDF4 instead of pyNIO since it seems to work much better with unlimited variables
fAnVars = {}
for key in {'TLAT', 'TLON', 'time_bounds', 'time'}:
print 'creating ', key
dataset['slope_percentile']['longName']='Comparison of slope'
dataset['slope_percentile']['units']='percentile of 1979-2014 slope'
dataset['var_diff']={'data':var_diff}
dataset['var_diff']['longName']='Difference of model mean and observed 1979-2014 detrended variance'
dataset['var_diff']['units']='Days'
dataset['slope_diff']={'data':slope_diff}
dataset['slope_diff']['longName']='Difference of model mean and observed 1979-2014 trend'
dataset['slope_diff']['units']='Days per year'
# save as a netcdf
import nio
f=nio.open_file(obs, 'r')
fnAnalysis='/Volumes/Pitcairn/seaicePPF/northernHemisphere/analysisOutput/obsModelComparison.nc'
fAn=Dataset(fnAnalysis, 'w',format='NETCDF4')
# create all the dimentions, set time to unlimited
for k in f.dimensions.keys():
if f.unlimited(k)==True:
fAn.createDimension(k, None)
else:
fAn.createDimension(k, f.dimensions[k])
fAn.createDimension('nm', numModels)
# use the netCDF4 instead of pyNIO since it seems to work much better with unlimited variables
fAnVars={}
for key in {'TLAT', 'TLON','time_bounds', 'time'}:
print 'creating ', key
analysisFiles = []
for fn in dirList:
analysisFiles.append(Dataset(fn, 'r'))
print fn, (datetime.now() - startTime)
analysisFiles2 = []
for fn in dirList2:
analysisFiles2.append(Dataset(fn, 'r'))
print fn, (datetime.now() - startTime)
numModels = len(dirList)
numYears = (2100 - 1850)
key = 'aice_d'
f = nio.open_file(
u'/Volumes/Pitcairn/seaicePPF/p/cesm0005/CESM-CAM5-BGC-LE/ice/proc/tseries/daily/aice_d/b.e11.B20TRC5CNBDRD.f09_g16.002.cice.h1.aice_d_nh.19200101-20051231.nc',
'r')
landMask = f.variables[key][0, :, :] > 120
fillVal = f.variables[key].__dict__['_FillValue']
del f
NI = landMask.shape[1]
NJ = landMask.shape[0]
NM = len(analysisFiles)
numSIF = np.nan * np.ones((numModels, numYears, NJ, NI))
first = np.nan * np.ones((numModels, numYears, NJ, NI))
last = np.nan * np.ones((numModels, numYears, NJ, NI))
i = 0
for fn in dirList:
for fn in dirList:
analysisFiles.append(Dataset(fn, 'r'))
print fn, (datetime.now()-startTime)
analysisFiles2=[]
for fn in dirList2:
analysisFiles2.append(Dataset(fn, 'r'))
print fn, (datetime.now()-startTime)
numModels=len(dirList)
numYears=(2100-1850)
key='aice_d'
f=nio.open_file(u'/Volumes/Pitcairn/seaicePPF/p/cesm0005/CESM-CAM5-BGC-LE/ice/proc/tseries/daily/aice_d/b.e11.B20TRC5CNBDRD.f09_g16.002.cice.h1.aice_d_nh.19200101-20051231.nc','r')
landMask=f.variables[key][0,:,:]>120
fillVal=f.variables[key].__dict__['_FillValue']
del f
NI=landMask.shape[1]
NJ=landMask.shape[0]
NM=len(analysisFiles)
numSIF=np.nan*np.ones((numModels,numYears,NJ,NI))
first=np.nan*np.ones((numModels,numYears,NJ,NI))
last=np.nan*np.ones((numModels,numYears,NJ,NI))
i=0
for fn in dirList:
f=Dataset(fn, 'r')
