def ncaverage(varname, files, outname):
"""
Create a netCDF file OUTNAME by averaging the lon/lat grids stored in
VARNAME in the input files FILES.
Return a dictionary of the attributes from the input files
so that the caller can create addiitonal application-specific metadata
to insert into the output file.
Checks are performed to make sure that input units are either consistent
or non-existent, and that the input grid shapes are the same, and the
endpoints of the coordinate variables match.
"""
history = []
now = datetime.now()
history.append(
"ncaverage at %04d-%02d-%02dT%02d:%02d:%02d, files..." %
(now.year, now.month, now.day, now.hour, now.minute, now.second))
units = None
attributes = dict()
first = 1
for file in files:
print "File: " + file
history.append(" + " + file)
ncobj = nh.nc3_open(file, 'r')
attr = nh.nc3_get_attributes(ncobj)
if varname + ':units' in attr:
newunit = attr[varname + ':units']
if units:
if newunit != units:
raise ValueError("Inconsistent units (" + newunits +
") on var " + varname + " in file " +
file + ", previous files were (" + units +
")")
units = newunit
attributes[file] = attr
# We need these for the first file and then on each subsequent file
a = ncobj.variables[varname]
latvec = ncobj.variables['latitude']
lonvec = ncobj.variables['longitude']
tvec = ncobj.variables['time']
# The first file is used to 1) set the shape of the output
# 2) capture some basic metadata used in writing the output file,
# which we start to write here while we've got access to the
# dimension variables.
if first:
first = 0
mean = numpy.zeros(a.shape, dtype=numpy.float32)
n = numpy.zeros(a.shape, dtype=numpy.int32)
if 'time:calendar' in attributes[file]:
t1 = netCDF3.num2date(tvec[0],
units=tvec.units,
calendar=tvec.calendar)
else:
t1 = netCDF3.num2date(tvec[0], units=tvec.units)
t2 = t1
ncout = nh.nc3_open(outname, 'w')
nh.nc3_set_timelatlon(ncout, 1, len(latvec), len(lonvec))
nh.nc3_add_data(ncout, 'latitude', latvec)
nh.nc3_add_data(ncout, 'longitude', lonvec)
# remember the endpoints of each CV and compute the step size for each
lat1 = latvec[0]
lat2 = latvec[len(latvec) - 1]
lon1 = lonvec[0]
lon2 = lonvec[len(lonvec) - 1]
dlat = abs(lat2 - lat1) / len(latvec)
dlon = abs(lon2 - lon1) / len(lonvec)
# Guard against incompatible files - shapes must be the
# same AND CV endpoint values must match within a fraction of a pixel.
if a.shape != mean.shape:
raise ValueError("Data in " + file +
" is a different shape from first file")
if (abs(latvec[0] - lat1) > 0.01 * dlat
or abs(latvec[len(latvec) - 1] - lat2) > 0.01 * dlat
or abs(lonvec[0] - lon1) > 0.01 * dlon
or abs(lonvec[len(lonvec) - 1] - lon2) > 0.01 * dlon):
raise ValueError("Coordinate endpoints in " + file +
" do not match first file")
# Keep track of the earliest and latest times
if 'time:calendar' in attributes[file]:
t = netCDF3.num2date(tvec[0],
units=tvec.units,
calendar=tvec.calendar)
else:
t = netCDF3.num2date(tvec[0], units=tvec.units)
if t < t1: t1 = t
if t > t2: t2 = t
# b is 1 unless there is a missing value, where it is 0
fillValue = ncobj.variables[varname]._FillValue
b = numpy.where(a == fillValue, 0, 1)
n = n + b
mean = mean + b * a
nh.nc3_close(ncobj)
# Before computing mean, set all values with 0 count to _FillValue, and
# the corresponding count to 1. This avoids division by zero and leaves
# those pixels set to _FillValue.
mean = numpy.where(n == 0, fillValue, mean)
n = numpy.where(n == 0, 1, n)
mean = mean / n
nh.nc3_set_var(ncout, varname)
nh.nc3_add_time(ncout, [t1])
nh.nc3_add_data(ncout, varname, mean)
attr = {}
attr['history'] = '\n'.join(history)
attr[varname + ':_FillValue'] = fillValue
if units:
attr[varname + ':units'] = units
nh.nc3_set_attributes(ncout, attr)
nh.nc3_close(ncout)
return attributes
def ncaverage(varname, files, outname):
"""
Create a netCDF file OUTNAME by averaging the lon/lat grids stored in
VARNAME in the input files FILES.
Return a dictionary of the attributes from the input files
so that the caller can create addiitonal application-specific metadata
to insert into the output file.
Checks are performed to make sure that input units are either consistent
or non-existent, and that the input grid shapes are the same, and the
endpoints of the coordinate variables match.
"""
history = []
now = datetime.now()
history.append("ncaverage at %04d-%02d-%02dT%02d:%02d:%02d, files..." % (now.year, now.month, now.day, now.hour, now.minute, now.second))
units = None
attributes = dict();
first = 1
for file in files:
print "File: "+file
history.append(" + "+file)
ncobj = nh.nc3_open(file,'r')
attr = nh.nc3_get_attributes(ncobj)
if varname+':units' in attr:
newunit = attr[varname+':units']
if units:
if newunit != units:
raise ValueError("Inconsistent units ("+newunits+") on var "+varname+" in file "+file+", previous files were ("+units+")")
units = newunit
attributes[file] = attr
# We need these for the first file and then on each subsequent file
a = ncobj.variables[varname]
latvec = ncobj.variables['latitude']
lonvec = ncobj.variables['longitude']
tvec = ncobj.variables['time']
# The first file is used to 1) set the shape of the output
# 2) capture some basic metadata used in writing the output file,
# which we start to write here while we've got access to the
# dimension variables.
if first:
first = 0
mean = numpy.zeros(a.shape, dtype=numpy.float32)
n = numpy.zeros(a.shape, dtype=numpy.int32)
if 'time:calendar' in attributes[file]:
t1 = netCDF3.num2date(tvec[0],units=tvec.units,calendar=tvec.calendar)
else:
t1 = netCDF3.num2date(tvec[0],units=tvec.units)
t2 = t1
ncout = nh.nc3_open(outname,'w')
nh.nc3_set_timelatlon(ncout,1,len(latvec),len(lonvec))
nh.nc3_add_data(ncout,'latitude',latvec)
nh.nc3_add_data(ncout,'longitude',lonvec)
# remember the endpoints of each CV and compute the step size for each
lat1 = latvec[0]
lat2 = latvec[len(latvec)-1]
lon1 = lonvec[0]
lon2 = lonvec[len(lonvec)-1]
dlat = abs(lat2-lat1)/len(latvec)
dlon = abs(lon2-lon1)/len(lonvec)
# Guard against incompatible files - shapes must be the
# same AND CV endpoint values must match within a fraction of a pixel.
if a.shape != mean.shape:
raise ValueError("Data in "+file+" is a different shape from first file")
if (abs(latvec[0]-lat1) > 0.01*dlat or
abs(latvec[len(latvec)-1]-lat2) > 0.01*dlat or
abs(lonvec[0]-lon1) > 0.01*dlon or
abs(lonvec[len(lonvec)-1]-lon2) > 0.01*dlon):
raise ValueError("Coordinate endpoints in "+file+" do not match first file")
# Keep track of the earliest and latest times
if 'time:calendar' in attributes[file]:
t = netCDF3.num2date(tvec[0],units=tvec.units,calendar=tvec.calendar)
else:
t = netCDF3.num2date(tvec[0],units=tvec.units)
if t < t1: t1 = t
if t > t2: t2 = t
# b is 1 unless there is a missing value, where it is 0
fillValue = ncobj.variables[varname]._FillValue
b = numpy.where(a == fillValue, 0, 1)
n = n+b
mean = mean+b*a
nh.nc3_close(ncobj)
# Before computing mean, set all values with 0 count to _FillValue, and
# the corresponding count to 1. This avoids division by zero and leaves
# those pixels set to _FillValue.
mean = numpy.where(n==0, fillValue, mean)
n = numpy.where(n==0, 1, n)
mean = mean/n
nh.nc3_set_var(ncout,varname)
nh.nc3_add_time(ncout,[t1])
nh.nc3_add_data(ncout,varname,mean)
attr = {}
attr['history'] = '\n'.join(history)
attr[varname+':_FillValue'] = fillValue
if units:
attr[varname+':units'] = units
nh.nc3_set_attributes(ncout,attr)
nh.nc3_close(ncout)
return attributes
def nctoflt(ncfile, fltstem, varname, iz=0):
"""Main function to process a netCDF file to binary flt
Output files have the stem name and suffix .flt and .hdr
If varname is 3D, then iz is the index of the first dimension used
to extract a 2D slice.
If the latitude runs south to north, then the grid is flipped before
being written
"""
ncobj = nh.nc3_open(ncfile,'r')
a = ncobj.variables[varname]
# Copy out into a numpy array and make sure we have only
# 2 dimensions and type float32.
b = numpy.float32(ncobj.variables[varname])
if len(b.shape) < 2 or len(b.shape) > 3:
raise ValueError("Only 2D and 3D data allowed (not "+len(b.shape)+"D)")
if len(b.shape) == 3:
b = numpy.float32(b[iz,::,::].reshape(b.shape[1], b.shape[2]))
fillValue = numpy.float32(ncobj.variables[varname]._FillValue)
latvec = ncobj.variables['latitude']
lonvec = ncobj.variables['longitude']
lat1 = latvec[0]
lat2 = latvec[len(latvec)-1]
# Reverse if latitude runs South to North
if lat1 < lat2:
x = lat2
lat2 = lat1
lat1 = x
b = b[::-1,]
lon1 = lonvec[0]
lon2 = lonvec[len(lonvec)-1]
dlat = abs(lat1-lat2)/(len(latvec)-1)
dlon = abs(lon2-lon1)/(len(lonvec)-1)
xll = lon1-dlon*0.5
yll = lat2-dlat*0.5
fltname = fltstem+'.flt'
if os.path.exists(fltname): os.unlink(fltname)
b.tofile(fltname)
f = file(fltstem+".hdr","w")
f.write("ncols %d\n" % b.shape[1])
f.write("nrows %d\n" % b.shape[0])
f.write("xllcorner %f\n" % xll)
f.write("yllcorner %f\n" % yll)
f.write("cellsize %f\n" % dlon)
f.write("NODATA_value %f\n" % fillValue)
if sys.byteorder == "little":
f.write("byteorder LSBFIRST\n")
else:
f.write("byteorder LSBLAST\n")
f.close()
attr = nh.nc3_get_attributes(ncobj)
nh.nc3_close(ncobj)
return attr
def flttonc(fltstem, ncfile, varname, yyyymmdd, units=None):
"""Main function to process a binary flt and hdr with name FLTSTEM
to netCDF file NCFILE with variable name VARNAME. Time is set to
the date corresponding to YYYYMMDD. Units can be provided optionally.
"""
d = datetime.strptime(yyyymmdd, "%Y%M%d")
# Read the metadata from the header
meta = {}
r = re.compile("^(\S+)\s+(\S+)$")
f = open(fltstem+'.hdr','r')
for line in f:
line=line.strip()
m = re.match(r, line)
if m:
if m.group(1).lower() == 'byteorder':
meta[m.group(1).lower()] = m.group(2)
else:
meta[m.group(1).lower()] = float(m.group(2))
f.close()
# Get the data and shape it appropriately
a=np.fromfile(fltstem+'.flt',dtype=np.float32)
a.reshape(int(meta['ncols']), int(meta['nrows']), 1)
# Make the lon and lat coordinate variables
if 'xllcenter' in meta:
lon = nr.create_vector(meta['xllcenter'], \
meta['ncols'], \
meta['cellsize'])
else:
lon = nr.create_vector(meta['xllcorner']+0.5*meta['cellsize'], \
meta['ncols'], \
meta['cellsize'])
if 'yllcenter' in meta:
lat = nr.create_vector(meta['yllcenter'], \
meta['nrows'], \
meta['cellsize'])
else:
lat = nr.create_vector(meta['yllcorner']+0.5*meta['cellsize'], \
meta['nrows'], \
meta['cellsize'])
# Reverse the latitude elements so they run from North to South
lat = lat[::-1]
# Make a :history global attribute
history = []
now = datetime.now()
history.append("flttonc at %04d-%02d-%02dT%02d:%02d:%02d" % (now.year, now.month, now.day, now.hour, now.minute, now.second))
history.append("Input file: "+fltstem+".flt")
attr = {}
if 'nodata_value' in meta:
attr[varname+':_FillValue'] = meta['nodata_value']
if units:
attr[varname+':units'] = units
attr['history'] = '\n'.join(history)
# Write the netCDF file
ncobj = nh.nc3_open(ncfile,'w')
nh.nc3_set_timelatlon(ncobj,1,len(lat),len(lon))
nh.nc3_set_var(ncobj,varname)
nh.nc3_set_var(ncobj,'wgs84',dims=()) # Grid mapping container
nh.nc3_add_time(ncobj,[d])
nh.nc3_add_data(ncobj,'latitude',lat)
nh.nc3_add_data(ncobj,'longitude',lon)
nh.nc3_add_data(ncobj,varname,a)
nh.nc3_set_attributes(ncobj,attr)
nh.nc3_close(ncobj)
return ncfile
def flttonc(fltstem, ncfile, varname, yyyymmdd, units=None):
"""Main function to process a binary flt and hdr with name FLTSTEM
to netCDF file NCFILE with variable name VARNAME. Time is set to
the date corresponding to YYYYMMDD. Units can be provided optionally.
"""
d = datetime.strptime(yyyymmdd, "%Y%M%d")
# Read the metadata from the header
meta = {}
r = re.compile("^(\S+)\s+(\S+)$")
f = open(fltstem + '.hdr', 'r')
for line in f:
line = line.strip()
m = re.match(r, line)
if m:
if m.group(1).lower() == 'byteorder':
meta[m.group(1).lower()] = m.group(2)
else:
meta[m.group(1).lower()] = float(m.group(2))
f.close()
# Get the data and shape it appropriately
a = np.fromfile(fltstem + '.flt', dtype=np.float32)
a.reshape(int(meta['ncols']), int(meta['nrows']), 1)
# Make the lon and lat coordinate variables
if 'xllcenter' in meta:
lon = nr.create_vector(meta['xllcenter'], \
meta['ncols'], \
meta['cellsize'])
else:
lon = nr.create_vector(meta['xllcorner']+0.5*meta['cellsize'], \
meta['ncols'], \
meta['cellsize'])
if 'yllcenter' in meta:
lat = nr.create_vector(meta['yllcenter'], \
meta['nrows'], \
meta['cellsize'])
else:
lat = nr.create_vector(meta['yllcorner']+0.5*meta['cellsize'], \
meta['nrows'], \
meta['cellsize'])
# Reverse the latitude elements so they run from North to South
lat = lat[::-1]
# Make a :history global attribute
history = []
now = datetime.now()
history.append(
"flttonc at %04d-%02d-%02dT%02d:%02d:%02d" %
(now.year, now.month, now.day, now.hour, now.minute, now.second))
history.append("Input file: " + fltstem + ".flt")
attr = {}
if 'nodata_value' in meta:
attr[varname + ':_FillValue'] = meta['nodata_value']
if units:
attr[varname + ':units'] = units
attr['history'] = '\n'.join(history)
# Write the netCDF file
ncobj = nh.nc3_open(ncfile, 'w')
nh.nc3_set_timelatlon(ncobj, 1, len(lat), len(lon))
nh.nc3_set_var(ncobj, varname)
nh.nc3_set_var(ncobj, 'wgs84', dims=()) # Grid mapping container
nh.nc3_add_time(ncobj, [d])
nh.nc3_add_data(ncobj, 'latitude', lat)
nh.nc3_add_data(ncobj, 'longitude', lon)
nh.nc3_add_data(ncobj, varname, a)
nh.nc3_set_attributes(ncobj, attr)
nh.nc3_close(ncobj)
return ncfile
def nctoflt(ncfile, fltstem, varname, iz=0):
"""Main function to process a netCDF file to binary flt
Output files have the stem name and suffix .flt and .hdr
If varname is 3D, then iz is the index of the first dimension used
to extract a 2D slice.
If the latitude runs south to north, then the grid is flipped before
being written
"""
ncobj = nh.nc3_open(ncfile, 'r')
a = ncobj.variables[varname]
# Copy out into a numpy array and make sure we have only
# 2 dimensions and type float32.
b = numpy.float32(ncobj.variables[varname])
if len(b.shape) < 2 or len(b.shape) > 3:
raise ValueError("Only 2D and 3D data allowed (not " + len(b.shape) +
"D)")
if len(b.shape) == 3:
b = numpy.float32(b[iz, ::, ::].reshape(b.shape[1], b.shape[2]))
fillValue = numpy.float32(ncobj.variables[varname]._FillValue)
latvec = ncobj.variables['latitude']
lonvec = ncobj.variables['longitude']
lat1 = latvec[0]
lat2 = latvec[len(latvec) - 1]
# Reverse if latitude runs South to North
if lat1 < lat2:
x = lat2
lat2 = lat1
lat1 = x
b = b[::-1, ]
lon1 = lonvec[0]
lon2 = lonvec[len(lonvec) - 1]
dlat = abs(lat1 - lat2) / (len(latvec) - 1)
dlon = abs(lon2 - lon1) / (len(lonvec) - 1)
xll = lon1 - dlon * 0.5
yll = lat2 - dlat * 0.5
fltname = fltstem + '.flt'
if os.path.exists(fltname): os.unlink(fltname)
b.tofile(fltname)
f = file(fltstem + ".hdr", "w")
f.write("ncols %d\n" % b.shape[1])
f.write("nrows %d\n" % b.shape[0])
f.write("xllcorner %f\n" % xll)
f.write("yllcorner %f\n" % yll)
f.write("cellsize %f\n" % dlon)
f.write("NODATA_value %f\n" % fillValue)
if sys.byteorder == "little":
f.write("byteorder LSBFIRST\n")
else:
f.write("byteorder LSBLAST\n")
f.close()
attr = nh.nc3_get_attributes(ncobj)
nh.nc3_close(ncobj)
return attr