def ncaverage(varname, files, outname): """ Create a netCDF file OUTNAME by averaging the lon/lat grids stored in VARNAME in the input files FILES. Return a dictionary of the attributes from the input files so that the caller can create addiitonal application-specific metadata to insert into the output file. Checks are performed to make sure that input units are either consistent or non-existent, and that the input grid shapes are the same, and the endpoints of the coordinate variables match. """ history = [] now = datetime.now() history.append( "ncaverage at %04d-%02d-%02dT%02d:%02d:%02d, files..." % (now.year, now.month, now.day, now.hour, now.minute, now.second)) units = None attributes = dict() first = 1 for file in files: print "File: " + file history.append(" + " + file) ncobj = nh.nc3_open(file, 'r') attr = nh.nc3_get_attributes(ncobj) if varname + ':units' in attr: newunit = attr[varname + ':units'] if units: if newunit != units: raise ValueError("Inconsistent units (" + newunits + ") on var " + varname + " in file " + file + ", previous files were (" + units + ")") units = newunit attributes[file] = attr # We need these for the first file and then on each subsequent file a = ncobj.variables[varname] latvec = ncobj.variables['latitude'] lonvec = ncobj.variables['longitude'] tvec = ncobj.variables['time'] # The first file is used to 1) set the shape of the output # 2) capture some basic metadata used in writing the output file, # which we start to write here while we've got access to the # dimension variables. if first: first = 0 mean = numpy.zeros(a.shape, dtype=numpy.float32) n = numpy.zeros(a.shape, dtype=numpy.int32) if 'time:calendar' in attributes[file]: t1 = netCDF3.num2date(tvec[0], units=tvec.units, calendar=tvec.calendar) else: t1 = netCDF3.num2date(tvec[0], units=tvec.units) t2 = t1 ncout = nh.nc3_open(outname, 'w') nh.nc3_set_timelatlon(ncout, 1, len(latvec), len(lonvec)) nh.nc3_add_data(ncout, 'latitude', latvec) nh.nc3_add_data(ncout, 'longitude', lonvec) # remember the endpoints of each CV and compute the step size for each lat1 = latvec[0] lat2 = latvec[len(latvec) - 1] lon1 = lonvec[0] lon2 = lonvec[len(lonvec) - 1] dlat = abs(lat2 - lat1) / len(latvec) dlon = abs(lon2 - lon1) / len(lonvec) # Guard against incompatible files - shapes must be the # same AND CV endpoint values must match within a fraction of a pixel. if a.shape != mean.shape: raise ValueError("Data in " + file + " is a different shape from first file") if (abs(latvec[0] - lat1) > 0.01 * dlat or abs(latvec[len(latvec) - 1] - lat2) > 0.01 * dlat or abs(lonvec[0] - lon1) > 0.01 * dlon or abs(lonvec[len(lonvec) - 1] - lon2) > 0.01 * dlon): raise ValueError("Coordinate endpoints in " + file + " do not match first file") # Keep track of the earliest and latest times if 'time:calendar' in attributes[file]: t = netCDF3.num2date(tvec[0], units=tvec.units, calendar=tvec.calendar) else: t = netCDF3.num2date(tvec[0], units=tvec.units) if t < t1: t1 = t if t > t2: t2 = t # b is 1 unless there is a missing value, where it is 0 fillValue = ncobj.variables[varname]._FillValue b = numpy.where(a == fillValue, 0, 1) n = n + b mean = mean + b * a nh.nc3_close(ncobj) # Before computing mean, set all values with 0 count to _FillValue, and # the corresponding count to 1. This avoids division by zero and leaves # those pixels set to _FillValue. mean = numpy.where(n == 0, fillValue, mean) n = numpy.where(n == 0, 1, n) mean = mean / n nh.nc3_set_var(ncout, varname) nh.nc3_add_time(ncout, [t1]) nh.nc3_add_data(ncout, varname, mean) attr = {} attr['history'] = '\n'.join(history) attr[varname + ':_FillValue'] = fillValue if units: attr[varname + ':units'] = units nh.nc3_set_attributes(ncout, attr) nh.nc3_close(ncout) return attributes
def flttonc(fltstem, ncfile, varname, yyyymmdd, units=None): """Main function to process a binary flt and hdr with name FLTSTEM to netCDF file NCFILE with variable name VARNAME. Time is set to the date corresponding to YYYYMMDD. Units can be provided optionally. """ d = datetime.strptime(yyyymmdd, "%Y%M%d") # Read the metadata from the header meta = {} r = re.compile("^(\S+)\s+(\S+)$") f = open(fltstem+'.hdr','r') for line in f: line=line.strip() m = re.match(r, line) if m: if m.group(1).lower() == 'byteorder': meta[m.group(1).lower()] = m.group(2) else: meta[m.group(1).lower()] = float(m.group(2)) f.close() # Get the data and shape it appropriately a=np.fromfile(fltstem+'.flt',dtype=np.float32) a.reshape(int(meta['ncols']), int(meta['nrows']), 1) # Make the lon and lat coordinate variables if 'xllcenter' in meta: lon = nr.create_vector(meta['xllcenter'], \ meta['ncols'], \ meta['cellsize']) else: lon = nr.create_vector(meta['xllcorner']+0.5*meta['cellsize'], \ meta['ncols'], \ meta['cellsize']) if 'yllcenter' in meta: lat = nr.create_vector(meta['yllcenter'], \ meta['nrows'], \ meta['cellsize']) else: lat = nr.create_vector(meta['yllcorner']+0.5*meta['cellsize'], \ meta['nrows'], \ meta['cellsize']) # Reverse the latitude elements so they run from North to South lat = lat[::-1] # Make a :history global attribute history = [] now = datetime.now() history.append("flttonc at %04d-%02d-%02dT%02d:%02d:%02d" % (now.year, now.month, now.day, now.hour, now.minute, now.second)) history.append("Input file: "+fltstem+".flt") attr = {} if 'nodata_value' in meta: attr[varname+':_FillValue'] = meta['nodata_value'] if units: attr[varname+':units'] = units attr['history'] = '\n'.join(history) # Write the netCDF file ncobj = nh.nc3_open(ncfile,'w') nh.nc3_set_timelatlon(ncobj,1,len(lat),len(lon)) nh.nc3_set_var(ncobj,varname) nh.nc3_set_var(ncobj,'wgs84',dims=()) # Grid mapping container nh.nc3_add_time(ncobj,[d]) nh.nc3_add_data(ncobj,'latitude',lat) nh.nc3_add_data(ncobj,'longitude',lon) nh.nc3_add_data(ncobj,varname,a) nh.nc3_set_attributes(ncobj,attr) nh.nc3_close(ncobj) return ncfile
def ncaverage(varname, files, outname): """ Create a netCDF file OUTNAME by averaging the lon/lat grids stored in VARNAME in the input files FILES. Return a dictionary of the attributes from the input files so that the caller can create addiitonal application-specific metadata to insert into the output file. Checks are performed to make sure that input units are either consistent or non-existent, and that the input grid shapes are the same, and the endpoints of the coordinate variables match. """ history = [] now = datetime.now() history.append("ncaverage at %04d-%02d-%02dT%02d:%02d:%02d, files..." % (now.year, now.month, now.day, now.hour, now.minute, now.second)) units = None attributes = dict(); first = 1 for file in files: print "File: "+file history.append(" + "+file) ncobj = nh.nc3_open(file,'r') attr = nh.nc3_get_attributes(ncobj) if varname+':units' in attr: newunit = attr[varname+':units'] if units: if newunit != units: raise ValueError("Inconsistent units ("+newunits+") on var "+varname+" in file "+file+", previous files were ("+units+")") units = newunit attributes[file] = attr # We need these for the first file and then on each subsequent file a = ncobj.variables[varname] latvec = ncobj.variables['latitude'] lonvec = ncobj.variables['longitude'] tvec = ncobj.variables['time'] # The first file is used to 1) set the shape of the output # 2) capture some basic metadata used in writing the output file, # which we start to write here while we've got access to the # dimension variables. if first: first = 0 mean = numpy.zeros(a.shape, dtype=numpy.float32) n = numpy.zeros(a.shape, dtype=numpy.int32) if 'time:calendar' in attributes[file]: t1 = netCDF3.num2date(tvec[0],units=tvec.units,calendar=tvec.calendar) else: t1 = netCDF3.num2date(tvec[0],units=tvec.units) t2 = t1 ncout = nh.nc3_open(outname,'w') nh.nc3_set_timelatlon(ncout,1,len(latvec),len(lonvec)) nh.nc3_add_data(ncout,'latitude',latvec) nh.nc3_add_data(ncout,'longitude',lonvec) # remember the endpoints of each CV and compute the step size for each lat1 = latvec[0] lat2 = latvec[len(latvec)-1] lon1 = lonvec[0] lon2 = lonvec[len(lonvec)-1] dlat = abs(lat2-lat1)/len(latvec) dlon = abs(lon2-lon1)/len(lonvec) # Guard against incompatible files - shapes must be the # same AND CV endpoint values must match within a fraction of a pixel. if a.shape != mean.shape: raise ValueError("Data in "+file+" is a different shape from first file") if (abs(latvec[0]-lat1) > 0.01*dlat or abs(latvec[len(latvec)-1]-lat2) > 0.01*dlat or abs(lonvec[0]-lon1) > 0.01*dlon or abs(lonvec[len(lonvec)-1]-lon2) > 0.01*dlon): raise ValueError("Coordinate endpoints in "+file+" do not match first file") # Keep track of the earliest and latest times if 'time:calendar' in attributes[file]: t = netCDF3.num2date(tvec[0],units=tvec.units,calendar=tvec.calendar) else: t = netCDF3.num2date(tvec[0],units=tvec.units) if t < t1: t1 = t if t > t2: t2 = t # b is 1 unless there is a missing value, where it is 0 fillValue = ncobj.variables[varname]._FillValue b = numpy.where(a == fillValue, 0, 1) n = n+b mean = mean+b*a nh.nc3_close(ncobj) # Before computing mean, set all values with 0 count to _FillValue, and # the corresponding count to 1. This avoids division by zero and leaves # those pixels set to _FillValue. mean = numpy.where(n==0, fillValue, mean) n = numpy.where(n==0, 1, n) mean = mean/n nh.nc3_set_var(ncout,varname) nh.nc3_add_time(ncout,[t1]) nh.nc3_add_data(ncout,varname,mean) attr = {} attr['history'] = '\n'.join(history) attr[varname+':_FillValue'] = fillValue if units: attr[varname+':units'] = units nh.nc3_set_attributes(ncout,attr) nh.nc3_close(ncout) return attributes
def flttonc(fltstem, ncfile, varname, yyyymmdd, units=None): """Main function to process a binary flt and hdr with name FLTSTEM to netCDF file NCFILE with variable name VARNAME. Time is set to the date corresponding to YYYYMMDD. Units can be provided optionally. """ d = datetime.strptime(yyyymmdd, "%Y%M%d") # Read the metadata from the header meta = {} r = re.compile("^(\S+)\s+(\S+)$") f = open(fltstem + '.hdr', 'r') for line in f: line = line.strip() m = re.match(r, line) if m: if m.group(1).lower() == 'byteorder': meta[m.group(1).lower()] = m.group(2) else: meta[m.group(1).lower()] = float(m.group(2)) f.close() # Get the data and shape it appropriately a = np.fromfile(fltstem + '.flt', dtype=np.float32) a.reshape(int(meta['ncols']), int(meta['nrows']), 1) # Make the lon and lat coordinate variables if 'xllcenter' in meta: lon = nr.create_vector(meta['xllcenter'], \ meta['ncols'], \ meta['cellsize']) else: lon = nr.create_vector(meta['xllcorner']+0.5*meta['cellsize'], \ meta['ncols'], \ meta['cellsize']) if 'yllcenter' in meta: lat = nr.create_vector(meta['yllcenter'], \ meta['nrows'], \ meta['cellsize']) else: lat = nr.create_vector(meta['yllcorner']+0.5*meta['cellsize'], \ meta['nrows'], \ meta['cellsize']) # Reverse the latitude elements so they run from North to South lat = lat[::-1] # Make a :history global attribute history = [] now = datetime.now() history.append( "flttonc at %04d-%02d-%02dT%02d:%02d:%02d" % (now.year, now.month, now.day, now.hour, now.minute, now.second)) history.append("Input file: " + fltstem + ".flt") attr = {} if 'nodata_value' in meta: attr[varname + ':_FillValue'] = meta['nodata_value'] if units: attr[varname + ':units'] = units attr['history'] = '\n'.join(history) # Write the netCDF file ncobj = nh.nc3_open(ncfile, 'w') nh.nc3_set_timelatlon(ncobj, 1, len(lat), len(lon)) nh.nc3_set_var(ncobj, varname) nh.nc3_set_var(ncobj, 'wgs84', dims=()) # Grid mapping container nh.nc3_add_time(ncobj, [d]) nh.nc3_add_data(ncobj, 'latitude', lat) nh.nc3_add_data(ncobj, 'longitude', lon) nh.nc3_add_data(ncobj, varname, a) nh.nc3_set_attributes(ncobj, attr) nh.nc3_close(ncobj) return ncfile