示例#1
0
def ncaverage(varname, files, outname):
    """
    Create a netCDF file OUTNAME by averaging the lon/lat grids stored in
    VARNAME in the input files FILES.
    Return a dictionary of the attributes from the input files
    so that the caller can create addiitonal application-specific metadata
    to insert into the output file.

    Checks are performed to make sure that input units are either consistent
    or non-existent, and that the input grid shapes are the same, and the
    endpoints of the coordinate variables match.
    """
    history = []
    now = datetime.now()
    history.append(
        "ncaverage at %04d-%02d-%02dT%02d:%02d:%02d, files..." %
        (now.year, now.month, now.day, now.hour, now.minute, now.second))
    units = None
    attributes = dict()
    first = 1
    for file in files:

        print "File: " + file
        history.append(" + " + file)
        ncobj = nh.nc3_open(file, 'r')
        attr = nh.nc3_get_attributes(ncobj)
        if varname + ':units' in attr:
            newunit = attr[varname + ':units']
            if units:
                if newunit != units:
                    raise ValueError("Inconsistent units (" + newunits +
                                     ") on var " + varname + " in file " +
                                     file + ", previous files were (" + units +
                                     ")")
            units = newunit
        attributes[file] = attr

        # We need these for the first file and then on each subsequent file
        a = ncobj.variables[varname]
        latvec = ncobj.variables['latitude']
        lonvec = ncobj.variables['longitude']
        tvec = ncobj.variables['time']

        # The first file is used to 1) set the shape of the output
        # 2) capture some basic metadata used in writing the output file,
        # which we start to write here while we've got access to the
        # dimension variables.
        if first:
            first = 0
            mean = numpy.zeros(a.shape, dtype=numpy.float32)
            n = numpy.zeros(a.shape, dtype=numpy.int32)
            if 'time:calendar' in attributes[file]:
                t1 = netCDF3.num2date(tvec[0],
                                      units=tvec.units,
                                      calendar=tvec.calendar)
            else:
                t1 = netCDF3.num2date(tvec[0], units=tvec.units)
            t2 = t1
            ncout = nh.nc3_open(outname, 'w')
            nh.nc3_set_timelatlon(ncout, 1, len(latvec), len(lonvec))
            nh.nc3_add_data(ncout, 'latitude', latvec)
            nh.nc3_add_data(ncout, 'longitude', lonvec)
            # remember the endpoints of each CV and compute the step size for each
            lat1 = latvec[0]
            lat2 = latvec[len(latvec) - 1]
            lon1 = lonvec[0]
            lon2 = lonvec[len(lonvec) - 1]
            dlat = abs(lat2 - lat1) / len(latvec)
            dlon = abs(lon2 - lon1) / len(lonvec)

        # Guard against incompatible files - shapes must be the
        # same AND CV endpoint values must match within a fraction of a pixel.
        if a.shape != mean.shape:
            raise ValueError("Data in " + file +
                             " is a different shape from first file")
        if (abs(latvec[0] - lat1) > 0.01 * dlat
                or abs(latvec[len(latvec) - 1] - lat2) > 0.01 * dlat
                or abs(lonvec[0] - lon1) > 0.01 * dlon
                or abs(lonvec[len(lonvec) - 1] - lon2) > 0.01 * dlon):
            raise ValueError("Coordinate endpoints in " + file +
                             " do not match first file")

        # Keep track of the earliest and latest times
        if 'time:calendar' in attributes[file]:
            t = netCDF3.num2date(tvec[0],
                                 units=tvec.units,
                                 calendar=tvec.calendar)
        else:
            t = netCDF3.num2date(tvec[0], units=tvec.units)
        if t < t1: t1 = t
        if t > t2: t2 = t

        # b is 1 unless there is a missing value, where it is 0
        fillValue = ncobj.variables[varname]._FillValue
        b = numpy.where(a == fillValue, 0, 1)
        n = n + b
        mean = mean + b * a
        nh.nc3_close(ncobj)

    # Before computing mean, set all values with 0 count to _FillValue, and
    # the corresponding count to 1.  This avoids division by zero and leaves
    # those pixels set to _FillValue.
    mean = numpy.where(n == 0, fillValue, mean)
    n = numpy.where(n == 0, 1, n)
    mean = mean / n

    nh.nc3_set_var(ncout, varname)
    nh.nc3_add_time(ncout, [t1])
    nh.nc3_add_data(ncout, varname, mean)
    attr = {}
    attr['history'] = '\n'.join(history)
    attr[varname + ':_FillValue'] = fillValue
    if units:
        attr[varname + ':units'] = units
    nh.nc3_set_attributes(ncout, attr)
    nh.nc3_close(ncout)
    return attributes
示例#2
0
def ncaverage(varname, files, outname):
    """
    Create a netCDF file OUTNAME by averaging the lon/lat grids stored in
    VARNAME in the input files FILES.
    Return a dictionary of the attributes from the input files
    so that the caller can create addiitonal application-specific metadata
    to insert into the output file.

    Checks are performed to make sure that input units are either consistent
    or non-existent, and that the input grid shapes are the same, and the
    endpoints of the coordinate variables match.
    """
    history = []
    now = datetime.now()
    history.append("ncaverage at %04d-%02d-%02dT%02d:%02d:%02d, files..." % (now.year, now.month, now.day, now.hour, now.minute, now.second))
    units = None
    attributes = dict();
    first = 1
    for file in files:

        print "File: "+file
        history.append(" + "+file)
        ncobj = nh.nc3_open(file,'r')
        attr = nh.nc3_get_attributes(ncobj)
        if varname+':units' in attr:
            newunit = attr[varname+':units']
            if units:
                if newunit != units:
                    raise ValueError("Inconsistent units ("+newunits+") on var "+varname+" in file "+file+", previous files were ("+units+")")
            units = newunit
        attributes[file] = attr

        # We need these for the first file and then on each subsequent file
        a = ncobj.variables[varname]
        latvec = ncobj.variables['latitude']
        lonvec = ncobj.variables['longitude']
        tvec = ncobj.variables['time']

        # The first file is used to 1) set the shape of the output
        # 2) capture some basic metadata used in writing the output file,
        # which we start to write here while we've got access to the 
        # dimension variables.
        if first:
            first = 0
            mean = numpy.zeros(a.shape, dtype=numpy.float32)
            n = numpy.zeros(a.shape, dtype=numpy.int32)
            if 'time:calendar' in attributes[file]:
                t1 = netCDF3.num2date(tvec[0],units=tvec.units,calendar=tvec.calendar)
            else:
                t1 = netCDF3.num2date(tvec[0],units=tvec.units)
            t2 = t1
            ncout = nh.nc3_open(outname,'w')
            nh.nc3_set_timelatlon(ncout,1,len(latvec),len(lonvec))
            nh.nc3_add_data(ncout,'latitude',latvec)
            nh.nc3_add_data(ncout,'longitude',lonvec)
            # remember the endpoints of each CV and compute the step size for each
            lat1 = latvec[0]
            lat2 = latvec[len(latvec)-1]
            lon1 = lonvec[0]
            lon2 = lonvec[len(lonvec)-1]
            dlat = abs(lat2-lat1)/len(latvec)
            dlon = abs(lon2-lon1)/len(lonvec)
       

        # Guard against incompatible files - shapes must be the 
        # same AND CV endpoint values must match within a fraction of a pixel.
        if a.shape != mean.shape:
            raise ValueError("Data in "+file+" is a different shape from first file")
        if (abs(latvec[0]-lat1) > 0.01*dlat or
            abs(latvec[len(latvec)-1]-lat2) > 0.01*dlat or
            abs(lonvec[0]-lon1) > 0.01*dlon or
            abs(lonvec[len(lonvec)-1]-lon2) > 0.01*dlon):
            raise ValueError("Coordinate endpoints in "+file+" do not match first file")

        # Keep track of the earliest and latest times
        if 'time:calendar' in attributes[file]:
            t = netCDF3.num2date(tvec[0],units=tvec.units,calendar=tvec.calendar)
        else:
            t = netCDF3.num2date(tvec[0],units=tvec.units)
        if t < t1: t1 = t
        if t > t2: t2 = t

        # b is 1 unless there is a missing value, where it is 0
        fillValue = ncobj.variables[varname]._FillValue
        b = numpy.where(a == fillValue, 0, 1)
        n = n+b
        mean = mean+b*a
        nh.nc3_close(ncobj)

    # Before computing mean, set all values with 0 count to _FillValue, and
    # the corresponding count to 1.  This avoids division by zero and leaves 
    # those pixels set to _FillValue.
    mean = numpy.where(n==0, fillValue, mean)
    n = numpy.where(n==0, 1, n)
    mean = mean/n

    nh.nc3_set_var(ncout,varname)
    nh.nc3_add_time(ncout,[t1])
    nh.nc3_add_data(ncout,varname,mean)
    attr = {}
    attr['history'] = '\n'.join(history)
    attr[varname+':_FillValue'] = fillValue
    if units:
        attr[varname+':units'] = units
    nh.nc3_set_attributes(ncout,attr)
    nh.nc3_close(ncout)
    return attributes
示例#3
0
def nctoflt(ncfile, fltstem, varname, iz=0):
    """Main function to process a netCDF file to binary flt

       Output files have the stem name and suffix .flt and .hdr

       If varname is 3D, then iz is the index of the first dimension used
       to extract a 2D slice.

       If the latitude runs south to north, then the grid is flipped before
       being written
    """

    ncobj = nh.nc3_open(ncfile,'r')
    a = ncobj.variables[varname]
    # Copy out into a numpy array and make sure we have only
    # 2 dimensions and type float32.
    b = numpy.float32(ncobj.variables[varname])
    if len(b.shape) < 2 or len(b.shape) > 3:
        raise ValueError("Only 2D and 3D data allowed (not "+len(b.shape)+"D)")
    if len(b.shape) == 3:
        b = numpy.float32(b[iz,::,::].reshape(b.shape[1], b.shape[2]))
    fillValue = numpy.float32(ncobj.variables[varname]._FillValue)

    latvec = ncobj.variables['latitude']
    lonvec = ncobj.variables['longitude']
    lat1 = latvec[0]
    lat2 = latvec[len(latvec)-1]
    # Reverse if latitude runs South to North
    if lat1 < lat2:
        x = lat2
        lat2 = lat1
        lat1 = x
        b = b[::-1,]
    lon1 = lonvec[0]
    lon2 = lonvec[len(lonvec)-1]
    
    dlat = abs(lat1-lat2)/(len(latvec)-1)
    dlon = abs(lon2-lon1)/(len(lonvec)-1)
    xll = lon1-dlon*0.5
    yll = lat2-dlat*0.5


    fltname = fltstem+'.flt'
    if os.path.exists(fltname): os.unlink(fltname)
    b.tofile(fltname)
    f = file(fltstem+".hdr","w")
    
    f.write("ncols %d\n" % b.shape[1])
    f.write("nrows %d\n" % b.shape[0])
    f.write("xllcorner %f\n" % xll)
    f.write("yllcorner %f\n" % yll)
    f.write("cellsize %f\n" % dlon)
    f.write("NODATA_value %f\n" % fillValue)
    if sys.byteorder == "little":
        f.write("byteorder LSBFIRST\n")
    else:
        f.write("byteorder LSBLAST\n")
    f.close()
    attr = nh.nc3_get_attributes(ncobj)
    nh.nc3_close(ncobj)
    return attr
示例#4
0
def flttonc(fltstem, ncfile, varname, yyyymmdd, units=None):
    """Main function to process a binary flt and hdr with name FLTSTEM
       to netCDF file NCFILE with variable name VARNAME.  Time is set to
       the date corresponding to YYYYMMDD.  Units can be provided optionally.
    """

    d = datetime.strptime(yyyymmdd, "%Y%M%d")

    # Read the metadata from the header
    meta = {}
    r = re.compile("^(\S+)\s+(\S+)$")
    f = open(fltstem+'.hdr','r')
    for line in f:
        line=line.strip()
        m = re.match(r, line)
        if m:
            if m.group(1).lower() == 'byteorder':
                meta[m.group(1).lower()] = m.group(2)
            else:
                meta[m.group(1).lower()] = float(m.group(2))
    f.close()

    # Get the data and shape it appropriately
    a=np.fromfile(fltstem+'.flt',dtype=np.float32)
    a.reshape(int(meta['ncols']), int(meta['nrows']), 1)

    # Make the lon and lat coordinate variables
    if 'xllcenter' in meta:
        lon = nr.create_vector(meta['xllcenter'], \
                               meta['ncols'], \
                               meta['cellsize'])
    else:
        lon = nr.create_vector(meta['xllcorner']+0.5*meta['cellsize'], \
                               meta['ncols'], \
                               meta['cellsize'])
    if 'yllcenter' in meta:
        lat = nr.create_vector(meta['yllcenter'], \
                               meta['nrows'], \
                               meta['cellsize'])
    else:
        lat = nr.create_vector(meta['yllcorner']+0.5*meta['cellsize'], \
                               meta['nrows'], \
                               meta['cellsize'])
    # Reverse the latitude elements so they run from North to South
    lat = lat[::-1] 

    # Make a :history global attribute
    history = []
    now = datetime.now()
    history.append("flttonc at %04d-%02d-%02dT%02d:%02d:%02d" % (now.year, now.month, now.day, now.hour, now.minute, now.second))
    history.append("Input file: "+fltstem+".flt")

    attr = {}
    if 'nodata_value' in meta:
        attr[varname+':_FillValue'] = meta['nodata_value']
    if units:
        attr[varname+':units'] = units
    attr['history'] = '\n'.join(history)

    # Write the netCDF file
    ncobj = nh.nc3_open(ncfile,'w')
    nh.nc3_set_timelatlon(ncobj,1,len(lat),len(lon))
    nh.nc3_set_var(ncobj,varname)
    nh.nc3_set_var(ncobj,'wgs84',dims=())  # Grid mapping container
    nh.nc3_add_time(ncobj,[d])
    nh.nc3_add_data(ncobj,'latitude',lat)
    nh.nc3_add_data(ncobj,'longitude',lon)
    nh.nc3_add_data(ncobj,varname,a)
    nh.nc3_set_attributes(ncobj,attr)
    nh.nc3_close(ncobj)
    return ncfile
示例#5
0
def flttonc(fltstem, ncfile, varname, yyyymmdd, units=None):
    """Main function to process a binary flt and hdr with name FLTSTEM
       to netCDF file NCFILE with variable name VARNAME.  Time is set to
       the date corresponding to YYYYMMDD.  Units can be provided optionally.
    """

    d = datetime.strptime(yyyymmdd, "%Y%M%d")

    # Read the metadata from the header
    meta = {}
    r = re.compile("^(\S+)\s+(\S+)$")
    f = open(fltstem + '.hdr', 'r')
    for line in f:
        line = line.strip()
        m = re.match(r, line)
        if m:
            if m.group(1).lower() == 'byteorder':
                meta[m.group(1).lower()] = m.group(2)
            else:
                meta[m.group(1).lower()] = float(m.group(2))
    f.close()

    # Get the data and shape it appropriately
    a = np.fromfile(fltstem + '.flt', dtype=np.float32)
    a.reshape(int(meta['ncols']), int(meta['nrows']), 1)

    # Make the lon and lat coordinate variables
    if 'xllcenter' in meta:
        lon = nr.create_vector(meta['xllcenter'], \
                               meta['ncols'], \
                               meta['cellsize'])
    else:
        lon = nr.create_vector(meta['xllcorner']+0.5*meta['cellsize'], \
                               meta['ncols'], \
                               meta['cellsize'])
    if 'yllcenter' in meta:
        lat = nr.create_vector(meta['yllcenter'], \
                               meta['nrows'], \
                               meta['cellsize'])
    else:
        lat = nr.create_vector(meta['yllcorner']+0.5*meta['cellsize'], \
                               meta['nrows'], \
                               meta['cellsize'])
    # Reverse the latitude elements so they run from North to South
    lat = lat[::-1]

    # Make a :history global attribute
    history = []
    now = datetime.now()
    history.append(
        "flttonc at %04d-%02d-%02dT%02d:%02d:%02d" %
        (now.year, now.month, now.day, now.hour, now.minute, now.second))
    history.append("Input file: " + fltstem + ".flt")

    attr = {}
    if 'nodata_value' in meta:
        attr[varname + ':_FillValue'] = meta['nodata_value']
    if units:
        attr[varname + ':units'] = units
    attr['history'] = '\n'.join(history)

    # Write the netCDF file
    ncobj = nh.nc3_open(ncfile, 'w')
    nh.nc3_set_timelatlon(ncobj, 1, len(lat), len(lon))
    nh.nc3_set_var(ncobj, varname)
    nh.nc3_set_var(ncobj, 'wgs84', dims=())  # Grid mapping container
    nh.nc3_add_time(ncobj, [d])
    nh.nc3_add_data(ncobj, 'latitude', lat)
    nh.nc3_add_data(ncobj, 'longitude', lon)
    nh.nc3_add_data(ncobj, varname, a)
    nh.nc3_set_attributes(ncobj, attr)
    nh.nc3_close(ncobj)
    return ncfile
示例#6
0
def nctoflt(ncfile, fltstem, varname, iz=0):
    """Main function to process a netCDF file to binary flt

       Output files have the stem name and suffix .flt and .hdr

       If varname is 3D, then iz is the index of the first dimension used
       to extract a 2D slice.

       If the latitude runs south to north, then the grid is flipped before
       being written
    """

    ncobj = nh.nc3_open(ncfile, 'r')
    a = ncobj.variables[varname]
    # Copy out into a numpy array and make sure we have only
    # 2 dimensions and type float32.
    b = numpy.float32(ncobj.variables[varname])
    if len(b.shape) < 2 or len(b.shape) > 3:
        raise ValueError("Only 2D and 3D data allowed (not " + len(b.shape) +
                         "D)")
    if len(b.shape) == 3:
        b = numpy.float32(b[iz, ::, ::].reshape(b.shape[1], b.shape[2]))
    fillValue = numpy.float32(ncobj.variables[varname]._FillValue)

    latvec = ncobj.variables['latitude']
    lonvec = ncobj.variables['longitude']
    lat1 = latvec[0]
    lat2 = latvec[len(latvec) - 1]
    # Reverse if latitude runs South to North
    if lat1 < lat2:
        x = lat2
        lat2 = lat1
        lat1 = x
        b = b[::-1, ]
    lon1 = lonvec[0]
    lon2 = lonvec[len(lonvec) - 1]

    dlat = abs(lat1 - lat2) / (len(latvec) - 1)
    dlon = abs(lon2 - lon1) / (len(lonvec) - 1)
    xll = lon1 - dlon * 0.5
    yll = lat2 - dlat * 0.5

    fltname = fltstem + '.flt'
    if os.path.exists(fltname): os.unlink(fltname)
    b.tofile(fltname)
    f = file(fltstem + ".hdr", "w")

    f.write("ncols %d\n" % b.shape[1])
    f.write("nrows %d\n" % b.shape[0])
    f.write("xllcorner %f\n" % xll)
    f.write("yllcorner %f\n" % yll)
    f.write("cellsize %f\n" % dlon)
    f.write("NODATA_value %f\n" % fillValue)
    if sys.byteorder == "little":
        f.write("byteorder LSBFIRST\n")
    else:
        f.write("byteorder LSBLAST\n")
    f.close()
    attr = nh.nc3_get_attributes(ncobj)
    nh.nc3_close(ncobj)
    return attr