Exemplo n.º 1
0
    def get_variable_names(self, filenames, data_type=None):
        try:
            from pyhdf.SD import SD
            from pyhdf.HDF import HDF
        except ImportError:
            raise ImportError("HDF support was not installed, please reinstall with pyhdf to read HDF files.")

        valid_variables = set([])
        for filename in filenames:
            # Do VD variables
            datafile = HDF(filename)
            vdata = datafile.vstart()
            variables = vdata.vdatainfo()
            # Assumes that latitude shape == longitude shape (it should):
            # dim_length = [var[3] for var in variables if var[0] == 'Latitude'][0]
            for var in variables:
                # if var[3] == dim_length:
                valid_variables.add(var[0])

            # Do SD variables:
            sd = SD(filename)
            datasets = sd.datasets()
            # if 'Height' in datasets:
            #     valid_shape = datasets['Height'][1]
            for var in datasets:
                    # if datasets[var][1] == valid_shape:
                valid_variables.add(var)

        return valid_variables
Exemplo n.º 2
0
def read_rrc(inpath):
    '''Read rrc data m*n from hdf file'''

    '''b1-5;b13-16 for MODIS Rrc
        Rrc_1238 Rrc_443-862 ozone senz solz for VIIRS rrc   
    '''  
    hdf = SD(inpath, SDC.READ)
    #dts = sorted(hdf.datasets().keys())
    modis_key = ['CorrRefl_01','CorrRefl_02','CorrRefl_03','CorrRefl_04','CorrRefl_05',
                 'CorrRefl_13','CorrRefl_14','CorrRefl_15','CorrRefl_16']
    viirs_key = ['Rrc_443','Rrc_486','Rrc_551','Rrc_671','Rrc_745','Rrc_862','Rrc_1238']
    mission = os.path.basename(inpath)[0]
    if mission =='A' or mission =='T':keys = modis_key
    elif mission=='V':keys = viirs_key
    else:keys = hdf.datasets().keys()
    for i,dt in enumerate(keys):
        print(i,dt)
        band = hdf.select(dt)[:,:]        
        if i==0:             
            limit = (band.shape[0],band.shape[1],len(keys))            
            rrc = np.zeros(limit,dtype = np.float)
            rrc[:,:,i] = band
        else:
            rrc[:,:,i] = band
    hdf.end()
    print(rrc.shape)
    return rrc
Exemplo n.º 3
0
def run(FILE_NAME):

    DATAFIELD_NAME = 'dHat'

    if USE_NETCDF4:
        from netCDF4 import Dataset    
        nc = Dataset(FILE_NAME)
        var = nc.variables[DATAFIELD_NAME]
        # This datafield has scale factor and add offset attributes, but no
        # fill value.  We'll turn off automatic scaling and do it ourselves.
        var.set_auto_maskandscale(False)
        data = nc.variables[DATAFIELD_NAME][:].astype(np.float64)

        # Retrieve scale/offset attributes.
        scale_factor = var.scale_factor
        add_offset = var.add_offset
    
        # Retrieve the geolocation data.
        latitude = nc.variables['geolocation'][:,:,0]
        longitude = nc.variables['geolocation'][:,:,1]
    else:
        from pyhdf.SD import SD, SDC
        hdf = SD(FILE_NAME, SDC.READ)
        
        ds = hdf.select(DATAFIELD_NAME)
        data = ds[:,:].astype(np.double)

        # Handle scale/osffset attributes.
        attrs = ds.attributes(full=1)
        sfa=attrs["scale_factor"]
        scale_factor = sfa[0]
        aoa=attrs["add_offset"]
        add_offset = aoa[0]

        # Retrieve the geolocation data.        
        geo = hdf.select('geolocation')
        latitude = geo[:,:,0]
        longitude = geo[:,:,1]

    data = data / scale_factor + add_offset
    
    # Draw an equidistant cylindrical projection using the high resolution
    # coastline database.
    m = Basemap(projection='cyl', resolution='h',
                llcrnrlat=30, urcrnrlat = 36,
                llcrnrlon=121, urcrnrlon = 133)
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(30, 37), labels=[1, 0, 0, 0])
    m.drawmeridians(np.arange(121, 133, 2), labels=[0, 0, 0, 1])
    m.pcolormesh(longitude, latitude, data, latlon=True)
    cb = m.colorbar()
    cb.set_label('Unit:mm')

    basename = os.path.basename(FILE_NAME)
    plt.title('{0}\n{1}'.format(basename, DATAFIELD_NAME))
    fig = plt.gcf()
    # plt.show()
    
    pngfile = "{0}.py.png".format(basename)
    fig.savefig(pngfile)
Exemplo n.º 4
0
 def load(self, fldname, **kwargs):
     """ Load Cali Current fields for a given day"""
     self._timeparams(**kwargs)
     
     if fldname == 'chl':
         filename = "/C%04i%03i_chl_mapped.hdf" % (self.yr, self.yd)
         #ncfieldname = 'chl_%04i_%03i' % (yr,yd)
         def scale(PV): return 10**(PV*0.015-2)
     elif fldname == 'sst':
         filename = "/M%04i%03i_sst_mapped.hdf" % (self.yr, self.yd)
         #ncfieldname = 'sst_%04i_%03i' % (yr,yd)            
         def scale(PV): return PV*0.15000001-3
     if not os.path.isfile(self.datadir + filename):
         print "Downloading " + filename
         self.download(fldname, self.jd)
         
     h = SD(self.datadir + filename,SDC.READ)        
     ncfieldname = h.datasets().keys()[0]
     fld =  h.select(ncfieldname)
     attr = fld.attributes()
     PV = fld[:].astype(np.float)
     PV[PV<0] = PV[PV<0]+256
     PV[PV==0]   = np.nan
     PV[PV==255] = np.nan
     setattr(self, fldname, scale(PV)[self.j1:self.j2, self.i1:self.i2])
Exemplo n.º 5
0
def main():

    varname_to_rpn_name = {
        "precipitation": "PR",
        "relativeError": "RERR"
    }

    varnames = list(varname_to_rpn_name.keys())

    target_dir = "/skynet3_rech1/huziy/from_hdf4"
    source_dir = "/st1_fs2/winger/Validation/TRMM/HDF_format"

    for f_name in os.listdir(source_dir):
        if not f_name.endswith("HDF"):
            continue

        path = os.path.join(source_dir, f_name)
        ds = SD(path)
        print(ds.datasets())
        target_path = os.path.join(target_dir, f_name + ".rpn")
        r_obj = RPN(target_path, mode="w")
        for varname in varnames:
            var_data = ds.select(varname)[0, :, :]
            r_obj.write_2D_field(
                name=varname_to_rpn_name[varname],
                data=var_data, label=varname, grid_type="L",
                ig = [25, 25, 4013, 18012])
        r_obj.close()
Exemplo n.º 6
0
def main(cal_file, with_cp):

    from pyhdf.SD import SD

    if with_cp:
        cmd = 'cp %s /home/noel/scratch/' % (cal_file)
        print "running "+cmd
        os.system(cmd)
        filename = os.path.basename(cal_file)
        cal_file = '/home/noel/scratch/' + filename
                        
    print 'Reading ' + cal_file 
    
    vars = ['Latitude', 'Longitude', 
            'Total_Attenuated_Backscatter_532', 'Attenuated_Backscatter_1064', 'Perpendicular_Attenuated_Backscatter_532',
            'Pressure', 'Temperature', 'Molecular_Number_Density', 'Tropopause_Height', 'Surface_Elevation']
    
    hdf = SD(cal_file)
    for var in vars:
        print 'Reading ' + var
        hdf_var = hdf.select(var)
        data = hdf_var.get()
        hdf_var.endaccess()
    hdf.end()
    
    print 'ok.'
    if with_cp:
        print 'Removing '+filename
        cmd = 'rm -f /home/noel/scratch/' + filename
        os.system(cmd)
Exemplo n.º 7
0
def rainfall_anunal_car(year):

    file = glob.glob('/Users/yuewang/Documents/DATA/atl/ATL_3B42V7_rain_accum.'+ str(year)+'*')

    rainfall_0 = []
    for i in file:
        atl =SD(i,SDC.READ)
        rainfall = atl.select('RAIN_TOTAL')
        rainfall_value = rainfall.get()
        rainfall_0.append(rainfall_value)
    
    rainfall_single = np.array(rainfall_0)
    rainfall_anunal = sum(rainfall_single)
    rainfall_anunal_car = rainfall_anunal[238:286,372:476]
    
# calculation none-zone mean value    
    ind = np.where(rainfall_anunal_car != 0)
    rf_annual = []
    for i,j in zip(*ind):
        mm = rainfall_anunal_car[i,j]
        rf_annual.append(mm)
    rf_annual = np.array(rf_annual)
    
    d = np.mean(rf_annual)
    
    return d
Exemplo n.º 8
0
def rainfall_anunal_GMX(year):

    file = glob.glob('/Users/yuewang/Documents/DATA/atl/ATL_3B42V7_rain_accum.'+ str(year)+'*')

    rainfall_0 = []
    for i in file:
        atl =SD(i,SDC.READ)
        rainfall = atl.select('RAIN_TOTAL')
        rainfall_value = rainfall.get()
        rainfall_0.append(rainfall_value)
    
    rainfall_single = np.array(rainfall_0)
    rainfall_anunal = sum(rainfall_single)
    rainfall_anunal_GMX = rainfall_anunal[280:320,340:400]
    
    ind = np.where(rainfall_anunal_GMX != 0)
    rf_annual = []
    for i,j in zip(*ind):
        mm = rainfall_anunal_GMX[i,j]
        rf_annual.append(mm)
    rf_annual = np.array(rf_annual)
    
    c = np.mean(rf_annual)
    
    return c
Exemplo n.º 9
0
    def test_1000m_to_250m(self):
        """Test the 1 km to 250 meter interpolation facility."""
        # gfilename = \
        #      "/san1/test/data/modis/MOD03_A12278_113638_2012278145123.hdf"
        gfilename = "/local_disk/src/python-geotiepoints/tests/MOD03_A12278_113638_2012278145123.hdf"
        # result_filename = \
        #      "/san1/test/data/modis/250m_lonlat_results.npz"
        result_filename = "/local_disk/src/python-geotiepoints/tests/250m_lonlat_results.npz"

        from pyhdf.SD import SD
        from pyhdf.error import HDF4Error

        try:
            gdata = SD(gfilename)
        except HDF4Error:
            print("Failed reading eos-hdf file %s" % gfilename)
            return

        lats = gdata.select("Latitude")[0:50, :]
        lons = gdata.select("Longitude")[0:50, :]

        verif = np.load(result_filename)
        vlons = verif['lons']
        vlats = verif['lats']
        tlons, tlats = modis1kmto250m(lons, lats)

        self.assert_(np.allclose(tlons, vlons, atol=0.05))
        self.assert_(np.allclose(tlats, vlats, atol=0.05))
Exemplo n.º 10
0
    def test_1000m_to_250m(self):
        """test the 1 km to 250 meter interpolation facility
        """
        gfilename_hdf = "testdata/MOD03_A12278_113638_2012278145123.hdf"
        gfilename = "testdata/250m_lonlat_section_input.npz"
        result_filename = "testdata/250m_lonlat_section_result.npz"

        from pyhdf.SD import SD
        from pyhdf.error import HDF4Error
        
        gdata = None
        try:
            gdata = SD(gfilename_hdf)
        except HDF4Error:
            print "Failed reading eos-hdf file %s" % gfilename_hdf
            try:
                indata = np.load(gfilename)
            except IOError:
                return

        if gdata:
            lats = gdata.select("Latitude")[20:50, :]
            lons = gdata.select("Longitude")[20:50, :]
        else:
            lats = indata['lat'] / 1000.
            lons = indata['lon'] / 1000.

        verif = np.load(result_filename)
        vlons = verif['lon'] / 1000.
        vlats = verif['lat'] / 1000.
        tlons, tlats = modis1kmto250m(lons, lats)

        self.assert_(np.allclose(tlons, vlons, atol=0.05))
        self.assert_(np.allclose(tlats, vlats, atol=0.05))
Exemplo n.º 11
0
def export_multi_fluid_LFM(argv):
	if (len(argv) >= 2):
		input_filename = argv[0]
		output_filename = argv[1]
		print input_filename
		sd = SD(input_filename, SDC.READ)

		grid = get_corners(sd)

		timesteps = 0

		# step = 1640000

		for key in sd.datasets().keys():
			shift = key.find('time_step')
			if shift == 0:
				if len(argv) == 3:
					step = argv[2]
					if key == 'time_step_'+str(step):
						export_timestep(sd, output_filename, key, grid)
				else:
					export_timestep(sd, output_filename, key, grid)
				timesteps += 1

		print 'timesteps found in file:', timesteps


	else:
		print 'usage: python lfm_split.py input_multi_timestep_hdf output_filename_prefix step(optional)'
Exemplo n.º 12
0
def load_standard_lfm_hdf(filename):
	""" Load the standard formated hdf which we want to emulate"""
	f = SD(filename, SDC.READ)
	X_grid = f.select('X_grid')
	Y_grid = f.select('Y_grid')
	Z_grid = f.select('Z_grid')

	# x_grid is size nkp1,njp1,nip1
	(nkp1,njp1,nip1) = X_grid[:].shape
	# The LFM reader expects i to vary fastest, then j, then k
	# However, the LFM pre-converted files store positions with k varying fastest (column-major)
	# Recommend saving in column-major format. If it fails, we can always switch.

	
	# i = 0; j = 0; k = 0
	# print 'printing standard first row'
	# for i in range(nip1):
	# 	print X_grid[k,j,i]/R_e

	# print 'printing j sweep'
	# i = 0; j = 0; k = 0;
	# for j in range(njp1):
	# 	print X_grid[k,j,i]/R_e

	# print 'printing k sweep'
	# i = 0; j = 0; k = 0;
	# for k in range(nkp1):
	# 	print X_grid[k,j,i]/R_e


	print 'standard nip1,njp1,nkp1 =', nip1,njp1,nkp1
	ni = nip1-1
	nj = njp1-1
	nk = nkp1-1
	print 'standard ni,nj,nk =', ni,nj,nk
Exemplo n.º 13
0
def write_interpolated(filename, f0, f1, fact, datasets):
    '''
    interpolate two hdf files f0 and f1 using factor fact, and
    write the result to filename
    '''

    hdf = SD(filename, SDC.WRITE|SDC.CREATE)
    for dataset in datasets:

        try:
            info = SD(f0).select(dataset).info()
        except:
            print >> stderr, 'Error loading %s in %s' % (dataset, f0)
            raise

        typ  = info[3]
        shp  = info[2]
        met0 = SD(f0).select(dataset).get()
        met1 = SD(f1).select(dataset).get()

        interp = (1-fact)*met0 + fact*met1

        interp = interp.astype({
                SDC.INT16: 'int16',
                SDC.FLOAT32: 'float32',
                SDC.FLOAT64: 'float64',
            }[typ])

        # write
        sds = hdf.create(dataset, typ, shp)
        sds[:] = interp[:]
        sds.endaccess()

    hdf.end()
def run(FILE_NAME):
    # Identify the data field.
    DATAFIELD_NAME = 'Longwave Flux (2.5R)'

    if USE_NETCDF4:
        from netCDF4 import Dataset
        nc = Dataset(FILE_NAME)
        data = nc.variables[DATAFIELD_NAME][:].astype(np.float64)
    else:
        from pyhdf.SD import SD, SDC
        hdf = SD(FILE_NAME, SDC.READ)
        
        # Read dataset.
        data2D = hdf.select(DATAFIELD_NAME)
        data = data2D[:,:]


    # Set fillvalue and units.
    # See "CERES Data Management System ES-4 Collection Guide" [1] and a sample
    # image by NASA [2] for details.  The fillvalue is 3.4028235E38.  Here, we
    # just use the max of the data.
    fillvalue = np.max(data)
    data[data == fillvalue] = np.nan
    datam = np.ma.masked_array(data, mask=np.isnan(data))
    
    # Set fillvalue and units.
    # See "CERES Data Management System ES-4 Collection Guide" [1] and a
    # sample image by NASA [2] for details.
    # The fillvalue is 3.4028235E38. Here, we use max value from the dataset.
    units = 'Watts/Meter^2'
    ysize, xsize = data.shape
    xinc = 360.0 / xsize
    yinc = 180.0 / ysize
    x0, x1 = (-180, 180)
    y0, y1 = (-90, 90)
    longitude = np.linspace(x0 + xinc/2, x1 - xinc/2, xsize)
    latitude = np.linspace(y0 + yinc/2, y1 - yinc/2, ysize)
    
    # Flip the latitude to run from 90 to -90
    latitude = latitude[::-1]
    
    # The data is global, so render in a global projection.
    m = Basemap(projection='cyl', resolution='l',
                llcrnrlat=-90, urcrnrlat=90,
                llcrnrlon=-180, urcrnrlon=180)
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(-90.,90,45))
    m.drawmeridians(np.arange(-180.,180,45), labels=[True,False,False,True])
    m.pcolormesh(longitude, latitude, datam, latlon=True)
    cb = m.colorbar()

    cb.set_label(units)

    basename = os.path.basename(FILE_NAME)
    plt.title('{0}\n{1}'.format(basename, DATAFIELD_NAME))
    fig = plt.gcf()
    # plt.show()
    pngfile = "{0}.py.png".format(basename)
    fig.savefig(pngfile)
Exemplo n.º 15
0
 def __init__(self, filename, filename_info, filetype_info):
     super(HDF4FileHandler, self).__init__(filename, filename_info, filetype_info)
     self.file_content = {}
     file_handle = SD(self.filename, SDC.READ)
     self._collect_attrs('', file_handle.attributes())
     for k, v in file_handle.datasets().items():
         self.collect_metadata(k, file_handle.select(k))
     del file_handle
Exemplo n.º 16
0
def print_dataset_2A12(*arg):

	FILE_NAME=arg[0]+'1B01.'+arg[1]
	hdf = SD(FILE_NAME, SDC.READ)

	'List available SDS datasets'
	for ds in hdf.datasets():
		print ds
Exemplo n.º 17
0
def readhdf(filename, fieldname, ignoreE = True):
    try:
        hdf = SD(filename, SDC.READ)
        data = hdf.select(fieldname)[:].copy()
        hdf.end()
    except Exception, e:
        if not ignoreE:print e
        data = Dataset(filename)[fieldname][:].copy()
Exemplo n.º 18
0
    def setup_grid(self):
        """Setup necessary variables for grid """

        if not os.path.isfile(self.datadir + self.gridfile):
            urllib.urlretrieve(self.dataurl + self.gridfile,
                               self.datadir + self.gridfile)
        g = SD(self.datadir + self.gridfile, SDC.READ)
        self.llat = g.select('Latitude')[:]
        self.llon = g.select('Longitude')[:]
Exemplo n.º 19
0
def print_dataset_1C21(*arg):

	FILE_NAME=arg[0]+'1C21.'+arg[1]
	print FILE_NAME
	hdf = SD(FILE_NAME, SDC.READ)

	'List available SDS datasets'
	for ds in hdf.datasets():
		print ds
def run(FILE_NAME):

    DATAFIELD_NAME = 'Temperature_MW_A'
    if USE_NETCDF4:
        from netCDF4 import Dataset    
        nc = Dataset(FILE_NAME)

        # The variable has a fill value, 
        # so netCDF4 converts it to a float64 masked array for us.
        data = nc.variables[DATAFIELD_NAME][11,:,:]
        latitude = nc.variables['Latitude'][:]
        longitude = nc.variables['Longitude'][:]

    else:
        from pyhdf.SD import SD, SDC
        hdf = SD(FILE_NAME, SDC.READ)

        # List available SDS datasets.
        # print hdf.datasets()

        # Read dataset.
        data3D = hdf.select(DATAFIELD_NAME)
        data = data3D[11,:,:]

        # Read geolocation dataset.
        lat = hdf.select('Latitude')
        latitude = lat[:,:]
        lon = hdf.select('Longitude')
        longitude = lon[:,:]

        # Handle fill value.
        attrs = data3D.attributes(full=1)
        fillvalue=attrs["_FillValue"]

        # fillvalue[0] is the attribute value.
        fv = fillvalue[0]
        data[data == fv] = np.nan
        data = np.ma.masked_array(data, np.isnan(data))

    
    # Draw an equidistant cylindrical projection using the low resolution
    # coastline database.
    m = Basemap(projection='cyl', resolution='l',
                llcrnrlat=-90, urcrnrlat = 90,
                llcrnrlon=-180, urcrnrlon = 180)
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(-90., 120., 30.), labels=[1, 0, 0, 0])
    m.drawmeridians(np.arange(-180., 181., 45.), labels=[0, 0, 0, 1])
    m.pcolormesh(longitude, latitude, data, latlon=True, alpha=0.90)
    cb = m.colorbar()
    cb.set_label('Unit:K')
    basename = os.path.basename(FILE_NAME)
    plt.title('{0}\n {1} at TempPrsLvls=11'.format(basename, DATAFIELD_NAME))
    fig = plt.gcf()
    # plt.show()
    pngfile = "{0}.{1}.py.png".format(basename, DATAFIELD_NAME)
    fig.savefig(pngfile)
Exemplo n.º 21
0
def aod(date, path, orbit, block):

    from constant import BAND_GREEN
    file_aerosol = _MIL2ASAE_fname(date, path, orbit)
    f = SD(file_aerosol)
    tau0 = f.select('RegMeanSpectralOptDepth').get()[block-1 , : , :, BAND_GREEN]
    tau0[tau0 == -9999] = np.mean(tau0[tau0 != -9999])

    return tau0
def run(FILE_NAME):

    DATAFIELD_NAME = 'SurfaceTemperature'

    # The dataset is (6144 x 6400).  Subset it to be around than 1K x 1K
    # Otherwise, the plot will skip processing some regions.
    rows = slice(0, 6144, 6)
    cols = slice(0, 6400, 6)

    if USE_NETCDF4:    
        from netCDF4 import Dataset
        nc = Dataset(FILE_NAME)

        data = nc.variables[DATAFIELD_NAME][rows, cols]
    
        # Retrieve the geolocation data.
        latitude = nc.variables['Latitude'][rows, cols]
        longitude = nc.variables['Longitude'][rows, cols]
        
    else:
        from pyhdf.SD import SD, SDC
        hdf = SD(FILE_NAME, SDC.READ)

        # Read dataset.
        data2D = hdf.select(DATAFIELD_NAME)
        data = data2D[rows,cols]

        # Read geolocation dataset.
        lat = hdf.select('Latitude')
        latitude = lat[rows,cols]
        lon = hdf.select('Longitude')
        longitude = lon[rows,cols]
        

    # Apply the fill value.  The valid minimum is zero, although there's no
    # attribute.
    data[data < 0] = np.nan
    data = np.ma.masked_array(data, np.isnan(data))
    
    # Render the data in a lambert azimuthal equal area projection.
    m = Basemap(projection='nplaea', resolution='l',
                boundinglat=60, lon_0=43)
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(50, 90, 10), labels=[1, 0, 0, 1])
    m.drawmeridians(np.arange(-180, 180, 30))
    x, y = m(longitude, latitude)
    m.pcolormesh(x, y, data)
    cb = m.colorbar()
    cb.set_label('Unknown')

    basename = os.path.basename(FILE_NAME)
    plt.title('{0}\n{1}'.format(basename, DATAFIELD_NAME))
    fig = plt.gcf()
    # plt.show()
    pngfile = "{0}.py.png".format(basename)
    fig.savefig(pngfile)
Exemplo n.º 23
0
def open_file(filename):
    A = SD(filename)

    # retrieve data SDS
    d = A.datasets()
    sds_name = d.keys()[0]  # name of sds. Dictionary method.
    sds = A.select(sds_name)
    pin = A.attributes()

    return sds, pin
Exemplo n.º 24
0
 def landmask(self):
     if not hasattr(self, "_landmask"):
         filename = os.path.basename(self.landurl)
         if not os.path.isfile(self.datadir + filename):
             urllib.urlretrieve(self.dataurl + self.landurl,
                                self.datadir + filename)
         h = SD(self.datadir + filename, SDC.READ)        
         ncfieldname = h.datasets().keys()[0]
         self._landmask =   h.select(ncfieldname)[:] == -1
     return self._landmask
Exemplo n.º 25
0
def run(FILE_NAME):
    
    # Identify the data field.
    DATAFIELD_NAME = 'AlbedoLocal'

    hdf = SD(FILE_NAME, SDC.READ)

    # Read dataset.
    data4D = hdf.select(DATAFIELD_NAME)

    # Convert 4-D data to 2-D data by subsetting.
    SOMBlockDim = 50;
    NBandDim = 0;
    data = data4D[SOMBlockDim,:,:,NBandDim].astype(np.double)

    # Read geolocation dataset from HDF-EOS2 dumper output.
    GEO_FILE_NAME = 'lat_MISR_TC_ALBEDO_P223_F05_lvl50.output'
    GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], 
                                 GEO_FILE_NAME)
    lat = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0])
    lat = lat.reshape(data.shape)
    
    GEO_FILE_NAME = 'lon_MISR_TC_ALBEDO_P223_F05_lvl50.output'
    GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], 
                                 GEO_FILE_NAME)
    lon = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0])
    lon = lon.reshape(data.shape)
        
    # Read attributes.
    attrs = data4D.attributes(full=1)
    fva=attrs["_FillValue"]
    _FillValue = fva[0]

    # Apply the fill value.
    data[data == _FillValue] = np.nan
    datam = np.ma.masked_array(data, mask=np.isnan(data))


    # Set the limit for the plot.
    m = Basemap(projection='cyl', resolution='h',
                llcrnrlat=np.min(lat), urcrnrlat = np.max(lat),
                llcrnrlon=np.min(lon), urcrnrlon = np.max(lon))
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(np.floor(np.min(lat)), np.ceil(np.max(lat)), 1), labels=[1, 0, 0, 0])
    m.drawmeridians(np.arange(np.floor(np.min(lon)), np.ceil(np.max(lon)), 1), labels=[0, 0, 0, 1])
    m.pcolormesh(lon, lat, datam, latlon=True)
    cb = m.colorbar()
    cb.set_label('No Unit')

    basename = os.path.basename(FILE_NAME)
    plt.title('{0}\n{1} at SOMBlockDim=50 NBandDim=0'.format(basename, DATAFIELD_NAME))
    fig = plt.gcf()
    # plt.show()
    pngfile = "{0}.l50.py.png".format(basename)
    fig.savefig(pngfile)
Exemplo n.º 26
0
def read_var_ikslice(filename,var_name,i,k,tc,pc):
    hdffile = SD(filename,SDC.READ)
    ni = hdffile.select('X_grid').ni-1
    nj = hdffile.select('X_grid').nj-1

    if var_name not in ['br','btheta','bphi','vr','vtheta','vphi','bt','bp','vt','vp']:
        var=hdffile.select(var_name+'_').get(start=(k,0,i),count=(1,nj,1)).squeeze()
    else:
        if var_name in ['br','btheta','bphi','bt','bp']:
            bx=hdffile.select('bx_').get(start=(k,0,i),count=(1,nj,1)).squeeze()
            by=hdffile.select('by_').get(start=(k,0,i),count=(1,nj,1)).squeeze()
            bz=hdffile.select('bz_').get(start=(k,0,i),count=(1,nj,1)).squeeze()

            if var_name=='br':
                var     = bx*cos(pc[k])*sin(tc) + by*sin(pc[k])*sin(tc) + bz*cos(tc)
            elif (var_name=='btheta' or var_name=='bt'):
                var = bx*cos(pc[k])*cos(tc) + by*sin(pc[k])*cos(tc) - bz*sin(tc)
            else:
                var   =-bx*sin(pc[k])            + by*cos(pc[k])
        else:
            vx=hdffile.select('vx_').get(start=(k,0,i),count=(1,nj,1)).squeeze()
            vy=hdffile.select('vy_').get(start=(k,0,i),count=(1,nj,1)).squeeze()
            vz=hdffile.select('vz_').get(start=(k,0,i),count=(1,nj,1)).squeeze()

            if var_name=='vr':
                var    = vx*cos(pc[k])*sin(tc) + vy*sin(pc[k])*sin(tc) + vz*cos(tc)
            elif (var_name=='vtheta' or var_name=='vt'):
                var = vx*cos(pc[k])*cos(tc) + vy*sin(pc[k])*cos(tc) - vz*sin(tc)
            else:
                var   =-vx*sin(pc[k])            + vy*cos(pc[k])
    hdffile.end()
    return(var)
Exemplo n.º 27
0
def get_lat_lon_modis(satscene, options):
    """Read lat and lon.
    """
    filename_tmpl = satscene.time_slot.strftime(options["geofile"])
    file_list = glob.glob(os.path.join(options["dir"], filename_tmpl))

    if len(file_list) == 0:
        # Try in the same directory as the data
        data_dir = os.path.split(options["filename"])[0]
        file_list = glob.glob(os.path.join(data_dir, filename_tmpl))

    if len(file_list) > 1:
        logger.warning("More than 1 geolocation file matching!")
        filename = max(file_list, key=lambda x: os.stat(x).st_mtime)
        coarse_resolution = 1000
    elif len(file_list) == 0:
        logger.warning("No geolocation file matching " + filename_tmpl
                       + " in " + options["dir"])
        logger.debug("Using 5km geolocation and interpolating")
        filename = options["filename"]
        coarse_resolution = 5000
    else:
        filename = file_list[0]
        coarse_resolution = 1000

    logger.debug("Loading geolocation file: " + str(filename)
                 + " at resolution " + str(coarse_resolution))

    resolution = options["resolution"]

    data = SD(str(filename))
    lat = data.select("Latitude")
    fill_value = lat.attributes()["_FillValue"]
    lat = np.ma.masked_equal(lat.get(), fill_value)
    lon = data.select("Longitude")
    fill_value = lon.attributes()["_FillValue"]
    lon = np.ma.masked_equal(lon.get(), fill_value)

    if resolution == coarse_resolution:
        return lat, lon

    cores = options["cores"]

    from geotiepoints import modis5kmto1km, modis1kmto500m, modis1kmto250m
    logger.debug("Interpolating from " + str(coarse_resolution)
                 + " to " + str(resolution))
    if coarse_resolution == 5000:
        lon, lat = modis5kmto1km(lon, lat)
    if resolution == 500:
        lon, lat = modis1kmto500m(lon, lat, cores)
    if resolution == 250:
        lon, lat = modis1kmto250m(lon, lat, cores)

    return lat, lon
def run(FILE_NAME):

    # Identify the data field.
    DATAFIELD_NAME = 'bsst'

    if USE_NETCDF4:
        from netCDF4 import Dataset
        nc = Dataset(FILE_NAME)
        # Subset the data to match the size of the swath geolocation fields.
        # Turn off autoscaling, we'll handle that ourselves due to non-standard
        # naming of the offset attribute.
        var = nc.variables[DATAFIELD_NAME]
        var.set_auto_maskandscale(False)
        lat = nc.variables['lat']
        lon = nc.variables['lon']
    else:
        from pyhdf.SD import SD, SDC
        hdf = SD(FILE_NAME, SDC.READ)

        # Read dataset.
        var = hdf.select(DATAFIELD_NAME)
        lat = hdf.select('lat')
        lon = hdf.select('lon')

    latitude = lat[::8]
    longitude = lon[::8]
    data = var[::8, ::8].astype(np.float64)
    
    # Apply the attributes.  By inspection, fill value is 0
    data[data==0] = np.nan
    data = data * var.scale_factor + var.add_off
    datam = np.ma.masked_array(data, mask=np.isnan(data))
    
    m = Basemap(projection='cyl', resolution='l',
                llcrnrlat=-90, urcrnrlat=90,
                llcrnrlon=-180, urcrnrlon=180)
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(-90, 91, 45))
    m.drawmeridians(np.arange(-180, 180, 45), labels=[True,False,False,True])
    m.pcolormesh(longitude, latitude, datam, latlon=True)

    cax = plt.axes([0.92, 0.3, 0.01, 0.4])
    cb = plt.colorbar(cax=cax)
    units = 'degrees-C'
    cb.set_label(units)    
    
    basename = os.path.basename(FILE_NAME)
    fig = plt.gcf()
    # plt.show()
    long_name = 'Sea Surface Temperature ('+DATAFIELD_NAME+')'
    fig.suptitle('{0}\n{1}'.format(basename, long_name))
    # plt.show()
    pngfile = "{0}.py.png".format(basename)
    fig.savefig(pngfile)
def run(FILE_NAME):

    # Identify the HDF-EOS2 swath data file.
    DATAFIELD_NAME = 'radiances'

    if USE_NETCDF4:
        from netCDF4 import Dataset    
        nc = Dataset(FILE_NAME)
        data = nc.variables['radiances'][:,:,567]
        latitude = nc.variables['Latitude'][:]
        longitude = nc.variables['Longitude'][:]
    else:
        from pyhdf.SD import SD, SDC
        hdf = SD(FILE_NAME, SDC.READ)

        # Read dataset.
        data3D = hdf.select(DATAFIELD_NAME)
        data = data3D[:,:,567]

        # Read geolocation dataset.
        lat = hdf.select('Latitude')
        latitude = lat[:,:]
        lon = hdf.select('Longitude')
        longitude = lon[:,:]
        

    
    # Replace the filled value with NaN, replace with a masked array.
    data[data == -9999] = np.nan
    datam = np.ma.masked_array(data, np.isnan(data))
    
 
    # Draw a polar stereographic projection using the low resolution coastline
    # database.
    m = Basemap(projection='spstere', resolution='l',
                boundinglat=-65, lon_0 = 180)
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(-80., -50., 5.))
    m.drawmeridians(np.arange(-180., 181., 20.), labels=[1, 0, 0, 1])
    x, y = m(longitude, latitude)
    m.pcolormesh(x, y, datam)

   # See page 101 of "AIRS Version 5.0 Released Files Description" document [1]
    # for unit specification.
    units = 'mW/m**2/cm**-1/sr'
    cb = m.colorbar()
    cb.set_label('Unit:'+units)
    
    basename = os.path.basename(FILE_NAME)
    plt.title('{0}\n {1} at channel=567'.format(basename, DATAFIELD_NAME))
    fig = plt.gcf()
    # plt.show()
    pngfile = "{0}.py.png".format(basename)
    fig.savefig(pngfile)
Exemplo n.º 30
0
def load_thin_modis(satscene, options):
    """Read modis data from file and load it into *satscene*.
    """
    filename = satscene.time_slot.strftime("thin_MYD021KM.A%Y%j.%H%M.005.NRT.hdf")
    filename = os.path.join(options["dir"], filename)
    
    data = SD(filename)

    datasets = ['EV_250_Aggr1km_RefSB',
                'EV_500_Aggr1km_RefSB',
                'EV_1KM_RefSB',
                'EV_1KM_Emissive']

    for dataset in datasets:
        subdata = data.select(dataset)
        band_names = subdata.attributes()["band_names"].split(",")
        if len(satscene.channels_to_load & set(band_names)) > 0:
            uncertainty = data.select(dataset+"_Uncert_Indexes")
            if dataset == 'EV_1KM_Emissive':
                array = calibrate_tb(subdata, uncertainty)
            else:
                array = calibrate_refl(subdata, uncertainty)
            for (i, band) in enumerate(band_names):
                if band in satscene.channels_to_load:
                    satscene[band] = array[i]

    mda = data.attributes()["CoreMetadata.0"]
    orbit_idx = mda.index("ORBITNUMBER")
    satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116]

    lat, lon = get_lat_lon(satscene, None)
    from pyresample import geometry
    satscene.area = geometry.SwathDefinition(lons=lon, lats=lat)

    # trimming out dead sensor lines
    if satscene.satname == "aqua":
        for band in ["6", "27"]:
            if not satscene[band].is_loaded() or satscene[band].data.mask.all():
                continue
            width = satscene[band].data.shape[1]
            height = satscene[band].data.shape[0]
            indices = satscene[band].data.mask.sum(1) < width
            if indices.sum() == height:
                continue
            satscene[band] = satscene[band].data[indices, :]
            satscene[band].area = geometry.SwathDefinition(
                lons=satscene.area.lons[indices,:],
                lats=satscene.area.lats[indices,:])
            satscene[band].area.area_id = ("swath_" + satscene.fullname + "_"
                                           + str(satscene.time_slot) + "_"
                                           + str(satscene[band].shape) + "_"
                                           + str(band))