Example #1
0
    def get_variable_names(self, filenames, data_type=None):
        try:
            from pyhdf.SD import SD
            from pyhdf.HDF import HDF
        except ImportError:
            raise ImportError("HDF support was not installed, please reinstall with pyhdf to read HDF files.")

        valid_variables = set([])
        for filename in filenames:
            # Do VD variables
            datafile = HDF(filename)
            vdata = datafile.vstart()
            variables = vdata.vdatainfo()
            # Assumes that latitude shape == longitude shape (it should):
            # dim_length = [var[3] for var in variables if var[0] == 'Latitude'][0]
            for var in variables:
                # if var[3] == dim_length:
                valid_variables.add(var[0])

            # Do SD variables:
            sd = SD(filename)
            datasets = sd.datasets()
            # if 'Height' in datasets:
            #     valid_shape = datasets['Height'][1]
            for var in datasets:
                    # if datasets[var][1] == valid_shape:
                valid_variables.add(var)

        return valid_variables
Example #2
0
def main():

    varname_to_rpn_name = {
        "precipitation": "PR",
        "relativeError": "RERR"
    }

    varnames = list(varname_to_rpn_name.keys())

    target_dir = "/skynet3_rech1/huziy/from_hdf4"
    source_dir = "/st1_fs2/winger/Validation/TRMM/HDF_format"

    for f_name in os.listdir(source_dir):
        if not f_name.endswith("HDF"):
            continue

        path = os.path.join(source_dir, f_name)
        ds = SD(path)
        print(ds.datasets())
        target_path = os.path.join(target_dir, f_name + ".rpn")
        r_obj = RPN(target_path, mode="w")
        for varname in varnames:
            var_data = ds.select(varname)[0, :, :]
            r_obj.write_2D_field(
                name=varname_to_rpn_name[varname],
                data=var_data, label=varname, grid_type="L",
                ig = [25, 25, 4013, 18012])
        r_obj.close()
def write_interpolated(filename, f0, f1, fact, datasets):
    '''
    interpolate two hdf files f0 and f1 using factor fact, and
    write the result to filename
    '''

    hdf = SD(filename, SDC.WRITE|SDC.CREATE)
    for dataset in datasets:

        try:
            info = SD(f0).select(dataset).info()
        except:
            print >> stderr, 'Error loading %s in %s' % (dataset, f0)
            raise

        typ  = info[3]
        shp  = info[2]
        met0 = SD(f0).select(dataset).get()
        met1 = SD(f1).select(dataset).get()

        interp = (1-fact)*met0 + fact*met1

        interp = interp.astype({
                SDC.INT16: 'int16',
                SDC.FLOAT32: 'float32',
                SDC.FLOAT64: 'float64',
            }[typ])

        # write
        sds = hdf.create(dataset, typ, shp)
        sds[:] = interp[:]
        sds.endaccess()

    hdf.end()
Example #4
0
def read_rrc(inpath):
    '''Read rrc data m*n from hdf file'''

    '''b1-5;b13-16 for MODIS Rrc
        Rrc_1238 Rrc_443-862 ozone senz solz for VIIRS rrc   
    '''  
    hdf = SD(inpath, SDC.READ)
    #dts = sorted(hdf.datasets().keys())
    modis_key = ['CorrRefl_01','CorrRefl_02','CorrRefl_03','CorrRefl_04','CorrRefl_05',
                 'CorrRefl_13','CorrRefl_14','CorrRefl_15','CorrRefl_16']
    viirs_key = ['Rrc_443','Rrc_486','Rrc_551','Rrc_671','Rrc_745','Rrc_862','Rrc_1238']
    mission = os.path.basename(inpath)[0]
    if mission =='A' or mission =='T':keys = modis_key
    elif mission=='V':keys = viirs_key
    else:keys = hdf.datasets().keys()
    for i,dt in enumerate(keys):
        print(i,dt)
        band = hdf.select(dt)[:,:]        
        if i==0:             
            limit = (band.shape[0],band.shape[1],len(keys))            
            rrc = np.zeros(limit,dtype = np.float)
            rrc[:,:,i] = band
        else:
            rrc[:,:,i] = band
    hdf.end()
    print(rrc.shape)
    return rrc
Example #5
0
def load_standard_lfm_hdf(filename):
	""" Load the standard formated hdf which we want to emulate"""
	f = SD(filename, SDC.READ)
	X_grid = f.select('X_grid')
	Y_grid = f.select('Y_grid')
	Z_grid = f.select('Z_grid')

	# x_grid is size nkp1,njp1,nip1
	(nkp1,njp1,nip1) = X_grid[:].shape
	# The LFM reader expects i to vary fastest, then j, then k
	# However, the LFM pre-converted files store positions with k varying fastest (column-major)
	# Recommend saving in column-major format. If it fails, we can always switch.

	
	# i = 0; j = 0; k = 0
	# print 'printing standard first row'
	# for i in range(nip1):
	# 	print X_grid[k,j,i]/R_e

	# print 'printing j sweep'
	# i = 0; j = 0; k = 0;
	# for j in range(njp1):
	# 	print X_grid[k,j,i]/R_e

	# print 'printing k sweep'
	# i = 0; j = 0; k = 0;
	# for k in range(nkp1):
	# 	print X_grid[k,j,i]/R_e


	print 'standard nip1,njp1,nkp1 =', nip1,njp1,nkp1
	ni = nip1-1
	nj = njp1-1
	nk = nkp1-1
	print 'standard ni,nj,nk =', ni,nj,nk
Example #6
0
def main(cal_file, with_cp):

    from pyhdf.SD import SD

    if with_cp:
        cmd = 'cp %s /home/noel/scratch/' % (cal_file)
        print "running "+cmd
        os.system(cmd)
        filename = os.path.basename(cal_file)
        cal_file = '/home/noel/scratch/' + filename
                        
    print 'Reading ' + cal_file 
    
    vars = ['Latitude', 'Longitude', 
            'Total_Attenuated_Backscatter_532', 'Attenuated_Backscatter_1064', 'Perpendicular_Attenuated_Backscatter_532',
            'Pressure', 'Temperature', 'Molecular_Number_Density', 'Tropopause_Height', 'Surface_Elevation']
    
    hdf = SD(cal_file)
    for var in vars:
        print 'Reading ' + var
        hdf_var = hdf.select(var)
        data = hdf_var.get()
        hdf_var.endaccess()
    hdf.end()
    
    print 'ok.'
    if with_cp:
        print 'Removing '+filename
        cmd = 'rm -f /home/noel/scratch/' + filename
        os.system(cmd)
Example #7
0
def export_multi_fluid_LFM(argv):
	if (len(argv) >= 2):
		input_filename = argv[0]
		output_filename = argv[1]
		print input_filename
		sd = SD(input_filename, SDC.READ)

		grid = get_corners(sd)

		timesteps = 0

		# step = 1640000

		for key in sd.datasets().keys():
			shift = key.find('time_step')
			if shift == 0:
				if len(argv) == 3:
					step = argv[2]
					if key == 'time_step_'+str(step):
						export_timestep(sd, output_filename, key, grid)
				else:
					export_timestep(sd, output_filename, key, grid)
				timesteps += 1

		print 'timesteps found in file:', timesteps


	else:
		print 'usage: python lfm_split.py input_multi_timestep_hdf output_filename_prefix step(optional)'
Example #8
0
    def test_1000m_to_250m(self):
        """Test the 1 km to 250 meter interpolation facility."""
        # gfilename = \
        #      "/san1/test/data/modis/MOD03_A12278_113638_2012278145123.hdf"
        gfilename = "/local_disk/src/python-geotiepoints/tests/MOD03_A12278_113638_2012278145123.hdf"
        # result_filename = \
        #      "/san1/test/data/modis/250m_lonlat_results.npz"
        result_filename = "/local_disk/src/python-geotiepoints/tests/250m_lonlat_results.npz"

        from pyhdf.SD import SD
        from pyhdf.error import HDF4Error

        try:
            gdata = SD(gfilename)
        except HDF4Error:
            print("Failed reading eos-hdf file %s" % gfilename)
            return

        lats = gdata.select("Latitude")[0:50, :]
        lons = gdata.select("Longitude")[0:50, :]

        verif = np.load(result_filename)
        vlons = verif['lons']
        vlats = verif['lats']
        tlons, tlats = modis1kmto250m(lons, lats)

        self.assert_(np.allclose(tlons, vlons, atol=0.05))
        self.assert_(np.allclose(tlats, vlats, atol=0.05))
    def test_1000m_to_250m(self):
        """test the 1 km to 250 meter interpolation facility
        """
        gfilename_hdf = "testdata/MOD03_A12278_113638_2012278145123.hdf"
        gfilename = "testdata/250m_lonlat_section_input.npz"
        result_filename = "testdata/250m_lonlat_section_result.npz"

        from pyhdf.SD import SD
        from pyhdf.error import HDF4Error
        
        gdata = None
        try:
            gdata = SD(gfilename_hdf)
        except HDF4Error:
            print "Failed reading eos-hdf file %s" % gfilename_hdf
            try:
                indata = np.load(gfilename)
            except IOError:
                return

        if gdata:
            lats = gdata.select("Latitude")[20:50, :]
            lons = gdata.select("Longitude")[20:50, :]
        else:
            lats = indata['lat'] / 1000.
            lons = indata['lon'] / 1000.

        verif = np.load(result_filename)
        vlons = verif['lon'] / 1000.
        vlats = verif['lat'] / 1000.
        tlons, tlats = modis1kmto250m(lons, lats)

        self.assert_(np.allclose(tlons, vlons, atol=0.05))
        self.assert_(np.allclose(tlats, vlats, atol=0.05))
def run(FILE_NAME):

    DATAFIELD_NAME = 'dHat'

    if USE_NETCDF4:
        from netCDF4 import Dataset    
        nc = Dataset(FILE_NAME)
        var = nc.variables[DATAFIELD_NAME]
        # This datafield has scale factor and add offset attributes, but no
        # fill value.  We'll turn off automatic scaling and do it ourselves.
        var.set_auto_maskandscale(False)
        data = nc.variables[DATAFIELD_NAME][:].astype(np.float64)

        # Retrieve scale/offset attributes.
        scale_factor = var.scale_factor
        add_offset = var.add_offset
    
        # Retrieve the geolocation data.
        latitude = nc.variables['geolocation'][:,:,0]
        longitude = nc.variables['geolocation'][:,:,1]
    else:
        from pyhdf.SD import SD, SDC
        hdf = SD(FILE_NAME, SDC.READ)
        
        ds = hdf.select(DATAFIELD_NAME)
        data = ds[:,:].astype(np.double)

        # Handle scale/osffset attributes.
        attrs = ds.attributes(full=1)
        sfa=attrs["scale_factor"]
        scale_factor = sfa[0]
        aoa=attrs["add_offset"]
        add_offset = aoa[0]

        # Retrieve the geolocation data.        
        geo = hdf.select('geolocation')
        latitude = geo[:,:,0]
        longitude = geo[:,:,1]

    data = data / scale_factor + add_offset
    
    # Draw an equidistant cylindrical projection using the high resolution
    # coastline database.
    m = Basemap(projection='cyl', resolution='h',
                llcrnrlat=30, urcrnrlat = 36,
                llcrnrlon=121, urcrnrlon = 133)
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(30, 37), labels=[1, 0, 0, 0])
    m.drawmeridians(np.arange(121, 133, 2), labels=[0, 0, 0, 1])
    m.pcolormesh(longitude, latitude, data, latlon=True)
    cb = m.colorbar()
    cb.set_label('Unit:mm')

    basename = os.path.basename(FILE_NAME)
    plt.title('{0}\n{1}'.format(basename, DATAFIELD_NAME))
    fig = plt.gcf()
    # plt.show()
    
    pngfile = "{0}.py.png".format(basename)
    fig.savefig(pngfile)
Example #11
0
def rainfall_anunal_car(year):

    file = glob.glob('/Users/yuewang/Documents/DATA/atl/ATL_3B42V7_rain_accum.'+ str(year)+'*')

    rainfall_0 = []
    for i in file:
        atl =SD(i,SDC.READ)
        rainfall = atl.select('RAIN_TOTAL')
        rainfall_value = rainfall.get()
        rainfall_0.append(rainfall_value)
    
    rainfall_single = np.array(rainfall_0)
    rainfall_anunal = sum(rainfall_single)
    rainfall_anunal_car = rainfall_anunal[238:286,372:476]
    
# calculation none-zone mean value    
    ind = np.where(rainfall_anunal_car != 0)
    rf_annual = []
    for i,j in zip(*ind):
        mm = rainfall_anunal_car[i,j]
        rf_annual.append(mm)
    rf_annual = np.array(rf_annual)
    
    d = np.mean(rf_annual)
    
    return d
Example #12
0
def rainfall_anunal_GMX(year):

    file = glob.glob('/Users/yuewang/Documents/DATA/atl/ATL_3B42V7_rain_accum.'+ str(year)+'*')

    rainfall_0 = []
    for i in file:
        atl =SD(i,SDC.READ)
        rainfall = atl.select('RAIN_TOTAL')
        rainfall_value = rainfall.get()
        rainfall_0.append(rainfall_value)
    
    rainfall_single = np.array(rainfall_0)
    rainfall_anunal = sum(rainfall_single)
    rainfall_anunal_GMX = rainfall_anunal[280:320,340:400]
    
    ind = np.where(rainfall_anunal_GMX != 0)
    rf_annual = []
    for i,j in zip(*ind):
        mm = rainfall_anunal_GMX[i,j]
        rf_annual.append(mm)
    rf_annual = np.array(rf_annual)
    
    c = np.mean(rf_annual)
    
    return c
Example #13
0
 def load(self, fldname, **kwargs):
     """ Load Cali Current fields for a given day"""
     self._timeparams(**kwargs)
     
     if fldname == 'chl':
         filename = "/C%04i%03i_chl_mapped.hdf" % (self.yr, self.yd)
         #ncfieldname = 'chl_%04i_%03i' % (yr,yd)
         def scale(PV): return 10**(PV*0.015-2)
     elif fldname == 'sst':
         filename = "/M%04i%03i_sst_mapped.hdf" % (self.yr, self.yd)
         #ncfieldname = 'sst_%04i_%03i' % (yr,yd)            
         def scale(PV): return PV*0.15000001-3
     if not os.path.isfile(self.datadir + filename):
         print "Downloading " + filename
         self.download(fldname, self.jd)
         
     h = SD(self.datadir + filename,SDC.READ)        
     ncfieldname = h.datasets().keys()[0]
     fld =  h.select(ncfieldname)
     attr = fld.attributes()
     PV = fld[:].astype(np.float)
     PV[PV<0] = PV[PV<0]+256
     PV[PV==0]   = np.nan
     PV[PV==255] = np.nan
     setattr(self, fldname, scale(PV)[self.j1:self.j2, self.i1:self.i2])
def run(FILE_NAME):
    # Identify the data field.
    DATAFIELD_NAME = 'Longwave Flux (2.5R)'

    if USE_NETCDF4:
        from netCDF4 import Dataset
        nc = Dataset(FILE_NAME)
        data = nc.variables[DATAFIELD_NAME][:].astype(np.float64)
    else:
        from pyhdf.SD import SD, SDC
        hdf = SD(FILE_NAME, SDC.READ)
        
        # Read dataset.
        data2D = hdf.select(DATAFIELD_NAME)
        data = data2D[:,:]


    # Set fillvalue and units.
    # See "CERES Data Management System ES-4 Collection Guide" [1] and a sample
    # image by NASA [2] for details.  The fillvalue is 3.4028235E38.  Here, we
    # just use the max of the data.
    fillvalue = np.max(data)
    data[data == fillvalue] = np.nan
    datam = np.ma.masked_array(data, mask=np.isnan(data))
    
    # Set fillvalue and units.
    # See "CERES Data Management System ES-4 Collection Guide" [1] and a
    # sample image by NASA [2] for details.
    # The fillvalue is 3.4028235E38. Here, we use max value from the dataset.
    units = 'Watts/Meter^2'
    ysize, xsize = data.shape
    xinc = 360.0 / xsize
    yinc = 180.0 / ysize
    x0, x1 = (-180, 180)
    y0, y1 = (-90, 90)
    longitude = np.linspace(x0 + xinc/2, x1 - xinc/2, xsize)
    latitude = np.linspace(y0 + yinc/2, y1 - yinc/2, ysize)
    
    # Flip the latitude to run from 90 to -90
    latitude = latitude[::-1]
    
    # The data is global, so render in a global projection.
    m = Basemap(projection='cyl', resolution='l',
                llcrnrlat=-90, urcrnrlat=90,
                llcrnrlon=-180, urcrnrlon=180)
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(-90.,90,45))
    m.drawmeridians(np.arange(-180.,180,45), labels=[True,False,False,True])
    m.pcolormesh(longitude, latitude, datam, latlon=True)
    cb = m.colorbar()

    cb.set_label(units)

    basename = os.path.basename(FILE_NAME)
    plt.title('{0}\n{1}'.format(basename, DATAFIELD_NAME))
    fig = plt.gcf()
    # plt.show()
    pngfile = "{0}.py.png".format(basename)
    fig.savefig(pngfile)
Example #15
0
def readhdf(filename, fieldname, ignoreE = True):
    try:
        hdf = SD(filename, SDC.READ)
        data = hdf.select(fieldname)[:].copy()
        hdf.end()
    except Exception, e:
        if not ignoreE:print e
        data = Dataset(filename)[fieldname][:].copy()
Example #16
0
def print_dataset_2A12(*arg):

	FILE_NAME=arg[0]+'1B01.'+arg[1]
	hdf = SD(FILE_NAME, SDC.READ)

	'List available SDS datasets'
	for ds in hdf.datasets():
		print ds
Example #17
0
 def __init__(self, filename, filename_info, filetype_info):
     super(HDF4FileHandler, self).__init__(filename, filename_info, filetype_info)
     self.file_content = {}
     file_handle = SD(self.filename, SDC.READ)
     self._collect_attrs('', file_handle.attributes())
     for k, v in file_handle.datasets().items():
         self.collect_metadata(k, file_handle.select(k))
     del file_handle
Example #18
0
def aod(date, path, orbit, block):

    from constant import BAND_GREEN
    file_aerosol = _MIL2ASAE_fname(date, path, orbit)
    f = SD(file_aerosol)
    tau0 = f.select('RegMeanSpectralOptDepth').get()[block-1 , : , :, BAND_GREEN]
    tau0[tau0 == -9999] = np.mean(tau0[tau0 != -9999])

    return tau0
def run(FILE_NAME):

    DATAFIELD_NAME = 'Temperature_MW_A'
    if USE_NETCDF4:
        from netCDF4 import Dataset    
        nc = Dataset(FILE_NAME)

        # The variable has a fill value, 
        # so netCDF4 converts it to a float64 masked array for us.
        data = nc.variables[DATAFIELD_NAME][11,:,:]
        latitude = nc.variables['Latitude'][:]
        longitude = nc.variables['Longitude'][:]

    else:
        from pyhdf.SD import SD, SDC
        hdf = SD(FILE_NAME, SDC.READ)

        # List available SDS datasets.
        # print hdf.datasets()

        # Read dataset.
        data3D = hdf.select(DATAFIELD_NAME)
        data = data3D[11,:,:]

        # Read geolocation dataset.
        lat = hdf.select('Latitude')
        latitude = lat[:,:]
        lon = hdf.select('Longitude')
        longitude = lon[:,:]

        # Handle fill value.
        attrs = data3D.attributes(full=1)
        fillvalue=attrs["_FillValue"]

        # fillvalue[0] is the attribute value.
        fv = fillvalue[0]
        data[data == fv] = np.nan
        data = np.ma.masked_array(data, np.isnan(data))

    
    # Draw an equidistant cylindrical projection using the low resolution
    # coastline database.
    m = Basemap(projection='cyl', resolution='l',
                llcrnrlat=-90, urcrnrlat = 90,
                llcrnrlon=-180, urcrnrlon = 180)
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(-90., 120., 30.), labels=[1, 0, 0, 0])
    m.drawmeridians(np.arange(-180., 181., 45.), labels=[0, 0, 0, 1])
    m.pcolormesh(longitude, latitude, data, latlon=True, alpha=0.90)
    cb = m.colorbar()
    cb.set_label('Unit:K')
    basename = os.path.basename(FILE_NAME)
    plt.title('{0}\n {1} at TempPrsLvls=11'.format(basename, DATAFIELD_NAME))
    fig = plt.gcf()
    # plt.show()
    pngfile = "{0}.{1}.py.png".format(basename, DATAFIELD_NAME)
    fig.savefig(pngfile)
Example #20
0
    def setup_grid(self):
        """Setup necessary variables for grid """

        if not os.path.isfile(self.datadir + self.gridfile):
            urllib.urlretrieve(self.dataurl + self.gridfile,
                               self.datadir + self.gridfile)
        g = SD(self.datadir + self.gridfile, SDC.READ)
        self.llat = g.select('Latitude')[:]
        self.llon = g.select('Longitude')[:]
Example #21
0
def print_dataset_1C21(*arg):

	FILE_NAME=arg[0]+'1C21.'+arg[1]
	print FILE_NAME
	hdf = SD(FILE_NAME, SDC.READ)

	'List available SDS datasets'
	for ds in hdf.datasets():
		print ds
def run(FILE_NAME):

    DATAFIELD_NAME = 'SurfaceTemperature'

    # The dataset is (6144 x 6400).  Subset it to be around than 1K x 1K
    # Otherwise, the plot will skip processing some regions.
    rows = slice(0, 6144, 6)
    cols = slice(0, 6400, 6)

    if USE_NETCDF4:    
        from netCDF4 import Dataset
        nc = Dataset(FILE_NAME)

        data = nc.variables[DATAFIELD_NAME][rows, cols]
    
        # Retrieve the geolocation data.
        latitude = nc.variables['Latitude'][rows, cols]
        longitude = nc.variables['Longitude'][rows, cols]
        
    else:
        from pyhdf.SD import SD, SDC
        hdf = SD(FILE_NAME, SDC.READ)

        # Read dataset.
        data2D = hdf.select(DATAFIELD_NAME)
        data = data2D[rows,cols]

        # Read geolocation dataset.
        lat = hdf.select('Latitude')
        latitude = lat[rows,cols]
        lon = hdf.select('Longitude')
        longitude = lon[rows,cols]
        

    # Apply the fill value.  The valid minimum is zero, although there's no
    # attribute.
    data[data < 0] = np.nan
    data = np.ma.masked_array(data, np.isnan(data))
    
    # Render the data in a lambert azimuthal equal area projection.
    m = Basemap(projection='nplaea', resolution='l',
                boundinglat=60, lon_0=43)
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(50, 90, 10), labels=[1, 0, 0, 1])
    m.drawmeridians(np.arange(-180, 180, 30))
    x, y = m(longitude, latitude)
    m.pcolormesh(x, y, data)
    cb = m.colorbar()
    cb.set_label('Unknown')

    basename = os.path.basename(FILE_NAME)
    plt.title('{0}\n{1}'.format(basename, DATAFIELD_NAME))
    fig = plt.gcf()
    # plt.show()
    pngfile = "{0}.py.png".format(basename)
    fig.savefig(pngfile)
Example #23
0
def open_file(filename):
    A = SD(filename)

    # retrieve data SDS
    d = A.datasets()
    sds_name = d.keys()[0]  # name of sds. Dictionary method.
    sds = A.select(sds_name)
    pin = A.attributes()

    return sds, pin
Example #24
0
 def landmask(self):
     if not hasattr(self, "_landmask"):
         filename = os.path.basename(self.landurl)
         if not os.path.isfile(self.datadir + filename):
             urllib.urlretrieve(self.dataurl + self.landurl,
                                self.datadir + filename)
         h = SD(self.datadir + filename, SDC.READ)        
         ncfieldname = h.datasets().keys()[0]
         self._landmask =   h.select(ncfieldname)[:] == -1
     return self._landmask
Example #25
0
def read_var_ikslice(filename,var_name,i,k,tc,pc):
    hdffile = SD(filename,SDC.READ)
    ni = hdffile.select('X_grid').ni-1
    nj = hdffile.select('X_grid').nj-1

    if var_name not in ['br','btheta','bphi','vr','vtheta','vphi','bt','bp','vt','vp']:
        var=hdffile.select(var_name+'_').get(start=(k,0,i),count=(1,nj,1)).squeeze()
    else:
        if var_name in ['br','btheta','bphi','bt','bp']:
            bx=hdffile.select('bx_').get(start=(k,0,i),count=(1,nj,1)).squeeze()
            by=hdffile.select('by_').get(start=(k,0,i),count=(1,nj,1)).squeeze()
            bz=hdffile.select('bz_').get(start=(k,0,i),count=(1,nj,1)).squeeze()

            if var_name=='br':
                var     = bx*cos(pc[k])*sin(tc) + by*sin(pc[k])*sin(tc) + bz*cos(tc)
            elif (var_name=='btheta' or var_name=='bt'):
                var = bx*cos(pc[k])*cos(tc) + by*sin(pc[k])*cos(tc) - bz*sin(tc)
            else:
                var   =-bx*sin(pc[k])            + by*cos(pc[k])
        else:
            vx=hdffile.select('vx_').get(start=(k,0,i),count=(1,nj,1)).squeeze()
            vy=hdffile.select('vy_').get(start=(k,0,i),count=(1,nj,1)).squeeze()
            vz=hdffile.select('vz_').get(start=(k,0,i),count=(1,nj,1)).squeeze()

            if var_name=='vr':
                var    = vx*cos(pc[k])*sin(tc) + vy*sin(pc[k])*sin(tc) + vz*cos(tc)
            elif (var_name=='vtheta' or var_name=='vt'):
                var = vx*cos(pc[k])*cos(tc) + vy*sin(pc[k])*cos(tc) - vz*sin(tc)
            else:
                var   =-vx*sin(pc[k])            + vy*cos(pc[k])
    hdffile.end()
    return(var)
def run(FILE_NAME):
    
    # Identify the data field.
    DATAFIELD_NAME = 'AlbedoLocal'

    hdf = SD(FILE_NAME, SDC.READ)

    # Read dataset.
    data4D = hdf.select(DATAFIELD_NAME)

    # Convert 4-D data to 2-D data by subsetting.
    SOMBlockDim = 50;
    NBandDim = 0;
    data = data4D[SOMBlockDim,:,:,NBandDim].astype(np.double)

    # Read geolocation dataset from HDF-EOS2 dumper output.
    GEO_FILE_NAME = 'lat_MISR_TC_ALBEDO_P223_F05_lvl50.output'
    GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], 
                                 GEO_FILE_NAME)
    lat = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0])
    lat = lat.reshape(data.shape)
    
    GEO_FILE_NAME = 'lon_MISR_TC_ALBEDO_P223_F05_lvl50.output'
    GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], 
                                 GEO_FILE_NAME)
    lon = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0])
    lon = lon.reshape(data.shape)
        
    # Read attributes.
    attrs = data4D.attributes(full=1)
    fva=attrs["_FillValue"]
    _FillValue = fva[0]

    # Apply the fill value.
    data[data == _FillValue] = np.nan
    datam = np.ma.masked_array(data, mask=np.isnan(data))


    # Set the limit for the plot.
    m = Basemap(projection='cyl', resolution='h',
                llcrnrlat=np.min(lat), urcrnrlat = np.max(lat),
                llcrnrlon=np.min(lon), urcrnrlon = np.max(lon))
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(np.floor(np.min(lat)), np.ceil(np.max(lat)), 1), labels=[1, 0, 0, 0])
    m.drawmeridians(np.arange(np.floor(np.min(lon)), np.ceil(np.max(lon)), 1), labels=[0, 0, 0, 1])
    m.pcolormesh(lon, lat, datam, latlon=True)
    cb = m.colorbar()
    cb.set_label('No Unit')

    basename = os.path.basename(FILE_NAME)
    plt.title('{0}\n{1} at SOMBlockDim=50 NBandDim=0'.format(basename, DATAFIELD_NAME))
    fig = plt.gcf()
    # plt.show()
    pngfile = "{0}.l50.py.png".format(basename)
    fig.savefig(pngfile)
Example #27
0
def get_lat_lon_modis(satscene, options):
    """Read lat and lon.
    """
    filename_tmpl = satscene.time_slot.strftime(options["geofile"])
    file_list = glob.glob(os.path.join(options["dir"], filename_tmpl))

    if len(file_list) == 0:
        # Try in the same directory as the data
        data_dir = os.path.split(options["filename"])[0]
        file_list = glob.glob(os.path.join(data_dir, filename_tmpl))

    if len(file_list) > 1:
        logger.warning("More than 1 geolocation file matching!")
        filename = max(file_list, key=lambda x: os.stat(x).st_mtime)
        coarse_resolution = 1000
    elif len(file_list) == 0:
        logger.warning("No geolocation file matching " + filename_tmpl
                       + " in " + options["dir"])
        logger.debug("Using 5km geolocation and interpolating")
        filename = options["filename"]
        coarse_resolution = 5000
    else:
        filename = file_list[0]
        coarse_resolution = 1000

    logger.debug("Loading geolocation file: " + str(filename)
                 + " at resolution " + str(coarse_resolution))

    resolution = options["resolution"]

    data = SD(str(filename))
    lat = data.select("Latitude")
    fill_value = lat.attributes()["_FillValue"]
    lat = np.ma.masked_equal(lat.get(), fill_value)
    lon = data.select("Longitude")
    fill_value = lon.attributes()["_FillValue"]
    lon = np.ma.masked_equal(lon.get(), fill_value)

    if resolution == coarse_resolution:
        return lat, lon

    cores = options["cores"]

    from geotiepoints import modis5kmto1km, modis1kmto500m, modis1kmto250m
    logger.debug("Interpolating from " + str(coarse_resolution)
                 + " to " + str(resolution))
    if coarse_resolution == 5000:
        lon, lat = modis5kmto1km(lon, lat)
    if resolution == 500:
        lon, lat = modis1kmto500m(lon, lat, cores)
    if resolution == 250:
        lon, lat = modis1kmto250m(lon, lat, cores)

    return lat, lon
def run(FILE_NAME):

    # Identify the HDF-EOS2 swath data file.
    DATAFIELD_NAME = 'radiances'

    if USE_NETCDF4:
        from netCDF4 import Dataset    
        nc = Dataset(FILE_NAME)
        data = nc.variables['radiances'][:,:,567]
        latitude = nc.variables['Latitude'][:]
        longitude = nc.variables['Longitude'][:]
    else:
        from pyhdf.SD import SD, SDC
        hdf = SD(FILE_NAME, SDC.READ)

        # Read dataset.
        data3D = hdf.select(DATAFIELD_NAME)
        data = data3D[:,:,567]

        # Read geolocation dataset.
        lat = hdf.select('Latitude')
        latitude = lat[:,:]
        lon = hdf.select('Longitude')
        longitude = lon[:,:]
        

    
    # Replace the filled value with NaN, replace with a masked array.
    data[data == -9999] = np.nan
    datam = np.ma.masked_array(data, np.isnan(data))
    
 
    # Draw a polar stereographic projection using the low resolution coastline
    # database.
    m = Basemap(projection='spstere', resolution='l',
                boundinglat=-65, lon_0 = 180)
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(-80., -50., 5.))
    m.drawmeridians(np.arange(-180., 181., 20.), labels=[1, 0, 0, 1])
    x, y = m(longitude, latitude)
    m.pcolormesh(x, y, datam)

   # See page 101 of "AIRS Version 5.0 Released Files Description" document [1]
    # for unit specification.
    units = 'mW/m**2/cm**-1/sr'
    cb = m.colorbar()
    cb.set_label('Unit:'+units)
    
    basename = os.path.basename(FILE_NAME)
    plt.title('{0}\n {1} at channel=567'.format(basename, DATAFIELD_NAME))
    fig = plt.gcf()
    # plt.show()
    pngfile = "{0}.py.png".format(basename)
    fig.savefig(pngfile)
def run(FILE_NAME):

    # Identify the data field.
    DATAFIELD_NAME = 'bsst'

    if USE_NETCDF4:
        from netCDF4 import Dataset
        nc = Dataset(FILE_NAME)
        # Subset the data to match the size of the swath geolocation fields.
        # Turn off autoscaling, we'll handle that ourselves due to non-standard
        # naming of the offset attribute.
        var = nc.variables[DATAFIELD_NAME]
        var.set_auto_maskandscale(False)
        lat = nc.variables['lat']
        lon = nc.variables['lon']
    else:
        from pyhdf.SD import SD, SDC
        hdf = SD(FILE_NAME, SDC.READ)

        # Read dataset.
        var = hdf.select(DATAFIELD_NAME)
        lat = hdf.select('lat')
        lon = hdf.select('lon')

    latitude = lat[::8]
    longitude = lon[::8]
    data = var[::8, ::8].astype(np.float64)
    
    # Apply the attributes.  By inspection, fill value is 0
    data[data==0] = np.nan
    data = data * var.scale_factor + var.add_off
    datam = np.ma.masked_array(data, mask=np.isnan(data))
    
    m = Basemap(projection='cyl', resolution='l',
                llcrnrlat=-90, urcrnrlat=90,
                llcrnrlon=-180, urcrnrlon=180)
    m.drawcoastlines(linewidth=0.5)
    m.drawparallels(np.arange(-90, 91, 45))
    m.drawmeridians(np.arange(-180, 180, 45), labels=[True,False,False,True])
    m.pcolormesh(longitude, latitude, datam, latlon=True)

    cax = plt.axes([0.92, 0.3, 0.01, 0.4])
    cb = plt.colorbar(cax=cax)
    units = 'degrees-C'
    cb.set_label(units)    
    
    basename = os.path.basename(FILE_NAME)
    fig = plt.gcf()
    # plt.show()
    long_name = 'Sea Surface Temperature ('+DATAFIELD_NAME+')'
    fig.suptitle('{0}\n{1}'.format(basename, long_name))
    # plt.show()
    pngfile = "{0}.py.png".format(basename)
    fig.savefig(pngfile)
Example #30
0
def load_thin_modis(satscene, options):
    """Read modis data from file and load it into *satscene*.
    """
    filename = satscene.time_slot.strftime("thin_MYD021KM.A%Y%j.%H%M.005.NRT.hdf")
    filename = os.path.join(options["dir"], filename)
    
    data = SD(filename)

    datasets = ['EV_250_Aggr1km_RefSB',
                'EV_500_Aggr1km_RefSB',
                'EV_1KM_RefSB',
                'EV_1KM_Emissive']

    for dataset in datasets:
        subdata = data.select(dataset)
        band_names = subdata.attributes()["band_names"].split(",")
        if len(satscene.channels_to_load & set(band_names)) > 0:
            uncertainty = data.select(dataset+"_Uncert_Indexes")
            if dataset == 'EV_1KM_Emissive':
                array = calibrate_tb(subdata, uncertainty)
            else:
                array = calibrate_refl(subdata, uncertainty)
            for (i, band) in enumerate(band_names):
                if band in satscene.channels_to_load:
                    satscene[band] = array[i]

    mda = data.attributes()["CoreMetadata.0"]
    orbit_idx = mda.index("ORBITNUMBER")
    satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116]

    lat, lon = get_lat_lon(satscene, None)
    from pyresample import geometry
    satscene.area = geometry.SwathDefinition(lons=lon, lats=lat)

    # trimming out dead sensor lines
    if satscene.satname == "aqua":
        for band in ["6", "27"]:
            if not satscene[band].is_loaded() or satscene[band].data.mask.all():
                continue
            width = satscene[band].data.shape[1]
            height = satscene[band].data.shape[0]
            indices = satscene[band].data.mask.sum(1) < width
            if indices.sum() == height:
                continue
            satscene[band] = satscene[band].data[indices, :]
            satscene[band].area = geometry.SwathDefinition(
                lons=satscene.area.lons[indices,:],
                lats=satscene.area.lats[indices,:])
            satscene[band].area.area_id = ("swath_" + satscene.fullname + "_"
                                           + str(satscene.time_slot) + "_"
                                           + str(satscene[band].shape) + "_"
                                           + str(band))
Example #31
0
 def set_data_shape(self):
     """
     use dataset set self.data_shape
     :return:
     """
     # 如果分辨率是 1000 米
     if self.resolution == 1000:
         satellite_type = ['AQUA', 'TERRA']
         if self.satellite in satellite_type:
             h4File = SD(self.in_file, SDC.READ)
             in_data_r250 = h4File.select('EV_250_Aggr1km_RefSB').get()
             self.data_shape = in_data_r250.shape[1:]
         else:
             raise ValueError('Cant read this satellite`s data.: {}'.format(
                 self.satellite))
     else:
         raise ValueError("Cant handle this resolution: ".format(
             self.resolution))
Example #32
0
 def set_file_attr(self):
     """
     根据 self.file_level_1 获取 L1 文件的属性
     set self.level_1_attr
     储存格式是字典
     :return:
     """
     if self.resolution == 1000:
         satellite_type1 = ['FY1C', 'FY1D']
         if self.satellite in satellite_type1:
             hdf4 = SD(self.in_file, SDC.READ)
             self.file_attr = attrs2dict(hdf4.attributes())
         else:
             raise ValueError('Cant read this satellite`s data.: {}'.format(
                 self.satellite))
     else:
         raise ValueError("Cant handle this resolution: ".format(
             self.resolution))
Example #33
0
def SD_wrapper(file_path):
    ds = None
    if 's3://' == file_path[0:5]:
        s3 = parse_s3_url(file_path)
        s3_client = boto3.client('s3')
        ds,tmpdir,fullFilename = hdf4_dataset_from_s3(s3_client,s3['bucket_name'],s3['prefix']+s3['prefix_end']+s3['resource'],filename=s3['resource'])
    else:
        ds = SD(file_path)
    return ds
Example #34
0
    def __init__(self, path):
        self.path = path
        self.file_handle = HDF(str(path))

        self.scientific_dataset = SD(str(path))
        datasets = self.scientific_dataset.datasets()
        dataset_dict = {
            key: Dataset(weakref.ref(self), key, *info)
            for key, info in datasets.items()
        }
        self.datasets = dataset_dict

        self.vdata_table = VS(self.file_handle)
        vdata_dict = {
            info[0]: VData(weakref.ref(self), *info)
            for info in self.vdata_table.vdatainfo()
        }
        self.vdata = vdata_dict
Example #35
0
 def __init__(self, filename):
     self.filename = filename
     self._succeed = True
     try:
         self.science_data = SD(self.filename)
     except:
         print('Modis READ ERROR......(myd021km):' + self.filename)
         print("Unexpected error:", sys.exc_info()[0])
         self._succeed = False
Example #36
0
def Get_h4_nc_Subdatasets(h4_nc_path, target_datasets):
    # 以只读方式打开文件
    hc_dataset = SD(h4_nc_path, SDC.READ)
    # 判断存放目标数据集名称的列表是否为空
    if len(target_datasets) == 0:
        print('The target dataset names is null.')
        return

    subdatasets_dict = dict()
    for dataset_name in target_datasets:
        temp_subdataset = hc_dataset.select(dataset_name)  # 获取的是一个pyhdf.SD.SDS对象
        if temp_subdataset is None:
            print("The subdataset:%s don't exist." % (dataset_name))
            continue

        subdatasets_dict[dataset_name] = temp_subdataset

    return subdatasets_dict
Example #37
0
def is_daily_product(hdf_filename):

    hdf_file = SD(hdf_filename, SDC.READ)

    # get CoreMetadata.0 attribute (which is a very long string) and just extract and analyze the substring
    # with the DAYNIGHT info:
    core_metadata0_string = hdf_file.attributes()['CoreMetadata.0']
    daynight_start_index = core_metadata0_string.find('DAYNIGHT')

    if daynight_start_index != -1:
        #print('test: |' + core_metadata0_string[daynight_start_index:daynight_start_index+120] + '|')
        daynight_info_block = core_metadata0_string[daynight_start_index:daynight_start_index+120]

        is_daily = (daynight_info_block.find('\"Day\"') != -1)
        #print(is_daily)
        return is_daily
    else:
        return false
def HDFread(filename, variable, Class=None):
    """
    Extract the data for non scientific data in V mode of hdf file
    """
    hdf = HDF(filename, HC.READ)

    # Initialize the SD, V and VS interfaces on the file.
    sd = SD(filename)
    vs = hdf.vstart()
    v = hdf.vgstart()

    # Encontrar el puto id de las Geolocation Fields
    if Class == None:
        ref = v.findclass('SWATH Vgroup')
    else:
        ref = v.findclass(Class)

    # Open all data of the class
    vg = v.attach(ref)
    # All fields in the class
    members = vg.tagrefs()

    nrecs = []
    names = []
    for tag, ref in members:
        # Vdata tag
        vd = vs.attach(ref)
        # nrecs, intmode, fields, size, name = vd.inquire()
        nrecs.append(vd.inquire()[0])  # number of records of the Vdata
        names.append(vd.inquire()[-1])  # name of the Vdata
        vd.detach()

    idx = names.index(variable)
    var = vs.attach(members[idx][1])
    V = var.read(nrecs[idx])
    var.detach()
    # Terminate V, VS and SD interfaces.
    v.end()
    vs.end()
    sd.end()
    # Close HDF file.
    hdf.close()

    return np.array(V).ravel()
Example #39
0
def read_band(file_path):
    DATAFIELD_NAME = 'EV_250_RefSB'

    hdf = SD(file_path, SDC.READ)

    file_name = path.splitext(path.basename(file_path))[0]
    band1_path = path.join(tmp_dir, file_name + "_band1.jpg")
    band2_path = path.join(tmp_dir, file_name + "_band2.jpg")

    if path.exists(band1_path) and path.exists(band2_path):
        pass
    else:
        # Read dataset.
        data = hdf.select(DATAFIELD_NAME)
        data2d = data.get().astype("double")

        # Retrieve attributes.
        attrs = data.attributes(full=1)
        aoa = attrs["radiance_offsets"]
        add_offset = aoa[0]
        fva = attrs["_FillValue"]
        _FillValue = fva[0]
        sfa = attrs["radiance_scales"]
        scale_factor = sfa[0]
        vra = attrs["valid_range"]
        valid_min = vra[0][0]
        valid_max = vra[0][1]

        invalid = np.logical_or(data2d > valid_max,
                                data2d < valid_min)
        invalid = np.logical_or(invalid, data2d == _FillValue)
        data2d[invalid] = np.nan

        data2d[0] = (data2d[0] - add_offset[0]) * scale_factor[0]
        data2d[1] = (data2d[1] - add_offset[1]) * scale_factor[1]
        data2d = np.ma.masked_array(data2d, np.isnan(data2d))

        band1_img = Image.fromarray(data2d[0].astype("uint8"))
        band2_img = Image.fromarray(data2d[1].astype("uint8"))

        band1_img.save(band1_path)
        band2_img.save(band2_path)

    return band1_path, band2_path
Example #40
0
    def _readGranule(self, filename):
        """Reads one GOCI granule."""

        # Don't fuss if the file cannot be opened
        # ---------------------------------------
        try:
            if self.verb:
                print "[] Working on " + filename
            hfile = SD(filename)
        except HDF4Error:
            if self.verb > 2:
                print "- %s: not recognized as an HDF file" % filename
            return

        # Read select variables (reshape to allow concatenation later)
        # ------------------------------------------------------------
        for sds in self.SDS:

            sds_ = sds.replace(' ', '_')

            if sds == 'Turbid_index':
                try:
                    v = hfile.select(sds).get()
                except HDF4Error:
                    v = np.full(hfile.select('AOD_550nm').get().shape, -999)
            else:
                v = hfile.select(sds).get()

            rank = len(v.shape)
            if rank == 2:
                self.__dict__[sds_].append(flipud(v))
            else:
                self.__dict__[sds_].append(v)

        #       Satellite name
        #       --------------
        if self.sat is None:
            self.sat = 'GOCI'

        #       Collection
        #       ----------
        if self.col is None:
            col = 1
            self.col = "%03d" % col
Example #41
0
    def read(self, filename, **kwargs):
        """Read the data"""
        from pyhdf.SD import SD
        import datetime

        # print "*** >>> Read the hdf-eos file!"
        root = SD(filename)

        # Get all the Attributes:
        # Common Attributes, Data Time,
        # Data Structure and Scene Coordinates
        for key in root.attributes().keys():
            self._eoshdf_info[key] = root.attributes()[key]

        # Start Time - datetime object
        starttime = datetime.datetime.strptime(self._eoshdf_info['Start Time'][0:13],
                                               "%Y%j%H%M%S")
        msec = float(self._eoshdf_info['Start Time'][13:16]) / 1000.
        self.starttime = starttime + datetime.timedelta(seconds=msec)

        # End Time - datetime object
        endtime = datetime.datetime.strptime(self._eoshdf_info['End Time'][0:13],
                                             "%Y%j%H%M%S")
        msec = float(self._eoshdf_info['End Time'][13:16]) / 1000.
        self.endtime = endtime + datetime.timedelta(seconds=msec)

        # What is the leading 'H' doing here?
        sensor_name = self._eoshdf_info['Sensor Name'][1:-1].lower()
        try:
            self.satid = EOS_SATELLITE[sensor_name]
        except KeyError:
            LOG.error("Failed setting the satellite id - sat-name = ",
                      sensor_name)

        self.orbit = self._eoshdf_info['Orbit Number']
        self.shape = (self._eoshdf_info['Number of Scan Control Points'],
                      self._eoshdf_info['Number of Pixel Control Points'])

        # try:
        if 1:
            value = root.select(self.name)
            attr = value.attributes()
            data = value.get()

            self.attr = attr
            band = data
            if self.name in FLAGS_QUALITY:
                self.data = band
            else:
                nodata = attr['bad_value_scaled']
                self.data = (np.ma.masked_equal(band, nodata) *
                             attr['slope'] + attr['intercept'])

            value.endaccess()
        # except:
        #    pass

        root.end()
        self.filled = True
 def extract_attributes(self):
     # extract the attributes from the data file
     try:
         granule = SD(self.data_path, 1)
     except TypeError:
         granule = SD(self.data_path.encode("utf-8"), 1)
     self.attributes = granule.attributes()
     granule.end()
Example #43
0
def load_data_from_files(filename):
    if not os.path.exists(filename):
        print("File {} does not exist, cannot load data.".format(filename))
        return
    elif not HDF.ishdf(filename):
        print("File {} is not in hdf4 file format, cannot load data.".format(
            filename))
        return

    f = SD(filename, SDC.READ)
    data_field = None
    for i, d in enumerate(f.datasets()):
        # print("{0}. {1}".format(i+1,d))
        if "NDVI" in d:
            data_field = d

    ndvi_data = f.select(data_field)
    data = np.array(ndvi_data.get())
    return data
def DescribeHDFvar(filename, variable):
    """
    Describes the info and atributes of HDF variable
    IMPUTS
    filename : complete path and filename
    variable : name of the variable to describe, type STR
    """

    file = SD(filename, SDC.READ)
    print('---------- ' + variable + ' ----------')
    sds_obj = file.select(variable)
    Var = sds_obj.get()
    sds_info = sds_obj.info()
    print(Var.shape)
    print(sds_info)
    print(sds_info[0], sds_info[1])
    print('sds attributes')
    pprint.pprint(sds_obj.attributes())
    file.end()
Example #45
0
    def __getHdfObj(self):
        """
        Return an insance of pyhdf.SD for read
        """
        try:
            hdf = SD(self.filename, SDC.READ)
        except HDF4Error as msg:
            sys.stderr.write('HDF4Error opening "' + self.filename +
                             '" for read.  Error message follows:' + str(msg))
            sys.stderr.write('Trying operation again.')
            sys.stderr.flush()

            # Maybe the I/O was busy.  Sleep for a bit & try again.
            import time
            time.sleep(1)

            hdf = SD(self.filename, SDC.READ)

        return hdf
Example #46
0
 def set_data_shape(self):
     """
     根据 self.satellite set self.dataset_shape
     :return:
     """
     # 如果分辨率是 1000 米
     if self.resolution == 1000:
         satellite_type1 = ['FY1C', 'FY1D']
         if self.satellite in satellite_type1:
             hdf4 = SD(self.in_file, SDC.READ)
             dn = hdf4.select('Earth_View')[0]
             self.data_shape = dn.shape
         else:
             raise ValueError('Cant read this satellite`s data.: {}'.format(
                 self.satellite))
     # elif self.resolution == 250:
     else:
         raise ValueError("Cant handle this resolution: ".format(
             self.resolution))
Example #47
0
 def set_file_attr(self):
     """
     get hdf5 file attrs self.file_attr
     :return:
     """
     if self.resolution == 1000:
         satellite_type = ['AQUA', 'TERRA']
         if self.satellite in satellite_type:
             try:
                 h4r = SD(self.in_file, SDC.READ)
                 self.file_attr = attrs2dict(h4r.attributes())
             except Exception as e:
                 print(str(e))
         else:
             raise ValueError('Cant read this satellite`s data.: {}'.format(
                 self.satellite))
     else:
         raise ValueError("Cant handle this resolution: ".format(
             self.resolution))
Example #48
0
    def create_lcm(self):

        for H, V in product(Hs, Vs):

            mcd_file = self.validate_file('MCD', self.year, self.fday, H, V)

            if mcd_file is not None:

                # open file
                mcdds = SD(os.path.join(mcd_data, mcd_file), SDC.READ)

                # load land cover mask
                lcm = mcdds.select(lc_type).get()
                mcdds.end()

                self.lcm[V * 2400:(V + 1) * 2400,
                         H * 2400:(H + 1) * 2400] = lcm

        return self.lcm
Example #49
0
def GetData(file, datafield):
    '''
    GetData(file,datafield) : read the data labeled datafield in file
    =================================================================
    
    Retrieve data from a HDF file    
    
    input : 
    ------
        file : name of input file
        datafield : label of required data field
    output:
    ------
        data3D : output array
    
    '''
    hdf = SD(file, SDC.READ)
    data3D = hdf.select(datafield)
    return data3D
Example #50
0
def read_var(fname, varname, normalized=False):
    f = SD(fname, SDC.READ)
    phi = f.select('fakeDim0')[:]
    theta = f.select('fakeDim1')[:]
    r = f.select('fakeDim2')[:]
    var = f.select('Data-Set-2')[:]
    f.end()

    if normalized:
        return (phi, theta, r, var)
    else:
        return (phi, theta, r * mas_units['length'], var * mas_units[varname])
Example #51
0
 def set_data_shape(self):
     """
     use dataset set self.data_shape
     :return:
     """
     # 如果分辨率是 13500 米
     if self.resolution == 13500:
         satellite_type = ['AQUA', 'TERRA']
         if self.satellite in satellite_type:
             h4r = SD(self.in_file, SDC.READ)
             in_data_r250 = h4r.select('Latitude').get()
             self.data_shape = in_data_r250.shape
             h4r.end()
         else:
             raise ValueError('Cant read this satellite`s data.: {}'.format(
                 self.satellite))
     else:
         raise ValueError("Cant handle this resolution: ".format(
             self.resolution))
Example #52
0
def load_hdf(filepath, year, day):
    try:
        hdf = SD(filepath, SDC.READ)
    except HDF4Error:
        print(f"Error {filepath}, downloading")
        os.remove(filepath)
        download_main(year, start_day=day, day_only=True)
        return load_hdf(filepath, year, day)
    else:
        return hdf
Example #53
0
class Hdf4File:
    """
    Base class for file formats using HDF4File format. The :class:`Hdf4File`
    wraps around the pyhdf.SD class to implement RAII.
    """
    def __init__(self, filename):
        """
        Open an HDF4 file for reading.

        Arguments:

            filename(str): The path to the file to open.
        """
        from pyhdf.SD import SD, SDC
        self.filename = filename
        self.file_handle = SD(self.filename, SDC.READ)

    def __del__(self):
        self.file_handle.end()
Example #54
0
    def get_lonlat(self, resolution, time_slot, cores=1):
        """Read lat and lon.
        """
        if (resolution, time_slot) in self.areas:
            return self.areas[resolution, time_slot]
        logger.debug("generating lon, lat at %d", resolution)
        if self.geofile is not None:
            coarse_resolution = 1000
            filename = self.geofile
        else:
            coarse_resolution = 5000
            logger.info("Using 5km geolocation and interpolating")
            filename = (self.datafiles.get(1000) or
                        self.datafiles.get(500) or
                        self.datafiles.get(250))

        logger.debug("Loading geolocation from file: " + str(filename)
                     + " at resolution " + str(coarse_resolution))

        data = SD(str(filename))
        lat = data.select("Latitude")
        fill_value = lat.attributes()["_FillValue"]
        lat = np.ma.masked_equal(lat.get(), fill_value)
        lon = data.select("Longitude")
        fill_value = lon.attributes()["_FillValue"]
        lon = np.ma.masked_equal(lon.get(), fill_value)

        if resolution == coarse_resolution:
            self.areas[resolution] = lon, lat
            return lon, lat

        from geotiepoints import modis5kmto1km, modis1kmto500m, modis1kmto250m
        logger.debug("Interpolating from " + str(coarse_resolution)
                     + " to " + str(resolution))
        if coarse_resolution == 5000:
            lon, lat = modis5kmto1km(lon, lat)
        if resolution == 500:
            lon, lat = modis1kmto500m(lon, lat, cores)
        if resolution == 250:
            lon, lat = modis1kmto250m(lon, lat, cores)

        self.areas[resolution, time_slot] = lon, lat
        return lon, lat
Example #55
0
class HDF4File:
    """
    Simplified interface for reading HDF4 files. It combines the SD and VS
    low-level interfaces.
    """

    # Attributes:
    #     variables(``list``): List of strings of variable names contained in
    #         this file.

    def __init__(self, path):
        self.path = path
        self.file_handle = HDF(str(path))

        self.scientific_dataset = SD(str(path))
        datasets = self.scientific_dataset.datasets()
        dataset_dict = {
            key: Dataset(weakref.ref(self), key, *info)
            for key, info in datasets.items()
        }
        self.datasets = dataset_dict

        self.vdata_table = VS(self.file_handle)
        vdata_dict = {
            info[0]: VData(weakref.ref(self), *info)
            for info in self.vdata_table.vdatainfo()
        }
        self.vdata = vdata_dict

    def __del__(self):
        if self.file_handle:
            self.file_handle.close()
            self.file_handle = None

    @property
    def variables(self):
        """
        Names of the variables available in this file.
        """
        return list(self.datasets.keys()) + list(self.vdata.keys())

    def __getattribute__(self, name):
        try:
            return object.__getattribute__(self, name)
        except AttributeError as error:
            datasets = object.__getattribute__(self, "datasets")
            if name in datasets:
                return datasets[name]
            vdata = object.__getattribute__(self, "vdata")
            if name in vdata:
                return vdata[name]
            raise error

    def __repr__(self):
        return f"HDF4File({self.path})"
Example #56
0
    def get_spectral_response(self):
        """
        return 光谱波数和响应值,1维,2维
        """
        if self.resolution == 13500:
            satellite_type1 = ['AQUA', 'TERRA']
            if self.satellite in satellite_type1:
                try:
                    h4r = SD(self.in_file, SDC.READ)
                    data_pre = h4r.select('radiances').get()
                    h4r.end()

                    # 非法值变成0 卷积时不贡献
                    condition = np.logical_or(data_pre < 0, data_pre > 200)
                    idx = np.where(condition)
                    #                     print len(idx[0])
                    if len(idx[0] > 0):
                        data_pre[idx] = 0
                    # 单位转换
                except Exception as e:
                    print(str(e))
                response = data_pre
                # 暂时取一个观测的光谱波数
                file_name = '{}_{}_SRF_Pub.txt'.format(self.satellite,
                                                       self.sensor)
                data_file = os.path.join(g_main_path, 'SRF', file_name)
                dtype = {
                    'names': ('wave_length', 'response'),
                    'formats': ('f4', 'f4')
                }
                if os.path.isfile(data_file):
                    datas = np.loadtxt(data_file, dtype=dtype)
                    # 波长转波数
                    wave_number = datas['wave_length']
            else:
                raise ValueError('Cant read this satellite`s data.: {}'.format(
                    self.satellite))
        else:
            raise ValueError(
                'Cant read this data, please check its resolution: {}'.format(
                    self.in_file))
        return wave_number, response
Example #57
0
    def get_sds(self, fieldnames=None):
        """Return specific or all SDS in the hdf file as dictionary.

        SDS arrays can be accessed using the 'data' key. Note that no scaling
        is applied to the data in get() method (use get_scaled() to achieve
        that). However, the scaling and missing data information can be
        accessed using the following keys:
            'scale_factor'
            'add_offset'
            '_FillValue'
        """

        # Convert scalar fieldnames to sequence
        if isinstance(fieldnames, basestring):
            fieldnames = (fieldnames, )

        # Open file to read SDs
        try:
            h4 = SD(self.filename, mode=SDC.READ)
            sclinfo = None
            if 'Slope_and_Offset_Usage' in h4.attributes():
                sclinfo = 'Slope_and_Offset_Usage'

            # Get all available SDS from file if fieldnames in not given
            if fieldnames is None:
                fieldnames = []
                for key in sorted(h4.datasets()):
                    fieldnames.append(key)

            # Create and empty dataset dictionary with all available
            # fields fill in data from SDS
            sds = dict.fromkeys(fieldnames, {})
            for key in sds:
                attrs = h4.select(key).attributes()
                if sclinfo:
                    attrs[sclinfo] = h4.attributes()[sclinfo]

                sds[key] = attrs
                sds[key]['data'] = h4.select(key).get()

            # Close hdf interface
            h4.end()
        except HDF4Error as e:
            raise HDF4Error(e)

        # Return raw (possibly un-calibrated) SDS/attributes dictionary
        return sds
Example #58
0
def retrieve_1C21(*arg):
    hdf = SD(arg[0], SDC.READ)

    Lon, Lat, date_beg, date_end, st, en = retrieve_ancillary(hdf)

    ' osRain includes rays #20 to #30'
    # data=hdf.select('osRain')[st:en,:,0] # [dBZ]

    ' normalSample correspond to dBZ in 140 levels'
    data = hdf.select('normalSample')[st:en, :, 100]  # [dBZ*100]
    data = data.astype(float)
    data[data <= -32700.] = np.nan
    data = data / 100.  # [dBZ]

    # foo = hdf.select('normalSample')[:, :, :]
    # print foo.shape
    # return 	Lon[:,20:31], Lat[:,20:31], [date_beg,date_end], osRain

    data = np.ma.masked_array(data, np.isnan(data))
    return Lon[st:en, :], Lat[st:en, :], [date_beg, date_end], data
def main(yaml_file):
    in_cfg = ReadInYaml(yaml_file)

    job_name = in_cfg.job_name
    ymd = in_cfg.ymd
    in_path = in_cfg.ipath
    out_path = in_cfg.opath
    for eachfile in in_path:
        h4r = SD(eachfile)
        ndsi_data = h4r.select('Day_CMG_Snow_Cover')[:]
        h4r.end()

        file_name = os.path.basename(eachfile) + '.png'
        out_fig = os.path.join(out_path, file_name)

        if not os.path.isdir(out_path):
            os.makedirs(out_path)

        print(out_fig)
        plot_map(job_name, ymd, ndsi_data, out_fig)
Example #60
0
def get_layer_information(cloudsat_filenames, get_quality=True, verbose=0):
    """ Returns
    CloudLayerType: -9: error, 0: non determined, 1-8 cloud types 
    CloudLayerBase: in km
    CloudLayerTop: in km
    CloudTypeQuality: valid range [0, 1]; if <get_quality>
    """

    all_info = {
        'CloudLayerType': [],
        'CloudLayerBase': [],
        'CloudLayerTop': []
    }

    if get_quality:
        all_info['CloudTypeQuality'] = []

    for cloudsat_path in cloudsat_filenames:

        sd = SD(cloudsat_path, SDC.READ)

        if verbose:
            # List available SDS datasets.
            print("hdf datasets:", sd.datasets())

        # get cloud types at each height
        for key, value in all_info.items():
            value.append(sd.select(key).get())

    for key, value in all_info.items():
        value = np.vstack(value)

        if key == 'CloudLayerType':
            all_info[key] = value.astype(np.int8)
        else:
            all_info[key] = value.astype(np.float16)

    if not get_quality:
        all_info['CloudTypeQuality'] = None

    return all_info