def test_1000m_to_250m(self): """test the 1 km to 250 meter interpolation facility """ gfilename_hdf = "testdata/MOD03_A12278_113638_2012278145123.hdf" gfilename = "testdata/250m_lonlat_section_input.npz" result_filename = "testdata/250m_lonlat_section_result.npz" from pyhdf.SD import SD from pyhdf.error import HDF4Error gdata = None try: gdata = SD(gfilename_hdf) except HDF4Error: print "Failed reading eos-hdf file %s" % gfilename_hdf try: indata = np.load(gfilename) except IOError: return if gdata: lats = gdata.select("Latitude")[20:50, :] lons = gdata.select("Longitude")[20:50, :] else: lats = indata['lat'] / 1000. lons = indata['lon'] / 1000. verif = np.load(result_filename) vlons = verif['lon'] / 1000. vlats = verif['lat'] / 1000. tlons, tlats = modis1kmto250m(lons, lats) self.assert_(np.allclose(tlons, vlons, atol=0.05)) self.assert_(np.allclose(tlats, vlats, atol=0.05))
def load_standard_lfm_hdf(filename): """ Load the standard formated hdf which we want to emulate""" f = SD(filename, SDC.READ) X_grid = f.select('X_grid') Y_grid = f.select('Y_grid') Z_grid = f.select('Z_grid') # x_grid is size nkp1,njp1,nip1 (nkp1,njp1,nip1) = X_grid[:].shape # The LFM reader expects i to vary fastest, then j, then k # However, the LFM pre-converted files store positions with k varying fastest (column-major) # Recommend saving in column-major format. If it fails, we can always switch. # i = 0; j = 0; k = 0 # print 'printing standard first row' # for i in range(nip1): # print X_grid[k,j,i]/R_e # print 'printing j sweep' # i = 0; j = 0; k = 0; # for j in range(njp1): # print X_grid[k,j,i]/R_e # print 'printing k sweep' # i = 0; j = 0; k = 0; # for k in range(nkp1): # print X_grid[k,j,i]/R_e print 'standard nip1,njp1,nkp1 =', nip1,njp1,nkp1 ni = nip1-1 nj = njp1-1 nk = nkp1-1 print 'standard ni,nj,nk =', ni,nj,nk
def read_var_point(filename,var_name,i,j,k,thetac,phic): thetac = thetac[j] phic = phic[k] hdffile = SD(filename,SDC.READ) if var_name not in ['br','btheta','bphi','vr','vtheta','vphi']: var=hdffile.select(var_name+'_').get(start=(k,j,i),count=(1,1,1)).squeeze() else: # R,theta,phi=r_theta_phi_uniform(filename) if var_name in ['br','btheta','bphi']: bx=hdffile.select('bx_').get(start=(k,j,i),count=(1,1,1)).squeeze() by=hdffile.select('by_').get(start=(k,j,i),count=(1,1,1)).squeeze() bz=hdffile.select('bz_').get(start=(k,j,i),count=(1,1,1)).squeeze() if var_name=='br': var = bx*cos(phic)*sin(thetac) + by*sin(phic)*sin(thetac) + bz*cos(thetac) elif var_name=='btheta': var = bx*cos(phic)*cos(thetac) + by*sin(phic)*cos(thetac) - bz*sin(thetac) else: var =-bx*sin(phic) + by*cos(phic) else: vx=hdffile.select('vx_').get(start=(k,j,i),count=(1,1,1)).squeeze() vy=hdffile.select('vy_').get(start=(k,j,i),count=(1,1,1)).squeeze() vz=hdffile.select('vz_').get(start=(k,j,i),count=(1,1,1)).squeeze() if var_name=='vr': var = vx*cos(phic)*sin(thetac) + vy*sin(phic)*sin(thetac) + vz*cos(thetac) elif var_name=='vtheta': var = vx*cos(phic)*cos(thetac) + vy*sin(phic)*cos(thetac) - vz*sin(thetac) else: var =-vx*sin(phic) + vy*cos(phic) hdffile.end() return(var)
def readMOD35L2(fname, geoloc_only=False): hdf_file = SD(HDFDIR + fname) if not geoloc_only: cloud_mask = hdf_file.select('Cloud_Mask').get() lon = hdf_file.select('Longitude').get() lat = hdf_file.select('Latitude').get() hdf_file.end() if not geoloc_only: cld_msk = uint8(cloud_mask[0]) cloud = cld_msk & 6 # 0, 2, 4, 6 land = cld_msk & 192 # 0, 64, 128, 192 cloud[cloud==0] = 1 # 0 -> Cloud cloud[cloud!=1] = 0 # 2, 4, 6 -> No cloud coast = land coast[coast==64] = 1 # 64 -> Coast coast[coast!=1] = 0 # 0, 128, 192 -> Not coast land[land!=0] = 1 # 64, 128, 192 -> Land, 0 -> Water return lon, lat, cloud, land, coast return lon, lat
def test_5_to_1(self): """test the 5km to 1km interpolation facility """ #gfilename = "testdata/MOD03_A12097_174256_2012097175435.hdf" gfilename = "/san1/test/data/modis/MOD03_A12097_174256_2012097175435.hdf" #filename = "testdata/MOD021km_A12097_174256_2012097175435.hdf" filename = "/san1/test/data/modis/MOD021km_A12097_174256_2012097175435.hdf" from pyhdf.SD import SD from pyhdf.error import HDF4Error try: gdata = SD(gfilename) data = SD(filename) except HDF4Error: print "Failed reading both eos-hdf files %s and %s" % (gfilename, filename) return glats = gdata.select("Latitude")[:] glons = gdata.select("Longitude")[:] lats = data.select("Latitude")[:] lons = data.select("Longitude")[:] tlons, tlats = modis5kmto1km(lons, lats) self.assert_(np.allclose(tlons, glons, atol=0.05)) self.assert_(np.allclose(tlats, glats, atol=0.05))
def read_var_islice(filename,var_name,i,thetac,phic): nk = phic.size nj = thetac.size phic = phic[:,None] thetac = thetac[None,:] hdffile = SD(filename,SDC.READ) if var_name not in ['br','btheta','bphi','vr','vtheta','vphi']: var=hdffile.select(var_name+'_').get(start=(0,0,i),count=(nk,nj,1)).squeeze() else: if var_name in ['br','btheta','bphi']: bx=hdffile.select('bx_').get(start=(0,0,i),count=(nk,nj,1)).squeeze() by=hdffile.select('by_').get(start=(0,0,i),count=(nk,nj,1)).squeeze() bz=hdffile.select('bz_').get(start=(0,0,i),count=(nk,nj,1)).squeeze() if var_name=='br': var = bx*cos(phic)*sin(thetac) + by*sin(phic)*sin(thetac) + bz*cos(thetac) elif var_name=='btheta': var = bx*cos(phic)*cos(thetac) + by*sin(phic)*cos(thetac) - bz*sin(thetac) else: var =-bx*sin(phic) + by*cos(phic) else: vx=hdffile.select('vx_').get(start=(0,0,i),count=(nk,nj,1)).squeeze() vy=hdffile.select('vy_').get(start=(0,0,i),count=(nk,nj,1)).squeeze() vz=hdffile.select('vz_').get(start=(0,0,i),count=(nk,nj,1)).squeeze() if var_name=='vr': var = vx*cos(phic)*sin(thetac) + vy*sin(phic)*sin(thetac) + vz*cos(thetac) elif var_name=='vtheta': var = vx*cos(phic)*cos(thetac) + vy*sin(phic)*cos(thetac) - vz*sin(thetac) else: var =-vx*sin(phic) + vy*cos(phic) hdffile.end() return(var)
def test_1000m_to_250m(self): """Test the 1 km to 250 meter interpolation facility.""" # gfilename = \ # "/san1/test/data/modis/MOD03_A12278_113638_2012278145123.hdf" gfilename = "/local_disk/src/python-geotiepoints/tests/MOD03_A12278_113638_2012278145123.hdf" # result_filename = \ # "/san1/test/data/modis/250m_lonlat_results.npz" result_filename = "/local_disk/src/python-geotiepoints/tests/250m_lonlat_results.npz" from pyhdf.SD import SD from pyhdf.error import HDF4Error try: gdata = SD(gfilename) except HDF4Error: print("Failed reading eos-hdf file %s" % gfilename) return lats = gdata.select("Latitude")[0:50, :] lons = gdata.select("Longitude")[0:50, :] verif = np.load(result_filename) vlons = verif['lons'] vlats = verif['lats'] tlons, tlats = modis1kmto250m(lons, lats) self.assert_(np.allclose(tlons, vlons, atol=0.05)) self.assert_(np.allclose(tlats, vlats, atol=0.05))
def run(FILE_NAME): DATAFIELD_NAME = 'dHat' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) var = nc.variables[DATAFIELD_NAME] # This datafield has scale factor and add offset attributes, but no # fill value. We'll turn off automatic scaling and do it ourselves. var.set_auto_maskandscale(False) data = nc.variables[DATAFIELD_NAME][:].astype(np.float64) # Retrieve scale/offset attributes. scale_factor = var.scale_factor add_offset = var.add_offset # Retrieve the geolocation data. latitude = nc.variables['geolocation'][:,:,0] longitude = nc.variables['geolocation'][:,:,1] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) ds = hdf.select(DATAFIELD_NAME) data = ds[:,:].astype(np.double) # Handle scale/osffset attributes. attrs = ds.attributes(full=1) sfa=attrs["scale_factor"] scale_factor = sfa[0] aoa=attrs["add_offset"] add_offset = aoa[0] # Retrieve the geolocation data. geo = hdf.select('geolocation') latitude = geo[:,:,0] longitude = geo[:,:,1] data = data / scale_factor + add_offset # Draw an equidistant cylindrical projection using the high resolution # coastline database. m = Basemap(projection='cyl', resolution='h', llcrnrlat=30, urcrnrlat = 36, llcrnrlon=121, urcrnrlon = 133) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(30, 37), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(121, 133, 2), labels=[0, 0, 0, 1]) m.pcolormesh(longitude, latitude, data, latlon=True) cb = m.colorbar() cb.set_label('Unit:mm') basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def run(FILE_NAME): DATAFIELD_NAME = 'Temperature_MW_A' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) # The variable has a fill value, # so netCDF4 converts it to a float64 masked array for us. data = nc.variables[DATAFIELD_NAME][11,:,:] latitude = nc.variables['Latitude'][:] longitude = nc.variables['Longitude'][:] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # List available SDS datasets. # print hdf.datasets() # Read dataset. data3D = hdf.select(DATAFIELD_NAME) data = data3D[11,:,:] # Read geolocation dataset. lat = hdf.select('Latitude') latitude = lat[:,:] lon = hdf.select('Longitude') longitude = lon[:,:] # Handle fill value. attrs = data3D.attributes(full=1) fillvalue=attrs["_FillValue"] # fillvalue[0] is the attribute value. fv = fillvalue[0] data[data == fv] = np.nan data = np.ma.masked_array(data, np.isnan(data)) # Draw an equidistant cylindrical projection using the low resolution # coastline database. m = Basemap(projection='cyl', resolution='l', llcrnrlat=-90, urcrnrlat = 90, llcrnrlon=-180, urcrnrlon = 180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90., 120., 30.), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180., 181., 45.), labels=[0, 0, 0, 1]) m.pcolormesh(longitude, latitude, data, latlon=True, alpha=0.90) cb = m.colorbar() cb.set_label('Unit:K') basename = os.path.basename(FILE_NAME) plt.title('{0}\n {1} at TempPrsLvls=11'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.{1}.py.png".format(basename, DATAFIELD_NAME) fig.savefig(pngfile)
def setup_grid(self): """Setup necessary variables for grid """ if not os.path.isfile(self.datadir + self.gridfile): urllib.urlretrieve(self.dataurl + self.gridfile, self.datadir + self.gridfile) g = SD(self.datadir + self.gridfile, SDC.READ) self.llat = g.select('Latitude')[:] self.llon = g.select('Longitude')[:]
def run(FILE_NAME): DATAFIELD_NAME = 'SurfaceTemperature' # The dataset is (6144 x 6400). Subset it to be around than 1K x 1K # Otherwise, the plot will skip processing some regions. rows = slice(0, 6144, 6) cols = slice(0, 6400, 6) if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) data = nc.variables[DATAFIELD_NAME][rows, cols] # Retrieve the geolocation data. latitude = nc.variables['Latitude'][rows, cols] longitude = nc.variables['Longitude'][rows, cols] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[rows,cols] # Read geolocation dataset. lat = hdf.select('Latitude') latitude = lat[rows,cols] lon = hdf.select('Longitude') longitude = lon[rows,cols] # Apply the fill value. The valid minimum is zero, although there's no # attribute. data[data < 0] = np.nan data = np.ma.masked_array(data, np.isnan(data)) # Render the data in a lambert azimuthal equal area projection. m = Basemap(projection='nplaea', resolution='l', boundinglat=60, lon_0=43) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(50, 90, 10), labels=[1, 0, 0, 1]) m.drawmeridians(np.arange(-180, 180, 30)) x, y = m(longitude, latitude) m.pcolormesh(x, y, data) cb = m.colorbar() cb.set_label('Unknown') basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def get_lat_lon_modis(satscene, options): """Read lat and lon. """ filename_tmpl = satscene.time_slot.strftime(options["geofile"]) file_list = glob.glob(os.path.join(options["dir"], filename_tmpl)) if len(file_list) == 0: # Try in the same directory as the data data_dir = os.path.split(options["filename"])[0] file_list = glob.glob(os.path.join(data_dir, filename_tmpl)) if len(file_list) > 1: logger.warning("More than 1 geolocation file matching!") filename = max(file_list, key=lambda x: os.stat(x).st_mtime) coarse_resolution = 1000 elif len(file_list) == 0: logger.warning("No geolocation file matching " + filename_tmpl + " in " + options["dir"]) logger.debug("Using 5km geolocation and interpolating") filename = options["filename"] coarse_resolution = 5000 else: filename = file_list[0] coarse_resolution = 1000 logger.debug("Loading geolocation file: " + str(filename) + " at resolution " + str(coarse_resolution)) resolution = options["resolution"] data = SD(str(filename)) lat = data.select("Latitude") fill_value = lat.attributes()["_FillValue"] lat = np.ma.masked_equal(lat.get(), fill_value) lon = data.select("Longitude") fill_value = lon.attributes()["_FillValue"] lon = np.ma.masked_equal(lon.get(), fill_value) if resolution == coarse_resolution: return lat, lon cores = options["cores"] from geotiepoints import modis5kmto1km, modis1kmto500m, modis1kmto250m logger.debug("Interpolating from " + str(coarse_resolution) + " to " + str(resolution)) if coarse_resolution == 5000: lon, lat = modis5kmto1km(lon, lat) if resolution == 500: lon, lat = modis1kmto500m(lon, lat, cores) if resolution == 250: lon, lat = modis1kmto250m(lon, lat, cores) return lat, lon
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'bsst' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) # Subset the data to match the size of the swath geolocation fields. # Turn off autoscaling, we'll handle that ourselves due to non-standard # naming of the offset attribute. var = nc.variables[DATAFIELD_NAME] var.set_auto_maskandscale(False) lat = nc.variables['lat'] lon = nc.variables['lon'] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. var = hdf.select(DATAFIELD_NAME) lat = hdf.select('lat') lon = hdf.select('lon') latitude = lat[::8] longitude = lon[::8] data = var[::8, ::8].astype(np.float64) # Apply the attributes. By inspection, fill value is 0 data[data==0] = np.nan data = data * var.scale_factor + var.add_off datam = np.ma.masked_array(data, mask=np.isnan(data)) m = Basemap(projection='cyl', resolution='l', llcrnrlat=-90, urcrnrlat=90, llcrnrlon=-180, urcrnrlon=180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90, 91, 45)) m.drawmeridians(np.arange(-180, 180, 45), labels=[True,False,False,True]) m.pcolormesh(longitude, latitude, datam, latlon=True) cax = plt.axes([0.92, 0.3, 0.01, 0.4]) cb = plt.colorbar(cax=cax) units = 'degrees-C' cb.set_label(units) basename = os.path.basename(FILE_NAME) fig = plt.gcf() # plt.show() long_name = 'Sea Surface Temperature ('+DATAFIELD_NAME+')' fig.suptitle('{0}\n{1}'.format(basename, long_name)) # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def run(FILE_NAME): # Identify the HDF-EOS2 swath data file. DATAFIELD_NAME = 'radiances' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) data = nc.variables['radiances'][:,:,567] latitude = nc.variables['Latitude'][:] longitude = nc.variables['Longitude'][:] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data3D = hdf.select(DATAFIELD_NAME) data = data3D[:,:,567] # Read geolocation dataset. lat = hdf.select('Latitude') latitude = lat[:,:] lon = hdf.select('Longitude') longitude = lon[:,:] # Replace the filled value with NaN, replace with a masked array. data[data == -9999] = np.nan datam = np.ma.masked_array(data, np.isnan(data)) # Draw a polar stereographic projection using the low resolution coastline # database. m = Basemap(projection='spstere', resolution='l', boundinglat=-65, lon_0 = 180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-80., -50., 5.)) m.drawmeridians(np.arange(-180., 181., 20.), labels=[1, 0, 0, 1]) x, y = m(longitude, latitude) m.pcolormesh(x, y, datam) # See page 101 of "AIRS Version 5.0 Released Files Description" document [1] # for unit specification. units = 'mW/m**2/cm**-1/sr' cb = m.colorbar() cb.set_label('Unit:'+units) basename = os.path.basename(FILE_NAME) plt.title('{0}\n {1} at channel=567'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def read_var(fname,varname,normalized=False): f = SD(fname,SDC.READ) phi = f.select('fakeDim0')[:] theta = f.select('fakeDim1')[:] r = f.select('fakeDim2')[:] var = f.select('Data-Set-2')[:] f.end() if normalized: return(phi,theta,r,var) else: return(phi,theta,r*mas_units['length'],var*mas_units[varname])
def load_hdf_spec(filename,ix,iy,data_name='MCD43GF_CMG'): """ Purpose: Simple hdf load file for MCD43GF files. Chose only specific indices to load Input: filename: full file path and name of the hdf file to load ix: array of indices to be loaded in the x direction iy: array of indices to be loaded in the y direction Output: dat: np.array of the requested data at the indices Keywords: data_name: (defaults to MCD43GF_CMG) the name of the dataset to load within the filename Dependencies: numpy pyhdf Required files: filename Example: >>b = load_hdf_spec(fp+'MCD43GF_geo_shortwave_193_2007.hdf',[200,201,202],[503,504,505,506]) >>b array([[ nan, nan, nan, nan], [ nan, nan, nan, nan], [ nan, nan, nan, nan]]) >>b.shape (3L, 4L) Modification History: Written (v1.0): Samuel LeBlanc, 2017-03-22, Santa Cruz, CA """ import numpy as np from pyhdf.SD import SD, SDC hdf = SD(filename, SDC.READ) if hasattr(ix,'__len__'): if (len(ix)-1)<(ix[-1]-ix[0]): raise ValueError('ix is not a contiguous array') if (len(iy)-1)<(iy[-1]-iy[0]): raise ValueError('iy is not a contiguous array') dat = hdf.select(data_name).get(start=(ix[0],iy[0]),count=(ix[-1]-ix[0]+1,iy[-1]-iy[0]+1)) else: dat = hdf.select(data_name).get(start=(ix,iy),count=(1,1)) dat = dat.astype(float) dat[dat==32767] = np.nan dat = dat/1000.0 return dat
def run(FILE_NAME): DATAFIELD_NAME = 'binDIDHmean' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) data = nc.variables[DATAFIELD_NAME][:].astype(np.float64) # Retrieve the geolocation data. latitude = nc.variables['Latitude'][:] longitude = nc.variables['Longitude'][:] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. ds = hdf.select(DATAFIELD_NAME) data = ds[:,:] # Read geolocation dataset. lat = hdf.select('Latitude') latitude = lat[:,:] lon = hdf.select('Longitude') longitude = lon[:,:] # The swath crosses the international dateline between row 6000 and 7000. # This causes the mesh to smear, so we'll adjust the longitude (modulus # 360 degrees, of course) both in the longitude array and in the basemap # definition to avoid that. longitude[longitude < -60] += 360 # Draw an equidistant cylindrical projection using the low resolution # coastline database. m = Basemap(projection='cyl', resolution='l', llcrnrlat=-90, urcrnrlat = 90, llcrnrlon=-60, urcrnrlon = 300) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90., 120., 30.), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180., 181., 45.), labels=[0, 0, 0, 1]) m.pcolormesh(longitude, latitude, data, latlon=True) cb = m.colorbar() cb.set_label('Unit:none') basename = os.path.basename(FILE_NAME) plt.title('{0}\n {1}'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def load_thin_modis(satscene, options): """Read modis data from file and load it into *satscene*. """ filename = satscene.time_slot.strftime("thin_MYD021KM.A%Y%j.%H%M.005.NRT.hdf") filename = os.path.join(options["dir"], filename) data = SD(filename) datasets = ['EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB', 'EV_1KM_Emissive'] for dataset in datasets: subdata = data.select(dataset) band_names = subdata.attributes()["band_names"].split(",") if len(satscene.channels_to_load & set(band_names)) > 0: uncertainty = data.select(dataset+"_Uncert_Indexes") if dataset == 'EV_1KM_Emissive': array = calibrate_tb(subdata, uncertainty) else: array = calibrate_refl(subdata, uncertainty) for (i, band) in enumerate(band_names): if band in satscene.channels_to_load: satscene[band] = array[i] mda = data.attributes()["CoreMetadata.0"] orbit_idx = mda.index("ORBITNUMBER") satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116] lat, lon = get_lat_lon(satscene, None) from pyresample import geometry satscene.area = geometry.SwathDefinition(lons=lon, lats=lat) # trimming out dead sensor lines if satscene.satname == "aqua": for band in ["6", "27"]: if not satscene[band].is_loaded() or satscene[band].data.mask.all(): continue width = satscene[band].data.shape[1] height = satscene[band].data.shape[0] indices = satscene[band].data.mask.sum(1) < width if indices.sum() == height: continue satscene[band] = satscene[band].data[indices, :] satscene[band].area = geometry.SwathDefinition( lons=satscene.area.lons[indices,:], lats=satscene.area.lats[indices,:]) satscene[band].area.area_id = ("swath_" + satscene.fullname + "_" + str(satscene.time_slot) + "_" + str(satscene[band].shape) + "_" + str(band))
def callback(body, message): """Do actual work.""" logger.info("body in callback() is %s" % body) # pull lat/lon, time path = body sd = SD(path) lat = N.array(sd.select('Latitude').get()) lon = N.array(sd.select('Longitude').get()) t = N.array(sd.select('Time').get()) sd.end() #logger.info("lat: %s" % str(lat.shape)) #logger.info("lon: %s" % str(lon.shape)) #logger.info("time: %s" % str(t.shape)) # build metadata json id = os.path.basename(path) md = { "id": id, "dataset": "AIRX2RET", "starttime": t[0,0], "endtime": t[44,29], "location": { "coordinates": [[ [ lon[0,0], lat[0,0] ], [ lon[0,29], lat[0,29] ], [ lon[44,29], lat[44,29] ], [ lon[44,0], lat[44,0] ], [ lon[0,0], lat[0,0] ], ]], "type": "polygon" }, "urls": "http://mozart/data/public/products/%s" % id } # publish pub_dir = '/data/public/products' ensure_dir(pub_dir) shutil.move(path, os.path.join(pub_dir, id)) # insert into ElasticSearch index = doctype = 'airs' conn = ES('http://localhost:9200') mapping = json.load(open('grq_mapping.json')) if not conn.indices.exists_index(index): conn.indices.create_index(index, mapping) conn.indices.put_mapping(doctype, mapping, index) ret = conn.index(md, index, doctype, md['id']) message.ack()
def run(FILE_NAME): DATAFIELD_NAME = 'CO Profiles Day' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) data = nc.variables[DATAFIELD_NAME][:, :, 1].astype(np.float64) latitude = nc.variables['Latitude'][:] longitude = nc.variables['Longitude'][:] pressure = nc.variables['Pressure Grid'][:] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data3D = hdf.select(DATAFIELD_NAME) data = data3D[:, :, 1].astype(np.float64) # Read coordinates. latitude = hdf.select('Latitude')[:] longitude = hdf.select('Longitude')[:] pressure = hdf.select('Pressure Grid')[:] # Replace the fill value with NaN data[data == -9999] = np.nan data = np.ma.masked_array(data, np.isnan(data)) m = Basemap(projection='cyl', resolution='l', llcrnrlat=-90, urcrnrlat=90, llcrnrlon=-180, urcrnrlon=180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90, 91, 45), labels=[True,False,False,True]) m.drawmeridians(np.arange(-180, 180, 45), labels=[True,False,False,True]) m.pcolormesh(longitude, latitude, data, latlon=True) cb = m.colorbar() cb.set_label('ppbv') basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1} at Pressure={2} hPa'.format(basename, DATAFIELD_NAME, pressure[1])) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def main(): varname_to_rpn_name = { "precipitation": "PR", "relativeError": "RERR" } varnames = list(varname_to_rpn_name.keys()) target_dir = "/skynet3_rech1/huziy/from_hdf4" source_dir = "/st1_fs2/winger/Validation/TRMM/HDF_format" for f_name in os.listdir(source_dir): if not f_name.endswith("HDF"): continue path = os.path.join(source_dir, f_name) ds = SD(path) print(ds.datasets()) target_path = os.path.join(target_dir, f_name + ".rpn") r_obj = RPN(target_path, mode="w") for varname in varnames: var_data = ds.select(varname)[0, :, :] r_obj.write_2D_field( name=varname_to_rpn_name[varname], data=var_data, label=varname, grid_type="L", ig = [25, 25, 4013, 18012]) r_obj.close()
def rainfall_anunal_GMX(year): file = glob.glob('/Users/yuewang/Documents/DATA/atl/ATL_3B42V7_rain_accum.'+ str(year)+'*') rainfall_0 = [] for i in file: atl =SD(i,SDC.READ) rainfall = atl.select('RAIN_TOTAL') rainfall_value = rainfall.get() rainfall_0.append(rainfall_value) rainfall_single = np.array(rainfall_0) rainfall_anunal = sum(rainfall_single) rainfall_anunal_GMX = rainfall_anunal[280:320,340:400] ind = np.where(rainfall_anunal_GMX != 0) rf_annual = [] for i,j in zip(*ind): mm = rainfall_anunal_GMX[i,j] rf_annual.append(mm) rf_annual = np.array(rf_annual) c = np.mean(rf_annual) return c
def read_rrc(inpath): '''Read rrc data m*n from hdf file''' '''b1-5;b13-16 for MODIS Rrc Rrc_1238 Rrc_443-862 ozone senz solz for VIIRS rrc ''' hdf = SD(inpath, SDC.READ) #dts = sorted(hdf.datasets().keys()) modis_key = ['CorrRefl_01','CorrRefl_02','CorrRefl_03','CorrRefl_04','CorrRefl_05', 'CorrRefl_13','CorrRefl_14','CorrRefl_15','CorrRefl_16'] viirs_key = ['Rrc_443','Rrc_486','Rrc_551','Rrc_671','Rrc_745','Rrc_862','Rrc_1238'] mission = os.path.basename(inpath)[0] if mission =='A' or mission =='T':keys = modis_key elif mission=='V':keys = viirs_key else:keys = hdf.datasets().keys() for i,dt in enumerate(keys): print(i,dt) band = hdf.select(dt)[:,:] if i==0: limit = (band.shape[0],band.shape[1],len(keys)) rrc = np.zeros(limit,dtype = np.float) rrc[:,:,i] = band else: rrc[:,:,i] = band hdf.end() print(rrc.shape) return rrc
def main(cal_file, with_cp): from pyhdf.SD import SD if with_cp: cmd = 'cp %s /home/noel/scratch/' % (cal_file) print "running "+cmd os.system(cmd) filename = os.path.basename(cal_file) cal_file = '/home/noel/scratch/' + filename print 'Reading ' + cal_file vars = ['Latitude', 'Longitude', 'Total_Attenuated_Backscatter_532', 'Attenuated_Backscatter_1064', 'Perpendicular_Attenuated_Backscatter_532', 'Pressure', 'Temperature', 'Molecular_Number_Density', 'Tropopause_Height', 'Surface_Elevation'] hdf = SD(cal_file) for var in vars: print 'Reading ' + var hdf_var = hdf.select(var) data = hdf_var.get() hdf_var.endaccess() hdf.end() print 'ok.' if with_cp: print 'Removing '+filename cmd = 'rm -f /home/noel/scratch/' + filename os.system(cmd)
def rainfall_anunal_car(year): file = glob.glob('/Users/yuewang/Documents/DATA/atl/ATL_3B42V7_rain_accum.'+ str(year)+'*') rainfall_0 = [] for i in file: atl =SD(i,SDC.READ) rainfall = atl.select('RAIN_TOTAL') rainfall_value = rainfall.get() rainfall_0.append(rainfall_value) rainfall_single = np.array(rainfall_0) rainfall_anunal = sum(rainfall_single) rainfall_anunal_car = rainfall_anunal[238:286,372:476] # calculation none-zone mean value ind = np.where(rainfall_anunal_car != 0) rf_annual = [] for i,j in zip(*ind): mm = rainfall_anunal_car[i,j] rf_annual.append(mm) rf_annual = np.array(rf_annual) d = np.mean(rf_annual) return d
def load(self, fldname, **kwargs): """ Load Cali Current fields for a given day""" self._timeparams(**kwargs) if fldname == 'chl': filename = "/C%04i%03i_chl_mapped.hdf" % (self.yr, self.yd) #ncfieldname = 'chl_%04i_%03i' % (yr,yd) def scale(PV): return 10**(PV*0.015-2) elif fldname == 'sst': filename = "/M%04i%03i_sst_mapped.hdf" % (self.yr, self.yd) #ncfieldname = 'sst_%04i_%03i' % (yr,yd) def scale(PV): return PV*0.15000001-3 if not os.path.isfile(self.datadir + filename): print "Downloading " + filename self.download(fldname, self.jd) h = SD(self.datadir + filename,SDC.READ) ncfieldname = h.datasets().keys()[0] fld = h.select(ncfieldname) attr = fld.attributes() PV = fld[:].astype(np.float) PV[PV<0] = PV[PV<0]+256 PV[PV==0] = np.nan PV[PV==255] = np.nan setattr(self, fldname, scale(PV)[self.j1:self.j2, self.i1:self.i2])
def read_amsr_hdf4(filename): from pyhdf.SD import SD, SDC from pyhdf.HDF import HDF, HC import pyhdf.VS retv = AmsrObject() h4file = SD(filename, SDC.READ) datasets = h4file.datasets() attributes = h4file.attributes() #for idx,attr in enumerate(attributes.keys()): # print idx, attr for sds in ["Longitude", "Latitude", "High_res_cloud"]: data = h4file.select(sds).get() if sds in ["Longitude", "Latitude"]: retv.all_arrays[sds.lower()] = data.ravel() elif sds in ["High_res_cloud"]: lwp_gain = h4file.select(sds).attributes()['Scale'] retv.all_arrays["lwp_mm"] = data.ravel() * lwp_gain #print h4file.select(sds).info() h4file = HDF(filename, SDC.READ) vs = h4file.vstart() data_info_list = vs.vdatainfo() #print "1D data compound/Vdata" for item in data_info_list: #1D data compound/Vdata name = item[0] #print name if name in ["Time"]: data_handle = vs.attach(name) data = np.array(data_handle[:]) retv.all_arrays["sec1993"] = data data_handle.detach() else: pass #print name #data = np.array(data_handle[:]) #attrinfo_dic = data_handle.attrinfo() #factor = data_handle.findattr('factor') #offset = data_handle.findattr('offset') #print data_handle.factor #data_handle.detach() #print data_handle.attrinfo() h4file.close() #for key in retv.all_arrays.keys(): # print key, retv.all_arrays[key] return retv
def read(self, filename, **kwargs): """Read the data""" from pyhdf.SD import SD import datetime #print "*** >>> Read the hdf-eos file!" root = SD(filename) # Get all the Attributes: # Common Attributes, Data Time, # Data Structure and Scene Coordinates for key in root.attributes().keys(): self._eoshdf_info[key] = root.attributes()[key] # Start Time - datetime object starttime = datetime.datetime.strptime(self._eoshdf_info['Start Time'][0:13], "%Y%j%H%M%S") msec = float(self._eoshdf_info['Start Time'][13:16])/1000. self.starttime = starttime + datetime.timedelta(seconds=msec) # End Time - datetime object endtime = datetime.datetime.strptime(self._eoshdf_info['End Time'][0:13], "%Y%j%H%M%S") msec = float(self._eoshdf_info['End Time'][13:16])/1000. self.endtime = endtime + datetime.timedelta(seconds=msec) # What is the leading 'H' doing here? sensor_name = self._eoshdf_info['Sensor Name'][1:-1].lower() try: self.satid = EOS_SATELLITE[sensor_name] except KeyError: LOG.error("Failed setting the satellite id - sat-name = ", sensor_name) self.orbit = self._eoshdf_info['Orbit Number'] self.shape = (self._eoshdf_info['Number of Scan Control Points'], self._eoshdf_info['Number of Pixel Control Points']) #try: if 1: value = root.select(self.name) attr = value.attributes() data = value.get() self.attr = attr band = data if self.name in FLAGS_QUALITY: self.data = band else: nodata = attr['bad_value_scaled'] self.data = (np.ma.masked_equal(band, nodata) * attr['slope'] + attr['intercept']) value.endaccess() #except: # pass root.end() self.filled= True
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'Longwave Flux (2.5R)' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) data = nc.variables[DATAFIELD_NAME][:].astype(np.float64) else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:,:] # Set fillvalue and units. # See "CERES Data Management System ES-4 Collection Guide" [1] and a sample # image by NASA [2] for details. The fillvalue is 3.4028235E38. Here, we # just use the max of the data. fillvalue = np.max(data) data[data == fillvalue] = np.nan datam = np.ma.masked_array(data, mask=np.isnan(data)) # Set fillvalue and units. # See "CERES Data Management System ES-4 Collection Guide" [1] and a # sample image by NASA [2] for details. # The fillvalue is 3.4028235E38. Here, we use max value from the dataset. units = 'Watts/Meter^2' ysize, xsize = data.shape xinc = 360.0 / xsize yinc = 180.0 / ysize x0, x1 = (-180, 180) y0, y1 = (-90, 90) longitude = np.linspace(x0 + xinc/2, x1 - xinc/2, xsize) latitude = np.linspace(y0 + yinc/2, y1 - yinc/2, ysize) # Flip the latitude to run from 90 to -90 latitude = latitude[::-1] # The data is global, so render in a global projection. m = Basemap(projection='cyl', resolution='l', llcrnrlat=-90, urcrnrlat=90, llcrnrlon=-180, urcrnrlon=180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90.,90,45)) m.drawmeridians(np.arange(-180.,180,45), labels=[True,False,False,True]) m.pcolormesh(longitude, latitude, datam, latlon=True) cb = m.colorbar() cb.set_label(units) basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def get_lat_lon_thin_modis(satscene, options): """Read lat and lon. """ filename = satscene.time_slot.strftime("thin_MYD03.A%Y%j.%H%M.005.NRT.hdf") filename = os.path.join(options["dir"], filename) data = SD(filename) lat = data.select("Latitude") fill_value = lat.attributes()["_FillValue"] lat = np.ma.masked_equal(lat.get(), fill_value) lon = data.select("Longitude") fill_value = lon.attributes()["_FillValue"] lon = np.ma.masked_equal(lon.get(), fill_value) return lat, lon
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'sur_refl_b01_1' if USE_GDAL: import gdal GRID_NAME = 'MODIS_Grid_500m_2D' gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format( FILE_NAME, GRID_NAME, DATAFIELD_NAME) gdset = gdal.Open(gname) data = gdset.ReadAsArray().astype(np.float64) # Construct the grid. x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform() nx, ny = (gdset.RasterXSize, gdset.RasterYSize) x = np.linspace(x0, x0 + xinc * nx, nx) y = np.linspace(y0, y0 + yinc * ny, ny) xv, yv = np.meshgrid(x, y) # In basemap, the sinusoidal projection is global, so we won't use it. # Instead we'll convert the grid back to lat/lons. sinu = pyproj.Proj("+proj=sinu +R=6371007.181 +nadgrids=@null +wktext") wgs84 = pyproj.Proj("+init=EPSG:4326") lon, lat = pyproj.transform(sinu, wgs84, xv, yv) # Read the attributes. meta = gdset.GetMetadata() long_name = meta['long_name'] units = meta['units'] _FillValue = np.float(meta['_FillValue']) scale_factor = np.float(meta['scale_factor']) valid_range = [np.float(x) for x in meta['valid_range'].split(', ')] del gdset else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:, :].astype(np.double) # Read geolocation dataset from HDF-EOS2 dumper output. GEO_FILE_NAME = 'lat_MOD09GA.A2007268.h10v08.005.2007272184810_MODIS_Grid_500m_2D.output' GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], GEO_FILE_NAME) lat = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0]) lat = lat.reshape(data.shape) GEO_FILE_NAME = 'lon_MOD09GA.A2007268.h10v08.005.2007272184810_MODIS_Grid_1km_2D.output' GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], GEO_FILE_NAME) lon = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0]) lon = lon.reshape(data.shape) # Read attributes. attrs = data2D.attributes(full=1) lna = attrs["long_name"] long_name = lna[0] vra = attrs["valid_range"] valid_range = vra[0] fva = attrs["_FillValue"] _FillValue = fva[0] sfa = attrs["scale_factor"] scale_factor = sfa[0] ua = attrs["units"] units = ua[0] invalid = np.logical_or(data > valid_range[1], data < valid_range[0]) invalid = np.logical_or(invalid, data == _FillValue) data[invalid] = np.nan data = data / scale_factor data = np.ma.masked_array(data, np.isnan(data)) m = Basemap(projection='cyl', resolution='h', llcrnrlat=-2.5, urcrnrlat=12.5, llcrnrlon=-82.5, urcrnrlon=-67.5) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(0, 15, 5), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-80, -65, 5), labels=[0, 0, 0, 1]) m.pcolormesh(lon, lat, data, latlon=True) cb = m.colorbar() cb.set_label(units) basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, long_name)) fig = plt.gcf() # plt.show() pngfile = "{0}.sur_refl_b01_1.py.png".format(basename) fig.savefig(pngfile)
import cartopy.crs as ccrs import numpy as np from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER FILE_NAME = 'D:/f_work/E_public/E1_wangwei/data/MOD16A2/MOD16A2.A2001009.h28v06.006.2017068170401.hdf' from pyhdf.SD import SD, SDC #from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Get global attribute dictionnary attr = hdf.attributes(full=1) # Get dataset dictionnary tables = hdf.datasets() DATAFIELD_NAME = tuple(tables.keys())[0] # Read dataset. data_raw = hdf.select(DATAFIELD_NAME) data = data_raw[:, :].astype(np.double) # Read lat/lon. xdim = hdf.select('XDim') lon = xdim[:].astype(np.double) ydim = hdf.select('YDim') lat = ydim[:].astype(np.double) # Retrieve attributes. attrs = data_raw.attributes(full=1) lna = attrs["long_name"] long_name = lna[0] aoa = attrs["add_offset"] add_offset = aoa[0]
def cimport_data(self, idata_name=('V50M', 'U50M')): '''Import ten-year data. Args: idata_name: the data name. Returns: self.dict_ref_wind: imported data. ''' year_index = range(self.start_year, self.end_year + 1) site_num = len(self.site_index) zero = str(0) for each_siteth in range(1, site_num + 1): self.dict_ref_wind[each_siteth] = {} for each_year in year_index: self.dict_ref_wind[each_siteth][each_year] = {} for each_month in WindData.month_name: self.dict_ref_wind[each_siteth][each_year][each_month] = np.empty((0,24), np.float32) for each_year in year_index: if ((each_year % 400 == 0) or ((each_year % 4 == 0) and (each_year % 100 != 0))): year_feature = WindData.leap_year else: year_feature = WindData.nonleap_year for each_month in WindData.month_name: if ((each_year == 2010) and (each_month in WindData.spl_month2010)): fnames = self.file_name + WindData.fnamesa else: fnames = self.file_name + WindData.fnamesb day_s = year_feature[each_month][1] day_e = year_feature[each_month][2] if year_feature[each_month][3]: for each_day in range(day_s, day_e + 1): fname = fnames + str(each_year) + zero + str(each_day) + WindData.fnamee print fname fid = SD(fname) tmpv = fid.select(idata_name[0])[:, :, :] tmpu = fid.select(idata_name[1])[:, :, :] fid.end() for each_siteth in range(1, site_num + 1): tmpv_site = tmpv[:, self.site_index[each_siteth - 1][0], \ self.site_index[each_siteth - 1][1]].reshape(1, -1) tmpu_site = tmpu[:, self.site_index[each_siteth - 1][0], \ self.site_index[each_siteth - 1][1]].reshape(1, -1) tmpwind_site = self.cuv2speed(tmpv_site, tmpu_site) self.dict_ref_wind[each_siteth][each_year][each_month] = \ np.vstack((self.dict_ref_wind[each_siteth][each_year][each_month], tmpwind_site)) else: for each_day in range(day_s, day_e + 1): fname = fnames + str(each_year) + str(each_day) + WindData.fnamee print fname fid = SD(fname) tmpv = fid.select(idata_name[0])[:, :, :] tmpu = fid.select(idata_name[1])[:, :, :] fid.end() for each_siteth in range(1, site_num + 1): tmpv_site = tmpv[:, self.site_index[each_siteth - 1][0], \ self.site_index[each_siteth-1][1]].reshape(1, -1) tmpu_site = tmpu[:, self.site_index[each_siteth - 1][0], \ self.site_index[each_siteth-1][1]].reshape(1, -1) tmpwind_site = self.cuv2speed(tmpv_site,tmpu_site) self.dict_ref_wind[each_siteth][each_year][each_month] = \ np.vstack((self.dict_ref_wind[each_siteth][each_year][each_month], tmpwind_site)) return self.dict_ref_wind
def run(FILE_NAME): GEO_FILE_NAME = 'MYD03.A2002226.0000.005.2009193071127.hdf' GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], GEO_FILE_NAME) DATAFIELD_NAME = 'EV_Band26' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) # Read dataset. data = nc.variables[DATAFIELD_NAME][:,:].astype(np.float64) # Retrieve the geolocation data from MYD03 product. nc_geo = Dataset(GEO_FILE_NAME) longitude = nc_geo.variables['Longitude'][:] latitude = nc_geo.variables['Latitude'][:] # Retrieve attributes. units = nc.variables[DATAFIELD_NAME].radiance_units long_name = nc.variables[DATAFIELD_NAME].long_name # The scale and offset attributes do not have standard names in this # case, so we have to apply the scaling equation ourselves. scale_factor = nc.variables[DATAFIELD_NAME].radiance_scales add_offset = nc.variables[DATAFIELD_NAME].radiance_offsets valid_range = nc.variables[DATAFIELD_NAME].valid_range _FillValue = nc.variables[DATAFIELD_NAME]._FillValue valid_min = valid_range[0] valid_max = valid_range[1] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:,:].astype(np.double) hdf_geo = SD(GEO_FILE_NAME, SDC.READ) # Read geolocation dataset from MOD03 product. lat = hdf_geo.select('Latitude') latitude = lat[:,:] lon = hdf_geo.select('Longitude') longitude = lon[:,:] # Retrieve attributes. attrs = data2D.attributes(full=1) lna=attrs["long_name"] long_name = lna[0] aoa=attrs["radiance_offsets"] add_offset = aoa[0] fva=attrs["_FillValue"] _FillValue = fva[0] sfa=attrs["radiance_scales"] scale_factor = sfa[0] vra=attrs["valid_range"] valid_min = vra[0][0] valid_max = vra[0][1] ua=attrs["radiance_units"] units = ua[0] invalid = np.logical_or(data > valid_max, data < valid_min) invalid = np.logical_or(invalid, data == _FillValue) data[invalid] = np.nan data = (data - add_offset) * scale_factor data = np.ma.masked_array(data, np.isnan(data)) # The data is close to the equator in Africa, so a global projection is # not needed. m = Basemap(projection='cyl', resolution='l', llcrnrlat=-5, urcrnrlat=30, llcrnrlon=5, urcrnrlon=45) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(0, 50, 10), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(0, 50., 10), labels=[0, 0, 0, 1]) m.pcolormesh(longitude, latitude, data, latlon=True) cb=m.colorbar() cb.set_label(units, fontsize=8) basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, 'Radiance derived from ' + long_name), fontsize=11) fig = plt.gcf() # plt.show() pngfile = "{0}.evband26.py.png".format(basename) fig.savefig(pngfile)
# 批量读取HDF文件,提取AOD值,并将结果添加到列表中 file_name = os.listdir(file_path) # 文件名 i = 0 for hdf in file_name: i = i + 1 HDF_FILE_URL = file_path + hdf file = SD(HDF_FILE_URL) data_sets_dic = file.datasets() ''' #输出数据集名称 for idx, sds in enumerate(data_sets_dic.keys()): print(idx, sds) ''' sds_obj1 = file.select('Longitude') # 选择经度 sds_obj2 = file.select('Latitude') # 选择纬度 sds_obj3 = file.select('Optical_Depth_Land_And_Ocean') # 产品质量最高的AOD数据集 longitude = sds_obj1.get() # 读取数据 latitude = sds_obj2.get() aod = sds_obj3.get() writer = pd.ExcelWriter(output_file_path + '%s.xlsx' % hdf) longitude = pd.DataFrame(longitude) # 格式转换 latitude = pd.DataFrame(latitude) aod = pd.DataFrame(aod) longitude.to_excel(writer, sheet_name='longitude') latitude.to_excel(writer, sheet_name='latitude') aod.to_excel(writer, sheet_name='aod') writer.close()
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'Snow_Cover' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) # Subset the data to match the size of the swath geolocation fields. rows = slice(5, 4060, 10) cols = slice(5, 2708, 10) data = nc.variables['Snow_Cover'][rows, cols] latitude = nc.variables['Latitude'][:] longitude = nc.variables['Longitude'][:] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:, :].astype(np.float64) # Read geolocation dataset from HDF-EOS2 dumper output. GEO_FILE_NAME = 'lat_MOD10_L2.A2000065.0040.005.2008235221207.output' GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], GEO_FILE_NAME) lat = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0]) latitude = lat.reshape(data.shape) GEO_FILE_NAME = 'lon_MOD10_L2.A2000065.0040.005.2008235221207.output' GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], GEO_FILE_NAME) lon = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0]) longitude = lon.reshape(data.shape) # Draw a polar stereographic projection using the low resolution coastline # database. m = Basemap(projection='npstere', resolution='l', boundinglat=64, lon_0=0) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(60., 81, 10.)) m.drawmeridians(np.arange(-180., 181., 30.), labels=[True, False, False, True]) # Use a discretized colormap since we have only two levels. cmap = mpl.colors.ListedColormap(['grey', 'mediumblue']) bounds = [0, 19.5, 39] norm = mpl.colors.BoundaryNorm(bounds, cmap.N) m.pcolormesh(longitude, latitude, data, latlon=True, cmap=cmap, norm=norm) # Must reset the alpha level to opaque for the colorbar. # See http://stackoverflow.com/questions/4478725/... # .../partially-transparent-scatter-plot-but-with-a-solid-color-bar color_bar = plt.colorbar() color_bar.set_alpha(1) color_bar.set_ticks([9.75, 29.25]) color_bar.set_ticklabels(['missing data', 'ocean']) color_bar.draw_all() basename = os.path.basename(FILE_NAME) long_name = 'Snow Cover' plt.title('{0}\n{1}'.format(basename, long_name)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def hdf_read_example(): "hdf_read_example(): function to read a HDF file, take an extract and plot it in a graph. Default is ifile and sdsName" ifile = 'C:\\Users\\Janie\\Downloads\\MOD09GA.A2004154.h19v10.005.2008152141836.hdf' sdsName = 'sur_refl_b01_1' ofile = '' xmin = 0 xmax = 2000 ymin = 0 ymax = 2000 # default plot value plot = 1 if options.ifile: ifile = options.ifile if options.ofile: ofile = options.ofile if options.xmin: xmin = options.xmin if options.xmax: xmax = options.xmax if options.ymin: ymin = options.ymin if options.ymax: ymax = options.ymax if options.sdsName: sdsName = options.sdsName # read sds md = SD(ifile, SDC.READ) sds = md.select(sdsName) if options.v: # o/p file datasets sys.stderr.write('ifile: %s\n' % (ifile)) md.datasets() if options.plot: # o/p file datasets plot = 1 if plot: ex = sds[xmin:xmax, ymin:ymax] np.clip(ex, 0., 10000, out=ex) imshow(ex) plt.colorbar(drawedges="True") plt.title('%s' % (sdsName)) plt.ylabel("Row") plt.xlabel("Col") if ofile: # will save as emf, eps, pdf, png, ps, raw, rgba, svg, svgz plt.savefig(ofile) else: plt.show()
# In[4]: from a301.scripts import week5_test week5_test.main() # In[5]: # Read the lats and lons from the MYD03 file generic_rad = a301.data_dir / Path('rad_file_2018_10_1.hdf') generic_m3 = a301.data_dir / Path('m3_file_2018_10_1.hdf') print(f'reading {generic_m3}') m3_file = SD(str(generic_m3), SDC.READ) lats = m3_file.select('Latitude').get() lons = m3_file.select('Longitude').get() m3_file.end() # In[6]: #Read ch30 from the generic_rad file rad_file = SD(str(generic_rad), SDC.READ) ch30 = rad_file.select('ch30').get() rad_file.end() # # Calculate chan 30 brightness temperature #
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'NDSI_Snow_Cover' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) # Subset the data to match the size of the swath geolocation fields. rows = slice(5, 4060, 10) cols = slice(5, 2708, 10) data = nc.variables[DATAFIELD_NAME][rows, cols] latitude = nc.variables['Latitude'][:] longitude = nc.variables['Longitude'][:] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) # Use the following for low resolution. # This is good for low memory machine. rows = slice(5, 4060, 10) cols = slice(5, 2708, 10) data = data2D[rows, cols] latitude = hdf.select('Latitude')[:] longitude = hdf.select('Longitude')[:] # Read dataset attribute. attrs = data2D.attributes(full=1) lna = attrs["long_name"] long_name= lna[0] # Use the following for high resolution. # This may not work for low memory machine. # data = data2D[:,:] # Read geolocation dataset from HDF-EOS2 dumper output. # GEO_FILE_NAME = 'lat_MOD10_L2.A2000065.0040.005.2008235221207.output' # lat = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0]) # latitude = lat.reshape(data.shape) # GEO_FILE_NAME = 'lon_MOD10_L2.A2000065.0040.005.2008235221207.output' # lon = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0]) # longitude = lon.reshape(data.shape) # Draw a polar stereographic projection using the low resolution coastline # database. m = Basemap(projection='npstere', resolution='l', boundinglat=64, lon_0 = 0) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(60.,81,10.)) m.drawmeridians(np.arange(-180.,181.,30.), labels=[True,False,False,True]) # Key: = 0-100=ndsi snow, 200=missing data, 201=no decision, 211=night, 237=inland water, 239=ocean, 250=cloud, 254=detector saturated, 255=fill # Use a discretized colormap since we have only two levels. cmap = colors.ListedColormap(['purple', 'blue']) # Define the bins and normalize for discrete colorbar. bounds = np.array([211.0,237.0,239.0]) norm = mpl.colors.BoundaryNorm(bounds, cmap.N) m.pcolormesh(longitude, latitude, data, latlon=True, cmap=cmap, norm=norm) color_bar = plt.colorbar() # Must reset the alpha level to opaque for the colorbar. # See http://stackoverflow.com/questions/4478725/... # .../partially-transparent-scatter-plot-but-with-a-solid-color-bar color_bar.set_alpha(1) # Put label in the middle. color_bar.set_ticks([224.0, 238.0]) color_bar.set_ticklabels(['night', 'inland water']) color_bar.draw_all() basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, long_name)) fig = plt.gcf() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def load_modis(): dataPath = "/home/mrilee/data/MODIS/" # MOD03.A2005349.2120.061.2017187042837.hdf MOD05_L2.A2005349.2120.061.2017294065852.hdf # MOD03.A2005349.2125.061.2017187042720.hdf MOD05_L2.A2005349.2125.061.2017294065400.hdf modis_base = "MOD05_L2." # 1 modis_item = "A2005349.2120.061.2017294065852" # 1 modis_time_start = "2005-12-15T21:20:00" modis_item = "A2005349.2125.061.2017294065400" modis_time_start = "2005-12-15T21:25:00" modis_geofilename = "MOD03.A2005349.2125.061.2017187042720.hdf" modis_suffix = ".hdf" modis_filename = modis_base + modis_item + modis_suffix fmt_suffix = ".h5" workFileName = "sketchG." + modis_base + modis_item + fmt_suffix key_across = 'Cell_Across_Swath_1km:mod05' key_along = 'Cell_Along_Swath_1km:mod05' print('loading: ', dataPath + modis_filename) hdf = SD(dataPath + modis_filename, SDC.READ) ds_wv_nir = hdf.select('Water_Vapor_Near_Infrared') data = ds_wv_nir.get() # MODIS_Swath_Type_GEO/Geolocation_Fields/ # Latitude hdf_geo = SD(dataPath + modis_geofilename, SDC.READ) print('hg info: ', hdf_geo.info()) # for idx,sds in enumerate(hdf_geo.datasets().keys()): # print(idx,sds) # hdf_geo_ds = hdf_geo.select[''] # hdf_geo_swath = hdf_geo.select('MODIS_Swath_Type_GEO') # hdf_geo_swath_gf = hdf_geo_swath['Geolocation_Fields'] hdf_geo_lat = hdf_geo.select('Latitude').get() hdf_geo_lon = hdf_geo.select('Longitude').get() print('hgl type ', type(hdf_geo_lat)) print('hgl shape ', hdf_geo_lat.shape, hdf_geo_lon.shape) print('hgl dtype ', hdf_geo_lat.dtype) # exit() add_offset = ds_wv_nir.attributes()['add_offset'] scale_factor = ds_wv_nir.attributes()['scale_factor'] print('scale_factor = %f, add_offset = %f.' % (scale_factor, add_offset)) data = (data - add_offset) * scale_factor print('data mnmx: ', np.amin(data), np.amax(data)) nAlong = ds_wv_nir.dimensions()[key_along] nAcross = ds_wv_nir.dimensions()[key_across] print('ds_wv_nir nAlong,nAcross: ', nAlong, nAcross) dt = np.array([modis_time_start], dtype='datetime64[ms]') t_resolution = 26 # 5 minutes resolution? 2+6+10+6+6 tid = ps.from_utc(dt.astype(np.int64), t_resolution) # print(np.arange(np.datetime64("2005-12-15T21:20:00"),np.datetime64("2005-12-15T21:25:00"))) # exit() fill_value = ds_wv_nir.attributes()['_FillValue'] mod_lat = hdf_geo_lat.flatten() mod_lon = hdf_geo_lon.flatten() mod_dat = data.flatten() return mod_lon, mod_lat, mod_dat # load_modis
def run(FILE_NAME): DATAFIELD_NAME = 'Sea_Ice_by_Reflectance_SP' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) ncvar = nc.variables[DATAFIELD_NAME] data = ncvar[:].astype(np.float64) gridmeta = getattr(nc, 'StructMetadata.0') else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:, :].astype(np.float64) # Read global attribute. fattrs = hdf.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] # Construct the grid. The needed information is in a global attribute # called 'StructMetadata.0'. Use regular expressions to tease out the # extents of the grid. ul_regex = re.compile( r'''UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = ul_regex.search(gridmeta) x0 = np.float(match.group('upper_left_x')) y0 = np.float(match.group('upper_left_y')) lr_regex = re.compile( r'''LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = lr_regex.search(gridmeta) x1 = np.float(match.group('lower_right_x')) y1 = np.float(match.group('lower_right_y')) ny, nx = data.shape x = np.linspace(x0, x1, nx) y = np.linspace(y0, y1, ny) xv, yv = np.meshgrid(x, y) # Reproject into latlon # Reproject the coordinates out of lamaz into lat/lon. lamaz = pyproj.Proj("+proj=laea +a=6371228 +lat_0=-90 +lon_0=0 +units=m") wgs84 = pyproj.Proj("+init=EPSG:4326") lon, lat = pyproj.transform(lamaz, wgs84, xv, yv) # Use a south polar azimuthal equal area projection. m = Basemap(projection='splaea', resolution='l', boundinglat=-20, lon_0=180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90, 0, 15), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180, 180, 45), labels=[0, 0, 0, 1]) # Use a discretized colormap since we have only a few levels. # 0=missing data # 1=no decision # 11=night # 25=land # 37=inland water # 39=ocean # 50=cloud # 200=sea ice # 253=no input tile expected # 254=non-production mask" lst = [ '#727272', '#b7b7b7', '#ffff96', '#00ff00', '#232375', '#232375', '#63c6ff', '#ff0000', '#3f3f3f', '#000000' ] cmap = mpl.colors.ListedColormap(lst) bounds = [0, 1, 11, 25, 37, 39, 50, 200, 253, 254, 255] norm = mpl.colors.BoundaryNorm(bounds, cmap.N) # Render only a subset of the mesh. rows = slice(500, 4000, 5) cols = slice(500, 4000, 5) m.pcolormesh(lon[rows, cols], lat[rows, cols], data[rows, cols], latlon=True, cmap=cmap, norm=norm) color_bar = plt.colorbar() color_bar.set_ticks([0.5, 5.5, 18, 31, 38, 44.5, 125, 226.5, 253.5, 254.5]) color_bar.set_ticklabels([ 'missing', 'no decision', 'night', 'land', 'inland water', 'ocean', 'cloud', 'sea ice', 'no input tile\nexpected', 'non-production\nmask' ]) color_bar.draw_all() basename = os.path.basename(FILE_NAME) long_name = DATAFIELD_NAME plt.title('{0}\n{1}'.format(basename, long_name)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
""" import os import matplotlib as mpl import matplotlib.pyplot as plt from mpl_toolkits.basemap import Basemap import numpy as np from pyhdf.SD import SD, SDC # Open HDF4 file. FILE_NAME = 'A2002185000000.L2_LAC_SST.hdf' hdf = SD(FILE_NAME, SDC.READ) # Read dataset. DATAFIELD_NAME = 'sst' data = hdf.select(DATAFIELD_NAME) lat = hdf.select('latitude') latnp = lat[:, :] lon = hdf.select('longitude')[:, :] lonnp = lon[:, :] # Data size is nLw_443[2724][1968] while lat/lon size is lat[2724][247]. # We need to blow up lat/lon values using control point columns parameter # to match data size. # Read global attributes gattrs = hdf.attributes(full=1) nopcp = gattrs['Number of Pixel Control Points'] cpc = hdf.select('cntl_pt_cols') step1 = cpc[2] - cpc[1] step2 = cpc[nopcp[0] - 1] - cpc[nopcp[0] - 2]
def __init__(self, p_myd021km_filename): self.succeed = False self.filename = p_myd021km_filename # 初始化图像 # 读取modis——hdf try: myd021km_data = SD(self.filename) # 1.SDS Reading ev_250_SDS = myd021km_data.select('EV_250_Aggr1km_RefSB') # 经纬度 ev_500_SDS = myd021km_data.select('EV_500_Aggr1km_RefSB') # 经纬度 ev_1kmRef_SDS = myd021km_data.select('EV_1KM_RefSB') # 经纬度 ev_1km_SDS = myd021km_data.select('EV_1KM_Emissive') # 经纬度 '''250''' ev_250 = ev_250_SDS.get() # 2 for key, value in ev_250_SDS.attributes().items(): if key == 'reflectance_offsets': ev_250_offset = value if key == 'reflectance_scales': ev_250_scale = value '''500''' ev_500 = ev_500_SDS.get() # 2 for key, value in ev_500_SDS.attributes().items(): if key == 'reflectance_offsets': ev_500_offset = value if key == 'reflectance_scales': ev_500_scale = value '''100Ref''' ev_1kmRef = ev_1kmRef_SDS.get() # 2 for key, value in ev_1kmRef_SDS.attributes().items(): if key == 'radiance_offsets': ev_1kmRef_offset = value if key == 'radiance_scales': ev_1kmRef_scale = value '''100EV''' ev_1km = ev_1km_SDS.get() # 2 for key, value in ev_1km_SDS.attributes().items(): if key == 'radiance_offsets': ev_1km_offset = value if key == 'radiance_scales': ev_1km_scale = value # pack self.band_num = ev_250.shape[0] + ev_500.shape[ 0] + ev_1kmRef.shape[0] + ev_1km.shape[0] self.width = ev_250.shape[1] self.height = ev_250.shape[2] self.all_band = np.empty([self.band_num, self.width, self.height], dtype=np.uint16) self.scale = np.empty(self.band_num) self.offset = np.empty(self.band_num) now_band = 0 # 250 for idx in range(ev_250.shape[0]): self.all_band[now_band, ...] = ev_250[idx, ...] self.scale[now_band] = ev_250_scale[idx] self.offset[now_band] = ev_250_offset[idx] now_band += 1 # print(now_band) # 500 for idx in range(ev_500.shape[0]): self.all_band[now_band, ...] = ev_500[idx, ...] self.scale[now_band] = ev_500_scale[idx] self.offset[now_band] = ev_500_offset[idx] now_band += 1 # print(now_band) # 1kmref for idx in range(ev_1kmRef.shape[0]): self.all_band[now_band, ...] = ev_1kmRef[idx, ...] self.scale[now_band] = ev_1kmRef_scale[idx] self.offset[now_band] = ev_1kmRef_offset[idx] now_band += 1 # print(now_band) # 1km for idx in range(ev_1km.shape[0]): self.all_band[now_band, ...] = ev_1km[idx, ...] self.scale[now_band] = ev_1km_scale[idx] self.offset[now_band] = ev_1km_offset[idx] now_band += 1 # print(now_band) self.succeed = True except: print('Modis READ ERROR......(myd021km):' + self.filename) print("Unexpected error:", sys.exc_info()[0]) self.isValid = False
481410057:{ 'name': 'Socorro Hueco', 'lat': 31.6675000, 'lon': -106.2880000 }, 481410058:{ 'name': 'Skyline Park', 'lat': 31.8939133, 'lon': -106.4258270 }, 481410693:{ 'name': 'VanBuren', 'lat': 31.8133700, 'lon': -106.4645200 }, 481411011:{ 'name': 'El Paso Delta', 'lat': 31.7584760, 'lon': -106.4073460 }, 481411021:{ 'name': 'Ojo De Agua', 'lat': 31.8624700, 'lon': -106.5473000 }, } base_dir = './hdf/20180101-20191101/' flist = sorted(os.listdir(base_dir)) features = [] # do this for each file for fname in flist: f = SD(base_dir + fname, SDC.READ) ### select sds obj lat_obj = f.select('Latitude') lon_obj = f.select('Longitude') lst_obj = f.select('LST') vt_obj = f.select('View_time') ### get sds data lat_data = lat_obj.get() lon_data = lon_obj.get() lst_data = lst_obj.get() vt_data = vt_obj.get() # filter hdf points by loacation bound inbound = [] year, day, hour = infoFromFilename(fname) for i in range(np.shape(lat_data)[0]): for j in range(np.shape(lat_data)[1]):
print(swath_info) # %% [markdown] # * Read the projection details and data # %% myd03_file_name = list(a301_lib.sat_data.glob("MYD03*2105*hdf"))[0] print(myd03_file_name) myd02_file_name = list(a301_lib.sat_data.glob("MYD02*2105*hdf"))[0] print(myd02_file_name) # %% # Read the lats and lons from the MYD03 file print(f"reading {myd03_file_name}") m3_file = SD(str(myd03_file_name), SDC.READ) lats = m3_file.select("Latitude").get() lons = m3_file.select("Longitude").get() m3_file.end() # %% the_file = SD(str(myd02_file_name), SDC.READ) longwave_data = the_file.select("EV_1KM_Emissive") longwave_bands = the_file.select("Band_1KM_Emissive") band_nums = longwave_bands.get() ch30_index = np.searchsorted(band_nums, 30.0) print(f"make sure our index datatime in int64: {ch30_index.dtype}") ch30_index = int(ch30_index) scales = longwave_data.attributes()["radiance_scales"] offsets = longwave_data.attributes()["radiance_offsets"] ch30_scale = scales[ch30_index] ch30_offset = offsets[ch30_index]
modz = [] modlat = [] modlon = [] plt.figure() for i in range(len(filelist)): data = SD(filelist[i], SDC.READ) time = filelist[i][filelist[i].find('MYD021KM.A') + 18:filelist[i].find('MYD021KM.A') + 22] for j in range(len(coorlist)): if coorlist[j].find(time) > 0: coor = SD(coorlist[j], SDC.READ) selected_sds = data.select('EV_250_Aggr1km_RefSB') selected_sds_attributes = selected_sds.attributes() for key, value in selected_sds_attributes.iteritems(): if key == 'reflectance_scales': reflectance_scales_250_Aggr1km_RefSB = np.asarray(value) if key == 'reflectance_offsets': reflectance_offsets_250_Aggr1km_RefSB = np.asarray(value) sds_data_250_Aggr1km_RefSB = selected_sds.get() selected_sds = data.select('EV_500_Aggr1km_RefSB') selected_sds_attributes = selected_sds.attributes() for key, value in selected_sds_attributes.iteritems(): if key == 'reflectance_scales':
def run(FILE_NAME, OUT_PATH): # Identify the data field. DATAFIELD_NAME = 'Feature_Classification_Flags' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) # Subset the data to match the size of the swath geolocation fields. # Turn off autoscaling, we'll handle that ourselves due to presence of # a valid range. var = nc.variables[DATAFIELD_NAME] data = var[:, :] print data.shape # Read geolocation datasets. lat = nc.variables['Latitude'][:] long = nc.variables['Longitude'][:] lat = np.squeeze(lat) long = np.squeeze(long) print lat.shape else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:, :] print data.shape # Read geolocation datasets. latitude = hdf.select('Latitude') lat = latitude[:] longitude = hdf.select('Longitude') long = longitude[:] # Extract Feature Type only (1-3 bits) through bitmask. data_feature_type = data & 7 # if datanew[datanew = 3 == 3 # datashifted = data >> 9 #data_sub_feature_type = datashifted & 7 #data_feature_type[data_feature_type > 3] = 0; #data_feature_type[data_feature_type < 3] = 0; #data_feature_type[data_feature_type == 3] = 1; #data_sub_feature_type[data_feature_type != 1] = 0 # data = data.equal(5,5,-9) # data. print data_feature_type.shape # for t in range[0] #print bin(datashiftednew[0][i]) # for i in range(0,len(data[:])): # # for t in range(0,len(data[i][:])): # # print 'record' # print i # # print datanew[0][i] # print data[i][t] # print format(data[i][t], '#017b') # print data_feature_type[i][t] # print format(data_feature_type[i][t], '#017b') # # print format(datashifted[i][t], '#017b') # print data_sub_feature_type[i][t] # print format(data_sub_feature_type[i][t], '#017b') # # break # Subset latitude values for the region of interest (40N to 62N). # See the output of CAL_LID_L2_VFM-ValStage1-V3-02.2011-12-31T23-18-11ZD.hdf.py example. lat = lat[:] long = long[:] size = lat.shape[0] print lat.shape[0] # You can visualize other blocks by changing subset parameters. # data2d = data[3500:3999, 0:164] # 20.2km to 30.1km # data2d = data[3500:3999, 165:1164] # 8.2km to 20.2km # data2d = data[3500:4000, 1165:] # -0.5km to 8.2km low_data2d = data_feature_type[:, 1165: 1455] #8.2 to -.5km # -0.5km to 8.2km med_data2d2 = data_feature_type[:, 165: 365] # 20.2km to 8.2km 8.2km to 20.2km # high_data2d2 = data_feature_type[:, 0:55 ] # 30.2 to 20.km 8.2km to 20.2km #data2d = high_data2d2 #print med_data2d2.shape #print high_data2d2.shape # data2d = np.concatenate((high_data2d2, med_data2d2), axis=1) data2d = np.concatenate((med_data2d2, low_data2d), axis=1) # data2d = np.concatenate((data2d, low_data2d), axis=1) print data2d.shape #data3d = np.reshape(data2d, (size, 15, 290)) #data_sub_feature_type = data3d[:,0,:] # Focus on cloud (=2) data only. # data[data > 2] = 0; #data[data < 2] = 0; #data[data == 2] = 1; # Focus on aersol (=3) data only. # Generate altitude data according to file specification [1]. # alt = np.zeros(55) #alt = np.zeros(200) # alt = np.zeros(290) #alt = np.zeros(545) alt = np.zeros(490) # You can visualize other blocks by changing subset parameters. # 20.2km to 30.1km # for i in range (0, 54): # alt[i] = 20.2 + i*0.18; # -0.5km to 8.2km for i in range(0, 290): print i alt[i] = -0.5 + i * 0.03 print "breaK" for i in range(290, 490): print i alt[i] = 8.2 + (i - 289) * 0.06 # for i in range (490, 545): # # print i # alt[i] = 20.2 + (i-490)*0.180 # Contour the data on a grid of longitude vs. pressure latitude, altitude = np.meshgrid(lat, alt) # Make a color map of fixed colors. cmap = colors.ListedColormap( ['gray', 'blue', 'yellow', 'red', 'green', 'brown', 'black', 'white']) # cmap = colors.ListedColormap(['white', 'red', 'orange','magenta','green','blue','gray', ]) # Define the bins and normalize. bounds = np.linspace(0, 8, num=9) #print bounds levels = [0, .99, 1.99, 2.99, 3.99, 4.99, 5.99, 6.99, 7.99] print levels print cmap.colors norm = mpl.colors.BoundaryNorm(bounds, cmap.N) # print norm.boundaries print levels long_name = 'Feature Type (Bits 1-3) at Altitude 0 - 20.2km' basename = os.path.basename(FILE_NAME) plt.contourf(latitude, altitude, np.rot90(data2d, 1), cmap=cmap, levels=levels) plt.title('{0}\n{1}\n\n\n\n'.format(basename, long_name), fontsize=10) plt.xlabel('Latitude (degrees)', fontsize=8) plt.ylabel('Altitude (km)', fontsize=8) xticks = plt.xticks(fontsize="10") plt.yticks(fontsize="10") xticks.set_xticks([0, 1, 2, 3, 4, 5, 6, 7]) #plt.subplots_adjust(right=0.87) axtwo = plt.twiny() axtwo.set_xlabel("Longitude (degrees)", fontsize=8) #axtwo.xaxis.set_label_coords(.5, 1.01) intlong = long.astype(int).flatten() longspace = int(math.floor((len(long[:]) / 7))) print "long space" print longspace print len(long[:]) intlong_reversed_arr = intlong[::-1] axtwo.set_xticks([0, 1, 2, 3, 4, 5, 6, 7]) axtwo.set_xticklabels(intlong_reversed_arr[::longspace], fontsize=10) print intlong_reversed_arr[::longspace] print intlong_reversed_arr[0] print intlong_reversed_arr[3743] print lat[::longspace] print lat[0] print lat[3743] print lat[2000] print long[2000] fig = plt.gcf() fig.set_figheight(4) fig.set_figwidth(6) plt.subplots_adjust(right=0.65) fig.tight_layout() # Create a second axes for the discrete colorbar. ax2 = fig.add_axes([0.88, 0.1, 0.01, 0.6]) cb = mpl.colorbar.ColorbarBase(ax2, cmap=cmap, boundaries=bounds) cb.ax.set_yticklabels([ 'invalid', 'clear air', 'cloud', 'aerosol', 'stratospheric\nfeature', 'surface', 'subsurface', 'no signal' ], fontsize=6) # plt.show() pngfile = OUT_PATH + '/' + "{0}.v.py_feature_type_0-20.2km.png".format( basename) fig.savefig(pngfile) fig.clf()
import datetime starttime = datetime.datetime.now() #long running #do something other # 文件读取 HDF_FILR_URL = "1011-1.hdf" file = SD(HDF_FILR_URL) # print(file.info()) datasets_dic = file.datasets() ''' for idx, sds in enumerate(datasets_dic.keys()): print(idx, sds) ''' sds_obj1 = file.select('Longitude') # select sds sds_obj2 = file.select('Latitude') # select sds sds_obj3 = file.select('Optical_Depth_Land_And_Ocean') # select sds longitude = sds_obj1.get() # get sds data latitude = sds_obj2.get() # get sds data aod = sds_obj3.get() # get sds data # 参数设置 longitude = pd.DataFrame(longitude) latitude = pd.DataFrame(latitude) aod = pd.DataFrame(aod) # print(aod.shape,aod.info()) #矩阵大小 ''' #print(aod[450][650])#先列后行 ''' #监测站 坐标设置