def get_variable_names(self, filenames, data_type=None): try: from pyhdf.SD import SD from pyhdf.HDF import HDF except ImportError: raise ImportError("HDF support was not installed, please reinstall with pyhdf to read HDF files.") valid_variables = set([]) for filename in filenames: # Do VD variables datafile = HDF(filename) vdata = datafile.vstart() variables = vdata.vdatainfo() # Assumes that latitude shape == longitude shape (it should): # dim_length = [var[3] for var in variables if var[0] == 'Latitude'][0] for var in variables: # if var[3] == dim_length: valid_variables.add(var[0]) # Do SD variables: sd = SD(filename) datasets = sd.datasets() # if 'Height' in datasets: # valid_shape = datasets['Height'][1] for var in datasets: # if datasets[var][1] == valid_shape: valid_variables.add(var) return valid_variables
def main(): varname_to_rpn_name = { "precipitation": "PR", "relativeError": "RERR" } varnames = list(varname_to_rpn_name.keys()) target_dir = "/skynet3_rech1/huziy/from_hdf4" source_dir = "/st1_fs2/winger/Validation/TRMM/HDF_format" for f_name in os.listdir(source_dir): if not f_name.endswith("HDF"): continue path = os.path.join(source_dir, f_name) ds = SD(path) print(ds.datasets()) target_path = os.path.join(target_dir, f_name + ".rpn") r_obj = RPN(target_path, mode="w") for varname in varnames: var_data = ds.select(varname)[0, :, :] r_obj.write_2D_field( name=varname_to_rpn_name[varname], data=var_data, label=varname, grid_type="L", ig = [25, 25, 4013, 18012]) r_obj.close()
def write_interpolated(filename, f0, f1, fact, datasets): ''' interpolate two hdf files f0 and f1 using factor fact, and write the result to filename ''' hdf = SD(filename, SDC.WRITE|SDC.CREATE) for dataset in datasets: try: info = SD(f0).select(dataset).info() except: print >> stderr, 'Error loading %s in %s' % (dataset, f0) raise typ = info[3] shp = info[2] met0 = SD(f0).select(dataset).get() met1 = SD(f1).select(dataset).get() interp = (1-fact)*met0 + fact*met1 interp = interp.astype({ SDC.INT16: 'int16', SDC.FLOAT32: 'float32', SDC.FLOAT64: 'float64', }[typ]) # write sds = hdf.create(dataset, typ, shp) sds[:] = interp[:] sds.endaccess() hdf.end()
def read_rrc(inpath): '''Read rrc data m*n from hdf file''' '''b1-5;b13-16 for MODIS Rrc Rrc_1238 Rrc_443-862 ozone senz solz for VIIRS rrc ''' hdf = SD(inpath, SDC.READ) #dts = sorted(hdf.datasets().keys()) modis_key = ['CorrRefl_01','CorrRefl_02','CorrRefl_03','CorrRefl_04','CorrRefl_05', 'CorrRefl_13','CorrRefl_14','CorrRefl_15','CorrRefl_16'] viirs_key = ['Rrc_443','Rrc_486','Rrc_551','Rrc_671','Rrc_745','Rrc_862','Rrc_1238'] mission = os.path.basename(inpath)[0] if mission =='A' or mission =='T':keys = modis_key elif mission=='V':keys = viirs_key else:keys = hdf.datasets().keys() for i,dt in enumerate(keys): print(i,dt) band = hdf.select(dt)[:,:] if i==0: limit = (band.shape[0],band.shape[1],len(keys)) rrc = np.zeros(limit,dtype = np.float) rrc[:,:,i] = band else: rrc[:,:,i] = band hdf.end() print(rrc.shape) return rrc
def load_standard_lfm_hdf(filename): """ Load the standard formated hdf which we want to emulate""" f = SD(filename, SDC.READ) X_grid = f.select('X_grid') Y_grid = f.select('Y_grid') Z_grid = f.select('Z_grid') # x_grid is size nkp1,njp1,nip1 (nkp1,njp1,nip1) = X_grid[:].shape # The LFM reader expects i to vary fastest, then j, then k # However, the LFM pre-converted files store positions with k varying fastest (column-major) # Recommend saving in column-major format. If it fails, we can always switch. # i = 0; j = 0; k = 0 # print 'printing standard first row' # for i in range(nip1): # print X_grid[k,j,i]/R_e # print 'printing j sweep' # i = 0; j = 0; k = 0; # for j in range(njp1): # print X_grid[k,j,i]/R_e # print 'printing k sweep' # i = 0; j = 0; k = 0; # for k in range(nkp1): # print X_grid[k,j,i]/R_e print 'standard nip1,njp1,nkp1 =', nip1,njp1,nkp1 ni = nip1-1 nj = njp1-1 nk = nkp1-1 print 'standard ni,nj,nk =', ni,nj,nk
def main(cal_file, with_cp): from pyhdf.SD import SD if with_cp: cmd = 'cp %s /home/noel/scratch/' % (cal_file) print "running "+cmd os.system(cmd) filename = os.path.basename(cal_file) cal_file = '/home/noel/scratch/' + filename print 'Reading ' + cal_file vars = ['Latitude', 'Longitude', 'Total_Attenuated_Backscatter_532', 'Attenuated_Backscatter_1064', 'Perpendicular_Attenuated_Backscatter_532', 'Pressure', 'Temperature', 'Molecular_Number_Density', 'Tropopause_Height', 'Surface_Elevation'] hdf = SD(cal_file) for var in vars: print 'Reading ' + var hdf_var = hdf.select(var) data = hdf_var.get() hdf_var.endaccess() hdf.end() print 'ok.' if with_cp: print 'Removing '+filename cmd = 'rm -f /home/noel/scratch/' + filename os.system(cmd)
def export_multi_fluid_LFM(argv): if (len(argv) >= 2): input_filename = argv[0] output_filename = argv[1] print input_filename sd = SD(input_filename, SDC.READ) grid = get_corners(sd) timesteps = 0 # step = 1640000 for key in sd.datasets().keys(): shift = key.find('time_step') if shift == 0: if len(argv) == 3: step = argv[2] if key == 'time_step_'+str(step): export_timestep(sd, output_filename, key, grid) else: export_timestep(sd, output_filename, key, grid) timesteps += 1 print 'timesteps found in file:', timesteps else: print 'usage: python lfm_split.py input_multi_timestep_hdf output_filename_prefix step(optional)'
def test_1000m_to_250m(self): """Test the 1 km to 250 meter interpolation facility.""" # gfilename = \ # "/san1/test/data/modis/MOD03_A12278_113638_2012278145123.hdf" gfilename = "/local_disk/src/python-geotiepoints/tests/MOD03_A12278_113638_2012278145123.hdf" # result_filename = \ # "/san1/test/data/modis/250m_lonlat_results.npz" result_filename = "/local_disk/src/python-geotiepoints/tests/250m_lonlat_results.npz" from pyhdf.SD import SD from pyhdf.error import HDF4Error try: gdata = SD(gfilename) except HDF4Error: print("Failed reading eos-hdf file %s" % gfilename) return lats = gdata.select("Latitude")[0:50, :] lons = gdata.select("Longitude")[0:50, :] verif = np.load(result_filename) vlons = verif['lons'] vlats = verif['lats'] tlons, tlats = modis1kmto250m(lons, lats) self.assert_(np.allclose(tlons, vlons, atol=0.05)) self.assert_(np.allclose(tlats, vlats, atol=0.05))
def test_1000m_to_250m(self): """test the 1 km to 250 meter interpolation facility """ gfilename_hdf = "testdata/MOD03_A12278_113638_2012278145123.hdf" gfilename = "testdata/250m_lonlat_section_input.npz" result_filename = "testdata/250m_lonlat_section_result.npz" from pyhdf.SD import SD from pyhdf.error import HDF4Error gdata = None try: gdata = SD(gfilename_hdf) except HDF4Error: print "Failed reading eos-hdf file %s" % gfilename_hdf try: indata = np.load(gfilename) except IOError: return if gdata: lats = gdata.select("Latitude")[20:50, :] lons = gdata.select("Longitude")[20:50, :] else: lats = indata['lat'] / 1000. lons = indata['lon'] / 1000. verif = np.load(result_filename) vlons = verif['lon'] / 1000. vlats = verif['lat'] / 1000. tlons, tlats = modis1kmto250m(lons, lats) self.assert_(np.allclose(tlons, vlons, atol=0.05)) self.assert_(np.allclose(tlats, vlats, atol=0.05))
def run(FILE_NAME): DATAFIELD_NAME = 'dHat' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) var = nc.variables[DATAFIELD_NAME] # This datafield has scale factor and add offset attributes, but no # fill value. We'll turn off automatic scaling and do it ourselves. var.set_auto_maskandscale(False) data = nc.variables[DATAFIELD_NAME][:].astype(np.float64) # Retrieve scale/offset attributes. scale_factor = var.scale_factor add_offset = var.add_offset # Retrieve the geolocation data. latitude = nc.variables['geolocation'][:,:,0] longitude = nc.variables['geolocation'][:,:,1] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) ds = hdf.select(DATAFIELD_NAME) data = ds[:,:].astype(np.double) # Handle scale/osffset attributes. attrs = ds.attributes(full=1) sfa=attrs["scale_factor"] scale_factor = sfa[0] aoa=attrs["add_offset"] add_offset = aoa[0] # Retrieve the geolocation data. geo = hdf.select('geolocation') latitude = geo[:,:,0] longitude = geo[:,:,1] data = data / scale_factor + add_offset # Draw an equidistant cylindrical projection using the high resolution # coastline database. m = Basemap(projection='cyl', resolution='h', llcrnrlat=30, urcrnrlat = 36, llcrnrlon=121, urcrnrlon = 133) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(30, 37), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(121, 133, 2), labels=[0, 0, 0, 1]) m.pcolormesh(longitude, latitude, data, latlon=True) cb = m.colorbar() cb.set_label('Unit:mm') basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def rainfall_anunal_car(year): file = glob.glob('/Users/yuewang/Documents/DATA/atl/ATL_3B42V7_rain_accum.'+ str(year)+'*') rainfall_0 = [] for i in file: atl =SD(i,SDC.READ) rainfall = atl.select('RAIN_TOTAL') rainfall_value = rainfall.get() rainfall_0.append(rainfall_value) rainfall_single = np.array(rainfall_0) rainfall_anunal = sum(rainfall_single) rainfall_anunal_car = rainfall_anunal[238:286,372:476] # calculation none-zone mean value ind = np.where(rainfall_anunal_car != 0) rf_annual = [] for i,j in zip(*ind): mm = rainfall_anunal_car[i,j] rf_annual.append(mm) rf_annual = np.array(rf_annual) d = np.mean(rf_annual) return d
def rainfall_anunal_GMX(year): file = glob.glob('/Users/yuewang/Documents/DATA/atl/ATL_3B42V7_rain_accum.'+ str(year)+'*') rainfall_0 = [] for i in file: atl =SD(i,SDC.READ) rainfall = atl.select('RAIN_TOTAL') rainfall_value = rainfall.get() rainfall_0.append(rainfall_value) rainfall_single = np.array(rainfall_0) rainfall_anunal = sum(rainfall_single) rainfall_anunal_GMX = rainfall_anunal[280:320,340:400] ind = np.where(rainfall_anunal_GMX != 0) rf_annual = [] for i,j in zip(*ind): mm = rainfall_anunal_GMX[i,j] rf_annual.append(mm) rf_annual = np.array(rf_annual) c = np.mean(rf_annual) return c
def load(self, fldname, **kwargs): """ Load Cali Current fields for a given day""" self._timeparams(**kwargs) if fldname == 'chl': filename = "/C%04i%03i_chl_mapped.hdf" % (self.yr, self.yd) #ncfieldname = 'chl_%04i_%03i' % (yr,yd) def scale(PV): return 10**(PV*0.015-2) elif fldname == 'sst': filename = "/M%04i%03i_sst_mapped.hdf" % (self.yr, self.yd) #ncfieldname = 'sst_%04i_%03i' % (yr,yd) def scale(PV): return PV*0.15000001-3 if not os.path.isfile(self.datadir + filename): print "Downloading " + filename self.download(fldname, self.jd) h = SD(self.datadir + filename,SDC.READ) ncfieldname = h.datasets().keys()[0] fld = h.select(ncfieldname) attr = fld.attributes() PV = fld[:].astype(np.float) PV[PV<0] = PV[PV<0]+256 PV[PV==0] = np.nan PV[PV==255] = np.nan setattr(self, fldname, scale(PV)[self.j1:self.j2, self.i1:self.i2])
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'Longwave Flux (2.5R)' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) data = nc.variables[DATAFIELD_NAME][:].astype(np.float64) else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:,:] # Set fillvalue and units. # See "CERES Data Management System ES-4 Collection Guide" [1] and a sample # image by NASA [2] for details. The fillvalue is 3.4028235E38. Here, we # just use the max of the data. fillvalue = np.max(data) data[data == fillvalue] = np.nan datam = np.ma.masked_array(data, mask=np.isnan(data)) # Set fillvalue and units. # See "CERES Data Management System ES-4 Collection Guide" [1] and a # sample image by NASA [2] for details. # The fillvalue is 3.4028235E38. Here, we use max value from the dataset. units = 'Watts/Meter^2' ysize, xsize = data.shape xinc = 360.0 / xsize yinc = 180.0 / ysize x0, x1 = (-180, 180) y0, y1 = (-90, 90) longitude = np.linspace(x0 + xinc/2, x1 - xinc/2, xsize) latitude = np.linspace(y0 + yinc/2, y1 - yinc/2, ysize) # Flip the latitude to run from 90 to -90 latitude = latitude[::-1] # The data is global, so render in a global projection. m = Basemap(projection='cyl', resolution='l', llcrnrlat=-90, urcrnrlat=90, llcrnrlon=-180, urcrnrlon=180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90.,90,45)) m.drawmeridians(np.arange(-180.,180,45), labels=[True,False,False,True]) m.pcolormesh(longitude, latitude, datam, latlon=True) cb = m.colorbar() cb.set_label(units) basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def readhdf(filename, fieldname, ignoreE = True): try: hdf = SD(filename, SDC.READ) data = hdf.select(fieldname)[:].copy() hdf.end() except Exception, e: if not ignoreE:print e data = Dataset(filename)[fieldname][:].copy()
def print_dataset_2A12(*arg): FILE_NAME=arg[0]+'1B01.'+arg[1] hdf = SD(FILE_NAME, SDC.READ) 'List available SDS datasets' for ds in hdf.datasets(): print ds
def __init__(self, filename, filename_info, filetype_info): super(HDF4FileHandler, self).__init__(filename, filename_info, filetype_info) self.file_content = {} file_handle = SD(self.filename, SDC.READ) self._collect_attrs('', file_handle.attributes()) for k, v in file_handle.datasets().items(): self.collect_metadata(k, file_handle.select(k)) del file_handle
def aod(date, path, orbit, block): from constant import BAND_GREEN file_aerosol = _MIL2ASAE_fname(date, path, orbit) f = SD(file_aerosol) tau0 = f.select('RegMeanSpectralOptDepth').get()[block-1 , : , :, BAND_GREEN] tau0[tau0 == -9999] = np.mean(tau0[tau0 != -9999]) return tau0
def run(FILE_NAME): DATAFIELD_NAME = 'Temperature_MW_A' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) # The variable has a fill value, # so netCDF4 converts it to a float64 masked array for us. data = nc.variables[DATAFIELD_NAME][11,:,:] latitude = nc.variables['Latitude'][:] longitude = nc.variables['Longitude'][:] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # List available SDS datasets. # print hdf.datasets() # Read dataset. data3D = hdf.select(DATAFIELD_NAME) data = data3D[11,:,:] # Read geolocation dataset. lat = hdf.select('Latitude') latitude = lat[:,:] lon = hdf.select('Longitude') longitude = lon[:,:] # Handle fill value. attrs = data3D.attributes(full=1) fillvalue=attrs["_FillValue"] # fillvalue[0] is the attribute value. fv = fillvalue[0] data[data == fv] = np.nan data = np.ma.masked_array(data, np.isnan(data)) # Draw an equidistant cylindrical projection using the low resolution # coastline database. m = Basemap(projection='cyl', resolution='l', llcrnrlat=-90, urcrnrlat = 90, llcrnrlon=-180, urcrnrlon = 180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90., 120., 30.), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180., 181., 45.), labels=[0, 0, 0, 1]) m.pcolormesh(longitude, latitude, data, latlon=True, alpha=0.90) cb = m.colorbar() cb.set_label('Unit:K') basename = os.path.basename(FILE_NAME) plt.title('{0}\n {1} at TempPrsLvls=11'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.{1}.py.png".format(basename, DATAFIELD_NAME) fig.savefig(pngfile)
def setup_grid(self): """Setup necessary variables for grid """ if not os.path.isfile(self.datadir + self.gridfile): urllib.urlretrieve(self.dataurl + self.gridfile, self.datadir + self.gridfile) g = SD(self.datadir + self.gridfile, SDC.READ) self.llat = g.select('Latitude')[:] self.llon = g.select('Longitude')[:]
def print_dataset_1C21(*arg): FILE_NAME=arg[0]+'1C21.'+arg[1] print FILE_NAME hdf = SD(FILE_NAME, SDC.READ) 'List available SDS datasets' for ds in hdf.datasets(): print ds
def run(FILE_NAME): DATAFIELD_NAME = 'SurfaceTemperature' # The dataset is (6144 x 6400). Subset it to be around than 1K x 1K # Otherwise, the plot will skip processing some regions. rows = slice(0, 6144, 6) cols = slice(0, 6400, 6) if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) data = nc.variables[DATAFIELD_NAME][rows, cols] # Retrieve the geolocation data. latitude = nc.variables['Latitude'][rows, cols] longitude = nc.variables['Longitude'][rows, cols] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[rows,cols] # Read geolocation dataset. lat = hdf.select('Latitude') latitude = lat[rows,cols] lon = hdf.select('Longitude') longitude = lon[rows,cols] # Apply the fill value. The valid minimum is zero, although there's no # attribute. data[data < 0] = np.nan data = np.ma.masked_array(data, np.isnan(data)) # Render the data in a lambert azimuthal equal area projection. m = Basemap(projection='nplaea', resolution='l', boundinglat=60, lon_0=43) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(50, 90, 10), labels=[1, 0, 0, 1]) m.drawmeridians(np.arange(-180, 180, 30)) x, y = m(longitude, latitude) m.pcolormesh(x, y, data) cb = m.colorbar() cb.set_label('Unknown') basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def open_file(filename): A = SD(filename) # retrieve data SDS d = A.datasets() sds_name = d.keys()[0] # name of sds. Dictionary method. sds = A.select(sds_name) pin = A.attributes() return sds, pin
def landmask(self): if not hasattr(self, "_landmask"): filename = os.path.basename(self.landurl) if not os.path.isfile(self.datadir + filename): urllib.urlretrieve(self.dataurl + self.landurl, self.datadir + filename) h = SD(self.datadir + filename, SDC.READ) ncfieldname = h.datasets().keys()[0] self._landmask = h.select(ncfieldname)[:] == -1 return self._landmask
def read_var_ikslice(filename,var_name,i,k,tc,pc): hdffile = SD(filename,SDC.READ) ni = hdffile.select('X_grid').ni-1 nj = hdffile.select('X_grid').nj-1 if var_name not in ['br','btheta','bphi','vr','vtheta','vphi','bt','bp','vt','vp']: var=hdffile.select(var_name+'_').get(start=(k,0,i),count=(1,nj,1)).squeeze() else: if var_name in ['br','btheta','bphi','bt','bp']: bx=hdffile.select('bx_').get(start=(k,0,i),count=(1,nj,1)).squeeze() by=hdffile.select('by_').get(start=(k,0,i),count=(1,nj,1)).squeeze() bz=hdffile.select('bz_').get(start=(k,0,i),count=(1,nj,1)).squeeze() if var_name=='br': var = bx*cos(pc[k])*sin(tc) + by*sin(pc[k])*sin(tc) + bz*cos(tc) elif (var_name=='btheta' or var_name=='bt'): var = bx*cos(pc[k])*cos(tc) + by*sin(pc[k])*cos(tc) - bz*sin(tc) else: var =-bx*sin(pc[k]) + by*cos(pc[k]) else: vx=hdffile.select('vx_').get(start=(k,0,i),count=(1,nj,1)).squeeze() vy=hdffile.select('vy_').get(start=(k,0,i),count=(1,nj,1)).squeeze() vz=hdffile.select('vz_').get(start=(k,0,i),count=(1,nj,1)).squeeze() if var_name=='vr': var = vx*cos(pc[k])*sin(tc) + vy*sin(pc[k])*sin(tc) + vz*cos(tc) elif (var_name=='vtheta' or var_name=='vt'): var = vx*cos(pc[k])*cos(tc) + vy*sin(pc[k])*cos(tc) - vz*sin(tc) else: var =-vx*sin(pc[k]) + vy*cos(pc[k]) hdffile.end() return(var)
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'AlbedoLocal' hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data4D = hdf.select(DATAFIELD_NAME) # Convert 4-D data to 2-D data by subsetting. SOMBlockDim = 50; NBandDim = 0; data = data4D[SOMBlockDim,:,:,NBandDim].astype(np.double) # Read geolocation dataset from HDF-EOS2 dumper output. GEO_FILE_NAME = 'lat_MISR_TC_ALBEDO_P223_F05_lvl50.output' GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], GEO_FILE_NAME) lat = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0]) lat = lat.reshape(data.shape) GEO_FILE_NAME = 'lon_MISR_TC_ALBEDO_P223_F05_lvl50.output' GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], GEO_FILE_NAME) lon = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0]) lon = lon.reshape(data.shape) # Read attributes. attrs = data4D.attributes(full=1) fva=attrs["_FillValue"] _FillValue = fva[0] # Apply the fill value. data[data == _FillValue] = np.nan datam = np.ma.masked_array(data, mask=np.isnan(data)) # Set the limit for the plot. m = Basemap(projection='cyl', resolution='h', llcrnrlat=np.min(lat), urcrnrlat = np.max(lat), llcrnrlon=np.min(lon), urcrnrlon = np.max(lon)) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(np.floor(np.min(lat)), np.ceil(np.max(lat)), 1), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(np.floor(np.min(lon)), np.ceil(np.max(lon)), 1), labels=[0, 0, 0, 1]) m.pcolormesh(lon, lat, datam, latlon=True) cb = m.colorbar() cb.set_label('No Unit') basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1} at SOMBlockDim=50 NBandDim=0'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.l50.py.png".format(basename) fig.savefig(pngfile)
def get_lat_lon_modis(satscene, options): """Read lat and lon. """ filename_tmpl = satscene.time_slot.strftime(options["geofile"]) file_list = glob.glob(os.path.join(options["dir"], filename_tmpl)) if len(file_list) == 0: # Try in the same directory as the data data_dir = os.path.split(options["filename"])[0] file_list = glob.glob(os.path.join(data_dir, filename_tmpl)) if len(file_list) > 1: logger.warning("More than 1 geolocation file matching!") filename = max(file_list, key=lambda x: os.stat(x).st_mtime) coarse_resolution = 1000 elif len(file_list) == 0: logger.warning("No geolocation file matching " + filename_tmpl + " in " + options["dir"]) logger.debug("Using 5km geolocation and interpolating") filename = options["filename"] coarse_resolution = 5000 else: filename = file_list[0] coarse_resolution = 1000 logger.debug("Loading geolocation file: " + str(filename) + " at resolution " + str(coarse_resolution)) resolution = options["resolution"] data = SD(str(filename)) lat = data.select("Latitude") fill_value = lat.attributes()["_FillValue"] lat = np.ma.masked_equal(lat.get(), fill_value) lon = data.select("Longitude") fill_value = lon.attributes()["_FillValue"] lon = np.ma.masked_equal(lon.get(), fill_value) if resolution == coarse_resolution: return lat, lon cores = options["cores"] from geotiepoints import modis5kmto1km, modis1kmto500m, modis1kmto250m logger.debug("Interpolating from " + str(coarse_resolution) + " to " + str(resolution)) if coarse_resolution == 5000: lon, lat = modis5kmto1km(lon, lat) if resolution == 500: lon, lat = modis1kmto500m(lon, lat, cores) if resolution == 250: lon, lat = modis1kmto250m(lon, lat, cores) return lat, lon
def run(FILE_NAME): # Identify the HDF-EOS2 swath data file. DATAFIELD_NAME = 'radiances' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) data = nc.variables['radiances'][:,:,567] latitude = nc.variables['Latitude'][:] longitude = nc.variables['Longitude'][:] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data3D = hdf.select(DATAFIELD_NAME) data = data3D[:,:,567] # Read geolocation dataset. lat = hdf.select('Latitude') latitude = lat[:,:] lon = hdf.select('Longitude') longitude = lon[:,:] # Replace the filled value with NaN, replace with a masked array. data[data == -9999] = np.nan datam = np.ma.masked_array(data, np.isnan(data)) # Draw a polar stereographic projection using the low resolution coastline # database. m = Basemap(projection='spstere', resolution='l', boundinglat=-65, lon_0 = 180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-80., -50., 5.)) m.drawmeridians(np.arange(-180., 181., 20.), labels=[1, 0, 0, 1]) x, y = m(longitude, latitude) m.pcolormesh(x, y, datam) # See page 101 of "AIRS Version 5.0 Released Files Description" document [1] # for unit specification. units = 'mW/m**2/cm**-1/sr' cb = m.colorbar() cb.set_label('Unit:'+units) basename = os.path.basename(FILE_NAME) plt.title('{0}\n {1} at channel=567'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'bsst' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) # Subset the data to match the size of the swath geolocation fields. # Turn off autoscaling, we'll handle that ourselves due to non-standard # naming of the offset attribute. var = nc.variables[DATAFIELD_NAME] var.set_auto_maskandscale(False) lat = nc.variables['lat'] lon = nc.variables['lon'] else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. var = hdf.select(DATAFIELD_NAME) lat = hdf.select('lat') lon = hdf.select('lon') latitude = lat[::8] longitude = lon[::8] data = var[::8, ::8].astype(np.float64) # Apply the attributes. By inspection, fill value is 0 data[data==0] = np.nan data = data * var.scale_factor + var.add_off datam = np.ma.masked_array(data, mask=np.isnan(data)) m = Basemap(projection='cyl', resolution='l', llcrnrlat=-90, urcrnrlat=90, llcrnrlon=-180, urcrnrlon=180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90, 91, 45)) m.drawmeridians(np.arange(-180, 180, 45), labels=[True,False,False,True]) m.pcolormesh(longitude, latitude, datam, latlon=True) cax = plt.axes([0.92, 0.3, 0.01, 0.4]) cb = plt.colorbar(cax=cax) units = 'degrees-C' cb.set_label(units) basename = os.path.basename(FILE_NAME) fig = plt.gcf() # plt.show() long_name = 'Sea Surface Temperature ('+DATAFIELD_NAME+')' fig.suptitle('{0}\n{1}'.format(basename, long_name)) # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def load_thin_modis(satscene, options): """Read modis data from file and load it into *satscene*. """ filename = satscene.time_slot.strftime("thin_MYD021KM.A%Y%j.%H%M.005.NRT.hdf") filename = os.path.join(options["dir"], filename) data = SD(filename) datasets = ['EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB', 'EV_1KM_Emissive'] for dataset in datasets: subdata = data.select(dataset) band_names = subdata.attributes()["band_names"].split(",") if len(satscene.channels_to_load & set(band_names)) > 0: uncertainty = data.select(dataset+"_Uncert_Indexes") if dataset == 'EV_1KM_Emissive': array = calibrate_tb(subdata, uncertainty) else: array = calibrate_refl(subdata, uncertainty) for (i, band) in enumerate(band_names): if band in satscene.channels_to_load: satscene[band] = array[i] mda = data.attributes()["CoreMetadata.0"] orbit_idx = mda.index("ORBITNUMBER") satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116] lat, lon = get_lat_lon(satscene, None) from pyresample import geometry satscene.area = geometry.SwathDefinition(lons=lon, lats=lat) # trimming out dead sensor lines if satscene.satname == "aqua": for band in ["6", "27"]: if not satscene[band].is_loaded() or satscene[band].data.mask.all(): continue width = satscene[band].data.shape[1] height = satscene[band].data.shape[0] indices = satscene[band].data.mask.sum(1) < width if indices.sum() == height: continue satscene[band] = satscene[band].data[indices, :] satscene[band].area = geometry.SwathDefinition( lons=satscene.area.lons[indices,:], lats=satscene.area.lats[indices,:]) satscene[band].area.area_id = ("swath_" + satscene.fullname + "_" + str(satscene.time_slot) + "_" + str(satscene[band].shape) + "_" + str(band))
def set_data_shape(self): """ use dataset set self.data_shape :return: """ # 如果分辨率是 1000 米 if self.resolution == 1000: satellite_type = ['AQUA', 'TERRA'] if self.satellite in satellite_type: h4File = SD(self.in_file, SDC.READ) in_data_r250 = h4File.select('EV_250_Aggr1km_RefSB').get() self.data_shape = in_data_r250.shape[1:] else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) else: raise ValueError("Cant handle this resolution: ".format( self.resolution))
def set_file_attr(self): """ 根据 self.file_level_1 获取 L1 文件的属性 set self.level_1_attr 储存格式是字典 :return: """ if self.resolution == 1000: satellite_type1 = ['FY1C', 'FY1D'] if self.satellite in satellite_type1: hdf4 = SD(self.in_file, SDC.READ) self.file_attr = attrs2dict(hdf4.attributes()) else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) else: raise ValueError("Cant handle this resolution: ".format( self.resolution))
def SD_wrapper(file_path): ds = None if 's3://' == file_path[0:5]: s3 = parse_s3_url(file_path) s3_client = boto3.client('s3') ds,tmpdir,fullFilename = hdf4_dataset_from_s3(s3_client,s3['bucket_name'],s3['prefix']+s3['prefix_end']+s3['resource'],filename=s3['resource']) else: ds = SD(file_path) return ds
def __init__(self, path): self.path = path self.file_handle = HDF(str(path)) self.scientific_dataset = SD(str(path)) datasets = self.scientific_dataset.datasets() dataset_dict = { key: Dataset(weakref.ref(self), key, *info) for key, info in datasets.items() } self.datasets = dataset_dict self.vdata_table = VS(self.file_handle) vdata_dict = { info[0]: VData(weakref.ref(self), *info) for info in self.vdata_table.vdatainfo() } self.vdata = vdata_dict
def __init__(self, filename): self.filename = filename self._succeed = True try: self.science_data = SD(self.filename) except: print('Modis READ ERROR......(myd021km):' + self.filename) print("Unexpected error:", sys.exc_info()[0]) self._succeed = False
def Get_h4_nc_Subdatasets(h4_nc_path, target_datasets): # 以只读方式打开文件 hc_dataset = SD(h4_nc_path, SDC.READ) # 判断存放目标数据集名称的列表是否为空 if len(target_datasets) == 0: print('The target dataset names is null.') return subdatasets_dict = dict() for dataset_name in target_datasets: temp_subdataset = hc_dataset.select(dataset_name) # 获取的是一个pyhdf.SD.SDS对象 if temp_subdataset is None: print("The subdataset:%s don't exist." % (dataset_name)) continue subdatasets_dict[dataset_name] = temp_subdataset return subdatasets_dict
def is_daily_product(hdf_filename): hdf_file = SD(hdf_filename, SDC.READ) # get CoreMetadata.0 attribute (which is a very long string) and just extract and analyze the substring # with the DAYNIGHT info: core_metadata0_string = hdf_file.attributes()['CoreMetadata.0'] daynight_start_index = core_metadata0_string.find('DAYNIGHT') if daynight_start_index != -1: #print('test: |' + core_metadata0_string[daynight_start_index:daynight_start_index+120] + '|') daynight_info_block = core_metadata0_string[daynight_start_index:daynight_start_index+120] is_daily = (daynight_info_block.find('\"Day\"') != -1) #print(is_daily) return is_daily else: return false
def HDFread(filename, variable, Class=None): """ Extract the data for non scientific data in V mode of hdf file """ hdf = HDF(filename, HC.READ) # Initialize the SD, V and VS interfaces on the file. sd = SD(filename) vs = hdf.vstart() v = hdf.vgstart() # Encontrar el puto id de las Geolocation Fields if Class == None: ref = v.findclass('SWATH Vgroup') else: ref = v.findclass(Class) # Open all data of the class vg = v.attach(ref) # All fields in the class members = vg.tagrefs() nrecs = [] names = [] for tag, ref in members: # Vdata tag vd = vs.attach(ref) # nrecs, intmode, fields, size, name = vd.inquire() nrecs.append(vd.inquire()[0]) # number of records of the Vdata names.append(vd.inquire()[-1]) # name of the Vdata vd.detach() idx = names.index(variable) var = vs.attach(members[idx][1]) V = var.read(nrecs[idx]) var.detach() # Terminate V, VS and SD interfaces. v.end() vs.end() sd.end() # Close HDF file. hdf.close() return np.array(V).ravel()
def read_band(file_path): DATAFIELD_NAME = 'EV_250_RefSB' hdf = SD(file_path, SDC.READ) file_name = path.splitext(path.basename(file_path))[0] band1_path = path.join(tmp_dir, file_name + "_band1.jpg") band2_path = path.join(tmp_dir, file_name + "_band2.jpg") if path.exists(band1_path) and path.exists(band2_path): pass else: # Read dataset. data = hdf.select(DATAFIELD_NAME) data2d = data.get().astype("double") # Retrieve attributes. attrs = data.attributes(full=1) aoa = attrs["radiance_offsets"] add_offset = aoa[0] fva = attrs["_FillValue"] _FillValue = fva[0] sfa = attrs["radiance_scales"] scale_factor = sfa[0] vra = attrs["valid_range"] valid_min = vra[0][0] valid_max = vra[0][1] invalid = np.logical_or(data2d > valid_max, data2d < valid_min) invalid = np.logical_or(invalid, data2d == _FillValue) data2d[invalid] = np.nan data2d[0] = (data2d[0] - add_offset[0]) * scale_factor[0] data2d[1] = (data2d[1] - add_offset[1]) * scale_factor[1] data2d = np.ma.masked_array(data2d, np.isnan(data2d)) band1_img = Image.fromarray(data2d[0].astype("uint8")) band2_img = Image.fromarray(data2d[1].astype("uint8")) band1_img.save(band1_path) band2_img.save(band2_path) return band1_path, band2_path
def _readGranule(self, filename): """Reads one GOCI granule.""" # Don't fuss if the file cannot be opened # --------------------------------------- try: if self.verb: print "[] Working on " + filename hfile = SD(filename) except HDF4Error: if self.verb > 2: print "- %s: not recognized as an HDF file" % filename return # Read select variables (reshape to allow concatenation later) # ------------------------------------------------------------ for sds in self.SDS: sds_ = sds.replace(' ', '_') if sds == 'Turbid_index': try: v = hfile.select(sds).get() except HDF4Error: v = np.full(hfile.select('AOD_550nm').get().shape, -999) else: v = hfile.select(sds).get() rank = len(v.shape) if rank == 2: self.__dict__[sds_].append(flipud(v)) else: self.__dict__[sds_].append(v) # Satellite name # -------------- if self.sat is None: self.sat = 'GOCI' # Collection # ---------- if self.col is None: col = 1 self.col = "%03d" % col
def read(self, filename, **kwargs): """Read the data""" from pyhdf.SD import SD import datetime # print "*** >>> Read the hdf-eos file!" root = SD(filename) # Get all the Attributes: # Common Attributes, Data Time, # Data Structure and Scene Coordinates for key in root.attributes().keys(): self._eoshdf_info[key] = root.attributes()[key] # Start Time - datetime object starttime = datetime.datetime.strptime(self._eoshdf_info['Start Time'][0:13], "%Y%j%H%M%S") msec = float(self._eoshdf_info['Start Time'][13:16]) / 1000. self.starttime = starttime + datetime.timedelta(seconds=msec) # End Time - datetime object endtime = datetime.datetime.strptime(self._eoshdf_info['End Time'][0:13], "%Y%j%H%M%S") msec = float(self._eoshdf_info['End Time'][13:16]) / 1000. self.endtime = endtime + datetime.timedelta(seconds=msec) # What is the leading 'H' doing here? sensor_name = self._eoshdf_info['Sensor Name'][1:-1].lower() try: self.satid = EOS_SATELLITE[sensor_name] except KeyError: LOG.error("Failed setting the satellite id - sat-name = ", sensor_name) self.orbit = self._eoshdf_info['Orbit Number'] self.shape = (self._eoshdf_info['Number of Scan Control Points'], self._eoshdf_info['Number of Pixel Control Points']) # try: if 1: value = root.select(self.name) attr = value.attributes() data = value.get() self.attr = attr band = data if self.name in FLAGS_QUALITY: self.data = band else: nodata = attr['bad_value_scaled'] self.data = (np.ma.masked_equal(band, nodata) * attr['slope'] + attr['intercept']) value.endaccess() # except: # pass root.end() self.filled = True
def extract_attributes(self): # extract the attributes from the data file try: granule = SD(self.data_path, 1) except TypeError: granule = SD(self.data_path.encode("utf-8"), 1) self.attributes = granule.attributes() granule.end()
def load_data_from_files(filename): if not os.path.exists(filename): print("File {} does not exist, cannot load data.".format(filename)) return elif not HDF.ishdf(filename): print("File {} is not in hdf4 file format, cannot load data.".format( filename)) return f = SD(filename, SDC.READ) data_field = None for i, d in enumerate(f.datasets()): # print("{0}. {1}".format(i+1,d)) if "NDVI" in d: data_field = d ndvi_data = f.select(data_field) data = np.array(ndvi_data.get()) return data
def DescribeHDFvar(filename, variable): """ Describes the info and atributes of HDF variable IMPUTS filename : complete path and filename variable : name of the variable to describe, type STR """ file = SD(filename, SDC.READ) print('---------- ' + variable + ' ----------') sds_obj = file.select(variable) Var = sds_obj.get() sds_info = sds_obj.info() print(Var.shape) print(sds_info) print(sds_info[0], sds_info[1]) print('sds attributes') pprint.pprint(sds_obj.attributes()) file.end()
def __getHdfObj(self): """ Return an insance of pyhdf.SD for read """ try: hdf = SD(self.filename, SDC.READ) except HDF4Error as msg: sys.stderr.write('HDF4Error opening "' + self.filename + '" for read. Error message follows:' + str(msg)) sys.stderr.write('Trying operation again.') sys.stderr.flush() # Maybe the I/O was busy. Sleep for a bit & try again. import time time.sleep(1) hdf = SD(self.filename, SDC.READ) return hdf
def set_data_shape(self): """ 根据 self.satellite set self.dataset_shape :return: """ # 如果分辨率是 1000 米 if self.resolution == 1000: satellite_type1 = ['FY1C', 'FY1D'] if self.satellite in satellite_type1: hdf4 = SD(self.in_file, SDC.READ) dn = hdf4.select('Earth_View')[0] self.data_shape = dn.shape else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) # elif self.resolution == 250: else: raise ValueError("Cant handle this resolution: ".format( self.resolution))
def set_file_attr(self): """ get hdf5 file attrs self.file_attr :return: """ if self.resolution == 1000: satellite_type = ['AQUA', 'TERRA'] if self.satellite in satellite_type: try: h4r = SD(self.in_file, SDC.READ) self.file_attr = attrs2dict(h4r.attributes()) except Exception as e: print(str(e)) else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) else: raise ValueError("Cant handle this resolution: ".format( self.resolution))
def create_lcm(self): for H, V in product(Hs, Vs): mcd_file = self.validate_file('MCD', self.year, self.fday, H, V) if mcd_file is not None: # open file mcdds = SD(os.path.join(mcd_data, mcd_file), SDC.READ) # load land cover mask lcm = mcdds.select(lc_type).get() mcdds.end() self.lcm[V * 2400:(V + 1) * 2400, H * 2400:(H + 1) * 2400] = lcm return self.lcm
def GetData(file, datafield): ''' GetData(file,datafield) : read the data labeled datafield in file ================================================================= Retrieve data from a HDF file input : ------ file : name of input file datafield : label of required data field output: ------ data3D : output array ''' hdf = SD(file, SDC.READ) data3D = hdf.select(datafield) return data3D
def read_var(fname, varname, normalized=False): f = SD(fname, SDC.READ) phi = f.select('fakeDim0')[:] theta = f.select('fakeDim1')[:] r = f.select('fakeDim2')[:] var = f.select('Data-Set-2')[:] f.end() if normalized: return (phi, theta, r, var) else: return (phi, theta, r * mas_units['length'], var * mas_units[varname])
def set_data_shape(self): """ use dataset set self.data_shape :return: """ # 如果分辨率是 13500 米 if self.resolution == 13500: satellite_type = ['AQUA', 'TERRA'] if self.satellite in satellite_type: h4r = SD(self.in_file, SDC.READ) in_data_r250 = h4r.select('Latitude').get() self.data_shape = in_data_r250.shape h4r.end() else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) else: raise ValueError("Cant handle this resolution: ".format( self.resolution))
def load_hdf(filepath, year, day): try: hdf = SD(filepath, SDC.READ) except HDF4Error: print(f"Error {filepath}, downloading") os.remove(filepath) download_main(year, start_day=day, day_only=True) return load_hdf(filepath, year, day) else: return hdf
class Hdf4File: """ Base class for file formats using HDF4File format. The :class:`Hdf4File` wraps around the pyhdf.SD class to implement RAII. """ def __init__(self, filename): """ Open an HDF4 file for reading. Arguments: filename(str): The path to the file to open. """ from pyhdf.SD import SD, SDC self.filename = filename self.file_handle = SD(self.filename, SDC.READ) def __del__(self): self.file_handle.end()
def get_lonlat(self, resolution, time_slot, cores=1): """Read lat and lon. """ if (resolution, time_slot) in self.areas: return self.areas[resolution, time_slot] logger.debug("generating lon, lat at %d", resolution) if self.geofile is not None: coarse_resolution = 1000 filename = self.geofile else: coarse_resolution = 5000 logger.info("Using 5km geolocation and interpolating") filename = (self.datafiles.get(1000) or self.datafiles.get(500) or self.datafiles.get(250)) logger.debug("Loading geolocation from file: " + str(filename) + " at resolution " + str(coarse_resolution)) data = SD(str(filename)) lat = data.select("Latitude") fill_value = lat.attributes()["_FillValue"] lat = np.ma.masked_equal(lat.get(), fill_value) lon = data.select("Longitude") fill_value = lon.attributes()["_FillValue"] lon = np.ma.masked_equal(lon.get(), fill_value) if resolution == coarse_resolution: self.areas[resolution] = lon, lat return lon, lat from geotiepoints import modis5kmto1km, modis1kmto500m, modis1kmto250m logger.debug("Interpolating from " + str(coarse_resolution) + " to " + str(resolution)) if coarse_resolution == 5000: lon, lat = modis5kmto1km(lon, lat) if resolution == 500: lon, lat = modis1kmto500m(lon, lat, cores) if resolution == 250: lon, lat = modis1kmto250m(lon, lat, cores) self.areas[resolution, time_slot] = lon, lat return lon, lat
class HDF4File: """ Simplified interface for reading HDF4 files. It combines the SD and VS low-level interfaces. """ # Attributes: # variables(``list``): List of strings of variable names contained in # this file. def __init__(self, path): self.path = path self.file_handle = HDF(str(path)) self.scientific_dataset = SD(str(path)) datasets = self.scientific_dataset.datasets() dataset_dict = { key: Dataset(weakref.ref(self), key, *info) for key, info in datasets.items() } self.datasets = dataset_dict self.vdata_table = VS(self.file_handle) vdata_dict = { info[0]: VData(weakref.ref(self), *info) for info in self.vdata_table.vdatainfo() } self.vdata = vdata_dict def __del__(self): if self.file_handle: self.file_handle.close() self.file_handle = None @property def variables(self): """ Names of the variables available in this file. """ return list(self.datasets.keys()) + list(self.vdata.keys()) def __getattribute__(self, name): try: return object.__getattribute__(self, name) except AttributeError as error: datasets = object.__getattribute__(self, "datasets") if name in datasets: return datasets[name] vdata = object.__getattribute__(self, "vdata") if name in vdata: return vdata[name] raise error def __repr__(self): return f"HDF4File({self.path})"
def get_spectral_response(self): """ return 光谱波数和响应值,1维,2维 """ if self.resolution == 13500: satellite_type1 = ['AQUA', 'TERRA'] if self.satellite in satellite_type1: try: h4r = SD(self.in_file, SDC.READ) data_pre = h4r.select('radiances').get() h4r.end() # 非法值变成0 卷积时不贡献 condition = np.logical_or(data_pre < 0, data_pre > 200) idx = np.where(condition) # print len(idx[0]) if len(idx[0] > 0): data_pre[idx] = 0 # 单位转换 except Exception as e: print(str(e)) response = data_pre # 暂时取一个观测的光谱波数 file_name = '{}_{}_SRF_Pub.txt'.format(self.satellite, self.sensor) data_file = os.path.join(g_main_path, 'SRF', file_name) dtype = { 'names': ('wave_length', 'response'), 'formats': ('f4', 'f4') } if os.path.isfile(data_file): datas = np.loadtxt(data_file, dtype=dtype) # 波长转波数 wave_number = datas['wave_length'] else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) else: raise ValueError( 'Cant read this data, please check its resolution: {}'.format( self.in_file)) return wave_number, response
def get_sds(self, fieldnames=None): """Return specific or all SDS in the hdf file as dictionary. SDS arrays can be accessed using the 'data' key. Note that no scaling is applied to the data in get() method (use get_scaled() to achieve that). However, the scaling and missing data information can be accessed using the following keys: 'scale_factor' 'add_offset' '_FillValue' """ # Convert scalar fieldnames to sequence if isinstance(fieldnames, basestring): fieldnames = (fieldnames, ) # Open file to read SDs try: h4 = SD(self.filename, mode=SDC.READ) sclinfo = None if 'Slope_and_Offset_Usage' in h4.attributes(): sclinfo = 'Slope_and_Offset_Usage' # Get all available SDS from file if fieldnames in not given if fieldnames is None: fieldnames = [] for key in sorted(h4.datasets()): fieldnames.append(key) # Create and empty dataset dictionary with all available # fields fill in data from SDS sds = dict.fromkeys(fieldnames, {}) for key in sds: attrs = h4.select(key).attributes() if sclinfo: attrs[sclinfo] = h4.attributes()[sclinfo] sds[key] = attrs sds[key]['data'] = h4.select(key).get() # Close hdf interface h4.end() except HDF4Error as e: raise HDF4Error(e) # Return raw (possibly un-calibrated) SDS/attributes dictionary return sds
def retrieve_1C21(*arg): hdf = SD(arg[0], SDC.READ) Lon, Lat, date_beg, date_end, st, en = retrieve_ancillary(hdf) ' osRain includes rays #20 to #30' # data=hdf.select('osRain')[st:en,:,0] # [dBZ] ' normalSample correspond to dBZ in 140 levels' data = hdf.select('normalSample')[st:en, :, 100] # [dBZ*100] data = data.astype(float) data[data <= -32700.] = np.nan data = data / 100. # [dBZ] # foo = hdf.select('normalSample')[:, :, :] # print foo.shape # return Lon[:,20:31], Lat[:,20:31], [date_beg,date_end], osRain data = np.ma.masked_array(data, np.isnan(data)) return Lon[st:en, :], Lat[st:en, :], [date_beg, date_end], data
def main(yaml_file): in_cfg = ReadInYaml(yaml_file) job_name = in_cfg.job_name ymd = in_cfg.ymd in_path = in_cfg.ipath out_path = in_cfg.opath for eachfile in in_path: h4r = SD(eachfile) ndsi_data = h4r.select('Day_CMG_Snow_Cover')[:] h4r.end() file_name = os.path.basename(eachfile) + '.png' out_fig = os.path.join(out_path, file_name) if not os.path.isdir(out_path): os.makedirs(out_path) print(out_fig) plot_map(job_name, ymd, ndsi_data, out_fig)
def get_layer_information(cloudsat_filenames, get_quality=True, verbose=0): """ Returns CloudLayerType: -9: error, 0: non determined, 1-8 cloud types CloudLayerBase: in km CloudLayerTop: in km CloudTypeQuality: valid range [0, 1]; if <get_quality> """ all_info = { 'CloudLayerType': [], 'CloudLayerBase': [], 'CloudLayerTop': [] } if get_quality: all_info['CloudTypeQuality'] = [] for cloudsat_path in cloudsat_filenames: sd = SD(cloudsat_path, SDC.READ) if verbose: # List available SDS datasets. print("hdf datasets:", sd.datasets()) # get cloud types at each height for key, value in all_info.items(): value.append(sd.select(key).get()) for key, value in all_info.items(): value = np.vstack(value) if key == 'CloudLayerType': all_info[key] = value.astype(np.int8) else: all_info[key] = value.astype(np.float16) if not get_quality: all_info['CloudTypeQuality'] = None return all_info