def hdf2txt(hdfFile, txtFile): try: hdf = SD(hdfFile) # open the HDF file attr = hdf.attributes() # get global attribute dictionary dsets = hdf.datasets() # get dataset dictionary except HDF4Error, msg: print "HDF4Error", msg
def __init__(self, filename): hdfFile = SD(filename) tmp = np.array (hdfFile.select(1)[:], dtype = np.int16) emp = hdfFile.select(1).getfillvalue() hdfFile.end() max = tmp.max() print "signal max : ", max tmp[tmp == emp] = max min = tmp.min() print "signal min : ", min rng = max - min self.image = self.rawimage = np.array((tmp-min)*1.0/rng*255, dtype=np.uint8)
def txtToHDF(txtFile, hdfFile, varName, attr): """Inputs: txtFile = name of .txt file (passed as string) hdfFile = name of .hdf file (passed as string) varName = name of dataset to be added (passed as string) attr = dataset attributes (passed as dictionary) txtFile indicates a dataset, and varName and attr give information about it. txtToHDF puts this information into an SD (Scientific Dataset) object and stores that object as in hdfFile, creating hdfFile if need be, otherwise updating it. """ # Catch pyhdf errors try: # Open HDF file in update mode, creating it if non existent. d = SD(hdfFile, SDC.WRITE|SDC.CREATE) # Open text file and get matrix dimensions on first line. txt = open(txtFile) ni, nj = map(int, txt.readline().split()) # Define HDF dataset of type SDC.FLOAT32 with those dimensions. v = d.create(varName, SDC.FLOAT32, (ni, nj)) # Assign attributes passed as argument inside dict `attr'. for attrName in attr.keys(): setattr(v, attrName, attr[attrName]) # Load variable with lines of data. Compute min and max # over the whole matrix. i = 0 while i < ni: elems = map(float, txt.readline().split()) v[i] = elems minE = min(elems) maxE = max(elems) if i: minVal = min(minVal, minE) maxVal = max(maxVal, maxE) else: minVal = minE maxVal = maxE i += 1 # Set variable min and max attributes. v.minVal = minVal v.maxVal = maxVal # Close dataset and file objects (not really necessary, # since closing is automatic when objects go out of scope. v.endaccess() d.end() txt.close() except HDF4Error, msg: print "HDF4Error:", msg
def main(): # open the hdf file for reading hdf=SD.SD(FILE_NAME) # read the sds data sds=hdf.select(SDS_NAME) data=sds.get() # turn [y,x] (HDF representation) data into [x,y] (numpy one) data=data.reshape(data.shape[1],data.shape[0]) # print out the data msg_out="" for i in range(X_LENGTH): for j in range(Y_LENGTH): msg_out+=str(data[i,j])+" " msg_out+="\n" print msg_out
def geo_intersects(filename, lonlat): """ Checks a geolocation file for overlap with a latitude longitude box :filename: name of file to check :lonlat: list, [leftlon, rightlon, botlat, toplat] :return: boolean, true if there was overlap """ logging.info("Checking %s for intersection with given lonlat" % filename) if filename[-4:] != '.hdf': logging.info("ERROR: %s is not an hdf file" % filename) return False try: hdf = SD.SD(filename) except: logging.info("ERROR: failed to load file: %s" % filename) return False lon = hdf.select('Longitude') lat = hdf.select('Latitude') dim1 = len(lon[:]) dim2 = len(lon[0]) minlon = float(lon[0][0]) maxlon = float(lon[dim1 - 1][dim2 - 1]) minlat = float(lat[dim1 - 1][dim2 - 1]) maxlat = float(lat[0][0]) if minlon > maxlon: logging.info( "File %s crosses dateline (not currently supported), skipping..." % filename) return False lonoverlap = minlon < lonlat[1] and maxlon > lonlat[0] latoverlap = minlat < lonlat[3] and maxlat > lonlat[2] intersects = lonoverlap and latoverlap if intersects: logging.info("File %s intersects given lonlat") else: logging.info("File %s does not intersect given lonlat") return intersects
def get_l2hdf_slope_intercept(ifile, prod): #--------------------------------------------------------------- slope_inter= np.asarray([1.0, 0.0]) ftype= hdf_cdf_version(ifile) if ftype == 'hdf4': f= SD(ifile) d1 = f.select(prod) d1Attr= d1.attributes() attNames= d1Attr.keys() attNames.sort() for nm in attNames: if nm == 'slope': slope_inter[0]= float(d1Attr[nm]) if nm == 'intercept': slope_inter[1]= float(d1Attr[nm]) if nm == 'scale_factor': slope_inter[0]= float(d1Attr[nm]) if nm == 'add_offset': slope_inter[1]= float(d1Attr[nm]) d1.endaccess() f.end() return slope_inter if ftype == 'hdf5': f = Dataset(ifile, 'r') group_names= f.groups.keys() #get group names (e.g. geophtsical_data_sets or navigation) for grp_name in group_names: var_name= f.groups[grp_name].variables.keys() #get names of objects within each group (e,g. chlor_a or longitude) for vn in var_name: if vn == prod: p = f.groups[grp_name].variables[vn] try: slope_inter= np.asarray([float(p.scale_factor), float(p.add_offset)]) except: print ( '\nDid not find slope intercept valules in l2 file. Using as default: slope = 1.0 and interecept = 0.0\n' ) f.close() return slope_inter
def __init__(self, filename): self.filename = os.path.basename(filename) self.filepath = os.path.realpath(filename) self._hdf_handle = SD.SD(self.filepath, SD.SDC.READ) # CREFLM_npp_d20141103_t1758468_e1800112.hdf fn = os.path.splitext(self.filename)[0] parts = fn.split("_") self.satellite = parts[1].lower() self.instrument = "viirs" # Parse out the datetime, making sure to add the microseconds and set the timezone to UTC begin_us = int(parts[3][-1]) * 100000 self.begin_time = datetime.strptime(parts[2][1:] + parts[3][1:-1], "%Y%m%d%H%M%S").replace(microsecond=begin_us) end_us = int(parts[4][-1]) * 100000 self.end_time = datetime.strptime(parts[2][1:] + parts[4][1:-1], "%Y%m%d%H%M%S").replace(microsecond=end_us) if self.end_time < self.begin_time: self.end_time += timedelta(days=1)
def __init__(self, filename): hdfFile = SD(filename) bo = re.compile('.*?=\s+([-]*\d*\.\d+).*',re.DOTALL) coord_string = hdfFile.__getattr__('ArchiveMetadata.0')[hdfFile.__getattr__('ArchiveMetadata.0').find('NORTHBOUNDINGCOORDINATE')] self.west = float(bo.match(coord_string.split('WESTBOUNDINGCOORDINATE')[1]).group(1)) self.east = float(bo.match(coord_string.split('EASTBOUNDINGCOORDINATE')[1]).group(1)) self.south = float(bo.match(coord_string.split('SOUTHBOUNDINGCOORDINATE')[1]).group(1)) self.north = float(bo.match(coord_string.split('NORTHBOUNDINGCOORDINATE')[1]).group(1)) tmp = np.array (hdfFile.select(1)[:], dtype = np.int16) emp = hdfFile.getfillvalue() tmp[tmp == emp] = 0 min = tmp.min() rng = tmp.max() - min self.image = np.array(float(tmp-min)/rng*255, dtype=np.uint8) hdfFile.end()
def __init__(self, fname): """Initialize the HDF file""" MXD35L2.__init__(self) print("> Reading: " + fname) name = fname start = fname.find('D35_L2.A') + len('D35_L2.A') end = start + len('YYYYDDD.HHMM') datetimestamp = DT.datetime.strptime(fname[start:end], '%Y%j.%H%M') hdf_file = SD.SD(fname) lon = hdf_file.select('Longitude').get() lat = hdf_file.select('Latitude').get() cloud_mask = NP.uint8(hdf_file.select('Cloud_Mask').get()[0]) hdf_file.end() cloud = cloud_mask & 6 # get bits 1 and 2 cloud[cloud == 0] = 1 # 00 = confident cloudy cloud[cloud != 1] = 0 water = cloud_mask & 192 # get bits 6 and 7 water[water == 0] = 1 # 00 = water water[water != 1] = 0 coast = cloud_mask & 192 # get bits 6 and 7 coast[coast == 64] = 1 # 01 = coastal coast[coast != 1] = 0 lon, lat, cloud, water, coast = autoFlip(lon, 'horizontal', lon, lat, cloud, water, coast) lon, lat, cloud, water, coast = autoFlip(lat, 'vertical', lon, lat, cloud, water, coast) self.name = name self.datetimestamp = datetimestamp self.lon = lon self.lat = lat self.cloud = cloud self.water = water self.coast = coast self.top = self.lat[0] self.bot = self.lat[-1] self.lef = self.lon[0] self.ryt = self.lon[-1]
def get_variables(file_path): all_vars = dict() try: f = SD.SD(file_path, SD.SDC.READ) ds = f.datasets() for ds_name in ds: #open dataset d = f.select(ds_name) units = None if d.attributes().get('units') == 'none' else d.attributes().get('units') details = "%s %s %s" %(" ".join(d.dimensions().keys()), str(d.attributes().get('long_name')), ds_name) details = details.replace("\n", " ").replace("/", " ").replace(".", " ").replace(":", " ") all_vars[sanitize(ds_name)] = dict(standard_name=ds_name, units=units, details=details) d.endaccess() f.end() return all_vars except Exception, e: print e # raise return
def hdf2tiff(file): hdf = SD(file) #打开HDF文件 attr = hdf.attributes(full=1) #HDF文件属性字典 attNames = attr.keys() attNames.sort() for name in attNames: t = attr[name] print name, t dsets = hdf.datasets() #HDF数据字典 dsNames = dsets.keys() dsNames.sort() for name in dsNames: ds = hdf.select(name) vAttr = ds.attributes() t = dsets[name] print name, vAttr, t im_data = hdf.select('precipitation').get() * 3 im_data = np.fliplr(im_data) im_data = np.transpose(im_data) np.putmask(im_data, im_data < 0, np.nan) im_width = 1440 im_height = 400 im_bands = 1 datatype = gdal.GDT_UInt16 sr = osr.SpatialReference() sr.ImportFromEPSG(4326) im_proj = sr.ExportToWkt() im_geotrans = (-180, 0.25, 0, 50, 0, -0.25) driver = gdal.GetDriverByName("GTiff") dataset = driver.Create( os.path.splitext(file)[0] + ".TIF", im_width, im_height, im_bands, datatype) dataset.SetGeoTransform(im_geotrans) dataset.SetProjection(im_proj) dataset.GetRasterBand(1).WriteArray(im_data) del dataset hdf.end()
def to_array(fname): # from HDF4 to HDF5 files fin = sd.SD(fname) # HDF4 files d = fin.datasets() #orbit = fin.select('id').get() secs00 = fin.select('time').get() lat = fin.select('lat').get() lon = fin.select('lon').get() elev = fin.select('elev').get() agc = fin.select('gain').get() energy = fin.select('wave_energy').get() reflect = fin.select('reflect').get() icesvar = fin.select('icesvar').get() #pressure = fin.select('pressure').get() #sat = fin.select('sat_cor_product').get() nrow, ncol = lon.shape[0], 9 orbit = np.empty(nrow, 'f8') revNo = int(os.path.split(fname)[-1].split('_')[2]) orbit.fill(revNo) secs00 += 43200 # move time 12 hours back (starting midnight) lon[lon < 0] += 360. # from -180/180 to 0/360 fout = tb.openFile(os.path.splitext(fname)[0] + '.h5', 'w') atom = tb.Float64Atom() shape = (nrow, ncol) filters = tb.Filters(complib='blosc', complevel=9) c = fout.createCArray('/', 'data', atom=atom, shape=shape, filters=filters) c[:, 0] = orbit c[:, 1] = secs00 c[:, 2] = lat c[:, 3] = lon c[:, 4] = elev c[:, 5] = agc c[:, 6] = energy c[:, 7] = reflect c[:, 8] = icesvar fout.close()
def read(filename, variables=None, datadict=None): """ Reads SD from a HDF4 file into a dictionary. :param str filename: The name (with path) of the HDF file to read. :param iterable names: A sequence of variable (dataset) names to read from the file (default None, causing all variables to be read). The names must appear exactly as in in the HDF file. :param dict datadict: Optional dictionary to add data to, otherwise a new, empty dictionary is created :return: A dictionary containing data for requested variables. Missing data is replaced by NaN. """ # Optional HDF import if not SD: raise ImportError( "HDF support was not installed, please reinstall with pyhdf to read HDF files." ) # List of required variable names. # Open the file. datafile = None try: datafile = SD.SD(filename) sd_variables = list(datafile.datasets().keys()) finally: if datafile is not None: datafile.end() if variables is None: requested_sd_variables = sd_variables else: requested_sd_variables = set(listify(variables)).intersection( set(sd_variables)) # Create dictionary to hold data arrays for returning. if datadict is None: datadict = {} # Get data. for variable in requested_sd_variables: datadict[variable] = HDF_SDS(filename, variable) return datadict
def __init__(self, filename): hdfFile = SD(filename) bo = re.compile('.*?=\s+([-]*\d*\.\d+).*', re.DOTALL) coord_string = hdfFile.__getattr__('ArchiveMetadata.0')[ hdfFile.__getattr__('ArchiveMetadata.0').find( 'NORTHBOUNDINGCOORDINATE')] self.west = float( bo.match(coord_string.split('WESTBOUNDINGCOORDINATE')[1]).group(1)) self.east = float( bo.match(coord_string.split('EASTBOUNDINGCOORDINATE')[1]).group(1)) self.south = float( bo.match( coord_string.split('SOUTHBOUNDINGCOORDINATE')[1]).group(1)) self.north = float( bo.match( coord_string.split('NORTHBOUNDINGCOORDINATE')[1]).group(1)) tmp = np.array(hdfFile.select(1)[:], dtype=np.int16) emp = hdfFile.getfillvalue() tmp[tmp == emp] = 0 min = tmp.min() rng = tmp.max() - min self.image = np.array(float(tmp - min) / rng * 255, dtype=np.uint8) hdfFile.end()
def rdhdf(hdf_filename): if (hdf_filename.endswith('h5')): x, y, z, f = rdh5(hdf_filename) return (x, y, z, f) x = np.array([]) y = np.array([]) z = np.array([]) f = np.array([]) # Open the HDF file sd_id = h4.SD(hdf_filename) #Read dataset. In all PSI hdf4 files, the #data is stored in "Data-Set-2": sds_id = sd_id.select('Data-Set-2') f = sds_id.get() #Get number of dimensions: ndims = np.ndim(f) #Get the scales: for i in range(0, ndims): dim = sds_id.dim(i) if i == 0: x = dim.getscale() elif i == 1: y = dim.getscale() elif i == 2: z = dim.getscale() sd_id.end() x = np.array(x) y = np.array(y) z = np.array(z) f = np.array(f) return (x, y, z, f)
def start_mapping(upload_result, url=False): print url if not url: print "processing the uploaded file result" filename = upload_result["name"] file_path = os.path.join(os.path.abspath(UPLOAD_DIR), filename) "Extracting starts here" # open the hdf file for reading hdf = SD.SD(file_path) print hdf # f = h5py.File(file_path, 'r') # print f.keys() # dataset = netCDF4.Dataset(file_path, mode='r') # print dataset.title return {"result": "this lib mapper working"} else: print "Processing the url provided" url = upload_result print url
def load_from_fileZ2D(self, file_name): print(file_name) hdf = SD.SD(file_name) sds = hdf.select("fakeDim0") self.th = sds.get() sds = hdf.select("fakeDim1") self.r = sds.get() sds = hdf.select("Data-Set-2") #density self.dd = (sds.get()).T sds = hdf.select("Data-Set-3") #total energy self.ee = (sds.get()).T sds = hdf.select("Data-Set-4") #ur self.ur = (sds.get()).T sds = hdf.select("Data-Set-5") #uth self.uth = (sds.get()).T sds = hdf.select("Data-Set-6") #uph self.uph = (sds.get()).T
def MODIS_file(filename,height_lim,zen_lim,t_bins): from pyhdf import SD import numpy as np f=SD.SD(filename) sds=f.select('cloud_top_temperature_1km') temp=0.01*(sds.get()[:2030,:1350]+15000)#sampling on the same grid as latitude sds=f.select('Cloud_Phase_Optical_Properties') phase=sds.get()[:2030,:1350] sds=f.select('Solar_Zenith_Day') zenith=0.009999999776482582*np.repeat(np.repeat(sds.get(),5,axis=0),5,axis=1).reshape(2030,1350) #consider getting rid of the fill values sds=f.select('cloud_top_height_1km') height=sds.get()[:2030,:1350] sds=f.select('Latitude') lat=np.repeat(np.repeat(sds.get(),5,axis=0),5,axis=1).reshape(2030,1350) #the above code extracts the relevant datasets from each MODIS file #filtering cloud by altitude heights=np.where((height<height_lim)&(((zenith<zen_lim)&(zenith>-zen_lim)))&((lat<-45)&(lat>-65)))#less than 4000 meteters #filtered by both positive and negative zenith angles #sample the new fields based on these thresholds temp=temp[heights] phase=phase[heights] zenith=zenith[heights] height=height[heights] lat=lat[heights] ice_loc=np.where(phase==3)#location of pixels in the liquid and ice fields liq_loc=np.where(phase==2) hist_water=np.histogram(temp[liq_loc],bins=t_bins)[0] hist_ice=np.histogram(temp[ice_loc],bins=t_bins)[0] #histograms of the temperatures of ice and liquid pixel numbers return hist_water,hist_ice,t_bins,np.histogram(zenith[ice_loc],bins=np.arange(-90,90,1))[0],np.histogram(zenith[liq_loc],bins=np.arange(-90,90,1))[0]
def read_ftir(FILE_NAME): """ Read NDACC HDF file. File and variable names to be specified by user when under name == main """ hdf = SD.SD(FILE_NAME) # read attributes lst = hdf.attributes() var = lst['DATA_VARIABLES'].split(';') data = [] #---------------------------------------- for v in var: #-------------------------------- try: sds = hdf.select(v) data.append(sds.get()) except: print('No SDS found: ' + v) var.pop(v) #-------------------------------- #---------------------------------------- return var, data
def process_files(self, dirname, out_file, latitude_range, longitude_range, csv_separator=";", delete_after=False): file_index = 1 for file in os.listdir(dirname): full_path = os.path.join(dirname, file) if file.endswith('.hdf'): print(f'Processing file... {file_index} (Path: {full_path})') data = self._process_file(SD(full_path, SDC.READ), latitude_range, longitude_range) BaseModisExtractor.write_to_csv(out_file, csv_separator, data) print( f'Processing finished for file... {file_index} (Path: {full_path})' ) if delete_after: for file in os.listdir(dirname): full_path = os.path.join(dirname, file) if file.endswith('.hdf'): os.remove(full_path) print(f'All results saved to {out_file}')
def L2_VFM_Reading(fpath): sd_obj = SD(fpath, SDC.READ) Vt_obj = HDF.HDF(fpath).vstart() m_data = Vt_obj.attach('metadata').read()[0] Height = np.array(m_data[-1]) # 583高度对应实际海拔 Lats = sd_obj.select('Latitude').get() Lons = sd_obj.select('Longitude').get() L_route = np.concatenate([Lats.T, Lons.T]).T target_rows = [] for location in L_route: distance = LonLat_Distance(location, LZU_LatLon) if distance < 50: target_rows.append(True) else: target_rows.append(False) VFM_basic = np.array(sd_obj.select('Feature_Classification_Flags').get()) VFM_basic = VFM_basic % 8 VFM_1 = np.reshape(VFM_basic[:, 0:165], (VFM_basic.shape[0] * 3, 55)) VFM_1 = np.repeat(VFM_1, 5, axis=0) VFM_2 = np.reshape(VFM_basic[:, 165:1165], (VFM_basic.shape[0] * 5, 200)) VFM_2 = np.repeat(VFM_2, 3, axis=0) VFM_3 = np.reshape(VFM_basic[:, 1165:5515], (VFM_basic.shape[0] * 15, 290)) VFM = np.concatenate((VFM_1, VFM_2, VFM_3), axis=1) target_rows_VFM = np.repeat(target_rows, 15) Rd_dic = {} Rd_dic['VFM'] = VFM Rd_dic_meta = { 'route': L_route, 'Lats': Lats, 'target rows': target_rows, 'Height': Height, 'target rows VFM': target_rows_VFM, } sd_obj.end() HDF.HDF(fpath).close() return Rd_dic, Rd_dic_meta
def look(self, path, mem_list, lp_list): data = {} for name in mem_list: # subdata sets type data tag = lp_list[name][0] ref = lp_list[name][1] if tag == HC.DFTAG_NDG: sd = SD(path) sds = sd.select(sd.reftoindex(ref)) data[name] = np.float64(sds.get()) sds.endaccess() sd.end() elif tag == HC.DFTAG_VH: #vd type data hdf = HDF(path) vs = hdf.vstart() vd = vs.attach(ref) nrecs, intmode, fields, size, name = vd.inquire() data[name] = np.full(nrecs, np.float64(vd.read()[0])) vs.end() hdf.close() return data
def work(file): sd = SD(file) lat = np.array(sd.select("Latitude")[:]) lon = np.array(sd.select("Longitude")[:]) cf = np.array(sd.select("Cloud_Fraction")[:]) t = np.array(sd.select("Scan_Start_Time")[:]) + initial_time dist = distance(lat, lon) ind = np.where(dist == dist.min()) x = ind[0] y = ind[1] lat = lat[x-1:x+2, y-1:y+2] lon = lon[x-1:x+2, y-1:y+2] cf = cf[x-1:x+2, y-1:y+2] t = t[x-1:x+2, y-1:y+2] dist = dist[x-1:x+2, y-1:y+2] d = dist / np.sum(dist) cf = weighted_average(cf, d) lat = weighted_average(lat, d) lon = weighted_average(lon, d) t = weighted_average(t, d) return (cf, lat, lon, t)
for i in range(RAT.shape[1]): Digitised[np.where(Digitised[:,2]==i),2] = digitizing(ROIS[6][np.where(Digitised[:,2]==i)],RAT[:,i]) for i in range(10000): Digitised[i,3] = digitizing(ROIS[-2][i],BT[:,CN1,ST,CP,Digitised[i,2],Digitised[i,0],Digitised[i,1]]) test[i] = TT[ABCN,Digitised[i,3],Digitised[i,2],Digitised[i,0],Digitised[i,1]] Texture = ROIS[-1]>=test return Texture.astype(int) AZM_Data = SD.SD('/Users/jesseloveridge/Documents/Summer Project/MISR_AM1_AZM_F01_01.hdf') #Thresholds from MISR dataset VZT = AZM_Data.select('Viewing Zenith Angle Thresholds')[:] SZAT = AZM_Data.select('Solar Zenith Angle Thresholds')[:] RAT = AZM_Data.select('Relative Azimuth Thresholds')[:,:] BT = AZM_Data.select('Brightness Thresholds')[:] TT = AZM_Data.select('Texture Thresholds')[:] os.chdir('/Volumes/Promise1/Jesse/MODIS_DATA/') #file lists requires this directory to be changed tau_list,rad_list = file_lists() #use a for loop here to provide indices for the file lists. tau_file = tau_list[index] rad_file = rad_list[index]
def get_hdf_data(fname): hdf = SD.SD(fname) info= hdf.datasets() #Lets see what is inside hdf.info() data_sets=[] for name in sorted(info.keys()): if name[0:4]=="Data": sds=hdf.select(name) long_name=sds.attributes()["long_name"] for i in range(len(sds.attributes()["long_name"])): if long_name[i:i+2]=="AT": junk=i-1 short_name=long_name[:junk] data_sets.append([name,long_name,short_name]) #Get the time from the last long name if long_name[junk+9:] != "********": time=float(long_name[junk+9:]) else: time=0.0 #Get the dimensions from the last data set dims=len((sds.info()[2])) #Get the coordinate system from the last data set coord_sys=sds.attributes()["coordsys"] if coord_sys=='spherical polar': coord_sys='spol' else: print(("get_hdf_data: I don't understand coordinate system ",coord_sys)) exit() #Now we know which of the datasets contain real data, we can extract all the data alldat={} alldat["Filename"]=fname alldat["Coord_sys"]=coord_sys alldat["Time"]=time alldat["Data"]={} alldat["Data_names"]=np.array(data_sets)[:,2] alldat["N_data"]=len(data_sets) alldat["Dims"]=dims #Loop over all the data sets in the hdf file - name each of the resulting dictionaries with the short name for i in range (len(data_sets)): print((data_sets[i][2])) sds=hdf.select(data_sets[i][0]) data = sds.get() c1=info[data_sets[i][0]][0][0] c2=info[data_sets[i][0]][0][1] sds=hdf.select(c1) x2=sds.get() sds=hdf.select(c2) x1=sds.get() alldat[data_sets[i][2]]={} alldat[data_sets[i][2]]=data alldat["r_cent"]=x1 alldat["theta_cent"]=x2 #HDF files only give us the centre of the grid, python needs the inner edges as well. r_edge=[] r_ratio=(x1[2]-x1[1])/(x1[1]-x1[0]) dr=(x1[1]-x1[0])/(0.5*(1.0+r_ratio)) r_edge.append(x1[0]-0.5*dr) print((r_edge[0],r_ratio)) for i in range(len(x1)-1): r_edge.append(r_edge[-1]+dr) dr=dr*r_ratio theta_edge=[] theta_ratio=(x2[2]-x2[1])/(x2[1]-x2[0]) dtheta=(x2[1]-x2[0])/(0.5*(1.0+theta_ratio)) theta_min=x2[0]-0.5*dtheta if theta_min<0.0: theta_min=0.0 theta_edge.append(theta_min) print((x2[0])) print((theta_edge[0],theta_ratio)) for i in range(len(x2)-1): theta_edge.append(theta_edge[-1]+dtheta) dtheta=dtheta*theta_ratio if (theta_edge[-1]+(x2[-1]-theta_edge[-1])*2.0)>(np.pi/2.0): x2[-1]=(theta_edge[-1]+(np.pi/2.0))/2.0 alldat["r_edge"]=r_edge alldat["theta_edge"]=theta_edge return(alldat)
import numpy as np import matplotlib as mpl import matplotlib.pyplot as plt import matplotlib.cm as cm from pyhdf.SD import * from bitstring import * import sys #np.set_printoptions(threshold='nan') _filename = 'C:\Users\AKO NA LNG\Desktop\Esquivel Files\School Files 2\Special Problem\MODIS\MOD35\MOD35_L2.A2015220.0215.006.2015220134621.hdf' mod35 = SD(_filename, SDC.READ) sds_name = 'Cloud_Mask' sds_index = mod35.select(sds_name) sds_data = sds_index.get() sds_data_0 = sds_data[0, :, :] sds_data_0_bin = sds_data_0.astype(dtype=np.uint8, order='K', casting='unsafe', subok=False, copy=False) for x in np.nditer(sds_data_0_bin, op_flags=['readwrite']): x[...] = np.right_shift(x, 6) fig = plt.figure() ax = fig.add_subplot(111) cmap = plt.cm.winter bounds = [0, 1, 2, 3]
print " vgroup:", vg0._name, "tag,ref:", tag, ref vg0.detach() # Unhandled tag else: print "unhandled tag,ref",tag,ref # Close vgroup vg.detach() # Open HDF file in readonly mode. filename = sys.argv[1] hdf = HDF(filename) # Initialize the SD, V and VS interfaces on the file. sd = SD(filename) vs = hdf.vstart() v = hdf.vgstart() # Scan all vgroups in the file. ref = -1 while 1: try: ref = v.getid(ref) except HDF4Error,msg: # no more vgroup break describevg(ref) # Terminate V, VS and SD interfaces. v.end() vs.end()
def mosaic(*arg,**args): # This function will take files tranfered in *arg and will mosaic them together and produce a new file # mosaic(file1,[file2,file3,...],endfile) # If only file1 and endfile is given, the function will only produce a copy without any other modification try: log=args["log"] except KeyError: log=logMod.Log("",nolog=True) if len(arg)>2: lfile = arg[:-1] # This is the list of the NAME of the files to merge newfilename = arg[-1] # This is the final file NAME # Should eventually check if files exists and can be read ***IMPROVE*** lfHDF = [] # This is the list of the FILES to merge latt = [] # This is the list of the ATTRIBUTE "StructMetadata.0" of the files for fil in lfile: try: a=SD(fil,SDC.READ) except TypeError: a=SD(fil.encode('ascii','ignore'),SDC.READ) lfHDF.append(a) #print("hoho") latt.append(atribute(lfHDF[-1].attributes()["StructMetadata.0"],fil,dsi=[0,])) ## Listing all the GRIDS that the new file will have gridlist = [] # This is the list of GRIDS to include in the final file for attOfF in latt: # Should check if any grid ***IMPROVE*** gridlist += attOfF.listgridname()[1] # listgridname return a list of all the grids name # remove double entry gridlist = list(set(gridlist)) ## Listing all the DATASETS that the new file will have dslist = [] # This is the list of DATASETS to include in the final file for attOfF in latt: # Should check if any grid ***IMPROVE*** dslist = attOfF.orderedDS() # remove double entry # dslist = list(set(dslist)) ## Validation of commoun information ############################################################################### # Some informations have to be the same for each file or else no mosaic can # # be made for exemple two files with not the same projection type can't be # # merged together. ***IMPROVE*** Maybe in the future we could transform file # # so that they have the same projection or some other thing. # ############################################################################### # List of parameter to check to insure that they are the same paramMustSim = ["Projection","ProjParams","SphereCode"] # Dictionary that will keep all the informations about every file paramMustSimDict = {} for grid in gridlist: # Verification of a grid first = True # Variable that will enable the construction of the dict paramdict = {} # Dictionary that keep the actual value that have to be the same for attOfF in latt: # Verification of a file bigG = attOfF.getgridbyname(grid) # Getting all the attributes in the grid of a file if bigG is not None: # If the grid exists in that file if first : # If this is the first time that a file is check for that grid first = False for p in paramMustSim: # Checking every parameters that must be the same paramdict[p] = bigG.variable[p] # Validation of same Dtype for each datafield go = bigG.GROUP["DataField"].OBJECT for r in go: paramdict[go[r].variable["DataFieldName"]]=go[r].variable["DataType"] else: # If it's not the first time that a file is check for that grid for p in paramMustSim: # Checking every parameters that must be the same if not paramdict[p]==bigG.variable[p]: # Stop merging and return error ***IMPROVE*** # Maybe do only the things that can be done ***IMPROVE*** log.log('e',Nom,"Error dataset are not compatible") # Validation of same Dtype for each datafield go=bigG.GROUP["DataField"].OBJECT for r in go: if not paramdict[go[r].variable["DataFieldName"]]==go[r].variable["DataType"]: # Stop merging and return error ***IMPROVE*** # Maybe do only the things that can be done ***IMPROVE*** log.log('e',Nom,"Error dataset are not compatible") # Keep all this info for later it's going to be useful paramMustSimDict[grid]=paramdict ## Determination of new informations ############################################################################### # Some properties have to be calculated in order to merge. This section is # # doing just that # ############################################################################### gridResolX={} # Getting the RESOLUTION in the X direction for each grid gridResolY={} # Getting the RESOLUTION in the Y direction for each grid extremeup={} # Getting the UPPER coordinates for each grid extremedown={} # Getting the LOWEST coordinates for each grid extremeleft={} # Getting the LEFTMOST coordinates for each grid extremeright={} # Getting the RIGHTMOST coordinates for each grid gridDimX={} # Getting the DIMENSIONS of X direction for each grid gridDimY={} # Getting the DIMENSIONS of Y direction for each grid NoValueDS={} # Getting the fill value of each dataset dtypeDS={} # Getting the DTYPE for each dataset dstogrid={} # Knowing wich is the GRID for each dataset filGridULC={} # Getting the upper left corner of each file for each grid for grid in gridlist: # For each grid filGridULC[grid]={} # Adding a dictionary for each grid that will contain information on every file for attOfF in latt: ### Determination of resolution of each grid # ***IMPROVE*** Should check if bigd is none bigG=attOfF.getgridbyname(grid) # Getting all the attributes in the grid of a file # Get extreme grid point ulp=eval(bigG.variable["UpperLeftPointMtrs"]) lrp=eval(bigG.variable["LowerRightMtrs"]) # Get grid dimmension dimx=int(bigG.variable["XDim"]) dimy=int(bigG.variable["YDim"]) # Calculate grid resolution gridResolX[grid]=(lrp[0]-ulp[0])/dimx gridResolY[grid]=(ulp[1]-lrp[1])/dimy ### Determination of new extreme coordinates for each grid # up try: if extremeup[grid]< ulp[1]: extremeup[grid]=ulp[1] except KeyError: extremeup[grid]=ulp[1] # down try: if extremedown[grid]> lrp[1]: extremedown[grid]=lrp[1] except KeyError: extremedown[grid]=lrp[1] # left try: if extremeleft[grid]> ulp[0]: extremeleft[grid]=ulp[0] except KeyError: extremeleft[grid]=ulp[0] # right try: if extremeright[grid]< lrp[0]: extremeright[grid]=lrp[0] except KeyError: extremeright[grid]=lrp[0] ### Detetermination of dataset to grid name if bigG is not None: go=bigG.GROUP["DataField"].OBJECT for r in go: dstogrid[ go[r].variable["DataFieldName"] ] = grid ## Determination of ULC for each grid in each file filGridULC[grid][attOfF.name] = ulp ## determination of new dimension for each grid gridDimY[grid] = int((extremeup[grid]-extremedown[grid])/gridResolY[grid]) gridDimX[grid] = int((extremeright[grid]-extremeleft[grid])/gridResolX[grid]) for ds in dslist: # For each dataset for sd in lfHDF: # For each hdf file # Try opening dataset try: sds = sd.select(eval(ds)) # Get fill value NoValueDS[ds] = sds.getfillvalue() # Get dtype dtypeDS[ds] = sds.info()[3] except: log.log('e',Nom,"no dataset") ## Start creating new file ############################################################################### # This is the actual part were stuf appens # ############################################################################### # This part is the same for every file in any circumstances ########## absolute ######################## # Open new file try: hdf = HDF(newfilename, HC.WRITE | HC.CREATE |HC.TRUNC) sd = SD(newfilename, SDC.WRITE | SDC.CREATE ) except TypeError: hdf = HDF(newfilename.encode('ascii','ignore'), HC.WRITE | HC.CREATE |HC.TRUNC) sd = SD(newfilename.encode('ascii','ignore'), SDC.WRITE | SDC.CREATE ) v=hdf.vgstart() vg={} vg1={} vg2={} ## rewrite the gridlist gridlist = [] for ds in dslist: if dstogrid[ds] not in gridlist: gridlist.append(dstogrid[ds]) for grid in gridlist: vg[grid]=v.attach(-1,write=1) vg[grid]._class="GRID" vg[grid]._name=eval(grid) vg1[grid]=v.attach(-1,write=1) vg2[grid]=v.attach(-1,write=1) vg1[grid]._class="GRID Vgroup" vg1[grid]._name="Data Fields" vg2[grid]._class="GRID Vgroup" vg2[grid]._name="Grid Attributes" vg[grid].insert(vg1[grid]) vg[grid].insert(vg2[grid]) ########## absolute ######################## # Create dataset with the right size for ds in dslist: theGrid=dstogrid[ds] # Get grid name of data set sds = sd.create(eval(ds),dtypeDS[ds],(gridDimY[theGrid],gridDimX[theGrid])) # Set fill value fv=NoValueDS[ds] try: sds.setfillvalue(NoValueDS[ds]) except OverflowError: log.log('e',Nom,"setfillvalue") sds.setfillvalue(0) ## write real data for fil in range(len(latt)): try: # Determine were the data will be writen ulc = filGridULC[theGrid][latt[fil].name] # Determine the position on the grid y = (extremeup[theGrid]-ulc[1])/(extremeup[theGrid]-extremedown[theGrid]) x = (ulc[0]-extremeleft[theGrid])/(extremeright[theGrid]-extremeleft[theGrid]) y = int(y*gridDimY[theGrid]) x = int(x*gridDimX[theGrid]) # read data from files osds = lfHDF[fil].select(eval(ds)) sh = osds[:].shape sds[y:y+sh[0],x:x+sh[1]] = osds[:] osds.endaccess() except: pass # Close sds vg1[dstogrid[ds]].add(HC.DFTAG_NDG,sds.ref()) sds.endaccess() for g in vg1: vg1[g].detach() vg2[g].detach() vg[g].detach() # Create attribute table for the file attstr="GROUP=GridStructure\n" gridcount=1 for gr in gridlist: # Start group grid attstr+="\tGROUP=GRID_%i\n"%gridcount # Add grid name attstr+="\t\tGridName=%s\n"%gr # Add dimention attstr+="\t\tXDim=%i\n"%gridDimX[gr] attstr+="\t\tYDim=%i\n"%gridDimY[gr] # Add UpperLeftPointMtrs attstr+="\t\tUpperLeftPointMtrs=(%f,%f)\n"%(extremeleft[gr],extremeup[gr]) # Add lrp attstr+="\t\tLowerRightMtrs=(%f,%f)\n"%(extremeright[gr],extremedown[gr]) # Add projection attstr+="\t\tProjection=%s\n"%paramMustSimDict[gr]["Projection"] # ProjParams attstr+="\t\tProjParams=%s\n"%paramMustSimDict[gr]["ProjParams"] # SphereCode attstr+="\t\tSphereCode=%s\n"%paramMustSimDict[gr]["SphereCode"] attstr+="""\t\tGROUP=Dimension \t\tEND_GROUP=Dimension \t\tGROUP=DataField\n""" ## Add data sets # create list of ds for current grid lsdsgr=[] dsnum=1 for ds in dslist: if dstogrid[ds] == gr: # Add object attstr+="\t\t\tOBJECT=DataField_%i\n"%dsnum # datafield name attstr+="\t\t\t\tDataFieldName=%s\n"%ds # datatype attstr+="\t\t\t\tDataType=%s\n"%paramMustSimDict[gr][ds] # dim attstr+='\t\t\t\tDimList=("YDim","XDim")\n' attstr+="\t\t\tEND_OBJECT=DataField_%i\n"%dsnum dsnum+=1 attstr+="\t\tEND_GROUP=DataField\n" attstr+="""\t\tGROUP=MergedFields \t\tEND_GROUP=MergedFields\n""" attstr+="\tEND_GROUP=GRID_%i\n"%gridcount gridcount+=1 attstr+="""END_GROUP=GridStructure GROUP=PointStructure END_GROUP=PointStructure END""" # adding attribute to new file att=sd.attr('StructMetadata.0') att.set(SDC.CHAR,attstr) sd.end() hdf.close() # This should return something somehow elif len(arg)>1: afile = arg[0] # This is the list of the NAME of the files to merge newfilename = arg[1] # This is the final file NAME # Create a copy from shutil import copyfile copyfile(afile,newfilename)
def doCompress(compType, value=0, v2=0): """Create and validate an HDF file using a compression scheme sepcified by the parameters""" # Build a significant file name if compType == SDC.COMP_NONE: fileName = "SDS.COMP_NONE" elif compType == SDC.COMP_RLE: fileName = "SDS.COMP_RLE" elif compType == SDC.COMP_SKPHUFF: fileName = "SDS.COMP_SKPHUFF.%d" % value elif compType == SDC.COMP_DEFLATE: fileName = "SDS.COMP_DEFLATE.%d" % value elif compType == SDC.COMP_SZIP: fileName = "SDS.COMP_SZIP" if value == SDC.COMP_SZIP_NN: fileName += ".NN" elif value == SDC.COMP_SZIP_EC: fileName += ".EC" else: print("illegal value") sys.exit(1) fileName += ".%s" % v2 else: print("illegal compType") sys.exit(1) fileName += ".hdf" SDS_NAME = "Data" fill_value = 0 #LENGTH = 9 #WIDTH = 6 # #data = numpy.array( ((100,100,200,200,300,400), # (100,100,200,200,300,400), # (100,100,200,200,300,400), # (300,300, 0,400,300,400), # (300,300, 0,400,300,400), # (300,300, 0,400,300,400), # (0, 0,600,600,300,400), # (500,500,600,600,300,400), # (0, 0,600,600,300,400)), NUMPY_DATATYPE) # The above dataset is used in the original NCSA example. # It is too small to show a significant size reduction after # compression. The following is used for a more realistic example. data = numpy.zeros((LENGTH, WIDTH), NUMPY_DATATYPE) for i in range(LENGTH): for j in range(WIDTH): data[i,j] = (i+j)*(i-j) # Create HDF file, wiping it out it it already exists. sd_id = SD(fileName, SDC.WRITE | SDC.CREATE | SDC.TRUNC) # Create dataset. sds_id = sd_id.create(SDS_NAME, HDF_DATATYPE, (LENGTH, WIDTH)) # Fill dataset will fill value. sds_id.setfillvalue(0) # Apply compression. try: sds_id.setcompress(compType, # compression type value, v2) # args depend on compression type except HDF4Error as msg: print(("Error compressing the dataset with params: " "(%d,%d,%d) : %s" % (compType, value, v2, msg))) sds_id.endaccess() sd_id.end() os.remove(fileName) return # Load data in the dataset. sds_id[:] = data # Close dataset. sds_id.endaccess() # Close hdf file to flush compressed data. sd_id.end() # Verify compressed data. # ###################### # Reopen file and select first dataset. sd_id = SD(fileName, SDC.READ) sds_id = sd_id.select(0) # Obtain compression info. compInfo = sds_id.getcompress() compType = compInfo[0] print("file : %s" % fileName) print(" size = %d" % os.path.getsize(fileName)) if compType == SDC.COMP_NONE: print(" compType = COMP_NONE") elif compType == SDC.COMP_RLE: print(" compType = COMP_RLE") elif compType == SDC.COMP_SKPHUFF: print(" compType = COMP_SKPHUFF") print(" dataSize = %d" % compInfo[1]) elif compType == SDC.COMP_DEFLATE: print(" compType = COMP_DEFLATE (GZIP)") print(" level = %d" % compInfo[1]) elif compType == SDC.COMP_SZIP: print(" compType = COMP_SZIP") optionMask = compInfo[1] if optionMask & SDC.COMP_SZIP_NN: print(" encoding scheme = NN") elif optionMask & SDC.COMP_SZIP_EC: print(" encoding scheme = EC") else: print(" unknown encoding scheme") sys.exit(1) pixelsPerBlock, pixelsPerScanline, bitsPerPixel, pixels = compInfo[2:] print(" pixelsPerBlock = %d" % pixelsPerBlock) print(" pixelsPerScanline = %d" % pixelsPerScanline) print(" bitsPerPixel = %d" % bitsPerPixel) print(" pixels = %d" % pixels) else: print(" unknown compression type") sys.exit(1) # Read dataset contents. out_data = sds_id[:] # Compare with original data. num_errs = 0 for i in range(LENGTH): for j in range(WIDTH): if data[i,j] != out_data[i,j]: print("bad value at %d,%d expected: %d got: %d" \ % (i,j,data[i,j],out_data[i,j])) num_errs += 1 # Close dataset and hdf file. sds_id.endaccess() sd_id.end() if num_errs == 0: print(" file validated") else: print(" file invalid : %d errors" % num_errs) print("")
pass try: os.unlink(tmp_path) except OSError,e: pass with open(path, 'r') as in_fp: with open(tmp_path_Z, 'w') as out_fp: out_fp.write(in_fp.read()) os.system('gunzip -f ' + tmp_path_Z) #m = hashlib.md5() #m.update(open(tmp_path).read()) #print 'tmp_path', tmp_path, m.hexdigest() sd = SD(tmp_path, SDC.READ) kt = sd.select('surfacePrecipitation') print kt.dimensions() data = kt[:, :].copy() print year, month, day, data.mean() acc.append([year, month, day, data]) if year != last_year: save() last_year = year save()
### Example from docs http://hdfeos.github.io/pyhdf/modules/SD.html#writing # Import SD and numpy. from pyhdf.SD import * from numpy import * fileName = 'template.hdf' # Create HDF file. hdfFile = SD(fileName, SDC.WRITE|SDC.CREATE) # Assign a few attributes at the file level hdfFile.author = 'It is me...' hdfFile.priority = 2 # Create a dataset named 'd1' to hold a 3x3 float array. d1 = hdfFile.create('d1', SDC.FLOAT32, (3,3)) # Set some attributs on 'd1' d1.description = 'Sample 3x3 float array' d1.units = 'celsius' # Name 'd1' dimensions and assign them attributes. dim1 = d1.dim(0) dim2 = d1.dim(1) dim1.setname('width') dim2.setname('height') dim1.units = 'm' dim2.units = 'cm' # Assign values to 'd1' d1[0] = (14.5, 12.8, 13.0) # row 1 d1[1:] = ((-1.3, 0.5, 4.8), # row 2 and (3.1, 0.0, 13.8)) # row 3
#!/usr/bin/env python2 # -*- coding: utf-8 -*- """ Created on Wed Jul 25 14:38:18 2018 @author: neeleshrampal """ """Edits have been made to incorparate a cloud fraction vs hISTROGRAM product""" """One must note that the product is likely to be conservative for cumulus cloud regions, due to its determination of partly cloudy reubis""" filename = '/Volumes/Promise1/Neelesh/MODIS_L3/2002/365/MOD08_D3.A2002365.061.2017280190741.hdf' from pyhdf import SD f = SD.SD(filename) datasets = f.datasets().keys() d1 = [] for key in datasets: # if 'Histo_vs_Pressure' in key: d1.append(key) # print d1 """the goal for today is to ensure that you can obtain the mean cloud fraction from grouping the data""" sds = f.select('Cloud_Fraction_Nadir_Day_Mean') data = sds.get()/1e4 c=np.where(data<0.0) data[c]=np.nan #f.close() dset = 'Cloud_Fraction_Nadir_Day_Pixel_Counts' sds = f.select(dset) data4 = np.array(sds.get(),dtype=float)
import numpy as np import matplotlib as mpl import matplotlib.pyplot as plt import matplotlib.cm as cm from pyhdf.SD import * from bitstring import * import sys # np.set_printoptions(threshold='nan') _filename = "C:\Users\AKO NA LNG\Desktop\Esquivel Files\School Files 2\Special Problem\MODIS\MOD35\MOD35_L2.A2015220.0215.006.2015220134621.hdf" mod35 = SD(_filename, SDC.READ) sds_name = "Cloud_Mask" sds_index = mod35.select(sds_name) sds_data = sds_index.get() sds_data_0 = sds_data[0, :, :] sds_data_0_bin = sds_data_0.astype(dtype=np.uint8, order="K", casting="unsafe", subok=False, copy=False) for x in np.nditer(sds_data_0_bin, op_flags=["readwrite"]): x[...] = np.right_shift(x, 6) fig = plt.figure() ax = fig.add_subplot(111) cmap = plt.cm.winter bounds = [0, 1, 2, 3] norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
#!/usr/bin/env python # -*- coding: utf-8 -*- import sys, Image import numpy as np from pyhdf.SD import * if __name__ == '__main__': filename = sys.argv[1] hdfFile = SD(filename) arrays = [] for i in range(3): data = hdfFile.select(i) tmp = np.array (data[:], dtype = np.int16) emp = data.getfillvalue() maxv = tmp.max() tmp[tmp == emp] = maxv minv = tmp.min() rng = maxv - minv arr = np.array((tmp-minv)*1.0/rng*255, dtype=np.uint8) im = Image.fromarray(arr) im.save(filename+"_" +str(i)+".png") arrays.append(arr) hdfFile.end()
def sdscreate(sd, name): # Create a simple 3x3 float array. sds = sd.create(name, SDC.FLOAT32, (3,3)) # Initialize array sds[:] = ((0,1,2),(3,4,5),(6,7,8)) # "close" dataset. sds.endaccess() # Create HDF file filename = 'inventory.hdf' hdf = HDF(filename, HC.WRITE|HC.CREATE) # Initialize the SD, V and VS interfaces on the file. sd = SD(filename, SDC.WRITE) # SD interface vs = hdf.vstart() # vdata interface v = hdf.vgstart() # vgroup interface # Create vdata named 'INVENTORY'. vdatacreate(vs, 'INVENTORY') # Create dataset named "ARR_3x3" sdscreate(sd, 'ARR_3x3') # Attach the vdata and the dataset. vd = vs.attach('INVENTORY') sds = sd.select('ARR_3x3') # Create vgroup named 'TOTAL'. vg = v.create('TOTAL')
def prnt(fname) : # Dictionnary used to convert from a numeric data type to its symbolic # representation typeTab = { SDC.CHAR: 'CHAR', SDC.CHAR8: 'CHAR8', SDC.UCHAR8: 'UCHAR8', SDC.INT8: 'INT8', SDC.UINT8: 'UINT8', SDC.INT16: 'INT16', SDC.UINT16: 'UINT16', SDC.INT32: 'INT32', SDC.UINT32: 'UINT32', SDC.FLOAT32: 'FLOAT32', SDC.FLOAT64: 'FLOAT64' } printf = sys.stdout.write def eol(n=1): printf("%s" % chr(10) * n) #hdfFile = sys.argv[1] # Get first command line argument hdfFile = fname try: # Catch pyhdf.SD errors # Open HDF file named on the command line f = SD(hdfFile) # Get global attribute dictionnary attr = f.attributes(full=1) # Get dataset dictionnary dsets = f.datasets() # File name, number of attributes and number of variables. printf("FILE INFO"); eol() printf("-------------"); eol() printf("%-25s%s" % ("File:", hdfFile)); eol() printf("%-25s%d" % (" file attributes:", len(attr))); eol() printf("%-25s%d" % (" datasets:", len(dsets))); eol() eol(); # Global attribute table. if len(attr) > 0: printf("File attributes"); eol(2) printf(" name idx type len value"); eol() printf(" -------------------- --- ------- --- -----"); eol() # Get list of attribute names and sort them lexically attNames = attr.keys() attNames.sort() for name in attNames: t = attr[name] # t[0] is the attribute value # t[1] is the attribute index number # t[2] is the attribute type # t[3] is the attribute length printf(" %-20s %3d %-7s %3d %s" % (name, t[1], typeTab[t[2]], t[3], t[0])); eol() eol() # Dataset table if len(dsets) > 0: printf("Datasets (idx:index num, na:n attributes, cv:coord var)"); eol(2) printf(" name idx type na cv dimension(s)"); eol() printf(" -------------------- --- ------- -- -- ------------"); eol() # Get list of dataset names and sort them lexically dsNames = dsets.keys() dsNames.sort() for name in dsNames: # Get dataset instance ds = f.select(name) # Retrieve the dictionary of dataset attributes so as # to display their number vAttr = ds.attributes() t = dsets[name] # t[0] is a tuple of dimension names # t[1] is a tuple of dimension lengths # t[2] is the dataset type # t[3] is the dataset index number printf(" %-20s %3d %-7s %2d %-2s " % (name, t[3], typeTab[t[2]], len(vAttr), ds.iscoordvar() and 'X' or '')) # Display dimension info. n = 0 for d in t[0]: printf("%s%s(%d)" % (n > 0 and ', ' or '', d, t[1][n])) n += 1 eol() eol() # Dataset info. if len(dsNames) > 0: printf("DATASET INFO"); eol() printf("-------------"); eol(2) for name in dsNames: # Access the dataset dsObj = f.select(name) # Get dataset attribute dictionnary dsAttr = dsObj.attributes(full=1) if len(dsAttr) > 0: printf("%s attributes" % name); eol(2) printf(" name idx type len value"); eol() printf(" -------------------- --- ------- --- -----"); eol() # Get the list of attribute names and sort them alphabetically. attNames = dsAttr.keys() attNames.sort() for nm in attNames: t = dsAttr[nm] # t[0] is the attribute value # t[1] is the attribute index number # t[2] is the attribute type # t[3] is the attribute length printf(" %-20s %3d %-7s %3d %s" % (nm, t[1], typeTab[t[2]], t[3], t[0])); eol() eol() # Get dataset dimension dictionnary dsDim = dsObj.dimensions(full=1) if len(dsDim) > 0: printf ("%s dimensions" % name); eol(2) printf(" name idx len unl type natt");eol() printf(" -------------------- --- ----- --- ------- ----");eol() # Get the list of dimension names and sort them alphabetically. dimNames = dsDim.keys() dimNames.sort() for nm in dimNames: t = dsDim[nm] # t[0] is the dimension length # t[1] is the dimension index number # t[2] is 1 if the dimension is unlimited, 0 if not # t[3] is the the dimension scale type, 0 if no scale # t[4] is the number of attributes printf(" %-20s %3d %5d %s %-7s %4d" % (nm, t[1], t[0], t[2] and "X" or " ", t[3] and typeTab[t[3]] or "", t[4])); eol() eol() except HDF4Error, msg: print "HDF4Error", msg
SDC.INT32: 'INT32', SDC.UINT32: 'UINT32', SDC.FLOAT32: 'FLOAT32', SDC.FLOAT64: 'FLOAT64' } printf = sys.stdout.write def eol(n=1): printf("%s" % chr(10) * n) hdfFile = sys.argv[1] # Get first command line argument try: # Catch pyhdf.SD errors # Open HDF file named on the command line f = SD(hdfFile) # Get global attribute dictionnary attr = f.attributes(full=1) # Get dataset dictionnary dsets = f.datasets() # File name, number of attributes and number of variables. printf("FILE INFO"); eol() printf("-------------"); eol() printf("%-25s%s" % ("File:", hdfFile)); eol() printf("%-25s%d" % (" file attributes:", len(attr))); eol() printf("%-25s%d" % (" datasets:", len(dsets))); eol() eol(); # Global attribute table. if len(attr) > 0:
def get_modis_latitude_longitude(modis_filename): hdf = SD(modis_filename) lat = hdf.select['Latitude'] lon = hdf.select['Longitude'] return (lat,lon)
def get_merra(fname, minlat, maxlat, minlon, maxlon, cdic, verbose=False): '''Read data from MERRA hdf file. Note that the Lon values should be between [0-360]. Hdf file with weather model data can be downloaded from http://disc.sci.gsfc.nasa.gov/daac-bin/FTPSubset.pl Args: * fname (str): Path to the grib file * minlat (np.float): Minimum latitude * maxlat (np.float): Maximum latitude * minlon (np.float): Minimum longitude * maxlon (np.float): Maximum longitude * cdic (np.float): Dictionary of constants Returns: * lvls (np.array): Pressure levels * latlist(np.array): Latitudes of the stations * lonlist(np.array): Longitudes of the stations * gph (np.array): Geopotential height * tmp (np.array): Temperature * vpr (np.array): Vapor pressure ''' if fname[-3:] == 'hdf': print(minlat) print(maxlat) print(minlon) print(maxlon) # Read the hdf file file = SD(fname) if verbose: print 'PROGRESS: READING HDF FILE' lvl = file.select('levels') rlvls = lvl.get() # Pressure levels are from lowest to highest lvls = [] for i in xrange( len(rlvls) ): # Reverse the pressure levels to be consistent with other GAMs index = len(rlvls) - i - 1 lvls.append(rlvls[index]) nlvls = len(lvls) lvls = np.array(lvls) alpha = cdic['Rv'] / cdic['Rd'] # Select latitutde and longitude lat = file.select('latitude') lon = file.select('longitude') lats = lat.get() lons = lon.get() mask1 = (lats > minlat) & (lats < maxlat) mask2 = (lons > minlon) & (lons < maxlon) [ii] = np.where(mask1 == True) [jj] = np.where(mask2 == True) del mask1 del mask2 iimemo = [] for m in xrange(len(ii)): for i in xrange(len(jj)): iimemo.append(ii[m]) jjmemo = [] for i in xrange(len(ii)): jjmemo.append(jj) jjmemo = np.array(jjmemo) jjmemo = jjmemo.flatten() iimemo = np.array(iimemo) ii = iimemo jj = jjmemo latlist = lats[ii] lonlist = lons[jj] nstn = len(latlist) # Create arrays for 3D storage gph = np.zeros((nlvls, nstn)) #Potential height tmp = gph.copy() #Temperature vpr = gph.copy() #Vapor pressure if verbose: print 'Number of stations:', nstn # Get data from files h = file.select('h') height = h.get()[0] qv = file.select('qv') humidity = qv.get()[0] sp = file.select('ps') spressure = sp.get()[0] t = file.select('t') temp = t.get()[0] # Lvls are in hecto pascals, convert to pascals lvls = 100.0 * lvls # Reverse altitude for i in xrange(nlvls): index = nlvls - i - 1 gph[i, :] = height[index][ii, jj] idx = np.zeros(nstn) for i in xrange(nstn): for m in xrange(nlvls): if spressure[ii[i]][jj[i]] > lvls[m]: idx[i] = m # extrapolation of temperature and humidity data at pressure levels under surface at each grid point tk = np.zeros(nstn) tb = np.zeros(nstn) for i in xrange(nstn): t = temp[:, ii[i], jj[i]] x = [lvls[idx[i]], lvls[idx[i] - 1]] y = [t[nlvls - idx[i] - 1], t[nlvls - idx[i]]] coef = np.polyfit(x, y, 1) tk[i] = coef[0] tb[i] = coef[1] hk = np.zeros(nstn) hb = np.zeros(nstn) for i in xrange(nstn): hum = humidity[:, ii[i], jj[i]] x = [lvls[idx[i]], lvls[idx[i] - 1]] y = [hum[nlvls - idx[i] - 1], hum[nlvls - idx[i]]] coef = np.polyfit(x, y, 1) hk[i] = coef[0] hb[i] = coef[1] #fill out the tmp and vpr array for i in xrange(nstn): id = np.int(idx[i] + 1) for n in xrange(id): tmp[n, i] = temp[nlvls - 1 - n, ii[i], jj[i]] exl = nlvls - id for m in xrange(exl): tmp[id + m, i] = tk[i] * lvls[id + m] + tb[i] for i in xrange(nstn): id = np.int(idx[i] + 1) for n in xrange(id): vpr[n, i] = humidity[nlvls - 1 - n, ii[i], jj[i]] exl = nlvls - id for m in xrange(exl): vpr[id + m, i] = hk[i] * lvls[id + m] + hb[i] memo = list(vpr) memo = np.array(memo) for i in xrange(nlvls): vpr[i, :] = memo[i, :] * lvls[i] * alpha / ( 1 + (alpha - 1) * memo[i, :]) # Close the hdf file file.end() if fname[-3:] == 'nc4': # Read the netcdf file file = netCDF4.Dataset(fname) if verbose: print 'PROGRESS: READING netcdf FILE' ncv = file.variables rlvls = ncv['lev'][:] # Pressure levels are from lowest to highest lvls = [] for i in xrange( len(rlvls) ): # Reverse the pressure levels to be consistent with other GAMs index = len(rlvls) - i - 1 lvls.append(rlvls[index]) nlvls = len(lvls) lvls = np.array(lvls) alpha = cdic['Rv'] / cdic['Rd'] # Select latitutde and longitude lats = ncv['lat'][:] lons = ncv['lon'][:] mask1 = (lats > minlat) & (lats < maxlat) mask2 = (lons > minlon) & (lons < maxlon) [ii] = np.where(mask1 == True) [jj] = np.where(mask2 == True) del mask1 del mask2 iimemo = [] for m in xrange(len(ii)): for i in xrange(len(jj)): iimemo.append(ii[m]) jjmemo = [] for i in xrange(len(ii)): jjmemo.append(jj) jjmemo = np.array(jjmemo) jjmemo = jjmemo.flatten() iimemo = np.array(iimemo) ii = iimemo jj = jjmemo latlist = lats[ii] lonlist = lons[jj] nstn = len(latlist) # Create arrays for 3D storage gph = np.zeros((nlvls, nstn)) #Potential height tmp = gph.copy() #Temperature vpr = gph.copy() #Vapor pressure if verbose: print 'Number of stations:', nstn # Get data from files height = ncv['H'][:] height = height[0] humidity = ncv['QV'][:] humidity = humidity[0] spressure = ncv['PS'][:] spressure = spressure[0] temp = ncv['T'][:] temp = temp[0] # Lvls are in hecto pascals, convert to pascals lvls = 100.0 * lvls # Reverse altitude for i in xrange(nlvls): index = nlvls - i - 1 gph[i, :] = height[index][ii, jj] idx = np.zeros(nstn) for i in xrange(nstn): for m in xrange(nlvls): if spressure[ii[i]][jj[i]] > lvls[m]: idx[i] = m # extrapolation of temperature and humidity data at pressure levels under surface at each grid point tk = np.zeros(nstn) tb = np.zeros(nstn) for i in xrange(nstn): t = temp[:, ii[i], jj[i]] x = [lvls[idx[i]], lvls[idx[i] - 1]] y = [t[nlvls - idx[i] - 1], t[nlvls - idx[i]]] coef = np.polyfit(x, y, 1) tk[i] = coef[0] tb[i] = coef[1] hk = np.zeros(nstn) hb = np.zeros(nstn) for i in xrange(nstn): hum = humidity[:, ii[i], jj[i]] x = [lvls[idx[i]], lvls[idx[i] - 1]] y = [hum[nlvls - idx[i] - 1], hum[nlvls - idx[i]]] coef = np.polyfit(x, y, 1) hk[i] = coef[0] hb[i] = coef[1] #fill out the tmp and vpr array for i in xrange(nstn): id = np.int(idx[i] + 1) for n in xrange(id): tmp[n, i] = temp[nlvls - 1 - n, ii[i], jj[i]] exl = nlvls - id for m in xrange(exl): tmp[id + m, i] = tk[i] * lvls[id + m] + tb[i] for i in xrange(nstn): id = np.int(idx[i] + 1) for n in xrange(id): vpr[n, i] = humidity[nlvls - 1 - n, ii[i], jj[i]] exl = nlvls - id for m in xrange(exl): vpr[id + m, i] = hk[i] * lvls[id + m] + hb[i] memo = list(vpr) memo = np.array(memo) for i in xrange(nlvls): vpr[i, :] = memo[i, :] * lvls[i] * alpha / ( 1 + (alpha - 1) * memo[i, :]) # Close the netcdf file file.close() # Send data return lvls, latlist, lonlist, gph, tmp, vpr
def doCompress(compType, value=0, v2=0): """Create and validate an HDF file using a compression scheme specified by the parameters""" # Build a significant file name if compType == SDC.COMP_NONE: fileName = "SDS.COMP_NONE" elif compType == SDC.COMP_RLE: fileName = "SDS.COMP_RLE" elif compType == SDC.COMP_SKPHUFF: fileName = "SDS.COMP_SKPHUFF.%d" % value elif compType == SDC.COMP_DEFLATE: fileName = "SDS.COMP_DEFLATE.%d" % value elif compType == SDC.COMP_SZIP: fileName = "SDS.COMP_SZIP" if value == SDC.COMP_SZIP_NN: fileName += ".NN" elif value == SDC.COMP_SZIP_EC: fileName += ".EC" else: print("illegal value") sys.exit(1) fileName += ".%s" % v2 else: print("illegal compType") sys.exit(1) fileName += ".hdf" SDS_NAME = "Data" fill_value = 0 #LENGTH = 9 #WIDTH = 6 # #data = numpy.array( ((100,100,200,200,300,400), # (100,100,200,200,300,400), # (100,100,200,200,300,400), # (300,300, 0,400,300,400), # (300,300, 0,400,300,400), # (300,300, 0,400,300,400), # (0, 0,600,600,300,400), # (500,500,600,600,300,400), # (0, 0,600,600,300,400)), NUMPY_DATATYPE) # The above dataset is used in the original NCSA example. # It is too small to show a significant size reduction after # compression. The following is used for a more realistic example. data = numpy.zeros((LENGTH, WIDTH), NUMPY_DATATYPE) for i in range(LENGTH): for j in range(WIDTH): data[i, j] = (i + j) * (i - j) # Create HDF file, wiping it out it it already exists. sd_id = SD(fileName, SDC.WRITE | SDC.CREATE | SDC.TRUNC) # Create dataset. sds_id = sd_id.create(SDS_NAME, HDF_DATATYPE, (LENGTH, WIDTH)) # Fill dataset will fill value. sds_id.setfillvalue(0) # Apply compression. try: sds_id.setcompress( compType, # compression type value, v2) # args depend on compression type except HDF4Error as msg: print(("Error compressing the dataset with params: " "(%d,%d,%d) : %s" % (compType, value, v2, msg))) sds_id.endaccess() sd_id.end() os.remove(fileName) return # Load data in the dataset. sds_id[:] = data # Close dataset. sds_id.endaccess() # Close hdf file to flush compressed data. sd_id.end() # Verify compressed data. # ###################### # Reopen file and select first dataset. sd_id = SD(fileName, SDC.READ) sds_id = sd_id.select(0) # Obtain compression info. compInfo = sds_id.getcompress() compType = compInfo[0] print("file : %s" % fileName) print(" size = %d" % os.path.getsize(fileName)) if compType == SDC.COMP_NONE: print(" compType = COMP_NONE") elif compType == SDC.COMP_RLE: print(" compType = COMP_RLE") elif compType == SDC.COMP_SKPHUFF: print(" compType = COMP_SKPHUFF") print(" dataSize = %d" % compInfo[1]) elif compType == SDC.COMP_DEFLATE: print(" compType = COMP_DEFLATE (GZIP)") print(" level = %d" % compInfo[1]) elif compType == SDC.COMP_SZIP: print(" compType = COMP_SZIP") optionMask = compInfo[1] if optionMask & SDC.COMP_SZIP_NN: print(" encoding scheme = NN") elif optionMask & SDC.COMP_SZIP_EC: print(" encoding scheme = EC") else: print(" unknown encoding scheme") sys.exit(1) pixelsPerBlock, pixelsPerScanline, bitsPerPixel, pixels = compInfo[2:] print(" pixelsPerBlock = %d" % pixelsPerBlock) print(" pixelsPerScanline = %d" % pixelsPerScanline) print(" bitsPerPixel = %d" % bitsPerPixel) print(" pixels = %d" % pixels) else: print(" unknown compression type") sys.exit(1) # Read dataset contents. out_data = sds_id[:] # Compare with original data. num_errs = 0 for i in range(LENGTH): for j in range(WIDTH): if data[i, j] != out_data[i, j]: print("bad value at %d,%d expected: %d got: %d" \ % (i,j,data[i,j],out_data[i,j])) num_errs += 1 # Close dataset and hdf file. sds_id.endaccess() sd_id.end() if num_errs == 0: print(" file validated") else: print(" file invalid : %d errors" % num_errs) print("")
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'Blue Radiance/RDQI' hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data3D = hdf.select(DATAFIELD_NAME) data = data3D[:,:,:] # Read attributes. attrs = data3D.attributes(full=1) fva=attrs["_FillValue"] _FillValue = fva[0] # Read geolocation dataset from another file. GEO_FILE_NAME = 'MISR_AM1_AGP_P117_F01_24.hdf' GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], GEO_FILE_NAME) hdf_geo = SD(GEO_FILE_NAME, SDC.READ) # Read geolocation dataset. lat3D = hdf_geo.select('GeoLatitude') lat = lat3D[:,:,:] lon3D = hdf_geo.select('GeoLongitude') lon = lon3D[:,:,:] # Read scale factor attribute. f = HDF(FILE_NAME, HC.READ) v = f.vgstart() vg = v.attach(8) # PyHDF cannot read attributes from Vgroup properly. # sfa = vg.attr('Scale Factor') # scale_factor = sfa.get() vg.detach() v.end() # Set it manually using HDFView. scale_factor = 0.047203224152326584 # We need to shift bits for "RDQI" to get "Blue Band "only. # See the page 84 of "MISR Data Products Specifications (rev. S)". # The document is available at [1]. datas = np.right_shift(data, 2); dataf = datas.astype(np.double) # Apply the fill value. dataf[data == _FillValue] = np.nan # Filter out values (> 16376) used for "Flag Data". # See Table 1.2 in "Level 1 Radiance Scaling and Conditioning # Algorithm Theoretical Basis" document [2]. dataf[datas > 16376] = np.nan datam = np.ma.masked_array(dataf, mask=np.isnan(dataf)) # Apply scale facotr. datam = scale_factor * datam; nblocks = data.shape[0] ydimsize = data.shape[1] xdimsize = data.shape[2] datam = datam.reshape(nblocks*ydimsize, xdimsize) lat = lat.reshape(nblocks*ydimsize, xdimsize) lon = lon.reshape(nblocks*ydimsize, xdimsize) # Set the limit for the plot. m = Basemap(projection='cyl', resolution='h', llcrnrlat=np.min(lat), urcrnrlat = np.max(lat), llcrnrlon=np.min(lon), urcrnrlon = np.max(lon)) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90., 120., 30.), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180., 181., 45.), labels=[0, 0, 0, 1]) m.pcolormesh(lon, lat, datam, latlon=True) cb = m.colorbar() cb.set_label(r'$Wm^{-2}sr^{-1}{\mu}m^{-1}$') basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, 'Blue Radiance')) fig = plt.gcf() # plt.show() pngfile = "{0}.py.agp.png".format(basename) fig.savefig(pngfile)
#colorbar() #title('OW') ############################################ #read AMSRE data ####---snow1: AMSREfolder = '/scratch/clisap/seaice/OWN_PRODUCTS/MELT_PONDS/GRID_AMSRE/' AMSREfile = AMSREfolder + 'asi-n6250-20110625-v5i.nc' AMSREfile2 = AMSREfolder + 'AMSR_E_L3_SeaIce12km_V12_20110625.hdf' ##ASI obj = NetCDFFile(AMSREfile) asi = array(obj.variables['icecon'][:], dtype=float32) / 10 asi[asi == -100] = nan ###NASA-TEAM2 AMSRE_NASA = SD.SD(AMSREfile2) nt2_key = 26 #'SI_12km_NH_89V_DSC' bt_key = 29 #'SI_12km_NH_ICECON_DSC' sd_ice_nt2 = AMSRE_NASA.select(nt2_key) ice_nt2 = sd_ice_nt2.get() ice_nt2 = array(ice_nt2, dtype=float)[::-1] ice_nt2[ice_nt2 >= 101] = nan ###NASA-BOOTSTRAP sd_ice_bt = AMSRE_NASA.select(bt_key) ice_bt = sd_ice_bt.get() # stored as difference to NT2 data (ice_bst-ice_nt2) ice_bt = array(ice_bt, dtype=float)[::-1] ice_bt = ice_bt + ice_nt2 ice_bt[ice_bt >= 101] = nan region = 'Arc'
def doCompress(compType, value=0, v2=0): """Create and validate an HDF file using a compression scheme sepcified by the parameters""" # Build a significant file name if compType == SDC.COMP_NONE: fileName = "SDS.COMP_NONE" elif compType == SDC.COMP_RLE: fileName = "SDS.COMP_RLE" elif compType == SDC.COMP_SKPHUFF: fileName = "SDS.COMP_SKPHUFF.%d" % value elif compType == SDC.COMP_DEFLATE: fileName = "SDS.COMP_DEFLATE.%d" % value elif compType == SDC.COMP_SZIP: fileName = "SDS.COMP_SZIP" if value == SDC.COMP_SZIP_NN: fileName += ".NN" elif value == SDC.COMP_SZIP_EC: fileName += ".EC" else: print "illegal value" sys.exit(1) fileName += ".%s" % v2 else: print "illegal compType" sys.exit(1) fileName += ".hdf" SDS_NAME = "Data" fill_value = 0 #LENGTH = 9 #WIDTH = 6 # #data = numpy.array( ((100,100,200,200,300,400), # (100,100,200,200,300,400), # (100,100,200,200,300,400), # (300,300, 0,400,300,400), # (300,300, 0,400,300,400), # (300,300, 0,400,300,400), # (0, 0,600,600,300,400), # (500,500,600,600,300,400), # (0, 0,600,600,300,400)), NUMPY_DATATYPE) # The above dataset is used in the original NCSA example. # It is too small to show a significant size reduction after # compression. The following is used for a more realistic example. data = numpy.zeros((LENGTH, WIDTH), NUMPY_DATATYPE) for i in range(LENGTH): for j in range(WIDTH): data[i,j] = (i+j)*(i-j) # Create HDF file, wiping it out it it already exists. sd_id = SD(fileName, SDC.WRITE | SDC.CREATE | SDC.TRUNC) # Create dataset. sds_id = sd_id.create(SDS_NAME, HDF_DATATYPE, (LENGTH, WIDTH)) # Fill dataset will fill value. sds_id.setfillvalue(0) # Apply compression. try: sds_id.setcompress(compType, # compression type value, v2) # args depend on compression type except HDF4Error, msg: print("Error compressing the dataset with params: " "(%d,%d,%d) : %s" % (compType, value, v2, msg)) sds_id.endaccess() sd_id.end() os.remove(fileName) return
def sdscreate(sd, name): # Create a simple 3x3 float array. sds = sd.create(name, SDC.FLOAT32, (3, 3)) # Initialize array sds[:] = ((0, 1, 2), (3, 4, 5), (6, 7, 8)) # "close" dataset. sds.endaccess() # Create HDF file filename = 'inventory.hdf' hdf = HDF(filename, HC.WRITE | HC.CREATE) # Initialize the SD, V and VS interfaces on the file. sd = SD(filename, SDC.WRITE) # SD interface vs = hdf.vstart() # vdata interface v = hdf.vgstart() # vgroup interface # Create vdata named 'INVENTORY'. vdatacreate(vs, 'INVENTORY') # Create dataset named "ARR_3x3" sdscreate(sd, 'ARR_3x3') # Attach the vdata and the dataset. vd = vs.attach('INVENTORY') sds = sd.select('ARR_3x3') # Create vgroup named 'TOTAL'. vg = v.create('TOTAL')
return # Load data in the dataset. sds_id[:] = data # Close dataset. sds_id.endaccess() # Close hdf file to flush compressed data. sd_id.end() # Verify compressed data. # ###################### # Reopen file and select first dataset. sd_id = SD(fileName, SDC.READ) sds_id = sd_id.select(0) # Obtain compression info. compInfo = sds_id.getcompress() compType = compInfo[0] print "file : %s" % fileName print " size = %d" % os.path.getsize(fileName) if compType == SDC.COMP_NONE: print " compType = COMP_NONE" elif compType == SDC.COMP_RLE: print " compType = COMP_RLE" elif compType == SDC.COMP_SKPHUFF: print " compType = COMP_SKPHUFF" print " dataSize = %d" % compInfo[1] elif compType == SDC.COMP_DEFLATE:
import numpy as np import matplotlib as mpl import matplotlib.pyplot as plt import matplotlib.cm as cm from pyhdf.SD import * import sys # np.set_printoptions(threshold='nan') _filename = "C:\Users\AKO NA LNG\Desktop\Esquivel Files\School Files 2\Special Problem\MODIS\MOD35\MOD35_L2.A2015348.0215.006.2015348134053.hdf" mod35 = SD(_filename, SDC.READ) sds_name_cm = "Cloud_Mask" sds_index_cm = mod35.select(sds_name_cm) cm_sds_data = sds_index_cm.get() sds_data_0 = cm_sds_data[0, :, :] sds_data_0_bin = sds_data_0.astype(dtype=np.uint8, order="K", casting="unsafe", subok=False, copy=False) for x in np.nditer(sds_data_0_bin, op_flags=["readwrite"]): x[...] = np.right_shift(x, 6) for y in np.nditer(sds_data_0_bin, op_flags=["readwrite"]): if y[...] == 3 or y[...] == 2: y[...] = 0
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'RegBestEstimateSpectralOptDepth' hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data4D = hdf.select(DATAFIELD_NAME) # Subset the Blue Band. 1=Blue, 2=Green, 3=Red, 4=NIR. data = data4D[:,:,:,0].astype(np.double) # Read geolocation dataset from HDF-EOS2 dumper output. GEO_FILE_NAME = 'lat_MISR_AM1_AS_AEROSOL_P004_O066234_F12_0022.output' GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], GEO_FILE_NAME) lat = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0]) lat = lat.reshape(data.shape) GEO_FILE_NAME = 'lon_MISR_AM1_AS_AEROSOL_P004_O066234_F12_0022.output' GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'], GEO_FILE_NAME) lon = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0]) lon = lon.reshape(data.shape) # Read attributes. attrs = data4D.attributes(full=1) fva=attrs["_FillValue"] _FillValue = fva[0] # Apply the fill value. data[data == _FillValue] = np.nan datam = np.ma.masked_array(data, mask=np.isnan(data)) nblocks = data.shape[0] ydimsize = data.shape[1] xdimsize = data.shape[2] datam = datam.reshape(nblocks*ydimsize, xdimsize) lat = lat.reshape(nblocks*ydimsize, xdimsize) lon = lon.reshape(nblocks*ydimsize, xdimsize) # Set the limit for the plot. m = Basemap(projection='cyl', resolution='h', llcrnrlat=np.min(lat), urcrnrlat = np.max(lat), llcrnrlon=np.min(lon), urcrnrlon = np.max(lon)) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90., 120., 30.), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180., 181., 45.), labels=[0, 0, 0, 1]) m.pcolormesh(lon, lat, datam, latlon=True) cb = m.colorbar() cb.set_label('No Unit') basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1} - Blue Band'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)