def read_vdata(FILENAME): """Reads all vdata fields in an HDF-EOS file and places those fields in a dictionary. PARAMETERS: ----------- FILENAME: Name of HDF-EOS file. OUTPUTS: -------- vdata_dict: Dictionary of vdata fields. """ # Prepare to read the data. f = HDF(FILENAME, SDC.READ) # Open the file vs = f.vstart() # Start the vdata interface data_info_list = vs.vdatainfo() # List the vdata fields vdata_fieldnames = [a[0] for a in data_info_list] # Get the names # Load the data, place in dictionary vdata_dict = {} for field in vdata_fieldnames: vdata_dict[field] = np.squeeze(np.asarray(vs.attach(field)[:])) # terminate the vdata interface, close the file. vs.end() f.close() return vdata_dict
def get_solor_angle(myd1km_filename_str): try: # open hdf myd021km_data = SD(myd1km_filename_str) # 1.SDS Reading zenith_sds = myd021km_data.select('SensorZenith') # 经纬度 zenith = zenith_sds.get() # 2 azimuth_sds = myd021km_data.select('SensorAzimuth') # 经纬度 azimuth = azimuth_sds.get() return zenith * 0.01, azimuth * 0.01 f = HDF(myd1km_filename_str, SDC.READ) vs = f.vstart() data_info_list = vs.vdatainfo() # Vset table # L1B swam matedata L1B_Swath_Matedata_VD = vs.attach('Level 1B Swath Metadata') # Read [Swath/scan type] sd_info = L1B_Swath_Matedata_VD.inquire() all_metadata = L1B_Swath_Matedata_VD.read(sd_info[0]) L1B_Swath_Matedata_VD.detach() # __del__ with handle this. vs.end() # Use with, so __del__ can do this. f.close() return all_metadata except HDF4Error: print("Unexpected error:(get_solor_angle)", sys.exc_info()[0]) print('READ ERROR......:' + myd1km_filename_str) return None
def h4lookup(path, swath = "Earth UV-2 Swath"): ''' only look-up datasets, ignore vdata and "WavelengthReferenceColumn" is that. ''' hdf = HDF(path) v = hdf.vgstart() s2_vg = v.attach(swath) geo_tag, geo_ref = s2_vg.tagrefs()[0] dat_tag, dat_ref = s2_vg.tagrefs()[1] s2_vg.detach() #-------------------------------------------- # found geoloaction & data fields #-------------------------------------------- geo_vgs = v.attach(geo_ref); dat_vgs = v.attach(dat_ref) gvg_tagrefs = geo_vgs.tagrefs(); dvg_tagrefs = dat_vgs.tagrefs() geo_vgs.detach(); dat_vgs.detach() tagrefs_list = gvg_tagrefs + dvg_tagrefs refs_dict = {} #-------------------------------------------- # create dict in which keys are names in hdf and values are refs #-------------------------------------------- sd = SD(path) for tr in tagrefs_list: tag, ref = tr if tag == HC.DFTAG_NDG: sds = sd.select(sd.reftoindex(ref)) refs_dict[sds.info()[0]] = ref sds.endaccess(); sd.end(); v.end(); hdf.close() return refs_dict
def load_vd(fname, varnames): '''return list containing vdata structures specified by list varnames, contained in hdf4 file with filename fname''' f = HDF(fname) data_list = [] vs = f.vstart() for name in varnames: vd = vs.attach(name) data_list.append(vd[:]) vd.detach() vs.end() f.close() return data_list
def read_data_from_hdf(inputfile): wavelengths = [] depth = [] downwelling_downcast = [] downwelling_upcast = [] upwelling_downcast = [] upwelling_upcast = [] # open the hdf file read-only # Initialize the SD, V and VS interfaces on the file. hdf = HDF(inputfile) vs = hdf.vstart() v = hdf.vgstart() # Attach and read the contents of the Profiler vgroup vg = v.attach(v.find('Profiler')) for tag, ref in vg.tagrefs(): assert(tag == HC.DFTAG_VH) vd = vs.attach(ref) nrecs, intmode, fields, size, name = vd.inquire() if name == "ED_hyperspectral_downcast": x = vd.read(nrecs) downwelling_downcast = np.asarray([i[3:] for i in x]) wavelengths = np.asarray([float(x) for x in fields[3:]]) depth = np.asarray([i[2] for i in x]) elif name == "ED_hyperspectral_upcast": downwelling_upcast = np.asarray([i[3:] for i in vd.read(nrecs)]) elif name == "LU_hyperspectral_downcast": upwelling_downcast = np.asarray([i[3:] for i in vd.read(nrecs)]) elif name == "LU_hyperspectral_upcast": upwelling_upcast = np.asarray([i[3:] for i in vd.read(nrecs)]) vd.detach() # Close vgroup vg.detach() #clean up v.end() vs.end() hdf.close() return wavelengths, depth, downwelling_downcast, downwelling_upcast, \ upwelling_downcast, upwelling_upcast
def HDFread1D(filename, variable): """ Read HDF file in vs in simple mode """ # Read the file f = HDF(filename) # Initialize v mode vs = f.vstart() # extrect data var = vs.attach(variable) Var = np.array(var[:]).ravel() # Close the file var.detach() vs.end() f.close() return Var
def HDFread(filename, variable, Class=None): """ Extract the data for non scientific data in V mode of hdf file """ hdf = HDF(filename, HC.READ) # Initialize the SD, V and VS interfaces on the file. sd = SD(filename) vs = hdf.vstart() v = hdf.vgstart() # Encontrar el puto id de las Geolocation Fields if Class == None: ref = v.findclass('SWATH Vgroup') else: ref = v.findclass(Class) # Open all data of the class vg = v.attach(ref) # All fields in the class members = vg.tagrefs() nrecs = [] names = [] for tag, ref in members: # Vdata tag vd = vs.attach(ref) # nrecs, intmode, fields, size, name = vd.inquire() nrecs.append(vd.inquire()[0]) # number of records of the Vdata names.append(vd.inquire()[-1]) # name of the Vdata vd.detach() idx = names.index(variable) var = vs.attach(members[idx][1]) V = var.read(nrecs[idx]) var.detach() # Terminate V, VS and SD interfaces. v.end() vs.end() sd.end() # Close HDF file. hdf.close() return np.array(V).ravel()
def get_train_data(self, index_tuple, band_select=[ 0, 1, 2, 3, 4, 6, 7, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37 ]): ''' [Varification & Night eliminite] :param index_tuple: :return: ''' # [band_num, rowsize] raw_data = self.get_by_index(index_tuple, verify=False) # Verify raw_data = raw_data[band_select, ...] valid_flag = np.ones([raw_data.shape[1]], dtype=bool) for idx in range(raw_data.shape[0]): valid_flag = valid_flag & (raw_data[idx, ...] > 32767) # Daytime mode try: # open hdf f = HDF(self.filename, SDC.READ) vs = f.vstart() data_info_list = vs.vdatainfo() # Vset table # L1B swam matedata L1B_Swath_Matedata_VD = vs.attach('Level 1B Swath Metadata') # Read [Swath/scan type] svath_matedata = L1B_Swath_Matedata_VD[:] for idx in range(valid_flag.shape[0]): if svath_matedata[int( (index_tuple[0][idx]) / 10)][2] == 'D ': valid_flag[idx] = True else: valid_flag[idx] = False L1B_Swath_Matedata_VD.detach() # __del__ with handle this. vs.end() # Use with, so __del__ can do this. f.close() except ValueError: print("Unexpected error:", sys.exc_info()[0]) print('READ ERROR......(%d):' % self.filename) return 0, 0, False raw_data = self.get_by_index(index_tuple, verify=False, scaled=True) raw_data = raw_data[band_select, ...] return raw_data, valid_flag
def get_swath_metadata(myd1km_filename_str): try: # open hdf f = HDF(myd1km_filename_str, SDC.READ) vs = f.vstart() data_info_list = vs.vdatainfo() # Vset table # L1B swam matedata L1B_Swath_Matedata_VD = vs.attach('Level 1B Swath Metadata') # Read [Swath/scan type] sd_info = L1B_Swath_Matedata_VD.inquire() all_metadata = L1B_Swath_Matedata_VD.read(sd_info[0]) L1B_Swath_Matedata_VD.detach() # __del__ with handle this. vs.end() # Use with, so __del__ can do this. f.close() return all_metadata except HDF4Error: print("Unexpected error:", sys.exc_info()[0]) print('READ ERROR......:' + myd1km_filename_str) return None
def look(self, path, mem_list, lp_list): data = {} for name in mem_list: # subdata sets type data tag = lp_list[name][0] ref = lp_list[name][1] if tag == HC.DFTAG_NDG: sd = SD(path) sds = sd.select(sd.reftoindex(ref)) data[name] = np.float64(sds.get()) sds.endaccess() sd.end() elif tag == HC.DFTAG_VH: #vd type data hdf = HDF(path) vs = hdf.vstart() vd = vs.attach(ref) nrecs, intmode, fields, size, name = vd.inquire() data[name] = np.full(nrecs, np.float64(vd.read()[0])) vs.end() hdf.close() return data
def read_vd_hdf2(file_in,var_in): '''Read data stored as a VD variable in input HDF-EOS file.''' '''Inputs are the file path and the required variable name.''' '''Outputs are the data (numpy array). ''' #--information from input file-- hdf = HDF(file_in) # the HDF file vs = hdf.vstart() # initialize VS interface on HDF file ### vdinfo = vs.vdatainfo() # return info about all vdatas vd = vs.attach(var_in) # open a vdata given its name dimsz = vd.inquire()[0] # return 5 elements: # 1. # of records # 2. interlace mode # 3. list of vdata field names # 4. size in bytes of the vdata record # 5. name of the vdata var_data = np.array(vd.read(dimsz)) #read a number of records vd.detach() # close the vdata vs.end() # terminate the vdata interface hdf.close() # close the HDF file return var_data,dimsz
def get_myd1km_sci_time(myd1km_filename_str): try: # open hdf f = HDF(myd1km_filename_str, SDC.READ) vs = f.vstart() data_info_list = vs.vdatainfo() # Vset table # L1B swam matedata L1B_Swath_Matedata_VD = vs.attach('Level 1B Swath Metadata') # Read [Swath/scan type] begin = L1B_Swath_Matedata_VD[0] begin = begin[4] end = L1B_Swath_Matedata_VD[-1] end = end[4] L1B_Swath_Matedata_VD.detach() # __del__ with handle this. vs.end() # Use with, so __del__ can do this. f.close() return begin, end, True except HDF4Error: print("Unexpected error:", sys.exc_info()[0]) print('READ ERROR......:' + myd1km_filename_str) return 0, 0, False
from pyhdf.HDF import * from pyhdf.VS import * f = HDF('inventory.hdf', # Open file 'inventory.hdf' in write mode HC.WRITE|HC.CREATE) # creating it if it does not exist vs = f.vstart() # init vdata interface vd = vs.attach('INVENTORY', 1) # attach vdata 'INVENTORY' in write mode # Update the `status' vdata attribute. The attribute length must not # change. We call the attribute info() method, which returns a list where # number of values (eg string length) is stored at index 2. # We then assign a left justified string of exactly that length. len = vd.attr('status').info()[2] vd.status = '%-*s' % (len, 'phase 3 done') # Update record at index 1 (second record) vd[1] = ('Z4367', 'surprise', 10, 3.1, 44.5) # Update record at index 4, and those after vd[4:] = ( ('QR231', 'toy', 12, 2.5, 45), ('R3389', 'robot', 3, 45, 2000), ('R3390', 'robot2', 8, 55, 2050) ) vd.detach() # "close" the vdata vs.end() # terminate the vdata interface f.close() # close the HDF file
def get_train_data_map(self, index_tuple, kernel_radius=5, band_select=[ 0, 1, 2, 3, 4, 6, 7, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37 ]): ''' [Varification & Night eliminite] :param index_tuple: :return: ''' # [band_num, rowsize] selected_band_num = len(band_select) kernel_size = 2 * kernel_radius + 1 # Verify valid_flag = np.ones([index_tuple[0].shape[0]], dtype=bool) # Daytime mode try: # open hdf f = HDF(self.filename, SDC.READ) vs = f.vstart() data_info_list = vs.vdatainfo() # Vset table # L1B swam matedata L1B_Swath_Matedata_VD = vs.attach('Level 1B Swath Metadata') # Read [Swath/scan type] svath_matedata = L1B_Swath_Matedata_VD[:] for idx in range(valid_flag.shape[0]): if svath_matedata[int( (index_tuple[0][idx]) / 10)][2] == 'D ': valid_flag[idx] = True else: valid_flag[idx] = False L1B_Swath_Matedata_VD.detach() # __del__ with handle this. vs.end() # Use with, so __del__ can do this. f.close() except ValueError: print("Unexpected error:", sys.exc_info()[0]) print('READ ERROR......(%d):' % self.filename) return 0, 0, False if np.sum(valid_flag) == 0: return None, valid_flag raw_data = np.empty([ index_tuple[0].shape[0], selected_band_num, kernel_size, kernel_size ]) # Create a Mat of kernel kernel_i = np.empty([kernel_size, kernel_size], dtype=np.int16) for i in range(kernel_size): for j in range(kernel_size): kernel_i[i, j] = i kernel_i -= kernel_radius kernel_i = np.reshape(kernel_i, kernel_i.size) kernel_j = np.transpose(kernel_i) kernel_j = np.reshape(kernel_j, kernel_j.size) for idx in range(index_tuple[0].shape[0]): if not valid_flag[idx]: continue temp_kernel_i = kernel_i + index_tuple[0][idx] if np.sum((temp_kernel_i < 0) | (temp_kernel_i >= self.width)) > 0: valid_flag[idx] = False continue temp_kernel_j = kernel_j + index_tuple[1][idx] if np.sum((temp_kernel_j < 0) | (temp_kernel_j >= self.height)) > 0: valid_flag[idx] = False continue temp = (self.all_band[..., temp_kernel_i, temp_kernel_j])[band_select, ...] if np.sum(temp[..., int(kernel_size * kernel_size * 0.5)] > 32767): valid_flag[idx] = False else: indeces = temp > 32767 temp[indeces] = 0 raw_data[idx, ...] = temp.reshape(selected_band_num, kernel_size, kernel_size) for idx in range(selected_band_num): raw_data[:, idx, :, :] = ( raw_data[:, idx, :, :] - self.offset[band_select[idx]]) * self.scale[band_select[idx]] return raw_data, valid_flag
# Attach the vdata and the dataset. vd = vs.attach('INVENTORY') sds = sd.select('ARR_3x3') # Create vgroup named 'TOTAL'. vg = v.create('TOTAL') # Add vdata to the vgroup vg.insert(vd) # We could also have written this: # vgroup.add(vd._tag, vd._refnum) # or this: # vgroup.add(HC.DFTAG_VH, vd._refnum) # Add dataset to the vgroup vg.add(HC.DFTAG_NDG, sds.ref()) # Close vgroup, vdata and dataset. vg.detach() # vgroup vd.detach() # vdata sds.endaccess() # dataset # Terminate V, VS and SD interfaces. v.end() # V interface vs.end() # VS interface sd.end() # SD interface # Close HDF file. hdf.close()
print "unhandled tag,ref",tag,ref # Close vgroup vg.detach() # Open HDF file in readonly mode. filename = sys.argv[1] hdf = HDF(filename) # Initialize the SD, V and VS interfaces on the file. sd = SD(filename) vs = hdf.vstart() v = hdf.vgstart() # Scan all vgroups in the file. ref = -1 while 1: try: ref = v.getid(ref) except HDF4Error,msg: # no more vgroup break describevg(ref) # Terminate V, VS and SD interfaces. v.end() vs.end() sd.end() # Close HDF file. hdf.close()
def mosaic(*arg,**args): # This function will take files tranfered in *arg and will mosaic them together and produce a new file # mosaic(file1,[file2,file3,...],endfile) # If only file1 and endfile is given, the function will only produce a copy without any other modification try: log=args["log"] except KeyError: log=logMod.Log("",nolog=True) if len(arg)>2: lfile = arg[:-1] # This is the list of the NAME of the files to merge newfilename = arg[-1] # This is the final file NAME # Should eventually check if files exists and can be read ***IMPROVE*** lfHDF = [] # This is the list of the FILES to merge latt = [] # This is the list of the ATTRIBUTE "StructMetadata.0" of the files for fil in lfile: try: a=SD(fil,SDC.READ) except TypeError: a=SD(fil.encode('ascii','ignore'),SDC.READ) lfHDF.append(a) #print("hoho") latt.append(atribute(lfHDF[-1].attributes()["StructMetadata.0"],fil,dsi=[0,])) ## Listing all the GRIDS that the new file will have gridlist = [] # This is the list of GRIDS to include in the final file for attOfF in latt: # Should check if any grid ***IMPROVE*** gridlist += attOfF.listgridname()[1] # listgridname return a list of all the grids name # remove double entry gridlist = list(set(gridlist)) ## Listing all the DATASETS that the new file will have dslist = [] # This is the list of DATASETS to include in the final file for attOfF in latt: # Should check if any grid ***IMPROVE*** dslist = attOfF.orderedDS() # remove double entry # dslist = list(set(dslist)) ## Validation of commoun information ############################################################################### # Some informations have to be the same for each file or else no mosaic can # # be made for exemple two files with not the same projection type can't be # # merged together. ***IMPROVE*** Maybe in the future we could transform file # # so that they have the same projection or some other thing. # ############################################################################### # List of parameter to check to insure that they are the same paramMustSim = ["Projection","ProjParams","SphereCode"] # Dictionary that will keep all the informations about every file paramMustSimDict = {} for grid in gridlist: # Verification of a grid first = True # Variable that will enable the construction of the dict paramdict = {} # Dictionary that keep the actual value that have to be the same for attOfF in latt: # Verification of a file bigG = attOfF.getgridbyname(grid) # Getting all the attributes in the grid of a file if bigG is not None: # If the grid exists in that file if first : # If this is the first time that a file is check for that grid first = False for p in paramMustSim: # Checking every parameters that must be the same paramdict[p] = bigG.variable[p] # Validation of same Dtype for each datafield go = bigG.GROUP["DataField"].OBJECT for r in go: paramdict[go[r].variable["DataFieldName"]]=go[r].variable["DataType"] else: # If it's not the first time that a file is check for that grid for p in paramMustSim: # Checking every parameters that must be the same if not paramdict[p]==bigG.variable[p]: # Stop merging and return error ***IMPROVE*** # Maybe do only the things that can be done ***IMPROVE*** log.log('e',Nom,"Error dataset are not compatible") # Validation of same Dtype for each datafield go=bigG.GROUP["DataField"].OBJECT for r in go: if not paramdict[go[r].variable["DataFieldName"]]==go[r].variable["DataType"]: # Stop merging and return error ***IMPROVE*** # Maybe do only the things that can be done ***IMPROVE*** log.log('e',Nom,"Error dataset are not compatible") # Keep all this info for later it's going to be useful paramMustSimDict[grid]=paramdict ## Determination of new informations ############################################################################### # Some properties have to be calculated in order to merge. This section is # # doing just that # ############################################################################### gridResolX={} # Getting the RESOLUTION in the X direction for each grid gridResolY={} # Getting the RESOLUTION in the Y direction for each grid extremeup={} # Getting the UPPER coordinates for each grid extremedown={} # Getting the LOWEST coordinates for each grid extremeleft={} # Getting the LEFTMOST coordinates for each grid extremeright={} # Getting the RIGHTMOST coordinates for each grid gridDimX={} # Getting the DIMENSIONS of X direction for each grid gridDimY={} # Getting the DIMENSIONS of Y direction for each grid NoValueDS={} # Getting the fill value of each dataset dtypeDS={} # Getting the DTYPE for each dataset dstogrid={} # Knowing wich is the GRID for each dataset filGridULC={} # Getting the upper left corner of each file for each grid for grid in gridlist: # For each grid filGridULC[grid]={} # Adding a dictionary for each grid that will contain information on every file for attOfF in latt: ### Determination of resolution of each grid # ***IMPROVE*** Should check if bigd is none bigG=attOfF.getgridbyname(grid) # Getting all the attributes in the grid of a file # Get extreme grid point ulp=eval(bigG.variable["UpperLeftPointMtrs"]) lrp=eval(bigG.variable["LowerRightMtrs"]) # Get grid dimmension dimx=int(bigG.variable["XDim"]) dimy=int(bigG.variable["YDim"]) # Calculate grid resolution gridResolX[grid]=(lrp[0]-ulp[0])/dimx gridResolY[grid]=(ulp[1]-lrp[1])/dimy ### Determination of new extreme coordinates for each grid # up try: if extremeup[grid]< ulp[1]: extremeup[grid]=ulp[1] except KeyError: extremeup[grid]=ulp[1] # down try: if extremedown[grid]> lrp[1]: extremedown[grid]=lrp[1] except KeyError: extremedown[grid]=lrp[1] # left try: if extremeleft[grid]> ulp[0]: extremeleft[grid]=ulp[0] except KeyError: extremeleft[grid]=ulp[0] # right try: if extremeright[grid]< lrp[0]: extremeright[grid]=lrp[0] except KeyError: extremeright[grid]=lrp[0] ### Detetermination of dataset to grid name if bigG is not None: go=bigG.GROUP["DataField"].OBJECT for r in go: dstogrid[ go[r].variable["DataFieldName"] ] = grid ## Determination of ULC for each grid in each file filGridULC[grid][attOfF.name] = ulp ## determination of new dimension for each grid gridDimY[grid] = int((extremeup[grid]-extremedown[grid])/gridResolY[grid]) gridDimX[grid] = int((extremeright[grid]-extremeleft[grid])/gridResolX[grid]) for ds in dslist: # For each dataset for sd in lfHDF: # For each hdf file # Try opening dataset try: sds = sd.select(eval(ds)) # Get fill value NoValueDS[ds] = sds.getfillvalue() # Get dtype dtypeDS[ds] = sds.info()[3] except: log.log('e',Nom,"no dataset") ## Start creating new file ############################################################################### # This is the actual part were stuf appens # ############################################################################### # This part is the same for every file in any circumstances ########## absolute ######################## # Open new file try: hdf = HDF(newfilename, HC.WRITE | HC.CREATE |HC.TRUNC) sd = SD(newfilename, SDC.WRITE | SDC.CREATE ) except TypeError: hdf = HDF(newfilename.encode('ascii','ignore'), HC.WRITE | HC.CREATE |HC.TRUNC) sd = SD(newfilename.encode('ascii','ignore'), SDC.WRITE | SDC.CREATE ) v=hdf.vgstart() vg={} vg1={} vg2={} ## rewrite the gridlist gridlist = [] for ds in dslist: if dstogrid[ds] not in gridlist: gridlist.append(dstogrid[ds]) for grid in gridlist: vg[grid]=v.attach(-1,write=1) vg[grid]._class="GRID" vg[grid]._name=eval(grid) vg1[grid]=v.attach(-1,write=1) vg2[grid]=v.attach(-1,write=1) vg1[grid]._class="GRID Vgroup" vg1[grid]._name="Data Fields" vg2[grid]._class="GRID Vgroup" vg2[grid]._name="Grid Attributes" vg[grid].insert(vg1[grid]) vg[grid].insert(vg2[grid]) ########## absolute ######################## # Create dataset with the right size for ds in dslist: theGrid=dstogrid[ds] # Get grid name of data set sds = sd.create(eval(ds),dtypeDS[ds],(gridDimY[theGrid],gridDimX[theGrid])) # Set fill value fv=NoValueDS[ds] try: sds.setfillvalue(NoValueDS[ds]) except OverflowError: log.log('e',Nom,"setfillvalue") sds.setfillvalue(0) ## write real data for fil in range(len(latt)): try: # Determine were the data will be writen ulc = filGridULC[theGrid][latt[fil].name] # Determine the position on the grid y = (extremeup[theGrid]-ulc[1])/(extremeup[theGrid]-extremedown[theGrid]) x = (ulc[0]-extremeleft[theGrid])/(extremeright[theGrid]-extremeleft[theGrid]) y = int(y*gridDimY[theGrid]) x = int(x*gridDimX[theGrid]) # read data from files osds = lfHDF[fil].select(eval(ds)) sh = osds[:].shape sds[y:y+sh[0],x:x+sh[1]] = osds[:] osds.endaccess() except: pass # Close sds vg1[dstogrid[ds]].add(HC.DFTAG_NDG,sds.ref()) sds.endaccess() for g in vg1: vg1[g].detach() vg2[g].detach() vg[g].detach() # Create attribute table for the file attstr="GROUP=GridStructure\n" gridcount=1 for gr in gridlist: # Start group grid attstr+="\tGROUP=GRID_%i\n"%gridcount # Add grid name attstr+="\t\tGridName=%s\n"%gr # Add dimention attstr+="\t\tXDim=%i\n"%gridDimX[gr] attstr+="\t\tYDim=%i\n"%gridDimY[gr] # Add UpperLeftPointMtrs attstr+="\t\tUpperLeftPointMtrs=(%f,%f)\n"%(extremeleft[gr],extremeup[gr]) # Add lrp attstr+="\t\tLowerRightMtrs=(%f,%f)\n"%(extremeright[gr],extremedown[gr]) # Add projection attstr+="\t\tProjection=%s\n"%paramMustSimDict[gr]["Projection"] # ProjParams attstr+="\t\tProjParams=%s\n"%paramMustSimDict[gr]["ProjParams"] # SphereCode attstr+="\t\tSphereCode=%s\n"%paramMustSimDict[gr]["SphereCode"] attstr+="""\t\tGROUP=Dimension \t\tEND_GROUP=Dimension \t\tGROUP=DataField\n""" ## Add data sets # create list of ds for current grid lsdsgr=[] dsnum=1 for ds in dslist: if dstogrid[ds] == gr: # Add object attstr+="\t\t\tOBJECT=DataField_%i\n"%dsnum # datafield name attstr+="\t\t\t\tDataFieldName=%s\n"%ds # datatype attstr+="\t\t\t\tDataType=%s\n"%paramMustSimDict[gr][ds] # dim attstr+='\t\t\t\tDimList=("YDim","XDim")\n' attstr+="\t\t\tEND_OBJECT=DataField_%i\n"%dsnum dsnum+=1 attstr+="\t\tEND_GROUP=DataField\n" attstr+="""\t\tGROUP=MergedFields \t\tEND_GROUP=MergedFields\n""" attstr+="\tEND_GROUP=GRID_%i\n"%gridcount gridcount+=1 attstr+="""END_GROUP=GridStructure GROUP=PointStructure END_GROUP=PointStructure END""" # adding attribute to new file att=sd.attr('StructMetadata.0') att.set(SDC.CHAR,attstr) sd.end() hdf.close() # This should return something somehow elif len(arg)>1: afile = arg[0] # This is the list of the NAME of the files to merge newfilename = arg[1] # This is the final file NAME # Create a copy from shutil import copyfile copyfile(afile,newfilename)
#----------------------------------------------------------------------------------------# # Read HDF Files (VD data) Latitude & Longitude f = HDF(file_path + file_name, SDC.READ) vs = f.vstart() Latitude = vs.attach('Latitude') Longitude = vs.attach('Longitude') a = Latitude[:] Latitude.detach() Longitude.detach() vs.end() f.close() #----------------------------------------------------------------------------------------# # SDS Data file = SD(file_path + file_name, SDC.READ) file_info = file.info() print(file_info) # number of sds and metadata print('---------- CloudLayerBase ----------') sds_obj = file.select('CloudLayerBase') # select sds CloudLayerBase = sds_obj.get()
for fieldName in vd._fields: # loop over all field names try: # instantiate field and obtain value of attribute 'unit' v = vd.field(fieldName).unit print("%s: %s" % (fieldName, v), end=' ') except: # no 'unit' attribute: ignore pass print("") print("") # Display table header. header = "%-7s %-12s %3s %4s %8s" % tuple(vd._fields) print("-" * len(header)) print(header) print("-" * len(header)) # Loop over the vdata records, displaying each record as a table row. # Current record position is 0 after attaching the vdata. while True: try: rec = vd.read() # read next record # equivalent to: # rec = vd[vd.tell()] print("%-7s %-12s %3d %4.1f %8.2f" % tuple(rec[0])) except HDF4Error: # end of vdata reached break vd.detach() # "close" the vdata vs.end() # terminate the vdata interface f.close() # close the HDF file
def PFlux(year1, month1, day1): filename = external_dir + "swepam_data_1day.hdf" hdf = HDF(filename) #Initialize the V interface on the HDF file. v = hdf.vgstart() vs = hdf.vstart() #Scan all vgroups in the file ref = -1 refnum = v.getid(ref) vg = v.attach(refnum) #print "----------------" #print "name:", vg._name, "class:",vg._class, "tag,ref:", #print vg._tag, vg._refnum # Show the number of members of each main object type. #print "members: ", vg._nmembers, #print "datasets:", vg.nrefs(HC.DFTAG_NDG), #print "vdataset: ", vg.nrefs(HC.DFTAG_VH), #print "vgroups: ", vg.nrefs(HC.DFTAG_VG) # Read the contents of the vgroup. members = vg.tagrefs() # Display info about each member. index = -1 for tag, ref in members: index += 1 # print "member index", index # Vdata tag if tag == HC.DFTAG_VH: vd = vs.attach(ref) nrecs, intmode, fields, size, name = vd.inquire() # print " vdata:",name, "tag,ref:",tag, ref # print " fields:",fields # print " nrecs:",nrecs vd.detach() # SDS tag elif tag == HC.DFTAG_NDG: sds = sd.select(sd.reftoindex(ref)) name, rank, dims, type, nattrs = sds.info() # print " dataset:",name, "tag,ref:", tag, ref # print " dims:",dims # print " type:",type sds.endaccess() # VS tag elif tag == HC.DFTAG_VG: vg0 = v.attach(ref) # print " vgroup:", vg0._name, "tag,ref:", tag, ref vg0.detach() # Unhandled tag else: print("unhandled tag,ref", tag, ref) # Close vgroup members = vg.tagrefs() (tag, ref) in members vd = vs.attach(ref) nrecs, intmode, fields, size, name = vd.inquire() alldata = vd.read(nrecs) vd.detach() vg.detach() v.end() hdf.close() data = np.array(alldata) # input # (y,m,d) = (1998,3,1) year = data[:, 0] day = data[:, 1] hr = data[:, 2] min = data[:, 3] sec = data[:, 4] fp_year = data[:, 5] fp_doy = data[:, 6] pdensity = data[:, 8] speed = data[:, 11] start = datetime.date(int(year[0]), 1, 1) + datetime.timedelta(int(day[0]) - 1) end = datetime.date(int(year[-1]), 1, 1) + datetime.timedelta(int(day[-1]) - 1) # print "star Date and End Date:", start, end delta1 = datetime.date(year1, month1, day1) - start index = delta1.days # print "index ....", index if index >= (nrecs - 1): print("ERROR: the input time is too new") # break elif index < 0: print("ERROR: the input time is too old") # break else: avePF = 0 num = 0 for i in range(0, 90): j = index - i if j < 0: print("ERROR: the index is out of the array") break else: if pdensity[j] > 0 and speed[j] > 0: avePF = avePF + pdensity[j] * speed[j] num = num + 1 avePF = avePF / num # print "the 90 days average proton flux is:",avePF, num return avePF
def genCloudSatFigure(filepath, data_field, title, label, plot_type, cmap): """Ingest CloudSat overpasses (HDF) and plot parameters of interest like snowfall rates as observed by the CPR. """ # Load and unpack HDF f = HDF(filepath) vs = f.vstart() Latitude = vs.attach('Latitude') Longitude = vs.attach('Longitude') Time = vs.attach('Profile_time') UTC = vs.attach('UTC_start') if plot_type == PlotType.SFCS: snowfall_rate_sfc = vs.attach('snowfall_rate_sfc') c = snowfall_rate_sfc[:] c = flatten(c) snowfall_rate_sfc.detach() if plot_type == PlotType.TEMP: EC_Height = vs.attach('EC_height') b = EC_Height[:] b = flatten(b) EC_Height.detach() a = Time[:] a = flatten(a) d = Longitude[:] d = flatten(d) utc_start = UTC[0][0] Latitude.detach() # "close" the vdata Longitude.detach() # "close" the vdata Time.detach() # "close" the vdata UTC.detach() # "close" the vdata vs.end() # terminate the vdata interface f.close() #---------- Read HDF Files ----------# cpr_2b_geoprof = SD(filepath, SDC.READ) offset = 28000 # Position along granule span = 1000 # Plot length if plot_type != PlotType.SFCS: if plot_type == PlotType.REFL or plot_type == PlotType.MASK or plot_type == PlotType.SNOW: cpr_2b_geoprof_height = cpr_2b_geoprof.select('Height') cpr_2b_geoprof_height_data = cpr_2b_geoprof_height.get() cpr_2b_geoprof_radar_reflectivity = cpr_2b_geoprof.select(data_field) cpr_2b_geoprof_radar_reflectivity_data = cpr_2b_geoprof_radar_reflectivity.get() if plot_type == PlotType.TEMP or plot_type == PlotType.SNOW: cpr_2b_geoprof_radar_reflectivity_data[cpr_2b_geoprof_radar_reflectivity_data < 0] = math.nan elif plot_type == PlotType.POWR: cpr_2b_geoprof_radar_reflectivity_data[cpr_2b_geoprof_radar_reflectivity_data < 0] = math.nan fillvalue = 15360 missing = -8888 img = np.zeros((span,125)) if plot_type == PlotType.REFL: img.fill(-30) factor = 1 if plot_type == PlotType.REFL: factor = 0.01 for i in np.arange(span): for j in np.arange(125): if plot_type == PlotType.TEMP: k = int( 125 * (b[j] + 5000) / 35000 ) else: k = int( 125 * (cpr_2b_geoprof_height_data[i+offset,j] + 5000) / 35000 ) if cpr_2b_geoprof_radar_reflectivity_data[i+offset,j] > -3000 and \ cpr_2b_geoprof_radar_reflectivity_data[i+offset,j] < 2100: img[i,k] = cpr_2b_geoprof_radar_reflectivity_data[i+offset,j] * factor # Begin plotting granule fig = plt.figure(figsize=(18, 6)) ax = plt.subplot(111) im = ax.imshow(img.T, interpolation='bilinear', cmap=cmap, origin='lower', extent=[0,200,-10,60]) plt.title(title) plt.ylabel('Height (km)') plt.xlabel('Time') plt.ylim(0,20) pylab.yticks([0,5,10,15,20],[0,5,10,15,20]) position_tick_labels = [str(round(a[offset]+utc_start, 3)), str(round(a[offset+200]+utc_start, 3)), str(round(a[offset+400]+utc_start, 3)), str(round(a[offset+600]+utc_start, 3)), str(round(a[offset+800]+utc_start, 3)), str(round(a[offset+1000]+utc_start, 3))] pylab.xticks([0,40,80,120,160, 200], position_tick_labels) divider = make_axes_locatable(ax) cax = divider.append_axes("right", size="3%", pad=0.10) if plot_type == PlotType.MASK: plt.colorbar(im, cax=cax, boundaries=[-10,0,10,20,30,40], ticks=[-10,0,10,20,30,40], label=label) elif plot_type == PlotType.TEMP: plt.colorbar(im, cax=cax, label=label, boundaries=[200, 220, 240, 260, 280], ticks=[200, 220, 240, 260, 280]) else: plt.colorbar(im, cax=cax, label=label) plt.savefig("cloudsat_radar_reflectivity.png") plt.show() else: fig = plt.figure(figsize=(15.5, 2.4)) c = c[offset:offset+span] index = np.arange(len(c)) plt.plot(index, c, color="#6699ff") plt.fill(index, c, color="#6699ff") plt.grid() plt.title("CloudSat Surface Snowfall") plt.xlabel("Time") plt.ylabel("Rate (mm / hr)") plt.ylim(0,3) plt.xlim(0,span) position_tick_labels = [str(round(a[offset]+utc_start, 3)), str(round(a[offset+200]+utc_start, 3)), str(round(a[offset+400]+utc_start, 3)), str(round(a[offset+600]+utc_start, 3)), str(round(a[offset+800]+utc_start, 3)), str(round(a[offset+1000]+utc_start, 3))] pylab.xticks([0,200,400,600,800,1000], position_tick_labels) plt.show()
def compress(*arg, **karg): # This function compress the file with a given compress parameter # compress(file,comression parameter) oldfile = arg[0] newfilename = arg[0] + "t" try: log = karg["log"] except KeyError: log = logMod.Log("", nolog=True) keepall = True # Should eventually check if files exists and can be read ***IMPROVE*** try: HDFF = SD(oldfile, SDC.READ) # This is the list of the FILES to merge except TypeError: HDFF = SD(oldfile.encode('ascii', 'ignore'), SDC.READ) # This is the list of the FILES to merge # This is the list of the ATTRIBUTE "StructMetadata.0" of the files attOfF = atribute(HDFF.attributes()["StructMetadata.0"], oldfile, dsi=[ 0, ]) ## Listing all the GRIDS that the new file will have gridlist = attOfF.listgridname()[ 1] # listgridname return a list of all the grids name ## Listing all the DATASETS that the new file will have # Should check if any grid ***IMPROVE*** dslist = attOfF.orderedDS() ## Validation of commoun information ############################################################################### # Some informations have to be the same for each file or else no mosaic can # # be made for exemple two files with not the same projection type can't be # # merged together. ***IMPROVE*** Maybe in the future we could transform file # # so that they have the same projection or some other thing. # ############################################################################### # List of parameter to check to insure that they are the same paramMustSim = ["Projection", "ProjParams", "SphereCode"] # Dictionary that will keep all the informations about every file paramMustSimDict = {} for grid in gridlist: # Verification of a grid paramdict = { } # Dictionary that keep the actual value that have to be the same bigG = attOfF.getgridbyname( grid) # Getting all the attributes in the grid of a file if bigG is not None: # If the grid exists in that file for p in paramMustSim: # Checking every parameters that must be the same paramdict[p] = bigG.variable[p] # Validation of same Dtype for each datafield go = bigG.GROUP["DataField"].OBJECT for r in go: paramdict[go[r].variable["DataFieldName"]] = go[r].variable[ "DataType"] # Keep all this info for later it's going to be useful paramMustSimDict[grid] = paramdict # LAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA ## Determination of new informations ############################################################################### # Some properties have to be calculated in order to merge. This section is # # doing just that # ############################################################################### gridResolX = {} # Getting the RESOLUTION in the X direction for each grid gridResolY = {} # Getting the RESOLUTION in the Y direction for each grid extremeup = {} # Getting the UPPER coordinates for each grid extremedown = {} # Getting the LOWEST coordinates for each grid extremeleft = {} # Getting the LEFTMOST coordinates for each grid extremeright = {} # Getting the RIGHTMOST coordinates for each grid gridDimX = {} # Getting the DIMENSIONS of X direction for each grid gridDimY = {} # Getting the DIMENSIONS of Y direction for each grid NoValueDS = {} # Getting the fill value of each dataset dtypeDS = {} # Getting the DTYPE for each dataset dstogrid = {} # Knowing wich is the GRID for each dataset filGridULC = {} # Getting the upper left corner of each file for each grid for grid in gridlist: # For each grid filGridULC[grid] = { } # Adding a dictionary for each grid that will contain information on every file ### Determination of resolution of each grid # ***IMPROVE*** Should check if bigd is none bigG = attOfF.getgridbyname( grid) # Getting all the attributes in the grid of a file # Get extreme grid point ulp = eval(bigG.variable["UpperLeftPointMtrs"]) lrp = eval(bigG.variable["LowerRightMtrs"]) # Get grid dimmension (PIXEL) dimx = int(bigG.variable["XDim"]) dimy = int(bigG.variable["YDim"]) # Calculate grid resolution gridResolX[grid] = (lrp[0] - ulp[0]) / dimx gridResolY[grid] = (ulp[1] - lrp[1]) / dimy ### Determination of new extreme coordinates for each grid # up extremeup[grid] = ulp[1] # down extremedown[grid] = lrp[1] # left extremeleft[grid] = ulp[0] # right extremeright[grid] = lrp[0] #print("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",int(LOC - UPC),int(RMC - LMC),"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n") ### Detetermination of dataset to grid name if bigG is not None: go = bigG.GROUP["DataField"].OBJECT for r in go: dstogrid[go[r].variable["DataFieldName"]] = grid ### Determination of new extreme coordinates for each grid ## Determination of ULC for each grid in each file filGridULC[grid][attOfF.name] = ulp ## determination of new dimension for each grid (pixel) gridDimY[grid] = int(dimy) gridDimX[grid] = int(dimx) for ds in dslist: # For each dataset # For each hdf file # Try opening dataset try: sds = HDFF.select(eval(ds)) # Get fill value NoValueDS[ds] = sds.getfillvalue() # Get dtype dtypeDS[ds] = sds.info()[3] except: log.log('e', Nom, "no dataset") ## Start creating new file ############################################################################### # This is the actual part were stuf appens # ############################################################################### # This part is the same for every file in any circumstances ########## absolute ######################## # Open new file try: hdf = HDF(newfilename, HC.WRITE | HC.CREATE | HC.TRUNC) sd = SD(newfilename, SDC.WRITE | SDC.CREATE) except TypeError: hdf = HDF(newfilename.encode('ascii', 'ignore'), HC.WRITE | HC.CREATE | HC.TRUNC) sd = SD(newfilename.encode('ascii', 'ignore'), SDC.WRITE | SDC.CREATE) v = hdf.vgstart() vg = {} vg1 = {} vg2 = {} ## rewrite the gridlist gridlist = [] for ds in dslist: if dstogrid[ds] not in gridlist: gridlist.append(dstogrid[ds]) for grid in gridlist: vg[grid] = v.attach(-1, write=1) vg[grid]._class = "GRID" vg[grid]._name = eval(grid) vg1[grid] = v.attach(-1, write=1) vg2[grid] = v.attach(-1, write=1) vg1[grid]._class = "GRID Vgroup" vg1[grid]._name = "Data Fields" vg2[grid]._class = "GRID Vgroup" vg2[grid]._name = "Grid Attributes" vg[grid].insert(vg1[grid]) vg[grid].insert(vg2[grid]) ########## absolute ######################## # Create dataset with the right size for ds in dslist: theGrid = dstogrid[ds] # Get grid name of data set # Make sure that the set is asked to be there sds = sd.create(eval(ds), dtypeDS[ds], (gridDimY[theGrid], gridDimX[theGrid])) # Set fill value fv = NoValueDS[ds] try: sds.setfillvalue(NoValueDS[ds]) except OverflowError: log.log('e', Nom, "dataset fillvaluet") sds.setfillvalue(0) # Set compression try: sds.setcompress(*arg[1]) # args depend on compression type except HDF4Error as msg: log.log('e', Nom, "Error compressing the dataset") sds_id.endaccess() sd_id.end() return ## write real data try: # read data from files osds = HDFF.select(eval(ds)) # And write it in the new sds sds[:, :] = osds[:, :] osds.endaccess() except: pass # Close sds vg1[dstogrid[ds]].add(HC.DFTAG_NDG, sds.ref()) sds.endaccess() for g in vg1: vg1[g].detach() vg2[g].detach() vg[g].detach() # Create attribute table for the file attstr = "GROUP=GridStructure\n" gridcount = 1 for gr in gridlist: # Start group grid attstr += "\tGROUP=GRID_%i\n" % gridcount # Add grid name attstr += "\t\tGridName=%s\n" % gr # Add dimention attstr += "\t\tXDim=%i\n" % gridDimX[gr] attstr += "\t\tYDim=%i\n" % gridDimY[gr] # Add UpperLeftPointMtrs attstr += "\t\tUpperLeftPointMtrs=(%f,%f)\n" % (extremeleft[gr], extremeup[gr]) # Add lrp attstr += "\t\tLowerRightMtrs=(%f,%f)\n" % (extremeright[gr], extremedown[gr]) # Add projection attstr += "\t\tProjection=%s\n" % paramMustSimDict[gr]["Projection"] # ProjParams attstr += "\t\tProjParams=%s\n" % paramMustSimDict[gr]["ProjParams"] # SphereCode attstr += "\t\tSphereCode=%s\n" % paramMustSimDict[gr]["SphereCode"] attstr += """\t\tGROUP=Dimension \t\tEND_GROUP=Dimension \t\tGROUP=DataField\n""" ## Add data sets # create list of ds for current grid lsdsgr = [] dsnum = 1 for ds in dslist: if dstogrid[ds] == gr: # Add object attstr += "\t\t\tOBJECT=DataField_%i\n" % dsnum # datafield name attstr += "\t\t\t\tDataFieldName=%s\n" % ds # datatype attstr += "\t\t\t\tDataType=%s\n" % paramMustSimDict[gr][ds] # dim attstr += '\t\t\t\tDimList=("YDim","XDim")\n' attstr += "\t\t\tEND_OBJECT=DataField_%i\n" % dsnum dsnum += 1 attstr += "\t\tEND_GROUP=DataField\n" attstr += """\t\tGROUP=MergedFields \t\tEND_GROUP=MergedFields\n""" attstr += "\tEND_GROUP=GRID_%i\n" % gridcount gridcount += 1 attstr += """END_GROUP=GridStructure GROUP=PointStructure END_GROUP=PointStructure END""" # adding attribute to new file att = sd.attr('StructMetadata.0') att.set(SDC.CHAR, attstr) sd.end() hdf.close() # delete old file os.remove(oldfile) # rename new file to old file name os.rename(newfilename, oldfile)