def read(self, filename, **kwargs): """Read the data""" from pyhdf.SD import SD import datetime # print "*** >>> Read the hdf-eos file!" root = SD(filename) # Get all the Attributes: # Common Attributes, Data Time, # Data Structure and Scene Coordinates for key in root.attributes().keys(): self._eoshdf_info[key] = root.attributes()[key] # Start Time - datetime object starttime = datetime.datetime.strptime(self._eoshdf_info['Start Time'][0:13], "%Y%j%H%M%S") msec = float(self._eoshdf_info['Start Time'][13:16]) / 1000. self.starttime = starttime + datetime.timedelta(seconds=msec) # End Time - datetime object endtime = datetime.datetime.strptime(self._eoshdf_info['End Time'][0:13], "%Y%j%H%M%S") msec = float(self._eoshdf_info['End Time'][13:16]) / 1000. self.endtime = endtime + datetime.timedelta(seconds=msec) # What is the leading 'H' doing here? sensor_name = self._eoshdf_info['Sensor Name'][1:-1].lower() try: self.satid = EOS_SATELLITE[sensor_name] except KeyError: LOG.error("Failed setting the satellite id - sat-name = ", sensor_name) self.orbit = self._eoshdf_info['Orbit Number'] self.shape = (self._eoshdf_info['Number of Scan Control Points'], self._eoshdf_info['Number of Pixel Control Points']) # try: if 1: value = root.select(self.name) attr = value.attributes() data = value.get() self.attr = attr band = data if self.name in FLAGS_QUALITY: self.data = band else: nodata = attr['bad_value_scaled'] self.data = (np.ma.masked_equal(band, nodata) * attr['slope'] + attr['intercept']) value.endaccess() # except: # pass root.end() self.filled = True
def read(self, filename, **kwargs): """Read the data""" from pyhdf.SD import SD import datetime #print "*** >>> Read the hdf-eos file!" root = SD(filename) # Get all the Attributes: # Common Attributes, Data Time, # Data Structure and Scene Coordinates for key in root.attributes().keys(): self._eoshdf_info[key] = root.attributes()[key] # Start Time - datetime object starttime = datetime.datetime.strptime(self._eoshdf_info['Start Time'][0:13], "%Y%j%H%M%S") msec = float(self._eoshdf_info['Start Time'][13:16])/1000. self.starttime = starttime + datetime.timedelta(seconds=msec) # End Time - datetime object endtime = datetime.datetime.strptime(self._eoshdf_info['End Time'][0:13], "%Y%j%H%M%S") msec = float(self._eoshdf_info['End Time'][13:16])/1000. self.endtime = endtime + datetime.timedelta(seconds=msec) # What is the leading 'H' doing here? sensor_name = self._eoshdf_info['Sensor Name'][1:-1].lower() try: self.satid = EOS_SATELLITE[sensor_name] except KeyError: LOG.error("Failed setting the satellite id - sat-name = ", sensor_name) self.orbit = self._eoshdf_info['Orbit Number'] self.shape = (self._eoshdf_info['Number of Scan Control Points'], self._eoshdf_info['Number of Pixel Control Points']) #try: if 1: value = root.select(self.name) attr = value.attributes() data = value.get() self.attr = attr band = data if self.name in FLAGS_QUALITY: self.data = band else: nodata = attr['bad_value_scaled'] self.data = (np.ma.masked_equal(band, nodata) * attr['slope'] + attr['intercept']) value.endaccess() #except: # pass root.end() self.filled= True
def shape(self): from pyhdf.SD import SD try: return self._shape except AttributeError: n = 0 for f in self.filenames: prod = SD(f) n += prod.attributes()['Number of Scans'] self._shape = (10 * n, prod.attributes()['Max Earth Frames']) return self._shape
def _l3read_hdf(self, fieldname='l3m_data'): self.vprint( "Reading hdf file") sd = SD(self.filename, SDC.READ) ds = sd.select(fieldname) field = ds[self.j1:self.j2, self.i1:self.i2].copy() intercept = ds.attributes()['Intercept'] slope = ds.attributes()['Slope'] try: nanval = ds.attributes()['Fill'] except: nanval = 65535 try: base = ds.attributes()['Base'] except KeyError: base = -999 if 'Scaled Data Maximum' in sd.attributes().keys(): self.maxval = sd.attributes()['Scaled Data Maximum'] self.minval = sd.attributes()['Scaled Data Minimum'] elif 'Suggested Image Scaling Maximum' in sd.attributes().keys(): self.maxval = sd.attributes()['Suggested Image Scaling Maximum'] self.minval = sd.attributes()['Suggested Image Scaling Minimum'] else: self.minval = self.vc[fld][2] self.maxval = self.vc[fld][3] start_iso = (pl.date2num(dtm( sd.attributes()['Period Start Year'],1,1)) + sd.attributes()['Period Start Day'] - 1) end_iso = (pl.date2num(dtm( sd.attributes()['Period End Year'],1,1)) + sd.attributes()['Period End Day'] - 1) self.jd = ((start_iso+end_iso)/2) self.date = pl.num2date(self.jd) return field,base,intercept,slope
def read_file_attr(in_file): try: hdf4 = SD(in_file, SDC.READ) return attrs2dict(hdf4.attributes()) except Exception as why: print 'ReadHDF4.read_file_attr: {}'.format(why) return
def prepare(self): """ Public: Handles reprojection of file, conversion of hdf into GeoTIFF """ data_file = SD(self.input_file, SDC.READ) extracted_data = list() for band in self.bands: band_data = data_file.select(band) extracted_data.append(band_data.get()) self.attributes = data_file.attributes() data_file.end() extracted_data = np.array(extracted_data) extracted_data[np.where( extracted_data <= self.low_thres)] = self.low_value extracted_data = np.log(extracted_data) extracted_data[np.where(extracted_data >= self.high_thres)] = HIGH_VAL indices = np.where((extracted_data > self.low_thres) & (extracted_data < self.high_thres)) extracted_data[indices] = (HIGH_VAL * (extracted_data[indices] - self.low_thres) / self.diff) extracted_data = extracted_data.astype(rasterio.uint8) file_name = self.input_file.split("/")[-1] self.prepare_thumbnail(extracted_data, file_name)
def ler_hdf_precipitacao(arquivo, pasta): diret = pasta + arquivo hdf = SD(diret, SDC.READ) #Conjuntos de dados disponíveis #print(hdf.datasets()) #Atributos #print(hdf.attributes()) #Obter resolução espacial e as coordenadas das "bordas" lista = hdf.attributes()['GridHeader'].replace(';', '').split('\n') atrs = {} for atr in [ 'LatitudeResolution=', 'LongitudeResolution=', 'NorthBoundingCoordinate=', 'SouthBoundingCoordinate=', 'EastBoundingCoordinate=', 'WestBoundingCoordinate=' ]: valor = [ float(lista[n][len(atr):]) for n, x in enumerate(lista) if atr in x ] atrs[atr] = valor[0] #Selecionar o conjunto de dados desejado ('precipitation'): dados de precipitação precip = hdf.select('precipitation') precip = precip[:] #Array com os dados de precipitação no formato nlon x nlat precip = np.transpose(precip) #Matriz transposta para termos nlat x nlon # Missing value = -9999.9; Set all the missing values less than 0 to NaNs np.putmask(precip, precip < 0, np.nan) return precip
def read_MODIS_file(fname): os.chdir(DataDir) hdf = SD(fname, SDC.READ) metadata = hdf.attributes() firepix = metadata['FirePix'] if firepix == 0: print('no fire pixel in %s' % (fname)) hdf.end() return [] print('%s fire pixel in %s' % (firepix, fname)) data = np.zeros((firepix, 5)) sds = hdf.select('FP_latitude') latdata = sds.get() sds = hdf.select('FP_longitude') londata = sds.get() sds = hdf.select('FP_power') FRPdata = sds.get() sds = hdf.select('FP_confidence') FRPcondata = sds.get() sds = hdf.select('FP_ViewZenAng') vzadata = sds.get() data[:, 0] = latdata data[:, 1] = londata data[:, 2] = FRPdata data[:, 3] = vzadata data[:, 4] = FRPcondata hdf.end() return data
def get_flip(filename): hdf = SD(filename, SDC.READ) global_attributes = hdf.attributes() if 'CoreMetadata.0' in global_attributes: # modis l1b coreMetaData = global_attributes['CoreMetadata.0'] odlParser = MiniODLParser() coreMetaDict = odlParser.parseFromString(coreMetaData, ['all']) platform = coreMetaDict['ASSOCIATEDPLATFORMSHORTNAME:VALUE'] dnflag = coreMetaDict['DAYNIGHTFLAG:VALUE'] print >> stderr, "platform ", platform print >> stderr, "dnflag ", dnflag if platform == 'Aqua' and dnflag == 'Day': print 'flipXY' elif platform == 'Terra' and dnflag == 'Night': print 'flipXY' else: startNode = global_attributes['Start Node'] print >> stderr, 'start node', startNode endNode = global_attributes['End Node'] print >> stderr, 'end node', endNode if startNode == 'Ascending' and endNode == 'Ascending': print 'flipXY'
def ancillary_interp_ozone(file, lon, lat, dataset='ozone', kind='linear'): from pyhdf.SD import SD, SDC f = SD(file, SDC.READ) datasets_dic = f.datasets() meta = f.attributes() sds_obj = f.select(dataset) data = sds_obj.get() f.end() f = None from numpy import linspace from scipy import interpolate ## make lons and lats for this file lons = linspace(meta["Westernmost Longitude"], meta["Easternmost Longitude"], num=meta['Number of Columns']) lats = linspace(meta["Northernmost Latitude"], meta["Southernmost Latitude"], num=meta['Number of Rows']) ## do interpolation in space if kind == 'nearest': xi, xret = min(enumerate(lons), key=lambda x: abs(x[1] - float(lon))) yi, yret = min(enumerate(lats), key=lambda x: abs(x[1] - float(lat))) uoz = data[yi, xi] / 1000. else: interp = interpolate.interp2d(lons, lats, data, kind=kind) uoz = (interp(lon, lat))[0] / 1000. anc_ozone = {'ozone': {'interp': uoz}} return (anc_ozone)
def get_sds(self, fieldnames=None): """Return specific or all SDS in the hdf file as dictionary. SDS arrays can be accessed using the 'data' key. Note that no scaling is applied to the data in get() method (use get_scaled() to achieve that). However, the scaling and missing data information can be accessed using the following keys: 'scale_factor' 'add_offset' '_FillValue' """ # Convert scalar fieldnames to sequence if isinstance(fieldnames, basestring): fieldnames = (fieldnames, ) # Open file to read SDs try: h4 = SD(self.filename, mode=SDC.READ) sclinfo = None if 'Slope_and_Offset_Usage' in h4.attributes(): sclinfo = 'Slope_and_Offset_Usage' # Get all available SDS from file if fieldnames in not given if fieldnames is None: fieldnames = [] for key in sorted(h4.datasets()): fieldnames.append(key) # Create and empty dataset dictionary with all available # fields fill in data from SDS sds = dict.fromkeys(fieldnames, {}) for key in sds: attrs = h4.select(key).attributes() if sclinfo: attrs[sclinfo] = h4.attributes()[sclinfo] sds[key] = attrs sds[key]['data'] = h4.select(key).get() # Close hdf interface h4.end() except HDF4Error as e: raise HDF4Error(e) # Return raw (possibly un-calibrated) SDS/attributes dictionary return sds
class HDFeos: def __init__(self, file_path): self.lats = None self.lons = None self.gring_lats = None self.gring_lons = None self.hdf = SD(file_path) self.file_path = file_path def read_laton(self): self.lons = self.hdf.select('Longitude').get().astype(numpy.double) self.lats = self.hdf.select('Latitude').get().astype(numpy.double) def get_metadata_group(self, group_name): metadata_group = {} keys = [s for s in self.hdf.attributes().keys() if group_name in s] for key in keys: string = self.hdf.attributes()[key] m = parse_hdfeos_metadata(string) metadata_group = {**metadata_group, **m} return metadata_group def create_sidecar(self, workers, cover_res, out_path): sids = staremaster.conversions.latlon2stare(self.lats, self.lons, resolution=-1, workers=workers, adapt_resolution=True) if not cover_res: cover_res = staremaster.conversions.min_level(sids) cover_sids = staremaster.conversions.gring2cover( self.gring_lats, self.gring_lons, cover_res) i = self.lats.shape[0] j = self.lats.shape[1] l = cover_sids.size sidecar = Sidecar(self.file_path, out_path) sidecar.write_dimensions(i, j, l, nom_res=self.nom_res) sidecar.write_lons(self.lons, nom_res=self.nom_res) sidecar.write_lats(self.lats, nom_res=self.nom_res) sidecar.write_sids(sids, nom_res=self.nom_res) sidecar.write_cover(cover_sids, nom_res=self.nom_res) return sidecar
def __init__(self, filename, filename_info, filetype_info): super(HDF4FileHandler, self).__init__(filename, filename_info, filetype_info) self.file_content = {} file_handle = SD(self.filename, SDC.READ) self._collect_attrs('', file_handle.attributes()) for k, v in file_handle.datasets().items(): self.collect_metadata(k, file_handle.select(k)) del file_handle
def get_colloc_crap(i, file_col): from pyhdf.SD import SD, SDC from libtools import dbg dbg(file_col) hdf = SD(file_col, SDC.READ) x = hdf.select('Master_Index_1')[i] a = hdf.select('Master_Index_2')[i] return x, a, hdf.attributes()['fname_AIRS'] #_WRS
def extract_attributes(self): # extract the attributes from the data file try: granule = SD(self.data_path, 1) except TypeError: granule = SD(self.data_path.encode("utf-8"), 1) self.attributes = granule.attributes() granule.end()
def get(self, fieldnames=[]): '''Returns specific or all SDS in the hdf file as dictionary. SDS arrays can be accessed using the 'data' key. Note that no scaling is applied to the data in get() method (use get_scaled() to achieve that). However, the scaling and missing data information can be accessed using the following keys: 'scale_factor' 'add_offset' '_FillValue' ''' if not isinstance(fieldnames, list): fieldnames = [fieldnames] try: # Open hdf4 interface in read mode h4 = SD(self.filename, mode=SDC.READ) sclinfo = None if 'Slope_and_Offset_Usage' in h4.attributes(): sclinfo = 'Slope_and_Offset_Usage' if len(fieldnames) == 0: # Get all available field names from hdf fieldnames = [] for key in sorted(h4.datasets()): fieldnames.append(key) # Create and empty dataset dictionary with all available # fields fill in data from SDS datasets = dict.fromkeys(fieldnames, {}) for key in datasets: attrs = h4.select(key).attributes() if sclinfo: attrs[sclinfo] = h4.attributes()[sclinfo] datasets[key] = attrs datasets[key]['data'] = h4.select(key).get() # Close hdf interface h4.end() except HDF4Error as e: print "HDF4Error", e sys.exit(1) # Return un-calibrated datasets/attributes dictionary return datasets
def open_file(filename): A = SD(filename) # retrieve data SDS d = A.datasets() sds_name = list(d.keys())[0] # name of sds. Dictionary method. sds = A.select(sds_name) pin = A.attributes() return sds, pin
def open_file(filename): A = SD(filename) # retrieve data SDS d = A.datasets() sds_name = d.keys()[0] # name of sds. Dictionary method. sds = A.select(sds_name) pin = A.attributes() return sds, pin
def get_core(filename): """ given the path to a Modis hdf4 file with a "CoreMetadata.0" attribute return that value as a string """ filename = str(filename) the_file = SD(filename, SDC.READ) metaDat = the_file.attributes()["CoreMetadata.0"] core_meta = str(metaDat).rstrip(" \t\r\n\0") the_file.end() return core_meta
def _populate_SD(self): """Populate SDs and their shape attributes""" try: h4 = SD(self.filename, mode=SDC.READ) self.sds = sorted(h4.datasets().keys()) self.attr.append(h4.attributes()) for k, v in sorted(h4.datasets().viewitems()): self.items.append((k, v[1])) h4.end() except HDF4Error as e: raise HDF4Error('{}: {}'.format(e, self.filename))
def readHDF4(inFile, bands=[1]): '''Reads HDF4 images as Raster instances using pyhdf''' from pyhdf.SD import SD, SDC g = SD(inFile, mode=SDC.READ) cellSize = g.attributes()['PixelSize'] m = g.attributes()['StructMetadata.0'] start = m.find('(', m.find('UpperLeft')) + 1 Xll = float(m[start:m.find(',', start)]) start = m.find(',', m.find('LowerRight')) + 1 Yll = float(m[start:m.find(')', start)]) rastList = [] for band in bands: b = g.select(band - 1) attribs = b.attributes() scale = 1.0 offset = 0.0 if attribs.has_key('scale_factor'): scale = attribs['scale_factor'] if attribs.has_key('add_offset'): offset = attribs['add_offset'] NoData = b.getfillvalue() arr = ma.masked_values(b.get(), NoData) arr = arr.astype('float32') arr = scale * (arr - offset) b.endaccess() rastList.append(raster.Raster(arr, cellSize, Xll, Yll, NoData)) g.end() return rastList
def load_thin_modis(satscene, options): """Read modis data from file and load it into *satscene*. """ filename = satscene.time_slot.strftime("thin_MYD021KM.A%Y%j.%H%M.005.NRT.hdf") filename = os.path.join(options["dir"], filename) data = SD(filename) datasets = ['EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB', 'EV_1KM_Emissive'] for dataset in datasets: subdata = data.select(dataset) band_names = subdata.attributes()["band_names"].split(",") if len(satscene.channels_to_load & set(band_names)) > 0: uncertainty = data.select(dataset+"_Uncert_Indexes") if dataset == 'EV_1KM_Emissive': array = calibrate_tb(subdata, uncertainty) else: array = calibrate_refl(subdata, uncertainty) for (i, band) in enumerate(band_names): if band in satscene.channels_to_load: satscene[band] = array[i] mda = data.attributes()["CoreMetadata.0"] orbit_idx = mda.index("ORBITNUMBER") satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116] lat, lon = get_lat_lon(satscene, None) from pyresample import geometry satscene.area = geometry.SwathDefinition(lons=lon, lats=lat) # trimming out dead sensor lines if satscene.satname == "aqua": for band in ["6", "27"]: if not satscene[band].is_loaded() or satscene[band].data.mask.all(): continue width = satscene[band].data.shape[1] height = satscene[band].data.shape[0] indices = satscene[band].data.mask.sum(1) < width if indices.sum() == height: continue satscene[band] = satscene[band].data[indices, :] satscene[band].area = geometry.SwathDefinition( lons=satscene.area.lons[indices,:], lats=satscene.area.lats[indices,:]) satscene[band].area.area_id = ("swath_" + satscene.fullname + "_" + str(satscene.time_slot) + "_" + str(satscene[band].shape) + "_" + str(band))
def getMetadata(filepath): metadata = {} hdf_file = SD(filepath, SDC.READ) dataset = gdal.Open(filepath, gdal.GA_ReadOnly) # Construct the grid. The needed information is in a global attribute # called 'StructMetadata.0'. Use regular expressions to tease out the # extents of the grid. fattrs = hdf_file.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] ul_regex = re.compile( r'''UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = ul_regex.search(gridmeta) x0 = np.float(match.group('upper_left_x')) y0 = np.float(match.group('upper_left_y')) lr_regex = re.compile( r'''LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = lr_regex.search(gridmeta) x1 = np.float(match.group('lower_right_x')) y1 = np.float(match.group('lower_right_y')) metadata['xmax'] = x0 metadata['xmin'] = x1 metadata['ymax'] = y1 metadata['ymin'] = y0 # determine the projection GCTP code from the grid metadata proj_regex = re.compile(r'''Projection=(?P<projection>\w+)''', re.VERBOSE) match = proj_regex.search(gridmeta) proj = match.group('projection') # support MODIS sinusoidal projection for now, add others later if proj == 'GCTP_SNSOID': sinu = pyproj.Proj("+proj=sinu +R=6371007.181 +nadgrids=@null +wktext") wgs84 = pyproj.Proj("+init=EPSG:4326") metadata['lonmin'], metadata['latmin'] = pyproj.transform( sinu, wgs84, x0, y0) metadata['lonmax'], metadata['latmax'] = pyproj.transform( sinu, wgs84, x1, y1) return metadata
def check_if_day(hdf_file): hdf_filename = sys.argv[1] hdf_file = SD(hdf_filename, SDC.READ) core_metadata0_string = hdf_file.attributes()['CoreMetadata.0'] daynight_start_index = core_metadata0_string.find('DAYNIGHT') #print('test: |' + core_metadata0_string[daynight_start_index:daynight_start_index+120] + '|') daynight_info_block = core_metadata0_string[daynight_start_index:daynight_start_index+120] is_day = (daynight_info_block.find('\"Day\"') != -1) #print(is_day) return is_day
def gethv(Datadir, x0, y0, **kwargs): files = glob.glob(Datadir + '2008.06.18/*.hdf') strhv = [] for Aerfile in files: ds = SD(Aerfile, SDC.READ) lookup1 = 'UpperLeftPointMtrs' lookup2 = 'LowerRightMtrs' StrMeta = ds.attributes()['StructMetadata.0'] for line in StrMeta.splitlines(): if lookup1 in line: xy = (line.split('('))[1].split(',') xul = np.float(xy[0]) yul = np.float((xy[1].split(')'))[0]) if lookup2 in line: xy = (line.split('('))[1].split(',') xlr = np.float(xy[0]) ylr = np.float((xy[1].split(')'))[0]) break Aobj = ds.select('Optical_Depth_047') AOD = Aobj.get() [nx, ny] = AOD[0, :, :].shape XDim = xul + (0.5 + np.arange(nx)) * (xlr - xul) / nx YDim = yul + (0.5 + np.arange(ny)) * (ylr - yul) / ny # YDim=np.flip(YDim) xv, yv = np.meshgrid(XDim, YDim) # In basemap, the sinusoidal projection is global, so we won't use it. # Instead we'll convert the grid back to lat/lons. sinu = pyproj.Proj("+proj=sinu +R=6371007.181 +nadgrids=@null +wktext") wgs84 = pyproj.Proj("+proj=latlong +R=6371007.181") lon, lat = pyproj.transform(sinu, wgs84, xv, yv) ds.end() if 'mindist' in kwargs: mindist = kwargs['mindist'] distance = (lat - y0)**2 + (lon - x0)**2 if np.min(distance) < mindist: strhv = np.append(strhv, ((Aerfile.split('/'))[-1].split('.'))[-4]) if 'dx' in kwargs: dx = kwargs['dx'] dy = kwargs['dy'] if (np.min(np.absolute(lat - y0)) < dy) & (np.min( np.absolute(lon - x0)) < dx): strhv = np.append(strhv, ((Aerfile.split('/'))[-1].split('.'))[-4]) return strhv
def read_modis_sd(filename): product = SD(filename) Nfire = product.attributes()['FirePix'] if Nfire == 0: return np.zeros((1, 8)) datasets = [ 'FP_power', 'FP_latitude', 'FP_longitude', 'FP_ViewZenAng', 'FP_MeanT21', 'FP_MAD_T21', 'FP_sample', 'FP_T21' ] datasetList = [] for sds in datasets: selection = product.select(sds).get() datasetList.append(selection) return (np.array(datasetList).transpose())
def HDF_proj_WKT(hdf_filepath): # returns the projection of the HDF file in Well Known Text (WKT) format # first determine the HDF type hdf_type = HDF_type(hdf_filepath) if hdf_type == 'hdf4': # for HDF4 assume corner coordinates are stored in the StructMetadata.0 section hdf_file = SD(hdf_filepath, SDC.READ) try: # access grid metadata section of StructMetadata.0 fattr = hdf_file.attributes(full=1) structmeta = fattr['StructMetadata.0'] gridmeta = structmeta[0] # determine the projection GCTP code from the grid metadata proj_regex = re.compile(r'''Projection=(?P<projection>\w+)''', re.VERBOSE) match = proj_regex.search(gridmeta) proj = match.group('projection') # support MODIS sinusoidal projection for now, add others later if proj == 'GCTP_SNSOID': sinu_proj4 = "+proj=sinu +R=6371007.181 +nadgrids=@null +wktext" srs = osr.SpatialReference() srs.ImportFromProj4(sinu_proj4) return srs.ExportToWkt() except: #prjfile = open('/home/rkalyana/GeoEDF/GeoEDF/connector/filter/modis/6933.prj', 'r') #prj_txt = prjfile.read() #srs = osr.SpatialReference() #srs.ImportFromESRI([prj_txt]) #prjfile.close() #return srs.ExportToWkt() raise GeoEDFError( 'Error determining the projection or unsupported projection') else: # HDF5 file; only SMAP files in EASE Grid 2.0 are supported at the moment hdf_file = h5py.File(hdf_filepath, mode='r') # check to see if this is a EASE Grid 2.0 file if 'EASE2_global_projection' in hdf_file.keys(): ease_proj4 = "+proj=cea +lat_0=0 +lon_0=0 +lat_ts=30 +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=m" srs = osr.SpatialReference() srs.ImportFromProj4(ease_proj4) return srs.ExportToWkt() else: raise GeoEDFError( 'Error determining the projection or unsupported projection')
def read_hdf4_info(input_file_path) : """ get information about variable names and attributes (both global and variable specific) from the given file. The file is assumed to exist and be a valid hdf4 file returns something in the form: { GLOBAL_ATTRS_KEY : a dictionary of attribute values keyed by the attribute names VAR_LIST_KEY : [list of variable names] VAR_INFO_KEY : { <var_name> : { SHAPE_KEY: (shape of variable data) VAR_ATTRS_KEY: a dictionary of attribute values keyed by the attribute names } } } TODO, depending on what changes need to be made for CF compliance this data structure may need to change a lot """ file_info = { } # open the file file_object = SD(input_file_path, SDC.READ) # get information on the global attributes in the file global_attrs = file_object.attributes() file_info[GLOBAL_ATTRS_KEY] = global_attrs # get information on the variables in the file variable_list = file_object.datasets().keys() file_info[VAR_LIST_KEY] = variable_list # for each variable in a file, get more specific information about it file_info[VAR_INFO_KEY] = { } sets_temp = file_object.datasets() # this should return a dictionary with entries for each variable in the form # <variable name>: ((dimension names), (data shape), type, index num) for var_name in variable_list : var_object = file_object.select(var_name) var_attrs = var_object.attributes() file_info[VAR_INFO_KEY][var_name] = { SHAPE_KEY: sets_temp[var_name][1], VAR_ATTRS_KEY: var_attrs, } return file_info, file_object
def set_file_attr(self): """ get hdf5 file attrs self.file_attr :return: """ if self.resolution == 13500: satellite_type = ['AQUA', 'TERRA'] if self.satellite in satellite_type: h4r = SD(self.in_file, SDC.READ) self.file_attr = attrs2dict(h4r.attributes()) else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) else: raise ValueError("Cant handle this resolution: ".format( self.resolution))
def read_amsr_hdf4(filename): from pyhdf.SD import SD, SDC from pyhdf.HDF import HDF, HC import pyhdf.VS retv = AmsrObject() h4file = SD(filename, SDC.READ) datasets = h4file.datasets() attributes = h4file.attributes() #for idx,attr in enumerate(attributes.keys()): # print idx, attr for sds in ["Longitude", "Latitude", "High_res_cloud"]: data = h4file.select(sds).get() if sds in ["Longitude", "Latitude"]: retv.all_arrays[sds.lower()] = data.ravel() elif sds in ["High_res_cloud"]: lwp_gain = h4file.select(sds).attributes()['Scale'] retv.all_arrays["lwp_mm"] = data.ravel() * lwp_gain #print h4file.select(sds).info() h4file = HDF(filename, SDC.READ) vs = h4file.vstart() data_info_list = vs.vdatainfo() #print "1D data compound/Vdata" for item in data_info_list: #1D data compound/Vdata name = item[0] #print name if name in ["Time"]: data_handle = vs.attach(name) data = np.array(data_handle[:]) retv.all_arrays["sec1993"] = data data_handle.detach() else: pass #print name #data = np.array(data_handle[:]) #attrinfo_dic = data_handle.attrinfo() #factor = data_handle.findattr('factor') #offset = data_handle.findattr('offset') #print data_handle.factor #data_handle.detach() #print data_handle.attrinfo() h4file.close() #for key in retv.all_arrays.keys(): # print key, retv.all_arrays[key] return retv
def get_corners(hdf: SD) -> Tuple[Tuple[float, float], Tuple[float, float]]: metadata_str = hdf.attributes()['StructMetadata.0'].replace('\x00', '') wtf = metadata_str.replace('\t', '').split('\n') points = [ x for x in wtf if 'UpperLeftPointMtrs' in x or 'LowerRightMtrs' in x ] assert len(points) == 4 a, b = points[0].split('=') assert a == 'UpperLeftPointMtrs' UpperLeftPointMtrs = eval(b) a, b = points[1].split('=') assert a == 'LowerRightMtrs' LowerRightMtrs = eval(b) return UpperLeftPointMtrs, LowerRightMtrs
class MODIS02File(object): """ """ def __init__(self, modis02fname): """ """ self._sd = SD(modis02fname, SDC.READ) def close(self): """ """ self._sd.end() def read_radiance(self, band): """ """ sds = self._sd.select('EV_1KM_Emissive') att = sds.attributes() bands = np.array(att['band_names'].split(','), dtype='int32') iband = int(np.where(bands == band)[0][0]) scale = att['radiance_scales'][iband] offset = att['radiance_offsets'][iband] fillvalue = att['_FillValue'] rad = (ma.masked_equal(sds[iband, :, :], fillvalue).astype('float32') \ - offset) * scale return rad def read_attributes(self): attrs = {} sdattrs = self._sd.attributes() coremd = sdattrs['CoreMetadata.0'].split('\n') for iline, line in enumerate(coremd): if 'OBJECT = RANGEENDINGDATE' in line: enddate = coremd[iline + 2].split('"')[1] elif 'OBJECT = RANGEENDINGTIME' in line: endtime = coremd[iline + 2].split('"')[1] elif 'OBJECT = RANGEBEGINNINGDATE' in line: begindate = coremd[iline + 2].split('"')[1] elif 'OBJECT = RANGEBEGINNINGTIME' in line: begintime = coremd[iline + 2].split('"')[1] elif 'OBJECT = ASSOCIATEDSENSORSHORTNAME' in line: attrs['sensor'] = coremd[iline + 3].split('"')[1] elif 'OBJECT = ASSOCIATEDPLATFORMSHORTNAME' in line: attrs['platform'] = coremd[iline + 3].split('"')[1] attrs['start_time'] = datetime.strptime(begindate + 'T' + begintime, '%Y-%m-%dT%H:%M:%S.%f') attrs['stop_time'] = datetime.strptime(enddate + 'T' + endtime, '%Y-%m-%dT%H:%M:%S.%f') return attrs
def set_file_attr(self): """ 根据 self.file_level_1 获取 L1 文件的属性 set self.level_1_attr 储存格式是字典 :return: """ if self.resolution == 1000: satellite_type1 = ['FY1C', 'FY1D'] if self.satellite in satellite_type1: hdf4 = SD(self.in_file, SDC.READ) self.file_attr = attrs2dict(hdf4.attributes()) else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) else: raise ValueError("Cant handle this resolution: ".format( self.resolution))
def is_daily_product(hdf_filename): hdf_file = SD(hdf_filename, SDC.READ) # get CoreMetadata.0 attribute (which is a very long string) and just extract and analyze the substring # with the DAYNIGHT info: core_metadata0_string = hdf_file.attributes()['CoreMetadata.0'] daynight_start_index = core_metadata0_string.find('DAYNIGHT') if daynight_start_index != -1: #print('test: |' + core_metadata0_string[daynight_start_index:daynight_start_index+120] + '|') daynight_info_block = core_metadata0_string[daynight_start_index:daynight_start_index+120] is_daily = (daynight_info_block.find('\"Day\"') != -1) #print(is_daily) return is_daily else: return false
def extractEVDCData(fileAbsPath): inFile = SD(fileAbsPath, SDC.READ) outFileName = os.path.split(fileAbsPath)[1][0:-3] + 'xml' outFile = open(outFileName, 'w') siteName = inFile.attributes(True)['DATA_LOCATION'][0] siteLatitude = inFile.select('LATITUDE.INSTRUMENT')[0] siteLongitude = inFile.select('LONGITUDE.INSTRUMENT')[0] siteElevation = inFile.select('ALTITUDE.INSTRUMENT')[0] outFile.write('<EVDC>\n') outFile.write(' <siteName>' + siteName + '</siteName>\n') outFile.write(' <siteLatitude>' + str(siteLatitude) + '</siteLatitude>\n') outFile.write(' <siteLongitude>' + str(siteLongitude) + '</siteLongitude>\n') outFile.write(' <siteElevation>' + str(siteElevation) + '</siteElevation>\n') timestamps = inFile.select('DATETIME') timestampCount = timestamps.info()[2] so2absorption = inFile.select('SO2.COLUMN_ABSORPTION.SOLAR') field = so2absorption.attributes(True)['VAR_NAME'][0] for i in range(0, timestampCount): referenceDate = datetime.date(2000,1,1) days = int(timestamps[i]) hours = int((timestamps[i]-days) * 24) minutes = int(((timestamps[i]-days)*24-hours) * 60) seconds = int(((((timestamps[i]-days)*24-hours)*60)-minutes) * 60) date = referenceDate + datetime.timedelta(days) measurementTime = date.strftime('%Y-%m-%dT') + str(hours).zfill(2) + ':' + str(minutes).zfill(2) + ':' + str(seconds).zfill(2) + 'Z' outFile.write(' <data>\n') outFile.write(' <timeStart>' + measurementTime + '</timeStart>\n') outFile.write(' <timeEnd>' + measurementTime + '</timeEnd>\n') outFile.write(' <field>'+ field + '</field>\n') outFile.write(' <value>' + str(so2absorption[i]) + '</value>\n') outFile.write(' </data>\n') outFile.write('</EVDC>') outFile.close() return [outFileName]
def read_calipso_hdf4(filename, retv): from pyhdf.SD import SD, SDC from pyhdf.HDF import HDF, HC import pyhdf.VS def convert_data(data): if len(data.shape) == 2: if data.shape[1] == 1: return data[:, 0] elif data.shape[0] == 1: return data[0, :] return data if filename is not None: h4file = SD(filename, SDC.READ) datasets = h4file.datasets() attributes = h4file.attributes() singleshotdata = {} for idx, dataset in enumerate(datasets.keys()): #non-goups if dataset in scip_these_larger_variables_until_needed.keys(): continue elif dataset[0:8] == 'Surface_': continue if dataset in ["ssNumber_Layers_Found", "ssLayer_Base_Altitude", "ssLayer_Top_Altitude"]: singleshotdata[dataset] = h4file.select(dataset).get() if dataset[0:2] == "ss": #already saved temporarly what we need continue name = dataset.lower() #print idx, dataset if dataset in atrain_match_names.keys(): name = atrain_match_names[dataset] data = np.array(h4file.select(dataset).get()) setattr(retv, name, data) if "ssNumber_Layers_Found" in singleshotdata.keys(): # Extract number of cloudy single shots (max 15) # plus average cloud base and top # in 5 km FOV logger.info("Reading single shot information") retv = rearrange_calipso_the_single_shot_info( retv, singleshotdata) return retv
def _read_hdf(self, fieldname='l3m_data'): sd = SD(self.filename, SDC.READ) ds = sd.select(fieldname) attrdict = ds.attributes() for d in attrdict.keys(): attrdict[d.lower()] = attrdict.pop(d) field = ds[self.j1:self.j2, self.i1:self.i2].copy() intercept = ds.attributes()['intercept'] slope = ds.attributes()['slope'] try: nanval = ds.attributes()['fill'] except: nanval = ds.attributes()['bad_value_scaled'] try: base = ds.attributes()['base'] except KeyError: base = -999 dstr = sd.attributes()['Start Time'] self.jd = pl.datestr2num(dstr[:4] + "-1-1") + float(dstr[4:7]) - 1 return field,base,intercept,slope,nanval
def write_interpolated(filename, f0, f1, fact, datasetNames): ''' interpolate two hdf files f0 and f1 using factor fact, and write the result to filename ''' hdf = SD(filename, SDC.WRITE|SDC.CREATE) for datasetName in datasetNames: try: info = SD(f0).select(datasetName).info() except: print >> stderr, 'Error loading %s in %s' % (datasetName, f0) raise typ = info[3] shp = info[2] sds_in1 = SD(f0).select(datasetName) met0 = sds_in1.get() met1 = SD(f1).select(datasetName).get() interp = (1-fact)*met0 + fact*met1 interp = interp.astype({ SDC.INT16: 'int16', SDC.FLOAT32: 'float32', SDC.FLOAT64: 'float64', }[typ]) # write sds = hdf.create(datasetName, typ, shp) sds[:] = interp[:] # copy attributes attr = sds_in1.attributes() if len(attr) > 0: for name in attr.keys(): setattr(sds, name, attr[name]) sds.endaccess() hdf.end()
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'Day_CMG_Snow_Cover' if USE_GDAL: import gdal GRID_NAME = 'MOD_CMG_Snow_5km' gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME, GRID_NAME, DATAFIELD_NAME) gdset = gdal.Open(gname) data = gdset.ReadAsArray() # Read projection parameters. x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform() nx, ny = (gdset.RasterXSize, gdset.RasterYSize) del gdset else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:,:].astype(np.float64) # Read global attribute. fattrs = hdf.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] # Read projection parameters. # The needed information is in a global attribute called 'StructMetadata.0'. # Use regular expressions to tease out the extents of the grid. ul_regex = re.compile(r'''UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = ul_regex.search(gridmeta) x0 = np.float(match.group('upper_left_x')) / 1e6 y0 = np.float(match.group('upper_left_y')) / 1e6 lr_regex = re.compile(r'''LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = lr_regex.search(gridmeta) x1 = np.float(match.group('lower_right_x')) / 1e6 y1 = np.float(match.group('lower_right_y')) / 1e6 ny, nx = data.shape xinc = (x1 - x0) / nx yinc = (y1 - y0) / ny # Construct the grid. It's already in lat/lon. x = np.linspace(x0, x0 + xinc*nx, nx) y = np.linspace(y0, y0 + yinc*ny, ny) lon, lat = np.meshgrid(x, y) m = Basemap(projection='cyl', resolution='l', llcrnrlat=-90, urcrnrlat = 90, llcrnrlon=-180, urcrnrlon = 180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90., 120., 30.), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180, 180., 45.), labels=[0, 0, 0, 1]) # Bin the data as follows: # 0% snow # 1-99% snow # 100% snow # lake ice (107) # night (111) # cloud-obscured water (250) # water mask (254) # fill (255) lst = ['#00ff00', '#888888', '#ffffff', '#ffafff', '#000000', '#63c6ff', '#0000cc', '#8928dd'] cmap = mpl.colors.ListedColormap(lst) bounds = [0, 1, 100, 107, 111, 250, 254, 255, 256] norm = mpl.colors.BoundaryNorm(bounds, cmap.N) # Render the image in the projected coordinate system. m.pcolormesh(lon[::2,::2], lat[::2,::2], data[::2,::2], latlon=True, cmap=cmap, norm=norm) long_name = 'Day CMG Snow Cover' basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, long_name)) fig = plt.gcf() color_bar = plt.colorbar(orientation='horizontal') color_bar.set_ticks([0.5, 50, 103, 109, 180, 252, 254.5, 255.5]) color_bar.set_ticklabels(['0%\nsnow', '1-99%\nsnow', '100%\nsnow', 'lake\nice', 'night', 'cloud\n-obscured\nwater', 'water\nmask', 'fill']) color_bar.draw_all() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
import matplotlib as mpl import matplotlib.pyplot as plt from mpl_toolkits.basemap import Basemap import numpy as np from pyhdf.SD import SD, SDC # Open HDF4 file. FILE_NAME = 'S1999001.L3m_DAY_CHL_chlor_a_9km.hdf' hdf = SD(FILE_NAME, SDC.READ) # The lat and lon should be calculated using lat and lon of southwest point. # Then, we need number of lines and columns to calculate the lat and lon # step. Assume even space between lat and lon points to get all lat and lon # data. # Extract southwest point from global attribute. gattrs = hdf.attributes(full=1) swlata = gattrs['SW Point Latitude'] swlat = swlata[0] swlona = gattrs['SW Point Longitude'] swlon = swlona[0] linesa = gattrs['Number of Lines'] nlat = linesa[0] colsa = gattrs['Number of Columns'] nlon = colsa[0] latstepa = gattrs['Latitude Step'] latstep = latstepa[0] lonstepa = gattrs['Longitude Step'] lonstep = lonstepa[0] nmlat = swlat + (nlat)*latstep;
def run(FILE_NAME): DATAFIELD_NAME = 'Local albedo average - 1 deg' from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data3D = hdf.select(DATAFIELD_NAME) data = data3D[:,:,3].astype(np.double) # Read file attribute. fattrs = hdf.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] # Read dataset attribute. attrs = data3D.attributes(full=1) fva=attrs["_FillValue"] _FillValue = fva[0] # Apply the fill value. data[data == _FillValue] = np.nan datam = np.ma.masked_array(data, mask=np.isnan(data)) # Construct the grid. The needed information is in a global attribute # called 'StructMetadata.0'. Use regular expressions to tease out the # extents of the grid. ul_regex = re.compile(r'''UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = ul_regex.search(gridmeta) x0 = np.float(match.group('upper_left_x')) / 1000000.0 y0 = np.float(match.group('upper_left_y')) / 1000000.0 lr_regex = re.compile(r'''LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = lr_regex.search(gridmeta) x1 = np.float(match.group('lower_right_x')) / 1000000.0 y1 = np.float(match.group('lower_right_y')) / 1000000.0 data = data.T nx, ny = data.shape x = np.linspace(x0, x1, nx) y = np.linspace(y0, y1, ny) lon, lat = np.meshgrid(x, y) m = Basemap(projection='cyl', resolution='l', llcrnrlat=-90, urcrnrlat = 90, llcrnrlon=-180, urcrnrlon = 180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90., 120., 30.), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180., 181., 45.), labels=[0, 0, 0, 1]) m.pcolormesh(lon, lat, datam, latlon=True) cb = m.colorbar() # Albedo doesn't have a unit according to the specification [1]. cb.set_label('No Units') basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1} at Band=3'.format(basename, DATAFIELD_NAME)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
class HDFEOSFileReader(BaseFileHandler): def __init__(self, filename, filename_info, filetype_info): self.filename = filename try: self.sd = SD(str(self.filename)) except HDF4Error as err: raise ValueError("Could not load data from " + str(self.filename) + ": " + str(err)) self.sd = SD(self.filename) self.mda = self.read_mda(self.sd.attributes()['CoreMetadata.0']) self.mda.update(self.read_mda( self.sd.attributes()['StructMetadata.0'])) self.mda.update(self.read_mda( self.sd.attributes()['ArchiveMetadata.0'])) @property def start_time(self): date = (self.mda['INVENTORYMETADATA']['RANGEDATETIME']['RANGEBEGINNINGDATE']['VALUE'] + ' ' + self.mda['INVENTORYMETADATA']['RANGEDATETIME']['RANGEBEGINNINGTIME']['VALUE']) return datetime.strptime(date, '%Y-%m-%d %H:%M:%S.%f') @property def end_time(self): date = (self.mda['INVENTORYMETADATA']['RANGEDATETIME']['RANGEENDINGDATE']['VALUE'] + ' ' + self.mda['INVENTORYMETADATA']['RANGEDATETIME']['RANGEENDINGTIME']['VALUE']) return datetime.strptime(date, '%Y-%m-%d %H:%M:%S.%f') def read_mda(self, attribute): lines = attribute.split('\n') mda = {} current_dict = mda path = [] for line in lines: if not line: continue if line == 'END': break key, val = line.split('=') key = key.strip() val = val.strip() try: val = eval(val) except NameError: pass if key in ['GROUP', 'OBJECT']: new_dict = {} path.append(val) current_dict[val] = new_dict current_dict = new_dict elif key in ['END_GROUP', 'END_OBJECT']: if val != path[-1]: raise SyntaxError path = path[:-1] current_dict = mda for item in path: current_dict = current_dict[item] elif key in ['CLASS', 'NUM_VAL']: pass else: current_dict[key] = val return mda
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'High_res_cloud' if USE_GDAL: import gdal GRID_NAME = 'GlobalGrid' gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME, GRID_NAME, DATAFIELD_NAME) gdset = gdal.Open(gname) data = gdset.ReadAsArray().astype(np.float64) # Read projection parameters. meta = gdset.GetMetadata() x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform() nx, ny = (gdset.RasterXSize, gdset.RasterYSize) # Read attributes. units = meta['Unit'] scale_factor = float(meta['Scale']) # Construct the grid. The projection is GEO, so this immediately # gives us latitude and longitude. x = np.linspace(x0, x0 + xinc*nx, nx) y = np.linspace(y0, y0 + yinc*ny, ny) del gdset else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:,:].astype(np.float64) # Read global attribute. fattrs = hdf.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] # Construct the grid. The needed information is in a global attribute # called 'StructMetadata.0'. Use regular expressions to tease out the # extents of the grid. ul_regex = re.compile(r'''UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = ul_regex.search(gridmeta) x0 = np.float(match.group('upper_left_x')) / 1e6 y0 = np.float(match.group('upper_left_y')) / 1e6 lr_regex = re.compile(r'''LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = lr_regex.search(gridmeta) x1 = np.float(match.group('lower_right_x')) / 1e6 y1 = np.float(match.group('lower_right_y')) / 1e6 ny, nx = data.shape x = np.linspace(x0, x1, nx) y = np.linspace(y0, y1, ny) # Retrieve attributes. attrs = data2D.attributes(full=1) sfa=attrs["Scale"] scale_factor = sfa[0] ua=attrs["Unit"] units = ua[0] # Apply the attributes information. data[data == -9999] = np.nan data = data * scale_factor data = np.ma.masked_array(data, np.isnan(data)) long_name = DATAFIELD_NAME longitude, latitude = np.meshgrid(x, y) m = Basemap(projection='cyl', resolution='l', lon_0=0, llcrnrlat=-90, urcrnrlat = 90, llcrnrlon=-180, urcrnrlon = 180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90, 91, 45), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180, 181, 45), labels=[0, 0, 0, 1]) m.pcolormesh(longitude, latitude, data, latlon=True) cb = m.colorbar() cb.set_label(units) basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, long_name)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def modis_oc(file_name, limit=None, flagLevel=None, param='sst'): # ;Example of setting up flag level for non SST images # ;--> cf http://oceancolor.gsfc.nasa.gov/DOCS/Ocean_Level-2_Data_Products.pdf # flagtable=REPLICATE(0B,32) ;32 bits flag # ;flaglevel[[0,1,2,3,4,5,8,9,10,12,14,15,16,17,18,19,21,22,23,25,26,28,29]]=1B ;RQ: remove 1 for table correspondance in documentation (table 3) # # IF (~exist(sfglevel)) THEN sfglevel=1 # # case (flaglevel) of # 1: flagTable[[ 1, 3, 5, 15,16, 19, 25,26]]=1B # 2: flagTable[[0,1, 3,4,5,8,9,10, 14,15,16, 19,21,22,23,25,26]]=1B # 3: flagTable[[0,1, 3,4,5,8,9,10, 14,15,16, 19,21,22,23,25,26]]=1B # else: flagTable[[0,1, 3,4,5,8,9,10, 14,15,16, 19,21,22,23,25,26]]=1B # endcase # #; flaglevel[[ 1, 3, 5, 12, 15,16, 19, 25,26]]=1B ;low value flag #; ;flaglevel[[0,1, 3,4,5,8,9,10,12,14,15,16, 19,21,22,23,25,26]]=1B ;operational flag (strong removal) #Read MODIS HDF4 data f = SD(file_name, SDC.READ) fattr=f.attributes() #Load coordinates lon=f.select('longitude') lat=f.select('latitude') #Shrink image ############# #scene dimensions info_sst=f.datasets()[param] dnames=info_sst[0] d=info_sst[1] lonvec=lon.get().reshape(d[0]*d[1]) latvec=lat.get().reshape(d[0]*d[1]) #Get points within domain if limit is not None : indvec,flagvec=in_limits(lonvec, latvec, limit) flagmat=flagvec.reshape(d[0],d[1]) rowsum=np.sum(flagmat, 0) colsum=np.sum(flagmat, 1) yflag=rowsum >= 1 xflag=colsum >= 1 xid=np.arange(d[0]) xid=xid.compress(xflag) xcnt=int(xid.size) xst=int(xid.min()) yid=np.arange(d[1]) yid=yid.compress(yflag) ycnt=int(yid.size) yst=int(yid.min()) #Shrink lon & lat lon_var=lon.get(start=[xst,yst], count=[xcnt,ycnt]) lat_var=lat.get(start=[xst,yst], count=[xcnt,ycnt]) #Load SST image ############### sst=f.select(param) attr=sst.attributes() slope=attr['slope'] intercept=attr['intercept'] flagValue=attr['bad_value_scaled'] sst_var=sst.get(start=[xst,yst], count=[xcnt,ycnt]) #Shrink sst image #Compute mask fg=f.select('qual_'+param) fg_var=fg.get(start=[xst,yst], count=[xcnt,ycnt]) if flagLevel is None : mask=sst_var == flagValue else : mask= (sst_var == flagValue) | (fg_var >= flagLevel) sst_var=np.ma.masked_array(sst_var*slope + intercept, mask=mask,type=float)
def load_generic(satscene, filename, resolution, cores): """Read modis data, generic part. """ try: data = SD(str(filename)) except HDF4Error as err: logger.warning("Could not load data from " + str(filename) + ": " + str(err)) return datadict = { 1000: ['EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB', 'EV_1KM_Emissive'], 500: ['EV_250_Aggr500_RefSB', 'EV_500_RefSB'], 250: ['EV_250_RefSB']} datasets = datadict[resolution] loaded_bands = [] # process by dataset, reflective and emissive datasets separately for dataset in datasets: subdata = data.select(dataset) band_names = subdata.attributes()["band_names"].split(",") if len(satscene.channels_to_load & set(band_names)) > 0: # get the relative indices of the desired channels indices = [i for i, band in enumerate(band_names) if band in satscene.channels_to_load] uncertainty = data.select(dataset + "_Uncert_Indexes") if dataset.endswith('Emissive'): array = calibrate_tb(subdata, uncertainty, indices, band_names) else: array = calibrate_refl(subdata, uncertainty, indices) for (i, idx) in enumerate(indices): satscene[band_names[idx]] = array[i] # fix the resolution to match the loaded data. satscene[band_names[idx]].resolution = resolution loaded_bands.append(band_names[idx]) # Get the orbit number if not satscene.orbit: mda = data.attributes()["CoreMetadata.0"] orbit_idx = mda.index("ORBITNUMBER") satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116] # Get the geolocation # if resolution != 1000: # logger.warning("Cannot load geolocation at this resolution (yet).") # return lat, lon = get_lat_lon(satscene, resolution, filename, cores) area = geometry.SwathDefinition(lons=lon, lats=lat) for band_name in loaded_bands: satscene[band_name].area = area # Trimming out dead sensor lines (detectors) on aqua: # (in addition channel 21 is noisy) if satscene.satname == "aqua": for band in ["6", "27", "36"]: if not satscene[band].is_loaded() or satscene[band].data.mask.all(): continue width = satscene[band].data.shape[1] height = satscene[band].data.shape[0] indices = satscene[band].data.mask.sum(1) < width if indices.sum() == height: continue satscene[band] = satscene[band].data[indices, :] satscene[band].area = geometry.SwathDefinition( lons=satscene[band].area.lons[indices, :], lats=satscene[band].area.lats[indices, :]) satscene[band].area.area_id = ("swath_" + satscene.fullname + "_" + str(satscene.time_slot) + "_" + str(satscene[band].shape) + "_" + str(band)) # Trimming out dead sensor lines (detectors) on terra: # (in addition channel 27, 30, 34, 35, and 36 are nosiy) if satscene.satname == "terra": for band in ["29"]: if not satscene[band].is_loaded() or satscene[band].data.mask.all(): continue width = satscene[band].data.shape[1] height = satscene[band].data.shape[0] indices = satscene[band].data.mask.sum(1) < width if indices.sum() == height: continue satscene[band] = satscene[band].data[indices, :] satscene[band].area = geometry.SwathDefinition( lons=satscene[band].area.lons[indices, :], lats=satscene[band].area.lats[indices, :]) satscene[band].area.area_id = ("swath_" + satscene.fullname + "_" + str(satscene.time_slot) + "_" + str(satscene[band].shape) + "_" + str(band))
class hdf (object): """wrapper for HDF4 dataset for comparison __call__ yields sequence of variable names __getitem__ returns individual variables ready for slicing to numpy arrays """ _hdf = None def __init__(self, filename, allowWrite=False): if pyhdf is None: LOG.error('pyhdf is not installed and is needed in order to read hdf4 files') assert(pyhdf is not None) mode = SDC.READ if allowWrite: mode = mode | SDC.WRITE self._hdf = SD(filename, mode) self.attributeCache = CaseInsensitiveAttributeCache(self) def __call__(self): "yield names of variables to be compared" return self._hdf.datasets().keys() # this returns a numpy array with a copy of the full, scaled # data for this variable, if the data type must be changed to allow # for scaling it will be (so the return type may not reflect the # type found in the original file) def __getitem__(self, name): # defaults scale_factor = 1.0 add_offset = 0.0 data_type = None scaling_method = None # get the variable object and use it to # get our raw data and scaling info variable_object = self.get_variable_object(name) raw_data_copy = variable_object[:] try : # TODO, this currently won't work with geocat data, work around it for now scale_factor, scale_factor_error, add_offset, add_offset_error, data_type = SDS.getcal(variable_object) except HDF4Error: # load just the scale factor and add offset information by hand temp = self.attributeCache.get_variable_attributes(name) if ADD_OFFSET_STR in temp.keys() : add_offset = temp[ADD_OFFSET_STR] data_type = np.dtype(type(add_offset)) if SCALE_FACTOR_STR in temp.keys() : scale_factor = temp[SCALE_FACTOR_STR] data_type = np.dtype(type(scale_factor)) if SCALE_METHOD_STR in temp.keys() : scaling_method = temp[SCALE_METHOD_STR] SDS.endaccess(variable_object) # don't do lots of work if we don't need to scale things if (scale_factor == 1.0) and (add_offset == 0.0) : return raw_data_copy # at the moment geocat has several scaling methods that don't match the normal standards for hdf """ please see constant.f90 for a more up to date version of this information: INTEGER(kind=int1) :: NO_SCALE ! 0 INTEGER(kind=int1) :: LINEAR_SCALE ! 1 INTEGER(kind=int1) :: LOG_SCALE ! 2 INTEGER(kind=int1) :: SQRT_SCALE ! 3 """ if (scaling_method == 0) : return raw_data_copy if not ((scaling_method is None) or (int(scaling_method) <= 1)) : LOG.warn ('Scaling method of \"' + str(scaling_method) + '\" will be ignored in favor of hdf standard method. ' + 'This may cause problems with data consistency') # if we don't have a data type something strange has gone wrong assert(not (data_type is None)) # get information about where the data is the missing value missing_val = self.missing_value(name) missing_mask = np.zeros(raw_data_copy.shape, dtype=np.bool) missing_mask[raw_data_copy == missing_val] = True # create the scaled version of the data scaled_data_copy = np.array(raw_data_copy, dtype=data_type) scaled_data_copy[~missing_mask] = (scaled_data_copy[~missing_mask] * scale_factor) + add_offset #TODO, type truncation issues? return scaled_data_copy def get_variable_object(self, name): return self._hdf.select(name) def missing_value(self, name): return self.get_attribute(name, fillValConst1) def create_new_variable(self, variablename, missingvalue=None, data=None, variabletocopyattributesfrom=None): """ create a new variable with the given name optionally set the missing value (fill value) and data to those given the created variable will be returned, or None if a variable could not be created """ raise IOUnimplimentedError('Unable to create variable in hdf file, this functionality is not yet available.') return None def add_attribute_data_to_variable(self, variableName, newAttributeName, newAttributeValue) : """ if the attribute exists for the given variable, set it to the new value if the attribute does not exist for the given variable, create it and set it to the new value """ raise IOUnimplimentedError('Unable add attribute to hdf file, this functionality is not yet available.') return def get_variable_attributes (self, variableName, caseInsensitive=True) : """ returns all the attributes associated with a variable name """ toReturn = None if caseInsensitive : toReturn = self.attributeCache.get_variable_attributes(variableName) else : toReturn = self.get_variable_object(variableName).attributes() return toReturn def get_attribute(self, variableName, attributeName, caseInsensitive=True) : """ returns the value of the attribute if it is available for this variable, or None """ toReturn = None if caseInsensitive : toReturn = self.attributeCache.get_variable_attribute(variableName, attributeName) else : temp_attributes = self.get_variable_attributes(variableName, caseInsensitive=False) if attributeName in temp_attributes : toReturn = temp_attributes[attributeName] return toReturn def get_global_attributes(self, caseInsensitive=True) : """ get a list of all the global attributes for this file or None """ toReturn = None if caseInsensitive : toReturn = self.attributeCache.get_global_attributes() else : toReturn = self._hdf.attributes() return toReturn def get_global_attribute(self, attributeName, caseInsensitive=True) : """ returns the value of a global attribute if it is available or None """ toReturn = None if caseInsensitive : toReturn = self.attributeCache.get_global_attribute(attributeName) else : if attributeName in self._hdf.attributes() : toReturn = self._hdf.attributes()[attributeName] return toReturn def is_loadable_type (self, name) : """ check to see if the indicated variable is a type that can be loaded """ # TODO, are there any bad types for these files? return True
def load(satscene, **kwargs): """Read data from file and load it into *satscene*. Load data into the *channels*. *Channels* is a list or a tuple containing channels we will load data into. If None, all channels are loaded. """ del kwargs conf = ConfigParser() conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg")) options = {} for option, value in conf.items(satscene.instrument_name+"-level3", raw = True): options[option] = value pathname = os.path.join(options["dir"], options['filename']) filename = satscene.time_slot.strftime(pathname) for prodname in GEO_PHYS_PRODUCTS + FLAGS_QUALITY: if prodname in satscene.channels_to_load: prod_chan = ModisEosHdfLevel2(prodname) prod_chan.read(filename) prod_chan.satid = satscene.satname.capitalize() prod_chan.resolution = 1000.0 prod_chan.shape = prod_chan.data.shape # All this for the netCDF writer: prod_chan.info['var_name'] = prodname prod_chan.info['var_data'] = prod_chan.data resolution_str = str(int(prod_chan.resolution))+'m' prod_chan.info['var_dim_names'] = ('y'+resolution_str, 'x'+resolution_str) prod_chan.info['long_name'] = prod_chan.attr['long_name'][:-1] try: prod_chan.info['standard_name'] = prod_chan.attr['standard_name'][:-1] except KeyError: pass valid_min = np.min(prod_chan.data) valid_max = np.max(prod_chan.data) prod_chan.info['valid_range'] = np.array([valid_min, valid_max]) prod_chan.info['resolution'] = prod_chan.resolution if prodname == 'l2_flags': # l2 flags definitions for i in range(1, 33): key = "f%02d_name"%i prod_chan.info[key] = prod_chan.attr[key][:-1] satscene.channels.append(prod_chan) if prodname in CHANNELS: satscene[prodname].info['units'] = '%' else: satscene[prodname].info['units'] = prod_chan.attr['units'][:-1] LOG.info("Loading modis lvl2 product '%s' done"%prodname) # Check if there are any bands to load: channels_to_load = False for bandname in CHANNELS: if bandname in satscene.channels_to_load: channels_to_load = True break if channels_to_load: #print "FILE: ", filename eoshdf = SD(filename) # Get all the Attributes: # Common Attributes, Data Time, # Data Structure and Scene Coordinates info = {} for key in eoshdf.attributes().keys(): info[key] = eoshdf.attributes()[key] dsets = eoshdf.datasets() selected_dsets = [] for bandname in CHANNELS: if (bandname in satscene.channels_to_load and bandname in dsets): value = eoshdf.select(bandname) selected_dsets.append(value) # Get only the selected datasets attr = value.attributes() band = value.get() nodata = attr['bad_value_scaled'] mask = np.equal(band, nodata) satscene[bandname] = (np.ma.masked_where(mask, band) * attr['slope'] + attr['intercept']) satscene[bandname].info['units'] = '%' satscene[bandname].info['long_name'] = attr['long_name'][:-1] for dset in selected_dsets: dset.endaccess() LOG.info("Loading modis lvl2 Remote Sensing Reflectances done") eoshdf.end() lat, lon = get_lat_lon(satscene, None) from pyresample import geometry satscene.area = geometry.SwathDefinition(lons=lon, lats=lat) #print "Variant: ", satscene.variant satscene.variant = 'regional' # Temporary fix! LOG.info("Loading modis data done.")
class HDF4Store(xmlplot.common.VariableStore,xmlstore.util.referencedobject): class Variable(xmlplot.common.Variable): def __init__(self,store,hdfvar): xmlplot.common.Variable.__init__(self,store) self.hdfvar = hdfvar self.info = self.hdfvar.info() def getName_raw(self): return self.info[0] def getDimensions_raw(self): dimnames = [] for idim in range(self.info[1]): dim = self.hdfvar.dim(idim) dimnames.append(dim.info()[0]) return dimnames def getLongName(self): atts = self.getProperties() if 'long_name' in atts: return atts['long_name'] return xmlplot.common.Variable.getLongName(self) def getUnit(self): atts = self.getProperties() if 'units' in atts: return atts['units'] return xmlplot.common.Variable.getUnit(self) def getShape(self): shape = self.info[2] if isinstance(shape,int): shape = (shape,) return shape def getDataType(self): return datatypes.get(self.info[3],None) def getProperties(self): return self.hdfvar.attributes() def getSlice(self,bounds=None,dataonly=False,transfercoordinatemask=True): dimnames = self.getDimensions_raw() shape = self.getShape() # Determine final slice if bounds is None: bounds = (Ellipsis,) newbounds = [] for bound,dimlen,dimname in zip(xmlplot.common.processEllipsis(bounds,shape),shape,dimnames): if isinstance(bound,int): # Integer value provided as index. assert bound>=-dimlen, 'Slice index %i lies below the lowest possible index for dimension %s (%i).' % (bound,dimname,-dimlen ) assert bound< dimlen, 'Slice index %i exceeds the highest possible index for dimension %s (%i).' % (bound,dimname, dimlen-1) if bound<0: bound += dimlen elif isinstance(bound,slice): start,stop,step = bound.indices(dimlen) bound = slice(start,stop,step) newbounds.append(bound) bounds = tuple(newbounds) # Get data dat = numpy.asarray(self.hdfvar[bounds]) # Mask fill value fillvalue = self.hdfvar.attributes().get('_FillValue',None) if fillvalue is None: fillvalue = self.hdfvar.attributes().get('Fill',None) if fillvalue is not None: dat = numpy.ma.array(dat,mask=(dat==fillvalue),copy=False) # Determine scale factor and offset, and cast data to acommodating type if needed. scale = self.hdfvar.attributes().get('scale_factor',None) offset = self.hdfvar.attributes().get('add_offset', None) if scale is not None or offset is not None and dat.dtype!=numpy.float: dat = dat.astype(numpy.float) if scale is not None: dat *= scale if offset is not None: dat += offset if dataonly: return dat newdimnames = [d for d,b in zip(dimnames,bounds) if not isinstance(b,int)] varslice = self.Slice(newdimnames) varslice.data = dat inewdim = 0 for dimname,bound in zip(dimnames,bounds): # Get the coordinate variable coordvar = self.store.getVariable_raw(dimname) if coordvar is None: # No coordinate variable available: use indices if not isinstance(bound,slice): continue coorddims = [dimname] coords = numpy.arange(bound.start,bound.stop,bound.step,dtype=numpy.float) else: # Coordinate variable present: use it. coorddims = list(coordvar.getDimensions()) # Debug check: see if all coordinate dimensions are also used by the variable. for cd in coorddims: assert cd in dimnames, 'Coordinate dimension %s is not used by this variable (it uses %s).' % (cd,', '.join(dimnames)) # Get coordinate values coordslice = [bounds[dimnames.index(cd)] for cd in coorddims] coords = coordvar.getSlice(coordslice, dataonly=True) # Get the list of coordinate dimensions after the ones with single index have been sliced out. newcoorddims = [cd for cd in coorddims if isinstance(bounds[dimnames.index(cd)],slice)] # Transfer the coordinate mask to the data if desired. coordmask = numpy.ma.getmask(coords) if transfercoordinatemask and coordmask is not numpy.ma.nomask: coordmask = xmlplot.common.broadcastSelective(coordmask,newcoorddims,dat.shape,newdimnames) if datamask is numpy.ma.nomask: datamask = coordmask else: datamask |= coordmask # If we take a single index for this dimension, it will not be included in the output. if not isinstance(bound,slice): continue # Coordinates should not have a mask - undo the masking. if coordmask is not numpy.ma.nomask: coords = numpy.ma.getdata(coords) # Auto-generate staggered coordinates coords_stag = xmlplot.common.stagger(coords) # Insert data dimensions where they are lacking in coordinate coords = xmlplot.common.broadcastSelective(coords, (dimname,),dat.shape, newdimnames) coords_stag = xmlplot.common.broadcastSelective(coords_stag,(dimname,),[l+1 for l in dat.shape],newdimnames) # Assign coordinate values varslice.coords [inewdim] = coords varslice.coords_stag[inewdim] = coords_stag inewdim += 1 return varslice def __init__(self,path): xmlplot.common.VariableStore.__init__(self) xmlstore.util.referencedobject.__init__(self) from pyhdf.SD import SD, SDC self.file = SD(str(path),SDC.READ) def getVariable_raw(self,varname): """Returns a Variable object for the given original short variable name. The method must be implemented by derived classes. """ if varname not in self.file.datasets().keys(): return None return self.Variable(self,self.file.select(varname)) def getVariableNames_raw(self): """Returns a list of original short names for all variables present in the store. The method must be implemented by derived classes. """ return self.file.datasets().keys() def getProperties(self): return self.file.attributes() def unlink(self): self.file.end()
def run(FILE_NAME): DATAFIELD_NAME = 'Gpp_1km' if USE_GDAL: import gdal GRID_NAME = 'MOD_Grid_MOD17A2' gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME, GRID_NAME, DATAFIELD_NAME) gdset = gdal.Open(gname) data = gdset.ReadAsArray().astype(np.float64) # Get any needed attributes. meta = gdset.GetMetadata() scale_factor = np.float(meta['scale_factor']) add_offset = np.float(meta['add_offset']) _FillValue = np.float(meta['_FillValue']) valid_range = [np.float(x) for x in meta['valid_range'].split(', ')] units = meta['units'] long_name = meta['long_name'] # Construct the grid. x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform() nx, ny = (gdset.RasterXSize, gdset.RasterYSize) x = np.linspace(x0, x0 + xinc*nx, nx) y = np.linspace(y0, y0 + yinc*ny, ny) xv, yv = np.meshgrid(x, y) del gdset else: if USE_NETCDF: from netCDF4 import Dataset nc = Dataset(FILE_NAME) ncvar = nc.variables[DATAFIELD_NAME] # The scaling equation isn't "scale * data + offset", # so turn automatic scaling off. ncvar.set_auto_maskandscale(False) data = ncvar[:].astype(np.float64) # Get any needed attributes. scale_factor = ncvar.scale_factor add_offset = ncvar.add_offset _FillValue = ncvar._FillValue valid_range = ncvar.valid_range units = ncvar.units long_name = ncvar.long_name gridmeta = getattr(nc, 'StructMetadata.0') else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:,:].astype(np.double) # Read attributes. attrs = data2D.attributes(full=1) lna=attrs["long_name"] long_name = lna[0] vra=attrs["valid_range"] valid_range = vra[0] aoa=attrs["add_offset"] add_offset = aoa[0] fva=attrs["_FillValue"] _FillValue = fva[0] sfa=attrs["scale_factor"] scale_factor = sfa[0] ua=attrs["units"] units = ua[0] fattrs = hdf.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] # Construct the grid. The needed information is in a global attribute # called 'StructMetadata.0'. Use regular expressions to tease out the # extents of the grid. In addition, the grid is in packed decimal # degrees, so we need to normalize to degrees. ul_regex = re.compile(r'''UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = ul_regex.search(gridmeta) x0 = np.float(match.group('upper_left_x')) y0 = np.float(match.group('upper_left_y')) lr_regex = re.compile(r'''LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = lr_regex.search(gridmeta) x1 = np.float(match.group('lower_right_x')) y1 = np.float(match.group('lower_right_y')) ny, nx = data.shape x = np.linspace(x0, x1, nx) y = np.linspace(y0, y1, ny) xv, yv = np.meshgrid(x, y) # In basemap, the sinusoidal projection is global, so we won't use it. # Instead we'll convert the grid back to lat/lons so we can use a local # projection. sinu = pyproj.Proj("+proj=sinu +R=6371007.181 +nadgrids=@null +wktext") wgs84 = pyproj.Proj("+init=EPSG:4326") lon, lat= pyproj.transform(sinu, wgs84, xv, yv) # Apply the attributes to the data. invalid = np.logical_or(data < valid_range[0], data > valid_range[1]) invalid = np.logical_or(invalid, data ==_FillValue) data[invalid] = np.nan data = (data - add_offset) * scale_factor data = np.ma.masked_array(data, np.isnan(data)) m = Basemap(projection='cyl', resolution='l', llcrnrlat=2.5, urcrnrlat=12.5, llcrnrlon=-87.5, urcrnrlon = -77.5) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(0, 15, 5), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-90, 75, 5), labels=[0, 0, 0, 1]) m.pcolormesh(lon, lat, data, latlon=True) cb = m.colorbar() cb.set_label(units) basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, long_name)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'Sea_Ice_by_Reflectance' if USE_GDAL: import gdal GRID_NAME = 'MOD_Grid_Seaice_1km' gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME, GRID_NAME, DATAFIELD_NAME) gdset = gdal.Open(gname) data = gdset.ReadAsArray() # Construct the grid. meta = gdset.GetMetadata() x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform() nx, ny = (gdset.RasterXSize, gdset.RasterYSize) del gdset else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:,:].astype(np.float64) # Read global attribute. fattrs = hdf.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] # Construct the grid. The needed information is in a global attribute # called 'StructMetadata.0'. Use regular expressions to tease out the # extents of the grid. ul_regex = re.compile(r'''UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = ul_regex.search(gridmeta) x0 = np.float(match.group('upper_left_x')) y0 = np.float(match.group('upper_left_y')) lr_regex = re.compile(r'''LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = lr_regex.search(gridmeta) x1 = np.float(match.group('lower_right_x')) y1 = np.float(match.group('lower_right_y')) ny, nx = data.shape xinc = (x1 - x0) / nx yinc = (y1 - y0) / ny x = np.linspace(x0, x0 + xinc*nx, nx) y = np.linspace(y0, y0 + yinc*ny, ny) xv, yv = np.meshgrid(x, y) # Reproject the coordinates out of lamaz into lat/lon. lamaz = pyproj.Proj("+proj=laea +a=6371228 +lat_0=90 +lon_0=0 +units=m") wgs84 = pyproj.Proj("+init=EPSG:4326") lon, lat= pyproj.transform(lamaz, wgs84, xv, yv) # Draw a lambert equal area azimuthal basemap. m = Basemap(projection='laea', resolution='l', lat_ts=50, lat_0=50, lon_0=150, width=2500000,height=2500000) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(50, 91, 10), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(110, 181, 10), labels=[0, 0, 0, 1]) # Use a discretized colormap since we have only a few levels. # 0=missing data # 1=no decision # 11=night # 25=land # 37=inland water # 39=ocean # 50=cloud # 200=sea ice # 253=no input tile expected # 254=non-production mask" # 255=fill lst = ['#727272', '#b7b7b7', '#ffff96', '#00ff00', '#232375', '#232375', '#63c6ff', '#ff0000', '#3f3f3f', '#000000', '#000000'] cmap = mpl.colors.ListedColormap(lst) bounds = [0, 1, 11, 25, 37, 39, 50, 200, 253, 254, 255, 256] norm = mpl.colors.BoundaryNorm(bounds, cmap.N) m.pcolormesh(lon, lat, data, latlon=True, cmap=cmap, norm=norm) color_bar = plt.colorbar() color_bar.set_ticks([0.5, 5.5, 18, 31, 38, 44.5, 125, 226.5, 253.5, 254.5, 255.5]) color_bar.set_ticklabels(['missing', 'no decision', 'night', 'land', 'inland water', 'ocean', 'cloud', 'sea ice', 'no input tile\nexpected', 'non-production\nmask', 'fill']) color_bar.draw_all() basename = os.path.basename(FILE_NAME) long_name = DATAFIELD_NAME plt.title('{0}\n{1}'.format(basename, long_name)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def run(FILE_NAME): DATAFIELD_NAME = 'Sea_Ice_by_Reflectance_SP' if USE_NETCDF4: from netCDF4 import Dataset nc = Dataset(FILE_NAME) ncvar = nc.variables[DATAFIELD_NAME] data = ncvar[:].astype(np.float64) gridmeta = getattr(nc, 'StructMetadata.0') else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:,:].astype(np.float64) # Read global attribute. fattrs = hdf.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] # Construct the grid. The needed information is in a global attribute # called 'StructMetadata.0'. Use regular expressions to tease out the # extents of the grid. ul_regex = re.compile(r'''UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = ul_regex.search(gridmeta) x0 = np.float(match.group('upper_left_x')) y0 = np.float(match.group('upper_left_y')) lr_regex = re.compile(r'''LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = lr_regex.search(gridmeta) x1 = np.float(match.group('lower_right_x')) y1 = np.float(match.group('lower_right_y')) ny, nx = data.shape x = np.linspace(x0, x1, nx) y = np.linspace(y0, y1, ny) xv, yv = np.meshgrid(x, y) # Reproject into latlon # Reproject the coordinates out of lamaz into lat/lon. lamaz = pyproj.Proj("+proj=laea +a=6371228 +lat_0=-90 +lon_0=0 +units=m") wgs84 = pyproj.Proj("+init=EPSG:4326") lon, lat= pyproj.transform(lamaz, wgs84, xv, yv) # Use a south polar azimuthal equal area projection. m = Basemap(projection='splaea', resolution='l', boundinglat=-20, lon_0=180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90, 0, 15), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180, 180, 45), labels=[0, 0, 0, 1]) # Use a discretized colormap since we have only a few levels. # 0=missing data # 1=no decision # 11=night # 25=land # 37=inland water # 39=ocean # 50=cloud # 200=sea ice # 253=no input tile expected # 254=non-production mask" lst = ['#727272', '#b7b7b7', '#ffff96', '#00ff00', '#232375', '#232375', '#63c6ff', '#ff0000', '#3f3f3f', '#000000'] cmap = mpl.colors.ListedColormap(lst) bounds = [0, 1, 11, 25, 37, 39, 50, 200, 253, 254, 255] norm = mpl.colors.BoundaryNorm(bounds, cmap.N) # Render only a subset of the mesh. rows = slice(500, 4000, 5) cols = slice(500, 4000, 5) m.pcolormesh(lon[rows,cols], lat[rows,cols], data[rows,cols], latlon=True, cmap=cmap, norm=norm) color_bar = plt.colorbar() color_bar.set_ticks([0.5, 5.5, 18, 31, 38, 44.5, 125, 226.5, 253.5, 254.5]) color_bar.set_ticklabels(['missing', 'no decision', 'night', 'land', 'inland water', 'ocean', 'cloud', 'sea ice', 'no input tile\nexpected', 'non-production\nmask']) color_bar.draw_all() basename = os.path.basename(FILE_NAME) long_name = DATAFIELD_NAME plt.title('{0}\n{1}'.format(basename, long_name)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def run(FILE_NAME): DATAFIELD_NAME = 'LST_Night_CMG' if USE_GDAL: # GDAL import gdal GRID_NAME = 'MODIS_8DAY_0.05DEG_CMG_LST' gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME, GRID_NAME, DATAFIELD_NAME) gdset = gdal.Open(gname) # data = gdset.ReadAsArray().astype(np.float64)[::4, ::4] data = gdset.ReadAsArray().astype(np.float64)[:,:] # Get any needed attributes. meta = gdset.GetMetadata() scale_factor = np.float(meta['scale_factor']) add_offset = np.float(meta['add_offset']) _FillValue = np.float(meta['_FillValue']) valid_range = [np.float(x) for x in meta['valid_range'].split(', ')] units = meta['units'] long_name = meta['long_name'] # Construct the grid. Subset by a factor of 4. x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform() # nx, ny = (gdset.RasterXSize / 4, gdset.RasterYSize / 4) nx, ny = (gdset.RasterXSize, gdset.RasterYSize) # x = np.linspace(x0, x0 + xinc*4*nx, nx) # y = np.linspace(y0, y0 + yinc*4*ny, ny) x = np.linspace(x0, x0 + xinc*nx, nx) y = np.linspace(y0, y0 + yinc*ny, ny) lon, lat = np.meshgrid(x, y) del gdset else: if USE_NETCDF: from netCDF4 import Dataset # The scaling equation isn't what netcdf4 expects, so turn it off. nc = Dataset(FILE_NAME) ncvar = nc.variables[DATAFIELD_NAME] ncvar.set_auto_maskandscale(False) data = ncvar[::4, ::4].astype(np.float64) # Get any needed attributes. scale_factor = ncvar.scale_factor add_offset = ncvar.add_offset _FillValue = ncvar._FillValue valid_range = ncvar.valid_range units = ncvar.units long_name = ncvar.long_name gridmeta = getattr(nc, 'StructMetadata.0') else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:,:].astype(np.double) # Read attributes. attrs = data2D.attributes(full=1) lna=attrs["long_name"] long_name = lna[0] vra=attrs["valid_range"] valid_range = vra[0] aoa=attrs["add_offset"] add_offset = aoa[0] fva=attrs["_FillValue"] _FillValue = fva[0] sfa=attrs["scale_factor"] scale_factor = sfa[0] ua=attrs["units"] units = ua[0] fattrs = hdf.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] # Construct the grid. The needed information is in a global attribute # called 'StructMetadata.0'. Use regular expressions to tease out the # extents of the grid. In addition, the grid is in packed decimal # degrees, so we need to normalize to degrees. ul_regex = re.compile(r'''UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = ul_regex.search(gridmeta) x0 = np.float(match.group('upper_left_x')) / 1e6 y0 = np.float(match.group('upper_left_y')) / 1e6 lr_regex = re.compile(r'''LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = lr_regex.search(gridmeta) x1 = np.float(match.group('lower_right_x')) / 1e6 y1 = np.float(match.group('lower_right_y')) / 1e6 ny, nx = data.shape x = np.linspace(x0, x1, nx) y = np.linspace(y0, y1, ny) lon, lat = np.meshgrid(x, y) # Apply the attributes to the data. invalid = np.logical_or(data < valid_range[0], data > valid_range[1]) invalid = np.logical_or(invalid, data ==_FillValue) data[invalid] = np.nan data = (data - add_offset) * scale_factor data = np.ma.masked_array(data, np.isnan(data)) m = Basemap(projection='cyl', resolution='l', llcrnrlat=-90, urcrnrlat=90, llcrnrlon=-180, urcrnrlon = 180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90, 91, 30), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180, 181, 45), labels=[0, 0, 0, 1]) m.pcolormesh(lon, lat, data, latlon=True) cb = m.colorbar() cb.set_label(units) basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, long_name)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
class ModisReader(Reader): pformat = "hdfeos_l1b" res = {"1": 1000, "Q": 250, "H": 500} def __init__(self, *args, **kwargs): Reader.__init__(self, *args, **kwargs) self.datafiles = {} self.geofile = None self.filename = None self.data = None self.areas = {} def load(self, satscene, *args, **kwargs): """Read data from file and load it into *satscene*. """ del args conf = ConfigParser() conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg")) options = dict(conf.items(satscene.instrument_name + "-level2", raw=True)) options["resolution"] = 1000 options["geofile"] = os.path.join(options["dir"], options["geofile"]) options.update(kwargs) if isinstance(kwargs.get("filename"), (list, set, tuple)): # we got the entire dataset. for fname in kwargs["filename"]: if fnmatch(os.path.basename(fname), "M?D02?km*"): resolution = self.res[os.path.basename(fname)[5]] self.datafiles[resolution] = fname elif fnmatch(os.path.basename(fname), "M?D03*"): self.geofile = fname elif ((kwargs.get("filename") is not None) and fnmatch(os.path.basename(options["filename"]), "M?D02?km*")): # read just one file logger.debug("Reading from file: " + str(options["filename"])) filename = options["filename"] resolution = self.res[os.path.basename(filename)[5]] self.datafiles[resolution] = filename else: # find files according to config resolution = int(options["resolution"]) or 1000 for res in [250, 500, 1000]: datafile = os.path.join(options['dir'], options["filename" + str(res)]) try: self.datafiles[res] = get_filename(datafile, satscene.time_slot) except IOError: self.datafiles[res] = None logger.warning("Can't find file for resolution %s with template: %s", str(res), datafile) try: self.geofile = get_filename(options["geofile"], satscene.time_slot) except IOError: self.geofile = None logger.warning("Can't find geofile with template: %s", options['geofile']) resolution = options["resolution"] cores = options.get("cores", max(multiprocessing.cpu_count() / 4, 1)) datadict = { 1000: ['EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB', 'EV_1KM_Emissive'], 500: ['EV_250_Aggr500_RefSB', 'EV_500_RefSB'], 250: ['EV_250_RefSB']} loaded_bands = [] # process by dataset, reflective and emissive datasets separately resolutions = [250, 500, 1000] for res in resolutions: if res < resolution: continue logger.debug("Working on resolution %d", res) self.filename = self.datafiles[res] logger.debug("Using " + str(cores) + " cores for interpolation") try: self.data = SD(str(self.filename)) except HDF4Error as err: logger.warning("Could not load data from " + str(self.filename) + ": " + str(err)) continue datasets = datadict[res] for dataset in datasets: subdata = self.data.select(dataset) band_names = subdata.attributes()["band_names"].split(",") if len(satscene.channels_to_load & set(band_names)) > 0: # get the relative indices of the desired channels indices = [i for i, band in enumerate(band_names) if band in satscene.channels_to_load] uncertainty = self.data.select(dataset + "_Uncert_Indexes") if dataset.endswith('Emissive'): array = calibrate_tb( subdata, uncertainty, indices, band_names) else: array = calibrate_refl(subdata, uncertainty, indices) for (i, idx) in enumerate(indices): if band_names[idx] in loaded_bands: continue satscene[band_names[idx]] = array[i] # fix the resolution to match the loaded data. satscene[band_names[idx]].resolution = res loaded_bands.append(band_names[idx]) # Get the orbit number if not satscene.orbit: mda = self.data.attributes()["CoreMetadata.0"] orbit_idx = mda.index("ORBITNUMBER") satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116] # Get the geolocation # if resolution != 1000: # logger.warning("Cannot load geolocation at this resolution (yet).") # return for band_name in loaded_bands: lon, lat = self.get_lonlat(satscene[band_name].resolution, cores) area = geometry.SwathDefinition(lons=lon, lats=lat) satscene[band_name].area = area # Trimming out dead sensor lines (detectors) on aqua: # (in addition channel 21 is noisy) if satscene.satname == "aqua": for band in ["6", "27", "36"]: if not satscene[band].is_loaded() or satscene[band].data.mask.all(): continue width = satscene[band].data.shape[1] height = satscene[band].data.shape[0] indices = satscene[band].data.mask.sum(1) < width if indices.sum() == height: continue satscene[band] = satscene[band].data[indices, :] satscene[band].area = geometry.SwathDefinition( lons=satscene[band].area.lons[indices, :], lats=satscene[band].area.lats[indices, :]) # Trimming out dead sensor lines (detectors) on terra: # (in addition channel 27, 30, 34, 35, and 36 are nosiy) if satscene.satname == "terra": for band in ["29"]: if not satscene[band].is_loaded() or satscene[band].data.mask.all(): continue width = satscene[band].data.shape[1] height = satscene[band].data.shape[0] indices = satscene[band].data.mask.sum(1) < width if indices.sum() == height: continue satscene[band] = satscene[band].data[indices, :] satscene[band].area = geometry.SwathDefinition( lons=satscene[band].area.lons[indices, :], lats=satscene[band].area.lats[indices, :]) for band_name in loaded_bands: band_uid = hashlib.sha1(satscene[band_name].data.mask).hexdigest() satscene[band_name].area.area_id = ("swath_" + satscene.fullname + "_" + str(satscene.time_slot) + "_" + str(satscene[ band_name].shape) + "_" + str(band_uid)) satscene[band_name].area_id = satscene[band_name].area.area_id def get_lonlat(self, resolution, cores=1): """Read lat and lon. """ if resolution in self.areas: return self.areas[resolution] logger.debug("generating lon, lat at %d", resolution) if self.geofile is not None: coarse_resolution = 1000 filename = self.geofile else: coarse_resolution = 5000 logger.info("Using 5km geolocation and interpolating") filename = (self.datafiles.get(1000) or self.datafiles.get(500) or self.datafiles.get(250)) logger.debug("Loading geolocation from file: " + str(filename) + " at resolution " + str(coarse_resolution)) data = SD(str(filename)) lat = data.select("Latitude") fill_value = lat.attributes()["_FillValue"] lat = np.ma.masked_equal(lat.get(), fill_value) lon = data.select("Longitude") fill_value = lon.attributes()["_FillValue"] lon = np.ma.masked_equal(lon.get(), fill_value) if resolution == coarse_resolution: self.areas[resolution] = lon, lat return lon, lat from geotiepoints import modis5kmto1km, modis1kmto500m, modis1kmto250m logger.debug("Interpolating from " + str(coarse_resolution) + " to " + str(resolution)) if coarse_resolution == 5000: lon, lat = modis5kmto1km(lon, lat) if resolution == 500: lon, lat = modis1kmto500m(lon, lat, cores) if resolution == 250: lon, lat = modis1kmto250m(lon, lat, cores) self.areas[resolution] = lon, lat return lon, lat # These have to be interpolated... def get_height(self): return self.data.select("Height") def get_sunz(self): return self.data.select("SolarZenith") def get_suna(self): return self.data.select("SolarAzimuth") def get_satz(self): return self.data.select("SensorZenith") def get_sata(self): return self.data.select("SensorAzimuth")
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = "SI_06km_NH_89V_DAY" if USE_GDAL: import gdal GRID_NAME = "NpPolarGrid06km" gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME, GRID_NAME, DATAFIELD_NAME) gdset = gdal.Open(gname) data = gdset.ReadAsArray().astype(np.float64) # Read projection parameters from global attribute. meta = gdset.GetMetadata() x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform() nx, ny = (gdset.RasterXSize, gdset.RasterYSize) del gdset else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:, :].astype(np.float64) # Read global attribute. fattrs = hdf.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] # Construct the grid. The needed information is in a global attribute # called 'StructMetadata.0'. Use regular expressions to tease out the # extents of the grid. ul_regex = re.compile( r"""UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)""", re.VERBOSE, ) match = ul_regex.search(gridmeta) x0 = np.float(match.group("upper_left_x")) y0 = np.float(match.group("upper_left_y")) lr_regex = re.compile( r"""LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)""", re.VERBOSE, ) match = lr_regex.search(gridmeta) x1 = np.float(match.group("lower_right_x")) y1 = np.float(match.group("lower_right_y")) ny, nx = data.shape xinc = (x1 - x0) / nx yinc = (y1 - y0) / ny # Apply the attributes information. # Ref: http://nsidc.org/data/docs/daac/ae_si6_6km_tbs.gd.html#2 data[data == 0] = np.nan data *= 0.1 data = np.ma.masked_array(data, np.isnan(data)) x = np.linspace(x0, x0 + xinc * nx, nx) y = np.linspace(y0, y0 + yinc * ny, ny) xv, yv = np.meshgrid(x, y) args = [ "+proj=stere", "+lat_0=90", "+lon_0=-45", "+lat_ts=70", "+k=1", "+es=0.006693883", "+a=6378273", "+x_0=0", "+y_0=0", "+ellps=WGS84", "+datum=WGS84", ] pstereo = pyproj.Proj(" ".join(args)) wgs84 = pyproj.Proj("+init=EPSG:4326") lon, lat = pyproj.transform(pstereo, wgs84, xv, yv) units = "K" long_name = DATAFIELD_NAME m = Basemap(projection="npstere", resolution="l", boundinglat=30, lon_0=0) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(0, 91, 20), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180, 181, 45), labels=[0, 0, 0, 1]) m.pcolormesh(lon, lat, data, latlon=True) cb = m.colorbar() cb.set_label(units) basename = os.path.basename(FILE_NAME) plt.title("{0}\n{1}".format(basename, long_name)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'Extent' if USE_GDAL: import gdal GRID_NAME = 'Northern Hemisphere' gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME, GRID_NAME, DATAFIELD_NAME) gdset = gdal.Open(gname) data = gdset.ReadAsArray() meta = gdset.GetMetadata() x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform() nx, ny = (gdset.RasterXSize, gdset.RasterYSize) del gdset else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:,:].astype(np.float64) # Read global attribute. fattrs = hdf.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] # Construct the grid. The needed information is in a global attribute # called 'StructMetadata.0'. Use regular expressions to tease out the # extents of the grid. ul_regex = re.compile(r'''UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = ul_regex.search(gridmeta) x0 = np.float(match.group('upper_left_x')) y0 = np.float(match.group('upper_left_y')) lr_regex = re.compile(r'''LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = lr_regex.search(gridmeta) x1 = np.float(match.group('lower_right_x')) y1 = np.float(match.group('lower_right_y')) ny, nx = data.shape xinc = (x1 - x0) / nx yinc = (y1 - y0) / ny x = np.linspace(x0, x0 + xinc*nx, nx) y = np.linspace(y0, y0 + yinc*ny, ny) xv, yv = np.meshgrid(x, y) # Reproject into WGS84 lamaz = pyproj.Proj("+proj=laea +a=6371228 +lat_0=90 +lon_0=0 +units=m") wgs84 = pyproj.Proj("+init=EPSG:4326") lon, lat= pyproj.transform(lamaz, wgs84, xv, yv) # Use a north polar azimuthal equal area projection. m = Basemap(projection='nplaea', resolution='l', boundinglat=40, lon_0=0) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(0, 90, 15), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180, 180, 45), labels=[0, 0, 0, 1]) # Bin the data as follows: # 0 -- snow-free land # 1-20% sea ice -- blue # 21-40% sea ice -- blue-cyan # 41-60% sea ice -- blue # 61-80% sea ice -- cyan-blue # 81-100% sea ice -- cyan # 101 -- permanent ice # 103 -- dry snow # 252 mixed pixels at coastlines # 255 ocean lst = ['#004400', '#0000ff', '#0044ff', '#0088ff', '#00ccff', '#00ffff', '#ffffff', '#440044', '#191919', '#000000', '#8888cc'] cmap = mpl.colors.ListedColormap(lst) bounds = [0, 1, 21, 41, 61, 81, 101, 103, 104, 252, 255] tickpts = [0.5, 11, 31, 51, 71, 91, 102, 103.5, 178, 253.5] norm = mpl.colors.BoundaryNorm(bounds, cmap.N) # The corners cause trouble, so chop them out. idx = slice(5, 721) m.pcolormesh(lon[idx, idx], lat[idx, idx], data[idx, idx], latlon=True, cmap=cmap, norm=norm) color_bar = plt.colorbar() color_bar.set_ticks(tickpts) color_bar.set_ticklabels(['snow-free\nland', '1-20% sea ice', '21-40% sea ice', '41-60% sea ice', '61-80% sea ice', '81-100% sea ice', 'permanent\nice', 'dry\nsnow', 'mixed pixels\nat coastlines', 'ocean']) color_bar.draw_all() basename = os.path.basename(FILE_NAME) long_name = DATAFIELD_NAME plt.title('{0}\n{1}'.format(basename, long_name)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = 'SI_12km_NH_ICECON_DAY' from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:,:].astype(np.float64) # Read global attribute. fattrs = hdf.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] # Construct the grid. The needed information is in a global attribute # called 'StructMetadata.0'. Use regular expressions to tease out the # extents of the grid. ul_regex = re.compile(r'''UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = ul_regex.search(gridmeta) x0 = np.float(match.group('upper_left_x')) y0 = np.float(match.group('upper_left_y')) lr_regex = re.compile(r'''LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = lr_regex.search(gridmeta) x1 = np.float(match.group('lower_right_x')) y1 = np.float(match.group('lower_right_y')) ny, nx = data.shape xinc = (x1 - x0) / nx yinc = (y1 - y0) / ny # Handle land mask. # http://nsidc.org/data/docs/daac/ae_si12_12km_seaice/data.html data[data > 100.0] = np.nan data = np.ma.masked_array(data, np.isnan(data)) # Construct the grid. # Reproject out of the GCTP stereographic into lat/lon. x = np.linspace(x0, x0 + xinc*nx, nx) y = np.linspace(y0, y0 + yinc*ny, ny) xv, yv = np.meshgrid(x, y) args = ["+proj=stere", "+lat_0=90", "+lon_0=-45", "+lat_ts=70", "+k=1", "+es=0.006693883", "+a=6378273", "+x_0=0", "+y_0=0", "+ellps=WGS84", "+datum=WGS84"] pstereo = pyproj.Proj(' '.join(args)) wgs84 = pyproj.Proj("+init=EPSG:4326") lon, lat= pyproj.transform(pstereo, wgs84, xv, yv) units = 'Percent' long_name = DATAFIELD_NAME m = Basemap(projection='npstere', resolution='l', boundinglat=30, lon_0 = 0) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(0, 91, 20), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180, 181, 45), labels=[0, 0, 0, 1]) m.pcolormesh(lon, lat, data, latlon=True) cb = m.colorbar() cb.set_label(units) basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, long_name)) fig = plt.gcf() # plt.show() pngfile = "{0}.n.py.png".format(basename) fig.savefig(pngfile)
def run(FILE_NAME): # Identify the data field. DATAFIELD_NAME = "TbOceanRain" if USE_GDAL: import gdal GRID_NAME = "MonthlyRainTotal_GeoGrid" gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME, GRID_NAME, DATAFIELD_NAME) gdset = gdal.Open(gname) data = gdset.ReadAsArray().astype(np.float64) # Apply the attributes information. meta = gdset.GetMetadata() # Construct the grid. The projection is GEO, so this immediately # gives us latitude and longitude. x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform() nx, ny = (gdset.RasterXSize, gdset.RasterYSize) x = np.linspace(x0, x0 + xinc * nx, nx) y = np.linspace(y0, y0 + yinc * ny, ny) longitude, latitude = np.meshgrid(x, y) del gdset else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:, :].astype(np.float64) # Read global attribute. fattrs = hdf.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] # Construct the grid. The needed information is in a global attribute # called 'StructMetadata.0'. Use regular expressions to tease out the # extents of the grid. In addition, the grid is in packed decimal # degrees, so we need to normalize to degrees. ul_regex = re.compile( r"""UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)""", re.VERBOSE, ) match = ul_regex.search(gridmeta) x0 = np.float(match.group("upper_left_x")) / 1e6 y0 = np.float(match.group("upper_left_y")) / 1e6 lr_regex = re.compile( r"""LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)""", re.VERBOSE, ) match = lr_regex.search(gridmeta) x1 = np.float(match.group("lower_right_x")) / 1e6 y1 = np.float(match.group("lower_right_y")) / 1e6 ny, nx = data.shape x = np.linspace(x0, x1, nx) y = np.linspace(y0, y1, ny) longitude, latitude = np.meshgrid(x, y) # Apply the attributes information. data[data == -1] = np.nan data = np.ma.masked_array(data, np.isnan(data)) long_name = DATAFIELD_NAME units = "mm" m = Basemap(projection="cyl", resolution="l", lon_0=0, llcrnrlat=-90, urcrnrlat=90, llcrnrlon=-180, urcrnrlon=180) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(-90, 91, 45), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-180, 181, 45), labels=[0, 0, 0, 1]) m.pcolormesh(longitude, latitude, data, latlon=True) cb = m.colorbar() cb.set_label(units) basename = os.path.basename(FILE_NAME) plt.title("{0}\n{1}".format(basename, long_name)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def run(FILE_NAME): DATAFIELD_NAME = 'NDVI_TOA' if USE_GDAL: # Gdal import gdal GRID_NAME = 'WELD_GRID' gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME, GRID_NAME, DATAFIELD_NAME) # Scale down the data by a factor of 5 so that low-memory machines # can handle it. gdset = gdal.Open(gname) data = gdset.ReadAsArray().astype(np.float64)[::5, ::5] # Get any needed attributes. meta = gdset.GetMetadata() scale = np.float(meta['scale_factor']) fillvalue = np.float(meta['_FillValue']) valid_range = [np.float(x) for x in meta['valid_range'].split(', ')] units = meta['units'] # Construct the grid. x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform() ny, nx = (gdset.RasterYSize / 5, gdset.RasterXSize / 5) x = np.linspace(x0, x0 + xinc*5*nx, nx) y = np.linspace(y0, y0 + yinc*5*ny, ny) xv, yv = np.meshgrid(x, y) del gdset else: if USE_NETCDF: from netCDF4 import Dataset # Scale down the data by a factor of 5 so that low-memory machines # can handle it. nc = Dataset(FILE_NAME) ncvar = nc.variables[DATAFIELD_NAME] ncvar.set_auto_maskandscale(False) data = ncvar[::5, ::5].astype(np.float64) # Get any needed attributes. scale = ncvar.scale_factor fillvalue = ncvar._FillValue valid_range = ncvar.valid_range units = ncvar.units gridmeta = getattr(nc, 'StructMetadata.0') else: from pyhdf.SD import SD, SDC hdf = SD(FILE_NAME, SDC.READ) # Read dataset. data2D = hdf.select(DATAFIELD_NAME) data = data2D[:,:].astype(np.double) # Scale down the data by a factor of 6 so that low-memory machines # can handle it. data = data[::6, ::6] # Read attributes. attrs = data2D.attributes(full=1) vra=attrs["valid_range"] valid_range = vra[0] fva=attrs["_FillValue"] fillvalue = fva[0] sfa=attrs["scale_factor"] scale = sfa[0] ua=attrs["units"] units = ua[0] fattrs = hdf.attributes(full=1) ga = fattrs["StructMetadata.0"] gridmeta = ga[0] # Construct the grid. The needed information is in a global attribute # called 'StructMetadata.0'. Use regular expressions to tease out the # extents of the grid. ul_regex = re.compile(r'''UpperLeftPointMtrs=\( (?P<upper_left_x>[+-]?\d+\.\d+) , (?P<upper_left_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = ul_regex.search(gridmeta) x0 = np.float(match.group('upper_left_x')) y0 = np.float(match.group('upper_left_y')) lr_regex = re.compile(r'''LowerRightMtrs=\( (?P<lower_right_x>[+-]?\d+\.\d+) , (?P<lower_right_y>[+-]?\d+\.\d+) \)''', re.VERBOSE) match = lr_regex.search(gridmeta) x1 = np.float(match.group('lower_right_x')) y1 = np.float(match.group('lower_right_y')) ny, nx = data.shape x = np.linspace(x0, x1, nx) y = np.linspace(y0, y1, ny) xv, yv = np.meshgrid(x, y) # Apply the attributes to the data. invalid = np.logical_or(data < valid_range[0], data > valid_range[1]) invalid = np.logical_or(invalid, data == fillvalue) data[invalid] = np.nan data = data * scale data = np.ma.masked_array(data, np.isnan(data)) # Convert the grid back to lat/lon. The 1st and 2nd standard parallels, # the center meridian, and the latitude of projected origin are in the # projection parameters contained in the "StructMetadata.0" global # attribute. The following regular expression could have been used to # retrieve them. # # Ref: HDF-EOS Library User's Guide for the EOSDIS Evolution and # Development (EED) Contract, Volume 2, Revision 02: Function # Reference Guide, pages 1-6 through 1-13. # # aea_regex = re.compile(r'''Projection=GCTP_ALBERS\s+ProjParams= # \(\d+,\d+ # (?P<stdpr1>[-]?\d+), # (?P<stdpr2>[-]?\d+), # (?P<centmer>[-]?\d+), # (?P<origlat>[-]?\d+) # ,0{7}\)''', re.VERBOSE) # aea = pyproj.Proj("+proj=aea +lat_1=29.5 +lat2=45.5 +lon_0=-96 +lat_0=23") wgs84 = pyproj.Proj("+init=EPSG:4326") lon, lat= pyproj.transform(aea, wgs84, xv, yv) m = Basemap(projection='aea', resolution='i', lat_1=29.5, lat_2=45.5, lon_0=-96, lat_0=23, llcrnrlat=37.5, urcrnrlat = 42.5, llcrnrlon=-127.5, urcrnrlon = -122.5) m.drawcoastlines(linewidth=0.5) m.drawparallels(np.arange(35, 45, 1), labels=[1, 0, 0, 0]) m.drawmeridians(np.arange(-130, -120, 1), labels=[0, 0, 0, 1]) m.pcolormesh(lon, lat, data, latlon=True) cb = m.colorbar() cb.set_label(units) long_name = DATAFIELD_NAME basename = os.path.basename(FILE_NAME) plt.title('{0}\n{1}'.format(basename, long_name)) fig = plt.gcf() # plt.show() pngfile = "{0}.py.png".format(basename) fig.savefig(pngfile)
def modis_sst(file_name, limit=None, flagLevel=None, param='sst'): #Setup flag tables for L2 data flagTable=np.zeros(32,dtype=bool) if flagLevel == 0 : flagTable[[1]]=True elif flagLevel == 1 :flagTable[[ 1, 3, 5, 15,16, 19, 25,26]]=True elif flagLevel == 2 : flagTable[[0,1, 3,4,5,8,9,10, 14,15,16, 19,21,22,23,25,26]]=True elif flagLevel >= 3 : flagTable[[0,1, 3,4,5,8,9,10, 14,15,16, 19,21,22,23,25,26]]=True flags=np.where(flagTable)[0] #Read MODIS HDF4 data f = SD(file_name, SDC.READ) fattr=f.attributes() #Get dimensions nScans=fattr['Number of Scan Lines'] sCtl=fattr['Number of Scan Control Points'] pCtl=fattr['Number of Pixel Control Points'] nPix=fattr['Pixels per Scan Line'] #Load coordinates #UPDATE THIS SECTION TO ALLOW CLEAN LOAD OF THE IMAGE... lonsel=f.select('longitude') latsel=f.select('latitude') # # pCtl_ind=np.arange(pCtl,dtype=np.float32) # p_ind=(pCtl-1)*np.arange(nPix,dtype=np.float32)/(nPix-1) # # sCtl_ind=np.arange(sCtl,dtype=np.float32) # s_ind=(sCtl-1)*np.arange(nScans,dtype=np.float32)/(nScans-1) # dum=interp2d2d(sCtl_ind, pCtl_ind, lonsel.get(), s_ind, p_ind) # dumlon=interp1d(p_ind, lonsel.get(), pCtl_ind, spline=True) # dumlat=interp1d(p_ind, lonsel.get(), pCtl_ind, spline=True) # shlon=shlat=lon.dimensions().values() #Shrink image ############# #scene dimensions info_sst=f.datasets()[param] dnames=info_sst[0] d=info_sst[1] lonvec=lonsel.get().reshape(d[0]*d[1]) latvec=latsel.get().reshape(d[0]*d[1]) #Get points within domain if limit is not None : indvec,flagvec=in_limits(lonvec, latvec, limit) flagmat=flagvec.reshape(d[0],d[1]) rowsum=np.sum(flagmat, 0) colsum=np.sum(flagmat, 1) yflag=rowsum >= 1 xflag=colsum >= 1 xid=np.arange(d[0]) xid=xid.compress(xflag) xcnt=np.int(xid.size) yid=np.arange(d[1]) yid=yid.compress(yflag) ycnt=np.int(yid.size) if xcnt == 0: raise Exception('Error : no longitude within limits') if ycnt == 0: raise Exception('Error : no latitude within limits') #Get start points xst=np.int(xid.min()) yst=np.int(yid.min()) #Shrink lon & lat lon_var=lonsel.get(start=[xst,yst], count=[xcnt,ycnt]) lat_var=latsel.get(start=[xst,yst], count=[xcnt,ycnt]) #Load SST image ############### sst=f.select(param) attr=sst.attributes() slope=attr['slope'] intercept=attr['intercept'] flagValue=attr['bad_value_scaled'] sst_var=sst.get(start=[xst,yst], count=[xcnt,ycnt]) #Shrink sst image #Compute mask if (param == 'sst') or (param == 'sst4') : fg=f.select('qual_'+param) fg_var=fg.get(start=[xst,yst], count=[xcnt,ycnt]) if flagLevel is None : mask=sst_var == flagValue else : mask= (sst_var == flagValue) | (fg_var >= flagLevel) elif param == 'chlor_a' : fg=f.select('l2_flags') fg_var=fg.get(start=[xst,yst], count=[xcnt,ycnt]).flatten() # dumvar=[False]*(xcnt*ycnt)#np.zeros((xcnt,ycnt,32),dtype=str).reshape((xcnt*ycnt*32)) dumfg=[[np.int(b) for b in np.binary_repr(f,32)[::-1]] for f in fg_var] #Rq : bits should be read from end to start dumvar=np.sum(np.array(dumfg)[:,flags],1) >= 1 # for i,f in enumerate(fg_var) : # dumvar[i] = (np.array([b for b in np.binary_repr(f,32)])[flags] == '1').any() mask=np.reshape(dumvar,(xcnt,ycnt)) #Check flags # plt.bar(np.arange(1,33),np.sum(dumfg,0)[::-1]/np.float64(xcnt*ycnt)); plt.show() sst_var=np.ma.masked_array(sst_var*slope + intercept, mask=mask,type=float) #Image reprojection to avoid bow tie effect # import pyresample as pr # swath_def=pr.geometry.SwathDefinition(lons=lon_var, lats=lat_var) # lons,lats=np.meshgrid(np.arange(lon_var.min(),lon_var.max(),0.005), np.arange( lat_var.min(),lat_var.max(),0.005)) # grid_def = pr.geometry.GridDefinition(lons=lons, lats=lats) # # area_id = 'NWMEd' # area_name = 'NWMed' # proj_id = 'cyl' # proj4_args = '+proj=eqc +lat_ts=0 +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +a=6378137 +b=6378137 +units=m' # x_size = 601 # y_size = 351 # area_extent = (5., 41., 11., 44.5) # area_def = pr.utils.get_area_def(area_id, area_name, proj_id, proj4_args, x_size, y_size, area_extent ) # res = pr.kd_tree.resample_gauss(swath_def, sst_var.data, grid_def, radius_of_influence=10000.,sigmas=500.,fill_value=None) #Quality control # sstfg=f.select('qual_sst') # l2fg=f.select('l2_flags') #Return output dictionary return {'lon':lon_var,'lat':lat_var,'sst':sst_var}