def parse_hdf(filename):
hdf_file = '{path}/data/{filename}'.format(
path=sys.path[0],
filename=filename
)
file = SD(hdf_file)
print log_string, 'file info: ', file.info()
datasets_dict = file.datasets()
print log_string, '数据集：'
for idx, sds in enumerate(datasets_dict):
sds_obj = file.select(sds)
data = sds_obj.get()
data_attr = sds_obj.attributes()
availabe_dict[sds] = data
print log_string, idx, sds, ' :', data.shape
file.end()
# http://hdfeos.org/zoo/index_openLAADS_Examples.php
import os
import matplotlib as mpl
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
import numpy as np
from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER
FILE_NAME = 'MOD08_D3.A2010001.006.2015041224130.hdf'
DATAFIELD_NAME = 'Cloud_Fraction_Mean'
from pyhdf.SD import SD, SDC
hdf = SD(FILE_NAME, SDC.READ)
print(hdf.info())
datasets_dic = hdf.datasets()
for idx,sds in enumerate(datasets_dic.keys()):
print(idx,sds)
# Read dataset.
data_raw = hdf.select(DATAFIELD_NAME)
data = data_raw[:,:].astype(np.double)
# Read lat/lon.
xdim = hdf.select('XDim')
lon = xdim[:].astype(np.double)
ydim = hdf.select('YDim')
#!/usr/bin/env python
# This is an example of how the pyhdf package can be used to access MODIS data
from pyhdf.SD import SD, SDC
import numpy as np
from hdf_functions import scaled_data, print_description
file_name = '../assets/example_data/MYD05_L2.A2017109.0000.006.NRT.hdf'
file = SD(file_name, SDC.READ)
print('Data size: ' + str(file.info()[0]) + ', ' + str(file.info()[1]))
datasets = file.datasets()
# Print data contents
print('\nData content:')
for key in datasets.keys():
print('\n* ' + key)
sds_obj = file.select(key)
print_description(sds_obj)
data = scaled_data(sds_obj)
print(data)
def load_modis():
dataPath = "/home/mrilee/data/MODIS/"
# MOD03.A2005349.2120.061.2017187042837.hdf MOD05_L2.A2005349.2120.061.2017294065852.hdf
# MOD03.A2005349.2125.061.2017187042720.hdf MOD05_L2.A2005349.2125.061.2017294065400.hdf
modis_base = "MOD05_L2."
# 1 modis_item = "A2005349.2120.061.2017294065852"
# 1 modis_time_start = "2005-12-15T21:20:00"
modis_item = "A2005349.2125.061.2017294065400"
modis_time_start = "2005-12-15T21:25:00"
modis_geofilename = "MOD03.A2005349.2125.061.2017187042720.hdf"
modis_suffix = ".hdf"
modis_filename = modis_base + modis_item + modis_suffix
fmt_suffix = ".h5"
workFileName = "sketchG." + modis_base + modis_item + fmt_suffix
key_across = 'Cell_Across_Swath_1km:mod05'
key_along = 'Cell_Along_Swath_1km:mod05'
print('loading: ', dataPath + modis_filename)
hdf = SD(dataPath + modis_filename, SDC.READ)
ds_wv_nir = hdf.select('Water_Vapor_Near_Infrared')
data = ds_wv_nir.get()
# MODIS_Swath_Type_GEO/Geolocation_Fields/
# Latitude
hdf_geo = SD(dataPath + modis_geofilename, SDC.READ)
print('hg info: ', hdf_geo.info())
# for idx,sds in enumerate(hdf_geo.datasets().keys()):
# print(idx,sds)
# hdf_geo_ds = hdf_geo.select['']
# hdf_geo_swath = hdf_geo.select('MODIS_Swath_Type_GEO')
# hdf_geo_swath_gf = hdf_geo_swath['Geolocation_Fields']
hdf_geo_lat = hdf_geo.select('Latitude').get()
hdf_geo_lon = hdf_geo.select('Longitude').get()
print('hgl type ', type(hdf_geo_lat))
print('hgl shape ', hdf_geo_lat.shape, hdf_geo_lon.shape)
print('hgl dtype ', hdf_geo_lat.dtype)
# exit()
add_offset = ds_wv_nir.attributes()['add_offset']
scale_factor = ds_wv_nir.attributes()['scale_factor']
print('scale_factor = %f, add_offset = %f.' % (scale_factor, add_offset))
data = (data - add_offset) * scale_factor
print('data mnmx: ', np.amin(data), np.amax(data))
nAlong = ds_wv_nir.dimensions()[key_along]
nAcross = ds_wv_nir.dimensions()[key_across]
print('ds_wv_nir nAlong,nAcross: ', nAlong, nAcross)
dt = np.array([modis_time_start], dtype='datetime64[ms]')
t_resolution = 26 # 5 minutes resolution? 2+6+10+6+6
tid = ps.from_utc(dt.astype(np.int64), t_resolution)
# print(np.arange(np.datetime64("2005-12-15T21:20:00"),np.datetime64("2005-12-15T21:25:00")))
# exit()
fill_value = ds_wv_nir.attributes()['_FillValue']
mod_lat = hdf_geo_lat.flatten()
mod_lon = hdf_geo_lon.flatten()
mod_dat = data.flatten()
return mod_lon, mod_lat, mod_dat # load_modis
#### File paths
file_name = "C:\\Users\\wb580236\\OneDrive - WBG\\Documents\\Python Scripts\\MCD19A2.A2021137.h18v08.006.2021139045000.hdf"
lat_lon_file = ""
##### Get h and v numbers from file name
h = file_name.split('\\')[6].split('.')[2][1:3]
v = file_name.split('\\')[6].split('.')[2][5:7]
##### Get origin Lat/Lon from the lat lon file
f = open(lat_lon_file, 'r')
for line in f.readlines():
if line[0:2] == h:
i = int(v) * 3
lat_0 = line[i + 1:i + 4]
lon_0 = 0
file = SD(file_name, SDC.READ)
print(file.info())
pdb.set_trace()
#### To create numpy array of lat/lon values
lat_lon = np.linspace(lat_0, lat_0 + 1.2, 1200)
datasets_dic = file.datasets()
#### Unwrap the data
for idx, sds in enumerate(datasets_dic.keys()):
print(idx, sds)
sds_obj = file.select('Optical_Depth_055')
data = sds_obj.get()
for key, value in sds_obj.attributes().items():
print(key, value)
if key == 'add_offset':
add_offset = value
if key == 'scale_factor':
fmt_suffix = ".h5"
workFileName = "sketchG." + modis_base + modis_item + fmt_suffix
key_across = 'Cell_Across_Swath_1km:mod05'
key_along = 'Cell_Along_Swath_1km:mod05'
hdf = SD(dataPath + modis_filename, SDC.READ)
ds_wv_nir = hdf.select('Water_Vapor_Near_Infrared')
data = ds_wv_nir.get()
# MODIS_Swath_Type_GEO/Geolocation_Fields/
# Latitude
hdf_geo = SD(dataPath + modis_geofilename, SDC.READ)
print('hg info: ', hdf_geo.info())
for idx, sds in enumerate(hdf_geo.datasets().keys()):
print(idx, sds)
# hdf_geo_ds = hdf_geo.select['']
# hdf_geo_swath = hdf_geo.select('MODIS_Swath_Type_GEO')
# hdf_geo_swath_gf = hdf_geo_swath['Geolocation_Fields']
hdf_geo_lat = hdf_geo.select('Latitude').get()
hdf_geo_lon = hdf_geo.select('Longitude').get()
print('hgl type ', type(hdf_geo_lat))
print('hgl shape ', hdf_geo_lat.shape, hdf_geo_lon.shape)
print('hgl dtype ', hdf_geo_lat.dtype)
# exit()
add_offset = ds_wv_nir.attributes()['add_offset']
scale_factor = ds_wv_nir.attributes()['scale_factor']
Latitude = vs.attach('Latitude')
Longitude = vs.attach('Longitude')
a = Latitude[:]
Latitude.detach()
Longitude.detach()
vs.end()
f.close()
#----------------------------------------------------------------------------------------#
# SDS Data
file = SD(file_path + file_name, SDC.READ)
file_info = file.info()
print(file_info) # number of sds and metadata
print('---------- CloudLayerBase ----------')
sds_obj = file.select('CloudLayerBase') # select sds
CloudLayerBase = sds_obj.get()
sds_info = sds_obj.info()
print(CloudLayerBase.shape)
print(sds_info)
print(sds_info[0], sds_info[1])
print('sds attributes')
pprint.pprint(sds_obj.attributes())
def plot_sd(SD):
# TODO:
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from matplotlib import ticker
name = SD.info()[0]
rank = SD.info()[1]
dims = SD.info()[2]
fill = SD._FillValue
scale = SD.scale_factor
offset = SD.add_offset
# unit = SD.units
data = SD.get()
# print data.shape, rank
scl_data = data * scale + offset
vrng = np.array(SD.valid_range) * scale + offset
if rank == 3:
nc = 3 # number of columns in figure
nr = dims[0] / nc + 1 if dims[0] % nc > 0 else dims[0] / nc
fig = plt.figure(figsize=(12, nr * 3))
gs = gridspec.GridSpec(nr, nc)
for i in range(0, dims[0]):
if data[i, :, :].max() == fill: continue
ax = fig.add_subplot(gs[i])
frame = plt.gca()
frame.axes.get_xaxis().set_ticks([])
frame.axes.get_yaxis().set_ticks([])
ax.set_title('{0}:{1}'.format(name, i + 1), fontsize=10)
mdata = np.ma.masked_where(data[i, :, :] == fill,
scl_data[i, :, :])
# print 'Reading array', i
# print mdata.min(), mdata.max(), mdata.ptp()
vrng = [mdata.min(), mdata.max()]
plt.imshow(mdata.T,
vmin=vrng[0],
vmax=vrng[1],
cmap=plt.cm.Spectral_r) # @UndefinedVariable
cb = plt.colorbar(orientation='vertical',
fraction=.03,
format='%0.2f')
tick_locator = ticker.MaxNLocator(nbins=6)
cb.locator = tick_locator
cb.update_ticks()
elif rank == 2:
if data.max() == fill:
# No valid sds in the selected field, skip further processing..
raise SystemExit('** W: No valid data in {0} for '\
'plotting **\n'.format(name))
else:
# Create masked array of the selected field filtering out fill_values
mdata = np.ma.masked_where(data == fill, scl_data)
fig = plt.figure(figsize=(4, 3))
ax = fig.add_subplot(111)
ax.set_title(name)
plt.imshow(mdata.T,
vmin=mdata.min(),
vmax=mdata.max(),
cmap=plt.cm.Spectral_r) # @UndefinedVariable
ax.set_aspect('equal')
plt.colorbar(orientation='vertical', fraction=.03)
fig.tight_layout()
plt.show()
from pyhdf.SD import SD, SDC
import os
import pprint
import arcpy
import re
import glob
import sys
import time
##############################################
#### Check out structure of HDf files
os.chdir('F:\Hexsim\Data\MODIS_LST_download')
file_name = 'MOD11A1.A2000065.h09v04.005.2007176143322.hdf'
file = SD(file_name, SDC.READ)
print file.info()
datasets_dic = file.datasets()
#Check datasets in HDF
for idx, sds in enumerate(datasets_dic.keys()):
print idx, sds
LST_obj = file.select('LST_Day_1km') #Select dataset of interest
data = LST_obj.get() #Get sds data
print(data) #Check content
#Check data attributes
pprint.pprint(LST_obj.attributes())
#Correct for scale factor
from __future__ import division
from pyhdf.SD import SD, SDC
import numpy as np
import matplotlib.pyplot as plt
import pprint
if __name__ == '__main__':
hdf_name = 'datas/MCD43A3C.A2019227.h27v05.006.2019236003508.hdf'
hdf_obj = SD(hdf_name)
print(hdf_obj.info())
data_dic = hdf_obj.datasets()
for idx, sds in enumerate(data_dic.keys()):
print(idx, sds)
bsa = hdf_obj.select('Albedo_WSA_Band1')
data = bsa.get()
data = np.transpose(data)
uee = [[82, 18, 25, 32, 25, 14, 25], [85, 18, 25, 32, 25, 14, 25],
[90, 18, 25, 32, 25, 14, 25], [100, 18, 25, 32, 25, 14, 25],
[125, 18, 25, 32, 0, 14, 25], [150, 18, 25, 32, 25, 14, 25],
[255, 18, 25, 32, 25, 14, 25]]
f_data = np.asfarray(data * 255 / 32767)
print(f_data)
plt.imshow(f_data, cmap='Blues')
plt.show()
os.chdir(newdir)
try:
yield
finally:
os.chdir(prevdir)
if __name__ == "__main__":
work_dir = Path(sys.argv[1]).resolve()
if not work_dir.is_dir():
raise ValueError(f"{str(work_dir)} doesn't exist")
print(f"cd to {work_dir}")
with cd(work_dir):
all_files = list(work_dir.glob("*hdf"))
print(f"found {all_files}")
out_dir = Path()/"out_dir"
out_dir.mkdir(parents=True, exist_ok=True)
for a_file in all_files:
print(f"reading {a_file}")
str_file = str(a_file)
the_sd = SD(str_file, SDC.READ)
print(the_sd.info())
datasets_dict = the_sd.datasets()
for idx, sds in enumerate(datasets_dict.keys()):
print(idx, sds)
out=the_sd.attributes()
print(list(out.keys()))
vdata_binsize.detach() # "close" the vdata
vdata_nclouds.detach() # "close" the vdata
vdata_nbins.detach() # "close" the vdata
vdata_nray.detach() # "close" the vdata
vdata_quality.detach() # "close" the vdata
vdata_layers.detach()
vs.end() # terminate the vdata interface
f.close()
#----------------------------------------------------------------------------------------#
# Get SDS info
file = SD(file_path+file_name, SDC.READ)
pprint.pprint( file.info() ) # number of sds and metadata
#----------------------------------------------------------------------------------------#
# get all sds names
datasets_dic = file.datasets()
#for idx,sds in enumerate(datasets_dic.keys()):
# print idx,sds
sds_dic = {}
for key, value in datasets_dic.items():
#print key, value, value[3]
sds_dic[value[3]] = key
pprint.pprint( sds_dic )
from pyhdf.SD import SD
import matplotlib.pyplot as plt
import numpy as np
import pprint
hdf = SD(
'/Users/didi/Documents/whvixd/personal/studyAI/dataset/modis/MOD17A2H/MOD17A2H.A2021313.h21v03.006.2021322084625.hdf'
)
print(hdf.info()) # 信息类别数
data = hdf.datasets()
# for idx, sds in enumerate(data.keys()):
# print(idx, sds)
# 数据获取
# Gpp_500m = hdf.select('Gpp_500m').get()
# print(Gpp_500m.shape)
# pprint.pprint(Gpp_500m.attributes())
# Lon = hdf.select('longitude')[:]
# print(Lon)
# 数据获取
# Map = hdf.select('cloud_fraction')
# data = Map.get()
# data = np.transpose(data) # 将数据进行转置
# 变量信息读取与输出
# pprint.pprint(Map.attributes())
# 属性
