def hdf2txt(hdfFile, txtFile):
try:
hdf = SD(hdfFile) # open the HDF file
attr = hdf.attributes() # get global attribute dictionary
dsets = hdf.datasets() # get dataset dictionary
except HDF4Error, msg:
print "HDF4Error", msg
def __init__(self, filename):
hdfFile = SD(filename)
tmp = np.array (hdfFile.select(1)[:], dtype = np.int16)
emp = hdfFile.select(1).getfillvalue()
hdfFile.end()
max = tmp.max()
print "signal max : ", max
tmp[tmp == emp] = max
min = tmp.min()
print "signal min : ", min
rng = max - min
self.image = self.rawimage = np.array((tmp-min)*1.0/rng*255, dtype=np.uint8)
def txtToHDF(txtFile, hdfFile, varName, attr):
"""Inputs:
txtFile = name of .txt file (passed as string)
hdfFile = name of .hdf file (passed as string)
varName = name of dataset to be added (passed as string)
attr = dataset attributes (passed as dictionary)
txtFile indicates a dataset, and varName and attr give information
about it. txtToHDF puts this information into an SD (Scientific
Dataset) object and stores that object as in hdfFile, creating
hdfFile if need be, otherwise updating it.
"""
# Catch pyhdf errors
try:
# Open HDF file in update mode, creating it if non existent.
d = SD(hdfFile, SDC.WRITE|SDC.CREATE)
# Open text file and get matrix dimensions on first line.
txt = open(txtFile)
ni, nj = map(int, txt.readline().split())
# Define HDF dataset of type SDC.FLOAT32 with those dimensions.
v = d.create(varName, SDC.FLOAT32, (ni, nj))
# Assign attributes passed as argument inside dict `attr'.
for attrName in attr.keys():
setattr(v, attrName, attr[attrName])
# Load variable with lines of data. Compute min and max
# over the whole matrix.
i = 0
while i < ni:
elems = map(float, txt.readline().split())
v[i] = elems
minE = min(elems)
maxE = max(elems)
if i:
minVal = min(minVal, minE)
maxVal = max(maxVal, maxE)
else:
minVal = minE
maxVal = maxE
i += 1
# Set variable min and max attributes.
v.minVal = minVal
v.maxVal = maxVal
# Close dataset and file objects (not really necessary,
# since closing is automatic when objects go out of scope.
v.endaccess()
d.end()
txt.close()
except HDF4Error, msg:
print "HDF4Error:", msg
def main():
# open the hdf file for reading
hdf=SD.SD(FILE_NAME)
# read the sds data
sds=hdf.select(SDS_NAME)
data=sds.get()
# turn [y,x] (HDF representation) data into [x,y] (numpy one)
data=data.reshape(data.shape[1],data.shape[0])
# print out the data
msg_out=""
for i in range(X_LENGTH):
for j in range(Y_LENGTH):
msg_out+=str(data[i,j])+" "
msg_out+="\n"
print msg_out
def geo_intersects(filename, lonlat):
"""
Checks a geolocation file for overlap with a latitude longitude box
:filename: name of file to check
:lonlat: list, [leftlon, rightlon, botlat, toplat]
:return: boolean, true if there was overlap
"""
logging.info("Checking %s for intersection with given lonlat" % filename)
if filename[-4:] != '.hdf':
logging.info("ERROR: %s is not an hdf file" % filename)
return False
try:
hdf = SD.SD(filename)
except:
logging.info("ERROR: failed to load file: %s" % filename)
return False
lon = hdf.select('Longitude')
lat = hdf.select('Latitude')
dim1 = len(lon[:])
dim2 = len(lon[0])
minlon = float(lon[0][0])
maxlon = float(lon[dim1 - 1][dim2 - 1])
minlat = float(lat[dim1 - 1][dim2 - 1])
maxlat = float(lat[0][0])
if minlon > maxlon:
logging.info(
"File %s crosses dateline (not currently supported), skipping..." %
filename)
return False
lonoverlap = minlon < lonlat[1] and maxlon > lonlat[0]
latoverlap = minlat < lonlat[3] and maxlat > lonlat[2]
intersects = lonoverlap and latoverlap
if intersects:
logging.info("File %s intersects given lonlat")
else:
logging.info("File %s does not intersect given lonlat")
return intersects
def get_l2hdf_slope_intercept(ifile, prod):
#---------------------------------------------------------------
slope_inter= np.asarray([1.0, 0.0])
ftype= hdf_cdf_version(ifile)
if ftype == 'hdf4':
f= SD(ifile)
d1 = f.select(prod)
d1Attr= d1.attributes()
attNames= d1Attr.keys()
attNames.sort()
for nm in attNames:
if nm == 'slope': slope_inter[0]= float(d1Attr[nm])
if nm == 'intercept': slope_inter[1]= float(d1Attr[nm])
if nm == 'scale_factor': slope_inter[0]= float(d1Attr[nm])
if nm == 'add_offset': slope_inter[1]= float(d1Attr[nm])
d1.endaccess()
f.end()
return slope_inter
if ftype == 'hdf5':
f = Dataset(ifile, 'r')
group_names= f.groups.keys() #get group names (e.g. geophtsical_data_sets or navigation)
for grp_name in group_names:
var_name= f.groups[grp_name].variables.keys() #get names of objects within each group (e,g. chlor_a or longitude)
for vn in var_name:
if vn == prod:
p = f.groups[grp_name].variables[vn]
try: slope_inter= np.asarray([float(p.scale_factor), float(p.add_offset)])
except: print ( '\nDid not find slope intercept valules in l2 file. Using as default: slope = 1.0 and interecept = 0.0\n' )
f.close()
return slope_inter
def __init__(self, filename):
self.filename = os.path.basename(filename)
self.filepath = os.path.realpath(filename)
self._hdf_handle = SD.SD(self.filepath, SD.SDC.READ)
# CREFLM_npp_d20141103_t1758468_e1800112.hdf
fn = os.path.splitext(self.filename)[0]
parts = fn.split("_")
self.satellite = parts[1].lower()
self.instrument = "viirs"
# Parse out the datetime, making sure to add the microseconds and set the timezone to UTC
begin_us = int(parts[3][-1]) * 100000
self.begin_time = datetime.strptime(parts[2][1:] + parts[3][1:-1], "%Y%m%d%H%M%S").replace(microsecond=begin_us)
end_us = int(parts[4][-1]) * 100000
self.end_time = datetime.strptime(parts[2][1:] + parts[4][1:-1], "%Y%m%d%H%M%S").replace(microsecond=end_us)
if self.end_time < self.begin_time:
self.end_time += timedelta(days=1)
def __init__(self, filename):
hdfFile = SD(filename)
bo = re.compile('.*?=\s+([-]*\d*\.\d+).*',re.DOTALL)
coord_string = hdfFile.__getattr__('ArchiveMetadata.0')[hdfFile.__getattr__('ArchiveMetadata.0').find('NORTHBOUNDINGCOORDINATE')]
self.west = float(bo.match(coord_string.split('WESTBOUNDINGCOORDINATE')[1]).group(1))
self.east = float(bo.match(coord_string.split('EASTBOUNDINGCOORDINATE')[1]).group(1))
self.south = float(bo.match(coord_string.split('SOUTHBOUNDINGCOORDINATE')[1]).group(1))
self.north = float(bo.match(coord_string.split('NORTHBOUNDINGCOORDINATE')[1]).group(1))
tmp = np.array (hdfFile.select(1)[:], dtype = np.int16)
emp = hdfFile.getfillvalue()
tmp[tmp == emp] = 0
min = tmp.min()
rng = tmp.max() - min
self.image = np.array(float(tmp-min)/rng*255, dtype=np.uint8)
hdfFile.end()
def __init__(self, fname):
"""Initialize the HDF file"""
MXD35L2.__init__(self)
print("> Reading: " + fname)
name = fname
start = fname.find('D35_L2.A') + len('D35_L2.A')
end = start + len('YYYYDDD.HHMM')
datetimestamp = DT.datetime.strptime(fname[start:end], '%Y%j.%H%M')
hdf_file = SD.SD(fname)
lon = hdf_file.select('Longitude').get()
lat = hdf_file.select('Latitude').get()
cloud_mask = NP.uint8(hdf_file.select('Cloud_Mask').get()[0])
hdf_file.end()
cloud = cloud_mask & 6 # get bits 1 and 2
cloud[cloud == 0] = 1 # 00 = confident cloudy
cloud[cloud != 1] = 0
water = cloud_mask & 192 # get bits 6 and 7
water[water == 0] = 1 # 00 = water
water[water != 1] = 0
coast = cloud_mask & 192 # get bits 6 and 7
coast[coast == 64] = 1 # 01 = coastal
coast[coast != 1] = 0
lon, lat, cloud, water, coast = autoFlip(lon, 'horizontal', lon, lat,
cloud, water, coast)
lon, lat, cloud, water, coast = autoFlip(lat, 'vertical', lon, lat,
cloud, water, coast)
self.name = name
self.datetimestamp = datetimestamp
self.lon = lon
self.lat = lat
self.cloud = cloud
self.water = water
self.coast = coast
self.top = self.lat[0]
self.bot = self.lat[-1]
self.lef = self.lon[0]
self.ryt = self.lon[-1]
def get_variables(file_path):
all_vars = dict()
try:
f = SD.SD(file_path, SD.SDC.READ)
ds = f.datasets()
for ds_name in ds:
#open dataset
d = f.select(ds_name)
units = None if d.attributes().get('units') == 'none' else d.attributes().get('units')
details = "%s %s %s" %(" ".join(d.dimensions().keys()), str(d.attributes().get('long_name')), ds_name)
details = details.replace("\n", " ").replace("/", " ").replace(".", " ").replace(":", " ")
all_vars[sanitize(ds_name)] = dict(standard_name=ds_name, units=units, details=details)
d.endaccess()
f.end()
return all_vars
except Exception, e:
print e
# raise
return
def hdf2tiff(file):
hdf = SD(file) #打开HDF文件
attr = hdf.attributes(full=1) #HDF文件属性字典
attNames = attr.keys()
attNames.sort()
for name in attNames:
t = attr[name]
print name, t
dsets = hdf.datasets() #HDF数据字典
dsNames = dsets.keys()
dsNames.sort()
for name in dsNames:
ds = hdf.select(name)
vAttr = ds.attributes()
t = dsets[name]
print name, vAttr, t
im_data = hdf.select('precipitation').get() * 3
im_data = np.fliplr(im_data)
im_data = np.transpose(im_data)
np.putmask(im_data, im_data < 0, np.nan)
im_width = 1440
im_height = 400
im_bands = 1
datatype = gdal.GDT_UInt16
sr = osr.SpatialReference()
sr.ImportFromEPSG(4326)
im_proj = sr.ExportToWkt()
im_geotrans = (-180, 0.25, 0, 50, 0, -0.25)
driver = gdal.GetDriverByName("GTiff")
dataset = driver.Create(
os.path.splitext(file)[0] + ".TIF", im_width, im_height, im_bands,
datatype)
dataset.SetGeoTransform(im_geotrans)
dataset.SetProjection(im_proj)
dataset.GetRasterBand(1).WriteArray(im_data)
del dataset
hdf.end()
def to_array(fname):
# from HDF4 to HDF5 files
fin = sd.SD(fname) # HDF4 files
d = fin.datasets()
#orbit = fin.select('id').get()
secs00 = fin.select('time').get()
lat = fin.select('lat').get()
lon = fin.select('lon').get()
elev = fin.select('elev').get()
agc = fin.select('gain').get()
energy = fin.select('wave_energy').get()
reflect = fin.select('reflect').get()
icesvar = fin.select('icesvar').get()
#pressure = fin.select('pressure').get()
#sat = fin.select('sat_cor_product').get()
nrow, ncol = lon.shape[0], 9
orbit = np.empty(nrow, 'f8')
revNo = int(os.path.split(fname)[-1].split('_')[2])
orbit.fill(revNo)
secs00 += 43200 # move time 12 hours back (starting midnight)
lon[lon < 0] += 360. # from -180/180 to 0/360
fout = tb.openFile(os.path.splitext(fname)[0] + '.h5', 'w')
atom = tb.Float64Atom()
shape = (nrow, ncol)
filters = tb.Filters(complib='blosc', complevel=9)
c = fout.createCArray('/', 'data', atom=atom, shape=shape, filters=filters)
c[:, 0] = orbit
c[:, 1] = secs00
c[:, 2] = lat
c[:, 3] = lon
c[:, 4] = elev
c[:, 5] = agc
c[:, 6] = energy
c[:, 7] = reflect
c[:, 8] = icesvar
fout.close()
def read(filename, variables=None, datadict=None):
"""
Reads SD from a HDF4 file into a dictionary.
:param str filename: The name (with path) of the HDF file to read.
:param iterable names: A sequence of variable (dataset) names to read from the
file (default None, causing all variables to be read). The names must appear exactly as in in the HDF file.
:param dict datadict: Optional dictionary to add data to, otherwise a new, empty dictionary is created
:return: A dictionary containing data for requested variables. Missing data is replaced by NaN.
"""
# Optional HDF import
if not SD:
raise ImportError(
"HDF support was not installed, please reinstall with pyhdf to read HDF files."
)
# List of required variable names.
# Open the file.
datafile = None
try:
datafile = SD.SD(filename)
sd_variables = list(datafile.datasets().keys())
finally:
if datafile is not None:
datafile.end()
if variables is None:
requested_sd_variables = sd_variables
else:
requested_sd_variables = set(listify(variables)).intersection(
set(sd_variables))
# Create dictionary to hold data arrays for returning.
if datadict is None:
datadict = {}
# Get data.
for variable in requested_sd_variables:
datadict[variable] = HDF_SDS(filename, variable)
return datadict
def __init__(self, filename):
hdfFile = SD(filename)
bo = re.compile('.*?=\s+([-]*\d*\.\d+).*', re.DOTALL)
coord_string = hdfFile.__getattr__('ArchiveMetadata.0')[
hdfFile.__getattr__('ArchiveMetadata.0').find(
'NORTHBOUNDINGCOORDINATE')]
self.west = float(
bo.match(coord_string.split('WESTBOUNDINGCOORDINATE')[1]).group(1))
self.east = float(
bo.match(coord_string.split('EASTBOUNDINGCOORDINATE')[1]).group(1))
self.south = float(
bo.match(
coord_string.split('SOUTHBOUNDINGCOORDINATE')[1]).group(1))
self.north = float(
bo.match(
coord_string.split('NORTHBOUNDINGCOORDINATE')[1]).group(1))
tmp = np.array(hdfFile.select(1)[:], dtype=np.int16)
emp = hdfFile.getfillvalue()
tmp[tmp == emp] = 0
min = tmp.min()
rng = tmp.max() - min
self.image = np.array(float(tmp - min) / rng * 255, dtype=np.uint8)
hdfFile.end()
def rdhdf(hdf_filename):
if (hdf_filename.endswith('h5')):
x, y, z, f = rdh5(hdf_filename)
return (x, y, z, f)
x = np.array([])
y = np.array([])
z = np.array([])
f = np.array([])
# Open the HDF file
sd_id = h4.SD(hdf_filename)
#Read dataset. In all PSI hdf4 files, the
#data is stored in "Data-Set-2":
sds_id = sd_id.select('Data-Set-2')
f = sds_id.get()
#Get number of dimensions:
ndims = np.ndim(f)
#Get the scales:
for i in range(0, ndims):
dim = sds_id.dim(i)
if i == 0:
x = dim.getscale()
elif i == 1:
y = dim.getscale()
elif i == 2:
z = dim.getscale()
sd_id.end()
x = np.array(x)
y = np.array(y)
z = np.array(z)
f = np.array(f)
return (x, y, z, f)
def start_mapping(upload_result, url=False):
print url
if not url:
print "processing the uploaded file result"
filename = upload_result["name"]
file_path = os.path.join(os.path.abspath(UPLOAD_DIR), filename)
"Extracting starts here"
# open the hdf file for reading
hdf = SD.SD(file_path)
print hdf
# f = h5py.File(file_path, 'r')
# print f.keys()
# dataset = netCDF4.Dataset(file_path, mode='r')
# print dataset.title
return {"result": "this lib mapper working"}
else:
print "Processing the url provided"
url = upload_result
print url
def load_from_fileZ2D(self, file_name):
print(file_name)
hdf = SD.SD(file_name)
sds = hdf.select("fakeDim0")
self.th = sds.get()
sds = hdf.select("fakeDim1")
self.r = sds.get()
sds = hdf.select("Data-Set-2") #density
self.dd = (sds.get()).T
sds = hdf.select("Data-Set-3") #total energy
self.ee = (sds.get()).T
sds = hdf.select("Data-Set-4") #ur
self.ur = (sds.get()).T
sds = hdf.select("Data-Set-5") #uth
self.uth = (sds.get()).T
sds = hdf.select("Data-Set-6") #uph
self.uph = (sds.get()).T
def MODIS_file(filename,height_lim,zen_lim,t_bins):
from pyhdf import SD
import numpy as np
f=SD.SD(filename)
sds=f.select('cloud_top_temperature_1km')
temp=0.01*(sds.get()[:2030,:1350]+15000)#sampling on the same grid as latitude
sds=f.select('Cloud_Phase_Optical_Properties')
phase=sds.get()[:2030,:1350]
sds=f.select('Solar_Zenith_Day')
zenith=0.009999999776482582*np.repeat(np.repeat(sds.get(),5,axis=0),5,axis=1).reshape(2030,1350)
#consider getting rid of the fill values
sds=f.select('cloud_top_height_1km')
height=sds.get()[:2030,:1350]
sds=f.select('Latitude')
lat=np.repeat(np.repeat(sds.get(),5,axis=0),5,axis=1).reshape(2030,1350)
#the above code extracts the relevant datasets from each MODIS file
#filtering cloud by altitude
heights=np.where((height<height_lim)&(((zenith<zen_lim)&(zenith>-zen_lim)))&((lat<-45)&(lat>-65)))#less than 4000 meteters
#filtered by both positive and negative zenith angles
#sample the new fields based on these thresholds
temp=temp[heights]
phase=phase[heights]
zenith=zenith[heights]
height=height[heights]
lat=lat[heights]
ice_loc=np.where(phase==3)#location of pixels in the liquid and ice fields
liq_loc=np.where(phase==2)
hist_water=np.histogram(temp[liq_loc],bins=t_bins)[0]
hist_ice=np.histogram(temp[ice_loc],bins=t_bins)[0]
#histograms of the temperatures of ice and liquid pixel numbers
return hist_water,hist_ice,t_bins,np.histogram(zenith[ice_loc],bins=np.arange(-90,90,1))[0],np.histogram(zenith[liq_loc],bins=np.arange(-90,90,1))[0]
def read_ftir(FILE_NAME):
""" Read NDACC HDF file. File and variable names to be specified by user
when under name == main """
hdf = SD.SD(FILE_NAME)
# read attributes
lst = hdf.attributes()
var = lst['DATA_VARIABLES'].split(';')
data = []
#----------------------------------------
for v in var:
#--------------------------------
try:
sds = hdf.select(v)
data.append(sds.get())
except:
print('No SDS found: ' + v)
var.pop(v)
#--------------------------------
#----------------------------------------
return var, data
def process_files(self,
dirname,
out_file,
latitude_range,
longitude_range,
csv_separator=";",
delete_after=False):
file_index = 1
for file in os.listdir(dirname):
full_path = os.path.join(dirname, file)
if file.endswith('.hdf'):
print(f'Processing file... {file_index} (Path: {full_path})')
data = self._process_file(SD(full_path, SDC.READ),
latitude_range, longitude_range)
BaseModisExtractor.write_to_csv(out_file, csv_separator, data)
print(
f'Processing finished for file... {file_index} (Path: {full_path})'
)
if delete_after:
for file in os.listdir(dirname):
full_path = os.path.join(dirname, file)
if file.endswith('.hdf'):
os.remove(full_path)
print(f'All results saved to {out_file}')
def L2_VFM_Reading(fpath):
sd_obj = SD(fpath, SDC.READ)
Vt_obj = HDF.HDF(fpath).vstart()
m_data = Vt_obj.attach('metadata').read()[0]
Height = np.array(m_data[-1]) # 583高度对应实际海拔
Lats = sd_obj.select('Latitude').get()
Lons = sd_obj.select('Longitude').get()
L_route = np.concatenate([Lats.T, Lons.T]).T
target_rows = []
for location in L_route:
distance = LonLat_Distance(location, LZU_LatLon)
if distance < 50:
target_rows.append(True)
else:
target_rows.append(False)
VFM_basic = np.array(sd_obj.select('Feature_Classification_Flags').get())
VFM_basic = VFM_basic % 8
VFM_1 = np.reshape(VFM_basic[:, 0:165], (VFM_basic.shape[0] * 3, 55))
VFM_1 = np.repeat(VFM_1, 5, axis=0)
VFM_2 = np.reshape(VFM_basic[:, 165:1165], (VFM_basic.shape[0] * 5, 200))
VFM_2 = np.repeat(VFM_2, 3, axis=0)
VFM_3 = np.reshape(VFM_basic[:, 1165:5515], (VFM_basic.shape[0] * 15, 290))
VFM = np.concatenate((VFM_1, VFM_2, VFM_3), axis=1)
target_rows_VFM = np.repeat(target_rows, 15)
Rd_dic = {}
Rd_dic['VFM'] = VFM
Rd_dic_meta = {
'route': L_route,
'Lats': Lats,
'target rows': target_rows,
'Height': Height,
'target rows VFM': target_rows_VFM,
}
sd_obj.end()
HDF.HDF(fpath).close()
return Rd_dic, Rd_dic_meta
def look(self, path, mem_list, lp_list):
data = {}
for name in mem_list: # subdata sets type data
tag = lp_list[name][0]
ref = lp_list[name][1]
if tag == HC.DFTAG_NDG:
sd = SD(path)
sds = sd.select(sd.reftoindex(ref))
data[name] = np.float64(sds.get())
sds.endaccess()
sd.end()
elif tag == HC.DFTAG_VH: #vd type data
hdf = HDF(path)
vs = hdf.vstart()
vd = vs.attach(ref)
nrecs, intmode, fields, size, name = vd.inquire()
data[name] = np.full(nrecs, np.float64(vd.read()[0]))
vs.end()
hdf.close()
return data
def work(file):
sd = SD(file)
lat = np.array(sd.select("Latitude")[:])
lon = np.array(sd.select("Longitude")[:])
cf = np.array(sd.select("Cloud_Fraction")[:])
t = np.array(sd.select("Scan_Start_Time")[:]) + initial_time
dist = distance(lat, lon)
ind = np.where(dist == dist.min())
x = ind[0]
y = ind[1]
lat = lat[x-1:x+2, y-1:y+2]
lon = lon[x-1:x+2, y-1:y+2]
cf = cf[x-1:x+2, y-1:y+2]
t = t[x-1:x+2, y-1:y+2]
dist = dist[x-1:x+2, y-1:y+2]
d = dist / np.sum(dist)
cf = weighted_average(cf, d)
lat = weighted_average(lat, d)
lon = weighted_average(lon, d)
t = weighted_average(t, d)
return (cf, lat, lon, t)
for i in range(RAT.shape[1]):
Digitised[np.where(Digitised[:,2]==i),2] = digitizing(ROIS[6][np.where(Digitised[:,2]==i)],RAT[:,i])
for i in range(10000):
Digitised[i,3] = digitizing(ROIS[-2][i],BT[:,CN1,ST,CP,Digitised[i,2],Digitised[i,0],Digitised[i,1]])
test[i] = TT[ABCN,Digitised[i,3],Digitised[i,2],Digitised[i,0],Digitised[i,1]]
Texture = ROIS[-1]>=test
return Texture.astype(int)
AZM_Data = SD.SD('/Users/jesseloveridge/Documents/Summer Project/MISR_AM1_AZM_F01_01.hdf') #Thresholds from MISR dataset
VZT = AZM_Data.select('Viewing Zenith Angle Thresholds')[:]
SZAT = AZM_Data.select('Solar Zenith Angle Thresholds')[:]
RAT = AZM_Data.select('Relative Azimuth Thresholds')[:,:]
BT = AZM_Data.select('Brightness Thresholds')[:]
TT = AZM_Data.select('Texture Thresholds')[:]
os.chdir('/Volumes/Promise1/Jesse/MODIS_DATA/') #file lists requires this directory to be changed
tau_list,rad_list = file_lists()
#use a for loop here to provide indices for the file lists.
tau_file = tau_list[index]
rad_file = rad_list[index]
def get_hdf_data(fname):
hdf = SD.SD(fname)
info= hdf.datasets()
#Lets see what is inside
hdf.info()
data_sets=[]
for name in sorted(info.keys()):
if name[0:4]=="Data":
sds=hdf.select(name)
long_name=sds.attributes()["long_name"]
for i in range(len(sds.attributes()["long_name"])):
if long_name[i:i+2]=="AT":
junk=i-1
short_name=long_name[:junk]
data_sets.append([name,long_name,short_name])
#Get the time from the last long name
if long_name[junk+9:] != "********":
time=float(long_name[junk+9:])
else:
time=0.0
#Get the dimensions from the last data set
dims=len((sds.info()[2]))
#Get the coordinate system from the last data set
coord_sys=sds.attributes()["coordsys"]
if coord_sys=='spherical polar':
coord_sys='spol'
else:
print(("get_hdf_data: I don't understand coordinate system ",coord_sys))
exit()
#Now we know which of the datasets contain real data, we can extract all the data
alldat={}
alldat["Filename"]=fname
alldat["Coord_sys"]=coord_sys
alldat["Time"]=time
alldat["Data"]={}
alldat["Data_names"]=np.array(data_sets)[:,2]
alldat["N_data"]=len(data_sets)
alldat["Dims"]=dims
#Loop over all the data sets in the hdf file - name each of the resulting dictionaries with the short name
for i in range (len(data_sets)):
print((data_sets[i][2]))
sds=hdf.select(data_sets[i][0])
data = sds.get()
c1=info[data_sets[i][0]][0][0]
c2=info[data_sets[i][0]][0][1]
sds=hdf.select(c1)
x2=sds.get()
sds=hdf.select(c2)
x1=sds.get()
alldat[data_sets[i][2]]={}
alldat[data_sets[i][2]]=data
alldat["r_cent"]=x1
alldat["theta_cent"]=x2
#HDF files only give us the centre of the grid, python needs the inner edges as well.
r_edge=[]
r_ratio=(x1[2]-x1[1])/(x1[1]-x1[0])
dr=(x1[1]-x1[0])/(0.5*(1.0+r_ratio))
r_edge.append(x1[0]-0.5*dr)
print((r_edge[0],r_ratio))
for i in range(len(x1)-1):
r_edge.append(r_edge[-1]+dr)
dr=dr*r_ratio
theta_edge=[]
theta_ratio=(x2[2]-x2[1])/(x2[1]-x2[0])
dtheta=(x2[1]-x2[0])/(0.5*(1.0+theta_ratio))
theta_min=x2[0]-0.5*dtheta
if theta_min<0.0:
theta_min=0.0
theta_edge.append(theta_min)
print((x2[0]))
print((theta_edge[0],theta_ratio))
for i in range(len(x2)-1):
theta_edge.append(theta_edge[-1]+dtheta)
dtheta=dtheta*theta_ratio
if (theta_edge[-1]+(x2[-1]-theta_edge[-1])*2.0)>(np.pi/2.0):
x2[-1]=(theta_edge[-1]+(np.pi/2.0))/2.0
alldat["r_edge"]=r_edge
alldat["theta_edge"]=theta_edge
return(alldat)
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from pyhdf.SD import *
from bitstring import *
import sys
#np.set_printoptions(threshold='nan')
_filename = 'C:\Users\AKO NA LNG\Desktop\Esquivel Files\School Files 2\Special Problem\MODIS\MOD35\MOD35_L2.A2015220.0215.006.2015220134621.hdf'
mod35 = SD(_filename, SDC.READ)
sds_name = 'Cloud_Mask'
sds_index = mod35.select(sds_name)
sds_data = sds_index.get()
sds_data_0 = sds_data[0, :, :]
sds_data_0_bin = sds_data_0.astype(dtype=np.uint8,
order='K',
casting='unsafe',
subok=False,
copy=False)
for x in np.nditer(sds_data_0_bin, op_flags=['readwrite']):
x[...] = np.right_shift(x, 6)
fig = plt.figure()
ax = fig.add_subplot(111)
cmap = plt.cm.winter
bounds = [0, 1, 2, 3]
print " vgroup:", vg0._name, "tag,ref:", tag, ref
vg0.detach()
# Unhandled tag
else:
print "unhandled tag,ref",tag,ref
# Close vgroup
vg.detach()
# Open HDF file in readonly mode.
filename = sys.argv[1]
hdf = HDF(filename)
# Initialize the SD, V and VS interfaces on the file.
sd = SD(filename)
vs = hdf.vstart()
v = hdf.vgstart()
# Scan all vgroups in the file.
ref = -1
while 1:
try:
ref = v.getid(ref)
except HDF4Error,msg: # no more vgroup
break
describevg(ref)
# Terminate V, VS and SD interfaces.
v.end()
vs.end()
def mosaic(*arg,**args):
# This function will take files tranfered in *arg and will mosaic them together and produce a new file
# mosaic(file1,[file2,file3,...],endfile)
# If only file1 and endfile is given, the function will only produce a copy without any other modification
try:
log=args["log"]
except KeyError:
log=logMod.Log("",nolog=True)
if len(arg)>2:
lfile = arg[:-1] # This is the list of the NAME of the files to merge
newfilename = arg[-1] # This is the final file NAME
# Should eventually check if files exists and can be read ***IMPROVE***
lfHDF = [] # This is the list of the FILES to merge
latt = [] # This is the list of the ATTRIBUTE "StructMetadata.0" of the files
for fil in lfile:
try:
a=SD(fil,SDC.READ)
except TypeError:
a=SD(fil.encode('ascii','ignore'),SDC.READ)
lfHDF.append(a)
#print("hoho")
latt.append(atribute(lfHDF[-1].attributes()["StructMetadata.0"],fil,dsi=[0,]))
## Listing all the GRIDS that the new file will have
gridlist = [] # This is the list of GRIDS to include in the final file
for attOfF in latt:
# Should check if any grid ***IMPROVE***
gridlist += attOfF.listgridname()[1] # listgridname return a list of all the grids name
# remove double entry
gridlist = list(set(gridlist))
## Listing all the DATASETS that the new file will have
dslist = [] # This is the list of DATASETS to include in the final file
for attOfF in latt:
# Should check if any grid ***IMPROVE***
dslist = attOfF.orderedDS()
# remove double entry
# dslist = list(set(dslist))
## Validation of commoun information
###############################################################################
# Some informations have to be the same for each file or else no mosaic can #
# be made for exemple two files with not the same projection type can't be #
# merged together. ***IMPROVE*** Maybe in the future we could transform file #
# so that they have the same projection or some other thing. #
###############################################################################
# List of parameter to check to insure that they are the same
paramMustSim = ["Projection","ProjParams","SphereCode"]
# Dictionary that will keep all the informations about every file
paramMustSimDict = {}
for grid in gridlist:
# Verification of a grid
first = True # Variable that will enable the construction of the dict
paramdict = {} # Dictionary that keep the actual value that have to be the same
for attOfF in latt:
# Verification of a file
bigG = attOfF.getgridbyname(grid) # Getting all the attributes in the grid of a file
if bigG is not None:
# If the grid exists in that file
if first :
# If this is the first time that a file is check for that grid
first = False
for p in paramMustSim:
# Checking every parameters that must be the same
paramdict[p] = bigG.variable[p]
# Validation of same Dtype for each datafield
go = bigG.GROUP["DataField"].OBJECT
for r in go:
paramdict[go[r].variable["DataFieldName"]]=go[r].variable["DataType"]
else:
# If it's not the first time that a file is check for that grid
for p in paramMustSim:
# Checking every parameters that must be the same
if not paramdict[p]==bigG.variable[p]:
# Stop merging and return error ***IMPROVE***
# Maybe do only the things that can be done ***IMPROVE***
log.log('e',Nom,"Error dataset are not compatible")
# Validation of same Dtype for each datafield
go=bigG.GROUP["DataField"].OBJECT
for r in go:
if not paramdict[go[r].variable["DataFieldName"]]==go[r].variable["DataType"]:
# Stop merging and return error ***IMPROVE***
# Maybe do only the things that can be done ***IMPROVE***
log.log('e',Nom,"Error dataset are not compatible")
# Keep all this info for later it's going to be useful
paramMustSimDict[grid]=paramdict
## Determination of new informations
###############################################################################
# Some properties have to be calculated in order to merge. This section is #
# doing just that #
###############################################################################
gridResolX={} # Getting the RESOLUTION in the X direction for each grid
gridResolY={} # Getting the RESOLUTION in the Y direction for each grid
extremeup={} # Getting the UPPER coordinates for each grid
extremedown={} # Getting the LOWEST coordinates for each grid
extremeleft={} # Getting the LEFTMOST coordinates for each grid
extremeright={} # Getting the RIGHTMOST coordinates for each grid
gridDimX={} # Getting the DIMENSIONS of X direction for each grid
gridDimY={} # Getting the DIMENSIONS of Y direction for each grid
NoValueDS={} # Getting the fill value of each dataset
dtypeDS={} # Getting the DTYPE for each dataset
dstogrid={} # Knowing wich is the GRID for each dataset
filGridULC={} # Getting the upper left corner of each file for each grid
for grid in gridlist:
# For each grid
filGridULC[grid]={} # Adding a dictionary for each grid that will contain information on every file
for attOfF in latt:
### Determination of resolution of each grid
# ***IMPROVE*** Should check if bigd is none
bigG=attOfF.getgridbyname(grid) # Getting all the attributes in the grid of a file
# Get extreme grid point
ulp=eval(bigG.variable["UpperLeftPointMtrs"])
lrp=eval(bigG.variable["LowerRightMtrs"])
# Get grid dimmension
dimx=int(bigG.variable["XDim"])
dimy=int(bigG.variable["YDim"])
# Calculate grid resolution
gridResolX[grid]=(lrp[0]-ulp[0])/dimx
gridResolY[grid]=(ulp[1]-lrp[1])/dimy
### Determination of new extreme coordinates for each grid
# up
try:
if extremeup[grid]< ulp[1]:
extremeup[grid]=ulp[1]
except KeyError:
extremeup[grid]=ulp[1]
# down
try:
if extremedown[grid]> lrp[1]:
extremedown[grid]=lrp[1]
except KeyError:
extremedown[grid]=lrp[1]
# left
try:
if extremeleft[grid]> ulp[0]:
extremeleft[grid]=ulp[0]
except KeyError:
extremeleft[grid]=ulp[0]
# right
try:
if extremeright[grid]< lrp[0]:
extremeright[grid]=lrp[0]
except KeyError:
extremeright[grid]=lrp[0]
### Detetermination of dataset to grid name
if bigG is not None:
go=bigG.GROUP["DataField"].OBJECT
for r in go:
dstogrid[ go[r].variable["DataFieldName"] ] = grid
## Determination of ULC for each grid in each file
filGridULC[grid][attOfF.name] = ulp
## determination of new dimension for each grid
gridDimY[grid] = int((extremeup[grid]-extremedown[grid])/gridResolY[grid])
gridDimX[grid] = int((extremeright[grid]-extremeleft[grid])/gridResolX[grid])
for ds in dslist:
# For each dataset
for sd in lfHDF:
# For each hdf file
# Try opening dataset
try:
sds = sd.select(eval(ds))
# Get fill value
NoValueDS[ds] = sds.getfillvalue()
# Get dtype
dtypeDS[ds] = sds.info()[3]
except:
log.log('e',Nom,"no dataset")
## Start creating new file
###############################################################################
# This is the actual part were stuf appens #
###############################################################################
# This part is the same for every file in any circumstances
########## absolute ########################
# Open new file
try:
hdf = HDF(newfilename, HC.WRITE | HC.CREATE |HC.TRUNC)
sd = SD(newfilename, SDC.WRITE | SDC.CREATE )
except TypeError:
hdf = HDF(newfilename.encode('ascii','ignore'), HC.WRITE | HC.CREATE |HC.TRUNC)
sd = SD(newfilename.encode('ascii','ignore'), SDC.WRITE | SDC.CREATE )
v=hdf.vgstart()
vg={}
vg1={}
vg2={}
## rewrite the gridlist
gridlist = []
for ds in dslist:
if dstogrid[ds] not in gridlist:
gridlist.append(dstogrid[ds])
for grid in gridlist:
vg[grid]=v.attach(-1,write=1)
vg[grid]._class="GRID"
vg[grid]._name=eval(grid)
vg1[grid]=v.attach(-1,write=1)
vg2[grid]=v.attach(-1,write=1)
vg1[grid]._class="GRID Vgroup"
vg1[grid]._name="Data Fields"
vg2[grid]._class="GRID Vgroup"
vg2[grid]._name="Grid Attributes"
vg[grid].insert(vg1[grid])
vg[grid].insert(vg2[grid])
########## absolute ########################
# Create dataset with the right size
for ds in dslist:
theGrid=dstogrid[ds]
# Get grid name of data set
sds = sd.create(eval(ds),dtypeDS[ds],(gridDimY[theGrid],gridDimX[theGrid]))
# Set fill value
fv=NoValueDS[ds]
try:
sds.setfillvalue(NoValueDS[ds])
except OverflowError:
log.log('e',Nom,"setfillvalue")
sds.setfillvalue(0)
## write real data
for fil in range(len(latt)):
try:
# Determine were the data will be writen
ulc = filGridULC[theGrid][latt[fil].name]
# Determine the position on the grid
y = (extremeup[theGrid]-ulc[1])/(extremeup[theGrid]-extremedown[theGrid])
x = (ulc[0]-extremeleft[theGrid])/(extremeright[theGrid]-extremeleft[theGrid])
y = int(y*gridDimY[theGrid])
x = int(x*gridDimX[theGrid])
# read data from files
osds = lfHDF[fil].select(eval(ds))
sh = osds[:].shape
sds[y:y+sh[0],x:x+sh[1]] = osds[:]
osds.endaccess()
except:
pass
# Close sds
vg1[dstogrid[ds]].add(HC.DFTAG_NDG,sds.ref())
sds.endaccess()
for g in vg1:
vg1[g].detach()
vg2[g].detach()
vg[g].detach()
# Create attribute table for the file
attstr="GROUP=GridStructure\n"
gridcount=1
for gr in gridlist:
# Start group grid
attstr+="\tGROUP=GRID_%i\n"%gridcount
# Add grid name
attstr+="\t\tGridName=%s\n"%gr
# Add dimention
attstr+="\t\tXDim=%i\n"%gridDimX[gr]
attstr+="\t\tYDim=%i\n"%gridDimY[gr]
# Add UpperLeftPointMtrs
attstr+="\t\tUpperLeftPointMtrs=(%f,%f)\n"%(extremeleft[gr],extremeup[gr])
# Add lrp
attstr+="\t\tLowerRightMtrs=(%f,%f)\n"%(extremeright[gr],extremedown[gr])
# Add projection
attstr+="\t\tProjection=%s\n"%paramMustSimDict[gr]["Projection"]
# ProjParams
attstr+="\t\tProjParams=%s\n"%paramMustSimDict[gr]["ProjParams"]
# SphereCode
attstr+="\t\tSphereCode=%s\n"%paramMustSimDict[gr]["SphereCode"]
attstr+="""\t\tGROUP=Dimension
\t\tEND_GROUP=Dimension
\t\tGROUP=DataField\n"""
## Add data sets
# create list of ds for current grid
lsdsgr=[]
dsnum=1
for ds in dslist:
if dstogrid[ds] == gr:
# Add object
attstr+="\t\t\tOBJECT=DataField_%i\n"%dsnum
# datafield name
attstr+="\t\t\t\tDataFieldName=%s\n"%ds
# datatype
attstr+="\t\t\t\tDataType=%s\n"%paramMustSimDict[gr][ds]
# dim
attstr+='\t\t\t\tDimList=("YDim","XDim")\n'
attstr+="\t\t\tEND_OBJECT=DataField_%i\n"%dsnum
dsnum+=1
attstr+="\t\tEND_GROUP=DataField\n"
attstr+="""\t\tGROUP=MergedFields
\t\tEND_GROUP=MergedFields\n"""
attstr+="\tEND_GROUP=GRID_%i\n"%gridcount
gridcount+=1
attstr+="""END_GROUP=GridStructure
GROUP=PointStructure
END_GROUP=PointStructure
END"""
# adding attribute to new file
att=sd.attr('StructMetadata.0')
att.set(SDC.CHAR,attstr)
sd.end()
hdf.close()
# This should return something somehow
elif len(arg)>1:
afile = arg[0] # This is the list of the NAME of the files to merge
newfilename = arg[1] # This is the final file NAME
# Create a copy
from shutil import copyfile
copyfile(afile,newfilename)
def doCompress(compType, value=0, v2=0):
"""Create and validate an HDF file using a compression scheme
sepcified by the parameters"""
# Build a significant file name
if compType == SDC.COMP_NONE:
fileName = "SDS.COMP_NONE"
elif compType == SDC.COMP_RLE:
fileName = "SDS.COMP_RLE"
elif compType == SDC.COMP_SKPHUFF:
fileName = "SDS.COMP_SKPHUFF.%d" % value
elif compType == SDC.COMP_DEFLATE:
fileName = "SDS.COMP_DEFLATE.%d" % value
elif compType == SDC.COMP_SZIP:
fileName = "SDS.COMP_SZIP"
if value == SDC.COMP_SZIP_NN:
fileName += ".NN"
elif value == SDC.COMP_SZIP_EC:
fileName += ".EC"
else:
print("illegal value")
sys.exit(1)
fileName += ".%s" % v2
else:
print("illegal compType")
sys.exit(1)
fileName += ".hdf"
SDS_NAME = "Data"
fill_value = 0
#LENGTH = 9
#WIDTH = 6
#
#data = numpy.array( ((100,100,200,200,300,400),
# (100,100,200,200,300,400),
# (100,100,200,200,300,400),
# (300,300, 0,400,300,400),
# (300,300, 0,400,300,400),
# (300,300, 0,400,300,400),
# (0, 0,600,600,300,400),
# (500,500,600,600,300,400),
# (0, 0,600,600,300,400)), NUMPY_DATATYPE)
# The above dataset is used in the original NCSA example.
# It is too small to show a significant size reduction after
# compression. The following is used for a more realistic example.
data = numpy.zeros((LENGTH, WIDTH), NUMPY_DATATYPE)
for i in range(LENGTH):
for j in range(WIDTH):
data[i,j] = (i+j)*(i-j)
# Create HDF file, wiping it out it it already exists.
sd_id = SD(fileName, SDC.WRITE | SDC.CREATE | SDC.TRUNC)
# Create dataset.
sds_id = sd_id.create(SDS_NAME, HDF_DATATYPE, (LENGTH, WIDTH))
# Fill dataset will fill value.
sds_id.setfillvalue(0)
# Apply compression.
try:
sds_id.setcompress(compType, # compression type
value, v2) # args depend on compression type
except HDF4Error as msg:
print(("Error compressing the dataset with params: "
"(%d,%d,%d) : %s" % (compType, value, v2, msg)))
sds_id.endaccess()
sd_id.end()
os.remove(fileName)
return
# Load data in the dataset.
sds_id[:] = data
# Close dataset.
sds_id.endaccess()
# Close hdf file to flush compressed data.
sd_id.end()
# Verify compressed data.
# ######################
# Reopen file and select first dataset.
sd_id = SD(fileName, SDC.READ)
sds_id = sd_id.select(0)
# Obtain compression info.
compInfo = sds_id.getcompress()
compType = compInfo[0]
print("file : %s" % fileName)
print(" size = %d" % os.path.getsize(fileName))
if compType == SDC.COMP_NONE:
print(" compType = COMP_NONE")
elif compType == SDC.COMP_RLE:
print(" compType = COMP_RLE")
elif compType == SDC.COMP_SKPHUFF:
print(" compType = COMP_SKPHUFF")
print(" dataSize = %d" % compInfo[1])
elif compType == SDC.COMP_DEFLATE:
print(" compType = COMP_DEFLATE (GZIP)")
print(" level = %d" % compInfo[1])
elif compType == SDC.COMP_SZIP:
print(" compType = COMP_SZIP")
optionMask = compInfo[1]
if optionMask & SDC.COMP_SZIP_NN:
print(" encoding scheme = NN")
elif optionMask & SDC.COMP_SZIP_EC:
print(" encoding scheme = EC")
else:
print(" unknown encoding scheme")
sys.exit(1)
pixelsPerBlock, pixelsPerScanline, bitsPerPixel, pixels = compInfo[2:]
print(" pixelsPerBlock = %d" % pixelsPerBlock)
print(" pixelsPerScanline = %d" % pixelsPerScanline)
print(" bitsPerPixel = %d" % bitsPerPixel)
print(" pixels = %d" % pixels)
else:
print(" unknown compression type")
sys.exit(1)
# Read dataset contents.
out_data = sds_id[:]
# Compare with original data.
num_errs = 0
for i in range(LENGTH):
for j in range(WIDTH):
if data[i,j] != out_data[i,j]:
print("bad value at %d,%d expected: %d got: %d" \
% (i,j,data[i,j],out_data[i,j]))
num_errs += 1
# Close dataset and hdf file.
sds_id.endaccess()
sd_id.end()
if num_errs == 0:
print(" file validated")
else:
print(" file invalid : %d errors" % num_errs)
print("")
pass
try:
os.unlink(tmp_path)
except OSError,e:
pass
with open(path, 'r') as in_fp:
with open(tmp_path_Z, 'w') as out_fp:
out_fp.write(in_fp.read())
os.system('gunzip -f ' + tmp_path_Z)
#m = hashlib.md5()
#m.update(open(tmp_path).read())
#print 'tmp_path', tmp_path, m.hexdigest()
sd = SD(tmp_path, SDC.READ)
kt = sd.select('surfacePrecipitation')
print kt.dimensions()
data = kt[:, :].copy()
print year, month, day, data.mean()
acc.append([year, month, day, data])
if year != last_year:
save()
last_year = year
save()
### Example from docs http://hdfeos.github.io/pyhdf/modules/SD.html#writing
# Import SD and numpy.
from pyhdf.SD import *
from numpy import *
fileName = 'template.hdf'
# Create HDF file.
hdfFile = SD(fileName, SDC.WRITE|SDC.CREATE)
# Assign a few attributes at the file level
hdfFile.author = 'It is me...'
hdfFile.priority = 2
# Create a dataset named 'd1' to hold a 3x3 float array.
d1 = hdfFile.create('d1', SDC.FLOAT32, (3,3))
# Set some attributs on 'd1'
d1.description = 'Sample 3x3 float array' d1.units = 'celsius'
# Name 'd1' dimensions and assign them attributes.
dim1 = d1.dim(0)
dim2 = d1.dim(1)
dim1.setname('width')
dim2.setname('height')
dim1.units = 'm'
dim2.units = 'cm'
# Assign values to 'd1'
d1[0] = (14.5, 12.8, 13.0) # row 1
d1[1:] = ((-1.3, 0.5, 4.8), # row 2 and
(3.1, 0.0, 13.8)) # row 3
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 25 14:38:18 2018
@author: neeleshrampal
"""
"""Edits have been made to incorparate a cloud fraction vs hISTROGRAM product"""
"""One must note that the product is likely to be conservative for cumulus cloud regions, due to its determination of partly cloudy reubis"""
filename = '/Volumes/Promise1/Neelesh/MODIS_L3/2002/365/MOD08_D3.A2002365.061.2017280190741.hdf'
from pyhdf import SD
f = SD.SD(filename)
datasets = f.datasets().keys()
d1 = []
for key in datasets:
#
if 'Histo_vs_Pressure' in key:
d1.append(key)
# print d1
"""the goal for today is to ensure that you can obtain the mean cloud fraction from grouping the data"""
sds = f.select('Cloud_Fraction_Nadir_Day_Mean')
data = sds.get()/1e4
c=np.where(data<0.0)
data[c]=np.nan
#f.close()
dset = 'Cloud_Fraction_Nadir_Day_Pixel_Counts'
sds = f.select(dset)
data4 = np.array(sds.get(),dtype=float)
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from pyhdf.SD import *
from bitstring import *
import sys
# np.set_printoptions(threshold='nan')
_filename = "C:\Users\AKO NA LNG\Desktop\Esquivel Files\School Files 2\Special Problem\MODIS\MOD35\MOD35_L2.A2015220.0215.006.2015220134621.hdf"
mod35 = SD(_filename, SDC.READ)
sds_name = "Cloud_Mask"
sds_index = mod35.select(sds_name)
sds_data = sds_index.get()
sds_data_0 = sds_data[0, :, :]
sds_data_0_bin = sds_data_0.astype(dtype=np.uint8, order="K", casting="unsafe", subok=False, copy=False)
for x in np.nditer(sds_data_0_bin, op_flags=["readwrite"]):
x[...] = np.right_shift(x, 6)
fig = plt.figure()
ax = fig.add_subplot(111)
cmap = plt.cm.winter
bounds = [0, 1, 2, 3]
norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys, Image
import numpy as np
from pyhdf.SD import *
if __name__ == '__main__':
filename = sys.argv[1]
hdfFile = SD(filename)
arrays = []
for i in range(3):
data = hdfFile.select(i)
tmp = np.array (data[:], dtype = np.int16)
emp = data.getfillvalue()
maxv = tmp.max()
tmp[tmp == emp] = maxv
minv = tmp.min()
rng = maxv - minv
arr = np.array((tmp-minv)*1.0/rng*255, dtype=np.uint8)
im = Image.fromarray(arr)
im.save(filename+"_" +str(i)+".png")
arrays.append(arr)
hdfFile.end()
def sdscreate(sd, name):
# Create a simple 3x3 float array.
sds = sd.create(name, SDC.FLOAT32, (3,3))
# Initialize array
sds[:] = ((0,1,2),(3,4,5),(6,7,8))
# "close" dataset.
sds.endaccess()
# Create HDF file
filename = 'inventory.hdf'
hdf = HDF(filename, HC.WRITE|HC.CREATE)
# Initialize the SD, V and VS interfaces on the file.
sd = SD(filename, SDC.WRITE) # SD interface
vs = hdf.vstart() # vdata interface
v = hdf.vgstart() # vgroup interface
# Create vdata named 'INVENTORY'.
vdatacreate(vs, 'INVENTORY')
# Create dataset named "ARR_3x3"
sdscreate(sd, 'ARR_3x3')
# Attach the vdata and the dataset.
vd = vs.attach('INVENTORY')
sds = sd.select('ARR_3x3')
# Create vgroup named 'TOTAL'.
vg = v.create('TOTAL')
def prnt(fname) :
# Dictionnary used to convert from a numeric data type to its symbolic
# representation
typeTab = {
SDC.CHAR: 'CHAR',
SDC.CHAR8: 'CHAR8',
SDC.UCHAR8: 'UCHAR8',
SDC.INT8: 'INT8',
SDC.UINT8: 'UINT8',
SDC.INT16: 'INT16',
SDC.UINT16: 'UINT16',
SDC.INT32: 'INT32',
SDC.UINT32: 'UINT32',
SDC.FLOAT32: 'FLOAT32',
SDC.FLOAT64: 'FLOAT64'
}
printf = sys.stdout.write
def eol(n=1):
printf("%s" % chr(10) * n)
#hdfFile = sys.argv[1] # Get first command line argument
hdfFile = fname
try: # Catch pyhdf.SD errors
# Open HDF file named on the command line
f = SD(hdfFile)
# Get global attribute dictionnary
attr = f.attributes(full=1)
# Get dataset dictionnary
dsets = f.datasets()
# File name, number of attributes and number of variables.
printf("FILE INFO"); eol()
printf("-------------"); eol()
printf("%-25s%s" % ("File:", hdfFile)); eol()
printf("%-25s%d" % (" file attributes:", len(attr))); eol()
printf("%-25s%d" % (" datasets:", len(dsets))); eol()
eol();
# Global attribute table.
if len(attr) > 0:
printf("File attributes"); eol(2)
printf(" name idx type len value"); eol()
printf(" -------------------- --- ------- --- -----"); eol()
# Get list of attribute names and sort them lexically
attNames = attr.keys()
attNames.sort()
for name in attNames:
t = attr[name]
# t[0] is the attribute value
# t[1] is the attribute index number
# t[2] is the attribute type
# t[3] is the attribute length
printf(" %-20s %3d %-7s %3d %s" %
(name, t[1], typeTab[t[2]], t[3], t[0])); eol()
eol()
# Dataset table
if len(dsets) > 0:
printf("Datasets (idx:index num, na:n attributes, cv:coord var)"); eol(2)
printf(" name idx type na cv dimension(s)"); eol()
printf(" -------------------- --- ------- -- -- ------------"); eol()
# Get list of dataset names and sort them lexically
dsNames = dsets.keys()
dsNames.sort()
for name in dsNames:
# Get dataset instance
ds = f.select(name)
# Retrieve the dictionary of dataset attributes so as
# to display their number
vAttr = ds.attributes()
t = dsets[name]
# t[0] is a tuple of dimension names
# t[1] is a tuple of dimension lengths
# t[2] is the dataset type
# t[3] is the dataset index number
printf(" %-20s %3d %-7s %2d %-2s " %
(name, t[3], typeTab[t[2]], len(vAttr),
ds.iscoordvar() and 'X' or ''))
# Display dimension info.
n = 0
for d in t[0]:
printf("%s%s(%d)" % (n > 0 and ', ' or '', d, t[1][n]))
n += 1
eol()
eol()
# Dataset info.
if len(dsNames) > 0:
printf("DATASET INFO"); eol()
printf("-------------"); eol(2)
for name in dsNames:
# Access the dataset
dsObj = f.select(name)
# Get dataset attribute dictionnary
dsAttr = dsObj.attributes(full=1)
if len(dsAttr) > 0:
printf("%s attributes" % name); eol(2)
printf(" name idx type len value"); eol()
printf(" -------------------- --- ------- --- -----"); eol()
# Get the list of attribute names and sort them alphabetically.
attNames = dsAttr.keys()
attNames.sort()
for nm in attNames:
t = dsAttr[nm]
# t[0] is the attribute value
# t[1] is the attribute index number
# t[2] is the attribute type
# t[3] is the attribute length
printf(" %-20s %3d %-7s %3d %s" %
(nm, t[1], typeTab[t[2]], t[3], t[0])); eol()
eol()
# Get dataset dimension dictionnary
dsDim = dsObj.dimensions(full=1)
if len(dsDim) > 0:
printf ("%s dimensions" % name); eol(2)
printf(" name idx len unl type natt");eol()
printf(" -------------------- --- ----- --- ------- ----");eol()
# Get the list of dimension names and sort them alphabetically.
dimNames = dsDim.keys()
dimNames.sort()
for nm in dimNames:
t = dsDim[nm]
# t[0] is the dimension length
# t[1] is the dimension index number
# t[2] is 1 if the dimension is unlimited, 0 if not
# t[3] is the the dimension scale type, 0 if no scale
# t[4] is the number of attributes
printf(" %-20s %3d %5d %s %-7s %4d" %
(nm, t[1], t[0], t[2] and "X" or " ",
t[3] and typeTab[t[3]] or "", t[4])); eol()
eol()
except HDF4Error, msg:
print "HDF4Error", msg
SDC.INT32: 'INT32',
SDC.UINT32: 'UINT32',
SDC.FLOAT32: 'FLOAT32',
SDC.FLOAT64: 'FLOAT64'
}
printf = sys.stdout.write
def eol(n=1):
printf("%s" % chr(10) * n)
hdfFile = sys.argv[1] # Get first command line argument
try: # Catch pyhdf.SD errors
# Open HDF file named on the command line
f = SD(hdfFile)
# Get global attribute dictionnary
attr = f.attributes(full=1)
# Get dataset dictionnary
dsets = f.datasets()
# File name, number of attributes and number of variables.
printf("FILE INFO"); eol()
printf("-------------"); eol()
printf("%-25s%s" % ("File:", hdfFile)); eol()
printf("%-25s%d" % (" file attributes:", len(attr))); eol()
printf("%-25s%d" % (" datasets:", len(dsets))); eol()
eol();
# Global attribute table.
if len(attr) > 0:
def get_modis_latitude_longitude(modis_filename): hdf = SD(modis_filename) lat = hdf.select['Latitude'] lon = hdf.select['Longitude'] return (lat,lon)
def get_merra(fname, minlat, maxlat, minlon, maxlon, cdic, verbose=False):
'''Read data from MERRA hdf file. Note that the Lon values
should be between [0-360]. Hdf file with weather model
data can be downloaded from
http://disc.sci.gsfc.nasa.gov/daac-bin/FTPSubset.pl
Args:
* fname (str): Path to the grib file
* minlat (np.float): Minimum latitude
* maxlat (np.float): Maximum latitude
* minlon (np.float): Minimum longitude
* maxlon (np.float): Maximum longitude
* cdic (np.float): Dictionary of constants
Returns:
* lvls (np.array): Pressure levels
* latlist(np.array): Latitudes of the stations
* lonlist(np.array): Longitudes of the stations
* gph (np.array): Geopotential height
* tmp (np.array): Temperature
* vpr (np.array): Vapor pressure
'''
if fname[-3:] == 'hdf':
print(minlat)
print(maxlat)
print(minlon)
print(maxlon)
# Read the hdf file
file = SD(fname)
if verbose:
print 'PROGRESS: READING HDF FILE'
lvl = file.select('levels')
rlvls = lvl.get() # Pressure levels are from lowest to highest
lvls = []
for i in xrange(
len(rlvls)
): # Reverse the pressure levels to be consistent with other GAMs
index = len(rlvls) - i - 1
lvls.append(rlvls[index])
nlvls = len(lvls)
lvls = np.array(lvls)
alpha = cdic['Rv'] / cdic['Rd']
# Select latitutde and longitude
lat = file.select('latitude')
lon = file.select('longitude')
lats = lat.get()
lons = lon.get()
mask1 = (lats > minlat) & (lats < maxlat)
mask2 = (lons > minlon) & (lons < maxlon)
[ii] = np.where(mask1 == True)
[jj] = np.where(mask2 == True)
del mask1
del mask2
iimemo = []
for m in xrange(len(ii)):
for i in xrange(len(jj)):
iimemo.append(ii[m])
jjmemo = []
for i in xrange(len(ii)):
jjmemo.append(jj)
jjmemo = np.array(jjmemo)
jjmemo = jjmemo.flatten()
iimemo = np.array(iimemo)
ii = iimemo
jj = jjmemo
latlist = lats[ii]
lonlist = lons[jj]
nstn = len(latlist)
# Create arrays for 3D storage
gph = np.zeros((nlvls, nstn)) #Potential height
tmp = gph.copy() #Temperature
vpr = gph.copy() #Vapor pressure
if verbose:
print 'Number of stations:', nstn
# Get data from files
h = file.select('h')
height = h.get()[0]
qv = file.select('qv')
humidity = qv.get()[0]
sp = file.select('ps')
spressure = sp.get()[0]
t = file.select('t')
temp = t.get()[0]
# Lvls are in hecto pascals, convert to pascals
lvls = 100.0 * lvls
# Reverse altitude
for i in xrange(nlvls):
index = nlvls - i - 1
gph[i, :] = height[index][ii, jj]
idx = np.zeros(nstn)
for i in xrange(nstn):
for m in xrange(nlvls):
if spressure[ii[i]][jj[i]] > lvls[m]:
idx[i] = m
# extrapolation of temperature and humidity data at pressure levels under surface at each grid point
tk = np.zeros(nstn)
tb = np.zeros(nstn)
for i in xrange(nstn):
t = temp[:, ii[i], jj[i]]
x = [lvls[idx[i]], lvls[idx[i] - 1]]
y = [t[nlvls - idx[i] - 1], t[nlvls - idx[i]]]
coef = np.polyfit(x, y, 1)
tk[i] = coef[0]
tb[i] = coef[1]
hk = np.zeros(nstn)
hb = np.zeros(nstn)
for i in xrange(nstn):
hum = humidity[:, ii[i], jj[i]]
x = [lvls[idx[i]], lvls[idx[i] - 1]]
y = [hum[nlvls - idx[i] - 1], hum[nlvls - idx[i]]]
coef = np.polyfit(x, y, 1)
hk[i] = coef[0]
hb[i] = coef[1]
#fill out the tmp and vpr array
for i in xrange(nstn):
id = np.int(idx[i] + 1)
for n in xrange(id):
tmp[n, i] = temp[nlvls - 1 - n, ii[i], jj[i]]
exl = nlvls - id
for m in xrange(exl):
tmp[id + m, i] = tk[i] * lvls[id + m] + tb[i]
for i in xrange(nstn):
id = np.int(idx[i] + 1)
for n in xrange(id):
vpr[n, i] = humidity[nlvls - 1 - n, ii[i], jj[i]]
exl = nlvls - id
for m in xrange(exl):
vpr[id + m, i] = hk[i] * lvls[id + m] + hb[i]
memo = list(vpr)
memo = np.array(memo)
for i in xrange(nlvls):
vpr[i, :] = memo[i, :] * lvls[i] * alpha / (
1 + (alpha - 1) * memo[i, :])
# Close the hdf file
file.end()
if fname[-3:] == 'nc4':
# Read the netcdf file
file = netCDF4.Dataset(fname)
if verbose:
print 'PROGRESS: READING netcdf FILE'
ncv = file.variables
rlvls = ncv['lev'][:] # Pressure levels are from lowest to highest
lvls = []
for i in xrange(
len(rlvls)
): # Reverse the pressure levels to be consistent with other GAMs
index = len(rlvls) - i - 1
lvls.append(rlvls[index])
nlvls = len(lvls)
lvls = np.array(lvls)
alpha = cdic['Rv'] / cdic['Rd']
# Select latitutde and longitude
lats = ncv['lat'][:]
lons = ncv['lon'][:]
mask1 = (lats > minlat) & (lats < maxlat)
mask2 = (lons > minlon) & (lons < maxlon)
[ii] = np.where(mask1 == True)
[jj] = np.where(mask2 == True)
del mask1
del mask2
iimemo = []
for m in xrange(len(ii)):
for i in xrange(len(jj)):
iimemo.append(ii[m])
jjmemo = []
for i in xrange(len(ii)):
jjmemo.append(jj)
jjmemo = np.array(jjmemo)
jjmemo = jjmemo.flatten()
iimemo = np.array(iimemo)
ii = iimemo
jj = jjmemo
latlist = lats[ii]
lonlist = lons[jj]
nstn = len(latlist)
# Create arrays for 3D storage
gph = np.zeros((nlvls, nstn)) #Potential height
tmp = gph.copy() #Temperature
vpr = gph.copy() #Vapor pressure
if verbose:
print 'Number of stations:', nstn
# Get data from files
height = ncv['H'][:]
height = height[0]
humidity = ncv['QV'][:]
humidity = humidity[0]
spressure = ncv['PS'][:]
spressure = spressure[0]
temp = ncv['T'][:]
temp = temp[0]
# Lvls are in hecto pascals, convert to pascals
lvls = 100.0 * lvls
# Reverse altitude
for i in xrange(nlvls):
index = nlvls - i - 1
gph[i, :] = height[index][ii, jj]
idx = np.zeros(nstn)
for i in xrange(nstn):
for m in xrange(nlvls):
if spressure[ii[i]][jj[i]] > lvls[m]:
idx[i] = m
# extrapolation of temperature and humidity data at pressure levels under surface at each grid point
tk = np.zeros(nstn)
tb = np.zeros(nstn)
for i in xrange(nstn):
t = temp[:, ii[i], jj[i]]
x = [lvls[idx[i]], lvls[idx[i] - 1]]
y = [t[nlvls - idx[i] - 1], t[nlvls - idx[i]]]
coef = np.polyfit(x, y, 1)
tk[i] = coef[0]
tb[i] = coef[1]
hk = np.zeros(nstn)
hb = np.zeros(nstn)
for i in xrange(nstn):
hum = humidity[:, ii[i], jj[i]]
x = [lvls[idx[i]], lvls[idx[i] - 1]]
y = [hum[nlvls - idx[i] - 1], hum[nlvls - idx[i]]]
coef = np.polyfit(x, y, 1)
hk[i] = coef[0]
hb[i] = coef[1]
#fill out the tmp and vpr array
for i in xrange(nstn):
id = np.int(idx[i] + 1)
for n in xrange(id):
tmp[n, i] = temp[nlvls - 1 - n, ii[i], jj[i]]
exl = nlvls - id
for m in xrange(exl):
tmp[id + m, i] = tk[i] * lvls[id + m] + tb[i]
for i in xrange(nstn):
id = np.int(idx[i] + 1)
for n in xrange(id):
vpr[n, i] = humidity[nlvls - 1 - n, ii[i], jj[i]]
exl = nlvls - id
for m in xrange(exl):
vpr[id + m, i] = hk[i] * lvls[id + m] + hb[i]
memo = list(vpr)
memo = np.array(memo)
for i in xrange(nlvls):
vpr[i, :] = memo[i, :] * lvls[i] * alpha / (
1 + (alpha - 1) * memo[i, :])
# Close the netcdf file
file.close()
# Send data
return lvls, latlist, lonlist, gph, tmp, vpr
def doCompress(compType, value=0, v2=0):
"""Create and validate an HDF file using a compression scheme
specified by the parameters"""
# Build a significant file name
if compType == SDC.COMP_NONE:
fileName = "SDS.COMP_NONE"
elif compType == SDC.COMP_RLE:
fileName = "SDS.COMP_RLE"
elif compType == SDC.COMP_SKPHUFF:
fileName = "SDS.COMP_SKPHUFF.%d" % value
elif compType == SDC.COMP_DEFLATE:
fileName = "SDS.COMP_DEFLATE.%d" % value
elif compType == SDC.COMP_SZIP:
fileName = "SDS.COMP_SZIP"
if value == SDC.COMP_SZIP_NN:
fileName += ".NN"
elif value == SDC.COMP_SZIP_EC:
fileName += ".EC"
else:
print("illegal value")
sys.exit(1)
fileName += ".%s" % v2
else:
print("illegal compType")
sys.exit(1)
fileName += ".hdf"
SDS_NAME = "Data"
fill_value = 0
#LENGTH = 9
#WIDTH = 6
#
#data = numpy.array( ((100,100,200,200,300,400),
# (100,100,200,200,300,400),
# (100,100,200,200,300,400),
# (300,300, 0,400,300,400),
# (300,300, 0,400,300,400),
# (300,300, 0,400,300,400),
# (0, 0,600,600,300,400),
# (500,500,600,600,300,400),
# (0, 0,600,600,300,400)), NUMPY_DATATYPE)
# The above dataset is used in the original NCSA example.
# It is too small to show a significant size reduction after
# compression. The following is used for a more realistic example.
data = numpy.zeros((LENGTH, WIDTH), NUMPY_DATATYPE)
for i in range(LENGTH):
for j in range(WIDTH):
data[i, j] = (i + j) * (i - j)
# Create HDF file, wiping it out it it already exists.
sd_id = SD(fileName, SDC.WRITE | SDC.CREATE | SDC.TRUNC)
# Create dataset.
sds_id = sd_id.create(SDS_NAME, HDF_DATATYPE, (LENGTH, WIDTH))
# Fill dataset will fill value.
sds_id.setfillvalue(0)
# Apply compression.
try:
sds_id.setcompress(
compType, # compression type
value,
v2) # args depend on compression type
except HDF4Error as msg:
print(("Error compressing the dataset with params: "
"(%d,%d,%d) : %s" % (compType, value, v2, msg)))
sds_id.endaccess()
sd_id.end()
os.remove(fileName)
return
# Load data in the dataset.
sds_id[:] = data
# Close dataset.
sds_id.endaccess()
# Close hdf file to flush compressed data.
sd_id.end()
# Verify compressed data.
# ######################
# Reopen file and select first dataset.
sd_id = SD(fileName, SDC.READ)
sds_id = sd_id.select(0)
# Obtain compression info.
compInfo = sds_id.getcompress()
compType = compInfo[0]
print("file : %s" % fileName)
print(" size = %d" % os.path.getsize(fileName))
if compType == SDC.COMP_NONE:
print(" compType = COMP_NONE")
elif compType == SDC.COMP_RLE:
print(" compType = COMP_RLE")
elif compType == SDC.COMP_SKPHUFF:
print(" compType = COMP_SKPHUFF")
print(" dataSize = %d" % compInfo[1])
elif compType == SDC.COMP_DEFLATE:
print(" compType = COMP_DEFLATE (GZIP)")
print(" level = %d" % compInfo[1])
elif compType == SDC.COMP_SZIP:
print(" compType = COMP_SZIP")
optionMask = compInfo[1]
if optionMask & SDC.COMP_SZIP_NN:
print(" encoding scheme = NN")
elif optionMask & SDC.COMP_SZIP_EC:
print(" encoding scheme = EC")
else:
print(" unknown encoding scheme")
sys.exit(1)
pixelsPerBlock, pixelsPerScanline, bitsPerPixel, pixels = compInfo[2:]
print(" pixelsPerBlock = %d" % pixelsPerBlock)
print(" pixelsPerScanline = %d" % pixelsPerScanline)
print(" bitsPerPixel = %d" % bitsPerPixel)
print(" pixels = %d" % pixels)
else:
print(" unknown compression type")
sys.exit(1)
# Read dataset contents.
out_data = sds_id[:]
# Compare with original data.
num_errs = 0
for i in range(LENGTH):
for j in range(WIDTH):
if data[i, j] != out_data[i, j]:
print("bad value at %d,%d expected: %d got: %d" \
% (i,j,data[i,j],out_data[i,j]))
num_errs += 1
# Close dataset and hdf file.
sds_id.endaccess()
sd_id.end()
if num_errs == 0:
print(" file validated")
else:
print(" file invalid : %d errors" % num_errs)
print("")
def run(FILE_NAME):
# Identify the data field.
DATAFIELD_NAME = 'Blue Radiance/RDQI'
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data3D = hdf.select(DATAFIELD_NAME)
data = data3D[:,:,:]
# Read attributes.
attrs = data3D.attributes(full=1)
fva=attrs["_FillValue"]
_FillValue = fva[0]
# Read geolocation dataset from another file.
GEO_FILE_NAME = 'MISR_AM1_AGP_P117_F01_24.hdf'
GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'],
GEO_FILE_NAME)
hdf_geo = SD(GEO_FILE_NAME, SDC.READ)
# Read geolocation dataset.
lat3D = hdf_geo.select('GeoLatitude')
lat = lat3D[:,:,:]
lon3D = hdf_geo.select('GeoLongitude')
lon = lon3D[:,:,:]
# Read scale factor attribute.
f = HDF(FILE_NAME, HC.READ)
v = f.vgstart()
vg = v.attach(8)
# PyHDF cannot read attributes from Vgroup properly.
# sfa = vg.attr('Scale Factor')
# scale_factor = sfa.get()
vg.detach()
v.end()
# Set it manually using HDFView.
scale_factor = 0.047203224152326584
# We need to shift bits for "RDQI" to get "Blue Band "only.
# See the page 84 of "MISR Data Products Specifications (rev. S)".
# The document is available at [1].
datas = np.right_shift(data, 2);
dataf = datas.astype(np.double)
# Apply the fill value.
dataf[data == _FillValue] = np.nan
# Filter out values (> 16376) used for "Flag Data".
# See Table 1.2 in "Level 1 Radiance Scaling and Conditioning
# Algorithm Theoretical Basis" document [2].
dataf[datas > 16376] = np.nan
datam = np.ma.masked_array(dataf, mask=np.isnan(dataf))
# Apply scale facotr.
datam = scale_factor * datam;
nblocks = data.shape[0]
ydimsize = data.shape[1]
xdimsize = data.shape[2]
datam = datam.reshape(nblocks*ydimsize, xdimsize)
lat = lat.reshape(nblocks*ydimsize, xdimsize)
lon = lon.reshape(nblocks*ydimsize, xdimsize)
# Set the limit for the plot.
m = Basemap(projection='cyl', resolution='h',
llcrnrlat=np.min(lat), urcrnrlat = np.max(lat),
llcrnrlon=np.min(lon), urcrnrlon = np.max(lon))
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(-90., 120., 30.), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(-180., 181., 45.), labels=[0, 0, 0, 1])
m.pcolormesh(lon, lat, datam, latlon=True)
cb = m.colorbar()
cb.set_label(r'$Wm^{-2}sr^{-1}{\mu}m^{-1}$')
basename = os.path.basename(FILE_NAME)
plt.title('{0}\n{1}'.format(basename, 'Blue Radiance'))
fig = plt.gcf()
# plt.show()
pngfile = "{0}.py.agp.png".format(basename)
fig.savefig(pngfile)
#colorbar()
#title('OW')
############################################
#read AMSRE data
####---snow1:
AMSREfolder = '/scratch/clisap/seaice/OWN_PRODUCTS/MELT_PONDS/GRID_AMSRE/'
AMSREfile = AMSREfolder + 'asi-n6250-20110625-v5i.nc'
AMSREfile2 = AMSREfolder + 'AMSR_E_L3_SeaIce12km_V12_20110625.hdf'
##ASI
obj = NetCDFFile(AMSREfile)
asi = array(obj.variables['icecon'][:], dtype=float32) / 10
asi[asi == -100] = nan
###NASA-TEAM2
AMSRE_NASA = SD.SD(AMSREfile2)
nt2_key = 26 #'SI_12km_NH_89V_DSC'
bt_key = 29 #'SI_12km_NH_ICECON_DSC'
sd_ice_nt2 = AMSRE_NASA.select(nt2_key)
ice_nt2 = sd_ice_nt2.get()
ice_nt2 = array(ice_nt2, dtype=float)[::-1]
ice_nt2[ice_nt2 >= 101] = nan
###NASA-BOOTSTRAP
sd_ice_bt = AMSRE_NASA.select(bt_key)
ice_bt = sd_ice_bt.get() # stored as difference to NT2 data (ice_bst-ice_nt2)
ice_bt = array(ice_bt, dtype=float)[::-1]
ice_bt = ice_bt + ice_nt2
ice_bt[ice_bt >= 101] = nan
region = 'Arc'
def doCompress(compType, value=0, v2=0):
"""Create and validate an HDF file using a compression scheme
sepcified by the parameters"""
# Build a significant file name
if compType == SDC.COMP_NONE:
fileName = "SDS.COMP_NONE"
elif compType == SDC.COMP_RLE:
fileName = "SDS.COMP_RLE"
elif compType == SDC.COMP_SKPHUFF:
fileName = "SDS.COMP_SKPHUFF.%d" % value
elif compType == SDC.COMP_DEFLATE:
fileName = "SDS.COMP_DEFLATE.%d" % value
elif compType == SDC.COMP_SZIP:
fileName = "SDS.COMP_SZIP"
if value == SDC.COMP_SZIP_NN:
fileName += ".NN"
elif value == SDC.COMP_SZIP_EC:
fileName += ".EC"
else:
print "illegal value"
sys.exit(1)
fileName += ".%s" % v2
else:
print "illegal compType"
sys.exit(1)
fileName += ".hdf"
SDS_NAME = "Data"
fill_value = 0
#LENGTH = 9
#WIDTH = 6
#
#data = numpy.array( ((100,100,200,200,300,400),
# (100,100,200,200,300,400),
# (100,100,200,200,300,400),
# (300,300, 0,400,300,400),
# (300,300, 0,400,300,400),
# (300,300, 0,400,300,400),
# (0, 0,600,600,300,400),
# (500,500,600,600,300,400),
# (0, 0,600,600,300,400)), NUMPY_DATATYPE)
# The above dataset is used in the original NCSA example.
# It is too small to show a significant size reduction after
# compression. The following is used for a more realistic example.
data = numpy.zeros((LENGTH, WIDTH), NUMPY_DATATYPE)
for i in range(LENGTH):
for j in range(WIDTH):
data[i,j] = (i+j)*(i-j)
# Create HDF file, wiping it out it it already exists.
sd_id = SD(fileName, SDC.WRITE | SDC.CREATE | SDC.TRUNC)
# Create dataset.
sds_id = sd_id.create(SDS_NAME, HDF_DATATYPE, (LENGTH, WIDTH))
# Fill dataset will fill value.
sds_id.setfillvalue(0)
# Apply compression.
try:
sds_id.setcompress(compType, # compression type
value, v2) # args depend on compression type
except HDF4Error, msg:
print("Error compressing the dataset with params: "
"(%d,%d,%d) : %s" % (compType, value, v2, msg))
sds_id.endaccess()
sd_id.end()
os.remove(fileName)
return
def sdscreate(sd, name):
# Create a simple 3x3 float array.
sds = sd.create(name, SDC.FLOAT32, (3, 3))
# Initialize array
sds[:] = ((0, 1, 2), (3, 4, 5), (6, 7, 8))
# "close" dataset.
sds.endaccess()
# Create HDF file
filename = 'inventory.hdf'
hdf = HDF(filename, HC.WRITE | HC.CREATE)
# Initialize the SD, V and VS interfaces on the file.
sd = SD(filename, SDC.WRITE) # SD interface
vs = hdf.vstart() # vdata interface
v = hdf.vgstart() # vgroup interface
# Create vdata named 'INVENTORY'.
vdatacreate(vs, 'INVENTORY')
# Create dataset named "ARR_3x3"
sdscreate(sd, 'ARR_3x3')
# Attach the vdata and the dataset.
vd = vs.attach('INVENTORY')
sds = sd.select('ARR_3x3')
# Create vgroup named 'TOTAL'.
vg = v.create('TOTAL')
return
# Load data in the dataset.
sds_id[:] = data
# Close dataset.
sds_id.endaccess()
# Close hdf file to flush compressed data.
sd_id.end()
# Verify compressed data.
# ######################
# Reopen file and select first dataset.
sd_id = SD(fileName, SDC.READ)
sds_id = sd_id.select(0)
# Obtain compression info.
compInfo = sds_id.getcompress()
compType = compInfo[0]
print "file : %s" % fileName
print " size = %d" % os.path.getsize(fileName)
if compType == SDC.COMP_NONE:
print " compType = COMP_NONE"
elif compType == SDC.COMP_RLE:
print " compType = COMP_RLE"
elif compType == SDC.COMP_SKPHUFF:
print " compType = COMP_SKPHUFF"
print " dataSize = %d" % compInfo[1]
elif compType == SDC.COMP_DEFLATE:
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from pyhdf.SD import *
import sys
# np.set_printoptions(threshold='nan')
_filename = "C:\Users\AKO NA LNG\Desktop\Esquivel Files\School Files 2\Special Problem\MODIS\MOD35\MOD35_L2.A2015348.0215.006.2015348134053.hdf"
mod35 = SD(_filename, SDC.READ)
sds_name_cm = "Cloud_Mask"
sds_index_cm = mod35.select(sds_name_cm)
cm_sds_data = sds_index_cm.get()
sds_data_0 = cm_sds_data[0, :, :]
sds_data_0_bin = sds_data_0.astype(dtype=np.uint8, order="K", casting="unsafe", subok=False, copy=False)
for x in np.nditer(sds_data_0_bin, op_flags=["readwrite"]):
x[...] = np.right_shift(x, 6)
for y in np.nditer(sds_data_0_bin, op_flags=["readwrite"]):
if y[...] == 3 or y[...] == 2:
y[...] = 0
def run(FILE_NAME):
# Identify the data field.
DATAFIELD_NAME = 'RegBestEstimateSpectralOptDepth'
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data4D = hdf.select(DATAFIELD_NAME)
# Subset the Blue Band. 1=Blue, 2=Green, 3=Red, 4=NIR.
data = data4D[:,:,:,0].astype(np.double)
# Read geolocation dataset from HDF-EOS2 dumper output.
GEO_FILE_NAME = 'lat_MISR_AM1_AS_AEROSOL_P004_O066234_F12_0022.output'
GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'],
GEO_FILE_NAME)
lat = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0])
lat = lat.reshape(data.shape)
GEO_FILE_NAME = 'lon_MISR_AM1_AS_AEROSOL_P004_O066234_F12_0022.output'
GEO_FILE_NAME = os.path.join(os.environ['HDFEOS_ZOO_DIR'],
GEO_FILE_NAME)
lon = np.genfromtxt(GEO_FILE_NAME, delimiter=',', usecols=[0])
lon = lon.reshape(data.shape)
# Read attributes.
attrs = data4D.attributes(full=1)
fva=attrs["_FillValue"]
_FillValue = fva[0]
# Apply the fill value.
data[data == _FillValue] = np.nan
datam = np.ma.masked_array(data, mask=np.isnan(data))
nblocks = data.shape[0]
ydimsize = data.shape[1]
xdimsize = data.shape[2]
datam = datam.reshape(nblocks*ydimsize, xdimsize)
lat = lat.reshape(nblocks*ydimsize, xdimsize)
lon = lon.reshape(nblocks*ydimsize, xdimsize)
# Set the limit for the plot.
m = Basemap(projection='cyl', resolution='h',
llcrnrlat=np.min(lat), urcrnrlat = np.max(lat),
llcrnrlon=np.min(lon), urcrnrlon = np.max(lon))
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(-90., 120., 30.), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(-180., 181., 45.), labels=[0, 0, 0, 1])
m.pcolormesh(lon, lat, datam, latlon=True)
cb = m.colorbar()
cb.set_label('No Unit')
basename = os.path.basename(FILE_NAME)
plt.title('{0}\n{1} - Blue Band'.format(basename, DATAFIELD_NAME))
fig = plt.gcf()
# plt.show()
pngfile = "{0}.py.png".format(basename)
fig.savefig(pngfile)
