def read(self, filename, **kwargs):
"""Read the data"""
from pyhdf.SD import SD
import datetime
# print "*** >>> Read the hdf-eos file!"
root = SD(filename)
# Get all the Attributes:
# Common Attributes, Data Time,
# Data Structure and Scene Coordinates
for key in root.attributes().keys():
self._eoshdf_info[key] = root.attributes()[key]
# Start Time - datetime object
starttime = datetime.datetime.strptime(self._eoshdf_info['Start Time'][0:13],
"%Y%j%H%M%S")
msec = float(self._eoshdf_info['Start Time'][13:16]) / 1000.
self.starttime = starttime + datetime.timedelta(seconds=msec)
# End Time - datetime object
endtime = datetime.datetime.strptime(self._eoshdf_info['End Time'][0:13],
"%Y%j%H%M%S")
msec = float(self._eoshdf_info['End Time'][13:16]) / 1000.
self.endtime = endtime + datetime.timedelta(seconds=msec)
# What is the leading 'H' doing here?
sensor_name = self._eoshdf_info['Sensor Name'][1:-1].lower()
try:
self.satid = EOS_SATELLITE[sensor_name]
except KeyError:
LOG.error("Failed setting the satellite id - sat-name = ",
sensor_name)
self.orbit = self._eoshdf_info['Orbit Number']
self.shape = (self._eoshdf_info['Number of Scan Control Points'],
self._eoshdf_info['Number of Pixel Control Points'])
# try:
if 1:
value = root.select(self.name)
attr = value.attributes()
data = value.get()
self.attr = attr
band = data
if self.name in FLAGS_QUALITY:
self.data = band
else:
nodata = attr['bad_value_scaled']
self.data = (np.ma.masked_equal(band, nodata) *
attr['slope'] + attr['intercept'])
value.endaccess()
# except:
# pass
root.end()
self.filled = True
def read(self, filename, **kwargs):
"""Read the data"""
from pyhdf.SD import SD
import datetime
#print "*** >>> Read the hdf-eos file!"
root = SD(filename)
# Get all the Attributes:
# Common Attributes, Data Time,
# Data Structure and Scene Coordinates
for key in root.attributes().keys():
self._eoshdf_info[key] = root.attributes()[key]
# Start Time - datetime object
starttime = datetime.datetime.strptime(self._eoshdf_info['Start Time'][0:13],
"%Y%j%H%M%S")
msec = float(self._eoshdf_info['Start Time'][13:16])/1000.
self.starttime = starttime + datetime.timedelta(seconds=msec)
# End Time - datetime object
endtime = datetime.datetime.strptime(self._eoshdf_info['End Time'][0:13],
"%Y%j%H%M%S")
msec = float(self._eoshdf_info['End Time'][13:16])/1000.
self.endtime = endtime + datetime.timedelta(seconds=msec)
# What is the leading 'H' doing here?
sensor_name = self._eoshdf_info['Sensor Name'][1:-1].lower()
try:
self.satid = EOS_SATELLITE[sensor_name]
except KeyError:
LOG.error("Failed setting the satellite id - sat-name = ",
sensor_name)
self.orbit = self._eoshdf_info['Orbit Number']
self.shape = (self._eoshdf_info['Number of Scan Control Points'],
self._eoshdf_info['Number of Pixel Control Points'])
#try:
if 1:
value = root.select(self.name)
attr = value.attributes()
data = value.get()
self.attr = attr
band = data
if self.name in FLAGS_QUALITY:
self.data = band
else:
nodata = attr['bad_value_scaled']
self.data = (np.ma.masked_equal(band, nodata) *
attr['slope'] + attr['intercept'])
value.endaccess()
#except:
# pass
root.end()
self.filled= True
def shape(self):
from pyhdf.SD import SD
try:
return self._shape
except AttributeError:
n = 0
for f in self.filenames:
prod = SD(f)
n += prod.attributes()['Number of Scans']
self._shape = (10 * n, prod.attributes()['Max Earth Frames'])
return self._shape
def _l3read_hdf(self, fieldname='l3m_data'):
self.vprint( "Reading hdf file")
sd = SD(self.filename, SDC.READ)
ds = sd.select(fieldname)
field = ds[self.j1:self.j2, self.i1:self.i2].copy()
intercept = ds.attributes()['Intercept']
slope = ds.attributes()['Slope']
try:
nanval = ds.attributes()['Fill']
except:
nanval = 65535
try:
base = ds.attributes()['Base']
except KeyError:
base = -999
if 'Scaled Data Maximum' in sd.attributes().keys():
self.maxval = sd.attributes()['Scaled Data Maximum']
self.minval = sd.attributes()['Scaled Data Minimum']
elif 'Suggested Image Scaling Maximum' in sd.attributes().keys():
self.maxval = sd.attributes()['Suggested Image Scaling Maximum']
self.minval = sd.attributes()['Suggested Image Scaling Minimum']
else:
self.minval = self.vc[fld][2]
self.maxval = self.vc[fld][3]
start_iso = (pl.date2num(dtm(
sd.attributes()['Period Start Year'],1,1)) +
sd.attributes()['Period Start Day'] - 1)
end_iso = (pl.date2num(dtm(
sd.attributes()['Period End Year'],1,1)) +
sd.attributes()['Period End Day'] - 1)
self.jd = ((start_iso+end_iso)/2)
self.date = pl.num2date(self.jd)
return field,base,intercept,slope
def read_file_attr(in_file):
try:
hdf4 = SD(in_file, SDC.READ)
return attrs2dict(hdf4.attributes())
except Exception as why:
print 'ReadHDF4.read_file_attr: {}'.format(why)
return
def prepare(self):
"""
Public:
Handles reprojection of file, conversion of hdf into GeoTIFF
"""
data_file = SD(self.input_file, SDC.READ)
extracted_data = list()
for band in self.bands:
band_data = data_file.select(band)
extracted_data.append(band_data.get())
self.attributes = data_file.attributes()
data_file.end()
extracted_data = np.array(extracted_data)
extracted_data[np.where(
extracted_data <= self.low_thres)] = self.low_value
extracted_data = np.log(extracted_data)
extracted_data[np.where(extracted_data >= self.high_thres)] = HIGH_VAL
indices = np.where((extracted_data > self.low_thres)
& (extracted_data < self.high_thres))
extracted_data[indices] = (HIGH_VAL *
(extracted_data[indices] - self.low_thres) /
self.diff)
extracted_data = extracted_data.astype(rasterio.uint8)
file_name = self.input_file.split("/")[-1]
self.prepare_thumbnail(extracted_data, file_name)
def ler_hdf_precipitacao(arquivo, pasta):
diret = pasta + arquivo
hdf = SD(diret, SDC.READ)
#Conjuntos de dados disponíveis
#print(hdf.datasets())
#Atributos
#print(hdf.attributes())
#Obter resolução espacial e as coordenadas das "bordas"
lista = hdf.attributes()['GridHeader'].replace(';', '').split('\n')
atrs = {}
for atr in [
'LatitudeResolution=', 'LongitudeResolution=',
'NorthBoundingCoordinate=', 'SouthBoundingCoordinate=',
'EastBoundingCoordinate=', 'WestBoundingCoordinate='
]:
valor = [
float(lista[n][len(atr):]) for n, x in enumerate(lista) if atr in x
]
atrs[atr] = valor[0]
#Selecionar o conjunto de dados desejado ('precipitation'): dados de precipitação
precip = hdf.select('precipitation')
precip = precip[:] #Array com os dados de precipitação no formato nlon x nlat
precip = np.transpose(precip) #Matriz transposta para termos nlat x nlon
# Missing value = -9999.9; Set all the missing values less than 0 to NaNs
np.putmask(precip, precip < 0, np.nan)
return precip
def read_MODIS_file(fname):
os.chdir(DataDir)
hdf = SD(fname, SDC.READ)
metadata = hdf.attributes()
firepix = metadata['FirePix']
if firepix == 0:
print('no fire pixel in %s' % (fname))
hdf.end()
return []
print('%s fire pixel in %s' % (firepix, fname))
data = np.zeros((firepix, 5))
sds = hdf.select('FP_latitude')
latdata = sds.get()
sds = hdf.select('FP_longitude')
londata = sds.get()
sds = hdf.select('FP_power')
FRPdata = sds.get()
sds = hdf.select('FP_confidence')
FRPcondata = sds.get()
sds = hdf.select('FP_ViewZenAng')
vzadata = sds.get()
data[:, 0] = latdata
data[:, 1] = londata
data[:, 2] = FRPdata
data[:, 3] = vzadata
data[:, 4] = FRPcondata
hdf.end()
return data
def get_flip(filename):
hdf = SD(filename, SDC.READ)
global_attributes = hdf.attributes()
if 'CoreMetadata.0' in global_attributes:
# modis l1b
coreMetaData = global_attributes['CoreMetadata.0']
odlParser = MiniODLParser()
coreMetaDict = odlParser.parseFromString(coreMetaData, ['all'])
platform = coreMetaDict['ASSOCIATEDPLATFORMSHORTNAME:VALUE']
dnflag = coreMetaDict['DAYNIGHTFLAG:VALUE']
print >> stderr, "platform ", platform
print >> stderr, "dnflag ", dnflag
if platform == 'Aqua' and dnflag == 'Day':
print 'flipXY'
elif platform == 'Terra' and dnflag == 'Night':
print 'flipXY'
else:
startNode = global_attributes['Start Node']
print >> stderr, 'start node', startNode
endNode = global_attributes['End Node']
print >> stderr, 'end node', endNode
if startNode == 'Ascending' and endNode == 'Ascending':
print 'flipXY'
def ancillary_interp_ozone(file, lon, lat, dataset='ozone', kind='linear'):
from pyhdf.SD import SD, SDC
f = SD(file, SDC.READ)
datasets_dic = f.datasets()
meta = f.attributes()
sds_obj = f.select(dataset)
data = sds_obj.get()
f.end()
f = None
from numpy import linspace
from scipy import interpolate
## make lons and lats for this file
lons = linspace(meta["Westernmost Longitude"],
meta["Easternmost Longitude"],
num=meta['Number of Columns'])
lats = linspace(meta["Northernmost Latitude"],
meta["Southernmost Latitude"],
num=meta['Number of Rows'])
## do interpolation in space
if kind == 'nearest':
xi, xret = min(enumerate(lons), key=lambda x: abs(x[1] - float(lon)))
yi, yret = min(enumerate(lats), key=lambda x: abs(x[1] - float(lat)))
uoz = data[yi, xi] / 1000.
else:
interp = interpolate.interp2d(lons, lats, data, kind=kind)
uoz = (interp(lon, lat))[0] / 1000.
anc_ozone = {'ozone': {'interp': uoz}}
return (anc_ozone)
def get_sds(self, fieldnames=None):
"""Return specific or all SDS in the hdf file as dictionary.
SDS arrays can be accessed using the 'data' key. Note that no scaling
is applied to the data in get() method (use get_scaled() to achieve
that). However, the scaling and missing data information can be
accessed using the following keys:
'scale_factor'
'add_offset'
'_FillValue'
"""
# Convert scalar fieldnames to sequence
if isinstance(fieldnames, basestring):
fieldnames = (fieldnames, )
# Open file to read SDs
try:
h4 = SD(self.filename, mode=SDC.READ)
sclinfo = None
if 'Slope_and_Offset_Usage' in h4.attributes():
sclinfo = 'Slope_and_Offset_Usage'
# Get all available SDS from file if fieldnames in not given
if fieldnames is None:
fieldnames = []
for key in sorted(h4.datasets()):
fieldnames.append(key)
# Create and empty dataset dictionary with all available
# fields fill in data from SDS
sds = dict.fromkeys(fieldnames, {})
for key in sds:
attrs = h4.select(key).attributes()
if sclinfo:
attrs[sclinfo] = h4.attributes()[sclinfo]
sds[key] = attrs
sds[key]['data'] = h4.select(key).get()
# Close hdf interface
h4.end()
except HDF4Error as e:
raise HDF4Error(e)
# Return raw (possibly un-calibrated) SDS/attributes dictionary
return sds
class HDFeos:
def __init__(self, file_path):
self.lats = None
self.lons = None
self.gring_lats = None
self.gring_lons = None
self.hdf = SD(file_path)
self.file_path = file_path
def read_laton(self):
self.lons = self.hdf.select('Longitude').get().astype(numpy.double)
self.lats = self.hdf.select('Latitude').get().astype(numpy.double)
def get_metadata_group(self, group_name):
metadata_group = {}
keys = [s for s in self.hdf.attributes().keys() if group_name in s]
for key in keys:
string = self.hdf.attributes()[key]
m = parse_hdfeos_metadata(string)
metadata_group = {**metadata_group, **m}
return metadata_group
def create_sidecar(self, workers, cover_res, out_path):
sids = staremaster.conversions.latlon2stare(self.lats,
self.lons,
resolution=-1,
workers=workers,
adapt_resolution=True)
if not cover_res:
cover_res = staremaster.conversions.min_level(sids)
cover_sids = staremaster.conversions.gring2cover(
self.gring_lats, self.gring_lons, cover_res)
i = self.lats.shape[0]
j = self.lats.shape[1]
l = cover_sids.size
sidecar = Sidecar(self.file_path, out_path)
sidecar.write_dimensions(i, j, l, nom_res=self.nom_res)
sidecar.write_lons(self.lons, nom_res=self.nom_res)
sidecar.write_lats(self.lats, nom_res=self.nom_res)
sidecar.write_sids(sids, nom_res=self.nom_res)
sidecar.write_cover(cover_sids, nom_res=self.nom_res)
return sidecar
def __init__(self, filename, filename_info, filetype_info):
super(HDF4FileHandler, self).__init__(filename, filename_info, filetype_info)
self.file_content = {}
file_handle = SD(self.filename, SDC.READ)
self._collect_attrs('', file_handle.attributes())
for k, v in file_handle.datasets().items():
self.collect_metadata(k, file_handle.select(k))
del file_handle
def __init__(self, filename, filename_info, filetype_info):
super(HDF4FileHandler, self).__init__(filename, filename_info, filetype_info)
self.file_content = {}
file_handle = SD(self.filename, SDC.READ)
self._collect_attrs('', file_handle.attributes())
for k, v in file_handle.datasets().items():
self.collect_metadata(k, file_handle.select(k))
del file_handle
def get_colloc_crap(i, file_col):
from pyhdf.SD import SD, SDC
from libtools import dbg
dbg(file_col)
hdf = SD(file_col, SDC.READ)
x = hdf.select('Master_Index_1')[i]
a = hdf.select('Master_Index_2')[i]
return x, a, hdf.attributes()['fname_AIRS'] #_WRS
def extract_attributes(self):
# extract the attributes from the data file
try:
granule = SD(self.data_path, 1)
except TypeError:
granule = SD(self.data_path.encode("utf-8"), 1)
self.attributes = granule.attributes()
granule.end()
def get(self, fieldnames=[]):
'''Returns specific or all SDS in the hdf file as dictionary.
SDS arrays can be accessed using the 'data' key. Note that no scaling
is applied to the data in get() method (use get_scaled() to achieve
that). However, the scaling and missing data information can be
accessed using the following keys:
'scale_factor'
'add_offset'
'_FillValue'
'''
if not isinstance(fieldnames, list):
fieldnames = [fieldnames]
try:
# Open hdf4 interface in read mode
h4 = SD(self.filename, mode=SDC.READ)
sclinfo = None
if 'Slope_and_Offset_Usage' in h4.attributes():
sclinfo = 'Slope_and_Offset_Usage'
if len(fieldnames) == 0:
# Get all available field names from hdf
fieldnames = []
for key in sorted(h4.datasets()):
fieldnames.append(key)
# Create and empty dataset dictionary with all available
# fields fill in data from SDS
datasets = dict.fromkeys(fieldnames, {})
for key in datasets:
attrs = h4.select(key).attributes()
if sclinfo:
attrs[sclinfo] = h4.attributes()[sclinfo]
datasets[key] = attrs
datasets[key]['data'] = h4.select(key).get()
# Close hdf interface
h4.end()
except HDF4Error as e:
print "HDF4Error", e
sys.exit(1)
# Return un-calibrated datasets/attributes dictionary
return datasets
def open_file(filename):
A = SD(filename)
# retrieve data SDS
d = A.datasets()
sds_name = list(d.keys())[0] # name of sds. Dictionary method.
sds = A.select(sds_name)
pin = A.attributes()
return sds, pin
def open_file(filename):
A = SD(filename)
# retrieve data SDS
d = A.datasets()
sds_name = d.keys()[0] # name of sds. Dictionary method.
sds = A.select(sds_name)
pin = A.attributes()
return sds, pin
def get_core(filename):
"""
given the path to a Modis hdf4 file with a "CoreMetadata.0" attribute
return that value as a string
"""
filename = str(filename)
the_file = SD(filename, SDC.READ)
metaDat = the_file.attributes()["CoreMetadata.0"]
core_meta = str(metaDat).rstrip(" \t\r\n\0")
the_file.end()
return core_meta
def _populate_SD(self):
"""Populate SDs and their shape attributes"""
try:
h4 = SD(self.filename, mode=SDC.READ)
self.sds = sorted(h4.datasets().keys())
self.attr.append(h4.attributes())
for k, v in sorted(h4.datasets().viewitems()):
self.items.append((k, v[1]))
h4.end()
except HDF4Error as e:
raise HDF4Error('{}: {}'.format(e, self.filename))
def readHDF4(inFile, bands=[1]):
'''Reads HDF4 images as Raster instances using pyhdf'''
from pyhdf.SD import SD, SDC
g = SD(inFile, mode=SDC.READ)
cellSize = g.attributes()['PixelSize']
m = g.attributes()['StructMetadata.0']
start = m.find('(', m.find('UpperLeft')) + 1
Xll = float(m[start:m.find(',', start)])
start = m.find(',', m.find('LowerRight')) + 1
Yll = float(m[start:m.find(')', start)])
rastList = []
for band in bands:
b = g.select(band - 1)
attribs = b.attributes()
scale = 1.0
offset = 0.0
if attribs.has_key('scale_factor'):
scale = attribs['scale_factor']
if attribs.has_key('add_offset'):
offset = attribs['add_offset']
NoData = b.getfillvalue()
arr = ma.masked_values(b.get(), NoData)
arr = arr.astype('float32')
arr = scale * (arr - offset)
b.endaccess()
rastList.append(raster.Raster(arr, cellSize, Xll, Yll, NoData))
g.end()
return rastList
def load_thin_modis(satscene, options):
"""Read modis data from file and load it into *satscene*.
"""
filename = satscene.time_slot.strftime("thin_MYD021KM.A%Y%j.%H%M.005.NRT.hdf")
filename = os.path.join(options["dir"], filename)
data = SD(filename)
datasets = ['EV_250_Aggr1km_RefSB',
'EV_500_Aggr1km_RefSB',
'EV_1KM_RefSB',
'EV_1KM_Emissive']
for dataset in datasets:
subdata = data.select(dataset)
band_names = subdata.attributes()["band_names"].split(",")
if len(satscene.channels_to_load & set(band_names)) > 0:
uncertainty = data.select(dataset+"_Uncert_Indexes")
if dataset == 'EV_1KM_Emissive':
array = calibrate_tb(subdata, uncertainty)
else:
array = calibrate_refl(subdata, uncertainty)
for (i, band) in enumerate(band_names):
if band in satscene.channels_to_load:
satscene[band] = array[i]
mda = data.attributes()["CoreMetadata.0"]
orbit_idx = mda.index("ORBITNUMBER")
satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116]
lat, lon = get_lat_lon(satscene, None)
from pyresample import geometry
satscene.area = geometry.SwathDefinition(lons=lon, lats=lat)
# trimming out dead sensor lines
if satscene.satname == "aqua":
for band in ["6", "27"]:
if not satscene[band].is_loaded() or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene.area.lons[indices,:],
lats=satscene.area.lats[indices,:])
satscene[band].area.area_id = ("swath_" + satscene.fullname + "_"
+ str(satscene.time_slot) + "_"
+ str(satscene[band].shape) + "_"
+ str(band))
def getMetadata(filepath):
metadata = {}
hdf_file = SD(filepath, SDC.READ)
dataset = gdal.Open(filepath, gdal.GA_ReadOnly)
# Construct the grid. The needed information is in a global attribute
# called 'StructMetadata.0'. Use regular expressions to tease out the
# extents of the grid.
fattrs = hdf_file.attributes(full=1)
ga = fattrs["StructMetadata.0"]
gridmeta = ga[0]
ul_regex = re.compile(
r'''UpperLeftPointMtrs=\(
(?P<upper_left_x>[+-]?\d+\.\d+)
,
(?P<upper_left_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = ul_regex.search(gridmeta)
x0 = np.float(match.group('upper_left_x'))
y0 = np.float(match.group('upper_left_y'))
lr_regex = re.compile(
r'''LowerRightMtrs=\(
(?P<lower_right_x>[+-]?\d+\.\d+)
,
(?P<lower_right_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = lr_regex.search(gridmeta)
x1 = np.float(match.group('lower_right_x'))
y1 = np.float(match.group('lower_right_y'))
metadata['xmax'] = x0
metadata['xmin'] = x1
metadata['ymax'] = y1
metadata['ymin'] = y0
# determine the projection GCTP code from the grid metadata
proj_regex = re.compile(r'''Projection=(?P<projection>\w+)''', re.VERBOSE)
match = proj_regex.search(gridmeta)
proj = match.group('projection')
# support MODIS sinusoidal projection for now, add others later
if proj == 'GCTP_SNSOID':
sinu = pyproj.Proj("+proj=sinu +R=6371007.181 +nadgrids=@null +wktext")
wgs84 = pyproj.Proj("+init=EPSG:4326")
metadata['lonmin'], metadata['latmin'] = pyproj.transform(
sinu, wgs84, x0, y0)
metadata['lonmax'], metadata['latmax'] = pyproj.transform(
sinu, wgs84, x1, y1)
return metadata
def check_if_day(hdf_file):
hdf_filename = sys.argv[1]
hdf_file = SD(hdf_filename, SDC.READ)
core_metadata0_string = hdf_file.attributes()['CoreMetadata.0']
daynight_start_index = core_metadata0_string.find('DAYNIGHT')
#print('test: |' + core_metadata0_string[daynight_start_index:daynight_start_index+120] + '|')
daynight_info_block = core_metadata0_string[daynight_start_index:daynight_start_index+120]
is_day = (daynight_info_block.find('\"Day\"') != -1)
#print(is_day)
return is_day
def gethv(Datadir, x0, y0, **kwargs):
files = glob.glob(Datadir + '2008.06.18/*.hdf')
strhv = []
for Aerfile in files:
ds = SD(Aerfile, SDC.READ)
lookup1 = 'UpperLeftPointMtrs'
lookup2 = 'LowerRightMtrs'
StrMeta = ds.attributes()['StructMetadata.0']
for line in StrMeta.splitlines():
if lookup1 in line:
xy = (line.split('('))[1].split(',')
xul = np.float(xy[0])
yul = np.float((xy[1].split(')'))[0])
if lookup2 in line:
xy = (line.split('('))[1].split(',')
xlr = np.float(xy[0])
ylr = np.float((xy[1].split(')'))[0])
break
Aobj = ds.select('Optical_Depth_047')
AOD = Aobj.get()
[nx, ny] = AOD[0, :, :].shape
XDim = xul + (0.5 + np.arange(nx)) * (xlr - xul) / nx
YDim = yul + (0.5 + np.arange(ny)) * (ylr - yul) / ny
# YDim=np.flip(YDim)
xv, yv = np.meshgrid(XDim, YDim)
# In basemap, the sinusoidal projection is global, so we won't use it.
# Instead we'll convert the grid back to lat/lons.
sinu = pyproj.Proj("+proj=sinu +R=6371007.181 +nadgrids=@null +wktext")
wgs84 = pyproj.Proj("+proj=latlong +R=6371007.181")
lon, lat = pyproj.transform(sinu, wgs84, xv, yv)
ds.end()
if 'mindist' in kwargs:
mindist = kwargs['mindist']
distance = (lat - y0)**2 + (lon - x0)**2
if np.min(distance) < mindist:
strhv = np.append(strhv,
((Aerfile.split('/'))[-1].split('.'))[-4])
if 'dx' in kwargs:
dx = kwargs['dx']
dy = kwargs['dy']
if (np.min(np.absolute(lat - y0)) < dy) & (np.min(
np.absolute(lon - x0)) < dx):
strhv = np.append(strhv,
((Aerfile.split('/'))[-1].split('.'))[-4])
return strhv
def read_modis_sd(filename):
product = SD(filename)
Nfire = product.attributes()['FirePix']
if Nfire == 0:
return np.zeros((1, 8))
datasets = [
'FP_power', 'FP_latitude', 'FP_longitude', 'FP_ViewZenAng',
'FP_MeanT21', 'FP_MAD_T21', 'FP_sample', 'FP_T21'
]
datasetList = []
for sds in datasets:
selection = product.select(sds).get()
datasetList.append(selection)
return (np.array(datasetList).transpose())
def HDF_proj_WKT(hdf_filepath):
# returns the projection of the HDF file in Well Known Text (WKT) format
# first determine the HDF type
hdf_type = HDF_type(hdf_filepath)
if hdf_type == 'hdf4':
# for HDF4 assume corner coordinates are stored in the StructMetadata.0 section
hdf_file = SD(hdf_filepath, SDC.READ)
try:
# access grid metadata section of StructMetadata.0
fattr = hdf_file.attributes(full=1)
structmeta = fattr['StructMetadata.0']
gridmeta = structmeta[0]
# determine the projection GCTP code from the grid metadata
proj_regex = re.compile(r'''Projection=(?P<projection>\w+)''',
re.VERBOSE)
match = proj_regex.search(gridmeta)
proj = match.group('projection')
# support MODIS sinusoidal projection for now, add others later
if proj == 'GCTP_SNSOID':
sinu_proj4 = "+proj=sinu +R=6371007.181 +nadgrids=@null +wktext"
srs = osr.SpatialReference()
srs.ImportFromProj4(sinu_proj4)
return srs.ExportToWkt()
except:
#prjfile = open('/home/rkalyana/GeoEDF/GeoEDF/connector/filter/modis/6933.prj', 'r')
#prj_txt = prjfile.read()
#srs = osr.SpatialReference()
#srs.ImportFromESRI([prj_txt])
#prjfile.close()
#return srs.ExportToWkt()
raise GeoEDFError(
'Error determining the projection or unsupported projection')
else: # HDF5 file; only SMAP files in EASE Grid 2.0 are supported at the moment
hdf_file = h5py.File(hdf_filepath, mode='r')
# check to see if this is a EASE Grid 2.0 file
if 'EASE2_global_projection' in hdf_file.keys():
ease_proj4 = "+proj=cea +lat_0=0 +lon_0=0 +lat_ts=30 +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=m"
srs = osr.SpatialReference()
srs.ImportFromProj4(ease_proj4)
return srs.ExportToWkt()
else:
raise GeoEDFError(
'Error determining the projection or unsupported projection')
def read_hdf4_info(input_file_path) :
"""
get information about variable names and attributes (both global and variable specific) from the
given file. The file is assumed to exist and be a valid hdf4 file
returns something in the form:
{
GLOBAL_ATTRS_KEY : a dictionary of attribute values keyed by the attribute names
VAR_LIST_KEY : [list of variable names]
VAR_INFO_KEY : {
<var_name> : {
SHAPE_KEY: (shape of variable data)
VAR_ATTRS_KEY: a dictionary of attribute values keyed by the attribute names
}
}
}
TODO, depending on what changes need to be made for CF compliance this data structure may need to change a lot
"""
file_info = { }
# open the file
file_object = SD(input_file_path, SDC.READ)
# get information on the global attributes in the file
global_attrs = file_object.attributes()
file_info[GLOBAL_ATTRS_KEY] = global_attrs
# get information on the variables in the file
variable_list = file_object.datasets().keys()
file_info[VAR_LIST_KEY] = variable_list
# for each variable in a file, get more specific information about it
file_info[VAR_INFO_KEY] = { }
sets_temp = file_object.datasets()
# this should return a dictionary with entries for each variable in the form
# <variable name>: ((dimension names), (data shape), type, index num)
for var_name in variable_list :
var_object = file_object.select(var_name)
var_attrs = var_object.attributes()
file_info[VAR_INFO_KEY][var_name] = {
SHAPE_KEY: sets_temp[var_name][1],
VAR_ATTRS_KEY: var_attrs,
}
return file_info, file_object
def set_file_attr(self):
"""
get hdf5 file attrs self.file_attr
:return:
"""
if self.resolution == 13500:
satellite_type = ['AQUA', 'TERRA']
if self.satellite in satellite_type:
h4r = SD(self.in_file, SDC.READ)
self.file_attr = attrs2dict(h4r.attributes())
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
else:
raise ValueError("Cant handle this resolution: ".format(
self.resolution))
def read_amsr_hdf4(filename):
from pyhdf.SD import SD, SDC
from pyhdf.HDF import HDF, HC
import pyhdf.VS
retv = AmsrObject()
h4file = SD(filename, SDC.READ)
datasets = h4file.datasets()
attributes = h4file.attributes()
#for idx,attr in enumerate(attributes.keys()):
# print idx, attr
for sds in ["Longitude", "Latitude", "High_res_cloud"]:
data = h4file.select(sds).get()
if sds in ["Longitude", "Latitude"]:
retv.all_arrays[sds.lower()] = data.ravel()
elif sds in ["High_res_cloud"]:
lwp_gain = h4file.select(sds).attributes()['Scale']
retv.all_arrays["lwp_mm"] = data.ravel() * lwp_gain
#print h4file.select(sds).info()
h4file = HDF(filename, SDC.READ)
vs = h4file.vstart()
data_info_list = vs.vdatainfo()
#print "1D data compound/Vdata"
for item in data_info_list:
#1D data compound/Vdata
name = item[0]
#print name
if name in ["Time"]:
data_handle = vs.attach(name)
data = np.array(data_handle[:])
retv.all_arrays["sec1993"] = data
data_handle.detach()
else:
pass
#print name
#data = np.array(data_handle[:])
#attrinfo_dic = data_handle.attrinfo()
#factor = data_handle.findattr('factor')
#offset = data_handle.findattr('offset')
#print data_handle.factor
#data_handle.detach()
#print data_handle.attrinfo()
h4file.close()
#for key in retv.all_arrays.keys():
# print key, retv.all_arrays[key]
return retv
def get_corners(hdf: SD) -> Tuple[Tuple[float, float], Tuple[float, float]]:
metadata_str = hdf.attributes()['StructMetadata.0'].replace('\x00', '')
wtf = metadata_str.replace('\t', '').split('\n')
points = [
x for x in wtf if 'UpperLeftPointMtrs' in x or 'LowerRightMtrs' in x
]
assert len(points) == 4
a, b = points[0].split('=')
assert a == 'UpperLeftPointMtrs'
UpperLeftPointMtrs = eval(b)
a, b = points[1].split('=')
assert a == 'LowerRightMtrs'
LowerRightMtrs = eval(b)
return UpperLeftPointMtrs, LowerRightMtrs
class MODIS02File(object):
""" """
def __init__(self, modis02fname):
""" """
self._sd = SD(modis02fname, SDC.READ)
def close(self):
""" """
self._sd.end()
def read_radiance(self, band):
""" """
sds = self._sd.select('EV_1KM_Emissive')
att = sds.attributes()
bands = np.array(att['band_names'].split(','), dtype='int32')
iband = int(np.where(bands == band)[0][0])
scale = att['radiance_scales'][iband]
offset = att['radiance_offsets'][iband]
fillvalue = att['_FillValue']
rad = (ma.masked_equal(sds[iband, :, :], fillvalue).astype('float32') \
- offset) * scale
return rad
def read_attributes(self):
attrs = {}
sdattrs = self._sd.attributes()
coremd = sdattrs['CoreMetadata.0'].split('\n')
for iline, line in enumerate(coremd):
if 'OBJECT = RANGEENDINGDATE' in line:
enddate = coremd[iline + 2].split('"')[1]
elif 'OBJECT = RANGEENDINGTIME' in line:
endtime = coremd[iline + 2].split('"')[1]
elif 'OBJECT = RANGEBEGINNINGDATE' in line:
begindate = coremd[iline + 2].split('"')[1]
elif 'OBJECT = RANGEBEGINNINGTIME' in line:
begintime = coremd[iline + 2].split('"')[1]
elif 'OBJECT = ASSOCIATEDSENSORSHORTNAME' in line:
attrs['sensor'] = coremd[iline + 3].split('"')[1]
elif 'OBJECT = ASSOCIATEDPLATFORMSHORTNAME' in line:
attrs['platform'] = coremd[iline + 3].split('"')[1]
attrs['start_time'] = datetime.strptime(begindate + 'T' + begintime,
'%Y-%m-%dT%H:%M:%S.%f')
attrs['stop_time'] = datetime.strptime(enddate + 'T' + endtime,
'%Y-%m-%dT%H:%M:%S.%f')
return attrs
def set_file_attr(self):
"""
根据 self.file_level_1 获取 L1 文件的属性
set self.level_1_attr
储存格式是字典
:return:
"""
if self.resolution == 1000:
satellite_type1 = ['FY1C', 'FY1D']
if self.satellite in satellite_type1:
hdf4 = SD(self.in_file, SDC.READ)
self.file_attr = attrs2dict(hdf4.attributes())
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
else:
raise ValueError("Cant handle this resolution: ".format(
self.resolution))
def is_daily_product(hdf_filename):
hdf_file = SD(hdf_filename, SDC.READ)
# get CoreMetadata.0 attribute (which is a very long string) and just extract and analyze the substring
# with the DAYNIGHT info:
core_metadata0_string = hdf_file.attributes()['CoreMetadata.0']
daynight_start_index = core_metadata0_string.find('DAYNIGHT')
if daynight_start_index != -1:
#print('test: |' + core_metadata0_string[daynight_start_index:daynight_start_index+120] + '|')
daynight_info_block = core_metadata0_string[daynight_start_index:daynight_start_index+120]
is_daily = (daynight_info_block.find('\"Day\"') != -1)
#print(is_daily)
return is_daily
else:
return false
def extractEVDCData(fileAbsPath):
inFile = SD(fileAbsPath, SDC.READ)
outFileName = os.path.split(fileAbsPath)[1][0:-3] + 'xml'
outFile = open(outFileName, 'w')
siteName = inFile.attributes(True)['DATA_LOCATION'][0]
siteLatitude = inFile.select('LATITUDE.INSTRUMENT')[0]
siteLongitude = inFile.select('LONGITUDE.INSTRUMENT')[0]
siteElevation = inFile.select('ALTITUDE.INSTRUMENT')[0]
outFile.write('<EVDC>\n')
outFile.write(' <siteName>' + siteName + '</siteName>\n')
outFile.write(' <siteLatitude>' + str(siteLatitude) + '</siteLatitude>\n')
outFile.write(' <siteLongitude>' + str(siteLongitude) + '</siteLongitude>\n')
outFile.write(' <siteElevation>' + str(siteElevation) + '</siteElevation>\n')
timestamps = inFile.select('DATETIME')
timestampCount = timestamps.info()[2]
so2absorption = inFile.select('SO2.COLUMN_ABSORPTION.SOLAR')
field = so2absorption.attributes(True)['VAR_NAME'][0]
for i in range(0, timestampCount):
referenceDate = datetime.date(2000,1,1)
days = int(timestamps[i])
hours = int((timestamps[i]-days) * 24)
minutes = int(((timestamps[i]-days)*24-hours) * 60)
seconds = int(((((timestamps[i]-days)*24-hours)*60)-minutes) * 60)
date = referenceDate + datetime.timedelta(days)
measurementTime = date.strftime('%Y-%m-%dT') + str(hours).zfill(2) + ':' + str(minutes).zfill(2) + ':' + str(seconds).zfill(2) + 'Z'
outFile.write(' <data>\n')
outFile.write(' <timeStart>' + measurementTime + '</timeStart>\n')
outFile.write(' <timeEnd>' + measurementTime + '</timeEnd>\n')
outFile.write(' <field>'+ field + '</field>\n')
outFile.write(' <value>' + str(so2absorption[i]) + '</value>\n')
outFile.write(' </data>\n')
outFile.write('</EVDC>')
outFile.close()
return [outFileName]
def read_calipso_hdf4(filename, retv):
from pyhdf.SD import SD, SDC
from pyhdf.HDF import HDF, HC
import pyhdf.VS
def convert_data(data):
if len(data.shape) == 2:
if data.shape[1] == 1:
return data[:, 0]
elif data.shape[0] == 1:
return data[0, :]
return data
if filename is not None:
h4file = SD(filename, SDC.READ)
datasets = h4file.datasets()
attributes = h4file.attributes()
singleshotdata = {}
for idx, dataset in enumerate(datasets.keys()):
#non-goups
if dataset in scip_these_larger_variables_until_needed.keys():
continue
elif dataset[0:8] == 'Surface_':
continue
if dataset in ["ssNumber_Layers_Found",
"ssLayer_Base_Altitude",
"ssLayer_Top_Altitude"]:
singleshotdata[dataset] = h4file.select(dataset).get()
if dataset[0:2] == "ss":
#already saved temporarly what we need
continue
name = dataset.lower()
#print idx, dataset
if dataset in atrain_match_names.keys():
name = atrain_match_names[dataset]
data = np.array(h4file.select(dataset).get())
setattr(retv, name, data)
if "ssNumber_Layers_Found" in singleshotdata.keys():
# Extract number of cloudy single shots (max 15)
# plus average cloud base and top
# in 5 km FOV
logger.info("Reading single shot information")
retv = rearrange_calipso_the_single_shot_info(
retv,
singleshotdata)
return retv
def _read_hdf(self, fieldname='l3m_data'):
sd = SD(self.filename, SDC.READ)
ds = sd.select(fieldname)
attrdict = ds.attributes()
for d in attrdict.keys(): attrdict[d.lower()] = attrdict.pop(d)
field = ds[self.j1:self.j2, self.i1:self.i2].copy()
intercept = ds.attributes()['intercept']
slope = ds.attributes()['slope']
try:
nanval = ds.attributes()['fill']
except:
nanval = ds.attributes()['bad_value_scaled']
try:
base = ds.attributes()['base']
except KeyError:
base = -999
dstr = sd.attributes()['Start Time']
self.jd = pl.datestr2num(dstr[:4] + "-1-1") + float(dstr[4:7]) - 1
return field,base,intercept,slope,nanval
def write_interpolated(filename, f0, f1, fact, datasetNames):
'''
interpolate two hdf files f0 and f1 using factor fact, and
write the result to filename
'''
hdf = SD(filename, SDC.WRITE|SDC.CREATE)
for datasetName in datasetNames:
try:
info = SD(f0).select(datasetName).info()
except:
print >> stderr, 'Error loading %s in %s' % (datasetName, f0)
raise
typ = info[3]
shp = info[2]
sds_in1 = SD(f0).select(datasetName)
met0 = sds_in1.get()
met1 = SD(f1).select(datasetName).get()
interp = (1-fact)*met0 + fact*met1
interp = interp.astype({
SDC.INT16: 'int16',
SDC.FLOAT32: 'float32',
SDC.FLOAT64: 'float64',
}[typ])
# write
sds = hdf.create(datasetName, typ, shp)
sds[:] = interp[:]
# copy attributes
attr = sds_in1.attributes()
if len(attr) > 0:
for name in attr.keys():
setattr(sds, name, attr[name])
sds.endaccess()
hdf.end()
def run(FILE_NAME):
# Identify the data field.
DATAFIELD_NAME = 'Day_CMG_Snow_Cover'
if USE_GDAL:
import gdal
GRID_NAME = 'MOD_CMG_Snow_5km'
gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME,
GRID_NAME,
DATAFIELD_NAME)
gdset = gdal.Open(gname)
data = gdset.ReadAsArray()
# Read projection parameters.
x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform()
nx, ny = (gdset.RasterXSize, gdset.RasterYSize)
del gdset
else:
from pyhdf.SD import SD, SDC
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data2D = hdf.select(DATAFIELD_NAME)
data = data2D[:,:].astype(np.float64)
# Read global attribute.
fattrs = hdf.attributes(full=1)
ga = fattrs["StructMetadata.0"]
gridmeta = ga[0]
# Read projection parameters.
# The needed information is in a global attribute called 'StructMetadata.0'.
# Use regular expressions to tease out the extents of the grid.
ul_regex = re.compile(r'''UpperLeftPointMtrs=\(
(?P<upper_left_x>[+-]?\d+\.\d+)
,
(?P<upper_left_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = ul_regex.search(gridmeta)
x0 = np.float(match.group('upper_left_x')) / 1e6
y0 = np.float(match.group('upper_left_y')) / 1e6
lr_regex = re.compile(r'''LowerRightMtrs=\(
(?P<lower_right_x>[+-]?\d+\.\d+)
,
(?P<lower_right_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = lr_regex.search(gridmeta)
x1 = np.float(match.group('lower_right_x')) / 1e6
y1 = np.float(match.group('lower_right_y')) / 1e6
ny, nx = data.shape
xinc = (x1 - x0) / nx
yinc = (y1 - y0) / ny
# Construct the grid. It's already in lat/lon.
x = np.linspace(x0, x0 + xinc*nx, nx)
y = np.linspace(y0, y0 + yinc*ny, ny)
lon, lat = np.meshgrid(x, y)
m = Basemap(projection='cyl', resolution='l',
llcrnrlat=-90, urcrnrlat = 90,
llcrnrlon=-180, urcrnrlon = 180)
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(-90., 120., 30.), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(-180, 180., 45.), labels=[0, 0, 0, 1])
# Bin the data as follows:
# 0% snow
# 1-99% snow
# 100% snow
# lake ice (107)
# night (111)
# cloud-obscured water (250)
# water mask (254)
# fill (255)
lst = ['#00ff00',
'#888888',
'#ffffff',
'#ffafff',
'#000000',
'#63c6ff',
'#0000cc',
'#8928dd']
cmap = mpl.colors.ListedColormap(lst)
bounds = [0, 1, 100, 107, 111, 250, 254, 255, 256]
norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
# Render the image in the projected coordinate system.
m.pcolormesh(lon[::2,::2], lat[::2,::2], data[::2,::2],
latlon=True, cmap=cmap, norm=norm)
long_name = 'Day CMG Snow Cover'
basename = os.path.basename(FILE_NAME)
plt.title('{0}\n{1}'.format(basename, long_name))
fig = plt.gcf()
color_bar = plt.colorbar(orientation='horizontal')
color_bar.set_ticks([0.5, 50, 103, 109, 180, 252, 254.5, 255.5])
color_bar.set_ticklabels(['0%\nsnow', '1-99%\nsnow', '100%\nsnow', 'lake\nice',
'night', 'cloud\n-obscured\nwater', 'water\nmask',
'fill'])
color_bar.draw_all()
# plt.show()
pngfile = "{0}.py.png".format(basename)
fig.savefig(pngfile)
import matplotlib as mpl import matplotlib.pyplot as plt from mpl_toolkits.basemap import Basemap import numpy as np from pyhdf.SD import SD, SDC # Open HDF4 file. FILE_NAME = 'S1999001.L3m_DAY_CHL_chlor_a_9km.hdf' hdf = SD(FILE_NAME, SDC.READ) # The lat and lon should be calculated using lat and lon of southwest point. # Then, we need number of lines and columns to calculate the lat and lon # step. Assume even space between lat and lon points to get all lat and lon # data. # Extract southwest point from global attribute. gattrs = hdf.attributes(full=1) swlata = gattrs['SW Point Latitude'] swlat = swlata[0] swlona = gattrs['SW Point Longitude'] swlon = swlona[0] linesa = gattrs['Number of Lines'] nlat = linesa[0] colsa = gattrs['Number of Columns'] nlon = colsa[0] latstepa = gattrs['Latitude Step'] latstep = latstepa[0] lonstepa = gattrs['Longitude Step'] lonstep = lonstepa[0] nmlat = swlat + (nlat)*latstep;
def run(FILE_NAME):
DATAFIELD_NAME = 'Local albedo average - 1 deg'
from pyhdf.SD import SD, SDC
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data3D = hdf.select(DATAFIELD_NAME)
data = data3D[:,:,3].astype(np.double)
# Read file attribute.
fattrs = hdf.attributes(full=1)
ga = fattrs["StructMetadata.0"]
gridmeta = ga[0]
# Read dataset attribute.
attrs = data3D.attributes(full=1)
fva=attrs["_FillValue"]
_FillValue = fva[0]
# Apply the fill value.
data[data == _FillValue] = np.nan
datam = np.ma.masked_array(data, mask=np.isnan(data))
# Construct the grid. The needed information is in a global attribute
# called 'StructMetadata.0'. Use regular expressions to tease out the
# extents of the grid.
ul_regex = re.compile(r'''UpperLeftPointMtrs=\(
(?P<upper_left_x>[+-]?\d+\.\d+)
,
(?P<upper_left_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = ul_regex.search(gridmeta)
x0 = np.float(match.group('upper_left_x')) / 1000000.0
y0 = np.float(match.group('upper_left_y')) / 1000000.0
lr_regex = re.compile(r'''LowerRightMtrs=\(
(?P<lower_right_x>[+-]?\d+\.\d+)
,
(?P<lower_right_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = lr_regex.search(gridmeta)
x1 = np.float(match.group('lower_right_x')) / 1000000.0
y1 = np.float(match.group('lower_right_y')) / 1000000.0
data = data.T
nx, ny = data.shape
x = np.linspace(x0, x1, nx)
y = np.linspace(y0, y1, ny)
lon, lat = np.meshgrid(x, y)
m = Basemap(projection='cyl', resolution='l',
llcrnrlat=-90, urcrnrlat = 90,
llcrnrlon=-180, urcrnrlon = 180)
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(-90., 120., 30.), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(-180., 181., 45.), labels=[0, 0, 0, 1])
m.pcolormesh(lon, lat, datam, latlon=True)
cb = m.colorbar()
# Albedo doesn't have a unit according to the specification [1].
cb.set_label('No Units')
basename = os.path.basename(FILE_NAME)
plt.title('{0}\n{1} at Band=3'.format(basename, DATAFIELD_NAME))
fig = plt.gcf()
# plt.show()
pngfile = "{0}.py.png".format(basename)
fig.savefig(pngfile)
class HDFEOSFileReader(BaseFileHandler):
def __init__(self, filename, filename_info, filetype_info):
self.filename = filename
try:
self.sd = SD(str(self.filename))
except HDF4Error as err:
raise ValueError("Could not load data from " + str(self.filename)
+ ": " + str(err))
self.sd = SD(self.filename)
self.mda = self.read_mda(self.sd.attributes()['CoreMetadata.0'])
self.mda.update(self.read_mda(
self.sd.attributes()['StructMetadata.0']))
self.mda.update(self.read_mda(
self.sd.attributes()['ArchiveMetadata.0']))
@property
def start_time(self):
date = (self.mda['INVENTORYMETADATA']['RANGEDATETIME']['RANGEBEGINNINGDATE']['VALUE'] + ' ' +
self.mda['INVENTORYMETADATA']['RANGEDATETIME']['RANGEBEGINNINGTIME']['VALUE'])
return datetime.strptime(date, '%Y-%m-%d %H:%M:%S.%f')
@property
def end_time(self):
date = (self.mda['INVENTORYMETADATA']['RANGEDATETIME']['RANGEENDINGDATE']['VALUE'] + ' ' +
self.mda['INVENTORYMETADATA']['RANGEDATETIME']['RANGEENDINGTIME']['VALUE'])
return datetime.strptime(date, '%Y-%m-%d %H:%M:%S.%f')
def read_mda(self, attribute):
lines = attribute.split('\n')
mda = {}
current_dict = mda
path = []
for line in lines:
if not line:
continue
if line == 'END':
break
key, val = line.split('=')
key = key.strip()
val = val.strip()
try:
val = eval(val)
except NameError:
pass
if key in ['GROUP', 'OBJECT']:
new_dict = {}
path.append(val)
current_dict[val] = new_dict
current_dict = new_dict
elif key in ['END_GROUP', 'END_OBJECT']:
if val != path[-1]:
raise SyntaxError
path = path[:-1]
current_dict = mda
for item in path:
current_dict = current_dict[item]
elif key in ['CLASS', 'NUM_VAL']:
pass
else:
current_dict[key] = val
return mda
def run(FILE_NAME):
# Identify the data field.
DATAFIELD_NAME = 'High_res_cloud'
if USE_GDAL:
import gdal
GRID_NAME = 'GlobalGrid'
gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME,
GRID_NAME,
DATAFIELD_NAME)
gdset = gdal.Open(gname)
data = gdset.ReadAsArray().astype(np.float64)
# Read projection parameters.
meta = gdset.GetMetadata()
x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform()
nx, ny = (gdset.RasterXSize, gdset.RasterYSize)
# Read attributes.
units = meta['Unit']
scale_factor = float(meta['Scale'])
# Construct the grid. The projection is GEO, so this immediately
# gives us latitude and longitude.
x = np.linspace(x0, x0 + xinc*nx, nx)
y = np.linspace(y0, y0 + yinc*ny, ny)
del gdset
else:
from pyhdf.SD import SD, SDC
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data2D = hdf.select(DATAFIELD_NAME)
data = data2D[:,:].astype(np.float64)
# Read global attribute.
fattrs = hdf.attributes(full=1)
ga = fattrs["StructMetadata.0"]
gridmeta = ga[0]
# Construct the grid. The needed information is in a global attribute
# called 'StructMetadata.0'. Use regular expressions to tease out the
# extents of the grid.
ul_regex = re.compile(r'''UpperLeftPointMtrs=\(
(?P<upper_left_x>[+-]?\d+\.\d+)
,
(?P<upper_left_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = ul_regex.search(gridmeta)
x0 = np.float(match.group('upper_left_x')) / 1e6
y0 = np.float(match.group('upper_left_y')) / 1e6
lr_regex = re.compile(r'''LowerRightMtrs=\(
(?P<lower_right_x>[+-]?\d+\.\d+)
,
(?P<lower_right_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = lr_regex.search(gridmeta)
x1 = np.float(match.group('lower_right_x')) / 1e6
y1 = np.float(match.group('lower_right_y')) / 1e6
ny, nx = data.shape
x = np.linspace(x0, x1, nx)
y = np.linspace(y0, y1, ny)
# Retrieve attributes.
attrs = data2D.attributes(full=1)
sfa=attrs["Scale"]
scale_factor = sfa[0]
ua=attrs["Unit"]
units = ua[0]
# Apply the attributes information.
data[data == -9999] = np.nan
data = data * scale_factor
data = np.ma.masked_array(data, np.isnan(data))
long_name = DATAFIELD_NAME
longitude, latitude = np.meshgrid(x, y)
m = Basemap(projection='cyl', resolution='l', lon_0=0,
llcrnrlat=-90, urcrnrlat = 90,
llcrnrlon=-180, urcrnrlon = 180)
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(-90, 91, 45), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(-180, 181, 45), labels=[0, 0, 0, 1])
m.pcolormesh(longitude, latitude, data, latlon=True)
cb = m.colorbar()
cb.set_label(units)
basename = os.path.basename(FILE_NAME)
plt.title('{0}\n{1}'.format(basename, long_name))
fig = plt.gcf()
# plt.show()
pngfile = "{0}.py.png".format(basename)
fig.savefig(pngfile)
def modis_oc(file_name,
limit=None,
flagLevel=None,
param='sst'):
# ;Example of setting up flag level for non SST images
# ;--> cf http://oceancolor.gsfc.nasa.gov/DOCS/Ocean_Level-2_Data_Products.pdf
# flagtable=REPLICATE(0B,32) ;32 bits flag
# ;flaglevel[[0,1,2,3,4,5,8,9,10,12,14,15,16,17,18,19,21,22,23,25,26,28,29]]=1B ;RQ: remove 1 for table correspondance in documentation (table 3)
#
# IF (~exist(sfglevel)) THEN sfglevel=1
#
# case (flaglevel) of
# 1: flagTable[[ 1, 3, 5, 15,16, 19, 25,26]]=1B
# 2: flagTable[[0,1, 3,4,5,8,9,10, 14,15,16, 19,21,22,23,25,26]]=1B
# 3: flagTable[[0,1, 3,4,5,8,9,10, 14,15,16, 19,21,22,23,25,26]]=1B
# else: flagTable[[0,1, 3,4,5,8,9,10, 14,15,16, 19,21,22,23,25,26]]=1B
# endcase
#
#; flaglevel[[ 1, 3, 5, 12, 15,16, 19, 25,26]]=1B ;low value flag
#; ;flaglevel[[0,1, 3,4,5,8,9,10,12,14,15,16, 19,21,22,23,25,26]]=1B ;operational flag (strong removal)
#Read MODIS HDF4 data
f = SD(file_name, SDC.READ)
fattr=f.attributes()
#Load coordinates
lon=f.select('longitude')
lat=f.select('latitude')
#Shrink image
#############
#scene dimensions
info_sst=f.datasets()[param]
dnames=info_sst[0]
d=info_sst[1]
lonvec=lon.get().reshape(d[0]*d[1])
latvec=lat.get().reshape(d[0]*d[1])
#Get points within domain
if limit is not None :
indvec,flagvec=in_limits(lonvec, latvec, limit)
flagmat=flagvec.reshape(d[0],d[1])
rowsum=np.sum(flagmat, 0)
colsum=np.sum(flagmat, 1)
yflag=rowsum >= 1
xflag=colsum >= 1
xid=np.arange(d[0])
xid=xid.compress(xflag)
xcnt=int(xid.size)
xst=int(xid.min())
yid=np.arange(d[1])
yid=yid.compress(yflag)
ycnt=int(yid.size)
yst=int(yid.min())
#Shrink lon & lat
lon_var=lon.get(start=[xst,yst], count=[xcnt,ycnt])
lat_var=lat.get(start=[xst,yst], count=[xcnt,ycnt])
#Load SST image
###############
sst=f.select(param)
attr=sst.attributes()
slope=attr['slope']
intercept=attr['intercept']
flagValue=attr['bad_value_scaled']
sst_var=sst.get(start=[xst,yst], count=[xcnt,ycnt]) #Shrink sst image
#Compute mask
fg=f.select('qual_'+param)
fg_var=fg.get(start=[xst,yst], count=[xcnt,ycnt])
if flagLevel is None :
mask=sst_var == flagValue
else :
mask= (sst_var == flagValue) | (fg_var >= flagLevel)
sst_var=np.ma.masked_array(sst_var*slope + intercept, mask=mask,type=float)
def load_generic(satscene, filename, resolution, cores):
"""Read modis data, generic part.
"""
try:
data = SD(str(filename))
except HDF4Error as err:
logger.warning("Could not load data from " + str(filename)
+ ": " + str(err))
return
datadict = {
1000: ['EV_250_Aggr1km_RefSB',
'EV_500_Aggr1km_RefSB',
'EV_1KM_RefSB',
'EV_1KM_Emissive'],
500: ['EV_250_Aggr500_RefSB',
'EV_500_RefSB'],
250: ['EV_250_RefSB']}
datasets = datadict[resolution]
loaded_bands = []
# process by dataset, reflective and emissive datasets separately
for dataset in datasets:
subdata = data.select(dataset)
band_names = subdata.attributes()["band_names"].split(",")
if len(satscene.channels_to_load & set(band_names)) > 0:
# get the relative indices of the desired channels
indices = [i for i, band in enumerate(band_names)
if band in satscene.channels_to_load]
uncertainty = data.select(dataset + "_Uncert_Indexes")
if dataset.endswith('Emissive'):
array = calibrate_tb(subdata, uncertainty, indices, band_names)
else:
array = calibrate_refl(subdata, uncertainty, indices)
for (i, idx) in enumerate(indices):
satscene[band_names[idx]] = array[i]
# fix the resolution to match the loaded data.
satscene[band_names[idx]].resolution = resolution
loaded_bands.append(band_names[idx])
# Get the orbit number
if not satscene.orbit:
mda = data.attributes()["CoreMetadata.0"]
orbit_idx = mda.index("ORBITNUMBER")
satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116]
# Get the geolocation
# if resolution != 1000:
# logger.warning("Cannot load geolocation at this resolution (yet).")
# return
lat, lon = get_lat_lon(satscene, resolution, filename, cores)
area = geometry.SwathDefinition(lons=lon, lats=lat)
for band_name in loaded_bands:
satscene[band_name].area = area
# Trimming out dead sensor lines (detectors) on aqua:
# (in addition channel 21 is noisy)
if satscene.satname == "aqua":
for band in ["6", "27", "36"]:
if not satscene[band].is_loaded() or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
satscene[band].area.area_id = ("swath_" + satscene.fullname + "_"
+ str(satscene.time_slot) + "_"
+ str(satscene[band].shape) + "_"
+ str(band))
# Trimming out dead sensor lines (detectors) on terra:
# (in addition channel 27, 30, 34, 35, and 36 are nosiy)
if satscene.satname == "terra":
for band in ["29"]:
if not satscene[band].is_loaded() or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
satscene[band].area.area_id = ("swath_" + satscene.fullname + "_"
+ str(satscene.time_slot) + "_"
+ str(satscene[band].shape) + "_"
+ str(band))
class hdf (object):
"""wrapper for HDF4 dataset for comparison
__call__ yields sequence of variable names
__getitem__ returns individual variables ready for slicing to numpy arrays
"""
_hdf = None
def __init__(self, filename, allowWrite=False):
if pyhdf is None:
LOG.error('pyhdf is not installed and is needed in order to read hdf4 files')
assert(pyhdf is not None)
mode = SDC.READ
if allowWrite:
mode = mode | SDC.WRITE
self._hdf = SD(filename, mode)
self.attributeCache = CaseInsensitiveAttributeCache(self)
def __call__(self):
"yield names of variables to be compared"
return self._hdf.datasets().keys()
# this returns a numpy array with a copy of the full, scaled
# data for this variable, if the data type must be changed to allow
# for scaling it will be (so the return type may not reflect the
# type found in the original file)
def __getitem__(self, name):
# defaults
scale_factor = 1.0
add_offset = 0.0
data_type = None
scaling_method = None
# get the variable object and use it to
# get our raw data and scaling info
variable_object = self.get_variable_object(name)
raw_data_copy = variable_object[:]
try :
# TODO, this currently won't work with geocat data, work around it for now
scale_factor, scale_factor_error, add_offset, add_offset_error, data_type = SDS.getcal(variable_object)
except HDF4Error:
# load just the scale factor and add offset information by hand
temp = self.attributeCache.get_variable_attributes(name)
if ADD_OFFSET_STR in temp.keys() :
add_offset = temp[ADD_OFFSET_STR]
data_type = np.dtype(type(add_offset))
if SCALE_FACTOR_STR in temp.keys() :
scale_factor = temp[SCALE_FACTOR_STR]
data_type = np.dtype(type(scale_factor))
if SCALE_METHOD_STR in temp.keys() :
scaling_method = temp[SCALE_METHOD_STR]
SDS.endaccess(variable_object)
# don't do lots of work if we don't need to scale things
if (scale_factor == 1.0) and (add_offset == 0.0) :
return raw_data_copy
# at the moment geocat has several scaling methods that don't match the normal standards for hdf
"""
please see constant.f90 for a more up to date version of this information:
INTEGER(kind=int1) :: NO_SCALE ! 0
INTEGER(kind=int1) :: LINEAR_SCALE ! 1
INTEGER(kind=int1) :: LOG_SCALE ! 2
INTEGER(kind=int1) :: SQRT_SCALE ! 3
"""
if (scaling_method == 0) :
return raw_data_copy
if not ((scaling_method is None) or (int(scaling_method) <= 1)) :
LOG.warn ('Scaling method of \"' + str(scaling_method) + '\" will be ignored in favor of hdf standard method. '
+ 'This may cause problems with data consistency')
# if we don't have a data type something strange has gone wrong
assert(not (data_type is None))
# get information about where the data is the missing value
missing_val = self.missing_value(name)
missing_mask = np.zeros(raw_data_copy.shape, dtype=np.bool)
missing_mask[raw_data_copy == missing_val] = True
# create the scaled version of the data
scaled_data_copy = np.array(raw_data_copy, dtype=data_type)
scaled_data_copy[~missing_mask] = (scaled_data_copy[~missing_mask] * scale_factor) + add_offset #TODO, type truncation issues?
return scaled_data_copy
def get_variable_object(self, name):
return self._hdf.select(name)
def missing_value(self, name):
return self.get_attribute(name, fillValConst1)
def create_new_variable(self, variablename, missingvalue=None, data=None, variabletocopyattributesfrom=None):
"""
create a new variable with the given name
optionally set the missing value (fill value) and data to those given
the created variable will be returned, or None if a variable could not
be created
"""
raise IOUnimplimentedError('Unable to create variable in hdf file, this functionality is not yet available.')
return None
def add_attribute_data_to_variable(self, variableName, newAttributeName, newAttributeValue) :
"""
if the attribute exists for the given variable, set it to the new value
if the attribute does not exist for the given variable, create it and set it to the new value
"""
raise IOUnimplimentedError('Unable add attribute to hdf file, this functionality is not yet available.')
return
def get_variable_attributes (self, variableName, caseInsensitive=True) :
"""
returns all the attributes associated with a variable name
"""
toReturn = None
if caseInsensitive :
toReturn = self.attributeCache.get_variable_attributes(variableName)
else :
toReturn = self.get_variable_object(variableName).attributes()
return toReturn
def get_attribute(self, variableName, attributeName, caseInsensitive=True) :
"""
returns the value of the attribute if it is available for this variable, or None
"""
toReturn = None
if caseInsensitive :
toReturn = self.attributeCache.get_variable_attribute(variableName, attributeName)
else :
temp_attributes = self.get_variable_attributes(variableName, caseInsensitive=False)
if attributeName in temp_attributes :
toReturn = temp_attributes[attributeName]
return toReturn
def get_global_attributes(self, caseInsensitive=True) :
"""
get a list of all the global attributes for this file or None
"""
toReturn = None
if caseInsensitive :
toReturn = self.attributeCache.get_global_attributes()
else :
toReturn = self._hdf.attributes()
return toReturn
def get_global_attribute(self, attributeName, caseInsensitive=True) :
"""
returns the value of a global attribute if it is available or None
"""
toReturn = None
if caseInsensitive :
toReturn = self.attributeCache.get_global_attribute(attributeName)
else :
if attributeName in self._hdf.attributes() :
toReturn = self._hdf.attributes()[attributeName]
return toReturn
def is_loadable_type (self, name) :
"""
check to see if the indicated variable is a type that can be loaded
"""
# TODO, are there any bad types for these files?
return True
def load(satscene, **kwargs):
"""Read data from file and load it into *satscene*. Load data into the
*channels*. *Channels* is a list or a tuple containing channels we will
load data into. If None, all channels are loaded.
"""
del kwargs
conf = ConfigParser()
conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg"))
options = {}
for option, value in conf.items(satscene.instrument_name+"-level3",
raw = True):
options[option] = value
pathname = os.path.join(options["dir"], options['filename'])
filename = satscene.time_slot.strftime(pathname)
for prodname in GEO_PHYS_PRODUCTS + FLAGS_QUALITY:
if prodname in satscene.channels_to_load:
prod_chan = ModisEosHdfLevel2(prodname)
prod_chan.read(filename)
prod_chan.satid = satscene.satname.capitalize()
prod_chan.resolution = 1000.0
prod_chan.shape = prod_chan.data.shape
# All this for the netCDF writer:
prod_chan.info['var_name'] = prodname
prod_chan.info['var_data'] = prod_chan.data
resolution_str = str(int(prod_chan.resolution))+'m'
prod_chan.info['var_dim_names'] = ('y'+resolution_str,
'x'+resolution_str)
prod_chan.info['long_name'] = prod_chan.attr['long_name'][:-1]
try:
prod_chan.info['standard_name'] = prod_chan.attr['standard_name'][:-1]
except KeyError:
pass
valid_min = np.min(prod_chan.data)
valid_max = np.max(prod_chan.data)
prod_chan.info['valid_range'] = np.array([valid_min, valid_max])
prod_chan.info['resolution'] = prod_chan.resolution
if prodname == 'l2_flags':
# l2 flags definitions
for i in range(1, 33):
key = "f%02d_name"%i
prod_chan.info[key] = prod_chan.attr[key][:-1]
satscene.channels.append(prod_chan)
if prodname in CHANNELS:
satscene[prodname].info['units'] = '%'
else:
satscene[prodname].info['units'] = prod_chan.attr['units'][:-1]
LOG.info("Loading modis lvl2 product '%s' done"%prodname)
# Check if there are any bands to load:
channels_to_load = False
for bandname in CHANNELS:
if bandname in satscene.channels_to_load:
channels_to_load = True
break
if channels_to_load:
#print "FILE: ", filename
eoshdf = SD(filename)
# Get all the Attributes:
# Common Attributes, Data Time,
# Data Structure and Scene Coordinates
info = {}
for key in eoshdf.attributes().keys():
info[key] = eoshdf.attributes()[key]
dsets = eoshdf.datasets()
selected_dsets = []
for bandname in CHANNELS:
if (bandname in satscene.channels_to_load and
bandname in dsets):
value = eoshdf.select(bandname)
selected_dsets.append(value)
# Get only the selected datasets
attr = value.attributes()
band = value.get()
nodata = attr['bad_value_scaled']
mask = np.equal(band, nodata)
satscene[bandname] = (np.ma.masked_where(mask, band) *
attr['slope'] + attr['intercept'])
satscene[bandname].info['units'] = '%'
satscene[bandname].info['long_name'] = attr['long_name'][:-1]
for dset in selected_dsets:
dset.endaccess()
LOG.info("Loading modis lvl2 Remote Sensing Reflectances done")
eoshdf.end()
lat, lon = get_lat_lon(satscene, None)
from pyresample import geometry
satscene.area = geometry.SwathDefinition(lons=lon, lats=lat)
#print "Variant: ", satscene.variant
satscene.variant = 'regional' # Temporary fix!
LOG.info("Loading modis data done.")
class HDF4Store(xmlplot.common.VariableStore,xmlstore.util.referencedobject):
class Variable(xmlplot.common.Variable):
def __init__(self,store,hdfvar):
xmlplot.common.Variable.__init__(self,store)
self.hdfvar = hdfvar
self.info = self.hdfvar.info()
def getName_raw(self):
return self.info[0]
def getDimensions_raw(self):
dimnames = []
for idim in range(self.info[1]):
dim = self.hdfvar.dim(idim)
dimnames.append(dim.info()[0])
return dimnames
def getLongName(self):
atts = self.getProperties()
if 'long_name' in atts: return atts['long_name']
return xmlplot.common.Variable.getLongName(self)
def getUnit(self):
atts = self.getProperties()
if 'units' in atts: return atts['units']
return xmlplot.common.Variable.getUnit(self)
def getShape(self):
shape = self.info[2]
if isinstance(shape,int): shape = (shape,)
return shape
def getDataType(self):
return datatypes.get(self.info[3],None)
def getProperties(self):
return self.hdfvar.attributes()
def getSlice(self,bounds=None,dataonly=False,transfercoordinatemask=True):
dimnames = self.getDimensions_raw()
shape = self.getShape()
# Determine final slice
if bounds is None: bounds = (Ellipsis,)
newbounds = []
for bound,dimlen,dimname in zip(xmlplot.common.processEllipsis(bounds,shape),shape,dimnames):
if isinstance(bound,int):
# Integer value provided as index.
assert bound>=-dimlen, 'Slice index %i lies below the lowest possible index for dimension %s (%i).' % (bound,dimname,-dimlen )
assert bound< dimlen, 'Slice index %i exceeds the highest possible index for dimension %s (%i).' % (bound,dimname, dimlen-1)
if bound<0: bound += dimlen
elif isinstance(bound,slice):
start,stop,step = bound.indices(dimlen)
bound = slice(start,stop,step)
newbounds.append(bound)
bounds = tuple(newbounds)
# Get data
dat = numpy.asarray(self.hdfvar[bounds])
# Mask fill value
fillvalue = self.hdfvar.attributes().get('_FillValue',None)
if fillvalue is None: fillvalue = self.hdfvar.attributes().get('Fill',None)
if fillvalue is not None: dat = numpy.ma.array(dat,mask=(dat==fillvalue),copy=False)
# Determine scale factor and offset, and cast data to acommodating type if needed.
scale = self.hdfvar.attributes().get('scale_factor',None)
offset = self.hdfvar.attributes().get('add_offset', None)
if scale is not None or offset is not None and dat.dtype!=numpy.float:
dat = dat.astype(numpy.float)
if scale is not None: dat *= scale
if offset is not None: dat += offset
if dataonly: return dat
newdimnames = [d for d,b in zip(dimnames,bounds) if not isinstance(b,int)]
varslice = self.Slice(newdimnames)
varslice.data = dat
inewdim = 0
for dimname,bound in zip(dimnames,bounds):
# Get the coordinate variable
coordvar = self.store.getVariable_raw(dimname)
if coordvar is None:
# No coordinate variable available: use indices
if not isinstance(bound,slice): continue
coorddims = [dimname]
coords = numpy.arange(bound.start,bound.stop,bound.step,dtype=numpy.float)
else:
# Coordinate variable present: use it.
coorddims = list(coordvar.getDimensions())
# Debug check: see if all coordinate dimensions are also used by the variable.
for cd in coorddims:
assert cd in dimnames, 'Coordinate dimension %s is not used by this variable (it uses %s).' % (cd,', '.join(dimnames))
# Get coordinate values
coordslice = [bounds[dimnames.index(cd)] for cd in coorddims]
coords = coordvar.getSlice(coordslice, dataonly=True)
# Get the list of coordinate dimensions after the ones with single index have been sliced out.
newcoorddims = [cd for cd in coorddims if isinstance(bounds[dimnames.index(cd)],slice)]
# Transfer the coordinate mask to the data if desired.
coordmask = numpy.ma.getmask(coords)
if transfercoordinatemask and coordmask is not numpy.ma.nomask:
coordmask = xmlplot.common.broadcastSelective(coordmask,newcoorddims,dat.shape,newdimnames)
if datamask is numpy.ma.nomask:
datamask = coordmask
else:
datamask |= coordmask
# If we take a single index for this dimension, it will not be included in the output.
if not isinstance(bound,slice): continue
# Coordinates should not have a mask - undo the masking.
if coordmask is not numpy.ma.nomask:
coords = numpy.ma.getdata(coords)
# Auto-generate staggered coordinates
coords_stag = xmlplot.common.stagger(coords)
# Insert data dimensions where they are lacking in coordinate
coords = xmlplot.common.broadcastSelective(coords, (dimname,),dat.shape, newdimnames)
coords_stag = xmlplot.common.broadcastSelective(coords_stag,(dimname,),[l+1 for l in dat.shape],newdimnames)
# Assign coordinate values
varslice.coords [inewdim] = coords
varslice.coords_stag[inewdim] = coords_stag
inewdim += 1
return varslice
def __init__(self,path):
xmlplot.common.VariableStore.__init__(self)
xmlstore.util.referencedobject.__init__(self)
from pyhdf.SD import SD, SDC
self.file = SD(str(path),SDC.READ)
def getVariable_raw(self,varname):
"""Returns a Variable object for the given original short variable name.
The method must be implemented by derived classes.
"""
if varname not in self.file.datasets().keys(): return None
return self.Variable(self,self.file.select(varname))
def getVariableNames_raw(self):
"""Returns a list of original short names for all variables present in the store.
The method must be implemented by derived classes.
"""
return self.file.datasets().keys()
def getProperties(self):
return self.file.attributes()
def unlink(self):
self.file.end()
def run(FILE_NAME):
DATAFIELD_NAME = 'Gpp_1km'
if USE_GDAL:
import gdal
GRID_NAME = 'MOD_Grid_MOD17A2'
gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME,
GRID_NAME,
DATAFIELD_NAME)
gdset = gdal.Open(gname)
data = gdset.ReadAsArray().astype(np.float64)
# Get any needed attributes.
meta = gdset.GetMetadata()
scale_factor = np.float(meta['scale_factor'])
add_offset = np.float(meta['add_offset'])
_FillValue = np.float(meta['_FillValue'])
valid_range = [np.float(x) for x in meta['valid_range'].split(', ')]
units = meta['units']
long_name = meta['long_name']
# Construct the grid.
x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform()
nx, ny = (gdset.RasterXSize, gdset.RasterYSize)
x = np.linspace(x0, x0 + xinc*nx, nx)
y = np.linspace(y0, y0 + yinc*ny, ny)
xv, yv = np.meshgrid(x, y)
del gdset
else:
if USE_NETCDF:
from netCDF4 import Dataset
nc = Dataset(FILE_NAME)
ncvar = nc.variables[DATAFIELD_NAME]
# The scaling equation isn't "scale * data + offset",
# so turn automatic scaling off.
ncvar.set_auto_maskandscale(False)
data = ncvar[:].astype(np.float64)
# Get any needed attributes.
scale_factor = ncvar.scale_factor
add_offset = ncvar.add_offset
_FillValue = ncvar._FillValue
valid_range = ncvar.valid_range
units = ncvar.units
long_name = ncvar.long_name
gridmeta = getattr(nc, 'StructMetadata.0')
else:
from pyhdf.SD import SD, SDC
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data2D = hdf.select(DATAFIELD_NAME)
data = data2D[:,:].astype(np.double)
# Read attributes.
attrs = data2D.attributes(full=1)
lna=attrs["long_name"]
long_name = lna[0]
vra=attrs["valid_range"]
valid_range = vra[0]
aoa=attrs["add_offset"]
add_offset = aoa[0]
fva=attrs["_FillValue"]
_FillValue = fva[0]
sfa=attrs["scale_factor"]
scale_factor = sfa[0]
ua=attrs["units"]
units = ua[0]
fattrs = hdf.attributes(full=1)
ga = fattrs["StructMetadata.0"]
gridmeta = ga[0]
# Construct the grid. The needed information is in a global attribute
# called 'StructMetadata.0'. Use regular expressions to tease out the
# extents of the grid. In addition, the grid is in packed decimal
# degrees, so we need to normalize to degrees.
ul_regex = re.compile(r'''UpperLeftPointMtrs=\(
(?P<upper_left_x>[+-]?\d+\.\d+)
,
(?P<upper_left_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = ul_regex.search(gridmeta)
x0 = np.float(match.group('upper_left_x'))
y0 = np.float(match.group('upper_left_y'))
lr_regex = re.compile(r'''LowerRightMtrs=\(
(?P<lower_right_x>[+-]?\d+\.\d+)
,
(?P<lower_right_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = lr_regex.search(gridmeta)
x1 = np.float(match.group('lower_right_x'))
y1 = np.float(match.group('lower_right_y'))
ny, nx = data.shape
x = np.linspace(x0, x1, nx)
y = np.linspace(y0, y1, ny)
xv, yv = np.meshgrid(x, y)
# In basemap, the sinusoidal projection is global, so we won't use it.
# Instead we'll convert the grid back to lat/lons so we can use a local
# projection.
sinu = pyproj.Proj("+proj=sinu +R=6371007.181 +nadgrids=@null +wktext")
wgs84 = pyproj.Proj("+init=EPSG:4326")
lon, lat= pyproj.transform(sinu, wgs84, xv, yv)
# Apply the attributes to the data.
invalid = np.logical_or(data < valid_range[0], data > valid_range[1])
invalid = np.logical_or(invalid, data ==_FillValue)
data[invalid] = np.nan
data = (data - add_offset) * scale_factor
data = np.ma.masked_array(data, np.isnan(data))
m = Basemap(projection='cyl', resolution='l',
llcrnrlat=2.5, urcrnrlat=12.5,
llcrnrlon=-87.5, urcrnrlon = -77.5)
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(0, 15, 5), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(-90, 75, 5), labels=[0, 0, 0, 1])
m.pcolormesh(lon, lat, data, latlon=True)
cb = m.colorbar()
cb.set_label(units)
basename = os.path.basename(FILE_NAME)
plt.title('{0}\n{1}'.format(basename, long_name))
fig = plt.gcf()
# plt.show()
pngfile = "{0}.py.png".format(basename)
fig.savefig(pngfile)
def run(FILE_NAME):
# Identify the data field.
DATAFIELD_NAME = 'Sea_Ice_by_Reflectance'
if USE_GDAL:
import gdal
GRID_NAME = 'MOD_Grid_Seaice_1km'
gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME,
GRID_NAME,
DATAFIELD_NAME)
gdset = gdal.Open(gname)
data = gdset.ReadAsArray()
# Construct the grid.
meta = gdset.GetMetadata()
x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform()
nx, ny = (gdset.RasterXSize, gdset.RasterYSize)
del gdset
else:
from pyhdf.SD import SD, SDC
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data2D = hdf.select(DATAFIELD_NAME)
data = data2D[:,:].astype(np.float64)
# Read global attribute.
fattrs = hdf.attributes(full=1)
ga = fattrs["StructMetadata.0"]
gridmeta = ga[0]
# Construct the grid. The needed information is in a global attribute
# called 'StructMetadata.0'. Use regular expressions to tease out the
# extents of the grid.
ul_regex = re.compile(r'''UpperLeftPointMtrs=\(
(?P<upper_left_x>[+-]?\d+\.\d+)
,
(?P<upper_left_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = ul_regex.search(gridmeta)
x0 = np.float(match.group('upper_left_x'))
y0 = np.float(match.group('upper_left_y'))
lr_regex = re.compile(r'''LowerRightMtrs=\(
(?P<lower_right_x>[+-]?\d+\.\d+)
,
(?P<lower_right_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = lr_regex.search(gridmeta)
x1 = np.float(match.group('lower_right_x'))
y1 = np.float(match.group('lower_right_y'))
ny, nx = data.shape
xinc = (x1 - x0) / nx
yinc = (y1 - y0) / ny
x = np.linspace(x0, x0 + xinc*nx, nx)
y = np.linspace(y0, y0 + yinc*ny, ny)
xv, yv = np.meshgrid(x, y)
# Reproject the coordinates out of lamaz into lat/lon.
lamaz = pyproj.Proj("+proj=laea +a=6371228 +lat_0=90 +lon_0=0 +units=m")
wgs84 = pyproj.Proj("+init=EPSG:4326")
lon, lat= pyproj.transform(lamaz, wgs84, xv, yv)
# Draw a lambert equal area azimuthal basemap.
m = Basemap(projection='laea', resolution='l', lat_ts=50,
lat_0=50, lon_0=150,
width=2500000,height=2500000)
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(50, 91, 10), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(110, 181, 10), labels=[0, 0, 0, 1])
# Use a discretized colormap since we have only a few levels.
# 0=missing data
# 1=no decision
# 11=night
# 25=land
# 37=inland water
# 39=ocean
# 50=cloud
# 200=sea ice
# 253=no input tile expected
# 254=non-production mask"
# 255=fill
lst = ['#727272',
'#b7b7b7',
'#ffff96',
'#00ff00',
'#232375',
'#232375',
'#63c6ff',
'#ff0000',
'#3f3f3f',
'#000000',
'#000000']
cmap = mpl.colors.ListedColormap(lst)
bounds = [0, 1, 11, 25, 37, 39, 50, 200, 253, 254, 255, 256]
norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
m.pcolormesh(lon, lat, data, latlon=True, cmap=cmap, norm=norm)
color_bar = plt.colorbar()
color_bar.set_ticks([0.5, 5.5, 18, 31, 38, 44.5, 125, 226.5, 253.5, 254.5, 255.5])
color_bar.set_ticklabels(['missing', 'no decision', 'night', 'land',
'inland water', 'ocean', 'cloud', 'sea ice',
'no input tile\nexpected', 'non-production\nmask',
'fill'])
color_bar.draw_all()
basename = os.path.basename(FILE_NAME)
long_name = DATAFIELD_NAME
plt.title('{0}\n{1}'.format(basename, long_name))
fig = plt.gcf()
# plt.show()
pngfile = "{0}.py.png".format(basename)
fig.savefig(pngfile)
def run(FILE_NAME):
DATAFIELD_NAME = 'Sea_Ice_by_Reflectance_SP'
if USE_NETCDF4:
from netCDF4 import Dataset
nc = Dataset(FILE_NAME)
ncvar = nc.variables[DATAFIELD_NAME]
data = ncvar[:].astype(np.float64)
gridmeta = getattr(nc, 'StructMetadata.0')
else:
from pyhdf.SD import SD, SDC
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data2D = hdf.select(DATAFIELD_NAME)
data = data2D[:,:].astype(np.float64)
# Read global attribute.
fattrs = hdf.attributes(full=1)
ga = fattrs["StructMetadata.0"]
gridmeta = ga[0]
# Construct the grid. The needed information is in a global attribute
# called 'StructMetadata.0'. Use regular expressions to tease out the
# extents of the grid.
ul_regex = re.compile(r'''UpperLeftPointMtrs=\(
(?P<upper_left_x>[+-]?\d+\.\d+)
,
(?P<upper_left_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = ul_regex.search(gridmeta)
x0 = np.float(match.group('upper_left_x'))
y0 = np.float(match.group('upper_left_y'))
lr_regex = re.compile(r'''LowerRightMtrs=\(
(?P<lower_right_x>[+-]?\d+\.\d+)
,
(?P<lower_right_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = lr_regex.search(gridmeta)
x1 = np.float(match.group('lower_right_x'))
y1 = np.float(match.group('lower_right_y'))
ny, nx = data.shape
x = np.linspace(x0, x1, nx)
y = np.linspace(y0, y1, ny)
xv, yv = np.meshgrid(x, y)
# Reproject into latlon
# Reproject the coordinates out of lamaz into lat/lon.
lamaz = pyproj.Proj("+proj=laea +a=6371228 +lat_0=-90 +lon_0=0 +units=m")
wgs84 = pyproj.Proj("+init=EPSG:4326")
lon, lat= pyproj.transform(lamaz, wgs84, xv, yv)
# Use a south polar azimuthal equal area projection.
m = Basemap(projection='splaea', resolution='l',
boundinglat=-20, lon_0=180)
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(-90, 0, 15), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(-180, 180, 45), labels=[0, 0, 0, 1])
# Use a discretized colormap since we have only a few levels.
# 0=missing data
# 1=no decision
# 11=night
# 25=land
# 37=inland water
# 39=ocean
# 50=cloud
# 200=sea ice
# 253=no input tile expected
# 254=non-production mask"
lst = ['#727272',
'#b7b7b7',
'#ffff96',
'#00ff00',
'#232375',
'#232375',
'#63c6ff',
'#ff0000',
'#3f3f3f',
'#000000']
cmap = mpl.colors.ListedColormap(lst)
bounds = [0, 1, 11, 25, 37, 39, 50, 200, 253, 254, 255]
norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
# Render only a subset of the mesh.
rows = slice(500, 4000, 5)
cols = slice(500, 4000, 5)
m.pcolormesh(lon[rows,cols], lat[rows,cols], data[rows,cols],
latlon=True, cmap=cmap, norm=norm)
color_bar = plt.colorbar()
color_bar.set_ticks([0.5, 5.5, 18, 31, 38, 44.5, 125, 226.5, 253.5, 254.5])
color_bar.set_ticklabels(['missing', 'no decision', 'night', 'land',
'inland water', 'ocean', 'cloud', 'sea ice',
'no input tile\nexpected',
'non-production\nmask'])
color_bar.draw_all()
basename = os.path.basename(FILE_NAME)
long_name = DATAFIELD_NAME
plt.title('{0}\n{1}'.format(basename, long_name))
fig = plt.gcf()
# plt.show()
pngfile = "{0}.py.png".format(basename)
fig.savefig(pngfile)
def run(FILE_NAME):
DATAFIELD_NAME = 'LST_Night_CMG'
if USE_GDAL:
# GDAL
import gdal
GRID_NAME = 'MODIS_8DAY_0.05DEG_CMG_LST'
gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME,
GRID_NAME,
DATAFIELD_NAME)
gdset = gdal.Open(gname)
# data = gdset.ReadAsArray().astype(np.float64)[::4, ::4]
data = gdset.ReadAsArray().astype(np.float64)[:,:]
# Get any needed attributes.
meta = gdset.GetMetadata()
scale_factor = np.float(meta['scale_factor'])
add_offset = np.float(meta['add_offset'])
_FillValue = np.float(meta['_FillValue'])
valid_range = [np.float(x) for x in meta['valid_range'].split(', ')]
units = meta['units']
long_name = meta['long_name']
# Construct the grid. Subset by a factor of 4.
x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform()
# nx, ny = (gdset.RasterXSize / 4, gdset.RasterYSize / 4)
nx, ny = (gdset.RasterXSize, gdset.RasterYSize)
# x = np.linspace(x0, x0 + xinc*4*nx, nx)
# y = np.linspace(y0, y0 + yinc*4*ny, ny)
x = np.linspace(x0, x0 + xinc*nx, nx)
y = np.linspace(y0, y0 + yinc*ny, ny)
lon, lat = np.meshgrid(x, y)
del gdset
else:
if USE_NETCDF:
from netCDF4 import Dataset
# The scaling equation isn't what netcdf4 expects, so turn it off.
nc = Dataset(FILE_NAME)
ncvar = nc.variables[DATAFIELD_NAME]
ncvar.set_auto_maskandscale(False)
data = ncvar[::4, ::4].astype(np.float64)
# Get any needed attributes.
scale_factor = ncvar.scale_factor
add_offset = ncvar.add_offset
_FillValue = ncvar._FillValue
valid_range = ncvar.valid_range
units = ncvar.units
long_name = ncvar.long_name
gridmeta = getattr(nc, 'StructMetadata.0')
else:
from pyhdf.SD import SD, SDC
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data2D = hdf.select(DATAFIELD_NAME)
data = data2D[:,:].astype(np.double)
# Read attributes.
attrs = data2D.attributes(full=1)
lna=attrs["long_name"]
long_name = lna[0]
vra=attrs["valid_range"]
valid_range = vra[0]
aoa=attrs["add_offset"]
add_offset = aoa[0]
fva=attrs["_FillValue"]
_FillValue = fva[0]
sfa=attrs["scale_factor"]
scale_factor = sfa[0]
ua=attrs["units"]
units = ua[0]
fattrs = hdf.attributes(full=1)
ga = fattrs["StructMetadata.0"]
gridmeta = ga[0]
# Construct the grid. The needed information is in a global attribute
# called 'StructMetadata.0'. Use regular expressions to tease out the
# extents of the grid. In addition, the grid is in packed decimal
# degrees, so we need to normalize to degrees.
ul_regex = re.compile(r'''UpperLeftPointMtrs=\(
(?P<upper_left_x>[+-]?\d+\.\d+)
,
(?P<upper_left_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = ul_regex.search(gridmeta)
x0 = np.float(match.group('upper_left_x')) / 1e6
y0 = np.float(match.group('upper_left_y')) / 1e6
lr_regex = re.compile(r'''LowerRightMtrs=\(
(?P<lower_right_x>[+-]?\d+\.\d+)
,
(?P<lower_right_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = lr_regex.search(gridmeta)
x1 = np.float(match.group('lower_right_x')) / 1e6
y1 = np.float(match.group('lower_right_y')) / 1e6
ny, nx = data.shape
x = np.linspace(x0, x1, nx)
y = np.linspace(y0, y1, ny)
lon, lat = np.meshgrid(x, y)
# Apply the attributes to the data.
invalid = np.logical_or(data < valid_range[0], data > valid_range[1])
invalid = np.logical_or(invalid, data ==_FillValue)
data[invalid] = np.nan
data = (data - add_offset) * scale_factor
data = np.ma.masked_array(data, np.isnan(data))
m = Basemap(projection='cyl', resolution='l',
llcrnrlat=-90, urcrnrlat=90,
llcrnrlon=-180, urcrnrlon = 180)
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(-90, 91, 30), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(-180, 181, 45), labels=[0, 0, 0, 1])
m.pcolormesh(lon, lat, data, latlon=True)
cb = m.colorbar()
cb.set_label(units)
basename = os.path.basename(FILE_NAME)
plt.title('{0}\n{1}'.format(basename, long_name))
fig = plt.gcf()
# plt.show()
pngfile = "{0}.py.png".format(basename)
fig.savefig(pngfile)
class ModisReader(Reader):
pformat = "hdfeos_l1b"
res = {"1": 1000,
"Q": 250,
"H": 500}
def __init__(self, *args, **kwargs):
Reader.__init__(self, *args, **kwargs)
self.datafiles = {}
self.geofile = None
self.filename = None
self.data = None
self.areas = {}
def load(self, satscene, *args, **kwargs):
"""Read data from file and load it into *satscene*.
"""
del args
conf = ConfigParser()
conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg"))
options = dict(conf.items(satscene.instrument_name + "-level2",
raw=True))
options["resolution"] = 1000
options["geofile"] = os.path.join(options["dir"], options["geofile"])
options.update(kwargs)
if isinstance(kwargs.get("filename"), (list, set, tuple)):
# we got the entire dataset.
for fname in kwargs["filename"]:
if fnmatch(os.path.basename(fname), "M?D02?km*"):
resolution = self.res[os.path.basename(fname)[5]]
self.datafiles[resolution] = fname
elif fnmatch(os.path.basename(fname), "M?D03*"):
self.geofile = fname
elif ((kwargs.get("filename") is not None) and
fnmatch(os.path.basename(options["filename"]), "M?D02?km*")):
# read just one file
logger.debug("Reading from file: " + str(options["filename"]))
filename = options["filename"]
resolution = self.res[os.path.basename(filename)[5]]
self.datafiles[resolution] = filename
else:
# find files according to config
resolution = int(options["resolution"]) or 1000
for res in [250, 500, 1000]:
datafile = os.path.join(options['dir'],
options["filename" + str(res)])
try:
self.datafiles[res] = get_filename(datafile,
satscene.time_slot)
except IOError:
self.datafiles[res] = None
logger.warning("Can't find file for resolution %s with template: %s",
str(res), datafile)
try:
self.geofile = get_filename(options["geofile"],
satscene.time_slot)
except IOError:
self.geofile = None
logger.warning("Can't find geofile with template: %s",
options['geofile'])
resolution = options["resolution"]
cores = options.get("cores", max(multiprocessing.cpu_count() / 4, 1))
datadict = {
1000: ['EV_250_Aggr1km_RefSB',
'EV_500_Aggr1km_RefSB',
'EV_1KM_RefSB',
'EV_1KM_Emissive'],
500: ['EV_250_Aggr500_RefSB',
'EV_500_RefSB'],
250: ['EV_250_RefSB']}
loaded_bands = []
# process by dataset, reflective and emissive datasets separately
resolutions = [250, 500, 1000]
for res in resolutions:
if res < resolution:
continue
logger.debug("Working on resolution %d", res)
self.filename = self.datafiles[res]
logger.debug("Using " + str(cores) + " cores for interpolation")
try:
self.data = SD(str(self.filename))
except HDF4Error as err:
logger.warning("Could not load data from " + str(self.filename)
+ ": " + str(err))
continue
datasets = datadict[res]
for dataset in datasets:
subdata = self.data.select(dataset)
band_names = subdata.attributes()["band_names"].split(",")
if len(satscene.channels_to_load & set(band_names)) > 0:
# get the relative indices of the desired channels
indices = [i for i, band in enumerate(band_names)
if band in satscene.channels_to_load]
uncertainty = self.data.select(dataset + "_Uncert_Indexes")
if dataset.endswith('Emissive'):
array = calibrate_tb(
subdata, uncertainty, indices, band_names)
else:
array = calibrate_refl(subdata, uncertainty, indices)
for (i, idx) in enumerate(indices):
if band_names[idx] in loaded_bands:
continue
satscene[band_names[idx]] = array[i]
# fix the resolution to match the loaded data.
satscene[band_names[idx]].resolution = res
loaded_bands.append(band_names[idx])
# Get the orbit number
if not satscene.orbit:
mda = self.data.attributes()["CoreMetadata.0"]
orbit_idx = mda.index("ORBITNUMBER")
satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116]
# Get the geolocation
# if resolution != 1000:
# logger.warning("Cannot load geolocation at this resolution (yet).")
# return
for band_name in loaded_bands:
lon, lat = self.get_lonlat(satscene[band_name].resolution, cores)
area = geometry.SwathDefinition(lons=lon, lats=lat)
satscene[band_name].area = area
# Trimming out dead sensor lines (detectors) on aqua:
# (in addition channel 21 is noisy)
if satscene.satname == "aqua":
for band in ["6", "27", "36"]:
if not satscene[band].is_loaded() or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
# Trimming out dead sensor lines (detectors) on terra:
# (in addition channel 27, 30, 34, 35, and 36 are nosiy)
if satscene.satname == "terra":
for band in ["29"]:
if not satscene[band].is_loaded() or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
for band_name in loaded_bands:
band_uid = hashlib.sha1(satscene[band_name].data.mask).hexdigest()
satscene[band_name].area.area_id = ("swath_" + satscene.fullname + "_"
+ str(satscene.time_slot) + "_"
+
str(satscene[
band_name].shape) + "_"
+ str(band_uid))
satscene[band_name].area_id = satscene[band_name].area.area_id
def get_lonlat(self, resolution, cores=1):
"""Read lat and lon.
"""
if resolution in self.areas:
return self.areas[resolution]
logger.debug("generating lon, lat at %d", resolution)
if self.geofile is not None:
coarse_resolution = 1000
filename = self.geofile
else:
coarse_resolution = 5000
logger.info("Using 5km geolocation and interpolating")
filename = (self.datafiles.get(1000) or
self.datafiles.get(500) or
self.datafiles.get(250))
logger.debug("Loading geolocation from file: " + str(filename)
+ " at resolution " + str(coarse_resolution))
data = SD(str(filename))
lat = data.select("Latitude")
fill_value = lat.attributes()["_FillValue"]
lat = np.ma.masked_equal(lat.get(), fill_value)
lon = data.select("Longitude")
fill_value = lon.attributes()["_FillValue"]
lon = np.ma.masked_equal(lon.get(), fill_value)
if resolution == coarse_resolution:
self.areas[resolution] = lon, lat
return lon, lat
from geotiepoints import modis5kmto1km, modis1kmto500m, modis1kmto250m
logger.debug("Interpolating from " + str(coarse_resolution)
+ " to " + str(resolution))
if coarse_resolution == 5000:
lon, lat = modis5kmto1km(lon, lat)
if resolution == 500:
lon, lat = modis1kmto500m(lon, lat, cores)
if resolution == 250:
lon, lat = modis1kmto250m(lon, lat, cores)
self.areas[resolution] = lon, lat
return lon, lat
# These have to be interpolated...
def get_height(self):
return self.data.select("Height")
def get_sunz(self):
return self.data.select("SolarZenith")
def get_suna(self):
return self.data.select("SolarAzimuth")
def get_satz(self):
return self.data.select("SensorZenith")
def get_sata(self):
return self.data.select("SensorAzimuth")
def run(FILE_NAME):
# Identify the data field.
DATAFIELD_NAME = "SI_06km_NH_89V_DAY"
if USE_GDAL:
import gdal
GRID_NAME = "NpPolarGrid06km"
gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME, GRID_NAME, DATAFIELD_NAME)
gdset = gdal.Open(gname)
data = gdset.ReadAsArray().astype(np.float64)
# Read projection parameters from global attribute.
meta = gdset.GetMetadata()
x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform()
nx, ny = (gdset.RasterXSize, gdset.RasterYSize)
del gdset
else:
from pyhdf.SD import SD, SDC
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data2D = hdf.select(DATAFIELD_NAME)
data = data2D[:, :].astype(np.float64)
# Read global attribute.
fattrs = hdf.attributes(full=1)
ga = fattrs["StructMetadata.0"]
gridmeta = ga[0]
# Construct the grid. The needed information is in a global attribute
# called 'StructMetadata.0'. Use regular expressions to tease out the
# extents of the grid.
ul_regex = re.compile(
r"""UpperLeftPointMtrs=\(
(?P<upper_left_x>[+-]?\d+\.\d+)
,
(?P<upper_left_y>[+-]?\d+\.\d+)
\)""",
re.VERBOSE,
)
match = ul_regex.search(gridmeta)
x0 = np.float(match.group("upper_left_x"))
y0 = np.float(match.group("upper_left_y"))
lr_regex = re.compile(
r"""LowerRightMtrs=\(
(?P<lower_right_x>[+-]?\d+\.\d+)
,
(?P<lower_right_y>[+-]?\d+\.\d+)
\)""",
re.VERBOSE,
)
match = lr_regex.search(gridmeta)
x1 = np.float(match.group("lower_right_x"))
y1 = np.float(match.group("lower_right_y"))
ny, nx = data.shape
xinc = (x1 - x0) / nx
yinc = (y1 - y0) / ny
# Apply the attributes information.
# Ref: http://nsidc.org/data/docs/daac/ae_si6_6km_tbs.gd.html#2
data[data == 0] = np.nan
data *= 0.1
data = np.ma.masked_array(data, np.isnan(data))
x = np.linspace(x0, x0 + xinc * nx, nx)
y = np.linspace(y0, y0 + yinc * ny, ny)
xv, yv = np.meshgrid(x, y)
args = [
"+proj=stere",
"+lat_0=90",
"+lon_0=-45",
"+lat_ts=70",
"+k=1",
"+es=0.006693883",
"+a=6378273",
"+x_0=0",
"+y_0=0",
"+ellps=WGS84",
"+datum=WGS84",
]
pstereo = pyproj.Proj(" ".join(args))
wgs84 = pyproj.Proj("+init=EPSG:4326")
lon, lat = pyproj.transform(pstereo, wgs84, xv, yv)
units = "K"
long_name = DATAFIELD_NAME
m = Basemap(projection="npstere", resolution="l", boundinglat=30, lon_0=0)
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(0, 91, 20), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(-180, 181, 45), labels=[0, 0, 0, 1])
m.pcolormesh(lon, lat, data, latlon=True)
cb = m.colorbar()
cb.set_label(units)
basename = os.path.basename(FILE_NAME)
plt.title("{0}\n{1}".format(basename, long_name))
fig = plt.gcf()
# plt.show()
pngfile = "{0}.py.png".format(basename)
fig.savefig(pngfile)
def run(FILE_NAME):
# Identify the data field.
DATAFIELD_NAME = 'Extent'
if USE_GDAL:
import gdal
GRID_NAME = 'Northern Hemisphere'
gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME,
GRID_NAME,
DATAFIELD_NAME)
gdset = gdal.Open(gname)
data = gdset.ReadAsArray()
meta = gdset.GetMetadata()
x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform()
nx, ny = (gdset.RasterXSize, gdset.RasterYSize)
del gdset
else:
from pyhdf.SD import SD, SDC
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data2D = hdf.select(DATAFIELD_NAME)
data = data2D[:,:].astype(np.float64)
# Read global attribute.
fattrs = hdf.attributes(full=1)
ga = fattrs["StructMetadata.0"]
gridmeta = ga[0]
# Construct the grid. The needed information is in a global attribute
# called 'StructMetadata.0'. Use regular expressions to tease out the
# extents of the grid.
ul_regex = re.compile(r'''UpperLeftPointMtrs=\(
(?P<upper_left_x>[+-]?\d+\.\d+)
,
(?P<upper_left_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = ul_regex.search(gridmeta)
x0 = np.float(match.group('upper_left_x'))
y0 = np.float(match.group('upper_left_y'))
lr_regex = re.compile(r'''LowerRightMtrs=\(
(?P<lower_right_x>[+-]?\d+\.\d+)
,
(?P<lower_right_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = lr_regex.search(gridmeta)
x1 = np.float(match.group('lower_right_x'))
y1 = np.float(match.group('lower_right_y'))
ny, nx = data.shape
xinc = (x1 - x0) / nx
yinc = (y1 - y0) / ny
x = np.linspace(x0, x0 + xinc*nx, nx)
y = np.linspace(y0, y0 + yinc*ny, ny)
xv, yv = np.meshgrid(x, y)
# Reproject into WGS84
lamaz = pyproj.Proj("+proj=laea +a=6371228 +lat_0=90 +lon_0=0 +units=m")
wgs84 = pyproj.Proj("+init=EPSG:4326")
lon, lat= pyproj.transform(lamaz, wgs84, xv, yv)
# Use a north polar azimuthal equal area projection.
m = Basemap(projection='nplaea', resolution='l',
boundinglat=40, lon_0=0)
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(0, 90, 15), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(-180, 180, 45), labels=[0, 0, 0, 1])
# Bin the data as follows:
# 0 -- snow-free land
# 1-20% sea ice -- blue
# 21-40% sea ice -- blue-cyan
# 41-60% sea ice -- blue
# 61-80% sea ice -- cyan-blue
# 81-100% sea ice -- cyan
# 101 -- permanent ice
# 103 -- dry snow
# 252 mixed pixels at coastlines
# 255 ocean
lst = ['#004400',
'#0000ff',
'#0044ff',
'#0088ff',
'#00ccff',
'#00ffff',
'#ffffff',
'#440044',
'#191919',
'#000000',
'#8888cc']
cmap = mpl.colors.ListedColormap(lst)
bounds = [0, 1, 21, 41, 61, 81, 101, 103, 104, 252, 255]
tickpts = [0.5, 11, 31, 51, 71, 91, 102, 103.5, 178, 253.5]
norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
# The corners cause trouble, so chop them out.
idx = slice(5, 721)
m.pcolormesh(lon[idx, idx], lat[idx, idx], data[idx, idx],
latlon=True, cmap=cmap, norm=norm)
color_bar = plt.colorbar()
color_bar.set_ticks(tickpts)
color_bar.set_ticklabels(['snow-free\nland',
'1-20% sea ice',
'21-40% sea ice',
'41-60% sea ice',
'61-80% sea ice',
'81-100% sea ice',
'permanent\nice',
'dry\nsnow',
'mixed pixels\nat coastlines',
'ocean'])
color_bar.draw_all()
basename = os.path.basename(FILE_NAME)
long_name = DATAFIELD_NAME
plt.title('{0}\n{1}'.format(basename, long_name))
fig = plt.gcf()
# plt.show()
pngfile = "{0}.py.png".format(basename)
fig.savefig(pngfile)
def run(FILE_NAME):
# Identify the data field.
DATAFIELD_NAME = 'SI_12km_NH_ICECON_DAY'
from pyhdf.SD import SD, SDC
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data2D = hdf.select(DATAFIELD_NAME)
data = data2D[:,:].astype(np.float64)
# Read global attribute.
fattrs = hdf.attributes(full=1)
ga = fattrs["StructMetadata.0"]
gridmeta = ga[0]
# Construct the grid. The needed information is in a global attribute
# called 'StructMetadata.0'. Use regular expressions to tease out the
# extents of the grid.
ul_regex = re.compile(r'''UpperLeftPointMtrs=\(
(?P<upper_left_x>[+-]?\d+\.\d+)
,
(?P<upper_left_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = ul_regex.search(gridmeta)
x0 = np.float(match.group('upper_left_x'))
y0 = np.float(match.group('upper_left_y'))
lr_regex = re.compile(r'''LowerRightMtrs=\(
(?P<lower_right_x>[+-]?\d+\.\d+)
,
(?P<lower_right_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = lr_regex.search(gridmeta)
x1 = np.float(match.group('lower_right_x'))
y1 = np.float(match.group('lower_right_y'))
ny, nx = data.shape
xinc = (x1 - x0) / nx
yinc = (y1 - y0) / ny
# Handle land mask.
# http://nsidc.org/data/docs/daac/ae_si12_12km_seaice/data.html
data[data > 100.0] = np.nan
data = np.ma.masked_array(data, np.isnan(data))
# Construct the grid.
# Reproject out of the GCTP stereographic into lat/lon.
x = np.linspace(x0, x0 + xinc*nx, nx)
y = np.linspace(y0, y0 + yinc*ny, ny)
xv, yv = np.meshgrid(x, y)
args = ["+proj=stere",
"+lat_0=90",
"+lon_0=-45",
"+lat_ts=70",
"+k=1",
"+es=0.006693883",
"+a=6378273",
"+x_0=0",
"+y_0=0",
"+ellps=WGS84",
"+datum=WGS84"]
pstereo = pyproj.Proj(' '.join(args))
wgs84 = pyproj.Proj("+init=EPSG:4326")
lon, lat= pyproj.transform(pstereo, wgs84, xv, yv)
units = 'Percent'
long_name = DATAFIELD_NAME
m = Basemap(projection='npstere', resolution='l', boundinglat=30, lon_0 = 0)
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(0, 91, 20), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(-180, 181, 45), labels=[0, 0, 0, 1])
m.pcolormesh(lon, lat, data, latlon=True)
cb = m.colorbar()
cb.set_label(units)
basename = os.path.basename(FILE_NAME)
plt.title('{0}\n{1}'.format(basename, long_name))
fig = plt.gcf()
# plt.show()
pngfile = "{0}.n.py.png".format(basename)
fig.savefig(pngfile)
def run(FILE_NAME):
# Identify the data field.
DATAFIELD_NAME = "TbOceanRain"
if USE_GDAL:
import gdal
GRID_NAME = "MonthlyRainTotal_GeoGrid"
gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME, GRID_NAME, DATAFIELD_NAME)
gdset = gdal.Open(gname)
data = gdset.ReadAsArray().astype(np.float64)
# Apply the attributes information.
meta = gdset.GetMetadata()
# Construct the grid. The projection is GEO, so this immediately
# gives us latitude and longitude.
x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform()
nx, ny = (gdset.RasterXSize, gdset.RasterYSize)
x = np.linspace(x0, x0 + xinc * nx, nx)
y = np.linspace(y0, y0 + yinc * ny, ny)
longitude, latitude = np.meshgrid(x, y)
del gdset
else:
from pyhdf.SD import SD, SDC
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data2D = hdf.select(DATAFIELD_NAME)
data = data2D[:, :].astype(np.float64)
# Read global attribute.
fattrs = hdf.attributes(full=1)
ga = fattrs["StructMetadata.0"]
gridmeta = ga[0]
# Construct the grid. The needed information is in a global attribute
# called 'StructMetadata.0'. Use regular expressions to tease out the
# extents of the grid. In addition, the grid is in packed decimal
# degrees, so we need to normalize to degrees.
ul_regex = re.compile(
r"""UpperLeftPointMtrs=\(
(?P<upper_left_x>[+-]?\d+\.\d+)
,
(?P<upper_left_y>[+-]?\d+\.\d+)
\)""",
re.VERBOSE,
)
match = ul_regex.search(gridmeta)
x0 = np.float(match.group("upper_left_x")) / 1e6
y0 = np.float(match.group("upper_left_y")) / 1e6
lr_regex = re.compile(
r"""LowerRightMtrs=\(
(?P<lower_right_x>[+-]?\d+\.\d+)
,
(?P<lower_right_y>[+-]?\d+\.\d+)
\)""",
re.VERBOSE,
)
match = lr_regex.search(gridmeta)
x1 = np.float(match.group("lower_right_x")) / 1e6
y1 = np.float(match.group("lower_right_y")) / 1e6
ny, nx = data.shape
x = np.linspace(x0, x1, nx)
y = np.linspace(y0, y1, ny)
longitude, latitude = np.meshgrid(x, y)
# Apply the attributes information.
data[data == -1] = np.nan
data = np.ma.masked_array(data, np.isnan(data))
long_name = DATAFIELD_NAME
units = "mm"
m = Basemap(projection="cyl", resolution="l", lon_0=0, llcrnrlat=-90, urcrnrlat=90, llcrnrlon=-180, urcrnrlon=180)
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(-90, 91, 45), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(-180, 181, 45), labels=[0, 0, 0, 1])
m.pcolormesh(longitude, latitude, data, latlon=True)
cb = m.colorbar()
cb.set_label(units)
basename = os.path.basename(FILE_NAME)
plt.title("{0}\n{1}".format(basename, long_name))
fig = plt.gcf()
# plt.show()
pngfile = "{0}.py.png".format(basename)
fig.savefig(pngfile)
def run(FILE_NAME):
DATAFIELD_NAME = 'NDVI_TOA'
if USE_GDAL:
# Gdal
import gdal
GRID_NAME = 'WELD_GRID'
gname = 'HDF4_EOS:EOS_GRID:"{0}":{1}:{2}'.format(FILE_NAME,
GRID_NAME,
DATAFIELD_NAME)
# Scale down the data by a factor of 5 so that low-memory machines
# can handle it.
gdset = gdal.Open(gname)
data = gdset.ReadAsArray().astype(np.float64)[::5, ::5]
# Get any needed attributes.
meta = gdset.GetMetadata()
scale = np.float(meta['scale_factor'])
fillvalue = np.float(meta['_FillValue'])
valid_range = [np.float(x) for x in meta['valid_range'].split(', ')]
units = meta['units']
# Construct the grid.
x0, xinc, _, y0, _, yinc = gdset.GetGeoTransform()
ny, nx = (gdset.RasterYSize / 5, gdset.RasterXSize / 5)
x = np.linspace(x0, x0 + xinc*5*nx, nx)
y = np.linspace(y0, y0 + yinc*5*ny, ny)
xv, yv = np.meshgrid(x, y)
del gdset
else:
if USE_NETCDF:
from netCDF4 import Dataset
# Scale down the data by a factor of 5 so that low-memory machines
# can handle it.
nc = Dataset(FILE_NAME)
ncvar = nc.variables[DATAFIELD_NAME]
ncvar.set_auto_maskandscale(False)
data = ncvar[::5, ::5].astype(np.float64)
# Get any needed attributes.
scale = ncvar.scale_factor
fillvalue = ncvar._FillValue
valid_range = ncvar.valid_range
units = ncvar.units
gridmeta = getattr(nc, 'StructMetadata.0')
else:
from pyhdf.SD import SD, SDC
hdf = SD(FILE_NAME, SDC.READ)
# Read dataset.
data2D = hdf.select(DATAFIELD_NAME)
data = data2D[:,:].astype(np.double)
# Scale down the data by a factor of 6 so that low-memory machines
# can handle it.
data = data[::6, ::6]
# Read attributes.
attrs = data2D.attributes(full=1)
vra=attrs["valid_range"]
valid_range = vra[0]
fva=attrs["_FillValue"]
fillvalue = fva[0]
sfa=attrs["scale_factor"]
scale = sfa[0]
ua=attrs["units"]
units = ua[0]
fattrs = hdf.attributes(full=1)
ga = fattrs["StructMetadata.0"]
gridmeta = ga[0]
# Construct the grid. The needed information is in a global attribute
# called 'StructMetadata.0'. Use regular expressions to tease out the
# extents of the grid.
ul_regex = re.compile(r'''UpperLeftPointMtrs=\(
(?P<upper_left_x>[+-]?\d+\.\d+)
,
(?P<upper_left_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = ul_regex.search(gridmeta)
x0 = np.float(match.group('upper_left_x'))
y0 = np.float(match.group('upper_left_y'))
lr_regex = re.compile(r'''LowerRightMtrs=\(
(?P<lower_right_x>[+-]?\d+\.\d+)
,
(?P<lower_right_y>[+-]?\d+\.\d+)
\)''', re.VERBOSE)
match = lr_regex.search(gridmeta)
x1 = np.float(match.group('lower_right_x'))
y1 = np.float(match.group('lower_right_y'))
ny, nx = data.shape
x = np.linspace(x0, x1, nx)
y = np.linspace(y0, y1, ny)
xv, yv = np.meshgrid(x, y)
# Apply the attributes to the data.
invalid = np.logical_or(data < valid_range[0], data > valid_range[1])
invalid = np.logical_or(invalid, data == fillvalue)
data[invalid] = np.nan
data = data * scale
data = np.ma.masked_array(data, np.isnan(data))
# Convert the grid back to lat/lon. The 1st and 2nd standard parallels,
# the center meridian, and the latitude of projected origin are in the
# projection parameters contained in the "StructMetadata.0" global
# attribute. The following regular expression could have been used to
# retrieve them.
#
# Ref: HDF-EOS Library User's Guide for the EOSDIS Evolution and
# Development (EED) Contract, Volume 2, Revision 02: Function
# Reference Guide, pages 1-6 through 1-13.
#
# aea_regex = re.compile(r'''Projection=GCTP_ALBERS\s+ProjParams=
# \(\d+,\d+
# (?P<stdpr1>[-]?\d+),
# (?P<stdpr2>[-]?\d+),
# (?P<centmer>[-]?\d+),
# (?P<origlat>[-]?\d+)
# ,0{7}\)''', re.VERBOSE)
#
aea = pyproj.Proj("+proj=aea +lat_1=29.5 +lat2=45.5 +lon_0=-96 +lat_0=23")
wgs84 = pyproj.Proj("+init=EPSG:4326")
lon, lat= pyproj.transform(aea, wgs84, xv, yv)
m = Basemap(projection='aea', resolution='i',
lat_1=29.5, lat_2=45.5, lon_0=-96, lat_0=23,
llcrnrlat=37.5, urcrnrlat = 42.5,
llcrnrlon=-127.5, urcrnrlon = -122.5)
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(35, 45, 1), labels=[1, 0, 0, 0])
m.drawmeridians(np.arange(-130, -120, 1), labels=[0, 0, 0, 1])
m.pcolormesh(lon, lat, data, latlon=True)
cb = m.colorbar()
cb.set_label(units)
long_name = DATAFIELD_NAME
basename = os.path.basename(FILE_NAME)
plt.title('{0}\n{1}'.format(basename, long_name))
fig = plt.gcf()
# plt.show()
pngfile = "{0}.py.png".format(basename)
fig.savefig(pngfile)
def modis_sst(file_name,
limit=None,
flagLevel=None,
param='sst'):
#Setup flag tables for L2 data
flagTable=np.zeros(32,dtype=bool)
if flagLevel == 0 : flagTable[[1]]=True
elif flagLevel == 1 :flagTable[[ 1, 3, 5, 15,16, 19, 25,26]]=True
elif flagLevel == 2 : flagTable[[0,1, 3,4,5,8,9,10, 14,15,16, 19,21,22,23,25,26]]=True
elif flagLevel >= 3 : flagTable[[0,1, 3,4,5,8,9,10, 14,15,16, 19,21,22,23,25,26]]=True
flags=np.where(flagTable)[0]
#Read MODIS HDF4 data
f = SD(file_name, SDC.READ)
fattr=f.attributes()
#Get dimensions
nScans=fattr['Number of Scan Lines']
sCtl=fattr['Number of Scan Control Points']
pCtl=fattr['Number of Pixel Control Points']
nPix=fattr['Pixels per Scan Line']
#Load coordinates
#UPDATE THIS SECTION TO ALLOW CLEAN LOAD OF THE IMAGE...
lonsel=f.select('longitude')
latsel=f.select('latitude')
#
# pCtl_ind=np.arange(pCtl,dtype=np.float32)
# p_ind=(pCtl-1)*np.arange(nPix,dtype=np.float32)/(nPix-1)
#
# sCtl_ind=np.arange(sCtl,dtype=np.float32)
# s_ind=(sCtl-1)*np.arange(nScans,dtype=np.float32)/(nScans-1)
# dum=interp2d2d(sCtl_ind, pCtl_ind, lonsel.get(), s_ind, p_ind)
# dumlon=interp1d(p_ind, lonsel.get(), pCtl_ind, spline=True)
# dumlat=interp1d(p_ind, lonsel.get(), pCtl_ind, spline=True)
# shlon=shlat=lon.dimensions().values()
#Shrink image
#############
#scene dimensions
info_sst=f.datasets()[param]
dnames=info_sst[0]
d=info_sst[1]
lonvec=lonsel.get().reshape(d[0]*d[1])
latvec=latsel.get().reshape(d[0]*d[1])
#Get points within domain
if limit is not None :
indvec,flagvec=in_limits(lonvec, latvec, limit)
flagmat=flagvec.reshape(d[0],d[1])
rowsum=np.sum(flagmat, 0)
colsum=np.sum(flagmat, 1)
yflag=rowsum >= 1
xflag=colsum >= 1
xid=np.arange(d[0])
xid=xid.compress(xflag)
xcnt=np.int(xid.size)
yid=np.arange(d[1])
yid=yid.compress(yflag)
ycnt=np.int(yid.size)
if xcnt == 0: raise Exception('Error : no longitude within limits')
if ycnt == 0: raise Exception('Error : no latitude within limits')
#Get start points
xst=np.int(xid.min())
yst=np.int(yid.min())
#Shrink lon & lat
lon_var=lonsel.get(start=[xst,yst], count=[xcnt,ycnt])
lat_var=latsel.get(start=[xst,yst], count=[xcnt,ycnt])
#Load SST image
###############
sst=f.select(param)
attr=sst.attributes()
slope=attr['slope']
intercept=attr['intercept']
flagValue=attr['bad_value_scaled']
sst_var=sst.get(start=[xst,yst], count=[xcnt,ycnt]) #Shrink sst image
#Compute mask
if (param == 'sst') or (param == 'sst4') :
fg=f.select('qual_'+param)
fg_var=fg.get(start=[xst,yst], count=[xcnt,ycnt])
if flagLevel is None :
mask=sst_var == flagValue
else :
mask= (sst_var == flagValue) | (fg_var >= flagLevel)
elif param == 'chlor_a' :
fg=f.select('l2_flags')
fg_var=fg.get(start=[xst,yst], count=[xcnt,ycnt]).flatten()
# dumvar=[False]*(xcnt*ycnt)#np.zeros((xcnt,ycnt,32),dtype=str).reshape((xcnt*ycnt*32))
dumfg=[[np.int(b) for b in np.binary_repr(f,32)[::-1]] for f in fg_var] #Rq : bits should be read from end to start
dumvar=np.sum(np.array(dumfg)[:,flags],1) >= 1
# for i,f in enumerate(fg_var) :
# dumvar[i] = (np.array([b for b in np.binary_repr(f,32)])[flags] == '1').any()
mask=np.reshape(dumvar,(xcnt,ycnt))
#Check flags
# plt.bar(np.arange(1,33),np.sum(dumfg,0)[::-1]/np.float64(xcnt*ycnt)); plt.show()
sst_var=np.ma.masked_array(sst_var*slope + intercept, mask=mask,type=float)
#Image reprojection to avoid bow tie effect
# import pyresample as pr
# swath_def=pr.geometry.SwathDefinition(lons=lon_var, lats=lat_var)
# lons,lats=np.meshgrid(np.arange(lon_var.min(),lon_var.max(),0.005), np.arange( lat_var.min(),lat_var.max(),0.005))
# grid_def = pr.geometry.GridDefinition(lons=lons, lats=lats)
#
# area_id = 'NWMEd'
# area_name = 'NWMed'
# proj_id = 'cyl'
# proj4_args = '+proj=eqc +lat_ts=0 +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +a=6378137 +b=6378137 +units=m'
# x_size = 601
# y_size = 351
# area_extent = (5., 41., 11., 44.5)
# area_def = pr.utils.get_area_def(area_id, area_name, proj_id, proj4_args, x_size, y_size, area_extent )
# res = pr.kd_tree.resample_gauss(swath_def, sst_var.data, grid_def, radius_of_influence=10000.,sigmas=500.,fill_value=None)
#Quality control
# sstfg=f.select('qual_sst')
# l2fg=f.select('l2_flags')
#Return output dictionary
return {'lon':lon_var,'lat':lat_var,'sst':sst_var}
