def _select_hdf_dataset(self, hdf_dataset_name, byte_dimension):
"""Load a dataset from HDF-EOS level 2 file."""
hdf_dataset = self.sd.select(hdf_dataset_name)
if byte_dimension == 0:
dataset = xr.DataArray(from_sds(hdf_dataset, chunks=CHUNK_SIZE),
dims=['i', 'y', 'x']).astype(np.uint8)
elif byte_dimension == 2:
dataset = xr.DataArray(from_sds(hdf_dataset, chunks=CHUNK_SIZE),
dims=['y', 'x', 'i']).astype(np.uint8)
# Reorder dimensions for consistency
dataset = dataset.transpose('i', 'y', 'x')
return dataset
def _mask_uncertain_pixels(self, array, uncertainty, band_index):
if not self._mask_saturated:
return array
band_uncertainty = from_sds(uncertainty,
chunks=CHUNK_SIZE)[band_index, :, :]
array = array.where(band_uncertainty < 15)
return array
def get_dataset(self, key, info, out=None, xslice=None, yslice=None):
"""Get the dataset designated by *key*."""
if key.name in [
'solar_zenith_angle', 'solar_azimuth_angle',
'satellite_zenith_angle', 'satellite_azimuth_angle'
]:
if key.name == 'solar_zenith_angle':
var = self.sd.select('SolarZenith')
if key.name == 'solar_azimuth_angle':
var = self.sd.select('SolarAzimuth')
if key.name == 'satellite_zenith_angle':
var = self.sd.select('SensorZenith')
if key.name == 'satellite_azimuth_angle':
var = self.sd.select('SensorAzimuth')
data = xr.DataArray(from_sds(var, chunks=CHUNK_SIZE),
dims=['y', 'x']).astype(np.float32)
data = data.where(data != var._FillValue)
data = data * np.float32(var.scale_factor)
data.attrs = info
return data
if key.name not in ['longitude', 'latitude']:
return
if (self.cache[key.resolution]['lons'] is None
or self.cache[key.resolution]['lats'] is None):
lons_id = DatasetID('longitude', resolution=key.resolution)
lats_id = DatasetID('latitude', resolution=key.resolution)
lons, lats = self.load([lons_id, lats_id],
interpolate=False,
raw=True)
if key.resolution != self.resolution:
from geotiepoints.geointerpolator import GeoInterpolator
lons, lats = self._interpolate([lons, lats], self.resolution,
lons_id.resolution,
GeoInterpolator)
lons = np.ma.masked_invalid(np.ascontiguousarray(lons))
lats = np.ma.masked_invalid(np.ascontiguousarray(lats))
self.cache[key.resolution]['lons'] = lons
self.cache[key.resolution]['lats'] = lats
if key.name == 'latitude':
data = self.cache[key.resolution]['lats'].filled(np.nan)
data = xr.DataArray(da.from_array(data,
chunks=(CHUNK_SIZE, CHUNK_SIZE)),
dims=['y', 'x'])
else:
data = self.cache[key.resolution]['lons'].filled(np.nan)
data = xr.DataArray(da.from_array(data,
chunks=(CHUNK_SIZE, CHUNK_SIZE)),
dims=['y', 'x'])
data.attrs = info
return data
def get_dataset(self, key, info, out=None, xslice=None, yslice=None):
"""Get the dataset designated by *key*."""
if key.name in ['solar_zenith_angle', 'solar_azimuth_angle',
'satellite_zenith_angle', 'satellite_azimuth_angle']:
if key.name == 'solar_zenith_angle':
var = self.sd.select('SolarZenith')
if key.name == 'solar_azimuth_angle':
var = self.sd.select('SolarAzimuth')
if key.name == 'satellite_zenith_angle':
var = self.sd.select('SensorZenith')
if key.name == 'satellite_azimuth_angle':
var = self.sd.select('SensorAzimuth')
data = xr.DataArray(from_sds(var, chunks=CHUNK_SIZE),
dims=['y', 'x']).astype(np.float32)
data = data.where(data != var._FillValue)
data = data * np.float32(var.scale_factor)
data.attrs = info
return data
if key.name not in ['longitude', 'latitude']:
return
if (self.cache[key.resolution]['lons'] is None or
self.cache[key.resolution]['lats'] is None):
lons_id = DatasetID('longitude',
resolution=key.resolution)
lats_id = DatasetID('latitude',
resolution=key.resolution)
lons, lats = self.load(
[lons_id, lats_id], interpolate=False, raw=True)
if key.resolution != self.resolution:
from geotiepoints.geointerpolator import GeoInterpolator
lons, lats = self._interpolate([lons, lats],
self.resolution,
lons_id.resolution,
GeoInterpolator)
lons = np.ma.masked_invalid(np.ascontiguousarray(lons))
lats = np.ma.masked_invalid(np.ascontiguousarray(lats))
self.cache[key.resolution]['lons'] = lons
self.cache[key.resolution]['lats'] = lats
if key.name == 'latitude':
data = self.cache[key.resolution]['lats'].filled(np.nan)
data = xr.DataArray(da.from_array(data, chunks=(CHUNK_SIZE, CHUNK_SIZE)),
dims=['y', 'x'])
else:
data = self.cache[key.resolution]['lons'].filled(np.nan)
data = xr.DataArray(da.from_array(data, chunks=(CHUNK_SIZE,
CHUNK_SIZE)),
dims=['y', 'x'])
data.attrs = info
return data
def load(self, file_key):
"""Load the data."""
var = self.sd.select(file_key)
data = xr.DataArray(from_sds(var, chunks=CHUNK_SIZE),
dims=['y', 'x']).astype(np.float32)
data = data.where(data != var._FillValue)
try:
data = data * np.float32(var.scale_factor)
except AttributeError:
pass
return data
def load_dataset(self, dataset_name, is_category=False):
"""Load the dataset from HDF EOS file."""
from satpy.readers.hdf4_utils import from_sds
dataset = self._read_dataset_in_file(dataset_name)
dask_arr = from_sds(dataset, chunks=CHUNK_SIZE)
dims = ('y', 'x') if dask_arr.ndim == 2 else None
data = xr.DataArray(dask_arr, dims=dims, attrs=dataset.attributes())
data = self._scale_and_mask_data_array(data, is_category=is_category)
return data
def load_dataset(self, dataset_name):
"""Load the dataset from HDF EOS file. """
from satpy.readers.hdf4_utils import from_sds
dataset = self._read_dataset_in_file(dataset_name)
fill_value = dataset._FillValue
scale_factor = np.float32(dataset.scale_factor)
data = xr.DataArray(from_sds(dataset, chunks=CHUNK_SIZE),
dims=['y', 'x']).astype(np.float32)
data_mask = data.where(data != fill_value)
data = data_mask * scale_factor
return data
def get_dataset(self, key, info):
"""Read data from file and return the corresponding projectables."""
if self.resolution != key['resolution']:
return
var_name, band_index = self._get_band_variable_name_and_index(
key["name"])
subdata = self.sd.select(var_name)
var_attrs = subdata.attributes()
uncertainty = self.sd.select(var_name + "_Uncert_Indexes")
array = xr.DataArray(from_sds(subdata,
chunks=CHUNK_SIZE)[band_index, :, :],
dims=['y', 'x']).astype(np.float32)
valid_range = var_attrs['valid_range']
valid_min = np.float32(valid_range[0])
valid_max = np.float32(valid_range[1])
if not self._mask_saturated:
array = self._fill_saturated(array, valid_max)
array = self._mask_invalid(array, valid_min, valid_max)
array = self._mask_uncertain_pixels(array, uncertainty, band_index)
projectable = self._calibrate_data(key, info, array, var_attrs,
band_index)
# if ((platform_name == 'Aqua' and key['name'] in ["6", "27", "36"]) or
# (platform_name == 'Terra' and key['name'] in ["29"])):
# height, width = projectable.shape
# row_indices = projectable.mask.sum(1) == width
# if row_indices.sum() != height:
# projectable.mask[row_indices, :] = True
# Get the orbit number
# if not satscene.orbit:
# mda = self.data.attributes()["CoreMetadata.0"]
# orbit_idx = mda.index("ORBITNUMBER")
# satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116]
# Trimming out dead sensor lines (detectors) on terra:
# (in addition channel 27, 30, 34, 35, and 36 are nosiy)
# if satscene.satname == "terra":
# for band in ["29"]:
# if not satscene[band].is_loaded() or satscene[band].data.mask.all():
# continue
# width = satscene[band].data.shape[1]
# height = satscene[band].data.shape[0]
# indices = satscene[band].data.mask.sum(1) < width
# if indices.sum() == height:
# continue
# satscene[band] = satscene[band].data[indices, :]
# satscene[band].area = geometry.SwathDefinition(
# lons=satscene[band].area.lons[indices, :],
# lats=satscene[band].area.lats[indices, :])
self._add_satpy_metadata(key, projectable)
return projectable
def _select_hdf_dataset(self, hdf_dataset_name, byte_dimension):
"""Load a dataset from HDF-EOS level 2 file."""
dataset = self.sd.select(hdf_dataset_name)
dask_arr = from_sds(dataset, chunks=CHUNK_SIZE)
attrs = dataset.attributes()
dims = ['y', 'x']
if byte_dimension == 0:
dims = ['i', 'y', 'x']
dask_arr = dask_arr.astype(np.uint8)
elif byte_dimension == 2:
dims = ['y', 'x', 'i']
dask_arr = dask_arr.astype(np.uint8)
dataset = xr.DataArray(dask_arr, dims=dims, attrs=attrs)
if 'i' in dataset.dims:
# Reorder dimensions for consistency
dataset = dataset.transpose('i', 'y', 'x')
return dataset
def get_dataset(self, dataset_id, dataset_info):
dataset_name = dataset_id.name
if dataset_name in HDFEOSGeoReader.DATASET_NAMES:
return HDFEOSGeoReader.get_dataset(self, dataset_id, dataset_info)
dataset_name_in_file = dataset_info['file_key']
# The dataset asked correspond to a given set of bits of the HDF EOS dataset
if 'bits' in dataset_info and 'byte' in dataset_info:
def bits_strip(bit_start, bit_count, value):
"""Extract specified bit from bit representation of integer value.
Parameters
----------
bit_start : int
Starting index of the bits to extract (first bit has index 0)
bit_count : int
Number of bits starting from bit_start to extract
value : int
Number from which to extract the bits
Returns
-------
int
Value of the extracted bits
"""
bit_mask = pow(2, bit_start + bit_count) - 1
return np.right_shift(np.bitwise_and(value, bit_mask),
bit_start)
hdf_dataset = self.sd.select(dataset_name_in_file)
dataset = xr.DataArray(from_sds(hdf_dataset, chunks=CHUNK_SIZE),
dims=['i', 'y', 'x']).astype(np.uint8)
bit_start = dataset_info['bits'][0]
bit_count = dataset_info['bits'][1]
dataset = bits_strip(bit_start, bit_count,
dataset[dataset_info['byte'], :, :])
else:
dataset = self.load_dataset(dataset_name)
return dataset
def load_dataset(self, dataset_name):
"""Load the dataset from HDF EOS file."""
from satpy.readers.hdf4_utils import from_sds
dataset = self._read_dataset_in_file(dataset_name)
fill_value = dataset._FillValue
dask_arr = from_sds(dataset, chunks=CHUNK_SIZE)
dims = ('y', 'x') if dask_arr.ndim == 2 else None
data = xr.DataArray(dask_arr, dims=dims, attrs=dataset.attributes())
# preserve integer data types if possible
if np.issubdtype(data.dtype, np.integer):
new_fill = fill_value
else:
new_fill = np.nan
data.attrs.pop('_FillValue', None)
good_mask = data != fill_value
scale_factor = data.attrs.get('scale_factor')
if scale_factor is not None:
data = data * scale_factor
data = data.where(good_mask, new_fill)
return data
def get_dataset(self, key, info):
"""Read data from file and return the corresponding projectables."""
datadict = {
1000: [
'EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB',
'EV_1KM_Emissive'
],
500: ['EV_250_Aggr500_RefSB', 'EV_500_RefSB'],
250: ['EV_250_RefSB']
}
platform_name = self.metadata['INVENTORYMETADATA'][
'ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][
'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER'][
'ASSOCIATEDPLATFORMSHORTNAME']['VALUE']
info.update({'platform_name': 'EOS-' + platform_name})
info.update({'sensor': 'modis'})
if self.resolution != key.resolution:
return
datasets = datadict[self.resolution]
for dataset in datasets:
subdata = self.sd.select(dataset)
var_attrs = subdata.attributes()
band_names = var_attrs["band_names"].split(",")
# get the relative indices of the desired channel
try:
index = band_names.index(key.name)
except ValueError:
continue
uncertainty = self.sd.select(dataset + "_Uncert_Indexes")
array = xr.DataArray(from_sds(subdata,
chunks=CHUNK_SIZE)[index, :, :],
dims=['y', 'x']).astype(np.float32)
valid_range = var_attrs['valid_range']
# Fill values:
# Data Value Meaning
# 65535 Fill Value (includes reflective band data at night mode
# and completely missing L1A scans)
# 65534 L1A DN is missing within a scan
# 65533 Detector is saturated
# 65532 Cannot compute zero point DN, e.g., SV is saturated
# 65531 Detector is dead (see comments below)
# 65530 RSB dn** below the minimum of the scaling range
# 65529 TEB radiance or RSB dn** exceeds the maximum of the
# scaling range
# 65528 Aggregation algorithm failure
# 65527 Rotation of Earth view Sector from nominal science
# collection position
# 65526 Calibration coefficient b1 could not be computed
# 65525 Subframe is dead
# 65524 Both sides of the PCLW electronics on simultaneously
# 65501 - 65523 (reserved for future use)
# 65500 NAD closed upper limit
array = array.where(array >= np.float32(valid_range[0]))
array = array.where(array <= np.float32(valid_range[1]))
array = array.where(
from_sds(uncertainty, chunks=CHUNK_SIZE)[index, :, :] < 15)
if key.calibration == 'brightness_temperature':
projectable = calibrate_bt(array, var_attrs, index, key.name)
info.setdefault('units', 'K')
info.setdefault('standard_name', 'toa_brightness_temperature')
elif key.calibration == 'reflectance':
projectable = calibrate_refl(array, var_attrs, index)
info.setdefault('units', '%')
info.setdefault('standard_name',
'toa_bidirectional_reflectance')
elif key.calibration == 'radiance':
projectable = calibrate_radiance(array, var_attrs, index)
info.setdefault('units', var_attrs.get('radiance_units'))
info.setdefault('standard_name',
'toa_outgoing_radiance_per_unit_wavelength')
elif key.calibration == 'counts':
projectable = calibrate_counts(array, var_attrs, index)
info.setdefault('units', 'counts')
info.setdefault('standard_name', 'counts') # made up
else:
raise ValueError("Unknown calibration for "
"key: {}".format(key))
projectable.attrs = info
# if ((platform_name == 'Aqua' and key.name in ["6", "27", "36"]) or
# (platform_name == 'Terra' and key.name in ["29"])):
# height, width = projectable.shape
# row_indices = projectable.mask.sum(1) == width
# if row_indices.sum() != height:
# projectable.mask[row_indices, :] = True
# Get the orbit number
# if not satscene.orbit:
# mda = self.data.attributes()["CoreMetadata.0"]
# orbit_idx = mda.index("ORBITNUMBER")
# satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116]
# Trimming out dead sensor lines (detectors) on terra:
# (in addition channel 27, 30, 34, 35, and 36 are nosiy)
# if satscene.satname == "terra":
# for band in ["29"]:
# if not satscene[band].is_loaded() or satscene[band].data.mask.all():
# continue
# width = satscene[band].data.shape[1]
# height = satscene[band].data.shape[0]
# indices = satscene[band].data.mask.sum(1) < width
# if indices.sum() == height:
# continue
# satscene[band] = satscene[band].data[indices, :]
# satscene[band].area = geometry.SwathDefinition(
# lons=satscene[band].area.lons[indices, :],
# lats=satscene[band].area.lats[indices, :])
return projectable
def get_dataset(self, key, info):
"""Read data from file and return the corresponding projectables."""
datadict = {
1000: ['EV_250_Aggr1km_RefSB',
'EV_500_Aggr1km_RefSB',
'EV_1KM_RefSB',
'EV_1KM_Emissive'],
500: ['EV_250_Aggr500_RefSB',
'EV_500_RefSB'],
250: ['EV_250_RefSB']}
platform_name = self.metadata['INVENTORYMETADATA']['ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][
'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER']['ASSOCIATEDPLATFORMSHORTNAME']['VALUE']
info.update({'platform_name': 'EOS-' + platform_name})
info.update({'sensor': 'modis'})
if self.resolution != key.resolution:
return
datasets = datadict[self.resolution]
for dataset in datasets:
subdata = self.sd.select(dataset)
var_attrs = subdata.attributes()
band_names = var_attrs["band_names"].split(",")
# get the relative indices of the desired channel
try:
index = band_names.index(key.name)
except ValueError:
continue
uncertainty = self.sd.select(dataset + "_Uncert_Indexes")
array = xr.DataArray(from_sds(subdata, chunks=CHUNK_SIZE)[index, :, :],
dims=['y', 'x']).astype(np.float32)
valid_range = var_attrs['valid_range']
array = array.where(array >= np.float32(valid_range[0]))
array = array.where(array <= np.float32(valid_range[1]))
array = array.where(from_sds(uncertainty, chunks=CHUNK_SIZE)[index, :, :] < 15)
if key.calibration == 'brightness_temperature':
projectable = calibrate_bt(array, var_attrs, index, key.name)
info.setdefault('units', 'K')
info.setdefault('standard_name', 'toa_brightness_temperature')
elif key.calibration == 'reflectance':
projectable = calibrate_refl(array, var_attrs, index)
info.setdefault('units', '%')
info.setdefault('standard_name',
'toa_bidirectional_reflectance')
elif key.calibration == 'radiance':
projectable = calibrate_radiance(array, var_attrs, index)
info.setdefault('units', var_attrs.get('radiance_units'))
info.setdefault('standard_name',
'toa_outgoing_radiance_per_unit_wavelength')
elif key.calibration == 'counts':
projectable = calibrate_counts(array, var_attrs, index)
info.setdefault('units', 'counts')
info.setdefault('standard_name', 'counts') # made up
else:
raise ValueError("Unknown calibration for "
"key: {}".format(key))
projectable.attrs = info
# if ((platform_name == 'Aqua' and key.name in ["6", "27", "36"]) or
# (platform_name == 'Terra' and key.name in ["29"])):
# height, width = projectable.shape
# row_indices = projectable.mask.sum(1) == width
# if row_indices.sum() != height:
# projectable.mask[row_indices, :] = True
# Get the orbit number
# if not satscene.orbit:
# mda = self.data.attributes()["CoreMetadata.0"]
# orbit_idx = mda.index("ORBITNUMBER")
# satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116]
# Get the geolocation
# if resolution != 1000:
# logger.warning("Cannot load geolocation at this resolution (yet).")
# return
# Trimming out dead sensor lines (detectors) on terra:
# (in addition channel 27, 30, 34, 35, and 36 are nosiy)
# if satscene.satname == "terra":
# for band in ["29"]:
# if not satscene[band].is_loaded() or satscene[band].data.mask.all():
# continue
# width = satscene[band].data.shape[1]
# height = satscene[band].data.shape[0]
# indices = satscene[band].data.mask.sum(1) < width
# if indices.sum() == height:
# continue
# satscene[band] = satscene[band].data[indices, :]
# satscene[band].area = geometry.SwathDefinition(
# lons=satscene[band].area.lons[indices, :],
# lats=satscene[band].area.lats[indices, :])
return projectable
def get_dataset(self, key, info):
"""Read data from file and return the corresponding projectables."""
datadict = {
1000: ['EV_250_Aggr1km_RefSB',
'EV_500_Aggr1km_RefSB',
'EV_1KM_RefSB',
'EV_1KM_Emissive'],
500: ['EV_250_Aggr500_RefSB',
'EV_500_RefSB'],
250: ['EV_250_RefSB']}
platform_name = self.metadata['INVENTORYMETADATA']['ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][
'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER']['ASSOCIATEDPLATFORMSHORTNAME']['VALUE']
info.update({'platform_name': 'EOS-' + platform_name})
info.update({'sensor': 'modis'})
if self.resolution != key.resolution:
return
datasets = datadict[self.resolution]
for dataset in datasets:
subdata = self.sd.select(dataset)
band_names = subdata.attributes()["band_names"].split(",")
# get the relative indices of the desired channel
try:
index = band_names.index(key.name)
except ValueError:
continue
uncertainty = self.sd.select(dataset + "_Uncert_Indexes")
array = xr.DataArray(from_sds(subdata, chunks=CHUNK_SIZE)[index, :, :],
dims=['y', 'x']).astype(np.float32)
valid_range = subdata.attributes()['valid_range']
array = array.where(array >= np.float32(valid_range[0]))
array = array.where(array <= np.float32(valid_range[1]))
array = array.where(from_sds(uncertainty, chunks=CHUNK_SIZE)[index, :, :] < 15)
if dataset.endswith('Emissive'):
projectable = calibrate_tb(array, subdata.attributes(), index, key.name)
else:
projectable = calibrate_refl(array, subdata.attributes(), index)
projectable.attrs = info
# if ((platform_name == 'Aqua' and key.name in ["6", "27", "36"]) or
# (platform_name == 'Terra' and key.name in ["29"])):
# height, width = projectable.shape
# row_indices = projectable.mask.sum(1) == width
# if row_indices.sum() != height:
# projectable.mask[row_indices, :] = True
# Get the orbit number
# if not satscene.orbit:
# mda = self.data.attributes()["CoreMetadata.0"]
# orbit_idx = mda.index("ORBITNUMBER")
# satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116]
# Get the geolocation
# if resolution != 1000:
# logger.warning("Cannot load geolocation at this resolution (yet).")
# return
# Trimming out dead sensor lines (detectors) on terra:
# (in addition channel 27, 30, 34, 35, and 36 are nosiy)
# if satscene.satname == "terra":
# for band in ["29"]:
# if not satscene[band].is_loaded() or satscene[band].data.mask.all():
# continue
# width = satscene[band].data.shape[1]
# height = satscene[band].data.shape[0]
# indices = satscene[band].data.mask.sum(1) < width
# if indices.sum() == height:
# continue
# satscene[band] = satscene[band].data[indices, :]
# satscene[band].area = geometry.SwathDefinition(
# lons=satscene[band].area.lons[indices, :],
# lats=satscene[band].area.lats[indices, :])
return projectable
