def _select_hdf_dataset(self, hdf_dataset_name, byte_dimension): """Load a dataset from HDF-EOS level 2 file.""" hdf_dataset = self.sd.select(hdf_dataset_name) if byte_dimension == 0: dataset = xr.DataArray(from_sds(hdf_dataset, chunks=CHUNK_SIZE), dims=['i', 'y', 'x']).astype(np.uint8) elif byte_dimension == 2: dataset = xr.DataArray(from_sds(hdf_dataset, chunks=CHUNK_SIZE), dims=['y', 'x', 'i']).astype(np.uint8) # Reorder dimensions for consistency dataset = dataset.transpose('i', 'y', 'x') return dataset
def _mask_uncertain_pixels(self, array, uncertainty, band_index): if not self._mask_saturated: return array band_uncertainty = from_sds(uncertainty, chunks=CHUNK_SIZE)[band_index, :, :] array = array.where(band_uncertainty < 15) return array
def get_dataset(self, key, info, out=None, xslice=None, yslice=None): """Get the dataset designated by *key*.""" if key.name in [ 'solar_zenith_angle', 'solar_azimuth_angle', 'satellite_zenith_angle', 'satellite_azimuth_angle' ]: if key.name == 'solar_zenith_angle': var = self.sd.select('SolarZenith') if key.name == 'solar_azimuth_angle': var = self.sd.select('SolarAzimuth') if key.name == 'satellite_zenith_angle': var = self.sd.select('SensorZenith') if key.name == 'satellite_azimuth_angle': var = self.sd.select('SensorAzimuth') data = xr.DataArray(from_sds(var, chunks=CHUNK_SIZE), dims=['y', 'x']).astype(np.float32) data = data.where(data != var._FillValue) data = data * np.float32(var.scale_factor) data.attrs = info return data if key.name not in ['longitude', 'latitude']: return if (self.cache[key.resolution]['lons'] is None or self.cache[key.resolution]['lats'] is None): lons_id = DatasetID('longitude', resolution=key.resolution) lats_id = DatasetID('latitude', resolution=key.resolution) lons, lats = self.load([lons_id, lats_id], interpolate=False, raw=True) if key.resolution != self.resolution: from geotiepoints.geointerpolator import GeoInterpolator lons, lats = self._interpolate([lons, lats], self.resolution, lons_id.resolution, GeoInterpolator) lons = np.ma.masked_invalid(np.ascontiguousarray(lons)) lats = np.ma.masked_invalid(np.ascontiguousarray(lats)) self.cache[key.resolution]['lons'] = lons self.cache[key.resolution]['lats'] = lats if key.name == 'latitude': data = self.cache[key.resolution]['lats'].filled(np.nan) data = xr.DataArray(da.from_array(data, chunks=(CHUNK_SIZE, CHUNK_SIZE)), dims=['y', 'x']) else: data = self.cache[key.resolution]['lons'].filled(np.nan) data = xr.DataArray(da.from_array(data, chunks=(CHUNK_SIZE, CHUNK_SIZE)), dims=['y', 'x']) data.attrs = info return data
def get_dataset(self, key, info, out=None, xslice=None, yslice=None): """Get the dataset designated by *key*.""" if key.name in ['solar_zenith_angle', 'solar_azimuth_angle', 'satellite_zenith_angle', 'satellite_azimuth_angle']: if key.name == 'solar_zenith_angle': var = self.sd.select('SolarZenith') if key.name == 'solar_azimuth_angle': var = self.sd.select('SolarAzimuth') if key.name == 'satellite_zenith_angle': var = self.sd.select('SensorZenith') if key.name == 'satellite_azimuth_angle': var = self.sd.select('SensorAzimuth') data = xr.DataArray(from_sds(var, chunks=CHUNK_SIZE), dims=['y', 'x']).astype(np.float32) data = data.where(data != var._FillValue) data = data * np.float32(var.scale_factor) data.attrs = info return data if key.name not in ['longitude', 'latitude']: return if (self.cache[key.resolution]['lons'] is None or self.cache[key.resolution]['lats'] is None): lons_id = DatasetID('longitude', resolution=key.resolution) lats_id = DatasetID('latitude', resolution=key.resolution) lons, lats = self.load( [lons_id, lats_id], interpolate=False, raw=True) if key.resolution != self.resolution: from geotiepoints.geointerpolator import GeoInterpolator lons, lats = self._interpolate([lons, lats], self.resolution, lons_id.resolution, GeoInterpolator) lons = np.ma.masked_invalid(np.ascontiguousarray(lons)) lats = np.ma.masked_invalid(np.ascontiguousarray(lats)) self.cache[key.resolution]['lons'] = lons self.cache[key.resolution]['lats'] = lats if key.name == 'latitude': data = self.cache[key.resolution]['lats'].filled(np.nan) data = xr.DataArray(da.from_array(data, chunks=(CHUNK_SIZE, CHUNK_SIZE)), dims=['y', 'x']) else: data = self.cache[key.resolution]['lons'].filled(np.nan) data = xr.DataArray(da.from_array(data, chunks=(CHUNK_SIZE, CHUNK_SIZE)), dims=['y', 'x']) data.attrs = info return data
def load(self, file_key): """Load the data.""" var = self.sd.select(file_key) data = xr.DataArray(from_sds(var, chunks=CHUNK_SIZE), dims=['y', 'x']).astype(np.float32) data = data.where(data != var._FillValue) try: data = data * np.float32(var.scale_factor) except AttributeError: pass return data
def load_dataset(self, dataset_name, is_category=False): """Load the dataset from HDF EOS file.""" from satpy.readers.hdf4_utils import from_sds dataset = self._read_dataset_in_file(dataset_name) dask_arr = from_sds(dataset, chunks=CHUNK_SIZE) dims = ('y', 'x') if dask_arr.ndim == 2 else None data = xr.DataArray(dask_arr, dims=dims, attrs=dataset.attributes()) data = self._scale_and_mask_data_array(data, is_category=is_category) return data
def load_dataset(self, dataset_name): """Load the dataset from HDF EOS file. """ from satpy.readers.hdf4_utils import from_sds dataset = self._read_dataset_in_file(dataset_name) fill_value = dataset._FillValue scale_factor = np.float32(dataset.scale_factor) data = xr.DataArray(from_sds(dataset, chunks=CHUNK_SIZE), dims=['y', 'x']).astype(np.float32) data_mask = data.where(data != fill_value) data = data_mask * scale_factor return data
def get_dataset(self, key, info): """Read data from file and return the corresponding projectables.""" if self.resolution != key['resolution']: return var_name, band_index = self._get_band_variable_name_and_index( key["name"]) subdata = self.sd.select(var_name) var_attrs = subdata.attributes() uncertainty = self.sd.select(var_name + "_Uncert_Indexes") array = xr.DataArray(from_sds(subdata, chunks=CHUNK_SIZE)[band_index, :, :], dims=['y', 'x']).astype(np.float32) valid_range = var_attrs['valid_range'] valid_min = np.float32(valid_range[0]) valid_max = np.float32(valid_range[1]) if not self._mask_saturated: array = self._fill_saturated(array, valid_max) array = self._mask_invalid(array, valid_min, valid_max) array = self._mask_uncertain_pixels(array, uncertainty, band_index) projectable = self._calibrate_data(key, info, array, var_attrs, band_index) # if ((platform_name == 'Aqua' and key['name'] in ["6", "27", "36"]) or # (platform_name == 'Terra' and key['name'] in ["29"])): # height, width = projectable.shape # row_indices = projectable.mask.sum(1) == width # if row_indices.sum() != height: # projectable.mask[row_indices, :] = True # Get the orbit number # if not satscene.orbit: # mda = self.data.attributes()["CoreMetadata.0"] # orbit_idx = mda.index("ORBITNUMBER") # satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116] # Trimming out dead sensor lines (detectors) on terra: # (in addition channel 27, 30, 34, 35, and 36 are nosiy) # if satscene.satname == "terra": # for band in ["29"]: # if not satscene[band].is_loaded() or satscene[band].data.mask.all(): # continue # width = satscene[band].data.shape[1] # height = satscene[band].data.shape[0] # indices = satscene[band].data.mask.sum(1) < width # if indices.sum() == height: # continue # satscene[band] = satscene[band].data[indices, :] # satscene[band].area = geometry.SwathDefinition( # lons=satscene[band].area.lons[indices, :], # lats=satscene[band].area.lats[indices, :]) self._add_satpy_metadata(key, projectable) return projectable
def _select_hdf_dataset(self, hdf_dataset_name, byte_dimension): """Load a dataset from HDF-EOS level 2 file.""" dataset = self.sd.select(hdf_dataset_name) dask_arr = from_sds(dataset, chunks=CHUNK_SIZE) attrs = dataset.attributes() dims = ['y', 'x'] if byte_dimension == 0: dims = ['i', 'y', 'x'] dask_arr = dask_arr.astype(np.uint8) elif byte_dimension == 2: dims = ['y', 'x', 'i'] dask_arr = dask_arr.astype(np.uint8) dataset = xr.DataArray(dask_arr, dims=dims, attrs=attrs) if 'i' in dataset.dims: # Reorder dimensions for consistency dataset = dataset.transpose('i', 'y', 'x') return dataset
def get_dataset(self, dataset_id, dataset_info): dataset_name = dataset_id.name if dataset_name in HDFEOSGeoReader.DATASET_NAMES: return HDFEOSGeoReader.get_dataset(self, dataset_id, dataset_info) dataset_name_in_file = dataset_info['file_key'] # The dataset asked correspond to a given set of bits of the HDF EOS dataset if 'bits' in dataset_info and 'byte' in dataset_info: def bits_strip(bit_start, bit_count, value): """Extract specified bit from bit representation of integer value. Parameters ---------- bit_start : int Starting index of the bits to extract (first bit has index 0) bit_count : int Number of bits starting from bit_start to extract value : int Number from which to extract the bits Returns ------- int Value of the extracted bits """ bit_mask = pow(2, bit_start + bit_count) - 1 return np.right_shift(np.bitwise_and(value, bit_mask), bit_start) hdf_dataset = self.sd.select(dataset_name_in_file) dataset = xr.DataArray(from_sds(hdf_dataset, chunks=CHUNK_SIZE), dims=['i', 'y', 'x']).astype(np.uint8) bit_start = dataset_info['bits'][0] bit_count = dataset_info['bits'][1] dataset = bits_strip(bit_start, bit_count, dataset[dataset_info['byte'], :, :]) else: dataset = self.load_dataset(dataset_name) return dataset
def load_dataset(self, dataset_name): """Load the dataset from HDF EOS file.""" from satpy.readers.hdf4_utils import from_sds dataset = self._read_dataset_in_file(dataset_name) fill_value = dataset._FillValue dask_arr = from_sds(dataset, chunks=CHUNK_SIZE) dims = ('y', 'x') if dask_arr.ndim == 2 else None data = xr.DataArray(dask_arr, dims=dims, attrs=dataset.attributes()) # preserve integer data types if possible if np.issubdtype(data.dtype, np.integer): new_fill = fill_value else: new_fill = np.nan data.attrs.pop('_FillValue', None) good_mask = data != fill_value scale_factor = data.attrs.get('scale_factor') if scale_factor is not None: data = data * scale_factor data = data.where(good_mask, new_fill) return data
def get_dataset(self, key, info): """Read data from file and return the corresponding projectables.""" datadict = { 1000: [ 'EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB', 'EV_1KM_Emissive' ], 500: ['EV_250_Aggr500_RefSB', 'EV_500_RefSB'], 250: ['EV_250_RefSB'] } platform_name = self.metadata['INVENTORYMETADATA'][ 'ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][ 'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER'][ 'ASSOCIATEDPLATFORMSHORTNAME']['VALUE'] info.update({'platform_name': 'EOS-' + platform_name}) info.update({'sensor': 'modis'}) if self.resolution != key.resolution: return datasets = datadict[self.resolution] for dataset in datasets: subdata = self.sd.select(dataset) var_attrs = subdata.attributes() band_names = var_attrs["band_names"].split(",") # get the relative indices of the desired channel try: index = band_names.index(key.name) except ValueError: continue uncertainty = self.sd.select(dataset + "_Uncert_Indexes") array = xr.DataArray(from_sds(subdata, chunks=CHUNK_SIZE)[index, :, :], dims=['y', 'x']).astype(np.float32) valid_range = var_attrs['valid_range'] # Fill values: # Data Value Meaning # 65535 Fill Value (includes reflective band data at night mode # and completely missing L1A scans) # 65534 L1A DN is missing within a scan # 65533 Detector is saturated # 65532 Cannot compute zero point DN, e.g., SV is saturated # 65531 Detector is dead (see comments below) # 65530 RSB dn** below the minimum of the scaling range # 65529 TEB radiance or RSB dn** exceeds the maximum of the # scaling range # 65528 Aggregation algorithm failure # 65527 Rotation of Earth view Sector from nominal science # collection position # 65526 Calibration coefficient b1 could not be computed # 65525 Subframe is dead # 65524 Both sides of the PCLW electronics on simultaneously # 65501 - 65523 (reserved for future use) # 65500 NAD closed upper limit array = array.where(array >= np.float32(valid_range[0])) array = array.where(array <= np.float32(valid_range[1])) array = array.where( from_sds(uncertainty, chunks=CHUNK_SIZE)[index, :, :] < 15) if key.calibration == 'brightness_temperature': projectable = calibrate_bt(array, var_attrs, index, key.name) info.setdefault('units', 'K') info.setdefault('standard_name', 'toa_brightness_temperature') elif key.calibration == 'reflectance': projectable = calibrate_refl(array, var_attrs, index) info.setdefault('units', '%') info.setdefault('standard_name', 'toa_bidirectional_reflectance') elif key.calibration == 'radiance': projectable = calibrate_radiance(array, var_attrs, index) info.setdefault('units', var_attrs.get('radiance_units')) info.setdefault('standard_name', 'toa_outgoing_radiance_per_unit_wavelength') elif key.calibration == 'counts': projectable = calibrate_counts(array, var_attrs, index) info.setdefault('units', 'counts') info.setdefault('standard_name', 'counts') # made up else: raise ValueError("Unknown calibration for " "key: {}".format(key)) projectable.attrs = info # if ((platform_name == 'Aqua' and key.name in ["6", "27", "36"]) or # (platform_name == 'Terra' and key.name in ["29"])): # height, width = projectable.shape # row_indices = projectable.mask.sum(1) == width # if row_indices.sum() != height: # projectable.mask[row_indices, :] = True # Get the orbit number # if not satscene.orbit: # mda = self.data.attributes()["CoreMetadata.0"] # orbit_idx = mda.index("ORBITNUMBER") # satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116] # Trimming out dead sensor lines (detectors) on terra: # (in addition channel 27, 30, 34, 35, and 36 are nosiy) # if satscene.satname == "terra": # for band in ["29"]: # if not satscene[band].is_loaded() or satscene[band].data.mask.all(): # continue # width = satscene[band].data.shape[1] # height = satscene[band].data.shape[0] # indices = satscene[band].data.mask.sum(1) < width # if indices.sum() == height: # continue # satscene[band] = satscene[band].data[indices, :] # satscene[band].area = geometry.SwathDefinition( # lons=satscene[band].area.lons[indices, :], # lats=satscene[band].area.lats[indices, :]) return projectable
def get_dataset(self, key, info): """Read data from file and return the corresponding projectables.""" datadict = { 1000: ['EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB', 'EV_1KM_Emissive'], 500: ['EV_250_Aggr500_RefSB', 'EV_500_RefSB'], 250: ['EV_250_RefSB']} platform_name = self.metadata['INVENTORYMETADATA']['ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][ 'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER']['ASSOCIATEDPLATFORMSHORTNAME']['VALUE'] info.update({'platform_name': 'EOS-' + platform_name}) info.update({'sensor': 'modis'}) if self.resolution != key.resolution: return datasets = datadict[self.resolution] for dataset in datasets: subdata = self.sd.select(dataset) var_attrs = subdata.attributes() band_names = var_attrs["band_names"].split(",") # get the relative indices of the desired channel try: index = band_names.index(key.name) except ValueError: continue uncertainty = self.sd.select(dataset + "_Uncert_Indexes") array = xr.DataArray(from_sds(subdata, chunks=CHUNK_SIZE)[index, :, :], dims=['y', 'x']).astype(np.float32) valid_range = var_attrs['valid_range'] array = array.where(array >= np.float32(valid_range[0])) array = array.where(array <= np.float32(valid_range[1])) array = array.where(from_sds(uncertainty, chunks=CHUNK_SIZE)[index, :, :] < 15) if key.calibration == 'brightness_temperature': projectable = calibrate_bt(array, var_attrs, index, key.name) info.setdefault('units', 'K') info.setdefault('standard_name', 'toa_brightness_temperature') elif key.calibration == 'reflectance': projectable = calibrate_refl(array, var_attrs, index) info.setdefault('units', '%') info.setdefault('standard_name', 'toa_bidirectional_reflectance') elif key.calibration == 'radiance': projectable = calibrate_radiance(array, var_attrs, index) info.setdefault('units', var_attrs.get('radiance_units')) info.setdefault('standard_name', 'toa_outgoing_radiance_per_unit_wavelength') elif key.calibration == 'counts': projectable = calibrate_counts(array, var_attrs, index) info.setdefault('units', 'counts') info.setdefault('standard_name', 'counts') # made up else: raise ValueError("Unknown calibration for " "key: {}".format(key)) projectable.attrs = info # if ((platform_name == 'Aqua' and key.name in ["6", "27", "36"]) or # (platform_name == 'Terra' and key.name in ["29"])): # height, width = projectable.shape # row_indices = projectable.mask.sum(1) == width # if row_indices.sum() != height: # projectable.mask[row_indices, :] = True # Get the orbit number # if not satscene.orbit: # mda = self.data.attributes()["CoreMetadata.0"] # orbit_idx = mda.index("ORBITNUMBER") # satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116] # Get the geolocation # if resolution != 1000: # logger.warning("Cannot load geolocation at this resolution (yet).") # return # Trimming out dead sensor lines (detectors) on terra: # (in addition channel 27, 30, 34, 35, and 36 are nosiy) # if satscene.satname == "terra": # for band in ["29"]: # if not satscene[band].is_loaded() or satscene[band].data.mask.all(): # continue # width = satscene[band].data.shape[1] # height = satscene[band].data.shape[0] # indices = satscene[band].data.mask.sum(1) < width # if indices.sum() == height: # continue # satscene[band] = satscene[band].data[indices, :] # satscene[band].area = geometry.SwathDefinition( # lons=satscene[band].area.lons[indices, :], # lats=satscene[band].area.lats[indices, :]) return projectable
def get_dataset(self, key, info): """Read data from file and return the corresponding projectables.""" datadict = { 1000: ['EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB', 'EV_1KM_Emissive'], 500: ['EV_250_Aggr500_RefSB', 'EV_500_RefSB'], 250: ['EV_250_RefSB']} platform_name = self.metadata['INVENTORYMETADATA']['ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][ 'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER']['ASSOCIATEDPLATFORMSHORTNAME']['VALUE'] info.update({'platform_name': 'EOS-' + platform_name}) info.update({'sensor': 'modis'}) if self.resolution != key.resolution: return datasets = datadict[self.resolution] for dataset in datasets: subdata = self.sd.select(dataset) band_names = subdata.attributes()["band_names"].split(",") # get the relative indices of the desired channel try: index = band_names.index(key.name) except ValueError: continue uncertainty = self.sd.select(dataset + "_Uncert_Indexes") array = xr.DataArray(from_sds(subdata, chunks=CHUNK_SIZE)[index, :, :], dims=['y', 'x']).astype(np.float32) valid_range = subdata.attributes()['valid_range'] array = array.where(array >= np.float32(valid_range[0])) array = array.where(array <= np.float32(valid_range[1])) array = array.where(from_sds(uncertainty, chunks=CHUNK_SIZE)[index, :, :] < 15) if dataset.endswith('Emissive'): projectable = calibrate_tb(array, subdata.attributes(), index, key.name) else: projectable = calibrate_refl(array, subdata.attributes(), index) projectable.attrs = info # if ((platform_name == 'Aqua' and key.name in ["6", "27", "36"]) or # (platform_name == 'Terra' and key.name in ["29"])): # height, width = projectable.shape # row_indices = projectable.mask.sum(1) == width # if row_indices.sum() != height: # projectable.mask[row_indices, :] = True # Get the orbit number # if not satscene.orbit: # mda = self.data.attributes()["CoreMetadata.0"] # orbit_idx = mda.index("ORBITNUMBER") # satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116] # Get the geolocation # if resolution != 1000: # logger.warning("Cannot load geolocation at this resolution (yet).") # return # Trimming out dead sensor lines (detectors) on terra: # (in addition channel 27, 30, 34, 35, and 36 are nosiy) # if satscene.satname == "terra": # for band in ["29"]: # if not satscene[band].is_loaded() or satscene[band].data.mask.all(): # continue # width = satscene[band].data.shape[1] # height = satscene[band].data.shape[0] # indices = satscene[band].data.mask.sum(1) < width # if indices.sum() == height: # continue # satscene[band] = satscene[band].data[indices, :] # satscene[band].area = geometry.SwathDefinition( # lons=satscene[band].area.lons[indices, :], # lats=satscene[band].area.lats[indices, :]) return projectable