def read_geo(self, key, info):
"""Read angles.
"""
if key.name in ['satellite_zenith_angle', 'satellite_azimuth_angle']:
if self.senazi is None or self.senzen is None:
self.senazi, self.senzen = self.angles("SatelliteAzimuthAngle",
"SatelliteZenithAngle")
if key.name == 'satellite_zenith_angle':
return Projectable(self.senzen,
copy=False,
name=key.name,
**self.mda)
else:
return Projectable(self.senazi,
copy=False,
name=key.name,
**self.mda)
if key.name in ['solar_zenith_angle', 'solar_azimuth_angle']:
if self.solazi is None or self.solzen is None:
self.solazi, self.solzen = self.angles("SolarAzimuthAngle",
"SolarZenithAngle")
if key.name == 'solar_zenith_angle':
return Projectable(self.solzen,
copy=False,
name=key.name,
**self.mda)
else:
return Projectable(self.solazi,
copy=False,
name=key.name,
**self.mda)
def __call__(self, projectables, optional_datasets=None, **info):
(band,) = projectables
factor = 8
#proj = Projectable(band[::factor, ::factor], copy=False, **band.info)
newshape = (band.shape[0] / factor, factor,
band.shape[1] / factor, factor)
proj = Projectable(band.reshape(newshape).mean(axis=3).mean(axis=1),
copy=False, **band.info)
self.apply_modifier_info(band, proj)
old_area = proj.info['area']
proj.info['area'] = AreaDefinition(old_area.area_id,
old_area.name,
old_area.proj_id,
old_area.proj_dict,
old_area.x_size / factor,
old_area.y_size / factor,
old_area.area_extent)
proj.info['resolution'] *= factor
proj.info['id'] = DatasetID(name=proj.info['id'].name,
resolution=proj.info['resolution'],
wavelength=proj.info['id'].wavelength,
polarization=proj.info['id'].polarization,
calibration=proj.info['id'].calibration,
modifiers=proj.info['id'].modifiers)
return proj
def get_dataset(self, key, info):
"""Get a dataset from the file."""
if key.name in CHANNEL_NAMES:
dataset = self.calibrate([key])[0]
else: # Get sun-sat angles
if key.name in ANGLES:
if isinstance(getattr(self, ANGLES[key.name]), np.ndarray):
dataset = Projectable(
getattr(self, ANGLES[key.name]),
copy=False)
else:
dataset = self.get_angles(key.name)
else:
logger.exception(
"Not a supported sun-sensor viewing angle: %s", key.name)
raise
# TODO get metadata
if self.lons is None or self.lats is None:
self.navigate()
if self.area is None:
self.area = SwathDefinition(self.lons, self.lats)
self.area.name = 'wla'
if not self._shape:
self._shape = dataset.shape
dataset.info['area'] = self.area
return dataset
def stack(datasets):
"""First dataset at the bottom."""
base = Projectable(datasets[0], copy=True)
for dataset in datasets[1:]:
base_mask = np.ma.getmaskarray(base)
other_mask = np.ma.getmaskarray(dataset)
base.mask = np.logical_and(base_mask, other_mask)
not_masked = np.logical_not(other_mask)
base[not_masked] = dataset[not_masked]
return base
def stack(datasets):
"""First dataset at the bottom."""
base = Projectable(datasets[0], copy=True)
for dataset in datasets[1:]:
base_mask = np.ma.getmaskarray(base)
other_mask = np.ma.getmaskarray(dataset)
base.mask = np.logical_and(base_mask, other_mask)
not_masked = np.logical_not(other_mask)
base[not_masked] = dataset[not_masked]
return base
def get_dataset(self, key, info):
avhrr_channel_index = {'1': 0,
'2': 1,
'3a': 2,
'3b': 2,
'4': 3,
'5': 4}
index = avhrr_channel_index[key.name]
mask = False
if key.name in ['3a', '3b'] and self._is3b is None:
ch3a = bfield(self._data["id"]["id"], 10)
self._is3b = np.logical_not(ch3a)
if key.name == '3a':
mask = np.tile(self._is3b, (1, 2048))
elif key.name == '3b':
mask = np.tile(np.logical_not(self._is3b), (1, 2048))
data = self._data["image_data"][:, :, index]
if key.calibration == 'counts':
return Projectable(data,
mask=mask,
area=self.get_lonlats(),
units='1')
pg_spacecraft = ''.join(self.platform_name.split()).lower()
jdays = (np.datetime64(self.start_time) - np.datetime64(str(
self.year) + '-01-01T00:00:00Z')) / np.timedelta64(1, 'D')
if index < 2 or key.name == '3a':
data = calibrate_solar(data, index, self.year, jdays,
pg_spacecraft)
units = '%'
if index > 2 or key.name == '3b':
if self.times is None:
self.times = time_seconds(self._data["timecode"], self.year)
line_numbers = (
np.round((self.times - self.times[-1]) /
np.timedelta64(166666667, 'ns'))).astype(np.int)
line_numbers -= line_numbers[0]
if self.prt is None:
self.prt, self.ict, self.space = self.get_telemetry()
chan = index + 1
data = calibrate_thermal(data, self.prt, self.ict[:, chan - 3],
self.space[:, chan - 3], line_numbers,
chan, pg_spacecraft)
units = 'K'
# TODO: check if entirely masked before returning
return Projectable(data, mask=mask, area=self.get_lonlats(), units=units)
def step_impl(context):
"""
:type context: behave.runner.Context
"""
from satpy.scene import Scene
from datetime import datetime
from satpy.projectable import Projectable
scn = Scene(platform_name="Suomi-NPP",
sensor="viirs",
start_time=datetime(2015, 3, 11, 11, 20),
end_time=datetime(2015, 3, 11, 11, 26))
scn["MyDataset"] = Projectable([[1, 2], [3, 4]])
scn["MyDataset2"] = Projectable([[5, 6], [7, 8]])
context.scene = scn
def __call__(self, projectables, optional_datasets=None, **info):
"""CO2 correction of the brightness temperature of the MSG 3.9um
channel.
.. math::
T4_CO2corr = (BT(IR3.9)^4 + Rcorr)^0.25
Rcorr = BT(IR10.8)^4 - (BT(IR10.8)-dt_CO2)^4
dt_CO2 = (BT(IR10.8)-BT(IR13.4))/4.0
"""
(ir_039, ir_108, ir_134) = projectables
LOG.info('Applying CO2 correction')
dt_co2 = (ir_108 - ir_134) / 4.0
rcorr = ir_108**4 - (ir_108 - dt_co2)**4
t4_co2corr = ir_039**4 + rcorr
t4_co2corr = np.ma.where(t4_co2corr > 0.0, t4_co2corr, 0)
t4_co2corr = t4_co2corr**0.25
info = ir_039.info.copy()
proj = Projectable(t4_co2corr, mask=t4_co2corr.mask, **info)
self.apply_modifier_info(ir_039, proj)
return proj
def compute(self, data,
weight_count=10000, weight_min=0.01, weight_distance_max=1.0,
weight_sum_min=-1.0, maximum_weight_mode=False,
**kwargs):
rows = self.cache["rows"]
cols = self.cache["cols"]
# if the data is scan based then check its metadata or the passed kwargs
# otherwise assume the entire input swath is one large "scanline"
rows_per_scan = getattr(data, "info", kwargs).get("rows_per_scan", data.shape[0])
num_valid_points, out_arrs = fornav(cols, rows, self.target_area_def, data, rows_per_scan=rows_per_scan,
weight_count=weight_count, weight_min=weight_min,
weight_distance_max=weight_distance_max, weight_sum_min=weight_sum_min,
maximum_weight_mode=maximum_weight_mode, **kwargs)
num_valid_points = num_valid_points[0]
out_arr = out_arrs[0]
grid_covered_ratio = num_valid_points / float(out_arr.size)
grid_covered = grid_covered_ratio > self.grid_coverage
if not grid_covered:
msg = "EWA resampling only found %f%% of the grid covered (need %f%%)" % (grid_covered_ratio * 100,
self.grid_coverage * 100)
raise RuntimeError(msg)
LOG.debug("EWA resampling found %f%% of the grid covered" % (grid_covered_ratio * 100))
info = getattr(data, "info", {})
info["mask"] = np.isnan(out_arr) if np.issubdtype(data.dtype, np.floating) else out_arr == fill
return Projectable(out_arr, **info)
def get_dataset(self, key, info):
"""Load a dataset
"""
if self.channel != key.name:
return
logger.debug('Reading %s.', key.name)
variable = self.nc[self.channel + '_radiance']
radiances = (np.ma.masked_equal(variable[:],
variable.attrs['_FillValue'], copy=False) *
variable.attrs['scale_factor'] +
variable.attrs['add_offset'])
units = variable.attrs['units']
if key.calibration == 'reflectance':
solar_flux = self.cal['solar_flux'][:]
d_index = np.ma.masked_equal(self.cal['detector_index'][:],
self.cal['detector_index'].attrs[
'_FillValue'],
copy=False)
idx = int(key.name[2:]) - 1
radiances /= solar_flux[idx, d_index]
radiances *= np.pi * 100
units = '%'
proj = Projectable(radiances,
copy=False,
units=units,
platform_name=self.platform_name,
sensor=self.sensor)
return proj
def get_dataset(self, ds_key, ds_info, out=None):
var_path = ds_info["file_key"]
fill_value = ds_info.get("fill_value", 65535)
data = self[var_path]
dtype = ds_info.get("dtype", np.float32)
mask = data == fill_value
data = data.astype(dtype) * self[var_path + "/attr/SCALE FACTOR"]
if ((ds_info.get('standard_name') == "longitude"
or ds_info.get('standard_name') == "latitude")
and ds_key.resolution == 10000):
# FIXME: Lower frequency channels need CoRegistration parameters
# applied
data = data[:, ::2]
mask = mask[:, ::2]
ds_info.update({
"name":
ds_key.name,
"id":
ds_key,
"units":
self[var_path + "/attr/UNIT"],
"platform":
self["/attr/PlatformShortName"].item(),
"sensor":
self["/attr/SensorShortName"].item(),
"start_orbit":
int(self["/attr/StartOrbitNumber"].item()),
"end_orbit":
int(self["/attr/StopOrbitNumber"].item()),
})
return Projectable(data, mask=mask, **ds_info)
def __call__(self, projectables, nonprojectables=None, **info):
if len(projectables) != 3:
raise ValueError("Expected 3 datasets, got %d" %
(len(projectables), ))
try:
the_data = np.rollaxis(
np.ma.dstack([projectable for projectable in projectables]),
axis=2)
except ValueError:
raise IncompatibleAreas
# info = projectables[0].info.copy()
# info.update(projectables[1].info)
# info.update(projectables[2].info)
info = combine_info(*projectables)
info.update(self.info)
info['id'] = DatasetID(self.info['name'])
# FIXME: should this be done here ?
info["wavelength_range"] = None
info.pop("units", None)
sensor = set()
for projectable in projectables:
current_sensor = projectable.info.get("sensor", None)
if current_sensor:
if isinstance(current_sensor, (str, bytes, six.text_type)):
sensor.add(current_sensor)
else:
sensor |= current_sensor
if len(sensor) == 0:
sensor = None
elif len(sensor) == 1:
sensor = list(sensor)[0]
info["sensor"] = sensor
info["mode"] = "RGB"
return Projectable(data=the_data, **info)
def get_angles(self, angle_id):
"""Get sun-satellite viewing angles"""
tic = datetime.now()
sunz40km = self._data["ang"][:, :, 0] * 1e-2
satz40km = self._data["ang"][:, :, 1] * 1e-2
azidiff40km = self._data["ang"][:, :, 2] * 1e-2
try:
from geotiepoints.interpolator import Interpolator
except ImportError:
logger.warning("Could not interpolate sun-sat angles, "
"python-geotiepoints missing.")
self.sunz, self.satz, self.azidiff = sunz40km, satz40km, azidiff40km
else:
cols40km = np.arange(24, 2048, 40)
cols1km = np.arange(2048)
lines = sunz40km.shape[0]
rows40km = np.arange(lines)
rows1km = np.arange(lines)
along_track_order = 1
cross_track_order = 3
satint = Interpolator(
[sunz40km, satz40km, azidiff40km], (rows40km, cols40km),
(rows1km, cols1km), along_track_order, cross_track_order)
self.sunz, self.satz, self.azidiff = satint.interpolate()
logger.debug("Interpolate sun-sat angles: time %s",
str(datetime.now() - tic))
return Projectable(getattr(self, ANGLES[angle_id]), copy=False)
def get_dataset(self, key, info=None):
"""Load a dataset
"""
if key in self.cache:
return self.cache[key]
# Type dictionary
typedict = {
"af": "flash_accumulation",
"afa": "accumulated_flash_area",
"afr": "flash_radiance",
"lgr": "radiance",
"lef": "radiance",
"lfl": "radiance"
}
#Get lightning data out of NetCDF container
logger.debug("KEY: %s" % key.name)
# Get grid dimensions from file
refdim = self.nc['grid_position'][:]
# Get number of lines and columns
self.nlines = int(refdim[2])
self.ncols = int(refdim[3])
# Create reference grid
grid = np.full((refdim[2], refdim[3]), np.NaN)
# Get product value
values = self.nc[typedict[key.name]][:]
rows = self.nc['row'][:]
cols = self.nc['column'][:]
# Convert xy coordinates to flattend indices
ids = np.ravel_multi_index([rows, cols], grid.shape)
# Replace NaN values with data
np.put(grid, ids, values)
# Correct for bottom left origin in LI row/column indices.
rotgrid = np.flipud(grid)
logger.debug('DATA SHAPE: %s' % str(rotgrid.shape))
# Rotate the grid by 90 degree clockwise
logger.warning("LI data has been roteted to fit to reference grid. \
Works only for test dataset")
rotgrid = np.rot90(rotgrid, 3)
logger.debug('[ Dimension ] : %s' % (refdim))
logger.debug("ROW/COLS: %d / %d" % (self.nlines, self.ncols))
logger.debug('[ Number of values ] : %d' % (len(values)))
logger.debug('[Min/Max] : <%d> / <%d>' %
(np.min(values), np.max(values)))
logger.debug("START: %s" % self.start_time)
logger.debug("END: %s" % self.end_time)
ds = (np.ma.masked_invalid(rotgrid[:]))
# Create projectable object
out = Projectable(ds, dtype=np.float32)
self.cache[key] = out
return (out)
def get_dataset(self, key, info, out=None):
to_return = out is None
if out is None:
nlines = int(self.data_info['number_of_lines'])
ncols = int(self.data_info['number_of_columns'])
out = Projectable(np.ma.empty((nlines, ncols), dtype=np.float32))
self.read_band(key, info, out)
if to_return:
from satpy.yaml_reader import Shuttle
return Shuttle(out.data, out.mask, out.info)
def get_dataset(self, key, info):
"""Load a dataset
"""
logger.debug('Reading %s.', key.name)
variable = self.nc[key.name]
ds = (np.ma.masked_equal(variable[:], variable.attrs['_FillValue']) *
(variable.attrs['scale_factor'] * 1.0) +
variable.attrs.get('add_offset', 0))
proj = Projectable(ds, copy=False, **info)
return proj
def __call__(self, projectables, optional_datasets=None, **info):
"""Boost vegetation effect thanks to NIR (0.8µm) band."""
(green, nir) = projectables
LOG.info('Boosting vegetation on green band')
proj = Projectable(green * 0.85 + nir * 0.15,
copy=False,
**green.info)
self.apply_modifier_info(green, proj)
return proj
def get_dataset(self, key, info):
if key.name not in ['longitude', 'latitude']:
return
if self.lons is None or self.lats is None:
lons_id = DatasetID('longitude', resolution=key.resolution)
lats_id = DatasetID('latitude', resolution=key.resolution)
lons, lats = self.load([lons_id, lats_id],
interpolate=False,
raw=True)
from geotiepoints.geointerpolator import GeoInterpolator
self.lons, self.lats = self._interpolate([lons, lats],
self.resolution,
lons_id.resolution,
GeoInterpolator)
if key.name == 'latitude':
return Projectable(self.lats, id=key, **info)
else:
return Projectable(self.lons, id=key, **info)
def __call__(self, projectables, nonprojectables=None, **info):
if len(projectables) != 3:
raise ValueError("Expected 3 datasets, got %d" %
(len(projectables), ))
# Collect information that is the same between the projectables
info = combine_info(*projectables)
# Update that information with configured information (including name)
info.update(self.info)
# Force certain pieces of metadata that we *know* to be true
info["wavelength"] = None
info["mode"] = self.info.get("mode", "RGB")
return Projectable(data=np.rollaxis(np.ma.dstack(
[projectable for projectable in projectables]),
axis=2),
**info)
def get_dataset(self, key, info):
"""Read data from file and return the corresponding projectables.
"""
datadict = {
1000: [
'EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB',
'EV_1KM_Emissive'
],
500: ['EV_250_Aggr500_RefSB', 'EV_500_RefSB'],
250: ['EV_250_RefSB']
}
platform_name = self.mda['INVENTORYMETADATA'][
'ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][
'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER'][
'ASSOCIATEDPLATFORMSHORTNAME']['VALUE']
if self.resolution != key.resolution:
return
datasets = datadict[self.resolution]
for dataset in datasets:
subdata = self.sd.select(dataset)
band_names = subdata.attributes()["band_names"].split(",")
# get the relative indices of the desired channel
try:
index = band_names.index(key.name)
except ValueError:
continue
uncertainty = self.sd.select(dataset + "_Uncert_Indexes")
if dataset.endswith('Emissive'):
array = calibrate_tb(subdata, uncertainty, [index], band_names)
else:
array = calibrate_refl(subdata, uncertainty, [index])
projectable = Projectable(array[0], id=key, mask=array[0].mask)
# if ((platform_name == 'Aqua' and key.name in ["6", "27", "36"]) or
# (platform_name == 'Terra' and key.name in ["29"])):
# height, width = projectable.shape
# row_indices = projectable.mask.sum(1) == width
# if row_indices.sum() != height:
# projectable.mask[row_indices, :] = True
return projectable
def get_dataset(self, ds_key, ds_info, out=None):
var_path = ds_info["file_key"]
fill_value = ds_info.get("fill_value", 65535)
data = self[var_path]
dtype = ds_info.get("dtype", np.float32)
mask = data == fill_value
data = data.astype(dtype) * self[var_path + "/attr/SCALE FACTOR"]
ds_info.update({
"name": ds_key.name,
"id": ds_key,
"units": self[var_path + "/attr/UNIT"],
"platform": self["/attr/PlatformShortName"].item(),
"sensor": self["/attr/SensorShortName"].item(),
"start_orbit": int(self["/attr/StartOrbitNumber"].item()),
"end_orbit": int(self["/attr/StopOrbitNumber"].item()),
})
return Projectable(data, mask=mask, **ds_info)
def get_dataset(self, key, info):
"""Load a dataset
"""
logger.debug('Reading %s.', key.name)
print self.filename, info['file_key']
try:
variable = self.nc[info['file_key']]
except KeyError:
return
ds = (np.ma.masked_equal(variable[:], variable.attrs['_FillValue']) *
(variable.attrs.get('scale_factor', 1) * 1.0) +
variable.attrs.get('add_offset', 0))
ds.mask = np.ma.getmaskarray(ds)
proj = Projectable(ds, copy=False, **info)
return proj
def __call__(self, projectables, optional_datasets=None, **info):
"""Get the corrected reflectance when removing Rayleigh scattering. Uses
pyspectral.
"""
from pyspectral.rayleigh import Rayleigh
(vis, ) = projectables
try:
(sata, satz, suna, sunz) = optional_datasets
except ValueError:
from pyorbital.astronomy import get_alt_az, sun_zenith_angle
from pyorbital.orbital import get_observer_look
sunalt, suna = get_alt_az(vis.info['start_time'],
*vis.info['area'].get_lonlats())
sunz = sun_zenith_angle(vis.info['start_time'],
*vis.info['area'].get_lonlats())
lons, lats = vis.info['area'].get_lonlats()
sata, satel = get_observer_look(vis.info['satellite_longitude'],
vis.info['satellite_latitude'],
vis.info['satellite_altitude'],
vis.info['start_time'], lons, lats,
0)
satz = 90 - satel
LOG.info('Removing Rayleigh scattering')
ssadiff = np.abs(suna - sata)
ssadiff = np.where(np.greater(ssadiff, 180), 360 - ssadiff, ssadiff)
corrector = Rayleigh(vis.info['platform_name'],
vis.info['sensor'],
atmosphere='us-standard',
rural_aerosol=False)
refl_cor_band = corrector.get_reflectance(sunz, satz, ssadiff,
vis.info['id'].wavelength[1],
vis)
proj = Projectable(vis - refl_cor_band, copy=False, **vis.info)
self.apply_modifier_info(vis, proj)
return proj
def get_dataset(self, key, info):
"""Load a dataset
"""
if key.name not in self.datasets:
return
if self.nc is None:
self.nc = h5netcdf.File(self.filename, 'r')
logger.debug('Reading %s.', key.name)
variable = self.nc[self.datasets[key.name]]
values = (np.ma.masked_equal(variable[:],
variable.attrs['_FillValue'], copy=False) *
variable.attrs.get('scale_factor', 1) +
variable.attrs.get('add_offset', 0))
units = variable.attrs['units']
l_step = self.nc.attrs['al_subsampling_factor']
c_step = self.nc.attrs['ac_subsampling_factor']
if c_step != 1 or l_step != 1:
from geotiepoints.interpolator import Interpolator
tie_lines = np.arange(
0, (values.shape[0] - 1) * l_step + 1, l_step)
tie_cols = np.arange(0, (values.shape[1] - 1) * c_step + 1, c_step)
lines = np.arange((values.shape[0] - 1) * l_step + 1)
cols = np.arange((values.shape[1] - 1) * c_step + 1)
along_track_order = 1
cross_track_order = 3
satint = Interpolator([values],
(tie_lines, tie_cols),
(lines, cols),
along_track_order,
cross_track_order)
(values, ) = satint.interpolate()
proj = Projectable(values,
copy=False,
units=units,
platform_name=self.platform_name,
sensor=self.sensor)
return proj
def load(self, keys, interpolate=True, raw=False):
projectables = []
for key in keys:
dataset = self.sd.select(key.name.capitalize())
fill_value = dataset.attributes()["_FillValue"]
try:
scale_factor = dataset.attributes()["scale_factor"]
except KeyError:
scale_factor = 1
data = np.ma.masked_equal(dataset.get(), fill_value) * scale_factor
# TODO: interpolate if needed
if key.resolution is not None and key.resolution < self.resolution and interpolate:
data = self._interpolate(data, self.resolution, key.resolution)
if raw:
projectables.append(data)
else:
projectables.append(Projectable(data, id=key))
return projectables
def __call__(self, projectables, optional_datasets=None, **info):
"""Get the reflectance part of an NIR channel. Not supposed to be used
for wavelength outside [3, 4] µm.
"""
try:
from pyspectral.near_infrared_reflectance import Calculator
except ImportError:
LOG.info("Couldn't load pyspectral")
raise
nir, tb11 = projectables
LOG.info('Getting reflective part of %s', nir.info['name'])
sun_zenith = None
tb13_4 = None
for dataset in optional_datasets:
if dataset.info["standard_name"] == "solar_zenith_angle":
sun_zenith = dataset
elif (dataset.info['units'] == 'K' and
"wavelengh" in dataset.info and
dataset.info["wavelength"][0] <= 13.4 <= dataset.info["wavelength"][2]):
tb13_4 = dataset
# Check if the sun-zenith angle was provided:
if sun_zenith is None:
from pyorbital.astronomy import sun_zenith_angle as sza
lons, lats = nir.info["area"].get_lonlats()
sun_zenith = sza(nir.info['start_time'], lons, lats)
refl39 = Calculator(nir.info['platform_name'],
nir.info['sensor'], nir.info['name'])
proj = Projectable(refl39.reflectance_from_tbs(sun_zenith, nir,
tb11, tb13_4),
**nir.info)
self.apply_modifier_info(nir, proj)
return proj
def get_dataset(self, key, info):
"""Load a dataset
"""
logger.debug('Reading in get_dataset%s.', key.name)
variable = self.nc["Rad"]
radiances = (np.ma.masked_equal(
variable[:], variable.attrs['_FillValue'], copy=False) *
variable.attrs['scale_factor'] +
variable.attrs['add_offset'])
units = variable.attrs['units']
self.calibrate(radiances, key)
proj = Projectable(radiances,
copy=False,
units=units,
platform_name=self.platform_name,
sensor=self.sensor)
self.nlines, self.ncols = radiances.shape
return proj
def read_geo(self, key, info):
"""Read angles.
"""
if key.name in ['satellite_zenith_angle', 'satellite_azimuth_angle']:
if self.senazi is None or self.senzen is None:
self.senazi, self.senzen = self.angles("SatelliteAzimuthAngle",
"SatelliteZenithAngle")
if key.name == 'satellite_zenith_angle':
return Projectable(self.senzen,
copy=False,
name=key.name,
**self.mda)
else:
return Projectable(self.senazi,
copy=False,
name=key.name,
**self.mda)
if key.name in ['solar_zenith_angle', 'solar_azimuth_angle']:
if self.solazi is None or self.solzen is None:
self.solazi, self.solzen = self.angles("SolarAzimuthAngle",
"SolarZenithAngle")
if key.name == 'solar_zenith_angle':
return Projectable(self.solzen,
copy=False,
name=key.name,
**self.mda)
else:
return Projectable(self.solazi,
copy=False,
name=key.name,
**self.mda)
if info.get('standard_name') in ['latitude', 'longitude']:
if self.lons is None or self.lats is None:
self.lons, self.lats = self.navigate()
mda = self.mda.copy()
mda.update(info)
if info['standard_name'] == 'longitude':
return Projectable(self.lons, copy=False, id=key, **mda)
else:
return Projectable(self.lats, copy=False, id=key, **mda)
def load(self, dataset_keys, area=None, start_time=None, end_time=None):
image_files = []
pattern = self.file_patterns[0]
prologue_file = None
epilogue_file = None
for filename in self.info['filenames']:
try:
file_info = parse(pattern, os.path.basename(filename))
except ValueError:
continue
if file_info["segment"] == "EPI":
epilogue_file = filename
elif file_info["segment"] == "PRO":
prologue_file = filename
else:
image_files.append(filename)
start_times = set()
datasets = DatasetDict()
area_converted_to_extent = False
area_extent = None
for ds in dataset_keys:
channel_files = []
for filename in image_files:
file_info = parse(pattern, os.path.basename(filename))
if file_info["dataset_name"] == ds.name:
channel_files.append(filename)
start_times.add(file_info['start_time'])
if not channel_files:
continue
kwargs = {}
if 'platform_name' in self.info:
kwargs['platform_name'] = self.info['platform_name']
# Convert area definitions to maximal area_extent
if not area_converted_to_extent and area is not None:
metadata = xrit.sat.load_files(prologue_file,
channel_files,
epilogue_file,
only_metadata=True,
**kwargs)
# otherwise use the default value (MSG3 extent at
# lon0=0.0), that is, do not pass default_extent=area_extent
area_extent = area_defs_to_extent(
[area], metadata.proj4_params)
area_converted_to_extent = True
try:
calibrate = 1
if ds.calibration == 'counts':
calibrate = 0
elif ds.calibration == 'radiance':
calibrate = 2
image = xrit.sat.load_files(prologue_file,
channel_files,
epilogue_file,
mask=True,
calibrate=calibrate,
**kwargs)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except CalibrationError:
LOGGER.warning(
"Loading non calibrated data since calibration failed.")
image = xrit.sat.load_files(prologue_file,
channel_files,
epilogue_file,
mask=True,
calibrate=False,
**kwargs)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except ReaderError as err:
# if dataset can't be found, go on with next dataset
LOGGER.error(str(err))
continue
if len(metadata.instruments) != 1:
sensor = None
else:
sensor = metadata.instruments[0]
units = {'ALBEDO(%)': '%',
'KELVIN': 'K'}
standard_names = {'1': 'counts',
'W m-2 sr-1 m-1':
'toa_outgoing_radiance_per_unit_wavelength',
'%': 'toa_bidirectional_reflectance',
'K':
'toa_brightness_temperature'}
unit = units.get(metadata.calibration_unit,
metadata.calibration_unit)
projectable = Projectable(
data,
name=ds.name,
units=unit,
standard_name=standard_names[unit],
sensor=sensor,
start_time=min(start_times),
id=ds)
# Build an area on the fly from the mipp metadata
proj_params = getattr(metadata, "proj4_params").split(" ")
proj_dict = {}
for param in proj_params:
key, val = param.split("=")
proj_dict[key] = val
if IS_PYRESAMPLE_LOADED:
# Build area_def on-the-fly
projectable.info["area"] = geometry.AreaDefinition(
str(metadata.area_extent) + str(data.shape),
"On-the-fly area", proj_dict["proj"], proj_dict,
data.shape[1], data.shape[0], metadata.area_extent)
else:
LOGGER.info("Could not build area, pyresample missing...")
datasets[ds] = projectable
return datasets
def load(self, keys):
"""Read data from file and return the corresponding projectables.
"""
datadict = {
1000: ['EV_250_Aggr1km_RefSB',
'EV_500_Aggr1km_RefSB',
'EV_1KM_RefSB',
'EV_1KM_Emissive'],
500: ['EV_250_Aggr500_RefSB',
'EV_500_RefSB'],
250: ['EV_250_RefSB']}
projectables = []
platform_name = self.mda['INVENTORYMETADATA']['ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][
'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER']['ASSOCIATEDPLATFORMSHORTNAME']['VALUE']
keys = [key for key in keys if key.resolution == self.resolution]
if len(keys) == 0:
logger.debug("Nothing to read in %s.", self.filename)
return projectables
datasets = datadict[self.resolution]
key_names = [key.name for key in keys]
for dataset in datasets:
subdata = self.sd.select(dataset)
band_names = subdata.attributes()["band_names"].split(",")
if len(set(key_names) & set(band_names)) > 0:
# get the relative indices of the desired channels
indices = [i for i, band in enumerate(band_names)
if band in key_names]
uncertainty = self.sd.select(dataset + "_Uncert_Indexes")
if dataset.endswith('Emissive'):
array = calibrate_tb(
subdata, uncertainty, indices, band_names)
else:
array = calibrate_refl(subdata, uncertainty, indices)
for (i, idx) in enumerate(indices):
dsid = [key for key in keys if key.name ==
band_names[idx]][0]
area = self.navigation_reader.get_lonlats(self.resolution)
projectable = Projectable(array[i], id=dsid, area=area)
if ((platform_name == 'Aqua' and dsid.name in ["6", "27", "36"]) or
(platform_name == 'Terra' and dsid.name in ["29"])):
height, width = projectable.shape
row_indices = projectable.mask.sum(1) == width
if row_indices.sum() == height:
continue
projectable.mask[indices, :] = True
projectables.append(projectable)
return projectables
# Get the orbit number
if not satscene.orbit:
mda = self.data.attributes()["CoreMetadata.0"]
orbit_idx = mda.index("ORBITNUMBER")
satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116]
# Get the geolocation
# if resolution != 1000:
# logger.warning("Cannot load geolocation at this resolution (yet).")
# return
for band_name in loaded_bands:
lon, lat = self.get_lonlat(satscene[band_name].resolution, cores)
area = geometry.SwathDefinition(lons=lon, lats=lat)
satscene[band_name].area = area
# Trimming out dead sensor lines (detectors) on aqua:
# (in addition channel 21 is noisy)
if satscene.satname == "aqua":
for band in ["6", "27", "36"]:
if not satscene[band].is_loaded() or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
# Trimming out dead sensor lines (detectors) on terra:
# (in addition channel 27, 30, 34, 35, and 36 are nosiy)
if satscene.satname == "terra":
for band in ["29"]:
if not satscene[band].is_loaded() or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
for band_name in loaded_bands:
band_uid = hashlib.sha1(satscene[band_name].data.mask).hexdigest()
satscene[band_name].area.area_id = ("swath_" + satscene.fullname + "_"
+ str(satscene.time_slot) + "_"
+
str(satscene[
band_name].shape) + "_"
+ str(band_uid))
satscene[band_name].area_id = satscene[band_name].area.area_id
def get_dataset(self, key, info):
"""Get calibrated channel data."""
if self.mdrs is None:
self._read_all(self.filename)
if key.name in ['longitude', 'latitude']:
lons, lats = self.get_full_lonlats()
# todo: make that datasets
if key.name == 'longitude':
return Projectable(lons, id=key, **info)
else:
return Projectable(lats, id=key, **info)
if key.calibration == 'counts':
raise ValueError('calibration=counts is not supported! ' +
'This reader cannot return counts')
elif key.calibration not in [
'reflectance', 'brightness_temperature', 'radiance'
]:
raise ValueError('calibration type ' + str(key.calibration) +
' is not supported!')
if key.name in ['3A', '3a'] and self.three_a_mask is None:
self.three_a_mask = ((self["FRAME_INDICATOR"] & 2**16) != 2**16)
if key.name in ['3B', '3b'] and self.three_b_mask is None:
self.three_b_mask = ((self["FRAME_INDICATOR"] & 2**16) != 0)
if key.name not in ["1", "2", "3a", "3A", "3b", "3B", "4", "5"]:
LOG.info("Can't load channel in eps_l1b: " + str(key.name))
return
if key.name == "1":
if key.calibration == 'reflectance':
array = np.ma.array(
radiance_to_refl(self["SCENE_RADIANCES"][:, 0, :],
self["CH1_SOLAR_FILTERED_IRRADIANCE"]))
else:
array = np.ma.array(self["SCENE_RADIANCES"][:, 0, :])
if key.name == "2":
if key.calibration == 'reflectance':
array = np.ma.array(
radiance_to_refl(self["SCENE_RADIANCES"][:, 1, :],
self["CH1_SOLAR_FILTERED_IRRADIANCE"]))
else:
array = np.ma.array(self["SCENE_RADIANCES"][:, 1, :])
if key.name.lower() == "3a":
if key.calibration == 'reflectance':
array = np.ma.array(
radiance_to_refl(self["SCENE_RADIANCES"][:, 2, :],
self["CH2_SOLAR_FILTERED_IRRADIANCE"]))
else:
array = np.ma.array(self["SCENE_RADIANCES"][:, 2, :])
mask = np.empty(array.shape, dtype=bool)
mask[:, :] = self.three_a_mask[:, np.newaxis]
array = np.ma.array(array, mask=mask, copy=False)
if key.name.lower() == "3b":
if key.calibration == 'brightness_temperature':
array = np.array(
radiance_to_bt(self["SCENE_RADIANCES"][:, 2, :],
self["CH3B_CENTRAL_WAVENUMBER"],
self["CH3B_CONSTANT1"],
self["CH3B_CONSTANT2_SLOPE"]))
else:
array = self["SCENE_RADIANCES"][:, 2, :]
mask = np.empty(array.shape, dtype=bool)
mask[:, :] = self.three_b_mask[:, np.newaxis]
array = np.ma.array(array, mask=mask, copy=False)
if key.name == "4":
if key.calibration == 'brightness_temperature':
array = np.ma.array(
radiance_to_bt(self["SCENE_RADIANCES"][:, 3, :],
self["CH4_CENTRAL_WAVENUMBER"],
self["CH4_CONSTANT1"],
self["CH4_CONSTANT2_SLOPE"]))
else:
array = np.ma.array(self["SCENE_RADIANCES"][:, 3, :])
if key.name == "5":
if key.calibration == 'brightness_temperature':
array = np.ma.array(
radiance_to_bt(self["SCENE_RADIANCES"][:, 4, :],
self["CH5_CENTRAL_WAVENUMBER"],
self["CH5_CONSTANT1"],
self["CH5_CONSTANT2_SLOPE"]))
else:
array = np.ma.array(self["SCENE_RADIANCES"][:, 4, :])
proj = Projectable(array, mask=array.mask, id=key)
return proj
def load(self, keys):
"""Read data from file and return the corresponding projectables.
"""
datadict = {
1000: [
'EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB',
'EV_1KM_Emissive'
],
500: ['EV_250_Aggr500_RefSB', 'EV_500_RefSB'],
250: ['EV_250_RefSB']
}
projectables = []
platform_name = self.mda['INVENTORYMETADATA'][
'ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][
'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER'][
'ASSOCIATEDPLATFORMSHORTNAME']['VALUE']
keys = [key for key in keys if key.resolution == self.resolution]
if len(keys) == 0:
logger.debug("Nothing to read in %s.", self.filename)
return projectables
datasets = datadict[self.resolution]
key_names = [key.name for key in keys]
for dataset in datasets:
subdata = self.sd.select(dataset)
band_names = subdata.attributes()["band_names"].split(",")
if len(set(key_names) & set(band_names)) > 0:
# get the relative indices of the desired channels
indices = [
i for i, band in enumerate(band_names) if band in key_names
]
uncertainty = self.sd.select(dataset + "_Uncert_Indexes")
if dataset.endswith('Emissive'):
array = calibrate_tb(subdata, uncertainty, indices,
band_names)
else:
array = calibrate_refl(subdata, uncertainty, indices)
for (i, idx) in enumerate(indices):
dsid = [
key for key in keys if key.name == band_names[idx]
][0]
area = self.navigation_reader.get_lonlats(self.resolution)
projectable = Projectable(array[i], id=dsid, area=area)
if ((platform_name == 'Aqua'
and dsid.name in ["6", "27", "36"]) or
(platform_name == 'Terra' and dsid.name in ["29"])):
height, width = projectable.shape
row_indices = projectable.mask.sum(1) == width
if row_indices.sum() == height:
continue
projectable.mask[indices, :] = True
projectables.append(projectable)
return projectables
# Get the orbit number
if not satscene.orbit:
mda = self.data.attributes()["CoreMetadata.0"]
orbit_idx = mda.index("ORBITNUMBER")
satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116]
# Get the geolocation
# if resolution != 1000:
# logger.warning("Cannot load geolocation at this resolution (yet).")
# return
for band_name in loaded_bands:
lon, lat = self.get_lonlat(satscene[band_name].resolution, cores)
area = geometry.SwathDefinition(lons=lon, lats=lat)
satscene[band_name].area = area
# Trimming out dead sensor lines (detectors) on aqua:
# (in addition channel 21 is noisy)
if satscene.satname == "aqua":
for band in ["6", "27", "36"]:
if not satscene[band].is_loaded(
) or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
# Trimming out dead sensor lines (detectors) on terra:
# (in addition channel 27, 30, 34, 35, and 36 are nosiy)
if satscene.satname == "terra":
for band in ["29"]:
if not satscene[band].is_loaded(
) or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
for band_name in loaded_bands:
band_uid = hashlib.sha1(satscene[band_name].data.mask).hexdigest()
satscene[band_name].area.area_id = (
"swath_" + satscene.fullname + "_" + str(satscene.time_slot) +
"_" + str(satscene[band_name].shape) + "_" + str(band_uid))
satscene[band_name].area_id = satscene[band_name].area.area_id
