def get_lonlats(self, navid, nav_info, lon_out=None, lat_out=None):
# TODO: read in place when lon_out and lat_out are provided
lons_id = DatasetID(nav_info['longitude_key'],
resolution=navid.resolution)
lats_id = DatasetID(nav_info['latitude_key'],
resolution=navid.resolution)
try:
lons = self.cache[lons_id]
lats = self.cache[lats_id]
except KeyError:
lons, lats = self.load(
[lons_id, lats_id], interpolate=False, raw=True)
from geotiepoints.geointerpolator import GeoInterpolator
lons, lats = self._interpolate(
[lons, lats], self.resolution, lons_id.resolution, GeoInterpolator)
self.cache[lons_id] = lons
self.cache[lats_id] = lats
if lon_out is not None:
lon_out[:] = lons[:]
if lat_out is not None:
lat_out[:] = lats[:]
#navid.name = self.mda['ARCHIVEDMETADATA']['LONGNAME']['VALUE']
return lons, lats
def __call__(self, projectables, nonprojectables=None, **info):
if len(projectables) != 3:
raise ValueError("Expected 3 datasets, got %d" %
(len(projectables), ))
try:
the_data = np.rollaxis(
np.ma.dstack([projectable for projectable in projectables]),
axis=2)
except ValueError:
raise IncompatibleAreas
# info = projectables[0].info.copy()
# info.update(projectables[1].info)
# info.update(projectables[2].info)
info = combine_info(*projectables)
info.update(self.info)
info['id'] = DatasetID(self.info['name'])
# FIXME: should this be done here ?
info["wavelength_range"] = None
info.pop("units", None)
sensor = set()
for projectable in projectables:
current_sensor = projectable.info.get("sensor", None)
if current_sensor:
if isinstance(current_sensor, (str, bytes, six.text_type)):
sensor.add(current_sensor)
else:
sensor |= current_sensor
if len(sensor) == 0:
sensor = None
elif len(sensor) == 1:
sensor = list(sensor)[0]
info["sensor"] = sensor
info["mode"] = "RGB"
return Projectable(data=the_data, **info)
def test_init_dict(self):
from satpy.readers import DatasetDict, DatasetID
regular_dict = {
DatasetID(name="test", wavelength=(0, 0.5, 1)): "1",
}
d = DatasetDict(regular_dict)
self.assertEqual(d, regular_dict)
def __call__(self, projectables, optional_datasets=None, **info):
(band,) = projectables
factor = 8
#proj = Projectable(band[::factor, ::factor], copy=False, **band.info)
newshape = (band.shape[0] / factor, factor,
band.shape[1] / factor, factor)
proj = Projectable(band.reshape(newshape).mean(axis=3).mean(axis=1),
copy=False, **band.info)
self.apply_modifier_info(band, proj)
old_area = proj.info['area']
proj.info['area'] = AreaDefinition(old_area.area_id,
old_area.name,
old_area.proj_id,
old_area.proj_dict,
old_area.x_size / factor,
old_area.y_size / factor,
old_area.area_extent)
proj.info['resolution'] *= factor
proj.info['id'] = DatasetID(name=proj.info['id'].name,
resolution=proj.info['resolution'],
wavelength=proj.info['id'].wavelength,
polarization=proj.info['id'].polarization,
calibration=proj.info['id'].calibration,
modifiers=proj.info['id'].modifiers)
return proj
def get_dataset(self, key, info):
if key.name not in ['longitude', 'latitude']:
return
if self.lons is None or self.lats is None:
lons_id = DatasetID('longitude', resolution=key.resolution)
lats_id = DatasetID('latitude', resolution=key.resolution)
lons, lats = self.load([lons_id, lats_id],
interpolate=False,
raw=True)
from geotiepoints.geointerpolator import GeoInterpolator
self.lons, self.lats = self._interpolate([lons, lats],
self.resolution,
lons_id.resolution,
GeoInterpolator)
if key.name == 'latitude':
return Projectable(self.lats, id=key, **info)
else:
return Projectable(self.lons, id=key, **info)
def apply_modifier_info(self, origin, destination):
mods = origin.info.get('modifiers', [])
if mods is None:
mods = []
else:
mods = list(mods)
mods.append(self.info['name'])
old_id = origin.info['id']
did = DatasetID(old_id.name, old_id.wavelength, old_id.resolution,
old_id.polarization, old_id.calibration, tuple(mods))
destination.info['modifiers'] = mods
destination.info['id'] = did
def _get_coordinates_for_dataset_key(self, dsid):
"""Get the coordinate dataset keys for *dsid*."""
ds_info = self.ids[dsid]
cids = []
for cinfo in ds_info.get('coordinates', []):
if isinstance(cinfo, dict):
cinfo['resolution'] = ds_info['resolution']
else:
# cid = self.get_dataset_key(cinfo)
cinfo = {'name': cinfo, 'resolution': ds_info['resolution']}
cid = DatasetID(**cinfo)
cids.append(self.get_dataset_key(cid))
return cids
def test_get_keys_by_datasetid(self):
from satpy.readers import DatasetDict, DatasetID
did_list = [
DatasetID(name="test", wavelength=(0, 0.5, 1), resolution=1000),
DatasetID(name="testh", wavelength=(0, 0.5, 1), resolution=500),
DatasetID(name="test2", wavelength=(1, 1.5, 2), resolution=1000)
]
val_list = ["1", "1h", "2"]
d = DatasetDict(dict(zip(did_list, val_list)))
self.assertIn(did_list[0],
d.get_keys_by_datasetid(DatasetID(wavelength=0.5)))
self.assertIn(did_list[1],
d.get_keys_by_datasetid(DatasetID(wavelength=0.5)))
self.assertIn(did_list[2],
d.get_keys_by_datasetid(DatasetID(wavelength=1.5)))
self.assertIn(did_list[0],
d.get_keys_by_datasetid(DatasetID(resolution=1000)))
self.assertIn(did_list[2],
d.get_keys_by_datasetid(DatasetID(resolution=1000)))
def get_dataset_ids(self):
"""Get the dataset ids from the config."""
ids = []
for dskey, dataset in self.datasets.items():
# Build each permutation/product of the dataset
id_kwargs = []
for key in DatasetID._fields:
val = dataset.get(key)
if key in ["wavelength", "modifiers"] and isinstance(
val, list):
# special case: wavelength can be [min, nominal, max]
# but is still considered 1 option
# it also needs to be a tuple so it can be used in
# a dictionary key (DatasetID)
id_kwargs.append((tuple(val), ))
elif key == "modifiers" and val is None:
# empty modifiers means no modifiers applied
id_kwargs.append((tuple(), ))
elif isinstance(val, (list, tuple, set)):
# this key has multiple choices
# (ex. 250 meter, 500 meter, 1000 meter resolutions)
id_kwargs.append(val)
elif isinstance(val, dict):
id_kwargs.append(val.keys())
else:
# this key only has one choice so make it a one
# item iterable
id_kwargs.append((val, ))
for id_params in itertools.product(*id_kwargs):
dsid = DatasetID(*id_params)
ids.append(dsid)
# create dataset infos specifically for this permutation
ds_info = dataset.copy()
for key in dsid._fields:
if isinstance(ds_info.get(key), dict):
ds_info.update(ds_info[key][getattr(dsid, key)])
# this is important for wavelength which was converted
# to a tuple
ds_info[key] = getattr(dsid, key)
self.ids[dsid] = ds_info
return ids
def test_get_item(self):
from satpy.readers import DatasetDict, DatasetID
regular_dict = {
DatasetID(name="test", wavelength=(0, 0.5, 1), resolution=1000):
"1",
DatasetID(name="testh", wavelength=(0, 0.5, 1), resolution=500):
"1h",
DatasetID(name="test2", wavelength=(1, 1.5, 2), resolution=1000):
"2",
}
d = DatasetDict(regular_dict)
self.assertEqual(d["test"], "1")
self.assertEqual(d[1.5], "2")
self.assertEqual(d[DatasetID(wavelength=1.5)], "2")
self.assertEqual(d[DatasetID(wavelength=0.5, resolution=1000)], "1")
self.assertEqual(d[DatasetID(wavelength=0.5, resolution=500)], "1h")
def compute(self, *requirements):
"""Compute all the composites contained in `requirements`.
"""
if not requirements:
requirements = self.wishlist.copy()
keepables = set()
for requirement in requirements:
if isinstance(
requirement,
DatasetID) and requirement.name not in self.compositors:
continue
elif not isinstance(
requirement,
DatasetID) and requirement not in self.compositors:
continue
if requirement in self.datasets:
continue
if isinstance(requirement, DatasetID):
requirement_name = requirement.name
else:
requirement_name = requirement
compositor = self.compositors[requirement_name]
# Compute any composites that this one depends on
keepables |= self.compute(*compositor.info["prerequisites"])
# Compute any composites that this composite might optionally
# depend on
if compositor.info["optional_prerequisites"]:
keepables |= self.compute(
*compositor.info["optional_prerequisites"])
# Resolve the simple name of a prereq to the fully qualified
prereq_datasets = [
self[prereq] for prereq in compositor.info["prerequisites"]
]
optional_datasets = [
self[prereq]
for prereq in compositor.info["optional_prerequisites"]
if prereq in self
]
compositor.info["prerequisites"] = [
ds.info["id"] for ds in prereq_datasets
]
compositor.info["optional_prerequisites"] = [
ds.info["id"] for ds in optional_datasets
]
try:
comp_projectable = compositor(
prereq_datasets,
optional_datasets=optional_datasets,
**self.info)
# validate the composite projectable
assert ("name" in comp_projectable.info)
comp_projectable.info.setdefault("resolution", None)
comp_projectable.info.setdefault("wavelength_range", None)
comp_projectable.info.setdefault("polarization", None)
comp_projectable.info.setdefault("calibration", None)
comp_projectable.info.setdefault("modifiers", None)
# FIXME: Should this be a requirement of anything creating a
# Dataset? Special handling by .info?
band_id = DatasetID(
name=comp_projectable.info["name"],
resolution=comp_projectable.info["resolution"],
wavelength=comp_projectable.info["wavelength_range"],
polarization=comp_projectable.info["polarization"],
calibration=comp_projectable.info["calibration"],
modifiers=comp_projectable.info["modifiers"])
comp_projectable.info["id"] = band_id
self.datasets[band_id] = comp_projectable
# update the wishlist with this new dataset id
if requirement_name in self.wishlist:
self.wishlist.remove(requirement_name)
self.wishlist.add(band_id)
except IncompatibleAreas:
LOG.debug(
"Composite '%s' could not be created because of"
" incompatible areas", requirement_name)
# FIXME: If a composite depends on this composite we need to
# notify the previous call
preservable_datasets = set(
compositor.info["prerequisites"] +
compositor.info["optional_prerequisites"])
for ds_id, projectable in self.datasets.items():
# FIXME: Can compositors use wavelengths or only names?
if ds_id in preservable_datasets:
keepables.add(ds_id)
return keepables
def get_lonlats(self, navid, nav_info, lon_out, lat_out):
"""Load an area.
"""
lon_out[:] = self.get_dataset(DatasetID('longitude'))
lat_out[:] = self.get_dataset(DatasetID('latitude'))
def _process_composite_config(self, composite_name, conf,
composite_type, sensor_id, composite_config, **kwargs):
compositors = self.compositors[sensor_id]
modifiers = self.modifiers[sensor_id]
options = conf[composite_type][composite_name]
try:
loader = options.pop('compositor')
except KeyError:
if composite_name in compositors or composite_name in modifiers:
return conf
raise ValueError("'compositor' missing or empty in %s" %
composite_config)
options['name'] = composite_name
# fix prerequisites in case of modifiers
for prereq_type in ['prerequisites', 'optional_prerequisites']:
prereqs = []
for item in options.get(prereq_type, []):
if isinstance(item, dict):
mods = item.get('modifiers', tuple())
key = DatasetID(item.get('name'),
item.get('wavelength'),
item.get('resolution'),
item.get('polarization'),
item.get('calibration'),
mods)
comp_id = DatasetID(item.get('name'),
item.get('wavelength'),
item.get('resolution'),
item.get('polarization'),
item.get('calibration'),
tuple())
comp_name = item.get('name')
mods = key.modifiers
for modifier in mods:
prev_comp_name = comp_name
prev_comp_id = comp_id
new_mods = tuple(
list(comp_id.modifiers or []) + [modifier])
comp_id = DatasetID(key.name,
key.wavelength,
key.resolution,
key.polarization,
key.calibration,
new_mods)
comp_name = key.name
try:
mloader, moptions = modifiers[modifier]
except KeyError:
self._process_composite_config(modifier, conf,
composite_type, sensor_id, composite_config, **kwargs)
mloader, moptions = modifiers[modifier]
moptions = moptions.copy()
moptions.update(**kwargs)
moptions['name'] = modifier
moptions['id'] = comp_id
moptions['prerequisites'] = (
[prev_comp_id] + moptions['prerequisites'])
moptions['sensor'] = sensor_id
compositors[comp_id] = mloader(**moptions)
prereqs.append(comp_id)
else:
prereqs.append(item)
options[prereq_type] = prereqs
if composite_type == 'composites':
options.update(**kwargs)
options['id'] = options['name']
comp = loader(**options)
compositors[composite_name] = comp
elif composite_type == 'modifiers':
modifiers[composite_name] = loader, options