def test_init_dict(self):
"""Test DatasetDict init with a regular dict argument."""
from satpy import DatasetDict
regular_dict = {
make_dataid(name="test", wavelength=(0, 0.5, 1)): "1",
}
d = DatasetDict(regular_dict)
self.assertEqual(d, regular_dict)
def test_composites(sensor_name):
from satpy import DatasetID, DatasetDict
# Composite ID -> (prereqs, optional_prereqs)
comps = {
DatasetID(name='comp1'): (['ds1'], []),
DatasetID(name='comp2'): (['ds1', 'ds2'], []),
DatasetID(name='comp3'): (['ds1', 'ds2', 'ds3'], []),
DatasetID(name='comp4'): (['comp2', 'ds3'], []),
DatasetID(name='comp5'): (['ds1', 'ds2'], ['ds3']),
DatasetID(name='comp6'): (['ds1', 'ds2'], ['comp2']),
DatasetID(name='comp7'): (['ds1', 'comp2'], ['ds2']),
DatasetID(name='comp8'): (['ds_NOPE', 'comp2'], []),
DatasetID(name='comp9'): (['ds1', 'comp2'], ['ds_NOPE']),
DatasetID(name='comp10'): ([DatasetID('ds1', modifiers=('mod1',)), 'comp2'], []),
DatasetID(name='comp11'): ([0.22, 0.48, 0.85], []),
DatasetID(name='comp12'): ([DatasetID(wavelength=0.22, modifiers=('mod1',)),
DatasetID(wavelength=0.48, modifiers=('mod1',)),
DatasetID(wavelength=0.85, modifiers=('mod1',))], []),
DatasetID(name='comp13'): ([DatasetID(name='ds5', modifiers=('res_change',))], []),
DatasetID(name='comp14'): (['ds1'], []),
DatasetID(name='comp15'): (['ds1', 'ds9_fail_load'], []),
DatasetID(name='comp16'): (['ds1'], ['ds9_fail_load']),
DatasetID(name='comp17'): (['ds1', 'comp15'], []),
DatasetID(name='comp18'): (['ds3',
DatasetID(name='ds4', modifiers=('mod1', 'mod3',)),
DatasetID(name='ds5', modifiers=('mod1', 'incomp_areas'))], []),
DatasetID(name='comp18_2'): (['ds3',
DatasetID(name='ds4', modifiers=('mod1', 'mod3',)),
DatasetID(name='ds5', modifiers=('mod1', 'incomp_areas_opt'))], []),
DatasetID(name='comp19'): ([DatasetID('ds5', modifiers=('res_change',)), 'comp13', 'ds2'], []),
DatasetID(name='comp20'): ([DatasetID(name='ds5', modifiers=('mod_opt_prereq',))], []),
DatasetID(name='comp21'): ([DatasetID(name='ds5', modifiers=('mod_bad_opt',))], []),
DatasetID(name='comp22'): ([DatasetID(name='ds5', modifiers=('mod_opt_only',))], []),
DatasetID(name='comp23'): ([0.8], []),
}
# Modifier name -> (prereqs (not including to-be-modified), opt_prereqs)
mods = {
'mod1': (['ds2'], []),
'mod2': (['comp3'], []),
'mod3': (['ds2'], []),
'res_change': ([], []),
'incomp_areas': (['ds1'], []),
'incomp_areas_opt': ([DatasetID(name='ds1', modifiers=('incomp_areas',))], ['ds2']),
'mod_opt_prereq': (['ds1'], ['ds2']),
'mod_bad_opt': (['ds1'], ['ds9_fail_load']),
'mod_opt_only': ([], ['ds2']),
'mod_wl': ([DatasetID(wavelength=0.2, modifiers=('mod1',))], []),
}
comps = {sensor_name: DatasetDict((k, _create_fake_compositor(k, *v)) for k, v in comps.items())}
mods = {sensor_name: dict((k, _create_fake_modifiers(k, *v)) for k, v in mods.items())}
return comps, mods
def _reader_load(self, dataset_keys):
from satpy import DatasetDict, Dataset
import numpy as np
dataset_ids = self.datasets
loaded_datasets = DatasetDict()
for k in dataset_keys:
if k == 'ds9_fail_load':
continue
for ds in dataset_ids:
if ds == k:
loaded_datasets[ds] = Dataset(data=np.arange(5),
**ds.to_dict())
return loaded_datasets
def convert_satpy_to_p2g_name(
self,
satpy_products: Iterable[Union[DataID]],
possible_p2g_names: Optional[list[str]] = None,
):
"""Get the P2G name for a series of Satpy names or DataIDs.
If a Satpy DataID does not have a Polar2Grid compatible name then
``None`` is yielded. A name is not compatible if requesting it would
produce a different product than the original DataID.
"""
from satpy import DatasetDict
if possible_p2g_names is None:
possible_p2g_names = self._user_products
satpy_id_dict = DatasetDict({x: x for x in satpy_products})
satpy_id_to_p2g_name = {}
for p2g_name in possible_p2g_names:
satpy_data_query = self._all_aliases.get(p2g_name, p2g_name)
try:
matching_satpy_id = satpy_id_dict[satpy_data_query]
except KeyError:
continue
if matching_satpy_id in satpy_id_to_p2g_name:
logger.warning(
"Multiple product names map to the same identifier in Satpy"
)
print(matching_satpy_id,
satpy_id_to_p2g_name[matching_satpy_id], p2g_name)
satpy_id_to_p2g_name[matching_satpy_id] = p2g_name
for satpy_product in satpy_products:
satpy_id_name = satpy_product["name"]
satpy_id_as_p2g_name = satpy_id_to_p2g_name.get(satpy_product)
satpy_name_is_p2g_name = satpy_id_name in possible_p2g_names
satpy_name_does_not_round_trip = satpy_id_dict[
satpy_product] != satpy_product
if satpy_id_as_p2g_name is None:
# We can't yield this name if it is also a P2G name or if
# asking Satpy for the name doesn't return the same DataID
# product. Otherwise users would ask for X and not get X.
if satpy_name_is_p2g_name or satpy_name_does_not_round_trip:
yield None
else:
yield satpy_id_name
continue
yield satpy_id_to_p2g_name.get(satpy_product, satpy_id_name)
def load(self, dataset_keys):
from satpy import DatasetDict
from xarray import DataArray
import numpy as np
dataset_ids = self.all_ids.keys()
loaded_datasets = DatasetDict()
for k in dataset_keys:
if k == 'ds9_fail_load':
continue
for ds in dataset_ids:
if ds == k:
loaded_datasets[ds] = DataArray(data=np.arange(25).reshape(5, 5),
attrs=ds.to_dict(),
dims=['y', 'x'])
return loaded_datasets
def __init__(self, readers, compositors, modifiers):
"""Collect Dataset generating information.
Collect the objects that generate and have information about Datasets
including objects that may depend on certain Datasets being generated.
This includes readers, compositors, and modifiers.
Args:
readers (dict): Reader name -> Reader Object
compositors (dict): Sensor name -> Composite ID -> Composite Object
modifiers (dict): Sensor name -> Modifier name -> (Modifier Class, modifier options)
"""
self.readers = readers
self.compositors = compositors
self.modifiers = modifiers
# we act as the root node of the tree
super(DependencyTree, self).__init__(None)
# keep a flat dictionary of nodes contained in the tree for better
# __contains__
self._all_nodes = DatasetDict()
def _load_config(self, composite_configs):
if not isinstance(composite_configs, (list, tuple)):
composite_configs = [composite_configs]
conf = {}
for composite_config in composite_configs:
with open(composite_config, 'r', encoding='utf-8') as conf_file:
conf = recursive_dict_update(
conf, yaml.load(conf_file, Loader=UnsafeLoader))
try:
sensor_name = conf['sensor_name']
except KeyError:
logger.debug('No "sensor_name" tag found in %s, skipping.',
composite_configs)
return
sensor_id = sensor_name.split('/')[-1]
sensor_deps = sensor_name.split('/')[:-1]
compositors = self.compositors.setdefault(sensor_id, DatasetDict())
modifiers = self.modifiers.setdefault(sensor_id, {})
for sensor_dep in reversed(sensor_deps):
if sensor_dep not in self.compositors or sensor_dep not in self.modifiers:
self.load_sensor_composites(sensor_dep)
if sensor_deps:
compositors.update(self.compositors[sensor_deps[-1]])
modifiers.update(self.modifiers[sensor_deps[-1]])
id_keys = self._get_sensor_id_keys(conf, sensor_id, sensor_deps)
mod_config_helper = _ModifierConfigHelper(modifiers, id_keys)
configured_modifiers = conf.get('modifiers', {})
mod_config_helper.parse_config(configured_modifiers, composite_configs)
comp_config_helper = _CompositeConfigHelper(compositors, id_keys)
configured_composites = conf.get('composites', {})
comp_config_helper.parse_config(configured_composites,
composite_configs)
def test_composites(sensor_name):
"""Create some test composites."""
from satpy import DatasetDict
# Composite ID -> (prereqs, optional_prereqs)
comps = {
make_cid(name='comp1'): (['ds1'], []),
make_cid(name='comp2'): (['ds1', 'ds2'], []),
make_cid(name='comp3'): (['ds1', 'ds2', 'ds3'], []),
make_cid(name='comp4'): (['comp2', 'ds3'], []),
make_cid(name='comp5'): (['ds1', 'ds2'], ['ds3']),
make_cid(name='comp6'): (['ds1', 'ds2'], ['comp2']),
make_cid(name='comp7'): (['ds1', 'comp2'], ['ds2']),
make_cid(name='comp8'): (['ds_NOPE', 'comp2'], []),
make_cid(name='comp9'): (['ds1', 'comp2'], ['ds_NOPE']),
make_cid(name='comp10'): ([make_dsq(name='ds1', modifiers=('mod1',)), 'comp2'], []),
make_cid(name='comp11'): ([0.22, 0.48, 0.85], []),
make_cid(name='comp12'): ([make_dsq(wavelength=0.22, modifiers=('mod1',)),
make_dsq(wavelength=0.48, modifiers=('mod1',)),
make_dsq(wavelength=0.85, modifiers=('mod1',))], []),
make_cid(name='comp13'): ([make_dsq(name='ds5', modifiers=('res_change',))], []),
make_cid(name='comp14'): (['ds1'], []),
make_cid(name='comp15'): (['ds1', 'ds9_fail_load'], []),
make_cid(name='comp16'): (['ds1'], ['ds9_fail_load']),
make_cid(name='comp17'): (['ds1', 'comp15'], []),
make_cid(name='comp18'): (['ds3',
make_dsq(name='ds4', modifiers=('mod1', 'mod3',)),
make_dsq(name='ds5', modifiers=('mod1', 'incomp_areas'))], []),
make_cid(name='comp18_2'): (['ds3',
make_dsq(name='ds4', modifiers=('mod1', 'mod3',)),
make_dsq(name='ds5', modifiers=('mod1', 'incomp_areas_opt'))], []),
make_cid(name='comp19'): ([make_dsq(name='ds5', modifiers=('res_change',)), 'comp13', 'ds2'], []),
make_cid(name='comp20'): ([make_dsq(name='ds5', modifiers=('mod_opt_prereq',))], []),
make_cid(name='comp21'): ([make_dsq(name='ds5', modifiers=('mod_bad_opt',))], []),
make_cid(name='comp22'): ([make_dsq(name='ds5', modifiers=('mod_opt_only',))], []),
make_cid(name='comp23'): ([0.8], []),
make_cid(name='static_image'): ([], []),
make_cid(name='comp24', resolution=500): ([make_dsq(name='ds11', resolution=500),
make_dsq(name='ds12', resolution=500)], []),
make_cid(name='comp24', resolution=1000): ([make_dsq(name='ds11', resolution=1000),
make_dsq(name='ds12', resolution=1000)], []),
make_cid(name='comp25', resolution=500): ([make_dsq(name='comp24', resolution=500),
make_dsq(name='ds5', resolution=500)], []),
make_cid(name='comp25', resolution=1000): ([make_dsq(name='comp24', resolution=1000),
make_dsq(name='ds5', resolution=1000)], []),
make_cid(name='ahi_green'): ([make_dsq(wavelength=30.5, modifiers=('sunz_corr', 'rayleigh_corr')),
make_dsq(wavelength=30.85, modifiers=('sunz_corr',))], [])
}
# Modifier name -> (prereqs (not including to-be-modified), opt_prereqs)
mods = {
'mod1': (['ds2'], []),
'mod2': (['comp3'], []),
'mod3': (['ds2'], []),
'res_change': ([], []),
'incomp_areas': (['ds1'], []),
'incomp_areas_opt': ([make_dataid(name='ds1', modifiers=('incomp_areas',))], ['ds2']),
'mod_opt_prereq': (['ds1'], ['ds2']),
'mod_bad_opt': (['ds1'], ['ds9_fail_load']),
'mod_opt_only': ([], ['ds2']),
'mod_wl': ([make_dsq(wavelength=0.2, modifiers=('mod1',))], []),
'sunz_corr': ([], ['sunz_angles_NOPE']),
'rayleigh_corr': ([make_dsq(wavelength=30.64, modifiers=('sunz_corr',))], [])
}
comps = {sensor_name: DatasetDict((k, _create_fake_compositor(k, *v)) for k, v in comps.items())}
mods = {sensor_name: dict((k, _create_fake_modifiers(k, *v)) for k, v in mods.items())}
return comps, mods
def test_init_noargs(self):
"""Test DatasetDict init with no arguments."""
from satpy import DatasetDict
d = DatasetDict()
self.assertIsInstance(d, dict)
