def test_dataid():
"""Test the DataID object."""
from satpy.dataset import DataID, WavelengthRange, ModifierTuple, ValueList
# Check that enum is translated to type.
did = make_dataid()
assert issubclass(did._id_keys['calibration']['type'], ValueList)
assert 'enum' not in did._id_keys['calibration']
# Check that None is never a valid value
did = make_dataid(name='cheese_shops', resolution=None)
assert 'resolution' not in did
assert 'None' not in did.__repr__()
with pytest.raises(ValueError):
make_dataid(name=None, resolution=1000)
# Check that defaults are applied correctly
assert did['modifiers'] == ModifierTuple()
# Check that from_dict creates a distinct instance...
did2 = did.from_dict(dict(name='cheese_shops', resolution=None))
assert did is not did2
# ...But is equal
assert did2 == did
# Check that the instance is immutable
with pytest.raises(TypeError):
did['resolution'] = 1000
# Check that a missing required field crashes
with pytest.raises(ValueError):
make_dataid(resolution=1000)
# Check to_dict
assert did.to_dict() == dict(name='cheese_shops', modifiers=tuple())
# Check repr
did = make_dataid(name='VIS008', resolution=111)
assert repr(did) == "DataID(name='VIS008', resolution=111, modifiers=())"
# Check inequality
default_id_keys_config = {'name': None,
'wavelength': {
'type': WavelengthRange,
},
'resolution': None,
'calibration': {
'enum': [
'reflectance',
'brightness_temperature',
'radiance',
'counts'
]
},
'modifiers': {
'default': ModifierTuple(),
'type': ModifierTuple,
},
}
assert DataID(default_id_keys_config, wavelength=10) != DataID(default_id_keys_config, name="VIS006")
def test_compare_no_wl(self):
"""Compare fully qualified wavelength ID to no wavelength ID."""
from satpy.dataset import DataID, default_id_keys_config as dikc
d1 = DataID(dikc, name="a", wavelength=(0.1, 0.2, 0.3))
d2 = DataID(dikc, name="a", wavelength=None)
# this happens when sorting IDs during dependency checks
self.assertFalse(d1 < d2)
self.assertTrue(d2 < d1)
def update_ds_ids_from_file_handlers(self):
"""Add or modify available dataset information.
Each file handler is consulted on whether or not it can load the
dataset with the provided information dictionary.
See
:meth:`satpy.readers.file_handlers.BaseFileHandler.available_datasets`
for more information.
"""
avail_datasets = self._file_handlers_available_datasets()
new_ids = {}
for is_avail, ds_info in avail_datasets:
# especially from the yaml config
coordinates = ds_info.get('coordinates')
if isinstance(coordinates, list):
# xarray doesn't like concatenating attributes that are
# lists: https://github.com/pydata/xarray/issues/2060
ds_info['coordinates'] = tuple(ds_info['coordinates'])
ds_info.setdefault('modifiers', tuple()) # default to no mods
# Create DataID for this dataset
ds_id = DataID(self._id_keys, **ds_info)
# all datasets
new_ids[ds_id] = ds_info
# available datasets
# False == we have the file type but it doesn't have this dataset
# None == we don't have the file type object to ask
if is_avail:
self.available_ids[ds_id] = ds_info
self.all_ids = new_ids
def id(self):
"""Return the DataID of the object."""
try:
return self.attrs['_satpy_id']
except KeyError:
id_keys = self.attrs.get('_satpy_id_keys', minimal_default_keys_config)
return DataID(id_keys, **self.attrs)
def test_dataid_copy():
"""Test copying a DataID."""
from satpy.dataset import DataID, default_id_keys_config as dikc
from copy import deepcopy
did = DataID(dikc, name="a", resolution=1000)
did2 = deepcopy(did)
assert did2 == did
assert did2.id_keys == did.id_keys
def _se(datasets, optional_datasets=None, ds_id=ds_id, **kwargs):
if ds_id['name'] == 'comp14':
# used as a test when composites update the dataset id with
# information from prereqs
ds_id = DataID(ds_id.id_keys, resolution=555, **ds_id)
if len(datasets) != len(prereqs):
raise ValueError("Not enough prerequisite datasets passed")
return DataArray(data=np.arange(75).reshape(5, 5, 3),
attrs=ds_id.to_dict(),
dims=['y', 'x', 'bands'],
coords={'bands': ['R', 'G', 'B']})
def test_dataid_equal_if_enums_different():
"""Check that dataids with different enums but same items are equal."""
from satpy.dataset import DataID, WavelengthRange, ModifierTuple
id_keys_config1 = {'name': None,
'wavelength': {
'type': WavelengthRange,
},
'resolution': None,
'calibration': {
'enum': [
'c1',
'c2',
'c3',
]
},
'modifiers': {
'default': ModifierTuple(),
'type': ModifierTuple,
},
}
id_keys_config2 = {'name': None,
'wavelength': {
'type': WavelengthRange,
},
'resolution': None,
'calibration': {
'enum': [
'c1',
'c1.5',
'c2',
'c2.5',
'c3'
]
},
'modifiers': {
'default': ModifierTuple(),
'type': ModifierTuple,
},
}
assert DataID(id_keys_config1, name='ni', calibration='c2') == DataID(id_keys_config2, name="ni", calibration='c2')
def __setitem__(self, key, value):
"""Support assigning 'Dataset' objects or dictionaries of metadata."""
value_dict = value
if hasattr(value, 'attrs'):
# xarray.DataArray objects
value_dict = value.attrs
# use value information to make a more complete DataID
if not isinstance(key, DataID):
if not isinstance(value_dict, dict):
raise ValueError(
"Key must be a DataID when value is not an xarray DataArray or dict"
)
old_key = key
try:
key = self.get_key(key)
except KeyError:
if isinstance(old_key, str):
new_name = old_key
else:
new_name = value_dict.get("name")
# this is a new key and it's not a full DataID tuple
if new_name is None and value_dict.get('wavelength') is None:
raise ValueError("One of 'name' or 'wavelength' attrs "
"values should be set.")
try:
id_keys = value_dict['_satpy_id'].id_keys
except KeyError:
try:
id_keys = value_dict['_satpy_id_keys']
except KeyError:
id_keys = minimal_default_keys_config
value_dict['name'] = new_name
key = DataID(id_keys, **value_dict)
if hasattr(value, 'attrs') and 'name' not in value.attrs:
value.attrs['name'] = new_name
# update the 'value' with the information contained in the key
try:
new_info = key.to_dict()
except AttributeError:
new_info = key
if isinstance(value_dict, dict):
value_dict.update(new_info)
if hasattr(value, 'attrs'):
if isinstance(key, DataID):
value.attrs['_satpy_id'] = key
return super(DatasetDict, self).__setitem__(key, value)
def test_basic_init(self):
"""Test basic ways of creating a DataID."""
from satpy.dataset import DataID, default_id_keys_config as dikc, minimal_default_keys_config as mdkc
did = DataID(dikc, name="a")
assert did['name'] == 'a'
assert did['modifiers'] == tuple()
DataID(dikc, name="a", wavelength=0.86)
DataID(dikc, name="a", resolution=1000)
DataID(dikc, name="a", calibration='radiance')
DataID(dikc, name="a", wavelength=0.86, resolution=250,
calibration='radiance')
DataID(dikc, name="a", wavelength=0.86, resolution=250,
calibration='radiance', modifiers=('sunz_corrected',))
with pytest.raises(ValueError):
DataID(dikc, wavelength=0.86)
did = DataID(mdkc, name='comp24', resolution=500)
assert did['resolution'] == 500
def load_ds_ids_from_config(self):
"""Get the dataset ids from the config."""
ids = []
for dataset in self.datasets.values():
# xarray doesn't like concatenating attributes that are lists
# https://github.com/pydata/xarray/issues/2060
if 'coordinates' in dataset and \
isinstance(dataset['coordinates'], list):
dataset['coordinates'] = tuple(dataset['coordinates'])
id_keys = get_keys_from_config(self._id_keys, dataset)
# Build each permutation/product of the dataset
id_kwargs = []
for key, idval in id_keys.items():
val = dataset.get(key, idval.get('default') if idval is not None else None)
val_type = None
if idval is not None:
val_type = idval.get('type')
if val_type is not None and issubclass(val_type, tuple):
# special case: wavelength can be [min, nominal, max]
# but is still considered 1 option
id_kwargs.append((val, ))
elif isinstance(val, (list, tuple, set)):
# this key has multiple choices
# (ex. 250 meter, 500 meter, 1000 meter resolutions)
id_kwargs.append(val)
elif isinstance(val, dict):
id_kwargs.append(val.keys())
else:
# this key only has one choice so make it a one
# item iterable
id_kwargs.append((val,))
for id_params in itertools.product(*id_kwargs):
dsid = DataID(id_keys, **dict(zip(id_keys, id_params)))
ids.append(dsid)
# create dataset infos specifically for this permutation
ds_info = dataset.copy()
for key in dsid.keys():
if isinstance(ds_info.get(key), dict):
ds_info.update(ds_info[key][dsid.get(key)])
# this is important for wavelength which was converted
# to a tuple
ds_info[key] = dsid.get(key)
self.all_ids[dsid] = ds_info
return ids
def make_dataid(**items):
"""Make a data id."""
return DataID(local_id_keys_config, **items)
def make_cid(**items):
"""Make a DataID with a minimal set of keys to id composites."""
return DataID(minimal_default_keys_config, **items)
def make_dataid(**items):
"""Make a DataID with default keys."""
return DataID(default_id_keys_config, **items)
def test_bad_calibration(self):
"""Test that asking for a bad calibration fails."""
from satpy.dataset import DataID, default_id_keys_config as dikc
with pytest.raises(ValueError):
DataID(dikc, name='C05', calibration='_bad_')
def test_id_query_interactions():
"""Test interactions between DataIDs and DataQuery's."""
from satpy.dataset import DataQuery, DataID, WavelengthRange, ModifierTuple, minimal_default_keys_config
default_id_keys_config = {'name': {
'required': True,
},
'wavelength': {
'type': WavelengthRange,
},
'resolution': None,
'calibration': {
'enum': [
'reflectance',
'brightness_temperature',
'radiance',
'counts'
]
},
'modifiers': {
'default': ModifierTuple(),
'type': ModifierTuple,
},
}
# Check hash equality
dq = DataQuery(modifiers=tuple(), name='cheese_shops')
did = DataID(default_id_keys_config, name='cheese_shops')
assert hash(dq) == hash(did)
# Check did filtering
did2 = DataID(default_id_keys_config, name='ni')
res = dq.filter_dataids([did2, did])
assert len(res) == 1
assert res[0] == did
dataid_container = [DataID(default_id_keys_config,
name='ds1',
resolution=250,
calibration='reflectance',
modifiers=tuple())]
dq = DataQuery(wavelength=0.22, modifiers=tuple())
assert len(dq.filter_dataids(dataid_container)) == 0
dataid_container = [DataID(minimal_default_keys_config,
name='natural_color')]
dq = DataQuery(name='natural_color', resolution=250)
assert len(dq.filter_dataids(dataid_container)) == 1
dq = make_dsq(wavelength=0.22, modifiers=('mod1',))
did = make_cid(name='static_image')
assert len(dq.filter_dataids([did])) == 0
# Check did sorting
dq = DataQuery(name='cheese_shops', wavelength=2, modifiers='*')
did = DataID(default_id_keys_config, name='cheese_shops', wavelength=(1, 2, 3))
did2 = DataID(default_id_keys_config, name='cheese_shops', wavelength=(1.1, 2.1, 3.1))
dsids, distances = dq.sort_dataids([did2, did])
assert list(dsids) == [did, did2]
assert np.allclose(distances, [0, 0.1])
dq = DataQuery(name='cheese_shops')
did = DataID(default_id_keys_config, name='cheese_shops', resolution=200)
did2 = DataID(default_id_keys_config, name='cheese_shops', resolution=400)
dsids, distances = dq.sort_dataids([did2, did])
assert list(dsids) == [did, did2]
assert distances[0] < distances[1]
did = DataID(default_id_keys_config, name='cheese_shops', calibration='counts')
did2 = DataID(default_id_keys_config, name='cheese_shops', calibration='reflectance')
dsids, distances = dq.sort_dataids([did2, did])
assert list(dsids) == [did2, did]
assert distances[0] < distances[1]
did = DataID(default_id_keys_config, name='cheese_shops', modifiers=tuple())
did2 = DataID(default_id_keys_config, name='cheese_shops', modifiers=tuple(['out_of_stock']))
dsids, distances = dq.sort_dataids([did2, did])
assert list(dsids) == [did, did2]
assert distances[0] < distances[1]
# Check (in)equality
assert DataQuery(wavelength=10) != DataID(default_id_keys_config, name="VIS006")