def get_dataset_key(self, key, available_only=False, **kwargs):
"""Get the fully qualified `DataID` matching `key`.
This will first search through available DataIDs, datasets that
should be possible to load, and fallback to "known" datasets, those
that are configured but aren't loadable from the provided files.
Providing ``available_only=True`` will stop this fallback behavior
and raise a ``KeyError`` exception if no available dataset is found.
Args:
key (str, float, DataID, DataQuery): Key to search for in this reader.
available_only (bool): Search only loadable datasets for the
provided key. Loadable datasets are always searched first,
but if ``available_only=False`` (default) then all known
datasets will be searched.
kwargs: See :func:`satpy.readers.get_key` for more information about
kwargs.
Returns:
Best matching DataID to the provided ``key``.
Raises:
KeyError: if no key match is found.
"""
try:
return get_key(key, self.available_ids.keys(), **kwargs)
except KeyError:
if available_only:
raise
return get_key(key, self.all_ids.keys(), **kwargs)
def test_fy3d_1km_resolutions(self):
"""Test loading data when only 1km resolutions are available."""
from satpy.tests.utils import make_dataid
from satpy.readers import load_reader, get_key
filenames = [
'tf2019071182739.FY3D-X_MERSI_1000M_L1B.HDF',
'tf2019071182739.FY3D-X_MERSI_GEO1K_L1B.HDF',
]
reader = load_reader(self.reader_configs)
files = reader.select_files_from_pathnames(filenames)
self.assertEqual(2, len(files))
reader.create_filehandlers(files)
# Make sure we have some files
self.assertTrue(reader.file_handlers)
# Verify that we have multiple resolutions for:
# - Bands 1-4 (visible)
# - Bands 24-25 (IR)
available_datasets = reader.available_dataset_ids
for band_name in ('1', '2', '3', '4', '24', '25'):
if band_name in ('24', '25'):
# don't know how to get radiance for IR bands
num_results = 2
else:
num_results = 3
ds_id = make_dataid(name=band_name, resolution=250)
with pytest.raises(KeyError):
res = get_key(ds_id, available_datasets,
num_results=num_results, best=False)
ds_id = make_dataid(name=band_name, resolution=1000)
res = get_key(ds_id, available_datasets,
num_results=num_results, best=False)
self.assertEqual(num_results, len(res))
res = reader.load(['1', '2', '3', '4', '5', '20', '24', '25'])
self.assertEqual(8, len(res))
self.assertEqual((2 * 10, 2048), res['1'].shape)
self.assertEqual('reflectance', res['1'].attrs['calibration'])
self.assertEqual('%', res['1'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['2'].shape)
self.assertEqual('reflectance', res['2'].attrs['calibration'])
self.assertEqual('%', res['2'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['3'].shape)
self.assertEqual('reflectance', res['3'].attrs['calibration'])
self.assertEqual('%', res['3'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['4'].shape)
self.assertEqual('reflectance', res['4'].attrs['calibration'])
self.assertEqual('%', res['4'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['5'].shape)
self.assertEqual('reflectance', res['5'].attrs['calibration'])
self.assertEqual('%', res['5'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['20'].shape)
self.assertEqual('brightness_temperature', res['20'].attrs['calibration'])
self.assertEqual('K', res['20'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['24'].shape)
self.assertEqual('brightness_temperature', res['24'].attrs['calibration'])
self.assertEqual('K', res['24'].attrs['units'])
self.assertEqual((2 * 10, 2048), res['25'].shape)
self.assertEqual('brightness_temperature', res['25'].attrs['calibration'])
self.assertEqual('K', res['25'].attrs['units'])
def test_fy4a_all_resolutions(self):
"""Test loading data when all resolutions are available."""
from satpy import DatasetID
from satpy.readers import load_reader, get_key
filenames = [
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_0500M_V0001.HDF',
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_1000M_V0001.HDF',
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_2000M_V0001.HDF',
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_4000M_V0001.HDF',
]
reader = load_reader(self.reader_configs)
files = reader.select_files_from_pathnames(filenames)
self.assertTrue(4, len(files))
reader.create_filehandlers(files)
# Make sure we have some files
self.assertTrue(reader.file_handlers)
available_datasets = reader.available_dataset_ids
# 500m
band_names = ['C' + '%02d' % ch for ch in np.linspace(2, 2, 1)]
for band_name in band_names:
ds_id = DatasetID(name=band_name, resolution=500)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(2, len(res))
# 1km
band_names = ['C' + '%02d' % ch for ch in np.linspace(1, 3, 3)]
for band_name in band_names:
ds_id = DatasetID(name=band_name, resolution=1000)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(2, len(res))
# 2km
band_names = ['C' + '%02d' % ch for ch in np.linspace(1, 7, 7)]
for band_name in band_names:
ds_id = DatasetID(name=band_name, resolution=2000)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
if band_name < 'C07':
self.assertEqual(2, len(res))
else:
self.assertEqual(3, len(res))
band_names = ['C' + '%02d' % ch for ch in np.linspace(1, 14, 14)]
res = reader.load(band_names)
self.assertEqual(14, len(res))
for band_name in band_names:
self.assertEqual((2, 5), res[band_name].shape)
if band_name < 'C07':
self.assertEqual('reflectance',
res[band_name].attrs['calibration'])
else:
self.assertEqual('brightness_temperature',
res[band_name].attrs['calibration'])
if band_name < 'C07':
self.assertEqual('%', res[band_name].attrs['units'])
else:
self.assertEqual('K', res[band_name].attrs['units'])
def test_fy4a_1km_resolutions(self):
"""Test loading data when only 1km resolutions are available."""
from satpy import DatasetID
from satpy.readers import load_reader, get_key
filenames = [
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_1000M_V0001.HDF',
]
reader = load_reader(self.reader_configs)
files = reader.select_files_from_pathnames(filenames)
self.assertTrue(1, len(files))
reader.create_filehandlers(files)
# Make sure we have some files
self.assertTrue(reader.file_handlers)
# Verify that the resolution is only 1km
available_datasets = reader.available_dataset_ids
band_names = ['C' + '%02d' % ch for ch in np.linspace(1, 3, 3)]
for band_name in band_names:
ds_id = DatasetID(name=band_name, resolution=500)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(0, len(res))
ds_id = DatasetID(name=band_name, resolution=1000)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(2, len(res))
ds_id = DatasetID(name=band_name, resolution=2000)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(0, len(res))
ds_id = DatasetID(name=band_name, resolution=4000)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(0, len(res))
res = reader.load(band_names)
self.assertEqual(3, len(res))
expected = {
1:
np.array([[2.01, 2.02, 2.03, 2.04, 2.05],
[2.06, 2.07, 2.08, 2.09, 2.1]]),
2:
np.array([[4.03, 4.06, 4.09, 4.12, 4.15],
[4.18, 4.21, 4.24, 4.27, 4.3]]),
3:
np.array([[6.05, 6.1, 6.15, 6.2, 6.25],
[6.3, 6.35, 6.4, 6.45, 6.5]])
}
for index, band_name in enumerate(band_names):
self.assertEqual(1, res[band_name].attrs['sensor'].islower())
self.assertEqual((2, 5), res[band_name].shape)
self.assertEqual('reflectance',
res[band_name].attrs['calibration'])
self.assertEqual('%', res[band_name].attrs['units'])
self.assertTrue(
np.allclose(res[band_name].values,
expected[index + 1],
equal_nan=True))
def test_fy4a_2km_resolutions(self):
"""Test loading data when only 2km resolutions are available."""
from satpy.tests.utils import make_dsq
from satpy.readers import load_reader, get_key
filenames = [
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_2000M_V0001.HDF',
]
reader = load_reader(self.reader_configs)
files = reader.select_files_from_pathnames(filenames)
self.assertEqual(1, len(files))
reader.create_filehandlers(files)
# Make sure we have some files
self.assertTrue(reader.file_handlers)
# Verify that the resolution is only 2km
available_datasets = reader.available_dataset_ids
band_names = ['C' + '%02d' % ch for ch in np.linspace(1, 7, 7)]
for band_name in band_names:
for resolution in [500, 1000, 4000]:
ds_q = make_dsq(name=band_name, resolution=resolution)
with pytest.raises(KeyError):
res = get_key(ds_q, available_datasets, num_results=0, best=False)
ds_q = make_dsq(name=band_name, resolution=2000)
res = get_key(ds_q, available_datasets, num_results=0, best=False)
if band_name < 'C07':
self.assertEqual(2, len(res))
else:
self.assertEqual(3, len(res))
res = reader.load(band_names)
self.assertEqual(7, len(res))
expected = {
1: np.array([[2.01, 2.02, 2.03, 2.04, 2.05], [2.06, 2.07, 2.08, 2.09, 2.1]]),
2: np.array([[4.03, 4.06, 4.09, 4.12, 4.15], [4.18, 4.21, 4.24, 4.27, 4.3]]),
3: np.array([[6.05, 6.1, 6.15, 6.2, 6.25], [6.3, 6.35, 6.4, 6.45, 6.5]]),
4: np.array([[8.07, 8.14, 8.21, 8.28, 8.35], [8.42, 8.49, 8.56, 8.63, 8.7]]),
5: np.array([[10.09, 10.18, 10.27, 10.36, 10.45], [10.54, 10.63, 10.72, 10.81, 10.9]]),
6: np.array([[12.11, 12.22, 12.33, 12.44, 12.55], [12.66, 12.77, 12.88, 12.99, 13.1]]),
7: np.array([[0.2, 0.3, 0.4, 0.5, 0.6], [0.7, 0.8, 0.9, 1., np.nan]])
}
for index, band_name in enumerate(band_names):
self.assertEqual((2, 5), res[band_name].shape)
if band_name < 'C07':
self.assertEqual('reflectance', res[band_name].attrs['calibration'])
else:
self.assertEqual('brightness_temperature', res[band_name].attrs['calibration'])
if band_name < 'C07':
self.assertEqual('%', res[band_name].attrs['units'])
else:
self.assertEqual('K', res[band_name].attrs['units'])
self.assertTrue(np.allclose(res[band_name].values, expected[index + 1], equal_nan=True))
def get_dataset_key(self, key, prefer_available=True, **kwargs):
"""Get the fully qualified `DatasetID` matching `key`.
See `satpy.readers.get_key` for more information about kwargs.
"""
if prefer_available:
try:
return get_key(key, self.available_ids.keys(), **kwargs)
except KeyError:
return get_key(key, self.all_ids.keys(), **kwargs)
return get_key(key, self.all_ids.keys(), **kwargs)
def get_dataset_key(self, key, **kwargs):
"""Get the fully qualified `DatasetID` matching `key`.
See `satpy.readers.get_key` for more information about kwargs.
"""
return get_key(key, self.ids.keys(), **kwargs)
def get_dataset_key(self, key, **kwargs):
"""Get the fully qualified `DatasetID` matching `key`.
See `satpy.readers.get_key` for more information about kwargs.
"""
return get_key(key, self.ids.keys(), **kwargs)
def test_fy4a_500m_resolutions(self):
"""Test loading data when only 500m resolutions are available."""
from satpy import DatasetID
from satpy.readers import load_reader, get_key
filenames = [
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_0500M_V0001.HDF',
]
reader = load_reader(self.reader_configs)
files = reader.select_files_from_pathnames(filenames)
self.assertTrue(1, len(files))
reader.create_filehandlers(files)
# Make sure we have some files
self.assertTrue(reader.file_handlers)
# Verify that the resolution is only 500m
available_datasets = reader.available_dataset_ids
band_names = ['C' + '%02d' % ch for ch in np.linspace(2, 2, 1)]
for band_name in band_names:
ds_id = DatasetID(name=band_name, resolution=500)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(2, len(res))
ds_id = DatasetID(name=band_name, resolution=1000)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(0, len(res))
ds_id = DatasetID(name=band_name, resolution=2000)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(0, len(res))
ds_id = DatasetID(name=band_name, resolution=4000)
res = get_key(ds_id, available_datasets, num_results=0, best=False)
self.assertEqual(0, len(res))
res = reader.load(band_names)
self.assertEqual(1, len(res))
expected = np.array([[2.01, 2.02, 2.03, 2.04, 2.05],
[2.06, 2.07, 2.08, 2.09, 2.1]])
for band_name in band_names:
self.assertEqual((2, 5), res[band_name].shape)
self.assertEqual('reflectance',
res[band_name].attrs['calibration'])
self.assertEqual('%', res[band_name].attrs['units'])
self.assertTrue(
np.allclose(res[band_name].values, expected, equal_nan=True))
def test_get_key(self):
"""Test 'get_key' special functions."""
from satpy import DatasetID
from satpy.readers import get_key
d = self.test_dict
res1 = get_key(DatasetID(name='test4'), d, calibration='radiance')
res2 = get_key(DatasetID(name='test4'),
d,
calibration='radiance',
num_results=0)
res3 = get_key(DatasetID(name='test4'),
d,
calibration='radiance',
num_results=3)
self.assertEqual(len(res2), 1)
self.assertEqual(len(res3), 1)
res2 = res2[0]
res3 = res3[0]
self.assertEqual(res1, res2)
self.assertEqual(res1, res3)
res1 = get_key('test4', d, polarization='V')
self.assertEqual(
res1,
DatasetID(name='test4', calibration='radiance', polarization='V'))
res1 = get_key(0.5, d, resolution=500)
self.assertEqual(
res1,
DatasetID(name='testh', wavelength=(0, 0.5, 1), resolution=500))
res1 = get_key('test6', d, level=100)
self.assertEqual(res1, DatasetID(name='test6', level=100))
res1 = get_key('test5', d)
res2 = get_key('test5', d, modifiers=('mod2', ))
res3 = get_key('test5', d, modifiers=(
'mod1',
'mod2',
))
self.assertEqual(res1, DatasetID(name='test5', modifiers=('mod2', )))
self.assertEqual(res1, res2)
self.assertNotEqual(res1, res3)
# more than 1 result when default is to ask for 1 result
self.assertRaises(KeyError, get_key, 'test4', d, best=False)
def test_get_key(self):
"""Test 'get_key' special functions."""
from satpy import DatasetID
from satpy.readers import get_key
d = self.test_dict
res1 = get_key(DatasetID(name='test4'), d, calibration='radiance')
res2 = get_key(DatasetID(name='test4'), d, calibration='radiance',
num_results=0)
res3 = get_key(DatasetID(name='test4'), d, calibration='radiance',
num_results=3)
self.assertEqual(len(res2), 1)
self.assertEqual(len(res3), 1)
res2 = res2[0]
res3 = res3[0]
self.assertEqual(res1, res2)
self.assertEqual(res1, res3)
res1 = get_key('test4', d, polarization='V')
self.assertEqual(res1, DatasetID(name='test4', calibration='radiance',
polarization='V'))
res1 = get_key(0.5, d, resolution=500)
self.assertEqual(res1, DatasetID(name='testh',
wavelength=(0, 0.5, 1),
resolution=500))
res1 = get_key('test6', d, level=100)
self.assertEqual(res1, DatasetID(name='test6',
level=100))
res1 = get_key('test5', d)
res2 = get_key('test5', d, modifiers=('mod2',))
res3 = get_key('test5', d, modifiers=('mod1', 'mod2',))
self.assertEqual(res1, DatasetID(name='test5',
modifiers=('mod2',)))
self.assertEqual(res1, res2)
self.assertNotEqual(res1, res3)
# more than 1 result when default is to ask for 1 result
self.assertRaises(KeyError, get_key, 'test4', d, best=False)
def _get_dataset_key(self, key, **kwargs):
from satpy.readers import get_key
return get_key(key, self.datasets, **kwargs)
def test_fy4a_all_resolutions(self):
"""Test loading data when all resolutions are available."""
from satpy.tests.utils import make_dsq
from satpy.readers import load_reader, get_key
filenames = [
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_0500M_V0001.HDF',
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_1000M_V0001.HDF',
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_2000M_V0001.HDF',
'FY4A-_AGRI--_N_REGC_1047E_L1-_FDI-_MULT_NOM_20190603003000_20190603003416_4000M_V0001.HDF',
]
reader = load_reader(self.reader_configs)
files = reader.select_files_from_pathnames(filenames)
self.assertEqual(4, len(files))
reader.create_filehandlers(files)
# Make sure we have some files
self.assertTrue(reader.file_handlers)
available_datasets = reader.available_dataset_ids
# 500m
band_names = ['C' + '%02d' % ch for ch in np.linspace(2, 2, 1)]
for band_name in band_names:
ds_q = make_dsq(name=band_name, resolution=500)
res = get_key(ds_q, available_datasets, num_results=0, best=False)
self.assertEqual(2, len(res))
# 1km
band_names = ['C' + '%02d' % ch for ch in np.linspace(1, 3, 3)]
for band_name in band_names:
ds_q = make_dsq(name=band_name, resolution=1000)
res = get_key(ds_q, available_datasets, num_results=0, best=False)
self.assertEqual(2, len(res))
# 2km
band_names = ['C' + '%02d' % ch for ch in np.linspace(1, 7, 7)]
for band_name in band_names:
ds_q = make_dsq(name=band_name, resolution=2000)
res = get_key(ds_q, available_datasets, num_results=0, best=False)
if band_name < 'C07':
self.assertEqual(2, len(res))
else:
self.assertEqual(3, len(res))
band_names = ['C' + '%02d' % ch for ch in np.linspace(1, 14, 14)]
res = reader.load(band_names)
self.assertEqual(14, len(res))
for band_name in band_names:
self.assertEqual((2, 5), res[band_name].shape)
if band_name < 'C07':
self.assertEqual('reflectance', res[band_name].attrs['calibration'])
else:
self.assertEqual('brightness_temperature', res[band_name].attrs['calibration'])
if band_name < 'C07':
self.assertEqual('%', res[band_name].attrs['units'])
else:
self.assertEqual('K', res[band_name].attrs['units'])
# check whether the data type of orbital_parameters is float
orbital_parameters = res[band_names[0]].attrs['orbital_parameters']
for attr in orbital_parameters:
self.assertEqual(type(orbital_parameters[attr]), float)
self.assertEqual(orbital_parameters['satellite_nominal_latitude'], 0.)
self.assertEqual(orbital_parameters['satellite_nominal_longitude'], 104.7)
self.assertEqual(orbital_parameters['satellite_nominal_altitude'], 3.5786E7)
def _get_dataset_key(self, key, **kwargs):
from satpy.readers import get_key
return get_key(key, self.datasets, **kwargs)
