def save_dataset(scns, fmat, fmat_config, renames):
"""Save one dataset to file, not doing the actual computation."""
obj = None
try:
with prepared_filename(fmat, renames) as filename:
res = fmat.get('resolution', None)
kwargs = fmat_config.copy()
kwargs.pop('fname_pattern', None)
kwargs.pop('dispatch', None)
if isinstance(fmat['product'], (tuple, list, set)):
kwargs.pop('format')
dsids = []
for prod in fmat['product']:
dsids.append(
DatasetID(name=prod, resolution=res, modifiers=None))
obj = scns[fmat['area']].save_datasets(datasets=dsids,
filename=filename,
compute=False,
**kwargs)
else:
dsid = DatasetID(name=fmat['product'],
resolution=res,
modifiers=None)
obj = scns[fmat['area']].save_dataset(dsid,
filename=filename,
compute=False,
**kwargs)
except KeyError as err:
LOG.info('Skipping %s: %s', fmat['product'], str(err))
else:
fmat_config['filename'] = renames.get(filename, filename)
return obj
def test_get_dataset_latlon(self, *mocks):
from satpy.dataset import DatasetID
lons = np.ones((3, 3))
lats = 2 * lons
reader = self._get_reader_mocked()
reader.get_lonlat.return_value = lons, lats
fh = self._get_fh_mocked(
reader=reader,
start_line=None,
end_line=None,
strip_invalid_coords=False,
interpolate_coords=True
)
# With interpolation of coordinates
for name, exp_data in zip(['longitude', 'latitude'], [lons, lats]):
key = DatasetID(name)
info = {'name': name, 'standard_name': 'my_standard_name'}
res = fh.get_dataset(key=key, info=info)
exp = xr.DataArray(exp_data,
name=res.name,
dims=('y', 'x'),
coords={'acq_time': ('y', [0, 1, 2])})
xr.testing.assert_equal(res, exp)
# Without interpolation of coordinates
fh.interpolate_coords = False
for name, exp_data in zip(['longitude', 'latitude'], [lons, lats]):
key = DatasetID(name)
info = {'name': name, 'standard_name': 'my_standard_name'}
res = fh.get_dataset(key=key, info=info)
self.assertTupleEqual(res.dims, ('y', 'x_every_eighth'))
def test_get_dataset_angles(self, get_angle, *mocks):
from satpy.dataset import DatasetID
from satpy.readers.avhrr_l1b_gaclac import ANGLES
ones = np.ones((3, 3))
get_angle.return_value = ones
reader = self._get_reader_mocked()
fh = self._get_fh_mocked(
reader=reader,
start_line=None,
end_line=None,
strip_invalid_coords=False,
interpolate_coords=True
)
# With interpolation of coordinates
for angle in ANGLES:
key = DatasetID(angle)
info = {'name': angle, 'standard_name': 'my_standard_name'}
res = fh.get_dataset(key=key, info=info)
exp = xr.DataArray(ones,
name=res.name,
dims=('y', 'x'),
coords={'acq_time': ('y', [0, 1, 2])})
xr.testing.assert_equal(res, exp)
# Without interpolation of coordinates
fh.interpolate_coords = False
for angle in ANGLES:
key = DatasetID(angle)
info = {'name': angle, 'standard_name': 'my_standard_name'}
res = fh.get_dataset(key=key, info=info)
self.assertTupleEqual(res.dims, ('y', 'x_every_eighth'))
def test_get_coordinates_for_dataset_keys(self):
"""Test getting coordinates for keys."""
ds_id1 = DatasetID(name='ch01',
wavelength=(0.5, 0.6, 0.7),
resolution=None,
polarization=None,
calibration='reflectance',
modifiers=())
ds_id2 = DatasetID(name='ch02',
wavelength=(0.7, 0.75, 0.8),
resolution=None,
polarization=None,
calibration='counts',
modifiers=())
lons = DatasetID(name='lons',
wavelength=None,
resolution=None,
polarization=None,
calibration=None,
modifiers=())
lats = DatasetID(name='lats',
wavelength=None,
resolution=None,
polarization=None,
calibration=None,
modifiers=())
res = self.reader._get_coordinates_for_dataset_keys(
[ds_id1, ds_id2, lons])
expected = {ds_id1: [lons, lats], ds_id2: [lons, lats], lons: []}
self.assertDictEqual(res, expected)
def setUp(self):
"""Create a test DatasetDict."""
from satpy.dataset import DatasetID
from satpy.readers import DatasetDict
self.regular_dict = regular_dict = {
DatasetID(name="test", wavelength=(0, 0.5, 1), resolution=1000):
"1",
DatasetID(name="testh", wavelength=(0, 0.5, 1), resolution=500):
"1h",
DatasetID(name="test2", wavelength=(1, 1.5, 2), resolution=1000):
"2",
DatasetID(name="test3",
wavelength=(1.2, 1.7, 2.2),
resolution=1000): "3",
DatasetID(name="test4", calibration="radiance", polarization="V"):
"4rad",
DatasetID(name="test4",
calibration="reflectance",
polarization="H"): "4refl",
DatasetID(name="test5", modifiers=('mod1', 'mod2')): "5_2mod",
DatasetID(name="test5", modifiers=('mod2', )): "5_1mod",
DatasetID(name='test6', level=100): '6_100',
DatasetID(name='test6', level=200): '6_200',
}
self.test_dict = DatasetDict(regular_dict)
def test_all_dataset_ids(self):
"""Check that all datasets ids are returned."""
self.assertSetEqual(set(self.reader.all_dataset_ids),
{DatasetID(name='ch02',
wavelength=(0.7, 0.75, 0.8),
resolution=None,
polarization=None,
calibration='counts',
modifiers=()),
DatasetID(name='ch01',
wavelength=(0.5, 0.6, 0.7),
resolution=None,
polarization=None,
calibration='reflectance',
modifiers=()),
DatasetID(name='lons',
wavelength=None,
resolution=None,
polarization=None,
calibration=None,
modifiers=()),
DatasetID(name='lats',
wavelength=None,
resolution=None,
polarization=None,
calibration=None,
modifiers=())})
def _process_composite_config(self, composite_name, conf, composite_type,
sensor_id, composite_config, **kwargs):
compositors = self.compositors[sensor_id]
modifiers = self.modifiers[sensor_id]
try:
options = conf[composite_type][composite_name]
loader = options.pop('compositor')
except KeyError:
if composite_name in compositors or composite_name in modifiers:
return conf
raise ValueError("'compositor' missing or empty in %s" %
composite_config)
options['name'] = composite_name
for prereq_type in ['prerequisites', 'optional_prerequisites']:
prereqs = []
for item in options.get(prereq_type, []):
if isinstance(item, dict):
key = DatasetID.from_dict(item)
prereqs.append(key)
else:
prereqs.append(item)
options[prereq_type] = prereqs
if composite_type == 'composites':
options.update(**kwargs)
key = DatasetID.from_dict(options)
comp = loader(**options)
compositors[key] = comp
elif composite_type == 'modifiers':
modifiers[composite_name] = loader, options
def _slice_datasets(self, dataset_ids, slice_key, new_area, area_only=True):
"""Slice scene in-place for the datasets specified."""
new_datasets = {}
datasets = (self[ds_id] for ds_id in dataset_ids)
for ds, parent_ds in dataset_walker(datasets):
ds_id = DatasetID.from_dict(ds.attrs)
# handle ancillary variables
pres = None
if parent_ds is not None:
pres = new_datasets[DatasetID.from_dict(parent_ds.attrs)]
if ds_id in new_datasets:
replace_anc(ds, pres)
continue
if area_only and ds.attrs.get('area') is None:
new_datasets[ds_id] = ds
replace_anc(ds, pres)
continue
if not isinstance(slice_key, dict):
# match dimension name to slice object
key = dict(zip(ds.dims, slice_key))
else:
key = slice_key
new_ds = ds.isel(**key)
if new_area is not None:
new_ds.attrs['area'] = new_area
new_datasets[ds_id] = new_ds
if parent_ds is None:
# don't use `__setitem__` because we don't want this to
# affect the existing wishlist/dep tree
self.datasets[ds_id] = new_ds
else:
replace_anc(new_ds, pres)
def get_dataset(self, key, info, out=None, xslice=None, yslice=None):
"""Get the dataset designated by *key*."""
if key.name in [
'solar_zenith_angle', 'solar_azimuth_angle',
'satellite_zenith_angle', 'satellite_azimuth_angle'
]:
if key.name == 'solar_zenith_angle':
var = self.sd.select('SolarZenith')
if key.name == 'solar_azimuth_angle':
var = self.sd.select('SolarAzimuth')
if key.name == 'satellite_zenith_angle':
var = self.sd.select('SensorZenith')
if key.name == 'satellite_azimuth_angle':
var = self.sd.select('SensorAzimuth')
data = xr.DataArray(from_sds(var, chunks=CHUNK_SIZE),
dims=['y', 'x']).astype(np.float32)
data = data.where(data != var._FillValue)
data = data * np.float32(var.scale_factor)
data.attrs = info
return data
if key.name not in ['longitude', 'latitude']:
return
if (self.cache[key.resolution]['lons'] is None
or self.cache[key.resolution]['lats'] is None):
lons_id = DatasetID('longitude', resolution=key.resolution)
lats_id = DatasetID('latitude', resolution=key.resolution)
lons, lats = self.load([lons_id, lats_id],
interpolate=False,
raw=True)
if key.resolution != self.resolution:
from geotiepoints.geointerpolator import GeoInterpolator
lons, lats = self._interpolate([lons, lats], self.resolution,
lons_id.resolution,
GeoInterpolator)
lons = np.ma.masked_invalid(np.ascontiguousarray(lons))
lats = np.ma.masked_invalid(np.ascontiguousarray(lats))
self.cache[key.resolution]['lons'] = lons
self.cache[key.resolution]['lats'] = lats
if key.name == 'latitude':
data = self.cache[key.resolution]['lats'].filled(np.nan)
data = xr.DataArray(da.from_array(data,
chunks=(CHUNK_SIZE, CHUNK_SIZE)),
dims=['y', 'x'])
else:
data = self.cache[key.resolution]['lons'].filled(np.nan)
data = xr.DataArray(da.from_array(data,
chunks=(CHUNK_SIZE, CHUNK_SIZE)),
dims=['y', 'x'])
data.attrs = info
return data
def test_compare_no_wl(self):
"""Compare fully qualified wavelength ID to no wavelength ID"""
from satpy.dataset import DatasetID
d1 = DatasetID(name="a", wavelength=(0.1, 0.2, 0.3))
d2 = DatasetID(name="a", wavelength=None)
# this happens when sorting IDs during dependency checks
self.assertFalse(d1 < d2)
self.assertTrue(d2 < d1)
def _resampled_scene(self, new_scn, destination_area, **resample_kwargs):
"""Resample `datasets` to the `destination` area."""
new_datasets = {}
datasets = list(new_scn.datasets.values())
max_area = None
if isinstance(destination_area, (str, six.text_type)):
destination_area = get_area_def(destination_area)
if hasattr(destination_area, 'freeze'):
try:
max_area = new_scn.max_area()
destination_area = destination_area.freeze(max_area)
except ValueError:
raise ValueError("No dataset areas available to freeze "
"DynamicAreaDefinition.")
resamplers = {}
for dataset, parent_dataset in dataset_walker(datasets):
ds_id = DatasetID.from_dict(dataset.attrs)
pres = None
if parent_dataset is not None:
pres = new_datasets[DatasetID.from_dict(parent_dataset.attrs)]
if ds_id in new_datasets:
replace_anc(dataset, pres)
continue
if dataset.attrs.get('area') is None:
if parent_dataset is None:
new_scn.datasets[ds_id] = dataset
else:
replace_anc(dataset, pres)
continue
LOG.debug("Resampling %s", ds_id)
source_area = dataset.attrs['area']
try:
slice_x, slice_y = source_area.get_area_slices(
destination_area)
source_area = source_area[slice_y, slice_x]
dataset = dataset.isel(x=slice_x, y=slice_y)
assert ('x', source_area.x_size) in dataset.sizes.items()
assert ('y', source_area.y_size) in dataset.sizes.items()
dataset.attrs['area'] = source_area
except NotImplementedError:
LOG.info("Not reducing data before resampling.")
if source_area not in resamplers:
key, resampler = prepare_resampler(source_area,
destination_area,
**resample_kwargs)
resamplers[source_area] = resampler
self.resamplers[key] = resampler
kwargs = resample_kwargs.copy()
kwargs['resampler'] = resamplers[source_area]
res = resample_dataset(dataset, destination_area, **kwargs)
new_datasets[ds_id] = res
if parent_dataset is None:
new_scn.datasets[ds_id] = res
else:
replace_anc(res, pres)
def test_get_file_handlers(self):
"""Test getting filehandler to load a dataset."""
ds_id1 = DatasetID(name='ch01', wavelength=(0.5, 0.6, 0.7),
resolution=None, polarization=None,
calibration='reflectance', modifiers=())
self.reader.file_handlers = {'ftype1': 'bla'}
self.assertEqual(self.reader._get_file_handlers(ds_id1), 'bla')
lons = DatasetID(name='lons', wavelength=None,
resolution=None, polarization=None,
calibration=None, modifiers=())
self.assertEqual(self.reader._get_file_handlers(lons), None)
def get_keys(self,
name_or_wl,
resolution=None,
polarization=None,
calibration=None,
modifiers=None):
# Get things that match at least the name_or_wl
if isinstance(name_or_wl, numbers.Number):
keys = [
k for k in self.keys()
if DatasetID.wavelength_match(k.wavelength, name_or_wl)
]
elif isinstance(name_or_wl, (str, six.text_type)):
keys = [
k for k in self.keys()
if DatasetID.name_match(k.name, name_or_wl)
]
else:
raise TypeError("First argument must be a wavelength or name")
if resolution is not None:
if not isinstance(resolution, (list, tuple)):
resolution = (resolution, )
keys = [
k for k in keys
if k.resolution is not None and k.resolution in resolution
]
if polarization is not None:
if not isinstance(polarization, (list, tuple)):
polarization = (polarization, )
keys = [
k for k in keys if k.polarization is not None
and k.polarization in polarization
]
if calibration is not None:
if not isinstance(calibration, (list, tuple)):
calibration = (calibration, )
keys = [
k for k in keys
if k.calibration is not None and k.calibration in calibration
]
if modifiers is not None:
keys = [
k for k in keys
if k.modifiers is not None and k.modifiers == modifiers
]
return keys
def filter_keys_by_dataset_id(did, key_container):
"""Filer provided key iterable by the provided `DatasetID`.
Note: The `modifiers` attribute of `did` should be `None` to allow for
**any** modifier in the results.
Args:
did (DatasetID): Query parameters to match in the `key_container`.
key_container (iterable): Set, list, tuple, or dict of `DatasetID`
keys.
Returns (list): List of keys matching the provided parameters in no
specific order.
"""
keys = iter(key_container)
for key in DATASET_KEYS:
if getattr(did, key) is not None:
if key == "wavelength":
keys = [k for k in keys
if (getattr(k, key) is not None and
DatasetID.wavelength_match(getattr(k, key),
getattr(did, key)))]
else:
keys = [k for k in keys
if getattr(k, key) is not None and getattr(k, key)
== getattr(did, key)]
return keys
def update_ds_ids_from_file_handlers(self):
"""Add or modify available dataset information.
Each file handler is consulted on whether or not it can load the
dataset with the provided information dictionary.
See
:meth:`satpy.readers.file_handlers.BaseFileHandler.available_datasets`
for more information.
"""
avail_datasets = self._file_handlers_available_datasets()
new_ids = {}
for is_avail, ds_info in avail_datasets:
# especially from the yaml config
coordinates = ds_info.get('coordinates')
if isinstance(coordinates, list):
# xarray doesn't like concatenating attributes that are
# lists: https://github.com/pydata/xarray/issues/2060
ds_info['coordinates'] = tuple(ds_info['coordinates'])
ds_info.setdefault('modifiers', tuple()) # default to no mods
ds_id = DatasetID.from_dict(ds_info)
# all datasets
new_ids[ds_id] = ds_info
# available datasets
# False == we have the file type but it doesn't have this dataset
# None == we don't have the file type object to ask
if is_avail:
self.available_ids[ds_id] = ds_info
self.all_ids = new_ids
def update_ds_ids_from_file_handlers(self):
"""Update DatasetIDs with information from loaded files.
This is useful, for example, if dataset resolution may change
depending on what files were loaded.
"""
for file_handlers in self.file_handlers.values():
fh = file_handlers[0]
# update resolution in the dataset IDs for this files resolution
res = getattr(fh, 'resolution', None)
if res is None:
continue
for ds_id, ds_info in list(self.ids.items()):
file_types = ds_info['file_type']
if not isinstance(file_types, list):
file_types = [file_types]
if fh.filetype_info['file_type'] not in file_types:
continue
if ds_id.resolution is not None:
continue
ds_info['resolution'] = res
new_id = DatasetID.from_dict(ds_info)
self.ids[new_id] = ds_info
del self.ids[ds_id]
def test_available_dataset_ids(self):
"""Get ids of the available datasets."""
self.reader.file_handlers = ['ftype1']
self.assertSetEqual(set(self.reader.available_dataset_ids),
{DatasetID(name='ch02',
wavelength=(0.7, 0.75, 0.8),
resolution=None,
polarization=None,
calibration='counts',
modifiers=()),
DatasetID(name='ch01',
wavelength=(0.5, 0.6, 0.7),
resolution=None,
polarization=None,
calibration='reflectance',
modifiers=())})
def test_init_dict(self):
"""Test DatasetDict init with a regular dict argument."""
from satpy.dataset import DatasetID
from satpy.readers import DatasetDict
regular_dict = {DatasetID(name="test", wavelength=(0, 0.5, 1)): "1", }
d = DatasetDict(regular_dict)
self.assertEqual(d, regular_dict)
def test_get_dataset_qual_flags(self, *mocks):
from satpy.dataset import DatasetID
qual_flags = np.ones((3, 7))
reader = self._get_reader_mocked()
reader.get_qual_flags.return_value = qual_flags
fh = self._get_fh_mocked(
reader=reader,
start_line=None,
end_line=None,
strip_invalid_coords=False,
interpolate_coords=True
)
key = DatasetID('qual_flags')
info = {'name': 'qual_flags'}
res = fh.get_dataset(key=key, info=info)
exp = xr.DataArray(qual_flags,
name=res.name,
dims=('y', 'num_flags'),
coords={'acq_time': ('y', [0, 1, 2]),
'num_flags': ['Scan line number',
'Fatal error flag',
'Insufficient data for calibration',
'Insufficient data for calibration',
'Solar contamination of blackbody in channels 3',
'Solar contamination of blackbody in channels 4',
'Solar contamination of blackbody in channels 5']})
xr.testing.assert_equal(res, exp)
def filter_keys_by_dataset_id(did, key_container):
"""Filer provided key iterable by the provided `DatasetID`.
Note: The `modifiers` attribute of `did` should be `None` to allow for
**any** modifier in the results.
Args:
did (DatasetID): Query parameters to match in the `key_container`.
key_container (iterable): Set, list, tuple, or dict of `DatasetID`
keys.
Returns (list): List of keys matching the provided parameters in no
specific order.
"""
keys = iter(key_container)
for key in DATASET_KEYS:
if getattr(did, key) is not None:
if key == "wavelength":
keys = [k for k in keys
if (getattr(k, key) is not None and
DatasetID.wavelength_match(getattr(k, key),
getattr(did, key)))]
else:
keys = [k for k in keys
if getattr(k, key) is not None and getattr(k, key)
== getattr(did, key)]
return keys
def get_dataset(self, key, info, out=None, xslice=None, yslice=None):
"""Get the dataset designated by *key*."""
if key.name in [
'solar_zenith_angle', 'solar_azimuth_angle',
'satellite_zenith_angle', 'satellite_azimuth_angle'
]:
if key.name == 'solar_zenith_angle':
var = self.sd.select('SolarZenith')
if key.name == 'solar_azimuth_angle':
var = self.sd.select('SolarAzimuth')
if key.name == 'satellite_zenith_angle':
var = self.sd.select('SensorZenith')
if key.name == 'satellite_azimuth_angle':
var = self.sd.select('SensorAzimuth')
mask = var[:] == var._FillValue
data = np.ma.masked_array(var[:] * var.scale_factor, mask=mask)
return Dataset(data, id=key, **info)
if key.name not in ['longitude', 'latitude']:
return
if (self.cache[key.resolution]['lons'] is None
or self.cache[key.resolution]['lats'] is None):
lons_id = DatasetID('longitude', resolution=key.resolution)
lats_id = DatasetID('latitude', resolution=key.resolution)
lons, lats = self.load([lons_id, lats_id],
interpolate=False,
raw=True)
if key.resolution != self.resolution:
from geotiepoints.geointerpolator import GeoInterpolator
lons, lats = self._interpolate([lons, lats], self.resolution,
lons_id.resolution,
GeoInterpolator)
lons = np.ma.masked_invalid(np.ascontiguousarray(lons))
lats = np.ma.masked_invalid(np.ascontiguousarray(lats))
self.cache[key.resolution]['lons'] = lons
self.cache[key.resolution]['lats'] = lats
if key.name == 'latitude':
return Dataset(self.cache[key.resolution]['lats'], id=key, **info)
else:
return Dataset(self.cache[key.resolution]['lons'], id=key, **info)
def test_get_area_def(self):
"""Test getting the area def."""
area = self.reader.get_area_def(DatasetID('HRV'))
self.assertEqual(area.area_extent,
(-45561979844414.07, -3720765401003.719, 45602912357076.38, 77771774058.38356))
self.assertEqual(area.proj_dict, {'a': 6378169.0,
'b': 6356583.8,
'h': 35785831.0,
'lon_0': 44,
'proj': 'geos',
'units': 'm'})
self.reader.fill_hrv = False
area = self.reader.get_area_def(DatasetID('HRV'))
self.assertEqual(area.defs[0].area_extent,
(-22017598561055.01, -2926674655354.9604, 23564847539690.22, 77771774058.38356))
self.assertEqual(area.defs[1].area_extent,
(-30793529275853.656, -3720765401003.719, 14788916824891.568, -2926674655354.9604))
def test_available_dataset_ids(self):
"""Get ids of the available datasets."""
loadables = self.reader.select_files_from_pathnames(['a001.bla'])
self.reader.create_filehandlers(loadables)
self.assertSetEqual(set(self.reader.available_dataset_ids),
{DatasetID(name='ch02',
wavelength=(0.7, 0.75, 0.8),
resolution=None,
polarization=None,
calibration='counts',
modifiers=()),
DatasetID(name='ch01',
wavelength=(0.5, 0.6, 0.7),
resolution=None,
polarization=None,
calibration='reflectance',
modifiers=())})
def _generate_composite(self, comp_node, keepables):
"""Collect all composite prereqs and create the specified composite.
Args:
comp_node (Node): Composite Node to generate a Dataset for
keepables (set): `set` to update if any datasets are needed
when generation is continued later. This can
happen if generation is delayed to incompatible
areas which would require resampling first.
"""
if comp_node.name in self.datasets:
# already loaded
return
compositor, prereqs, optional_prereqs = comp_node.data
try:
prereq_datasets = self._get_prereq_datasets(
comp_node.name,
prereqs,
keepables,
)
except KeyError:
return
optional_datasets = self._get_prereq_datasets(
comp_node.name,
optional_prereqs,
keepables,
skip=True
)
try:
composite = compositor(prereq_datasets,
optional_datasets=optional_datasets,
**self.attrs)
cid = DatasetID.from_dict(composite.attrs)
self.datasets[cid] = composite
# update the node with the computed DatasetID
if comp_node.name in self.wishlist:
self.wishlist.remove(comp_node.name)
self.wishlist.add(cid)
comp_node.name = cid
except IncompatibleAreas:
LOG.warning("Delaying generation of %s "
"because of incompatible areas",
str(compositor.id))
preservable_datasets = set(self.datasets.keys())
prereq_ids = set(p.name for p in prereqs)
opt_prereq_ids = set(p.name for p in optional_prereqs)
keepables |= preservable_datasets & (prereq_ids | opt_prereq_ids)
# even though it wasn't generated keep a list of what
# might be needed in other compositors
keepables.add(comp_node.name)
return
def test_get_coordinates_for_dataset_key_without(self):
"""Test getting coordinates for a key without coordinates."""
ds_id = DatasetID(name='lons',
wavelength=None,
resolution=None,
polarization=None,
calibration=None,
modifiers=())
res = self.reader._get_coordinates_for_dataset_key(ds_id)
self.assertListEqual(res, [])
def test_get_area_def(self):
"""Test getting the area def."""
area = self.reader.get_area_def(DatasetID('VIS006'))
self.assertEqual(area.proj_dict, {'a': 6378169.0,
'b': 6356583.8,
'h': 35785831.0,
'lon_0': 44.0,
'proj': 'geos',
'units': 'm'})
self.assertEqual(area.area_extent,
(-77771774058.38356, -3720765401003.719,
30310525626438.438, 77771774058.38356))
# Data shifted by 1.5km to N-W
self.reader.mda['offset_corrected'] = False
area = self.reader.get_area_def(DatasetID('VIS006'))
self.assertEqual(area.area_extent,
(-77771772558.38356, -3720765402503.719,
30310525627938.438, 77771772558.38356))
def test_get_coordinates_for_dataset_key(self):
"""Test getting coordinates for a key."""
ds_id = DatasetID(name='ch01', wavelength=(0.5, 0.6, 0.7),
resolution=None, polarization=None,
calibration='reflectance', modifiers=())
res = self.reader._get_coordinates_for_dataset_key(ds_id)
self.assertListEqual(res,
[DatasetID(name='lons',
wavelength=None,
resolution=None,
polarization=None,
calibration=None,
modifiers=()),
DatasetID(name='lats',
wavelength=None,
resolution=None,
polarization=None,
calibration=None,
modifiers=())])
def test_get_dataset_channels(self, get_channel, *mocks):
from satpy.dataset import DatasetID
# Mock reader and file handler
fh = self._get_fh_mocked(
reader=self._get_reader_mocked(),
chn_dict={'1': 0, '5': 0},
start_line=None,
end_line=None,
strip_invalid_coords=False,
filename_info={'orbit_number': 123},
sensor='sensor',
)
# Test calibration to reflectance as well as attributes.
counts = np.ones((3, 3))
get_channel.return_value = counts
key = DatasetID('1', calibration='reflectance')
info = {'name': '1', 'standard_name': 'my_standard_name'}
res = fh.get_dataset(key=key, info=info)
exp = xr.DataArray(da.ones((3, 3)),
name=res.name,
dims=('y', 'x'),
coords={'acq_time': ('y', [0, 1, 2])},
attrs={'name': '1',
'platform_name': 'spacecraft_name',
'orbit_number': 123,
'sensor': 'sensor',
'orbital_parameters': {'tle': 'tle'},
'midnight_scanline': 1,
'missing_scanlines': [1, 2, 3],
'foo': 'bar',
'standard_name': 'my_standard_name'})
exp.coords['acq_time'].attrs['long_name'] = 'Mean scanline acquisition time'
xr.testing.assert_identical(res, exp)
get_channel.assert_called_with(key)
# Counts & brightness temperature: Similar, just check _get_channel() call
for key in [DatasetID('1', calibration='counts'),
DatasetID('5', calibration='brightness_temperature')]:
fh.get_dataset(key=key, info={'name': 1})
get_channel.assert_called_with(key)
def test_get_channel(self):
from satpy.dataset import DatasetID
counts = np.moveaxis(np.array([[[1, 2, 3],
[4, 5, 6]]]), 0, 2)
calib_channels = 2 * counts
reader = self._get_reader_mocked()
reader.get_counts.return_value = counts
reader.get_calibrated_channels.return_value = calib_channels
fh = self._get_fh_mocked(reader=reader, counts=None, calib_channels=None,
chn_dict={'1': 0})
key = DatasetID('1', calibration='counts')
# Counts
res = fh._get_channel(key=key)
np.testing.assert_array_equal(res, [[1, 2, 3],
[4, 5, 6]])
np.testing.assert_array_equal(fh.counts, counts)
# Reflectance and Brightness Temperature
for calib in ['reflectance', 'brightness_temperature']:
key = DatasetID('1', calibration=calib)
res = fh._get_channel(key=key)
np.testing.assert_array_equal(res, [[2, 4, 6],
[8, 10, 12]])
np.testing.assert_array_equal(fh.calib_channels, calib_channels)
# Invalid
key = DatasetID('7', calibration='coffee')
self.assertRaises(ValueError, fh._get_channel, key=key)
# Buffering
reader.get_counts.reset_mock()
key = DatasetID('1', calibration='counts')
fh._get_channel(key=key)
reader.get_counts.assert_not_called()
reader.get_calibrated_channels.reset_mock()
for calib in ['reflectance', 'brightness_temperature']:
key = DatasetID('1', calibration=calib)
fh._get_channel(key)
reader.get_calibrated_channels.assert_not_called()
def test_get_area_def(self):
"""Test getting the area def."""
area = self.reader.get_area_def(DatasetID('VIS006'))
proj_dict = area.proj_dict
self.assertEqual(proj_dict['a'], 6378169.0)
self.assertEqual(proj_dict['b'], 6356583.8)
self.assertEqual(proj_dict['h'], 35785831.0)
self.assertEqual(proj_dict['lon_0'], 44.0)
self.assertEqual(proj_dict['proj'], 'geos')
self.assertEqual(proj_dict['units'], 'm')
self.assertEqual(area.area_extent,
(-77771774058.38356, -3720765401003.719,
30310525626438.438, 77771774058.38356))
# Data shifted by 1.5km to N-W
self.reader.mda['offset_corrected'] = False
area = self.reader.get_area_def(DatasetID('VIS006'))
self.assertEqual(area.area_extent,
(-77771772558.38356, -3720765402503.719,
30310525627938.438, 77771772558.38356))
def test_get_datasets_by_id(self):
"""Check getting datasets by id."""
from satpy.dataset import DatasetID
dsid = DatasetID('ch01')
res = self.reader.get_ds_ids_by_id(dsid)
self.assertEqual(len(res), 1)
self.assertEqual(res[0].name, 'ch01')
dsid = DatasetID(wavelength=.6)
res = self.reader.get_ds_ids_by_id(dsid)
self.assertEqual(len(res), 1)
self.assertEqual(res[0].name, 'ch01')
dsid = DatasetID('ch01', .6)
res = self.reader.get_ds_ids_by_id(dsid)
self.assertEqual(len(res), 1)
self.assertEqual(res[0].name, 'ch01')
dsid = DatasetID('ch01', .1)
self.assertRaises(KeyError, self.reader.get_ds_ids_by_id, dsid)
def _process_composite_config(self, composite_name, conf,
composite_type, sensor_id, composite_config, **kwargs):
compositors = self.compositors[sensor_id]
modifiers = self.modifiers[sensor_id]
try:
options = conf[composite_type][composite_name]
loader = options.pop('compositor')
except KeyError:
if composite_name in compositors or composite_name in modifiers:
return conf
raise ValueError("'compositor' missing or empty in {0}. Option keys = {1}".format(
composite_config, str(options.keys())))
options['name'] = composite_name
for prereq_type in ['prerequisites', 'optional_prerequisites']:
prereqs = []
for item in options.get(prereq_type, []):
if isinstance(item, dict):
# we want this prerequisite to act as a query with
# 'modifiers' being None otherwise it will be an empty
# tuple
item.setdefault('modifiers', None)
key = DatasetID.from_dict(item)
prereqs.append(key)
else:
prereqs.append(item)
options[prereq_type] = prereqs
if composite_type == 'composites':
options.update(**kwargs)
key = DatasetID.from_dict(options)
comp = loader(**options)
compositors[key] = comp
elif composite_type == 'modifiers':
modifiers[composite_name] = loader, options
def update_ds_ids_from_file_handlers(self):
"""Update DatasetIDs with information from loaded files.
This is useful, for example, if dataset resolution may change
depending on what files were loaded.
"""
for file_handlers in self.file_handlers.values():
fh = file_handlers[0]
# update resolution in the dataset IDs for this files resolution
res = getattr(fh, 'resolution', None)
if res is None:
continue
for ds_id, ds_info in list(self.ids.items()):
if fh.filetype_info['file_type'] != ds_info['file_type']:
continue
if ds_id.resolution is not None:
continue
ds_info['resolution'] = res
new_id = DatasetID.from_dict(ds_info)
self.ids[new_id] = ds_info
del self.ids[ds_id]
