def test_cache_get_angles(self, input_func, input2_func, exp_equal_sun,
exp_num_zarr, tmpdir):
"""Test get_angles when caching is enabled."""
from satpy.modifiers.angles import (
STATIC_EARTH_INERTIAL_DATETIME,
_get_sensor_angles_from_sat_pos,
_get_valid_lonlats,
get_angles,
)
# Patch methods
data = input_func()
additional_cache = exp_num_zarr > 4
# Compute angles
from pyorbital.orbital import get_observer_look
with mock.patch("satpy.modifiers.angles.get_observer_look", wraps=get_observer_look) as gol, \
satpy.config.set(cache_lonlats=True, cache_sensor_angles=True, cache_dir=str(tmpdir)):
res = get_angles(data)
assert all(isinstance(x, xr.DataArray) for x in res)
# call again, should be cached
new_data = input2_func(data)
res2 = get_angles(new_data)
assert all(isinstance(x, xr.DataArray) for x in res2)
res, res2 = da.compute(res, res2)
for r1, r2 in zip(res[:2], res2[:2]):
if additional_cache:
pytest.raises(AssertionError, np.testing.assert_allclose,
r1, r2)
else:
np.testing.assert_allclose(r1, r2)
for r1, r2 in zip(res[2:], res2[2:]):
if exp_equal_sun:
np.testing.assert_allclose(r1, r2)
else:
pytest.raises(AssertionError, np.testing.assert_allclose,
r1, r2)
zarr_dirs = glob(str(tmpdir / "*.zarr"))
assert len(
zarr_dirs
) == exp_num_zarr # two for lon/lat, one for sata, one for satz
_get_sensor_angles_from_sat_pos.cache_clear()
_get_valid_lonlats.cache_clear()
zarr_dirs = glob(str(tmpdir / "*.zarr"))
assert len(zarr_dirs) == 0
assert gol.call_count == data.data.blocks.size * (
int(additional_cache) + 1)
args = gol.call_args_list[0][0]
assert args[:4] == (10.0, 0.0, 12345.678,
STATIC_EARTH_INERTIAL_DATETIME)
exp_sat_lon = 10.1 if additional_cache else 10.0
args = gol.call_args_list[-1][0]
assert args[:4] == (exp_sat_lon, 0.0, 12345.678,
STATIC_EARTH_INERTIAL_DATETIME)
def test_no_cache_dir_fails(self, tmp_path):
"""Test that 'cache_dir' not being set fails."""
from satpy.modifiers.angles import _get_sensor_angles_from_sat_pos, get_angles
data = _get_angle_test_data()
with pytest.raises(RuntimeError), \
satpy.config.set(cache_lonlats=True, cache_sensor_angles=True, cache_dir=None):
get_angles(data)
with pytest.raises(RuntimeError), \
satpy.config.set(cache_lonlats=True, cache_sensor_angles=True, cache_dir=None):
_get_sensor_angles_from_sat_pos.cache_clear()
def test_get_angles_satpos_preference(self, forced_preference):
"""Test that 'actual' satellite position is used for generating sensor angles."""
from satpy.modifiers.angles import get_angles
input_data1 = _get_angle_test_data()
# add additional satellite position metadata
input_data1.attrs["orbital_parameters"]["nadir_longitude"] = 9.0
input_data1.attrs["orbital_parameters"]["nadir_latitude"] = 0.01
input_data1.attrs["orbital_parameters"][
"satellite_actual_longitude"] = 9.5
input_data1.attrs["orbital_parameters"][
"satellite_actual_latitude"] = 0.005
input_data1.attrs["orbital_parameters"][
"satellite_actual_altitude"] = 12345679
input_data2 = input_data1.copy(deep=True)
input_data2.attrs = deepcopy(input_data1.attrs)
input_data2.attrs["orbital_parameters"]["nadir_longitude"] = 9.1
input_data2.attrs["orbital_parameters"]["nadir_latitude"] = 0.02
input_data2.attrs["orbital_parameters"][
"satellite_actual_longitude"] = 9.5
input_data2.attrs["orbital_parameters"][
"satellite_actual_latitude"] = 0.005
input_data2.attrs["orbital_parameters"][
"satellite_actual_altitude"] = 12345679
from pyorbital.orbital import get_observer_look
with mock.patch("satpy.modifiers.angles.get_observer_look", wraps=get_observer_look) as gol, \
satpy.config.set(sensor_angles_position_preference=forced_preference):
angles1 = get_angles(input_data1)
da.compute(angles1)
angles2 = get_angles(input_data2)
da.compute(angles2)
# get_observer_look should have been called once per array chunk
assert gol.call_count == input_data1.data.blocks.size * 2
if forced_preference == "actual":
exp_call = mock.call(9.5, 0.005, 12345.679,
input_data1.attrs["start_time"], mock.ANY,
mock.ANY, 0)
all_same_calls = [exp_call] * gol.call_count
gol.assert_has_calls(all_same_calls)
# the dask arrays should have the same name to prove they are the same computation
for angle_arr1, angle_arr2 in zip(angles1, angles2):
assert angle_arr1.data.name == angle_arr2.data.name
else:
# nadir 1
gol.assert_any_call(9.0, 0.01, 12345.679,
input_data1.attrs["start_time"], mock.ANY,
mock.ANY, 0)
# nadir 2
gol.assert_any_call(9.1, 0.02, 12345.679,
input_data1.attrs["start_time"], mock.ANY,
mock.ANY, 0)
def test_cache_get_angles(self, input_func, num_normalized_chunks,
exp_zarr_chunks, input2_func, exp_equal_sun,
exp_num_zarr, force_bad_glob, tmp_path):
"""Test get_angles when caching is enabled."""
from satpy.modifiers.angles import STATIC_EARTH_INERTIAL_DATETIME, get_angles
# Patch methods
data = input_func()
additional_cache = exp_num_zarr > 4
# Compute angles
from pyorbital.orbital import get_observer_look
with mock.patch("satpy.modifiers.angles.get_observer_look", wraps=get_observer_look) as gol, \
satpy.config.set(cache_lonlats=True, cache_sensor_angles=True, cache_dir=str(tmp_path)), \
warnings.catch_warnings(record=True) as caught_warnings:
res = get_angles(data)
self._check_cached_result(res, exp_zarr_chunks)
# call again, should be cached
new_data = input2_func(data)
with _mock_glob_if(force_bad_glob):
res2 = get_angles(new_data)
self._check_cached_result(res2, exp_zarr_chunks)
res_numpy, res2_numpy = da.compute(res, res2)
for r1, r2 in zip(res_numpy[:2], res2_numpy[:2]):
_assert_allclose_if(not additional_cache, r1, r2)
for r1, r2 in zip(res_numpy[2:], res2_numpy[2:]):
_assert_allclose_if(exp_equal_sun, r1, r2)
self._check_cache_and_clear(tmp_path, exp_num_zarr)
if "odd_chunks" in input_func.__name__:
assert any(w.category is PerformanceWarning
for w in caught_warnings)
else:
assert not any(w.category is PerformanceWarning
for w in caught_warnings)
assert gol.call_count == num_normalized_chunks * (
int(additional_cache) + 1)
args = gol.call_args_list[0][0]
assert args[:4] == (10.0, 0.0, 12345.678,
STATIC_EARTH_INERTIAL_DATETIME)
exp_sat_lon = 10.1 if additional_cache else 10.0
args = gol.call_args_list[-1][0]
assert args[:4] == (exp_sat_lon, 0.0, 12345.678,
STATIC_EARTH_INERTIAL_DATETIME)
def __call__(self, projectables, optional_datasets=None, **info):
"""Get the corrected reflectance when removing Rayleigh scattering.
Uses pyspectral.
"""
from pyspectral.rayleigh import Rayleigh
if not optional_datasets or len(optional_datasets) != 4:
vis, red = self.match_data_arrays(projectables)
sata, satz, suna, sunz = get_angles(vis)
else:
vis, red, sata, satz, suna, sunz = self.match_data_arrays(
projectables + optional_datasets)
# get the dask array underneath
sata = sata.data
satz = satz.data
suna = suna.data
sunz = sunz.data
# First make sure the two azimuth angles are in the range 0-360:
sata = sata % 360.
suna = suna % 360.
ssadiff = da.absolute(suna - sata)
ssadiff = da.minimum(ssadiff, 360 - ssadiff)
del sata, suna
atmosphere = self.attrs.get('atmosphere', 'us-standard')
aerosol_type = self.attrs.get('aerosol_type', 'marine_clean_aerosol')
rayleigh_key = (vis.attrs['platform_name'],
vis.attrs['sensor'], atmosphere, aerosol_type)
logger.info("Removing Rayleigh scattering with atmosphere '%s' and "
"aerosol type '%s' for '%s'",
atmosphere, aerosol_type, vis.attrs['name'])
if rayleigh_key not in self._rayleigh_cache:
corrector = Rayleigh(vis.attrs['platform_name'], vis.attrs['sensor'],
atmosphere=atmosphere,
aerosol_type=aerosol_type)
self._rayleigh_cache[rayleigh_key] = corrector
else:
corrector = self._rayleigh_cache[rayleigh_key]
try:
refl_cor_band = corrector.get_reflectance(sunz, satz, ssadiff,
vis.attrs['name'],
red.data)
except (KeyError, IOError):
logger.warning("Could not get the reflectance correction using band name: %s", vis.attrs['name'])
logger.warning("Will try use the wavelength, however, this may be ambiguous!")
refl_cor_band = corrector.get_reflectance(sunz, satz, ssadiff,
vis.attrs['wavelength'][1],
red.data)
proj = vis - refl_cor_band
proj.attrs = vis.attrs
self.apply_modifier_info(vis, proj)
return proj
def _extract_angle_data_arrays(self, datasets, optional_datasets):
all_datasets = datasets + optional_datasets
if len(all_datasets) == 1:
vis = self.match_data_arrays(datasets)[0]
return vis, get_angles(vis)
if len(all_datasets) == 5:
vis, *angles = self.match_data_arrays(datasets + optional_datasets)
return vis, angles
raise ValueError("Not sure how to handle provided dependencies. "
"Either all 4 angles must be provided or none of "
"of them.")
def test_get_angles(self, input_func):
"""Test sun and satellite angle calculation."""
from satpy.modifiers.angles import get_angles
data = input_func()
from pyorbital.orbital import get_observer_look
with mock.patch("satpy.modifiers.angles.get_observer_look", wraps=get_observer_look) as gol:
angles = get_angles(data)
assert all(isinstance(x, xr.DataArray) for x in angles)
da.compute(angles)
# get_observer_look should have been called once per array chunk
assert gol.call_count == data.data.blocks.size
# Check arguments of get_orbserver_look() call, especially the altitude
# unit conversion from meters to kilometers
args = gol.call_args[0]
assert args[:4] == (10.0, 0.0, 12345.678, data.attrs["start_time"])