def test_check_numpy_cache(self, xr_Dataset, np_load):
"""Test that cache stored in .npz is converted to zarr."""
from satpy.resample import KDTreeResampler
data, source_area, swath_data, source_swath, target_area = get_test_data(
)
resampler = KDTreeResampler(source_area, target_area)
zarr_out = mock.MagicMock()
xr_Dataset.return_value = zarr_out
try:
the_dir = tempfile.mkdtemp()
kwargs = {}
np_path = resampler._create_cache_filename(the_dir,
prefix='resample_lut-',
fmt='.npz',
mask=None,
**kwargs)
zarr_path = resampler._create_cache_filename(the_dir,
prefix='nn_lut-',
fmt='.zarr',
mask=None,
**kwargs)
resampler._check_numpy_cache(the_dir)
np_load.assert_not_called()
zarr_out.to_zarr.assert_not_called()
with open(np_path, 'w') as fid:
fid.write("42")
resampler._check_numpy_cache(the_dir)
np_load.assert_called_once_with(np_path, 'r')
zarr_out.to_zarr.assert_called_once_with(zarr_path)
finally:
shutil.rmtree(the_dir)
def test_kd_resampling(self, resampler, create_filename, load, savez):
"""Test the kd resampler."""
import numpy as np
import dask.array as da
from satpy.resample import KDTreeResampler
from pyresample.geometry import SwathDefinition
source_area = mock.MagicMock()
source_swath = SwathDefinition(
da.arange(5, chunks=5), da.arange(5, chunks=5))
target_area = mock.MagicMock()
resampler = KDTreeResampler(source_swath, target_area)
resampler.precompute(
mask=da.arange(5, chunks=5).astype(np.bool), cache_dir='.')
resampler.resampler.get_neighbour_info.assert_called()
# swath definitions should not be cached
self.assertFalse(len(savez.mock_calls), 0)
resampler.resampler.reset_mock()
resampler = KDTreeResampler(source_area, target_area)
resampler.precompute()
resampler.resampler.get_neighbour_info.assert_called_with(mask=None)
try:
the_dir = tempfile.mkdtemp()
resampler = KDTreeResampler(source_area, target_area)
create_filename.return_value = os.path.join(the_dir, 'test_cache.npz')
load.side_effect = IOError()
resampler.precompute(cache_dir=the_dir)
# assert data was saved to the on-disk cache
self.assertEqual(len(savez.mock_calls), 1)
# assert that load was called to try to load something from disk
self.assertEqual(len(load.mock_calls), 1)
# we should have cached things in-memory
self.assertEqual(len(resampler._index_caches), 1)
nbcalls = len(resampler.resampler.get_neighbour_info.mock_calls)
# test reusing the resampler
load.side_effect = None
class FakeNPZ(dict):
def close(self):
pass
load.return_value = FakeNPZ(valid_input_index=1,
valid_output_index=2,
index_array=3,
distance_array=4)
resampler.precompute(cache_dir=the_dir)
# we already have things cached in-memory, no need to save again
self.assertEqual(len(savez.mock_calls), 1)
# we already have things cached in-memory, don't need to load
self.assertEqual(len(load.mock_calls), 1)
# we should have cached things in-memory
self.assertEqual(len(resampler._index_caches), 1)
self.assertEqual(len(resampler.resampler.get_neighbour_info.mock_calls), nbcalls)
# test loading saved resampler
resampler = KDTreeResampler(source_area, target_area)
resampler.precompute(cache_dir=the_dir)
self.assertEqual(len(load.mock_calls), 2)
self.assertEqual(len(resampler.resampler.get_neighbour_info.mock_calls), nbcalls)
# we should have cached things in-memory now
self.assertEqual(len(resampler._index_caches), 1)
finally:
shutil.rmtree(the_dir)
data = mock.MagicMock()
data.name = 'hej'
data.data = [1, 2, 3]
fill_value = 8
resampler.compute(data, fill_value=fill_value)
resampler.resampler.get_sample_from_neighbour_info.assert_called_with(data, fill_value)
def test_kd_resampling(self, resampler, create_filename, from_zarr,
xr_dset, cnc):
"""Test the kd resampler."""
import numpy as np
import dask.array as da
from satpy.resample import KDTreeResampler
data, source_area, swath_data, source_swath, target_area = get_test_data(
)
mock_dset = mock.MagicMock()
xr_dset.return_value = mock_dset
resampler = KDTreeResampler(source_swath, target_area)
resampler.precompute(mask=da.arange(5, chunks=5).astype(np.bool),
cache_dir='.')
resampler.resampler.get_neighbour_info.assert_called()
# swath definitions should not be cached
self.assertFalse(len(mock_dset.to_zarr.mock_calls), 0)
resampler.resampler.reset_mock()
cnc.assert_called_once()
resampler = KDTreeResampler(source_area, target_area)
resampler.precompute()
resampler.resampler.get_neighbour_info.assert_called_with(mask=None)
try:
the_dir = tempfile.mkdtemp()
resampler = KDTreeResampler(source_area, target_area)
create_filename.return_value = os.path.join(
the_dir, 'test_cache.zarr')
from_zarr.side_effect = IOError()
resampler.precompute(cache_dir=the_dir)
# assert data was saved to the on-disk cache
self.assertEqual(len(mock_dset.to_zarr.mock_calls), 1)
# assert that from_zarr was called to try to from_zarr something from disk
self.assertEqual(len(from_zarr.mock_calls), 1)
# we should have cached things in-memory
self.assertEqual(len(resampler._index_caches), 1)
nbcalls = len(resampler.resampler.get_neighbour_info.mock_calls)
# test reusing the resampler
from_zarr.side_effect = None
class FakeZarr(dict):
def close(self):
pass
def astype(self, dtype):
pass
from_zarr.return_value = FakeZarr(valid_input_index=1,
valid_output_index=2,
index_array=3,
distance_array=4)
resampler.precompute(cache_dir=the_dir)
# we already have things cached in-memory, no need to save again
self.assertEqual(len(mock_dset.to_zarr.mock_calls), 1)
# we already have things cached in-memory, don't need to load
self.assertEqual(len(from_zarr.mock_calls), 1)
# we should have cached things in-memory
self.assertEqual(len(resampler._index_caches), 1)
self.assertEqual(
len(resampler.resampler.get_neighbour_info.mock_calls),
nbcalls)
# test loading saved resampler
resampler = KDTreeResampler(source_area, target_area)
resampler.precompute(cache_dir=the_dir)
self.assertEqual(len(from_zarr.mock_calls), 4)
self.assertEqual(
len(resampler.resampler.get_neighbour_info.mock_calls),
nbcalls)
# we should have cached things in-memory now
self.assertEqual(len(resampler._index_caches), 1)
finally:
shutil.rmtree(the_dir)
fill_value = 8
resampler.compute(data, fill_value=fill_value)
resampler.resampler.get_sample_from_neighbour_info.assert_called_with(
data, fill_value)
def test_kd_resampling(self):
"""Test the kd resampler."""
from satpy.resample import KDTreeResampler
source_area = mock.MagicMock()
target_area = mock.MagicMock()
with mock.patch('satpy.resample.XArrayResamplerNN'):
resampler = KDTreeResampler(source_area, target_area)
resampler.precompute()
resampler.resampler.get_neighbour_info.assert_called_with()
try:
the_dir = tempfile.mkdtemp()
with mock.patch('satpy.resample.KDTreeResampler._create_cache_filename') as create_filename:
resampler = KDTreeResampler(source_area, target_area)
create_filename.return_value = os.path.join(the_dir, 'test_cache.npz')
with mock.patch('satpy.resample.np.load') as load:
with mock.patch('satpy.resample.np.savez') as savez:
load.side_effect = IOError()
resampler.precompute(cache_dir=the_dir)
# assert saving
self.assertEqual(len(savez.mock_calls), 1)
nbcalls = len(resampler.resampler.get_neighbour_info.mock_calls)
# test reusing the resampler
load.side_effect = None
class FakeNPZ(dict):
def close(self):
pass
load.return_value = FakeNPZ(valid_input_index=1,
valid_output_index=2,
index_array=3,
distance_array=4)
self.assertEqual(len(savez.mock_calls), 1)
resampler.precompute(cache_dir=the_dir)
self.assertEqual(len(load.mock_calls), 1)
self.assertEqual(len(resampler.resampler.get_neighbour_info.mock_calls), nbcalls)
# test loading saved resampler
resampler = KDTreeResampler(source_area, target_area)
resampler.precompute(cache_dir=the_dir)
self.assertEqual(len(load.mock_calls), 2)
self.assertEqual(len(resampler.resampler.get_neighbour_info.mock_calls), nbcalls)
finally:
shutil.rmtree(the_dir)
data = mock.MagicMock()
data.name = 'hej'
data.data = [1, 2, 3]
fill_value = 8
resampler.compute(data, fill_value=fill_value)
resampler.resampler.get_sample_from_neighbour_info.assert_called_with(data, fill_value)
data.attrs = {'_FillValue': 8}
resampler.compute(data)
resampler.resampler.get_sample_from_neighbour_info.assert_called_with(data, fill_value)
def test_kd_resampling(self):
"""Test the kd resampler."""
from satpy.resample import KDTreeResampler
source_area = mock.MagicMock()
target_area = mock.MagicMock()
with mock.patch('satpy.resample.XArrayResamplerNN'):
resampler = KDTreeResampler(source_area, target_area)
resampler.precompute()
resampler.resampler.get_neighbour_info.assert_called_with()
try:
the_dir = tempfile.mkdtemp()
with mock.patch(
'satpy.resample.KDTreeResampler._create_cache_filename'
) as create_filename:
resampler = KDTreeResampler(source_area, target_area)
create_filename.return_value = os.path.join(
the_dir, 'test_cache.npz')
with mock.patch('satpy.resample.np.load') as load:
with mock.patch('satpy.resample.np.savez') as savez:
load.side_effect = IOError()
resampler.precompute(cache_dir=the_dir)
# assert saving
self.assertEqual(len(savez.mock_calls), 1)
nbcalls = len(resampler.resampler.
get_neighbour_info.mock_calls)
# test reusing the resampler
load.side_effect = None
class FakeNPZ(dict):
def close(self):
pass
load.return_value = FakeNPZ(valid_input_index=1,
valid_output_index=2,
index_array=3,
distance_array=4)
self.assertEqual(len(savez.mock_calls), 1)
resampler.precompute(cache_dir=the_dir)
self.assertEqual(len(load.mock_calls), 1)
self.assertEqual(
len(resampler.resampler.get_neighbour_info.
mock_calls), nbcalls)
# test loading saved resampler
resampler = KDTreeResampler(
source_area, target_area)
resampler.precompute(cache_dir=the_dir)
self.assertEqual(len(load.mock_calls), 2)
self.assertEqual(
len(resampler.resampler.get_neighbour_info.
mock_calls), nbcalls)
finally:
shutil.rmtree(the_dir)
data = mock.MagicMock()
data.name = 'hej'
data.data = [1, 2, 3]
fill_value = 8
resampler.compute(data, fill_value=fill_value)
resampler.resampler.get_sample_from_neighbour_info.assert_called_with(
data, fill_value)
data.attrs = {'_FillValue': 8}
resampler.compute(data)
resampler.resampler.get_sample_from_neighbour_info.assert_called_with(
data, fill_value)