def test_merge_tiles_horizontal(self):
tile1 = Tile()
tile1.latitudes = np.ma.array([0.0, 1.0, 2.0, 3.0])
tile1.longitudes = np.ma.array([0.0, -1.0, -2.0])
tile1.times = np.ma.array([0])
tile1.data = np.ma.arange(12).reshape((1, 4, 3))
tile2 = Tile()
tile2.latitudes = np.ma.array([0.0, 1.0, 2.0, 3.0])
tile2.longitudes = np.ma.array([-3.0, -4.0, -5.0])
tile2.times = np.ma.array([0])
tile2.data = np.ma.arange(12, 24).reshape((1, 4, 3))
from nexustiles.model.nexusmodel import merge_tiles
times, lats, longs, data = merge_tiles([tile1, tile2])
self.assertTrue(np.ma.allequal(times, np.array([0])))
self.assertTrue(np.ma.allequal(lats, np.array([0.0, 1.0, 2.0, 3.0])))
self.assertTrue(
np.ma.allequal(longs, np.array([-5.0, -4.0, -3.0, -2.0, -1.0,
0.0])))
expected = np.ma.array(
[[[14, 13, 12, 2, 1, 0], [17, 16, 15, 5, 4, 3],
[20, 19, 18, 8, 7, 6], [23, 22, 21, 11, 10, 9]]],
mask=False)
self.assertTrue(
np.ma.allequal(np.ma.getmaskarray(data),
np.ma.getmaskarray(expected)))
self.assertTrue(np.ma.allequal(data, expected))
def test_multiple_tiles_same_long(self):
tile1 = Tile()
tile1.latitudes = np.ma.array([0.0, 1.0, 2.0, 3.0])
tile1.longitudes = np.ma.array([0.0, -1.0, -2.0])
tile1.times = np.ma.array([0L])
tile1.data = np.ma.arange(12).reshape((1, 4, 3))
tile2 = Tile()
tile2.latitudes = np.ma.array([4.0, 5.0, 6.0, 7.0])
tile2.longitudes = np.ma.array([0.0, -1.0, -2.0])
tile2.times = np.ma.array([0L])
tile2.data = np.ma.arange(12, 24).reshape((1, 4, 3))
self.assertAlmostEqual(1, get_approximate_value_for_lat_lon([tile1, tile2], 0.4, -1))
def test_merge_tiles_overlapping(self):
tile1 = Tile()
tile1.latitudes = np.ma.array([0.0, 1.0, 2.0, 3.0])
tile1.longitudes = np.ma.array([0.0, -1.0, -2.0])
tile1.times = np.ma.array([0])
tile1.data = np.ma.arange(12).reshape((1, 4, 3))
tile2 = Tile()
tile2.latitudes = np.ma.array([0.0, 1.0, 2.0, 3.0])
tile2.longitudes = np.ma.array([0.0, -1.0, -2.0])
tile2.times = np.ma.array([0])
tile2.data = np.ma.arange(12, 24).reshape((1, 4, 3))
from nexustiles.model.nexusmodel import merge_tiles
self.assertRaises(Exception, lambda _: merge_tiles([tile1, tile2]))
def test_multi_variable_satellite_to_satellite_matchup(test_dir, test_tile,
test_matchup_args):
"""
Test multi-variable satellite to satellite matchup functionality.
"""
test_tile.latitudes = np.array([0, 20], dtype=np.float32)
test_tile.longitudes = np.array([0, 20], dtype=np.float32)
test_tile.times = [1627490285]
test_tile.data = np.array([[[[1.10, 2.10], [3.10, 4.10]]],
[[[11.0, 21.0], [31.0, 41.0]]]])
test_tile.is_multi = True
test_tile.variables = [
TileVariable('wind_speed', 'wind_speed'),
TileVariable('wind_dir', 'wind_direction')
]
test_matchup_args['tile_service_factory'] = setup_mock_tile_service(
test_tile)
with mock.patch(
'webservice.algorithms_spark.Matchup.edge_endpoints.getEndpointByName'
) as mock_edge_endpoints:
mock_edge_endpoints.return_value = None
# Open the edge response json. We want to convert these points
# to tile points so we can test sat to sat matchup
edge_json = json.load(
open(os.path.join(test_dir, 'edge_response.json')))
points = [
wkt.loads(result['point']) for result in edge_json['results']
]
matchup_tile = Tile()
matchup_tile.variables = [
TileVariable('sst', 'sea_surface_temperature'),
TileVariable('wind_dir', 'wind_direction')
]
matchup_tile.latitudes = np.array([point.y for point in points],
dtype=np.float32)
matchup_tile.longitudes = np.array([point.x for point in points],
dtype=np.float32)
matchup_tile.times = [edge_json['results'][0]['time']]
matchup_tile.data = np.array([[[[10.0, 0, 0], [0, 20.0, 0],
[0, 0, 30.0]]],
[[[100.0, 0, 0], [0, 200.0, 0],
[0, 0, 300.0]]]])
# matchup_tile.get_indices = lambda: [[0, 0, 0], [0, 1, 1], [0, 2, 2]]
matchup_tile.is_multi = True
test_matchup_args['tile_service_factory'](
).find_tiles_in_polygon.return_value = [matchup_tile]
generator = matchup.match_satellite_to_insitu(**test_matchup_args)
matchup_result = list(generator)
assert len(matchup_result) == 2
assert len(matchup_result[0]) == 2
assert len(matchup_result[1]) == 2
assert len(matchup_result[0][0].data) == 2
assert len(matchup_result[0][1].data) == 2
assert len(matchup_result[1][0].data) == 2
assert len(matchup_result[1][1].data) == 2
def test_masked_tile(self):
tile = Tile()
tile.bbox = BBox(30.5, 37.5, -51.5, -36.5)
tile.latitudes = np.ma.arange(30.5, 38.5, 1.0)
tile.longitudes = np.ma.arange(-51.5, -35.5, 1.0)
tile.times = np.ma.array([0])
tile.data = np.ma.arange(128.0).reshape((1, 8, 16))
# tile.latitudes [ 30.5 31.5 32.5 33.5 34.5 35.5 36.5 37.5]
tile.latitudes = np.ma.masked_outside(tile.latitudes, 35, 45)
# tile.latitudes [-- -- -- -- -- 35.5 36.5 37.5]
# tile.longitudes [-51.5 -50.5 -49.5 -48.5 -47.5 -46.5 -45.5 -44.5 -43.5 -42.5 -41.5 -40.5 -39.5 -38.5 -37.5 -36.5]
tile.longitudes = np.ma.masked_outside(tile.longitudes, -50, -40)
# tile.longitudes [-- -- -49.5 -48.5 -47.5 -46.5 -45.5 -44.5 -43.5 -42.5 -41.5 -40.5 -- -- -- --]
# Tile no longer contains 35, -50
self.assertFalse(tile.contains_point(35, -50))
def setUp(self):
tile = Tile()
tile.bbox = BBox(-1.0, 1.0, -2.0, 2.0)
tile.latitudes = np.ma.array([-1.0, -0.5, 0, .5, 1.0])
tile.longitudes = np.ma.array([-2.0, -1.0, 0, 1.0, 2.0])
tile.times = np.ma.array([0L])
tile.data = np.ma.arange(25.0).reshape((1, 5, 5))
tile.meta_data = {"std": np.ma.arange(25.0).reshape((1, 5, 5))}
attrs = {'find_tile_by_polygon_and_most_recent_day_of_year.return_value': [tile]}
self.tile_service = Mock(spec=NexusTileService, **attrs)
def test_lat_exact_lon_approx(self):
tile = Tile()
tile.bbox = BBox(-1.0, 1.0, -2.0, 2.0)
tile.latitudes = np.ma.array([-1.0, -0.5, 0, .5, 1.0])
tile.longitudes = np.ma.array([-2.0, -1.0, 0, 1.0, 2.0])
tile.times = np.ma.array([0L])
tile.data = np.ma.arange(25.0).reshape((1, 5, 5))
# -2 -1 0 1 2
# -1.0 [[[0. 1. 2. 3. 4. ]
# -0.5 [5. 6. 7. 8. 9. ]
# 0. [10. 11. 12. 13. 14.]
# 0.5 [15. 16. 17. 18. 19.]
# 1.0 [20. 21. 22. 23. 24.]]]
self.assertAlmostEqual(18.0, get_approximate_value_for_lat_lon([tile], 0.5, 1.01))
def test_lon_greater_than_bounds(self):
tile = Tile()
tile.bbox = BBox(-1.0, 1.0, -2.0, 2.0)
tile.latitudes = np.ma.array([-1.0, -0.5, 0, .5, 1.0])
tile.longitudes = np.ma.array([-2.0, -1.0, 0, 1.0, 2.0])
tile.times = np.ma.array([0L])
tile.data = np.ma.arange(25.0).reshape((1, 5, 5))
# -2 -1 0 1 2
# -1.0 [[[0. 1. 2. 3. 4. ]
# -0.5 [5. 6. 7. 8. 9. ]
# 0. [10. 11. 12. 13. 14.]
# 0.5 [15. 16. 17. 18. 19.]
# 1.0 [20. 21. 22. 23. 24.]]]
self.assertTrue(np.isnan(get_approximate_value_for_lat_lon([tile], 0, 3)))
def test_update_tile_stats(self):
tile = Tile()
tile.bbox = BBox(-1.0, 1.0, -2.0, 2.0)
tile.latitudes = np.ma.array([-1.0, -0.5, 0, .5, 1.0])
tile.longitudes = np.ma.array([-2.0, -1.0, 0, 1.0, 2.0])
tile.times = np.ma.array([0])
tile.data = np.ma.arange(25.0).reshape((1, 5, 5))
# -2 -1 0 1 2
# -1.0 [[[0. 1. 2. 3. 4. ]
# -0.5 [5. 6. 7. 8. 9. ]
# 0. [10. 11. 12. 13. 14.]
# 0.5 [15. 16. 17. 18. 19.]
# 1.0 [20. 21. 22. 23. 24.]]]
tile.update_stats()
self.assertAlmostEqual(0.0, tile.tile_stats.min)
self.assertAlmostEqual(24.0, tile.tile_stats.max)
self.assertAlmostEqual(12.0, tile.tile_stats.mean)
self.assertEqual(25, tile.tile_stats.count)
def test_match_satellite_to_insitu(test_dir, test_tile, test_matchup_args):
"""
Test the test_match_satellite_to_insitu and ensure the matchup is
done as expected, where the tile points and in-situ points are all
known and the expected matchup points have been hand-calculated.
This test case mocks out all external dependencies, so Solr,
Cassandra, HTTP insitu requests, etc are all mocked.
The test points are as follows:
X (0, 20) X (20, 20)
O (5, 15)
O (10, 10)
O (18, 3)
X (0, 0) X (20, 0)
The 'X' points are the primary satellite points and the 'O' points
are the secondary satellite or insitu points
Visual inspection reveals that primary point (0, 20) should match
with secondary point (5, 15) and primary point (20, 0) should match
with (18, 3)
"""
test_tile.variables = [TileVariable('sst', 'sea_surface_temperature')]
test_tile.latitudes = np.array([0, 20], dtype=np.float32)
test_tile.longitudes = np.array([0, 20], dtype=np.float32)
test_tile.times = [1627490285]
test_tile.data = np.array([[[11.0, 21.0], [31.0, 41.0]]])
test_tile.get_indices = lambda: [[0, 0, 0], [0, 0, 1], [0, 1, 0],
[0, 1, 1]]
test_tile.is_multi = False
tile_ids = [1]
polygon_wkt = 'POLYGON((-34.98 29.54, -30.1 29.54, -30.1 31.00, -34.98 31.00, -34.98 29.54))'
primary_ds_name = 'primary-ds-name'
matchup_ds_names = 'test'
parameter = 'sst'
depth_min = 0.0
depth_max = 1.0
time_tolerance = 3.0
radius_tolerance = 1000000.0
platforms = '1,2,3,4,5,6,7,8,9'
with mock.patch(
'webservice.algorithms_spark.Matchup.edge_endpoints.getEndpointByName'
) as mock_edge_endpoints:
# Test the satellite->insitu branch
# By mocking the getEndpointsByName function we are forcing
# Matchup to think this dummy matchup dataset is an insitu
# dataset
mock_edge_endpoints.return_value = {'url': 'http://test-edge-url'}
matchup.query_edge = lambda *args, **kwargs: json.load(
open(os.path.join(test_dir, 'edge_response.json')))
match_args = dict(
tile_ids=tile_ids,
primary_b=MockSparkParam(primary_ds_name),
matchup_b=MockSparkParam(matchup_ds_names),
parameter_b=MockSparkParam(parameter),
tt_b=MockSparkParam(time_tolerance),
rt_b=MockSparkParam(radius_tolerance),
platforms_b=MockSparkParam(platforms),
bounding_wkt_b=MockSparkParam(polygon_wkt),
depth_min_b=MockSparkParam(depth_min),
depth_max_b=MockSparkParam(depth_max),
tile_service_factory=setup_mock_tile_service(test_tile))
generator = matchup.match_satellite_to_insitu(**match_args)
def validate_matchup_result(matchup_result, insitu_matchup):
"""
The matchup results for satellite->insitu vs
satellite->satellite are almost exactly the same so they
can be validated using the same logic. They are the same
because they represent the same data, except one test is in
tile format (sat to sat) and one is in edge point format
(insitu). The only difference is the data field is different
for satellite data.
"""
# There should be two primary matchup points
assert len(matchup_result) == 2
# Each primary point matched with 1 matchup point
assert len(matchup_result[0]) == 2
assert len(matchup_result[1]) == 2
# Check that the satellite point was matched to the expected secondary point
assert matchup_result[0][1].latitude == 3.0
assert matchup_result[0][1].longitude == 18.0
assert matchup_result[1][1].latitude == 15.0
assert matchup_result[1][1].longitude == 5.0
# Check that the secondary points have the expected values
if insitu_matchup:
assert matchup_result[0][1].data[0].variable_value == 30.0
assert matchup_result[1][1].data[0].variable_value == 10.0
assert matchup_result[0][1].data[
0].variable_name == 'sea_water_temperature'
assert matchup_result[1][1].data[
0].variable_name == 'sea_water_temperature'
else:
assert matchup_result[0][1].data[0].variable_value == 30.0
assert matchup_result[1][1].data[0].variable_value == 10.0
assert matchup_result[0][1].data[0].variable_name == 'sst'
assert matchup_result[0][1].data[
0].cf_variable_name == 'sea_surface_temperature'
assert matchup_result[1][1].data[0].variable_name == 'sst'
assert matchup_result[1][1].data[
0].cf_variable_name == 'sea_surface_temperature'
# Check that the satellite points have the expected values
assert matchup_result[0][0].data[0].variable_value == 21.0
assert matchup_result[1][0].data[0].variable_value == 31.0
assert matchup_result[0][0].data[0].variable_name == 'sst'
assert matchup_result[0][0].data[
0].cf_variable_name == 'sea_surface_temperature'
assert matchup_result[1][0].data[0].variable_name == 'sst'
assert matchup_result[1][0].data[
0].cf_variable_name == 'sea_surface_temperature'
insitu_matchup_result = list(generator)
validate_matchup_result(insitu_matchup_result, insitu_matchup=True)
# Test the satellite->satellite branch
# By mocking the getEndpointsByName function to return None we
# are forcing Matchup to think this dummy matchup dataset is
# satellite dataset
mock_edge_endpoints.return_value = None
# Open the edge response json. We want to convert these points
# to tile points so we can test sat to sat matchup
edge_json = json.load(
open(os.path.join(test_dir, 'edge_response.json')))
points = [
wkt.loads(result['point']) for result in edge_json['results']
]
matchup_tile = Tile()
matchup_tile.variables = [
TileVariable('sst', 'sea_surface_temperature')
]
matchup_tile.latitudes = np.array([point.y for point in points],
dtype=np.float32)
matchup_tile.longitudes = np.array([point.x for point in points],
dtype=np.float32)
matchup_tile.times = [edge_json['results'][0]['time']]
matchup_tile.data = np.array([[[10.0, 0, 0], [0, 20.0, 0],
[0, 0, 30.0]]])
matchup_tile.get_indices = lambda: [[0, 0, 0], [0, 1, 1], [0, 2, 2]]
matchup_tile.is_multi = False
match_args['tile_service_factory'](
).find_tiles_in_polygon.return_value = [matchup_tile]
generator = matchup.match_satellite_to_insitu(**match_args)
sat_matchup_result = list(generator)
validate_matchup_result(sat_matchup_result, insitu_matchup=False)