def test_merge_tiles_horizontal(self):
        tile1 = Tile()
        tile1.latitudes = np.ma.array([0.0, 1.0, 2.0, 3.0])
        tile1.longitudes = np.ma.array([0.0, -1.0, -2.0])
        tile1.times = np.ma.array([0])
        tile1.data = np.ma.arange(12).reshape((1, 4, 3))

        tile2 = Tile()
        tile2.latitudes = np.ma.array([0.0, 1.0, 2.0, 3.0])
        tile2.longitudes = np.ma.array([-3.0, -4.0, -5.0])
        tile2.times = np.ma.array([0])
        tile2.data = np.ma.arange(12, 24).reshape((1, 4, 3))

        from nexustiles.model.nexusmodel import merge_tiles

        times, lats, longs, data = merge_tiles([tile1, tile2])

        self.assertTrue(np.ma.allequal(times, np.array([0])))
        self.assertTrue(np.ma.allequal(lats, np.array([0.0, 1.0, 2.0, 3.0])))
        self.assertTrue(
            np.ma.allequal(longs, np.array([-5.0, -4.0, -3.0, -2.0, -1.0,
                                            0.0])))
        expected = np.ma.array(
            [[[14, 13, 12, 2, 1, 0], [17, 16, 15, 5, 4, 3],
              [20, 19, 18, 8, 7, 6], [23, 22, 21, 11, 10, 9]]],
            mask=False)
        self.assertTrue(
            np.ma.allequal(np.ma.getmaskarray(data),
                           np.ma.getmaskarray(expected)))
        self.assertTrue(np.ma.allequal(data, expected))
    def test_multiple_tiles_same_long(self):
        tile1 = Tile()
        tile1.latitudes = np.ma.array([0.0, 1.0, 2.0, 3.0])
        tile1.longitudes = np.ma.array([0.0, -1.0, -2.0])
        tile1.times = np.ma.array([0L])
        tile1.data = np.ma.arange(12).reshape((1, 4, 3))

        tile2 = Tile()
        tile2.latitudes = np.ma.array([4.0, 5.0, 6.0, 7.0])
        tile2.longitudes = np.ma.array([0.0, -1.0, -2.0])
        tile2.times = np.ma.array([0L])
        tile2.data = np.ma.arange(12, 24).reshape((1, 4, 3))

        self.assertAlmostEqual(1, get_approximate_value_for_lat_lon([tile1, tile2], 0.4, -1))
    def test_merge_tiles_overlapping(self):
        tile1 = Tile()
        tile1.latitudes = np.ma.array([0.0, 1.0, 2.0, 3.0])
        tile1.longitudes = np.ma.array([0.0, -1.0, -2.0])
        tile1.times = np.ma.array([0])
        tile1.data = np.ma.arange(12).reshape((1, 4, 3))

        tile2 = Tile()
        tile2.latitudes = np.ma.array([0.0, 1.0, 2.0, 3.0])
        tile2.longitudes = np.ma.array([0.0, -1.0, -2.0])
        tile2.times = np.ma.array([0])
        tile2.data = np.ma.arange(12, 24).reshape((1, 4, 3))

        from nexustiles.model.nexusmodel import merge_tiles

        self.assertRaises(Exception, lambda _: merge_tiles([tile1, tile2]))
Пример #4
0
def test_multi_variable_satellite_to_satellite_matchup(test_dir, test_tile,
                                                       test_matchup_args):
    """
    Test multi-variable satellite to satellite matchup functionality.
    """
    test_tile.latitudes = np.array([0, 20], dtype=np.float32)
    test_tile.longitudes = np.array([0, 20], dtype=np.float32)
    test_tile.times = [1627490285]
    test_tile.data = np.array([[[[1.10, 2.10], [3.10, 4.10]]],
                               [[[11.0, 21.0], [31.0, 41.0]]]])
    test_tile.is_multi = True
    test_tile.variables = [
        TileVariable('wind_speed', 'wind_speed'),
        TileVariable('wind_dir', 'wind_direction')
    ]
    test_matchup_args['tile_service_factory'] = setup_mock_tile_service(
        test_tile)

    with mock.patch(
            'webservice.algorithms_spark.Matchup.edge_endpoints.getEndpointByName'
    ) as mock_edge_endpoints:
        mock_edge_endpoints.return_value = None
        # Open the edge response json. We want to convert these points
        # to tile points so we can test sat to sat matchup
        edge_json = json.load(
            open(os.path.join(test_dir, 'edge_response.json')))
        points = [
            wkt.loads(result['point']) for result in edge_json['results']
        ]

        matchup_tile = Tile()
        matchup_tile.variables = [
            TileVariable('sst', 'sea_surface_temperature'),
            TileVariable('wind_dir', 'wind_direction')
        ]
        matchup_tile.latitudes = np.array([point.y for point in points],
                                          dtype=np.float32)
        matchup_tile.longitudes = np.array([point.x for point in points],
                                           dtype=np.float32)
        matchup_tile.times = [edge_json['results'][0]['time']]
        matchup_tile.data = np.array([[[[10.0, 0, 0], [0, 20.0, 0],
                                        [0, 0, 30.0]]],
                                      [[[100.0, 0, 0], [0, 200.0, 0],
                                        [0, 0, 300.0]]]])
        # matchup_tile.get_indices = lambda: [[0, 0, 0], [0, 1, 1], [0, 2, 2]]
        matchup_tile.is_multi = True

        test_matchup_args['tile_service_factory'](
        ).find_tiles_in_polygon.return_value = [matchup_tile]

        generator = matchup.match_satellite_to_insitu(**test_matchup_args)
        matchup_result = list(generator)
        assert len(matchup_result) == 2
        assert len(matchup_result[0]) == 2
        assert len(matchup_result[1]) == 2
        assert len(matchup_result[0][0].data) == 2
        assert len(matchup_result[0][1].data) == 2
        assert len(matchup_result[1][0].data) == 2
        assert len(matchup_result[1][1].data) == 2
    def test_masked_tile(self):
        tile = Tile()
        tile.bbox = BBox(30.5, 37.5, -51.5, -36.5)
        tile.latitudes = np.ma.arange(30.5, 38.5, 1.0)
        tile.longitudes = np.ma.arange(-51.5, -35.5, 1.0)
        tile.times = np.ma.array([0])
        tile.data = np.ma.arange(128.0).reshape((1, 8, 16))

        # tile.latitudes [ 30.5  31.5  32.5  33.5  34.5  35.5  36.5  37.5]
        tile.latitudes = np.ma.masked_outside(tile.latitudes, 35, 45)
        # tile.latitudes [-- -- -- -- -- 35.5 36.5 37.5]

        # tile.longitudes [-51.5 -50.5 -49.5 -48.5 -47.5 -46.5 -45.5 -44.5 -43.5 -42.5 -41.5 -40.5 -39.5  -38.5 -37.5 -36.5]
        tile.longitudes = np.ma.masked_outside(tile.longitudes, -50, -40)
        # tile.longitudes [-- -- -49.5 -48.5 -47.5 -46.5 -45.5 -44.5 -43.5 -42.5 -41.5 -40.5 -- -- -- --]

        # Tile no longer contains 35, -50
        self.assertFalse(tile.contains_point(35, -50))
Пример #6
0
    def setUp(self):
        tile = Tile()
        tile.bbox = BBox(-1.0, 1.0, -2.0, 2.0)
        tile.latitudes = np.ma.array([-1.0, -0.5, 0, .5, 1.0])
        tile.longitudes = np.ma.array([-2.0, -1.0, 0, 1.0, 2.0])
        tile.times = np.ma.array([0L])
        tile.data = np.ma.arange(25.0).reshape((1, 5, 5))
        tile.meta_data = {"std": np.ma.arange(25.0).reshape((1, 5, 5))}

        attrs = {'find_tile_by_polygon_and_most_recent_day_of_year.return_value': [tile]}
        self.tile_service = Mock(spec=NexusTileService, **attrs)
    def test_lat_exact_lon_approx(self):
        tile = Tile()
        tile.bbox = BBox(-1.0, 1.0, -2.0, 2.0)
        tile.latitudes = np.ma.array([-1.0, -0.5, 0, .5, 1.0])
        tile.longitudes = np.ma.array([-2.0, -1.0, 0, 1.0, 2.0])
        tile.times = np.ma.array([0L])
        tile.data = np.ma.arange(25.0).reshape((1, 5, 5))

        #         -2   -1   0    1    2
        # -1.0 [[[0.   1.   2.   3.   4. ]
        # -0.5   [5.   6.   7.   8.   9. ]
        #  0.    [10.  11.  12.  13.  14.]
        #  0.5   [15.  16.  17.  18.  19.]
        #  1.0   [20.  21.  22.  23.  24.]]]

        self.assertAlmostEqual(18.0, get_approximate_value_for_lat_lon([tile], 0.5, 1.01))
    def test_lon_greater_than_bounds(self):
        tile = Tile()
        tile.bbox = BBox(-1.0, 1.0, -2.0, 2.0)
        tile.latitudes = np.ma.array([-1.0, -0.5, 0, .5, 1.0])
        tile.longitudes = np.ma.array([-2.0, -1.0, 0, 1.0, 2.0])
        tile.times = np.ma.array([0L])
        tile.data = np.ma.arange(25.0).reshape((1, 5, 5))

        #         -2   -1   0    1    2
        # -1.0 [[[0.   1.   2.   3.   4. ]
        # -0.5   [5.   6.   7.   8.   9. ]
        #  0.    [10.  11.  12.  13.  14.]
        #  0.5   [15.  16.  17.  18.  19.]
        #  1.0   [20.  21.  22.  23.  24.]]]

        self.assertTrue(np.isnan(get_approximate_value_for_lat_lon([tile], 0, 3)))
    def test_update_tile_stats(self):
        tile = Tile()
        tile.bbox = BBox(-1.0, 1.0, -2.0, 2.0)
        tile.latitudes = np.ma.array([-1.0, -0.5, 0, .5, 1.0])
        tile.longitudes = np.ma.array([-2.0, -1.0, 0, 1.0, 2.0])
        tile.times = np.ma.array([0])
        tile.data = np.ma.arange(25.0).reshape((1, 5, 5))

        #         -2   -1   0    1    2
        # -1.0 [[[0.   1.   2.   3.   4. ]
        # -0.5   [5.   6.   7.   8.   9. ]
        #  0.    [10.  11.  12.  13.  14.]
        #  0.5   [15.  16.  17.  18.  19.]
        #  1.0   [20.  21.  22.  23.  24.]]]

        tile.update_stats()

        self.assertAlmostEqual(0.0, tile.tile_stats.min)
        self.assertAlmostEqual(24.0, tile.tile_stats.max)
        self.assertAlmostEqual(12.0, tile.tile_stats.mean)
        self.assertEqual(25, tile.tile_stats.count)
Пример #10
0
def test_match_satellite_to_insitu(test_dir, test_tile, test_matchup_args):
    """
    Test the test_match_satellite_to_insitu and ensure the matchup is
    done as expected, where the tile points and in-situ points are all
    known and the expected matchup points have been hand-calculated.

    This test case mocks out all external dependencies, so Solr,
    Cassandra, HTTP insitu requests, etc are all mocked.

    The test points are as follows:

    X (0, 20)                         X (20, 20)

            O (5, 15)



                     O (10, 10)




                             O (18, 3)

    X (0, 0)                        X (20, 0)

    The 'X' points are the primary satellite points and the 'O' points
    are the secondary satellite or insitu points

    Visual inspection reveals that primary point (0, 20) should match
    with secondary point (5, 15) and primary point (20, 0) should match
    with (18, 3)
    """
    test_tile.variables = [TileVariable('sst', 'sea_surface_temperature')]
    test_tile.latitudes = np.array([0, 20], dtype=np.float32)
    test_tile.longitudes = np.array([0, 20], dtype=np.float32)
    test_tile.times = [1627490285]
    test_tile.data = np.array([[[11.0, 21.0], [31.0, 41.0]]])
    test_tile.get_indices = lambda: [[0, 0, 0], [0, 0, 1], [0, 1, 0],
                                     [0, 1, 1]]
    test_tile.is_multi = False

    tile_ids = [1]
    polygon_wkt = 'POLYGON((-34.98 29.54, -30.1 29.54, -30.1 31.00, -34.98 31.00, -34.98 29.54))'
    primary_ds_name = 'primary-ds-name'
    matchup_ds_names = 'test'
    parameter = 'sst'
    depth_min = 0.0
    depth_max = 1.0
    time_tolerance = 3.0
    radius_tolerance = 1000000.0
    platforms = '1,2,3,4,5,6,7,8,9'

    with mock.patch(
            'webservice.algorithms_spark.Matchup.edge_endpoints.getEndpointByName'
    ) as mock_edge_endpoints:
        # Test the satellite->insitu branch
        # By mocking the getEndpointsByName function we are forcing
        # Matchup to think this dummy matchup dataset is an insitu
        # dataset
        mock_edge_endpoints.return_value = {'url': 'http://test-edge-url'}
        matchup.query_edge = lambda *args, **kwargs: json.load(
            open(os.path.join(test_dir, 'edge_response.json')))

        match_args = dict(
            tile_ids=tile_ids,
            primary_b=MockSparkParam(primary_ds_name),
            matchup_b=MockSparkParam(matchup_ds_names),
            parameter_b=MockSparkParam(parameter),
            tt_b=MockSparkParam(time_tolerance),
            rt_b=MockSparkParam(radius_tolerance),
            platforms_b=MockSparkParam(platforms),
            bounding_wkt_b=MockSparkParam(polygon_wkt),
            depth_min_b=MockSparkParam(depth_min),
            depth_max_b=MockSparkParam(depth_max),
            tile_service_factory=setup_mock_tile_service(test_tile))

        generator = matchup.match_satellite_to_insitu(**match_args)

        def validate_matchup_result(matchup_result, insitu_matchup):
            """
            The matchup results for satellite->insitu vs
            satellite->satellite are almost exactly the same so they
            can be validated using the same logic. They are the same
            because they represent the same data, except one test is in
            tile format (sat to sat) and one is in edge point format
            (insitu). The only difference is the data field is different
            for satellite data.
            """
            # There should be two primary matchup points
            assert len(matchup_result) == 2
            # Each primary point matched with 1 matchup point
            assert len(matchup_result[0]) == 2
            assert len(matchup_result[1]) == 2
            # Check that the satellite point was matched to the expected secondary point
            assert matchup_result[0][1].latitude == 3.0
            assert matchup_result[0][1].longitude == 18.0
            assert matchup_result[1][1].latitude == 15.0
            assert matchup_result[1][1].longitude == 5.0
            # Check that the secondary points have the expected values
            if insitu_matchup:
                assert matchup_result[0][1].data[0].variable_value == 30.0
                assert matchup_result[1][1].data[0].variable_value == 10.0
                assert matchup_result[0][1].data[
                    0].variable_name == 'sea_water_temperature'
                assert matchup_result[1][1].data[
                    0].variable_name == 'sea_water_temperature'
            else:
                assert matchup_result[0][1].data[0].variable_value == 30.0
                assert matchup_result[1][1].data[0].variable_value == 10.0
                assert matchup_result[0][1].data[0].variable_name == 'sst'
                assert matchup_result[0][1].data[
                    0].cf_variable_name == 'sea_surface_temperature'
                assert matchup_result[1][1].data[0].variable_name == 'sst'
                assert matchup_result[1][1].data[
                    0].cf_variable_name == 'sea_surface_temperature'
            # Check that the satellite points have the expected values
            assert matchup_result[0][0].data[0].variable_value == 21.0
            assert matchup_result[1][0].data[0].variable_value == 31.0
            assert matchup_result[0][0].data[0].variable_name == 'sst'
            assert matchup_result[0][0].data[
                0].cf_variable_name == 'sea_surface_temperature'
            assert matchup_result[1][0].data[0].variable_name == 'sst'
            assert matchup_result[1][0].data[
                0].cf_variable_name == 'sea_surface_temperature'

        insitu_matchup_result = list(generator)
        validate_matchup_result(insitu_matchup_result, insitu_matchup=True)

        # Test the satellite->satellite branch
        # By mocking the getEndpointsByName function to return None we
        # are forcing Matchup to think this dummy matchup dataset is
        # satellite dataset
        mock_edge_endpoints.return_value = None

        # Open the edge response json. We want to convert these points
        # to tile points so we can test sat to sat matchup
        edge_json = json.load(
            open(os.path.join(test_dir, 'edge_response.json')))
        points = [
            wkt.loads(result['point']) for result in edge_json['results']
        ]

        matchup_tile = Tile()
        matchup_tile.variables = [
            TileVariable('sst', 'sea_surface_temperature')
        ]
        matchup_tile.latitudes = np.array([point.y for point in points],
                                          dtype=np.float32)
        matchup_tile.longitudes = np.array([point.x for point in points],
                                           dtype=np.float32)
        matchup_tile.times = [edge_json['results'][0]['time']]
        matchup_tile.data = np.array([[[10.0, 0, 0], [0, 20.0, 0],
                                       [0, 0, 30.0]]])
        matchup_tile.get_indices = lambda: [[0, 0, 0], [0, 1, 1], [0, 2, 2]]
        matchup_tile.is_multi = False

        match_args['tile_service_factory'](
        ).find_tiles_in_polygon.return_value = [matchup_tile]

        generator = matchup.match_satellite_to_insitu(**match_args)

        sat_matchup_result = list(generator)
        validate_matchup_result(sat_matchup_result, insitu_matchup=False)