def test_all_constraints_in_4d(self): ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.get_non_masked_points() sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, pres=50.0, t=dt.datetime(1984, 8, 29)) # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction h_sep = 1000 # 15m altitude separation a_sep = 15 # 1 day (and a little bit) time separation t_sep = "P1dT1M" # Pressure constraint is 50/40 < p_sep < 60/50 p_sep = 1.22 constraint = SepConstraintKdtree(h_sep=h_sep, a_sep=a_sep, p_sep=p_sep, t_sep=t_sep) index = HaversineDistanceKDTreeIndex() index.index_data(None, ug_data_points, None) constraint.haversine_distance_kd_tree_index = index # This should leave us with 9 points: [[ 22, 23, 24] # [ 27, 28, 29] # [ 32, 33, 34]] ref_vals = np.array([27.0, 28.0, 29.0, 32.0, 33.0, 34.0]) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = np.sort(new_points.vals) eq_(ref_vals.size, new_vals.size) assert np.equal(ref_vals, new_vals).all()
def test_horizontal_constraint_in_2d_with_missing_values(self): ug_data = mock.make_regular_2d_ungridded_data_with_missing_values() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=0) coord_map = None # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() sample_points = None # Not used index.index_data(sample_points, ug_data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index for i, sample_point in ug_data.as_data_frame(time_index=False, name='vals').iterrows(): new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals if np.isnan(sample_point.vals): ref_vals = np.array([]) else: ref_vals = np.array([sample_point.vals]) eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_2d(self): from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import Metadata ug_data = mock.make_regular_2d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_points = UngriddedData(np.array([0.0]), Metadata(), CoordList([Coord(np.array([7.5]), Metadata(standard_name='latitude')), Coord(np.array([-2.5]), Metadata(standard_name='longitude'))])) sample_points_view = sample_points.as_data_frame(time_index=False, name='vals').dropna(axis=1) # sample_point = HyperPoint(lat=7.5, lon=-2.5) # sample_points = HyperPointList([sample_point]) coord_map = None # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, ug_data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index # This should leave us with 4 points ref_vals = np.array([10, 11, 13, 14]) indices = constraint.haversine_distance_kd_tree_index.find_points_within_distance_sample(sample_points_view, 400) new_vals = ug_data.data.flat[indices] eq_(ref_vals.size, len(new_vals[0])) assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_4d(self): ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_points = pd.DataFrame( data={ 'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0], 'time': [dt.datetime(1984, 8, 29)] }) coord_map = None # Constraint distance selects the central three points. constraint = SepConstraintKdtree(h_sep=1000) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, ug_data_points, coord_map) constraint.haversine_distance_kd_tree_index = index # This should leave us with 30 points ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten() new_points = constraint.constrain_points(sample_points.iloc[0], ug_data_points) new_vals = np.sort(new_points.vals) eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_all_constraints_in_4d(self): ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_point = pd.DataFrame(data={'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0], 'air_pressure': [50.0], 'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]}).iloc[0] # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction h_sep = 1000 # 15m altitude separation a_sep = 15 # 1 day (and a little bit) time separation t_sep = 'P1dT1M' # Pressure constraint is 50/40 < p_sep < 60/50 p_sep = 1.22 constraint = SepConstraintKdtree(h_sep=h_sep, a_sep=a_sep, p_sep=p_sep, t_sep=t_sep) index = HaversineDistanceKDTreeIndex() index.index_data(None, ug_data_points, None) constraint.haversine_distance_kd_tree_index = index # This should leave us with 9 points: [[ 22, 23, 24] # [ 27, 28, 29] # [ 32, 33, 34]] ref_vals = np.array([27., 28., 29., 32., 33., 34.]) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = np.sort(new_points.vals) eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_for_same_3d_grids_returns_original_data(self): # Create sample and data cubes that include a time coordinate with the dimensions in reverse of normal order. sample_cube = gridded_data.make_from_cube( mock.make_mock_cube(lat_dim_length=5, lon_dim_length=3, time_dim_length=2, dim_order=["time", "lon", "lat"]) ) data_cube = gridded_data.make_from_cube( mock.make_mock_cube(lat_dim_length=5, lon_dim_length=3, time_dim_length=2, dim_order=["time", "lon", "lat"]) ) data_points = data_cube.get_non_masked_points() sample_points = sample_cube.get_all_points() coord_map = make_coord_map(sample_cube, data_cube) # Make separation constraint small enough to include only the corresponding point in the data cube. constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index for idx, sample_point in enumerate(sample_points): out_points = constraint.constrain_points(sample_point, data_points) # Two times for each spatial position. assert len(out_points) == 2 assert data_points[idx].val[0] in [p.val[0] for p in out_points]
def test_horizontal_constraint_in_2d_when_lats_are_the_same_produces_a_balanced_tree(self): ug_data = mock.make_regular_2d_ungridded_data(lat_dim_length=1001, lat_max=10, lat_min=10) ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_points = pd.DataFrame(data={'longitude': [-2.5], 'latitude': [7.5]}) coord_map = None # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, ug_data_points, coord_map, leafsize=2) depth = self.get_max_depth(index.index.tree, 0) assert_that(depth, is_(2), "Depth is 2, there are three unique values -10, 0, 10")
def test_horizontal_constraint_in_2d_when_lats_are_the_same_produces_a_balanced_tree(self): ug_data = mock.make_regular_2d_ungridded_data(lat_dim_length=1001, lat_max=10, lat_min=10) ug_data_points = ug_data.get_non_masked_points() sample_point = HyperPoint(lat=7.5, lon=-2.5) sample_points = HyperPointList([sample_point]) coord_map = None # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, ug_data_points, coord_map, leafsize=2) depth = self.get_max_depth(index.index.tree, 0) assert_that(depth, is_(2), "Depth is 2, there are three unique values -10, 0, 10")
def test_all_constraints_in_4d(self): ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_point = pd.DataFrame( data={ 'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0], 'air_pressure': [50.0], 'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))] }).iloc[0] # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction h_sep = 1000 # 15m altitude separation a_sep = 15 # 1 day (and a little bit) time separation t_sep = 'P1dT1M' # Pressure constraint is 50/40 < p_sep < 60/50 p_sep = 1.22 constraint = SepConstraintKdtree(h_sep=h_sep, a_sep=a_sep, p_sep=p_sep, t_sep=t_sep) index = HaversineDistanceKDTreeIndex() index.index_data(None, ug_data_points, None) constraint.haversine_distance_kd_tree_index = index # This should leave us with 9 points: [[ 22, 23, 24] # [ 27, 28, 29] # [ 32, 33, 34]] ref_vals = np.array([27., 28., 29., 32., 33., 34.]) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = np.sort(new_points.vals) eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_for_same_2d_grids_returns_original_data(self): # Simple case of lat/lon grid with dimensions in that order. sample_cube = gridded_data.make_from_cube(mock.make_mock_cube()) data_cube = gridded_data.make_from_cube(mock.make_mock_cube()) data_points = data_cube.get_non_masked_points() sample_points = sample_cube.get_all_points() coord_map = make_coord_map(sample_cube, data_cube) # Make separation constraint small enough to include only the corresponding point in the data cube. constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index for idx, sample_point in enumerate(sample_points): out_points = constraint.constrain_points(sample_point, data_points) assert len(out_points) == 1 assert out_points[0].val[0] == data_points[idx].val[0]
def test_horizontal_constraint_in_4d(self): ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_points = pd.DataFrame(data={'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0], 'time': [dt.datetime(1984, 8, 29)]}) coord_map = None # Constraint distance selects the central three points. constraint = SepConstraintKdtree(h_sep=1000) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, ug_data_points, coord_map) constraint.haversine_distance_kd_tree_index = index # This should leave us with 30 points ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten() new_points = constraint.constrain_points(sample_points.iloc[0], ug_data_points) new_vals = np.sort(new_points.vals) eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_4d(self): ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.get_non_masked_points() sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, t=dt.datetime(1984, 8, 29)) sample_points = HyperPointList([sample_point]) coord_map = None # Constraint distance selects the central three points. constraint = SepConstraintKdtree(h_sep=1000) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, ug_data_points, coord_map) constraint.haversine_distance_kd_tree_index = index # This should leave us with 30 points ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten() new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = np.sort(new_points.vals) eq_(ref_vals.size, new_vals.size) assert np.equal(ref_vals, new_vals).all()
def test_horizontal_constraint_in_2d(self): ug_data = mock.make_regular_2d_ungridded_data() ug_data_points = ug_data.get_non_masked_points() sample_point = HyperPoint(lat=7.5, lon=-2.5) sample_points = HyperPointList([sample_point]) coord_map = None # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, ug_data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index # This should leave us with 4 points ref_vals = np.array([10, 11, 13, 14]) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals eq_(ref_vals.size, new_vals.size) assert np.equal(ref_vals, new_vals).all()
def test_horizontal_constraint_in_2d_with_missing_values(self): ug_data = mock.make_regular_2d_ungridded_data_with_missing_values() ug_data_points = ug_data.get_non_masked_points() coord_map = None # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() sample_points = None # Not used index.index_data(sample_points, ug_data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index for sample_point in ug_data.get_all_points(): new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals if sample_point.val[0] is np.ma.masked: ref_vals = np.array([]) else: ref_vals = np.array([sample_point.val]) eq_(ref_vals.size, new_vals.size) assert np.equal(ref_vals, new_vals).all()
def test_horizontal_constraint_in_2d_when_lats_are_the_same_produces_a_balanced_tree( self): ug_data = mock.make_regular_2d_ungridded_data(lat_dim_length=1001, lat_max=10, lat_min=10) ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_points = pd.DataFrame(data={ 'longitude': [-2.5], 'latitude': [7.5] }) coord_map = None # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, ug_data_points, coord_map, leafsize=2) depth = self.get_max_depth(index.index.tree, 0) assert_that(depth, is_(2), "Depth is 2, there are three unique values -10, 0, 10")
def test_horizontal_constraint_in_2d(self): from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import Metadata ug_data = mock.make_regular_2d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_points = UngriddedData( np.array([0.0]), Metadata(), CoordList([ Coord(np.array([7.5]), Metadata(standard_name='latitude')), Coord(np.array([-2.5]), Metadata(standard_name='longitude')) ])) sample_points_view = sample_points.as_data_frame( time_index=False, name='vals').dropna(axis=1) # sample_point = HyperPoint(lat=7.5, lon=-2.5) # sample_points = HyperPointList([sample_point]) coord_map = None # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, ug_data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index # This should leave us with 4 points ref_vals = np.array([10, 11, 13, 14]) indices = constraint.haversine_distance_kd_tree_index.find_points_within_distance_sample( sample_points_view, 400) new_vals = ug_data.data.flat[indices] eq_(ref_vals.size, len(new_vals[0])) assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_2d_with_missing_values(self): # Test with standard 2d grids but with missing data. sample_cube = gridded_data.make_from_cube(mock.make_mock_cube()) data_cube = gridded_data.make_from_cube(mock.make_square_5x3_2d_cube_with_missing_data()) data_points = data_cube.get_non_masked_points() sample_points = sample_cube.get_all_points() coord_map = make_coord_map(sample_cube, data_cube) # Make separation constraint small enough to include only the corresponding point in the data cube. constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index for idx, sample_point in enumerate(sample_points): out_points = constraint.constrain_points(sample_point, data_points) if data_points[idx].val[0] is np.ma.masked: assert len(out_points) == 0 else: assert len(out_points) == 1 assert out_points[0].val[0] == data_points[idx].val[0]