def test_pressure_constraint_in_4d(self): from cis.collocation.col_implementations import SepConstraintKdtree import datetime as dt import numpy as np ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_point = pd.Series({ 'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0], 'air_pressure': [24.0], 'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))] }) constraint = SepConstraintKdtree(p_sep=2) # This should leave us with 20 points: [ 6. 7. 8. 9. 10.] # [ 11. 12. 13. 14. 15.] # [ 16. 17. 18. 19. 20.] # [ 21. 22. 23. 24. 25.] ref_vals = np.array([ 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., 17., 18., 19., 20., 21., 22., 23., 24., 25. ]) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_alt_constraint_in_4d(self): from cis.collocation.col_implementations import SepConstraintKdtree import datetime as dt import numpy as np ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_point = pd.Series({ 'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0], 'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))] }) # 15m altitude separation a_sep = 15 constraint = SepConstraintKdtree(a_sep=a_sep) # This should leave us with 15 points: [ 21. 22. 23. 24. 25.] # [ 26. 27. 28. 29. 30.] # [ 31. 32. 33. 34. 35.] ref_vals = np.array([ 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., 35. ]) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_time_constraint_in_4d(self): from cis.collocation.col_implementations import SepConstraintKdtree import datetime as dt import numpy as np ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_point = pd.Series({ 'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0], 'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))] }) # 1 day (and a little bit) time seperation constraint = SepConstraintKdtree(t_sep='P1dT1M') # This should leave us with 30 points ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten() new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_for_same_3d_grids_returns_original_data(self): # Create sample and data cubes that include a time coordinate with the dimensions in reverse of normal order. sample_cube = gridded_data.make_from_cube( mock.make_mock_cube(lat_dim_length=5, lon_dim_length=3, time_dim_length=2, dim_order=["time", "lon", "lat"]) ) data_cube = gridded_data.make_from_cube( mock.make_mock_cube(lat_dim_length=5, lon_dim_length=3, time_dim_length=2, dim_order=["time", "lon", "lat"]) ) data_points = data_cube.get_non_masked_points() sample_points = sample_cube.get_all_points() coord_map = make_coord_map(sample_cube, data_cube) # Make separation constraint small enough to include only the corresponding point in the data cube. constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index for idx, sample_point in enumerate(sample_points): out_points = constraint.constrain_points(sample_point, data_points) # Two times for each spatial position. assert len(out_points) == 2 assert data_points[idx].val[0] in [p.val[0] for p in out_points]
def test_horizontal_constraint_in_2d_with_missing_values(self): ug_data = mock.make_regular_2d_ungridded_data_with_missing_values() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=0) coord_map = None # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() sample_points = None # Not used index.index_data(sample_points, ug_data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index for i, sample_point in ug_data.as_data_frame(time_index=False, name='vals').iterrows(): new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals if np.isnan(sample_point.vals): ref_vals = np.array([]) else: ref_vals = np.array([sample_point.vals]) eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_all_constraints_in_4d(self): ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_point = pd.DataFrame(data={'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0], 'air_pressure': [50.0], 'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]}).iloc[0] # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction h_sep = 1000 # 15m altitude separation a_sep = 15 # 1 day (and a little bit) time separation t_sep = 'P1dT1M' # Pressure constraint is 50/40 < p_sep < 60/50 p_sep = 1.22 constraint = SepConstraintKdtree(h_sep=h_sep, a_sep=a_sep, p_sep=p_sep, t_sep=t_sep) index = HaversineDistanceKDTreeIndex() index.index_data(None, ug_data_points, None) constraint.haversine_distance_kd_tree_index = index # This should leave us with 9 points: [[ 22, 23, 24] # [ 27, 28, 29] # [ 32, 33, 34]] ref_vals = np.array([27., 28., 29., 32., 33., 34.]) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = np.sort(new_points.vals) eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_all_constraints_in_4d(self): ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.get_non_masked_points() sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, pres=50.0, t=dt.datetime(1984, 8, 29)) # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction h_sep = 1000 # 15m altitude separation a_sep = 15 # 1 day (and a little bit) time separation t_sep = "P1dT1M" # Pressure constraint is 50/40 < p_sep < 60/50 p_sep = 1.22 constraint = SepConstraintKdtree(h_sep=h_sep, a_sep=a_sep, p_sep=p_sep, t_sep=t_sep) index = HaversineDistanceKDTreeIndex() index.index_data(None, ug_data_points, None) constraint.haversine_distance_kd_tree_index = index # This should leave us with 9 points: [[ 22, 23, 24] # [ 27, 28, 29] # [ 32, 33, 34]] ref_vals = np.array([27.0, 28.0, 29.0, 32.0, 33.0, 34.0]) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = np.sort(new_points.vals) eq_(ref_vals.size, new_vals.size) assert np.equal(ref_vals, new_vals).all()
def test_horizontal_constraint_in_4d(self): ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_points = pd.DataFrame( data={ 'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0], 'time': [dt.datetime(1984, 8, 29)] }) coord_map = None # Constraint distance selects the central three points. constraint = SepConstraintKdtree(h_sep=1000) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, ug_data_points, coord_map) constraint.haversine_distance_kd_tree_index = index # This should leave us with 30 points ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten() new_points = constraint.constrain_points(sample_points.iloc[0], ug_data_points) new_vals = np.sort(new_points.vals) eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_pressure_constraint_in_4d(self): from cis.collocation.col_implementations import SepConstraintKdtree import datetime as dt import numpy as np ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.get_non_masked_points() sample_point = HyperPoint(0.0, 0.0, 50.0, 24.0, dt.datetime(1984, 8, 29)) constraint = SepConstraintKdtree(p_sep=2) # This should leave us with 20 points: [ 6. 7. 8. 9. 10.] # [ 11. 12. 13. 14. 15.] # [ 16. 17. 18. 19. 20.] # [ 21. 22. 23. 24. 25.] ref_vals = np.array( [ 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, ] ) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals eq_(ref_vals.size, new_vals.size) assert np.equal(ref_vals, new_vals).all()
def test_all_constraints_in_4d(self): ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_point = pd.DataFrame( data={ 'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0], 'air_pressure': [50.0], 'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))] }).iloc[0] # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction h_sep = 1000 # 15m altitude separation a_sep = 15 # 1 day (and a little bit) time separation t_sep = 'P1dT1M' # Pressure constraint is 50/40 < p_sep < 60/50 p_sep = 1.22 constraint = SepConstraintKdtree(h_sep=h_sep, a_sep=a_sep, p_sep=p_sep, t_sep=t_sep) index = HaversineDistanceKDTreeIndex() index.index_data(None, ug_data_points, None) constraint.haversine_distance_kd_tree_index = index # This should leave us with 9 points: [[ 22, 23, 24] # [ 27, 28, 29] # [ 32, 33, 34]] ref_vals = np.array([27., 28., 29., 32., 33., 34.]) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = np.sort(new_points.vals) eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_time_constraint_in_4d(self): from cis.collocation.col_implementations import SepConstraintKdtree import datetime as dt import numpy as np ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.get_non_masked_points() sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, t=dt.datetime(1984, 8, 29)) # 1 day (and a little bit) time seperation constraint = SepConstraintKdtree(t_sep="P1dT1M") # This should leave us with 30 points ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten() new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals eq_(ref_vals.size, new_vals.size) assert np.equal(ref_vals, new_vals).all()
def test_horizontal_constraint_for_same_2d_grids_returns_original_data(self): # Simple case of lat/lon grid with dimensions in that order. sample_cube = gridded_data.make_from_cube(mock.make_mock_cube()) data_cube = gridded_data.make_from_cube(mock.make_mock_cube()) data_points = data_cube.get_non_masked_points() sample_points = sample_cube.get_all_points() coord_map = make_coord_map(sample_cube, data_cube) # Make separation constraint small enough to include only the corresponding point in the data cube. constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index for idx, sample_point in enumerate(sample_points): out_points = constraint.constrain_points(sample_point, data_points) assert len(out_points) == 1 assert out_points[0].val[0] == data_points[idx].val[0]
def test_time_constraint_in_4d(self): from cis.collocation.col_implementations import SepConstraintKdtree import datetime as dt import numpy as np ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_point = pd.Series({'longitude': [0.0], 'latitude': [0.0], 'altitude':[50.0], 'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]}) # 1 day (and a little bit) time seperation constraint = SepConstraintKdtree(t_sep='P1dT1M') # This should leave us with 30 points ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten() new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_4d(self): ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.get_non_masked_points() sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, t=dt.datetime(1984, 8, 29)) sample_points = HyperPointList([sample_point]) coord_map = None # Constraint distance selects the central three points. constraint = SepConstraintKdtree(h_sep=1000) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, ug_data_points, coord_map) constraint.haversine_distance_kd_tree_index = index # This should leave us with 30 points ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten() new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = np.sort(new_points.vals) eq_(ref_vals.size, new_vals.size) assert np.equal(ref_vals, new_vals).all()
def test_horizontal_constraint_in_4d(self): ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_points = pd.DataFrame(data={'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0], 'time': [dt.datetime(1984, 8, 29)]}) coord_map = None # Constraint distance selects the central three points. constraint = SepConstraintKdtree(h_sep=1000) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, ug_data_points, coord_map) constraint.haversine_distance_kd_tree_index = index # This should leave us with 30 points ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten() new_points = constraint.constrain_points(sample_points.iloc[0], ug_data_points) new_vals = np.sort(new_points.vals) eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_2d(self): ug_data = mock.make_regular_2d_ungridded_data() ug_data_points = ug_data.get_non_masked_points() sample_point = HyperPoint(lat=7.5, lon=-2.5) sample_points = HyperPointList([sample_point]) coord_map = None # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, ug_data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index # This should leave us with 4 points ref_vals = np.array([10, 11, 13, 14]) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals eq_(ref_vals.size, new_vals.size) assert np.equal(ref_vals, new_vals).all()
def test_pressure_constraint_in_4d(self): from cis.collocation.col_implementations import SepConstraintKdtree import datetime as dt import numpy as np ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_point = pd.Series({'longitude': [0.0], 'latitude': [0.0], 'altitude':[50.0], 'air_pressure': [24.0], 'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]}) constraint = SepConstraintKdtree(p_sep=2) # This should leave us with 20 points: [ 6. 7. 8. 9. 10.] # [ 11. 12. 13. 14. 15.] # [ 16. 17. 18. 19. 20.] # [ 21. 22. 23. 24. 25.] ref_vals = np.array([6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., 17., 18., 19., 20., 21., 22., 23., 24., 25.]) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_2d_with_missing_values(self): ug_data = mock.make_regular_2d_ungridded_data_with_missing_values() ug_data_points = ug_data.get_non_masked_points() coord_map = None # One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees # in each direction constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() sample_points = None # Not used index.index_data(sample_points, ug_data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index for sample_point in ug_data.get_all_points(): new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals if sample_point.val[0] is np.ma.masked: ref_vals = np.array([]) else: ref_vals = np.array([sample_point.val]) eq_(ref_vals.size, new_vals.size) assert np.equal(ref_vals, new_vals).all()
def test_alt_constraint_in_4d(self): from cis.collocation.col_implementations import SepConstraintKdtree import datetime as dt import numpy as np ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.get_non_masked_points() sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, t=dt.datetime(1984, 8, 29)) # 15m altitude separation a_sep = 15 constraint = SepConstraintKdtree(a_sep=a_sep) # This should leave us with 15 points: [ 21. 22. 23. 24. 25.] # [ 26. 27. 28. 29. 30.] # [ 31. 32. 33. 34. 35.] ref_vals = np.array([21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, 33.0, 34.0, 35.0]) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals eq_(ref_vals.size, new_vals.size) assert np.equal(ref_vals, new_vals).all()
def test_horizontal_constraint_in_2d_with_missing_values(self): # Test with standard 2d grids but with missing data. sample_cube = gridded_data.make_from_cube(mock.make_mock_cube()) data_cube = gridded_data.make_from_cube(mock.make_square_5x3_2d_cube_with_missing_data()) data_points = data_cube.get_non_masked_points() sample_points = sample_cube.get_all_points() coord_map = make_coord_map(sample_cube, data_cube) # Make separation constraint small enough to include only the corresponding point in the data cube. constraint = SepConstraintKdtree(h_sep=400) index = HaversineDistanceKDTreeIndex() index.index_data(sample_points, data_points, coord_map, leafsize=2) constraint.haversine_distance_kd_tree_index = index for idx, sample_point in enumerate(sample_points): out_points = constraint.constrain_points(sample_point, data_points) if data_points[idx].val[0] is np.ma.masked: assert len(out_points) == 0 else: assert len(out_points) == 1 assert out_points[0].val[0] == data_points[idx].val[0]
def test_alt_constraint_in_4d(self): from cis.collocation.col_implementations import SepConstraintKdtree import datetime as dt import numpy as np ug_data = mock.make_regular_4d_ungridded_data() ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1) sample_point = pd.Series({'longitude': [0.0], 'latitude': [0.0], 'altitude':[50.0], 'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]}) # 15m altitude separation a_sep = 15 constraint = SepConstraintKdtree(a_sep=a_sep) # This should leave us with 15 points: [ 21. 22. 23. 24. 25.] # [ 26. 27. 28. 29. 30.] # [ 31. 32. 33. 34. 35.] ref_vals = np.array([21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., 35.]) new_points = constraint.constrain_points(sample_point, ug_data_points) new_vals = new_points.vals eq_(ref_vals.size, new_vals.size) assert (np.equal(ref_vals, new_vals).all())