def test_all_constraints_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, pres=50.0, t=dt.datetime(1984, 8, 29))
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
h_sep = 1000
# 15m altitude separation
a_sep = 15
# 1 day (and a little bit) time separation
t_sep = "P1dT1M"
# Pressure constraint is 50/40 < p_sep < 60/50
p_sep = 1.22
constraint = SepConstraintKdtree(h_sep=h_sep, a_sep=a_sep, p_sep=p_sep, t_sep=t_sep)
index = HaversineDistanceKDTreeIndex()
index.index_data(None, ug_data_points, None)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 9 points: [[ 22, 23, 24]
# [ 27, 28, 29]
# [ 32, 33, 34]]
ref_vals = np.array([27.0, 28.0, 29.0, 32.0, 33.0, 34.0])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = np.sort(new_points.vals)
eq_(ref_vals.size, new_vals.size)
assert np.equal(ref_vals, new_vals).all()
def test_horizontal_constraint_in_2d_with_missing_values(self):
ug_data = mock.make_regular_2d_ungridded_data_with_missing_values()
ug_data_points = ug_data.as_data_frame(time_index=False,
name='vals').dropna(axis=0)
coord_map = None
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
sample_points = None # Not used
index.index_data(sample_points, ug_data_points, coord_map, leafsize=2)
constraint.haversine_distance_kd_tree_index = index
for i, sample_point in ug_data.as_data_frame(time_index=False,
name='vals').iterrows():
new_points = constraint.constrain_points(sample_point,
ug_data_points)
new_vals = new_points.vals
if np.isnan(sample_point.vals):
ref_vals = np.array([])
else:
ref_vals = np.array([sample_point.vals])
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_2d(self):
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import Metadata
ug_data = mock.make_regular_2d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1)
sample_points = UngriddedData(np.array([0.0]), Metadata(),
CoordList([Coord(np.array([7.5]), Metadata(standard_name='latitude')),
Coord(np.array([-2.5]), Metadata(standard_name='longitude'))]))
sample_points_view = sample_points.as_data_frame(time_index=False, name='vals').dropna(axis=1)
# sample_point = HyperPoint(lat=7.5, lon=-2.5)
# sample_points = HyperPointList([sample_point])
coord_map = None
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, ug_data_points, coord_map, leafsize=2)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 4 points
ref_vals = np.array([10, 11, 13, 14])
indices = constraint.haversine_distance_kd_tree_index.find_points_within_distance_sample(sample_points_view, 400)
new_vals = ug_data.data.flat[indices]
eq_(ref_vals.size, len(new_vals[0]))
assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False,
name='vals').dropna(axis=1)
sample_points = pd.DataFrame(
data={
'longitude': [0.0],
'latitude': [0.0],
'altitude': [50.0],
'time': [dt.datetime(1984, 8, 29)]
})
coord_map = None
# Constraint distance selects the central three points.
constraint = SepConstraintKdtree(h_sep=1000)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, ug_data_points, coord_map)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 30 points
ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten()
new_points = constraint.constrain_points(sample_points.iloc[0],
ug_data_points)
new_vals = np.sort(new_points.vals)
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_all_constraints_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1)
sample_point = pd.DataFrame(data={'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0],
'air_pressure': [50.0],
'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]}).iloc[0]
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
h_sep = 1000
# 15m altitude separation
a_sep = 15
# 1 day (and a little bit) time separation
t_sep = 'P1dT1M'
# Pressure constraint is 50/40 < p_sep < 60/50
p_sep = 1.22
constraint = SepConstraintKdtree(h_sep=h_sep, a_sep=a_sep, p_sep=p_sep, t_sep=t_sep)
index = HaversineDistanceKDTreeIndex()
index.index_data(None, ug_data_points, None)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 9 points: [[ 22, 23, 24]
# [ 27, 28, 29]
# [ 32, 33, 34]]
ref_vals = np.array([27., 28., 29., 32., 33., 34.])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = np.sort(new_points.vals)
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_for_same_3d_grids_returns_original_data(self):
# Create sample and data cubes that include a time coordinate with the dimensions in reverse of normal order.
sample_cube = gridded_data.make_from_cube(
mock.make_mock_cube(lat_dim_length=5, lon_dim_length=3, time_dim_length=2, dim_order=["time", "lon", "lat"])
)
data_cube = gridded_data.make_from_cube(
mock.make_mock_cube(lat_dim_length=5, lon_dim_length=3, time_dim_length=2, dim_order=["time", "lon", "lat"])
)
data_points = data_cube.get_non_masked_points()
sample_points = sample_cube.get_all_points()
coord_map = make_coord_map(sample_cube, data_cube)
# Make separation constraint small enough to include only the corresponding point in the data cube.
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, data_points, coord_map, leafsize=2)
constraint.haversine_distance_kd_tree_index = index
for idx, sample_point in enumerate(sample_points):
out_points = constraint.constrain_points(sample_point, data_points)
# Two times for each spatial position.
assert len(out_points) == 2
assert data_points[idx].val[0] in [p.val[0] for p in out_points]
def test_horizontal_constraint_in_2d_when_lats_are_the_same_produces_a_balanced_tree(self):
ug_data = mock.make_regular_2d_ungridded_data(lat_dim_length=1001, lat_max=10, lat_min=10)
ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1)
sample_points = pd.DataFrame(data={'longitude': [-2.5], 'latitude': [7.5]})
coord_map = None
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, ug_data_points, coord_map, leafsize=2)
depth = self.get_max_depth(index.index.tree, 0)
assert_that(depth, is_(2), "Depth is 2, there are three unique values -10, 0, 10")
def test_horizontal_constraint_in_2d_when_lats_are_the_same_produces_a_balanced_tree(self):
ug_data = mock.make_regular_2d_ungridded_data(lat_dim_length=1001, lat_max=10, lat_min=10)
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=7.5, lon=-2.5)
sample_points = HyperPointList([sample_point])
coord_map = None
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, ug_data_points, coord_map, leafsize=2)
depth = self.get_max_depth(index.index.tree, 0)
assert_that(depth, is_(2), "Depth is 2, there are three unique values -10, 0, 10")
def test_all_constraints_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False,
name='vals').dropna(axis=1)
sample_point = pd.DataFrame(
data={
'longitude': [0.0],
'latitude': [0.0],
'altitude': [50.0],
'air_pressure': [50.0],
'time':
[cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]
}).iloc[0]
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
h_sep = 1000
# 15m altitude separation
a_sep = 15
# 1 day (and a little bit) time separation
t_sep = 'P1dT1M'
# Pressure constraint is 50/40 < p_sep < 60/50
p_sep = 1.22
constraint = SepConstraintKdtree(h_sep=h_sep,
a_sep=a_sep,
p_sep=p_sep,
t_sep=t_sep)
index = HaversineDistanceKDTreeIndex()
index.index_data(None, ug_data_points, None)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 9 points: [[ 22, 23, 24]
# [ 27, 28, 29]
# [ 32, 33, 34]]
ref_vals = np.array([27., 28., 29., 32., 33., 34.])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = np.sort(new_points.vals)
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_for_same_2d_grids_returns_original_data(self):
# Simple case of lat/lon grid with dimensions in that order.
sample_cube = gridded_data.make_from_cube(mock.make_mock_cube())
data_cube = gridded_data.make_from_cube(mock.make_mock_cube())
data_points = data_cube.get_non_masked_points()
sample_points = sample_cube.get_all_points()
coord_map = make_coord_map(sample_cube, data_cube)
# Make separation constraint small enough to include only the corresponding point in the data cube.
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, data_points, coord_map, leafsize=2)
constraint.haversine_distance_kd_tree_index = index
for idx, sample_point in enumerate(sample_points):
out_points = constraint.constrain_points(sample_point, data_points)
assert len(out_points) == 1
assert out_points[0].val[0] == data_points[idx].val[0]
def test_horizontal_constraint_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1)
sample_points = pd.DataFrame(data={'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0],
'time': [dt.datetime(1984, 8, 29)]})
coord_map = None
# Constraint distance selects the central three points.
constraint = SepConstraintKdtree(h_sep=1000)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, ug_data_points, coord_map)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 30 points
ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten()
new_points = constraint.constrain_points(sample_points.iloc[0], ug_data_points)
new_vals = np.sort(new_points.vals)
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, t=dt.datetime(1984, 8, 29))
sample_points = HyperPointList([sample_point])
coord_map = None
# Constraint distance selects the central three points.
constraint = SepConstraintKdtree(h_sep=1000)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, ug_data_points, coord_map)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 30 points
ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten()
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = np.sort(new_points.vals)
eq_(ref_vals.size, new_vals.size)
assert np.equal(ref_vals, new_vals).all()
def test_horizontal_constraint_in_2d(self):
ug_data = mock.make_regular_2d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=7.5, lon=-2.5)
sample_points = HyperPointList([sample_point])
coord_map = None
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, ug_data_points, coord_map, leafsize=2)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 4 points
ref_vals = np.array([10, 11, 13, 14])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert np.equal(ref_vals, new_vals).all()
def test_horizontal_constraint_in_2d_with_missing_values(self):
ug_data = mock.make_regular_2d_ungridded_data_with_missing_values()
ug_data_points = ug_data.get_non_masked_points()
coord_map = None
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
sample_points = None # Not used
index.index_data(sample_points, ug_data_points, coord_map, leafsize=2)
constraint.haversine_distance_kd_tree_index = index
for sample_point in ug_data.get_all_points():
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
if sample_point.val[0] is np.ma.masked:
ref_vals = np.array([])
else:
ref_vals = np.array([sample_point.val])
eq_(ref_vals.size, new_vals.size)
assert np.equal(ref_vals, new_vals).all()
def test_horizontal_constraint_in_2d_with_missing_values(self):
ug_data = mock.make_regular_2d_ungridded_data_with_missing_values()
ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=0)
coord_map = None
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
sample_points = None # Not used
index.index_data(sample_points, ug_data_points, coord_map, leafsize=2)
constraint.haversine_distance_kd_tree_index = index
for i, sample_point in ug_data.as_data_frame(time_index=False, name='vals').iterrows():
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
if np.isnan(sample_point.vals):
ref_vals = np.array([])
else:
ref_vals = np.array([sample_point.vals])
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_2d_when_lats_are_the_same_produces_a_balanced_tree(
self):
ug_data = mock.make_regular_2d_ungridded_data(lat_dim_length=1001,
lat_max=10,
lat_min=10)
ug_data_points = ug_data.as_data_frame(time_index=False,
name='vals').dropna(axis=1)
sample_points = pd.DataFrame(data={
'longitude': [-2.5],
'latitude': [7.5]
})
coord_map = None
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, ug_data_points, coord_map, leafsize=2)
depth = self.get_max_depth(index.index.tree, 0)
assert_that(depth, is_(2),
"Depth is 2, there are three unique values -10, 0, 10")
def test_horizontal_constraint_in_2d(self):
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import Metadata
ug_data = mock.make_regular_2d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False,
name='vals').dropna(axis=1)
sample_points = UngriddedData(
np.array([0.0]), Metadata(),
CoordList([
Coord(np.array([7.5]), Metadata(standard_name='latitude')),
Coord(np.array([-2.5]), Metadata(standard_name='longitude'))
]))
sample_points_view = sample_points.as_data_frame(
time_index=False, name='vals').dropna(axis=1)
# sample_point = HyperPoint(lat=7.5, lon=-2.5)
# sample_points = HyperPointList([sample_point])
coord_map = None
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, ug_data_points, coord_map, leafsize=2)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 4 points
ref_vals = np.array([10, 11, 13, 14])
indices = constraint.haversine_distance_kd_tree_index.find_points_within_distance_sample(
sample_points_view, 400)
new_vals = ug_data.data.flat[indices]
eq_(ref_vals.size, len(new_vals[0]))
assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_2d_with_missing_values(self):
# Test with standard 2d grids but with missing data.
sample_cube = gridded_data.make_from_cube(mock.make_mock_cube())
data_cube = gridded_data.make_from_cube(mock.make_square_5x3_2d_cube_with_missing_data())
data_points = data_cube.get_non_masked_points()
sample_points = sample_cube.get_all_points()
coord_map = make_coord_map(sample_cube, data_cube)
# Make separation constraint small enough to include only the corresponding point in the data cube.
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, data_points, coord_map, leafsize=2)
constraint.haversine_distance_kd_tree_index = index
for idx, sample_point in enumerate(sample_points):
out_points = constraint.constrain_points(sample_point, data_points)
if data_points[idx].val[0] is np.ma.masked:
assert len(out_points) == 0
else:
assert len(out_points) == 1
assert out_points[0].val[0] == data_points[idx].val[0]
