def test_pressure_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraint
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(0.0, 0.0, 50.0, 24.0,
dt.datetime(1984, 8, 29))
constraint = SepConstraint(p_sep=2)
# This should leave us with 20 points: [ 6. 7. 8. 9. 10.]
# [ 11. 12. 13. 14. 15.]
# [ 16. 17. 18. 19. 20.]
# [ 21. 22. 23. 24. 25.]
ref_vals = np.array([
6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., 17., 18., 19.,
20., 21., 22., 23., 24., 25.
])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_coordinates_outside_grid_in_col_ungridded_to_ungridded_in_2d(
self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, nn_pressure, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
sample_points = HyperPointList()
sample_points.append(
HyperPoint(lat=0.0,
lon=0.0,
pres=0.1,
t=dt.datetime(1984, 8, 29, 8, 34)))
sample_points.append(
HyperPoint(lat=0.0,
lon=0.0,
pres=91.0,
t=dt.datetime(1984, 9, 2, 1, 23)))
sample_points.append(
HyperPoint(lat=0.0,
lon=0.0,
pres=890.0,
t=dt.datetime(1984, 9, 4, 15, 54)))
sample_points = UngriddedData.from_points_array(sample_points)
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(),
nn_pressure())[0]
eq_(new_data.data[0], 1.0)
eq_(new_data.data[1], 46.0)
eq_(new_data.data[2], 46.0)
def test_pressure_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraintKdtree
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False,
name='vals').dropna(axis=1)
sample_point = pd.Series({
'longitude': [0.0],
'latitude': [0.0],
'altitude': [50.0],
'air_pressure': [24.0],
'time':
[cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]
})
constraint = SepConstraintKdtree(p_sep=2)
# This should leave us with 20 points: [ 6. 7. 8. 9. 10.]
# [ 11. 12. 13. 14. 15.]
# [ 16. 17. 18. 19. 20.]
# [ 21. 22. 23. 24. 25.]
ref_vals = np.array([
6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., 17., 18., 19.,
20., 21., 22., 23., 24., 25.
])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_all_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraint
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, pres=50.0, t=dt.datetime(1984, 8, 29))
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
h_sep = 1000
# 15m altitude seperation
a_sep = 15
# 1 day (and a little bit) time seperation
t_sep = 'P1DT1M'
# Pressure constraint is 50/40 < p_sep < 60/50
p_sep = 1.22
constraint = SepConstraint(h_sep=h_sep, a_sep=a_sep, p_sep=p_sep, t_sep=t_sep)
# This should leave us with 9 points: [[ 22, 23, 24]
# [ 27, 28, 29]
# [ 32, 33, 34]]
ref_vals = np.array([27., 28., 29., 32., 33., 34.])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_alt_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraintKdtree
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False,
name='vals').dropna(axis=1)
sample_point = pd.Series({
'longitude': [0.0],
'latitude': [0.0],
'altitude': [50.0],
'time':
[cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]
})
# 15m altitude separation
a_sep = 15
constraint = SepConstraintKdtree(a_sep=a_sep)
# This should leave us with 15 points: [ 21. 22. 23. 24. 25.]
# [ 26. 27. 28. 29. 30.]
# [ 31. 32. 33. 34. 35.]
ref_vals = np.array([
21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33.,
34., 35.
])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False,
name='vals').dropna(axis=1)
sample_points = pd.DataFrame(
data={
'longitude': [0.0],
'latitude': [0.0],
'altitude': [50.0],
'time': [dt.datetime(1984, 8, 29)]
})
coord_map = None
# Constraint distance selects the central three points.
constraint = SepConstraintKdtree(h_sep=1000)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, ug_data_points, coord_map)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 30 points
ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten()
new_points = constraint.constrain_points(sample_points.iloc[0],
ug_data_points)
new_vals = np.sort(new_points.vals)
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_alt_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraint
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=0.0,
lon=0.0,
alt=50.0,
t=dt.datetime(1984, 8, 29))
# 15m altitude seperation
a_sep = 15
constraint = SepConstraint(a_sep=a_sep)
# This should leave us with 15 points: [ 21. 22. 23. 24. 25.]
# [ 26. 27. 28. 29. 30.]
# [ 31. 32. 33. 34. 35.]
ref_vals = np.array([
21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33.,
34., 35.
])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_basic_col_in_4d(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, nn_altitude, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
sample_points = HyperPointList()
sample_points.append(
HyperPoint(lat=1.0,
lon=1.0,
alt=12.0,
t=dt.datetime(1984, 8, 29, 8, 34)))
sample_points.append(
HyperPoint(lat=4.0,
lon=4.0,
alt=34.0,
t=dt.datetime(1984, 9, 2, 1, 23)))
sample_points.append(
HyperPoint(lat=-4.0,
lon=-4.0,
alt=89.0,
t=dt.datetime(1984, 9, 4, 15, 54)))
sample_points = UngriddedData.from_points_array(sample_points)
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(),
nn_altitude())[0]
eq_(new_data.data[0], 6.0)
eq_(new_data.data[1], 16.0)
eq_(new_data.data[2], 46.0)
def test_all_constraints_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, pres=50.0, t=dt.datetime(1984, 8, 29))
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
h_sep = 1000
# 15m altitude separation
a_sep = 15
# 1 day (and a little bit) time separation
t_sep = "P1dT1M"
# Pressure constraint is 50/40 < p_sep < 60/50
p_sep = 1.22
constraint = SepConstraintKdtree(h_sep=h_sep, a_sep=a_sep, p_sep=p_sep, t_sep=t_sep)
index = HaversineDistanceKDTreeIndex()
index.index_data(None, ug_data_points, None)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 9 points: [[ 22, 23, 24]
# [ 27, 28, 29]
# [ 32, 33, 34]]
ref_vals = np.array([27.0, 28.0, 29.0, 32.0, 33.0, 34.0])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = np.sort(new_points.vals)
eq_(ref_vals.size, new_vals.size)
assert np.equal(ref_vals, new_vals).all()
def test_all_constraints_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1)
sample_point = pd.DataFrame(data={'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0],
'air_pressure': [50.0],
'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]}).iloc[0]
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
h_sep = 1000
# 15m altitude separation
a_sep = 15
# 1 day (and a little bit) time separation
t_sep = 'P1dT1M'
# Pressure constraint is 50/40 < p_sep < 60/50
p_sep = 1.22
constraint = SepConstraintKdtree(h_sep=h_sep, a_sep=a_sep, p_sep=p_sep, t_sep=t_sep)
index = HaversineDistanceKDTreeIndex()
index.index_data(None, ug_data_points, None)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 9 points: [[ 22, 23, 24]
# [ 27, 28, 29]
# [ 32, 33, 34]]
ref_vals = np.array([27., 28., 29., 32., 33., 34.])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = np.sort(new_points.vals)
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_time_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraintKdtree
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False,
name='vals').dropna(axis=1)
sample_point = pd.Series({
'longitude': [0.0],
'latitude': [0.0],
'altitude': [50.0],
'time':
[cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]
})
# 1 day (and a little bit) time seperation
constraint = SepConstraintKdtree(t_sep='P1dT1M')
# This should leave us with 30 points
ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten()
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_basic_col_with_incompatible_points_throws_a_TypeError(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, nn_pressure, SepConstraintKdtree
ug_data = mock.make_regular_4d_ungridded_data()
# Make sample points with no time dimension specified
sample_points = UngriddedData.from_points_array(
[HyperPoint(1.0, 1.0), HyperPoint(4.0, 4.0), HyperPoint(-4.0, -4.0)])
col = GeneralUngriddedCollocator()
with self.assertRaises(AttributeError):
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(), nn_pressure())[0]
def test_already_collocated_in_col_ungridded_to_ungridded_in_2d(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, nn_pressure, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
sample_points = UngriddedData.from_points_array(
[HyperPoint(lat=0.0, lon=0.0, pres=80.0, t=dt.datetime(1984, 9, 4, 15, 54))])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(), nn_pressure())[0]
eq_(new_data.data[0], 41.0)
def test_GIVEN_ungridded_data_WHEN_call_as_data_frame_THEN_returns_valid_data_frame(self):
from cis.test.util.mock import make_regular_4d_ungridded_data
from datetime import datetime
ug_data = make_regular_4d_ungridded_data()
df = ug_data.as_data_frame()
assert_that(df['rainfall_flux'][5] == 6)
assert_that(df['latitude'][17] == 0)
assert_that(df['latitude'].ix[datetime(1984,8,31)][0] == 10)
assert_that(df['rainfall_flux'].median() == 25.5)
def test_basic_col_in_4d(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, mean, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
# Note - This isn't actually used for averaging
sample_points = UngriddedData.from_points_array(
[HyperPoint(lat=1.0, lon=1.0, alt=12.0, t=dt.datetime(1984, 8, 29, 8, 34))])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(), mean())[0]
eq_(new_data.data[0], 25.5)
def test_GIVEN_ungridded_data_WHEN_call_as_data_frame_THEN_returns_valid_data_frame(
self):
from cis.test.util.mock import make_regular_4d_ungridded_data
from datetime import datetime
ug_data = make_regular_4d_ungridded_data()
df = ug_data.as_data_frame()
assert_that(df['rainfall_flux'][5] == 6)
assert_that(df['latitude'][17] == 0)
assert_that(df.loc[datetime(1984, 8, 31), 'latitude'][0] == 10)
assert_that(df['rainfall_flux'].median() == 25.5)
def test_basic_col_with_incompatible_points_throws_a_TypeError(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, nn_pressure, SepConstraintKdtree
ug_data = mock.make_regular_4d_ungridded_data()
# Make sample points with no time dimension specified
sample_points = UngriddedData.from_points_array([
HyperPoint(1.0, 1.0),
HyperPoint(4.0, 4.0),
HyperPoint(-4.0, -4.0)
])
col = GeneralUngriddedCollocator()
with self.assertRaises(AttributeError):
new_data = col.collocate(sample_points, ug_data,
SepConstraintKdtree(), nn_pressure())[0]
def test_already_collocated_in_col_ungridded_to_ungridded_in_2d(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, nn_pressure, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
sample_points = UngriddedData.from_points_array([
HyperPoint(lat=0.0,
lon=0.0,
pres=80.0,
t=dt.datetime(1984, 9, 4, 15, 54))
])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(),
nn_pressure())[0]
eq_(new_data.data[0], 41.0)
def test_coordinates_outside_grid_in_col_ungridded_to_ungridded_in_2d(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, nn_pressure, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
sample_points = HyperPointList()
sample_points.append(HyperPoint(lat=0.0, lon=0.0, pres=0.1, t=dt.datetime(1984, 8, 29, 8, 34)))
sample_points.append(HyperPoint(lat=0.0, lon=0.0, pres=91.0, t=dt.datetime(1984, 9, 2, 1, 23)))
sample_points.append(HyperPoint(lat=0.0, lon=0.0, pres=890.0, t=dt.datetime(1984, 9, 4, 15, 54)))
sample_points = UngriddedData.from_points_array(sample_points)
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(), nn_pressure())[0]
eq_(new_data.data[0], 1.0)
eq_(new_data.data[1], 46.0)
eq_(new_data.data[2], 46.0)
def test_basic_col_in_4d(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, nn_altitude, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
sample_points = HyperPointList()
sample_points.append(HyperPoint(lat=1.0, lon=1.0, alt=12.0, t=dt.datetime(1984, 8, 29, 8, 34)))
sample_points.append(HyperPoint(lat=4.0, lon=4.0, alt=34.0, t=dt.datetime(1984, 9, 2, 1, 23)))
sample_points.append(HyperPoint(lat=-4.0, lon=-4.0, alt=89.0, t=dt.datetime(1984, 9, 4, 15, 54)))
sample_points = UngriddedData.from_points_array(sample_points)
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(), nn_altitude())[0]
eq_(new_data.data[0], 6.0)
eq_(new_data.data[1], 16.0)
eq_(new_data.data[2], 46.0)
def test_basic_col_in_4d(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, mean, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
# Note - This isn't actually used for averaging
sample_points = UngriddedData.from_points_array([
HyperPoint(lat=1.0,
lon=1.0,
alt=12.0,
t=dt.datetime(1984, 8, 29, 8, 34))
])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(),
mean())[0]
eq_(new_data.data[0], 25.5)
def test_coordinates_exactly_between_points_in_col_ungridded_to_ungridded_in_2d(self):
"""
This works out the edge case where the points are exactly in the middle or two or more datapoints.
The nn_pressure algorithm will start with the first point as the nearest and iterates through the
points finding any points which are closer than the current closest. If two distances were exactly
the same the first point to be chosen.
"""
from cis.collocation.col_implementations import GeneralUngriddedCollocator, nn_pressure, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
# Choose a time at midday
sample_points = UngriddedData.from_points_array(
[HyperPoint(lat=0.0, lon=0.0, pres=8, t=dt.datetime(1984, 8, 29, 12))])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(), nn_pressure())[0]
eq_(new_data.data[0], 1.0)
def test_pressure_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraintKdtree
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(0.0, 0.0, 50.0, 24.0, dt.datetime(1984, 8, 29))
constraint = SepConstraintKdtree(p_sep=2)
# This should leave us with 20 points: [ 6. 7. 8. 9. 10.]
# [ 11. 12. 13. 14. 15.]
# [ 16. 17. 18. 19. 20.]
# [ 21. 22. 23. 24. 25.]
ref_vals = np.array(
[
6.0,
7.0,
8.0,
9.0,
10.0,
11.0,
12.0,
13.0,
14.0,
15.0,
16.0,
17.0,
18.0,
19.0,
20.0,
21.0,
22.0,
23.0,
24.0,
25.0,
]
)
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert np.equal(ref_vals, new_vals).all()
def test_averaging_basic_col_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
# Note - This isn't actually used for averaging
sample_points = UngriddedData.from_points_array(
[HyperPoint(lat=1.0, lon=1.0, alt=12.0, t=dt.datetime(1984, 8, 29, 8, 34))])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(), moments())
means = new_data[0]
std_dev = new_data[1]
no_points = new_data[2]
eq_(means.name(), 'rainfall_flux')
eq_(std_dev.name(), 'Corrected sample standard deviation of TOTAL RAINFALL RATE: LS+CONV KG/M2/S')
eq_(no_points.name(), 'Number of points used to calculate the mean of TOTAL RAINFALL RATE: LS+CONV KG/M2/S')
assert means.coords()
assert std_dev.coords()
assert no_points.coords()
def test_averaging_basic_col_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
# Note - This isn't actually used for averaging
sample_points = UngriddedData.from_points_array(
[HyperPoint(lat=1.0, lon=1.0, alt=12.0, t=dt.datetime(1984, 8, 29, 8, 34))])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, DummyConstraint(), moments())
means = new_data[0]
std_dev = new_data[1]
no_points = new_data[2]
eq_(means.name(), 'rain')
eq_(std_dev.name(), 'rain_std_dev')
eq_(no_points.name(), 'rain_num_points')
assert means.coords()
assert std_dev.coords()
assert no_points.coords()
def test_basic_col_in_4d_with_pressure_not_altitude(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, moments, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
# Note - This isn't actually used for averaging
sample_points = UngriddedData.from_points_array(
[HyperPoint(lat=1.0, lon=1.0, pres=12.0, t=dt.datetime(1984, 8, 29, 8, 34))])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(), moments())
means = new_data[0]
std_dev = new_data[1]
no_points = new_data[2]
eq_(means.data[0], 25.5)
assert_almost_equal(std_dev.data[0], np.sqrt(212.5))
eq_(no_points.data[0], 50)
def test_all_constraints_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False,
name='vals').dropna(axis=1)
sample_point = pd.DataFrame(
data={
'longitude': [0.0],
'latitude': [0.0],
'altitude': [50.0],
'air_pressure': [50.0],
'time':
[cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]
}).iloc[0]
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
h_sep = 1000
# 15m altitude separation
a_sep = 15
# 1 day (and a little bit) time separation
t_sep = 'P1dT1M'
# Pressure constraint is 50/40 < p_sep < 60/50
p_sep = 1.22
constraint = SepConstraintKdtree(h_sep=h_sep,
a_sep=a_sep,
p_sep=p_sep,
t_sep=t_sep)
index = HaversineDistanceKDTreeIndex()
index.index_data(None, ug_data_points, None)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 9 points: [[ 22, 23, 24]
# [ 27, 28, 29]
# [ 32, 33, 34]]
ref_vals = np.array([27., 28., 29., 32., 33., 34.])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = np.sort(new_points.vals)
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_time_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraint
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, t=dt.datetime(1984, 8, 29))
# 1 day (and a little bit) time seperation
constraint = SepConstraint(t_sep='P1dT1M')
# This should leave us with 30 points
ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten()
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraint
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, t=dt.datetime(1984, 8, 29))
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraint(h_sep=1000)
# This should leave us with 30 points
ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten()
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_time_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraintKdtree
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1)
sample_point = pd.Series({'longitude': [0.0], 'latitude': [0.0], 'altitude':[50.0],
'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]})
# 1 day (and a little bit) time seperation
constraint = SepConstraintKdtree(t_sep='P1dT1M')
# This should leave us with 30 points
ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten()
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1)
sample_points = pd.DataFrame(data={'longitude': [0.0], 'latitude': [0.0], 'altitude': [50.0],
'time': [dt.datetime(1984, 8, 29)]})
coord_map = None
# Constraint distance selects the central three points.
constraint = SepConstraintKdtree(h_sep=1000)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, ug_data_points, coord_map)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 30 points
ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten()
new_points = constraint.constrain_points(sample_points.iloc[0], ug_data_points)
new_vals = np.sort(new_points.vals)
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, t=dt.datetime(1984, 8, 29))
sample_points = HyperPointList([sample_point])
coord_map = None
# Constraint distance selects the central three points.
constraint = SepConstraintKdtree(h_sep=1000)
index = HaversineDistanceKDTreeIndex()
index.index_data(sample_points, ug_data_points, coord_map)
constraint.haversine_distance_kd_tree_index = index
# This should leave us with 30 points
ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten()
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = np.sort(new_points.vals)
eq_(ref_vals.size, new_vals.size)
assert np.equal(ref_vals, new_vals).all()
def test_all_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraint
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=0.0,
lon=0.0,
alt=50.0,
pres=50.0,
t=dt.datetime(1984, 8, 29))
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
h_sep = 1000
# 15m altitude seperation
a_sep = 15
# 1 day (and a little bit) time seperation
t_sep = 'P1DT1M'
# Pressure constraint is 50/40 < p_sep < 60/50
p_sep = 1.22
constraint = SepConstraint(h_sep=h_sep,
a_sep=a_sep,
p_sep=p_sep,
t_sep=t_sep)
# This should leave us with 9 points: [[ 22, 23, 24]
# [ 27, 28, 29]
# [ 32, 33, 34]]
ref_vals = np.array([27., 28., 29., 32., 33., 34.])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_basic_col_in_4d_with_pressure_not_altitude(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, moments, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
# Note - This isn't actually used for averaging
sample_points = UngriddedData.from_points_array([
HyperPoint(lat=1.0,
lon=1.0,
pres=12.0,
t=dt.datetime(1984, 8, 29, 8, 34))
])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(),
moments())
means = new_data[0]
std_dev = new_data[1]
no_points = new_data[2]
eq_(means.data[0], 25.5)
assert_almost_equal(std_dev.data[0], np.sqrt(212.5))
eq_(no_points.data[0], 50)
def test_time_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraint
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=0.0,
lon=0.0,
alt=50.0,
t=dt.datetime(1984, 8, 29))
# 1 day (and a little bit) time seperation
constraint = SepConstraint(t_sep='P1dT1M')
# This should leave us with 30 points
ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten()
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_pressure_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraintKdtree
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1)
sample_point = pd.Series({'longitude': [0.0], 'latitude': [0.0], 'altitude':[50.0], 'air_pressure': [24.0],
'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]})
constraint = SepConstraintKdtree(p_sep=2)
# This should leave us with 20 points: [ 6. 7. 8. 9. 10.]
# [ 11. 12. 13. 14. 15.]
# [ 16. 17. 18. 19. 20.]
# [ 21. 22. 23. 24. 25.]
ref_vals = np.array([6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., 17., 18., 19., 20., 21., 22., 23.,
24., 25.])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_coordinates_exactly_between_points_in_col_ungridded_to_ungridded_in_2d(
self):
"""
This works out the edge case where the points are exactly in the middle or two or more datapoints.
The nn_pressure algorithm will start with the first point as the nearest and iterates through the
points finding any points which are closer than the current closest. If two distances were exactly
the same the first point to be chosen.
"""
from cis.collocation.col_implementations import GeneralUngriddedCollocator, nn_pressure, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
# Choose a time at midday
sample_points = UngriddedData.from_points_array([
HyperPoint(lat=0.0,
lon=0.0,
pres=8,
t=dt.datetime(1984, 8, 29, 12))
])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(),
nn_pressure())[0]
eq_(new_data.data[0], 1.0)
def test_alt_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraint
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=0.0, lon=0.0, alt=50.0, t=dt.datetime(1984, 8, 29))
# 15m altitude seperation
a_sep = 15
constraint = SepConstraint(a_sep=a_sep)
# This should leave us with 15 points: [ 21. 22. 23. 24. 25.]
# [ 26. 27. 28. 29. 30.]
# [ 31. 32. 33. 34. 35.]
ref_vals = np.array([21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., 35.])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_alt_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraintKdtree
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=1)
sample_point = pd.Series({'longitude': [0.0], 'latitude': [0.0], 'altitude':[50.0],
'time': [cis_standard_time_unit.date2num(dt.datetime(1984, 8, 29))]})
# 15m altitude separation
a_sep = 15
constraint = SepConstraintKdtree(a_sep=a_sep)
# This should leave us with 15 points: [ 21. 22. 23. 24. 25.]
# [ 26. 27. 28. 29. 30.]
# [ 31. 32. 33. 34. 35.]
ref_vals = np.array([21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., 35.])
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_4d(self):
from cis.collocation.col_implementations import SepConstraint
import datetime as dt
import numpy as np
ug_data = mock.make_regular_4d_ungridded_data()
ug_data_points = ug_data.get_non_masked_points()
sample_point = HyperPoint(lat=0.0,
lon=0.0,
alt=50.0,
t=dt.datetime(1984, 8, 29))
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraint(h_sep=1000)
# This should leave us with 30 points
ref_vals = np.reshape(np.arange(50) + 1.0, (10, 5))[:, 1:4].flatten()
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
