def test_aggregation_two_dims_using_moments_kernel(self):
self.kernel = moments()
data1 = make_regular_2d_ungridded_data_with_missing_values()
data2 = make_regular_2d_ungridded_data_with_missing_values()
data2.metadata._name = 'snow'
data2._data += 10
data = UngriddedDataList([data1, data2])
grid = {'y': AggregationGrid(-12.5, 12.5, 15, False), 'x': AggregationGrid(-7.5, 7.5, 10, False)}
agg = Aggregator(data, grid)
output = agg.aggregate_ungridded(self.kernel)
expect_mean = numpy.array([[4.4, 4.5], [35.0 / 3, 13.5]])
expect_stddev = numpy.array([[numpy.sqrt(9.3), numpy.sqrt(4.5)],
[numpy.sqrt(13.0 / 3), numpy.sqrt(4.5)]])
expect_count = numpy.array([[5, 2], [3, 2]])
assert isinstance(output, GriddedDataList)
assert len(output) == 6
mean_1, stddev_1, count_1, mean_2, stddev_2, count_2 = output
assert mean_1.var_name == 'rain'
assert stddev_1.var_name == 'rain_std_dev'
assert count_1.var_name == 'rain_num_points'
assert mean_2.var_name == 'snow'
assert stddev_2.var_name == 'snow_std_dev'
assert count_2.var_name == 'snow_num_points'
assert_arrays_almost_equal(mean_1.data, expect_mean)
assert_arrays_almost_equal(stddev_1.data, expect_stddev)
assert_arrays_almost_equal(count_1.data, expect_count)
assert_arrays_almost_equal(mean_2.data, expect_mean + 10)
assert_arrays_almost_equal(stddev_2.data, expect_stddev)
assert_arrays_almost_equal(count_2.data, expect_count)
def test_aggregation_two_dims_using_moments_kernel(self):
self.kernel = moments()
data1 = make_regular_2d_ungridded_data_with_missing_values()
data2 = make_regular_2d_ungridded_data_with_missing_values()
data2.metadata._name = 'snow'
data2._data += 10
data = UngriddedDataList([data1, data2])
grid = {'y': slice(-12.5, 12.5, 15), 'x': slice(-7.5, 7.5, 10)}
output = data.aggregate(how=self.kernel, **grid)
expect_mean = numpy.array([[4.4, 4.5], [35.0 / 3, 13.5]])
expect_stddev = numpy.array([[numpy.sqrt(9.3),
numpy.sqrt(4.5)],
[numpy.sqrt(13.0 / 3),
numpy.sqrt(4.5)]])
expect_count = numpy.array([[5, 2], [3, 2]])
assert isinstance(output, GriddedDataList)
assert len(output) == 6
mean_1, stddev_1, count_1, mean_2, stddev_2, count_2 = output
assert mean_1.var_name == 'rain'
assert stddev_1.var_name == 'rain_std_dev'
assert count_1.var_name == 'rain_num_points'
assert mean_2.var_name == 'snow'
assert stddev_2.var_name == 'snow_std_dev'
assert count_2.var_name == 'snow_num_points'
assert_arrays_almost_equal(mean_1.data, expect_mean)
assert_arrays_almost_equal(stddev_1.data, expect_stddev)
assert_arrays_almost_equal(count_1.data, expect_count)
assert_arrays_almost_equal(mean_2.data, expect_mean + 10)
assert_arrays_almost_equal(stddev_2.data, expect_stddev)
assert_arrays_almost_equal(count_2.data, expect_count)
def test_list_ungridded_ungridded_box_mean(self):
ug_data_1 = mock.make_regular_2d_ungridded_data()
ug_data_2 = mock.make_regular_2d_ungridded_data(data_offset=3)
ug_data_2.long_name = 'TOTAL SNOWFALL RATE: LS+CONV KG/M2/S'
ug_data_2.standard_name = 'snowfall_rate'
ug_data_2.metadata._name = 'snow'
data_list = UngriddedDataList([ug_data_1, ug_data_2])
sample_points = mock.make_regular_2d_ungridded_data()
constraint = SepConstraintKdtree('500km')
kernel = moments()
col = GeneralUngriddedCollocator()
output = col.collocate(sample_points, data_list, constraint, kernel)
expected_result = np.array(list(range(1, 16)))
expected_stddev = np.array(15 * [float('inf')])
expected_n = np.array(15 * [1])
assert len(output) == 6
assert isinstance(output, UngriddedDataList)
assert output[3].var_name == 'snow'
assert output[4].var_name == 'snow_std_dev'
assert output[5].var_name == 'snow_num_points'
assert np.allclose(output[0].data, expected_result)
assert np.allclose(output[1].data, expected_stddev)
assert np.allclose(output[2].data, expected_n)
assert np.allclose(output[3].data, expected_result + 3)
assert np.allclose(output[4].data, expected_stddev)
assert np.allclose(output[5].data, expected_n)
def test_list_gridded_ungridded_box_moments(self):
data1 = make_from_cube(mock.make_mock_cube())
data1.name = lambda: 'Name1'
data1.var_name = 'var_name1'
data1._standard_name = 'y_wind'
data2 = make_from_cube(mock.make_mock_cube(data_offset=3))
data2.name = lambda: 'Name1'
data2.var_name = 'var_name2'
data2._standard_name = 'x_wind'
data_list = GriddedDataList([data1, data2])
sample = UngriddedData.from_points_array(
[HyperPoint(lat=1.0, lon=1.0, alt=12.0, t=dt.datetime(1984, 8, 29, 8, 34)),
HyperPoint(lat=3.0, lon=3.0, alt=7.0, t=dt.datetime(1984, 8, 29, 8, 34)),
HyperPoint(lat=-1.0, lon=-1.0, alt=5.0, t=dt.datetime(1984, 8, 29, 8, 34))])
constraint = SepConstraintKdtree('500km')
kernel = moments()
col = GeneralUngriddedCollocator()
output = col.collocate(sample, data_list, constraint, kernel)
expected_result = np.array([28.0/3, 10.0, 20.0/3])
expected_stddev = np.array([1.52752523, 1.82574186, 1.52752523])
expected_n = np.array([3, 4, 3])
assert len(output) == 6
assert isinstance(output, UngriddedDataList)
assert np.allclose(output[0].data, expected_result)
assert np.allclose(output[1].data, expected_stddev)
assert np.allclose(output[2].data, expected_n)
assert np.allclose(output[3].data, expected_result + 3)
assert np.allclose(output[4].data, expected_stddev)
assert np.allclose(output[5].data, expected_n)
def test_aggregation_one_dim_using_moments_kernel(self):
self.kernel = moments()
data1 = make_regular_2d_ungridded_data_with_missing_values()
data2 = make_regular_2d_ungridded_data_with_missing_values()
data2.metadata._name = 'snow'
data2._data += 10
data = UngriddedDataList([data1, data2])
grid = {'y': slice(-12.5, 12.5, 12.5)}
output = data.aggregate(how=self.kernel, **grid)
expect_mean = numpy.array([[3.2], [11]])
expect_stddev = numpy.array([[numpy.sqrt(3.7)], [numpy.sqrt(26.0 / 3)]])
expect_count = numpy.array([[5], [7]])
assert isinstance(output, GriddedDataList)
assert len(output) == 6
mean_1, stddev_1, count_1, mean_2, stddev_2, count_2 = output
assert mean_1.var_name == 'rain'
assert stddev_1.var_name == 'rain_std_dev'
assert count_1.var_name == 'rain_num_points'
assert mean_2.var_name == 'snow'
assert stddev_2.var_name == 'snow_std_dev'
assert count_2.var_name == 'snow_num_points'
assert_arrays_almost_equal(mean_1.data, expect_mean)
assert_arrays_almost_equal(stddev_1.data, expect_stddev)
assert_arrays_almost_equal(count_1.data, expect_count)
assert_arrays_almost_equal(mean_2.data, expect_mean + 10)
assert_arrays_almost_equal(stddev_2.data, expect_stddev)
assert_arrays_almost_equal(count_2.data, expect_count)
def test_list_ungridded_ungridded_box_mean(self):
ug_data_1 = mock.make_regular_2d_ungridded_data()
ug_data_2 = mock.make_regular_2d_ungridded_data(data_offset=3)
ug_data_2.long_name = 'TOTAL SNOWFALL RATE: LS+CONV KG/M2/S'
ug_data_2.standard_name = 'snowfall_flux'
ug_data_2.metadata._name = 'snow'
data_list = UngriddedDataList([ug_data_1, ug_data_2])
sample_points = mock.make_regular_2d_ungridded_data()
constraint = SepConstraintKdtree('500km')
kernel = moments()
col = GeneralUngriddedCollocator()
output = col.collocate(sample_points, data_list, constraint, kernel)
expected_result = np.array(list(range(1, 16)))
expected_n = np.array(15 * [1])
assert len(output) == 6
assert isinstance(output, UngriddedDataList)
assert output[3].var_name == 'snow'
assert output[4].var_name == 'snow_std_dev'
assert output[5].var_name == 'snow_num_points'
assert np.allclose(output[0].data, expected_result)
assert all(output[1].data.mask)
assert np.allclose(output[2].data, expected_n)
assert np.allclose(output[3].data, expected_result + 3)
assert all(output[4].data.mask)
assert np.allclose(output[5].data, expected_n)
def test_basic_col_in_4d_with_pressure_not_altitude(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, moments, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
# Note - This isn't actually used for averaging
sample_points = UngriddedData.from_points_array(
[HyperPoint(lat=1.0, lon=1.0, pres=12.0, t=dt.datetime(1984, 8, 29, 8, 34))])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(), moments())
means = new_data[0]
std_dev = new_data[1]
no_points = new_data[2]
eq_(means.data[0], 25.5)
assert_almost_equal(std_dev.data[0], np.sqrt(212.5))
eq_(no_points.data[0], 50)
def test_averaging_basic_col_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
# Note - This isn't actually used for averaging
sample_points = UngriddedData.from_points_array(
[HyperPoint(lat=1.0, lon=1.0, alt=12.0, t=dt.datetime(1984, 8, 29, 8, 34))])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(), moments())
means = new_data[0]
std_dev = new_data[1]
no_points = new_data[2]
eq_(means.name(), 'rainfall_flux')
eq_(std_dev.name(), 'Corrected sample standard deviation of TOTAL RAINFALL RATE: LS+CONV KG/M2/S')
eq_(no_points.name(), 'Number of points used to calculate the mean of TOTAL RAINFALL RATE: LS+CONV KG/M2/S')
assert means.coords()
assert std_dev.coords()
assert no_points.coords()
def test_averaging_basic_col_in_4d(self):
ug_data = mock.make_regular_4d_ungridded_data()
# Note - This isn't actually used for averaging
sample_points = UngriddedData.from_points_array(
[HyperPoint(lat=1.0, lon=1.0, alt=12.0, t=dt.datetime(1984, 8, 29, 8, 34))])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, DummyConstraint(), moments())
means = new_data[0]
std_dev = new_data[1]
no_points = new_data[2]
eq_(means.name(), 'rain')
eq_(std_dev.name(), 'rain_std_dev')
eq_(no_points.name(), 'rain_num_points')
assert means.coords()
assert std_dev.coords()
assert no_points.coords()
def test_ungridded_ungridded_box_moments(self):
data = mock.make_regular_2d_ungridded_data()
sample = UngriddedData.from_points_array(
[HyperPoint(lat=1.0, lon=1.0, alt=12.0, t=dt.datetime(1984, 8, 29, 8, 34)),
HyperPoint(lat=3.0, lon=3.0, alt=7.0, t=dt.datetime(1984, 8, 29, 8, 34)),
HyperPoint(lat=-1.0, lon=-1.0, alt=5.0, t=dt.datetime(1984, 8, 29, 8, 34))])
constraint = SepConstraintKdtree('500km')
kernel = moments()
col = GeneralUngriddedCollocator()
output = col.collocate(sample, data, constraint, kernel)
expected_result = np.array([28.0/3, 10.0, 20.0/3])
expected_stddev = np.array([1.52752523, 1.82574186, 1.52752523])
expected_n = np.array([3, 4, 3])
assert len(output) == 3
assert isinstance(output, UngriddedDataList)
assert np.allclose(output[0].data, expected_result)
assert np.allclose(output[1].data, expected_stddev)
assert np.allclose(output[2].data, expected_n)
def test_ungridded_ungridded_box_moments_no_missing_data_for_missing_sample(self):
data = mock.make_regular_2d_ungridded_data()
sample = UngriddedData.from_points_array(
[HyperPoint(lat=1.0, lon=1.0, alt=12.0, t=dt.datetime(1984, 8, 29, 8, 34)),
HyperPoint(lat=3.0, lon=3.0, alt=7.0, t=dt.datetime(1984, 8, 29, 8, 34)),
HyperPoint(lat=-1.0, lon=-1.0, alt=5.0, t=dt.datetime(1984, 8, 29, 8, 34))])
constraint = SepConstraintKdtree('500km')
kernel = moments()
sample_mask = [False, True, False]
sample.data = np.ma.array([0, 0, 0], mask=sample_mask)
col = GeneralUngriddedCollocator(missing_data_for_missing_sample=False)
output = col.collocate(sample, data, constraint, kernel)
assert len(output) == 3
assert isinstance(output, UngriddedDataList)
assert not any(output[0].data.mask)
assert not any(output[1].data.mask)
assert not any(output[2].data.mask)
def test_ungridded_ungridded_box_moments(self):
data = mock.make_regular_2d_ungridded_data()
sample = UngriddedData.from_points_array(
[HyperPoint(lat=1.0, lon=1.0, alt=12.0, t=dt.datetime(1984, 8, 29, 8, 34)),
HyperPoint(lat=3.0, lon=3.0, alt=7.0, t=dt.datetime(1984, 8, 29, 8, 34)),
HyperPoint(lat=-1.0, lon=-1.0, alt=5.0, t=dt.datetime(1984, 8, 29, 8, 34))])
constraint = SepConstraintKdtree('500km')
kernel = moments()
col = GeneralUngriddedCollocator()
output = col.collocate(sample, data, constraint, kernel)
expected_result = np.array([28.0/3, 10.0, 20.0/3])
expected_stddev = np.array([1.52752523, 1.82574186, 1.52752523])
expected_n = np.array([3, 4, 3])
assert len(output) == 3
assert isinstance(output, UngriddedDataList)
assert np.allclose(output[0].data, expected_result)
assert np.allclose(output[1].data, expected_stddev)
assert np.allclose(output[2].data, expected_n)
def test_aggregation_one_dim_using_moments_kernel(self):
self.kernel = moments()
data = make_regular_2d_ungridded_data_with_missing_values()
grid = {'y': AggregationGrid(-12.5, 12.5, 12.5, False)}
agg = Aggregator(data, grid)
output = agg.aggregate_ungridded(self.kernel)
expect_mean = numpy.array([[3.2], [11]])
expect_stddev = numpy.array([[numpy.sqrt(3.7)], [numpy.sqrt(26 / 3.0)]])
expect_count = numpy.array([[5], [7]])
assert isinstance(output, GriddedDataList)
assert len(output) == 3
actual_mean, actual_stddev, actual_count = output
assert actual_mean.var_name == 'rain'
assert actual_stddev.var_name == 'rain_std_dev'
assert actual_count.var_name == 'rain_num_points'
assert_arrays_almost_equal(actual_mean.data, expect_mean)
assert_arrays_almost_equal(actual_stddev.data, expect_stddev)
assert_arrays_almost_equal(actual_count.data, expect_count)
def test_aggregation_two_dims_using_moments_kernel(self):
self.kernel = moments()
data = make_regular_2d_ungridded_data_with_missing_values()
grid = {'y': AggregationGrid(-12.5, 12.5, 15, False), 'x': AggregationGrid(-7.5, 7.5, 10, False)}
agg = Aggregator(data, grid)
output = agg.aggregate_ungridded(self.kernel)
expect_mean = numpy.array([[4.4, 4.5], [35.0 / 3, 13.5]])
expect_stddev = numpy.array([[numpy.sqrt(9.3), numpy.sqrt(4.5)],
[numpy.sqrt(13.0 / 3), numpy.sqrt(4.5)]])
expect_count = numpy.array([[5, 2], [3, 2]])
assert isinstance(output, GriddedDataList)
assert len(output) == 3
actual_mean, actual_stddev, actual_count = output
assert actual_mean.var_name == 'rain'
assert actual_stddev.var_name == 'rain_std_dev'
assert actual_count.var_name == 'rain_num_points'
assert_arrays_almost_equal(actual_mean.data, expect_mean)
assert_arrays_almost_equal(actual_stddev.data, expect_stddev)
assert_arrays_almost_equal(actual_count.data, expect_count)
def test_basic_col_in_4d_with_pressure_not_altitude(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, moments, SepConstraintKdtree
import datetime as dt
ug_data = mock.make_regular_4d_ungridded_data()
# Note - This isn't actually used for averaging
sample_points = UngriddedData.from_points_array([
HyperPoint(lat=1.0,
lon=1.0,
pres=12.0,
t=dt.datetime(1984, 8, 29, 8, 34))
])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, ug_data, SepConstraintKdtree(),
moments())
means = new_data[0]
std_dev = new_data[1]
no_points = new_data[2]
eq_(means.data[0], 25.5)
assert_almost_equal(std_dev.data[0], np.sqrt(212.5))
eq_(no_points.data[0], 50)
def test_single_moments(self):
constraint = BinningCubeCellConstraint()
kernel = moments()
single_moments(constraint, kernel)
def __init__(self):
self.moments = moments()
