def single_point_results_in_single_value_in_cell_with_pressure_with_cube_with_pressure(con, kernel):
sample_cube = make_square_5x3_2d_cube_with_pressure()
data_point = make_dummy_ungridded_data_single_point(0.5, 0.5, 1.2, pressure=1.0)
col = GeneralGriddedCollocator(fill_value=-999.9)
out_cube = col.collocate(points=sample_cube, data=data_point, constraint=con, kernel=kernel)[0]
expected_result = numpy.array([[[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9]],
[[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9]],
[[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9],
[-999.9, 1.2, -999.9, -999.9, -999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9]],
[[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9]],
[[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9, -999.9, -999.9, -999.9, -999.9]]])
print(out_cube.data.filled())
assert_arrays_equal(out_cube.data.filled(), expected_result)
def list_moments(constraint, kernel):
col = GeneralGriddedCollocator()
sample = make_square_5x3_2d_cube()
data1 = make_regular_2d_ungridded_data(10, -10, 10, 6, -5, 5)
data2 = make_regular_2d_ungridded_data(10, -10, 10, 6, -5, 5)
data2.metadata._name = 'snow'
data2.data *= 2
output = col.collocate(sample, UngriddedDataList([data1, data2]), constraint, kernel)
assert len(output) == 6
assert output[0].var_name == 'rain'
assert output[1].var_name == 'rain_std_dev'
assert output[2].var_name == 'rain_num_points'
assert output[3].var_name == 'snow'
assert output[4].var_name == 'snow_std_dev'
assert output[5].var_name == 'snow_num_points'
expected_data1 = numpy.array([[4.5, 6.5, 8.5],
[16.5, 18.5, 20.5],
[28.5, 30.5, 32.5],
[40.5, 42.5, 44.5],
[52.5, 54.5, 56.5]])
expected_data2 = 2 * expected_data1
expected_stddev1 = numpy.ones((5, 3)) * 3.5118845842842465
expected_stddev2 = expected_stddev1 * 2
expected_num = numpy.ones((5, 3)) * 4
assert numpy.array_equal(output[0].data, expected_data1)
assert numpy.allclose(output[1].data, expected_stddev1)
assert numpy.array_equal(output[2].data, expected_num)
assert numpy.array_equal(output[3].data, expected_data2)
assert numpy.allclose(output[4].data, expected_stddev2)
assert numpy.array_equal(output[5].data, expected_num)
def test_data_with_no_standard_name(self):
sample_cube = make_mock_cube()
data_points = make_dummy_1d_ungridded_data()
col = GeneralGriddedCollocator(fill_value=-999.9)
con = CubeCellConstraint()
out_cube = col.collocate(points=sample_cube, data=data_points, constraint=con, kernel=SlowMean())[0]
def point_on_180_is_included_in_lower_bound(con, kernel):
sample_cube = make_square_NxM_2d_cube_with_time(start_lon=-135, end_lon=135, lon_point_count=4)
data_point = make_dummy_ungridded_data_single_point(0, 180.0, 1.2)
col = GeneralGriddedCollocator(fill_value=-999.9)
out_cube = col.collocate(points=sample_cube, data=data_point, constraint=con, kernel=kernel)[0]
expected_result = numpy.array([[-999.9, -999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9, -999.9],
[1.2, -999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9, -999.9]])
assert_arrays_almost_equal(out_cube.data.filled(), expected_result)
def two_points_in_a_cell_results_in_mean_value_in_cell(con, kernel):
sample_cube = make_mock_cube()
data_point = make_dummy_ungridded_data_two_points_with_different_values(0.5, 0.5, 1.2, 1.4)
col = GeneralGriddedCollocator(fill_value=-999.9)
out_cube = col.collocate(points=sample_cube, data=data_point, constraint=con, kernel=kernel)[0]
expected_result = numpy.array([[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, 1.3, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9]])
assert_arrays_almost_equal(out_cube.data.filled(), expected_result)
def single_point_outside_grid_and_one_inside_excludes_outside_using_binned_only(con, kernel):
sample_cube = make_mock_cube()
data_point = make_dummy_ungridded_data_single_point([0.5, 99], [0.5, 99], [1.2, 5])
col = GeneralGriddedCollocator(fill_value=-999.9)
out_cube = col.collocate(points=sample_cube, data=data_point, constraint=con, kernel=kernel)[0]
expected_result = numpy.array([[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, 1.2, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9]])
assert_arrays_equal(out_cube.data.filled(), expected_result)
def point_on_a_lat_lon_boundary_appears_in_highest_cell(con, kernel):
sample_cube = make_mock_cube()
data_point = make_dummy_ungridded_data_single_point(2.5, 2.5, 1.2)
col = GeneralGriddedCollocator(fill_value=-999.9)
out_cube = col.collocate(points=sample_cube, data=data_point, constraint=con, kernel=kernel)[0]
expected_result = numpy.array([[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, 1.2],
[-999.9, -999.9, -999.9]])
assert_arrays_almost_equal(out_cube.data.filled(), expected_result)
def single_point_on_grid_corner_is_counted_once(con, kernel):
sample_cube = make_mock_cube()
data_point = make_dummy_ungridded_data_single_point(10, 5, 1.2)
col = GeneralGriddedCollocator(fill_value=-999.9)
out_cube = col.collocate(points=sample_cube, data=data_point, constraint=con, kernel=kernel)[0]
expected_result = numpy.array([[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, 1.2]])
assert_arrays_equal(out_cube.data.filled(), expected_result)
def single_masked_point_results_in_single_value_in_cell_using_kernel_and_con(con, kernel):
sample_cube = make_mock_cube()
data_point = make_dummy_ungridded_data_single_point(0.5, 0.5, 1.2, mask=True)
col = GeneralGriddedCollocator(fill_value=-999.9)
out_cube = col.collocate(points=sample_cube, data=data_point, constraint=con, kernel=kernel)[0]
expected_result = numpy.array([[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9]])
assert_arrays_equal(out_cube.data.filled(), expected_result)
def multiple_points_inside_grid_and_outside(con, kernel):
sample_cube = make_mock_cube()
data_point = make_dummy_ungridded_data_single_point([0.5, 99, 0.6, 3.0, -9], [0.5, 99, 0.6, 0.5, -3],
[1.2, 5, 3.4, 5, 8])
col = GeneralGriddedCollocator(fill_value=-999.9)
out_cube = col.collocate(points=sample_cube, data=data_point, constraint=con, kernel=kernel)[0]
expected_result = numpy.array([[8, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, 2.3, -999.9],
[-999.9, 5, -999.9],
[-999.9, -999.9, -999.9]])
assert_arrays_equal(out_cube.data.filled(), expected_result)
def test_gridded_gridded_bin_when_sample_has_dimension_data_doesnt(self):
# JASCIS-204
from cis.data_io.gridded_data import make_from_cube
sample = make_from_cube(make_mock_cube(time_dim_length=7, dim_order=['lat', 'lon', 'time']))
data = make_from_cube(make_mock_cube(lat_dim_length=11, lon_dim_length=13,
time_dim_length=0, dim_order=['time', 'lon', 'lat']))
col = GeneralGriddedCollocator()
constraint = BinningCubeCellConstraint()
kernel = mean()
out_cube = col.collocate(points=sample, data=data, constraint=constraint, kernel=kernel)
assert out_cube[0].shape == (5, 3)
def aggregate_ungridded(self, kernel):
"""
Performs aggregation for ungridded data by first generating a new grid, converting it into a cube, then
collocating using the appropriate kernel and a cube cell constraint
"""
new_cube_coords = []
new_cube_shape = []
i = 0
for coord in self.data.coords():
grid, guessed_axis = self.get_grid(coord)
if grid is None:
new_coord = self._make_fully_collapsed_coord(coord)
if grid is not None and isnan(grid.delta):
# Issue a warning and then still collapse fully
logging.warning('Coordinate ' + guessed_axis + ' was given without a grid. No need to specify '
'coordinates for complete collapse, all coordinates without a grid specified are '
'automatically collapsed for ungridded aggregation.')
new_coord = self._make_fully_collapsed_coord(coord)
if grid is not None and not isnan(grid.delta):
new_coord = self._make_partially_collapsed_coord(coord, grid, guessed_axis)
new_cube_coords.append((new_coord, i))
new_cube_shape.append(len(new_coord.points))
i += 1
if len(self._grid) != 0:
raise CoordinateNotFoundError('No coordinate found that matches {}. Please check the coordinate '
'name.'.format(self._grid.keys()))
dummy_data = numpy.reshape(numpy.arange(int(numpy.prod(new_cube_shape))) + 1.0, tuple(new_cube_shape))
aggregation_cube = iris.cube.Cube(dummy_data, dim_coords_and_dims=new_cube_coords)
collocator = GeneralGriddedCollocator()
constraint = BinnedCubeCellOnlyConstraint()
aggregated_cube = collocator.collocate(aggregation_cube, self.data, constraint, kernel)
self._add_max_min_bounds_for_collapsed_coords(aggregated_cube, self.data)
# We need to rename any variables which clash with coordinate names otherwise they will not output correctly, we
# prepend it with 'aggregated_' to make it clear which variable has been aggregated (the original coordinate
# value will not have been.)
for idx, d in enumerate(aggregated_cube):
if d.var_name in [coord.var_name for coord in aggregation_cube.coords()]:
new_name = "aggregated_" + d.var_name
aggregated_cube[idx].rename(new_name)
aggregated_cube[idx].var_name = new_name
logging.warning("Variable {} clashes with a coordinate variable name and has been renamed to: {}"
.format(d.var_name, new_name))
return aggregated_cube
def can_collocate_list_of_data(constraint, kernel):
col = GeneralGriddedCollocator()
sample = make_square_5x3_2d_cube()
data1 = make_regular_2d_ungridded_data(10, -10, 10, 6, -5, 5, 0)
data2 = make_regular_2d_ungridded_data(10, -10, 10, 6, -5, 5, 10)
output = col.collocate(sample, UngriddedDataList([data1, data2]), constraint, kernel)
assert len(output) == 2
expected_data2 = numpy.array([[14.5, 16.5, 18.5],
[26.5, 28.5, 30.5],
[38.5, 40.5, 42.5],
[50.5, 52.5, 54.5],
[62.5, 64.5, 66.5]])
expected_data1 = expected_data2 - 10
assert_arrays_equal(output[0].data, expected_data1)
assert_arrays_equal(output[1].data, expected_data2)
def test_fill_value_for_cube_cell_constraint_default_fill_value(self):
sample_cube = make_mock_cube()
data_point = make_dummy_ungridded_data_single_point(99, 99, 0.0)
col = GeneralGriddedCollocator()
con = CubeCellConstraint()
out_cube = col.collocate(points=sample_cube, data=data_point, constraint=con, kernel=SlowMean())[0]
expected_result = numpy.array([[float('nan'), float('nan'), float('nan')],
[float('nan'), float('nan'), float('nan')],
[float('nan'), float('nan'), float('nan')],
[float('nan'), float('nan'), float('nan')],
[float('nan'), float('nan'), float('nan')]])
numpy.testing.assert_array_equal(out_cube.data.filled(), expected_result)
def test_fill_value_for_cube_cell_constraint(self):
sample_cube = make_mock_cube()
data_point = make_dummy_ungridded_data_single_point(99, 99, 0.0)
col = GeneralGriddedCollocator(fill_value=-999.9)
con = CubeCellConstraint()
out_cube = col.collocate(points=sample_cube, data=data_point, constraint=con, kernel=SlowMean())[0]
expected_result = numpy.array([[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9]])
assert numpy.array_equal(out_cube.data.filled(), expected_result)
def single_point_results_in_single_value_in_masked_cell_using_kernel_and_con_missing_for_masked_false(con, kernel):
mask = [[False, False, False],
[False, False, False],
[False, True, False],
[False, False, False],
[False, False, False]]
sample_cube = make_mock_cube(mask=mask)
data_point = make_dummy_ungridded_data_single_point(0.5, 0.5, 1.2)
col = GeneralGriddedCollocator(fill_value=-999.9, missing_data_for_missing_sample=False)
out_cube = col.collocate(points=sample_cube, data=data_point, constraint=con, kernel=kernel)[0]
expected_result = numpy.array([[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, 1.2, -999.9],
[-999.9, -999.9, -999.9],
[-999.9, -999.9, -999.9]])
assert_arrays_equal(out_cube.data.filled(), expected_result)
def single_moments(constraint, kernel):
col = GeneralGriddedCollocator()
sample = make_square_5x3_2d_cube()
data = make_regular_2d_ungridded_data(10, -9.9, 9.9, 6, -4.9, 4.9, 10)
output = col.collocate(sample, data, constraint, kernel)
expected_data = numpy.array([[14.5, 16.5, 18.5],
[26.5, 28.5, 30.5],
[38.5, 40.5, 42.5],
[50.5, 52.5, 54.5],
[62.5, 64.5, 66.5]])
expected_stddev = numpy.ones((5, 3)) * 3.5118845842842465
expected_num = numpy.ones((5, 3)) * 4
assert len(output) == 3
assert isinstance(output, GriddedDataList)
assert output[0].var_name == 'rain'
assert output[1].var_name == 'rain_std_dev'
assert output[2].var_name == 'rain_num_points'
assert_arrays_equal(output[0].data, expected_data)
assert numpy.allclose(output[1].data, expected_stddev)
assert numpy.array_equal(output[2].data, expected_num)
