def test_GIVEN_single_variable_WHEN_aggregate_THEN_DataWriter_called_correctly(self):
variables = 'var_name'
filenames = 'filename'
output_file = 'output.hdf'
kernel = 'mean'
grid = None
input_data = GriddedDataList([make_from_cube(make_square_5x3_2d_cube())])
output_data = make_from_cube(make_square_5x3_2d_cube() + 1)
mock_data_reader = DataReader()
mock_data_reader.read_data_list = MagicMock(return_value=input_data)
mock_data_writer = DataWriter()
mock_data_writer.write_data = Mock()
mock_aggregator = Aggregator(None, None)
mock_aggregator.aggregate_gridded = MagicMock(return_value=output_data) # Return the modified data array
aggregate = Aggregate(grid, output_file, data_reader=mock_data_reader, data_writer=mock_data_writer)
aggregate._create_aggregator = MagicMock(return_value=mock_aggregator)
aggregate.aggregate(variables, filenames, None, kernel)
assert_that(mock_data_writer.write_data.call_count, is_(1))
written_data = mock_data_writer.write_data.call_args[0][0]
written_filename = mock_data_writer.write_data.call_args[0][1]
assert_that(written_data.data.tolist(), is_([[2, 3, 4], [5, 6, 7], [8, 9, 10], [11, 12, 13], [14, 15, 16]]))
assert_that(written_filename, is_(output_file))
def test_GIVEN_multiple_variable_gridded_WHEN_read_data_THEN_GriddedDataList_returned(
self):
variables = ['var1', 'var2']
filenames = 'filename1'
product = None
gridded_data = make_square_5x3_2d_cube()
gridded_data.__class__ = GriddedData
get_data_func = MagicMock(return_value=gridded_data)
reader = DataReader(get_data_func=get_data_func)
data = reader.read_data_list(filenames, variables, product)
# Check the data read function is called correctly
assert_that(get_data_func.call_count, is_(2))
first_call_args = get_data_func.call_args_list[0][0]
second_call_args = get_data_func.call_args_list[1][0]
assert_that(first_call_args[0], is_([filenames]))
assert_that(first_call_args[1], is_(variables[0]))
assert_that(second_call_args[1], is_(variables[1]))
assert_that(first_call_args[2], is_(product))
# Check the data we got back is as expected
assert_that(data, instance_of(GriddedDataList))
assert_that(data[0].data.tolist(),
is_(make_square_5x3_2d_cube().data.tolist()))
assert_that(data[1].data.tolist(), is_(data[0].data.tolist()))
def test_GIVEN_multiple_variables_WHEN_subset_THEN_DataWriter_called_correctly(self):
variables = ['var_name1', 'var_name2']
filename = 'filename'
xmin, xmax = 0, 5
ymin, ymax = -5, 5
limits = {'x': SubsetLimits(xmin, xmax, False),
'y': SubsetLimits(ymin, ymax, False)}
output_file = 'output.hdf'
def _mock_subset(data, constraint):
# Modify the data slightly so we can be sure it's passed in correctly
for var in data:
var.data += 1
return data
mock_subsetter = Subsetter()
mock_subsetter.subset = _mock_subset
mock_data_reader = DataReader()
mock_data_reader.read_data_list = MagicMock(return_value=GriddedDataList([make_square_5x3_2d_cube(),
make_square_5x3_2d_cube()]))
mock_data_writer = DataWriter()
mock_data_writer.write_data = MagicMock()
subset = Subset(limits, output_file, subsetter=mock_subsetter,
data_reader=mock_data_reader, data_writer=mock_data_writer)
subset.subset(variables, filename, product=None)
assert_that(mock_data_writer.write_data.call_count, is_(1))
written_data = mock_data_writer.write_data.call_args[0][0]
written_filename = mock_data_writer.write_data.call_args[0][1]
assert_that(written_data[0].data.tolist(), is_([[2, 3, 4], [5, 6, 7], [8, 9, 10], [11, 12, 13], [14, 15, 16]]))
assert_that(written_data[0].data.tolist(), written_data[1].data.tolist())
assert_that(written_filename, is_(output_file))
def test_GIVEN_gridded_datagroups_WHEN_read_datagroups_THEN_data_returned_in_list(
self):
datagroup_1 = {
'variables': ['var1', 'var2'],
'filenames': ['filename1.nc'],
'product': None
}
datagroup_2 = {
'variables': ['var3'],
'filenames': ['filename2.nc'],
'product': 'cis'
}
var1 = make_from_cube(make_square_5x3_2d_cube())
var2 = make_from_cube(make_square_5x3_2d_cube())
var3 = make_from_cube(make_square_5x3_2d_cube())
get_data_func = MagicMock(side_effect=[var1, var2, var3])
get_var_func = MagicMock(side_effect=lambda f: {
'filename1.nc': ['var1', 'var2'],
'filename2.nc': ['var3']
}[f])
reader = DataReader(get_data_func=get_data_func,
get_variables_func=get_var_func)
data = reader.read_datagroups([datagroup_1, datagroup_2])
assert_that(len(data), is_(3))
assert_that(data[0], is_(var1))
assert_that(data[1], is_(var2))
assert_that(data[2], is_(var3))
def test_GIVEN_not_enough_aliases_WHEN_read_datagroups_THEN_raises_ValueError(self):
datagroup = {'variables': ['var1', 'var2'],
'filenames': ['filename1.nc'],
'product': None,
'aliases': ['alias1']}
var1 = make_from_cube(make_square_5x3_2d_cube())
var2 = make_from_cube(make_square_5x3_2d_cube())
get_data_func = MagicMock(side_effect=[var1, var2])
get_var_func = MagicMock(side_effect=['var1', 'var2'])
reader = DataReader(get_data_func=get_data_func, get_variables_func=get_var_func)
with self.assertRaises(ValueError):
data = reader.read_datagroups([datagroup])
def test_already_collocated_in_col_gridded_to_ungridded_in_2d(self):
cube = make_from_cube(mock.make_square_5x3_2d_cube())
# This point already exists on the cube with value 5 - which shouldn't be a problem
sample_points = UngriddedData.from_points_array([HyperPoint(0.0, 0.0)])
col = GriddedUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, 'nn')[0]
eq_(new_data.data[0], 8.0)
def test_GIVEN_single_variable_WHEN_subset_THEN_Subsetter_called_correctly(self):
variable = 'var_name'
filename = 'filename'
xmin, xmax = 0, 5
ymin, ymax = -5, 5
limits = {'x': SubsetLimits(xmin, xmax, False),
'y': SubsetLimits(ymin, ymax, False)}
output_file = 'output.hdf'
mock_data_reader = DataReader()
mock_data_reader.read_data_list = MagicMock(return_value=make_from_cube(make_square_5x3_2d_cube()))
mock_data_writer = DataWriter()
mock_data_writer.write_data = Mock()
mock_subsetter = Subsetter()
mock_subsetter.subset = MagicMock(side_effect=lambda *args: args[0]) # Return the data array unmodified
subset = Subset(limits, output_file, subsetter=mock_subsetter,
data_reader=mock_data_reader, data_writer=mock_data_writer)
subset.subset(variable, filename, product=None)
assert_that(mock_subsetter.subset.call_count, is_(1))
called_data = mock_subsetter.subset.call_args[0][0]
called_constraint = mock_subsetter.subset.call_args[0][1]
assert_that(called_data.data.tolist(),
is_([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12], [13, 14, 15]]))
assert_that(called_constraint, instance_of(GriddedSubsetConstraint))
assert_that(called_constraint._limits['latitude'][1:3], is_((ymin, ymax)))
assert_that(called_constraint._limits['longitude'][1:3], is_((xmin, xmax)))
def test_already_collocated_in_col_gridded_to_ungridded_in_2d(self):
cube = make_from_cube(mock.make_square_5x3_2d_cube())
# This point already exists on the cube with value 5 - which shouldn't be a problem
sample_points = UngriddedData.from_points_array([HyperPoint(0.0, 0.0)])
col = GriddedUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, 'nn')[0]
eq_(new_data.data[0], 8.0)
def list_moments(constraint, kernel):
col = GeneralGriddedCollocator()
sample = make_square_5x3_2d_cube()
data1 = make_regular_2d_ungridded_data(10, -10, 10, 6, -5, 5)
data2 = make_regular_2d_ungridded_data(10, -10, 10, 6, -5, 5)
data2.metadata._name = 'snow'
data2.data *= 2
output = col.collocate(sample, UngriddedDataList([data1, data2]), constraint, kernel)
assert len(output) == 6
assert output[0].var_name == 'rain'
assert output[1].var_name == 'rain_std_dev'
assert output[2].var_name == 'rain_num_points'
assert output[3].var_name == 'snow'
assert output[4].var_name == 'snow_std_dev'
assert output[5].var_name == 'snow_num_points'
expected_data1 = numpy.array([[4.5, 6.5, 8.5],
[16.5, 18.5, 20.5],
[28.5, 30.5, 32.5],
[40.5, 42.5, 44.5],
[52.5, 54.5, 56.5]])
expected_data2 = 2 * expected_data1
expected_stddev1 = numpy.ones((5, 3)) * 3.5118845842842465
expected_stddev2 = expected_stddev1 * 2
expected_num = numpy.ones((5, 3)) * 4
assert numpy.array_equal(output[0].data, expected_data1)
assert numpy.allclose(output[1].data, expected_stddev1)
assert numpy.array_equal(output[2].data, expected_num)
assert numpy.array_equal(output[3].data, expected_data2)
assert numpy.allclose(output[4].data, expected_stddev2)
assert numpy.array_equal(output[5].data, expected_num)
def test_GIVEN_not_enough_aliases_WHEN_read_datagroups_THEN_raises_ValueError(
self):
datagroup = {
'variables': ['var1', 'var2'],
'filenames': ['filename1.nc'],
'product': None,
'aliases': ['alias1']
}
var1 = make_from_cube(make_square_5x3_2d_cube())
var2 = make_from_cube(make_square_5x3_2d_cube())
get_data_func = MagicMock(side_effect=[var1, var2])
get_var_func = MagicMock(side_effect=['var1', 'var2'])
reader = DataReader(get_data_func=get_data_func,
get_variables_func=get_var_func)
with self.assertRaises(ValueError):
data = reader.read_datagroups([datagroup])
def test_already_collocated_in_col_gridded_to_ungridded_in_2d(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, nn_gridded
cube = gridded_data.make_from_cube(mock.make_square_5x3_2d_cube())
# This point already exists on the cube with value 5 - which shouldn't be a problem
sample_points = UngriddedData.from_points_array([HyperPoint(0.0, 0.0)])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, nn_gridded())[0]
eq_(new_data.data[0], 8.0)
def test_basic_col_gridded_to_ungridded_using_li_in_2d(self):
cube = make_from_cube(mock.make_square_5x3_2d_cube())
sample_points = UngriddedData.from_points_array(
[HyperPoint(1.0, 1.0), HyperPoint(4.0, 4.0), HyperPoint(-4.0, -4.0)])
col = GriddedUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, 'lin')[0]
assert_almost_equal(new_data.data[0], 8.8)
assert_almost_equal(new_data.data[1], 11.2)
assert_almost_equal(new_data.data[2], 4.8)
def test_coordinates_outside_grid_in_col_gridded_to_ungridded_in_2d(self):
cube = make_from_cube(mock.make_square_5x3_2d_cube())
sample_points = UngriddedData.from_points_array(
[HyperPoint(5.5, 5.5), HyperPoint(-5.5, 5.5), HyperPoint(5.5, -5.5), HyperPoint(-5.5, -5.5)])
col = GriddedUngriddedCollocator(extrapolate=True)
new_data = col.collocate(sample_points, cube, None, 'nn')[0]
eq_(new_data.data[0], 12.0)
eq_(new_data.data[1], 6.0)
eq_(new_data.data[2], 10.0)
eq_(new_data.data[3], 4.0)
def test_GIVEN_aliases_missing_WHEN_read_datagroups_THEN_read_OK_aliases_default_to_var_names(self):
datagroup = {'variables': ['var1'],
'filenames': ['filename1.nc'],
'product': None}
var1 = make_from_cube(make_square_5x3_2d_cube())
get_data_func = MagicMock(side_effect=[var1])
get_var_func = MagicMock(side_effect=['var1'])
reader = DataReader(get_data_func=get_data_func, get_variables_func=get_var_func)
data = reader.read_datagroups([datagroup])
assert_that(data[0].var_name, is_('dummy'))
def test_can_iterate_over_all_points(self):
gd = gridded_data.make_from_cube(mock.make_square_5x3_2d_cube())
hpv = gd.get_all_points()
count = 0
vals = []
for p in hpv:
count += 1
vals.append(p.val[0])
assert(count == 15)
assert(vals == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])
def test_GIVEN_aliases_WHEN_read_datagroups_THEN_output_data_has_aliases(self):
datagroup = {'variables': ['var1'],
'filenames': ['filename1.nc'],
'product': None,
'aliases': ['alias1']}
get_data_func = MagicMock(return_value=make_from_cube(make_square_5x3_2d_cube()))
get_var_func = MagicMock(return_value=['var1'])
reader = DataReader(get_data_func=get_data_func, get_variables_func=get_var_func)
data = reader.read_datagroups([datagroup])
assert_that(data[0].alias, is_('alias1'))
def test_GIVEN_multiple_variable_mix_of_gridded_ungridded_WHEN_read_data_THEN_raises_TypeError(self):
variables = ['var1', 'var2']
filenames = 'filename1'
product = None
gridded_data = make_square_5x3_2d_cube()
gridded_data.__class__ = GriddedData
ungridded_data = make_regular_2d_ungridded_data()
get_data_func = MagicMock(side_effect=[gridded_data, ungridded_data])
reader = DataReader(get_data_func=get_data_func)
with self.assertRaises(TypeError):
data = reader.read_data_list(filenames, variables, product)[0]
def test_GIVEN_gridded_datagroups_WHEN_read_datagroups_THEN_data_returned_in_list(self):
datagroup_1 = {'variables': ['var1', 'var2'],
'filenames': ['filename1.nc'],
'product': None}
datagroup_2 = {'variables': ['var3'],
'filenames': ['filename2.nc'],
'product': 'cis'}
var1 = make_from_cube(make_square_5x3_2d_cube())
var2 = make_from_cube(make_square_5x3_2d_cube())
var3 = make_from_cube(make_square_5x3_2d_cube())
get_data_func = MagicMock(side_effect=[var1, var2, var3])
get_var_func = MagicMock(side_effect=lambda f: {'filename1.nc': ['var1', 'var2'],
'filename2.nc': ['var3']}[f])
reader = DataReader(get_data_func=get_data_func, get_variables_func=get_var_func)
data = reader.read_datagroups([datagroup_1, datagroup_2])
assert_that(isinstance(data, GriddedDataList))
assert_that(len(data), is_(3))
assert_that(data[0], is_(var1))
assert_that(data[1], is_(var2))
assert_that(data[2], is_(var3))
def test_GIVEN_single_variable_gridded_WHEN_read_data_THEN_GriddedData_returned(self):
variables = 'var1'
filenames = 'filename1'
product = None
gridded_data = make_square_5x3_2d_cube()
gridded_data.__class__ = GriddedData
get_data_func = MagicMock(return_value=gridded_data)
reader = DataReader(get_data_func=get_data_func)
data = reader.read_data_list(filenames, variables, product)[0]
# Check the data read function is called correctly
assert_that(get_data_func.call_count, is_(1))
call_args = get_data_func.call_args_list[0][0]
assert_that(call_args[0], is_([filenames]))
assert_that(call_args[1], is_(variables))
assert_that(call_args[2], is_(product))
# Check the data we got back is as expected
assert_that(data, instance_of(GriddedData))
assert_that(data.data.tolist(), is_(make_square_5x3_2d_cube().data.tolist()))
def test_basic_col_gridded_to_ungridded_using_li_in_2d(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, li
cube = gridded_data.make_from_cube(mock.make_square_5x3_2d_cube())
sample_points = UngriddedData.from_points_array(
[HyperPoint(1.0, 1.0), HyperPoint(4.0, 4.0), HyperPoint(-4.0, -4.0)])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, li())[0]
assert_almost_equal(new_data.data[0], 8.8)
assert_almost_equal(new_data.data[1], 11.2)
assert_almost_equal(new_data.data[2], 4.8)
def test_coordinates_outside_grid_in_col_gridded_to_ungridded_in_2d(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, nn_gridded
cube = gridded_data.make_from_cube(mock.make_square_5x3_2d_cube())
sample_points = UngriddedData.from_points_array(
[HyperPoint(5.5, 5.5), HyperPoint(-5.5, 5.5), HyperPoint(5.5, -5.5), HyperPoint(-5.5, -5.5)])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, nn_gridded())[0]
eq_(new_data.data[0], 12.0)
eq_(new_data.data[1], 6.0)
eq_(new_data.data[2], 10.0)
eq_(new_data.data[3], 4.0)
def test_GIVEN_multiple_variable_mix_of_gridded_ungridded_WHEN_read_data_THEN_raises_TypeError(
self):
variables = ['var1', 'var2']
filenames = 'filename1'
product = None
gridded_data = make_square_5x3_2d_cube()
gridded_data.__class__ = GriddedData
ungridded_data = make_regular_2d_ungridded_data()
get_data_func = MagicMock(side_effect=[gridded_data, ungridded_data])
reader = DataReader(get_data_func=get_data_func)
with self.assertRaises(TypeError):
data = reader.read_data_list(filenames, variables, product)[0]
def test_negative_lon_points_in_2d_dont_matter(self):
"""
This is exactly the same test as above, except we ommit the point with negative longitude, this makes the
collocator wrap the longitude coordinate and gives a slightly different interpolation result...
"""
cube = make_from_cube(mock.make_square_5x3_2d_cube())
sample_points = UngriddedData.from_points_array(
[HyperPoint(1.0, 1.0), HyperPoint(4.0, 4.0)])
col = GriddedUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, 'lin')[0]
assert_almost_equal(new_data.data[0], 8.8)
assert_almost_equal(new_data.data[1], 11.2)
def test_basic_col_gridded_to_ungridded_using_li_in_2d(self):
cube = make_from_cube(mock.make_square_5x3_2d_cube())
sample_points = UngriddedData.from_points_array([
HyperPoint(1.0, 1.0),
HyperPoint(4.0, 4.0),
HyperPoint(-4.0, -4.0)
])
col = GriddedUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, 'lin')[0]
assert_almost_equal(new_data.data[0], 8.8)
assert_almost_equal(new_data.data[1], 11.2)
assert_almost_equal(new_data.data[2], 4.8)
def test_negative_lon_points_in_2d_dont_matter(self):
"""
This is exactly the same test as above, except we ommit the point with negative longitude, this makes the
collocator wrap the longitude coordinate and gives a slightly different interpolation result...
"""
cube = make_from_cube(mock.make_square_5x3_2d_cube())
sample_points = UngriddedData.from_points_array(
[HyperPoint(1.0, 1.0), HyperPoint(4.0, 4.0)])
col = GriddedUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, 'lin')[0]
assert_almost_equal(new_data.data[0], 8.8)
assert_almost_equal(new_data.data[1], 11.2)
def test_basic_col_gridded_to_ungridded_in_2d(self):
cube = make_from_cube(mock.make_square_5x3_2d_cube())
sample_points = UngriddedData.from_points_array(
[HyperPoint(lat=1.0, lon=1.0),
HyperPoint(lat=4.0, lon=4.0),
HyperPoint(lat=-4.0, lon=-4.0)])
col = GriddedUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, 'nn')[0]
eq_(new_data.data[0], 8.0) # float(cube[2,1].data))
eq_(new_data.data[1], 12.0) # float(cube[3,2].data))
eq_(new_data.data[2], 4.0) # float(cube[1,0].data))
def test_basic_col_gridded_to_ungridded_in_2d(self):
from cis.collocation.col_implementations import GeneralUngriddedCollocator, nn_gridded
cube = gridded_data.make_from_cube(mock.make_square_5x3_2d_cube())
sample_points = UngriddedData.from_points_array(
[HyperPoint(lat=1.0, lon=1.0),
HyperPoint(lat=4.0, lon=4.0),
HyperPoint(lat=-4.0, lon=-4.0)])
col = GeneralUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, nn_gridded())[0]
eq_(new_data.data[0], 8.0) # float(cube[2,1].data))
eq_(new_data.data[1], 12.0) # float(cube[3,2].data))
eq_(new_data.data[2], 4.0) # float(cube[1,0].data))
def test_2d_cube(self):
from cis.test.util.mock import make_square_5x3_2d_cube
from cis.data_io.ungridded_data import UngriddedData
from cis.data_io.hyperpoint import HyperPoint
cube = make_square_5x3_2d_cube()
sample_points = UngriddedData.from_points_array(
[HyperPoint(1.0, 1.0), HyperPoint(4.0, 4.0), HyperPoint(-4.0, -4.0)])
interpolator = GriddedUngriddedInterpolator(cube, sample_points, 'lin')
values = interpolator(cube)
wanted = np.asarray([8.8, 11.2, 4.8])
assert_array_almost_equal(values, wanted)
def test_basic_col_gridded_to_ungridded_in_2d(self):
cube = make_from_cube(mock.make_square_5x3_2d_cube())
sample_points = UngriddedData.from_points_array([
HyperPoint(lat=1.0, lon=1.0),
HyperPoint(lat=4.0, lon=4.0),
HyperPoint(lat=-4.0, lon=-4.0)
])
col = GriddedUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, 'nn')[0]
eq_(new_data.data[0], 8.0) # float(cube[2,1].data))
eq_(new_data.data[1], 12.0) # float(cube[3,2].data))
eq_(new_data.data[2], 4.0) # float(cube[1,0].data))
def test_GIVEN_aliases_missing_WHEN_read_datagroups_THEN_read_OK_aliases_default_to_var_names(
self):
datagroup = {
'variables': ['var1'],
'filenames': ['filename1.nc'],
'product': None
}
var1 = make_from_cube(make_square_5x3_2d_cube())
get_data_func = MagicMock(side_effect=[var1])
get_var_func = MagicMock(side_effect=['var1'])
reader = DataReader(get_data_func=get_data_func,
get_variables_func=get_var_func)
data = reader.read_datagroups([datagroup])
assert_that(data[0].var_name, is_('dummy'))
def test_coordinates_outside_grid_in_col_gridded_to_ungridded_in_2d(self):
cube = make_from_cube(mock.make_square_5x3_2d_cube())
sample_points = UngriddedData.from_points_array([
HyperPoint(5.5, 5.5),
HyperPoint(-5.5, 5.5),
HyperPoint(5.5, -5.5),
HyperPoint(-5.5, -5.5)
])
col = GriddedUngriddedCollocator(extrapolate=True)
new_data = col.collocate(sample_points, cube, None, 'nn')[0]
eq_(new_data.data[0], 12.0)
eq_(new_data.data[1], 6.0)
eq_(new_data.data[2], 10.0)
eq_(new_data.data[3], 4.0)
def test_GIVEN_GriddedData_WHEN_collocate_THEN_collocator_called_correctly(self):
sample = gridded_data.make_from_cube(mock.make_square_5x3_2d_cube())
out_name = "output-name"
col_name = "dummy"
col_options = {}
mock_collocator = MagicMock()
mock_collocator.collocate = MagicMock(return_value=GriddedDataList([sample]))
mock_constraint = MagicMock()
mock_kernel = MagicMock()
mock_collocator_factory = CollocatorFactory()
mock_collocator_factory.get_collocator_instances_for_method = MagicMock(
return_value=(mock_collocator, mock_constraint, mock_kernel)
)
col = Collocate(sample, out_name, collocator_factory=mock_collocator_factory)
data = gridded_data.make_from_cube(mock.make_square_5x3_2d_cube())
output = col.collocate(data, col_name, col_options)
assert_that(mock_collocator_factory.get_collocator_instances_for_method.call_count, is_(1))
factory_call_args = mock_collocator_factory.get_collocator_instances_for_method.call_args_list[0][0]
assert_that(factory_call_args[0], is_(col_name))
assert_that(factory_call_args[1], is_(None))
col_options["missing_data_for_missing_sample"] = False
assert_that(factory_call_args[2], is_(col_options))
assert_that(factory_call_args[3], is_(None))
assert_that(factory_call_args[4], is_(True))
assert_that(factory_call_args[5], is_(True))
assert_that(mock_collocator.collocate.call_count, is_(1))
collocator_args = mock_collocator.collocate.call_args_list[0][0]
assert_that(collocator_args[0], is_(sample))
assert_that(collocator_args[1], is_(data))
assert_that(collocator_args[2], is_(mock_constraint))
assert_that(collocator_args[3], is_(mock_kernel))
assert_that(isinstance(output, GriddedDataList))
assert_that(len(output), is_(1))
assert_that(output[0].data.flatten().tolist(), is_(sample.data.flatten().tolist()))
def can_collocate_list_of_data(constraint, kernel):
col = GeneralGriddedCollocator()
sample = make_square_5x3_2d_cube()
data1 = make_regular_2d_ungridded_data(10, -10, 10, 6, -5, 5, 0)
data2 = make_regular_2d_ungridded_data(10, -10, 10, 6, -5, 5, 10)
output = col.collocate(sample, UngriddedDataList([data1, data2]), constraint, kernel)
assert len(output) == 2
expected_data2 = numpy.array([[14.5, 16.5, 18.5],
[26.5, 28.5, 30.5],
[38.5, 40.5, 42.5],
[50.5, 52.5, 54.5],
[62.5, 64.5, 66.5]])
expected_data1 = expected_data2 - 10
assert_arrays_equal(output[0].data, expected_data1)
assert_arrays_equal(output[1].data, expected_data2)
def test_GIVEN_gridded_and_ungridded_datagroups_WHEN_read_datagroups_THEN_raises_TypeError(self):
datagroup_1 = {'variables': ['var1'],
'filenames': ['filename1.nc'],
'product': None}
datagroup_2 = {'variables': ['var3'],
'filenames': ['filename2.nc'],
'product': 'cis'}
var1 = make_from_cube(make_square_5x3_2d_cube())
var2 = make_regular_2d_ungridded_data()
get_data_func = MagicMock(side_effect=[var1, var2])
get_var_func = MagicMock(side_effect=lambda f: {'filename1.nc': ['var1'],
'filename2.nc': ['var3']}[f])
reader = DataReader(get_data_func=get_data_func, get_variables_func=get_var_func)
with self.assertRaises(TypeError):
reader.read_datagroups([datagroup_1, datagroup_2])
def test_GIVEN_aliases_WHEN_read_datagroups_THEN_output_data_has_aliases(
self):
datagroup = {
'variables': ['var1'],
'filenames': ['filename1.nc'],
'product': None,
'aliases': ['alias1']
}
get_data_func = MagicMock(
return_value=make_from_cube(make_square_5x3_2d_cube()))
get_var_func = MagicMock(return_value=['var1'])
reader = DataReader(get_data_func=get_data_func,
get_variables_func=get_var_func)
data = reader.read_datagroups([datagroup])
assert_that(data[0].alias, is_('alias1'))
def test_coordinates_exactly_between_points_in_col_gridded_to_ungridded_in_2d(self):
"""
This works out the edge case where the points are exactly in the middle or two or more datapoints.
Iris seems to count a point as 'belonging' to a datapoint if it is greater than a datapoint cell's lower
bound and less than or equal to it's upper bound. Where a cell is an imaginary boundary around a
datapoint which divides the grid.
"""
cube = make_from_cube(mock.make_square_5x3_2d_cube())
sample_points = UngriddedData.from_points_array(
[HyperPoint(2.5, 2.5), HyperPoint(-2.5, 2.5), HyperPoint(2.5, -2.5), HyperPoint(-2.5, -2.5)])
col = GriddedUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, 'nn')[0]
eq_(new_data.data[0], 8.0)
eq_(new_data.data[1], 5.0)
eq_(new_data.data[2], 7.0)
eq_(new_data.data[3], 4.0)
def test_2d_cube(self):
from cis.test.util.mock import make_square_5x3_2d_cube
from cis.data_io.ungridded_data import UngriddedData
from cis.data_io.hyperpoint import HyperPoint
cube = make_square_5x3_2d_cube()
sample_points = UngriddedData.from_points_array([
HyperPoint(1.0, 1.0),
HyperPoint(4.0, 4.0),
HyperPoint(-4.0, -4.0)
])
interpolator = GriddedUngriddedInterpolator(cube, sample_points, 'lin')
values = interpolator(cube)
wanted = np.asarray([8.8, 11.2, 4.8])
assert_array_almost_equal(values, wanted)
def single_moments(constraint, kernel):
col = GeneralGriddedCollocator()
sample = make_square_5x3_2d_cube()
data = make_regular_2d_ungridded_data(10, -9.9, 9.9, 6, -4.9, 4.9, 10)
output = col.collocate(sample, data, constraint, kernel)
expected_data = numpy.array([[14.5, 16.5, 18.5],
[26.5, 28.5, 30.5],
[38.5, 40.5, 42.5],
[50.5, 52.5, 54.5],
[62.5, 64.5, 66.5]])
expected_stddev = numpy.ones((5, 3)) * 3.5118845842842465
expected_num = numpy.ones((5, 3)) * 4
assert len(output) == 3
assert isinstance(output, GriddedDataList)
assert output[0].var_name == 'rain'
assert output[1].var_name == 'rain_std_dev'
assert output[2].var_name == 'rain_num_points'
assert_arrays_equal(output[0].data, expected_data)
assert numpy.allclose(output[1].data, expected_stddev)
assert numpy.array_equal(output[2].data, expected_num)
def test_coordinates_exactly_between_points_in_col_gridded_to_ungridded_in_2d(
self):
"""
This works out the edge case where the points are exactly in the middle or two or more datapoints.
Iris seems to count a point as 'belonging' to a datapoint if it is greater than a datapoint cell's lower
bound and less than or equal to it's upper bound. Where a cell is an imaginary boundary around a
datapoint which divides the grid.
"""
cube = make_from_cube(mock.make_square_5x3_2d_cube())
sample_points = UngriddedData.from_points_array([
HyperPoint(2.5, 2.5),
HyperPoint(-2.5, 2.5),
HyperPoint(2.5, -2.5),
HyperPoint(-2.5, -2.5)
])
col = GriddedUngriddedCollocator()
new_data = col.collocate(sample_points, cube, None, 'nn')[0]
eq_(new_data.data[0], 8.0)
eq_(new_data.data[1], 5.0)
eq_(new_data.data[2], 7.0)
eq_(new_data.data[3], 4.0)
def test_GIVEN_single_variable_WHEN_subset_THEN_DataReader_called_correctly(self):
variable = 'var_name'
filename = 'filename'
output_file = 'output.hdf'
xmin, xmax = 0, 5
ymin, ymax = -5, 5
limits = {'x': SubsetLimits(xmin, xmax, False),
'y': SubsetLimits(ymin, ymax, False)}
mock_data_reader = DataReader()
mock_data_reader.read_data_list = MagicMock(return_value=make_from_cube(make_square_5x3_2d_cube()))
mock_data_writer = DataWriter()
mock_data_writer.write_data = Mock()
mock_subsetter = Subsetter()
mock_subsetter.subset = lambda *args: args[0] # Return the data array unmodified
subset = Subset(limits, output_file, subsetter=mock_subsetter,
data_reader=mock_data_reader, data_writer=mock_data_writer)
subset.subset(variable, filename, product=None)
assert_that(mock_data_reader.read_data_list.call_count, is_(1))
assert_that(mock_data_reader.read_data_list.call_args[0][0], filename)
assert_that(mock_data_reader.read_data_list.call_args[0][1], variable)
def test_GIVEN_multiple_variables_and_filenames_WHEN_aggregate_THEN_Aggregate_called_correctly(self):
variables = ['var_name1', 'var_name2']
filenames = ['filename1', 'filename2']
output_file = 'output.hdf'
kernel = 'mean'
grid = 'grid'
input_data = GriddedDataList(2 * [make_from_cube(make_square_5x3_2d_cube())])
output_data = input_data
mock_data_reader = DataReader()
mock_data_reader.read_data_list = MagicMock(return_value=input_data)
mock_data_writer = DataWriter()
mock_data_writer.write_data = Mock()
mock_aggregator = Aggregator(None, None)
mock_aggregator.aggregate_gridded = MagicMock(return_value=output_data) # Return the data array unmodified
aggregate = Aggregate(grid, output_file, data_reader=mock_data_reader, data_writer=mock_data_writer)
aggregate._create_aggregator = MagicMock(return_value=mock_aggregator)
aggregate.aggregate(variables, filenames, None, kernel)
assert_that(mock_aggregator.aggregate_gridded.call_count, is_(1))
assert_that(mock_aggregator.aggregate_gridded.call_args[0][0], kernel)
