def setUp(self):
self.process = PostProcessBNG()
class TestPostProcessBNG(TestController):
def assert_that_arrays_almost_the_same(self, expected_values, resulting_values, message=""):
assert_that(resulting_values.shape, is_(expected_values.shape), "Array shapes for {}".format(message))
assert_that(np.ma.allclose(expected_values[:], resulting_values[:], atol=1.0e-15),
is_(True),
"{} arrays not almost the same. Expected\n {}\n was\n {}"
.format(message, expected_values[:], resulting_values[:]))
assert_that(np.array_equal(np.ma.getmaskarray(expected_values[:]), np.ma.getmaskarray(resulting_values[:])),
is_(True),
"{} arrays have the same mask. Expected\n {}\n was\n {}"
.format(message, expected_values[:], resulting_values[:]))
def assert_that_var_meta_data(self, actual_variable, long_name, standard_name, units, message):
assert_that(actual_variable.long_name, is_(long_name), "long name of {}".format(message))
assert_that(actual_variable.standard_name, is_(standard_name), "standard name of {}".format(message))
assert_that(actual_variable.units, is_(units), "units of {}".format(message))
def create_reference_file(self, expected_lats, expected_lons, expected_x, expected_y):
reference_file = netCDF4.Dataset("ref", mode="w", diskless=True)
reference_file.createDimension('x', len(expected_x))
reference_file.createDimension('y', len(expected_y))
reference_file.createVariable('lat', 'f4', ('y', 'x'))
reference_file.createVariable('lon', 'f4', ('y', 'x'))
reference_file.createVariable('x', 'f4', ('x',))
reference_file.createVariable('y', 'f4', ('y',))
reference_file.variables['lat'][:] = expected_lats
reference_file.variables['lon'][:] = expected_lons
reference_file.variables['x'][:] = expected_x
reference_file.variables['y'][:] = expected_y
return reference_file
def create_input_file(self, in_lats, in_lons, in_values, in_time=None, in_pusedo=None):
file_in_handle = netCDF4.Dataset("input", mode="w", diskless=True)
file_in_handle.createDimension('x', in_lats.shape[0])
file_in_handle.createDimension('y', in_lats.shape[1])
if in_time is not None:
file_in_handle.createDimension('time', in_time.shape[0])
if in_pusedo is not None:
file_in_handle.createDimension('pusedo', in_pusedo.shape[0])
lat_var = file_in_handle.createVariable('latitude', 'f4', ('y', 'x'))
lat_var[:] = in_lats
lon_var = file_in_handle.createVariable('longitude', 'f4', ('y', 'x'))
lon_var[:] = in_lons
if in_time is None:
values_var = file_in_handle.createVariable('values', 'f4', ('y', 'x'), fill_value=-99999.0)
else:
time_var = file_in_handle.createVariable('time', 'f4', ('time',))
time_var[:] = in_time
if in_pusedo is None:
values_var = file_in_handle.createVariable('values', 'f4', ('time', 'y', 'x'), fill_value=-99999.0)
else:
pusedo_var = file_in_handle.createVariable('pusedo', 'f4', ('pusedo',))
pusedo_var[:] = in_pusedo
values_var = file_in_handle.createVariable('values', 'f4', ('pusedo', 'time', 'y', 'x'), fill_value=-99999.0)
values_var[:] = in_values
return file_in_handle
def setUp(self):
self.process = PostProcessBNG()
def tearDown(self):
self.process.close()
def assert_that_variables_are_as_expected(self, expected_lats, expected_lons, expected_values, expected_x,
expected_y, expected_t=None, expected_pusedo=None):
out_variables = self.process.output_file_handle.variables
self.assert_that_arrays_almost_the_same(expected_values, out_variables['values'], "values")
assert_that(out_variables['values'].getncattr('_FillValue'), is_(-99999.0), "fill value for values")
self.assert_that_arrays_almost_the_same(expected_lats, out_variables['latitude'], "latitudes")
self.assert_that_var_meta_data(out_variables['latitude'], 'latitude', 'latitude', 'degrees_north', "latitude")
self.assert_that_arrays_almost_the_same(expected_lons, out_variables['longitude'], "longitudes")
self.assert_that_arrays_almost_the_same(expected_x, out_variables['x'], "x variable")
self.assert_that_arrays_almost_the_same(expected_y, out_variables['y'], "y variable")
if expected_t is not None:
# deliberate upper case of T for time
assert_that(self.process.output_file_handle.dimensions, has_key('Time'), "Time is in dimensions")
self.assert_that_arrays_almost_the_same(expected_t, out_variables['Time'], "Time variable")
if expected_pusedo is not None:
self.assert_that_arrays_almost_the_same(expected_pusedo, out_variables['pusedo'], "pusedo variable")
assert_that(self.process.output_file_handle.variables, has_key('crs'), "crs is in variable")
def test_GIVEN_file_with_3x2_points_in_WHEN_convert_THEN_return_file_with_all_3x2_points_populated(self):
expected_x = np.array([0, 1000, 2000])
expected_y = np.array([0, 1000])
expected_lats = np.array(
[[49.766807231896, 49.7674713664211, 49.768133858763],
[49.775759046806, 49.7764233904388, 49.7770860913769]])
expected_lons = np.array(
[[-7.5571598420827, -7.54333897801594, -7.52951760137346],
[-7.55818671478591, -7.54436332897671, -7.53053943025975]])
in_lats = np.array(
[[49.766807231896, 49.7674713664211, 49.768133858763, 49.775759046806, 49.7764233904388, 49.7770860913769]])
in_lons = np.array(
[[-7.5571598, -7.54333898, -7.529518, -7.5581867, -7.544363, -7.53054]])
expected_values = np.array([[0.0, 1.0, 2.0],
[1.0, 1.1, 2.1]])
in_values = expected_values.flatten()
self.process.input_file_handle = self.create_input_file(in_lats, in_lons, in_values)
self.process.reference_file_handle = self.create_reference_file(expected_lats, expected_lons, expected_x, expected_y)
self.process.output_file_handle = netCDF4.Dataset("output", mode="w", diskless=True)
self.process.convert_jules_1d_to_thredds_2d_for_chess()
self.assert_that_variables_are_as_expected(expected_lats, expected_lons, expected_values, expected_x,
expected_y)
def test_GIVEN_file_with_3x2_points_and_points_missing_WHEN_convert_THEN_return_file_with_all_3x2_points_populated_and_missing_values_for_non_populated(self):
expected_x = np.array([0, 1000, 2000])
expected_y = np.array([0, 1000])
expected_lats = np.array(
[[49.766807231896, 49.7674713664211, 49.768133858763],
[49.775759046806, 49.7764233904388, 49.7770860913769]])
expected_lons = np.array(
[[-7.5571598420827, -7.54333897801594, -7.52951760137346],
[-7.55818671478591, -7.54436332897671, -7.53053943025975]])
in_lats = np.array(
[[49.766807231896, 49.7674713664211, 49.768133858763, 49.7764233904388, 49.7770860913769]])
in_lons = np.array(
[[-7.5571598, -7.54333898, -7.529518, -7.544363, -7.53054]])
expected_values = np.ma.array([[0.0, 1.0, 2.0],
[1.0, 1.1, 2.1]],
mask=[[False, False, False],
[True, False, False]])
in_values = [[0.0, 1.0, 2.0, 1.1, 2.1]]
self.process.input_file_handle = self.create_input_file(in_lats, in_lons, in_values)
self.process.reference_file_handle = self.create_reference_file(expected_lats, expected_lons, expected_x, expected_y)
self.process.output_file_handle = netCDF4.Dataset("output", mode="w", diskless=True)
self.process.convert_jules_1d_to_thredds_2d_for_chess()
self.assert_that_variables_are_as_expected(expected_lats, expected_lons, expected_values, expected_x,
expected_y)
def test_GIVEN_file_with_1x1_point_WHEN_convert_THEN_return_file_with_all_1x1_points_populated(self):
ref_x = np.array([0, 1000, 2000])
ref_y = np.array([1, 1001, 2001])
ref_lats = np.array(
[[49.11, 49.21, 49.31],
[49.12, 49.22, 49.32],
[49.13, 49.23, 49.33]])
ref_lons = np.array(
[[7.11, 7.21, 7.31],
[7.12, 7.22, 7.32],
[7.13, 7.23, 7.33]])
in_lats = np.array([[49.22]])
in_lons = np.array([[7.22]])
expected_x = np.array([1000])
expected_y = np.array([1001])
expected_lats = np.array([[49.22]])
expected_lons = np.array([[7.22]])
expected_values = np.array([[6]])
self.process.input_file_handle = self.create_input_file(in_lats, in_lons, expected_values)
self.process.reference_file_handle = self.create_reference_file(ref_lats, ref_lons, ref_x, ref_y)
self.process.output_file_handle = netCDF4.Dataset("output", mode="w", diskless=True)
self.process.convert_jules_1d_to_thredds_2d_for_chess()
self.assert_that_variables_are_as_expected(expected_lats, expected_lons, expected_values, expected_x,
expected_y)
def test_GIVEN_file_with_1x1_point_with_a_masked_value_WHEN_convert_THEN_return_file_with_all_1x1_points_populated_as_a_masked_value(self):
ref_x = np.array([0, 1000, 2000])
ref_y = np.array([1, 1001, 2001])
ref_lats = np.array(
[[49.11, 49.21, 49.31],
[49.12, 49.22, 49.32],
[49.13, 49.23, 49.33]])
ref_lons = np.array(
[[7.11, 7.21, 7.31],
[7.12, 7.22, 7.32],
[7.13, 7.23, 7.33]])
in_lats = np.array([[49.22]])
in_lons = np.array([[7.22]])
expected_x = np.array([1000])
expected_y = np.array([1001])
expected_lats = np.array([[49.22]])
expected_lons = np.array([[7.22]])
expected_values = np.ma.array([[6]], mask=[[True]])
self.process.input_file_handle = self.create_input_file(in_lats, in_lons, expected_values)
self.process.reference_file_handle = self.create_reference_file(ref_lats, ref_lons, ref_x, ref_y)
self.process.output_file_handle = netCDF4.Dataset("output", mode="w", diskless=True)
self.process.convert_jules_1d_to_thredds_2d_for_chess()
self.assert_that_variables_are_as_expected(expected_lats, expected_lons, expected_values, expected_x,
expected_y)
def test_GIVEN_file_with_3x2x1_points_in_WHEN_convert_THEN_return_file_with_all_3x2x1_points_populated_and_time_var_is_Time(self):
ref_x = np.array([0, 1000, 2000])
ref_y = np.array([1, 1001, 2001])
ref_lats = np.array(
[[49.11, 49.21, 49.31],
[49.12, 49.22, 49.32],
[49.13, 49.23, 49.33]])
ref_lons = np.array(
[[7.11, 7.21, 7.31],
[7.12, 7.22, 7.32],
[7.13, 7.23, 7.33]])
expected_x = np.array([1000, 2000])
expected_y = np.array([1001])
expected_time = np.array([4.0, 5.0, 6.0])
expected_lats = np.array([[49.22, 49.32]])
expected_lons = np.array([[7.22, 7.32]])
expected_values = np.array([[[6, 7]], [[8, 9]], [[9, 10]]])
in_lats = expected_lats
in_lons = expected_lons
in_time = expected_time
in_values = expected_values.flatten()
self.process.input_file_handle = self.create_input_file(in_lats, in_lons, in_values, in_time)
self.process.reference_file_handle = self.create_reference_file(ref_lats, ref_lons, ref_x, ref_y)
self.process.output_file_handle = netCDF4.Dataset("output", mode="w", diskless=True)
self.process.convert_jules_1d_to_thredds_2d_for_chess()
self.assert_that_variables_are_as_expected(expected_lats, expected_lons, expected_values, expected_x,
expected_y, expected_time)
def test_GIVEN_file_with_4x3x2x1_points_in_WHEN_convert_THEN_return_file_with_all_4x3x2x1_points_populated_and_time_var_is_Time(self):
ref_x = np.array([1000, 2000])
ref_y = np.array([1001])
ref_lats = np.array(
[[49.22, 49.32]])
ref_lons = np.array(
[[7.22, 7.32]])
expected_x = np.array([1000, 2000])
expected_y = np.array([1001])
expected_pusedo = np.array([10.0, 11.0, 12.0, 13.0])
expected_time = np.array([4.0, 5.0, 6.0])
expected_lats = np.array([[49.22, 49.32]])
expected_lons = np.array([[7.22, 7.32]])
expected_values = np.arange(24).reshape((4, 3, 1, 2))
in_lats = expected_lats
in_lons = expected_lons
in_time = expected_time
in_pusedo = expected_pusedo
in_values = expected_values.flatten()
self.process.input_file_handle = self.create_input_file(in_lats, in_lons, in_values, in_time, in_pusedo)
self.process.reference_file_handle = self.create_reference_file(ref_lats, ref_lons, ref_x, ref_y)
self.process.output_file_handle = netCDF4.Dataset("output", mode="w", diskless=True)
self.process.convert_jules_1d_to_thredds_2d_for_chess()
self.assert_that_variables_are_as_expected(expected_lats, expected_lons, expected_values, expected_x,
expected_y, expected_time, expected_pusedo)
def test_GIVEN_file_with_no_points_WHEN_convert_THEN_return_file_with_no_points_populated(self):
ref_x = np.array([0, 1000, 2000])
ref_y = np.array([1, 1001, 2001])
ref_lats = np.array(
[[49.11, 49.21, 49.31],
[49.12, 49.22, 49.32],
[49.13, 49.23, 49.33]])
ref_lons = np.array(
[[7.11, 7.21, 7.31],
[7.12, 7.22, 7.32],
[7.13, 7.23, 7.33]])
in_lats = np.array([[]])
in_lons = np.array([[]])
in_values = np.array([[]])
self.process.input_file_handle = self.create_input_file(in_lats, in_lons, in_values)
self.process.reference_file_handle = self.create_reference_file(ref_lats, ref_lons, ref_x, ref_y)
self.process.output_file_handle = netCDF4.Dataset("output", mode="w", diskless=True)
assert_that(self.process.convert_jules_1d_to_thredds_2d_for_chess, raises(ProcessingError))
def test_GIVEN_file_with_meta_data_WHEN_convert_THEN_meta_data_transfered_and_crs_mapping_added(self):
ref_x = np.array([0, 1000, 2000])
ref_y = np.array([1, 1001, 2001])
ref_lats = np.array(
[[49.11, 49.21, 49.31],
[49.12, 49.22, 49.32],
[49.13, 49.23, 49.33]])
ref_lons = np.array(
[[7.11, 7.21, 7.31],
[7.12, 7.22, 7.32],
[7.13, 7.23, 7.33]])
in_lats = np.array([[49.22]])
in_lons = np.array([[7.22]])
in_values = np.array([[6]])
expected_metadata_value = "some metadata"
expected_global_metadata_value = "some globalmetadata"
self.process.input_file_handle = self.create_input_file(in_lats, in_lons, in_values)
self.process.input_file_handle.variables['values'].metadata_value = expected_metadata_value
self.process.input_file_handle.metadata_value = expected_global_metadata_value
self.process.reference_file_handle = self.create_reference_file(ref_lats, ref_lons, ref_x, ref_y)
self.process.output_file_handle = netCDF4.Dataset("output", mode="w", diskless=True)
self.process.convert_jules_1d_to_thredds_2d_for_chess(verbose=True)
assert_that(self.process.output_file_handle.variables['values'].ncattrs(), has_item('metadata_value'), "variable meta data items exists")
assert_that(self.process.output_file_handle.variables['values'].metadata_value, is_(expected_metadata_value), "variable metadata is set correctly")
assert_that(self.process.output_file_handle.ncattrs(), has_item('metadata_value'), "global meta data items exists")
assert_that(self.process.output_file_handle.metadata_value, is_(expected_global_metadata_value), "global metadata is set correctly")
assert_that(self.process.output_file_handle.ncattrs(), has_item('grid_mapping'), "grid_mapping items exists")
assert_that(self.process.output_file_handle.grid_mapping, is_('crs'), "grid mapping is set to crs")
def test_GIVEN_file_with_time_bounds_in_WHEN_convert_THEN_return_file_with_time_bound_in(self):
ref_x = np.array([1000, 2000])
ref_y = np.array([1001])
ref_lats = np.array(
[[49.22, 49.32]])
ref_lons = np.array(
[[7.22, 7.32]])
expected_x = np.array([1000, 2000])
expected_y = np.array([1001])
expected_time = np.array([4.0, 5.0, 6.0])
expected_lats = np.array([[49.22, 49.32]])
expected_lons = np.array([[7.22, 7.32]])
expected_values = np.arange(6).reshape((3, 1, 2))
in_lats = expected_lats
in_lons = expected_lons
in_time = expected_time
in_values = expected_values.flatten()
self.process.input_file_handle = self.create_input_file(in_lats, in_lons, in_values, in_time)
self.process.reference_file_handle = self.create_reference_file(ref_lats, ref_lons, ref_x, ref_y)
self.process.output_file_handle = netCDF4.Dataset("output", mode="w", diskless=True)
self.process.input_file_handle.createDimension('nt', 2)
bnds = self.process.input_file_handle.createVariable('time_bounds', 'f4', ('time', 'nt'))
expected_bnds = np.arange(2 * 3).reshape((3, 2))
bnds[:] = expected_bnds
self.process.convert_jules_1d_to_thredds_2d_for_chess()
self.assert_that_variables_are_as_expected(expected_lats, expected_lons, expected_values, expected_x,
expected_y, expected_time)
self.assert_that_arrays_almost_the_same(bnds,
self.process.output_file_handle.variables['time_bounds'],
"time bounds")
def test_GIVEN_file__WHEN_convert_THEN_check_max_min_range_is_correct(self):
ref_x = np.array([0, 1000, 2000])
ref_y = np.array([1, 1001, 2001])
ref_lats = np.array(
[[49.11, 49.21, 49.31],
[49.12, 49.22, 49.32],
[49.13, 49.23, 49.33]])
ref_lons = np.array(
[[7.11, 7.21, 7.31],
[7.12, 7.22, 7.32],
[7.13, 7.23, 7.33]])
expected_x = np.array([1000, 2000])
expected_y = np.array([1001])
expected_time = np.array([4.0, 5.0, 6.0])
expected_lats = np.array([[49.22, 49.32]])
expected_lons = np.array([[7.22, 7.32]])
expected_min = 0.8
expected_max = 22
expected_values = np.ma.array([[[10000, 7]], [[expected_min, 9]], [[expected_max, 10]]],
mask=[[[True, False]], [[False, False]], [[False, False]]])
in_lats = expected_lats
in_lons = expected_lons
in_time = expected_time
in_values = expected_values.flatten()
self.process.input_file_handle = self.create_input_file(in_lats, in_lons, in_values, in_time)
self.process.reference_file_handle = self.create_reference_file(ref_lats, ref_lons, ref_x, ref_y)
self.process.output_file_handle = netCDF4.Dataset("output", mode="w", diskless=True)
self.process.convert_jules_1d_to_thredds_2d_for_chess()
actual_range = self.process.output_file_handle.variables["values"].actual_range
assert_that(actual_range[0], close_to(expected_min, 1e-4), "actual range min")
assert_that(actual_range[1], close_to( expected_max, 1e-4), "actual range max")
def test_GIVEN_file_with_all_values_missing_WHEN_convert_THEN_max_min_range_are_not_in_file(self):
ref_x = np.array([0, 1000, 2000])
ref_y = np.array([1, 1001, 2001])
ref_lats = np.array(
[[49.11, 49.21, 49.31],
[49.12, 49.22, 49.32],
[49.13, 49.23, 49.33]])
ref_lons = np.array(
[[7.11, 7.21, 7.31],
[7.12, 7.22, 7.32],
[7.13, 7.23, 7.33]])
expected_x = np.array([1000, 2000])
expected_y = np.array([1001])
expected_time = np.array([4.0, 5.0, 6.0])
expected_lats = np.array([[49.22, 49.32]])
expected_lons = np.array([[7.22, 7.32]])
expected_min = 0.8
expected_max = 22
expected_values = np.ma.array([[[6, 7]], [[expected_min, 9]], [[expected_max, 10]]],
mask=[[[True, True]], [[True, True]], [[True, True]]])
in_lats = expected_lats
in_lons = expected_lons
in_time = expected_time
in_values = expected_values.flatten()
self.process.input_file_handle = self.create_input_file(in_lats, in_lons, in_values, in_time)
self.process.reference_file_handle = self.create_reference_file(ref_lats, ref_lons, ref_x, ref_y)
self.process.output_file_handle = netCDF4.Dataset("output", mode="w", diskless=True)
self.process.convert_jules_1d_to_thredds_2d_for_chess()
assert_that(self.process.output_file_handle.variables["values"].ncattrs(), is_not(has_item('actual_range')), "actual range should not be in file because there is not data")
