def test_run_total_column_water_vapour(self):
self.dataset = xr.Dataset()
data = DefaultData.create_default_array_3d(3,
3,
11,
np.float32,
fill_value=np.NaN)
data[0, :, :] = 11.52932
data[1, :, :] = 11.529235
data[2, :, :] = 11.52744
self.dataset["amsre.nwp.log_surface_pressure"] = Variable(
["matchup_count", "ny", "nx"], data)
data = DefaultData.create_default_array_4d(3,
3,
60,
11,
np.float32,
fill_value=np.NaN)
data[0, :, 1, 1] = np.float32(
np.array([
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
1.3609621E-8, 3.7061102E-7, 4.273528E-6, 2.6397798E-5,
3.5652188E-6, 1.5417224E-4, 1.9648654E-4, 9.139678E-5,
8.232285E-6, 6.5089694E-7, 4.0974975E-9, 0.0
]))
data[1, :, 1, 1] = np.float32(
np.array([
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 1.0075588E-7, 1.054102E-6, 1.1424428E-5, 2.2602183E-6,
1.546818E-4, 1.9482679E-4, 8.667531E-5, 4.7594845E-6,
2.708914E-9, 0.0, 0.0
]))
data[2, :, 1, 1] = np.float32(
np.array([
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
2.7519864E-9, 4.7594305E-8, 1.3165399E-6, 1.2988783E-4,
8.855922E-6, 3.5680281E-7, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
]))
self.dataset["amsre.nwp.cloud_liquid_water"] = Variable(
["matchup_count", "cloud_layers", "ny", "nx"], data)
prep_data = self.preprocessor.run(self.dataset)
variable = prep_data.variables["amsre.nwp.total_column_liquid_water"]
self.assertEqual((11, ), variable.shape)
self.assertAlmostEqual(0.064784802, variable.data[0], 8)
self.assertAlmostEqual(0.059696566, variable.data[1], 8)
self.assertAlmostEqual(0.028990015, variable.data[2], 8)
def test_run(self):
dataset = xr.Dataset()
for i in range(0, len(self.BT_VARIABLE_NAMES)):
data = DefaultData.create_default_vector(13,
np.float32,
fill_value=26)
dataset[self.BT_VARIABLE_NAMES[i]] = Variable(["matchup"], data)
bt_bias_correction = BtBiasCorrection()
bt_bias_correction.run(dataset)
self.assertAlmostEqual(
26.3524, dataset["amsre.brightness_temperature6V"].data[0], 6)
self.assertAlmostEqual(
26.0793, dataset["amsre.brightness_temperature6H"].data[1], 6)
self.assertAlmostEqual(
25.8907, dataset["amsre.brightness_temperature10V"].data[2], 5)
self.assertAlmostEqual(
25.2479, dataset["amsre.brightness_temperature10H"].data[3], 6)
self.assertAlmostEqual(
26.6228, dataset["amsre.brightness_temperature18V"].data[4], 5)
self.assertAlmostEqual(
26.2794, dataset["amsre.brightness_temperature18H"].data[5], 6)
self.assertAlmostEqual(
26.02, dataset["amsre.brightness_temperature23V"].data[6], 5)
self.assertAlmostEqual(
25.7331, dataset["amsre.brightness_temperature23H"].data[7], 5)
self.assertAlmostEqual(
25.646, dataset["amsre.brightness_temperature36V"].data[8], 6)
self.assertAlmostEqual(
26.1464, dataset["amsre.brightness_temperature36H"].data[9], 5)
def test_run_average_variables_masks_invalid(self):
fill_value = -79
data = DefaultData.create_default_array_3d(9, 9, 4, np.float32)
data[0, 2, :] = 104
data[0, 3, :] = 105
data[0, 4, :] = 106
data[0, 5, :] = 107
data[0, 6, :] = 108
data[0, 3, 4] = fill_value
data[0, 4, 6] = fill_value
variable = Variable(["matchup_count", "ny", "nx"], data)
variable.attrs["_FillValue"] = fill_value
self.dataset["amsre.brightness_temperature6H"] = variable
flag_coding = FlagCoding(4)
prep_data = self.preprocessor.run(self.dataset,
flag_coding=flag_coding)
variable = prep_data.variables["amsre.brightness_temperature6H"]
self.assertEqual((4, ), variable.shape)
self.assertAlmostEqual(106.0, variable.data[0], 5)
flags = flag_coding.get_flags()
self.assertEqual(0, flags[0])
def test_create_default_array(self):
array = DefaultData.create_default_array(5,
4,
np.float32,
fill_value=np.NaN)
self.assertEqual((4, 5), array.shape)
self.assertTrue(np.isnan(array.data[0, 1]))
def test_run_stddev_variables_too_many_fill_values(self):
data = DefaultData.create_default_array_3d(21, 21, 5, np.float32)
for x in range(0, 21):
for y in range(0, 21):
data[:, y, x] = x + y
data[1, :, 1] = -81
data[1, :, 2] = -81
data[1, :, 5] = -81
variable = Variable(["matchup_count", "ny", "nx"], data)
variable.attrs["_FillValue"] = -81
self.dataset["amsre.brightness_temperature36V"] = variable
flag_coding = FlagCoding(5)
prep_data = self.preprocessor.run(self.dataset,
flag_coding=flag_coding)
variable = prep_data.variables[
"amsre.brightness_temperature36V_stddev"]
self.assertEqual((5, ), variable.shape)
self.assertAlmostEqual(8.563489, variable.data[0], 7)
self.assertAlmostEqual(-81, variable.data[1], 7)
self.assertAlmostEqual(8.563489, variable.data[2], 7)
flags = flag_coding.get_flags()
self.assertEqual(1, flags[1])
def test_run_stddev_variables_masks_fill_value(self):
data = DefaultData.create_default_array_3d(21, 21, 5, np.float32)
for x in range(0, 21):
for y in range(0, 21):
data[:, y, x] = (x + y) * 1.0
data[:, 1, 1] = -80
data[:, 10, 10] = -80
data[:, 15, 15] = -80
data[:, 20, 20] = -80
variable = Variable(["matchup_count", "ny", "nx"], data)
variable.attrs["_FillValue"] = -80
self.dataset["amsre.brightness_temperature23H"] = variable
flag_coding = FlagCoding(5)
prep_data = self.preprocessor.run(self.dataset,
flag_coding=flag_coding)
variable = prep_data.variables[
"amsre.brightness_temperature23H_stddev"]
self.assertEqual((5, ), variable.shape)
self.assertAlmostEqual(8.4922457, variable.data[1], 7)
flags = flag_coding.get_flags()
self.assertEqual(0, flags[1])
def _create_vector_float32_variable(num_matchups):
array = DefaultData.create_default_vector(num_matchups,
np.float32,
fill_value=np.NaN)
variable = Variable(["matchup"], array)
variable.attrs["_FillValue"] = np.NaN
return variable
def setUp(self):
self.preprocessor = Preprocessor()
self.dataset = xr.Dataset()
data = DefaultData.create_default_array_3d(5, 5, 11, np.int8)
self.dataset["amsre.nwp.log_surface_pressure"] = Variable(
["macwenameifdifferentlyhere", "ny_test", "nx_test"], data)
data = DefaultData.create_default_array_4d(5,
5,
60,
11,
np.float32,
fill_value=np.NaN)
self.dataset["amsre.nwp.cloud_liquid_water"] = Variable([
"macwenameifdifferentlyhere", "cloud_layers", "ny_test", "nx_test"
], data)
def test_run_windspeed_and_relative_angle(self):
data = DefaultData.create_default_array_3d(3,
3,
11,
np.float32,
fill_value=np.NaN)
data[0, :, :] = 2
data[1, :, :] = 3
data[2, :, :] = 4
self.dataset["amsre.nwp.10m_east_wind_component"] = Variable(
["matchup_count", "ny", "nx"], data)
data = DefaultData.create_default_array_3d(3,
3,
11,
np.float32,
fill_value=np.NaN)
data[0, :, :] = 5
data[1, :, :] = 6
data[2, :, :] = 7
self.dataset["amsre.nwp.10m_north_wind_component"] = Variable(
["matchup_count", "ny", "nx"], data)
data = DefaultData.create_default_array_3d(3,
3,
11,
np.float32,
fill_value=np.NaN)
data[0, :, :] = 8
data[1, :, :] = 9
data[2, :, :] = 10
self.dataset["amsre.satellite_azimuth_angle"] = Variable(
["matchup_count", "ny", "nx"], data)
prep_data = self.preprocessor.run(self.dataset)
variable = prep_data.variables["amsre.nwp.abs_wind_speed"]
self.assertEqual((11, ), variable.shape)
self.assertAlmostEqual(5.3851647, variable.data[0], 7)
self.assertAlmostEqual(6.7082038, variable.data[1], 7)
self.assertAlmostEqual(8.0622578, variable.data[2], 7)
variable = prep_data.variables["relative_angle"]
self.assertEqual((11, ), variable.shape)
self.assertAlmostEqual(346.19858, variable.data[0], 5)
self.assertAlmostEqual(342.43494, variable.data[1], 5)
self.assertAlmostEqual(340.25513, variable.data[2], 5)
def test_scale_data_no_scaling(self):
array = DefaultData.create_default_array(6, 12, np.int16)
array[0, 0] = 22
array[1, 0] = 23
variable = Variable(["y", "x"], array)
MmdReader._scale_data(variable)
self.assertEqual(22, variable.data[0, 0])
self.assertEqual(23, variable.data[1, 0])
def test_get_default_fill_value(self):
self.assertEqual(-127, DefaultData.get_default_fill_value(np.int8))
self.assertEqual(255, DefaultData.get_default_fill_value(np.uint8))
self.assertEqual(-32767, DefaultData.get_default_fill_value(np.int16))
self.assertEqual(np.uint16(-1),
DefaultData.get_default_fill_value(np.uint16))
self.assertEqual(-2147483647,
DefaultData.get_default_fill_value(np.int32))
self.assertEqual(4294967295,
DefaultData.get_default_fill_value(np.uint32))
self.assertEqual(-9223372036854775806,
DefaultData.get_default_fill_value(np.int64))
self.assertEqual(np.float32(9.96921E36),
DefaultData.get_default_fill_value(np.float32))
self.assertEqual(9.969209968386869E36,
DefaultData.get_default_fill_value(np.float64))
def test_run_dimension_squeezed_variables(self):
data = DefaultData.create_default_array_3d(1, 1, 7, np.int32)
data[3, 0, 0] = 78
self.dataset["insitu.time"] = Variable(["matchup_count", "ny", "nx"],
data)
data = DefaultData.create_default_array_3d(1, 1, 7, np.float32)
data[4, 0, 0] = 0.89
self.dataset["insitu.lon"] = Variable(["matchup_count", "ny", "nx"],
data)
prep_data = self.preprocessor.run(self.dataset)
variable = prep_data.variables["insitu.time"]
self.assertEqual((7, ), variable.shape)
self.assertEqual(78, variable.data[3])
variable = prep_data.variables["insitu.lon"]
self.assertEqual((7, ), variable.shape)
self.assertAlmostEqual(0.89, variable.data[4], 7)
def test_scale_data_only_offset(self):
array = DefaultData.create_default_array(6, 10, np.int16)
array[4, 0] = 22
array[5, 0] = 23
variable = Variable(["y", "x"], array)
variable.attrs["OFFSET"] = -10
MmdReader._scale_data(variable)
self.assertAlmostEqual(12, variable.data[4, 0], 8)
self.assertAlmostEqual(13, variable.data[5, 0], 8)
def test_run_fetch_center_pixel_convert_to_Celsius_and_apply_bias(self):
data = DefaultData.create_default_array_3d(5, 5, 11, np.float32)
data[5, 2, 2] = 279.6853
self.dataset["amsre.nwp.sea_surface_temperature"] = Variable(
["matchup_count", "ny", "nx"], data)
prep_data = self.preprocessor.run(self.dataset)
variable = prep_data.variables["amsre.nwp.sea_surface_temperature"]
self.assertEqual((11, ), variable.shape)
self.assertAlmostEqual(6.4853086, variable.data[5], 7)
def test_scale_data_only_scaling(self):
array = DefaultData.create_default_array(6, 11, np.int16)
array[2, 0] = 22
array[3, 0] = 23
variable = Variable(["y", "x"], array)
variable.attrs["SCALE_FACTOR"] = 0.2
MmdReader._scale_data(variable)
self.assertAlmostEqual(4.4, variable.data[2, 0], 8)
self.assertAlmostEqual(4.6, variable.data[3, 0], 8)
def test_run_fetch_center_pixel_variables(self):
data = DefaultData.create_default_array_3d(5, 5, 11, np.int16)
data[5, 2, 2] = 101
self.dataset["amsre.pixel_data_quality6V"] = Variable(
["matchup_count", "ny", "nx"], data)
data = DefaultData.create_default_array_3d(5, 5, 11, np.float64)
data[6, 2, 2] = 1.02
self.dataset["amsre.nwp.total_column_water_vapour"] = Variable(
["matchup_count", "ny", "nx"], data)
prep_data = self.preprocessor.run(self.dataset)
variable = prep_data.variables["amsre.pixel_data_quality6V"]
self.assertEqual((11, ), variable.shape)
self.assertEqual(101, variable.data[5])
variable = prep_data.variables["amsre.nwp.total_column_water_vapour"]
self.assertEqual((11, ), variable.shape)
self.assertAlmostEqual(1.02, variable.data[6], 8)
def test_scale_data_scaling_and_offset(self):
array = DefaultData.create_default_array(6, 9, np.int16)
array[0, 1] = 22
array[1, 1] = 23
variable = Variable(["y", "x"], array)
variable.attrs["SCALE_FACTOR"] = 0.2
variable.attrs["OFFSET"] = 1.6
MmdReader._scale_data(variable)
self.assertAlmostEqual(6.0, variable.data[0, 1], 8)
self.assertAlmostEqual(6.2, variable.data[1, 1], 8)
def calculate_pressure_levels(self, sea_level_pressure):
num_levels = len(self.A_COEFFS) - 1
num_measures = len(sea_level_pressure)
target_data = DefaultData.create_default_array(num_levels,
num_measures,
np.float32,
fill_value=np.NaN)
for i in range(0, num_measures):
p_f = np.float64(
sea_level_pressure[i]) * self.B_COEFFS + self.A_COEFFS
target_data[i, :] = p_f[1:61] - p_f[0:60]
return target_data
def _create_2d_float_variable(max_iterations,
num_matchups,
dims_names=None):
array = DefaultData.create_default_array(max_iterations,
num_matchups,
np.float32,
dims_names=dims_names,
fill_value=np.NaN)
if dims_names is None:
variable = Variable(["matchup", "iterations"], array)
else:
variable = Variable(dims_names, array)
variable.attrs["_FillValue"] = np.NaN
return variable
def test_create_T_A(self):
dataset = xr.Dataset()
for i in range(0, len(self.BT_VARIABLE_NAMES)):
data = DefaultData.create_default_vector(13,
np.float32,
fill_value=i * 10)
dataset[self.BT_VARIABLE_NAMES[i]] = Variable(["matchup_count"],
data)
T_A = self.retrieval.create_T_A(dataset)
self.assertAlmostEqual(0, T_A[0, 0], 8)
self.assertAlmostEqual(10, T_A[0, 1], 8)
self.assertAlmostEqual(20, T_A[0, 2], 8)
self.assertAlmostEqual(30, T_A[0, 3], 8)
self.assertAlmostEqual(90, T_A[0, 9], 8)
def test_run_stddev_variables(self):
data = DefaultData.create_default_array_3d(21, 21, 5, np.float32)
for x in range(0, 21):
for y in range(0, 21):
data[:, y, x] = (x + y) * 0.23
variable = Variable(["matchup_count", "ny", "nx"], data)
variable.attrs["_FillValue"] = -79
self.dataset["amsre.brightness_temperature23V"] = variable
flag_coding = FlagCoding(5)
prep_data = self.preprocessor.run(self.dataset,
flag_coding=flag_coding)
variable = prep_data.variables[
"amsre.brightness_temperature23V_stddev"]
self.assertEqual((5, ), variable.shape)
self.assertAlmostEqual(1.9696023, variable.data[0], 7)
flags = flag_coding.get_flags()
self.assertEqual(0, flags[0])
def test_run_average_variables(self):
data = DefaultData.create_default_array_3d(7, 7, 5, np.float32)
data[0, 1, :] = 102
data[0, 2, :] = 101
data[0, 3, :] = 103
data[0, 4, :] = 105
data[0, 5, :] = 104
variable = Variable(["matchup_count", "ny", "nx"], data)
variable.attrs["_FillValue"] = -78
self.dataset["amsre.brightness_temperature6V"] = variable
flag_coding = FlagCoding(5)
prep_data = self.preprocessor.run(self.dataset,
flag_coding=flag_coding)
variable = prep_data.variables["amsre.brightness_temperature6V"]
self.assertEqual((5, ), variable.shape)
self.assertEqual(103, variable.data[0])
flags = flag_coding.get_flags()
self.assertEqual(0, flags[0])
def _add_fill_value_attributes(variable, target_variable_name):
if "brightness_temperature" in target_variable_name:
offset = variable.attrs["OFFSET"]
scale_factor = variable.attrs["SCALE_FACTOR"]
variable.attrs["_FillValue"] = -32768.0 * scale_factor + offset
return
if "insitu." in target_variable_name:
variable.attrs["_FillValue"] = -32768
return
if "source" in variable.attrs:
variable.attrs["_FillValue"] = 2e20
return
if not "FillValue" in variable.attrs:
default_fill = DefaultData.get_default_fill_value(variable.dtype)
scale_factor, offset = MmdReader.get_scale_and_offset(variable)
if scale_factor != 1.0 or offset != 0.0:
variable.attrs["_FillValue"] = default_fill * scale_factor + offset
else:
variable.attrs["_FillValue"] = default_fill
return
def _create_variable(self, data_type):
array = DefaultData.create_default_array(2, 2, data_type)
variable = Variable(["y", "x"], array)
return variable
def _create_dataset_with_variable(self, variable_name):
dataset = xr.Dataset()
array = DefaultData.create_default_array(2, 2, np.int16)
variable = Variable(["y", "x"], array)
dataset[variable_name] = variable
return dataset
def test_add_fill_value_attributes_no_scaling_no_fill_value_float32(self):
variable = self._create_variable(np.float32)
MmdReader._add_fill_value_attributes(variable, "don't care")
self.assertEqual(DefaultData.get_default_fill_value(np.float32),
variable.attrs["_FillValue"])
def _create_vector_uint8_variable(num_matchups):
array = DefaultData.create_default_vector(num_matchups, np.uint8)
variable = Variable(["matchup"], array)
variable.attrs["_FillValue"] = DefaultData.get_default_fill_value(
np.uint8)
return variable