def test_number_constants_with_scientific_notation(self):
expression = 'a + 257e-3'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = xr.Variable(['x'], np.asarray([1, 2, 3, 4, 0.01]))
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['x'], [1.257, 2.257, 3.257, 4.257, 0.267])
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_not_with_vertical_1D(self):
expression = '~ a' # the meaning is "not a"
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = get_vertical_one_dim_bool_variable()
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['y'], np.asarray([t, f, t]))
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_inverse_tangent(self):
expression = 'arctan(tan(a))'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = xr.Variable(('x',), [1, 0.5, 0])
R._load_virtual_variable(ds, 'v')
expected = ds['a']
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_replace_pi(self):
expression = 'a*pi'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = xr.Variable(['x'], np.asarray([1, 2, 3, 4, 0.01]))
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['x'], [3.14159265359, 6.28318530718, 9.42477796077, 12.5663706144, 0.0314159265359])
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_absolute_value(self):
expression = 'abs(a)'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = xr.Variable(('x',), [-2.2, -3.4, 4.5 * -1])
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(('x',), [2.2, 3.4, 4.5])
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_square_root(self):
expression = 'sqrt(a)'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = xr.Variable(('x',), [4, 9, math.pow(12, 2)])
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(('x',), [2, 3, 12])
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_unary_negation(self):
expression = '- a'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = get_two_dim_int_variable()
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['y', 'x'], np.asarray(
[[-10, -20, -30, -40], [-13, -23, -33, -43], [-16, -26, -36, -46]]))
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_modulo(self):
expression = 'a % 7'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = get_two_dim_int_variable()
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['y', 'x'], np.asarray(
[[3, 6, 2, 5], [6, 2, 5, 1], [2, 5, 1, 4]]))
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_power(self):
expression = 'a ** 2'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = get_two_dim_int_variable()
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['y', 'x'], np.asarray(
[[100, 400, 900, 1600], [169, 529, 1089, 1849], [256, 676, 1296, 2116]]))
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_or_with_vertical_1D(self):
expression = 'a | b' # the meaning is "a or b"
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = get_vertical_one_dim_bool_variable()
ds['b'] = get_two_dim_bool_variable()
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['y', 'x'], np.asarray([[t, f, f, t], [t, t, t, t], [t, f, f, f]]))
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_exponential_minus_one(self):
expression = 'expm1(a)'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = xr.Variable(('x',), [2.2, 3.4, 4.5])
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(('x',), [math.expm1(2.2),
math.expm1(3.4),
math.expm1(4.5), ])
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_natural_logarithm_1_plus_x(self):
expression = 'log1p(a)'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = xr.Variable(('x',), [2.2, 3.4, 4.5])
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(('x',), [math.log1p(2.2),
math.log1p(3.4),
math.log1p(4.5), ])
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_inverse_hyperbolic_tangent(self):
expression = 'arctanh(a)'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = xr.Variable(('x',), [0.2, 0.4, 0.5])
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(('x',), [math.atanh(0.2),
math.atanh(0.4),
math.atanh(0.5), ])
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_equal(self):
expression = 'a == 5'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = get_three_dim_int_variable()
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['z', 'y', 'x'], np.asarray(
[[[f, f, f, f], [f, f, f, t], [f, f, t, f]],
[[f, t, f, f], [t, f, f, f], [f, f, f, f]]]))
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_multiplication(self):
expression = 'a * b'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = get_two_dim_int_variable()
ds['b'] = get_vertical_one_dim_int_variable()
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['y', 'x'], np.asarray(
[[2000, 4000, 6000, 8000], [3900, 6900, 9900, 12900], [6400, 10400, 14400, 18400]]))
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_subtraction(self):
expression = 'b - a'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = get_two_dim_int_variable()
ds['b'] = get_vertical_one_dim_int_variable()
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['y', 'x'], np.asarray(
[[190, 180, 170, 160], [287, 277, 267, 257], [384, 374, 364, 354]]))
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_addition(self):
expression = 'a + b'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = get_two_dim_int_variable()
ds['b'] = get_vertical_one_dim_int_variable()
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['y', 'x'], np.asarray(
[[210, 220, 230, 240], [313, 323, 333, 343], [416, 426, 436, 446]]))
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_inverse_hyperbolic_cosine(self):
expression = 'arccosh(a)'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = xr.Variable(('x',), [2, 4, 5])
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(('x',), [math.acosh(2),
math.acosh(4),
math.acosh(5), ])
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_dont_replace_pi_if_it_part_of_a_word(self):
expression = 'lpit + pit + lpi'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['lpit'] = xr.Variable(['x'], np.asarray([1, 2, 3, 4, 0.01]))
ds['pit'] = xr.Variable(['x'], np.asarray([1, 2, 3, 4, 0.01]))
ds['lpi'] = xr.Variable(['x'], np.asarray([1, 2, 3, 4, 0.01]))
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['x'], [3, 6, 9, 12, 0.03])
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_arctan2(self):
expression = 'arctan2(a, b)'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = xr.Variable(('x',), [0.2, 0.4, 0.5])
ds['b'] = xr.Variable(('x',), [0.8, 0.7, 0.6])
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(('x',), [math.atan2(0.2, 0.8),
math.atan2(0.4, 0.7),
math.atan2(0.5, 0.6), ])
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_sinus(self):
expression = 'sin(a)'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = get_one_dim_radians()
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['x'], np.asarray(
[0,
0.70710678,
0.989821441881,
0.70710678,
-0.70710678,
-0.989821441881]))
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_cosine(self):
expression = 'cos(a)'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = get_one_dim_radians()
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['x'], np.asarray(
[1,
0.70710678,
0.142314838273,
-0.70710678,
-0.70710678,
-0.142314838273]))
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_tangent(self):
expression = 'tan(a)'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = get_one_dim_radians()
R._load_virtual_variable(ds, 'v')
expected = xr.Variable(['x'], np.asarray(
[0,
1,
6.95515277177,
-1,
1,
6.95515277177]))
tu.assert_array_equals_with_index_error_message(self, expected, ds['v'])
def test_division(self):
expression = 'b / a'
ds = xr.Dataset()
ds['v'] = create_virtual_variable(expression)
ds['a'] = get_two_dim_int_variable()
ds['b'] = get_vertical_one_dim_int_variable()
R._load_virtual_variable(ds, 'v')
loaded_ = ds['v']
# if two arrays of type integer are divided, the result data type can be integer or float
# the expected preparation depends on returned datatype
if np.issubdtype(loaded_.dtype, np.integer):
expected = xr.Variable(['y', 'x'], np.asarray(
[[20, 10, 6, 5],
[23, 13, 9, 6],
[25, 15, 11, 8]]))
else:
expected = xr.Variable(['y', 'x'], np.asarray(
[[20, 10, 6.666666666666667, 5],
[23.076923076923077, 13.043478260869565, 9.090909090909092, 6.976744186046512],
[25, 15.384615384615385, 11.11111111111111, 8.695652173913043]]))
tu.assert_array_equals_with_index_error_message(self, expected, loaded_)
class FCDRReaderTest(ut.TestCase):
def setUp(self):
self.fcdr_reader = FCDRReader()
def test_adding_2_three_dimensional_variables(self):
ds = xr.Dataset()
ds['a'] = create_three_dim_variable()
ds['b'] = create_three_dim_variable()
v_var = create_virtual_variable("a + b")
ds["v_var"] = v_var
self.fcdr_reader._load_virtual_variable(ds, 'v_var')
self.assertTrue('v_var' in ds)
virtual_loaded = ds['v_var']
self.assertEqual(('z', 'y', 'x'), virtual_loaded.dims)
self.assertEqual((4, 2, 3), virtual_loaded.shape)
self.assertEqual(v_var.attrs, virtual_loaded.attrs)
expected = np.asarray([[[2, 4, 6], [2.2, 4.2, 6.2]], [[22, 24, 26], [22.2, 24.2, 26.2]], [[42, 44, 46], [42.2, 44.2, 46.2]], [[62, 64, 66], [62.2, 64.2, 66.2]]])
actual = ds['v_var'].values
self.assertEqual(type(expected), type(actual))
tu.assert_array_equals_with_index_error_message(self, expected, actual)
def test_adding_1_three_dimensional_variable_and_1_two_dimensional_variable(self):
ds = xr.Dataset()
ds['a'] = create_three_dim_variable()
ds['b'] = create_two_dim_variable()
v_var = create_virtual_variable("a + b")
ds["v_var"] = v_var
self.fcdr_reader._load_virtual_variable(ds, 'v_var')
self.assertTrue('v_var' in ds)
virtual_loaded = ds['v_var']
self.assertEqual(('z', 'y', 'x'), virtual_loaded.dims)
self.assertEqual((4, 2, 3), virtual_loaded.shape)
self.assertEqual(v_var.attrs, virtual_loaded.attrs)
expected = np.asarray([[[2, 4, 6], [5.1, 7.1, 9.1]], [[12, 14, 16], [15.1, 17.1, 19.1]], [[22, 24, 26], [25.1, 27.1, 29.1]], [[32, 34, 36], [35.1, 37.1, 39.1]]])
actual = virtual_loaded.values
self.assertEqual(type(expected), type(actual))
tu.assert_array_equals_with_index_error_message(self, expected, actual)
def test_adding_1_three_dimensional_variable_and_1_one_dimensional_variable(self):
ds = xr.Dataset()
ds['a'] = create_three_dim_variable()
ds['b'] = create_one_dim_variable()
v_var = create_virtual_variable("a + b")
ds["v_var"] = v_var
self.fcdr_reader._load_virtual_variable(ds, 'v_var')
self.assertTrue('v_var' in ds)
virtual_loaded = ds['v_var']
self.assertEqual(('z', 'y', 'x'), virtual_loaded.dims)
self.assertEqual((4, 2, 3), virtual_loaded.shape)
self.assertEqual(v_var.attrs, virtual_loaded.attrs)
expected = np.asarray([[[2.3, 4.3, 6.3], [2.4, 4.4, 6.4]], [[12.3, 14.3, 16.3], [12.4, 14.4, 16.4]], [[22.3, 24.3, 26.3], [22.4, 24.4, 26.4]], [[32.3, 34.3, 36.3], [32.4, 34.4, 36.4]]])
actual = virtual_loaded.values
self.assertEqual(type(expected), type(actual))
tu.assert_array_equals_with_index_error_message(self, expected, actual)
def test_adding_1_three_dimensional_variable_and_1_vertical_one_dimensional_variable(self):
ds = xr.Dataset()
ds['a'] = create_three_dim_variable()
ds['b'] = create_vertical_one_dim_variable()
v_var = create_virtual_variable("a + b")
ds["v_var"] = v_var
self.fcdr_reader._load_virtual_variable(ds, 'v_var')
self.assertTrue('v_var' in ds)
virtual_loaded = ds['v_var']
self.assertEqual(('z', 'y', 'x'), virtual_loaded.dims)
self.assertEqual((4, 2, 3), virtual_loaded.shape)
self.assertEqual(v_var.attrs, virtual_loaded.attrs)
expected = np.asarray([[[6, 7, 8], [7.1, 8.1, 9.1]], [[16, 17, 18], [17.1, 18.1, 19.1]], [[26, 27, 28], [27.1, 28.1, 29.1]], [[36, 37, 38], [37.1, 38.1, 39.1]]])
actual = virtual_loaded.values
self.assertEqual(type(expected), type(actual))
tu.assert_array_equals_with_index_error_message(self, expected, actual)
def test_multiplying_1_three_dimensional_variable_and_1_scalar_variable(self):
ds = xr.Dataset()
ds['a'] = create_three_dim_variable()
ds['b'] = create_scalar_variable(2.7)
v_var = create_virtual_variable("a * b")
ds["v_var"] = v_var
self.fcdr_reader._load_virtual_variable(ds, 'v_var')
self.assertTrue('v_var' in ds)
virtual_loaded = ds['v_var']
self.assertEqual(('z', 'y', 'x'), virtual_loaded.dims)
self.assertEqual((4, 2, 3), virtual_loaded.shape)
self.assertEqual(v_var.attrs, virtual_loaded.attrs)
expected = np.asarray(
[[[2.7, 5.4, 8.1], [2.97, 5.67, 8.37]], [[29.7, 32.4, 35.1], [29.97, 32.67, 35.37]], [[56.7, 59.4, 62.1], [56.97, 59.67, 62.37]], [[83.7, 86.4, 89.1], [83.97, 86.67, 89.37]]])
actual = virtual_loaded.values
self.assertEqual(type(expected), type(actual))
tu.assert_array_equals_with_index_error_message(self, expected, actual)
def test_multiplying_2d_array_and_tiepoint_array(self):
ds = xr.Dataset()
ds['a'] = create_two_dim_ones(10)
ds['b'] = create_two_dim_tie_points_variable()
ds["v_var"] = create_virtual_variable("a * b")
self.fcdr_reader._load_virtual_variable(ds, 'v_var')
def test_prepare_virtual_variables(self):
ds = xr.Dataset()
ds['a'] = create_three_dim_variable()
ds['b'] = create_two_dim_variable()
ds["v_var"] = create_virtual_variable("a + b")
self.fcdr_reader._load_virtual_variable(ds, 'v_var')
self.assertEqual(3, len(ds.data_vars))
expected = np.full([4, 2, 3], 1, np.int32)
actual = ds['v_var'].values
self.assertEqual(type(expected), type(actual))
self.assertEqual(expected.shape, actual.shape)
def test_is_already_loaded(self):
virtual_variable = create_virtual_variable("alpha * beta")
self.assertFalse(self.fcdr_reader._is_already_loaded(virtual_variable))
variable = create_three_dim_variable()
self.assertTrue(self.fcdr_reader._is_already_loaded(variable))
def test_get_num_data_elements_virtual(self):
virtual_variable = create_virtual_variable("alpha * beta")
self.assertEqual(1, self.fcdr_reader._get_num_data_elements(virtual_variable))
def test_get_num_data_elements_one_d(self):
one_d_variable = create_one_dim_variable()
self.assertEqual(3, self.fcdr_reader._get_num_data_elements(one_d_variable))
def test_get_num_data_elements_two_d(self):
two_d_variable = create_two_dim_variable()
self.assertEqual(6, self.fcdr_reader._get_num_data_elements(two_d_variable))
def test_get_num_data_elements_three_d(self):
three_d_variable = create_three_dim_variable()
self.assertEqual(24, self.fcdr_reader._get_num_data_elements(three_d_variable))
class FCDRReaderMviriVirtualVariablesTest(unittest.TestCase):
def setUp(self):
self.fcdr_reader = FCDRReader()
fcdr_writer = FCDRWriter()
self.dataset = fcdr_writer.createTemplateFull("MVIRI", 5000)
def testCalculate_sensitivitiy_solar_irradiance(self):
self.dataset["distance_sun_earth"].data = 1.0166579484939575
self.dataset["count_vis"].data[:, :] = 63
self.dataset["mean_count_space_vis"] = 4.961684375
self.dataset["a0_vis"].data = 0.9800095200636486
self.dataset["a1_vis"].data = 0.01179638707394702
self.dataset["a2_vis"].data = 0.001179638707394702
self.dataset["years_since_launch"].data = 8.830136986301369
self.dataset["solar_zenith_angle"].data[:, :] = 49.3436
self.dataset["solar_irradiance_vis"].data = 688.144781045
self.fcdr_reader._load_virtual_variable(
self.dataset, "sensitivity_solar_irradiance_vis")
virtual_variable = self.dataset["sensitivity_solar_irradiance_vis"]
self.assertIsNotNone(virtual_variable)
self.assertEqual((5000, 5000), virtual_variable.shape)
self.assertAlmostEqual(0.00071843801878964449,
virtual_variable.data[0, 0])
def testCalculate_sensitivitiy_count_vis(self):
self.dataset["distance_sun_earth"].data = 1.0166579484939575
self.dataset["a0_vis"].data = 0.9800095200636486
self.dataset["a1_vis"].data = 0.01179638707394702
self.dataset["a2_vis"].data = 0.02179638707394702
self.dataset["years_since_launch"].data = 8.830136986301369
self.dataset["solar_zenith_angle"].data[:, :] = 35.4024
self.dataset["solar_irradiance_vis"].data = 688.144781045
self.fcdr_reader._load_virtual_variable(self.dataset,
"sensitivity_count_vis")
virtual_variable = self.dataset["sensitivity_count_vis"]
self.assertIsNotNone(virtual_variable)
self.assertEqual((500, 500), virtual_variable.shape)
self.assertAlmostEqual(0.016114772327457307, virtual_variable.data[1,
1])
def testCalculate_sensitivitiy_count_space(self):
self.dataset["distance_sun_earth"].data = 1.0166579484939575
self.dataset["a0_vis"].data = 0.9800095200636486
self.dataset["a1_vis"].data = 0.01179638707394702
self.dataset["a2_vis"].data = 0.01379638707394702
self.dataset["years_since_launch"].data = 8.830136986301369
self.dataset["solar_zenith_angle"].data[:, :] = 26.9212
self.dataset["solar_irradiance_vis"].data = 688.144781045
self.fcdr_reader._load_virtual_variable(self.dataset,
"sensitivity_count_space")
virtual_variable = self.dataset["sensitivity_count_space"]
self.assertIsNotNone(virtual_variable)
self.assertEqual((500, 500), virtual_variable.shape)
self.assertAlmostEqual(-0.011430569161934807, virtual_variable.data[2,
2])
def testCalculate_sensitivitiy_a0_vis(self):
self.dataset["distance_sun_earth"].data = 1.0166579484939575
self.dataset["count_vis"].data[:, :] = 14
self.dataset["mean_count_space_vis"] = 4.961684375
self.dataset["solar_zenith_angle"].data[:, :] = 18.2038
self.dataset["solar_irradiance_vis"].data = 688.144781045
self.fcdr_reader._load_virtual_variable(self.dataset,
"sensitivity_a0_vis")
virtual_variable = self.dataset["sensitivity_a0_vis"]
self.assertIsNotNone(virtual_variable)
self.assertEqual((5000, 5000), virtual_variable.shape)
self.assertAlmostEqual(0.044895821172659549, virtual_variable.data[3,
3])
def testCalculate_sensitivitiy_a1_vis(self):
self.dataset["distance_sun_earth"].data = 1.0166579484939575
self.dataset["years_since_launch"].data = 8.830136986301369
self.dataset["count_vis"].data[:, :] = 24
self.dataset["mean_count_space_vis"] = 4.961684375
self.dataset["solar_zenith_angle"].data[:, :] = 22.1907
self.dataset["solar_irradiance_vis"].data = 688.144781045
self.fcdr_reader._load_virtual_variable(self.dataset,
"sensitivity_a1_vis")
virtual_variable = self.dataset["sensitivity_a1_vis"]
self.assertIsNotNone(virtual_variable)
self.assertEqual((5000, 5000), virtual_variable.shape)
self.assertAlmostEqual(0.85671563074783341, virtual_variable.data[4,
4])
def testCalculate_sensitivitiy_a2_vis(self):
self.dataset["distance_sun_earth"].data = 1.0166579484939575
self.dataset["years_since_launch"].data = 8.830136986301369
self.dataset["count_vis"].data[:, :] = 24
self.dataset["mean_count_space_vis"] = 4.961684375
self.dataset["solar_zenith_angle"].data[:, :] = 22.1907
self.dataset["solar_irradiance_vis"].data = 688.144781045
self.fcdr_reader._load_virtual_variable(self.dataset,
"sensitivity_a2_vis")
virtual_variable = self.dataset["sensitivity_a2_vis"]
self.assertIsNotNone(virtual_variable)
self.assertEqual((5000, 5000), virtual_variable.shape)
self.assertAlmostEqual(7.5649163778089505, virtual_variable.data[4, 4])