def evaluate_cmd(main_arguments): """ Main routine for handling calls to the evaluation command :param main_arguments: The command line arguments (minus the eval command) """ from cis.evaluate import Calculator data_reader = DataReader() data_list = data_reader.read_datagroups(main_arguments.datagroups) calculator = Calculator() result = calculator.evaluate(data_list, main_arguments.expr, main_arguments.output_var, main_arguments.units, main_arguments.attributes) result.save_data(main_arguments.output)
class TestCalculator(unittest.TestCase): def setUp(self): self.calc = Calculator() self.data = GriddedDataList([make_from_cube(mock.make_mock_cube())]) self.data[0].var_name = 'var_name' def _make_two_gridded(self): data1 = make_from_cube(mock.make_mock_cube()) data2 = make_from_cube(mock.make_mock_cube(data_offset=10)) data1.var_name = 'var1' data2._var_name = 'var2' data1.filenames = ['filename1'] data2.filenames = ['filename2'] self.data = [data1, data2] self.data = GriddedDataList([data1, data2]) def _make_two_ungridded_data(self): data1 = mock.make_regular_2d_ungridded_data_with_missing_values() data2 = mock.make_regular_2d_ungridded_data_with_missing_values() data1.metadata._name = 'var1' data2.metadata._name = 'var2' data1.filenames = ['filename1'] data2.filenames = ['filename2'] self.data = [data1, data2] def test_GIVEN_expr_with_double_underscores_WHEN_calculate_THEN_raises_EvaluationError(self): expr = "[c for c in ().__class__.__base__.__subclasses__() if c.__name__ " \ "== 'catch_warnings'][0]()._module.__builtins__['__import__']('os')" with self.assertRaises(EvaluationError): self.calc.evaluate(self.data, expr) def test_GIVEN_expr_using_disallowed_builtins_WHEN_calculate_THEN_raises_EvaluationError(self): expr = 'open("path")' with self.assertRaises(EvaluationError): self.calc.evaluate(self.data, expr) def test_GIVEN_expr_using_numpy_WHEN_calculate_THEN_allowed(self): expr = 'numpy.log(var_name)' self.calc.evaluate(self.data, expr) def test_GIVEN_expr_using_allowed_builtins_WHEN_calculate_THEN_allowed(self): expr = 'var_name + sum(sum(var_name))' self.calc.evaluate(self.data, expr) def test_GIVEN_two_cubes_and_basic_addition_WHEN_calculate_THEN_addition_successful(self): self._make_two_gridded() expr = 'var1 + var2' res = self.calc.evaluate(self.data, expr) expected = numpy.array([[12, 14, 16], [18, 20, 22], [24, 26, 28], [30, 32, 34], [36, 38, 40]]) assert_that(numpy.array_equal(res.data, expected)) def test_GIVEN_two_cubes_interpolated_WHEN_calculate_THEN_interpolation_successful(self): self._make_two_gridded() # Simulate the use case of interpolating between two wavelengths # 550 -> [600] -> 670 expr = 'var1 + (var2 - var1) * (600 - 550) / (670 - 550)' res = self.calc.evaluate(self.data, expr) expected = numpy.array([[5, 6, 7], [8, 9, 10], [11, 12, 13], [14, 15, 16], [17, 18, 19]]) + 1.0 / 6 assert_that(numpy.allclose(res.data, expected)) def test_GIVEN_ungridded_data_basic_addition_WHEN_calculate_THEN_addition_successful(self): data1 = mock.make_regular_2d_ungridded_data() data2 = mock.make_regular_2d_ungridded_data() data1.metadata._name = 'var1' data2.metadata._name = 'var2' self.data = [data1, data2] expr = 'var1 + var2' res = self.calc.evaluate(self.data, expr) expected = 2 * self.data[1].data.flatten() assert_that(numpy.array_equal(res.data, expected)) def test_GIVEN_ungridded_data_one_flattened_WHEN_calculate_THEN_compared_on_flattened_grid(self): data1 = mock.make_regular_2d_ungridded_data() data2 = mock.make_regular_2d_ungridded_data() data2._data = data2.data_flattened for coord in data2.coords(): coord._data = coord.data_flattened data1.metadata._name = 'var1' data2.metadata._name = 'var2' self.data = [data1, data2] expr = 'var1 + var2' res = self.calc.evaluate(self.data, expr) expected = 2 * self.data[1].data assert_that(numpy.array_equal(res.data, expected)) assert_that(res.data.shape, is_((15,))) def test_GIVEN_ungridded_missing_values_WHEN_calculate_THEN_missing_values_preserved(self): data = mock.make_regular_2d_ungridded_data_with_missing_values() data.metadata._name = 'var1' self.data = [data] expr = 'var1 + 10' res = self.calc.evaluate(self.data, expr) expected = numpy.ma.masked_invalid([[11, 12, 13], [14, float('Nan'), 16], [17, 18, float('Nan')], [20, 21, 22], [float('Nan'), 24, 25]]).flatten() compare_masked_arrays(res.data, expected) def test_GIVEN_two_variables_WHEN_calculate_THEN_variable_values_not_changed(self): # This is to fix an issue where the calculator was taking a reference to the input list and therefore changing # its value when output. data1 = mock.make_regular_2d_ungridded_data() data2 = mock.make_regular_2d_ungridded_data() data1.metadata._name = 'var1' data2.metadata._name = 'var2' self.data = [data1, data2] expr = 'var1 + var2' res = self.calc.evaluate(self.data, expr) assert_that(numpy.array_equal(data1.data, data2.data)) def test_GIVEN_two_cubes_basic_addition_WHEN_calculate_THEN_metadata_correct(self): self._make_two_gridded() expr = 'var1 + alias2' self.data[1].alias = 'alias2' res = self.calc.evaluate(self.data, expr) expected_var_name = 'calculated_variable' expected_standard_name = None expected_long_name = 'Calculated value for expression "%s"' % expr assert_that(isinstance(res, GriddedData)) assert_that(res.var_name, is_(expected_var_name)) assert_that(res.standard_name, is_(expected_standard_name)) assert_that(res.long_name, is_(expected_long_name)) assert_that(res.alias, is_(res.var_name)) def test_GIVEN_variables_from_same_file_WHEN_calculate_THEN_history_added(self): self._make_two_ungridded_data() self.data[1].alias = 'alias2' self.data[1].filenames = self.data[0].filenames expr = 'var1 + alias2' res = self.calc.evaluate(self.data, expr) expected_history = "Evaluated using CIS version " + __version__ + \ "\nExpression evaluated: 'var1 + alias2'" + \ "\nwith variables: 'var1' from files ['filename1']," + \ "\n'var2' (as 'alias2') from files ['filename1']." # Do an ends_with comparison because history starts with timestamp assert_that(res.history, ends_with(expected_history)) def test_GIVEN_variables_from_different_files_WHEN_calculate_THEN_history_added(self): self._make_two_gridded() expr = 'var1 + var2' res = self.calc.evaluate(self.data, expr) expected_history = "Evaluated using CIS version " + __version__ + \ "\nExpression evaluated: 'var1 + var2'" + \ "\nwith variables: 'var1' from files ['filename1']," + \ "\n'var2' from files ['filename2']." # Do an ends_with comparison because history starts with timestamp assert_that(res.history, ends_with(expected_history)) def test_GIVEN_output_var_name_WHEN_calculate_THEN_output_uses_var_name(self): self._make_two_gridded() expr = 'var1 + var2' res = self.calc.evaluate(self.data, expr, output_var='var_out_name') assert_that(res.var_name, is_('var_out_name')) def test_GIVEN_operation_results_in_wrong_shape_THEN_raises_EvaluationError(self): data1 = mock.make_regular_2d_ungridded_data() data1.metadata._name = 'var1' self.data = [data1] expr = 'numpy.mean(var1)' with self.assertRaises(EvaluationError): res = self.calc.evaluate(self.data, expr) def test_GIVEN_variables_not_compatible_shape_THEN_raises_EvaluationError(self): data1 = mock.make_regular_2d_ungridded_data() data2 = mock.make_regular_2d_ungridded_data(lat_dim_length=6) data1.metadata._name = 'var1' data2.metadata._name = 'var2' self.data = [data1, data2] expr = 'var1 + var2' with self.assertRaises(EvaluationError): res = self.calc.evaluate(self.data, expr) def test_GIVEN_units_string_WHEN_calculate_ungridded_THEN_units_set_on_output(self): self._make_two_ungridded_data() expr = 'var1 + var2' units = 'kg m^-3' res = self.calc.evaluate(self.data, expr, units=units) assert_that(res.units, is_(units)) def test_GIVEN_units_string_WHEN_calculate_gridded_THEN_units_set_on_output(self): self._make_two_gridded() expr = 'var1 + var2' units = 'kg m^-3' res = self.calc.evaluate(self.data, expr, units=units) assert_that(res.units, is_(units)) def test_GIVEN_attributes_WHEN_calculate_ungridded_THEN_attributes_set_on_output(self): self._make_two_ungridded_data() expr = 'var1 + var2' attributes = {'att1': 'val1', 'att2': 'val2'} res = self.calc.evaluate(self.data, expr, attributes=attributes) assert_that(res.attributes['att1'], is_('val1')) assert_that(res.attributes['att2'], is_('val2')) def test_GIVEN_attributes_WHEN_calculate_gridded_THEN_attributes_set_on_output(self): self._make_two_gridded() expr = 'var1 + var2' attributes = {'att1': 'val1', 'att2': 'val2'} res = self.calc.evaluate(self.data, expr, attributes=attributes) assert_that(res.attributes['att1'], is_('val1')) assert_that(res.attributes['att2'], is_('val2'))