Example #1
0
class TestEvaluation(unittest.TestCase):
    def setUp(self):
        self.eval = Evaluation(None, [], [])

        lat = np.array([10, 12, 14, 16, 18])
        lon = np.array([100, 102, 104, 106, 108])
        time = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)])
        flat_array = np.array(range(300))
        value = flat_array.reshape(12, 5, 5)
        self.variable = 'prec'
        self.other_var = 'temp'
        self.test_dataset = Dataset(lat, lon, time, value, self.variable)
        self.another_test_dataset = Dataset(lat, lon, time, value, 
                self.other_var)

    def test_init(self):
        self.assertEquals(self.eval.ref_dataset, None)
        self.assertEquals(self.eval.target_datasets, [])
        self.assertEquals(self.eval.metrics, [])
        self.assertEquals(self.eval.unary_metrics, [])

    def test_full_init(self):
        self.eval = Evaluation(
                        self.test_dataset,           
                        [self.test_dataset, self.another_test_dataset], 
                        [Bias(), Bias(), TemporalStdDev()])                    

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

        # Make sure the two target datasets were added properly
        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, self.other_var)

        # Make sure the three metrics were added properly
        # The two Bias metrics are "binary" metrics
        self.assertEqual(len(self.eval.metrics), 2)
        # TemporalStdDev is a "unary" metric and should be stored as such
        self.assertEqual(len(self.eval.unary_metrics), 1)

    def test_invalid_ref_dataset(self):
        with self.assertRaises(TypeError):
            self.eval.ref_dataset = "This isn't a Dataset object"

    def test_valid_subregion(self):
        bound = Bounds(
                -10, 10, 
                -20, 20, 
                dt.datetime(2000, 1, 1), dt.datetime(2001, 1, 1))

        self.eval.subregions = [bound, bound]
        self.assertEquals(len(self.eval.subregions), 2)

    def test_invalid_subregion_bound(self):
        bound = "This is not a bounds object"

        with self.assertRaises(TypeError):
            self.eval.subregions = [bound]

    def test_add_ref_dataset(self):
        self.eval = Evaluation(self.test_dataset, [], [])

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

    def test_add_dataset(self):
        self.eval.add_dataset(self.test_dataset)

        self.assertEqual(self.eval.target_datasets[0].variable, 
                self.variable)

    def test_add_datasets(self):
        self.eval.add_datasets([self.test_dataset, self.another_test_dataset])

        self.assertEqual(len(self.eval.target_datasets), 2)
        self.assertEqual(self.eval.target_datasets[0].variable, 
                self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, 
                self.other_var)

    def test_add_metric(self):
        # Add a "binary" metric
        self.assertEqual(len(self.eval.metrics), 0)
        self.eval.add_metric(Bias())
        self.assertEqual(len(self.eval.metrics), 1)

        # Add a "unary" metric
        self.assertEqual(len(self.eval.unary_metrics), 0)
        self.eval.add_metric(TemporalStdDev())
        self.assertEqual(len(self.eval.unary_metrics), 1)

    def test_add_metrics(self):
        self.assertEqual(len(self.eval.metrics), 0)
        self.eval.add_metrics([Bias(), Bias()])
        self.assertEqual(len(self.eval.metrics), 2)
    
    def test_bias_output_shape(self):
        bias_eval = Evaluation(self.test_dataset, [self.another_test_dataset], [Bias()])
        bias_eval.run()
        input_shape = tuple(self.test_dataset.values.shape)
        bias_results_shape = tuple(bias_eval.results[0][0].shape)
        self.assertEqual(input_shape, bias_results_shape)
Example #2
0
class TestEvaluation(unittest.TestCase):

    def setUp(self):
        self.eval = Evaluation(None, [], [])

        lat = np.array([10, 12, 14, 16, 18])
        lon = np.array([100, 102, 104, 106, 108])
        time = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)])
        flat_array = np.array(range(300))
        value = flat_array.reshape(12, 5, 5)
        self.variable = 'prec'
        self.other_var = 'temp'
        self.test_dataset = Dataset(lat, lon, time, value, self.variable)
        self.another_test_dataset = Dataset(lat, lon, time, value,
                                            self.other_var)

    def test_init(self):
        self.assertEquals(self.eval.ref_dataset, None)
        self.assertEquals(self.eval.target_datasets, [])
        self.assertEquals(self.eval.metrics, [])
        self.assertEquals(self.eval.unary_metrics, [])

    def test_full_init(self):
        self.eval = Evaluation(
            self.test_dataset,
            [self.test_dataset, self.another_test_dataset],
            [Bias(), Bias(), TemporalStdDev()])
        ref_dataset = self.test_dataset
        target_datasets = [self.test_dataset, self.another_test_dataset]
        metrics = [Bias(), Bias()]
        unary_metrics = [TemporalStdDev()]

        self.eval = Evaluation(ref_dataset,
                               target_datasets,
                               metrics + unary_metrics)

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

        # Make sure the two target datasets were added properly
        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, self.other_var)

        # Make sure the three metrics were added properly
        # The two Bias metrics are "binary" metrics
        self.assertEqual(len(self.eval.metrics), 2)
        # TemporalStdDev is a "unary" metric and should be stored as such
        self.assertEqual(len(self.eval.unary_metrics), 1)
        self.eval.run()
        out_str = (
            "<Evaluation - ref_dataset: {}, "
            "target_dataset(s): {}, "
            "binary_metric(s): {}, "
            "unary_metric(s): {}, "
            "subregion(s): {}>"
        ).format(
            str(self.test_dataset),
            [str(ds) for ds in target_datasets],
            [str(m) for m in metrics],
            [str(u) for u in unary_metrics],
            None
        )
        self.assertEqual(str(self.eval), out_str)

    def test_valid_ref_dataset_setter(self):
        self.eval.ref_dataset = self.another_test_dataset
        self.assertEqual(self.eval.ref_dataset.variable,
                         self.another_test_dataset.variable)

    def test_invalid_ref_dataset(self):
        with self.assertRaises(TypeError):
            self.eval.ref_dataset = "This isn't a Dataset object"

    def test_valid_subregion(self):
        bound = Bounds(
            lat_min=-10, lat_max=10,
            lon_min=-20, lon_max=20,
            start=dt.datetime(2000, 1, 1), end=dt.datetime(2001, 1, 1))

        self.eval.subregions = [bound, bound]
        self.assertEquals(len(self.eval.subregions), 2)

    def test_invalid_subregion_bound(self):
        bound = "This is not a bounds object"

        with self.assertRaises(TypeError):
            self.eval.subregions = [bound]

    def test_add_ref_dataset(self):
        self.eval = Evaluation(self.test_dataset, [], [])

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

    def test_add_valid_dataset(self):
        self.eval.add_dataset(self.test_dataset)

        self.assertEqual(self.eval.target_datasets[0].variable,
                         self.variable)

    def test_add_invalid_dataset(self):
        with self.assertRaises(TypeError):
            self.eval.add_dataset('This is an invalid dataset')

    def test_add_datasets(self):
        self.eval.add_datasets([self.test_dataset, self.another_test_dataset])

        self.assertEqual(len(self.eval.target_datasets), 2)
        self.assertEqual(self.eval.target_datasets[0].variable,
                         self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable,
                         self.other_var)

    def test_add_valid_metric(self):
        # Add a "binary" metric
        self.assertEqual(len(self.eval.metrics), 0)
        self.eval.add_metric(Bias())
        self.assertEqual(len(self.eval.metrics), 1)

        # Add a "unary" metric
        self.assertEqual(len(self.eval.unary_metrics), 0)
        self.eval.add_metric(TemporalStdDev())
        self.assertEqual(len(self.eval.unary_metrics), 1)

    def test_add_invalid_metric(self):
        with self.assertRaises(TypeError):
            self.eval.add_metric('This is an invalid metric')

    def test_add_metrics(self):
        self.assertEqual(len(self.eval.metrics), 0)
        self.eval.add_metrics([Bias(), Bias()])
        self.assertEqual(len(self.eval.metrics), 2)

    def test_invalid_evaluation_run(self):
        self.eval = Evaluation(None, [], [])
        self.assertEqual(self.eval.run(), None)

    def test_bias_output_shape(self):
        bias_eval = Evaluation(self.test_dataset, [
                               self.another_test_dataset], [Bias()])
        bias_eval.run()
        input_shape = tuple(self.test_dataset.values.shape)
        bias_results_shape = tuple(bias_eval.results[0][0].shape)
        self.assertEqual(input_shape, bias_results_shape)

    def test_result_shape(self):
        bias_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset,
                self.another_test_dataset],
            [Bias(), Bias()]
        )
        bias_eval.run()

        # Expected result shape is
        # [bias, bias] where bias.shape[0] = number of datasets
        self.assertTrue(len(bias_eval.results) == 2)
        self.assertTrue(bias_eval.results[0].shape[0] == 3)

    def test_unary_result_shape(self):
        new_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset,
                self.another_test_dataset, self.another_test_dataset],
            [TemporalStdDev()]
        )
        new_eval.run()

        # Expected result shape is
        # [stddev] where stddev.shape[0] = number of datasets

        self.assertTrue(len(new_eval.unary_results) == 1)
        self.assertTrue(new_eval.unary_results[0].shape[0] == 5)

    def test_subregion_result_shape(self):
        bound = Bounds(
            lat_min=10, lat_max=18,
            lon_min=100, lon_max=108,
            start=dt.datetime(2000, 1, 1), end=dt.datetime(2000, 3, 1))

        bias_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset],
            [Bias()],
            [bound]
        )
        bias_eval.run()

        # Expected result shape is
        # [
        #       [   # Subregions cause this extra layer
        #           [number of targets, bias.run(reference, target1).shape]
        #       ]
        #   ],
        self.assertTrue(len(bias_eval.results) == 1)
        self.assertTrue(len(bias_eval.results[0]) == 1)
        self.assertTrue(bias_eval.results[0][0].shape[0] == 2)
        self.assertTrue(isinstance(bias_eval.results, type([])))

    def test_subregion_unary_result_shape(self):
        bound = Bounds(
            lat_min=10, lat_max=18,
            lon_min=100, lon_max=108,
            start=dt.datetime(2000, 1, 1), end=dt.datetime(2000, 3, 1))

        new_eval = Evaluation(
            self.test_dataset,
            [self.another_test_dataset, self.another_test_dataset],
            [TemporalStdDev(), TemporalStdDev()],
            [bound, bound, bound, bound, bound]
        )
        new_eval.run()

        # Expected result shape is
        # [
        #       [   # Subregions cause this extra layer
        #           [3, temporalstddev.run(reference).shape],
        #       ]
        # ]

        # 5 = number of subregions
        self.assertTrue(len(new_eval.unary_results) == 5)
        # number of metrics
        self.assertTrue(len(new_eval.unary_results[0]) == 2)
        self.assertTrue(isinstance(new_eval.unary_results, type([])))
        # number of datasets (ref + target)
        self.assertTrue(new_eval.unary_results[0][0].shape[0] == 3)
Example #3
0
class TestEvaluation(unittest.TestCase):
    def setUp(self):
        self.eval = Evaluation(None, [], [])

        lat = np.array([10, 12, 14, 16, 18])
        lon = np.array([100, 102, 104, 106, 108])
        time = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)])
        flat_array = np.array(range(300))
        value = flat_array.reshape(12, 5, 5)
        self.variable = 'prec'
        self.other_var = 'temp'
        self.test_dataset = Dataset(lat, lon, time, value, self.variable)
        self.another_test_dataset = Dataset(lat, lon, time, value,
                                            self.other_var)

    def test_init(self):
        self.assertEquals(self.eval.ref_dataset, None)
        self.assertEquals(self.eval.target_datasets, [])
        self.assertEquals(self.eval.metrics, [])
        self.assertEquals(self.eval.unary_metrics, [])

    def test_full_init(self):
        self.eval = Evaluation(
            self.test_dataset, [self.test_dataset, self.another_test_dataset],
            [Bias(), Bias(), TemporalStdDev()])

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

        # Make sure the two target datasets were added properly
        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, self.other_var)

        # Make sure the three metrics were added properly
        # The two Bias metrics are "binary" metrics
        self.assertEqual(len(self.eval.metrics), 2)
        # TemporalStdDev is a "unary" metric and should be stored as such
        self.assertEqual(len(self.eval.unary_metrics), 1)

    def test_invalid_ref_dataset(self):
        with self.assertRaises(TypeError):
            self.eval.ref_dataset = "This isn't a Dataset object"

    def test_valid_subregion(self):
        bound = Bounds(-10, 10, -20, 20, dt.datetime(2000, 1, 1),
                       dt.datetime(2001, 1, 1))

        self.eval.subregions = [bound, bound]
        self.assertEquals(len(self.eval.subregions), 2)

    def test_invalid_subregion_bound(self):
        bound = "This is not a bounds object"

        with self.assertRaises(TypeError):
            self.eval.subregions = [bound]

    def test_add_ref_dataset(self):
        self.eval = Evaluation(self.test_dataset, [], [])

        self.assertEqual(self.eval.ref_dataset.variable, self.variable)

    def test_add_dataset(self):
        self.eval.add_dataset(self.test_dataset)

        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)

    def test_add_datasets(self):
        self.eval.add_datasets([self.test_dataset, self.another_test_dataset])

        self.assertEqual(len(self.eval.target_datasets), 2)
        self.assertEqual(self.eval.target_datasets[0].variable, self.variable)
        self.assertEqual(self.eval.target_datasets[1].variable, self.other_var)

    def test_add_metric(self):
        # Add a "binary" metric
        self.assertEqual(len(self.eval.metrics), 0)
        self.eval.add_metric(Bias())
        self.assertEqual(len(self.eval.metrics), 1)

        # Add a "unary" metric
        self.assertEqual(len(self.eval.unary_metrics), 0)
        self.eval.add_metric(TemporalStdDev())
        self.assertEqual(len(self.eval.unary_metrics), 1)

    def test_add_metrics(self):
        self.assertEqual(len(self.eval.metrics), 0)
        self.eval.add_metrics([Bias(), Bias()])
        self.assertEqual(len(self.eval.metrics), 2)

    def test_bias_output_shape(self):
        bias_eval = Evaluation(self.test_dataset, [self.another_test_dataset],
                               [Bias()])
        bias_eval.run()
        input_shape = tuple(self.test_dataset.values.shape)
        bias_results_shape = tuple(bias_eval.results[0][0].shape)
        self.assertEqual(input_shape, bias_results_shape)