def test_sigmoid(self): from helpers import metrics pred = [self.sigmoid(actual) for actual in self.actuals] print(pred) crps = metrics.mean_crps(self.thresholds, pred, self.actuals) print("Sigmoids: answer = {0}".format(crps)) assert_almost_equal(crps, 0.36, 2)
def test_all_ones(self): from helpers import metrics pred = [[1.] * len(self.thresholds) for _ in self.actuals] print(pred) crps = metrics.mean_crps(self.thresholds, pred, self.actuals) print("All one predictions: answer = {0}".format(crps)) # all one for actual=3 is wrong for threshold=2 i.e. 1 threshold expected = (1 + 0 + 3 + 1) / 20.0 assert_almost_equal(crps, expected)
def test_invalid_cdf(self): from helpers import metrics pred = [metrics._heavyside(self.thresholds, actual) for actual in self.actuals] pred[1] = [0, 0.5, 0.3, 0.8, 1.0] # not a valid CDF print(pred) crps = metrics.mean_crps(self.thresholds, pred, self.actuals) print("Invalid CDF for one case: answer = {0}".format(crps)) assert_almost_equal(crps, 1.0 / 4) # since one case is wrong
def test_all_zeros(self): from helpers import metrics pred = [[0.] * len(self.thresholds) for _ in self.actuals] print(pred) crps = metrics.mean_crps(self.thresholds, pred, self.actuals) print("All zero predictions: answer = {0}".format(crps)) # all zero for actual=3 is wrong for thresholds 4,6,8,10 expected = (4 + 5 + 2 + 4) / 20.0 assert_almost_equal(crps, expected)
def test_exact_answer(self): from helpers import metrics # if the user supplies the exact answers, CRPS is zero. pred = [metrics._heavyside(self.thresholds, actual) for actual in self.actuals] print(pred) crps = metrics.mean_crps(self.thresholds, pred, self.actuals) print("Exact answer = {0}".format(crps)) assert_almost_equal(crps, 0.)