def testMultiMetric(self): ms1 = MetricSpec(field='a', metric='trivial', inferenceElement='prediction', params={'errorMetric': 'aae', 'window': 1000, 'steps': 1}) ms2 = MetricSpec(metric='trivial', inferenceElement='prediction', field='a', params={'window': 10, 'steps': 1, 'errorMetric': 'rmse'}) metric1000 = getModule(ms1) metric10 = getModule(ms2) # create multi metric multi = MetricMulti(weights=[0.2, 0.8], metrics=[metric10, metric1000]) multi.verbosity = 1 # create reference metrics (must be diff from metrics above used in MultiMetric, as they keep history) metric1000ref = getModule(ms1) metric10ref = getModule(ms2) gt = range(500, 1000) p = range(500) for i in xrange(len(gt)): v10=metric10ref.addInstance(gt[i], p[i]) v1000=metric1000ref.addInstance(gt[i], p[i]) if v10 is None or v1000 is None: check=None else: check=0.2*float(v10) + 0.8*float(v1000) metricValue = multi.addInstance(gt[i], p[i]) self.assertEqual(check, metricValue, "iter i= %s gt=%s pred=%s multi=%s sub1=%s sub2=%s" % (i, gt[i], p[i], metricValue, v10, v1000))
def testNegativeLogLikelihood(self): # make sure negativeLogLikelihood returns correct LL numbers # mock objects for ClassifierInput and ModelResult (see opfutils.py) class MockClassifierInput(object): def __init__(self, bucketIdx): self.bucketIndex = bucketIdx class MockModelResult(object): def __init__(self, bucketll, bucketIdx): self.inferences = {'multiStepBucketLikelihoods': {1: bucketll}} self.classifierInput = MockClassifierInput(bucketIdx) bucketLL = {0: 1.0, 1: 0, 2: 0, 3: 0} # model prediction as a dictionary gt_bucketIdx = 0 # bucket index for ground truth negLL = getModule(MetricSpec("negativeLogLikelihood", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY})) negLL.addInstance(0, 0, record = None, result=MockModelResult(bucketLL, gt_bucketIdx)) target = 0.0 # -log(1.0) self.assertAlmostEqual(negLL.getMetric()["value"], target) bucketLL = {0: 0.5, 1: 0.5, 2: 0, 3: 0} # model prediction as a dictionary gt_bucketIdx = 0 # bucket index for ground truth negLL = getModule(MetricSpec("negativeLogLikelihood", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY})) negLL.addInstance(0, 0, record = None, result=MockModelResult(bucketLL, gt_bucketIdx)) target = 0.6931471 # -log(0.5) self.assertTrue(abs(negLL.getMetric()["value"]-target) < OPFMetricsTest.DELTA) # test accumulated negLL for multiple steps bucketLL = [] bucketLL.append({0: 1, 1: 0, 2: 0, 3: 0}) bucketLL.append({0: 0, 1: 1, 2: 0, 3: 0}) bucketLL.append({0: 0, 1: 0, 2: 1, 3: 0}) bucketLL.append({0: 0, 1: 0, 2: 0, 3: 1}) gt_bucketIdx = [0, 2, 1, 3] negLL = getModule(MetricSpec("negativeLogLikelihood", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY})) for i in xrange(len(bucketLL)): negLL.addInstance(0, 0, record = None, result=MockModelResult(bucketLL[i], gt_bucketIdx[i])) target = 5.756462 self.assertTrue(abs(negLL.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testTwoGramScalarsStepsGreaterOne(self): """Two gram scalars test with step size other than 1""" oneGram = getModule(MetricSpec("two_gram", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY,\ "window":100, "predictionField":"test", "errorMetric":"acc", "steps": 2})) # Sequences of 0,1,2,3,4,0,1,2,3,4,... encodings = [np.zeros(10) for i in range(5)] for i in range(len(encodings)): encoding = encodings[i] encoding[i] = 1 gt = [i%5 for i in range(1000)] res = [] for i in xrange(len(gt)): if i == 20: # Make sure we don"t barf with missing values oneGram.addInstance(np.zeros(10), prediction=None, record={"test":None}) else: # Feed in next groundTruth oneGram.addInstance(encodings[i%5], prediction=None, record={"test":gt[i]}) res.append(oneGram.getMetric()["value"]) target = 1.0 self.assertTrue(abs(oneGram.getMetric()["value"]-target) \ < OPFMetricsTest.DELTA)
def testTwoGramStrings(self): """One gram string test""" oneGram = getModule(MetricSpec("two_gram", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"acc", "predictionField":"test"})) # Sequences of "0", "1", "2", "3", "4", "0", "1", ... gt = [str(i%5) for i in range(1000)] encodings = [np.zeros(10) for i in range(5)] for i in range(len(encodings)): encoding = encodings[i] encoding[i] = 1 # Make every 5th element random newElem = 100 for i in range(5, 1000, 5): gt[i] = str(newElem) newElem += 20 res = [] for i in xrange(len(gt)): if i==20: # Make sure we don"t barf with missing values oneGram.addInstance(np.zeros(10), prediction=None, record={"test":None}) else: oneGram.addInstance(encodings[i%5], prediction=None, record={"test":gt[i]}) res.append(oneGram.getMetric()["value"]) target = .8 self.assertTrue(abs(oneGram.getMetric()["value"]-target)\ < OPFMetricsTest.DELTA)
def testMovingModeAccuracy(self): """Moving mode Accuracy metric test""" movingModeACC = getModule( MetricSpec( "moving_mode", None, None, {"verbosity": OPFMetricsTest.VERBOSITY, "window": 100, "mode_window": 3, "errorMetric": "acc"}, ) ) # Should initially asymptote to .5 # Then after 900 should go to 0.0 as the predictions will always be offset gt = [i / 4 for i in range(900)] gt.extend([2 * i / 4 for i in range(100)]) p = [i for i in range(1000)] res = [] for i in xrange(len(gt)): movingModeACC.addInstance(gt[i], p[i]) res.append(movingModeACC.getMetric()["value"]) # Make sure that there is no point where the average acc is <.5 self.assertTrue(min(res[1:899]) == 0.5) # Make sure that after the statistics switch the acc goes to 0.0 self.assertTrue(max(res[900:]) <= 0.5) # Make sure that the statistics change is still noticeable while it # is in the window self.assertTrue(res[998] > 0.0) target = 0.0 self.assertTrue(abs(movingModeACC.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testAAE(self): aae = getModule(MetricSpec("aae", None, None, {"verbosity": OPFMetricsTest.VERBOSITY})) gt = [9, 4, 5, 6] p = [0, 13, 8, 3] for i in xrange(len(gt)): aae.addInstance(gt[i], p[i]) target = 6.0 self.assertTrue(abs(aae.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testNegLLMultiplePrediction(self): # In cases where the ground truth has multiple possible outcomes, make sure # that the prediction that captures ground truth distribution has best LL # and models that gives single prediction (either most likely outcome or # average outcome) has worse LL # mock objects for ClassifierInput and ModelResult (see opfutils.py) class MockClassifierInput(object): def __init__(self, bucketIdx): self.bucketIndex = bucketIdx class MockModelResult(object): def __init__(self, bucketll, bucketIdx): self.inferences = {'multiStepBucketLikelihoods': {1: bucketll}} self.classifierInput = MockClassifierInput(bucketIdx) # the ground truth lies in bucket 0 with p=0.45, in bucket 1 with p=0.0 # and in bucket 2 with p=0.55 gt_bucketIdx = [0]*45+[2]*55 # compare neg log-likelihood for three type of model predictions # a model that predicts ground truth distribution prediction_gt = {0: 0.45, 1: 0, 2: 0.55} # a model that predicts only the most likely outcome prediction_ml = {0: 0.0, 1: 0, 2: 1.0} # a model that only predicts mean (bucket 1) prediction_mean = {0: 0, 1: 1, 2: 0} negLL_gt = getModule(MetricSpec("negativeLogLikelihood", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY})) negLL_ml = getModule(MetricSpec("negativeLogLikelihood", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY})) negLL_mean = getModule(MetricSpec("negativeLogLikelihood", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY})) for i in xrange(len(gt_bucketIdx)): negLL_gt.addInstance(0, 0, record = None, result=MockModelResult(prediction_gt, gt_bucketIdx[i])) negLL_ml.addInstance(0, 0, record = None, result=MockModelResult(prediction_ml, gt_bucketIdx[i])) negLL_mean.addInstance(0, 0, record = None, result=MockModelResult(prediction_mean, gt_bucketIdx[i])) self.assertTrue(negLL_gt.getMetric()["value"] < negLL_ml.getMetric()["value"]) self.assertTrue(negLL_gt.getMetric()["value"] < negLL_mean.getMetric()["value"])
def testWindowedRMSE(self): wrmse = getModule(MetricSpec("rmse", None, None, {"verbosity": OPFMetricsTest.VERBOSITY, "window": 3})) gt = [9, 4, 4, 100, 44] p = [0, 13, 4, 6, 7] for gv, pv in zip(gt, p): wrmse.addInstance(gv, pv) target = 58.324 self.assertTrue(abs(wrmse.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testAverageError(self): """Ave Error""" err = getModule(MetricSpec("avg_err", None, None, {"verbosity": OPFMetricsTest.VERBOSITY})) gt = [1, 1, 2, 3, 4, 5] p = [0, 1, 2, 4, 5, 6] for i in xrange(len(gt)): err.addInstance(gt[i], p[i]) target = 2.0 / 3.0 self.assertTrue(abs(err.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testAccuracy(self): """Accuracy""" acc = getModule(MetricSpec("acc", None, None, {"verbosity": OPFMetricsTest.VERBOSITY})) gt = [0, 1, 2, 3, 4, 5] p = [0, 1, 2, 4, 5, 6] for i in xrange(len(gt)): acc.addInstance(gt[i], p[i]) target = 0.5 self.assertTrue(abs(acc.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testNRMSE(self): nrmse = getModule(MetricSpec("nrmse", None, None, {"verbosity": OPFMetricsTest.VERBOSITY})) gt = [9, 4, 5, 6] p = [0, 13, 8, 3] for i in xrange(len(gt)): nrmse.addInstance(gt[i], p[i]) target = 3.5856858280031814 self.assertAlmostEqual(nrmse.getMetric()["value"], target)
def testAAE(self): aae = getModule(MetricSpec("aae", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY})) gt = [9, 4, 5, 6] p = [0, 13, 8, 3] for i in xrange(len(gt)): aae.addInstance(gt[i], p[i]) target = 6.0 self.assertTrue(abs(aae.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testWindowedAAE(self): """Windowed AAE""" waae = getModule(MetricSpec("aae", None, None, {"verbosity": OPFMetricsTest.VERBOSITY, "window": 1})) gt = [9, 4, 5, 6] p = [0, 13, 8, 3] for i in xrange(len(gt)): waae.addInstance(gt[i], p[i]) target = 3.0 self.assertTrue(abs(waae.getMetric()["value"] - target) < OPFMetricsTest.DELTA, "Got %s" % waae.getMetric())
def testTrivialAccuracy(self): trivialaccuracy = getModule(MetricSpec("trivial", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY,"errorMetric":"acc"})) gt = [str(i/4+1) for i in range(100)] p = [str(i) for i in range(100)] for i in xrange(len(gt)): trivialaccuracy.addInstance(gt[i], p[i]) target = .75 self.assertTrue(abs(trivialaccuracy.getMetric()["value"]-target) \ < OPFMetricsTest.DELTA)
def testTrivialAAE(self): trivialaae = getModule(MetricSpec("trivial", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY,"errorMetric":"aae"})) gt = [i/4+1 for i in range(100)] p = [i for i in range(100)] for i in xrange(len(gt)): trivialaae.addInstance(gt[i], p[i]) target = .25 self.assertTrue(abs(trivialaae.getMetric()["value"]-target) \ < OPFMetricsTest.DELTA)
def testAverageError(self): """Ave Error""" err = getModule(MetricSpec("avg_err", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY})) gt = [1, 1, 2, 3, 4, 5] p = [0, 1, 2, 4, 5, 6] for i in xrange(len(gt)): err.addInstance(gt[i], p[i]) target = (2.0/3.0) self.assertTrue(abs(err.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testAccuracy(self): """Accuracy""" acc = getModule(MetricSpec("acc", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY})) gt = [0, 1, 2, 3, 4, 5] p = [0, 1, 2, 4, 5, 6] for i in xrange(len(gt)): acc.addInstance(gt[i], p[i]) target = 0.5 self.assertTrue(abs(acc.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testLongWindowRMSE(self): """RMSE""" rmse = getModule(MetricSpec("rmse", None, None, {"verbosity": OPFMetricsTest.VERBOSITY, "window": 100})) gt = [9, 4, 5, 6] p = [0, 13, 8, 3] for i in xrange(len(gt)): rmse.addInstance(gt[i], p[i]) target = 6.71 self.assertTrue(abs(rmse.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testWindowedTrivialAAE (self): """Trivial Average Error metric test""" trivialAveErr = getModule(MetricSpec("trivial", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY,"errorMetric":"avg_err"})) gt = [str(i/4+1) for i in range(100)] p = [str(i) for i in range(100)] for i in xrange(len(gt)): trivialAveErr.addInstance(gt[i], p[i]) target = .25 self.assertTrue(abs(trivialAveErr.getMetric()["value"]-target)\ < OPFMetricsTest.DELTA)
def testWindowedRMSE(self): wrmse = getModule(MetricSpec("rmse", None, None, {"verbosity": OPFMetricsTest.VERBOSITY, "window":3})) gt = [9, 4, 4, 100, 44] p = [0, 13, 4, 6, 7] for gv, pv in zip(gt, p): wrmse.addInstance(gv, pv) target = 58.324 self.assertTrue (abs(wrmse.getMetric()["value"]-target)\ < OPFMetricsTest.DELTA)
def testWindowedAAE(self): """Windowed AAE""" waae = getModule(MetricSpec("aae", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY, "window":1})) gt = [9, 4, 5, 6] p = [0, 13, 8, 3] for i in xrange(len(gt)): waae.addInstance(gt[i], p[i]) target = 3.0 self.assertTrue( abs(waae.getMetric()["value"]-target) \ < OPFMetricsTest.DELTA, "Got %s" %waae.getMetric())
def testNRMSE(self): nrmse = getModule( MetricSpec("nrmse", None, None, {"verbosity": OPFMetricsTest.VERBOSITY})) gt = [9, 4, 5, 6] p = [0, 13, 8, 3] for i in xrange(len(gt)): nrmse.addInstance(gt[i], p[i]) target = 3.5856858280031814 self.assertAlmostEqual(nrmse.getMetric()["value"], target)
def testMultiMetric(self): ms1 = MetricSpec(field='a', metric='trivial', inferenceElement='prediction', params={ 'errorMetric': 'aae', 'window': 1000, 'steps': 1 }) ms2 = MetricSpec(metric='trivial', inferenceElement='prediction', field='a', params={ 'window': 10, 'steps': 1, 'errorMetric': 'rmse' }) metric1000 = getModule(ms1) metric10 = getModule(ms2) # create multi metric multi = MetricMulti(weights=[0.2, 0.8], metrics=[metric10, metric1000]) multi.verbosity = 1 # create reference metrics (must be diff from metrics above used in MultiMetric, as they keep history) metric1000ref = getModule(ms1) metric10ref = getModule(ms2) gt = range(500, 1000) p = range(500) for i in xrange(len(gt)): v10 = metric10ref.addInstance(gt[i], p[i]) v1000 = metric1000ref.addInstance(gt[i], p[i]) if v10 is None or v1000 is None: check = None else: check = 0.2 * float(v10) + 0.8 * float(v1000) metricValue = multi.addInstance(gt[i], p[i]) self.assertEqual( check, metricValue, "iter i= %s gt=%s pred=%s multi=%s sub1=%s sub2=%s" % (i, gt[i], p[i], metricValue, v10, v1000))
def testLongWindowRMSE(self): """RMSE""" rmse = getModule(MetricSpec("rmse", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100})) gt = [9, 4, 5, 6] p = [0, 13, 8, 3] for i in xrange(len(gt)): rmse.addInstance(gt[i], p[i]) target = 6.71 self.assertTrue(abs(rmse.getMetric()["value"]-target)\ < OPFMetricsTest.DELTA)
def testMultistepProbability(self): """Multistep with probabilities metric test""" msp = getModule(MetricSpec("multiStepProbability", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"aae", "steps":3})) gt = [5 for i in range(1000)] p = [{3: {i: .3, 5: .7}} for i in range(1000)] for i in xrange(len(gt)): msp.addInstance(gt[i], p[i]) #((999-5)(1000-5)/2-(899-5)(900-5)/2)*.3/100 target = 283.35 self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testWindowedAverageError(self): """Windowed Ave Error""" err = getModule(MetricSpec("avg_err", None, None, \ {"verbosity" : OPFMetricsTest.VERBOSITY, "window":2})) gt = [0, 1, 2, 3, 4, 5] p = [0, 1, 2, 4, 5, 6] for i in range(len(gt)): err.addInstance(gt[i], p[i]) target = 1.0 self.assertTrue( abs(err.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testMultistepAAE(self): """Multistep AAE metric test""" msp = getModule(MetricSpec("multiStep", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"aae", "steps": 3})) # Make each ground truth 1 greater than the prediction gt = [i+1 for i in range(100)] p = [{3: {i: .7, 5: 0.3}} for i in range(100)] for i in xrange(len(gt)): msp.addInstance(gt[i], p[i]) target = 1 self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testMultistepProbabilityMultipleSteps(self): """Multistep with probabilities metric test, predicting 2 different step sizes""" msp = getModule(MetricSpec("multiStepProbability", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"aae", "steps": [1,3]})) gt = [5 for i in range(1000)] p = [{3: {i: .3, 5: .7}, 1: {5: 1.0}} for i in range(1000)] for i in xrange(len(gt)): msp.addInstance(gt[i], p[i]) #(((999-5)(1000-5)/2-(899-5)(900-5)/2)*.3/100) / 2 # / 2 because the 1-step prediction is 100% accurate target = 283.35/2 self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testMultistepAAEMultipleSteps(self): """Multistep AAE metric test, predicting 2 different step sizes""" msp = getModule(MetricSpec("multiStep", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"aae", "steps": [3,6]})) # Make each 3 step prediction +1 over ground truth and each 6 step # prediction +0.5 over ground truth gt = [i for i in range(100)] p = [{3: {i+1: .7, 5: 0.3}, 6: {i+0.5: .7, 5: 0.3}} for i in range(100)] for i in xrange(len(gt)): msp.addInstance(gt[i], p[i]) target = 0.75 # average of +1 error and 0.5 error self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testMovingMeanRMSE(self): """Moving mean RMSE metric test""" movingMeanRMSE = getModule(MetricSpec("moving_mean", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "mean_window":3, "errorMetric":"rmse"})) gt = [i for i in range(890)] gt.extend([2*i for i in range(110)]) p = [i for i in range(1000)] res = [] for i in xrange(len(gt)): movingMeanRMSE.addInstance(gt[i], p[i]) res.append(movingMeanRMSE.getMetric()["value"]) self.assertTrue(max(res[1:890]) == 2.0) self.assertTrue(min(res[891:])>=4.0) target = 4.0 self.assertTrue(abs(movingMeanRMSE.getMetric()["value"]-target) \ < OPFMetricsTest.DELTA)
def __constructMetricsModules(self, metricSpecs): """ Creates the required metrics modules Parameters: ----------------------------------------------------------------------- metricSpecs: A sequence of MetricSpec objects that specify which metric modules to instantiate """ if not metricSpecs: return self.__metricSpecs = metricSpecs for spec in metricSpecs: if not InferenceElement.validate(spec.inferenceElement): raise ValueError("Invalid inference element for metric spec: %r" %spec) self.__metrics.append(metrics.getModule(spec)) self.__metricLabels.append(spec.getLabel())
def testTwoGramStrings(self): """One gram string test""" oneGram = getModule( MetricSpec( "two_gram", None, None, { "verbosity": OPFMetricsTest.VERBOSITY, "window": 100, "errorMetric": "acc", "predictionField": "test" })) # Sequences of "0", "1", "2", "3", "4", "0", "1", ... gt = [str(i % 5) for i in range(1000)] encodings = [np.zeros(10) for i in range(5)] for i in range(len(encodings)): encoding = encodings[i] encoding[i] = 1 # Make every 5th element random newElem = 100 for i in range(5, 1000, 5): gt[i] = str(newElem) newElem += 20 res = [] for i in xrange(len(gt)): if i == 20: # Make sure we don"t barf with missing values oneGram.addInstance(np.zeros(10), prediction=None, record={"test": None}) else: oneGram.addInstance(encodings[i % 5], prediction=None, record={"test": gt[i]}) res.append(oneGram.getMetric()["value"]) target = .8 self.assertTrue(abs(oneGram.getMetric()["value"]-target)\ < OPFMetricsTest.DELTA)
def testMovingModeAccuracy(self): """Moving mode Accuracy metric test""" movingModeACC = getModule(MetricSpec("moving_mode", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "mode_window":3, "errorMetric":"acc"})) #Should initially asymptote to .5 #Then after 900 should go to 0.0 as the predictions will always be offset gt = [i/4 for i in range(900)] gt.extend([2*i/4 for i in range(100)]) p = [i for i in range(1000)] res = [] for i in xrange(len(gt)): movingModeACC.addInstance(gt[i], p[i]) res.append(movingModeACC.getMetric()["value"]) #Make sure that there is no point where the average acc is <.5 self.assertTrue(min(res[1:899]) == .5) #Make sure that after the statistics switch the acc goes to 0.0 self.assertTrue(max(res[900:])<=.5) #Make sure that the statistics change is still noticeable while it #is in the window self.assertTrue(res[998]>0.0) target = 0.0 self.assertTrue(abs(movingModeACC.getMetric()["value"]-target)\ < OPFMetricsTest.DELTA)
def testMovingModeAverageError(self): """Moving mode Average Error metric test""" movingModeAvgErr = getModule(MetricSpec("moving_mode", None, None, {"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "mode_window":3, "errorMetric":"avg_err"})) #Should initially assymptote to .5 #Then after 900 should go to 1.0 as the predictions will always be offset gt = [i/4 for i in range(900)] gt.extend([2*i/4 for i in range(100)]) p = [i for i in range(1000)] res = [] for i in xrange(len(gt)): movingModeAvgErr.addInstance(gt[i], p[i]) res.append(movingModeAvgErr.getMetric()["value"]) #Make sure that there is no point where the average error is >.5 self.assertTrue(max(res[1:890]) == .5) #Make sure that after the statistics switch the error goes to 1.0 self.assertTrue(min(res[891:])>=.5) #Make sure that the statistics change is still noticeable while it is #in the window self.assertTrue(res[998]<1.0) target = 1.0 self.assertTrue(abs(movingModeAvgErr.getMetric()["value"]-target)\ < OPFMetricsTest.DELTA)
printTPRegionParams(model._getTPRegion()) inputData = "%s/%s.csv" % (DATA_DIR, dataSet.replace(" ", "_")) sensor = model._getSensorRegion() encoderList = sensor.getSelf().encoder.getEncoderList() if sensor.getSelf().disabledEncoder is not None: classifier_encoder = sensor.getSelf().disabledEncoder.getEncoderList() classifier_encoder = classifier_encoder[0] else: classifier_encoder = None _METRIC_SPECS = getMetricSpecs(predictedField, stepsAhead=_options.stepsAhead) metric = metrics.getModule(_METRIC_SPECS[0]) metricsManager = MetricsManager(_METRIC_SPECS, model.getFieldInfo(), model.getInferenceType()) if plot: plotCount = 1 plotHeight = max(plotCount * 3, 6) fig = plt.figure(figsize=(14, plotHeight)) gs = gridspec.GridSpec(plotCount, 1) plt.title(predictedField) plt.ylabel('Data') plt.xlabel('Timed') plt.tight_layout() plt.ion() print "Load dataset: ", dataSet
print model._spLearningEnabled printTPRegionParams(model._getTPRegion()) inputData = "%s/%s.csv" % (DATA_DIR, dataSet.replace(" ", "_")) sensor = model._getSensorRegion() encoderList = sensor.getSelf().encoder.getEncoderList() if sensor.getSelf().disabledEncoder is not None: classifier_encoder = sensor.getSelf().disabledEncoder.getEncoderList() classifier_encoder = classifier_encoder[0] else: classifier_encoder = None _METRIC_SPECS = getMetricSpecs(predictedField, stepsAhead=_options.stepsAhead) metric = metrics.getModule(_METRIC_SPECS[0]) metricsManager = MetricsManager(_METRIC_SPECS, model.getFieldInfo(), model.getInferenceType()) if plot: plotCount = 1 plotHeight = max(plotCount * 3, 6) fig = plt.figure(figsize=(14, plotHeight)) gs = gridspec.GridSpec(plotCount, 1) plt.title(predictedField) plt.ylabel('Data') plt.xlabel('Timed') plt.tight_layout() plt.ion() print "Load dataset: ", dataSet
def testNegativeLogLikelihood(self): # make sure negativeLogLikelihood returns correct LL numbers # mock objects for ClassifierInput and ModelResult (see opfutils.py) class MockClassifierInput(object): def __init__(self, bucketIdx): self.bucketIndex = bucketIdx class MockModelResult(object): def __init__(self, bucketll, bucketIdx): self.inferences = {'multiStepBucketLikelihoods': {1: bucketll}} self.classifierInput = MockClassifierInput(bucketIdx) bucketLL = { 0: 1.0, 1: 0, 2: 0, 3: 0 } # model prediction as a dictionary gt_bucketIdx = 0 # bucket index for ground truth negLL = getModule( MetricSpec("negativeLogLikelihood", None, None, {"verbosity": OPFMetricsTest.VERBOSITY})) negLL.addInstance(0, 0, record=None, result=MockModelResult(bucketLL, gt_bucketIdx)) target = 0.0 # -log(1.0) self.assertAlmostEqual(negLL.getMetric()["value"], target) bucketLL = { 0: 0.5, 1: 0.5, 2: 0, 3: 0 } # model prediction as a dictionary gt_bucketIdx = 0 # bucket index for ground truth negLL = getModule( MetricSpec("negativeLogLikelihood", None, None, {"verbosity": OPFMetricsTest.VERBOSITY})) negLL.addInstance(0, 0, record=None, result=MockModelResult(bucketLL, gt_bucketIdx)) target = 0.6931471 # -log(0.5) self.assertTrue( abs(negLL.getMetric()["value"] - target) < OPFMetricsTest.DELTA) # test accumulated negLL for multiple steps bucketLL = [] bucketLL.append({0: 1, 1: 0, 2: 0, 3: 0}) bucketLL.append({0: 0, 1: 1, 2: 0, 3: 0}) bucketLL.append({0: 0, 1: 0, 2: 1, 3: 0}) bucketLL.append({0: 0, 1: 0, 2: 0, 3: 1}) gt_bucketIdx = [0, 2, 1, 3] negLL = getModule( MetricSpec("negativeLogLikelihood", None, None, {"verbosity": OPFMetricsTest.VERBOSITY})) for i in xrange(len(bucketLL)): negLL.addInstance(0, 0, record=None, result=MockModelResult(bucketLL[i], gt_bucketIdx[i])) target = 5.756462 self.assertTrue( abs(negLL.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testCustomErrorMetric(self): customFunc = """def getError(pred,ground,tools): return abs(pred-ground)""" customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc, "errorWindow":3})) gt = [9, 4, 5, 6] p = [0, 13, 8, 3] for i in xrange(len(gt)): aggErr = customEM.addInstance(gt[i], p[i]) target = 5.0 delta = 0.001 # insure that addInstance returns the aggregate error - other # uber metrics depend on this behavior. self.assertEqual(aggErr, customEM.getMetric()["value"]) self.assertTrue(abs(customEM.getMetric()["value"]-target) < delta) customFunc = """def getError(pred,ground,tools): sum = 0 for i in range(min(3,tools.getBufferLen())): sum+=abs(tools.getPrediction(i)-tools.getGroundTruth(i)) return sum/3""" customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc})) gt = [9, 4, 5, 6] p = [0, 13, 8, 3] for i in xrange(len(gt)): customEM.addInstance(gt[i], p[i]) target = 5.0 delta = 0.001 self.assertTrue(abs(customEM.getMetric()["value"]-target) < delta) # Test custom error metric helper functions # Test getPrediction # Not-Windowed storeWindow=4 failed = False for lookBack in range(3): customFunc = """def getError(pred,ground,tools): return tools.getPrediction(%d)""" % lookBack customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc})) gt = [i for i in range(100)] p = [2*i for i in range(100)] t1 = [3*i for i in range(100)] t2 = [str(4*i) for i in range(100)] for i in xrange(len(gt)): curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]} if i < lookBack: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue( not failed, "An exception should have been generated, but wasn't") else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue( customEM.getMetric()["value"] == p[i-lookBack]) #Windowed for lookBack in range(5): customFunc = """def getError(pred,ground,tools): return tools.getPrediction(%d)""" % lookBack customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc,"storeWindow":storeWindow})) gt = [i for i in range(100)] p = [2*i for i in range(100)] t1 = [3*i for i in range(100)] t2 = [str(4*i) for i in range(100)] for i in xrange(len(gt)): curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]} if lookBack>=storeWindow-1: pass if i < lookBack or lookBack>=storeWindow: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue (not failed , "An exception should have been generated, but wasn't") else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue (customEM.getMetric()["value"] == p[i-lookBack]) #Test getGroundTruth #Not-Windowed for lookBack in range(3): customFunc = """def getError(pred,ground,tools): return tools.getGroundTruth(%d)""" % lookBack customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc})) gt = [i for i in range(100)] p = [2*i for i in range(100)] t1 = [3*i for i in range(100)] t2 = [str(4*i) for i in range(100)] for i in xrange(len(gt)): curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]} if i < lookBack: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue( not failed , "An exception should have been generated, but wasn't") else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue (customEM.getMetric()["value"] == gt[i-lookBack]) #Windowed for lookBack in range(5): customFunc = """def getError(pred,ground,tools): return tools.getGroundTruth(%d)""" % lookBack customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc,"storeWindow":storeWindow})) gt = [i for i in range(100)] p = [2*i for i in range(100)] t1 = [3*i for i in range(100)] t2 = [str(4*i) for i in range(100)] for i in xrange(len(gt)): curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]} if i < lookBack or lookBack>=storeWindow: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue( not failed , "An exception should have been generated, but wasn't") else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue( customEM.getMetric()["value"] == gt[i-lookBack]) #Test getFieldValue #Not-Windowed Scalar for lookBack in range(3): customFunc = """def getError(pred,ground,tools): return tools.getFieldValue(%d,"test1")""" % lookBack customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc})) gt = [i for i in range(100)] p = [2*i for i in range(100)] t1 = [3*i for i in range(100)] t2 = [str(4*i) for i in range(100)] for i in xrange(len(gt)): curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]} if i < lookBack: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue( not failed , "An exception should have been generated, but wasn't") else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue (customEM.getMetric()["value"] == t1[i-lookBack]) #Windowed Scalar for lookBack in range(3): customFunc = """def getError(pred,ground,tools): return tools.getFieldValue(%d,"test1")""" % lookBack customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc,"storeWindow":storeWindow})) gt = [i for i in range(100)] p = [2*i for i in range(100)] t1 = [3*i for i in range(100)] t2 = [str(4*i) for i in range(100)] for i in xrange(len(gt)): curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]} if i < lookBack or lookBack>=storeWindow: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue (not failed , "An exception should have been generated, but wasn't") else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue( customEM.getMetric()["value"] == t1[i-lookBack]) #Not-Windowed category for lookBack in range(3): customFunc = """def getError(pred,ground,tools): return tools.getFieldValue(%d,"test1")""" % lookBack customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc})) gt = [i for i in range(100)] p = [2*i for i in range(100)] t1 = [3*i for i in range(100)] t2 = [str(4*i) for i in range(100)] for i in xrange(len(gt)): curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]} if i < lookBack: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue( not failed , "An exception should have been generated, but wasn't") else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue (customEM.getMetric()["value"] == t1[i-lookBack]) #Windowed category for lookBack in range(3): customFunc = """def getError(pred,ground,tools): return tools.getFieldValue(%d,"test1")""" % lookBack customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc,"storeWindow":storeWindow})) gt = [i for i in range(100)] p = [2*i for i in range(100)] t1 = [3*i for i in range(100)] t2 = [str(4*i) for i in range(100)] for i in xrange(len(gt)): curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]} if i < lookBack or lookBack>=storeWindow: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue (not failed , "An exception should have been generated, but wasn't") else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue (customEM.getMetric()["value"] == t1[i-lookBack]) #Test getBufferLen #Not-Windowed customFunc = """def getError(pred,ground,tools): return tools.getBufferLen()""" customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc})) gt = [i for i in range(100)] p = [2*i for i in range(100)] t1 = [3*i for i in range(100)] t2 = [str(4*i) for i in range(100)] for i in xrange(len(gt)): curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]} customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue (customEM.getMetric()["value"] == i+1) #Windowed customFunc = """def getError(pred,ground,tools): return tools.getBufferLen()""" customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc,"storeWindow":storeWindow})) gt = [i for i in range(100)] p = [2*i for i in range(100)] t1 = [3*i for i in range(100)] t2 = [str(4*i) for i in range(100)] for i in xrange(len(gt)): curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]} customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue (customEM.getMetric()["value"] == min(i+1, 4)) #Test initialization edge cases try: customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc,"errorWindow":0})) self.assertTrue (False , "error Window of 0 should fail self.assertTrue") except: pass try: customEM = getModule(MetricSpec("custom_error_metric", None, None, {"customFuncSource":customFunc,"storeWindow":0})) self.assertTrue (False , "error Window of 0 should fail self.assertTrue") except: pass
def testCustomErrorMetric(self): customFunc = """def getError(pred,ground,tools): return abs(pred-ground)""" customEM = getModule( MetricSpec("custom_error_metric", None, None, { "customFuncSource": customFunc, "errorWindow": 3 })) gt = [9, 4, 5, 6] p = [0, 13, 8, 3] for i in xrange(len(gt)): aggErr = customEM.addInstance(gt[i], p[i]) target = 5.0 delta = 0.001 # insure that addInstance returns the aggregate error - other # uber metrics depend on this behavior. self.assertEqual(aggErr, customEM.getMetric()["value"]) self.assertTrue(abs(customEM.getMetric()["value"] - target) < delta) customFunc = """def getError(pred,ground,tools): sum = 0 for i in range(min(3,tools.getBufferLen())): sum+=abs(tools.getPrediction(i)-tools.getGroundTruth(i)) return sum/3""" customEM = getModule( MetricSpec("custom_error_metric", None, None, {"customFuncSource": customFunc})) gt = [9, 4, 5, 6] p = [0, 13, 8, 3] for i in xrange(len(gt)): customEM.addInstance(gt[i], p[i]) target = 5.0 delta = 0.001 self.assertTrue(abs(customEM.getMetric()["value"] - target) < delta) # Test custom error metric helper functions # Test getPrediction # Not-Windowed storeWindow = 4 failed = False for lookBack in range(3): customFunc = """def getError(pred,ground,tools): return tools.getPrediction(%d)""" % lookBack customEM = getModule( MetricSpec("custom_error_metric", None, None, {"customFuncSource": customFunc})) gt = [i for i in range(100)] p = [2 * i for i in range(100)] t1 = [3 * i for i in range(100)] t2 = [str(4 * i) for i in range(100)] for i in xrange(len(gt)): curRecord = { "pred": p[i], "ground": gt[i], "test1": t1[i], "test2": t2[i] } if i < lookBack: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue( not failed, "An exception should have been generated, but wasn't" ) else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue( customEM.getMetric()["value"] == p[i - lookBack]) #Windowed for lookBack in range(5): customFunc = """def getError(pred,ground,tools): return tools.getPrediction(%d)""" % lookBack customEM = getModule( MetricSpec("custom_error_metric", None, None, { "customFuncSource": customFunc, "storeWindow": storeWindow })) gt = [i for i in range(100)] p = [2 * i for i in range(100)] t1 = [3 * i for i in range(100)] t2 = [str(4 * i) for i in range(100)] for i in xrange(len(gt)): curRecord = { "pred": p[i], "ground": gt[i], "test1": t1[i], "test2": t2[i] } if lookBack >= storeWindow - 1: pass if i < lookBack or lookBack >= storeWindow: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue( not failed, "An exception should have been generated, but wasn't" ) else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue( customEM.getMetric()["value"] == p[i - lookBack]) #Test getGroundTruth #Not-Windowed for lookBack in range(3): customFunc = """def getError(pred,ground,tools): return tools.getGroundTruth(%d)""" % lookBack customEM = getModule( MetricSpec("custom_error_metric", None, None, {"customFuncSource": customFunc})) gt = [i for i in range(100)] p = [2 * i for i in range(100)] t1 = [3 * i for i in range(100)] t2 = [str(4 * i) for i in range(100)] for i in xrange(len(gt)): curRecord = { "pred": p[i], "ground": gt[i], "test1": t1[i], "test2": t2[i] } if i < lookBack: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue( not failed, "An exception should have been generated, but wasn't" ) else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue( customEM.getMetric()["value"] == gt[i - lookBack]) #Windowed for lookBack in range(5): customFunc = """def getError(pred,ground,tools): return tools.getGroundTruth(%d)""" % lookBack customEM = getModule( MetricSpec("custom_error_metric", None, None, { "customFuncSource": customFunc, "storeWindow": storeWindow })) gt = [i for i in range(100)] p = [2 * i for i in range(100)] t1 = [3 * i for i in range(100)] t2 = [str(4 * i) for i in range(100)] for i in xrange(len(gt)): curRecord = { "pred": p[i], "ground": gt[i], "test1": t1[i], "test2": t2[i] } if i < lookBack or lookBack >= storeWindow: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue( not failed, "An exception should have been generated, but wasn't" ) else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue( customEM.getMetric()["value"] == gt[i - lookBack]) #Test getFieldValue #Not-Windowed Scalar for lookBack in range(3): customFunc = """def getError(pred,ground,tools): return tools.getFieldValue(%d,"test1")""" % lookBack customEM = getModule( MetricSpec("custom_error_metric", None, None, {"customFuncSource": customFunc})) gt = [i for i in range(100)] p = [2 * i for i in range(100)] t1 = [3 * i for i in range(100)] t2 = [str(4 * i) for i in range(100)] for i in xrange(len(gt)): curRecord = { "pred": p[i], "ground": gt[i], "test1": t1[i], "test2": t2[i] } if i < lookBack: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue( not failed, "An exception should have been generated, but wasn't" ) else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue( customEM.getMetric()["value"] == t1[i - lookBack]) #Windowed Scalar for lookBack in range(3): customFunc = """def getError(pred,ground,tools): return tools.getFieldValue(%d,"test1")""" % lookBack customEM = getModule( MetricSpec("custom_error_metric", None, None, { "customFuncSource": customFunc, "storeWindow": storeWindow })) gt = [i for i in range(100)] p = [2 * i for i in range(100)] t1 = [3 * i for i in range(100)] t2 = [str(4 * i) for i in range(100)] for i in xrange(len(gt)): curRecord = { "pred": p[i], "ground": gt[i], "test1": t1[i], "test2": t2[i] } if i < lookBack or lookBack >= storeWindow: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue( not failed, "An exception should have been generated, but wasn't" ) else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue( customEM.getMetric()["value"] == t1[i - lookBack]) #Not-Windowed category for lookBack in range(3): customFunc = """def getError(pred,ground,tools): return tools.getFieldValue(%d,"test1")""" % lookBack customEM = getModule( MetricSpec("custom_error_metric", None, None, {"customFuncSource": customFunc})) gt = [i for i in range(100)] p = [2 * i for i in range(100)] t1 = [3 * i for i in range(100)] t2 = [str(4 * i) for i in range(100)] for i in xrange(len(gt)): curRecord = { "pred": p[i], "ground": gt[i], "test1": t1[i], "test2": t2[i] } if i < lookBack: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue( not failed, "An exception should have been generated, but wasn't" ) else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue( customEM.getMetric()["value"] == t1[i - lookBack]) #Windowed category for lookBack in range(3): customFunc = """def getError(pred,ground,tools): return tools.getFieldValue(%d,"test1")""" % lookBack customEM = getModule( MetricSpec("custom_error_metric", None, None, { "customFuncSource": customFunc, "storeWindow": storeWindow })) gt = [i for i in range(100)] p = [2 * i for i in range(100)] t1 = [3 * i for i in range(100)] t2 = [str(4 * i) for i in range(100)] for i in xrange(len(gt)): curRecord = { "pred": p[i], "ground": gt[i], "test1": t1[i], "test2": t2[i] } if i < lookBack or lookBack >= storeWindow: try: customEM.addInstance(gt[i], p[i], curRecord) failed = True except: self.assertTrue( not failed, "An exception should have been generated, but wasn't" ) else: customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue( customEM.getMetric()["value"] == t1[i - lookBack]) #Test getBufferLen #Not-Windowed customFunc = """def getError(pred,ground,tools): return tools.getBufferLen()""" customEM = getModule( MetricSpec("custom_error_metric", None, None, {"customFuncSource": customFunc})) gt = [i for i in range(100)] p = [2 * i for i in range(100)] t1 = [3 * i for i in range(100)] t2 = [str(4 * i) for i in range(100)] for i in xrange(len(gt)): curRecord = { "pred": p[i], "ground": gt[i], "test1": t1[i], "test2": t2[i] } customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue(customEM.getMetric()["value"] == i + 1) #Windowed customFunc = """def getError(pred,ground,tools): return tools.getBufferLen()""" customEM = getModule( MetricSpec("custom_error_metric", None, None, { "customFuncSource": customFunc, "storeWindow": storeWindow })) gt = [i for i in range(100)] p = [2 * i for i in range(100)] t1 = [3 * i for i in range(100)] t2 = [str(4 * i) for i in range(100)] for i in xrange(len(gt)): curRecord = { "pred": p[i], "ground": gt[i], "test1": t1[i], "test2": t2[i] } customEM.addInstance(gt[i], p[i], curRecord) self.assertTrue(customEM.getMetric()["value"] == min(i + 1, 4)) #Test initialization edge cases try: customEM = getModule( MetricSpec("custom_error_metric", None, None, { "customFuncSource": customFunc, "errorWindow": 0 })) self.assertTrue(False, "error Window of 0 should fail self.assertTrue") except: pass try: customEM = getModule( MetricSpec("custom_error_metric", None, None, { "customFuncSource": customFunc, "storeWindow": 0 })) self.assertTrue(False, "error Window of 0 should fail self.assertTrue") except: pass