def testMultiMetric(self):
ms1 = MetricSpec(field='a', metric='trivial', inferenceElement='prediction', params={'errorMetric': 'aae', 'window': 1000, 'steps': 1})
ms2 = MetricSpec(metric='trivial', inferenceElement='prediction', field='a', params={'window': 10, 'steps': 1, 'errorMetric': 'rmse'})
metric1000 = getModule(ms1)
metric10 = getModule(ms2)
# create multi metric
multi = MetricMulti(weights=[0.2, 0.8], metrics=[metric10, metric1000])
multi.verbosity = 1
# create reference metrics (must be diff from metrics above used in MultiMetric, as they keep history)
metric1000ref = getModule(ms1)
metric10ref = getModule(ms2)
gt = range(500, 1000)
p = range(500)
for i in xrange(len(gt)):
v10=metric10ref.addInstance(gt[i], p[i])
v1000=metric1000ref.addInstance(gt[i], p[i])
if v10 is None or v1000 is None:
check=None
else:
check=0.2*float(v10) + 0.8*float(v1000)
metricValue = multi.addInstance(gt[i], p[i])
self.assertEqual(check, metricValue, "iter i= %s gt=%s pred=%s multi=%s sub1=%s sub2=%s" % (i, gt[i], p[i], metricValue, v10, v1000))
def testNegativeLogLikelihood(self):
# make sure negativeLogLikelihood returns correct LL numbers
# mock objects for ClassifierInput and ModelResult (see opfutils.py)
class MockClassifierInput(object):
def __init__(self, bucketIdx):
self.bucketIndex = bucketIdx
class MockModelResult(object):
def __init__(self, bucketll, bucketIdx):
self.inferences = {'multiStepBucketLikelihoods': {1: bucketll}}
self.classifierInput = MockClassifierInput(bucketIdx)
bucketLL = {0: 1.0, 1: 0, 2: 0, 3: 0} # model prediction as a dictionary
gt_bucketIdx = 0 # bucket index for ground truth
negLL = getModule(MetricSpec("negativeLogLikelihood", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY}))
negLL.addInstance(0, 0, record = None,
result=MockModelResult(bucketLL, gt_bucketIdx))
target = 0.0 # -log(1.0)
self.assertAlmostEqual(negLL.getMetric()["value"], target)
bucketLL = {0: 0.5, 1: 0.5, 2: 0, 3: 0} # model prediction as a dictionary
gt_bucketIdx = 0 # bucket index for ground truth
negLL = getModule(MetricSpec("negativeLogLikelihood", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY}))
negLL.addInstance(0, 0, record = None,
result=MockModelResult(bucketLL, gt_bucketIdx))
target = 0.6931471 # -log(0.5)
self.assertTrue(abs(negLL.getMetric()["value"]-target)
< OPFMetricsTest.DELTA)
# test accumulated negLL for multiple steps
bucketLL = []
bucketLL.append({0: 1, 1: 0, 2: 0, 3: 0})
bucketLL.append({0: 0, 1: 1, 2: 0, 3: 0})
bucketLL.append({0: 0, 1: 0, 2: 1, 3: 0})
bucketLL.append({0: 0, 1: 0, 2: 0, 3: 1})
gt_bucketIdx = [0, 2, 1, 3]
negLL = getModule(MetricSpec("negativeLogLikelihood", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY}))
for i in xrange(len(bucketLL)):
negLL.addInstance(0, 0, record = None,
result=MockModelResult(bucketLL[i], gt_bucketIdx[i]))
target = 5.756462
self.assertTrue(abs(negLL.getMetric()["value"]-target)
< OPFMetricsTest.DELTA)
def testTwoGramScalarsStepsGreaterOne(self):
"""Two gram scalars test with step size other than 1"""
oneGram = getModule(MetricSpec("two_gram", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY,\
"window":100, "predictionField":"test",
"errorMetric":"acc", "steps": 2}))
# Sequences of 0,1,2,3,4,0,1,2,3,4,...
encodings = [np.zeros(10) for i in range(5)]
for i in range(len(encodings)):
encoding = encodings[i]
encoding[i] = 1
gt = [i%5 for i in range(1000)]
res = []
for i in xrange(len(gt)):
if i == 20:
# Make sure we don"t barf with missing values
oneGram.addInstance(np.zeros(10), prediction=None,
record={"test":None})
else:
# Feed in next groundTruth
oneGram.addInstance(encodings[i%5], prediction=None,
record={"test":gt[i]})
res.append(oneGram.getMetric()["value"])
target = 1.0
self.assertTrue(abs(oneGram.getMetric()["value"]-target) \
< OPFMetricsTest.DELTA)
def testTwoGramStrings(self):
"""One gram string test"""
oneGram = getModule(MetricSpec("two_gram", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"acc",
"predictionField":"test"}))
# Sequences of "0", "1", "2", "3", "4", "0", "1", ...
gt = [str(i%5) for i in range(1000)]
encodings = [np.zeros(10) for i in range(5)]
for i in range(len(encodings)):
encoding = encodings[i]
encoding[i] = 1
# Make every 5th element random
newElem = 100
for i in range(5, 1000, 5):
gt[i] = str(newElem)
newElem += 20
res = []
for i in xrange(len(gt)):
if i==20:
# Make sure we don"t barf with missing values
oneGram.addInstance(np.zeros(10), prediction=None,
record={"test":None})
else:
oneGram.addInstance(encodings[i%5], prediction=None,
record={"test":gt[i]})
res.append(oneGram.getMetric()["value"])
target = .8
self.assertTrue(abs(oneGram.getMetric()["value"]-target)\
< OPFMetricsTest.DELTA)
def testMovingModeAccuracy(self):
"""Moving mode Accuracy metric test"""
movingModeACC = getModule(
MetricSpec(
"moving_mode",
None,
None,
{"verbosity": OPFMetricsTest.VERBOSITY, "window": 100, "mode_window": 3, "errorMetric": "acc"},
)
)
# Should initially asymptote to .5
# Then after 900 should go to 0.0 as the predictions will always be offset
gt = [i / 4 for i in range(900)]
gt.extend([2 * i / 4 for i in range(100)])
p = [i for i in range(1000)]
res = []
for i in xrange(len(gt)):
movingModeACC.addInstance(gt[i], p[i])
res.append(movingModeACC.getMetric()["value"])
# Make sure that there is no point where the average acc is <.5
self.assertTrue(min(res[1:899]) == 0.5)
# Make sure that after the statistics switch the acc goes to 0.0
self.assertTrue(max(res[900:]) <= 0.5)
# Make sure that the statistics change is still noticeable while it
# is in the window
self.assertTrue(res[998] > 0.0)
target = 0.0
self.assertTrue(abs(movingModeACC.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testTwoGramScalarsStepsGreaterOne(self):
"""Two gram scalars test with step size other than 1"""
oneGram = getModule(MetricSpec("two_gram", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY,\
"window":100, "predictionField":"test",
"errorMetric":"acc", "steps": 2}))
# Sequences of 0,1,2,3,4,0,1,2,3,4,...
encodings = [np.zeros(10) for i in range(5)]
for i in range(len(encodings)):
encoding = encodings[i]
encoding[i] = 1
gt = [i%5 for i in range(1000)]
res = []
for i in xrange(len(gt)):
if i == 20:
# Make sure we don"t barf with missing values
oneGram.addInstance(np.zeros(10), prediction=None,
record={"test":None})
else:
# Feed in next groundTruth
oneGram.addInstance(encodings[i%5], prediction=None,
record={"test":gt[i]})
res.append(oneGram.getMetric()["value"])
target = 1.0
self.assertTrue(abs(oneGram.getMetric()["value"]-target) \
< OPFMetricsTest.DELTA)
def testAAE(self):
aae = getModule(MetricSpec("aae", None, None, {"verbosity": OPFMetricsTest.VERBOSITY}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
aae.addInstance(gt[i], p[i])
target = 6.0
self.assertTrue(abs(aae.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testNegLLMultiplePrediction(self):
# In cases where the ground truth has multiple possible outcomes, make sure
# that the prediction that captures ground truth distribution has best LL
# and models that gives single prediction (either most likely outcome or
# average outcome) has worse LL
# mock objects for ClassifierInput and ModelResult (see opfutils.py)
class MockClassifierInput(object):
def __init__(self, bucketIdx):
self.bucketIndex = bucketIdx
class MockModelResult(object):
def __init__(self, bucketll, bucketIdx):
self.inferences = {'multiStepBucketLikelihoods': {1: bucketll}}
self.classifierInput = MockClassifierInput(bucketIdx)
# the ground truth lies in bucket 0 with p=0.45, in bucket 1 with p=0.0
# and in bucket 2 with p=0.55
gt_bucketIdx = [0]*45+[2]*55
# compare neg log-likelihood for three type of model predictions
# a model that predicts ground truth distribution
prediction_gt = {0: 0.45, 1: 0, 2: 0.55}
# a model that predicts only the most likely outcome
prediction_ml = {0: 0.0, 1: 0, 2: 1.0}
# a model that only predicts mean (bucket 1)
prediction_mean = {0: 0, 1: 1, 2: 0}
negLL_gt = getModule(MetricSpec("negativeLogLikelihood", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY}))
negLL_ml = getModule(MetricSpec("negativeLogLikelihood", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY}))
negLL_mean = getModule(MetricSpec("negativeLogLikelihood", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY}))
for i in xrange(len(gt_bucketIdx)):
negLL_gt.addInstance(0, 0, record = None,
result=MockModelResult(prediction_gt, gt_bucketIdx[i]))
negLL_ml.addInstance(0, 0, record = None,
result=MockModelResult(prediction_ml, gt_bucketIdx[i]))
negLL_mean.addInstance(0, 0, record = None,
result=MockModelResult(prediction_mean, gt_bucketIdx[i]))
self.assertTrue(negLL_gt.getMetric()["value"] < negLL_ml.getMetric()["value"])
self.assertTrue(negLL_gt.getMetric()["value"] < negLL_mean.getMetric()["value"])
def testWindowedRMSE(self):
wrmse = getModule(MetricSpec("rmse", None, None, {"verbosity": OPFMetricsTest.VERBOSITY, "window": 3}))
gt = [9, 4, 4, 100, 44]
p = [0, 13, 4, 6, 7]
for gv, pv in zip(gt, p):
wrmse.addInstance(gv, pv)
target = 58.324
self.assertTrue(abs(wrmse.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testAverageError(self):
"""Ave Error"""
err = getModule(MetricSpec("avg_err", None, None, {"verbosity": OPFMetricsTest.VERBOSITY}))
gt = [1, 1, 2, 3, 4, 5]
p = [0, 1, 2, 4, 5, 6]
for i in xrange(len(gt)):
err.addInstance(gt[i], p[i])
target = 2.0 / 3.0
self.assertTrue(abs(err.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testAccuracy(self):
"""Accuracy"""
acc = getModule(MetricSpec("acc", None, None, {"verbosity": OPFMetricsTest.VERBOSITY}))
gt = [0, 1, 2, 3, 4, 5]
p = [0, 1, 2, 4, 5, 6]
for i in xrange(len(gt)):
acc.addInstance(gt[i], p[i])
target = 0.5
self.assertTrue(abs(acc.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testNRMSE(self):
nrmse = getModule(MetricSpec("nrmse", None, None, {"verbosity": OPFMetricsTest.VERBOSITY}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
nrmse.addInstance(gt[i], p[i])
target = 3.5856858280031814
self.assertAlmostEqual(nrmse.getMetric()["value"], target)
def testAAE(self):
aae = getModule(MetricSpec("aae", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
aae.addInstance(gt[i], p[i])
target = 6.0
self.assertTrue(abs(aae.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testWindowedAAE(self):
"""Windowed AAE"""
waae = getModule(MetricSpec("aae", None, None, {"verbosity": OPFMetricsTest.VERBOSITY, "window": 1}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
waae.addInstance(gt[i], p[i])
target = 3.0
self.assertTrue(abs(waae.getMetric()["value"] - target) < OPFMetricsTest.DELTA, "Got %s" % waae.getMetric())
def testTrivialAccuracy(self):
trivialaccuracy = getModule(MetricSpec("trivial", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY,"errorMetric":"acc"}))
gt = [str(i/4+1) for i in range(100)]
p = [str(i) for i in range(100)]
for i in xrange(len(gt)):
trivialaccuracy.addInstance(gt[i], p[i])
target = .75
self.assertTrue(abs(trivialaccuracy.getMetric()["value"]-target) \
< OPFMetricsTest.DELTA)
def testTrivialAccuracy(self):
trivialaccuracy = getModule(MetricSpec("trivial", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY,"errorMetric":"acc"}))
gt = [str(i/4+1) for i in range(100)]
p = [str(i) for i in range(100)]
for i in xrange(len(gt)):
trivialaccuracy.addInstance(gt[i], p[i])
target = .75
self.assertTrue(abs(trivialaccuracy.getMetric()["value"]-target) \
< OPFMetricsTest.DELTA)
def testTrivialAAE(self):
trivialaae = getModule(MetricSpec("trivial", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY,"errorMetric":"aae"}))
gt = [i/4+1 for i in range(100)]
p = [i for i in range(100)]
for i in xrange(len(gt)):
trivialaae.addInstance(gt[i], p[i])
target = .25
self.assertTrue(abs(trivialaae.getMetric()["value"]-target) \
< OPFMetricsTest.DELTA)
def testTrivialAAE(self):
trivialaae = getModule(MetricSpec("trivial", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY,"errorMetric":"aae"}))
gt = [i/4+1 for i in range(100)]
p = [i for i in range(100)]
for i in xrange(len(gt)):
trivialaae.addInstance(gt[i], p[i])
target = .25
self.assertTrue(abs(trivialaae.getMetric()["value"]-target) \
< OPFMetricsTest.DELTA)
def testAverageError(self):
"""Ave Error"""
err = getModule(MetricSpec("avg_err", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY}))
gt = [1, 1, 2, 3, 4, 5]
p = [0, 1, 2, 4, 5, 6]
for i in xrange(len(gt)):
err.addInstance(gt[i], p[i])
target = (2.0/3.0)
self.assertTrue(abs(err.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testAccuracy(self):
"""Accuracy"""
acc = getModule(MetricSpec("acc", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY}))
gt = [0, 1, 2, 3, 4, 5]
p = [0, 1, 2, 4, 5, 6]
for i in xrange(len(gt)):
acc.addInstance(gt[i], p[i])
target = 0.5
self.assertTrue(abs(acc.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testLongWindowRMSE(self):
"""RMSE"""
rmse = getModule(MetricSpec("rmse", None, None, {"verbosity": OPFMetricsTest.VERBOSITY, "window": 100}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
rmse.addInstance(gt[i], p[i])
target = 6.71
self.assertTrue(abs(rmse.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testWindowedTrivialAAE (self):
"""Trivial Average Error metric test"""
trivialAveErr = getModule(MetricSpec("trivial", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY,"errorMetric":"avg_err"}))
gt = [str(i/4+1) for i in range(100)]
p = [str(i) for i in range(100)]
for i in xrange(len(gt)):
trivialAveErr.addInstance(gt[i], p[i])
target = .25
self.assertTrue(abs(trivialAveErr.getMetric()["value"]-target)\
< OPFMetricsTest.DELTA)
def testWindowedTrivialAAE (self):
"""Trivial Average Error metric test"""
trivialAveErr = getModule(MetricSpec("trivial", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY,"errorMetric":"avg_err"}))
gt = [str(i/4+1) for i in range(100)]
p = [str(i) for i in range(100)]
for i in xrange(len(gt)):
trivialAveErr.addInstance(gt[i], p[i])
target = .25
self.assertTrue(abs(trivialAveErr.getMetric()["value"]-target)\
< OPFMetricsTest.DELTA)
def testWindowedRMSE(self):
wrmse = getModule(MetricSpec("rmse", None, None,
{"verbosity": OPFMetricsTest.VERBOSITY, "window":3}))
gt = [9, 4, 4, 100, 44]
p = [0, 13, 4, 6, 7]
for gv, pv in zip(gt, p):
wrmse.addInstance(gv, pv)
target = 58.324
self.assertTrue (abs(wrmse.getMetric()["value"]-target)\
< OPFMetricsTest.DELTA)
def testWindowedAAE(self):
"""Windowed AAE"""
waae = getModule(MetricSpec("aae", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":1}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
waae.addInstance(gt[i], p[i])
target = 3.0
self.assertTrue( abs(waae.getMetric()["value"]-target) \
< OPFMetricsTest.DELTA, "Got %s" %waae.getMetric())
def testNRMSE(self):
nrmse = getModule(
MetricSpec("nrmse", None, None,
{"verbosity": OPFMetricsTest.VERBOSITY}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
nrmse.addInstance(gt[i], p[i])
target = 3.5856858280031814
self.assertAlmostEqual(nrmse.getMetric()["value"], target)
def testMultiMetric(self):
ms1 = MetricSpec(field='a',
metric='trivial',
inferenceElement='prediction',
params={
'errorMetric': 'aae',
'window': 1000,
'steps': 1
})
ms2 = MetricSpec(metric='trivial',
inferenceElement='prediction',
field='a',
params={
'window': 10,
'steps': 1,
'errorMetric': 'rmse'
})
metric1000 = getModule(ms1)
metric10 = getModule(ms2)
# create multi metric
multi = MetricMulti(weights=[0.2, 0.8], metrics=[metric10, metric1000])
multi.verbosity = 1
# create reference metrics (must be diff from metrics above used in MultiMetric, as they keep history)
metric1000ref = getModule(ms1)
metric10ref = getModule(ms2)
gt = range(500, 1000)
p = range(500)
for i in xrange(len(gt)):
v10 = metric10ref.addInstance(gt[i], p[i])
v1000 = metric1000ref.addInstance(gt[i], p[i])
if v10 is None or v1000 is None:
check = None
else:
check = 0.2 * float(v10) + 0.8 * float(v1000)
metricValue = multi.addInstance(gt[i], p[i])
self.assertEqual(
check, metricValue,
"iter i= %s gt=%s pred=%s multi=%s sub1=%s sub2=%s" %
(i, gt[i], p[i], metricValue, v10, v1000))
def testLongWindowRMSE(self):
"""RMSE"""
rmse = getModule(MetricSpec("rmse", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
rmse.addInstance(gt[i], p[i])
target = 6.71
self.assertTrue(abs(rmse.getMetric()["value"]-target)\
< OPFMetricsTest.DELTA)
def testMultistepProbability(self):
"""Multistep with probabilities metric test"""
msp = getModule(MetricSpec("multiStepProbability", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"aae",
"steps":3}))
gt = [5 for i in range(1000)]
p = [{3: {i: .3, 5: .7}} for i in range(1000)]
for i in xrange(len(gt)):
msp.addInstance(gt[i], p[i])
#((999-5)(1000-5)/2-(899-5)(900-5)/2)*.3/100
target = 283.35
self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testWindowedAverageError(self):
"""Windowed Ave Error"""
err = getModule(MetricSpec("avg_err", None, None, \
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":2}))
gt = [0, 1, 2, 3, 4, 5]
p = [0, 1, 2, 4, 5, 6]
for i in range(len(gt)):
err.addInstance(gt[i], p[i])
target = 1.0
self.assertTrue(
abs(err.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testMultistepProbability(self):
"""Multistep with probabilities metric test"""
msp = getModule(MetricSpec("multiStepProbability", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"aae",
"steps":3}))
gt = [5 for i in range(1000)]
p = [{3: {i: .3, 5: .7}} for i in range(1000)]
for i in xrange(len(gt)):
msp.addInstance(gt[i], p[i])
#((999-5)(1000-5)/2-(899-5)(900-5)/2)*.3/100
target = 283.35
self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testMultistepAAE(self):
"""Multistep AAE metric test"""
msp = getModule(MetricSpec("multiStep", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"aae",
"steps": 3}))
# Make each ground truth 1 greater than the prediction
gt = [i+1 for i in range(100)]
p = [{3: {i: .7, 5: 0.3}} for i in range(100)]
for i in xrange(len(gt)):
msp.addInstance(gt[i], p[i])
target = 1
self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testMultistepAAE(self):
"""Multistep AAE metric test"""
msp = getModule(MetricSpec("multiStep", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"aae",
"steps": 3}))
# Make each ground truth 1 greater than the prediction
gt = [i+1 for i in range(100)]
p = [{3: {i: .7, 5: 0.3}} for i in range(100)]
for i in xrange(len(gt)):
msp.addInstance(gt[i], p[i])
target = 1
self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testMultistepProbabilityMultipleSteps(self):
"""Multistep with probabilities metric test, predicting 2 different step
sizes"""
msp = getModule(MetricSpec("multiStepProbability", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100,
"errorMetric":"aae", "steps": [1,3]}))
gt = [5 for i in range(1000)]
p = [{3: {i: .3, 5: .7},
1: {5: 1.0}} for i in range(1000)]
for i in xrange(len(gt)):
msp.addInstance(gt[i], p[i])
#(((999-5)(1000-5)/2-(899-5)(900-5)/2)*.3/100) / 2
# / 2 because the 1-step prediction is 100% accurate
target = 283.35/2
self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testMultistepProbabilityMultipleSteps(self):
"""Multistep with probabilities metric test, predicting 2 different step
sizes"""
msp = getModule(MetricSpec("multiStepProbability", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100,
"errorMetric":"aae", "steps": [1,3]}))
gt = [5 for i in range(1000)]
p = [{3: {i: .3, 5: .7},
1: {5: 1.0}} for i in range(1000)]
for i in xrange(len(gt)):
msp.addInstance(gt[i], p[i])
#(((999-5)(1000-5)/2-(899-5)(900-5)/2)*.3/100) / 2
# / 2 because the 1-step prediction is 100% accurate
target = 283.35/2
self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testMultistepAAEMultipleSteps(self):
"""Multistep AAE metric test, predicting 2 different step sizes"""
msp = getModule(MetricSpec("multiStep", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"aae",
"steps": [3,6]}))
# Make each 3 step prediction +1 over ground truth and each 6 step
# prediction +0.5 over ground truth
gt = [i for i in range(100)]
p = [{3: {i+1: .7, 5: 0.3},
6: {i+0.5: .7, 5: 0.3}} for i in range(100)]
for i in xrange(len(gt)):
msp.addInstance(gt[i], p[i])
target = 0.75 # average of +1 error and 0.5 error
self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testMultistepAAEMultipleSteps(self):
"""Multistep AAE metric test, predicting 2 different step sizes"""
msp = getModule(MetricSpec("multiStep", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "errorMetric":"aae",
"steps": [3,6]}))
# Make each 3 step prediction +1 over ground truth and each 6 step
# prediction +0.5 over ground truth
gt = [i for i in range(100)]
p = [{3: {i+1: .7, 5: 0.3},
6: {i+0.5: .7, 5: 0.3}} for i in range(100)]
for i in xrange(len(gt)):
msp.addInstance(gt[i], p[i])
target = 0.75 # average of +1 error and 0.5 error
self.assertTrue(abs(msp.getMetric()["value"]-target) < OPFMetricsTest.DELTA)
def testMovingMeanRMSE(self):
"""Moving mean RMSE metric test"""
movingMeanRMSE = getModule(MetricSpec("moving_mean", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "mean_window":3,
"errorMetric":"rmse"}))
gt = [i for i in range(890)]
gt.extend([2*i for i in range(110)])
p = [i for i in range(1000)]
res = []
for i in xrange(len(gt)):
movingMeanRMSE.addInstance(gt[i], p[i])
res.append(movingMeanRMSE.getMetric()["value"])
self.assertTrue(max(res[1:890]) == 2.0)
self.assertTrue(min(res[891:])>=4.0)
target = 4.0
self.assertTrue(abs(movingMeanRMSE.getMetric()["value"]-target) \
< OPFMetricsTest.DELTA)
def testMovingMeanRMSE(self):
"""Moving mean RMSE metric test"""
movingMeanRMSE = getModule(MetricSpec("moving_mean", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "mean_window":3,
"errorMetric":"rmse"}))
gt = [i for i in range(890)]
gt.extend([2*i for i in range(110)])
p = [i for i in range(1000)]
res = []
for i in xrange(len(gt)):
movingMeanRMSE.addInstance(gt[i], p[i])
res.append(movingMeanRMSE.getMetric()["value"])
self.assertTrue(max(res[1:890]) == 2.0)
self.assertTrue(min(res[891:])>=4.0)
target = 4.0
self.assertTrue(abs(movingMeanRMSE.getMetric()["value"]-target) \
< OPFMetricsTest.DELTA)
def __constructMetricsModules(self, metricSpecs):
"""
Creates the required metrics modules
Parameters:
-----------------------------------------------------------------------
metricSpecs:
A sequence of MetricSpec objects that specify which metric modules to
instantiate
"""
if not metricSpecs:
return
self.__metricSpecs = metricSpecs
for spec in metricSpecs:
if not InferenceElement.validate(spec.inferenceElement):
raise ValueError("Invalid inference element for metric spec: %r" %spec)
self.__metrics.append(metrics.getModule(spec))
self.__metricLabels.append(spec.getLabel())
def __constructMetricsModules(self, metricSpecs):
"""
Creates the required metrics modules
Parameters:
-----------------------------------------------------------------------
metricSpecs:
A sequence of MetricSpec objects that specify which metric modules to
instantiate
"""
if not metricSpecs:
return
self.__metricSpecs = metricSpecs
for spec in metricSpecs:
if not InferenceElement.validate(spec.inferenceElement):
raise ValueError("Invalid inference element for metric spec: %r" %spec)
self.__metrics.append(metrics.getModule(spec))
self.__metricLabels.append(spec.getLabel())
def testTwoGramStrings(self):
"""One gram string test"""
oneGram = getModule(
MetricSpec(
"two_gram", None, None, {
"verbosity": OPFMetricsTest.VERBOSITY,
"window": 100,
"errorMetric": "acc",
"predictionField": "test"
}))
# Sequences of "0", "1", "2", "3", "4", "0", "1", ...
gt = [str(i % 5) for i in range(1000)]
encodings = [np.zeros(10) for i in range(5)]
for i in range(len(encodings)):
encoding = encodings[i]
encoding[i] = 1
# Make every 5th element random
newElem = 100
for i in range(5, 1000, 5):
gt[i] = str(newElem)
newElem += 20
res = []
for i in xrange(len(gt)):
if i == 20:
# Make sure we don"t barf with missing values
oneGram.addInstance(np.zeros(10),
prediction=None,
record={"test": None})
else:
oneGram.addInstance(encodings[i % 5],
prediction=None,
record={"test": gt[i]})
res.append(oneGram.getMetric()["value"])
target = .8
self.assertTrue(abs(oneGram.getMetric()["value"]-target)\
< OPFMetricsTest.DELTA)
def testMovingModeAccuracy(self):
"""Moving mode Accuracy metric test"""
movingModeACC = getModule(MetricSpec("moving_mode", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "mode_window":3,
"errorMetric":"acc"}))
#Should initially asymptote to .5
#Then after 900 should go to 0.0 as the predictions will always be offset
gt = [i/4 for i in range(900)]
gt.extend([2*i/4 for i in range(100)])
p = [i for i in range(1000)]
res = []
for i in xrange(len(gt)):
movingModeACC.addInstance(gt[i], p[i])
res.append(movingModeACC.getMetric()["value"])
#Make sure that there is no point where the average acc is <.5
self.assertTrue(min(res[1:899]) == .5)
#Make sure that after the statistics switch the acc goes to 0.0
self.assertTrue(max(res[900:])<=.5)
#Make sure that the statistics change is still noticeable while it
#is in the window
self.assertTrue(res[998]>0.0)
target = 0.0
self.assertTrue(abs(movingModeACC.getMetric()["value"]-target)\
< OPFMetricsTest.DELTA)
def testMovingModeAverageError(self):
"""Moving mode Average Error metric test"""
movingModeAvgErr = getModule(MetricSpec("moving_mode", None, None,
{"verbosity" : OPFMetricsTest.VERBOSITY, "window":100, "mode_window":3,
"errorMetric":"avg_err"}))
#Should initially assymptote to .5
#Then after 900 should go to 1.0 as the predictions will always be offset
gt = [i/4 for i in range(900)]
gt.extend([2*i/4 for i in range(100)])
p = [i for i in range(1000)]
res = []
for i in xrange(len(gt)):
movingModeAvgErr.addInstance(gt[i], p[i])
res.append(movingModeAvgErr.getMetric()["value"])
#Make sure that there is no point where the average error is >.5
self.assertTrue(max(res[1:890]) == .5)
#Make sure that after the statistics switch the error goes to 1.0
self.assertTrue(min(res[891:])>=.5)
#Make sure that the statistics change is still noticeable while it is
#in the window
self.assertTrue(res[998]<1.0)
target = 1.0
self.assertTrue(abs(movingModeAvgErr.getMetric()["value"]-target)\
< OPFMetricsTest.DELTA)
printTPRegionParams(model._getTPRegion())
inputData = "%s/%s.csv" % (DATA_DIR, dataSet.replace(" ", "_"))
sensor = model._getSensorRegion()
encoderList = sensor.getSelf().encoder.getEncoderList()
if sensor.getSelf().disabledEncoder is not None:
classifier_encoder = sensor.getSelf().disabledEncoder.getEncoderList()
classifier_encoder = classifier_encoder[0]
else:
classifier_encoder = None
_METRIC_SPECS = getMetricSpecs(predictedField,
stepsAhead=_options.stepsAhead)
metric = metrics.getModule(_METRIC_SPECS[0])
metricsManager = MetricsManager(_METRIC_SPECS, model.getFieldInfo(),
model.getInferenceType())
if plot:
plotCount = 1
plotHeight = max(plotCount * 3, 6)
fig = plt.figure(figsize=(14, plotHeight))
gs = gridspec.GridSpec(plotCount, 1)
plt.title(predictedField)
plt.ylabel('Data')
plt.xlabel('Timed')
plt.tight_layout()
plt.ion()
print "Load dataset: ", dataSet
print model._spLearningEnabled
printTPRegionParams(model._getTPRegion())
inputData = "%s/%s.csv" % (DATA_DIR, dataSet.replace(" ", "_"))
sensor = model._getSensorRegion()
encoderList = sensor.getSelf().encoder.getEncoderList()
if sensor.getSelf().disabledEncoder is not None:
classifier_encoder = sensor.getSelf().disabledEncoder.getEncoderList()
classifier_encoder = classifier_encoder[0]
else:
classifier_encoder = None
_METRIC_SPECS = getMetricSpecs(predictedField, stepsAhead=_options.stepsAhead)
metric = metrics.getModule(_METRIC_SPECS[0])
metricsManager = MetricsManager(_METRIC_SPECS, model.getFieldInfo(),
model.getInferenceType())
if plot:
plotCount = 1
plotHeight = max(plotCount * 3, 6)
fig = plt.figure(figsize=(14, plotHeight))
gs = gridspec.GridSpec(plotCount, 1)
plt.title(predictedField)
plt.ylabel('Data')
plt.xlabel('Timed')
plt.tight_layout()
plt.ion()
print "Load dataset: ", dataSet
def testNegativeLogLikelihood(self):
# make sure negativeLogLikelihood returns correct LL numbers
# mock objects for ClassifierInput and ModelResult (see opfutils.py)
class MockClassifierInput(object):
def __init__(self, bucketIdx):
self.bucketIndex = bucketIdx
class MockModelResult(object):
def __init__(self, bucketll, bucketIdx):
self.inferences = {'multiStepBucketLikelihoods': {1: bucketll}}
self.classifierInput = MockClassifierInput(bucketIdx)
bucketLL = {
0: 1.0,
1: 0,
2: 0,
3: 0
} # model prediction as a dictionary
gt_bucketIdx = 0 # bucket index for ground truth
negLL = getModule(
MetricSpec("negativeLogLikelihood", None, None,
{"verbosity": OPFMetricsTest.VERBOSITY}))
negLL.addInstance(0,
0,
record=None,
result=MockModelResult(bucketLL, gt_bucketIdx))
target = 0.0 # -log(1.0)
self.assertAlmostEqual(negLL.getMetric()["value"], target)
bucketLL = {
0: 0.5,
1: 0.5,
2: 0,
3: 0
} # model prediction as a dictionary
gt_bucketIdx = 0 # bucket index for ground truth
negLL = getModule(
MetricSpec("negativeLogLikelihood", None, None,
{"verbosity": OPFMetricsTest.VERBOSITY}))
negLL.addInstance(0,
0,
record=None,
result=MockModelResult(bucketLL, gt_bucketIdx))
target = 0.6931471 # -log(0.5)
self.assertTrue(
abs(negLL.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
# test accumulated negLL for multiple steps
bucketLL = []
bucketLL.append({0: 1, 1: 0, 2: 0, 3: 0})
bucketLL.append({0: 0, 1: 1, 2: 0, 3: 0})
bucketLL.append({0: 0, 1: 0, 2: 1, 3: 0})
bucketLL.append({0: 0, 1: 0, 2: 0, 3: 1})
gt_bucketIdx = [0, 2, 1, 3]
negLL = getModule(
MetricSpec("negativeLogLikelihood", None, None,
{"verbosity": OPFMetricsTest.VERBOSITY}))
for i in xrange(len(bucketLL)):
negLL.addInstance(0,
0,
record=None,
result=MockModelResult(bucketLL[i],
gt_bucketIdx[i]))
target = 5.756462
self.assertTrue(
abs(negLL.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
def testCustomErrorMetric(self):
customFunc = """def getError(pred,ground,tools):
return abs(pred-ground)"""
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc, "errorWindow":3}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
aggErr = customEM.addInstance(gt[i], p[i])
target = 5.0
delta = 0.001
# insure that addInstance returns the aggregate error - other
# uber metrics depend on this behavior.
self.assertEqual(aggErr, customEM.getMetric()["value"])
self.assertTrue(abs(customEM.getMetric()["value"]-target) < delta)
customFunc = """def getError(pred,ground,tools):
sum = 0
for i in range(min(3,tools.getBufferLen())):
sum+=abs(tools.getPrediction(i)-tools.getGroundTruth(i))
return sum/3"""
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
customEM.addInstance(gt[i], p[i])
target = 5.0
delta = 0.001
self.assertTrue(abs(customEM.getMetric()["value"]-target) < delta)
# Test custom error metric helper functions
# Test getPrediction
# Not-Windowed
storeWindow=4
failed = False
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getPrediction(%d)""" % lookBack
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc}))
gt = [i for i in range(100)]
p = [2*i for i in range(100)]
t1 = [3*i for i in range(100)]
t2 = [str(4*i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]}
if i < lookBack:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue( not failed,
"An exception should have been generated, but wasn't")
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue( customEM.getMetric()["value"] == p[i-lookBack])
#Windowed
for lookBack in range(5):
customFunc = """def getError(pred,ground,tools):
return tools.getPrediction(%d)""" % lookBack
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc,"storeWindow":storeWindow}))
gt = [i for i in range(100)]
p = [2*i for i in range(100)]
t1 = [3*i for i in range(100)]
t2 = [str(4*i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]}
if lookBack>=storeWindow-1:
pass
if i < lookBack or lookBack>=storeWindow:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue (not failed ,
"An exception should have been generated, but wasn't")
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue (customEM.getMetric()["value"] == p[i-lookBack])
#Test getGroundTruth
#Not-Windowed
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getGroundTruth(%d)""" % lookBack
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc}))
gt = [i for i in range(100)]
p = [2*i for i in range(100)]
t1 = [3*i for i in range(100)]
t2 = [str(4*i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]}
if i < lookBack:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue( not failed ,
"An exception should have been generated, but wasn't")
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue (customEM.getMetric()["value"] == gt[i-lookBack])
#Windowed
for lookBack in range(5):
customFunc = """def getError(pred,ground,tools):
return tools.getGroundTruth(%d)""" % lookBack
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc,"storeWindow":storeWindow}))
gt = [i for i in range(100)]
p = [2*i for i in range(100)]
t1 = [3*i for i in range(100)]
t2 = [str(4*i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]}
if i < lookBack or lookBack>=storeWindow:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue( not failed ,
"An exception should have been generated, but wasn't")
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue( customEM.getMetric()["value"] == gt[i-lookBack])
#Test getFieldValue
#Not-Windowed Scalar
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getFieldValue(%d,"test1")""" % lookBack
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc}))
gt = [i for i in range(100)]
p = [2*i for i in range(100)]
t1 = [3*i for i in range(100)]
t2 = [str(4*i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]}
if i < lookBack:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue( not failed ,
"An exception should have been generated, but wasn't")
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue (customEM.getMetric()["value"] == t1[i-lookBack])
#Windowed Scalar
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getFieldValue(%d,"test1")""" % lookBack
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc,"storeWindow":storeWindow}))
gt = [i for i in range(100)]
p = [2*i for i in range(100)]
t1 = [3*i for i in range(100)]
t2 = [str(4*i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]}
if i < lookBack or lookBack>=storeWindow:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue (not failed ,
"An exception should have been generated, but wasn't")
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue( customEM.getMetric()["value"] == t1[i-lookBack])
#Not-Windowed category
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getFieldValue(%d,"test1")""" % lookBack
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc}))
gt = [i for i in range(100)]
p = [2*i for i in range(100)]
t1 = [3*i for i in range(100)]
t2 = [str(4*i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]}
if i < lookBack:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue( not failed ,
"An exception should have been generated, but wasn't")
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue (customEM.getMetric()["value"] == t1[i-lookBack])
#Windowed category
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getFieldValue(%d,"test1")""" % lookBack
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc,"storeWindow":storeWindow}))
gt = [i for i in range(100)]
p = [2*i for i in range(100)]
t1 = [3*i for i in range(100)]
t2 = [str(4*i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]}
if i < lookBack or lookBack>=storeWindow:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue (not failed ,
"An exception should have been generated, but wasn't")
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue (customEM.getMetric()["value"] == t1[i-lookBack])
#Test getBufferLen
#Not-Windowed
customFunc = """def getError(pred,ground,tools):
return tools.getBufferLen()"""
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc}))
gt = [i for i in range(100)]
p = [2*i for i in range(100)]
t1 = [3*i for i in range(100)]
t2 = [str(4*i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]}
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue (customEM.getMetric()["value"] == i+1)
#Windowed
customFunc = """def getError(pred,ground,tools):
return tools.getBufferLen()"""
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc,"storeWindow":storeWindow}))
gt = [i for i in range(100)]
p = [2*i for i in range(100)]
t1 = [3*i for i in range(100)]
t2 = [str(4*i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {"pred":p[i], "ground":gt[i], "test1":t1[i], "test2":t2[i]}
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue (customEM.getMetric()["value"] == min(i+1, 4))
#Test initialization edge cases
try:
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc,"errorWindow":0}))
self.assertTrue (False , "error Window of 0 should fail self.assertTrue")
except:
pass
try:
customEM = getModule(MetricSpec("custom_error_metric", None, None,
{"customFuncSource":customFunc,"storeWindow":0}))
self.assertTrue (False , "error Window of 0 should fail self.assertTrue")
except:
pass
def testCustomErrorMetric(self):
customFunc = """def getError(pred,ground,tools):
return abs(pred-ground)"""
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"errorWindow": 3
}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
aggErr = customEM.addInstance(gt[i], p[i])
target = 5.0
delta = 0.001
# insure that addInstance returns the aggregate error - other
# uber metrics depend on this behavior.
self.assertEqual(aggErr, customEM.getMetric()["value"])
self.assertTrue(abs(customEM.getMetric()["value"] - target) < delta)
customFunc = """def getError(pred,ground,tools):
sum = 0
for i in range(min(3,tools.getBufferLen())):
sum+=abs(tools.getPrediction(i)-tools.getGroundTruth(i))
return sum/3"""
customEM = getModule(
MetricSpec("custom_error_metric", None, None,
{"customFuncSource": customFunc}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
customEM.addInstance(gt[i], p[i])
target = 5.0
delta = 0.001
self.assertTrue(abs(customEM.getMetric()["value"] - target) < delta)
# Test custom error metric helper functions
# Test getPrediction
# Not-Windowed
storeWindow = 4
failed = False
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getPrediction(%d)""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None,
{"customFuncSource": customFunc}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == p[i - lookBack])
#Windowed
for lookBack in range(5):
customFunc = """def getError(pred,ground,tools):
return tools.getPrediction(%d)""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"storeWindow": storeWindow
}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if lookBack >= storeWindow - 1:
pass
if i < lookBack or lookBack >= storeWindow:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == p[i - lookBack])
#Test getGroundTruth
#Not-Windowed
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getGroundTruth(%d)""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None,
{"customFuncSource": customFunc}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == gt[i - lookBack])
#Windowed
for lookBack in range(5):
customFunc = """def getError(pred,ground,tools):
return tools.getGroundTruth(%d)""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"storeWindow": storeWindow
}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack or lookBack >= storeWindow:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == gt[i - lookBack])
#Test getFieldValue
#Not-Windowed Scalar
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getFieldValue(%d,"test1")""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None,
{"customFuncSource": customFunc}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == t1[i - lookBack])
#Windowed Scalar
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getFieldValue(%d,"test1")""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"storeWindow": storeWindow
}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack or lookBack >= storeWindow:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == t1[i - lookBack])
#Not-Windowed category
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getFieldValue(%d,"test1")""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None,
{"customFuncSource": customFunc}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == t1[i - lookBack])
#Windowed category
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getFieldValue(%d,"test1")""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"storeWindow": storeWindow
}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack or lookBack >= storeWindow:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == t1[i - lookBack])
#Test getBufferLen
#Not-Windowed
customFunc = """def getError(pred,ground,tools):
return tools.getBufferLen()"""
customEM = getModule(
MetricSpec("custom_error_metric", None, None,
{"customFuncSource": customFunc}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(customEM.getMetric()["value"] == i + 1)
#Windowed
customFunc = """def getError(pred,ground,tools):
return tools.getBufferLen()"""
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"storeWindow": storeWindow
}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(customEM.getMetric()["value"] == min(i + 1, 4))
#Test initialization edge cases
try:
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"errorWindow": 0
}))
self.assertTrue(False,
"error Window of 0 should fail self.assertTrue")
except:
pass
try:
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"storeWindow": 0
}))
self.assertTrue(False,
"error Window of 0 should fail self.assertTrue")
except:
pass
