# computed for this experiment
'metrics':[],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
'loggedMetrics': ['.*aae.*'],
}
# Add multi-step prediction metrics
for steps in config['predictionSteps']:
control['metrics'].append(
MetricSpec(field=config['predictedField'], metric='multiStep',
inferenceElement='multiStepBestPredictions',
params={'errorMetric': 'aae', 'window': 1000, 'steps': steps}))
control['metrics'].append(
MetricSpec(field=config['predictedField'], metric='trivial',
inferenceElement='prediction',
params={'errorMetric': 'aae', 'window': 1000, 'steps': steps}))
control['metrics'].append(
MetricSpec(field=config['predictedField'], metric='multiStep',
inferenceElement='multiStepBestPredictions',
params={'errorMetric': 'altMAPE', 'window': 1000, 'steps': steps}))
control['metrics'].append(
MetricSpec(field=config['predictedField'], metric='trivial',
inferenceElement='prediction',
params={'errorMetric': 'altMAPE', 'window': 1000, 'steps': steps}))
#
# iterationCount of -1 = iterate over the entire dataset
'iterationCount':
-1,
# A dictionary containing all the supplementary parameters for inference
"inferenceArgs": {
'predictedField': config['predictedField'],
'predictionSteps': config['predictionSteps']
},
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics': [
MetricSpec(field=config['predictedField'],
metric=metricName,
inferenceElement='prediction',
params={'window': config['windowSize']}),
MetricSpec(field=config['predictedField'],
metric='trivial',
inferenceElement='prediction',
params={
'errorMetric': metricName,
'window': config['windowSize']
}),
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
-1,
# A dictionary containing all the supplementary parameters for inference
"inferenceArgs": {
u'inputPredictedField': 'auto',
u'predictedField': u'kw_energy_consumption',
u'predictionSteps': [10]
},
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics': [
MetricSpec(field=u'kw_energy_consumption',
metric='multiStep',
inferenceElement='multiStepBestPredictions',
params={
'window': 1000,
'steps': [10],
'errorMetric': 'aae'
}),
MetricSpec(field=u'kw_energy_consumption',
metric='multiStep',
inferenceElement='multiStepBestPredictions',
params={
'window': 1000,
'steps': [10],
'errorMetric': 'altMAPE'
})
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
u'version': 1},
# Iteration count: maximum number of iterations. Each iteration corresponds
# to one record from the (possibly aggregated) dataset. The task is
# terminated when either number of iterations reaches iterationCount or
# all records in the (possibly aggregated) database have been processed,
# whichever occurs first.
#
# iterationCount of -1 = iterate over the entire dataset
#'iterationCount' : ITERATION_COUNT,
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics':[
MetricSpec(field=u'attendance', metric='multiStep',
inferenceElement='multiStepBestPredictions',
params={'window': 1000, 'steps': [0], 'errorMetric': 'aae'}),
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
'loggedMetrics': ['.*'],
}
descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
control=control)
},
# Iteration count: maximum number of iterations. Each iteration corresponds
# to one record from the (possibly aggregated) dataset. The task is
# terminated when either number of iterations reaches iterationCount or
# all records in the (possibly aggregated) database have been processed,
# whichever occurs first.
#
# iterationCount of -1 = iterate over the entire dataset
#'iterationCount' : ITERATION_COUNT,
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics': [
MetricSpec(field=u'attendance',
inferenceElement=InferenceElement.prediction,
metric='aae',
params={'window': 1000}),
MetricSpec(field=u'attendance',
inferenceElement=InferenceElement.prediction,
metric='trivial_aae',
params={'window': 1000}),
MetricSpec(field=u'attendance',
inferenceElement=InferenceElement.prediction,
metric='nupicScore_scalar',
params={
'frequencyWindow': 1000,
'movingAverageWindow': 1000
}),
MetricSpec(field=u'attendance',
inferenceElement=InferenceElement.prediction,
metric='nupicScore_scalar',
#
# iterationCount of -1 = iterate over the entire dataset
'iterationCount':
-1,
# A dictionary containing all the supplementary parameters for inference
"inferenceArgs": {
u'predictedField': u'f',
'predictionSteps': [1]
},
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics': [
MetricSpec(field=u'f',
metric='aae',
inferenceElement='prediction',
params={'window': 100}),
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
'loggedMetrics': ['.*nupicScore.*'],
}
################################################################################
################################################################################
descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
control=control)
'iterationCount':
20,
# A dictionary containing all the supplementary parameters for inference
"inferenceArgs": {
u'predictedField': u'value',
u'predictionSteps': [1, 5]
},
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics': [
MetricSpec(field=u'value',
metric='multiStep',
inferenceElement='multiStepBestPredictions',
params={
'window': 10,
'steps': 1,
'errorMetric': 'aae'
}),
MetricSpec(field=u'value',
metric='multiStep',
inferenceElement='multiStepBestPredictions',
params={
'window': 10,
'steps': 5,
'errorMetric': 'aae'
}),
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
plt.xlabel('time [s]')
plt.ylabel('failures')
DESCRIPTION = (
"Starts a NuPIC model from the model params returned by the swarm\n"
"and pushes each line of input from the gym into the model. Results\n"
"are written to an output file (default) or plotted dynamically if\n"
"the --plot option is specified.\n"
"NOTE: You must run ./swarm.py before this, because model parameters\n"
"are required to run NuPIC.\n")
_METRIC_SPECS = (
MetricSpec(field='class',
metric='multiStep',
inferenceElement='multiStepBestPredictions',
params={
'errorMetric': 'aae',
'window': 1000,
'steps': 1
}),
MetricSpec(field='class',
metric='trivial',
inferenceElement='prediction',
params={
'errorMetric': 'aae',
'window': 1000,
'steps': 1
}),
MetricSpec(field='class',
metric='multiStep',
inferenceElement='multiStepBestPredictions',
params={
-1,
# A dictionary containing all the supplementary parameters for inference
"inferenceArgs": {
u'inputPredictedField': 'auto',
u'predictedField': u'vaccine_name',
u'predictionSteps': [1]
},
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics': [
MetricSpec(field=u'vaccine_name',
metric='multiStep',
inferenceElement='multiStepBestPredictions',
params={
'window': 1000,
'steps': [1],
'errorMetric': 'avg_err'
})
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
'loggedMetrics': ['.*'],
}
descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
control=control)
# iterationCount of -1 = iterate over the entire dataset
#'iterationCount' : ITERATION_COUNT,
# A dictionary containing all the supplementary parameters for inference
"inferenceArgs": {
u'predictedField': u'consumption',
u'predictionSteps': [0]
},
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics': [
MetricSpec(field=u'consumption',
metric='multiStep',
inferenceElement='multiStepBestPredictions',
params={
'window': 1000,
'steps': [0],
'errorMetric': 'avg_err'
})
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
'loggedMetrics': ['.*'],
}
################################################################################
################################################################################
def _testMetricsMgr():
print "*Testing Metrics Managers*..."
from nupic.data.fieldmeta import (FieldMetaInfo, FieldMetaType,
FieldMetaSpecial)
from nupic.frameworks.opf.metrics import MetricSpec
from nupic.frameworks.opf.opfutils import ModelResult, SensorInput
onlineMetrics = (MetricSpec(metric="aae", inferenceElement='', \
field="consumption", params={}),)
print "TESTING METRICS MANAGER (BASIC PLUMBING TEST)..."
modelFieldMetaInfo = (FieldMetaInfo(name='temperature',
type=FieldMetaType.float,
special=FieldMetaSpecial.none),
FieldMetaInfo(name='consumption',
type=FieldMetaType.float,
special=FieldMetaSpecial.none))
# -----------------------------------------------------------------------
# Test to make sure that invalid InferenceElements are caught
try:
MetricsManager(metricSpecs=onlineMetrics,
fieldInfo=modelFieldMetaInfo,
inferenceType=InferenceType.TemporalNextStep)
except ValueError:
print "Caught bad inference element: PASS"
print
onlineMetrics = (MetricSpec(metric="aae",
inferenceElement=InferenceElement.prediction,
field="consumption",
params={}), )
temporalMetrics = MetricsManager(
metricSpecs=onlineMetrics,
fieldInfo=modelFieldMetaInfo,
inferenceType=InferenceType.TemporalNextStep)
inputs = [
{
'groundTruthRow': [9, 7],
'predictionsDict': {
InferenceType.TemporalNextStep: [12, 17]
}
},
{
'groundTruthRow': [12, 17],
'predictionsDict': {
InferenceType.TemporalNextStep: [14, 19]
}
},
{
'groundTruthRow': [14, 20],
'predictionsDict': {
InferenceType.TemporalNextStep: [16, 21]
}
},
{
'groundTruthRow': [9, 7],
'predictionsDict': {
InferenceType.TemporalNextStep: None
}
},
]
for element in inputs:
groundTruthRow = element['groundTruthRow']
tPredictionRow = element['predictionsDict'][
InferenceType.TemporalNextStep]
result = ModelResult(sensorInput=SensorInput(dataRow=groundTruthRow,
dataEncodings=None,
sequenceReset=0,
category=None),
inferences={'prediction': tPredictionRow})
temporalMetrics.update(result)
assert temporalMetrics.getMetrics().values()[0] == 15.0 / 3.0, \
"Expected %f, got %f" %(15.0/3.0,
temporalMetrics.getMetrics().values()[0])
print "ok"
return
# to one record from the (possibly aggregated) dataset. The task is
# terminated when either number of iterations reaches iterationCount or
# all records in the (possibly aggregated) database have been processed,
# whichever occurs first.
#
# iterationCount of -1 = iterate over the entire dataset
'iterationCount' : -1,
# A dictionary containing all the supplementary parameters for inference
"inferenceArgs":{u'predictedField': u'consumption', u'predictionSteps': [1, 5]},
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics':[
MetricSpec(field=u'consumption', metric='multiStep', inferenceElement='multiStepBestPredictions', params={'window': 1000, 'steps': [1, 5], 'errorMetric': 'aae'}),
MetricSpec(field=u'consumption', metric='multiStep', inferenceElement='multiStepBestPredictions', params={'window': 1000, 'steps': [1, 5], 'errorMetric': 'altMAPE'}),
MetricSpec(field=u'consumption', metric='grokScore_scalar', inferenceElement='encodings', params={'frequencyWindow': 1000, 'movingAverageWindow': 1000}),
MetricSpec(field=u'consumption', metric='grokScore_scalar', inferenceElement='encodings', params={'frequencyWindow': 1000})
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
'loggedMetrics': ['.*'],
}
################################################################################
################################################################################
# iterationCount of -1 = iterate over the entire dataset
'iterationCount': -1,
# Task Control parameters for OPFTaskDriver (per opfTaskControlSchema.json)
'taskControl': {
# Iteration cycle list consisting of opftaskdriver.IterationPhaseSpecXXXXX
# instances.
'iterationCycle': [
#IterationPhaseSpecLearnOnly(1000),
IterationPhaseSpecLearnAndInfer(1000, inferenceArgs=None),
#IterationPhaseSpecInferOnly(10, inferenceArgs=None),
],
'metrics': [
MetricSpec(field=u'consumption',
inferenceElement='prediction',
metric='aae',
params={'window': 200}),
MetricSpec(field=u'consumption',
inferenceElement='encodings',
metric='grokScore_scalar',
params={
'frequencyWindow': 200,
'movingAverageWindow': 200
}),
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
},
# Iteration count: maximum number of iterations. Each iteration corresponds
# to one record from the (possibly aggregated) dataset. The task is
# terminated when either number of iterations reaches iterationCount or
# all records in the (possibly aggregated) database have been processed,
# whichever occurs first.
#
# iterationCount of -1 = iterate over the entire dataset
#'iterationCount' : ITERATION_COUNT,
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics': [
MetricSpec(field=u'consumption',
inferenceElement=InferenceElement.prediction,
metric='rmse'),
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
'loggedMetrics': ['.*nupicScore.*'],
}
################################################################################
################################################################################
descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
control=control)
# all records in the (possibly aggregated) database have been processed,
# whichever occurs first.
#
# iterationCount of -1 = iterate over the entire dataset
'iterationCount' : -1,
# A dictionary containing all the supplementary parameters for inference
"inferenceArgs":{u'predictedField': u'classification', u'predictionSteps': [0]},
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics':[
MetricSpec(field='classification', metric='multiStep',
inferenceElement='multiStepBestPredictions',
params={'errorMetric': config['errorMetric'],
'window': 100,
'steps': 0}),
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
'loggedMetrics': ['.*'],
}
################################################################################
################################################################################
descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
# Task Control parameters for OPFTaskDriver (per opfTaskControlSchema.json)
'taskControl': {
# Iteration cycle list consisting of opf_task_driver.IterationPhaseSpecXXXXX
# instances.
'iterationCycle': [
#IterationPhaseSpecLearnOnly(1000),
IterationPhaseSpecLearnAndInfer(1000),
#IterationPhaseSpecInferOnly(10),
],
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics': [
MetricSpec(metric='avg_err',
inferenceElement='classification',
params={'window': 200}),
MetricSpec(metric='neg_auc',
inferenceElement='classConfidences',
params={
'window': 200,
'computeEvery': 10
}),
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
'loggedMetrics': ['.*avg_err.*', '.*auc.*'],
# iterationCount of -1 = iterate over the entire dataset
'iterationCount': -1,
# Task Control parameters for OPFTaskDriver (per opfTaskControlSchema.json)
'taskControl': {
# Iteration cycle list consisting of opf_task_driver.IterationPhaseSpecXXXXX
# instances.
'iterationCycle': [
#IterationPhaseSpecLearnOnly(1000),
IterationPhaseSpecLearnAndInfer(1000, inferenceArgs=None),
#IterationPhaseSpecInferOnly(10, inferenceArgs=None),
],
'metrics': [
MetricSpec(field='consumption',
inferenceElement='prediction',
metric='aae',
params={'window': 200}),
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
'loggedMetrics': ['.*nupicScore.*', ".*aae.*"],
# Callbacks for experimentation/research (optional)
'callbacks': {
# Callbacks to be called at the beginning of a task, before model iterations.
# Signature: callback(<reference to OPFExperiment>); returns nothing
# 'setup' : [htmPredictionModelControlEnableSPLearningCb, htmPredictionModelControlEnableTPLearningCb],
from nupic.frameworks.opf.predictionmetricsmanager import MetricsManager
import nupic_anomaly_output
MODEL_NAME = "TiltTemp2009"
DATA_DIR = "./data"
MODEL_PARAMS_DIR = "./model_params"
PREDICTED_FIELD = "Tilt_06"
# '7/2/10 0:00'
DATE_FORMAT = "%m/%d/%y %H:%M"
_METRIC_SPECS = (
MetricSpec(field=PREDICTED_FIELD,
metric="multiStep",
inferenceElement="multiStepBestPredictions",
params={
"errorMetric": "aae",
"window": 1000,
"steps": 1
}),
MetricSpec(field=PREDICTED_FIELD,
metric="trivial",
inferenceElement="prediction",
params={
"errorMetric": "aae",
"window": 1000,
"steps": 1
}),
MetricSpec(field=PREDICTED_FIELD,
metric="multiStep",
inferenceElement="multiStepBestPredictions",
params={
# Task Control parameters for OPFTaskDriver (per opfTaskControlSchema.json)
'taskControl' : {
# Iteration cycle list consisting of opftaskdriver.IterationPhaseSpecXXXXX
# instances.
'iterationCycle' : [
#IterationPhaseSpecLearnOnly(1000),
IterationPhaseSpecLearnAndInfer(1000, dict(predictedField="consumption")),
#IterationPhaseSpecInferOnly(10),
],
'metrics' :[
MetricSpec(metric='rmse',
field="consumption",
inferenceElement=InferenceElement.prediction),
],
# Callbacks for experimentation/research (optional)
'callbacks' : {
# Callbacks to be called at the beginning of a task, before model iterations.
# Signature: callback(<reference to OPFExperiment>); returns nothing
'setup' : [],
# Callbacks to be called after every learning/inference iteration
# Signature: callback(<reference to OPFExperiment>); returns nothing
'postIter' : [],
# Callbacks to be called when the experiment task is finished
# Signature: callback(<reference to OPFExperiment>); returns nothing
'iterationCount':
-1,
# A dictionary containing all the supplementary parameters for inference
"inferenceArgs": {
u'predictedField': u'c1',
u'predictionSteps': [1]
},
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics': [
MetricSpec(field=u'c1',
metric='multiStep',
inferenceElement='multiStepBestPredictions',
params={
'window': 1000,
'steps': [1],
'errorMetric': 'altMAPE'
}),
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
'loggedMetrics': ['.*'],
}
descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
control=control)
def testCustomErrorMetric(self):
customFunc = """def getError(pred,ground,tools):
return abs(pred-ground)"""
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"errorWindow": 3
}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
aggErr = customEM.addInstance(gt[i], p[i])
target = 5.0
delta = 0.001
# insure that addInstance returns the aggregate error - other
# uber metrics depend on this behavior.
self.assertEqual(aggErr, customEM.getMetric()["value"])
self.assertTrue(abs(customEM.getMetric()["value"] - target) < delta)
customFunc = """def getError(pred,ground,tools):
sum = 0
for i in range(min(3,tools.getBufferLen())):
sum+=abs(tools.getPrediction(i)-tools.getGroundTruth(i))
return sum/3"""
customEM = getModule(
MetricSpec("custom_error_metric", None, None,
{"customFuncSource": customFunc}))
gt = [9, 4, 5, 6]
p = [0, 13, 8, 3]
for i in xrange(len(gt)):
customEM.addInstance(gt[i], p[i])
target = 5.0
delta = 0.001
self.assertTrue(abs(customEM.getMetric()["value"] - target) < delta)
# Test custom error metric helper functions
# Test getPrediction
# Not-Windowed
storeWindow = 4
failed = False
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getPrediction(%d)""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None,
{"customFuncSource": customFunc}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == p[i - lookBack])
#Windowed
for lookBack in range(5):
customFunc = """def getError(pred,ground,tools):
return tools.getPrediction(%d)""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"storeWindow": storeWindow
}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if lookBack >= storeWindow - 1:
pass
if i < lookBack or lookBack >= storeWindow:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == p[i - lookBack])
#Test getGroundTruth
#Not-Windowed
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getGroundTruth(%d)""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None,
{"customFuncSource": customFunc}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == gt[i - lookBack])
#Windowed
for lookBack in range(5):
customFunc = """def getError(pred,ground,tools):
return tools.getGroundTruth(%d)""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"storeWindow": storeWindow
}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack or lookBack >= storeWindow:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == gt[i - lookBack])
#Test getFieldValue
#Not-Windowed Scalar
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getFieldValue(%d,"test1")""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None,
{"customFuncSource": customFunc}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == t1[i - lookBack])
#Windowed Scalar
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getFieldValue(%d,"test1")""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"storeWindow": storeWindow
}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack or lookBack >= storeWindow:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == t1[i - lookBack])
#Not-Windowed category
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getFieldValue(%d,"test1")""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None,
{"customFuncSource": customFunc}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == t1[i - lookBack])
#Windowed category
for lookBack in range(3):
customFunc = """def getError(pred,ground,tools):
return tools.getFieldValue(%d,"test1")""" % lookBack
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"storeWindow": storeWindow
}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
if i < lookBack or lookBack >= storeWindow:
try:
customEM.addInstance(gt[i], p[i], curRecord)
failed = True
except:
self.assertTrue(
not failed,
"An exception should have been generated, but wasn't"
)
else:
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(
customEM.getMetric()["value"] == t1[i - lookBack])
#Test getBufferLen
#Not-Windowed
customFunc = """def getError(pred,ground,tools):
return tools.getBufferLen()"""
customEM = getModule(
MetricSpec("custom_error_metric", None, None,
{"customFuncSource": customFunc}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(customEM.getMetric()["value"] == i + 1)
#Windowed
customFunc = """def getError(pred,ground,tools):
return tools.getBufferLen()"""
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"storeWindow": storeWindow
}))
gt = [i for i in range(100)]
p = [2 * i for i in range(100)]
t1 = [3 * i for i in range(100)]
t2 = [str(4 * i) for i in range(100)]
for i in xrange(len(gt)):
curRecord = {
"pred": p[i],
"ground": gt[i],
"test1": t1[i],
"test2": t2[i]
}
customEM.addInstance(gt[i], p[i], curRecord)
self.assertTrue(customEM.getMetric()["value"] == min(i + 1, 4))
#Test initialization edge cases
try:
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"errorWindow": 0
}))
self.assertTrue(False,
"error Window of 0 should fail self.assertTrue")
except:
pass
try:
customEM = getModule(
MetricSpec("custom_error_metric", None, None, {
"customFuncSource": customFunc,
"storeWindow": 0
}))
self.assertTrue(False,
"error Window of 0 should fail self.assertTrue")
except:
pass
#
# iterationCount of -1 = iterate over the entire dataset
'iterationCount':
-1,
# A dictionary containing all the supplementary parameters for inference
"inferenceArgs": {
u'predictedField': u'f',
u'predictionSteps': [1]
},
# Metrics: A list of MetricSpecs that instantiate the metrics that are
# computed for this experiment
'metrics': [
MetricSpec(field=u'f',
metric='passThruPrediction',
inferenceElement='anomalyScore',
params={'window': 1000}),
],
# Logged Metrics: A sequence of regular expressions that specify which of
# the metrics from the Inference Specifications section MUST be logged for
# every prediction. The regex's correspond to the automatically generated
# metric labels. This is similar to the way the optimization metric is
# specified in permutations.py.
'loggedMetrics': ['.*'],
}
descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
control=control)
def testNegativeLogLikelihood(self):
# make sure negativeLogLikelihood returns correct LL numbers
# mock objects for ClassifierInput and ModelResult (see opf_utils.py)
class MockClassifierInput(object):
def __init__(self, bucketIdx):
self.bucketIndex = bucketIdx
class MockModelResult(object):
def __init__(self, bucketll, bucketIdx):
self.inferences = {'multiStepBucketLikelihoods': {1: bucketll}}
self.classifierInput = MockClassifierInput(bucketIdx)
bucketLL = {
0: 1.0,
1: 0,
2: 0,
3: 0
} # model prediction as a dictionary
gt_bucketIdx = 0 # bucket index for ground truth
negLL = getModule(
MetricSpec("negativeLogLikelihood", None, None,
{"verbosity": OPFMetricsTest.VERBOSITY}))
negLL.addInstance(0,
0,
record=None,
result=MockModelResult(bucketLL, gt_bucketIdx))
target = 0.0 # -log(1.0)
self.assertAlmostEqual(negLL.getMetric()["value"], target)
bucketLL = {
0: 0.5,
1: 0.5,
2: 0,
3: 0
} # model prediction as a dictionary
gt_bucketIdx = 0 # bucket index for ground truth
negLL = getModule(
MetricSpec("negativeLogLikelihood", None, None,
{"verbosity": OPFMetricsTest.VERBOSITY}))
negLL.addInstance(0,
0,
record=None,
result=MockModelResult(bucketLL, gt_bucketIdx))
target = 0.6931471 # -log(0.5)
self.assertTrue(
abs(negLL.getMetric()["value"] - target) < OPFMetricsTest.DELTA)
# test accumulated negLL for multiple steps
bucketLL = []
bucketLL.append({0: 1, 1: 0, 2: 0, 3: 0})
bucketLL.append({0: 0, 1: 1, 2: 0, 3: 0})
bucketLL.append({0: 0, 1: 0, 2: 1, 3: 0})
bucketLL.append({0: 0, 1: 0, 2: 0, 3: 1})
gt_bucketIdx = [0, 2, 1, 3]
negLL = getModule(
MetricSpec("negativeLogLikelihood", None, None,
{"verbosity": OPFMetricsTest.VERBOSITY}))
for i in range(len(bucketLL)):
negLL.addInstance(0,
0,
record=None,
result=MockModelResult(bucketLL[i],
gt_bucketIdx[i]))
target = 5.756462
self.assertTrue(
abs(negLL.getMetric()["value"] - target) < OPFMetricsTest.DELTA)