예제 #1
0
  # computed for this experiment
  'metrics':[],
    
  # Logged Metrics: A sequence of regular expressions that specify which of
  # the metrics from the Inference Specifications section MUST be logged for
  # every prediction. The regex's correspond to the automatically generated
  # metric labels. This is similar to the way the optimization metric is
  # specified in permutations.py.
  'loggedMetrics': ['.*aae.*'],
}

# Add multi-step prediction metrics
for steps in config['predictionSteps']:
  control['metrics'].append(    
      MetricSpec(field=config['predictedField'], metric='multiStep', 
                 inferenceElement='multiStepBestPredictions', 
                 params={'errorMetric': 'aae', 'window': 1000, 'steps': steps}))
  control['metrics'].append(    
      MetricSpec(field=config['predictedField'], metric='trivial', 
                 inferenceElement='prediction', 
                 params={'errorMetric': 'aae', 'window': 1000, 'steps': steps}))
  control['metrics'].append(    
      MetricSpec(field=config['predictedField'], metric='multiStep', 
                 inferenceElement='multiStepBestPredictions', 
                 params={'errorMetric': 'altMAPE', 'window': 1000, 'steps': steps}))
  control['metrics'].append(    
      MetricSpec(field=config['predictedField'], metric='trivial', 
                 inferenceElement='prediction', 
                 params={'errorMetric': 'altMAPE', 'window': 1000, 'steps': steps}))

예제 #2
0
    #
    # iterationCount of -1 = iterate over the entire dataset
    'iterationCount':
    -1,

    # A dictionary containing all the supplementary parameters for inference
    "inferenceArgs": {
        'predictedField': config['predictedField'],
        'predictionSteps': config['predictionSteps']
    },

    # Metrics: A list of MetricSpecs that instantiate the metrics that are
    # computed for this experiment
    'metrics': [
        MetricSpec(field=config['predictedField'],
                   metric=metricName,
                   inferenceElement='prediction',
                   params={'window': config['windowSize']}),
        MetricSpec(field=config['predictedField'],
                   metric='trivial',
                   inferenceElement='prediction',
                   params={
                       'errorMetric': metricName,
                       'window': config['windowSize']
                   }),
    ],

    # Logged Metrics: A sequence of regular expressions that specify which of
    # the metrics from the Inference Specifications section MUST be logged for
    # every prediction. The regex's correspond to the automatically generated
    # metric labels. This is similar to the way the optimization metric is
    # specified in permutations.py.
예제 #3
0
    -1,

    # A dictionary containing all the supplementary parameters for inference
    "inferenceArgs": {
        u'inputPredictedField': 'auto',
        u'predictedField': u'kw_energy_consumption',
        u'predictionSteps': [10]
    },

    # Metrics: A list of MetricSpecs that instantiate the metrics that are
    # computed for this experiment
    'metrics': [
        MetricSpec(field=u'kw_energy_consumption',
                   metric='multiStep',
                   inferenceElement='multiStepBestPredictions',
                   params={
                       'window': 1000,
                       'steps': [10],
                       'errorMetric': 'aae'
                   }),
        MetricSpec(field=u'kw_energy_consumption',
                   metric='multiStep',
                   inferenceElement='multiStepBestPredictions',
                   params={
                       'window': 1000,
                       'steps': [10],
                       'errorMetric': 'altMAPE'
                   })
    ],

    # Logged Metrics: A sequence of regular expressions that specify which of
    # the metrics from the Inference Specifications section MUST be logged for
예제 #4
0
      u'version': 1},

  # Iteration count: maximum number of iterations.  Each iteration corresponds
  # to one record from the (possibly aggregated) dataset.  The task is
  # terminated when either number of iterations reaches iterationCount or
  # all records in the (possibly aggregated) database have been processed,
  # whichever occurs first.
  #
  # iterationCount of -1 = iterate over the entire dataset
  #'iterationCount' : ITERATION_COUNT,

  # Metrics: A list of MetricSpecs that instantiate the metrics that are
  # computed for this experiment
  'metrics':[
    MetricSpec(field=u'attendance', metric='multiStep', 
               inferenceElement='multiStepBestPredictions', 
               params={'window': 1000, 'steps': [0], 'errorMetric': 'aae'}),
  ],

  # Logged Metrics: A sequence of regular expressions that specify which of
  # the metrics from the Inference Specifications section MUST be logged for
  # every prediction. The regex's correspond to the automatically generated
  # metric labels. This is similar to the way the optimization metric is
  # specified in permutations.py.
  'loggedMetrics': ['.*'],
}



descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
                                                control=control)
예제 #5
0
    },

    # Iteration count: maximum number of iterations.  Each iteration corresponds
    # to one record from the (possibly aggregated) dataset.  The task is
    # terminated when either number of iterations reaches iterationCount or
    # all records in the (possibly aggregated) database have been processed,
    # whichever occurs first.
    #
    # iterationCount of -1 = iterate over the entire dataset
    #'iterationCount' : ITERATION_COUNT,

    # Metrics: A list of MetricSpecs that instantiate the metrics that are
    # computed for this experiment
    'metrics': [
        MetricSpec(field=u'attendance',
                   inferenceElement=InferenceElement.prediction,
                   metric='aae',
                   params={'window': 1000}),
        MetricSpec(field=u'attendance',
                   inferenceElement=InferenceElement.prediction,
                   metric='trivial_aae',
                   params={'window': 1000}),
        MetricSpec(field=u'attendance',
                   inferenceElement=InferenceElement.prediction,
                   metric='nupicScore_scalar',
                   params={
                       'frequencyWindow': 1000,
                       'movingAverageWindow': 1000
                   }),
        MetricSpec(field=u'attendance',
                   inferenceElement=InferenceElement.prediction,
                   metric='nupicScore_scalar',
예제 #6
0
    #
    # iterationCount of -1 = iterate over the entire dataset
    'iterationCount':
    -1,

    # A dictionary containing all the supplementary parameters for inference
    "inferenceArgs": {
        u'predictedField': u'f',
        'predictionSteps': [1]
    },

    # Metrics: A list of MetricSpecs that instantiate the metrics that are
    # computed for this experiment
    'metrics': [
        MetricSpec(field=u'f',
                   metric='aae',
                   inferenceElement='prediction',
                   params={'window': 100}),
    ],

    # Logged Metrics: A sequence of regular expressions that specify which of
    # the metrics from the Inference Specifications section MUST be logged for
    # every prediction. The regex's correspond to the automatically generated
    # metric labels. This is similar to the way the optimization metric is
    # specified in permutations.py.
    'loggedMetrics': ['.*nupicScore.*'],
}

################################################################################
################################################################################
descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
                                                control=control)
예제 #7
0
    'iterationCount':
    20,

    # A dictionary containing all the supplementary parameters for inference
    "inferenceArgs": {
        u'predictedField': u'value',
        u'predictionSteps': [1, 5]
    },

    # Metrics: A list of MetricSpecs that instantiate the metrics that are
    # computed for this experiment
    'metrics': [
        MetricSpec(field=u'value',
                   metric='multiStep',
                   inferenceElement='multiStepBestPredictions',
                   params={
                       'window': 10,
                       'steps': 1,
                       'errorMetric': 'aae'
                   }),
        MetricSpec(field=u'value',
                   metric='multiStep',
                   inferenceElement='multiStepBestPredictions',
                   params={
                       'window': 10,
                       'steps': 5,
                       'errorMetric': 'aae'
                   }),
    ],

    # Logged Metrics: A sequence of regular expressions that specify which of
    # the metrics from the Inference Specifications section MUST be logged for
예제 #8
0
plt.xlabel('time [s]')
plt.ylabel('failures')

DESCRIPTION = (
    "Starts a NuPIC model from the model params returned by the swarm\n"
    "and pushes each line of input from the gym into the model. Results\n"
    "are written to an output file (default) or plotted dynamically if\n"
    "the --plot option is specified.\n"
    "NOTE: You must run ./swarm.py before this, because model parameters\n"
    "are required to run NuPIC.\n")

_METRIC_SPECS = (
    MetricSpec(field='class',
               metric='multiStep',
               inferenceElement='multiStepBestPredictions',
               params={
                   'errorMetric': 'aae',
                   'window': 1000,
                   'steps': 1
               }),
    MetricSpec(field='class',
               metric='trivial',
               inferenceElement='prediction',
               params={
                   'errorMetric': 'aae',
                   'window': 1000,
                   'steps': 1
               }),
    MetricSpec(field='class',
               metric='multiStep',
               inferenceElement='multiStepBestPredictions',
               params={
    -1,

    # A dictionary containing all the supplementary parameters for inference
    "inferenceArgs": {
        u'inputPredictedField': 'auto',
        u'predictedField': u'vaccine_name',
        u'predictionSteps': [1]
    },

    # Metrics: A list of MetricSpecs that instantiate the metrics that are
    # computed for this experiment
    'metrics': [
        MetricSpec(field=u'vaccine_name',
                   metric='multiStep',
                   inferenceElement='multiStepBestPredictions',
                   params={
                       'window': 1000,
                       'steps': [1],
                       'errorMetric': 'avg_err'
                   })
    ],

    # Logged Metrics: A sequence of regular expressions that specify which of
    # the metrics from the Inference Specifications section MUST be logged for
    # every prediction. The regex's correspond to the automatically generated
    # metric labels. This is similar to the way the optimization metric is
    # specified in permutations.py.
    'loggedMetrics': ['.*'],
}

descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
                                                control=control)
예제 #10
0
    # iterationCount of -1 = iterate over the entire dataset
    #'iterationCount' : ITERATION_COUNT,

    # A dictionary containing all the supplementary parameters for inference
    "inferenceArgs": {
        u'predictedField': u'consumption',
        u'predictionSteps': [0]
    },

    # Metrics: A list of MetricSpecs that instantiate the metrics that are
    # computed for this experiment
    'metrics': [
        MetricSpec(field=u'consumption',
                   metric='multiStep',
                   inferenceElement='multiStepBestPredictions',
                   params={
                       'window': 1000,
                       'steps': [0],
                       'errorMetric': 'avg_err'
                   })
    ],

    # Logged Metrics: A sequence of regular expressions that specify which of
    # the metrics from the Inference Specifications section MUST be logged for
    # every prediction. The regex's correspond to the automatically generated
    # metric labels. This is similar to the way the optimization metric is
    # specified in permutations.py.
    'loggedMetrics': ['.*'],
}

################################################################################
################################################################################
예제 #11
0
def _testMetricsMgr():
    print "*Testing Metrics Managers*..."
    from nupic.data.fieldmeta import (FieldMetaInfo, FieldMetaType,
                                      FieldMetaSpecial)

    from nupic.frameworks.opf.metrics import MetricSpec
    from nupic.frameworks.opf.opfutils import ModelResult, SensorInput
    onlineMetrics = (MetricSpec(metric="aae", inferenceElement='', \
                                field="consumption", params={}),)

    print "TESTING METRICS MANAGER (BASIC PLUMBING TEST)..."

    modelFieldMetaInfo = (FieldMetaInfo(name='temperature',
                                        type=FieldMetaType.float,
                                        special=FieldMetaSpecial.none),
                          FieldMetaInfo(name='consumption',
                                        type=FieldMetaType.float,
                                        special=FieldMetaSpecial.none))

    # -----------------------------------------------------------------------
    # Test to make sure that invalid InferenceElements are caught
    try:
        MetricsManager(metricSpecs=onlineMetrics,
                       fieldInfo=modelFieldMetaInfo,
                       inferenceType=InferenceType.TemporalNextStep)
    except ValueError:
        print "Caught bad inference element: PASS"

    print
    onlineMetrics = (MetricSpec(metric="aae",
                                inferenceElement=InferenceElement.prediction,
                                field="consumption",
                                params={}), )

    temporalMetrics = MetricsManager(
        metricSpecs=onlineMetrics,
        fieldInfo=modelFieldMetaInfo,
        inferenceType=InferenceType.TemporalNextStep)

    inputs = [
        {
            'groundTruthRow': [9, 7],
            'predictionsDict': {
                InferenceType.TemporalNextStep: [12, 17]
            }
        },
        {
            'groundTruthRow': [12, 17],
            'predictionsDict': {
                InferenceType.TemporalNextStep: [14, 19]
            }
        },
        {
            'groundTruthRow': [14, 20],
            'predictionsDict': {
                InferenceType.TemporalNextStep: [16, 21]
            }
        },
        {
            'groundTruthRow': [9, 7],
            'predictionsDict': {
                InferenceType.TemporalNextStep: None
            }
        },
    ]

    for element in inputs:
        groundTruthRow = element['groundTruthRow']
        tPredictionRow = element['predictionsDict'][
            InferenceType.TemporalNextStep]

        result = ModelResult(sensorInput=SensorInput(dataRow=groundTruthRow,
                                                     dataEncodings=None,
                                                     sequenceReset=0,
                                                     category=None),
                             inferences={'prediction': tPredictionRow})

        temporalMetrics.update(result)

    assert temporalMetrics.getMetrics().values()[0] == 15.0 / 3.0, \
            "Expected %f, got %f" %(15.0/3.0,
                                    temporalMetrics.getMetrics().values()[0])
    print "ok"

    return
예제 #12
0
  # to one record from the (possibly aggregated) dataset.  The task is
  # terminated when either number of iterations reaches iterationCount or
  # all records in the (possibly aggregated) database have been processed,
  # whichever occurs first.
  #
  # iterationCount of -1 = iterate over the entire dataset
  'iterationCount' : -1,


  # A dictionary containing all the supplementary parameters for inference
  "inferenceArgs":{u'predictedField': u'consumption', u'predictionSteps': [1, 5]},

  # Metrics: A list of MetricSpecs that instantiate the metrics that are
  # computed for this experiment
  'metrics':[
    MetricSpec(field=u'consumption', metric='multiStep', inferenceElement='multiStepBestPredictions', params={'window': 1000, 'steps': [1, 5], 'errorMetric': 'aae'}),
    MetricSpec(field=u'consumption', metric='multiStep', inferenceElement='multiStepBestPredictions', params={'window': 1000, 'steps': [1, 5], 'errorMetric': 'altMAPE'}),
    MetricSpec(field=u'consumption', metric='grokScore_scalar', inferenceElement='encodings', params={'frequencyWindow': 1000, 'movingAverageWindow': 1000}),
    MetricSpec(field=u'consumption', metric='grokScore_scalar', inferenceElement='encodings', params={'frequencyWindow': 1000})
  ],

  # Logged Metrics: A sequence of regular expressions that specify which of
  # the metrics from the Inference Specifications section MUST be logged for
  # every prediction. The regex's correspond to the automatically generated
  # metric labels. This is similar to the way the optimization metric is
  # specified in permutations.py.
  'loggedMetrics': ['.*'],
}

################################################################################
################################################################################
예제 #13
0
            # iterationCount of -1 = iterate over the entire dataset
            'iterationCount': -1,

            # Task Control parameters for OPFTaskDriver (per opfTaskControlSchema.json)
            'taskControl': {

                # Iteration cycle list consisting of opftaskdriver.IterationPhaseSpecXXXXX
                # instances.
                'iterationCycle': [
                    #IterationPhaseSpecLearnOnly(1000),
                    IterationPhaseSpecLearnAndInfer(1000, inferenceArgs=None),
                    #IterationPhaseSpecInferOnly(10, inferenceArgs=None),
                ],
                'metrics': [
                    MetricSpec(field=u'consumption',
                               inferenceElement='prediction',
                               metric='aae',
                               params={'window': 200}),
                    MetricSpec(field=u'consumption',
                               inferenceElement='encodings',
                               metric='grokScore_scalar',
                               params={
                                   'frequencyWindow': 200,
                                   'movingAverageWindow': 200
                               }),
                ],

                # Logged Metrics: A sequence of regular expressions that specify which of
                # the metrics from the Inference Specifications section MUST be logged for
                # every prediction. The regex's correspond to the automatically generated
                # metric labels. This is similar to the way the optimization metric is
                # specified in permutations.py.
예제 #14
0
    },

    # Iteration count: maximum number of iterations.  Each iteration corresponds
    # to one record from the (possibly aggregated) dataset.  The task is
    # terminated when either number of iterations reaches iterationCount or
    # all records in the (possibly aggregated) database have been processed,
    # whichever occurs first.
    #
    # iterationCount of -1 = iterate over the entire dataset
    #'iterationCount' : ITERATION_COUNT,

    # Metrics: A list of MetricSpecs that instantiate the metrics that are
    # computed for this experiment
    'metrics': [
        MetricSpec(field=u'consumption',
                   inferenceElement=InferenceElement.prediction,
                   metric='rmse'),
    ],

    # Logged Metrics: A sequence of regular expressions that specify which of
    # the metrics from the Inference Specifications section MUST be logged for
    # every prediction. The regex's correspond to the automatically generated
    # metric labels. This is similar to the way the optimization metric is
    # specified in permutations.py.
    'loggedMetrics': ['.*nupicScore.*'],
}

################################################################################
################################################################################
descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
                                                control=control)
예제 #15
0
  # all records in the (possibly aggregated) database have been processed,
  # whichever occurs first.
  #
  # iterationCount of -1 = iterate over the entire dataset
  'iterationCount' : -1,


  # A dictionary containing all the supplementary parameters for inference
  "inferenceArgs":{u'predictedField': u'classification', u'predictionSteps': [0]},

  # Metrics: A list of MetricSpecs that instantiate the metrics that are
  # computed for this experiment
  'metrics':[
    MetricSpec(field='classification', metric='multiStep', 
               inferenceElement='multiStepBestPredictions', 
               params={'errorMetric': config['errorMetric'], 
                       'window': 100,
                       'steps': 0}),
  ],

  # Logged Metrics: A sequence of regular expressions that specify which of
  # the metrics from the Inference Specifications section MUST be logged for
  # every prediction. The regex's correspond to the automatically generated
  # metric labels. This is similar to the way the optimization metric is
  # specified in permutations.py.
  'loggedMetrics': ['.*'],
}

################################################################################
################################################################################
descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
예제 #16
0
파일: description.py 프로젝트: emmaai/nupic
            # Task Control parameters for OPFTaskDriver (per opfTaskControlSchema.json)
            'taskControl': {

                # Iteration cycle list consisting of opf_task_driver.IterationPhaseSpecXXXXX
                # instances.
                'iterationCycle': [
                    #IterationPhaseSpecLearnOnly(1000),
                    IterationPhaseSpecLearnAndInfer(1000),
                    #IterationPhaseSpecInferOnly(10),
                ],

                # Metrics: A list of MetricSpecs that instantiate the metrics that are
                # computed for this experiment
                'metrics': [
                    MetricSpec(metric='avg_err',
                               inferenceElement='classification',
                               params={'window': 200}),
                    MetricSpec(metric='neg_auc',
                               inferenceElement='classConfidences',
                               params={
                                   'window': 200,
                                   'computeEvery': 10
                               }),
                ],

                # Logged Metrics: A sequence of regular expressions that specify which of
                # the metrics from the Inference Specifications section MUST be logged for
                # every prediction. The regex's correspond to the automatically generated
                # metric labels. This is similar to the way the optimization metric is
                # specified in permutations.py.
                'loggedMetrics': ['.*avg_err.*', '.*auc.*'],
예제 #17
0
            # iterationCount of -1 = iterate over the entire dataset
            'iterationCount': -1,

            # Task Control parameters for OPFTaskDriver (per opfTaskControlSchema.json)
            'taskControl': {

                # Iteration cycle list consisting of opf_task_driver.IterationPhaseSpecXXXXX
                # instances.
                'iterationCycle': [
                    #IterationPhaseSpecLearnOnly(1000),
                    IterationPhaseSpecLearnAndInfer(1000, inferenceArgs=None),
                    #IterationPhaseSpecInferOnly(10, inferenceArgs=None),
                ],
                'metrics': [
                    MetricSpec(field='consumption',
                               inferenceElement='prediction',
                               metric='aae',
                               params={'window': 200}),
                ],

                # Logged Metrics: A sequence of regular expressions that specify which of
                # the metrics from the Inference Specifications section MUST be logged for
                # every prediction. The regex's correspond to the automatically generated
                # metric labels. This is similar to the way the optimization metric is
                # specified in permutations.py.
                'loggedMetrics': ['.*nupicScore.*', ".*aae.*"],

                # Callbacks for experimentation/research (optional)
                'callbacks': {
                    # Callbacks to be called at the beginning of a task, before model iterations.
                    # Signature: callback(<reference to OPFExperiment>); returns nothing
                    #        'setup' : [htmPredictionModelControlEnableSPLearningCb, htmPredictionModelControlEnableTPLearningCb],
예제 #18
0
from nupic.frameworks.opf.predictionmetricsmanager import MetricsManager

import nupic_anomaly_output

MODEL_NAME = "TiltTemp2009"
DATA_DIR = "./data"
MODEL_PARAMS_DIR = "./model_params"
PREDICTED_FIELD = "Tilt_06"
# '7/2/10 0:00'
DATE_FORMAT = "%m/%d/%y %H:%M"

_METRIC_SPECS = (
    MetricSpec(field=PREDICTED_FIELD,
               metric="multiStep",
               inferenceElement="multiStepBestPredictions",
               params={
                   "errorMetric": "aae",
                   "window": 1000,
                   "steps": 1
               }),
    MetricSpec(field=PREDICTED_FIELD,
               metric="trivial",
               inferenceElement="prediction",
               params={
                   "errorMetric": "aae",
                   "window": 1000,
                   "steps": 1
               }),
    MetricSpec(field=PREDICTED_FIELD,
               metric="multiStep",
               inferenceElement="multiStepBestPredictions",
               params={
예제 #19
0
파일: base.py 프로젝트: zyh329/nupic

      # Task Control parameters for OPFTaskDriver (per opfTaskControlSchema.json)
      'taskControl' : {

        # Iteration cycle list consisting of opftaskdriver.IterationPhaseSpecXXXXX
        # instances.
        'iterationCycle' : [
          #IterationPhaseSpecLearnOnly(1000),
          IterationPhaseSpecLearnAndInfer(1000, dict(predictedField="consumption")),
          #IterationPhaseSpecInferOnly(10),
        ],

        'metrics' :[
          MetricSpec(metric='rmse',
                     field="consumption",
                     inferenceElement=InferenceElement.prediction),
        ],

        # Callbacks for experimentation/research (optional)
        'callbacks' : {
          # Callbacks to be called at the beginning of a task, before model iterations.
          # Signature: callback(<reference to OPFExperiment>); returns nothing
          'setup' : [],

          # Callbacks to be called after every learning/inference iteration
          # Signature: callback(<reference to OPFExperiment>); returns nothing
          'postIter' : [],

          # Callbacks to be called when the experiment task is finished
          # Signature: callback(<reference to OPFExperiment>); returns nothing
예제 #20
0
파일: base.py 프로젝트: pulinagrawal/nupic
    'iterationCount':
    -1,

    # A dictionary containing all the supplementary parameters for inference
    "inferenceArgs": {
        u'predictedField': u'c1',
        u'predictionSteps': [1]
    },

    # Metrics: A list of MetricSpecs that instantiate the metrics that are
    # computed for this experiment
    'metrics': [
        MetricSpec(field=u'c1',
                   metric='multiStep',
                   inferenceElement='multiStepBestPredictions',
                   params={
                       'window': 1000,
                       'steps': [1],
                       'errorMetric': 'altMAPE'
                   }),
    ],

    # Logged Metrics: A sequence of regular expressions that specify which of
    # the metrics from the Inference Specifications section MUST be logged for
    # every prediction. The regex's correspond to the automatically generated
    # metric labels. This is similar to the way the optimization metric is
    # specified in permutations.py.
    'loggedMetrics': ['.*'],
}

descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
                                                control=control)
예제 #21
0
    def testCustomErrorMetric(self):
        customFunc = """def getError(pred,ground,tools):
                      return abs(pred-ground)"""

        customEM = getModule(
            MetricSpec("custom_error_metric", None, None, {
                "customFuncSource": customFunc,
                "errorWindow": 3
            }))
        gt = [9, 4, 5, 6]
        p = [0, 13, 8, 3]
        for i in xrange(len(gt)):
            aggErr = customEM.addInstance(gt[i], p[i])
        target = 5.0
        delta = 0.001

        # insure that addInstance returns the aggregate error - other
        # uber metrics depend on this behavior.
        self.assertEqual(aggErr, customEM.getMetric()["value"])
        self.assertTrue(abs(customEM.getMetric()["value"] - target) < delta)

        customFunc = """def getError(pred,ground,tools):
        sum = 0
        for i in range(min(3,tools.getBufferLen())):
          sum+=abs(tools.getPrediction(i)-tools.getGroundTruth(i))
        return sum/3"""

        customEM = getModule(
            MetricSpec("custom_error_metric", None, None,
                       {"customFuncSource": customFunc}))
        gt = [9, 4, 5, 6]
        p = [0, 13, 8, 3]
        for i in xrange(len(gt)):
            customEM.addInstance(gt[i], p[i])
        target = 5.0
        delta = 0.001
        self.assertTrue(abs(customEM.getMetric()["value"] - target) < delta)

        # Test custom error metric helper functions
        # Test getPrediction
        # Not-Windowed
        storeWindow = 4
        failed = False
        for lookBack in range(3):
            customFunc = """def getError(pred,ground,tools):
          return tools.getPrediction(%d)""" % lookBack

            customEM = getModule(
                MetricSpec("custom_error_metric", None, None,
                           {"customFuncSource": customFunc}))
            gt = [i for i in range(100)]
            p = [2 * i for i in range(100)]
            t1 = [3 * i for i in range(100)]
            t2 = [str(4 * i) for i in range(100)]

            for i in xrange(len(gt)):
                curRecord = {
                    "pred": p[i],
                    "ground": gt[i],
                    "test1": t1[i],
                    "test2": t2[i]
                }
                if i < lookBack:
                    try:
                        customEM.addInstance(gt[i], p[i], curRecord)
                        failed = True
                    except:
                        self.assertTrue(
                            not failed,
                            "An exception should have been generated, but wasn't"
                        )
                else:
                    customEM.addInstance(gt[i], p[i], curRecord)
                    self.assertTrue(
                        customEM.getMetric()["value"] == p[i - lookBack])
        #Windowed
        for lookBack in range(5):
            customFunc = """def getError(pred,ground,tools):
          return tools.getPrediction(%d)""" % lookBack

            customEM = getModule(
                MetricSpec("custom_error_metric", None, None, {
                    "customFuncSource": customFunc,
                    "storeWindow": storeWindow
                }))
            gt = [i for i in range(100)]
            p = [2 * i for i in range(100)]
            t1 = [3 * i for i in range(100)]
            t2 = [str(4 * i) for i in range(100)]

            for i in xrange(len(gt)):
                curRecord = {
                    "pred": p[i],
                    "ground": gt[i],
                    "test1": t1[i],
                    "test2": t2[i]
                }
                if lookBack >= storeWindow - 1:
                    pass
                if i < lookBack or lookBack >= storeWindow:
                    try:
                        customEM.addInstance(gt[i], p[i], curRecord)
                        failed = True
                    except:
                        self.assertTrue(
                            not failed,
                            "An exception should have been generated, but wasn't"
                        )
                else:
                    customEM.addInstance(gt[i], p[i], curRecord)
                    self.assertTrue(
                        customEM.getMetric()["value"] == p[i - lookBack])

        #Test getGroundTruth
        #Not-Windowed
        for lookBack in range(3):
            customFunc = """def getError(pred,ground,tools):
          return tools.getGroundTruth(%d)""" % lookBack

            customEM = getModule(
                MetricSpec("custom_error_metric", None, None,
                           {"customFuncSource": customFunc}))
            gt = [i for i in range(100)]
            p = [2 * i for i in range(100)]
            t1 = [3 * i for i in range(100)]
            t2 = [str(4 * i) for i in range(100)]

            for i in xrange(len(gt)):
                curRecord = {
                    "pred": p[i],
                    "ground": gt[i],
                    "test1": t1[i],
                    "test2": t2[i]
                }
                if i < lookBack:
                    try:
                        customEM.addInstance(gt[i], p[i], curRecord)
                        failed = True
                    except:
                        self.assertTrue(
                            not failed,
                            "An exception should have been generated, but wasn't"
                        )
                else:
                    customEM.addInstance(gt[i], p[i], curRecord)
                    self.assertTrue(
                        customEM.getMetric()["value"] == gt[i - lookBack])
        #Windowed
        for lookBack in range(5):
            customFunc = """def getError(pred,ground,tools):
          return tools.getGroundTruth(%d)""" % lookBack

            customEM = getModule(
                MetricSpec("custom_error_metric", None, None, {
                    "customFuncSource": customFunc,
                    "storeWindow": storeWindow
                }))
            gt = [i for i in range(100)]
            p = [2 * i for i in range(100)]
            t1 = [3 * i for i in range(100)]
            t2 = [str(4 * i) for i in range(100)]

            for i in xrange(len(gt)):
                curRecord = {
                    "pred": p[i],
                    "ground": gt[i],
                    "test1": t1[i],
                    "test2": t2[i]
                }
                if i < lookBack or lookBack >= storeWindow:
                    try:
                        customEM.addInstance(gt[i], p[i], curRecord)
                        failed = True
                    except:
                        self.assertTrue(
                            not failed,
                            "An exception should have been generated, but wasn't"
                        )
                else:
                    customEM.addInstance(gt[i], p[i], curRecord)
                    self.assertTrue(
                        customEM.getMetric()["value"] == gt[i - lookBack])

        #Test getFieldValue
        #Not-Windowed Scalar
        for lookBack in range(3):
            customFunc = """def getError(pred,ground,tools):
          return tools.getFieldValue(%d,"test1")""" % lookBack

            customEM = getModule(
                MetricSpec("custom_error_metric", None, None,
                           {"customFuncSource": customFunc}))
            gt = [i for i in range(100)]
            p = [2 * i for i in range(100)]
            t1 = [3 * i for i in range(100)]
            t2 = [str(4 * i) for i in range(100)]

            for i in xrange(len(gt)):
                curRecord = {
                    "pred": p[i],
                    "ground": gt[i],
                    "test1": t1[i],
                    "test2": t2[i]
                }
                if i < lookBack:
                    try:
                        customEM.addInstance(gt[i], p[i], curRecord)
                        failed = True
                    except:
                        self.assertTrue(
                            not failed,
                            "An exception should have been generated, but wasn't"
                        )
                else:
                    customEM.addInstance(gt[i], p[i], curRecord)
                    self.assertTrue(
                        customEM.getMetric()["value"] == t1[i - lookBack])
        #Windowed Scalar
        for lookBack in range(3):
            customFunc = """def getError(pred,ground,tools):
          return tools.getFieldValue(%d,"test1")""" % lookBack

            customEM = getModule(
                MetricSpec("custom_error_metric", None, None, {
                    "customFuncSource": customFunc,
                    "storeWindow": storeWindow
                }))
            gt = [i for i in range(100)]
            p = [2 * i for i in range(100)]
            t1 = [3 * i for i in range(100)]
            t2 = [str(4 * i) for i in range(100)]

            for i in xrange(len(gt)):
                curRecord = {
                    "pred": p[i],
                    "ground": gt[i],
                    "test1": t1[i],
                    "test2": t2[i]
                }
                if i < lookBack or lookBack >= storeWindow:
                    try:
                        customEM.addInstance(gt[i], p[i], curRecord)
                        failed = True
                    except:
                        self.assertTrue(
                            not failed,
                            "An exception should have been generated, but wasn't"
                        )
                else:
                    customEM.addInstance(gt[i], p[i], curRecord)
                    self.assertTrue(
                        customEM.getMetric()["value"] == t1[i - lookBack])
        #Not-Windowed category
        for lookBack in range(3):
            customFunc = """def getError(pred,ground,tools):
          return tools.getFieldValue(%d,"test1")""" % lookBack

            customEM = getModule(
                MetricSpec("custom_error_metric", None, None,
                           {"customFuncSource": customFunc}))
            gt = [i for i in range(100)]
            p = [2 * i for i in range(100)]
            t1 = [3 * i for i in range(100)]
            t2 = [str(4 * i) for i in range(100)]

            for i in xrange(len(gt)):
                curRecord = {
                    "pred": p[i],
                    "ground": gt[i],
                    "test1": t1[i],
                    "test2": t2[i]
                }
                if i < lookBack:
                    try:
                        customEM.addInstance(gt[i], p[i], curRecord)
                        failed = True
                    except:
                        self.assertTrue(
                            not failed,
                            "An exception should have been generated, but wasn't"
                        )
                else:
                    customEM.addInstance(gt[i], p[i], curRecord)
                    self.assertTrue(
                        customEM.getMetric()["value"] == t1[i - lookBack])

        #Windowed category
        for lookBack in range(3):
            customFunc = """def getError(pred,ground,tools):
          return tools.getFieldValue(%d,"test1")""" % lookBack

            customEM = getModule(
                MetricSpec("custom_error_metric", None, None, {
                    "customFuncSource": customFunc,
                    "storeWindow": storeWindow
                }))
            gt = [i for i in range(100)]
            p = [2 * i for i in range(100)]
            t1 = [3 * i for i in range(100)]
            t2 = [str(4 * i) for i in range(100)]

            for i in xrange(len(gt)):
                curRecord = {
                    "pred": p[i],
                    "ground": gt[i],
                    "test1": t1[i],
                    "test2": t2[i]
                }
                if i < lookBack or lookBack >= storeWindow:
                    try:
                        customEM.addInstance(gt[i], p[i], curRecord)
                        failed = True
                    except:
                        self.assertTrue(
                            not failed,
                            "An exception should have been generated, but wasn't"
                        )
                else:
                    customEM.addInstance(gt[i], p[i], curRecord)
                    self.assertTrue(
                        customEM.getMetric()["value"] == t1[i - lookBack])
        #Test getBufferLen
        #Not-Windowed
        customFunc = """def getError(pred,ground,tools):
        return tools.getBufferLen()"""

        customEM = getModule(
            MetricSpec("custom_error_metric", None, None,
                       {"customFuncSource": customFunc}))
        gt = [i for i in range(100)]
        p = [2 * i for i in range(100)]
        t1 = [3 * i for i in range(100)]
        t2 = [str(4 * i) for i in range(100)]

        for i in xrange(len(gt)):
            curRecord = {
                "pred": p[i],
                "ground": gt[i],
                "test1": t1[i],
                "test2": t2[i]
            }
            customEM.addInstance(gt[i], p[i], curRecord)
            self.assertTrue(customEM.getMetric()["value"] == i + 1)
        #Windowed
        customFunc = """def getError(pred,ground,tools):
        return tools.getBufferLen()"""

        customEM = getModule(
            MetricSpec("custom_error_metric", None, None, {
                "customFuncSource": customFunc,
                "storeWindow": storeWindow
            }))
        gt = [i for i in range(100)]
        p = [2 * i for i in range(100)]
        t1 = [3 * i for i in range(100)]
        t2 = [str(4 * i) for i in range(100)]

        for i in xrange(len(gt)):
            curRecord = {
                "pred": p[i],
                "ground": gt[i],
                "test1": t1[i],
                "test2": t2[i]
            }
            customEM.addInstance(gt[i], p[i], curRecord)
            self.assertTrue(customEM.getMetric()["value"] == min(i + 1, 4))

        #Test initialization edge cases
        try:
            customEM = getModule(
                MetricSpec("custom_error_metric", None, None, {
                    "customFuncSource": customFunc,
                    "errorWindow": 0
                }))
            self.assertTrue(False,
                            "error Window of 0 should fail self.assertTrue")
        except:
            pass

        try:
            customEM = getModule(
                MetricSpec("custom_error_metric", None, None, {
                    "customFuncSource": customFunc,
                    "storeWindow": 0
                }))
            self.assertTrue(False,
                            "error Window of 0 should fail self.assertTrue")
        except:
            pass
예제 #22
0
    #
    # iterationCount of -1 = iterate over the entire dataset
    'iterationCount':
    -1,

    # A dictionary containing all the supplementary parameters for inference
    "inferenceArgs": {
        u'predictedField': u'f',
        u'predictionSteps': [1]
    },

    # Metrics: A list of MetricSpecs that instantiate the metrics that are
    # computed for this experiment
    'metrics': [
        MetricSpec(field=u'f',
                   metric='passThruPrediction',
                   inferenceElement='anomalyScore',
                   params={'window': 1000}),
    ],

    # Logged Metrics: A sequence of regular expressions that specify which of
    # the metrics from the Inference Specifications section MUST be logged for
    # every prediction. The regex's correspond to the automatically generated
    # metric labels. This is similar to the way the optimization metric is
    # specified in permutations.py.
    'loggedMetrics': ['.*'],
}

descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
                                                control=control)
예제 #23
0
    def testNegativeLogLikelihood(self):
        # make sure negativeLogLikelihood returns correct LL numbers

        # mock objects for ClassifierInput and ModelResult (see opf_utils.py)
        class MockClassifierInput(object):
            def __init__(self, bucketIdx):
                self.bucketIndex = bucketIdx

        class MockModelResult(object):
            def __init__(self, bucketll, bucketIdx):
                self.inferences = {'multiStepBucketLikelihoods': {1: bucketll}}
                self.classifierInput = MockClassifierInput(bucketIdx)

        bucketLL = {
            0: 1.0,
            1: 0,
            2: 0,
            3: 0
        }  # model prediction as a dictionary
        gt_bucketIdx = 0  # bucket index for ground truth
        negLL = getModule(
            MetricSpec("negativeLogLikelihood", None, None,
                       {"verbosity": OPFMetricsTest.VERBOSITY}))
        negLL.addInstance(0,
                          0,
                          record=None,
                          result=MockModelResult(bucketLL, gt_bucketIdx))
        target = 0.0  # -log(1.0)
        self.assertAlmostEqual(negLL.getMetric()["value"], target)

        bucketLL = {
            0: 0.5,
            1: 0.5,
            2: 0,
            3: 0
        }  # model prediction as a dictionary
        gt_bucketIdx = 0  # bucket index for ground truth
        negLL = getModule(
            MetricSpec("negativeLogLikelihood", None, None,
                       {"verbosity": OPFMetricsTest.VERBOSITY}))
        negLL.addInstance(0,
                          0,
                          record=None,
                          result=MockModelResult(bucketLL, gt_bucketIdx))
        target = 0.6931471  # -log(0.5)
        self.assertTrue(
            abs(negLL.getMetric()["value"] - target) < OPFMetricsTest.DELTA)

        # test accumulated negLL for multiple steps
        bucketLL = []
        bucketLL.append({0: 1, 1: 0, 2: 0, 3: 0})
        bucketLL.append({0: 0, 1: 1, 2: 0, 3: 0})
        bucketLL.append({0: 0, 1: 0, 2: 1, 3: 0})
        bucketLL.append({0: 0, 1: 0, 2: 0, 3: 1})

        gt_bucketIdx = [0, 2, 1, 3]
        negLL = getModule(
            MetricSpec("negativeLogLikelihood", None, None,
                       {"verbosity": OPFMetricsTest.VERBOSITY}))
        for i in range(len(bucketLL)):
            negLL.addInstance(0,
                              0,
                              record=None,
                              result=MockModelResult(bucketLL[i],
                                                     gt_bucketIdx[i]))
        target = 5.756462
        self.assertTrue(
            abs(negLL.getMetric()["value"] - target) < OPFMetricsTest.DELTA)