Exemple #1
0
    def initModel(self):
        if os.path.exists(os.path.abspath(self.model_path)):
            self.model = ModelFactory.loadFromCheckpoint(os.path.relpath(self.model_path))
        else:
            self.model = ModelFactory.create(self.model_params)

        predicted_field = self.model_params["predictedField"]
        if predicted_field:
            self.model.enableInference({"predictedField": predicted_field})
Exemple #2
0
    def initModel(self):
        if os.path.exists(os.path.abspath(self.model_path)):
            self.model = ModelFactory.loadFromCheckpoint(
                os.path.relpath(self.model_path))
        else:
            self.model = ModelFactory.create(self.model_params)

        predicted_field = self.model_params['predictedField']
        if predicted_field:
            self.model.enableInference({'predictedField': predicted_field})
    def __init__(self,name,admin_in,admin_out,sensor_spec, sensors_dir,sensor_in,store,swarm):
        threading.Thread.__init__(self)
        #self.config = config
        self.sensor_in = sensor_in
        self.store = store
        self.swarm = swarm
        self.name = name
        self.brain_available = False
        threading.Thread.__init__(self)
        Sensor. __init__(self,name=name,admin_in=admin_in, admin_out=admin_out,sensor_spec=sensor_spec, sensors_dir=sensors_dir)
        swarm_config_path = sensors_dir  + sensor_in +'/stores/' + store + '/swarms/' + swarm +'/'
        #store_path = sensors_dir  + sensor_in +'/stores/' + store + '/out.csv'
        #model = ModelFactory.loadFromCheckpoint('/home/hans/cortical_one_var/sensors/cpu/stores/store_3/swarms/swarm_1/model_save')

        print swarm_config_path

        #load original swarm config file
        with open(swarm_config_path + 'swarm_config.json')as json_file:
            self.swarm_config = json.load(json_file)
            print(self.swarm_config)

        self.swarm_config_ng = SwarmConfig(self.swarm_config)

        print self.swarm_config_ng.get_predicted_field()


        #if there is a 'brain', then tae the existing brain
        self.possible_brain_path = str(swarm_config_path +  'model_save')
        if os.path.exists(self.possible_brain_path):
            possible_brain_2 = '/home/hans/cortical_one_var/sensors/cpu/stores/store_3/swarms/swarm_1/model_save'
            print "load existing brain..."
            print self.possible_brain_path
            #model = ModelFactory.loadFromCheckpoint(possible_brain_2)
            model = ModelFactory.loadFromCheckpoint(self.possible_brain_path)
            #use this case to add the availabilty of a 'brain' (???!!!) to your annuncement

        else:

            #laod model configuration
            model = ModelFactory.create(getModelParamsFromFileNG(swarm_config_path))

            #configure prediction
            model.enableInference({"predictedField": self.swarm_config_ng.get_predicted_field()})

        self.connection_sensor_in = stomp.Connection()
        self.connection_sensor_in.set_listener(name=self.name, lstnr=AbstractSensorListener(self.name,topic = '/topic/' +self.sensor_in,config=self.swarm_config_ng,model=model))
        self.connection_sensor_in.start()
        self.connection_sensor_in.connect(self.user, self.password, wait=True)
        #self.connection_sensor_in.connect('admin', 'password', wait=True)
        self.abstract_listener = self.connection_sensor_in.get_listener(name=self.name)
        self.connection_sensor_in.subscribe(destination='/topic/' +self.sensor_in, id=2, ack='auto')

        self.values = []

        self.self_announcement()
def getModel(flag):

    modelDir = os.getcwd() + '/model/%s' % flag

    _LOGGER.info(modelDir)

    if os.path.exists(modelDir):
        _LOGGER.info('model exists')
        model = ModelFactory.loadFromCheckpoint(modelDir)
        return model
    else:
        _LOGGER.info('creating new model')
        model = ModelFactory.create(MODEL_PARAMS)
        model.save(modelDir)
        return model
def getModel(flag):

    modelDir = os.getcwd() + "/model/%s" % flag

    _LOGGER.info(modelDir)

    if os.path.exists(modelDir):
        _LOGGER.info("model exists")
        model = ModelFactory.loadFromCheckpoint(modelDir)
        return model
    else:
        _LOGGER.info("creating new model")
        model = ModelFactory.create(MODEL_PARAMS)
        model.save(modelDir)
        return model
    def __init__(self, predictStep, enablePredict, maxValue, minValue,
                 minResolution):
        # initial the parameters and data variables.
        self.predictStep = predictStep
        self.enablePredict = enablePredict
        self.metricData = xrange(int(minValue), int(maxValue),
                                 int((maxValue - minValue) / minResolution))
        self.maxValue = maxValue
        self.minValue = minValue
        self.minResolution = minResolution
        self.timestamp = None
        self.actualValue = None
        self.predictValue = None
        self.anomalyScore = None
        self.modelResult = None
        self.output = None

        # get the model parameters.
        self.parameters = getScalarMetricWithTimeOfDayAnomalyParams(
            self.metricData, self.minValue, self.maxValue, self.minResolution)
        # make sure the result contains the predictions.
        self.parameters["modelConfig"]["modelParams"][
            "clEnable"] = self.enablePredict
        # so we can modify the predict step by do that:
        self.parameters["modelConfig"]["modelParams"]["clParams"][
            "steps"] = self.predictStep
        # create the model
        self.model = ModelFactory.create(self.parameters["modelConfig"])
        self.model.enableInference(self.parameters["inferenceArgs"])
Exemple #7
0
  def configureDetector(self, probationaryPeriodData):
        # Load the model params JSON
    probationaryPeriod = probationaryPeriodData.shape[0]

    paramsPath = os.path.join(os.path.split(__file__)[0],
                "modelParams",
                "model_params.json")
    with open(paramsPath) as fp:
      modelParams = json.load(fp)

    self.sensorParams = modelParams["modelParams"]["sensorParams"]\
                                   ["encoders"]["value"]

    # RDSE - resolution calculation
    resolution = max(0.001,
                     (self.inputMax - self.inputMin) / \
                     self.sensorParams.pop("numBuckets")
                    )
    self.sensorParams["resolution"] = resolution

    self.model = ModelFactory.create(modelParams)

    self.model.enableInference({"predictedField": "value"})

    # The anomaly likelihood object
    numentaLearningPeriod = math.floor(probationaryPeriod / 2.0)
    self.anomalyLikelihood = AnomalyLikelihood(probationaryPeriod,
                                               numentaLearningPeriod)
 def __init__(self):
     """MODEL_PARAMS = {
         # Your choice
     }"""
     self.model = ModelFactory.create(MODEL_PARAMS)
     self.lastInput = -1
     super(HelloModel, self).__init__(self.model)
def run_experiment():
	import model_params
	model = ModelFactory.create(MODEL_PARAMS)
	model.enableInference({ "predictedField": "event" })

	with open(IN_PATH, "rb") as event_input, open(OUT_PATH, "w") as event_output:
		csv_writer = csv.writer(event_output)
		csv_writer.writerow(['timestamp', 'thread_id', 'event', 'anomalyScore',
			'prediction1', 'likelihood1', 'prediction2', 'likelihood2'])

		csv_reader = csv.reader(event_input)
		# skip headers
		csv_reader.next()
		csv_reader.next()
		csv_reader.next()

		# the real data
		for row in csv_reader:
			timestamp = dateutil.parser.parse(row[0])
			thread_id = row[1]
			event = row[2]
			result = model.run({"timestamp": timestamp, "thread_id": thread_id, "event": event, })

			# Print to files / command line
			output_row = [timestamp, thread_id, event, result.inferences['anomalyScore']]
			predictions = result.inferences['multiStepPredictions'][1]
			for prediction, likelihood in sorted(predictions.items(), key=itemgetter(1), reverse=True):
				output_row.append(prediction)
				output_row.append(likelihood)

			csv_writer.writerow(output_row)

			if csv_reader.line_num % 100 == 0:
				print 'Wrote %s lines of output...' % csv_reader.line_num
Exemple #10
0
def run():
    if SMOOTH:
        inputDataDir = "../preprocessing/smoothed_data"
    else:
        inputDataDir = "../preprocessing/formatted_data"

    model = ModelFactory.create(MODEL_PARAMS)
    model.enableInference({"predictedField": "channel_value"})

    for channel in CHANNELS:
        for movement in MOTOR:

            inputFile = "%s/%s_%s.csv" % (inputDataDir, movement, channel)
            with open(inputFile, "rb") as input:
                csvReader = csv.reader(input)

                # skip header rows
                csvReader.next()
                csvReader.next()
                csvReader.next()

                # train model
                for row in csvReader:
                    channel_value = float(row[0])
                    model.run({"channel_value": channel_value})

                # save model
                modelDir = "%s/%s_%s_%s" % (CWD, MODEL_PREFIX, movement,
                                            channel)
                model.save(modelDir)
                print "Saved new model in %s" % modelDir
Exemple #11
0
    def predict(self, model):
        # model input data is 
        cla_model_input=dict()
        cla_model_input["rssi"]    = float(model.rssi_raw)
        cla_model_input["quality"] = float(model.quality_raw)

        if not self.cla_model:
            # Create the model for predicting WiFi usage
            # The shifter will align prediction and actual values.
            s1 = int(round(time.time() * 1000))
            self.cla_model = ModelFactory.create(model_params.MODEL_PARAMS)
            self.cla_model.enableInference({'predictedField': 'rssi'})
            self.shifter = InferenceShifter()        
            s2 = int(round(time.time() * 1000))
            #print "Predictor construction %s,%d,%d,%d" % (self, s1, s2, abs(s2-s1))

        # Shift 'x' time periods into the future
        # Compare 'current' with 'predicted - pos(x)' - how do we compare
        # The point is to be able to Generalize on patterns that it hasn't seen yet
        result = self.shifter.shift(self.cla_model.run(cla_model_input))
        inference = result.inferences['multiStepBestPredictions'][self.timestep]

        # Store away current rssi information on the model
        if inference is not None:
            model.rssi_predicted =  int(inference)
            model.rssi_delta = abs(model.rssi_raw - model.rssi_predicted)
            model.zone_predicted.assign_rssi_zone(model.rssi_predicted) 
Exemple #12
0
def run(fname="verylargeseism_out"):
    print("reading %s" % input_file)
    print("using this model: ", model_params)
    model = ModelFactory.create(model_params)
    model.enableInference({"predictedField": "acc"})
    print("created nupic model in :%f min" % ((clk.time() - t0) / 60.))
    print("will output predictions and anomalies \n in this file: %s" % fname)

    output = NuPICFileOutput(fname, show_anomaly_score=True)
    with open(input_file, "rb") as data_input:
        csv_reader = csv.reader(data_input)

        # skip header rows
        csv_reader.next()
        csv_reader.next()
        csv_reader.next()

        # the real data
        for row in csv_reader:
            time = float(row[0])
            acc_value = float(row[1])
            result = model.run({"acc": acc_value})
            output.write(time, acc_value, result, prediction_step=PSTEPS)

    output.close()
    print("time is :%f min" % ((clk.time() - t0) / 60.))
Exemple #13
0
    def __init__(self, numPredictions, resultsDir):
        random.seed(43)
        self.numPredictions = numPredictions

        if not os.path.exists(resultsDir):
            os.makedirs(resultsDir)

        self.resultsFile = open(os.path.join(resultsDir, "0.log"), 'w')

        self.model = ModelFactory.create(MODEL_PARAMS)
        self.model.enableInference({"predictedField": "element"})
        self.shifter = InferenceShifter()
        self.mapping = getEncoderMapping(self.model)

        self.correct = []
        self.numPredictedActiveCells = []
        self.numPredictedInactiveCells = []
        self.numUnpredictedActiveColumns = []

        self.iteration = 0
        self.perturbed = False
        self.randoms = []
        self.verbosity = 1

        self.dataset = HighOrderDataset(numPredictions=self.numPredictions)
        self.sequences = []
        self.currentSequence = []
        self.replenish_sequence()
Exemple #14
0
 def __init__(self):
     """MODEL_PARAMS = {
         # Your choice
     }"""
     self.model = ModelFactory.create(MODEL_PARAMS)
     self.lastInput = -1
     super(HelloModel, self).__init__(self.model)
Exemple #15
0
def do_teach():

    model = ModelFactory.create(getModelParamsFromName())
    model.enableInference({"predictedField": "s1"})

    target = open("teach.csv", 'w')
    spamwriter = csv.writer(target, delimiter=',',
                                quotechar='|', quoting=csv.QUOTE_MINIMAL)
    spamwriter.writerow(['s1', 's2', 's3', 's4', 'predicts1'])

    with open('out.csv', 'rb') as csvfile:

        spamreader = csv.reader(csvfile, delimiter=',', quotechar='"')
        for row in spamreader:
            print ', '.join(row)

            result = model.run({
                "s1": float(row[0]),
                "s2": float(row[1]),
                "s3": float(row[2]),
                "s4": float(row[3])
            })
            print result.rawInput
            print result.inferences['multiStepBestPredictions']

            spamwriter.writerow([float(row[0]), float(row[1]), float(row[2]), float(row[3]), result.inferences['multiStepBestPredictions'][4]])


    model.save(os.getcwd() + "/model_save")
Exemple #16
0
    def _createModel(cls, stats, replaceParams):
        """Instantiate and configure an OPF model

    :param dict stats: Metric data stats per stats_schema.json in the
      unicorn_backend package.
    :param sequence replaceParams: Parameter replacement PATH REPLACEMENT pairs
    :returns: OPF Model instance
    """
        # Generate swarm params
        swarmParams = getScalarMetricWithTimeOfDayAnomalyParams(
            metricData=[0],
            minVal=stats["min"],
            maxVal=stats["max"],
            minResolution=stats.get("minResolution"))

        for path, replacement in replaceParams:
            _recurseDictAndReplace(swarmParams,
                                   path.split(_REPLACE_PATH_SEPARATOR),
                                   replacement)

        model = ModelFactory.create(modelConfig=swarmParams["modelConfig"])
        model.enableLearning()
        model.enableInference(swarmParams["inferenceArgs"])

        return model
    def test_hotgym_anomalyScore_stays_below_50_perc_after_110_rows(self):
        """
    Tests that the hotgym anomalyScore values stays below 50% after feeding in
    110 rows of data.
    """
        model = ModelFactory.create(
            rec_center_hourly_model_params.MODEL_PARAMS)
        model.enableInference({"predictedField": "kw_energy_consumption"})
        inputFile = open(CSV_DATA, "rb")
        csvReader = csv.reader(inputFile)
        # skip header rows
        csvReader.next()
        csvReader.next()
        csvReader.next()

        rowCount = 0

        for row in csvReader:
            rowCount += 1
            timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
            consumption = float(row[1])
            result = model.run({
                "timestamp": timestamp,
                "kw_energy_consumption": consumption
            })
            anomalyScore = result.inferences["anomalyScore"]
            print "row %i: %f" % (rowCount, anomalyScore)
            if rowCount >= START_AT_ROW:
                unittest.TestCase.assertGreater(
                    self, ANOMALY_THRESHOLD, anomalyScore,
                    "Anomaly score exceeded threshold of %f after %i rows of data."
                    % (ANOMALY_THRESHOLD, rowCount))
                break

        inputFile.close()
Exemple #18
0
def run_experiment():
    generate_data.run()
    swarm_over_data()
    import model_params
    model = ModelFactory.create(model_params.MODEL_PARAMS)
    model.enableInference({"predictedField": "sine"})
    output = NuPICPlotOutput("sine_out", show_anomaly_score=True)

    with open("sine.csv", "rb") as sine_input:
        csv_reader = csv.reader(sine_input)

        # Skip headers
        csv_reader.next()
        csv_reader.next()
        csv_reader.next()

        # Real Data
        for row in csv_reader:
            angle = float(row[0])
            sine_value = float(row[1])

            result = model.run({"sine": sine_value})
            output.write(angle, sine_value, result)

    output.close()
Exemple #19
0
    def initialize(self):
        calcRange = abs(self.inputMax - self.inputMin)
        calcPad = calcRange * .2

        self.inputMin = self.inputMin - calcPad
        self.inputMax = self.inputMax + calcPad
        # Load the model params JSON

        paramsPath = os.path.join(
            os.path.split(__file__)[0], "modelParams", "model_params.json")
        with open(paramsPath) as fp:
            modelParams = json.load(fp)

        self.sensorParams = modelParams["modelParams"]["sensorParams"]\
                                       ["encoders"]["value"]

        # RDSE - resolution calculation
        resolution = max(0.001,
                         (self.inputMax - self.inputMin) / \
                         self.sensorParams.pop("numBuckets")
                        )
        self.sensorParams["resolution"] = resolution

        self.model = ModelFactory.create(modelParams)

        self.model.enableInference({"predictedField": "value"})

        # Initialize the anomaly likelihood object
        numentaLearningPeriod = math.floor(self.probationaryPeriod / 2.0)
        self.anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood(
            claLearningPeriod=numentaLearningPeriod,
            estimationSamples=self.probationaryPeriod - numentaLearningPeriod,
            reestimationPeriod=100)
Exemple #20
0
 def __init__(self):
     """Constr."""
     self.model = ModelFactory.create(ModelParams.MODEL_PARAMS)
     self.model.enableInference({'predictedField': 'cpuUtil'})
     self.headers = [
         "timestamp", "cpuUtil", "ramUtil", "diskUtil", "numUsers"
     ]
Exemple #21
0
  def __init__(self, numPredictions, resultsDir):
    random.seed(43)
    self.numPredictions = numPredictions

    if not os.path.exists(resultsDir):
      os.makedirs(resultsDir)

    self.resultsFile = open(os.path.join(resultsDir, "0.log"), 'w')

    self.model = ModelFactory.create(MODEL_PARAMS)
    self.model.enableInference({"predictedField": "element"})
    self.shifter = InferenceShifter()
    self.mapping = getEncoderMapping(self.model)

    self.correct = []
    self.numPredictedActiveCells = []
    self.numPredictedInactiveCells = []
    self.numUnpredictedActiveColumns = []

    self.iteration = 0
    self.perturbed = False
    self.randoms = []
    self.verbosity = 1

    self.dataset = HighOrderDataset(numPredictions=self.numPredictions)
    self.sequences = []
    self.currentSequence = []
    self.replenish_sequence()
Exemple #22
0
def run():
    if SMOOTH:
        inputDataDir = "../preprocessing/smoothed_data"
    else:
        inputDataDir = "../preprocessing/formatted_data"

    model = ModelFactory.create(MODEL_PARAMS)
    model.enableInference({"predictedField": "channel_value"})

    for channel in CHANNELS:
        for movement in MOTOR:

            inputFile = "%s/%s_%s.csv" % (inputDataDir, movement, channel)
            with open(inputFile, "rb") as input:
                csvReader = csv.reader(input)

                # skip header rows
                csvReader.next()
                csvReader.next()
                csvReader.next()

                # train model
                for row in csvReader:
                  channel_value = float(row[0])
                  model.run({"channel_value": channel_value})

                # save model
                modelDir = "%s/%s_%s_%s" % (CWD, MODEL_PREFIX, movement, channel)
                model.save(modelDir)
                print "Saved new model in %s" % modelDir
    def __init__(self,
                 modelConfig,
                 inferenceArgs,
                 metricSpecs,
                 sourceSpec,
                 sinkSpec=None):
        """Initialization.

    Args:
      modelConfig: The model config dict.
      metricSpecs: A sequence of MetricSpec instances.
      sourceSpec: Path to the source CSV file.
      sinkSpec: Path to the sink CSV file.
    """
        self.model = ModelFactory.create(modelConfig)
        self.model.enableInference(inferenceArgs)
        self.metricsManager = MetricsManager(metricSpecs,
                                             self.model.getFieldInfo(),
                                             self.model.getInferenceType())
        self.sink = None
        if sinkSpec is not None:
            # TODO: make this work - sinkSpec not yet supported.
            raise NotImplementedError('The sinkSpec is not yet implemented.')
            #self.sink = BasicPredictionLogger(
            #    self.model.getFieldInfo(), sinkSpec, 'myOutput',
            #    self.model.getInferenceType())
            #self.sink.setLoggedMetrics(
            #    self.metricsManager.getMetricLabels())
        self.datasetReader = BasicDatasetReader(sourceSpec)
Exemple #24
0
def run_seism_experiment():
    input_file = "eQnoise.csv"
    generate_data.run(input_file)
    print("time is :%f secs" % ((clk.time() - t0) / 60.))
    model_params = swarm_over_data()
    print("time is :%f secs" % ((clk.time() - t0) / 60.))
    print(model_params)
    if PLOT:
        pass
        #output = NuPICPlotOutput("sine3_output", show_anomaly_score=True)
    else:
        output = NuPICFileOutput("eQnoise_output", show_anomaly_score=True)
    print("time is :%f min" % ((clk.time() - t0) / 60.))
    model = ModelFactory.create(model_params)
    model.enableInference({"predictedField": "acc"})

    with open(input_file, "rb") as data_input:
        csv_reader = csv.reader(data_input)

        # skip header rows
        csv_reader.next()
        csv_reader.next()
        csv_reader.next()

        # the real data
        for row in csv_reader:
            time = float(row[0])
            acc_value = float(row[1])
            result = model.run({"acc": acc_value})
            output.write(time, acc_value, result, prediction_step=PSTEPS)

    output.close()
    print("time is :%f min" % ((clk.time() - t0) / 60.))
Exemple #25
0
def run_experiment():
    # generate_data.run()
    # swarm_over_data()

    from model_params import model_params
    model = ModelFactory.create(model_params.MODEL_PARAMS)
    model.enableInference({"predictedField": "sine"})

    output = nupic_output.NuPICPlotOutput("sine")
    #output = nupic_output.NuPICFileOutput(["sine"])
    #shifter = InferenceShifter()

    with open("sine.csv", "rb") as sine_input:
        csv_reader = csv.reader(sine_input)
        csv_reader.next()
        csv_reader.next()
        csv_reader.next()

        for row in csv_reader:
            angle = float(row[0])
            sine_value = float(row[1])

            result = model.run({"sine": sine_value})
            print result
            #result = shifter.shift(result)

            prediction = result.inferences["multiStepBestPredictions"][1]
            anomalyScore = result.inferences["anomalyScore"]
            output.write(angle, sine_value, prediction, anomalyScore)

    output.close()
Exemple #26
0
def run_sine_experiment():
    input_file = "sine.csv"
    generate_data.run(input_file)
    model_params = swarm_over_data()
    if PLOT:
        output = NuPICPlotOutput("sine_output", show_anomaly_score=True)
    else:
        output = NuPICFileOutput("sine_output", show_anomaly_score=True)
    model = ModelFactory.create(model_params)
    model.enableInference({"predictedField": "sine"})

    with open(input_file, "rb") as sine_input:
        csv_reader = csv.reader(sine_input)

        # skip header rows
        csv_reader.next()
        csv_reader.next()
        csv_reader.next()

        # the real data
        for row in csv_reader:
            angle = float(row[0])
            sine_value = float(row[1])
            result = model.run({"sine": sine_value})
            output.write(angle, sine_value, result, prediction_step=1)

    output.close()
Exemple #27
0
  def initialize(self):
    calcRange = abs(self.inputMax - self.inputMin)
    calcPad = calcRange * .2

    self.inputMin = self.inputMin - calcPad
    self.inputMax = self.inputMax + calcPad
    # Load the model params JSON

    paramsPath = os.path.join(os.path.split(__file__)[0],
                "modelParams",
                "model_params.json")
    with open(paramsPath) as fp:
      modelParams = json.load(fp)

    self.sensorParams = modelParams["modelParams"]["sensorParams"]\
                                   ["encoders"]["value"]

    # RDSE - resolution calculation
    resolution = max(0.001,
                     (self.inputMax - self.inputMin) / \
                     self.sensorParams.pop("numBuckets")
                    )
    self.sensorParams["resolution"] = resolution

    self.model = ModelFactory.create(modelParams)

    self.model.enableInference({"predictedField": "value"})

    # The anomaly likelihood object
    numentaLearningPeriod = math.floor(self.probationaryPeriod / 2.0)
    self.anomalyLikelihood = AnomalyLikelihood(self.probationaryPeriod,
                                               numentaLearningPeriod)
Exemple #28
0
    def testModelParams(self):
        """
    Test that clusterParams loads returns a valid dict that can be instantiated
    as a CLAModel.
    """
        params = getScalarMetricWithTimeOfDayAnomalyParams([0],
                                                           minVal=23.42,
                                                           maxVal=23.420001)

        encodersDict = (
            params['modelConfig']['modelParams']['sensorParams']['encoders'])

        model = ModelFactory.create(modelConfig=params['modelConfig'])
        self.assertIsInstance(
            model, CLAModel, "JSON returned cannot be used to create a model")

        # Ensure we have a time of day field
        self.assertIsNotNone(encodersDict['c0_timeOfDay'])

        # Ensure resolution doesn't get too low
        if encodersDict['c1']['type'] == 'RandomDistributedScalarEncoder':
            self.assertGreaterEqual(encodersDict['c1']['resolution'], 0.001,
                                    "Resolution is too low")

        # Ensure tm_cpp returns correct json file
        params = getScalarMetricWithTimeOfDayAnomalyParams(
            [0], tmImplementation="tm_cpp")
        self.assertEqual(
            params['modelConfig']['modelParams']['tpParams']['temporalImp'],
            "tm_cpp", "Incorrect json for tm_cpp tmImplementation")

        # Ensure incorrect tmImplementation throws exception
        with self.assertRaises(ValueError):
            getScalarMetricWithTimeOfDayAnomalyParams([0], tmImplementation="")
Exemple #29
0
  def testModelParams(self):
    """
    Test that clusterParams loads returns a valid dict that can be instantiated
    as a CLAModel.
    """
    params = getScalarMetricWithTimeOfDayAnomalyParams([0],
                                                       minVal=23.42,
                                                       maxVal=23.420001)

    encodersDict= (
      params['modelConfig']['modelParams']['sensorParams']['encoders'])

    model = ModelFactory.create(modelConfig=params['modelConfig'])
    self.assertIsInstance(model,
                          CLAModel,
                          "JSON returned cannot be used to create a model")

    # Ensure we have a time of day field
    self.assertIsNotNone(encodersDict['c0_timeOfDay'])

    # Ensure resolution doesn't get too low
    if encodersDict['c1']['type'] == 'RandomDistributedScalarEncoder':
      self.assertGreaterEqual(encodersDict['c1']['resolution'], 0.001,
                              "Resolution is too low")

    # Ensure tm_cpp returns correct json file
    params = getScalarMetricWithTimeOfDayAnomalyParams([0], tmImplementation="tm_cpp")
    self.assertEqual(params['modelConfig']['modelParams']['tpParams']['temporalImp'], "tm_cpp",
                     "Incorrect json for tm_cpp tmImplementation")

    # Ensure incorrect tmImplementation throws exception
    with self.assertRaises(ValueError):
        getScalarMetricWithTimeOfDayAnomalyParams([0], tmImplementation="")
  def _createModel(cls, stats, replaceParams):
    """Instantiate and configure an OPF model

    :param dict stats: Metric data stats per stats_schema.json in the
      unicorn_backend package.
    :param sequence replaceParams: Parameter replacement PATH REPLACEMENT pairs
    :returns: OPF Model instance
    """
    # Generate swarm params
    swarmParams = getScalarMetricWithTimeOfDayAnomalyParams(
      metricData=[0],
      minVal=stats["min"],
      maxVal=stats["max"],
      minResolution=stats.get("minResolution"))

    for path, replacement in replaceParams:
      _recurseDictAndReplace(swarmParams,
                             path.split(_REPLACE_PATH_SEPARATOR),
                             replacement)

    model = ModelFactory.create(modelConfig=swarmParams["modelConfig"])
    model.enableLearning()
    model.enableInference(swarmParams["inferenceArgs"])

    return model
Exemple #31
0
    def POST(self, name):
        """
    /models/{name}

    schema:
    {
      "modelParams": dict containing model parameters
      "predictedFieldName": str
    }

    returns:
    {"success":name}
    """
        global g_models

        data = json.loads(web.data())
        modelParams = data["modelParams"]
        predictedFieldName = data["predictedFieldName"]

        if name in g_models.keys():
            raise web.badrequest("Model with name <%s> already exists" % name)

        model = ModelFactory.create(modelParams)
        model.enableInference({'predictedField': predictedFieldName})
        g_models[name] = model

        return json.dumps({"success": name})
  def testCloneModelWithCheckpoint(self):
    checkpointMgr = ModelCheckpointMgr()

    modelID = uuid.uuid1().hex
    destModelID = uuid.uuid1().hex

    # Create the source model with meta-info only (no checkpoint)
    modelDef = {'a': 1, 'b': 2, 'c':3}
    checkpointMgr.define(modelID, modelDef)

    # Create a model that we can clone
    model1 = ModelFactory.create(self._getModelParams("variant1"))
    checkpointMgr.save(modelID, model1, attributes="attributes1")

    # Clone the source model
    checkpointMgr.clone(modelID, destModelID)

    # Discard the source model checkpoint
    checkpointMgr.remove(modelID)

    # Verify that the destination model's definition is the same as the
    # source model's
    destModelDef = checkpointMgr.loadModelDefinition(destModelID)
    self.assertDictEqual(destModelDef, modelDef)

    # Verify that the destination model's attributes match the source's
    attributes = checkpointMgr.loadCheckpointAttributes(destModelID)
    self.assertEqual(attributes, "attributes1")

    # Attempt to load the cloned model from checkpoint
    model = checkpointMgr.load(destModelID)
    self.assertEqual(str(model.getFieldInfo()), str(model1.getFieldInfo()))
Exemple #33
0
  def load(self, modelID):
    """ Retrieve a model instance from checkpoint.

    param modelID: unique model ID

    retval: an OPF model instance

    raises:
      ModelNotFound if the model checkpoint hasn't been saved yet or if this
        model's entry doesn't exist in the checkpoint archive
    """
    startTime = time.time()

    checkpointStoreDirPath = self._getCurrentCheckpointRealPath(modelID)

    modelInstanceDirPath = os.path.join(checkpointStoreDirPath,
                                        self._CHECKPOINT_INSTANCE_DIR_NAME)

    model = ModelFactory.loadFromCheckpoint(modelInstanceDirPath)

    self._logger.info(
      "{TAG:MCKPT.LOAD} Loaded model=%s: duration=%ss; directory=%s",
      modelID, time.time() - startTime, checkpointStoreDirPath)

    return model
  def reset(self, params, repetition):
    random.seed(params['seed'])

    if params['dataset'] == 'simple':
      self.dataset = SimpleDataset()
    elif params['dataset'] == 'reber':
      self.dataset = ReberDataset(maxLength=params['max_length'])
    elif params['dataset'] == 'high-order':
      self.dataset = HighOrderDataset(numPredictions=params['num_predictions'])
    else:
      raise Exception("Dataset not found")

    # if not os.path.exists(resultsDir):
    #   os.makedirs(resultsDir)
    # self.resultsFile = open(os.path.join(resultsDir, "0.log"), 'w')
    if params['verbosity'] > 0:
      print " initializing HTM model..."
    self.model = ModelFactory.create(MODEL_PARAMS)
    self.model.enableInference({"predictedField": "element"})
    self.shifter = InferenceShifter()
    self.mapping = getEncoderMapping(self.model)

    self.currentSequence = self.dataset.generateSequence()
    self.numPredictedActiveCells = []
    self.numPredictedInactiveCells = []
    self.numUnpredictedActiveColumns = []

    self.currentSequence = self.dataset.generateSequence()
    self.perturbed = False
    self.randoms = []
    self.verbosity = 1
    self.sequenceCounter = 0
Exemple #35
0
  def POST(self, name):
    """
    /models/{name}

    schema:
    {
      "modelParams": dict containing model parameters
      "predictedFieldName": str
    }

    returns:
    {"success":name}
    """
    global g_models

    data = json.loads(web.data())
    modelParams = data["modelParams"]
    predictedFieldName = data["predictedFieldName"]

    if name in g_models.keys():
      raise web.badrequest("Model with name <%s> already exists" % name)

    model = ModelFactory.create(modelParams)
    model.enableInference({'predictedField': predictedFieldName})
    g_models[name] = model

    return json.dumps({"success": name})
Exemple #36
0
def create_downstream_model(intersections):
    """
    Just model those downstream sensors as separate inputs
    :param intersection:
    :return:
    """
    model_params = getModelParamsFromName('3104_3044', clear=True)
    intersection = locations_collection.find_one({'intersection_number': intersections[0],
                                                  'neighbours_sensors': {'$exists': True}})
    if intersection is None:
        sys.exit('No such intersection exists or it has no neighbours_sensors')

    multi_encoder = {
        'type': 'MultiEncoder',
        'fieldname': 'lanes',
        'encoderDescriptions': {}
    }
    for i in intersection['neighbours_sensors'][intersections[1]]['to']:
        i = str(i)
        multi_encoder['encoderDescriptions'][i] = get_sensor_encoder(i, 250)
    for i in get_time_encoders():
        model_params['modelParams']['sensorParams']['encoders'][i['name']] = i
    import json
    model_params['modelParams']['sensorParams']['encoders']['lanes'] = multi_encoder
    print json.dumps(model_params['modelParams']['sensorParams'], indent=4)
    # return None, intersection
    model = ModelFactory.create(model_params)
    model.enableInference({'predictedField': 'lanes'})
    return model, intersection
def run_sine_experiment():
  input_file = "sine.csv"
  generate_data.run(input_file)
  model_params = swarm_over_data()
  if PLOT:
    output = NuPICPlotOutput("sine_output", show_anomaly_score=True)
  else:
    output = NuPICFileOutput("sine_output", show_anomaly_score=True)
  model = ModelFactory.create(model_params)
  model.enableInference({"predictedField": "sine"})

  with open(input_file, "rb") as sine_input:
    csv_reader = csv.reader(sine_input)

    # skip header rows
    csv_reader.next()
    csv_reader.next()
    csv_reader.next()

    # the real data
    for row in csv_reader:
      angle = float(row[0])
      sine_value = float(row[1])
      result = model.run({"sine": sine_value})
      output.write(angle, sine_value, result, prediction_step=1)

  output.close()
    def testCloneModelWithCheckpoint(self):
        checkpointMgr = ModelCheckpointMgr()

        modelID = uuid.uuid1().hex
        destModelID = uuid.uuid1().hex

        # Create the source model with meta-info only (no checkpoint)
        modelDef = {'a': 1, 'b': 2, 'c': 3}
        checkpointMgr.define(modelID, modelDef)

        # Create a model that we can clone
        model1 = ModelFactory.create(self._getModelParams("variant1"))
        checkpointMgr.save(modelID, model1, attributes="attributes1")

        # Clone the source model
        checkpointMgr.clone(modelID, destModelID)

        # Discard the source model checkpoint
        checkpointMgr.remove(modelID)

        # Verify that the destination model's definition is the same as the
        # source model's
        destModelDef = checkpointMgr.loadModelDefinition(destModelID)
        self.assertDictEqual(destModelDef, modelDef)

        # Verify that the destination model's attributes match the source's
        attributes = checkpointMgr.loadCheckpointAttributes(destModelID)
        self.assertEqual(attributes, "attributes1")

        # Attempt to load the cloned model from checkpoint
        model = checkpointMgr.load(destModelID)
        self.assertEqual(str(model.getFieldInfo()), str(model1.getFieldInfo()))
Exemple #39
0
    def initialize(self):
        # Get config params, setting the RDSE resolution
        rangePadding = abs(self.inputMax - self.inputMin) * 0.2
        modelParams = getScalarMetricWithTimeOfDayAnomalyParams(
            metricData=[0],
            minVal=self.inputMin - rangePadding,
            maxVal=self.inputMax + rangePadding,
            minResolution=0.001,
            tmImplementation="cpp")["modelConfig"]

        self._setupEncoderParams(
            modelParams["modelParams"]["sensorParams"]["encoders"])

        self.model = ModelFactory.create(modelParams)

        self.model.enableInference({"predictedField": "value"})

        if self.useLikelihood:
            # Initialize the anomaly likelihood object
            numentaLearningPeriod = math.floor(self.probationaryPeriod / 2.0)
            self.anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood(
                claLearningPeriod=numentaLearningPeriod,
                estimationSamples=self.probationaryPeriod -
                numentaLearningPeriod,
                reestimationPeriod=100)
  def test_hotgym_anomalyScore_stays_below_50_perc_after_110_rows(self):
    """
    Tests that the hotgym anomalyScore values stays below 50% after feeding in
    110 rows of data.
    """
    model = ModelFactory.create(rec_center_hourly_model_params.MODEL_PARAMS)
    model.enableInference({"predictedField": "kw_energy_consumption"})
    inputFile = open(CSV_DATA, "rb")
    csvReader = csv.reader(inputFile)
    # skip header rows
    csvReader.next()
    csvReader.next()
    csvReader.next()

    rowCount = 0

    for row in csvReader:
      rowCount += 1
      timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
      consumption = float(row[1])
      result = model.run({
        "timestamp": timestamp,
        "kw_energy_consumption": consumption
      })
      anomalyScore = result.inferences["anomalyScore"]
      print "row %i: %f" % (rowCount, anomalyScore)
      if rowCount >= START_AT_ROW:
        unittest.TestCase.assertGreater(self, ANOMALY_THRESHOLD, anomalyScore,
            "Anomaly score exceeded threshold of %f after %i rows of data." % (
            ANOMALY_THRESHOLD, rowCount))
        break

    inputFile.close()
Exemple #41
0
def createModel():
    # create the model with the parameters that from the swarm training.
    model = ModelFactory.create(heartBeat_model_params.MODEL_PARAMS)
    model.enableInference({
        "predictedField": "a"
    })
    return model
Exemple #42
0
    def initialize(self):
        # Get config params, setting the RDSE resolution
        rangePadding = abs(self.inputMax - self.inputMin) * 0.2
        modelParams = getScalarMetricWithTimeOfDayAnomalyParams(
            metricData=[0],
            minVal=self.inputMin - rangePadding,
            maxVal=self.inputMax + rangePadding,
            minResolution=0.001,
            tmImplementation="cpp",
        )["modelConfig"]

        self._setupEncoderParams(modelParams["modelParams"]["sensorParams"]["encoders"])

        self.model = ModelFactory.create(modelParams)

        self.model.enableInference({"predictedField": "value"})

        if self.useLikelihood:
            # Initialize the anomaly likelihood object
            numentaLearningPeriod = math.floor(self.probationaryPeriod / 2.0)
            self.anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood(
                claLearningPeriod=numentaLearningPeriod,
                estimationSamples=self.probationaryPeriod - numentaLearningPeriod,
                reestimationPeriod=100,
            )
def createModel(verbosity, categories):
    model = ModelFactory.create(addCategoryEncoder(model_params.MODEL_PARAMS, categories))
    model.enableInference({"predictedField": "token"})

    if verbosity:
        print(model)

    return model
Exemple #44
0
def createModel(verbose):
    params = model_params.MODEL_PARAMS
    if verbose:
        print "Model parameters:"
        print params
    model = ModelFactory.create(params)
    model.enableInference({"predictedField": "vector"})
    return model
Exemple #45
0
 def create_model(self, params):
     # create model
     self.model = ModelFactory.create(params)
     # set init params
     self.model.enableLearning()
     self.model.enableInference({'predictedField': 'value'})
     # init metrics
     self.init_metrics()
Exemple #46
0
def do_teach_ng(sensors_dir_ng, sensor, store, swarm):
    swarm_config_path = sensors_dir_ng + sensor + '/stores/' + store + '/swarms/' + swarm + '/'
    store_path = sensors_dir_ng + sensor + '/stores/' + store + '/out.csv'

    print swarm_config_path

    #load original swarm config file
    with open(swarm_config_path + 'swarm_config.json') as json_file:
        swarm_config = json.load(json_file)
        print(swarm_config)

    print swarm_config['inferenceArgs']['predictedField']

    #laod model configuration
    model = ModelFactory.create(getModelParamsFromFileNG(swarm_config_path))

    #configure prediction
    model.enableInference(
        {"predictedField": swarm_config['inferenceArgs']['predictedField']})

    target = open(swarm_config_path + "teach.csv", 'w')
    spamwriter = csv.writer(target,
                            delimiter=',',
                            quotechar='|',
                            quoting=csv.QUOTE_MINIMAL)
    fieldNames = map(lambda x: x['fieldName'], swarm_config['includedFields'])
    columnNames = copy.copy(fieldNames)
    presteps = swarm_config['inferenceArgs']['predictionSteps']
    for prediction in presteps:
        columnNames.append(swarm_config['inferenceArgs']['predictedField'] +
                           '_predicted_' + str(prediction))
    spamwriter.writerow(columnNames)
    with open(store_path, 'rb') as csvfile:
        spamreader = csv.reader(csvfile, delimiter=',', quotechar='"')
        for row in spamreader:
            print ', '.join(row)

            #prepare input for model
            run_set = {}
            for index, name in enumerate(fieldNames):
                run_set[name] = float(row[index])

            #run the model
            result = model.run(run_set)

            print result.rawInput
            print result.inferences['multiStepBestPredictions']

            #spamwriter.writerow([float(row[0]), float(row[1]), float(row[2]), float(row[3]), result.inferences['multiStepBestPredictions'][4]])
            out_row = []
            for index, name in enumerate(fieldNames):
                out_row.append(float(row[index]))
            for index, predstep in enumerate(presteps):
                out_row.append(
                    result.inferences['multiStepBestPredictions'][predstep])
            spamwriter.writerow(out_row)

    model.save(str(swarm_config_path + "/model_save"))
Exemple #47
0
    def __init__(self, config):

        # Instantiate NuPIC model
        model_params = base_model_params.MODEL_PARAMS

        # Set resolution
        model_params['modelParams']['sensorParams']['encoders']['value']['resolution'] = config['resolution']

        # Override other Nupic parameters:
        model_params['modelParams'] = update_dict(model_params['modelParams'], config['nupic_model_params'])

        # Create model and enable inference on it
        self.model = ModelFactory.create(model_params)
        self.model.enableInference({'predictedField': 'value'})

        # The shifter is used to bring the predictions to the actual time frame
        self.shifter = InferenceShifter()

        # The anomaly likelihood object
        self.anomalyLikelihood = AnomalyLikelihood()

        # Set stream source
        self.stream = config['stream']

        # Setup class variables
        self.db = redis.Redis('localhost')
        self.seconds_per_request = config['seconds_per_request']
        self.webhook = config['webhook']
        self.anomaly_threshold = config['anomaly_threshold']
        self.likelihood_threshold = config['likelihood_threshold']
        self.domain = config['domain']
        self.alert = False # Toogle when we get above threshold

        # Setup logging
        self.logger =  logger or logging.getLogger(__name__)
        handler = logging.handlers.RotatingFileHandler(os.environ['LOG_DIR']+"/monitor_%s.log" % self.stream.name,
                                                       maxBytes=1024*1024,
                                                       backupCount=4,
                                                      )

        handler.setFormatter(logging.Formatter('[%(levelname)s/%(processName)s][%(asctime)s] %(name)s %(message)s'))
        handler.setLevel(logging.INFO)
        self.logger.addHandler(handler)
        self.logger.setLevel(logging.INFO)

        self.logger.info("=== Settings ===")
        self.logger.info("Webhook: %s", self.webhook)
        self.logger.info("Domain: %s", self.domain)
        self.logger.info("Seconds per request: %d", self.seconds_per_request)

        # Write metadata to Redis
        try:
            # Save in redis with key = 'results:monitor_id' and value = 'time, status, actual, prediction, anomaly'
            self.db.set('name:%s' % self.stream.id, self.stream.name)
            self.db.set('value_label:%s' % self.stream.id, self.stream.value_label)
            self.db.set('value_unit:%s' % self.stream.id, self.stream.value_unit)
        except Exception:
            self.logger.warn("Could not write results to redis.", exc_info=True)
Exemple #48
0
def createModel():
    model = ModelFactory.create(category_test.config)
    model.enableInference({
        "predictedField": "category"
        })
    # model.enableInference({
    #     "predictedField": "kw_energy_consumption"
    #     })
    return model
def createModel(metric):
  """
    Fetch the model params for the specific metric and create a new HTM model

    :param metric: AvogadroAgent metric class

    :returns: An HTM Model
  """
  return ModelFactory.create(metric.MODEL_PARAMS)
Exemple #50
0
 def load_model(self, modelSaveDir):
     try:
         print "Loading model from %s..." % modelSaveDir
         model = ModelFactory.loadFromCheckpoint(modelSaveDir)
         self._init_model(model)
     except:
         print "ERR", "load_model", sys.exc_info()
         traceback.print_exc()
         raise
def runAnomaly(options):
  """
  Create and run a CLA Model on the given dataset (based on the hotgym anomaly
  client in NuPIC).
  """
  # Load the model params JSON
  with open("model_params.json") as fp:
    modelParams = json.load(fp)

  # Update the min/max value for the encoder
  sensorParams = modelParams['modelParams']['sensorParams']
  sensorParams['encoders']['value']['maxval'] = options.max
  sensorParams['encoders']['value']['minval'] = options.min

  model = ModelFactory.create(modelParams)
  model.enableInference({'predictedField': 'value'})
  with open (options.inputFile) as fin:
    
    # Open file and setup headers
    reader = csv.reader(fin)
    csvWriter = csv.writer(open(options.outputFile,"wb"))
    csvWriter.writerow(["timestamp", "value",
                        "anomaly_score", "likelihood_score"])
    headers = reader.next()
    
    # The anomaly likelihood object
    anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood()
    
    # Iterate through each record in the CSV file
    print "Starting processing at",datetime.datetime.now()
    for i, record in enumerate(reader, start=1):
      
      # Convert input data to a dict so we can pass it into the model
      inputData = dict(zip(headers, record))
      inputData["value"] = float(inputData["value"])
      inputData["dttm"] = dateutil.parser.parse(inputData["dttm"])
      
      # Send it to the CLA and get back the raw anomaly score
      result = model.run(inputData)
      anomalyScore = result.inferences['anomalyScore']
      
      # Compute the Anomaly Likelihood
      likelihood = anomalyLikelihood.anomalyProbability(
        inputData["value"], anomalyScore, inputData["dttm"])
      if likelihood > 0.9999:
        print "Anomaly detected:",inputData['dttm'],inputData['value'],likelihood

      # Write results to the output CSV file
      csvWriter.writerow([inputData["dttm"], inputData["value"],
                          anomalyScore, likelihood])

      # Progress report
      if (i%500) == 0: print i,"records processed"

  print "Completed processing",i,"records at",datetime.datetime.now()
  print "Anomaly scores for",options.inputFile,
  print "have been written to",options.outputFile
Exemple #52
0
 def createModelFromParams(self, modelParams):
     """
     Creates a model fromt the provided model parameters file
     :param modelParams:
     :return:
     """
     model = ModelFactory.create(modelParams)
     model.enableInference({"predictedField": self.fieldToPredict})
     return model
 def createModel(self, modelParams, predictedField):
     """
 Creates a model.
 @param modelParams: () The modelParams used for running the model.
 @param predictedField: (string) The predictedField.
 """
     model = ModelFactory.create(modelParams)
     model.enableInference({"predictedField": predictedField})
     self.model = model
Exemple #54
0
def createModel(seed=1956):
    params = model_params.MODEL_PARAMS
    params['modelParams']['spParams']['seed'] = seed

    # Randomly change encoder resolution to be in [0.78, 0.98], i.e. 0.88 +/- 0.1
    params['modelParams']['sensorParams']['encoders']['consumption'][
        'resolution'] = 0.78 + numpy.random.random() * .2

    return ModelFactory.create(model_params.MODEL_PARAMS)
def createModel(metric):
  """
    Fetch the model params for the specific metric and create a new HTM model

    :param metric: AvogadroAgent metric class

    :returns: An HTM Model
  """
  return ModelFactory.create(metric.MODEL_PARAMS)
def initialize():
    global model, shifter
    
    model = ModelFactory.create(model_params.MODEL_PARAMS)
    model.enableInference({'predictedField': 'x'})
    print "model initialized !"
    
    shifter = InferenceShifter()
    print "shifter initialized !"
Exemple #57
0
def createModel(seed=1956):
  params = model_params.MODEL_PARAMS
  params['modelParams']['spParams']['seed'] = seed

  # Randomly change encoder resolution to be in [0.78, 0.98], i.e. 0.88 +/- 0.1
  params['modelParams']['sensorParams']['encoders']['consumption'][
    'resolution'] = 0.78 + numpy.random.random()*.2

  return ModelFactory.create(model_params.MODEL_PARAMS)