def initModel(self): if os.path.exists(os.path.abspath(self.model_path)): self.model = ModelFactory.loadFromCheckpoint(os.path.relpath(self.model_path)) else: self.model = ModelFactory.create(self.model_params) predicted_field = self.model_params["predictedField"] if predicted_field: self.model.enableInference({"predictedField": predicted_field})
def initModel(self): if os.path.exists(os.path.abspath(self.model_path)): self.model = ModelFactory.loadFromCheckpoint( os.path.relpath(self.model_path)) else: self.model = ModelFactory.create(self.model_params) predicted_field = self.model_params['predictedField'] if predicted_field: self.model.enableInference({'predictedField': predicted_field})
def __init__(self,name,admin_in,admin_out,sensor_spec, sensors_dir,sensor_in,store,swarm): threading.Thread.__init__(self) #self.config = config self.sensor_in = sensor_in self.store = store self.swarm = swarm self.name = name self.brain_available = False threading.Thread.__init__(self) Sensor. __init__(self,name=name,admin_in=admin_in, admin_out=admin_out,sensor_spec=sensor_spec, sensors_dir=sensors_dir) swarm_config_path = sensors_dir + sensor_in +'/stores/' + store + '/swarms/' + swarm +'/' #store_path = sensors_dir + sensor_in +'/stores/' + store + '/out.csv' #model = ModelFactory.loadFromCheckpoint('/home/hans/cortical_one_var/sensors/cpu/stores/store_3/swarms/swarm_1/model_save') print swarm_config_path #load original swarm config file with open(swarm_config_path + 'swarm_config.json')as json_file: self.swarm_config = json.load(json_file) print(self.swarm_config) self.swarm_config_ng = SwarmConfig(self.swarm_config) print self.swarm_config_ng.get_predicted_field() #if there is a 'brain', then tae the existing brain self.possible_brain_path = str(swarm_config_path + 'model_save') if os.path.exists(self.possible_brain_path): possible_brain_2 = '/home/hans/cortical_one_var/sensors/cpu/stores/store_3/swarms/swarm_1/model_save' print "load existing brain..." print self.possible_brain_path #model = ModelFactory.loadFromCheckpoint(possible_brain_2) model = ModelFactory.loadFromCheckpoint(self.possible_brain_path) #use this case to add the availabilty of a 'brain' (???!!!) to your annuncement else: #laod model configuration model = ModelFactory.create(getModelParamsFromFileNG(swarm_config_path)) #configure prediction model.enableInference({"predictedField": self.swarm_config_ng.get_predicted_field()}) self.connection_sensor_in = stomp.Connection() self.connection_sensor_in.set_listener(name=self.name, lstnr=AbstractSensorListener(self.name,topic = '/topic/' +self.sensor_in,config=self.swarm_config_ng,model=model)) self.connection_sensor_in.start() self.connection_sensor_in.connect(self.user, self.password, wait=True) #self.connection_sensor_in.connect('admin', 'password', wait=True) self.abstract_listener = self.connection_sensor_in.get_listener(name=self.name) self.connection_sensor_in.subscribe(destination='/topic/' +self.sensor_in, id=2, ack='auto') self.values = [] self.self_announcement()
def getModel(flag): modelDir = os.getcwd() + '/model/%s' % flag _LOGGER.info(modelDir) if os.path.exists(modelDir): _LOGGER.info('model exists') model = ModelFactory.loadFromCheckpoint(modelDir) return model else: _LOGGER.info('creating new model') model = ModelFactory.create(MODEL_PARAMS) model.save(modelDir) return model
def getModel(flag): modelDir = os.getcwd() + "/model/%s" % flag _LOGGER.info(modelDir) if os.path.exists(modelDir): _LOGGER.info("model exists") model = ModelFactory.loadFromCheckpoint(modelDir) return model else: _LOGGER.info("creating new model") model = ModelFactory.create(MODEL_PARAMS) model.save(modelDir) return model
def __init__(self, predictStep, enablePredict, maxValue, minValue, minResolution): # initial the parameters and data variables. self.predictStep = predictStep self.enablePredict = enablePredict self.metricData = xrange(int(minValue), int(maxValue), int((maxValue - minValue) / minResolution)) self.maxValue = maxValue self.minValue = minValue self.minResolution = minResolution self.timestamp = None self.actualValue = None self.predictValue = None self.anomalyScore = None self.modelResult = None self.output = None # get the model parameters. self.parameters = getScalarMetricWithTimeOfDayAnomalyParams( self.metricData, self.minValue, self.maxValue, self.minResolution) # make sure the result contains the predictions. self.parameters["modelConfig"]["modelParams"][ "clEnable"] = self.enablePredict # so we can modify the predict step by do that: self.parameters["modelConfig"]["modelParams"]["clParams"][ "steps"] = self.predictStep # create the model self.model = ModelFactory.create(self.parameters["modelConfig"]) self.model.enableInference(self.parameters["inferenceArgs"])
def configureDetector(self, probationaryPeriodData): # Load the model params JSON probationaryPeriod = probationaryPeriodData.shape[0] paramsPath = os.path.join(os.path.split(__file__)[0], "modelParams", "model_params.json") with open(paramsPath) as fp: modelParams = json.load(fp) self.sensorParams = modelParams["modelParams"]["sensorParams"]\ ["encoders"]["value"] # RDSE - resolution calculation resolution = max(0.001, (self.inputMax - self.inputMin) / \ self.sensorParams.pop("numBuckets") ) self.sensorParams["resolution"] = resolution self.model = ModelFactory.create(modelParams) self.model.enableInference({"predictedField": "value"}) # The anomaly likelihood object numentaLearningPeriod = math.floor(probationaryPeriod / 2.0) self.anomalyLikelihood = AnomalyLikelihood(probationaryPeriod, numentaLearningPeriod)
def __init__(self): """MODEL_PARAMS = { # Your choice }""" self.model = ModelFactory.create(MODEL_PARAMS) self.lastInput = -1 super(HelloModel, self).__init__(self.model)
def run_experiment(): import model_params model = ModelFactory.create(MODEL_PARAMS) model.enableInference({ "predictedField": "event" }) with open(IN_PATH, "rb") as event_input, open(OUT_PATH, "w") as event_output: csv_writer = csv.writer(event_output) csv_writer.writerow(['timestamp', 'thread_id', 'event', 'anomalyScore', 'prediction1', 'likelihood1', 'prediction2', 'likelihood2']) csv_reader = csv.reader(event_input) # skip headers csv_reader.next() csv_reader.next() csv_reader.next() # the real data for row in csv_reader: timestamp = dateutil.parser.parse(row[0]) thread_id = row[1] event = row[2] result = model.run({"timestamp": timestamp, "thread_id": thread_id, "event": event, }) # Print to files / command line output_row = [timestamp, thread_id, event, result.inferences['anomalyScore']] predictions = result.inferences['multiStepPredictions'][1] for prediction, likelihood in sorted(predictions.items(), key=itemgetter(1), reverse=True): output_row.append(prediction) output_row.append(likelihood) csv_writer.writerow(output_row) if csv_reader.line_num % 100 == 0: print 'Wrote %s lines of output...' % csv_reader.line_num
def run(): if SMOOTH: inputDataDir = "../preprocessing/smoothed_data" else: inputDataDir = "../preprocessing/formatted_data" model = ModelFactory.create(MODEL_PARAMS) model.enableInference({"predictedField": "channel_value"}) for channel in CHANNELS: for movement in MOTOR: inputFile = "%s/%s_%s.csv" % (inputDataDir, movement, channel) with open(inputFile, "rb") as input: csvReader = csv.reader(input) # skip header rows csvReader.next() csvReader.next() csvReader.next() # train model for row in csvReader: channel_value = float(row[0]) model.run({"channel_value": channel_value}) # save model modelDir = "%s/%s_%s_%s" % (CWD, MODEL_PREFIX, movement, channel) model.save(modelDir) print "Saved new model in %s" % modelDir
def predict(self, model): # model input data is cla_model_input=dict() cla_model_input["rssi"] = float(model.rssi_raw) cla_model_input["quality"] = float(model.quality_raw) if not self.cla_model: # Create the model for predicting WiFi usage # The shifter will align prediction and actual values. s1 = int(round(time.time() * 1000)) self.cla_model = ModelFactory.create(model_params.MODEL_PARAMS) self.cla_model.enableInference({'predictedField': 'rssi'}) self.shifter = InferenceShifter() s2 = int(round(time.time() * 1000)) #print "Predictor construction %s,%d,%d,%d" % (self, s1, s2, abs(s2-s1)) # Shift 'x' time periods into the future # Compare 'current' with 'predicted - pos(x)' - how do we compare # The point is to be able to Generalize on patterns that it hasn't seen yet result = self.shifter.shift(self.cla_model.run(cla_model_input)) inference = result.inferences['multiStepBestPredictions'][self.timestep] # Store away current rssi information on the model if inference is not None: model.rssi_predicted = int(inference) model.rssi_delta = abs(model.rssi_raw - model.rssi_predicted) model.zone_predicted.assign_rssi_zone(model.rssi_predicted)
def run(fname="verylargeseism_out"): print("reading %s" % input_file) print("using this model: ", model_params) model = ModelFactory.create(model_params) model.enableInference({"predictedField": "acc"}) print("created nupic model in :%f min" % ((clk.time() - t0) / 60.)) print("will output predictions and anomalies \n in this file: %s" % fname) output = NuPICFileOutput(fname, show_anomaly_score=True) with open(input_file, "rb") as data_input: csv_reader = csv.reader(data_input) # skip header rows csv_reader.next() csv_reader.next() csv_reader.next() # the real data for row in csv_reader: time = float(row[0]) acc_value = float(row[1]) result = model.run({"acc": acc_value}) output.write(time, acc_value, result, prediction_step=PSTEPS) output.close() print("time is :%f min" % ((clk.time() - t0) / 60.))
def __init__(self, numPredictions, resultsDir): random.seed(43) self.numPredictions = numPredictions if not os.path.exists(resultsDir): os.makedirs(resultsDir) self.resultsFile = open(os.path.join(resultsDir, "0.log"), 'w') self.model = ModelFactory.create(MODEL_PARAMS) self.model.enableInference({"predictedField": "element"}) self.shifter = InferenceShifter() self.mapping = getEncoderMapping(self.model) self.correct = [] self.numPredictedActiveCells = [] self.numPredictedInactiveCells = [] self.numUnpredictedActiveColumns = [] self.iteration = 0 self.perturbed = False self.randoms = [] self.verbosity = 1 self.dataset = HighOrderDataset(numPredictions=self.numPredictions) self.sequences = [] self.currentSequence = [] self.replenish_sequence()
def do_teach(): model = ModelFactory.create(getModelParamsFromName()) model.enableInference({"predictedField": "s1"}) target = open("teach.csv", 'w') spamwriter = csv.writer(target, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) spamwriter.writerow(['s1', 's2', 's3', 's4', 'predicts1']) with open('out.csv', 'rb') as csvfile: spamreader = csv.reader(csvfile, delimiter=',', quotechar='"') for row in spamreader: print ', '.join(row) result = model.run({ "s1": float(row[0]), "s2": float(row[1]), "s3": float(row[2]), "s4": float(row[3]) }) print result.rawInput print result.inferences['multiStepBestPredictions'] spamwriter.writerow([float(row[0]), float(row[1]), float(row[2]), float(row[3]), result.inferences['multiStepBestPredictions'][4]]) model.save(os.getcwd() + "/model_save")
def _createModel(cls, stats, replaceParams): """Instantiate and configure an OPF model :param dict stats: Metric data stats per stats_schema.json in the unicorn_backend package. :param sequence replaceParams: Parameter replacement PATH REPLACEMENT pairs :returns: OPF Model instance """ # Generate swarm params swarmParams = getScalarMetricWithTimeOfDayAnomalyParams( metricData=[0], minVal=stats["min"], maxVal=stats["max"], minResolution=stats.get("minResolution")) for path, replacement in replaceParams: _recurseDictAndReplace(swarmParams, path.split(_REPLACE_PATH_SEPARATOR), replacement) model = ModelFactory.create(modelConfig=swarmParams["modelConfig"]) model.enableLearning() model.enableInference(swarmParams["inferenceArgs"]) return model
def test_hotgym_anomalyScore_stays_below_50_perc_after_110_rows(self): """ Tests that the hotgym anomalyScore values stays below 50% after feeding in 110 rows of data. """ model = ModelFactory.create( rec_center_hourly_model_params.MODEL_PARAMS) model.enableInference({"predictedField": "kw_energy_consumption"}) inputFile = open(CSV_DATA, "rb") csvReader = csv.reader(inputFile) # skip header rows csvReader.next() csvReader.next() csvReader.next() rowCount = 0 for row in csvReader: rowCount += 1 timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT) consumption = float(row[1]) result = model.run({ "timestamp": timestamp, "kw_energy_consumption": consumption }) anomalyScore = result.inferences["anomalyScore"] print "row %i: %f" % (rowCount, anomalyScore) if rowCount >= START_AT_ROW: unittest.TestCase.assertGreater( self, ANOMALY_THRESHOLD, anomalyScore, "Anomaly score exceeded threshold of %f after %i rows of data." % (ANOMALY_THRESHOLD, rowCount)) break inputFile.close()
def run_experiment(): generate_data.run() swarm_over_data() import model_params model = ModelFactory.create(model_params.MODEL_PARAMS) model.enableInference({"predictedField": "sine"}) output = NuPICPlotOutput("sine_out", show_anomaly_score=True) with open("sine.csv", "rb") as sine_input: csv_reader = csv.reader(sine_input) # Skip headers csv_reader.next() csv_reader.next() csv_reader.next() # Real Data for row in csv_reader: angle = float(row[0]) sine_value = float(row[1]) result = model.run({"sine": sine_value}) output.write(angle, sine_value, result) output.close()
def initialize(self): calcRange = abs(self.inputMax - self.inputMin) calcPad = calcRange * .2 self.inputMin = self.inputMin - calcPad self.inputMax = self.inputMax + calcPad # Load the model params JSON paramsPath = os.path.join( os.path.split(__file__)[0], "modelParams", "model_params.json") with open(paramsPath) as fp: modelParams = json.load(fp) self.sensorParams = modelParams["modelParams"]["sensorParams"]\ ["encoders"]["value"] # RDSE - resolution calculation resolution = max(0.001, (self.inputMax - self.inputMin) / \ self.sensorParams.pop("numBuckets") ) self.sensorParams["resolution"] = resolution self.model = ModelFactory.create(modelParams) self.model.enableInference({"predictedField": "value"}) # Initialize the anomaly likelihood object numentaLearningPeriod = math.floor(self.probationaryPeriod / 2.0) self.anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood( claLearningPeriod=numentaLearningPeriod, estimationSamples=self.probationaryPeriod - numentaLearningPeriod, reestimationPeriod=100)
def __init__(self): """Constr.""" self.model = ModelFactory.create(ModelParams.MODEL_PARAMS) self.model.enableInference({'predictedField': 'cpuUtil'}) self.headers = [ "timestamp", "cpuUtil", "ramUtil", "diskUtil", "numUsers" ]
def __init__(self, modelConfig, inferenceArgs, metricSpecs, sourceSpec, sinkSpec=None): """Initialization. Args: modelConfig: The model config dict. metricSpecs: A sequence of MetricSpec instances. sourceSpec: Path to the source CSV file. sinkSpec: Path to the sink CSV file. """ self.model = ModelFactory.create(modelConfig) self.model.enableInference(inferenceArgs) self.metricsManager = MetricsManager(metricSpecs, self.model.getFieldInfo(), self.model.getInferenceType()) self.sink = None if sinkSpec is not None: # TODO: make this work - sinkSpec not yet supported. raise NotImplementedError('The sinkSpec is not yet implemented.') #self.sink = BasicPredictionLogger( # self.model.getFieldInfo(), sinkSpec, 'myOutput', # self.model.getInferenceType()) #self.sink.setLoggedMetrics( # self.metricsManager.getMetricLabels()) self.datasetReader = BasicDatasetReader(sourceSpec)
def run_seism_experiment(): input_file = "eQnoise.csv" generate_data.run(input_file) print("time is :%f secs" % ((clk.time() - t0) / 60.)) model_params = swarm_over_data() print("time is :%f secs" % ((clk.time() - t0) / 60.)) print(model_params) if PLOT: pass #output = NuPICPlotOutput("sine3_output", show_anomaly_score=True) else: output = NuPICFileOutput("eQnoise_output", show_anomaly_score=True) print("time is :%f min" % ((clk.time() - t0) / 60.)) model = ModelFactory.create(model_params) model.enableInference({"predictedField": "acc"}) with open(input_file, "rb") as data_input: csv_reader = csv.reader(data_input) # skip header rows csv_reader.next() csv_reader.next() csv_reader.next() # the real data for row in csv_reader: time = float(row[0]) acc_value = float(row[1]) result = model.run({"acc": acc_value}) output.write(time, acc_value, result, prediction_step=PSTEPS) output.close() print("time is :%f min" % ((clk.time() - t0) / 60.))
def run_experiment(): # generate_data.run() # swarm_over_data() from model_params import model_params model = ModelFactory.create(model_params.MODEL_PARAMS) model.enableInference({"predictedField": "sine"}) output = nupic_output.NuPICPlotOutput("sine") #output = nupic_output.NuPICFileOutput(["sine"]) #shifter = InferenceShifter() with open("sine.csv", "rb") as sine_input: csv_reader = csv.reader(sine_input) csv_reader.next() csv_reader.next() csv_reader.next() for row in csv_reader: angle = float(row[0]) sine_value = float(row[1]) result = model.run({"sine": sine_value}) print result #result = shifter.shift(result) prediction = result.inferences["multiStepBestPredictions"][1] anomalyScore = result.inferences["anomalyScore"] output.write(angle, sine_value, prediction, anomalyScore) output.close()
def run_sine_experiment(): input_file = "sine.csv" generate_data.run(input_file) model_params = swarm_over_data() if PLOT: output = NuPICPlotOutput("sine_output", show_anomaly_score=True) else: output = NuPICFileOutput("sine_output", show_anomaly_score=True) model = ModelFactory.create(model_params) model.enableInference({"predictedField": "sine"}) with open(input_file, "rb") as sine_input: csv_reader = csv.reader(sine_input) # skip header rows csv_reader.next() csv_reader.next() csv_reader.next() # the real data for row in csv_reader: angle = float(row[0]) sine_value = float(row[1]) result = model.run({"sine": sine_value}) output.write(angle, sine_value, result, prediction_step=1) output.close()
def initialize(self): calcRange = abs(self.inputMax - self.inputMin) calcPad = calcRange * .2 self.inputMin = self.inputMin - calcPad self.inputMax = self.inputMax + calcPad # Load the model params JSON paramsPath = os.path.join(os.path.split(__file__)[0], "modelParams", "model_params.json") with open(paramsPath) as fp: modelParams = json.load(fp) self.sensorParams = modelParams["modelParams"]["sensorParams"]\ ["encoders"]["value"] # RDSE - resolution calculation resolution = max(0.001, (self.inputMax - self.inputMin) / \ self.sensorParams.pop("numBuckets") ) self.sensorParams["resolution"] = resolution self.model = ModelFactory.create(modelParams) self.model.enableInference({"predictedField": "value"}) # The anomaly likelihood object numentaLearningPeriod = math.floor(self.probationaryPeriod / 2.0) self.anomalyLikelihood = AnomalyLikelihood(self.probationaryPeriod, numentaLearningPeriod)
def testModelParams(self): """ Test that clusterParams loads returns a valid dict that can be instantiated as a CLAModel. """ params = getScalarMetricWithTimeOfDayAnomalyParams([0], minVal=23.42, maxVal=23.420001) encodersDict = ( params['modelConfig']['modelParams']['sensorParams']['encoders']) model = ModelFactory.create(modelConfig=params['modelConfig']) self.assertIsInstance( model, CLAModel, "JSON returned cannot be used to create a model") # Ensure we have a time of day field self.assertIsNotNone(encodersDict['c0_timeOfDay']) # Ensure resolution doesn't get too low if encodersDict['c1']['type'] == 'RandomDistributedScalarEncoder': self.assertGreaterEqual(encodersDict['c1']['resolution'], 0.001, "Resolution is too low") # Ensure tm_cpp returns correct json file params = getScalarMetricWithTimeOfDayAnomalyParams( [0], tmImplementation="tm_cpp") self.assertEqual( params['modelConfig']['modelParams']['tpParams']['temporalImp'], "tm_cpp", "Incorrect json for tm_cpp tmImplementation") # Ensure incorrect tmImplementation throws exception with self.assertRaises(ValueError): getScalarMetricWithTimeOfDayAnomalyParams([0], tmImplementation="")
def testModelParams(self): """ Test that clusterParams loads returns a valid dict that can be instantiated as a CLAModel. """ params = getScalarMetricWithTimeOfDayAnomalyParams([0], minVal=23.42, maxVal=23.420001) encodersDict= ( params['modelConfig']['modelParams']['sensorParams']['encoders']) model = ModelFactory.create(modelConfig=params['modelConfig']) self.assertIsInstance(model, CLAModel, "JSON returned cannot be used to create a model") # Ensure we have a time of day field self.assertIsNotNone(encodersDict['c0_timeOfDay']) # Ensure resolution doesn't get too low if encodersDict['c1']['type'] == 'RandomDistributedScalarEncoder': self.assertGreaterEqual(encodersDict['c1']['resolution'], 0.001, "Resolution is too low") # Ensure tm_cpp returns correct json file params = getScalarMetricWithTimeOfDayAnomalyParams([0], tmImplementation="tm_cpp") self.assertEqual(params['modelConfig']['modelParams']['tpParams']['temporalImp'], "tm_cpp", "Incorrect json for tm_cpp tmImplementation") # Ensure incorrect tmImplementation throws exception with self.assertRaises(ValueError): getScalarMetricWithTimeOfDayAnomalyParams([0], tmImplementation="")
def POST(self, name): """ /models/{name} schema: { "modelParams": dict containing model parameters "predictedFieldName": str } returns: {"success":name} """ global g_models data = json.loads(web.data()) modelParams = data["modelParams"] predictedFieldName = data["predictedFieldName"] if name in g_models.keys(): raise web.badrequest("Model with name <%s> already exists" % name) model = ModelFactory.create(modelParams) model.enableInference({'predictedField': predictedFieldName}) g_models[name] = model return json.dumps({"success": name})
def testCloneModelWithCheckpoint(self): checkpointMgr = ModelCheckpointMgr() modelID = uuid.uuid1().hex destModelID = uuid.uuid1().hex # Create the source model with meta-info only (no checkpoint) modelDef = {'a': 1, 'b': 2, 'c':3} checkpointMgr.define(modelID, modelDef) # Create a model that we can clone model1 = ModelFactory.create(self._getModelParams("variant1")) checkpointMgr.save(modelID, model1, attributes="attributes1") # Clone the source model checkpointMgr.clone(modelID, destModelID) # Discard the source model checkpoint checkpointMgr.remove(modelID) # Verify that the destination model's definition is the same as the # source model's destModelDef = checkpointMgr.loadModelDefinition(destModelID) self.assertDictEqual(destModelDef, modelDef) # Verify that the destination model's attributes match the source's attributes = checkpointMgr.loadCheckpointAttributes(destModelID) self.assertEqual(attributes, "attributes1") # Attempt to load the cloned model from checkpoint model = checkpointMgr.load(destModelID) self.assertEqual(str(model.getFieldInfo()), str(model1.getFieldInfo()))
def load(self, modelID): """ Retrieve a model instance from checkpoint. param modelID: unique model ID retval: an OPF model instance raises: ModelNotFound if the model checkpoint hasn't been saved yet or if this model's entry doesn't exist in the checkpoint archive """ startTime = time.time() checkpointStoreDirPath = self._getCurrentCheckpointRealPath(modelID) modelInstanceDirPath = os.path.join(checkpointStoreDirPath, self._CHECKPOINT_INSTANCE_DIR_NAME) model = ModelFactory.loadFromCheckpoint(modelInstanceDirPath) self._logger.info( "{TAG:MCKPT.LOAD} Loaded model=%s: duration=%ss; directory=%s", modelID, time.time() - startTime, checkpointStoreDirPath) return model
def reset(self, params, repetition): random.seed(params['seed']) if params['dataset'] == 'simple': self.dataset = SimpleDataset() elif params['dataset'] == 'reber': self.dataset = ReberDataset(maxLength=params['max_length']) elif params['dataset'] == 'high-order': self.dataset = HighOrderDataset(numPredictions=params['num_predictions']) else: raise Exception("Dataset not found") # if not os.path.exists(resultsDir): # os.makedirs(resultsDir) # self.resultsFile = open(os.path.join(resultsDir, "0.log"), 'w') if params['verbosity'] > 0: print " initializing HTM model..." self.model = ModelFactory.create(MODEL_PARAMS) self.model.enableInference({"predictedField": "element"}) self.shifter = InferenceShifter() self.mapping = getEncoderMapping(self.model) self.currentSequence = self.dataset.generateSequence() self.numPredictedActiveCells = [] self.numPredictedInactiveCells = [] self.numUnpredictedActiveColumns = [] self.currentSequence = self.dataset.generateSequence() self.perturbed = False self.randoms = [] self.verbosity = 1 self.sequenceCounter = 0
def create_downstream_model(intersections): """ Just model those downstream sensors as separate inputs :param intersection: :return: """ model_params = getModelParamsFromName('3104_3044', clear=True) intersection = locations_collection.find_one({'intersection_number': intersections[0], 'neighbours_sensors': {'$exists': True}}) if intersection is None: sys.exit('No such intersection exists or it has no neighbours_sensors') multi_encoder = { 'type': 'MultiEncoder', 'fieldname': 'lanes', 'encoderDescriptions': {} } for i in intersection['neighbours_sensors'][intersections[1]]['to']: i = str(i) multi_encoder['encoderDescriptions'][i] = get_sensor_encoder(i, 250) for i in get_time_encoders(): model_params['modelParams']['sensorParams']['encoders'][i['name']] = i import json model_params['modelParams']['sensorParams']['encoders']['lanes'] = multi_encoder print json.dumps(model_params['modelParams']['sensorParams'], indent=4) # return None, intersection model = ModelFactory.create(model_params) model.enableInference({'predictedField': 'lanes'}) return model, intersection
def testCloneModelWithCheckpoint(self): checkpointMgr = ModelCheckpointMgr() modelID = uuid.uuid1().hex destModelID = uuid.uuid1().hex # Create the source model with meta-info only (no checkpoint) modelDef = {'a': 1, 'b': 2, 'c': 3} checkpointMgr.define(modelID, modelDef) # Create a model that we can clone model1 = ModelFactory.create(self._getModelParams("variant1")) checkpointMgr.save(modelID, model1, attributes="attributes1") # Clone the source model checkpointMgr.clone(modelID, destModelID) # Discard the source model checkpoint checkpointMgr.remove(modelID) # Verify that the destination model's definition is the same as the # source model's destModelDef = checkpointMgr.loadModelDefinition(destModelID) self.assertDictEqual(destModelDef, modelDef) # Verify that the destination model's attributes match the source's attributes = checkpointMgr.loadCheckpointAttributes(destModelID) self.assertEqual(attributes, "attributes1") # Attempt to load the cloned model from checkpoint model = checkpointMgr.load(destModelID) self.assertEqual(str(model.getFieldInfo()), str(model1.getFieldInfo()))
def initialize(self): # Get config params, setting the RDSE resolution rangePadding = abs(self.inputMax - self.inputMin) * 0.2 modelParams = getScalarMetricWithTimeOfDayAnomalyParams( metricData=[0], minVal=self.inputMin - rangePadding, maxVal=self.inputMax + rangePadding, minResolution=0.001, tmImplementation="cpp")["modelConfig"] self._setupEncoderParams( modelParams["modelParams"]["sensorParams"]["encoders"]) self.model = ModelFactory.create(modelParams) self.model.enableInference({"predictedField": "value"}) if self.useLikelihood: # Initialize the anomaly likelihood object numentaLearningPeriod = math.floor(self.probationaryPeriod / 2.0) self.anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood( claLearningPeriod=numentaLearningPeriod, estimationSamples=self.probationaryPeriod - numentaLearningPeriod, reestimationPeriod=100)
def test_hotgym_anomalyScore_stays_below_50_perc_after_110_rows(self): """ Tests that the hotgym anomalyScore values stays below 50% after feeding in 110 rows of data. """ model = ModelFactory.create(rec_center_hourly_model_params.MODEL_PARAMS) model.enableInference({"predictedField": "kw_energy_consumption"}) inputFile = open(CSV_DATA, "rb") csvReader = csv.reader(inputFile) # skip header rows csvReader.next() csvReader.next() csvReader.next() rowCount = 0 for row in csvReader: rowCount += 1 timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT) consumption = float(row[1]) result = model.run({ "timestamp": timestamp, "kw_energy_consumption": consumption }) anomalyScore = result.inferences["anomalyScore"] print "row %i: %f" % (rowCount, anomalyScore) if rowCount >= START_AT_ROW: unittest.TestCase.assertGreater(self, ANOMALY_THRESHOLD, anomalyScore, "Anomaly score exceeded threshold of %f after %i rows of data." % ( ANOMALY_THRESHOLD, rowCount)) break inputFile.close()
def createModel(): # create the model with the parameters that from the swarm training. model = ModelFactory.create(heartBeat_model_params.MODEL_PARAMS) model.enableInference({ "predictedField": "a" }) return model
def initialize(self): # Get config params, setting the RDSE resolution rangePadding = abs(self.inputMax - self.inputMin) * 0.2 modelParams = getScalarMetricWithTimeOfDayAnomalyParams( metricData=[0], minVal=self.inputMin - rangePadding, maxVal=self.inputMax + rangePadding, minResolution=0.001, tmImplementation="cpp", )["modelConfig"] self._setupEncoderParams(modelParams["modelParams"]["sensorParams"]["encoders"]) self.model = ModelFactory.create(modelParams) self.model.enableInference({"predictedField": "value"}) if self.useLikelihood: # Initialize the anomaly likelihood object numentaLearningPeriod = math.floor(self.probationaryPeriod / 2.0) self.anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood( claLearningPeriod=numentaLearningPeriod, estimationSamples=self.probationaryPeriod - numentaLearningPeriod, reestimationPeriod=100, )
def createModel(verbosity, categories): model = ModelFactory.create(addCategoryEncoder(model_params.MODEL_PARAMS, categories)) model.enableInference({"predictedField": "token"}) if verbosity: print(model) return model
def createModel(verbose): params = model_params.MODEL_PARAMS if verbose: print "Model parameters:" print params model = ModelFactory.create(params) model.enableInference({"predictedField": "vector"}) return model
def create_model(self, params): # create model self.model = ModelFactory.create(params) # set init params self.model.enableLearning() self.model.enableInference({'predictedField': 'value'}) # init metrics self.init_metrics()
def do_teach_ng(sensors_dir_ng, sensor, store, swarm): swarm_config_path = sensors_dir_ng + sensor + '/stores/' + store + '/swarms/' + swarm + '/' store_path = sensors_dir_ng + sensor + '/stores/' + store + '/out.csv' print swarm_config_path #load original swarm config file with open(swarm_config_path + 'swarm_config.json') as json_file: swarm_config = json.load(json_file) print(swarm_config) print swarm_config['inferenceArgs']['predictedField'] #laod model configuration model = ModelFactory.create(getModelParamsFromFileNG(swarm_config_path)) #configure prediction model.enableInference( {"predictedField": swarm_config['inferenceArgs']['predictedField']}) target = open(swarm_config_path + "teach.csv", 'w') spamwriter = csv.writer(target, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) fieldNames = map(lambda x: x['fieldName'], swarm_config['includedFields']) columnNames = copy.copy(fieldNames) presteps = swarm_config['inferenceArgs']['predictionSteps'] for prediction in presteps: columnNames.append(swarm_config['inferenceArgs']['predictedField'] + '_predicted_' + str(prediction)) spamwriter.writerow(columnNames) with open(store_path, 'rb') as csvfile: spamreader = csv.reader(csvfile, delimiter=',', quotechar='"') for row in spamreader: print ', '.join(row) #prepare input for model run_set = {} for index, name in enumerate(fieldNames): run_set[name] = float(row[index]) #run the model result = model.run(run_set) print result.rawInput print result.inferences['multiStepBestPredictions'] #spamwriter.writerow([float(row[0]), float(row[1]), float(row[2]), float(row[3]), result.inferences['multiStepBestPredictions'][4]]) out_row = [] for index, name in enumerate(fieldNames): out_row.append(float(row[index])) for index, predstep in enumerate(presteps): out_row.append( result.inferences['multiStepBestPredictions'][predstep]) spamwriter.writerow(out_row) model.save(str(swarm_config_path + "/model_save"))
def __init__(self, config): # Instantiate NuPIC model model_params = base_model_params.MODEL_PARAMS # Set resolution model_params['modelParams']['sensorParams']['encoders']['value']['resolution'] = config['resolution'] # Override other Nupic parameters: model_params['modelParams'] = update_dict(model_params['modelParams'], config['nupic_model_params']) # Create model and enable inference on it self.model = ModelFactory.create(model_params) self.model.enableInference({'predictedField': 'value'}) # The shifter is used to bring the predictions to the actual time frame self.shifter = InferenceShifter() # The anomaly likelihood object self.anomalyLikelihood = AnomalyLikelihood() # Set stream source self.stream = config['stream'] # Setup class variables self.db = redis.Redis('localhost') self.seconds_per_request = config['seconds_per_request'] self.webhook = config['webhook'] self.anomaly_threshold = config['anomaly_threshold'] self.likelihood_threshold = config['likelihood_threshold'] self.domain = config['domain'] self.alert = False # Toogle when we get above threshold # Setup logging self.logger = logger or logging.getLogger(__name__) handler = logging.handlers.RotatingFileHandler(os.environ['LOG_DIR']+"/monitor_%s.log" % self.stream.name, maxBytes=1024*1024, backupCount=4, ) handler.setFormatter(logging.Formatter('[%(levelname)s/%(processName)s][%(asctime)s] %(name)s %(message)s')) handler.setLevel(logging.INFO) self.logger.addHandler(handler) self.logger.setLevel(logging.INFO) self.logger.info("=== Settings ===") self.logger.info("Webhook: %s", self.webhook) self.logger.info("Domain: %s", self.domain) self.logger.info("Seconds per request: %d", self.seconds_per_request) # Write metadata to Redis try: # Save in redis with key = 'results:monitor_id' and value = 'time, status, actual, prediction, anomaly' self.db.set('name:%s' % self.stream.id, self.stream.name) self.db.set('value_label:%s' % self.stream.id, self.stream.value_label) self.db.set('value_unit:%s' % self.stream.id, self.stream.value_unit) except Exception: self.logger.warn("Could not write results to redis.", exc_info=True)
def createModel(): model = ModelFactory.create(category_test.config) model.enableInference({ "predictedField": "category" }) # model.enableInference({ # "predictedField": "kw_energy_consumption" # }) return model
def createModel(metric): """ Fetch the model params for the specific metric and create a new HTM model :param metric: AvogadroAgent metric class :returns: An HTM Model """ return ModelFactory.create(metric.MODEL_PARAMS)
def load_model(self, modelSaveDir): try: print "Loading model from %s..." % modelSaveDir model = ModelFactory.loadFromCheckpoint(modelSaveDir) self._init_model(model) except: print "ERR", "load_model", sys.exc_info() traceback.print_exc() raise
def runAnomaly(options): """ Create and run a CLA Model on the given dataset (based on the hotgym anomaly client in NuPIC). """ # Load the model params JSON with open("model_params.json") as fp: modelParams = json.load(fp) # Update the min/max value for the encoder sensorParams = modelParams['modelParams']['sensorParams'] sensorParams['encoders']['value']['maxval'] = options.max sensorParams['encoders']['value']['minval'] = options.min model = ModelFactory.create(modelParams) model.enableInference({'predictedField': 'value'}) with open (options.inputFile) as fin: # Open file and setup headers reader = csv.reader(fin) csvWriter = csv.writer(open(options.outputFile,"wb")) csvWriter.writerow(["timestamp", "value", "anomaly_score", "likelihood_score"]) headers = reader.next() # The anomaly likelihood object anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood() # Iterate through each record in the CSV file print "Starting processing at",datetime.datetime.now() for i, record in enumerate(reader, start=1): # Convert input data to a dict so we can pass it into the model inputData = dict(zip(headers, record)) inputData["value"] = float(inputData["value"]) inputData["dttm"] = dateutil.parser.parse(inputData["dttm"]) # Send it to the CLA and get back the raw anomaly score result = model.run(inputData) anomalyScore = result.inferences['anomalyScore'] # Compute the Anomaly Likelihood likelihood = anomalyLikelihood.anomalyProbability( inputData["value"], anomalyScore, inputData["dttm"]) if likelihood > 0.9999: print "Anomaly detected:",inputData['dttm'],inputData['value'],likelihood # Write results to the output CSV file csvWriter.writerow([inputData["dttm"], inputData["value"], anomalyScore, likelihood]) # Progress report if (i%500) == 0: print i,"records processed" print "Completed processing",i,"records at",datetime.datetime.now() print "Anomaly scores for",options.inputFile, print "have been written to",options.outputFile
def createModelFromParams(self, modelParams): """ Creates a model fromt the provided model parameters file :param modelParams: :return: """ model = ModelFactory.create(modelParams) model.enableInference({"predictedField": self.fieldToPredict}) return model
def createModel(self, modelParams, predictedField): """ Creates a model. @param modelParams: () The modelParams used for running the model. @param predictedField: (string) The predictedField. """ model = ModelFactory.create(modelParams) model.enableInference({"predictedField": predictedField}) self.model = model
def createModel(seed=1956): params = model_params.MODEL_PARAMS params['modelParams']['spParams']['seed'] = seed # Randomly change encoder resolution to be in [0.78, 0.98], i.e. 0.88 +/- 0.1 params['modelParams']['sensorParams']['encoders']['consumption'][ 'resolution'] = 0.78 + numpy.random.random() * .2 return ModelFactory.create(model_params.MODEL_PARAMS)
def initialize(): global model, shifter model = ModelFactory.create(model_params.MODEL_PARAMS) model.enableInference({'predictedField': 'x'}) print "model initialized !" shifter = InferenceShifter() print "shifter initialized !"
def createModel(seed=1956): params = model_params.MODEL_PARAMS params['modelParams']['spParams']['seed'] = seed # Randomly change encoder resolution to be in [0.78, 0.98], i.e. 0.88 +/- 0.1 params['modelParams']['sensorParams']['encoders']['consumption'][ 'resolution'] = 0.78 + numpy.random.random()*.2 return ModelFactory.create(model_params.MODEL_PARAMS)