def runModel(model):
inputFilePath = "disease_person1.csv"
inputFile = open(inputFilePath,"rb")
csvReader = csv.reader(inputFile)
#skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
output = nupic_output.NuPICPlotOutput("ECG")
counter = 0
for row in csvReader:
counter += 1
if(counter % 100 == 0):
print "Read %i lines..." % counter
timestamp = datetime.datetime.strptime(row[0],DATE_FORMAT)
#timestamp = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-6]
value = int(row[1])
result = model.run({
"timestamp": timestamp,
"value": value
})
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
output.write(timestamp, value,prediction, anomalyScore)
inputFile.close()
output.close()
def runModel(model, file_name):
'''
Run predictions with a given model.
Args:
model: Opf model.
'''
input_file = open(file_name, "rb")
csv_reader = csv.reader(input_file)
csv_reader.next()
csv_reader.next()
csv_reader.next()
shifter = InferenceShifter()
output = nupic_output.NuPICPlotOutput("Sine", show_anomaly_score=True)
for row in csv_reader:
angle = float(row[0])
sine = float(row[1])
result = model.run({"angle": angle, "sine": sine})
result = shifter.shift(result)
inference = result.inferences['multiStepBestPredictions'][1]
anomaly_score = result.inferences['anomalyScore']
output.write(sine, inference, anomaly_score)
input_file.close()
output.close()
def __init__(self, model, inputs):
super(HotGym, self).__init__(model)
self.model = model
self.inputs = inputs
self.metricsManager = MetricsManager(_METRIC_SPECS,
model.getFieldInfo(),
model.getInferenceType())
self.counter = 0
self.timestampStr = ""
self.consumptionStr = ""
self.shifter = InferenceShifter()
self.dates = deque(maxlen=WINDOW)
self.convertedDates = deque(maxlen=WINDOW)
self.actualValues = deque([0.0] * WINDOW, maxlen=WINDOW)
self.predictedValues = deque([0.0] * WINDOW, maxlen=WINDOW)
self.linesInitialized = False
self.actualLines = None
self.predictedLines = None
self.shouldScheduleDraw = True
fig = plt.figure(figsize=(6.1, 11))
plotCount = 1
gs = gridspec.GridSpec(plotCount, 1)
self.graph = fig.add_subplot(gs[0, 0])
plt.xlabel('Date')
plt.ylabel('Consumption (kW)')
plt.tight_layout()
def __init__(self, numPredictions, resultsDir):
random.seed(43)
self.numPredictions = numPredictions
if not os.path.exists(resultsDir):
os.makedirs(resultsDir)
self.resultsFile = open(os.path.join(resultsDir, "0.log"), 'w')
self.model = ModelFactory.create(MODEL_PARAMS)
self.model.enableInference({"predictedField": "element"})
self.shifter = InferenceShifter()
self.mapping = getEncoderMapping(self.model)
self.correct = []
self.numPredictedActiveCells = []
self.numPredictedInactiveCells = []
self.numUnpredictedActiveColumns = []
self.iteration = 0
self.perturbed = False
self.randoms = []
self.verbosity = 1
self.dataset = HighOrderDataset(numPredictions=self.numPredictions)
self.sequences = []
self.currentSequence = []
self.replenish_sequence()
def runModel(model):
inputFilePath = "disease_person1.csv"
inputFile = open(inputFilePath, "rb")
csvReader = csv.reader(inputFile)
#skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
output = nupic_output.NuPICPlotOutput("ECG")
counter = 0
for row in csvReader:
counter += 1
if (counter % 100 == 0):
print "Read %i lines..." % counter
timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
#timestamp = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-6]
value = int(row[1])
result = model.run({"timestamp": timestamp, "value": value})
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
output.write(timestamp, value, prediction, anomalyScore)
inputFile.close()
output.close()
def predict(self, model):
# model input data is
cla_model_input=dict()
cla_model_input["rssi"] = float(model.rssi_raw)
cla_model_input["quality"] = float(model.quality_raw)
if not self.cla_model:
# Create the model for predicting WiFi usage
# The shifter will align prediction and actual values.
s1 = int(round(time.time() * 1000))
self.cla_model = ModelFactory.create(model_params.MODEL_PARAMS)
self.cla_model.enableInference({'predictedField': 'rssi'})
self.shifter = InferenceShifter()
s2 = int(round(time.time() * 1000))
#print "Predictor construction %s,%d,%d,%d" % (self, s1, s2, abs(s2-s1))
# Shift 'x' time periods into the future
# Compare 'current' with 'predicted - pos(x)' - how do we compare
# The point is to be able to Generalize on patterns that it hasn't seen yet
result = self.shifter.shift(self.cla_model.run(cla_model_input))
inference = result.inferences['multiStepBestPredictions'][self.timestep]
# Store away current rssi information on the model
if inference is not None:
model.rssi_predicted = int(inference)
model.rssi_delta = abs(model.rssi_raw - model.rssi_predicted)
model.zone_predicted.assign_rssi_zone(model.rssi_predicted)
def runModel(model, data, field, plot, logLikelihood):
fieldIndex = data.headers().index(field)
datetimeIndex = data.headers().index(DATETIME_FIELDNAME)
shifter = InferenceShifter()
if plot:
output = nupic_anomaly_output.NuPICPlotOutput(field, logLikelihood)
else:
output = nupic_anomaly_output.NuPICFileOutput(field, logLikelihood)
for dataPoint in data.data():
dateString = dataPoint[datetimeIndex]
timestamp = datetime.datetime.strptime(dateString, DATE_FORMAT)
value = dataPoint[fieldIndex]
if value is not None:
result = model.run({
"timestamp": timestamp,
"value": value
})
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
output.write(timestamp, value, prediction, anomalyScore)
output.close()
def runIoThroughNupic(inputPath, model, modelName, plot=False):
with open(inputPath, "rb") as inputFile:
csvReader = csv.reader(inputFile)
# skip header rows
headers = csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
if plot:
output = nupic_output.NuPICPlotOutput(modelName)
else:
output = nupic_output.NuPICFileOutput(modelName, path="data")
counter = 0
for row in csvReader:
assert len(row) == len(headers)
counter += 1
if (counter % 100 == 0):
print "Read %i lines..." % counter
row = [float(row[0])] + [int(val) for val in row[1:]]
input_row = dict(zip(headers, row))
result = model.run(input_row)
if plot:
seconds = input_row["seconds"]
actual = input_row[PREDICTED_BUCKET]
shifter.shift(result)
predicted = result.inferences["multiStepBestPredictions"][1]
output.write([seconds], [actual], [predicted])
else:
output.write(input_row, result)
output.close()
def runIoThroughNupic(inputData, model, gymName, plot):
inputFile = open(inputData, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
if plot:
output = nupic_output.NuPICPlotOutput([gymName])
else:
output = nupic_output.NuPICFileOutput([gymName])
counter = 0
for row in csvReader:
counter += 1
if (counter % 100 == 0):
print "Read %i lines..." % counter
timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
consumption = float(row[1])
result = model.run({
"timestamp": timestamp,
"kw_energy_consumption": consumption
})
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
output.write([timestamp], [consumption], [prediction])
inputFile.close()
output.close()
def runIoThroughNupic(inputData, model, gymName, plot, load):
"""
Handles looping over the input data and passing each row into the given model
object, as well as extracting the result object and passing it into an output
handler.
:param inputData: file path to input data CSV
:param model: OPF Model object
:param gymName: Gym name, used for output handler naming
:param plot: Whether to use matplotlib or not. If false, uses file output.
"""
inputFile = open(inputData, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
if plot:
output = nupic_anomaly_output.NuPICPlotOutput(gymName)
else:
output = nupic_anomaly_output.NuPICFileOutput(gymName)
counter = 0
# using dummy time to debug
timestamp = datetime.datetime.strptime(csvReader.next()[0], DATE_FORMAT)
print("DEBUG_PRINT: initiali time", timestamp)
for row in csvReader:
counter += 1
if (counter % 100 == 0):
print "Read %i lines..." % counter
timestamp = timestamp + datetime.timedelta(microseconds=10000)
#timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
consumption = float(row[2])
rawValue = float(row[1])
result = model.run({
"timestamp": timestamp,
"wavelet_value": consumption
})
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
output.write(timestamp, consumption, prediction, anomalyScore,
rawValue)
if not load:
print("saving model for MODEL_DIR")
model.save(MODEL_DIR)
inputFile.close()
output.close()
return model
def runHarddriveAnomaly(plot):
shifter = InferenceShifter()
if plot:
output = nupic_anomaly_output.NuPICPlotOutput('Harddrive Learning')
else:
output = nupic_anomaly_output.NuPICFileOutput('Harddrive Learning')
_LOGGER.info('start with anomaly detection...')
model = getModel('good')
model.enableInference({'predictedField': 'class'})
_LOGGER.info('read data file...')
data = pd.read_csv('harddrive-smart-data-test.csv')
dictionary = data.transpose().to_dict().values()
for row in dictionary:
csvWriter = csv.writer(open(_OUTPUT_PATH, "wa"))
csvWriter.writerow(["class", "anomaly_score"])
result = model.run(row)
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences['anomalyScore']
output.write('', result.rawInput["class"], prediction, anomalyScore)
csvWriter.writerow([row["class"], anomalyScore])
if anomalyScore > _ANOMALY_THRESHOLD:
_LOGGER.info("Anomaly detected at [%s]. Anomaly score: %f.",
result.rawInput["class"], anomalyScore)
def runIoThroughNupic(inputPath, model, modelName, plot=False):
with open(inputPath, "rb") as inputFile:
csvReader = csv.reader(inputFile)
# skip header rows
headers = csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
if plot:
output = nupic_output.NuPICPlotOutput(modelName)
else:
output = nupic_output.NuPICFileOutput(modelName, path="data")
counter = 0
for row in csvReader:
assert len(row) == len(headers)
counter += 1
if (counter % 100 == 0):
print "Read %i lines..." % counter
row = [float(row[0])] + [int(val) for val in row[1:]]
input_row = dict(zip(headers, row))
result = model.run(input_row)
if plot:
seconds = input_row["seconds"]
actual = input_row[PREDICTED_BUCKET]
shifter.shift(result)
predicted = result.inferences["multiStepBestPredictions"][1]
output.write([seconds], [actual], [predicted])
else:
output.write(input_row, result)
output.close()
def runHarddriveAnomaly(plot):
shifter = InferenceShifter()
if plot:
output = nupic_anomaly_output.NuPICPlotOutput("Harddrive Learning")
else:
output = nupic_anomaly_output.NuPICFileOutput("Harddrive Learning")
_LOGGER.info("start with anomaly detection...")
model = getModel("good")
model.enableInference({"predictedField": "class"})
_LOGGER.info("read data file...")
data = pd.read_csv("harddrive-smart-data-test.csv")
dictionary = data.transpose().to_dict().values()
for row in dictionary:
csvWriter = csv.writer(open(_OUTPUT_PATH, "wa"))
csvWriter.writerow(["class", "anomaly_score"])
result = model.run(row)
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
output.write("", result.rawInput["class"], prediction, anomalyScore)
csvWriter.writerow([row["class"], anomalyScore])
if anomalyScore > _ANOMALY_THRESHOLD:
_LOGGER.info("Anomaly detected at [%s]. Anomaly score: %f.", result.rawInput["class"], anomalyScore)
def runModel(model, inputFilePath):
inputFile = open(inputFilePath, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
output = nupic_anomaly_output.NuPICFileOutput("weight_output")
shifter = InferenceShifter()
counter = 0
for row in csvReader:
counter += 1
if(counter % 20 == 0):
print "Read %i lines..." % counter
weight_date = datetime.datetime.strptime("2013-01-08 12:00:00", DATE_FORMAT)
weight = row[2]
result = model.run({
"weight": weight,
"weight_date": weight_date
})
result = shifter.shift(result)
prediction = str(result.inferences["multiStepBestPredictions"][1])
anomalyScore = int(result.inferences["anomalyScore"])
output.write(weight_date, weight, prediction, anomalyScore)
def __init__(self, metricSpecs, fieldInfo, inferenceType):
"""
Constructs a Metrics Manager
Parameters:
-----------------------------------------------------------------------
metricSpecs: A sequence of MetricSpecs that specify which metrics should
be calculated
inferenceType: An opf_utils.inferenceType value that specifies the inference
type of the associated model. This affects how metrics are
calculated. FOR EXAMPLE, temporal models save the inference
from the previous timestep to match it to the ground truth
value in the current timestep
"""
self.__metricSpecs = []
self.__metrics = []
self.__metricLabels = []
# Maps field names to indices. Useful for looking up input/predictions by
# field name
self.__fieldNameIndexMap = dict( [(info.name, i) \
for i, info in enumerate(fieldInfo)] )
self.__constructMetricsModules(metricSpecs)
self.__currentGroundTruth = None
self.__currentInference = None
self.__currentResult = None
self.__isTemporal = InferenceType.isTemporal(inferenceType)
if self.__isTemporal:
self.__inferenceShifter = InferenceShifter()
def __init__(self, *args, **kwargs):
super(NuPICPlotOutput, self).__init__(*args, **kwargs)
# turn matplotlib interactive mode on (ion)
plt.ion()
plt.figure(figsize=(14, 10))
gs = gridspec.GridSpec(2, 1, height_ratios=[3,1])
# plot title, legend, etc
plt.title('Sine prediction example')
plt.ylabel('Sine (rad)')
# The shifter will align prediction and actual values.
self.shifter = InferenceShifter()
# Keep the last WINDOW predicted and actual values for plotting.
self.actual_history = deque([0.0] * WINDOW, maxlen=360)
self.predicted_history = deque([0.0] * WINDOW, maxlen=360)
if self.show_anomaly_score:
self.anomaly_score = deque([0.0] * WINDOW, maxlen=360)
# Initialize the plot lines that we will update with each new record.
if self.show_anomaly_score:
plt.subplot(gs[0])
self.actual_line, = plt.plot(range(WINDOW), self.actual_history)
self.predicted_line, = plt.plot(range(WINDOW), self.predicted_history)
plt.legend(tuple(['actual','predicted']), loc=3)
if self.show_anomaly_score:
plt.subplot(gs[1])
self.anomaly_score_line, = plt.plot(range(WINDOW), self.anomaly_score, 'r-')
plt.legend(tuple(['anomaly score']), loc=3)
# Set the y-axis range.
self.actual_line.axes.set_ylim(-1, 1)
self.predicted_line.axes.set_ylim(-1, 1)
if self.show_anomaly_score:
self.anomaly_score_line.axes.set_ylim(-1, 1)
def runModel(model):
inputFile = open("rec-center-hourly.csv", "rb")
csvReader = csv.reader(inputFile)
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
output = nupic_output.NuPICPlotOutput(['REC center'])
counter = 0
for row in csvReader:
counter += 1
if (counter % 100 == 0):
print("Read %i lines", counter)
timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
consumption = float(row[1])
result = model.run({
"timestamp": timestamp,
"kw_energy_consumption": consumption
})
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
output.write([timestamp], [consumption], [prediction])
inputFile.close()
output.close()
def runModel(model):
inputFilePath = "./rec-center-hourly.csv"
inputFile = open(inputFilePath, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
#output = nupic_output.NuPICFileOutput(["Rec Center"])
output = nupic_output.NuPICPlotOutput(["Rec Center"])
counter = 0
for row in csvReader:
counter += 1
if (counter % 100 == 0) :
print "Read %i lines..." % counter
timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
consumption = float(row[1])
result = model.run({
"timestamp": timestamp,
"kw_energy_consumption": consumption
})
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
output.write([timestamp], [consumption], [prediction])
inputFile.close()
output.close()
def runIoThroughNupic(inputData, model, gymName, plot):
inputFile = open(inputData, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
if plot:
output = nupic_anomaly_output.NuPICPlotOutput(gymName)
else:
output = nupic_anomaly_output.NuPICFileOutput(gymName)
counter = 0
for row in csvReader:
counter += 1
if (counter % 100 == 0):
print "Read %i lines..." % counter
timestamp = dateutil.parser.parse(row[0])
ecg = float(row[1])
result = model.run({
"timestamp": timestamp,
"ECG1": ecg
})
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
output.write(timestamp, ecg, prediction, anomalyScore)
inputFile.close()
output.close()
def runIoThroughNupic(inputData, model, roadName, plot):
inputFile = open(inputData, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
if plot:
output = nupic_output.NuPICPlotOutput([roadName])
else:
output = nupic_output.NuPICFileOutput([roadName])
counter = 0
for row in csvReader:
counter += 1
if (counter % 100 == 0):
print "Read %i lines..." % counter
timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
traffic_count = float(row[1])
result = model.run({
"timestamp": timestamp,
"hourly_traffic_count": traffic_count
})
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
output.write([timestamp], [traffic_count], [prediction])
inputFile.close()
output.close()
def runIoThroughNupic(inputData, model, gymName, plot, load):
"""
Handles looping over the input data and passing each row into the given model
object, as well as extracting the result object and passing it into an output
handler.
:param inputData: file path to input data CSV
:param model: OPF Model object
:param gymName: Gym name, used for output handler naming
:param plot: Whether to use matplotlib or not. If false, uses file output.
"""
inputFile = open(inputData, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
if plot:
output = nupic_anomaly_output.NuPICPlotOutput(gymName)
else:
output = nupic_anomaly_output.NuPICFileOutput(gymName)
counter = 0
# using dummy time to debug
timestamp = datetime.datetime.strptime(csvReader.next()[0], DATE_FORMAT)
print("DEBUG_PRINT: initiali time", timestamp)
for row in csvReader:
counter += 1
if (counter % 100 == 0):
print "Read %i lines..." % counter
timestamp = timestamp + datetime.timedelta(microseconds=10000)
#timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
consumption = float(row[2])
rawValue = float(row[1])
result = model.run({
"timestamp": timestamp,
"wavelet_value": consumption
})
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
output.write(timestamp, consumption, prediction, anomalyScore, rawValue)
if not load:
print("saving model for MODEL_DIR")
model.save(MODEL_DIR)
inputFile.close()
output.close()
return model
def testDumpsObject(self):
testClass = InferenceShifter()
testClass.a = 5
testClass.b = {'b': (17,)}
encoded = json.dumps(testClass, sort_keys=True)
self.assertEqual(
encoded,
'{"_inferenceBuffer": null, "a": 5, "b": {"b": {"py/tuple": [17]}}, '
'"py/object": "nupic.data.inference_shifter.InferenceShifter"}')
def run_io_through_nupic(input_data, model, file_name, plot, print_results):
input_file = open(input_data, "rb")
csv_reader = csv.reader(input_file)
# skip header rows
csv_reader.next()
csv_reader.next()
csv_reader.next()
shifter = InferenceShifter()
if plot:
output = nupic_output.NuPICPlotOutput([file_name])
else:
output = nupic_output.NuPICFileOutput([file_name])
metrics_manager = MetricsManager(_METRIC_SPECS, model.getFieldInfo(),
model.getInferenceType())
counter = 0
timestamp = None
consumption = None
result = None
for row in csv_reader:
counter += 1
timestamp = datetime.datetime.strptime(row[1], DATE_FORMAT)
consumption = int(row[2])
amount = float(row[0])
result = model.run({
"amount": amount,
"date": timestamp,
"tte": consumption
})
result.metrics = metrics_manager.update(result)
if counter % 100 == 0 or counter % 384 == 0:
print "Read %i lines..." % counter
print ("After %i records, rmse=%f" % (counter,
result.metrics["multiStepBestPredictions:multiStep:"
"errorMetric='rmse':steps=1:window=1000:"
"field=tte"]))
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
output.write([timestamp], [consumption], [prediction])
if print_results:
print("date:", timestamp.strftime("%y-%m-%d"), "actual:", consumption, "predicted:", prediction)
if plot and counter % 20 == 0:
output.refresh_gui()
#if plot and counter % 1000 == 0:
# break
input_file.close()
output.close()
def testDumpsObject(self):
testClass = InferenceShifter()
testClass.a = 5
testClass.b = {'b': (17, )}
encoded = json.dumps(testClass, sort_keys=True)
self.assertEqual(
encoded,
'{"_inferenceBuffer": null, "a": 5, "b": {"b": {"py/tuple": [17]}}, '
'"py/object": "nupic.data.inference_shifter.InferenceShifter"}')
def runDataThroughNupic(MODELS, anomaly_helper, inputData, systemName):
ANOMALY_OBJ = nupic_output.NuPICFileOutput(systemName)
ANOMALY_OBJ.anomalyLikelihoodHelper = anomaly_helper
ANOMALY_LIKELIHOOD = [0.0 for i in range(len(MODEL_NAMES))]
inputFile = open(inputData, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
counter = 0
metricsManager = [0 for i in range(len(MODELS))]
for row in csvReader:
counter += 1
timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
for model_index in range(len(MODELS)):
data = float(row[model_index + 1])
inference_type = MODEL_NAMES[model_index]
result = MODELS[model_index].run({
"timestamp": timestamp,
inference_type: data
})
if counter % 20 == 0:
print(str(counter) + ":" + inference_type)
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
anomalyLikelihood = ANOMALY_OBJ.get_anomaly_likelihood(
timestamp, data, prediction, anomalyScore)
ANOMALY_LIKELIHOOD[model_index] = anomalyLikelihood
ANOMALY_OBJ.write(timestamp, ANOMALY_LIKELIHOOD)
inputFile.close()
ANOMALY_OBJ.close()
print("Saving Anomaly Object")
path = os.path.join(os.getcwd(), "objects/")
if os.path.isdir(path) is False:
os.mkdir('objects')
with open('objects/anomaly_object.pkl', 'wb') as o:
pickle.dump(ANOMALY_OBJ.anomalyLikelihoodHelper, o)
def __init__(self, writer):
"""
writer: Non-temporal prediction log writer conforming to
PredictionWriterIface interface.
"""
self.__logger = logging.getLogger(".".join(
['com.numenta', self.__class__.__module__, self.__class__.__name__]))
self.__writer = writer
self.__inferenceShifter = InferenceShifter()
return
def runModel(model, csv_path, outputCSVFile, outputPlotFile):
"""
Runs HTM model with input data
:param model : input HTM model
:param csv_path : path to csv dataset file
:param outputCSVFile : output csv file
:param outputPlotFile: output plot file
"""
# get input csv file and read it
inputFilePath = csv_path
inputFile = open(inputFilePath, "rb")
csvReader = csv.reader(inputFile)
# skip first 3 header rows
csvReader.next()
csvReader.next()
csvReader.next()
# plot prediction
shifter = InferenceShifter()
# loop through data
counter = 0
for row in csvReader:
counter += 1
# print after every 100 iterations
if (counter % 100 == 0):
print("Read %i lines..." % counter)
timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
value = float(row[1])
result = model.run({"timestamp": timestamp, "value": value})
# plotting prediction and anomaly detection
plot_result = shifter.shift(result)
plot_prediction = plot_result.inferences["multiStepBestPredictions"][1]
plot_anomalyScore = plot_result.inferences["anomalyScore"]
outputPlotFile.write(timestamp, value, plot_prediction,
plot_anomalyScore)
# output csv for anomaly detection
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
outputCSVFile.write(timestamp, value, prediction, anomalyScore)
# close all files after usage
inputFile.close()
outputCSVFile.close()
outputPlotFile.close()
return result
def RunPOSPrediction():
# Create the model for predicting POS.
model = ModelFactory.create(model_params.MODEL_PARAMS)
model.enableInference({'predictedField': 'pos'})
# The shifter will align prediction and actual values.
shifter = InferenceShifter()
# Keep the last WINDOW predicted and actual values for plotting.
#actHistory = deque([0.0] * WINDOW, maxlen=60)
#predHistory = deque([0.0] * WINDOW, maxlen=60)
# Initialize the plot lines that we will update with each new record.
#actline, = plt.plot(range(WINDOW), actHistory)
#predline, = plt.plot(range(WINDOW), predHistory)
# Set the y-axis range.
#actline.axes.set_ylim(0, 100)
#predline.axes.set_ylim(0, 100)
corpus = nltk.Text(word.lower() for word in brown.words())
part_of_speech_list = nltk.pos_tag(corpus)
for i, word_pos in enumerate(part_of_speech_list):
s = time.time()
# Get the CPU usage.
#cpu = psutil.cpu_percent()
word, pos = word_pos
if i == len(part_of_speech_list) - 1:
break
next_word, next_pos = part_of_speech_list[i + 1]
# Run the input through the model and shift the resulting prediction.
modelInput = {'pos': pos}
result = shifter.shift(model.run(modelInput))
# Update the trailing predicted and actual value deques.
inference = result.inferences['multiStepBestPredictions'][5]
if inference is not None:
print("Actual: %s/%s, Predicted: %s/%s" % (next_word, result.rawInput['pos'], "?", inference))
#if inference is not None:
# actHistory.append(result.rawInput['cpu'])
# predHistory.append(inference)
# Redraw the chart with the new data.
#actline.set_ydata(actHistory) # update the data
#predline.set_ydata(predHistory) # update the data
#plt.draw()
#plt.legend( ('actual','predicted') )
sleep(1)
def runIoThroughNupic(ip, port, model, metric_name, plot):
"""
Handles looping over the input data and passing each row into the given model
object, as well as extracting the result object and passing it into an output
handler.
:param ip: IP address of the muse server
:param port: port of the muse server
:param model: OPF Model object
:param metric_name: metric name, used for output handler naming
:param plot: Whether to use matplotlib or not. If false, uses file output.
"""
client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
client.bind((ip, port))
shifter = InferenceShifter()
if plot:
output = nupic_anomaly_output.NuPICPlotOutput(metric_name)
else:
output = nupic_anomaly_output.NuPICFileOutput(metric_name)
counter = 0
while 1:
counter += 1
if (counter % 100 == 0):
print "Read %i lines..." % counter
data = json.loads(client.recv(1024))
print data
# data in nano-volts (10E9)
metric_value = float(data['channel_values'][0] * 10E8)
result = model.run({
"metric_value": metric_value
})
#print metric_value
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
# TODO: timestamp data BEFORE sending it.
t = datetime.datetime.now().strftime(DATE_FORMAT)
timestamp = datetime.datetime.strptime(t, DATE_FORMAT)
# write output
output.write(timestamp, metric_value, prediction, anomalyScore)
output.close()
def runIoThroughNupic(inputData, model, metric, sensor, patientId, plot):
"""
Handles looping over the input data and passing each row into the given model
object, as well as extracting the result object and passing it into an output
handler.
:param inputData: file path to input data CSV
:param model: OPF Model object
:param csvName: CSV name, used for output handler naming
:param plot: Whether to use matplotlib or not. If false, uses file output.
"""
inputFile = open(inputData, "rb")
csvReader = csv.reader(inputFile.read().splitlines())
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
csvName = "%s_%s_%s" % (metric, sensor, patientId)
print "running model with model_params '%s'" % csvName
shifter = InferenceShifter()
if plot:
output = nupic_anomaly_output.NuPICPlotOutput(csvName)
else:
if not os.path.exists(MODEL_RESULTS_DIR):
os.makedirs(MODEL_RESULTS_DIR)
output = nupic_anomaly_output.NuPICFileOutput("%s/%s" % (MODEL_RESULTS_DIR,
csvName))
counter = 0
for row in csvReader:
counter += 1
if (counter % 100 == 0):
print "Read %i lines..." % counter
metric_value = float(row[0])
result = model.run({
"metric_value": metric_value
})
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][0]
anomalyScore = result.inferences["anomalyScore"]
output.write(counter, metric_value, prediction, anomalyScore)
output.close()
inputFile.close()
def runIoThroughNupic(inputData, model, metric, sensor, patientId, plot):
"""
Handles looping over the input data and passing each row into the given model
object, as well as extracting the result object and passing it into an output
handler.
:param inputData: file path to input data CSV
:param model: OPF Model object
:param csvName: CSV name, used for output handler naming
:param plot: Whether to use matplotlib or not. If false, uses file output.
"""
inputFile = open(inputData, "rb")
csvReader = csv.reader(inputFile.read().splitlines())
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
csvName = "%s_%s_%s" % (metric, sensor, patientId)
print "running model with model_params '%s'" % csvName
shifter = InferenceShifter()
if plot:
output = nupic_anomaly_output.NuPICPlotOutput(csvName)
else:
if not os.path.exists(MODEL_RESULTS_DIR):
os.makedirs(MODEL_RESULTS_DIR)
output = nupic_anomaly_output.NuPICFileOutput("%s/%s" % (MODEL_RESULTS_DIR,
csvName))
counter = 0
for row in csvReader:
counter += 1
if (counter % 100 == 0):
print "Read %i lines..." % counter
metric_value = float(row[0])
result = model.run({
"metric_value": metric_value
})
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][0]
anomalyScore = result.inferences["anomalyScore"]
output.write(counter, metric_value, prediction, anomalyScore)
output.close()
inputFile.close()
def runModel(model, inputFilePath):
inputFile = open(inputFilePath, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
output = nupic_anomaly_output.NuPICFileOutput("disease_forecast")
shifter = InferenceShifter()
counter = 0
actualCount = 0
predictCount = 0
miss = 0
hit = 0
row_count = 0
for row in csvReader:
if row_count > 1000:
break
counter += 1
if (counter % 10 == 0):
print "Read %i lines..." % counter
disease_date = datetime.datetime.strptime(row[5], DATE_FORMAT)
disease_name = str(row[2])
result = model.run({
"disease_name": disease_name,
"disease_date": disease_date
})
result = shifter.shift(result)
prediction = str(result.inferences["multiStepBestPredictions"][20])
anomalyScore = int(result.inferences["anomalyScore"])
output.write(disease_date, disease_name, prediction, anomalyScore)
if prediction == "Malaria":
predictCount += 1
if disease_name == "Malaria" and prediction != None:
actualCount += 1
if prediction != None:
if disease_name == prediction:
hit += 1
else:
miss += 1
row_count += 1
print counter, row[
0], "Actual: ", disease_name, "Predicted: ", prediction, "------", anomalyScore
print "\n Number of actuals: ", actualCount, " \n Number of predictions: ", predictCount
print "\n hits: ", hit, "\n misses: ", miss - 20
def runDataThroughModel(model, dataFrame):
shifter = InferenceShifter()
anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood()
out = []
for index, row in dataFrame.iterrows():
timestamp = datetime.strptime(row["timestamp"], DATE_FORMAT)
value = int(row["value"])
result = model.run({"timestamp": timestamp, "value": value})
if index % 100 == 0:
print "Read %i lines..." % index
result = shifter.shift(result)
resultOut = convertToWritableOutput(result, anomalyLikelihood)
out.append(resultOut)
return pd.DataFrame(out)
def runModel(model, inputFilePath):
inputFile = open(inputFilePath, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
output = nupic_anomaly_output.NuPICFileOutput("disease_forecast")
shifter = InferenceShifter()
counter = 0
actualCount = 0
predictCount = 0
miss = 0
hit = 0
row_count = 0
for row in csvReader:
if row_count > 1000:
break
counter += 1
if(counter % 10 == 0):
print "Read %i lines..." % counter
disease_date = datetime.datetime.strptime(row[5], DATE_FORMAT)
disease_name = str(row[2])
result = model.run({
"disease_name": disease_name,
"disease_date": disease_date
})
result = shifter.shift(result)
prediction = str(result.inferences["multiStepBestPredictions"][20])
anomalyScore = int(result.inferences["anomalyScore"])
output.write(disease_date, disease_name, prediction, anomalyScore)
if prediction == "Malaria":
predictCount += 1
if disease_name == "Malaria" and prediction != None:
actualCount += 1
if prediction != None:
if disease_name == prediction:
hit += 1
else:
miss += 1
row_count += 1
print counter, row[0], "Actual: ", disease_name, "Predicted: ", prediction, "------", anomalyScore
print"\n Number of actuals: ", actualCount," \n Number of predictions: ", predictCount
print "\n hits: ", hit,"\n misses: ", miss-20
def run():
"""Poll CPU usage, make predictions, and plot the results. Runs forever."""
# Create the model for predicting CPU usage.
model = ModelFactory.create(model_params.MODEL_PARAMS)
model.enableInference({'predictedField': 'x'})
# The shifter will align prediction and actual values.
shifter = InferenceShifter()
# Keep the last WINDOW predicted and actual values for plotting.
actHistory = deque([0.0] * WINDOW, maxlen=60)
predHistory = deque([0.0] * WINDOW, maxlen=60)
# Initialize the plot lines that we will update with each new record.
actline, = plt.plot(range(WINDOW), actHistory)
predline, = plt.plot(range(WINDOW), predHistory)
# Set the y-axis range.
actline.axes.set_ylim(0, 100)
predline.axes.set_ylim(0, 100)
while True:
s = time.time()
# Get the CPU usage.
#cpu = psutil.cpu_percent()
x = s_gen.next()
y = s_gen.next()
# Run the input through the model and shift the resulting prediction.
modelInput = {'x': x,'y':y}
result = shifter.shift(model.run(modelInput))
# Update the trailing predicted and actual value deques.
inference = result.inferences['multiStepBestPredictions'][5]
if inference is not None:
actHistory.append(result.rawInput['x'])
predHistory.append(inference)
# Redraw the chart with the new data.
actline.set_ydata(actHistory) # update the data
predline.set_ydata(predHistory) # update the data
plt.draw()
plt.legend( ('actual','predicted') )
# Make sure we wait a total of 2 seconds per iteration.
try:
plt.pause(SECONDS_PER_STEP)
except:
pass
def runModel(model):
inputFilePath = "datasets/rec-center-hourly.csv"
inputFile = open(inputFilePath, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
#output = nupic_output.NuPICFileOutput(["Rec Center"])
#output = nupic_output.NuPICPlotOutput(["Rec Center"])
counter = 0
for row in csvReader:
counter += 1
if (counter % 100 == 0) :
print "Read %i lines..." % counter
if counter > 900:
#model.disableLearning()
timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
consumption = float(row[1])
category = float(row[2])
result = model.run({
"timestamp": timestamp,
"kw_energy_consumption": consumption,
"category": 0.0,
"_learning": False
})
#print model.isLearningEnabled()
else:
timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
consumption = float(row[1])
category = float(row[2])
result = model.run({
"timestamp": timestamp,
"kw_energy_consumption": consumption,
"category": category,
"_learning": True
})
result = shifter.shift(result)
#print timestamp, consumption, category, result.inferences['multiStepBestPredictions']
print timestamp, category, result.inferences['multiStepBestPredictions'][1]
#output.write([timestamp], [consumption], [prediction])
inputFile.close()
def runIoThroughNupic(inputData, model, name, plot):
inputFile = open(inputData, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
if len(plot) == 0:
for field in SWARM_DESCRIPTION["includedFields"]:
plot.append(field["fieldName"])
output = nupic_output.NuPICFileOutput(name, plot)
else:
output = nupic_output.NuPICPlotOutput(name, plot)
metricsManager = MetricsManager(_METRIC_SPECS, model.getFieldInfo(),
model.getInferenceType())
counter = 0
for row in csvReader:
counter += 1
data = {}
fldCounter = 0
for field in SWARM_DESCRIPTION["includedFields"]:
data[field["fieldName"]] = translate_data(field["fieldType"], row[fldCounter])
fldCounter += 1
result = model.run(data)
result.metrics = metricsManager.update(result)
if options.verbose is not None and counter % 100 == 0:
print "Read %i lines..." % counter
print ("After %i records, 1-step altMAPE=%f" % (counter,
result.metrics["multiStepBestPredictions:multiStep:"
"errorMetric='altMAPE':steps=1:window=1000:"
"field="+PREDICTED_FIELD]))
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
vals = []
for field in plot:
vals.append(data[field])
output.write(vals, prediction)
inputFile.close()
output.close()
def __init__(self, model, inputs):
super(HotGym, self).__init__(model)
self.model = model
self.inputs = inputs
self.metricsManager = MetricsManager(_METRIC_SPECS, model.getFieldInfo(),
model.getInferenceType())
self.counter = 0
self.timestampStr = ""
self.consumptionStr = ""
self.shifter = InferenceShifter()
self.dates = deque(maxlen=WINDOW)
self.convertedDates = deque(maxlen=WINDOW)
self.actualValues = deque([0.0] * WINDOW, maxlen=WINDOW)
self.predictedValues = deque([0.0] * WINDOW, maxlen=WINDOW)
self.linesInitialized = False
self.actualLines = None
self.predictedLines = None
self.shouldScheduleDraw = True
fig = plt.figure(figsize=(6.1, 11))
plotCount = 1
gs = gridspec.GridSpec(plotCount, 1)
self.graph = fig.add_subplot(gs[0, 0])
plt.xlabel('Date')
plt.ylabel('Consumption (kW)')
plt.tight_layout()
def __init__(self, numPredictions, resultsDir):
random.seed(43)
self.numPredictions = numPredictions
if not os.path.exists(resultsDir):
os.makedirs(resultsDir)
self.resultsFile = open(os.path.join(resultsDir, "0.log"), 'w')
self.model = ModelFactory.create(MODEL_PARAMS)
self.model.enableInference({"predictedField": "element"})
self.shifter = InferenceShifter()
self.mapping = getEncoderMapping(self.model)
self.correct = []
self.numPredictedActiveCells = []
self.numPredictedInactiveCells = []
self.numUnpredictedActiveColumns = []
self.iteration = 0
self.perturbed = False
self.randoms = []
self.verbosity = 1
self.dataset = HighOrderDataset(numPredictions=self.numPredictions)
self.sequences = []
self.currentSequence = []
self.replenish_sequence()
def reset(self, params, repetition):
random.seed(params['seed'])
if params['dataset'] == 'simple':
self.dataset = SimpleDataset()
elif params['dataset'] == 'reber':
self.dataset = ReberDataset(maxLength=params['max_length'])
elif params['dataset'] == 'high-order':
self.dataset = HighOrderDataset(numPredictions=params['num_predictions'])
else:
raise Exception("Dataset not found")
# if not os.path.exists(resultsDir):
# os.makedirs(resultsDir)
# self.resultsFile = open(os.path.join(resultsDir, "0.log"), 'w')
if params['verbosity'] > 0:
print " initializing HTM model..."
self.model = ModelFactory.create(MODEL_PARAMS)
self.model.enableInference({"predictedField": "element"})
self.shifter = InferenceShifter()
self.mapping = getEncoderMapping(self.model)
self.currentSequence = self.dataset.generateSequence()
self.numPredictedActiveCells = []
self.numPredictedInactiveCells = []
self.numUnpredictedActiveColumns = []
self.currentSequence = self.dataset.generateSequence()
self.perturbed = False
self.randoms = []
self.verbosity = 1
self.sequenceCounter = 0
def __init__(self, metricSpecs, fieldInfo, inferenceType):
"""
Constructs a Metrics Manager
Parameters:
-----------------------------------------------------------------------
metricSpecs: A sequence of MetricSpecs that specify which metrics should
be calculated
inferenceType: An opfutils.inferenceType value that specifies the inference
type of the associated model. This affects how metrics are
calculated. FOR EXAMPLE, temporal models save the inference
from the previous timestep to match it to the ground truth
value in the current timestep
"""
self.__metricSpecs = []
self.__metrics = []
self.__metricLabels = []
# Maps field names to indices. Useful for looking up input/predictions by
# field name
self.__fieldNameIndexMap = dict( [(info.name, i) \
for i, info in enumerate(fieldInfo)] )
self.__constructMetricsModules(metricSpecs)
self.__currentGroundTruth = None
self.__currentInference = None
self.__isTemporal = InferenceType.isTemporal(inferenceType)
if self.__isTemporal:
self.__inferenceShifter = InferenceShifter()
def __init__(self, *args, **kwargs):
super(NuPICPlotOutput, self).__init__(*args, **kwargs)
# turn matplotlib interactive mode on (ion)
plt.ion()
plt.figure(figsize=(14, 10))
gs = gridspec.GridSpec(2, 1, height_ratios=[3,1])
# plot title, legend, etc
plt.title('Sine prediction example')
plt.ylabel('Sine (rad)')
# The shifter will align prediction and actual values.
self.shifter = InferenceShifter()
# Keep the last WINDOW predicted and actual values for plotting.
self.actual_history = deque([0.0] * WINDOW, maxlen=360)
self.predicted_history = deque([0.0] * WINDOW, maxlen=360)
if self.show_anomaly_score:
self.anomaly_score = deque([0.0] * WINDOW, maxlen=360)
# Initialize the plot lines that we will update with each new record.
if self.show_anomaly_score:
plt.subplot(gs[0])
self.actual_line, = plt.plot(range(WINDOW), self.actual_history)
self.predicted_line, = plt.plot(range(WINDOW), self.predicted_history)
plt.legend(tuple(['actual','predicted']), loc=3)
if self.show_anomaly_score:
plt.subplot(gs[1])
self.anomaly_score_line, = plt.plot(range(WINDOW), self.anomaly_score, 'r-')
plt.legend(tuple(['anomaly score']), loc=3)
# Set the y-axis range.
MAX_VALUE=0.0005
self.actual_line.axes.set_ylim(-MAX_VALUE, MAX_VALUE)
self.predicted_line.axes.set_ylim(-MAX_VALUE, MAX_VALUE)
if self.show_anomaly_score:
self.anomaly_score_line.axes.set_ylim(-1, 1)
def runCPU():
"""Poll CPU usage, make predictions, and plot the results. Runs forever."""
# Create the model for predicting CPU usage.
model = ModelFactory.create(model_params.MODEL_PARAMS)
model.enableInference({'predictedField': 'cpu'})
# Create a metrics manager for computing an error metric.
metricsManager = MetricsManager(METRIC_SPECS, model.getFieldInfo(),
model.getInferenceType())
# The shifter will align prediction and actual values.
shifter = InferenceShifter()
# Keep the last WINDOW predicted and actual values for plotting.
actHistory = deque([0.0] * WINDOW, maxlen=60)
predHistory = deque([0.0] * WINDOW, maxlen=60)
# Initialize the plot lines that we will update with each new record.
actline, = plot(range(WINDOW), actHistory)
predline, = plot(range(WINDOW), predHistory)
# Set the y-axis range.
actline.axes.set_ylim(0, 100)
predline.axes.set_ylim(0, 100)
while True:
s = time.time()
# Get the CPU usage.
cpu = psutil.cpu_percent()
# Run the input through the model and shift the resulting prediction.
modelInput = {'cpu': cpu}
result = shifter.shift(model.run(modelInput))
# Compute an error metric (not currently used).
result.metrics = metricsManager.update(result)
# Update the trailing predicted and actual value deques.
inference = result.inferences['multiStepBestPredictions'][5]
if inference is not None:
actHistory.append(result.rawInput['cpu'])
predHistory.append(inference)
# Redraw the chart with the new data.
actline.set_ydata(actHistory) # update the data
predline.set_ydata(predHistory) # update the data
draw()
# Make sure we wait a total of 2 seconds per iteration.
time.sleep(SECONDS_PER_STEP - (time.time() - s))
def runIoThroughNupic(ip, port, model, metric_name, plot):
"""
Handles looping over the input data and passing each row into the given model
object, as well as extracting the result object and passing it into an output
handler.
:param ip: IP address of the muse server
:param port: port of the muse server
:param model: OPF Model object
:param metric_name: metric name, used for output handler naming
:param plot: Whether to use matplotlib or not. If false, uses file output.
"""
client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
client.bind((ip, port))
shifter = InferenceShifter()
if plot:
output = nupic_anomaly_output.NuPICPlotOutput(metric_name)
else:
output = nupic_anomaly_output.NuPICFileOutput(metric_name)
counter = 0
while 1:
counter += 1
if (counter % 100 == 0):
print "Read %i lines..." % counter
data = json.loads(client.recv(1024))
print data
# data in nano-volts (10E9)
metric_value = float(data['channel_values'][0] * 10E8)
result = model.run({"metric_value": metric_value})
#print metric_value
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
# TODO: timestamp data BEFORE sending it.
t = datetime.datetime.now().strftime(DATE_FORMAT)
timestamp = datetime.datetime.strptime(t, DATE_FORMAT)
# write output
output.write(timestamp, metric_value, prediction, anomalyScore)
output.close()
def runIoThroughNupic(inputData, model, gymName, plot):
inputFile = open(inputData, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
if plot:
output = nupic_output.NuPICPlotOutput([gymName])
else:
output = nupic_output.NuPICFileOutput([gymName])
metricsManager = MetricsManager(_METRIC_SPECS, model.getFieldInfo(),
model.getInferenceType())
counter = 0
for row in csvReader:
counter += 1
timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
consumption = float(row[1])
result = model.run({
"timestamp": timestamp,
"kw_energy_consumption": consumption
})
result.metrics = metricsManager.update(result)
if counter % 100 == 0:
print "Read %i lines..." % counter
print("After %i records, 1-step altMAPE=%f" %
(counter,
result.metrics["multiStepBestPredictions:multiStep:"
"errorMetric='altMAPE':steps=1:window=1000:"
"field=kw_energy_consumption"]))
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
output.write([timestamp], [consumption], [prediction])
if plot and counter % 20 == 0:
output.refreshGUI()
inputFile.close()
output.close()
def runModel(inputFilePath, low_model):
ep = Evaluate(predict_number=3)
region_sate = RegionState()
csv_data = get_csv_data(inputFilePath)
shifter_high = InferenceShifter()
shifter_low = InferenceShifter()
counter = 0
before_value = None
before_predvalue = []
for i in range(2):
for row in csv_data:
counter += 1
if counter % 10 == 0:
tp = low_model._getTPRegion()
tp.getSelf().resetSequenceStates()
# data準備
data = get_formated_data(row)
if before_value:
data["high_low"] = "high" if before_value < data["low"] else "low"
# run cla model
low_result = low_model.run(data)
# region_sate.save(data['low'], low_model, low_result)
# low prediction
low_result = shifter_low.shift(low_result)
high_low_prediction = low_result.inferences["multiStepBestPredictions"][1]
anomalyScore = low_result.inferences["anomalyScore"]
# # evaluate high/low
if before_value:
print i, counter,
ep.evaluate_high_low(data, high_low_prediction, anomalyScore)
# save now_value
# TODO: InferenceShifter を上手く使った方法に変更したい.
before_value = data["low"]
# region_sate.show(low_model)
return low_model
def testObjectWithNonStringKeys(self):
testClass = InferenceShifter()
testClass.a = 5
testClass.b = {(4, 5): (17,)}
encoded = json.dumps(testClass, sort_keys=True)
self.assertEqual(
encoded,
'{"_inferenceBuffer": null, "a": 5, "b": {"py/dict/keys": '
'["{\\"py/tuple\\": [4, 5]}"], "{\\"py/tuple\\": [4, 5]}": '
'{"py/tuple": [17]}}, "py/object": '
'"nupic.data.inference_shifter.InferenceShifter"}')
decoded = json.loads(encoded)
self.assertEqual(decoded.a, 5)
self.assertEqual(type(decoded.b), dict)
self.assertEqual(len(decoded.b.keys()), 1)
self.assertTupleEqual(decoded.b.keys()[0], (4, 5))
self.assertTupleEqual(decoded.b[(4, 5)], (17,))
def runIoThroughNupic(inputData, model, gymName, plot):
inputFile = open(inputData, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
if plot:
output = nupic_output.NuPICPlotOutput([gymName])
else:
output = nupic_output.NuPICFileOutput([gymName])
metricsManager = MetricsManager(_METRIC_SPECS, model.getFieldInfo(),
model.getInferenceType())
counter = 0
for row in csvReader:
counter += 1
timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
consumption = float(row[1])
result = model.run({
"timestamp": timestamp,
"kw_energy_consumption": consumption
})
result.metrics = metricsManager.update(result)
if counter % 100 == 0:
print "Read %i lines..." % counter
print ("After %i records, 1-step altMAPE=%f" % (counter,
result.metrics["multiStepBestPredictions:multiStep:"
"errorMetric='altMAPE':steps=1:window=1000:"
"field=kw_energy_consumption"]))
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
output.write([timestamp], [consumption], [prediction])
if plot and counter % 20 == 0:
output.refreshGUI()
inputFile.close()
output.close()
def runModel(model, inputFilePath):
inputFile = open(inputFilePath, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
output = nupic_output.NuPICPlotOutput("Vaccination")
shifter = InferenceShifter()
counter = 0
actualCount = 0
predictCount = 0
miss = 0
hit = 0
for row in csvReader:
counter += 1
if(counter % 10 == 0):
print "Read %i lines..." % counter
vaccine_date = datetime.datetime.strptime(row[2], DATE_FORMAT)
vaccine_name = str(row[1])
result = model.run({
"vaccine_date": vaccine_date,
"vaccine_name": vaccine_name
})
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
#output.write([vaccine_date], [vaccine_name], [prediction])
print len(vaccine_name)
output.write(vaccine_date, len(vaccine_name), len(prediction), anomalyScore)
if prediction == "Yellow Fever":
predictCount += 1
if vaccine_name == "Yellow Fever":
actualCount += 1
if vaccine_name == prediction:
hit += 1
else:
miss += 1
print counter, "community member_id: ", row[0], "Actual: ", vaccine_name, "Predicted: ", prediction, "------", anomalyScore
print"\n Number of actuals: ", actualCount," \n Number of predictions: ", predictCount
print "\n hits: ", hit,"\n misses: ", miss
def runIoThroughNupic(inputData, model, gymName, plot):
"""
Handles looping over the input data and passing each row into the given model
object, as well as extracting the result object and passing it into an output
handler.
:param inputData: file path to input data CSV
:param model: OPF Model object
:param gymName: Gym name, used for output handler naming
:param plot: Whether to use matplotlib or not. If false, uses file output.
"""
# t0 = time()
# # model = load(open('model.pkl', 'rb'))
# model = ModelFactory.loadFromCheckpoint('/home/magarwal/logoDetective/core/anomalyDetection/nupic/nupic/examples/opf/clients/hotgym/anomaly/one_gym/model_save/model.pkl')
# print 'time taken in loading model = ',time()-t0
inputFile = open(inputData, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
if plot:
output = nupic_anomaly_output.NuPICPlotOutput(gymName)
else:
output = nupic_anomaly_output.NuPICFileOutput(gymName)
counter = 0
for row in csvReader:
counter += 1
if (counter % 1000 == 0):
print "Read %i lines..." % counter
timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
feat = float(row[1])
result = model.run({"time_start": timestamp, FEAT_NAME: feat})
if plot:
result = shifter.shift(result)
# print 'result = ',result
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
output.write(timestamp, feat, prediction, anomalyScore)
inputFile.close()
output.close()
def testObjectWithNonStringKeys(self):
testClass = InferenceShifter()
testClass.a = 5
testClass.b = {(4, 5): (17, )}
encoded = json.dumps(testClass, sort_keys=True)
self.assertEqual(
encoded,
'{"_inferenceBuffer": null, "a": 5, "b": {"py/dict/keys": '
'["{\\"py/tuple\\": [4, 5]}"], "{\\"py/tuple\\": [4, 5]}": '
'{"py/tuple": [17]}}, "py/object": '
'"nupic.data.inference_shifter.InferenceShifter"}')
decoded = json.loads(encoded)
self.assertEqual(decoded.a, 5)
self.assertEqual(type(decoded.b), dict)
self.assertEqual(len(decoded.b.keys()), 1)
self.assertTupleEqual(decoded.b.keys()[0], (4, 5))
self.assertTupleEqual(decoded.b[(4, 5)], (17, ))
def __init__(self, config):
# Instantiate NuPIC model
model_params = base_model_params.MODEL_PARAMS
# Set resolution
model_params['modelParams']['sensorParams']['encoders']['value']['resolution'] = config['resolution']
# Override other Nupic parameters:
model_params['modelParams'] = update_dict(model_params['modelParams'], config['nupic_model_params'])
# Create model and enable inference on it
self.model = ModelFactory.create(model_params)
self.model.enableInference({'predictedField': 'value'})
# The shifter is used to bring the predictions to the actual time frame
self.shifter = InferenceShifter()
# The anomaly likelihood object
self.anomalyLikelihood = AnomalyLikelihood()
# Set stream source
self.stream = config['stream']
# Setup class variables
self.db = redis.Redis('localhost')
self.seconds_per_request = config['seconds_per_request']
self.webhook = config['webhook']
self.anomaly_threshold = config['anomaly_threshold']
self.likelihood_threshold = config['likelihood_threshold']
self.domain = config['domain']
self.alert = False # Toogle when we get above threshold
# Setup logging
self.logger = logger or logging.getLogger(__name__)
handler = logging.handlers.RotatingFileHandler(os.environ['LOG_DIR']+"/monitor_%s.log" % self.stream.name,
maxBytes=1024*1024,
backupCount=4,
)
handler.setFormatter(logging.Formatter('[%(levelname)s/%(processName)s][%(asctime)s] %(name)s %(message)s'))
handler.setLevel(logging.INFO)
self.logger.addHandler(handler)
self.logger.setLevel(logging.INFO)
self.logger.info("=== Settings ===")
self.logger.info("Webhook: %s", self.webhook)
self.logger.info("Domain: %s", self.domain)
self.logger.info("Seconds per request: %d", self.seconds_per_request)
# Write metadata to Redis
try:
# Save in redis with key = 'results:monitor_id' and value = 'time, status, actual, prediction, anomaly'
self.db.set('name:%s' % self.stream.id, self.stream.name)
self.db.set('value_label:%s' % self.stream.id, self.stream.value_label)
self.db.set('value_unit:%s' % self.stream.id, self.stream.value_unit)
except Exception:
self.logger.warn("Could not write results to redis.", exc_info=True)
def runCPU():
"""Poll CPU usage, make predictions, and plot the results. Runs forever."""
# Create the model for predicting CPU usage.
model = ModelFactory.create(model_params.MODEL_PARAMS)
model.enableInference({'predictedField': 'cpu'})
# The shifter will align prediction and actual values.
shifter = InferenceShifter()
# Keep the last WINDOW predicted and actual values for plotting.
actHistory = deque([0.0] * WINDOW, maxlen=60)
predHistory = deque([0.0] * WINDOW, maxlen=60)
# Initialize the plot lines that we will update with each new record.
actline, = plt.plot(range(WINDOW), actHistory)
predline, = plt.plot(range(WINDOW), predHistory)
# Set the y-axis range.
actline.axes.set_ylim(0, 100)
predline.axes.set_ylim(0, 100)
while True:
s = time.time()
# Get the CPU usage.
cpu = psutil.cpu_percent()
# Run the input through the model and shift the resulting prediction.
modelInput = {'cpu': cpu}
result = shifter.shift(model.run(modelInput))
# Update the trailing predicted and actual value deques.
inference = result.inferences['multiStepBestPredictions'][5]
if inference is not None:
actHistory.append(result.rawInput['cpu'])
predHistory.append(inference)
# Redraw the chart with the new data.
actline.set_ydata(actHistory) # update the data
predline.set_ydata(predHistory) # update the data
plt.draw()
plt.legend(('actual', 'predicted'))
# Make sure we wait a total of 2 seconds per iteration.
try:
plt.pause(SECONDS_PER_STEP)
except:
pass
def runIoThroughNupic(inputData, model, InputName, plot):
"""
Handles looping over the input data and passing each row into the given model
object, as well as extracting the result object and passing it into an output
handler.
:param inputData: file path to input data CSV
:param model: OPF Model object
:param InputName: Gym name, used for output handler naming
:param plot: Whether to use matplotlib or not. If false, uses file output.
"""
inputFile = open(inputData, "rb")
csvReader = csv.reader(inputFile)
# skip header rows
csvReader.next()
csvReader.next()
csvReader.next()
shifter = InferenceShifter()
if plot:
output = nupic_anomaly_output.NuPICPlotOutput(InputName)
else:
output = nupic_anomaly_output.NuPICFileOutput(InputName)
counter = 0
for row in csvReader:
counter += 1
if (counter % 100 == 0):
print "Read %i lines..." % counter
timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
PredFldNm = float(row[1])
result = model.run({
"c0": timestamp,
"c1": PredFldNm
})
if plot:
result = shifter.shift(result)
prediction = result.inferences["multiStepBestPredictions"][1]
anomalyScore = result.inferences["anomalyScore"]
output.write(timestamp, PredFldNm, prediction, anomalyScore)
inputFile.close()
output.close()
class NuPICPlotOutput(NuPICOutput):
def __init__(self, *args, **kwargs):
super(NuPICPlotOutput, self).__init__(*args, **kwargs)
# turn matplotlib interactive mode on (ion)
plt.ion()
plt.figure(figsize=(14, 10))
gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1])
# plot title, legend, etc
plt.title("Sine prediction example")
plt.ylabel("Sine (rad)")
# The shifter will align prediction and actual values.
self.shifter = InferenceShifter()
# Keep the last WINDOW predicted and actual values for plotting.
self.actual_history = deque([0.0] * WINDOW, maxlen=360)
self.predicted_history = deque([0.0] * WINDOW, maxlen=360)
if self.show_anomaly_score:
self.anomaly_score = deque([0.0] * WINDOW, maxlen=360)
# Initialize the plot lines that we will update with each new record.
if self.show_anomaly_score:
plt.subplot(gs[0])
self.actual_line, = plt.plot(range(WINDOW), self.actual_history)
self.predicted_line, = plt.plot(range(WINDOW), self.predicted_history)
plt.legend(tuple(["actual", "predicted"]), loc=3)
if self.show_anomaly_score:
plt.subplot(gs[1])
self.anomaly_score_line, = plt.plot(range(WINDOW), self.anomaly_score, "r-")
plt.legend(tuple(["anomaly score"]), loc=3)
# Set the y-axis range.
self.actual_line.axes.set_ylim(-1, 1)
self.predicted_line.axes.set_ylim(-1, 1)
if self.show_anomaly_score:
self.anomaly_score_line.axes.set_ylim(-1, 1)
def write(self, index, value, prediction_result, prediction_step=1):
shifted_result = self.shifter.shift(prediction_result)
# shifted_result = prediction_result
# Update the trailing predicted and actual value deques.
inference = shifted_result.inferences["multiStepBestPredictions"][prediction_step]
if inference is not None:
self.actual_history.append(shifted_result.rawInput["sine"])
self.predicted_history.append(inference)
if self.show_anomaly_score:
anomaly_score = prediction_result.inferences["anomalyScore"]
self.anomaly_score.append(anomaly_score)
# Redraw the chart with the new data.
self.actual_line.set_ydata(self.actual_history) # update the data
self.predicted_line.set_ydata(self.predicted_history) # update the data
if self.show_anomaly_score:
self.anomaly_score_line.set_ydata(self.anomaly_score) # update the data
plt.draw()
plt.tight_layout()
def close(self):
plt.ioff()
plt.show()
def __init__(self, metricSpecs, fieldInfo, inferenceType):
self.__metricSpecs = []
self.__metrics = []
self.__metricLabels = []
# Maps field names to indices. Useful for looking up input/predictions by
# field name
self.__fieldNameIndexMap = dict( [(info.name, i) \
for i, info in enumerate(fieldInfo)] )
self.__constructMetricsModules(metricSpecs)
self.__currentGroundTruth = None
self.__currentInference = None
self.__currentResult = None
self.__isTemporal = InferenceType.isTemporal(inferenceType)
if self.__isTemporal:
self.__inferenceShifter = InferenceShifter()