def runNetwork(network):
"""
"""
sensorRegion = network.regions["sensor"]
spatialPoolerRegion = network.regions["spatialPoolerRegion"]
temporalPoolerRegion = network.regions["temporalPoolerRegion"]
classifierRegion = network.regions["classifierRegion"]
prevPredictedColumns = []
for i, data in enumerate(get_input_data() * 100):
# add data
sensorRegion.getSelf().dataSource.push(data)
# Run the network for a single iteration
network.run(1)
# Calculate the anomaly score using the active columns
# and previous predicted columns
activeColumns = spatialPoolerRegion.getOutputData("bottomUpOut").nonzero()[0]
anomalyScore = computeAnomalyScore(activeColumns, prevPredictedColumns)
prevPredictedColumns = copy.deepcopy(activeColumns)
# Classifier
activeCells = temporalPoolerRegion.getOutputData("bottomUpOut").nonzero()[0]
res = classifierRegion.getSelf().customCompute(
recordNum=i,
patternNZ=activeCells,
classification={
'bucketIdx': createClassifierEncoder().getBucketIndices(data)[0] ,
'actValue': data['y']}
)
predict = res['actualValues'][res[0].tolist().index(max(res[0]))]
rate = max(res[0])
print '%s x:%d y:%s p:%s rate:%5.2f anomaly:%5.2f' % (i, data['x'], data['y'], predict, rate, anomalyScore)
def runNetwork(network, writer):
"""Run the network and write output to writer.
:param network: a Network instance to run
:param writer: a csv.writer instance to write output to
"""
sensorRegion = network.regions["sensor"]
spatialPoolerRegion = network.regions["spatialPoolerRegion"]
temporalPoolerRegion = network.regions["temporalPoolerRegion"]
prevPredictedColumns = []
i = 0
for _ in xrange(_NUM_RECORDS):
# Run the network for a single iteration
network.run(1)
activeColumns = spatialPoolerRegion.getOutputData(
"bottomUpOut").nonzero()[0]
# Calculate the anomaly score using the active columns
# and previous predicted columns
anomalyScore = computeAnomalyScore(activeColumns, prevPredictedColumns)
# Write out the anomaly score along with the record number and consumption
# value.
consumption = sensorRegion.getOutputData("sourceOut")[0]
writer.writerow((i, consumption, anomalyScore))
# Store the predicted columns for the next timestep
predictedColumns = temporalPoolerRegion.getOutputData(
"topDownOut").nonzero()[0]
prevPredictedColumns = copy.deepcopy(predictedColumns)
i += 1
def _calc_anomaly(self):
"""
各層のanomalyを計算
"""
score = 0
anomalyScore = {}
for name in self.dest_region_params.keys():
#sp_bottomUpOut = self.network.regions["sp_"+name].getOutputData("bottomUpOut").nonzero()[0]
sp_bottomUpOut = self.network.regions["tp_"+name].getInputData("bottomUpIn").nonzero()[0]
if self.prevPredictedColumns.has_key(name):
score = computeAnomalyScore(sp_bottomUpOut, self.prevPredictedColumns[name])
#topdown_predict = self.network.regions["TP"].getSelf()._tfdr.topDownCompute().copy().nonzero()[0]
topdown_predict = self.network.regions["tp_"+name].getSelf()._tfdr.topDownCompute().nonzero()[0]
self.prevPredictedColumns[name] = copy.deepcopy(topdown_predict)
anomalyScore[name] = score
return anomalyScore
def _calc_anomaly(self):
"""
各層のanomalyを計算
"""
score = 0
anomalyScore = {}
for name in set( itertools.chain.from_iterable( self.net_structure.values() )):
#sp_bottomUpOut = self.network.regions["sp_"+name].getOutputData("bottomUpOut").nonzero()[0]
sp_bottomUpOut = self.network.regions["tp_"+name].getInputData("bottomUpIn").nonzero()[0]
if self.prevPredictedColumns.has_key(name):
score = computeAnomalyScore(sp_bottomUpOut, self.prevPredictedColumns[name])
#topdown_predict = self.network.regions["TP"].getSelf()._tfdr.topDownCompute().copy().nonzero()[0]
topdown_predict = self.network.regions["tp_"+name].getSelf()._tfdr.topDownCompute().nonzero()[0]
self.prevPredictedColumns[name] = copy.deepcopy(topdown_predict)
anomalyScore[name] = score
return anomalyScore
def _calc_anomaly(self):
"""
各層のanomalyを計算
"""
import copy
import itertools
from nupic.algorithms.anomaly import computeAnomalyScore
score = 0
anomalyScore = {}
for name in set( itertools.chain.from_iterable( self.net_structure.values() )):
sp_bottomUpOut = self.network.regions["sp_"+name].getOutputData("bottomUpOut").nonzero()[0]
if self.prevPredictedColumns.has_key(name):
score = computeAnomalyScore(sp_bottomUpOut, self.prevPredictedColumns[name])
#topdown_predict = self.network.regions["TP"].getSelf()._tfdr.topDownCompute().copy().nonzero()[0]
topdown_predict = self.network.regions["tp_"+name].getSelf()._tfdr.topDownCompute().nonzero()[0]
self.prevPredictedColumns[name] = copy.deepcopy(topdown_predict)
anomalyScore[name] = score
return anomalyScore
def runNetwork(network):
"""
"""
sensorRegion = network.regions["sensor"]
spatialPoolerRegion = network.regions["spatialPoolerRegion"]
temporalPoolerRegion = network.regions["temporalPoolerRegion"]
classifierRegion = network.regions["classifierRegion"]
prevPredictedColumns = []
for i, data in enumerate(get_input_data() * 100):
# add data
sensorRegion.getSelf().dataSource.push(data)
# Run the network for a single iteration
network.run(1)
# Calculate the anomaly score using the active columns
# and previous predicted columns
activeColumns = spatialPoolerRegion.getOutputData(
"bottomUpOut").nonzero()[0]
anomalyScore = computeAnomalyScore(activeColumns, prevPredictedColumns)
prevPredictedColumns = copy.deepcopy(activeColumns)
# Classifier
activeCells = temporalPoolerRegion.getOutputData(
"bottomUpOut").nonzero()[0]
res = classifierRegion.getSelf().customCompute(
recordNum=i,
patternNZ=activeCells,
classification={
'bucketIdx':
createClassifierEncoder().getBucketIndices(data)[0],
'actValue': data['y']
})
predict = res['actualValues'][res[0].tolist().index(max(res[0]))]
rate = max(res[0])
print '%s x:%d y:%s p:%s rate:%5.2f anomaly:%5.2f' % (
i, data['x'], data['y'], predict, rate, anomalyScore)
def constructClassificationRecord(self, inputs):
"""
Construct a _CLAClassificationRecord based on the state of the model
passed in through the inputs.
Types for self.classificationVectorType:
1 - TP active cells in learn state
2 - SP columns concatenated with error from TP column predictions and SP
"""
# Count the number of unpredicted columns
allSPColumns = inputs["spBottomUpOut"]
activeSPColumns = allSPColumns.nonzero()[0]
score = computeAnomalyScore(activeSPColumns, self._prevPredictedColumns)
spSize = len(allSPColumns)
allTPCells = inputs['tpTopDownOut']
tpSize = len(inputs['tpLrnActiveStateT'])
classificationVector = numpy.array([])
if self.classificationVectorType == 1:
# Classification Vector: [---TP Cells---]
classificationVector = numpy.zeros(tpSize)
activeCellMatrix = inputs["tpLrnActiveStateT"].reshape(tpSize, 1)
activeCellIdx = numpy.where(activeCellMatrix > 0)[0]
if activeCellIdx.shape[0] > 0:
classificationVector[numpy.array(activeCellIdx, dtype=numpy.uint16)] = 1
elif self.classificationVectorType == 2:
# Classification Vecotr: [---SP---|---(TP-SP)----]
classificationVector = numpy.zeros(spSize+spSize)
if activeSPColumns.shape[0] > 0:
classificationVector[activeSPColumns] = 1.0
errorColumns = numpy.setdiff1d(self._prevPredictedColumns,
activeSPColumns)
if errorColumns.shape[0] > 0:
errorColumnIndexes = ( numpy.array(errorColumns, dtype=numpy.uint16) +
spSize )
classificationVector[errorColumnIndexes] = 1.0
else:
raise TypeError("Classification vector type must be either 'tpc' or"
" 'sp_tpe', current value is %s" % (self.classificationVectorType))
# Store the state for next time step
numPredictedCols = len(self._prevPredictedColumns)
predictedColumns = allTPCells.nonzero()[0]
self._prevPredictedColumns = copy.deepcopy(predictedColumns)
if self._anomalyVectorLength is None:
self._anomalyVectorLength = len(classificationVector)
result = _CLAClassificationRecord(
ROWID=self._iteration, #__numRunCalls called
#at beginning of model.run
anomalyScore=score,
anomalyVector=classificationVector.nonzero()[0].tolist(),
anomalyLabel=[]
)
return result
def testComputeAnomalyScorePartialMatch(self):
score = anomaly.computeAnomalyScore(array([2, 3, 6]), array([3, 5, 7]))
self.assertAlmostEqual(score, 2.0 / 3.0)
def testComputeAnomalyScoreNoMatch(self):
score = anomaly.computeAnomalyScore(array([2, 4, 6]), array([3, 5, 7]))
self.assertAlmostEqual(score, 1.0)
def testComputeAnomalyScorePerfectMatch(self):
score = anomaly.computeAnomalyScore(array([3, 5, 7]), array([3, 5, 7]))
self.assertAlmostEqual(score, 0.0)
def testComputeAnomalyScoreNoActive(self):
score = anomaly.computeAnomalyScore(array([]), array([3, 5]))
self.assertAlmostEqual(score, 1.0)
def testComputeAnomalyScoreNoActiveOrPredicted(self): score = anomaly.computeAnomalyScore(array([]), array([])) self.assertAlmostEqual(score, 0.0)
def constructClassificationRecord(self, inputs):
"""
Construct a _CLAClassificationRecord based on the state of the model
passed in through the inputs.
Types for self.classificationVectorType:
1 - TP active cells in learn state
2 - SP columns concatenated with error from TP column predictions and SP
"""
# Count the number of unpredicted columns
allSPColumns = inputs["spBottomUpOut"]
activeSPColumns = allSPColumns.nonzero()[0]
score = computeAnomalyScore(activeSPColumns,
self._prevPredictedColumns)
spSize = len(allSPColumns)
allTPCells = inputs['tpTopDownOut']
tpSize = len(inputs['tpLrnActiveStateT'])
classificationVector = numpy.array([])
if self.classificationVectorType == 1:
# Classification Vector: [---TP Cells---]
classificationVector = numpy.zeros(tpSize)
activeCellMatrix = inputs["tpLrnActiveStateT"].reshape(tpSize, 1)
activeCellIdx = numpy.where(activeCellMatrix > 0)[0]
if activeCellIdx.shape[0] > 0:
classificationVector[numpy.array(activeCellIdx,
dtype=numpy.uint16)] = 1
elif self.classificationVectorType == 2:
# Classification Vecotr: [---SP---|---(TP-SP)----]
classificationVector = numpy.zeros(spSize + spSize)
if activeSPColumns.shape[0] > 0:
classificationVector[activeSPColumns] = 1.0
errorColumns = numpy.setdiff1d(self._prevPredictedColumns,
activeSPColumns)
if errorColumns.shape[0] > 0:
errorColumnIndexes = (
numpy.array(errorColumns, dtype=numpy.uint16) + spSize)
classificationVector[errorColumnIndexes] = 1.0
else:
raise TypeError(
"Classification vector type must be either 'tpc' or"
" 'sp_tpe', current value is %s" %
(self.classificationVectorType))
# Store the state for next time step
numPredictedCols = len(self._prevPredictedColumns)
predictedColumns = allTPCells.nonzero()[0]
self._prevPredictedColumns = copy.deepcopy(predictedColumns)
if self._anomalyVectorLength is None:
self._anomalyVectorLength = len(classificationVector)
result = _CLAClassificationRecord(
ROWID=self._iteration, #__numRunCalls called
#at beginning of model.run
anomalyScore=score,
anomalyVector=classificationVector.nonzero()[0].tolist(),
anomalyLabel=[])
return result
def testComputeAnomalyScorePartialMatch(self): score = anomaly.computeAnomalyScore(array([2, 3, 6]), array([3, 5, 7])) self.assertAlmostEqual(score, 2.0 / 3.0)
def testComputeAnomalyScoreNoMatch(self): score = anomaly.computeAnomalyScore(array([2, 4, 6]), array([3, 5, 7])) self.assertAlmostEqual(score, 1.0)
def testComputeAnomalyScorePerfectMatch(self): score = anomaly.computeAnomalyScore(array([3, 5, 7]), array([3, 5, 7])) self.assertAlmostEqual(score, 0.0)
def testComputeAnomalyScoreNoActive(self): score = anomaly.computeAnomalyScore(array([]), array([3, 5])) self.assertAlmostEqual(score, 1.0)
def testComputeAnomalyScoreNoActiveOrPredicted(self):
score = anomaly.computeAnomalyScore(array([]), array([]))
self.assertAlmostEqual(score, 0.0)
def main():
def calc(sensor, sp, tp):
sensor.prepareInputs()
sensor.compute()
sp.prepareInputs()
sp.compute()
tp.prepareInputs()
tp.compute()
# ファイル渡すだけなら, 絶対パスに変換しているだけ.
#trainFile = findDataset(_DATA_PATH)
trainFile = os.path.abspath(_DATA_PATH)
print trainFile
# TODO: sensor layerには, dataSourceの形で渡す必要がある.
# 後から渡すような形にするためには, DataBuffer@nupic/frameworks/opf/clamodel.py のように,
# getNextRecordDict methodを持った, instanceを作れば良いのかな?
dataSource = FileRecordStream(streamID=trainFile)
# print dataSource
# for i in range(2000):
# print dataSource.getNextRecordDict()
network = createNetwork(dataSource)
#
sensorRegion = network.regions["sensor"]
SPRegion = network.regions["SP"]
TPRegion = network.regions["TP"]
classifier = network.regions["Classifier"]
prevPredictedColumns = []
# TODO: ここの内容, OPFとの違いをちゃんと押さえる.
#for calc_num in xrange(10000):
f = open(trainFile, "rb")
csvReader = csv.reader(f)
csvReader.next()
csvReader.next()
csvReader.next()
#for calc_num in xrange(10000):
for calc_num, row in enumerate(csvReader):
gym = row[0]
timestamp = datetime.datetime.strptime(row[1], "%Y-%m-%d %H:%M:%S.0")
consumption = float(row[3])
input_data = {
'timestamp': timestamp,
'consumption': consumption,
'gym': gym
}
sensorRegion.getSelf().dataSource.push(input_data)
# TODO: このrunって実際何をやってるんだ? どこに書いてあるか探す.
# 多分, 各Regionの_computeでやってることをまとめて実行してるのだと思う.
network.run(1)
#calc(sensorRegion, SPRegion, TPRegion)
#
print
print "####################################"
print
print "==== EC layer ===="
consumption = sensorRegion.getOutputData("sourceOut")[0]
print 'sourceOut',consumption
categoryOut = sensorRegion.getOutputData("categoryOut")[0]
print 'categoryOut',categoryOut
# spatialTopDownOut = sensorRegion.getOutputData("spatialTopDownOut")[0]
# print 'spatialTopDownOut',spatialTopDownOut
# temporalTopDownOut = sensorRegion.getOutputData("temporalTopDownOut")[0]
# print 'temporalTopDownOut',temporalTopDownOut
# SPへの入力(encode後)
print
print "==== SP layer ===="
sp_bottomUpIn = SPRegion.getInputData("bottomUpIn").nonzero()[0]
print 'bottomUpIn', sp_bottomUpIn[:10]
sp_bottomUpOut = SPRegion.getOutputData("bottomUpOut").nonzero()[0]
print 'bottomUpOut', sp_bottomUpOut[:10]
# sp_input = SPRegion.getInputData("topDownIn").nonzero()[0]
# print 'topDownIn',sp_input[:10]
# topDownOut = SPRegion.getOutputData("topDownOut").nonzero()[0]
# print 'topDownOut', topDownOut[:10]
# spatialTopDownOut = SPRegion.getOutputData("spatialTopDownOut").nonzero()[0]
# print 'spatialTopDownOut', spatialTopDownOut[:10]
# temporalTopDownOut = SPRegion.getOutputData("temporalTopDownOut").nonzero()[0]
# print 'temporalTopDownOut', temporalTopDownOut[:10]
# anomalyScore = SPRegion.getOutputData("anomalyScore").nonzero()[0]
# print 'anomalyScore', anomalyScore[:10]
# anomaly
# TPへのinput前
print
print "==== TP layer ===="
# TPへのinput後 ※ topdownmdoeをfalseにしたらできたが..
tp_bottomUpIn = TPRegion.getInputData("bottomUpIn").nonzero()[0]
print 'bottomUpIn', sorted(tp_bottomUpIn)[:10]
tp_bottomUpOut = TPRegion.getOutputData("bottomUpOut").nonzero()[0]
print 'bottomUpOut', tp_bottomUpOut[:10]
# tp_topDownOut = TPRegion.getOutputData("topDownOut").nonzero()[0]
# print 'topDownOut', tp_topDownOut[:10]
# predictedColumns = TPRegion.getOutputData("lrnActiveStateT").nonzero()[0]
# print 'lrnActiveStateT',predictedColumns[:10]
# predictedColumns = TPRegion.getOutputData("anomalyScore").nonzero()[0]
# print 'anomalyScore',predictedColumns[:10]
#
print
print "==== Anomaly ===="
print 'sp_bottomUpOut', sp_bottomUpOut[:10]
print 'prevPredictedColumns', prevPredictedColumns[:10]
anomalyScore = computeAnomalyScore(sp_bottomUpOut, prevPredictedColumns)
print 'anomalyScore', anomalyScore
print
print "==== Predict ===="
topdown_predict = TPRegion.getSelf()._tfdr.topDownCompute().copy().nonzero()[0]
print 'tp_topdown_predict', sorted(topdown_predict)[:10]
prevPredictedColumns = copy.deepcopy(topdown_predict)
categoryIn = classifier.getInputData("categoryIn").nonzero()[0]
print 'categoryIn', categoryIn[:10]
cl_bottomUpIn = classifier.getInputData("bottomUpIn").nonzero()[0]
print 'bottomUpIn', cl_bottomUpIn[:10]
# predict consumption
# enc_list = sensorRegion.getSelf().encoder.getEncoderList()
# bucketIdx = enc_list[0].getBucketIndices(consumption)[0]
# classificationIn = {
# 'bucketIdx': bucketIdx,
# 'actValue': float(consumption)
# }
# predict gym
enc_list = sensorRegion.getSelf().disabledEncoder.getEncoderList()
bucketIdx = enc_list[0].getBucketIndices(gym)[0]
classificationIn = {
'bucketIdx': bucketIdx,
'actValue': gym
}
clResults = classifier.getSelf().customCompute(
recordNum=calc_num,
patternNZ=tp_bottomUpOut,
classification=classificationIn
)
max_index = [i for i, j in enumerate(clResults[1] ) if j == max(clResults[1] )]
print 'predict value: ', clResults['actualValues'][max_index[0]]
print 'actual value: ', gym
print
"""