def getProtosAtTime(self, timestep):
collection = ExperimentDB.getExperimentDB(self.name)
experimentData = collection.find_one({"_id":timestep})
experimentData['protos'] = []
predictedField = self._modelDescription["predictedField"]
predictedFieldIndex = self.getFieldNames().index(predictedField)
dists = json.loads(experimentData['classificationDist'])
for distId in json.loads(experimentData["classificationIdx"]):
distSurroundingValues = collection.find({"_id": {
"$gt": distId-10, "$lt": distId+10
}}).sort('_id', pymongo.ASCENDING)
experimentData['protos'].append(dict(
ids=[],
actual=[],
prediction=[],
anomaly=[],
anomalyLabel=[],
dist=dists.pop(0),
index=distId
))
protosId = len(experimentData['protos']) - 1
for distSurroundingValue in distSurroundingValues:
inferences = json.loads(distSurroundingValue["inferences"])
actual = distSurroundingValue["actual"]
inference = inferences[InferenceElement.multiStepBestPredictions]
step = min(inference.iterkeys())
prediction =inference[step]
if prediction is None:
prediction = 0.0
anomaly = inferences[InferenceElement.anomalyScore]
anomalyLabel = inferences[InferenceElement.anomalyLabel]
experimentData['protos'][protosId]["ids"].append(distSurroundingValue["_id"])
experimentData['protos'][protosId]["actual"].append(actual)
experimentData['protos'][protosId]["prediction"].append(prediction)
experimentData['protos'][protosId]["anomaly"].append(anomaly)
experimentData['protos'][protosId]["anomalyLabel"].append(anomalyLabel)
return experimentData
def _runExperimentLoop(self, queue):
collection = ExperimentDB.getExperimentDB(self.name)
while self._maxiterations == -1 or self._iteration <= self._maxiterations:
try:
# Get next record
record = self._dataset.getNextRecord()
except StopIteration:
self._isFinished.set()
return None
if self._stop.isSet():
break
# Feed record to model and get prediction
modelResult = self._model.run(record)
if modelResult is None:
continue
if modelResult.inferences[InferenceElement.anomalyVector] is not None:
modelResult.inferences[InferenceElement.anomalyVector] = \
modelResult.inferences[InferenceElement.anomalyVector].nonzero()[0].tolist()
distances = self._model._classifier_distances #classifier.getSelf().getLatestDistances()
sortedDistIdx = []
sortedDists = []
if distances is not None and len(distances) > 0:
sortedDistIdx = distances.argsort()
sortedDists = distances[sortedDistIdx[:5]].tolist()
idList = self._model._classifier_indexes #classifier.getSelf().getParameter('categoryRecencyList')
if len(idList) > 0:
sortedDistIdx = [ \
idList[i] + self._model._classificationDelay - 1\
for i in sortedDistIdx[:min(5, len(sortedDistIdx))]]
else:
sortedDistIdx = []
#matrix = classifier.getSelf()._knn._Memory
#print matrix.shape
#print "Index: %s" % (sorted)
#if len(sorted) > 0:
# print matrix.getRow(int(sorted[0]))
# print matrix.getRow(int(sorted[0])).nonzero()[0]
predictedField = self._modelDescription["predictedField"]
predictedFieldIndex = self.getFieldNames().index(predictedField)
modelResult.inferences['encodings'] = None
modelResult.sensorInput.dataEncodings = None
actual = modelResult.sensorInput.dataRow[predictedFieldIndex]
dbelem = {"_id":self._iteration,
"actual": actual,
"inferences": json.dumps(modelResult.inferences),
"classificationIdx":json.dumps(sortedDistIdx),
"classificationDist":json.dumps(sortedDists)
}
collection.insert(dbelem)
self._dataQ.put(dbelem)
self._iteration += 1
gevent.sleep(0)
self._isFinished.set()
def getDetailsAtTime(self, timestep):
collection = ExperimentDB.getExperimentDB(self.name)
return collection.find_one({"_id":timestep})
def getDataAtTime(self, dataInput):
timestep = int(dataInput['timestep'])
collection = ExperimentDB.getExperimentDB(self.name)
experimentData = collection.find_one({"_id": timestep})
return experimentData
def _runExperimentLoadLoop(self, queue):
collection = ExperimentDB.getExperimentDB(self.name)
experimentData = collection.find()
for record in experimentData:
self._dataQ.put(record)
gevent.sleep(0)
def _runExperimentLoop(self, queue):
self.prevFieldPred = {}
self._model.resetSequenceStates()
cOut = os.fdopen(os.open("/tmp/cerebro.cout", os.O_RDWR | os.O_CREAT), 'w+')
oldC = os.dup(1)
collection = ExperimentDB.getExperimentDB(self.name)
while self._maxiterations == -1 or self._iteration <= self._maxiterations:
try:
# Get next record
record = self._dataset.getNextRecord()
except StopIteration:
self._isFinished.set()
return None
if self._stop.isSet():
break
# Feed record to model and get prediction. Capture all the stdout as well
os.dup2(cOut.fileno(), 1)
modelResult = self._model.run(record)
os.dup2(oldC, 1)
cOut.seek(0)
verboseOutput = cOut.read()
cOut.truncate(0)
modelResult.inferences['encodings'] = None
modelResult.sensorInput.dataEncodings = None
model = self._model
sensor = model._getSensorRegion()
sp = model._getSPRegion()
tp = model._getTPRegion()
cl = model._getClassifierRegion()
spImp = None
tpImp = None
if sp is not None:
spImp = sp.getSelf()._sfdr
if tp is not None:
tpImp = tp.getSelf()._tfdr
clImp = cl.getSelf()._claClassifier
#Copy all the pertinent data
sourceScalars = copy.deepcopy(sensor.getOutputData('sourceOut'))
sensorBits = sensor.getOutputData('dataOut')
sensorBUOut = sensorBits.nonzero()[0].tolist()
SPBUOut = []
nConnectedInputs = []
overlaps = []
if spImp is not None:
SPBUOut = sp.getOutputData('bottomUpOut').nonzero()[0].tolist()
nConnectedInputs = spImp._allConnectedM.nNonZerosPerRow()[SPBUOut].astype('int32').tolist()
overlaps = zip(SPBUOut,
spImp._overlapsNoBoost[SPBUOut].astype('int32').tolist())
TPTDOut = tp.getOutputData('topDownOut') if tp else None
sensorTDIn = sensor.getInputData('temporalTopDownIn')
permanences = {}
predictedCols = ()
predictedConfidences = ()
tpInfActiveCells = ()
tpLrnActiveCells = ()
tpPredCells = []
if TPTDOut is not None:
predictedCols = TPTDOut.nonzero()[0].tolist()
predictedConfidences = TPTDOut[predictedCols].tolist()
tpInfActiveCells = self._formatActiveCells(tpImp.infActiveState['t'])
tpLrnActiveCells = self._formatActiveCells(tpImp.lrnActiveState['t'])
tpInfPredT_1 = self._formatActiveCells(tpImp.infPredictedState['t-1'])
tpInfPredT = self._formatActiveCells(tpImp.infPredictedState['t'])
tpPredCells = tpImp.infPredictedState['t'].nonzero()[0].tolist()
sensorPredBits = []
if sensorTDIn is not None:
sensorPredBits = sensorTDIn
if self.prevPredictedCols is None:
self.prevPredictedCols = []
self.prevTPPredictedCells = []
self.prevPredictedConfs = []
self.prevTPPredicted = []
clPattern = clImp._patternNZHistory[-1]
step = clImp.steps[0]
bitHistories = {}
fieldActivations = {}
fieldPredictions = {}
for fieldName, (start, stop) in self.fieldRanges.iteritems():
nzBits = sensorBits[start:stop].nonzero()[0]
fieldActivations[fieldName] = nzBits.tolist()
nzBits = sensorPredBits[start:stop].nonzero()[0]
fieldPredictions[fieldName] = nzBits.tolist()
predictedField = self._modelDescription["predictedField"]
predictedFieldIndex = self.getFieldNames().index(self.predictedField)
actual = modelResult.sensorInput.dataRow[predictedFieldIndex]
dthandler = lambda obj: obj.isoformat() if isinstance(obj,
datetime.datetime) \
else None
record = {"_id":self._iteration,
"actual": actual,
"SPBUOut":SPBUOut,
"overlaps":overlaps,
"predictedCols": self.prevPredictedCols,
"tpInfActive": tpInfActiveCells,
"tpLrnActive": tpLrnActiveCells,
"tpPredicted": self.prevTPPredictedCells,
"tpInfPredT_1":tpInfPredT_1,
"tpInfPredT":tpInfPredT,
"permanences": permanences,
"overlaps": overlaps,
"inferences": json.dumps(modelResult.inferences),
"record":json.dumps(modelResult.rawInput,
default=dthandler),
"fieldActivations":fieldActivations,
#TODO: for some reason, field predictions don't need to be shifted??
"fieldPredictions": fieldPredictions,
"verboseOutput": verboseOutput,
}
collection.insert(record)
self._dataQ.put(record)
self.prevPredictedCols = predictedCols
self.prevTPPredictedCells = tpPredCells
self.prevPredictedConfs = predictedConfidences
#self.prevTPPredicted = tpPredCells
self.prevTPPredicted = None
self.prevFieldPred = fieldPredictions
self._iteration += 1
gevent.sleep(0)
os.close(oldC)
cOut.close()
self._isFinished.set()
def getDataAtTime(self, dataInput):
timestep = int(dataInput['timestep'])
collection = ExperimentDB.getExperimentDB(self.name)
experimentData = collection.find_one({"_id":timestep})
return experimentData
def _runExperimentLoadLoop(self, queue):
collection = ExperimentDB.getExperimentDB(self.name)
experimentData = collection.find()
for record in experimentData:
self._dataQ.put(record)
gevent.sleep(0)
