def getProtosAtTime(self, timestep): collection = ExperimentDB.getExperimentDB(self.name) experimentData = collection.find_one({"_id":timestep}) experimentData['protos'] = [] predictedField = self._modelDescription["predictedField"] predictedFieldIndex = self.getFieldNames().index(predictedField) dists = json.loads(experimentData['classificationDist']) for distId in json.loads(experimentData["classificationIdx"]): distSurroundingValues = collection.find({"_id": { "$gt": distId-10, "$lt": distId+10 }}).sort('_id', pymongo.ASCENDING) experimentData['protos'].append(dict( ids=[], actual=[], prediction=[], anomaly=[], anomalyLabel=[], dist=dists.pop(0), index=distId )) protosId = len(experimentData['protos']) - 1 for distSurroundingValue in distSurroundingValues: inferences = json.loads(distSurroundingValue["inferences"]) actual = distSurroundingValue["actual"] inference = inferences[InferenceElement.multiStepBestPredictions] step = min(inference.iterkeys()) prediction =inference[step] if prediction is None: prediction = 0.0 anomaly = inferences[InferenceElement.anomalyScore] anomalyLabel = inferences[InferenceElement.anomalyLabel] experimentData['protos'][protosId]["ids"].append(distSurroundingValue["_id"]) experimentData['protos'][protosId]["actual"].append(actual) experimentData['protos'][protosId]["prediction"].append(prediction) experimentData['protos'][protosId]["anomaly"].append(anomaly) experimentData['protos'][protosId]["anomalyLabel"].append(anomalyLabel) return experimentData
def _runExperimentLoop(self, queue): collection = ExperimentDB.getExperimentDB(self.name) while self._maxiterations == -1 or self._iteration <= self._maxiterations: try: # Get next record record = self._dataset.getNextRecord() except StopIteration: self._isFinished.set() return None if self._stop.isSet(): break # Feed record to model and get prediction modelResult = self._model.run(record) if modelResult is None: continue if modelResult.inferences[InferenceElement.anomalyVector] is not None: modelResult.inferences[InferenceElement.anomalyVector] = \ modelResult.inferences[InferenceElement.anomalyVector].nonzero()[0].tolist() distances = self._model._classifier_distances #classifier.getSelf().getLatestDistances() sortedDistIdx = [] sortedDists = [] if distances is not None and len(distances) > 0: sortedDistIdx = distances.argsort() sortedDists = distances[sortedDistIdx[:5]].tolist() idList = self._model._classifier_indexes #classifier.getSelf().getParameter('categoryRecencyList') if len(idList) > 0: sortedDistIdx = [ \ idList[i] + self._model._classificationDelay - 1\ for i in sortedDistIdx[:min(5, len(sortedDistIdx))]] else: sortedDistIdx = [] #matrix = classifier.getSelf()._knn._Memory #print matrix.shape #print "Index: %s" % (sorted) #if len(sorted) > 0: # print matrix.getRow(int(sorted[0])) # print matrix.getRow(int(sorted[0])).nonzero()[0] predictedField = self._modelDescription["predictedField"] predictedFieldIndex = self.getFieldNames().index(predictedField) modelResult.inferences['encodings'] = None modelResult.sensorInput.dataEncodings = None actual = modelResult.sensorInput.dataRow[predictedFieldIndex] dbelem = {"_id":self._iteration, "actual": actual, "inferences": json.dumps(modelResult.inferences), "classificationIdx":json.dumps(sortedDistIdx), "classificationDist":json.dumps(sortedDists) } collection.insert(dbelem) self._dataQ.put(dbelem) self._iteration += 1 gevent.sleep(0) self._isFinished.set()
def getDetailsAtTime(self, timestep): collection = ExperimentDB.getExperimentDB(self.name) return collection.find_one({"_id":timestep})
def getDataAtTime(self, dataInput): timestep = int(dataInput['timestep']) collection = ExperimentDB.getExperimentDB(self.name) experimentData = collection.find_one({"_id": timestep}) return experimentData
def _runExperimentLoadLoop(self, queue): collection = ExperimentDB.getExperimentDB(self.name) experimentData = collection.find() for record in experimentData: self._dataQ.put(record) gevent.sleep(0)
def _runExperimentLoop(self, queue): self.prevFieldPred = {} self._model.resetSequenceStates() cOut = os.fdopen(os.open("/tmp/cerebro.cout", os.O_RDWR | os.O_CREAT), 'w+') oldC = os.dup(1) collection = ExperimentDB.getExperimentDB(self.name) while self._maxiterations == -1 or self._iteration <= self._maxiterations: try: # Get next record record = self._dataset.getNextRecord() except StopIteration: self._isFinished.set() return None if self._stop.isSet(): break # Feed record to model and get prediction. Capture all the stdout as well os.dup2(cOut.fileno(), 1) modelResult = self._model.run(record) os.dup2(oldC, 1) cOut.seek(0) verboseOutput = cOut.read() cOut.truncate(0) modelResult.inferences['encodings'] = None modelResult.sensorInput.dataEncodings = None model = self._model sensor = model._getSensorRegion() sp = model._getSPRegion() tp = model._getTPRegion() cl = model._getClassifierRegion() spImp = None tpImp = None if sp is not None: spImp = sp.getSelf()._sfdr if tp is not None: tpImp = tp.getSelf()._tfdr clImp = cl.getSelf()._claClassifier #Copy all the pertinent data sourceScalars = copy.deepcopy(sensor.getOutputData('sourceOut')) sensorBits = sensor.getOutputData('dataOut') sensorBUOut = sensorBits.nonzero()[0].tolist() SPBUOut = [] nConnectedInputs = [] overlaps = [] if spImp is not None: SPBUOut = sp.getOutputData('bottomUpOut').nonzero()[0].tolist() nConnectedInputs = spImp._allConnectedM.nNonZerosPerRow()[SPBUOut].astype('int32').tolist() overlaps = zip(SPBUOut, spImp._overlapsNoBoost[SPBUOut].astype('int32').tolist()) TPTDOut = tp.getOutputData('topDownOut') if tp else None sensorTDIn = sensor.getInputData('temporalTopDownIn') permanences = {} predictedCols = () predictedConfidences = () tpInfActiveCells = () tpLrnActiveCells = () tpPredCells = [] if TPTDOut is not None: predictedCols = TPTDOut.nonzero()[0].tolist() predictedConfidences = TPTDOut[predictedCols].tolist() tpInfActiveCells = self._formatActiveCells(tpImp.infActiveState['t']) tpLrnActiveCells = self._formatActiveCells(tpImp.lrnActiveState['t']) tpInfPredT_1 = self._formatActiveCells(tpImp.infPredictedState['t-1']) tpInfPredT = self._formatActiveCells(tpImp.infPredictedState['t']) tpPredCells = tpImp.infPredictedState['t'].nonzero()[0].tolist() sensorPredBits = [] if sensorTDIn is not None: sensorPredBits = sensorTDIn if self.prevPredictedCols is None: self.prevPredictedCols = [] self.prevTPPredictedCells = [] self.prevPredictedConfs = [] self.prevTPPredicted = [] clPattern = clImp._patternNZHistory[-1] step = clImp.steps[0] bitHistories = {} fieldActivations = {} fieldPredictions = {} for fieldName, (start, stop) in self.fieldRanges.iteritems(): nzBits = sensorBits[start:stop].nonzero()[0] fieldActivations[fieldName] = nzBits.tolist() nzBits = sensorPredBits[start:stop].nonzero()[0] fieldPredictions[fieldName] = nzBits.tolist() predictedField = self._modelDescription["predictedField"] predictedFieldIndex = self.getFieldNames().index(self.predictedField) actual = modelResult.sensorInput.dataRow[predictedFieldIndex] dthandler = lambda obj: obj.isoformat() if isinstance(obj, datetime.datetime) \ else None record = {"_id":self._iteration, "actual": actual, "SPBUOut":SPBUOut, "overlaps":overlaps, "predictedCols": self.prevPredictedCols, "tpInfActive": tpInfActiveCells, "tpLrnActive": tpLrnActiveCells, "tpPredicted": self.prevTPPredictedCells, "tpInfPredT_1":tpInfPredT_1, "tpInfPredT":tpInfPredT, "permanences": permanences, "overlaps": overlaps, "inferences": json.dumps(modelResult.inferences), "record":json.dumps(modelResult.rawInput, default=dthandler), "fieldActivations":fieldActivations, #TODO: for some reason, field predictions don't need to be shifted?? "fieldPredictions": fieldPredictions, "verboseOutput": verboseOutput, } collection.insert(record) self._dataQ.put(record) self.prevPredictedCols = predictedCols self.prevTPPredictedCells = tpPredCells self.prevPredictedConfs = predictedConfidences #self.prevTPPredicted = tpPredCells self.prevTPPredicted = None self.prevFieldPred = fieldPredictions self._iteration += 1 gevent.sleep(0) os.close(oldC) cOut.close() self._isFinished.set()
def getDataAtTime(self, dataInput): timestep = int(dataInput['timestep']) collection = ExperimentDB.getExperimentDB(self.name) experimentData = collection.find_one({"_id":timestep}) return experimentData