def recomputeOccupancies(sensorId): sensorObj = SensorDb.getSensorObj(sensorId) if sensorObj.getSensorStatus() == PURGING: return {"status": "NOK", "ErrorMessage": "Sensor is PURGING"} elif sensorObj.getSensorStatus() == RECOMPUTING: return {"status": "NOK", "ErrorMessage": "Sensor is RECOMPUTING"} else: SensorDb.setSensorStatus(sensorId, RECOMPUTING) return {"status": "OK", "sensors": SensorDb.getAllSensors()}
def purgeSensors(): """ Run in an infinite loop purging all the sensors that are marked for purging. This is invoked as a process from main. """ from Defines import PURGING from Defines import RECOMPUTING import time while True: for sensor in DbCollections.getSensors().find(): sensorObj = Sensor(sensor) if sensorObj.getSensorStatus() == PURGING: SensorDb.purgeSensor(sensorObj) elif sensorObj.getSensorStatus() == RECOMPUTING: RecomputeOccupancies.recomputeOccupanciesWorker( sensorObj.getSensorId()) time.sleep(30)
def disconnectSensor(sensorId): """ Send a sensor a command to exit. URL Path: sensorId -- the session ID of the login session. URL Args: None Request Body: Contains authentication information for the agent that is authorized to arm and disarm the sensor: - agentName: Name of the agent to arm/disarm sensor. - key : password of the agent to arm/disarm the sensor. HTTP Return Codes: - 200 OK: invocation was successful. - 403 Forbidden: authentication failure - 400 Bad request: Sensor is not a streaming sensor. Example Invocation: :: params = {} params["agentName"] = "NIST_ESC" params["key"] = "ESC_PASS" r = requests.post("https://"+ host + ":" + str(443) + "/sensorcontrol/disconnectSensor/" + self.sensorId + "/LTE:70315780:713315880",data=json.dumps(params),verify=False) """ try: util.debugPrint("disconnectSensor: sensorId " + sensorId) requestStr = request.data if requestStr is None: abort(400) accountData = json.loads(requestStr) if not authentication.authenticateSensorAgent(accountData): abort(403) sensorConfig = SensorDb.getSensorObj(sensorId) if sensorConfig is None: abort(404) if not sensorConfig.isStreamingEnabled(): abort(400) sendCommandToSensor( sensorId, json.dumps({ "sensorId": sensorId, "command": "exit" })) return jsonify({STATUS: OK}) except: print "Unexpected error:", sys.exc_info()[0] print sys.exc_info() traceback.print_exc() util.logStackTrace(sys.exc_info()) raise
def getSensorDataSummary(sensorId, locationMessage): sensor = SensorDb.getSensorObj(sensorId) if sensor is None: return {STATUS: NOK, ERROR_MESSAGE: "Sensor Not found"} measurementType = sensor.getMeasurementType() tzId = locationMessage[TIME_ZONE_KEY] acquisitionCount = LocationMessage.getMessageCount(locationMessage) util.debugPrint("AquistionCount " + str(acquisitionCount)) if acquisitionCount == 0: return { "status": "OK", "minOccupancy": 0, "tStartReadings": 0, "tStartLocalTime": 0, "tStartLocalTimeFormattedTimeStamp": "UNKNOWN", "tStartDayBoundary": 0, "tEndDayBoundary": 0, "tEndReadings": 0, "tEndLocalTimeFormattedTimeStamp": "UNKNOWN", "maxOccupancy": 0, "measurementType": measurementType, "isStreamingEnabled": sensor.isStreamingEnabled(), "sensorStatus": sensor.getSensorStatus(), COUNT: 0 } minTime = LocationMessage.getFirstDataMessageTimeStamp(locationMessage) maxTime = LocationMessage.getLastDataMessageTimeStamp(locationMessage) tStartDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( minTime, tzId) (minLocalTime, tStartLocalTimeTzName) = timezone.getLocalTime(minTime, tzId) tEndDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( maxTime, tzId) tstampMin = timezone.formatTimeStampLong(minTime, tzId) tstampMax = timezone.formatTimeStampLong(maxTime, tzId) retval = { "status": "OK", "maxOccupancy": 0, "minOccupancy": 0, "tStartReadings": minTime, "tStartLocalTime": minLocalTime, "tStartLocalTimeFormattedTimeStamp": tstampMin, "tStartDayBoundary": tStartDayBoundary, "tEndDayBoundary": tEndDayBoundary, "tEndReadings": maxTime, "tEndLocalTimeFormattedTimeStamp": tstampMax, "measurementType": measurementType, "isStreamingEnabled": sensor.isStreamingEnabled(), "sensorStatus": sensor.getSensorStatus(), COUNT: acquisitionCount } return retval
def testArmSensor(sensorId, sessionId): """ URL Path: sessionId -- the session ID of the login session. sensorId -- the sensorId URL Args: None Request Body: - agentName: Name of the agent to arm/disarm sensor. - key : Key (password) of the agent to arm/disarm the sensor. HTTP Return Codes: - 200 OK: invocation was successful. - 403 Forbidden: authentication failure - 400 Bad request: Sensor is not a streaming sensor. Example Invocation: :: params = {} params["agentName"] = "NIST_ESC" params["key"] = "ESC_PASS" url = "https://{host}:8443/admin/armSensor/{self.sensorId}" r = requests.post(url, data=json.dumps(params), verify=False) """ try: if not authentication.checkSessionId(sessionId, ADMIN): abort(403) sensorConfig = SensorDb.getSensorObj(sensorId) if sensorConfig is None: abort(404) if not sensorConfig.isStreamingEnabled(): abort(400) persistent = request.args.get("persistent") if persistent is None: persistent = "false" DataStreamSharedState.sendCommandToSensor( sensorId, json.dumps({ "sensorId": sensorId, "command": "arm", "persistent": persistent })) return jsonify({STATUS: OK}) except: print "Unexpected error:", sys.exc_info()[0] print sys.exc_info() traceback.print_exc() util.logStackTrace(sys.exc_info()) raise
def getCaptureEvents(): retval = {} captureEventCount = {} for sensorId in SensorDb.getAllSensorIds(): captureEvent = DbCollections.getCaptureDb().find({SENSOR_ID: sensorId}) if captureEvent is None: captureEventCount[sensorId] = 0 else: captureEventCount[sensorId] = captureEvent.count() retval[STATUS] = "OK" retval["captureEventCount"] = captureEventCount return retval
def postCaptureEvent(): """ Handle post of a capture event from a sensor URL Path: - None URL Parameters: - None Request Body: - CaptureEvent JSON structure which includes the sensor ID and sensor key. These are used for verifying the request. See MSOD specification for definition of CaptureEvent structure. HTTP Return Codes: - 200 OK. A JSON Document containing {"status":"OK"} is returned. """ try: requestStr = request.data if requestStr is None or requestStr == "": util.debugPrint("postCaptureEvent - request body not found") abort(400) util.debugPrint("postCaptureEvent " + requestStr) captureEvent = json.loads(requestStr) if SENSOR_ID not in captureEvent or SENSOR_KEY not in captureEvent: util.debugPrint("postCaptureEvent - missing a required field") abort(400) sensorId = captureEvent[SENSOR_ID] sensorConfig = SensorDb.getSensorObj(sensorId) if sensorConfig is None: util.debugPrint("postCaptureEvent - sensor not found") abort(404) sensorKey = captureEvent[SENSOR_KEY] if not authentication.authenticateSensor(sensorId, sensorKey): abort(403) return jsonify(CaptureDb.insertEvent(sensorId, captureEvent)) except: print "Unexpected error:", sys.exc_info()[0] print sys.exc_info() traceback.print_exc() util.logStackTrace(sys.exc_info()) raise
def getSocketServerPort(sensorId): retval = {} global memCache if memCache is None: memCache = MemCache() sensor = SensorDb.getSensorObj(sensorId) print "sensorStatus ", sensor.getSensorStatus() if sensor is None or sensor.getSensorStatus() != ENABLED \ or not sensor.isStreamingEnabled(): retval["port"] = -1 return retval retval["port"] = STREAMING_SERVER_PORT return retval
def getFreqBandsWorker(sessionId): try: if not Config.isConfigured(): util.debugPrint("Please configure system") return make_response("Please configure system", 500) if not authentication.checkSessionId(sessionId, ADMIN): return make_response("Session not found.", 403) return jsonify(SensorDb.getFreqBands()) except: print "Unexpected error:", sys.exc_info()[0] print sys.exc_info() traceback.print_exc() util.logStackTrace(sys.exc_info()) raise
def updateSensorWorker(sessionId): try: util.debugPrint("updateSensor") if not Config.isConfigured(): util.debugPrint("Please configure system") return make_response("Please configure system", 500) if not authentication.checkSessionId(sessionId, ADMIN): return make_response("Session not found.", 403) requestStr = request.data sensorConfig = json.loads(requestStr) return jsonify(SensorDb.updateSensor(sensorConfig)) except: print "Unexpected error:", sys.exc_info()[0] print sys.exc_info() traceback.print_exc() util.logStackTrace(sys.exc_info()) raise
def getSensorInfoWorker(sessionId): try: if not authentication.checkSessionId(sessionId, ADMIN): return make_response("Session not found", 403) lastMessageFlagStr = request.args.get("getFirstLastMessages") if lastMessageFlagStr is not None and lastMessageFlagStr == "true": lastMessageFlag = True else: lastMessageFlag = False response = SensorDb.getSensors(getMessageDates=lastMessageFlag) return jsonify(response) except: print "Unexpected error:", sys.exc_info()[0] print sys.exc_info() traceback.print_exc() util.logStackTrace(sys.exc_info()) raise
def resetNoiseFloor(sensorId, noiseFloor): if SessionLock.isAcquired(): return { "status": "NOK", "StatusMessage": "Session is locked. Try again later." } SessionLock.acquire() try: dataMessages = DbCollections.getDataMessages(sensorId).find( {SENSOR_ID: sensorId}) if dataMessages is None: return {"status": "OK", "StatusMessage": "No Data Found"} SessionLock.release() for jsonData in dataMessages: DbCollections.getDataMessages(sensorId).update( {"_id": jsonData["_id"]}, {"$set": jsonData}, upsert=False) finally: SessionLock.release() return {"status": "OK", "sensors": SensorDb.getAllSensors()}
def getDailyMaxMinMeanStats(sensorId, lat, lon, alt, tstart, ndays, sys2detect, fmin, fmax, subBandMinFreq, subBandMaxFreq): locationMessage = DbCollections.getLocationMessages().find_one({ SENSOR_ID: sensorId, LAT: lat, LON: lon, ALT: alt }) if locationMessage is None: return {STATUS: NOK, ERROR_MESSAGE: "Location Information Not Found"} locationMessageId = str(locationMessage["_id"]) tZId = locationMessage[TIME_ZONE_KEY] tmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(tstart, tZId) startMessage = DbCollections.getDataMessages(sensorId).find_one() result = {} result[STATUS] = OK values = {} for day in range(0, ndays): tstart = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( tmin + day * SECONDS_PER_DAY, tZId) tend = tstart + SECONDS_PER_DAY queryString = { LOCATION_MESSAGE_ID: locationMessageId, TIME: { '$gte': tstart, '$lte': tend }, FREQ_RANGE: msgutils.freqRange(sys2detect, fmin, fmax) } cur = DbCollections.getDataMessages(sensorId).find(queryString) # cur.batch_size(20) if startMessage['mType'] == FFT_POWER: stats = compute_daily_max_min_mean_stats_for_fft_power(cur) else: stats = compute_daily_max_min_mean_median_stats_for_swept_freq( cur, subBandMinFreq, subBandMaxFreq) # gap in readings. continue. if stats is None: continue (cutoff, dailyStat) = stats values[day * 24] = dailyStat # Now compute the next interval after the last one (if one exists) tend = tmin + SECONDS_PER_DAY * ndays queryString = { LOCATION_MESSAGE_ID: locationMessageId, TIME: { '$gte': tend }, FREQ_RANGE: msgutils.freqRange(sys2detect, fmin, fmax) } msg = DbCollections.getDataMessages(sensorId).find_one(queryString) if msg is None: result["nextTmin"] = tmin else: nextTmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( msg[TIME], tZId) result["nextTmin"] = nextTmin # Now compute the previous interval before this one. prevMessage = msgutils.getPrevAcquisition(startMessage) if prevMessage is not None: newTmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( prevMessage[TIME] - SECONDS_PER_DAY * ndays, tZId) queryString = { LOCATION_MESSAGE_ID: locationMessageId, TIME: { '$gte': newTmin }, FREQ_RANGE: msgutils.freqRange(sys2detect, fmin, fmax) } msg = DbCollections.getDataMessages(sensorId).find_one(queryString) else: msg = startMessage sensor = SensorDb.getSensorObj(sensorId) channelCount = sensor.getChannelCount(sys2detect, fmin, fmax) result[STATUS] = OK result["prevTmin"] = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( msg[TIME], tZId) result["tmin"] = tmin result["maxFreq"] = fmin result["minFreq"] = fmax result["cutoff"] = cutoff result[CHANNEL_COUNT] = channelCount result["startDate"] = timezone.formatTimeStampLong(tmin, tZId) result["values"] = values util.debugPrint(result) return result
def getHourlyMaxMinMeanStats(sensorId, startTime, sys2detect, fmin, fmax, subBandMinFreq, subBandMaxFreq, sessionId): sensor = SensorDb.getSensor(sensorId) if sensor is None: return {STATUS: NOK, ERROR_MESSAGE: "Sensor Not Found"} tstart = int(startTime) fmin = int(subBandMinFreq) fmax = int(subBandMaxFreq) freqRange = msgutils.freqRange(sys2detect, fmin, fmax) queryString = { SENSOR_ID: sensorId, TIME: { '$gte': tstart }, FREQ_RANGE: freqRange } util.debugPrint(queryString) startMessage = DbCollections.getDataMessages(sensorId).find_one( queryString) if startMessage is None: errorStr = "Start Message Not Found" util.debugPrint(errorStr) response = {STATUS: NOK, ERROR_MESSAGE: "No data found"} return response locationMessageId = DataMessage.getLocationMessageId(startMessage) retval = {STATUS: OK} values = {} locationMessage = DbCollections.getLocationMessages().find_one( {"_id": locationMessageId}) tZId = LocationMessage.getTimeZone(locationMessage) tmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( tstart, LocationMessage.getTimeZone(locationMessage)) for hour in range(0, 23): dataMessages = DbCollections.getDataMessages(sensorId).find({ "t": { "$gte": tmin + hour * SECONDS_PER_HOUR }, "t": { "$lte": (hour + 1) * SECONDS_PER_HOUR }, FREQ_RANGE: freqRange }) if dataMessages is not None: stats = compute_stats_for_fft_power(dataMessages) (nChannels, maxFreq, minFreq, cutoff, result) = stats values[hour] = result retval["values"] = values # Now compute the next interval after the last one (if one exists) tend = tmin + SECONDS_PER_DAY queryString = { SENSOR_ID: sensorId, TIME: { '$gte': tend }, FREQ_RANGE: freqRange } msg = DbCollections.getDataMessages(sensorId).find_one(queryString) if msg is None: result["nextTmin"] = tmin else: nextTmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( msg[TIME], tZId) result["nextTmin"] = nextTmin # Now compute the previous interval before this one. prevMessage = msgutils.getPrevAcquisition(startMessage) if prevMessage is not None: newTmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( prevMessage[TIME] - SECONDS_PER_DAY, tZId) queryString = { SENSOR_ID: sensorId, TIME: { '$gte': newTmin }, FREQ_RANGE: msgutils.freqRange(sys2detect, fmin, fmax) } msg = DbCollections.getDataMessages(sensorId).find_one(queryString) else: msg = startMessage result[STATUS] = OK result["prevTmin"] = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( msg[TIME], tZId) result["tmin"] = tmin result["maxFreq"] = maxFreq result["minFreq"] = minFreq result["cutoff"] = cutoff result[CHANNEL_COUNT] = nChannels result["startDate"] = timezone.formatTimeStampLong(tmin, tZId) result["values"] = values return result
def getSensorData(ws): """ Handle sensor data streaming requests from the web browser. """ try: util.debugPrint("DataStreamng:getSensorData") global memCache if memCache is None: memCache = MemCache() token = ws.receive() print "token = ", token parts = token.split(":") if parts is None or len(parts) < 5: ws.close() return sessionId = parts[0] if not authentication.checkSessionId(sessionId, "user"): ws.close() return import SessionLock if SessionLock.findSessionByRemoteAddr(sessionId) is not None: ws.send(dumps({"status": "Streaming session already open"})) ws.close() return sensorId = parts[1] systemToDetect = parts[2] minFreq = int(parts[3]) maxFreq = int(parts[4]) util.debugPrint("sensorId " + sensorId) memCache.incrementStreamingListenerCount(sensorId) sensorObj = SensorDb.getSensorObj(sensorId) if sensorObj is None: ws.send(dumps({"status": "Sensor not found: " + sensorId})) bandName = systemToDetect + ":" + str(minFreq) + ":" + str(maxFreq) util.debugPrint("isStreamingEnabled = " + str(sensorObj.isStreamingEnabled())) lastDataMessage = memCache.loadLastDataMessage(sensorId, bandName) key = sensorId + ":" + bandName if key not in lastDataMessage or not sensorObj.isStreamingEnabled(): ws.send( dumps({ "status": "NO_DATA: Data message not found or streaming not enabled" })) else: ws.send(dumps({"status": "OK"})) util.debugPrint("DataStreaming lastDataMessage: " + str(lastDataMessage[key])) ws.send(str(lastDataMessage[key])) lastdatatime = -1 drift = 0 while True: secondsPerFrame = sensorObj.getStreamingSecondsPerFrame() lastdataseen = memCache.loadLastDataSeenTimeStamp( sensorId, bandName) if key in lastdataseen and lastdatatime != lastdataseen[key]: lastdatatime = lastdataseen[key] sensordata = memCache.loadSensorData(sensorId, bandName) memCache.incrementDataConsumedCounter(sensorId, bandName) currentTime = time.time() lastdatasent = currentTime drift = drift + (currentTime - lastdatasent) - secondsPerFrame ws.send(sensordata[key]) # If we drifted, send the last reading again to fill in. if drift < 0: drift = 0 if drift > secondsPerFrame: if APPLY_DRIFT_CORRECTION: util.debugPrint("Drift detected") ws.send(sensordata[key]) drift = 0 sleepTime = secondsPerFrame gevent.sleep(sleepTime) except: traceback.print_exc() ws.close() util.debugPrint("Error writing to websocket") finally: memCache.decrementStreamingListenerCount(sensorId)
def readFromInput(bbuf, conn): util.debugPrint("DataStreaming:readFromInput") soc = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sensorCommandDispatcherPid = None memCache = MemCache() try: while True: lengthString = "" while True: lastChar = bbuf.readChar() if lastChar is None: time.sleep(0.1) return if len(lengthString) > 1000: raise Exception("Formatting error") if lastChar == '{': headerLength = int(lengthString.rstrip()) break else: lengthString += str(lastChar) jsonStringBytes = "{" while len(jsonStringBytes) < headerLength: jsonStringBytes += str(bbuf.readChar()) jsonData = json.loads(jsonStringBytes) if not any(k in jsonData for k in (TYPE, SENSOR_ID, SENSOR_KEY)): err = "Sensor Data Stream: Missing a required field" util.errorPrint(err) util.errorPrint("Invalid message -- closing connection: " + json.dumps(jsonData, indent=4)) raise Exception("Invalid message") return sensorId = jsonData[SENSOR_ID] global mySensorId if mySensorId is None: mySensorId = sensorId elif mySensorId != sensorId: raise Exception("Sensor ID mismatch " + mySensorId + " / " + sensorId) sensorKey = jsonData[SENSOR_KEY] if not authentication.authenticateSensor(sensorId, sensorKey): util.debugPrint("jsonData " + json.dumps(jsonData, indent=4)) util.errorPrint("Sensor authentication failed: " + sensorId + " sensorKey " + sensorKey) raise Exception("Authentication failure") return if memCache.getStreamingServerPid(sensorId) == -1: memCache.setStreamingServerPid(sensorId) elif memCache.getStreamingServerPid(sensorId) != os.getpid(): util.errorPrint( "Handling connection for this sensor already " + str(os.getpid())) try: os.kill(memCache.getStreamingServerPid(sensorId), signal.SIGKILL) memCache.setStreamingServerPid(sensorId) except: print "Unexpected error:", sys.exc_info()[0] print sys.exc_info() traceback.print_exc() util.logStackTrace(sys.exc_info()) util.debugPrint( "Problem killing process " + str(memCache.getStreamingServerPid(sensorId))) if memCache.getStreamingCommandDispatcherPid(sensorId) != -1: try: os.kill( memCache.getStreamingCommandDispatcherPid( sensorId), signal.SIGKILL) except: memCache.removeStreamingCommandDispatcherPid(sensorId) util.debugPrint("Process not found. Proceeding ") util.debugPrint("DataStreaming: Message = " + dumps(jsonData, sort_keys=True, indent=4)) sensorObj = SensorDb.getSensorObj(sensorId) if not sensorObj.isStreamingEnabled( ) or sensorObj.getStreamingParameters() is None: raise Exception("Streaming is not enabled") return # the last time a data message was inserted if jsonData[TYPE] == DATA: util.debugPrint("pubsubPort: " + str(memCache.getPubSubPort(sensorId))) if "Sys2Detect" not in jsonData: jsonData[SYS_TO_DETECT] = "LTE" DataMessage.init(jsonData) t = Process(target=runSensorCommandDispatchWorker, args=(conn, sensorId)) t.start() sensorCommandDispatcherPid = t.pid childPids.append(sensorCommandDispatcherPid) cutoff = DataMessage.getThreshold(jsonData) n = DataMessage.getNumberOfFrequencyBins(jsonData) sensorId = DataMessage.getSensorId(jsonData) lastDataMessageReceivedAt[sensorId] = time.time() lastDataMessageOriginalTimeStamp[ sensorId] = DataMessage.getTime(jsonData) # Check if the measurement type reported by the sensor # matches that in the sensordb measurementType = DataMessage.getMeasurementType(jsonData) if sensorObj.getMeasurementType() != measurementType: err = "Measurement type mismatch " err += sensorObj.getMeasurementType() err += " / " err += measurementType raise Exception(err) # Check if the time per measurement reported by the sensor # matches that in the sensordb timePerMeasurement = sensorObj.getStreamingSecondsPerFrame() util.debugPrint("StreamingServer: timePerMeasurement " + str(timePerMeasurement)) if timePerMeasurement != DataMessage.getTimePerMeasurement( jsonData): err = "TimePerMeasurement mismatch " err += str(timePerMeasurement) err += "/" err += str(DataMessage.getTimePerMeasurement(jsonData)) raise Exception(err) # The sampling interval for write to the database. s = sensorObj.getStreamingSamplingIntervalSeconds() streamingSamplingIntervalSeconds = s # The number of measurements per capture measurementsPerCapture = int(streamingSamplingIntervalSeconds / timePerMeasurement) util.debugPrint("StreamingServer: measurementsPerCapture " + str(measurementsPerCapture)) # The number of power value samples per capture. samplesPerCapture = int( (streamingSamplingIntervalSeconds / timePerMeasurement) * n) # The number of spectrums per frame sent to the browser. spectrumsPerFrame = 1 jsonData[SPECTRUMS_PER_FRAME] = spectrumsPerFrame # The streaming filter of the sensor (MAX_HOLD or AVG) jsonData[STREAMING_FILTER] = sensorObj.getStreamingFilter() # The band name sys2detect:minfreq:maxfreq string for the # reported measurement. bandName = DataMessage.getFreqRange(jsonData) # Keep a copy of the last data message for periodic insertion # into the db memCache.setLastDataMessage(sensorId, bandName, json.dumps(jsonData)) # captureBufferCounter is a pointer into the capture buffer. captureBufferCounter = 0 powerArrayCounter = 0 timingCounter = 0 # initialize the "prev occupancy array" prevOccupancyArray = [-1 for i in range(0, n)] occupancyArray = [0 for i in range(0, n)] occupancyTimer = time.time() if sensorId not in lastDataMessage: lastDataMessage[sensorId] = jsonData powerVal = [0 for i in range(0, n)] startTime = time.time() sensorObj = SensorDb.getSensorObj(sensorId) if sensorObj is None: raise Exception("Sensor not found") if sensorObj.getSensorStatus() == DISABLED: bbuf.close() raise Exception("Sensor is disabled") if not sensorObj.isStreamingEnabled(): raise Exception("Streaming is disabled") enb = sensorObj.isStreamingCaptureEnabled() isStreamingCaptureEnabled = enb util.debugPrint("isStreamingCaptureEnabled : " + str(enb) + " samplesPerCapture " + str(samplesPerCapture)) if isStreamingCaptureEnabled: sensorData = [0 for i in range(0, samplesPerCapture)] while True: data = bbuf.readByte() if isStreamingCaptureEnabled: sensorData[captureBufferCounter] = data powerVal[powerArrayCounter] = data now = time.time() if isStreamingCaptureEnabled and captureBufferCounter + 1 == samplesPerCapture: # Buffer is full so push the data into mongod. util.debugPrint("Inserting Data message") captureBufferCounter = 0 # Time offset since the last data message was received. timeOffset = time.time( ) - lastDataMessageReceivedAt[sensorId] # Offset the capture by the time since the DataMessage header was received. lastDataMessage[sensorId]["t"] = int( lastDataMessageOriginalTimeStamp[sensorId] + int(timeOffset)) lastDataMessage[sensorId][ "nM"] = measurementsPerCapture lastDataMessage[sensorId]["mPar"]["td"] = int( now - occupancyTimer) lastDataMessage[sensorId]["mPar"][ "tm"] = timePerMeasurement headerStr = json.dumps(lastDataMessage[sensorId], indent=4) util.debugPrint("StreamingServer: headerStr " + headerStr) headerLength = len(headerStr) if isStreamingCaptureEnabled: # Start the db operation in a seperate process p = Process(target=populate_db.put_data, args=(headerStr, headerLength), kwargs={ "filedesc": None, "powers": sensorData }) p.start() lastDataMessageInsertedAt[sensorId] = time.time() occupancyTimer = time.time() else: captureBufferCounter = captureBufferCounter + 1 if data > cutoff: occupancyArray[powerArrayCounter] = 1 else: occupancyArray[powerArrayCounter] = 0 # print "occupancyArray", occupancyArray if (powerArrayCounter + 1) == n: # Get the occupancy subscription counter. if memCache.getSubscriptionCount(sensorId) != 0: if not np.array_equal(occupancyArray, prevOccupancyArray): port = memCache.getPubSubPort(sensorId) soc.sendto( json.dumps({sensorId: occupancyArray}), ("localhost", port)) prevOccupancyArray = np.array(occupancyArray) # sending data as CSV values to the browser listenerCount = memCache.getStreamingListenerCount( sensorId) if listenerCount > 0: sensordata = str(powerVal)[1:-1].replace(" ", "") memCache.setSensorData(sensorId, bandName, sensordata) # Record the occupancy for the measurements. # Allow for 10% jitter. if timingCounter == 1000 and checkForDataRate: if (((now - startTime) / 1000.0 < timePerMeasurement / 2) or ((now - startTime) / 1000.0 > timePerMeasurement * 2)): print " delta ", now - startTime, "global counter ", powerArrayCounter util.errorPrint( "Data coming in too fast or too slow - sensor configuration problem." ) raise Exception( "Data coming in too fast - sensor configuration problem." ) else: startTime = now lastdataseen = now if listenerCount > 0: memCache.setLastDataSeenTimeStamp( sensorId, bandName, lastdataseen) powerArrayCounter = 0 else: powerArrayCounter = powerArrayCounter + 1 timingCounter = timingCounter + 1 elif jsonData[TYPE] == SYS: util.debugPrint( "DataStreaming: Got a System message -- adding to the database" ) populate_db.put_data(jsonStringBytes, headerLength) elif jsonData[TYPE] == LOC: util.debugPrint( "DataStreaming: Got a Location Message -- adding to the database" ) populate_db.put_data(jsonStringBytes, headerLength) finally: util.debugPrint("Closing sockets for sensorId " + sensorId) memCache.removeStreamingServerPid(sensorId) port = memCache.getSensorArmPort(sensorId) sendCommandToSensor( sensorId, json.dumps({ "sensorId": sensorId, "command": "exit" })) memCache.releaseSensorArmPort(sensorId) bbuf.close() time.sleep(1) soc.close() # kill the command dispatcher for good measure. try: if sensorCommandDispatcherPid is not None: os.kill(sensorCommandDispatcherPid, signal.SIGKILL) except: util.debugPrint("Process not found. " + str(sensorCommandDispatcherPid))
def getBandDataSummary(sensorId, locationMessage, sys2detect, minFreq, maxFreq, mintime, dayCount=None): sensor = SensorDb.getSensorObj(sensorId) if sensor is None: return {STATUS: NOK, ERROR_MESSAGE: "Sensor Not found"} measurementType = sensor.getMeasurementType() tzId = locationMessage[TIME_ZONE_KEY] locationMessageId = str(locationMessage["_id"]) freqRange = msgutils.freqRange(sys2detect, minFreq, maxFreq) count = LocationMessage.getBandCount(locationMessage, freqRange) if count == 0: return { FREQ_RANGE: freqRange, COUNT: 0, "minFreq": minFreq, "maxFreq": maxFreq, SYSTEM_TO_DETECT: sys2detect } else: minOccupancy = LocationMessage.getMinBandOccupancy( locationMessage, freqRange) maxOccupancy = LocationMessage.getMaxBandOccupancy( locationMessage, freqRange) count = LocationMessage.getBandCount(locationMessage, freqRange) meanOccupancy = LocationMessage.getMeanOccupancy( locationMessage, freqRange) minTime = LocationMessage.getFirstMessageTimeStampForBand( locationMessage, freqRange) maxTime = LocationMessage.getLastMessageTimeStampForBand( locationMessage, freqRange) maxTimes = timezone.getLocalTime(maxTime, tzId) (tEndReadingsLocalTime, tEndReadingsLocalTimeTzName) = maxTimes tEndDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( maxTime, tzId) tStartDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( minTime, tzId) tstampMin = timezone.formatTimeStampLong(minTime, tzId) tstampMax = timezone.formatTimeStampLong(maxTime, tzId) retval = { "tStartDayBoundary": tStartDayBoundary, "tEndDayBoundary": tEndDayBoundary, "tStartReadings": minTime, "tStartLocalTime": minTime, "tStartLocalTimeFormattedTimeStamp": tstampMin, "tEndReadings": maxTime, "tEndReadingsLocalTime": maxTime, "tEndLocalTimeFormattedTimeStamp": tstampMax, "tEndDayBoundary": tEndDayBoundary, "maxOccupancy": maxOccupancy, "meanOccupancy": meanOccupancy, "minOccupancy": minOccupancy, "maxFreq": maxFreq, "minFreq": minFreq, SYSTEM_TO_DETECT: sys2detect, FREQ_RANGE: freqRange, "measurementType": measurementType, "active": sensor.isBandActive(sys2detect, minFreq, maxFreq), COUNT: count } return retval
def put_data(jsonString, headerLength, filedesc=None, powers=None, streamOccupancies=None): """ put data in the database. jsonString starts with {. If filedesc is None then the data part of the message is appended to the message (immediately follows it). Otherwise, the data is read from filedesc. """ start_time = time.time() if filedesc is None: # We are not reading from a file: # Assume we are given the message in the string with the data # tacked at the end of it. jsonStringBytes = jsonString[0:headerLength] else: jsonStringBytes = jsonString util.debugPrint("jsonStringBytes = " + jsonStringBytes) jsonData = json.loads(jsonStringBytes) sensorId = jsonData[SENSOR_ID] sensorKey = jsonData[SENSOR_KEY] if not authentication.authenticateSensor(sensorId, sensorKey): raise Exception("Sensor Authentication Failure") sensorObj = SensorDb.getSensorObj(sensorId) if not sensorObj.getSensorStatus() == ENABLED: raise Exception("Sensor is disabled") # remove the sensor key from metadata for safety. del jsonData[SENSOR_KEY] locationPosts = DbCollections.getLocationMessages() systemPosts = DbCollections.getSystemMessages() dataPosts = DbCollections.getDataMessages(sensorId) db = DbCollections.getSpectrumDb() currentLocalTime = time.time() Message.setInsertionTime(jsonData, currentLocalTime) if jsonData[TYPE] == SYS: # see if this system message already exists in the DB to avoid duplicates. query = {SENSOR_ID: jsonData[SENSOR_ID], "t": jsonData["t"]} found = systemPosts.find_one(query) if CAL in jsonData: calStr = jsonData[CAL] # Ugly!! Need to fix this. if calStr != "N/A": n = jsonData[CAL]["mPar"]["n"] nM = jsonData[CAL]["nM"] sensorId = jsonData[SENSOR_ID] if n * nM != 0: dataType = jsonData[CAL][DATA_TYPE] lengthToRead = n * nM if filedesc is not None: messageBytes = readDataFromFileDesc( filedesc, dataType, lengthToRead) elif powers is None: messageBytes = jsonString[headerLength:] else: # TODO -- deal with the other data types here messageBytes = struct.pack('%sb' % len(powers), *powers) fs = gridfs.GridFS(db, jsonData[SENSOR_ID] + "_data") key = fs.put(messageBytes) jsonData[CAL][DATA_KEY] = str(key) if found is None: systemPosts.ensure_index([('t', pymongo.DESCENDING)]) systemPosts.insert(jsonData) else: util.debugPrint("not inserting duplicate system post") end_time = time.time() util.debugPrint("Insertion time " + str(end_time - start_time)) sensorObj.updateSystemMessageTimeStamp(Message.getTime(jsonData)) SensorDb.updateSensor(sensorObj.getJson(), False, False) elif jsonData[TYPE] == LOC: print(json.dumps(jsonData, sort_keys=True, indent=4)) sensorId = jsonData[SENSOR_ID] t = jsonData['t'] lat = jsonData[LAT] lon = jsonData[LON] alt = jsonData[ALT] query = {SENSOR_ID: sensorId, LAT: lat, LON: lon, ALT: alt} locMsg = locationPosts.find_one(query) if locMsg is not None: print "Location Post already exists - not updating " return (to_zone, timeZoneName) = timezone.getLocalTimeZoneFromGoogle(t, lat, lon) # If google returned null, then override with local information if to_zone is None: if TIME_ZONE_KEY in jsonData: to_zone = jsonData[TIME_ZONE_KEY] else: raise Exception("ERROR: Unable to determine timeZone ") else: jsonData[TIME_ZONE_KEY] = to_zone # insert the loc message into the database. db.locationMessages.ensure_index([('t', pymongo.DESCENDING)]) locationPosts.insert(jsonData) end_time = time.time() sensorObj.updateLocationMessageTimeStamp(Message.getTime(jsonData)) SensorDb.updateSensor(sensorObj.getJson(), False, False) print "inserted Location Message. Insertion time " + str(end_time - start_time) elif jsonData[TYPE] == DATA: # BUG BUG -- we need to fix this. Need new data. if SYS_TO_DETECT not in jsonData: jsonData[SYS_TO_DETECT] = "LTE" # Fix up issue with sys2detect - should have no spaces. # BUGBUG -- this is ugly. Should reject the data. jsonData[SYS_TO_DETECT] = jsonData[SYS_TO_DETECT].replace(" ", "") DataMessage.init(jsonData) freqRange = DataMessage.getFreqRange(jsonData) if freqRange not in sensorObj.getThreshold(): raise Exception("ERROR: Frequency Band " + freqRange + " not found") lastSystemPost = systemPosts.find_one({ SENSOR_ID: sensorId, "t": { "$lte": Message.getTime(jsonData) } }) lastLocationPost = locationPosts.find_one({ SENSOR_ID: sensorId, "t": { "$lte": Message.getTime(jsonData) } }) if lastLocationPost is None or lastSystemPost is None: raise Exception("Location post or system post not found for " + sensorId) # Check for duplicates query = {SENSOR_ID: sensorId, "t": Message.getTime(jsonData)} found = DbCollections.getDataMessages(sensorId).find_one(query) # record the location message associated with the data. DataMessage.setLocationMessageId(jsonData, str(lastLocationPost['_id'])) DataMessage.setSystemMessageId(jsonData, str(lastSystemPost['_id'])) # prev data message. nM = DataMessage.getNumberOfMeasurements(jsonData) n = DataMessage.getNumberOfFrequencyBins(jsonData) lengthToRead = n * nM dataType = DataMessage.getDataType(jsonData) if lengthToRead != 0: if filedesc is not None: messageBytes = readDataFromFileDesc(filedesc, dataType, lengthToRead) elif powers is None: messageBytes = jsonString[headerLength:] else: # TODO - deal with the other data types here. messageBytes = struct.pack("%sb" % len(powers), *powers) occupancyBytes = None if streamOccupancies is not None: occupancyBytes = struct.pack("%sb" % len(streamOccupancies), *streamOccupancies) # Note: The data needs to be read before it is rejected. if found is not None: util.debugPrint("ignoring duplicate data message") return if lengthToRead != 0: fs = gridfs.GridFS(db, sensorId + "_data") key = fs.put(messageBytes) DataMessage.setDataKey(jsonData, str(key)) if occupancyBytes is not None: key = fs.put(occupancyBytes) DataMessage.setOccupancyKey(jsonData, str(key)) DataMessage.setOccupancyVectorLength(jsonData, len(occupancyBytes)) cutoff = DataMessage.getThreshold(jsonData) sensorMeasurementType = SensorDb.getSensor(sensorId)[MEASUREMENT_TYPE] if DataMessage.getMeasurementType(jsonData) != sensorMeasurementType: raise Exception( "MeasurementType Mismatch between sensor and DataMessage") #dataPosts.ensure_index([('t', pymongo.ASCENDING)]) maxPower = -1000 minPower = 1000 if DataMessage.getMeasurementType(jsonData) == FFT_POWER: occupancyCount = [0 for i in range(0, nM)] if powers is None: powerVal = np.array(np.zeros(n * nM)) else: powerVal = np.array(powers) # unpack the power array. if dataType == BINARY_INT8 and powers is None: for i in range(0, lengthToRead): powerVal[i] = struct.unpack('b', messageBytes[i:i + 1])[0] maxPower = np.max(powerVal) minPower = np.min(powerVal) powerArray = powerVal.reshape(nM, n) for i in range(0, nM): occupancyCount[i] = float( len(filter(lambda x: x >= cutoff, powerArray[i, :]))) / float(n) minOccupancy = np.min(occupancyCount) maxOccupancy = np.max(occupancyCount) meanOccupancy = np.mean(occupancyCount) medianOccupancy = np.median(occupancyCount) DataMessage.setMaxOccupancy(jsonData, maxOccupancy) DataMessage.setMeanOccupancy(jsonData, meanOccupancy) DataMessage.setMinOccupancy(jsonData, minOccupancy) DataMessage.setMedianOccupancy(jsonData, medianOccupancy) if DataMessage.isProcessed(jsonData): sensorObj.updateMinOccupancy(freqRange, minOccupancy) sensorObj.updateMaxOccupancy(freqRange, maxOccupancy) sensorObj.updateOccupancyCount(freqRange, meanOccupancy) LocationMessage.updateMaxBandOccupancy(lastLocationPost, freqRange, maxOccupancy) LocationMessage.updateMinBandOccupancy(lastLocationPost, freqRange, minOccupancy) LocationMessage.updateOccupancySum(lastLocationPost, freqRange, meanOccupancy) else: if dataType == ASCII: powerVal = eval(messageBytes) else: for i in range(0, lengthToRead): powerVal[i] = struct.unpack('f', messageBytes[i:i + 4])[0] maxPower = np.max(powerVal) minPower = np.min(powerVal) occupancyCount = float(len(filter(lambda x: x >= cutoff, powerVal))) occupancy = occupancyCount / float(len(powerVal)) DataMessage.setOccupancy(jsonData, occupancy) if DataMessage.isProcessed(jsonData): sensorObj.updateMinOccupancy(freqRange, occupancy) sensorObj.updateMaxOccupancy(freqRange, occupancy) sensorObj.updateOccupancyCount(freqRange, occupancy) LocationMessage.updateMaxBandOccupancy(lastLocationPost, freqRange, occupancy) LocationMessage.updateMinBandOccupancy(lastLocationPost, freqRange, occupancy) LocationMessage.updateOccupancySum(lastLocationPost, freqRange, occupancy) sensorObj.updateTime(freqRange, Message.getTime(jsonData)) sensorObj.updateDataMessageTimeStamp(Message.getTime(jsonData)) SensorDb.updateSensor(sensorObj.getJson(), False, False) DataMessage.setMaxPower(jsonData, maxPower) DataMessage.setMinPower(jsonData, minPower) #if filedesc is not None: # print json.dumps(jsonData, sort_keys=True, indent=4) if DataMessage.isProcessed(jsonData): dataPosts.insert(jsonData) else: DbCollections.getUnprocessedDataMessages(sensorId).insert(jsonData) # Update location specific information for this sensor. LocationMessage.addFreqRange(lastLocationPost, freqRange) LocationMessage.setMessageTimeStampForBand(lastLocationPost, freqRange, Message.getTime(jsonData)) LocationMessage.incrementMessageCount(lastLocationPost) LocationMessage.incrementBandCount(lastLocationPost, freqRange) LocationMessage.setMinMaxPower(lastLocationPost, minPower, maxPower) locationPostId = lastLocationPost["_id"] del lastLocationPost["_id"] locationPosts.update({"_id": locationPostId}, {"$set": lastLocationPost}, upsert=False) end_time = time.time() if filedesc is not None: print " Insertion time " + str(end_time - start_time)
def getDataSummaryForAllBands(sensorId, locationMessage, tmin=None, dayCount=None): """ get the summary of the data corresponding to the location message. """ # tmin and tmax are the min and the max of the time range of interest. locationMessageId = str(locationMessage["_id"]) tzId = locationMessage[TIME_ZONE_KEY] sensor = SensorDb.getSensor(sensorId) if sensor is None: return {STATUS: NOK, ERROR_MESSAGE: "Sensor Not found in SensorDb"} bands = sensor[THRESHOLDS] if len(bands.keys()) == 0: return {STATUS: NOK, ERROR_MESSAGE: "Sensor has no bands"} measurementType = sensor[MEASUREMENT_TYPE] bandStatistics = [] query = {SENSOR_ID: sensorId, "locationMessageId": locationMessageId} msg = DbCollections.getDataMessages(sensorId).find_one(query) if msg is None: for key in bands.keys(): band = bands[key] minFreq = band[MIN_FREQ_HZ] maxFreq = band[MAX_FREQ_HZ] sys2detect = band[SYSTEM_TO_DETECT] isActive = band[ACTIVE] bandInfo = { "tStartDayBoundary": 0, "tEndDayBoundary": 0, "tStartReadings": 0, "tStartLocalTime": 0, "tStartLocalTimeFormattedTimeStamp": UNKNOWN, "tEndReadings": 0, "tEndReadingsLocalTime": 0, "tEndLocalTimeFormattedTimeStamp": UNKNOWN, "tEndDayBoundary": 0, "maxOccupancy": 0, "meanOccupancy": 0, "minOccupancy": 0, "maxFreq": maxFreq, "minFreq": minFreq, SYSTEM_TO_DETECT: sys2detect, "measurementType": measurementType, "active": isActive, COUNT: 0 } bandStatistics.append(bandInfo) return {STATUS: "OK", "bands": bandStatistics} if tmin is None and dayCount is None: query = {SENSOR_ID: sensorId, "locationMessageId": locationMessageId} tmin = msgutils.getDayBoundaryTimeStamp(msg) mintime = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( int(tmin), tzId) elif tmin is not None and dayCount is None: mintime = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( int(tmin), tzId) else: mintime = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( int(tmin), tzId) for key in bands.keys(): band = bands[key] minFreq = band[MIN_FREQ_HZ] maxFreq = band[MAX_FREQ_HZ] sys2detect = band[SYSTEM_TO_DETECT] bandSummary = getBandDataSummary(sensorId, locationMessage, sys2detect, minFreq, maxFreq, mintime, dayCount=dayCount) bandStatistics.append(bandSummary) return {STATUS: "OK", "bands": bandStatistics}
def setupSensor(sensorConfigName): import SensorDb sensorConfig = json.load(open(sensorConfigName)) SensorDb.addSensor(sensorConfig)
def recomputeOccupanciesWorker(sensorId): # Clean out the summary stats. util.debugPrint("recomputeOccupanciesWorker " + sensorId) sensorObj = SensorDb.getSensorObj(sensorId) if sensorObj is None: return try: sensorObj.cleanSensorStats() cur = DbCollections.getDataMessages(sensorId).find() if cur is None or cur.count() == 0: return locationMessages = DbCollections.getLocationMessages().find( {SENSOR_ID: sensorId}) for locationMessage in locationMessages: lid = locationMessage["_id"] LocationMessage.clean(locationMessage) util.debugPrint("Location Message " + json.dumps(locationMessage, indent=4)) DbCollections.getLocationMessages().update( {"_id": lid}, {"$set": locationMessage}, upsert=False) for jsonData in cur: freqRange = DataMessage.getFreqRange(jsonData) # TODO -- recompute the occupancies. for data message. DataMessage.resetThreshold(jsonData) dataMsgId = jsonData["_id"] del jsonData["_id"] DbCollections.getDataMessages(sensorId).update({"_id": dataMsgId}, {"$set": jsonData}, upsert=False) minPower = DataMessage.getMinPower(jsonData) maxPower = DataMessage.getMaxPower(jsonData) lastLocationPost = msgutils.getLocationMessage(jsonData) if DataMessage.getMeasurementType(jsonData) == FFT_POWER: minOccupancy = DataMessage.getMinOccupancy(jsonData) maxOccupancy = DataMessage.getMaxOccupancy(jsonData) meanOccupancy = DataMessage.getMeanOccupancy(jsonData) sensorObj.updateMinOccupancy(freqRange, minOccupancy) sensorObj.updateMaxOccupancy(freqRange, maxOccupancy) sensorObj.updateOccupancyCount(freqRange, meanOccupancy) LocationMessage.updateMaxBandOccupancy(lastLocationPost, freqRange, maxOccupancy) LocationMessage.updateMinBandOccupancy(lastLocationPost, freqRange, minOccupancy) LocationMessage.updateOccupancySum(lastLocationPost, freqRange, meanOccupancy) else: occupancy = DataMessage.getOccupancy(jsonData) sensorObj.updateMinOccupancy(freqRange, occupancy) sensorObj.updateMaxOccupancy(freqRange, occupancy) sensorObj.updateOccupancyCount(freqRange, occupancy) LocationMessage.updateMaxBandOccupancy(lastLocationPost, freqRange, occupancy) LocationMessage.updateMinBandOccupancy(lastLocationPost, freqRange, occupancy) LocationMessage.updateOccupancySum(lastLocationPost, freqRange, occupancy) DbCollections.getLocationMessages().update( {"_id": lastLocationPost["_id"]}, {"$set": lastLocationPost}, upsert=False) except: print "Unexpected error:", sys.exc_info()[0] print sys.exc_info() traceback.print_exc() util.logStackTrace(sys.exc_info()) finally: SensorDb.setSensorStatus(sensorId, ENABLED)
def runGarbageCollector(sensorId): SessionLock.acquire() try: userCount = SessionLock.getUserSessionCount() if userCount != 0: return {"status": "NOK", "ErrorMessage": "Active user session detected"} sensorObj = SensorDb.getSensorObj(sensorId) if sensorObj is None: return {"status": "NOK", "ErrorMessage": "Sensor Not found"} if sensorObj.getSensorStatus() != DISABLED: return {"status": "NOK", "ErrorMessage": "Sensor is ENABLED -- DISABLE it first"} dataRetentionDuration = sensorObj.getSensorDataRetentionDurationMonths( ) dataRetentionTime = dataRetentionDuration * 30 * SECONDS_PER_DAY cur = DbCollections.getDataMessages(sensorId).find( {SENSOR_ID: sensorId}) #dataMessages = cur.sort('t', pymongo.ASCENDING) currentTime = time.time() for msg in cur: insertionTime = Message.getInsertionTime(msg) if currentTime - dataRetentionTime >= insertionTime: DbCollections.getDataMessages(sensorId).remove(msg) msgutils.removeData(msg) else: break # Now redo our book keeping summary fields. #dataMessages = cur.sort('t', pymongo.ASCENDING) locationMessages = DbCollections.getLocationMessages().find( {SENSOR_ID: sensorId}) for locationMessage in locationMessages: insertionTime = Message.getInsertionTime(locationMessage) if currentTime - dataRetentionTime >= insertionTime: DbCollections.getLocationMessages().remove(msg) else: LocationMessage.clean(locationMessage) DbCollections.getLocationMessages().update( {"_id": locationMessage["_id"]}, {"$set": locationMessage}, upsert=False) sensorObj.cleanSensorStats() # Update the summary statistics. cur = DbCollections.getDataMessages(sensorId).find( {SENSOR_ID: sensorId}) for jsonData in cur: freqRange = DataMessage.getFreqRange(jsonData) minPower = DataMessage.getMinPower(jsonData) maxPower = DataMessage.getMaxPower(jsonData) messageId = DataMessage.getLocationMessageId(jsonData) lastLocationPost = DbCollections.getLocationMessages().find_one({"_id": messageId}) if DataMessage.getMeasurementType(jsonData) == FFT_POWER: minOccupancy = DataMessage.getMinOccupancy(jsonData) maxOccupancy = DataMessage.getMaxOccupancy(jsonData) meanOccupancy = DataMessage.getMeanOccupancy(jsonData) sensorObj.updateMinOccupancy(freqRange, minOccupancy) sensorObj.updateMaxOccupancy(freqRange, maxOccupancy) sensorObj.updateOccupancyCount(freqRange, meanOccupancy) LocationMessage.updateMaxBandOccupancy(lastLocationPost, freqRange, maxOccupancy) LocationMessage.updateMinBandOccupancy(lastLocationPost, freqRange, minOccupancy) LocationMessage.updateOccupancySum(lastLocationPost, freqRange, meanOccupancy) else: occupancy = DataMessage.getOccupancy(jsonData) sensorObj.updateMinOccupancy(freqRange, occupancy) sensorObj.updateMaxOccupancy(freqRange, occupancy) sensorObj.updateOccupancyCount(freqRange, occupancy) LocationMessage.updateMaxBandOccupancy(lastLocationPost, freqRange, occupancy) LocationMessage.updateMinBandOccupancy(lastLocationPost, freqRange, occupancy) LocationMessage.updateOccupancySum(lastLocationPost, freqRange, occupancy) sensorObj.updateTime(freqRange, Message.getTime(jsonData)) sensorObj.updateDataMessageTimeStamp(Message.getTime(jsonData)) SensorDb.updateSensor(sensorObj.getJson(), False, False) DbCollections.getLocationMessages().update( {"_id": lastLocationPost["_id"]}, {"$set": lastLocationPost}, upsert=False) # Garbage collect the unprocessed data messages. cur = DbCollections.getUnprocessedDataMessages(sensorId).find({SENSOR_ID: sensorId}) if cur is not None: dataMessages = cur.sort('t', pymongo.ASCENDING) for msg in dataMessages: insertionTime = Message.getInsertionTime(msg) if currentTime - dataRetentionTime >= insertionTime: DbCollections.getUnprocessedDataMessages(sensorId).remove(msg) else: break DbCollections.dropDailyOccupancyCache(sensorId) return {"status": "OK", "sensors": SensorDb.getAllSensors()} finally: SessionLock.release()