def disconnectSensor(sensorId):
"""
Send a sensor a command to exit.
URL Path:
sensorId -- the session ID of the login session.
URL Args: None
Request Body:
Contains authentication information for the agent that is authorized
to arm and disarm the sensor:
- agentName: Name of the agent to arm/disarm sensor.
- key : password of the agent to arm/disarm the sensor.
HTTP Return Codes:
- 200 OK: invocation was successful.
- 403 Forbidden: authentication failure
- 400 Bad request: Sensor is not a streaming sensor.
Example Invocation:
::
params = {}
params["agentName"] = "NIST_ESC"
params["key"] = "ESC_PASS"
r = requests.post("https://"+ host + ":" + str(443) + "/sensorcontrol/disconnectSensor/" + self.sensorId + "/LTE:70315780:713315880",data=json.dumps(params),verify=False)
"""
try:
util.debugPrint("disconnectSensor: sensorId " + sensorId)
requestStr = request.data
if requestStr is None:
abort(400)
accountData = json.loads(requestStr)
if not authentication.authenticateSensorAgent(accountData):
abort(403)
sensorConfig = SensorDb.getSensorObj(sensorId)
if sensorConfig is None:
abort(404)
if not sensorConfig.isStreamingEnabled():
abort(400)
sendCommandToSensor(
sensorId, json.dumps({
"sensorId": sensorId,
"command": "exit"
}))
return jsonify({STATUS: OK})
except:
print "Unexpected error:", sys.exc_info()[0]
print sys.exc_info()
traceback.print_exc()
util.logStackTrace(sys.exc_info())
raise
def getSensorDataSummary(sensorId, locationMessage):
sensor = SensorDb.getSensorObj(sensorId)
if sensor is None:
return {STATUS: NOK, ERROR_MESSAGE: "Sensor Not found"}
measurementType = sensor.getMeasurementType()
tzId = locationMessage[TIME_ZONE_KEY]
acquisitionCount = LocationMessage.getMessageCount(locationMessage)
util.debugPrint("AquistionCount " + str(acquisitionCount))
if acquisitionCount == 0:
return {
"status": "OK",
"minOccupancy": 0,
"tStartReadings": 0,
"tStartLocalTime": 0,
"tStartLocalTimeFormattedTimeStamp": "UNKNOWN",
"tStartDayBoundary": 0,
"tEndDayBoundary": 0,
"tEndReadings": 0,
"tEndLocalTimeFormattedTimeStamp": "UNKNOWN",
"maxOccupancy": 0,
"measurementType": measurementType,
"isStreamingEnabled": sensor.isStreamingEnabled(),
"sensorStatus": sensor.getSensorStatus(),
COUNT: 0
}
minTime = LocationMessage.getFirstDataMessageTimeStamp(locationMessage)
maxTime = LocationMessage.getLastDataMessageTimeStamp(locationMessage)
tStartDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
minTime, tzId)
(minLocalTime,
tStartLocalTimeTzName) = timezone.getLocalTime(minTime, tzId)
tEndDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
maxTime, tzId)
tstampMin = timezone.formatTimeStampLong(minTime, tzId)
tstampMax = timezone.formatTimeStampLong(maxTime, tzId)
retval = {
"status": "OK",
"maxOccupancy": 0,
"minOccupancy": 0,
"tStartReadings": minTime,
"tStartLocalTime": minLocalTime,
"tStartLocalTimeFormattedTimeStamp": tstampMin,
"tStartDayBoundary": tStartDayBoundary,
"tEndDayBoundary": tEndDayBoundary,
"tEndReadings": maxTime,
"tEndLocalTimeFormattedTimeStamp": tstampMax,
"measurementType": measurementType,
"isStreamingEnabled": sensor.isStreamingEnabled(),
"sensorStatus": sensor.getSensorStatus(),
COUNT: acquisitionCount
}
return retval
def recomputeOccupancies(sensorId):
sensorObj = SensorDb.getSensorObj(sensorId)
if sensorObj.getSensorStatus() == PURGING:
return {"status": "NOK", "ErrorMessage": "Sensor is PURGING"}
elif sensorObj.getSensorStatus() == RECOMPUTING:
return {"status": "NOK", "ErrorMessage": "Sensor is RECOMPUTING"}
else:
SensorDb.setSensorStatus(sensorId, RECOMPUTING)
return {"status": "OK", "sensors": SensorDb.getAllSensors()}
def testArmSensor(sensorId, sessionId):
"""
URL Path:
sessionId -- the session ID of the login session.
sensorId -- the sensorId
URL Args: None
Request Body:
- agentName: Name of the agent to arm/disarm sensor.
- key : Key (password) of the agent to arm/disarm the sensor.
HTTP Return Codes:
- 200 OK: invocation was successful.
- 403 Forbidden: authentication failure
- 400 Bad request: Sensor is not a streaming sensor.
Example Invocation:
::
params = {}
params["agentName"] = "NIST_ESC"
params["key"] = "ESC_PASS"
url = "https://{host}:8443/admin/armSensor/{self.sensorId}"
r = requests.post(url, data=json.dumps(params), verify=False)
"""
try:
if not authentication.checkSessionId(sessionId, ADMIN):
abort(403)
sensorConfig = SensorDb.getSensorObj(sensorId)
if sensorConfig is None:
abort(404)
if not sensorConfig.isStreamingEnabled():
abort(400)
persistent = request.args.get("persistent")
if persistent is None:
persistent = "false"
DataStreamSharedState.sendCommandToSensor(
sensorId,
json.dumps({
"sensorId": sensorId,
"command": "arm",
"persistent": persistent
}))
return jsonify({STATUS: OK})
except:
print "Unexpected error:", sys.exc_info()[0]
print sys.exc_info()
traceback.print_exc()
util.logStackTrace(sys.exc_info())
raise
def postCaptureEvent():
"""
Handle post of a capture event from a sensor
URL Path:
- None
URL Parameters:
- None
Request Body:
- CaptureEvent JSON structure which includes the sensor ID and sensor key.
These are used for verifying the request. See MSOD specification for definition of
CaptureEvent structure.
HTTP Return Codes:
- 200 OK. A JSON Document containing {"status":"OK"} is returned.
"""
try:
requestStr = request.data
if requestStr is None or requestStr == "":
util.debugPrint("postCaptureEvent - request body not found")
abort(400)
util.debugPrint("postCaptureEvent " + requestStr)
captureEvent = json.loads(requestStr)
if SENSOR_ID not in captureEvent or SENSOR_KEY not in captureEvent:
util.debugPrint("postCaptureEvent - missing a required field")
abort(400)
sensorId = captureEvent[SENSOR_ID]
sensorConfig = SensorDb.getSensorObj(sensorId)
if sensorConfig is None:
util.debugPrint("postCaptureEvent - sensor not found")
abort(404)
sensorKey = captureEvent[SENSOR_KEY]
if not authentication.authenticateSensor(sensorId, sensorKey):
abort(403)
return jsonify(CaptureDb.insertEvent(sensorId, captureEvent))
except:
print "Unexpected error:", sys.exc_info()[0]
print sys.exc_info()
traceback.print_exc()
util.logStackTrace(sys.exc_info())
raise
def getSocketServerPort(sensorId):
retval = {}
global memCache
if memCache is None:
memCache = MemCache()
sensor = SensorDb.getSensorObj(sensorId)
print "sensorStatus ", sensor.getSensorStatus()
if sensor is None or sensor.getSensorStatus() != ENABLED \
or not sensor.isStreamingEnabled():
retval["port"] = -1
return retval
retval["port"] = STREAMING_SERVER_PORT
return retval
def recomputeOccupanciesWorker(sensorId):
# Clean out the summary stats.
util.debugPrint("recomputeOccupanciesWorker " + sensorId)
sensorObj = SensorDb.getSensorObj(sensorId)
if sensorObj is None:
return
try:
sensorObj.cleanSensorStats()
cur = DbCollections.getDataMessages(sensorId).find()
if cur is None or cur.count() == 0:
return
locationMessages = DbCollections.getLocationMessages().find(
{SENSOR_ID: sensorId})
for locationMessage in locationMessages:
lid = locationMessage["_id"]
LocationMessage.clean(locationMessage)
util.debugPrint("Location Message " +
json.dumps(locationMessage, indent=4))
DbCollections.getLocationMessages().update(
{"_id": lid}, {"$set": locationMessage}, upsert=False)
for jsonData in cur:
freqRange = DataMessage.getFreqRange(jsonData)
# TODO -- recompute the occupancies. for data message.
DataMessage.resetThreshold(jsonData)
dataMsgId = jsonData["_id"]
del jsonData["_id"]
DbCollections.getDataMessages(sensorId).update({"_id": dataMsgId},
{"$set": jsonData},
upsert=False)
minPower = DataMessage.getMinPower(jsonData)
maxPower = DataMessage.getMaxPower(jsonData)
lastLocationPost = msgutils.getLocationMessage(jsonData)
if DataMessage.getMeasurementType(jsonData) == FFT_POWER:
minOccupancy = DataMessage.getMinOccupancy(jsonData)
maxOccupancy = DataMessage.getMaxOccupancy(jsonData)
meanOccupancy = DataMessage.getMeanOccupancy(jsonData)
sensorObj.updateMinOccupancy(freqRange, minOccupancy)
sensorObj.updateMaxOccupancy(freqRange, maxOccupancy)
sensorObj.updateOccupancyCount(freqRange, meanOccupancy)
LocationMessage.updateMaxBandOccupancy(lastLocationPost,
freqRange, maxOccupancy)
LocationMessage.updateMinBandOccupancy(lastLocationPost,
freqRange, minOccupancy)
LocationMessage.updateOccupancySum(lastLocationPost, freqRange,
meanOccupancy)
else:
occupancy = DataMessage.getOccupancy(jsonData)
sensorObj.updateMinOccupancy(freqRange, occupancy)
sensorObj.updateMaxOccupancy(freqRange, occupancy)
sensorObj.updateOccupancyCount(freqRange, occupancy)
LocationMessage.updateMaxBandOccupancy(lastLocationPost,
freqRange, occupancy)
LocationMessage.updateMinBandOccupancy(lastLocationPost,
freqRange, occupancy)
LocationMessage.updateOccupancySum(lastLocationPost, freqRange,
occupancy)
DbCollections.getLocationMessages().update(
{"_id": lastLocationPost["_id"]}, {"$set": lastLocationPost},
upsert=False)
except:
print "Unexpected error:", sys.exc_info()[0]
print sys.exc_info()
traceback.print_exc()
util.logStackTrace(sys.exc_info())
finally:
SensorDb.setSensorStatus(sensorId, ENABLED)
def getDailyMaxMinMeanStats(sensorId, lat, lon, alt, tstart, ndays, sys2detect,
fmin, fmax, subBandMinFreq, subBandMaxFreq):
locationMessage = DbCollections.getLocationMessages().find_one({
SENSOR_ID: sensorId,
LAT: lat,
LON: lon,
ALT: alt
})
if locationMessage is None:
return {STATUS: NOK, ERROR_MESSAGE: "Location Information Not Found"}
locationMessageId = str(locationMessage["_id"])
tZId = locationMessage[TIME_ZONE_KEY]
tmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(tstart, tZId)
startMessage = DbCollections.getDataMessages(sensorId).find_one()
result = {}
result[STATUS] = OK
values = {}
for day in range(0, ndays):
tstart = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
tmin + day * SECONDS_PER_DAY, tZId)
tend = tstart + SECONDS_PER_DAY
queryString = {
LOCATION_MESSAGE_ID: locationMessageId,
TIME: {
'$gte': tstart,
'$lte': tend
},
FREQ_RANGE: msgutils.freqRange(sys2detect, fmin, fmax)
}
cur = DbCollections.getDataMessages(sensorId).find(queryString)
# cur.batch_size(20)
if startMessage['mType'] == FFT_POWER:
stats = compute_daily_max_min_mean_stats_for_fft_power(cur)
else:
stats = compute_daily_max_min_mean_median_stats_for_swept_freq(
cur, subBandMinFreq, subBandMaxFreq)
# gap in readings. continue.
if stats is None:
continue
(cutoff, dailyStat) = stats
values[day * 24] = dailyStat
# Now compute the next interval after the last one (if one exists)
tend = tmin + SECONDS_PER_DAY * ndays
queryString = {
LOCATION_MESSAGE_ID: locationMessageId,
TIME: {
'$gte': tend
},
FREQ_RANGE: msgutils.freqRange(sys2detect, fmin, fmax)
}
msg = DbCollections.getDataMessages(sensorId).find_one(queryString)
if msg is None:
result["nextTmin"] = tmin
else:
nextTmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
msg[TIME], tZId)
result["nextTmin"] = nextTmin
# Now compute the previous interval before this one.
prevMessage = msgutils.getPrevAcquisition(startMessage)
if prevMessage is not None:
newTmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
prevMessage[TIME] - SECONDS_PER_DAY * ndays, tZId)
queryString = {
LOCATION_MESSAGE_ID: locationMessageId,
TIME: {
'$gte': newTmin
},
FREQ_RANGE: msgutils.freqRange(sys2detect, fmin, fmax)
}
msg = DbCollections.getDataMessages(sensorId).find_one(queryString)
else:
msg = startMessage
sensor = SensorDb.getSensorObj(sensorId)
channelCount = sensor.getChannelCount(sys2detect, fmin, fmax)
result[STATUS] = OK
result["prevTmin"] = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
msg[TIME], tZId)
result["tmin"] = tmin
result["maxFreq"] = fmin
result["minFreq"] = fmax
result["cutoff"] = cutoff
result[CHANNEL_COUNT] = channelCount
result["startDate"] = timezone.formatTimeStampLong(tmin, tZId)
result["values"] = values
util.debugPrint(result)
return result
def readFromInput(bbuf, conn):
util.debugPrint("DataStreaming:readFromInput")
soc = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sensorCommandDispatcherPid = None
memCache = MemCache()
try:
while True:
lengthString = ""
while True:
lastChar = bbuf.readChar()
if lastChar is None:
time.sleep(0.1)
return
if len(lengthString) > 1000:
raise Exception("Formatting error")
if lastChar == '{':
headerLength = int(lengthString.rstrip())
break
else:
lengthString += str(lastChar)
jsonStringBytes = "{"
while len(jsonStringBytes) < headerLength:
jsonStringBytes += str(bbuf.readChar())
jsonData = json.loads(jsonStringBytes)
if not any(k in jsonData for k in (TYPE, SENSOR_ID, SENSOR_KEY)):
err = "Sensor Data Stream: Missing a required field"
util.errorPrint(err)
util.errorPrint("Invalid message -- closing connection: " +
json.dumps(jsonData, indent=4))
raise Exception("Invalid message")
return
sensorId = jsonData[SENSOR_ID]
global mySensorId
if mySensorId is None:
mySensorId = sensorId
elif mySensorId != sensorId:
raise Exception("Sensor ID mismatch " + mySensorId + " / " +
sensorId)
sensorKey = jsonData[SENSOR_KEY]
if not authentication.authenticateSensor(sensorId, sensorKey):
util.debugPrint("jsonData " + json.dumps(jsonData, indent=4))
util.errorPrint("Sensor authentication failed: " + sensorId +
" sensorKey " + sensorKey)
raise Exception("Authentication failure")
return
if memCache.getStreamingServerPid(sensorId) == -1:
memCache.setStreamingServerPid(sensorId)
elif memCache.getStreamingServerPid(sensorId) != os.getpid():
util.errorPrint(
"Handling connection for this sensor already " +
str(os.getpid()))
try:
os.kill(memCache.getStreamingServerPid(sensorId),
signal.SIGKILL)
memCache.setStreamingServerPid(sensorId)
except:
print "Unexpected error:", sys.exc_info()[0]
print sys.exc_info()
traceback.print_exc()
util.logStackTrace(sys.exc_info())
util.debugPrint(
"Problem killing process " +
str(memCache.getStreamingServerPid(sensorId)))
if memCache.getStreamingCommandDispatcherPid(sensorId) != -1:
try:
os.kill(
memCache.getStreamingCommandDispatcherPid(
sensorId), signal.SIGKILL)
except:
memCache.removeStreamingCommandDispatcherPid(sensorId)
util.debugPrint("Process not found. Proceeding ")
util.debugPrint("DataStreaming: Message = " +
dumps(jsonData, sort_keys=True, indent=4))
sensorObj = SensorDb.getSensorObj(sensorId)
if not sensorObj.isStreamingEnabled(
) or sensorObj.getStreamingParameters() is None:
raise Exception("Streaming is not enabled")
return
# the last time a data message was inserted
if jsonData[TYPE] == DATA:
util.debugPrint("pubsubPort: " +
str(memCache.getPubSubPort(sensorId)))
if "Sys2Detect" not in jsonData:
jsonData[SYS_TO_DETECT] = "LTE"
DataMessage.init(jsonData)
t = Process(target=runSensorCommandDispatchWorker,
args=(conn, sensorId))
t.start()
sensorCommandDispatcherPid = t.pid
childPids.append(sensorCommandDispatcherPid)
cutoff = DataMessage.getThreshold(jsonData)
n = DataMessage.getNumberOfFrequencyBins(jsonData)
sensorId = DataMessage.getSensorId(jsonData)
lastDataMessageReceivedAt[sensorId] = time.time()
lastDataMessageOriginalTimeStamp[
sensorId] = DataMessage.getTime(jsonData)
# Check if the measurement type reported by the sensor
# matches that in the sensordb
measurementType = DataMessage.getMeasurementType(jsonData)
if sensorObj.getMeasurementType() != measurementType:
err = "Measurement type mismatch "
err += sensorObj.getMeasurementType()
err += " / "
err += measurementType
raise Exception(err)
# Check if the time per measurement reported by the sensor
# matches that in the sensordb
timePerMeasurement = sensorObj.getStreamingSecondsPerFrame()
util.debugPrint("StreamingServer: timePerMeasurement " +
str(timePerMeasurement))
if timePerMeasurement != DataMessage.getTimePerMeasurement(
jsonData):
err = "TimePerMeasurement mismatch "
err += str(timePerMeasurement)
err += "/"
err += str(DataMessage.getTimePerMeasurement(jsonData))
raise Exception(err)
# The sampling interval for write to the database.
s = sensorObj.getStreamingSamplingIntervalSeconds()
streamingSamplingIntervalSeconds = s
# The number of measurements per capture
measurementsPerCapture = int(streamingSamplingIntervalSeconds /
timePerMeasurement)
util.debugPrint("StreamingServer: measurementsPerCapture " +
str(measurementsPerCapture))
# The number of power value samples per capture.
samplesPerCapture = int(
(streamingSamplingIntervalSeconds / timePerMeasurement) *
n)
# The number of spectrums per frame sent to the browser.
spectrumsPerFrame = 1
jsonData[SPECTRUMS_PER_FRAME] = spectrumsPerFrame
# The streaming filter of the sensor (MAX_HOLD or AVG)
jsonData[STREAMING_FILTER] = sensorObj.getStreamingFilter()
# The band name sys2detect:minfreq:maxfreq string for the
# reported measurement.
bandName = DataMessage.getFreqRange(jsonData)
# Keep a copy of the last data message for periodic insertion
# into the db
memCache.setLastDataMessage(sensorId, bandName,
json.dumps(jsonData))
# captureBufferCounter is a pointer into the capture buffer.
captureBufferCounter = 0
powerArrayCounter = 0
timingCounter = 0
# initialize the "prev occupancy array"
prevOccupancyArray = [-1 for i in range(0, n)]
occupancyArray = [0 for i in range(0, n)]
occupancyTimer = time.time()
if sensorId not in lastDataMessage:
lastDataMessage[sensorId] = jsonData
powerVal = [0 for i in range(0, n)]
startTime = time.time()
sensorObj = SensorDb.getSensorObj(sensorId)
if sensorObj is None:
raise Exception("Sensor not found")
if sensorObj.getSensorStatus() == DISABLED:
bbuf.close()
raise Exception("Sensor is disabled")
if not sensorObj.isStreamingEnabled():
raise Exception("Streaming is disabled")
enb = sensorObj.isStreamingCaptureEnabled()
isStreamingCaptureEnabled = enb
util.debugPrint("isStreamingCaptureEnabled : " + str(enb) +
" samplesPerCapture " + str(samplesPerCapture))
if isStreamingCaptureEnabled:
sensorData = [0 for i in range(0, samplesPerCapture)]
while True:
data = bbuf.readByte()
if isStreamingCaptureEnabled:
sensorData[captureBufferCounter] = data
powerVal[powerArrayCounter] = data
now = time.time()
if isStreamingCaptureEnabled and captureBufferCounter + 1 == samplesPerCapture:
# Buffer is full so push the data into mongod.
util.debugPrint("Inserting Data message")
captureBufferCounter = 0
# Time offset since the last data message was received.
timeOffset = time.time(
) - lastDataMessageReceivedAt[sensorId]
# Offset the capture by the time since the DataMessage header was received.
lastDataMessage[sensorId]["t"] = int(
lastDataMessageOriginalTimeStamp[sensorId] +
int(timeOffset))
lastDataMessage[sensorId][
"nM"] = measurementsPerCapture
lastDataMessage[sensorId]["mPar"]["td"] = int(
now - occupancyTimer)
lastDataMessage[sensorId]["mPar"][
"tm"] = timePerMeasurement
headerStr = json.dumps(lastDataMessage[sensorId],
indent=4)
util.debugPrint("StreamingServer: headerStr " +
headerStr)
headerLength = len(headerStr)
if isStreamingCaptureEnabled:
# Start the db operation in a seperate process
p = Process(target=populate_db.put_data,
args=(headerStr, headerLength),
kwargs={
"filedesc": None,
"powers": sensorData
})
p.start()
lastDataMessageInsertedAt[sensorId] = time.time()
occupancyTimer = time.time()
else:
captureBufferCounter = captureBufferCounter + 1
if data > cutoff:
occupancyArray[powerArrayCounter] = 1
else:
occupancyArray[powerArrayCounter] = 0
# print "occupancyArray", occupancyArray
if (powerArrayCounter + 1) == n:
# Get the occupancy subscription counter.
if memCache.getSubscriptionCount(sensorId) != 0:
if not np.array_equal(occupancyArray,
prevOccupancyArray):
port = memCache.getPubSubPort(sensorId)
soc.sendto(
json.dumps({sensorId: occupancyArray}),
("localhost", port))
prevOccupancyArray = np.array(occupancyArray)
# sending data as CSV values to the browser
listenerCount = memCache.getStreamingListenerCount(
sensorId)
if listenerCount > 0:
sensordata = str(powerVal)[1:-1].replace(" ", "")
memCache.setSensorData(sensorId, bandName,
sensordata)
# Record the occupancy for the measurements.
# Allow for 10% jitter.
if timingCounter == 1000 and checkForDataRate:
if (((now - startTime) / 1000.0 <
timePerMeasurement / 2)
or ((now - startTime) / 1000.0 >
timePerMeasurement * 2)):
print " delta ", now - startTime, "global counter ", powerArrayCounter
util.errorPrint(
"Data coming in too fast or too slow - sensor configuration problem."
)
raise Exception(
"Data coming in too fast - sensor configuration problem."
)
else:
startTime = now
lastdataseen = now
if listenerCount > 0:
memCache.setLastDataSeenTimeStamp(
sensorId, bandName, lastdataseen)
powerArrayCounter = 0
else:
powerArrayCounter = powerArrayCounter + 1
timingCounter = timingCounter + 1
elif jsonData[TYPE] == SYS:
util.debugPrint(
"DataStreaming: Got a System message -- adding to the database"
)
populate_db.put_data(jsonStringBytes, headerLength)
elif jsonData[TYPE] == LOC:
util.debugPrint(
"DataStreaming: Got a Location Message -- adding to the database"
)
populate_db.put_data(jsonStringBytes, headerLength)
finally:
util.debugPrint("Closing sockets for sensorId " + sensorId)
memCache.removeStreamingServerPid(sensorId)
port = memCache.getSensorArmPort(sensorId)
sendCommandToSensor(
sensorId, json.dumps({
"sensorId": sensorId,
"command": "exit"
}))
memCache.releaseSensorArmPort(sensorId)
bbuf.close()
time.sleep(1)
soc.close()
# kill the command dispatcher for good measure.
try:
if sensorCommandDispatcherPid is not None:
os.kill(sensorCommandDispatcherPid, signal.SIGKILL)
except:
util.debugPrint("Process not found. " +
str(sensorCommandDispatcherPid))
def getBandDataSummary(sensorId,
locationMessage,
sys2detect,
minFreq,
maxFreq,
mintime,
dayCount=None):
sensor = SensorDb.getSensorObj(sensorId)
if sensor is None:
return {STATUS: NOK, ERROR_MESSAGE: "Sensor Not found"}
measurementType = sensor.getMeasurementType()
tzId = locationMessage[TIME_ZONE_KEY]
locationMessageId = str(locationMessage["_id"])
freqRange = msgutils.freqRange(sys2detect, minFreq, maxFreq)
count = LocationMessage.getBandCount(locationMessage, freqRange)
if count == 0:
return {
FREQ_RANGE: freqRange,
COUNT: 0,
"minFreq": minFreq,
"maxFreq": maxFreq,
SYSTEM_TO_DETECT: sys2detect
}
else:
minOccupancy = LocationMessage.getMinBandOccupancy(
locationMessage, freqRange)
maxOccupancy = LocationMessage.getMaxBandOccupancy(
locationMessage, freqRange)
count = LocationMessage.getBandCount(locationMessage, freqRange)
meanOccupancy = LocationMessage.getMeanOccupancy(
locationMessage, freqRange)
minTime = LocationMessage.getFirstMessageTimeStampForBand(
locationMessage, freqRange)
maxTime = LocationMessage.getLastMessageTimeStampForBand(
locationMessage, freqRange)
maxTimes = timezone.getLocalTime(maxTime, tzId)
(tEndReadingsLocalTime, tEndReadingsLocalTimeTzName) = maxTimes
tEndDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
maxTime, tzId)
tStartDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
minTime, tzId)
tstampMin = timezone.formatTimeStampLong(minTime, tzId)
tstampMax = timezone.formatTimeStampLong(maxTime, tzId)
retval = {
"tStartDayBoundary": tStartDayBoundary,
"tEndDayBoundary": tEndDayBoundary,
"tStartReadings": minTime,
"tStartLocalTime": minTime,
"tStartLocalTimeFormattedTimeStamp": tstampMin,
"tEndReadings": maxTime,
"tEndReadingsLocalTime": maxTime,
"tEndLocalTimeFormattedTimeStamp": tstampMax,
"tEndDayBoundary": tEndDayBoundary,
"maxOccupancy": maxOccupancy,
"meanOccupancy": meanOccupancy,
"minOccupancy": minOccupancy,
"maxFreq": maxFreq,
"minFreq": minFreq,
SYSTEM_TO_DETECT: sys2detect,
FREQ_RANGE: freqRange,
"measurementType": measurementType,
"active": sensor.isBandActive(sys2detect, minFreq, maxFreq),
COUNT: count
}
return retval
def put_data(jsonString,
headerLength,
filedesc=None,
powers=None,
streamOccupancies=None):
"""
put data in the database. jsonString starts with {. If filedesc is None
then the data part of the message is appended to the message (immediately follows it).
Otherwise, the data is read from filedesc.
"""
start_time = time.time()
if filedesc is None:
# We are not reading from a file:
# Assume we are given the message in the string with the data
# tacked at the end of it.
jsonStringBytes = jsonString[0:headerLength]
else:
jsonStringBytes = jsonString
util.debugPrint("jsonStringBytes = " + jsonStringBytes)
jsonData = json.loads(jsonStringBytes)
sensorId = jsonData[SENSOR_ID]
sensorKey = jsonData[SENSOR_KEY]
if not authentication.authenticateSensor(sensorId, sensorKey):
raise Exception("Sensor Authentication Failure")
sensorObj = SensorDb.getSensorObj(sensorId)
if not sensorObj.getSensorStatus() == ENABLED:
raise Exception("Sensor is disabled")
# remove the sensor key from metadata for safety.
del jsonData[SENSOR_KEY]
locationPosts = DbCollections.getLocationMessages()
systemPosts = DbCollections.getSystemMessages()
dataPosts = DbCollections.getDataMessages(sensorId)
db = DbCollections.getSpectrumDb()
currentLocalTime = time.time()
Message.setInsertionTime(jsonData, currentLocalTime)
if jsonData[TYPE] == SYS:
# see if this system message already exists in the DB to avoid duplicates.
query = {SENSOR_ID: jsonData[SENSOR_ID], "t": jsonData["t"]}
found = systemPosts.find_one(query)
if CAL in jsonData:
calStr = jsonData[CAL]
# Ugly!! Need to fix this.
if calStr != "N/A":
n = jsonData[CAL]["mPar"]["n"]
nM = jsonData[CAL]["nM"]
sensorId = jsonData[SENSOR_ID]
if n * nM != 0:
dataType = jsonData[CAL][DATA_TYPE]
lengthToRead = n * nM
if filedesc is not None:
messageBytes = readDataFromFileDesc(
filedesc, dataType, lengthToRead)
elif powers is None:
messageBytes = jsonString[headerLength:]
else:
# TODO -- deal with the other data types here
messageBytes = struct.pack('%sb' % len(powers),
*powers)
fs = gridfs.GridFS(db, jsonData[SENSOR_ID] + "_data")
key = fs.put(messageBytes)
jsonData[CAL][DATA_KEY] = str(key)
if found is None:
systemPosts.ensure_index([('t', pymongo.DESCENDING)])
systemPosts.insert(jsonData)
else:
util.debugPrint("not inserting duplicate system post")
end_time = time.time()
util.debugPrint("Insertion time " + str(end_time - start_time))
sensorObj.updateSystemMessageTimeStamp(Message.getTime(jsonData))
SensorDb.updateSensor(sensorObj.getJson(), False, False)
elif jsonData[TYPE] == LOC:
print(json.dumps(jsonData, sort_keys=True, indent=4))
sensorId = jsonData[SENSOR_ID]
t = jsonData['t']
lat = jsonData[LAT]
lon = jsonData[LON]
alt = jsonData[ALT]
query = {SENSOR_ID: sensorId, LAT: lat, LON: lon, ALT: alt}
locMsg = locationPosts.find_one(query)
if locMsg is not None:
print "Location Post already exists - not updating "
return
(to_zone,
timeZoneName) = timezone.getLocalTimeZoneFromGoogle(t, lat, lon)
# If google returned null, then override with local information
if to_zone is None:
if TIME_ZONE_KEY in jsonData:
to_zone = jsonData[TIME_ZONE_KEY]
else:
raise Exception("ERROR: Unable to determine timeZone ")
else:
jsonData[TIME_ZONE_KEY] = to_zone
# insert the loc message into the database.
db.locationMessages.ensure_index([('t', pymongo.DESCENDING)])
locationPosts.insert(jsonData)
end_time = time.time()
sensorObj.updateLocationMessageTimeStamp(Message.getTime(jsonData))
SensorDb.updateSensor(sensorObj.getJson(), False, False)
print "inserted Location Message. Insertion time " + str(end_time -
start_time)
elif jsonData[TYPE] == DATA:
# BUG BUG -- we need to fix this. Need new data.
if SYS_TO_DETECT not in jsonData:
jsonData[SYS_TO_DETECT] = "LTE"
# Fix up issue with sys2detect - should have no spaces.
# BUGBUG -- this is ugly. Should reject the data.
jsonData[SYS_TO_DETECT] = jsonData[SYS_TO_DETECT].replace(" ", "")
DataMessage.init(jsonData)
freqRange = DataMessage.getFreqRange(jsonData)
if freqRange not in sensorObj.getThreshold():
raise Exception("ERROR: Frequency Band " + freqRange +
" not found")
lastSystemPost = systemPosts.find_one({
SENSOR_ID: sensorId,
"t": {
"$lte": Message.getTime(jsonData)
}
})
lastLocationPost = locationPosts.find_one({
SENSOR_ID: sensorId,
"t": {
"$lte": Message.getTime(jsonData)
}
})
if lastLocationPost is None or lastSystemPost is None:
raise Exception("Location post or system post not found for " +
sensorId)
# Check for duplicates
query = {SENSOR_ID: sensorId, "t": Message.getTime(jsonData)}
found = DbCollections.getDataMessages(sensorId).find_one(query)
# record the location message associated with the data.
DataMessage.setLocationMessageId(jsonData,
str(lastLocationPost['_id']))
DataMessage.setSystemMessageId(jsonData, str(lastSystemPost['_id']))
# prev data message.
nM = DataMessage.getNumberOfMeasurements(jsonData)
n = DataMessage.getNumberOfFrequencyBins(jsonData)
lengthToRead = n * nM
dataType = DataMessage.getDataType(jsonData)
if lengthToRead != 0:
if filedesc is not None:
messageBytes = readDataFromFileDesc(filedesc, dataType,
lengthToRead)
elif powers is None:
messageBytes = jsonString[headerLength:]
else:
# TODO - deal with the other data types here.
messageBytes = struct.pack("%sb" % len(powers), *powers)
occupancyBytes = None
if streamOccupancies is not None:
occupancyBytes = struct.pack("%sb" % len(streamOccupancies),
*streamOccupancies)
# Note: The data needs to be read before it is rejected.
if found is not None:
util.debugPrint("ignoring duplicate data message")
return
if lengthToRead != 0:
fs = gridfs.GridFS(db, sensorId + "_data")
key = fs.put(messageBytes)
DataMessage.setDataKey(jsonData, str(key))
if occupancyBytes is not None:
key = fs.put(occupancyBytes)
DataMessage.setOccupancyKey(jsonData, str(key))
DataMessage.setOccupancyVectorLength(jsonData, len(occupancyBytes))
cutoff = DataMessage.getThreshold(jsonData)
sensorMeasurementType = SensorDb.getSensor(sensorId)[MEASUREMENT_TYPE]
if DataMessage.getMeasurementType(jsonData) != sensorMeasurementType:
raise Exception(
"MeasurementType Mismatch between sensor and DataMessage")
#dataPosts.ensure_index([('t', pymongo.ASCENDING)])
maxPower = -1000
minPower = 1000
if DataMessage.getMeasurementType(jsonData) == FFT_POWER:
occupancyCount = [0 for i in range(0, nM)]
if powers is None:
powerVal = np.array(np.zeros(n * nM))
else:
powerVal = np.array(powers)
# unpack the power array.
if dataType == BINARY_INT8 and powers is None:
for i in range(0, lengthToRead):
powerVal[i] = struct.unpack('b', messageBytes[i:i + 1])[0]
maxPower = np.max(powerVal)
minPower = np.min(powerVal)
powerArray = powerVal.reshape(nM, n)
for i in range(0, nM):
occupancyCount[i] = float(
len(filter(lambda x: x >= cutoff,
powerArray[i, :]))) / float(n)
minOccupancy = np.min(occupancyCount)
maxOccupancy = np.max(occupancyCount)
meanOccupancy = np.mean(occupancyCount)
medianOccupancy = np.median(occupancyCount)
DataMessage.setMaxOccupancy(jsonData, maxOccupancy)
DataMessage.setMeanOccupancy(jsonData, meanOccupancy)
DataMessage.setMinOccupancy(jsonData, minOccupancy)
DataMessage.setMedianOccupancy(jsonData, medianOccupancy)
if DataMessage.isProcessed(jsonData):
sensorObj.updateMinOccupancy(freqRange, minOccupancy)
sensorObj.updateMaxOccupancy(freqRange, maxOccupancy)
sensorObj.updateOccupancyCount(freqRange, meanOccupancy)
LocationMessage.updateMaxBandOccupancy(lastLocationPost,
freqRange, maxOccupancy)
LocationMessage.updateMinBandOccupancy(lastLocationPost,
freqRange, minOccupancy)
LocationMessage.updateOccupancySum(lastLocationPost, freqRange,
meanOccupancy)
else:
if dataType == ASCII:
powerVal = eval(messageBytes)
else:
for i in range(0, lengthToRead):
powerVal[i] = struct.unpack('f', messageBytes[i:i + 4])[0]
maxPower = np.max(powerVal)
minPower = np.min(powerVal)
occupancyCount = float(len(filter(lambda x: x >= cutoff,
powerVal)))
occupancy = occupancyCount / float(len(powerVal))
DataMessage.setOccupancy(jsonData, occupancy)
if DataMessage.isProcessed(jsonData):
sensorObj.updateMinOccupancy(freqRange, occupancy)
sensorObj.updateMaxOccupancy(freqRange, occupancy)
sensorObj.updateOccupancyCount(freqRange, occupancy)
LocationMessage.updateMaxBandOccupancy(lastLocationPost,
freqRange, occupancy)
LocationMessage.updateMinBandOccupancy(lastLocationPost,
freqRange, occupancy)
LocationMessage.updateOccupancySum(lastLocationPost, freqRange,
occupancy)
sensorObj.updateTime(freqRange, Message.getTime(jsonData))
sensorObj.updateDataMessageTimeStamp(Message.getTime(jsonData))
SensorDb.updateSensor(sensorObj.getJson(), False, False)
DataMessage.setMaxPower(jsonData, maxPower)
DataMessage.setMinPower(jsonData, minPower)
#if filedesc is not None:
# print json.dumps(jsonData, sort_keys=True, indent=4)
if DataMessage.isProcessed(jsonData):
dataPosts.insert(jsonData)
else:
DbCollections.getUnprocessedDataMessages(sensorId).insert(jsonData)
# Update location specific information for this sensor.
LocationMessage.addFreqRange(lastLocationPost, freqRange)
LocationMessage.setMessageTimeStampForBand(lastLocationPost, freqRange,
Message.getTime(jsonData))
LocationMessage.incrementMessageCount(lastLocationPost)
LocationMessage.incrementBandCount(lastLocationPost, freqRange)
LocationMessage.setMinMaxPower(lastLocationPost, minPower, maxPower)
locationPostId = lastLocationPost["_id"]
del lastLocationPost["_id"]
locationPosts.update({"_id": locationPostId},
{"$set": lastLocationPost},
upsert=False)
end_time = time.time()
if filedesc is not None:
print " Insertion time " + str(end_time - start_time)
def getSensorData(ws):
"""
Handle sensor data streaming requests from the web browser.
"""
try:
util.debugPrint("DataStreamng:getSensorData")
global memCache
if memCache is None:
memCache = MemCache()
token = ws.receive()
print "token = ", token
parts = token.split(":")
if parts is None or len(parts) < 5:
ws.close()
return
sessionId = parts[0]
if not authentication.checkSessionId(sessionId, "user"):
ws.close()
return
import SessionLock
if SessionLock.findSessionByRemoteAddr(sessionId) is not None:
ws.send(dumps({"status": "Streaming session already open"}))
ws.close()
return
sensorId = parts[1]
systemToDetect = parts[2]
minFreq = int(parts[3])
maxFreq = int(parts[4])
util.debugPrint("sensorId " + sensorId)
memCache.incrementStreamingListenerCount(sensorId)
sensorObj = SensorDb.getSensorObj(sensorId)
if sensorObj is None:
ws.send(dumps({"status": "Sensor not found: " + sensorId}))
bandName = systemToDetect + ":" + str(minFreq) + ":" + str(maxFreq)
util.debugPrint("isStreamingEnabled = " +
str(sensorObj.isStreamingEnabled()))
lastDataMessage = memCache.loadLastDataMessage(sensorId, bandName)
key = sensorId + ":" + bandName
if key not in lastDataMessage or not sensorObj.isStreamingEnabled():
ws.send(
dumps({
"status":
"NO_DATA: Data message not found or streaming not enabled"
}))
else:
ws.send(dumps({"status": "OK"}))
util.debugPrint("DataStreaming lastDataMessage: " +
str(lastDataMessage[key]))
ws.send(str(lastDataMessage[key]))
lastdatatime = -1
drift = 0
while True:
secondsPerFrame = sensorObj.getStreamingSecondsPerFrame()
lastdataseen = memCache.loadLastDataSeenTimeStamp(
sensorId, bandName)
if key in lastdataseen and lastdatatime != lastdataseen[key]:
lastdatatime = lastdataseen[key]
sensordata = memCache.loadSensorData(sensorId, bandName)
memCache.incrementDataConsumedCounter(sensorId, bandName)
currentTime = time.time()
lastdatasent = currentTime
drift = drift + (currentTime -
lastdatasent) - secondsPerFrame
ws.send(sensordata[key])
# If we drifted, send the last reading again to fill in.
if drift < 0:
drift = 0
if drift > secondsPerFrame:
if APPLY_DRIFT_CORRECTION:
util.debugPrint("Drift detected")
ws.send(sensordata[key])
drift = 0
sleepTime = secondsPerFrame
gevent.sleep(sleepTime)
except:
traceback.print_exc()
ws.close()
util.debugPrint("Error writing to websocket")
finally:
memCache.decrementStreamingListenerCount(sensorId)
def runGarbageCollector(sensorId):
SessionLock.acquire()
try:
userCount = SessionLock.getUserSessionCount()
if userCount != 0:
return {"status": "NOK",
"ErrorMessage": "Active user session detected"}
sensorObj = SensorDb.getSensorObj(sensorId)
if sensorObj is None:
return {"status": "NOK", "ErrorMessage": "Sensor Not found"}
if sensorObj.getSensorStatus() != DISABLED:
return {"status": "NOK",
"ErrorMessage": "Sensor is ENABLED -- DISABLE it first"}
dataRetentionDuration = sensorObj.getSensorDataRetentionDurationMonths(
)
dataRetentionTime = dataRetentionDuration * 30 * SECONDS_PER_DAY
cur = DbCollections.getDataMessages(sensorId).find(
{SENSOR_ID: sensorId})
#dataMessages = cur.sort('t', pymongo.ASCENDING)
currentTime = time.time()
for msg in cur:
insertionTime = Message.getInsertionTime(msg)
if currentTime - dataRetentionTime >= insertionTime:
DbCollections.getDataMessages(sensorId).remove(msg)
msgutils.removeData(msg)
else:
break
# Now redo our book keeping summary fields.
#dataMessages = cur.sort('t', pymongo.ASCENDING)
locationMessages = DbCollections.getLocationMessages().find(
{SENSOR_ID: sensorId})
for locationMessage in locationMessages:
insertionTime = Message.getInsertionTime(locationMessage)
if currentTime - dataRetentionTime >= insertionTime:
DbCollections.getLocationMessages().remove(msg)
else:
LocationMessage.clean(locationMessage)
DbCollections.getLocationMessages().update(
{"_id": locationMessage["_id"]}, {"$set": locationMessage},
upsert=False)
sensorObj.cleanSensorStats()
# Update the summary statistics.
cur = DbCollections.getDataMessages(sensorId).find(
{SENSOR_ID: sensorId})
for jsonData in cur:
freqRange = DataMessage.getFreqRange(jsonData)
minPower = DataMessage.getMinPower(jsonData)
maxPower = DataMessage.getMaxPower(jsonData)
messageId = DataMessage.getLocationMessageId(jsonData)
lastLocationPost = DbCollections.getLocationMessages().find_one({"_id": messageId})
if DataMessage.getMeasurementType(jsonData) == FFT_POWER:
minOccupancy = DataMessage.getMinOccupancy(jsonData)
maxOccupancy = DataMessage.getMaxOccupancy(jsonData)
meanOccupancy = DataMessage.getMeanOccupancy(jsonData)
sensorObj.updateMinOccupancy(freqRange, minOccupancy)
sensorObj.updateMaxOccupancy(freqRange, maxOccupancy)
sensorObj.updateOccupancyCount(freqRange, meanOccupancy)
LocationMessage.updateMaxBandOccupancy(lastLocationPost, freqRange,
maxOccupancy)
LocationMessage.updateMinBandOccupancy(lastLocationPost, freqRange,
minOccupancy)
LocationMessage.updateOccupancySum(lastLocationPost, freqRange,
meanOccupancy)
else:
occupancy = DataMessage.getOccupancy(jsonData)
sensorObj.updateMinOccupancy(freqRange, occupancy)
sensorObj.updateMaxOccupancy(freqRange, occupancy)
sensorObj.updateOccupancyCount(freqRange, occupancy)
LocationMessage.updateMaxBandOccupancy(lastLocationPost, freqRange,
occupancy)
LocationMessage.updateMinBandOccupancy(lastLocationPost, freqRange,
occupancy)
LocationMessage.updateOccupancySum(lastLocationPost, freqRange,
occupancy)
sensorObj.updateTime(freqRange, Message.getTime(jsonData))
sensorObj.updateDataMessageTimeStamp(Message.getTime(jsonData))
SensorDb.updateSensor(sensorObj.getJson(), False, False)
DbCollections.getLocationMessages().update(
{"_id": lastLocationPost["_id"]}, {"$set": lastLocationPost},
upsert=False)
# Garbage collect the unprocessed data messages.
cur = DbCollections.getUnprocessedDataMessages(sensorId).find({SENSOR_ID: sensorId})
if cur is not None:
dataMessages = cur.sort('t', pymongo.ASCENDING)
for msg in dataMessages:
insertionTime = Message.getInsertionTime(msg)
if currentTime - dataRetentionTime >= insertionTime:
DbCollections.getUnprocessedDataMessages(sensorId).remove(msg)
else:
break
DbCollections.dropDailyOccupancyCache(sensorId)
return {"status": "OK", "sensors": SensorDb.getAllSensors()}
finally:
SessionLock.release()