def startStreamingServer(port):
"""
Start the streaming server and accept connections.
"""
global memCache
if memCache is None:
memCache = MemCache()
soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
l_onoff = 1
l_linger = 0
soc.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', l_onoff, l_linger))
portAssigned = False
for p in range(port, port + 10, 2):
try:
print 'Trying port ', p
soc.bind(('0.0.0.0', p))
soc.listen(10)
socketServerPort = p
memCache.setSocketServerPort(p)
portAssigned = True
util.debugPrint("DataStreaming: Bound to port " + str(p))
break
except:
print sys.exc_info()
traceback.print_exc()
util.debugPrint("DataStreaming: Bind failed - retry")
if portAssigned:
global occupancyQueue
socketServer = startSocketServer(soc, socketServerPort)
socketServer.start()
else:
util.errorPrint(
"DataStreaming: Streaming disabled on worker - no port found.")
def notifyConfigChange(sensorId):
memCache = MemCache()
port = memCache.getSensorArmPort(sensorId)
soc = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
soc.sendto(json.dumps({
"sensorId": sensorId,
"command": "retune"
}), ("localhost", port))
soc.close()
def restartSensor(sensorId):
memCache = MemCache()
pid = memCache.getStreamingServerPid(sensorId)
if pid != -1:
try:
util.debugPrint("restartSensor: sensorId " + sensorId + " pid " +
str(pid) + " sending sigint")
os.kill(pid, signal.SIGINT)
except:
util.errorPrint("restartSensor: Pid " + str(pid) + " not found")
else:
util.debugPrint("restartSensor: pid not found")
def workerProc(conn):
global bbuf
global memCache
if memCache is None:
memCache = MemCache()
bbuf = BBuf(conn)
readFromInput(bbuf, conn)
def deleteCaptureEvents(sensorId, startDate, sessionId):
"""
Delete the events from the capture db.
Send a message to the sensor to do the same.
"""
try:
if not authentication.checkSessionId(sessionId, ADMIN):
util.debugPrint("deleteCaptureEvents: failed authentication")
abort(403)
sdate = int(startDate)
if sdate < 0:
util.debugPrint("deleteCaptureEvents: illegal param")
abort(400)
else:
CaptureDb.deleteCaptureDb(sensorId, sdate)
global memCache
if memCache is None:
memCache = MemCache()
command = json.dumps({
"sensorId": sensorId,
"timestamp": sdate,
"command": "garbage_collect"
})
sendCommandToSensor(sensorId, command)
return jsonify({STATUS: "OK"})
except:
print "Unexpected error:", sys.exc_info()[0]
print sys.exc_info()
traceback.print_exc()
util.logStackTrace(sys.exc_info())
raise
def runSensorCommandDispatchWorker(conn, sensorId):
soc = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
global memCache
memCache = MemCache()
port = memCache.getSensorArmPort(sensorId)
memCache.setStreamingCommandDispatcherPid(sensorId)
soc.bind(("localhost", port))
util.debugPrint("runSensorCommandDispatchWorker: port = " + str(port) +
" pid " + str(os.getpid()))
try:
while True:
command, addr = soc.recvfrom(1024)
if command is None or command == "":
break
util.debugPrint("runSensorArmWorker: got a message " +
str(command))
conn.send(command.encode())
commandJson = json.loads(command)
if commandJson['command'] == 'retune' or commandJson[
'command'] == 'exit':
break
finally:
util.debugPrint("runSensorCommandDispatchWorker: closing socket")
soc.close()
memCache.removeStreamingCommandDispatcherPid(sensorId)
time.sleep(1)
conn.close()
os._exit(0)
def dataStream(ws):
"""
Handle the data stream from a sensor.
"""
print "Got a connection"
bbuf = MyByteBuffer(ws)
global memCache
if memCache is None:
memCache = MemCache()
readFromInput(bbuf, True)
def getSocketServerPort(sensorId):
retval = {}
global memCache
if memCache is None:
memCache = MemCache()
sensor = SensorDb.getSensorObj(sensorId)
print "sensorStatus ", sensor.getSensorStatus()
if sensor is None or sensor.getSensorStatus() != ENABLED \
or not sensor.isStreamingEnabled():
retval["port"] = -1
return retval
retval["port"] = STREAMING_SERVER_PORT
return retval
def readFromInput(bbuf, conn):
util.debugPrint("DataStreaming:readFromInput")
soc = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sensorCommandDispatcherPid = None
memCache = MemCache()
try:
while True:
lengthString = ""
while True:
lastChar = bbuf.readChar()
if lastChar is None:
time.sleep(0.1)
return
if len(lengthString) > 1000:
raise Exception("Formatting error")
if lastChar == '{':
headerLength = int(lengthString.rstrip())
break
else:
lengthString += str(lastChar)
jsonStringBytes = "{"
while len(jsonStringBytes) < headerLength:
jsonStringBytes += str(bbuf.readChar())
jsonData = json.loads(jsonStringBytes)
if not any(k in jsonData for k in (TYPE, SENSOR_ID, SENSOR_KEY)):
err = "Sensor Data Stream: Missing a required field"
util.errorPrint(err)
util.errorPrint("Invalid message -- closing connection: " +
json.dumps(jsonData, indent=4))
raise Exception("Invalid message")
return
sensorId = jsonData[SENSOR_ID]
global mySensorId
if mySensorId is None:
mySensorId = sensorId
elif mySensorId != sensorId:
raise Exception("Sensor ID mismatch " + mySensorId + " / " +
sensorId)
sensorKey = jsonData[SENSOR_KEY]
if not authentication.authenticateSensor(sensorId, sensorKey):
util.debugPrint("jsonData " + json.dumps(jsonData, indent=4))
util.errorPrint("Sensor authentication failed: " + sensorId +
" sensorKey " + sensorKey)
raise Exception("Authentication failure")
return
if memCache.getStreamingServerPid(sensorId) == -1:
memCache.setStreamingServerPid(sensorId)
elif memCache.getStreamingServerPid(sensorId) != os.getpid():
util.errorPrint(
"Handling connection for this sensor already " +
str(os.getpid()))
try:
os.kill(memCache.getStreamingServerPid(sensorId),
signal.SIGKILL)
memCache.setStreamingServerPid(sensorId)
except:
print "Unexpected error:", sys.exc_info()[0]
print sys.exc_info()
traceback.print_exc()
util.logStackTrace(sys.exc_info())
util.debugPrint(
"Problem killing process " +
str(memCache.getStreamingServerPid(sensorId)))
if memCache.getStreamingCommandDispatcherPid(sensorId) != -1:
try:
os.kill(
memCache.getStreamingCommandDispatcherPid(
sensorId), signal.SIGKILL)
except:
memCache.removeStreamingCommandDispatcherPid(sensorId)
util.debugPrint("Process not found. Proceeding ")
util.debugPrint("DataStreaming: Message = " +
dumps(jsonData, sort_keys=True, indent=4))
sensorObj = SensorDb.getSensorObj(sensorId)
if not sensorObj.isStreamingEnabled(
) or sensorObj.getStreamingParameters() is None:
raise Exception("Streaming is not enabled")
return
# the last time a data message was inserted
if jsonData[TYPE] == DATA:
util.debugPrint("pubsubPort: " +
str(memCache.getPubSubPort(sensorId)))
if "Sys2Detect" not in jsonData:
jsonData[SYS_TO_DETECT] = "LTE"
DataMessage.init(jsonData)
t = Process(target=runSensorCommandDispatchWorker,
args=(conn, sensorId))
t.start()
sensorCommandDispatcherPid = t.pid
childPids.append(sensorCommandDispatcherPid)
cutoff = DataMessage.getThreshold(jsonData)
n = DataMessage.getNumberOfFrequencyBins(jsonData)
sensorId = DataMessage.getSensorId(jsonData)
lastDataMessageReceivedAt[sensorId] = time.time()
lastDataMessageOriginalTimeStamp[
sensorId] = DataMessage.getTime(jsonData)
# Check if the measurement type reported by the sensor
# matches that in the sensordb
measurementType = DataMessage.getMeasurementType(jsonData)
if sensorObj.getMeasurementType() != measurementType:
err = "Measurement type mismatch "
err += sensorObj.getMeasurementType()
err += " / "
err += measurementType
raise Exception(err)
# Check if the time per measurement reported by the sensor
# matches that in the sensordb
timePerMeasurement = sensorObj.getStreamingSecondsPerFrame()
util.debugPrint("StreamingServer: timePerMeasurement " +
str(timePerMeasurement))
if timePerMeasurement != DataMessage.getTimePerMeasurement(
jsonData):
err = "TimePerMeasurement mismatch "
err += str(timePerMeasurement)
err += "/"
err += str(DataMessage.getTimePerMeasurement(jsonData))
raise Exception(err)
# The sampling interval for write to the database.
s = sensorObj.getStreamingSamplingIntervalSeconds()
streamingSamplingIntervalSeconds = s
# The number of measurements per capture
measurementsPerCapture = int(streamingSamplingIntervalSeconds /
timePerMeasurement)
util.debugPrint("StreamingServer: measurementsPerCapture " +
str(measurementsPerCapture))
# The number of power value samples per capture.
samplesPerCapture = int(
(streamingSamplingIntervalSeconds / timePerMeasurement) *
n)
# The number of spectrums per frame sent to the browser.
spectrumsPerFrame = 1
jsonData[SPECTRUMS_PER_FRAME] = spectrumsPerFrame
# The streaming filter of the sensor (MAX_HOLD or AVG)
jsonData[STREAMING_FILTER] = sensorObj.getStreamingFilter()
# The band name sys2detect:minfreq:maxfreq string for the
# reported measurement.
bandName = DataMessage.getFreqRange(jsonData)
# Keep a copy of the last data message for periodic insertion
# into the db
memCache.setLastDataMessage(sensorId, bandName,
json.dumps(jsonData))
# captureBufferCounter is a pointer into the capture buffer.
captureBufferCounter = 0
powerArrayCounter = 0
timingCounter = 0
# initialize the "prev occupancy array"
prevOccupancyArray = [-1 for i in range(0, n)]
occupancyArray = [0 for i in range(0, n)]
occupancyTimer = time.time()
if sensorId not in lastDataMessage:
lastDataMessage[sensorId] = jsonData
powerVal = [0 for i in range(0, n)]
startTime = time.time()
sensorObj = SensorDb.getSensorObj(sensorId)
if sensorObj is None:
raise Exception("Sensor not found")
if sensorObj.getSensorStatus() == DISABLED:
bbuf.close()
raise Exception("Sensor is disabled")
if not sensorObj.isStreamingEnabled():
raise Exception("Streaming is disabled")
enb = sensorObj.isStreamingCaptureEnabled()
isStreamingCaptureEnabled = enb
util.debugPrint("isStreamingCaptureEnabled : " + str(enb) +
" samplesPerCapture " + str(samplesPerCapture))
if isStreamingCaptureEnabled:
sensorData = [0 for i in range(0, samplesPerCapture)]
while True:
data = bbuf.readByte()
if isStreamingCaptureEnabled:
sensorData[captureBufferCounter] = data
powerVal[powerArrayCounter] = data
now = time.time()
if isStreamingCaptureEnabled and captureBufferCounter + 1 == samplesPerCapture:
# Buffer is full so push the data into mongod.
util.debugPrint("Inserting Data message")
captureBufferCounter = 0
# Time offset since the last data message was received.
timeOffset = time.time(
) - lastDataMessageReceivedAt[sensorId]
# Offset the capture by the time since the DataMessage header was received.
lastDataMessage[sensorId]["t"] = int(
lastDataMessageOriginalTimeStamp[sensorId] +
int(timeOffset))
lastDataMessage[sensorId][
"nM"] = measurementsPerCapture
lastDataMessage[sensorId]["mPar"]["td"] = int(
now - occupancyTimer)
lastDataMessage[sensorId]["mPar"][
"tm"] = timePerMeasurement
headerStr = json.dumps(lastDataMessage[sensorId],
indent=4)
util.debugPrint("StreamingServer: headerStr " +
headerStr)
headerLength = len(headerStr)
if isStreamingCaptureEnabled:
# Start the db operation in a seperate process
p = Process(target=populate_db.put_data,
args=(headerStr, headerLength),
kwargs={
"filedesc": None,
"powers": sensorData
})
p.start()
lastDataMessageInsertedAt[sensorId] = time.time()
occupancyTimer = time.time()
else:
captureBufferCounter = captureBufferCounter + 1
if data > cutoff:
occupancyArray[powerArrayCounter] = 1
else:
occupancyArray[powerArrayCounter] = 0
# print "occupancyArray", occupancyArray
if (powerArrayCounter + 1) == n:
# Get the occupancy subscription counter.
if memCache.getSubscriptionCount(sensorId) != 0:
if not np.array_equal(occupancyArray,
prevOccupancyArray):
port = memCache.getPubSubPort(sensorId)
soc.sendto(
json.dumps({sensorId: occupancyArray}),
("localhost", port))
prevOccupancyArray = np.array(occupancyArray)
# sending data as CSV values to the browser
listenerCount = memCache.getStreamingListenerCount(
sensorId)
if listenerCount > 0:
sensordata = str(powerVal)[1:-1].replace(" ", "")
memCache.setSensorData(sensorId, bandName,
sensordata)
# Record the occupancy for the measurements.
# Allow for 10% jitter.
if timingCounter == 1000 and checkForDataRate:
if (((now - startTime) / 1000.0 <
timePerMeasurement / 2)
or ((now - startTime) / 1000.0 >
timePerMeasurement * 2)):
print " delta ", now - startTime, "global counter ", powerArrayCounter
util.errorPrint(
"Data coming in too fast or too slow - sensor configuration problem."
)
raise Exception(
"Data coming in too fast - sensor configuration problem."
)
else:
startTime = now
lastdataseen = now
if listenerCount > 0:
memCache.setLastDataSeenTimeStamp(
sensorId, bandName, lastdataseen)
powerArrayCounter = 0
else:
powerArrayCounter = powerArrayCounter + 1
timingCounter = timingCounter + 1
elif jsonData[TYPE] == SYS:
util.debugPrint(
"DataStreaming: Got a System message -- adding to the database"
)
populate_db.put_data(jsonStringBytes, headerLength)
elif jsonData[TYPE] == LOC:
util.debugPrint(
"DataStreaming: Got a Location Message -- adding to the database"
)
populate_db.put_data(jsonStringBytes, headerLength)
finally:
util.debugPrint("Closing sockets for sensorId " + sensorId)
memCache.removeStreamingServerPid(sensorId)
port = memCache.getSensorArmPort(sensorId)
sendCommandToSensor(
sensorId, json.dumps({
"sensorId": sensorId,
"command": "exit"
}))
memCache.releaseSensorArmPort(sensorId)
bbuf.close()
time.sleep(1)
soc.close()
# kill the command dispatcher for good measure.
try:
if sensorCommandDispatcherPid is not None:
os.kill(sensorCommandDispatcherPid, signal.SIGKILL)
except:
util.debugPrint("Process not found. " +
str(sensorCommandDispatcherPid))
def getSensorData(ws):
"""
Handle sensor data streaming requests from the web browser.
"""
try:
util.debugPrint("DataStreamng:getSensorData")
global memCache
if memCache is None:
memCache = MemCache()
token = ws.receive()
print "token = ", token
parts = token.split(":")
if parts is None or len(parts) < 5:
ws.close()
return
sessionId = parts[0]
if not authentication.checkSessionId(sessionId, "user"):
ws.close()
return
import SessionLock
if SessionLock.findSessionByRemoteAddr(sessionId) is not None:
ws.send(dumps({"status": "Streaming session already open"}))
ws.close()
return
sensorId = parts[1]
systemToDetect = parts[2]
minFreq = int(parts[3])
maxFreq = int(parts[4])
util.debugPrint("sensorId " + sensorId)
memCache.incrementStreamingListenerCount(sensorId)
sensorObj = SensorDb.getSensorObj(sensorId)
if sensorObj is None:
ws.send(dumps({"status": "Sensor not found: " + sensorId}))
bandName = systemToDetect + ":" + str(minFreq) + ":" + str(maxFreq)
util.debugPrint("isStreamingEnabled = " +
str(sensorObj.isStreamingEnabled()))
lastDataMessage = memCache.loadLastDataMessage(sensorId, bandName)
key = sensorId + ":" + bandName
if key not in lastDataMessage or not sensorObj.isStreamingEnabled():
ws.send(
dumps({
"status":
"NO_DATA: Data message not found or streaming not enabled"
}))
else:
ws.send(dumps({"status": "OK"}))
util.debugPrint("DataStreaming lastDataMessage: " +
str(lastDataMessage[key]))
ws.send(str(lastDataMessage[key]))
lastdatatime = -1
drift = 0
while True:
secondsPerFrame = sensorObj.getStreamingSecondsPerFrame()
lastdataseen = memCache.loadLastDataSeenTimeStamp(
sensorId, bandName)
if key in lastdataseen and lastdatatime != lastdataseen[key]:
lastdatatime = lastdataseen[key]
sensordata = memCache.loadSensorData(sensorId, bandName)
memCache.incrementDataConsumedCounter(sensorId, bandName)
currentTime = time.time()
lastdatasent = currentTime
drift = drift + (currentTime -
lastdatasent) - secondsPerFrame
ws.send(sensordata[key])
# If we drifted, send the last reading again to fill in.
if drift < 0:
drift = 0
if drift > secondsPerFrame:
if APPLY_DRIFT_CORRECTION:
util.debugPrint("Drift detected")
ws.send(sensordata[key])
drift = 0
sleepTime = secondsPerFrame
gevent.sleep(sleepTime)
except:
traceback.print_exc()
ws.close()
util.debugPrint("Error writing to websocket")
finally:
memCache.decrementStreamingListenerCount(sensorId)