def updateSensorWorker(sessionId):
try:
util.debugPrint("updateSensor")
if not Config.isConfigured():
util.debugPrint("Please configure system")
return make_response("Please configure system", 500)
if not authentication.checkSessionId(sessionId, ADMIN):
return make_response("Session not found.", 403)
requestStr = request.data
sensorConfig = json.loads(requestStr)
return jsonify(SensorDb.updateSensor(sensorConfig))
except:
print "Unexpected error:", sys.exc_info()[0]
print sys.exc_info()
traceback.print_exc()
util.logStackTrace(sys.exc_info())
raise
def put_data(jsonString,
headerLength,
filedesc=None,
powers=None,
streamOccupancies=None):
"""
put data in the database. jsonString starts with {. If filedesc is None
then the data part of the message is appended to the message (immediately follows it).
Otherwise, the data is read from filedesc.
"""
start_time = time.time()
if filedesc is None:
# We are not reading from a file:
# Assume we are given the message in the string with the data
# tacked at the end of it.
jsonStringBytes = jsonString[0:headerLength]
else:
jsonStringBytes = jsonString
util.debugPrint("jsonStringBytes = " + jsonStringBytes)
jsonData = json.loads(jsonStringBytes)
sensorId = jsonData[SENSOR_ID]
sensorKey = jsonData[SENSOR_KEY]
if not authentication.authenticateSensor(sensorId, sensorKey):
raise Exception("Sensor Authentication Failure")
sensorObj = SensorDb.getSensorObj(sensorId)
if not sensorObj.getSensorStatus() == ENABLED:
raise Exception("Sensor is disabled")
# remove the sensor key from metadata for safety.
del jsonData[SENSOR_KEY]
locationPosts = DbCollections.getLocationMessages()
systemPosts = DbCollections.getSystemMessages()
dataPosts = DbCollections.getDataMessages(sensorId)
db = DbCollections.getSpectrumDb()
currentLocalTime = time.time()
Message.setInsertionTime(jsonData, currentLocalTime)
if jsonData[TYPE] == SYS:
# see if this system message already exists in the DB to avoid duplicates.
query = {SENSOR_ID: jsonData[SENSOR_ID], "t": jsonData["t"]}
found = systemPosts.find_one(query)
if CAL in jsonData:
calStr = jsonData[CAL]
# Ugly!! Need to fix this.
if calStr != "N/A":
n = jsonData[CAL]["mPar"]["n"]
nM = jsonData[CAL]["nM"]
sensorId = jsonData[SENSOR_ID]
if n * nM != 0:
dataType = jsonData[CAL][DATA_TYPE]
lengthToRead = n * nM
if filedesc is not None:
messageBytes = readDataFromFileDesc(
filedesc, dataType, lengthToRead)
elif powers is None:
messageBytes = jsonString[headerLength:]
else:
# TODO -- deal with the other data types here
messageBytes = struct.pack('%sb' % len(powers),
*powers)
fs = gridfs.GridFS(db, jsonData[SENSOR_ID] + "_data")
key = fs.put(messageBytes)
jsonData[CAL][DATA_KEY] = str(key)
if found is None:
systemPosts.ensure_index([('t', pymongo.DESCENDING)])
systemPosts.insert(jsonData)
else:
util.debugPrint("not inserting duplicate system post")
end_time = time.time()
util.debugPrint("Insertion time " + str(end_time - start_time))
sensorObj.updateSystemMessageTimeStamp(Message.getTime(jsonData))
SensorDb.updateSensor(sensorObj.getJson(), False, False)
elif jsonData[TYPE] == LOC:
print(json.dumps(jsonData, sort_keys=True, indent=4))
sensorId = jsonData[SENSOR_ID]
t = jsonData['t']
lat = jsonData[LAT]
lon = jsonData[LON]
alt = jsonData[ALT]
query = {SENSOR_ID: sensorId, LAT: lat, LON: lon, ALT: alt}
locMsg = locationPosts.find_one(query)
if locMsg is not None:
print "Location Post already exists - not updating "
return
(to_zone,
timeZoneName) = timezone.getLocalTimeZoneFromGoogle(t, lat, lon)
# If google returned null, then override with local information
if to_zone is None:
if TIME_ZONE_KEY in jsonData:
to_zone = jsonData[TIME_ZONE_KEY]
else:
raise Exception("ERROR: Unable to determine timeZone ")
else:
jsonData[TIME_ZONE_KEY] = to_zone
# insert the loc message into the database.
db.locationMessages.ensure_index([('t', pymongo.DESCENDING)])
locationPosts.insert(jsonData)
end_time = time.time()
sensorObj.updateLocationMessageTimeStamp(Message.getTime(jsonData))
SensorDb.updateSensor(sensorObj.getJson(), False, False)
print "inserted Location Message. Insertion time " + str(end_time -
start_time)
elif jsonData[TYPE] == DATA:
# BUG BUG -- we need to fix this. Need new data.
if SYS_TO_DETECT not in jsonData:
jsonData[SYS_TO_DETECT] = "LTE"
# Fix up issue with sys2detect - should have no spaces.
# BUGBUG -- this is ugly. Should reject the data.
jsonData[SYS_TO_DETECT] = jsonData[SYS_TO_DETECT].replace(" ", "")
DataMessage.init(jsonData)
freqRange = DataMessage.getFreqRange(jsonData)
if freqRange not in sensorObj.getThreshold():
raise Exception("ERROR: Frequency Band " + freqRange +
" not found")
lastSystemPost = systemPosts.find_one({
SENSOR_ID: sensorId,
"t": {
"$lte": Message.getTime(jsonData)
}
})
lastLocationPost = locationPosts.find_one({
SENSOR_ID: sensorId,
"t": {
"$lte": Message.getTime(jsonData)
}
})
if lastLocationPost is None or lastSystemPost is None:
raise Exception("Location post or system post not found for " +
sensorId)
# Check for duplicates
query = {SENSOR_ID: sensorId, "t": Message.getTime(jsonData)}
found = DbCollections.getDataMessages(sensorId).find_one(query)
# record the location message associated with the data.
DataMessage.setLocationMessageId(jsonData,
str(lastLocationPost['_id']))
DataMessage.setSystemMessageId(jsonData, str(lastSystemPost['_id']))
# prev data message.
nM = DataMessage.getNumberOfMeasurements(jsonData)
n = DataMessage.getNumberOfFrequencyBins(jsonData)
lengthToRead = n * nM
dataType = DataMessage.getDataType(jsonData)
if lengthToRead != 0:
if filedesc is not None:
messageBytes = readDataFromFileDesc(filedesc, dataType,
lengthToRead)
elif powers is None:
messageBytes = jsonString[headerLength:]
else:
# TODO - deal with the other data types here.
messageBytes = struct.pack("%sb" % len(powers), *powers)
occupancyBytes = None
if streamOccupancies is not None:
occupancyBytes = struct.pack("%sb" % len(streamOccupancies),
*streamOccupancies)
# Note: The data needs to be read before it is rejected.
if found is not None:
util.debugPrint("ignoring duplicate data message")
return
if lengthToRead != 0:
fs = gridfs.GridFS(db, sensorId + "_data")
key = fs.put(messageBytes)
DataMessage.setDataKey(jsonData, str(key))
if occupancyBytes is not None:
key = fs.put(occupancyBytes)
DataMessage.setOccupancyKey(jsonData, str(key))
DataMessage.setOccupancyVectorLength(jsonData, len(occupancyBytes))
cutoff = DataMessage.getThreshold(jsonData)
sensorMeasurementType = SensorDb.getSensor(sensorId)[MEASUREMENT_TYPE]
if DataMessage.getMeasurementType(jsonData) != sensorMeasurementType:
raise Exception(
"MeasurementType Mismatch between sensor and DataMessage")
#dataPosts.ensure_index([('t', pymongo.ASCENDING)])
maxPower = -1000
minPower = 1000
if DataMessage.getMeasurementType(jsonData) == FFT_POWER:
occupancyCount = [0 for i in range(0, nM)]
if powers is None:
powerVal = np.array(np.zeros(n * nM))
else:
powerVal = np.array(powers)
# unpack the power array.
if dataType == BINARY_INT8 and powers is None:
for i in range(0, lengthToRead):
powerVal[i] = struct.unpack('b', messageBytes[i:i + 1])[0]
maxPower = np.max(powerVal)
minPower = np.min(powerVal)
powerArray = powerVal.reshape(nM, n)
for i in range(0, nM):
occupancyCount[i] = float(
len(filter(lambda x: x >= cutoff,
powerArray[i, :]))) / float(n)
minOccupancy = np.min(occupancyCount)
maxOccupancy = np.max(occupancyCount)
meanOccupancy = np.mean(occupancyCount)
medianOccupancy = np.median(occupancyCount)
DataMessage.setMaxOccupancy(jsonData, maxOccupancy)
DataMessage.setMeanOccupancy(jsonData, meanOccupancy)
DataMessage.setMinOccupancy(jsonData, minOccupancy)
DataMessage.setMedianOccupancy(jsonData, medianOccupancy)
if DataMessage.isProcessed(jsonData):
sensorObj.updateMinOccupancy(freqRange, minOccupancy)
sensorObj.updateMaxOccupancy(freqRange, maxOccupancy)
sensorObj.updateOccupancyCount(freqRange, meanOccupancy)
LocationMessage.updateMaxBandOccupancy(lastLocationPost,
freqRange, maxOccupancy)
LocationMessage.updateMinBandOccupancy(lastLocationPost,
freqRange, minOccupancy)
LocationMessage.updateOccupancySum(lastLocationPost, freqRange,
meanOccupancy)
else:
if dataType == ASCII:
powerVal = eval(messageBytes)
else:
for i in range(0, lengthToRead):
powerVal[i] = struct.unpack('f', messageBytes[i:i + 4])[0]
maxPower = np.max(powerVal)
minPower = np.min(powerVal)
occupancyCount = float(len(filter(lambda x: x >= cutoff,
powerVal)))
occupancy = occupancyCount / float(len(powerVal))
DataMessage.setOccupancy(jsonData, occupancy)
if DataMessage.isProcessed(jsonData):
sensorObj.updateMinOccupancy(freqRange, occupancy)
sensorObj.updateMaxOccupancy(freqRange, occupancy)
sensorObj.updateOccupancyCount(freqRange, occupancy)
LocationMessage.updateMaxBandOccupancy(lastLocationPost,
freqRange, occupancy)
LocationMessage.updateMinBandOccupancy(lastLocationPost,
freqRange, occupancy)
LocationMessage.updateOccupancySum(lastLocationPost, freqRange,
occupancy)
sensorObj.updateTime(freqRange, Message.getTime(jsonData))
sensorObj.updateDataMessageTimeStamp(Message.getTime(jsonData))
SensorDb.updateSensor(sensorObj.getJson(), False, False)
DataMessage.setMaxPower(jsonData, maxPower)
DataMessage.setMinPower(jsonData, minPower)
#if filedesc is not None:
# print json.dumps(jsonData, sort_keys=True, indent=4)
if DataMessage.isProcessed(jsonData):
dataPosts.insert(jsonData)
else:
DbCollections.getUnprocessedDataMessages(sensorId).insert(jsonData)
# Update location specific information for this sensor.
LocationMessage.addFreqRange(lastLocationPost, freqRange)
LocationMessage.setMessageTimeStampForBand(lastLocationPost, freqRange,
Message.getTime(jsonData))
LocationMessage.incrementMessageCount(lastLocationPost)
LocationMessage.incrementBandCount(lastLocationPost, freqRange)
LocationMessage.setMinMaxPower(lastLocationPost, minPower, maxPower)
locationPostId = lastLocationPost["_id"]
del lastLocationPost["_id"]
locationPosts.update({"_id": locationPostId},
{"$set": lastLocationPost},
upsert=False)
end_time = time.time()
if filedesc is not None:
print " Insertion time " + str(end_time - start_time)
def runGarbageCollector(sensorId):
SessionLock.acquire()
try:
userCount = SessionLock.getUserSessionCount()
if userCount != 0:
return {"status": "NOK",
"ErrorMessage": "Active user session detected"}
sensorObj = SensorDb.getSensorObj(sensorId)
if sensorObj is None:
return {"status": "NOK", "ErrorMessage": "Sensor Not found"}
if sensorObj.getSensorStatus() != DISABLED:
return {"status": "NOK",
"ErrorMessage": "Sensor is ENABLED -- DISABLE it first"}
dataRetentionDuration = sensorObj.getSensorDataRetentionDurationMonths(
)
dataRetentionTime = dataRetentionDuration * 30 * SECONDS_PER_DAY
cur = DbCollections.getDataMessages(sensorId).find(
{SENSOR_ID: sensorId})
#dataMessages = cur.sort('t', pymongo.ASCENDING)
currentTime = time.time()
for msg in cur:
insertionTime = Message.getInsertionTime(msg)
if currentTime - dataRetentionTime >= insertionTime:
DbCollections.getDataMessages(sensorId).remove(msg)
msgutils.removeData(msg)
else:
break
# Now redo our book keeping summary fields.
#dataMessages = cur.sort('t', pymongo.ASCENDING)
locationMessages = DbCollections.getLocationMessages().find(
{SENSOR_ID: sensorId})
for locationMessage in locationMessages:
insertionTime = Message.getInsertionTime(locationMessage)
if currentTime - dataRetentionTime >= insertionTime:
DbCollections.getLocationMessages().remove(msg)
else:
LocationMessage.clean(locationMessage)
DbCollections.getLocationMessages().update(
{"_id": locationMessage["_id"]}, {"$set": locationMessage},
upsert=False)
sensorObj.cleanSensorStats()
# Update the summary statistics.
cur = DbCollections.getDataMessages(sensorId).find(
{SENSOR_ID: sensorId})
for jsonData in cur:
freqRange = DataMessage.getFreqRange(jsonData)
minPower = DataMessage.getMinPower(jsonData)
maxPower = DataMessage.getMaxPower(jsonData)
messageId = DataMessage.getLocationMessageId(jsonData)
lastLocationPost = DbCollections.getLocationMessages().find_one({"_id": messageId})
if DataMessage.getMeasurementType(jsonData) == FFT_POWER:
minOccupancy = DataMessage.getMinOccupancy(jsonData)
maxOccupancy = DataMessage.getMaxOccupancy(jsonData)
meanOccupancy = DataMessage.getMeanOccupancy(jsonData)
sensorObj.updateMinOccupancy(freqRange, minOccupancy)
sensorObj.updateMaxOccupancy(freqRange, maxOccupancy)
sensorObj.updateOccupancyCount(freqRange, meanOccupancy)
LocationMessage.updateMaxBandOccupancy(lastLocationPost, freqRange,
maxOccupancy)
LocationMessage.updateMinBandOccupancy(lastLocationPost, freqRange,
minOccupancy)
LocationMessage.updateOccupancySum(lastLocationPost, freqRange,
meanOccupancy)
else:
occupancy = DataMessage.getOccupancy(jsonData)
sensorObj.updateMinOccupancy(freqRange, occupancy)
sensorObj.updateMaxOccupancy(freqRange, occupancy)
sensorObj.updateOccupancyCount(freqRange, occupancy)
LocationMessage.updateMaxBandOccupancy(lastLocationPost, freqRange,
occupancy)
LocationMessage.updateMinBandOccupancy(lastLocationPost, freqRange,
occupancy)
LocationMessage.updateOccupancySum(lastLocationPost, freqRange,
occupancy)
sensorObj.updateTime(freqRange, Message.getTime(jsonData))
sensorObj.updateDataMessageTimeStamp(Message.getTime(jsonData))
SensorDb.updateSensor(sensorObj.getJson(), False, False)
DbCollections.getLocationMessages().update(
{"_id": lastLocationPost["_id"]}, {"$set": lastLocationPost},
upsert=False)
# Garbage collect the unprocessed data messages.
cur = DbCollections.getUnprocessedDataMessages(sensorId).find({SENSOR_ID: sensorId})
if cur is not None:
dataMessages = cur.sort('t', pymongo.ASCENDING)
for msg in dataMessages:
insertionTime = Message.getInsertionTime(msg)
if currentTime - dataRetentionTime >= insertionTime:
DbCollections.getUnprocessedDataMessages(sensorId).remove(msg)
else:
break
DbCollections.dropDailyOccupancyCache(sensorId)
return {"status": "OK", "sensors": SensorDb.getAllSensors()}
finally:
SessionLock.release()