def put_data(jsonString,
headerLength,
filedesc=None,
powers=None,
streamOccupancies=None):
"""
put data in the database. jsonString starts with {. If filedesc is None
then the data part of the message is appended to the message (immediately follows it).
Otherwise, the data is read from filedesc.
"""
start_time = time.time()
if filedesc is None:
# We are not reading from a file:
# Assume we are given the message in the string with the data
# tacked at the end of it.
jsonStringBytes = jsonString[0:headerLength]
else:
jsonStringBytes = jsonString
util.debugPrint("jsonStringBytes = " + jsonStringBytes)
jsonData = json.loads(jsonStringBytes)
sensorId = jsonData[SENSOR_ID]
sensorKey = jsonData[SENSOR_KEY]
if not authentication.authenticateSensor(sensorId, sensorKey):
raise Exception("Sensor Authentication Failure")
sensorObj = SensorDb.getSensorObj(sensorId)
if not sensorObj.getSensorStatus() == ENABLED:
raise Exception("Sensor is disabled")
# remove the sensor key from metadata for safety.
del jsonData[SENSOR_KEY]
locationPosts = DbCollections.getLocationMessages()
systemPosts = DbCollections.getSystemMessages()
dataPosts = DbCollections.getDataMessages(sensorId)
db = DbCollections.getSpectrumDb()
currentLocalTime = time.time()
Message.setInsertionTime(jsonData, currentLocalTime)
if jsonData[TYPE] == SYS:
# see if this system message already exists in the DB to avoid duplicates.
query = {SENSOR_ID: jsonData[SENSOR_ID], "t": jsonData["t"]}
found = systemPosts.find_one(query)
if CAL in jsonData:
calStr = jsonData[CAL]
# Ugly!! Need to fix this.
if calStr != "N/A":
n = jsonData[CAL]["mPar"]["n"]
nM = jsonData[CAL]["nM"]
sensorId = jsonData[SENSOR_ID]
if n * nM != 0:
dataType = jsonData[CAL][DATA_TYPE]
lengthToRead = n * nM
if filedesc is not None:
messageBytes = readDataFromFileDesc(
filedesc, dataType, lengthToRead)
elif powers is None:
messageBytes = jsonString[headerLength:]
else:
# TODO -- deal with the other data types here
messageBytes = struct.pack('%sb' % len(powers),
*powers)
fs = gridfs.GridFS(db, jsonData[SENSOR_ID] + "_data")
key = fs.put(messageBytes)
jsonData[CAL][DATA_KEY] = str(key)
if found is None:
systemPosts.ensure_index([('t', pymongo.DESCENDING)])
systemPosts.insert(jsonData)
else:
util.debugPrint("not inserting duplicate system post")
end_time = time.time()
util.debugPrint("Insertion time " + str(end_time - start_time))
sensorObj.updateSystemMessageTimeStamp(Message.getTime(jsonData))
SensorDb.updateSensor(sensorObj.getJson(), False, False)
elif jsonData[TYPE] == LOC:
print(json.dumps(jsonData, sort_keys=True, indent=4))
sensorId = jsonData[SENSOR_ID]
t = jsonData['t']
lat = jsonData[LAT]
lon = jsonData[LON]
alt = jsonData[ALT]
query = {SENSOR_ID: sensorId, LAT: lat, LON: lon, ALT: alt}
locMsg = locationPosts.find_one(query)
if locMsg is not None:
print "Location Post already exists - not updating "
return
(to_zone,
timeZoneName) = timezone.getLocalTimeZoneFromGoogle(t, lat, lon)
# If google returned null, then override with local information
if to_zone is None:
if TIME_ZONE_KEY in jsonData:
to_zone = jsonData[TIME_ZONE_KEY]
else:
raise Exception("ERROR: Unable to determine timeZone ")
else:
jsonData[TIME_ZONE_KEY] = to_zone
# insert the loc message into the database.
db.locationMessages.ensure_index([('t', pymongo.DESCENDING)])
locationPosts.insert(jsonData)
end_time = time.time()
sensorObj.updateLocationMessageTimeStamp(Message.getTime(jsonData))
SensorDb.updateSensor(sensorObj.getJson(), False, False)
print "inserted Location Message. Insertion time " + str(end_time -
start_time)
elif jsonData[TYPE] == DATA:
# BUG BUG -- we need to fix this. Need new data.
if SYS_TO_DETECT not in jsonData:
jsonData[SYS_TO_DETECT] = "LTE"
# Fix up issue with sys2detect - should have no spaces.
# BUGBUG -- this is ugly. Should reject the data.
jsonData[SYS_TO_DETECT] = jsonData[SYS_TO_DETECT].replace(" ", "")
DataMessage.init(jsonData)
freqRange = DataMessage.getFreqRange(jsonData)
if freqRange not in sensorObj.getThreshold():
raise Exception("ERROR: Frequency Band " + freqRange +
" not found")
lastSystemPost = systemPosts.find_one({
SENSOR_ID: sensorId,
"t": {
"$lte": Message.getTime(jsonData)
}
})
lastLocationPost = locationPosts.find_one({
SENSOR_ID: sensorId,
"t": {
"$lte": Message.getTime(jsonData)
}
})
if lastLocationPost is None or lastSystemPost is None:
raise Exception("Location post or system post not found for " +
sensorId)
# Check for duplicates
query = {SENSOR_ID: sensorId, "t": Message.getTime(jsonData)}
found = DbCollections.getDataMessages(sensorId).find_one(query)
# record the location message associated with the data.
DataMessage.setLocationMessageId(jsonData,
str(lastLocationPost['_id']))
DataMessage.setSystemMessageId(jsonData, str(lastSystemPost['_id']))
# prev data message.
nM = DataMessage.getNumberOfMeasurements(jsonData)
n = DataMessage.getNumberOfFrequencyBins(jsonData)
lengthToRead = n * nM
dataType = DataMessage.getDataType(jsonData)
if lengthToRead != 0:
if filedesc is not None:
messageBytes = readDataFromFileDesc(filedesc, dataType,
lengthToRead)
elif powers is None:
messageBytes = jsonString[headerLength:]
else:
# TODO - deal with the other data types here.
messageBytes = struct.pack("%sb" % len(powers), *powers)
occupancyBytes = None
if streamOccupancies is not None:
occupancyBytes = struct.pack("%sb" % len(streamOccupancies),
*streamOccupancies)
# Note: The data needs to be read before it is rejected.
if found is not None:
util.debugPrint("ignoring duplicate data message")
return
if lengthToRead != 0:
fs = gridfs.GridFS(db, sensorId + "_data")
key = fs.put(messageBytes)
DataMessage.setDataKey(jsonData, str(key))
if occupancyBytes is not None:
key = fs.put(occupancyBytes)
DataMessage.setOccupancyKey(jsonData, str(key))
DataMessage.setOccupancyVectorLength(jsonData, len(occupancyBytes))
cutoff = DataMessage.getThreshold(jsonData)
sensorMeasurementType = SensorDb.getSensor(sensorId)[MEASUREMENT_TYPE]
if DataMessage.getMeasurementType(jsonData) != sensorMeasurementType:
raise Exception(
"MeasurementType Mismatch between sensor and DataMessage")
#dataPosts.ensure_index([('t', pymongo.ASCENDING)])
maxPower = -1000
minPower = 1000
if DataMessage.getMeasurementType(jsonData) == FFT_POWER:
occupancyCount = [0 for i in range(0, nM)]
if powers is None:
powerVal = np.array(np.zeros(n * nM))
else:
powerVal = np.array(powers)
# unpack the power array.
if dataType == BINARY_INT8 and powers is None:
for i in range(0, lengthToRead):
powerVal[i] = struct.unpack('b', messageBytes[i:i + 1])[0]
maxPower = np.max(powerVal)
minPower = np.min(powerVal)
powerArray = powerVal.reshape(nM, n)
for i in range(0, nM):
occupancyCount[i] = float(
len(filter(lambda x: x >= cutoff,
powerArray[i, :]))) / float(n)
minOccupancy = np.min(occupancyCount)
maxOccupancy = np.max(occupancyCount)
meanOccupancy = np.mean(occupancyCount)
medianOccupancy = np.median(occupancyCount)
DataMessage.setMaxOccupancy(jsonData, maxOccupancy)
DataMessage.setMeanOccupancy(jsonData, meanOccupancy)
DataMessage.setMinOccupancy(jsonData, minOccupancy)
DataMessage.setMedianOccupancy(jsonData, medianOccupancy)
if DataMessage.isProcessed(jsonData):
sensorObj.updateMinOccupancy(freqRange, minOccupancy)
sensorObj.updateMaxOccupancy(freqRange, maxOccupancy)
sensorObj.updateOccupancyCount(freqRange, meanOccupancy)
LocationMessage.updateMaxBandOccupancy(lastLocationPost,
freqRange, maxOccupancy)
LocationMessage.updateMinBandOccupancy(lastLocationPost,
freqRange, minOccupancy)
LocationMessage.updateOccupancySum(lastLocationPost, freqRange,
meanOccupancy)
else:
if dataType == ASCII:
powerVal = eval(messageBytes)
else:
for i in range(0, lengthToRead):
powerVal[i] = struct.unpack('f', messageBytes[i:i + 4])[0]
maxPower = np.max(powerVal)
minPower = np.min(powerVal)
occupancyCount = float(len(filter(lambda x: x >= cutoff,
powerVal)))
occupancy = occupancyCount / float(len(powerVal))
DataMessage.setOccupancy(jsonData, occupancy)
if DataMessage.isProcessed(jsonData):
sensorObj.updateMinOccupancy(freqRange, occupancy)
sensorObj.updateMaxOccupancy(freqRange, occupancy)
sensorObj.updateOccupancyCount(freqRange, occupancy)
LocationMessage.updateMaxBandOccupancy(lastLocationPost,
freqRange, occupancy)
LocationMessage.updateMinBandOccupancy(lastLocationPost,
freqRange, occupancy)
LocationMessage.updateOccupancySum(lastLocationPost, freqRange,
occupancy)
sensorObj.updateTime(freqRange, Message.getTime(jsonData))
sensorObj.updateDataMessageTimeStamp(Message.getTime(jsonData))
SensorDb.updateSensor(sensorObj.getJson(), False, False)
DataMessage.setMaxPower(jsonData, maxPower)
DataMessage.setMinPower(jsonData, minPower)
#if filedesc is not None:
# print json.dumps(jsonData, sort_keys=True, indent=4)
if DataMessage.isProcessed(jsonData):
dataPosts.insert(jsonData)
else:
DbCollections.getUnprocessedDataMessages(sensorId).insert(jsonData)
# Update location specific information for this sensor.
LocationMessage.addFreqRange(lastLocationPost, freqRange)
LocationMessage.setMessageTimeStampForBand(lastLocationPost, freqRange,
Message.getTime(jsonData))
LocationMessage.incrementMessageCount(lastLocationPost)
LocationMessage.incrementBandCount(lastLocationPost, freqRange)
LocationMessage.setMinMaxPower(lastLocationPost, minPower, maxPower)
locationPostId = lastLocationPost["_id"]
del lastLocationPost["_id"]
locationPosts.update({"_id": locationPostId},
{"$set": lastLocationPost},
upsert=False)
end_time = time.time()
if filedesc is not None:
print " Insertion time " + str(end_time - start_time)
def getDataSummaryForAllBands(sensorId,
locationMessage,
tmin=None,
dayCount=None):
"""
get the summary of the data corresponding to the location message.
"""
# tmin and tmax are the min and the max of the time range of interest.
locationMessageId = str(locationMessage["_id"])
tzId = locationMessage[TIME_ZONE_KEY]
sensor = SensorDb.getSensor(sensorId)
if sensor is None:
return {STATUS: NOK, ERROR_MESSAGE: "Sensor Not found in SensorDb"}
bands = sensor[THRESHOLDS]
if len(bands.keys()) == 0:
return {STATUS: NOK, ERROR_MESSAGE: "Sensor has no bands"}
measurementType = sensor[MEASUREMENT_TYPE]
bandStatistics = []
query = {SENSOR_ID: sensorId, "locationMessageId": locationMessageId}
msg = DbCollections.getDataMessages(sensorId).find_one(query)
if msg is None:
for key in bands.keys():
band = bands[key]
minFreq = band[MIN_FREQ_HZ]
maxFreq = band[MAX_FREQ_HZ]
sys2detect = band[SYSTEM_TO_DETECT]
isActive = band[ACTIVE]
bandInfo = {
"tStartDayBoundary": 0,
"tEndDayBoundary": 0,
"tStartReadings": 0,
"tStartLocalTime": 0,
"tStartLocalTimeFormattedTimeStamp": UNKNOWN,
"tEndReadings": 0,
"tEndReadingsLocalTime": 0,
"tEndLocalTimeFormattedTimeStamp": UNKNOWN,
"tEndDayBoundary": 0,
"maxOccupancy": 0,
"meanOccupancy": 0,
"minOccupancy": 0,
"maxFreq": maxFreq,
"minFreq": minFreq,
SYSTEM_TO_DETECT: sys2detect,
"measurementType": measurementType,
"active": isActive,
COUNT: 0
}
bandStatistics.append(bandInfo)
return {STATUS: "OK", "bands": bandStatistics}
if tmin is None and dayCount is None:
query = {SENSOR_ID: sensorId, "locationMessageId": locationMessageId}
tmin = msgutils.getDayBoundaryTimeStamp(msg)
mintime = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
int(tmin), tzId)
elif tmin is not None and dayCount is None:
mintime = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
int(tmin), tzId)
else:
mintime = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
int(tmin), tzId)
for key in bands.keys():
band = bands[key]
minFreq = band[MIN_FREQ_HZ]
maxFreq = band[MAX_FREQ_HZ]
sys2detect = band[SYSTEM_TO_DETECT]
bandSummary = getBandDataSummary(sensorId,
locationMessage,
sys2detect,
minFreq,
maxFreq,
mintime,
dayCount=dayCount)
bandStatistics.append(bandSummary)
return {STATUS: "OK", "bands": bandStatistics}
def getHourlyMaxMinMeanStats(sensorId, startTime, sys2detect, fmin, fmax,
subBandMinFreq, subBandMaxFreq, sessionId):
sensor = SensorDb.getSensor(sensorId)
if sensor is None:
return {STATUS: NOK, ERROR_MESSAGE: "Sensor Not Found"}
tstart = int(startTime)
fmin = int(subBandMinFreq)
fmax = int(subBandMaxFreq)
freqRange = msgutils.freqRange(sys2detect, fmin, fmax)
queryString = {
SENSOR_ID: sensorId,
TIME: {
'$gte': tstart
},
FREQ_RANGE: freqRange
}
util.debugPrint(queryString)
startMessage = DbCollections.getDataMessages(sensorId).find_one(
queryString)
if startMessage is None:
errorStr = "Start Message Not Found"
util.debugPrint(errorStr)
response = {STATUS: NOK, ERROR_MESSAGE: "No data found"}
return response
locationMessageId = DataMessage.getLocationMessageId(startMessage)
retval = {STATUS: OK}
values = {}
locationMessage = DbCollections.getLocationMessages().find_one(
{"_id": locationMessageId})
tZId = LocationMessage.getTimeZone(locationMessage)
tmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
tstart, LocationMessage.getTimeZone(locationMessage))
for hour in range(0, 23):
dataMessages = DbCollections.getDataMessages(sensorId).find({
"t": {
"$gte": tmin + hour * SECONDS_PER_HOUR
},
"t": {
"$lte": (hour + 1) * SECONDS_PER_HOUR
},
FREQ_RANGE:
freqRange
})
if dataMessages is not None:
stats = compute_stats_for_fft_power(dataMessages)
(nChannels, maxFreq, minFreq, cutoff, result) = stats
values[hour] = result
retval["values"] = values
# Now compute the next interval after the last one (if one exists)
tend = tmin + SECONDS_PER_DAY
queryString = {
SENSOR_ID: sensorId,
TIME: {
'$gte': tend
},
FREQ_RANGE: freqRange
}
msg = DbCollections.getDataMessages(sensorId).find_one(queryString)
if msg is None:
result["nextTmin"] = tmin
else:
nextTmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
msg[TIME], tZId)
result["nextTmin"] = nextTmin
# Now compute the previous interval before this one.
prevMessage = msgutils.getPrevAcquisition(startMessage)
if prevMessage is not None:
newTmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
prevMessage[TIME] - SECONDS_PER_DAY, tZId)
queryString = {
SENSOR_ID: sensorId,
TIME: {
'$gte': newTmin
},
FREQ_RANGE: msgutils.freqRange(sys2detect, fmin, fmax)
}
msg = DbCollections.getDataMessages(sensorId).find_one(queryString)
else:
msg = startMessage
result[STATUS] = OK
result["prevTmin"] = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
msg[TIME], tZId)
result["tmin"] = tmin
result["maxFreq"] = maxFreq
result["minFreq"] = minFreq
result["cutoff"] = cutoff
result[CHANNEL_COUNT] = nChannels
result["startDate"] = timezone.formatTimeStampLong(tmin, tZId)
result["values"] = values
return result