def getEvents(sensorId, startTime, days): captureDb = DbCollections.getCaptureEventDb(sensorId) if startTime > 0: query = {SENSOR_ID: sensorId, "t": {"$gte": startTime}} captureEvent = captureDb.find_one(query) if captureEvent is None: return {STATUS: OK, "events": []} locationMessage = msgutils.getLocationMessage(captureEvent) if locationMessage is None: return {STATUS: NOK, "ErrorMessage": "Location message not found"} tZId = locationMessage[TIME_ZONE_KEY] timeStamp = captureEvent['t'] startTimeDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( timeStamp, tZId) endTime = startTimeDayBoundary + days * SECONDS_PER_DAY query = {"t": {"$gte": startTimeDayBoundary}, "t": {"$lte": endTime}} else: query = {} captureEvent = captureDb.find_one() if captureEvent is None: return {STATUS: OK, "events": []} locationMessage = msgutils.getLocationMessage(captureEvent) if locationMessage is None: return {STATUS: NOK, "ErrorMessage": "Location message not found"} timeStamp = captureEvent['t'] tZId = locationMessage[TIME_ZONE_KEY] startTime = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( timeStamp, tZId) if days > 0: endTime = startTime + days * SECONDS_PER_DAY query = {"t": {"$gte": startTime}, "t": {"$lte": endTime}} else: query = {"t": {"$gte": startTime}} found = captureDb.find(query) retval = [] if found is not None: for value in found: del value["_id"] retval.append(value) return {STATUS: OK, "events": retval}
def getPrevDayBoundary(msg): """ get the previous acquisition day boundary. """ prevMsg = getPrevAcquisition(msg) if prevMsg is None: locationMessage = getLocationMessage(msg) return timezone.getDayBoundaryTimeStampFromUtcTimeStamp( msg['t'], locationMessage[TIME_ZONE_KEY]) locationMessage = msgutils.getLocationMessage(prevMsg) timeZone = locationMessage[TIME_ZONE_KEY] return timezone.getDayBoundaryTimeStampFromUtcTimeStamp( prevMsg['t'], timeZone)
def insertEvent(sensorId, captureEvent): """ Insert an event in the capture database. """ locationMessage = msgutils.getLocationMessage(captureEvent) if locationMessage is None: return {STATUS: NOK, "ErrorMessage": "Location message not found"} tZId = locationMessage[TIME_ZONE_KEY] del captureEvent[SENSOR_KEY] captureDb = DbCollections.getCaptureEventDb(sensorId) captureTime = captureEvent["t"] captureEvent["formattedTimeStamp"] = timezone.formatTimeStampLong( captureTime, tZId) captureDb.ensure_index([('t', pymongo.DESCENDING)]) captureDb.insert(captureEvent) return {STATUS: OK}
def recomputeOccupanciesWorker(sensorId): # Clean out the summary stats. util.debugPrint("recomputeOccupanciesWorker " + sensorId) sensorObj = SensorDb.getSensorObj(sensorId) if sensorObj is None: return try: sensorObj.cleanSensorStats() cur = DbCollections.getDataMessages(sensorId).find() if cur is None or cur.count() == 0: return locationMessages = DbCollections.getLocationMessages().find( {SENSOR_ID: sensorId}) for locationMessage in locationMessages: lid = locationMessage["_id"] LocationMessage.clean(locationMessage) util.debugPrint("Location Message " + json.dumps(locationMessage, indent=4)) DbCollections.getLocationMessages().update( {"_id": lid}, {"$set": locationMessage}, upsert=False) for jsonData in cur: freqRange = DataMessage.getFreqRange(jsonData) # TODO -- recompute the occupancies. for data message. DataMessage.resetThreshold(jsonData) dataMsgId = jsonData["_id"] del jsonData["_id"] DbCollections.getDataMessages(sensorId).update({"_id": dataMsgId}, {"$set": jsonData}, upsert=False) minPower = DataMessage.getMinPower(jsonData) maxPower = DataMessage.getMaxPower(jsonData) lastLocationPost = msgutils.getLocationMessage(jsonData) if DataMessage.getMeasurementType(jsonData) == FFT_POWER: minOccupancy = DataMessage.getMinOccupancy(jsonData) maxOccupancy = DataMessage.getMaxOccupancy(jsonData) meanOccupancy = DataMessage.getMeanOccupancy(jsonData) sensorObj.updateMinOccupancy(freqRange, minOccupancy) sensorObj.updateMaxOccupancy(freqRange, maxOccupancy) sensorObj.updateOccupancyCount(freqRange, meanOccupancy) LocationMessage.updateMaxBandOccupancy(lastLocationPost, freqRange, maxOccupancy) LocationMessage.updateMinBandOccupancy(lastLocationPost, freqRange, minOccupancy) LocationMessage.updateOccupancySum(lastLocationPost, freqRange, meanOccupancy) else: occupancy = DataMessage.getOccupancy(jsonData) sensorObj.updateMinOccupancy(freqRange, occupancy) sensorObj.updateMaxOccupancy(freqRange, occupancy) sensorObj.updateOccupancyCount(freqRange, occupancy) LocationMessage.updateMaxBandOccupancy(lastLocationPost, freqRange, occupancy) LocationMessage.updateMinBandOccupancy(lastLocationPost, freqRange, occupancy) LocationMessage.updateOccupancySum(lastLocationPost, freqRange, occupancy) DbCollections.getLocationMessages().update( {"_id": lastLocationPost["_id"]}, {"$set": lastLocationPost}, upsert=False) except: print "Unexpected error:", sys.exc_info()[0] print sys.exc_info() traceback.print_exc() util.logStackTrace(sys.exc_info()) finally: SensorDb.setSensorStatus(sensorId, ENABLED)
def getOccupanciesByDate(sensorId, sys2detect, minFreq, maxFreq, startDate, timeOfDay, seconds, sessionId): freqRange = msgutils.freqRange(sys2detect, minFreq, maxFreq) dataMessages = DbCollections.getDataMessages(sensorId) dataMessage = dataMessages.find_one({}) if dataMessages is None: return {STATUS: "NOK", STATUS_MESSAGE: "No Data Found"} locationMessage = msgutils.getLocationMessage(dataMessage) timeString = startDate + " " + timeOfDay startTime = timezone.parseTime(timeString, locationMessage[TIME_ZONE_KEY]) endTime = startTime + seconds query = { SENSOR_ID: sensorId, FREQ_RANGE: freqRange, "$and": [{ TIME: { "$gte": startTime } }, { TIME: { "$lte": endTime } }] } # print query cur = dataMessages.find(query) if cur is None or cur.count() == 0: return {STATUS: "NOK", STATUS_MESSAGE: "No Data Found"} occupancyFileName = sessionId + "/" + sensorId + ":" + freqRange + ".occupancy." + str( startTime) + "-" + str(seconds) + ".txt" if not os.path.exists( util.getPath(STATIC_GENERATED_FILE_LOCATION) + sessionId): os.mkdir(util.getPath(STATIC_GENERATED_FILE_LOCATION) + sessionId) occupancyFileUrl = Config.getGeneratedDataPath() + "/" + occupancyFileName occupancyFilePath = util.getPath( STATIC_GENERATED_FILE_LOCATION) + occupancyFileName occupancyFile = open(occupancyFilePath, "w") timeFileName = sessionId + "/" + sensorId + ":" + freqRange + ".time." + str( startTime) + "-" + str(seconds) + ".txt" if not os.path.exists( util.getPath(STATIC_GENERATED_FILE_LOCATION) + sessionId): os.mkdir(util.getPath(STATIC_GENERATED_FILE_LOCATION) + sessionId) timeFileUrl = Config.getGeneratedDataPath() + "/" + timeFileName timeFilePath = util.getPath(STATIC_GENERATED_FILE_LOCATION) + timeFileName timeFile = open(timeFilePath, "w") tm = None timeSinceStart = 0 try: for dataMessage in cur: del dataMessage["_id"] print dumps(dataMessage, indent=4) nM = DataMessage.getNumberOfMeasurements(dataMessage) td = DataMessage.getMeasurementDuration(dataMessage) tm = DataMessage.getTimePerMeasurement(dataMessage) occupancyEndTime = dataMessage[TIME] occupancyStartTime = occupancyEndTime - nM * tm occupancyData = msgutils.getOccupancyData(dataMessage) secondsPerEntry = float(td) / float(nM) if startTime <= occupancyStartTime and endTime >= occupancyEndTime: sindex = 0 findex = nM elif startTime > occupancyStartTime and endTime < occupancyEndTime: sindex = (startTime - occupancyStartTime) / secondsPerEntry findex = nM - (occupancyEndTime - endTime) / secondsPerEntry elif startTime >= occupancyStartTime: sindex = (startTime - occupancyStartTime) / secondsPerEntry findex = nM elif endTime <= occupancyEndTime: sindex = 0 findex = nM - (occupancyEndTime - endTime) / secondsPerEntry timeSinceStart = timeSinceStart + sindex * tm print "sindex/findex", sindex, findex for i in range(sindex, findex): occupancy = str(int(occupancyData[i])) occupancyFile.write(occupancy + "\n") for i in range(sindex, findex): timeFile.write(str(timeSinceStart) + "\n") timeSinceStart = timeSinceStart + tm occupancyFile.close() timeFile.close() return { STATUS: "OK", OCCUPANCY_FILE_URL: occupancyFileUrl, TIME_FILE_URL: timeFileUrl } except: print "Unexpected error:", sys.exc_info()[0] print sys.exc_info() traceback.print_exc() util.logStackTrace(sys.exc_info()) finally: occupancyFile.close()
def generateSingleAcquisitionSpectrogramAndOccupancyForFFTPower( sensorId, sessionId, threshold, startTime, minFreq, maxFreq, leftBound, rightBound): util.debugPrint( "generateSingleAcquisitionSpectrogramAndOccupancyForFFTPower " + " sensorId = " + sensorId + " leftBound = " + str(leftBound) + " rightBound = " + str(rightBound)) dataMessages = DbCollections.getDataMessages(sensorId) chWidth = Config.getScreenConfig()[CHART_WIDTH] chHeight = Config.getScreenConfig()[CHART_HEIGHT] if dataMessages is None: return {STATUS: NOK, ERROR_MESSAGE: "Data message collection found "} msg = dataMessages.find_one({SENSOR_ID: sensorId, "t": startTime}) if msg is None: return { STATUS: NOK, ERROR_MESSAGE: "No data message found at " + str(int(startTime)) } if threshold is None: cutoff = DataMessage.getThreshold(msg) else: cutoff = int(threshold) startTime = DataMessage.getTime(msg) fs = gridfs.GridFS(DbCollections.getSpectrumDb(), msg[SENSOR_ID] + "_data") sensorId = msg[SENSOR_ID] messageBytes = fs.get(ObjectId(msg[DATA_KEY])).read() util.debugPrint("Read " + str(len(messageBytes))) spectrogramFile = sessionId + "/" + sensorId + "." + str( startTime) + "." + str(leftBound) + "." + str(rightBound) + "." + str( cutoff) spectrogramFilePath = util.getPath( STATIC_GENERATED_FILE_LOCATION) + spectrogramFile if leftBound < 0 or rightBound < 0: util.debugPrint("Bounds to exlude must be >= 0") return {STATUS: NOK, ERROR_MESSAGE: "Invalid bounds specified"} measurementDuration = DataMessage.getMeasurementDuration(msg) miliSecondsPerMeasurement = float(measurementDuration * 1000) / float( DataMessage.getNumberOfMeasurements(msg)) leftColumnsToExclude = int(leftBound / miliSecondsPerMeasurement) rightColumnsToExclude = int(rightBound / miliSecondsPerMeasurement) if leftColumnsToExclude + rightColumnsToExclude >= DataMessage.getNumberOfMeasurements( msg): util.debugPrint("leftColumnToExclude " + str(leftColumnsToExclude) + " rightColumnsToExclude " + str(rightColumnsToExclude)) return {STATUS: NOK, ERROR_MESSAGE: "Invalid bounds"} util.debugPrint("LeftColumns to exclude " + str(leftColumnsToExclude) + " right columns to exclude " + str(rightColumnsToExclude)) noiseFloor = DataMessage.getNoiseFloor(msg) nM = DataMessage.getNumberOfMeasurements( msg) - leftColumnsToExclude - rightColumnsToExclude n = DataMessage.getNumberOfFrequencyBins(msg) locationMessage = msgutils.getLocationMessage(msg) lengthToRead = n * DataMessage.getNumberOfMeasurements(msg) # Read the power values power = msgutils.getData(msg) powerVal = np.array(power[n * leftColumnsToExclude:lengthToRead - n * rightColumnsToExclude]) minTime = float( leftColumnsToExclude * miliSecondsPerMeasurement) / float(1000) spectrogramData = powerVal.reshape(nM, n) maxpower = msgutils.getMaxPower(msg) if maxpower < cutoff: maxpower = cutoff # generate the spectrogram as an image. if (not os.path.exists(spectrogramFilePath + ".png")) or\ DebugFlags.getDisableSessionIdCheckFlag(): dirname = util.getPath(STATIC_GENERATED_FILE_LOCATION) + sessionId if not os.path.exists(dirname): os.makedirs( util.getPath(STATIC_GENERATED_FILE_LOCATION) + sessionId) fig = plt.figure(figsize=(chWidth, chHeight)) # aspect ratio frame1 = plt.gca() frame1.axes.get_xaxis().set_visible(False) frame1.axes.get_yaxis().set_visible(False) cmap = plt.cm.spectral cmap.set_under(UNDER_CUTOFF_COLOR) fig = plt.imshow(np.transpose(spectrogramData), interpolation='none', origin='lower', aspect="auto", vmin=cutoff, vmax=maxpower, cmap=cmap) util.debugPrint("Generated fig " + spectrogramFilePath + ".png") plt.savefig(spectrogramFilePath + '.png', bbox_inches='tight', pad_inches=0, dpi=100) plt.clf() plt.close() else: util.debugPrint("File exists -- not regenerating") # generate the occupancy data for the measurement. occupancyCount = [0 for i in range(0, nM)] for i in range(0, nM): occupancyCount[i] = float( len(filter(lambda x: x >= cutoff, spectrogramData[i, :]))) / float(n) timeArray = [ int((i + leftColumnsToExclude) * miliSecondsPerMeasurement) for i in range(0, nM) ] # get the size of the generated png. reader = png.Reader(filename=spectrogramFilePath + ".png") (width, height, pixels, metadata) = reader.read() if (not os.path.exists(spectrogramFilePath + ".cbar.png")) or \ DebugFlags.getDisableSessionIdCheckFlag(): # generate the colorbar as a separate image. norm = mpl.colors.Normalize(vmin=cutoff, vmax=maxpower) fig = plt.figure(figsize=(chWidth * 0.2, chHeight * 1.22)) # aspect ratio ax1 = fig.add_axes([0.0, 0, 0.1, 1]) mpl.colorbar.ColorbarBase(ax1, cmap=cmap, norm=norm, orientation='vertical') plt.savefig(spectrogramFilePath + '.cbar.png', bbox_inches='tight', pad_inches=0, dpi=50) plt.clf() plt.close() nextAcquisition = msgutils.getNextAcquisition(msg) prevAcquisition = msgutils.getPrevAcquisition(msg) if nextAcquisition is not None: nextAcquisitionTime = DataMessage.getTime(nextAcquisition) else: nextAcquisitionTime = DataMessage.getTime(msg) if prevAcquisition is not None: prevAcquisitionTime = DataMessage.getTime(prevAcquisition) else: prevAcquisitionTime = DataMessage.getTime(msg) tz = locationMessage[TIME_ZONE_KEY] timeDelta = DataMessage.getMeasurementDuration( msg) - float(leftBound) / float(1000) - float(rightBound) / float(1000) meanOccupancy = np.mean(occupancyCount) maxOccupancy = np.max(occupancyCount) minOccupancy = np.min(occupancyCount) medianOccupancy = np.median(occupancyCount) result = { "spectrogram": Config.getGeneratedDataPath() + "/" + spectrogramFile + ".png", "cbar": Config.getGeneratedDataPath() + "/" + spectrogramFile + ".cbar.png", "maxPower": maxpower, "cutoff": cutoff, "noiseFloor": noiseFloor, "minPower": msgutils.getMinPower(msg), "maxFreq": DataMessage.getFmax(msg), "minFreq": DataMessage.getFmin(msg), "minTime": minTime, "timeDelta": timeDelta, "measurementsPerAcquisition": DataMessage.getNumberOfMeasurements(msg), "binsPerMeasurement": DataMessage.getNumberOfFrequencyBins(msg), "measurementCount": nM, "maxOccupancy": maxOccupancy, "minOccupancy": minOccupancy, "meanOccupancy": meanOccupancy, "medianOccupancy": medianOccupancy, "currentAcquisition": DataMessage.getTime(msg), "prevAcquisition": prevAcquisitionTime, "nextAcquisition": nextAcquisitionTime, "formattedDate": timezone.formatTimeStampLong(DataMessage.getTime(msg), tz), "image_width": float(width), "image_height": float(height) } # Now put in the occupancy data result[STATUS] = OK util.debugPrint( "generateSingleAcquisitionSpectrogramAndOccupancyForFFTPower:returning (abbreviated): " + str(result)) result["timeArray"] = timeArray result["occupancyArray"] = occupancyCount return result
def generatePowerVsTimeForSweptFrequency(sensorId, startTime, freqHz, sessionId): """ generate a power vs. time plot for swept frequency readings. The plot is generated for a period of one day. """ chWidth = Config.getScreenConfig()[CHART_WIDTH] chHeight = Config.getScreenConfig()[CHART_HEIGHT] dataMessages = DbCollections.getDataMessages(sensorId) if dataMessages is None: return { STATUS: NOK, ERROR_MESSAGE: "Data Message Collection not found" } msg = dataMessages.find_one({ SENSOR_ID: sensorId, "t": { "$gt": int(startTime) } }) (maxFreq, minFreq) = msgutils.getMaxMinFreq(msg) locationMessage = msgutils.getLocationMessage(msg) timeZone = locationMessage[TIME_ZONE_KEY] if freqHz > maxFreq: freqHz = maxFreq if freqHz < minFreq: freqHz = minFreq n = int(msg["mPar"]["n"]) freqIndex = int( float(freqHz - minFreq) / float(maxFreq - minFreq) * float(n)) powerArray = [] timeArray = [] startTime = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( msg['t'], timeZone) while True: data = msgutils.getData(msg) powerArray.append(data[freqIndex]) timeArray.append(float(msg['t'] - startTime) / float(3600)) nextMsg = msgutils.getNextAcquisition(msg) if nextMsg is None: break elif nextMsg['t'] - startTime > SECONDS_PER_DAY: break else: msg = nextMsg plt.figure(figsize=(chWidth, chHeight)) plt.xlim([0, 23]) freqMHz = float(freqHz) / 1E6 title = "Power vs. Time at " + str(freqMHz) + " MHz" plt.title(title) xlabel = "Time (H) from start of day" plt.xlabel(xlabel) ylabel = "Signal Power (dBm)" plt.ylabel(ylabel) plt.xlim([0, 23]) plt.scatter(timeArray, powerArray) spectrumFile = sessionId + "/" + msg[SENSOR_ID] + "." + str( startTime) + "." + str(freqMHz) + ".power.png" spectrumFilePath = util.getPath( STATIC_GENERATED_FILE_LOCATION) + spectrumFile plt.savefig(spectrumFilePath, pad_inches=0, dpi=100) plt.clf() plt.close() retval = { STATUS: OK, "powervstime": Config.getGeneratedDataPath() + "/" + spectrumFile, "timeArray": timeArray, "powerValues": powerArray, "title": title, "xlabel": xlabel, "ylabel": ylabel } return retval
def generateZipFile(sensorId, startTime, days, sys2detect, minFreq, maxFreq, dumpFileNamePrefix, sessionId): util.debugPrint("generateZipFile: " + sensorId + "/" + str(days) + "/" + str(minFreq) + "/" + str(maxFreq) + "/" + sessionId) dumpFileName = sessionId + "/" + dumpFileNamePrefix + ".txt" zipFileName = sessionId + "/" + dumpFileNamePrefix + ".zip" dirname = util.getPath(STATIC_GENERATED_FILE_LOCATION + sessionId) if not os.path.exists(dirname): os.makedirs(dirname) dumpFilePath = util.getPath(STATIC_GENERATED_FILE_LOCATION) + dumpFileName zipFilePath = util.getPath(STATIC_GENERATED_FILE_LOCATION) + zipFileName if os.path.exists(dumpFilePath): os.remove(dumpFilePath) if os.path.exists(zipFilePath): os.remove(zipFilePath) endTime = int(startTime) + int(days) * SECONDS_PER_DAY freqRange = msgutils.freqRange(sys2detect, int(minFreq), int(maxFreq)) query = {SENSOR_ID: sensorId, "t": {"$lte": int(endTime)}, "t": {"$gte": int(startTime)}, FREQ_RANGE: freqRange} firstMessage = DbCollections.getDataMessages(sensorId).find_one(query) if firstMessage is None: util.debugPrint("No data found") return locationMessage = msgutils.getLocationMessage(firstMessage) if locationMessage is None: util.debugPrint("generateZipFileForDownload: No location info found") return systemMessage = DbCollections.getSystemMessages().find_one({SENSOR_ID: sensorId}) if systemMessage is None: util.debugPrint("generateZipFileForDownload: No system info found") return dumpFile = open(dumpFilePath, "a") zipFile = zipfile.ZipFile(zipFilePath, mode="w") try: # Write out the system message. data = msgutils.getCalData(systemMessage) systemMessage[DATA_TYPE] = ASCII if CAL in systemMessage and DATA_KEY in systemMessage[CAL]: del systemMessage[CAL][DATA_KEY] del systemMessage["_id"] systemMessageString = json.dumps(systemMessage, sort_keys=False, indent=4) length = len(systemMessageString) dumpFile.write(str(length)) dumpFile.write("\n") dumpFile.write(systemMessageString) if data is not None: dataString = str(data) dumpFile.write(dataString) # Write out the location message. del locationMessage["_id"] locationMessageString = json.dumps(locationMessage, sort_keys=False, indent=4) locationMessageLength = len(locationMessageString) dumpFile.write(str(locationMessageLength)) dumpFile.write("\n") dumpFile.write(locationMessageString) # Write out the data messages one at a time c = DbCollections.getDataMessages(sensorId).find(query) for dataMessage in c: data = msgutils.getData(dataMessage) # delete fields we don't want to export del dataMessage["_id"] del dataMessage["locationMessageId"] del dataMessage[DATA_KEY] del dataMessage["cutoff"] dataMessage["Compression"] = "None" dataMessageString = json.dumps(dataMessage, sort_keys=False, indent=4) length = len(dataMessageString) dumpFile.write(str(length)) dumpFile.write("\n") dumpFile.write(dataMessageString) if dataMessage[DATA_TYPE] == ASCII: dumpFile.write(str(data)) elif dataMessage[DATA_TYPE] == BINARY_INT8: for dataByte in data: dumpFile.write(struct.pack('b', dataByte)) elif dataMessage[DATA_TYPE] == BINARY_INT16: for dataWord in data: dumpFile.write(struct.pack('i', dataWord)) elif dataMessage[DATA_TYPE] == BINARY_FLOAT32: for dataWord in data: dumpFile.write(struct.pack('f', dataWord)) zipFile.write(dumpFilePath, arcname=dumpFileNamePrefix + ".txt", compress_type=zipfile.ZIP_DEFLATED) zipFile.close() except: print "Unexpected error:", sys.exc_info()[0] print sys.exc_info() traceback.print_exc() util.logStackTrace(sys.exc_info()) finally: dumpFile.close() os.remove(dumpFilePath) zipFile.close()