def generateOccupancyForFFTPower(msg, fileNamePrefix): chWidth = Config.getScreenConfig()[CHART_WIDTH] chHeight = Config.getScreenConfig()[CHART_HEIGHT] measurementDuration = DataMessage.getMeasurementDuration(msg) nM = DataMessage.getNumberOfMeasurements(msg) n = DataMessage.getNumberOfFrequencyBins(msg) cutoff = DataMessage.getThreshold(msg) # miliSecondsPerMeasurement = float(measurementDuration * 1000) / float(nM) spectrogramData = msgutils.getData(msg) # Generate the occupancy stats for the acquisition. occupancyCount = [0 for i in range(0, nM)] for i in range(0, nM): occupancyCount[i] = float( len(filter(lambda x: x >= cutoff, spectrogramData[i, :]))) / float(n) * 100 timeArray = [i for i in range(0, nM)] minOccupancy = np.minimum(occupancyCount) maxOccupancy = np.maximum(occupancyCount) plt.figure(figsize=(chWidth, chHeight)) plt.axes([0, measurementDuration * 1000, minOccupancy, maxOccupancy]) plt.xlim([0, measurementDuration]) plt.plot(timeArray, occupancyCount, "g.") plt.xlabel("Time (s) since start of acquisition") plt.ylabel("Band Occupancy (%)") plt.title("Band Occupancy; Cutoff: " + str(cutoff)) occupancyFilePath = util.getPath( STATIC_GENERATED_FILE_LOCATION) + fileNamePrefix + '.occupancy.png' plt.savefig(occupancyFilePath) plt.clf() plt.close() # plt.close('all') return fileNamePrefix + ".occupancy.png"
def resetThreshold(jsonData): newThreshold = _getThreshold(jsonData) if newThreshold != getThreshold(jsonData): jsonData['cutoff'] = newThreshold cutoff = newThreshold powerVal = np.array(msgutils.getData(jsonData)) n = getNumberOfFrequencyBins(jsonData) nM = getNumberOfMeasurements(jsonData) if getMeasurementType(jsonData) == FFT_POWER: occupancyCount = [0 for i in range(0, nM)] # unpack the power array. powerArray = powerVal.reshape(nM, n) for i in range(0, nM): occupancyCount[i] = float( len(filter(lambda x: x >= cutoff, powerArray[i, :]))) / float(n) setMaxOccupancy(jsonData, float(np.max(occupancyCount))) setMeanOccupancy(jsonData, float(np.mean(occupancyCount))) setMinOccupancy(jsonData, float(np.min(occupancyCount))) setMedianOccupancy(jsonData, float(np.median(occupancyCount))) else: occupancyCount = float(len(filter(lambda x: x >= cutoff, powerVal))) setOccupancy(jsonData, occupancyCount / float(len(powerVal))) return True else: return False
def generateSpectrumForFFTPower(msg, milisecOffset, sessionId): chWidth = Config.getScreenConfig()[CHART_WIDTH] chHeight = Config.getScreenConfig()[CHART_HEIGHT] startTime = msg["t"] nM = int(msg["nM"]) n = int(msg["mPar"]["n"]) measurementDuration = int(msg["mPar"]["td"]) miliSecondsPerMeasurement = float( measurementDuration * MILISECONDS_PER_SECOND) / float(nM) powerVal = np.array(msgutils.getData(msg)) spectrogramData = np.transpose(powerVal.reshape(nM, n)) col = int(milisecOffset / miliSecondsPerMeasurement) util.debugPrint("Col = " + str(col)) spectrumData = spectrogramData[:, col] maxFreq = msg["mPar"]["fStop"] minFreq = msg["mPar"]["fStart"] nSteps = len(spectrumData) freqDelta = float(maxFreq - minFreq) / float(1E6) / nSteps freqArray = [ float(minFreq) / float(1E6) + i * freqDelta for i in range(0, nSteps) ] plt.figure(figsize=(chWidth, chHeight)) plt.scatter(freqArray, spectrumData, color='red', label='Signal Power') # TODO -- fix this when the sensor is calibrated. wnI = msg[NOISE_FLOOR] noiseFloorData = [wnI for i in range(0, len(spectrumData))] plt.scatter(freqArray, noiseFloorData, color='black', label="Noise Floor") xlabel = "Freq (MHz)" ylabel = "Power (dBm)" plt.xlabel(xlabel) plt.ylabel(ylabel) locationMessage = DbCollections.getLocationMessages().find_one( {"_id": ObjectId(msg["locationMessageId"])}) t = msg["t"] + milisecOffset / float(MILISECONDS_PER_SECOND) tz = locationMessage[TIME_ZONE_KEY] title = "Spectrum at " + timezone.formatTimeStampLong(t, tz) plt.title(title) spectrumFile = sessionId + "/" + msg[SENSOR_ID] + "." + str( startTime) + "." + str(milisecOffset) + ".spectrum.png" spectrumFilePath = util.getPath( STATIC_GENERATED_FILE_LOCATION) + spectrumFile plt.savefig(spectrumFilePath, pad_inches=0, dpi=100) plt.clf() plt.close() # plt.close("all") retval = { "status": "OK", "spectrum": Config.getGeneratedDataPath() + "/" + spectrumFile, "freqArray": freqArray, "spectrumData": spectrumData.tolist(), "noiseFloorData": noiseFloorData, "title": title, "xlabel": xlabel, "ylabel": ylabel } return retval
def trimSpectrumToSubBand(msg, subBandMinFreq, subBandMaxFreq): """ Trim spectrum to a sub band of a measurement band. """ data = msgutils.getData(msg) n = msg["mPar"]["n"] minFreq = msg["mPar"]["fStart"] maxFreq = msg["mPar"]["fStop"] freqRangePerReading = float(maxFreq - minFreq) / float(n) endReadingsToIgnore = int((maxFreq - subBandMaxFreq) / freqRangePerReading) topReadingsToIgnore = int((subBandMinFreq - minFreq) / freqRangePerReading) if topReadingsToIgnore == 0 and endReadingsToIgnore == 0: # if reporting the whole array just wrap and return it. powerArray = np.array(data) else: # Otherwise, slice and return it. powerArray = np.array([ data[i] for i in range(topReadingsToIgnore, n - endReadingsToIgnore) ]) return powerArray
def generateSingleAcquisitionSpectrogramAndOccupancyForFFTPower( sensorId, sessionId, threshold, startTime, minFreq, maxFreq, leftBound, rightBound): util.debugPrint( "generateSingleAcquisitionSpectrogramAndOccupancyForFFTPower " + " sensorId = " + sensorId + " leftBound = " + str(leftBound) + " rightBound = " + str(rightBound)) dataMessages = DbCollections.getDataMessages(sensorId) chWidth = Config.getScreenConfig()[CHART_WIDTH] chHeight = Config.getScreenConfig()[CHART_HEIGHT] if dataMessages is None: return {STATUS: NOK, ERROR_MESSAGE: "Data message collection found "} msg = dataMessages.find_one({SENSOR_ID: sensorId, "t": startTime}) if msg is None: return { STATUS: NOK, ERROR_MESSAGE: "No data message found at " + str(int(startTime)) } if threshold is None: cutoff = DataMessage.getThreshold(msg) else: cutoff = int(threshold) startTime = DataMessage.getTime(msg) fs = gridfs.GridFS(DbCollections.getSpectrumDb(), msg[SENSOR_ID] + "_data") sensorId = msg[SENSOR_ID] messageBytes = fs.get(ObjectId(msg[DATA_KEY])).read() util.debugPrint("Read " + str(len(messageBytes))) spectrogramFile = sessionId + "/" + sensorId + "." + str( startTime) + "." + str(leftBound) + "." + str(rightBound) + "." + str( cutoff) spectrogramFilePath = util.getPath( STATIC_GENERATED_FILE_LOCATION) + spectrogramFile if leftBound < 0 or rightBound < 0: util.debugPrint("Bounds to exlude must be >= 0") return {STATUS: NOK, ERROR_MESSAGE: "Invalid bounds specified"} measurementDuration = DataMessage.getMeasurementDuration(msg) miliSecondsPerMeasurement = float(measurementDuration * 1000) / float( DataMessage.getNumberOfMeasurements(msg)) leftColumnsToExclude = int(leftBound / miliSecondsPerMeasurement) rightColumnsToExclude = int(rightBound / miliSecondsPerMeasurement) if leftColumnsToExclude + rightColumnsToExclude >= DataMessage.getNumberOfMeasurements( msg): util.debugPrint("leftColumnToExclude " + str(leftColumnsToExclude) + " rightColumnsToExclude " + str(rightColumnsToExclude)) return {STATUS: NOK, ERROR_MESSAGE: "Invalid bounds"} util.debugPrint("LeftColumns to exclude " + str(leftColumnsToExclude) + " right columns to exclude " + str(rightColumnsToExclude)) noiseFloor = DataMessage.getNoiseFloor(msg) nM = DataMessage.getNumberOfMeasurements( msg) - leftColumnsToExclude - rightColumnsToExclude n = DataMessage.getNumberOfFrequencyBins(msg) locationMessage = msgutils.getLocationMessage(msg) lengthToRead = n * DataMessage.getNumberOfMeasurements(msg) # Read the power values power = msgutils.getData(msg) powerVal = np.array(power[n * leftColumnsToExclude:lengthToRead - n * rightColumnsToExclude]) minTime = float( leftColumnsToExclude * miliSecondsPerMeasurement) / float(1000) spectrogramData = powerVal.reshape(nM, n) maxpower = msgutils.getMaxPower(msg) if maxpower < cutoff: maxpower = cutoff # generate the spectrogram as an image. if (not os.path.exists(spectrogramFilePath + ".png")) or\ DebugFlags.getDisableSessionIdCheckFlag(): dirname = util.getPath(STATIC_GENERATED_FILE_LOCATION) + sessionId if not os.path.exists(dirname): os.makedirs( util.getPath(STATIC_GENERATED_FILE_LOCATION) + sessionId) fig = plt.figure(figsize=(chWidth, chHeight)) # aspect ratio frame1 = plt.gca() frame1.axes.get_xaxis().set_visible(False) frame1.axes.get_yaxis().set_visible(False) cmap = plt.cm.spectral cmap.set_under(UNDER_CUTOFF_COLOR) fig = plt.imshow(np.transpose(spectrogramData), interpolation='none', origin='lower', aspect="auto", vmin=cutoff, vmax=maxpower, cmap=cmap) util.debugPrint("Generated fig " + spectrogramFilePath + ".png") plt.savefig(spectrogramFilePath + '.png', bbox_inches='tight', pad_inches=0, dpi=100) plt.clf() plt.close() else: util.debugPrint("File exists -- not regenerating") # generate the occupancy data for the measurement. occupancyCount = [0 for i in range(0, nM)] for i in range(0, nM): occupancyCount[i] = float( len(filter(lambda x: x >= cutoff, spectrogramData[i, :]))) / float(n) timeArray = [ int((i + leftColumnsToExclude) * miliSecondsPerMeasurement) for i in range(0, nM) ] # get the size of the generated png. reader = png.Reader(filename=spectrogramFilePath + ".png") (width, height, pixels, metadata) = reader.read() if (not os.path.exists(spectrogramFilePath + ".cbar.png")) or \ DebugFlags.getDisableSessionIdCheckFlag(): # generate the colorbar as a separate image. norm = mpl.colors.Normalize(vmin=cutoff, vmax=maxpower) fig = plt.figure(figsize=(chWidth * 0.2, chHeight * 1.22)) # aspect ratio ax1 = fig.add_axes([0.0, 0, 0.1, 1]) mpl.colorbar.ColorbarBase(ax1, cmap=cmap, norm=norm, orientation='vertical') plt.savefig(spectrogramFilePath + '.cbar.png', bbox_inches='tight', pad_inches=0, dpi=50) plt.clf() plt.close() nextAcquisition = msgutils.getNextAcquisition(msg) prevAcquisition = msgutils.getPrevAcquisition(msg) if nextAcquisition is not None: nextAcquisitionTime = DataMessage.getTime(nextAcquisition) else: nextAcquisitionTime = DataMessage.getTime(msg) if prevAcquisition is not None: prevAcquisitionTime = DataMessage.getTime(prevAcquisition) else: prevAcquisitionTime = DataMessage.getTime(msg) tz = locationMessage[TIME_ZONE_KEY] timeDelta = DataMessage.getMeasurementDuration( msg) - float(leftBound) / float(1000) - float(rightBound) / float(1000) meanOccupancy = np.mean(occupancyCount) maxOccupancy = np.max(occupancyCount) minOccupancy = np.min(occupancyCount) medianOccupancy = np.median(occupancyCount) result = { "spectrogram": Config.getGeneratedDataPath() + "/" + spectrogramFile + ".png", "cbar": Config.getGeneratedDataPath() + "/" + spectrogramFile + ".cbar.png", "maxPower": maxpower, "cutoff": cutoff, "noiseFloor": noiseFloor, "minPower": msgutils.getMinPower(msg), "maxFreq": DataMessage.getFmax(msg), "minFreq": DataMessage.getFmin(msg), "minTime": minTime, "timeDelta": timeDelta, "measurementsPerAcquisition": DataMessage.getNumberOfMeasurements(msg), "binsPerMeasurement": DataMessage.getNumberOfFrequencyBins(msg), "measurementCount": nM, "maxOccupancy": maxOccupancy, "minOccupancy": minOccupancy, "meanOccupancy": meanOccupancy, "medianOccupancy": medianOccupancy, "currentAcquisition": DataMessage.getTime(msg), "prevAcquisition": prevAcquisitionTime, "nextAcquisition": nextAcquisitionTime, "formattedDate": timezone.formatTimeStampLong(DataMessage.getTime(msg), tz), "image_width": float(width), "image_height": float(height) } # Now put in the occupancy data result[STATUS] = OK util.debugPrint( "generateSingleAcquisitionSpectrogramAndOccupancyForFFTPower:returning (abbreviated): " + str(result)) result["timeArray"] = timeArray result["occupancyArray"] = occupancyCount return result
def generatePowerVsTimeForSweptFrequency(sensorId, startTime, freqHz, sessionId): """ generate a power vs. time plot for swept frequency readings. The plot is generated for a period of one day. """ chWidth = Config.getScreenConfig()[CHART_WIDTH] chHeight = Config.getScreenConfig()[CHART_HEIGHT] dataMessages = DbCollections.getDataMessages(sensorId) if dataMessages is None: return { STATUS: NOK, ERROR_MESSAGE: "Data Message Collection not found" } msg = dataMessages.find_one({ SENSOR_ID: sensorId, "t": { "$gt": int(startTime) } }) (maxFreq, minFreq) = msgutils.getMaxMinFreq(msg) locationMessage = msgutils.getLocationMessage(msg) timeZone = locationMessage[TIME_ZONE_KEY] if freqHz > maxFreq: freqHz = maxFreq if freqHz < minFreq: freqHz = minFreq n = int(msg["mPar"]["n"]) freqIndex = int( float(freqHz - minFreq) / float(maxFreq - minFreq) * float(n)) powerArray = [] timeArray = [] startTime = timezone.getDayBoundaryTimeStampFromUtcTimeStamp( msg['t'], timeZone) while True: data = msgutils.getData(msg) powerArray.append(data[freqIndex]) timeArray.append(float(msg['t'] - startTime) / float(3600)) nextMsg = msgutils.getNextAcquisition(msg) if nextMsg is None: break elif nextMsg['t'] - startTime > SECONDS_PER_DAY: break else: msg = nextMsg plt.figure(figsize=(chWidth, chHeight)) plt.xlim([0, 23]) freqMHz = float(freqHz) / 1E6 title = "Power vs. Time at " + str(freqMHz) + " MHz" plt.title(title) xlabel = "Time (H) from start of day" plt.xlabel(xlabel) ylabel = "Signal Power (dBm)" plt.ylabel(ylabel) plt.xlim([0, 23]) plt.scatter(timeArray, powerArray) spectrumFile = sessionId + "/" + msg[SENSOR_ID] + "." + str( startTime) + "." + str(freqMHz) + ".power.png" spectrumFilePath = util.getPath( STATIC_GENERATED_FILE_LOCATION) + spectrumFile plt.savefig(spectrumFilePath, pad_inches=0, dpi=100) plt.clf() plt.close() retval = { STATUS: OK, "powervstime": Config.getGeneratedDataPath() + "/" + spectrumFile, "timeArray": timeArray, "powerValues": powerArray, "title": title, "xlabel": xlabel, "ylabel": ylabel } return retval
def generatePowerVsTimeForFFTPower(sensorId, startTime, leftBound, rightBound, freqHz, sessionId): """ Generate a power vs. time plot for FFTPower readings. The plot is generated for one acquistion. """ chWidth = Config.getScreenConfig()[CHART_WIDTH] chHeight = Config.getScreenConfig()[CHART_HEIGHT] msg = DbCollections.getDataMessages(sensorId).find_one({ SENSOR_ID: sensorId, "t": int(startTime) }) if msg is None: errorMessage = "Message not found" util.debugPrint(errorMessage) return {STATUS: NOK, ERROR_MESSAGE: errorMessage} n = int(msg["mPar"]["n"]) measurementDuration = msg["mPar"]["td"] miliSecondsPerMeasurement = float( measurementDuration * MILISECONDS_PER_SECOND) / float(msg["nM"]) leftColumnsToExclude = int(leftBound / miliSecondsPerMeasurement) rightColumnsToExclude = int(rightBound / miliSecondsPerMeasurement) if leftColumnsToExclude + rightColumnsToExclude >= msg["nM"]: util.debugPrint("leftColumnToExclude " + str(leftColumnsToExclude) + " rightColumnsToExclude " + str(rightColumnsToExclude)) return None nM = int(msg["nM"]) - leftColumnsToExclude - rightColumnsToExclude power = np.array(msgutils.getData(msg)) lengthToRead = int(n * msg["nM"]) powerVal = power[n * leftColumnsToExclude:lengthToRead - n * rightColumnsToExclude] spectrogramData = np.transpose(powerVal.reshape(nM, n)) maxFreq = msg["mPar"]["fStop"] minFreq = msg["mPar"]["fStart"] freqDeltaPerIndex = float(maxFreq - minFreq) / float(n) row = int((freqHz - minFreq) / freqDeltaPerIndex) util.debugPrint("row = " + str(row)) if row < 0: util.debugPrint("WARNING: row < 0") row = 0 powerValues = spectrogramData[row, :] timeArray = [ float((leftColumnsToExclude + i) * miliSecondsPerMeasurement) / float(MILISECONDS_PER_SECOND) for i in range(0, nM) ] plt.figure(figsize=(chWidth, chHeight)) plt.xlim([ float(leftBound) / float(MILISECONDS_PER_SECOND), float(measurementDuration * MILISECONDS_PER_SECOND - rightBound) / float(MILISECONDS_PER_SECOND) ]) plt.scatter(timeArray, powerValues) freqMHz = float(freqHz) / 1E6 title = "Power vs. Time at " + str(freqMHz) + " MHz" plt.title(title) spectrumFile = sessionId + "/" + msg[SENSOR_ID] + "." + str(startTime) + "." + str(leftBound) + "." + str(rightBound) + \ "." + str(freqMHz) + ".power.png" spectrumFilePath = util.getPath( STATIC_GENERATED_FILE_LOCATION) + spectrumFile xlabel = "Time (s) from start of acquistion" ylabel = "Signal Power (dBm)" plt.xlabel(xlabel) plt.ylabel(ylabel) plt.savefig(spectrumFilePath, pad_inches=0, dpi=100) plt.clf() plt.close() retval = { "powervstime": Config.getGeneratedDataPath() + "/" + spectrumFile, "powerValues": powerValues.tolist(), "timeArray": timeArray, "title": title, "xlabel": xlabel, "ylabel": ylabel } retval[STATUS] = OK return retval
def generateZipFile(sensorId, startTime, days, sys2detect, minFreq, maxFreq, dumpFileNamePrefix, sessionId): util.debugPrint("generateZipFile: " + sensorId + "/" + str(days) + "/" + str(minFreq) + "/" + str(maxFreq) + "/" + sessionId) dumpFileName = sessionId + "/" + dumpFileNamePrefix + ".txt" zipFileName = sessionId + "/" + dumpFileNamePrefix + ".zip" dirname = util.getPath(STATIC_GENERATED_FILE_LOCATION + sessionId) if not os.path.exists(dirname): os.makedirs(dirname) dumpFilePath = util.getPath(STATIC_GENERATED_FILE_LOCATION) + dumpFileName zipFilePath = util.getPath(STATIC_GENERATED_FILE_LOCATION) + zipFileName if os.path.exists(dumpFilePath): os.remove(dumpFilePath) if os.path.exists(zipFilePath): os.remove(zipFilePath) endTime = int(startTime) + int(days) * SECONDS_PER_DAY freqRange = msgutils.freqRange(sys2detect, int(minFreq), int(maxFreq)) query = {SENSOR_ID: sensorId, "t": {"$lte": int(endTime)}, "t": {"$gte": int(startTime)}, FREQ_RANGE: freqRange} firstMessage = DbCollections.getDataMessages(sensorId).find_one(query) if firstMessage is None: util.debugPrint("No data found") return locationMessage = msgutils.getLocationMessage(firstMessage) if locationMessage is None: util.debugPrint("generateZipFileForDownload: No location info found") return systemMessage = DbCollections.getSystemMessages().find_one({SENSOR_ID: sensorId}) if systemMessage is None: util.debugPrint("generateZipFileForDownload: No system info found") return dumpFile = open(dumpFilePath, "a") zipFile = zipfile.ZipFile(zipFilePath, mode="w") try: # Write out the system message. data = msgutils.getCalData(systemMessage) systemMessage[DATA_TYPE] = ASCII if CAL in systemMessage and DATA_KEY in systemMessage[CAL]: del systemMessage[CAL][DATA_KEY] del systemMessage["_id"] systemMessageString = json.dumps(systemMessage, sort_keys=False, indent=4) length = len(systemMessageString) dumpFile.write(str(length)) dumpFile.write("\n") dumpFile.write(systemMessageString) if data is not None: dataString = str(data) dumpFile.write(dataString) # Write out the location message. del locationMessage["_id"] locationMessageString = json.dumps(locationMessage, sort_keys=False, indent=4) locationMessageLength = len(locationMessageString) dumpFile.write(str(locationMessageLength)) dumpFile.write("\n") dumpFile.write(locationMessageString) # Write out the data messages one at a time c = DbCollections.getDataMessages(sensorId).find(query) for dataMessage in c: data = msgutils.getData(dataMessage) # delete fields we don't want to export del dataMessage["_id"] del dataMessage["locationMessageId"] del dataMessage[DATA_KEY] del dataMessage["cutoff"] dataMessage["Compression"] = "None" dataMessageString = json.dumps(dataMessage, sort_keys=False, indent=4) length = len(dataMessageString) dumpFile.write(str(length)) dumpFile.write("\n") dumpFile.write(dataMessageString) if dataMessage[DATA_TYPE] == ASCII: dumpFile.write(str(data)) elif dataMessage[DATA_TYPE] == BINARY_INT8: for dataByte in data: dumpFile.write(struct.pack('b', dataByte)) elif dataMessage[DATA_TYPE] == BINARY_INT16: for dataWord in data: dumpFile.write(struct.pack('i', dataWord)) elif dataMessage[DATA_TYPE] == BINARY_FLOAT32: for dataWord in data: dumpFile.write(struct.pack('f', dataWord)) zipFile.write(dumpFilePath, arcname=dumpFileNamePrefix + ".txt", compress_type=zipfile.ZIP_DEFLATED) zipFile.close() except: print "Unexpected error:", sys.exc_info()[0] print sys.exc_info() traceback.print_exc() util.logStackTrace(sys.exc_info()) finally: dumpFile.close() os.remove(dumpFilePath) zipFile.close()