def getSensorDataSummary(sensorId, locationMessage):
sensor = SensorDb.getSensorObj(sensorId)
if sensor is None:
return {STATUS: NOK, ERROR_MESSAGE: "Sensor Not found"}
measurementType = sensor.getMeasurementType()
tzId = locationMessage[TIME_ZONE_KEY]
acquisitionCount = LocationMessage.getMessageCount(locationMessage)
util.debugPrint("AquistionCount " + str(acquisitionCount))
if acquisitionCount == 0:
return {
"status": "OK",
"minOccupancy": 0,
"tStartReadings": 0,
"tStartLocalTime": 0,
"tStartLocalTimeFormattedTimeStamp": "UNKNOWN",
"tStartDayBoundary": 0,
"tEndDayBoundary": 0,
"tEndReadings": 0,
"tEndLocalTimeFormattedTimeStamp": "UNKNOWN",
"maxOccupancy": 0,
"measurementType": measurementType,
"isStreamingEnabled": sensor.isStreamingEnabled(),
"sensorStatus": sensor.getSensorStatus(),
COUNT: 0
}
minTime = LocationMessage.getFirstDataMessageTimeStamp(locationMessage)
maxTime = LocationMessage.getLastDataMessageTimeStamp(locationMessage)
tStartDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
minTime, tzId)
(minLocalTime,
tStartLocalTimeTzName) = timezone.getLocalTime(minTime, tzId)
tEndDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
maxTime, tzId)
tstampMin = timezone.formatTimeStampLong(minTime, tzId)
tstampMax = timezone.formatTimeStampLong(maxTime, tzId)
retval = {
"status": "OK",
"maxOccupancy": 0,
"minOccupancy": 0,
"tStartReadings": minTime,
"tStartLocalTime": minLocalTime,
"tStartLocalTimeFormattedTimeStamp": tstampMin,
"tStartDayBoundary": tStartDayBoundary,
"tEndDayBoundary": tEndDayBoundary,
"tEndReadings": maxTime,
"tEndLocalTimeFormattedTimeStamp": tstampMax,
"measurementType": measurementType,
"isStreamingEnabled": sensor.isStreamingEnabled(),
"sensorStatus": sensor.getSensorStatus(),
COUNT: acquisitionCount
}
return retval
def generateSpectrumForSweptFrequency(msg, sessionId, minFreq, maxFreq):
try:
chWidth = Config.getScreenConfig()[CHART_WIDTH]
chHeight = Config.getScreenConfig()[CHART_HEIGHT]
spectrumData = msgutils.trimSpectrumToSubBand(msg, minFreq, maxFreq)
noiseFloorData = msgutils.trimNoiseFloorToSubBand(
msg, minFreq, maxFreq)
nSteps = len(spectrumData)
freqDelta = float(maxFreq - minFreq) / float(1E6) / nSteps
freqArray = [
float(minFreq) / float(1E6) + i * freqDelta
for i in range(0, nSteps)
]
plt.figure(figsize=(chWidth, chHeight))
plt1 = plt.scatter(freqArray,
spectrumData,
color='red',
label="Signal Power")
plt2 = plt.scatter(freqArray,
noiseFloorData,
color='black',
label="Noise Floor")
plt.legend(handles=[plt1, plt2])
xlabel = "Freq (MHz)"
plt.xlabel(xlabel)
ylabel = "Power (dBm)"
plt.ylabel(ylabel)
locationMessage = DbCollections.getLocationMessages().find_one(
{"_id": ObjectId(msg["locationMessageId"])})
t = msg["t"]
tz = locationMessage[TIME_ZONE_KEY]
title = "Spectrum at " + timezone.formatTimeStampLong(t, tz)
plt.title(title)
spectrumFile = sessionId + "/" + msg[SENSOR_ID] + "." + str(
msg['t']) + "." + str(minFreq) + "." + str(
maxFreq) + ".spectrum.png"
spectrumFilePath = util.getPath(
STATIC_GENERATED_FILE_LOCATION) + spectrumFile
plt.savefig(spectrumFilePath, pad_inches=0, dpi=100)
plt.clf()
plt.close()
# plt.close("all")
urlPrefix = Config.getGeneratedDataPath()
retval = {
"status": "OK",
"spectrum": urlPrefix + "/" + spectrumFile,
"freqArray": freqArray,
"spectrumData": spectrumData.tolist(),
"noiseFloorData": noiseFloorData.tolist(),
"title": title,
"xlabel": xlabel,
"ylabel": ylabel
}
return retval
except:
print "Unexpected error:", sys.exc_info()[0]
print sys.exc_info()
traceback.print_exc()
raise
def generateSpectrumForFFTPower(msg, milisecOffset, sessionId):
chWidth = Config.getScreenConfig()[CHART_WIDTH]
chHeight = Config.getScreenConfig()[CHART_HEIGHT]
startTime = msg["t"]
nM = int(msg["nM"])
n = int(msg["mPar"]["n"])
measurementDuration = int(msg["mPar"]["td"])
miliSecondsPerMeasurement = float(
measurementDuration * MILISECONDS_PER_SECOND) / float(nM)
powerVal = np.array(msgutils.getData(msg))
spectrogramData = np.transpose(powerVal.reshape(nM, n))
col = int(milisecOffset / miliSecondsPerMeasurement)
util.debugPrint("Col = " + str(col))
spectrumData = spectrogramData[:, col]
maxFreq = msg["mPar"]["fStop"]
minFreq = msg["mPar"]["fStart"]
nSteps = len(spectrumData)
freqDelta = float(maxFreq - minFreq) / float(1E6) / nSteps
freqArray = [
float(minFreq) / float(1E6) + i * freqDelta for i in range(0, nSteps)
]
plt.figure(figsize=(chWidth, chHeight))
plt.scatter(freqArray, spectrumData, color='red', label='Signal Power')
# TODO -- fix this when the sensor is calibrated.
wnI = msg[NOISE_FLOOR]
noiseFloorData = [wnI for i in range(0, len(spectrumData))]
plt.scatter(freqArray, noiseFloorData, color='black', label="Noise Floor")
xlabel = "Freq (MHz)"
ylabel = "Power (dBm)"
plt.xlabel(xlabel)
plt.ylabel(ylabel)
locationMessage = DbCollections.getLocationMessages().find_one(
{"_id": ObjectId(msg["locationMessageId"])})
t = msg["t"] + milisecOffset / float(MILISECONDS_PER_SECOND)
tz = locationMessage[TIME_ZONE_KEY]
title = "Spectrum at " + timezone.formatTimeStampLong(t, tz)
plt.title(title)
spectrumFile = sessionId + "/" + msg[SENSOR_ID] + "." + str(
startTime) + "." + str(milisecOffset) + ".spectrum.png"
spectrumFilePath = util.getPath(
STATIC_GENERATED_FILE_LOCATION) + spectrumFile
plt.savefig(spectrumFilePath, pad_inches=0, dpi=100)
plt.clf()
plt.close()
# plt.close("all")
retval = {
"status": "OK",
"spectrum": Config.getGeneratedDataPath() + "/" + spectrumFile,
"freqArray": freqArray,
"spectrumData": spectrumData.tolist(),
"noiseFloorData": noiseFloorData,
"title": title,
"xlabel": xlabel,
"ylabel": ylabel
}
return retval
def __init__(self, num_ch, file_name, sensorLoc, sensorSys, sensor_id, sensor_key, center_freq, bandwidth, meas_duration, atten, samp_rate, avoid_LO):
gr.sync_block.__init__(self,
name="jsonfile_sink",
in_sig=[(bytes, num_ch)],
out_sig=None)
# Establish ssl socket connection to server
self.num_ch = num_ch
# Open file for output stream
self.file_name = file_name
self.f = open(self.file_name, 'wb')
self.srvr = blocks.file_descriptor_sink(self.num_ch * gr.sizeof_char, self.f.fileno())
self.sensorLoc = sensorLoc
self.sensorSys = sensorSys
self.sensor_id = sensor_id
self.sensor_key = sensor_key
self.center_freq = center_freq
self.bandwidth = bandwidth
self.meas_duration = meas_duration
self.atten = atten
self.samp_rate = samp_rate
self.avoid = avoid_LO
print ("Sensor ID: ", self.sensor_id)
# Send location and system info to server
loc_msg = self.read_json_from_file(self.sensorLoc)
sys_msg = self.read_json_from_file(self.sensorSys)
ts = long(round(getLocalUtcTimeStamp()))
print 'Serial no.', self.sensor_id
loc_msg['t'] = ts
loc_msg['SensorID'] = self.sensor_id
sys_msg['t'] = ts
sys_msg['SensorID'] = self.sensor_id
self.post_msg(loc_msg)
self.post_msg(sys_msg)
# Form data header
ts = long(round(getLocalUtcTimeStamp()))
f_start = self.center_freq - self.bandwidth / 2.0
if self.avoid:
#self.center_freq = self.center_freq + self.samp_rate/4.0
#print "Avoiding LO...\nShifting center Frequency to", self.center_freq
f_start = (self.center_freq - self.samp_rate/4) - self.bandwidth/2.0
f_stop = f_start + self.bandwidth
if self.avoid:
print "Avoiding LO, frequencies are shifted to: [",f_start/1e6, "MHz-",f_stop/1e6,"MHz ]"
mpar = Struct(fStart=f_start, fStop=f_stop, n=self.num_ch, td=-1, tm=self.meas_duration, Det='Average', Atten=self.atten)
# Need to add a field for overflow indicator
data_hdr = Struct(Ver='1.0.12', Type='Data', SensorID=self.sensor_id, SensorKey=self.sensor_key, t=ts, Sys2Detect='LTE', Sensitivity='Low', mType='FFT-Power', t1=ts, a=1, nM=-1, Ta=-1, OL='NaN', wnI=-77.0, Comment='Using hard-coded (not detected) system noise power for wnI', Processed='False', DataType = 'Binary - int8', ByteOrder='N/A', Compression='None', mPar=mpar)
self.post_msg(data_hdr)
self.f.flush()
date_str = formatTimeStampLong(ts, loc_msg['TimeZone'])
print date_str, "fc =", self.center_freq/1e6, "MHz. Writing data to file", self.file_name
def main_loop(tb):
if not tb.set_freq(tb.center_freq):
print "Failed to set frequency to", tb.center_freq
sys.exit(1)
print "Set frequency to", tb.center_freq/1e6, "MHz\n"
time.sleep(0.25)
sensor_id = "HackRF"
# Establish ssl socket connection to server
r = requests.post('https://'+tb.dest_host+':8443/sensordata/getStreamingPort/'+sensor_id, verify=False)
print 'server response:', r.text
response = r.json()
print 'socket port =', response['port']
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tb.s = s = ssl.wrap_socket(sock)#, ca_certs='dummy.crt', cert_reqs=ssl.CERT_NONE)
s.connect((tb.dest_host, response['port']))
tb.set_sock(s)
# Send location and system info to server
loc_msg = tb.read_json_from_file('sensor.loc')
sys_msg = tb.read_json_from_file('sensor.sys')
ts = long(round(getLocalUtcTimeStamp()))
sensor_key = sys_msg['SensorKey']
print 'Sensor Key', sensor_key
#print 'Serial no.', sensor_id
loc_msg['t'] = ts
loc_msg['SensorID'] = sensor_id
sys_msg['t'] = ts
sys_msg['SensorID'] = sensor_id
tb.send_obj(loc_msg)
tb.send_obj(sys_msg)
# Form data header
ts = long(round(getLocalUtcTimeStamp()))
f_start = tb.center_freq - tb.bandwidth/2.0
if tb.avoid_LO:
f_start = (tb.center_freq - tb.samp_rate/4) - tb.bandwidth/2.0
f_stop = f_start + tb.bandwidth
if tb.avoid_LO:
print "Avoiding LO, frequencies are shifted to: [",f_start/1e6, "MHz-",f_stop/1e6,"MHz ]"
mpar = Struct(fStart=f_start, fStop=f_stop, n=tb.num_ch, td=-1, tm=tb.meas_duration, Det='Average', Atten=tb.atten)
# Need to add a field for overflow indicator
data = Struct(Ver='1.0.12', Type='Data', SensorID='HackRF', SensorKey='12345', t=ts, Sys2Detect='LTE', Sensitivity='Low', mType='FFT-Power', t1=ts, a=1, nM=-1, Ta=-1, OL='NaN', wnI=-77.0, Comment='Using hard-coded (not detected) system noise power for wnI', Processed='False', DataType = 'Binary - int8', ByteOrder='N/A', Compression='None', mPar=mpar)
tb.send_obj(data)
date_str = formatTimeStampLong(ts, loc_msg['TimeZone'])
print date_str, "fc =", tb.center_freq/1e6, "MHz. Sending data to", tb.dest_host
# Start flow graph
tb.start()
tb.wait()
tb.s.close()
print 'Closed socket'
def main_loop(tb):
sensor_id = "TestSensor"
r = requests.post('https://' + tb.dest_host +
'/sensordata/getStreamingPort/' + sensor_id,
verify=False)
print 'server response:', r.text
response = r.json()
print 'socket port =', response['port']
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tb.s = s = ssl.wrap_socket(sock)
s.connect((tb.dest_host, response['port']))
tb.set_sock(s)
# Send location and system info to server
loc_msg = tb.read_json_from_file('sensor.loc')
loc_msg["SensorID"] = sensor_id
sys_msg = tb.read_json_from_file('sensor.sys')
sys_msg["SensorID"] = sensor_id
ts = long(round(getLocalUtcTimeStamp()))
print 'Serial no.', sensor_id
loc_msg['t'] = ts
loc_msg['SensorID'] = sensor_id
sys_msg['t'] = ts
sys_msg['SensorID'] = sensor_id
tb.send_obj(loc_msg)
tb.send_obj(sys_msg)
# Form data header
ts = long(round(getLocalUtcTimeStamp()))
f_start = tb.center_freq - tb.bandwidth / 2.0
f_stop = f_start + tb.bandwidth
# Note -- dummy atten
mpar = Struct(fStart=f_start,
fStop=f_stop,
n=tb.num_ch,
td=-1,
tm=tb.meas_duration,
Det='Average' if tb.det_type == 'avg' else 'Peak',
Atten=tb.atten)
# Need to add a field for overflow indicator
data = Struct(Ver='1.0.12', Type='Data', SensorID=sensor_id, SensorKey='NaN', t=ts, Sys2Detect='LTE', \
Sensitivity='Low', mType='FFT-Power', t1=ts, a=1, nM=-1, Ta=-1, OL='NaN', wnI=-77.0, \
Comment='Using hard-coded (not detected) system noise power for wnI', \
Processed='False', DataType = 'Binary - int8', ByteOrder='N/A', Compression='None', mPar=mpar)
tb.send_obj(data)
date_str = formatTimeStampLong(ts, loc_msg['TimeZone'])
print date_str, "fc =", tb.center_freq / 1e6, "MHz. Sending data to", tb.dest_host
# Start flow graph
tb.start()
tb.wait()
tb.s.close()
print 'Closed socket'
def __init__(self, num_ch, dest_host, sensorLoc, sensorSys, sensor_id, sensor_key, center_freq, bandwidth, meas_duration, atten):
gr.sync_block.__init__(self,
name="sslsocket_sink",
in_sig=[(bytes, num_ch)],
out_sig=None)
# Establish ssl socket connection to server
self.num_ch = num_ch
self.dest_host = dest_host
self.sensorLoc = sensorLoc
self.sensorSys = sensorSys
self.sensor_id = sensor_id
self.sensor_key = sensor_key
self.center_freq = center_freq
self.bandwidth = bandwidth
self.meas_duration = meas_duration
self.atten = atten
r = requests.post('https://'+self.dest_host+':8443/sensordata/getStreamingPort/'+self.sensor_id, verify=False)
print 'server response:', r.text
response = r.json()
print 'socket port =', response['port']
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock = ssl.wrap_socket(sock, cert_reqs=ssl.CERT_NONE)
self.sock.connect((self.dest_host, response['port']))
print ("Sensor ID: ", self.sensor_id)
# Send location and system info to server
loc_msg = self.read_json_from_file(self.sensorLoc)
sys_msg = self.read_json_from_file(self.sensorSys)
ts = long(round(getLocalUtcTimeStamp()))
print 'Serial no.', self.sensor_id
loc_msg['t'] = ts
loc_msg['SensorID'] = self.sensor_id
sys_msg['t'] = ts
sys_msg['SensorID'] = self.sensor_id
self.send_obj(loc_msg)
self.send_obj(sys_msg)
# Form data header
ts = long(round(getLocalUtcTimeStamp()))
f_start = self.center_freq - self.bandwidth / 2.0
f_stop = f_start + self.bandwidth
mpar = Struct(fStart=f_start, fStop=f_stop, n=self.num_ch, td=-1, tm=self.meas_duration, Det='Average', Atten=self.atten)
# Need to add a field for overflow indicator
data = Struct(Ver='1.0.12', Type='Data', SensorID=self.sensor_id, SensorKey=self.sensor_key, t=ts, Sys2Detect='LTE', Sensitivity='Low', mType='FFT-Power', t1=ts, a=1, nM=-1, Ta=-1, OL='NaN', wnI=-77.0, Comment='Using hard-coded (not detected) system noise power for wnI', Processed='False', DataType = 'Binary - int8', ByteOrder='N/A', Compression='None', mPar=mpar)
self.send_obj(data)
date_str = formatTimeStampLong(ts, loc_msg['TimeZone'])
print date_str, "fc =", self.center_freq/1e6, "MHz. Sending data to", self.dest_host
def insertEvent(sensorId, captureEvent):
"""
Insert an event in the capture database.
"""
locationMessage = msgutils.getLocationMessage(captureEvent)
if locationMessage is None:
return {STATUS: NOK, "ErrorMessage": "Location message not found"}
tZId = locationMessage[TIME_ZONE_KEY]
del captureEvent[SENSOR_KEY]
captureDb = DbCollections.getCaptureEventDb(sensorId)
captureTime = captureEvent["t"]
captureEvent["formattedTimeStamp"] = timezone.formatTimeStampLong(
captureTime, tZId)
captureDb.ensure_index([('t', pymongo.DESCENDING)])
captureDb.insert(captureEvent)
return {STATUS: OK}
def main_loop(tb):
if not tb.set_freq(tb.center_freq):
print "Failed to set frequency to", tb.center_freq
sys.exit(1)
print "Set frequency to", tb.center_freq/1e6, "MHz"
time.sleep(0.25)
# Establish connection to server
r = requests.post('http://'+tb.dest_host+':8000/sensordata/getStreamingPort')
response = r.json()
print 'socket port =', response['port']
tb.s = s = jsocket.JsonClient(address=tb.dest_host, port=response['port'])
s.connect()
tb.set_fd(s.conn.fileno())
# Send location and system info to server
loc_msg = tb.read_json_from_file('sensor.loc')
sys_msg = tb.read_json_from_file('sensor.sys')
ts = long(round(getLocalUtcTimeStamp()))
sensor_id = tb.u.get_usrp_info()['rx_serial']
print 'Serial no.', sensor_id
loc_msg['t'] = ts
loc_msg['SensorID'] = sensor_id
sys_msg['t'] = ts
sys_msg['SensorID'] = sensor_id
s.send_obj(loc_msg)
s.send_obj(sys_msg)
# Form data header
ts = long(round(getLocalUtcTimeStamp()))
f_start = tb.center_freq - tb.bandwidth / 2.0
f_stop = f_start + tb.bandwidth
mpar = Struct(fStart=f_start, fStop=f_stop, n=tb.num_ch, td=-1, tm=tb.meas_duration, Det='Average', Atten=tb.atten)
# Need to add a field for overflow indicator
data = Struct(Ver='1.0.12', Type='Data', SensorID=sensor_id, SensorKey='NaN', t=ts, Sys2Detect='LTE', Sensitivity='Low', mType='FFT-Power', t1=ts, a=1, nM=-1, Ta=-1, OL='NaN', wnI=-77.0, Comment='Using hard-coded (not detected) system noise power for wnI', Processed='False', DataType = 'Binary - int8', ByteOrder='N/A', Compression='None', mPar=mpar)
s.send_obj(data)
date_str = formatTimeStampLong(ts, loc_msg['TimeZone'])
print date_str, "fc =", tb.center_freq/1e6, "MHz. Sending data to", tb.dest_host
# Execute flow graph and wait for it to stop
tb.start()
tb.wait()
tb.s.close()
print 'Closed socket'
def generateSingleAcquisitionSpectrogramAndOccupancyForFFTPower(
sensorId, sessionId, threshold, startTime, minFreq, maxFreq, leftBound,
rightBound):
util.debugPrint(
"generateSingleAcquisitionSpectrogramAndOccupancyForFFTPower " +
" sensorId = " + sensorId + " leftBound = " + str(leftBound) +
" rightBound = " + str(rightBound))
dataMessages = DbCollections.getDataMessages(sensorId)
chWidth = Config.getScreenConfig()[CHART_WIDTH]
chHeight = Config.getScreenConfig()[CHART_HEIGHT]
if dataMessages is None:
return {STATUS: NOK, ERROR_MESSAGE: "Data message collection found "}
msg = dataMessages.find_one({SENSOR_ID: sensorId, "t": startTime})
if msg is None:
return {
STATUS: NOK,
ERROR_MESSAGE: "No data message found at " + str(int(startTime))
}
if threshold is None:
cutoff = DataMessage.getThreshold(msg)
else:
cutoff = int(threshold)
startTime = DataMessage.getTime(msg)
fs = gridfs.GridFS(DbCollections.getSpectrumDb(), msg[SENSOR_ID] + "_data")
sensorId = msg[SENSOR_ID]
messageBytes = fs.get(ObjectId(msg[DATA_KEY])).read()
util.debugPrint("Read " + str(len(messageBytes)))
spectrogramFile = sessionId + "/" + sensorId + "." + str(
startTime) + "." + str(leftBound) + "." + str(rightBound) + "." + str(
cutoff)
spectrogramFilePath = util.getPath(
STATIC_GENERATED_FILE_LOCATION) + spectrogramFile
if leftBound < 0 or rightBound < 0:
util.debugPrint("Bounds to exlude must be >= 0")
return {STATUS: NOK, ERROR_MESSAGE: "Invalid bounds specified"}
measurementDuration = DataMessage.getMeasurementDuration(msg)
miliSecondsPerMeasurement = float(measurementDuration * 1000) / float(
DataMessage.getNumberOfMeasurements(msg))
leftColumnsToExclude = int(leftBound / miliSecondsPerMeasurement)
rightColumnsToExclude = int(rightBound / miliSecondsPerMeasurement)
if leftColumnsToExclude + rightColumnsToExclude >= DataMessage.getNumberOfMeasurements(
msg):
util.debugPrint("leftColumnToExclude " + str(leftColumnsToExclude) +
" rightColumnsToExclude " + str(rightColumnsToExclude))
return {STATUS: NOK, ERROR_MESSAGE: "Invalid bounds"}
util.debugPrint("LeftColumns to exclude " + str(leftColumnsToExclude) +
" right columns to exclude " + str(rightColumnsToExclude))
noiseFloor = DataMessage.getNoiseFloor(msg)
nM = DataMessage.getNumberOfMeasurements(
msg) - leftColumnsToExclude - rightColumnsToExclude
n = DataMessage.getNumberOfFrequencyBins(msg)
locationMessage = msgutils.getLocationMessage(msg)
lengthToRead = n * DataMessage.getNumberOfMeasurements(msg)
# Read the power values
power = msgutils.getData(msg)
powerVal = np.array(power[n * leftColumnsToExclude:lengthToRead -
n * rightColumnsToExclude])
minTime = float(
leftColumnsToExclude * miliSecondsPerMeasurement) / float(1000)
spectrogramData = powerVal.reshape(nM, n)
maxpower = msgutils.getMaxPower(msg)
if maxpower < cutoff:
maxpower = cutoff
# generate the spectrogram as an image.
if (not os.path.exists(spectrogramFilePath + ".png")) or\
DebugFlags.getDisableSessionIdCheckFlag():
dirname = util.getPath(STATIC_GENERATED_FILE_LOCATION) + sessionId
if not os.path.exists(dirname):
os.makedirs(
util.getPath(STATIC_GENERATED_FILE_LOCATION) + sessionId)
fig = plt.figure(figsize=(chWidth, chHeight)) # aspect ratio
frame1 = plt.gca()
frame1.axes.get_xaxis().set_visible(False)
frame1.axes.get_yaxis().set_visible(False)
cmap = plt.cm.spectral
cmap.set_under(UNDER_CUTOFF_COLOR)
fig = plt.imshow(np.transpose(spectrogramData),
interpolation='none',
origin='lower',
aspect="auto",
vmin=cutoff,
vmax=maxpower,
cmap=cmap)
util.debugPrint("Generated fig " + spectrogramFilePath + ".png")
plt.savefig(spectrogramFilePath + '.png',
bbox_inches='tight',
pad_inches=0,
dpi=100)
plt.clf()
plt.close()
else:
util.debugPrint("File exists -- not regenerating")
# generate the occupancy data for the measurement.
occupancyCount = [0 for i in range(0, nM)]
for i in range(0, nM):
occupancyCount[i] = float(
len(filter(lambda x: x >= cutoff,
spectrogramData[i, :]))) / float(n)
timeArray = [
int((i + leftColumnsToExclude) * miliSecondsPerMeasurement)
for i in range(0, nM)
]
# get the size of the generated png.
reader = png.Reader(filename=spectrogramFilePath + ".png")
(width, height, pixels, metadata) = reader.read()
if (not os.path.exists(spectrogramFilePath + ".cbar.png")) or \
DebugFlags.getDisableSessionIdCheckFlag():
# generate the colorbar as a separate image.
norm = mpl.colors.Normalize(vmin=cutoff, vmax=maxpower)
fig = plt.figure(figsize=(chWidth * 0.2,
chHeight * 1.22)) # aspect ratio
ax1 = fig.add_axes([0.0, 0, 0.1, 1])
mpl.colorbar.ColorbarBase(ax1,
cmap=cmap,
norm=norm,
orientation='vertical')
plt.savefig(spectrogramFilePath + '.cbar.png',
bbox_inches='tight',
pad_inches=0,
dpi=50)
plt.clf()
plt.close()
nextAcquisition = msgutils.getNextAcquisition(msg)
prevAcquisition = msgutils.getPrevAcquisition(msg)
if nextAcquisition is not None:
nextAcquisitionTime = DataMessage.getTime(nextAcquisition)
else:
nextAcquisitionTime = DataMessage.getTime(msg)
if prevAcquisition is not None:
prevAcquisitionTime = DataMessage.getTime(prevAcquisition)
else:
prevAcquisitionTime = DataMessage.getTime(msg)
tz = locationMessage[TIME_ZONE_KEY]
timeDelta = DataMessage.getMeasurementDuration(
msg) - float(leftBound) / float(1000) - float(rightBound) / float(1000)
meanOccupancy = np.mean(occupancyCount)
maxOccupancy = np.max(occupancyCount)
minOccupancy = np.min(occupancyCount)
medianOccupancy = np.median(occupancyCount)
result = {
"spectrogram":
Config.getGeneratedDataPath() + "/" + spectrogramFile + ".png",
"cbar":
Config.getGeneratedDataPath() + "/" + spectrogramFile + ".cbar.png",
"maxPower": maxpower,
"cutoff": cutoff,
"noiseFloor": noiseFloor,
"minPower": msgutils.getMinPower(msg),
"maxFreq": DataMessage.getFmax(msg),
"minFreq": DataMessage.getFmin(msg),
"minTime": minTime,
"timeDelta": timeDelta,
"measurementsPerAcquisition": DataMessage.getNumberOfMeasurements(msg),
"binsPerMeasurement": DataMessage.getNumberOfFrequencyBins(msg),
"measurementCount": nM,
"maxOccupancy": maxOccupancy,
"minOccupancy": minOccupancy,
"meanOccupancy": meanOccupancy,
"medianOccupancy": medianOccupancy,
"currentAcquisition": DataMessage.getTime(msg),
"prevAcquisition": prevAcquisitionTime,
"nextAcquisition": nextAcquisitionTime,
"formattedDate": timezone.formatTimeStampLong(DataMessage.getTime(msg),
tz),
"image_width": float(width),
"image_height": float(height)
}
# Now put in the occupancy data
result[STATUS] = OK
util.debugPrint(
"generateSingleAcquisitionSpectrogramAndOccupancyForFFTPower:returning (abbreviated): "
+ str(result))
result["timeArray"] = timeArray
result["occupancyArray"] = occupancyCount
return result
def getBandDataSummary(sensorId,
locationMessage,
sys2detect,
minFreq,
maxFreq,
mintime,
dayCount=None):
sensor = SensorDb.getSensorObj(sensorId)
if sensor is None:
return {STATUS: NOK, ERROR_MESSAGE: "Sensor Not found"}
measurementType = sensor.getMeasurementType()
tzId = locationMessage[TIME_ZONE_KEY]
locationMessageId = str(locationMessage["_id"])
freqRange = msgutils.freqRange(sys2detect, minFreq, maxFreq)
count = LocationMessage.getBandCount(locationMessage, freqRange)
if count == 0:
return {
FREQ_RANGE: freqRange,
COUNT: 0,
"minFreq": minFreq,
"maxFreq": maxFreq,
SYSTEM_TO_DETECT: sys2detect
}
else:
minOccupancy = LocationMessage.getMinBandOccupancy(
locationMessage, freqRange)
maxOccupancy = LocationMessage.getMaxBandOccupancy(
locationMessage, freqRange)
count = LocationMessage.getBandCount(locationMessage, freqRange)
meanOccupancy = LocationMessage.getMeanOccupancy(
locationMessage, freqRange)
minTime = LocationMessage.getFirstMessageTimeStampForBand(
locationMessage, freqRange)
maxTime = LocationMessage.getLastMessageTimeStampForBand(
locationMessage, freqRange)
maxTimes = timezone.getLocalTime(maxTime, tzId)
(tEndReadingsLocalTime, tEndReadingsLocalTimeTzName) = maxTimes
tEndDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
maxTime, tzId)
tStartDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
minTime, tzId)
tstampMin = timezone.formatTimeStampLong(minTime, tzId)
tstampMax = timezone.formatTimeStampLong(maxTime, tzId)
retval = {
"tStartDayBoundary": tStartDayBoundary,
"tEndDayBoundary": tEndDayBoundary,
"tStartReadings": minTime,
"tStartLocalTime": minTime,
"tStartLocalTimeFormattedTimeStamp": tstampMin,
"tEndReadings": maxTime,
"tEndReadingsLocalTime": maxTime,
"tEndLocalTimeFormattedTimeStamp": tstampMax,
"tEndDayBoundary": tEndDayBoundary,
"maxOccupancy": maxOccupancy,
"meanOccupancy": meanOccupancy,
"minOccupancy": minOccupancy,
"maxFreq": maxFreq,
"minFreq": minFreq,
SYSTEM_TO_DETECT: sys2detect,
FREQ_RANGE: freqRange,
"measurementType": measurementType,
"active": sensor.isBandActive(sys2detect, minFreq, maxFreq),
COUNT: count
}
return retval
def main_loop(tb):
tb.set_next_freq()
time.sleep(0.25)
# Send location and system info to server
loc_msg = tb.read_json_from_file('sensor.loc')
sys_msg = tb.read_json_from_file('sensor.sys')
ts = long(round(getLocalUtcTimeStamp()))
sensor_id = tb.u.get_usrp_info()['rx_serial']
loc_msg['t'] = ts
loc_msg['SensorID'] = sensor_id
sys_msg['t'] = ts
sys_msg['SensorID'] = sensor_id
tb.post_msg(loc_msg)
tb.post_msg(sys_msg)
data_file_path = '/tmp/temp.dat'
num_bands = len(tb.center_freq)
pause_duration = tb.acq_period / num_bands - tb.dwell_delay
n = 0
while 1:
# Form data header
ts = long(round(getLocalUtcTimeStamp()))
if n==0:
t1s = ts
center_freq = tb.center_freq[tb.band_ind]
bandwidth = tb.bandwidth[tb.band_ind]
f_start = center_freq - bandwidth / 2.0
f_stop = f_start + bandwidth
mpar = Struct(fStart=f_start, fStop=f_stop, n=tb.num_ch, td=tb.dwell_delay, Det='Average', Atten=tb.atten)
num_vectors_expected = int(tb.dwell_delay / tb.meas_duration)
# Need to add a field for overflow indicator
data_hdr = Struct(Ver='1.0.12', Type='Data', SensorID=sensor_id, SensorKey='NaN', t=ts, Sys2Detect='LTE', Sensitivity='Low', mType='FFT-Power', t1=t1s, a=n/num_bands+1, nM=num_vectors_expected, Ta=tb.acq_period, OL='NaN', wnI=-77.0, Comment='Using hard-coded (not detected) system noise power for wnI', Processed='False', DataType = 'Binary - int8', ByteOrder='N/A', Compression='None', mPar=mpar)
date_str = formatTimeStampLong(ts, loc_msg['TimeZone'])
print date_str, "fc =", center_freq/1e6, "MHz. Writing data to file..."
# Execute flow graph and wait for it to stop
f = open(data_file_path,'wb')
tb.set_fd(f.fileno())
tb.set_bin2ch_map(tb.bin2ch_map[tb.band_ind])
tb.start()
tb.wait()
# Check the number of power vectors generated and pad if necessary
num_vectors_written = tb.stats.nitems_written(0)
print '\nNum output items:', num_vectors_written
if num_vectors_written != num_vectors_expected:
print 'Warning: Unexpected number of power vectors generated'
if num_vectors_written < num_vectors_expected:
pad_len = (num_vectors_expected - num_vectors_written) * tb.num_ch
pad = array.array('b', [127]*pad_len)
f.write(pad.tostring())
f.close()
# Post data file
tb.post_data_msg(data_hdr, data_file_path)
# Tune to next frequency and pause
tb.set_next_freq()
print "Pause..."
time.sleep(max(0, ts + tb.acq_period / num_bands - getLocalUtcTimeStamp()))
tb.head.reset()
n += 1
def run(self):
global f, f_start, f_stop, bw, date_str, loc, ch_mag_sq_avg_plt, spec_gram, first_data_rcvd, Locked, Paused
def get_data(c, MSGLEN):
msg = ''
while len(msg) < MSGLEN:
chunk = c.recv(MSGLEN - len(msg))
if not chunk:
break
msg = msg + chunk
return msg
#host = socket.gethostname() # Get local machine name
#host = "localhost"
host = "192.168.0.1"
port = 4001
s = jsocket.JsonServer(address=host,
port=port) # Create a socket server
print 'Listening for client connection...'
try:
s.accept_connection() # Establish connection with client
except KeyboardInterrupt:
s.close()
print 'Closed socket'
return
Locked = False
try:
msg = s.read_obj()
except RuntimeError:
print 'Error reading from socket'
s.close()
print 'Closed socket'
return
while msg['Type'] != 'Data':
if msg['Type'] == 'Loc':
loc = msg
print loc, '\n'
elif msg['Type'] == 'Sys':
sys = msg
print sys, '\n'
else:
print 'Warning: Received unknown message type'
msg = s.read_obj()
data_hdr = msg
meas_ts = data_hdr['t']
if data_hdr['mType'] == 'FFT-Power':
f_start = data_hdr['mPar']['fStart']
f_stop = data_hdr['mPar']['fStop']
num_ch = data_hdr['mPar']['n']
meas_dur = data_hdr['mPar']['tm']
fc = (f_start + f_stop) / 2.0
bw = f_stop - f_start
f = map(lambda i: (f_start + i * bw / num_ch) / 1e6, range(num_ch))
nmeas = max(1, int(math.ceil(
1.0 / meas_dur))) # Read 1-sec worth of measurements at a time.
spec_gram = np.zeros((num_ch, SPEC_GRAM_DUR_SEC / meas_dur), np.int8)
f.append(f_stop / 1e6) ### Why needed?
# Obtain time of acquisition in local time of sensor.
# Subtract any DST offset that time.localtime() will add.
date_str = formatTimeStampLong(meas_ts, loc['TimeZone'])
print date_str + ', fc =', fc / 1e6, 'MHz; bw =', bw / 1e6, 'MHz; N_ch =', num_ch, '; T_meas = {0:0.3f} ms'.format(
meas_dur * 1000)
item_size = spec_gram.dtype.itemsize
while windowOpen:
data = get_data(s.conn, item_size * num_ch * nmeas)
if len(data) < item_size * num_ch * nmeas:
print 'Received partial measurement block'
s.close()
print 'Closed socket'
return
ch_power_dBm = np.fromstring(data, np.int8)
ch_power_dBm.resize(nmeas, num_ch)
if not Paused:
Locked = True
# Rotate new data in to right-most columns of spec_gram array.
spec_gram[:, 0:-nmeas] = spec_gram[:, nmeas:]
spec_gram[:, -nmeas:] = ch_power_dBm.transpose()
# Compute average of received frames.
ch_mag_sq = pow(10, (ch_power_dBm - 30) / 10.0)
ch_mag_sq_avg = ch_mag_sq.mean(axis=0)
# Repeat the first element of ch_mag_sq_avg for plotting only.
ch_mag_sq_avg_plt = np.repeat(ch_mag_sq_avg, [2] + [1] *
(len(ch_mag_sq_avg) - 1))
Locked = False
first_data_rcvd = True
s.close()
print 'Closed socket'
def generateSingleDaySpectrogramAndOccupancyForSweptFrequency(
sensorId, lat, lon, alt, sessionId, startTime, sys2detect, fstart,
fstop, subBandMinFreq, subBandMaxFreq, cutoff):
"""
Generate single day spectrogram and occupancy for SweptFrequency
Parameters:
- msg: the data message
- sessionId: login session id.
- startTime: absolute start time.
- sys2detect: the system to detect.
- fstart: start frequency.
- fstop: stop frequency
- subBandMinFreq: min freq of subband.
- subBandMaxFreq: max freq of subband.
- cutoff: occupancy threshold.
"""
try:
chWidth = Config.getScreenConfig()[CHART_WIDTH]
chHeight = Config.getScreenConfig()[CHART_HEIGHT]
locationMessage = DbCollections.getLocationMessages().find_one({
SENSOR_ID:
sensorId,
LAT:
lat,
LON:
lon,
ALT:
alt
})
if locationMessage is None:
return {STATUS: NOK, ERROR_MESSAGE: "Location message not found"}
tz = locationMessage[TIME_ZONE_KEY]
startTimeUtc = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
startTime, tz)
startMsg = DbCollections.\
getDataMessages(sensorId).find_one(
{TIME:{"$gte":startTimeUtc},
LOCATION_MESSAGE_ID:str(locationMessage["_id"]),
FREQ_RANGE:msgutils.freqRange(sys2detect, fstart, fstop)})
if startMsg is None:
util.debugPrint("Not found")
return {STATUS: NOK, ERROR_MESSAGE: "Data Not Found"}
if DataMessage.getTime(startMsg) - startTimeUtc > SECONDS_PER_DAY:
util.debugPrint("Not found - outside day boundary: " +
str(startMsg['t'] - startTimeUtc))
return {
STATUS: NOK,
ERROR_MESSAGE: "Not found - outside day boundary."
}
msg = startMsg
sensorId = msg[SENSOR_ID]
powerValues = msgutils.trimSpectrumToSubBand(msg, subBandMinFreq,
subBandMaxFreq)
vectorLength = len(powerValues)
if cutoff is None:
cutoff = DataMessage.getThreshold(msg)
else:
cutoff = int(cutoff)
spectrogramFile = sessionId + "/" + sensorId + "." + str(
startTimeUtc) + "." + str(cutoff) + "." + str(
subBandMinFreq) + "." + str(subBandMaxFreq)
spectrogramFilePath = util.getPath(
STATIC_GENERATED_FILE_LOCATION) + spectrogramFile
powerVal = np.array(
[cutoff for i in range(0, MINUTES_PER_DAY * vectorLength)])
spectrogramData = powerVal.reshape(vectorLength, MINUTES_PER_DAY)
# artificial power value when sensor is off.
sensorOffPower = np.transpose(
np.array([2000 for i in range(0, vectorLength)]))
prevMessage = msgutils.getPrevAcquisition(msg)
if prevMessage is None:
util.debugPrint("prevMessage not found")
prevMessage = msg
prevAcquisition = sensorOffPower
else:
prevAcquisitionTime = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
prevMessage['t'], tz)
util.debugPrint("prevMessage[t] " + str(prevMessage['t']) +
" msg[t] " + str(msg['t']) + " prevDayBoundary " +
str(prevAcquisitionTime))
prevAcquisition = np.transpose(
np.array(
msgutils.trimSpectrumToSubBand(prevMessage, subBandMinFreq,
subBandMaxFreq)))
occupancy = []
timeArray = []
maxpower = -1000
minpower = 1000
# Note that we are starting with the first message.
count = 1
while True:
acquisition = msgutils.trimSpectrumToSubBand(
msg, subBandMinFreq, subBandMaxFreq)
occupancyCount = float(
len(filter(lambda x: x >= cutoff, acquisition)))
occupancyVal = occupancyCount / float(len(acquisition))
occupancy.append(occupancyVal)
minpower = np.minimum(minpower, msgutils.getMinPower(msg))
maxpower = np.maximum(maxpower, msgutils.getMaxPower(msg))
if prevMessage['t1'] != msg['t1']:
# GAP detected so fill it with sensorOff
sindex = get_index(DataMessage.getTime(prevMessage),
startTimeUtc)
if get_index(DataMessage.getTime(prevMessage),
startTimeUtc) < 0:
sindex = 0
for i in range(
sindex,
get_index(DataMessage.getTime(msg), startTimeUtc)):
spectrogramData[:, i] = sensorOffPower
elif DataMessage.getTime(prevMessage) > startTimeUtc:
# Prev message is the same tstart and prevMessage is in the range of interest.
# Sensor was not turned off.
# fill forward using the prev acquisition.
for i in range(
get_index(DataMessage.getTime(prevMessage),
startTimeUtc),
get_index(msg["t"], startTimeUtc)):
spectrogramData[:, i] = prevAcquisition
else:
# forward fill from prev acquisition to the start time
# with the previous power value
for i in range(
0, get_index(DataMessage.getTime(msg), startTimeUtc)):
spectrogramData[:, i] = prevAcquisition
colIndex = get_index(DataMessage.getTime(msg), startTimeUtc)
spectrogramData[:, colIndex] = acquisition
timeArray.append(
float(DataMessage.getTime(msg) - startTimeUtc) / float(3600))
prevMessage = msg
prevAcquisition = acquisition
msg = msgutils.getNextAcquisition(msg)
if msg is None:
lastMessage = prevMessage
for i in range(
get_index(DataMessage.getTime(prevMessage),
startTimeUtc), MINUTES_PER_DAY):
spectrogramData[:, i] = sensorOffPower
break
elif DataMessage.getTime(msg) - startTimeUtc >= SECONDS_PER_DAY:
if msg['t1'] == prevMessage['t1']:
for i in range(
get_index(DataMessage.getTime(prevMessage),
startTimeUtc), MINUTES_PER_DAY):
spectrogramData[:, i] = prevAcquisition
else:
for i in range(
get_index(DataMessage.getTime(prevMessage),
startTimeUtc), MINUTES_PER_DAY):
spectrogramData[:, i] = sensorOffPower
lastMessage = prevMessage
break
count = count + 1
# generate the spectrogram as an image.
if not os.path.exists(spectrogramFilePath + ".png"):
fig = plt.figure(figsize=(chWidth, chHeight))
frame1 = plt.gca()
frame1.axes.get_xaxis().set_visible(False)
frame1.axes.get_yaxis().set_visible(False)
cmap = plt.cm.spectral
cmap.set_under(UNDER_CUTOFF_COLOR)
cmap.set_over(OVER_CUTOFF_COLOR)
dirname = util.getPath(STATIC_GENERATED_FILE_LOCATION) + sessionId
if maxpower < cutoff:
maxpower = cutoff
minpower = cutoff
if not os.path.exists(dirname):
os.makedirs(dirname)
fig = plt.imshow(spectrogramData,
interpolation='none',
origin='lower',
aspect='auto',
vmin=cutoff,
vmax=maxpower,
cmap=cmap)
util.debugPrint("Generated fig")
plt.savefig(spectrogramFilePath + '.png',
bbox_inches='tight',
pad_inches=0,
dpi=100)
plt.clf()
plt.close()
else:
util.debugPrint("File exists - not generating image")
util.debugPrint("FileName: " + spectrogramFilePath + ".png")
util.debugPrint("Reading " + spectrogramFilePath + ".png")
# get the size of the generated png.
reader = png.Reader(filename=spectrogramFilePath + ".png")
(width, height, pixels, metadata) = reader.read()
util.debugPrint("width = " + str(width) + " height = " + str(height))
# generate the colorbar as a separate image.
if not os.path.exists(spectrogramFilePath + ".cbar.png"):
norm = mpl.colors.Normalize(vmin=cutoff, vmax=maxpower)
fig = plt.figure(figsize=(chWidth * 0.3, chHeight * 1.2))
ax1 = fig.add_axes([0.0, 0, 0.1, 1])
mpl.colorbar.ColorbarBase(ax1,
cmap=cmap,
norm=norm,
orientation='vertical')
plt.savefig(spectrogramFilePath + '.cbar.png',
bbox_inches='tight',
pad_inches=0,
dpi=50)
plt.clf()
plt.close()
else:
util.debugPrint(spectrogramFilePath + ".cbar.png" +
" exists -- not generating")
localTime, tzName = timezone.getLocalTime(startTimeUtc, tz)
# step back for 24 hours.
prevAcquisitionTime = msgutils.getPrevDayBoundary(startMsg)
nextAcquisitionTime = msgutils.getNextDayBoundary(lastMessage)
meanOccupancy = np.mean(occupancy)
maxOccupancy = np.max(occupancy)
minOccupancy = np.min(occupancy)
medianOccupancy = np.median(occupancy)
result = {
"spectrogram":
Config.getGeneratedDataPath() + "/" + spectrogramFile + ".png",
"cbar": Config.getGeneratedDataPath() + "/" + spectrogramFile +
".cbar.png",
"maxPower": maxpower,
"maxOccupancy": maxOccupancy,
"minOccupancy": minOccupancy,
"meanOccupancy": meanOccupancy,
"medianOccupancy": medianOccupancy,
"cutoff": cutoff,
"aquisitionCount": count,
"minPower": minpower,
"tStartTimeUtc": startTimeUtc,
"timeDelta": HOURS_PER_DAY,
"prevAcquisition": prevAcquisitionTime,
"nextAcquisition": nextAcquisitionTime,
"formattedDate": timezone.formatTimeStampLong(startTimeUtc, tz),
"image_width": float(width),
"image_height": float(height)
}
result["timeArray"] = timeArray
result["occupancyArray"] = occupancy
if "ENBW" in lastMessage["mPar"]:
enbw = lastMessage["mPar"]["ENBW"]
result["ENBW"] = enbw
if "RBW" in lastMessage["mPar"]:
rbw = lastMessage["mPar"]["RBW"]
result["RBW"] = rbw
result[STATUS] = OK
util.debugPrint(result)
return result
except:
print "Unexpected error:", sys.exc_info()[0]
print sys.exc_info()
traceback.print_exc()
util.logStackTrace(sys.exc_info())
raise
def getOneDayStats(sensorId, lat, lon, alt, startTime, sys2detect, minFreq,
maxFreq):
"""
Generate and return a JSON structure with the one day statistics.
startTime is the start time in UTC
sys2detect is the system to detect.
minFreq is the minimum frequency of the frequency band of interest.
maxFreq is the maximum frequency of the frequency band of interest.
"""
locationMessage = DbCollections.getLocationMessages().find_one({
SENSOR_ID: sensorId,
LAT: lat,
LON: lon,
ALT: alt
})
if locationMessage is None:
return {STATUS: NOK, ERROR_MESSAGE: "No location information"}
freqRange = msgutils.freqRange(sys2detect, minFreq, maxFreq)
mintime = int(startTime)
maxtime = mintime + SECONDS_PER_DAY
tzId = locationMessage[TIME_ZONE_KEY]
mintime = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(startTime, tzId)
maxtime = mintime + SECONDS_PER_DAY
query = {
SENSOR_ID: sensorId,
LOCATION_MESSAGE_ID: str(locationMessage["_id"]),
"t": {
"$lte": maxtime,
"$gte": mintime
},
FREQ_RANGE: freqRange
}
cur = DbCollections.getDataMessages(sensorId).find(query)
if cur is None or cur.count() == 0:
return {STATUS: NOK, ERROR_MESSAGE: "Data messages not found"}
res = {}
values = {}
res["formattedDate"] = timezone.formatTimeStampLong(
mintime, locationMessage[TIME_ZONE_KEY])
acquisitionCount = cur.count()
prevMsg = None
for msg in cur:
if prevMsg is None:
prevMsg = msgutils.getPrevAcquisition(msg)
channelCount = msg["mPar"]["n"]
measurementsPerAcquisition = msg["nM"]
cutoff = msg["cutoff"]
values[int(msg["t"] - mintime)] = {
"t": msg["t"],
"maxPower": msg["maxPower"],
"minPower": msg["minPower"],
"maxOccupancy": msg["maxOccupancy"],
"minOccupancy": msg["minOccupancy"],
"meanOccupancy": msg["meanOccupancy"],
"medianOccupancy": msg["medianOccupancy"]
}
query = {SENSOR_ID: sensorId, "t": {"$gt": maxtime}, FREQ_RANGE: freqRange}
msg = DbCollections.getDataMessages(sensorId).find_one(query)
if msg is not None:
nextDay = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
msg["t"], tzId)
else:
nextDay = mintime
if prevMsg is not None:
prevDayBoundary = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
prevMsg["t"], tzId)
query = {
SENSOR_ID: sensorId,
"t": {
"$gte": prevDayBoundary
},
FREQ_RANGE: freqRange
}
msg = DbCollections.getDataMessages(sensorId).find_one(query)
prevDay = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
msg["t"], tzId)
else:
prevDay = mintime
res["nextIntervalStart"] = nextDay
res["prevIntervalStart"] = prevDay
res["currentIntervalStart"] = mintime
res[CHANNEL_COUNT] = channelCount
res["measurementsPerAcquisition"] = measurementsPerAcquisition
res[ACQUISITION_COUNT] = acquisitionCount
res["cutoff"] = cutoff
res["values"] = values
res[STATUS] = OK
return res
def getHourlyMaxMinMeanStats(sensorId, startTime, sys2detect, fmin, fmax,
subBandMinFreq, subBandMaxFreq, sessionId):
sensor = SensorDb.getSensor(sensorId)
if sensor is None:
return {STATUS: NOK, ERROR_MESSAGE: "Sensor Not Found"}
tstart = int(startTime)
fmin = int(subBandMinFreq)
fmax = int(subBandMaxFreq)
freqRange = msgutils.freqRange(sys2detect, fmin, fmax)
queryString = {
SENSOR_ID: sensorId,
TIME: {
'$gte': tstart
},
FREQ_RANGE: freqRange
}
util.debugPrint(queryString)
startMessage = DbCollections.getDataMessages(sensorId).find_one(
queryString)
if startMessage is None:
errorStr = "Start Message Not Found"
util.debugPrint(errorStr)
response = {STATUS: NOK, ERROR_MESSAGE: "No data found"}
return response
locationMessageId = DataMessage.getLocationMessageId(startMessage)
retval = {STATUS: OK}
values = {}
locationMessage = DbCollections.getLocationMessages().find_one(
{"_id": locationMessageId})
tZId = LocationMessage.getTimeZone(locationMessage)
tmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
tstart, LocationMessage.getTimeZone(locationMessage))
for hour in range(0, 23):
dataMessages = DbCollections.getDataMessages(sensorId).find({
"t": {
"$gte": tmin + hour * SECONDS_PER_HOUR
},
"t": {
"$lte": (hour + 1) * SECONDS_PER_HOUR
},
FREQ_RANGE:
freqRange
})
if dataMessages is not None:
stats = compute_stats_for_fft_power(dataMessages)
(nChannels, maxFreq, minFreq, cutoff, result) = stats
values[hour] = result
retval["values"] = values
# Now compute the next interval after the last one (if one exists)
tend = tmin + SECONDS_PER_DAY
queryString = {
SENSOR_ID: sensorId,
TIME: {
'$gte': tend
},
FREQ_RANGE: freqRange
}
msg = DbCollections.getDataMessages(sensorId).find_one(queryString)
if msg is None:
result["nextTmin"] = tmin
else:
nextTmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
msg[TIME], tZId)
result["nextTmin"] = nextTmin
# Now compute the previous interval before this one.
prevMessage = msgutils.getPrevAcquisition(startMessage)
if prevMessage is not None:
newTmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
prevMessage[TIME] - SECONDS_PER_DAY, tZId)
queryString = {
SENSOR_ID: sensorId,
TIME: {
'$gte': newTmin
},
FREQ_RANGE: msgutils.freqRange(sys2detect, fmin, fmax)
}
msg = DbCollections.getDataMessages(sensorId).find_one(queryString)
else:
msg = startMessage
result[STATUS] = OK
result["prevTmin"] = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
msg[TIME], tZId)
result["tmin"] = tmin
result["maxFreq"] = maxFreq
result["minFreq"] = minFreq
result["cutoff"] = cutoff
result[CHANNEL_COUNT] = nChannels
result["startDate"] = timezone.formatTimeStampLong(tmin, tZId)
result["values"] = values
return result
def __init__(self, num_ch, file_name, sensorLoc, sensorSys, sensor_id,
sensor_key, center_freq, bandwidth, meas_duration, atten,
samp_rate, avoid_LO):
gr.sync_block.__init__(self,
name="jsonfile_sink",
in_sig=[(bytes, num_ch)],
out_sig=None)
# Establish ssl socket connection to server
self.num_ch = num_ch
# Open file for output stream
self.file_name = file_name
self.f = open(self.file_name, 'wb')
self.srvr = blocks.file_descriptor_sink(self.num_ch * gr.sizeof_char,
self.f.fileno())
self.sensorLoc = sensorLoc
self.sensorSys = sensorSys
self.sensor_id = sensor_id
self.sensor_key = sensor_key
self.center_freq = center_freq
self.bandwidth = bandwidth
self.meas_duration = meas_duration
self.atten = atten
self.samp_rate = samp_rate
self.avoid = avoid_LO
print("Sensor ID: ", self.sensor_id)
# Send location and system info to server
loc_msg = self.read_json_from_file(self.sensorLoc)
sys_msg = self.read_json_from_file(self.sensorSys)
ts = long(round(getLocalUtcTimeStamp()))
print 'Serial no.', self.sensor_id
loc_msg['t'] = ts
loc_msg['SensorID'] = self.sensor_id
sys_msg['t'] = ts
sys_msg['SensorID'] = self.sensor_id
self.post_msg(loc_msg)
self.post_msg(sys_msg)
# Form data header
ts = long(round(getLocalUtcTimeStamp()))
f_start = self.center_freq - self.bandwidth / 2.0
if self.avoid:
#self.center_freq = self.center_freq + self.samp_rate/4.0
#print "Avoiding LO...\nShifting center Frequency to", self.center_freq
f_start = (self.center_freq -
self.samp_rate / 4) - self.bandwidth / 2.0
f_stop = f_start + self.bandwidth
if self.avoid:
print "Avoiding LO, frequencies are shifted to: [", f_start / 1e6, "MHz-", f_stop / 1e6, "MHz ]"
mpar = Struct(fStart=f_start,
fStop=f_stop,
n=self.num_ch,
td=-1,
tm=self.meas_duration,
Det='Average',
Atten=self.atten)
# Need to add a field for overflow indicator
data_hdr = Struct(
Ver='1.0.12',
Type='Data',
SensorID=self.sensor_id,
SensorKey=self.sensor_key,
t=ts,
Sys2Detect='LTE',
Sensitivity='Low',
mType='FFT-Power',
t1=ts,
a=1,
nM=-1,
Ta=-1,
OL='NaN',
wnI=-77.0,
Comment=
'Using hard-coded (not detected) system noise power for wnI',
Processed='False',
DataType='Binary - int8',
ByteOrder='N/A',
Compression='None',
mPar=mpar)
self.post_msg(data_hdr)
self.f.flush()
date_str = formatTimeStampLong(ts, loc_msg['TimeZone'])
print date_str, "fc =", self.center_freq / 1e6, "MHz. Writing data to file", self.file_name
def main_loop(tb):
if not tb.set_freq(tb.center_freq):
print "Failed to set frequency to", tb.center_freq
sys.exit(1)
print "Set frequency to", tb.center_freq / 1e6, "MHz"
time.sleep(0.25)
# Open file for output stream
f = open(tb.file_name, 'wb')
tb.set_fd(f.fileno())
# Write location and system info to file
loc_msg = tb.read_json_from_file('sensor.loc')
sys_msg = tb.read_json_from_file('sensor.sys')
ts = long(round(getLocalUtcTimeStamp()))
sensor_id = tb.u.get_usrp_info()['rx_serial']
print 'Serial no.', sensor_id
loc_msg['t'] = ts
loc_msg['SensorID'] = sensor_id
sys_msg['t'] = ts
sys_msg['SensorID'] = sensor_id
tb.post_msg(loc_msg, f)
tb.post_msg(sys_msg, f)
# Form data header
ts = long(round(getLocalUtcTimeStamp()))
f_start = tb.center_freq - tb.bandwidth / 2.0
f_stop = f_start + tb.bandwidth
mpar = Struct(fStart=f_start,
fStop=f_stop,
n=tb.num_ch,
td=-1,
tm=tb.meas_duration,
Det='Average',
Atten=tb.atten)
# Need to add a field for overflow indicator
data_hdr = Struct(
Ver='1.0.12',
Type='Data',
SensorID=sensor_id,
SensorKey='NaN',
t=ts,
Sys2Detect='LTE',
Sensitivity='Low',
mType='FFT-Power',
t1=ts,
a=1,
nM=-1,
Ta=-1,
OL='NaN',
wnI=-77.0,
Comment='Using hard-coded (not detected) system noise power for wnI',
Processed='False',
DataType='Binary - int8',
ByteOrder='N/A',
Compression='None',
mPar=mpar)
tb.post_msg(data_hdr, f)
f.flush()
date_str = formatTimeStampLong(ts, loc_msg['TimeZone'])
print date_str, "fc =", tb.center_freq / 1e6, "MHz. Writing data to file", tb.file_name
# Execute flow graph and wait for it to stop
tb.start()
tb.wait()
f.close()
tf = long(round(getLocalUtcTimeStamp()))
date_str = formatTimeStampLong(tf, loc_msg['TimeZone'])
print 'Closed file', date_str
def getDailyMaxMinMeanStats(sensorId, lat, lon, alt, tstart, ndays, sys2detect,
fmin, fmax, subBandMinFreq, subBandMaxFreq):
locationMessage = DbCollections.getLocationMessages().find_one({
SENSOR_ID: sensorId,
LAT: lat,
LON: lon,
ALT: alt
})
if locationMessage is None:
return {STATUS: NOK, ERROR_MESSAGE: "Location Information Not Found"}
locationMessageId = str(locationMessage["_id"])
tZId = locationMessage[TIME_ZONE_KEY]
tmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(tstart, tZId)
startMessage = DbCollections.getDataMessages(sensorId).find_one()
result = {}
result[STATUS] = OK
values = {}
for day in range(0, ndays):
tstart = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
tmin + day * SECONDS_PER_DAY, tZId)
tend = tstart + SECONDS_PER_DAY
queryString = {
LOCATION_MESSAGE_ID: locationMessageId,
TIME: {
'$gte': tstart,
'$lte': tend
},
FREQ_RANGE: msgutils.freqRange(sys2detect, fmin, fmax)
}
cur = DbCollections.getDataMessages(sensorId).find(queryString)
# cur.batch_size(20)
if startMessage['mType'] == FFT_POWER:
stats = compute_daily_max_min_mean_stats_for_fft_power(cur)
else:
stats = compute_daily_max_min_mean_median_stats_for_swept_freq(
cur, subBandMinFreq, subBandMaxFreq)
# gap in readings. continue.
if stats is None:
continue
(cutoff, dailyStat) = stats
values[day * 24] = dailyStat
# Now compute the next interval after the last one (if one exists)
tend = tmin + SECONDS_PER_DAY * ndays
queryString = {
LOCATION_MESSAGE_ID: locationMessageId,
TIME: {
'$gte': tend
},
FREQ_RANGE: msgutils.freqRange(sys2detect, fmin, fmax)
}
msg = DbCollections.getDataMessages(sensorId).find_one(queryString)
if msg is None:
result["nextTmin"] = tmin
else:
nextTmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
msg[TIME], tZId)
result["nextTmin"] = nextTmin
# Now compute the previous interval before this one.
prevMessage = msgutils.getPrevAcquisition(startMessage)
if prevMessage is not None:
newTmin = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
prevMessage[TIME] - SECONDS_PER_DAY * ndays, tZId)
queryString = {
LOCATION_MESSAGE_ID: locationMessageId,
TIME: {
'$gte': newTmin
},
FREQ_RANGE: msgutils.freqRange(sys2detect, fmin, fmax)
}
msg = DbCollections.getDataMessages(sensorId).find_one(queryString)
else:
msg = startMessage
sensor = SensorDb.getSensorObj(sensorId)
channelCount = sensor.getChannelCount(sys2detect, fmin, fmax)
result[STATUS] = OK
result["prevTmin"] = timezone.getDayBoundaryTimeStampFromUtcTimeStamp(
msg[TIME], tZId)
result["tmin"] = tmin
result["maxFreq"] = fmin
result["minFreq"] = fmax
result["cutoff"] = cutoff
result[CHANNEL_COUNT] = channelCount
result["startDate"] = timezone.formatTimeStampLong(tmin, tZId)
result["values"] = values
util.debugPrint(result)
return result
def main_loop(tb):
if not tb.set_freq(tb.center_freq):
print "Failed to set frequency to", tb.center_freq
sys.exit(1)
print "Set frequency to", tb.center_freq / 1e6, "MHz\n"
time.sleep(0.25)
sensor_id = "HackRF"
# Establish ssl socket connection to server
r = requests.post('https://' + tb.dest_host +
':8443/sensordata/getStreamingPort/' + sensor_id,
verify=False)
print 'server response:', r.text
response = r.json()
print 'socket port =', response['port']
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tb.s = s = ssl.wrap_socket(
sock) #, ca_certs='dummy.crt', cert_reqs=ssl.CERT_NONE)
s.connect((tb.dest_host, response['port']))
tb.set_sock(s)
# Send location and system info to server
loc_msg = tb.read_json_from_file('sensor.loc')
sys_msg = tb.read_json_from_file('sensor.sys')
ts = long(round(getLocalUtcTimeStamp()))
sensor_key = sys_msg['SensorKey']
print 'Sensor Key', sensor_key
#print 'Serial no.', sensor_id
loc_msg['t'] = ts
loc_msg['SensorID'] = sensor_id
sys_msg['t'] = ts
sys_msg['SensorID'] = sensor_id
tb.send_obj(loc_msg)
tb.send_obj(sys_msg)
# Form data header
ts = long(round(getLocalUtcTimeStamp()))
f_start = tb.center_freq - tb.bandwidth / 2.0
if tb.avoid_LO:
f_start = (tb.center_freq - tb.samp_rate / 4) - tb.bandwidth / 2.0
f_stop = f_start + tb.bandwidth
if tb.avoid_LO:
print "Avoiding LO, frequencies are shifted to: [", f_start / 1e6, "MHz-", f_stop / 1e6, "MHz ]"
mpar = Struct(fStart=f_start,
fStop=f_stop,
n=tb.num_ch,
td=-1,
tm=tb.meas_duration,
Det='Average',
Atten=tb.atten)
# Need to add a field for overflow indicator
data = Struct(
Ver='1.0.12',
Type='Data',
SensorID='HackRF',
SensorKey='12345',
t=ts,
Sys2Detect='LTE',
Sensitivity='Low',
mType='FFT-Power',
t1=ts,
a=1,
nM=-1,
Ta=-1,
OL='NaN',
wnI=-77.0,
Comment='Using hard-coded (not detected) system noise power for wnI',
Processed='False',
DataType='Binary - int8',
ByteOrder='N/A',
Compression='None',
mPar=mpar)
tb.send_obj(data)
date_str = formatTimeStampLong(ts, loc_msg['TimeZone'])
print date_str, "fc =", tb.center_freq / 1e6, "MHz. Sending data to", tb.dest_host
# Start flow graph
tb.start()
tb.wait()
tb.s.close()
print 'Closed socket'
def main_loop(tb):
tb.set_next_freq()
time.sleep(0.25)
# Send location and system info to server
loc_msg = tb.read_json_from_file('sensor.loc')
sys_msg = tb.read_json_from_file('sensor.sys')
ts = long(round(getLocalUtcTimeStamp()))
sensor_id = tb.u.get_usrp_info()['rx_serial']
loc_msg['t'] = ts
loc_msg['SensorID'] = sensor_id
sys_msg['t'] = ts
sys_msg['SensorID'] = sensor_id
tb.post_msg(loc_msg)
tb.post_msg(sys_msg)
data_file_path = '/tmp/temp.dat'
num_bands = len(tb.center_freq)
pause_duration = tb.acq_period / num_bands - tb.dwell_delay
n = 0
while 1:
# Form data header
ts = long(round(getLocalUtcTimeStamp()))
if n == 0:
t1s = ts
center_freq = tb.center_freq[tb.band_ind]
bandwidth = tb.bandwidth[tb.band_ind]
f_start = center_freq - bandwidth / 2.0
f_stop = f_start + bandwidth
mpar = Struct(fStart=f_start,
fStop=f_stop,
n=tb.num_ch,
td=tb.dwell_delay,
Det='Average',
Atten=tb.atten)
num_vectors_expected = int(tb.dwell_delay / tb.meas_duration)
# Need to add a field for overflow indicator
data_hdr = Struct(
Ver='1.0.12',
Type='Data',
SensorID=sensor_id,
SensorKey='NaN',
t=ts,
Sys2Detect='LTE',
Sensitivity='Low',
mType='FFT-Power',
t1=t1s,
a=n / num_bands + 1,
nM=num_vectors_expected,
Ta=tb.acq_period,
OL='NaN',
wnI=-77.0,
Comment=
'Using hard-coded (not detected) system noise power for wnI',
Processed='False',
DataType='Binary - int8',
ByteOrder='N/A',
Compression='None',
mPar=mpar)
date_str = formatTimeStampLong(ts, loc_msg['TimeZone'])
print date_str, "fc =", center_freq / 1e6, "MHz. Writing data to file..."
# Execute flow graph and wait for it to stop
f = open(data_file_path, 'wb')
tb.set_fd(f.fileno())
tb.set_bin2ch_map(tb.bin2ch_map[tb.band_ind])
tb.start()
tb.wait()
# Check the number of power vectors generated and pad if necessary
num_vectors_written = tb.stats.nitems_written(0)
print '\nNum output items:', num_vectors_written
if num_vectors_written != num_vectors_expected:
print 'Warning: Unexpected number of power vectors generated'
if num_vectors_written < num_vectors_expected:
pad_len = (num_vectors_expected -
num_vectors_written) * tb.num_ch
pad = array.array('b', [127] * pad_len)
f.write(pad.tostring())
f.close()
# Post data file
tb.post_data_msg(data_hdr, data_file_path)
# Tune to next frequency and pause
tb.set_next_freq()
print "Pause..."
time.sleep(
max(0, ts + tb.acq_period / num_bands - getLocalUtcTimeStamp()))
tb.head.reset()
n += 1
def __init__(self, num_ch, dest_host, sensorLoc, sensorSys, sensor_id,
sensor_key, center_freq, bandwidth, meas_duration, atten):
gr.sync_block.__init__(self,
name="sslsocket_sink",
in_sig=[(bytes, num_ch)],
out_sig=None)
# Establish ssl socket connection to server
self.num_ch = num_ch
self.dest_host = dest_host
self.sensorLoc = sensorLoc
self.sensorSys = sensorSys
self.sensor_id = sensor_id
self.sensor_key = sensor_key
self.center_freq = center_freq
self.bandwidth = bandwidth
self.meas_duration = meas_duration
self.atten = atten
r = requests.post('https://' + self.dest_host +
':8443/sensordata/getStreamingPort/' +
self.sensor_id,
verify=False)
print 'server response:', r.text
response = r.json()
print 'socket port =', response['port']
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock = ssl.wrap_socket(sock, cert_reqs=ssl.CERT_NONE)
self.sock.connect((self.dest_host, response['port']))
print("Sensor ID: ", self.sensor_id)
# Send location and system info to server
loc_msg = self.read_json_from_file(self.sensorLoc)
sys_msg = self.read_json_from_file(self.sensorSys)
ts = long(round(getLocalUtcTimeStamp()))
print 'Serial no.', self.sensor_id
loc_msg['t'] = ts
loc_msg['SensorID'] = self.sensor_id
sys_msg['t'] = ts
sys_msg['SensorID'] = self.sensor_id
self.send_obj(loc_msg)
self.send_obj(sys_msg)
# Form data header
ts = long(round(getLocalUtcTimeStamp()))
f_start = self.center_freq - self.bandwidth / 2.0
f_stop = f_start + self.bandwidth
mpar = Struct(fStart=f_start,
fStop=f_stop,
n=self.num_ch,
td=-1,
tm=self.meas_duration,
Det='Average',
Atten=self.atten)
# Need to add a field for overflow indicator
data = Struct(
Ver='1.0.12',
Type='Data',
SensorID=self.sensor_id,
SensorKey=self.sensor_key,
t=ts,
Sys2Detect='LTE',
Sensitivity='Low',
mType='FFT-Power',
t1=ts,
a=1,
nM=-1,
Ta=-1,
OL='NaN',
wnI=-77.0,
Comment=
'Using hard-coded (not detected) system noise power for wnI',
Processed='False',
DataType='Binary - int8',
ByteOrder='N/A',
Compression='None',
mPar=mpar)
self.send_obj(data)
date_str = formatTimeStampLong(ts, loc_msg['TimeZone'])
print date_str, "fc =", self.center_freq / 1e6, "MHz. Sending data to", self.dest_host