def dump_results():
dynamo_table = Table("weightresult")
all_cloud_results_output = open("all_cloud_results.csv", "w")
fog_results_output = open("fog_results.csv", "w")
for s_string in ["all_cloud_results", "cloud_vs_fog"]:
item = dynamo_table.getItem({
"environment": "roomA",
"sensor": s_string
})
for result in item["results"]:
output_line = ",".join([
result["gateway_A_subject"], result["gateway_B_subject"],
result["gateway_C_subject"],
str(result["Comm_pi_pi"]),
str(result["Comm_pi_lambda"]),
str(result["Compu_pi"]),
str(result["Lambda_ExecTime"]),
str(result["w_1"]),
str(result["w_2"]),
str(result["data_bytes_features"]),
str(result["data_bytes_entire"]),
str(result["number_of_sensors"]),
str(result["Error"])
])
if ("fog_or_cloud" in result
and result["fog_or_cloud"] != "all_cloud"):
fog_results_output.write(output_line + "\n")
else:
all_cloud_results_output.write(output_line + "\n")
def main():
figureNumber = sys.argv[1]
oldTime = 0
resultTable = Table('weightresult')
resultItem = resultTable.getItem({
'environment': 'roomA',
'sensor': 'sensorA&B&C'
})
plotBandwidth(resultItem)
plotLatency(resultItem)
plotCosts(resultItem)
plotAccuracy(resultItem, figureNumber)
plt.show()
def sample_size():
'''
Listens to the GUI for the sample size of the experiment.
Uploads this number to DynamoDB so that the Pi's can access it.
'''
tStart = time.time()
table = Table('SampleSize')
item = table.getItem({
'forum' : '1',
'subject' : 'PC1'
})
item['timeStamp'] = Decimal(tStart)
# item['sampleSize'] = e.get() # e.get is the number you entered in the window
item['sampleSize'] = Decimal(e.get())
table.addItem(item)
if aIsReady:
ready = ready + 1
oldTimeA = timeA
if bIsReady:
ready = ready + 1
oldTimeB = timeB
if cIsReady:
ready = ready + 1
oldTimeC = timeC
# This means that all gateways are ready
if ready >= 3:
ready = 0
print("Now triggering lambda...")
break
# The Trigger_A table makes Lambda start processing the data.
# The table needs to be specified in the Lambda function config.
lambdaTriggerTable = Table('Trigger_A')
item = lambdaTriggerTable.getItem({'forum': 'roomA', 'subject': '1'})
tEnd = time.time()
item['timeStamp'] = Decimal(str(tEnd))
lambdaTriggerTable.addItem(item)
print("Lambda triggered!")
# if KeyboardInterrupt:
# print("Closing Lambda Trigger")
# sys.exit()
def lambda_handler(event, context):
# Fetch the DynamoDB resource
tStart = time.time()
# Change: Getting the number of samples from the 'SampleSize' table is tricky
# When we'll have multiple Pi's, keeping track of this number will be buggy
# For this reason, I'm setting the value of 'datanum' to the number of items
# that we're going to get from the table containing the aggregated sensor data
# Initialize helper variables
featurenum = 3
collectornum = 2
betam = np.zeros((featurenum, collectornum))
dataBytesFeatures = 0
numSensors = 0
# Fetch the features calculated by Gateway A
table_A = Table('sensingdata_A')
itemKey = {'forum': 'roomA', 'subject': 'sensorA'}
item_A = table_A.getItem(itemKey)
betam[0][0] = item_A['feature_A']
betam[1][0] = item_A['feature_B']
betam[2][0] = item_A['feature_C']
# dataBytesFeatures += item_A['data_bytes']
# numSensors += item_A['number_of_sensors']
# Fetch the features calculated by Gateway B
table_B = Table('sensingdata_B')
itemKey = {'forum': 'roomB', 'subject': 'sensorB'}
item_B = table_B.getItem(itemKey)
betam[0][1] = item_B['feature_A']
betam[1][1] = item_B['feature_B']
betam[2][1] = item_B['feature_C']
# dataBytesFeatures += item_B['data_bytes']
# numSensors += item_B['number_of_sensors']
# Fetch the aggregated data from Gateway C
table_C = Table('sensingdata_C')
itemKey = {'forum': 'roomC', 'subject': 'sensorC'}
item_C = table_C.getItem(itemKey)
aggregatedData = item_C['aggregated_data']
#numSensors += item_C['number_of_sensors']
datanum = len(aggregatedData)
X = np.zeros((datanum, featurenum))
y = np.zeros((datanum, 1))
for i in range(datanum):
X[i][0] = aggregatedData[i]['X_1']
X[i][1] = aggregatedData[i]['X_2']
X[i][2] = aggregatedData[i]['X_3']
y[i][0] = aggregatedData[i]['Y']
#data_bytes = item_C['data_bytes']
def prox_simplex(y):
# projection onto simplex
n = len(y)
val = -np.sort(-y)
suppt_v = np.cumsum(val) - np.arange(1, n + 1, 1) * val
k_act = np.sum(suppt_v < 1)
lam = (np.sum(val[0:k_act]) - 1.0) / k_act
x = np.maximum(y - lam, 0.0)
return x
def combine(y, X, betam):
K = betam.shape[1]
w = np.ones((K, )) / K
maxit = 1000
tol = 1e-3
Xb = np.dot(X, betam)
step = 1.0 / np.max(np.linalg.svd(Xb, full_matrices=0,
compute_uv=0))**2
for it in range(maxit):
prev_w = np.copy(w)
res = y - np.dot(np.matrix(Xb), np.matrix(w).T)
grad = -np.dot(np.matrix(Xb).T, np.matrix(res))
w -= step * np.squeeze(np.asarray(grad.T))
w = prox_simplex(w)
if np.linalg.norm(w - prev_w) / (1e-20 +
np.linalg.norm(prev_w)) < tol:
break
return w
w = combine(y, X, betam)
w_temp = [decimal.Decimal(str(w[i])) for i in range(collectornum)]
wb = np.dot(np.matrix(betam), np.matrix(w).T)
Predict_y = np.dot(np.matrix(X), wb)
Predict_y_array = np.squeeze(np.asarray(Predict_y))
MSE = np.sqrt(np.sum((y - np.squeeze(np.asarray(Predict_y)))**2)) / datanum
MSE_temp = decimal.Decimal(str(MSE))
tEnd = time.time()
Lambda_ExecTime = tEnd - tStart
tEnd_temp = decimal.Decimal(str(tEnd))
Lambda_ExecTime_temp = decimal.Decimal(str(Lambda_ExecTime))
Predict_y_array = Predict_y_array.tolist()
y = y.tolist()
for i in range(len(Predict_y_array)):
y[i] = decimal.Decimal(str(y[i][0]))
Predict_y_array[i] = decimal.Decimal(str(Predict_y_array[i]))
table = Table('weightresult')
resultData = {
'environment': 'roomA',
'sensor': 'sensorA&B&C',
'w_1': w_temp[0],
'w_2': w_temp[1],
'Prediction': Predict_y_array,
'Real_Data': y,
'Error': MSE_temp,
'Lambda_ExecTime': Lambda_ExecTime_temp,
'Time': tEnd_temp
}
item = table.addItem(resultData)
# Record this run
resultData.pop('environment', None)
resultData.pop('sensor', None)
resultData.pop('Prediction', None)
resultData.pop('Real_Data', None)
record = table.getItem({'environment': 'roomA', 'sensor': 'expResults'})
results = record['results']
results.append(resultData)
item = table.addItem(record)
def main(tableLetter, sleepTime):
"""
Runs the experiment as indicated below:
1) Listens for a trigger on the 'SampleSize' table on DynamoDB
2) On the trigger, starts listening and reading from bluetooth (port 1)
3) Collects additional data from its sensors
4) Calculates the features of the data
5) Uploads the features to DynamoDB
6) Visualizes these results using an animated matplotlib figure.
"""
# Establish a connection to the 'SampleSize' table
table = Table('SampleSize')
oldSizeTime = 0 # Placeholder. The value will be overwritten by a time stamp
while True:
# Break out of the inner while-loop only when the table has been updated
sense.set_pixels(LED.threeDots('green', 'G'))
stayInLoop = True
key = {
'forum' : '1',
'subject' : 'PC1'
}
while stayInLoop:
try:
stayInLoop, timeStamp = table.compareValues(key, 'timeStamp', oldSizeTime, True)
# Sleep because pinging AWS is costs $$$
time.sleep(sleepTime)
except KeyboardInterrupt:
print("Shutting down...")
sense.set_pixels(LED.pluses('black'))
sys.exit()
oldSizeTime = timeStamp
numDataPoints = table.getItem(key)['sampleSize']
numFeatures = 3
# Listen for incoming bluetooth data on port 1
sense.set_pixels(LED.arrowReceive('orange', 'black'))
timeOne = time.time()
btTime, dataFromBT = BT.listenOnBluetooth(1)
timeTwo = time.time()
# Edit: timeTwo - timeOne != btTime since some time is spent waiting on the sockets
# Make the LEDs show the computation state
sense.set_pixels(LED.diamond('blue'))
# Transform the received bluetooth data to numpy arrays
targetMatrix, designMatrix = bluetoothDataToNPArrays(dataFromBT, numDataPoints, numFeatures)
# Aggregate the bluetooth data, with data collected from the Gateway Pi
sense.set_pixels(LED.pluses('green'))
targetMatrix, designMatrix = collectData(targetMatrix, designMatrix, numDataPoints)
# Signify the computation state
sense.set_pixels(LED.diamond('blue'))
# Calculate how many sensors were used.
# Number of received readings divided by the number of readings per sensor.
numSensors = targetMatrix.shape[0] / numDataPoints
# Calculate the features if the gateway has permission to do so
if calculateFeatures[tableLetter]:
features = gradientDescent(targetMatrix, designMatrix, numFeatures, numDataPoints)
timeThree = time.time()
compTime = timeThree - timeTwo
# Upload data to DynamoDB
sense.set_pixels(LED.arrowSend('blue', 'black'))
if calculateFeatures[tableLetter]:
uploadTime = uploadToDB(tableLetter, features, btTime, compTime, numSensors)
else:
# Make sure that targetMatrix and designMatrix get read in the correct order
uploadTime = uploadToDB(tableLetter, [targetMatrix, designMatrix], btTime, compTime, numSensors)
# Reset the state of the LED
sense.set_pixels(LED.xCross('red'))
def uploadToDB(tableLetter, data, btTime, compTime, numSensors):
"""
Uploads the features and the latencies to DynamoDB
Param(s):
(char) Denotes the table, e.g. A, B
(numpy array) Features calculated from the dataset
(int) Time taken to load bluetooth data in seconds
(int) Time taken to compute the features in seconds
Returns an int showing the time taken to upload to DynamoDB in seconds
"""
startTime = time.time()
table = Table('sensingdata_' + tableLetter)
room = 'roomA'
sensor = 'sensor' + tableLetter
# Prepare the upload payload
item = table.getItem({
'forum' : room,
'subject' : sensor
})
# If the Gateway was designated to calculate features...
if calculateFeatures[tableLetter]:
item['feature_A'] = Decimal(str(float(data[0][0])))
item['feature_B'] = Decimal(str(float(data[1][0])))
item['feature_C'] = Decimal(str(float(data[2][0])))
sizeOfDataInBytes = data.nbytes
# Otherwise, the Gateway was meant to aggregate data without calculations
else:
targetMatrix = data[0]
designMatrix = data[1]
# Numpy indexes follow the [row][column] convention
# ndarray.shape returns the dimensions as a (#OfRows, #OfColumns)
# Both of our matrices have the same number of rows, hence one measure is enough
numOfRows = designMatrix.shape[0]
aggregatedItems = []
for i in range(numOfRows):
currentItem = {}
currentItem['X_1'] = Decimal(str(designMatrix[i][0])) # Time
currentItem['X_2'] = Decimal(str(designMatrix[i][1])) # Pressure
currentItem['X_3'] = Decimal(str(designMatrix[i][2])) # Humidity
currentItem['Y'] = Decimal(str(targetMatrix[i][0])) # Temperature
aggregatedItems.append(currentItem)
sizeOfDataInBytes = designMatrix.nbytes + targetMatrix.nbytes
item['aggregated_data'] = aggregatedItems
# Upload this document to DynamoDB
item['Comm_pi_pi'] = Decimal(str(btTime))
item['Compu_pi'] = Decimal(str(compTime))
item['data_bytes'] = Decimal(str(sizeOfDataInBytes))
table.addItem(item)
# Attach a time stamp and the size of the file to the header item in DynamoDB
endTime = time.time()
uploadDuration = endTime - startTime
item['Comm_pi_lambda'] = Decimal(str(uploadDuration))
item['timeStamp'] = Decimal(str(endTime))
item['number_of_sensors'] = Decimal(str(numSensors))
table.addItem(item)
# Log the experiment run to DynamoDB, regardless of gateway type
item.pop('aggregated_data', None)
item.pop('forum', None)
item.pop('subject', None)
# newData = []
# labels = []
# for key in item.keys():
# labels.append(key)
# newData.append(item[key])
record = table.getItem({'forum' : 'roomA', 'subject' : 'records'})
try:
data = record['data']
except KeyError:
record['data'] = []
data.append(item)
# record['data_labels'] = labels
table.addItem(record)
#print(" ".join(["Uploaded", sizeOfDataInBytes, "bytes of data to DynamoDB"]))
print("Uploaded ", str(sizeOfDataInBytes), " bytes of data to DynamoDB")
return uploadDuration