def dump_results(): dynamo_table = Table("weightresult") all_cloud_results_output = open("all_cloud_results.csv", "w") fog_results_output = open("fog_results.csv", "w") for s_string in ["all_cloud_results", "cloud_vs_fog"]: item = dynamo_table.getItem({ "environment": "roomA", "sensor": s_string }) for result in item["results"]: output_line = ",".join([ result["gateway_A_subject"], result["gateway_B_subject"], result["gateway_C_subject"], str(result["Comm_pi_pi"]), str(result["Comm_pi_lambda"]), str(result["Compu_pi"]), str(result["Lambda_ExecTime"]), str(result["w_1"]), str(result["w_2"]), str(result["data_bytes_features"]), str(result["data_bytes_entire"]), str(result["number_of_sensors"]), str(result["Error"]) ]) if ("fog_or_cloud" in result and result["fog_or_cloud"] != "all_cloud"): fog_results_output.write(output_line + "\n") else: all_cloud_results_output.write(output_line + "\n")
def main(): figureNumber = sys.argv[1] oldTime = 0 resultTable = Table('weightresult') resultItem = resultTable.getItem({ 'environment': 'roomA', 'sensor': 'sensorA&B&C' }) plotBandwidth(resultItem) plotLatency(resultItem) plotCosts(resultItem) plotAccuracy(resultItem, figureNumber) plt.show()
def dump_data(): table_names = ["sensingdata_A", "sensingdata_B", "sensingdata_C"] for table_name in table_names: print("Scanning", table_name) dynamo_table = Table(table_name) cloud_compute_output = open(table_name + "_cloud_compute.csv", "w") local_compute_output = open(table_name + "_local_compute.csv", "w") i = 0 for page in dynamo_table.getAllItems(): for item in page["Items"]: try: if "aggregated_data" in item.keys(): output_line = ",".join([ item["subject"]["S"], str(len(item["aggregated_data"]["L"])) ]) cloud_compute_output.write(output_line + "\n") for reading in item["aggregated_data"]["L"]: output_line = ",".join([ reading["M"]["X_1"]["N"], reading["M"]["X_2"]["N"], reading["M"]["X_3"]["N"], reading["M"]["Y"]["N"] ]) cloud_compute_output.write(output_line + "\n") elif table_name in {"sensingdata_A", "sensingdata_B"}: output_line = ",".join([ item["forum"]["S"], item["subject"]["S"], item["timeStamp"]["N"], item["feature_A"]["N"], item["feature_B"]["N"], item["feature_C"]["N"], item["Comm_pi_pi"]["N"], item["data_bytes"]["N"], item["Comm_pi_lambda"]["N"], item["number_of_sensors"]["N"], item["Compu_pi"]["N"] ]) local_compute_output.write(output_line + "\n") except KeyError: print(item.keys()) break i += 1 if (i % 50 == 0): print("Appended", i, "documents")
def initializeTable(nameOfTable, hashKey, rangeKey, hashVal, rangeVal): ''' Helper method for initializing tables. Assumes all inputs are strings Param(s): (String) Name of the table being created (String) Name of the hash key. Assumed to be AWS type 'S' (String) Name of the range key. Assumed to be AWS type 'S' (String) Value of the hash key for the first item (String) Value of the range key for the first item ''' table = Table(nameOfTable, [hashKey, 'S'], [rangeKey, 'S']) table.addItem({hashKey: hashVal, rangeKey: rangeVal})
def sample_size(): ''' Listens to the GUI for the sample size of the experiment. Uploads this number to DynamoDB so that the Pi's can access it. ''' tStart = time.time() table = Table('SampleSize') item = table.getItem({ 'forum' : '1', 'subject' : 'PC1' }) item['timeStamp'] = Decimal(tStart) # item['sampleSize'] = e.get() # e.get is the number you entered in the window item['sampleSize'] = Decimal(e.get()) table.addItem(item)
import time import sys from DynamoDBUtility import Table from decimal import Decimal #_______________________________________________________________________________ oldTimeA = 0 oldTimeB = 0 oldTimeC = 0 queryA = {'forum': 'roomA', 'subject': 'sensorA'} queryB = {'forum': 'roomA', 'subject': 'sensorB'} queryC = {'forum': 'roomA', 'subject': 'sensorC'} tableA = Table('sensingdata_A') tableB = Table('sensingdata_B') tableC = Table( 'sensingdata_C') # I changed this since I don't see latency_C's purpose while True: ready = 0 # This while-loop is only escaped once all the Gateway Pi's are done while True: aIsReady, timeA = tableA.compareValues(queryA, 'timeStamp', oldTimeA, False) bIsReady, timeB = tableB.compareValues(queryB, 'timeStamp', oldTimeB, False) cIsReady, timeC = tableC.compareValues(queryC, 'timeStamp', oldTimeC,
def lambda_handler(event, context): # Fetch the DynamoDB resource tStart = time.time() # Change: Getting the number of samples from the 'SampleSize' table is tricky # When we'll have multiple Pi's, keeping track of this number will be buggy # For this reason, I'm setting the value of 'datanum' to the number of items # that we're going to get from the table containing the aggregated sensor data # Initialize helper variables featurenum = 3 collectornum = 2 betam = np.zeros((featurenum, collectornum)) dataBytesFeatures = 0 numSensors = 0 # Fetch the features calculated by Gateway A table_A = Table('sensingdata_A') itemKey = {'forum': 'roomA', 'subject': 'sensorA'} item_A = table_A.getItem(itemKey) betam[0][0] = item_A['feature_A'] betam[1][0] = item_A['feature_B'] betam[2][0] = item_A['feature_C'] # dataBytesFeatures += item_A['data_bytes'] # numSensors += item_A['number_of_sensors'] # Fetch the features calculated by Gateway B table_B = Table('sensingdata_B') itemKey = {'forum': 'roomB', 'subject': 'sensorB'} item_B = table_B.getItem(itemKey) betam[0][1] = item_B['feature_A'] betam[1][1] = item_B['feature_B'] betam[2][1] = item_B['feature_C'] # dataBytesFeatures += item_B['data_bytes'] # numSensors += item_B['number_of_sensors'] # Fetch the aggregated data from Gateway C table_C = Table('sensingdata_C') itemKey = {'forum': 'roomC', 'subject': 'sensorC'} item_C = table_C.getItem(itemKey) aggregatedData = item_C['aggregated_data'] #numSensors += item_C['number_of_sensors'] datanum = len(aggregatedData) X = np.zeros((datanum, featurenum)) y = np.zeros((datanum, 1)) for i in range(datanum): X[i][0] = aggregatedData[i]['X_1'] X[i][1] = aggregatedData[i]['X_2'] X[i][2] = aggregatedData[i]['X_3'] y[i][0] = aggregatedData[i]['Y'] #data_bytes = item_C['data_bytes'] def prox_simplex(y): # projection onto simplex n = len(y) val = -np.sort(-y) suppt_v = np.cumsum(val) - np.arange(1, n + 1, 1) * val k_act = np.sum(suppt_v < 1) lam = (np.sum(val[0:k_act]) - 1.0) / k_act x = np.maximum(y - lam, 0.0) return x def combine(y, X, betam): K = betam.shape[1] w = np.ones((K, )) / K maxit = 1000 tol = 1e-3 Xb = np.dot(X, betam) step = 1.0 / np.max(np.linalg.svd(Xb, full_matrices=0, compute_uv=0))**2 for it in range(maxit): prev_w = np.copy(w) res = y - np.dot(np.matrix(Xb), np.matrix(w).T) grad = -np.dot(np.matrix(Xb).T, np.matrix(res)) w -= step * np.squeeze(np.asarray(grad.T)) w = prox_simplex(w) if np.linalg.norm(w - prev_w) / (1e-20 + np.linalg.norm(prev_w)) < tol: break return w w = combine(y, X, betam) w_temp = [decimal.Decimal(str(w[i])) for i in range(collectornum)] wb = np.dot(np.matrix(betam), np.matrix(w).T) Predict_y = np.dot(np.matrix(X), wb) Predict_y_array = np.squeeze(np.asarray(Predict_y)) MSE = np.sqrt(np.sum((y - np.squeeze(np.asarray(Predict_y)))**2)) / datanum MSE_temp = decimal.Decimal(str(MSE)) tEnd = time.time() Lambda_ExecTime = tEnd - tStart tEnd_temp = decimal.Decimal(str(tEnd)) Lambda_ExecTime_temp = decimal.Decimal(str(Lambda_ExecTime)) Predict_y_array = Predict_y_array.tolist() y = y.tolist() for i in range(len(Predict_y_array)): y[i] = decimal.Decimal(str(y[i][0])) Predict_y_array[i] = decimal.Decimal(str(Predict_y_array[i])) table = Table('weightresult') resultData = { 'environment': 'roomA', 'sensor': 'sensorA&B&C', 'w_1': w_temp[0], 'w_2': w_temp[1], 'Prediction': Predict_y_array, 'Real_Data': y, 'Error': MSE_temp, 'Lambda_ExecTime': Lambda_ExecTime_temp, 'Time': tEnd_temp } item = table.addItem(resultData) # Record this run resultData.pop('environment', None) resultData.pop('sensor', None) resultData.pop('Prediction', None) resultData.pop('Real_Data', None) record = table.getItem({'environment': 'roomA', 'sensor': 'expResults'}) results = record['results'] results.append(resultData) item = table.addItem(record)
def main(tableLetter, sleepTime): """ Runs the experiment as indicated below: 1) Listens for a trigger on the 'SampleSize' table on DynamoDB 2) On the trigger, starts listening and reading from bluetooth (port 1) 3) Collects additional data from its sensors 4) Calculates the features of the data 5) Uploads the features to DynamoDB 6) Visualizes these results using an animated matplotlib figure. """ # Establish a connection to the 'SampleSize' table table = Table('SampleSize') oldSizeTime = 0 # Placeholder. The value will be overwritten by a time stamp while True: # Break out of the inner while-loop only when the table has been updated sense.set_pixels(LED.threeDots('green', 'G')) stayInLoop = True key = { 'forum' : '1', 'subject' : 'PC1' } while stayInLoop: try: stayInLoop, timeStamp = table.compareValues(key, 'timeStamp', oldSizeTime, True) # Sleep because pinging AWS is costs $$$ time.sleep(sleepTime) except KeyboardInterrupt: print("Shutting down...") sense.set_pixels(LED.pluses('black')) sys.exit() oldSizeTime = timeStamp numDataPoints = table.getItem(key)['sampleSize'] numFeatures = 3 # Listen for incoming bluetooth data on port 1 sense.set_pixels(LED.arrowReceive('orange', 'black')) timeOne = time.time() btTime, dataFromBT = BT.listenOnBluetooth(1) timeTwo = time.time() # Edit: timeTwo - timeOne != btTime since some time is spent waiting on the sockets # Make the LEDs show the computation state sense.set_pixels(LED.diamond('blue')) # Transform the received bluetooth data to numpy arrays targetMatrix, designMatrix = bluetoothDataToNPArrays(dataFromBT, numDataPoints, numFeatures) # Aggregate the bluetooth data, with data collected from the Gateway Pi sense.set_pixels(LED.pluses('green')) targetMatrix, designMatrix = collectData(targetMatrix, designMatrix, numDataPoints) # Signify the computation state sense.set_pixels(LED.diamond('blue')) # Calculate how many sensors were used. # Number of received readings divided by the number of readings per sensor. numSensors = targetMatrix.shape[0] / numDataPoints # Calculate the features if the gateway has permission to do so if calculateFeatures[tableLetter]: features = gradientDescent(targetMatrix, designMatrix, numFeatures, numDataPoints) timeThree = time.time() compTime = timeThree - timeTwo # Upload data to DynamoDB sense.set_pixels(LED.arrowSend('blue', 'black')) if calculateFeatures[tableLetter]: uploadTime = uploadToDB(tableLetter, features, btTime, compTime, numSensors) else: # Make sure that targetMatrix and designMatrix get read in the correct order uploadTime = uploadToDB(tableLetter, [targetMatrix, designMatrix], btTime, compTime, numSensors) # Reset the state of the LED sense.set_pixels(LED.xCross('red'))
def uploadToDB(tableLetter, data, btTime, compTime, numSensors): """ Uploads the features and the latencies to DynamoDB Param(s): (char) Denotes the table, e.g. A, B (numpy array) Features calculated from the dataset (int) Time taken to load bluetooth data in seconds (int) Time taken to compute the features in seconds Returns an int showing the time taken to upload to DynamoDB in seconds """ startTime = time.time() table = Table('sensingdata_' + tableLetter) room = 'roomA' sensor = 'sensor' + tableLetter # Prepare the upload payload item = table.getItem({ 'forum' : room, 'subject' : sensor }) # If the Gateway was designated to calculate features... if calculateFeatures[tableLetter]: item['feature_A'] = Decimal(str(float(data[0][0]))) item['feature_B'] = Decimal(str(float(data[1][0]))) item['feature_C'] = Decimal(str(float(data[2][0]))) sizeOfDataInBytes = data.nbytes # Otherwise, the Gateway was meant to aggregate data without calculations else: targetMatrix = data[0] designMatrix = data[1] # Numpy indexes follow the [row][column] convention # ndarray.shape returns the dimensions as a (#OfRows, #OfColumns) # Both of our matrices have the same number of rows, hence one measure is enough numOfRows = designMatrix.shape[0] aggregatedItems = [] for i in range(numOfRows): currentItem = {} currentItem['X_1'] = Decimal(str(designMatrix[i][0])) # Time currentItem['X_2'] = Decimal(str(designMatrix[i][1])) # Pressure currentItem['X_3'] = Decimal(str(designMatrix[i][2])) # Humidity currentItem['Y'] = Decimal(str(targetMatrix[i][0])) # Temperature aggregatedItems.append(currentItem) sizeOfDataInBytes = designMatrix.nbytes + targetMatrix.nbytes item['aggregated_data'] = aggregatedItems # Upload this document to DynamoDB item['Comm_pi_pi'] = Decimal(str(btTime)) item['Compu_pi'] = Decimal(str(compTime)) item['data_bytes'] = Decimal(str(sizeOfDataInBytes)) table.addItem(item) # Attach a time stamp and the size of the file to the header item in DynamoDB endTime = time.time() uploadDuration = endTime - startTime item['Comm_pi_lambda'] = Decimal(str(uploadDuration)) item['timeStamp'] = Decimal(str(endTime)) item['number_of_sensors'] = Decimal(str(numSensors)) table.addItem(item) # Log the experiment run to DynamoDB, regardless of gateway type item.pop('aggregated_data', None) item.pop('forum', None) item.pop('subject', None) # newData = [] # labels = [] # for key in item.keys(): # labels.append(key) # newData.append(item[key]) record = table.getItem({'forum' : 'roomA', 'subject' : 'records'}) try: data = record['data'] except KeyError: record['data'] = [] data.append(item) # record['data_labels'] = labels table.addItem(record) #print(" ".join(["Uploaded", sizeOfDataInBytes, "bytes of data to DynamoDB"])) print("Uploaded ", str(sizeOfDataInBytes), " bytes of data to DynamoDB") return uploadDuration