def create_all_histograms(): create_datadir_if_not_exists(GRAPHPATH) keysize = "6" shardings = [False, True] _number_shardings = len(shardings) middleware_latencies = np.zeros( (_number_shardings, 2, 3)) # the "two" is for the number of middlewares client_latencies = np.zeros( (_number_shardings, 2, 3, 3) ) # the "two" and "three" is for middlewares and clients respectively # Iterate through keysizes for jdx, sharding in enumerate(shardings): print("Sharding: ", sharding) print("Keysize is: ", keysize) # ONCE FOR THE MIDDLEWARE for repetition in range(3): for middleware in [1, 2]: # Get throughput and latency from the file middleware_filename = get_pattern_exp5_1_middleware( middleware=middleware, keysize=keysize, repetition=repetition, sharding=sharding) try: df = parse_log(BASEPATH + middleware_filename) tmp_latencies_mw = get_histogram_latency_log_dataframe_middleware( df).T # print("ATT") # print(jdx, idx, middleware, repetition) # print(type(jdx), type(idx), type(middleware), type(repetition)) middleware_latencies[ jdx, idx, :, middleware - 1, repetition] = get_avg_25_50_75_90_99_percentiles( tmp_latencies_mw) except Exception as e: # print("WRONG WITH THE FOLLOWING CONFIG: ", client_filename) print("WRONG WITH THE FOLLOWING CONFIG: ", middleware_filename) print(e) continue # ONCE FOR THE CLIENT for client_idx, client in enumerate( ['Client1', 'Client2', 'Client3']): for middleware in [1, 2]: # TODO: Do it for each individual middleware! # Trying to open and read the file! client_filename = get_pattern_exp5_1_client( keysize=keysize, middleware=middleware, repetition=repetition, client=client, sharding=sharding) try: _, tmp_latencies_client_get = read_client_histogram_as_dataframe( filepath=BASEPATH + client_filename) # print("At") # print(jdx, idx, middleware, client_idx, repetition) # print(type(jdx), type(idx), type(middleware), type(client_idx), type(repetition)) set_df, get_df = read_client_histogram_as_dataframe( tmp_latencies_client_get) # print("Indecies: ", jdx, idx, :, middleware - 1, client_idx, repetition) if np.isnan(out).any(): print("Out is: ", out) client_latencies[ jdx, idx, :, middleware - 1, client_idx, repetition] = client_latencies[jdx, idx, :, middleware - 1, client_idx, 0] else: client_latencies[jdx, idx, :, middleware - 1, client_idx, repetition] = out except Exception as e: # print("WRONG WITH THE FOLLOWING CONFIG: ", client_filename) print("WRONG WITH THE FOLLOWING CONFIG: ", middleware_filename) print(e) assert False continue # Normalizing the graphs to the appropriate shapes by taking means (and calculating stddevs) mean_client_latencies = np.mean(client_latencies, axis=1, keepdims=True) mean_client_latencies = np.mean(mean_client_latencies, axis=2, keepdims=True) mean_middleware_latencies = np.mean(middleware_latencies, axis=1, keepdims=True) mean_client_latencies = mean_client_latencies.squeeze() mean_middleware_latencies = mean_middleware_latencies.squeeze() print("Squeezed") print(mean_client_latencies.shape) print(mean_middleware_latencies.shape) client_means = np.mean(mean_client_latencies[jdx, :, :, :], axis=2) client_stddevs = np.std(mean_client_latencies[jdx, :, :, :], axis=2) mw_means = np.mean(mean_middleware_latencies[jdx, :, :, :], axis=2) mw_stddev = np.std(mean_middleware_latencies[jdx, :, :, :], axis=2) print(mw_means) print(client_means) print("Means and Stddev") print(client_means.shape) print(client_stddevs.shape) print(mw_means.shape) print(mw_stddev.shape) create_multiple_histogram_plot( keys=multikeys, means=client_means, stddevs=client_stddevs, filepath=GRAPHPATH + "exp5_1_client_percentile_plots_sharded_{}".format(sharding)) create_multiple_histogram_plot( keys=multikeys, means=mw_means, stddevs=mw_stddev, filepath=GRAPHPATH + "exp5_1_mw_percentile_plots_sharded_{}".format(sharding))
def create_queue_barplots(): create_datadir_if_not_exists(GRAPHPATH) middleware_threads = [(2**x) for x in range(3, 7)] virtual_clients = [(2**x) for x in range(0, 6)] total_virtual_clients = len([(2**x) for x in range(0, 6)]) total_middleware_threads = len([(2**x) for x in range(3, 7)]) # The following are the different values we're going to plot: # time_labels = [ 'Time to Enqueue', 'Time in Queue', 'Time Queue to Server', 'Time at Server', 'Time Server to Client' ] print("Time labels have shape: ", len(time_labels)) for write in [0, 1]: for _vc in range(0, 6): vc = (2**_vc) middleware_latencies = np.zeros( (total_middleware_threads, len(time_labels), 1, 3)) # 3 repetitions, 2 middlewares for _mt, mt in enumerate(middleware_threads): # ONCE FOR THE MIDDLEWARE for repetition in range(3): for middleware in [1]: # Get throughput and latency from the file middleware_filename = get_pattern_exp3_1_middleware( virtualclientthreads=vc, middlewarethreads=mt, repetition=repetition, write=write, ) try: df = parse_log(BASEPATH + middleware_filename) middleware_latencies[ _mt, :, middleware - 1, repetition] = get_average_queue_components(df) # print("item is: ", middleware_latencies[_mt, :, middleware - 1, repetition]) # print(middleware_latencies) except Exception as e: # print("WRONG WITH THE FOLLOWING CONFIG: ", client_filename) print("WRONG WITH THE FOLLOWING CONFIG: ", middleware_filename) print(e) continue # Normalizing the graphs to the appropriate shapes by taking means (and calculating stddevs) mean_middleware_latencies = np.mean(middleware_latencies, axis=2, keepdims=True) stddev_middleware_latencies = np.std(mean_middleware_latencies, axis=3, keepdims=True) mean_middleware_latencies = np.mean(mean_middleware_latencies, axis=3, keepdims=True) mw_means = mean_middleware_latencies.squeeze().T mw_stddev = stddev_middleware_latencies.squeeze().T print("Means and Stddev") print(mw_means.shape) print(mw_stddev.shape) print(len(time_labels)) print(time_labels) print(mw_means) create_multiple_histogram_plot( keys=middleware_threads, means=mw_means.T, stddevs=mw_stddev.T, filepath=GRAPHPATH + "exp3_1_mw_percentile_plots_writes_{}__vc_{}".format( write, vc), is_queue=True)