def create_all_histograms():
create_datadir_if_not_exists(GRAPHPATH)
keysize = "6"
shardings = [False, True]
_number_shardings = len(shardings)
middleware_latencies = np.zeros(
(_number_shardings, 2,
3)) # the "two" is for the number of middlewares
client_latencies = np.zeros(
(_number_shardings, 2, 3, 3)
) # the "two" and "three" is for middlewares and clients respectively
# Iterate through keysizes
for jdx, sharding in enumerate(shardings):
print("Sharding: ", sharding)
print("Keysize is: ", keysize)
# ONCE FOR THE MIDDLEWARE
for repetition in range(3):
for middleware in [1, 2]:
# Get throughput and latency from the file
middleware_filename = get_pattern_exp5_1_middleware(
middleware=middleware,
keysize=keysize,
repetition=repetition,
sharding=sharding)
try:
df = parse_log(BASEPATH + middleware_filename)
tmp_latencies_mw = get_histogram_latency_log_dataframe_middleware(
df).T
# print("ATT")
# print(jdx, idx, middleware, repetition)
# print(type(jdx), type(idx), type(middleware), type(repetition))
middleware_latencies[
jdx, idx, :, middleware - 1,
repetition] = get_avg_25_50_75_90_99_percentiles(
tmp_latencies_mw)
except Exception as e:
# print("WRONG WITH THE FOLLOWING CONFIG: ", client_filename)
print("WRONG WITH THE FOLLOWING CONFIG: ",
middleware_filename)
print(e)
continue
# ONCE FOR THE CLIENT
for client_idx, client in enumerate(
['Client1', 'Client2', 'Client3']):
for middleware in [1, 2]:
# TODO: Do it for each individual middleware!
# Trying to open and read the file!
client_filename = get_pattern_exp5_1_client(
keysize=keysize,
middleware=middleware,
repetition=repetition,
client=client,
sharding=sharding)
try:
_, tmp_latencies_client_get = read_client_histogram_as_dataframe(
filepath=BASEPATH + client_filename)
# print("At")
# print(jdx, idx, middleware, client_idx, repetition)
# print(type(jdx), type(idx), type(middleware), type(client_idx), type(repetition))
set_df, get_df = read_client_histogram_as_dataframe(
tmp_latencies_client_get)
# print("Indecies: ", jdx, idx, :, middleware - 1, client_idx, repetition)
if np.isnan(out).any():
print("Out is: ", out)
client_latencies[
jdx, idx, :, middleware - 1, client_idx,
repetition] = client_latencies[jdx, idx, :,
middleware - 1,
client_idx, 0]
else:
client_latencies[jdx, idx, :, middleware - 1,
client_idx, repetition] = out
except Exception as e:
# print("WRONG WITH THE FOLLOWING CONFIG: ", client_filename)
print("WRONG WITH THE FOLLOWING CONFIG: ",
middleware_filename)
print(e)
assert False
continue
# Normalizing the graphs to the appropriate shapes by taking means (and calculating stddevs)
mean_client_latencies = np.mean(client_latencies,
axis=1,
keepdims=True)
mean_client_latencies = np.mean(mean_client_latencies,
axis=2,
keepdims=True)
mean_middleware_latencies = np.mean(middleware_latencies,
axis=1,
keepdims=True)
mean_client_latencies = mean_client_latencies.squeeze()
mean_middleware_latencies = mean_middleware_latencies.squeeze()
print("Squeezed")
print(mean_client_latencies.shape)
print(mean_middleware_latencies.shape)
client_means = np.mean(mean_client_latencies[jdx, :, :, :], axis=2)
client_stddevs = np.std(mean_client_latencies[jdx, :, :, :], axis=2)
mw_means = np.mean(mean_middleware_latencies[jdx, :, :, :], axis=2)
mw_stddev = np.std(mean_middleware_latencies[jdx, :, :, :], axis=2)
print(mw_means)
print(client_means)
print("Means and Stddev")
print(client_means.shape)
print(client_stddevs.shape)
print(mw_means.shape)
print(mw_stddev.shape)
create_multiple_histogram_plot(
keys=multikeys,
means=client_means,
stddevs=client_stddevs,
filepath=GRAPHPATH +
"exp5_1_client_percentile_plots_sharded_{}".format(sharding))
create_multiple_histogram_plot(
keys=multikeys,
means=mw_means,
stddevs=mw_stddev,
filepath=GRAPHPATH +
"exp5_1_mw_percentile_plots_sharded_{}".format(sharding))
def create_queue_barplots():
create_datadir_if_not_exists(GRAPHPATH)
middleware_threads = [(2**x) for x in range(3, 7)]
virtual_clients = [(2**x) for x in range(0, 6)]
total_virtual_clients = len([(2**x) for x in range(0, 6)])
total_middleware_threads = len([(2**x) for x in range(3, 7)])
# The following are the different values we're going to plot:
#
time_labels = [
'Time to Enqueue', 'Time in Queue', 'Time Queue to Server',
'Time at Server', 'Time Server to Client'
]
print("Time labels have shape: ", len(time_labels))
for write in [0, 1]:
for _vc in range(0, 6):
vc = (2**_vc)
middleware_latencies = np.zeros(
(total_middleware_threads, len(time_labels), 1,
3)) # 3 repetitions, 2 middlewares
for _mt, mt in enumerate(middleware_threads):
# ONCE FOR THE MIDDLEWARE
for repetition in range(3):
for middleware in [1]:
# Get throughput and latency from the file
middleware_filename = get_pattern_exp3_1_middleware(
virtualclientthreads=vc,
middlewarethreads=mt,
repetition=repetition,
write=write,
)
try:
df = parse_log(BASEPATH + middleware_filename)
middleware_latencies[
_mt, :, middleware - 1,
repetition] = get_average_queue_components(df)
# print("item is: ", middleware_latencies[_mt, :, middleware - 1, repetition])
# print(middleware_latencies)
except Exception as e:
# print("WRONG WITH THE FOLLOWING CONFIG: ", client_filename)
print("WRONG WITH THE FOLLOWING CONFIG: ",
middleware_filename)
print(e)
continue
# Normalizing the graphs to the appropriate shapes by taking means (and calculating stddevs)
mean_middleware_latencies = np.mean(middleware_latencies,
axis=2,
keepdims=True)
stddev_middleware_latencies = np.std(mean_middleware_latencies,
axis=3,
keepdims=True)
mean_middleware_latencies = np.mean(mean_middleware_latencies,
axis=3,
keepdims=True)
mw_means = mean_middleware_latencies.squeeze().T
mw_stddev = stddev_middleware_latencies.squeeze().T
print("Means and Stddev")
print(mw_means.shape)
print(mw_stddev.shape)
print(len(time_labels))
print(time_labels)
print(mw_means)
create_multiple_histogram_plot(
keys=middleware_threads,
means=mw_means.T,
stddevs=mw_stddev.T,
filepath=GRAPHPATH +
"exp3_1_mw_percentile_plots_writes_{}__vc_{}".format(
write, vc),
is_queue=True)