def bytes_graph():
data = {}
for fn in glob.glob(sys.argv[1] + '/*.counters.txt'):
key = [key for n, key in fn_keys.items() if n in fn][0]
c, p, _ = eval(open(fn).read())
c = sum([int(b.split('\n')[-1]) * bytes_units for b in c])
p = sum([int(b.split('\n')[-1]) * bytes_units for b in p])
data[key] = p, c
y = [data[t] for t in types]
x = np.array([[0, 1] for i in xrange(len(y))])
options = DotMap()
options.plot_type = 'BAR'
options.bar_labels.show = False
options.legend.options.labels = [
'ToR VOQ', 'VOQ + Resize', 'VOQ + reTCP', 'ADU', 'ADU + Resize',
'ADU + reTCP', 'Fixed Sched.', 'Packet (10G)', 'Packet (20G)',
'Packet (40G)', 'Packet (80G)'
]
options.legend.options.fontsize = 12
options.legend.options.ncol = 4
options.series.color_groups = [0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 3]
options.y.limits = [0, 85]
options.y.label_offset = [-.07, -.11]
options.x.ticks.major.show = False
options.x.ticks.major.labels = DotMap(text=['Packet', 'Circuit'])
options.y.ticks.major.labels = DotMap(locations=[0, 15, 30, 45, 60])
options.output_fn = 'graphs/utilization.pdf'
options.x.label.xlabel = 'Which switch'
options.y.label.ylabel = 'Bytes sent (GB)'
plot(x, y, options)
def plot_lat_vs_util(all_tpts,
all_lats,
fln,
ylb,
num_racks,
ylm=None,
odr=path.join(PROGDIR, "graphs"),
lbs=23):
# Calculate utilization.
all_utls = [[
get_util_for_tpt(tpt_Gbps_c, num_racks) for tpt_Gbps_c in tpts_Gbps_c
] for tpts_Gbps_c in all_tpts]
options = dotmap.DotMap()
options.legend.options.fontsize = lbs
options.legend.options.labels = [
"Static buffers (vary size)", "Dynamic buffers (vary $\\tau$)",
"Dynamic buffers + reTCP (vary $\\tau$)"
][:len(all_tpts)]
options.output_fn = path.join(odr, "{}.pdf".format(fln))
options.plot_type = "LINE"
options.series_options = [
dotmap.DotMap(marker="o", markersize=10, linewidth=5)
for _ in xrange(len(all_utls))
]
options.x.label.xlabel = "Average circuit utilization (%)"
options.y.label.ylabel = "{} latency ($\mu$s)".format(ylb)
ylm = (ylm if ylm is not None else 1.2 *
max([max(line) for line in all_lats]))
options.y.limits = [0, ylm]
simpleplotlib.plot(all_utls, all_lats, options)
def plot_lat(keys,
latencies,
fln,
ylb,
ylm=None,
xlr=0,
xtk_locs=None,
odr=path.join(PROGDIR, "graphs"),
flt=lambda key: True):
# Sort the data based on the x-values (keys).
keys, latencies = zip(
*sorted(zip(keys, latencies), key=lambda p: int(p[0])))
# Filter.
keys, latencies = zip(*[(k, l) for k, l in zip(keys, latencies) if flt(k)])
x = [keys for _ in xrange(len(latencies[0]))]
y = zip(*latencies)
print("")
print("raw latency data for: {}".format(fln))
print("{}:".format(ylb.strip("\n")))
print(" all: {}".format(", ".join(
["({}: {})".format(a, b) for a, b in zip(x[0], y[0])])))
print(" circuit: {}".format(". ".join(
["({}: {})".format(a, b) for a, b in zip(x[1], y[1])])))
print(" packet: {}".format(", ".join(
["({}: {})".format(a, b) for a, b in zip(x[2], y[2])])))
print("")
options = dotmap.DotMap()
options.legend.options.loc = "upper left"
options.legend.options.labels = ["both NWs", "circuit NW", "packet NW"]
options.legend.options.fontsize = 20
options.output_fn = path.join(odr, "{}.pdf".format(fln))
options.plot_type = "LINE"
options.series_options = [
dotmap.DotMap(marker="o", markersize=10, linewidth=5)
for _ in xrange(len(x))
]
options.x.label.fontsize = options.y.label.fontsize = 20
options.x.label.xlabel = "Buffer size (packets)" if "static" in fln \
else "Early buffer resizing ($\mu$s)"
options.x.ticks.major.options.labelsize = \
options.y.ticks.major.options.labelsize = 20
options.x.ticks.major.labels = \
dotmap.DotMap(locations=[4, 8, 16, 32, 64, 128]) \
if "static" in fln else dotmap.DotMap(locations=keys)
if xtk_locs is not None:
options.y.ticks.major.labels = dotmap.DotMap(locations=xtk_locs)
options.x.ticks.major.labels.options.rotation = xlr
options.x.ticks.major.labels.options.rotation_mode = "anchor"
options.x.ticks.major.labels.options.horizontalalignment = \
"center" if xlr == 0 else "right"
options.y.label.ylabel = "{} latency ($\mu$s)".format(ylb)
ylm = ylm if ylm is not None else 1.2 * max([max(line) for line in y])
options.y.limits = [0, ylm]
simpleplotlib.plot(x, y, options)
def graph_big_tp(data):
x = [sorted([p[0] for p in d if p[2] > SMALL_FLOW_MAX_SIZE]) for d in data]
y = [[float(j) / (len(x[i]) - 1) * 100 for j in xrange(len(x[i]))]
for i in xrange(len(x))]
options = get_default_plot_options(x, y)
options.x.limits = [0, 10]
options.output_fn = 'graphs/elephant_tp_cdf.pdf'
options.x.label.xlabel = 'Throughput (Gbps)'
plot(x, y, options)
def graph_big_durs(data):
x = [
sorted([p[1] * dur_units for p in d if p[2] > SMALL_FLOW_MAX_SIZE])
for d in data
]
y = [[float(j) / (len(x[i]) - 1) * 100 for j in xrange(len(x[i]))]
for i in xrange(len(x))]
options = get_default_plot_options(x, y)
options.output_fn = 'graphs/elephant_fct_cdf.pdf'
options.x.label.xlabel = 'Flow-completion time (ms)'
plot(x, y, options)
def graph_tail(data):
x = np.array([[0] for i in xrange(len(data))])
y = [np.percentile(d, 99) for d in data]
options = get_default_plot_options(x, y)
options.y.limits = [0, 1500]
options.output_fn = 'graphs/hdfs_99th.pdf'
options.y.label.ylabel = '99th percent. writes (ms)'
options.y.ticks.major.show = False
del options.legend.options.ncol
del options.legend.order
plot(x, y, options)
def graph_throughput(data):
x = np.array([[0] for i in xrange(len(data))])
y = data
options = get_default_plot_options(x, y)
options.horizontal_lines.lines = [80 * 8 + 10 * 8]
options.legend.options.fontsize = 18
options.y.label_offset = [-0.01, -.13]
options.y.limits = [0, 1100]
options.output_fn = 'graphs/hdfs_throughput.pdf'
options.y.label.ylabel = 'Agg. tput. (Gbps)'
options.y.ticks.major.show = False
plot(x, y, options)
def graph_wct(data):
x = data
y = [[float(j) / (len(x[i]) - 1) * 100 for j in xrange(len(x[i]))]
for i in xrange(len(x))]
options = get_default_plot_options(x, y)
options.plot_type = 'LINE'
options.legend.options.labels = ['HDFS', 'reHDFS']
options.series_options = [DotMap(linewidth=5) for i in range(len(x))]
options.output_fn = 'graphs/hdfs_writes_cdf.pdf'
options.x.label.xlabel = 'HDFS write completion time (ms)'
options.y.label.ylabel = 'CDF (%)'
del options.series.color_groups
del options.legend.options.ncol
del options.x.ticks.major.show
plot(x, y, options)
def plot_util_vs_latency(tpts, latencies, fln):
x = [[
min(
j /
(0.9 * 1. /
(python_config.NUM_RACKS - 1) * python_config.CIRCUIT_BW_Gbps) *
100, 100.0) for j in t
] for t in tpts]
y = [zip(*l)[0] for l in latencies]
options = dotmap.DotMap()
options.plot_type = "LINE"
options.legend.options.labels = [
"Static buffers (vary size)", "Dynamic buffers (vary $\\tau$)",
"reTCP", "reTCP + dynamic buffers (vary $\\tau$)"
]
options.legend.options.fontsize = 19
options.series_options = [
dotmap.DotMap(marker="o", markersize=10, linewidth=5)
for _ in xrange(len(x))
]
options.series_options[2].marker = "x"
options.series_options[2].s = 100
del options.series_options[2].markersize
options.series_options[2].zorder = 10
options.output_fn = \
path.join(PROGDIR, "graphs", "throughput_vs_latency99.pdf") \
if "99" in fln \
else path.join(PROGDIR, "graphs", "throughput_vs_latency.pdf")
options.x.label.xlabel = "Circuit utilization (%)"
options.y.label.ylabel = "99th percent. latency ($\mu$s)" if "99" in fln \
else "Median latency ($\mu$s)"
options.y.limits = [0, 1000] if "99" in fln else [0, 600]
options.y.ticks.major.labels = \
dotmap.DotMap(locations=[0, 200, 400, 600, 800, 1000]) \
if "99" in fln else \
dotmap.DotMap(locations=[0, 100, 200, 300, 400, 500, 600])
simpleplotlib.plot(x, y, options)
def bytes_graph():
data = {}
for fn in glob.glob(sys.argv[1] + '/*.counters.txt'):
key = 'reHDFS+' if 'reHDFS' in fn else 'HDFS+'
key += [k for n, k in fn_keys.items() if n in fn][0]
c, p, _ = eval(open(fn).read())
c = sum([int(b.split('\n')[-1]) * bytes_units for b in c])
p = sum([int(b.split('\n')[-1]) * bytes_units for b in p])
data[key] = p, c
y = [data[t] for t in types]
x = np.array([[0, 1] for i in xrange(len(y))])
options = get_default_plot_options(x, y)
options.bar_labels.show = False
options.legend.options.fontsize = 18
options.y.label_offset = [-.07, -.18]
options.y.limits = [0, 40]
options.x.ticks.major.labels = DotMap(text=['Packet', 'Circuit'])
options.y.ticks.major.labels = DotMap(locations=[0, 5, 10, 15, 20, 25])
options.output_fn = 'graphs/hdfs_utilization.pdf'
options.x.label.xlabel = 'Which switch'
options.y.label.ylabel = 'Bytes sent (GB)'
plot(x, y, options)
def plot_seq(data,
fln,
odr=path.join(PROGDIR, "..", "graphs"),
ins=None,
flt=lambda idx, label: True,
order=None,
xlm=None,
ylm=None,
chunk_mode=None,
voq_agg=False):
assert not voq_agg or chunk_mode is None, \
"voq_agg=True requires chunk_mode=None!"
assert not voq_agg or ins is None, "voq_agg=True requires ins=None!"
plot_voqs = False
if chunk_mode is None:
# Plot aggregate metrics for all chunks from all flows in each
# experiment.
seq_ys = data["seqs"]
seq_xs = [xrange(len(seq_ys[idx])) for idx in xrange(len(seq_ys))]
keys = data["keys"]
if voq_agg:
# First, create a list of x values for each list of VOQ results.
# Then, split apart the seq_xs and the seq_ys.
voq_xs, voq_ys = zip(*[(xrange(len(voq_ys)), voq_ys)
for voq_ys in data["voqs"]])
plot_voqs = True
else:
# Include the "optimal" and "packet only" lines.
lines = [([x for x in xrange(len(seq_ys))], seq_ys)
for seq_ys in data["seqs"][0:2]]
if chunk_mode == "best":
# Plot the best chunk from any flow in each experiment.
lines.extend(data["chunks_best"].values())
keys = data["keys"]
else:
# Plot a specific chunk from all flows in a single experiment. Do
# not use a legend because each line will correspond to a separate
# flow instead of a separate experiment.
exps_data = data["chunks_selected_chunk{}".format(chunk_mode)]
num_exps = len(exps_data)
assert num_exps == 1, \
("When using chunk_mode={}, there should be exactly one "
"experiment, but there actually are: {}").format(
chunk_mode, num_exps)
# Extracts a flow's src ID.
get_src_id = lambda f: int(f.split(".")[-1])
# Sort the results by the flow src ID, then split the flows and
# their results. Do ".values()[0]" because we assume that, if we are
# here, then there is only a single experiment (see above).
flws, results_by_chunk = zip(
*sorted(exps_data.values()[0].items(),
key=lambda p: get_src_id(p[0][0])))
# Combine the separate VOQ length results. This assumes that all of
# the flows were sent from the same rack to the same rack. Start by
# extracting the xs and VOQ lengths for each flow and turning them
# into single-point pairs.
voqs = [
zip(voq_xs, voq_ys)
for _, _, voq_xs, voq_ys, _ in results_by_chunk
]
# Flatten the per-flow VOQ length results into one master list.
voqs = [pair for voqs_flw in voqs for pair in voqs_flw]
# Sort the VOQ length results by their x-values.
voqs = sorted(voqs, key=lambda val: val[0])
# Split the voq_xs and voq_ys into separate lists.
voq_xs, voq_ys = zip(*voqs)
# Since there is only a single experiment (see above) yet the rest
# of the code is generalized to multiple experiments, wrap the
# single experiment results in a list.
voq_xs = [voq_xs]
voq_ys = [voq_ys]
plot_voqs = True
# Remove the VOQ lengths from the results, then add them to the list
# of all lines (which currently includes the "optimal" and "packet
# only" lines).
lines.extend([(seq_xs, seq_ys)
for seq_xs, seq_ys, _, _, _ in results_by_chunk])
# Convert each of the flows to a src ID.
keys = (data["keys"][0:2] +
["flow {}".format(get_src_id(flw[0])) for flw in flws])
seq_xs, seq_ys = zip(*lines)
if plot_voqs:
# Convert from tuples to lists to enable insertion.
voq_xs = list(voq_xs)
voq_ys = list(voq_ys)
# Insert empty lists for the "optimal" and "packet only" lines. These
# will be removed later.
voq_xs.insert(0, None)
voq_xs.insert(0, None)
voq_ys.insert(0, None)
voq_ys.insert(0, None)
else:
voq_xs = [None for _ in seq_xs]
voq_ys = [None for _ in seq_ys]
# Format the legend labels.
lls = []
for k in keys:
try:
k_int = int(k)
if "static" in fln:
lls += ["%s packets" % k_int]
else:
lls += ["%s $\mu$s" % k_int]
except ValueError:
lls += [k]
options = dotmap.DotMap()
options.plot_type = "LINE" if chunk_mode is None else "SCATTER"
options.legend.options.loc = "center right"
options.legend.options.labels = lls
options.legend.options.fontsize = 18
options.output_fn = path.join(odr, "{}.pdf".format(fln))
if xlm is not None:
options.x.limits = xlm
if ylm is not None:
options.y.limits = ylm
options.x.label.xlabel = "Time ($\mu$s)"
options.y.label.ylabel = "Expected TCP sequence\nnumber ($\\times$1000)"
options.x.label.fontsize = options.y.label.fontsize = 18
options.x.ticks.major.options.labelsize = \
options.y.ticks.major.options.labelsize = 18
options.x.axis.show = options.y.axis.show = True
options.x.axis.color = options.y.axis.color = "black"
circuit_bounds = data["circuit_bounds"]
options.vertical_lines.lines = circuit_bounds
shaded = []
for i in xrange(0, len(circuit_bounds), 2):
shaded.append((circuit_bounds[i], circuit_bounds[i + 1]))
options.vertical_shaded.limits = shaded
options.vertical_shaded.options.alpha = 0.1
options.vertical_shaded.options.color = "blue"
if ins is not None:
# Enable an inset.
options.inset.show = True
options.inset.options.zoom_level = 4
options.inset.options.corners = (2, 3)
options.inset.options.location = "center right"
options.inset.options.marker.options.color = "black"
xlm_ins, ylm_ins = ins
options.inset.options.x.limits = xlm_ins
options.inset.options.y.limits = ylm_ins
if flt is not None:
# Pick only the lines that we want.
seq_xs, seq_ys, voq_xs, voq_ys, options.legend.options.labels = zip(
*[(sx, sy, vx, vy, l) for (idx, (sx, sy, vx, vy, l)) in enumerate(
zip(seq_xs, seq_ys, voq_xs, voq_ys,
options.legend.options.labels)) if flt(idx, l)])
if order is not None:
# Reorder the lines.
real_seq_xs = []
real_seq_ys = []
real_voq_xs = []
real_voq_ys = []
real_ls = []
for item in order:
idx = 0
found = False
for possibility in options.legend.options.labels:
if possibility.startswith(item):
found = True
break
idx += 1
if found:
real_seq_xs.append(seq_xs[idx])
real_seq_ys.append(seq_ys[idx])
real_voq_xs.append(voq_xs[idx])
real_voq_ys.append(voq_ys[idx])
real_ls.append(options.legend.options.labels[idx])
seq_xs = real_seq_xs
seq_ys = real_seq_ys
voq_xs = real_voq_xs
voq_ys = real_voq_ys
options.legend.options.labels = real_ls
# Set series options. Do this after filtering so that we have an accurate
# count of the number of series.
if chunk_mode is None:
options.series_options = [
dotmap.DotMap(color="C{}".format(idx), linewidth=2)
for idx in xrange(len(seq_xs))
]
else:
options.series_options = [
dotmap.DotMap(s=6, edgecolors="none") for _ in xrange(len(seq_xs))
]
# Set legend options. Do this after filtering so that we have an accurate
# count of the number of series. Use 1 column if there are 10 or fewer
# lines, otherwise use 2 columns.
options.legend.options.ncol, options.legend.options.bbox_to_anchor = \
(1, (1.4, 0.5)) if len(seq_xs) <= 10 else (2, (1.65, 0.5))
if plot_voqs:
# If we are going to plot a second y-axis, then shift the legend to the
# right.
offset_x, offset_y = options.legend.options.bbox_to_anchor
options.legend.options.bbox_to_anchor = (offset_x + 0.1, offset_y)
simpleplotlib.plot(seq_xs, seq_ys, options)
if plot_voqs:
# Plot the VOQ length on a second y-axis. Modify the active figure
# instance to include a totally separate line on a second y-axis. We
# cannot use simpleplotlib's built-in y2 functionality because we are
# not plotting a y2 line for each y line.
options2 = simpleplotlib.default_options.copy()
options2.output_fn = options.output_fn
options2.plot_type = "LINE"
options2.series2_options = [dotmap.DotMap() for _ in voq_xs]
if voq_agg:
# Configure the VOQ line to be dashed and the same color as the
# corresponding sequence number lines.
for idx in xrange(len(options2.series2_options)):
options2.series2_options[idx].color = \
options.series_options[idx].color
options2.series2_options[idx].linestyle = "dashed"
else:
# We are in chunk_mode==#, so there is only a single
# experiment.
options2.series2_options[-1].linewidth = 1
options2.series2_options[-1].color = "black"
options2.series2_options[-1].alpha = 0.5
options2.series2_options[-1].marker = "o"
options2.series2_options[-1].markeredgecolor = "none"
options2.series2_options[-1].markersize = 3
options2.x.limits = options.x.limits
options2.x.margin = options2.y2.margin = \
simpleplotlib.default_options.x.margin
options2.y2.axis.color = options.y.axis.color
options2.y2.label.fontsize = \
options.y.label.fontsize
options2.y2.label.ylabel = "VOQ length (packets)"
options2.y2.ticks.major.options.labelsize = \
options.y.ticks.major.options.labelsize
# As a final step, remove the VOQ lines corresponding to "optimal" and
# "packet only", which are None.
voq_xs, voq_ys, options2.series2_options = zip(
*[(vx, vy, o)
for vx, vy, o in zip(voq_xs, voq_ys, options2.series2_options)
if vx is not None])
ax2 = pyplot.gca().twinx()
simpleplotlib.plot_data(ax2, voq_xs, voq_ys, options2,
options2.series2_options)
simpleplotlib.apply_options_to_axis(ax2.xaxis, voq_xs, options2.x)
simpleplotlib.apply_options_to_axis(ax2.yaxis, voq_ys, options2.y2)
# Overwrite the original graph.
pyplot.savefig(options2["output_fn"],
bbox_inches="tight",
pad_inches=0)
def main():
assert len(sys.argv) == 2, "Expected one argument: experiment data file"
edf = sys.argv[1]
if not path.isfile(edf):
print("The first argument must be a file, but is: {}".format(edf))
sys.exit(-1)
# Specify and create the output directory.
odr = path.join(PROGDIR, "graphs", "nsdi2020")
if path.exists(odr):
if not path.isdir(odr):
print("Output directory exists and is a file: {}".format(odr))
sys.exit(-1)
else:
os.makedirs(odr)
# { num flows : { q len : list of results } }
data = defaultdict(lambda: defaultdict(dict))
with open(edf) as f:
for line in f:
# date, swt_us, num_flows, q_len_p, delay_2way_ns, fct_us, num_rtx, \
# num_rto, num_syns, avt_rtt_us = line.strip().split()
date, num_flows, swt_us, q_len_p, delay_2way_ns, fct_us, num_rtx, \
num_rto, num_syns, avt_rtt_us = line.strip().split()
# Convert the queue length from being in terms of 1500-byte packets
# to being in terms of 9000-bytes packets.
q_len_p = float(q_len_p) * (float(MSS_B_EXP) / float(MSS_B_TARGET))
swt_us = float(swt_us)
fct_s = float(fct_us) / 1e6
record = (date, float(delay_2way_ns), fct_s, float(num_rtx),
float(num_rto), float(num_syns), float(avt_rtt_us))
# Store this record if it is the smallest FCT for this q len.
if ((swt_us in data[num_flows][q_len_p]
and fct_s < data[num_flows][q_len_p][swt_us][2])
or swt_us not in data[num_flows][q_len_p]):
data[num_flows][q_len_p][swt_us] = record
for num_flows, q_len_results in data.items():
# for q_len_p, swt_us_results in q_len_results.items():
# for swt_us, val in swt_us_results.items():
# print val
# assert False
# { q len : list of pairs (switch time, FCT) }.items()
lines = {
q_len_p:
[(swt_us, fct_s)
for swt_us, (_, _, fct_s, _, _, _, _) in swt_us_results.items()]
for q_len_p, swt_us_results in q_len_results.items()
}.items()
# Pick only the lines we want.
lines = [(q_len_p, res) for q_len_p, res in lines
if q_len_p in CHOSEN_QUEUE_CAPS]
# Sort the datapoints based on their x-valies.
lines = sorted(lines, key=lambda a: a[0])
# lbls: list of q lens
# lines: list of lists of pairs of (switch time, FCT)
lbls, lines = zip(*lines)
# xs: list of lists of switch times
# ys: list of lists of FCTs
xs = [[p[0] for p in sorted(val, key=lambda a: a[0])] for val in lines]
ys = [[p[1] for p in sorted(val, key=lambda a: a[0])] for val in lines]
options = DotMap()
options.plot_type = "LINE"
if len(CHOSEN_QUEUE_CAPS) > 1:
options.legend.options.labels = [
"{} packets".format(int(round(lbl))) for lbl in lbls
]
options.legend.options.fontsize = 18
options.legend.options.ncol = 1
options.series_options = [
DotMap(linewidth=2, marker="o") for _ in range(len(xs))
]
options.output_fn = path.join(odr,
"sim-{}-flows.pdf".format(num_flows))
options.x.label.xlabel = "Circuit uptime ($\mu$s)"
options.y.label.ylabel = "Flow completion time (s)"
options.x.label.fontsize = options.y.label.fontsize = 18
options.x.ticks.major.options.labelsize = \
options.y.ticks.major.options.labelsize = 18
options.x.log = True
options.x.axis.show = options.y.axis.show = True
options.x.axis.color = options.y.axis.color = "black"
options.x.axis.stretch = 1.35
# Flip the x-axis.
options.x.limits = (max([max(vals) for vals in xs]) * 1.5,
min([min(vals) for vals in xs]) * 0.5)
options.y.limits = (0, max([max(vals) for vals in ys]) + 4)
plot(xs, ys, options)
def plot_circuit_util(keys,
tpts_Gbps_c,
fln,
xlb,
num_racks,
odr=path.join(PROGDIR, "graphs"),
srt=True,
xlr=0,
lbs=23,
flt=lambda key: True,
order=None):
""" srt: sort, xlr: x label rotation (degrees), lbs: bar label size """
if srt:
# Sort the data based on the x-values (keys).
keys, tpts_Gbps_c = zip(*sorted(zip(keys, tpts_Gbps_c)))
# Filter.
keys, tpts_Gbps_c = zip(*[(key, tpt_Gbps_c)
for key, tpt_Gbps_c in zip(keys, tpts_Gbps_c)
if flt(key)])
# Convert circuit throughput into circuit utilization. The throughput values
# are averages over the entire experiment, including when the circuit
# network was not active: they were computed by dividing the number of bytes
# transported by the circuit network over the course the entire experiment
# by the length of the experiment. To calculate the average circuit
# utilization, we divide the achieved average throughput by the maximum
# possible average throughput. The maximum possible average throughput is
# equal to the circuit bandwidth multiplied by the fraction of the
# experiment during which the circuit network was in use. This is equal to
# the fraction of the schedule during which a circuit was assigned between
# the racks in question (1 / (num_racks - 1)) times the fraction of that
# time during which the circuit was up (reconfiguration time / day length =
# 0.9). Therefore:
#
# util = throughput = throughput * (num_racks - 1)
# ------------------------------- ----------------------------
# bandwidth * 1 * 0.9 0.9 * bandwidth
# -------------
# num_racks - 1
#
# Finally, we multiply by 100 to convert to a percent.
utls = [
tpt_Gbps_c * (num_racks - 1) / (0.9 * python_config.CIRCUIT_BW_Gbps) *
100 for tpt_Gbps_c in tpts_Gbps_c
]
if order is not None:
# Reorder the lines.
real_keys = []
real_utls = []
for item in order:
for idx, possibility in enumerate(keys):
if possibility == item:
real_keys.append(possibility)
real_utls.append(utls[idx])
break
keys = real_keys
utls = real_utls
print("")
print("raw util data for: {}".format(fln))
print("{}:".format(xlb))
print(" {}".format(", ".join(
["({}: {})".format(key, utl) for key, utl in zip(keys, utls)])))
print("")
options = dotmap.DotMap()
options.plot_type = "BAR"
options.legend.options.fontsize = 20
options.bar_labels.format_string = "%1.0f"
options.bar_labels.options.fontsize = lbs
options.output_fn = path.join(odr, "{}.pdf".format(fln))
options.y.limits = (0, 100)
options.x.label.fontsize = options.y.label.fontsize = 20
options.x.label.xlabel = xlb
options.y.label.ylabel = "Average circuit\nutilization (%)"
options.x.ticks.major.options.labelsize = \
options.y.ticks.major.options.labelsize = 20
options.x.ticks.major.labels = dotmap.DotMap(text=keys)
options.x.ticks.major.labels.options.rotation = xlr
options.x.ticks.major.labels.options.rotation_mode = "anchor"
options.x.ticks.major.labels.options.horizontalalignment = \
"center" if xlr == 0 else "right"
options.y.ticks.major.show = True
options.x.ticks.major.show = False
simpleplotlib.plot([np.arange(len(utls))], [utls], options)
def plot_circuit_util(keys,
tpts_Gbps_c,
fln,
xlb,
num_racks,
odr=path.join(PROGDIR, "graphs"),
srt=True,
xlr=0,
lbs=23,
flt=lambda key: True,
order=None):
""" srt: sort, xlr: x label rotation (degrees), lbs: bar label size """
if srt:
# Sort the data based on the x-values (keys).
keys, tpts_Gbps_c = zip(*sorted(zip(keys, tpts_Gbps_c)))
# Filter.
keys, tpts_Gbps_c = zip(*[(key, tpt_Gbps_c)
for key, tpt_Gbps_c in zip(keys, tpts_Gbps_c)
if flt(key)])
# Calculate utilization.
utls = [
get_util_for_tpt(tpt_Gbps_c, num_racks) for tpt_Gbps_c in tpts_Gbps_c
]
if order is not None:
# Reorder the lines.
real_keys = []
real_utls = []
for item in order:
for idx, possibility in enumerate(keys):
if possibility == item:
real_keys.append(possibility)
real_utls.append(utls[idx])
break
keys = real_keys
utls = real_utls
print("")
print("raw util data for: {}".format(fln))
print("{}:".format(xlb))
print(" {}".format(", ".join(
["({}: {})".format(key, utl) for key, utl in zip(keys, utls)])))
print("")
options = dotmap.DotMap()
options.plot_type = "BAR"
options.legend.options.fontsize = 20
options.bar_labels.format_string = "%1.0f"
options.bar_labels.options.fontsize = lbs
options.output_fn = path.join(odr, "{}.pdf".format(fln))
options.y.limits = (0, 100)
options.x.label.fontsize = options.y.label.fontsize = 20
options.x.label.xlabel = xlb
options.y.label.ylabel = "Average circuit\nutilization (%)"
options.x.ticks.major.options.labelsize = \
options.y.ticks.major.options.labelsize = 20
options.x.ticks.major.labels = dotmap.DotMap(text=keys)
options.x.ticks.major.labels.options.rotation = xlr
options.x.ticks.major.labels.options.rotation_mode = "anchor"
options.x.ticks.major.labels.options.horizontalalignment = \
"center" if xlr == 0 else "right"
options.y.ticks.major.show = True
options.x.ticks.major.show = False
simpleplotlib.plot([np.arange(len(utls))], [utls], options)
return sorted(data)
if __name__ == '__main__':
if not os.path.isdir(sys.argv[1]):
print 'first arg must be dir'
sys.exit(-1)
db = shelve.open(sys.argv[1] + '/delay_shelve.db')
db['data'] = get_data()
x = [zip(*db['data'])[0]]
y = [zip(*db['data'])[1]]
options = DotMap()
options.plot_type = 'LINE'
options.legend.options.fontsize = 19
options.series_options = [
DotMap(marker='o', markersize=10, linewidth=5) for i in range(len(x))
]
options.output_fn = 'graphs/delay_sensitivity_rtt.pdf'
options.x.label.xlabel = '# of RTTs in a day'
options.y.label.ylabel = 'Median lat. improve. (%)'
options.x.limits = [0, 19]
options.y.limits = [0, 55]
options.x.ticks.major.labels = DotMap(
locations=[0, 1, 2, 3, 4, 5, 6, 9, 18])
options.y.ticks.major.labels = DotMap(locations=[0, 10, 20, 30, 40, 50])
plot(x, y, options)
db.close()
