scale = 6000
#
# memory use at various lobsters scales
#
bfmtfn = lambda x: '%1.1fGB' % (x * 1e-9) if x >= 1e9 else '%1.0fMB' % (
x * 1e-6) if x >= 1e6 else '%1.10kB' % (
x * 1e-3) if x >= 1e3 else '%1.0fB' % x if x > 0 else "No overhead"
base = common.load('lobsters').query(
'until == 256 & scale == %d & metric == "sojourn"' % (scale))
base["opmem"] = base["opmem"] / (1024 * 1024 * 1024)
prune_limit = base.query(
'memlimit != 0 & durable == False').reset_index()['memlimit'].min()
print('Using %s memory limit as representative for lobsters' %
(common.bts(prune_limit * 1024 * 1024 * 1024)))
prune = base.query('durable == False & memlimit == %f' % (prune_limit))
full = base.query('partial == False & memlimit == 0')
fig, mem = plt.subplots()
xs = [
"Noria",
"Noria without partial",
]
xticks = [x for x in range(len(xs))]
#
# memory used
#
# first, find smallest supported memlimit at 1M
vote = common.load('vote', only_good=False)
data = vote.query('op == "all" & clients == 4 & write_every == 100')
data = data.query(
'until == 256 & distribution == "skewed" & metric == "sojourn"')
for load in [1000000, 250000]:
d = data.query('target == %d' % load)
fvmrss = d.query('partial == False')['vmrss'].item()
fopmem = d.query('partial == False')['opmem'].item()
low = d.query('memlimit != 0')
low = low.query('achieved >= 0.99 * target & p95 < 20')
low_mem = low.reset_index()['memlimit'].min()
print('Using %s memory limit as representative for vote @ %d' %
(common.bts(low_mem * 1024 * 1024 * 1024), load))
# bah -- floating point
low = low.query('memlimit == %.7f' % low_mem)
low_opmem = low['opmem'].item()
perc = 100.0 * low_opmem / fopmem
print('vote @ %d, limit %s: opmem / full opmem = %.1f%%' %
(load, common.bts(low_mem * 1024 * 1024 * 1024), perc))
print(' vmrss use = %.1f%%' %
(100.0 * low['vmrss'].item() / fvmrss))
with open(
os.path.join(os.path.dirname(__file__), '..', 'benchmarks', 'results',
'vote-formula', 'results.log'), 'r') as f:
data = pd.read_table(f)
% (plot_scale))
# estimate base table size
est = base.query('memlimit == %f' % 0.25)
no_dur = est.query('durable == False')['vmrss'].item()
dur = est.query('durable == True')['vmrss'].item()
delta = no_dur - dur
for limit in limits:
limit /= 1024.0
d = data.query('memlimit == %f' % limit).reset_index()
opmem = base.query('durable == False & memlimit == %f' %
(limit))['vmrss'].item()
lo.plot(d["latency"], d["pct"], color=colors[len(limits) - i - 1])
hi.plot(d["latency"],
d["pct"],
color=colors[len(limits) - i - 1],
label='%s + %s' % (common.bts(delta), common.bts(opmem - delta)))
i += 1
hi.set_ylim(75, 101)
hi.set_yticks([75, 90, 95, 100])
hi.set_ylabel("CDF [\\%]")
hi.set_xscale('log')
hi.set_xlim(4, 100)
hi.set_xticks([5, 10, 20, 50])
hi.set_xticklabels(["5ms", "10ms", "20ms", "50ms"])
hi.legend(loc='lower right', title='Base table + view VmRSS')
lo.set_ylim(0, 75)
lo.set_yticks([0, 25, 50, 75])
lo.set_xlim(1, 8)
lo.set_xticks([1, 2, 4, 6, 8])
base = vote.query('op == "all" & clients == 4 & write_every == 100 & until == 256 & distribution == "skewed" & metric == "sojourn"')
data = base.query('durable == False').sort_index().reset_index()
limits = data.groupby('memlimit').tail(1)
limits = [l for l in limits["memlimit"]]
limits = [448 / 1024.0, 384 / 1024.0, 320 / 1024.0, 256 / 1024.0]
dur_limit = 448 / 1024.0
colors = common.memlimit_colors(len(limits))
i = 0
dur = base.query('durable == True & target == 1000000 & memlimit == %f' % dur_limit)['vmrss'].item()
no_dur = base.query('durable == False & target == 1000000 & memlimit == %f' % dur_limit)['vmrss'].item()
delta = no_dur - dur
fopmem = data.query('memlimit == 0 & target == 1000000 & partial == False')['opmem'].item()
for limit in limits:
d = data.query('memlimit == %f' % limit).reset_index()
opmem = d.query('target == 1000000')["vmrss"].item()
ax.plot(d["achieved"], d["p95"], '.-', color = colors[len(limits) - i - 1], label = '%s + %s' % (common.bts(delta), common.bts(opmem - delta)))
i += 1
opmem = d.query('partial == True & target == 1000000')['opmem'].item()
print('vote @ 1M, limit %s: opmem / full opmem = %.1f%%' % (common.bts(limit * 1024 * 1024 * 1024) if limit != 0 else "none", 100.0 * opmem / fopmem))
ax.xaxis.set_major_formatter(common.kfmt)
ax.set_ylim(0, 50)
ax.legend(title = "VmRSS @ 1M/s")
ax.set_xlabel("Achieved throughput [requests per second]")
ax.set_ylabel("95-th \\%-ile latency [ms]")
fig.tight_layout()
plt.savefig("{}.pdf".format(sys.argv[1]), format="pdf")
# for thesis presentation backup slides:
# plt.savefig("{}.png".format(sys.argv[1]), format="png", dpi=256)
scale = 6000
#
# memory use at various lobsters scales
#
bfmtfn = lambda x: '%1.1fGB' % (x * 1e-9) if x >= 1e9 else '%1.0fMB' % (
x * 1e-6) if x >= 1e6 else '%1.10kB' % (
x * 1e-3) if x >= 1e3 else '%1.0fB' % x if x > 0 else "No overhead"
base = common.load('lobsters').query(
'until == 256 & scale == %d & metric == "sojourn"' % (scale))
base["vmrss"] = base["vmrss"] / (1024 * 1024 * 1024)
prune_limit = base.query(
'memlimit != 0 & durable == False').reset_index()['memlimit'].min()
print('Using %s memory limit as representative for lobsters' %
(common.bts(prune_limit * 1024 * 1024 * 1024)))
prune_limit_dur = base.query(
'memlimit != 0 & durable == True').reset_index()['memlimit'].min()
print('Using %s memory limit as representative for durable lobsters' %
(common.bts(prune_limit_dur * 1024 * 1024 * 1024)))
prune = base.query('durable == False & memlimit == %f' % (prune_limit))
prune_dur = base.query('durable == True & memlimit == %f' % (prune_limit_dur))
partial = base.query('partial == True & memlimit == 0')
full = base.query('partial == False & memlimit == 0')
fig, mem = plt.subplots()
xs = [
"Noria",
"Noria without partial",
]