def do_membar(path):
df = parse_to_csv(path/'mem-pytorch')
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'color3']):
# only draw one batch size: the largest one
df['Model'], df['BatchSize'] = df.Network.str.split('_').str
df.BatchSize.replace({'small': 1, 'medium': 5, 'large': 10}, inplace=True)
df['BatchSize'] = pd.to_numeric(df.BatchSize)
df = df.reset_index().loc[df.reset_index().groupby(['Model'])['BatchSize'].idxmax()]
df = df.drop(['index', 'BatchSize', 'Network'], axis=1)
df = df.rename(columns={'Model': 'Network'})
# sort values
df = df.sort_values('Network', ascending=False)
ax = plot_mem(df)
ax.set_xlabel('Memory Usage (GB)')
ax.set_ylabel('')
ax.legend(fontsize='xx-small',frameon=False)
ax.tick_params(axis='x', which='major', labelsize=8)
ax.tick_params(axis='y', which='major', labelsize=8, length=2)
ax.yaxis.label.set_size(8)
#ax.xaxis.tick_top()
#fig.tight_layout()
#fig.subplots_adjust(top=1, bottom=0, left=0, right=1)
fig = ax.figure
fig.set_size_inches(3.25, 2.5, forward=True)
fig.savefig('/tmp/workspace/mem-pytorch.pdf',
dpi=300, bbox_inches='tight', pad_inches = .015)
plt.close()
def prepare_paper(path='logs/nsdi19'):
path = Path(path)
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'line12']):
# also use color
# fifo = ju.load_trace(path/'card266'/'salus'/'trace.csv')
fifo = load_data(path / 'card272' / 'case0' / 'fifo', 'case0.output')
srtf = load_data(path / 'card266' / 'salus', 'card266.output')
srtf_refine = ju.load_refine(path / 'card266' / 'salus')
fair = load_data(path / 'card272' / 'case2' / 'salus', 'case2.output')
pack = load_data(path / 'card272' / 'case1' / 'salus', 'case1.output')
fig, ax = plt.subplots()
fig.set_size_inches(3.25, 1.5, forward=True)
jcts = pd.DataFrame({
'FIFO': fifo.JCT,
'SRTF': srtf.JCT,
'PACK': pack.JCT,
'FAIR': fair.JCT
})
plot_jcts(jcts, ax=ax, marker='', linewidth=1)
fig.tight_layout()
fig.savefig('/tmp/workspace/card272-jct.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
plt.close()
return fifo, srtf, srtf_refine, fair, pack
def do_srtf3(path):
srtf = load_data(path / 'card272' / 'case5' / 'salus', 'case5.output')
df = cm.load_mem(path / 'card274/srtf/alloc.output')
offset_local = srtf.Queued.min()
# timeline logs in US/Eastern, but server logs in UTC
# convert offset from US/Eastern to UTC
offset_server = offset_local.tz_localize('US/Eastern').tz_convert(
'UTC').tz_localize(None)
# select data in range
jnos = [14, 15]
st_sec = 468.4
ed_sec = 471
st = offset_server + pd.to_timedelta(st_sec, 's')
ed = offset_server + pd.to_timedelta(ed_sec, 's')
srtf = srtf[srtf.No.isin(jnos)]
df = df[(df.timestamp >= st) & (df.timestamp <= ed)]
df = df[df.Sess.isin(srtf.Sess.unique())]
sess2Model = srtf.set_index('Sess').Model.str.split('.', expand=True)
sess2Model['Label'] = sess2Model.apply(
lambda x: '#{}: {}'.format(x[3], x[0]), axis=1)
df['Sess'] = df.Sess.map(sess2Model.Label)
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'line12']):
fig, ax = plt.subplots()
ax.set_prop_cycle(None)
#next(ax._get_lines.prop_cycler)
# renormalize offset when plot
cm.plot_all(df, ax=ax, linewidth=1, marker='')
ax.set_ylim(bottom=0)
ax.xaxis.set_major_locator(pu.MaxNLocator(nbins=3))
ax.minorticks_off()
pu.cleanup_axis_bytes(ax.yaxis, maxN=5)
ax.set_ylabel('Memory Usage')
ax.set_xlabel('Time (s)')
ax.legend().remove()
#fig.tight_layout()
fig.set_size_inches(1.625, 1.5, forward=True)
fig.savefig('/tmp/workspace/card274-srtf-mem.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
plt.close()
def do_singlemem(path):
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'line12']):
# a single mem
df = cm.load_mem(path / 'exp1' / 'alloc.output')
ax = cm.plot_mem(df, linewidth=.8, color='k')
pu.cleanup_axis_bytes(ax.yaxis, maxN=4)
ax.set_xlabel('Time (s)')
ax.set_ylabel('TensorFlow\nMemory Usage')
ax.set_ylim(bottom=0, top=12 * (1024**3))
ax.set_xlim(left=1.5, right=7)
fig = ax.figure
fig.set_size_inches(3.25, 1.5, forward=True)
fig.savefig('/tmp/workspace/exp1.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
plt.close()
def prepare_paper(path='logs/nsdi19'):
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'gray']):
df = load_data(path)
fig, ax = plt.subplots()
fig.set_size_inches(3.25, 1.2, forward=True)
#plot_eval_pit_vs_speed(df, ax=ax)
#ax.set_xlabel('Time (s)')
#ax.yaxis.label.set_size(8)
plot_ratio(df, ax=ax, linewidth=1)
ax.set_ylabel('CDF')
fig.tight_layout()
fig.savefig('/tmp/workspace/card260.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
plt.close()
def prepare_paper(path='logs/nsdi19'):
path = Path(path)
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'color3']):
# fifo = ju.load_trace(path/'card266'/'salus'/'trace.csv')
df = load_data(path/'card271')
fig, ax = plt.subplots()
fig.set_size_inches(3.25, 1.3, forward=True)
# set col order
df = df[['Network', 'Salus', 'TF']]
ax = plot_makespan(df, ax=ax)
ax.tick_params(axis='x', rotation=0)
ax.set_xlabel('')
fig.tight_layout()
fig.savefig('/tmp/workspace/card271.pdf', dpi=300)
plt.close()
return df
def do_singlemem(path):
# a single mem
df = load_pytorch(path/'mem-pytorch'/'resnet101_75_output.txt')
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'line12']):
ax = mem.plot_cs(df.set_index('timestamp').act * 1024**2, linewidth=.8,
marker='', color='k')
ax.set_xlabel('Time (s)')
ax.set_ylabel('PyTorch\nMemory Usage')
pu.cleanup_axis_bytes(ax.yaxis, maxN=5)
ax.set_ylim(bottom=0,
top=12 * (1024**3)
)
ax.set_xlim(left=1,
right=8
)
ax.legend().remove()
fig = ax.figure
fig.set_size_inches(3.25, 1.5, forward=True)
fig.savefig('/tmp/workspace/exp1-pytorch.pdf',
dpi=300, bbox_inches='tight', pad_inches = .015)
plt.close()
def prepare_paper(path=path):
df, iters = load_data(path / 'card275' / 'case2')
fst, fed = iters.loc[iters.index[0], ['start_iter', 'end_iter']]
sst, sed = iters.loc[iters.index[-2], ['start_iter', 'end_iter']]
beforeAlloc = df[(df.timestamp < fst)]
# fAlloc = df[(df.timestamp >= fst) & (df.timestamp < fed)]
sAlloc = df[(df.timestamp >= sst) & (df.timestamp < sed)]
KB = 1024
MB = 1024 * KB
model = beforeAlloc
ephemeral = sAlloc[sAlloc.Size >= MB]
framework = sAlloc[sAlloc.Size < MB]
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'color3']):
fig, axs = plt.subplots(ncols=2,
nrows=1,
squeeze=False,
gridspec_kw={'width_ratios': [3, 2]})
fig.set_size_inches(3.25, 1.5, forward=True)
plot_size_cdf(framework,
axs[0][0],
axs[0][1],
label='Framework',
marker=None,
markevery=0.1,
linestyle='dashed',
linewidth=1)
plot_size_cdf(model,
axs[0][0],
axs[0][1],
label='Model',
marker=None,
markevery=0.1,
linestyle='dotted',
linewidth=1,
cumsum_kws={'zorder': 10})
plot_size_cdf(ephemeral,
axs[0][0],
axs[0][1],
label='Ephemeral',
marker=None,
markevery=0.05,
linestyle='-',
markersize=3,
linewidth=1)
axs[0][0].set_xlim(left=1)
axs[0][0].set_xscale('log', basex=2)
axs[0][0].set_xticks([1, 2**8, 2**16, 2**24, 2**32])
axs[0][0].tick_params(axis='x',
# labelsize='xx-small'
)
bytesformatter = pu.FuncFormatter(
lambda x, pos: pu.bytes2human(x, format='%(value).0f%(symbol)s'))
axs[0][0].xaxis.set_major_formatter(bytesformatter)
#pu.cleanup_axis_bytes(axs[0][0].xaxis, maxN=5, format=)
axs[0][1].set_ylim(bottom=1)
axs[0][1].set_yscale('log', basey=2)
axs[0][1].set_yticks([1, 2**8, 2**16, 2**24, 2**32, 2**37])
axs[0][1].tick_params(axis='y',
# labelsize='xx-small'
)
bytesformatter = pu.FuncFormatter(
lambda x, pos: pu.bytes2human(x, format='%(value).0f%(symbol)s'))
axs[0][1].yaxis.set_major_formatter(bytesformatter)
#pu.cleanup_axis_bytes(axs[0][0].xaxis, maxN=5, format=)
# legend at the bottom
handles, labels = axs[0][0].get_legend_handles_labels()
fig.legend(handles,
labels,
loc='lower center',
ncol=3,
bbox_to_anchor=(0, -0.1, 1, 1),
bbox_transform=fig.transFigure)
fig.tight_layout()
fig.savefig('/tmp/workspace/card275.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
plt.close()
def do_srtf2(path):
srtf = load_data(path / 'card266' / 'salus', 'card266.output')
srtf_refine = ju.load_refine(path / 'card266' / 'salus')
offset_local = srtf.Queued.min()
#srtf = ju.update_start_time_using_refine(srtf, srtf_refine, offset=offset_local)
jnos = [34, 80, 82, 86, 93, 94]
new_jnos = {n: i for i, n in enumerate(jnos)}
st_sec = 2651
ed_sec = 2910
srtf = srtf[srtf.No.isin(jnos)]
#colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2']
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'color3']):
# first do a refine plot to update 'Started' in srtf
ax = plt.gca()
ju.plot_refine(ax,
srtf,
srtf_refine,
offset=offset_local,
new_jnos=new_jnos,
plot_offset=-st_sec)
plt.close()
fig, ax = plt.subplots()
monochrome = cycler('color', ['0.0'])
ax.set_prop_cycle(monochrome)
ju.plot_timeline(srtf.drop(['LaneId'], axis=1),
ax=ax,
linewidth=2.5,
offset=offset_local,
new_jnos=new_jnos,
plot_offset=-st_sec)
ju.plot_refine(ax,
srtf,
srtf_refine,
offset=offset_local,
new_jnos=new_jnos,
plot_offset=-st_sec)
ax.set_xlim([0, ed_sec - st_sec])
# add a legend
ax.legend(handles=[
mpatches.Patch(color='#b6b6b6', label='Queuing'),
mpatches.Patch(color='black', label='Active')
],
bbox_to_anchor=(0.85, 0.03),
loc='lower right')
ax.set_ylabel('Job #')
ax.yaxis.set_ticks([0, 1, 2, 3, 4, 5])
fig.set_size_inches(4.875, 1.5, forward=True)
fig.savefig('/tmp/workspace/card274-srtf-compute.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
plt.close()
def do_fair(path):
fair = load_data(path / 'card272' / 'case4' / 'salus', 'case4.output')
df = cm.load_mem(path / 'card274/fair/alloc.output')
offset_local = fair.Queued.min()
# timeline logs in US/Eastern, but server logs in UTC
# convert offset from US/Eastern to UTC
offset_server = offset_local.tz_localize('US/Eastern').tz_convert(
'UTC').tz_localize(None)
# select sess name after calc the offset
# use jno
jnos = [1, 2, 4, 5]
#srtf = srtf[srtf.No.isin(jnos)]
#srtf_refine = srtf_refine[srtf_refine.No.isin(jnos)]
# select data in range
st_sec = 158
ed_sec = 164
st = offset_server + pd.to_timedelta(st_sec, 's')
ed = offset_server + pd.to_timedelta(ed_sec, 's')
df = df[(df.timestamp >= st) & (df.timestamp <= ed)]
df = df[df.Sess.isin(fair[fair.No.isin(jnos)].Sess.unique())]
sess2Model = fair[fair.No.isin(jnos)].set_index('Sess').Model.str.split(
'.', expand=True)
sess2Model['Label'] = sess2Model.apply(
lambda x: '#{}: {}'.format(x[3], x[0]), axis=1)
df['Sess'] = df.Sess.map(sess2Model.Label)
#colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2']
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'line12']):
fig, ax = plt.subplots()
cm.plot_all(df,
offset=offset_server,
ax=ax,
mem_kws={'use_marker': True},
markevery=0.05,
linestyle='-',
linewidth=.75)
ax.set_ylim(bottom=0)
#axs[1].legend().remove()
# Use axs[0]'s legend, but put it at axs[1]'s legend's place
fig.subplots_adjust(bottom=0.15, hspace=0.05)
ax.legend(
#*axs[0].get_legend_handles_labels(),
loc="lower center",
frameon=False,
bbox_to_anchor=[0.5, 1],
#bbox_transform=fig.transFigure,
fontsize='x-small',
ncol=4
#mode='expand'
)
ax.xaxis.set_major_locator(pu.MaxNLocator(nbins=5))
ax.minorticks_off()
pu.cleanup_axis_bytes(ax.yaxis, maxN=5)
ax.set_ylabel('Memory Usage')
ax.set_xlabel('Time (s)')
#fig.tight_layout()
fig.set_size_inches(6.5, 1.6, forward=True)
fig.savefig('/tmp/workspace/card274-fair.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
def do_srtf(path):
srtf = load_data(path / 'card272' / 'case5' / 'salus', 'case5.output')
srtf_refine = ju.load_refine(path / 'card272' / 'case5' / 'salus')
df = cm.load_mem(path / 'card274/srtf/alloc.output')
offset_local = srtf.Queued.min()
# timeline logs in US/Eastern, but server logs in UTC
# convert offset from US/Eastern to UTC
offset_server = offset_local.tz_localize('US/Eastern').tz_convert(
'UTC').tz_localize(None)
# select sess name after calc the offset
# use jno = [13, 14, 15]
jnos = [13, 14, 15]
srtf = srtf[srtf.No.isin(jnos)]
#srtf_refine = srtf_refine[srtf_refine.No.isin(jnos)]
# select data in range
st_sec = 400
ed_sec = 515
st = offset_server + pd.to_timedelta(st_sec, 's')
ed = offset_server + pd.to_timedelta(ed_sec, 's')
df = df[(df.timestamp >= st) & (df.timestamp <= ed)]
df = df[df.Sess.isin(srtf.Sess.unique())]
sess2Model = srtf.set_index('Sess').Model.str.split('.', expand=True)
sess2Model['Label'] = sess2Model.apply(
lambda x: '#{}: {}'.format(x[3], x[0]), axis=1)
df['Sess'] = df.Sess.map(sess2Model.Label)
#colors = ['#1f77b4', '#ff7f0e', '#2ca02c']
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'color3']):
fig, axs = plt.subplots(nrows=2,
sharex=True,
gridspec_kw={'height_ratios': [1, 4]})
_, _, sessProps = ju.plot_timeline(srtf.drop(['LaneId'], axis=1),
ax=axs[0],
linewidth=2.5,
returnSessProps=True,
offset=offset_local)
ju.plot_refine(axs[0], srtf, srtf_refine, offset=offset_local)
#axs[0].set_ylabel('Job No.')
# srtf is selected by simply set axis limit, because they are not easy to cut in data
axs[0].set_xlim([st_sec, ed_sec])
axs[0].set_ylim([12.45, 15.5])
#axs[0].set_ylabel('Job', rotation=0, labelpad=20)
sessProps = {
row.Label: sessProps[sess]
for sess, row in sess2Model.iterrows()
}
cm.plot_all(df, offset=offset_server, ax=axs[1], sessProps=sessProps)
axs[1].set_ylim(bottom=0)
#axs[1].legend().remove()
# Use axs[0]'s legend, but put it at axs[1]'s legend's place
fig.subplots_adjust(bottom=0.15, hspace=0.05)
axs[1].legend(
#*axs[0].get_legend_handles_labels(),
loc="lower center",
frameon=False,
bbox_to_anchor=[0.5, 1],
#bbox_transform=fig.transFigure,
fontsize='x-small',
ncol=4
#mode='expand'
)
axs[1].xaxis.set_major_locator(pu.MaxNLocator(nbins=5))
axs[1].minorticks_off()
pu.cleanup_axis_bytes(axs[1].yaxis, maxN=5)
axs[1].set_ylabel('Memory Usage')
axs[1].set_xlabel('Time (s)')
#fig.tight_layout()
fig.set_size_inches(6.5, 2, forward=True)
fig.savefig('/tmp/workspace/card274-srtf.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
def prepare_paper(path):
path = Path(path)
df = load_jcts(path)
# load in exp data
exp = load_cardlog(path)
# Drop anything without exp data
df = df.assign(Salus=exp).dropna()
# TODO: fix deep speech, which runs even faster than TF
#df = df.query('not index.str.contains("speech")')
df = df[~df.index.str.contains("speech")]
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'hatchbar']):
# add color to the cycler
cycle = mpl.rcParams['axes.prop_cycle'].by_key()
cycle['color'] = [
'ed7d31', '244185', '8cb5df', '000000', 'dcedd0', '006d2c'
]
mpl.rcParams['axes.prop_cycle'] = mpl.cycler(**cycle)
# override color cycle
#plt.rc('axes', prop_cycle=cycler('color', ['ed7d31', '000000', '8cb5df', 'dcedd0']))
fig, axs = plt.subplots(ncols=2, gridspec_kw={'width_ratios': [4, 1]})
fig.set_size_inches(6.5, 1.85, forward=True)
axs[0].set_prop_cycle(None)
cycle = axs[0]._get_lines.prop_cycler
for _, prop in zip(range(6), cycle):
print(prop)
# set plot bar order
order = ['Salus', 'TF', 'MPS']
df = df[order]
ax = plot_latency(df, ax=axs[0])
for _, prop in zip(range(6), cycle):
print(prop)
ax.set_ylim([0, 60])
#ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha='right')
# Draw utilization, current number is hard coded
# we ran 42 jobs, 3 of each (without speech, which is wierd)
# so 42 for TF
ut = pd.DataFrame([42, 6, 1], index=['TF', 'MPS', 'Salus'])
ax = ut.plot.bar(ax=axs[1],
color='w',
edgecolor='k',
hatch='////',
linewidth=1)
#ax.tick_params(axis='x', labelrotation=0)
ax.legend().remove()
ax.set_ylabel('# of GPUs needed')
fig.tight_layout()
fig.savefig('/tmp/workspace/card270.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.005)
plt.close()
# load Salus
salus_jcts = []
for f in (path/'salus/2').iterdir():
salus_jcts.append(parse_jct(f))
data.loc['Salus'] = [np.mean(salus_jcts), np.max(salus_jcts)]
return data
#%%
path = 'logs/nsdi19/exp6_2'
data = load_exp62(path)
data = data[['Average JCT']]
data = data[data.index != 'FIFO']
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'gray']):
ax = data.plot.bar(hatch='////', color='w', edgecolor='k')
fig = ax.figure
ax.tick_params(axis='x', labelsize=7, rotation=0)
ax.set_ylim(0, 150)
#ax.set_xlabel('')
ax.set_ylabel('Time (s)')
ax.legend(loc='upper left', bbox_to_anchor=(0, 1.1),)
ax.annotate('{:.2f}'.format(data.loc['SP+MPS+OC']['Average JCT']),
xy=[data.index.get_loc('SP+MPS+OC'), 150],
xytext=[0, 7], textcoords='offset points',
size=7,
horizontalalignment='center', verticalalignment='top')
