def load_data(path):
path = Path(path)
df = cm.load_mem(path / 'alloc.output')
# GPU only
df = df[df.Allocator.str.contains("GPU")]
iters = cm.load_iters(path / 'alloc.output')
iters = iters[iters.MainIter]
iters = iters.pivot_table(values='timestamp',
index=[
'StepId',
'GraphId',
'Sess',
],
columns='evt',
aggfunc='first').reset_index()
# select only a single step
#st, ed = iters.loc[iters.index[0], ['start_iter', 'end_iter']]
#df = df[(df.timestamp >= st) & (df.timestamp < ed)]
# alloc sizes
return df, iters
def do_srtf3(path):
srtf = load_data(path / 'card272' / 'case5' / 'salus', 'case5.output')
df = cm.load_mem(path / 'card274/srtf/alloc.output')
offset_local = srtf.Queued.min()
# timeline logs in US/Eastern, but server logs in UTC
# convert offset from US/Eastern to UTC
offset_server = offset_local.tz_localize('US/Eastern').tz_convert(
'UTC').tz_localize(None)
# select data in range
jnos = [14, 15]
st_sec = 468.4
ed_sec = 471
st = offset_server + pd.to_timedelta(st_sec, 's')
ed = offset_server + pd.to_timedelta(ed_sec, 's')
srtf = srtf[srtf.No.isin(jnos)]
df = df[(df.timestamp >= st) & (df.timestamp <= ed)]
df = df[df.Sess.isin(srtf.Sess.unique())]
sess2Model = srtf.set_index('Sess').Model.str.split('.', expand=True)
sess2Model['Label'] = sess2Model.apply(
lambda x: '#{}: {}'.format(x[3], x[0]), axis=1)
df['Sess'] = df.Sess.map(sess2Model.Label)
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'line12']):
fig, ax = plt.subplots()
ax.set_prop_cycle(None)
#next(ax._get_lines.prop_cycler)
# renormalize offset when plot
cm.plot_all(df, ax=ax, linewidth=1, marker='')
ax.set_ylim(bottom=0)
ax.xaxis.set_major_locator(pu.MaxNLocator(nbins=3))
ax.minorticks_off()
pu.cleanup_axis_bytes(ax.yaxis, maxN=5)
ax.set_ylabel('Memory Usage')
ax.set_xlabel('Time (s)')
ax.legend().remove()
#fig.tight_layout()
fig.set_size_inches(1.625, 1.5, forward=True)
fig.savefig('/tmp/workspace/card274-srtf-mem.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
plt.close()
def do_singlemem(path):
with plt.style.context(['seaborn-paper', 'mypaper', 'line12']):
# a single mem
df = cm.load_mem(path / 'exp1' / 'alloc.output')
ax = cm.plot_mem(df, linewidth=.5, markevery=400, color='k')
pu.cleanup_axis_bytes(ax.yaxis, maxN=4)
ax.set_xlabel('Time (s)')
ax.set_ylabel('TensorFlow\nMemory Usage')
ax.set_ylim(bottom=0, top=12 * (1024**3))
ax.set_xlim(left=1.5, right=7)
fig = ax.figure
fig.set_size_inches(3.25, 1.5, forward=True)
fig.savefig('/tmp/workspace/exp1.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
plt.close()
def do_fair(path):
fair = load_data(path / 'card272' / 'case4' / 'salus', 'case4.output')
df = cm.load_mem(path / 'card274/fair/alloc.output')
offset_local = fair.Queued.min()
# timeline logs in US/Eastern, but server logs in UTC
# convert offset from US/Eastern to UTC
offset_server = offset_local.tz_localize('US/Eastern').tz_convert(
'UTC').tz_localize(None)
# select sess name after calc the offset
# use jno
jnos = [1, 2, 4, 5]
#srtf = srtf[srtf.No.isin(jnos)]
#srtf_refine = srtf_refine[srtf_refine.No.isin(jnos)]
# select data in range
st_sec = 158
ed_sec = 164
st = offset_server + pd.to_timedelta(st_sec, 's')
ed = offset_server + pd.to_timedelta(ed_sec, 's')
df = df[(df.timestamp >= st) & (df.timestamp <= ed)]
df = df[df.Sess.isin(fair[fair.No.isin(jnos)].Sess.unique())]
sess2Model = fair[fair.No.isin(jnos)].set_index('Sess').Model.str.split(
'.', expand=True)
sess2Model['Label'] = sess2Model.apply(
lambda x: '#{}: {}'.format(x[3], x[0]), axis=1)
df['Sess'] = df.Sess.map(sess2Model.Label)
#colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2']
with plt.style.context(['seaborn-paper', 'mypaper', 'line12']):
fig, ax = plt.subplots()
cm.plot_all(df,
offset=offset_server,
ax=ax,
markevery=0.05,
linestyle='-',
linewidth=.75)
ax.set_ylim(bottom=0)
#axs[1].legend().remove()
# Use axs[0]'s legend, but put it at axs[1]'s legend's place
fig.subplots_adjust(bottom=0.15, hspace=0.05)
ax.legend(
#*axs[0].get_legend_handles_labels(),
loc="lower center",
frameon=False,
bbox_to_anchor=[0.5, 1],
#bbox_transform=fig.transFigure,
fontsize='x-small',
ncol=4
#mode='expand'
)
ax.xaxis.set_major_locator(pu.MaxNLocator(nbins=5))
ax.minorticks_off()
pu.cleanup_axis_bytes(ax.yaxis, maxN=5)
ax.set_ylabel('Memory Usage')
ax.set_xlabel('Time (s)')
#fig.tight_layout()
fig.set_size_inches(6.5, 1.6, forward=True)
fig.savefig('/tmp/workspace/card274-fair.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
def do_srtf(path):
srtf = load_data(path / 'card272' / 'case5' / 'salus', 'case5.output')
srtf_refine = ju.load_refine(path / 'card272' / 'case5' / 'salus')
df = cm.load_mem(path / 'card274/srtf/alloc.output')
offset_local = srtf.Queued.min()
# timeline logs in US/Eastern, but server logs in UTC
# convert offset from US/Eastern to UTC
offset_server = offset_local.tz_localize('US/Eastern').tz_convert(
'UTC').tz_localize(None)
# select sess name after calc the offset
# use jno = [13, 14, 15]
jnos = [13, 14, 15]
srtf = srtf[srtf.No.isin(jnos)]
#srtf_refine = srtf_refine[srtf_refine.No.isin(jnos)]
# select data in range
st_sec = 400
ed_sec = 515
st = offset_server + pd.to_timedelta(st_sec, 's')
ed = offset_server + pd.to_timedelta(ed_sec, 's')
df = df[(df.timestamp >= st) & (df.timestamp <= ed)]
df = df[df.Sess.isin(srtf.Sess.unique())]
sess2Model = srtf.set_index('Sess').Model.str.split('.', expand=True)
sess2Model['Label'] = sess2Model.apply(
lambda x: '#{}: {}'.format(x[3], x[0]), axis=1)
df['Sess'] = df.Sess.map(sess2Model.Label)
#colors = ['#1f77b4', '#ff7f0e', '#2ca02c']
with plt.style.context(['seaborn-paper', 'mypaper', 'color3']):
fig, axs = plt.subplots(nrows=2,
sharex=True,
gridspec_kw={'height_ratios': [1, 4]})
_, _, sessProps = ju.plot_timeline(srtf.drop(['LaneId'], axis=1),
ax=axs[0],
linewidth=2.5,
returnSessProps=True,
offset=offset_local)
ju.plot_refine(axs[0], srtf, srtf_refine, offset=offset_local)
#axs[0].set_ylabel('Job No.')
# srtf is selected by simply set axis limit, because they are not easy to cut in data
axs[0].set_xlim([st_sec, ed_sec])
axs[0].set_ylim([12.45, 15.5])
#axs[0].set_ylabel('Job', rotation=0, labelpad=20)
sessProps = {
row.Label: sessProps[sess]
for sess, row in sess2Model.iterrows()
}
cm.plot_all(df, offset=offset_server, ax=axs[1], sessProps=sessProps)
axs[1].set_ylim(bottom=0)
#axs[1].legend().remove()
# Use axs[0]'s legend, but put it at axs[1]'s legend's place
fig.subplots_adjust(bottom=0.15, hspace=0.05)
axs[1].legend(
#*axs[0].get_legend_handles_labels(),
loc="lower center",
frameon=False,
bbox_to_anchor=[0.5, 1],
#bbox_transform=fig.transFigure,
fontsize='x-small',
ncol=4
#mode='expand'
)
axs[1].xaxis.set_major_locator(pu.MaxNLocator(nbins=5))
axs[1].minorticks_off()
pu.cleanup_axis_bytes(axs[1].yaxis, maxN=5)
axs[1].set_ylabel('Memory Usage')
axs[1].set_xlabel('Time (s)')
#fig.tight_layout()
fig.set_size_inches(6.5, 2, forward=True)
fig.savefig('/tmp/workspace/card274-srtf.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
