def plot_size_cdf(df,
cdf_ax,
cumsum_ax,
cdf_kws=None,
cumsum_kws=None,
**kwargs):
sizes = df[df.Sign > 0].Size
if cdf_kws is None:
cdf_kws = {}
cdf_kws.update(kwargs)
pu.cdf(sizes, ax=cdf_ax, **cdf_kws)
if cumsum_kws is None:
cumsum_kws = {}
cumsum_kws.update(kwargs)
sizes.sort_values().reset_index(drop=True).cumsum().plot(ax=cumsum_ax,
**cumsum_kws)
pu.cleanup_axis_bytes(cdf_ax.xaxis, format='%(value).0f %(symbol)s')
cdf_ax.set_ylabel('CDF')
cdf_ax.set_xlabel('Memory Allocation Size')
pu.cleanup_axis_bytes(cumsum_ax.yaxis,
maxN=5,
format='%(value).0f %(symbol)s')
cumsum_ax.set_ylabel('Cumsum of Sizes')
cumsum_ax.set_xlabel('Memory Allocations')
def draw_memmap(mapstr, lc=None, ax=None, **kwargs):
"""Draw memmap str using matplotlib
mapstr is like:
0x7f2a76000000, 2211840, 1, 256;0x7f2a7621c000, 8200192, 0, 1024;
"""
if ax is None:
_, ax = plt.subplots()
ax.set_xlim([0, 17179869184])
ax.set_ylim([-0.2, 0.2])
args = {'linewidths': 100}
args.update(kwargs)
newlc = False
if lc is None:
lc = LineCollection([], **args)
newlc = True
mapdf = parse_mapstr(mapstr)
mapdf, lc = draw_on_lc(mapdf, lc)
ax.set_xlim([mapdf.NOrigin.min(), mapdf.NEnd.max()])
if newlc:
ax.add_collection(lc, autolim=False)
pu.cleanup_axis_bytes(ax.xaxis)
return lc
def __init__(self, df, label=None):
self.df = df
self.label = label
if self.label is None:
self.label = 'id = {index} size = {Size} timestamp = {timestamp}'
self.fig = plt.gcf()
self.fig.set_size_inches([25, 15], forward=True)
gs = gridspec.GridSpec(2, 1, height_ratios=[20, 1])
self._ax = self.fig.add_subplot(gs[0, 0])
self._ax.set_ylim([-0.2, 0.2])
# create artists
self._lc = draw_memmap('0x0, 17179869184, 0;', ax=self._ax)
self._text = self._ax.text(0.5,
0.8,
"",
ha='center',
va='baseline',
fontsize=25,
transform=self._ax.transAxes)
# lc and texts for allocators
self._lcalloc = LineCollection([[(0, 0.1), (17179869184, 0.1)]],
linewidths=10)
self._ax.add_collection(self._lcalloc)
self._textalloc = []
pu.cleanup_axis_bytes(self._ax.xaxis)
init_frame = 0
self._step(init_frame)
# add a slider
axstep = self.fig.add_subplot(gs[1, 0],
facecolor='lightgoldenrodyellow')
self._stepSlider = Slider(axstep,
'ID',
0,
len(df),
closedmax=False,
valinit=init_frame,
valfmt='%d',
valstep=1,
dragging=True)
def update(val):
self._step(int(val))
self.fig.canvas.draw_idle()
self._stepSlider.on_changed(update)
# listen key press
self.fig.canvas.mpl_connect('key_press_event',
lambda evt: self._keydown(evt))
def plot_mem(df, marker=False, offset=None, return_offset=False, **kwargs):
df, offset = normalize_time(df, offset)
cumsum = calc_cumsum(df)
ax = plot_cs(cumsum, **kwargs)
if isinstance(marker, bool) and marker:
ax = plot_marker(df, cumsum, ax=ax)
pu.cleanup_axis_bytes(ax.yaxis)
if return_offset:
return ax, offset
return ax
def do_srtf3(path):
srtf = load_data(path / 'card272' / 'case5' / 'salus', 'case5.output')
df = cm.load_mem(path / 'card274/srtf/alloc.output')
offset_local = srtf.Queued.min()
# timeline logs in US/Eastern, but server logs in UTC
# convert offset from US/Eastern to UTC
offset_server = offset_local.tz_localize('US/Eastern').tz_convert(
'UTC').tz_localize(None)
# select data in range
jnos = [14, 15]
st_sec = 468.4
ed_sec = 471
st = offset_server + pd.to_timedelta(st_sec, 's')
ed = offset_server + pd.to_timedelta(ed_sec, 's')
srtf = srtf[srtf.No.isin(jnos)]
df = df[(df.timestamp >= st) & (df.timestamp <= ed)]
df = df[df.Sess.isin(srtf.Sess.unique())]
sess2Model = srtf.set_index('Sess').Model.str.split('.', expand=True)
sess2Model['Label'] = sess2Model.apply(
lambda x: '#{}: {}'.format(x[3], x[0]), axis=1)
df['Sess'] = df.Sess.map(sess2Model.Label)
pu.matplotlib_fixes()
with plt.style.context(['seaborn-paper', 'mypaper', 'line12']):
fig, ax = plt.subplots()
ax.set_prop_cycle(None)
#next(ax._get_lines.prop_cycler)
# renormalize offset when plot
cm.plot_all(df, ax=ax, linewidth=1, marker='')
ax.set_ylim(bottom=0)
ax.xaxis.set_major_locator(pu.MaxNLocator(nbins=3))
ax.minorticks_off()
pu.cleanup_axis_bytes(ax.yaxis, maxN=5)
ax.set_ylabel('Memory Usage')
ax.set_xlabel('Time (s)')
ax.legend().remove()
#fig.tight_layout()
fig.set_size_inches(1.625, 1.5, forward=True)
fig.savefig('/tmp/workspace/card274-srtf-mem.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
plt.close()
def do_singlemem(path):
with plt.style.context(['seaborn-paper', 'mypaper', 'line12']):
# a single mem
df = cm.load_mem(path / 'exp1' / 'alloc.output')
ax = cm.plot_mem(df, linewidth=.5, markevery=400, color='k')
pu.cleanup_axis_bytes(ax.yaxis, maxN=4)
ax.set_xlabel('Time (s)')
ax.set_ylabel('TensorFlow\nMemory Usage')
ax.set_ylim(bottom=0, top=12 * (1024**3))
ax.set_xlim(left=1.5, right=7)
fig = ax.figure
fig.set_size_inches(3.25, 1.5, forward=True)
fig.savefig('/tmp/workspace/exp1.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
plt.close()
def do_singlemem(path):
with plt.style.context(['seaborn-paper', 'mypaper', 'line12']):
# a single mem
df = load_pytorch(path / 'mem-pytorch' / 'resnet101_75_output.txt')
ax = mem.plot_cs(df.set_index('timestamp').act * 1024**2,
linewidth=.5,
markevery=200,
color='k')
ax.set_xlabel('Time (s)')
ax.set_ylabel('PyTorch\nMemory Usage')
pu.cleanup_axis_bytes(ax.yaxis, maxN=5)
ax.set_ylim(bottom=0, top=12 * (1024**3))
ax.set_xlim(left=1, right=8)
ax.legend().remove()
fig = ax.figure
fig.set_size_inches(3.25, 1.5, forward=True)
fig.savefig('/tmp/workspace/exp1-pytorch.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
plt.close()
def do_fair(path):
fair = load_data(path / 'card272' / 'case4' / 'salus', 'case4.output')
df = cm.load_mem(path / 'card274/fair/alloc.output')
offset_local = fair.Queued.min()
# timeline logs in US/Eastern, but server logs in UTC
# convert offset from US/Eastern to UTC
offset_server = offset_local.tz_localize('US/Eastern').tz_convert(
'UTC').tz_localize(None)
# select sess name after calc the offset
# use jno
jnos = [1, 2, 4, 5]
#srtf = srtf[srtf.No.isin(jnos)]
#srtf_refine = srtf_refine[srtf_refine.No.isin(jnos)]
# select data in range
st_sec = 158
ed_sec = 164
st = offset_server + pd.to_timedelta(st_sec, 's')
ed = offset_server + pd.to_timedelta(ed_sec, 's')
df = df[(df.timestamp >= st) & (df.timestamp <= ed)]
df = df[df.Sess.isin(fair[fair.No.isin(jnos)].Sess.unique())]
sess2Model = fair[fair.No.isin(jnos)].set_index('Sess').Model.str.split(
'.', expand=True)
sess2Model['Label'] = sess2Model.apply(
lambda x: '#{}: {}'.format(x[3], x[0]), axis=1)
df['Sess'] = df.Sess.map(sess2Model.Label)
#colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2']
with plt.style.context(['seaborn-paper', 'mypaper', 'line12']):
fig, ax = plt.subplots()
cm.plot_all(df,
offset=offset_server,
ax=ax,
markevery=0.05,
linestyle='-',
linewidth=.75)
ax.set_ylim(bottom=0)
#axs[1].legend().remove()
# Use axs[0]'s legend, but put it at axs[1]'s legend's place
fig.subplots_adjust(bottom=0.15, hspace=0.05)
ax.legend(
#*axs[0].get_legend_handles_labels(),
loc="lower center",
frameon=False,
bbox_to_anchor=[0.5, 1],
#bbox_transform=fig.transFigure,
fontsize='x-small',
ncol=4
#mode='expand'
)
ax.xaxis.set_major_locator(pu.MaxNLocator(nbins=5))
ax.minorticks_off()
pu.cleanup_axis_bytes(ax.yaxis, maxN=5)
ax.set_ylabel('Memory Usage')
ax.set_xlabel('Time (s)')
#fig.tight_layout()
fig.set_size_inches(6.5, 1.6, forward=True)
fig.savefig('/tmp/workspace/card274-fair.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
def do_srtf(path):
srtf = load_data(path / 'card272' / 'case5' / 'salus', 'case5.output')
srtf_refine = ju.load_refine(path / 'card272' / 'case5' / 'salus')
df = cm.load_mem(path / 'card274/srtf/alloc.output')
offset_local = srtf.Queued.min()
# timeline logs in US/Eastern, but server logs in UTC
# convert offset from US/Eastern to UTC
offset_server = offset_local.tz_localize('US/Eastern').tz_convert(
'UTC').tz_localize(None)
# select sess name after calc the offset
# use jno = [13, 14, 15]
jnos = [13, 14, 15]
srtf = srtf[srtf.No.isin(jnos)]
#srtf_refine = srtf_refine[srtf_refine.No.isin(jnos)]
# select data in range
st_sec = 400
ed_sec = 515
st = offset_server + pd.to_timedelta(st_sec, 's')
ed = offset_server + pd.to_timedelta(ed_sec, 's')
df = df[(df.timestamp >= st) & (df.timestamp <= ed)]
df = df[df.Sess.isin(srtf.Sess.unique())]
sess2Model = srtf.set_index('Sess').Model.str.split('.', expand=True)
sess2Model['Label'] = sess2Model.apply(
lambda x: '#{}: {}'.format(x[3], x[0]), axis=1)
df['Sess'] = df.Sess.map(sess2Model.Label)
#colors = ['#1f77b4', '#ff7f0e', '#2ca02c']
with plt.style.context(['seaborn-paper', 'mypaper', 'color3']):
fig, axs = plt.subplots(nrows=2,
sharex=True,
gridspec_kw={'height_ratios': [1, 4]})
_, _, sessProps = ju.plot_timeline(srtf.drop(['LaneId'], axis=1),
ax=axs[0],
linewidth=2.5,
returnSessProps=True,
offset=offset_local)
ju.plot_refine(axs[0], srtf, srtf_refine, offset=offset_local)
#axs[0].set_ylabel('Job No.')
# srtf is selected by simply set axis limit, because they are not easy to cut in data
axs[0].set_xlim([st_sec, ed_sec])
axs[0].set_ylim([12.45, 15.5])
#axs[0].set_ylabel('Job', rotation=0, labelpad=20)
sessProps = {
row.Label: sessProps[sess]
for sess, row in sess2Model.iterrows()
}
cm.plot_all(df, offset=offset_server, ax=axs[1], sessProps=sessProps)
axs[1].set_ylim(bottom=0)
#axs[1].legend().remove()
# Use axs[0]'s legend, but put it at axs[1]'s legend's place
fig.subplots_adjust(bottom=0.15, hspace=0.05)
axs[1].legend(
#*axs[0].get_legend_handles_labels(),
loc="lower center",
frameon=False,
bbox_to_anchor=[0.5, 1],
#bbox_transform=fig.transFigure,
fontsize='x-small',
ncol=4
#mode='expand'
)
axs[1].xaxis.set_major_locator(pu.MaxNLocator(nbins=5))
axs[1].minorticks_off()
pu.cleanup_axis_bytes(axs[1].yaxis, maxN=5)
axs[1].set_ylabel('Memory Usage')
axs[1].set_xlabel('Time (s)')
#fig.tight_layout()
fig.set_size_inches(6.5, 2, forward=True)
fig.savefig('/tmp/workspace/card274-srtf.pdf',
dpi=300,
bbox_inches='tight',
pad_inches=.015)
def memory_usage(logs,
iter_times=None,
beginning=None,
starts=None,
ends=None,
mem_type=None,
unified_ylabel=False,
smoother=None,
xformatter=None,
per_sess=False,
show_avg=None,
ax=None):
if beginning is None:
beginning = get_beginning(logs)
if show_avg is None:
show_avg = not per_sess
# Prepare ticket -> session map
ticket2sess = {}
for l in logs:
if l.type != 'mem_pre':
continue
if l.ticket not in ticket2sess:
ticket2sess[l.ticket] = l.sess
elif l.sess != ticket2sess[l.ticket]:
print('WARNING: ticket {} reused: previous {}, now: {}'.format(
l.ticket, ticket2sess[l.ticket], l.sess))
mem_usages = [
l for l in logs if l.type == 'mem_alloc' or l.type == 'mem_dealloc'
]
mem_activities = []
for m in mem_usages:
if m.addr == '0x0' or m.addr == '0':
continue
if m.type == 'mem_alloc':
mem_activities.append({
'timestamp': m.timestamp,
'size': m.size,
'mem_type': m.block['mem_type'],
'alloc_inst': m.block['alloc_inst'],
'session': ticket2sess[m.block['ticket']]
})
elif m.type == 'mem_dealloc':
mem_activities.append({
'timestamp': m.timestamp,
'size': -m.size,
'mem_type': m.block['mem_type'],
'alloc_inst': m.block['alloc_inst'],
'session': ticket2sess[m.block['ticket']]
})
else:
raise ValueError("Unexpected value: ", m)
df = pd.DataFrame(mem_activities)
if len(df) == 0:
return df, None
if mem_type is not None:
df = df[df['mem_type'] == mem_type]
df = df.set_index('timestamp').sort_index()
# Save beginning
if beginning is None:
beginning = df.index[0]
nrows = len(df['mem_type'].unique())
if ax is None:
fig, axs = plt.subplots(nrows=nrows,
ncols=1,
sharex=True,
squeeze=False)
# Make axes into a 1D array
axs = axs.reshape(-1)
else:
axs = [ax]
series = []
pending_avg = []
for (name, group), ax in zip(df.groupby('mem_type'), axs):
if per_sess:
sessionUsages = {}
for k, gg in group.groupby('session'):
sessionUsages[k] = gg['size'].cumsum()
ss = pd.DataFrame(sessionUsages).fillna(method='ffill').fillna(0)
else:
ss = group['size'].cumsum()
# Do range restriction
rstarts = None
rends = None
# Restrict x axis to iteration timaxes, must be done after cumsum, otherwise there
# will be negative number
if iter_times is not None:
rstarts = iter_times[0][0]
rends = iter_times[-1][1]
rstarts = starts if starts is not None else rstarts
rends = ends if ends is not None else rends
if rstarts is not None and rends is not None:
ss = ss.loc[rstarts:rends]
elif rstarts is not None:
ss = ss.loc[rstarts:]
elif rends is not None:
ss = ss.loc[:rends]
ss.index = ss.index - beginning
if show_avg:
ss2 = ss.resample('100us').interpolate(method='time')
pending_avg.append((ss2.mean(), ax))
series.append(ss)
if smoother:
if show_avg:
ss = smoother(ss, ss2)
else:
ss = smoother(ss)
# Change to timedelta after iteration restriction.
# for some reason slicing doesn't work on Timedeltas
# Pandas doesn't support irregular Timedelta intervals on xaxis
# see https://stackoverflow.com/questions/40621710/axis-interval-spacing-when-plotting-with-pandas-timedelta
# and https://github.com/pandas-dev/pandas/issues/8711
ss.index = ss.index.astype(int)
if per_sess:
ss.plot.area(ax=ax, linewidth=0)
else:
ss.plot(ax=ax)
ax.legend().remove()
pu.cleanup_axis_bytes(ax.yaxis)
if not unified_ylabel:
ax.set_ylabel('Memory Usage')
# Adjust x axis
if unified_ylabel:
axs[-1].xaxis.label.set_visible(False)
else:
axs[-1].set_xlabel('Time (s)')
axs[-1].autoscale(axis='x')
# xlim = axs[-1].get_xlim()
axs[-1].set_xlim(left=0)
pu.cleanup_axis_timedelta(axs[-1].xaxis, xformatter)
# Draw avg line after adjust xaxis
if show_avg:
for d, ax in pending_avg:
pu.axhlines(d, linestyle='--', ax=ax)
def format_coord(x, y):
return 'x={:.4f}, y={:.1f} MB'.format(x, y / 1024 / 1024)
axs[-1].format_coord = format_coord
fig = axs[-1].figure
#fig.tight_layout()
if unified_ylabel:
fig.text(0.5, 0.02, 'Time (s)', ha='center')
fig.text(0.02,
0.5,
'Memory Usage (bytes)',
va='center',
rotation='vertical')
fig.subplots_adjust(left=0.1, bottom=0.13)
return df, beginning, fig
def __init__(self,
df,
colormap,
ptr2sess,
label=None,
doPreprocess=False,
hdf=None):
self.label = label
if self.label is None:
self.label = 'id = {index} timestamp = {timestamp}'
self.colormap = colormap
self.ptr2sess = ptr2sess
self._do_preprocess = doPreprocess
self._hdf = hdf
if doPreprocess:
self.data = preprocess_memmap2(df, self.colormap, self.ptr2sess)
else:
self.data = df
self.fig = plt.gcf()
self.fig.set_size_inches([25, 15], forward=True)
gs = gridspec.GridSpec(2, 1, height_ratios=[20, 1])
self._ax = self.fig.add_subplot(gs[0, 0])
self._ax.set_ylim([-0.2, 0.2])
# create artists
self._lc = draw_memmap('0x0, 17179869184, 0, 0;', ax=self._ax)
self._text = self._ax.text(0.5,
0.8,
"",
ha='center',
va='baseline',
fontsize=25,
transform=self._ax.transAxes)
# lc and texts for bins
self._lcalloc = LineCollection([[(0, 0.1), (17179869184, 0.1)]],
linewidths=10)
self._ax.add_collection(self._lcalloc)
self._textalloc = []
pu.cleanup_axis_bytes(self._ax.xaxis)
init_frame = 0
self._step(init_frame)
# session legend
self._ax.legend(handles=[
mpatches.Patch(color=c, label=sess)
for sess, c in self.colormap.items()
])
# add a slider
axstep = self.fig.add_subplot(gs[1, 0],
facecolor='lightgoldenrodyellow')
self._stepSlider = Slider(axstep,
'ID',
0,
len(self.data) - 1,
valinit=init_frame,
valfmt='%d',
valstep=1,
dragging=True)
def update(val):
self._step(int(val))
self.fig.canvas.draw_idle()
self._stepSlider.on_changed(update)
# timer
self._timer = None
# listen key press
self.fig.canvas.mpl_connect('key_press_event',
lambda evt: self._keydown(evt))
