def plot_size_cdf(df, cdf_ax, cumsum_ax, cdf_kws=None, cumsum_kws=None, **kwargs): sizes = df[df.Sign > 0].Size if cdf_kws is None: cdf_kws = {} cdf_kws.update(kwargs) pu.cdf(sizes, ax=cdf_ax, **cdf_kws) if cumsum_kws is None: cumsum_kws = {} cumsum_kws.update(kwargs) sizes.sort_values().reset_index(drop=True).cumsum().plot(ax=cumsum_ax, **cumsum_kws) pu.cleanup_axis_bytes(cdf_ax.xaxis, format='%(value).0f %(symbol)s') cdf_ax.set_ylabel('CDF') cdf_ax.set_xlabel('Memory Allocation Size') pu.cleanup_axis_bytes(cumsum_ax.yaxis, maxN=5, format='%(value).0f %(symbol)s') cumsum_ax.set_ylabel('Cumsum of Sizes') cumsum_ax.set_xlabel('Memory Allocations')
def draw_memmap(mapstr, lc=None, ax=None, **kwargs): """Draw memmap str using matplotlib mapstr is like: 0x7f2a76000000, 2211840, 1, 256;0x7f2a7621c000, 8200192, 0, 1024; """ if ax is None: _, ax = plt.subplots() ax.set_xlim([0, 17179869184]) ax.set_ylim([-0.2, 0.2]) args = {'linewidths': 100} args.update(kwargs) newlc = False if lc is None: lc = LineCollection([], **args) newlc = True mapdf = parse_mapstr(mapstr) mapdf, lc = draw_on_lc(mapdf, lc) ax.set_xlim([mapdf.NOrigin.min(), mapdf.NEnd.max()]) if newlc: ax.add_collection(lc, autolim=False) pu.cleanup_axis_bytes(ax.xaxis) return lc
def __init__(self, df, label=None): self.df = df self.label = label if self.label is None: self.label = 'id = {index} size = {Size} timestamp = {timestamp}' self.fig = plt.gcf() self.fig.set_size_inches([25, 15], forward=True) gs = gridspec.GridSpec(2, 1, height_ratios=[20, 1]) self._ax = self.fig.add_subplot(gs[0, 0]) self._ax.set_ylim([-0.2, 0.2]) # create artists self._lc = draw_memmap('0x0, 17179869184, 0;', ax=self._ax) self._text = self._ax.text(0.5, 0.8, "", ha='center', va='baseline', fontsize=25, transform=self._ax.transAxes) # lc and texts for allocators self._lcalloc = LineCollection([[(0, 0.1), (17179869184, 0.1)]], linewidths=10) self._ax.add_collection(self._lcalloc) self._textalloc = [] pu.cleanup_axis_bytes(self._ax.xaxis) init_frame = 0 self._step(init_frame) # add a slider axstep = self.fig.add_subplot(gs[1, 0], facecolor='lightgoldenrodyellow') self._stepSlider = Slider(axstep, 'ID', 0, len(df), closedmax=False, valinit=init_frame, valfmt='%d', valstep=1, dragging=True) def update(val): self._step(int(val)) self.fig.canvas.draw_idle() self._stepSlider.on_changed(update) # listen key press self.fig.canvas.mpl_connect('key_press_event', lambda evt: self._keydown(evt))
def plot_mem(df, marker=False, offset=None, return_offset=False, **kwargs): df, offset = normalize_time(df, offset) cumsum = calc_cumsum(df) ax = plot_cs(cumsum, **kwargs) if isinstance(marker, bool) and marker: ax = plot_marker(df, cumsum, ax=ax) pu.cleanup_axis_bytes(ax.yaxis) if return_offset: return ax, offset return ax
def do_srtf3(path): srtf = load_data(path / 'card272' / 'case5' / 'salus', 'case5.output') df = cm.load_mem(path / 'card274/srtf/alloc.output') offset_local = srtf.Queued.min() # timeline logs in US/Eastern, but server logs in UTC # convert offset from US/Eastern to UTC offset_server = offset_local.tz_localize('US/Eastern').tz_convert( 'UTC').tz_localize(None) # select data in range jnos = [14, 15] st_sec = 468.4 ed_sec = 471 st = offset_server + pd.to_timedelta(st_sec, 's') ed = offset_server + pd.to_timedelta(ed_sec, 's') srtf = srtf[srtf.No.isin(jnos)] df = df[(df.timestamp >= st) & (df.timestamp <= ed)] df = df[df.Sess.isin(srtf.Sess.unique())] sess2Model = srtf.set_index('Sess').Model.str.split('.', expand=True) sess2Model['Label'] = sess2Model.apply( lambda x: '#{}: {}'.format(x[3], x[0]), axis=1) df['Sess'] = df.Sess.map(sess2Model.Label) pu.matplotlib_fixes() with plt.style.context(['seaborn-paper', 'mypaper', 'line12']): fig, ax = plt.subplots() ax.set_prop_cycle(None) #next(ax._get_lines.prop_cycler) # renormalize offset when plot cm.plot_all(df, ax=ax, linewidth=1, marker='') ax.set_ylim(bottom=0) ax.xaxis.set_major_locator(pu.MaxNLocator(nbins=3)) ax.minorticks_off() pu.cleanup_axis_bytes(ax.yaxis, maxN=5) ax.set_ylabel('Memory Usage') ax.set_xlabel('Time (s)') ax.legend().remove() #fig.tight_layout() fig.set_size_inches(1.625, 1.5, forward=True) fig.savefig('/tmp/workspace/card274-srtf-mem.pdf', dpi=300, bbox_inches='tight', pad_inches=.015) plt.close()
def do_singlemem(path): with plt.style.context(['seaborn-paper', 'mypaper', 'line12']): # a single mem df = cm.load_mem(path / 'exp1' / 'alloc.output') ax = cm.plot_mem(df, linewidth=.5, markevery=400, color='k') pu.cleanup_axis_bytes(ax.yaxis, maxN=4) ax.set_xlabel('Time (s)') ax.set_ylabel('TensorFlow\nMemory Usage') ax.set_ylim(bottom=0, top=12 * (1024**3)) ax.set_xlim(left=1.5, right=7) fig = ax.figure fig.set_size_inches(3.25, 1.5, forward=True) fig.savefig('/tmp/workspace/exp1.pdf', dpi=300, bbox_inches='tight', pad_inches=.015) plt.close()
def do_singlemem(path): with plt.style.context(['seaborn-paper', 'mypaper', 'line12']): # a single mem df = load_pytorch(path / 'mem-pytorch' / 'resnet101_75_output.txt') ax = mem.plot_cs(df.set_index('timestamp').act * 1024**2, linewidth=.5, markevery=200, color='k') ax.set_xlabel('Time (s)') ax.set_ylabel('PyTorch\nMemory Usage') pu.cleanup_axis_bytes(ax.yaxis, maxN=5) ax.set_ylim(bottom=0, top=12 * (1024**3)) ax.set_xlim(left=1, right=8) ax.legend().remove() fig = ax.figure fig.set_size_inches(3.25, 1.5, forward=True) fig.savefig('/tmp/workspace/exp1-pytorch.pdf', dpi=300, bbox_inches='tight', pad_inches=.015) plt.close()
def do_fair(path): fair = load_data(path / 'card272' / 'case4' / 'salus', 'case4.output') df = cm.load_mem(path / 'card274/fair/alloc.output') offset_local = fair.Queued.min() # timeline logs in US/Eastern, but server logs in UTC # convert offset from US/Eastern to UTC offset_server = offset_local.tz_localize('US/Eastern').tz_convert( 'UTC').tz_localize(None) # select sess name after calc the offset # use jno jnos = [1, 2, 4, 5] #srtf = srtf[srtf.No.isin(jnos)] #srtf_refine = srtf_refine[srtf_refine.No.isin(jnos)] # select data in range st_sec = 158 ed_sec = 164 st = offset_server + pd.to_timedelta(st_sec, 's') ed = offset_server + pd.to_timedelta(ed_sec, 's') df = df[(df.timestamp >= st) & (df.timestamp <= ed)] df = df[df.Sess.isin(fair[fair.No.isin(jnos)].Sess.unique())] sess2Model = fair[fair.No.isin(jnos)].set_index('Sess').Model.str.split( '.', expand=True) sess2Model['Label'] = sess2Model.apply( lambda x: '#{}: {}'.format(x[3], x[0]), axis=1) df['Sess'] = df.Sess.map(sess2Model.Label) #colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2'] with plt.style.context(['seaborn-paper', 'mypaper', 'line12']): fig, ax = plt.subplots() cm.plot_all(df, offset=offset_server, ax=ax, markevery=0.05, linestyle='-', linewidth=.75) ax.set_ylim(bottom=0) #axs[1].legend().remove() # Use axs[0]'s legend, but put it at axs[1]'s legend's place fig.subplots_adjust(bottom=0.15, hspace=0.05) ax.legend( #*axs[0].get_legend_handles_labels(), loc="lower center", frameon=False, bbox_to_anchor=[0.5, 1], #bbox_transform=fig.transFigure, fontsize='x-small', ncol=4 #mode='expand' ) ax.xaxis.set_major_locator(pu.MaxNLocator(nbins=5)) ax.minorticks_off() pu.cleanup_axis_bytes(ax.yaxis, maxN=5) ax.set_ylabel('Memory Usage') ax.set_xlabel('Time (s)') #fig.tight_layout() fig.set_size_inches(6.5, 1.6, forward=True) fig.savefig('/tmp/workspace/card274-fair.pdf', dpi=300, bbox_inches='tight', pad_inches=.015)
def do_srtf(path): srtf = load_data(path / 'card272' / 'case5' / 'salus', 'case5.output') srtf_refine = ju.load_refine(path / 'card272' / 'case5' / 'salus') df = cm.load_mem(path / 'card274/srtf/alloc.output') offset_local = srtf.Queued.min() # timeline logs in US/Eastern, but server logs in UTC # convert offset from US/Eastern to UTC offset_server = offset_local.tz_localize('US/Eastern').tz_convert( 'UTC').tz_localize(None) # select sess name after calc the offset # use jno = [13, 14, 15] jnos = [13, 14, 15] srtf = srtf[srtf.No.isin(jnos)] #srtf_refine = srtf_refine[srtf_refine.No.isin(jnos)] # select data in range st_sec = 400 ed_sec = 515 st = offset_server + pd.to_timedelta(st_sec, 's') ed = offset_server + pd.to_timedelta(ed_sec, 's') df = df[(df.timestamp >= st) & (df.timestamp <= ed)] df = df[df.Sess.isin(srtf.Sess.unique())] sess2Model = srtf.set_index('Sess').Model.str.split('.', expand=True) sess2Model['Label'] = sess2Model.apply( lambda x: '#{}: {}'.format(x[3], x[0]), axis=1) df['Sess'] = df.Sess.map(sess2Model.Label) #colors = ['#1f77b4', '#ff7f0e', '#2ca02c'] with plt.style.context(['seaborn-paper', 'mypaper', 'color3']): fig, axs = plt.subplots(nrows=2, sharex=True, gridspec_kw={'height_ratios': [1, 4]}) _, _, sessProps = ju.plot_timeline(srtf.drop(['LaneId'], axis=1), ax=axs[0], linewidth=2.5, returnSessProps=True, offset=offset_local) ju.plot_refine(axs[0], srtf, srtf_refine, offset=offset_local) #axs[0].set_ylabel('Job No.') # srtf is selected by simply set axis limit, because they are not easy to cut in data axs[0].set_xlim([st_sec, ed_sec]) axs[0].set_ylim([12.45, 15.5]) #axs[0].set_ylabel('Job', rotation=0, labelpad=20) sessProps = { row.Label: sessProps[sess] for sess, row in sess2Model.iterrows() } cm.plot_all(df, offset=offset_server, ax=axs[1], sessProps=sessProps) axs[1].set_ylim(bottom=0) #axs[1].legend().remove() # Use axs[0]'s legend, but put it at axs[1]'s legend's place fig.subplots_adjust(bottom=0.15, hspace=0.05) axs[1].legend( #*axs[0].get_legend_handles_labels(), loc="lower center", frameon=False, bbox_to_anchor=[0.5, 1], #bbox_transform=fig.transFigure, fontsize='x-small', ncol=4 #mode='expand' ) axs[1].xaxis.set_major_locator(pu.MaxNLocator(nbins=5)) axs[1].minorticks_off() pu.cleanup_axis_bytes(axs[1].yaxis, maxN=5) axs[1].set_ylabel('Memory Usage') axs[1].set_xlabel('Time (s)') #fig.tight_layout() fig.set_size_inches(6.5, 2, forward=True) fig.savefig('/tmp/workspace/card274-srtf.pdf', dpi=300, bbox_inches='tight', pad_inches=.015)
def memory_usage(logs, iter_times=None, beginning=None, starts=None, ends=None, mem_type=None, unified_ylabel=False, smoother=None, xformatter=None, per_sess=False, show_avg=None, ax=None): if beginning is None: beginning = get_beginning(logs) if show_avg is None: show_avg = not per_sess # Prepare ticket -> session map ticket2sess = {} for l in logs: if l.type != 'mem_pre': continue if l.ticket not in ticket2sess: ticket2sess[l.ticket] = l.sess elif l.sess != ticket2sess[l.ticket]: print('WARNING: ticket {} reused: previous {}, now: {}'.format( l.ticket, ticket2sess[l.ticket], l.sess)) mem_usages = [ l for l in logs if l.type == 'mem_alloc' or l.type == 'mem_dealloc' ] mem_activities = [] for m in mem_usages: if m.addr == '0x0' or m.addr == '0': continue if m.type == 'mem_alloc': mem_activities.append({ 'timestamp': m.timestamp, 'size': m.size, 'mem_type': m.block['mem_type'], 'alloc_inst': m.block['alloc_inst'], 'session': ticket2sess[m.block['ticket']] }) elif m.type == 'mem_dealloc': mem_activities.append({ 'timestamp': m.timestamp, 'size': -m.size, 'mem_type': m.block['mem_type'], 'alloc_inst': m.block['alloc_inst'], 'session': ticket2sess[m.block['ticket']] }) else: raise ValueError("Unexpected value: ", m) df = pd.DataFrame(mem_activities) if len(df) == 0: return df, None if mem_type is not None: df = df[df['mem_type'] == mem_type] df = df.set_index('timestamp').sort_index() # Save beginning if beginning is None: beginning = df.index[0] nrows = len(df['mem_type'].unique()) if ax is None: fig, axs = plt.subplots(nrows=nrows, ncols=1, sharex=True, squeeze=False) # Make axes into a 1D array axs = axs.reshape(-1) else: axs = [ax] series = [] pending_avg = [] for (name, group), ax in zip(df.groupby('mem_type'), axs): if per_sess: sessionUsages = {} for k, gg in group.groupby('session'): sessionUsages[k] = gg['size'].cumsum() ss = pd.DataFrame(sessionUsages).fillna(method='ffill').fillna(0) else: ss = group['size'].cumsum() # Do range restriction rstarts = None rends = None # Restrict x axis to iteration timaxes, must be done after cumsum, otherwise there # will be negative number if iter_times is not None: rstarts = iter_times[0][0] rends = iter_times[-1][1] rstarts = starts if starts is not None else rstarts rends = ends if ends is not None else rends if rstarts is not None and rends is not None: ss = ss.loc[rstarts:rends] elif rstarts is not None: ss = ss.loc[rstarts:] elif rends is not None: ss = ss.loc[:rends] ss.index = ss.index - beginning if show_avg: ss2 = ss.resample('100us').interpolate(method='time') pending_avg.append((ss2.mean(), ax)) series.append(ss) if smoother: if show_avg: ss = smoother(ss, ss2) else: ss = smoother(ss) # Change to timedelta after iteration restriction. # for some reason slicing doesn't work on Timedeltas # Pandas doesn't support irregular Timedelta intervals on xaxis # see https://stackoverflow.com/questions/40621710/axis-interval-spacing-when-plotting-with-pandas-timedelta # and https://github.com/pandas-dev/pandas/issues/8711 ss.index = ss.index.astype(int) if per_sess: ss.plot.area(ax=ax, linewidth=0) else: ss.plot(ax=ax) ax.legend().remove() pu.cleanup_axis_bytes(ax.yaxis) if not unified_ylabel: ax.set_ylabel('Memory Usage') # Adjust x axis if unified_ylabel: axs[-1].xaxis.label.set_visible(False) else: axs[-1].set_xlabel('Time (s)') axs[-1].autoscale(axis='x') # xlim = axs[-1].get_xlim() axs[-1].set_xlim(left=0) pu.cleanup_axis_timedelta(axs[-1].xaxis, xformatter) # Draw avg line after adjust xaxis if show_avg: for d, ax in pending_avg: pu.axhlines(d, linestyle='--', ax=ax) def format_coord(x, y): return 'x={:.4f}, y={:.1f} MB'.format(x, y / 1024 / 1024) axs[-1].format_coord = format_coord fig = axs[-1].figure #fig.tight_layout() if unified_ylabel: fig.text(0.5, 0.02, 'Time (s)', ha='center') fig.text(0.02, 0.5, 'Memory Usage (bytes)', va='center', rotation='vertical') fig.subplots_adjust(left=0.1, bottom=0.13) return df, beginning, fig
def __init__(self, df, colormap, ptr2sess, label=None, doPreprocess=False, hdf=None): self.label = label if self.label is None: self.label = 'id = {index} timestamp = {timestamp}' self.colormap = colormap self.ptr2sess = ptr2sess self._do_preprocess = doPreprocess self._hdf = hdf if doPreprocess: self.data = preprocess_memmap2(df, self.colormap, self.ptr2sess) else: self.data = df self.fig = plt.gcf() self.fig.set_size_inches([25, 15], forward=True) gs = gridspec.GridSpec(2, 1, height_ratios=[20, 1]) self._ax = self.fig.add_subplot(gs[0, 0]) self._ax.set_ylim([-0.2, 0.2]) # create artists self._lc = draw_memmap('0x0, 17179869184, 0, 0;', ax=self._ax) self._text = self._ax.text(0.5, 0.8, "", ha='center', va='baseline', fontsize=25, transform=self._ax.transAxes) # lc and texts for bins self._lcalloc = LineCollection([[(0, 0.1), (17179869184, 0.1)]], linewidths=10) self._ax.add_collection(self._lcalloc) self._textalloc = [] pu.cleanup_axis_bytes(self._ax.xaxis) init_frame = 0 self._step(init_frame) # session legend self._ax.legend(handles=[ mpatches.Patch(color=c, label=sess) for sess, c in self.colormap.items() ]) # add a slider axstep = self.fig.add_subplot(gs[1, 0], facecolor='lightgoldenrodyellow') self._stepSlider = Slider(axstep, 'ID', 0, len(self.data) - 1, valinit=init_frame, valfmt='%d', valstep=1, dragging=True) def update(val): self._step(int(val)) self.fig.canvas.draw_idle() self._stepSlider.on_changed(update) # timer self._timer = None # listen key press self.fig.canvas.mpl_connect('key_press_event', lambda evt: self._keydown(evt))