def load_tf(path):
logs = pl.load_file(path)
df = pd.DataFrame(l.__dict__ for l in logs)
df = df[df.type.isin(tf_events)].drop(['level', 'loc', 'entry_type'],
axis=1)
# make sure step is int
df['step'] = df.step.astype(int)
# discard some internal or async op: _SOURCE, _Recv, _Send
ignored = ['_Recv', '_Send']
df = df[~df.kernel.isin(ignored)]
df = df[df.op != '_SOURCE']
steptf = df.pivot_table(values='timestamp',
index=['step', 'op', 'kernel'],
columns='type',
aggfunc='first').reset_index()
# add a name column
def name(row):
return '{}[{}]'.format(row['op'], row['kernel'])
steptf['name'] = steptf.apply(name, axis=1).values
# reorder
steptf = steptf[['step', 'name', 'op', 'kernel'] + tf_events]
return steptf.sort_values(by=tf_events).reset_index(drop=True)
def load_memmap(path):
logs = pl.load_file(path)
df = pd.DataFrame(l.__dict__ for l in logs)
df = df[df.type == 'memmap']
df = df.drop(['level', 'loc', 'entry_type', 'thread', 'type'], axis=1)
# make sure size is int
df['Size'] = df.Size.astype(int)
return df
def load_enditermap(path):
df = pd.DataFrame(l.__dict__ for l in pl.load_file(path))
#df = df[df.type == 'generic_evt']
df = df.query(
'type == "generic_evt" & (evt == "start_iter" | evt == "end_iter")')
df = df.drop(['level', 'loc', 'entry_type', 'thread', 'type'], axis=1)
df = df.dropna(how='all', axis=1)
# get only end_iter events for memmory map
return df[df.evt == 'end_iter']
def load_sessiter(path, return_logs=False, parallel_workers=0):
try:
logs = pl.load_file(path, parallel_workers)
except TypeError:
logs = path
df = pd.DataFrame(l.__dict__ for l in logs)
df = df[df.type == 'generic_evt']
df = df[df.evt.isin(['start_iter', 'end_iter'])]
df = df.drop(['entry_type', 'level', 'loc', 'thread'],
axis=1).dropna(axis=1)
enditers = df[df.evt == 'end_iter']
mainiters = df[df.GraphId.isin(enditers.GraphId)]
return mainiters
def load_failure(path):
logs = pl.load_file(str(path))
df = pd.DataFrame(l.__dict__ for l in logs)
df = df[df.type == 'optracing_evt']
df = df.drop(['entry_type', 'level', 'loc', 'thread', 'type'], axis=1)
# make sure step is int
df['step'] = df.step.astype(int)
ss = select_steps(df)
step25 = df[df.step.isin(ss)]
# discard unneeded event
step25 = step25[step25.evt == 'done']
# add a name column
def name(row):
return '{}[{}]'.format(row['op'], row['kernel'])
step25['name'] = step25.apply(name, axis=1).values
return step25
def wrapped(config):
local_dir = os.path.join(config.log_dir, name)
if not os.path.isdir(local_dir):
return None
log_file = os.path.join(local_dir, 'alloc.output')
iter_file = os.path.join(local_dir, 'mem-iter.output')
logs = None
iters = None
if mem:
try:
logs = pl.load_file(log_file)
except IOError:
pass
if os.path.isfile(iter_file):
sessstart, iters = pn.parse_iterations(iter_file)
fig = func(config, local_dir, logs, iters)
# fig.set_size_inches(2.35, 2.35, forward=True)
return fig
def load_salus(path, filter_step=True):
logs = pl.load_file(path)
df = pd.DataFrame(l.__dict__ for l in logs)
df = df[df.type == 'optracing_evt']
df = df.drop(['entry_type', 'level', 'loc', 'thread', 'type'], axis=1)
# make sure step is int
df['step'] = df.step.astype(int)
if filter_step:
ss = select_steps(df)
step25 = df[df.step.isin(ss)]
else:
step25 = df
# discard some internal or async op: _SOURCE, _Recv, _Send
ignored = ['_Recv', '_Send']
step25 = step25[~step25.kernel.isin(ignored)]
step25 = step25[step25.op != '_SOURCE']
# discard unneeded event
step25 = step25[step25.evt != 'scheduled']
# convert evt values to columns
step = step25.pivot_table(values='timestamp',
index=['step', 'op', 'kernel', 'sess'],
columns='evt',
aggfunc='last').reset_index()
# add a name column
def name(row):
return '{}[{}]'.format(row['op'], row['kernel'])
step['name'] = step.apply(name, axis=1).values
# reorder
step = step[['sess', 'step', 'name', 'op', 'kernel'] + salus_events]
# sort
return step.sort_values(by=salus_events).reset_index(drop=True)