def do_mem(logdir, network, batch_size):
"""Do basic JCT on workload"""
batch_num = 20
if network == "speech":
batch_num = 5
logger.info(f'Measuring memory for {network}_{batch_size} for {batch_num} iter')
ex = "salus" if FLAGS.use_salus else "tf"
final_dst = logdir / ex / WTL.from_name(network).canonical_name(RunConfig(batch_size, batch_num, None))
with atomic_directory(final_dst) as outputdir:
if not FLAGS.use_salus:
logger.info(' Running on TF')
wl = WTL.create(network, batch_size, batch_num, Executor.TF)
wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '1'
wl.env['TF_CPP_MIN_LOG_LEVEL'] = ''
run_tf(outputdir, wl)
# filter and move file to a more convinent name
for f in pathlib.Path(outputdir).iterdir():
with f.with_name('alloc.output').open('w') as file:
grep = execute(['egrep', r"] (\+|-)", f.name], stdout=file, cwd=str(f.parent))
grep.wait()
f.unlink()
break
else:
scfg = maybe_forced_preset(presets.AllocProf)
scfg.logconf = "memop"
scfg.output_dir = outputdir
server = SalusServer(scfg)
with server.run():
logger.info(' Running on Salus')
WTL.block_run(network, batch_size, batch_num, Executor.Salus, outputdir / 'rpc.output')
return final_dst
def case2():
scfg = maybe_forced_preset(presets.OpTracing)
scfg.logconf = 'memop'
scfg.env['TF_CPP_MIN_VLOG_LEVEL'] = '1'
scfg.env['TF_CPP_MIN_LOG_LEVEL'] = ''
scfg.save_outerr = True
run_seq(scfg.copy(output_dir=FLAGS.save_dir/'case2'),
WTL.create("inception3", 100, 20),
WTL.create("resnet50", 50, 20))
# filter the TF allocator output
f = FLAGS.save_dir/'case1'/'server.stderr'
with f.with_name('tfalloc.output').open('w') as file:
grep = execute(['egrep', r"] (\+|-)", f.name], stdout=file, cwd=str(f.parent))
grep.wait()
def test():
"""Measure TF inference memory usage"""
wl = WTL.create("inception3eval", 50, 50, executor=Executor.TF)
wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '1'
wl.env['TF_CPP_MIN_LOG_LEVEL'] = ''
outdir = FLAGS.save_dir/'test'/wl.canonical_name
run_tf(outdir, wl)
# filter and move file to a more convinent name
for f in outdir.iterdir():
with f.with_name('alloc.output').open('w') as file:
grep = execute(['egrep', r"] (\+|-)", f.name], stdout=file, cwd=str(f.parent))
grep.wait()
f.unlink()
break
def do_mem(logdir, network, batch_size):
"""Do basic JCT on workload"""
batch_num = 10
logger.info(f'Saving model checkpoint for {network}_{batch_size} for {batch_num} iter')
final_dst = logdir / 'tf' / WTL.from_name(network).canonical_name(RunConfig(batch_size, batch_num, None))
with atomic_directory(final_dst) as outputdir:
logger.info(' Running on TF')
wl = WTL.create(network, batch_size, batch_num, Executor.TF)
wl.env['SALUS_SAVE_MODEL'] = '1'
run_tf(outputdir, wl)
# filter and move file to a more convinent name
for f in pathlib.Path(outputdir).iterdir():
with f.with_name('alloc.output').open('w') as file:
grep = execute(['egrep', r"] (\+|-)", f.name], stdout=file, cwd=str(f.parent))
grep.wait()
f.unlink()
break
return final_dst