def tfmps(argv):
# type: (Sequence[str]) -> None
batch_sizes = [int(v) for v in argv[1:]]
if not batch_sizes:
batch_sizes = [1, 2, 4, 8]
for idx, bs in enumerate(batch_sizes):
with tempfile.TemporaryDirectory() as td:
# create a background training job
train_wl, pipetrain = create_train(Executor.TF, idx, td)
train_wl.extra_args += ['--min_mem']
# create the foreground inference job
wl, pipe = create_infer(Executor.TF, bs, td)
wl.extra_args += ['--min_mem']
run_tf(FLAGS.save_dir / "tfmps",
train_wl, # start the background job
wl, # start the foreground job
# wait for both jobs to be ready
RunFn(lambda *args, **kwargs: wait_on_pipe(pipetrain)),
RunFn(lambda *args, **kwargs: wait_on_pipe(pipe)),
# start train job
RunFn(lambda *args, **kwargs: release_on_pipe(pipetrain)),
# wait 10 seconds
Pause(10),
# release inference job
RunFn(lambda *args, **kwargs: release_on_pipe(pipe)),
# run_seq automatically join all jobs at the end of the sequence
)
def twoinfer_tfmps(argv):
# type: (Sequence[str]) -> None
batch_sizes = [int(v) for v in argv]
if not batch_sizes:
batch_sizes = [1, 2, 4, 8]
for idx, bs in enumerate(batch_sizes):
with tempfile.TemporaryDirectory() as td:
# create the foreground inference job
wl1, pipe1 = create_infer(Executor.TF, bs, td)
wl1.extra_args += ['--min_mem']
# create the foreground inference job
wl2, pipe2 = create_infer(Executor.TF, bs, td)
wl2.extra_args += ['--min_mem']
run_tf(FLAGS.save_dir / "twoinfer" / "tfmps",
wl1, # start the background job
wl2, # start the foreground job
# wait for both jobs to be ready
RunFn(lambda *args, **kwargs: wait_on_pipe(pipe1)),
RunFn(lambda *args, **kwargs: wait_on_pipe(pipe2)),
# start train job
RunFn(lambda *args, **kwargs: release_on_pipe(pipe1)),
# release inference job
RunFn(lambda *args, **kwargs: release_on_pipe(pipe2)),
# run_seq automatically join all jobs at the end of the sequence
)
def do_mem(logdir, network, batch_size):
"""Do basic JCT on workload"""
batch_num = 20
if network == "speech":
batch_num = 5
logger.info(f'Measuring memory for {network}_{batch_size} for {batch_num} iter')
ex = "salus" if FLAGS.use_salus else "tf"
final_dst = logdir / ex / WTL.from_name(network).canonical_name(RunConfig(batch_size, batch_num, None))
with atomic_directory(final_dst) as outputdir:
if not FLAGS.use_salus:
logger.info(' Running on TF')
wl = WTL.create(network, batch_size, batch_num, Executor.TF)
wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '1'
wl.env['TF_CPP_MIN_LOG_LEVEL'] = ''
run_tf(outputdir, wl)
# filter and move file to a more convinent name
for f in pathlib.Path(outputdir).iterdir():
with f.with_name('alloc.output').open('w') as file:
grep = execute(['egrep', r"] (\+|-)", f.name], stdout=file, cwd=str(f.parent))
grep.wait()
f.unlink()
break
else:
scfg = maybe_forced_preset(presets.AllocProf)
scfg.logconf = "memop"
scfg.output_dir = outputdir
server = SalusServer(scfg)
with server.run():
logger.info(' Running on Salus')
WTL.block_run(network, batch_size, batch_num, Executor.Salus, outputdir / 'rpc.output')
return final_dst
def main(argv):
scfg = maybe_forced_preset(presets.OpTracing)
name, bs, bn = 'vgg11', 25, 10
if len(argv) > 0:
name = argv[0]
if len(argv) > 1:
bs = argv[1]
bs = try_with_default(int, bs, ValueError)(bs)
if len(argv) > 2:
bn = int(argv[2])
def create_wl(ex):
return WTL.create(name, bs, bn, executor=ex)
# Run on Salus
wl = create_wl(Executor.Salus)
run_seq(scfg.copy(output_dir=FLAGS.save_dir / "salus" / '1'), wl)
if FLAGS.also_tf:
wl = create_wl(Executor.TF)
wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '1'
wl.env['TF_CPP_MIN_LOG_LEVEL'] = ''
run_tf(FLAGS.save_dir / "tf", wl)
# filter and move file to a more convinent name
for f in (FLAGS.save_dir / "tf").iterdir():
f.rename(f.with_name('perf.output'))
break
def main(argv):
scfg = maybe_forced_preset(presets.OpTracing)
name, bs = 'vgg11', 25
if len(argv) > 0:
name = argv[0]
if len(argv) > 1:
bs = int(argv[1])
def create_wl(ex):
return WTL.create(name, bs, 10, executor=ex)
# Run on TF
wl = create_wl(Executor.TF)
wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '2'
wl.env['TF_CPP_MIN_LOG_LEVEL'] = ''
run_tf(FLAGS.save_dir / 'tf', wl)
# Run on Salus
wl = create_wl(Executor.Salus)
run_seq(scfg.copy(output_dir=FLAGS.save_dir / "salus" / '1'), wl)
return
# Run 2 on Salus
run_seq(scfg.copy(output_dir=FLAGS.save_dir / "salus" / '2'),
create_wl(Executor.Salus),
create_wl(Executor.Salus),
)
def main(argv):
scfg = maybe_forced_preset(presets.AllocProf)
if argv:
run_seq(scfg.copy(output_dir=FLAGS.save_dir),
*parse_actions_from_cmd(argv))
return
run_seq(scfg.copy(output_dir=FLAGS.save_dir),
WTL.create("alexnet", 25, 200),
WTL.create("alexnet", 25, 200),
WTL.create("alexnet", 25, 200),
WTL.create("alexnet", 25, 200),
WTL.create("alexnet", 25, 200),
)
run_tf(FLAGS.save_dir,
WTL.create("alexnet", 25, 200, executor=Executor.TF),
Pause.Wait,
WTL.create("alexnet", 25, 200, executor=Executor.TF),
Pause.Wait,
WTL.create("alexnet", 25, 200, executor=Executor.TF),
Pause.Wait,
WTL.create("alexnet", 25, 200, executor=Executor.TF),
Pause.Wait,
WTL.create("alexnet", 25, 200, executor=Executor.TF),
)
def case1():
scfg = maybe_forced_preset(presets.MostEfficient)
run_tf(FLAGS.save_dir/'case1',
WTL.create("inception3eval", 1, 1000, executor=Executor.TF))
run_seq(scfg.copy(output_dir=FLAGS.save_dir/'case1'),
WTL.create("inception3eval", 1, 1000))
def alone_tf(_argv):
# run tf
# the foreground inference job
wl = create_infer(Executor.TF, 10)
wl.extra_args += ['--min_mem']
run_tf(FLAGS.save_dir / "alone", wl)
# the background training job
wl = create_train(Executor.TF, 0)
wl.extra_args += ['--min_mem']
run_tf(FLAGS.save_dir / "alone", wl)
def case2():
# Run on TF
wl = WTL.create("super_res", 128, 20, executor=Executor.TF)
wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '2'
wl.env['TF_CPP_MIN_LOG_LEVEL'] = ''
run_tf(FLAGS.save_dir/'case2'/'tf', wl)
scfg = maybe_forced_preset(presets.OpTracing)
scfg.logconf = 'memop'
run_seq(scfg.copy(output_dir=FLAGS.save_dir/'case2'/'salus'),
WTL.create("super_res", 128, 20))
def tfmps2(argv):
# type: (Sequence[str]) -> None
name = "alexneteval"
if len(argv) > 1:
name = argv[0]
batch_sizes = [int(v) for v in argv[1:]]
if not batch_sizes:
batch_sizes = [1, 2, 4, 8]
batch_num = 300
# batch_sizes = [1, 2, 4, 8, 16, 32]
# batch_sizes = [1024, 1536, 2048, 4096]
for idx, bs in enumerate(batch_sizes):
with tempfile.TemporaryDirectory() as td:
# create a background training job, the batch number has no effect here,
# only used to distinguish different runs
trainWl = WTL.create('inception4', 50, 100 + idx, executor=Executor.TF)
# make sure it runs long enough
trainWl.env['SALUS_ITER_SECONDS'] = '300'
trainWl.extra_args += ['--min_mem']
# create a pipe to signal trainWl
pipetrain = str(pathlib.Path(td).joinpath('fifotrain'))
os.mkfifo(pipetrain)
trainWl.env['SALUS_WAIT_FOR_SIGNAL'] = pipetrain
# create the foreground inference job
wl = WTL.create(name, bs, batch_num, executor=Executor.TF)
set_env(wl)
wl.env['SALUS_ITER_SECONDS'] = '150'
wl.extra_args += ['--min_mem']
pipe = str(pathlib.Path(td).joinpath('fifo'))
os.mkfifo(pipe)
wl.env['SALUS_WAIT_FOR_SIGNAL'] = pipe
run_tf(FLAGS.save_dir / "tfmps2" / (name + "-inception4"),
wl, # start the foreground job
Pause(20),
trainWl, # start the background job
# wait for both jobs to be ready
RunFn(lambda *args, **kwargs: wait_on_pipe(pipetrain)),
RunFn(lambda *args, **kwargs: wait_on_pipe(pipe)),
# start train job
RunFn(lambda *args, **kwargs: release_on_pipe(pipetrain)),
# wait 10 seconds
Pause(10),
# release inference job
RunFn(lambda *args, **kwargs: release_on_pipe(pipe)),
# run_seq automatically join all jobs at the end of the sequence
)
def main(argv):
wls = select_workloads(argv)
for wl in wls:
if wl.name in ['speech', 'mnistsf', 'mnistcv', 'mnistlg', 'seq2seq']:
continue
if (FLAGS.save_dir / wl.canonical_name).exists():
continue
wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '1'
wl.env['TF_CPP_MIN_LOG_LEVEL'] = ''
run_tf(FLAGS.save_dir / wl.canonical_name, wl)
def test():
"""Measure TF inference memory usage"""
wl = WTL.create("inception3eval", 50, 50, executor=Executor.TF)
wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '1'
wl.env['TF_CPP_MIN_LOG_LEVEL'] = ''
outdir = FLAGS.save_dir/'test'/wl.canonical_name
run_tf(outdir, wl)
# filter and move file to a more convinent name
for f in outdir.iterdir():
with f.with_name('alloc.output').open('w') as file:
grep = execute(['egrep', r"] (\+|-)", f.name], stdout=file, cwd=str(f.parent))
grep.wait()
f.unlink()
break
def main(argv):
name, bs, bn = 'vgg11', 25, 10
if len(argv) > 0:
name = argv[0]
if len(argv) > 1:
bs = argv[1]
bs = try_with_default(int, bs, ValueError)(bs)
if len(argv) > 2:
bn = int(argv[2])
def create_wl(ex):
return WTL.create(name, bs, bn, executor=ex)
wl = create_wl(Executor.TF)
# wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '1'
# wl.env['TF_CPP_MIN_LOG_LEVEL'] = ''
run_tf(FLAGS.save_dir / "tf", wl)
def do_mem(logdir, network, batch_size):
"""Do basic JCT on workload"""
batch_num = 10
logger.info(f'Saving model checkpoint for {network}_{batch_size} for {batch_num} iter')
final_dst = logdir / WTL.from_name(network).canonical_name(RunConfig(batch_size, batch_num, None))
with atomic_directory(final_dst) as outputdir:
logger.info(' Running on TF')
wl = WTL.create(network, batch_size, batch_num, Executor.TF)
wl.env['SALUS_SAVE_MODEL'] = '1'
model_dir = pathlib.Path('~/../symbiotic/peifeng/tf_cnn_benchmarks_models/legacy_checkpoint_models')
model_dir = model_dir.expanduser().resolve()
wl.env['SALUS_TFBENCH_EVAL_MODEL_DIR'] = str(model_dir)
run_tf(outputdir, wl)
return final_dst
def do_mem(logdir, network, batch_size):
"""Do basic JCT on workload"""
batch_num = 10
logger.info(f'Saving model checkpoint for {network}_{batch_size} for {batch_num} iter')
final_dst = logdir / 'tf' / WTL.from_name(network).canonical_name(RunConfig(batch_size, batch_num, None))
with atomic_directory(final_dst) as outputdir:
logger.info(' Running on TF')
wl = WTL.create(network, batch_size, batch_num, Executor.TF)
wl.env['SALUS_SAVE_MODEL'] = '1'
run_tf(outputdir, wl)
# filter and move file to a more convinent name
for f in pathlib.Path(outputdir).iterdir():
with f.with_name('alloc.output').open('w') as file:
grep = execute(['egrep', r"] (\+|-)", f.name], stdout=file, cwd=str(f.parent))
grep.wait()
f.unlink()
break
return final_dst
def main(argv):
# type: (Sequence[str]) -> None
model_dir = pathlib.Path('~/../symbiotic/peifeng/tf_cnn_benchmarks_models/legacy_checkpoint_models')
model_dir = model_dir.expanduser().resolve()
saved_model_dir = pathlib.Path('~/../symbiotic/peifeng/tf_cnn_benchmarks_models/saved_models')
saved_model_dir = saved_model_dir.expanduser().resolve()
for wl in select_workloads(argv, batch_size=1, batch_num=1, executor=Executor.TF):
if wl.wtl.runnerCls is not TFBenchmarkRunner:
logger.info(f'Skipping {wl.name}')
continue
if not wl.name.endswith('eval'):
logger.info(f'Skipping {wl.name}')
continue
logger.info(f"**** Saving SavedModel: {wl.canonical_name}")
logger.info(f"**** Location: {FLAGS.save_dir}")
wl.env['SALUS_TFBENCH_EVAL_MODEL_DIR'] = str(model_dir)
wl.env['SALUS_TFBENCH_EVAL_SAVED_MODEL_DIR'] = str(saved_model_dir)
run_tf(FLAGS.save_dir, wl)
def main(argv):
scfg = maybe_forced_preset(presets.MostEfficient)
scfg.scheduler = 'pack'
# Then run on tf
run_tf(FLAGS.save_dir / "resnet152_75-alone",
WTL.create("resnet152", 75, 11, executor=Executor.TF),
)
return
run_tf(FLAGS.save_dir / "resnet101_50-alone",
WTL.create("resnet101", 50, 31, executor=Executor.TF),
)
run_tf(FLAGS.save_dir / "resnet101_50",
WTL.create("resnet101", 50, 31, executor=Executor.TF),
WTL.create("resnet101", 50, 31, executor=Executor.TF),
)
run_tf(FLAGS.save_dir / "resnet152_75",
WTL.create("resnet152", 75, 22, executor=Executor.TF),
WTL.create("resnet152", 75, 22, executor=Executor.TF),
)
def main(argv):
scfg = maybe_forced_preset(presets.MostEfficient)
name, bs, bn = 'alexnet', 25, 2307
if len(argv) > 0:
name = argv[0]
if len(argv) > 1:
bs = int(argv[1])
if len(argv) > 2:
bn = int(argv[2])
def create_wl(ex):
wl = WTL.create(name, bs, bn, executor=ex)
if FLAGS.use_oc:
wl.env['TF_GPU_ALLOCATOR'] = 'cuda_managed'
return wl
if FLAGS.using_mps:
# Run 2 alexnet_25 on TF
logdirname = 'tf-mps-oc' if FLAGS.use_oc else 'tf-mps'
run_tf(FLAGS.save_dir / logdirname,
create_wl(Executor.TF),
create_wl(Executor.TF),
)
return
# Run alexnet_25 on Salus
wl = create_wl(Executor.Salus)
run_seq(scfg.copy(output_dir=FLAGS.save_dir / "salus" / '1'), wl)
# Run 2 alexnet_25 on Salus
run_seq(scfg.copy(output_dir=FLAGS.save_dir / "salus" / '2'),
create_wl(Executor.Salus),
create_wl(Executor.Salus),
)
# Run alexnet_25 on TF
wl = create_wl(Executor.TF)
run_tf(FLAGS.save_dir / 'tf', wl)
# Run 2 alexnet_25 on TF
run_tf(FLAGS.save_dir / 'tf-nomps',
create_wl(Executor.TF),
create_wl(Executor.TF),
)
def tfop(_):
wl = WTL.create("inception3", 100, 10, executor=Executor.TF)
wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '2'
wl.env['TF_CPP_MIN_LOG_LEVEL'] = ''
run_tf(FLAGS.save_dir / 'tfop', wl)
def case2(scfg):
run_tf(scfg.copy(output_dir=FLAGS.save_dir),
WTL.create("inception3", 100, 10, executor=Executor.TF))
