def tfmps(argv): # type: (Sequence[str]) -> None batch_sizes = [int(v) for v in argv[1:]] if not batch_sizes: batch_sizes = [1, 2, 4, 8] for idx, bs in enumerate(batch_sizes): with tempfile.TemporaryDirectory() as td: # create a background training job train_wl, pipetrain = create_train(Executor.TF, idx, td) train_wl.extra_args += ['--min_mem'] # create the foreground inference job wl, pipe = create_infer(Executor.TF, bs, td) wl.extra_args += ['--min_mem'] run_tf(FLAGS.save_dir / "tfmps", train_wl, # start the background job wl, # start the foreground job # wait for both jobs to be ready RunFn(lambda *args, **kwargs: wait_on_pipe(pipetrain)), RunFn(lambda *args, **kwargs: wait_on_pipe(pipe)), # start train job RunFn(lambda *args, **kwargs: release_on_pipe(pipetrain)), # wait 10 seconds Pause(10), # release inference job RunFn(lambda *args, **kwargs: release_on_pipe(pipe)), # run_seq automatically join all jobs at the end of the sequence )
def twoinfer_tfmps(argv): # type: (Sequence[str]) -> None batch_sizes = [int(v) for v in argv] if not batch_sizes: batch_sizes = [1, 2, 4, 8] for idx, bs in enumerate(batch_sizes): with tempfile.TemporaryDirectory() as td: # create the foreground inference job wl1, pipe1 = create_infer(Executor.TF, bs, td) wl1.extra_args += ['--min_mem'] # create the foreground inference job wl2, pipe2 = create_infer(Executor.TF, bs, td) wl2.extra_args += ['--min_mem'] run_tf(FLAGS.save_dir / "twoinfer" / "tfmps", wl1, # start the background job wl2, # start the foreground job # wait for both jobs to be ready RunFn(lambda *args, **kwargs: wait_on_pipe(pipe1)), RunFn(lambda *args, **kwargs: wait_on_pipe(pipe2)), # start train job RunFn(lambda *args, **kwargs: release_on_pipe(pipe1)), # release inference job RunFn(lambda *args, **kwargs: release_on_pipe(pipe2)), # run_seq automatically join all jobs at the end of the sequence )
def do_mem(logdir, network, batch_size): """Do basic JCT on workload""" batch_num = 20 if network == "speech": batch_num = 5 logger.info(f'Measuring memory for {network}_{batch_size} for {batch_num} iter') ex = "salus" if FLAGS.use_salus else "tf" final_dst = logdir / ex / WTL.from_name(network).canonical_name(RunConfig(batch_size, batch_num, None)) with atomic_directory(final_dst) as outputdir: if not FLAGS.use_salus: logger.info(' Running on TF') wl = WTL.create(network, batch_size, batch_num, Executor.TF) wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '1' wl.env['TF_CPP_MIN_LOG_LEVEL'] = '' run_tf(outputdir, wl) # filter and move file to a more convinent name for f in pathlib.Path(outputdir).iterdir(): with f.with_name('alloc.output').open('w') as file: grep = execute(['egrep', r"] (\+|-)", f.name], stdout=file, cwd=str(f.parent)) grep.wait() f.unlink() break else: scfg = maybe_forced_preset(presets.AllocProf) scfg.logconf = "memop" scfg.output_dir = outputdir server = SalusServer(scfg) with server.run(): logger.info(' Running on Salus') WTL.block_run(network, batch_size, batch_num, Executor.Salus, outputdir / 'rpc.output') return final_dst
def main(argv): scfg = maybe_forced_preset(presets.OpTracing) name, bs, bn = 'vgg11', 25, 10 if len(argv) > 0: name = argv[0] if len(argv) > 1: bs = argv[1] bs = try_with_default(int, bs, ValueError)(bs) if len(argv) > 2: bn = int(argv[2]) def create_wl(ex): return WTL.create(name, bs, bn, executor=ex) # Run on Salus wl = create_wl(Executor.Salus) run_seq(scfg.copy(output_dir=FLAGS.save_dir / "salus" / '1'), wl) if FLAGS.also_tf: wl = create_wl(Executor.TF) wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '1' wl.env['TF_CPP_MIN_LOG_LEVEL'] = '' run_tf(FLAGS.save_dir / "tf", wl) # filter and move file to a more convinent name for f in (FLAGS.save_dir / "tf").iterdir(): f.rename(f.with_name('perf.output')) break
def main(argv): scfg = maybe_forced_preset(presets.OpTracing) name, bs = 'vgg11', 25 if len(argv) > 0: name = argv[0] if len(argv) > 1: bs = int(argv[1]) def create_wl(ex): return WTL.create(name, bs, 10, executor=ex) # Run on TF wl = create_wl(Executor.TF) wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '2' wl.env['TF_CPP_MIN_LOG_LEVEL'] = '' run_tf(FLAGS.save_dir / 'tf', wl) # Run on Salus wl = create_wl(Executor.Salus) run_seq(scfg.copy(output_dir=FLAGS.save_dir / "salus" / '1'), wl) return # Run 2 on Salus run_seq(scfg.copy(output_dir=FLAGS.save_dir / "salus" / '2'), create_wl(Executor.Salus), create_wl(Executor.Salus), )
def main(argv): scfg = maybe_forced_preset(presets.AllocProf) if argv: run_seq(scfg.copy(output_dir=FLAGS.save_dir), *parse_actions_from_cmd(argv)) return run_seq(scfg.copy(output_dir=FLAGS.save_dir), WTL.create("alexnet", 25, 200), WTL.create("alexnet", 25, 200), WTL.create("alexnet", 25, 200), WTL.create("alexnet", 25, 200), WTL.create("alexnet", 25, 200), ) run_tf(FLAGS.save_dir, WTL.create("alexnet", 25, 200, executor=Executor.TF), Pause.Wait, WTL.create("alexnet", 25, 200, executor=Executor.TF), Pause.Wait, WTL.create("alexnet", 25, 200, executor=Executor.TF), Pause.Wait, WTL.create("alexnet", 25, 200, executor=Executor.TF), Pause.Wait, WTL.create("alexnet", 25, 200, executor=Executor.TF), )
def case1(): scfg = maybe_forced_preset(presets.MostEfficient) run_tf(FLAGS.save_dir/'case1', WTL.create("inception3eval", 1, 1000, executor=Executor.TF)) run_seq(scfg.copy(output_dir=FLAGS.save_dir/'case1'), WTL.create("inception3eval", 1, 1000))
def alone_tf(_argv): # run tf # the foreground inference job wl = create_infer(Executor.TF, 10) wl.extra_args += ['--min_mem'] run_tf(FLAGS.save_dir / "alone", wl) # the background training job wl = create_train(Executor.TF, 0) wl.extra_args += ['--min_mem'] run_tf(FLAGS.save_dir / "alone", wl)
def case2(): # Run on TF wl = WTL.create("super_res", 128, 20, executor=Executor.TF) wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '2' wl.env['TF_CPP_MIN_LOG_LEVEL'] = '' run_tf(FLAGS.save_dir/'case2'/'tf', wl) scfg = maybe_forced_preset(presets.OpTracing) scfg.logconf = 'memop' run_seq(scfg.copy(output_dir=FLAGS.save_dir/'case2'/'salus'), WTL.create("super_res", 128, 20))
def tfmps2(argv): # type: (Sequence[str]) -> None name = "alexneteval" if len(argv) > 1: name = argv[0] batch_sizes = [int(v) for v in argv[1:]] if not batch_sizes: batch_sizes = [1, 2, 4, 8] batch_num = 300 # batch_sizes = [1, 2, 4, 8, 16, 32] # batch_sizes = [1024, 1536, 2048, 4096] for idx, bs in enumerate(batch_sizes): with tempfile.TemporaryDirectory() as td: # create a background training job, the batch number has no effect here, # only used to distinguish different runs trainWl = WTL.create('inception4', 50, 100 + idx, executor=Executor.TF) # make sure it runs long enough trainWl.env['SALUS_ITER_SECONDS'] = '300' trainWl.extra_args += ['--min_mem'] # create a pipe to signal trainWl pipetrain = str(pathlib.Path(td).joinpath('fifotrain')) os.mkfifo(pipetrain) trainWl.env['SALUS_WAIT_FOR_SIGNAL'] = pipetrain # create the foreground inference job wl = WTL.create(name, bs, batch_num, executor=Executor.TF) set_env(wl) wl.env['SALUS_ITER_SECONDS'] = '150' wl.extra_args += ['--min_mem'] pipe = str(pathlib.Path(td).joinpath('fifo')) os.mkfifo(pipe) wl.env['SALUS_WAIT_FOR_SIGNAL'] = pipe run_tf(FLAGS.save_dir / "tfmps2" / (name + "-inception4"), wl, # start the foreground job Pause(20), trainWl, # start the background job # wait for both jobs to be ready RunFn(lambda *args, **kwargs: wait_on_pipe(pipetrain)), RunFn(lambda *args, **kwargs: wait_on_pipe(pipe)), # start train job RunFn(lambda *args, **kwargs: release_on_pipe(pipetrain)), # wait 10 seconds Pause(10), # release inference job RunFn(lambda *args, **kwargs: release_on_pipe(pipe)), # run_seq automatically join all jobs at the end of the sequence )
def main(argv): wls = select_workloads(argv) for wl in wls: if wl.name in ['speech', 'mnistsf', 'mnistcv', 'mnistlg', 'seq2seq']: continue if (FLAGS.save_dir / wl.canonical_name).exists(): continue wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '1' wl.env['TF_CPP_MIN_LOG_LEVEL'] = '' run_tf(FLAGS.save_dir / wl.canonical_name, wl)
def test(): """Measure TF inference memory usage""" wl = WTL.create("inception3eval", 50, 50, executor=Executor.TF) wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '1' wl.env['TF_CPP_MIN_LOG_LEVEL'] = '' outdir = FLAGS.save_dir/'test'/wl.canonical_name run_tf(outdir, wl) # filter and move file to a more convinent name for f in outdir.iterdir(): with f.with_name('alloc.output').open('w') as file: grep = execute(['egrep', r"] (\+|-)", f.name], stdout=file, cwd=str(f.parent)) grep.wait() f.unlink() break
def main(argv): name, bs, bn = 'vgg11', 25, 10 if len(argv) > 0: name = argv[0] if len(argv) > 1: bs = argv[1] bs = try_with_default(int, bs, ValueError)(bs) if len(argv) > 2: bn = int(argv[2]) def create_wl(ex): return WTL.create(name, bs, bn, executor=ex) wl = create_wl(Executor.TF) # wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '1' # wl.env['TF_CPP_MIN_LOG_LEVEL'] = '' run_tf(FLAGS.save_dir / "tf", wl)
def do_mem(logdir, network, batch_size): """Do basic JCT on workload""" batch_num = 10 logger.info(f'Saving model checkpoint for {network}_{batch_size} for {batch_num} iter') final_dst = logdir / WTL.from_name(network).canonical_name(RunConfig(batch_size, batch_num, None)) with atomic_directory(final_dst) as outputdir: logger.info(' Running on TF') wl = WTL.create(network, batch_size, batch_num, Executor.TF) wl.env['SALUS_SAVE_MODEL'] = '1' model_dir = pathlib.Path('~/../symbiotic/peifeng/tf_cnn_benchmarks_models/legacy_checkpoint_models') model_dir = model_dir.expanduser().resolve() wl.env['SALUS_TFBENCH_EVAL_MODEL_DIR'] = str(model_dir) run_tf(outputdir, wl) return final_dst
def do_mem(logdir, network, batch_size): """Do basic JCT on workload""" batch_num = 10 logger.info(f'Saving model checkpoint for {network}_{batch_size} for {batch_num} iter') final_dst = logdir / 'tf' / WTL.from_name(network).canonical_name(RunConfig(batch_size, batch_num, None)) with atomic_directory(final_dst) as outputdir: logger.info(' Running on TF') wl = WTL.create(network, batch_size, batch_num, Executor.TF) wl.env['SALUS_SAVE_MODEL'] = '1' run_tf(outputdir, wl) # filter and move file to a more convinent name for f in pathlib.Path(outputdir).iterdir(): with f.with_name('alloc.output').open('w') as file: grep = execute(['egrep', r"] (\+|-)", f.name], stdout=file, cwd=str(f.parent)) grep.wait() f.unlink() break return final_dst
def main(argv): # type: (Sequence[str]) -> None model_dir = pathlib.Path('~/../symbiotic/peifeng/tf_cnn_benchmarks_models/legacy_checkpoint_models') model_dir = model_dir.expanduser().resolve() saved_model_dir = pathlib.Path('~/../symbiotic/peifeng/tf_cnn_benchmarks_models/saved_models') saved_model_dir = saved_model_dir.expanduser().resolve() for wl in select_workloads(argv, batch_size=1, batch_num=1, executor=Executor.TF): if wl.wtl.runnerCls is not TFBenchmarkRunner: logger.info(f'Skipping {wl.name}') continue if not wl.name.endswith('eval'): logger.info(f'Skipping {wl.name}') continue logger.info(f"**** Saving SavedModel: {wl.canonical_name}") logger.info(f"**** Location: {FLAGS.save_dir}") wl.env['SALUS_TFBENCH_EVAL_MODEL_DIR'] = str(model_dir) wl.env['SALUS_TFBENCH_EVAL_SAVED_MODEL_DIR'] = str(saved_model_dir) run_tf(FLAGS.save_dir, wl)
def main(argv): scfg = maybe_forced_preset(presets.MostEfficient) scfg.scheduler = 'pack' # Then run on tf run_tf(FLAGS.save_dir / "resnet152_75-alone", WTL.create("resnet152", 75, 11, executor=Executor.TF), ) return run_tf(FLAGS.save_dir / "resnet101_50-alone", WTL.create("resnet101", 50, 31, executor=Executor.TF), ) run_tf(FLAGS.save_dir / "resnet101_50", WTL.create("resnet101", 50, 31, executor=Executor.TF), WTL.create("resnet101", 50, 31, executor=Executor.TF), ) run_tf(FLAGS.save_dir / "resnet152_75", WTL.create("resnet152", 75, 22, executor=Executor.TF), WTL.create("resnet152", 75, 22, executor=Executor.TF), )
def main(argv): scfg = maybe_forced_preset(presets.MostEfficient) name, bs, bn = 'alexnet', 25, 2307 if len(argv) > 0: name = argv[0] if len(argv) > 1: bs = int(argv[1]) if len(argv) > 2: bn = int(argv[2]) def create_wl(ex): wl = WTL.create(name, bs, bn, executor=ex) if FLAGS.use_oc: wl.env['TF_GPU_ALLOCATOR'] = 'cuda_managed' return wl if FLAGS.using_mps: # Run 2 alexnet_25 on TF logdirname = 'tf-mps-oc' if FLAGS.use_oc else 'tf-mps' run_tf(FLAGS.save_dir / logdirname, create_wl(Executor.TF), create_wl(Executor.TF), ) return # Run alexnet_25 on Salus wl = create_wl(Executor.Salus) run_seq(scfg.copy(output_dir=FLAGS.save_dir / "salus" / '1'), wl) # Run 2 alexnet_25 on Salus run_seq(scfg.copy(output_dir=FLAGS.save_dir / "salus" / '2'), create_wl(Executor.Salus), create_wl(Executor.Salus), ) # Run alexnet_25 on TF wl = create_wl(Executor.TF) run_tf(FLAGS.save_dir / 'tf', wl) # Run 2 alexnet_25 on TF run_tf(FLAGS.save_dir / 'tf-nomps', create_wl(Executor.TF), create_wl(Executor.TF), )
def tfop(_): wl = WTL.create("inception3", 100, 10, executor=Executor.TF) wl.env['TF_CPP_MIN_VLOG_LEVEL'] = '2' wl.env['TF_CPP_MIN_LOG_LEVEL'] = '' run_tf(FLAGS.save_dir / 'tfop', wl)
def case2(scfg): run_tf(scfg.copy(output_dir=FLAGS.save_dir), WTL.create("inception3", 100, 10, executor=Executor.TF))