def main(positional_arguments): # Command-line arguments like '--distortions False' are equivalent to # '--distortions=True False', where False is a positional argument. To prevent # this from silently running with distortions, we do not allow positional # arguments. assert len(positional_arguments) >= 1 if len(positional_arguments) > 1: raise ValueError('Received unknown positional arguments: %s' % positional_arguments[1:]) options = make_options_from_flags(FLAGS) params = benchmark_cnn.make_params_from_flags() params = params._replace(batch_size=options.batch_size) params = params._replace(model='MY_GTSRB') params = params._replace(num_epochs=options.num_epochs) params = params._replace(num_gpus=options.num_gpus) params = params._replace(data_format='NHWC') params = params._replace(train_dir=options.checkpoint_folder) params = params._replace(allow_growth=True) params = params._replace(variable_update='replicated') params = params._replace(local_parameter_device='gpu') params = params._replace(use_tf_layers=False) # params = params._replace(all_reduce_spec='nccl') # params = params._replace(bottom_file=options.bottom_file) # params = params._replace(affine_files=options.affine_files) # params = params._replace(affine_classes=options.affine_classes) params = params._replace(optimizer=options.optimizer) params = params._replace(weight_decay=options.weight_decay) #params = params._replace(print_training_accuracy=True) params = params._replace(backbone_model_path=options.backbone_model_path) # Summary and Save & load checkpoints. # params = params._replace(summary_verbosity=1) # params = params._replace(save_summaries_steps=10) # params = params._replace(save_model_secs=3600) # save every 1 hour params = params._replace(save_model_secs=60) #save every 5 min params = benchmark_cnn.setup(params) #testtest(params) #exit(0) if 'test' in options.data_dir: dataset = GTSRBTestDataset(options) else: dataset = GTSRBDataset(options) model = Model_Builder(options.model_name, dataset.num_classes, options, params) bench = benchmark_cnn.BenchmarkCNN(params, dataset=dataset, model=model) tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) bench.print_info() bench.run() tf.reset_default_graph()
def main(positional_arguments): # Command-line arguments like '--distortions False' are equivalent to # '--distortions=True False', where False is a positional argument. To prevent # this from silently running with distortions, we do not allow positional # arguments. assert len(positional_arguments) >= 1 if len(positional_arguments) > 1: raise ValueError('Received unknown positional arguments: %s' % positional_arguments[1:]) params = benchmark_cnn.make_params_from_flags() params = benchmark_cnn.setup(params) bench = benchmark_cnn.BenchmarkCNN(params) tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) bench.print_info() with log_context(LOGGER_URL, LOGGER_USRENAME, LOGGER_PASSWORD, LOGGER_DB, LOGGER_SERIES, machine=LOGGER_VM): bench.run()
def main(positional_arguments): # Command-line arguments like '--distortions False' are equivalent to # '--distortions=True False', where False is a positional argument. To prevent # this from silently running with distortions, we do not allow positional # arguments. assert len(positional_arguments) >= 1 if len(positional_arguments) > 1: raise ValueError('Received unknown positional arguments: %s' % positional_arguments[1:]) params = benchmark_cnn.make_params_from_flags() params = benchmark_cnn.setup(params) import sys if params.enable_dmo == True: if LoadFileSystem() == False: sys.exit(-1) else : print("\n*******DMO enabled********\n") # sys.exit(0) bench = benchmark_cnn.BenchmarkCNN(params) tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) bench.print_info() bench.run()
def main(positional_arguments): # Command-line arguments like '--distortions False' are equivalent to # '--distortions=True False', where False is a positional argument. To prevent # this from silently running with distortions, we do not allow positional # arguments. assert len(positional_arguments) >= 1 if len(positional_arguments) > 1: raise ValueError('Received unknown positional arguments: %s' % positional_arguments[1:]) params = benchmark_cnn.make_params_from_flags() params = benchmark_cnn.setup(params) bench = benchmark_cnn.BenchmarkCNN(params) tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) bench.print_info() print('num_inter_threads: ' + str(params.num_inter_threads)) print('num_intra_threads: ' + str(params.num_intra_threads)) print('datasets_num_private_threads: ' + str(params.datasets_num_private_threads)) print('datasets_use_prefetch: ' + str(params.datasets_use_prefetch)) print('datasets_prefetch_buffer_size: ' + str(params.datasets_prefetch_buffer_size)) bench.run()
def main(positional_arguments): # Command-line arguments like '--distortions False' are equivalent to # '--distortions=True False', where False is a positional argument. To prevent # this from silently running with distortions, we do not allow positional # arguments. assert len(positional_arguments) >= 1 if len(positional_arguments) > 1: raise ValueError('Received unknown positional arguments: %s' % positional_arguments[1:]) params = benchmark_cnn.make_params_from_flags() # Print ENV Variables tf.logging.debug('=' * 20 + ' Environment Variables ' + '=' * 20) for k, v in os.environ.items(): tf.logging.debug('{}: {}'.format(k, v)) with mlperf.mlperf_logger(absl_flags.FLAGS.ml_perf_compliance_logging, params.model): params = benchmark_cnn.setup(params) bench = benchmark_cnn.BenchmarkCNN(params) tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) bench.print_info() bench.run()
def main(_): setup() bench = BenchmarkCNN() tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) bench.print_info() bench.run()
def main(_): params = benchmark_cnn.make_params_from_flags() params = benchmark_cnn.setup(params) bench = benchmark_cnn.BenchmarkCNN(params) tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) bench.print_info() bench.run()
def main(_): # Build benchmark_cnn model params = benchmark_cnn.make_params_from_flags() params, sess_config = benchmark_cnn.setup(params) bench = benchmark_cnn.BenchmarkCNN(params) # Print informaton tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) bench.print_info() # Build single-GPU benchmark_cnn model with tf.Graph().as_default() as single_gpu_graph: bench.build_model() def run(sess, num_iters, tensor_or_op_name_to_replica_names, num_workers, worker_id, num_replicas_per_worker): fetches = { 'global_step': tensor_or_op_name_to_replica_names[bench.global_step.name][0], 'cost': tensor_or_op_name_to_replica_names[bench.cost.name][0], 'train_op': tensor_or_op_name_to_replica_names[bench.train_op.name][0], } if isinstance(bench.lr, tf.Tensor): fetches['lr'] = tensor_or_op_name_to_replica_names[ bench.lr.name][0] start = time.time() for i in range(num_iters): results = sess.run(fetches) if i % FLAGS.log_frequency == 0: end = time.time() throughput = float(FLAGS.log_frequency) / float(end - start) parallax.log.info( "global step: %d, lr: %f, loss: %f, " "throughput: %f steps/sec" % (results['global_step'], results['lr'] if 'lr' in results else bench.lr, results['cost'], throughput)) start = time.time() config = parallax_config.build_config() config.sess_config = sess_config parallax.parallel_run(single_gpu_graph, run, FLAGS.resource_info_file, FLAGS.max_steps, sync=FLAGS.sync, parallax_config=config)
def main(positional_arguments): assert len(positional_arguments) >= 1 if len(positional_arguments) > 1: raise ValueError('Received unknown positional arguments: %s' % positional_arguments[1:]) options = make_options_from_flags(FLAGS) params = benchmark_cnn.make_params_from_flags() params = params._replace(batch_size=options.batch_size) params = params._replace(model='MY_GTSRB') params = params._replace(num_epochs=options.num_epochs) params = params._replace(num_gpus=options.num_gpus) params = params._replace(data_format='NHWC') params = params._replace(train_dir=options.checkpoint_folder) params = params._replace(allow_growth=True) params = params._replace(variable_update='replicated') params = params._replace(local_parameter_device='gpu') params = params._replace(use_tf_layers=False) # params = params._replace(all_reduce_spec='nccl') # params = params._replace(bottom_file=options.bottom_file) # params = params._replace(affine_files=options.affine_files) # params = params._replace(affine_classes=options.affine_classes) params = params._replace(optimizer=options.optimizer) params = params._replace(weight_decay=options.weight_decay) params = params._replace(print_training_accuracy=True) params = params._replace(backbone_model_path=options.backbone_model_path) # Summary and Save & load checkpoints. # params = params._replace(summary_verbosity=1) # params = params._replace(save_summaries_steps=10) params = params._replace(save_model_secs=3600) # save every 1 hour # params = params._replace(save_model_secs=300) #save every 5 min params = benchmark_cnn.setup(params) dataset = CifarDataset(options) model = Model_Builder(options.model_name, dataset.num_classes, options, params) bench = benchmark_cnn.BenchmarkCNN(params, dataset=dataset, model=model) tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) bench.print_info() bench.run()
def main(positional_arguments): # Command-line arguments like '--distortions False' are equivalent to # '--distortions=True False', where False is a positional argument. To prevent # this from silently running with distortions, we do not allow positional # arguments. assert len(positional_arguments) >= 1 if len(positional_arguments) > 1: raise ValueError('Received unknown positional arguments: %s' % positional_arguments[1:]) params = benchmark_cnn.make_params_from_flags() params = benchmark_cnn.setup(params) bench = benchmark_cnn.BenchmarkCNN(params) tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) run_benchmark(bench, absl_flags.FLAGS.iters_per_step)
def main(_): if FLAGS.winograd_nonfused: os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1' else: os.environ.pop('TF_ENABLE_WINOGRAD_NONFUSED', None) if FLAGS.autotune_threshold: os.environ['TF_AUTOTUNE_THRESHOLD'] = str(FLAGS.autotune_threshold) os.environ['TF_SYNC_ON_FINISH'] = str(int(FLAGS.sync_on_finish)) argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) bench = BenchmarkCNN() tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) bench.print_info() bench.run()
def main(extra_flags): # extra_flags is a list of command line arguments, excluding those defined # in tf.flags.FLAGS. extra_flags[0] is always the program name. It is an error # to supply flags not defined with tf.flags.FLAGS, so we raise an ValueError # in that case. assert len(extra_flags) >= 1 if len(extra_flags) > 1: raise ValueError('Received unknown flags: %s' % extra_flags[1:]) params = benchmark_cnn.make_params_from_flags() benchmark_cnn.setup(params) bench = benchmark_cnn.BenchmarkCNN(params) tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) bench.print_info() bench.run()
def main(positional_arguments): # Command-line arguments like '--distortions False' are equivalent to # '--distortions=True False', where False is a positional argument. To prevent # this from silently running with distortions, we do not allow positional # arguments. assert len(positional_arguments) >= 1 if len(positional_arguments) > 1: raise ValueError("Received unknown positional arguments: %s" % positional_arguments[1:]) params = benchmark_cnn.make_params_from_flags() with mlperf.mlperf_logger(absl_flags.FLAGS.ml_perf_compliance_logging, params.model): params = benchmark_cnn.setup(params) bench = benchmark_cnn.BenchmarkCNN(params) tfversion = cnn_util.tensorflow_version_tuple() log_fn("TensorFlow: %i.%i" % (tfversion[0], tfversion[1])) bench.print_info() bench.run()
def main(_): # Build benchmark_cnn model params = benchmark_cnn.make_params_from_flags() params, sess_config = benchmark_cnn.setup(params) bench = benchmark_cnn.BenchmarkCNN(params) # Print informaton tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) bench.print_info() # Build single-GPU benchmark_cnn model single_gpu_graph = tf.Graph() with single_gpu_graph.as_default(): bench.build_model() config = parallax_config.build_config() config.sess_config = sess_config sess, num_workers, worker_id, num_replicas_per_worker = \ parallax.parallel_run(single_gpu_graph, FLAGS.resource_info_file, sync=FLAGS.sync, parallax_config=config) fetches = { 'global_step': bench.global_step, 'cost': bench.cost, 'train_op': bench.train_op, } start = time.time() for i in range(FLAGS.max_steps): results = sess.run(fetches) if (i + 1) % FLAGS.log_frequency == 0: end = time.time() throughput = float(FLAGS.log_frequency) / float(end - start) parallax.log.info( "global step: %d, loss: %f, throughput: %f steps/sec" % (results['global_step'][0] + 1, results['cost'][0], throughput)) start = time.time()
def train(train_args): """ Train network train_args : dict Json dict with the user's configuration parameters. Can be loaded with json.loads() or with yaml.safe_load() """ run_results = { "status": "ok", "user_args": train_args, "machine_config": {}, "training": {}, "evaluation": {} } # Remove possible existing model and log files for f in os.listdir(cfg.MODELS_DIR): file_path = os.path.join(cfg.MODELS_DIR, f) try: if os.path.isfile(file_path): os.unlink(file_path) except Exception as e: print(e) # Declare training arguments kwargs = { 'model': yaml.safe_load(train_args.model).split(' ')[0], 'num_gpus': yaml.safe_load(train_args.num_gpus), 'num_epochs': yaml.safe_load(train_args.num_epochs), 'batch_size': yaml.safe_load(train_args.batch_size_per_device), 'optimizer': yaml.safe_load(train_args.optimizer), 'local_parameter_device': 'cpu', 'variable_update': 'parameter_server' } # Locate training data and check if the selected network fits it # For real data check whether the right data was mounted to the right place and if not download it (cifar10 only) if yaml.safe_load(train_args.dataset) != 'Synthetic data': data_name = yaml.safe_load(train_args.dataset) if data_name == 'cifar10': locate_cifar10() if data_name == 'imagenet': locate_imagenet() kwargs['data_name'] = data_name if data_name == 'imagenet_mini': locate_imagenet_mini() kwargs['data_name'] = 'imagenet' verify_selected_model(kwargs['model'], kwargs['data_name']) kwargs['data_dir'] = '{}/{}'.format(cfg.DATA_DIR, data_name) else: verify_selected_model(kwargs['model'], 'imagenet') # If no GPU is available or the gpu option is set to 0 run CPU mode if num_local_gpus == 0 or kwargs['num_gpus'] == 0: kwargs['device'] = 'cpu' kwargs['data_format'] = 'NHWC' # cpu data format kwargs[ 'num_gpus'] = 1 # Important: tensorflow uses this also to specify the number of CPUs else: kwargs['device'] = 'gpu' kwargs['data_format'] = 'NCHW' # Add training info to run_results but not the directories run_results["training"].update(kwargs) if run_results["training"]["device"] == "cpu": del run_results["training"]["num_gpus"] # avoid misleading info kwargs['train_dir'] = cfg.MODELS_DIR kwargs['benchmark_log_dir'] = cfg.MODELS_DIR # Setup and run the benchmark model params = benchmark.make_params(**kwargs) try: params = benchmark.setup(params) bench = benchmark.BenchmarkCNN(params) except ValueError as param_ex: raise BadRequest( "ValueError in parameter setup: {}. Params: {}".format( param_ex, params)) tf_version = '.'.join( [str(x) for x in cnn_util.tensorflow_version_tuple()]) run_results["training"]["tf_version"] = tf_version # Run benchmark and measure total execution time bench.print_info() start_time_global = datetime.datetime.now().strftime(time_fmt) try: bench.run() except ValueError as ve: raise BadRequest('ValueError in benchmark execution: {}'.format(ve)) end_time_global = datetime.datetime.now().strftime(time_fmt) # Read training and metric log files and store training results training_file = '{}/training.log'.format(cfg.MODELS_DIR) os.rename('{}/benchmark_run.log'.format(cfg.MODELS_DIR), training_file) run_parameters, machine_config = parse_logfile_training(training_file) run_results['training'].update(run_parameters) run_results["machine_config"] = machine_config metric_file = '{}/metric.log'.format(cfg.MODELS_DIR) run_results['training']['result'] = {} run_results['training']['result']['global_start_time'] = start_time_global run_results['training']['result']['global_end_time'] = end_time_global start, end, avg_examples = parse_metric_file(metric_file) run_results["training"]["result"][ "average_examples_per_sec"] = avg_examples run_results['training']['result']['execution_start_time'] = start run_results['training']['result']['execution_end_time'] = end ## Evaluation ## if yaml.safe_load(train_args.evaluation): run_results["evaluation"] = {} kwargs_eval = { 'model': kwargs['model'], 'num_gpus': kwargs['num_gpus'], 'device': kwargs['device'], 'data_format': kwargs['data_format'], 'benchmark_log_dir': kwargs['benchmark_log_dir'], 'train_dir': kwargs['train_dir'], 'eval': True # 'eval_dir': cfg.DATA_DIR, } run_results['evaluation']['device'] = kwargs_eval['device'] if run_results['evaluation']['device'] == 'gpu': run_results['evaluation']['num_gpus'] = kwargs_eval[ 'num_gpus'] # only for GPU to avoid confusion # Locate data if yaml.safe_load(train_args.dataset) != 'Synthetic data': kwargs_eval['data_name'] = kwargs['data_name'] kwargs_eval['data_dir'] = kwargs['data_dir'] # Setup and run the evaluation params_eval = benchmark.make_params(**kwargs_eval) try: params_eval = benchmark.setup(params_eval) evaluation = benchmark.BenchmarkCNN(params_eval) except ValueError as param_ex: raise BadRequest("ValueError: {}".format(param_ex)) evaluation.print_info() start_time_global = datetime.datetime.now().strftime(time_fmt) evaluation.run() end_time_global = datetime.datetime.now().strftime(time_fmt) # Read log files and get evaluation results os.rename('{}/benchmark_run.log'.format(cfg.MODELS_DIR), '{}/evaluation.log'.format(cfg.MODELS_DIR)) evaluation_file = '{}/evaluation.log'.format(cfg.MODELS_DIR) run_parameters = parse_logfile_evaluation(evaluation_file) run_results['evaluation'].update(run_parameters) logfile = '{}/metric.log'.format(cfg.MODELS_DIR) run_results['evaluation']['result'] = {} run_results['evaluation']['result'][ 'global_start_time'] = start_time_global run_results['evaluation']['result'][ 'global_end_time'] = end_time_global with open(logfile, "r") as f: for line in f: l = json.loads(line) if l["name"] == "eval_average_examples_per_sec": run_results["evaluation"]['result'][ "average_examples_per_sec"] = l["value"] if l["name"] == "eval_top_1_accuracy": run_results["evaluation"]['result']["top_1_accuracy"] = l[ "value"] if l["name"] == "eval_top_5_accuracy": run_results["evaluation"]['result']["top_5_accuracy"] = l[ "value"] return run_results
def train(**train_kwargs): """ Train network train_args : dict """ print("[DEBUG] train(**train_kwargs) - train_kwargs: %s" % (train_kwargs)) if cfg.DEBUG_MODEL else '' # use the schema schema = cfg.get_train_args_schema() # deserialize key-word arguments train_args = schema.load(train_kwargs) train_keys = train_args.keys() # log the dataset name # dataset options: ['synthetic_data', 'imagnet_mini', 'imagenet'] dataset_name = 'synthetic_data' if cfg.BENCHMARK_TYPE == 'benchmark': if train_args['flavor'] == 'synthetic': benchmark_flavor = 'synthetic' dataset_name = 'synthetic_data' if train_args['flavor'] == 'dataset': benchmark_flavor = 'dataset' dataset_name = 'imagenet_mini' if cfg.BENCHMARK_TYPE == 'pro': benchmark_flavor = 'pro' dataset_name = train_args['dataset'] # log the Tensorflow version tf_version = '.'.join( [str(x) for x in cnn_util.tensorflow_version_tuple()]) # Declare training arguments for tf_cnn_benchmarks. # Defaults are from config.py kwargs = {} kwargs[ 'num_gpus'] = train_args['num_gpus'] if 'num_gpus' in train_keys else 0 kwargs['num_epochs'] = (train_args['num_epochs'] if 'num_epochs' in train_keys else cfg.NUM_EPOCHS) kwargs['optimizer'] = (train_args['optimizer'] if 'optimizer' in train_keys else cfg.OPTIMIZER) kwargs['use_fp16'] = (train_args['use_fp16'] if 'use_fp16' in train_keys else cfg.USE_FP16) kwargs['local_parameter_device'] = 'cpu' kwargs['variable_update'] = 'parameter_server' kwargs['allow_growth'] = True kwargs['print_training_accuracy'] = True # how often print training info kwargs['display_every'] = 10 if kwargs['num_epochs'] < 1.0 else 100 # If no GPU is available or the gpu option is set to 0, run CPU mode if num_local_gpus == 0 or kwargs['num_gpus'] == 0: kwargs['device'] = 'cpu' kwargs['data_format'] = 'NHWC' # cpu data format kwargs[ 'num_gpus'] = 1 # Important: tensorflow uses this also to specify the number of CPUs else: kwargs['device'] = 'gpu' kwargs['data_format'] = 'NCHW' if dataset_name != 'synthetic_data': kwargs['data_name'] = (dataset_name if dataset_name != 'imagenet_mini' else 'imagenet') kwargs['data_dir'] = os.path.join(cfg.DATA_DIR, dataset_name) # Log training info configured for benchmark_cnn in the run_results run_results = { 'machine_config': {}, 'benchmark': { 'version': get_metadata()['Version'], 'flavor': benchmark_flavor, 'docker_base_image': cfg.DOCKER_BASE_IMAGE, 'dataset': dataset_name, 'tf_version': tf_version }, 'training': { 'num_gpus': 0, 'optimizer': '', 'use_fp16': '', 'local_parameter_device': '', 'variable_update': '', 'allow_growth': '', 'device': '', 'data_format': '', 'models': [] }, } # Update run_results with values configured for tf_cnn_benchmarks (kwargs) results_train_keys = run_results["training"].keys() kwargs_keys = kwargs.keys() for key in results_train_keys: if key in kwargs_keys: run_results['training'][key] = kwargs[key] # In kwargs num_gpus=1 also for CPU, update num_gpus in run_results to 0 if run_results["training"]["device"] == "cpu": run_results["training"]["num_gpus"] = 0 # avoid misleading info # Log information about the machine (CPU, GPU, memory): run_results['machine_config'] = mutils.get_machine_config() # Let's measure the total time, including download of data start_time_global = datetime.datetime.now().strftime(cfg.TIME_FORMAT) # Locate training dataset # For real data, check whether the data was mounted to the right place # and if not, download it (imagenet_mini, cifar10, NOT imagenet!) if dataset_name == 'cifar10': mutils.locate_cifar10() if dataset_name == 'imagenet_mini': mutils.locate_imagenet_mini() if dataset_name == 'imagenet': mutils.locate_imagenet() if cfg.BENCHMARK_TYPE == 'pro': train_pro.train(train_args, kwargs, run_results) else: train_sd.train(kwargs, run_results) end_time_global = datetime.datetime.now().strftime(cfg.TIME_FORMAT) run_results['global_start_time'] = start_time_global run_results['global_end_time'] = end_time_global end_time_global = mutils.timestr_to_stamp(end_time_global, cfg.TIME_FORMAT) start_time_global = mutils.timestr_to_stamp(start_time_global, cfg.TIME_FORMAT) run_results['global_execution_time_sec'] = (end_time_global - start_time_global) return run_results
def main(positional_arguments): # Command-line arguments like '--distortions False' are equivalent to # '--distortions=True False', where False is a positional argument. To prevent # this from silently running with distortions, we do not allow positional # arguments. # For DGX servers use hierarchical_copy=True argument assert len(positional_arguments) >= 1 if len(positional_arguments) > 1: raise ValueError('Received unknown positional arguments: %s' % positional_arguments[1:]) tests_models = [ { 'num_gpus': None, 'batch_size': 64, 'variable_update': 'parameter_server', 'model': 'inception3' }, { 'num_gpus': None, 'batch_size': 64, 'variable_update': 'parameter_server', 'model': 'resnet50' }, { 'num_gpus': None, 'batch_size': 32, 'variable_update': 'parameter_server', 'model': 'resnet152' }, #batch=64 crashes { 'num_gpus': None, 'batch_size': 64, 'variable_update': 'replicated', 'model': 'vgg16' }, { 'num_gpus': None, 'batch_size': 512, 'variable_update': 'replicated', 'model': 'alexnet' } ] test_gpus = [1, 2, 4, 8] stats = [] for test in tests_models: for num_gpus in test_gpus: test['num_gpus'] = num_gpus params = benchmark_cnn.make_params_from_flags() params = benchmark_cnn.setup(params) # force --hierarchical_copy to False when using 1 GPU if num_gpus == 1: params = params._replace(hierarchical_copy=False) params = params._replace(num_gpus=test['num_gpus'], batch_size=test['batch_size'], model=test['model'], variable_update=test['variable_update']) bench = benchmark_cnn.BenchmarkCNN(params) tfversion = cnn_util.tensorflow_version_tuple() log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) bench.print_info() results = bench.run() # result # { # 'average_wall_time': 0.6646941304206848, # 'images_per_sec': 385.1395525908701, # 'last_average_loss': 7.256145, # 'num_steps': 100, # 'num_workers': 1 # } stats.append({'test': test.copy(), 'result': results}) # summary print('summary:') print('==========') pprint.pprint(stats) print('==========') s = '' for i in range(len(test_gpus)): for j in range(len(tests_models)): s += str(stats[i + j * len(test_gpus)]['result']['images_per_sec']) s += ', ' s += '\n' print(s) print('==========')