def testEvalResults(self): samples = resnet_benchmark.MakeSamplesFromEvalOutput( self.metadata_input, self.contents, 0) golden = [ Sample('Eval Loss', 3.86324, '', self.metadata_output), Sample('Top 1 Accuracy', 32.751465, '%', self.metadata_output), Sample('Top 5 Accuracy', 58.825684, '%', self.metadata_output) ] self.assertEqual(samples, golden)
def testEvalResults(self): samples = resnet_benchmark.MakeSamplesFromEvalOutput( self.metadata, self.contents, 0) golden = [ Sample( 'Eval Loss', 3.86324, '', {'epoch': 4.000479577971386, 'elapsed_seconds': 0, 'step': 5005, 'num_examples_per_epoch': 1251.1}), Sample( 'Top 1 Accuracy', 32.751465, '%', {'epoch': 4.000479577971386, 'elapsed_seconds': 0, 'step': 5005, 'num_examples_per_epoch': 1251.1}), Sample( 'Top 5 Accuracy', 58.825684, '%', {'epoch': 4.000479577971386, 'elapsed_seconds': 0, 'step': 5005, 'num_examples_per_epoch': 1251.1}) ] self.assertEqual(samples, golden)
def Run(benchmark_spec): """Run Inception V3 on the cluster. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. Returns: A list of sample.Sample objects. """ _UpdateBenchmarkSpecWithFlags(benchmark_spec) vm = benchmark_spec.vms[0] inception3_benchmark_script = ( 'tpu/models/experimental/inception/inception_v3.py') inception3_benchmark_cmd = ( '{env_cmd} && python {script} ' '--learning_rate={learning_rate} ' '--iterations={iterations} ' '--use_tpu={use_tpu} ' '--use_data={use_data} ' '--train_steps_per_eval={steps_per_eval} ' '--data_dir={data_dir} ' '--model_dir={model_dir} ' '--save_checkpoints_secs={save_checkpoints_secs} ' '--train_batch_size={train_batch_size} ' '--eval_batch_size={eval_batch_size} ' '--precision={precision}'.format( env_cmd=benchmark_spec.env_cmd, script=inception3_benchmark_script, learning_rate=benchmark_spec.learning_rate, iterations=benchmark_spec.iterations, use_tpu=bool(benchmark_spec.tpus), use_data=benchmark_spec.use_data, steps_per_eval=benchmark_spec.steps_per_eval, data_dir=benchmark_spec.data_dir, model_dir=benchmark_spec.model_dir, save_checkpoints_secs=benchmark_spec.save_checkpoints_secs, train_batch_size=benchmark_spec.train_batch_size, eval_batch_size=benchmark_spec.eval_batch_size, precision=benchmark_spec.precision)) if FLAGS.tf_device == 'gpu': inception3_benchmark_cmd = '{env} {cmd}'.format( env=tensorflow.GetEnvironmentVars(vm), cmd=inception3_benchmark_cmd) samples = [] metadata = _CreateMetadataDict(benchmark_spec) elapsed_seconds = 0 steps_per_eval = benchmark_spec.steps_per_eval train_steps = benchmark_spec.train_steps for step in range(steps_per_eval, train_steps + steps_per_eval, steps_per_eval): step = min(step, train_steps) inception3_benchmark_cmd_step = '{cmd} --train_steps={step}'.format( cmd=inception3_benchmark_cmd, step=step) if benchmark_spec.mode in ('train', 'train_and_eval'): if benchmark_spec.tpus: tpu = benchmark_spec.tpu_groups['train'].GetName() num_shards = '--num_shards={}'.format( benchmark_spec.tpu_groups['train'].GetNumShards()) else: tpu = num_shards = '' inception3_benchmark_train_cmd = ( '{cmd} --tpu={tpu} --mode=train {num_shards}'.format( cmd=inception3_benchmark_cmd_step, tpu=tpu, num_shards=num_shards)) start = time.time() stdout, stderr = vm.RobustRemoteCommand( inception3_benchmark_train_cmd, should_log=True) elapsed_seconds += (time.time() - start) samples.extend( mnist_benchmark.MakeSamplesFromTrainOutput( metadata, stdout + stderr, elapsed_seconds, step)) if benchmark_spec.mode in ('train_and_eval', 'eval'): if benchmark_spec.tpus: tpu = benchmark_spec.tpu_groups['eval'].GetName() num_shards = '--num_shards={}'.format( benchmark_spec.tpu_groups['eval'].GetNumShards()) else: tpu = num_shards = '' inception3_benchmark_eval_cmd = ( '{cmd} --tpu={tpu} --mode=eval {num_shards}'.format( cmd=inception3_benchmark_cmd_step, tpu=tpu, num_shards=num_shards)) stdout, stderr = vm.RobustRemoteCommand( inception3_benchmark_eval_cmd, should_log=True) samples.extend( resnet_benchmark.MakeSamplesFromEvalOutput( metadata, stdout + stderr, elapsed_seconds)) return samples