def benchmark_optimizer(self, optimizer, num_iters):
"""Optimizer benchmark with Bidirectional LSTM model on IMDB data.
Args:
optimizer: The optimizer instance to be benchmarked.
num_iters: The number of iterations to run for performance measurement.
"""
model, train_x, train_y = bidirect_imdb_lstm_config()
metrics, wall_time, extras = benchmark_util.measure_performance(
model,
x=train_x,
y=train_y,
batch_size=512,
optimizer=optimizer,
loss="binary_crossentropy",
metrics=["accuracy"])
name = benchmark_util.get_benchmark_name(self._get_name())
metadata = {
"implementation": name[0],
"model_name": "optimizers",
"parameters": "lstm.512",
}
extras.update(metadata)
self.report_benchmark(iters=num_iters,
wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_cnn_cifar10_bs_1024_gpu_2(self):
"""Measure performance with batch_size=1024, gpu=2 and
distribution_strategy=`mirrored`.
"""
batch_size = 1024
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
num_gpus=2,
distribution_strategy='mirrored',
epochs=self.epochs,
optimizer=tf.keras.optimizers.RMSprop(learning_rate=0.0001,
decay=1e-6),
loss='categorical_crossentropy',
metrics=['accuracy'])
metadata = benchmark_util.get_keras_examples_metadata(
'cnn', batch_size)
extras.update(metadata)
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_irnn_mnist_bs_1024(self):
"""Measure performance with batch_size=1024."""
batch_size = 1024
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
optimizer=tf.keras.optimizers.RMSprop(learning_rate=self.learning_rate),
loss='categorical_crossentropy',
metrics=['accuracy'])
self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
def benchmark_mnist_convnet(self, batch_size, run_iters):
"""Benchmark for Convnet model on synthetic mnist data."""
convnet_x = np.random.random((5000, 28, 28, 1))
convnet_y = np.random.random((5000, 10))
metrics, wall_time, extras = benchmark_util.measure_performance(
self._mnist_convnet,
x=convnet_x,
y=convnet_y,
batch_size=batch_size,
run_iters=run_iters,
optimizer=_OPTIMIZER,
loss=_LOSS)
self.report_benchmark(
iters=run_iters, wall_time=wall_time, metrics=metrics, extras=extras)
def benchmark_imdb_lstm(self, batch_size, run_iters):
"""Benchmark for LSTM model on synthetic imdb review dataset."""
lstm_x = np.random.randint(0, 1999, size=(2500, 100))
lstm_y = np.random.random((2500, 1))
metrics, wall_time, extras = benchmark_util.measure_performance(
self._imdb_lstm,
x=lstm_x,
y=lstm_y,
batch_size=batch_size,
run_iters=run_iters,
optimizer=_OPTIMIZER,
loss=_LOSS)
self.report_benchmark(
iters=run_iters, wall_time=wall_time, metrics=metrics, extras=extras)
def benchmark_mlp_reuters_bs_512(self):
"""Measure performance with batch_size=512."""
batch_size = 512
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
epochs=self.epochs,
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
def benchmark_mnist_mlp(self, batch_size, run_iters):
"""Benchmark for MLP model on synthetic mnist data."""
mlp_x = np.random.random((5000, 784))
mlp_y = np.random.random((5000, 10))
metrics, wall_time, extras = benchmark_util.measure_performance(
self._mnist_mlp,
x=mlp_x,
y=mlp_y,
batch_size=batch_size,
run_iters=run_iters,
optimizer=_OPTIMIZER,
loss=_LOSS)
self.report_benchmark(
iters=run_iters, wall_time=wall_time, metrics=metrics, extras=extras)
def benchmark_bidirect_lstm_imdb_bs_512(self):
"""Measure performance with batch_size=512."""
batch_size = 512
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.imdb_x,
y=self.imdb_y,
batch_size=batch_size,
optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_hrnn_mnist_bs_256(self):
"""Measure performance with batch_size=256."""
batch_size = 256
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_text_classification_bs_512(self):
"""Measure performance with batch_size=512."""
batch_size = 512
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.imdb_x,
y=self.imdb_y,
batch_size=batch_size,
optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_text_classification_bs_256(self):
"""Measure performance with batch_size=256."""
batch_size = 256
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.imdb_x,
y=self.imdb_y,
batch_size=batch_size,
optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
metadata = benchmark_util.get_keras_examples_metadata(
'transformer', batch_size)
extras.update(metadata)
self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
def benchmark_antirectifier_bs_512(self):
"""Measure performance with batch_size=512."""
batch_size = 512
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
optimizer="rmsprop",
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=["sparse_categorical_accuracy"])
metadata = benchmark_util.get_keras_examples_metadata(
"antirectifier", batch_size)
extras.update(metadata)
self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
def benchmark_antirectifier_bs_256(self):
"""Measure performance with batch_size=256."""
batch_size = 256
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
optimizer="rmsprop",
loss=tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True),
metrics=["sparse_categorical_accuracy"])
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_conv_mnist_bs_256(self):
"""Measure performance with batch_size=256 and run_iters=3."""
batch_size = 256
run_iters = 3
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
run_iters=run_iters,
epochs=self.epochs,
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
self.report_benchmark(
iters=run_iters, wall_time=wall_time, metrics=metrics, extras=extras)
def benchmark_cnn_cifar10_bs_512(self):
"""Measure performance with batch_size=512."""
batch_size = 512
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
epochs=self.epochs,
optimizer=tf.keras.optimizers.RMSprop(learning_rate=0.0001,
decay=1e-6),
loss='categorical_crossentropy',
metrics=['accuracy'])
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_hrnn_mnist_bs_1024(self):
"""Measure performance with batch_size=1024."""
batch_size = 1024
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
metadata = benchmark_util.get_keras_examples_metadata(
'hierarchical_rnn', batch_size)
extras.update(metadata)
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_bidirect_lstm_imdb_bs_256(self):
"""Measure performance with batch_size=256."""
batch_size = 256
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.imdb_x,
y=self.imdb_y,
batch_size=batch_size,
optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
metadata = benchmark_util.get_keras_examples_metadata(
'bidirectional_lstm', batch_size)
extras.update(metadata)
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_mlp_reuters_bs_256(self):
"""Measure performance with batch_size=256."""
batch_size = 256
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
epochs=self.epochs,
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
metadata = benchmark_util.get_keras_examples_metadata(
'mlp', batch_size)
extras.update(metadata)
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_irnn_mnist_bs_512(self):
"""Measure performance with batch_size=512."""
batch_size = 512
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
optimizer=tf.keras.optimizers.RMSprop(
learning_rate=self.learning_rate),
loss='categorical_crossentropy',
metrics=['accuracy'])
metadata = benchmark_util.get_keras_examples_metadata(
'irnn', batch_size)
extras.update(metadata)
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_pixel_cnn_bs_512(self):
"""Measure performance with batch_size=512 and run_iters=4."""
batch_size = 512
run_iters = 4
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
run_iters=run_iters,
optimizer="rmsprop",
loss=tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True),
metrics=["sparse_categorical_accuracy"])
self.report_benchmark(iters=run_iters,
wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_bidirect_lstm_imdb_bs_512_gpu_2(self):
"""Measure performance with batch_size=512, gpu=2 and
distribution_strategy=`mirrored`.
"""
batch_size = 512
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.imdb_x,
y=self.imdb_y,
batch_size=batch_size,
num_gpus=2,
distribution_strategy='mirrored',
optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_hrnn_mnist_bs_1024_gpu_2(self):
"""Measure performance with batch_size=1024, gpu=2 and
distribution_strategy='mirrored'
"""
batch_size = 1024
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
num_gpus=2,
distribution_strategy='mirrored',
optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_bidirect_lstm_imdb(self, batch_size, run_iters):
"""Benchmark for Bidirectional LSTM on IMDB."""
# Load dataset.
(x_train,
y_train), _ = tf.keras.datasets.imdb.load_data(num_words=_MAX_FEATURE)
x_train = tf.keras.preprocessing.sequence.pad_sequences(
x_train, maxlen=_MAX_LEN)
results = benchmark_util.measure_performance(
self._lstm_imdb_model,
x=x_train,
y=y_train,
batch_size=batch_size,
run_iters=run_iters,
optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
self.report_benchmark(iters=run_iters,
wall_time=results['wall_time'],
extras=results)
def benchmark_text_classification_bs_512_gpu_2(self):
"""Measure performance with batch_size=512, gpu=1 and
distribution_strategy='mirrored'
"""
batch_size = 512
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.imdb_x,
y=self.imdb_y,
batch_size=batch_size,
num_gpus=2,
distribution_strategy='mirrored',
optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
metadata = benchmark_util.get_keras_examples_metadata(
'transformer', batch_size)
extras.update(metadata)
self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
def benchmark_antirectifier_bs_512_gpu_2(self):
"""Measure performance with batch_size=512, gpu=2 and
distribution_strategy=`mirrored`.
"""
batch_size = 512
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
num_gpus=2,
distribution_strategy="mirrored",
optimizer="rmsprop",
loss=tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True),
metrics=["sparse_categorical_accuracy"])
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_conv_mnist_bs_256_gpu_2(self):
"""Measure performance with batch_size=256, run_iters=3, gpu=2 and
distribution_strategy='mirrored'
"""
batch_size = 256
run_iters = 3
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
run_iters=run_iters,
num_gpus=2,
distribution_strategy='mirrored',
epochs=self.epochs,
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
self.report_benchmark(
iters=run_iters, wall_time=wall_time, metrics=metrics, extras=extras)
def benchmark_text_classification_bs_512_gpu_2(self):
"""Measure performance with batch_size=512, run_iters=4, gpu=1 and
distribution_strategy='mirrored'
"""
batch_size = 512
run_iters = 4
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.imdb_x,
y=self.imdb_y,
batch_size=batch_size,
run_iters=run_iters,
num_gpus=2,
distribution_strategy='mirrored',
optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
self.report_benchmark(iters=run_iters,
wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_irnn_mnist_bs_1024_gpu_3(self):
"""Measure performance with batch_size=1024, run_iters=3, gpu=3 and
distribution_strategy='mirrored'
"""
batch_size = 1024
run_iters = 3
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
run_iters=run_iters,
num_gpus=3,
distribution_strategy='mirrored',
optimizer=tf.keras.optimizers.RMSprop(
learning_rate=self.learning_rate),
loss='categorical_crossentropy',
metrics=['accuracy'])
self.report_benchmark(iters=run_iters,
wall_time=wall_time,
metrics=metrics,
extras=extras)
def benchmark_mlp_reuters_bs_512_gpu_2(self):
"""Measure performance with batch_size=512, gpu=2 and
distribution_strategy='mirrored'
"""
batch_size = 512
metrics, wall_time, extras = benchmark_util.measure_performance(
self._build_model,
x=self.x_train,
y=self.y_train,
batch_size=batch_size,
num_gpus=2,
distribution_strategy='mirrored',
epochs=self.epochs,
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
metadata = benchmark_util.get_keras_examples_metadata(
'mlp', batch_size)
extras.update(metadata)
self.report_benchmark(wall_time=wall_time,
metrics=metrics,
extras=extras)