def test_gru_fewer_parameters_than_lstm(self):
def _gru_fn():
return tf.keras.layers.GRU(10, return_sequences=True)
def _lstm_fn():
return tf.keras.layers.LSTM(10, return_sequences=True)
gru_model = models.create_recurrent_model(10, 10, 1, _gru_fn, 'gru')
lstm_model = models.create_recurrent_model(10, 10, 1, _lstm_fn, 'lstm')
self.assertLess(gru_model.count_params(), lstm_model.count_params())
def test_gru_constructs(self):
def _recurrent_layer_fn():
return tf.keras.layers.GRU(10, return_sequences=True)
model = models.create_recurrent_model(10, 10, 2, _recurrent_layer_fn,
'rnn-gru')
self.assertIsInstance(model, tf.keras.Model)
self.assertEqual('rnn-gru', model.name)
def run_experiment():
"""Runs the training experiment."""
vocab = _create_vocab()
(stackoverflow_train, stackoverflow_val,
stackoverflow_test) = construct_word_level_datasets(vocab)
num_training_steps = FLAGS.num_training_examples / FLAGS.batch_size
def _lstm_fn():
return tf.keras.layers.LSTM(FLAGS.latent_size, return_sequences=True)
model = models.create_recurrent_model(FLAGS.vocab_size,
FLAGS.embedding_size,
FLAGS.num_layers, _lstm_fn,
'stackoverflow-lstm')
model.compile(loss=tf.keras.losses.sparse_categorical_crossentropy,
optimizer=tf.keras.optimizers.SGD(
learning_rate=FLAGS.learning_rate,
use_momentum=FLAGS.use_momentum),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
train_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'train_results.csv')
test_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'test_results.csv')
train_csv_logger = tf.keras.callbacks.CSVLogger(train_results_path)
test_csv_logger = tf.keras.callbacks.CSVLogger(test_results_path)
model.fit(stackoverflow_train,
steps_per_epoch=num_training_steps,
epochs=25,
verbose=1,
validation_data=stackoverflow_val,
callbacks=[train_csv_logger])
score = model.evaluate_generator(stackoverflow_test,
verbose=1,
callbacks=[test_csv_logger])
print('Final test loss: %.4f' % score[0])
print('Final test accuracy: %.4f' % score[1])
def run_experiment():
"""Runs the training experiment."""
try:
tf.io.gfile.makedirs(
os.path.join(FLAGS.root_output_dir, FLAGS.exp_name))
except tf.errors.OpError:
pass
train_set, validation_set, test_set = (
dataset.construct_word_level_datasets(
vocab_size=FLAGS.vocab_size,
client_batch_size=FLAGS.batch_size,
client_epochs_per_round=1,
max_seq_len=FLAGS.sequence_length,
max_elements_per_user=FLAGS.max_elements_per_user,
centralized_train=True,
shuffle_buffer_size=None,
num_validation_examples=FLAGS.num_validation_examples,
num_test_examples=FLAGS.num_test_examples))
recurrent_model = tf.keras.layers.LSTM if FLAGS.lstm else tf.keras.layers.GRU
def _layer_fn():
return recurrent_model(FLAGS.latent_size, return_sequences=True)
pad, oov, _, eos = dataset.get_special_tokens(FLAGS.vocab_size)
model = models.create_recurrent_model(
FLAGS.vocab_size,
FLAGS.embedding_size,
FLAGS.num_layers,
_layer_fn,
'stackoverflow-recurrent',
shared_embedding=FLAGS.shared_embedding)
logging.info('Training model: %s', model.summary())
optimizer = utils_impl.create_optimizer_from_flags('centralized')
model.compile(
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
optimizer=optimizer,
metrics=[
metrics.MaskedCategoricalAccuracy([pad], 'accuracy_with_oov'),
metrics.MaskedCategoricalAccuracy([pad, oov], 'accuracy_no_oov'),
metrics.MaskedCategoricalAccuracy([pad, oov, eos],
'accuracy_no_oov_no_eos')
])
train_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'train_results')
test_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'test_results')
train_csv_logger = AtomicCSVLogger(train_results_path)
test_csv_logger = AtomicCSVLogger(test_results_path)
log_dir = os.path.join(FLAGS.root_output_dir, 'logdir', FLAGS.exp_name)
try:
tf.io.gfile.makedirs(log_dir)
tf.io.gfile.makedirs(train_results_path)
tf.io.gfile.makedirs(test_results_path)
except tf.errors.OpError:
pass # log_dir already exists.
train_tensorboard_callback = tf.keras.callbacks.TensorBoard(
log_dir=log_dir,
write_graph=True,
update_freq=FLAGS.tensorboard_update_frequency)
test_tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir)
results_file = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'results.csv.bz2')
# Write the hyperparameters to a CSV:
hparam_dict = collections.OrderedDict([(name, FLAGS[name].value)
for name in hparam_flags])
hparam_dict['results_file'] = results_file
hparams_file = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'hparams.csv')
utils_impl.atomic_write_to_csv(pd.Series(hparam_dict), hparams_file)
model.fit(train_set,
epochs=FLAGS.epochs,
verbose=1,
steps_per_epoch=FLAGS.steps_per_epoch,
validation_data=validation_set,
callbacks=[train_csv_logger, train_tensorboard_callback])
score = model.evaluate(
test_set,
verbose=1,
callbacks=[test_csv_logger, test_tensorboard_callback])
logging.info('Final test loss: %.4f', score[0])
logging.info('Final test accuracy: %.4f', score[1])
def run_experiment():
"""Runs the training experiment."""
vocab = _create_vocab()
(stackoverflow_train, stackoverflow_val,
stackoverflow_test) = construct_word_level_datasets(vocab)
num_training_steps = FLAGS.num_training_examples / FLAGS.batch_size
def _lstm_fn():
return tf.keras.layers.LSTM(FLAGS.latent_size, return_sequences=True)
model = models.create_recurrent_model(FLAGS.vocab_size,
FLAGS.embedding_size,
FLAGS.num_layers, _lstm_fn,
'stackoverflow-lstm')
if FLAGS.optimizer == 'sgd':
optimizer = tf.keras.optimizers.SGD(learning_rate=FLAGS.learning_rate,
momentum=FLAGS.momentum)
if FLAGS.optimizer == 'adagrad':
optimizer = tf.keras.optimizers.Adagrad(
learning_rate=FLAGS.learning_rate)
model.compile(loss=tf.keras.losses.sparse_categorical_crossentropy,
optimizer=optimizer,
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
train_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'train_results')
test_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'test_results')
train_csv_logger = AtomicCSVLogger(train_results_path)
test_csv_logger = AtomicCSVLogger(test_results_path)
log_dir = os.path.join(FLAGS.root_output_dir, 'logdir', FLAGS.exp_name)
try:
tf.io.gfile.makedirs(log_dir)
tf.io.gfile.makedirs(train_results_path)
tf.io.gfile.makedirs(test_results_path)
except tf.errors.OpError:
pass # log_dir already exists.
tensorboard_callback = tf.keras.callbacks.TensorBoard(
log_dir=log_dir,
write_graph=True,
update_freq=FLAGS.tensorboard_update_frequency)
results_file = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'results.csv.bz2')
# Write the hyperparameters to a CSV:
hparam_dict = collections.OrderedDict([(name, FLAGS[name].value)
for name in hparam_flags])
hparam_dict['results_file'] = results_file
hparams_file = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'hparams.csv')
utils_impl.atomic_write_to_csv(pd.Series(hparam_dict), hparams_file)
model.fit(stackoverflow_train,
steps_per_epoch=num_training_steps,
epochs=25,
verbose=1,
validation_data=stackoverflow_val,
callbacks=[train_csv_logger, tensorboard_callback])
score = model.evaluate_generator(
stackoverflow_test,
verbose=1,
callbacks=[test_csv_logger, tensorboard_callback])
print('Final test loss: %.4f' % score[0])
print('Final test accuracy: %.4f' % score[1])
def run_experiment():
"""Runs the training experiment."""
training_set, validation_set, test_set = (
dataset.construct_word_level_datasets(
vocab_size=FLAGS.vocab_size,
batch_size=FLAGS.batch_size,
client_epochs_per_round=1,
max_seq_len=FLAGS.sequence_length,
max_training_elements_per_user=-1,
num_validation_examples=FLAGS.num_validation_examples,
num_test_examples=FLAGS.num_test_examples))
centralized_train = training_set.create_tf_dataset_from_all_clients()
def _lstm_fn():
return tf.keras.layers.LSTM(FLAGS.latent_size, return_sequences=True)
model = models.create_recurrent_model(
FLAGS.vocab_size,
FLAGS.embedding_size,
FLAGS.num_layers,
_lstm_fn,
'stackoverflow-lstm',
shared_embedding=FLAGS.shared_embedding)
logging.info('Training model: %s', model.summary())
optimizer = utils_impl.create_optimizer_from_flags('centralized')
model.compile(loss=tf.keras.losses.sparse_categorical_crossentropy,
optimizer=optimizer,
weighted_metrics=['acc'])
train_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'train_results')
test_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'test_results')
train_csv_logger = AtomicCSVLogger(train_results_path)
test_csv_logger = AtomicCSVLogger(test_results_path)
log_dir = os.path.join(FLAGS.root_output_dir, 'logdir', FLAGS.exp_name)
try:
tf.io.gfile.makedirs(log_dir)
tf.io.gfile.makedirs(train_results_path)
tf.io.gfile.makedirs(test_results_path)
except tf.errors.OpError:
pass # log_dir already exists.
train_tensorboard_callback = tf.keras.callbacks.TensorBoard(
log_dir=log_dir,
write_graph=True,
update_freq=FLAGS.tensorboard_update_frequency)
test_tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir)
results_file = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'results.csv.bz2')
# Write the hyperparameters to a CSV:
hparam_dict = collections.OrderedDict([(name, FLAGS[name].value)
for name in hparam_flags])
hparam_dict['results_file'] = results_file
hparams_file = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name,
'hparams.csv')
utils_impl.atomic_write_to_csv(pd.Series(hparam_dict), hparams_file)
oov, bos, eos, pad = dataset.get_special_tokens(FLAGS.vocab_size)
class_weight = {x: 1.0 for x in range(FLAGS.vocab_size)}
class_weight[oov] = 0.0 # No credit for predicting OOV.
class_weight[bos] = 0.0 # Shouldn't matter since this is never a target.
class_weight[eos] = 1.0 # Model should learn to predict end of sentence.
class_weight[pad] = 0.0 # No credit for predicting pad.
model.fit(centralized_train,
epochs=FLAGS.epochs,
verbose=1,
class_weight=class_weight,
validation_data=validation_set,
callbacks=[train_csv_logger, train_tensorboard_callback])
score = model.evaluate(
test_set,
verbose=1,
callbacks=[test_csv_logger, test_tensorboard_callback])
logging.info('Final test loss: %.4f', score[0])
logging.info('Final test accuracy: %.4f', score[1])
def test_dense_fn_raises(self):
def _dense_layer_fn():
return tf.keras.layers.Dense(10)
with self.assertRaisesRegex(ValueError, 'tf.keras.layers.RNN'):
models.create_recurrent_model(10, 10, 1, _dense_layer_fn, 'dense')