def test_gru_fewer_parameters_than_lstm(self): def _gru_fn(): return tf.keras.layers.GRU(10, return_sequences=True) def _lstm_fn(): return tf.keras.layers.LSTM(10, return_sequences=True) gru_model = models.create_recurrent_model(10, 10, 1, _gru_fn, 'gru') lstm_model = models.create_recurrent_model(10, 10, 1, _lstm_fn, 'lstm') self.assertLess(gru_model.count_params(), lstm_model.count_params())
def test_gru_constructs(self): def _recurrent_layer_fn(): return tf.keras.layers.GRU(10, return_sequences=True) model = models.create_recurrent_model(10, 10, 2, _recurrent_layer_fn, 'rnn-gru') self.assertIsInstance(model, tf.keras.Model) self.assertEqual('rnn-gru', model.name)
def run_experiment(): """Runs the training experiment.""" vocab = _create_vocab() (stackoverflow_train, stackoverflow_val, stackoverflow_test) = construct_word_level_datasets(vocab) num_training_steps = FLAGS.num_training_examples / FLAGS.batch_size def _lstm_fn(): return tf.keras.layers.LSTM(FLAGS.latent_size, return_sequences=True) model = models.create_recurrent_model(FLAGS.vocab_size, FLAGS.embedding_size, FLAGS.num_layers, _lstm_fn, 'stackoverflow-lstm') model.compile(loss=tf.keras.losses.sparse_categorical_crossentropy, optimizer=tf.keras.optimizers.SGD( learning_rate=FLAGS.learning_rate, use_momentum=FLAGS.use_momentum), metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]) train_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'train_results.csv') test_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'test_results.csv') train_csv_logger = tf.keras.callbacks.CSVLogger(train_results_path) test_csv_logger = tf.keras.callbacks.CSVLogger(test_results_path) model.fit(stackoverflow_train, steps_per_epoch=num_training_steps, epochs=25, verbose=1, validation_data=stackoverflow_val, callbacks=[train_csv_logger]) score = model.evaluate_generator(stackoverflow_test, verbose=1, callbacks=[test_csv_logger]) print('Final test loss: %.4f' % score[0]) print('Final test accuracy: %.4f' % score[1])
def run_experiment(): """Runs the training experiment.""" try: tf.io.gfile.makedirs( os.path.join(FLAGS.root_output_dir, FLAGS.exp_name)) except tf.errors.OpError: pass train_set, validation_set, test_set = ( dataset.construct_word_level_datasets( vocab_size=FLAGS.vocab_size, client_batch_size=FLAGS.batch_size, client_epochs_per_round=1, max_seq_len=FLAGS.sequence_length, max_elements_per_user=FLAGS.max_elements_per_user, centralized_train=True, shuffle_buffer_size=None, num_validation_examples=FLAGS.num_validation_examples, num_test_examples=FLAGS.num_test_examples)) recurrent_model = tf.keras.layers.LSTM if FLAGS.lstm else tf.keras.layers.GRU def _layer_fn(): return recurrent_model(FLAGS.latent_size, return_sequences=True) pad, oov, _, eos = dataset.get_special_tokens(FLAGS.vocab_size) model = models.create_recurrent_model( FLAGS.vocab_size, FLAGS.embedding_size, FLAGS.num_layers, _layer_fn, 'stackoverflow-recurrent', shared_embedding=FLAGS.shared_embedding) logging.info('Training model: %s', model.summary()) optimizer = utils_impl.create_optimizer_from_flags('centralized') model.compile( loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), optimizer=optimizer, metrics=[ metrics.MaskedCategoricalAccuracy([pad], 'accuracy_with_oov'), metrics.MaskedCategoricalAccuracy([pad, oov], 'accuracy_no_oov'), metrics.MaskedCategoricalAccuracy([pad, oov, eos], 'accuracy_no_oov_no_eos') ]) train_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'train_results') test_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'test_results') train_csv_logger = AtomicCSVLogger(train_results_path) test_csv_logger = AtomicCSVLogger(test_results_path) log_dir = os.path.join(FLAGS.root_output_dir, 'logdir', FLAGS.exp_name) try: tf.io.gfile.makedirs(log_dir) tf.io.gfile.makedirs(train_results_path) tf.io.gfile.makedirs(test_results_path) except tf.errors.OpError: pass # log_dir already exists. train_tensorboard_callback = tf.keras.callbacks.TensorBoard( log_dir=log_dir, write_graph=True, update_freq=FLAGS.tensorboard_update_frequency) test_tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir) results_file = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'results.csv.bz2') # Write the hyperparameters to a CSV: hparam_dict = collections.OrderedDict([(name, FLAGS[name].value) for name in hparam_flags]) hparam_dict['results_file'] = results_file hparams_file = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'hparams.csv') utils_impl.atomic_write_to_csv(pd.Series(hparam_dict), hparams_file) model.fit(train_set, epochs=FLAGS.epochs, verbose=1, steps_per_epoch=FLAGS.steps_per_epoch, validation_data=validation_set, callbacks=[train_csv_logger, train_tensorboard_callback]) score = model.evaluate( test_set, verbose=1, callbacks=[test_csv_logger, test_tensorboard_callback]) logging.info('Final test loss: %.4f', score[0]) logging.info('Final test accuracy: %.4f', score[1])
def run_experiment(): """Runs the training experiment.""" vocab = _create_vocab() (stackoverflow_train, stackoverflow_val, stackoverflow_test) = construct_word_level_datasets(vocab) num_training_steps = FLAGS.num_training_examples / FLAGS.batch_size def _lstm_fn(): return tf.keras.layers.LSTM(FLAGS.latent_size, return_sequences=True) model = models.create_recurrent_model(FLAGS.vocab_size, FLAGS.embedding_size, FLAGS.num_layers, _lstm_fn, 'stackoverflow-lstm') if FLAGS.optimizer == 'sgd': optimizer = tf.keras.optimizers.SGD(learning_rate=FLAGS.learning_rate, momentum=FLAGS.momentum) if FLAGS.optimizer == 'adagrad': optimizer = tf.keras.optimizers.Adagrad( learning_rate=FLAGS.learning_rate) model.compile(loss=tf.keras.losses.sparse_categorical_crossentropy, optimizer=optimizer, metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]) train_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'train_results') test_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'test_results') train_csv_logger = AtomicCSVLogger(train_results_path) test_csv_logger = AtomicCSVLogger(test_results_path) log_dir = os.path.join(FLAGS.root_output_dir, 'logdir', FLAGS.exp_name) try: tf.io.gfile.makedirs(log_dir) tf.io.gfile.makedirs(train_results_path) tf.io.gfile.makedirs(test_results_path) except tf.errors.OpError: pass # log_dir already exists. tensorboard_callback = tf.keras.callbacks.TensorBoard( log_dir=log_dir, write_graph=True, update_freq=FLAGS.tensorboard_update_frequency) results_file = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'results.csv.bz2') # Write the hyperparameters to a CSV: hparam_dict = collections.OrderedDict([(name, FLAGS[name].value) for name in hparam_flags]) hparam_dict['results_file'] = results_file hparams_file = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'hparams.csv') utils_impl.atomic_write_to_csv(pd.Series(hparam_dict), hparams_file) model.fit(stackoverflow_train, steps_per_epoch=num_training_steps, epochs=25, verbose=1, validation_data=stackoverflow_val, callbacks=[train_csv_logger, tensorboard_callback]) score = model.evaluate_generator( stackoverflow_test, verbose=1, callbacks=[test_csv_logger, tensorboard_callback]) print('Final test loss: %.4f' % score[0]) print('Final test accuracy: %.4f' % score[1])
def run_experiment(): """Runs the training experiment.""" training_set, validation_set, test_set = ( dataset.construct_word_level_datasets( vocab_size=FLAGS.vocab_size, batch_size=FLAGS.batch_size, client_epochs_per_round=1, max_seq_len=FLAGS.sequence_length, max_training_elements_per_user=-1, num_validation_examples=FLAGS.num_validation_examples, num_test_examples=FLAGS.num_test_examples)) centralized_train = training_set.create_tf_dataset_from_all_clients() def _lstm_fn(): return tf.keras.layers.LSTM(FLAGS.latent_size, return_sequences=True) model = models.create_recurrent_model( FLAGS.vocab_size, FLAGS.embedding_size, FLAGS.num_layers, _lstm_fn, 'stackoverflow-lstm', shared_embedding=FLAGS.shared_embedding) logging.info('Training model: %s', model.summary()) optimizer = utils_impl.create_optimizer_from_flags('centralized') model.compile(loss=tf.keras.losses.sparse_categorical_crossentropy, optimizer=optimizer, weighted_metrics=['acc']) train_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'train_results') test_results_path = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'test_results') train_csv_logger = AtomicCSVLogger(train_results_path) test_csv_logger = AtomicCSVLogger(test_results_path) log_dir = os.path.join(FLAGS.root_output_dir, 'logdir', FLAGS.exp_name) try: tf.io.gfile.makedirs(log_dir) tf.io.gfile.makedirs(train_results_path) tf.io.gfile.makedirs(test_results_path) except tf.errors.OpError: pass # log_dir already exists. train_tensorboard_callback = tf.keras.callbacks.TensorBoard( log_dir=log_dir, write_graph=True, update_freq=FLAGS.tensorboard_update_frequency) test_tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir) results_file = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'results.csv.bz2') # Write the hyperparameters to a CSV: hparam_dict = collections.OrderedDict([(name, FLAGS[name].value) for name in hparam_flags]) hparam_dict['results_file'] = results_file hparams_file = os.path.join(FLAGS.root_output_dir, FLAGS.exp_name, 'hparams.csv') utils_impl.atomic_write_to_csv(pd.Series(hparam_dict), hparams_file) oov, bos, eos, pad = dataset.get_special_tokens(FLAGS.vocab_size) class_weight = {x: 1.0 for x in range(FLAGS.vocab_size)} class_weight[oov] = 0.0 # No credit for predicting OOV. class_weight[bos] = 0.0 # Shouldn't matter since this is never a target. class_weight[eos] = 1.0 # Model should learn to predict end of sentence. class_weight[pad] = 0.0 # No credit for predicting pad. model.fit(centralized_train, epochs=FLAGS.epochs, verbose=1, class_weight=class_weight, validation_data=validation_set, callbacks=[train_csv_logger, train_tensorboard_callback]) score = model.evaluate( test_set, verbose=1, callbacks=[test_csv_logger, test_tensorboard_callback]) logging.info('Final test loss: %.4f', score[0]) logging.info('Final test accuracy: %.4f', score[1])
def test_dense_fn_raises(self): def _dense_layer_fn(): return tf.keras.layers.Dense(10) with self.assertRaisesRegex(ValueError, 'tf.keras.layers.RNN'): models.create_recurrent_model(10, 10, 1, _dense_layer_fn, 'dense')