def test_toy_model(self): if self.run_epoch is None or self.score_dataset is None: self.skipTest("RunEpochTester: run_epoch_fn and score_dataset_fn " "must be provided.") self.assertIsNotNone(self.run_epoch) self.assertIsNotNone(self.score_dataset) with tf.Session(graph=self.lm.graph) as sess: tf.set_random_seed(42) sess.run(tf.global_variables_initializer()) bi = utils.rnnlm_batch_generator(self.train_ids, 5, 10) self.run_epoch(self.lm, sess, bi, learning_rate=0.3, train=True, verbose=True, tick_s=1.0) train_loss = self.score_dataset(self.lm, sess, self.train_ids, name="Train set") test_loss = self.score_dataset(self.lm, sess, self.test_ids, name="Test set") # This is a *really* simple dataset, so you should have no trouble # getting almost perfect scores. self.assertFalse(train_loss is None) self.assertFalse(test_loss is None) self.assertLessEqual(train_loss, 0.1) self.assertLessEqual(test_loss, 0.2)
def score_dataset(lm, session, ids, name="Data"): # For scoring, we can use larger batches to speed things up. bi = utils.rnnlm_batch_generator(ids, batch_size=100, max_time=100) cost = run_epoch(lm, session, bi, learning_rate=0.0, train=False, verbose=False, tick_s=3600) print("{:s}: avg. loss: {:.03f} (perplexity: {:.02f})".format(name, cost, np.exp(cost))) return cost
def score_each_step(lm, session, ids): #no batching bi = utils.rnnlm_batch_generator(ids, batch_size=100, max_time=100) for i, (w, y) in enumerate(bi): if i == 0: h = session.run(lm.initial_h_, {lm.input_w_: w}) feed_dict = { lm.input_w_: w, lm.target_y_: y, lm.learning_rate_: 0.002, lm.use_dropout_: False, lm.initial_h_: h } cost, h, _ = session.run([loss, lm.final_h_, train_op], feed_dict=feed_dict)
def test_rnnlm_batch_generator(self): ids = np.arange(15) batches = list( utils.rnnlm_batch_generator(ids, batch_size=2, max_time=3)) self.assertEqual(len(batches), 3) self.assertTrue(np.array_equal(batches[0][0], [[0, 1, 2], [7, 8, 9]])) # x0 self.assertTrue(np.array_equal(batches[0][1], [[1, 2, 3], [8, 9, 10]])) # y0 self.assertTrue( np.array_equal(batches[1][0], [[3, 4, 5], [10, 11, 12]])) # x1 self.assertTrue( np.array_equal(batches[1][1], [[4, 5, 6], [11, 12, 13]])) # y1 self.assertTrue(np.array_equal(batches[2][0], [[6], [13]])) # x2 self.assertTrue(np.array_equal(batches[2][1], [[7], [14]])) # y2
def run_training(train_ids, test_ids, tf_savedir, model_params, max_time=100, batch_size=256, learning_rate=0.002, num_epochs=20): #V = len(words_to_ids.keys()) # Training parameters ## add parameter sets for each attack/defense configuration #max_time = 25 #batch_size = 100 #learning_rate = 0.01 #num_epochs = 10 # Model parameters #model_params = dict(V=vocab.size, #H=200, #softmax_ns=200, #num_layers=2) #model_params = dict(V=len(words_to_ids.keys()), #H=1024, #softmax_ns=len(words_to_ids.keys()), #num_layers=2) #model_params = dict(V=V, H=H, softmax_ns=softmax_ns, num_layers=num_layers) #TF_SAVEDIR = "/tmp/artificial_hotel_reviews/a4_model" TF_SAVEDIR = tf_savedir checkpoint_filename = os.path.join(TF_SAVEDIR, "rnnlm") trained_filename = os.path.join(TF_SAVEDIR, "rnnlm_trained") # Will print status every this many seconds #print_interval = 5 print_interval = 30 lm = rnnlm.RNNLM(**model_params) lm.BuildCoreGraph() lm.BuildTrainGraph() # Explicitly add global initializer and variable saver to LM graph with lm.graph.as_default(): initializer = tf.global_variables_initializer() saver = tf.train.Saver() # Clear old log directory shutil.rmtree(TF_SAVEDIR, ignore_errors=True) if not os.path.isdir(TF_SAVEDIR): os.makedirs(TF_SAVEDIR) with tf.Session(graph=lm.graph) as session: # Seed RNG for repeatability #tf.set_random_seed(42) session.run(initializer) #check trainable variables #variables_names = [v.name for v in tf.trainable_variables()] #values = session.run(variables_names) #for k, v in zip(variables_names, values): #print("Variable: ", k) #print("Shape: ", v.shape) #print(v) for epoch in range(1, num_epochs + 1): t0_epoch = time.time() bi = utils.rnnlm_batch_generator(train_ids, batch_size, max_time) print("[epoch {:d}] Starting epoch {:d}".format(epoch, epoch)) # Run a training epoch. run_epoch(lm, session, batch_iterator=bi, train=True, verbose=True, tick_s=10, learning_rate=learning_rate) print("[epoch {:d}] Completed in {:s}".format( epoch, utils.pretty_timedelta(since=t0_epoch))) # Save a checkpoint saver.save(session, checkpoint_filename, global_step=epoch) ## # score_dataset will run a forward pass over the entire dataset # and report perplexity scores. This can be slow (around 1/2 to # 1/4 as long as a full epoch), so you may want to comment it out # to speed up training on a slow machine. Be sure to run it at the # end to evaluate your score. print("[epoch {:d}]".format(epoch), end=" ") score_dataset(lm, session, train_ids, name="Train set") print("[epoch {:d}]".format(epoch), end=" ") score_dataset(lm, session, test_ids, name="Test set") print("") # Save final model saver.save(session, trained_filename) return trained_filename