def test_toy_model(self):
if self.run_epoch is None or self.score_dataset is None:
self.skipTest("RunEpochTester: run_epoch_fn and score_dataset_fn "
"must be provided.")
self.assertIsNotNone(self.run_epoch)
self.assertIsNotNone(self.score_dataset)
with tf.Session(graph=self.lm.graph) as sess:
tf.set_random_seed(42)
sess.run(tf.global_variables_initializer())
bi = utils.rnnlm_batch_generator(self.train_ids, 5, 10)
self.run_epoch(self.lm,
sess,
bi,
learning_rate=0.3,
train=True,
verbose=True,
tick_s=1.0)
train_loss = self.score_dataset(self.lm,
sess,
self.train_ids,
name="Train set")
test_loss = self.score_dataset(self.lm,
sess,
self.test_ids,
name="Test set")
# This is a *really* simple dataset, so you should have no trouble
# getting almost perfect scores.
self.assertFalse(train_loss is None)
self.assertFalse(test_loss is None)
self.assertLessEqual(train_loss, 0.1)
self.assertLessEqual(test_loss, 0.2)
def score_dataset(lm, session, ids, name="Data"):
# For scoring, we can use larger batches to speed things up.
bi = utils.rnnlm_batch_generator(ids, batch_size=100, max_time=100)
cost = run_epoch(lm, session, bi,
learning_rate=0.0, train=False,
verbose=False, tick_s=3600)
print("{:s}: avg. loss: {:.03f} (perplexity: {:.02f})".format(name, cost, np.exp(cost)))
return cost
def score_each_step(lm, session, ids):
#no batching
bi = utils.rnnlm_batch_generator(ids, batch_size=100, max_time=100)
for i, (w, y) in enumerate(bi):
if i == 0:
h = session.run(lm.initial_h_, {lm.input_w_: w})
feed_dict = {
lm.input_w_: w,
lm.target_y_: y,
lm.learning_rate_: 0.002,
lm.use_dropout_: False,
lm.initial_h_: h
}
cost, h, _ = session.run([loss, lm.final_h_, train_op],
feed_dict=feed_dict)
def test_rnnlm_batch_generator(self):
ids = np.arange(15)
batches = list(
utils.rnnlm_batch_generator(ids, batch_size=2, max_time=3))
self.assertEqual(len(batches), 3)
self.assertTrue(np.array_equal(batches[0][0],
[[0, 1, 2], [7, 8, 9]])) # x0
self.assertTrue(np.array_equal(batches[0][1],
[[1, 2, 3], [8, 9, 10]])) # y0
self.assertTrue(
np.array_equal(batches[1][0], [[3, 4, 5], [10, 11, 12]])) # x1
self.assertTrue(
np.array_equal(batches[1][1], [[4, 5, 6], [11, 12, 13]])) # y1
self.assertTrue(np.array_equal(batches[2][0], [[6], [13]])) # x2
self.assertTrue(np.array_equal(batches[2][1], [[7], [14]])) # y2
def run_training(train_ids,
test_ids,
tf_savedir,
model_params,
max_time=100,
batch_size=256,
learning_rate=0.002,
num_epochs=20):
#V = len(words_to_ids.keys())
# Training parameters
## add parameter sets for each attack/defense configuration
#max_time = 25
#batch_size = 100
#learning_rate = 0.01
#num_epochs = 10
# Model parameters
#model_params = dict(V=vocab.size,
#H=200,
#softmax_ns=200,
#num_layers=2)
#model_params = dict(V=len(words_to_ids.keys()),
#H=1024,
#softmax_ns=len(words_to_ids.keys()),
#num_layers=2)
#model_params = dict(V=V, H=H, softmax_ns=softmax_ns, num_layers=num_layers)
#TF_SAVEDIR = "/tmp/artificial_hotel_reviews/a4_model"
TF_SAVEDIR = tf_savedir
checkpoint_filename = os.path.join(TF_SAVEDIR, "rnnlm")
trained_filename = os.path.join(TF_SAVEDIR, "rnnlm_trained")
# Will print status every this many seconds
#print_interval = 5
print_interval = 30
lm = rnnlm.RNNLM(**model_params)
lm.BuildCoreGraph()
lm.BuildTrainGraph()
# Explicitly add global initializer and variable saver to LM graph
with lm.graph.as_default():
initializer = tf.global_variables_initializer()
saver = tf.train.Saver()
# Clear old log directory
shutil.rmtree(TF_SAVEDIR, ignore_errors=True)
if not os.path.isdir(TF_SAVEDIR):
os.makedirs(TF_SAVEDIR)
with tf.Session(graph=lm.graph) as session:
# Seed RNG for repeatability
#tf.set_random_seed(42)
session.run(initializer)
#check trainable variables
#variables_names = [v.name for v in tf.trainable_variables()]
#values = session.run(variables_names)
#for k, v in zip(variables_names, values):
#print("Variable: ", k)
#print("Shape: ", v.shape)
#print(v)
for epoch in range(1, num_epochs + 1):
t0_epoch = time.time()
bi = utils.rnnlm_batch_generator(train_ids, batch_size, max_time)
print("[epoch {:d}] Starting epoch {:d}".format(epoch, epoch))
# Run a training epoch.
run_epoch(lm,
session,
batch_iterator=bi,
train=True,
verbose=True,
tick_s=10,
learning_rate=learning_rate)
print("[epoch {:d}] Completed in {:s}".format(
epoch, utils.pretty_timedelta(since=t0_epoch)))
# Save a checkpoint
saver.save(session, checkpoint_filename, global_step=epoch)
##
# score_dataset will run a forward pass over the entire dataset
# and report perplexity scores. This can be slow (around 1/2 to
# 1/4 as long as a full epoch), so you may want to comment it out
# to speed up training on a slow machine. Be sure to run it at the
# end to evaluate your score.
print("[epoch {:d}]".format(epoch), end=" ")
score_dataset(lm, session, train_ids, name="Train set")
print("[epoch {:d}]".format(epoch), end=" ")
score_dataset(lm, session, test_ids, name="Test set")
print("")
# Save final model
saver.save(session, trained_filename)
return trained_filename
