def create_examples(input_string, ):
"""
from the input, produce examples where the input is a sequence of integers
representing a string of characters, and the target is the character immediately
following the input sequence
"""
sequences = []
targets = []
depths = []
Config.char2int = {c: i for i, c in enumerate(sorted(set(input_string)))}
# ToDo Discuss with Ben how we want to train on text shorter than the window size?
# Get all examples
if Config.dataset_size == -1:
# iterate over the file window by window
i = 0
while i + Config.sequence_length + 1 < len(input_string):
sequences += [[
Config.char2int[c]
for c in input_string[i:i + Config.sequence_length]
]]
depths.append(Config.sequence_length)
targets += [
Config.char2int[input_string[i + Config.sequence_length]]
]
i += 1
else:
# get size many examples
for z in range(Config.dataset_size):
# get a random starting point
r = random.choice(
range(len(input_string) - Config.sequence_length - 1))
sequences.append([
Config.char2int[c]
for c in input_string[r:r + Config.sequence_length]
])
depths.append(Config.sequence_length)
targets.append(
Config.char2int[input_string[r + Config.sequence_length]])
assert (len(sequences) == len(targets))
# Define how to randomly split the input data into train and test
shuffled_list = list(range(len(sequences)))
random.shuffle(shuffled_list)
# Determine whether to do a validation split
if Config.perform_validation():
split_point = int(Config.training_split_ratio * len(sequences))
else:
split_point = len(sequences)
Config.Train.x = [sequences[idx] for idx in shuffled_list[:split_point]]
Config.Train.depth = [depths[idx] for idx in shuffled_list[:split_point]]
Config.Train.t = list(
map(lambda idx: _build_target_vector(targets[idx]),
shuffled_list[:split_point]))
if Config.perform_validation():
Config.Validation.x = [
sequences[idx] for idx in shuffled_list[split_point:]
]
Config.Validation.depth = [
depths[idx] for idx in shuffled_list[split_point:]
]
Config.Validation.t = list(
map(lambda idx: _build_target_vector(targets[idx]),
shuffled_list[split_point:]))
def run_training():
net_features = network.construct()
input_x = net_features["X"]
target = net_features["target"]
seq_len = net_features["seq_len"]
# Setup the training procedure
cross_h = tf.nn.softmax_cross_entropy_with_logits(
logits=net_features["output"], labels=target)
loss_op = tf.reduce_sum(cross_h)
optimizer = tf.train.AdamOptimizer(
learning_rate=Config.Train.learning_rate)
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(loss_op, tvars), 5.)
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0.0))
train_op = optimizer.apply_gradients(zip(grads, tvars),
global_step=global_step)
# train_op = optimizer.minimize(loss_op)
sess = tf.Session()
if Config.Train.restore:
Config.import_model(sess)
else:
sess.run(tf.global_variables_initializer())
num_batches = 0
for epoch in range(0, Config.Train.num_epochs):
# Shuffle the batches for each epoch
shuffled_list = list(range(Config.Train.size()))
random.shuffle(shuffled_list)
train_err = 0
for batch in range(0, Config.Train.num_batch()):
end_batch = min((batch + 1) * Config.batch_size,
Config.Train.size())
start_batch = max(0, end_batch - Config.batch_size)
# Use the randomized batches
train_x = list(
map(lambda idx: Config.Train.x[idx],
shuffled_list[start_batch:end_batch]))
train_t = list(
map(lambda idx: Config.Train.t[idx],
shuffled_list[start_batch:end_batch]))
seqlen = list(
map(lambda idx: Config.Train.depth[idx],
shuffled_list[start_batch:end_batch]))
_, err = sess.run([train_op, loss_op],
feed_dict={
input_x: train_x,
target: train_t,
seq_len: seqlen
})
train_err += err
num_batches += 1
BATCH_PRINT_FREQUENCY = 1000
if num_batches % BATCH_PRINT_FREQUENCY == 0:
print("Epoch %d: Total Batches %d: Last Batch Error: %0.3f" %
(epoch, num_batches, err))
# ToDo It would be nice to add perplexity here.
logging.info("EPOCH #%05d COMPLETED" % epoch)
train_err /= Config.Train.num_batch()
logging.info("Epoch %05d: Average Batch Training Error: \t\t%0.3f" %
(epoch, train_err))
if Config.perform_validation():
test_err = _calculate_validation_error(sess, loss_op, input_x,
target, seq_len)
logging.info(
"Epoch %05d: Average Batch Verification Error: \t%0.3f" %
(epoch, test_err))
if epoch % Config.Train.checkpoint_frequency == 0:
Config.export_model(sess, epoch)
sess.close()