def run_test():
# test batch generation
print('download and read data')
filename = maybe_download('text8.zip', 31344016)
# Read data
text = read_data(filename)
# create datasets
valid_size = 1000
valid_text = text[:valid_size]
train_text = text[valid_size:]
# train_size = len(train_text)
# create batch generators
train_batches = BatchGenerator(train_text, BATCH_SIZE, MIN_CHARS_IN_BATCH, MAX_CHARS_IN_BATCH, reverse_encoder_input=True)
valid_batches = BatchGenerator(valid_text, 1, MIN_CHARS_IN_BATCH, MAX_CHARS_IN_BATCH)
# print(BatchGenerator.characters(train_batches.next()[0]))
print('test main batch generator')
e_bs, d_bs, dw_bs = train_batches.next()
print(BatchGenerator.batches2string(e_bs))
print(BatchGenerator.batches2string(d_bs))
BatchGenerator.verify_weights(d_bs, dw_bs)
e_bs, d_bs, dw_bs = train_batches.next()
print(BatchGenerator.batches2string(e_bs))
print(BatchGenerator.batches2string(d_bs))
BatchGenerator.verify_weights(d_bs, dw_bs)
e_bs, d_bs, dw_bs = valid_batches.next()
print(BatchGenerator.batches2string(e_bs))
print(BatchGenerator.batches2string(d_bs))
BatchGenerator.verify_weights(d_bs, dw_bs)
e_bs, d_bs, dw_bs = valid_batches.next()
print(BatchGenerator.batches2string(e_bs))
print(BatchGenerator.batches2string(d_bs))
BatchGenerator.verify_weights(d_bs, dw_bs)
print('test random english generator')
random_batch = RandomWordsBatchGenerator(2, MIN_CHARS_IN_BATCH, MAX_CHARS_IN_BATCH,
reverse_encoder_input=False)
for _ in range(10):
e_bs, d_bs, dw_bs = random_batch.next()
print(BatchGenerator.batches2string(e_bs))
print(BatchGenerator.batches2string(d_bs))
BatchGenerator.verify_weights(d_bs, dw_bs)
print('test random string gen with padding')
random_str_batch = ReverseStringBatchGenerator(1, 8, 8,
reverse_encoder_input=False)
e_bs, d_bs, dw_bs = random_str_batch.next()
print(BatchGenerator.batches2string(e_bs))
print(BatchGenerator.batches2string(d_bs))
BatchGenerator.verify_weights(d_bs, dw_bs)
random_str_batch = ReverseStringBatchGenerator(2, 8, 16,
reverse_encoder_input=False)
e_bs, d_bs, dw_bs = random_str_batch.next()
print(BatchGenerator.batches2string(e_bs))
print(BatchGenerator.batches2string(d_bs))
BatchGenerator.verify_weights(d_bs, dw_bs)
def train():
graph = tf.Graph()
inputs, labels, sequence_lengths, optimizer, loss, predictions, summary_op, saver = model(
graph)
train_writer = tf.train.SummaryWriter('tensorboard/train', graph)
test_writer = tf.train.SummaryWriter('tensorboard/test', graph)
with tf.Session(graph=graph) as sess:
sess.run(tf.initialize_all_variables())
print("Training...")
train_batches = BatchGenerator(settings.BATCH_SIZE)
test_batches = BatchGenerator(3000, settings.SEQUENCE_LIMIT)
test_inputs, test_labels, test_sequence_lengths = test_batches.next()
for step in xrange(settings.EPOCH):
if step % 1000 != 1:
train_inputs, train_labels, train_sequence_lengths = train_batches.next(
)
feed_dict = {
inputs: train_inputs,
labels: train_labels,
sequence_lengths: train_sequence_lengths
}
_, train_loss, train_predictions, summary = sess.run(
[optimizer, loss, predictions, summary_op],
feed_dict=feed_dict)
train_writer.add_summary(summary, step)
train_writer.flush()
else:
test_feed_dict = {
inputs: test_inputs,
labels: test_labels,
sequence_lengths: test_sequence_lengths
}
test_loss, summary = sess.run([loss, summary_op],
feed_dict=test_feed_dict)
test_writer.add_summary(summary, step)
test_writer.flush()
if step % 1000 == 0:
print('-----------Step %d:-------------' % step)
print('Training set:')
print(' Loss : ', train_loss)
print(' Input : ', train_inputs[0])
print(' Label : ', utils.onehot2sticker(train_labels))
print(' Prediction : ',
utils.onehot2sticker(train_predictions))
if step % 10000 == 0:
# Save the variables to disk.
save_path = saver.save(sess,
"checkpoints/" + settings.CKPT_NAME)
print("Model saved in file: %s" % save_path)
def train(num_layers, units_per_layer):
print('download and read data')
filename = maybe_download('text8.zip', 31344016)
with tf.Session(graph=tf.Graph()) as validation_session:
validation_model = ReverseSeqValidationSummaryModel(validation_session.graph)
validation_session.run(tf.initialize_all_variables())
with tf.Session(graph=tf.Graph()) as sess:
# Create model.
print("Creating %d layers of %d units." % (num_layers, units_per_layer))
model = create_model(sess, num_layers, units_per_layer, False, DECODER_FEED_PREVIOUS)
# Read data
text = read_data(filename)
# create datasets
valid_size = 10000
valid_text = text[:valid_size]
train_text = text[valid_size:]
# create batch generators
validation_batch = BatchGenerator(valid_text, 1, MIN_CHARS_IN_BATCH, MAX_CHARS_IN_BATCH,
reverse_encoder_input=REVERSE_ENCODER_INPUT)
if TRAIN_BATCH_TYPE == UseTrainBatchType.use_english_words:
train_batch = BatchGenerator(train_text, BATCH_SIZE, MIN_CHARS_IN_BATCH, MAX_CHARS_IN_BATCH,
reverse_encoder_input=REVERSE_ENCODER_INPUT)
elif TRAIN_BATCH_TYPE == UseTrainBatchType.use_random_train_words:
train_batch = RandomWordsBatchGenerator(BATCH_SIZE, MIN_CHARS_IN_BATCH, MAX_CHARS_IN_BATCH,
reverse_encoder_input=REVERSE_ENCODER_INPUT)
else:
train_batch = ReverseStringBatchGenerator(BATCH_SIZE, MIN_CHARS_IN_BATCH, MAX_CHARS_IN_BATCH,
reverse_encoder_input=REVERSE_ENCODER_INPUT)
validation_batch = ReverseStringBatchGenerator(1, MIN_CHARS_IN_BATCH, MAX_CHARS_IN_BATCH,
reverse_encoder_input=REVERSE_ENCODER_INPUT)
# This is the training loop.
step_time, loss = 0.0, 0.0
current_step = model.global_step.eval() + 1
print('starting from step %i' % current_step)
previous_losses = []
enc_state = model.initial_enc_state.eval()
run_data_dir = run_data_directory(num_layers, units_per_layer)
while True:
# Get a batch and make a step.
start_time = time.time()
encoder_inputs, decoder_inputs, decoder_weights = train_batch.next()
_, step_loss, enc_state = model.step(sess, current_step, encoder_inputs, decoder_inputs, decoder_weights,
enc_state, DROPOUT_PROB, False)
step_time += (time.time() - start_time) / STEPS_PER_CHECKPOINT
loss += step_loss / STEPS_PER_CHECKPOINT
current_step += 1
# Once in a while, we save checkpoint, print statistics, and run evals.
if current_step % STEPS_PER_CHECKPOINT == 0:
# Print statistics for the previous epoch.
perplexity = math.exp(loss) if loss < 300 else float('inf')
print("global step %d learning rate %.4f step-time %.2f loss %.3f perplexity "
"%.2f" % (model.global_step.eval(), model.learning_rate.eval(),
step_time, loss, perplexity))
# Decrease learning rate if no improvement was seen over last 3 times.
if len(previous_losses) > 2 and loss > max(previous_losses[-3:]):
sess.run(model.learning_rate_decay_op)
previous_losses.append(loss)
# Save checkpoint and zero timer and loss.
checkpoint_path = os.path.join(run_data_dir, 'state')
model.saver.save(sess, checkpoint_path, global_step=model.global_step)
step_time, loss = 0.0, 0.0
# Run evals on validation set and print their perplexity.
val_perp = validate_sentence(sess, model, validation_batch, enc_state, current_step)
summary_str = validation_model.merged_validation.eval(
{validation_model.validation_perp: val_perp if val_perp < 500 else 500 },
validation_session)
model.summ_writer.add_summary(summary_str, current_step)
sys.stdout.flush()
best_val_loss = np.inf
best_val_acc = 0.
best_val_auc = 0.
best_epoch = 0
best_acc_epoch = 0
wait = 0
total_time = 0.
print('Training...')
for epoch in range(NB_EPOCH):
t = time.time()
adj_train, train_labels, train_u_indices, \
train_v_indices, val_labels, val_u_indices, val_v_indices = batch_gen.next()
# global normalization
support = []
support_t = []
adj_train_int = sp.csr_matrix(adj_train, dtype=np.int32)
for i in range(NUMCLASSES):
# build individual binary rating matrices (supports) for each rating
support_unnormalized = sp.csr_matrix(adj_train_int == i + 1,
dtype=np.float32)
if support_unnormalized.nnz == 0 and DATASET != 'yahoo_music':
# yahoo music has dataset split with not all ratings types present in training set.
# this produces empty adjacency matrices for these ratings.
sys.exit(