'''

Calculate the cross-entropy loss based on labels and logits input

'''

# one-hot encode labels

onehot_labels = tf.one_hot(labels, depth=FLAGS.num_classes)

# Yang16 uses log likelyhood equivalent to the cross entropy used here

cross_entropy_loss = tf.losses.softmax_cross_entropy(onehot_labels=onehot_labels,

logits=logits)

tf.summary.scalar('batch_loss', cross_entropy_loss)

'''

Calculate gradients and update parameters based on loss

'''

# get all trainaible variables

trained_vars = tf.trainable_variables()

# great utils function to calculate the parameters of the model and print them out

count_parameters(trained_vars)

# get gradients for parameter given loss

gradients = tf.gradients(loss, trained_vars)

# Gradient clipping (described in paper?): Clips values of multiple tensors by the ratio of the sum of their norms.

clipped_grads, global_norm = tf.clip_by_global_norm(gradients, FLAGS.max_grad_norm)

# save global norm

tf.summary.scalar('global_grad_norm', global_norm)

# Add gradients and vars to summary

# for gradient, var in list(zip(clipped_grads, trained_vars)):

# if 'attention' in var.name:

# tf.summary.histogram(var.name + '/gradient', gradient)

# tf.summary.histogram(var.name, var)

# Define optimizer

# Returns and create (if necessary) the global step tensor.

global_step = tf.train.get_or_create_global_step()

# define the optimizer rmsprop; paper uses different optimizer: SGD with momentum 0.9

optimizer = tf.train.RMSPropOptimizer(FLAGS.learning_rate)

# get apply gradients operation

train_op = optimizer.apply_gradients(zip(clipped_grads, trained_vars),

name='train_op',

global_step=global_step)

'''

Calculates average batch accuracy, overall accuracy and returns some operations

'''

# get index of best predictions

predictions = tf.argmax(logits, axis=-1)

# check agreement between prediction and labels (as integer)

correct_preds = tf.equal(predictions, tf.cast(labels, tf.int64))

# calcualte average accuracy of batch data point

batch_acc = tf.reduce_mean(tf.cast(correct_preds, tf.float32))

# save accuracy

tf.summary.scalar('batch_accuracy', batch_acc)

# calculates overall accuracy

# acc_update: An operation that increments the total and count variables appropriately and whose value matches accuracy

total_acc, acc_update = tf.metrics.accuracy(labels, predictions, name='metrics/acc')

# intersting: we get all variables related to metrics scope and initialize

metrics_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope="metrics")

metrics_init = tf.variables_initializer(var_list=metrics_vars)

# load the word_to_index encoded vocabulary

vocab = read_vocab(FLAGS.vocab)

# create embedding matrix of size (vocab,emb_size)

glove_embs = load_glove(FLAGS.embedding_file,FLAGS.emb_size, vocab)

print('input embeddings shape: ',glove_embs.shape)

# read data

data_reader = DataReader(train_file=FLAGS.train_data_file,

dev_file=FLAGS.dev_data_file,

test_file=FLAGS.test_data_file,

num_classes=FLAGS.num_classes)

config = tf.ConfigProto(allow_soft_placement=FLAGS.allow_soft_placement)

tf.reset_default_graph()

sess = tf.Session(config=config)

model = Model(cell_dim=FLAGS.cell_dim,

att_dim=FLAGS.att_dim,

vocab_size=len(vocab),

emb_size=FLAGS.emb_size,

num_classes=FLAGS.num_classes,

dropout_rate=FLAGS.dropout_rate,

pretrained_embs=glove_embs)

# calculate loss

loss = loss_fn(model.labels, model.logits)

total_loss, loss_update = tf.metrics.mean(loss,name='metrics/losss')

# calculates gradients

train_op, global_step = train_fn(loss)

# calculates metrics and merges all

batch_acc, total_acc, acc_update, metrics_init = eval_fn(model.labels, model.logits)

summary_op = tf.summary.merge_all()

summary_total = tf.summary.merge(

[tf.summary.scalar('total_batch_accuracy', total_acc),

tf.summary.scalar("total_batch_loss",total_loss)

])

sess.run(tf.global_variables_initializer())

# The graph described by sess.graph will be displayed by TensorBoard

train_writer.add_graph(sess.graph)

# save all variables

saver = tf.train.Saver(max_to_keep=FLAGS.num_checkpoints)

print('\n{}> Start training'.format(datetime.now()))

epoch = 0

valid_step = 0

test_step = 0

train_test_prop = len(data_reader.train_data) / len(data_reader.test_data)

test_batch_size = int(FLAGS.batch_size / train_test_prop)

best_acc = float('-inf')

while epoch < FLAGS.num_epochs:

epoch += 1

print('\n{}> Epoch: {}'.format(datetime.now(), epoch))

# we newly initialize metrics tensors each epoch, each evaluation

sess.run(metrics_init)

# each data point/doc in batch contains a list of sentences, encoded with index

for batch_docs, batch_labels in data_reader.read_train_set(FLAGS.batch_size, shuffle=True):

# do a batch

_step, _, _loss, _acc = sess.run([global_step, train_op, loss, batch_acc],

feed_dict=model.get_feed_dict(batch_docs, batch_labels, training=True))

# each display_step steps evaluate metric variables and add to train_writer, training is false to disables dropout

if _step % FLAGS.display_step == 0: #

_summary,_loss_save,_acc_save,_,_ = sess.run([summary_op,loss,batch_acc, acc_update,loss_update],

feed_dict=model.get_feed_dict(batch_docs, batch_labels))

train_writer.add_summary(_summary, global_step=_step)

last_step_epoch = _step

# evaluate avg batch metrics

total_acc_train,total_loss_train,summary_total_train = sess.run([total_acc,total_loss,summary_total])

train_writer.add_summary(summary_total_train,global_step=last_step_epoch)

mlflow.log_metrics({'avg_batch_accuracy':total_acc_train,'avg_batch_loss':total_loss_train},step=last_step_epoch)

print('Avg training accuracy = {:.2f}'.format(total_acc_train))

print('Avg training loss = {:.2f}'.format(total_loss_train))

# we newly initialize metrics tensors each epoch, each evaluation

sess.run(metrics_init)

# for each epoch calculate metrics for valid set

for batch_docs, batch_labels in data_reader.read_valid_set(test_batch_size):

_loss, _acc, _,_ = sess.run([loss, batch_acc, acc_update,loss_update],

feed_dict=model.get_feed_dict(batch_docs, batch_labels))

total_acc_valid,total_loss_valid,summary_total_valid = sess.run([total_acc,total_loss,summary_total])

valid_writer.add_summary(summary_total_train,global_step=last_step_epoch)

mlflow.log_metrics({'avg_valid_accuracy':total_acc_valid,'avg_valid_loss':total_loss_valid},step=last_step_epoch)

print('Avg validation accuracy = {:.2f}'.format(total_acc_valid))

print('Avg validation loss = {:.2f}'.format(total_loss_valid))

# we newly initialize metrics tensors each epoch, each evaluation

sess.run(metrics_init)

# for each epoch calculate metrics for test set

for batch_docs, batch_labels in data_reader.read_test_set(test_batch_size):

_loss, _acc, _,_ = sess.run([loss, batch_acc, acc_update,loss_update],

feed_dict=model.get_feed_dict(batch_docs, batch_labels))

total_acc_test,total_loss_test,summary_total_test = sess.run([total_acc,total_loss,summary_total])

test_writer.add_summary(summary_total_test,global_step=last_step_epoch)

mlflow.log_metrics({'avg_test_accuracy':total_acc_test,'avg_test_loss':total_loss_test},step=last_step_epoch)

print('Avg validation accuracy = {:.2f}'.format(total_acc_test))

print('Avg validation loss = {:.2f}'.format(total_loss_test))

# keep track of best test accuracy, if epoch improved, save all variables

if total_acc_test > best_acc:

best_acc = total_acc_test

saver.save(sess, FLAGS.checkpoint_dir)

print('Best testing accuracy = {:.2f}'.format(test_acc))

print("{} Optimization Finished!".format(datetime.now()))