gpu_options = tf.GPUOptions(allow_growth=True, per_process_gpu_memory_fraction=0.3)
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement,
gpu_options=gpu_options)
with tf.Session(config=session_conf) as sess:
model = MemN2N_KV(memory_representation=FLAGS.memory_representation, window_size=FLAGS.window_size,
encoding=encoding,
batch_size=batch_size, vocab_size=vocab_size, vocab=word2idx,
glove=FLAGS.glove,
dropout=FLAGS.dropout,
sentence_size=sentence_size, memory_size=memory_size,
candidates_size=max_candidates_size,
embedding_size=FLAGS.embedding_size,
starter_learning_rate=FLAGS.learning_rate,
l2=FLAGS.l2, l2_lambda=FLAGS.l2_lambda,
gradient_clipping=FLAGS.gradient_clipping, max_grad_norm=FLAGS.max_grad_norm,
gradient_noise=FLAGS.add_gradient_noise, anneal_noise=FLAGS.anneal_noise,
hops=FLAGS.hops, reader=FLAGS.reader,
session=sess,
logger=logger)
writer = tf.summary.FileWriter(log_dir, sess.graph)
saver = tf.train.Saver(max_to_keep=2)
x = []
training_accuracies = []
testing_accuracies = []
global_step,
20000,
0.96,
staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
epsilon=FLAGS.epsilon)
with tf.Session() as sess:
model = MemN2N_KV(batch_size=batch_size,
vocab_size=vocab_size,
query_size=sentence_size,
story_size=sentence_size,
memory_key_size=memory_size,
feature_size=FLAGS.feature_size,
memory_value_size=memory_size,
embedding_size=FLAGS.embedding_size,
hops=FLAGS.hops,
reader=FLAGS.reader,
l2_lambda=FLAGS.l2_lambda)
grads_and_vars = optimizer.compute_gradients(model.loss_op)
grads_and_vars = [(tf.clip_by_norm(g, FLAGS.max_grad_norm), v)
for g, v in grads_and_vars if g is not None]
grads_and_vars = [(add_gradient_noise(g), v)
for g, v in grads_and_vars]
nil_grads_and_vars = []
for g, v in grads_and_vars:
def testBatchSize():
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
global_step = tf.Variable(0, name="global_step", trainable=False)
# decay learning rate
starter_learning_rate = FLAGS.learning_rate
learning_rate = tf.train.exponential_decay(starter_learning_rate,
global_step,
20000,
0.96,
staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
epsilon=FLAGS.epsilon)
config = tf.ConfigProto(allow_soft_placement=True)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
model = MemN2N_KV(batch_size=batch_size,
vocab_size=vocab_size,
query_size=sentence_size,
story_size=sentence_size,
memory_key_size=memory_size,
feature_size=FLAGS.feature_size,
memory_value_size=memory_size,
embedding_size=FLAGS.embedding_size,
hops=FLAGS.hops,
reader=FLAGS.reader,
l2_lambda=FLAGS.l2_lambda)
grads_and_vars = optimizer.compute_gradients(model.loss_op)
grads_and_vars = [(tf.clip_by_norm(g, FLAGS.max_grad_norm), v)
for g, v in grads_and_vars if g is not None]
grads_and_vars = [(add_gradient_noise(g), v)
for g, v in grads_and_vars]
nil_grads_and_vars = []
for g, v in grads_and_vars:
if v.name in model._nil_vars:
nil_grads_and_vars.append((zero_nil_slot(g), v))
else:
nil_grads_and_vars.append((g, v))
train_op = optimizer.apply_gradients(nil_grads_and_vars,
name="train_op",
global_step=global_step)
sess.run(tf.global_variables_initializer())
def train_step(s, q, a):
feed_dict = {
model._memory_value: s,
model._query: q,
model._memory_key: s,
model._labels: a,
model.keep_prob: FLAGS.keep_prob
}
_, step, predict_op = sess.run(
[train_op, global_step, model.predict_op], feed_dict)
return predict_op
def test_step(s, q):
feed_dict = {
model._query: q,
model._memory_key: s,
model._memory_value: s,
model.keep_prob: 1
}
preds = sess.run(model.predict_op, feed_dict)
return preds
for t in range(1, FLAGS.epochs + 1):
np.random.shuffle(batches)
train_preds = []
#for start in range(0, n_train, batch_size):
for start, end in batches:
#end = start + batch_size
s = trainS[start:end]
q = trainQ[start:end]
a = trainA[start:end]
predict_op = train_step(s, q, a)
train_preds += list(predict_op)
train_preds = test_step(trainS, trainQ)
train_acc = metrics.accuracy_score(train_labels, train_preds)
trainaccuracy.append(train_acc)
# print('-----------------------')
# print('Epoch', t)
# print('Training Accuracy: {0:.2f}'.format(train_acc))
# print('-----------------------')
val_preds = test_step(valS, valQ)
val_acc = metrics.accuracy_score(np.array(val_preds),
val_labels)
validationaccuracy.append(val_acc)
# print(val_preds)
# print('-----------------------')
# print('Epoch', t)
# print('Validation Accuracy:', val_acc)
# print('-----------------------')
# test on train dataset
train_preds = test_step(trainS, trainQ)
train_acc = metrics.accuracy_score(train_labels, train_preds)
train_acc = '{0:.2f}'.format(train_acc)
# eval dataset
val_preds = test_step(valS, valQ)
val_acc = metrics.accuracy_score(val_labels, val_preds)
val_acc = '{0:.2f}'.format(val_acc)
# batchACCURACY.append(val_acc)
# testing dataset
test_preds = test_step(testS, testQ)
test_acc = metrics.accuracy_score(test_labels, test_preds)
test_acc = '{0:.2f}'.format(test_acc)
print("Testing Accuracy: {}".format(test_acc))