def main():
dialogues_train = map(lambda x: x, train_babi)
dialogues_train_eval = map(lambda x: [x[-1]], train_babi) \
if args.predict_last_turn_only \
else map(lambda x: x, train_babi)
dialogues_test = map(lambda x: [x[-1]], test_plus) \
if args.predict_last_turn_only \
else map(lambda x: x, test_plus)
data_train = reduce(lambda x, y: x + y, dialogues_train, [])
data_train_eval = reduce(lambda x, y: x + y, dialogues_train_eval, [])
data_test = reduce(lambda x, y: x + y, dialogues_test, [])
train_s, train_q, train_a = vectorize_data_dialog(data_train, word_idx,
answer_idx,
sentence_size,
memory_size)
train_eval_s, train_eval_q, train_eval_a = vectorize_data_dialog(
data_train_eval, word_idx, answer_idx, sentence_size, memory_size)
test_s, test_q, test_a = vectorize_data_dialog(data_test, word_idx,
answer_idx, sentence_size,
memory_size)
print("Training Size (dialogues)", len(dialogues_train))
print("Training/Evaluation Size (dialogues)", len(dialogues_train_eval))
print("Testing Size (dialogues)", len(dialogues_test))
print("Training Size (stories)", len(data_train))
print("Training/Evaluation Size (stories)", len(data_train_eval))
print("Testing Size (stories)", len(data_test))
tf.set_random_seed(FLAGS.random_state)
batch_size = FLAGS.batch_size
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=FLAGS.learning_rate)
batches = zip(range(0,
len(data_train) - batch_size, batch_size),
range(batch_size, len(data_train), batch_size))
batches = [(start, end) for start, end in batches]
with tf.Session() as sess:
model = MemN2N(batch_size,
vocab_size,
sentence_size,
memory_size,
FLAGS.embedding_size,
answers_vectorized,
session=sess,
hops=FLAGS.hops,
max_grad_norm=FLAGS.max_grad_norm,
optimizer=optimizer)
best_accuracy_per_epoch = train_model(
model, (train_s, train_q, train_a),
(train_eval_s, train_eval_q, train_eval_a),
(test_s, test_q, test_a), batches)
return best_accuracy_per_epoch
def main(in_train_dialogue_name, in_testset_size, in_fold_number,
in_dataset_shuffle):
trainset_idx = [get_global_dialogue_index(in_train_dialogue_name)]
testset_idx = []
testset_counter = in_fold_number * in_testset_size
while len(testset_idx) != in_testset_size:
current_idx = in_dataset_shuffle[testset_counter]
if current_idx not in trainset_idx:
testset_idx.append(current_idx)
testset_counter += 1
dialogues_train = map(lambda x: all_dialogues[x], trainset_idx)
dialogues_test = map(lambda x: all_dialogues[x], testset_idx)
data_train = reduce(lambda x, y: x + y, dialogues_train, [])
data_test = reduce(lambda x, y: x + y, dialogues_test, [])
train_s, train_q, train_a = vectorize_data_dialog(data_train, word_idx,
answer_idx,
sentence_size,
memory_size)
test_s, test_q, test_a = vectorize_data_dialog(data_test, word_idx,
answer_idx, sentence_size,
memory_size)
print("Training Size (dialogues)", len(dialogues_train))
print("Testing Size (dialogues)", len(dialogues_test))
print("Training Size (stories)", len(data_train))
print("Testing Size (stories)", len(data_test))
tf.set_random_seed(FLAGS.random_state)
batch_size = FLAGS.batch_size
optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate,
epsilon=FLAGS.epsilon)
batches = zip(range(0,
len(data_train) - batch_size, batch_size),
range(batch_size, len(data_train), batch_size))
batches = [(start, end) for start, end in batches]
with tf.Session() as sess:
model = MemN2N(batch_size,
vocab_size,
sentence_size,
memory_size,
FLAGS.embedding_size,
answer_vocab_size=answer_vocab_size,
session=sess,
hops=FLAGS.hops,
max_grad_norm=FLAGS.max_grad_norm,
optimizer=optimizer)
best_accuracy_per_epoch = train_model(model,
(train_s, train_q, train_a),
(test_s, test_q, test_a),
batches)
return best_accuracy_per_epoch
def main(in_trainset_size, in_split_number):
# train/validation/test sets
all_dialogues_idx = reduce(lambda x, y: x + [y], range(len(all_dialogues)),
[])
random.shuffle(all_dialogues_idx)
trainset_idx = all_dialogues_idx[in_split_number *
in_trainset_size:in_split_number *
in_trainset_size + in_trainset_size]
testset_idx = filter(lambda x: x not in trainset_idx, all_dialogues_idx)
dialogues_train = map(lambda x: all_dialogues[x], trainset_idx)
dialogues_test = map(lambda x: all_dialogues[x], testset_idx)
data_train = reduce(lambda x, y: x + y, dialogues_train, [])
data_test = reduce(lambda x, y: x + y, dialogues_test, [])
train_s, train_q, train_a = vectorize_data_dialog(data_train, word_idx,
answer_idx,
sentence_size,
memory_size)
test_s, test_q, test_a = vectorize_data_dialog(data_test, word_idx,
answer_idx, sentence_size,
memory_size)
print("Training Size (dialogues)", len(dialogues_train))
print("Testing Size (dialogues)", len(dialogues_test))
print("Training Size (stories)", len(data_train))
print("Testing Size (stories)", len(data_test))
tf.set_random_seed(FLAGS.random_state)
batch_size = FLAGS.batch_size
optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate,
epsilon=FLAGS.epsilon)
batches = zip(range(0,
len(data_train) - batch_size, batch_size),
range(batch_size, len(data_train), batch_size))
batches = [(start, end) for start, end in batches]
with tf.Session() as sess:
model = MemN2N(batch_size,
vocab_size,
sentence_size,
memory_size,
FLAGS.embedding_size,
answer_vocab_size=answer_vocab_size,
session=sess,
hops=FLAGS.hops,
max_grad_norm=FLAGS.max_grad_norm,
optimizer=optimizer)
best_accuracy_per_epoch = train_model(model,
(train_s, train_q, train_a),
(test_s, test_q, test_a),
batches)
return best_accuracy_per_epoch
def main():
data_train = reduce(lambda x, y: x + y, train, [])
data_test = reduce(lambda x, y: x + y, test, [])
train_s, train_q, train_a = vectorize_data_dialog(data_train, word_idx,
answer_idx,
sentence_size,
memory_size)
test_s, test_q, test_a = vectorize_data_dialog(data_test, word_idx,
answer_idx, sentence_size,
memory_size)
print("Training Size (dialogues)", len(train))
print("Testing Size (dialogues)", len(test))
print("Training Size (stories)", len(data_train))
print("Testing Size (stories)", len(data_test))
tf.set_random_seed(FLAGS.random_state)
batch_size = FLAGS.batch_size
optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate,
epsilon=FLAGS.epsilon)
batches = zip(range(0,
len(data_train) - batch_size, batch_size),
range(batch_size, len(data_train), batch_size))
batches = [(start, end) for start, end in batches]
with tf.Session() as sess:
model = MemN2N(batch_size,
vocab_size,
sentence_size,
memory_size,
FLAGS.embedding_size,
answer_vocab_size=answer_vocab_size,
session=sess,
hops=FLAGS.hops,
max_grad_norm=FLAGS.max_grad_norm,
optimizer=optimizer)
best_accuracy_per_epoch = train_model(model,
(train_s, train_q, train_a),
(test_s, test_q, test_a),
batches)
return best_accuracy_per_epoch
val_labels = np.argmax(valA, axis=1)
tf.set_random_seed(FLAGS.random_state)
batch_size = FLAGS.batch_size
optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate,
epsilon=FLAGS.epsilon)
batches = zip(range(0, n_train - batch_size, batch_size),
range(batch_size, n_train, batch_size))
with tf.Session() as sess:
model = MemN2N(batch_size,
vocab_size,
sentence_size,
memory_size,
FLAGS.embedding_size,
session=sess,
hops=FLAGS.hops,
max_grad_norm=FLAGS.max_grad_norm,
optimizer=optimizer,
l2=FLAGS.regularization,
nonlin=tf.nn.relu)
writer = tf.train.SummaryWriter(get_log_dir_name(), sess.graph)
for t in range(1, FLAGS.epochs + 1):
np.random.shuffle(batches)
total_cost = 0.0
for start in range(0, n_train, batch_size):
end = start + batch_size
s = trainS[start:end]
q = trainQ[start:end]
def main(task_id):
tf.flags.DEFINE_integer("hops", 3, "")
tf.flags.DEFINE_integer("epochs", 100, "")
tf.flags.DEFINE_integer("embedding_size", 100, "")
tf.flags.DEFINE_float("learning_rate", 0.01, "")
tf.flags.DEFINE_float("anneal_rate", 25, "")
tf.flags.DEFINE_float("anneal_stop_epoch", 100, "")
tf.flags.DEFINE_float("max_grad_norm", 40.0, "")
tf.flags.DEFINE_integer("random_state", None, "")
tf.flags.DEFINE_integer("evaluation_interval", 10, "")
tf.flags.DEFINE_integer("batch_size", 32, "")
tf.flags.DEFINE_integer("memory_size", 50, "")
tf.flags.DEFINE_integer("task_id", task_id, "")
tf.flags.DEFINE_string("data_dir", "data/tasks_1-20_v1-2/hn/", "")
FLAGS = tf.flags.FLAGS
print("Started Task:", FLAGS.task_id)
train, test = load_task(FLAGS.data_dir, FLAGS.task_id)
data = train + test
vocab = sorted(
reduce(lambda x, y: x | y, (set(list(chain.from_iterable(s)) + q + a)
for s, q, a in data)))
max_story_size, mean_story_size, sentence_size, query_size = get_sizes(
data)
memory_size = min(FLAGS.memory_size, max_story_size)
word_idx = dict((c, i + 1) for i, c in enumerate(vocab))
inverted_word_idx = dict((i + 1, c) for i, c in enumerate(vocab))
for i in range(memory_size):
word_idx['time{}'.format(i + 1)] = 'time{}'.format(i + 1)
vocab_size = len(word_idx) + 1
sentence_size = max(query_size, sentence_size)
sentence_size += 1
S, Q, A = vectorize_data(train, word_idx, sentence_size, memory_size)
trainS, valS, trainQ, valQ, trainA, valA = train_test_split(
S, Q, A, test_size=.1, random_state=FLAGS.random_state)
testS, testQ, testA = vectorize_data(test, word_idx, sentence_size,
memory_size)
n_test, n_train, n_val = get_shapes(testS, trainS, valS)
train_labels, test_labels, val_labels = get_labels(trainA, testA, valA)
tf.set_random_seed(FLAGS.random_state)
batch_size, batches = get_batches(n_train, FLAGS)
with tf.Session() as sess:
model = MemN2N(batch_size,
vocab_size,
sentence_size,
memory_size,
FLAGS.embedding_size,
session=sess,
hops=FLAGS.hops,
max_grad_norm=FLAGS.max_grad_norm)
for t in range(1, FLAGS.epochs + 1):
lr = get_updated_learning_rate(t, FLAGS)
np.random.shuffle(batches)
total_cost = get_total_cost(batches, model, trainS, trainQ, trainA,
lr)
if t % FLAGS.evaluation_interval == 0:
train_predictions = get_train_predictions(
FLAGS, trainS, trainQ, n_train, batch_size, model)
validation_predictions = model.predict(valS, valQ)
training_accuracy = metrics.accuracy_score(
np.array(train_predictions), train_labels)
validation_accuracy = metrics.accuracy_score(
validation_predictions, val_labels)
print("******************************************")
print('Epoch', t)
print('Training Accuracy:', training_accuracy)
print('Validation Accuracy:', validation_accuracy)
print('Total Cost:', total_cost)
print("******************************************")
# Saving Model
# input_dic = {"stories":model._stories, "queries":model._queries, "answers":model._answers, "learning_rate":model._lr}
# output_dic = {model._logits}
# tf.saved_model.simple_save(
# sess, "./saved_models/task_"+FLAGS.task_id, input_dic, output_dic
# )
test_predictions = model.predict(testS, testQ)
testing_accuracy = metrics.accuracy_score(test_predictions,
test_labels)
print("Final Testing Accuracy:", testing_accuracy)
print("")
printTests(1, 6, test, test_predictions, test_labels,
inverted_word_idx)
else:
os.mkdir(FLAGS.test_dir)
os.mkdir(lead3_dir)
os.mkdir(mem_dir)
os.mkdir(lead3_dir + '/system')
os.mkdir(lead3_dir + '/model')
os.mkdir(mem_dir + '/system')
os.mkdir(mem_dir + '/model')
print('Created test directory', FLAGS.test_dir)
if os.path.exists('se.txt'):
os.remove('se.txt')
model = MemN2N(batch_size, vocab_size, sentence_size, memory_size, FLAGS.embedding_size, next_element,
FLAGS.keep_prob, l2_para=FLAGS.l2, word_init=FLAGS.word2vec_init,
hidden_size=FLAGS.hidden_size,
training=False,
session=sess,
hops=FLAGS.hops, max_grad_norm=FLAGS.max_grad_norm)
print('created model')
saver = tf.train.Saver()
saver.restore(sess, FLAGS.train_dir)
names=[]
cnt = -1 # file index
for epoch in range(1, FLAGS.epochs + 1):
epoch_start_time = time.time()
total_cost = 0.0
sess.run(iterator.initializer, feed_dict={filenames: test_filenames})
while True:
try:
epsilon=FLAGS.epsilon)
batches = zip(range(0, n_train - batch_size, batch_size),
range(batch_size, n_train, batch_size))
batches = [(start, end) for start, end in batches]
linear_start = FLAGS.linear_start
last_train_acc, last_val_acc = None, None
with tf.Session() as sess:
model = MemN2N(batch_size,
vocab_size,
sentence_size,
memory_size,
FLAGS.embedding_size,
session=sess,
weight_tying=FLAGS.weight_tying,
hops=FLAGS.hops,
max_grad_norm=FLAGS.max_grad_norm,
optimizer=optimizer,
global_step=global_step)
for t in range(1, FLAGS.epochs + 1):
np.random.shuffle(batches)
total_cost = 0.0
if t > FLAGS.ls_epoch:
linear_start = False
for start, end in batches:
s = trainS[start:end]
q = trainQ[start:end]
a = trainA[start:end]
best_val_accs = [-1] * len(tasks)
best_val_epochs = [-1] * len(tasks)
best_val_update_epoch = -1
stop_early = False
best_train_accs = []
with tf.Session() as sess:
print(batch_size, vocab_size, sentence_size, memory_size,
FLAGS.embedding_size, FLAGS.hops, FLAGS.max_grad_norm,
FLAGS.regularization, FLAGS.learning_rate, FLAGS.epsilon)
model = MemN2N(batch_size,
vocab_size,
sentence_size,
memory_size,
FLAGS.embedding_size,
session=sess,
hops=FLAGS.hops,
max_grad_norm=FLAGS.max_grad_norm,
l2=FLAGS.regularization,
lr=FLAGS.learning_rate,
epsilon=FLAGS.epsilon,
nonlin=tf.nn.relu)
writers = gen_writers(sess, get_log_dir_name())
for i in range(1, FLAGS.epochs + 1):
np.random.shuffle(batches)
total_cost = 0.0
for start, end in batches:
end = start + batch_size
s = trainS[start:end]
q = trainQ[start:end]
def run_memn2n_joint_training(tensorflow_commandline_flags):
# parsing commandline input for defining tasks to log with each observation
calc_accuracy_tasks = []
if tensorflow_commandline_flags.sigopt_calc_accuracy_tasks.split is not None:
calc_accuracy_tasks = [
int(i) for i in
tensorflow_commandline_flags.sigopt_calc_accuracy_tasks.split(",")
]
# preprocessing data
# load all train/test data
ids = range(1, 21)
train, test = [], []
for i in ids:
tr, te = load_task(tensorflow_commandline_flags.data_dir, i)
train.append(tr)
test.append(te)
data = list(chain.from_iterable(train + test))
vocab = sorted(
reduce(lambda x, y: x | y, (set(list(chain.from_iterable(s)) + q + a)
for s, q, a in data)))
word_idx = dict((c, i + 1) for i, c in enumerate(vocab))
max_story_size = max(map(len, (s for s, _, _ in data)))
mean_story_size = int(np.mean([len(s) for s, _, _ in data]))
sentence_size = max(map(len, chain.from_iterable(s for s, _, _ in data)))
query_size = max(map(len, (q for _, q, _ in data)))
sentence_size = max(query_size, sentence_size) # for the position
sentence_size += 1 # +1 for time words
logging.debug("setting up sigopt experiment")
sigopt_experiment_definition, e2e_memnn_experiment = sigopt_memn2n_experiment_setup.setup_sigopt_memn2n_experiment(
tensorflow_commandline_flags)
while e2e_memnn_experiment.progress.observation_count < e2e_memnn_experiment.observation_budget:
logging.info("starting new observation cycle")
logging.info("observation number: %d",
e2e_memnn_experiment.progress.observation_count)
logging.debug("getting sigopt suggestions")
suggestions = sigopt_experiment_definition.get_suggestions(
e2e_memnn_experiment)
memory_size = suggestions.assignments[ParametersList.MEMORY_SIZE.value]
# Add time words/indexes
for i in range(memory_size):
word_idx['time{}'.format(i + 1)] = 'time{}'.format(i + 1)
vocab_size = len(word_idx) + 1 # +1 for nil word
logging.info("Longest sentence length %d", sentence_size)
logging.info("Longest story length %d", max_story_size)
logging.info("Average story length %d", mean_story_size)
logging.info("transforming data")
# train/validation/test sets
trainS = []
valS = []
trainQ = []
valQ = []
trainA = []
valA = []
for task in train:
S, Q, A = vectorize_data(task, word_idx, sentence_size,
memory_size)
ts, vs, tq, vq, ta, va = cross_validation.train_test_split(
S,
Q,
A,
test_size=0.1,
random_state=tensorflow_commandline_flags.random_state)
trainS.append(ts)
trainQ.append(tq)
trainA.append(ta)
valS.append(vs)
valQ.append(vq)
valA.append(va)
trainS = reduce(lambda a, b: np.vstack((a, b)), (x for x in trainS))
trainQ = reduce(lambda a, b: np.vstack((a, b)), (x for x in trainQ))
trainA = reduce(lambda a, b: np.vstack((a, b)), (x for x in trainA))
valS = reduce(lambda a, b: np.vstack((a, b)), (x for x in valS))
valQ = reduce(lambda a, b: np.vstack((a, b)), (x for x in valQ))
valA = reduce(lambda a, b: np.vstack((a, b)), (x for x in valA))
testS, testQ, testA = vectorize_data(list(chain.from_iterable(test)),
word_idx, sentence_size,
memory_size)
n_train = trainS.shape[0]
n_val = valS.shape[0]
n_test = testS.shape[0]
logging.info("Training Size: %d", n_train)
logging.info("Validation Size: %d", n_val)
logging.info("Testing Size: %d", n_test)
train_labels = np.argmax(trainA, axis=1)
test_labels = np.argmax(testA, axis=1)
val_labels = np.argmax(valA, axis=1)
tf.set_random_seed(tensorflow_commandline_flags.random_state)
batch_size = tensorflow_commandline_flags.batch_size
# This avoids feeding 1 task after another, instead each batch has a random sampling of tasks
batches = zip(range(0, n_train - batch_size, batch_size),
range(batch_size, n_train, batch_size))
batches = [(start, end) for start, end in batches]
optimizer = sigopt_memn2n_experiment_setup.string_to_optimizer_object(
suggestions.assignments[ParametersList.OPTIMIZER.value],
suggestions.assignments)
with tf.Session(config=tf.ConfigProto(
log_device_placement=True)) as sess:
model = MemN2N(
batch_size,
vocab_size,
sentence_size,
memory_size=memory_size,
embedding_size=suggestions.assignments[
ParametersList.WORD_EMBEDDING.value],
optimizer=optimizer,
session=sess,
hops=suggestions.assignments[ParametersList.HOP_SIZE.value],
max_grad_norm=tensorflow_commandline_flags.max_grad_norm)
logging.info("batch training memory network")
for i in range(1, tensorflow_commandline_flags.epochs + 1):
logging.debug("epoch number %d", i)
logging.debug("observation count %d",
e2e_memnn_experiment.progress.observation_count)
np.random.shuffle(batches)
total_cost = 0.0
for start, end in batches:
s = trainS[start:end]
q = trainQ[start:end]
a = trainA[start:end]
cost_t = model.batch_fit(s, q, a)
total_cost += cost_t
if i % tensorflow_commandline_flags.evaluation_interval == 0:
logging.info(
"calculating training and validation accuracy.")
train_accs = []
for start in range(0, n_train, int(n_train / 20)):
end = start + int(n_train / 20)
s = trainS[start:end]
q = trainQ[start:end]
pred = model.predict(s, q)
acc = metrics.accuracy_score(pred,
train_labels[start:end])
train_accs.append(acc)
logging.debug("Training accuracy %f",
np.average(train_accs))
val_accs = []
for start in range(0, n_val, int(n_val / 20)):
end = start + int(n_val / 20)
s = valS[start:end]
q = valQ[start:end]
pred = model.predict(s, q)
acc = metrics.accuracy_score(pred,
val_labels[start:end])
val_accs.append(acc)
logging.debug("Validation accuracy %f",
np.average(val_accs))
test_accs = []
for start in range(0, n_test, int(n_test / 20)):
end = start + int(n_test / 20)
s = testS[start:end]
q = testQ[start:end]
pred = model.predict(s, q)
acc = metrics.accuracy_score(pred,
test_labels[start:end])
test_accs.append(acc)
logging.info('-----------------------')
logging.info('Total Cost: %d', total_cost)
task_accuracies = []
t = 1
for t1, t2, t3 in zip(train_accs, val_accs, test_accs):
logging.info("Task %d", t)
logging.info("Training Accuracy %f", t1)
logging.info("Validation Accuracy %f", t2)
logging.info("Testing Accuracy %f", t3)
if t in calc_accuracy_tasks:
task_accuracies.append(t3)
t += 1
logging.info('-----------------------')
# log task test accuracies with current observation
metadata_dict = {}
for t, task_name in enumerate(calc_accuracy_tasks):
metadata_dict[str(task_name)] = task_accuracies[t]
test_accs_average = np.average(test_accs)
logging.debug("creating sigopt observation")
try:
e2e_memnn_experiment, current_observation = sigopt_experiment_definition.update_experiment_metadata(
e2e_memnn_experiment, suggestions,
test_accs_average, metadata_dict)
except ConnectionError as error:
logging.debug("connection problem: %s", str(error))
conn = Connection(
client_token=tensorflow_commandline_flags.
sigopt_connection_token)
conn.experiments(
e2e_memnn_experiment.id).observations().create(
suggestion=suggestions.id,
value=test_accs_average,
metadata=metadata_dict)
e2e_memnn_experiment = conn.experiments(
e2e_memnn_experiment.id).fetch()
tf.reset_default_graph()
logging.info(
"Sig opt best parameters: %s",
sigopt_experiment_definition.get_best_suggestions(
e2e_memnn_experiment))
def main(args=sys.argv[1:]):
args = parse_args(args)
logging.basicConfig(level=args.logging,
format='%(asctime)s\t%(levelname)-8s\t%(message)s')
output_path = os.path.join(
args.output_dir_path,
#'%s_%s' % (get_git_revision_short_hash(), datetime.datetime.now().time().isoformat())
datetime.datetime.now().date().isoformat() + '_' +
datetime.datetime.now().time().isoformat(),
)
optimizer = tf.train.AdamOptimizer(learning_rate=args.learning_rate)
encodings = {
'bow_encoding': bow_encoding,
'position_encoding': position_encoding,
}
nonlins = {
'relu': tf.nn.relu,
}
if args.nonlin is not None:
nonlin = nonlins[args.nonlin]
else:
nonlin = None
tf.set_random_seed(args.random_seed)
if not args.joint:
for task_id in args.task_ids:
with tf.Graph().as_default():
with tf.Session(graph=tf.get_default_graph()) as sess:
train_data, val_data, test_data, word_idx, reverse_word_idx, vocab_size, sentence_size, memory_size = load_data(
args.data_path, [task_id], args.dim_memory,
args.num_caches, args.random_seed)
print(args.temporal_encoding)
model = MemN2N(
args.batch_size,
vocab_size,
sentence_size,
memory_size,
args.num_caches,
args.dim_emb,
word_idx,
reverse_word_idx,
args.num_hops,
args.max_grad_norm,
share_type=args.share_type,
nonlin=nonlin,
optimizer=optimizer,
initializer=tf.random_normal_initializer(
stddev=args.init_stddev),
encoding=encodings[args.encoding_type],
temporal_encoding=args.temporal_encoding,
session=sess,
)
saver = tf.train.Saver()
params = list_to_path(vars(args).values())
saver.save(sess, './save/model_%s' % params)
train_loop(model, train_data, val_data, args.batch_size,
args.num_epochs, args.val_freq)
d = {'vocab_dict': word_idx}
for f in test_data:
test_accs, test_attendance_acc, test_preds, test_probs = evaluate_per_question(
model, test_data[f], output_path)
test_task_name = os.path.basename(
os.path.splitext(f)[0])
d.update({
'%s_test_preds' % test_task_name:
test_preds,
'%s_test_probs' % test_task_name:
test_probs[0],
'%s_test_r' % test_task_name:
test_probs[1],
'%s_test_acc' % test_task_name:
test_accs,
#'%s_test_acc_0' % test_task_name: test_accs[0],
#'%s_test_acc_1' % test_task_name: test_accs[1],
#'%s_test_acc_2' % test_task_name: test_accs[2],
#'%s_test_acc_3' % test_task_name: test_accs[3],
'%s_test_attendance_accs' % test_task_name:
test_attendance_acc,
})
vars_args = vars(args)
del vars_args['task_ids']
vars_args['task_ids'] = [task_id]
d.update(**vars_args)
np.save(output_path + '_%d' % task_id, d)
else:
with tf.Graph().as_default():
with tf.Session(graph=tf.get_default_graph()) as sess:
train_data, val_data, test_data, word_idx, reverse_word_idx, vocab_size, sentence_size, memory_size = load_data(
args.data_path, args.task_ids, args.dim_memory,
args.num_caches, args.random_seed)
print(args.temporal_encoding)
model = MemN2N(
args.batch_size,
vocab_size,
sentence_size,
memory_size,
args.num_caches,
args.dim_emb,
word_idx,
reverse_word_idx,
args.num_hops,
args.max_grad_norm,
share_type=args.share_type,
nonlin=nonlin,
optimizer=optimizer,
initializer=tf.random_normal_initializer(
stddev=args.init_stddev),
encoding=encodings[args.encoding_type],
temporal_encoding=args.temporal_encoding,
session=sess,
)
saver = tf.train.Saver()
params = list_to_path(vars(args).values())
saver.save(
sess,
'/home/kayleeburns/memory/memn2n/ckpts/consistency/model_%s'
% params)
train_loop(model, train_data, val_data, args.batch_size,
args.num_epochs, args.val_freq)
d = {'vocab_dict': word_idx}
# FIND TEST ACCURACY
for f in test_data:
test_accs, test_attendance_acc, test_preds, test_probs = evaluate_per_question(
model, test_data[f], output_path)
test_task_name = os.path.basename(os.path.splitext(f)[0])
d.update({
'%s_test_preds' % test_task_name: test_preds,
'%s_test_probs' % test_task_name: test_probs,
'%s_test_acc' % test_task_name: test_accs,
#'%s_test_acc_0' % test_task_name: test_accs[0],
#'%s_test_acc_1' % test_task_name: test_accs[1],
#'%s_test_acc_2' % test_task_name: test_accs[2],
#'%s_test_acc_3' % test_task_name: test_accs[3],
#'%s_test_attendance_accs' % test_task_name: test_attendance_acc,
})
d.update(**vars(args))
np.save(output_path, d)
print("Validation Size", n_val)
print("Testing Size", n_test)
train_labels = np.argmax(trainA, axis=1)
test_labels = np.argmax(testA, axis=1)
val_labels = np.argmax(valA, axis=1)
tf.set_random_seed(FLAGS.random_state)
batch_size = FLAGS.batch_size
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
with tf.Session() as sess:
model = MemN2N(
batch_size,
vocab_size,
sentence_size,
memory_size,
FLAGS.embedding_size,
session=sess, # max_gradient_norm=FLAGS.max_gradient_norm,
hops=FLAGS.num_hops,
optimizer=optimizer)
for t in range(1, FLAGS.num_epochs + 1):
total_cost = 0.0
for start in range(0, n_train, batch_size):
end = start + batch_size
s = trainS[start:end]
q = trainQ[start:end]
a = trainA[start:end]
cost_t = model.batch_fit(s, q, a)
total_cost += cost_t
if t % FLAGS.evaluation_interval == 0:
def run_task(task_id):
print("Started Task:", task_id)
# task data
train, test = load_task(FLAGS.data_dir, task_id)
data = train + test
vocab = sorted(
reduce(lambda x, y: x | y, (set(list(chain.from_iterable(s)) + q + a)
for s, q, a in data)))
word_idx = dict((c, i + 1) for i, c in enumerate(vocab))
reverse_lookup = {v: k for (k, v) in word_idx.items()}
lookup_vocab = ['nil']
print(reverse_lookup)
print(word_idx)
for i in range(1, len(reverse_lookup) + 1):
lookup_vocab.append(reverse_lookup[i])
max_story_size = max(map(len, (s for s, _, _ in data)))
mean_story_size = int(np.mean([len(s) for s, _, _ in data]))
sentence_size = max(map(len, chain.from_iterable(s for s, _, _ in data)))
query_size = max(map(len, (q for _, q, _ in data)))
memory_size = min(FLAGS.memory_size, max_story_size)
# Add time words/indexes
for i in range(memory_size):
word_idx['time{}'.format(i + 1)] = 'time{}'.format(i + 1)
print(len(word_idx))
print(word_idx)
vocab_size = len(word_idx) + 1 # +1 for nil word
sentence_size = max(query_size, sentence_size) # for the position
sentence_size += 1 # +1 for time words
print("Longest sentence length", sentence_size)
print("Longest story length", max_story_size)
print("Average story length", mean_story_size)
# train/validation/test sets
S, Q, A = vectorize_data(train, word_idx, sentence_size, memory_size)
trainS, valS, trainQ, valQ, trainA, valA = cross_validation.train_test_split(
S, Q, A, test_size=.1, random_state=FLAGS.random_state)
testS, testQ, testA = vectorize_data(test, word_idx, sentence_size,
memory_size)
#print(testS[0])
print("Training set shape", trainS.shape)
# params
n_train = trainS.shape[0]
n_test = testS.shape[0]
n_val = valS.shape[0]
print("Training Size", n_train)
print("Validation Size", n_val)
print("Testing Size", n_test)
train_labels = np.argmax(trainA, axis=1)
val_labels = np.argmax(valA, axis=1)
test_labels = np.argmax(testA, axis=1)
tf.set_random_seed(FLAGS.random_state)
batch_size = FLAGS.batch_size
batches = zip(range(0, n_train - batch_size, batch_size),
range(batch_size, n_train, batch_size))
batches = [(start, end) for start, end in batches]
train_accuracies = []
validation_accuracies = []
test_accuracies = []
max_testAccuracy = 0
max_trainAccuracy = 0
max_valAccuracy = 0
best_test_prob_hops = 0
best_test_prob_vocab = 0
best_pred_word = 0
best_true_word = 0
best_test_A1 = 0
best_test_C = 0
best_lookup_vocab = 0
for run_id in range(FLAGS.NoOfRuns):
print('Run Number: ' + str(run_id))
max_epoch_trainAccuracy = 0
max_epoch_valAccuracy = 0
with tf.Session() as sess:
model = MemN2N(batch_size,
vocab_size,
sentence_size,
memory_size,
FLAGS.embedding_size,
session=sess,
hops=FLAGS.hops,
max_grad_norm=FLAGS.max_grad_norm,
regularization=FLAGS.regularization,
nonlin=FLAGS.nonlin)
for t in range(1, FLAGS.epochs + 1):
# Stepped learning rate
if t - 1 <= FLAGS.anneal_stop_epoch:
anneal = 2.0**((t - 1) // FLAGS.anneal_rate)
else:
anneal = 2.0**(FLAGS.anneal_stop_epoch //
FLAGS.anneal_rate)
lr = FLAGS.learning_rate / anneal
np.random.shuffle(batches)
total_cost = 0.0
for start, end in batches:
s = trainS[start:end]
q = trainQ[start:end]
a = trainA[start:end]
cost_t = model.batch_fit(s, q, a, lr)
total_cost += cost_t
if t % FLAGS.evaluation_interval == 0:
train_preds = []
for start in range(0, n_train, batch_size):
end = start + batch_size
s = trainS[start:end]
q = trainQ[start:end]
pred = model.predict(s, q)
train_preds += list(pred)
val_preds, valid_prob_vocab, valid_prob_hops, valid_A1, valid_C = model.predict_prob_instrument(
valS, valQ)
train_acc = metrics.accuracy_score(np.array(train_preds),
train_labels)
val_acc = metrics.accuracy_score(val_preds, val_labels)
print('-----------------------')
print('Epoch', t)
print('Total Cost:', total_cost)
print('Training Accuracy:', train_acc)
print('Validation Accuracy:', val_acc)
print('-----------------------')
if (val_acc > max_epoch_valAccuracy):
max_epoch_trainAccuracy = train_acc
max_epoch_valAccuracy = val_acc
test_preds, test_prob_vocab, test_prob_hops, test_A1, test_C = model.predict_prob_instrument(
testS, testQ)
pred_word = [reverse_lookup[i] for i in test_preds]
true_word = [reverse_lookup[i] for i in test_labels]
test_acc = metrics.accuracy_score(test_preds, test_labels)
train_accuracies.append(max_trainAccuracy)
validation_accuracies.append(max_valAccuracy)
test_accuracies.append(test_acc)
if (test_acc > max_testAccuracy):
max_testAccuracy = test_acc
max_trainAccuracy = max_epoch_trainAccuracy
max_valAccuracy = max_epoch_valAccuracy
best_test_prob_hops = test_prob_hops
best_test_prob_vocab = test_prob_vocab
best_pred_word = pred_word
best_true_word = true_word
best_test_A1 = test_A1
best_test_C = test_C
best_lookup_vocab = lookup_vocab
print("Test Accuracy: ", test_acc)
iou = calculate_iou(test_prob_hops, test, task_id)
print('IoU: ' + str(iou))
print("Best Testing Accuracy:", max_testAccuracy)
# save test files
np.save(logs_dir + 'task_' + str(task_id) + '_attention',
best_test_prob_hops)
np.save(logs_dir + 'task_' + str(task_id) + '_vocab_prob',
best_test_prob_vocab)
np.save(logs_dir + 'task_' + str(task_id) + '_pred', best_pred_word)
np.save(logs_dir + 'task_' + str(task_id) + '_truth', best_true_word)
np.save(logs_dir + 'task_' + str(task_id) + '_A', best_test_A1)
np.save(logs_dir + 'task_' + str(task_id) + '_C', np.array(best_test_C))
np.save(logs_dir + 'task_' + str(task_id) + '_lookupvocab',
best_lookup_vocab)
iou = calculate_iou(best_test_prob_hops, test, task_id)
print('Best IoU: ' + str(iou))
return max_trainAccuracy, max_valAccuracy, max_testAccuracy, iou
# Make only 1 GPU available to CUDA
os.environ[
'CUDA_VISIBLE_DEVICES'] = "0" # Comma separated indexes of GPUs to use - GPUs are indexed from 0 to 7 on the workstation
# Set the options to limit the memory allocation on GPUs
tf_gpu_options = tf.GPUOptions(allow_growth=True,
per_process_gpu_memory_fraction=0.5)
sess = tf.Session(config=tf.ConfigProto(gpu_options=tf_gpu_options))
model = MemN2N(config["batch"],
config["vocab_size"],
config["sentence_size"],
config["memory_size"],
config["embedding_size"],
session=sess,
hops=config["hops"],
max_grad_norm=config["max_grad_norm"],
l2=config["regularization"],
lr=config["lr"],
epsilon=config["epsilon"],
nonlin=tf.nn.relu,
restoreLoc=restore_location)
# Uncomment to see if the weights were loaded correctly
# print(sess.run(model.A))
def get_pred(testS, testQ):
# print(testS)
ps = model.predict_proba(testS, testQ)
op = model.predict_test(testS, testQ)
tf.flags.DEFINE_float("max_grad_norm", 40.0, "Clip gradients to this norm.")
tf.flags.DEFINE_integer("evaluation_interval", 1,
"Evaluate and print results every x epochs")
tf.flags.DEFINE_integer("batch_size", 32, "Batch size for training.")
tf.flags.DEFINE_integer("hops", 3, "Number of hops in the Memory Network.")
tf.flags.DEFINE_integer("epochs", 50, "Number of epochs to train for.")
tf.flags.DEFINE_integer("embedding_size", 20,
"Embedding size for embedding matrices.")
tf.flags.DEFINE_integer("memory_size", 50, "Maximum size of memory.")
tf.flags.DEFINE_integer("class_size", 6, "Number of classes.")
tf.flags.DEFINE_integer("random_state", 20, "Random state.")
tf.flags.DEFINE_string("dataset", "dblp", "Directory containing bAbI tasks")
tf.flags.DEFINE_integer("sentence_size", "15", "maximum sentence size")
tf.flags.DEFINE_string("nonlin", "None", "nonlinear function ")
tf.flags.DEFINE_string("checkpoint_dir", "checkpoint",
"Directory name to save the checkpoints [checkpoints]")
tf.flags.DEFINE_string("encoder", "avg", " the approach of sentence encoder")
tf.flags.DEFINE_string("filter_sizes", "3",
"the filter sizes in convolutional neural networks")
tf.flags.DEFINE_float("dropout_keep_prob", 1.0, "Dropout keep probability")
tf.flags.DEFINE_integer("share_memory_size", 20, "Share memory size")
tf.flags.DEFINE_string("embedding_file", "doc.txt", "document embedding file")
FLAGS = tf.flags.FLAGS
print("class_size:", FLAGS.class_size)
with tf.Session() as sess:
reader = TextReader(FLAGS)
model = MemN2N(session=sess, reader=reader, config=FLAGS)
model.train()
train_labels = np.argmax(trainA, axis=1)
test_labels = np.argmax(testA, axis=1)
val_labels = np.argmax(valA, axis=1)
tf.set_random_seed(FLAGS.random_state)
batch_size = FLAGS.batch_size
batches = list(
zip(range(0, n_train - batch_size, batch_size),
range(batch_size, n_train, batch_size)))
with tf.Session() as sess:
model = MemN2N(batch_size,
vocab_size,
sentence_size,
memory_size,
FLAGS.embedding_size,
session=sess,
hops=FLAGS.hops,
max_grad_norm=FLAGS.max_grad_norm)
for t in range(1, FLAGS.epochs + 1):
# Stepped learning rate
if t - 1 <= FLAGS.anneal_stop_epoch:
anneal = 2.0**((t - 1) // FLAGS.anneal_rate)
else:
anneal = 2.0**(FLAGS.anneal_stop_epoch // FLAGS.anneal_rate)
lr = FLAGS.learning_rate / anneal
np.random.shuffle(batches)
total_cost = 0.0
for start, end in batches:
s = trainS[start:end]
def run_memn2n_single_training(tensorflow_commandline_flags):
logging.info("Started Task: %s", str(tensorflow_commandline_flags.task_id))
# preprocessing data before training memory network
# task data
train, test = load_task(tensorflow_commandline_flags.data_dir,
tensorflow_commandline_flags.task_id)
data = train + test
vocab = sorted(
reduce(lambda x, y: x | y, (set(list(chain.from_iterable(s)) + q + a)
for s, q, a in data)))
word_idx = dict((c, i + 1) for i, c in enumerate(vocab))
max_story_size = max(map(len, (s for s, _, _ in data)))
mean_story_size = int(np.mean([len(s) for s, _, _ in data]))
sentence_size = max(map(len, chain.from_iterable(s for s, _, _ in data)))
query_size = max(map(len, (q for _, q, _ in data)))
sentence_size = max(query_size, sentence_size) # for the position
sentence_size += 1 # +1 for time words
sigopt_experiment_definition, e2e_memnn_experiment = sigopt_memn2n_experiment_setup.setup_sigopt_memn2n_experiment(
tensorflow_commandline_flags)
while e2e_memnn_experiment.progress.observation_count < e2e_memnn_experiment.observation_budget:
logging.info("observation number: %d",
e2e_memnn_experiment.progress.observation_count)
suggestions = sigopt_experiment_definition.get_suggestions(
e2e_memnn_experiment)
memory_size = suggestions.assignments[ParametersList.MEMORY_SIZE.value]
# Add time words/indexes
for i in range(memory_size):
word_idx['time{}'.format(i + 1)] = 'time{}'.format(i + 1)
vocab_size = len(word_idx) + 1 # +1 for nil word
logging.info("Longest sentence length %d", sentence_size)
logging.info("Longest story length %d", max_story_size)
logging.info("Average story length %d", mean_story_size)
# train/validation/test sets
S, Q, A = vectorize_data(train, word_idx, sentence_size, memory_size)
trainS, valS, trainQ, valQ, trainA, valA = cross_validation.train_test_split(
S,
Q,
A,
test_size=.1,
random_state=tensorflow_commandline_flags.random_state)
testS, testQ, testA = vectorize_data(test, word_idx, sentence_size,
memory_size)
# params
n_train = trainS.shape[0]
n_test = testS.shape[0]
n_val = valS.shape[0]
logging.info("Training Size %d", n_train)
logging.info("Validation Size %d", n_val)
logging.info("Testing Size %d", n_test)
train_labels = np.argmax(trainA, axis=1)
test_labels = np.argmax(testA, axis=1)
val_labels = np.argmax(valA, axis=1)
tf.set_random_seed(tensorflow_commandline_flags.random_state)
batch_size = tensorflow_commandline_flags.batch_size
batches = zip(range(0, n_train - batch_size, batch_size),
range(batch_size, n_train, batch_size))
batches = [(start, end) for start, end in batches]
optimizer = sigopt_memn2n_experiment_setup.string_to_optimizer_object(
suggestions.assignments[ParametersList.OPTIMIZER.value],
suggestions.assignments)
with tf.Session(config=tf.ConfigProto(
log_device_placement=True)) as sess:
model = MemN2N(
batch_size,
vocab_size,
sentence_size,
memory_size=memory_size,
embedding_size=suggestions.assignments[
ParametersList.WORD_EMBEDDING.value],
optimizer=optimizer,
session=sess,
hops=suggestions.assignments[ParametersList.HOP_SIZE.value],
max_grad_norm=tensorflow_commandline_flags.max_grad_norm)
for t in range(1, tensorflow_commandline_flags.epochs + 1):
logging.info("epoch number: %d", t)
logging.info("observation number: %d",
e2e_memnn_experiment.progress.observation_count)
np.random.shuffle(batches)
total_cost = 0.0
for start, end in batches:
s = trainS[start:end]
q = trainQ[start:end]
a = trainA[start:end]
cost_t = model.batch_fit(s, q, a)
total_cost += cost_t
if t % tensorflow_commandline_flags.evaluation_interval == 0:
train_preds = []
for start in range(0, n_train, batch_size):
end = start + batch_size
s = trainS[start:end]
q = trainQ[start:end]
pred = model.predict(s, q)
train_preds += list(pred)
val_preds = model.predict(valS, valQ)
train_acc = metrics.accuracy_score(np.array(train_preds),
train_labels)
val_acc = metrics.accuracy_score(val_preds, val_labels)
test_preds = model.predict(testS, testQ)
test_acc = metrics.accuracy_score(test_preds, test_labels)
logging.info('-----------------------')
logging.info('Epoch %d', t)
logging.info('Total Cost: %f', total_cost)
logging.info('Training Accuracy: %f', train_acc)
logging.info('Validation Accuracy: %f', val_acc)
logging.info('Test Accuracy: %f', test_acc)
logging.info('-----------------------')
e2e_memnn_experiment, observation = sigopt_experiment_definition.update_experiment(
e2e_memnn_experiment, suggestions, test_acc)
# reset computation graph to create new mm model
tf.reset_default_graph()
logging.info(
"Sig opt best parameters: %s",
sigopt_experiment_definition.get_best_suggestions(
e2e_memnn_experiment))
def main(
in_trainset_size,
in_testset_size,
in_fold_number,
in_dataset_shuffle
):
max_dialogue_length = max(map(len, all_dialogues_babi))
longer_dialogues_idx = filter(
lambda x: len(all_dialogues_babi[x]) == max_dialogue_length,
in_dataset_shuffle
)
trainset_idx = []
for train_dialogue_counter in range(in_trainset_size * in_fold_number, in_trainset_size * (in_fold_number + 1)):
trainset_idx.append(longer_dialogues_idx[train_dialogue_counter % len(longer_dialogues_idx)])
testset_idx = []
for test_dialogue_counter in range(in_testset_size * in_fold_number, in_testset_size * (in_fold_number + 1)):
testset_idx.append(in_dataset_shuffle[test_dialogue_counter % len(in_dataset_shuffle)])
dialogues_train = map(lambda x: all_dialogues_babi[x], trainset_idx)
# testing on API calls only?
dialogues_test = map(lambda x: [all_dialogues_babi_plus[x][-1]], testset_idx)
data_train = reduce(lambda x, y: x + y, dialogues_train, [])
data_test = reduce(lambda x, y: x + y, dialogues_test, [])
train_s, train_q, train_a = vectorize_data_dialog(
data_train,
word_idx,
answer_idx,
sentence_size,
memory_size
)
test_s, test_q, test_a = vectorize_data_dialog(
data_test,
word_idx,
answer_idx,
sentence_size,
memory_size
)
print("Training Size (dialogues)", len(dialogues_train))
print("Testing Size (dialogues)", len(dialogues_test))
print("Training Size (stories)", len(data_train))
print("Testing Size (stories)", len(data_test))
tf.set_random_seed(FLAGS.random_state)
batch_size = FLAGS.batch_size
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=FLAGS.learning_rate # ,
# epsilon=FLAGS.epsilon
)
batches = zip(
range(0, len(data_train) - batch_size, batch_size),
range(batch_size, len(data_train), batch_size)
)
batches = [(start, end) for start, end in batches]
with tf.Session() as sess:
model = MemN2N(
batch_size,
vocab_size,
sentence_size,
memory_size,
FLAGS.embedding_size,
answers_vectorized,
session=sess,
hops=FLAGS.hops,
max_grad_norm=FLAGS.max_grad_norm,
optimizer=optimizer
)
best_accuracy_per_epoch = train_model(
model,
(train_s, train_q, train_a),
(test_s, test_q, test_a),
batches
)
return best_accuracy_per_epoch
val_batches = zip(range(0, n_val - batch_size + 1, batch_size),
range(batch_size, n_val + 1, batch_size))
val_batches = [(start, end) for start, end in val_batches]
if (FLAGS.use_testset):
test_batches = zip(range(0, n_test - batch_size + 1, batch_size),
range(batch_size, n_test + 1, batch_size))
test_batches = [(start, end) for start, end in test_batches]
with tf.Graph().as_default() as graph:
with tf.Session() as sess:
model = MemN2N(batch_size,
vocab_size,
ans_vec_size,
sentence_size,
memory_size,
FLAGS.embedding_size,
FLAGS.memn2n_vector_size,
FLAGS.loss_norm,
session=sess,
hops=FLAGS.hops,
max_grad_norm=FLAGS.max_grad_norm,
optimizer=optimizer)
saver = XlsxExporter(story)
header = {}
header['Story'] = story
header['Learning Rate'] = str(FLAGS.learning_rate)
header['Training Size'] = str(n_train)
header['Validation Size'] = str(n_val)
header['Sentence Size'] = str(sentence_size)
header['Answer Vector Size'] = str(ans_vec_size)
header['MemN2N Vector Size'] = str(FLAGS.memn2n_vector_size)
