def test_restore(self):
current_dir = os.path.dirname(__file__)
ckpts_dir = os.path.join(current_dir, 'checkpoints')
model1_output, model1_memview = None, None
sample_input = np.random.uniform(0, 1, (2, 5, 10)).astype(np.float32)
sample_seq_len = 5
graph1 = tf.Graph()
with graph1.as_default():
with tf.Session(graph=graph1) as session1:
computer = DNC(DummyController,
10,
20,
10,
10,
64,
2,
batch_size=2)
session1.run(tf.initialize_all_variables())
saved_weights = session1.run([
computer.controller.nn_output_weights,
computer.controller.interface_weights,
computer.controller.mem_output_weights,
computer.controller.W, computer.controller.b
])
computer.save(session1, ckpts_dir, 'test-restore')
graph2 = tf.Graph()
with graph2.as_default():
with tf.Session(graph=graph2) as session2:
computer = DNC(DummyController,
10,
20,
10,
10,
64,
2,
batch_size=2)
session2.run(tf.initialize_all_variables())
computer.restore(session2, ckpts_dir, 'test-restore')
restored_weights = session2.run([
computer.controller.nn_output_weights,
computer.controller.interface_weights,
computer.controller.mem_output_weights,
computer.controller.W, computer.controller.b
])
self.assertTrue(
np.product([
np.array_equal(restored_weights[i], saved_weights[i])
for i in range(5)
]))
def test_save(self):
graph = tf.Graph()
with graph.as_default():
with tf.compat.v1.Session(graph=graph) as session:
computer = DNC(DummyController,
10,
20,
10,
10,
64,
2,
batch_size=2)
session.run(tf.compat.v1.global_variables_initializer())
current_dir = os.path.dirname(__file__)
ckpts_dir = os.path.join(current_dir, 'checkpoints')
computer.save(session, ckpts_dir, 'test-save')
self.assertTrue(True)
last_100_losses.append(loss_value)
# summerizer.add_summary(summary, i)
if summerize:
llprint("\n\tAvg. Cross-Entropy: %.7f\n" % (np.mean(last_100_losses)))
end_time_100 = time.time()
elapsed_time = (end_time_100 - start_time_100) / 60
avg_counter += 1
avg_100_time += (1. / avg_counter) * (elapsed_time - avg_100_time)
estimated_time = (avg_100_time * ((end - i) / 100.)) / 60.
print("\tAvg. 100 iterations time: %.2f minutes" % (avg_100_time))
print("\tApprox. time to completion: %.2f hours" % (estimated_time))
start_time_100 = time.time()
last_100_losses = []
if take_checkpoint:
llprint("\nSaving Checkpoint ... "),
ncomputer.save(session, ckpts_dir, 'step-%d' % (i))
llprint("Done!\n")
except KeyboardInterrupt:
llprint("\nSaving Checkpoint ... "),
ncomputer.save(session, ckpts_dir, 'step-%d' % (i))
llprint("Done!\n")
sys.exit(0)
summerize_op if summerize else no_summerize
],
feed_dict={
ncomputer.input_data: np.array(input_data),
ncomputer.target_output: np.array(target_output),
ncomputer.sequence_length:
np.array(input_data).shape[1]
})
last_100_losses.append(loss_value)
if summary:
summerizer.add_summary(summary, i)
for key in mem_views_values:
mem_views_values[key] = mem_views_values[key].tolist()
if summerize:
llprint("\n\tAvg. Logistic Loss: %.4f\n" %
(np.mean(last_100_losses)))
last_100_losses = []
if mem_summarize:
with open(mem_logs_dir + "/" + str(i) + ".json", "w") as f:
json.dump(mem_views_values,
f,
ensure_ascii=False,
separators=(',', ':'))
if take_checkpoint:
llprint("\nSaving Checkpoint ... "),
ncomputer.save(session, ckpts_dir, 'task1-3-step-%d' % (i))
llprint("Done!\n")
],
feed_dict={
ncomputer.input_data: np.array(input_data),
ncomputer.target_output: np.array(target_output),
ncomputer.sequence_length:
np.array(input_data).shape[1],
ncomputer.input_mode: np.array(input_modes)
})
# for kkk in range(len(mem_views_values["inputs"][0])):
# print(kkk, mem_views_values["inputs"][0][kkk])
# print(o[0][kkk])
last_100_losses.append(loss_value)
if summary:
summerizer.add_summary(summary, i)
for key in mem_views_values:
mem_views_values[key] = mem_views_values[key].tolist()
if summerize:
llprint("\n\tAvg. Logistic Loss: %.4f\n" %
(np.mean(last_100_losses)))
last_100_losses = []
# if mem_summarize:
# with open(mem_logs_dir+"/graph2-data2-step-"+str(i)+".json", "w") as f:
# json.dump(mem_views_values, f, ensure_ascii=False, separators=(',', ':'))
if take_checkpoint:
llprint("\nSaving Checkpoint ... "),
ncomputer.save(session, ckpts_dir,
'graph2-data2-3-step-%d' % (i))
llprint("Done!\n")
def main():
"""
Train the DNC to take answer questions from the DREAM dataset.
:return: None.
"""
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
dirname = os.path.dirname(__file__)
ckpts_dir = os.path.join(dirname, 'checkpoints/')
data_dir = os.path.join(dirname, 'data', 'encoded')
tb_logs_dir = os.path.join(dirname, 'logs')
llprint("Loading Data ... ")
lexicon_dict = load(os.path.join(data_dir, 'lexicon-dict.pkl'))
data_files = os.listdir(os.path.join(data_dir, 'train'))
llprint("Done!\n")
batch_size = 1
input_size = output_size = len(lexicon_dict)
sequence_max_length = 100
word_space_size = len(lexicon_dict)
words_count = 256
word_size = 64
read_heads = 4
learning_rate = 1e-4
momentum = 0.9
from_checkpoint = None
iterations = 100000
start_step = 0
options, _ = getopt.getopt(sys.argv[1:], '',
['checkpoint=', 'iterations=', 'start='])
for opt in options:
if opt[0] == '--checkpoint':
from_checkpoint = opt[1]
print("Checkpoint found")
elif opt[0] == '--iterations':
iterations = int(opt[1])
elif opt[0] == '--start':
start_step = int(opt[1])
graph = tf.Graph()
with graph.as_default():
with tf.compat.v1.Session(graph=graph) as session:
llprint("Building Computational Graph ... ")
optimizer = tf.compat.v1.train.RMSPropOptimizer(learning_rate,
momentum=momentum)
summarizer = tf.compat.v1.summary.FileWriter(
tb_logs_dir, session.graph)
ncomputer = DNC(RecurrentController, input_size, output_size,
sequence_max_length, words_count, word_size,
read_heads, batch_size)
output, _ = ncomputer.get_outputs()
loss_weights = tf.compat.v1.placeholder(tf.float32,
[batch_size, None, 1])
loss = tf.reduce_mean(
loss_weights * tf.nn.softmax_cross_entropy_with_logits(
logits=output, labels=ncomputer.target_output))
summaries = []
gradients = optimizer.compute_gradients(loss)
for i, (grad, var) in enumerate(gradients):
if grad is not None:
gradients[i] = (tf.clip_by_value(grad, -10, 10), var)
for (grad, var) in gradients:
if grad is not None:
summaries.append(
tf.compat.v1.summary.histogram(var.name + '/grad',
grad))
apply_gradients = optimizer.apply_gradients(gradients)
summaries.append(tf.compat.v1.summary.scalar("Loss", loss))
summarize_op = tf.compat.v1.summary.merge(summaries)
no_summarize = tf.no_op()
llprint("Done!\n")
llprint("Initializing Variables ... ")
session.run(tf.compat.v1.global_variables_initializer())
llprint("Done!\n")
if from_checkpoint is not None:
llprint("Restoring Checkpoint %s ... " % from_checkpoint)
ncomputer.restore(session, ckpts_dir, from_checkpoint)
llprint("Done!\n")
elif os.path.exists(ckpts_dir):
checkpoints = os.listdir(ckpts_dir)
if len(checkpoints) != 0 and any("step-" in s
for s in checkpoints):
checkpoint_numbers = [
int(checkpoint[checkpoint.find("-") + 1:])
for checkpoint in checkpoints
if checkpoint[checkpoint.find("-") + 1:].isnumeric()
]
checkpoint_numbers.sort()
ncomputer.restore(session, ckpts_dir,
f"step-{checkpoint_numbers[-1]}")
start = checkpoint_numbers[-1]
end = 100000
last_100_losses = []
if not 'start' in locals():
start = 0
end = 100000
if from_checkpoint is not None:
start = int(from_checkpoint[from_checkpoint.find("-") + 1:])
start_time_100 = time.time()
end_time_100 = None
avg_100_time = 0.
avg_counter = 0
for i in range(start, end + 1):
try:
llprint("\rIteration %d/%d" % (i, end))
sample = np.random.choice(data_files, 1)
with open(os.path.join(data_dir, 'train', sample[0])) as f:
sample = json.load(f)
input_data, target_output, seq_len, weights = prepare_sample(
sample, lexicon_dict['='], word_space_size,
lexicon_dict)
summarize = (i % 100 == 0)
take_checkpoint = (i != 0) and (i % 200 == 0)
#For debugging
outputs, _ = ncomputer.get_outputs()
softmaxed = tf.nn.softmax(outputs)
loss_value, _, summary, softmax_output = session.run(
[
loss, apply_gradients,
summarize_op if summarize else no_summarize,
softmaxed
],
feed_dict={
ncomputer.input_data: input_data,
ncomputer.target_output: target_output,
ncomputer.sequence_length: seq_len,
loss_weights: weights
})
softmax_output = np.squeeze(softmax_output, axis=0)
given_answers = np.argmax(softmax_output, axis=1)
words = []
for an_array in target_output[0]:
for word in np.where(an_array == 1):
words.extend([
list(lexicon_dict.keys())[np.where(
an_array == 1)[0][0]]
])
last_100_losses.append(loss_value)
if summarize:
print("\n\tLoss value: ", loss_value)
print("\tTarget output: ", words)
print("\tOutput: ", [
list(lexicon_dict.keys())[num]
for num in given_answers
])
summarizer.add_summary(summary, i)
llprint("\tAvg. Cross-Entropy: %.7f\n" %
(np.mean(last_100_losses)))
end_time_100 = time.time()
elapsed_time = (end_time_100 - start_time_100) / 60
avg_counter += 1
avg_100_time += (1. / avg_counter) * (elapsed_time -
avg_100_time)
estimated_time = (avg_100_time *
((end - i) / 100.)) / 60.
print("\tAvg. 100 iterations time: %.2f minutes" %
avg_100_time)
print("\tApprox. time to completion: %.2f hours\n" %
estimated_time)
start_time_100 = time.time()
last_100_losses = []
if take_checkpoint:
llprint("\nSaving Checkpoint ... line 237 "),
ncomputer.save(session, ckpts_dir, 'step-%d' % i)
llprint("Done!\n")
except KeyboardInterrupt:
llprint("\nSaving Checkpoint ... "),
ncomputer.save(session, ckpts_dir, 'step-%d' % i)
llprint("Done!\n")
sys.exit(0)
def main():
"""
Train the DNC to take a word and list its instances of vowels in order of occurrence.
:return: None.
"""
dirname = os.path.dirname(__file__)
ckpts_dir = os.path.join(dirname, 'checkpoints')
data_dir = os.path.join(dirname, 'data', 'encoded')
tb_logs_dir = os.path.join(dirname, 'logs')
llprint("Loading Data ... ")
lexicon_dict = load(os.path.join(data_dir, 'lexicon-dict.pkl'))
data = load(os.path.join(data_dir, 'train', 'train.pkl'))
llprint("Done!\n")
batch_size = 1
input_size = output_size = len(lexicon_dict)
sequence_max_length = 100
dict_size = len(lexicon_dict)
words_count = 256
word_size = 64
read_heads = 4
learning_rate = 1e-4
momentum = 0.9
from_checkpoint = None
iterations = 100000
start_step = 0
options, _ = getopt.getopt(sys.argv[1:], '',
['checkpoint=', 'iterations=', 'start='])
for opt in options:
if opt[0] == '--checkpoint':
from_checkpoint = opt[1]
elif opt[0] == '--iterations':
iterations = int(opt[1])
elif opt[0] == '--start':
start_step = int(opt[1])
graph = tf.Graph()
with graph.as_default():
with tf.compat.v1.Session(graph=graph) as session:
llprint("Building Computational Graph ... ")
optimizer = tf.compat.v1.train.RMSPropOptimizer(learning_rate,
momentum=momentum)
summarizer = tf.compat.v1.summary.FileWriter(
tb_logs_dir, session.graph)
ncomputer = DNC(RecurrentController, input_size, output_size,
sequence_max_length, words_count, word_size,
read_heads, batch_size)
output, _ = ncomputer.get_outputs()
loss_weights = tf.compat.v1.placeholder(tf.float32,
[batch_size, None, 1])
loss = tf.reduce_mean(
loss_weights * tf.nn.softmax_cross_entropy_with_logits(
logits=output, labels=ncomputer.target_output))
summaries = []
gradients = optimizer.compute_gradients(loss)
for i, (grad, var) in enumerate(gradients):
if grad is not None:
gradients[i] = (tf.clip_by_value(grad, -10, 10), var)
for (grad, var) in gradients:
if grad is not None:
summaries.append(
tf.compat.v1.summary.histogram(var.name + '/grad',
grad))
apply_gradients = optimizer.apply_gradients(gradients)
summaries.append(tf.compat.v1.summary.scalar("Loss", loss))
summarize_op = tf.compat.v1.summary.merge(summaries)
no_summarize = tf.no_op()
llprint("Done!\n")
llprint("Initializing Variables ... ")
session.run(tf.compat.v1.global_variables_initializer())
llprint("Done!\n")
if from_checkpoint is not None:
llprint("Restoring Checkpoint %s ... " % from_checkpoint)
ncomputer.restore(session, ckpts_dir, from_checkpoint)
llprint("Done!\n")
last_100_losses = []
start = 0 if start_step == 0 else start_step + 1
end = start_step + iterations + 1
start_time_100 = time.time()
avg_100_time = 0.
avg_counter = 0
for i in range(start, end + 1):
try:
llprint("\rIteration %d/%d" % (i, end))
sample = np.random.choice(data, 1)
input_data, target_output, seq_len, weights = prepare_sample(
sample, lexicon_dict['#'], dict_size)
summarize = (i % 100 == 0)
take_checkpoint = (i != 0) and (i % end == 0)
loss_value, _, summary = session.run(
[
loss, apply_gradients,
summarize_op if summarize else no_summarize
],
feed_dict={
ncomputer.input_data: input_data,
ncomputer.target_output: target_output,
ncomputer.sequence_length: seq_len,
loss_weights: weights
})
last_100_losses.append(loss_value)
if summarize:
summarizer.add_summary(summary, i)
llprint("\n\tAvg. Cross-Entropy: %.7f\n" %
(np.mean(last_100_losses)))
end_time_100 = time.time()
elapsed_time = (end_time_100 - start_time_100) / 60
avg_counter += 1
avg_100_time += (1. / avg_counter) * (elapsed_time -
avg_100_time)
estimated_time = (avg_100_time *
((end - i) / 100.)) / 60.
print("\tAvg. 100 iterations time: %.2f minutes" %
avg_100_time)
print("\tApprox. time to completion: %.2f hours" %
estimated_time)
start_time_100 = time.time()
last_100_losses = []
if take_checkpoint:
llprint("\nSaving Checkpoint ... "),
ncomputer.save(session, ckpts_dir, 'step-%d' % i)
llprint("Done!\n")
except KeyboardInterrupt:
llprint("\nSaving Checkpoint ... "),
ncomputer.save(session, ckpts_dir, 'step-%d' % i)
llprint("Done!\n")
sys.exit(0)
loss_value, _, summary, mem_view, out = session.run(
[
loss, apply_gradients,
summerize_op if summerize else no_summerize,
packed_memory_view, squashed_output
],
feed_dict={
ncomputer.input_data: one_big_input,
ncomputer.target_output: one_big_output,
ncomputer.sequence_length: one_big_input.shape[1]
})
last_100_losses.append(loss_value)
if summary:
summerizer.add_summary(summary, i)
if i % visualize_step == 0:
visualize_op(one_big_input, out, mem_view, images_dir, i)
if summerize:
llprint("\n\tAvg. Logistic Loss: %.4f\n" %
(np.mean(last_100_losses)))
last_100_losses = []
if take_checkpoint:
llprint("\nSaving Checkpoint ... "),
ncomputer.save(
session, ckpts_dir, 'model-%dx%d-r%d-s%d' %
(words_count, word_size, read_heads, i))
llprint("Done!\n")
