def test_restore(self): current_dir = os.path.dirname(__file__) ckpts_dir = os.path.join(current_dir, 'checkpoints') model1_output, model1_memview = None, None sample_input = np.random.uniform(0, 1, (2, 5, 10)).astype(np.float32) sample_seq_len = 5 graph1 = tf.Graph() with graph1.as_default(): with tf.Session(graph=graph1) as session1: computer = DNC(DummyController, 10, 20, 10, 10, 64, 2, batch_size=2) session1.run(tf.initialize_all_variables()) saved_weights = session1.run([ computer.controller.nn_output_weights, computer.controller.interface_weights, computer.controller.mem_output_weights, computer.controller.W, computer.controller.b ]) computer.save(session1, ckpts_dir, 'test-restore') graph2 = tf.Graph() with graph2.as_default(): with tf.Session(graph=graph2) as session2: computer = DNC(DummyController, 10, 20, 10, 10, 64, 2, batch_size=2) session2.run(tf.initialize_all_variables()) computer.restore(session2, ckpts_dir, 'test-restore') restored_weights = session2.run([ computer.controller.nn_output_weights, computer.controller.interface_weights, computer.controller.mem_output_weights, computer.controller.W, computer.controller.b ]) self.assertTrue( np.product([ np.array_equal(restored_weights[i], saved_weights[i]) for i in range(5) ]))
def test_save(self): graph = tf.Graph() with graph.as_default(): with tf.compat.v1.Session(graph=graph) as session: computer = DNC(DummyController, 10, 20, 10, 10, 64, 2, batch_size=2) session.run(tf.compat.v1.global_variables_initializer()) current_dir = os.path.dirname(__file__) ckpts_dir = os.path.join(current_dir, 'checkpoints') computer.save(session, ckpts_dir, 'test-save') self.assertTrue(True)
last_100_losses.append(loss_value) # summerizer.add_summary(summary, i) if summerize: llprint("\n\tAvg. Cross-Entropy: %.7f\n" % (np.mean(last_100_losses))) end_time_100 = time.time() elapsed_time = (end_time_100 - start_time_100) / 60 avg_counter += 1 avg_100_time += (1. / avg_counter) * (elapsed_time - avg_100_time) estimated_time = (avg_100_time * ((end - i) / 100.)) / 60. print("\tAvg. 100 iterations time: %.2f minutes" % (avg_100_time)) print("\tApprox. time to completion: %.2f hours" % (estimated_time)) start_time_100 = time.time() last_100_losses = [] if take_checkpoint: llprint("\nSaving Checkpoint ... "), ncomputer.save(session, ckpts_dir, 'step-%d' % (i)) llprint("Done!\n") except KeyboardInterrupt: llprint("\nSaving Checkpoint ... "), ncomputer.save(session, ckpts_dir, 'step-%d' % (i)) llprint("Done!\n") sys.exit(0)
summerize_op if summerize else no_summerize ], feed_dict={ ncomputer.input_data: np.array(input_data), ncomputer.target_output: np.array(target_output), ncomputer.sequence_length: np.array(input_data).shape[1] }) last_100_losses.append(loss_value) if summary: summerizer.add_summary(summary, i) for key in mem_views_values: mem_views_values[key] = mem_views_values[key].tolist() if summerize: llprint("\n\tAvg. Logistic Loss: %.4f\n" % (np.mean(last_100_losses))) last_100_losses = [] if mem_summarize: with open(mem_logs_dir + "/" + str(i) + ".json", "w") as f: json.dump(mem_views_values, f, ensure_ascii=False, separators=(',', ':')) if take_checkpoint: llprint("\nSaving Checkpoint ... "), ncomputer.save(session, ckpts_dir, 'task1-3-step-%d' % (i)) llprint("Done!\n")
], feed_dict={ ncomputer.input_data: np.array(input_data), ncomputer.target_output: np.array(target_output), ncomputer.sequence_length: np.array(input_data).shape[1], ncomputer.input_mode: np.array(input_modes) }) # for kkk in range(len(mem_views_values["inputs"][0])): # print(kkk, mem_views_values["inputs"][0][kkk]) # print(o[0][kkk]) last_100_losses.append(loss_value) if summary: summerizer.add_summary(summary, i) for key in mem_views_values: mem_views_values[key] = mem_views_values[key].tolist() if summerize: llprint("\n\tAvg. Logistic Loss: %.4f\n" % (np.mean(last_100_losses))) last_100_losses = [] # if mem_summarize: # with open(mem_logs_dir+"/graph2-data2-step-"+str(i)+".json", "w") as f: # json.dump(mem_views_values, f, ensure_ascii=False, separators=(',', ':')) if take_checkpoint: llprint("\nSaving Checkpoint ... "), ncomputer.save(session, ckpts_dir, 'graph2-data2-3-step-%d' % (i)) llprint("Done!\n")
def main(): """ Train the DNC to take answer questions from the DREAM dataset. :return: None. """ os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) dirname = os.path.dirname(__file__) ckpts_dir = os.path.join(dirname, 'checkpoints/') data_dir = os.path.join(dirname, 'data', 'encoded') tb_logs_dir = os.path.join(dirname, 'logs') llprint("Loading Data ... ") lexicon_dict = load(os.path.join(data_dir, 'lexicon-dict.pkl')) data_files = os.listdir(os.path.join(data_dir, 'train')) llprint("Done!\n") batch_size = 1 input_size = output_size = len(lexicon_dict) sequence_max_length = 100 word_space_size = len(lexicon_dict) words_count = 256 word_size = 64 read_heads = 4 learning_rate = 1e-4 momentum = 0.9 from_checkpoint = None iterations = 100000 start_step = 0 options, _ = getopt.getopt(sys.argv[1:], '', ['checkpoint=', 'iterations=', 'start=']) for opt in options: if opt[0] == '--checkpoint': from_checkpoint = opt[1] print("Checkpoint found") elif opt[0] == '--iterations': iterations = int(opt[1]) elif opt[0] == '--start': start_step = int(opt[1]) graph = tf.Graph() with graph.as_default(): with tf.compat.v1.Session(graph=graph) as session: llprint("Building Computational Graph ... ") optimizer = tf.compat.v1.train.RMSPropOptimizer(learning_rate, momentum=momentum) summarizer = tf.compat.v1.summary.FileWriter( tb_logs_dir, session.graph) ncomputer = DNC(RecurrentController, input_size, output_size, sequence_max_length, words_count, word_size, read_heads, batch_size) output, _ = ncomputer.get_outputs() loss_weights = tf.compat.v1.placeholder(tf.float32, [batch_size, None, 1]) loss = tf.reduce_mean( loss_weights * tf.nn.softmax_cross_entropy_with_logits( logits=output, labels=ncomputer.target_output)) summaries = [] gradients = optimizer.compute_gradients(loss) for i, (grad, var) in enumerate(gradients): if grad is not None: gradients[i] = (tf.clip_by_value(grad, -10, 10), var) for (grad, var) in gradients: if grad is not None: summaries.append( tf.compat.v1.summary.histogram(var.name + '/grad', grad)) apply_gradients = optimizer.apply_gradients(gradients) summaries.append(tf.compat.v1.summary.scalar("Loss", loss)) summarize_op = tf.compat.v1.summary.merge(summaries) no_summarize = tf.no_op() llprint("Done!\n") llprint("Initializing Variables ... ") session.run(tf.compat.v1.global_variables_initializer()) llprint("Done!\n") if from_checkpoint is not None: llprint("Restoring Checkpoint %s ... " % from_checkpoint) ncomputer.restore(session, ckpts_dir, from_checkpoint) llprint("Done!\n") elif os.path.exists(ckpts_dir): checkpoints = os.listdir(ckpts_dir) if len(checkpoints) != 0 and any("step-" in s for s in checkpoints): checkpoint_numbers = [ int(checkpoint[checkpoint.find("-") + 1:]) for checkpoint in checkpoints if checkpoint[checkpoint.find("-") + 1:].isnumeric() ] checkpoint_numbers.sort() ncomputer.restore(session, ckpts_dir, f"step-{checkpoint_numbers[-1]}") start = checkpoint_numbers[-1] end = 100000 last_100_losses = [] if not 'start' in locals(): start = 0 end = 100000 if from_checkpoint is not None: start = int(from_checkpoint[from_checkpoint.find("-") + 1:]) start_time_100 = time.time() end_time_100 = None avg_100_time = 0. avg_counter = 0 for i in range(start, end + 1): try: llprint("\rIteration %d/%d" % (i, end)) sample = np.random.choice(data_files, 1) with open(os.path.join(data_dir, 'train', sample[0])) as f: sample = json.load(f) input_data, target_output, seq_len, weights = prepare_sample( sample, lexicon_dict['='], word_space_size, lexicon_dict) summarize = (i % 100 == 0) take_checkpoint = (i != 0) and (i % 200 == 0) #For debugging outputs, _ = ncomputer.get_outputs() softmaxed = tf.nn.softmax(outputs) loss_value, _, summary, softmax_output = session.run( [ loss, apply_gradients, summarize_op if summarize else no_summarize, softmaxed ], feed_dict={ ncomputer.input_data: input_data, ncomputer.target_output: target_output, ncomputer.sequence_length: seq_len, loss_weights: weights }) softmax_output = np.squeeze(softmax_output, axis=0) given_answers = np.argmax(softmax_output, axis=1) words = [] for an_array in target_output[0]: for word in np.where(an_array == 1): words.extend([ list(lexicon_dict.keys())[np.where( an_array == 1)[0][0]] ]) last_100_losses.append(loss_value) if summarize: print("\n\tLoss value: ", loss_value) print("\tTarget output: ", words) print("\tOutput: ", [ list(lexicon_dict.keys())[num] for num in given_answers ]) summarizer.add_summary(summary, i) llprint("\tAvg. Cross-Entropy: %.7f\n" % (np.mean(last_100_losses))) end_time_100 = time.time() elapsed_time = (end_time_100 - start_time_100) / 60 avg_counter += 1 avg_100_time += (1. / avg_counter) * (elapsed_time - avg_100_time) estimated_time = (avg_100_time * ((end - i) / 100.)) / 60. print("\tAvg. 100 iterations time: %.2f minutes" % avg_100_time) print("\tApprox. time to completion: %.2f hours\n" % estimated_time) start_time_100 = time.time() last_100_losses = [] if take_checkpoint: llprint("\nSaving Checkpoint ... line 237 "), ncomputer.save(session, ckpts_dir, 'step-%d' % i) llprint("Done!\n") except KeyboardInterrupt: llprint("\nSaving Checkpoint ... "), ncomputer.save(session, ckpts_dir, 'step-%d' % i) llprint("Done!\n") sys.exit(0)
def main(): """ Train the DNC to take a word and list its instances of vowels in order of occurrence. :return: None. """ dirname = os.path.dirname(__file__) ckpts_dir = os.path.join(dirname, 'checkpoints') data_dir = os.path.join(dirname, 'data', 'encoded') tb_logs_dir = os.path.join(dirname, 'logs') llprint("Loading Data ... ") lexicon_dict = load(os.path.join(data_dir, 'lexicon-dict.pkl')) data = load(os.path.join(data_dir, 'train', 'train.pkl')) llprint("Done!\n") batch_size = 1 input_size = output_size = len(lexicon_dict) sequence_max_length = 100 dict_size = len(lexicon_dict) words_count = 256 word_size = 64 read_heads = 4 learning_rate = 1e-4 momentum = 0.9 from_checkpoint = None iterations = 100000 start_step = 0 options, _ = getopt.getopt(sys.argv[1:], '', ['checkpoint=', 'iterations=', 'start=']) for opt in options: if opt[0] == '--checkpoint': from_checkpoint = opt[1] elif opt[0] == '--iterations': iterations = int(opt[1]) elif opt[0] == '--start': start_step = int(opt[1]) graph = tf.Graph() with graph.as_default(): with tf.compat.v1.Session(graph=graph) as session: llprint("Building Computational Graph ... ") optimizer = tf.compat.v1.train.RMSPropOptimizer(learning_rate, momentum=momentum) summarizer = tf.compat.v1.summary.FileWriter( tb_logs_dir, session.graph) ncomputer = DNC(RecurrentController, input_size, output_size, sequence_max_length, words_count, word_size, read_heads, batch_size) output, _ = ncomputer.get_outputs() loss_weights = tf.compat.v1.placeholder(tf.float32, [batch_size, None, 1]) loss = tf.reduce_mean( loss_weights * tf.nn.softmax_cross_entropy_with_logits( logits=output, labels=ncomputer.target_output)) summaries = [] gradients = optimizer.compute_gradients(loss) for i, (grad, var) in enumerate(gradients): if grad is not None: gradients[i] = (tf.clip_by_value(grad, -10, 10), var) for (grad, var) in gradients: if grad is not None: summaries.append( tf.compat.v1.summary.histogram(var.name + '/grad', grad)) apply_gradients = optimizer.apply_gradients(gradients) summaries.append(tf.compat.v1.summary.scalar("Loss", loss)) summarize_op = tf.compat.v1.summary.merge(summaries) no_summarize = tf.no_op() llprint("Done!\n") llprint("Initializing Variables ... ") session.run(tf.compat.v1.global_variables_initializer()) llprint("Done!\n") if from_checkpoint is not None: llprint("Restoring Checkpoint %s ... " % from_checkpoint) ncomputer.restore(session, ckpts_dir, from_checkpoint) llprint("Done!\n") last_100_losses = [] start = 0 if start_step == 0 else start_step + 1 end = start_step + iterations + 1 start_time_100 = time.time() avg_100_time = 0. avg_counter = 0 for i in range(start, end + 1): try: llprint("\rIteration %d/%d" % (i, end)) sample = np.random.choice(data, 1) input_data, target_output, seq_len, weights = prepare_sample( sample, lexicon_dict['#'], dict_size) summarize = (i % 100 == 0) take_checkpoint = (i != 0) and (i % end == 0) loss_value, _, summary = session.run( [ loss, apply_gradients, summarize_op if summarize else no_summarize ], feed_dict={ ncomputer.input_data: input_data, ncomputer.target_output: target_output, ncomputer.sequence_length: seq_len, loss_weights: weights }) last_100_losses.append(loss_value) if summarize: summarizer.add_summary(summary, i) llprint("\n\tAvg. Cross-Entropy: %.7f\n" % (np.mean(last_100_losses))) end_time_100 = time.time() elapsed_time = (end_time_100 - start_time_100) / 60 avg_counter += 1 avg_100_time += (1. / avg_counter) * (elapsed_time - avg_100_time) estimated_time = (avg_100_time * ((end - i) / 100.)) / 60. print("\tAvg. 100 iterations time: %.2f minutes" % avg_100_time) print("\tApprox. time to completion: %.2f hours" % estimated_time) start_time_100 = time.time() last_100_losses = [] if take_checkpoint: llprint("\nSaving Checkpoint ... "), ncomputer.save(session, ckpts_dir, 'step-%d' % i) llprint("Done!\n") except KeyboardInterrupt: llprint("\nSaving Checkpoint ... "), ncomputer.save(session, ckpts_dir, 'step-%d' % i) llprint("Done!\n") sys.exit(0)
loss_value, _, summary, mem_view, out = session.run( [ loss, apply_gradients, summerize_op if summerize else no_summerize, packed_memory_view, squashed_output ], feed_dict={ ncomputer.input_data: one_big_input, ncomputer.target_output: one_big_output, ncomputer.sequence_length: one_big_input.shape[1] }) last_100_losses.append(loss_value) if summary: summerizer.add_summary(summary, i) if i % visualize_step == 0: visualize_op(one_big_input, out, mem_view, images_dir, i) if summerize: llprint("\n\tAvg. Logistic Loss: %.4f\n" % (np.mean(last_100_losses))) last_100_losses = [] if take_checkpoint: llprint("\nSaving Checkpoint ... "), ncomputer.save( session, ckpts_dir, 'model-%dx%d-r%d-s%d' % (words_count, word_size, read_heads, i)) llprint("Done!\n")