def plot(seq_length): i,o = tasks.copy(8,seq_length) weights,outputs = do_task(i) plt.figure(1,figsize=(20,7)) plt.subplot(311) plt.imshow(i.T,interpolation='nearest') plt.subplot(312) plt.imshow(o.T,interpolation='nearest') plt.subplot(313) plt.imshow(outputs.T,interpolation='nearest') plt.show()
def plot(seq_length): i, o = tasks.copy(8, seq_length) weights, outputs = do_task(i) plt.figure(1, figsize=(20, 7)) plt.subplot(311) plt.imshow(i.T, interpolation='nearest') plt.subplot(312) plt.imshow(o.T, interpolation='nearest') plt.subplot(313) plt.imshow(outputs.T, interpolation='nearest') plt.show()
def plot_weights(seq_length): i, o = tasks.copy(8, seq_length) weights, outputs = do_task(i) plt.figure(1, figsize=(20, 20)) plt.imshow(weights.T[100:123], interpolation='nearest', cmap=cm.gray) plt.show()
from tasks import copy # any task could be analyzed sys.path.append('Utils') from Utils import Neural_GPU epochs = 5 iterations = 100 model = Neural_GPU.Neural_GPU() sess = tf.Session() sess.run(tf.global_variables_initializer()) for epoch in range(1, epochs + 1): train_cost = [] for iteration in range(iterations): trainX, trainY = copy(model.batch_size, model.sequence_size, model.vocab_size) _, cost = sess.run( [model.train_op, model.loss], feed_dict={ model.inputs: trainX, model.targets: trainY, model.training: True, model.learning_rate: 0.01, model.keep_prob: 0.94 }) assert not np.isnan(cost), 'Epoch {}, Iteration {}'.format( epoch, iteration + 1) train_cost.append(cost) print( 'Epoch {}, Average Training Loss {:.4f}, Final Training Loss {:.4f}' .format(epoch, np.mean(train_cost), cost))
mem_size = 128, mem_width = 20, output_size = 8 ) max_sequences = 100000 patience = 20000 patience_increase = 3 improvement_threshold = 0.995 best_score = np.inf test_score = 0. score = None alpha = 0.95 for counter in xrange(max_sequences): # length = np.random.randint(int(20 * (min(counter,50000)/float(50000))**2) +1) + 1 length = np.random.randint(1,21) i,o = tasks.copy(8,length) if score == None: score = train(i,o) else: score = alpha * score + (1 - alpha) * train(i,o) print "round:", counter, "score:", score if score < best_score: # improve patience if loss improvement is good enough if score < best_score * improvement_threshold: patience = max(patience, counter * patience_increase) P.save(model_out) best_score = score if patience <= counter: break
model_out = sys.argv[1] P, train = make_train(input_size=8, mem_size=128, mem_width=20, output_size=8) max_sequences = 100000 patience = 20000 patience_increase = 3 improvement_threshold = 0.995 best_score = np.inf test_score = 0. score = None alpha = 0.95 for counter in xrange(max_sequences): length = np.random.randint( int(20 * (min(counter, 50000) / float(50000))**2) + 1) + 1 i, o = tasks.copy(8, length) if score == None: score = train(i, o) else: score = alpha * score + (1 - alpha) * train(i, o) print score if score < best_score: # improve patience if loss improvement is good enough if score < best_score * improvement_threshold: patience = max(patience, counter * patience_increase) P.save(model_out) best_score = score if patience <= counter: break
) print "Done. (%0.3f s)"%(time.time() - start_time) print P.parameter_count() return P, P_learn, train, test if __name__ == "__main__": model_out = sys.argv[1] P, P_learn, train, test = make_functions( input_size=width + 2, mem_size=128, mem_width=20, output_size=width + 2 ) test_set = [ tasks.copy(10, l+1, width) for l in xrange(max_sequence_length) ] def validation_test(): return sum(test(i,o) for i,o in test_set) / (10 * len(test_set)) best_score = validation_test() for counter in xrange(max_sequences): length = np.random.randint(max_sequence_length) + 1 i, o = tasks.copy(batch_size, length, width) score = train(i, o) if (counter + 1) % validation_frequency == 0: validation_score = validation_test() print validation_score, if validation_score < best_score: best_score = validation_score P.save('model.pkl')
plt.show() def error_rate(expected_output, actual_output): """ Given the expected output data and the actual output data by the NTM, compute the average error rate between the two. """ abs_diff = abs(expected_output - actual_output) num_elements = expected_output.shape[0] * expected_output.shape[1] return sum(sum(abs_diff)) / num_elements if __name__ == "__main__": if len(sys.argv) != 3: print "Usage: python test.py <model_filename> <sequence_length>" sys.exit() # Make the NTM model and load the parameters P, do_task = run_model.make_model() P.load(sys.argv[1]) # Randomly generate a copy task and perform the task input_data, expected_output = tasks.copy(8, int(sys.argv[2])) weights, actual_output = do_task(input_data) # Plot the outputs and compute the error_rate plot(input_data, expected_output, actual_output) print "The average error rate was " + str(error_rate(expected_output, actual_output))
def plot_weights(seq_length): i,o = tasks.copy(8,seq_length) weights,outputs = do_task(i) plt.figure(1,figsize=(20,20)) plt.imshow(weights.T[100:123],interpolation='nearest',cmap=cm.gray) plt.show()
test = theano.function(inputs=[input_seqs, output_seqs], outputs=bits_loss) print "Done. (%0.3f s)" % (time.time() - start_time) print P.parameter_count() return P, P_learn, train, test if __name__ == "__main__": model_out = sys.argv[1] P, P_learn, train, test = make_functions(input_size=width + 2, mem_size=128, mem_width=20, output_size=width + 2) test_set = [ tasks.copy(10, l + 1, width) for l in xrange(max_sequence_length) ] def validation_test(): return sum(test(i, o) for i, o in test_set) / (10 * len(test_set)) best_score = validation_test() for counter in xrange(max_sequences): length = np.random.randint(max_sequence_length) + 1 i, o = tasks.copy(batch_size, length, width) score = train(i, o) if (counter + 1) % validation_frequency == 0: validation_score = validation_test() print validation_score, if validation_score < best_score:
plt.show() def error_rate(expected_output, actual_output): """ Given the expected output data and the actual output data by the NTM, compute the average error rate between the two. """ abs_diff = abs(expected_output - actual_output) num_elements = expected_output.shape[0] * expected_output.shape[1] return sum(sum(abs_diff)) / num_elements if __name__ == "__main__": if len(sys.argv) != 3: print "Usage: python test.py <model_filename> <sequence_length>" sys.exit() # Make the NTM model and load the parameters P, do_task = run_model.make_model() P.load(sys.argv[1]) # Randomly generate a copy task and perform the task input_data, expected_output = tasks.copy(8, int(sys.argv[2])) weights, actual_output = do_task(input_data) # Plot the outputs and compute the error_rate plot(input_data, expected_output, actual_output) print "The average error rate was " + str( error_rate(expected_output, actual_output))