def plot(seq_length):
i,o = tasks.copy(8,seq_length)
weights,outputs = do_task(i)
plt.figure(1,figsize=(20,7))
plt.subplot(311)
plt.imshow(i.T,interpolation='nearest')
plt.subplot(312)
plt.imshow(o.T,interpolation='nearest')
plt.subplot(313)
plt.imshow(outputs.T,interpolation='nearest')
plt.show()
def plot(seq_length):
i, o = tasks.copy(8, seq_length)
weights, outputs = do_task(i)
plt.figure(1, figsize=(20, 7))
plt.subplot(311)
plt.imshow(i.T, interpolation='nearest')
plt.subplot(312)
plt.imshow(o.T, interpolation='nearest')
plt.subplot(313)
plt.imshow(outputs.T, interpolation='nearest')
plt.show()
def plot_weights(seq_length):
i, o = tasks.copy(8, seq_length)
weights, outputs = do_task(i)
plt.figure(1, figsize=(20, 20))
plt.imshow(weights.T[100:123], interpolation='nearest', cmap=cm.gray)
plt.show()
from tasks import copy # any task could be analyzed
sys.path.append('Utils')
from Utils import Neural_GPU
epochs = 5
iterations = 100
model = Neural_GPU.Neural_GPU()
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for epoch in range(1, epochs + 1):
train_cost = []
for iteration in range(iterations):
trainX, trainY = copy(model.batch_size, model.sequence_size,
model.vocab_size)
_, cost = sess.run(
[model.train_op, model.loss],
feed_dict={
model.inputs: trainX,
model.targets: trainY,
model.training: True,
model.learning_rate: 0.01,
model.keep_prob: 0.94
})
assert not np.isnan(cost), 'Epoch {}, Iteration {}'.format(
epoch, iteration + 1)
train_cost.append(cost)
print(
'Epoch {}, Average Training Loss {:.4f}, Final Training Loss {:.4f}'
.format(epoch, np.mean(train_cost), cost))
mem_size = 128, mem_width = 20, output_size = 8 ) max_sequences = 100000 patience = 20000 patience_increase = 3 improvement_threshold = 0.995 best_score = np.inf test_score = 0. score = None alpha = 0.95 for counter in xrange(max_sequences): # length = np.random.randint(int(20 * (min(counter,50000)/float(50000))**2) +1) + 1 length = np.random.randint(1,21) i,o = tasks.copy(8,length) if score == None: score = train(i,o) else: score = alpha * score + (1 - alpha) * train(i,o) print "round:", counter, "score:", score if score < best_score: # improve patience if loss improvement is good enough if score < best_score * improvement_threshold: patience = max(patience, counter * patience_increase) P.save(model_out) best_score = score if patience <= counter: break
model_out = sys.argv[1]
P, train = make_train(input_size=8,
mem_size=128,
mem_width=20,
output_size=8)
max_sequences = 100000
patience = 20000
patience_increase = 3
improvement_threshold = 0.995
best_score = np.inf
test_score = 0.
score = None
alpha = 0.95
for counter in xrange(max_sequences):
length = np.random.randint(
int(20 * (min(counter, 50000) / float(50000))**2) + 1) + 1
i, o = tasks.copy(8, length)
if score == None: score = train(i, o)
else: score = alpha * score + (1 - alpha) * train(i, o)
print score
if score < best_score:
# improve patience if loss improvement is good enough
if score < best_score * improvement_threshold:
patience = max(patience, counter * patience_increase)
P.save(model_out)
best_score = score
if patience <= counter: break
)
print "Done. (%0.3f s)"%(time.time() - start_time)
print P.parameter_count()
return P, P_learn, train, test
if __name__ == "__main__":
model_out = sys.argv[1]
P, P_learn, train, test = make_functions(
input_size=width + 2,
mem_size=128,
mem_width=20,
output_size=width + 2
)
test_set = [ tasks.copy(10, l+1, width) for l in xrange(max_sequence_length) ]
def validation_test():
return sum(test(i,o) for i,o in test_set) / (10 * len(test_set))
best_score = validation_test()
for counter in xrange(max_sequences):
length = np.random.randint(max_sequence_length) + 1
i, o = tasks.copy(batch_size, length, width)
score = train(i, o)
if (counter + 1) % validation_frequency == 0:
validation_score = validation_test()
print validation_score,
if validation_score < best_score:
best_score = validation_score
P.save('model.pkl')
plt.show()
def error_rate(expected_output, actual_output):
"""
Given the expected output data and the actual output data by the NTM,
compute the average error rate between the two.
"""
abs_diff = abs(expected_output - actual_output)
num_elements = expected_output.shape[0] * expected_output.shape[1]
return sum(sum(abs_diff)) / num_elements
if __name__ == "__main__":
if len(sys.argv) != 3:
print "Usage: python test.py <model_filename> <sequence_length>"
sys.exit()
# Make the NTM model and load the parameters
P, do_task = run_model.make_model()
P.load(sys.argv[1])
# Randomly generate a copy task and perform the task
input_data, expected_output = tasks.copy(8, int(sys.argv[2]))
weights, actual_output = do_task(input_data)
# Plot the outputs and compute the error_rate
plot(input_data, expected_output, actual_output)
print "The average error rate was " + str(error_rate(expected_output, actual_output))
def plot_weights(seq_length):
i,o = tasks.copy(8,seq_length)
weights,outputs = do_task(i)
plt.figure(1,figsize=(20,20))
plt.imshow(weights.T[100:123],interpolation='nearest',cmap=cm.gray)
plt.show()
test = theano.function(inputs=[input_seqs, output_seqs], outputs=bits_loss)
print "Done. (%0.3f s)" % (time.time() - start_time)
print P.parameter_count()
return P, P_learn, train, test
if __name__ == "__main__":
model_out = sys.argv[1]
P, P_learn, train, test = make_functions(input_size=width + 2,
mem_size=128,
mem_width=20,
output_size=width + 2)
test_set = [
tasks.copy(10, l + 1, width) for l in xrange(max_sequence_length)
]
def validation_test():
return sum(test(i, o) for i, o in test_set) / (10 * len(test_set))
best_score = validation_test()
for counter in xrange(max_sequences):
length = np.random.randint(max_sequence_length) + 1
i, o = tasks.copy(batch_size, length, width)
score = train(i, o)
if (counter + 1) % validation_frequency == 0:
validation_score = validation_test()
print validation_score,
if validation_score < best_score:
plt.show()
def error_rate(expected_output, actual_output):
"""
Given the expected output data and the actual output data by the NTM,
compute the average error rate between the two.
"""
abs_diff = abs(expected_output - actual_output)
num_elements = expected_output.shape[0] * expected_output.shape[1]
return sum(sum(abs_diff)) / num_elements
if __name__ == "__main__":
if len(sys.argv) != 3:
print "Usage: python test.py <model_filename> <sequence_length>"
sys.exit()
# Make the NTM model and load the parameters
P, do_task = run_model.make_model()
P.load(sys.argv[1])
# Randomly generate a copy task and perform the task
input_data, expected_output = tasks.copy(8, int(sys.argv[2]))
weights, actual_output = do_task(input_data)
# Plot the outputs and compute the error_rate
plot(input_data, expected_output, actual_output)
print "The average error rate was " + str(
error_rate(expected_output, actual_output))
