net.addConnection(hidden_bias)
net.addLayer(hidden_layer)
net.addConnection(hidden_to_output)
net.addConnection(output_bias)
net.addLayer(output_layer)
net.addObjective(objective)
net.setInputLayer(input_layer)
net.setTargetLayer(target_layer)
net.setOutputLayer(output_layer)
net.setObjective(objective)
net.randomize()
print "Done"
print "Creating a trainer..."
trainer = SGDTrainer(net,
learning_rate=0.1,
momentum=0.0,
weight_decay=0.00001)
# trainer = PSOTrainer(net, number_particles=20, initial_weight_range=(-5.0,5.0))
print "Done"
print "Training..."
for i in range(1000000):
trainer.trainBatch(training_set)
# print "Iteration", i, "\tObjetive =", trainer.global_best
print "Iteration", i, "\tObjetive =", net.getObjective()
print "Results:"
print net.getOutput()
net.addLayer(inp)
net.addLayer(tgt)
net.addConnection(i2r)
net.addConnection(rbias)
net.addRecurrentLayer(rec)
net.addConnection(r2o)
net.addConnection(obias)
net.addLayer(out)
net.addObjective(obj)
net.setInputLayer(inp)
net.setTargetLayer(tgt)
net.setOutputLayer(out)
net.setObjective(obj)
sgd = SGDTrainer(net, learning_rate=0.1, momentum=0.9)
seq = np.array([[[1], [1], [0]], [[1], [0], [0]], [[1], [1], [0]],
[[0], [0], [0]], [[0], [1], [0]], [[0], [0], [0]],
[[1], [1], [0]], [[1], [0], [0]], [[1], [1], [0]],
[[0], [0], [0]], [[0], [1], [0]], [[0], [0], [0]],
[[1], [1], [0]], [[1], [0], [0]], [[1], [1], [0]],
[[0], [0], [0]], [[0], [1], [0]], [[0], [0], [0]],
[[1], [1], [0]], [[1], [0], [0]], [[1], [1], [0]],
[[0], [0], [0]], [[0], [1], [0]], [[0], [0], [0]]])
tar = np.array([[[1], [1], [0]], [[0], [1], [0]], [[1], [0], [0]],
[[1], [0], [0]], [[1], [1], [0]], [[1], [1], [0]],
[[0], [0], [0]], [[1], [0], [0]], [[0], [1], [0]],
[[0], [1], [0]], [[0], [0], [0]], [[0], [0], [0]],
[[1], [1], [0]], [[0], [1], [0]], [[1], [0], [0]],
[[1], [0], [0]], [[1], [1], [0]], [[1], [1], [0]],
def run():
# Parameters
RNN_SIZE = 50
LEARNING_RATE = 0.00001
MOMENTUM = 0.5
WEIGHT_DECAY = 0.0001
START_EPOCH = 6750
NUMBER_EPOCHS = 15000
print "Building network and trainer..."
# Get the model and trainer
# rnn_net = buildNetwork(RNN_SIZE)
rnn_net = loadNetwork('RNN_model_snapshot_6750.pkl')
trainer = SGDTrainer(rnn_net, learning_rate = LEARNING_RATE, momentum = MOMENTUM, weight_decay = WEIGHT_DECAY)
print
training_filename = 'training_dataset.pkl'
validation_filename = 'validation_dataset.pkl'
test_filename = 'test_dataset.pkl'
print "Loading training data..."
f=open(training_filename, 'rb')
training_dataset = pickle.load(f)
f.close()
print "Loading validation data..."
f=open(validation_filename, 'rb')
validation_dataset = pickle.load(f)
f.close()
print "Loading test data..."
f=open(test_filename, 'rb')
test_dataset = pickle.load(f)
f.close()
print
print "Training and getting objective!"
print
# Let's classify things first and see how well it's starting off as
training_objective = rnn_net.getSequenceObjective(training_dataset)
validation_objective = rnn_net.getSequenceObjective(validation_dataset)
print "Step # 0:\tTraining Objective =", training_objective,", \t Validation Objective =", validation_objective
objective_file = open('RNN_model_objectives.txt', 'a')
# objective_file.write('Epoch Number, Training Objective, Validation Objective\n')
# objective_file.write(str(0) + ',' + str(training_objective) + ',' + str(validation_objective) + '\n')
# Let's start training!
for i in range(START_EPOCH, NUMBER_EPOCHS):
trainer.trainBatch(training_dataset)
training_objective = rnn_net.getSequenceObjective(training_dataset)
validation_objective = rnn_net.getSequenceObjective(validation_dataset)
print "Step #", i+1 ,":\tTraining Objective = ", training_objective,", \t Validation Objective =", validation_objective
objective_file.write(str(i+1) + ',' + str(training_objective) + ',' + str(validation_objective) + '\n')
objective_file.flush()
if (i+1)%25 == 0:
# Save the model!
f=open('RNN_model_snapshot_'+str(i+1)+'.pkl','wb')
# But don't save the dataset, cuz that part's big!
# rnn_net.setInput(np.zeros((0,0)))
# rnn_net.setTarget(np.zeros((0,0)))
pickle.dump(rnn_net, f)
f.close()
objective_file.close()
net.addLayer(target_layer)
net.addConnection(conn1)
net.addConnection(bias1)
net.addLayer(hidden_layer)
net.addConnection(conn2)
net.addConnection(bias2)
net.addLayer(output_layer)
net.addObjective(objective)
net.setInputLayer(input_layer)
net.setTargetLayer(target_layer)
net.setInput(dataset)
net.setTarget(targets)
net.setOutputLayer(output_layer)
net.setObjective(objective)
# Randomize these connections
net.randomize()
# Set the input and targets
# Everything from here on out deals directly with net
# Create a trainer
trainer = SGDTrainer(net, learning_rate=0.1)
for i in range(10000):
trainer.trainBatch((dataset, targets))
print i, net.getObjective()
print net.getOutput()
net.addLayer(target_layer)
net.addConnection(input_to_hidden)
net.addConnection(hidden_bias)
net.addLayer(hidden_layer)
net.addConnection(hidden_to_output)
net.addConnection(output_bias)
net.addLayer(output_layer)
net.addObjective(objective)
net.setInputLayer(input_layer)
net.setTargetLayer(target_layer)
net.setOutputLayer(output_layer)
net.setObjective(objective)
net.randomize()
print "Done"
print "Creating a trainer..."
trainer = SGDTrainer(net, learning_rate=0.1, momentum=0.0, weight_decay=0.00001)
# trainer = PSOTrainer(net, number_particles=20, initial_weight_range=(-5.0,5.0))
print "Done"
print "Training..."
for i in range(1000000):
trainer.trainBatch(training_set)
# print "Iteration", i, "\tObjetive =", trainer.global_best
print "Iteration", i, "\tObjetive =", net.getObjective()
print "Results:"
print net.getOutput()