def train_rnn_binary_add():
print ''
print 'Training RNN for binary add'
print ''
x_train, t_train = generate_binary_add_data(50, int_max=100)
x_val, t_val = generate_binary_add_data(20, int_max=200)
in_dim = x_train[0].shape[1]
out_dim = t_train[0].shape[1]
hid_dim = 20
net = nn.NeuralNet(hid_dim, out_dim)
net.add_layer(0, nonlin_type=ly.NONLIN_NAME_LINEAR)
net.set_loss(ls.LOSS_NAME_SQUARED)
rnn_net = rnn.RnnHybridNetwork(
rnn.RNN(in_dim, hid_dim, nonlin_type=ly.NONLIN_NAME_TANH), net)
print rnn_net
rnn_learner = rnn.SequenceLearner(rnn_net)
rnn_learner.load_data(x_train, t_train, x_val=x_val, t_val=t_val)
# rnn_learner.train_gradient_descent(learn_rate=1e-2, momentum=0.5, iprint=10, max_iters=200, max_grad_norm=10)
# rnn_learner.train_gradient_descent(learn_rate=1e-3, momentum=0.9, iprint=10, max_iters=200)
# rnn_learner.train_gradient_descent(learn_rate=1e-2, momentum=0, iprint=10, max_iters=200, adagrad_start_iter=10)
rnn_learner.train_sgd(minibatch_size=1,
learn_rate=1e-1,
momentum=0.9,
iprint=100,
adagrad_start_iter=1,
max_iters=2000,
max_grad_norm=1)
return rnn_net
def create_default_rnn_on_nn(add_noise=False):
in_dim = 3
net_hid_dim = 2
rnn_in_dim = 2
hid_dim = 2
out_dim = 3
net = nn.NeuralNet(in_dim, hid_dim)
net.add_layer(net_hid_dim,
nonlin_type=ly.NONLIN_NAME_TANH,
dropout=(0.5 if add_noise else 0))
net.add_layer(0, nonlin_type=ly.NONLIN_NAME_SIGMOID)
rnn_net = rnn.RnnOnNeuralNet(
net,
rnn.RNN(in_dim=rnn_in_dim,
out_dim=hid_dim,
nonlin_type=ly.NONLIN_NAME_TANH))
predict_net = nn.NeuralNet(hid_dim, out_dim)
predict_net.add_layer(0, nonlin_type=ly.NONLIN_NAME_LINEAR)
predict_net.set_loss(ls.LOSS_NAME_SQUARED)
rnn_predict_net = rnn.RnnHybridNetwork(rnn_net, predict_net)
return rnn_predict_net
def train_rnn_on_nn_ae():
print ''
print 'Training RNN autoencoder'
print ''
x_train, _ = generate_binary_add_data(50, int_max=64)
x_val, _ = generate_binary_add_data(20, int_max=64)
in_dim = x_train[0].shape[1]
out_dim = in_dim
hid_dim = 10
out_hid_dim = 5
in_hid_dim = 5
net = nn.NeuralNet(hid_dim, out_dim)
net.add_layer(out_hid_dim, nonlin_type=ly.NONLIN_NAME_RELU)
net.add_layer(0, nonlin_type=ly.NONLIN_NAME_LINEAR)
net.set_loss(ls.LOSS_NAME_SQUARED)
dec = rnn.RnnHybridNetwork(
rnn.RNN(out_dim=hid_dim, nonlin_type=ly.NONLIN_NAME_RELU), net)
enc_net = nn.NeuralNet(in_dim, in_hid_dim)
enc_net.add_layer(0, nonlin_type=ly.NONLIN_NAME_RELU)
enc = rnn.RnnOnNeuralNet(
enc_net,
rnn.RNN(in_dim=in_hid_dim,
out_dim=hid_dim,
nonlin_type=ly.NONLIN_NAME_RELU))
ae = rnn.RnnAutoEncoder(encoder=enc, decoder=dec)
print ae
rnn_learner = rnn.SequenceLearner(ae)
# rnn_learner.load_data(x_train, revert_sequence(x_train), x_val=x_val, t_val=revert_sequence(x_val))
rnn_learner.load_data(x_train, x_train, x_val=x_val, t_val=x_val)
# rnn_learner.train_gradient_descent(learn_rate=1e-2, momentum=0.5, iprint=10, max_iters=200, max_grad_norm=10)
# rnn_learner.train_gradient_descent(learn_rate=1e-3, momentum=0.9, iprint=10, max_iters=200)
# rnn_learner.train_gradient_descent(learn_rate=1e-2, momentum=0, iprint=10, max_iters=200, adagrad_start_iter=10)
rnn_learner.train_sgd(minibatch_size=1,
learn_rate=1e-1,
momentum=0.9,
iprint=100,
adagrad_start_iter=1,
max_iters=10000,
max_grad_norm=1)
return ae
def create_default_rnn_ae(add_noise=False):
in_dim = 3
hid_dim = 2
net_hid_dim = 2
net = nn.NeuralNet(hid_dim, in_dim)
net.add_layer(net_hid_dim,
ly.NONLIN_NAME_TANH,
dropout=(0.5 if add_noise else 0))
net.add_layer(0, ly.NONLIN_NAME_LINEAR)
net.set_loss(ls.LOSS_NAME_SQUARED)
dec = rnn.RnnHybridNetwork(rnn.RNN(out_dim=hid_dim), net)
enc = rnn.RNN(in_dim=in_dim, out_dim=hid_dim)
return rnn.RnnAutoEncoder(encoder=enc, decoder=dec)
def create_default_rnn_hybrid(add_noise=False, has_input=True):
in_dim = 3 if has_input else None
hid_dim = 2
out_dim = 2
net_hid_dim = 2
net = nn.NeuralNet(hid_dim, out_dim)
net.add_layer(net_hid_dim,
nonlin_type=ly.NONLIN_NAME_TANH,
dropout=(0 if not add_noise else 0.5))
net.add_layer(0, nonlin_type=ly.NONLIN_NAME_LINEAR)
net.set_loss(ls.LOSS_NAME_SQUARED)
rnn_net = rnn.RNN(in_dim, hid_dim)
if not has_input:
rnn_net.b = gnp.randn(hid_dim)
return rnn.RnnHybridNetwork(rnn_net, net)
def f_create_void():
return rnn.RnnHybridNetwork()