def main():
parser = argparse.ArgumentParser()
parser.add_argument('', '--in-dim', type=int, default=4)
parser.add_argument('', '--rnn-units', type=int, default=50)
parser.add_argument('', '--rnn-layers', type=int, default=2)
parser.add_argument('', '--language-model', action="store_true", default=False)
parser.add_argument('', '--z-norm', action="store_true", default=False)
parser.add_argument('', '--n-epochs', type=int, default=2000)
parser.add_argument('data', type=str)
parser.add_argument('model', type=str)
args = parser.parse_args()
options = dict()
options['in_dim'] = args.in_dim
options['rnn_units'] = args.rnn_units
options['rnn_layers'] = args.rnn_layers
options['language_model'] = args.language_model
options['z_norm'] = args.z_norm
options['n_epochs'] = args.n_epochs
options['fname'] = args.data
options['model'] = args.model
with open('%s.pkl'%args.model, 'wb') as f:
pkl.dump(options, f)
# build model
params, tparams, cost, inps, opt_ret = build_model(options)
# build optimizer
grads = tensor.grad(cost.mean(), wrt=tparams)
f_grad_shared, f_update = rmsprop(tparams, grads, inps, cost)
trainer = BHIterator(fname=options['data'], batch_size=8, shuffle=True)
for ee in xrange(options['n_epochs']):
print 'Epoch', ee
uidx = 0
for x, y in trainer:
uidx += 1
x_, x_mask = prepare_timeseries(x)
# TODO: add some noise to the input
cc = f_grad_shared(x_, x_mask, x_, x_mask)
f_update()
print 'Update', uidx, 'Cost', cc
# pull the latest parameters
params = unzipp(tparams)
numpy.savez('%s_iter_%d'%(args.model,ee), params)
numpy.savez(args.model, params)
def _create_optimizer(self):
if self.optimizer_type == optimizers.L_BFGS:
self.optimizer = l_bfgs(self.loss, self.iterations)
elif self.optimizer_type == optimizers.ADAM:
self.optimizer = adam(self.learning_rate)
self.train_op = self.optimizer.minimize(self.loss)
elif self.optimizer_type == optimizers.ADAGRAD:
self.optimizer = adagrad(self.learning_rate)
self.train_op = self.optimizer.minimize(self.loss)
elif self.optimizer_type == optimizers.GRADIENT_DESCENT:
self.optimizer = gradient_descent(self.learning_rate)
self.train_op = self.optimizer.minimize(self.loss)
elif self.optimizer_type == optimizers.RMSPROP:
self.optimizer = rmsprop(self.learning_rate)
self.train_op = self.optimizer.minimize(self.loss)
elif self.optimizer_type == optimizers.ADADELTA:
self.optimizer = adadelta(self.learning_rate)
self.train_op = self.optimizer.minimize(self.loss)
else:
raise "Unsupported optimizer"
## interactive plot for tracking during learning
plt.ion()
fig, axarr = plt.subplots(5, 4, figsize=(13, 9))
## Create Batches, Call Optimizer
batch_idxs = cv.make_batches(N_data, batch_size)
cur_dir = np.zeros(N_weights)
means = pred_fun(W, train_images)
from optimizers import adam, rmsprop
def sub_loss_grad(x, i):
return loss_grad(x,
train_images[batch_idxs[i%len(batch_idxs)]],
samps[batch_idxs[i%len(batch_idxs)]])
W = rmsprop(sub_loss_grad,
x = W, callback=callback,
step_size = .006,
gamma = .6,
eps = 1e-8,
num_iters = 200)
####################################################################
#examine output
####################################################################
print "Final Loss: ", loss_fun(best_w, train_images, samps)
params = pred_fun(best_w, train_images)
means = params[:, :8]
variances = params[:, -8:]
i = 10
def compare_moments(i):
print "samp comparison, idx = %d "%i
print " {0:5} | {1:6} | {2:6} | {3:6} | {4:6} ".format(
if __name__ == '__main__':
import optimizers
n_in, n_out = 10, 1
X, y, output, params = generate_rnn(n_in, n_out, 50)
output = output[-1, :]
lr = T.scalar(name='lr', dtype=dtype)
# minimize binary crossentropy
xent = -y * T.log(output) - (1 - y) * T.log(1 - output)
cost = xent.mean()
updates = optimizers.rmsprop(cost, params, lr)
t_sets = 10
X_datas = [np.asarray(rng.rand(20, n_in) > 0.5, dtype=dtype) for _ in range(t_sets)]
y_datas = [np.asarray(rng.rand(1, n_out) > 0.5, dtype=dtype) for _ in range(t_sets)]
train = theano.function([X, y, lr], [cost], updates=updates)
test = theano.function([X], [output])
l = 0.1
n_train = 1000
cost = sum([train(X_data, y_data, 0)[0] for X_data, y_data in zip(X_datas, y_datas)])
print('Before training:', cost)
for i in range(n_train):
if __name__ == '__main__':
import optimizers
n_in, n_out = 10, 1
X, y, output, params = generate_rnn(n_in, n_out, 50)
output = output[-1, :]
lr = T.scalar(name='lr', dtype=dtype)
# minimize binary crossentropy
xent = -y * T.log(output) - (1 - y) * T.log(1 - output)
cost = xent.mean()
updates = optimizers.rmsprop(cost, params, lr)
t_sets = 10
X_datas = [
np.asarray(rng.rand(20, n_in) > 0.5, dtype=dtype)
for _ in range(t_sets)
]
y_datas = [
np.asarray(rng.rand(1, n_out) > 0.5, dtype=dtype)
for _ in range(t_sets)
]
train = theano.function([X, y, lr], [cost], updates=updates)
test = theano.function([X], [output])
l = 0.1