def main():
parser = argparse.ArgumentParser()
parser.add_argument("--profile",action="store_true")
parser.add_argument("--unittest",action="store_true")
parser.add_argument("--epochs",type=int,default=10)
args = parser.parse_args()
batchsize = 64
Xshape = (batchsize, 3, 32, 32)
X = cgt.tensor4("X", fixed_shape = Xshape)
y = cgt.vector("y", fixed_shape = (batchsize,), dtype='i4')
conv1 = nn.SpatialConvolution(3, 32, kernelshape=(5,5), pad=(2,2),
weight_init=nn.IIDGaussian(std=1e-4))(X)
relu1 = nn.rectify(conv1)
pool1 = nn.max_pool_2d(relu1, kernelshape=(3,3), stride=(2,2))
conv2 = nn.SpatialConvolution(32, 32, kernelshape=(5,5), pad=(2,2),
weight_init=nn.IIDGaussian(std=0.01))(pool1)
relu2 = nn.rectify(conv2)
pool2 = nn.max_pool_2d(relu2, kernelshape=(3,3), stride=(2,2))
conv3 = nn.SpatialConvolution(32, 64, kernelshape=(5,5), pad=(2,2),
weight_init=nn.IIDGaussian(std=0.01))(pool2)
pool3 = nn.max_pool_2d(conv3, kernelshape=(3,3), stride=(2,2))
relu3 = nn.rectify(pool3)
d0,d1,d2,d3 = relu3.shape
flatlayer = relu3.reshape([d0,d1*d2*d3])
nfeats = cgt.infer_shape(flatlayer)[1]
ip1 = nn.Affine(nfeats, 10)(flatlayer)
logprobs = nn.logsoftmax(ip1)
loss = -logprobs[cgt.arange(batchsize), y].mean()
params = nn.get_parameters(loss)
updates = rmsprop_updates(loss, params, stepsize=1e-3)
train = cgt.function(inputs=[X, y], outputs=[loss], updates=updates)
if args.profile: cgt.profiler.start()
data = fetch_dataset("http://rll.berkeley.edu/cgt-data/cifar10.npz")
Xtrain = data["X_train"]
ytrain = data["y_train"]
print fmt_row(10, ["Epoch","Train NLL","Train Err","Test NLL","Test Err","Epoch Time"])
for i_epoch in xrange(args.epochs):
for start in xrange(0, Xtrain.shape[0], batchsize):
tstart = time.time()
end = start+batchsize
print train(Xtrain[start:end], ytrain[start:end]), time.time()-tstart
if start > batchsize*5: break
# elapsed = time.time() - tstart
# trainerr, trainloss = computeloss(Xtrain[:len(Xtest)], ytrain[:len(Xtest)])
# testerr, testloss = computeloss(Xtest, ytest)
# print fmt_row(10, [i_epoch, trainloss, trainerr, testloss, testerr, elapsed])
if args.profile:
cgt.profiler.print_stats()
return
if args.unittest:
break
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--epochs", type=int, default=10)
parser.add_argument("--profile", action="store_true")
parser.add_argument("--dropout", action="store_true")
parser.add_argument("--stepsize", type=float, default=.001)
parser.add_argument("--model", choices=["dense", "conv"], default="dense")
parser.add_argument("--unittest", action="store_true")
parser.add_argument("--grad_check", action="store_true")
args = parser.parse_args()
if args.grad_check: cgt.set_precision("quad")
# from mldata.org http://mldata.org/repository/data/viewslug/mnist-original/
# converted to npz
mnist = fetch_dataset("http://rll.berkeley.edu/cgt-data/mnist.npz")
Xdata = (mnist["X"] / 255.).astype(cgt.floatX)
ydata = mnist["y"]
np.random.seed(0)
if args.model == "conv":
Xdata = Xdata.reshape(-1, 1, 28, 28)
Xtrain = Xdata[0:60000]
ytrain = ydata[0:60000]
Xtest = Xdata[60000:70000]
ytest = ydata[60000:70000]
sortinds = np.random.permutation(60000)
Xtrain = Xtrain[sortinds]
ytrain = ytrain[sortinds]
X = cgt.tensor4("X",
fixed_shape=(None, 1, 28,
28)) if args.model == "conv" else cgt.matrix(
"X", fixed_shape=(None, 28 * 28))
y = cgt.vector("y", dtype='i8')
if args.model == "dense":
p_drop_input, p_drop_hidden = (0.2, 0.5) if args.dropout else (0, 0)
w_h = init_weights(784, 256)
w_h2 = init_weights(256, 256)
w_o = init_weights(256, 10)
pofy_drop = dense_model(X, w_h, w_h2, w_o, p_drop_input, p_drop_hidden)
pofy_nodrop = dense_model(X, w_h, w_h2, w_o, 0., 0.)
params = [w_h, w_h2, w_o]
elif args.model == "conv":
p_drop_conv, p_drop_hidden = (0.2, 0.5) if args.dropout else (0, 0)
w = init_weights(32, 1, 3, 3)
w2 = init_weights(64, 32, 3, 3)
w3 = init_weights(128, 64, 3, 3)
w4 = init_weights(128 * 2 * 2, 625)
w_o = init_weights(625, 10)
pofy_drop = convnet_model(X, w, w2, w3, w4, w_o, p_drop_conv,
p_drop_hidden)
pofy_nodrop = convnet_model(X, w, w2, w3, w4, w_o, 0., 0.)
params = [w, w2, w3, w4, w_o]
else:
raise RuntimeError("Unreachable")
cost_drop = -cgt.mean(categorical.loglik(y, pofy_drop))
updates = rmsprop_updates(cost_drop, params, stepsize=args.stepsize)
y_nodrop = cgt.argmax(pofy_nodrop, axis=1)
cost_nodrop = -cgt.mean(categorical.loglik(y, pofy_nodrop))
err_nodrop = cgt.cast(cgt.not_equal(y_nodrop, y), cgt.floatX).mean()
train = cgt.function(inputs=[X, y], outputs=[], updates=updates)
computeloss = cgt.function(inputs=[X, y],
outputs=[err_nodrop, cost_nodrop])
batch_size = 128
from cgt.tests import gradcheck_model
if args.grad_check:
cost_nodrop = cgt.core.clone(cost_nodrop, {
X: Xtrain[:1],
y: ytrain[:1]
})
print "doing gradient check..."
print "------------------------------------"
gradcheck_model(cost_nodrop, params[0:1])
print "success!"
return
if args.profile: cgt.profiler.start()
print fmt_row(10, [
"Epoch", "Train NLL", "Train Err", "Test NLL", "Test Err", "Epoch Time"
])
for i_epoch in xrange(args.epochs):
tstart = time.time()
for start in xrange(0, Xtrain.shape[0], batch_size):
end = start + batch_size
train(Xtrain[start:end], ytrain[start:end])
if args.unittest: return
elapsed = time.time() - tstart
trainerr, trainloss = computeloss(Xtrain[:len(Xtest)],
ytrain[:len(Xtest)])
testerr, testloss = computeloss(Xtest, ytest)
print fmt_row(
10, [i_epoch, trainloss, trainerr, testloss, testerr, elapsed])
if args.profile: cgt.execution.profiler.print_stats()
# ================================================================
# Main script
# ================================================================
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--unittest", action="store_true")
args = parser.parse_args()
# Load data
# -----------------------
mnist = fetch_dataset("http://rll.berkeley.edu/cgt-data/mnist.npz")
Xdata = (mnist["X"] / 255.0).astype(cgt.floatX)
ydata = mnist["y"]
Ntrain = 1000 if args.unittest else 10000
Xtrain = Xdata[0:Ntrain]
ytrain = ydata[0:Ntrain]
sortinds = np.random.permutation(Ntrain)
Xtrain = Xtrain[sortinds]
ytrain = ytrain[sortinds]
batch_size = 128
cgt.update_config(backend="native")
# Make symbolic variables
# -----------------------
# ================================================================
# Main script
# ================================================================
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--unittest", action="store_true")
args = parser.parse_args()
# Load data
# -----------------------
mnist = fetch_dataset("http://rll.berkeley.edu/cgt-data/mnist.npz")
Xdata = (mnist["X"] / 255.).astype(cgt.floatX)
ydata = mnist["y"]
Ntrain = 1000 if args.unittest else 10000
Xtrain = Xdata[0:Ntrain]
ytrain = ydata[0:Ntrain]
sortinds = np.random.permutation(Ntrain)
Xtrain = Xtrain[sortinds]
ytrain = ytrain[sortinds]
batch_size = 128
cgt.update_config(backend="native")
# Make symbolic variables
# -----------------------
def main():
import argparse
parser=argparse.ArgumentParser()
parser.add_argument("--epochs",type=int,default=10)
parser.add_argument("--profile",action="store_true")
parser.add_argument("--dropout",action="store_true")
parser.add_argument("--stepsize",type=float, default=.001)
parser.add_argument("--model",choices=["dense","conv"],default="dense")
parser.add_argument("--unittest",action="store_true")
parser.add_argument("--grad_check",action="store_true")
parser.add_argument("--devtype",choices=["cpu","gpu"],default="cpu")
args = parser.parse_args()
if args.grad_check: cgt.set_precision("quad")
# from mldata.org http://mldata.org/repository/data/viewslug/mnist-original/
# converted to npz
mnist = fetch_dataset("http://rll.berkeley.edu/cgt-data/mnist.npz")
Xdata = (mnist["X"]/255.).astype(cgt.floatX)
ydata = mnist["y"]
np.random.seed(0)
cgt.update_config(default_device=cgt.core.Device(devtype=args.devtype), backend="native")
if args.model=="conv":
Xdata = Xdata.reshape(-1, 1, 28, 28)
Xtrain = Xdata[0:60000]
ytrain = ydata[0:60000]
Xtest = Xdata[60000:70000]
ytest = ydata[60000:70000]
sortinds = np.random.permutation(60000)
Xtrain = Xtrain[sortinds]
ytrain = ytrain[sortinds]
X = cgt.tensor4("X",fixed_shape=(None,1,28,28)) if args.model=="conv" else cgt.matrix("X", fixed_shape=(None,28*28))
y = cgt.vector("y",dtype='i8')
if args.model == "dense":
p_drop_input,p_drop_hidden = (0.2, 0.5) if args.dropout else (0,0)
w_h = init_weights(784, 256)
w_h2 = init_weights(256, 256)
w_o = init_weights(256, 10)
pofy_drop = dense_model(X, w_h, w_h2, w_o, p_drop_input, p_drop_hidden)
pofy_nodrop = dense_model(X, w_h, w_h2, w_o, 0., 0.)
params = [w_h, w_h2, w_o]
elif args.model == "conv":
p_drop_conv,p_drop_hidden = (0.2, 0.5) if args.dropout else (0,0)
w = init_weights(32, 1, 3, 3)
w2 = init_weights(64, 32, 3, 3)
w3 = init_weights(128, 64, 3, 3)
w4 = init_weights(128 * 2 * 2, 625)
w_o = init_weights(625, 10)
pofy_drop = convnet_model(X, w, w2, w3, w4, w_o, p_drop_conv, p_drop_hidden)
pofy_nodrop = convnet_model(X, w, w2, w3, w4, w_o, 0., 0.)
params = [w, w2, w3, w4, w_o]
else:
raise RuntimeError("Unreachable")
cost_drop = -cgt.mean(categorical.loglik(y, pofy_drop))
updates = rmsprop_updates(cost_drop, params, stepsize=args.stepsize)
y_nodrop = cgt.argmax(pofy_nodrop, axis=1)
cost_nodrop = -cgt.mean(categorical.loglik(y, pofy_nodrop))
err_nodrop = cgt.cast(cgt.not_equal(y_nodrop, y), cgt.floatX).mean()
train = cgt.function(inputs=[X, y], outputs=[], updates=updates)
computeloss = cgt.function(inputs=[X, y], outputs=[err_nodrop,cost_nodrop])
batch_size=128
from cgt.tests import gradcheck_model
if args.grad_check:
cost_nodrop = cgt.core.clone(cost_nodrop, {X:Xtrain[:1],y:ytrain[:1]})
print "doing gradient check..."
print "------------------------------------"
gradcheck_model(cost_nodrop, params[0:1])
print "success!"
return
if args.profile: cgt.profiler.start()
print fmt_row(10, ["Epoch","Train NLL","Train Err","Test NLL","Test Err","Epoch Time"])
for i_epoch in xrange(args.epochs):
tstart = time.time()
for start in xrange(0, Xtrain.shape[0], batch_size):
end = start+batch_size
train(Xtrain[start:end], ytrain[start:end])
if args.unittest: return
elapsed = time.time() - tstart
trainerr, trainloss = computeloss(Xtrain[:len(Xtest)], ytrain[:len(Xtest)])
testerr, testloss = computeloss(Xtest, ytest)
print fmt_row(10, [i_epoch, trainloss, trainerr, testloss, testerr, elapsed])
if args.profile: cgt.execution.profiler.print_stats()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--profile", action="store_true")
parser.add_argument("--unittest", action="store_true")
parser.add_argument("--epochs", type=int, default=10)
args = parser.parse_args()
batchsize = 64
Xshape = (batchsize, 3, 32, 32)
X = cgt.tensor4("X", fixed_shape=Xshape)
y = cgt.vector("y", fixed_shape=(batchsize, ), dtype='i4')
conv1 = nn.SpatialConvolution(3,
32,
kernelshape=(5, 5),
pad=(2, 2),
weight_init=nn.IIDGaussian(std=1e-4))(X)
relu1 = nn.rectify(conv1)
pool1 = nn.max_pool_2d(relu1, kernelshape=(3, 3), stride=(2, 2))
conv2 = nn.SpatialConvolution(32,
32,
kernelshape=(5, 5),
pad=(2, 2),
weight_init=nn.IIDGaussian(std=0.01))(pool1)
relu2 = nn.rectify(conv2)
pool2 = nn.max_pool_2d(relu2, kernelshape=(3, 3), stride=(2, 2))
conv3 = nn.SpatialConvolution(32,
64,
kernelshape=(5, 5),
pad=(2, 2),
weight_init=nn.IIDGaussian(std=0.01))(pool2)
pool3 = nn.max_pool_2d(conv3, kernelshape=(3, 3), stride=(2, 2))
relu3 = nn.rectify(pool3)
d0, d1, d2, d3 = relu3.shape
flatlayer = relu3.reshape([d0, d1 * d2 * d3])
nfeats = cgt.infer_shape(flatlayer)[1]
ip1 = nn.Affine(nfeats, 10)(flatlayer)
logprobs = nn.logsoftmax(ip1)
loss = -logprobs[cgt.arange(batchsize), y].mean()
params = nn.get_parameters(loss)
updates = rmsprop_updates(loss, params, stepsize=1e-3)
train = cgt.function(inputs=[X, y], outputs=[loss], updates=updates)
if args.profile: cgt.profiler.start()
data = fetch_dataset("http://rll.berkeley.edu/cgt-data/cifar10.npz")
Xtrain = data["X_train"]
ytrain = data["y_train"]
print fmt_row(10, [
"Epoch", "Train NLL", "Train Err", "Test NLL", "Test Err", "Epoch Time"
])
for i_epoch in xrange(args.epochs):
for start in xrange(0, Xtrain.shape[0], batchsize):
tstart = time.time()
end = start + batchsize
print train(Xtrain[start:end],
ytrain[start:end]), time.time() - tstart
if start > batchsize * 5: break
# elapsed = time.time() - tstart
# trainerr, trainloss = computeloss(Xtrain[:len(Xtest)], ytrain[:len(Xtest)])
# testerr, testloss = computeloss(Xtest, ytest)
# print fmt_row(10, [i_epoch, trainloss, trainerr, testloss, testerr, elapsed])
if args.profile:
cgt.profiler.print_stats()
return
if args.unittest:
break
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--batch_size", default=100)
parser.add_argument("--nlayers", default=1, type=int, help="number of hidden layers in MLP before output layers")
parser.add_argument("--hdim", default=500, type=int, help="dimension of hidden layer")
parser.add_argument("--zdim", default=2, type=int, help="dimension of continuous latent variable")
parser.add_argument("--lmbda", default=0.001, type=float, help="weight decay coefficient")
parser.add_argument("--lr", default=0.01, type=float, help="learning rate")
parser.add_argument("--epochs", default=1000, type=int, help="number of passes over dataset")
parser.add_argument("--print_every", default=100, type=int, help="how often to print cost")
parser.add_argument("--outfile", default="vae_model.pk", help="output file to save model to")
args = parser.parse_args()
print(args)
if args.epochs > 100:
print("NOTE: training might take a while. You may want to first sanity check by setting --epochs to something like 20 (manifold will be fuzzy).")
# set up dataset
mnist = fetch_dataset("http://rll.berkeley.edu/cgt-data/mnist.npz")
X = (mnist["X"]/255.).astype(cgt.floatX)
y = mnist["y"]
np.random.seed(0)
sortinds = np.random.permutation(70000)
X = X[sortinds]
y = y[sortinds]
train_x = X[0:50000]
train_y = y[0:50000]
valid_x = X[50000:60000]
valid_y = y[50000:60000]
# run SGVB algorithm
model = VAE(train_x.shape[1], args, dec="bernoulli")
expcost = None
num_train_batches = train_x.shape[0] / args.batch_size
num_valid_batches = valid_x.shape[0] / args.batch_size
valid_freq = num_train_batches
for b in xrange(args.epochs * num_train_batches):
k = b % num_train_batches
x = train_x[k * args.batch_size:(k + 1) * args.batch_size, :]
eps = np.random.randn(x.shape[0], args.zdim).astype(cgt.floatX)
cost = model.train(x, eps)
if not expcost:
expcost = cost
else:
expcost = 0.01 * cost + 0.99 * expcost
if (b + 1) % args.print_every == 0:
print("iter %d, cost %f, expcost %f" % (b + 1, cost, expcost))
if (b + 1) % valid_freq == 0:
valid_cost = 0
for l in xrange(num_valid_batches):
x_val = valid_x[l * args.batch_size:(l + 1) * args.batch_size, :]
eps_val = np.zeros((x_val.shape[0], args.zdim), dtype=cgt.floatX)
valid_cost = valid_cost + model.test(x_val, eps_val)
valid_cost = valid_cost / num_valid_batches
print("valid cost: %f" % valid_cost)
# XXX fix pickling of cgt models
#print("saving final model")
#with open(args.outfile, "wb") as f:
#pickle.dump(model, f, protocol=pickle.HIGHEST_PROTOCOL)
# XXX use this to sample, should later be able to compile f(z) = y directly (See Issue #18)
newz = cgt.matrix("newz", dtype=cgt.floatX)
newy = cgt.core.clone(model.dec_mlp.out, {model.enc_mlp.out:newz})
decode = cgt.function(
[newz],
newy
)
S = (28, 28)
M = 20
manifold = np.zeros((S[0]*M, S[1]*M), dtype=cgt.floatX)
for z1 in xrange(M):
for z2 in xrange(M):
print z1, z2
z = np.zeros((1, 2))
# pass unit square through inverse Gaussian CDF
z[0, 0] = norm.ppf(z1 * 1.0/M + 1.0/(M * 2))
z[0, 1] = norm.ppf(z2 * 1.0/M + 1.0/(M * 2))
z = np.array(z, dtype=cgt.floatX)
x_hat = decode(z)
x_hat = x_hat.reshape(S)
manifold[z1 * S[0]:(z1 + 1) * S[0],
z2 * S[1]:(z2 + 1) * S[1]] = x_hat
plt.imshow(manifold, cmap="Greys_r")
plt.axis("off")
plt.show()
