def test_devices(): cgt.set_precision("double") cgt.update_config(backend="native") N = 10 K = 3 compile_info = cgt.compilation.get_compile_info() cuda_enabled = compile_info["CGT_ENABLE_CUDA"] if not cuda_enabled: raise SkipTest("cuda disabled") Xval = np.random.randn(N, K).astype(cgt.floatX) wval = np.random.randn(K).astype(cgt.floatX) bval = np.asarray(np.random.randn()).astype(cgt.floatX) yval = np.random.randn(N).astype(cgt.floatX) with cgt.scoped_update_config(default_device=cgt.Device(devtype="gpu")): X_nk = cgt.shared(Xval, "X", device=cgt.Device(devtype="gpu")) y_n = cgt.shared(yval, "y") w_k = cgt.shared(wval, "w") b = cgt.shared(bval, name="b") print "bval", bval ypred = cgt.dot(cgt.square(X_nk), w_k) + b err = cgt.sum(cgt.sin(ypred - y_n)) g = cgt.grad(err, [w_k, b]) outputs = [err] + g f = cgt.function([], [err] + g) results = f() print results assert np.allclose(results[0], np.sin(np.square(Xval).dot(wval) + bval - yval).sum())
args = parser.parse_args() # Load data # ----------------------- mnist = fetch_dataset("http://rll.berkeley.edu/cgt-data/mnist.npz") Xdata = (mnist["X"] / 255.0).astype(cgt.floatX) ydata = mnist["y"] Ntrain = 1000 if args.unittest else 10000 Xtrain = Xdata[0:Ntrain] ytrain = ydata[0:Ntrain] sortinds = np.random.permutation(Ntrain) Xtrain = Xtrain[sortinds] ytrain = ytrain[sortinds] batch_size = 128 cgt.update_config(backend="native") # Make symbolic variables # ----------------------- def build_fc_return_loss(X, y): """ Build fully connected network and return loss """ np.random.seed(0) h1 = nn.rectify(nn.Affine(28 * 28, 256, weight_init=nn.IIDGaussian(std=0.1))(X)) h2 = nn.rectify(nn.Affine(256, 256, weight_init=nn.IIDGaussian(std=0.1))(h1)) logprobs = nn.logsoftmax(nn.Affine(256, 10, weight_init=nn.IIDGaussian(std=0.1))(h2)) neglogliks = -logprobs[cgt.arange(X.shape[0]), y] loss = neglogliks.mean() return loss
args = parser.parse_args() # Load data # ----------------------- mnist = fetch_dataset("http://rll.berkeley.edu/cgt-data/mnist.npz") Xdata = (mnist["X"] / 255.).astype(cgt.floatX) ydata = mnist["y"] Ntrain = 1000 if args.unittest else 10000 Xtrain = Xdata[0:Ntrain] ytrain = ydata[0:Ntrain] sortinds = np.random.permutation(Ntrain) Xtrain = Xtrain[sortinds] ytrain = ytrain[sortinds] batch_size = 128 cgt.update_config(backend="native") # Make symbolic variables # ----------------------- def build_fc_return_loss(X, y): """ Build fully connected network and return loss """ np.random.seed(0) h1 = nn.rectify( nn.Affine(28 * 28, 256, weight_init=nn.IIDGaussian(std=.1))(X)) h2 = nn.rectify( nn.Affine(256, 256, weight_init=nn.IIDGaussian(std=.1))(h1)) logprobs = nn.logsoftmax(
def main(): import argparse parser=argparse.ArgumentParser() parser.add_argument("--epochs",type=int,default=10) parser.add_argument("--profile",action="store_true") parser.add_argument("--dropout",action="store_true") parser.add_argument("--stepsize",type=float, default=.001) parser.add_argument("--model",choices=["dense","conv"],default="dense") parser.add_argument("--unittest",action="store_true") parser.add_argument("--grad_check",action="store_true") parser.add_argument("--devtype",choices=["cpu","gpu"],default="cpu") args = parser.parse_args() if args.grad_check: cgt.set_precision("quad") # from mldata.org http://mldata.org/repository/data/viewslug/mnist-original/ # converted to npz mnist = fetch_dataset("http://rll.berkeley.edu/cgt-data/mnist.npz") Xdata = (mnist["X"]/255.).astype(cgt.floatX) ydata = mnist["y"] np.random.seed(0) cgt.update_config(default_device=cgt.core.Device(devtype=args.devtype), backend="native") if args.model=="conv": Xdata = Xdata.reshape(-1, 1, 28, 28) Xtrain = Xdata[0:60000] ytrain = ydata[0:60000] Xtest = Xdata[60000:70000] ytest = ydata[60000:70000] sortinds = np.random.permutation(60000) Xtrain = Xtrain[sortinds] ytrain = ytrain[sortinds] X = cgt.tensor4("X",fixed_shape=(None,1,28,28)) if args.model=="conv" else cgt.matrix("X", fixed_shape=(None,28*28)) y = cgt.vector("y",dtype='i8') if args.model == "dense": p_drop_input,p_drop_hidden = (0.2, 0.5) if args.dropout else (0,0) w_h = init_weights(784, 256) w_h2 = init_weights(256, 256) w_o = init_weights(256, 10) pofy_drop = dense_model(X, w_h, w_h2, w_o, p_drop_input, p_drop_hidden) pofy_nodrop = dense_model(X, w_h, w_h2, w_o, 0., 0.) params = [w_h, w_h2, w_o] elif args.model == "conv": p_drop_conv,p_drop_hidden = (0.2, 0.5) if args.dropout else (0,0) w = init_weights(32, 1, 3, 3) w2 = init_weights(64, 32, 3, 3) w3 = init_weights(128, 64, 3, 3) w4 = init_weights(128 * 2 * 2, 625) w_o = init_weights(625, 10) pofy_drop = convnet_model(X, w, w2, w3, w4, w_o, p_drop_conv, p_drop_hidden) pofy_nodrop = convnet_model(X, w, w2, w3, w4, w_o, 0., 0.) params = [w, w2, w3, w4, w_o] else: raise RuntimeError("Unreachable") cost_drop = -cgt.mean(categorical.loglik(y, pofy_drop)) updates = rmsprop_updates(cost_drop, params, stepsize=args.stepsize) y_nodrop = cgt.argmax(pofy_nodrop, axis=1) cost_nodrop = -cgt.mean(categorical.loglik(y, pofy_nodrop)) err_nodrop = cgt.cast(cgt.not_equal(y_nodrop, y), cgt.floatX).mean() train = cgt.function(inputs=[X, y], outputs=[], updates=updates) computeloss = cgt.function(inputs=[X, y], outputs=[err_nodrop,cost_nodrop]) batch_size=128 from cgt.tests import gradcheck_model if args.grad_check: cost_nodrop = cgt.core.clone(cost_nodrop, {X:Xtrain[:1],y:ytrain[:1]}) print "doing gradient check..." print "------------------------------------" gradcheck_model(cost_nodrop, params[0:1]) print "success!" return if args.profile: cgt.profiler.start() print fmt_row(10, ["Epoch","Train NLL","Train Err","Test NLL","Test Err","Epoch Time"]) for i_epoch in xrange(args.epochs): tstart = time.time() for start in xrange(0, Xtrain.shape[0], batch_size): end = start+batch_size train(Xtrain[start:end], ytrain[start:end]) if args.unittest: return elapsed = time.time() - tstart trainerr, trainloss = computeloss(Xtrain[:len(Xtest)], ytrain[:len(Xtest)]) testerr, testloss = computeloss(Xtest, ytest) print fmt_row(10, [i_epoch, trainloss, trainerr, testloss, testerr, elapsed]) if args.profile: cgt.execution.profiler.print_stats()
def main(): parser = argparse.ArgumentParser() parser.add_argument("--profile",action="store_true") parser.add_argument("--unittest",action="store_true") parser.add_argument("--epochs",type=int,default=10) parser.add_argument("--devtype",choices=["cpu","gpu"],default="cpu") args = parser.parse_args() cgt.update_config(default_device=cgt.core.Device(devtype=args.devtype), backend="native") batchsize = 64 Xshape = (batchsize, 3, 32, 32) X = cgt.tensor4("X", fixed_shape = Xshape) y = cgt.vector("y", fixed_shape = (batchsize,), dtype='i4') conv1 = nn.SpatialConvolution(3, 32, kernelshape=(5,5), pad=(2,2), weight_init=nn.IIDGaussian(std=1e-4))(X) relu1 = nn.rectify(conv1) pool1 = nn.max_pool_2d(relu1, kernelshape=(3,3), stride=(2,2)) conv2 = nn.SpatialConvolution(32, 32, kernelshape=(5,5), pad=(2,2), weight_init=nn.IIDGaussian(std=0.01))(pool1) relu2 = nn.rectify(conv2) pool2 = nn.max_pool_2d(relu2, kernelshape=(3,3), stride=(2,2)) conv3 = nn.SpatialConvolution(32, 64, kernelshape=(5,5), pad=(2,2), weight_init=nn.IIDGaussian(std=0.01))(pool2) pool3 = nn.max_pool_2d(conv3, kernelshape=(3,3), stride=(2,2)) relu3 = nn.rectify(pool3) d0,d1,d2,d3 = relu3.shape flatlayer = relu3.reshape([d0,d1*d2*d3]) nfeats = cgt.infer_shape(flatlayer)[1] ip1 = nn.Affine(nfeats, 10)(flatlayer) logprobs = nn.logsoftmax(ip1) loss = -logprobs[cgt.arange(batchsize), y].mean() params = nn.get_parameters(loss) updates = rmsprop_updates(loss, params, stepsize=1e-3) train = cgt.function(inputs=[X, y], outputs=[loss], updates=updates) if args.profile: cgt.profiler.start() data = fetch_dataset("http://rll.berkeley.edu/cgt-data/cifar10.npz") Xtrain = data["X_train"] ytrain = data["y_train"] print fmt_row(10, ["Epoch","Train NLL","Train Err","Test NLL","Test Err","Epoch Time"]) for i_epoch in xrange(args.epochs): for start in xrange(0, Xtrain.shape[0], batchsize): tstart = time.time() end = start+batchsize print train(Xtrain[start:end], ytrain[start:end]), time.time()-tstart if start > batchsize*5: break # elapsed = time.time() - tstart # trainerr, trainloss = computeloss(Xtrain[:len(Xtest)], ytrain[:len(Xtest)]) # testerr, testloss = computeloss(Xtest, ytest) # print fmt_row(10, [i_epoch, trainloss, trainerr, testloss, testerr, elapsed]) if args.profile: cgt.profiler.print_stats() return if args.unittest: break
def __init__(self, model="dense", im_size=[28, 28], dropout=True, devtype="cpu", grad_check=True, reg=0): if grad_check: cgt.set_precision("quad") self.model = model self.reg = reg np.random.seed(0) cgt.update_config(default_device=cgt.core.Device(devtype=devtype), backend="native") print(model) # MLP with 1 hidden layer if model == "dense1": self.Xsize = 2 * im_size[0] * im_size[1] + im_size[0] + im_size[1] self.X = cgt.matrix("X", fixed_shape=(None, self.Xsize)) self.y = cgt.vector("y", dtype='i8') self.p_drop_input, self.p_drop_hidden = (0.2, 0.5) if dropout else (0, 0) self.w_h = init_weights(self.Xsize, 256) self.w_o = init_weights(256, 8) self.pofy_drop = dense_model1(self.X, self.w_h, self.w_o, self.p_drop_input, self.p_drop_hidden) self.pofy_nodrop = dense_model1(self.X, self.w_h, self.w_o, 0., 0.) self.params = [self.w_h, self.w_o] self.l1 = cgt.abs(self.w_h).sum() + cgt.abs(self.w_o).sum() self.cost_drop = -cgt.mean( categorical.loglik(self.y, self.pofy_drop)) + self.reg * self.l1 # MLP with 2 hidden layers elif model == "dense2": self.Xsize = 2 * im_size[0] * im_size[1] + im_size[0] + im_size[1] self.X = cgt.matrix("X", fixed_shape=(None, self.Xsize)) self.y = cgt.vector("y", dtype='i8') self.p_drop_input, self.p_drop_hidden = (0.2, 0.5) if dropout else (0, 0) self.w_h = init_weights(self.Xsize, 256) self.w_h2 = init_weights(256, 256) self.w_o = init_weights(256, 8) self.pofy_drop = dense_model2(self.X, self.w_h, self.w_h2, self.w_o, self.p_drop_input, self.p_drop_hidden) self.pofy_nodrop = dense_model2(self.X, self.w_h, self.w_h2, self.w_o, 0., 0.) self.params = [self.w_h, self.w_h2, self.w_o] self.l1 = cgt.abs(self.w_h).sum() + cgt.abs( self.w_h2).sum() + cgt.abs(self.w_o).sum() self.cost_drop = -cgt.mean( categorical.loglik(self.y, self.pofy_drop)) + self.reg * self.l1 # MLP with 3 hidden layers elif model == "dense3": self.Xsize = 2 * im_size[0] * im_size[1] + im_size[0] + im_size[1] self.X = cgt.matrix("X", fixed_shape=(None, self.Xsize)) self.y = cgt.vector("y", dtype='i8') self.p_drop_input, self.p_drop_hidden = ( 0.0, [0.5, 0.5, 0.5]) if dropout else (0, [0, 0, 0]) self.w_h = init_weights(self.Xsize, 256) self.w_h2 = init_weights(256, 256) self.w_h3 = init_weights(256, 256) self.w_o = init_weights(256, 8) self.pofy_drop = dense_model3(self.X, self.w_h, self.w_h2, self.w_h3, self.w_o, self.p_drop_input, self.p_drop_hidden) self.pofy_nodrop = dense_model3(self.X, self.w_h, self.w_h2, self.w_h3, self.w_o, 0., [0., 0., 0.]) self.params = [self.w_h, self.w_h2, self.w_h3, self.w_o] self.l1 = cgt.abs(self.w_h).sum() + cgt.abs(self.w_h2).sum() + cgt.abs(self.w_h3).sum() + \ cgt.abs(self.w_o).sum() self.cost_drop = -cgt.mean( categorical.loglik(self.y, self.pofy_drop)) + self.reg * self.l1 else: raise RuntimeError("Unknown Model") self.y_nodrop = cgt.argmax(self.pofy_nodrop, axis=1) self.cost_nodrop = -cgt.mean( categorical.loglik(self.y, self.pofy_nodrop)) self.err_nodrop = cgt.cast(cgt.not_equal(self.y_nodrop, self.y), cgt.floatX).mean() self.computeloss = cgt.function( inputs=[self.X, self.y], outputs=[self.err_nodrop, self.cost_nodrop]) self.y_out = cgt.function(inputs=[self.X], outputs=[self.y_nodrop]) self.updates = rmsprop_updates(self.cost_drop, self.params) self.train = cgt.function(inputs=[self.X, self.y], outputs=[], updates=self.updates)