def test_setting_weights():
X = cgt.matrix("X", fixed_shape=(None, 28*28))
model = build_model(X, 0.0)
nnbuilder.set_all_weights(model, 'mnist.p')
y = cgt.vector("y", dtype='i8')
cost = -cgt.mean(categorical.loglik(y, model))
selected_number = cgt.argmax(model, axis=1)
err_nodrop = cgt.cast(cgt.not_equal(selected_number, y), cgt.floatX).mean()
computeloss = cgt.function(inputs=[X, y], outputs=[err_nodrop, cost])
Xdata, ydata = load_data()
Xtrain = Xdata[0:60000]
ytrain = ydata[0:60000]
Xtest = Xdata[60000:70000]
ytest = ydata[60000:70000]
sortinds = np.random.permutation(60000)
Xtrain = Xtrain[sortinds]
ytrain = ytrain[sortinds]
print fmt_row(10, ["Epoch","Train NLL","Train Err","Test NLL","Test Err","Epoch Time"])
for i_epoch in xrange(3):
tstart = time.time()
elapsed = time.time() - tstart
trainerr, trainloss = computeloss(Xtrain[:len(Xtest)], ytrain[:len(Xtest)])
testerr, testloss = computeloss(Xtest, ytest)
print fmt_row(10, [i_epoch, trainloss, trainerr, testloss, testerr, elapsed])
def make_prediction(self, max_label_length, ground_labels_basis_btc):
context_i_bf = parameter(init_array(IIDGaussian(0.1), (self.batch_size, self.feature_size)), name=None)
state_i_bf = parameter(init_array(IIDGaussian(0.1), (self.batch_size, self.decoder_size)), name=None)
char_list = []
for iter_step in range(0, max_label_length): #Is this right?
prev_out_bc = ground_labels_basis_btc[:, iter_step, :]
state_i_bf = self.get_decoder_state(context_i_bf, prev_out_bc, state_i_bf)
context_i_bf = self.get_context(state_i_bf)
this_character_dist = self.get_character_distribution(state_i_bf, context_i_bf)
char_list.append(cgt.argmax(this_character_dist, axis=1))
final = cgt.dimshuffle(cgt.stack(char_list), [1, 0])
return final
def main():
print("Loading data...")
X = cgt.matrix("X", fixed_shape=(None, 28*28))
y = cgt.vector("y", dtype='i8')
model = build_model(X, 0.0)
loss = -cgt.mean(categorical.loglik(y, model))
updates = nn.rmsprop(loss, nn.get_parameters(loss), 0.01)
train = cgt.function(inputs=[X, y], outputs=[], updates=updates)
y_nodrop = cgt.argmax(model, axis=1)
cost_nodrop = -cgt.mean(categorical.loglik(y, model))
err_nodrop = cgt.cast(cgt.not_equal(y_nodrop, y), cgt.floatX).mean()
computeloss = cgt.function(inputs=[X, y], outputs=[err_nodrop, cost_nodrop])
batch_size=128
Xdata, ydata = load_data()
Xtrain = Xdata[0:60000]
ytrain = ydata[0:60000]
Xtest = Xdata[60000:70000]
ytest = ydata[60000:70000]
sortinds = np.random.permutation(60000)
Xtrain = Xtrain[sortinds]
ytrain = ytrain[sortinds]
print fmt_row(10, ["Epoch","Train NLL","Train Err","Test NLL","Test Err","Epoch Time"])
for i_epoch in xrange(3):
tstart = time.time()
for start in xrange(0, Xtrain.shape[0], batch_size):
end = start+batch_size
train(Xtrain[start:end], ytrain[start:end])
elapsed = time.time() - tstart
trainerr, trainloss = computeloss(Xtrain[:len(Xtest)], ytrain[:len(Xtest)])
testerr, testloss = computeloss(Xtest, ytest)
print fmt_row(10, [i_epoch, trainloss, trainerr, testloss, testerr, elapsed])
nnbuilder.save_weights(model, 'mnist')
def main():
X = cgt.matrix(name='data', dtype=cgt.floatX, fixed_shape=(None, 2212))
y = cgt.vector("y", dtype='i8')
model = build_nn(X)
loss = -cgt.mean(categorical.loglik(y, model))
updates = nn.adagrad(loss, nn.get_parameters(loss), 0.01)
y_nodrop = cgt.argmax(model, axis=1)
cost_nodrop = -cgt.mean(categorical.loglik(y, model))
err_nodrop = cgt.cast(cgt.not_equal(y_nodrop, y), cgt.floatX).mean()
train = cgt.function(inputs=[X, y], outputs=[], updates=updates)
computeloss = cgt.function(inputs=[X, y], outputs=[err_nodrop, cost_nodrop])
batch_size = 20
Xdata, ydata = load_data()
Xtrain = Xdata[0:5200]
ytrain = ydata[0:5200]
Xtest = Xdata[5200:5573]
ytest = ydata[5200:5573]
sortinds = np.random.permutation(5200)
Xtrain = Xtrain[sortinds]
ytrain = ytrain[sortinds]
print fmt_row(10, ["Epoch","Train NLL","Train Err","Test NLL","Test Err","Epoch Time"])
for i_epoch in xrange(20):
tstart = time.time()
for start in xrange(0, Xtrain.shape[0], batch_size):
end = start+batch_size
train(Xtrain[start:end], ytrain[start:end])
elapsed = time.time() - tstart
trainerr, trainloss = computeloss(Xtrain[:len(Xtest)], ytrain[:len(Xtest)])
testerr, testloss = computeloss(Xtest, ytest)
print fmt_row(10, [i_epoch, trainloss, trainerr, testloss, testerr, elapsed])
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--epochs", type=int, default=10)
parser.add_argument("--profile", action="store_true")
parser.add_argument("--dropout", action="store_true")
parser.add_argument("--stepsize", type=float, default=.001)
parser.add_argument("--model", choices=["dense", "conv"], default="dense")
parser.add_argument("--unittest", action="store_true")
parser.add_argument("--grad_check", action="store_true")
args = parser.parse_args()
if args.grad_check: cgt.set_precision("quad")
# from mldata.org http://mldata.org/repository/data/viewslug/mnist-original/
# converted to npz
mnist = fetch_dataset("http://rll.berkeley.edu/cgt-data/mnist.npz")
Xdata = (mnist["X"] / 255.).astype(cgt.floatX)
ydata = mnist["y"]
np.random.seed(0)
if args.model == "conv":
Xdata = Xdata.reshape(-1, 1, 28, 28)
Xtrain = Xdata[0:60000]
ytrain = ydata[0:60000]
Xtest = Xdata[60000:70000]
ytest = ydata[60000:70000]
sortinds = np.random.permutation(60000)
Xtrain = Xtrain[sortinds]
ytrain = ytrain[sortinds]
X = cgt.tensor4("X",
fixed_shape=(None, 1, 28,
28)) if args.model == "conv" else cgt.matrix(
"X", fixed_shape=(None, 28 * 28))
y = cgt.vector("y", dtype='i8')
if args.model == "dense":
p_drop_input, p_drop_hidden = (0.2, 0.5) if args.dropout else (0, 0)
w_h = init_weights(784, 256)
w_h2 = init_weights(256, 256)
w_o = init_weights(256, 10)
pofy_drop = dense_model(X, w_h, w_h2, w_o, p_drop_input, p_drop_hidden)
pofy_nodrop = dense_model(X, w_h, w_h2, w_o, 0., 0.)
params = [w_h, w_h2, w_o]
elif args.model == "conv":
p_drop_conv, p_drop_hidden = (0.2, 0.5) if args.dropout else (0, 0)
w = init_weights(32, 1, 3, 3)
w2 = init_weights(64, 32, 3, 3)
w3 = init_weights(128, 64, 3, 3)
w4 = init_weights(128 * 2 * 2, 625)
w_o = init_weights(625, 10)
pofy_drop = convnet_model(X, w, w2, w3, w4, w_o, p_drop_conv,
p_drop_hidden)
pofy_nodrop = convnet_model(X, w, w2, w3, w4, w_o, 0., 0.)
params = [w, w2, w3, w4, w_o]
else:
raise RuntimeError("Unreachable")
cost_drop = -cgt.mean(categorical.loglik(y, pofy_drop))
updates = rmsprop_updates(cost_drop, params, stepsize=args.stepsize)
y_nodrop = cgt.argmax(pofy_nodrop, axis=1)
cost_nodrop = -cgt.mean(categorical.loglik(y, pofy_nodrop))
err_nodrop = cgt.cast(cgt.not_equal(y_nodrop, y), cgt.floatX).mean()
train = cgt.function(inputs=[X, y], outputs=[], updates=updates)
computeloss = cgt.function(inputs=[X, y],
outputs=[err_nodrop, cost_nodrop])
batch_size = 128
from cgt.tests import gradcheck_model
if args.grad_check:
cost_nodrop = cgt.core.clone(cost_nodrop, {
X: Xtrain[:1],
y: ytrain[:1]
})
print "doing gradient check..."
print "------------------------------------"
gradcheck_model(cost_nodrop, params[0:1])
print "success!"
return
if args.profile: cgt.profiler.start()
print fmt_row(10, [
"Epoch", "Train NLL", "Train Err", "Test NLL", "Test Err", "Epoch Time"
])
for i_epoch in xrange(args.epochs):
tstart = time.time()
for start in xrange(0, Xtrain.shape[0], batch_size):
end = start + batch_size
train(Xtrain[start:end], ytrain[start:end])
if args.unittest: return
elapsed = time.time() - tstart
trainerr, trainloss = computeloss(Xtrain[:len(Xtest)],
ytrain[:len(Xtest)])
testerr, testloss = computeloss(Xtest, ytest)
print fmt_row(
10, [i_epoch, trainloss, trainerr, testloss, testerr, elapsed])
if args.profile: cgt.execution.profiler.print_stats()
nn.SpatialConvolution(1, 32, kernelshape=(3,3), stride=(1,1), pad=(1,1), weight_init=nn.IIDGaussian(std=.1))(X)
)
pool1 = nn.max_pool_2d(conv1, kernelshape=(2,2), stride=(2,2))
conv2 = nn.rectify(
nn.SpatialConvolution(32, 32, kernelshape=(3,3), stride=(1,1), pad=(1,1), weight_init=nn.IIDGaussian(std=.1))(pool1)
)
pool2 = nn.max_pool_2d(conv2, kernelshape=(2,2), stride=(2,2))
d0, d1, d2, d3 = pool2.shape
flat = pool2.reshape([d0, d1*d2*d3])
nfeats = cgt.infer_shape(flat)[1]
probs = nn.softmax(nn.Affine(nfeats, 10)(flat))
cost = -categorical.loglik(y, probs).mean()
y_preds = cgt.argmax(probs, axis=1)
err = cgt.cast(cgt.not_equal(y, y_preds), cgt.floatX).mean()
params = nn.get_parameters(cost)
updates = nn.sgd(cost, params, 1e-3)
# training function
f = cgt.function(inputs=[X, y], outputs=[], updates=updates)
# compute the cost and error
cost_and_err = cgt.function(inputs=[X, y], outputs=[cost, err])
for i in xrange(epochs):
t0 = time.time()
for start in xrange(0, Xtrain.shape[0], batch_size):
end = batch_size + start
f(Xtrainimg[start:end], ytrain[start:end])
def main():
import argparse
parser=argparse.ArgumentParser()
parser.add_argument("--epochs",type=int,default=10)
parser.add_argument("--profile",action="store_true")
parser.add_argument("--dropout",action="store_true")
parser.add_argument("--stepsize",type=float, default=.001)
parser.add_argument("--model",choices=["dense","conv"],default="dense")
parser.add_argument("--unittest",action="store_true")
parser.add_argument("--grad_check",action="store_true")
parser.add_argument("--devtype",choices=["cpu","gpu"],default="cpu")
args = parser.parse_args()
if args.grad_check: cgt.set_precision("quad")
# from mldata.org http://mldata.org/repository/data/viewslug/mnist-original/
# converted to npz
mnist = fetch_dataset("http://rll.berkeley.edu/cgt-data/mnist.npz")
Xdata = (mnist["X"]/255.).astype(cgt.floatX)
ydata = mnist["y"]
np.random.seed(0)
cgt.update_config(default_device=cgt.core.Device(devtype=args.devtype), backend="native")
if args.model=="conv":
Xdata = Xdata.reshape(-1, 1, 28, 28)
Xtrain = Xdata[0:60000]
ytrain = ydata[0:60000]
Xtest = Xdata[60000:70000]
ytest = ydata[60000:70000]
sortinds = np.random.permutation(60000)
Xtrain = Xtrain[sortinds]
ytrain = ytrain[sortinds]
X = cgt.tensor4("X",fixed_shape=(None,1,28,28)) if args.model=="conv" else cgt.matrix("X", fixed_shape=(None,28*28))
y = cgt.vector("y",dtype='i8')
if args.model == "dense":
p_drop_input,p_drop_hidden = (0.2, 0.5) if args.dropout else (0,0)
w_h = init_weights(784, 256)
w_h2 = init_weights(256, 256)
w_o = init_weights(256, 10)
pofy_drop = dense_model(X, w_h, w_h2, w_o, p_drop_input, p_drop_hidden)
pofy_nodrop = dense_model(X, w_h, w_h2, w_o, 0., 0.)
params = [w_h, w_h2, w_o]
elif args.model == "conv":
p_drop_conv,p_drop_hidden = (0.2, 0.5) if args.dropout else (0,0)
w = init_weights(32, 1, 3, 3)
w2 = init_weights(64, 32, 3, 3)
w3 = init_weights(128, 64, 3, 3)
w4 = init_weights(128 * 2 * 2, 625)
w_o = init_weights(625, 10)
pofy_drop = convnet_model(X, w, w2, w3, w4, w_o, p_drop_conv, p_drop_hidden)
pofy_nodrop = convnet_model(X, w, w2, w3, w4, w_o, 0., 0.)
params = [w, w2, w3, w4, w_o]
else:
raise RuntimeError("Unreachable")
cost_drop = -cgt.mean(categorical.loglik(y, pofy_drop))
updates = rmsprop_updates(cost_drop, params, stepsize=args.stepsize)
y_nodrop = cgt.argmax(pofy_nodrop, axis=1)
cost_nodrop = -cgt.mean(categorical.loglik(y, pofy_nodrop))
err_nodrop = cgt.cast(cgt.not_equal(y_nodrop, y), cgt.floatX).mean()
train = cgt.function(inputs=[X, y], outputs=[], updates=updates)
computeloss = cgt.function(inputs=[X, y], outputs=[err_nodrop,cost_nodrop])
batch_size=128
from cgt.tests import gradcheck_model
if args.grad_check:
cost_nodrop = cgt.core.clone(cost_nodrop, {X:Xtrain[:1],y:ytrain[:1]})
print "doing gradient check..."
print "------------------------------------"
gradcheck_model(cost_nodrop, params[0:1])
print "success!"
return
if args.profile: cgt.profiler.start()
print fmt_row(10, ["Epoch","Train NLL","Train Err","Test NLL","Test Err","Epoch Time"])
for i_epoch in xrange(args.epochs):
tstart = time.time()
for start in xrange(0, Xtrain.shape[0], batch_size):
end = start+batch_size
train(Xtrain[start:end], ytrain[start:end])
if args.unittest: return
elapsed = time.time() - tstart
trainerr, trainloss = computeloss(Xtrain[:len(Xtest)], ytrain[:len(Xtest)])
testerr, testloss = computeloss(Xtest, ytest)
print fmt_row(10, [i_epoch, trainloss, trainerr, testloss, testerr, elapsed])
if args.profile: cgt.execution.profiler.print_stats()
def argmax(x, axis=None, keepdims=False):
return cgt.argmax(x, axis=axis, keepdims=keepdims)
def test_the_test_problem():
#Works
batch_size = 32 # How many samples do you want to batch.
feat_t_steps = 20 # How many 10ms sound clips.
feat_num_features = 10 # The dimension of the 10ms clips.
max_label_length = feat_t_steps # The maximal label length of the transcription. includes start character.
num_out_classes = 27
num_out_classes_true = num_out_classes + 2
num_batches = 756
num_epochs = 30
feats = cgt.tensor3(fixed_shape=(batch_size, feat_t_steps, feat_num_features))
ground_labels_basis = cgt.tensor3(fixed_shape=(batch_size, max_label_length, num_out_classes_true))
last_time = time.time()
print 'initializing temporal dense layer'
d1 = nnbuilder.temporalDenseLayer(feats, num_units=128, activation=cgt.sigmoid)
#d2 = nnbuilder.temporalDenseLayer(d1, num_units=128, activation=cgt.sigmoid)
d3 = nnbuilder.temporalDenseLayer(d1, num_units=num_out_classes_true, activation=nnbuilder.linear)
out = nn.three_d_softmax(d3, axis=2)
log_probs = None
for iter_step in range(0, max_label_length):
this_character_dist_bc = out[:, iter_step, :]
prev_out_bc = ground_labels_basis[:, iter_step, :]
log_probs_pre = prev_out_bc * this_character_dist_bc
log_probs_pre = cgt.log(cgt.sum(log_probs_pre, axis=1))
if log_probs is None:
log_probs = cgt.sum(log_probs_pre)
else:
log_probs += cgt.sum(log_probs_pre)
log_probs = -log_probs
print 'that took ' + str(time.time() - last_time) + ' seconds'
last_time = time.time()
print 'compiling objective function'
updates = nn.rmsprop(log_probs, nn.get_parameters(log_probs), learning_rate=0.01)
pred_train = cgt.function([feats, ground_labels_basis], [], updates=updates)
pred_fun = cgt.function([feats, ground_labels_basis], [log_probs])
most_likely_chars = cgt.argmax(out, axis=1)
actual_predictions = cgt.function([feats, ground_labels_basis], [most_likely_chars])
print 'that took ' + str(time.time() - last_time) + ' seconds'
test_data = np.load('test_data.npy')
test_labels = np.load('test_labels.npy')
data_mean = np.mean(test_data)
data_sd = np.mean(test_data)
print 'now training'
for one_epoch in range(0, num_epochs):
trained = 0
last_time = time.time()
print 'starting epoch ' + str(one_epoch)
for batch_iter in range(0, num_batches):
batch, labels_basis = normalize_batch_and_labels(test_data, batch_iter, feat_t_steps, data_mean, data_sd,
test_labels, num_out_classes_true)
pred_train(batch, labels_basis)
for batch_iter in range(0, num_batches):
batch, labels_basis = normalize_batch_and_labels(test_data, batch_iter, feat_t_steps, data_mean, data_sd,
test_labels, num_out_classes_true)
trained += pred_fun(batch, labels_basis)[0]
trained = trained/batch_iter
print 'train loss is ' + str(trained)
print 'that took ' + str(time.time() - last_time) + ' seconds'
act_pred = actual_predictions(batch, labels_basis)[0]
print 'an actual prediction is '
print act_pred
def __init__(self,
model="dense",
im_size=[28, 28],
dropout=True,
devtype="cpu",
grad_check=True,
reg=0):
if grad_check: cgt.set_precision("quad")
self.model = model
self.reg = reg
np.random.seed(0)
cgt.update_config(default_device=cgt.core.Device(devtype=devtype),
backend="native")
print(model)
# MLP with 1 hidden layer
if model == "dense1":
self.Xsize = 2 * im_size[0] * im_size[1] + im_size[0] + im_size[1]
self.X = cgt.matrix("X", fixed_shape=(None, self.Xsize))
self.y = cgt.vector("y", dtype='i8')
self.p_drop_input, self.p_drop_hidden = (0.2,
0.5) if dropout else (0,
0)
self.w_h = init_weights(self.Xsize, 256)
self.w_o = init_weights(256, 8)
self.pofy_drop = dense_model1(self.X, self.w_h, self.w_o,
self.p_drop_input,
self.p_drop_hidden)
self.pofy_nodrop = dense_model1(self.X, self.w_h, self.w_o, 0., 0.)
self.params = [self.w_h, self.w_o]
self.l1 = cgt.abs(self.w_h).sum() + cgt.abs(self.w_o).sum()
self.cost_drop = -cgt.mean(
categorical.loglik(self.y,
self.pofy_drop)) + self.reg * self.l1
# MLP with 2 hidden layers
elif model == "dense2":
self.Xsize = 2 * im_size[0] * im_size[1] + im_size[0] + im_size[1]
self.X = cgt.matrix("X", fixed_shape=(None, self.Xsize))
self.y = cgt.vector("y", dtype='i8')
self.p_drop_input, self.p_drop_hidden = (0.2,
0.5) if dropout else (0,
0)
self.w_h = init_weights(self.Xsize, 256)
self.w_h2 = init_weights(256, 256)
self.w_o = init_weights(256, 8)
self.pofy_drop = dense_model2(self.X, self.w_h, self.w_h2,
self.w_o, self.p_drop_input,
self.p_drop_hidden)
self.pofy_nodrop = dense_model2(self.X, self.w_h, self.w_h2,
self.w_o, 0., 0.)
self.params = [self.w_h, self.w_h2, self.w_o]
self.l1 = cgt.abs(self.w_h).sum() + cgt.abs(
self.w_h2).sum() + cgt.abs(self.w_o).sum()
self.cost_drop = -cgt.mean(
categorical.loglik(self.y,
self.pofy_drop)) + self.reg * self.l1
# MLP with 3 hidden layers
elif model == "dense3":
self.Xsize = 2 * im_size[0] * im_size[1] + im_size[0] + im_size[1]
self.X = cgt.matrix("X", fixed_shape=(None, self.Xsize))
self.y = cgt.vector("y", dtype='i8')
self.p_drop_input, self.p_drop_hidden = (
0.0, [0.5, 0.5, 0.5]) if dropout else (0, [0, 0, 0])
self.w_h = init_weights(self.Xsize, 256)
self.w_h2 = init_weights(256, 256)
self.w_h3 = init_weights(256, 256)
self.w_o = init_weights(256, 8)
self.pofy_drop = dense_model3(self.X, self.w_h, self.w_h2,
self.w_h3, self.w_o,
self.p_drop_input,
self.p_drop_hidden)
self.pofy_nodrop = dense_model3(self.X, self.w_h, self.w_h2,
self.w_h3, self.w_o, 0.,
[0., 0., 0.])
self.params = [self.w_h, self.w_h2, self.w_h3, self.w_o]
self.l1 = cgt.abs(self.w_h).sum() + cgt.abs(self.w_h2).sum() + cgt.abs(self.w_h3).sum() + \
cgt.abs(self.w_o).sum()
self.cost_drop = -cgt.mean(
categorical.loglik(self.y,
self.pofy_drop)) + self.reg * self.l1
else:
raise RuntimeError("Unknown Model")
self.y_nodrop = cgt.argmax(self.pofy_nodrop, axis=1)
self.cost_nodrop = -cgt.mean(
categorical.loglik(self.y, self.pofy_nodrop))
self.err_nodrop = cgt.cast(cgt.not_equal(self.y_nodrop, self.y),
cgt.floatX).mean()
self.computeloss = cgt.function(
inputs=[self.X, self.y],
outputs=[self.err_nodrop, self.cost_nodrop])
self.y_out = cgt.function(inputs=[self.X], outputs=[self.y_nodrop])
self.updates = rmsprop_updates(self.cost_drop, self.params)
self.train = cgt.function(inputs=[self.X, self.y],
outputs=[],
updates=self.updates)
def argmax(x, axis=None, keepdims=False):
return cgt.argmax(x, axis=axis, keepdims=keepdims)
