def train(dataset_x,
dataset_y,
model,
optimiser,
criterion,
epoch,
batch_size,
mode='train'):
progress = make_progressbar('Training ({}) epoch #{}'.format(mode, epoch),
len(dataset_x))
progress.start()
shuffle = np.random.permutation(len(dataset_x))
for j in range(dataset_x.shape[0] // batch_size):
indices = shuffle[j * batch_size:(j + 1) * batch_size]
mini_batch_input = dataset_x[indices].astype(df.floatX)
mini_batch_targets = dataset_y[indices].astype(df.floatX)
if mode == 'train':
model.zero_grad_parameters()
model.accumulate_gradients(mini_batch_input, mini_batch_targets,
criterion)
optimiser.update_parameters(model)
elif mode == 'stats':
model.accumulate_statistics(mini_batch_input)
else:
assert False, "Mode should be either 'train' or 'stats'"
progress.update(j * batch_size + len(mini_batch_input))
progress.finish()
def train(Xtrain, ytrain, model, optimiser, criterion, epoch, batch_size, mode='train'):
progress = make_progressbar('Training ({}) epoch #{}'.format(mode, epoch), len(Xtrain))
progress.start()
shuffle = np.random.permutation(len(Xtrain))
for ibatch in range(len(Xtrain) // 2 // batch_size):
indices = shuffle[ibatch*batch_size*2 : (ibatch+1)*batch_size*2]
Xleft = Xtrain[indices[:batch_size]].astype(df.floatX)
yleft = ytrain[indices[:batch_size]].astype(df.floatX)
Xright = Xtrain[indices[batch_size:]].astype(df.floatX)
yright = ytrain[indices[batch_size:]].astype(df.floatX)
# Need to put the targets into a column because of the way BCE works.
y = (yleft == yright)[:,None].astype(df.floatX)
if mode == 'train':
model.zero_grad_parameters()
model.accumulate_gradients((Xleft, Xright), y, criterion)
optimiser.update_parameters(model)
elif mode == 'stats':
model.accumulate_statistics((Xleft, Xright))
else:
assert False, "Mode should be either 'train' or 'stats'"
progress.update(ibatch*batch_size*2 + len(y))
progress.finish()
def validate(Xdata, ydata, model, epoch, batch_size):
progress = make_progressbar('Testing epoch #{}'.format(epoch), len(Xdata))
progress.start()
nerrors = 0
ntrials = 0
# In theory, we should check all pairs.
# But for MNIST, that'd be roughly 10k**2/2 ~ 50M
# which is too much for this example.
# So instead, we do all pairs in each batch, which for a batch of 100
# is ~ 100**2/2 * 10k/100 ~ 500k
for ibatch in range((len(Xdata) + batch_size - 1) // batch_size):
# Note: numpy correctly handles the size of the last minibatch.
batch_indices = np.arange(ibatch * batch_size,
min((ibatch + 1) * batch_size, len(Xdata)))
batchleft, batchright = map(list,
zip(*combinations(batch_indices, r=2)))
# But then, for each batch we've got 100**2/2 ~ 5k pairs,
# thus we need to go through them in batch-mode again (cuz GPU memory)
for i in range((len(batchleft) + batch_size - 1) // batch_size):
Xleft = Xdata[batchleft[i * batch_size:(i + 1) *
batch_size]].astype(df.floatX)
yleft = ydata[batchleft[i * batch_size:(i + 1) *
batch_size]].astype(df.floatX)
Xright = Xdata[batchright[i * batch_size:(i + 1) *
batch_size]].astype(df.floatX)
yright = ydata[batchright[i * batch_size:(i + 1) *
batch_size]].astype(df.floatX)
preds = 0.5 < np.squeeze(model.forward((Xleft, Xright)))
nerrors += sum(preds != (yleft == yright))
ntrials += len(preds)
progress.update(ibatch * batch_size + len(batch_indices))
progress.finish()
accuracy = 1 - float(nerrors) / ntrials
print(
"Epoch #{}, Classification accuracy: {:.2%} ({} errors out of {} tests)"
.format(epoch, accuracy, nerrors, ntrials))
def validate(dataset_x, dataset_y, model, epoch, batch_size):
progress = make_progressbar('Testing epoch #{}'.format(epoch), len(dataset_x))
progress.start()
logloss = 0.
for j in range((dataset_x.shape[0] + batch_size - 1) // batch_size):
# Note: numpy correctly handles the size of the last minibatch.
mini_batch_input = dataset_x[j*batch_size : (j+1)*batch_size].astype(th.config.floatX)
mini_batch_targets = dataset_y[j*batch_size : (j+1)*batch_size].astype(th.config.floatX)
mini_batch_prediction = model.forward(mini_batch_input)
logloss += multiclass_log_loss(mini_batch_targets, mini_batch_prediction, normalize=False)
progress.update(j * batch_size + len(mini_batch_input))
progress.finish()
print("Epoch #{}, Logloss: {:.5f}".format(epoch, logloss/dataset_x.shape[0]))
def run(optim):
progress = make_progressbar('Training with ' + str(optim), 5)
progress.start()
model = net()
model.training()
for epoch in range(5):
train(Xtrain, ytrain, model, optim, criterion, batch_size, 'train')
train(Xtrain, ytrain, model, optim, criterion, batch_size, 'stats')
progress.update(epoch + 1)
progress.finish()
model.evaluate()
nll, _ = test(Xtrain, ytrain, model, batch_size)
_, nerr = test(Xval, yval, model, batch_size)
print("Trainset NLL: {:.2f}".format(nll))
print("Testset errors: {}".format(nerr))
def validate(dataset_x, dataset_y, model, epoch, batch_size):
progress = make_progressbar('Testing epoch #{}'.format(epoch), len(dataset_x))
progress.start()
nerrors = 0
for j in range((dataset_x.shape[0] + batch_size - 1) // batch_size):
# Note: numpy correctly handles the size of the last minibatch.
mini_batch_input = dataset_x[j*batch_size : (j+1)*batch_size].astype(df.floatX)
mini_batch_targets = dataset_y[j*batch_size : (j+1)*batch_size]
mini_batch_prediction = np.argmax(model.forward(mini_batch_input), axis=1)
nerrors += sum(mini_batch_targets != mini_batch_prediction)
progress.update(j*batch_size + len(mini_batch_input))
progress.finish()
accuracy = 1 - float(nerrors)/dataset_x.shape[0]
print("Epoch #{}, Classification accuracy: {:.2%} ({} errors)".format(epoch, accuracy, nerrors))
def run(optim):
progress = make_progressbar("Training with " + str(optim), 5)
progress.start()
model = net()
model.training()
for epoch in range(5):
train(Xtrain, ytrain, model, optim, criterion, batch_size, "train")
train(Xtrain, ytrain, model, optim, criterion, batch_size, "stats")
progress.update(epoch + 1)
progress.finish()
model.evaluate()
nll, _ = test(Xtrain, ytrain, model, batch_size)
_, nerr = test(Xval, yval, model, batch_size)
print("Trainset NLL: {:.2f}".format(nll))
print("Testset errors: {}".format(nerr))
def validate(dataset_x, dataset_y, model, epoch, batch_size):
progress = make_progressbar('Testing epoch #{}'.format(epoch), len(dataset_x))
progress.start()
nerrors = 0
for j in range((dataset_x.shape[0] + batch_size - 1) // batch_size):
# Note: numpy correctly handles the size of the last minibatch.
mini_batch_input = dataset_x[j*batch_size : (j+1)*batch_size].astype(th.config.floatX)
mini_batch_targets = dataset_y[j*batch_size : (j+1)*batch_size].astype(th.config.floatX)
mini_batch_prediction = np.argmax(model.forward(mini_batch_input), axis=1)
nerrors += sum(mini_batch_targets != mini_batch_prediction)
progress.update(j * batch_size)
progress.finish()
accuracy = 1 - float(nerrors)/dataset_x.shape[0]
print("Epoch #{}, Classification accuracy: {:.2%} ({} errors)".format(epoch, accuracy, nerrors))
def validate(Xdata, ydata, model, epoch, batch_size):
progress = make_progressbar('Testing epoch #{}'.format(epoch), len(Xdata))
progress.start()
nerrors = 0
ntrials = 0
# In theory, we should check all pairs.
# But for MNIST, that'd be roughly 10k**2/2 ~ 50M
# which is too much for this example.
# So instead, we do all pairs in each batch, which for a batch of 100
# is ~ 100**2/2 * 10k/100 ~ 500k
for ibatch in range((len(Xdata) + batch_size - 1) // batch_size):
# Note: numpy correctly handles the size of the last minibatch.
batch_indices = np.arange(ibatch*batch_size, min((ibatch+1)*batch_size, len(Xdata)))
batchleft, batchright = map(list, zip(*combinations(batch_indices, r=2)))
# But then, for each batch we've got 100**2/2 ~ 5k pairs,
# thus we need to go through them in batch-mode again (cuz GPU memory)
for i in range((len(batchleft) + batch_size - 1) // batch_size):
Xleft = Xdata[batchleft [i*batch_size : (i+1)*batch_size]].astype(df.floatX)
yleft = ydata[batchleft [i*batch_size : (i+1)*batch_size]].astype(df.floatX)
Xright = Xdata[batchright[i*batch_size : (i+1)*batch_size]].astype(df.floatX)
yright = ydata[batchright[i*batch_size : (i+1)*batch_size]].astype(df.floatX)
preds = 0.5 < np.squeeze(model.forward((Xleft, Xright)))
nerrors += sum(preds != (yleft == yright))
ntrials += len(preds)
progress.update(ibatch*batch_size + len(batch_indices))
progress.finish()
accuracy = 1 - float(nerrors)/ntrials
print("Epoch #{}, Classification accuracy: {:.2%} ({} errors out of {} tests)".format(epoch, accuracy, nerrors, ntrials))
def validate(X, y_c, y_f, model, epoch, batch_size):
progress = make_progressbar('Testing epoch #{}'.format(epoch), len(X))
progress.start()
nerrors_c, nerrors_f = 0, 0
for ibatch in range((len(X) + batch_size - 1) // batch_size):
# Note: numpy correctly handles the size of the last minibatch.
Xbatch = X[ibatch*batch_size : (ibatch+1)*batch_size].astype(df.floatX)
ybatch_c = y_c[ibatch*batch_size : (ibatch+1)*batch_size].astype(df.floatX)
ybatch_f = y_f[ibatch*batch_size : (ibatch+1)*batch_size].astype(df.floatX)
(preds_c, preds_f) = map(lambda py: np.argmax(py, axis=1), model.forward(Xbatch))
nerrors_c += sum(ybatch_c != preds_c)
nerrors_f += sum(ybatch_f != preds_f)
progress.update(ibatch*batch_size + len(Xbatch))
progress.finish()
accuracy_c = 1 - float(nerrors_c)/len(X)
accuracy_f = 1 - float(nerrors_f)/len(X)
print("Epoch #{}, Coarse accuracy: {:.2%} ({} errors)".format(epoch, accuracy_c, nerrors_c))
print("Epoch #{}, Fine accuracy: {:.2%} ({} errors)".format(epoch, accuracy_f, nerrors_f))
def train(dataset_x, dataset_y, model, optimiser, criterion, epoch, batch_size, mode='train'):
progress = make_progressbar('Training ({}) epoch #{}'.format(mode, epoch), len(dataset_x))
progress.start()
shuffle = np.random.permutation(len(dataset_x))
for j in range(dataset_x.shape[0] // batch_size):
indices = shuffle[j*batch_size : (j+1)*batch_size]
mini_batch_input = dataset_x[indices].astype(df.floatX)
mini_batch_targets = dataset_y[indices].astype(df.floatX)
if mode == 'train':
model.zero_grad_parameters()
model.accumulate_gradients(mini_batch_input, mini_batch_targets, criterion)
optimiser.update_parameters(model)
elif mode == 'stats':
model.accumulate_statistics(mini_batch_input)
else:
assert False, "Mode should be either 'train' or 'stats'"
progress.update(j*batch_size + len(mini_batch_input))
progress.finish()
def train(X, y_c, y_f, model, optim, crit, epoch, batch_size, mode='train'):
progress = make_progressbar('Training ({}) epoch #{}'.format(mode, epoch), len(X))
progress.start()
shuffle = np.random.permutation(len(X))
for ibatch in range(len(X) // batch_size):
indices = shuffle[ibatch*batch_size : (ibatch+1)*batch_size]
Xbatch = X[indices].astype(df.floatX)
ybatch_c = y_c[indices].astype(df.floatX)
ybatch_f = y_f[indices].astype(df.floatX)
if mode == 'train':
model.zero_grad_parameters()
model.accumulate_gradients(Xbatch, (ybatch_c, ybatch_f), crit)
optim.update_parameters(model)
elif mode == 'stats':
model.accumulate_statistics(Xbatch)
else:
assert False, "Mode should be either 'train' or 'stats'"
progress.update(ibatch*batch_size + len(Xbatch))
progress.finish()
