for t in (gamma, beta, mu, variance))
else:
gamma, beta, mu, variance = (t.dimshuffle(0, 'x', 'x', 'x')
for t in (gamma, beta, mu, variance))
w_next = w * gamma / (T.sqrt(variance + epsilon))
w_next = T.clip(w_next, -limits[i], limits[i])
plain_params.append([w_next, b_next])
plain_params.append(params[8])
return plain_params
plain_params = remove_batchnorm(params)
z = feedForward(x, plain_params)
y = T.argmax(z, axis=1)
# compile theano functions
predict = theano.function([x], y)
batch_size = 250
# test
running_accuracy = 0.0
batches = 0
for start in range(0, 10000, batch_size):
x_batch = x_test[start:start + batch_size]
t_batch = labels[start:start + batch_size]
running_accuracy += np.mean(predict(x_batch) == t_batch)
batches += 1
test_accuracy = running_accuracy / batches
print('test accuracy = ' + repr(test_accuracy))
layers.saveParams('normalized_weights.save', plain_params)
def dropout_removal(params):
updates = []
#For FC layers, I do not compensate as it turns out it is better this way
updates.append((params[1][0],0.8*params[1][0]))
updates.append((params[3][0],0.7*params[3][0]))
updates.append((params[5][0],0.7*params[5][0]))
return updates
z, _ = feedForward(x, params)
y = T.argmax(z, axis=1)
dropout_updates = dropout_removal(params)
# compile theano functions
predict = theano.function([x], y)
remove_it = theano.function([],[],updates=dropout_updates)
batch_size = 250
# test
remove_it()
running_accuracy =0.0
batches = 0
for start in range(0,10000,batch_size):
x_batch = x_test[start:start+batch_size]
t_batch = labels[start:start+batch_size]
running_accuracy += np.mean(predict(x_batch) == t_batch)
batches+=1
test_accuracy = running_accuracy/batches
print('test accuracy = ' + repr(test_accuracy))
layers.saveParams('dropout_removed_88p12.save',params)
print("Params have been saved.")
#random crop
padded = np.pad(x_batch, ((0, 0), (0, 0), (4, 4), (4, 4)),
mode='constant')
random_cropped = np.zeros(x_batch.shape, dtype=np.float32)
crops = np.random.random_integers(0, high=8, size=(batch_size, 2))
for r in range(batch_size):
random_cropped[r, :, :, :] = padded[r, :,
crops[r, 0]:(crops[r, 0] + 32),
crops[r, 1]:(crops[r, 1] + 32)]
#train
cost = train(random_cropped, t_batch, lr, 1)
running_cost = running_cost + cost
batches = batches + 1
total_loss = running_cost / batches
# test
running_accuracy = 0.0
batches = 0
for start in range(0, 10000, batch_size):
x_batch = x_test[start:start + batch_size]
t_batch = labels[start:start + batch_size]
running_accuracy += np.mean(predict(x_batch, 0) == t_batch)
batches += 1
test_accuracy = running_accuracy / batches
print('loss = ' + repr(total_loss))
print('test accuracy = ' + repr(test_accuracy))
layers.saveParams('vgg_from_scratch_new_gpu.save', params)
print('Params have been saved')
if ((i + 1) % 100 == 0):
lr = 0.1
#train
indices = np.random.permutation(60000)
running_cost = 0.0
batches = 0
for start in range(0, 60000, batch_size):
x_batch = x_train[indices[start:start + batch_size]]
t_batch = t_train[indices[start:start + batch_size]]
cost = train(x_batch, t_batch, lr, 1)
running_cost = running_cost + cost
batches = batches + 1
total_loss = running_cost / batches
# test
labels = np.argmax(t_test, axis=1)
running_accuracy = 0.0
batches = 0
for start in range(0, 10000, batch_size):
x_batch = x_test[start:start + batch_size]
t_batch = labels[start:start + batch_size]
running_accuracy += np.mean(predict(x_batch, 0) == t_batch)
batches += 1
test_accuracy = running_accuracy / batches
print('loss = ' + repr(total_loss))
print('test accuracy = ' + repr(test_accuracy))
layers.saveParams('mnist_pretrained.save', params)
print('Params have been saved')
running_cost = running_cost + cost
batches = batches + 1
total_loss = running_cost / batches
# test
running_accuracy = 0.0
batches = 0
for start in range(0, 10000, batch_size):
x_batch = x_test[start:start + batch_size]
t_batch = labels[start:start + batch_size]
running_accuracy += np.mean(predict(x_batch, 0) == t_batch)
batches += 1
test_accuracy = running_accuracy / batches
accFile.write(
np.array_str(total_loss) + ' ' + np.array_str(test_accuracy) + '\n')
for res in runningGradientStats:
for stat in res:
gradStatFile.write(
np.array_str(stat[0].get_value()) + ' ' +
np.array_str(stat[1].get_value()) + '\n')
for res in runningActGrad:
for stat in res:
gradStatFile.write(
np.array_str(stat[0].get_value()) + ' ' +
np.array_str(stat[1].get_value()) + '\n')
gradStatFile.write('\n')
print(np.array_str(total_loss) + ' ' + np.array_str(test_accuracy))
layers.saveParams('pretrained_params.save', params)
print("Params have been saved")