for t in (gamma, beta, mu, variance)) else: gamma, beta, mu, variance = (t.dimshuffle(0, 'x', 'x', 'x') for t in (gamma, beta, mu, variance)) w_next = w * gamma / (T.sqrt(variance + epsilon)) w_next = T.clip(w_next, -limits[i], limits[i]) plain_params.append([w_next, b_next]) plain_params.append(params[8]) return plain_params plain_params = remove_batchnorm(params) z = feedForward(x, plain_params) y = T.argmax(z, axis=1) # compile theano functions predict = theano.function([x], y) batch_size = 250 # test running_accuracy = 0.0 batches = 0 for start in range(0, 10000, batch_size): x_batch = x_test[start:start + batch_size] t_batch = labels[start:start + batch_size] running_accuracy += np.mean(predict(x_batch) == t_batch) batches += 1 test_accuracy = running_accuracy / batches print('test accuracy = ' + repr(test_accuracy)) layers.saveParams('normalized_weights.save', plain_params)
def dropout_removal(params): updates = [] #For FC layers, I do not compensate as it turns out it is better this way updates.append((params[1][0],0.8*params[1][0])) updates.append((params[3][0],0.7*params[3][0])) updates.append((params[5][0],0.7*params[5][0])) return updates z, _ = feedForward(x, params) y = T.argmax(z, axis=1) dropout_updates = dropout_removal(params) # compile theano functions predict = theano.function([x], y) remove_it = theano.function([],[],updates=dropout_updates) batch_size = 250 # test remove_it() running_accuracy =0.0 batches = 0 for start in range(0,10000,batch_size): x_batch = x_test[start:start+batch_size] t_batch = labels[start:start+batch_size] running_accuracy += np.mean(predict(x_batch) == t_batch) batches+=1 test_accuracy = running_accuracy/batches print('test accuracy = ' + repr(test_accuracy)) layers.saveParams('dropout_removed_88p12.save',params) print("Params have been saved.")
#random crop padded = np.pad(x_batch, ((0, 0), (0, 0), (4, 4), (4, 4)), mode='constant') random_cropped = np.zeros(x_batch.shape, dtype=np.float32) crops = np.random.random_integers(0, high=8, size=(batch_size, 2)) for r in range(batch_size): random_cropped[r, :, :, :] = padded[r, :, crops[r, 0]:(crops[r, 0] + 32), crops[r, 1]:(crops[r, 1] + 32)] #train cost = train(random_cropped, t_batch, lr, 1) running_cost = running_cost + cost batches = batches + 1 total_loss = running_cost / batches # test running_accuracy = 0.0 batches = 0 for start in range(0, 10000, batch_size): x_batch = x_test[start:start + batch_size] t_batch = labels[start:start + batch_size] running_accuracy += np.mean(predict(x_batch, 0) == t_batch) batches += 1 test_accuracy = running_accuracy / batches print('loss = ' + repr(total_loss)) print('test accuracy = ' + repr(test_accuracy)) layers.saveParams('vgg_from_scratch_new_gpu.save', params) print('Params have been saved')
if ((i + 1) % 100 == 0): lr = 0.1 #train indices = np.random.permutation(60000) running_cost = 0.0 batches = 0 for start in range(0, 60000, batch_size): x_batch = x_train[indices[start:start + batch_size]] t_batch = t_train[indices[start:start + batch_size]] cost = train(x_batch, t_batch, lr, 1) running_cost = running_cost + cost batches = batches + 1 total_loss = running_cost / batches # test labels = np.argmax(t_test, axis=1) running_accuracy = 0.0 batches = 0 for start in range(0, 10000, batch_size): x_batch = x_test[start:start + batch_size] t_batch = labels[start:start + batch_size] running_accuracy += np.mean(predict(x_batch, 0) == t_batch) batches += 1 test_accuracy = running_accuracy / batches print('loss = ' + repr(total_loss)) print('test accuracy = ' + repr(test_accuracy)) layers.saveParams('mnist_pretrained.save', params) print('Params have been saved')
running_cost = running_cost + cost batches = batches + 1 total_loss = running_cost / batches # test running_accuracy = 0.0 batches = 0 for start in range(0, 10000, batch_size): x_batch = x_test[start:start + batch_size] t_batch = labels[start:start + batch_size] running_accuracy += np.mean(predict(x_batch, 0) == t_batch) batches += 1 test_accuracy = running_accuracy / batches accFile.write( np.array_str(total_loss) + ' ' + np.array_str(test_accuracy) + '\n') for res in runningGradientStats: for stat in res: gradStatFile.write( np.array_str(stat[0].get_value()) + ' ' + np.array_str(stat[1].get_value()) + '\n') for res in runningActGrad: for stat in res: gradStatFile.write( np.array_str(stat[0].get_value()) + ' ' + np.array_str(stat[1].get_value()) + '\n') gradStatFile.write('\n') print(np.array_str(total_loss) + ' ' + np.array_str(test_accuracy)) layers.saveParams('pretrained_params.save', params) print("Params have been saved")