train_x = train_x.reshape((-1, 1, 28, 28))
valid_x = valid_x.reshape((-1, 1, 28, 28))
test_x = test_x.reshape((-1, 1, 28, 28))
elif dataset == 'cifar10':
filename = 'cifar10.pkl'
train_x, train_y, test_x, test_y = load_cifar10(filename)
train_x = train_x.reshape((-1, 3, 32, 32))
test_x = test_x.reshape((-1, 3, 32, 32))
valid_x = test_x.copy()
valid_y = test_y.copy()
if model == 'hyperCNN':
model = HyperCNN(lbda=lbda,
perdatapoint=perdatapoint,
prior=prior,
coupling=coupling,
dataset=dataset,
opt=opt)
elif model == 'CNN':
model = MCdropoutCNN(dataset=dataset, opt=opt)
elif model == 'CNN_spatial_dropout':
model = MCdropoutCNN(dropout='spatial', dataset=dataset, opt=opt)
elif model == 'CNN_dropout':
model = MCdropoutCNN(dropout=1, dataset=dataset, opt=opt)
else:
raise Exception('no model named `{}`'.format(model))
recs = train_model(model, train_x[:size], train_y[:size], valid_x, valid_y,
lr0, lrdecay, bs, epochs)
from helpers import plot_dict
def active_learning(acquisition_iterations):
bh_iterations = 100
nb_classes = 10
Queries = 10
all_accuracy = 0
acquisition_iterations = 98
filename = '../../mnist.pkl.gz'
train_x, train_y, valid_x, valid_y, test_x, test_y = load_mnist(filename)
train_x = train_x.reshape(50000, 1, 28, 28)
valid_x = valid_x.reshape(10000, 1, 28, 28)
test_x = test_x.reshape(10000, 1, 28, 28)
train_x, train_y, pool_x, pool_y = split_train_pool_data(train_x, train_y)
train_y_multiclass = train_y.argmax(1)
train_x, train_y = get_initial_training_data(train_x, train_y_multiclass)
print("Initial Training Data", train_x.shape)
model = HyperCNN(lbda=lbda,
perdatapoint=perdatapoint,
prior=prior,
kernel_width=4,
pad='valid',
stride=1,
coupling=coupling)
recs = train_model(model.train_func, model.predict, train_x[:size],
train_y[:size], valid_x, valid_y, lr0, lrdecay, bs,
epochs)
test_accuracy = test_model(model.predict_proba, test_x, test_y)
print("Test Accuracy", test_accuracy)
all_accuracy = test_accuracy
for i in range(acquisition_iterations):
print('POOLING ITERATION', i)
pool_subset = 2000
pool_subset_dropout = np.asarray(
random.sample(range(0, pool_x.shape[0]), pool_subset))
X_pool_Dropout = pool_x[pool_subset_dropout, :, :, :]
y_pool_Dropout = pool_y[pool_subset_dropout]
score_All = np.zeros(shape=(X_pool_Dropout.shape[0], nb_classes))
for d in range(bh_iterations):
bh_score = model.predict_proba(X_pool_Dropout)
score_All = score_All + bh_score
Avg_Pi = np.divide(score_All, bh_iterations)
Log_Avg_Pi = np.log2(Avg_Pi)
Entropy_Avg_Pi = -np.multiply(Avg_Pi, Log_Avg_Pi)
Entropy_Average_Pi = np.sum(Entropy_Avg_Pi, axis=1)
U_X = Entropy_Average_Pi
sort_values = U_X.flatten()
x_pool_index = sort_values.argsort()[-Queries:][::-1]
Pooled_X = X_pool_Dropout[x_pool_index, :, :, :]
Pooled_Y = y_pool_Dropout[x_pool_index]
delete_Pool_X = np.delete(pool_x, (pool_subset_dropout), axis=0)
delete_Pool_Y = np.delete(pool_y, (pool_subset_dropout), axis=0)
delete_Pool_X_Dropout = np.delete(X_pool_Dropout, (x_pool_index),
axis=0)
delete_Pool_Y_Dropout = np.delete(y_pool_Dropout, (x_pool_index),
axis=0)
pool_x = np.concatenate((pool_x, X_pool_Dropout), axis=0)
pool_y = np.concatenate((pool_y, y_pool_Dropout), axis=0)
train_x = np.concatenate((train_x, Pooled_X), axis=0)
train_y = np.concatenate((train_y, Pooled_Y), axis=0)
print("Training Set Size", train_x.shape)
if 0: # don't warm start
model = HyperCNN(lbda=lbda,
perdatapoint=perdatapoint,
prior=prior,
kernel_width=4,
pad='valid',
stride=1,
coupling=coupling)
recs = train_model(model.train_func, model.predict, train_x[:size],
train_y[:size], valid_x, valid_y, lr0, lrdecay, bs,
epochs)
test_accuracy = test_model(model.predict_proba, test_x, test_y)
print("Test Accuracy", test_accuracy)
all_accuracy = np.append(all_accuracy, test_accuracy)
return all_accuracy
def active_learning(acquisition_iterations):
bh_iterations = 100
nb_classes = 10
Queries = 10
all_accuracy = 0
acquisition_iterations = 98
filename = '../../mnist.pkl.gz'
train_x, train_y, valid_x, valid_y, test_x, test_y = load_mnist(filename)
train_x = train_x.reshape(50000, 1, 28, 28)
valid_x = valid_x.reshape(10000, 1, 28, 28)
test_x = test_x.reshape(10000, 1, 28, 28)
train_x, train_y, pool_x, pool_y = split_train_pool_data(train_x, train_y)
train_y_multiclass = train_y.argmax(1)
train_x, train_y = get_initial_training_data(train_x, train_y_multiclass)
print("Initial Training Data", train_x.shape)
model = HyperCNN(lbda=lbda,
perdatapoint=perdatapoint,
prior=prior,
kernel_width=4,
pad='valid',
stride=1,
coupling=coupling)
recs = train_model(model.train_func, model.predict, train_x[:size],
train_y[:size], valid_x, valid_y, lr0, lrdecay, bs,
epochs)
test_accuracy = test_model(model.predict_proba, test_x, test_y)
all_accuracy = test_accuracy
print("Training Set Size", train_x.shape)
print("Test Accuracy", test_accuracy)
for i in range(acquisition_iterations):
print('POOLING ITERATION', i)
pool_subset = 2000
pool_subset_dropout = np.asarray(
random.sample(range(0, pool_x.shape[0]), pool_subset))
X_pool_Dropout = pool_x[pool_subset_dropout, :, :, :]
y_pool_Dropout = pool_y[pool_subset_dropout]
All_BH_Classes = np.zeros(shape=(X_pool_Dropout.shape[0], 1))
for d in range(bh_iterations):
bh_score = model.predict(X_pool_Dropout)
bh_score = np.array([bh_score]).T
All_BH_Classes = np.append(All_BH_Classes, bh_score, axis=1)
Variation = np.zeros(shape=(X_pool_Dropout.shape[0]))
for t in range(X_pool_Dropout.shape[0]):
L = np.array([0])
for d_iter in range(bh_iterations):
L = np.append(L, All_BH_Classes[t, d_iter + 1])
Predicted_Class, Mode = mode(L[1:])
v = np.array([1 - Mode / float(bh_iterations)])
Variation[t] = v
sort_values = Variation.flatten()
x_pool_index = sort_values.argsort()[-Queries:][::-1]
Pooled_X = X_pool_Dropout[x_pool_index, :, :, :]
Pooled_Y = y_pool_Dropout[x_pool_index]
delete_Pool_X = np.delete(pool_x, (pool_subset_dropout), axis=0)
delete_Pool_Y = np.delete(pool_y, (pool_subset_dropout), axis=0)
delete_Pool_X_Dropout = np.delete(X_pool_Dropout, (x_pool_index),
axis=0)
delete_Pool_Y_Dropout = np.delete(y_pool_Dropout, (x_pool_index),
axis=0)
pool_x = np.concatenate((pool_x, X_pool_Dropout), axis=0)
pool_y = np.concatenate((pool_y, y_pool_Dropout), axis=0)
train_x = np.concatenate((train_x, Pooled_X), axis=0)
train_y = np.concatenate((train_y, Pooled_Y), axis=0)
if 0: # don't warm start
model = HyperCNN(lbda=lbda,
perdatapoint=perdatapoint,
prior=prior,
kernel_width=4,
pad='valid',
stride=1,
coupling=coupling)
recs = train_model(model.train_func, model.predict, train_x[:size],
train_y[:size], valid_x, valid_y, lr0, lrdecay, bs,
epochs)
test_accuracy = test_model(model.predict_proba, test_x, test_y)
print("Training Set Size", train_x.shape)
print("Test Accuracy", test_accuracy)
all_accuracy = np.append(all_accuracy, test_accuracy)
return all_accuracy
def active_learning(acquisition_iterations):
t0 = time.time()
bh_iterations = 100
nb_classes = 10
Queries = 10
all_accuracy = 0
acquisition_iterations = 98
filename = '../../mnist.pkl.gz'
train_x, train_y, valid_x, valid_y, test_x, test_y = load_mnist(filename)
train_x = train_x.reshape(50000,1,28,28)
valid_x = valid_x.reshape(10000,1,28,28)
test_x = test_x.reshape(10000,1,28,28)
train_x, train_y, pool_x, pool_y = split_train_pool_data(train_x, train_y)
train_y_multiclass = train_y.argmax(1)
train_x, train_y = get_initial_training_data(train_x, train_y_multiclass)
train_y = train_y.astype('float32')
print "Initial Training Data", train_x.shape
# select model
if arch == 'hyperCNN':
model = HyperCNN(lbda=lbda,
perdatapoint=perdatapoint,
prior=prior,
coupling=coupling,
kernel_width=4,
pad='valid',
stride=1,
extra_linear=extra_linear)
#dataset=dataset)
elif arch == 'CNN':
model = MCdropoutCNN(kernel_width=4,
pad='valid',
stride=1)
elif arch == 'CNN_spatial_dropout':
model = MCdropoutCNN(dropout='spatial',
kernel_width=4,
pad='valid',
stride=1)
elif arch == 'CNN_dropout':
model = MCdropoutCNN(dropout=1,
kernel_width=4,
pad='valid',
stride=1)
else:
raise Exception('no model named `{}`'.format(model))
if save:
model.save(save_path + '_params_init.npy')
# TODO: pretraining
if params_reset == 'pretrained': # train the model to 100% train accuracy on the initial train set
# TODO: we could also try training to 100% every time...
# TODO: and the last time, we should train until overfitting
# TODO: we also need to consider cross-validating the prior (do we even USE a prior for the dropout net?? we're supposed to!!!)
# TODO: ...and we could also use the validation set for early-stopping after every acquisition
tr_acc = 0.
epochs = 0
print "pretraining..."
while tr_acc < 1.:
epochs += 1
print " epoch", epochs
tr_acc = train_epoch(model.train_func,model.predict,
train_x[:size],train_y[:size],
valid_x,valid_y,
lr0,lrdecay,bs)
model.add_reset('pretrained')
if save:
model.save(save_path + '_params_pretrained.npy')
print "pretraining completed"
else:
recs = train_model(model.train_func,model.predict,
train_x[:size],train_y[:size],
valid_x,valid_y,
lr0,lrdecay,bs,epochs)
valid_accuracy = test_model(model.predict_proba, valid_x, valid_y)
print " valid Accuracy", valid_accuracy
all_valid_accuracy = valid_accuracy
test_accuracy = test_model(model.predict_proba, test_x, test_y)
print " Test Accuracy", test_accuracy
all_accuracy = test_accuracy
for i in range(acquisition_iterations):
print'time', time.time() - t0
print'POOLING ITERATION', i
pool_subset = pool_size
pool_subset_dropout = np.asarray(random.sample(range(0,pool_x.shape[0]), pool_subset))
X_pool_Dropout = pool_x[pool_subset_dropout, :, :, :]
y_pool_Dropout = pool_y[pool_subset_dropout]
#####################################3
# BEGIN ACQUISITION
if acq == 'bald':
score_All = np.zeros(shape=(X_pool_Dropout.shape[0], nb_classes))
All_Entropy_BH = np.zeros(shape=X_pool_Dropout.shape[0])
all_bh_classes = np.zeros(shape=(X_pool_Dropout.shape[0], bh_iterations))
for d in range(bh_iterations):
bh_score = model.predict_proba(X_pool_Dropout)
score_All = score_All + bh_score
bh_score_log = np.log2(bh_score)
Entropy_Compute = - np.multiply(bh_score, bh_score_log)
Entropy_Per_BH = np.sum(Entropy_Compute, axis=1)
All_Entropy_BH = All_Entropy_BH + Entropy_Per_BH
bh_classes = np.max(bh_score, axis=1)
all_bh_classes[:, d] = bh_classes
### for plotting uncertainty
predicted_class = np.max(all_bh_classes, axis=1)
predicted_class_std = np.std(all_bh_classes, axis=1)
Avg_Pi = np.divide(score_All, bh_iterations)
Log_Avg_Pi = np.log2(Avg_Pi)
Entropy_Avg_Pi = - np.multiply(Avg_Pi, Log_Avg_Pi)
Entropy_Average_Pi = np.sum(Entropy_Avg_Pi, axis=1)
G_X = Entropy_Average_Pi
Average_Entropy = np.divide(All_Entropy_BH, bh_iterations)
F_X = Average_Entropy
U_X = G_X - F_X
sort_values = U_X.flatten()
x_pool_index = sort_values.argsort()[-Queries:][::-1]
#print x_pool_index.shape # 10
#assert False
elif acq == 'max_ent':
score_All = np.zeros(shape=(X_pool_Dropout.shape[0], nb_classes))
for d in range(bh_iterations):
bh_score = model.predict_proba(X_pool_Dropout)
score_All = score_All + bh_score
Avg_Pi = np.divide(score_All, bh_iterations)
Log_Avg_Pi = np.log2(Avg_Pi)
Entropy_Avg_Pi = - np.multiply(Avg_Pi, Log_Avg_Pi)
Entropy_Average_Pi = np.sum(Entropy_Avg_Pi, axis=1)
U_X = Entropy_Average_Pi
sort_values = U_X.flatten()
x_pool_index = sort_values.argsort()[-Queries:][::-1]
elif acq == 'var_ratio':
All_BH_Classes = np.zeros(shape=(X_pool_Dropout.shape[0],1))
for d in range(bh_iterations):
bh_score = model.predict(X_pool_Dropout)
bh_score = np.array([bh_score]).T
All_BH_Classes = np.append(All_BH_Classes, bh_score, axis=1)
Variation = np.zeros(shape=(X_pool_Dropout.shape[0]))
for t in range(X_pool_Dropout.shape[0]):
L = np.array([0])
for d_iter in range(bh_iterations):
L = np.append(L, All_BH_Classes[t, d_iter+1])
Predicted_Class, Mode = mode(L[1:])
v = np.array( [1 - Mode/float(bh_iterations)])
Variation[t] = v
sort_values = Variation.flatten()
x_pool_index = sort_values.argsort()[-Queries:][::-1]
elif acq == 'mean_std':
All_Dropout_Scores = np.zeros(shape=(X_Pool_Dropout.shape[0], nb_classes))
for d in range(dropout_iterations):
dropout_score = model.predict_stochastic(X_Pool_Dropout,batch_size=batch_size, verbose=1)
All_Dropout_Scores = np.append(All_Dropout_Scores, dropout_score, axis=1)
All_Std = np.zeros(shape=(X_Pool_Dropout.shape[0],nb_classes))
BayesSegnet_Sigma = np.zeros(shape=(X_Pool_Dropout.shape[0],1))
for t in range(X_Pool_Dropout.shape[0]):
for r in range(nb_classes):
L = np.array([0])
L = np.append(L, All_Dropout_Scores[t, r+10])
L_std = np.std(L[1:])
All_Std[t,r] = L_std
E = All_Std[t,:]
BayesSegnet_Sigma[t,0] = sum(E)
a_1d = BayesSegnet_Sigma.flatten()
x_pool_index = a_1d.argsort()[-Queries:][::-1]
elif acq == 'random':
#x_pool_index = np.asarray(random.sample(range(0, 38000), Queries))
x_pool_index = np.random.choice(range(pool_size), Queries, replace=False)
# END ACQUISITION
#####################################3
Pooled_X = X_pool_Dropout[x_pool_index, :, :, :]
Pooled_Y = y_pool_Dropout[x_pool_index]
delete_Pool_X = np.delete(pool_x, (pool_subset_dropout), axis=0)
delete_Pool_Y = np.delete(pool_y, (pool_subset_dropout), axis=0)
delete_Pool_X_Dropout = np.delete(X_pool_Dropout, (x_pool_index), axis=0)
delete_Pool_Y_Dropout = np.delete(y_pool_Dropout, (x_pool_index), axis=0)
pool_x = np.concatenate((pool_x, X_pool_Dropout), axis=0)
pool_y = np.concatenate((pool_y, y_pool_Dropout), axis=0)
train_x = np.concatenate((train_x, Pooled_X), axis=0)
train_y = np.concatenate((train_y, Pooled_Y), axis=0).astype('float32')
#print pool_x.shape, Pooled_X.shape, train_x.shape
#assert False
if params_reset == 'random':# don't warm start (TODO!)
if arch == 'hyperCNN':
model = HyperCNN(lbda=lbda,
perdatapoint=perdatapoint,
prior=prior,
coupling=coupling,
kernel_width=4,
pad='valid',
stride=1,
extra_linear=extra_linear)
#dataset=dataset)
elif arch == 'CNN':
model = MCdropoutCNN(kernel_width=4,
pad='valid',
stride=1)
elif arch == 'CNN_spatial_dropout':
model = MCdropoutCNN(dropout='spatial',
kernel_width=4,
pad='valid',
stride=1)
elif arch == 'CNN_dropout':
model = MCdropoutCNN(dropout=1,
kernel_width=4,
pad='valid',
stride=1)
else:
raise Exception('no model named `{}`'.format(model))
elif params_reset == 'deterministic':
model.call_reset('init')
elif params_reset == 'pretrained':
model.call_reset('pretrained')
recs = train_model(model.train_func,model.predict,
train_x[:size],train_y[:size],
valid_x,valid_y,
lr0,lrdecay,bs,epochs)
valid_accuracy = test_model(model.predict_proba, valid_x, valid_y)
print " Valid Accuracy", valid_accuracy
all_valid_accuracy = np.append(all_valid_accuracy, valid_accuracy)
if test_eval:
test_accuracy = test_model(model.predict_proba, test_x, test_y)
print " Test Accuracy", test_accuracy
all_accuracy = np.append(all_accuracy, test_accuracy)
return all_accuracy, all_valid_accuracy