#train on a subset only
x_train = x_train[:1000]
y_train = y_train[:1000]
epsilon = 2e-4 # step size
tau = 200 # number of steps to take before the reject/accept step
burn_in = 100
sample_every = 30
N_ensemble = 20 #number of models to create
N_restarts = 5 #use multiple intitialisations
# use multiple intitialisations
ensemble = []
with open('save/tmp/losses.dat', 'w') as f:
print('', f)
for i in range(N_restarts):
N = N_ensemble // N_restarts
mcmc.reset_model(model)
es, losses, weights, accept_ratio = test_run(model, x_train, y_train,
N, epsilon, tau, burn_in,
sample_every)
ensemble += es
print('Accept ratio:', accept_ratio)
with open('save/tmp/losses.dat', 'a') as f:
for l in losses:
print(l, file=f)
name = U.gen_save_name('save/mnist_hmc_ensemble_run.pkl')
with open(name, 'wb') as f:
pickle.dump(ensemble, f)
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = Sequential()
model.add(
Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
#model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
#model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
fname = U.gen_save_name('save/mnist_cnn_no_drop_run.h5')
model.save(fname)
def run(x_real,
x_real_labels,
x_to_adv,
x_to_adv_labels,
x_adv_labels,
x_plus_noise,
x_plus_noise_labels,
x_advs_plot,
attack_params,
adv_save_num=15,
fname='rcc_results_{}',
batch_size=5,
N_data=200):
dists_ls = []
fpr_entropies = []
tpr_entropies = []
fpr_balds = []
tpr_balds = []
prec_entropies = []
rec_entropies = []
prec_balds = []
rec_balds = []
AUC_entropies = []
AUC_balds = []
AP_entropies = []
AP_balds = []
#records on succesful values
fpr_entropies_succ = []
tpr_entropies_succ = []
fpr_balds_succ = []
tpr_balds_succ = []
prec_entropies_succ = []
rec_entropies_succ = []
prec_balds_succ = []
rec_balds_succ = []
AUC_entropies_succ = []
AUC_balds_succ = []
AP_entropies_succ = []
AP_balds_succ = []
accs = []
modelnames = []
for i, (name, m) in enumerate(models_to_eval):
modelnames.append(name)
input_t = K.placeholder(shape=(None, 224, 224, 3))
wrap = CallableModelWrapper(m, 'probs')
x_adv = create_adv_examples(wrap, input_t, x_to_adv, attack_params)
#check the examples are really adversarial
preds = np.concatenate([
m.predict(x).argmax(axis=1)
for x in batch_gen(x_adv, batch_size=args.batch_size)
],
axis=0)
acc = np.mean(np.equal(preds, x_to_adv_labels.argmax(axis=1)))
print("Accuracy on adv examples:", acc)
accs.append(acc)
succ_adv_inds = np.logical_not(
np.equal(preds, x_to_adv_labels.argmax(
axis=1))) #seperate out succesful adv examples
dists = U.batch_L_norm_distances(x_to_adv, x_adv, ord=2)
noise = np.random.random(size=x_plus_noise.shape)
noise /= (dists * np.linalg.norm(
noise.reshape(x_plus_noise.shape[0], -1), axis=1))[:, None, None,
None]
x_plus_noise += noise
x_plus_noise = np.clip(x_plus_noise, 0, 1)
x_synth = np.concatenate([x_real, x_adv, x_plus_noise])
y_synth = np.array(x_real_labels + x_adv_labels + x_plus_noise_labels)
#x_synth, y_synth = shuffle_dataset(x_synth, y_synth) no points
dists_ls.append(dists)
succ_adv_inds = np.concatenate([
np.ones(len(x_real_labels)), succ_adv_inds,
np.ones(len(x_plus_noise_labels))
]).astype(np.bool)
# save the adverserial examples to plot
x_advs_plot = x_advs_plot + [
U.tile_images([x_adv[i] for i in range(adv_save_num)],
horizontal=False)
]
batches = U.batches_generator(x_synth, y_synth, batch_size=batch_size)
# get the entropy and bald on this task
try:
#we can now clean up the adv tensor
del input_t
del adv_tensor
except:
pass #if these aren't defined, ignore
entropy = []
bald = []
for j, (bx, by) in enumerate(batches):
print('Evaluating entropy/bald: batch ', j)
if hasattr(m, 'get_results'):
_, e, b = m.get_results(bx)
else:
res = m.predict(bx)
e = np.sum(-res * np.log(res + 1e-6), axis=1)
b = np.zeros(e.shape) # undefined
entropy.append(e)
bald.append(b)
entropy = np.concatenate(entropy, axis=0)
bald = np.concatenate(bald, axis=0)
fpr_entropy, tpr_entropy, _ = roc_curve(y_synth, entropy, pos_label=1)
fpr_bald, tpr_bald, _ = roc_curve(y_synth, bald, pos_label=1)
prec_entr, rec_entr, _ = precision_recall_curve(y_synth,
entropy,
pos_label=1)
prec_bald, rec_bald, _ = precision_recall_curve(y_synth,
bald,
pos_label=1)
AUC_entropy = roc_auc_score(y_synth, entropy)
AUC_bald = roc_auc_score(y_synth, bald)
AP_entropy = average_precision_score(y_synth, entropy)
AP_bald = average_precision_score(y_synth, bald)
fpr_entropies.append(fpr_entropy)
tpr_entropies.append(tpr_entropy)
prec_entropies.append(prec_entr)
rec_entropies.append(rec_entr)
prec_balds.append(prec_bald)
rec_balds.append(rec_bald)
fpr_balds.append(fpr_bald)
tpr_balds.append(tpr_bald)
AUC_entropies.append(AUC_entropy)
AUC_balds.append(AUC_bald)
AP_entropies.append(AP_entropy)
AP_balds.append(AP_bald)
#record stats on succesful adv examples only
y_synth = y_synth[succ_adv_inds]
entropy = entropy[succ_adv_inds]
bald = bald[succ_adv_inds]
fpr_entropy, tpr_entropy, _ = roc_curve(y_synth, entropy, pos_label=1)
fpr_bald, tpr_bald, _ = roc_curve(y_synth, bald, pos_label=1)
prec_entr, rec_entr, _ = precision_recall_curve(y_synth,
entropy,
pos_label=1)
prec_bald, rec_bald, _ = precision_recall_curve(y_synth,
bald,
pos_label=1)
AUC_entropy = roc_auc_score(y_synth, entropy)
AUC_bald = roc_auc_score(y_synth, bald)
AP_entropy = average_precision_score(y_synth, entropy)
AP_bald = average_precision_score(y_synth, bald)
fpr_entropies_succ.append(fpr_entropy)
tpr_entropies_succ.append(tpr_entropy)
prec_entropies_succ.append(prec_entr)
rec_entropies_succ.append(rec_entr)
prec_balds_succ.append(prec_bald)
rec_balds_succ.append(rec_bald)
fpr_balds_succ.append(fpr_bald)
tpr_balds_succ.append(tpr_bald)
AUC_entropies_succ.append(AUC_entropy)
AUC_balds_succ.append(AUC_bald)
AP_entropies_succ.append(AP_entropy)
AP_balds_succ.append(AP_bald)
fname = U.gen_save_name(fname.format(attack_params["method"]))
with h5py.File(fname, 'w') as f:
#record some meta-data in case i forget what i was doing
f.create_dataset('attack', data=json.dumps(attack_params))
f.create_dataset('dists', data=np.array(dists_ls))
f.create_dataset('N_data', data=N_data)
for i, name in enumerate(modelnames):
g = f.create_group(name)
g.create_dataset('entropy_fpr', data=fpr_entropies[i])
g.create_dataset('entropy_tpr', data=tpr_entropies[i])
g.create_dataset('bald_fpr', data=fpr_balds[i])
g.create_dataset('bald_tpr', data=tpr_balds[i])
g.create_dataset('entropy_prec', data=prec_entropies[i])
g.create_dataset('entropy_rec', data=rec_entropies[i])
g.create_dataset('bald_prec', data=prec_balds[i])
g.create_dataset('bald_rec', data=rec_balds[i])
g.create_dataset('entropy_AUC', data=AUC_entropies[i])
g.create_dataset('bald_AUC', data=AUC_balds[i])
g.create_dataset('entropy_AP', data=AP_entropies[i])
g.create_dataset('bald_AP', data=AP_balds[i])
g.create_dataset('entropy_fpr_succ', data=fpr_entropies_succ[i])
g.create_dataset('entropy_tpr_succ', data=tpr_entropies_succ[i])
g.create_dataset('bald_fpr_succ', data=fpr_balds_succ[i])
g.create_dataset('bald_tpr_succ', data=tpr_balds_succ[i])
g.create_dataset('entropy_prec_succ', data=prec_entropies_succ[i])
g.create_dataset('entropy_rec_succ', data=rec_entropies_succ[i])
g.create_dataset('bald_prec_succ', data=prec_balds_succ[i])
g.create_dataset('bald_rec_succ', data=rec_balds_succ[i])
g.create_dataset('entropy_AUC_succ', data=AUC_entropies_succ[i])
g.create_dataset('bald_AUC_succ', data=AUC_balds_succ[i])
g.create_dataset('entropy_AP_succ', data=AP_entropies_succ[i])
g.create_dataset('bald_AP_succ', data=AP_balds_succ[i])
g.create_dataset('adv_accuracy', data=accs[i])
f.create_dataset('example_imgs',
data=np.concatenate(x_advs_plot, axis=1))
if __name__ == '__main__':
mode = 'rnet_cdrop'
if mode == 'vgg':
model = define_model()
wname = 'save/cats_dogs_vgg_w_run.h5'
elif mode == 'rnet':
model = define_model_resnet()
wname = 'save/cats_dogs_rn50_w_run.h5'
elif mode == 'rnet_cdrop':
model = define_cdropout_model_resnet()
wname = 'save/cats_dogs_rn50_cdrop_w_run.h5'
model.compile(loss='categorical_crossentropy',
metrics=['accuracy'],
optimizer='adam')
with h5py.File(H5PATH, 'r') as f:
x_tr = f['train']['X']
y_tr = f['train']['Y']
x_te = f['test']['X']
y_te = f['test']['Y']
model.fit(x_tr,
y_tr,
epochs=40,
validation_data=(x_te, y_te),
shuffle='batch')
name = U.gen_save_name(wname)
model.save_weights(name)
x_train = mmnist['x_train'][:1000]
y_train = mmnist['y_train'][:1000]
epsilon = 2e-4 # step size
tau = 200 # number of steps to take before the reject/accept step
burn_in = 100
sample_every = 30
N_ensemble = 20 #number of models to create
N_restarts = 5 #use multiple intitialisations
# use multiple intitialisations
ensemble = []
with open('save/tmp/losses.dat', 'w') as f:
print('', f)
for i in range(N_restarts):
N = N_ensemble // N_restarts
mcmc.reset_model(model)
es, losses, weights, accept_ratio = test_run(model, x_train, y_train,
N, epsilon, tau, burn_in,
sample_every)
ensemble += es
print('Accept ratio:', accept_ratio)
with open('save/tmp/losses.dat', 'a') as f:
for l in losses:
print(l, file=f)
mmnist.close()
name = U.gen_save_name('save/manifold_hmc_ensemble_run.pkl')
with open(name, 'wb') as f:
pickle.dump(ensemble, f)
dropout_regularizer=DROPOUT_REGULARIZER,
))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(
ConcreteDropout(
Dense(128, activation=act_fn),
weight_regularizer=WEIGHT_REGULARIZER,
dropout_regularizer=DROPOUT_REGULARIZER,
))
model.add(
ConcreteDropout(
Dense(num_classes, activation='softmax'),
weight_regularizer=WEIGHT_REGULARIZER,
dropout_regularizer=DROPOUT_REGULARIZER,
))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(),
metrics=['accuracy'])
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(x_test, y_test),
callbacks=[TrackConcreteDropoutP(model)],
shuffle='batch') #check the p values of the c d are converging.
fname = U.gen_save_name('save/manifold_mnist_cdrop_cnn_run.h5')
save_model(model, fname)