def experiment(network_model, reshape_mode='mlp'):
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
digits_data = utils.load_processed_data('combined_testing_data')
digits_data2 = utils.load_processed_data('digits_og_and_optimal')
taus = [13, 14, 15]
digits = list(map(reshape_fun, [digits_data[t] for t in taus]))
digits = list(map(utils.normalize_data, digits))
digits_og = reshape_fun(digits_data2['lecunn'])
digits_og = utils.normalize_data(digits_og)
d_labels = utils.create_one_hot(digits_data['labels'].astype('uint'))
d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint'))
ensemble_size = 20
epochs = 3
trials = 5
mnist_correct = []
mnist_wrong = []
digits_wrong = []
digits_correct = []
d2_wrong = []
d2_correct = []
for t in range(trials):
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(ensemble_size):
t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation(
xtrain, ytrain, (1 / 6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble(
[network_model], pop_per_type=ensemble_size, merge_type="Average")
losses = list(
map(
lambda m: ann.adveserial_loss(
klosses.categorical_crossentropy, m, eps=0.01),
model_list))
train_model.compile(optimizer='adam', loss=losses, metrics=['acc'])
train_model.fit(l_xtrain,
l_ytrain,
epochs=epochs,
validation_data=(l_xval, l_yval))
mnist_preds = merge_model.predict([xtest] * ensemble_size)
correct, wrong = bin_entropies(mnist_preds, ytest)
mnist_correct.extend(correct)
mnist_wrong.extend(wrong)
d2_preds = merge_model.predict([digits_og] * ensemble_size)
correct, wrong = bin_entropies(d2_preds, d2_labels)
d2_correct.extend(correct)
d2_wrong.extend(wrong)
for d in digits:
digits_preds = merge_model.predict([d] * ensemble_size)
correct, wrong = bin_entropies(digits_preds, d_labels)
digits_wrong.extend(wrong)
digits_correct.extend(correct)
ensemble = {
'mnist_correct': mnist_correct,
'mnist_wrong': mnist_wrong,
'digits_correct': digits_correct,
'digits_wrong': digits_wrong,
'lecunn_correct': d2_correct,
'lecunn_wrong': d2_wrong
}
return ensemble
def experiment(network_model, reshape_mode = 'mlp'):
reshape_funs = {
"conv" : lambda d : d.reshape(-1,28,28,1),
"mlp" : lambda d : d.reshape(-1,784)
}
xtrain,ytrain,xtest,ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest)
custom_digits_dict = utils.load_processed_data("combined_testing_data")
digits_labels = custom_digits_dict['labels']
digits_taus = list(custom_digits_dict.keys())[:-1]
digits_data = list(map(reshape_fun, [custom_digits_dict[t] for t in digits_taus]))
digits_data = list(map(utils.normalize_data, digits_data))
digits_labels = utils.create_one_hot(digits_labels.astype('uint'))
for t in range(trials):
print("===== START TRIAL %s =====" % (t + 1))
# Preparing Results
# Classification Error
ensemble_cerror = []
#t_ensemble_adv_cerror = []
digits_cerror = []
#t_digits_adv_cerror = []
# Prediction Entropy
entropy_ensemble = []
#t_entropy_adv_ensemble = []
digits_entropy = []
#t_digits_adv_entropy = []
# Voting entropy
#entropy_vote = []
#entropy_adv_vote = []
#digits_vote = []
#digits_adv_entropy = []
epochs = 5
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(ensemble_size):
t_xtrain,t_ytrain,t_xval,t_yval = utils.create_validation(xtrain,ytrain,(1/6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
# Without adveserial training
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=ensemble_size, merge_type="Average")
losses = list(
map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list)
)
train_model.compile(optimizer="adam", loss=losses, metrics = ['acc'])
train_model.fit(x=l_xtrain,y=l_ytrain, verbose=1,batch_size=100, epochs = epochs,validation_data=(l_xval,l_yval))
merge_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc'])
c_error = ann.test_model(merge_model, [xtest]*ensemble_size, ytest, metric = 'accuracy' )
entropy = ann.test_model(merge_model, [xtest]*ensemble_size, ytest, metric = 'entropy' )
ensemble_cerror.append(c_error)
entropy_ensemble.append(entropy)
#entropy_vote.append(calc_vote_entropy(model_list,m,xtest))
d_cerror,d_entropy = test_digits(merge_model,model_list,ensemble_size,digits_data,digits_labels)
digits_cerror.append(d_cerror)
digits_entropy.append(d_entropy)
#digits_vote.append(d_vote)
# Adveserial training
#inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=m, merge_type="Average")
#losses = list(
# map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list)
#)
#train_model.compile(optimizer="adam", loss=losses, metrics = ['acc'])
#train_model.fit(x=l_xtrain, y=l_ytrain, verbose=1, epochs = epochs ,validation_data=(l_xval,l_yval))
#c_error = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'c_error' )
#entropy = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'entropy' )
#
#t_ensemble_adv_cerror.append(c_error)
#t_entropy_adv_ensemble.append(entropy)
#
#d_cerror,d_entropy,d_vote = test_digits(merge_model,model_list,m,digits_data,digits_labels)
#
#t_digits_adv_cerror.append(d_cerror)
#t_digits_adv_entropy.append(d_entropy)
filename1 = 'adv_mnist_results_5-20-50-trial%s' % t
filename2 = 'adv_digits_results_5-20-50-trial%s' % t
mnist_results = {
'ensemble_cerror' : ensemble_cerror,
'ensemble_entropy' : entropy_ensemble
#'ensemble_adv_cerror' : ensemble_adv_cerror,
#'ensemble_adv_entropy' : entropy_adv_ensemble,
#'voting_entropy' : entropy_vote
#'voting_adv_entropy' : entropy_adv_vote
}
digits_results = {
'ensemble_cerror' : digits_cerror,
'ensemble_entropy' : digits_entropy
#'ensemble_adv_cerror' : digits_adv_cerror,
#'ensemble_adv_entropy' : digits_adv_entropy,
#'voting_entropy' : digits_vote
#'voting_adv_entropy' : digits_adv_vote
}
utils.save_processed_data(mnist_results,filename1)
utils.save_processed_data(digits_results,filename2)
return digits_taus, mnist_results, digits_results
{
"type" : "Flatten"
},
{
"type" : "Dense",
"units" : 10,
"activation" : "softmax"
}
]
}
"""
utils.setup_gpu_session(True)
xtrain, ytrain, xtest, ytest = utils.load_mnist()
xtrain = xtrain.reshape(60000, 28, 28, 1)
xtest = xtest.reshape(10000, 28, 28, 1)
model_conf = json.loads(network_model1)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble(
[model_conf])
ensemble_size = len(model_list)
lossfunctions = [
ann.adveserial_loss(klosses.categorical_crossentropy, m)
for m in model_list
]
train_model.compile(optimizer="adam", loss=lossfunctions, metrics=["accuracy"])
train_model.fit([xtrain] * ensemble_size, [ytrain] * ensemble_size,
verbose=1,
validation_data=([xtest] * ensemble_size,
[ytest] * ensemble_size),
epochs=3)
