def __init__(self,
sess,
data,
length,
attack='FGSM',
dataset="_test_set_",
num_filters=64,
batch_size=128,
epochs=10):
self.__data = data
self.__image_rows = data.x_train.shape[1]
self.__image_cols = data.x_train.shape[2]
self.__channels = data.x_train.shape[3]
self.__nb_classes = data.y_train.shape[1]
self.__attack = attack
self._length = length
self.__sess = sess
self.__batch = batch_size
self.__epochs = epochs
self.__dataset = data.dataset_name
self._test_or_val_dataset = dataset
self._attack_dir = "./adv_attacks/adversarial_images"
if dataset == "_test_set_":
self.idx_adv = helpers.load_pkl(
os.path.join(self._attack_dir, "example_idx.pkl"))
else:
self.idx_adv = helpers.load_pkl(
os.path.join(self._attack_dir, "validation_idx.pkl"))
self.surrogate_model = Sequential()
self.surrogate_model.add(
Conv2D(32, (3, 3),
padding='same',
input_shape=self.__data.x_train.shape[1:]))
self.surrogate_model.add(Activation('relu'))
self.surrogate_model.add(Conv2D(32, (3, 3)))
self.surrogate_model.add(Activation('relu'))
self.surrogate_model.add(MaxPooling2D(pool_size=(2, 2)))
self.surrogate_model.add(Dropout(0.25))
self.surrogate_model.add(Conv2D(64, (3, 3), padding='same'))
self.surrogate_model.add(Activation('relu'))
self.surrogate_model.add(Conv2D(64, (3, 3)))
self.surrogate_model.add(Activation('relu'))
self.surrogate_model.add(MaxPooling2D(pool_size=(2, 2)))
self.surrogate_model.add(Dropout(0.25))
self.surrogate_model.add(Flatten())
self.surrogate_model.add(Dense(512))
self.surrogate_model.add(Activation('relu'))
self.surrogate_model.add(Dropout(0.5))
self.surrogate_model.add(Dense(10))
self.surrogate_model.add(Activation('softmax'))
self.surrogate_model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
def _load_images(self, attack_str, dataset='_test_set_'):
if attack_str == "":
path = os.path.join(
self._attack_dir,
self.__dataset.lower() + dataset + self.__attack.lower() +
'.plk')
imgs = helpers.load_pkl(path)
else:
path = os.path.join(
self._attack_dir,
self.__dataset.lower() + dataset + attack_str + '.plk')
imgs = helpers.load_pkl(path)
# imgs = helpers.load_pkl(self.__dataset.lower() + dataset + attack_str + '.plk')
return imgs
def __init__(self, dataset):
"""
Realizes the MultiMagNet's experiments.
# Attribute
dataset: 'MNIST' or 'CIFAR'
"""
path = os.path.join("./adv_attacks/adversarial_images/example_idx.pkl")
self._idx_adv = helpers.load_pkl(path)
self._sess = tf.Session()
self._sess.as_default()
self._data = Data(dataset_name=dataset)
print("\nDataset loaded.")
def get_thresholds(self,
drop_rate=0.001,
p=2,
tau="RE",
plot_rec_images=False,
load_thresholds=True):
"""
Predicts the 'data' using the selected autoencoders,
returning their respective reconstruction errors thresholds.
"""
s = array(self.repository)
if (self.__number > s.size):
raise Exception(
"Number_team_members: {0} is bigger than the number of avaiable models into repository: {1}."
.format(self.__number, s.size))
else:
print(
'\nTotal members into repository: {0}\nNumber of members chosen: {1}'
.format(s.size, self.__number))
self.team = np.random.choice(s, size=self.__number, replace=False)
thresholds = []
num = round(drop_rate * len(self.__data.x_val))
for i in range(self.team.size):
# load pre-computed autoencoder threshold (if it exists)
path = os.path.join(
"./team_techniques/models/thresholds", self.team[i].name +
"_" + str(drop_rate) + "_" + "RE" + "_.plk")
try:
if not load_thresholds:
raise Exception(
"load_thresholds parameter set False. Computing the thresholds manually."
)
threshold = helpers.load_pkl(path)
if type(threshold) == type(None):
raise Exception
else:
print("Threshold of autoencoder {0} loaded.".format(
self.team[i].name))
except:
autoencoder = self.load_autoencoder(self.team[i], "RE")
rec = autoencoder.predict(self.__data.x_val)
if plot_rec_images == True:
rec_ex = autoencoder.predict(self.__data.x_test[:10])
helpers.plot_images(self.__data.x_test[:10], rec_ex[:10],
rec_ex.shape)
print(
'Reconstructing images using {0} model ({1}/{2}).'.format(
self.team[i].name, i + 1, self.team.size))
if self.__data.x_val.shape[1:] != rec.shape[1:]:
rec = rec.reshape(
rec.shape[0], self.__data.x_val.shape[1],
self.__data.x_val.shape[2],
self.__data.x_val.shape[3]).astype('float32')
diff = np.abs(self.__data.x_val - rec)
marks = np.mean(np.power(diff, p), axis=(1, 2, 3))
marks_iset = np.sort(marks)
threshold = marks_iset[-num]
path = os.path.join(
"./team_techniques/models/thresholds", self.team[i].name +
"_" + str(drop_rate) + "_" + "RE" + "_.plk")
try:
helpers.save_pkl(threshold, path)
except:
print(
"It was not possible to save {0} autoencoder threshold."
.format(self.team[i].name))
del autoencoder
thresholds.append(threshold)
if tau == "minRE":
thresholds = [np.min(thresholds)] * self.__number
return thresholds
def get_thresholds_pd(self,
classifier,
drop_rate=0.001,
T=10,
p=2,
tau="RE",
plot_rec_images=False,
load_thresholds=True,
metric='JSD'):
"""
Predicts the 'data' using the selected autoencoders,
returning their respective probability divergence thresholds.
"""
self.team = self.get_team()
thresholds = []
num = round(drop_rate * len(self.__data.x_val))
model = helpers.get_logits(classifier.model)
sft = Sequential()
sft.add(Lambda(lambda X: softmax(X, axis=1), input_shape=(10, )))
for i in range(self.team.size):
# load pre-computed autoencoder threshold (if it exists)
path = os.path.join(
"./team_techniques/models/thresholds", self.team[i].name +
"_" + str(drop_rate) + "_" + str(T) + "_" + metric + "_.plk")
try:
if not load_thresholds:
raise Exception(
"load_thresholds parameter set False. Computing the thresholds manually."
)
threshold = helpers.load_pkl(path)
if type(threshold) == type(None):
raise Exception
else:
print("Threshold of autoencoder {0} loaded.".format(
self.team[i].name))
except:
autoencoder = self.load_autoencoder(self.team[i], metric)
rec = autoencoder.predict(self.__data.x_val)
if plot_rec_images == True:
rec_ex = autoencoder.predict(self.__data.x_test[:10])
helpers.plot_images(self.__data.x_test[:10], rec_ex[:10],
rec_ex.shape)
del rec_ex
print(
'Reconstructing images using {0} model ({1}/{2}).'.format(
self.team[i].name, i + 1, self.team.size))
if self.__data.x_val.shape[1:] != rec.shape[1:]:
rec = rec.reshape(
rec.shape[0], self.__data.x_val.shape[1],
self.__data.x_val.shape[2],
self.__data.x_val.shape[3]).astype('float32')
# marks = np.mean(np.power(np.abs(model.predict(self.__data.x_val) - model.predict(rec)), 1), axis=1)
oc = sft.predict(model.predict(self.__data.x_val) / T)
rc = sft.predict(model.predict(rec) / T)
# print("OC[0]: {0}\nRC[0]: {1}".format(oc[0], rc[0]))
# print(oc.shape, rc.shape)
if metric == 'JSD':
marks = [JSD(oc[j], rc[j]) for j in range(len(rc))]
elif metric == 'DKL':
from scipy.stats import entropy
marks = [
entropy(pk=rc[j], qk=oc[j]) for j in range(len(rc))
]
marks_iset = np.sort(marks)
threshold = marks_iset[-num]
path = os.path.join(
"./team_techniques/models/thresholds",
self.team[i].name + "_" + str(drop_rate) + "_" + str(T) +
"_" + metric + "_.plk")
try:
helpers.save_pkl(threshold, path)
except:
print(
"It was not possible to save {0} autoencoder threshold."
.format(self.team[i].name))
del autoencoder
thresholds.append(threshold)
if tau == "minRE":
thresholds = [np.min(thresholds)] * self.__number
return thresholds
def tuning_team_parameters(self, attack, *args, classifier=None):
print("\nStarting validation process...\n")
classifier = Classifier(self._sess,
self._data,
epochs=350,
learning_rate=0.01,
batch_size=32)
classifier.execute()
path = os.path.join(
"./adv_attacks/adversarial_images",
self._data.dataset_name.lower() + "_val_set_" + attack.lower() +
".plk")
val_set_adv = helpers.load_pkl(path)
path = os.path.join(
"./adv_attacks/adversarial_images/validation_idx.pkl")
idx = helpers.load_pkl(path)
val_set_leg = self._data.x_val[idx]
_, x_val, y_val, _ = helpers.join_test_sets(self._data.x_test,
val_set_adv,
self._data.y_test,
len(val_set_leg),
idx=idx)
import itertools
combinations = list(itertools.product(*args))
print(len(combinations))
team_stats = np.zeros((len(combinations), 3))
parameters = [[0 for x in range(5)] for y in range(len(combinations))]
k = 0
for combination in combinations:
reduction_models = parameters[k][0] = combination[0]
drop_rate = parameters[k][1] = combination[1]
tau = parameters[k][2] = combination[2]
metric = parameters[k][3] = combination[3]
if self._data.dataset_name == "CIFAR":
T = parameters[k][4] = combination[4]
team = Assembly_Team(self._sess, self._data, reduction_models)
if metric == "RE":
thresholds = team.get_thresholds(tau=tau,
drop_rate=drop_rate,
p=1,
plot_rec_images=False,
load_thresholds=False)
val_marks = Image_Reduction.apply_techniques(x_val, team, p=1)
else:
thresholds = team.get_thresholds_pd(tau=tau,
classifier=classifier,
T=T,
drop_rate=drop_rate,
p=1,
plot_rec_images=False,
load_thresholds=False,
metric=metric)
val_marks = Image_Reduction.apply_techniques_pd(x_val,
team,
classifier,
T=T,
p=1,
metric=metric)
y_pred, _ = poll_votes(x_val, y_val, val_marks, thresholds,
reduction_models)
print(
"\nEXPERIMENT USING {0} DATASET: {1} Input Images 'x', {2} Attack, p = {3}, reduction models = {4}, drop_rate = {5}\n, T = {6}"
.format(self._data.dataset_name, len(x_val), attack, 1,
reduction_models, drop_rate, T))
team_stats[k, 0], team_stats[k, 1], team_stats[
k, 2], _, _, cm = helpers.get_cm_and_statistics(y_val, y_pred)
print(
'Threshold used: {0}\nConfusion Matrix:\n{1}\nACC: {2}, Positive Precision: {3}, Negative Precision: {4}'
.format(thresholds, cm, team_stats[k, 0], team_stats[k, 1],
team_stats[k, 2]))
k = k + 1
max_acc = max(team_stats[:, 0])
index = np.argmax(team_stats[:, 0])
print(
"\nBest accuracy of {0:.3} was obtained by the following MultiMagNet's hyperparameters:\n{1}"
.format(max_acc, parameters[index]))