def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Initialize a tf session
session = tf.Session()
k.set_session(session)
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
# Create simple CNN
model = Sequential()
model.add(Conv2D(4, kernel_size=(5, 5), activation='relu', input_shape=(28, 28, 1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(10, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adam(lr=0.01),
metrics=['accuracy'])
# Get classifier
krc = KerasClassifier((0, 1), model, use_logits=False)
krc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=2)
# First attack
cl2m = CarliniL2Method(classifier=krc, targeted=True, max_iter=100, binary_search_steps=10,
learning_rate=2e-2, initial_const=3, decay=1e-2)
params = {'y': random_targets(y_test, krc.nb_classes)}
x_test_adv = cl2m.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second attack
cl2m = CarliniL2Method(classifier=krc, targeted=False, max_iter=100, binary_search_steps=10,
learning_rate=2e-2, initial_const=3, decay=1e-2)
params = {'y': random_targets(y_test, krc.nb_classes)}
x_test_adv = cl2m.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((target != y_pred_adv).all())
# Third attack
cl2m = CarliniL2Method(classifier=krc, targeted=False, max_iter=100, binary_search_steps=10,
learning_rate=2e-2, initial_const=3, decay=1e-2)
params = {}
x_test_adv = cl2m.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
y_pred = np.argmax(krc.predict(x_test), axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Initialize a tf session
session = tf.Session()
k.set_session(session)
# Get MNIST
batch_size, nb_train, nb_test = 10, 10, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
# Create simple CNN
model = Sequential()
model.add(
Conv2D(4,
kernel_size=(5, 5),
activation='relu',
input_shape=(28, 28, 1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(10, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(lr=0.01),
metrics=['accuracy'])
# Get classifier
krc = KerasClassifier((0, 1), model, use_logits=False)
krc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=2)
# Attack
# TODO Launch with all possible attacks
attack_params = {
"attacker": "newtonfool",
"attacker_params": {
"max_iter": 20
}
}
up = UniversalPerturbation(krc)
x_train_adv = up.generate(x_train, **attack_params)
self.assertTrue((up.fooling_rate >= 0.2) or not up.converged)
x_test_adv = x_test + up.v
self.assertFalse((x_test == x_test_adv).all())
train_y_pred = np.argmax(krc.predict(x_train_adv), axis=1)
test_y_pred = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == test_y_pred).all())
self.assertFalse((np.argmax(y_train, axis=1) == train_y_pred).all())
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
# Create simple CNN
model = Sequential()
model.add(
Conv2D(4,
kernel_size=(5, 5),
activation='relu',
input_shape=(28, 28, 1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(10, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(lr=0.01),
metrics=['accuracy'])
# Get classifier
krc = KerasClassifier((0, 1), model, use_logits=False)
krc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=2)
# Attack
nf = NewtonFool(krc)
nf.set_params(max_iter=5)
x_test_adv = nf.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
y_pred = krc.predict(x_test)
y_pred_adv = krc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= y_pred_adv_max).all())
def test_binary_input_detector(self):
"""
Test the binary input detector end-to-end.
:return:
"""
# Initialize a tf session
session = tf.Session()
k.set_session(session)
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:NB_TRAIN], y_train[:NB_TRAIN]
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
input_shape = x_train.shape[1:]
nb_classes = 10
# Create simple CNN
model = Sequential()
model.add(
Conv2D(4,
kernel_size=(5, 5),
activation='relu',
input_shape=input_shape))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(nb_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(lr=0.01),
metrics=['accuracy'])
# Create classifier and train it:
classifier = KerasClassifier((0, 1), model, use_logits=False)
classifier.fit(x_train, y_train, nb_epochs=5, batch_size=128)
# Generate adversarial samples:
attacker = FastGradientMethod(classifier, eps=0.1)
x_train_adv = attacker.generate(x_train[:nb_train])
x_test_adv = attacker.generate(x_test[:nb_test])
# Compile training data for detector:
x_train_detector = np.concatenate((x_train[:nb_train], x_train_adv),
axis=0)
y_train_detector = np.concatenate(
(np.array([[1, 0]] * nb_train), np.array([[0, 1]] * nb_train)),
axis=0)
# Create a simple CNN for the detector.
# Note: we use the same architecture as for the classifier, except for the number of outputs (=2)
model = Sequential()
model.add(
Conv2D(4,
kernel_size=(5, 5),
activation='relu',
input_shape=input_shape))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(2, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(lr=0.01),
metrics=['accuracy'])
# Create detector and train it:
detector = BinaryInputDetector(
KerasClassifier((0, 1), model, use_logits=False))
detector.fit(x_train_detector,
y_train_detector,
nb_epochs=2,
batch_size=128)
# Apply detector on clean and adversarial test data:
test_detection = np.argmax(detector(x_test), axis=1)
test_adv_detection = np.argmax(detector(x_test_adv), axis=1)
# Assert there is at least one true positive and negative:
nb_true_positives = len(np.where(test_adv_detection == 1)[0])
nb_true_negatives = len(np.where(test_detection == 0)[0])
self.assertTrue(nb_true_positives > 0)
self.assertTrue(nb_true_negatives > 0)
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Initialize a tf session
session = tf.Session()
k.set_session(session)
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
# Create simple CNN
model = Sequential()
model.add(
Conv2D(4,
kernel_size=(5, 5),
activation='relu',
input_shape=(28, 28, 1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(10, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(lr=0.01),
metrics=['accuracy'])
# Get classifier
krc = KerasClassifier((0, 1), model, use_logits=False)
krc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=10)
# First attack
cl2m = CarliniL2Method(classifier=krc,
targeted=True,
max_iter=100,
binary_search_steps=1,
learning_rate=1,
initial_const=10,
decay=0)
params = {'y': random_targets(y_test, krc.nb_classes)}
x_test_adv = cl2m.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
print("CW2 Target: %s" % target)
print("CW2 Actual: %s" % y_pred_adv)
print("CW2 Success Rate: %f" %
(sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target == y_pred_adv).any())
# Second attack
cl2m = CarliniL2Method(classifier=krc,
targeted=False,
max_iter=100,
binary_search_steps=1,
learning_rate=1,
initial_const=10,
decay=0)
params = {'y': random_targets(y_test, krc.nb_classes)}
x_test_adv = cl2m.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
print("CW2 Target: %s" % target)
print("CW2 Actual: %s" % y_pred_adv)
print("CW2 Success Rate: %f" %
(sum(target != y_pred_adv) / float(len(target))))
self.assertTrue((target != y_pred_adv).any())
# Third attack
cl2m = CarliniL2Method(classifier=krc,
targeted=False,
max_iter=100,
binary_search_steps=1,
learning_rate=1,
initial_const=10,
decay=0)
params = {}
x_test_adv = cl2m.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(krc.predict(x_test), axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
print("CW2 Target: %s" % y_pred)
print("CW2 Actual: %s" % y_pred_adv)
print("CW2 Success Rate: %f" %
(sum(y_pred != y_pred_adv) / float(len(y_pred))))
self.assertTrue((y_pred != y_pred_adv).any())