def test_lenet_mnist_coverage_cpu():
context.set_context(mode=context.GRAPH_MODE, device_target="CPU")
# load network
net = Net()
model = Model(net)
# initialize fuzz test with training dataset
training_data = (np.random.random((10000, 10))*20).astype(np.float32)
model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, training_data)
# fuzz test with original test data
# get test data
test_data = (np.random.random((2000, 10))*20).astype(np.float32)
test_labels = np.random.randint(0, 10, 2000).astype(np.int32)
model_fuzz_test.test_adequacy_coverage_calculate(test_data)
LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc())
LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc())
LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
# generate adv_data
loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
attack = FastGradientSignMethod(net, eps=0.3, loss_fn=loss)
adv_data = attack.batch_generate(test_data, test_labels, batch_size=32)
model_fuzz_test.test_adequacy_coverage_calculate(adv_data,
bias_coefficient=0.5)
LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc())
LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc())
LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
def test_lenet_mnist_coverage_ascend():
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
# load network
net = Net()
model = Model(net)
# initialize fuzz test with training dataset
training_data = (np.random.random((10000, 10))*20).astype(np.float32)
model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, training_data)
# fuzz test with original test data
# get test data
test_data = (np.random.random((2000, 10))*20).astype(np.float32)
test_labels = np.random.randint(0, 10, 2000)
test_labels = (np.eye(10)[test_labels]).astype(np.float32)
model_fuzz_test.test_adequacy_coverage_calculate(test_data)
LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc())
LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc())
LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
# generate adv_data
attack = FastGradientSignMethod(net, eps=0.3)
adv_data = attack.batch_generate(test_data, test_labels, batch_size=32)
model_fuzz_test.test_adequacy_coverage_calculate(adv_data,
bias_coefficient=0.5)
LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc())
LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc())
LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
def test_fast_gradient_sign_method():
"""
Fast gradient sign method unit test.
"""
input_np = np.asarray([[0.1, 0.2, 0.7]], np.float32)
label = np.asarray([2], np.int32)
label = np.eye(3)[label].astype(np.float32)
attack = FastGradientSignMethod(Net())
ms_adv_x = attack.generate(input_np, label)
assert np.any(ms_adv_x != input_np), 'Fast gradient sign method: generate' \
' value must not be equal to' \
' original value.'
def test_lenet_mnist_coverage():
context.set_context(mode=context.GRAPH_MODE, device_target="CPU")
# upload trained network
ckpt_name = './trained_ckpt_file/checkpoint_lenet-10_1875.ckpt'
net = LeNet5()
load_dict = load_checkpoint(ckpt_name)
load_param_into_net(net, load_dict)
model = Model(net)
# get training data
data_list = "./MNIST_unzip/train"
batch_size = 32
ds = generate_mnist_dataset(data_list, batch_size, sparse=True)
train_images = []
for data in ds.create_tuple_iterator():
images = data[0].astype(np.float32)
train_images.append(images)
train_images = np.concatenate(train_images, axis=0)
# initialize fuzz test with training dataset
model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, train_images)
# fuzz test with original test data
# get test data
data_list = "./MNIST_unzip/test"
batch_size = 32
ds = generate_mnist_dataset(data_list, batch_size, sparse=True)
test_images = []
test_labels = []
for data in ds.create_tuple_iterator():
images = data[0].astype(np.float32)
labels = data[1]
test_images.append(images)
test_labels.append(labels)
test_images = np.concatenate(test_images, axis=0)
test_labels = np.concatenate(test_labels, axis=0)
model_fuzz_test.test_adequacy_coverage_calculate(test_images)
LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc())
LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc())
LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
# generate adv_data
loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
attack = FastGradientSignMethod(net, eps=0.3, loss_fn=loss)
adv_data = attack.batch_generate(test_images, test_labels, batch_size=32)
model_fuzz_test.test_adequacy_coverage_calculate(adv_data,
bias_coefficient=0.5)
LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc())
LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc())
LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
def test_fast_gradient_sign_method():
"""
FGSM-Attack test
"""
context.set_context(mode=context.GRAPH_MODE)
# get network
net = resnet50_cifar10(10)
# create test data
test_images = np.random.rand(64, 3, 224, 224).astype(np.float32)
test_labels = np.random.randint(10, size=64).astype(np.int32)
# attacking
loss_fn = CrossEntropyLoss()
attack = FastGradientSignMethod(net, eps=0.1, loss_fn=loss_fn)
adv_data = attack.batch_generate(test_images, test_labels, batch_size=32)
assert np.any(adv_data != test_images)
def test_ead():
"""UT for ensemble adversarial defense."""
num_classes = 10
batch_size = 64
sparse = False
context.set_context(mode=context.GRAPH_MODE)
context.set_context(device_target='Ascend')
# create test data
inputs = np.random.rand(batch_size, 1, 32, 32).astype(np.float32)
labels = np.random.randint(num_classes, size=batch_size).astype(np.int32)
if not sparse:
labels = np.eye(num_classes)[labels].astype(np.float32)
net = Net()
loss_fn = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=sparse)
optimizer = Momentum(net.trainable_params(), 0.001, 0.9)
net = Net()
fgsm = FastGradientSignMethod(net)
pgd = ProjectedGradientDescent(net)
ead = EnsembleAdversarialDefense(net, [fgsm, pgd],
loss_fn=loss_fn,
optimizer=optimizer)
LOGGER.set_level(logging.DEBUG)
LOGGER.debug(TAG, '---start ensemble adversarial defense--')
loss = ead.defense(inputs, labels)
LOGGER.debug(TAG, '---end ensemble adversarial defense--')
assert np.any(loss >= 0.0)
def __init__(self,
network,
eps=0.3,
eps_iter=0.1,
bounds=(0.0, 1.0),
is_targeted=False,
nb_iter=5,
loss_fn=None):
super(BasicIterativeMethod, self).__init__(network,
eps=eps,
eps_iter=eps_iter,
bounds=bounds,
nb_iter=nb_iter,
loss_fn=loss_fn)
self._is_targeted = check_param_type('is_targeted', is_targeted, bool)
self._attack = FastGradientSignMethod(self._network,
eps=self._eps_iter,
bounds=self._bounds,
is_targeted=self._is_targeted,
loss_fn=loss_fn)
def __init__(self,
network,
loss_fn=None,
optimizer=None,
bounds=(0.0, 1.0),
replace_ratio=0.5,
eps=0.1):
attack = FastGradientSignMethod(network,
eps=eps,
alpha=None,
bounds=bounds)
super(NaturalAdversarialDefense,
self).__init__(network, [attack],
loss_fn=loss_fn,
optimizer=optimizer,
bounds=bounds,
replace_ratio=replace_ratio)
class BasicIterativeMethod(IterativeGradientMethod):
"""
The Basic Iterative Method attack, an iterative FGSM method to generate
adversarial examples.
References: `A. Kurakin, I. Goodfellow, and S. Bengio, "Adversarial examples
in the physical world," in ICLR, 2017 <https://arxiv.org/abs/1607.02533>`_
Args:
network (Cell): Target model.
eps (float): Proportion of adversarial perturbation generated by the
attack to data range. Default: 0.3.
eps_iter (float): Proportion of single-step adversarial perturbation
generated by the attack to data range. Default: 0.1.
bounds (tuple): Upper and lower bounds of data, indicating the data range.
In form of (clip_min, clip_max). Default: (0.0, 1.0).
is_targeted (bool): If True, targeted attack. If False, untargeted
attack. Default: False.
nb_iter (int): Number of iteration. Default: 5.
loss_fn (Loss): Loss function for optimization. Default: None.
attack (class): The single step gradient method of each iteration. In
this class, FGSM is used.
Examples:
>>> attack = BasicIterativeMethod(network)
"""
def __init__(self,
network,
eps=0.3,
eps_iter=0.1,
bounds=(0.0, 1.0),
is_targeted=False,
nb_iter=5,
loss_fn=None):
super(BasicIterativeMethod, self).__init__(network,
eps=eps,
eps_iter=eps_iter,
bounds=bounds,
nb_iter=nb_iter,
loss_fn=loss_fn)
self._is_targeted = check_param_type('is_targeted', is_targeted, bool)
self._attack = FastGradientSignMethod(self._network,
eps=self._eps_iter,
bounds=self._bounds,
is_targeted=self._is_targeted,
loss_fn=loss_fn)
def generate(self, inputs, labels):
"""
Simple iterative FGSM method to generate adversarial examples.
Args:
inputs (numpy.ndarray): Benign input samples used as references to
create adversarial examples.
labels (numpy.ndarray): Original/target labels.
Returns:
numpy.ndarray, generated adversarial examples.
Examples:
>>> adv_x = attack.generate([[0.3, 0.2, 0.6],
>>> [0.3, 0.2, 0.4]],
>>> [[0, 0, 1, 0, 0, 0, 0, 0, 0, 0],
>>> [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]])
"""
inputs, labels = check_pair_numpy_param('inputs', inputs, 'labels',
labels)
arr_x = inputs
if self._bounds is not None:
clip_min, clip_max = self._bounds
clip_diff = clip_max - clip_min
for _ in range(self._nb_iter):
if 'self.prob' in globals():
d_inputs = _transform_inputs(inputs, self.prob)
else:
d_inputs = inputs
adv_x = self._attack.generate(d_inputs, labels)
perturs = np.clip(adv_x - arr_x, (0 - self._eps) * clip_diff,
self._eps * clip_diff)
adv_x = arr_x + perturs
inputs = adv_x
else:
for _ in range(self._nb_iter):
if 'self.prob' in globals():
d_inputs = _transform_inputs(inputs, self.prob)
else:
d_inputs = inputs
adv_x = self._attack.generate(d_inputs, labels)
adv_x = np.clip(adv_x, arr_x - self._eps, arr_x + self._eps)
inputs = adv_x
return adv_x
def test_fast_gradient_sign_method():
"""
FGSM-Attack test
"""
# upload trained network
ckpt_name = './trained_ckpt_file/checkpoint_lenet-10_1875.ckpt'
net = LeNet5()
load_dict = load_checkpoint(ckpt_name)
load_param_into_net(net, load_dict)
# get test data
data_list = "./MNIST_unzip/test"
batch_size = 32
ds = generate_mnist_dataset(data_list, batch_size, sparse=False)
# prediction accuracy before attack
model = Model(net)
batch_num = 3 # the number of batches of attacking samples
test_images = []
test_labels = []
predict_labels = []
i = 0
for data in ds.create_tuple_iterator():
i += 1
images = data[0].astype(np.float32)
labels = data[1]
test_images.append(images)
test_labels.append(labels)
pred_labels = np.argmax(model.predict(Tensor(images)).asnumpy(),
axis=1)
predict_labels.append(pred_labels)
if i >= batch_num:
break
predict_labels = np.concatenate(predict_labels)
true_labels = np.argmax(np.concatenate(test_labels), axis=1)
accuracy = np.mean(np.equal(predict_labels, true_labels))
LOGGER.info(TAG, "prediction accuracy before attacking is : %s", accuracy)
# attacking
attack = FastGradientSignMethod(net, eps=0.3)
start_time = time.clock()
adv_data = attack.batch_generate(np.concatenate(test_images),
np.concatenate(test_labels),
batch_size=32)
stop_time = time.clock()
np.save('./adv_data', adv_data)
pred_logits_adv = model.predict(Tensor(adv_data)).asnumpy()
# rescale predict confidences into (0, 1).
pred_logits_adv = softmax(pred_logits_adv, axis=1)
pred_labels_adv = np.argmax(pred_logits_adv, axis=1)
accuracy_adv = np.mean(np.equal(pred_labels_adv, true_labels))
LOGGER.info(TAG, "prediction accuracy after attacking is : %s",
accuracy_adv)
attack_evaluate = AttackEvaluate(
np.concatenate(test_images).transpose(0, 2, 3, 1),
np.concatenate(test_labels), adv_data.transpose(0, 2, 3, 1),
pred_logits_adv)
LOGGER.info(TAG, 'mis-classification rate of adversaries is : %s',
attack_evaluate.mis_classification_rate())
LOGGER.info(TAG, 'The average confidence of adversarial class is : %s',
attack_evaluate.avg_conf_adv_class())
LOGGER.info(TAG, 'The average confidence of true class is : %s',
attack_evaluate.avg_conf_true_class())
LOGGER.info(
TAG, 'The average distance (l0, l2, linf) between original '
'samples and adversarial samples are: %s',
attack_evaluate.avg_lp_distance())
LOGGER.info(
TAG, 'The average structural similarity between original '
'samples and adversarial samples are: %s', attack_evaluate.avg_ssim())
LOGGER.info(TAG, 'The average costing time is %s',
(stop_time - start_time) / (batch_num * batch_size))
