def test_lenet_mnist_coverage_cpu():
context.set_context(mode=context.GRAPH_MODE, device_target="CPU")
# load network
net = Net()
model = Model(net)
# initialize fuzz test with training dataset
neuron_num = 10
segmented_num = 1000
training_data = (np.random.random((10000, 10)) * 20).astype(np.float32)
model_fuzz_test = ModelCoverageMetrics(model, neuron_num, segmented_num,
training_data)
# fuzz test with original test data
# get test data
test_data = (np.random.random((2000, 10)) * 20).astype(np.float32)
test_labels = np.random.randint(0, 10, 2000).astype(np.int32)
model_fuzz_test.calculate_coverage(test_data)
LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc())
LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc())
LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
# generate adv_data
loss = SoftmaxCrossEntropyWithLogits(sparse=True)
attack = FastGradientSignMethod(net, eps=0.3, loss_fn=loss)
adv_data = attack.batch_generate(test_data, test_labels, batch_size=32)
model_fuzz_test.calculate_coverage(adv_data, bias_coefficient=0.5)
LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc())
LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc())
LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
def test_lenet_mnist_coverage_ascend():
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
# load network
net = Net()
model = Model(net)
# initialize fuzz test with training dataset
training_data = (np.random.random((10000, 10))*20).astype(np.float32)
model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, training_data)
# fuzz test with original test data
# get test data
test_data = (np.random.random((2000, 10))*20).astype(np.float32)
test_labels = np.random.randint(0, 10, 2000)
test_labels = (np.eye(10)[test_labels]).astype(np.float32)
model_fuzz_test.calculate_coverage(test_data)
LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc())
LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc())
LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
# generate adv_data
attack = FastGradientSignMethod(net, eps=0.3)
adv_data = attack.batch_generate(test_data, test_labels, batch_size=32)
model_fuzz_test.calculate_coverage(adv_data, bias_coefficient=0.5)
LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc())
LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc())
LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
def test_lenet_mnist_coverage():
# upload trained network
ckpt_path = '../common/networks/lenet5/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt'
net = LeNet5()
load_dict = load_checkpoint(ckpt_path)
load_param_into_net(net, load_dict)
model = Model(net)
# get training data
data_list = "../common/dataset/MNIST/train"
batch_size = 32
ds = generate_mnist_dataset(data_list, batch_size, sparse=True)
train_images = []
for data in ds.create_tuple_iterator(output_numpy=True):
images = data[0].astype(np.float32)
train_images.append(images)
train_images = np.concatenate(train_images, axis=0)
# initialize fuzz test with training dataset
model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, train_images)
# fuzz test with original test data
# get test data
data_list = "../common/dataset/MNIST/test"
batch_size = 32
ds = generate_mnist_dataset(data_list, batch_size, sparse=True)
test_images = []
test_labels = []
for data in ds.create_tuple_iterator(output_numpy=True):
images = data[0].astype(np.float32)
labels = data[1]
test_images.append(images)
test_labels.append(labels)
test_images = np.concatenate(test_images, axis=0)
test_labels = np.concatenate(test_labels, axis=0)
model_fuzz_test.calculate_coverage(test_images)
LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc())
LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc())
LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
# generate adv_data
loss = SoftmaxCrossEntropyWithLogits(sparse=True)
attack = FastGradientSignMethod(net, eps=0.3, loss_fn=loss)
adv_data = attack.batch_generate(test_images, test_labels, batch_size=32)
model_fuzz_test.calculate_coverage(adv_data, bias_coefficient=0.5)
LOGGER.info(TAG, 'KMNC of this adv data is : %s', model_fuzz_test.get_kmnc())
LOGGER.info(TAG, 'NBC of this adv data is : %s', model_fuzz_test.get_nbc())
LOGGER.info(TAG, 'SNAC of this adv data is : %s', model_fuzz_test.get_snac())
def test_fast_gradient_sign_method():
"""
FGSM-Attack test
"""
context.set_context(mode=context.GRAPH_MODE)
# get network
net = resnet50_cifar10(10)
# create test data
test_images = np.random.rand(64, 3, 224, 224).astype(np.float32)
test_labels = np.random.randint(10, size=64).astype(np.int32)
# attacking
loss_fn = CrossEntropyLoss()
attack = FastGradientSignMethod(net, eps=0.1, loss_fn=loss_fn)
adv_data = attack.batch_generate(test_images, test_labels, batch_size=32)
assert np.any(adv_data != test_images)
def test_nad_method():
"""
NAD-Defense test.
"""
mnist_path = "../../common/dataset/MNIST"
batch_size = 32
# 1. train original model
ds_train = generate_mnist_dataset(os.path.join(mnist_path, "train"),
batch_size=batch_size,
repeat_size=1)
net = LeNet5()
loss = SoftmaxCrossEntropyWithLogits(sparse=True)
opt = nn.Momentum(net.trainable_params(), 0.01, 0.09)
model = Model(net, loss, opt, metrics=None)
model.train(10,
ds_train,
callbacks=[LossMonitor()],
dataset_sink_mode=False)
# 2. get test data
ds_test = generate_mnist_dataset(os.path.join(mnist_path, "test"),
batch_size=batch_size,
repeat_size=1)
inputs = []
labels = []
for data in ds_test.create_tuple_iterator():
inputs.append(data[0].asnumpy().astype(np.float32))
labels.append(data[1].asnumpy())
inputs = np.concatenate(inputs)
labels = np.concatenate(labels)
# 3. get accuracy of test data on original model
net.set_train(False)
acc_list = []
batchs = inputs.shape[0] // batch_size
for i in range(batchs):
batch_inputs = inputs[i * batch_size:(i + 1) * batch_size]
batch_labels = labels[i * batch_size:(i + 1) * batch_size]
logits = net(Tensor(batch_inputs)).asnumpy()
label_pred = np.argmax(logits, axis=1)
acc_list.append(np.mean(batch_labels == label_pred))
LOGGER.info(TAG, 'accuracy of TEST data on original model is : %s',
np.mean(acc_list))
# 4. get adv of test data
attack = FastGradientSignMethod(net, eps=0.3, loss_fn=loss)
adv_data = attack.batch_generate(inputs, labels)
LOGGER.info(TAG, 'adv_data.shape is : %s', adv_data.shape)
# 5. get accuracy of adv data on original model
acc_list = []
batchs = adv_data.shape[0] // batch_size
for i in range(batchs):
batch_inputs = adv_data[i * batch_size:(i + 1) * batch_size]
batch_labels = labels[i * batch_size:(i + 1) * batch_size]
logits = net(Tensor(batch_inputs)).asnumpy()
label_pred = np.argmax(logits, axis=1)
acc_list.append(np.mean(batch_labels == label_pred))
LOGGER.info(TAG, 'accuracy of adv data on original model is : %s',
np.mean(acc_list))
# 6. defense
ds_train = generate_mnist_dataset(os.path.join(mnist_path, "train"),
batch_size=batch_size,
repeat_size=1)
inputs_train = []
labels_train = []
for data in ds_train.create_tuple_iterator():
inputs_train.append(data[0].asnumpy().astype(np.float32))
labels_train.append(data[1].asnumpy())
inputs_train = np.concatenate(inputs_train)
labels_train = np.concatenate(labels_train)
net.set_train()
nad = NaturalAdversarialDefense(net,
loss_fn=loss,
optimizer=opt,
bounds=(0.0, 1.0),
eps=0.3)
nad.batch_defense(inputs_train, labels_train, batch_size=32, epochs=10)
# 7. get accuracy of test data on defensed model
net.set_train(False)
acc_list = []
batchs = inputs.shape[0] // batch_size
for i in range(batchs):
batch_inputs = inputs[i * batch_size:(i + 1) * batch_size]
batch_labels = labels[i * batch_size:(i + 1) * batch_size]
logits = net(Tensor(batch_inputs)).asnumpy()
label_pred = np.argmax(logits, axis=1)
acc_list.append(np.mean(batch_labels == label_pred))
LOGGER.info(TAG, 'accuracy of TEST data on defensed model is : %s',
np.mean(acc_list))
# 8. get accuracy of adv data on defensed model
acc_list = []
batchs = adv_data.shape[0] // batch_size
for i in range(batchs):
batch_inputs = adv_data[i * batch_size:(i + 1) * batch_size]
batch_labels = labels[i * batch_size:(i + 1) * batch_size]
logits = net(Tensor(batch_inputs)).asnumpy()
label_pred = np.argmax(logits, axis=1)
acc_list.append(np.mean(batch_labels == label_pred))
LOGGER.info(TAG, 'accuracy of adv data on defensed model is : %s',
np.mean(acc_list))
def test_fast_gradient_sign_method():
"""
FGSM-Attack test for CPU device.
"""
# upload trained network
ckpt_path = '../../../common/networks/lenet5/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt'
net = LeNet5()
load_dict = load_checkpoint(ckpt_path)
load_param_into_net(net, load_dict)
# get test data
data_list = "../../../common/dataset/MNIST/test"
batch_size = 32
ds = generate_mnist_dataset(data_list, batch_size)
# prediction accuracy before attack
model = Model(net)
batch_num = 3 # the number of batches of attacking samples
test_images = []
test_labels = []
predict_labels = []
i = 0
for data in ds.create_tuple_iterator(output_numpy=True):
i += 1
images = data[0].astype(np.float32)
labels = data[1]
test_images.append(images)
test_labels.append(labels)
pred_labels = np.argmax(model.predict(Tensor(images)).asnumpy(),
axis=1)
predict_labels.append(pred_labels)
if i >= batch_num:
break
predict_labels = np.concatenate(predict_labels)
true_labels = np.concatenate(test_labels)
accuracy = np.mean(np.equal(predict_labels, true_labels))
LOGGER.info(TAG, "prediction accuracy before attacking is : %s", accuracy)
# attacking
loss = SoftmaxCrossEntropyWithLogits(sparse=True)
attack = FastGradientSignMethod(net, eps=0.3, loss_fn=loss)
start_time = time.clock()
adv_data = attack.batch_generate(np.concatenate(test_images),
true_labels, batch_size=32)
stop_time = time.clock()
np.save('./adv_data', adv_data)
pred_logits_adv = model.predict(Tensor(adv_data)).asnumpy()
# rescale predict confidences into (0, 1).
pred_logits_adv = softmax(pred_logits_adv, axis=1)
pred_labels_adv = np.argmax(pred_logits_adv, axis=1)
accuracy_adv = np.mean(np.equal(pred_labels_adv, true_labels))
LOGGER.info(TAG, "prediction accuracy after attacking is : %s", accuracy_adv)
attack_evaluate = AttackEvaluate(np.concatenate(test_images).transpose(0, 2, 3, 1),
np.eye(10)[true_labels],
adv_data.transpose(0, 2, 3, 1),
pred_logits_adv)
LOGGER.info(TAG, 'mis-classification rate of adversaries is : %s',
attack_evaluate.mis_classification_rate())
LOGGER.info(TAG, 'The average confidence of adversarial class is : %s',
attack_evaluate.avg_conf_adv_class())
LOGGER.info(TAG, 'The average confidence of true class is : %s',
attack_evaluate.avg_conf_true_class())
LOGGER.info(TAG, 'The average distance (l0, l2, linf) between original '
'samples and adversarial samples are: %s',
attack_evaluate.avg_lp_distance())
LOGGER.info(TAG, 'The average structural similarity between original '
'samples and adversarial samples are: %s',
attack_evaluate.avg_ssim())
LOGGER.info(TAG, 'The average costing time is %s',
(stop_time - start_time)/(batch_num*batch_size))