def test_lenet_mnist_coverage_cpu(): context.set_context(mode=context.GRAPH_MODE, device_target="CPU") # load network net = Net() model = Model(net) # initialize fuzz test with training dataset neuron_num = 10 segmented_num = 1000 training_data = (np.random.random((10000, 10)) * 20).astype(np.float32) model_fuzz_test = ModelCoverageMetrics(model, neuron_num, segmented_num, training_data) # fuzz test with original test data # get test data test_data = (np.random.random((2000, 10)) * 20).astype(np.float32) test_labels = np.random.randint(0, 10, 2000).astype(np.int32) model_fuzz_test.calculate_coverage(test_data) LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc()) LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc()) LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac()) # generate adv_data loss = SoftmaxCrossEntropyWithLogits(sparse=True) attack = FastGradientSignMethod(net, eps=0.3, loss_fn=loss) adv_data = attack.batch_generate(test_data, test_labels, batch_size=32) model_fuzz_test.calculate_coverage(adv_data, bias_coefficient=0.5) LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc()) LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc()) LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
def test_lenet_mnist_coverage_ascend(): context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") # load network net = Net() model = Model(net) # initialize fuzz test with training dataset training_data = (np.random.random((10000, 10))*20).astype(np.float32) model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, training_data) # fuzz test with original test data # get test data test_data = (np.random.random((2000, 10))*20).astype(np.float32) test_labels = np.random.randint(0, 10, 2000) test_labels = (np.eye(10)[test_labels]).astype(np.float32) model_fuzz_test.calculate_coverage(test_data) LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc()) LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc()) LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac()) # generate adv_data attack = FastGradientSignMethod(net, eps=0.3) adv_data = attack.batch_generate(test_data, test_labels, batch_size=32) model_fuzz_test.calculate_coverage(adv_data, bias_coefficient=0.5) LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc()) LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc()) LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
def test_lenet_mnist_coverage(): # upload trained network ckpt_path = '../common/networks/lenet5/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt' net = LeNet5() load_dict = load_checkpoint(ckpt_path) load_param_into_net(net, load_dict) model = Model(net) # get training data data_list = "../common/dataset/MNIST/train" batch_size = 32 ds = generate_mnist_dataset(data_list, batch_size, sparse=True) train_images = [] for data in ds.create_tuple_iterator(output_numpy=True): images = data[0].astype(np.float32) train_images.append(images) train_images = np.concatenate(train_images, axis=0) # initialize fuzz test with training dataset model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, train_images) # fuzz test with original test data # get test data data_list = "../common/dataset/MNIST/test" batch_size = 32 ds = generate_mnist_dataset(data_list, batch_size, sparse=True) test_images = [] test_labels = [] for data in ds.create_tuple_iterator(output_numpy=True): images = data[0].astype(np.float32) labels = data[1] test_images.append(images) test_labels.append(labels) test_images = np.concatenate(test_images, axis=0) test_labels = np.concatenate(test_labels, axis=0) model_fuzz_test.calculate_coverage(test_images) LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc()) LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc()) LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac()) # generate adv_data loss = SoftmaxCrossEntropyWithLogits(sparse=True) attack = FastGradientSignMethod(net, eps=0.3, loss_fn=loss) adv_data = attack.batch_generate(test_images, test_labels, batch_size=32) model_fuzz_test.calculate_coverage(adv_data, bias_coefficient=0.5) LOGGER.info(TAG, 'KMNC of this adv data is : %s', model_fuzz_test.get_kmnc()) LOGGER.info(TAG, 'NBC of this adv data is : %s', model_fuzz_test.get_nbc()) LOGGER.info(TAG, 'SNAC of this adv data is : %s', model_fuzz_test.get_snac())
def test_fast_gradient_sign_method(): """ FGSM-Attack test """ context.set_context(mode=context.GRAPH_MODE) # get network net = resnet50_cifar10(10) # create test data test_images = np.random.rand(64, 3, 224, 224).astype(np.float32) test_labels = np.random.randint(10, size=64).astype(np.int32) # attacking loss_fn = CrossEntropyLoss() attack = FastGradientSignMethod(net, eps=0.1, loss_fn=loss_fn) adv_data = attack.batch_generate(test_images, test_labels, batch_size=32) assert np.any(adv_data != test_images)
def test_nad_method(): """ NAD-Defense test. """ mnist_path = "../../common/dataset/MNIST" batch_size = 32 # 1. train original model ds_train = generate_mnist_dataset(os.path.join(mnist_path, "train"), batch_size=batch_size, repeat_size=1) net = LeNet5() loss = SoftmaxCrossEntropyWithLogits(sparse=True) opt = nn.Momentum(net.trainable_params(), 0.01, 0.09) model = Model(net, loss, opt, metrics=None) model.train(10, ds_train, callbacks=[LossMonitor()], dataset_sink_mode=False) # 2. get test data ds_test = generate_mnist_dataset(os.path.join(mnist_path, "test"), batch_size=batch_size, repeat_size=1) inputs = [] labels = [] for data in ds_test.create_tuple_iterator(): inputs.append(data[0].asnumpy().astype(np.float32)) labels.append(data[1].asnumpy()) inputs = np.concatenate(inputs) labels = np.concatenate(labels) # 3. get accuracy of test data on original model net.set_train(False) acc_list = [] batchs = inputs.shape[0] // batch_size for i in range(batchs): batch_inputs = inputs[i * batch_size:(i + 1) * batch_size] batch_labels = labels[i * batch_size:(i + 1) * batch_size] logits = net(Tensor(batch_inputs)).asnumpy() label_pred = np.argmax(logits, axis=1) acc_list.append(np.mean(batch_labels == label_pred)) LOGGER.info(TAG, 'accuracy of TEST data on original model is : %s', np.mean(acc_list)) # 4. get adv of test data attack = FastGradientSignMethod(net, eps=0.3, loss_fn=loss) adv_data = attack.batch_generate(inputs, labels) LOGGER.info(TAG, 'adv_data.shape is : %s', adv_data.shape) # 5. get accuracy of adv data on original model acc_list = [] batchs = adv_data.shape[0] // batch_size for i in range(batchs): batch_inputs = adv_data[i * batch_size:(i + 1) * batch_size] batch_labels = labels[i * batch_size:(i + 1) * batch_size] logits = net(Tensor(batch_inputs)).asnumpy() label_pred = np.argmax(logits, axis=1) acc_list.append(np.mean(batch_labels == label_pred)) LOGGER.info(TAG, 'accuracy of adv data on original model is : %s', np.mean(acc_list)) # 6. defense ds_train = generate_mnist_dataset(os.path.join(mnist_path, "train"), batch_size=batch_size, repeat_size=1) inputs_train = [] labels_train = [] for data in ds_train.create_tuple_iterator(): inputs_train.append(data[0].asnumpy().astype(np.float32)) labels_train.append(data[1].asnumpy()) inputs_train = np.concatenate(inputs_train) labels_train = np.concatenate(labels_train) net.set_train() nad = NaturalAdversarialDefense(net, loss_fn=loss, optimizer=opt, bounds=(0.0, 1.0), eps=0.3) nad.batch_defense(inputs_train, labels_train, batch_size=32, epochs=10) # 7. get accuracy of test data on defensed model net.set_train(False) acc_list = [] batchs = inputs.shape[0] // batch_size for i in range(batchs): batch_inputs = inputs[i * batch_size:(i + 1) * batch_size] batch_labels = labels[i * batch_size:(i + 1) * batch_size] logits = net(Tensor(batch_inputs)).asnumpy() label_pred = np.argmax(logits, axis=1) acc_list.append(np.mean(batch_labels == label_pred)) LOGGER.info(TAG, 'accuracy of TEST data on defensed model is : %s', np.mean(acc_list)) # 8. get accuracy of adv data on defensed model acc_list = [] batchs = adv_data.shape[0] // batch_size for i in range(batchs): batch_inputs = adv_data[i * batch_size:(i + 1) * batch_size] batch_labels = labels[i * batch_size:(i + 1) * batch_size] logits = net(Tensor(batch_inputs)).asnumpy() label_pred = np.argmax(logits, axis=1) acc_list.append(np.mean(batch_labels == label_pred)) LOGGER.info(TAG, 'accuracy of adv data on defensed model is : %s', np.mean(acc_list))
def test_fast_gradient_sign_method(): """ FGSM-Attack test for CPU device. """ # upload trained network ckpt_path = '../../../common/networks/lenet5/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt' net = LeNet5() load_dict = load_checkpoint(ckpt_path) load_param_into_net(net, load_dict) # get test data data_list = "../../../common/dataset/MNIST/test" batch_size = 32 ds = generate_mnist_dataset(data_list, batch_size) # prediction accuracy before attack model = Model(net) batch_num = 3 # the number of batches of attacking samples test_images = [] test_labels = [] predict_labels = [] i = 0 for data in ds.create_tuple_iterator(output_numpy=True): i += 1 images = data[0].astype(np.float32) labels = data[1] test_images.append(images) test_labels.append(labels) pred_labels = np.argmax(model.predict(Tensor(images)).asnumpy(), axis=1) predict_labels.append(pred_labels) if i >= batch_num: break predict_labels = np.concatenate(predict_labels) true_labels = np.concatenate(test_labels) accuracy = np.mean(np.equal(predict_labels, true_labels)) LOGGER.info(TAG, "prediction accuracy before attacking is : %s", accuracy) # attacking loss = SoftmaxCrossEntropyWithLogits(sparse=True) attack = FastGradientSignMethod(net, eps=0.3, loss_fn=loss) start_time = time.clock() adv_data = attack.batch_generate(np.concatenate(test_images), true_labels, batch_size=32) stop_time = time.clock() np.save('./adv_data', adv_data) pred_logits_adv = model.predict(Tensor(adv_data)).asnumpy() # rescale predict confidences into (0, 1). pred_logits_adv = softmax(pred_logits_adv, axis=1) pred_labels_adv = np.argmax(pred_logits_adv, axis=1) accuracy_adv = np.mean(np.equal(pred_labels_adv, true_labels)) LOGGER.info(TAG, "prediction accuracy after attacking is : %s", accuracy_adv) attack_evaluate = AttackEvaluate(np.concatenate(test_images).transpose(0, 2, 3, 1), np.eye(10)[true_labels], adv_data.transpose(0, 2, 3, 1), pred_logits_adv) LOGGER.info(TAG, 'mis-classification rate of adversaries is : %s', attack_evaluate.mis_classification_rate()) LOGGER.info(TAG, 'The average confidence of adversarial class is : %s', attack_evaluate.avg_conf_adv_class()) LOGGER.info(TAG, 'The average confidence of true class is : %s', attack_evaluate.avg_conf_true_class()) LOGGER.info(TAG, 'The average distance (l0, l2, linf) between original ' 'samples and adversarial samples are: %s', attack_evaluate.avg_lp_distance()) LOGGER.info(TAG, 'The average structural similarity between original ' 'samples and adversarial samples are: %s', attack_evaluate.avg_ssim()) LOGGER.info(TAG, 'The average costing time is %s', (stop_time - start_time)/(batch_num*batch_size))