def test_lenet_mnist_coverage_cpu(): context.set_context(mode=context.GRAPH_MODE, device_target="CPU") # load network net = Net() model = Model(net) # initialize fuzz test with training dataset training_data = (np.random.random((10000, 10))*20).astype(np.float32) model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, training_data) # fuzz test with original test data # get test data test_data = (np.random.random((2000, 10))*20).astype(np.float32) test_labels = np.random.randint(0, 10, 2000).astype(np.int32) model_fuzz_test.test_adequacy_coverage_calculate(test_data) LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc()) LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc()) LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac()) # generate adv_data loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) attack = FastGradientSignMethod(net, eps=0.3, loss_fn=loss) adv_data = attack.batch_generate(test_data, test_labels, batch_size=32) model_fuzz_test.test_adequacy_coverage_calculate(adv_data, bias_coefficient=0.5) LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc()) LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc()) LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
def test_lenet_mnist_coverage_ascend(): context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") # load network net = Net() model = Model(net) # initialize fuzz test with training dataset training_data = (np.random.random((10000, 10))*20).astype(np.float32) model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, training_data) # fuzz test with original test data # get test data test_data = (np.random.random((2000, 10))*20).astype(np.float32) test_labels = np.random.randint(0, 10, 2000) test_labels = (np.eye(10)[test_labels]).astype(np.float32) model_fuzz_test.test_adequacy_coverage_calculate(test_data) LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc()) LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc()) LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac()) # generate adv_data attack = FastGradientSignMethod(net, eps=0.3) adv_data = attack.batch_generate(test_data, test_labels, batch_size=32) model_fuzz_test.test_adequacy_coverage_calculate(adv_data, bias_coefficient=0.5) LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc()) LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc()) LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
def test_fast_gradient_sign_method(): """ Fast gradient sign method unit test. """ input_np = np.asarray([[0.1, 0.2, 0.7]], np.float32) label = np.asarray([2], np.int32) label = np.eye(3)[label].astype(np.float32) attack = FastGradientSignMethod(Net()) ms_adv_x = attack.generate(input_np, label) assert np.any(ms_adv_x != input_np), 'Fast gradient sign method: generate' \ ' value must not be equal to' \ ' original value.'
def test_lenet_mnist_coverage(): context.set_context(mode=context.GRAPH_MODE, device_target="CPU") # upload trained network ckpt_name = './trained_ckpt_file/checkpoint_lenet-10_1875.ckpt' net = LeNet5() load_dict = load_checkpoint(ckpt_name) load_param_into_net(net, load_dict) model = Model(net) # get training data data_list = "./MNIST_unzip/train" batch_size = 32 ds = generate_mnist_dataset(data_list, batch_size, sparse=True) train_images = [] for data in ds.create_tuple_iterator(): images = data[0].astype(np.float32) train_images.append(images) train_images = np.concatenate(train_images, axis=0) # initialize fuzz test with training dataset model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, train_images) # fuzz test with original test data # get test data data_list = "./MNIST_unzip/test" batch_size = 32 ds = generate_mnist_dataset(data_list, batch_size, sparse=True) test_images = [] test_labels = [] for data in ds.create_tuple_iterator(): images = data[0].astype(np.float32) labels = data[1] test_images.append(images) test_labels.append(labels) test_images = np.concatenate(test_images, axis=0) test_labels = np.concatenate(test_labels, axis=0) model_fuzz_test.test_adequacy_coverage_calculate(test_images) LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc()) LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc()) LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac()) # generate adv_data loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) attack = FastGradientSignMethod(net, eps=0.3, loss_fn=loss) adv_data = attack.batch_generate(test_images, test_labels, batch_size=32) model_fuzz_test.test_adequacy_coverage_calculate(adv_data, bias_coefficient=0.5) LOGGER.info(TAG, 'KMNC of this test is : %s', model_fuzz_test.get_kmnc()) LOGGER.info(TAG, 'NBC of this test is : %s', model_fuzz_test.get_nbc()) LOGGER.info(TAG, 'SNAC of this test is : %s', model_fuzz_test.get_snac())
def test_fast_gradient_sign_method(): """ FGSM-Attack test """ context.set_context(mode=context.GRAPH_MODE) # get network net = resnet50_cifar10(10) # create test data test_images = np.random.rand(64, 3, 224, 224).astype(np.float32) test_labels = np.random.randint(10, size=64).astype(np.int32) # attacking loss_fn = CrossEntropyLoss() attack = FastGradientSignMethod(net, eps=0.1, loss_fn=loss_fn) adv_data = attack.batch_generate(test_images, test_labels, batch_size=32) assert np.any(adv_data != test_images)
def test_ead(): """UT for ensemble adversarial defense.""" num_classes = 10 batch_size = 64 sparse = False context.set_context(mode=context.GRAPH_MODE) context.set_context(device_target='Ascend') # create test data inputs = np.random.rand(batch_size, 1, 32, 32).astype(np.float32) labels = np.random.randint(num_classes, size=batch_size).astype(np.int32) if not sparse: labels = np.eye(num_classes)[labels].astype(np.float32) net = Net() loss_fn = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=sparse) optimizer = Momentum(net.trainable_params(), 0.001, 0.9) net = Net() fgsm = FastGradientSignMethod(net) pgd = ProjectedGradientDescent(net) ead = EnsembleAdversarialDefense(net, [fgsm, pgd], loss_fn=loss_fn, optimizer=optimizer) LOGGER.set_level(logging.DEBUG) LOGGER.debug(TAG, '---start ensemble adversarial defense--') loss = ead.defense(inputs, labels) LOGGER.debug(TAG, '---end ensemble adversarial defense--') assert np.any(loss >= 0.0)
def __init__(self, network, eps=0.3, eps_iter=0.1, bounds=(0.0, 1.0), is_targeted=False, nb_iter=5, loss_fn=None): super(BasicIterativeMethod, self).__init__(network, eps=eps, eps_iter=eps_iter, bounds=bounds, nb_iter=nb_iter, loss_fn=loss_fn) self._is_targeted = check_param_type('is_targeted', is_targeted, bool) self._attack = FastGradientSignMethod(self._network, eps=self._eps_iter, bounds=self._bounds, is_targeted=self._is_targeted, loss_fn=loss_fn)
def __init__(self, network, loss_fn=None, optimizer=None, bounds=(0.0, 1.0), replace_ratio=0.5, eps=0.1): attack = FastGradientSignMethod(network, eps=eps, alpha=None, bounds=bounds) super(NaturalAdversarialDefense, self).__init__(network, [attack], loss_fn=loss_fn, optimizer=optimizer, bounds=bounds, replace_ratio=replace_ratio)
class BasicIterativeMethod(IterativeGradientMethod): """ The Basic Iterative Method attack, an iterative FGSM method to generate adversarial examples. References: `A. Kurakin, I. Goodfellow, and S. Bengio, "Adversarial examples in the physical world," in ICLR, 2017 <https://arxiv.org/abs/1607.02533>`_ Args: network (Cell): Target model. eps (float): Proportion of adversarial perturbation generated by the attack to data range. Default: 0.3. eps_iter (float): Proportion of single-step adversarial perturbation generated by the attack to data range. Default: 0.1. bounds (tuple): Upper and lower bounds of data, indicating the data range. In form of (clip_min, clip_max). Default: (0.0, 1.0). is_targeted (bool): If True, targeted attack. If False, untargeted attack. Default: False. nb_iter (int): Number of iteration. Default: 5. loss_fn (Loss): Loss function for optimization. Default: None. attack (class): The single step gradient method of each iteration. In this class, FGSM is used. Examples: >>> attack = BasicIterativeMethod(network) """ def __init__(self, network, eps=0.3, eps_iter=0.1, bounds=(0.0, 1.0), is_targeted=False, nb_iter=5, loss_fn=None): super(BasicIterativeMethod, self).__init__(network, eps=eps, eps_iter=eps_iter, bounds=bounds, nb_iter=nb_iter, loss_fn=loss_fn) self._is_targeted = check_param_type('is_targeted', is_targeted, bool) self._attack = FastGradientSignMethod(self._network, eps=self._eps_iter, bounds=self._bounds, is_targeted=self._is_targeted, loss_fn=loss_fn) def generate(self, inputs, labels): """ Simple iterative FGSM method to generate adversarial examples. Args: inputs (numpy.ndarray): Benign input samples used as references to create adversarial examples. labels (numpy.ndarray): Original/target labels. Returns: numpy.ndarray, generated adversarial examples. Examples: >>> adv_x = attack.generate([[0.3, 0.2, 0.6], >>> [0.3, 0.2, 0.4]], >>> [[0, 0, 1, 0, 0, 0, 0, 0, 0, 0], >>> [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]]) """ inputs, labels = check_pair_numpy_param('inputs', inputs, 'labels', labels) arr_x = inputs if self._bounds is not None: clip_min, clip_max = self._bounds clip_diff = clip_max - clip_min for _ in range(self._nb_iter): if 'self.prob' in globals(): d_inputs = _transform_inputs(inputs, self.prob) else: d_inputs = inputs adv_x = self._attack.generate(d_inputs, labels) perturs = np.clip(adv_x - arr_x, (0 - self._eps) * clip_diff, self._eps * clip_diff) adv_x = arr_x + perturs inputs = adv_x else: for _ in range(self._nb_iter): if 'self.prob' in globals(): d_inputs = _transform_inputs(inputs, self.prob) else: d_inputs = inputs adv_x = self._attack.generate(d_inputs, labels) adv_x = np.clip(adv_x, arr_x - self._eps, arr_x + self._eps) inputs = adv_x return adv_x
def test_fast_gradient_sign_method(): """ FGSM-Attack test """ # upload trained network ckpt_name = './trained_ckpt_file/checkpoint_lenet-10_1875.ckpt' net = LeNet5() load_dict = load_checkpoint(ckpt_name) load_param_into_net(net, load_dict) # get test data data_list = "./MNIST_unzip/test" batch_size = 32 ds = generate_mnist_dataset(data_list, batch_size, sparse=False) # prediction accuracy before attack model = Model(net) batch_num = 3 # the number of batches of attacking samples test_images = [] test_labels = [] predict_labels = [] i = 0 for data in ds.create_tuple_iterator(): i += 1 images = data[0].astype(np.float32) labels = data[1] test_images.append(images) test_labels.append(labels) pred_labels = np.argmax(model.predict(Tensor(images)).asnumpy(), axis=1) predict_labels.append(pred_labels) if i >= batch_num: break predict_labels = np.concatenate(predict_labels) true_labels = np.argmax(np.concatenate(test_labels), axis=1) accuracy = np.mean(np.equal(predict_labels, true_labels)) LOGGER.info(TAG, "prediction accuracy before attacking is : %s", accuracy) # attacking attack = FastGradientSignMethod(net, eps=0.3) start_time = time.clock() adv_data = attack.batch_generate(np.concatenate(test_images), np.concatenate(test_labels), batch_size=32) stop_time = time.clock() np.save('./adv_data', adv_data) pred_logits_adv = model.predict(Tensor(adv_data)).asnumpy() # rescale predict confidences into (0, 1). pred_logits_adv = softmax(pred_logits_adv, axis=1) pred_labels_adv = np.argmax(pred_logits_adv, axis=1) accuracy_adv = np.mean(np.equal(pred_labels_adv, true_labels)) LOGGER.info(TAG, "prediction accuracy after attacking is : %s", accuracy_adv) attack_evaluate = AttackEvaluate( np.concatenate(test_images).transpose(0, 2, 3, 1), np.concatenate(test_labels), adv_data.transpose(0, 2, 3, 1), pred_logits_adv) LOGGER.info(TAG, 'mis-classification rate of adversaries is : %s', attack_evaluate.mis_classification_rate()) LOGGER.info(TAG, 'The average confidence of adversarial class is : %s', attack_evaluate.avg_conf_adv_class()) LOGGER.info(TAG, 'The average confidence of true class is : %s', attack_evaluate.avg_conf_true_class()) LOGGER.info( TAG, 'The average distance (l0, l2, linf) between original ' 'samples and adversarial samples are: %s', attack_evaluate.avg_lp_distance()) LOGGER.info( TAG, 'The average structural similarity between original ' 'samples and adversarial samples are: %s', attack_evaluate.avg_ssim()) LOGGER.info(TAG, 'The average costing time is %s', (stop_time - start_time) / (batch_num * batch_size))