Пример #1
0
def test_genattack_numpy(request: Any) -> None:
    class Model:
        def __call__(self, inputs: Any) -> Any:
            return inputs.mean(axis=(2, 3))

    model = Model()
    with pytest.raises(ValueError):
        fbn.NumPyModel(model, bounds=(0, 1), data_format="foo")

    fmodel = fbn.NumPyModel(model, bounds=(0, 1))
    x, y = ep.astensors(
        *fbn.samples(
            fmodel, dataset="imagenet", batchsize=16, data_format="channels_first"
        )
    )

    with pytest.raises(ValueError, match="data_format"):
        fbn.attacks.GenAttack(reduced_dims=(2, 2)).run(
            fmodel, x, fbn.TargetedMisclassification(y), epsilon=0.3
        )

    with pytest.raises(ValueError, match="channel_axis"):
        fbn.attacks.GenAttack(channel_axis=2, reduced_dims=(2, 2)).run(
            fmodel, x, fbn.TargetedMisclassification(y), epsilon=0.3
        )
Пример #2
0
def main() -> None:
    # instantiate a model (could also be a TensorFlow or JAX model)
    model = models.resnet18(pretrained=True).eval()
    preprocessing = dict(mean=[0.485, 0.456, 0.406],
                         std=[0.229, 0.224, 0.225],
                         axis=-3)
    fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)

    # get data and test the model
    # wrapping the tensors with ep.astensors is optional, but it allows
    # us to work with EagerPy tensors in the following
    images, labels = ep.astensors(
        *samples(fmodel, dataset="imagenet", batchsize=16))
    clean_acc = accuracy(fmodel, images, labels) * 100
    print(f"clean accuracy:  {clean_acc:.1f} %")

    # the attack trys a combination of specified rotations and translations to an image
    # stops early if adversarial shifts and translations for all images are found
    attack = fa.SpatialAttack(
        max_translation=6,  # 6px so x in [x-6, x+6] and y in [y-6, y+6]
        num_translations=6,  # number of translations in x, y.
        max_rotation=20,  # +- rotation in degrees
        num_rotations=5,  # number of rotations
        # max total iterations = num_rotations * num_translations**2
    )

    # report the success rate of the attack (percentage of samples that could
    # be adversarially perturbed) and the robust accuracy (the remaining accuracy
    # of the model when it is attacked)
    xp_, _, success = attack(fmodel, images, labels)
    suc = success.float32().mean().item() * 100
    print(f"attack success:  {suc:.1f} %"
          " (for the specified rotation and translation bounds)")
    print(f"robust accuracy: {100 - suc:.1f} %"
          " (for the specified rotation and translation bounds)")
Пример #3
0
def test_astensors_tensor(t: Tensor) -> None:
    ts = (t, t + 1, t + 2)
    ys = ep.astensors(*ts)
    assert isinstance(ys, tuple)
    assert len(ts) == len(ys)
    for ti, yi in zip(ts, ys):
        assert (ti == yi).all()
Пример #4
0
def accuracy(fmodel: Model, inputs: Any, labels: Any) -> float:
    inputs_, labels_ = ep.astensors(inputs, labels)
    del inputs, labels

    predictions = fmodel(inputs_).argmax(axis=-1)
    accuracy = (predictions == labels_).float32().mean()
    return accuracy.item()
Пример #5
0
    def accuracy(self, inputs, labels):
        inputs_, labels_ = ep.astensors(inputs, labels)
        del inputs, labels

        self.model.eval()  # make sure in evaluating mode ...
        predictions = self.fmodel(inputs_).argmax(axis=-1)
        accuracy = (predictions == labels_)
        return accuracy.sum().item()
Пример #6
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--steps',
                        type=int,
                        default=20000,
                        help='Iteration of BA')
    parser.add_argument('--targeted',
                        action='store',
                        default=False,
                        help='For targeted attack')

    args = parser.parse_args()

    model = Net()
    model.load_state_dict(torch.load('mnist_cnn.pt'))
    model.eval()

    preprocessing = dict(mean=0.1307, std=0.3081)
    fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)

    fmodel = fmodel.transform_bounds((0, 1))
    assert fmodel.bounds == (0, 1)

    images, labels = ep.astensors(
        *samples(fmodel, dataset="mnist", batchsize=10))

    print('Model accuracy on clean examples: {}'.format(
        accuracy(fmodel, images, labels)))

    if args.targeted:
        target_class = (labels + 7) % 10
        criterion = fb.criteria.TargetedMisclassification(target_class)
    else:
        criterion = fb.criteria.Misclassification(labels)

    attack = fa.BoundaryAttack(steps=args.steps, tensorboard=None)
    epsilons = np.linspace(0.01, 10, 20)
    raw, clipped, success = attack(fmodel, images, labels, epsilons=epsilons)

    robust_accuracy = 1 - success.float32().mean(axis=-1)

    plt.plot(epsilons, robust_accuracy.numpy())
    plt.xlabel("Epsilons")
    plt.ylabel("Robust Accuracy")
    plt.savefig('mnist_BA_robust_acc.jpg')
    plt.show()

    mean_distance = []
    for i in range(len(clipped)):
        dist = np.mean(fb.distances.l2(clipped[i], images).numpy())
        mean_distance.append(dist)

    plt.plot(epsilons, mean_distance)
    plt.xlabel('Epsilons')
    plt.ylabel('Mean L2 distance')
    plt.savefig("mnist_BA_mean_L2distance.jpg")
    plt.show()
Пример #7
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--steps',
                        type=int,
                        default=10000,
                        help='Iteration of BA')
    parser.add_argument('--targeted',
                        action='store',
                        default=False,
                        help='For targeted attack')

    args = parser.parse_args()

    model = Net()
    model.load_state_dict(torch.load('mnist_cnn.pt'))
    model.eval()

    preprocessing = dict(mean=0.1307, std=0.3081)
    fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)

    fmodel = fmodel.transform_bounds((0, 1))
    assert fmodel.bounds == (0, 1)

    images, labels = ep.astensors(
        *samples(fmodel, dataset="mnist", batchsize=10))

    print('Model accuracy on clean examples: {}'.format(
        accuracy(fmodel, images, labels)))
    epsilons = np.linspace(0.01, 10, 20)

    boundary_attack = fa.BoundaryAttack(steps=args.steps, tensorboard=None)
    _, _, ba_success = boundary_attack(fmodel,
                                       images,
                                       labels,
                                       epsilons=epsilons)

    ba_robust_accuracy = 1 - ba_success.float32().mean(axis=-1)

    random_attack = fa.L2RepeatedAdditiveGaussianNoiseAttack(
        repeats=args.steps)
    _, _, ra_success = random_attack(fmodel, images, labels, epsilons=epsilons)
    ra_robust_accuracy = 1 - ra_success.float32().mean(axis=-1)

    legends = ["Boundary Attack", "Random Attack"]
    plt.plot(epsilons, ba_robust_accuracy.numpy())
    plt.plot(epsilons, ra_robust_accuracy.numpy())
    plt.legend(legends, loc='upper right')
    plt.xlabel("Perturbation Norm (L2)")
    plt.ylabel("Robust Accuracy")
    plt.title("{} Queries".format(args.steps))
    plt.savefig('mnist_robust_acc.jpg')
    plt.show()
Пример #8
0
def run_attacks(MODEL_DIR, res_path):
    rel_dirs = [x for x in os.listdir(MODEL_DIR) if '2020' in x]
    alpha = [re.findall('a=([0-9, \.]*)_', d)[0] for d in rel_dirs]
    res = dict.fromkeys(alpha)
    learner = prep_learner()

    for model_path, curr_alpha in tqdm(zip(rel_dirs, alpha), total=len(alpha)):
        conf.save_path = Path(path.join(MODEL_DIR, model_path))
        fix_str = [
            x for x in os.listdir(path.join(MODEL_DIR, model_path))
            if 'model' in x
        ][0][8:]
        learner.load_state(conf,
                           fix_str,
                           model_only=True,
                           from_save_folder=True)

        # probs
        set_probes(learner)

        for model in learner.models:
            model = torch.nn.DataParallel(model.cuda(),
                                          device_ids=list(range(4)))
            model.eval()

        res[curr_alpha] = dict()
        for (attack,
             eps), attack_name in tqdm(zip(attack_list, attack_list_names),
                                       desc='attaking ' + str(curr_alpha),
                                       total=len(attack_list)):
            fmodel = JointModelEP(
                [PyTorchModel(m, bounds=(0, 1)) for m in learner.models],
                'cuda')
            attack = attack()
            success_tot = []
            for images, labels in tqdm(learner.eval_loader,
                                       total=len(learner.eval_loader),
                                       desc=attack_name):
                images, labels = ep.astensors(images.to('cuda'),
                                              labels.to('cuda'))
                _, _, success = attack(fmodel, images, labels, epsilons=eps)
                success_tot.append(success)
            success_tot = ep.concatenate(success_tot, -1)

            # calculate and report the robust accuracy
            robust_accuracy = 1 - success_tot.float32().mean(axis=-1)
            for epsilon, acc in zip(eps, robust_accuracy):
                res[curr_alpha][attack_name + '_' + str(epsilon)] = acc.item()

            pickle.dump(res, open(res_path, 'wb'))
        pickle.dump(res, open(res_path, 'wb'))
Пример #9
0
def test_blur_numpy(request: Any) -> None:
    class Model:
        def __call__(self, inputs: Any) -> Any:
            return inputs.mean(axis=(2, 3))

    model = Model()
    with pytest.raises(ValueError):
        fbn.NumPyModel(model, bounds=(0, 1), data_format="foo")

    fmodel = fbn.NumPyModel(model, bounds=(0, 1))
    x, y = ep.astensors(*fbn.samples(
        fmodel, dataset="imagenet", batchsize=16,
        data_format="channels_first"))
    with pytest.raises(ValueError, match="data_format"):
        fbn.attacks.GaussianBlurAttack()(fmodel, x, y, epsilons=None)
Пример #10
0
def main():

    parser = argparse.ArgumentParser()
    parser.add_argument('--steps',
                        type=int,
                        default=1000,
                        help='Maximum number of steps to perform')
    parser.add_argument('--targeted',
                        action='store',
                        default=False,
                        help='For targeted attack')

    args = parser.parse_args()

    model = Net()
    model.load_state_dict(torch.load('mnist_cnn.pt'))
    model.eval()

    preprocessing = dict(mean=0.1307, std=0.3081)
    fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)

    fmodel = fmodel.transform_bounds((0, 1))
    assert fmodel.bounds == (0, 1)

    images, labels = ep.astensors(
        *samples(fmodel, dataset="mnist", batchsize=10))

    print('Model accuracy on clean examples: {}'.format(
        accuracy(fmodel, images, labels)))

    if args.targeted:
        target_class = (labels + 7) % 10
        criterion = fb.criteria.TargetedMisclassification(target_class)
    else:
        criterion = fb.criteria.Misclassification(labels)

    attack = fa.L2DeepFoolAttack(steps=args.steps)
    epsilons = None
    raw, clipped, success = attack(fmodel, images, labels, epsilons=epsilons)

    robust_accuracy = 1 - success.float32().mean()
    print("Robust Accuracy", robust_accuracy.item())

    dist = np.mean(fb.distances.l2(clipped, images).numpy())
    print("Average perturbation norm", dist)
Пример #11
0
def evaluate_l2(
    fmodel: Model,
    inputs: Any,
    labels: Any,
    *,
    attacks: List[Attack],
    epsilons: List[L2],
) -> Tuple[Any, Any]:
    x, y = ep.astensors(inputs, labels)
    del inputs, labels

    attack_success = np.zeros((len(attacks), len(epsilons), len(x)), dtype=np.float32)

    for i, attack in enumerate(attacks):
        sig = signature(type(attack).__init__)
        minimizing = "epsilon" not in sig.parameters

        if minimizing:
            # TODO: support hyperparameters
            xp = attack(fmodel, x, y)
            predictions = fmodel(xp).argmax(axis=-1)
            correct = (predictions == y).float32().numpy().astype(np.bool)
            perturbations = xp - x
            norms = flatten(perturbations).square().sum(axis=-1).sqrt().numpy()
            for j, epsilon in enumerate(epsilons):
                attack_success[i, j] = np.logical_and(
                    np.logical_not(correct), norms <= epsilon
                )
        else:
            for j, epsilon in enumerate(epsilons):
                attack.epsilon = epsilon  # type: ignore
                xp = attack(fmodel, x, y)
                predictions = fmodel(xp).argmax(axis=-1)
                correct = (predictions == y).float32().numpy().astype(np.bool)
                perturbations = xp - x
                norms = flatten(perturbations).square().sum(axis=-1).sqrt().numpy()
                # TODO: relax this norm check or pass a slightly stricter norm to the attack
                attack_success[i, j] = np.logical_and(
                    np.logical_not(correct), norms <= epsilon
                ).astype(np.float32)

    robust_accuracy = 1.0 - attack_success.max(axis=0).mean(axis=-1)
    return attack_success, robust_accuracy
Пример #12
0
    def attack_one_batch(fmodel, images, labels, iter=0, verbose=True):
        images, labels = ep.astensors(images, labels)

        raw_advs, clipped_advs, success = attack(fmodel,
                                                 images,
                                                 labels,
                                                 epsilons=epsilons)
        if verbose: print("===" * 8, iter, "===" * 8)
        if verbose:
            robust_accuracy = 1 - success.float32().mean(axis=-1)
            print("robust accuracy for perturbations with")
            for eps, acc in zip(epsilons, robust_accuracy):
                print(f"  Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")

        if verbose:
            fig = plt.gcf()
            os.makedirs("./image/", exist_ok=True)
            for i in range(len(raw_advs)):
                img_v = raw_advs[i].raw
                torchvision.utils.save_image(
                    img_v,
                    f'./image/{str(iter).zfill(4)}_{str(i).zfill(3)}_.png')
        return [x.raw for x in raw_advs]  #
Пример #13
0
def main() -> None:
    # instantiate a model (could also be a TensorFlow or JAX model)
    model = models.resnet18(pretrained=True).eval()
    preprocessing = dict(mean=[0.485, 0.456, 0.406],
                         std=[0.229, 0.224, 0.225],
                         axis=-3)
    fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)

    # get data and test the model
    # wrapping the tensors with ep.astensors is optional, but it allows
    # us to work with EagerPy tensors in the following
    images, labels = ep.astensors(
        *samples(fmodel, dataset="imagenet", batchsize=16))
    clean_acc = accuracy(fmodel, images, labels)
    print(f"clean accuracy:  {clean_acc * 100:.1f} %")

    # replace the gradient with the gradient from another model
    model2 = fmodel  # demo, we just use the same model

    # TODO: this is still a bit annoying because we need
    # to overwrite run to get the labels
    class Attack(LinfPGD):
        def value_and_grad(self, loss_fn, x):
            val1 = loss_fn(x)
            loss_fn2 = self.get_loss_fn(model2, self.labels)
            _, grad2 = ep.value_and_grad(loss_fn2, x)
            return val1, grad2

        def run(self, model, inputs, criterion, *, epsilon, **kwargs):
            criterion_ = get_criterion(criterion)
            self.labels = criterion_.labels
            return super().run(model,
                               inputs,
                               criterion_,
                               epsilon=epsilon,
                               **kwargs)

    # apply the attack
    attack = Attack()
    epsilons = [
        0.0,
        0.0002,
        0.0005,
        0.0008,
        0.001,
        0.0015,
        0.002,
        0.003,
        0.01,
        0.1,
        0.3,
        0.5,
        1.0,
    ]
    raw_advs, clipped_advs, success = attack(fmodel,
                                             images,
                                             labels,
                                             epsilons=epsilons)

    # calculate and report the robust accuracy (the accuracy of the model when
    # it is attacked)
    robust_accuracy = 1 - success.float32().mean(axis=-1)
    print("robust accuracy for perturbations with")
    for eps, acc in zip(epsilons, robust_accuracy):
        print(f"  Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")

    # we can also manually check this
    # we will use the clipped advs instead of the raw advs, otherwise
    # we would need to check if the perturbation sizes are actually
    # within the specified epsilon bound
    print()
    print("we can also manually check this:")
    print()
    print("robust accuracy for perturbations with")
    for eps, advs_ in zip(epsilons, clipped_advs):
        acc2 = accuracy(fmodel, advs_, labels)
        print(f"  Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
        print("    perturbation sizes:")
        perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2,
                                                               3)).numpy()
        print("    ", str(perturbation_sizes).replace("\n", "\n" + "    "))
        if acc2 == 0:
            break
Пример #14
0
def main() -> None:
    # instantiate a model (could also be a TensorFlow or JAX model)
    #model = models.resnet18(pretrained=True).eval()
    #model=torch.load('/data1/zyh/copycat/Framework/cifar_model.pth')

    model = AlexNet()
    path = "./cifar_net.pth"
    #path = '/data1/zyh/copycat/Framework/cifar_model.pth'
    #model.load_state_dict(torch.load('/data1/zyh/copycat/Framework/cifar_model.pth'))
    #pretrained_dict = {k: v for k, v in model_pretrained.items() if k in model_dict}
    #model_dict.update(pretrained_dict)
    #model.load_state_dict(state_dict)
    model.load_state_dict(torch.load(path), strict=True)
    model = model.to(device)
    model.eval()

    print(type(model))
    #preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3)
    preprocessing = dict(mean=[0.5] * 3, std=[0.5] * 3, axis=-3)
    fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)

    # get data and test the model
    # wrapping the tensors with ep.astensors is optional, but it allows
    # us to work with EagerPy tensors in the following
    test_dataset = torchvision.datasets.CIFAR10(
        root='~/.torch/',
        train=False,
        #transform = transforms.Compose([transforms.Resize((256,256)),transforms.ToTensor()]),
        transform=transforms.Compose([transforms.ToTensor()]),
        download=True)
    #     test_dataset .data = test_dataset.data[:128*5]

    test_loader = torch.utils.data.DataLoader(
        dataset=test_dataset,
        batch_size=128,  #该参数表示每次读取的批样本个数
        shuffle=False)  #该参数表示读取时是否打乱样本顺序
    # 创建迭代器
    #data_iter = iter(test_loader)

    #images, labels = next(data_iter)
    # 当迭代开始时, 队列和线程开始读取数据
    #images, labels = data_iter.next()
    #im=images
    #images=im.resize(100,3,128,128)
    with torch.no_grad():
        all_clean_acc_foolbox = []

        ## native predict
        predict_func(test_loader, model)

        for ii, (imgs, lbls) in tqdm.tqdm(enumerate(test_loader),
                                          total=len(test_loader)):
            imgs = imgs.to(device)
            lbls = lbls.to(device)

            images, labels = ep.astensors(imgs, lbls)

            ##  calc with foolbox
            pred_lbl_foolbox = fmodel(images)
            clean_acc_one = accuracy(fmodel, imgs, lbls)
            all_clean_acc_foolbox.append(clean_acc_one)

        clean_acc = sum(all_clean_acc_foolbox) / len(all_clean_acc_foolbox)

    print(f"clean accuracy:  {clean_acc * 100:.1f} %")

    # apply the attack
    attack = LinfPGD()
    '''epsilons = [
        0.0,
        0.0002,
        0.0005,
        0.0008,
        0.001,
        0.0015,
        0.002,
        0.003,
        0.01,
        0.1,
        0.3,
        0.5,
        1.0,
    ]'''
    epsilons = [
        0.0005,
        0.001,
        0.002,
        0.01,
        0.1,
    ]

    def attack_one_batch(fmodel, images, labels, iter=0, verbose=True):
        images, labels = ep.astensors(images, labels)

        raw_advs, clipped_advs, success = attack(fmodel,
                                                 images,
                                                 labels,
                                                 epsilons=epsilons)
        if verbose: print("===" * 8, iter, "===" * 8)
        if verbose:
            robust_accuracy = 1 - success.float32().mean(axis=-1)
            print("robust accuracy for perturbations with")
            for eps, acc in zip(epsilons, robust_accuracy):
                print(f"  Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")

        if verbose:
            fig = plt.gcf()
            os.makedirs("./image/", exist_ok=True)
            for i in range(len(raw_advs)):
                img_v = raw_advs[i].raw
                torchvision.utils.save_image(
                    img_v,
                    f'./image/{str(iter).zfill(4)}_{str(i).zfill(3)}_.png')
        return [x.raw for x in raw_advs]  #

    print("====" * 8, "start attack", "====" * 8)
    collection_adv = []
    collection_gt = []
    for ii, (imgs, lbls) in tqdm.tqdm(enumerate(test_loader),
                                      total=len(test_loader)):
        imgs = imgs.to(device)
        lbls = lbls.to(device)

        #         images, labels = ep.astensors(images,labels)
        adv_ret = attack_one_batch(fmodel=fmodel,
                                   images=imgs,
                                   labels=lbls,
                                   iter=ii,
                                   verbose=True)

        collection_adv.append(torch.stack(adv_ret))
        collection_gt.append(lbls.cpu())

    print("====" * 8, "start evaluation", "====" * 8)
    with torch.no_grad():

        adv_total_dataset = torch.cat(collection_adv, dim=1)
        lbl_total_dataset = torch.cat(collection_gt).to(device)

        #         print (adv_total_dataset.mean(dim=(1,2,3,4)),"the mean if each eps")
        for (eps, ep_adv_dataset) in zip(epsilons, adv_total_dataset):
            #             print ("eps:",eps,"===>"*8)
            #             print (ep_adv_dataset.mean(),"each...")
            advs_ = ep_adv_dataset.to(device)
            acc2 = accuracy(fmodel, advs_, lbl_total_dataset)
            print(f"  Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
            dataset = torch.utils.data.TensorDataset(ep_adv_dataset,
                                                     lbl_total_dataset)
            dl = torch.utils.data.DataLoader(dataset, batch_size=128)
            predict_func(dl, model)
from foolbox import PyTorchModel, accuracy, samples
import foolbox.attacks as fa
import numpy as np

if __name__ == "__main__":
    # instantiate a model (could also be a TensorFlow or JAX model)
    model = models.resnet18(pretrained=True).eval()
    preprocessing = dict(mean=[0.485, 0.456, 0.406],
                         std=[0.229, 0.224, 0.225],
                         axis=-3)
    fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)

    # get data and test the model
    # wrapping the tensors with ep.astensors is optional, but it allows
    # us to work with EagerPy tensors in the following
    images, labels = ep.astensors(
        *samples(fmodel, dataset="imagenet", batchsize=16))
    clean_acc = accuracy(fmodel, images, labels)
    print(f"clean accuracy:  {clean_acc * 100:.1f} %")
    print("")

    attacks = [
        fa.FGSM(),
        fa.LinfPGD(),
        fa.LinfBasicIterativeAttack(),
        fa.LinfAdditiveUniformNoiseAttack(),
        fa.LinfDeepFoolAttack(),
    ]

    epsilons = [
        0.0,
        0.0005,
Пример #16
0
advacc = 0.0
if __name__ == "__main__":
    total = 0
    attack_success = 0
    start = time.time()
    print("==> Testing %s.." %
          (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(start))))

    # 4 make adversarial sample
    for batch_idx, (images, labels) in enumerate(rawloader):
        total += labels.size(0)
        nepochs = nepochs + 1

        if batch_idx > 10000:
            break
        images, labels = ep.astensors(images.cuda(), labels.cuda())

        # TODO 注:转换类型,否则损失函数报错
        labels = labels.astype(dtype=torch.long)

        # TODO 计算对抗样本
        advs, _, success = attack(fmodel, images, labels, epsilons=epsilons)
        # calculate and report the robust accuracy
        robust_accuracy = 1 - success.float32().mean(axis=-1)

        rawacc += accuracy(fmodel, images, labels)

        for adv, label, acc in zip(advs, labels, robust_accuracy):
            if not os.path.exists(os.path.join(opt.outf, str(label.item()))):
                os.mkdir(os.path.join(opt.outf, str(label.item())))
            advacc = advacc + acc.item() * 100.0
Пример #17
0

	model.fit(x_train,y_train, epochs=5)

	model.evaluate(x_test,y_test,verbose=2)

	#instantiate the model
	fmodel=TensorFlowModel(model, bounds=(0,1))

	#get data and test the model
	#wrapping the tensors with ep.astensors is optional, but it allows
	#us to work with EagerPy tensors in the following

	##########################################################
	images, labels = samples(fmodel, dataset="mnist", batchsize=100)
	images1, labels1=ep.astensors(*samples(fmodel, dataset="mnist", batchsize=100))
	print(accuracy(fmodel, images1, labels1))


	predict=fmodel(images).numpy()
	tf.nn.softmax(predict).numpy()
	correct_pred=tf.math.argmax(predict,1)
	print(correct_pred)

	#print(images)
	images_arr=np.array(images)

	#print(images_arr)
	#print(images_arr.shape) #16,28,28,1

Пример #18
0
def main() -> None:
    # instantiate a model (could also be a TensorFlow or JAX model)
    model = models.resnet18(pretrained=True).eval()
    preprocessing = dict(mean=[0.485, 0.456, 0.406],
                         std=[0.229, 0.224, 0.225],
                         axis=-3)
    fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)

    # get data and test the model
    # wrapping the tensors with ep.astensors is optional, but it allows
    # us to work with EagerPy tensors in the following
    images, labels = ep.astensors(
        *samples(fmodel, dataset="imagenet", batchsize=16))
    clean_acc = accuracy(fmodel, images, labels)
    print(f"clean accuracy:  {clean_acc * 100:.1f} %")

    # apply the attack
    attack = LinfPGD()
    epsilons = [
        0.0,
        0.0002,
        0.0005,
        0.0008,
        0.001,
        0.0015,
        0.002,
        0.003,
        0.01,
        0.1,
        0.3,
        0.5,
        1.0,
    ]
    raw_advs, clipped_advs, success = attack(fmodel,
                                             images,
                                             labels,
                                             epsilons=epsilons)

    # calculate and report the robust accuracy (the accuracy of the model when
    # it is attacked)
    robust_accuracy = 1 - success.float32().mean(axis=-1)
    print("robust accuracy for perturbations with")
    for eps, acc in zip(epsilons, robust_accuracy):
        print(f"  Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")

    # we can also manually check this
    # we will use the clipped advs instead of the raw advs, otherwise
    # we would need to check if the perturbation sizes are actually
    # within the specified epsilon bound
    print()
    print("we can also manually check this:")
    print()
    print("robust accuracy for perturbations with")
    for eps, advs_ in zip(epsilons, clipped_advs):
        acc2 = accuracy(fmodel, advs_, labels)
        print(f"  Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
        print("    perturbation sizes:")
        perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2,
                                                               3)).numpy()
        print("    ", str(perturbation_sizes).replace("\n", "\n" + "    "))
        if acc2 == 0:
            break
Пример #19
0
def foolbox_attack(filter=None,
                   filter_preserve='low',
                   free_parm='eps',
                   plot_num=None):
    # get model.
    model = get_model()
    model = nn.DataParallel(model).to(device)
    model = model.eval()

    preprocessing = dict(mean=[0.485, 0.456, 0.406],
                         std=[0.229, 0.224, 0.225],
                         axis=-3)
    fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)

    if plot_num:
        free_parm = ''
        val_loader = get_val_loader(plot_num)
    else:
        # Load images.
        val_loader = get_val_loader(args.attack_batch_size)

    if 'eps' in free_parm:
        epsilons = [0.001, 0.003, 0.005, 0.008, 0.01, 0.1]
    else:
        epsilons = [0.01]
    if 'step' in free_parm:
        steps = [1, 5, 10, 30, 40, 50]
    else:
        steps = [args.iteration]

    for step in steps:
        # Adversarial attack.
        if args.attack_type == 'LinfPGD':
            attack = LinfPGD(steps=step)
        elif args.attack_type == 'FGSM':
            attack = FGSM()

        clean_acc = 0.0

        for i, data in enumerate(val_loader, 0):

            # Samples (attack_batch_size * attack_epochs) images for adversarial attack.
            if i >= args.attack_epochs:
                break

            images, labels = data[0].to(device), data[1].to(device)
            if step == steps[0]:
                clean_acc += (get_acc(
                    fmodel, images, labels
                )) / args.attack_epochs  # accumulate for attack epochs.

            _images, _labels = ep.astensors(images, labels)
            raw_advs, clipped_advs, success = attack(fmodel,
                                                     _images,
                                                     _labels,
                                                     epsilons=epsilons)

            if plot_num:
                grad = torch.from_numpy(
                    raw_advs[0].numpy()).to(device) - images
                grad = grad.clone().detach_()
                return grad

            if filter:
                robust_accuracy = torch.empty(len(epsilons))
                for eps_id in range(len(epsilons)):
                    grad = torch.from_numpy(
                        raw_advs[eps_id].numpy()).to(device) - images
                    grad = grad.clone().detach_()
                    freq = dct.dct_2d(grad)
                    if filter_preserve == 'low':
                        mask = torch.zeros(freq.size()).to(device)
                        mask[:, :, :filter, :filter] = 1
                    elif filter_preserve == 'high':
                        mask = torch.zeros(freq.size()).to(device)
                        mask[:, :, filter:, filter:] = 1
                    masked_freq = torch.mul(freq, mask)
                    new_grad = dct.idct_2d(masked_freq)
                    x_adv = torch.clamp(images + new_grad, 0, 1).detach_()

                    robust_accuracy[eps_id] = (get_acc(fmodel, x_adv, labels))
            else:
                robust_accuracy = 1 - success.float32().mean(axis=-1)
            if i == 0:
                robust_acc = robust_accuracy / args.attack_epochs
            else:
                robust_acc += robust_accuracy / args.attack_epochs

        if step == steps[0]:
            print("sample size is : ",
                  args.attack_batch_size * args.attack_epochs)
            print(f"clean accuracy:  {clean_acc * 100:.1f} %")
            print(
                f"Model {args.model} robust accuracy for {args.attack_type} perturbations with"
            )
        for eps, acc in zip(epsilons, robust_acc):
            print(
                f"  Step {step}, Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %"
            )
        print('  -------------------')
Пример #20
0
baseline_model.compile(
    loss='sparse_categorical_crossentropy',
    optimizer=tf.keras.optimizers.Adam(0.001),
    metrics=['accuracy'],
)

baseline_model.fit(ds_train,
                   epochs=1,
                   validation_data=ds_test,
                   steps_per_epoch=7500 // batch_size,
                   validation_steps=2500 // batch_size,
                   callbacks=[tensorboard_callback])

for images, labels in ds_train.take(1):  # only take first element of dataset
    images_ex = ep.astensors(images)
    labels_ex = ep.astensors(labels)

fmodel = fb.TensorFlowModel(baseline_model, bounds=(0, 1))

attacks = [
    fa.FGSM(),
    fa.LinfPGD(),
    fa.LinfBasicIterativeAttack(),
    fa.LinfAdditiveUniformNoiseAttack(),
    fa.LinfDeepFoolAttack(),
]

attacks_names = [
    "FGSM", "LinfPGD", "LinfBasicIterativeAttack",
    "LinfAdditiveUniformNoiseAttack", "LinfDeepFoolAttack"
Пример #21
0
def main() -> None:
    # instantiate a model (could also be a TensorFlow or JAX model)
    #model = models.resnet18(pretrained=True).eval()
    #model=torch.load('/data1/zyh/copycat/Framework/cifar_model.pth')

    model =AlexNet()
    path = "./cifar_net.pth"
    #path = '/data1/zyh/copycat/Framework/cifar_model.pth'
    #model.load_state_dict(torch.load('/data1/zyh/copycat/Framework/cifar_model.pth'))
    #pretrained_dict = {k: v for k, v in model_pretrained.items() if k in model_dict}
    #model_dict.update(pretrained_dict)
    #model.load_state_dict(state_dict)
    model.load_state_dict(torch.load(path),strict=True)
    model.eval()

    print(type(model))
    #preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3)
    preprocessing = dict(mean=[0.5]*3, std=[0.5]*3, axis=-3)
    fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)


    # get data and test the model
    # wrapping the tensors with ep.astensors is optional, but it allows
    # us to work with EagerPy tensors in the following
    #test_dataset = torchvision.datasets.CIFAR10(root='~/.torch/',
    #                                         train=True,
    #                                         #transform = transforms.Compose([transforms.Resize((256,256)),transforms.ToTensor()]),
    #                                         transform = transforms.Compose([transforms.ToTensor()]),
    #                                         download=True)
    #test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
    #                                       batch_size=128, #该参数表示每次读取的批样本个数
    #                                       shuffle=False) #该参数表示读取时是否打乱样本顺序
    #                                       # 创建迭代器
    #data_iter = iter(test_loader)

    #images, labels = next(data_iter)
    # 当迭代开始时, 队列和线程开始读取数据
    #images, labels = data_iter.next()
    #images=images.to(device)
    #labels=labels.to(device)
    #im=images
    #images=im.resize(100,3,128,128)
    images, labels = ep.astensors(*samples(fmodel, dataset="cifar10", batchsize=16))
    #images, labels = ep.astensors(*samples(fmodel, dataset="imagenet", batchsize=16))
    #print(images.shape)
    clean_acc = accuracy(fmodel, images, labels)
    
    print(f"clean accuracy:  {clean_acc * 100:.1f} %")

    # apply the attack
    attack = LinfPGD()
    '''epsilons = [
        0.0,
        0.0002,
        0.0005,
        0.0008,
        0.001,
        0.0015,
        0.002,
        0.003,
        0.01,
        0.1,
        0.3,
        0.5,
        1.0,
    ]'''
    epsilons = [
        0.0005,
        0.001,
        0.002,
        0.01,
        0.1,
    ]
    raw_advs, clipped_advs, success = attack(fmodel, images, labels, epsilons=epsilons)
    print(type(raw_advs))
    print("atest")
    # calculate and report the robust accuracy (the accuracy of the model when
    # it is attacked)
    robust_accuracy = 1 - success.float32().mean(axis=-1)
    print("robust accuracy for perturbations with")
    for eps, acc in zip(epsilons, robust_accuracy):
        print(f"  Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")

    # we can also manually check this
    # we will use the clipped advs instead of the raw advs, otherwise
    # we would need to check if the perturbation sizes are actually
    # within the specified epsilon bound
    print()
    print("we can also manually check this:")
    print()
    print("robust accuracy for perturbations with")
    for eps, advs_ in zip(epsilons, clipped_advs):
        acc2 = accuracy(fmodel, advs_, labels)
        print(f"  Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
        print("    perturbation sizes:")
        perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2, 3)).numpy()
        print("    ", str(perturbation_sizes).replace("\n", "\n" + "    "))
        if acc2 == 0:
            break
    fig = plt.gcf()
    os.makedirs("./image/",exist_ok=True)
    for i in range(len(raw_advs)):
        img_v = raw_advs[i].raw
        torchvision.utils.save_image(img_v, './image/'+str(i) +'.png')
Пример #22
0
 model.params.analysis_save_dir = os.path.join(
     model.params.analysis_out_dir, 'savefiles')
 if not os.path.exists(model.params.analysis_save_dir):
     os.makedirs(model.params.analysis_save_dir)
 model.to(params.device)
 model.load_checkpoint()
 fmodel = PyTorchModel(model.eval(), bounds=(0, 1))
 print('\n', '~' * 79)
 num_batches = len(test_loader.dataset) // model.params.batch_size
 attack_success = np.zeros(
     (len(attacks), len(epsilons), num_batches, model.params.batch_size),
     dtype=np.bool)
 for batch_index, (data, target) in enumerate(test_loader):
     data = model.preprocess_data(data.to(model.params.device))
     target = target.to(model.params.device)
     images, labels = ep.astensors(*(data, target))
     del data
     del target
     print(
         f'Model type: {model.params.model_type} [{model_index+1} out of {len(log_files)}]'
     )
     print(f'Batch {batch_index+1} out of {num_batches}')
     print(f'accuracy {accuracy(fmodel, images, labels)}')
     for attack_index, attack in enumerate(attacks):
         advs, inputs, success = attack(fmodel,
                                        images,
                                        labels,
                                        epsilons=epsilons)
         assert success.shape == (len(epsilons), len(images))
         success_ = success.numpy()
         assert success_.dtype == np.bool
Пример #23
0
 994: 'stinkhorn, carrion fungus',
 995: 'earthstar',
 996: 'hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa',
 997: 'bolete',
 998: 'ear, spike, capitulum',
 999: 'toilet tissue, toilet paper, bathroom tissue'}


if __name__ == '__main__':
    advs_save_path = 'adv_data/'
    r_save_path = 'r_data/'
    model = models.resnet152(pretrained=True).eval()
    preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3)
    # preprocessing = dict(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], axis=-3)
    fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
    images, labels, file_names = ep.astensors(*samples(fmodel, dataset="imagenet", batchsize=1, vis=True))
    file_names = file_names.raw

    prediction = accuracy(fmodel, images, labels, True).raw.cpu().detach().numpy()

    attack = FGSM()
    # attack = LinfPGD()
    # attack = L2DeepFoolAttack()
    epsilons = [0.0, 0.001, 0.01, 0.03, 0.1, 0.3, 0.5, 1.0]
    advs, _, _ = attack(fmodel, images, labels, epsilons=epsilons)

    for i in range(len(epsilons)):
        r = advs[0] - advs[i]
        attack_labels = accuracy(fmodel, advs[i], labels, True).raw.cpu().detach().numpy()

        imgs_path = advs_save_path + str(epsilons[i]) + '/'
# 	0.03,
# 	0.1,
# 	0.3,
# 	0.5,
# 	1.0,
# ]

#print(x_test[0],y_test[0])
foolbox_model = foolbox.models.TensorFlowModel(
    model=model,
    bounds=(0, 1),
)
foolbox_model = foolbox_model.transform_bounds((0, 1))
assert foolbox_model.bounds == (0, 1)

images = ep.astensors(x_test[0:16])
labels = ep.astensors(y_test[0:16])

#doesn't work right
attack = foolbox.attacks.LinfPGD()
_, advs, success = attack(foolbox_model, images, labels, epsilons=0.03)

print(success)

# The end accuracy was 0.9809 compared to 0.9496 when not using KFold.
#
# The predictions for the first 5:
# array([[5.43997092e-23, 4.90765706e-14, 2.98600290e-15, 6.97289174e-03,
#         4.53084188e-32, 9.93027031e-01, 6.42024900e-24, 4.53798596e-17,
#         2.48018923e-16, 5.83970931e-16],
#        [9.99999881e-01, 1.24962863e-16, 1.35353190e-07, 9.39377114e-16,