def setUp(self): # Set master seed master_seed(seed=1234)
def setUp(self): master_seed(seed=42)
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the # Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import numpy as np import pytest from tests.utils import ARTTestException, master_seed from art.attacks.poisoning.backdoor_attack_dgm_trail import BackdoorAttackDGMTrail from art.estimators.generation.tensorflow import TensorFlow2Generator master_seed(1234, set_tensorflow=True) @pytest.fixture def x_target(): return np.random.random_sample((28, 28, 1)) @pytest.mark.skip_framework("keras", "pytorch", "scikitlearn", "mxnet", "kerastf") def test_poison_estimator_trail(art_warning, get_default_mnist_subset, image_dl_gan, x_target): try: (train_images, y_train_images), _ = get_default_mnist_subset train_images = train_images * (2.0 / 255) - 1.0
def setUpClass(cls): master_seed(seed=301) (x_train, y_train), (x_test, y_test), min_, max_ = load_mnist() y_train = np.argmax(y_train, axis=1) y_test = np.argmax(y_test, axis=1) zero_or_four = np.logical_or(y_train == 4, y_train == 0, y_train == 9) x_train = x_train[zero_or_four] y_train = y_train[zero_or_four] tr_labels = np.zeros((y_train.shape[0], 2)) tr_labels[y_train == 0] = np.array([1, 0]) tr_labels[y_train == 4] = np.array([0, 1]) y_train = tr_labels zero_or_four = np.logical_or(y_test == 4, y_test == 0) x_test = x_test[zero_or_four] y_test = y_test[zero_or_four] te_labels = np.zeros((y_test.shape[0], 2)) te_labels[y_test == 0] = np.array([1, 0]) te_labels[y_test == 4] = np.array([0, 1]) y_test = te_labels n_samples_train = x_train.shape[0] n_features_train = x_train.shape[1] * x_train.shape[2] * x_train.shape[3] n_samples_test = x_test.shape[0] n_features_test = x_test.shape[1] * x_test.shape[2] * x_test.shape[3] x_train = x_train.reshape(n_samples_train, n_features_train) x_test = x_test.reshape(n_samples_test, n_features_test) x_train = x_train[:NB_TRAIN] y_train = y_train[:NB_TRAIN] trusted_data = x_test[:NB_TRUSTED] trusted_labels = y_test[:NB_TRUSTED] x_test = x_test[NB_TRUSTED:] y_test = y_test[NB_TRUSTED:] valid_data = x_test[:NB_VALID] valid_labels = y_test[:NB_VALID] x_test = x_test[NB_VALID:] y_test = y_test[NB_VALID:] clean_prov = np.random.randint(NB_DEVICES - 1, size=x_train.shape[0]) p_train = np.eye(NB_DEVICES)[clean_prov] no_defense = ScikitlearnSVC(model=SVC(kernel=kernel, gamma="auto"), clip_values=(min_, max_)) no_defense.fit(x=x_train, y=y_train) poison_points = np.random.randint(no_defense._model.support_vectors_.shape[0], size=NB_POISON) all_poison_init = np.copy(no_defense._model.support_vectors_[poison_points]) poison_labels = np.array([1, 1]) - no_defense.predict(all_poison_init) svm_attack = PoisoningAttackSVM( classifier=no_defense, x_train=x_train, y_train=y_train, step=0.1, eps=1.0, x_val=valid_data, y_val=valid_labels, ) poisoned_data, _ = svm_attack.poison(all_poison_init, y=poison_labels) # Stack on poison to data and add provenance of bad actor all_data = np.vstack([x_train, poisoned_data]) all_labels = np.vstack([y_train, poison_labels]) poison_prov = np.zeros((NB_POISON, NB_DEVICES)) poison_prov[:, NB_DEVICES - 1] = 1 all_p = np.vstack([p_train, poison_prov]) model = SVC(kernel=kernel, gamma="auto") cls.mnist = ( (all_data, all_labels, all_p), (x_test, y_test), (trusted_data, trusted_labels), (valid_data, valid_labels), (min_, max_), ) cls.classifier = SklearnClassifier(model=model, clip_values=(min_, max_)) cls.classifier.fit(all_data, all_labels) cls.defence_trust = ProvenanceDefense( cls.classifier, all_data, all_labels, all_p, x_val=trusted_data, y_val=trusted_labels, eps=0.1, ) cls.defence_no_trust = ProvenanceDefense(cls.classifier, all_data, all_labels, all_p, eps=0.1)
def setup_class(self): master_seed(seed=1234) # Small data for testing x1 = np.array( [ -1.0376293e-03, -1.0681478e-03, -1.0986663e-03, -1.1291848e-03, -1.1291848e-03, -1.1291848e-03, -1.1902219e-03, -1.1597034e-03, -1.1902219e-03, -1.1291848e-03, -1.1291848e-03, -1.0681478e-03, -9.1555528e-04, ] * 100 ) x2 = np.array( [ -1.8311106e-04, -1.2207404e-04, -6.1037019e-05, 0.0000000e00, 3.0518509e-05, 0.0000000e00, -3.0518509e-05, 0.0000000e00, 0.0000000e00, 9.1555528e-05, 2.1362957e-04, 3.3570360e-04, 4.2725913e-04, 4.5777764e-04, -1.8311106e-04, ] * 100 ) x3 = np.array( [ -8.2399976e-04, -7.0192572e-04, -5.4933317e-04, -4.2725913e-04, -3.6622211e-04, -2.7466659e-04, -2.1362957e-04, 5.4933317e-04, 5.7985168e-04, 6.1037019e-04, 6.7140721e-04, 7.0192572e-04, 6.7140721e-04, -1.5259255e-04, ] * 100 ) self.x = np.array([x1, x2, x3])
def setUp(self): master_seed(1234) super().setUp()
def setUp(cls): master_seed(seed=42)
def setUp(self): master_seed(seed=1234)
def create_scikit_model_weights(): master_seed(1234) model_list = { "decisionTreeClassifier": DecisionTreeClassifier(), "extraTreeClassifier": ExtraTreeClassifier(), "adaBoostClassifier": AdaBoostClassifier(), "baggingClassifier": BaggingClassifier(), "extraTreesClassifier": ExtraTreesClassifier(n_estimators=10), "gradientBoostingClassifier": GradientBoostingClassifier(n_estimators=10), "randomForestClassifier": RandomForestClassifier(n_estimators=10), "logisticRegression": LogisticRegression(solver="lbfgs", multi_class="auto"), "svc": SVC(gamma="auto"), "linearSVC": LinearSVC(), } clipped_models = { model_name: SklearnClassifier(model=model, clip_values=(0, 1)) for model_name, model in model_list.items() } unclipped_models = { model_name: SklearnClassifier(model=model) for model_name, model in model_list.items() } (x_train_iris, y_train_iris), (_, _), _, _ = load_dataset("iris") for model_name, model in clipped_models.items(): model.fit(x=x_train_iris, y=y_train_iris) pickle.dump( model, open( os.path.join( os.path.dirname(os.path.dirname(__file__)), "utils/resources/models/scikit/", model_name + "iris_clipped.sav", ), "wb", ), ) for model_name, model in unclipped_models.items(): model.fit(x=x_train_iris, y=y_train_iris) pickle.dump( model, open( os.path.join( os.path.dirname(os.path.dirname(__file__)), "utils/resources/models/scikit/", model_name + "iris_unclipped.sav", ), "wb", ), )
def setUp(self): master_seed(seed=42, set_tensorflow=True)
if TYPE_CHECKING: import torch logger = logging.getLogger(__name__) deep_learning_frameworks = [ "keras", "tensorflow1", "tensorflow2", "tensorflow2v1", "pytorch", "kerastf", "mxnet", "jax" ] non_deep_learning_frameworks = ["scikitlearn"] art_supported_frameworks = [] art_supported_frameworks.extend(deep_learning_frameworks) art_supported_frameworks.extend(non_deep_learning_frameworks) master_seed(1234) def get_default_framework(): import tensorflow as tf if tf.__version__[0] == "2": default_framework = "tensorflow2" else: default_framework = "tensorflow1" return default_framework def pytest_addoption(parser): parser.addoption(
def _test_backend_mnist(self, classifier, x_train, y_train, x_test, y_test): base_success_rate = 0.1 num_iter = 5 regularization = 100 batch_size = 5 eps = 0.3 # Test Wasserstein with wasserstein ball and wasserstein norm attack = Wasserstein( classifier, regularization=regularization, max_iter=num_iter, conjugate_sinkhorn_max_iter=num_iter, projected_sinkhorn_max_iter=num_iter, norm="wasserstein", ball="wasserstein", targeted=False, p=2, eps_iter=2, eps_factor=1.05, eps=eps, eps_step=0.1, kernel_size=5, batch_size=batch_size, ) x_train_adv = attack.generate(x_train) x_test_adv = attack.generate(x_test) self.assertFalse((x_train_adv == x_train).all()) self.assertFalse((x_test_adv == x_test).all()) train_y_pred = get_labels_np_array(classifier.predict(x_train_adv)).astype(float) test_y_pred = get_labels_np_array(classifier.predict(x_test_adv)).astype(float) train_success_rate = ( np.sum(np.argmax(train_y_pred, axis=1) != np.argmax(classifier.predict(x_train), axis=1)) / y_train.shape[0] ) self.assertGreaterEqual(train_success_rate, base_success_rate) test_success_rate = ( np.sum(np.argmax(test_y_pred, axis=1) != np.argmax(classifier.predict(x_test), axis=1)) / y_test.shape[0] ) self.assertGreaterEqual(test_success_rate, base_success_rate) # Test Wasserstein with wasserstein ball and l2 norm attack = Wasserstein( classifier, regularization=regularization, max_iter=num_iter, conjugate_sinkhorn_max_iter=num_iter, projected_sinkhorn_max_iter=num_iter, norm="2", ball="wasserstein", targeted=False, p=2, eps_iter=2, eps_factor=1.05, eps=eps, eps_step=0.1, kernel_size=5, batch_size=batch_size, ) x_train_adv = attack.generate(x_train) x_test_adv = attack.generate(x_test) train_y_pred = get_labels_np_array(classifier.predict(x_train_adv)).astype(float) test_y_pred = get_labels_np_array(classifier.predict(x_test_adv)).astype(float) train_success_rate = ( np.sum(np.argmax(train_y_pred, axis=1) != np.argmax(classifier.predict(x_train), axis=1)) / y_train.shape[0] ) self.assertGreaterEqual(train_success_rate, base_success_rate) test_success_rate = ( np.sum(np.argmax(test_y_pred, axis=1) != np.argmax(classifier.predict(x_test), axis=1)) / y_test.shape[0] ) self.assertGreaterEqual(test_success_rate, 0) # Test Wasserstein with wasserstein ball and inf norm attack = Wasserstein( classifier, regularization=regularization, max_iter=num_iter, conjugate_sinkhorn_max_iter=num_iter, projected_sinkhorn_max_iter=num_iter, norm="inf", ball="wasserstein", targeted=False, p=2, eps_iter=2, eps_factor=1.05, eps=eps, eps_step=0.1, kernel_size=5, batch_size=batch_size, ) x_train_adv = attack.generate(x_train) x_test_adv = attack.generate(x_test) train_y_pred = get_labels_np_array(classifier.predict(x_train_adv)).astype(float) test_y_pred = get_labels_np_array(classifier.predict(x_test_adv)).astype(float) train_success_rate = ( np.sum(np.argmax(train_y_pred, axis=1) != np.argmax(classifier.predict(x_train), axis=1)) / y_train.shape[0] ) self.assertGreaterEqual(train_success_rate, base_success_rate) test_success_rate = ( np.sum(np.argmax(test_y_pred, axis=1) != np.argmax(classifier.predict(x_test), axis=1)) / y_test.shape[0] ) self.assertGreaterEqual(test_success_rate, 0) # Test Wasserstein with wasserstein ball and l1 norm attack = Wasserstein( classifier, regularization=regularization, max_iter=num_iter, conjugate_sinkhorn_max_iter=num_iter, projected_sinkhorn_max_iter=num_iter, norm="1", ball="wasserstein", targeted=False, p=2, eps_iter=2, eps_factor=1.05, eps=eps, eps_step=0.1, kernel_size=5, batch_size=batch_size, ) x_train_adv = attack.generate(x_train) x_test_adv = attack.generate(x_test) train_y_pred = get_labels_np_array(classifier.predict(x_train_adv)).astype(float) test_y_pred = get_labels_np_array(classifier.predict(x_test_adv)).astype(float) train_success_rate = ( np.sum(np.argmax(train_y_pred, axis=1) != np.argmax(classifier.predict(x_train), axis=1)) / y_train.shape[0] ) self.assertGreaterEqual(train_success_rate, base_success_rate) test_success_rate = ( np.sum(np.argmax(test_y_pred, axis=1) != np.argmax(classifier.predict(x_test), axis=1)) / y_test.shape[0] ) self.assertGreaterEqual(test_success_rate, 0) # Test Wasserstein with l2 ball and wasserstein norm attack = Wasserstein( classifier, regularization=regularization, max_iter=num_iter, conjugate_sinkhorn_max_iter=num_iter, projected_sinkhorn_max_iter=num_iter, norm="wasserstein", ball="2", targeted=False, p=2, eps_iter=2, eps_factor=1.05, eps=eps, eps_step=0.05, kernel_size=5, batch_size=batch_size, ) x_train_adv = attack.generate(x_train) x_test_adv = attack.generate(x_test) train_y_pred = get_labels_np_array(classifier.predict(x_train_adv)).astype(float) test_y_pred = get_labels_np_array(classifier.predict(x_test_adv)).astype(float) train_success_rate = ( np.sum(np.argmax(train_y_pred, axis=1) != np.argmax(classifier.predict(x_train), axis=1)) / y_train.shape[0] ) self.assertGreaterEqual(train_success_rate, 0) test_success_rate = ( np.sum(np.argmax(test_y_pred, axis=1) != np.argmax(classifier.predict(x_test), axis=1)) / y_test.shape[0] ) self.assertGreaterEqual(test_success_rate, 0) # Test Wasserstein with l1 ball and wasserstein norm attack = Wasserstein( classifier, regularization=regularization, max_iter=num_iter, conjugate_sinkhorn_max_iter=num_iter, projected_sinkhorn_max_iter=num_iter, norm="wasserstein", ball="1", targeted=False, p=2, eps_iter=2, eps_factor=1.05, eps=eps, eps_step=0.1, kernel_size=5, batch_size=batch_size, ) x_train_adv = attack.generate(x_train) x_test_adv = attack.generate(x_test) train_y_pred = get_labels_np_array(classifier.predict(x_train_adv)).astype(float) test_y_pred = get_labels_np_array(classifier.predict(x_test_adv)).astype(float) train_success_rate = ( np.sum(np.argmax(train_y_pred, axis=1) != np.argmax(classifier.predict(x_train), axis=1)) / y_train.shape[0] ) self.assertGreaterEqual(train_success_rate, 0) test_success_rate = ( np.sum(np.argmax(test_y_pred, axis=1) != np.argmax(classifier.predict(x_test), axis=1)) / y_test.shape[0] ) self.assertGreaterEqual(test_success_rate, 0) # Test Wasserstein with inf ball and Wasserstein norm attack = Wasserstein( classifier, regularization=regularization, max_iter=num_iter, conjugate_sinkhorn_max_iter=num_iter, projected_sinkhorn_max_iter=num_iter, norm="wasserstein", ball="inf", targeted=False, p=2, eps_iter=2, eps_factor=1.05, eps=eps, eps_step=0.1, kernel_size=5, batch_size=batch_size, ) x_train_adv = attack.generate(x_train) x_test_adv = attack.generate(x_test) train_y_pred = get_labels_np_array(classifier.predict(x_train_adv)).astype(float) test_y_pred = get_labels_np_array(classifier.predict(x_test_adv)).astype(float) train_success_rate = ( np.sum(np.argmax(train_y_pred, axis=1) != np.argmax(classifier.predict(x_train), axis=1)) / y_train.shape[0] ) self.assertGreaterEqual(train_success_rate, base_success_rate) test_success_rate = ( np.sum(np.argmax(test_y_pred, axis=1) != np.argmax(classifier.predict(x_test), axis=1)) / y_test.shape[0] ) self.assertGreaterEqual(test_success_rate, base_success_rate) # Test Wasserstein with targeted attack master_seed(1234) attack = Wasserstein( classifier, regularization=regularization, max_iter=num_iter, conjugate_sinkhorn_max_iter=num_iter, projected_sinkhorn_max_iter=num_iter, norm="wasserstein", ball="wasserstein", targeted=True, p=2, eps_iter=2, eps_factor=1.05, eps=eps, eps_step=0.1, kernel_size=5, batch_size=batch_size, ) train_y_rand = random_targets(y_train, nb_classes=10) test_y_rand = random_targets(y_test, nb_classes=10) x_train_adv = attack.generate(x_train, train_y_rand) x_test_adv = attack.generate(x_test, test_y_rand) train_y_pred = get_labels_np_array(classifier.predict(x_train_adv)).astype(float) test_y_pred = get_labels_np_array(classifier.predict(x_test_adv)).astype(float) train_success_rate = ( np.sum(np.argmax(train_y_pred, axis=1) == np.argmax(train_y_rand, axis=1)) / y_train.shape[0] ) self.assertGreaterEqual(train_success_rate, base_success_rate) test_success_rate = np.sum(np.argmax(test_y_pred, axis=1) == np.argmax(test_y_rand, axis=1)) / y_test.shape[0] self.assertGreaterEqual(test_success_rate, 0) # Test Wasserstein with p-wasserstein=1 and kernel_size=3 attack = Wasserstein( classifier, regularization=regularization, max_iter=num_iter, conjugate_sinkhorn_max_iter=num_iter, projected_sinkhorn_max_iter=num_iter, norm="wasserstein", ball="wasserstein", targeted=False, p=1, eps_iter=2, eps_factor=1.05, eps=eps, eps_step=0.1, kernel_size=3, batch_size=batch_size, ) x_train_adv = attack.generate(x_train) x_test_adv = attack.generate(x_test) train_y_pred = get_labels_np_array(classifier.predict(x_train_adv)).astype(float) test_y_pred = get_labels_np_array(classifier.predict(x_test_adv)).astype(float) train_success_rate = ( np.sum(np.argmax(train_y_pred, axis=1) != np.argmax(classifier.predict(x_train), axis=1)) / y_train.shape[0] ) self.assertTrue(train_success_rate >= base_success_rate) test_success_rate = ( np.sum(np.argmax(test_y_pred, axis=1) != np.argmax(classifier.predict(x_test), axis=1)) / y_test.shape[0] ) self.assertTrue(test_success_rate >= base_success_rate)
def setUp(self): master_seed(seed=42) super().setUp()
def test_augment_images_with_patch(random_location, image_format, fix_get_mnist_subset): (x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist) = fix_get_mnist_subset master_seed() if image_format == "NHWC": patch = np.ones(shape=(4, 4, 1)) * 0.5 x = x_train_mnist[0:3] channels_first = False elif image_format == "NCHW": patch = np.ones(shape=(1, 4, 4)) * 0.5 x = np.transpose(x_train_mnist[0:3], (0, 3, 1, 2)) channels_first = True patched_images, transformations = DPatch._augment_images_with_patch( x=x, patch=patch, random_location=random_location, channels_first=channels_first) if random_location: transformation_expected = { "i_x_1": 0, "i_y_1": 2, "i_x_2": 4, "i_y_2": 6 } patched_images_column = [ 0.0, 0.0, 0.5, 0.5, 0.5, 0.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ] else: transformation_expected = { "i_x_1": 0, "i_y_1": 0, "i_x_2": 4, "i_y_2": 4 } patched_images_column = [ 0.5, 0.5, 0.5, 0.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ] assert transformations[1] == transformation_expected if image_format == "NCHW": patched_images = np.transpose(patched_images, (0, 2, 3, 1)) np.testing.assert_array_equal(patched_images[1, 2, :, 0], patched_images_column)
def _test_framework_vs_numpy(self, classifier): # Test PGD with np.inf norm attack_np = ProjectedGradientDescentNumpy( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=np.inf, targeted=False, num_random_init=0, batch_size=3, random_eps=False, ) x_train_adv_np = attack_np.generate(self.x_train_mnist) x_test_adv_np = attack_np.generate(self.x_test_mnist) attack_fw = ProjectedGradientDescent( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=np.inf, targeted=False, num_random_init=0, batch_size=3, random_eps=False, ) x_train_adv_fw = attack_fw.generate(self.x_train_mnist) x_test_adv_fw = attack_fw.generate(self.x_test_mnist) # Test self.assertAlmostEqual(np.mean(x_train_adv_np - self.x_train_mnist), np.mean(x_train_adv_fw - self.x_train_mnist), places=6) self.assertAlmostEqual(np.mean(x_test_adv_np - self.x_test_mnist), np.mean(x_test_adv_fw - self.x_test_mnist), places=6) # Test PGD with L1 norm attack_np = ProjectedGradientDescentNumpy( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=1, targeted=False, num_random_init=0, batch_size=3, random_eps=False, ) x_train_adv_np = attack_np.generate(self.x_train_mnist) x_test_adv_np = attack_np.generate(self.x_test_mnist) attack_fw = ProjectedGradientDescent( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=1, targeted=False, num_random_init=0, batch_size=3, random_eps=False, ) x_train_adv_fw = attack_fw.generate(self.x_train_mnist) x_test_adv_fw = attack_fw.generate(self.x_test_mnist) # Test self.assertAlmostEqual(np.mean(x_train_adv_np - self.x_train_mnist), np.mean(x_train_adv_fw - self.x_train_mnist), places=6) self.assertAlmostEqual(np.mean(x_test_adv_np - self.x_test_mnist), np.mean(x_test_adv_fw - self.x_test_mnist), places=6) # Test PGD with L2 norm attack_np = ProjectedGradientDescentNumpy( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=2, targeted=False, num_random_init=0, batch_size=3, random_eps=False, ) x_train_adv_np = attack_np.generate(self.x_train_mnist) x_test_adv_np = attack_np.generate(self.x_test_mnist) attack_fw = ProjectedGradientDescent( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=2, targeted=False, num_random_init=0, batch_size=3, random_eps=False, ) x_train_adv_fw = attack_fw.generate(self.x_train_mnist) x_test_adv_fw = attack_fw.generate(self.x_test_mnist) # Test self.assertAlmostEqual(np.mean(x_train_adv_np - self.x_train_mnist), np.mean(x_train_adv_fw - self.x_train_mnist), places=6) self.assertAlmostEqual(np.mean(x_test_adv_np - self.x_test_mnist), np.mean(x_test_adv_fw - self.x_test_mnist), places=6) # Test PGD with True targeted attack_np = ProjectedGradientDescentNumpy( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=np.inf, targeted=True, num_random_init=0, batch_size=3, random_eps=False, ) x_train_adv_np = attack_np.generate(self.x_train_mnist, self.y_train_mnist) x_test_adv_np = attack_np.generate(self.x_test_mnist, self.y_test_mnist) attack_fw = ProjectedGradientDescent( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=np.inf, targeted=True, num_random_init=0, batch_size=3, random_eps=False, ) x_train_adv_fw = attack_fw.generate(self.x_train_mnist, self.y_train_mnist) x_test_adv_fw = attack_fw.generate(self.x_test_mnist, self.y_test_mnist) # Test self.assertAlmostEqual(np.mean(x_train_adv_np - self.x_train_mnist), np.mean(x_train_adv_fw - self.x_train_mnist), places=6) self.assertAlmostEqual(np.mean(x_test_adv_np - self.x_test_mnist), np.mean(x_test_adv_fw - self.x_test_mnist), places=6) # Test PGD with num_random_init=2 master_seed(1234) attack_np = ProjectedGradientDescentNumpy( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=np.inf, targeted=False, num_random_init=2, batch_size=3, random_eps=False, ) x_train_adv_np = attack_np.generate(self.x_train_mnist) x_test_adv_np = attack_np.generate(self.x_test_mnist) master_seed(1234) attack_fw = ProjectedGradientDescent( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=np.inf, targeted=False, num_random_init=2, batch_size=3, random_eps=False, ) x_train_adv_fw = attack_fw.generate(self.x_train_mnist) x_test_adv_fw = attack_fw.generate(self.x_test_mnist) # Test self.assertAlmostEqual(np.mean(x_train_adv_np - self.x_train_mnist), np.mean(x_train_adv_fw - self.x_train_mnist), places=6) self.assertAlmostEqual(np.mean(x_test_adv_np - self.x_test_mnist), np.mean(x_test_adv_fw - self.x_test_mnist), places=6) # Test PGD with random_eps=True master_seed(1234) attack_np = ProjectedGradientDescentNumpy( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=np.inf, targeted=False, num_random_init=0, batch_size=3, random_eps=True, ) x_train_adv_np = attack_np.generate(self.x_train_mnist) x_test_adv_np = attack_np.generate(self.x_test_mnist) master_seed(1234) attack_fw = ProjectedGradientDescent( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=np.inf, targeted=False, num_random_init=0, batch_size=3, random_eps=True, ) x_train_adv_fw = attack_fw.generate(self.x_train_mnist) x_test_adv_fw = attack_fw.generate(self.x_test_mnist) # Test self.assertAlmostEqual(np.mean(x_train_adv_np - self.x_train_mnist), np.mean(x_train_adv_fw - self.x_train_mnist), places=6) self.assertAlmostEqual(np.mean(x_test_adv_np - self.x_test_mnist), np.mean(x_test_adv_fw - self.x_test_mnist), places=6) # Test the masking 1 master_seed(1234) attack_np = ProjectedGradientDescentNumpy( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=np.inf, targeted=False, num_random_init=1, batch_size=3, random_eps=True, ) mask = np.random.binomial(n=1, p=0.5, size=np.prod(self.x_train_mnist.shape)) mask = mask.reshape(self.x_train_mnist.shape).astype(np.float32) x_train_adv_np = attack_np.generate(self.x_train_mnist, mask=mask) mask = np.random.binomial(n=1, p=0.5, size=np.prod(self.x_test_mnist.shape)) mask = mask.reshape(self.x_test_mnist.shape).astype(np.float32) x_test_adv_np = attack_np.generate(self.x_test_mnist, mask=mask) master_seed(1234) attack_fw = ProjectedGradientDescent( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=np.inf, targeted=False, num_random_init=1, batch_size=3, random_eps=True, ) mask = np.random.binomial(n=1, p=0.5, size=np.prod(self.x_train_mnist.shape)) mask = mask.reshape(self.x_train_mnist.shape).astype(np.float32) x_train_adv_fw = attack_fw.generate(self.x_train_mnist, mask=mask) mask = np.random.binomial(n=1, p=0.5, size=np.prod(self.x_test_mnist.shape)) mask = mask.reshape(self.x_test_mnist.shape).astype(np.float32) x_test_adv_fw = attack_fw.generate(self.x_test_mnist, mask=mask) # Test self.assertAlmostEqual(np.mean(x_train_adv_np - self.x_train_mnist), np.mean(x_train_adv_fw - self.x_train_mnist), places=6) self.assertAlmostEqual(np.mean(x_test_adv_np - self.x_test_mnist), np.mean(x_test_adv_fw - self.x_test_mnist), places=6) # Test the masking 2 master_seed(1234) attack_np = ProjectedGradientDescentNumpy( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=np.inf, targeted=False, num_random_init=1, batch_size=3, random_eps=True, ) mask = np.random.binomial(n=1, p=0.5, size=np.prod(self.x_train_mnist.shape[1:])) mask = mask.reshape(self.x_train_mnist.shape[1:]).astype(np.float32) x_train_adv_np = attack_np.generate(self.x_train_mnist, mask=mask) mask = np.random.binomial(n=1, p=0.5, size=np.prod(self.x_test_mnist.shape[1:])) mask = mask.reshape(self.x_test_mnist.shape[1:]).astype(np.float32) x_test_adv_np = attack_np.generate(self.x_test_mnist, mask=mask) master_seed(1234) attack_fw = ProjectedGradientDescent( classifier, eps=1.0, eps_step=0.1, max_iter=5, norm=np.inf, targeted=False, num_random_init=1, batch_size=3, random_eps=True, ) mask = np.random.binomial(n=1, p=0.5, size=np.prod(self.x_train_mnist.shape[1:])) mask = mask.reshape(self.x_train_mnist.shape[1:]).astype(np.float32) x_train_adv_fw = attack_fw.generate(self.x_train_mnist, mask=mask) mask = np.random.binomial(n=1, p=0.5, size=np.prod(self.x_test_mnist.shape[1:])) mask = mask.reshape(self.x_test_mnist.shape[1:]).astype(np.float32) x_test_adv_fw = attack_fw.generate(self.x_test_mnist, mask=mask) # Test self.assertAlmostEqual(np.mean(x_train_adv_np - self.x_train_mnist), np.mean(x_train_adv_fw - self.x_train_mnist), places=6) self.assertAlmostEqual(np.mean(x_test_adv_np - self.x_test_mnist), np.mean(x_test_adv_fw - self.x_test_mnist), places=6)
def setUpClass(cls): master_seed(seed=1234, set_tensorflow=True) super().setUpClass()
def setUp(self): master_seed(seed=301) self.backdoor_path = os.path.join(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), "data", "backdoors", "alert.png") super().setUp()
def setUp(self): master_seed(seed=1234, set_tensorflow=True) super().setUp()
def setUpClass(cls): master_seed(seed=1234) cls.setUpIRIS()
def setUpClass(cls): master_seed(seed=1234) super().setUpClass()
def setUp(self): master_seed(seed=301)
def test_augment_images_with_patch(art_warning, random_location, image_format, fix_get_mnist_subset): try: (x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist) = fix_get_mnist_subset # TODO this master_seed should be removed as it is already set in conftest.py. expected values will # need to be updated accordingly master_seed() if image_format == "NHWC": patch = np.ones(shape=(4, 4, 1)) * 0.5 x = x_train_mnist[0:3] channels_first = False elif image_format == "NCHW": patch = np.ones(shape=(1, 4, 4)) * 0.5 x = np.transpose(x_train_mnist[0:3], (0, 3, 1, 2)) channels_first = True patched_images, transformations = DPatch._augment_images_with_patch( x=x, patch=patch, random_location=random_location, channels_first=channels_first) if random_location: transformation_expected = { "i_x_1": 0, "i_y_1": 2, "i_x_2": 4, "i_y_2": 6 } patched_images_column = [ 0.0, 0.0, 0.5, 0.5, 0.5, 0.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ] else: transformation_expected = { "i_x_1": 0, "i_y_1": 0, "i_x_2": 4, "i_y_2": 4 } patched_images_column = [ 0.5, 0.5, 0.5, 0.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ] assert transformations[1] == transformation_expected if image_format == "NCHW": patched_images = np.transpose(patched_images, (0, 2, 3, 1)) np.testing.assert_array_equal(patched_images[1, 2, :, 0], patched_images_column) except ARTTestException as e: art_warning(e)
def test_tf_faster_rcnn(art_warning, get_mnist_dataset): try: master_seed(seed=1234, set_tensorflow=True) # Only import if object detection module is available from art.estimators.object_detection.tensorflow_faster_rcnn import TensorFlowFasterRCNN # Define object detector images = tf.placeholder(tf.float32, shape=[1, 28, 28, 1]) obj_dec = TensorFlowFasterRCNN(images=images) # Get test data (_, _), (x_test_mnist, y_test_mnist) = get_mnist_dataset x_test_mnist = x_test_mnist[:1] # First test predict result = obj_dec.predict(x_test_mnist) assert list(result[0].keys()) == ["boxes", "labels", "scores"] assert result[0]["boxes"].shape == (300, 4) expected_detection_boxes = np.asarray([0.008862, 0.003788, 0.070454, 0.175931]) np.testing.assert_array_almost_equal(result[0]["boxes"][2, :], expected_detection_boxes, decimal=3) assert result[0]["scores"].shape == (300,) expected_detection_scores = np.asarray( [ 2.196349e-04, 7.968055e-05, 7.811916e-05, 7.334248e-05, 6.868376e-05, 6.861838e-05, 6.756858e-05, 6.331169e-05, 6.313509e-05, 6.222352e-05, ] ) np.testing.assert_array_almost_equal(result[0]["scores"][:10], expected_detection_scores, decimal=3) assert result[0]["labels"].shape == (300,) expected_detection_classes = np.asarray([37, 15, 15, 66, 15, 15, 15, 63, 2, 66]) np.testing.assert_array_almost_equal(result[0]["labels"][:10], expected_detection_classes) # Then test loss gradient # Create labels y = [{"boxes": result[0]["boxes"], "labels": result[0]["labels"], "scores": np.ones_like(result[0]["labels"])}] # Compute gradients grads = obj_dec.loss_gradient(x_test_mnist[:1], y) assert grads.shape == (1, 28, 28, 1) expected_gradients = np.asarray( [ [-0.00298723], [-0.0039893], [-0.00036253], [0.01038542], [0.01455704], [0.00995643], [0.00424966], [0.00470569], [0.00666382], [0.0028694], [0.00525351], [0.00889174], [0.0071413], [0.00618231], [0.00598106], [0.0072665], [0.00708815], [0.00286943], [0.00411595], [0.00788978], [0.00587319], [0.00808631], [0.01018151], [0.00867905], [0.00820272], [0.00124911], [-0.0042593], [0.02380728], ] ) np.testing.assert_array_almost_equal(grads[0, 0, :, :], expected_gradients, decimal=2) # Then test loss gradient with standard format # Create labels result_tf = obj_dec.predict(x_test_mnist, standardise_output=False) result = obj_dec.predict(x_test_mnist, standardise_output=True) from art.estimators.object_detection.utils import convert_tf_to_pt result_pt = convert_tf_to_pt(y=result_tf, height=x_test_mnist.shape[1], width=x_test_mnist.shape[2]) np.testing.assert_array_equal(result[0]["boxes"], result_pt[0]["boxes"]) np.testing.assert_array_equal(result[0]["labels"], result_pt[0]["labels"]) np.testing.assert_array_equal(result[0]["scores"], result_pt[0]["scores"]) y = [{"boxes": result[0]["boxes"], "labels": result[0]["labels"], "scores": np.ones_like(result[0]["labels"])}] # Compute gradients grads = obj_dec.loss_gradient(x_test_mnist[:1], y, standardise_output=True) assert grads.shape == (1, 28, 28, 1) expected_gradients = np.asarray( [ [-0.00095965], [-0.00265362], [-0.00031886], [0.01132964], [0.01674244], [0.01262039], [0.0063345], [0.00673249], [0.00618648], [0.00422678], [0.00542425], [0.00814896], [0.00919153], [0.01068758], [0.00929435], [0.00877143], [0.00747379], [0.0050377], [0.00656254], [0.00799547], [0.0051057], [0.00714598], [0.01090685], [0.00787637], [0.00709959], [0.00047201], [-0.00460457], [0.02629307], ] ) np.testing.assert_array_almost_equal(grads[0, 0, :, :], expected_gradients, decimal=2) obj_dec._sess.close() tf.reset_default_graph() except ARTTestException as e: art_warning(e)