def test_iterative_saliency(art_warning, fix_get_mnist_subset, image_dl_estimator_for_attack): try: classifier = image_dl_estimator_for_attack(FastGradientMethod) expected_values_axis_1 = { "nb_perturbed_frames": ExpectedValue(np.asarray([10, 1, 2, 12, 16, 1, 2, 7, 4, 11, 5]), 2) } expected_values_axis_2 = { "nb_perturbed_frames": ExpectedValue(np.asarray([11, 1, 2, 6, 14, 2, 2, 13, 4, 8, 4]), 2) } attacker = FastGradientMethod(classifier, eps=0.3, batch_size=128) attack = FrameSaliencyAttack(classifier, attacker, "iterative_saliency") backend_check_adverse_frames(attack, fix_get_mnist_subset, expected_values_axis_1) # test with non-default frame index: attack = FrameSaliencyAttack(classifier, attacker, "iterative_saliency", frame_index=2) backend_check_adverse_frames(attack, fix_get_mnist_subset, expected_values_axis_2) except ARTTestException as e: art_warning(e)
def test_minimal_perturbations_images(fix_get_mnist_subset, get_image_classifier_list_for_attack): classifier_list = get_image_classifier_list_for_attack(FastGradientMethod) # TODO this if statement must be removed once we have a classifier for both image and tabular data if classifier_list is None: logging.warning( "Couldn't perform this test because no classifier is defined") return for classifier in classifier_list: attack = FastGradientMethod(classifier, eps=1.0, batch_size=11) attack_params = {"minimal": True, "eps_step": 0.1, "eps": 5.0} attack.set_params(**attack_params) expected_values = { "x_test_mean": ExpectedValue(0.03896513, 0.01), "x_test_min": ExpectedValue(-0.30000000, 0.00001), "x_test_max": ExpectedValue(0.30000000, 0.00001), "y_test_pred_adv_expected": ExpectedValue(np.asarray([4, 2, 4, 7, 0, 4, 7, 2, 0, 7, 0]), 2), } backend_check_adverse_values(attack, fix_get_mnist_subset, expected_values)
def test_iterative_saliency(fix_get_mnist_subset, image_dl_estimator_for_attack): classifier_list = image_dl_estimator_for_attack(FastGradientMethod) # TODO this if statement must be removed once we have a classifier for both image and tabular data if classifier_list is None: logging.warning( "Couldn't perform this test because no classifier is defined") return expected_values_axis_1 = { "nb_perturbed_frames": ExpectedValue(np.asarray([10, 1, 2, 12, 16, 1, 2, 7, 4, 11, 5]), 2) } expected_values_axis_2 = { "nb_perturbed_frames": ExpectedValue(np.asarray([11, 1, 2, 6, 14, 2, 2, 13, 4, 8, 4]), 2) } for classifier in classifier_list: attacker = FastGradientMethod(classifier, eps=0.3, batch_size=128) attack = FrameSaliencyAttack(classifier, attacker, "iterative_saliency") backend_check_adverse_frames(attack, fix_get_mnist_subset, expected_values_axis_1) # test with non-default frame index: attack = FrameSaliencyAttack(classifier, attacker, "iterative_saliency", frame_index=2) backend_check_adverse_frames(attack, fix_get_mnist_subset, expected_values_axis_2)
def test_one_shot(fix_get_mnist_subset, image_dl_estimator_for_attack): classifier_list = image_dl_estimator_for_attack(FastGradientMethod) # TODO this if statement must be removed once we have a classifier for both image and tabular data if classifier_list is None: logging.warning( "Couldn't perform this test because no classifier is defined") return # for the one-shot method, frame saliency attack should resort to plain FastGradientMethod expected_values = { "x_test_mean": ExpectedValue(0.2346725, 0.002), "x_test_min": ExpectedValue(-1.0, 0.00001), "x_test_max": ExpectedValue(1.0, 0.00001), "y_test_pred_adv_expected": ExpectedValue(np.asarray([4, 4, 4, 7, 7, 4, 7, 2, 2, 3, 0]), 2), } for classifier in classifier_list: attacker = FastGradientMethod(classifier, eps=1, batch_size=128) attack = FrameSaliencyAttack(classifier, attacker, "one_shot") backend_check_adverse_values(attack, fix_get_mnist_subset, expected_values)
def test_minimal_perturbations_images(art_warning, fix_get_mnist_subset, image_dl_estimator_for_attack): try: classifier = image_dl_estimator_for_attack(FastGradientMethod) expected_values = { "x_test_mean": ExpectedValue(0.03896513, 0.01), "x_test_min": ExpectedValue(-0.30000000, 0.00001), "x_test_max": ExpectedValue(0.30000000, 0.00001), "y_test_pred_adv_expected": ExpectedValue(np.asarray([4, 2, 4, 7, 0, 4, 7, 2, 0, 7, 0]), 2), } attack = FastGradientMethod(classifier, eps=1.0, batch_size=11) # Test eps of float type attack_params = {"minimal": True, "eps_step": 0.1, "eps": 5.0} attack.set_params(**attack_params) backend_check_adverse_values(attack, fix_get_mnist_subset, expected_values) # Test eps of array type 1 (_, _, x_test_mnist, _) = fix_get_mnist_subset eps = np.ones(shape=x_test_mnist.shape) * 5.0 eps_step = np.ones_like(eps) * 0.1 attack_params = {"minimal": True, "eps_step": eps_step, "eps": eps} attack.set_params(**attack_params) backend_check_adverse_values(attack, fix_get_mnist_subset, expected_values) # Test eps of array type 2 eps = np.ones(shape=x_test_mnist.shape[1:]) * 5.0 eps_step = np.ones_like(eps) * 0.1 attack_params = {"minimal": True, "eps_step": eps_step, "eps": eps} attack.set_params(**attack_params) backend_check_adverse_values(attack, fix_get_mnist_subset, expected_values) # Test eps of array type 3 eps = np.ones(shape=x_test_mnist.shape[2:]) * 5.0 eps_step = np.ones_like(eps) * 0.1 attack_params = {"minimal": True, "eps_step": eps_step, "eps": eps} attack.set_params(**attack_params) backend_check_adverse_values(attack, fix_get_mnist_subset, expected_values) # Test eps of array type 4 eps = np.ones(shape=x_test_mnist.shape[3:]) * 5.0 eps_step = np.ones_like(eps) * 0.1 attack_params = {"minimal": True, "eps_step": eps_step, "eps": eps} attack.set_params(**attack_params) backend_check_adverse_values(attack, fix_get_mnist_subset, expected_values) except ARTTestException as e: art_warning(e)
def test_fit_generator(is_tf_version_2, get_default_mnist_subset, get_image_classifier_list): (x_train_mnist, y_train_mnist), (x_test_mnist, y_test_mnist) = get_default_mnist_subset if not is_tf_version_2: classifier, sess = get_image_classifier_list(one_classifier=True) # Create TensorFlow data generator x_tensor = tf.convert_to_tensor( x_train_mnist.reshape(10, 100, 28, 28, 1)) y_tensor = tf.convert_to_tensor(y_train_mnist.reshape(10, 100, 10)) dataset = tf.data.Dataset.from_tensor_slices((x_tensor, y_tensor)) iterator = dataset.make_initializable_iterator() data_gen = TensorFlowDataGenerator(sess=sess, iterator=iterator, iterator_type='initializable', iterator_arg={}, size=1000, batch_size=100) expected_values = {"post_fit_accuracy": ExpectedValue(0.65, 0.02)} backend_test_fit_generator(expected_values, classifier, data_gen, get_default_mnist_subset, nb_epochs=2)
def test_one_shot(art_warning, fix_get_mnist_subset, image_dl_estimator_for_attack): try: classifier = image_dl_estimator_for_attack(FastGradientMethod) # for the one-shot method, frame saliency attack should resort to plain FastGradientMethod expected_values = { "x_test_mean": ExpectedValue(0.2346725, 0.002), "x_test_min": ExpectedValue(-1.0, 0.00001), "x_test_max": ExpectedValue(1.0, 0.00001), "y_test_pred_adv_expected": ExpectedValue(np.asarray([4, 4, 4, 7, 7, 4, 7, 2, 2, 3, 0]), 2), } attacker = FastGradientMethod(classifier, eps=1.0, batch_size=128) attack = FrameSaliencyAttack(classifier, attacker, "one_shot") backend_check_adverse_values(attack, fix_get_mnist_subset, expected_values) except ARTTestException as e: art_warning(e)
def test_fit_generator(get_default_mnist_subset, default_batch_size, get_image_classifier_list): (x_train_mnist, y_train_mnist), (x_test_mnist, y_test_mnist) = get_default_mnist_subset gen = generator_fit(x_train_mnist, y_train_mnist, batch_size=default_batch_size) data_gen = KerasDataGenerator(iterator=gen, size=x_train_mnist.shape[0], batch_size=default_batch_size) classifier, _ = get_image_classifier_list(one_classifier=True) expected_values = {"pre_fit_accuracy": ExpectedValue(0.32, 0.06), "post_fit_accuracy": ExpectedValue(0.36, 0.06)} backend_test_fit_generator(expected_values, classifier, data_gen, get_default_mnist_subset, nb_epochs=3)
def test_class_gradient(get_default_mnist_subset, get_image_classifier_list): classifier, _ = get_image_classifier_list(one_classifier=True) expected_values = { "expected_gradients_1_all_labels": ExpectedValue( np.asarray( [ -1.0557447e-03, -1.0079544e-03, -7.7426434e-04, 1.7387432e-03, 2.1773507e-03, 5.0880699e-05, 1.6497371e-03, 2.6113100e-03, 6.0904310e-03, 4.1080985e-04, 2.5268078e-03, -3.6661502e-04, -3.0568996e-03, -1.1665225e-03, 3.8904310e-03, 3.1726385e-04, 1.3203260e-03, -1.1720930e-04, -1.4315104e-03, -4.7676818e-04, 9.7251288e-04, 0.0000000e00, 0.0000000e00, 0.0000000e00, 0.0000000e00, 0.0000000e00, 0.0000000e00, 0.0000000e00, ] ), 4, ), "expected_gradients_2_all_labels": ExpectedValue( np.asarray( [ -0.00367321, -0.0002892, 0.00037825, -0.00053344, 0.00192121, 0.00112047, 0.0023135, 0.0, 0.0, -0.00391743, -0.0002264, 0.00238103, -0.00073711, 0.00270405, 0.00389043, 0.00440818, -0.00412769, -0.00441794, 0.00081916, -0.00091284, 0.00119645, -0.00849089, 0.00547925, 0.0, 0.0, 0.0, 0.0, 0.0, ] ), 4, ), "expected_gradients_1_label5": ExpectedValue( np.asarray( [ -1.0557447e-03, -1.0079544e-03, -7.7426434e-04, 1.7387432e-03, 2.1773507e-03, 5.0880699e-05, 1.6497371e-03, 2.6113100e-03, 6.0904310e-03, 4.1080985e-04, 2.5268078e-03, -3.6661502e-04, -3.0568996e-03, -1.1665225e-03, 3.8904310e-03, 3.1726385e-04, 1.3203260e-03, -1.1720930e-04, -1.4315104e-03, -4.7676818e-04, 9.7251288e-04, 0.0000000e00, 0.0000000e00, 0.0000000e00, 0.0000000e00, 0.0000000e00, 0.0000000e00, 0.0000000e00, ] ), 4, ), "expected_gradients_2_label5": ExpectedValue( np.asarray( [ -0.00367321, -0.0002892, 0.00037825, -0.00053344, 0.00192121, 0.00112047, 0.0023135, 0.0, 0.0, -0.00391743, -0.0002264, 0.00238103, -0.00073711, 0.00270405, 0.00389043, 0.00440818, -0.00412769, -0.00441794, 0.00081916, -0.00091284, 0.00119645, -0.00849089, 0.00547925, 0.0, 0.0, 0.0, 0.0, 0.0, ] ), 4, ), "expected_gradients_1_labelArray": ExpectedValue( np.asarray( [ 5.0867125e-03, 4.8564528e-03, 6.1040390e-03, 8.6531248e-03, -6.0958797e-03, -1.4114540e-02, -7.1085989e-04, -5.0330814e-04, 1.2943064e-02, 8.2416134e-03, -1.9859476e-04, -9.8109958e-05, -3.8902222e-03, -1.2945873e-03, 7.5137997e-03, 1.7720886e-03, 3.1399424e-04, 2.3657181e-04, -3.0891625e-03, -1.0211229e-03, 2.0828887e-03, 0.0000000e00, 0.0000000e00, 0.0000000e00, 0.0000000e00, 0.0000000e00, 0.0000000e00, 0.0000000e00, ] ), 4, ), "expected_gradients_2_labelArray": ExpectedValue( np.asarray( [ -0.00195835, -0.00134457, -0.00307221, -0.00340564, 0.00175022, -0.00239714, -0.00122619, 0.0, 0.0, -0.00520899, -0.00046105, 0.00414874, -0.00171095, 0.00429184, 0.0075138, 0.00792442, 0.0019566, 0.00035517, 0.00504575, -0.00037397, 0.00022343, -0.00530034, 0.0020528, 0.0, 0.0, 0.0, 0.0, 0.0, ] ), 4, ), } labels = np.asarray( [ 3, 4, 4, 0, 1, 1, 1, 2, 3, 4, 4, 2, 2, 0, 0, 4, 0, 1, 2, 0, 3, 4, 2, 2, 3, 3, 0, 1, 3, 0, 3, 2, 3, 4, 1, 3, 3, 3, 2, 1, 3, 4, 2, 3, 4, 1, 4, 0, 4, 1, 1, 4, 1, 4, 0, 1, 0, 0, 4, 0, 4, 2, 3, 1, 2, 2, 4, 3, 4, 2, 2, 4, 4, 2, 1, 3, 2, 1, 4, 1, 0, 1, 2, 1, 2, 1, 2, 1, 1, 4, 1, 2, 4, 0, 4, 1, 2, 1, 1, 3, ] ) backend_test_class_gradient(get_default_mnist_subset, classifier, expected_values, labels)
def test_loss_gradient(get_default_mnist_subset, get_image_classifier_list): expected_values = { "expected_gradients_1": ExpectedValue( np.asarray( [ 0.0559206, 0.05338925, 0.0648919, 0.07925165, -0.04029291, -0.11281465, 0.01850601, 0.00325054, 0.08163195, 0.03333949, 0.031766, -0.02420463, -0.07815556, -0.04698735, 0.10711591, 0.04086434, -0.03441073, 0.01071284, -0.04229195, -0.01386157, 0.02827487, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ] ), 4, ), "expected_gradients_2": ExpectedValue( np.asarray( [ 0.00210803, 0.00213919, 0.00520981, 0.00548001, -0.0023059, 0.00432077, 0.00274945, 0.0, 0.0, -0.0583441, -0.00616604, 0.0526219, -0.02373985, 0.05273106, 0.10711591, 0.12773865, 0.0689289, 0.01337799, 0.10032021, 0.01681096, -0.00028647, -0.05588859, 0.01474165, 0.0, 0.0, 0.0, 0.0, 0.0, ] ), 4, ), } backend_test_loss_gradient(get_default_mnist_subset, get_image_classifier_list, expected_values)
def test_norm_images(norm, fix_get_mnist_subset, get_image_classifier_list_for_attack): classifier_list = get_image_classifier_list_for_attack(FastGradientMethod) # TODO this if statement must be removed once we have a classifier for both image and tabular data if classifier_list is None: logging.warning( "Couldn't perform this test because no classifier is defined") return elif norm == np.inf: expected_values = { "x_test_mean": ExpectedValue(0.2346725, 0.002), "x_test_min": ExpectedValue(-1.0, 0.00001), "x_test_max": ExpectedValue(1.0, 0.00001), "y_test_pred_adv_expected": ExpectedValue(np.asarray([4, 4, 4, 7, 7, 4, 7, 2, 2, 3, 0]), 2), } elif norm == 1: expected_values = { "x_test_mean": ExpectedValue(0.00051374, 0.002), "x_test_min": ExpectedValue(-0.01486498, 0.001), "x_test_max": ExpectedValue(0.014761963, 0.001), "y_test_pred_adv_expected": ExpectedValue(np.asarray([7, 1, 1, 4, 4, 1, 4, 4, 4, 4, 4]), 4), } elif norm == 2: expected_values = { "x_test_mean": ExpectedValue(0.007636416, 0.001), "x_test_min": ExpectedValue(-0.211054801, 0.001), "x_test_max": ExpectedValue(0.209592223, 0.001), "y_test_pred_adv_expected": ExpectedValue(np.asarray([7, 2, 4, 4, 4, 7, 7, 4, 0, 4, 4]), 2), } for classifier in classifier_list: attack = FastGradientMethod(classifier, eps=1, norm=norm, batch_size=128) backend_check_adverse_values(attack, fix_get_mnist_subset, expected_values)
def test_norm_images(art_warning, norm, fix_get_mnist_subset, image_dl_estimator_for_attack): try: classifier = image_dl_estimator_for_attack(FastGradientMethod) if norm == np.inf: expected_values = { "x_test_mean": ExpectedValue(0.2346725, 0.002), "x_test_min": ExpectedValue(-1.0, 0.00001), "x_test_max": ExpectedValue(1.0, 0.00001), "y_test_pred_adv_expected": ExpectedValue(np.asarray([4, 4, 4, 7, 7, 4, 7, 2, 2, 3, 0]), 2), } elif norm == 1: expected_values = { "x_test_mean": ExpectedValue(0.00051374, 0.002), "x_test_min": ExpectedValue(-0.01486498, 0.001), "x_test_max": ExpectedValue(0.014761963, 0.001), "y_test_pred_adv_expected": ExpectedValue(np.asarray([7, 1, 1, 4, 4, 1, 4, 4, 4, 4, 4]), 4), } elif norm == 2: expected_values = { "x_test_mean": ExpectedValue(0.007636416, 0.001), "x_test_min": ExpectedValue(-0.211054801, 0.001), "x_test_max": ExpectedValue(0.209592223, 0.001), "y_test_pred_adv_expected": ExpectedValue(np.asarray([7, 2, 4, 4, 4, 7, 7, 4, 0, 4, 4]), 2), } attack = FastGradientMethod(classifier, eps=1.0, norm=norm, batch_size=128) backend_check_adverse_values(attack, fix_get_mnist_subset, expected_values) except ARTTestException as e: art_warning(e)
def test_loss_gradient(get_default_mnist_subset, get_image_classifier_list): expected_values = { "expected_gradients_1": ExpectedValue( np.asarray([ 5.59206062e-04, 5.33892540e-04, 6.48919027e-04, 7.92516454e-04, -4.02929145e-04, -1.12814642e-03, 1.85060024e-04, 3.25053406e-05, 8.16319487e-04, 3.33394884e-04, 3.17659928e-04, -2.42046357e-04, -7.81555660e-04, -4.69873514e-04, 1.07115903e-03, 4.08643362e-04, -3.44107364e-04, 1.07128391e-04, -4.22919547e-04, -1.38615724e-04, 2.82748661e-04, 0.00000000e00, 0.00000000e00, 0.00000000e00, 0.00000000e00, 0.00000000e00, 0.00000000e00, 0.00000000e00, ]), 4, ), "expected_gradients_2": ExpectedValue( np.asarray([ 2.10802755e-05, 2.13919120e-05, 5.20980720e-05, 5.48000680e-05, -2.30590031e-05, 4.32076595e-05, 2.74944887e-05, 0.00000000e00, 0.00000000e00, -5.83440997e-04, -6.16604229e-05, 5.26219024e-04, -2.37398461e-04, 5.27310593e-04, 1.07115903e-03, 1.27738668e-03, 6.89289009e-04, 1.33779933e-04, 1.00320193e-03, 1.68109560e-04, -2.86467184e-06, -5.58885862e-04, 1.47416518e-04, 0.00000000e00, 0.00000000e00, 0.00000000e00, 0.00000000e00, 0.00000000e00, ]), 4, ), } backend_test_loss_gradient(get_default_mnist_subset, get_image_classifier_list, expected_values)
def test_class_gradient(get_image_classifier_list, get_default_mnist_subset): (x_train_mnist, y_train_mnist), (x_test_mnist, y_test_mnist) = get_default_mnist_subset classifier_logits, _ = get_image_classifier_list(one_classifier=True, from_logits=True) expected_values = { "expected_gradients_1_all_labels": ExpectedValue( np.asarray([ -0.03347399, -0.03195872, -0.02650188, 0.04111874, 0.08676253, 0.03339913, 0.06925241, 0.09387045, 0.15184258, -0.00684002, 0.05070481, 0.01409407, -0.03632583, 0.00151133, 0.05102589, 0.00766463, -0.00898967, 0.00232938, -0.00617045, -0.00201032, 0.00410065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ]), 4, ), "expected_gradients_2_all_labels": ExpectedValue( np.asarray([ -0.09723657, -0.00240533, 0.02445251, -0.00035474, 0.04765627, 0.04286841, 0.07209076, 0.0, 0.0, -0.07938144, -0.00142567, 0.02882954, -0.00049514, 0.04170151, 0.05102589, 0.09544909, -0.04401167, -0.06158172, 0.03359772, -0.00838454, 0.01722163, -0.13376027, 0.08206709, 0.0, 0.0, 0.0, 0.0, 0.0, ]), 4, ), "expected_gradients_1_label5": ExpectedValue( np.asarray([ -0.03347399, -0.03195872, -0.02650188, 0.04111874, 0.08676253, 0.03339913, 0.06925241, 0.09387045, 0.15184258, -0.00684002, 0.05070481, 0.01409407, -0.03632583, 0.00151133, 0.05102589, 0.00766463, -0.00898967, 0.00232938, -0.00617045, -0.00201032, 0.00410065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ]), 4, ), "expected_gradients_2_label5": ExpectedValue( np.asarray([ -0.09723657, -0.00240533, 0.02445251, -0.00035474, 0.04765627, 0.04286841, 0.07209076, 0.0, 0.0, -0.07938144, -0.00142567, 0.02882954, -0.00049514, 0.04170151, 0.05102589, 0.09544909, -0.04401167, -0.06158172, 0.03359772, -0.00838454, 0.01722163, -0.13376027, 0.08206709, 0.0, 0.0, 0.0, 0.0, 0.0, ]), 4, ), "expected_gradients_1_labelArray": ExpectedValue( np.asarray([ 0.06860766, 0.065502, 0.08539103, 0.13868105, -0.05520725, -0.18788849, 0.02264893, 0.02980516, 0.2226511, 0.11288887, -0.00678776, 0.02045561, -0.03120914, 0.00642691, 0.08449504, 0.02848018, -0.03251382, 0.00854315, -0.02354656, -0.00767687, 0.01565931, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ]), 4, ), "expected_gradients_2_labelArray": ExpectedValue( np.asarray([ -0.0487146, -0.0171556, -0.03161772, -0.0420007, 0.03360246, -0.01864819, 0.00315916, 0.0, 0.0, -0.07631349, -0.00374462, 0.04229517, -0.01131879, 0.05044588, 0.08449504, 0.12417868, 0.07536847, 0.03906382, 0.09467953, 0.00543209, -0.00504872, -0.03366479, -0.00385999, 0.0, 0.0, 0.0, 0.0, 0.0, ]), 4, ), } labels = np.random.randint(5, size=x_test_mnist.shape[0]) backend_test_class_gradient(get_default_mnist_subset, classifier_logits, expected_values, labels)