def test_black_box_keras_loss(art_warning, get_iris_dataset): try: (x_train, y_train), (_, _) = get_iris_dataset # This test creates a framework-specific (keras) model because it needs to check both the case of a string-based # loss and a class-based loss, and therefore cannot use the generic fixture get_tabular_classifier_list model = keras.models.Sequential() model.add(keras.layers.Dense(8, input_dim=4, activation="relu")) model.add(keras.layers.Dense(3, activation="softmax")) model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) model.fit(x_train, y_train, epochs=150, batch_size=10) classifier = KerasClassifier(model) attack = MembershipInferenceBlackBox(classifier, input_type="loss") backend_check_membership_accuracy(attack, get_iris_dataset, attack_train_ratio, 0.15) model2 = keras.models.Sequential() model2.add(keras.layers.Dense(12, input_dim=4, activation="relu")) model2.add(keras.layers.Dense(3, activation="softmax")) model2.compile(loss=keras.losses.CategoricalCrossentropy(), optimizer="adam", metrics=["accuracy"]) model2.fit(x_train, y_train, epochs=150, batch_size=10) classifier = KerasClassifier(model2) attack = MembershipInferenceBlackBox(classifier, input_type="loss") backend_check_membership_accuracy(attack, get_iris_dataset, attack_train_ratio, 0.15) except ARTTestException as e: art_warning(e)
def test_black_box_tabular(art_warning, model_type, tabular_dl_estimator_for_attack, get_iris_dataset): try: classifier = tabular_dl_estimator_for_attack(MembershipInferenceBlackBox) attack = MembershipInferenceBlackBox(classifier, attack_model_type=model_type) backend_check_membership_accuracy(attack, get_iris_dataset, attack_train_ratio, 0.25) except ARTTestException as e: art_warning(e)
def test_black_box_tabular_prob_nn(art_warning, tabular_dl_estimator_for_attack, get_iris_dataset): try: classifier = tabular_dl_estimator_for_attack(MembershipInferenceBlackBox) attack = MembershipInferenceBlackBox(classifier, attack_model_type="nn") backend_check_membership_probabilities(attack, get_iris_dataset, attack_train_ratio) except ARTTestException as e: art_warning(e)
def test_black_box_image(art_warning, get_default_mnist_subset, image_dl_estimator_for_attack): try: classifier = image_dl_estimator_for_attack(MembershipInferenceBlackBox) attack = MembershipInferenceBlackBox(classifier) backend_check_membership_accuracy(attack, get_default_mnist_subset, attack_train_ratio, 0.25) except ARTTestException as e: art_warning(e)
def test_black_box_loss_tabular(art_warning, model_type, tabular_dl_estimator_for_attack, get_iris_dataset): try: classifier = tabular_dl_estimator_for_attack(MembershipInferenceBlackBox) if type(classifier).__name__ == "PyTorchClassifier" or type(classifier).__name__ == "TensorFlowV2Classifier": attack = MembershipInferenceBlackBox(classifier, input_type="loss", attack_model_type=model_type) backend_check_membership_accuracy(attack, get_iris_dataset, attack_train_ratio, 0.25) except ARTTestException as e: art_warning(e)
def test_black_box_with_model(art_warning, tabular_dl_estimator_for_attack, estimator_for_attack, get_iris_dataset): try: classifier = tabular_dl_estimator_for_attack(MembershipInferenceBlackBox) attack_model = estimator_for_attack(num_features=2 * num_classes_iris) attack = MembershipInferenceBlackBox(classifier, attack_model=attack_model) backend_check_membership_accuracy(attack, get_iris_dataset, attack_train_ratio, 0.25) except ARTTestException as e: art_warning(e)
def test_errors(art_warning, tabular_dl_estimator_for_attack, get_iris_dataset): try: classifier = tabular_dl_estimator_for_attack(MembershipInferenceBlackBox) (x_train, y_train), (x_test, y_test) = get_iris_dataset with pytest.raises(ValueError): MembershipInferenceBlackBox(classifier, attack_model_type="a") with pytest.raises(ValueError): MembershipInferenceBlackBox(classifier, input_type="a") attack = MembershipInferenceBlackBox(classifier) with pytest.raises(ValueError): attack.fit(x_train, y_test, x_test, y_test) with pytest.raises(ValueError): attack.fit(x_train, y_train, x_test, y_train) with pytest.raises(ValueError): attack.infer(x_train, y_test) except ARTTestException as e: art_warning(e)
def test_errors(get_tabular_classifier_list, get_iris_dataset): classifier_list = get_tabular_classifier_list(MembershipInferenceBlackBox) if not classifier_list: logging.warning( "Couldn't perform this test because no classifier is defined") return (x_train, y_train), (x_test, y_test) = get_iris_dataset with pytest.raises(ValueError): MembershipInferenceBlackBox(classifier_list[0], attack_model_type="a") with pytest.raises(ValueError): MembershipInferenceBlackBox(classifier_list[0], input_type="a") attack = MembershipInferenceBlackBox(classifier_list[0]) with pytest.raises(ValueError): attack.fit(x_train, y_test, x_test, y_test) with pytest.raises(ValueError): attack.fit(x_train, y_train, x_test, y_train) with pytest.raises(ValueError): attack.infer(x_train, y_test)
def test_black_box_tabular_rf(get_tabular_classifier_list, get_iris_dataset): classifier_list = get_tabular_classifier_list(MembershipInferenceBlackBox) if not classifier_list: logging.warning( "Couldn't perform this test because no classifier is defined") return for classifier in classifier_list: attack = MembershipInferenceBlackBox(classifier, attack_model_type="rf") backend_check_membership_accuracy(attack, get_iris_dataset, attack_train_ratio, 0.1)
def test_black_box_loss_regression(art_warning, model_type, get_diabetes_dataset): try: from sklearn import linear_model (x_train_diabetes, y_train_diabetes), _ = get_diabetes_dataset regr_model = linear_model.LinearRegression() regr_model.fit(x_train_diabetes, y_train_diabetes) regressor = ScikitlearnRegressor(regr_model) attack = MembershipInferenceBlackBox(regressor, input_type="loss", attack_model_type=model_type) backend_check_membership_accuracy(attack, get_diabetes_dataset, attack_train_ratio, 0.25) except ARTTestException as e: art_warning(e)
def test_black_box_image(get_default_mnist_subset, image_dl_estimator_for_attack): classifier_list = image_dl_estimator_for_attack( MembershipInferenceBlackBox) if not classifier_list: logging.warning( "Couldn't perform this test because no classifier is defined") return for classifier in classifier_list: attack = MembershipInferenceBlackBox(classifier) backend_check_membership_accuracy(attack, get_default_mnist_subset, attack_train_ratio, 0.03)
def test_black_box_loss_tabular(model_type, get_tabular_classifier_list, get_iris_dataset): classifier_list = get_tabular_classifier_list(MembershipInferenceBlackBox) if not classifier_list: logging.warning( "Couldn't perform this test because no classifier is defined") return for classifier in classifier_list: if type(classifier).__name__ == "PyTorchClassifier" or type( classifier).__name__ == "TensorFlowV2Classifier": attack = MembershipInferenceBlackBox(classifier, input_type="loss", attack_model_type=model_type) backend_check_membership_accuracy(attack, get_iris_dataset, attack_train_ratio, 0.15)
def test_black_box_with_model(get_tabular_classifier_list, get_attack_classifier_list, get_iris_dataset): classifier_list = get_tabular_classifier_list(MembershipInferenceBlackBox) if not classifier_list: logging.warning( "Couldn't perform this test because no classifier is defined") return attack_model_list = get_attack_classifier_list(num_features=2 * num_classes_iris) if not attack_model_list: logging.warning( "Couldn't perform this test because no attack model is defined") return for classifier in classifier_list: for attack_model in attack_model_list: print(type(attack_model).__name__) attack = MembershipInferenceBlackBox(classifier, attack_model=attack_model) backend_check_membership_accuracy(attack, get_iris_dataset, attack_train_ratio, 0.03)
def test_shadow_model_bb_attack(art_warning): try: (x_target, y_target), (x_shadow, y_shadow), _, _ = load_nursery(test_set=0.5) target_train_size = len(x_target) // 2 x_target_train = x_target[:target_train_size] y_target_train = y_target[:target_train_size] x_target_test = x_target[target_train_size:] y_target_test = y_target[target_train_size:] model = RandomForestClassifier(random_state=7) model.fit(x_target_train, y_target_train) art_classifier = ScikitlearnRandomForestClassifier(model) shadow_models = ShadowModels(art_classifier, num_shadow_models=1, random_state=7) shadow_dataset = shadow_models.generate_shadow_dataset( x_shadow, to_categorical(y_shadow, 4)) (mem_x, mem_y, mem_pred), (nonmem_x, nonmem_y, nonmem_pred) = shadow_dataset attack = MembershipInferenceBlackBox(art_classifier, attack_model_type="rf") attack.fit(mem_x, mem_y, nonmem_x, nonmem_y, mem_pred, nonmem_pred) mem_infer = attack.infer(x_target_train, y_target_train) nonmem_infer = attack.infer(x_target_test, y_target_test) mem_acc = np.sum(mem_infer) / len(mem_infer) nonmem_acc = 1 - (np.sum(nonmem_infer) / len(nonmem_infer)) accuracy = (mem_acc * len(mem_infer) + nonmem_acc * len(nonmem_infer)) / (len(mem_infer) + len(nonmem_infer)) assert accuracy == pytest.approx(0.7, abs=0.2) except ARTTestException as e: art_warning(e)