def test_errors(art_warning, tabular_dl_estimator_for_attack, get_iris_dataset): try: classifier = tabular_dl_estimator_for_attack(MembershipInferenceBlackBox) (x_train, y_train), (x_test, y_test) = get_iris_dataset with pytest.raises(ValueError): MembershipInferenceBlackBox(classifier, attack_model_type="a") with pytest.raises(ValueError): MembershipInferenceBlackBox(classifier, input_type="a") attack = MembershipInferenceBlackBox(classifier) with pytest.raises(ValueError): attack.fit(x_train, y_test, x_test, y_test) with pytest.raises(ValueError): attack.fit(x_train, y_train, x_test, y_train) with pytest.raises(ValueError): attack.infer(x_train, y_test) except ARTTestException as e: art_warning(e)
def test_errors(get_tabular_classifier_list, get_iris_dataset): classifier_list = get_tabular_classifier_list(MembershipInferenceBlackBox) if not classifier_list: logging.warning( "Couldn't perform this test because no classifier is defined") return (x_train, y_train), (x_test, y_test) = get_iris_dataset with pytest.raises(ValueError): MembershipInferenceBlackBox(classifier_list[0], attack_model_type="a") with pytest.raises(ValueError): MembershipInferenceBlackBox(classifier_list[0], input_type="a") attack = MembershipInferenceBlackBox(classifier_list[0]) with pytest.raises(ValueError): attack.fit(x_train, y_test, x_test, y_test) with pytest.raises(ValueError): attack.fit(x_train, y_train, x_test, y_train) with pytest.raises(ValueError): attack.infer(x_train, y_test)
def test_shadow_model_bb_attack(art_warning): try: (x_target, y_target), (x_shadow, y_shadow), _, _ = load_nursery(test_set=0.5) target_train_size = len(x_target) // 2 x_target_train = x_target[:target_train_size] y_target_train = y_target[:target_train_size] x_target_test = x_target[target_train_size:] y_target_test = y_target[target_train_size:] model = RandomForestClassifier(random_state=7) model.fit(x_target_train, y_target_train) art_classifier = ScikitlearnRandomForestClassifier(model) shadow_models = ShadowModels(art_classifier, num_shadow_models=1, random_state=7) shadow_dataset = shadow_models.generate_shadow_dataset( x_shadow, to_categorical(y_shadow, 4)) (mem_x, mem_y, mem_pred), (nonmem_x, nonmem_y, nonmem_pred) = shadow_dataset attack = MembershipInferenceBlackBox(art_classifier, attack_model_type="rf") attack.fit(mem_x, mem_y, nonmem_x, nonmem_y, mem_pred, nonmem_pred) mem_infer = attack.infer(x_target_train, y_target_train) nonmem_infer = attack.infer(x_target_test, y_target_test) mem_acc = np.sum(mem_infer) / len(mem_infer) nonmem_acc = 1 - (np.sum(nonmem_infer) / len(nonmem_infer)) accuracy = (mem_acc * len(mem_infer) + nonmem_acc * len(nonmem_infer)) / (len(mem_infer) + len(nonmem_infer)) assert accuracy == pytest.approx(0.7, abs=0.2) except ARTTestException as e: art_warning(e)