def test_errors(art_warning, tabular_dl_estimator_for_attack, get_iris_dataset):
try:
classifier = tabular_dl_estimator_for_attack(MembershipInferenceBlackBox)
(x_train, y_train), (x_test, y_test) = get_iris_dataset
with pytest.raises(ValueError):
MembershipInferenceBlackBox(classifier, attack_model_type="a")
with pytest.raises(ValueError):
MembershipInferenceBlackBox(classifier, input_type="a")
attack = MembershipInferenceBlackBox(classifier)
with pytest.raises(ValueError):
attack.fit(x_train, y_test, x_test, y_test)
with pytest.raises(ValueError):
attack.fit(x_train, y_train, x_test, y_train)
with pytest.raises(ValueError):
attack.infer(x_train, y_test)
except ARTTestException as e:
art_warning(e)
def test_errors(get_tabular_classifier_list, get_iris_dataset):
classifier_list = get_tabular_classifier_list(MembershipInferenceBlackBox)
if not classifier_list:
logging.warning(
"Couldn't perform this test because no classifier is defined")
return
(x_train, y_train), (x_test, y_test) = get_iris_dataset
with pytest.raises(ValueError):
MembershipInferenceBlackBox(classifier_list[0], attack_model_type="a")
with pytest.raises(ValueError):
MembershipInferenceBlackBox(classifier_list[0], input_type="a")
attack = MembershipInferenceBlackBox(classifier_list[0])
with pytest.raises(ValueError):
attack.fit(x_train, y_test, x_test, y_test)
with pytest.raises(ValueError):
attack.fit(x_train, y_train, x_test, y_train)
with pytest.raises(ValueError):
attack.infer(x_train, y_test)
def test_shadow_model_bb_attack(art_warning):
try:
(x_target, y_target), (x_shadow,
y_shadow), _, _ = load_nursery(test_set=0.5)
target_train_size = len(x_target) // 2
x_target_train = x_target[:target_train_size]
y_target_train = y_target[:target_train_size]
x_target_test = x_target[target_train_size:]
y_target_test = y_target[target_train_size:]
model = RandomForestClassifier(random_state=7)
model.fit(x_target_train, y_target_train)
art_classifier = ScikitlearnRandomForestClassifier(model)
shadow_models = ShadowModels(art_classifier,
num_shadow_models=1,
random_state=7)
shadow_dataset = shadow_models.generate_shadow_dataset(
x_shadow, to_categorical(y_shadow, 4))
(mem_x, mem_y, mem_pred), (nonmem_x, nonmem_y,
nonmem_pred) = shadow_dataset
attack = MembershipInferenceBlackBox(art_classifier,
attack_model_type="rf")
attack.fit(mem_x, mem_y, nonmem_x, nonmem_y, mem_pred, nonmem_pred)
mem_infer = attack.infer(x_target_train, y_target_train)
nonmem_infer = attack.infer(x_target_test, y_target_test)
mem_acc = np.sum(mem_infer) / len(mem_infer)
nonmem_acc = 1 - (np.sum(nonmem_infer) / len(nonmem_infer))
accuracy = (mem_acc * len(mem_infer) + nonmem_acc *
len(nonmem_infer)) / (len(mem_infer) + len(nonmem_infer))
assert accuracy == pytest.approx(0.7, abs=0.2)
except ARTTestException as e:
art_warning(e)
