def test_black_box_keras_loss(art_warning, get_iris_dataset):
try:
(x_train, y_train), (_, _) = get_iris_dataset
# This test creates a framework-specific (keras) model because it needs to check both the case of a string-based
# loss and a class-based loss, and therefore cannot use the generic fixture get_tabular_classifier_list
model = keras.models.Sequential()
model.add(keras.layers.Dense(8, input_dim=4, activation="relu"))
model.add(keras.layers.Dense(3, activation="softmax"))
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
model.fit(x_train, y_train, epochs=150, batch_size=10)
classifier = KerasClassifier(model)
attack = MembershipInferenceBlackBox(classifier, input_type="loss")
backend_check_membership_accuracy(attack, get_iris_dataset,
attack_train_ratio, 0.15)
model2 = keras.models.Sequential()
model2.add(keras.layers.Dense(12, input_dim=4, activation="relu"))
model2.add(keras.layers.Dense(3, activation="softmax"))
model2.compile(loss=keras.losses.CategoricalCrossentropy(),
optimizer="adam",
metrics=["accuracy"])
model2.fit(x_train, y_train, epochs=150, batch_size=10)
classifier = KerasClassifier(model2)
attack = MembershipInferenceBlackBox(classifier, input_type="loss")
backend_check_membership_accuracy(attack, get_iris_dataset,
attack_train_ratio, 0.15)
except ARTTestException as e:
art_warning(e)
def test_black_box_tabular(art_warning, model_type, tabular_dl_estimator_for_attack, get_iris_dataset):
try:
classifier = tabular_dl_estimator_for_attack(MembershipInferenceBlackBox)
attack = MembershipInferenceBlackBox(classifier, attack_model_type=model_type)
backend_check_membership_accuracy(attack, get_iris_dataset, attack_train_ratio, 0.25)
except ARTTestException as e:
art_warning(e)
def test_black_box_tabular_prob_nn(art_warning, tabular_dl_estimator_for_attack, get_iris_dataset):
try:
classifier = tabular_dl_estimator_for_attack(MembershipInferenceBlackBox)
attack = MembershipInferenceBlackBox(classifier, attack_model_type="nn")
backend_check_membership_probabilities(attack, get_iris_dataset, attack_train_ratio)
except ARTTestException as e:
art_warning(e)
def test_black_box_image(art_warning, get_default_mnist_subset, image_dl_estimator_for_attack):
try:
classifier = image_dl_estimator_for_attack(MembershipInferenceBlackBox)
attack = MembershipInferenceBlackBox(classifier)
backend_check_membership_accuracy(attack, get_default_mnist_subset, attack_train_ratio, 0.25)
except ARTTestException as e:
art_warning(e)
def test_black_box_loss_tabular(art_warning, model_type, tabular_dl_estimator_for_attack, get_iris_dataset):
try:
classifier = tabular_dl_estimator_for_attack(MembershipInferenceBlackBox)
if type(classifier).__name__ == "PyTorchClassifier" or type(classifier).__name__ == "TensorFlowV2Classifier":
attack = MembershipInferenceBlackBox(classifier, input_type="loss", attack_model_type=model_type)
backend_check_membership_accuracy(attack, get_iris_dataset, attack_train_ratio, 0.25)
except ARTTestException as e:
art_warning(e)
def test_black_box_with_model(art_warning, tabular_dl_estimator_for_attack, estimator_for_attack, get_iris_dataset):
try:
classifier = tabular_dl_estimator_for_attack(MembershipInferenceBlackBox)
attack_model = estimator_for_attack(num_features=2 * num_classes_iris)
attack = MembershipInferenceBlackBox(classifier, attack_model=attack_model)
backend_check_membership_accuracy(attack, get_iris_dataset, attack_train_ratio, 0.25)
except ARTTestException as e:
art_warning(e)
def test_errors(art_warning, tabular_dl_estimator_for_attack, get_iris_dataset):
try:
classifier = tabular_dl_estimator_for_attack(MembershipInferenceBlackBox)
(x_train, y_train), (x_test, y_test) = get_iris_dataset
with pytest.raises(ValueError):
MembershipInferenceBlackBox(classifier, attack_model_type="a")
with pytest.raises(ValueError):
MembershipInferenceBlackBox(classifier, input_type="a")
attack = MembershipInferenceBlackBox(classifier)
with pytest.raises(ValueError):
attack.fit(x_train, y_test, x_test, y_test)
with pytest.raises(ValueError):
attack.fit(x_train, y_train, x_test, y_train)
with pytest.raises(ValueError):
attack.infer(x_train, y_test)
except ARTTestException as e:
art_warning(e)
def test_errors(get_tabular_classifier_list, get_iris_dataset):
classifier_list = get_tabular_classifier_list(MembershipInferenceBlackBox)
if not classifier_list:
logging.warning(
"Couldn't perform this test because no classifier is defined")
return
(x_train, y_train), (x_test, y_test) = get_iris_dataset
with pytest.raises(ValueError):
MembershipInferenceBlackBox(classifier_list[0], attack_model_type="a")
with pytest.raises(ValueError):
MembershipInferenceBlackBox(classifier_list[0], input_type="a")
attack = MembershipInferenceBlackBox(classifier_list[0])
with pytest.raises(ValueError):
attack.fit(x_train, y_test, x_test, y_test)
with pytest.raises(ValueError):
attack.fit(x_train, y_train, x_test, y_train)
with pytest.raises(ValueError):
attack.infer(x_train, y_test)
def test_black_box_tabular_rf(get_tabular_classifier_list, get_iris_dataset):
classifier_list = get_tabular_classifier_list(MembershipInferenceBlackBox)
if not classifier_list:
logging.warning(
"Couldn't perform this test because no classifier is defined")
return
for classifier in classifier_list:
attack = MembershipInferenceBlackBox(classifier,
attack_model_type="rf")
backend_check_membership_accuracy(attack, get_iris_dataset,
attack_train_ratio, 0.1)
def test_black_box_loss_regression(art_warning, model_type, get_diabetes_dataset):
try:
from sklearn import linear_model
(x_train_diabetes, y_train_diabetes), _ = get_diabetes_dataset
regr_model = linear_model.LinearRegression()
regr_model.fit(x_train_diabetes, y_train_diabetes)
regressor = ScikitlearnRegressor(regr_model)
attack = MembershipInferenceBlackBox(regressor, input_type="loss", attack_model_type=model_type)
backend_check_membership_accuracy(attack, get_diabetes_dataset, attack_train_ratio, 0.25)
except ARTTestException as e:
art_warning(e)
def test_black_box_image(get_default_mnist_subset,
image_dl_estimator_for_attack):
classifier_list = image_dl_estimator_for_attack(
MembershipInferenceBlackBox)
if not classifier_list:
logging.warning(
"Couldn't perform this test because no classifier is defined")
return
for classifier in classifier_list:
attack = MembershipInferenceBlackBox(classifier)
backend_check_membership_accuracy(attack, get_default_mnist_subset,
attack_train_ratio, 0.03)
def test_black_box_loss_tabular(model_type, get_tabular_classifier_list,
get_iris_dataset):
classifier_list = get_tabular_classifier_list(MembershipInferenceBlackBox)
if not classifier_list:
logging.warning(
"Couldn't perform this test because no classifier is defined")
return
for classifier in classifier_list:
if type(classifier).__name__ == "PyTorchClassifier" or type(
classifier).__name__ == "TensorFlowV2Classifier":
attack = MembershipInferenceBlackBox(classifier,
input_type="loss",
attack_model_type=model_type)
backend_check_membership_accuracy(attack, get_iris_dataset,
attack_train_ratio, 0.15)
def test_black_box_with_model(get_tabular_classifier_list,
get_attack_classifier_list, get_iris_dataset):
classifier_list = get_tabular_classifier_list(MembershipInferenceBlackBox)
if not classifier_list:
logging.warning(
"Couldn't perform this test because no classifier is defined")
return
attack_model_list = get_attack_classifier_list(num_features=2 *
num_classes_iris)
if not attack_model_list:
logging.warning(
"Couldn't perform this test because no attack model is defined")
return
for classifier in classifier_list:
for attack_model in attack_model_list:
print(type(attack_model).__name__)
attack = MembershipInferenceBlackBox(classifier,
attack_model=attack_model)
backend_check_membership_accuracy(attack, get_iris_dataset,
attack_train_ratio, 0.03)
def test_shadow_model_bb_attack(art_warning):
try:
(x_target, y_target), (x_shadow,
y_shadow), _, _ = load_nursery(test_set=0.5)
target_train_size = len(x_target) // 2
x_target_train = x_target[:target_train_size]
y_target_train = y_target[:target_train_size]
x_target_test = x_target[target_train_size:]
y_target_test = y_target[target_train_size:]
model = RandomForestClassifier(random_state=7)
model.fit(x_target_train, y_target_train)
art_classifier = ScikitlearnRandomForestClassifier(model)
shadow_models = ShadowModels(art_classifier,
num_shadow_models=1,
random_state=7)
shadow_dataset = shadow_models.generate_shadow_dataset(
x_shadow, to_categorical(y_shadow, 4))
(mem_x, mem_y, mem_pred), (nonmem_x, nonmem_y,
nonmem_pred) = shadow_dataset
attack = MembershipInferenceBlackBox(art_classifier,
attack_model_type="rf")
attack.fit(mem_x, mem_y, nonmem_x, nonmem_y, mem_pred, nonmem_pred)
mem_infer = attack.infer(x_target_train, y_target_train)
nonmem_infer = attack.infer(x_target_test, y_target_test)
mem_acc = np.sum(mem_infer) / len(mem_infer)
nonmem_acc = 1 - (np.sum(nonmem_infer) / len(nonmem_infer))
accuracy = (mem_acc * len(mem_infer) + nonmem_acc *
len(nonmem_infer)) / (len(mem_infer) + len(nonmem_infer))
assert accuracy == pytest.approx(0.7, abs=0.2)
except ARTTestException as e:
art_warning(e)
