def test_shadow_model_default_randomisation(art_warning):
try:
(x_train, y_train), (_, _), _, _ = load_nursery(test_set=0.2)
model = RandomForestClassifier(random_state=7)
model.fit(x_train, y_train)
art_classifier = ScikitlearnRandomForestClassifier(model)
shadow_models = ShadowModels(art_classifier,
num_shadow_models=1,
random_state=7)
shadow_dataset = shadow_models.generate_synthetic_shadow_dataset(
art_classifier,
dataset_size=40,
max_features_randomized=8,
min_confidence=0.2,
max_retries=15,
random_record_fn=None,
randomize_features_fn=None,
)
(mem_x, mem_y, mem_pred), (nonmem_x, nonmem_y,
nonmem_pred) = shadow_dataset
assert len(mem_x) == len(mem_y)
assert len(mem_y) == len(mem_pred)
assert len(nonmem_x) == len(nonmem_y)
assert len(nonmem_y) == len(nonmem_pred)
assert len(mem_x) + len(mem_y) == 40
except ARTTestException as e:
art_warning(e)
def setUpClass(cls):
master_seed(seed=1234)
super().setUpClass()
cls.sklearn_model = RandomForestClassifier(n_estimators=10)
cls.classifier = ScikitlearnRandomForestClassifier(model=cls.sklearn_model)
cls.classifier.fit(x=cls.x_train_iris, y=cls.y_train_iris)
def test_shadow_model_bb_attack(art_warning):
try:
(x_target, y_target), (x_shadow,
y_shadow), _, _ = load_nursery(test_set=0.5)
target_train_size = len(x_target) // 2
x_target_train = x_target[:target_train_size]
y_target_train = y_target[:target_train_size]
x_target_test = x_target[target_train_size:]
y_target_test = y_target[target_train_size:]
model = RandomForestClassifier(random_state=7)
model.fit(x_target_train, y_target_train)
art_classifier = ScikitlearnRandomForestClassifier(model)
shadow_models = ShadowModels(art_classifier,
num_shadow_models=1,
random_state=7)
shadow_dataset = shadow_models.generate_shadow_dataset(
x_shadow, to_categorical(y_shadow, 4))
(mem_x, mem_y, mem_pred), (nonmem_x, nonmem_y,
nonmem_pred) = shadow_dataset
attack = MembershipInferenceBlackBox(art_classifier,
attack_model_type="rf")
attack.fit(mem_x, mem_y, nonmem_x, nonmem_y, mem_pred, nonmem_pred)
mem_infer = attack.infer(x_target_train, y_target_train)
nonmem_infer = attack.infer(x_target_test, y_target_test)
mem_acc = np.sum(mem_infer) / len(mem_infer)
nonmem_acc = 1 - (np.sum(nonmem_infer) / len(nonmem_infer))
accuracy = (mem_acc * len(mem_infer) + nonmem_acc *
len(nonmem_infer)) / (len(mem_infer) + len(nonmem_infer))
assert accuracy == pytest.approx(0.7, abs=0.2)
except ARTTestException as e:
art_warning(e)
def test_synthetic_shadow_model(art_warning):
try:
(x_train, y_train), (_, _), _, _ = load_nursery(test_set=0.2)
model = RandomForestClassifier(random_state=7)
model.fit(x_train, y_train)
art_classifier = ScikitlearnRandomForestClassifier(model)
shadow_models = ShadowModels(art_classifier,
num_shadow_models=1,
random_state=7)
record_rng = np.random.default_rng(seed=7)
def random_record() -> np.ndarray:
children_values = [0.0, 0.33333333, 0.66666667, 1.0]
categorical_features = [3, 5, 4, 3, 2, 3, 3]
empty_record = np.zeros(1 + np.sum(categorical_features))
empty_record[0] = record_rng.choice(children_values)
offset = 1
for feature_options in categorical_features:
chosen_option = record_rng.integers(feature_options)
empty_record[offset + chosen_option] = 1.0
offset += feature_options
return empty_record
def randomize_features(record: np.ndarray,
num_features: int) -> np.ndarray:
children_values = [0.0, 0.33333333, 0.66666667, 1.0]
categorical_features = [3, 5, 4, 3, 2, 3, 3]
new_record = record.copy()
for feature in record_rng.choice(8, size=num_features):
if feature == 0:
new_record[0] = record_rng.choice(children_values)
else:
cat_feature = feature - 1
one_hot = np.zeros(categorical_features[cat_feature])
one_hot[record_rng.integers(
categorical_features[cat_feature])] = 1.0
feature_offset = 1 + np.sum(
categorical_features[:cat_feature], dtype=np.int64)
new_record[feature_offset:feature_offset +
categorical_features[cat_feature]] = one_hot
return new_record
shadow_dataset = shadow_models.generate_synthetic_shadow_dataset(
art_classifier,
40,
max_features_randomized=8,
random_record_fn=random_record,
randomize_features_fn=randomize_features,
)
(mem_x, mem_y, mem_pred), (nonmem_x, nonmem_y,
nonmem_pred) = shadow_dataset
assert len(mem_x) == len(mem_y)
assert len(mem_y) == len(mem_pred)
assert len(nonmem_x) == len(nonmem_y)
assert len(nonmem_y) == len(nonmem_pred)
assert len(mem_x) + len(mem_y) == 40
except ARTTestException as e:
art_warning(e)
def test_type(self):
self.assertIsInstance(
self.classifier, type(SklearnClassifier(model=self.sklearn_model)))
with self.assertRaises(TypeError):
ScikitlearnRandomForestClassifier(model="sklearn_model")