def test_general_wines_lr(wine_dataset):
"""
Check whether the produced adversaries are correct,
given Logistic Regression classifier and sklearn wines dataset.
"""
(x_train, y_train, x_valid, y_valid), _, clip_values = wine_dataset
# Setup classifier
lr_clf = LogisticRegression(penalty="none")
lr_clf.fit(x_train, y_train)
clf_slr = ScikitlearnLogisticRegression(model=lr_clf,
clip_values=clip_values)
lpf_slr = LowProFool(classifier=clf_slr, n_steps=80, eta=0.1, lambd=1.25)
lpf_slr.fit_importances(x_train, y_train)
sample = x_valid
# Draw targets different from original labels and then save as one-hot encoded.
target = np.eye(3)[np.array(
y_valid.apply(
lambda x: np.random.choice([i for i in [0, 1, 2] if i != x])))]
adversaries = lpf_slr.generate(x=sample, y=target)
expected = np.argmax(target, axis=1)
predicted = np.argmax(lr_clf.predict_proba(adversaries), axis=1)
correct = expected == predicted
success_rate = np.sum(correct) / correct.shape[0]
expected = 0.75
logger.info(
"[Wines, Scikit-learn Logistic Regression] success rate of adversarial attack (expected >{:.2f}):"
" {:.2f}%".format(expected * 100, success_rate * 100))
assert success_rate > expected
def setUpClass(cls):
master_seed(seed=1234)
super().setUpClass()
cls.sklearn_model = LogisticRegression(
verbose=0, C=1, solver="newton-cg", dual=False, fit_intercept=True, multi_class="ovr"
)
cls.classifier = ScikitlearnLogisticRegression(model=cls.sklearn_model)
cls.classifier.fit(x=cls.x_train_iris, y=cls.y_train_iris)
def setUpClass(cls):
master_seed(seed=1234)
super().setUpClass()
binary_class_index = np.argmax(cls.y_train_iris, axis=1) < 2
x_train_binary = cls.x_train_iris[
binary_class_index,
]
y_train_binary = cls.y_train_iris[binary_class_index,][:, [0, 1]]
cls.sklearn_model = LogisticRegression(
verbose=0, C=1, solver="newton-cg", dual=False, fit_intercept=True, multi_class="ovr"
)
cls.classifier = ScikitlearnLogisticRegression(model=cls.sklearn_model)
cls.classifier.fit(x=x_train_binary, y=y_train_binary)
def test_database_reconstruction_logistic_regression(get_iris_dataset):
(x_train_iris, y_train_iris), (x_test_iris, y_test_iris) = get_iris_dataset
y_train_iris = np.array([np.argmax(y) for y in y_train_iris])
y_test_iris = np.array([np.argmax(y) for y in y_test_iris])
x_private = x_test_iris[0, :].reshape(1, -1)
y_private = y_test_iris[0]
x_input = np.vstack((x_train_iris, x_private))
y_input = np.hstack((y_train_iris, y_private))
nb_private = LogisticRegression()
nb_private.fit(x_input, y_input)
estimator_private = ScikitlearnLogisticRegression(model=nb_private)
recon = DatabaseReconstruction(estimator=estimator_private)
x_recon, y_recon = recon.reconstruct(x_train_iris, y_train_iris)
assert x_recon is not None
assert x_recon.shape == (1, 4)
assert y_recon.shape == (1, 3)
assert np.isclose(x_recon, x_private, rtol=0.05).all()
assert np.argmax(y_recon, axis=1) == y_private
def test_type(self):
self.assertIsInstance(
self.classifier, type(SklearnClassifier(model=self.sklearn_model)))
with self.assertRaises(TypeError):
ScikitlearnLogisticRegression(model="sklearn_model")
def test_clipping(iris_dataset):
"""
Check weather adversaries are clipped properly.
"""
(x_train, y_train, x_valid, y_valid), _, clip_values = iris_dataset
# Setup classifier
lr_clf = LogisticRegression(penalty="none")
lr_clf.fit(x_train, y_train)
# Dataset min-max clipping values
bottom_min, top_max = clip_values
clf_slr_min_max = ScikitlearnLogisticRegression(model=lr_clf,
clip_values=(bottom_min,
top_max))
# Clip values
bottom_custom = -3
top_custom = 3
clf_slr_custom = ScikitlearnLogisticRegression(model=lr_clf,
clip_values=(bottom_custom,
top_custom))
# Setting up LowProFool classes with different hyper-parameters
lpf_min_max_default = LowProFool(classifier=clf_slr_min_max,
n_steps=45,
eta=0.02,
lambd=1.5)
lpf_min_max_high_eta = LowProFool(classifier=clf_slr_min_max,
n_steps=45,
eta=100000,
lambd=1.5)
lpf_custom_default = LowProFool(classifier=clf_slr_custom,
n_steps=45,
eta=0.02,
lambd=1.5)
lpf_custom_high_eta = LowProFool(classifier=clf_slr_custom,
n_steps=45,
eta=100000,
lambd=1.5)
lpf_min_max_default.fit_importances(x_train, y_train)
lpf_min_max_high_eta.fit_importances(x_train, y_train)
lpf_custom_default.fit_importances(x_train, y_train)
lpf_custom_high_eta.fit_importances(x_train, y_train)
# Generating adversaries
sample = np.array([[5.5, 2.4, 3.7, 1.0]])
target = np.array([[0.0, 0.0, 1.0]])
adversaries_min_max_default = lpf_min_max_default.generate(x=sample,
y=target)
adversaries_min_max_high_eta = lpf_min_max_high_eta.generate(x=sample,
y=target)
adversaries_custom_default = lpf_custom_default.generate(x=sample,
y=target)
adversaries_custom_high_eta = lpf_custom_high_eta.generate(x=sample,
y=target)
# Checking whether adversaries were clipped properly
eps = 1e-6
is_valid_1 = (
(bottom_min - eps).to_numpy() <=
adversaries_min_max_default).all() and (
(top_max + eps).to_numpy() >= adversaries_min_max_default).all()
is_valid_2 = (
(bottom_min - eps).to_numpy() <=
adversaries_min_max_high_eta).all() and (
(top_max + eps).to_numpy() >= adversaries_min_max_high_eta).all()
is_valid_3 = (
(bottom_custom - eps) <= adversaries_custom_default).all() and (
(top_custom + eps) >= adversaries_custom_default).all()
is_valid_4 = (bottom_custom - eps <= adversaries_custom_high_eta).all(
) and (top_custom + eps >= adversaries_custom_high_eta).all()
is_clipping_valid = is_valid_1 and is_valid_2 and is_valid_3 and is_valid_4
if is_clipping_valid:
logger.info(
"[Iris flower, Scikit-learn Logistic Regression] Clipping is valid."
)
else:
logger.info(
"[Iris flower, Scikit-learn Logistic Regression] Clipping is invalid."
)
assert is_valid_1
assert is_valid_2
assert is_valid_3
assert is_valid_4
def test_fit_importances(iris_dataset):
"""
Check whether feature importance is calculated properly.
"""
(x_train, y_train, x_valid, y_valid), _, clip_values = iris_dataset
def pearson_correlations(x, y):
correlations = [pearsonr(x[:, col], y)[0] for col in range(x.shape[1])]
absolutes = np.abs(np.array(correlations))
result = absolutes / np.power(np.sum(absolutes**2), 0.5)
return result
# Setup classifier
lr_clf = LogisticRegression(penalty="none")
lr_clf.fit(x_train, y_train)
clf_slr = ScikitlearnLogisticRegression(model=lr_clf,
clip_values=clip_values)
# User defined vector
vector = pearson_correlations(np.array(x_train), np.array(y_train))
# 3 different instances of LowProFool, using 3 different ways of specifying importance
# Default version - using pearson correlation under the hood
lpf_slr_default = LowProFool(classifier=clf_slr,
n_steps=45,
eta=0.02,
lambd=1.5,
importance="pearson")
# Predefined vector, passed in LowProFool initialization
lpf_slr_vec = LowProFool(classifier=clf_slr,
n_steps=45,
eta=0.02,
lambd=1.5,
importance=vector)
# User defined function
lpf_slr_fun = LowProFool(classifier=clf_slr,
n_steps=45,
eta=0.02,
lambd=1.5,
importance=pearson_correlations)
lpf_slr_default.fit_importances(x_train, y_train)
lpf_slr_vec.fit_importances(x_train, y_train)
lpf_slr_fun.fit_importances(x_train, y_train, normalize=False)
importance_default = lpf_slr_default.importance_vec
importance_vec_init = lpf_slr_vec.importance_vec
importance_function = lpf_slr_fun.importance_vec
# Predefined vector passed while fitting
lpf_slr_default.fit_importances(x_train, y_train, importance_array=vector)
importance_vec_fit = lpf_slr_default.importance_vec
# Vector normalization
vector_norm = vector / np.sum(vector)
is_default_valid = (vector_norm == importance_default).all()
is_custom_fun_valid = (vector == importance_function).all()
is_vec_init_valid = (vector_norm == importance_vec_init).all()
is_vec_fit_valid = (vector_norm == importance_vec_fit).all()
logger.info(
"[Iris flower, Scikit-learn Logistic Regression] Importance fitting test:"
)
if not is_default_valid:
logger.info("Fitting importance by default is invalid")
elif not is_custom_fun_valid:
logger.info("Fitting importance with custom function is invalid")
elif not is_vec_init_valid:
logger.info(
"Fitting importance with vector provided in initializer is invalid"
)
elif not is_vec_fit_valid:
logger.info(
"Fitting importance with vector provided in fit_importances() is invalid"
)
else:
logger.info(
"Fitting importance with all available methods went successfully")
assert is_default_valid
assert is_custom_fun_valid
assert is_vec_init_valid
assert is_vec_fit_valid