def test_LightGBM(self):
train_data = lightgbm.Dataset(self.x_train, label=np.argmax(self.y_train, axis=1))
test_data = lightgbm.Dataset(self.x_test, label=np.argmax(self.y_test, axis=1))
parameters = {
"objective": "multiclass",
"num_class": self.n_classes,
"metric": "multi_logloss",
"is_unbalance": "true",
"boosting": "gbdt",
"num_leaves": 5,
"feature_fraction": 0.5,
"bagging_fraction": 0.5,
"bagging_freq": 0,
"learning_rate": 0.05,
"verbose": 0,
}
model = lightgbm.train(
parameters, train_data, valid_sets=test_data, num_boost_round=2, early_stopping_rounds=10
)
classifier = LightGBMClassifier(model=model)
rt = RobustnessVerificationTreeModelsCliqueMethod(classifier=classifier)
average_bound, verified_error = rt.verify(
x=self.x_test, y=self.y_test, eps_init=0.3, nb_search_steps=10, max_clique=2, max_level=2
)
self.assertEqual(average_bound, 0.07634765624999999)
self.assertEqual(verified_error, 0.85)
def test_XGBoost(self):
model = XGBClassifier(n_estimators=4, max_depth=6)
model.fit(self.x_train, np.argmax(self.y_train, axis=1))
classifier = XGBoostClassifier(model=model, nb_features=self.n_features, nb_classes=self.n_classes)
rt = RobustnessVerificationTreeModelsCliqueMethod(classifier=classifier)
average_bound, verified_error = rt.verify(
x=self.x_test, y=self.y_test, eps_init=0.3, nb_search_steps=10, max_clique=2, max_level=2
)
self.assertEqual(average_bound, 0.03186914062500001)
self.assertEqual(verified_error, 0.99)
def test_ExtraTrees(self):
model = ExtraTreesClassifier(n_estimators=4, max_depth=6)
model.fit(self.x_train, np.argmax(self.y_train, axis=1))
classifier = SklearnClassifier(model=model)
rt = RobustnessVerificationTreeModelsCliqueMethod(classifier=classifier)
average_bound, verified_error = rt.verify(
x=self.x_test, y=self.y_test, eps_init=0.3, nb_search_steps=10, max_clique=2, max_level=2
)
self.assertEqual(average_bound, 0.05406445312499999)
self.assertEqual(verified_error, 0.96)
def test_GradientBoosting(self):
model = GradientBoostingClassifier(n_estimators=4, max_depth=6)
model.fit(self.x_train, np.argmax(self.y_train, axis=1))
classifier = SklearnClassifier(model=model)
rt = RobustnessVerificationTreeModelsCliqueMethod(classifier=classifier)
average_bound, verified_error = rt.verify(
x=self.x_test, y=self.y_test, eps_init=0.3, nb_search_steps=10, max_clique=2, max_level=2
)
self.assertAlmostEqual(average_bound, 0.009, delta=0.0002)
self.assertEqual(verified_error, 1.0)
def test_RandomForest(self):
model = RandomForestClassifier(n_estimators=4, max_depth=6)
model.fit(self.x_train, np.argmax(self.y_train, axis=1))
classifier = SklearnClassifier(model=model)
rt = RobustnessVerificationTreeModelsCliqueMethod(
classifier=classifier, verbose=False)
average_bound, verified_error = rt.verify(x=self.x_test,
y=self.y_test,
eps_init=0.3,
nb_search_steps=10,
max_clique=2,
max_level=2)
self.assertEqual(average_bound, 0.016482421874999993)
self.assertEqual(verified_error, 1.0)
def test_XGBoost(self):
model = XGBClassifier(n_estimators=4,
max_depth=6,
objective="multi:softprob",
eval_metric="merror")
model.fit(self.x_train, np.argmax(self.y_train, axis=1))
classifier = XGBoostClassifier(model=model,
nb_features=self.n_features,
nb_classes=self.n_classes)
rt = RobustnessVerificationTreeModelsCliqueMethod(
classifier=classifier)
average_bound, verified_error = rt.verify(x=self.x_test,
y=self.y_test,
eps_init=0.3,
nb_search_steps=10,
max_clique=2,
max_level=2)
self.assertEqual(average_bound, 0.0011425781249999997)
self.assertEqual(verified_error, 1.0)