def test_lime_stability_indices_good_behaviour(self):
np.random.seed(1)
rf = RandomForestClassifier(n_estimators=500)
rf.fit(self.train, self.labels_train)
i = np.random.randint(0, self.test.shape[0])
explainer = LimeTabularExplainerOvr(self.train,
mode="classification",
feature_names=self.feature_names,
class_names=self.target_names,
discretize_continuous=True)
indices = explainer.check_stability(data_row=self.test[i],
predict_fn=rf.predict_proba,
num_features=2,
n_calls=10,
index_verbose=True,
verbose=True,
model_regressor=None)
self.assertIsNotNone(indices)
csi, vsi = indices
self.assertTrue((csi >= 0) & (csi <= 100),
"CSI Index value is not in the range [0,100]")
self.assertTrue((vsi >= 0) & (vsi <= 100),
"VSI Index value is not in the range [0,100]")
def test_lime_explainer_entropy_discretizer(self):
np.random.seed(1)
rf = RandomForestClassifier(n_estimators=500)
rf.fit(self.train, self.labels_train)
i = np.random.randint(0, self.test.shape[0])
explainer = LimeTabularExplainerOvr(self.train,
feature_names=self.feature_names,
class_names=self.target_names,
training_labels=self.labels_train,
discretize_continuous=True,
discretizer='entropy')
exp = explainer.explain_instance(self.test[i],
rf.predict_proba,
num_features=2)
self.assertIsNotNone(exp)
keys = [x[0] for x in exp.as_list()]
print(keys)
self.assertEqual(1,
sum([1 if 'petal width' in x else 0 for x in keys]),
"Petal Width is a major feature")
self.assertEqual(1,
sum([1 if 'petal length' in x else 0 for x in keys]),
"Petal Length is a major feature")
def test_lime_explainer_good_regressor(self):
np.random.seed(1)
rf = RandomForestClassifier(n_estimators=500)
rf.fit(self.train, self.labels_train)
i = np.random.randint(0, self.test.shape[0])
explainer = LimeTabularExplainerOvr(self.train,
mode="classification",
feature_names=self.feature_names,
class_names=self.target_names,
discretize_continuous=True)
exp = explainer.explain_instance(self.test[i],
rf.predict_proba,
num_features=2,
model_regressor=LinearRegression())
self.assertIsNotNone(exp)
keys = [x[0] for x in exp.as_list()]
self.assertEqual(1,
sum([1 if 'petal width' in x else 0 for x in keys]),
"Petal Width is a major feature")
self.assertEqual(1,
sum([1 if 'petal length' in x else 0 for x in keys]),
"Petal Length is a major feature")
def testFeatureNamesAndCategoricalFeats(self):
training_data = np.array([[0., 1.], [1., 0.]])
explainer = LimeTabularExplainerOvr(training_data=training_data)
self.assertEqual(explainer.feature_names, ['0', '1'])
self.assertEqual(explainer.categorical_features, [0, 1])
explainer = LimeTabularExplainerOvr(training_data=training_data,
feature_names=np.array(
['one', 'two']))
self.assertEqual(explainer.feature_names, ['one', 'two'])
explainer = LimeTabularExplainerOvr(training_data=training_data,
categorical_features=np.array([0]),
discretize_continuous=False)
self.assertEqual(explainer.categorical_features, [0])
def test_lime_explainer_bad_regressor(self):
rf = RandomForestClassifier(n_estimators=500)
rf.fit(self.train, self.labels_train)
lasso = Lasso(alpha=1, fit_intercept=True)
i = np.random.randint(0, self.test.shape[0])
with self.assertRaises(TypeError):
explainer = LimeTabularExplainerOvr(
self.train,
mode="classification",
feature_names=self.feature_names,
class_names=self.target_names,
discretize_continuous=True)
exp = explainer.explain_instance(
self.test[i], # noqa:F841
rf.predict_proba,
num_features=2,
top_labels=1,
model_regressor=lasso)
def test_lime_explainer_sparse_synthetic_data(self):
n_features = 20
X, y = make_multilabel_classification(n_samples=100,
sparse=True,
n_features=n_features,
n_classes=1,
n_labels=2)
rf = RandomForestClassifier(n_estimators=500)
rf.fit(X, y)
instance = np.random.randint(0, X.shape[0])
feature_names = ["feature" + str(i) for i in range(n_features)]
explainer = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True)
exp = explainer.explain_instance(X[instance], rf.predict_proba)
self.assertIsNotNone(exp)
self.assertEqual(10, len(exp.as_list()))
def test_lime_explainer_good_regressor_synthetic_data(self):
X, y = make_classification(n_samples=1000,
n_features=20,
n_informative=2,
n_redundant=2,
random_state=10)
rf = RandomForestClassifier(n_estimators=500)
rf.fit(X, y)
instance = np.random.randint(0, X.shape[0])
feature_names = ["feature" + str(i) for i in range(20)]
explainer = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True)
exp = explainer.explain_instance(X[instance], rf.predict_proba)
self.assertIsNotNone(exp)
self.assertEqual(10, len(exp.as_list()))
def test_lime_stability_indices_model_error(self):
np.random.seed(1)
rf = RandomForestClassifier(n_estimators=500)
rf.fit(self.train, self.labels_train)
lin_regr = LinearRegression(fit_intercept=True)
i = np.random.randint(0, self.test.shape[0])
explainer = LimeTabularExplainerOvr(self.train,
mode="classification",
feature_names=self.feature_names,
class_names=self.target_names,
discretize_continuous=True)
with self.assertRaises(LocalModelError):
exp = explainer.check_stability(self.test[i],
rf.predict_proba,
num_features=2,
n_calls=10,
model_regressor=lin_regr)
def testFeatureValues(self):
training_data = np.array([[0, 0, 2], [1, 1, 0], [0, 2, 2], [1, 3, 0]])
explainer = LimeTabularExplainerOvr(training_data=training_data,
categorical_features=[0, 1, 2])
self.assertEqual(set(explainer.feature_values[0]), {0, 1})
self.assertEqual(set(explainer.feature_values[1]), {0, 1, 2, 3})
self.assertEqual(set(explainer.feature_values[2]), {0, 2})
assert_array_equal(explainer.feature_frequencies[0], np.array([.5,
.5]))
assert_array_equal(explainer.feature_frequencies[1],
np.array([.25, .25, .25, .25]))
assert_array_equal(explainer.feature_frequencies[2], np.array([.5,
.5]))
def test_lime_explainer_with_data_stats(self):
np.random.seed(1)
rf = RandomForestClassifier(n_estimators=500)
rf.fit(self.train, self.labels_train)
i = np.random.randint(0, self.test.shape[0])
# Generate stats using a quartile descritizer
descritizer = QuartileDiscretizer(self.train, [],
self.feature_names,
self.target_names,
random_state=20)
d_means = descritizer.means
d_stds = descritizer.stds
d_mins = descritizer.mins
d_maxs = descritizer.maxs
d_bins = descritizer.bins(self.train, self.target_names)
# Compute feature values and frequencies of all columns
cat_features = np.arange(self.train.shape[1])
discretized_training_data = descritizer.discretize(self.train)
feature_values = {}
feature_frequencies = {}
for feature in cat_features:
column = discretized_training_data[:, feature]
feature_count = collections.Counter(column)
values, frequencies = map(list, zip(*(feature_count.items())))
feature_values[feature] = values
feature_frequencies[feature] = frequencies
# Convert bins to list from array
d_bins_revised = {}
index = 0
for bin in d_bins:
d_bins_revised[index] = bin.tolist()
index = index + 1
# Descritized stats
data_stats = {}
data_stats["means"] = d_means
data_stats["stds"] = d_stds
data_stats["maxs"] = d_maxs
data_stats["mins"] = d_mins
data_stats["bins"] = d_bins_revised
data_stats["feature_values"] = feature_values
data_stats["feature_frequencies"] = feature_frequencies
data = np.zeros((2, len(self.feature_names)))
explainer = LimeTabularExplainerOvr(data,
feature_names=self.feature_names,
random_state=10,
training_data_stats=data_stats,
training_labels=self.target_names)
exp = explainer.explain_instance(self.test[i],
rf.predict_proba,
num_features=2,
model_regressor=LinearRegression())
self.assertIsNotNone(exp)
keys = [x[0] for x in exp.as_list()]
self.assertEqual(1,
sum([1 if 'petal width' in x else 0 for x in keys]),
"Petal Width is a major feature")
self.assertEqual(1,
sum([1 if 'petal length' in x else 0 for x in keys]),
"Petal Length is a major feature")
def test_lime_tabular_explainer_not_equal_random_state(self):
X, y = make_classification(n_samples=1000,
n_features=20,
n_informative=2,
n_redundant=2,
random_state=10)
rf = RandomForestClassifier(n_estimators=500, random_state=10)
rf.fit(X, y)
instance = np.random.RandomState(10).randint(0, X.shape[0])
feature_names = ["feature" + str(i) for i in range(20)]
# ----------------------------------------------------------------------
# -------------------------Quartile Discretizer-------------------------
# ----------------------------------------------------------------------
# ---------------------------------[1]----------------------------------
discretizer = QuartileDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_1 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=10)
exp_1 = explainer_1.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
discretizer = QuartileDiscretizer(X, [],
feature_names,
y,
random_state=10)
explainer_2 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=10)
exp_2 = explainer_2.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
self.assertTrue(exp_1.as_map() != exp_2.as_map())
# ---------------------------------[2]----------------------------------
discretizer = QuartileDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_1 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=20)
exp_1 = explainer_1.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
discretizer = QuartileDiscretizer(X, [],
feature_names,
y,
random_state=10)
explainer_2 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=10)
exp_2 = explainer_2.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
self.assertTrue(exp_1.as_map() != exp_2.as_map())
# ---------------------------------[3]----------------------------------
discretizer = QuartileDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_1 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=20)
exp_1 = explainer_1.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
discretizer = QuartileDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_2 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=10)
exp_2 = explainer_2.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
self.assertTrue(exp_1.as_map() != exp_2.as_map())
# ---------------------------------[4]----------------------------------
discretizer = QuartileDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_1 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=20)
exp_1 = explainer_1.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
discretizer = QuartileDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_2 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=20)
exp_2 = explainer_2.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
self.assertFalse(exp_1.as_map() != exp_2.as_map())
# ----------------------------------------------------------------------
# --------------------------Decile Discretizer--------------------------
# ----------------------------------------------------------------------
# ---------------------------------[1]----------------------------------
discretizer = DecileDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_1 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=10)
exp_1 = explainer_1.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
discretizer = DecileDiscretizer(X, [],
feature_names,
y,
random_state=10)
explainer_2 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=10)
exp_2 = explainer_2.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
self.assertTrue(exp_1.as_map() != exp_2.as_map())
# ---------------------------------[2]----------------------------------
discretizer = DecileDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_1 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=20)
exp_1 = explainer_1.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
discretizer = DecileDiscretizer(X, [],
feature_names,
y,
random_state=10)
explainer_2 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=10)
exp_2 = explainer_2.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
self.assertTrue(exp_1.as_map() != exp_2.as_map())
# ---------------------------------[3]----------------------------------
discretizer = DecileDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_1 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=20)
exp_1 = explainer_1.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
discretizer = DecileDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_2 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=10)
exp_2 = explainer_2.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
self.assertTrue(exp_1.as_map() != exp_2.as_map())
# ---------------------------------[4]----------------------------------
discretizer = DecileDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_1 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=20)
exp_1 = explainer_1.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
discretizer = DecileDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_2 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=20)
exp_2 = explainer_2.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
self.assertFalse(exp_1.as_map() != exp_2.as_map())
# ----------------------------------------------------------------------
# --------------------------Entropy Discretizer-------------------------
# ----------------------------------------------------------------------
# ---------------------------------[1]----------------------------------
discretizer = EntropyDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_1 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=10)
exp_1 = explainer_1.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
discretizer = EntropyDiscretizer(X, [],
feature_names,
y,
random_state=10)
explainer_2 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=10)
exp_2 = explainer_2.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
self.assertTrue(exp_1.as_map() != exp_2.as_map())
# ---------------------------------[2]----------------------------------
discretizer = EntropyDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_1 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=20)
exp_1 = explainer_1.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
discretizer = EntropyDiscretizer(X, [],
feature_names,
y,
random_state=10)
explainer_2 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=10)
exp_2 = explainer_2.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
self.assertTrue(exp_1.as_map() != exp_2.as_map())
# ---------------------------------[3]----------------------------------
discretizer = EntropyDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_1 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=20)
exp_1 = explainer_1.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
discretizer = EntropyDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_2 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=10)
exp_2 = explainer_2.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
self.assertTrue(exp_1.as_map() != exp_2.as_map())
# ---------------------------------[4]----------------------------------
discretizer = EntropyDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_1 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=20)
exp_1 = explainer_1.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
discretizer = EntropyDiscretizer(X, [],
feature_names,
y,
random_state=20)
explainer_2 = LimeTabularExplainerOvr(X,
feature_names=feature_names,
discretize_continuous=True,
discretizer=discretizer,
random_state=20)
exp_2 = explainer_2.explain_instance(X[instance],
rf.predict_proba,
num_samples=500)
self.assertFalse(exp_1.as_map() != exp_2.as_map())