def test_lowercase_tokens(self):
def _predict_fn(sentences):
return np.random.uniform(0., 1., [len(list(sentences)), 2])
sentence = 'It is a great movie but it is also somewhat bad .'
counterfactuals = [
'It is an ok movie but its also somewhat bad .',
'It is a terrible movie but it is also somewhat bad .',
'It is a good movie but it is also somewhat bad .',
'It was a good movie but it is also somewhat bad .',
'It was a great film but it is also somewhat bad .',
'It was a great show but it is bad also somewhat bad .',
'It was the great movie but it is also somewhat bad .',
'It was a movie but is somewhat bad .',
'It was a movie and also it is somewhat bad .',
'It was a movie but also it is very bad .',
'It was a great but also it is bad .',
'There is a good movie but also is somewhat bad .',
'is a great movie but also it is somewhat bad .',
'is a great movie but also it is somewhat .',
'is a great movie also it is somewhat bad .',
'is a great also it is somewhat .'
]
explanation_lowercase = lemon.explain(sentence,
counterfactuals,
_predict_fn,
class_to_explain=1,
lowercase_tokens=True,
return_model=True)
# Check that the number of model coefficients is equal to the number of
# unique tokens in the original sentence.
tokens = [token.lower() for token in sentence.split()]
unique_tokens = set(tokens)
self.assertLen(explanation_lowercase.model.coef_, len(unique_tokens))
# Check that the importance value for 'It' and 'it' are the same.
self.assertEqual(explanation_lowercase.feature_importance[0],
explanation_lowercase.feature_importance[6])
explanation_not_lowercase = lemon.explain(sentence,
counterfactuals,
_predict_fn,
class_to_explain=1,
lowercase_tokens=False,
return_model=True)
# Check that the number of model coefficients is equal to the number of
# unique tokens in the original sentence.
tokens = sentence.split()
unique_tokens = set(tokens)
self.assertLen(explanation_not_lowercase.model.coef_,
len(unique_tokens))
# Check that the importance value for 'It' and 'it' are not the same.
self.assertNotEqual(explanation_not_lowercase.feature_importance[0],
explanation_not_lowercase.feature_importance[6])
def test_explain_returns_explanation_with_intercept(self):
"""Tests if the explanation contains an intercept value."""
def _predict_fn(sentences):
return np.random.uniform(0., 1., [len(list(sentences)), 2])
explanation = lemon.explain('Test sentence', ['Test counterfactual'],
_predict_fn,
class_to_explain=1)
self.assertNotEqual(explanation.intercept, 0.)
def test_explain_returns_explanation_with_prediction(self):
"""Tests if the explanation contains a prediction."""
def _predict_fn(sentences):
return np.random.uniform(0., 1., [len(list(sentences)), 2])
explanation = lemon.explain('Test sentence', ['Test counterfactual'],
_predict_fn,
class_to_explain=1,
return_prediction=True)
self.assertIsNotNone(explanation.prediction)
def test_duplicate_tokens(self):
"""Checks the explanation for a sentence with duplicate tokens."""
def _predict_fn(sentences):
return np.random.uniform(0., 1., [len(list(sentences)), 2])
sentence = 'it is a great movie but it is also somewhat bad .'
counterfactuals = [
'it is an ok movie but its also somewhat bad .',
'it is a terrible movie but it is also somewhat bad .',
'it is a good movie but it is also somewhat bad .',
'it was a good movie but it also somewhat bad .',
'it was a great film but it is also somewhat bad .',
'it was a great show but it is bad also somewhat bad .',
'it was the great movie but it is also somewhat bad .',
'it was a movie but is somewhat bad .',
'it was a movie and also it is somewhat bad .',
'it was a movie but also it is very bad .',
'it was a great but also it is bad .',
'There is a good movie but also is somewhat bad .',
'is a great movie but also it somewhat bad .',
'is a great movie but also it is somewhat .',
'is a great movie also it is somewhat bad .',
'is a great also it is somewhat .'
]
explanation = lemon.explain(sentence,
counterfactuals,
_predict_fn,
class_to_explain=1,
return_model=True)
# Check that the number of model coefficients is equal to the number of
# unique tokens in the original sentence.
tokens = sentence.split()
unique_tokens = set(tokens)
self.assertLen(explanation.model.coef_, len(unique_tokens))
# Check that the importance value for 'it' and 'it' are the same.
self.assertEqual(explanation.feature_importance[0],
explanation.feature_importance[6])
# Check that the importance value for 'is' and 'is' are the same.
self.assertEqual(explanation.feature_importance[1],
explanation.feature_importance[7])
print(explanation.feature_importance)
def test_explain(self, sentence, counterfactuals, positive_token,
negative_token, num_classes, class_to_explain):
"""Tests explaining text classifiers with various output dimensions."""
def _predict_fn(sentences):
"""Mock prediction function."""
predictions = []
np.random.seed(0)
for sentence in sentences:
probs = np.random.uniform(0., 1., num_classes)
# To check if LEMON finds the right positive/negative correlations.
if negative_token in sentence:
probs[class_to_explain] = probs[class_to_explain] - 1.
if positive_token in sentence:
probs[class_to_explain] = probs[class_to_explain] + 1.
predictions.append(probs)
predictions = np.stack(predictions, axis=0)
if num_classes == 1:
return special.expit(predictions)
else:
return special.softmax(predictions, axis=-1)
explanation = lemon.explain(sentence,
counterfactuals,
_predict_fn,
class_to_explain,
tokenizer=str.split)
self.assertLen(explanation.feature_importance, len(sentence.split()))
# The positive word should have the highest attribution score.
positive_token_idx = sentence.split().index(positive_token)
self.assertEqual(positive_token_idx,
np.argmax(explanation.feature_importance))
# The negative word should have the lowest attribution score.
negative_token_idx = sentence.split().index(negative_token)
self.assertEqual(negative_token_idx,
np.argmin(explanation.feature_importance))