def test_neighbors():
# test Neighbors class
neighbor = Neighbors(nlp, w_prob=-20)
n = neighbor.neighbors('book')
# most similar word must be the word itself with a similarity score of 1
assert n[0][0].orth_ == 'book'
assert n[0][1] == 1.
# similarity score list needs to be descending
similarity_score = [x[1] for x in n]
assert sorted(similarity_score, reverse=True) == similarity_score
def test_neighbors():
# test inputs
w_prob = -15.
tag = 'NN'
top_n = 10
neighbor = Neighbors(nlp, w_prob=w_prob)
n = neighbor.neighbors('book', tag, top_n)
# The word itself is excluded from the array with similar words
assert 'book' not in n['words']
assert len(n['words']) == top_n
assert len(n['words']) == len(n['similarities'])
# similarity score list needs to be descending
similarity_score = n['similarities']
assert (np.sort(similarity_score[::-1]) - similarity_score).sum() == 0.0
def _load_AnchorText(path: Union[str, os.PathLike], predictor: Callable,
meta: dict) -> 'AnchorText':
# load the spacy model
import spacy
nlp = spacy.load(Path(path, 'nlp'))
with open(Path(path, 'explainer.dill'), 'rb') as f:
explainer = dill.load(f)
explainer.nlp = nlp
# explainer._synonyms_generator contains spacy Lexemes which contain unserializable Cython constructs
# so we re-initialize the object here
# TODO: this is slow to re-initialize, try optimzing
from alibi.explainers.anchor_text import Neighbors
explainer._synonyms_generator = Neighbors(nlp_obj=nlp)
explainer.reset_predictor(predictor)
return explainer