def test_neighbors(): # test Neighbors class neighbor = Neighbors(nlp, w_prob=-20) n = neighbor.neighbors('book') # most similar word must be the word itself with a similarity score of 1 assert n[0][0].orth_ == 'book' assert n[0][1] == 1. # similarity score list needs to be descending similarity_score = [x[1] for x in n] assert sorted(similarity_score, reverse=True) == similarity_score
def test_neighbors(): # test inputs w_prob = -15. tag = 'NN' top_n = 10 neighbor = Neighbors(nlp, w_prob=w_prob) n = neighbor.neighbors('book', tag, top_n) # The word itself is excluded from the array with similar words assert 'book' not in n['words'] assert len(n['words']) == top_n assert len(n['words']) == len(n['similarities']) # similarity score list needs to be descending similarity_score = n['similarities'] assert (np.sort(similarity_score[::-1]) - similarity_score).sum() == 0.0
def _load_AnchorText(path: Union[str, os.PathLike], predictor: Callable, meta: dict) -> 'AnchorText': # load the spacy model import spacy nlp = spacy.load(Path(path, 'nlp')) with open(Path(path, 'explainer.dill'), 'rb') as f: explainer = dill.load(f) explainer.nlp = nlp # explainer._synonyms_generator contains spacy Lexemes which contain unserializable Cython constructs # so we re-initialize the object here # TODO: this is slow to re-initialize, try optimzing from alibi.explainers.anchor_text import Neighbors explainer._synonyms_generator = Neighbors(nlp_obj=nlp) explainer.reset_predictor(predictor) return explainer