def test_default(spacy_doc): result = ke.yake(spacy_doc) assert isinstance(result, list) and len(result) > 0 assert all(isinstance(ts, tuple) and len(ts) == 2 for ts in result) assert all( isinstance(ts[0], compat.unicode_) and isinstance(ts[1], float) for ts in result )
def test_empty_doc(empty_spacy_doc): result = ke.yake(empty_spacy_doc) assert isinstance(result, list) assert len(result) == 0
#open data from .txt file with open('news_article.txt', 'r') as file: data = file.read().replace('\n', '') article = data.replace(u'\xa0', u' ') #create doc object doc = textacy.make_spacy_doc(article, lang='en_core_web_sm') #KEYTERM EXTRACTION #Each algorithm returns a list of tuples, containg the keyterm and a score textrank = ke.textrank(doc,normalize="lemma") yake = ke.yake(doc,normalize="lemma") scake = ke.scake(doc,normalize="lemma") sgrank = ke.sgrank(doc,normalize="lemma") #separate terms and relevany scores terms_textrank, scores_textrank = decompose_keyterms(textrank) terms_yake, scores_yake = decompose_keyterms(yake) terms_scake, scores_scake = decompose_keyterms(scake) terms_sgrank, scores_sgrank = decompose_keyterms(sgrank) #save results to dataframe df = keyterm_dataframe(scake,'scake') print(df)
def test_window_size(spacy_doc): result_2 = ke.yake(spacy_doc, window_size=2) result_4 = ke.yake(spacy_doc, window_size=4) assert len(result_2) > 0 and len(result_4) > 0 assert result_2 != result_4
def test_topn_float(spacy_doc): result = ke.yake(spacy_doc, topn=0.2) assert len(result) > 0 with pytest.raises(ValueError): _ = ke.yake(spacy_doc, topn=2.0)
def test_n_topn(spacy_doc): for n in (5, 25): result = ke.yake(spacy_doc, topn=n) assert 0 < len(result) <= n
def test_ngrams_2_3(spacy_doc): result = ke.yake(spacy_doc, ngrams=(2, 3)) assert len(result) > 0 assert all(2 <= len(term.split()) <= 3 for term, _ in result)
def test_ngrams_1(spacy_doc): result = ke.yake(spacy_doc, ngrams=1) assert len(result) > 0 assert all(len(term.split()) == 1 for term, _ in result)
def test_normalize_lemma(spacy_doc): result = ke.yake(spacy_doc, normalize="lemma") assert len(result) > 0 assert any(term != term.lower() for term, _ in result)
def test_normalize_lower(spacy_doc): result = ke.yake(spacy_doc, normalize="lower") assert len(result) > 0 assert all(term == term.lower() for term, _ in result)