def test_stopwords(): from b4msa.textmodel import TextModel tm = TextModel(lang='es', del_dup=False) text = tm.text_transformations('como esta mi carro') print(text) text1 = tm.lang.transform(text, stopwords='delete') print(text1) assert text1 == '~carro~' text1 = tm.lang.transform(text, stopwords='group') print(text1) assert text1 == '~_sw~_sw~_sw~carro~'
def test_stopwords(): from b4msa.textmodel import TextModel tm = TextModel(lang='es', del_dup=False) text = tm.text_transformations('como esta mi carro') print(text) text1 = tm.lang.transform(text, stopwords='delete') print(text1) assert text1 == '~carro~' text1 = tm.lang.transform(text, stopwords='group') print(text1) assert text1 == '~_sw~_sw~_sw~carro~'
text = 'I like playing football' output = [] for word in text.split(): w = stemmer.stem(word) output.append(w) output = " ".join(output) output text = 'I like playing football on Saturday' words = text.split() n = 3 n_grams = [] for a in zip(*[words[i:] for i in range(n)]): n_grams.append("~".join(a)) n_grams text = 'I like playing' q = 4 q_grams = [] for a in zip(*[text[i:] for i in range(q)]): q_grams.append("".join(a)) q_grams text = 'I like playing football with @mgraffg' tm = TextModel(token_list=[-1, 5], lang='english', usr_option=OPTION_GROUP, stemming=True) tm.text_transformations(text) tm.tokenize(text)