def test_vectorizer_extractor():
pytest.importorskip('sklearn')
stim = TextStim(join(TEXT_DIR, 'scandal.txt'))
result = TextVectorizerExtractor().transform(stim).to_df()
assert 'woman' in result.columns
assert result['woman'][0] == 3
from sklearn.feature_extraction.text import TfidfVectorizer
custom_vectorizer = TfidfVectorizer()
ext = TextVectorizerExtractor(vectorizer=custom_vectorizer)
stim2 = TextStim(join(TEXT_DIR, 'simple_text.txt'))
result = merge_results(ext.transform([stim, stim2]), format='wide',
extractor_names='multi')
assert ('TextVectorizerExtractor', 'woman') in result.columns
assert np.allclose(0.129568189476,
result[('TextVectorizerExtractor', 'woman')][0])
ext = TextVectorizerExtractor(vectorizer='CountVectorizer',
analyzer='char_wb',
ngram_range=(2, 2))
result = ext.transform(stim).to_df()
assert 'wo' in result.columns
assert result['wo'][0] == 6
def test_vectorizer_extractor():
pytest.importorskip('sklearn')
stim = TextStim(join(TEXT_DIR, 'scandal.txt'))
result = TextVectorizerExtractor().transform(stim).to_df()
assert 'woman' in result.columns
assert result['woman'][0] == 3
from sklearn.feature_extraction.text import TfidfVectorizer
custom_vectorizer = TfidfVectorizer()
ext = TextVectorizerExtractor(vectorizer=custom_vectorizer)
stim2 = TextStim(join(TEXT_DIR, 'simple_text.txt'))
result = merge_results(ext.transform([stim, stim2]))
assert ('TextVectorizerExtractor', 'woman') in result.columns
assert 0.129568189476 in result[('TextVectorizerExtractor', 'woman')]
ext = TextVectorizerExtractor(vectorizer='CountVectorizer',
analyzer='char_wb',
ngram_range=(2, 2))
result = ext.transform(stim).to_df()
assert 'wo' in result.columns
assert result['wo'][0] == 6