def test_multi_column_tfidf_vectorizer_one_column_zero_output_tokens(kwargs, output_shape): """Tests that a TF-IDF document-term matrix is still returned when only one column breaks""" corpus = np.array( [ ["Cats eat rats.", "Rats are mammals."], ["Dogs chase cats.", "Rats are mammals."], ["People like dogs.", "Rats are mammals."], ["People hate rats.", "Rats are mammals."], ] ) vec = MultiColumnTfidfVectorizer(**kwargs) output = vec.fit_transform(corpus) assert output.shape == output_shape
def test_multi_column_tfidf_vectorizer(): vec = MultiColumnTfidfVectorizer() output = vec.fit_transform(corpus) assert isinstance(output, sp.coo.coo_matrix) observed = output.todense() expected = np.hstack( [ TfidfVectorizer().fit_transform(corpus[:, 0]).todense(), TfidfVectorizer().fit_transform(corpus[:, 1]).todense(), ] ) np.testing.assert_array_equal(observed, expected)
def test_multi_column_tfidf_vectorizer_zero_output_tokens_ignore_zero_vocab_on(kwargs, data, shape): """Tests for empty matrix when no terms remain after pruning""" vec = MultiColumnTfidfVectorizer(**kwargs) output = vec.fit_transform(data) assert output.shape == shape
def test_multi_column_tfidf_vectorizer_vocabulary_sizes_small(): vocabulary_sizes = [TfidfVectorizer().fit_transform(corpus[:, i]).shape[1] - 1 for i in range(corpus.shape[1])] vectorizer = MultiColumnTfidfVectorizer(vocabulary_sizes=vocabulary_sizes) observed = vectorizer.fit_transform(corpus) assert observed.shape[1] == sum(vocabulary_sizes) assert sp.issparse(observed)
def test_multi_column_tfidf_vectorizer_zero_output_tokens_ignore_zero_vocab_off(kwargs, data): """Tests for ValueError when no terms remain after pruning and `ignore_overpruned_columns=False`""" with pytest.raises(ValueError): vec = MultiColumnTfidfVectorizer(**kwargs) vec.fit_transform(data)