def test_hdp_dense_input(): """Dense and sparse input should be the same""" X = make_uniform_doc_word_matrix(n_topics=10, words_per_topic=3, docs_per_topic=3) dense_X = X.todense() array_X = X.toarray() params = { 'n_topic_truncate': 20, 'n_doc_truncate': 5, 'learning_method': 'batch', 'max_iter': 10, 'random_state': 1, } hdp1 = HierarchicalDirichletProcess(**params) transformed_1 = hdp1.fit_transform(dense_X) hdp2 = HierarchicalDirichletProcess(**params) transformed_2 = hdp2.fit_transform(array_X) hdp3 = HierarchicalDirichletProcess(**params) transformed_3 = hdp3.fit_transform(X) assert_almost_equal(transformed_1, transformed_2) assert_almost_equal(transformed_2, transformed_3)
def test_likelihood_check(): """Test enable doc_likelihood check The result should be the same no matter it is True or False. """ X = make_uniform_doc_word_matrix(n_topics=10, words_per_topic=3, docs_per_topic=3) params = { 'n_topic_truncate': 20, 'n_doc_truncate': 5, 'learning_method': 'batch', 'max_iter': 10, 'random_state': 1, 'check_doc_likelihood': True, 'evaluate_every': 1, } hdp1 = HierarchicalDirichletProcess(**params) ret1 = hdp1.fit_transform(X) params['check_doc_likelihood'] = False hdp2 = HierarchicalDirichletProcess(**params) ret2 = hdp2.fit_transform(X) assert_almost_equal(ret1, ret2)
def test_uniform_diag_matrix(self): """Test diag matrix with make uniform matrix""" n_topics = self.rand.randint(100, 200) params = { 'n_topics': n_topics, 'words_per_topic': 1, 'docs_per_topic': 1, } matrix = make_uniform_doc_word_matrix(**params) dense = matrix.toarray() assert_array_equal(dense, np.eye(n_topics))
def test_shuffle_uniform_diag_matrix(self): """Test suffle diag matrix with make uniform matrix""" n_topics = self.rand.randint(100, 200) params = { 'n_topics': n_topics, 'words_per_topic': 1, 'docs_per_topic': 1, 'shuffle': True } matrix = make_uniform_doc_word_matrix(**params) dense = matrix.toarray() diag_shift = False for idx in range(n_topics): if dense[idx, idx] < 1.: diag_shift = True break assert_true(diag_shift)
def test_make_uniform_matrix(self): """Test words per document """ n_topics = self.rand.randint(100, 200) words_per_topic = self.rand.randint(10, 20) docs_per_topic = self.rand.randint(100, 2000) params = { 'n_topics': n_topics, 'words_per_topic': words_per_topic, 'docs_per_topic': docs_per_topic, } matrix = make_uniform_doc_word_matrix(**params) dense = matrix.toarray() assert_equal(dense.shape[0], n_topics * docs_per_topic) assert_equal(dense.shape[1], n_topics * words_per_topic) row_sum = np.sum(dense, axis=1) doc_topics = np.repeat(words_per_topic, n_topics * docs_per_topic) assert_array_equal(row_sum, doc_topics)
def test_hdp_transform(): """Test HDP transform""" X = make_uniform_doc_word_matrix(n_topics=10, words_per_topic=3, docs_per_topic=3) params = { 'n_topic_truncate': 20, 'n_doc_truncate': 5, 'learning_method': 'batch', 'max_iter': 10, } hdp = HierarchicalDirichletProcess(**params) assert_raises_regexp(NotFittedError, r"^no 'lambda_' attribute", hdp.transform, X) hdp.fit(X) transformed = hdp.transform(X) assert_equal(transformed.shape[0], X.shape[0]) assert_equal(transformed.shape[1], 20)
def test_hdp_fit_transform(): """Test HDP fit_transform""" X = make_uniform_doc_word_matrix(n_topics=10, words_per_topic=3, docs_per_topic=3) params = { 'n_topic_truncate': 20, 'n_doc_truncate': 5, 'learning_method': 'batch', 'max_iter': 10, 'random_state': 1, } hdp1 = HierarchicalDirichletProcess(**params) hdp1.fit(X) transformed_1 = hdp1.transform(X) hdp2 = HierarchicalDirichletProcess(**params) transformed_2 = hdp2.fit_transform(X) assert_almost_equal(transformed_1, transformed_2)
def test_hdp_invalid_parameters(): """Test HDP Invalid paramters """ X = make_uniform_doc_word_matrix(n_topics=10, words_per_topic=3, docs_per_topic=3) hdp1 = HierarchicalDirichletProcess(n_topic_truncate=10, n_doc_truncate=3, max_iter=-1, random_state=0) assert_raises_regexp(ValueError, r"^Invalid ", hdp1.fit, X) hdp2 = HierarchicalDirichletProcess(n_topic_truncate=10, n_doc_truncate=3, learning_method='na', random_state=0) assert_raises_regexp(ValueError, r"^Invalid 'learning_method'", hdp2.fit, X)
def test_hdp_topic_distribution(): """Test HDP topic_distribution""" X = make_uniform_doc_word_matrix(n_topics=10, words_per_topic=3, docs_per_topic=3) params = { 'n_topic_truncate': 20, 'n_doc_truncate': 5, 'learning_method': 'batch', 'max_iter': 10, 'random_state': 1, } hdp = HierarchicalDirichletProcess(**params) assert_raises_regexp(NotFittedError, r"^no 'lambda_' attribute", hdp.topic_distribution) hdp.fit(X) topic_distr = hdp.topic_distribution() assert_almost_equal(np.sum(topic_distr), 1.0)
def test_partial_fit_after_fit(): """Test run partial_fit after fit partial_fit should reset global parameters """ X = make_uniform_doc_word_matrix(n_topics=10, words_per_topic=3, docs_per_topic=3) params = { 'n_topic_truncate': 20, 'n_doc_truncate': 5, 'learning_method': 'batch', 'max_iter': 10, 'random_state': 1, } hdp1 = HierarchicalDirichletProcess(**params) hdp1.fit(X) hdp1.partial_fit(X) hdp2 = HierarchicalDirichletProcess(**params) hdp2.partial_fit(X) assert_almost_equal(hdp1.transform(X), hdp2.transform(X))