def test_sample_ranking_with_no_exceptions(self):
"""
Test if ranking is built without any exception.
"""
sample1 = Indexable(1, "this is an indexable metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable metadata")
self.rank.build_rank([sample1, sample2, sample3])
def __init__(self,
docid,
indexable_data,
original_data,
highlighted_data=None):
Indexable.__init__(self, docid, indexable_data)
self.original_data = original_data
self.highlighted_data = highlighted_data
def test_sample_indexing_and_validate_items(self):
sample1 = Indexable(1, "this is an indexable metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable metadata")
self.index.build_index([sample1, sample2, sample3])
expected_term_index = defaultdict(int)
expected_term_index['indexable'] = [0, 1, 2]
self.assertItemsEqual(self.index.term_index['indexable'], expected_term_index['indexable'])
def test_one_term_search(self):
"""
Test if the search for one term returns expected results.
"""
sample1 = Indexable(1, "this is an indexable metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable super metadata")
expected_indices = [1, 2]
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(["super"])
self.assertItemsEqual(search_results, expected_indices)
def test_mixed_valid_invalid_term_search(self):
"""
Test if the search returns when there are valid and invalid terms mixed.
"""
sample1 = Indexable(1, "this is an indexable simple metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable metadata")
expected_indices = []
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(["not_valid_term", "super"])
self.assertItemsEqual(search_results, expected_indices)
def test_three_terms_search_with_stop_words(self):
"""
Test if the search for stop words returns expected results.
"""
sample1 = Indexable(1, "this is an indexable simple metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable super metadata")
expected_indices = [0, 1, 2]
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(["this", "is", "metadata"])
self.assertItemsEqual(search_results, expected_indices)
def test_stop_word_search(self):
"""
Test if stop words are correctly ignored.
"""
sample1 = Indexable(1, "this is an indexable metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable super metadata")
expected_indices = []
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(["this"])
self.assertItemsEqual(search_results, expected_indices)
def test_doc_frequency_matrix_with_sample2(self):
"""
Test if document frequency matrix is correctly built.
"""
sample1 = Indexable(1, "the sky is blue")
sample2 = Indexable(2, "the sun is bright")
self.rank.build_rank([sample1, sample2])
expected_vocab_indices = {'blue': 0, 'sun': 2, 'bright': 3, 'sky': 1}
expected_tf = np.array([[1, 1, 0, 0], [0, 0, 1, 1]])
self.assertEqual(self.rank.vocabulary, expected_vocab_indices)
np.testing.assert_array_equal(self.rank.ft_matrix.todense(),
expected_tf)
def test_doc_inverse_term_frequency_vector2(self):
"""
Test if document inverse term frequency vector is correctly built.
"""
sample1 = Indexable(1, "the sky is blue")
sample2 = Indexable(2, "the sun is bright")
self.rank.build_rank([sample1, sample2])
expected_idf = [1.40546511, 1.40546511, 1.40546511, 1.40546511]
expected_tf_idf = [[0.70710678, 0.70710678, 0, 0],
[0, 0, 0.70710678, 0.70710678]]
np.testing.assert_almost_equal(self.rank.ifd_diag_matrix.diagonal(),
expected_idf, 4)
np.testing.assert_almost_equal(self.rank.tf_idf_matrix.todense(),
expected_tf_idf, 4)
def __init__(self,
docid,
indexable_data,
original_data,
url,
user_id,
time_of_visit,
relevance,
keywords,
highlighted_data=None):
Indexable.__init__(self, docid, indexable_data)
self.original_data = original_data
self.highlighted_data = highlighted_data
self.url = url
self.user_id = user_id
self.time_of_visit = time_of_visit
self.relevance = relevance
self.keywords = keywords
def test_doc_inverse_term_frequency_vector1(self):
"""
Test if document inverse term frequency vector is correctly built.
"""
sample1 = Indexable(1, "this is an indexable metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable metadata")
self.rank.build_rank([sample1, sample2, sample3])
expected_idf = [1., 1., 1.28768207, 1.69314718, 1.69314718]
expected_tf_idf = [[0.52284231, 0.52284231, 0.67325467, 0, 0],
[0.39148397, 0.39148397, 0.50410689, 0.66283998, 0],
[0.45329466, 0.45329466, 0, 0, 0.76749457]]
np.testing.assert_almost_equal(self.rank.ifd_diag_matrix.diagonal(),
expected_idf, 4)
np.testing.assert_almost_equal(self.rank.tf_idf_matrix.todense(),
expected_tf_idf, 4)
def test_score_computation(self):
"""
Test if document score is correctly calculated.
"""
sample1 = Indexable(1, "the sky is blue")
self.rank.build_rank([sample1])
np.testing.assert_almost_equal(self.rank.compute_rank(0, ["blue"]),
0.707106, 5)
np.testing.assert_almost_equal(self.rank.compute_rank(0, ["sky"]),
0.7071067, 5)
np.testing.assert_almost_equal(
self.rank.compute_rank(0, ["blue", "sky"]), 1.414213, 5)
def test_doc_frequency_matrix_with_sample1(self):
"""
Test if document frequency matrix is correctly built.
"""
sample1 = Indexable(1, "this is an indexable metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable metadata")
self.rank.build_rank([sample1, sample2, sample3])
expected_vocab_indices = {
'an': 2,
'super': 3,
'indexable': 1,
'metadata': 0,
'another': 4
}
expected_tf = np.array([[1, 1, 1, 0, 0], [1, 1, 1, 1, 0],
[1, 1, 0, 0, 1]])
self.assertEqual(self.rank.vocabulary, expected_vocab_indices)
np.testing.assert_array_equal(self.rank.ft_matrix.todense(),
expected_tf)
def test_sample_index_words_count(self):
sample = Indexable(1, "this is an indexable metadata, that is an indexable super metadata")
expected_words_count = defaultdict(int)
expected_words_count['this'] = 1
expected_words_count['that'] = 1
expected_words_count['super'] = 1
expected_words_count['is'] = 2
expected_words_count['metadata,'] = 1 # Exact terms are not yet processed.
expected_words_count['metadata'] = 1
expected_words_count['indexable'] = 2
expected_words_count['an'] = 2
self.assertItemsEqual(sample.words_count, expected_words_count)
def __init__(self, docid, indexable_data, original_data, highlighted_data = None):
Indexable.__init__(self, docid, indexable_data)
self.original_data = original_data
self.highlighted_data = highlighted_data