def setUp(self):
normalizer = in3120.BrainDeadNormalizer()
tokenizer = in3120.BrainDeadTokenizer()
corpus = in3120.InMemoryCorpus()
corpus.add_document(
in3120.InMemoryDocument(0, {
"title": "the foo",
"static_quality_score": 0.9
}))
corpus.add_document(
in3120.InMemoryDocument(1, {
"title": "the foo",
"static_quality_score": 0.2
}))
corpus.add_document(
in3120.InMemoryDocument(2, {
"title": "the foo foo",
"static_quality_score": 0.2
}))
corpus.add_document(in3120.InMemoryDocument(3, {"title": "the bar"}))
corpus.add_document(
in3120.InMemoryDocument(4, {"title": "the bar bar"}))
corpus.add_document(in3120.InMemoryDocument(5, {"title": "the baz"}))
corpus.add_document(in3120.InMemoryDocument(6, {"title": "the baz"}))
corpus.add_document(
in3120.InMemoryDocument(7, {"title": "the baz baz"}))
index = in3120.InMemoryInvertedIndex(corpus, ["title"], normalizer,
tokenizer)
self.__ranker = in3120.BetterRanker(corpus, index)
def test_mesh_corpus(self):
corpus = in3120.InMemoryCorpus("../data/mesh.txt")
index = in3120.InMemoryInvertedIndex(corpus, ["body"],
self.__normalizer,
self.__tokenizer)
self.assertEqual(len(list(index["hydrogen"])), 8)
self.assertEqual(len(list(index["hydrocephalus"])), 2)
def repl_a():
print("Building inverted index from Cranfield corpus...")
normalizer = in3120.BrainDeadNormalizer()
tokenizer = in3120.BrainDeadTokenizer()
corpus = in3120.InMemoryCorpus(data_path("cran.xml"))
index = in3120.InMemoryInvertedIndex(corpus, ["body"], normalizer, tokenizer)
print("Enter one or more index terms and inspect their posting lists.")
simple_repl("terms", lambda ts: {t: list(index.get_postings_iterator(t)) for t in index.get_terms(ts)})
def test_uses_yield(self):
import types
corpus = in3120.InMemoryCorpus()
corpus.add_document(in3120.InMemoryDocument(0, {"a": "foo bar"}))
index = in3120.InMemoryInvertedIndex(corpus, ["a"], self.__normalizer,
self.__tokenizer)
engine = in3120.SimpleSearchEngine(corpus, index)
ranker = in3120.BrainDeadRanker()
matches = engine.evaluate("foo", {}, ranker)
self.assertIsInstance(matches, types.GeneratorType,
"Are you using yield?")
def test_mesh_corpus(self):
normalizer = in3120.BrainDeadNormalizer()
tokenizer = in3120.BrainDeadTokenizer()
corpus = in3120.InMemoryCorpus("../data/mesh.txt")
index = in3120.InMemoryInvertedIndex(corpus, ["body"], normalizer,
tokenizer)
self.__process_query_with_two_terms(corpus, index, "HIV pROtein",
self.__merger.intersection,
[11316, 11319, 11320, 11321])
self.__process_query_with_two_terms(
corpus, index, "water Toxic", self.__merger.union,
[3078, 8138, 8635, 9379, 14472, 18572, 23234, 23985] +
[i for i in range(25265, 25282)])
def repl_d_2():
print("Indexing English news corpus...")
normalizer = in3120.BrainDeadNormalizer()
tokenizer = in3120.BrainDeadTokenizer()
corpus = in3120.InMemoryCorpus(data_path("en.txt"))
index = in3120.InMemoryInvertedIndex(corpus, ["body"], normalizer, tokenizer)
ranker = in3120.BetterRanker(corpus, index)
engine = in3120.SimpleSearchEngine(corpus, index)
options = {"debug": False, "hit_count": 5, "match_threshold": 0.5}
print("Enter a query and find matching documents.")
print(f"Lookup options are {options}.")
print(f"Tokenizer is {tokenizer.__class__.__name__}.")
print(f"Ranker is {ranker.__class__.__name__}.")
simple_repl("query", lambda q: list(engine.evaluate(q, options, ranker)))
def test_index_shingled_mesh_corpus(self):
tokenizer = in3120.ShingleGenerator(3)
corpus = in3120.InMemoryCorpus("../data/mesh.txt")
index = in3120.InMemoryInvertedIndex(corpus, ["body"],
self.__normalizer, tokenizer)
engine = in3120.SimpleSearchEngine(corpus, index)
self.__process_query_verify_matches("orGAnik kEMmistry", engine, {
"match_threshold": 0.1,
"hit_count": 10
}, (10, 8.0, [4408, 4410, 4411, 16980, 16981]))
self.__process_query_verify_matches("synndrome", engine, {
"match_threshold": 0.1,
"hit_count": 10
}, (10, 7.0, [1275]))
def test_synthetic_corpus(self):
from itertools import product, combinations_with_replacement
corpus = in3120.InMemoryCorpus()
words = ("".join(term) for term in product("bcd", "aei", "jkl"))
texts = (" ".join(word)
for word in combinations_with_replacement(words, 3))
for text in texts:
corpus.add_document(
in3120.InMemoryDocument(corpus.size(), {"a": text}))
index = in3120.InMemoryInvertedIndex(corpus, ["a"], self.__normalizer,
self.__tokenizer)
engine = in3120.SimpleSearchEngine(corpus, index)
epsilon = 0.0001
self.__process_query_verify_matches("baj BAJ baj", engine, {
"match_threshold": 1.0,
"hit_count": 27
}, (27, 9.0, [0]))
self.__process_query_verify_matches("baj caj", engine, {
"match_threshold": 1.0,
"hit_count": 100
}, (27, None, None))
self.__process_query_verify_matches("baj caj daj", engine, {
"match_threshold": 2 / 3 + epsilon,
"hit_count": 100
}, (79, None, None))
self.__process_query_verify_matches("baj caj", engine, {
"match_threshold": 2 / 3 + epsilon,
"hit_count": 100
}, (100, 3.0, [0, 9, 207, 2514]))
self.__process_query_verify_matches("baj cek dil", engine, {
"match_threshold": 1.0,
"hit_count": 10
}, (1, 3.0, [286]))
self.__process_query_verify_matches("baj cek dil", engine, {
"match_threshold": 1.0,
"hit_count": 10
}, (1, None, None))
self.__process_query_verify_matches("baj cek dil", engine, {
"match_threshold": 2 / 3 + epsilon,
"hit_count": 80
}, (79, 3.0, [13, 26, 273, 286, 377, 3107, 3198]))
self.__process_query_verify_matches("baj xxx yyy", engine, {
"match_threshold": 2 / 3 + epsilon,
"hit_count": 100
}, (0, None, None))
self.__process_query_verify_matches("baj xxx yyy", engine, {
"match_threshold": 2 / 3 - epsilon,
"hit_count": 100
}, (100, None, None))
def test_mesh_corpus(self):
corpus = in3120.InMemoryCorpus("../data/mesh.txt")
index = in3120.InMemoryInvertedIndex(corpus, ["body"],
self.__normalizer,
self.__tokenizer)
engine = in3120.SimpleSearchEngine(corpus, index)
query = "polluTION Water"
self.__process_two_term_query_verify_matches(query, engine, {
"match_threshold": 0.1,
"hit_count": 10
}, (10, [25274, 25275, 25276]))
self.__process_two_term_query_verify_matches(query, engine, {
"match_threshold": 1.0,
"hit_count": 10
}, (3, [25274, 25275, 25276]))
def test_multiple_fields(self):
document = in3120.InMemoryDocument(
0, {
'felt1': 'Dette er en test. Test, sa jeg. TEST!',
'felt2': 'test er det',
'felt3': 'test TEsT',
})
corpus = in3120.InMemoryCorpus()
corpus.add_document(document)
index = in3120.InMemoryInvertedIndex(corpus, ['felt1', 'felt3'],
self.__normalizer,
self.__tokenizer)
posting = next(index.get_postings_iterator('test'))
self.assertEqual(posting.document_id, 0)
self.assertEqual(posting.term_frequency, 5)
def test_access_postings(self):
corpus = in3120.InMemoryCorpus()
corpus.add_document(
in3120.InMemoryDocument(0, {"body": "this is a Test"}))
corpus.add_document(
in3120.InMemoryDocument(1, {"body": "test TEST prØve"}))
index = in3120.InMemoryInvertedIndex(corpus, ["body"],
self.__normalizer,
self.__tokenizer)
self.assertListEqual(list(index.get_terms("PRøvE wtf tesT")),
["prøve", "wtf", "test"])
self.assertListEqual([(p.document_id, p.term_frequency)
for p in index["prøve"]], [(1, 1)])
self.assertListEqual([(p.document_id, p.term_frequency)
for p in index.get_postings_iterator("wtf")], [])
self.assertListEqual([(p.document_id, p.term_frequency)
for p in index["test"]], [(0, 1), (1, 2)])
self.assertEqual(index.get_document_frequency("wtf"), 0)
self.assertEqual(index.get_document_frequency("prøve"), 1)
self.assertEqual(index.get_document_frequency("test"), 2)
def test_document_at_a_time_traversal_mesh_corpus(self):
from typing import Iterator, List, Tuple, Set
class AccessLoggedCorpus(in3120.Corpus):
def __init__(self, wrapped: in3120.Corpus):
self.__wrapped = wrapped
self.__accesses = set()
def __iter__(self):
return iter(self.__wrapped)
def size(self) -> int:
return self.__wrapped.size()
def get_document(self, document_id: int) -> in3120.Document:
self.__accesses.add(document_id)
return self.__wrapped.get_document(document_id)
def get_history(self) -> Set[int]:
return self.__accesses
class AccessLoggedIterator(Iterator[in3120.Posting]):
def __init__(self, term: str, accesses: List[Tuple[str, int]],
wrapped: Iterator[in3120.Posting]):
self.__term = term
self.__accesses = accesses
self.__wrapped = wrapped
def __next__(self):
posting = next(self.__wrapped)
self.__accesses.append((self.__term, posting.document_id))
return posting
class AccessLoggedInvertedIndex(in3120.InvertedIndex):
def __init__(self, wrapped: in3120.InvertedIndex):
self.__wrapped = wrapped
self.__accesses = []
def get_terms(self, buffer: str) -> Iterator[str]:
return self.__wrapped.get_terms(buffer)
def get_postings_iterator(self,
term: str) -> Iterator[in3120.Posting]:
return AccessLoggedIterator(
term, self.__accesses,
self.__wrapped.get_postings_iterator(term))
def get_document_frequency(self, term: str) -> int:
return self.__wrapped.get_document_frequency(term)
def get_history(self) -> List[Tuple[str, int]]:
return self.__accesses
corpus1 = in3120.InMemoryCorpus("../data/mesh.txt")
corpus2 = AccessLoggedCorpus(corpus1)
index = AccessLoggedInvertedIndex(
in3120.InMemoryInvertedIndex(corpus1, ["body"], self.__normalizer,
self.__tokenizer))
engine = in3120.SimpleSearchEngine(corpus2, index)
ranker = in3120.BrainDeadRanker()
query = "Water polluTION"
options = {"match_threshold": 0.5, "hit_count": 1, "debug": False}
matches = list(engine.evaluate(query, options, ranker))
self.assertIsNotNone(matches)
history = corpus2.get_history()
self.assertListEqual(
list(history),
[25274]) # Only the document in the result set should be accessed.
ordering1 = [
(
'water', 3078
), # Document-at-a-time ordering if evaluated as "water pollution".
('pollution', 788),
('pollution', 789),
('pollution', 790),
('pollution', 8079),
('water', 8635),
('pollution', 23837),
('water', 9379),
('water', 23234),
('water', 25265),
('pollution', 25274),
('water', 25266),
('water', 25267),
('water', 25268),
('water', 25269),
('water', 25270),
('water', 25271),
('water', 25272),
('water', 25273),
('water', 25274),
('water', 25275),
('pollution', 25275),
('water', 25276),
('pollution', 25276),
('water', 25277),
('water', 25278),
('water', 25279),
('water', 25280),
('water', 25281)
]
ordering2 = [
(
'pollution', 788
), # Document-at-a-time ordering if evaluated as "pollution water".
('water', 3078),
('pollution', 789),
('pollution', 790),
('pollution', 8079),
('water', 8635),
('pollution', 23837),
('water', 9379),
('water', 23234),
('water', 25265),
('pollution', 25274),
('water', 25266),
('water', 25267),
('water', 25268),
('water', 25269),
('water', 25270),
('water', 25271),
('water', 25272),
('water', 25273),
('water', 25274),
('pollution', 25275),
('water', 25275),
('pollution', 25276),
('water', 25276),
('water', 25277),
('water', 25278),
('water', 25279),
('water', 25280),
('water', 25281)
]
history = index.get_history()
self.assertTrue(
history == ordering1 or history
== ordering2) # Strict. Advanced implementations might fail.
