Example #1
0
 def test_uses_yield(self):
     import types
     corpus = in3120.InMemoryCorpus()
     corpus.add_document(in3120.InMemoryDocument(0, {"a": "foo bar"}))
     index = in3120.InMemoryInvertedIndex(corpus, ["a"], self.__normalizer,
                                          self.__tokenizer)
     engine = in3120.SimpleSearchEngine(corpus, index)
     ranker = in3120.BrainDeadRanker()
     matches = engine.evaluate("foo", {}, ranker)
     self.assertIsInstance(matches, types.GeneratorType,
                           "Are you using yield?")
Example #2
0
 def __process_two_term_query_verify_matches(self, query, engine, options,
                                             expected):
     ranker = in3120.BrainDeadRanker()
     hits, winners = expected
     matches = list(engine.evaluate(query, options, ranker))
     matches = [(m["score"], m["document"].document_id) for m in matches]
     self.assertEqual(len(matches), hits)
     for (score, winner) in matches[:len(winners)]:
         self.assertEqual(score, 2.0)
         self.assertIn(winner, winners)
     for (score, contender) in matches[len(winners):]:
         self.assertEqual(score, 1.0)
Example #3
0
def repl_d_1():
    print("Indexing MeSH corpus...")
    normalizer = in3120.BrainDeadNormalizer()
    tokenizer = in3120.ShingleGenerator(3)
    corpus = in3120.InMemoryCorpus(data_path("mesh.txt"))
    index = in3120.InMemoryInvertedIndex(corpus, ["body"], normalizer, tokenizer)
    ranker = in3120.BrainDeadRanker()
    engine = in3120.SimpleSearchEngine(corpus, index)
    options = {"debug": False, "hit_count": 5, "match_threshold": 0.5}
    print("Enter a query and find matching documents.")
    print(f"Lookup options are {options}.")
    print(f"Tokenizer is {tokenizer.__class__.__name__}.")
    print(f"Ranker is {ranker.__class__.__name__}.")
    simple_repl("query", lambda q: list(engine.evaluate(q, options, ranker)))
Example #4
0
 def __process_query_verify_matches(self, query, engine, options, expected):
     from itertools import takewhile
     ranker = in3120.BrainDeadRanker()
     hits, score, winners = expected
     matches = list(engine.evaluate(query, options, ranker))
     matches = [(m["score"], m["document"].document_id) for m in matches]
     self.assertEqual(len(matches), hits)
     if matches:
         for i in range(1, hits):
             self.assertGreaterEqual(matches[i - 1][0], matches[i][0])
         if score:
             self.assertEqual(matches[0][0], score)
         if winners:
             top = takewhile(lambda m: m[0] == matches[0][0], matches)
             self.assertListEqual(winners,
                                  list(sorted([m[1] for m in top])))
Example #5
0
 def setUp(self):
     self.__ranker = in3120.BrainDeadRanker()
Example #6
0
    def test_document_at_a_time_traversal_mesh_corpus(self):
        from typing import Iterator, List, Tuple, Set

        class AccessLoggedCorpus(in3120.Corpus):
            def __init__(self, wrapped: in3120.Corpus):
                self.__wrapped = wrapped
                self.__accesses = set()

            def __iter__(self):
                return iter(self.__wrapped)

            def size(self) -> int:
                return self.__wrapped.size()

            def get_document(self, document_id: int) -> in3120.Document:
                self.__accesses.add(document_id)
                return self.__wrapped.get_document(document_id)

            def get_history(self) -> Set[int]:
                return self.__accesses

        class AccessLoggedIterator(Iterator[in3120.Posting]):
            def __init__(self, term: str, accesses: List[Tuple[str, int]],
                         wrapped: Iterator[in3120.Posting]):
                self.__term = term
                self.__accesses = accesses
                self.__wrapped = wrapped

            def __next__(self):
                posting = next(self.__wrapped)
                self.__accesses.append((self.__term, posting.document_id))
                return posting

        class AccessLoggedInvertedIndex(in3120.InvertedIndex):
            def __init__(self, wrapped: in3120.InvertedIndex):
                self.__wrapped = wrapped
                self.__accesses = []

            def get_terms(self, buffer: str) -> Iterator[str]:
                return self.__wrapped.get_terms(buffer)

            def get_postings_iterator(self,
                                      term: str) -> Iterator[in3120.Posting]:
                return AccessLoggedIterator(
                    term, self.__accesses,
                    self.__wrapped.get_postings_iterator(term))

            def get_document_frequency(self, term: str) -> int:
                return self.__wrapped.get_document_frequency(term)

            def get_history(self) -> List[Tuple[str, int]]:
                return self.__accesses

        corpus1 = in3120.InMemoryCorpus("../data/mesh.txt")
        corpus2 = AccessLoggedCorpus(corpus1)
        index = AccessLoggedInvertedIndex(
            in3120.InMemoryInvertedIndex(corpus1, ["body"], self.__normalizer,
                                         self.__tokenizer))
        engine = in3120.SimpleSearchEngine(corpus2, index)
        ranker = in3120.BrainDeadRanker()
        query = "Water  polluTION"
        options = {"match_threshold": 0.5, "hit_count": 1, "debug": False}
        matches = list(engine.evaluate(query, options, ranker))
        self.assertIsNotNone(matches)
        history = corpus2.get_history()
        self.assertListEqual(
            list(history),
            [25274])  # Only the document in the result set should be accessed.
        ordering1 = [
            (
                'water', 3078
            ),  # Document-at-a-time ordering if evaluated as "water pollution".
            ('pollution', 788),
            ('pollution', 789),
            ('pollution', 790),
            ('pollution', 8079),
            ('water', 8635),
            ('pollution', 23837),
            ('water', 9379),
            ('water', 23234),
            ('water', 25265),
            ('pollution', 25274),
            ('water', 25266),
            ('water', 25267),
            ('water', 25268),
            ('water', 25269),
            ('water', 25270),
            ('water', 25271),
            ('water', 25272),
            ('water', 25273),
            ('water', 25274),
            ('water', 25275),
            ('pollution', 25275),
            ('water', 25276),
            ('pollution', 25276),
            ('water', 25277),
            ('water', 25278),
            ('water', 25279),
            ('water', 25280),
            ('water', 25281)
        ]
        ordering2 = [
            (
                'pollution', 788
            ),  # Document-at-a-time ordering if evaluated as "pollution water".
            ('water', 3078),
            ('pollution', 789),
            ('pollution', 790),
            ('pollution', 8079),
            ('water', 8635),
            ('pollution', 23837),
            ('water', 9379),
            ('water', 23234),
            ('water', 25265),
            ('pollution', 25274),
            ('water', 25266),
            ('water', 25267),
            ('water', 25268),
            ('water', 25269),
            ('water', 25270),
            ('water', 25271),
            ('water', 25272),
            ('water', 25273),
            ('water', 25274),
            ('pollution', 25275),
            ('water', 25275),
            ('pollution', 25276),
            ('water', 25276),
            ('water', 25277),
            ('water', 25278),
            ('water', 25279),
            ('water', 25280),
            ('water', 25281)
        ]
        history = index.get_history()
        self.assertTrue(
            history == ordering1 or history
            == ordering2)  # Strict. Advanced implementations might fail.