class TeeTestCase(unittest.TestCase):
def setUp(self):
self.d1 = Document({"id": "1",
"body": "A test of a unique filter pipeline."})
def test_document(self):
words = []
for word in self.d1.words():
words.append(word)
self.assertEqual(len(words), 8)
self.assertEqual(words[1], "test")
def document_test(self, words):
self.assertEqual(len(words), 8)
self.assertEqual(words[0], "a")
self.assertEqual(words[1], "test")
self.assertEqual(words[2], "of")
self.assertEqual(words[3], "a")
def test_tee(self):
identity = Identity()
t = Tee(identity)
words = list(t.process(self.d1.words()))
self.document_test(words)
words2 = list(t.alternate())
self.document_test(words2)
class HtmlCleanerTestCase(unittest.TestCase):
def setUp(self):
self.d1 = Document({"id": "1",
"body": "A test of a html cleaner pipeline.<br/> Another sentence."})
self.docs = [self.d1]
def test_document(self):
words = []
for word in self.d1.words():
words.append(word)
self.assertEqual(len(words), 13)
self.assertEqual(words[1], "test")
def test_html_cleaner_docs(self):
r = HtmlCleaner()
docs = list(r.process(self.docs))
words = list(docs[0].words())
self.assertEqual(len(words), 11)
self.assertEqual(words[6], "pipeline")
self.assertEqual(words[7], ".")
self.assertEqual(words[8], "another")
def test_html_cleaner_strings(self):
r = HtmlCleaner()
docs = list(r.process([self.d1["body"]]))
words = docs[0]
self.assertEqual(len(words), 53)
self.assertEqual(words[33], ".")
self.assertEqual(words[34], "\n")
self.assertEqual(words[35], " ")
class PipelineTestCase(unittest.TestCase):
"""
Test a simple pipeline with several pipeline modules
"""
def setUp(self):
self.d1 = Document({"id": "1", "body": "A stopword test."})
self.d2 = Document({"id": "2", "body": "A stopword test and a unique test."})
self.corpus = Corpus([self.d1, self.d2])
def test_pipeline(self):
recorder = Recorder([])
pipeline = GeneratorPipeline([Uniq(), recorder, StopwordFilter()])
words = pipeline.process(self.d2.words())
word_list = list(words)
self.assertEqual(len(word_list), 4)
self.assertEqual(word_list[0], "stopword")
self.assertEqual(word_list[1], "test")
self.assertEqual(word_list[2], "unique")
self.assertEqual(word_list[3], ".")
word_list = recorder.get_data()
#print word_list
self.assertEqual(len(word_list), 6)
self.assertEqual(word_list[0], "a")
self.assertEqual(word_list[1], "stopword")
self.assertEqual(word_list[2], "test")
self.assertEqual(word_list[3], "and")
self.assertEqual(word_list[4], "unique")
self.assertEqual(word_list[5], ".")
class StopwordFilterTestCase(unittest.TestCase):
def setUp(self):
self.d1 = Document({"id": "1", "body": "A stopword test."},
[StopwordFilter()])
def test_document(self):
words = []
for word in self.d1.words():
words.append(word)
self.assertEqual(len(words), 3)
self.assertEqual(words[0], "stopword")
class RedisUniqTestCase(unittest.TestCase):
def setUp(self):
self.d1 = Document({"id": "1",
"body": "A test of a unique filter pipeline."})
def test_document(self):
words = []
for word in self.d1.words():
words.append(word)
self.assertEqual(len(words), 8)
self.assertEqual(words[1], "test")
def test_redis_uniq(self):
r = RedisUniq()
words = list(r.process(self.d1.words()))
self.assertEqual(len(words), 7)
self.assertEqual(words[0], "a")
self.assertEqual(words[1], "test")
self.assertEqual(words[2], "of")
self.assertEqual(words[3], "unique")
class StdoutWriterTestCase(unittest.TestCase):
def setUp(self):
self.d1 = Document({"id": "1",
"body": "A test of a unique filter pipeline."})
def test_document(self):
words = []
for word in self.d1.words():
words.append(word)
self.assertEqual(len(words), 8)
self.assertEqual(words[1], "test")
class DocumentTestCase(unittest.TestCase):
def setUp(self):
self.d1 = Document({"id": "1", "body": "This this."})
self.d2 = Document({"id": "2", "body": "This is another test document."})
self.d3 = Document({"id": "3", "body": "Two words."})
self.d4 = Document({"id": "3", "body": "Derp a derp.\nA derp derp."})
def test_init(self):
d1 = Document({"id": "1", "body": "This this."})
self.assertEqual(d1.doc_id, "1")
self.assertEqual(str(d1), "This this.")
d2 = Document("This this.")
self.assertEqual(str(d1), "This this.")
def test___str__(self):
self.assertEqual(str(self.d1), "This this.")
self.assertEqual(str(self.d2), "This is another test document.")
self.assertEqual(str(self.d3), "Two words.")
def test_update_text(self):
self.assertEqual(str(self.d1), "This this.")
self.d1.update_text("more words")
self.assertEqual(str(self.d1), "more words")
def test_words(self):
words = list(self.d4.words())
self.assertEqual(len(words), 8)
self.assertEqual(words[0], "derp")
self.assertEqual(words[1], "a")
self.assertEqual(words[2], "derp")
self.assertEqual(words[3], ".")
self.assertEqual(words[4], "a")
self.assertEqual(words[5], "derp")
self.assertEqual(words[6], "derp")
self.assertEqual(words[7], ".")
def test_to_ngrams(self):
ngrams = self.d4.to_ngrams(2)
self.assertEqual(ngrams, [("derp", "a"), ("a", "derp"), ("derp", "."),
(".", "a"), ("a", "derp"), ("derp", "derp"),
("derp", ".")])