def test_token_stream(self):
stream = MongoStreamer(self.db_setup, tokenizer=self.tokenizer)
stream.cursor = self.mock_cursor
token_benchmark = [['doomed', 'failure'], ['set', 'success']]
id_benchmark = ['a', '1']
token_result = []
for each in stream.token_stream(cache_list=['doc_id']):
token_result.append(each)
self.assertEqual(token_benchmark, token_result)
self.assertEqual(id_benchmark, stream.__dict__['doc_id_cache'])
def test_token_stream(self):
stream = MongoStreamer(self.db_setup, tokenizer=self.tokenizer)
stream.cursor = self.mock_cursor
token_benchmark = [["doomed", "failure"], ["set", "success"]]
id_benchmark = ["a", "1"]
token_result = []
for each in stream.token_stream(cache_list=["doc_id"]):
token_result.append(each)
self.assertEqual(token_benchmark, token_result)
self.assertEqual(id_benchmark, stream.__dict__["doc_id_cache"])
def test_token_stream(self):
stream = MongoStreamer(self.db_setup,
tokenizer=self.tokenizer)
stream.cursor = self.mock_cursor
token_benchmark = [['doomed', 'failure'],
['set', 'success']]
id_benchmark = ['a', '1']
token_result = []
for each in stream.token_stream(cache_list=['doc_id']):
token_result.append(each)
self.assertEqual(token_benchmark, token_result)
self.assertEqual(id_benchmark, stream.__dict__['doc_id_cache'])
