def test_task_done_with_restart(self): """Test that items are not deleted before task_done.""" q = SQLiteQueue(path=self.path, auto_commit=False) for i in range(1, 11): q.put(i) self.assertEqual(1, q.get()) self.assertEqual(2, q.get()) # size is correct before task_done self.assertEqual(8, q.qsize()) q.task_done() # make sure the size still correct self.assertEqual(8, q.qsize()) self.assertEqual(3, q.get()) # without task done del q q = SQLiteQueue(path=self.path, auto_commit=False) # After restart, the qsize and head item are the same self.assertEqual(8, q.qsize()) # After restart, the queue still works self.assertEqual(3, q.get()) self.assertEqual(7, q.qsize())
def test_multiple_consumers(self): """Test sqlqueue can be used by multiple consumers.""" queue = SQLiteQueue(path=self.path, multithreading=True, auto_commit=self.auto_commit) def producer(): for x in range(1000): queue.put('var%d' % x) task_done_if_required(queue) def consumer(): for _ in range(100): data = queue.get(block=True) self.assertTrue('var' in data) p = Thread(target=producer) p.start() consumers = [] for _ in range(10): t = Thread(target=consumer) t.start() consumers.append(t) for t in consumers: t.join() self.assertEqual(0, queue.qsize())
def test_open_close_1000(self): """Write 1000 items, close, reopen checking if all items are there""" q = SQLiteQueue(self.path) for i in range(1000): q.put('var%d' % i) self.assertEqual(1000, q.qsize()) del q q = SQLiteQueue(self.path) self.assertEqual(1000, q.qsize()) for i in range(1000): data = q.get() self.assertEqual('var%d' % i, data) # assert adding another one still works q.put('foobar') data = q.get() self.assertEqual('foobar', data)
def test_open_close_single(self): """Write 1 item, close, reopen checking if same item is there""" q = SQLiteQueue(self.path, auto_commit=self.auto_commit) q.put(b'var1') del q q = SQLiteQueue(self.path) self.assertEqual(1, q.qsize()) self.assertEqual(b'var1', q.get())
def benchmark_sqlite_write(self): """Writing <BENCHMARK_COUNT> items.""" self.path = tempfile.mkdtemp('b_sql_10000') q = SQLiteQueue(self.path, auto_commit=False) for i in range(BENCHMARK_COUNT): q.put('bench%d' % i) assert q.qsize() == BENCHMARK_COUNT
def benchmark_sqlite_read_write_false(self): """Writing and reading <BENCHMARK_COUNT> items(1 task_done).""" self.path = tempfile.mkdtemp('b_sql_10000') q = SQLiteQueue(self.path, auto_commit=False) for i in range(BENCHMARK_COUNT): q.put('bench%d' % i) for i in range(BENCHMARK_COUNT): q.get() q.task_done() assert q.qsize() == 0
def test_multiple_consumers(self): """Test sqlqueue can be used by multiple consumers.""" queue = SQLiteQueue(path=self.path, multithreading=True, auto_commit=self.auto_commit) def producer(): for x in range(1000): queue.put('var%d' % x) counter = [] # Set all to 0 for _ in range(1000): counter.append(0) def consumer(index): for i in range(200): data = queue.get(block=True) self.assertTrue('var' in data) counter[index * 200 + i] = data p = Thread(target=producer) p.start() consumers = [] for index in range(5): t = Thread(target=consumer, args=(index, )) t.start() consumers.append(t) p.join() for t in consumers: t.join() self.assertEqual(0, queue.qsize()) for x in range(1000): self.assertNotEqual(0, counter[x], "not 0 for counter's index %s" % x) self.assertEqual(len(set(counter)), len(counter))