def test_task_done_with_restart(self):
"""Test that items are not deleted before task_done."""
q = SQLiteQueue(path=self.path, auto_commit=False)
for i in range(1, 11):
q.put(i)
self.assertEqual(1, q.get())
self.assertEqual(2, q.get())
# size is correct before task_done
self.assertEqual(8, q.qsize())
q.task_done()
# make sure the size still correct
self.assertEqual(8, q.qsize())
self.assertEqual(3, q.get())
# without task done
del q
q = SQLiteQueue(path=self.path, auto_commit=False)
# After restart, the qsize and head item are the same
self.assertEqual(8, q.qsize())
# After restart, the queue still works
self.assertEqual(3, q.get())
self.assertEqual(7, q.qsize())
def test_multiple_consumers(self):
"""Test sqlqueue can be used by multiple consumers."""
queue = SQLiteQueue(path=self.path,
multithreading=True,
auto_commit=self.auto_commit)
def producer():
for x in range(1000):
queue.put('var%d' % x)
task_done_if_required(queue)
def consumer():
for _ in range(100):
data = queue.get(block=True)
self.assertTrue('var' in data)
p = Thread(target=producer)
p.start()
consumers = []
for _ in range(10):
t = Thread(target=consumer)
t.start()
consumers.append(t)
for t in consumers:
t.join()
self.assertEqual(0, queue.qsize())
def test_open_close_1000(self):
"""Write 1000 items, close, reopen checking if all items are there"""
q = SQLiteQueue(self.path)
for i in range(1000):
q.put('var%d' % i)
self.assertEqual(1000, q.qsize())
del q
q = SQLiteQueue(self.path)
self.assertEqual(1000, q.qsize())
for i in range(1000):
data = q.get()
self.assertEqual('var%d' % i, data)
# assert adding another one still works
q.put('foobar')
data = q.get()
self.assertEqual('foobar', data)
def test_open_close_single(self):
"""Write 1 item, close, reopen checking if same item is there"""
q = SQLiteQueue(self.path, auto_commit=self.auto_commit)
q.put(b'var1')
del q
q = SQLiteQueue(self.path)
self.assertEqual(1, q.qsize())
self.assertEqual(b'var1', q.get())
def benchmark_sqlite_write(self):
"""Writing <BENCHMARK_COUNT> items."""
self.path = tempfile.mkdtemp('b_sql_10000')
q = SQLiteQueue(self.path, auto_commit=False)
for i in range(BENCHMARK_COUNT):
q.put('bench%d' % i)
assert q.qsize() == BENCHMARK_COUNT
def benchmark_sqlite_read_write_false(self):
"""Writing and reading <BENCHMARK_COUNT> items(1 task_done)."""
self.path = tempfile.mkdtemp('b_sql_10000')
q = SQLiteQueue(self.path, auto_commit=False)
for i in range(BENCHMARK_COUNT):
q.put('bench%d' % i)
for i in range(BENCHMARK_COUNT):
q.get()
q.task_done()
assert q.qsize() == 0
def test_multiple_consumers(self):
"""Test sqlqueue can be used by multiple consumers."""
queue = SQLiteQueue(path=self.path,
multithreading=True,
auto_commit=self.auto_commit)
def producer():
for x in range(1000):
queue.put('var%d' % x)
counter = []
# Set all to 0
for _ in range(1000):
counter.append(0)
def consumer(index):
for i in range(200):
data = queue.get(block=True)
self.assertTrue('var' in data)
counter[index * 200 + i] = data
p = Thread(target=producer)
p.start()
consumers = []
for index in range(5):
t = Thread(target=consumer, args=(index, ))
t.start()
consumers.append(t)
p.join()
for t in consumers:
t.join()
self.assertEqual(0, queue.qsize())
for x in range(1000):
self.assertNotEqual(0, counter[x],
"not 0 for counter's index %s" % x)
self.assertEqual(len(set(counter)), len(counter))
