def test_get(self):
producers = (
# id, priority, iterable
(1, 1, [1] * 10),
(2, 2, [2] * 10),
(3, 3, [3] * 10),
)
expected = [
3, # .3 0 0.0
2, # .3 .5 0.0
1, # .3 .5 1.0
3, # .6 .5 1.0
2, # .6 1.0 1.0
3, # 1.0 1.0 1.0
3, # 1.3 1.0 1.0
2, # 1.3 1.5 1.0
1, # 1.3 1.5 2.0
3, # 1.6 1.5 2.0
2, # 1.6 2.0 2.0
3, # 2.0 2.0 2.0
]
pq = priorityqueue.PriorityQueue()
for pid, prio, itr in producers:
pq.add_producer(pid, prio, itr)
dd(pq)
rst = []
for _ in expected:
val = pq.get()
rst.append(val)
self.assertEqual(expected, rst)
def test_bench(self):
pq = priorityqueue.PriorityQueue()
ntimes = 10240
nq = 1024
n_thread = 3
ths = []
def worker():
for _ in range(ntimes):
pq.get()
for i in range(1, nq + 1):
pq.add_producer(i, i, yield_forever())
with ututil.Timer() as t:
for i in range(n_thread):
th = threadutil.start_daemon_thread(worker)
ths.append(th)
for th in ths:
th.join()
us_per_call = t.spent() / ntimes / n_thread * 1000 * 1000
dd(us_per_call, 'us/call')
self.assertLess(us_per_call, 50)
def test_concurrent(self):
pq = priorityqueue.PriorityQueue()
ntimes = 10240
nq = 3
n_thread = 3
ths = []
def worker():
for _ in range(ntimes):
pq.get()
for i in range(1, nq + 1):
pq.add_producer(i, i, yield_forever())
for i in range(n_thread):
th = threadutil.start_daemon_thread(worker)
ths.append(th)
for th in ths:
th.join()
consumed = []
got = 0
for i in range(1, nq + 1):
q = pq.producer_by_id[i]
consumed.append(q.consumed)
dd('get:', q.stat)
got += q.stat['get']
self.assertEqual(ntimes * n_thread, got)
dd('consumed:', consumed)
self.assertAlmostEqual(consumed[0], consumed[1])
self.assertAlmostEqual(consumed[1], consumed[2])
def test_remove_queue(self):
pq = priorityqueue.PriorityQueue()
pq.add_producer(1, 1, [1, 1])
pq.remove_producer(1)
self.assertRaises(priorityqueue.Empty, pq.get)
pq.add_producer(1, 1, [1, 1])
pq.add_producer(10, 10, [10, 10])
self.assertEqual(10, pq.get())
pq.remove_producer(10)
self.assertEqual(1, pq.get())
def test_add_queue(self):
pq = priorityqueue.PriorityQueue()
pq.add_producer(1, 1, [1, 1])
self.assertEqual(1, pq.get())
# re-add should update iterable
pq.add_producer(1, 1, [2, 2])
self.assertEqual(2, pq.get())
# new queue with higher priority comes first
pq.add_producer(2, 10, [10, 10])
self.assertEqual(10, pq.get())
def test_add_queue_update_priority(self):
pq = priorityqueue.PriorityQueue()
pq.add_producer(1, 1, [1, 1, 1])
self.assertEqual(1, pq.get())
pq.add_producer(2, 2, [2, 2, 2])
self.assertEqual(2, pq.get())
self.assertEqual(2, pq.get())
# re-add should update priority, iterable and adjust heap
pq.add_producer(1, 10, [10, 10, 10])
dd(pq)
dd(pq.consumable_heap)
self.assertEqual(10, pq.get())
def test_remove_queue_inexistent(self):
pq = priorityqueue.PriorityQueue()
self.assertRaises(KeyError, pq.remove_producer, 1)
pq.remove_producer(1, ignore_not_found=True)
from pykit import priorityqueue
if __name__ == '__main__':
pq = priorityqueue.PriorityQueue()
pq.add_producer(2018, 5, ['11', '12', '13', '14', '15', '16', '17'])
pq.add_producer(2017, 3, ['21', '22', '23', '24', '25', '26', '27'])
for _ in range(8):
print pq.get()
print
while True:
try:
print pq.get()
except priorityqueue.Empty as e:
break