def test_queue_task_done(self):
# Test to make sure a queue task completed successfully.
q = toro.JoinableQueue()
try:
q.task_done()
except ValueError:
pass
else:
self.fail("Did not detect task count going negative")
def test_queue_join_timeout(self):
q = toro.JoinableQueue()
q.put(1)
st = time.time()
with assert_raises(toro.Timeout):
yield q.join(deadline=timedelta(seconds=0.1))
duration = time.time() - st
self.assertAlmostEqual(0.1, duration, places=1)
self.assertEqual(1, q.unfinished_tasks)
def test_queue_join(self):
q = toro.JoinableQueue()
yield q.put('foo')
yield q.put('bar')
self.assertEqual(2, q.unfinished_tasks)
future = q.join()
q.task_done()
self.assertEqual(1, q.unfinished_tasks)
q.task_done()
self.assertEqual(0, q.unfinished_tasks)
yield future
def test_queue_join_callback(self):
# Test that callbacks passed to join() run immediately after task_done()
q = toro.JoinableQueue()
history = []
q.put('foo')
q.put('foo')
q.join().add_done_callback(make_callback('join', history))
q.task_done()
history.append('task_done1')
q.task_done()
history.append('task_done2')
self.assertEqual(['task_done1', 'join', 'task_done2'], history)
def test_queue_join(self):
# Test that a queue join()s successfully, and before anything else
# (done twice for insurance).
q = toro.JoinableQueue()
yield self.queue_join_test(q)
yield self.queue_join_test(q)
try:
q.task_done()
except ValueError:
pass
else:
self.fail("Did not detect task count going negative")
def test_io_loop(self):
global_loop = self.io_loop
custom_loop = IOLoop()
self.assertNotEqual(global_loop, custom_loop)
q = toro.JoinableQueue(io_loop=custom_loop)
def callback(future):
assert future.result() == 'foo'
custom_loop.stop()
q.get().add_done_callback(callback)
q.put('foo')
custom_loop.start()
def test_queue_join_clear(self):
# Verify that join() blocks again after a task is added
q = toro.JoinableQueue()
q.put_nowait('foo')
q.task_done()
# The _finished Event is set
yield q.join()
yield q.join()
# Unset the event
q.put_nowait('bar')
with assert_raises(toro.Timeout):
yield q.join(deadline=timedelta(seconds=0.1))
q.task_done()
yield q.join() # The Event is set again.
def spider(base_url, concurrency):
q = toro.JoinableQueue()
sem = toro.BoundedSemaphore(concurrency)
start = time.time()
fetching, fetched = set(), set()
@gen.coroutine
def fetch_url():
current_url = yield q.get()
try:
if current_url in fetching:
return
print 'fetching', current_url
fetching.add(current_url)
urls = yield get_links_from_url(current_url)
fetched.add(current_url)
for new_url in urls:
# Only follow links beneath the base URL
if new_url.startswith(base_url):
yield q.put(new_url)
finally:
q.task_done()
sem.release()
@gen.coroutine
def worker():
while True:
yield sem.acquire()
# Launch a subtask
fetch_url()
q.put(base_url)
# Start worker, then wait for the work queue to be empty.
worker()
yield q.join(deadline=timedelta(seconds=300))
assert fetching == fetched
print 'Done in %d seconds, fetched %s URLs.' % (
time.time() - start, len(fetched))
"""A classic producer-consumer example for using :class:`~toro.JoinableQueue`.
"""
# start-file
from tornado import ioloop, gen
import toro
q = toro.JoinableQueue(maxsize=3)
@gen.coroutine
def producer():
for item in range(10):
print 'Sending', item
yield q.put(item)
@gen.coroutine
def consumer():
while True:
item = yield q.get()
print '\t\t', 'Got', item
q.task_done()
if __name__ == '__main__':
producer()
consumer()
loop = ioloop.IOLoop.current()
def stop(future):
loop.stop()
def test_str(self):
q = toro.JoinableQueue()
self.assertTrue('JoinableQueue' in str(q))
self.assertFalse('tasks' in str(q))
q.put('foo')
self.assertTrue('tasks' in str(q))
def test_issue_45(self):
# Test that join() exits immediately if not jobs were put into the queue
# From Gevent's test_issue_45()
self.switch_expected = False
q = toro.JoinableQueue()
yield q.join()
def q(self):
'''
任务量控制队列
'''
return toro.JoinableQueue(maxsize=self.queue_size)
