def test_wait_really_done(self):
# there is possibility that some tasks in the pending list
# became done but their callbacks haven't all been called yet
@tasks.coroutine
def coro1():
yield
@tasks.coroutine
def coro2():
yield
yield
a = tasks.Task(coro1(), loop=self.loop)
b = tasks.Task(coro2(), loop=self.loop)
task = tasks.Task(tasks.wait([b, a],
return_when=tasks.FIRST_COMPLETED,
loop=self.loop),
loop=self.loop)
done, pending = self.loop.run_until_complete(task)
self.assertEqual({a, b}, done)
self.assertTrue(a.done())
self.assertIsNone(a.result())
self.assertTrue(b.done())
self.assertIsNone(b.result())
def test_as_completed(self):
@tasks.coroutine
def sleeper(dt, x):
yield from tasks.sleep(dt, loop=self.loop)
return x
a = sleeper(0.1, 'a')
b = sleeper(0.1, 'b')
c = sleeper(0.15, 'c')
@tasks.coroutine
def foo():
values = []
for f in tasks.as_completed([b, c, a], loop=self.loop):
values.append((yield from f))
return values
t0 = time.monotonic()
res = self.loop.run_until_complete(tasks.Task(foo(), loop=self.loop))
t1 = time.monotonic()
self.assertTrue(t1 - t0 >= 0.14)
self.assertTrue('a' in res[:2])
self.assertTrue('b' in res[:2])
self.assertEqual(res[2], 'c')
# Doing it again should take no time and exercise a different path.
t0 = time.monotonic()
res = self.loop.run_until_complete(tasks.Task(foo(), loop=self.loop))
t1 = time.monotonic()
self.assertTrue(t1 - t0 <= 0.01)
def test_get_with_waiting_putters(self):
q = queues.Queue(loop=self.loop, maxsize=1)
tasks.Task(q.put('a'), loop=self.loop)
tasks.Task(q.put('b'), loop=self.loop)
test_utils.run_briefly(self.loop)
self.assertEqual(self.loop.run_until_complete(q.get()), 'a')
self.assertEqual(self.loop.run_until_complete(q.get()), 'b')
def test_POST_FILES_WITH_DATA(self):
url = self.server.url('method', 'post')
with open(__file__) as f:
r = self.event_loop.run_until_complete(
tasks.Task(
api.request('post',
url,
data={'test': 'true'},
files={'some': f})))
content = self.event_loop.run_until_complete(
tasks.Task(r.read(True)))
self.assertEqual(2, len(content['multipart-data']))
self.assertEqual('test', content['multipart-data'][0]['name'])
self.assertEqual('true', content['multipart-data'][0]['data'])
f.seek(0)
filename = os.path.split(f.name)[-1]
self.assertEqual('some', content['multipart-data'][1]['name'])
self.assertEqual(filename,
content['multipart-data'][1]['filename'])
self.assertEqual(f.read(), content['multipart-data'][1]['data'])
self.assertEqual(r.status, 200)
def test_POST_FILES_DEFLATE(self):
url = self.server.url('method', 'post')
with open(__file__) as f:
r = self.event_loop.run_until_complete(
tasks.Task(
api.request('post',
url,
files={'some': f},
chunked=1024,
compress='deflate')))
content = self.event_loop.run_until_complete(
tasks.Task(r.read(True)))
f.seek(0)
filename = os.path.split(f.name)[-1]
self.assertEqual('deflate', content['compression'])
self.assertEqual(1, len(content['multipart-data']))
self.assertEqual('some', content['multipart-data'][0]['name'])
self.assertEqual(filename,
content['multipart-data'][0]['filename'])
self.assertEqual(f.read(), content['multipart-data'][0]['data'])
self.assertEqual(r.status, 200)
def test_wait_with_exception(self):
def gen():
when = yield
self.assertAlmostEqual(0.1, when)
when = yield 0
self.assertAlmostEqual(0.15, when)
yield 0.15
loop = test_utils.TestLoop(gen)
self.addCleanup(loop.close)
a = tasks.Task(tasks.sleep(0.1, loop=loop), loop=loop)
@tasks.coroutine
def sleeper():
yield from tasks.sleep(0.15, loop=loop)
raise ZeroDivisionError('really')
b = tasks.Task(sleeper(), loop=loop)
@tasks.coroutine
def foo():
done, pending = yield from tasks.wait([b, a], loop=loop)
self.assertEqual(len(done), 2)
self.assertEqual(pending, set())
errors = set(f for f in done if f.exception() is not None)
self.assertEqual(len(errors), 1)
loop.run_until_complete(tasks.Task(foo(), loop=loop))
self.assertAlmostEqual(0.15, loop.time())
loop.run_until_complete(tasks.Task(foo(), loop=loop))
self.assertAlmostEqual(0.15, loop.time())
def test_wait_concurrent_complete(self):
def gen():
when = yield
self.assertAlmostEqual(0.1, when)
when = yield 0
self.assertAlmostEqual(0.15, when)
when = yield 0
self.assertAlmostEqual(0.1, when)
yield 0.1
loop = test_utils.TestLoop(gen)
self.addCleanup(loop.close)
a = tasks.Task(tasks.sleep(0.1, loop=loop), loop=loop)
b = tasks.Task(tasks.sleep(0.15, loop=loop), loop=loop)
done, pending = loop.run_until_complete(
tasks.wait([b, a], timeout=0.1, loop=loop))
self.assertEqual(done, set([a]))
self.assertEqual(pending, set([b]))
self.assertAlmostEqual(0.1, loop.time())
# move forward to close generator
loop.advance_time(10)
loop.run_until_complete(tasks.wait([a, b], loop=loop))
def test_wait_first_completed(self):
def gen():
when = yield
self.assertAlmostEqual(10.0, when)
when = yield 0
self.assertAlmostEqual(0.1, when)
yield 0.1
loop = test_utils.TestLoop(gen)
self.addCleanup(loop.close)
a = tasks.Task(tasks.sleep(10.0, loop=loop), loop=loop)
b = tasks.Task(tasks.sleep(0.1, loop=loop), loop=loop)
task = tasks.Task(tasks.wait([b, a],
return_when=tasks.FIRST_COMPLETED,
loop=loop),
loop=loop)
done, pending = loop.run_until_complete(task)
self.assertEqual({b}, done)
self.assertEqual({a}, pending)
self.assertFalse(a.done())
self.assertTrue(b.done())
self.assertIsNone(b.result())
self.assertAlmostEqual(0.1, loop.time())
# move forward to close generator
loop.advance_time(10)
loop.run_until_complete(tasks.wait([a, b], loop=loop))
def test_wait_first_exception_in_wait(self):
def gen():
when = yield
self.assertAlmostEqual(10.0, when)
when = yield 0
self.assertAlmostEqual(0.01, when)
yield 0.01
loop = test_utils.TestLoop(gen)
self.addCleanup(loop.close)
# first_exception, exception during waiting
a = tasks.Task(tasks.sleep(10.0, loop=loop), loop=loop)
@tasks.coroutine
def exc():
yield from tasks.sleep(0.01, loop=loop)
raise ZeroDivisionError('err')
b = tasks.Task(exc(), loop=loop)
task = tasks.wait([b, a], return_when=tasks.FIRST_EXCEPTION, loop=loop)
done, pending = loop.run_until_complete(task)
self.assertEqual({b}, done)
self.assertEqual({a}, pending)
self.assertAlmostEqual(0.01, loop.time())
# move forward to close generator
loop.advance_time(10)
loop.run_until_complete(tasks.wait([a, b], loop=loop))
def test_wait_with_global_loop(self):
def gen():
when = yield
self.assertAlmostEqual(0.01, when)
when = yield 0
self.assertAlmostEqual(0.015, when)
yield 0.015
loop = test_utils.TestLoop(gen)
self.addCleanup(loop.close)
a = tasks.Task(tasks.sleep(0.01, loop=loop), loop=loop)
b = tasks.Task(tasks.sleep(0.015, loop=loop), loop=loop)
@tasks.coroutine
def foo():
done, pending = yield from tasks.wait([b, a])
self.assertEqual(done, set([a, b]))
self.assertEqual(pending, set())
return 42
events.set_event_loop(loop)
try:
res = loop.run_until_complete(tasks.Task(foo(), loop=loop))
finally:
events.set_event_loop(None)
self.assertEqual(res, 42)
def test_wait(self):
def gen():
when = yield
self.assertAlmostEqual(0.1, when)
when = yield 0
self.assertAlmostEqual(0.15, when)
yield 0.15
loop = test_utils.TestLoop(gen)
self.addCleanup(loop.close)
a = tasks.Task(tasks.sleep(0.1, loop=loop), loop=loop)
b = tasks.Task(tasks.sleep(0.15, loop=loop), loop=loop)
@tasks.coroutine
def foo():
done, pending = yield from tasks.wait([b, a], loop=loop)
self.assertEqual(done, set([a, b]))
self.assertEqual(pending, set())
return 42
res = loop.run_until_complete(tasks.Task(foo(), loop=loop))
self.assertEqual(res, 42)
self.assertAlmostEqual(0.15, loop.time())
# Doing it again should take no time and exercise a different path.
res = loop.run_until_complete(tasks.Task(foo(), loop=loop))
self.assertAlmostEqual(0.15, loop.time())
self.assertEqual(res, 42)
def test_chunked(self):
r = self.event_loop.run_until_complete(
tasks.Task(api.request('get', self.server.url('chunked'))))
self.assertEqual(r.status, 200)
self.assertEqual(r.headers['Transfer-Encoding'], 'chunked')
content = self.event_loop.run_until_complete(tasks.Task(r.read(True)))
self.assertEqual(content['path'], '/chunked')
def test_wait_with_exception(self):
a = tasks.Task(tasks.sleep(0.1, loop=self.loop), loop=self.loop)
@tasks.coroutine
def sleeper():
yield from tasks.sleep(0.15, loop=self.loop)
raise ZeroDivisionError('really')
b = tasks.Task(sleeper(), loop=self.loop)
@tasks.coroutine
def foo():
done, pending = yield from tasks.wait([b, a], loop=self.loop)
self.assertEqual(len(done), 2)
self.assertEqual(pending, set())
errors = set(f for f in done if f.exception() is not None)
self.assertEqual(len(errors), 1)
t0 = time.monotonic()
self.loop.run_until_complete(tasks.Task(foo(), loop=self.loop))
t1 = time.monotonic()
self.assertTrue(t1 - t0 >= 0.14)
t0 = time.monotonic()
self.loop.run_until_complete(tasks.Task(foo(), loop=self.loop))
t1 = time.monotonic()
self.assertTrue(t1 - t0 <= 0.01)
def test_wait_concurrent_complete(self):
a = tasks.Task(tasks.sleep(0.1, loop=self.loop), loop=self.loop)
b = tasks.Task(tasks.sleep(0.15, loop=self.loop), loop=self.loop)
done, pending = self.loop.run_until_complete(
tasks.wait([b, a], timeout=0.1, loop=self.loop))
self.assertEqual(done, set([a]))
self.assertEqual(pending, set([b]))
def test_POST_DATA(self):
url = self.server.url('method', 'post')
r = self.event_loop.run_until_complete(
tasks.Task(api.request('post', url, data={'some': 'data'})))
self.assertEqual(r.status, 200)
content = self.event_loop.run_until_complete(tasks.Task(r.read(True)))
self.assertEqual({'some': ['data']}, content['form'])
self.assertEqual(r.status, 200)
def test_encoding(self):
r = self.event_loop.run_until_complete(
tasks.Task(
api.request('get', self.server.url('encoding', 'deflate'))))
self.assertEqual(r.status, 200)
r = self.event_loop.run_until_complete(
tasks.Task(api.request('get', self.server.url('encoding',
'gzip'))))
self.assertEqual(r.status, 200)
def test_stream(self):
wstream, response_fut = self.event_loop.run_until_complete(
tasks.Task(api.stream('get', self.server.url('method', 'get'))))
r = self.event_loop.run_until_complete(tasks.Task(response_fut))
content = self.event_loop.run_until_complete(tasks.Task(r.read()))
content = content.decode()
self.assertEqual(r.status, 200)
self.assertIn('"method": "GET"', content)
def test_notify(self):
cond = locks.Condition(loop=self.loop)
result = []
@tasks.coroutine
def c1(result):
yield from cond.acquire()
if (yield from cond.wait()):
result.append(1)
cond.release()
return True
@tasks.coroutine
def c2(result):
yield from cond.acquire()
if (yield from cond.wait()):
result.append(2)
cond.release()
return True
@tasks.coroutine
def c3(result):
yield from cond.acquire()
if (yield from cond.wait()):
result.append(3)
cond.release()
return True
t1 = tasks.Task(c1(result), loop=self.loop)
t2 = tasks.Task(c2(result), loop=self.loop)
t3 = tasks.Task(c3(result), loop=self.loop)
test_utils.run_briefly(self.loop)
self.assertEqual([], result)
self.loop.run_until_complete(cond.acquire())
cond.notify(1)
cond.release()
test_utils.run_briefly(self.loop)
self.assertEqual([1], result)
self.loop.run_until_complete(cond.acquire())
cond.notify(1)
cond.notify(2048)
cond.release()
test_utils.run_briefly(self.loop)
self.assertEqual([1, 2, 3], result)
self.assertTrue(t1.done())
self.assertTrue(t1.result())
self.assertTrue(t2.done())
self.assertTrue(t2.result())
self.assertTrue(t3.done())
self.assertTrue(t3.result())
def test_acquire(self):
lock = locks.Lock(loop=self.loop)
result = []
self.assertTrue(self.loop.run_until_complete(lock.acquire()))
@tasks.coroutine
def c1(result):
if (yield from lock.acquire()):
result.append(1)
return True
@tasks.coroutine
def c2(result):
if (yield from lock.acquire()):
result.append(2)
return True
@tasks.coroutine
def c3(result):
if (yield from lock.acquire()):
result.append(3)
return True
t1 = tasks.Task(c1(result), loop=self.loop)
t2 = tasks.Task(c2(result), loop=self.loop)
test_utils.run_briefly(self.loop)
self.assertEqual([], result)
lock.release()
test_utils.run_briefly(self.loop)
self.assertEqual([1], result)
test_utils.run_briefly(self.loop)
self.assertEqual([1], result)
t3 = tasks.Task(c3(result), loop=self.loop)
lock.release()
test_utils.run_briefly(self.loop)
self.assertEqual([1, 2], result)
lock.release()
test_utils.run_briefly(self.loop)
self.assertEqual([1, 2, 3], result)
self.assertTrue(t1.done())
self.assertTrue(t1.result())
self.assertTrue(t2.done())
self.assertTrue(t2.result())
self.assertTrue(t3.done())
self.assertTrue(t3.result())
def test_get_cancelled_race(self):
q = queues.Queue(loop=self.loop)
t1 = tasks.Task(q.get(), loop=self.loop)
t2 = tasks.Task(q.get(), loop=self.loop)
test_utils.run_briefly(self.loop)
t1.cancel()
test_utils.run_briefly(self.loop)
self.assertTrue(t1.done())
q.put_nowait('a')
test_utils.run_briefly(self.loop)
self.assertEqual(t2.result(), 'a')
def test_put_cancelled_race(self):
q = queues.Queue(loop=self.loop, maxsize=1)
tasks.Task(q.put('a'), loop=self.loop)
tasks.Task(q.put('c'), loop=self.loop)
t = tasks.Task(q.put('b'), loop=self.loop)
test_utils.run_briefly(self.loop)
t.cancel()
test_utils.run_briefly(self.loop)
self.assertTrue(t.done())
self.assertEqual(q.get_nowait(), 'a')
self.assertEqual(q.get_nowait(), 'c')
def test_exception_waiter(self):
buffer = parsers.DataBuffer(loop=self.loop)
@tasks.coroutine
def set_err():
buffer.set_exception(ValueError())
t1 = tasks.Task(buffer.read(), loop=self.loop)
t2 = tasks.Task(set_err(), loop=self.loop)
self.loop.run_until_complete(tasks.wait([t1, t2], loop=self.loop))
self.assertRaises(ValueError, t1.result)
def test_wait_first_completed(self):
a = tasks.Task(tasks.sleep(10.0, loop=self.loop), loop=self.loop)
b = tasks.Task(tasks.sleep(0.1, loop=self.loop), loop=self.loop)
task = tasks.Task(tasks.wait([b, a],
return_when=tasks.FIRST_COMPLETED,
loop=self.loop),
loop=self.loop)
done, pending = self.loop.run_until_complete(task)
self.assertEqual({b}, done)
self.assertEqual({a}, pending)
self.assertFalse(a.done())
self.assertTrue(b.done())
self.assertIsNone(b.result())
def test_task_class(self):
@tasks.coroutine
def notmuch():
return 'ok'
t = tasks.Task(notmuch(), loop=self.loop)
self.loop.run_until_complete(t)
self.assertTrue(t.done())
self.assertEqual(t.result(), 'ok')
self.assertIs(t._loop, self.loop)
loop = events.new_event_loop()
t = tasks.Task(notmuch(), loop=loop)
self.assertIs(t._loop, loop)
loop.close()
def test_cancel_done_future(self):
fut1 = futures.Future(loop=self.loop)
fut2 = futures.Future(loop=self.loop)
fut3 = futures.Future(loop=self.loop)
@tasks.coroutine
def task():
yield from fut1
try:
yield from fut2
except futures.CancelledError:
pass
yield from fut3
t = tasks.Task(task(), loop=self.loop)
test_utils.run_briefly(self.loop)
fut1.set_result(None)
t.cancel()
test_utils.run_once(self.loop) # process fut1 result, delay cancel
self.assertFalse(t.done())
test_utils.run_once(self.loop) # cancel fut2, but coro still alive
self.assertFalse(t.done())
test_utils.run_briefly(self.loop) # cancel fut3
self.assertTrue(t.done())
self.assertEqual(fut1.result(), None)
self.assertTrue(fut2.cancelled())
self.assertTrue(fut3.cancelled())
self.assertTrue(t.cancelled())
def test_clear_with_waiters(self):
ev = locks.Event(loop=self.loop)
result = []
@tasks.coroutine
def c1(result):
if (yield from ev.wait()):
result.append(1)
return True
t = tasks.Task(c1(result), loop=self.loop)
test_utils.run_briefly(self.loop)
self.assertEqual([], result)
ev.set()
ev.clear()
self.assertFalse(ev.is_set())
ev.set()
ev.set()
self.assertEqual(1, len(ev._waiters))
test_utils.run_briefly(self.loop)
self.assertEqual([1], result)
self.assertEqual(0, len(ev._waiters))
self.assertTrue(t.done())
self.assertTrue(t.result())
def add_getter():
q = queues.Queue(loop=self.loop)
# Start a task that waits to get.
tasks.Task(q.get(), loop=self.loop)
# Let it start waiting.
yield from tasks.sleep(0.1, loop=self.loop)
self.assertTrue('_getters[1]' in fn(q))
def queue_get():
self.loop.call_later(0.01, q.get_nowait)
queue_put_task = tasks.Task(queue_put(), loop=self.loop)
yield from started.wait()
self.assertFalse(finished)
yield from queue_put_task
self.assertTrue(finished)
def test_coroutine_non_gen_function_return_future(self):
fut = futures.Future(loop=self.loop)
@tasks.coroutine
def func():
return fut
@tasks.coroutine
def coro():
fut.set_result('test')
t1 = tasks.Task(func(), loop=self.loop)
t2 = tasks.Task(coro(), loop=self.loop)
res = self.loop.run_until_complete(t1)
self.assertEqual(res, 'test')
self.assertIsNone(t2.result())
def test_step_result_future(self):
# If coroutine returns future, task waits on this future.
class Fut(futures.Future):
def __init__(self, *args, **kwds):
self.cb_added = False
super().__init__(*args, **kwds)
def add_done_callback(self, fn):
self.cb_added = True
super().add_done_callback(fn)
fut = Fut(loop=self.loop)
result = None
@tasks.coroutine
def wait_for_future():
nonlocal result
result = yield from fut
t = tasks.Task(wait_for_future(), loop=self.loop)
test_utils.run_briefly(self.loop)
self.assertTrue(fut.cb_added)
res = object()
fut.set_result(res)
test_utils.run_briefly(self.loop)
self.assertIs(res, result)
self.assertTrue(t.done())
self.assertIsNone(t.result())