def test_as_completed(loop):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
x = c.submit(inc, 1)
y = c.submit(inc, 2)
z = c.submit(inc, 1)
seq = as_completed([x, y, z])
assert isinstance(seq, Iterator)
assert set(seq) == {x, y, z}
assert list(as_completed([])) == []
def test_as_completed(client):
x = client.submit(inc, 1)
y = client.submit(inc, 2)
z = client.submit(inc, 1)
seq = as_completed([x, y, z])
assert seq.count() == 3
assert isinstance(seq, Iterator)
assert set(seq) == {x, y, z}
assert seq.count() == 0
assert list(as_completed([])) == []
def test_as_completed(client):
x = client.submit(inc, 1)
y = client.submit(inc, 2)
z = client.submit(inc, 1)
seq = as_completed([x, y, z])
assert seq.count() == 3
assert isinstance(seq, Iterator)
assert set(seq) == {x, y, z}
assert seq.count() == 0
assert list(as_completed([])) == []
def test_as_completed_cancel(loop):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
x = c.submit(sleep, 1)
y = c.submit(inc, 1)
ac = as_completed([x, y])
x.cancel()
assert next(ac) is y
with pytest.raises(Empty):
ac.queue.get(timeout=0.1)
assert list(as_completed([x, y, x])) == [y]
def wait(collections, client=None, return_when=FIRST_COMPLETED):
"""Calculates collections on client.
Parameters
----------
collections : list of dask tasks
client : None or distributed.client.Client
if None uses distributed.client.default_client()
return_when : int
if == FIRST_COMPLETED returns when first task completes
if == ALL_COMPLETED returns when all tasks completed
Currently supports only FIRST_COMPLETED.
Returns
-------
tuple : (result, index, unfinished_futures)
"""
if return_when not in (FIRST_COMPLETED, ALL_COMPLETED):
raise ValueError(
"Unknown value for 'return_when'." +
"Expected {} or {}.".format(FIRST_COMPLETED, ALL_COMPLETED) +
"Received {}.".format(return_when))
if return_when == ALL_COMPLETED:
raise NotImplementedError("Support for ALL_COMPLETED not implemented.")
client = client or dc.default_client()
futures = client.compute(collections)
f = dc.as_completed(futures).__next__()
i = futures.index(f)
del futures[i]
res = f.result()
del f
return res, i, futures
def test_as_completed_with_actor(client):
class Counter:
n = 0
def __init__(self):
self.n = 0
def increment(self):
time.sleep(0.1)
self.n += 1
return self.n
future = client.submit(Counter, actor=True)
counter = future.result()
assert counter.n == 0
futures = []
for _ in range(3):
futures.append(counter.increment())
results = []
for future in as_completed(futures):
results.append(future.result())
assert counter.n == 3
assert sum(results) == 6 # i.e. future.result() returns 1, 2, 3
# ToDo: test other `as_completed` arguments, e.g.:
# for future, n in as_completed(futures, with_results=True):
# results.append(n)
def test_as_completed_is_empty(client):
ac = as_completed()
assert ac.is_empty()
x = client.submit(inc, 1)
ac.add(x)
assert not ac.is_empty()
assert next(ac) is x
assert ac.is_empty()
def test_as_completed_cancel(client):
x = client.submit(inc, 1)
y = client.submit(inc, 1)
ac = as_completed([x, y])
x.cancel()
assert next(ac) is x or y
assert next(ac) is y or x
with pytest.raises(Empty):
ac.queue.get(timeout=0.1)
res = list(as_completed([x, y, x]))
assert len(res) == 3
assert set(res) == {x, y}
assert res.count(x) == 2
async def test_async_for_py2_equivalent(c, s, a, b):
futures = c.map(sleep, [0.01] * 3, pure=False)
seq = as_completed(futures)
x, y, z = [el async for el in seq]
assert x.done()
assert y.done()
assert z.done()
assert x.key != y.key
def test_as_completed_is_empty(client):
ac = as_completed()
assert ac.is_empty()
x = client.submit(inc, 1)
ac.add(x)
assert not ac.is_empty()
assert next(ac) is x
assert ac.is_empty()
def test_as_completed_cancel(client):
x = client.submit(inc, 1)
y = client.submit(inc, 1)
ac = as_completed([x, y])
x.cancel()
assert next(ac) is x or y
assert next(ac) is y or x
with pytest.raises(Empty):
ac.queue.get(timeout=0.1)
res = list(as_completed([x, y, x]))
assert len(res) == 3
assert set(res) == {x, y}
assert res.count(x) == 2
async def test_as_completed_error_async(c, s, a, b):
x = c.submit(throws, 1)
y = c.submit(inc, 1)
ac = as_completed([x, y])
result = {el async for el in ac}
assert result == {x, y}
assert x.status == "error"
assert y.status == "finished"
def test_as_completed_error(client):
x = client.submit(throws, 1)
y = client.submit(inc, 1)
ac = as_completed([x, y])
result = set(ac)
assert result == {x, y}
assert x.status == "error"
assert y.status == "finished"
def test_as_completed_is_empty(loop):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
ac = as_completed()
assert ac.is_empty()
x = c.submit(inc, 1)
ac.add(x)
assert not ac.is_empty()
assert next(ac) is x
assert ac.is_empty()
def test_as_completed_error(client):
x = client.submit(throws, 1)
y = client.submit(inc, 1)
ac = as_completed([x, y])
result = set(ac)
assert result == {x, y}
assert x.status == 'error'
assert y.status == 'finished'
def test_as_completed_with_results(client):
x = client.submit(throws, 1)
y = client.submit(inc, 5)
z = client.submit(inc, 1)
ac = as_completed([x, y, z], with_results=True)
y.cancel()
with pytest.raises(RuntimeError) as exc:
res = list(ac)
assert str(exc.value) == 'hello!'
def test_as_completed_cancel(loop):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
x = c.submit(inc, 1)
y = c.submit(inc, 1)
ac = as_completed([x, y])
x.cancel()
assert next(ac) is x or y
assert next(ac) is y or x
with pytest.raises(Empty):
ac.queue.get(timeout=0.1)
res = list(as_completed([x, y, x]))
assert len(res) == 3
assert set(res) == {x, y}
assert res.count(x) == 2
def test_as_completed_with_results(client):
x = client.submit(throws, 1)
y = client.submit(inc, 5)
z = client.submit(inc, 1)
ac = as_completed([x, y, z], with_results=True)
y.cancel()
with pytest.raises(RuntimeError) as exc:
res = list(ac)
assert str(exc.value) == "hello!"
async def test_clear(c, s, a, b):
futures = c.map(inc, range(3))
ac = as_completed(futures)
await wait(futures)
ac.clear()
with pytest.raises(StopAsyncIteration):
await ac.__anext__()
del futures
while s.tasks:
await asyncio.sleep(0.3)
async def test_as_completed_with_results_async(c, s, a, b):
x = c.submit(throws, 1)
y = c.submit(inc, 5)
z = c.submit(inc, 1)
ac = as_completed([x, y, z], with_results=True)
await y.cancel()
with pytest.raises(RuntimeError) as exc:
async for _ in ac:
pass
assert str(exc.value) == "hello!"
def test_as_completed_update(client):
total = 0
todo = list(range(10))
expected = sum(map(inc, todo))
ac = as_completed([])
while todo or not ac.is_empty():
if todo:
work, todo = todo[:4], todo[4:]
ac.update(client.map(inc, work))
batch = ac.next_batch(block=True)
total += sum(r.result() for r in batch)
assert total == expected
def test_as_completed_error(loop):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
x = c.submit(throws, 1)
y = c.submit(inc, 1)
ac = as_completed([x, y])
result = set(ac)
assert result == {x, y}
assert x.status == 'error'
assert y.status == 'finished'
def test_as_completed_with_results_async(c, s, a, b):
x = c.submit(throws, 1)
y = c.submit(inc, 5)
z = c.submit(inc, 1)
ac = as_completed([x, y, z], with_results=True)
y.cancel()
with pytest.raises(RuntimeError) as exc:
first = yield ac.__anext__()
second = yield ac.__anext__()
third = yield ac.__anext__()
assert str(exc.value) == "hello!"
def test_as_completed_error_async(c, s, a, b):
x = c.submit(throws, 1)
y = c.submit(inc, 1)
ac = as_completed([x, y])
first = yield ac.__anext__()
second = yield ac.__anext__()
result = {first, second}
assert result == {x, y}
assert x.status == "error"
assert y.status == "finished"
def test_as_completed_update(client):
total = 0
todo = list(range(10))
expected = sum(map(inc, todo))
ac = as_completed([])
while todo or not ac.is_empty():
if todo:
work, todo = todo[:4], todo[4:]
ac.update(client.map(inc, work))
batch = ac.next_batch(block=True)
total += sum(r.result() for r in batch)
assert total == expected
def test_as_completed_error_async(c, s, a, b):
x = c.submit(throws, 1)
y = c.submit(inc, 1)
ac = as_completed([x, y])
first = yield ac.__anext__()
second = yield ac.__anext__()
result = {first, second}
assert result == {x, y}
assert x.status == 'error'
assert y.status == 'finished'
def test_as_completed_with_results(loop):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
x = c.submit(throws, 1)
y = c.submit(inc, 5)
z = c.submit(inc, 1)
ac = as_completed([x, y, z], with_results=True)
y.cancel()
with pytest.raises(RuntimeError) as exc:
res = list(ac)
assert str(exc.value) == 'hello!'
def test_as_completed_with_results_async(c, s, a, b):
x = c.submit(throws, 1)
y = c.submit(inc, 5)
z = c.submit(inc, 1)
ac = as_completed([x, y, z], with_results=True)
y.cancel()
with pytest.raises(RuntimeError) as exc:
first = yield ac.__anext__()
second = yield ac.__anext__()
third = yield ac.__anext__()
assert str(exc.value) == 'hello!'
def test_as_completed_add(client):
total = 0
expected = sum(map(inc, range(10)))
futures = client.map(inc, range(10))
ac = as_completed(futures)
for future in ac:
result = future.result()
total += result
if random.random() < 0.5:
future = client.submit(add, future, 10)
ac.add(future)
expected += result + 10
assert total == expected
def test_as_completed_add(client):
total = 0
expected = sum(map(inc, range(10)))
futures = client.map(inc, range(10))
ac = as_completed(futures)
for future in ac:
result = future.result()
total += result
if random.random() < 0.5:
future = client.submit(add, future, 10)
ac.add(future)
expected += result + 10
assert total == expected
async def test_str(c, s, a, b):
futures = c.map(inc, range(3))
ac = as_completed(futures)
assert "waiting=3" in str(ac)
assert "waiting=3" in repr(ac)
assert "done=0" in str(ac)
assert "done=0" in repr(ac)
await ac.__anext__()
start = time()
while "done=2" not in str(ac):
await asyncio.sleep(0.01)
assert time() < start + 2
def test_as_completed_update(loop):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
total = 0
todo = list(range(10))
expected = sum(map(inc, todo))
ac = as_completed([])
while todo or not ac.is_empty():
if todo:
work, todo = todo[:4], todo[4:]
ac.update(c.map(inc, work))
batch = ac.next_batch(block=True)
total += sum(r.result() for r in batch)
assert total == expected
def test_as_completed_repeats(client):
ac = as_completed()
x = client.submit(inc, 1)
ac.add(x)
ac.add(x)
assert next(ac) is x
assert next(ac) is x
with pytest.raises(StopIteration):
next(ac)
ac.add(x)
assert next(ac) is x
def test_async_for_py2_equivalent(c, s, a, b):
futures = c.map(sleep, [0.01] * 3, pure=False)
seq = as_completed(futures)
x = yield seq.__anext__()
y = yield seq.__anext__()
z = yield seq.__anext__()
assert x.done()
assert y.done()
assert z.done()
assert x.key != y.key
with pytest.raises(StopAsyncIteration):
yield seq.__anext__()
def test_as_completed_repeats(client):
ac = as_completed()
x = client.submit(inc, 1)
ac.add(x)
ac.add(x)
assert next(ac) is x
assert next(ac) is x
with pytest.raises(StopIteration):
next(ac)
ac.add(x)
assert next(ac) is x
def test_async_for_py2_equivalent(c, s, a, b):
futures = c.map(sleep, [0.01] * 3, pure=False)
seq = as_completed(futures)
x = yield seq.__anext__()
y = yield seq.__anext__()
z = yield seq.__anext__()
assert x.done()
assert y.done()
assert z.done()
assert x.key != y.key
with pytest.raises(StopAsyncIteration):
yield seq.__anext__()
def test_as_completed_add(loop):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
total = 0
expected = sum(map(inc, range(10)))
futures = c.map(inc, range(10))
ac = as_completed(futures)
for future in ac:
result = future.result()
total += result
if random.random() < 0.5:
future = c.submit(add, future, 10)
ac.add(future)
expected += result + 10
assert total == expected
def test_as_completed_cancel_last(client):
w = client.submit(inc, 0.3)
x = client.submit(inc, 1)
y = client.submit(inc, 0.3)
async def _():
await asyncio.sleep(0.1)
await w.cancel(asynchronous=True)
await y.cancel(asynchronous=True)
client.loop.add_callback(_)
ac = as_completed([x, y])
result = set(ac)
assert result == {x, y}
def test_as_completed_repeats(loop):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
ac = as_completed()
x = c.submit(inc, 1)
ac.add(x)
ac.add(x)
assert next(ac) is x
assert next(ac) is x
with pytest.raises(StopIteration):
next(ac)
ac.add(x)
assert next(ac) is x
def test_as_completed_cancel_last(client):
w = client.submit(inc, 0.3)
x = client.submit(inc, 1)
y = client.submit(inc, 0.3)
@gen.coroutine
def _():
yield gen.sleep(0.1)
yield w.cancel(asynchronous=True)
yield y.cancel(asynchronous=True)
client.loop.add_callback(_)
ac = as_completed([x, y])
result = set(ac)
assert result == {x, y}
def test_as_completed_cancel_last(client):
w = client.submit(inc, 0.3)
x = client.submit(inc, 1)
y = client.submit(inc, 0.3)
@gen.coroutine
def _():
yield gen.sleep(0.1)
yield w.cancel(asynchronous=True)
yield y.cancel(asynchronous=True)
client.loop.add_callback(_)
ac = as_completed([x, y])
result = set(ac)
assert result == {x, y}
def test_as_completed_with_results_no_raise(client):
x = client.submit(throws, 1)
y = client.submit(inc, 5)
z = client.submit(inc, 1)
ac = as_completed([x, y, z], with_results=True, raise_errors=False)
y.cancel()
res = list(ac)
dd = {r[0]: r[1:] for r in res}
assert set(dd.keys()) == {y, x, z}
assert x.status == 'error'
assert y.status == 'cancelled'
assert z.status == 'finished'
assert isinstance(dd[y][0], CancelledError)
assert isinstance(dd[x][0][1], RuntimeError)
assert dd[z][0] == 2
def test_as_completed_with_results_no_raise(client):
x = client.submit(throws, 1)
y = client.submit(inc, 5)
z = client.submit(inc, 1)
ac = as_completed([x, y, z], with_results=True, raise_errors=False)
y.cancel()
res = list(ac)
dd = {r[0]: r[1:] for r in res}
assert set(dd.keys()) == {y, x, z}
assert x.status == "error"
assert y.status == "cancelled"
assert z.status == "finished"
assert isinstance(dd[y][0], CancelledError) or dd[y][0] == 6
assert isinstance(dd[x][0][1], RuntimeError)
assert dd[z][0] == 2
def test_as_completed_with_results_no_raise_async(c, s, a, b):
x = c.submit(throws, 1)
y = c.submit(inc, 5)
z = c.submit(inc, 1)
ac = as_completed([x, y, z], with_results=True, raise_errors=False)
y.cancel()
first = yield ac.__anext__()
second = yield ac.__anext__()
third = yield ac.__anext__()
res = [first, second, third]
dd = {r[0]: r[1:] for r in res}
assert set(dd.keys()) == {y, x, z}
assert x.status == 'error'
assert y.status == 'cancelled'
assert z.status == 'finished'
assert isinstance(dd[y][0], CancelledError)
assert isinstance(dd[x][0][1], RuntimeError)
assert dd[z][0] == 2
def test_as_completed_with_non_futures(client):
with pytest.raises(TypeError):
list(as_completed([1, 2, 3]))