def test_AllProgress(e, s, a, b):
x, y, z = e.map(inc, [1, 2, 3])
xx, yy, zz = e.map(dec, [x, y, z])
yield _wait([x, y, z])
p = AllProgress(s)
assert p.all['inc'] == {x.key, y.key, z.key}
assert p.in_memory['inc'] == {x.key, y.key, z.key}
assert p.released == {}
assert p.erred == {}
yield _wait([xx, yy, zz])
assert p.all['dec'] == {xx.key, yy.key, zz.key}
assert p.in_memory['dec'] == {xx.key, yy.key, zz.key}
assert p.released == {}
assert p.erred == {}
s.client_releases_keys(client=e.id, keys=[x.key, y.key, z.key])
assert p.released['inc'] == {x.key, y.key, z.key}
assert p.all['inc'] == {x.key, y.key, z.key}
assert p.all['dec'] == {xx.key, yy.key, zz.key}
xxx = e.submit(div, 1, 0)
yield _wait([xxx])
assert p.erred == {'div': {xxx.key}}
s.client_releases_keys(client=e.id, keys=[xx.key, yy.key, zz.key])
for c in [p.all, p.in_memory, p.released, p.erred]:
assert 'inc' not in c
assert 'dec' not in c
def f(x):
return x
for i in range(4):
future = e.submit(f, i)
import gc
gc.collect()
yield gen.sleep(1)
yield _wait([future])
assert p.in_memory == {'f': {future.key}}
yield e._restart()
for c in [p.all, p.in_memory, p.released, p.erred]:
assert not c
x = e.submit(div, 1, 2)
yield _wait([x])
p.validate()
def test_AllProgress(e, s, a, b):
x, y, z = e.map(inc, [1, 2, 3])
xx, yy, zz = e.map(dec, [x, y, z])
yield _wait([x, y, z])
p = AllProgress(s)
assert p.all['inc'] == {x.key, y.key, z.key}
assert p.in_memory['inc'] == {x.key, y.key, z.key}
assert p.released == {}
assert p.erred == {}
yield _wait([xx, yy, zz])
assert p.all['dec'] == {xx.key, yy.key, zz.key}
assert p.in_memory['dec'] == {xx.key, yy.key, zz.key}
assert p.released == {}
assert p.erred == {}
s.client_releases_keys(client=e.id, keys=[x.key, y.key, z.key])
assert p.released['inc'] == {x.key, y.key, z.key}
assert p.all['inc'] == {x.key, y.key, z.key}
assert p.all['dec'] == {xx.key, yy.key, zz.key}
xxx = e.submit(div, 1, 0)
yield _wait([xxx])
assert p.erred == {'div': {xxx.key}}
s.client_releases_keys(client=e.id, keys=[xx.key, yy.key, zz.key])
for c in [p.all, p.in_memory, p.released, p.erred]:
assert 'inc' not in c
assert 'dec' not in c
def f(x):
return x
for i in range(4):
future = e.submit(f, i)
import gc; gc.collect()
yield gen.sleep(1)
yield _wait([future])
assert p.in_memory == {'f': {future.key}}
yield e._restart()
for c in [p.all, p.in_memory, p.released, p.erred]:
assert not c
x = e.submit(div, 1, 2)
yield _wait([x])
p.validate()
async def test_AllProgress_lost_key(c, s, a, b):
p = AllProgress(s)
futures = c.map(inc, range(5))
await wait(futures)
assert len(p.state["memory"]["inc"]) == 5
await a.close()
await b.close()
while len(p.state["memory"]["inc"]) > 0:
await asyncio.sleep(0.01)
def test_AllProgress_lost_key(c, s, a, b, timeout=None):
p = AllProgress(s)
futures = c.map(inc, range(5))
yield wait(futures)
assert len(p.state["memory"]["inc"]) == 5
yield a._close()
yield b._close()
start = time()
while len(p.state["memory"]["inc"]) > 0:
yield gen.sleep(0.1)
assert time() < start + 5
def test_AllProgress_lost_key(e, s, a, b, timeout=None):
p = AllProgress(s)
futures = e.map(inc, range(5))
yield _wait(futures)
assert len(p.in_memory['inc']) == 5
yield a._close()
yield b._close()
start = time()
while len(p.in_memory['inc']) > 0:
yield gen.sleep(0.1)
assert time() < start + 2
def test_AllProgress(c, s, a, b):
x, y, z = c.map(inc, [1, 2, 3])
xx, yy, zz = c.map(dec, [x, y, z])
yield wait([x, y, z])
p = AllProgress(s)
assert p.all["inc"] == {x.key, y.key, z.key}
assert p.state["memory"]["inc"] == {x.key, y.key, z.key}
assert p.state["released"] == {}
assert p.state["erred"] == {}
assert "inc" in p.nbytes
assert isinstance(p.nbytes["inc"], int)
assert p.nbytes["inc"] > 0
yield wait([xx, yy, zz])
assert p.all["dec"] == {xx.key, yy.key, zz.key}
assert p.state["memory"]["dec"] == {xx.key, yy.key, zz.key}
assert p.state["released"] == {}
assert p.state["erred"] == {}
assert p.nbytes["inc"] == p.nbytes["dec"]
t = c.submit(sum, [x, y, z])
yield t
keys = {x.key, y.key, z.key}
del x, y, z
import gc
gc.collect()
while any(k in s.who_has for k in keys):
yield gen.sleep(0.01)
assert p.state["released"]["inc"] == keys
assert p.all["inc"] == keys
assert p.all["dec"] == {xx.key, yy.key, zz.key}
if "inc" in p.nbytes:
assert p.nbytes["inc"] == 0
xxx = c.submit(div, 1, 0)
yield wait([xxx])
assert p.state["erred"] == {"div": {xxx.key}}
tkey = t.key
del xx, yy, zz, t
import gc
gc.collect()
while tkey in s.tasks:
yield gen.sleep(0.01)
for coll in [p.all, p.nbytes] + list(p.state.values()):
assert "inc" not in coll
assert "dec" not in coll
def f(x):
return x
for i in range(4):
future = c.submit(f, i)
import gc
gc.collect()
yield gen.sleep(1)
yield wait([future])
assert p.state["memory"] == {"f": {future.key}}
yield c._restart()
for coll in [p.all] + list(p.state.values()):
assert not coll
x = c.submit(div, 1, 2)
yield wait([x])
assert set(p.all) == {"div"}
assert all(set(d) == {"div"} for d in p.state.values())
def test_AllProgress(c, s, a, b):
x, y, z = c.map(inc, [1, 2, 3])
xx, yy, zz = c.map(dec, [x, y, z])
yield _wait([x, y, z])
p = AllProgress(s)
assert p.all['inc'] == {x.key, y.key, z.key}
assert p.state['memory']['inc'] == {x.key, y.key, z.key}
assert p.state['released'] == {}
assert p.state['erred'] == {}
assert 'inc' in p.nbytes
assert isinstance(p.nbytes['inc'], int)
assert p.nbytes['inc'] > 0
yield _wait([xx, yy, zz])
assert p.all['dec'] == {xx.key, yy.key, zz.key}
assert p.state['memory']['dec'] == {xx.key, yy.key, zz.key}
assert p.state['released'] == {}
assert p.state['erred'] == {}
assert p.nbytes['inc'] == p.nbytes['dec']
t = c.submit(sum, [x, y, z])
yield t._result()
keys = {x.key, y.key, z.key}
del x, y, z
import gc
gc.collect()
while any(k in s.who_has for k in keys):
yield gen.sleep(0.01)
assert p.state['released']['inc'] == keys
assert p.all['inc'] == keys
assert p.all['dec'] == {xx.key, yy.key, zz.key}
if 'inc' in p.nbytes:
assert p.nbytes['inc'] == 0
xxx = c.submit(div, 1, 0)
yield _wait([xxx])
assert p.state['erred'] == {'div': {xxx.key}}
tkey = t.key
del xx, yy, zz, t
import gc
gc.collect()
while tkey in s.task_state:
yield gen.sleep(0.01)
for coll in [p.all, p.nbytes] + list(p.state.values()):
assert 'inc' not in coll
assert 'dec' not in coll
def f(x):
return x
for i in range(4):
future = c.submit(f, i)
import gc
gc.collect()
yield gen.sleep(1)
yield _wait([future])
assert p.state['memory'] == {'f': {future.key}}
yield c._restart()
for coll in [p.all] + list(p.state.values()):
assert not coll
x = c.submit(div, 1, 2)
yield _wait([x])
assert set(p.all) == {'div'}
assert all(set(d) == {'div'} for d in p.state.values())
