def test_AllProgress(e, s, a, b): x, y, z = e.map(inc, [1, 2, 3]) xx, yy, zz = e.map(dec, [x, y, z]) yield _wait([x, y, z]) p = AllProgress(s) assert p.all['inc'] == {x.key, y.key, z.key} assert p.in_memory['inc'] == {x.key, y.key, z.key} assert p.released == {} assert p.erred == {} yield _wait([xx, yy, zz]) assert p.all['dec'] == {xx.key, yy.key, zz.key} assert p.in_memory['dec'] == {xx.key, yy.key, zz.key} assert p.released == {} assert p.erred == {} s.client_releases_keys(client=e.id, keys=[x.key, y.key, z.key]) assert p.released['inc'] == {x.key, y.key, z.key} assert p.all['inc'] == {x.key, y.key, z.key} assert p.all['dec'] == {xx.key, yy.key, zz.key} xxx = e.submit(div, 1, 0) yield _wait([xxx]) assert p.erred == {'div': {xxx.key}} s.client_releases_keys(client=e.id, keys=[xx.key, yy.key, zz.key]) for c in [p.all, p.in_memory, p.released, p.erred]: assert 'inc' not in c assert 'dec' not in c def f(x): return x for i in range(4): future = e.submit(f, i) import gc gc.collect() yield gen.sleep(1) yield _wait([future]) assert p.in_memory == {'f': {future.key}} yield e._restart() for c in [p.all, p.in_memory, p.released, p.erred]: assert not c x = e.submit(div, 1, 2) yield _wait([x]) p.validate()
def test_AllProgress(e, s, a, b): x, y, z = e.map(inc, [1, 2, 3]) xx, yy, zz = e.map(dec, [x, y, z]) yield _wait([x, y, z]) p = AllProgress(s) assert p.all['inc'] == {x.key, y.key, z.key} assert p.in_memory['inc'] == {x.key, y.key, z.key} assert p.released == {} assert p.erred == {} yield _wait([xx, yy, zz]) assert p.all['dec'] == {xx.key, yy.key, zz.key} assert p.in_memory['dec'] == {xx.key, yy.key, zz.key} assert p.released == {} assert p.erred == {} s.client_releases_keys(client=e.id, keys=[x.key, y.key, z.key]) assert p.released['inc'] == {x.key, y.key, z.key} assert p.all['inc'] == {x.key, y.key, z.key} assert p.all['dec'] == {xx.key, yy.key, zz.key} xxx = e.submit(div, 1, 0) yield _wait([xxx]) assert p.erred == {'div': {xxx.key}} s.client_releases_keys(client=e.id, keys=[xx.key, yy.key, zz.key]) for c in [p.all, p.in_memory, p.released, p.erred]: assert 'inc' not in c assert 'dec' not in c def f(x): return x for i in range(4): future = e.submit(f, i) import gc; gc.collect() yield gen.sleep(1) yield _wait([future]) assert p.in_memory == {'f': {future.key}} yield e._restart() for c in [p.all, p.in_memory, p.released, p.erred]: assert not c x = e.submit(div, 1, 2) yield _wait([x]) p.validate()
async def test_AllProgress_lost_key(c, s, a, b): p = AllProgress(s) futures = c.map(inc, range(5)) await wait(futures) assert len(p.state["memory"]["inc"]) == 5 await a.close() await b.close() while len(p.state["memory"]["inc"]) > 0: await asyncio.sleep(0.01)
def test_AllProgress_lost_key(c, s, a, b, timeout=None): p = AllProgress(s) futures = c.map(inc, range(5)) yield wait(futures) assert len(p.state["memory"]["inc"]) == 5 yield a._close() yield b._close() start = time() while len(p.state["memory"]["inc"]) > 0: yield gen.sleep(0.1) assert time() < start + 5
def test_AllProgress_lost_key(e, s, a, b, timeout=None): p = AllProgress(s) futures = e.map(inc, range(5)) yield _wait(futures) assert len(p.in_memory['inc']) == 5 yield a._close() yield b._close() start = time() while len(p.in_memory['inc']) > 0: yield gen.sleep(0.1) assert time() < start + 2
def test_AllProgress(c, s, a, b): x, y, z = c.map(inc, [1, 2, 3]) xx, yy, zz = c.map(dec, [x, y, z]) yield wait([x, y, z]) p = AllProgress(s) assert p.all["inc"] == {x.key, y.key, z.key} assert p.state["memory"]["inc"] == {x.key, y.key, z.key} assert p.state["released"] == {} assert p.state["erred"] == {} assert "inc" in p.nbytes assert isinstance(p.nbytes["inc"], int) assert p.nbytes["inc"] > 0 yield wait([xx, yy, zz]) assert p.all["dec"] == {xx.key, yy.key, zz.key} assert p.state["memory"]["dec"] == {xx.key, yy.key, zz.key} assert p.state["released"] == {} assert p.state["erred"] == {} assert p.nbytes["inc"] == p.nbytes["dec"] t = c.submit(sum, [x, y, z]) yield t keys = {x.key, y.key, z.key} del x, y, z import gc gc.collect() while any(k in s.who_has for k in keys): yield gen.sleep(0.01) assert p.state["released"]["inc"] == keys assert p.all["inc"] == keys assert p.all["dec"] == {xx.key, yy.key, zz.key} if "inc" in p.nbytes: assert p.nbytes["inc"] == 0 xxx = c.submit(div, 1, 0) yield wait([xxx]) assert p.state["erred"] == {"div": {xxx.key}} tkey = t.key del xx, yy, zz, t import gc gc.collect() while tkey in s.tasks: yield gen.sleep(0.01) for coll in [p.all, p.nbytes] + list(p.state.values()): assert "inc" not in coll assert "dec" not in coll def f(x): return x for i in range(4): future = c.submit(f, i) import gc gc.collect() yield gen.sleep(1) yield wait([future]) assert p.state["memory"] == {"f": {future.key}} yield c._restart() for coll in [p.all] + list(p.state.values()): assert not coll x = c.submit(div, 1, 2) yield wait([x]) assert set(p.all) == {"div"} assert all(set(d) == {"div"} for d in p.state.values())
def test_AllProgress(c, s, a, b): x, y, z = c.map(inc, [1, 2, 3]) xx, yy, zz = c.map(dec, [x, y, z]) yield _wait([x, y, z]) p = AllProgress(s) assert p.all['inc'] == {x.key, y.key, z.key} assert p.state['memory']['inc'] == {x.key, y.key, z.key} assert p.state['released'] == {} assert p.state['erred'] == {} assert 'inc' in p.nbytes assert isinstance(p.nbytes['inc'], int) assert p.nbytes['inc'] > 0 yield _wait([xx, yy, zz]) assert p.all['dec'] == {xx.key, yy.key, zz.key} assert p.state['memory']['dec'] == {xx.key, yy.key, zz.key} assert p.state['released'] == {} assert p.state['erred'] == {} assert p.nbytes['inc'] == p.nbytes['dec'] t = c.submit(sum, [x, y, z]) yield t._result() keys = {x.key, y.key, z.key} del x, y, z import gc gc.collect() while any(k in s.who_has for k in keys): yield gen.sleep(0.01) assert p.state['released']['inc'] == keys assert p.all['inc'] == keys assert p.all['dec'] == {xx.key, yy.key, zz.key} if 'inc' in p.nbytes: assert p.nbytes['inc'] == 0 xxx = c.submit(div, 1, 0) yield _wait([xxx]) assert p.state['erred'] == {'div': {xxx.key}} tkey = t.key del xx, yy, zz, t import gc gc.collect() while tkey in s.task_state: yield gen.sleep(0.01) for coll in [p.all, p.nbytes] + list(p.state.values()): assert 'inc' not in coll assert 'dec' not in coll def f(x): return x for i in range(4): future = c.submit(f, i) import gc gc.collect() yield gen.sleep(1) yield _wait([future]) assert p.state['memory'] == {'f': {future.key}} yield c._restart() for coll in [p.all] + list(p.state.values()): assert not coll x = c.submit(div, 1, 2) yield _wait([x]) assert set(p.all) == {'div'} assert all(set(d) == {'div'} for d in p.state.values())