def f(c, a, b): e1 = Executor((c.ip, c.port), start=False, loop=loop) yield e1._start() assert isinstance(e1.center, rpc) assert isinstance(e1.scheduler, Scheduler) s = Scheduler((c.ip, c.port)) yield s.sync_center() done = s.start() e2 = Executor(s, start=False, loop=loop) yield e2._start() assert isinstance(e2.center, rpc) assert isinstance(e2.scheduler, Scheduler) s.listen(8042) e3 = Executor(('127.0.0.1', s.port), start=False, loop=loop) yield e3._start() assert isinstance(e3.center, rpc) assert isinstance(e3.scheduler, rpc) s.stop() yield e1._shutdown() yield e2._shutdown() yield e3._shutdown()
def f(): yield a._start() yield b._start() while len(c.ncores) < 2: yield gen.sleep(0.01) try: e1 = Executor((c.ip, c.port), start=False, loop=loop) yield e1._start() e2 = Executor(e1.scheduler, start=False, loop=loop) yield e2._start() x = e1.submit(inc, 1) y = e2.submit(inc, 2) xx = yield x._result() yy = yield y._result() assert xx == 2 assert yy == 3 yield e1._restart() assert x.cancelled() assert y.cancelled() finally: yield a._close() yield b._close() yield e1._shutdown(fast=True) yield e2._shutdown(fast=True) c.stop()
def f(): yield a._start() yield b._start() while len(c.ncores) < 2: yield gen.sleep(0.01) try: e1 = Executor((c.ip, c.port), start=False, loop=loop) yield e1._start() e2 = Executor(scheduler=e1.scheduler, start=False, loop=loop) yield e2._start() x = e1.submit(inc, 1) y = e2.submit(inc, 2) xx = yield x._result() yy = yield y._result() assert xx == 2 assert yy == 3 yield e1._restart() assert x.cancelled() assert y.cancelled() finally: yield a._close() yield b._close() yield e1._shutdown(fast=True) yield e2._shutdown(fast=True) c.stop()
def test__scatter(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() d = yield e._scatter({'y': 20}) assert isinstance(d['y'], Future) assert a.data.get('y') == 20 or b.data.get('y') == 20 assert (a.address in s.who_has['y'] or b.address in s.who_has['y']) assert s.who_has['y'] assert s.nbytes == {'y': sizeof(20)} yy = yield e._gather([d['y']]) assert yy == [20] [x] = yield e._scatter([10]) assert isinstance(x, Future) assert a.data.get(x.key) == 10 or b.data.get(x.key) == 10 xx = yield e._gather([x]) assert s.who_has[x.key] assert (a.address in s.who_has[x.key] or b.address in s.who_has[x.key]) assert s.nbytes == {'y': sizeof(20), x.key: sizeof(10)} assert xx == [10] z = e.submit(add, x, d['y']) # submit works on RemoteData result = yield z._result() assert result == 10 + 20 result = yield e._gather([z, x]) assert result == [30, 10] yield e._shutdown()
def f(c, a, b): e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() L = e.map(inc, range(5), workers={a.ip}) yield _wait(L) assert set(a.data) == {x.key for x in L} assert not b.data for x in L: assert e.scheduler.restrictions[x.key] == {a.ip} L = e.map(inc, [10, 11, 12], workers=[{a.ip}, {a.ip, b.ip}, {b.ip}]) yield _wait(L) assert e.scheduler.restrictions[L[0].key] == {a.ip} assert e.scheduler.restrictions[L[1].key] == {a.ip, b.ip} assert e.scheduler.restrictions[L[2].key] == {b.ip} with pytest.raises(ValueError): e.map(inc, [10, 11, 12], workers=[{a.ip}]) yield e._shutdown()
def f(): yield a._start() yield b._start() e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() assert e.scheduler.ncores == {a.worker_address: 2, b.worker_address: 2} x = e.submit(inc, 1) y = e.submit(inc, x) yield y._result() cc = rpc(ip=c.ip, port=c.port) who_has = yield cc.who_has() try: assert e.scheduler.who_has == who_has assert set(e.scheduler.who_has) == {x.key, y.key} yield e._restart() assert len(e.scheduler.stacks) == 2 assert len(e.scheduler.processing) == 2 who_has = yield cc.who_has() assert not who_has assert not e.scheduler.who_has assert x.cancelled() assert y.cancelled() finally: yield a._close() yield b._close() yield e._shutdown(fast=True) c.stop()
def f(c, a, b): e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() d = yield e._scatter({'y': 20}) assert isinstance(d['y'], Future) assert a.data.get('y') == 20 or b.data.get('y') == 20 assert (a.address in e.scheduler.who_has['y'] or b.address in e.scheduler.who_has['y']) assert c.who_has['y'] assert e.scheduler.nbytes == {'y': sizeof(20)} yy = yield e._gather([d['y']]) assert yy == [20] [x] = yield e._scatter([10]) assert isinstance(x, Future) assert a.data.get(x.key) == 10 or b.data.get(x.key) == 10 xx = yield e._gather([x]) assert c.who_has[x.key] assert (a.address in e.scheduler.who_has[x.key] or b.address in e.scheduler.who_has[x.key]) assert e.scheduler.nbytes == {'y': sizeof(20), x.key: sizeof(10)} assert xx == [10] z = e.submit(add, x, d['y']) # submit works on RemoteData result = yield z._result() assert result == 10 + 20 result = yield e._gather([z, x]) assert result == [30, 10] yield e._shutdown()
def test_restrictions_map(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() L = e.map(inc, range(5), workers={a.ip}) yield _wait(L) assert set(a.data) == {x.key for x in L} assert not b.data for x in L: assert s.restrictions[x.key] == {a.ip} L = e.map(inc, [10, 11, 12], workers=[{a.ip}, {a.ip, b.ip}, {b.ip}]) yield _wait(L) assert s.restrictions[L[0].key] == {a.ip} assert s.restrictions[L[1].key] == {a.ip, b.ip} assert s.restrictions[L[2].key] == {b.ip} with pytest.raises(ValueError): e.map(inc, [10, 11, 12], workers=[{a.ip}]) yield e._shutdown()
def test_future(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() x = e.submit(inc, 10) assert str(x.key) in repr(x) assert str(x.status) in repr(x) yield e._shutdown()
def f(c, a, b): e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() x = e.submit(randint, 0, 1000, pure=True) xx = yield x._result() f = Executor((c.ip, c.port), start=False, loop=loop) yield f._start() y = f.submit(randint, 0, 1000, pure=True) yy = yield y._result() assert xx == yy yield e._shutdown() yield f._shutdown()
def test_upload_file_exception(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() with tmp_text('myfile.py', 'syntax-error!') as fn: with pytest.raises(SyntaxError): yield e._upload_file(fn) yield e._shutdown()
def f(c, a, b): e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() [x] = yield e._get({'x': (inc, 1)}, ['x']) import gc; gc.collect() assert e.refcount['x'] == 0 yield e._shutdown()
def test_directed_scatter(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() yield e._scatter([1, 2, 3], workers=[a.address]) assert len(a.data) == 3 assert not b.data yield e._shutdown()
def test_get_releases_data(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() [x] = yield e._get({'x': (inc, 1)}, ['x']) import gc; gc.collect() assert e.refcount['x'] == 0 yield e._shutdown()
def f(c, a, b): e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() with tmp_text('myfile.py', 'syntax-error!') as fn: with pytest.raises(SyntaxError): yield e._upload_file(fn) yield e._shutdown()
def f(c, a, b): e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() dsk = {'x': (div, 1, 0), 'y': (inc, 'x')} with pytest.raises(ZeroDivisionError): y = yield e._get(dsk, 'y') yield e._shutdown()
def test_two_consecutive_executors_share_results(s, a, b): from random import randint e = Executor((s.ip, s.port), start=False) yield e._start() x = e.submit(randint, 0, 1000, pure=True) xx = yield x._result() f = Executor((s.ip, s.port), start=False) yield f._start() y = f.submit(randint, 0, 1000, pure=True) yy = yield y._result() assert xx == yy yield e._shutdown() yield f._shutdown()
def f(c, a, b): e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() yield e._scatter([1, 2, 3], workers=[a.address]) assert len(a.data) == 3 assert not b.data yield e._shutdown()
def test_get_with_error(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() dsk = {'x': (div, 1, 0), 'y': (inc, 'x')} with pytest.raises(ZeroDivisionError): y = yield e._get(dsk, 'y') yield e._shutdown()
def f(c, a, b): e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() L = [e.submit(inc, i) for i in range(10)] yield _wait(L) assert a.data and b.data yield e._shutdown()
def test_many_submits_spread_evenly(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() L = [e.submit(inc, i) for i in range(10)] yield _wait(L) assert a.data and b.data yield e._shutdown()
def f(c, a, b): e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() from time import sleep x = e.submit(sleep, 3) yield x._result() yield e._shutdown()
def test_remote_submit_on_Future(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() x = e.submit(lambda x: x + 1, 1) y = e.submit(lambda x: x + 1, x) result = yield y._result() assert result == 3 yield e._shutdown()
def f(c, a, b): e = Executor((c.ip, c.port), delete_batch_time=0, start=False, loop=loop) yield e._start() from time import sleep x = e.submit(sleep, 3) yield x._result() yield e._shutdown()
def test_submit_then_get_with_Future(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() x = e.submit(slowinc, 1) dsk = {'y': (inc, x)} result = yield e._get(dsk, 'y') assert result == 3 yield e._shutdown()
def test_remote_scheduler(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() assert isinstance(e.scheduler_stream, IOStream) assert s.streams x = e.submit(inc, 1) result = yield x._result() yield e._shutdown()
def test_traceback(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() x = e.submit(div, 1, 0) tb = yield x._traceback() if sys.version_info[0] >= 3: assert any('x / y' in line for line in tb) yield e._shutdown()
def test_aliases(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() x = e.submit(inc, 1) dsk = {'y': x} result = yield e._get(dsk, 'y') assert result == 2 yield e._shutdown()
def f(c, a, b): e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() x = e.submit(inc, 1) dsk = {'y': x} result = yield e._get(dsk, 'y') assert result == 2 yield e._shutdown()
def test_badly_serialized_input(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() o = BadlySerializedObject() future = e.submit(inc, o) futures = e.map(inc, range(10)) L = yield e._gather(futures) assert list(L) == list(map(inc, range(10)))
def f(c, a, b): e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() x = e.submit(div, 1, 0) tb = yield x._traceback() if sys.version_info[0] >= 3: assert any('x / y' in line for line in tb) yield e._shutdown()
def test_Future_exception(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() x = e.submit(div, 1, 0) result = yield x._exception() assert isinstance(result, ZeroDivisionError) x = e.submit(div, 1, 1) result = yield x._exception() assert result is None yield e._shutdown()
def f(c, a, b): e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() x = e.submit(div, 1, 0) result = yield x._exception() assert isinstance(result, ZeroDivisionError) x = e.submit(div, 1, 1) result = yield x._exception() assert result is None yield e._shutdown()
def test_multiple_executors(s, a, b): a = Executor((s.ip, s.port), start=False) yield a._start() b = Executor((s.ip, s.port), start=False) yield b._start() x = a.submit(inc, 1) y = b.submit(inc, 2) assert x.executor is a assert y.executor is b xx = yield x._result() yy = yield y._result() assert xx == 2 assert yy == 3 z = a.submit(add, x, y) assert z.executor is a zz = yield z._result() assert zz == 5 yield a._shutdown() yield b._shutdown()
def f(c, a, b): e = Executor((c.ip, c.port), start=False, loop=loop) yield e._start() dsk = {('x', 0): (inc, 1), 5: (inc, 2)} x = yield e._get(dsk, ('x', 0)) y = yield e._get(dsk, 5) assert x == 2 assert y == 3 yield e._shutdown()
def f(c, a, b): a = Executor((c.ip, c.port), start=False, loop=loop) yield a._start() b = Executor(scheduler=a.scheduler, start=False, loop=loop) yield b._start() x = a.submit(inc, 1) y = b.submit(inc, 2) assert x.executor is a assert y.executor is b xx = yield x._result() yy = yield y._result() assert xx == 2 assert yy == 3 z = a.submit(add, x, y) assert z.executor is a zz = yield z._result() assert zz == 5 yield a._shutdown() yield b._shutdown()
def test_get_with_non_list_key(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() dsk = {('x', 0): (inc, 1), 5: (inc, 2)} x = yield e._get(dsk, ('x', 0)) y = yield e._get(dsk, 5) assert x == 2 assert y == 3 yield e._shutdown()
def test_restrictions_get(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() dsk = {'x': 1, 'y': (inc, 'x'), 'z': (inc, 'y')} restrictions = {'y': {a.ip}, 'z': {b.ip}} result = yield e._get(dsk, ['y', 'z'], restrictions) assert result == [2, 3] assert 'y' in a.data assert 'z' in b.data yield e._shutdown()
def dont_test_bad_restrictions_raise_exception(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() z = e.submit(inc, 2, workers={'bad-address'}) try: yield z._result() assert False except ValueError as e: assert 'bad-address' in str(e) assert z.key in str(e) yield e._shutdown()
def test_submit_naming(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() a = e.submit(inc, 1) b = e.submit(inc, 1) assert a.event is b.event c = e.submit(inc, 1, pure=False) assert c.key != a.key yield e._shutdown()
def test_nbytes_determines_worker(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() x = e.submit(identity, 1, workers=[a.address[0]]) y = e.submit(identity, tuple(range(100)), workers=[b.address[0]]) yield e._gather([x, y]) z = e.submit(lambda x, y: None, x, y) yield z._result() assert s.who_has[z.key] == {b.address} yield e._shutdown()
def test_gc(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() x = e.submit(inc, 10) result = yield x._result() assert s.who_has[x.key] x.__del__() yield e._shutdown() assert not s.who_has[x.key]
def test_nbytes(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() [x] = yield e._scatter([1]) assert s.nbytes == {x.key: sizeof(1)} y = e.submit(inc, x) yield y._result() assert s.nbytes == {x.key: sizeof(1), y.key: sizeof(2)} yield e._shutdown()
def test_remote_scatter_gather(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() x, y, z = yield e._scatter([1, 2, 3]) assert x.key in a.data or x.key in b.data assert y.key in a.data or y.key in b.data assert z.key in a.data or z.key in b.data xx, yy, zz = yield e._gather([x, y, z]) assert (xx, yy, zz) == (1, 2, 3) yield e._shutdown()
def test_get(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() result = yield e._get({'x': (inc, 1)}, 'x') assert result == 2 result = yield e._get({'x': (inc, 1)}, ['x']) assert result == [2] result = yield e._get({}, []) assert result == [] yield e._shutdown()
def test_allow_restrictions(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() x = e.submit(inc, 1, workers=a.ip) yield x._result() assert s.who_has[x.key] == {a.address} assert not s.loose_restrictions x = e.submit(inc, 2, workers=a.ip, allow_other_workers=True) yield x._result() assert s.who_has[x.key] == {a.address} assert x.key in s.loose_restrictions L = e.map(inc, range(3, 13), workers=a.ip, allow_other_workers=True) yield _wait(L) assert all(s.who_has[f.key] == {a.address} for f in L) assert {f.key for f in L}.issubset(s.loose_restrictions) """ x = e.submit(inc, 14, workers='127.0.0.3') with ignoring(gen.TimeoutError): yield gen.with_timeout(timedelta(seconds=0.1), x._result()) assert False assert not s.who_has[x.key] assert x.key not in s.loose_restrictions """ x = e.submit(inc, 15, workers='127.0.0.3', allow_other_workers=True) yield x._result() assert s.who_has[x.key] assert x.key in s.loose_restrictions L = e.map(inc, range(15, 25), workers='127.0.0.3', allow_other_workers=True) yield _wait(L) assert all(s.who_has[f.key] for f in L) assert {f.key for f in L}.issubset(s.loose_restrictions) with pytest.raises(ValueError): e.submit(inc, 1, allow_other_workers=True) with pytest.raises(ValueError): e.map(inc, [1], allow_other_workers=True) with pytest.raises(TypeError): e.submit(inc, 20, workers='127.0.0.1', allow_other_workers='Hello!') with pytest.raises(TypeError): e.map(inc, [20], workers='127.0.0.1', allow_other_workers='Hello!')
def test_restrictions_submit(s, a, b): e = Executor((s.ip, s.port), start=False) yield e._start() x = e.submit(inc, 1, workers={a.ip}) y = e.submit(inc, x, workers={b.ip}) yield _wait([x, y]) assert s.restrictions[x.key] == {a.ip} assert x.key in a.data assert s.restrictions[y.key] == {b.ip} assert y.key in b.data yield e._shutdown()