def test_restrictions_map(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = e.map(inc, range(5), workers={a.ip})
yield _wait(L)
assert set(a.data) == {x.key for x in L}
assert not b.data
for x in L:
assert s.restrictions[x.key] == {a.ip}
L = e.map(inc, [10, 11, 12], workers=[{a.ip},
{a.ip, b.ip},
{b.ip}])
yield _wait(L)
assert s.restrictions[L[0].key] == {a.ip}
assert s.restrictions[L[1].key] == {a.ip, b.ip}
assert s.restrictions[L[2].key] == {b.ip}
with pytest.raises(ValueError):
e.map(inc, [10, 11, 12], workers=[{a.ip}])
yield e._shutdown()
def test_restrictions_map(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = e.map(inc, range(5), workers={a.ip})
yield _wait(L)
assert set(a.data) == {x.key for x in L}
assert not b.data
for x in L:
assert s.restrictions[x.key] == {a.ip}
L = e.map(inc, [10, 11, 12], workers=[{a.ip},
{a.ip, b.ip},
{b.ip}])
yield _wait(L)
assert s.restrictions[L[0].key] == {a.ip}
assert s.restrictions[L[1].key] == {a.ip, b.ip}
assert s.restrictions[L[2].key] == {b.ip}
with pytest.raises(ValueError):
e.map(inc, [10, 11, 12], workers=[{a.ip}])
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
L = e.map(inc, range(5), workers={a.ip})
yield _wait(L)
assert set(a.data) == {x.key for x in L}
assert not b.data
for x in L:
assert e.restrictions[x.key] == {a.ip}
L = e.map(inc, [10, 11, 12], workers=[{a.ip},
{a.ip, b.ip},
{b.ip}])
yield _wait(L)
assert e.restrictions[L[0].key] == {a.ip}
assert e.restrictions[L[1].key] == {a.ip, b.ip}
assert e.restrictions[L[2].key] == {b.ip}
with pytest.raises(ValueError):
e.map(inc, [10, 11, 12], workers=[{a.ip}])
yield e._shutdown()
def test_submit_errors():
def f(a, b, c):
pass
e = Executor('127.0.0.1:8787', start=False)
with pytest.raises(TypeError):
e.submit(1, 2, 3)
with pytest.raises(TypeError):
e.map([1, 2, 3])
def test_submit_errors(loop):
def f(a, b, c):
pass
e = Executor('127.0.0.1:8787', start=False, loop=loop)
with pytest.raises(TypeError):
e.submit(1, 2, 3)
with pytest.raises(TypeError):
e.map([1, 2, 3])
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
L = e.map(assert_list, [[1, 2, 3], [4]])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list_kwarg, [[1, 2, 3], [4]], z=[10])
result = yield e._gather(L)
assert all(result)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
L = e.map(assert_list, [[1, 2, 3], [4]])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list_kwarg, [[1, 2, 3], [4]], z=[10])
result = yield e._gather(L)
assert all(result)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
L1 = e.map(inc, range(5))
L2 = e.map(inc, range(5))
assert [x.key for x in L1] == [x.key for x in L2]
L3 = e.map(inc, [1, 1, 1, 1])
assert len({x.event for x in L3}) == 1
L4 = e.map(inc, [1, 1, 1, 1], pure=False)
assert len({x.event for x in L4}) == 4
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
L1 = e.map(inc, range(5))
L2 = e.map(inc, range(5))
assert [x.key for x in L1] == [x.key for x in L2]
L3 = e.map(inc, [1, 1, 1, 1])
assert len({x.event for x in L3}) == 1
L4 = e.map(inc, [1, 1, 1, 1], pure=False)
assert len({x.event for x in L4}) == 4
def test_map_naming(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L1 = e.map(inc, range(5))
L2 = e.map(inc, range(5))
assert [x.key for x in L1] == [x.key for x in L2]
L3 = e.map(inc, [1, 1, 1, 1])
assert len({x.event for x in L3}) == 1
L4 = e.map(inc, [1, 1, 1, 1], pure=False)
assert len({x.event for x in L4}) == 4
yield e._shutdown()
def test_map_on_futures_with_kwargs(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
def f(x, y=10):
return x + y
futures = e.map(inc, range(10))
futures2 = e.map(f, futures, y=20)
results = yield e._gather(futures2)
assert results == [i + 1 + 20 for i in range(10)]
future = e.submit(inc, 100)
future2 = e.submit(f, future, y=200)
result = yield future2._result()
assert result == 100 + 1 + 200
def test_map_naming(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L1 = e.map(inc, range(5))
L2 = e.map(inc, range(5))
assert [x.key for x in L1] == [x.key for x in L2]
L3 = e.map(inc, [1, 1, 1, 1])
assert len({x.event for x in L3}) == 1
L4 = e.map(inc, [1, 1, 1, 1], pure=False)
assert len({x.event for x in L4}) == 4
yield e._shutdown()
def test_map_on_futures_with_kwargs(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
def f(x, y=10):
return x + y
futures = e.map(inc, range(10))
futures2 = e.map(f, futures, y=20)
results = yield e._gather(futures2)
assert results == [i + 1 + 20 for i in range(10)]
future = e.submit(inc, 100)
future2 = e.submit(f, future, y=200)
result = yield future2._result()
assert result == 100 + 1 + 200
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
L = e.map(assert_list, [[1, 2, 3], [4]])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list, [[1, 2, 3], [4]], z=[10])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list, [[1, 2, 3], [4]], [[]] * 3)
result = yield e._gather(L)
assert all(result)
yield e._shutdown()
def test_pragmatic_move_small_data_to_large_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
lists = e.map(lambda n: list(range(n)), [10] * 10, pure=False)
sums = e.map(sum, lists)
total = e.submit(sum, sums)
def f(x, y):
return None
results = e.map(f, lists, [total] * 10)
yield _wait([total])
yield _wait(results)
for l, r in zip(lists, results):
assert s.who_has[l.key] == s.who_has[r.key]
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
lists = e.map(lambda n: list(range(n)), [10] * 10, pure=False)
sums = e.map(sum, lists)
total = e.submit(sum, sums)
def f(x, y):
return None
results = e.map(f, lists, [total] * 10)
yield _wait([total])
yield _wait(results)
for l, r in zip(lists, results):
assert e.scheduler.who_has[l.key] == e.scheduler.who_has[r.key]
yield e._shutdown()
def test_pragmatic_move_small_data_to_large_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
lists = e.map(lambda n: list(range(n)), [10] * 10, pure=False)
sums = e.map(sum, lists)
total = e.submit(sum, sums)
def f(x, y):
return None
results = e.map(f, lists, [total] * 10)
yield _wait([total])
yield _wait(results)
for l, r in zip(lists, results):
assert s.who_has[l.key] == s.who_has[r.key]
yield e._shutdown()
def test_map_quotes(s, a, b):
def assert_list(x, z=[]):
return isinstance(x, list) and isinstance(z, list)
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = e.map(assert_list, [[1, 2, 3], [4]])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list, [[1, 2, 3], [4]], z=[10])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list, [[1, 2, 3], [4]], [[]] * 3)
result = yield e._gather(L)
assert all(result)
yield e._shutdown()
def test_map_quotes(s, a, b):
def assert_list(x, z=[]):
return isinstance(x, list) and isinstance(z, list)
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = e.map(assert_list, [[1, 2, 3], [4]])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list, [[1, 2, 3], [4]], z=[10])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list, [[1, 2, 3], [4]], [[]] * 3)
result = yield e._gather(L)
assert all(result)
yield e._shutdown()
def test_badly_serialized_input(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
o = BadlySerializedObject()
future = e.submit(inc, o)
futures = e.map(inc, range(10))
L = yield e._gather(futures)
assert list(L) == list(map(inc, range(10)))
def test_failed_worker_without_warning(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = e.map(inc, range(10))
yield _wait(L)
ncores1 = s.ncores.copy()
a.process.terminate()
start = time()
while not a.process.is_alive():
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while s.ncores == ncores1:
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while len(s.ncores) < 2:
yield gen.sleep(0.01)
assert time() - start < 5
yield _wait(L)
L2 = e.map(inc, range(10, 20))
yield _wait(L2)
assert all(len(keys) > 0 for keys in s.has_what.values())
ncores2 = s.ncores.copy()
yield e._restart()
L = e.map(inc, range(10))
yield _wait(L)
assert all(len(keys) > 0 for keys in s.has_what.values())
assert not (set(ncores2) & set(s.ncores)) # no overlap
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
L = e.map(inc, range(5), workers={a.ip})
yield _wait(L)
assert set(a.data) == {x.key for x in L}
assert not b.data
for x in L:
assert e.restrictions[x.key] == {a.ip}
L = e.map(inc, [10, 11, 12], workers=[{a.ip}, {a.ip, b.ip}, {b.ip}])
yield _wait(L)
assert e.restrictions[L[0].key] == {a.ip}
assert e.restrictions[L[1].key] == {a.ip, b.ip}
assert e.restrictions[L[2].key] == {b.ip}
with pytest.raises(ValueError):
e.map(inc, [10, 11, 12], workers=[{a.ip}])
yield e._shutdown()
def test_stress_1(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
n = 2**6
seq = e.map(inc, range(n))
while len(seq) > 1:
yield gen.sleep(0.1)
seq = [e.submit(add, seq[i], seq[i + 1])
for i in range(0, len(seq), 2)]
result = yield seq[0]._result()
assert result == sum(map(inc, range(n)))
yield e._shutdown()
def test_stress_1(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
n = 2**6
seq = e.map(inc, range(n))
while len(seq) > 1:
yield gen.sleep(0.1)
seq = [e.submit(add, seq[i], seq[i + 1])
for i in range(0, len(seq), 2)]
result = yield seq[0]._result()
assert result == sum(map(inc, range(n)))
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
n = 2**6
seq = e.map(inc, range(n))
while len(seq) > 1:
yield gen.sleep(0.1)
seq = [e.submit(add, seq[i], seq[i + 1])
for i in range(0, len(seq), 2)]
result = yield seq[0]._result()
assert result == sum(map(inc, range(n)))
yield e._shutdown()
def test_gather_robust_to_missing_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x, y, z = e.map(inc, range(3))
yield _wait([x, y, z]) # everything computed
for q in [x, y]:
if q.key in a.data:
del a.data[q.key]
if q.key in b.data:
del b.data[q.key]
xx, yy, zz = yield e._gather([x, y, z])
assert (xx, yy, zz) == (1, 2, 3)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
x, y, z = e.map(inc, range(3))
yield _wait([x, y, z]) # everything computed
for q in [x, y]:
if q.key in a.data:
del a.data[q.key]
if q.key in b.data:
del b.data[q.key]
xx, yy, zz = yield e._gather([x, y, z])
assert (xx, yy, zz) == (1, 2, 3)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
x, y, z = e.map(inc, range(3))
yield _wait([x, y, z]) # everything computed
for q in [x, y]:
if q.key in a.data:
del a.data[q.key]
if q.key in b.data:
del b.data[q.key]
xx, yy, zz = yield e._gather([x, y, z])
assert (xx, yy, zz) == (1, 2, 3)
yield e._shutdown()
def test_gather_robust_to_missing_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x, y, z = e.map(inc, range(3))
yield _wait([x, y, z]) # everything computed
for q in [x, y]:
if q.key in a.data:
del a.data[q.key]
if q.key in b.data:
del b.data[q.key]
xx, yy, zz = yield e._gather([x, y, z])
assert (xx, yy, zz) == (1, 2, 3)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
n = 2**6
seq = e.map(inc, range(n))
while len(seq) > 1:
yield gen.sleep(0.1)
seq = [
e.submit(add, seq[i], seq[i + 1])
for i in range(0, len(seq), 2)
]
result = yield seq[0]._result()
assert result == sum(map(inc, range(n)))
yield e._shutdown()
def test_allow_restrictions(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 1, workers=a.ip)
yield x._result()
assert s.who_has[x.key] == {a.address}
assert not s.loose_restrictions
x = e.submit(inc, 2, workers=a.ip, allow_other_workers=True)
yield x._result()
assert s.who_has[x.key] == {a.address}
assert x.key in s.loose_restrictions
L = e.map(inc, range(3, 13), workers=a.ip, allow_other_workers=True)
yield _wait(L)
assert all(s.who_has[f.key] == {a.address} for f in L)
assert {f.key for f in L}.issubset(s.loose_restrictions)
"""
x = e.submit(inc, 14, workers='127.0.0.3')
with ignoring(gen.TimeoutError):
yield gen.with_timeout(timedelta(seconds=0.1), x._result())
assert False
assert not s.who_has[x.key]
assert x.key not in s.loose_restrictions
"""
x = e.submit(inc, 15, workers='127.0.0.3', allow_other_workers=True)
yield x._result()
assert s.who_has[x.key]
assert x.key in s.loose_restrictions
L = e.map(inc, range(15, 25), workers='127.0.0.3', allow_other_workers=True)
yield _wait(L)
assert all(s.who_has[f.key] for f in L)
assert {f.key for f in L}.issubset(s.loose_restrictions)
with pytest.raises(ValueError):
e.submit(inc, 1, allow_other_workers=True)
with pytest.raises(ValueError):
e.map(inc, [1], allow_other_workers=True)
with pytest.raises(TypeError):
e.submit(inc, 20, workers='127.0.0.1', allow_other_workers='Hello!')
with pytest.raises(TypeError):
e.map(inc, [20], workers='127.0.0.1', allow_other_workers='Hello!')
def test_allow_restrictions(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 1, workers=a.ip)
yield x._result()
assert s.who_has[x.key] == {a.address}
assert not s.loose_restrictions
x = e.submit(inc, 2, workers=a.ip, allow_other_workers=True)
yield x._result()
assert s.who_has[x.key] == {a.address}
assert x.key in s.loose_restrictions
L = e.map(inc, range(3, 13), workers=a.ip, allow_other_workers=True)
yield _wait(L)
assert all(s.who_has[f.key] == {a.address} for f in L)
assert {f.key for f in L}.issubset(s.loose_restrictions)
"""
x = e.submit(inc, 14, workers='127.0.0.3')
with ignoring(gen.TimeoutError):
yield gen.with_timeout(timedelta(seconds=0.1), x._result())
assert False
assert not s.who_has[x.key]
assert x.key not in s.loose_restrictions
"""
x = e.submit(inc, 15, workers='127.0.0.3', allow_other_workers=True)
yield x._result()
assert s.who_has[x.key]
assert x.key in s.loose_restrictions
L = e.map(inc, range(15, 25), workers='127.0.0.3', allow_other_workers=True)
yield _wait(L)
assert all(s.who_has[f.key] for f in L)
assert {f.key for f in L}.issubset(s.loose_restrictions)
with pytest.raises(ValueError):
e.submit(inc, 1, allow_other_workers=True)
with pytest.raises(ValueError):
e.map(inc, [1], allow_other_workers=True)
with pytest.raises(TypeError):
e.submit(inc, 20, workers='127.0.0.1', allow_other_workers='Hello!')
with pytest.raises(TypeError):
e.map(inc, [20], workers='127.0.0.1', allow_other_workers='Hello!')
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
L1 = e.map(inc, range(5))
assert len(L1) == 5
assert isdistinct(x.key for x in L1)
assert all(isinstance(x, Future) for x in L1)
result = yield L1[0]._result()
assert result == inc(0)
assert len(e.dask) == 5
L2 = e.map(inc, L1)
result = yield L2[1]._result()
assert result == inc(inc(1))
assert len(e.dask) == 10
assert L1[0].key in e.dask[L2[0].key]
total = e.submit(sum, L2)
result = yield total._result()
assert result == sum(map(inc, map(inc, range(5))))
L3 = e.map(add, L1, L2)
result = yield L3[1]._result()
assert result == inc(1) + inc(inc(1))
L4 = e.map(add, range(3), range(4))
results = yield e._gather(L4)
if sys.version_info[0] >= 3:
assert results == list(map(add, range(3), range(4)))
def f(x, y=10):
return x + y
L5 = e.map(f, range(5), y=5)
results = yield e._gather(L5)
assert results == list(range(5, 10))
y = e.submit(f, 10)
L6 = e.map(f, range(5), y=y)
results = yield e._gather(L6)
assert results == list(range(20, 25))
yield e._shutdown()
def test_map(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L1 = e.map(inc, range(5))
assert len(L1) == 5
assert isdistinct(x.key for x in L1)
assert all(isinstance(x, Future) for x in L1)
result = yield L1[0]._result()
assert result == inc(0)
assert len(s.dask) == 5
L2 = e.map(inc, L1)
result = yield L2[1]._result()
assert result == inc(inc(1))
assert len(s.dask) == 10
assert L1[0].key in s.dask[L2[0].key]
total = e.submit(sum, L2)
result = yield total._result()
assert result == sum(map(inc, map(inc, range(5))))
L3 = e.map(add, L1, L2)
result = yield L3[1]._result()
assert result == inc(1) + inc(inc(1))
L4 = e.map(add, range(3), range(4))
results = yield e._gather(L4)
if sys.version_info[0] >= 3:
assert results == list(map(add, range(3), range(4)))
def f(x, y=10):
return x + y
L5 = e.map(f, range(5), y=5)
results = yield e._gather(L5)
assert results == list(range(5, 10))
y = e.submit(f, 10)
L6 = e.map(f, range(5), y=y)
results = yield e._gather(L6)
assert results == list(range(20, 25))
yield e._shutdown()
