def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
x_dsk = {('x', i, j): np.random.random((3, 3)) for i in range(3)
for j in range(2)}
y_dsk = {('y', i, j): np.random.random((3, 3)) for i in range(2)
for j in range(3)}
x_futures = yield e._scatter(x_dsk)
y_futures = yield e._scatter(y_dsk)
dt = np.random.random(0).dtype
x_local = da.Array(x_dsk, 'x', ((3, 3, 3), (3, 3)), dt)
y_local = da.Array(y_dsk, 'y', ((3, 3), (3, 3, 3)), dt)
x_remote = da.Array(x_futures, 'x', ((3, 3, 3), (3, 3)), dt)
y_remote = da.Array(y_futures, 'y', ((3, 3), (3, 3, 3)), dt)
exprs = [lambda x, y: x.T + y,
lambda x, y: x.mean() + y.mean(),
lambda x, y: x.dot(y).std(axis=0),
lambda x, y: x - x.mean(axis=1)[:, None]]
for expr in exprs:
local = expr(x_local, y_local)
local_results = dask.get(local.dask, local._keys())
local_result = da.Array._finalize(local, local_results)
remote = expr(x_remote, y_remote)
remote_results = yield e._get(remote.dask, remote._keys())
remote_result = da.Array._finalize(remote, remote_results)
assert np.all(local_result == remote_result)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
remote_dfs = e.map(lambda x: x, dfs)
ddf = yield _futures_to_dask_dataframe(e, remote_dfs, divisions=True)
assert isinstance(ddf, dd.DataFrame)
assert ddf.divisions == (0, 30, 60, 80)
expr = ddf.x.sum()
result = yield e._get(expr.dask, expr._keys())
assert result == [sum([df.x.sum() for df in dfs])]
yield e._shutdown()
def test__futures_to_dask_dataframe(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
remote_dfs = e.map(identity, dfs)
ddf = yield _futures_to_dask_dataframe(remote_dfs, divisions=True,
executor=e)
assert isinstance(ddf, dd.DataFrame)
assert ddf.divisions == (0, 30, 60, 80)
expr = ddf.x.sum()
result = yield e._get(expr.dask, expr._keys())
assert result == [sum([df.x.sum() for df in dfs])]
yield e._shutdown()
def test__futures_to_dask_dataframe(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
remote_dfs = e.map(identity, dfs)
ddf = yield _futures_to_dask_dataframe(remote_dfs,
divisions=True,
executor=e)
assert isinstance(ddf, dd.DataFrame)
assert ddf.divisions == (0, 30, 60, 80)
expr = ddf.x.sum()
result = yield e._get(expr.dask, expr._keys())
assert result == [sum([df.x.sum() for df in dfs])]
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
arrays = e.map(np.ones, [(5, 5)] * 6)
y = yield _stack(arrays, axis=0)
assert y.shape == (6, 5, 5)
assert y.chunks == ((1, 1, 1, 1, 1, 1), (5,), (5,))
y_results = yield e._get(y.dask, y._keys())
yy = da.Array._finalize(y, y_results)
assert isinstance(yy, np.ndarray)
assert yy.shape == y.shape
assert (yy == 1).all()
yield e._shutdown()
def test__futures_to_dask_array(s, a, b):
import dask.array as da
e = Executor((s.ip, s.port), start=False)
yield e._start()
remote_arrays = [[[
e.submit(np.full, (2, 3, 4), i + j + k) for i in range(2)
] for j in range(2)] for k in range(4)]
x = yield _futures_to_dask_array(remote_arrays, executor=e)
assert x.chunks == ((2, 2, 2, 2), (3, 3), (4, 4))
assert x.dtype == np.full((), 0).dtype
assert isinstance(x, da.Array)
expr = x.sum()
result = yield e._get(expr.dask, expr._keys())
assert isinstance(result[0], np.number)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
remote_arrays = [[[e.submit(np.full, (2, 3, 4), i + j + k)
for i in range(2)]
for j in range(2)]
for k in range(4)]
x = yield _futures_to_dask_array(remote_arrays, executor=e)
assert x.chunks == ((2, 2, 2, 2), (3, 3), (4, 4))
assert x.dtype == np.full((), 0).dtype
assert isinstance(x, da.Array)
expr = x.sum()
result = yield e._get(expr.dask, expr._keys())
assert isinstance(result[0], np.number)
yield e._shutdown()
