def test__futures_to_collection(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
remote_dfs = e.map(identity, dfs)
ddf = yield _futures_to_collection(remote_dfs, divisions=True)
ddf2 = yield _futures_to_dask_dataframe(remote_dfs, divisions=True)
assert isinstance(ddf, dd.DataFrame)
assert ddf.dask == ddf2.dask
remote_arrays = e.map(np.arange, range(3, 5))
x = yield _futures_to_collection(remote_arrays)
y = yield _futures_to_dask_array(remote_arrays)
assert type(x) == type(y)
assert x.dask == y.dask
remote_lists = yield e._scatter([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
b = yield _futures_to_collection(remote_lists)
c = yield _futures_to_dask_bag(remote_lists)
assert type(b) == type(c)
assert b.dask == b.dask
yield e._shutdown()
def test__futures_to_collection(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
remote_dfs = e.map(identity, dfs)
ddf = yield _futures_to_collection(remote_dfs, divisions=True)
ddf2 = yield _futures_to_dask_dataframe(remote_dfs, divisions=True)
assert isinstance(ddf, dd.DataFrame)
assert ddf.dask == ddf2.dask
remote_arrays = e.map(np.arange, range(3, 5))
x = yield _futures_to_collection(remote_arrays)
y = yield _futures_to_dask_array(remote_arrays)
assert type(x) == type(y)
assert x.dask == y.dask
remote_lists = yield e._scatter([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
b = yield _futures_to_collection(remote_lists)
c = yield _futures_to_dask_bag(remote_lists)
assert type(b) == type(c)
assert b.dask == b.dask
yield e._shutdown()
def test__futures_to_dask_array(c, s, a, b):
import dask.array as da
remote_arrays = [[[c.submit(np.full, (2, 3, 4), i + j + k) for i in range(2)] for j in range(2)] for k in range(4)]
x = yield _futures_to_dask_array(remote_arrays, client=c)
assert x.chunks == ((2, 2, 2, 2), (3, 3), (4, 4))
assert x.dtype == np.full((), 0).dtype
assert isinstance(x, da.Array)
expr = x.sum()
result = yield c._get(expr.dask, expr._keys())
assert isinstance(result[0], np.number)
def test__futures_to_dask_array(c, s, a, b):
import dask.array as da
remote_arrays = [[[
c.submit(np.full, (2, 3, 4), i + j + k) for i in range(2)
] for j in range(2)] for k in range(4)]
x = yield _futures_to_dask_array(remote_arrays, client=c)
assert x.chunks == ((2, 2, 2, 2), (3, 3), (4, 4))
assert x.dtype == np.full((), 0).dtype
assert isinstance(x, da.Array)
expr = x.sum()
result = yield c._get(expr.dask, expr._keys())
assert isinstance(result[0], np.number)
def test__futures_to_dask_array(s, a, b):
import dask.array as da
e = Executor((s.ip, s.port), start=False)
yield e._start()
remote_arrays = [[[
e.submit(np.full, (2, 3, 4), i + j + k) for i in range(2)
] for j in range(2)] for k in range(4)]
x = yield _futures_to_dask_array(remote_arrays, executor=e)
assert x.chunks == ((2, 2, 2, 2), (3, 3), (4, 4))
assert x.dtype == np.full((), 0).dtype
assert isinstance(x, da.Array)
expr = x.sum()
result = yield e._get(expr.dask, expr._keys())
assert isinstance(result[0], np.number)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
remote_arrays = [[[e.submit(np.full, (2, 3, 4), i + j + k)
for i in range(2)]
for j in range(2)]
for k in range(4)]
x = yield _futures_to_dask_array(remote_arrays, executor=e)
assert x.chunks == ((2, 2, 2, 2), (3, 3), (4, 4))
assert x.dtype == np.full((), 0).dtype
assert isinstance(x, da.Array)
expr = x.sum()
result = yield e._get(expr.dask, expr._keys())
assert isinstance(result[0], np.number)
yield e._shutdown()
def test__futures_to_collection(c, s, a, b):
remote_dfs = c.map(identity, dfs)
ddf = yield _futures_to_collection(remote_dfs, divisions=True)
ddf2 = yield _futures_to_dask_dataframe(remote_dfs, divisions=True)
assert isinstance(ddf, dd.DataFrame)
assert ddf.dask == ddf2.dask
remote_arrays = c.map(np.arange, range(3, 5))
x = yield _futures_to_collection(remote_arrays)
y = yield _futures_to_dask_array(remote_arrays)
assert type(x) == type(y)
assert x.dask == y.dask
remote_lists = yield c._scatter([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
b = yield _futures_to_collection(remote_lists)
c = yield _futures_to_dask_bag(remote_lists)
assert type(b) == type(c)
assert b.dask == b.dask
def test__futures_to_collection(c, s, a, b):
remote_dfs = c.map(identity, dfs)
ddf = yield _futures_to_collection(remote_dfs, divisions=True)
ddf2 = yield _futures_to_dask_dataframe(remote_dfs, divisions=True)
assert isinstance(ddf, dd.DataFrame)
assert ddf.dask == ddf2.dask
remote_arrays = c.map(np.arange, range(3, 5))
x = yield _futures_to_collection(remote_arrays)
y = yield _futures_to_dask_array(remote_arrays)
assert type(x) == type(y)
assert x.dask == y.dask
remote_lists = yield c._scatter([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
b = yield _futures_to_collection(remote_lists)
c = yield _futures_to_dask_bag(remote_lists)
assert type(b) == type(c)
assert b.dask == b.dask
