def test_futures_to_dask_dataframe(loop):
with cluster() as (c, [a, b]):
with Executor(('127.0.0.1', c['port']), loop=loop) as e:
remote_dfs = e.map(lambda x: x, dfs)
ddf = futures_to_dask_dataframe(remote_dfs, divisions=True)
assert isinstance(ddf, dd.DataFrame)
assert ddf.x.sum().compute(get=e.get) == sum([df.x.sum() for df in dfs])
ddf2 = futures_to_collection(remote_dfs, divisions=True)
assert type(ddf) == type(ddf2)
assert ddf.dask == ddf2.dask
def test_futures_to_dask_dataframe(loop):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
remote_dfs = c.map(lambda x: x, dfs)
ddf = futures_to_dask_dataframe(remote_dfs, divisions=True)
assert isinstance(ddf, dd.DataFrame)
assert ddf.x.sum().compute(get=c.get) == sum([df.x.sum() for df in dfs])
ddf2 = futures_to_collection(remote_dfs, divisions=True)
assert type(ddf) == type(ddf2)
assert ddf.dask == ddf2.dask
def test_futures_to_dask_dataframe(loop):
with cluster() as (c, [a, b]):
with Executor(('127.0.0.1', c['port']), loop=loop) as e:
remote_dfs = e.map(lambda x: x, dfs)
ddf = futures_to_dask_dataframe(remote_dfs, divisions=True)
assert isinstance(ddf, dd.DataFrame)
assert ddf.x.sum().compute(get=e.get) == sum(
[df.x.sum() for df in dfs])
ddf2 = futures_to_collection(remote_dfs, divisions=True)
assert type(ddf) == type(ddf2)
assert ddf.dask == ddf2.dask
def test_futures_to_dask_array(loop):
with cluster() as (c, [a, b]):
with Client(("127.0.0.1", c["port"]), loop=loop) as c:
remote_arrays = [[c.submit(np.full, (3, 3), i + j) for i in range(3)] for j in range(3)]
x = futures_to_dask_array(remote_arrays, client=c)
assert x.chunks == ((3, 3, 3), (3, 3, 3))
assert x.dtype == np.full((), 0).dtype
assert x.sum().compute(get=c.get) == 162
assert (x + x.T).sum().compute(get=c.get) == 162 * 2
y = futures_to_collection(remote_arrays, client=c)
assert x.dask == y.dask
def test_futures_to_dask_array(loop):
with cluster() as (c, [a, b]):
with Client(('127.0.0.1', c['port']), loop=loop) as c:
remote_arrays = [[
c.submit(np.full, (3, 3), i + j) for i in range(3)
] for j in range(3)]
x = futures_to_dask_array(remote_arrays, client=c)
assert x.chunks == ((3, 3, 3), (3, 3, 3))
assert x.dtype == np.full((), 0).dtype
assert x.sum().compute(get=c.get) == 162
assert (x + x.T).sum().compute(get=c.get) == 162 * 2
y = futures_to_collection(remote_arrays, client=c)
assert x.dask == y.dask
def test_futures_to_dask_array(loop):
with cluster() as (c, [a, b]):
with Executor(('127.0.0.1', c['port']), loop=loop) as e:
remote_arrays = [[e.submit(np.full, (3, 3), i + j)
for i in range(3)]
for j in range(3)]
x = futures_to_dask_array(remote_arrays, executor=e)
assert x.chunks == ((3, 3, 3), (3, 3, 3))
assert x.dtype == np.full((), 0).dtype
assert x.sum().compute(get=e.get) == 162
assert (x + x.T).sum().compute(get=e.get) == 162 * 2
y = futures_to_collection(remote_arrays, executor=e)
assert x.dask == y.dask
