def test_futures_to_dask_dataframe(loop): with cluster() as (c, [a, b]): with Executor(('127.0.0.1', c['port']), loop=loop) as e: remote_dfs = e.map(lambda x: x, dfs) ddf = futures_to_dask_dataframe(remote_dfs, divisions=True) assert isinstance(ddf, dd.DataFrame) assert ddf.x.sum().compute(get=e.get) == sum([df.x.sum() for df in dfs]) ddf2 = futures_to_collection(remote_dfs, divisions=True) assert type(ddf) == type(ddf2) assert ddf.dask == ddf2.dask
def test_futures_to_dask_dataframe(loop): with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as c: remote_dfs = c.map(lambda x: x, dfs) ddf = futures_to_dask_dataframe(remote_dfs, divisions=True) assert isinstance(ddf, dd.DataFrame) assert ddf.x.sum().compute(get=c.get) == sum([df.x.sum() for df in dfs]) ddf2 = futures_to_collection(remote_dfs, divisions=True) assert type(ddf) == type(ddf2) assert ddf.dask == ddf2.dask
def test_futures_to_dask_dataframe(loop): with cluster() as (c, [a, b]): with Executor(('127.0.0.1', c['port']), loop=loop) as e: remote_dfs = e.map(lambda x: x, dfs) ddf = futures_to_dask_dataframe(remote_dfs, divisions=True) assert isinstance(ddf, dd.DataFrame) assert ddf.x.sum().compute(get=e.get) == sum( [df.x.sum() for df in dfs]) ddf2 = futures_to_collection(remote_dfs, divisions=True) assert type(ddf) == type(ddf2) assert ddf.dask == ddf2.dask
def test_futures_to_dask_array(loop): with cluster() as (c, [a, b]): with Client(("127.0.0.1", c["port"]), loop=loop) as c: remote_arrays = [[c.submit(np.full, (3, 3), i + j) for i in range(3)] for j in range(3)] x = futures_to_dask_array(remote_arrays, client=c) assert x.chunks == ((3, 3, 3), (3, 3, 3)) assert x.dtype == np.full((), 0).dtype assert x.sum().compute(get=c.get) == 162 assert (x + x.T).sum().compute(get=c.get) == 162 * 2 y = futures_to_collection(remote_arrays, client=c) assert x.dask == y.dask
def test_futures_to_dask_array(loop): with cluster() as (c, [a, b]): with Client(('127.0.0.1', c['port']), loop=loop) as c: remote_arrays = [[ c.submit(np.full, (3, 3), i + j) for i in range(3) ] for j in range(3)] x = futures_to_dask_array(remote_arrays, client=c) assert x.chunks == ((3, 3, 3), (3, 3, 3)) assert x.dtype == np.full((), 0).dtype assert x.sum().compute(get=c.get) == 162 assert (x + x.T).sum().compute(get=c.get) == 162 * 2 y = futures_to_collection(remote_arrays, client=c) assert x.dask == y.dask
def test_futures_to_dask_array(loop): with cluster() as (c, [a, b]): with Executor(('127.0.0.1', c['port']), loop=loop) as e: remote_arrays = [[e.submit(np.full, (3, 3), i + j) for i in range(3)] for j in range(3)] x = futures_to_dask_array(remote_arrays, executor=e) assert x.chunks == ((3, 3, 3), (3, 3, 3)) assert x.dtype == np.full((), 0).dtype assert x.sum().compute(get=e.get) == 162 assert (x + x.T).sum().compute(get=e.get) == 162 * 2 y = futures_to_collection(remote_arrays, executor=e) assert x.dask == y.dask