Python _futures_to_dask_array 예제들

예제 #1

파일: test_collections.py 프로젝트: canavandl/distributed

def test__futures_to_collection(s, a, b):
    e = Executor((s.ip, s.port), start=False)
    yield e._start()

    remote_dfs = e.map(identity, dfs)
    ddf = yield _futures_to_collection(remote_dfs, divisions=True)
    ddf2 = yield _futures_to_dask_dataframe(remote_dfs, divisions=True)
    assert isinstance(ddf, dd.DataFrame)

    assert ddf.dask == ddf2.dask

    remote_arrays = e.map(np.arange, range(3, 5))
    x = yield _futures_to_collection(remote_arrays)
    y = yield _futures_to_dask_array(remote_arrays)

    assert type(x) == type(y)
    assert x.dask == y.dask

    remote_lists = yield e._scatter([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    b = yield _futures_to_collection(remote_lists)
    c = yield _futures_to_dask_bag(remote_lists)

    assert type(b) == type(c)
    assert b.dask == b.dask

    yield e._shutdown()

예제 #2

파일: test_collections.py 프로젝트: ogrisel/distributed

def test__futures_to_collection(s, a, b):
    e = Executor((s.ip, s.port), start=False)
    yield e._start()

    remote_dfs = e.map(identity, dfs)
    ddf = yield _futures_to_collection(remote_dfs, divisions=True)
    ddf2 = yield _futures_to_dask_dataframe(remote_dfs, divisions=True)
    assert isinstance(ddf, dd.DataFrame)

    assert ddf.dask == ddf2.dask

    remote_arrays = e.map(np.arange, range(3, 5))
    x = yield _futures_to_collection(remote_arrays)
    y = yield _futures_to_dask_array(remote_arrays)

    assert type(x) == type(y)
    assert x.dask == y.dask

    remote_lists = yield e._scatter([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    b = yield _futures_to_collection(remote_lists)
    c = yield _futures_to_dask_bag(remote_lists)

    assert type(b) == type(c)
    assert b.dask == b.dask

    yield e._shutdown()

예제 #3

파일: test_collections.py 프로젝트: dask/distributed

def test__futures_to_dask_array(c, s, a, b):
    import dask.array as da

    remote_arrays = [[[c.submit(np.full, (2, 3, 4), i + j + k) for i in range(2)] for j in range(2)] for k in range(4)]

    x = yield _futures_to_dask_array(remote_arrays, client=c)
    assert x.chunks == ((2, 2, 2, 2), (3, 3), (4, 4))
    assert x.dtype == np.full((), 0).dtype

    assert isinstance(x, da.Array)
    expr = x.sum()
    result = yield c._get(expr.dask, expr._keys())
    assert isinstance(result[0], np.number)

예제 #4

파일: test_collections.py 프로젝트: zegami/distributed

def test__futures_to_dask_array(c, s, a, b):
    import dask.array as da
    remote_arrays = [[[
        c.submit(np.full, (2, 3, 4), i + j + k) for i in range(2)
    ] for j in range(2)] for k in range(4)]

    x = yield _futures_to_dask_array(remote_arrays, client=c)
    assert x.chunks == ((2, 2, 2, 2), (3, 3), (4, 4))
    assert x.dtype == np.full((), 0).dtype

    assert isinstance(x, da.Array)
    expr = x.sum()
    result = yield c._get(expr.dask, expr._keys())
    assert isinstance(result[0], np.number)

예제 #5

def test__futures_to_dask_array(s, a, b):
    import dask.array as da
    e = Executor((s.ip, s.port), start=False)
    yield e._start()

    remote_arrays = [[[
        e.submit(np.full, (2, 3, 4), i + j + k) for i in range(2)
    ] for j in range(2)] for k in range(4)]

    x = yield _futures_to_dask_array(remote_arrays, executor=e)
    assert x.chunks == ((2, 2, 2, 2), (3, 3), (4, 4))
    assert x.dtype == np.full((), 0).dtype

    assert isinstance(x, da.Array)
    expr = x.sum()
    result = yield e._get(expr.dask, expr._keys())
    assert isinstance(result[0], np.number)

    yield e._shutdown()

예제 #6

파일: test_collections.py 프로젝트: freeman-lab/distributed

    def f(c, a, b):
        e = Executor((c.ip, c.port), start=False, loop=loop)
        yield e._start()

        remote_arrays = [[[e.submit(np.full, (2, 3, 4), i + j + k)
                            for i in range(2)]
                            for j in range(2)]
                            for k in range(4)]

        x = yield _futures_to_dask_array(remote_arrays, executor=e)
        assert x.chunks == ((2, 2, 2, 2), (3, 3), (4, 4))
        assert x.dtype == np.full((), 0).dtype

        assert isinstance(x, da.Array)
        expr = x.sum()
        result = yield e._get(expr.dask, expr._keys())
        assert isinstance(result[0], np.number)

        yield e._shutdown()

예제 #7

파일: test_collections.py 프로젝트: zegami/distributed

def test__futures_to_collection(c, s, a, b):
    remote_dfs = c.map(identity, dfs)
    ddf = yield _futures_to_collection(remote_dfs, divisions=True)
    ddf2 = yield _futures_to_dask_dataframe(remote_dfs, divisions=True)
    assert isinstance(ddf, dd.DataFrame)

    assert ddf.dask == ddf2.dask

    remote_arrays = c.map(np.arange, range(3, 5))
    x = yield _futures_to_collection(remote_arrays)
    y = yield _futures_to_dask_array(remote_arrays)

    assert type(x) == type(y)
    assert x.dask == y.dask

    remote_lists = yield c._scatter([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    b = yield _futures_to_collection(remote_lists)
    c = yield _futures_to_dask_bag(remote_lists)

    assert type(b) == type(c)
    assert b.dask == b.dask

예제 #8

파일: test_collections.py 프로젝트: amosonn/distributed

def test__futures_to_collection(c, s, a, b):
    remote_dfs = c.map(identity, dfs)
    ddf = yield _futures_to_collection(remote_dfs, divisions=True)
    ddf2 = yield _futures_to_dask_dataframe(remote_dfs, divisions=True)
    assert isinstance(ddf, dd.DataFrame)

    assert ddf.dask == ddf2.dask

    remote_arrays = c.map(np.arange, range(3, 5))
    x = yield _futures_to_collection(remote_arrays)
    y = yield _futures_to_dask_array(remote_arrays)

    assert type(x) == type(y)
    assert x.dask == y.dask

    remote_lists = yield c._scatter([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    b = yield _futures_to_collection(remote_lists)
    c = yield _futures_to_dask_bag(remote_lists)

    assert type(b) == type(c)
    assert b.dask == b.dask