예제 #1
0
def test_multiple_executors_restart(s, a, b):
    e1 = Executor((s.ip, s.port), start=False)
    yield e1._start()
    e2 = Executor((s.ip, s.port), start=False)
    yield e2._start()

    x = e1.submit(inc, 1)
    y = e2.submit(inc, 2)
    xx = yield x._result()
    yy = yield y._result()
    assert xx == 2
    assert yy == 3

    yield e1._restart()

    assert x.cancelled()
    assert y.cancelled()

    yield e1._shutdown(fast=True)
    yield e2._shutdown(fast=True)
def test_multiple_executors_restart(s, a, b):
    e1 = Executor((s.ip, s.port), start=False)
    yield e1._start()
    e2 = Executor((s.ip, s.port), start=False)
    yield e2._start()

    x = e1.submit(inc, 1)
    y = e2.submit(inc, 2)
    xx = yield x._result()
    yy = yield y._result()
    assert xx == 2
    assert yy == 3

    yield e1._restart()

    assert x.cancelled()
    assert y.cancelled()

    yield e1._shutdown(fast=True)
    yield e2._shutdown(fast=True)
    def test_framework_runs(self):
        with MesosCluster() as cluster:
            time.sleep(2)
            driver = DistributedDriver().create_driver(DistributedScheduler)
            driver.start()
            time.sleep(5)

            expect(cluster).to(have_activated_slaves(1))
            expect(cluster).to(have_framework_name('distributed-framework'))

            # distributed test - this probably doesnt belong here
            executor = Executor('127.0.0.1:8787')
            A = executor.map(lambda x: x**2, range(10))
            B = executor.map(lambda x: -x, A)
            total = executor.submit(sum, B)
            expect(total.result()).to(equal(-285))
            driver.stop()
예제 #4
0
def test__futures_to_dask_array(s, a, b):
    import dask.array as da
    e = Executor((s.ip, s.port), start=False)
    yield e._start()

    remote_arrays = [[[
        e.submit(np.full, (2, 3, 4), i + j + k) for i in range(2)
    ] for j in range(2)] for k in range(4)]

    x = yield _futures_to_dask_array(remote_arrays, executor=e)
    assert x.chunks == ((2, 2, 2, 2), (3, 3), (4, 4))
    assert x.dtype == np.full((), 0).dtype

    assert isinstance(x, da.Array)
    expr = x.sum()
    result = yield e._get(expr.dask, expr._keys())
    assert isinstance(result[0], np.number)

    yield e._shutdown()
예제 #5
0
    def f(c, a, b):
        e = Executor((c.ip, c.port), start=False, loop=loop)
        yield e._start()

        remote_arrays = [[[e.submit(np.full, (2, 3, 4), i + j + k)
                            for i in range(2)]
                            for j in range(2)]
                            for k in range(4)]

        x = yield _futures_to_dask_array(remote_arrays, executor=e)
        assert x.chunks == ((2, 2, 2, 2), (3, 3), (4, 4))
        assert x.dtype == np.full((), 0).dtype

        assert isinstance(x, da.Array)
        expr = x.sum()
        result = yield e._get(expr.dask, expr._keys())
        assert isinstance(result[0], np.number)

        yield e._shutdown()
예제 #6
0
    from conf_analysis.meg import artifacts, preprocessing
    from conf_analysis.behavior import empirical, metadata, keymap
    import mne, locale
    import numpy as np
    import pickle
    locale.setlocale(locale.LC_ALL, "en_US")
    result = {}
    raw = mne.io.read_raw_ctf(filename, system_clock='ignore')
    trials = preprocessing.blocks(raw, full_file_cache=True)
    trl, bl = trials['trial'], trials['block']
    bcnt = 0
    for b in np.unique(bl):
        if len(trl[bl == b]) >= 75:
            result[b] = bcnt
            bcnt += 1
        print((b, bcnt))
    return result


block_map = {}
for snum in range(1, 16):
    filenames = [metadata.get_raw_filename(snum, b) for b in range(4)]
    block_map[snum] = {}
    for session, filename in enumerate(filenames):
        block_map[snum][session] = executor.submit(do_one, filename)

diagnostics.progress(block_map)
block_map = executor.gather(block_map)

pickle.dump(block_map, open('blockmap.pickle', 'w'))
예제 #7
0
#!/home/vagrant/miniconda3/bin/python

import os
from distributed import Executor


def count_words(filename):
    with open(filename, "r", encoding="latin-1") as f:
        data = f.read()

    return len(data.split(" "))


def remote_word_count():
    root_dir = "/enron/lay-k"

    word_count = 0
    for directory, subdir, filenames in os.walk(root_dir):
        for filename in filenames:
            word_count += count_words(os.path.join(directory, filename))

    return word_count


executor = Executor("127.0.0.1:8786")

words = executor.submit(remote_word_count)

print("\n\n Word Count on remote machine is: ", words.result())
예제 #8
0
#!/home/vagrant/miniconda3/bin/python

import os
from distributed import Executor

def count_words(filename):
    with open(filename, "r", encoding = "latin-1") as f:
        data = f.read()

    return len(data.split(" "))


def remote_word_count():
    root_dir = "/enron/lay-k"

    word_count = 0
    for directory, subdir, filenames in os.walk(root_dir):
        for filename in filenames:
            word_count += count_words(os.path.join(directory, filename))

    return word_count


executor = Executor("127.0.0.1:8786")

words = executor.submit(remote_word_count)

print("\n\n Word Count on remote machine is: ", words.result())