def test_queue(c, s, a, b):
    x = yield Queue('x')
    y = yield Queue('y')
    xx = yield Queue('x')
    assert x.client is c

    future = c.submit(inc, 1)

    yield x.put(future)
    yield y.put(future)
    future2 = yield xx.get()
    assert future.key == future2.key

    with pytest.raises(gen.TimeoutError):
        yield x.get(timeout=0.1)

    del future, future2

    yield gen.sleep(0.1)
    assert s.task_state  # future still present in y's queue
    yield y.get()  # burn future

    start = time()
    while s.task_state:
        yield gen.sleep(0.01)
        assert time() < start + 5
def test_Future_knows_status_immediately(c, s, a, b):
    x = yield c.scatter(123)
    q = yield Queue('q')
    yield q.put(x)

    c2 = yield Client(s.address, asynchronous=True)
    q2 = yield Queue('q', client=c2)
    future = yield q2.get()
    assert future.status == 'finished'

    x = c.submit(div, 1, 0)
    yield wait(x)
    yield q.put(x)

    future2 = yield q2.get()
    assert future2.status == 'error'
    with pytest.raises(Exception):
        yield future2

    start = time()
    while True:  # we learn about the true error eventually
        try:
            yield future2
        except ZeroDivisionError:
            break
        except Exception:
            assert time() < start + 5
            yield gen.sleep(0.05)

    yield c2.close()
Esempio n. 3
0
def test_erred_future(c, s, a, b):
    future = c.submit(div, 1, 0)
    q = Queue()
    yield q.put(future)
    yield gen.sleep(0.1)
    future2 = yield q.get()
    with pytest.raises(ZeroDivisionError):
        yield future2.result()
Esempio n. 4
0
class DistStatusReporter(object):
    """Report status through the training scheduler.

    Example
    -------
    >>> @autogluon_method
    >>> def train_func(config, reporter):
    ...     reporter(accuracy=0.1)
    """
    def __init__(self, remote=None):
        self._queue = Queue(client=remote)
        self._stop = Variable(client=remote)
        self._stop.set(False)
        self._continue_semaphore = DistSemaphore(0, remote)

    def __call__(self, **kwargs):
        """Report updated training status.
        Pass in `done=True` when the training job is completed.

        Args:
            kwargs: Latest training result status.

        Example
        _______
        >>> reporter(accuracy=1, training_iters=4)
        """
        logger.debug('Reporting {}'.format(json.dumps(kwargs)))
        try:
            self._queue.put(kwargs.copy())
        except RuntimeError:
            return
        self._continue_semaphore.acquire()
        if self._stop.get():
            raise AutoGluonEarlyStop('Stopping!')

    def fetch(self, block=True):
        try:
            kwargs = self._queue.get()
        except CommClosedError:
            return {}
        return kwargs

    def terminate(self):
        self._stop.set(True)
        self._continue_semaphore.release()

    def move_on(self):
        self._continue_semaphore.release()

    def save_dict(self, **state_dict):
        raise NotImplementedError

    def get_dict(self):
        raise NotImplementedError

    def __repr__(self):
        reprstr = self.__class__.__name__
        return reprstr
Esempio n. 5
0
def test_picklability(c, s, a, b):
    q = Queue()

    def f(x):
        q.put(x + 1)

    yield c.submit(f, 10)
    result = yield q.get()
    assert result == 11
def test_picklability(c, s, a, b):
    q = Queue()

    def f(x):
        q.put(x + 1)

    yield c.submit(f, 10)
    result = yield q.get()
    assert result == 11
Esempio n. 7
0
def test_picklability_sync(client):
    q = Queue()

    def f(x):
        q.put(x + 1)

    client.submit(f, 10).result()

    assert q.get() == 11
Esempio n. 8
0
def test_picklability_sync(client):
    q = Queue()

    def f(x):
        q.put(x + 1)

    client.submit(f, 10).result()

    assert q.get() == 11
Esempio n. 9
0
def test_close(c, s, a, b):
    q = Queue()

    while q.name not in s.extensions['queues'].queues:
        yield gen.sleep(0.01)

    q.close()
    q.close()

    while q.name in s.extensions['queues'].queues:
        yield gen.sleep(0.01)
Esempio n. 10
0
def test_erred_future(c, s, a, b):
    future = c.submit(div, 1, 0)
    q = Queue()
    yield q.put(future)
    yield gen.sleep(0.1)
    future2 = yield q.get()
    with pytest.raises(ZeroDivisionError):
        yield future2.result()

    exc = yield future2.exception()
    assert isinstance(exc, ZeroDivisionError)
Esempio n. 11
0
 def f(i):
     with worker_client() as c:
         q = Queue('x', client=c)
         for _ in range(100):
             future = q.get()
             x = future.result()
             y = c.submit(inc, x)
             q.put(y)
             sleep(0.01)
         result = q.get().result()
         return result
Esempio n. 12
0
def test_close(c, s, a, b):
    q = Queue()

    while q.name not in s.extensions['queues'].queues:
        yield gen.sleep(0.01)

    q.close()
    q.close()

    while q.name in s.extensions['queues'].queues:
        yield gen.sleep(0.01)
def test_picklability_sync(loop):
    with cluster() as (s, [a, b]):
        with Client(s['address']) as c:
            q = Queue()

            def f(x):
                q.put(x + 1)

            c.submit(f, 10).result()

            assert q.get() == 11
Esempio n. 14
0
def test_queue_with_data(c, s, a, b):
    x = yield Queue("x")
    xx = yield Queue("x")
    assert x.client is c

    yield x.put((1, "hello"))
    data = yield xx.get()

    assert data == (1, "hello")

    with pytest.raises(gen.TimeoutError):
        yield x.get(timeout=0.1)
def test_queue_with_data(c, s, a, b):
    x = yield Queue('x')
    xx = yield Queue('x')
    assert x.client is c

    yield x.put([1, 'hello'])
    data = yield xx.get()

    assert data == [1, 'hello']

    with pytest.raises(gen.TimeoutError):
        yield x.get(timeout=0.1)
Esempio n. 16
0
def test_sync(client):
    future = client.submit(lambda x: x + 1, 10)
    x = Queue("x")
    xx = Queue("x")
    x.put(future)
    assert x.qsize() == 1
    assert xx.qsize() == 1
    future2 = xx.get()

    assert future2.result() == 11
Esempio n. 17
0
def test_Queue(c, s, a, b):
    assert s.address.startswith("tls://")

    x = yield Queue("x")
    y = yield Queue("y")

    size = yield x.qsize()
    assert size == 0

    future = c.submit(inc, 1)

    yield x.put(future)

    future2 = yield x.get()
    assert future.key == future2.key
Esempio n. 18
0
def test_Queue(c, s, a, b):
    assert s.address.startswith('tls://')

    x = Queue('x')
    y = Queue('y')

    size = yield x.qsize()
    assert size == 0

    future = c.submit(inc, 1)

    yield x.put(future)

    future2 = yield x.get()
    assert future.key == future2.key
def test_sync(loop):
    with cluster() as (s, [a, b]):
        with Client(s['address']) as c:
            future = c.submit(lambda x: x + 1, 10)
            x = Queue('x')
            xx = Queue('x')
            x.put(future)
            assert x.qsize() == 1
            assert xx.qsize() == 1
            future2 = xx.get()

            assert future2.result() == 11
Esempio n. 20
0
def test_timeout(c, s, a, b):
    q = Queue('v', maxsize=1)

    start = time()
    with pytest.raises(gen.TimeoutError):
        yield q.get(timeout=0.3)
    stop = time()
    assert 0.2 < stop - start < 2.0

    yield q.put(1)

    start = time()
    with pytest.raises(gen.TimeoutError):
        yield q.put(2, timeout=0.3)
    stop = time()
    assert 0.1 < stop - start < 2.0
Esempio n. 21
0
def test_timeout(c, s, a, b):
    q = Queue('v', maxsize=1)

    start = time()
    with pytest.raises(gen.TimeoutError):
        yield q.get(timeout=0.3)
    stop = time()
    assert 0.2 < stop - start < 2.0

    yield q.put(1)

    start = time()
    with pytest.raises(gen.TimeoutError):
        yield q.put(2, timeout=0.3)
    stop = time()
    assert 0.1 < stop - start < 2.0
Esempio n. 22
0
def test_queue_in_task(loop):
    # Ensure that we can create a Queue inside a task on a
    # worker in a separate Python process than the client
    with popen(["dask-scheduler", "--no-dashboard"]):
        with popen(["dask-worker", "127.0.0.1:8786"]):
            with Client("tcp://127.0.0.1:8786", loop=loop) as c:
                c.wait_for_workers(1)

                x = Queue("x")
                x.put(123)

                def foo():
                    y = Queue("x")
                    return y.get()

                result = c.submit(foo).result()
                assert result == 123
def test_hold_futures(s, a, b):
    c1 = yield Client(s.address, asynchronous=True)
    future = c1.submit(lambda x: x + 1, 10)
    q1 = yield Queue('q')
    yield q1.put(future)
    del q1
    yield c1.close()

    yield gen.sleep(0.1)

    c2 = yield Client(s.address, asynchronous=True)
    q2 = yield Queue('q')
    future2 = yield q2.get()
    result = yield future2

    assert result == 11
    yield c2.close()
Esempio n. 24
0
def test_close(c, s, a, b):
    q = yield Queue()

    q.close()
    q.close()

    while q.name in s.extensions["queues"].queues:
        yield gen.sleep(0.01)
Esempio n. 25
0
def start_and_attach_server(spec,
                            job_name=None,
                            task_index=None,
                            dask_worker=None):
    server = tf.train.Server(spec, job_name=job_name, task_index=task_index)
    dask_worker.tensorflow_server = server
    dask_worker.tensorflow_queue = Queue()
    return 'OK'
class DistSemaphore(object):
    def __init__(self, value, remote=None):
        self._queue = Queue(client=remote)
        for i in range(value):
            self._queue.put(1)

    def acquire(self):
        try:
            _ = self._queue.get()
        except distributed.comm.core.CommClosedError:
            pass

    def release(self):
        self._queue.put(1)

    def __repr__(self):
        reprstr = self.__class__.__name__
        return reprstr
Esempio n. 27
0
async def test_picklability(c, s, a, b):
    q = Queue()

    def f(x):
        q.put(x + 1)

    await c.submit(f, 10)
    result = await q.get()
    assert result == 11
Esempio n. 28
0
def test_get_many(c, s, a, b):
    x = yield Queue("x")
    xx = yield Queue("x")

    yield x.put(1)
    yield x.put(2)
    yield x.put(3)

    data = yield xx.get(batch=True)
    assert data == [1, 2, 3]

    yield x.put(1)
    yield x.put(2)
    yield x.put(3)

    data = yield xx.get(batch=2)
    assert data == [1, 2]

    with pytest.raises(TimeoutError):
        data = yield asyncio.wait_for(xx.get(batch=2), 0.1)
Esempio n. 29
0
def test_erred_future(c, s, a, b):
    future = c.submit(div, 1, 0)
    q = yield Queue()
    yield q.put(future)
    yield gen.sleep(0.1)
    future2 = yield q.get()
    with pytest.raises(ZeroDivisionError):
        yield future2.result()

    exc = yield future2.exception()
    assert isinstance(exc, ZeroDivisionError)
Esempio n. 30
0
def test_get_many(c, s, a, b):
    x = yield Queue("x")
    xx = yield Queue("x")

    yield x.put(1)
    yield x.put(2)
    yield x.put(3)

    data = yield xx.get(batch=True)
    assert data == [1, 2, 3]

    yield x.put(1)
    yield x.put(2)
    yield x.put(3)

    data = yield xx.get(batch=2)
    assert data == [1, 2]

    with pytest.raises(gen.TimeoutError):
        data = yield gen.with_timeout(timedelta(seconds=0.100), xx.get(batch=2))
Esempio n. 31
0
def test_same_futures(c, s, a, b):
    q = Queue('x')
    future = yield c.scatter(123)

    for i in range(5):
        yield q.put(future)

    assert s.wants_what['queue-x'] == {future.key}

    for i in range(4):
        future2 = yield q.get()
        assert s.wants_what['queue-x'] == {future.key}
        yield gen.sleep(0.05)
        assert s.wants_what['queue-x'] == {future.key}

    yield q.get()

    start = time()
    while s.wants_what['queue-x']:
        yield gen.sleep(0.01)
        assert time() - start < 2
def test_same_futures(c, s, a, b):
    q = Queue('x')
    future = yield c.scatter(123)

    for i in range(5):
        yield q.put(future)

    assert s.wants_what['queue-x'] == {future.key}

    for i in range(4):
        future2 = yield q.get()
        assert s.wants_what['queue-x'] == {future.key}
        yield gen.sleep(0.05)
        assert s.wants_what['queue-x'] == {future.key}

    yield q.get()

    start = time()
    while s.wants_what['queue-x']:
        yield gen.sleep(0.01)
        assert time() - start < 2
Esempio n. 33
0
def test_sync(client):
    future = client.submit(lambda x: x + 1, 10)
    x = Queue('x')
    xx = Queue('x')
    x.put(future)
    assert x.qsize() == 1
    assert xx.qsize() == 1
    future2 = xx.get()

    assert future2.result() == 11
Esempio n. 34
0
    def queue(self, maxsize: int = 0, client: Client = None) -> Queue:
        """
        Creates an executor-compatible Queue object which can share state
        across tasks.

        Args:
            - maxsize (int, optional): `maxsize` for the Queue; defaults to 0
                (interpreted as no size limitation)
            - client (dask.distributed.Client, optional): which client to
                associate the Queue with; defaults to `self.client`
        """
        q = Queue(maxsize=maxsize, client=client or self.client)
        return q
Esempio n. 35
0
async def test_2220(c, s, a, b):
    q = Queue()

    def put():
        q.put(55)

    def get():
        print(q.get())

    fut = c.submit(put)
    res = c.submit(get)

    await c.gather([res, fut])
Esempio n. 36
0
def test_2220(c, s, a, b):
    q = Queue()

    def put():
        q.put(55)

    def get():
        print(q.get())

    fut = c.submit(put)
    res = c.submit(get)

    yield [res, fut]
def test_race(c, s, *workers):
    def f(i):
        with worker_client() as c:
            q = Queue('x', client=c)
            for _ in range(100):
                future = q.get()
                x = future.result()
                y = c.submit(inc, x)
                q.put(y)
                sleep(0.01)
            result = q.get().result()
            return result

    q = Queue('x', client=c)
    L = yield c.scatter(range(5))
    for future in L:
        yield q.put(future)

    futures = c.map(f, range(5))
    results = yield c.gather(futures)
    assert all(r > 80 for r in results)
    qsize = yield q.qsize()
    assert not qsize
Esempio n. 38
0
def test_race(c, s, *workers):
    def f(i):
        with worker_client() as c:
            q = Queue('x', client=c)
            for _ in range(100):
                future = q.get()
                x = future.result()
                y = c.submit(inc, x)
                q.put(y)
                sleep(0.01)
            result = q.get().result()
            return result

    q = Queue('x', client=c)
    L = yield c.scatter(range(5))
    for future in L:
        yield q.put(future)

    futures = c.map(f, range(5))
    results = yield c.gather(futures)
    assert all(r > 50 for r in results)
    assert sum(results) == 510
    qsize = yield q.qsize()
    assert not qsize
Esempio n. 39
0
 def f(i):
     with worker_client() as c:
         q = Queue("x", client=c)
         for _ in range(100):
             future = q.get()
             x = future.result()
             y = c.submit(inc, x)
             q.put(y)
             sleep(0.01)
         result = q.get().result()
         return result
Esempio n. 40
0
def test_Queue(c, s, a, b):
    assert s.address.startswith('tls://')

    x = Queue('x')
    y = Queue('y')

    size = yield x.qsize()
    assert size == 0

    future = c.submit(inc, 1)

    yield x.put(future)

    future2 = yield x.get()
    assert future.key == future2.key