def test_queue(c, s, a, b):
x = yield Queue('x')
y = yield Queue('y')
xx = yield Queue('x')
assert x.client is c
future = c.submit(inc, 1)
yield x.put(future)
yield y.put(future)
future2 = yield xx.get()
assert future.key == future2.key
with pytest.raises(gen.TimeoutError):
yield x.get(timeout=0.1)
del future, future2
yield gen.sleep(0.1)
assert s.task_state # future still present in y's queue
yield y.get() # burn future
start = time()
while s.task_state:
yield gen.sleep(0.01)
assert time() < start + 5
def test_Future_knows_status_immediately(c, s, a, b):
x = yield c.scatter(123)
q = yield Queue('q')
yield q.put(x)
c2 = yield Client(s.address, asynchronous=True)
q2 = yield Queue('q', client=c2)
future = yield q2.get()
assert future.status == 'finished'
x = c.submit(div, 1, 0)
yield wait(x)
yield q.put(x)
future2 = yield q2.get()
assert future2.status == 'error'
with pytest.raises(Exception):
yield future2
start = time()
while True: # we learn about the true error eventually
try:
yield future2
except ZeroDivisionError:
break
except Exception:
assert time() < start + 5
yield gen.sleep(0.05)
yield c2.close()
def test_erred_future(c, s, a, b):
future = c.submit(div, 1, 0)
q = Queue()
yield q.put(future)
yield gen.sleep(0.1)
future2 = yield q.get()
with pytest.raises(ZeroDivisionError):
yield future2.result()
class DistStatusReporter(object):
"""Report status through the training scheduler.
Example
-------
>>> @autogluon_method
>>> def train_func(config, reporter):
... reporter(accuracy=0.1)
"""
def __init__(self, remote=None):
self._queue = Queue(client=remote)
self._stop = Variable(client=remote)
self._stop.set(False)
self._continue_semaphore = DistSemaphore(0, remote)
def __call__(self, **kwargs):
"""Report updated training status.
Pass in `done=True` when the training job is completed.
Args:
kwargs: Latest training result status.
Example
_______
>>> reporter(accuracy=1, training_iters=4)
"""
logger.debug('Reporting {}'.format(json.dumps(kwargs)))
try:
self._queue.put(kwargs.copy())
except RuntimeError:
return
self._continue_semaphore.acquire()
if self._stop.get():
raise AutoGluonEarlyStop('Stopping!')
def fetch(self, block=True):
try:
kwargs = self._queue.get()
except CommClosedError:
return {}
return kwargs
def terminate(self):
self._stop.set(True)
self._continue_semaphore.release()
def move_on(self):
self._continue_semaphore.release()
def save_dict(self, **state_dict):
raise NotImplementedError
def get_dict(self):
raise NotImplementedError
def __repr__(self):
reprstr = self.__class__.__name__
return reprstr
def test_picklability(c, s, a, b):
q = Queue()
def f(x):
q.put(x + 1)
yield c.submit(f, 10)
result = yield q.get()
assert result == 11
def test_picklability(c, s, a, b):
q = Queue()
def f(x):
q.put(x + 1)
yield c.submit(f, 10)
result = yield q.get()
assert result == 11
def test_picklability_sync(client):
q = Queue()
def f(x):
q.put(x + 1)
client.submit(f, 10).result()
assert q.get() == 11
def test_picklability_sync(client):
q = Queue()
def f(x):
q.put(x + 1)
client.submit(f, 10).result()
assert q.get() == 11
def test_close(c, s, a, b):
q = Queue()
while q.name not in s.extensions['queues'].queues:
yield gen.sleep(0.01)
q.close()
q.close()
while q.name in s.extensions['queues'].queues:
yield gen.sleep(0.01)
def test_erred_future(c, s, a, b):
future = c.submit(div, 1, 0)
q = Queue()
yield q.put(future)
yield gen.sleep(0.1)
future2 = yield q.get()
with pytest.raises(ZeroDivisionError):
yield future2.result()
exc = yield future2.exception()
assert isinstance(exc, ZeroDivisionError)
def f(i):
with worker_client() as c:
q = Queue('x', client=c)
for _ in range(100):
future = q.get()
x = future.result()
y = c.submit(inc, x)
q.put(y)
sleep(0.01)
result = q.get().result()
return result
def test_close(c, s, a, b):
q = Queue()
while q.name not in s.extensions['queues'].queues:
yield gen.sleep(0.01)
q.close()
q.close()
while q.name in s.extensions['queues'].queues:
yield gen.sleep(0.01)
def test_picklability_sync(loop):
with cluster() as (s, [a, b]):
with Client(s['address']) as c:
q = Queue()
def f(x):
q.put(x + 1)
c.submit(f, 10).result()
assert q.get() == 11
def test_queue_with_data(c, s, a, b):
x = yield Queue("x")
xx = yield Queue("x")
assert x.client is c
yield x.put((1, "hello"))
data = yield xx.get()
assert data == (1, "hello")
with pytest.raises(gen.TimeoutError):
yield x.get(timeout=0.1)
def test_queue_with_data(c, s, a, b):
x = yield Queue('x')
xx = yield Queue('x')
assert x.client is c
yield x.put([1, 'hello'])
data = yield xx.get()
assert data == [1, 'hello']
with pytest.raises(gen.TimeoutError):
yield x.get(timeout=0.1)
def test_sync(client):
future = client.submit(lambda x: x + 1, 10)
x = Queue("x")
xx = Queue("x")
x.put(future)
assert x.qsize() == 1
assert xx.qsize() == 1
future2 = xx.get()
assert future2.result() == 11
def test_Queue(c, s, a, b):
assert s.address.startswith("tls://")
x = yield Queue("x")
y = yield Queue("y")
size = yield x.qsize()
assert size == 0
future = c.submit(inc, 1)
yield x.put(future)
future2 = yield x.get()
assert future.key == future2.key
def test_Queue(c, s, a, b):
assert s.address.startswith('tls://')
x = Queue('x')
y = Queue('y')
size = yield x.qsize()
assert size == 0
future = c.submit(inc, 1)
yield x.put(future)
future2 = yield x.get()
assert future.key == future2.key
def test_sync(loop):
with cluster() as (s, [a, b]):
with Client(s['address']) as c:
future = c.submit(lambda x: x + 1, 10)
x = Queue('x')
xx = Queue('x')
x.put(future)
assert x.qsize() == 1
assert xx.qsize() == 1
future2 = xx.get()
assert future2.result() == 11
def test_timeout(c, s, a, b):
q = Queue('v', maxsize=1)
start = time()
with pytest.raises(gen.TimeoutError):
yield q.get(timeout=0.3)
stop = time()
assert 0.2 < stop - start < 2.0
yield q.put(1)
start = time()
with pytest.raises(gen.TimeoutError):
yield q.put(2, timeout=0.3)
stop = time()
assert 0.1 < stop - start < 2.0
def test_timeout(c, s, a, b):
q = Queue('v', maxsize=1)
start = time()
with pytest.raises(gen.TimeoutError):
yield q.get(timeout=0.3)
stop = time()
assert 0.2 < stop - start < 2.0
yield q.put(1)
start = time()
with pytest.raises(gen.TimeoutError):
yield q.put(2, timeout=0.3)
stop = time()
assert 0.1 < stop - start < 2.0
def test_queue_in_task(loop):
# Ensure that we can create a Queue inside a task on a
# worker in a separate Python process than the client
with popen(["dask-scheduler", "--no-dashboard"]):
with popen(["dask-worker", "127.0.0.1:8786"]):
with Client("tcp://127.0.0.1:8786", loop=loop) as c:
c.wait_for_workers(1)
x = Queue("x")
x.put(123)
def foo():
y = Queue("x")
return y.get()
result = c.submit(foo).result()
assert result == 123
def test_hold_futures(s, a, b):
c1 = yield Client(s.address, asynchronous=True)
future = c1.submit(lambda x: x + 1, 10)
q1 = yield Queue('q')
yield q1.put(future)
del q1
yield c1.close()
yield gen.sleep(0.1)
c2 = yield Client(s.address, asynchronous=True)
q2 = yield Queue('q')
future2 = yield q2.get()
result = yield future2
assert result == 11
yield c2.close()
def test_close(c, s, a, b):
q = yield Queue()
q.close()
q.close()
while q.name in s.extensions["queues"].queues:
yield gen.sleep(0.01)
def start_and_attach_server(spec,
job_name=None,
task_index=None,
dask_worker=None):
server = tf.train.Server(spec, job_name=job_name, task_index=task_index)
dask_worker.tensorflow_server = server
dask_worker.tensorflow_queue = Queue()
return 'OK'
class DistSemaphore(object):
def __init__(self, value, remote=None):
self._queue = Queue(client=remote)
for i in range(value):
self._queue.put(1)
def acquire(self):
try:
_ = self._queue.get()
except distributed.comm.core.CommClosedError:
pass
def release(self):
self._queue.put(1)
def __repr__(self):
reprstr = self.__class__.__name__
return reprstr
async def test_picklability(c, s, a, b):
q = Queue()
def f(x):
q.put(x + 1)
await c.submit(f, 10)
result = await q.get()
assert result == 11
def test_get_many(c, s, a, b):
x = yield Queue("x")
xx = yield Queue("x")
yield x.put(1)
yield x.put(2)
yield x.put(3)
data = yield xx.get(batch=True)
assert data == [1, 2, 3]
yield x.put(1)
yield x.put(2)
yield x.put(3)
data = yield xx.get(batch=2)
assert data == [1, 2]
with pytest.raises(TimeoutError):
data = yield asyncio.wait_for(xx.get(batch=2), 0.1)
def test_erred_future(c, s, a, b):
future = c.submit(div, 1, 0)
q = yield Queue()
yield q.put(future)
yield gen.sleep(0.1)
future2 = yield q.get()
with pytest.raises(ZeroDivisionError):
yield future2.result()
exc = yield future2.exception()
assert isinstance(exc, ZeroDivisionError)
def test_get_many(c, s, a, b):
x = yield Queue("x")
xx = yield Queue("x")
yield x.put(1)
yield x.put(2)
yield x.put(3)
data = yield xx.get(batch=True)
assert data == [1, 2, 3]
yield x.put(1)
yield x.put(2)
yield x.put(3)
data = yield xx.get(batch=2)
assert data == [1, 2]
with pytest.raises(gen.TimeoutError):
data = yield gen.with_timeout(timedelta(seconds=0.100), xx.get(batch=2))
def test_same_futures(c, s, a, b):
q = Queue('x')
future = yield c.scatter(123)
for i in range(5):
yield q.put(future)
assert s.wants_what['queue-x'] == {future.key}
for i in range(4):
future2 = yield q.get()
assert s.wants_what['queue-x'] == {future.key}
yield gen.sleep(0.05)
assert s.wants_what['queue-x'] == {future.key}
yield q.get()
start = time()
while s.wants_what['queue-x']:
yield gen.sleep(0.01)
assert time() - start < 2
def test_same_futures(c, s, a, b):
q = Queue('x')
future = yield c.scatter(123)
for i in range(5):
yield q.put(future)
assert s.wants_what['queue-x'] == {future.key}
for i in range(4):
future2 = yield q.get()
assert s.wants_what['queue-x'] == {future.key}
yield gen.sleep(0.05)
assert s.wants_what['queue-x'] == {future.key}
yield q.get()
start = time()
while s.wants_what['queue-x']:
yield gen.sleep(0.01)
assert time() - start < 2
def test_sync(client):
future = client.submit(lambda x: x + 1, 10)
x = Queue('x')
xx = Queue('x')
x.put(future)
assert x.qsize() == 1
assert xx.qsize() == 1
future2 = xx.get()
assert future2.result() == 11
def queue(self, maxsize: int = 0, client: Client = None) -> Queue:
"""
Creates an executor-compatible Queue object which can share state
across tasks.
Args:
- maxsize (int, optional): `maxsize` for the Queue; defaults to 0
(interpreted as no size limitation)
- client (dask.distributed.Client, optional): which client to
associate the Queue with; defaults to `self.client`
"""
q = Queue(maxsize=maxsize, client=client or self.client)
return q
async def test_2220(c, s, a, b):
q = Queue()
def put():
q.put(55)
def get():
print(q.get())
fut = c.submit(put)
res = c.submit(get)
await c.gather([res, fut])
def test_2220(c, s, a, b):
q = Queue()
def put():
q.put(55)
def get():
print(q.get())
fut = c.submit(put)
res = c.submit(get)
yield [res, fut]
def test_race(c, s, *workers):
def f(i):
with worker_client() as c:
q = Queue('x', client=c)
for _ in range(100):
future = q.get()
x = future.result()
y = c.submit(inc, x)
q.put(y)
sleep(0.01)
result = q.get().result()
return result
q = Queue('x', client=c)
L = yield c.scatter(range(5))
for future in L:
yield q.put(future)
futures = c.map(f, range(5))
results = yield c.gather(futures)
assert all(r > 80 for r in results)
qsize = yield q.qsize()
assert not qsize
def test_race(c, s, *workers):
def f(i):
with worker_client() as c:
q = Queue('x', client=c)
for _ in range(100):
future = q.get()
x = future.result()
y = c.submit(inc, x)
q.put(y)
sleep(0.01)
result = q.get().result()
return result
q = Queue('x', client=c)
L = yield c.scatter(range(5))
for future in L:
yield q.put(future)
futures = c.map(f, range(5))
results = yield c.gather(futures)
assert all(r > 50 for r in results)
assert sum(results) == 510
qsize = yield q.qsize()
assert not qsize
def f(i):
with worker_client() as c:
q = Queue("x", client=c)
for _ in range(100):
future = q.get()
x = future.result()
y = c.submit(inc, x)
q.put(y)
sleep(0.01)
result = q.get().result()
return result
def test_Queue(c, s, a, b):
assert s.address.startswith('tls://')
x = Queue('x')
y = Queue('y')
size = yield x.qsize()
assert size == 0
future = c.submit(inc, 1)
yield x.put(future)
future2 = yield x.get()
assert future.key == future2.key