def test_timeout(c, s, a, b):
v = Variable('v')
start = time()
with pytest.raises(gen.TimeoutError):
yield v.get(timeout=0.1)
assert 0.05 < time() - start < 2.0
def test_timeout_sync(client):
v = Variable("v")
start = time()
with pytest.raises(gen.TimeoutError):
v.get(timeout=0.1)
stop = time()
assert 0.1 < stop - start < 2.0
def test_timeout_sync(client):
v = Variable('v')
start = time()
with pytest.raises(gen.TimeoutError):
v.get(timeout=0.1)
stop = time()
assert 0.1 < stop - start < 2.0
def test_race(c, s, *workers):
NITERS = 50
def f(i):
with worker_client() as c:
v = Variable('x', client=c)
for _ in range(NITERS):
future = v.get()
x = future.result()
y = c.submit(inc, x)
v.set(y)
sleep(0.01 * random.random())
result = v.get().result()
sleep(0.1) # allow fire-and-forget messages to clear
return result
v = Variable('x', client=c)
x = yield c.scatter(1)
yield v.set(x)
futures = c.map(f, range(15))
results = yield c.gather(futures)
assert all(r > NITERS * 0.8 for r in results)
start = time()
while len(s.wants_what['variable-x']) != 1:
yield gen.sleep(0.01)
assert time() - start < 2
async def test_Future_knows_status_immediately(c, s, a, b):
x = await c.scatter(123)
v = Variable("x")
await v.set(x)
c2 = await Client(s.address, asynchronous=True)
v2 = Variable("x", client=c2)
future = await v2.get()
assert future.status == "finished"
x = c.submit(div, 1, 0)
await wait(x)
await v.set(x)
future2 = await v2.get()
assert future2.status == "error"
with pytest.raises(Exception):
await future2
start = time()
while True: # we learn about the true error eventually
try:
await future2
except ZeroDivisionError:
break
except Exception:
assert time() < start + 5
await asyncio.sleep(0.05)
await c2.close()
def test_race(c, s, *workers):
def f(i):
with worker_client() as c:
v = Variable('x', client=c)
for _ in range(100):
future = v.get()
x = future.result()
y = c.submit(inc, x)
v.set(y)
sleep(0.01)
result = v.get().result()
sleep(0.1) # allow fire-and-forget messages to clear
return result
v = Variable('x', client=c)
x = yield c._scatter(1)
yield v._set(x)
futures = c.map(f, range(10))
results = yield c._gather(futures)
assert all(r > 80 for r in results)
start = time()
while len(s.wants_what['variable-x']) != 1:
yield gen.sleep(0.01)
if not time() - start < 2:
import pdb
pdb.set_trace()
def test_race(c, s, *workers):
NITERS = 50
def f(i):
with worker_client() as c:
v = Variable("x", client=c)
for _ in range(NITERS):
future = v.get()
x = future.result()
y = c.submit(inc, x)
v.set(y)
sleep(0.01 * random.random())
result = v.get().result()
sleep(0.1) # allow fire-and-forget messages to clear
return result
v = Variable("x", client=c)
x = yield c.scatter(1)
yield v.set(x)
futures = c.map(f, range(15))
results = yield c.gather(futures)
assert all(r > NITERS * 0.8 for r in results)
start = time()
while len(s.wants_what["variable-x"]) != 1:
yield gen.sleep(0.01)
assert time() - start < 2
def test_pickleable(client):
v = Variable("v")
def f(x):
v.set(x + 1)
client.submit(f, 10).result()
assert v.get() == 11
def test_timeout(c, s, a, b):
v = Variable('v')
start = time()
with pytest.raises(gen.TimeoutError):
yield v.get(timeout=0.1)
stop = time()
assert 0.1 < stop - start < 2.0
def test_pickleable(client):
v = Variable('v')
def f(x):
v.set(x + 1)
client.submit(f, 10).result()
assert v.get() == 11
def test_sync(client):
future = client.submit(lambda x: x + 1, 10)
x = Variable('x')
xx = Variable('x')
x.set(future)
future2 = xx.get()
assert future2.result() == 11
async def test_delete_unset_variable(c, s, a, b):
x = Variable()
assert x.client is c
with captured_logger(logging.getLogger("distributed.utils")) as logger:
x.delete()
await c.close()
text = logger.getvalue()
assert "KeyError" not in text
def test_timeout(c, s, a, b):
v = Variable("v")
start = time()
with pytest.raises(gen.TimeoutError):
yield v.get(timeout=0.1)
stop = time()
assert 0.1 < stop - start < 2.0
def test_erred_future(c, s, a, b):
future = c.submit(div, 1, 0)
var = Variable()
yield var.set(future)
yield gen.sleep(0.1)
future2 = yield var.get()
with pytest.raises(ZeroDivisionError):
yield future2.result()
class DistStatusReporter(object):
"""Report status through the training scheduler.
Example
-------
>>> @autogluon_method
>>> def train_func(config, reporter):
... reporter(accuracy=0.1)
"""
def __init__(self, remote=None):
self._queue = Queue(client=remote)
self._stop = Variable(client=remote)
self._stop.set(False)
self._continue_semaphore = DistSemaphore(0, remote)
def __call__(self, **kwargs):
"""Report updated training status.
Pass in `done=True` when the training job is completed.
Args:
kwargs: Latest training result status.
Example
_______
>>> reporter(accuracy=1, training_iters=4)
"""
logger.debug('Reporting {}'.format(json.dumps(kwargs)))
try:
self._queue.put(kwargs.copy())
except RuntimeError:
return
self._continue_semaphore.acquire()
if self._stop.get():
raise AutoGluonEarlyStop('Stopping!')
def fetch(self, block=True):
try:
kwargs = self._queue.get()
except CommClosedError:
return {}
return kwargs
def terminate(self):
self._stop.set(True)
self._continue_semaphore.release()
def move_on(self):
self._continue_semaphore.release()
def save_dict(self, **state_dict):
raise NotImplementedError
def get_dict(self):
raise NotImplementedError
def __repr__(self):
reprstr = self.__class__.__name__
return reprstr
def test_queue_with_data(c, s, a, b):
x = Variable('x')
xx = Variable('x')
assert x.client is c
yield x.set((1, 'hello'))
data = yield xx.get()
assert data == (1, 'hello')
async def test_queue_with_data(c, s, a, b):
x = Variable("x")
xx = Variable("x")
assert x.client is c
await x.set((1, "hello"))
data = await xx.get()
assert data == (1, "hello")
def test_timeout_sync(loop):
with cluster() as (s, [a, b]):
with Client(s['address']) as c:
v = Variable('v')
start = time()
with pytest.raises(gen.TimeoutError):
v.get(timeout=0.1)
stop = time()
assert 0.1 < stop - start < 2.0
async def test_timeout_get(c, s, a, b):
v = Variable("v")
tornado_future = v.get()
vv = Variable("v")
await vv.set(1)
result = await tornado_future
assert result == 1
def test_pickleable(loop):
with cluster() as (s, [a, b]):
with Client(s['address']) as c:
v = Variable('v')
def f(x):
v.set(x + 1)
c.submit(f, 10).result()
assert v.get() == 11
def _pre_start_yield(self) -> None:
from distributed import Variable
is_inproc = self.client.scheduler.address.startswith(
"inproc") # type: ignore
if self.address is not None or is_inproc:
self._futures = weakref.WeakSet()
self._should_run_var = Variable(f"prefect-{uuid.uuid4().hex}",
client=self.client)
self._should_run_var.set(True)
def test_timeout_get(c, s, a, b):
v = Variable('v')
tornado_future = v.get()
vv = Variable('v')
yield vv.set(1)
result = yield tornado_future
assert result == 1
def test_future_erred_sync(client):
future = client.submit(div, 1, 0)
var = Variable()
var.set(future)
sleep(0.1)
future2 = var.get()
with pytest.raises(ZeroDivisionError):
future2.result()
def test_erred_future(c, s, a, b):
future = c.submit(div, 1, 0)
var = Variable()
yield var.set(future)
yield gen.sleep(0.1)
future2 = yield var.get()
with pytest.raises(ZeroDivisionError):
yield future2.result()
exc = yield future2.exception()
assert isinstance(exc, ZeroDivisionError)
def test_future_erred_sync(client):
future = client.submit(div, 1, 0)
var = Variable()
var.set(future)
sleep(0.1)
future2 = var.get()
with pytest.raises(ZeroDivisionError):
future2.result()
def f(i):
with worker_client() as c:
v = Variable('x', client=c)
for _ in range(NITERS):
future = v.get()
x = future.result()
y = c.submit(inc, x)
v.set(y)
sleep(0.01 * random.random())
result = v.get().result()
sleep(0.1) # allow fire-and-forget messages to clear
return result
def test_future_erred_sync(loop):
with cluster() as (s, [a, b]):
with Client(s['address']) as c:
future = c.submit(div, 1, 0)
var = Variable()
var.set(future)
sleep(0.1)
future2 = var.get()
with pytest.raises(ZeroDivisionError):
future2.result()
def test_timeout_sync(client):
v = Variable("v")
start = IOLoop.current().time()
with pytest.raises(TimeoutError):
v.get(timeout=0.2)
stop = IOLoop.current().time()
if WINDOWS:
assert 0.1 < stop - start < 2.0
else:
assert 0.2 < stop - start < 2.0
with pytest.raises(TimeoutError):
yield v.get(timeout=0.01)
def test_Future_knows_status_immediately(c, s, a, b):
x = yield c.scatter(123)
v = Variable('x')
yield v.set(x)
c2 = yield Client(s.address, asynchronous=True)
v2 = Variable('x', client=c2)
future = yield v2.get()
assert future.status == 'finished'
x = c.submit(div, 1, 0)
yield wait(x)
yield v.set(x)
future2 = yield v2.get()
assert future2.status == 'error'
with pytest.raises(Exception):
yield future2
start = time()
while True: # we learn about the true error eventually
try:
yield future2
except ZeroDivisionError:
break
except Exception:
assert time() < start + 5
yield gen.sleep(0.05)
yield c2.close()
def _pre_start_yield(self) -> None:
from distributed import Variable
is_inproc = self.client.scheduler.address.startswith(
"inproc") # type: ignore
if self.address is not None or is_inproc:
self._futures = weakref.WeakSet()
self._should_run_var = Variable(f"prefect-{uuid.uuid4().hex}",
client=self.client)
self._should_run_var.set(True)
self._watch_dask_events_task = asyncio.run_coroutine_threadsafe(
self._watch_dask_events(),
self.client.loop.asyncio_loop # type: ignore
)
def test_timeout(c, s, a, b):
v = Variable("v")
start = IOLoop.current().time()
with pytest.raises(TimeoutError):
yield v.get(timeout=0.2)
stop = IOLoop.current().time()
if WINDOWS: # timing is weird with asyncio and Windows
assert 0.1 < stop - start < 2.0
else:
assert 0.2 < stop - start < 2.0
with pytest.raises(TimeoutError):
yield v.get(timeout=0.01)
async def test_variables_do_not_leak_client(c, s, a, b):
# https://github.com/dask/distributed/issues/3899
clients_pre = set(s.clients)
# setup variable with future
x = Variable("x")
future = c.submit(inc, 1)
await x.set(future)
# complete teardown
x.delete()
start = time()
while set(s.clients) != clients_pre:
await asyncio.sleep(0.01)
assert time() < start + 5
async def test_race(c, s, *workers):
NITERS = 50
def f(i):
with worker_client() as c:
v = Variable("x", client=c)
for _ in range(NITERS):
future = v.get()
x = future.result()
y = c.submit(inc, x)
v.set(y)
sleep(0.01 * random.random())
result = v.get().result()
sleep(0.1) # allow fire-and-forget messages to clear
return result
v = Variable("x", client=c)
x = await c.scatter(1)
await v.set(x)
futures = c.map(f, range(15))
results = await c.gather(futures)
while len(s.wants_what["variable-x"]) != 1:
await asyncio.sleep(0.01)
def test_hold_futures(s, a, b):
c1 = yield Client(s.address, asynchronous=True)
future = c1.submit(lambda x: x + 1, 10)
x1 = Variable('x')
yield x1.set(future)
del x1
yield c1.close()
yield gen.sleep(0.1)
c2 = yield Client(s.address, asynchronous=True)
x2 = Variable('x')
future2 = yield x2.get()
result = yield future2
assert result == 11
yield c2.close()
def test_variable_in_task(loop):
# Ensure that we can create a Variable inside a task on a
# worker in a separate Python process than the client
with popen(["dask-scheduler", "--no-dashboard"]):
with popen(["dask-worker", "127.0.0.1:8786"]):
with Client("tcp://127.0.0.1:8786", loop=loop) as c:
c.wait_for_workers(1)
x = Variable("x")
x.set(123)
def foo():
y = Variable("x")
return y.get()
result = c.submit(foo).result()
assert result == 123
async def test_hold_futures(s, a, b):
c1 = await Client(s.address, asynchronous=True)
future = c1.submit(lambda x: x + 1, 10)
x1 = Variable("x")
await x1.set(future)
del x1
await c1.close()
await asyncio.sleep(0.1)
c2 = await Client(s.address, asynchronous=True)
x2 = Variable("x")
future2 = await x2.get()
result = await future2
assert result == 11
await c2.close()
def test_sync(client):
future = client.submit(lambda x: x + 1, 10)
x = Variable("x")
xx = Variable("x")
x.set(future)
future2 = xx.get()
assert future2.result() == 11
def _maybe_run(var_name: str, fn: Callable, *args: Any, **kwargs: Any) -> Any:
"""Check if the task should run against a `distributed.Variable` before
starting the task. This offers stronger guarantees than distributed's
current cancellation mechanism, which only cancels pending tasks."""
# In certain configurations, the way distributed unpickles variables can
# lead to excess client connections being created. To avoid this issue we
# manually lookup the variable by name.
from distributed import Variable, get_client
var = Variable(var_name, client=get_client())
try:
should_run = var.get(timeout=0)
except Exception:
# Errors here indicate the get operation timed out, which can happen if
# the variable is undefined (usually indicating the flow runner has
# stopped or the cluster is shutting down).
should_run = False
if should_run:
return fn(*args, **kwargs)
async def test_erred_future(c, s, a, b):
future = c.submit(div, 1, 0)
var = Variable()
await var.set(future)
await asyncio.sleep(0.1)
future2 = await var.get()
with pytest.raises(ZeroDivisionError):
await future2.result()
exc = await future2.exception()
assert isinstance(exc, ZeroDivisionError)
def fit(self, X, y=None):
w = get_worker()
dsk_lock = Lock(self.lock_name, client=w.client)
dsk_counter = Variable(self.counter_name, client=w.client)
dsk_killed_workers = Variable(self.killed_workers_name, client=w.client)
for e in list(w.executing):
should_die = False
t = literal_eval(e)
with dsk_lock:
c = dsk_counter.get()
dsk_counter.set(c + 1)
killed_workers = dsk_killed_workers.get()
if c > self.min_complete and t not in killed_workers:
killed_workers[t] = True
should_die = True
dsk_killed_workers.set(killed_workers)
if should_die:
os.kill(os.getpid(), 9)
return self
def test_sync(loop):
with cluster() as (s, [a, b]):
with Client(s['address']) as c:
future = c.submit(lambda x: x + 1, 10)
x = Variable('x')
xx = Variable('x')
x.set(future)
future2 = xx.get()
assert future2.result() == 11
def test_Future_knows_status_immediately(c, s, a, b):
x = yield c.scatter(123)
v = Variable('x')
yield v.set(x)
c2 = yield Client(s.address, asynchronous=True)
v2 = Variable('x', client=c2)
future = yield v2.get()
assert future.status == 'finished'
x = c.submit(div, 1, 0)
yield wait(x)
yield v.set(x)
future2 = yield v2.get()
assert future2.status == 'error'
with pytest.raises(Exception):
yield future2
start = time()
while True: # we learn about the true error eventually
try:
yield future2
except ZeroDivisionError:
break
except Exception:
assert time() < start + 5
yield gen.sleep(0.05)
yield c2.close()
def test_cleanup(c, s, a, b):
v = Variable('v')
vv = Variable('v')
x = c.submit(lambda x: x + 1, 10)
y = c.submit(lambda x: x + 1, 20)
x_key = x.key
yield v.set(x)
del x
yield gen.sleep(0.1)
t_future = xx = vv._get()
yield gen.moment
v._set(y)
future = yield t_future
assert future.key == x_key
result = yield future
assert result == 11
def test_variable(c, s, a, b):
x = Variable('x')
xx = Variable('x')
assert x.client is c
future = c.submit(inc, 1)
yield x.set(future)
future2 = yield xx.get()
assert future.key == future2.key
del future, future2
yield gen.sleep(0.1)
assert s.tasks # future still present
x.delete()
start = time()
while s.tasks:
yield gen.sleep(0.01)
assert time() < start + 5
