def test_pause_executor(c, s, a):
memory = psutil.Process().memory_info().rss
a.memory_limit = memory / 0.5 + 200e6
np = pytest.importorskip("numpy")
def f():
x = np.ones(int(400e6), dtype="u1")
sleep(1)
with captured_logger(logging.getLogger("distributed.worker")) as logger:
future = c.submit(f)
futures = c.map(slowinc, range(30), delay=0.1)
start = time()
while not a.paused:
yield gen.sleep(0.01)
assert time() < start + 4, (
format_bytes(psutil.Process().memory_info().rss),
format_bytes(a.memory_limit),
len(a.data),
)
out = logger.getvalue()
assert "memory" in out.lower()
assert "pausing" in out.lower()
assert sum(f.status == "finished" for f in futures) < 4
yield wait(futures)
async def test_pip_install_fails(c, s, a, b):
with captured_logger("distributed.diagnostics.plugin",
level=logging.ERROR) as logger:
with mock.patch(
"distributed.diagnostics.plugin.subprocess.Popen.communicate",
return_value=(b"", b"error"),
) as p1:
with mock.patch("distributed.diagnostics.plugin.subprocess.Popen",
return_value=p1) as p2:
p1.communicate.return_value = (
b"",
b"Could not find a version that satisfies the requirement not-a-package",
)
p1.wait.return_value = 1
await c.register_worker_plugin(
PipInstall(packages=["not-a-package"]))
assert "not-a-package" in logger.getvalue()
async def test_heartbeat_comm_closed(cleanup, monkeypatch, reconnect):
with captured_logger("distributed.worker", level=logging.WARNING) as logger:
async with await Scheduler() as s:
def bad_heartbeat_worker(*args, **kwargs):
raise CommClosedError()
async with await Worker(s.address, reconnect=reconnect) as w:
# Trigger CommClosedError during worker heartbeat
monkeypatch.setattr(
w.scheduler, "heartbeat_worker", bad_heartbeat_worker
)
await w.heartbeat()
if reconnect:
assert w.status == "running"
else:
assert w.status == "closed"
assert "Heartbeat to scheduler failed" in logger.getvalue()
def test_pause_executor(c, s, a):
memory = psutil.Process().memory_info().rss
a.memory_limit = memory / 0.8 + 200e6
np = pytest.importorskip('numpy')
def f():
x = np.ones(int(300e6), dtype='u1')
sleep(1)
with captured_logger(logging.getLogger('distributed.worker')) as logger:
future = c.submit(f)
futures = c.map(slowinc, range(10), delay=0.1)
yield gen.sleep(0.3)
assert a.paused
out = logger.getvalue()
assert 'memory' in out.lower()
assert 'pausing' in out.lower()
assert sum(f.status == 'finished' for f in futures) < 4
yield wait(futures)
def test_pause_executor(c, s, a):
memory = psutil.Process().memory_info().rss
a.memory_limit = memory / 0.8 + 200e6
np = pytest.importorskip('numpy')
def f():
x = np.ones(int(300e6), dtype='u1')
sleep(1)
with captured_logger(logging.getLogger('distributed.worker')) as logger:
future = c.submit(f)
futures = c.map(slowinc, range(10), delay=0.1)
yield gen.sleep(0.3)
assert a.paused, (format_bytes(psutil.Process().memory_info().rss),
format_bytes(a.memory_limit))
out = logger.getvalue()
assert 'memory' in out.lower()
assert 'pausing' in out.lower()
assert sum(f.status == 'finished' for f in futures) < 4
yield wait(futures)