def test_stress_scatter_death(c, s, *workers):
import random
s.allowed_failures = 1000
np = pytest.importorskip('numpy')
L = yield c.scatter([np.random.random(10000) for i in range(len(workers))])
yield c._replicate(L, n=2)
adds = [delayed(slowadd, pure=True)(random.choice(L),
random.choice(L),
delay=0.05,
dask_key_name='slowadd-1-%d' % i)
for i in range(50)]
adds = [delayed(slowadd, pure=True)(a, b, delay=0.02,
dask_key_name='slowadd-2-%d' % i)
for i, (a, b) in enumerate(sliding_window(2, adds))]
futures = c.compute(adds)
L = adds = None
alive = list(workers)
from distributed.scheduler import logger
for i in range(7):
yield gen.sleep(0.1)
try:
s.validate_state()
except Exception as c:
logger.exception(c)
if config.get('log-on-err'):
import pdb
pdb.set_trace()
else:
raise
w = random.choice(alive)
yield w._close()
alive.remove(w)
try:
yield gen.with_timeout(timedelta(seconds=25), c._gather(futures))
except gen.TimeoutError:
ws = {w.address: w for w in workers if w.status != 'closed'}
print(s.processing)
print(ws)
print(futures)
try:
worker = [w for w in ws.values() if w.waiting_for_data][0]
except Exception:
pass
if config.get('log-on-err'):
import pdb
pdb.set_trace()
else:
raise
except CancelledError:
pass
finally:
futures = None
def test_stress_scatter_death(c, s, *workers):
import random
s.allowed_failures = 1000
np = pytest.importorskip('numpy')
L = yield c.scatter([np.random.random(10000) for i in range(len(workers))])
yield c._replicate(L, n=2)
adds = [delayed(slowadd, pure=True)(random.choice(L),
random.choice(L),
delay=0.05,
dask_key_name='slowadd-1-%d' % i)
for i in range(50)]
adds = [delayed(slowadd, pure=True)(a, b, delay=0.02,
dask_key_name='slowadd-2-%d' % i)
for i, (a, b) in enumerate(sliding_window(2, adds))]
futures = c.compute(adds)
L = adds = None
alive = list(workers)
from distributed.scheduler import logger
for i in range(7):
yield gen.sleep(0.1)
try:
s.validate_state()
except Exception as c:
logger.exception(c)
if config.get('log-on-err'):
import pdb
pdb.set_trace()
else:
raise
w = random.choice(alive)
yield w._close()
alive.remove(w)
try:
yield gen.with_timeout(timedelta(seconds=25), c._gather(futures))
except gen.TimeoutError:
ws = {w.address: w for w in workers if w.status != 'closed'}
print(s.processing)
print(ws)
print(futures)
try:
worker = [w for w in ws.values() if w.waiting_for_data][0]
except Exception:
pass
if config.get('log-on-err'):
import pdb
pdb.set_trace()
else:
raise
except CancelledError:
pass
finally:
futures = None
async def test_stress_scatter_death(c, s, *workers):
import random
s.allowed_failures = 1000
np = pytest.importorskip("numpy")
L = await c.scatter([np.random.random(10000) for i in range(len(workers))])
await c.replicate(L, n=2)
adds = [
delayed(slowadd, pure=True)(
random.choice(L),
random.choice(L),
delay=0.05,
dask_key_name="slowadd-1-%d" % i,
) for i in range(50)
]
adds = [
delayed(slowadd, pure=True)(a,
b,
delay=0.02,
dask_key_name="slowadd-2-%d" % i)
for i, (a, b) in enumerate(sliding_window(2, adds))
]
futures = c.compute(adds)
L = adds = None
alive = list(workers)
from distributed.scheduler import logger
for i in range(7):
await asyncio.sleep(0.1)
try:
s.validate_state()
except Exception as c:
logger.exception(c)
if config.get("log-on-err"):
import pdb
pdb.set_trace()
else:
raise
w = random.choice(alive)
await w.close()
alive.remove(w)
with suppress(CancelledError):
await c.gather(futures)
futures = None
def test_stress_scatter_death(c, s, *workers):
import random
np = pytest.importorskip('numpy')
L = yield c._scatter(
[np.random.random(10000) for i in range(len(workers))])
yield c._replicate(L, n=2)
adds = [
delayed(slowadd, pure=True)(random.choice(L),
random.choice(L),
delay=0.05) for i in range(50)
]
adds = [
delayed(slowadd, pure=True)(a, b, delay=0.02)
for a, b in sliding_window(2, adds)
]
futures = c.compute(adds)
alive = list(workers)
from distributed.scheduler import logger
for i in range(7):
yield gen.sleep(0.1)
try:
s.validate_state()
except Exception as c:
logger.exception(c)
import pdb
pdb.set_trace()
w = random.choice(alive)
yield w._close()
alive.remove(w)
try:
yield gen.with_timeout(timedelta(seconds=10), c._gather(futures))
except gen.TimeoutError:
import pdb
pdb.set_trace()
except CancelledError:
pass
def test_stress_scatter_death(c, s, *workers):
import random
np = pytest.importorskip('numpy')
L = yield c._scatter([np.random.random(10000) for i in range(len(workers))])
yield c._replicate(L, n=2)
adds = [delayed(slowadd, pure=True)(random.choice(L),
random.choice(L),
delay=0.05)
for i in range(50)]
adds = [delayed(slowadd, pure=True)(a, b, delay=0.02)
for a, b in sliding_window(2, adds)]
futures = c.compute(adds)
alive = list(workers)
from distributed.scheduler import logger
for i in range(7):
yield gen.sleep(0.1)
try:
s.validate_state()
except Exception as c:
logger.exception(c)
import pdb; pdb.set_trace()
w = random.choice(alive)
yield w._close()
alive.remove(w)
try:
yield gen.with_timeout(timedelta(seconds=10), c._gather(futures))
except gen.TimeoutError:
import pdb; pdb.set_trace()
except CancelledError:
pass
