def test_stress_scatter_death(c, s, *workers): import random s.allowed_failures = 1000 np = pytest.importorskip('numpy') L = yield c.scatter([np.random.random(10000) for i in range(len(workers))]) yield c._replicate(L, n=2) adds = [delayed(slowadd, pure=True)(random.choice(L), random.choice(L), delay=0.05, dask_key_name='slowadd-1-%d' % i) for i in range(50)] adds = [delayed(slowadd, pure=True)(a, b, delay=0.02, dask_key_name='slowadd-2-%d' % i) for i, (a, b) in enumerate(sliding_window(2, adds))] futures = c.compute(adds) L = adds = None alive = list(workers) from distributed.scheduler import logger for i in range(7): yield gen.sleep(0.1) try: s.validate_state() except Exception as c: logger.exception(c) if config.get('log-on-err'): import pdb pdb.set_trace() else: raise w = random.choice(alive) yield w._close() alive.remove(w) try: yield gen.with_timeout(timedelta(seconds=25), c._gather(futures)) except gen.TimeoutError: ws = {w.address: w for w in workers if w.status != 'closed'} print(s.processing) print(ws) print(futures) try: worker = [w for w in ws.values() if w.waiting_for_data][0] except Exception: pass if config.get('log-on-err'): import pdb pdb.set_trace() else: raise except CancelledError: pass finally: futures = None
def test_stress_scatter_death(c, s, *workers): import random s.allowed_failures = 1000 np = pytest.importorskip('numpy') L = yield c.scatter([np.random.random(10000) for i in range(len(workers))]) yield c._replicate(L, n=2) adds = [delayed(slowadd, pure=True)(random.choice(L), random.choice(L), delay=0.05, dask_key_name='slowadd-1-%d' % i) for i in range(50)] adds = [delayed(slowadd, pure=True)(a, b, delay=0.02, dask_key_name='slowadd-2-%d' % i) for i, (a, b) in enumerate(sliding_window(2, adds))] futures = c.compute(adds) L = adds = None alive = list(workers) from distributed.scheduler import logger for i in range(7): yield gen.sleep(0.1) try: s.validate_state() except Exception as c: logger.exception(c) if config.get('log-on-err'): import pdb pdb.set_trace() else: raise w = random.choice(alive) yield w._close() alive.remove(w) try: yield gen.with_timeout(timedelta(seconds=25), c._gather(futures)) except gen.TimeoutError: ws = {w.address: w for w in workers if w.status != 'closed'} print(s.processing) print(ws) print(futures) try: worker = [w for w in ws.values() if w.waiting_for_data][0] except Exception: pass if config.get('log-on-err'): import pdb pdb.set_trace() else: raise except CancelledError: pass finally: futures = None
async def test_stress_scatter_death(c, s, *workers): import random s.allowed_failures = 1000 np = pytest.importorskip("numpy") L = await c.scatter([np.random.random(10000) for i in range(len(workers))]) await c.replicate(L, n=2) adds = [ delayed(slowadd, pure=True)( random.choice(L), random.choice(L), delay=0.05, dask_key_name="slowadd-1-%d" % i, ) for i in range(50) ] adds = [ delayed(slowadd, pure=True)(a, b, delay=0.02, dask_key_name="slowadd-2-%d" % i) for i, (a, b) in enumerate(sliding_window(2, adds)) ] futures = c.compute(adds) L = adds = None alive = list(workers) from distributed.scheduler import logger for i in range(7): await asyncio.sleep(0.1) try: s.validate_state() except Exception as c: logger.exception(c) if config.get("log-on-err"): import pdb pdb.set_trace() else: raise w = random.choice(alive) await w.close() alive.remove(w) with suppress(CancelledError): await c.gather(futures) futures = None
def test_stress_scatter_death(c, s, *workers): import random np = pytest.importorskip('numpy') L = yield c._scatter( [np.random.random(10000) for i in range(len(workers))]) yield c._replicate(L, n=2) adds = [ delayed(slowadd, pure=True)(random.choice(L), random.choice(L), delay=0.05) for i in range(50) ] adds = [ delayed(slowadd, pure=True)(a, b, delay=0.02) for a, b in sliding_window(2, adds) ] futures = c.compute(adds) alive = list(workers) from distributed.scheduler import logger for i in range(7): yield gen.sleep(0.1) try: s.validate_state() except Exception as c: logger.exception(c) import pdb pdb.set_trace() w = random.choice(alive) yield w._close() alive.remove(w) try: yield gen.with_timeout(timedelta(seconds=10), c._gather(futures)) except gen.TimeoutError: import pdb pdb.set_trace() except CancelledError: pass
def test_stress_scatter_death(c, s, *workers): import random np = pytest.importorskip('numpy') L = yield c._scatter([np.random.random(10000) for i in range(len(workers))]) yield c._replicate(L, n=2) adds = [delayed(slowadd, pure=True)(random.choice(L), random.choice(L), delay=0.05) for i in range(50)] adds = [delayed(slowadd, pure=True)(a, b, delay=0.02) for a, b in sliding_window(2, adds)] futures = c.compute(adds) alive = list(workers) from distributed.scheduler import logger for i in range(7): yield gen.sleep(0.1) try: s.validate_state() except Exception as c: logger.exception(c) import pdb; pdb.set_trace() w = random.choice(alive) yield w._close() alive.remove(w) try: yield gen.with_timeout(timedelta(seconds=10), c._gather(futures)) except gen.TimeoutError: import pdb; pdb.set_trace() except CancelledError: pass