def test_restart_fast(loop): with cluster_center(nanny=True) as (c, [a, b]): with Executor(('127.0.0.1', c['port'])) as e: L = e.map(sleep, range(10)) start = time() e.restart() assert not e.scheduler.dask assert time() - start < 5 assert all(x.status == 'cancelled' for x in L) x = e.submit(inc, 1) assert x.result() == 2
def test_gather_scatter(loop): with cluster_center() as (c, [a, b]): data = {'x': 1, 'y': 2, 'z': 3} addr = '127.0.0.1:%d' % c['port'] rds = scatter(addr, data) assert set(rds) == {'x', 'y', 'z'} names = sorted(rds) data2 = gather(addr, rds) data2 = dict(zip(rds, data2)) assert data == data2 delete(addr, ['x']) with pytest.raises(KeyError): gather(addr, ['x']) clear(addr) with pytest.raises(KeyError): gather(addr, ['y'])
def test_gather_scatter(loop): with cluster_center() as (c, [a, b]): data = {'x': 1, 'y': 2, 'z': 3} addr = '127.0.0.1', c['port'] rds = scatter(addr, data) assert set(rds) == {'x', 'y', 'z'} names = sorted(rds) data2 = gather(addr, rds) data2 = dict(zip(rds, data2)) assert data == data2 delete(addr, ['x']) with pytest.raises(KeyError): gather(addr, ['x']) clear(addr) with pytest.raises(KeyError): gather(addr, ['y'])
def test_gather_scatter(loop): with cluster_center() as (c, [a, b]): data = {'x': 1, 'y': 2, 'z': 3} addr = '127.0.0.1', c['port'] rds = scatter(addr, data) assert all(isinstance(rd, RemoteData) for rd in rds.values()) assert set(rds) == set(data) assert all(k == v.key for k, v in rds.items()) names = sorted(rds) data2 = gather(addr, [rds[name] for name in names]) data2 = dict(zip(names, data2)) assert data == data2 delete(addr, ['x']) with pytest.raises(KeyError): gather(addr, ['x']) clear(addr) with pytest.raises(KeyError): gather(addr, ['y'])
def test_restart_sync(loop): with cluster_center(nanny=True) as (c, [a, b]): with Executor(('127.0.0.1', c['port']), loop=loop) as e: assert len(e.scheduler.has_what) == 2 x = e.submit(div, 1, 2) x.result() assert e.scheduler.who_has e.restart() assert not e.scheduler.who_has assert x.cancelled() with pytest.raises(CancelledError): x.result() assert (set(e.scheduler.stacks) == set(e.scheduler.processing) == set(e.scheduler.ncores)) assert len(e.scheduler.stacks) == 2 y = e.submit(div, 1, 3) assert y.result() == 1 / 3
def test_resource_monitor(loop): with cluster_center(nanny=True) as (c, [a, b]): with scheduler(c['port']) as sport: with Executor(('127.0.0.1', sport)) as e: rm1 = ResourceMonitor(interval=0.01) with Executor(('127.0.0.1', c['port'])) as e: rm2 = ResourceMonitor(interval=0.01) rm3 = ResourceMonitor(('127.0.0.1', sport), interval=0.01) for rm in [rm1, rm2, rm3]: for k in ['cpu', 'memory', 'host']: assert k in rm.cds.data start = time() while not rm.cds.data['cpu']: loop.run_sync(lambda: gen.sleep(0.05)) assert time() < start + 2 assert (len(rm.cds.data['cpu']) == len(rm.cds.data['host']) == len(rm.cds.data['memory']) == 2) assert isinstance(rm.figure, Figure) rm.stream.close()
def test_resource_monitor(loop): with cluster_center(nanny=True) as (c, [a, b]): with scheduler(c['port']) as sport: with Executor(('127.0.0.1', sport)) as e: rm1 = ResourceMonitor(interval=0.01) with Executor(('127.0.0.1', c['port'])) as e: rm2 = ResourceMonitor(interval=0.01) rm3 = ResourceMonitor(('127.0.0.1', sport), interval=0.01) for rm in [rm1, rm2, rm3]: for k in ['cpu', 'memory', 'host']: assert k in rm.cds.data start = time() while not rm.cds.data['cpu']: loop.run_sync(lambda: gen.sleep(0.05)) assert time() < start + 2 assert (len(rm.cds.data['cpu']) == len( rm.cds.data['host']) == len(rm.cds.data['memory']) == 2) assert isinstance(rm.figure, Figure) rm.stream.close()
def test_gather_errors_voluminously(loop): with cluster_center() as (c, [a, b]): try: gather(('127.0.0.1', c['port']), ['x', 'y', 'z']) except KeyError as e: assert set(e.args) == {'x', 'y', 'z'}
def test_scheduler(loop): with cluster_center() as (c, [a, b]): with scheduler(c['port']) as sport: r = rpc(ip='127.0.0.1', port=sport) resp = loop.run_sync(r.ncores) assert len(resp) == 2