Ejemplo n.º 1
0
def test_request_more_than_one(loop):
    with SGECluster(scheduler_port=0) as cluster:
        adapt = Adaptive(cluster)
        with Client(cluster, loop=loop) as client:
            futures = client.map(slowinc, range(1000), delay=0.2)
            while len(cluster.scheduler.workers) < 3:
                sleep(0.1)
Ejemplo n.º 2
0
def test_dont_request_if_not_enough_tasks(loop):
    with SGECluster(scheduler_port=0) as cluster:
        adapt = Adaptive(cluster)
        with Client(cluster, loop=loop) as client:
            cluster.scheduler.task_duration['slowinc'] = 1000
            future = client.submit(slowinc, 1, delay=1000)

            for i in range(50):
                sleep(0.1)
                assert len(cluster.workers) < 2
Ejemplo n.º 3
0
def test_dont_over_request(loop, interval):
    with SGECluster(scheduler_port=0) as cluster:
        adapt = Adaptive(cluster)
        with Client(cluster, loop=loop) as client:
            future = client.submit(inc, 1)
            assert future.result() == 2
            assert len(cluster.scheduler.workers) == 1

            for i in range(5):
                sleep(0.2)
                assert len(cluster.scheduler.workers) == 1
Ejemplo n.º 4
0
def test_dont_request_on_many_short_tasks(loop):
    with SGECluster(scheduler_port=0) as cluster:
        adapt = Adaptive(cluster, interval=50, startup_cost=10)
        with Client(cluster, loop=loop) as client:
            cluster.scheduler.task_duration['slowinc'] = 0.001
            futures = client.map(slowinc, range(1000), delay=0.001)

            while not cluster.scheduler.workers:
                sleep(0.01)

            for i in range(20):
                sleep(0.1)
                assert len(cluster.workers) < 2
Ejemplo n.º 5
0
def test_dont_request_if_idle(loop):
    with SGECluster(scheduler_port=0) as cluster:
        cluster.start_workers(1)
        with Client(cluster, loop=loop) as client:
            while not cluster.scheduler.workers:
                sleep(0.1)
            futures = client.map(slowinc,
                                 range(1000),
                                 delay=0.2,
                                 workers=first(cluster.scheduler.workers))
            adapt = Adaptive(cluster, interval=2000)

            for i in range(60):
                sleep(0.1)
                assert len(cluster.workers) < 5
Ejemplo n.º 6
0
def test_adaptive_memory(loop):
    with SGECluster(scheduler_port=0) as cluster:
        adapt = Adaptive(cluster=cluster)
        with Client(cluster, loop=loop) as client:
            future = client.submit(inc, 1, resources={'memory': 1e9})
            assert future.result() == 2
            assert len(cluster.scheduler.ncores) > 0
            r = list(cluster.scheduler.worker_resources.values())[0]
            assert r['memory'] > 1e9

            del future

            start = time()
            while client.ncores():
                sleep(0.3)
                assert time() < start + 10
            """ # TODO: jobs aren't shutting down when process ends
Ejemplo n.º 7
0
def test_adaptive_memory(loop):
    with SGECluster(scheduler_port=0, cleanup_interval=100) as cluster:
        adapt = Adaptive(cluster)
        with Client(cluster, loop=loop) as client:
            future = client.submit(inc, 1, resources={'memory': 1e9})
            assert future.result() == 2
            assert len(cluster.scheduler.ncores) > 0
            r = list(cluster.scheduler.worker_resources.values())[0]
            assert r['memory'] > 1e9

            del future

            start = time()
            while client.ncores():
                sleep(0.3)
                assert time() < start + 10

            start = time()
            while cluster.workers:
                sleep(0.1)
                assert time() < start + 10
Ejemplo n.º 8
0
def test_adaptive_normal_tasks(loop):
    with SGECluster(scheduler_port=0) as cluster:
        adapt = Adaptive(cluster)
        with Client(cluster, loop=loop) as client:
            future = client.submit(inc, 1)
            assert future.result() == 2
Ejemplo n.º 9
0
def test_order_warns(loop):
    with SGECluster(scheduler_port=0) as cluster:
        scheduler = cluster.scheduler
        with pytest.warns(FutureWarning):
            adapt = Adaptive(scheduler, cluster)