def test_update_state_with_processing(loop):
s = Scheduler()
s.start(0)
s.add_worker(address=alice, ncores=1, coerce_address=False)
s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (inc, 'y')},
keys=['z'],
dependencies={'y': {'x'}, 'x': set(), 'z': {'y'}},
client='client')
s.mark_task_finished('x', alice, nbytes=10, type=dumps(int))
assert s.waiting == {'z': {'y'}}
assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'z': set()}
assert list(s.ready) == []
assert s.who_wants == {'z': {'client'}}
assert s.wants_what == {'client': {'z'}}
assert s.who_has == {'x': {alice}}
s.update_graph(tasks={'a': (inc, 'x'), 'b': (add,'a','y'), 'c': (inc, 'z')},
keys=['b', 'c'],
dependencies={'a': {'x'}, 'b': {'a', 'y'}, 'c': {'z'}},
client='client')
assert s.waiting == {'z': {'y'}, 'b': {'a', 'y'}, 'c': {'z'}}
assert s.stacks[alice] == ['a']
assert not s.ready
assert s.waiting_data == {'x': {'y', 'a'}, 'y': {'z', 'b'}, 'z': {'c'},
'a': {'b'}, 'b': set(), 'c': set()}
assert s.who_wants == {'b': {'client'}, 'c': {'client'}, 'z': {'client'}}
assert s.wants_what == {'client': {'b', 'c', 'z'}}
s.stop()
def test_update_state_respects_data_in_memory(loop):
s = Scheduler()
s.start(0)
s.add_worker(address=alice, ncores=1, coerce_address=False)
s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
keys=['y'],
dependencies={'y': {'x'}, 'x': set()},
client='client')
s.mark_task_finished('x', alice, nbytes=10, type=dumps(int),
compute_start=10, compute_stop=11)
s.ensure_occupied(alice)
s.mark_task_finished('y', alice, nbytes=10, type=dumps(int),
compute_start=11, compute_stop=12)
s.ensure_occupied(alice)
assert s.released == {'x'}
assert s.who_has == {'y': {alice}}
s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (add, 'y', 'x')},
keys=['z'],
dependencies={'y': {'x'}, 'z': {'y', 'x'}},
client='client')
assert s.released == set()
assert s.waiting == {'z': {'x'}}
assert s.processing[alice] == {'x'} # x was released, need to recompute
assert s.waiting_data == {'x': {'z'}, 'y': {'z'}, 'z': set()}
assert s.who_wants == {'y': {'client'}, 'z': {'client'}}
assert s.wants_what == {'client': {'y', 'z'}}
s.stop()
def test_update_state_supports_recomputing_released_results(loop):
s = Scheduler()
s.start(0)
s.add_worker(address=alice, ncores=1, coerce_address=False)
s.update_graph(tasks={
'x': 1,
'y': (inc, 'x'),
'z': (inc, 'x')
},
keys=['z'],
dependencies={
'y': {'x'},
'x': set(),
'z': {'y'}
},
client='client')
s.mark_task_finished('x',
alice,
nbytes=10,
type=dumps(int),
compute_start=10,
compute_stop=11)
s.ensure_occupied(alice)
s.mark_task_finished('y',
alice,
nbytes=10,
type=dumps(int),
compute_start=10,
compute_stop=11)
s.ensure_occupied(alice)
s.mark_task_finished('z',
alice,
nbytes=10,
type=dumps(int),
compute_start=10,
compute_stop=11)
s.ensure_occupied(alice)
assert not s.waiting
assert not s.ready
assert s.waiting_data == {'z': set()}
assert s.who_has == {'z': {alice}}
s.update_graph(tasks={
'x': 1,
'y': (inc, 'x')
},
keys=['y'],
dependencies={'y': {'x'}},
client='client')
assert s.waiting == {'y': {'x'}}
assert s.waiting_data == {'x': {'y'}, 'y': set(), 'z': set()}
assert s.who_wants == {'z': {'client'}, 'y': {'client'}}
assert s.wants_what == {'client': {'y', 'z'}}
assert set(s.processing[alice]) == {'x'}
s.stop()
def f(c, a, b):
e1 = Executor((c.ip, c.port), start=False, loop=loop)
yield e1._start()
assert isinstance(e1.center, rpc)
assert isinstance(e1.scheduler, Scheduler)
s = Scheduler((c.ip, c.port))
yield s.sync_center()
done = s.start()
e2 = Executor(s, start=False, loop=loop)
yield e2._start()
assert isinstance(e2.center, rpc)
assert isinstance(e2.scheduler, Scheduler)
s.listen(8042)
e3 = Executor(('127.0.0.1', s.port), start=False, loop=loop)
yield e3._start()
assert isinstance(e3.center, rpc)
assert isinstance(e3.scheduler, rpc)
s.stop()
yield e1._shutdown()
yield e2._shutdown()
yield e3._shutdown()
def test_update_state_supports_recomputing_released_results(loop):
s = Scheduler()
s.start(0)
s.add_worker(address=alice, ncores=1, coerce_address=False)
s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (inc, 'x')},
keys=['z'],
dependencies={'y': {'x'}, 'x': set(), 'z': {'y'}},
client='client')
s.mark_task_finished('x', alice, nbytes=10, type=dumps(int))
s.mark_task_finished('y', alice, nbytes=10, type=dumps(int))
s.mark_task_finished('z', alice, nbytes=10, type=dumps(int))
assert not s.waiting
assert not s.ready
assert s.waiting_data == {'z': set()}
assert s.who_has == {'z': {alice}}
s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
keys=['y'],
dependencies={'y': {'x'}},
client='client')
assert s.waiting == {'y': {'x'}}
assert s.waiting_data == {'x': {'y'}, 'y': set(), 'z': set()}
assert s.who_wants == {'z': {'client'}, 'y': {'client'}}
assert s.wants_what == {'client': {'y', 'z'}}
assert s.processing[alice] == {'x'}
s.stop()
def test_update_state_respects_data_in_memory(loop):
s = Scheduler()
s.start(0)
s.add_worker(address='alice', ncores=1)
s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
keys=['y'],
dependencies={'y': {'x'}, 'x': set()},
client='client')
s.mark_task_finished('x', 'alice', nbytes=10, type=int)
s.mark_task_finished('y', 'alice', nbytes=10, type=int)
assert s.who_has == {'y': {'alice'}}
s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (add, 'y', 'x')},
keys=['z'],
dependencies={'y': {'x'}, 'z': {'y', 'x'}},
client='client')
assert s.waiting == {'z': {'x'}}
assert s.processing['alice'] == {'x'} # x was released, need to recompute
assert s.waiting_data == {'x': {'z'}, 'y': {'z'}, 'z': set()}
assert s.who_wants == {'y': {'client'}, 'z': {'client'}}
assert s.wants_what == {'client': {'y', 'z'}}
assert s.in_play == {'x', 'y', 'z'}
s.stop()
def test_persist_taskstate():
s = Scheduler(validate=True, persist_file='persist_test')
s.start(0)
assert s.persist_scheduler
s.update_graph(tasks={
'x': dumps_task((inc, 1)),
'y': dumps_task((inc, 'x')),
'z': dumps_task((inc, 2))
},
keys=['y'],
dependencies={
'y': 'x',
'x': [],
'z': []
},
client='client')
taskstates = s.tasks
s.close()
s.stop()
del s
s = Scheduler(validate=True, persist_file='persist_test')
s.start(0)
assert ([taskstates.keys()] == [s.tasks.keys()]
and [x.state for x in taskstates.values()
] == [x.state for x in s.tasks.values()])
s.close()
for f in glob.glob("persist_test*"):
os.remove(f)
def f(c, a, b):
e1 = Executor((c.ip, c.port), start=False, loop=loop)
yield e1._start()
assert isinstance(e1.center, rpc)
assert isinstance(e1.scheduler, Scheduler)
s = Scheduler((c.ip, c.port))
yield s.sync_center()
done = s.start()
e2 = Executor(s, start=False, loop=loop)
yield e2._start()
assert isinstance(e2.center, rpc)
assert isinstance(e2.scheduler, Scheduler)
s.listen(8042)
e3 = Executor(('127.0.0.1', s.port), start=False, loop=loop)
yield e3._start()
assert isinstance(e3.center, rpc)
assert isinstance(e3.scheduler, rpc)
s.stop()
yield e1._shutdown()
yield e2._shutdown()
yield e3._shutdown()
def test_update_state_with_processing(loop):
s = Scheduler()
s.start(0)
s.add_worker(address=alice, ncores=1, coerce_address=False)
s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (inc, 'y')},
keys=['z'],
dependencies={'y': {'x'}, 'x': set(), 'z': {'y'}},
client='client')
s.mark_task_finished('x', alice, nbytes=10, type=dumps(int))
assert s.waiting == {'z': {'y'}}
assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'z': set()}
assert list(s.ready) == []
assert s.who_wants == {'z': {'client'}}
assert s.wants_what == {'client': {'z'}}
assert s.who_has == {'x': {alice}}
s.update_graph(tasks={'a': (inc, 'x'), 'b': (add,'a','y'), 'c': (inc, 'z')},
keys=['b', 'c'],
dependencies={'a': {'x'}, 'b': {'a', 'y'}, 'c': {'z'}},
client='client')
assert s.waiting == {'z': {'y'}, 'b': {'a', 'y'}, 'c': {'z'}}
assert s.stacks[alice] == ['a']
assert not s.ready
assert s.waiting_data == {'x': {'y', 'a'}, 'y': {'z', 'b'}, 'z': {'c'},
'a': {'b'}, 'b': set(), 'c': set()}
assert s.who_wants == {'b': {'client'}, 'c': {'client'}, 'z': {'client'}}
assert s.wants_what == {'client': {'b', 'c', 'z'}}
s.stop()
def test_update_state_respects_data_in_memory(loop):
s = Scheduler()
s.start(0)
s.add_worker(address=alice, ncores=1, coerce_address=False)
s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
keys=['y'],
dependencies={'y': {'x'}, 'x': set()},
client='client')
s.mark_task_finished('x', alice, nbytes=10, type=dumps(int))
s.mark_task_finished('y', alice, nbytes=10, type=dumps(int))
assert s.released == {'x'}
assert s.who_has == {'y': {alice}}
s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (add, 'y', 'x')},
keys=['z'],
dependencies={'y': {'x'}, 'z': {'y', 'x'}},
client='client')
assert s.released == set()
assert s.waiting == {'z': {'x'}}
assert s.processing[alice] == {'x'} # x was released, need to recompute
assert s.waiting_data == {'x': {'z'}, 'y': {'z'}, 'z': set()}
assert s.who_wants == {'y': {'client'}, 'z': {'client'}}
assert s.wants_what == {'client': {'y', 'z'}}
s.stop()
def test_scheduler_file():
with tmpfile() as fn:
s = Scheduler(scheduler_file=fn)
s.start(8009)
w = Worker(scheduler_file=fn)
yield w._start()
assert s.workers == {w.address}
yield w._close()
s.stop()
def test_file_descriptors_dont_leak(loop):
psutil = pytest.importorskip('psutil')
proc = psutil.Process()
before = proc.num_fds()
s = Scheduler()
s.start(0)
w = Worker(s.ip, s.port)
@gen.coroutine
def f():
yield w._start(0)
yield w._close()
loop.run_sync(f)
during = proc.num_fds()
s.stop()
s.close()
start = time()
while proc.num_fds() > before:
loop.run_sync(lambda: gen.sleep(0.01))
assert time() < start + 5
def test_file_descriptors_dont_leak(loop):
psutil = pytest.importorskip('psutil')
proc = psutil.Process()
before = proc.num_fds()
s = Scheduler()
s.start(0)
w = Worker(s.ip, s.port)
@gen.coroutine
def f():
yield w._start(0)
yield w._close()
loop.run_sync(f)
during = proc.num_fds()
s.stop()
s.close()
start = time()
while proc.num_fds() > before:
loop.run_sync(lambda: gen.sleep(0.01))
assert time() < start + 5
def test_update_state(loop):
s = Scheduler()
s.start(0)
s.add_worker(address=alice, ncores=1, coerce_address=False)
s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
keys=['y'],
dependencies={'y': 'x', 'x': set()},
client='client')
s.mark_task_finished('x', alice, nbytes=10, type=dumps(int),
compute_start=10, compute_stop=11)
s.ensure_occupied(alice)
assert set(s.processing[alice]) == {'y'}
assert set(s.rprocessing['y']) == {alice}
assert not s.ready
assert s.who_wants == {'y': {'client'}}
assert s.wants_what == {'client': {'y'}}
s.update_graph(tasks={'a': 1, 'z': (add, 'y', 'a')},
keys=['z'],
dependencies={'z': {'y', 'a'}},
client='client')
assert s.tasks == {'x': 1, 'y': (inc, 'x'), 'a': 1, 'z': (add, 'y', 'a')}
assert s.dependencies == {'x': set(), 'a': set(), 'y': {'x'}, 'z': {'a', 'y'}}
assert s.dependents == {'z': set(), 'y': {'z'}, 'a': {'z'}, 'x': {'y'}}
assert s.waiting == {'z': {'a', 'y'}}
assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'a': {'z'}, 'z': set()}
assert s.who_wants == {'z': {'client'}, 'y': {'client'}}
assert s.wants_what == {'client': {'y', 'z'}}
assert 'a' in s.ready or 'a' in s.processing[alice]
s.stop()
def test_update_state(loop):
s = Scheduler()
s.start(0)
s.add_worker(address='alice', ncores=1)
s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
keys=['y'],
dependencies={'y': 'x', 'x': set()},
client='client')
s.mark_task_finished('x', 'alice', nbytes=10, type=int)
assert s.processing['alice'] == {'y'}
assert not s.ready
assert s.who_wants == {'y': {'client'}}
assert s.wants_what == {'client': {'y'}}
s.update_graph(tasks={'a': 1, 'z': (add, 'y', 'a')},
keys=['z'],
dependencies={'z': {'y', 'a'}},
client='client')
assert s.tasks == {'x': 1, 'y': (inc, 'x'), 'a': 1, 'z': (add, 'y', 'a')}
assert s.dependencies == {'x': set(), 'a': set(), 'y': {'x'}, 'z': {'a', 'y'}}
assert s.dependents == {'z': set(), 'y': {'z'}, 'a': {'z'}, 'x': {'y'}}
assert s.waiting == {'z': {'a', 'y'}}
assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'a': {'z'}, 'z': set()}
assert s.who_wants == {'z': {'client'}, 'y': {'client'}}
assert s.wants_what == {'client': {'y', 'z'}}
assert list(s.ready) == ['a']
assert s.in_play == {'a', 'x', 'y', 'z'}
s.stop()
