def test_update_state_respects_data_in_memory(loop): s = Scheduler() s.start(0) s.add_worker(address=alice, ncores=1, coerce_address=False) s.update_graph(tasks={'x': 1, 'y': (inc, 'x')}, keys=['y'], dependencies={'y': {'x'}, 'x': set()}, client='client') s.mark_task_finished('x', alice, nbytes=10, type=dumps(int), compute_start=10, compute_stop=11) s.ensure_occupied(alice) s.mark_task_finished('y', alice, nbytes=10, type=dumps(int), compute_start=11, compute_stop=12) s.ensure_occupied(alice) assert s.released == {'x'} assert s.who_has == {'y': {alice}} s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (add, 'y', 'x')}, keys=['z'], dependencies={'y': {'x'}, 'z': {'y', 'x'}}, client='client') assert s.released == set() assert s.waiting == {'z': {'x'}} assert s.processing[alice] == {'x'} # x was released, need to recompute assert s.waiting_data == {'x': {'z'}, 'y': {'z'}, 'z': set()} assert s.who_wants == {'y': {'client'}, 'z': {'client'}} assert s.wants_what == {'client': {'y', 'z'}} s.stop()
def test_update_state_with_processing(loop): s = Scheduler() s.start(0) s.add_worker(address=alice, ncores=1, coerce_address=False) s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (inc, 'y')}, keys=['z'], dependencies={'y': {'x'}, 'x': set(), 'z': {'y'}}, client='client') s.mark_task_finished('x', alice, nbytes=10, type=dumps(int)) assert s.waiting == {'z': {'y'}} assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'z': set()} assert list(s.ready) == [] assert s.who_wants == {'z': {'client'}} assert s.wants_what == {'client': {'z'}} assert s.who_has == {'x': {alice}} s.update_graph(tasks={'a': (inc, 'x'), 'b': (add,'a','y'), 'c': (inc, 'z')}, keys=['b', 'c'], dependencies={'a': {'x'}, 'b': {'a', 'y'}, 'c': {'z'}}, client='client') assert s.waiting == {'z': {'y'}, 'b': {'a', 'y'}, 'c': {'z'}} assert s.stacks[alice] == ['a'] assert not s.ready assert s.waiting_data == {'x': {'y', 'a'}, 'y': {'z', 'b'}, 'z': {'c'}, 'a': {'b'}, 'b': set(), 'c': set()} assert s.who_wants == {'b': {'client'}, 'c': {'client'}, 'z': {'client'}} assert s.wants_what == {'client': {'b', 'c', 'z'}} s.stop()
def test_add_worker_is_idempotent(loop): s = Scheduler(loop=loop) s.start(0) s.add_worker(address=alice, ncores=1, coerce_address=False) ncores = s.ncores.copy() s.add_worker(address=alice, coerce_address=False) assert s.ncores == s.ncores
def test_update_state_supports_recomputing_released_results(loop): s = Scheduler() s.start(0) s.add_worker(address=alice, ncores=1, coerce_address=False) s.update_graph(tasks={ 'x': 1, 'y': (inc, 'x'), 'z': (inc, 'x') }, keys=['z'], dependencies={ 'y': {'x'}, 'x': set(), 'z': {'y'} }, client='client') s.mark_task_finished('x', alice, nbytes=10, type=dumps(int), compute_start=10, compute_stop=11) s.ensure_occupied(alice) s.mark_task_finished('y', alice, nbytes=10, type=dumps(int), compute_start=10, compute_stop=11) s.ensure_occupied(alice) s.mark_task_finished('z', alice, nbytes=10, type=dumps(int), compute_start=10, compute_stop=11) s.ensure_occupied(alice) assert not s.waiting assert not s.ready assert s.waiting_data == {'z': set()} assert s.who_has == {'z': {alice}} s.update_graph(tasks={ 'x': 1, 'y': (inc, 'x') }, keys=['y'], dependencies={'y': {'x'}}, client='client') assert s.waiting == {'y': {'x'}} assert s.waiting_data == {'x': {'y'}, 'y': set(), 'z': set()} assert s.who_wants == {'z': {'client'}, 'y': {'client'}} assert s.wants_what == {'client': {'y', 'z'}} assert set(s.processing[alice]) == {'x'} s.stop()
def test_update_state_supports_recomputing_released_results(loop): s = Scheduler() s.start(0) s.add_worker(address=alice, ncores=1, coerce_address=False) s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (inc, 'x')}, keys=['z'], dependencies={'y': {'x'}, 'x': set(), 'z': {'y'}}, client='client') s.mark_task_finished('x', alice, nbytes=10, type=dumps(int)) s.mark_task_finished('y', alice, nbytes=10, type=dumps(int)) s.mark_task_finished('z', alice, nbytes=10, type=dumps(int)) assert not s.waiting assert not s.ready assert s.waiting_data == {'z': set()} assert s.who_has == {'z': {alice}} s.update_graph(tasks={'x': 1, 'y': (inc, 'x')}, keys=['y'], dependencies={'y': {'x'}}, client='client') assert s.waiting == {'y': {'x'}} assert s.waiting_data == {'x': {'y'}, 'y': set(), 'z': set()} assert s.who_wants == {'z': {'client'}, 'y': {'client'}} assert s.wants_what == {'client': {'y', 'z'}} assert s.processing[alice] == {'x'} s.stop()
def test_update_state_respects_data_in_memory(loop): s = Scheduler() s.start(0) s.add_worker(address='alice', ncores=1) s.update_graph(tasks={'x': 1, 'y': (inc, 'x')}, keys=['y'], dependencies={'y': {'x'}, 'x': set()}, client='client') s.mark_task_finished('x', 'alice', nbytes=10, type=int) s.mark_task_finished('y', 'alice', nbytes=10, type=int) assert s.who_has == {'y': {'alice'}} s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (add, 'y', 'x')}, keys=['z'], dependencies={'y': {'x'}, 'z': {'y', 'x'}}, client='client') assert s.waiting == {'z': {'x'}} assert s.processing['alice'] == {'x'} # x was released, need to recompute assert s.waiting_data == {'x': {'z'}, 'y': {'z'}, 'z': set()} assert s.who_wants == {'y': {'client'}, 'z': {'client'}} assert s.wants_what == {'client': {'y', 'z'}} assert s.in_play == {'x', 'y', 'z'} s.stop()
def test_update_state_with_processing(loop): s = Scheduler() s.start(0) s.add_worker(address=alice, ncores=1, coerce_address=False) s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (inc, 'y')}, keys=['z'], dependencies={'y': {'x'}, 'x': set(), 'z': {'y'}}, client='client') s.mark_task_finished('x', alice, nbytes=10, type=dumps(int)) assert s.waiting == {'z': {'y'}} assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'z': set()} assert list(s.ready) == [] assert s.who_wants == {'z': {'client'}} assert s.wants_what == {'client': {'z'}} assert s.who_has == {'x': {alice}} s.update_graph(tasks={'a': (inc, 'x'), 'b': (add,'a','y'), 'c': (inc, 'z')}, keys=['b', 'c'], dependencies={'a': {'x'}, 'b': {'a', 'y'}, 'c': {'z'}}, client='client') assert s.waiting == {'z': {'y'}, 'b': {'a', 'y'}, 'c': {'z'}} assert s.stacks[alice] == ['a'] assert not s.ready assert s.waiting_data == {'x': {'y', 'a'}, 'y': {'z', 'b'}, 'z': {'c'}, 'a': {'b'}, 'b': set(), 'c': set()} assert s.who_wants == {'b': {'client'}, 'c': {'client'}, 'z': {'client'}} assert s.wants_what == {'client': {'b', 'c', 'z'}} s.stop()
def test_update_state_respects_data_in_memory(loop): s = Scheduler() s.start(0) s.add_worker(address=alice, ncores=1, coerce_address=False) s.update_graph(tasks={'x': 1, 'y': (inc, 'x')}, keys=['y'], dependencies={'y': {'x'}, 'x': set()}, client='client') s.mark_task_finished('x', alice, nbytes=10, type=dumps(int)) s.mark_task_finished('y', alice, nbytes=10, type=dumps(int)) assert s.released == {'x'} assert s.who_has == {'y': {alice}} s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (add, 'y', 'x')}, keys=['z'], dependencies={'y': {'x'}, 'z': {'y', 'x'}}, client='client') assert s.released == set() assert s.waiting == {'z': {'x'}} assert s.processing[alice] == {'x'} # x was released, need to recompute assert s.waiting_data == {'x': {'z'}, 'y': {'z'}, 'z': set()} assert s.who_wants == {'y': {'client'}, 'z': {'client'}} assert s.wants_what == {'client': {'y', 'z'}} s.stop()
def test_update_state(loop): s = Scheduler() s.start(0) s.add_worker(address=alice, ncores=1, coerce_address=False) s.update_graph(tasks={'x': 1, 'y': (inc, 'x')}, keys=['y'], dependencies={'y': 'x', 'x': set()}, client='client') s.mark_task_finished('x', alice, nbytes=10, type=dumps(int), compute_start=10, compute_stop=11) s.ensure_occupied(alice) assert set(s.processing[alice]) == {'y'} assert set(s.rprocessing['y']) == {alice} assert not s.ready assert s.who_wants == {'y': {'client'}} assert s.wants_what == {'client': {'y'}} s.update_graph(tasks={'a': 1, 'z': (add, 'y', 'a')}, keys=['z'], dependencies={'z': {'y', 'a'}}, client='client') assert s.tasks == {'x': 1, 'y': (inc, 'x'), 'a': 1, 'z': (add, 'y', 'a')} assert s.dependencies == {'x': set(), 'a': set(), 'y': {'x'}, 'z': {'a', 'y'}} assert s.dependents == {'z': set(), 'y': {'z'}, 'a': {'z'}, 'x': {'y'}} assert s.waiting == {'z': {'a', 'y'}} assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'a': {'z'}, 'z': set()} assert s.who_wants == {'z': {'client'}, 'y': {'client'}} assert s.wants_what == {'client': {'y', 'z'}} assert 'a' in s.ready or 'a' in s.processing[alice] s.stop()
def test_update_state(loop): s = Scheduler() s.start(0) s.add_worker(address='alice', ncores=1) s.update_graph(tasks={'x': 1, 'y': (inc, 'x')}, keys=['y'], dependencies={'y': 'x', 'x': set()}, client='client') s.mark_task_finished('x', 'alice', nbytes=10, type=int) assert s.processing['alice'] == {'y'} assert not s.ready assert s.who_wants == {'y': {'client'}} assert s.wants_what == {'client': {'y'}} s.update_graph(tasks={'a': 1, 'z': (add, 'y', 'a')}, keys=['z'], dependencies={'z': {'y', 'a'}}, client='client') assert s.tasks == {'x': 1, 'y': (inc, 'x'), 'a': 1, 'z': (add, 'y', 'a')} assert s.dependencies == {'x': set(), 'a': set(), 'y': {'x'}, 'z': {'a', 'y'}} assert s.dependents == {'z': set(), 'y': {'z'}, 'a': {'z'}, 'x': {'y'}} assert s.waiting == {'z': {'a', 'y'}} assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'a': {'z'}, 'z': set()} assert s.who_wants == {'z': {'client'}, 'y': {'client'}} assert s.wants_what == {'client': {'y', 'z'}} assert list(s.ready) == ['a'] assert s.in_play == {'a', 'x', 'y', 'z'} s.stop()