def test_update_state_respects_data_in_memory(loop):
s = Scheduler()
s.start(0)
s.add_worker(address=alice, ncores=1, coerce_address=False)
s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
keys=['y'],
dependencies={'y': {'x'}, 'x': set()},
client='client')
s.mark_task_finished('x', alice, nbytes=10, type=dumps(int),
compute_start=10, compute_stop=11)
s.ensure_occupied(alice)
s.mark_task_finished('y', alice, nbytes=10, type=dumps(int),
compute_start=11, compute_stop=12)
s.ensure_occupied(alice)
assert s.released == {'x'}
assert s.who_has == {'y': {alice}}
s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (add, 'y', 'x')},
keys=['z'],
dependencies={'y': {'x'}, 'z': {'y', 'x'}},
client='client')
assert s.released == set()
assert s.waiting == {'z': {'x'}}
assert s.processing[alice] == {'x'} # x was released, need to recompute
assert s.waiting_data == {'x': {'z'}, 'y': {'z'}, 'z': set()}
assert s.who_wants == {'y': {'client'}, 'z': {'client'}}
assert s.wants_what == {'client': {'y', 'z'}}
s.stop()
def test_update_state_respects_data_in_memory(loop):
s = Scheduler()
s.start(0)
s.add_worker(address=alice, ncores=1, coerce_address=False)
s.update_graph(tasks={
'x': 1,
'y': (inc, 'x')
},
keys=['y'],
dependencies={
'y': {'x'},
'x': set()
},
client='client')
s.mark_task_finished('x',
alice,
nbytes=10,
type=dumps(int),
compute_start=10,
compute_stop=11)
s.ensure_occupied(alice)
s.mark_task_finished('y',
alice,
nbytes=10,
type=dumps(int),
compute_start=11,
compute_stop=12)
s.ensure_occupied(alice)
assert s.released == {'x'}
assert s.who_has == {'y': {alice}}
s.update_graph(tasks={
'x': 1,
'y': (inc, 'x'),
'z': (add, 'y', 'x')
},
keys=['z'],
dependencies={
'y': {'x'},
'z': {'y', 'x'}
},
client='client')
assert s.released == set()
assert s.waiting == {'z': {'x'}}
assert set(s.processing[alice]) == {'x'
} # x was released need to recompute
assert set(s.rprocessing['x']) == {alice
} # x was released need to recompute
assert s.waiting_data == {'x': {'z'}, 'y': {'z'}, 'z': set()}
assert s.who_wants == {'y': {'client'}, 'z': {'client'}}
assert s.wants_what == {'client': {'y', 'z'}}
s.stop()
def test_update_state(loop):
s = Scheduler()
s.start(0)
s.add_worker(address=alice, ncores=1, coerce_address=False)
s.update_graph(tasks={
'x': 1,
'y': (inc, 'x')
},
keys=['y'],
dependencies={
'y': 'x',
'x': set()
},
client='client')
s.mark_task_finished('x',
alice,
nbytes=10,
type=dumps(int),
compute_start=10,
compute_stop=11)
s.ensure_occupied(alice)
assert set(s.processing[alice]) == {'y'}
assert set(s.rprocessing['y']) == {alice}
assert not s.ready
assert s.who_wants == {'y': {'client'}}
assert s.wants_what == {'client': {'y'}}
s.update_graph(tasks={
'a': 1,
'z': (add, 'y', 'a')
},
keys=['z'],
dependencies={'z': {'y', 'a'}},
client='client')
assert s.tasks == {'x': 1, 'y': (inc, 'x'), 'a': 1, 'z': (add, 'y', 'a')}
assert s.dependencies == {
'x': set(),
'a': set(),
'y': {'x'},
'z': {'a', 'y'}
}
assert s.dependents == {'z': set(), 'y': {'z'}, 'a': {'z'}, 'x': {'y'}}
assert s.waiting == {'z': {'a', 'y'}}
assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'a': {'z'}, 'z': set()}
assert s.who_wants == {'z': {'client'}, 'y': {'client'}}
assert s.wants_what == {'client': {'y', 'z'}}
assert 'a' in s.ready or 'a' in s.processing[alice]
s.stop()
def test_update_state(loop):
s = Scheduler()
s.start(0)
s.add_worker(address=alice, ncores=1, coerce_address=False)
s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
keys=['y'],
dependencies={'y': 'x', 'x': set()},
client='client')
s.mark_task_finished('x', alice, nbytes=10, type=dumps(int),
compute_start=10, compute_stop=11)
s.ensure_occupied(alice)
assert set(s.processing[alice]) == {'y'}
assert set(s.rprocessing['y']) == {alice}
assert not s.ready
assert s.who_wants == {'y': {'client'}}
assert s.wants_what == {'client': {'y'}}
s.update_graph(tasks={'a': 1, 'z': (add, 'y', 'a')},
keys=['z'],
dependencies={'z': {'y', 'a'}},
client='client')
assert s.tasks == {'x': 1, 'y': (inc, 'x'), 'a': 1, 'z': (add, 'y', 'a')}
assert s.dependencies == {'x': set(), 'a': set(), 'y': {'x'}, 'z': {'a', 'y'}}
assert s.dependents == {'z': set(), 'y': {'z'}, 'a': {'z'}, 'x': {'y'}}
assert s.waiting == {'z': {'a', 'y'}}
assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'a': {'z'}, 'z': set()}
assert s.who_wants == {'z': {'client'}, 'y': {'client'}}
assert s.wants_what == {'client': {'y', 'z'}}
assert 'a' in s.ready or 'a' in s.processing[alice]
s.stop()
