Exemplo n.º 1
0
def test_update_state_respects_data_in_memory(loop):
    s = Scheduler()
    s.start(0)
    s.add_worker(address=alice, ncores=1, coerce_address=False)
    s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
                   keys=['y'],
                   dependencies={'y': {'x'}, 'x': set()},
                   client='client')

    s.mark_task_finished('x', alice, nbytes=10, type=dumps(int),
                         compute_start=10, compute_stop=11)
    s.ensure_occupied(alice)
    s.mark_task_finished('y', alice, nbytes=10, type=dumps(int),
                         compute_start=11, compute_stop=12)
    s.ensure_occupied(alice)

    assert s.released == {'x'}
    assert s.who_has == {'y': {alice}}

    s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (add, 'y', 'x')},
                   keys=['z'],
                   dependencies={'y': {'x'}, 'z': {'y', 'x'}},
                   client='client')

    assert s.released == set()
    assert s.waiting == {'z': {'x'}}
    assert s.processing[alice] == {'x'}  # x was released, need to recompute
    assert s.waiting_data == {'x': {'z'}, 'y': {'z'}, 'z': set()}
    assert s.who_wants == {'y': {'client'}, 'z': {'client'}}
    assert s.wants_what == {'client': {'y', 'z'}}

    s.stop()
Exemplo n.º 2
0
def test_update_state_with_processing(loop):
    s = Scheduler()
    s.start(0)
    s.add_worker(address=alice, ncores=1, coerce_address=False)
    s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (inc, 'y')},
                   keys=['z'],
                   dependencies={'y': {'x'}, 'x': set(), 'z': {'y'}},
                   client='client')

    s.mark_task_finished('x', alice, nbytes=10, type=dumps(int))

    assert s.waiting == {'z': {'y'}}
    assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'z': set()}
    assert list(s.ready) == []

    assert s.who_wants == {'z': {'client'}}
    assert s.wants_what == {'client': {'z'}}

    assert s.who_has == {'x': {alice}}

    s.update_graph(tasks={'a': (inc, 'x'), 'b': (add,'a','y'), 'c': (inc, 'z')},
                   keys=['b', 'c'],
                   dependencies={'a': {'x'}, 'b': {'a', 'y'}, 'c': {'z'}},
                   client='client')

    assert s.waiting == {'z': {'y'}, 'b': {'a', 'y'}, 'c': {'z'}}
    assert s.stacks[alice] == ['a']
    assert not s.ready
    assert s.waiting_data == {'x': {'y', 'a'}, 'y': {'z', 'b'}, 'z': {'c'},
                              'a': {'b'}, 'b': set(), 'c': set()}

    assert s.who_wants == {'b': {'client'}, 'c': {'client'}, 'z': {'client'}}
    assert s.wants_what == {'client': {'b', 'c', 'z'}}

    s.stop()
Exemplo n.º 3
0
def test_add_worker_is_idempotent(loop):
    s = Scheduler(loop=loop)
    s.start(0)
    s.add_worker(address=alice, ncores=1, coerce_address=False)
    ncores = s.ncores.copy()
    s.add_worker(address=alice, coerce_address=False)
    assert s.ncores == s.ncores
Exemplo n.º 4
0
def test_update_state_supports_recomputing_released_results(loop):
    s = Scheduler()
    s.start(0)
    s.add_worker(address=alice, ncores=1, coerce_address=False)
    s.update_graph(tasks={
        'x': 1,
        'y': (inc, 'x'),
        'z': (inc, 'x')
    },
                   keys=['z'],
                   dependencies={
                       'y': {'x'},
                       'x': set(),
                       'z': {'y'}
                   },
                   client='client')

    s.mark_task_finished('x',
                         alice,
                         nbytes=10,
                         type=dumps(int),
                         compute_start=10,
                         compute_stop=11)
    s.ensure_occupied(alice)
    s.mark_task_finished('y',
                         alice,
                         nbytes=10,
                         type=dumps(int),
                         compute_start=10,
                         compute_stop=11)
    s.ensure_occupied(alice)
    s.mark_task_finished('z',
                         alice,
                         nbytes=10,
                         type=dumps(int),
                         compute_start=10,
                         compute_stop=11)
    s.ensure_occupied(alice)

    assert not s.waiting
    assert not s.ready
    assert s.waiting_data == {'z': set()}

    assert s.who_has == {'z': {alice}}

    s.update_graph(tasks={
        'x': 1,
        'y': (inc, 'x')
    },
                   keys=['y'],
                   dependencies={'y': {'x'}},
                   client='client')

    assert s.waiting == {'y': {'x'}}
    assert s.waiting_data == {'x': {'y'}, 'y': set(), 'z': set()}
    assert s.who_wants == {'z': {'client'}, 'y': {'client'}}
    assert s.wants_what == {'client': {'y', 'z'}}
    assert set(s.processing[alice]) == {'x'}

    s.stop()
Exemplo n.º 5
0
def test_update_state_supports_recomputing_released_results(loop):
    s = Scheduler()
    s.start(0)
    s.add_worker(address=alice, ncores=1, coerce_address=False)
    s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (inc, 'x')},
                   keys=['z'],
                   dependencies={'y': {'x'}, 'x': set(), 'z': {'y'}},
                   client='client')

    s.mark_task_finished('x', alice, nbytes=10, type=dumps(int))
    s.mark_task_finished('y', alice, nbytes=10, type=dumps(int))
    s.mark_task_finished('z', alice, nbytes=10, type=dumps(int))

    assert not s.waiting
    assert not s.ready
    assert s.waiting_data == {'z': set()}

    assert s.who_has == {'z': {alice}}

    s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
                   keys=['y'],
                   dependencies={'y': {'x'}},
                   client='client')

    assert s.waiting == {'y': {'x'}}
    assert s.waiting_data == {'x': {'y'}, 'y': set(), 'z': set()}
    assert s.who_wants == {'z': {'client'}, 'y': {'client'}}
    assert s.wants_what == {'client': {'y', 'z'}}
    assert s.processing[alice] == {'x'}

    s.stop()
Exemplo n.º 6
0
def test_update_state_respects_data_in_memory(loop):
    s = Scheduler()
    s.start(0)
    s.add_worker(address='alice', ncores=1)
    s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
                   keys=['y'],
                   dependencies={'y': {'x'}, 'x': set()},
                   client='client')

    s.mark_task_finished('x', 'alice', nbytes=10, type=int)
    s.mark_task_finished('y', 'alice', nbytes=10, type=int)

    assert s.who_has == {'y': {'alice'}}

    s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (add, 'y', 'x')},
                   keys=['z'],
                   dependencies={'y': {'x'}, 'z': {'y', 'x'}},
                   client='client')

    assert s.waiting == {'z': {'x'}}
    assert s.processing['alice'] == {'x'}  # x was released, need to recompute
    assert s.waiting_data == {'x': {'z'}, 'y': {'z'}, 'z': set()}
    assert s.who_wants == {'y': {'client'}, 'z': {'client'}}
    assert s.wants_what == {'client': {'y', 'z'}}
    assert s.in_play == {'x', 'y', 'z'}

    s.stop()
Exemplo n.º 7
0
def test_update_state_with_processing(loop):
    s = Scheduler()
    s.start(0)
    s.add_worker(address=alice, ncores=1, coerce_address=False)
    s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (inc, 'y')},
                   keys=['z'],
                   dependencies={'y': {'x'}, 'x': set(), 'z': {'y'}},
                   client='client')

    s.mark_task_finished('x', alice, nbytes=10, type=dumps(int))

    assert s.waiting == {'z': {'y'}}
    assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'z': set()}
    assert list(s.ready) == []

    assert s.who_wants == {'z': {'client'}}
    assert s.wants_what == {'client': {'z'}}

    assert s.who_has == {'x': {alice}}

    s.update_graph(tasks={'a': (inc, 'x'), 'b': (add,'a','y'), 'c': (inc, 'z')},
                   keys=['b', 'c'],
                   dependencies={'a': {'x'}, 'b': {'a', 'y'}, 'c': {'z'}},
                   client='client')

    assert s.waiting == {'z': {'y'}, 'b': {'a', 'y'}, 'c': {'z'}}
    assert s.stacks[alice] == ['a']
    assert not s.ready
    assert s.waiting_data == {'x': {'y', 'a'}, 'y': {'z', 'b'}, 'z': {'c'},
                              'a': {'b'}, 'b': set(), 'c': set()}

    assert s.who_wants == {'b': {'client'}, 'c': {'client'}, 'z': {'client'}}
    assert s.wants_what == {'client': {'b', 'c', 'z'}}

    s.stop()
Exemplo n.º 8
0
def test_update_state_respects_data_in_memory(loop):
    s = Scheduler()
    s.start(0)
    s.add_worker(address=alice, ncores=1, coerce_address=False)
    s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
                   keys=['y'],
                   dependencies={'y': {'x'}, 'x': set()},
                   client='client')

    s.mark_task_finished('x', alice, nbytes=10, type=dumps(int))
    s.mark_task_finished('y', alice, nbytes=10, type=dumps(int))

    assert s.released == {'x'}
    assert s.who_has == {'y': {alice}}

    s.update_graph(tasks={'x': 1, 'y': (inc, 'x'), 'z': (add, 'y', 'x')},
                   keys=['z'],
                   dependencies={'y': {'x'}, 'z': {'y', 'x'}},
                   client='client')

    assert s.released == set()
    assert s.waiting == {'z': {'x'}}
    assert s.processing[alice] == {'x'}  # x was released, need to recompute
    assert s.waiting_data == {'x': {'z'}, 'y': {'z'}, 'z': set()}
    assert s.who_wants == {'y': {'client'}, 'z': {'client'}}
    assert s.wants_what == {'client': {'y', 'z'}}

    s.stop()
Exemplo n.º 9
0
def test_update_state(loop):
    s = Scheduler()
    s.start(0)
    s.add_worker(address=alice, ncores=1, coerce_address=False)
    s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
                   keys=['y'],
                   dependencies={'y': 'x', 'x': set()},
                   client='client')

    s.mark_task_finished('x', alice, nbytes=10, type=dumps(int),
            compute_start=10, compute_stop=11)
    s.ensure_occupied(alice)

    assert set(s.processing[alice]) == {'y'}
    assert set(s.rprocessing['y']) == {alice}
    assert not s.ready
    assert s.who_wants == {'y': {'client'}}
    assert s.wants_what == {'client': {'y'}}

    s.update_graph(tasks={'a': 1, 'z': (add, 'y', 'a')},
                   keys=['z'],
                   dependencies={'z': {'y', 'a'}},
                   client='client')


    assert s.tasks == {'x': 1, 'y': (inc, 'x'), 'a': 1, 'z': (add, 'y', 'a')}
    assert s.dependencies == {'x': set(), 'a': set(), 'y': {'x'}, 'z': {'a', 'y'}}
    assert s.dependents == {'z': set(), 'y': {'z'}, 'a': {'z'}, 'x': {'y'}}

    assert s.waiting == {'z': {'a', 'y'}}
    assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'a': {'z'}, 'z': set()}

    assert s.who_wants == {'z': {'client'}, 'y': {'client'}}
    assert s.wants_what == {'client': {'y', 'z'}}

    assert 'a' in s.ready or 'a' in s.processing[alice]

    s.stop()
Exemplo n.º 10
0
def test_update_state(loop):
    s = Scheduler()
    s.start(0)
    s.add_worker(address='alice', ncores=1)
    s.update_graph(tasks={'x': 1, 'y': (inc, 'x')},
                   keys=['y'],
                   dependencies={'y': 'x', 'x': set()},
                   client='client')

    s.mark_task_finished('x', 'alice', nbytes=10, type=int)

    assert s.processing['alice'] == {'y'}
    assert not s.ready
    assert s.who_wants == {'y': {'client'}}
    assert s.wants_what == {'client': {'y'}}

    s.update_graph(tasks={'a': 1, 'z': (add, 'y', 'a')},
                   keys=['z'],
                   dependencies={'z': {'y', 'a'}},
                   client='client')


    assert s.tasks == {'x': 1, 'y': (inc, 'x'), 'a': 1, 'z': (add, 'y', 'a')}
    assert s.dependencies == {'x': set(), 'a': set(), 'y': {'x'}, 'z': {'a', 'y'}}
    assert s.dependents == {'z': set(), 'y': {'z'}, 'a': {'z'}, 'x': {'y'}}

    assert s.waiting == {'z': {'a', 'y'}}
    assert s.waiting_data == {'x': {'y'}, 'y': {'z'}, 'a': {'z'}, 'z': set()}

    assert s.who_wants == {'z': {'client'}, 'y': {'client'}}
    assert s.wants_what == {'client': {'y', 'z'}}

    assert list(s.ready) == ['a']
    assert s.in_play == {'a', 'x', 'y', 'z'}

    s.stop()