def test_decide_worker_with_loose_restrictions(): dependencies = {'x': set()} alice, bob, charlie = 'alice:8000', 'bob:8000', 'charlie:8000' stacks = {alice: [1, 2, 3], bob: [], charlie: [1]} stack_duration = {alice: 3, bob: 0, charlie: 1} processing = {alice: dict(), bob: dict(), charlie: dict()} who_has = {} nbytes = {} ncores = {alice: 1, bob: 1, charlie: 1} restrictions = {'x': {'alice', 'charlie'}} result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, set(), nbytes, ncores, 'x') assert result == charlie result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, {'x'}, nbytes, ncores, 'x') assert result == charlie restrictions = {'x': {'david', 'ethel'}} result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, set(), nbytes, ncores, 'x') assert result is None restrictions = {'x': {'david', 'ethel'}} result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, {'x'}, nbytes, ncores, 'x') assert result == bob
def test_decide_worker_with_loose_restrictions(): dependencies = {'x': set()} alice, bob, charlie = 'alice:8000', 'bob:8000', 'charlie:8000' stacks = {alice: [1, 2, 3], bob: [], charlie: [1]} who_has = {} nbytes = {} restrictions = {'x': {'alice', 'charlie'}} result = decide_worker(dependencies, stacks, who_has, restrictions, set(), nbytes, 'x') assert result == charlie result = decide_worker(dependencies, stacks, who_has, restrictions, {'x'}, nbytes, 'x') assert result == charlie restrictions = {'x': {'david', 'ethel'}} result = decide_worker(dependencies, stacks, who_has, restrictions, set(), nbytes, 'x') assert result is None restrictions = {'x': {'david', 'ethel'}} result = decide_worker(dependencies, stacks, who_has, restrictions, {'x'}, nbytes, 'x') assert result == bob
def test_decide_worker_with_restrictions(): dependencies = {'x': set()} alice, bob, charlie = 'alice:8000', 'bob:8000', 'charlie:8000' stacks = {alice: [], bob: [], charlie: []} processing = {alice: dict(), bob: dict(), charlie: dict()} who_has = {} restrictions = {'x': {'alice', 'charlie'}} nbytes = {} ncores = {alice: 1, bob: 1, charlie: 1} result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, set(), nbytes, ncores, 'x') assert result in {alice, charlie} stacks = {alice: [1, 2, 3], bob: [], charlie: [4, 5, 6]} result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, set(), nbytes, ncores, 'x') assert result in {alice, charlie} dependencies = {'x': {'y'}} who_has = {'y': {bob}} nbytes = {'y': 0} result = decide_worker(dependencies, stacks, stack_duration, processing, who_has, {}, restrictions, set(), nbytes, ncores, 'x') assert result in {alice, charlie}
def test_decide_worker_with_loose_restrictions(): dependencies = {'x': set()} alice, bob, charlie = ('alice', 8000), ('bob', 8000), ('charlie', 8000) stacks = {alice: [1, 2, 3], bob: [], charlie: [1]} who_has = {} nbytes = {} restrictions = {'x': {'alice', 'charlie'}} result = decide_worker(dependencies, stacks, who_has, restrictions, set(), nbytes, 'x') assert result == charlie result = decide_worker(dependencies, stacks, who_has, restrictions, {'x'}, nbytes, 'x') assert result == charlie restrictions = {'x': {'david', 'ethel'}} with pytest.raises(ValueError): result = decide_worker(dependencies, stacks, who_has, restrictions, set(), nbytes, 'x') restrictions = {'x': {'david', 'ethel'}} result = decide_worker(dependencies, stacks, who_has, restrictions, {'x'}, nbytes, 'x') assert result == bob
def test_decide_worker_with_restrictions(): dependencies = {'x': set()} alice, bob, charlie = ('alice', 8000), ('bob', 8000), ('charlie', 8000) stacks = {alice: [], bob: [], charlie: []} who_has = {} restrictions = {'x': {'alice', 'charlie'}} result = decide_worker(dependencies, stacks, who_has, restrictions, 'x') assert result in {alice, charlie} stacks = {alice: [1, 2, 3], bob: [], charlie: [4, 5, 6]} result = decide_worker(dependencies, stacks, who_has, restrictions, 'x') assert result in {alice, charlie} dependencies = {'x': {'y'}} who_has = {'y': {bob}} result = decide_worker(dependencies, stacks, who_has, restrictions, 'x') assert result in {alice, charlie}
def test_decide_worker_with_many_independent_leaves(): dsk = merge({('y', i): (inc, ('x', i)) for i in range(100)}, {('x', i): i for i in range(100)}) dependencies, dependents = get_deps(dsk) stacks = {'alice': [], 'bob': []} who_has = merge({('x', i * 2): {'alice'} for i in range(50)}, {('x', i * 2 + 1): {'bob'} for i in range(50)}) for key in dsk: worker = decide_worker(dependencies, stacks, who_has, {}, key) stacks[worker].append(key) nhits = (len([k for k in stacks['alice'] if 'alice' in who_has[('x', k[1])]]) + len([k for k in stacks['bob'] if 'bob' in who_has[('x', k[1])]])) assert nhits > 90
def test_decide_worker_with_many_independent_leaves(): dsk = merge({('y', i): (inc, ('x', i)) for i in range(100)}, {('x', i): i for i in range(100)}) dependencies, dependents = get_deps(dsk) stacks = {alice: [], bob: []} who_has = merge({('x', i * 2): {alice} for i in range(50)}, {('x', i * 2 + 1): {bob} for i in range(50)}) nbytes = {k: 0 for k in who_has} for key in dsk: worker = decide_worker(dependencies, stacks, who_has, {}, set(), nbytes, key) stacks[worker].append(key) nhits = (len([k for k in stacks[alice] if alice in who_has[('x', k[1])]]) + len([k for k in stacks[bob] if bob in who_has[('x', k[1])]])) assert nhits > 90
def test_decide_worker_without_stacks(): assert not decide_worker({'x': []}, [], {}, {}, set(), {}, 'x')
def test_decide_worker_without_stacks(): assert not decide_worker({'x': []}, {}, {}, {}, {}, {}, {}, set(), {}, {}, 'x')
def test_decide_worker_without_stacks(): with pytest.raises(ValueError): result = decide_worker({'x': []}, [], {}, {}, set(), {}, 'x')