def test_multiprogress(s, a, b): sched, report = Queue(), Queue(); s.handle_queues(sched, report) s.update_graph(tasks=valmap(dumps_task, {'x-1': (inc, 1), 'x-2': (inc, 'x-1'), 'x-3': (inc, 'x-2'), 'y-1': (dec, 'x-3'), 'y-2': (dec, 'y-1')}), keys=['y-2'], dependencies={'x-2': ['x-1'], 'x-3': ['x-2'], 'y-1': ['x-3'], 'y-2': ['y-1']}) p = MultiProgress(['y-2'], scheduler=s, func=key_split) yield p.setup() assert p.keys == {'x': {'x-1', 'x-2', 'x-3'}, 'y': {'y-1', 'y-2'}} while True: msg = yield report.get() if msg['op'] == 'key-in-memory' and msg['key'] == 'x-3': break assert p.keys == {'x': set(), 'y': {'y-1', 'y-2'}} while True: msg = yield report.get() if msg['op'] == 'key-in-memory' and msg['key'] == 'y-2': break assert p.keys == {'x': set(), 'y': set()} assert p.status == 'finished'
def test_multiprogress(c, s, a, b): x1 = c.submit(f, 1) x2 = c.submit(f, x1) x3 = c.submit(f, x2) y1 = c.submit(g, x3) y2 = c.submit(g, y1) p = MultiProgress([y2], scheduler=s, complete=True) yield p.setup() assert p.all_keys == { 'f': {f.key for f in [x1, x2, x3]}, 'g': {f.key for f in [y1, y2]} } yield x3 assert p.keys['f'] == set() yield y2 assert p.keys == {'f': set(), 'g': set()} assert p.status == 'finished'
def test_multiprogress(s, a, b): sched, report = Queue(), Queue(); s.handle_queues(sched, report) s.update_graph(dsk={'x-1': (inc, 1), 'x-2': (inc, 'x-1'), 'x-3': (inc, 'x-2'), 'y-1': (dec, 'x-3'), 'y-2': (dec, 'y-1')}, keys=['y-2']) p = MultiProgress(['y-2'], scheduler=s, func=lambda s: s.split('-')[0]) yield p.setup() assert p.keys == {'x': {'x-1', 'x-2', 'x-3'}, 'y': {'y-1', 'y-2'}} while True: msg = yield report.get() if msg['op'] == 'key-in-memory' and msg['key'] == 'x-3': break assert p.keys == {'x': set(), 'y': {'y-1', 'y-2'}} while True: msg = yield report.get() if msg['op'] == 'key-in-memory' and msg['key'] == 'y-2': break assert p.keys == {'x': set(), 'y': set()} assert p.status == 'finished'
async def test_multiprogress(c, s, a, b): x1 = c.submit(f, 1) x2 = c.submit(f, x1) x3 = c.submit(f, x2) y1 = c.submit(g, x3) y2 = c.submit(g, y1) p = MultiProgress([y2], scheduler=s, complete=True) await p.setup() assert p.all_keys == { "f": {f.key for f in [x1, x2, x3]}, "g": {f.key for f in [y1, y2]}, } await x3 assert p.keys["f"] == set() await y2 assert p.keys == {"f": set(), "g": set()} assert p.status == "finished"
def test_multiprogress(s, a, b): sched, report = Queue(), Queue() s.handle_queues(sched, report) s.update_graph(tasks=valmap( dumps_task, { 'x-1': (inc, 1), 'x-2': (inc, 'x-1'), 'x-3': (inc, 'x-2'), 'y-1': (dec, 'x-3'), 'y-2': (dec, 'y-1') }), keys=['y-2'], dependencies={ 'x-2': ['x-1'], 'x-3': ['x-2'], 'y-1': ['x-3'], 'y-2': ['y-1'] }) p = MultiProgress(['y-2'], scheduler=s, func=key_split) yield p.setup() assert p.keys == {'x': {'x-1', 'x-2', 'x-3'}, 'y': {'y-1', 'y-2'}} while True: msg = yield report.get() if msg['op'] == 'key-in-memory' and msg['key'] == 'x-3': break assert p.keys == {'x': set(), 'y': {'y-1', 'y-2'}} while True: msg = yield report.get() if msg['op'] == 'key-in-memory' and msg['key'] == 'y-2': break assert p.keys == {'x': set(), 'y': set()} assert p.status == 'finished'
def test_multiprogress(c, s, a, b): x1 = c.submit(f, 1) x2 = c.submit(f, x1) x3 = c.submit(f, x2) y1 = c.submit(g, x3) y2 = c.submit(g, y1) p = MultiProgress([y2], scheduler=s, complete=True) yield p.setup() assert p.all_keys == {'f': {f.key for f in [x1, x2, x3]}, 'g': {f.key for f in [y1, y2]}} yield x3 assert p.keys['f'] == set() yield y2 assert p.keys == {'f': set(), 'g': set()} assert p.status == 'finished'
async def setup(scheduler): p = MultiProgress(keys, scheduler, complete=complete, func=func) await p.setup() return p