def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
yield s._sync_center()
done = s.start()
sched, report = Queue(), Queue(); s.handle_queues(sched, report)
msg = yield report.get(); assert msg['op'] == 'stream-start'
class Bad(SchedulerPlugin):
def task_finished(self, scheduler, key, worker, nbytes):
raise Exception()
bad = Bad()
s.add_plugin(bad)
sched.put_nowait({'op': 'update-graph',
'dsk': {'x': (inc, 1),
'y': (inc, 'x'),
'z': (inc, 'y')},
'keys': ['z']})
while True: # normal execution
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'z':
break
sched.put_nowait({'op': 'close'})
yield done
def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
yield s._sync_center()
done = s.start()
sched, report = Queue(), Queue(); s.handle_queues(sched, report)
msg = yield report.get(); assert msg['op'] == 'stream-start'
class Counter(SchedulerPlugin):
def start(self, scheduler):
scheduler.add_plugin(self)
self.count = 0
def task_finished(self, scheduler, key, worker, nbytes):
self.count += 1
counter = Counter()
counter.start(s)
assert counter.count == 0
sched.put_nowait({'op': 'update-graph',
'dsk': {'x': (inc, 1),
'y': (inc, 'x'),
'z': (inc, 'y')},
'keys': ['z']})
while True:
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'z':
break
assert counter.count == 3
sched.put_nowait({'op': 'close'})
yield done
def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
yield s._sync_center()
done = s.start()
sched, report = Queue(), Queue(); s.handle_queues(sched, report)
msg = yield report.get(); assert msg['op'] == 'stream-start'
s.update_graph(dsk={'x': (inc, 1),
'y': (inc, 'x'),
'z': (inc, 'y')},
keys=['z'])
progress = TextProgressBar(['z'], scheduler=s)
progress.start()
assert progress.all_keys == {'x', 'y', 'z'}
assert progress.keys == {'x', 'y', 'z'}
while True:
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'z':
break
assert progress.keys == set()
check_bar_completed(capsys)
assert progress not in s.plugins
sched.put_nowait({'op': 'close'})
yield done
def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
yield s._sync_center()
done = s.start()
sched, report = Queue(), Queue(); s.handle_queues(sched, report)
msg = yield report.get(); assert msg['op'] == 'stream-start'
s.update_graph(dsk={'x': (div, 1, 0)},
keys=['x'])
progress = TextProgressBar(['x'], scheduler=s)
progress.start()
while True:
msg = yield report.get()
if msg.get('key') == 'x':
break
assert progress.status == 'error'
assert not progress._timer.is_alive()
progress = TextProgressBar(['x'], scheduler=s)
progress.start()
assert progress.status == 'error'
assert not progress._timer or not progress._timer.is_alive()
sched.put_nowait({'op': 'close'})
yield done
def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
yield s._sync_center()
done = s.start()
sched, report = Queue(), Queue(); s.handle_queues(sched, report)
msg = yield report.get(); assert msg['op'] == 'stream-start'
s.update_graph(dsk={'x-1': (inc, 1),
'x-2': (inc, 'x-1'),
'x-3': (inc, 'x-2'),
'y-1': (dec, 'x-3'),
'y-2': (dec, 'y-1'),
'e': (throws, 'y-2'),
'other': (inc, 123)},
keys=['e'])
while True:
msg = yield report.get()
if msg['op'] == 'task-erred' and msg['key'] == 'e':
break
p = MultiProgressWidget(['e'], scheduler=s, complete=True)
assert set(concat(p.all_keys.values())) == {'x-1', 'x-2', 'x-3', 'y-1',
'y-2', 'e'}
assert all(b.value == 1.0 for b in p.bars.values())
assert p.texts['x'].value == '3 / 3'
assert p.texts['y'].value == '2 / 2'
sched.put_nowait({'op': 'close'})
yield done
def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
yield s._sync_center()
done = s.start()
sched, report = Queue(), Queue(); s.handle_queues(sched, report)
msg = yield report.get(); assert msg['op'] == 'stream-start'
s.update_graph(dsk={'x-1': (inc, 1),
'x-2': (inc, 'x-1'),
'x-3': (inc, 'x-2'),
'y-1': (dec, 'x-3'),
'y-2': (dec, 'y-1'),
'e': (throws, 'y-2'),
'other': (inc, 123)},
keys=['e'])
while True:
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'y-2':
break
p = MultiProgressWidget(['x-1', 'x-2', 'x-3'], scheduler=s)
assert set(concat(p.all_keys.values())).issuperset({'x-1', 'x-2', 'x-3'})
assert 'x' in p.bars
sched.put_nowait({'op': 'close'})
yield done
def f(c, a, b):
s = Scheduler((c.ip, c.port))
yield s._sync_center()
done = s.start()
sched, report = Queue(), Queue()
s.handle_queues(sched, report)
msg = yield report.get()
assert msg['op'] == 'stream-start'
# Test update graph
s.put({'op': 'update-graph',
'dsk': {'x': (inc, 1),
'y': (inc, 'x'),
'z': (inc, 'y')},
'keys': ['z']})
while True:
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'z':
break
assert a.data.get('x') == 2 or b.data.get('x') == 2
# Test erring tasks
s.put({'op': 'update-graph',
'dsk': {'a': (div, 1, 0),
'b': (inc, 'a')},
'keys': ['a', 'b']})
while True:
msg = yield report.get()
if msg['op'] == 'task-erred' and msg['key'] == 'b':
break
# Test missing data
s.put({'op': 'missing-data',
'missing': ['z']})
while True:
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'z':
break
# Test missing data without being informed
for w in [a, b]:
if 'z' in w.data:
del w.data['z']
s.put({'op': 'update-graph',
'dsk': {'zz': (inc, 'z')},
'keys': ['zz']})
while True:
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'zz':
break
s.put({'op': 'close'})
yield done
def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
yield s._sync_center()
done = s.start()
sched, report = Queue(), Queue(); s.handle_queues(sched, report)
msg = yield report.get(); assert msg['op'] == 'stream-start'
p = TextProgressBar([], scheduler=s)
p.start()
assert p.status == 'finished'
check_bar_completed(capsys)
def f(c, a, b):
s = Scheduler((c.ip, c.port))
yield s._sync_center()
done = s.start()
sched, report = Queue(), Queue()
s.handle_queues(sched, report)
msg = yield report.get()
assert msg['op'] == 'stream-start'
# Test update graph
sched.put_nowait({'op': 'update-graph',
'dsk': {'x': (inc, 1),
'y': (inc, 'x'),
'z': (inc, 'y')},
'keys': ['z']})
while True:
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'z':
break
slen, rlen = len(s.scheduler_queues), len(s.report_queues)
sched2, report2 = Queue(), Queue()
s.handle_queues(sched2, report2)
assert slen + 1 == len(s.scheduler_queues)
assert rlen + 1 == len(s.report_queues)
sched2.put_nowait({'op': 'update-graph',
'dsk': {'a': (inc, 10)},
'keys': ['a']})
for q in [report, report2]:
while True:
msg = yield q.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'a':
break
sched.put_nowait({'op': 'close'})
yield done
def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
yield s._sync_center()
done = s.start()
sched, report = Queue(), Queue(); s.handle_queues(sched, report)
msg = yield report.get(); assert msg['op'] == 'stream-start'
s.update_graph(dsk={'x': (inc, 1),
'y': (inc, 'x'),
'z': (inc, 'y')},
keys=['z'])
bars = [Progress(keys=['z'], scheduler=s) for i in range(10)]
while True:
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'z':
break
assert all(b.status == 'finished' for b in bars)
sched.put_nowait({'op': 'close'})
yield done
def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
yield s._sync_center()
done = s.start()
sched, report = Queue(), Queue(); s.handle_queues(sched, report)
msg = yield report.get(); assert msg['op'] == 'stream-start'
s.update_graph(dsk={'x-1': (inc, 1),
'x-2': (inc, 'x-1'),
'x-3': (inc, 'x-2'),
'y-1': (dec, 'x-3'),
'y-2': (dec, 'y-1')},
keys=['y-2'])
p = MultiProgress(['y-2'], scheduler=s, func=lambda s: s.split('-')[0])
assert p.keys == {'x': {'x-1', 'x-2', 'x-3'},
'y': {'y-1', 'y-2'}}
while True:
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'x-3':
break
assert p.keys == {'x': set(),
'y': {'y-1', 'y-2'}}
while True:
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'y-2':
break
assert p.keys == {'x': set(),
'y': set()}
assert p.status == 'finished'
sched.put_nowait({'op': 'close'})
yield done
def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
yield s._sync_center()
done = s.start()
sched, report = Queue(), Queue(); s.handle_queues(sched, report)
msg = yield report.get(); assert msg['op'] == 'stream-start'
s.update_graph(dsk={'x': (inc, 1),
'y': (inc, 'x'),
'z': (inc, 'y')},
keys=['z'])
progress = ProgressWidget(['z'], scheduler=s)
while True:
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'z':
break
progress._update()
assert progress.bar.value == 1.0
assert 's' in progress.bar.description
sched.put_nowait({'op': 'close'})
yield done
def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
yield s._sync_center()
done = s.start()
sched, report = Queue(), Queue(); s.handle_queues(sched, report)
msg = yield report.get(); assert msg['op'] == 'stream-start'
s.update_graph(dsk={'x-1': (inc, 1),
'x-2': (inc, 'x-1'),
'x-3': (inc, 'x-2'),
'y-1': (dec, 'x-3'),
'y-2': (dec, 'y-1'),
'e': (throws, 'y-2'),
'other': (inc, 123)},
keys=['e'])
p = MultiProgressWidget(['e'], scheduler=s)
assert p.keys == {'x': {'x-1', 'x-2', 'x-3'},
'y': {'y-1', 'y-2'},
'e': {'e'}}
while True:
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'x-3':
break
assert p.keys == {'x': set(),
'y': {'y-1', 'y-2'},
'e': {'e'}}
p._update()
assert p.bars['x'].value == 1.0
assert p.bars['y'].value == 0.0
assert p.bars['e'].value == 0.0
assert '3 / 3' in p.texts['x'].value
assert '0 / 2' in p.texts['y'].value
assert '0 / 1' in p.texts['e'].value
while True:
msg = yield report.get()
if msg['op'] == 'key-in-memory' and msg['key'] == 'y-2':
break
p._update()
assert p.bars['x'].value == 1.0
assert p.bars['y'].value == 1.0
assert p.bars['e'].value == 0.0
assert p.keys == {'x': set(),
'y': set(),
'e': {'e'}}
while True:
msg = yield report.get()
if msg['op'] == 'task-erred' and msg['key'] == 'e':
break
assert p.bars['x'].bar_style == 'success'
assert p.bars['y'].bar_style == 'success'
assert p.bars['e'].bar_style == 'danger'
assert p.status == 'error'
sched.put_nowait({'op': 'close'})
yield done