def test_eventstream(c, s, *workers):
pytest.importorskip("bokeh")
es = EventStream()
s.add_plugin(es)
assert es.buffer == []
futures = c.map(div, [1] * 10, range(10))
total = c.submit(sum, futures[1:])
yield wait(total)
yield wait(futures)
assert len(es.buffer) == 11
from distributed.diagnostics.progress_stream import task_stream_append
lists = {
name: collections.deque(maxlen=100)
for name in
"start duration key name color worker worker_thread y alpha".split()
}
workers = dict()
for msg in es.buffer:
task_stream_append(lists, msg, workers)
assert len([n for n in lists["name"] if n.startswith("transfer")]) == 2
for name, color in zip(lists["name"], lists["color"]):
if name == "transfer":
assert color == "red"
assert any(c == "black" for c in lists["color"])
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
i = 0
try:
stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
while True:
msgs = yield read(stream)
if not msgs:
continue
last_seen[0] = time()
for msg in msgs:
if 'compute_start' in msg:
deque.append(msg)
times.append(msg['compute_start'])
index.append(i)
i += 1
if msg.get('transfer_start') is not None:
index.append(i)
i += 1
if msg.get('disk_load_start') is not None:
index.append(i)
i += 1
task_stream_append(rectangles, msg, workers)
except StreamClosedError:
pass # don't log StreamClosedErrors
except Exception as e:
logger.exception(e)
finally:
try:
sys.exit(0)
except:
pass
def test_task_stream_append():
msgs = [{'status': 'OK', 'compute_start': 10, 'compute_stop': 20,
'key':'inc-1', 'thread': 5855, 'worker':'127.0.0.1:9999'},
{'status': 'OK', 'compute_start': 15, 'compute_stop': 25,
'key':'inc-2', 'thread': 6000, 'worker':'127.0.0.1:9999'},
{'status': 'error', 'compute_start': 10, 'compute_stop': 14,
'key':'inc-3', 'thread': 6000, 'worker':'127.0.0.1:9999'},
{'status': 'OK', 'compute_start': 10, 'compute_stop': 30,
'transfer_start': 8, 'transfer_stop': 10,
'key':'add-1', 'thread': 4000, 'worker':'127.0.0.2:9999'}]
lists = {name: [] for name in
'start duration key name color worker worker_thread y alpha'.split()}
workers = {'127.0.0.1:9999-5855': 0}
for msg in msgs:
task_stream_append(lists, msg, workers)
assert len(lists['start']) == len(msgs) + 1 # +1 for a transfer
assert len(workers) == 3
assert set(workers) == set(lists['worker_thread'])
assert set(workers.values()) == set(range(len(workers)))
assert lists['color'][-1] == '#FF0020'
L = lists['color']
assert L[0] == L[1]
assert L[2] == 'black'
assert L[3] != L[0]
def test_eventstream(c, s, *workers):
pytest.importorskip('bokeh')
es = EventStream()
s.add_plugin(es)
assert es.buffer == []
futures = c.map(div, [1] * 10, range(10))
total = c.submit(sum, futures[1:])
yield wait(total)
yield wait(futures)
assert len(es.buffer) == 11
from distributed.bokeh import messages
from distributed.diagnostics.progress_stream import task_stream_append
lists = deepcopy(messages['task-events']['rectangles'])
workers = dict()
for msg in es.buffer:
task_stream_append(lists, msg, workers)
assert len([n for n in lists['name'] if n.startswith('transfer')]) == 2
for name, color in zip(lists['name'], lists['color']):
if name == 'transfer':
assert color == 'red'
assert any(c == 'black' for c in lists['color'])
def test_eventstream(c, s, *workers):
pytest.importorskip('bokeh')
es = EventStream()
s.add_plugin(es)
assert es.buffer == []
futures = c.map(div, [1] * 10, range(10))
total = c.submit(sum, futures[1:])
yield wait(total)
yield wait(futures)
assert len(es.buffer) == 11
from distributed.bokeh import messages
from distributed.diagnostics.progress_stream import task_stream_append
lists = deepcopy(messages['task-events']['rectangles'])
workers = dict()
for msg in es.buffer:
task_stream_append(lists, msg, workers)
assert len([n for n in lists['name'] if n.startswith('transfer')]) == 2
for name, color in zip(lists['name'], lists['color']):
if name == 'transfer':
assert color == 'red'
assert any(c == 'black' for c in lists['color'])
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
i = 0
try:
stream = yield eventstream("%(host)s:%(tcp-port)d" % options, 0.100)
while True:
msgs = yield read(stream)
if not msgs:
continue
last_seen[0] = time()
for msg in msgs:
if "compute_start" in msg:
deque.append(msg)
times.append(msg["compute_start"])
index.append(i)
i += 1
if msg.get("transfer_start") is not None:
index.append(i)
i += 1
if msg.get("disk_load_start") is not None:
index.append(i)
i += 1
task_stream_append(rectangles, msg, workers)
except StreamClosedError:
pass # don't log StreamClosedErrors
except Exception as e:
logger.exception(e)
finally:
try:
sys.exit(0)
except:
pass
def test_task_stream_append():
msgs = [{
'status': 'OK',
'compute_start': 10,
'compute_stop': 20,
'key': 'inc-1',
'thread': 5855,
'worker': '127.0.0.1:9999'
}, {
'status': 'OK',
'compute_start': 15,
'compute_stop': 25,
'key': 'inc-2',
'thread': 6000,
'worker': '127.0.0.1:9999'
}, {
'status': 'error',
'compute_start': 10,
'compute_stop': 14,
'key': 'inc-3',
'thread': 6000,
'worker': '127.0.0.1:9999'
}, {
'status': 'OK',
'compute_start': 10,
'compute_stop': 30,
'transfer_start': 8,
'transfer_stop': 10,
'key': 'add-1',
'thread': 4000,
'worker': '127.0.0.2:9999'
}]
lists = {
name: []
for name in
'start duration key name color worker worker_thread y alpha'.split()
}
workers = {'127.0.0.1:9999-5855': 0}
for msg in msgs:
task_stream_append(lists, msg, workers)
assert len(lists['start']) == len(msgs) + 1 # +1 for a transfer
assert len(workers) == 3
assert set(workers) == set(lists['worker_thread'])
assert set(workers.values()) == set(range(len(workers)))
assert lists['color'][-1] == '#FF0020'
L = lists['color']
assert L[0] == L[1]
assert L[2] == 'black'
assert L[3] != L[0]
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
i = 0
try:
stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
while True:
msgs = yield read(stream)
if not msgs:
continue
last_seen[0] = time()
for msg in msgs:
if 'startstops' in msg:
deque.append(msg)
count = task_stream_append(rectangles, msg, workers)
for _ in range(count):
index.append(i)
i += 1
except StreamClosedError:
pass # don't log StreamClosedErrors
except Exception as e:
logger.exception(e)
finally:
try:
sys.exit(0)
except:
pass
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
i = 0
try:
addr = options['scheduler-address']
comm = yield eventstream(addr, 0.100)
while True:
msgs = yield comm.read()
if not msgs:
continue
last_seen[0] = time()
for msg in msgs:
if 'startstops' in msg:
deque.append(msg)
count = task_stream_append(rectangles, msg, workers)
for _ in range(count):
index.append(i)
i += 1
except CommClosedError:
pass # don't log CommClosedErrors
except Exception as e:
logger.exception(e)
