def test_progress_quads():
msg = {'all': {'inc': 5, 'dec': 1, 'add': 4},
'memory': {'inc': 2, 'dec': 0, 'add': 1},
'erred': {'inc': 0, 'dec': 1, 'add': 0},
'released': {'inc': 1, 'dec': 0, 'add': 1}}
_incrementing_index_cache.clear()
d = progress_quads(msg, nrows=2)
color = d.pop('color')
assert len(set(color)) == 3
assert d == {'name': ['inc', 'add', 'dec'],
'show-name': ['inc', 'add', 'dec'],
'left': [0, 0, 1],
'right': [0.9, 0.9, 1.9],
'top': [0, -1, 0],
'bottom': [-.8, -1.8, -.8],
'all': [5, 4, 1],
'released': [1, 1, 0],
'memory': [2, 1, 0],
'erred': [0, 0, 1],
'done': ['3 / 5', '2 / 4', '1 / 1'],
'released-loc': [.9 * 1 / 5, .25 * 0.9, 1.0],
'memory-loc': [.9 * 3 / 5, .5 * 0.9, 1.0],
'erred-loc': [.9 * 3 / 5, .5 * 0.9, 1.9]}
def test_progress_quads():
msg = {'all': {'inc': 5, 'dec': 1, 'add': 4},
'memory': {'inc': 2, 'dec': 0, 'add': 1},
'erred': {'inc': 0, 'dec': 1, 'add': 0},
'released': {'inc': 1, 'dec': 0, 'add': 1},
'processing': {'inc': 1, 'dec': 0, 'add': 2}}
d = progress_quads(msg, nrows=2)
color = d.pop('color')
assert len(set(color)) == 3
expected = {'name': ['inc', 'add', 'dec'],
'show-name': ['inc', 'add', 'dec'],
'left': [0, 0, 1],
'right': [0.9, 0.9, 1.9],
'top': [0, -1, 0],
'bottom': [-.8, -1.8, -.8],
'all': [5, 4, 1],
'released': [1, 1, 0],
'memory': [2, 1, 0],
'erred': [0, 0, 1],
'processing': [1, 2, 0],
'done': ['3 / 5', '2 / 4', '1 / 1'],
'released-loc': [.9 * 1 / 5, .25 * 0.9, 1.0],
'memory-loc': [.9 * 3 / 5, .5 * 0.9, 1.0],
'erred-loc': [.9 * 3 / 5, .5 * 0.9, 1.9],
'processing-loc': [.9 * 4 / 5, 1 * 0.9, 1 * 0.9 + 1]}
assert d == expected
def test_progress_stream(e, s, a, b):
futures = e.map(div, [1] * 10, range(10))
x = 1
for i in range(5):
x = do(inc)(x)
future = e.compute(x)
yield _wait(futures + [future])
stream = yield progress_stream(s.address, interval=0.010)
msg = yield read(stream)
assert msg == {'all': {'div': 10, 'inc': 5, 'finalize': 1},
'erred': {'div': 1},
'in_memory': {'div': 9, 'finalize': 1},
'released': {'div': 1, 'inc': 5}}
d = progress_quads(msg)
assert d == {'name': ['div', 'inc', 'finalize'],
'all': [10, 5, 1],
'in_memory': [9, 0, 1],
'in_memory_right': [1, 1, 1],
'fraction': ['10 / 10', '5 / 5', '1 / 1'],
'erred': [1, 0, 0],
'erred_left': [0.9, 1, 1],
'released': [1, 5, 0],
'released_right': [0.1, 1, 0],
'top': [0.7, 1.7, 2.7],
'center': [0.5, 1.5, 2.5],
'bottom': [0.3, 1.3, 2.3]}
stream.close()
def test_progress_quads():
msg = {
'all': {
'inc': 5,
'dec': 1
},
'in_memory': {
'inc': 2,
'dec': 0
},
'erred': {
'inc': 0,
'dec': 1
},
'released': {
'inc': 1,
'dec': 0
}
}
d = progress_quads(msg)
assert d['name'] == ['inc', 'dec']
assert d['all'] == [5, 1]
assert d['in_memory'] == [2, 0]
assert d['fraction'] == ['3 / 5', '0 / 1']
assert d['erred'] == [0, 1]
assert d['released'] == [1, 0]
assert d['released_right'] == [1 / 5, 0]
assert d['in_memory_right'] == [3 / 5, 0]
assert d['erred_left'] == [1, 0]
def test_progress_stream(c, s, a, b):
futures = c.map(div, [1] * 10, range(10))
x = 1
for i in range(5):
x = delayed(inc)(x)
future = c.compute(x)
yield wait(futures + [future])
comm = yield progress_stream(s.address, interval=0.010)
msg = yield comm.read()
nbytes = msg.pop('nbytes')
assert msg == {
'all': {
'div': 10,
'inc': 5
},
'erred': {
'div': 1
},
'memory': {
'div': 9,
'inc': 1
},
'released': {
'inc': 4
}
}
assert set(nbytes) == set(msg['all'])
assert all(v > 0 for v in nbytes.values())
assert progress_quads(msg)
yield comm.close()
def test_progress_quads_too_many():
keys = ['x-%d' % i for i in range(1000)]
msg = {'all': {k: 1 for k in keys},
'memory': {k: 0 for k in keys},
'erred': {k: 0 for k in keys},
'released': {k: 0 for k in keys}}
d = progress_quads(msg, nrows=6, ncols=3)
assert len(d['name']) == 6 * 3
def progress_update():
with log_errors():
msg = messages['progress']
d = progress_quads(msg)
progress_source.data.update(d)
progress_plot.title.text = ("Progress -- total: %(total)s, "
"in-memory: %(in-memory)s, processing: %(processing)s, "
"ready: %(ready)s, waiting: %(waiting)s, failed: %(failed)s"
% messages['tasks']['deque'][-1])
def test_progress_quads_too_many():
keys = ['x-%d' % i for i in range(1000)]
msg = {'all': {k: 1 for k in keys},
'memory': {k: 0 for k in keys},
'erred': {k: 0 for k in keys},
'released': {k: 0 for k in keys}}
d = progress_quads(msg, nrows=6, ncols=3)
assert len(d['name']) == 6 * 3
def test_progress_quads_many_functions():
funcnames = ['fn%d' % i for i in range(1000)]
msg = {'all': {fn: 1 for fn in funcnames},
'memory': {fn: 1 for fn in funcnames},
'erred': {fn: 0 for fn in funcnames},
'released': {fn: 0 for fn in funcnames}}
d = progress_quads(msg, nrows=2)
color = d.pop('color')
assert len(set(color)) < 100
def test_progress_quads_many_functions():
funcnames = ['fn%d' % i for i in range(1000)]
msg = {'all': {fn: 1 for fn in funcnames},
'memory': {fn: 1 for fn in funcnames},
'erred': {fn: 0 for fn in funcnames},
'released': {fn: 0 for fn in funcnames}}
d = progress_quads(msg, nrows=2)
color = d.pop('color')
assert len(set(color)) < 100
def update(self, messages):
with log_errors():
msg = messages['progress']
if not msg:
return
d = progress_quads(msg)
self.source.data.update(d)
if messages['tasks']['deque']:
self.root.title.text = ("Progress -- total: %(total)s, "
"in-memory: %(in-memory)s, processing: %(processing)s, "
"waiting: %(waiting)s, failed: %(failed)s"
% messages['tasks']['deque'][-1])
def update(self, messages):
with log_errors():
msg = messages['progress']
if not msg:
return
d = progress_quads(msg)
self.source.data.update(d)
if messages['tasks']['deque']:
self.root.title.text = ("Progress -- total: %(total)s, "
"in-memory: %(in-memory)s, processing: %(processing)s, "
"waiting: %(waiting)s, failed: %(failed)s"
% messages['tasks']['deque'][-1])
def test_progress_quads():
msg = {
'all': {
'inc': 5,
'dec': 1,
'add': 4
},
'memory': {
'inc': 2,
'dec': 0,
'add': 1
},
'erred': {
'inc': 0,
'dec': 1,
'add': 0
},
'released': {
'inc': 1,
'dec': 0,
'add': 1
},
'processing': {
'inc': 1,
'dec': 0,
'add': 2
}
}
d = progress_quads(msg, nrows=2)
color = d.pop('color')
assert len(set(color)) == 3
expected = {
'name': ['inc', 'add', 'dec'],
'show-name': ['inc', 'add', 'dec'],
'left': [0, 0, 1],
'right': [0.9, 0.9, 1.9],
'top': [0, -1, 0],
'bottom': [-.8, -1.8, -.8],
'all': [5, 4, 1],
'released': [1, 1, 0],
'memory': [2, 1, 0],
'erred': [0, 0, 1],
'processing': [1, 2, 0],
'done': ['3 / 5', '2 / 4', '1 / 1'],
'released-loc': [.9 * 1 / 5, .25 * 0.9, 1.0],
'memory-loc': [.9 * 3 / 5, .5 * 0.9, 1.0],
'erred-loc': [.9 * 3 / 5, .5 * 0.9, 1.9],
'processing-loc': [.9 * 4 / 5, 1 * 0.9, 1 * 0.9 + 1]
}
assert d == expected
def test_progress_quads():
msg = {
"all": {
"inc": 5,
"dec": 1,
"add": 4
},
"memory": {
"inc": 2,
"dec": 0,
"add": 1
},
"erred": {
"inc": 0,
"dec": 1,
"add": 0
},
"released": {
"inc": 1,
"dec": 0,
"add": 1
},
"processing": {
"inc": 1,
"dec": 0,
"add": 2
},
}
d = progress_quads(msg, nrows=2)
color = d.pop("color")
assert len(set(color)) == 3
expected = {
"name": ["inc", "add", "dec"],
"show-name": ["inc", "add", "dec"],
"left": [0, 0, 1],
"right": [0.9, 0.9, 1.9],
"top": [0, -1, 0],
"bottom": [-0.8, -1.8, -0.8],
"all": [5, 4, 1],
"released": [1, 1, 0],
"memory": [2, 1, 0],
"erred": [0, 0, 1],
"processing": [1, 2, 0],
"done": ["3 / 5", "2 / 4", "1 / 1"],
"released-loc": [0.9 * 1 / 5, 0.25 * 0.9, 1.0],
"memory-loc": [0.9 * 3 / 5, 0.5 * 0.9, 1.0],
"erred-loc": [0.9 * 3 / 5, 0.5 * 0.9, 1.9],
"processing-loc": [0.9 * 4 / 5, 1 * 0.9, 1 * 0.9 + 1],
}
assert d == expected
def progress_update():
with log_errors():
msg = messages['progress']
d = progress_quads(msg)
progress_source.data.update(d)
progress_plot.title.text = ("Progress -- total: %(total)s, "
"in-memory: %(in-memory)s, processing: %(processing)s, "
"ready: %(ready)s, waiting: %(waiting)s, failed: %(failed)s"
% messages['tasks']['deque'][-1])
nb = nbytes_bar(msg['nbytes'])
nbytes_task_source.data.update(nb)
nbytes_task_plot.title.text = \
"Memory Use: %0.2f MB" % (sum(msg['nbytes'].values()) / 1e6)
def test_progress_stream(e, s, a, b):
futures = e.map(div, [1] * 10, range(10))
x = 1
for i in range(5):
x = do(inc)(x)
future = e.compute(x)
yield _wait(futures + [future])
stream = yield progress_stream(s.address, interval=0.010)
msg = yield read(stream)
assert msg == {
'all': {
'div': 10,
'inc': 5,
'finalize': 1
},
'erred': {
'div': 1
},
'in_memory': {
'div': 9,
'finalize': 1
},
'released': {
'div': 1,
'inc': 5
}
}
d = progress_quads(msg)
assert d == {
'name': ['div', 'inc', 'finalize'],
'all': [10, 5, 1],
'in_memory': [9, 0, 1],
'in_memory_right': [1, 1, 1],
'fraction': ['10 / 10', '5 / 5', '1 / 1'],
'erred': [1, 0, 0],
'erred_left': [0.9, 1, 1],
'released': [1, 5, 0],
'released_right': [0.1, 1, 0],
'top': [0.7, 1.7, 2.7],
'center': [0.5, 1.5, 2.5],
'bottom': [0.3, 1.3, 2.3]
}
stream.close()
def progress_update():
with log_errors():
msg = messages['progress']
d = progress_quads(msg)
progress_source.data.update(d)
progress_plot.title.text = (
"Progress -- total: %(total)s, "
"in-memory: %(in-memory)s, processing: %(processing)s, "
"ready: %(ready)s, waiting: %(waiting)s, failed: %(failed)s" %
messages['tasks']['deque'][-1])
nb = nbytes_bar(msg['nbytes'])
nbytes_task_source.data.update(nb)
nbytes_task_plot.title.text = \
"Memory Use: %0.2f MB" % (sum(msg['nbytes'].values()) / 1e6)
def test_progress_quads():
msg = {'all': {'inc': 5, 'dec': 1},
'in_memory': {'inc': 2, 'dec': 0},
'erred': {'inc': 0, 'dec': 1},
'released': {'inc': 1, 'dec': 0}}
d = progress_quads(msg)
assert d['name'] == ['inc', 'dec']
assert d['all'] == [5, 1]
assert d['in_memory'] == [2, 0]
assert d['fraction'] == ['3 / 5', '0 / 1']
assert d['erred'] == [0, 1]
assert d['released'] == [1, 0]
assert d['released_right'] == [1/5, 0]
assert d['in_memory_right'] == [3 / 5, 0]
assert d['erred_left'] == [1, 0]
def test_progress_quads_too_many():
keys = ["x-%d" % i for i in range(1000)]
msg = {
"all": {k: 1
for k in keys},
"memory": {k: 0
for k in keys},
"erred": {k: 0
for k in keys},
"released": {k: 0
for k in keys},
"processing": {k: 0
for k in keys},
}
d = progress_quads(msg, nrows=6, ncols=3)
assert len(d["name"]) == 6 * 3
def test_progress_quads_many_functions():
funcnames = ["fn%d" % i for i in range(1000)]
msg = {
"all": {fn: 1
for fn in funcnames},
"memory": {fn: 1
for fn in funcnames},
"erred": {fn: 0
for fn in funcnames},
"released": {fn: 0
for fn in funcnames},
"processing": {fn: 0
for fn in funcnames},
}
d = progress_quads(msg, nrows=2)
color = d.pop("color")
assert len(set(color)) < 100
def test_progress_quads():
msg = {'all': {'inc': 5, 'dec': 1, 'add': 4},
'memory': {'inc': 2, 'dec': 0, 'add': 1},
'erred': {'inc': 0, 'dec': 1, 'add': 0},
'released': {'inc': 1, 'dec': 0, 'add': 1}}
d = progress_quads(msg, nrows=2)
assert d == {'name': ['inc', 'add', 'dec'],
'show-name': ['inc', 'add', 'dec'],
'left': [0, 0, 1],
'right': [0.9, 0.9, 1.9],
'top': [0, -1, 0],
'bottom': [-.8, -1.8, -.8],
'all': [5, 4, 1],
'released': [1, 1, 0],
'memory': [2, 1, 0],
'erred': [0, 0, 1],
'done': ['3 / 5', '2 / 4', '1 / 1'],
'released-loc': [.9 * 1 / 5, .25 * 0.9, 1.0],
'memory-loc': [.9 * 3 / 5, .5 * 0.9, 1.0],
'erred-loc': [.9 * 3 / 5, .5 * 0.9, 1.9]}
def test_progress_stream(c, s, a, b):
futures = c.map(div, [1] * 10, range(10))
x = 1
for i in range(5):
x = do(inc)(x)
future = c.compute(x)
yield _wait(futures + [future])
stream = yield progress_stream(s.address, interval=0.010)
msg = yield read(stream)
nbytes = msg.pop('nbytes')
assert msg == {'all': {'div': 10, 'inc': 5, 'finalize': 1},
'erred': {'div': 1},
'memory': {'div': 9, 'finalize': 1},
'released': {'inc': 5}}
assert set(nbytes) == set(msg['all'])
assert all(v > 0 for v in nbytes.values())
assert progress_quads(msg)
stream.close()
async def test_progress_stream(c, s, a, b):
futures = c.map(div, [1] * 10, range(10))
x = 1
for i in range(5):
x = delayed(inc)(x)
future = c.compute(x)
await wait(futures + [future])
comm = await progress_stream(s.address, interval=0.010)
msg = await comm.read()
nbytes = msg.pop("nbytes")
assert msg == {
"all": {
"div": 10,
"inc": 5
},
"erred": {
"div": 1
},
"memory": {
"div": 9,
"inc": 1
},
"released": {
"inc": 4
},
"processing": {},
}
assert set(nbytes) == set(msg["all"])
assert all(v > 0 for v in nbytes.values())
assert progress_quads(msg)
await comm.close()
def test_progress_stream(e, s, a, b):
futures = e.map(div, [1] * 10, range(10))
x = 1
for i in range(5):
x = do(inc)(x)
future = e.compute(x)
yield _wait(futures + [future])
stream = yield progress_stream(s.address, interval=0.010)
msg = yield read(stream)
nbytes = msg.pop('nbytes')
assert msg == {
'all': {
'div': 10,
'inc': 5,
'finalize': 1
},
'erred': {
'div': 1
},
'memory': {
'div': 9,
'finalize': 1
},
'released': {
'inc': 5
}
}
assert set(nbytes) == set(msg['all'])
assert all(v > 0 for v in nbytes.values())
assert progress_quads(msg)
stream.close()
def __init__(self, **kwargs):
data = progress_quads(dict(all={}, memory={}, erred={}, released={}))
self.source = ColumnDataSource(data=data)
x_range = DataRange1d()
y_range = Range1d(-8, 0)
self.root = Plot(id='bk-task-progress-plot',
x_range=x_range,
y_range=y_range,
toolbar_location=None,
**kwargs)
self.root.add_glyph(
self.source,
Quad(top='top',
bottom='bottom',
left='left',
right='right',
fill_color="#aaaaaa",
line_color="#aaaaaa",
fill_alpha=0.2))
self.root.add_glyph(
self.source,
Quad(top='top',
bottom='bottom',
left='left',
right='released-loc',
fill_color="color",
line_color="color",
fill_alpha=0.6))
self.root.add_glyph(
self.source,
Quad(top='top',
bottom='bottom',
left='released-loc',
right='memory-loc',
fill_color="color",
line_color="color",
fill_alpha=1.0))
self.root.add_glyph(
self.source,
Quad(top='top',
bottom='bottom',
left='erred-loc',
right='erred-loc',
fill_color='#000000',
line_color='#000000',
fill_alpha=0.3))
self.root.add_glyph(
self.source,
Text(text='show-name',
y='bottom',
x='left',
x_offset=5,
text_font_size=value('10pt')))
self.root.add_glyph(
self.source,
Text(text='done',
y='bottom',
x='right',
x_offset=-5,
text_align='right',
text_font_size=value('10pt')))
hover = HoverTool(point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 14px; font-weight: bold;">Name:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">All:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@all</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Memory:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@memory</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Erred:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@erred</span>
</div>
""")
self.root.add_tools(hover)
def progress_update():
with log_errors():
msg = messages['progress']
d = progress_quads(msg)
progress_source.data.update(d)
def __init__(self, **kwargs):
data = progress_quads(dict(all={}, memory={}, erred={}, released={}))
self.source = ColumnDataSource(data=data)
x_range = DataRange1d()
y_range = Range1d(-8, 0)
self.root = Plot(
id='bk-task-progress-plot',
x_range=x_range, y_range=y_range, toolbar_location=None, **kwargs
)
self.root.add_glyph(
self.source,
Quad(top='top', bottom='bottom', left='left', right='right',
fill_color="#aaaaaa", line_color="#aaaaaa", fill_alpha=0.2)
)
self.root.add_glyph(
self.source,
Quad(top='top', bottom='bottom', left='left', right='released-loc',
fill_color="color", line_color="color", fill_alpha=0.6)
)
self.root.add_glyph(
self.source,
Quad(top='top', bottom='bottom', left='released-loc',
right='memory-loc', fill_color="color", line_color="color",
fill_alpha=1.0)
)
self.root.add_glyph(
self.source,
Quad(top='top', bottom='bottom', left='erred-loc',
right='erred-loc', fill_color='#000000', line_color='#000000',
fill_alpha=0.3)
)
self.root.add_glyph(
self.source,
Text(text='show-name', y='bottom', x='left', x_offset=5,
text_font_size=value('10pt'))
)
self.root.add_glyph(
self.source,
Text(text='done', y='bottom', x='right', x_offset=-5,
text_align='right', text_font_size=value('10pt'))
)
hover = HoverTool(
point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 14px; font-weight: bold;">Name:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">All:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@all</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Memory:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@memory</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Erred:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@erred</span>
</div>
"""
)
self.root.add_tools(hover)
def progress_update():
with log_errors():
msg = messages['progress']
d = progress_quads(msg)
progress_source.data.update(d)
