def make_plot(yname, line_color, below_axis=True, left_axis=True, right_axis=False, border_fill_color="white"):
""" Returns a tuple (plot, [obj1...objN]); the former can be added
to a GridPlot, and the latter is added to the plotcontext.
"""
plot = Plot(
x_range=DataRange1d(),
y_range=DataRange1d(),
min_border=15,
border_fill_color=border_fill_color,
border_fill_alpha=0.1,
toolbar_location='above',
plot_width=300,
plot_height=300,
responsive='box'
)
if below_axis:
plot.add_layout(LinearAxis(), 'below')
else:
plot.add_layout(LinearAxis(), 'above')
if left_axis:
plot.add_layout(LinearAxis(), 'left')
if right_axis:
plot.add_layout(LinearAxis(), 'right')
plot.add_glyph(source, Line(x="x", y=yname, line_color=line_color))
plot.add_tools(PanTool(), WheelZoomTool(), BoxSelectTool())
return plot
def modify_doc(doc):
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("data", "s.data"))))
button = Button(css_classes=["foo"])
button.js_on_click(CustomJS(args=dict(s=source), code="s.patch({'x': [[1, 100]]})"))
doc.add_root(column(button, plot))
def get_ws_plot(
data,
plot_height=300,
plot_width=800,
column_names=cs.column_names_weewx,
column_time=cs.time_column_name_weewx,
border_left=200,
):
data_seconds = data
data_seconds.iloc[:, 0] = data_seconds.iloc[:, 0] * 1000
plot = Plot(
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_width=plot_width,
plot_height=plot_height,
min_border_left=border_left,
**kwargs
)
add_glyphs_to_plot(column_names, column_time, data_seconds, plot)
plot.add_layout(DatetimeAxis(), "below")
plot.add_tools(PanTool(), WheelZoomTool(), ResizeTool(), CrosshairTool())
return plot
def make_plot(xname, yname, xax=False, yax=False):
mbl = 40 if yax else 0
mbb = 40 if xax else 0
plot = Plot(
x_range=xdr, y_range=ydr, background_fill_color="#efe8e2",
border_fill_color='white', title="", h_symmetry=False, v_symmetry=False,
plot_width=200 + mbl, plot_height=200 + mbb, min_border_left=2+mbl, min_border_right=2,
min_border_top=2, min_border_bottom=2+mbb)
circle = Circle(x=xname, y=yname, fill_color="color", fill_alpha=0.2, size=4, line_color="color")
r = plot.add_glyph(source, circle)
xdr.renderers.append(r)
ydr.renderers.append(r)
xticker = BasicTicker()
if xax:
xaxis = LinearAxis()
plot.add_layout(xaxis, 'below')
xticker = xaxis.ticker
plot.add_layout(Grid(dimension=0, ticker=xticker))
yticker = BasicTicker()
if yax:
yaxis = LinearAxis()
plot.add_layout(yaxis, 'left')
yticker = yaxis.ticker
plot.add_layout(Grid(dimension=1, ticker=yticker))
plot.add_tools(PanTool(), WheelZoomTool())
return plot
def test_patches_hover_still_works_when_a_seleciton_is_preselcted(output_file_url, selenium):
# This tests an edge case interaction when Patches (specifically) is used
# with a tool that requires hit testing e.g. HitTool AND a selection is
# pre-made on the data source driving it.
plot = Plot(
x_range=Range1d(0, 100),
y_range=Range1d(0, 100),
min_border=0
)
source = ColumnDataSource(dict(
xs=[[0, 50, 50, 0], [50, 100, 100, 50]],
ys=[[0, 0, 100, 100], [0, 0, 100, 100]],
color=['pink', 'blue']
))
source.selected = {
'0d': {'glyph': None, 'indices': []},
'1d': {'indices': [1]},
'2d': {}
}
plot.add_glyph(source, Patches(xs='xs', ys='ys', fill_color='color'))
plot.add_tools(HoverTool())
plot.add_layout(LinearAxis(), 'below')
plot.add_layout(LinearAxis(), 'left')
save(plot)
selenium.get(output_file_url)
assert has_no_console_errors(selenium)
# Hover plot and test no error
canvas = selenium.find_element_by_tag_name('canvas')
actions = ActionChains(selenium)
actions.move_to_element_with_offset(canvas, 100, 100)
actions.perform()
# If this assertion fails then there were likely errors on hover
assert has_no_console_errors(selenium)
def altitude_profile(data):
plot = Plot(title="%s - Altitude Profile" % title, plot_width=800, plot_height=400)
plot.x_range = DataRange1d()
plot.y_range = DataRange1d()
xaxis = LinearAxis(axis_label="Distance (km)")
plot.add_layout(xaxis, 'below')
yaxis = LinearAxis(axis_label="Altitude (m)")
plot.add_layout(yaxis, 'left')
xgrid = Grid(plot=plot, dimension=0, ticker=xaxis.ticker)
ygrid = Grid(plot=plot, dimension=1, ticker=yaxis.ticker)
plot.renderers.extend([xgrid, ygrid])
plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
X, Y = data.dist, data.alt
y0 = min(Y)
patches_source = ColumnDataSource(dict(
xs=[[X[i], X[i+1], X[i+1], X[i]] for i in range(len(X[:-1])) ],
ys=[[y0, y0, Y[i+1], Y[i]] for i in range(len(Y[:-1])) ],
color=data.colors[:-1]
))
patches = Patches(xs="xs", ys="ys", fill_color="color", line_color="color")
plot.add_glyph(patches_source, patches)
line_source = ColumnDataSource(dict(x=data.dist, y=data.alt))
line = Line(x='x', y='y', line_color="black", line_width=1)
plot.add_glyph(line_source, line)
return plot
def make_plot():
xdr = DataRange1d()
ydr = DataRange1d()
plot = Plot(x_range=xdr, y_range=ydr, plot_width=400, plot_height=400)
plot.title.text = "Product downloads"
line = Line(x="dates", y="downloads", line_color="blue")
plot.add_glyph(source, line)
circle = Circle(x="dates", y="downloads", fill_color="red")
plot.add_glyph(source, circle)
xaxis = DatetimeAxis()
plot.add_layout(xaxis, 'below')
yaxis = LinearAxis()
plot.add_layout(yaxis, 'left')
plot.add_layout(Grid(dimension=0, ticker=xaxis.ticker))
plot.add_layout(Grid(dimension=1, ticker=yaxis.ticker))
plot.add_tools(HoverTool(tooltips=dict(downloads="@downloads")))
return plot, source
def classical_gear(module, large_teeth, small_teeth):
xdr = Range1d(start=-300, end=150)
ydr = Range1d(start=-100, end=100)
plot = Plot(
title=None,
x_range=xdr, y_range=ydr,
plot_width=800, plot_height=800
)
plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
radius = pitch_radius(module, large_teeth)
angle = 0
glyph = Gear(
x=-radius, y=0,
module=module, teeth=large_teeth, angle=angle,
fill_color=fill_color[0], line_color=line_color
)
plot.add_glyph(glyph)
radius = pitch_radius(module, small_teeth)
angle = half_tooth(small_teeth)
glyph = Gear(
x=radius, y=0,
module=module, teeth=small_teeth, angle=angle,
fill_color=fill_color[1], line_color=line_color
)
plot.add_glyph(glyph)
return plot
def __init__(self, **kwargs):
names = ['processes', 'disk-read', 'cores', 'cpu', 'disk-write',
'memory', 'last-seen', 'memory_percent', 'host']
self.source = ColumnDataSource({k: [] for k in names})
columns = {name: TableColumn(field=name,
title=name.replace('_percent', ' %'))
for name in names}
cnames = ['host', 'cores', 'processes', 'memory', 'cpu', 'memory_percent']
formatters = {'cpu': NumberFormatter(format='0.0 %'),
'memory_percent': NumberFormatter(format='0.0 %'),
'memory': NumberFormatter(format='0 b'),
'latency': NumberFormatter(format='0.00000'),
'last-seen': NumberFormatter(format='0.000'),
'disk-read': NumberFormatter(format='0 b'),
'disk-write': NumberFormatter(format='0 b'),
'net-send': NumberFormatter(format='0 b'),
'net-recv': NumberFormatter(format='0 b')}
table = DataTable(
source=self.source, columns=[columns[n] for n in cnames],
)
for name in cnames:
if name in formatters:
table.columns[cnames.index(name)].formatter = formatters[name]
mem_plot = Plot(
title=Title(text="Memory Usage (%)"), toolbar_location=None,
x_range=Range1d(start=0, end=1), y_range=Range1d(start=-0.1, end=0.1),
**kwargs
)
mem_plot.add_glyph(
self.source,
Circle(x='memory_percent', y=0, size=10, fill_alpha=0.5)
)
mem_plot.add_layout(LinearAxis(), 'below')
hover = HoverTool(
point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 10px; font-family: Monaco, monospace;">@host: </span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@memory_percent</span>
</div>
"""
)
mem_plot.add_tools(hover, BoxSelectTool())
if 'sizing_mode' in kwargs:
sizing_mode = {'sizing_mode': kwargs['sizing_mode']}
else:
sizing_mode = {}
self.root = column(mem_plot, table, id='bk-worker-table', **sizing_mode)
def _make_plot():
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1]))
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_glyph(source, Rect(x='x', y='y', width=0.9, height=0.9))
plot.add_tools(ZoomOutTool())
code = RECORD("xrstart", "p.x_range.start") + RECORD("xrend", "p.x_range.end") + RECORD("yrstart", "p.y_range.start") + RECORD("yrend", "p.y_range.end")
plot.add_tools(CustomAction(callback=CustomJS(args=dict(p=plot), code=code)))
plot.toolbar_sticky = False
return plot
def modify_doc(doc):
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1], val=["a", "b"]))
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_glyph(source, Circle(x='x', y='y', size=20))
plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("data", "s.data"))))
slider = Slider(start=0, end=10, value=1, title="bar", css_classes=["foo"], width=300)
def cb(attr, old, new):
source.data['val'] = [old, new]
slider.on_change('value', cb)
doc.add_root(column(slider, plot))
def modify_doc(doc):
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1], val=["a", "b"]))
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_glyph(source, Circle(x='x', y='y', size=20))
plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("data", "s.data"))))
text_input = TextInput(css_classes=["foo"])
def cb(attr, old, new):
source.data['val'] = [old, new]
text_input.on_change('value', cb)
doc.add_root(column(text_input, plot))
def modify_doc(doc):
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1]))
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_glyph(source, Circle(x='x', y='y', size=20))
plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("data", "s.data"))))
button = Button(css_classes=['foo'])
def cb(event):
source.data=dict(x=[10, 20], y=[10, 10])
button.on_event('button_click', cb)
doc.add_root(column(button, plot))
def modify_doc(doc):
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1], val=["a", "b"]))
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_glyph(source, Circle(x='x', y='y', size=20))
plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("data", "s.data"))))
group = RadioButtonGroup(labels=LABELS, css_classes=["foo"])
def cb(active):
source.data['val'] = [active, "b"]
group.on_click(cb)
doc.add_root(column(group, plot))
def sample_gear():
xdr = Range1d(start=-30, end=30)
ydr = Range1d(start=-30, end=30)
plot = Plot(title=None, x_range=xdr, y_range=ydr, plot_width=800, plot_height=800)
plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
glyph = Gear(x=0, y=0, module=5, teeth=8, angle=0, shaft_size=0.2, fill_color=fill_color[2], line_color=line_color)
plot.add_glyph(glyph)
return plot
def modify_doc(doc):
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("indices", "s.selected.indices"))))
table = DataTable(columns=[
TableColumn(field="x"),
TableColumn(field="y")
], source=source, editable=False)
doc.add_root(column(plot, table))
def modify_doc(doc):
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("data", "s.data"))))
table = DataTable(columns=[
TableColumn(field="x", title="x", sortable=True),
TableColumn(field="y", title="y", sortable=True, editor=NumberEditor())
], source=source, editable=True)
doc.add_root(column(plot, table))
def _make_plot(num_objects=0, add=True, drag=True):
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1]))
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 3), y_range=Range1d(0, 3), min_border=0)
renderer = plot.add_glyph(source, Circle(x='x', y='y'))
tool = PointDrawTool(num_objects=num_objects, add=add, drag=drag, renderers=[renderer])
plot.add_tools(tool)
plot.toolbar.active_multi = tool
code = RECORD("x", "source.data.x") + RECORD("y", "source.data.y")
plot.add_tools(CustomAction(callback=CustomJS(args=dict(source=source), code=code)))
plot.toolbar_sticky = False
return plot
def _make_plot(dimensions="both", num_objects=0):
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1], width=[0.5, 0.5], height=[0.5, 0.5]))
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 3), y_range=Range1d(0, 3), min_border=0)
renderer = plot.add_glyph(source, Rect(x='x', y='y', width='width', height='height'))
tool = BoxEditTool(dimensions=dimensions, num_objects=num_objects, renderers=[renderer])
plot.add_tools(tool)
plot.toolbar.active_multi = tool
code = RECORD("x", "source.data.x") + RECORD("y", "source.data.y") + RECORD("width", "source.data.width") + RECORD("height", "source.data.height")
plot.add_tools(CustomAction(callback=CustomJS(args=dict(source=source), code=code)))
plot.toolbar_sticky = False
return plot
def _make_plot():
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1]))
r = Range1d(start=0.4, end=0.6)
plot = Plot(plot_height=400, plot_width=1100, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_glyph(source, Rect(x='x', y='y', width=0.9, height=0.9))
tool = RangeTool(x_range=r)
plot.add_tools(tool)
plot.min_border_right = 100
code = RECORD("start", "t.x_range.start") + RECORD("end", "t.x_range.end")
plot.add_tools(CustomAction(callback=CustomJS(args=dict(t=tool), code=code)))
plot.toolbar_sticky = False
return plot
def make_plot(xname, yname, xax=False, yax=False, text=None):
plot = Plot(
x_range=xdr,
y_range=ydr,
background_fill="#efe8e2",
border_fill="white",
title="",
min_border=2,
h_symmetry=False,
v_symmetry=False,
plot_width=250,
plot_height=250,
)
circle = Circle(x=xname, y=yname, fill_color="color", fill_alpha=0.2, size=4, line_color="color")
r = plot.add_glyph(source, circle)
xdr.renderers.append(r)
ydr.renderers.append(r)
xticker = BasicTicker()
if xax:
xaxis = LinearAxis()
plot.add_layout(xaxis, "below")
xticker = xaxis.ticker
plot.add_layout(Grid(dimension=0, ticker=xticker))
yticker = BasicTicker()
if yax:
yaxis = LinearAxis()
plot.add_layout(yaxis, "left")
yticker = yaxis.ticker
plot.add_layout(Grid(dimension=1, ticker=yticker))
plot.add_tools(PanTool(), WheelZoomTool())
if text:
text = " ".join(text.split("_"))
text = Text(
x={"field": "xcenter", "units": "screen"},
y={"field": "ycenter", "units": "screen"},
text=[text],
angle=pi / 4,
text_font_style="bold",
text_baseline="top",
text_color="#ffaaaa",
text_alpha=0.7,
text_align="center",
text_font_size="28pt",
)
plot.add_glyph(text_source, text)
return plot
def modify_doc(doc):
data = {'x': [1,2,3,4], 'y': [10,20,30,40]}
source = ColumnDataSource(data)
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("data", "s.data"))))
table = DataTable(columns=[
TableColumn(field="x"),
TableColumn(field="y", editor=NumberEditor())
], source=source, editable=True)
doc.add_root(column(plot, table))
def modify_doc(doc):
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1], val=["a", "b"]))
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_glyph(source, Circle(x='x', y='y', size=20))
plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("data", "s.data"))))
input_box = AutocompleteInput(css_classes=["foo"])
input_box.title = "title"
input_box.value = "400"
input_box.completions = ["100001", "12344556", "12344557", "3194567289", "209374209374"]
def cb(attr, old, new):
source.data['val'] = [old, new]
input_box.on_change('value', cb)
doc.add_root(column(input_box, plot))
def large_plot(n):
from bokeh.models import (
Plot, LinearAxis, Grid, GlyphRenderer,
ColumnDataSource, DataRange1d, PanTool, ZoomInTool, ZoomOutTool, WheelZoomTool, BoxZoomTool,
BoxSelectTool, SaveTool, ResetTool
)
from bokeh.models.layouts import Column
from bokeh.models.glyphs import Line
col = Column()
objects = set([col])
for i in xrange(n):
source = ColumnDataSource(data=dict(x=[0, i + 1], y=[0, i + 1]))
xdr = DataRange1d()
ydr = DataRange1d()
plot = Plot(x_range=xdr, y_range=ydr)
xaxis = LinearAxis()
plot.add_layout(xaxis, "below")
yaxis = LinearAxis()
plot.add_layout(yaxis, "left")
xgrid = Grid(dimension=0)
plot.add_layout(xgrid, "center")
ygrid = Grid(dimension=1)
plot.add_layout(ygrid, "center")
tickers = [xaxis.ticker, xaxis.formatter, yaxis.ticker, yaxis.formatter]
glyph = Line(x='x', y='y')
renderer = GlyphRenderer(data_source=source, glyph=glyph)
plot.renderers.append(renderer)
pan = PanTool()
zoom_in = ZoomInTool()
zoom_out = ZoomOutTool()
wheel_zoom = WheelZoomTool()
box_zoom = BoxZoomTool()
box_select = BoxSelectTool()
save = SaveTool()
reset = ResetTool()
tools = [pan, zoom_in, zoom_out, wheel_zoom, box_zoom, box_select, save, reset]
plot.add_tools(*tools)
col.children.append(plot)
objects |= set([
xdr, ydr,
xaxis, yaxis,
xgrid, ygrid,
renderer, renderer.view, glyph,
source, source.selected, source.selection_policy,
plot, plot.x_scale, plot.y_scale, plot.toolbar, plot.title,
box_zoom.overlay, box_select.overlay,
] + tickers + tools)
return col, objects
class MemoryUsage(DashboardComponent):
""" The memory usage across the cluster, grouped by task type """
def __init__(self, **kwargs):
self.source = ColumnDataSource(data=dict(
name=[], left=[], right=[], center=[], color=[],
percent=[], MB=[], text=[])
)
self.root = Plot(
id='bk-nbytes-plot', x_range=DataRange1d(), y_range=DataRange1d(),
toolbar_location=None, outline_line_color=None, **kwargs
)
self.root.add_glyph(
self.source,
Quad(top=1, bottom=0, left='left', right='right',
fill_color='color', fill_alpha=1)
)
self.root.add_layout(LinearAxis(), 'left')
self.root.add_layout(LinearAxis(), 'below')
hover = HoverTool(
point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 14px; font-weight: bold;">Name:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Percent:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@percent</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">MB:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@MB</span>
</div>
"""
)
self.root.add_tools(hover)
def update(self, messages):
with log_errors():
msg = messages['progress']
if not msg:
return
nb = nbytes_bar(msg['nbytes'])
self.source.data.update(nb)
self.root.title.text = \
"Memory Use: %0.2f MB" % (sum(msg['nbytes'].values()) / 1e6)
def modify_doc(doc):
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1], val=["a", "b"]))
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_glyph(source, Circle(x='x', y='y'))
plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("data", "s.data"))))
spinner = Spinner(low=-1, high=10, step=0.1, value=4, css_classes=["foo"])
def cb(attr, old, new):
source.data['val'] = [old, new]
spinner.on_change('value', cb)
doc.add_root(column(spinner, plot))
return doc
def modify_doc(doc):
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1]))
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 3), y_range=Range1d(0, 3), min_border=0)
renderer = plot.add_glyph(source, Circle(x='x', y='y'))
tool = PointDrawTool(renderers=[renderer])
plot.add_tools(tool)
plot.toolbar.active_multi = tool
div = Div(text='False')
def cb(attr, old, new):
if cds_data_almost_equal(new, expected):
div.text = 'True'
source.on_change('data', cb)
code = RECORD("matches", "div.text")
plot.add_tools(CustomAction(callback=CustomJS(args=dict(div=div), code=code)))
doc.add_root(column(plot, div))
def _make_plot(num_objects=0, drag=True, vertices=False):
source = ColumnDataSource(dict(xs=[[1, 2]], ys=[[1, 1]]))
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 3), y_range=Range1d(0, 3), min_border=0)
renderer = plot.add_glyph(source, MultiLine(xs='xs', ys='ys'))
tool = PolyDrawTool(num_objects=num_objects, drag=drag, renderers=[renderer])
if vertices:
psource = ColumnDataSource(dict(x=[], y=[]))
prenderer = plot.add_glyph(psource, Circle(x='x', y='y', size=10))
tool.vertex_renderer = prenderer
plot.add_tools(tool)
plot.toolbar.active_multi = tool
code = RECORD("xs", "source.data.xs") + RECORD("ys", "source.data.ys")
plot.add_tools(CustomAction(callback=CustomJS(args=dict(source=source), code=code)))
plot.toolbar_sticky = False
return plot
def modify_doc(doc):
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("data", "s.data"))))
table = DataTable(columns=[
TableColumn(field="x", title="x", sortable=True),
TableColumn(field="y", title="y", sortable=True)
], source=source, editable=False)
button = Button(css_classes=["foo"])
def cb():
source.stream({'x': [100], 'y': [100]})
button.on_click(cb)
doc.add_root(column(plot, table, button))
def modify_doc(doc):
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1]))
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
plot.add_glyph(source, Circle(x='x', y='y', size=20))
plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("data", "s.data"))))
button = Dropdown(label="Dropdown button", menu=items, css_classes=["foo"])
def cb(event):
item = event.item
if item == "item_1_value":
source.data = dict(x=[10, 20], y=[10, 10])
elif item == "item_2_value":
source.data = dict(x=[100, 200], y=[100, 100])
elif item == "item_3_value":
source.data = dict(x=[1000, 2000], y=[1000, 1000])
button.on_event('menu_item_click', cb)
doc.add_root(column(button, plot))
# Add the tools
tooltips = [
("Manufacturer", "@manufacturer"),
("Model", "@model"),
("Displacement", "@displ"),
("Year", "@year"),
("Cylinders", "@cyl"),
("Transmission", "@trans"),
("Drive", "@drv"),
("Class", "@class"),
]
cty_hover_tool = HoverTool(renderers=[cty],
tooltips=tooltips + [("City MPG", "@cty")])
hwy_hover_tool = HoverTool(renderers=[hwy],
tooltips=tooltips + [("Highway MPG", "@hwy")])
select_tool = BoxSelectTool(renderers=[cty, hwy], dimensions='width')
plot.add_tools(cty_hover_tool, hwy_hover_tool, select_tool)
layout = Column(plot, data_table)
doc = Document()
doc.add_root(layout)
if __name__ == "__main__":
doc.validate()
filename = "data_tables.html"
with open(filename, "w") as f:
f.write(file_html(doc, INLINE, "Data Tables"))
print("Wrote %s" % filename)
view(filename)
def large_plot(n):
from bokeh.models import (
BoxSelectTool,
BoxZoomTool,
Column,
ColumnDataSource,
DataRange1d,
GlyphRenderer,
Grid,
Line,
LinearAxis,
PanTool,
Plot,
ResetTool,
SaveTool,
WheelZoomTool,
ZoomInTool,
ZoomOutTool,
)
col = Column()
objects = {col}
for i in range(n):
source = ColumnDataSource(data=dict(x=[0, i + 1], y=[0, i + 1]))
xdr = DataRange1d()
ydr = DataRange1d()
plot = Plot(x_range=xdr, y_range=ydr)
xaxis = LinearAxis()
plot.add_layout(xaxis, "below")
yaxis = LinearAxis()
plot.add_layout(yaxis, "left")
xgrid = Grid(dimension=0)
plot.add_layout(xgrid, "center")
ygrid = Grid(dimension=1)
plot.add_layout(ygrid, "center")
tickers = [
xaxis.ticker, xaxis.formatter, yaxis.ticker, yaxis.formatter
]
glyph = Line(x='x', y='y')
renderer = GlyphRenderer(data_source=source, glyph=glyph)
plot.renderers.append(renderer)
pan = PanTool()
zoom_in = ZoomInTool()
zoom_out = ZoomOutTool()
wheel_zoom = WheelZoomTool()
box_zoom = BoxZoomTool()
box_select = BoxSelectTool()
save = SaveTool()
reset = ResetTool()
tools = [
pan, zoom_in, zoom_out, wheel_zoom, box_zoom, box_select, save,
reset
]
plot.add_tools(*tools)
col.children.append(plot)
objects |= set([
xdr,
ydr,
xaxis,
xaxis.major_label_policy,
yaxis,
yaxis.major_label_policy,
xgrid,
ygrid,
renderer,
renderer.view,
glyph,
source,
source.selected,
source.selection_policy,
plot,
plot.x_scale,
plot.y_scale,
plot.toolbar,
plot.title,
box_zoom.overlay,
box_select.overlay,
] + tickers + tools)
return col, objects
tools="",
toolbar_location=None)
plot.renderers.append(graph)
output_file("networkx_graph.html")
show(plot)
#### Interactive graph
plot = Plot(plot_width=400,
plot_height=400,
x_range=Range1d(-1.1, 1.1),
y_range=Range1d(-1.1, 1.1))
plot.title.text = "Graph Interaction Demonstration"
plot.add_tools(HoverTool(tooltips=None), TapTool(), BoxSelectTool())
graph_renderer = from_networkx(G, nx.circular_layout, scale=1, center=(0, 0))
graph_renderer.node_renderer.glyph = Circle(size=15, fill_color=Spectral4[0])
graph_renderer.node_renderer.selection_glyph = Circle(size=15,
fill_color=Spectral4[2])
graph_renderer.node_renderer.hover_glyph = Circle(size=15,
fill_color=Spectral4[1])
graph_renderer.edge_renderer.glyph = MultiLine(line_color="#CCCCCC",
line_alpha=0.8,
line_width=5)
graph_renderer.edge_renderer.selection_glyph = MultiLine(
line_color=Spectral4[2], line_width=5)
graph_renderer.edge_renderer.hover_glyph = MultiLine(line_color=Spectral4[1],
def dendrogram(D, figargs=None):
'''Creates a dendrogram plot.
This plot can show full structure of a given dendrogram.
Args:
D (henchman.selection.Dendrogram): An initialized dendrogram object
Examples:
>>> from henchman.selection import Dendrogram
>>> from henchman.plotting import show
>>> import henchman.plotting as hplot
>>> D = Dendrogram(X)
>>> plot = hplot.dendrogram(D)
>>> show(plot)
'''
if figargs is None:
return lambda figargs: dendrogram(D, figargs=figargs)
G = nx.Graph()
vertices_source = ColumnDataSource(
pd.DataFrame({
'index': D.columns.keys(),
'desc': list(D.columns.values())
}))
edges_source = ColumnDataSource(
pd.DataFrame(D.edges[0]).rename(columns={
1: 'end',
0: 'start'
}))
step_source = ColumnDataSource(
pd.DataFrame({
'step': [0],
'thresh': [D.threshlist[0]],
'components': [len(D.graphs[0])]
}))
G.add_nodes_from([str(x) for x in vertices_source.data['index']])
G.add_edges_from(
zip([str(x) for x in edges_source.data['start']],
[str(x) for x in edges_source.data['end']]))
graph_renderer = from_networkx(G,
nx.circular_layout,
scale=1,
center=(0, 0))
graph_renderer.node_renderer.data_source = vertices_source
graph_renderer.node_renderer.view = CDSView(source=vertices_source)
graph_renderer.edge_renderer.data_source = edges_source
graph_renderer.edge_renderer.view = CDSView(source=edges_source)
plot = Plot(plot_width=400,
plot_height=400,
x_range=Range1d(-1.1, 1.1),
y_range=Range1d(-1.1, 1.1))
plot.title.text = "Feature Connectivity"
graph_renderer.node_renderer.glyph = Circle(size=5,
fill_color=Spectral4[0])
graph_renderer.node_renderer.selection_glyph = Circle(
size=15, fill_color=Spectral4[2])
graph_renderer.edge_renderer.data_source = edges_source
graph_renderer.edge_renderer.glyph = MultiLine(line_color="#CCCCCC",
line_alpha=0.6,
line_width=.5)
graph_renderer.edge_renderer.selection_glyph = MultiLine(
line_color=Spectral4[2], line_width=3)
graph_renderer.node_renderer.hover_glyph = Circle(size=5,
fill_color=Spectral4[1])
graph_renderer.selection_policy = NodesAndLinkedEdges()
graph_renderer.inspection_policy = NodesAndLinkedEdges()
plot.renderers.append(graph_renderer)
plot.add_tools(
HoverTool(tooltips=[
("feature", "@desc"),
("index", "@index"),
]), TapTool(), BoxZoomTool(), SaveTool(), ResetTool())
plot = _modify_plot(plot, figargs)
if figargs['static']:
return plot
def modify_doc(doc, D, figargs):
data_table = DataTable(source=step_source,
columns=[
TableColumn(field='step', title='Step'),
TableColumn(field='thresh', title='Thresh'),
TableColumn(field='components',
title='Components')
],
height=50,
width=400)
def callback(attr, old, new):
try:
edges = D.edges[slider.value]
edges_source.data = ColumnDataSource(
pd.DataFrame(edges).rename(columns={
1: 'end',
0: 'start'
})).data
step_source.data = ColumnDataSource({
'step': [slider.value],
'thresh': [D.threshlist[slider.value]],
'components': [len(D.graphs[slider.value])]
}).data
except Exception as e:
print(e)
slider = Slider(start=0,
end=(len(D.edges) - 1),
value=0,
step=1,
title="Step")
slider.on_change('value', callback)
doc.add_root(column(slider, data_table, plot))
return lambda doc: modify_doc(doc, D, figargs)
def interactive_plot(network,
network_name,
layout_func='fruchterman_reingold'):
plot = Plot(plot_width=800,
plot_height=800,
x_range=Range1d(-1.1, 1.1),
y_range=Range1d(-1.1, 1.1))
plot.title.text = network_name
plot.add_tools(
HoverTool(
tooltips=[('bird',
'@index'), ("age",
"@Age"), ("sex",
"@Sex"), ("location",
'@Location')]),
TapTool(), BoxSelectTool(), BoxZoomTool(), ResetTool(), PanTool(),
WheelZoomTool())
if layout_func == 'fruchterman_reingold':
graph_renderer = graphs.from_networkx(network,
nx.fruchterman_reingold_layout,
scale=1,
center=(0, 0))
elif layout_func == 'spring':
graph_renderer = graphs.from_networkx(network,
nx.spring_layout,
scale=1,
center=(0, 0))
elif layout_func == 'circular':
graph_renderer = graphs.from_networkx(network,
nx.circular_layout,
scale=1,
center=(0, 0))
elif layout_func == 'kamada':
graph_renderer = graphs.from_networkx(network,
nx.kamada_kawai_layout,
scale=1,
center=(0, 0))
elif layout_func == 'spectral':
graph_renderer = graphs.from_networkx(network,
nx.spectral_layout,
scale=1,
center=(0, 0))
else:
graph_renderer = graphs.from_networkx(network,
nx.fruchterman_reingold_layout,
scale=1,
center=(0, 0))
centrality = nx.algorithms.centrality.betweenness_centrality(network)
""" first element are nodes again """
_, nodes_centrality = zip(*centrality.items())
max_centraliy = max(nodes_centrality)
c_centrality = [7 + 15 * t / max_centraliy for t in nodes_centrality]
import community #python-louvain
partition = community.best_partition(network)
p_, nodes_community = zip(*partition.items())
community_colors = [
'#e41a1c', '#377eb8', '#4daf4a', '#984ea3', '#ff7f00', '#ffff33',
'#a65628', '#b3cde3', '#ccebc5', '#decbe4', '#fed9a6', '#ffffcc',
'#e5d8bd', '#fddaec', '#1b9e77', '#d95f02', '#7570b3', '#e7298a',
'#66a61e', '#e6ab02', '#a6761d', '#666666'
]
colors = [
community_colors[t % len(community_colors)] for t in nodes_community
]
_, Sex = zip(*nx.get_node_attributes(network, 'Sex').items())
_, Age = zip(*nx.get_node_attributes(network, 'Age').items())
_, Location = zip(*nx.get_node_attributes(network, 'Location').items())
graph_renderer.node_renderer.data_source.add(c_centrality, 'centrality')
graph_renderer.node_renderer.data_source.add(Sex, 'Sex')
graph_renderer.node_renderer.data_source.add(Age, 'Age')
graph_renderer.node_renderer.data_source.add(Location, 'Location')
graph_renderer.node_renderer.data_source.add(colors, 'colors')
graph_renderer.node_renderer.glyph = Circle(size='centrality',
fill_color='colors')
graph_renderer.node_renderer.selection_glyph = Circle(
size='centrality', fill_color=Spectral4[2])
graph_renderer.node_renderer.hover_glyph = Circle(size=15,
fill_color=Spectral4[1])
_, edge_weights = zip(*nx.get_edge_attributes(network, 'weight').items())
max_weights = max(edge_weights)
c_weights = [7 + 2 * (t / max_weights) for t in edge_weights]
graph_renderer.edge_renderer.data_source.add(c_weights, 'weight')
graph_renderer.edge_renderer.glyph = MultiLine(line_color="#757474",
line_alpha=0.2,
line_width='weight')
graph_renderer.edge_renderer.selection_glyph = MultiLine(
line_color=Spectral4[2], line_width='weight')
graph_renderer.edge_renderer.hover_glyph = MultiLine(
line_color=Spectral4[1], line_width='weight')
graph_renderer.selection_policy = graphs.NodesAndLinkedEdges()
graph_inspection_policy = graphs.NodesOnly()
#graph_renderer.inspection_policy = graphs.EdgesAndLinkedNodes()
plot.renderers.append(graph_renderer)
#output_file("interactive_graphs.html")
return plot
from bokeh.util.browser import view
output_file("tile_source_example.html", title="Tile Source Example")
# set to roughly full extent of web mercator projection
x_range = Range1d(start=-200000, end=2000000)
y_range = Range1d(start=800000, end=7000000)
# create tile source from templated url
tile_options = {}
tile_options['url'] = 'http://c.tile.openstreetmap.org/{z}/{x}/{y}.png'
tile_source = WMTSTileSource(**tile_options)
# instantiate plot and add tile source
p = Plot(x_range=x_range, y_range=y_range, height=800, width=800)
p.add_tools(WheelZoomTool(), PanTool(), BoxZoomTool(match_aspect=True))
tile_renderer_options = {}
p.add_tile(tile_source, **tile_renderer_options)
doc = Document()
doc.add_root(p)
if __name__ == "__main__":
doc.validate()
filename = "tile_source.html"
with open(filename, "w") as f:
f.write(file_html(doc, INLINE, "Tile Source Example"))
print("Wrote %s" % filename)
view(filename)
toolbar_location="left")
county_patches = Patches(xs="county_xs",
ys="county_ys",
fill_color="county_colors",
fill_alpha=0.7,
line_color="white",
line_width=0.5)
plot.add_glyph(county_source, county_patches)
state_patches = Patches(xs="state_xs",
ys="state_ys",
fill_alpha=0.0,
line_color="#884444",
line_width=2)
plot.add_glyph(state_source, state_patches)
plot.add_tools(ResizeTool())
doc = Document()
doc.add(plot)
if __name__ == "__main__":
filename = "choropleth.html"
with open(filename, "w") as f:
f.write(
file_html(doc, INLINE,
"Choropleth of all US counties, Unemployment 2009"))
print("Wrote %s" % filename)
view(filename)
def create(palm):
connected = False
current_message = None
stream_t = 0
doc = curdoc()
# Streaked and reference waveforms plot
waveform_plot = Plot(
title=Title(text="eTOF waveforms"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
waveform_plot.toolbar.logo = None
waveform_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool())
# ---- axes
waveform_plot.add_layout(LinearAxis(axis_label="Photon energy, eV"),
place="below")
waveform_plot.add_layout(LinearAxis(axis_label="Intensity",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
waveform_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
waveform_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- line glyphs
waveform_source = ColumnDataSource(
dict(x_str=[], y_str=[], x_ref=[], y_ref=[]))
waveform_ref_line = waveform_plot.add_glyph(
waveform_source, Line(x="x_ref", y="y_ref", line_color="blue"))
waveform_str_line = waveform_plot.add_glyph(
waveform_source, Line(x="x_str", y="y_str", line_color="red"))
# ---- legend
waveform_plot.add_layout(
Legend(items=[("reference",
[waveform_ref_line]), ("streaked",
[waveform_str_line])]))
waveform_plot.legend.click_policy = "hide"
# Cross-correlation plot
xcorr_plot = Plot(
title=Title(text="Waveforms cross-correlation"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
xcorr_plot.toolbar.logo = None
xcorr_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool())
# ---- axes
xcorr_plot.add_layout(LinearAxis(axis_label="Energy shift, eV"),
place="below")
xcorr_plot.add_layout(LinearAxis(axis_label="Cross-correlation",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
xcorr_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
xcorr_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- line glyphs
xcorr_source = ColumnDataSource(dict(lags=[], xcorr=[]))
xcorr_plot.add_glyph(xcorr_source,
Line(x="lags", y="xcorr", line_color="purple"))
# ---- vertical span
xcorr_center_span = Span(location=0, dimension="height")
xcorr_plot.add_layout(xcorr_center_span)
# Delays plot
pulse_delay_plot = Plot(
title=Title(text="Pulse delays"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
pulse_delay_plot.toolbar.logo = None
pulse_delay_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool())
# ---- axes
pulse_delay_plot.add_layout(LinearAxis(axis_label="Pulse number"),
place="below")
pulse_delay_plot.add_layout(
LinearAxis(axis_label="Pulse delay (uncalib), eV",
major_label_orientation="vertical"),
place="left",
)
# ---- grid lines
pulse_delay_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
pulse_delay_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- line glyphs
pulse_delay_source = ColumnDataSource(dict(x=[], y=[]))
pulse_delay_plot.add_glyph(pulse_delay_source,
Line(x="x", y="y", line_color="steelblue"))
# Pulse lengths plot
pulse_length_plot = Plot(
title=Title(text="Pulse lengths"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
pulse_length_plot.toolbar.logo = None
pulse_length_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool())
# ---- axes
pulse_length_plot.add_layout(LinearAxis(axis_label="Pulse number"),
place="below")
pulse_length_plot.add_layout(
LinearAxis(axis_label="Pulse length (uncalib), eV",
major_label_orientation="vertical"),
place="left",
)
# ---- grid lines
pulse_length_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
pulse_length_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- line glyphs
pulse_length_source = ColumnDataSource(dict(x=[], y=[]))
pulse_length_plot.add_glyph(pulse_length_source,
Line(x="x", y="y", line_color="steelblue"))
# Image buffer slider
def buffer_slider_callback(_attr, _old, new):
message = receiver.data_buffer[new]
doc.add_next_tick_callback(partial(update, message=message))
buffer_slider = Slider(
start=0,
end=59,
value=0,
step=1,
title="Buffered Image",
callback_policy="throttle",
callback_throttle=500,
)
buffer_slider.on_change("value", buffer_slider_callback)
# Connect toggle button
def connect_toggle_callback(state):
nonlocal connected
if state:
connected = True
connect_toggle.label = "Connecting"
connect_toggle.button_type = "default"
else:
connected = False
connect_toggle.label = "Connect"
connect_toggle.button_type = "default"
connect_toggle = Toggle(label="Connect", button_type="default", width=250)
connect_toggle.on_click(connect_toggle_callback)
# Intensity stream reset button
def reset_button_callback():
nonlocal stream_t
stream_t = 1 # keep the latest point in order to prevent full axis reset
reset_button = Button(label="Reset", button_type="default", width=250)
reset_button.on_click(reset_button_callback)
# Stream update coroutine
async def update(message):
nonlocal stream_t
if connected and receiver.state == "receiving":
y_ref = message[receiver.reference].value[np.newaxis, :]
y_str = message[receiver.streaked].value[np.newaxis, :]
delay, length, debug_data = palm.process({
"0": y_ref,
"1": y_str
},
debug=True)
prep_data, lags, corr_res_uncut, _ = debug_data
waveform_source.data.update(
x_str=palm.energy_range,
y_str=prep_data["1"][0, :],
x_ref=palm.energy_range,
y_ref=prep_data["0"][0, :],
)
xcorr_source.data.update(lags=lags, xcorr=corr_res_uncut[0, :])
xcorr_center_span.location = delay[0]
pulse_delay_source.stream({
"x": [stream_t],
"y": [delay]
},
rollover=120)
pulse_length_source.stream({
"x": [stream_t],
"y": [length]
},
rollover=120)
stream_t += 1
# Periodic callback to fetch data from receiver
async def internal_periodic_callback():
nonlocal current_message
if waveform_plot.inner_width is None:
# wait for the initialization to finish, thus skip this periodic callback
return
if connected:
if receiver.state == "polling":
connect_toggle.label = "Polling"
connect_toggle.button_type = "warning"
elif receiver.state == "stopped":
connect_toggle.label = "Not available"
connect_toggle.button_type = "danger"
elif receiver.state == "receiving":
connect_toggle.label = "Receiving"
connect_toggle.button_type = "success"
# Set slider to the right-most position
if len(receiver.data_buffer) > 1:
buffer_slider.end = len(receiver.data_buffer) - 1
buffer_slider.value = len(receiver.data_buffer) - 1
if receiver.data_buffer:
current_message = receiver.data_buffer[-1]
doc.add_next_tick_callback(partial(update, message=current_message))
doc.add_periodic_callback(internal_periodic_callback, 1000)
# assemble
tab_layout = column(
row(
column(waveform_plot, xcorr_plot),
Spacer(width=30),
column(buffer_slider, row(connect_toggle, reset_button)),
),
row(pulse_delay_plot, Spacer(width=10), pulse_length_plot),
)
return Panel(child=tab_layout, title="Stream")
class TaskProgress(DashboardComponent):
""" Progress bars per task type """
def __init__(self, **kwargs):
data = progress_quads(dict(all={}, memory={}, erred={}, released={}))
self.source = ColumnDataSource(data=data)
x_range = DataRange1d()
y_range = Range1d(-8, 0)
self.root = Plot(id='bk-task-progress-plot',
x_range=x_range,
y_range=y_range,
toolbar_location=None,
**kwargs)
self.root.add_glyph(
self.source,
Quad(top='top',
bottom='bottom',
left='left',
right='right',
fill_color="#aaaaaa",
line_color="#aaaaaa",
fill_alpha=0.2))
self.root.add_glyph(
self.source,
Quad(top='top',
bottom='bottom',
left='left',
right='released-loc',
fill_color="color",
line_color="color",
fill_alpha=0.6))
self.root.add_glyph(
self.source,
Quad(top='top',
bottom='bottom',
left='released-loc',
right='memory-loc',
fill_color="color",
line_color="color",
fill_alpha=1.0))
self.root.add_glyph(
self.source,
Quad(top='top',
bottom='bottom',
left='erred-loc',
right='erred-loc',
fill_color='#000000',
line_color='#000000',
fill_alpha=0.3))
self.root.add_glyph(
self.source,
Text(text='show-name',
y='bottom',
x='left',
x_offset=5,
text_font_size=value('10pt')))
self.root.add_glyph(
self.source,
Text(text='done',
y='bottom',
x='right',
x_offset=-5,
text_align='right',
text_font_size=value('10pt')))
hover = HoverTool(point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 14px; font-weight: bold;">Name:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">All:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@all</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Memory:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@memory</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Erred:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@erred</span>
</div>
""")
self.root.add_tools(hover)
def update(self, messages):
with log_errors():
msg = messages['progress']
if not msg:
return
d = progress_quads(msg)
self.source.data.update(d)
if messages['tasks']['deque']:
self.root.title.text = (
"Progress -- total: %(total)s, "
"in-memory: %(in-memory)s, processing: %(processing)s, "
"waiting: %(waiting)s, failed: %(failed)s" %
messages['tasks']['deque'][-1])
class TaskStream(DashboardComponent):
""" Task Stream
The start and stop time of tasks as they occur on each core of the cluster.
"""
def __init__(self, n_rectangles=1000, clear_interval=20000, **kwargs):
"""
kwargs are applied to the bokeh.models.plots.Plot constructor
"""
self.n_rectangles = n_rectangles
self.clear_interval = clear_interval
self.source = ColumnDataSource(data=dict(start=[],
duration=[],
key=[],
name=[],
color=[],
worker=[],
y=[],
worker_thread=[],
alpha=[]))
x_range = DataRange1d()
y_range = DataRange1d(range_padding=0)
self.root = Plot(title=Title(text="Task Stream"),
id='bk-task-stream-plot',
x_range=x_range,
y_range=y_range,
toolbar_location="above",
min_border_right=35,
**kwargs)
self.root.add_glyph(
self.source,
Rect(x="start",
y="y",
width="duration",
height=0.8,
fill_color="color",
line_color="color",
line_alpha=0.6,
fill_alpha="alpha",
line_width=3))
self.root.add_layout(DatetimeAxis(axis_label="Time"), "below")
ticker = BasicTicker(num_minor_ticks=0)
self.root.add_layout(
LinearAxis(axis_label="Worker Core", ticker=ticker), "left")
self.root.add_layout(
Grid(dimension=1, grid_line_alpha=0.4, ticker=ticker))
hover = HoverTool(point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 12px; font-weight: bold;">@name:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@duration</span>
<span style="font-size: 10px;">ms</span>
</div>
""")
# export = ExportTool()
# export.register_plot(self.root)
self.root.add_tools(
hover,
# export,
BoxZoomTool(),
ResetTool(reset_size=False),
PanTool(dimensions="width"),
WheelZoomTool(dimensions="width"))
# Required for update callback
self.task_stream_index = [0]
def update(self, messages):
with log_errors():
index = messages['task-events']['index']
old = rectangles = messages['task-events']['rectangles']
if not index or index[-1] == self.task_stream_index[0]:
return
ind = bisect(index, self.task_stream_index[0])
rectangles = {
k: [v[i] for i in range(ind, len(index))]
for k, v in rectangles.items()
}
self.task_stream_index[0] = index[-1]
# If there has been a significant delay then clear old rectangles
if rectangles['start']:
last_end = old['start'][ind - 1] + old['duration'][ind - 1]
if min(rectangles['start']) > last_end + self.clear_interval:
self.source.data.update(rectangles)
return
self.source.stream(rectangles, self.n_rectangles)
for each in pos.values():
range_x[0] = min(range_x[0], int(each[0]))
range_x[1] = max(range_x[1], int(each[0]))
range_y[0] = min(range_y[0], int(each[1]))
range_y[1] = max(range_y[1], int(each[1]))
range_x = [range_x[0] - 50, range_x[1] + 150]
range_y = [range_y[0] - 50, range_y[1] + 50]
p1 = Plot(plot_width=700,
plot_height=700,
x_range=Range1d(range_x[0], range_x[1]),
y_range=Range1d(range_y[0], range_y[1]))
p1.title.text = "DAG"
p.title.text_font_size = '20pt'
node_hover_tool = HoverTool(tooltips=[("task", "@index")])
p1.add_tools(node_hover_tool, BoxZoomTool(), ResetTool())
global graph
graph = from_networkx(G, pos, scale=1, center=(0, 0))
colors = []
indexs = []
for i, each in enumerate(nodes):
indexs.append(i)
colors.append('blue')
graph.node_renderer.data_source.data['colors'] = colors
graph.node_renderer.data_source.data['name'] = nodes
graph.node_renderer.glyph = Circle(size=15, fill_color='colors')
graph.edge_renderer.glyph = MultiLine(line_color="black",
line_alpha=0.8,
text_baseline="middle",
text_font_size="9pt",
text_font_style="italic",
text_color="silver")
no_olympics = plot.add_glyph(no_olympics_glyph)
tooltips = """
<div>
<span style="font-size: 15px;">@Name</span>
<span style="font-size: 10px; color: #666;">(@Abbrev)</span>
</div>
<div>
<span style="font-size: 17px; font-weight: bold;">@Time{0.00}</span>
<span style="font-size: 10px; color: #666;">@Year</span>
</div>
<div style="font-size: 11px; color: #666;">@{MetersBack}{0.00} meters behind</div>
"""
hover = HoverTool(tooltips=tooltips, renderers=[medal])
plot.add_tools(hover)
doc = Document()
doc.add_root(plot)
if __name__ == "__main__":
filename = "sprint.html"
with open(filename, "w") as f:
f.write(file_html(doc, INLINE, title))
print("Wrote %s" % filename)
view(filename)
def main(doc=curdoc()):
print("start main function!")
class mq():
def __init__(self, outfname, subs, server, port, timeout, looptimeout):
self.OUTFNAME = outfname
self.subs = subs
self.server = server
self.port = port
self.timeout = timeout
self.looptimeout = looptimeout
self.outf = open(OUTFNAME, 'a')
self.client = mqtt.Client()
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
try:
self.client.connect(self.server, self.port, self.timeout)
except:
print("connection failed")
# The callback for when the client receives a CONNACK response from the server.
def on_connect(self, client, userdata, flags, rc):
# Subscribing in on_connect() means that if we lose the connection and
# reconnect then subscriptions will be renewed.
subres = client.subscribe(self.subs, qos=1)
print("Connected with result code " + str(rc))
# The callback for when a PUBLISH message is received from the server.
def on_message(self, client, userdata, msg):
start_messages = []
for each in tasks:
start_messages.append(each + " starts")
end_messages = []
for each in tasks:
end_messages.append(each + " ends")
# start_messages = ['localpro starts', 'aggregate0 starts', 'aggregate1 starts', 'aggregate2 starts',
# 'simpledetector0 starts', 'simpledetector1 starts', 'simpledetector2 starts', 'astutedetector0 starts',
# 'astutedetector1 starts', 'astutedetector2 starts', 'fusioncenter0 starts', 'fusioncenter1 starts',
# 'fusioncenter2 starts', 'globalfusion starts']
# end_messages = ['localpro ends', 'aggregate0 ends', 'aggregate1 ends', 'aggregate2 ends',
# 'simpledetector0 ends', 'simpledetector1 ends', 'simpledetector2 ends', 'astutedetector0 ends',
# 'astutedetector1 ends', 'astutedetector2 ends', 'fusioncenter0 ends', 'fusioncenter1 ends',
# 'fusioncenter2 ends', 'globalfusion ends']
message = msg.payload.decode()
global start_time
global finish_time
global total_time
print('--------------')
print(message)
print('--------------')
if message in start_messages:
index = start_messages.index(message)
color = "red"
doc.add_next_tick_callback(
partial(update3, index=index, color=color))
elif message in end_messages:
index = end_messages.index(message)
color1 = "blue"
doc.add_next_tick_callback(
partial(update3, index=index, color=color1))
elif message.startswith('mappings'):
print('---- Receive task mapping')
doc.add_next_tick_callback(
partial(update5,
new=message,
old=source6_df,
attr=source6.data))
doc.add_next_tick_callback(
partial(update4,
new=message,
old=source5_df,
attr=data_table2.source))
elif message.startswith('runtime'):
print('---- Receive runtime statistics')
doc.add_next_tick_callback(
partial(update8,
new=message,
old=source8_df,
attr=data_table4.source))
elif message.startswith('global'):
print('-----Receive global information')
doc.add_next_tick_callback(
partial(update7,
new=message,
old=source7_df,
attr=data_table3.source))
def retrieve_tasks(dag_info_file):
config_file = open(dag_info_file, 'r')
dag_size = int(config_file.readline())
tasks = {}
for i, line in enumerate(config_file, 1):
dag_line = line.strip().split(" ")
tasks[dag_line[0]] = i
if i == dag_size:
break
return tasks
def k8s_get_nodes(node_info_file):
"""read the node info from the file input
Args:
node_info_file (str): path of ``node.txt``
Returns:
dict: node information
"""
nodes = {}
node_file = open(node_info_file, "r")
for i, line in enumerate(node_file):
node_line = line.strip().split(" ")
nodes[i] = [node_line[0], node_line[1]]
return nodes
def repeatlist(it, count):
return islice(cycle(it), count)
def update():
m.client.loop(timeout=0.5)
# @gen.coroutine
# def update1(top,bottom,right,left,color):
# source.stream(dict(top=[top], bottom=[bottom],left=[left],right=[right],color=[color],line_color=["black"], line_width=[2]))
@gen.coroutine
def update2(x, y, time):
source1.stream(
dict(x=[x], y=[y], time=[time], text_font_style=['bold']))
@gen.coroutine
def update3(index, color):
colors = []
for i, each in enumerate(nodes):
if i == index:
colors.append(color)
continue
colors.append(each)
# print("DEBUG update3 colors:")
# print(colors)
graph.node_renderer.data_source.data['colors'] = colors
@gen.coroutine
def update4(attr, old, new):
assigned_info = new.split(' ')[1:]
for info in assigned_info:
tmp = info.split(':')
new_source5_df.loc[new_source5_df.task_names == tmp[0],
'assigned'] = tmp[1]
new_source5_df.loc[new_source5_df.task_names == tmp[0],
'as_time'] = convert_time(tmp[2])
source5.data = {
'task_id': new_source5_df.task_id,
'task_names': new_source5_df.task_names,
'assigned': new_source5_df.assigned,
'as_time': new_source5_df.as_time
}
@gen.coroutine
def update5(attr, old, new):
assigned_info = new.split(' ')[1:]
for info in assigned_info:
tmp = info.split(':')
t = '_T' + str(tasks[tmp[0]])
n = 'N' + str(node_short.index(tmp[1]))
new_source6_df.loc[new_source6_df.nodes == n,
'assigned_task'] = new_source6_df.loc[
new_source6_df.nodes == n,
'assigned_task'] + t
source6.data = {
'x': new_source6_df.x,
'y': new_source6_df.y,
'color': new_source6_df.color,
'nodes': new_source6_df.nodes,
'x_label': new_source6_df.x_label,
'y_label': new_source6_df.y_label,
'assigned_task': new_source6_df.assigned_task
}
@gen.coroutine
def update7(attr, old, new):
print("Call update7!")
tmp = new.split(' ')[1:]
home_id = tmp[0]
if tmp[1] == 'start':
data = ['N/A', 'N/A', tmp[2], tmp[0], convert_time(tmp[3])]
new_source7_df.loc[len(new_source7_df), :] = data
else:
new_source7_df.loc[(new_source7_df.home_id == tmp[0]) &
(new_source7_df.global_input == tmp[2]),
'end_times'] = convert_time(tmp[3])
tmp1 = new_source7_df.loc[(new_source7_df.home_id == tmp[0]) &
(new_source7_df.global_input == tmp[2]),
'end_times']
tmp2 = new_source7_df.loc[(new_source7_df.home_id == tmp[0]) &
(new_source7_df.global_input == tmp[2]),
'start_times']
new_source7_df.loc[(new_source7_df.home_id == tmp[0]) &
(new_source7_df.global_input == tmp[2]),
'exec_times'] = time_delta(tmp1, tmp2)
source7.data = {
'home_id': new_source7_df.home_id,
'global_input': new_source7_df.global_input,
'start_times': new_source7_df.start_times,
'end_times': new_source7_df.end_times,
'exec_times': new_source7_df.exec_times,
}
@gen.coroutine
def update8(attr, old, new):
tmp = new.split(' ')[1:]
if tmp[0] == 'enter':
data = [
'N/A', 'N/A',
convert_time(tmp[3]), 'N/A', 'N/A', tmp[2], 'N/A', tmp[1]
]
new_source8_df.loc[len(new_source8_df), :] = data
elif tmp[0] == 'exec':
new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_exec_times'] = convert_time(tmp[3])
tmp1 = new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_exec_times']
tmp2 = new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_enter_times']
new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_waiting_times'] = time_delta(tmp1, tmp2)
else:
new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_finish_times'] = convert_time(tmp[3])
tmp1 = new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_finish_times']
tmp2 = new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_enter_times']
tmp3 = new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_exec_times']
new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_elapse_times'] = time_delta(tmp1, tmp2)
new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_duration_times'] = time_delta(
tmp1, tmp3)
source8.data = {
'local_duration_times': new_source8_df.local_duration_times,
'local_elapse_times': new_source8_df.local_elapse_times,
'local_enter_times': new_source8_df.local_enter_times,
'local_exec_times': new_source8_df.local_exec_times,
'local_finish_times': new_source8_df.local_finish_times,
'local_input': new_source8_df.local_input,
'local_waiting_times': new_source8_df.local_waiting_times,
'task_name': new_source8_df.task_name
}
def convert_time(t):
return datetime.datetime.fromtimestamp(
float(t)).strftime("%d.%m.%y %H:%M:%S")
def time_delta(end, start):
tmp1 = datetime.datetime.strptime(end.iloc[0], "%d.%m.%y %H:%M:%S")
tmp2 = datetime.datetime.strptime(start.iloc[0], "%d.%m.%y %H:%M:%S")
delta = (tmp1 - tmp2).total_seconds()
return delta
###################################################################################################
global OUTFNAME, SERVER_IP, SUBSCRIPTIONS, DAG_PATH, NODE_PATH
OUTFNAME = 'demo_original.html'
# SERVER_IP = 'test.mosquitto.org'
SERVER_IP = 'test.mosquitto.org'
SUBSCRIPTIONS = 'JUPITER'
DAG_PATH = 'configuration.txt'
NODE_PATH = '../../nodes.txt'
global start_time, finish_time, total_time, offset, input_num
start_time = []
finish_time = 0
total_time = 0
offset = 0
input_num = 0
global source, source1, source2, source3, source4, source5, source6, source5_df, nodes, m, p, p1
source = ColumnDataSource(data=dict(top=[0],
bottom=[0],
left=[0],
right=[0],
color=["#9ecae1"],
line_color=["black"],
line_width=[2]))
source1 = ColumnDataSource(
data=dict(x=[8], y=[3.5], time=[''], text_font_style=['bold']))
global nodes, num_nodes, MAX_X, MAX_Y, tasks
nodes = k8s_get_nodes(NODE_PATH)
num_nodes = len(nodes)
MAX_X = 10
MAX_Y = 12
tasks = retrieve_tasks(DAG_PATH)
num_tasks = len(tasks)
# doc = curdoc()
doc.title = 'CIRCE Visualization'
m = mq(outfname=OUTFNAME,
subs=SUBSCRIPTIONS,
server=SERVER_IP,
port=1883,
timeout=60,
looptimeout=1)
###################################################################################################################################
global data_table, data_table2, source5_df, new_source5_df, source6_df, new_source6_df
node_id = ['N' + str(i) for i in nodes.keys()]
node_short = [i[0] for i in nodes.values()]
node_full = [i[1] for i in nodes.values()]
source4 = ColumnDataSource(
dict(node_id=node_id, node_short=node_short, node_full=node_full))
columns = [
TableColumn(field="node_id", title="Node ID"),
TableColumn(field="node_short", title="Node Name"),
TableColumn(field="node_full", title="Full Name")
]
data_table = DataTable(source=source4,
columns=columns,
width=400,
height=230,
selectable=True)
title1 = Div(text='Node Information',
style={
'font-size': '15pt',
'color': 'black',
'text-align': 'center'
},
width=400,
height=20)
assignment = ['N/A'] * len(tasks.keys())
assign_time = ['N/A'] * len(tasks.keys())
task_id = ['T' + str(i) for i in tasks.values()]
source5 = ColumnDataSource(data=dict(task_id=task_id,
task_names=list(tasks.keys()),
assigned=assignment,
as_time=assign_time))
columns2 = [
TableColumn(field="task_id", title="Task ID"),
TableColumn(field="task_names", title="Tasks Names"),
TableColumn(field="assigned", title="Assigned Node"),
TableColumn(field="as_time", title="Assigned Time")
]
data_table2 = DataTable(source=source5,
columns=columns2,
width=400,
height=230,
selectable=True,
editable=True)
title2 = Div(text='Task Mapping Information',
style={
'font-size': '15pt',
'color': 'black',
'text-align': 'center'
},
width=400,
height=20)
source5_df = source5.to_df()
new_source5_df = source5.to_df()
data_table2.on_change('source', lambda attr, old, new: update4())
###################################################################################################################################
points = set()
while len(points) < num_nodes:
ix = np.random.randint(1, MAX_X)
iy = np.random.randint(1, MAX_Y)
points.add((ix, iy))
x = [i[0] for i in points]
y = [i[1] for i in points]
x_label = [i - 0.3 for i in x]
y_label = [i - 0.2 for i in y]
c = brewer["Spectral"][9]
color = list(repeatlist(c, num_nodes))
assigned_task = [""] * len(points)
p = figure(x_range=(0, MAX_X + 1),
y_range=(0, MAX_Y + 1),
plot_width=500,
plot_height=600)
p.background_fill_color = "#EEEDED"
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None
p.xaxis.axis_label = 'Digital Ocean Clusters'
p.xaxis.axis_label_text_font_size = '20pt'
source6 = ColumnDataSource(data=dict(x=x,
y=y,
color=color,
nodes=node_id,
x_label=x_label,
y_label=y_label,
assigned_task=assigned_task))
lab = LabelSet(x='x_label',
y='y_label',
text='nodes',
text_font_style='bold',
text_color='black',
source=source6)
p.add_layout(lab)
w = p.circle(x='x',
y='y',
radius=0.8,
fill_color='color',
line_color='color',
source=source6)
p.add_tools(
HoverTool(tooltips=[
("node_id", "@nodes"),
("assigned_task", "@assigned_task"),
]))
source6_df = source6.to_df()
new_source6_df = source6.to_df()
###################################################################################################################################
global data_table3, data_table4, source7_df, new_source7_df, source8_df, new_source8_df
home_id = []
global_input = []
start_times = []
end_times = []
exec_times = []
source7 = ColumnDataSource(
dict(home_id=home_id,
global_input=global_input,
start_times=start_times,
end_times=end_times,
exec_times=exec_times))
columns = [
TableColumn(field="home_id", title="Home ID"),
TableColumn(field="global_input", title="Global Input Name"),
TableColumn(field="start_times", title="Enter time"),
TableColumn(field="end_times", title="Finish tme"),
TableColumn(field="exec_times", title="Make span [s]")
]
data_table3 = DataTable(source=source7,
columns=columns,
width=600,
height=580,
selectable=True)
title3 = Div(text='Global Input Information',
style={
'font-size': '15pt',
'color': 'black',
'text-align': 'center'
},
width=600,
height=20)
source7_df = source7.to_df()
new_source7_df = source7.to_df()
data_table3.on_change('source', lambda attr, old, new: update7())
task_name = []
local_input = []
local_enter_times = []
local_exec_times = []
local_finish_times = []
local_elapse_times = []
local_duration_times = []
local_waiting_times = []
source8 = ColumnDataSource(
dict(task_name=task_name,
local_input=local_input,
local_enter_times=local_enter_times,
local_exec_times=local_exec_times,
local_finish_times=local_finish_times,
local_elapse_times=local_elapse_times,
local_duration_times=local_duration_times,
local_waiting_times=local_waiting_times))
columns = [
TableColumn(field="task_name", title="Task name"),
TableColumn(field="local_input", title="Local Input"),
TableColumn(field="local_enter_times", title="Enter time"),
TableColumn(field="local_exec_times", title="Exec time"),
TableColumn(field="local_finish_times", title="Finish time"),
TableColumn(field="local_elapse_times", title="Elapse time"),
TableColumn(field="local_duration_times", title="Duration time"),
TableColumn(field="local_waiting_times", title="Waiting time")
]
data_table4 = DataTable(source=source8,
columns=columns,
width=900,
height=580,
selectable=True)
title4 = Div(text='Local Input Information',
style={
'font-size': '15pt',
'color': 'black',
'text-align': 'center'
},
width=600,
height=20)
source8_df = source8.to_df()
new_source8_df = source8.to_df()
data_table4.on_change('source', lambda attr, old, new: update8())
###################################################################################################################################
file = open('configuration.txt', 'r')
lines = file.readlines()
lines.pop(0)
nodes = get_dag_nodes(lines)
links = get_dag_links(lines)
print(nodes)
print(links)
G = nx.DiGraph()
G.add_nodes_from(nodes)
G.add_edges_from(links)
pos = graphviz_layout(G, prog='dot')
# calculate the range for task graph
range_x = [0, 0]
range_y = [0, 0]
for each in pos.values():
range_x[0] = min(range_x[0], int(each[0]))
range_x[1] = max(range_x[1], int(each[0]))
range_y[0] = min(range_y[0], int(each[1]))
range_y[1] = max(range_y[1], int(each[1]))
range_x = [range_x[0] - 50, range_x[1] + 150]
range_y = [range_y[0] - 50, range_y[1] + 50]
p1 = Plot(plot_width=700,
plot_height=700,
x_range=Range1d(range_x[0], range_x[1]),
y_range=Range1d(range_y[0], range_y[1]))
p1.title.text = "Network Anomaly Detection Task Graph"
p.title.text_font_size = '20pt'
node_hover_tool = HoverTool(tooltips=[("task", "@index")])
p1.add_tools(node_hover_tool, BoxZoomTool(), ResetTool())
global graph
graph = from_networkx(G, pos, scale=1, center=(0, 0))
# print("DEBUG data_source:")
# print(graph.node_renderer.data_source)
# print(graph.edge_renderer.data_source.data)
# print("DEBUG keys:")
# print(pos.keys())
# print("DEBUG values:")
# print(pos.values())
colors = []
indexs = []
for i, each in enumerate(nodes):
indexs.append(i)
colors.append('blue')
graph.node_renderer.data_source.data['colors'] = colors
graph.node_renderer.glyph = Circle(size=15, fill_color='colors')
graph.edge_renderer.glyph = MultiLine(line_color="black",
line_alpha=0.8,
line_width=1)
p1.renderers.append(graph)
# add labels to each node
x, y = zip(*graph.layout_provider.graph_layout.values())
print(x, y)
node_labels = nx.get_node_attributes(G, 'index')
source = ColumnDataSource({'x': x, 'y': y, 'task': tuple(nodes)})
labels = LabelSet(x='x',
y='y',
text='task',
source=source,
background_fill_color='white',
x_offset=(-30),
y_offset=8,
text_font_size="9pt")
p1.renderers.append(labels)
###################################################################################################################################
# node_info = column(title1,widgetbox(data_table,width=400,height=280))
# task_mapping_info = column(title2,widgetbox(data_table2,width=400,height=280))
p2 = layout([
title1,
widgetbox(data_table, width=400, height=280), title2,
widgetbox(data_table2, width=400, height=280)
],
sizing_mode='fixed',
width=400,
height=600)
p3 = layout([title3, widgetbox(data_table3)], sizing_mode='fixed')
p4 = layout([title4, widgetbox(data_table4)], sizing_mode='fixed')
doc_layout = column(row(p2, p), row(p1), row(p3, p4))
doc.add_root(doc_layout)
doc.add_periodic_callback(update, 50)
return
def radialplot(tree,
node_color='node_color',
node_size='node_size',
node_alpha='node_alpha',
edge_color='edge_color',
edge_alpha='edge_alpha',
edge_width='edge_width',
hover_var='hover_var',
figsize=(500, 500),
**kwargs):
""" Plots unrooted radial tree.
Parameters
----------
tree : instance of skbio.TreeNode
Input tree for plotting.
node_color : str
Name of variable in `tree` to color nodes.
node_size : str
Name of variable in `tree` that specifies the radius of nodes.
node_alpha : str
Name of variable in `tree` to specify node transparency.
edge_color : str
Name of variable in `tree` to color edges.
edge_alpha : str
Name of variable in `tree` to specify edge transparency.
edge_width : str
Name of variable in `tree` to specify edge width.
hover_var : str
Name of variable in `tree` to display in the hover menu.
figsize : tuple, int
Size of resulting figure. default: (500, 500)
**kwargs: dict
Plotting options to pass into bokeh.models.Plot
Returns
-------
bokeh.models.Plot
Interactive plotting instance.
Notes
-----
This assumes that the tree is strictly bifurcating.
See also
--------
bifurcate
"""
# TODO: Add in example doc string
# This entire function was motivated by
# http://chuckpr.github.io/blog/trees2.html
t = UnrootedDendrogram.from_tree(tree)
nodes = t.coords(figsize[0], figsize[1])
# fill in all of the node attributes
def _retreive(tree, x, default):
return pd.Series(
{n.name: getattr(n, x, default)
for n in tree.levelorder()})
# default node color to light grey
nodes[node_color] = _retreive(t, node_color, default='#D3D3D3')
nodes[node_size] = _retreive(t, node_size, default=1)
nodes[node_alpha] = _retreive(t, node_alpha, default=1)
nodes[hover_var] = _retreive(t, hover_var, default=None)
edges = nodes[['child0', 'child1']]
edges = edges.dropna(subset=['child0', 'child1'])
edges = edges.unstack()
edges = pd.DataFrame({
'src_node': edges.index.get_level_values(1),
'dest_node': edges.values
})
edges['x0'] = [nodes.loc[n].x for n in edges.src_node]
edges['x1'] = [nodes.loc[n].x for n in edges.dest_node]
edges['y0'] = [nodes.loc[n].y for n in edges.src_node]
edges['y1'] = [nodes.loc[n].y for n in edges.dest_node]
ns = [n.name for n in t.levelorder(include_self=True)]
attrs = pd.DataFrame(index=ns)
# default edge color to black
attrs[edge_color] = _retreive(t, edge_color, default='#000000')
attrs[edge_width] = _retreive(t, edge_width, default=1)
attrs[edge_alpha] = _retreive(t, edge_alpha, default=1)
edges = pd.merge(edges,
attrs,
left_on='dest_node',
right_index=True,
how='outer')
edges = edges.dropna(subset=['src_node'])
node_glyph = Circle(x="x",
y="y",
radius=node_size,
fill_color=node_color,
fill_alpha=node_alpha)
edge_glyph = Segment(x0="x0",
y0="y0",
x1="x1",
y1="y1",
line_color=edge_color,
line_alpha=edge_alpha,
line_width=edge_width)
def df2ds(df):
return ColumnDataSource(ColumnDataSource.from_df(df))
ydr = DataRange1d(range_padding=0.05)
xdr = DataRange1d(range_padding=0.05)
plot = Plot(x_range=xdr, y_range=ydr, **kwargs)
plot.add_glyph(df2ds(edges), edge_glyph)
ns = plot.add_glyph(df2ds(nodes), node_glyph)
tooltip = [("Feature ID", "@index")]
if hover_var is not None:
tooltip += [(hover_var, "@" + hover_var)]
hover = HoverTool(renderers=[ns], tooltips=tooltip)
plot.add_tools(hover, BoxZoomTool(), ResetTool())
return plot
def plot_data(data_df, connections, year, geoSource_new, df):
d_yr = df[df['Years'] == year]
data_df_countries = d_yr.merge(data_df,
left_on='Origin_Country',
right_on='Country')
#data_df_countries = node_dt #.drop_duplicates(subset=None, keep='first', inplace=True)
connections_df = connections
# node_source = ColumnDataSource(data_df_countries[["country_id","Country", "Longitude", "Latitude"]])
node_source = ColumnDataSource(data_df_countries)
edge_source = ColumnDataSource(connections_df[["start", "end"]])
node_renderer = GlyphRenderer(data_source=node_source,
glyph=node_glyph,
selection_glyph=node_selection,
nonselection_glyph=node_nonselection)
## Create edge_renderer
edge_renderer = GlyphRenderer(data_source=edge_source,
glyph=edge_glyph,
hover_glyph=edge_hover,
selection_glyph=edge_selection,
nonselection_glyph=edge_nonselection)
## Create layout_provider
graph_layout = dict(
zip(data_df_countries.country_id.astype(str),
zip(data_df_countries.Longitude, data_df_countries.Latitude)))
layout_provider = StaticLayoutProvider(graph_layout=graph_layout)
## Create graph renderer
graph = GraphRenderer(edge_renderer=edge_renderer,
node_renderer=node_renderer,
layout_provider=layout_provider,
inspection_policy=NodesAndLinkedEdges(),
selection_policy=NodesAndLinkedEdges())
plot = Plot(x_range=Range1d(-150, 150),
y_range=Range1d(15, 75),
plot_width=800,
plot_height=600,
background_fill_color=Set3_12[4],
background_fill_alpha=0.2)
plot.title.text = "Human Trafficing Visualization for " + str(year)
plot.add_glyph(
geoSource_new,
Patches(xs='xs',
ys='ys',
line_color='grey',
line_width=.2,
fill_color={
'field': 'Tier',
'transform': mapper2
},
fill_alpha=0.40))
plot.renderers.append(graph)
plot.add_layout(LinearAxis(axis_label="Latitude"), "below")
plot.add_layout(LinearAxis(axis_label="Longitude"), "left")
# tooltips="""
# <div>
# <div>
# <span style="font-size: 17px; font-weight: bold;">@desc</span>
# <span style="font-size: 15px; color: #966;">[$index]</span>
# </div>
# <div>
# <span style="font-size: 15px;">Country Involved</span>
# <span style="font-size: 10px; color: #696;">@Country</span>
# </div>
# </div>
# """,
hover = HoverTool(
show_arrow=True, # tooltips= # [("Country Involved: ", "@Country")],
tooltips="""
<div>
<div>
<span style="font-size: 13px;">Country Info</span>
<span style="font-size: 12px; color: #696;">@Destination_Country, @Type_Of_Trafficking</span>
</div>
</div>
""",
renderers=[node_renderer],
name='Test')
hover_no_tooltips = HoverTool(tooltips=None, renderers=[graph])
box_zoom = BoxZoomTool()
plot.add_tools(hover, hover_no_tooltips, box_zoom, TapTool(),
BoxSelectTool(), ResetTool(), WheelZoomTool())
plot.toolbar.active_inspect = [hover, hover_no_tooltips]
plot.toolbar.active_drag = box_zoom
plot.outline_line_color = "navy"
plot.outline_line_alpha = 0.3
plot.outline_line_width = 1
select_overlay = plot.select_one(BoxSelectTool).overlay
select_overlay.fill_color = "firebrick"
select_overlay.line_color = None
zoom_overlay = plot.select_one(BoxZoomTool).overlay
zoom_overlay.line_color = "olive"
zoom_overlay.line_width = 3
zoom_overlay.line_dash = "solid"
zoom_overlay.fill_color = None
plot.add_tile(STAMEN_TONER_LABELS)
return plot
N = 200
x = linspace(-2 * pi, 2 * pi, N)
y = sin(x)*exp(-x)
# Create an array of synthetic times, starting at the current time, and extending 24hrs
times = (linspace(0, 24*3600, N) + time.time()) * 1000
source = ColumnDataSource(data=dict(x=x, y=y, times=times))
plot = Plot(min_border=80, plot_width=800, plot_height=350, background_fill_color="#efefef")
circle = Circle(x="times", y="y", fill_color="red", size=3, line_color=None, fill_alpha=0.5)
plot.add_glyph(source, circle)
plot.add_layout(DatetimeAxis(), 'below')
plot.add_layout(DatetimeAxis(), 'left')
plot.add_tools(PanTool(), WheelZoomTool(zoom_on_axis=False, speed=1/5000.))
doc = Document()
doc.add_root(plot)
if __name__ == "__main__":
doc.validate()
filename = "dateaxis.html"
with open(filename, "w") as f:
f.write(file_html(doc, INLINE, "Date Axis Example"))
print("Wrote %s" % filename)
view(filename)
# Add the circle
renderer_source = sources['_%s' % years[0]]
circle_glyph = Circle(x='fertility',
y='life',
size='population',
fill_color='region_color',
fill_alpha=0.8,
line_color='#7c7e71',
line_width=0.5,
line_alpha=0.5)
circle_renderer = plot.add_glyph(renderer_source, circle_glyph)
# Add the hover (only against the circle and not other plot elements)
tooltips = "@index"
plot.add_tools(HoverTool(tooltips=tooltips, renderers=[circle_renderer]))
# Add the legend
text_x = 7
text_y = 95
for i, region in enumerate(regions):
plot.add_glyph(
Text(x=text_x,
y=text_y,
text=[region],
text_font_size='10pt',
text_color='#666666'))
plot.add_glyph(
Circle(x=text_x - 0.1,
y=text_y + 2,
fill_color=Spectral6[i],
def create(palm):
doc = curdoc()
# Calibration averaged waveforms per photon energy
waveform_plot = Plot(
title=Title(text="eTOF calibration waveforms"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=760,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location='right',
)
# ---- tools
waveform_plot.toolbar.logo = None
waveform_plot_hovertool = HoverTool(
tooltips=[("energy, eV", '@en'), ("eTOF bin", '$x{0.}')])
waveform_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool(), waveform_plot_hovertool)
# ---- axes
waveform_plot.add_layout(LinearAxis(axis_label='eTOF time bin'),
place='below')
waveform_plot.add_layout(LinearAxis(axis_label='Intensity',
major_label_orientation='vertical'),
place='left')
# ---- grid lines
waveform_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
waveform_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- multiline glyphs
waveform_ref_source = ColumnDataSource(dict(xs=[], ys=[], en=[]))
waveform_ref_multiline = waveform_plot.add_glyph(
waveform_ref_source, MultiLine(xs='xs', ys='ys', line_color='blue'))
waveform_str_source = ColumnDataSource(dict(xs=[], ys=[], en=[]))
waveform_str_multiline = waveform_plot.add_glyph(
waveform_str_source, MultiLine(xs='xs', ys='ys', line_color='red'))
# ---- legend
waveform_plot.add_layout(
Legend(items=[(
"reference",
[waveform_ref_multiline]), ("streaked",
[waveform_str_multiline])]))
waveform_plot.legend.click_policy = "hide"
# ---- vertical spans
photon_peak_ref_span = Span(location=0,
dimension='height',
line_dash='dashed',
line_color='blue')
photon_peak_str_span = Span(location=0,
dimension='height',
line_dash='dashed',
line_color='red')
waveform_plot.add_layout(photon_peak_ref_span)
waveform_plot.add_layout(photon_peak_str_span)
# Calibration fit plot
fit_plot = Plot(
title=Title(text="eTOF calibration fit"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location='right',
)
# ---- tools
fit_plot.toolbar.logo = None
fit_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(), ResetTool())
# ---- axes
fit_plot.add_layout(LinearAxis(axis_label='Photoelectron peak shift'),
place='below')
fit_plot.add_layout(LinearAxis(axis_label='Photon energy, eV',
major_label_orientation='vertical'),
place='left')
# ---- grid lines
fit_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
fit_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- circle glyphs
fit_ref_circle_source = ColumnDataSource(dict(x=[], y=[]))
fit_ref_circle = fit_plot.add_glyph(
fit_ref_circle_source, Circle(x='x', y='y', line_color='blue'))
fit_str_circle_source = ColumnDataSource(dict(x=[], y=[]))
fit_str_circle = fit_plot.add_glyph(fit_str_circle_source,
Circle(x='x', y='y', line_color='red'))
# ---- line glyphs
fit_ref_line_source = ColumnDataSource(dict(x=[], y=[]))
fit_ref_line = fit_plot.add_glyph(fit_ref_line_source,
Line(x='x', y='y', line_color='blue'))
fit_str_line_source = ColumnDataSource(dict(x=[], y=[]))
fit_str_line = fit_plot.add_glyph(fit_str_line_source,
Line(x='x', y='y', line_color='red'))
# ---- legend
fit_plot.add_layout(
Legend(items=[
("reference", [fit_ref_circle, fit_ref_line]),
("streaked", [fit_str_circle, fit_str_line]),
]))
fit_plot.legend.click_policy = "hide"
# Calibration results datatables
def datatable_ref_source_callback(_attr, _old, new):
for en, ps, use in zip(new['energy'], new['peak_pos_ref'],
new['use_in_fit']):
palm.etofs['0'].calib_data.loc[
en, 'calib_tpeak'] = ps if ps != 'NaN' else np.nan
palm.etofs['0'].calib_data.loc[en, 'use_in_fit'] = use
calib_res = {}
for etof_key in palm.etofs:
calib_res[etof_key] = palm.etofs[etof_key].fit_calibration_curve()
update_calibration_plot(calib_res)
datatable_ref_source = ColumnDataSource(
dict(energy=['', '', ''],
peak_pos_ref=['', '', ''],
use_in_fit=[True, True, True]))
datatable_ref_source.on_change('data', datatable_ref_source_callback)
datatable_ref = DataTable(
source=datatable_ref_source,
columns=[
TableColumn(field='energy',
title="Photon Energy, eV",
editor=IntEditor()),
TableColumn(field='peak_pos_ref',
title="Reference Peak",
editor=IntEditor()),
TableColumn(field='use_in_fit',
title=" ",
editor=CheckboxEditor(),
width=80),
],
index_position=None,
editable=True,
height=300,
width=250,
)
def datatable_str_source_callback(_attr, _old, new):
for en, ps, use in zip(new['energy'], new['peak_pos_str'],
new['use_in_fit']):
palm.etofs['1'].calib_data.loc[
en, 'calib_tpeak'] = ps if ps != 'NaN' else np.nan
palm.etofs['1'].calib_data.loc[en, 'use_in_fit'] = use
calib_res = {}
for etof_key in palm.etofs:
calib_res[etof_key] = palm.etofs[etof_key].fit_calibration_curve()
update_calibration_plot(calib_res)
datatable_str_source = ColumnDataSource(
dict(energy=['', '', ''],
peak_pos_str=['', '', ''],
use_in_fit=[True, True, True]))
datatable_str_source.on_change('data', datatable_str_source_callback)
datatable_str = DataTable(
source=datatable_str_source,
columns=[
TableColumn(field='energy',
title="Photon Energy, eV",
editor=IntEditor()),
TableColumn(field='peak_pos_str',
title="Streaked Peak",
editor=IntEditor()),
TableColumn(field='use_in_fit',
title=" ",
editor=CheckboxEditor(),
width=80),
],
index_position=None,
editable=True,
height=350,
width=250,
)
# eTOF calibration folder path text input
def path_textinput_callback(_attr, _old, _new):
path_periodic_update()
update_load_dropdown_menu()
path_textinput = TextInput(title="eTOF calibration path:",
value=os.path.join(os.path.expanduser('~')),
width=510)
path_textinput.on_change('value', path_textinput_callback)
# eTOF calibration eco scans dropdown
def scans_dropdown_callback(_attr, _old, new):
scans_dropdown.label = new
scans_dropdown = Dropdown(label="ECO scans",
button_type='default',
menu=[])
scans_dropdown.on_change('value', scans_dropdown_callback)
# ---- etof scans periodic update
def path_periodic_update():
new_menu = []
if os.path.isdir(path_textinput.value):
for entry in os.scandir(path_textinput.value):
if entry.is_file() and entry.name.endswith('.json'):
new_menu.append((entry.name, entry.name))
scans_dropdown.menu = sorted(new_menu, reverse=True)
doc.add_periodic_callback(path_periodic_update, 5000)
# Calibrate button
def calibrate_button_callback():
try:
palm.calibrate_etof_eco(eco_scan_filename=os.path.join(
path_textinput.value, scans_dropdown.value))
except Exception:
palm.calibrate_etof(folder_name=path_textinput.value)
datatable_ref_source.data.update(
energy=palm.etofs['0'].calib_data.index.tolist(),
peak_pos_ref=palm.etofs['0'].calib_data['calib_tpeak'].tolist(),
use_in_fit=palm.etofs['0'].calib_data['use_in_fit'].tolist(),
)
datatable_str_source.data.update(
energy=palm.etofs['0'].calib_data.index.tolist(),
peak_pos_str=palm.etofs['1'].calib_data['calib_tpeak'].tolist(),
use_in_fit=palm.etofs['1'].calib_data['use_in_fit'].tolist(),
)
def update_calibration_plot(calib_res):
etof_ref = palm.etofs['0']
etof_str = palm.etofs['1']
shift_val = 0
etof_ref_wf_shifted = []
etof_str_wf_shifted = []
for wf_ref, wf_str in zip(etof_ref.calib_data['waveform'],
etof_str.calib_data['waveform']):
shift_val -= max(wf_ref.max(), wf_str.max())
etof_ref_wf_shifted.append(wf_ref + shift_val)
etof_str_wf_shifted.append(wf_str + shift_val)
waveform_ref_source.data.update(
xs=len(etof_ref.calib_data) *
[list(range(etof_ref.internal_time_bins))],
ys=etof_ref_wf_shifted,
en=etof_ref.calib_data.index.tolist(),
)
waveform_str_source.data.update(
xs=len(etof_str.calib_data) *
[list(range(etof_str.internal_time_bins))],
ys=etof_str_wf_shifted,
en=etof_str.calib_data.index.tolist(),
)
photon_peak_ref_span.location = etof_ref.calib_t0
photon_peak_str_span.location = etof_str.calib_t0
def plot_fit(time, calib_a, calib_b):
time_fit = np.linspace(np.nanmin(time), np.nanmax(time), 100)
en_fit = (calib_a / time_fit)**2 + calib_b
return time_fit, en_fit
def update_plot(calib_results, circle, line):
(a, c), x, y = calib_results
x_fit, y_fit = plot_fit(x, a, c)
circle.data.update(x=x, y=y)
line.data.update(x=x_fit, y=y_fit)
update_plot(calib_res['0'], fit_ref_circle_source, fit_ref_line_source)
update_plot(calib_res['1'], fit_str_circle_source, fit_str_line_source)
calib_const_div.text = """
a_str = {:.2f}<br>
b_str = {:.2f}<br>
<br>
a_ref = {:.2f}<br>
b_ref = {:.2f}
""".format(etof_str.calib_a, etof_str.calib_b, etof_ref.calib_a,
etof_ref.calib_b)
calibrate_button = Button(label="Calibrate eTOF",
button_type='default',
width=250)
calibrate_button.on_click(calibrate_button_callback)
# Photon peak noise threshold value text input
def phot_peak_noise_thr_textinput_callback(_attr, old, new):
try:
new_value = float(new)
if new_value > 0:
for etof in palm.etofs.values():
etof.photon_peak_noise_thr = new_value
else:
phot_peak_noise_thr_textinput.value = old
except ValueError:
phot_peak_noise_thr_textinput.value = old
phot_peak_noise_thr_textinput = TextInput(
title='Photon peak noise threshold:', value=str(1))
phot_peak_noise_thr_textinput.on_change(
'value', phot_peak_noise_thr_textinput_callback)
# Electron peak noise threshold value text input
def el_peak_noise_thr_textinput_callback(_attr, old, new):
try:
new_value = float(new)
if new_value > 0:
for etof in palm.etofs.values():
etof.electron_peak_noise_thr = new_value
else:
el_peak_noise_thr_textinput.value = old
except ValueError:
el_peak_noise_thr_textinput.value = old
el_peak_noise_thr_textinput = TextInput(
title='Electron peak noise threshold:', value=str(10))
el_peak_noise_thr_textinput.on_change(
'value', el_peak_noise_thr_textinput_callback)
# Save calibration button
def save_button_callback():
palm.save_etof_calib(path=path_textinput.value)
update_load_dropdown_menu()
save_button = Button(label="Save", button_type='default', width=250)
save_button.on_click(save_button_callback)
# Load calibration button
def load_dropdown_callback(_attr, _old, new):
if new:
palm.load_etof_calib(os.path.join(path_textinput.value, new))
datatable_ref_source.data.update(
energy=palm.etofs['0'].calib_data.index.tolist(),
peak_pos_ref=palm.etofs['0'].calib_data['calib_tpeak'].tolist(
),
use_in_fit=palm.etofs['0'].calib_data['use_in_fit'].tolist(),
)
datatable_str_source.data.update(
energy=palm.etofs['0'].calib_data.index.tolist(),
peak_pos_str=palm.etofs['1'].calib_data['calib_tpeak'].tolist(
),
use_in_fit=palm.etofs['1'].calib_data['use_in_fit'].tolist(),
)
# Drop selection, so that this callback can be triggered again on the same dropdown menu
# item from the user perspective
load_dropdown.value = ''
def update_load_dropdown_menu():
new_menu = []
calib_file_ext = '.palm_etof'
if os.path.isdir(path_textinput.value):
for entry in os.scandir(path_textinput.value):
if entry.is_file() and entry.name.endswith((calib_file_ext)):
new_menu.append(
(entry.name[:-len(calib_file_ext)], entry.name))
load_dropdown.button_type = 'default'
load_dropdown.menu = sorted(new_menu, reverse=True)
else:
load_dropdown.button_type = 'danger'
load_dropdown.menu = new_menu
doc.add_next_tick_callback(update_load_dropdown_menu)
doc.add_periodic_callback(update_load_dropdown_menu, 5000)
load_dropdown = Dropdown(label="Load", menu=[], width=250)
load_dropdown.on_change('value', load_dropdown_callback)
# eTOF fitting equation
fit_eq_div = Div(
text="""Fitting equation:<br><br><img src="/palm/static/5euwuy.gif">"""
)
# Calibration constants
calib_const_div = Div(text="""
a_str = {}<br>
b_str = {}<br>
<br>
a_ref = {}<br>
b_ref = {}
""".format(0, 0, 0, 0))
# assemble
tab_layout = column(
row(
column(waveform_plot, fit_plot),
Spacer(width=30),
column(
path_textinput,
scans_dropdown,
calibrate_button,
phot_peak_noise_thr_textinput,
el_peak_noise_thr_textinput,
row(save_button, load_dropdown),
row(datatable_ref, datatable_str),
calib_const_div,
fit_eq_div,
),
))
return Panel(child=tab_layout, title="eTOF Calibration")
class MemoryUsage(DashboardComponent):
""" The memory usage across the cluster, grouped by task type """
def __init__(self, **kwargs):
self.source = ColumnDataSource(data=dict(
name=[],
left=[],
right=[],
center=[],
color=[],
percent=[],
MB=[],
text=[],
))
self.root = Plot(id="bk-nbytes-plot",
x_range=DataRange1d(),
y_range=DataRange1d(),
toolbar_location=None,
outline_line_color=None,
**kwargs)
self.root.add_glyph(
self.source,
Quad(
top=1,
bottom=0,
left="left",
right="right",
fill_color="color",
fill_alpha=1,
),
)
self.root.add_layout(LinearAxis(), "left")
self.root.add_layout(LinearAxis(), "below")
hover = HoverTool(
point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 14px; font-weight: bold;">Name:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Percent:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@percent</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">MB:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@MB</span>
</div>
""",
)
self.root.add_tools(hover)
@without_property_validation
def update(self, messages):
with log_errors():
msg = messages["progress"]
if not msg:
return
nb = nbytes_bar(msg["nbytes"])
self.source.data.update(nb)
self.root.title.text = "Memory Use: %0.2f MB" % (
sum(msg["nbytes"].values()) / 1e6)
# genre = Select(title="Genre", value="All", options=genre_tags)
# not working yet
graph_renderer = from_networkx(G, nx.spring_layout, scale=1, center=(0, 0))
callback = CustomJS(args=dict(source=graph_renderer.node_renderer.data_source),
code="""
console.log(cb_data.source)
var inds = cb_data.source.selected['1d'].indices;
var artist = cb_data.source.data.Img[inds]
window.alert(artist);
""")
node_hover_tool = HoverTool(tooltips=tips)
node_select_tool = TapTool(callback=callback)
plot.add_tools(node_hover_tool, WheelZoomTool(), PanTool(), ResetTool(),
node_select_tool)
graph_renderer.node_renderer.glyph = Circle(size=node_size,
fill_color=Spectral4[0])
graph_renderer.edge_renderer.glyph = MultiLine(line_color="black",
line_alpha=0.6,
line_width=1)
plot.renderers.append(graph_renderer)
# def select_tags():
# genre_val = genre.value
# selected = G.node
# if genre_val != "All":
# selected = selected[selected.Genre.str.contains(genre_val) == True]
# return selected
#
def main(
keywords: str,
pdf_dir: str,
):
"""Automated EDI Tagging of Course Syllabi"""
keywords_path = keywords
keywords_df = pd.read_csv(keywords_path, sep='\t')
# Store the mapping of concept to keyword in a dictionary
concept_dict = {}
# Store the syllabi data in a separate dict
data_dict = {}
# Parse the keywords file into a dictionary
for concept in keywords_df["Concept"]:
# Store the concept in the dictionary
if concept not in concept_dict:
concept_dict[concept] = {}
# Store the keywords associated with a concept
concept_df = keywords_df[keywords_df["Concept"] == concept]
keywords = concept_df["Keywords"].values[0]
# Keywords should be a comma-separated list
keywords_list = keywords.split(",")
# Add the keywords to the dictionary along with word counters
concept_dict[concept] = {k: 0 for k in keywords_list}
# Iterate through the pdfs directory
for file in os.listdir(pdf_dir):
# Store the relative path to the file
file_path = os.path.join(pdf_dir, file)
# Identify the file extension
ext = os.path.splitext(file)[1]
file_name = os.path.splitext(file)[0]
year = file_name.split("_")[0]
term = file_name.split("_")[1]
department = file_name.split("_")[2]
course = file_name.split("_")[3]
# If it's a text file
if ext == ".txt":
txt_file = open(file_path, "rt")
file_content = txt_file.read().strip().lower()
# If it's a pdf file
elif ext == ".pdf":
continue
# Add course to dict
data_dict[course] = {}
for concept in concept_dict:
# Add concept to course
data_dict[course][concept] = {}
for keyword in concept_dict[concept]:
# Count occurrence of the keyword
keyword_count = file_content.count(keyword)
# If not 0, Add the count to the concept dictionary
if keyword_count > 0:
# Increment counters
data_dict[course][concept][keyword] = keyword_count
# If no keywords were found, remove this concept from the data
if len(data_dict[course][concept]) == 0:
data_dict[course].pop(concept)
#-----------------------------------------------
# Create network graph
# Start with an empty undirected graph
G = nx.Graph()
# Or add 1 level dict
#G = nx.Graph(data_dict)
for course in data_dict.keys():
for concept in data_dict[course].keys():
for keyword in data_dict[course][concept].keys():
G.add_edge(course, concept)
G.add_edge(concept, keyword)
# Use the boken accessory function
gr = from_networkx(G, nx.spring_layout, scale=1, center=(0, 0))
#gr = from_networkx(G, nx.shell_layout, center=(0, 0))
# Check data
# print(gr.node_renderer.data_source.data)
# Style nodes
node_attr = {}
base_size = 10
keyword_color = Category10[4][0]
concept_color = Category10[4][1]
course_color = Category10[4][2]
# Initialize default values for nodes
node_attr["size"] = {node_name: base_size for node_name in G.nodes}
node_attr["color"] = {node_name: keyword_color for node_name in G.nodes}
node_attr["concept_num"] = {node_name: 0 for node_name in G.nodes}
node_attr["kw_num"] = {node_name: 0 for node_name in G.nodes}
for node in G.nodes:
# Check if it's a course
if node in data_dict:
node_attr["color"][node] = course_color
node_attr["size"][node] = len(data_dict[node]) * base_size
node_attr["concept_num"][node] = len(data_dict[node])
node_attr["kw_num"][node] = count_elements(data_dict[node])
# Check if it's a concept
else:
for course in data_dict:
for concept in data_dict[course]:
if node == concept:
node_attr["size"][node] = len(
data_dict[course][node]) * base_size
node_attr["color"][node] = concept_color
node_attr["concept_num"][node] = 1
node_attr["kw_num"][node] = count_elements(
data_dict[course][node])
break
# gr.node_renderer.data_source.data is a dictionary
gr.node_renderer.data_source.data['size'] = list(
node_attr["size"].values())
gr.node_renderer.data_source.data['color'] = list(
node_attr["color"].values())
gr.node_renderer.data_source.data['concept_num'] = list(
node_attr["concept_num"].values())
gr.node_renderer.data_source.data['kw_num'] = list(
node_attr["kw_num"].values())
gr.node_renderer.glyph = Circle(
size='size',
fill_color='color',
)
#-----------------------------------------------
# Plotting
# Plot setup
plot = Plot(plot_width=600, plot_height=400)
plot.title.text = "EDI Tagging | Department of History | Fall 2020"
node_hover_tool = HoverTool(
tooltips=[("Name", "@index"), ("Concepts",
"@concept_num"), ("Keywords",
"@kw_num")])
plot.add_tools(node_hover_tool, BoxZoomTool(), ResetTool())
# Add the graph to the plot
plot.renderers.append(gr)
# Save to file
html = file_html(plot, CDN, "tmp.html")
with open("edi-tagging-graph.html", "w") as outfile:
outfile.write(html)
import networkx as nx
from bokeh.io import output_file, show
from bokeh.models import (BoxSelectTool, Circle, EdgesAndLinkedNodes, HoverTool,
MultiLine, NodesAndLinkedEdges, Plot, Range1d, TapTool,)
from bokeh.palettes import Spectral4
from bokeh.plotting import from_networkx
G=nx.karate_club_graph()
plot = Plot(plot_width=400, plot_height=400,
x_range=Range1d(-1.1,1.1), y_range=Range1d(-1.1,1.1))
plot.title.text = "Graph Interaction Demonstration"
plot.add_tools(HoverTool(tooltips=None), TapTool(), BoxSelectTool())
graph_renderer = from_networkx(G, nx.circular_layout, scale=1, center=(0,0))
graph_renderer.node_renderer.glyph = Circle(size=15, fill_color=Spectral4[0])
graph_renderer.node_renderer.selection_glyph = Circle(size=15, fill_color=Spectral4[2])
graph_renderer.node_renderer.hover_glyph = Circle(size=15, fill_color=Spectral4[1])
graph_renderer.edge_renderer.glyph = MultiLine(line_color="#CCCCCC", line_alpha=0.8, line_width=5)
graph_renderer.edge_renderer.selection_glyph = MultiLine(line_color=Spectral4[2], line_width=5)
graph_renderer.edge_renderer.hover_glyph = MultiLine(line_color=Spectral4[1], line_width=5)
graph_renderer.selection_policy = NodesAndLinkedEdges()
graph_renderer.inspection_policy = EdgesAndLinkedNodes()
plot.renderers.append(graph_renderer)
def create(palm):
doc = curdoc()
# Calibration averaged waveforms per photon energy
waveform_plot = Plot(
title=Title(text="eTOF calibration waveforms"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=760,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
waveform_plot.toolbar.logo = None
waveform_plot_hovertool = HoverTool(
tooltips=[("energy, eV", "@en"), ("eTOF bin", "$x{0.}")])
waveform_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool(), waveform_plot_hovertool)
# ---- axes
waveform_plot.add_layout(LinearAxis(axis_label="eTOF time bin"),
place="below")
waveform_plot.add_layout(LinearAxis(axis_label="Intensity",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
waveform_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
waveform_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- multiline glyphs
waveform_ref_source = ColumnDataSource(dict(xs=[], ys=[], en=[]))
waveform_ref_multiline = waveform_plot.add_glyph(
waveform_ref_source, MultiLine(xs="xs", ys="ys", line_color="blue"))
waveform_str_source = ColumnDataSource(dict(xs=[], ys=[], en=[]))
waveform_str_multiline = waveform_plot.add_glyph(
waveform_str_source, MultiLine(xs="xs", ys="ys", line_color="red"))
# ---- legend
waveform_plot.add_layout(
Legend(items=[(
"reference",
[waveform_ref_multiline]), ("streaked",
[waveform_str_multiline])]))
waveform_plot.legend.click_policy = "hide"
# ---- vertical spans
photon_peak_ref_span = Span(location=0,
dimension="height",
line_dash="dashed",
line_color="blue")
photon_peak_str_span = Span(location=0,
dimension="height",
line_dash="dashed",
line_color="red")
waveform_plot.add_layout(photon_peak_ref_span)
waveform_plot.add_layout(photon_peak_str_span)
# Calibration fit plot
fit_plot = Plot(
title=Title(text="eTOF calibration fit"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
fit_plot.toolbar.logo = None
fit_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(), ResetTool())
# ---- axes
fit_plot.add_layout(LinearAxis(axis_label="Photoelectron peak shift"),
place="below")
fit_plot.add_layout(LinearAxis(axis_label="Photon energy, eV",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
fit_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
fit_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- circle glyphs
fit_ref_circle_source = ColumnDataSource(dict(x=[], y=[]))
fit_ref_circle = fit_plot.add_glyph(
fit_ref_circle_source, Circle(x="x", y="y", line_color="blue"))
fit_str_circle_source = ColumnDataSource(dict(x=[], y=[]))
fit_str_circle = fit_plot.add_glyph(fit_str_circle_source,
Circle(x="x", y="y", line_color="red"))
# ---- line glyphs
fit_ref_line_source = ColumnDataSource(dict(x=[], y=[]))
fit_ref_line = fit_plot.add_glyph(fit_ref_line_source,
Line(x="x", y="y", line_color="blue"))
fit_str_line_source = ColumnDataSource(dict(x=[], y=[]))
fit_str_line = fit_plot.add_glyph(fit_str_line_source,
Line(x="x", y="y", line_color="red"))
# ---- legend
fit_plot.add_layout(
Legend(items=[
("reference", [fit_ref_circle, fit_ref_line]),
("streaked", [fit_str_circle, fit_str_line]),
]))
fit_plot.legend.click_policy = "hide"
# Calibration results datatables
def datatable_ref_source_callback(_attr, _old_value, new_value):
for en, ps, use in zip(new_value["energy"], new_value["peak_pos_ref"],
new_value["use_in_fit"]):
palm.etofs["0"].calib_data.loc[
en, "calib_tpeak"] = ps if ps != "NaN" else np.nan
palm.etofs["0"].calib_data.loc[en, "use_in_fit"] = use
calib_res = {}
for etof_key in palm.etofs:
calib_res[etof_key] = palm.etofs[etof_key].fit_calibration_curve()
update_calibration_plot(calib_res)
datatable_ref_source = ColumnDataSource(
dict(energy=["", "", ""],
peak_pos_ref=["", "", ""],
use_in_fit=[True, True, True]))
datatable_ref_source.on_change("data", datatable_ref_source_callback)
datatable_ref = DataTable(
source=datatable_ref_source,
columns=[
TableColumn(field="energy",
title="Photon Energy, eV",
editor=IntEditor()),
TableColumn(field="peak_pos_ref",
title="Reference Peak",
editor=IntEditor()),
TableColumn(field="use_in_fit",
title=" ",
editor=CheckboxEditor(),
width=80),
],
index_position=None,
editable=True,
height=300,
width=250,
)
def datatable_str_source_callback(_attr, _old_value, new_value):
for en, ps, use in zip(new_value["energy"], new_value["peak_pos_str"],
new_value["use_in_fit"]):
palm.etofs["1"].calib_data.loc[
en, "calib_tpeak"] = ps if ps != "NaN" else np.nan
palm.etofs["1"].calib_data.loc[en, "use_in_fit"] = use
calib_res = {}
for etof_key in palm.etofs:
calib_res[etof_key] = palm.etofs[etof_key].fit_calibration_curve()
update_calibration_plot(calib_res)
datatable_str_source = ColumnDataSource(
dict(energy=["", "", ""],
peak_pos_str=["", "", ""],
use_in_fit=[True, True, True]))
datatable_str_source.on_change("data", datatable_str_source_callback)
datatable_str = DataTable(
source=datatable_str_source,
columns=[
TableColumn(field="energy",
title="Photon Energy, eV",
editor=IntEditor()),
TableColumn(field="peak_pos_str",
title="Streaked Peak",
editor=IntEditor()),
TableColumn(field="use_in_fit",
title=" ",
editor=CheckboxEditor(),
width=80),
],
index_position=None,
editable=True,
height=350,
width=250,
)
# eTOF calibration folder path text input
def path_textinput_callback(_attr, _old_value, _new_value):
path_periodic_update()
update_load_dropdown_menu()
path_textinput = TextInput(title="eTOF calibration path:",
value=os.path.join(os.path.expanduser("~")),
width=510)
path_textinput.on_change("value", path_textinput_callback)
# eTOF calibration eco scans dropdown
def scans_dropdown_callback(event):
scans_dropdown.label = event.item
scans_dropdown = Dropdown(label="ECO scans",
button_type="default",
menu=[])
scans_dropdown.on_click(scans_dropdown_callback)
# ---- etof scans periodic update
def path_periodic_update():
new_menu = []
if os.path.isdir(path_textinput.value):
for entry in os.scandir(path_textinput.value):
if entry.is_file() and entry.name.endswith(".json"):
new_menu.append((entry.name, entry.name))
scans_dropdown.menu = sorted(new_menu, reverse=True)
doc.add_periodic_callback(path_periodic_update, 5000)
path_tab = Panel(child=column(
path_textinput,
scans_dropdown,
),
title="Path")
upload_div = Div(text="Upload ECO scan (top) and all hdf5 files (bottom):")
# ECO scan upload FileInput
def eco_fileinput_callback(_attr, _old, new):
with io.BytesIO(base64.b64decode(new)) as eco_scan:
data = json.load(eco_scan)
print(data)
eco_fileinput = FileInput(accept=".json", disabled=True)
eco_fileinput.on_change("value", eco_fileinput_callback)
# HDF5 upload FileInput
def hdf5_fileinput_callback(_attr, _old, new):
for base64_str in new:
with io.BytesIO(base64.b64decode(base64_str)) as hdf5_file:
with h5py.File(hdf5_file, "r") as h5f:
print(h5f.keys())
hdf5_fileinput = FileInput(accept=".hdf5,.h5",
multiple=True,
disabled=True)
hdf5_fileinput.on_change("value", hdf5_fileinput_callback)
upload_tab = Panel(child=column(upload_div, eco_fileinput, hdf5_fileinput),
title="Upload")
# Calibrate button
def calibrate_button_callback():
try:
palm.calibrate_etof_eco(eco_scan_filename=os.path.join(
path_textinput.value, scans_dropdown.label))
except Exception:
palm.calibrate_etof(folder_name=path_textinput.value)
datatable_ref_source.data.update(
energy=palm.etofs["0"].calib_data.index.tolist(),
peak_pos_ref=palm.etofs["0"].calib_data["calib_tpeak"].tolist(),
use_in_fit=palm.etofs["0"].calib_data["use_in_fit"].tolist(),
)
datatable_str_source.data.update(
energy=palm.etofs["0"].calib_data.index.tolist(),
peak_pos_str=palm.etofs["1"].calib_data["calib_tpeak"].tolist(),
use_in_fit=palm.etofs["1"].calib_data["use_in_fit"].tolist(),
)
def update_calibration_plot(calib_res):
etof_ref = palm.etofs["0"]
etof_str = palm.etofs["1"]
shift_val = 0
etof_ref_wf_shifted = []
etof_str_wf_shifted = []
for wf_ref, wf_str in zip(etof_ref.calib_data["waveform"],
etof_str.calib_data["waveform"]):
shift_val -= max(wf_ref.max(), wf_str.max())
etof_ref_wf_shifted.append(wf_ref + shift_val)
etof_str_wf_shifted.append(wf_str + shift_val)
waveform_ref_source.data.update(
xs=len(etof_ref.calib_data) *
[list(range(etof_ref.internal_time_bins))],
ys=etof_ref_wf_shifted,
en=etof_ref.calib_data.index.tolist(),
)
waveform_str_source.data.update(
xs=len(etof_str.calib_data) *
[list(range(etof_str.internal_time_bins))],
ys=etof_str_wf_shifted,
en=etof_str.calib_data.index.tolist(),
)
photon_peak_ref_span.location = etof_ref.calib_t0
photon_peak_str_span.location = etof_str.calib_t0
def plot_fit(time, calib_a, calib_b):
time_fit = np.linspace(np.nanmin(time), np.nanmax(time), 100)
en_fit = (calib_a / time_fit)**2 + calib_b
return time_fit, en_fit
def update_plot(calib_results, circle, line):
(a, c), x, y = calib_results
x_fit, y_fit = plot_fit(x, a, c)
circle.data.update(x=x, y=y)
line.data.update(x=x_fit, y=y_fit)
update_plot(calib_res["0"], fit_ref_circle_source, fit_ref_line_source)
update_plot(calib_res["1"], fit_str_circle_source, fit_str_line_source)
calib_const_div.text = f"""
a_str = {etof_str.calib_a:.2f}<br>
b_str = {etof_str.calib_b:.2f}<br>
<br>
a_ref = {etof_ref.calib_a:.2f}<br>
b_ref = {etof_ref.calib_b:.2f}
"""
calibrate_button = Button(label="Calibrate eTOF",
button_type="default",
width=250)
calibrate_button.on_click(calibrate_button_callback)
# Photon peak noise threshold value text input
def phot_peak_noise_thr_spinner_callback(_attr, old_value, new_value):
if new_value > 0:
for etof in palm.etofs.values():
etof.photon_peak_noise_thr = new_value
else:
phot_peak_noise_thr_spinner.value = old_value
phot_peak_noise_thr_spinner = Spinner(title="Photon peak noise threshold:",
value=1,
step=0.1)
phot_peak_noise_thr_spinner.on_change(
"value", phot_peak_noise_thr_spinner_callback)
# Electron peak noise threshold value text input
def el_peak_noise_thr_spinner_callback(_attr, old_value, new_value):
if new_value > 0:
for etof in palm.etofs.values():
etof.electron_peak_noise_thr = new_value
else:
el_peak_noise_thr_spinner.value = old_value
el_peak_noise_thr_spinner = Spinner(title="Electron peak noise threshold:",
value=10,
step=0.1)
el_peak_noise_thr_spinner.on_change("value",
el_peak_noise_thr_spinner_callback)
# Save calibration button
def save_button_callback():
palm.save_etof_calib(path=path_textinput.value)
update_load_dropdown_menu()
save_button = Button(label="Save", button_type="default", width=250)
save_button.on_click(save_button_callback)
# Load calibration button
def load_dropdown_callback(event):
new_value = event.item
if new_value:
palm.load_etof_calib(os.path.join(path_textinput.value, new_value))
datatable_ref_source.data.update(
energy=palm.etofs["0"].calib_data.index.tolist(),
peak_pos_ref=palm.etofs["0"].calib_data["calib_tpeak"].tolist(
),
use_in_fit=palm.etofs["0"].calib_data["use_in_fit"].tolist(),
)
datatable_str_source.data.update(
energy=palm.etofs["0"].calib_data.index.tolist(),
peak_pos_str=palm.etofs["1"].calib_data["calib_tpeak"].tolist(
),
use_in_fit=palm.etofs["1"].calib_data["use_in_fit"].tolist(),
)
def update_load_dropdown_menu():
new_menu = []
calib_file_ext = ".palm_etof"
if os.path.isdir(path_textinput.value):
for entry in os.scandir(path_textinput.value):
if entry.is_file() and entry.name.endswith((calib_file_ext)):
new_menu.append(
(entry.name[:-len(calib_file_ext)], entry.name))
load_dropdown.button_type = "default"
load_dropdown.menu = sorted(new_menu, reverse=True)
else:
load_dropdown.button_type = "danger"
load_dropdown.menu = new_menu
doc.add_next_tick_callback(update_load_dropdown_menu)
doc.add_periodic_callback(update_load_dropdown_menu, 5000)
load_dropdown = Dropdown(label="Load", menu=[], width=250)
load_dropdown.on_click(load_dropdown_callback)
# eTOF fitting equation
fit_eq_div = Div(
text="""Fitting equation:<br><br><img src="/palm/static/5euwuy.gif">"""
)
# Calibration constants
calib_const_div = Div(text=f"""
a_str = {0}<br>
b_str = {0}<br>
<br>
a_ref = {0}<br>
b_ref = {0}
""")
# assemble
tab_layout = column(
row(
column(waveform_plot, fit_plot),
Spacer(width=30),
column(
Tabs(tabs=[path_tab, upload_tab]),
calibrate_button,
phot_peak_noise_thr_spinner,
el_peak_noise_thr_spinner,
row(save_button, load_dropdown),
row(datatable_ref, datatable_str),
calib_const_div,
fit_eq_div,
),
))
return Panel(child=tab_layout, title="eTOF Calibration")
title="Time (in Days)",
width=500,
margin=(10, 10, 10, 20))
start_vals = [
Sb[0] / 2.3, Eb[0], Ia_ukb[0], Ia_kb[0], Is_nhb[0], Is_hb[0], Rb[0] / 2.3,
Db[0]
]
current_source = ColumnDataSource(data=dict(sizes=start_vals))
#updating the node sizes
graph_renderer.node_renderer.data_source.add(current_source.data['sizes'],
'size')
graph_renderer.node_renderer.glyph = Circle(size='size', fill_color='color')
#when edge is hovered over, will display a description of the movement of individuals along that edge
hover_tool = HoverTool(tooltips=[("Path Movement", "@edge_names")])
plot.add_tools(hover_tool, TapTool(), BoxSelectTool(), ResetTool())
####### Bar Graph
proportion_pops = [
Sb[0] / 1000, Eb[0] / 1000, Ia_ukb[0] / 1000, Ia_kb[0] / 1000,
Is_nhb[0] / 1000, Is_hb[0] / 1000, Rb[0] / 1000, Db[0] / 1000
]
bar_source = ColumnDataSource(
data=dict(tall=proportion_pops, names=class_names, colors=Colorblind8))
bargraph = figure(x_range=class_names,
y_range=Range1d(0, 1.04),
title="Proportion of Population in Each Class",
tools=("reset, box_zoom"),
height=600,
margin=(15, 10, 10, 10))
bargraph.vbar(x='names',
class MemoryUse(DashboardComponent):
""" The memory usage across the cluster, grouped by task type """
def __init__(self, scheduler, **kwargs):
self.scheduler = scheduler
ps = [p for p in scheduler.plugins if isinstance(p, AllProgress)]
if ps:
self.plugin = ps[0]
else:
self.plugin = AllProgress(scheduler)
self.source = ColumnDataSource(data=dict(name=[],
left=[],
right=[],
center=[],
color=[],
percent=[],
MB=[],
text=[]))
self.root = Plot(id='bk-nbytes-plot',
x_range=DataRange1d(),
y_range=DataRange1d(),
toolbar_location=None,
outline_line_color=None,
**kwargs)
self.root.add_glyph(
self.source,
Quad(top=1,
bottom=0,
left='left',
right='right',
fill_color='color',
fill_alpha=1))
self.root.add_layout(LinearAxis(), 'left')
self.root.add_layout(LinearAxis(), 'below')
hover = HoverTool(point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 14px; font-weight: bold;">Name:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Percent:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@percent</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">MB:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@MB</span>
</div>
""")
self.root.add_tools(hover)
def update(self):
with log_errors():
nb = nbytes_bar(self.plugin.nbytes)
update(self.source, nb)
self.root.title.text = \
"Memory Use: %0.2f MB" % (sum(self.plugin.nbytes.values()) / 1e6)
def __init__(self, **kwargs):
names = [
'processes', 'disk-read', 'cores', 'cpu', 'disk-write', 'memory',
'last-seen', 'memory_percent', 'host'
]
self.source = ColumnDataSource({k: [] for k in names})
columns = {
name: TableColumn(field=name, title=name.replace('_percent', ' %'))
for name in names
}
cnames = [
'host', 'cores', 'processes', 'memory', 'cpu', 'memory_percent'
]
formatters = {
'cpu': NumberFormatter(format='0.0 %'),
'memory_percent': NumberFormatter(format='0.0 %'),
'memory': NumberFormatter(format='0 b'),
'latency': NumberFormatter(format='0.00000'),
'last-seen': NumberFormatter(format='0.000'),
'disk-read': NumberFormatter(format='0 b'),
'disk-write': NumberFormatter(format='0 b'),
'net-send': NumberFormatter(format='0 b'),
'net-recv': NumberFormatter(format='0 b')
}
table = DataTable(
source=self.source,
columns=[columns[n] for n in cnames],
)
for name in cnames:
if name in formatters:
table.columns[cnames.index(name)].formatter = formatters[name]
mem_plot = Plot(title=Title(text="Memory Usage (%)"),
toolbar_location=None,
x_range=Range1d(start=0, end=1),
y_range=Range1d(start=-0.1, end=0.1),
**kwargs)
mem_plot.add_glyph(
self.source,
Circle(x='memory_percent', y=0, size=10, fill_alpha=0.5))
mem_plot.add_layout(LinearAxis(), 'below')
hover = HoverTool(point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 10px; font-family: Monaco, monospace;">@host: </span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@memory_percent</span>
</div>
""")
mem_plot.add_tools(hover, BoxSelectTool())
if 'sizing_mode' in kwargs:
sizing_mode = {'sizing_mode': kwargs['sizing_mode']}
else:
sizing_mode = {}
self.root = column(mem_plot,
table,
id='bk-worker-table',
**sizing_mode)
def create(self):
manufacturers = sorted(mpg["manufacturer"].unique())
models = sorted(mpg["model"].unique())
transmissions = sorted(mpg["trans"].unique())
drives = sorted(mpg["drv"].unique())
classes = sorted(mpg["class"].unique())
manufacturer_select = Select(title="Manufacturer:", value="All", options=["All"] + manufacturers)
manufacturer_select.on_change('value', self.on_manufacturer_change)
model_select = Select(title="Model:", value="All", options=["All"] + models)
model_select.on_change('value', self.on_model_change)
transmission_select = Select(title="Transmission:", value="All", options=["All"] + transmissions)
transmission_select.on_change('value', self.on_transmission_change)
drive_select = Select(title="Drive:", value="All", options=["All"] + drives)
drive_select.on_change('value', self.on_drive_change)
class_select = Select(title="Class:", value="All", options=["All"] + classes)
class_select.on_change('value', self.on_class_change)
columns = [
TableColumn(field="manufacturer", title="Manufacturer", editor=SelectEditor(options=manufacturers), formatter=StringFormatter(font_style="bold")),
TableColumn(field="model", title="Model", editor=StringEditor(completions=models)),
TableColumn(field="displ", title="Displacement", editor=NumberEditor(step=0.1), formatter=NumberFormatter(format="0.0")),
TableColumn(field="year", title="Year", editor=IntEditor()),
TableColumn(field="cyl", title="Cylinders", editor=IntEditor()),
TableColumn(field="trans", title="Transmission", editor=SelectEditor(options=transmissions)),
TableColumn(field="drv", title="Drive", editor=SelectEditor(options=drives)),
TableColumn(field="class", title="Class", editor=SelectEditor(options=classes)),
TableColumn(field="cty", title="City MPG", editor=IntEditor()),
TableColumn(field="hwy", title="Highway MPG", editor=IntEditor()),
]
data_table = DataTable(source=self.source, columns=columns, editable=True, width=1300)
plot = Plot(title=None, x_range= DataRange1d(), y_range=DataRange1d(), plot_width=1000, plot_height=300)
# Set up x & y axis
plot.add_layout(LinearAxis(), 'below')
yaxis = LinearAxis()
plot.add_layout(yaxis, 'left')
plot.add_layout(Grid(dimension=1, ticker=yaxis.ticker))
# Add Glyphs
cty_glyph = Circle(x="index", y="cty", fill_color="#396285", size=8, fill_alpha=0.5, line_alpha=0.5)
hwy_glyph = Circle(x="index", y="hwy", fill_color="#CE603D", size=8, fill_alpha=0.5, line_alpha=0.5)
cty = plot.add_glyph(self.source, cty_glyph)
hwy = plot.add_glyph(self.source, hwy_glyph)
# Add the tools
tooltips = [
("Manufacturer", "@manufacturer"),
("Model", "@model"),
("Displacement", "@displ"),
("Year", "@year"),
("Cylinders", "@cyl"),
("Transmission", "@trans"),
("Drive", "@drv"),
("Class", "@class"),
]
cty_hover_tool = HoverTool(renderers=[cty], tooltips=tooltips + [("City MPG", "@cty")])
hwy_hover_tool = HoverTool(renderers=[hwy], tooltips=tooltips + [("Highway MPG", "@hwy")])
select_tool = BoxSelectTool(renderers=[cty, hwy], dimensions='width')
plot.add_tools(cty_hover_tool, hwy_hover_tool, select_tool)
controls = WidgetBox(manufacturer_select, model_select, transmission_select, drive_select, class_select)
top_panel = Row(controls, plot)
layout = Column(top_panel, data_table)
return layout
PanTool, WheelZoomTool)
from bokeh.resources import INLINE
x = arange(-2 * pi, 2 * pi, 0.1)
y = sin(x)
source = ColumnDataSource(data=dict(x=x, y=y))
xdr = DataRange1d()
ydr = DataRange1d()
plot = Plot(x_range=xdr, y_range=ydr, min_border=80)
circle = Circle(x="x", y="y", fill_color="red", size=5, line_color="black")
plot.add_glyph(source, circle)
plot.add_layout(LinearAxis(), 'below')
plot.add_layout(LinearAxis(), 'left')
plot.add_tools(PanTool(), WheelZoomTool())
doc = Document()
doc.add_root(plot)
if __name__ == "__main__":
filename = "glyph1.html"
with open(filename, "w") as f:
f.write(file_html(doc, INLINE, "Glyph Plot"))
print("Wrote %s" % filename)
view(filename)
line_width=2,
line_dash="dashed")
line = plot.add_glyph(source, line_glyph)
circle = Circle(x="x",
y="y2",
size=6,
line_color="red",
fill_color="orange",
fill_alpha=0.6)
circle = plot.add_glyph(source, circle)
pan = PanTool()
wheel_zoom = WheelZoomTool()
preview_save = SaveTool()
plot.add_tools(pan, wheel_zoom, preview_save)
# Add axes (Note it's important to add these before adding legends in side panels)
plot.add_layout(LinearAxis(), 'below')
plot.add_layout(LinearAxis(), 'left')
#plot.add_layout(LinearAxis(), 'right') - Due to a bug cannot have two things on the right side
from bokeh.core.enums import LegendLocation
# Add legends in names positions e.g. 'top_right', 'top_left' (see plot for all)
for location in LegendLocation:
legend = Legend(legends=[(location, [line]), ("other", [circle])],
location=location,
orientation="vertical")
plot.add_layout(legend)
def create():
det_data = {}
roi_selection = {}
upload_div = Div(text="Open .cami file:")
def upload_button_callback(_attr, _old, new):
with io.StringIO(base64.b64decode(new).decode()) as file:
h5meta_list = pyzebra.parse_h5meta(file)
file_list = h5meta_list["filelist"]
filelist.options = file_list
filelist.value = file_list[0]
upload_button = FileInput(accept=".cami")
upload_button.on_change("value", upload_button_callback)
def update_image(index=None):
if index is None:
index = index_spinner.value
current_image = det_data["data"][index]
proj_v_line_source.data.update(x=np.arange(0, IMAGE_W) + 0.5,
y=np.mean(current_image, axis=0))
proj_h_line_source.data.update(x=np.mean(current_image, axis=1),
y=np.arange(0, IMAGE_H) + 0.5)
image_source.data.update(
h=[np.zeros((1, 1))],
k=[np.zeros((1, 1))],
l=[np.zeros((1, 1))],
)
image_source.data.update(image=[current_image])
if auto_toggle.active:
im_max = int(np.max(current_image))
im_min = int(np.min(current_image))
display_min_spinner.value = im_min
display_max_spinner.value = im_max
image_glyph.color_mapper.low = im_min
image_glyph.color_mapper.high = im_max
def update_overview_plot():
h5_data = det_data["data"]
n_im, n_y, n_x = h5_data.shape
overview_x = np.mean(h5_data, axis=1)
overview_y = np.mean(h5_data, axis=2)
overview_plot_x_image_source.data.update(image=[overview_x], dw=[n_x])
overview_plot_y_image_source.data.update(image=[overview_y], dw=[n_y])
if frame_button_group.active == 0: # Frame
overview_plot_x.axis[1].axis_label = "Frame"
overview_plot_y.axis[1].axis_label = "Frame"
overview_plot_x_image_source.data.update(y=[0], dh=[n_im])
overview_plot_y_image_source.data.update(y=[0], dh=[n_im])
elif frame_button_group.active == 1: # Omega
overview_plot_x.axis[1].axis_label = "Omega"
overview_plot_y.axis[1].axis_label = "Omega"
om = det_data["rot_angle"]
om_start = om[0]
om_end = (om[-1] - om[0]) * n_im / (n_im - 1)
overview_plot_x_image_source.data.update(y=[om_start], dh=[om_end])
overview_plot_y_image_source.data.update(y=[om_start], dh=[om_end])
def filelist_callback(_attr, _old, new):
nonlocal det_data
det_data = pyzebra.read_detector_data(new)
index_spinner.value = 0
index_spinner.high = det_data["data"].shape[0] - 1
update_image(0)
update_overview_plot()
filelist = Select()
filelist.on_change("value", filelist_callback)
def index_spinner_callback(_attr, _old, new):
update_image(new)
index_spinner = Spinner(title="Image index:", value=0, low=0)
index_spinner.on_change("value", index_spinner_callback)
plot = Plot(
x_range=Range1d(0, IMAGE_W, bounds=(0, IMAGE_W)),
y_range=Range1d(0, IMAGE_H, bounds=(0, IMAGE_H)),
plot_height=IMAGE_H * 3,
plot_width=IMAGE_W * 3,
toolbar_location="left",
)
# ---- tools
plot.toolbar.logo = None
# ---- axes
plot.add_layout(LinearAxis(), place="above")
plot.add_layout(LinearAxis(major_label_orientation="vertical"),
place="right")
# ---- grid lines
plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- rgba image glyph
image_source = ColumnDataSource(
dict(
image=[np.zeros((IMAGE_H, IMAGE_W), dtype="float32")],
h=[np.zeros((1, 1))],
k=[np.zeros((1, 1))],
l=[np.zeros((1, 1))],
x=[0],
y=[0],
dw=[IMAGE_W],
dh=[IMAGE_H],
))
h_glyph = Image(image="h", x="x", y="y", dw="dw", dh="dh", global_alpha=0)
k_glyph = Image(image="k", x="x", y="y", dw="dw", dh="dh", global_alpha=0)
l_glyph = Image(image="l", x="x", y="y", dw="dw", dh="dh", global_alpha=0)
plot.add_glyph(image_source, h_glyph)
plot.add_glyph(image_source, k_glyph)
plot.add_glyph(image_source, l_glyph)
image_glyph = Image(image="image", x="x", y="y", dw="dw", dh="dh")
plot.add_glyph(image_source, image_glyph, name="image_glyph")
# ---- projections
proj_v = Plot(
x_range=plot.x_range,
y_range=DataRange1d(),
plot_height=200,
plot_width=IMAGE_W * 3,
toolbar_location=None,
)
proj_v.add_layout(LinearAxis(major_label_orientation="vertical"),
place="right")
proj_v.add_layout(LinearAxis(major_label_text_font_size="0pt"),
place="below")
proj_v.add_layout(Grid(dimension=0, ticker=BasicTicker()))
proj_v.add_layout(Grid(dimension=1, ticker=BasicTicker()))
proj_v_line_source = ColumnDataSource(dict(x=[], y=[]))
proj_v.add_glyph(proj_v_line_source,
Line(x="x", y="y", line_color="steelblue"))
proj_h = Plot(
x_range=DataRange1d(),
y_range=plot.y_range,
plot_height=IMAGE_H * 3,
plot_width=200,
toolbar_location=None,
)
proj_h.add_layout(LinearAxis(), place="above")
proj_h.add_layout(LinearAxis(major_label_text_font_size="0pt"),
place="left")
proj_h.add_layout(Grid(dimension=0, ticker=BasicTicker()))
proj_h.add_layout(Grid(dimension=1, ticker=BasicTicker()))
proj_h_line_source = ColumnDataSource(dict(x=[], y=[]))
proj_h.add_glyph(proj_h_line_source,
Line(x="x", y="y", line_color="steelblue"))
# add tools
hovertool = HoverTool(tooltips=[("intensity",
"@image"), ("h", "@h"), ("k",
"@k"), ("l",
"@l")])
box_edit_source = ColumnDataSource(dict(x=[], y=[], width=[], height=[]))
box_edit_glyph = Rect(x="x",
y="y",
width="width",
height="height",
fill_alpha=0,
line_color="red")
box_edit_renderer = plot.add_glyph(box_edit_source, box_edit_glyph)
boxedittool = BoxEditTool(renderers=[box_edit_renderer], num_objects=1)
def box_edit_callback(_attr, _old, new):
if new["x"]:
h5_data = det_data["data"]
x_val = np.arange(h5_data.shape[0])
left = int(np.floor(new["x"][0]))
right = int(np.ceil(new["x"][0] + new["width"][0]))
bottom = int(np.floor(new["y"][0]))
top = int(np.ceil(new["y"][0] + new["height"][0]))
y_val = np.sum(h5_data[:, bottom:top, left:right], axis=(1, 2))
else:
x_val = []
y_val = []
roi_avg_plot_line_source.data.update(x=x_val, y=y_val)
box_edit_source.on_change("data", box_edit_callback)
wheelzoomtool = WheelZoomTool(maintain_focus=False)
plot.add_tools(
PanTool(),
BoxZoomTool(),
wheelzoomtool,
ResetTool(),
hovertool,
boxedittool,
)
plot.toolbar.active_scroll = wheelzoomtool
# shared frame range
frame_range = DataRange1d()
det_x_range = DataRange1d()
overview_plot_x = Plot(
title=Title(text="Projections on X-axis"),
x_range=det_x_range,
y_range=frame_range,
plot_height=400,
plot_width=400,
toolbar_location="left",
)
# ---- tools
wheelzoomtool = WheelZoomTool(maintain_focus=False)
overview_plot_x.toolbar.logo = None
overview_plot_x.add_tools(
PanTool(),
BoxZoomTool(),
wheelzoomtool,
ResetTool(),
)
overview_plot_x.toolbar.active_scroll = wheelzoomtool
# ---- axes
overview_plot_x.add_layout(LinearAxis(axis_label="Coordinate X, pix"),
place="below")
overview_plot_x.add_layout(LinearAxis(axis_label="Frame",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
overview_plot_x.add_layout(Grid(dimension=0, ticker=BasicTicker()))
overview_plot_x.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- rgba image glyph
overview_plot_x_image_source = ColumnDataSource(
dict(image=[np.zeros((1, 1), dtype="float32")],
x=[0],
y=[0],
dw=[1],
dh=[1]))
overview_plot_x_image_glyph = Image(image="image",
x="x",
y="y",
dw="dw",
dh="dh")
overview_plot_x.add_glyph(overview_plot_x_image_source,
overview_plot_x_image_glyph,
name="image_glyph")
det_y_range = DataRange1d()
overview_plot_y = Plot(
title=Title(text="Projections on Y-axis"),
x_range=det_y_range,
y_range=frame_range,
plot_height=400,
plot_width=400,
toolbar_location="left",
)
# ---- tools
wheelzoomtool = WheelZoomTool(maintain_focus=False)
overview_plot_y.toolbar.logo = None
overview_plot_y.add_tools(
PanTool(),
BoxZoomTool(),
wheelzoomtool,
ResetTool(),
)
overview_plot_y.toolbar.active_scroll = wheelzoomtool
# ---- axes
overview_plot_y.add_layout(LinearAxis(axis_label="Coordinate Y, pix"),
place="below")
overview_plot_y.add_layout(LinearAxis(axis_label="Frame",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
overview_plot_y.add_layout(Grid(dimension=0, ticker=BasicTicker()))
overview_plot_y.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- rgba image glyph
overview_plot_y_image_source = ColumnDataSource(
dict(image=[np.zeros((1, 1), dtype="float32")],
x=[0],
y=[0],
dw=[1],
dh=[1]))
overview_plot_y_image_glyph = Image(image="image",
x="x",
y="y",
dw="dw",
dh="dh")
overview_plot_y.add_glyph(overview_plot_y_image_source,
overview_plot_y_image_glyph,
name="image_glyph")
def frame_button_group_callback(_active):
update_overview_plot()
frame_button_group = RadioButtonGroup(labels=["Frames", "Omega"], active=0)
frame_button_group.on_click(frame_button_group_callback)
roi_avg_plot = Plot(
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=IMAGE_H * 3,
plot_width=IMAGE_W * 3,
toolbar_location="left",
)
# ---- tools
roi_avg_plot.toolbar.logo = None
# ---- axes
roi_avg_plot.add_layout(LinearAxis(), place="below")
roi_avg_plot.add_layout(LinearAxis(major_label_orientation="vertical"),
place="left")
# ---- grid lines
roi_avg_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
roi_avg_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
roi_avg_plot_line_source = ColumnDataSource(dict(x=[], y=[]))
roi_avg_plot.add_glyph(roi_avg_plot_line_source,
Line(x="x", y="y", line_color="steelblue"))
cmap_dict = {
"gray": Greys256,
"gray_reversed": Greys256[::-1],
"plasma": Plasma256,
"cividis": Cividis256,
}
def colormap_callback(_attr, _old, new):
image_glyph.color_mapper = LinearColorMapper(palette=cmap_dict[new])
overview_plot_x_image_glyph.color_mapper = LinearColorMapper(
palette=cmap_dict[new])
overview_plot_y_image_glyph.color_mapper = LinearColorMapper(
palette=cmap_dict[new])
colormap = Select(title="Colormap:", options=list(cmap_dict.keys()))
colormap.on_change("value", colormap_callback)
colormap.value = "plasma"
radio_button_group = RadioButtonGroup(labels=["nb", "nb_bi"], active=0)
STEP = 1
# ---- colormap auto toggle button
def auto_toggle_callback(state):
if state:
display_min_spinner.disabled = True
display_max_spinner.disabled = True
else:
display_min_spinner.disabled = False
display_max_spinner.disabled = False
update_image()
auto_toggle = Toggle(label="Auto Range",
active=True,
button_type="default")
auto_toggle.on_click(auto_toggle_callback)
# ---- colormap display max value
def display_max_spinner_callback(_attr, _old_value, new_value):
display_min_spinner.high = new_value - STEP
image_glyph.color_mapper.high = new_value
display_max_spinner = Spinner(
title="Maximal Display Value:",
low=0 + STEP,
value=1,
step=STEP,
disabled=auto_toggle.active,
)
display_max_spinner.on_change("value", display_max_spinner_callback)
# ---- colormap display min value
def display_min_spinner_callback(_attr, _old_value, new_value):
display_max_spinner.low = new_value + STEP
image_glyph.color_mapper.low = new_value
display_min_spinner = Spinner(
title="Minimal Display Value:",
high=1 - STEP,
value=0,
step=STEP,
disabled=auto_toggle.active,
)
display_min_spinner.on_change("value", display_min_spinner_callback)
def hkl_button_callback():
index = index_spinner.value
setup_type = "nb_bi" if radio_button_group.active else "nb"
h, k, l = calculate_hkl(det_data, index, setup_type)
image_source.data.update(h=[h], k=[k], l=[l])
hkl_button = Button(label="Calculate hkl (slow)")
hkl_button.on_click(hkl_button_callback)
selection_list = TextAreaInput(rows=7)
def selection_button_callback():
nonlocal roi_selection
selection = [
int(np.floor(det_x_range.start)),
int(np.ceil(det_x_range.end)),
int(np.floor(det_y_range.start)),
int(np.ceil(det_y_range.end)),
int(np.floor(frame_range.start)),
int(np.ceil(frame_range.end)),
]
filename_id = filelist.value[-8:-4]
if filename_id in roi_selection:
roi_selection[f"{filename_id}"].append(selection)
else:
roi_selection[f"{filename_id}"] = [selection]
selection_list.value = str(roi_selection)
selection_button = Button(label="Add selection")
selection_button.on_click(selection_button_callback)
# Final layout
layout_image = column(
gridplot([[proj_v, None], [plot, proj_h]], merge_tools=False),
row(index_spinner))
colormap_layout = column(colormap, auto_toggle, display_max_spinner,
display_min_spinner)
hkl_layout = column(radio_button_group, hkl_button)
layout_overview = column(
gridplot(
[[overview_plot_x, overview_plot_y]],
toolbar_options=dict(logo=None),
merge_tools=True,
),
frame_button_group,
)
tab_layout = row(
column(
upload_div,
upload_button,
filelist,
layout_image,
row(colormap_layout, hkl_layout),
),
column(
roi_avg_plot,
layout_overview,
row(selection_button, selection_list),
),
)
return Panel(child=tab_layout, title="Data Viewer")
