def create_multiselect(cont):
"""
Create a multiselect box for each continent, for the "All" tab.
Args:
cont (str): Continent name.
Returns:
multi_select (:obj:`MultiSelect`): Multiselect box.
"""
multi_select = MultiSelect(title=cont, value=[], options=by_cont[cont])
multi_select.on_change("value", multi_update)
return multi_select
def layout_hook(self):
""" Hook for layout creation. """
options = [(v, v) for v in np.unique(self.context.data[self.coord])]
layout = MultiSelect(title=self.coord,
value=[options[0][0]],
options=options)
layout.size = min(len(options), self.max_elements)
layout.on_change("value", self.on_selected_coord_change)
self.coord_vals = [options[0][0]]
self.context._update_handlers()
return layout
def create():
det_data = {}
fit_params = {}
js_data = ColumnDataSource(
data=dict(content=["", ""], fname=["", ""], ext=["", ""]))
def proposal_textinput_callback(_attr, _old, new):
proposal = new.strip()
for zebra_proposals_path in pyzebra.ZEBRA_PROPOSALS_PATHS:
proposal_path = os.path.join(zebra_proposals_path, proposal)
if os.path.isdir(proposal_path):
# found it
break
else:
raise ValueError(f"Can not find data for proposal '{proposal}'.")
file_list = []
for file in os.listdir(proposal_path):
if file.endswith((".ccl", ".dat")):
file_list.append((os.path.join(proposal_path, file), file))
file_select.options = file_list
file_open_button.disabled = False
file_append_button.disabled = False
proposal_textinput = TextInput(title="Proposal number:", width=210)
proposal_textinput.on_change("value", proposal_textinput_callback)
def _init_datatable():
scan_list = [s["idx"] for s in det_data]
hkl = [f'{s["h"]} {s["k"]} {s["l"]}' for s in det_data]
export = [s.get("active", True) for s in det_data]
scan_table_source.data.update(
scan=scan_list,
hkl=hkl,
fit=[0] * len(scan_list),
export=export,
)
scan_table_source.selected.indices = []
scan_table_source.selected.indices = [0]
merge_options = [(str(i), f"{i} ({idx})")
for i, idx in enumerate(scan_list)]
merge_from_select.options = merge_options
merge_from_select.value = merge_options[0][0]
file_select = MultiSelect(title="Available .ccl/.dat files:",
width=210,
height=250)
def file_open_button_callback():
nonlocal det_data
det_data = []
for f_name in file_select.value:
with open(f_name) as file:
base, ext = os.path.splitext(f_name)
if det_data:
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data,
monitor_spinner.value)
pyzebra.merge_datasets(det_data, append_data)
else:
det_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(det_data, monitor_spinner.value)
pyzebra.merge_duplicates(det_data)
js_data.data.update(fname=[base, base])
_init_datatable()
append_upload_button.disabled = False
file_open_button = Button(label="Open New", width=100, disabled=True)
file_open_button.on_click(file_open_button_callback)
def file_append_button_callback():
for f_name in file_select.value:
with open(f_name) as file:
_, ext = os.path.splitext(f_name)
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data, monitor_spinner.value)
pyzebra.merge_datasets(det_data, append_data)
_init_datatable()
file_append_button = Button(label="Append", width=100, disabled=True)
file_append_button.on_click(file_append_button_callback)
def upload_button_callback(_attr, _old, new):
nonlocal det_data
det_data = []
for f_str, f_name in zip(new, upload_button.filename):
with io.StringIO(base64.b64decode(f_str).decode()) as file:
base, ext = os.path.splitext(f_name)
if det_data:
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data,
monitor_spinner.value)
pyzebra.merge_datasets(det_data, append_data)
else:
det_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(det_data, monitor_spinner.value)
pyzebra.merge_duplicates(det_data)
js_data.data.update(fname=[base, base])
_init_datatable()
append_upload_button.disabled = False
upload_div = Div(text="or upload new .ccl/.dat files:",
margin=(5, 5, 0, 5))
upload_button = FileInput(accept=".ccl,.dat", multiple=True, width=200)
upload_button.on_change("value", upload_button_callback)
def append_upload_button_callback(_attr, _old, new):
for f_str, f_name in zip(new, append_upload_button.filename):
with io.StringIO(base64.b64decode(f_str).decode()) as file:
_, ext = os.path.splitext(f_name)
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data, monitor_spinner.value)
pyzebra.merge_datasets(det_data, append_data)
_init_datatable()
append_upload_div = Div(text="append extra files:", margin=(5, 5, 0, 5))
append_upload_button = FileInput(accept=".ccl,.dat",
multiple=True,
width=200,
disabled=True)
append_upload_button.on_change("value", append_upload_button_callback)
def monitor_spinner_callback(_attr, old, new):
if det_data:
pyzebra.normalize_dataset(det_data, new)
_update_plot(_get_selected_scan())
monitor_spinner = Spinner(title="Monitor:",
mode="int",
value=100_000,
low=1,
width=145)
monitor_spinner.on_change("value", monitor_spinner_callback)
def _update_table():
fit_ok = [(1 if "fit" in scan else 0) for scan in det_data]
scan_table_source.data.update(fit=fit_ok)
def _update_plot(scan):
scan_motor = scan["scan_motor"]
y = scan["counts"]
x = scan[scan_motor]
plot.axis[0].axis_label = scan_motor
plot_scatter_source.data.update(x=x,
y=y,
y_upper=y + np.sqrt(y),
y_lower=y - np.sqrt(y))
fit = scan.get("fit")
if fit is not None:
x_fit = np.linspace(x[0], x[-1], 100)
plot_fit_source.data.update(x=x_fit, y=fit.eval(x=x_fit))
x_bkg = []
y_bkg = []
xs_peak = []
ys_peak = []
comps = fit.eval_components(x=x_fit)
for i, model in enumerate(fit_params):
if "linear" in model:
x_bkg = x_fit
y_bkg = comps[f"f{i}_"]
elif any(val in model
for val in ("gaussian", "voigt", "pvoigt")):
xs_peak.append(x_fit)
ys_peak.append(comps[f"f{i}_"])
plot_bkg_source.data.update(x=x_bkg, y=y_bkg)
plot_peak_source.data.update(xs=xs_peak, ys=ys_peak)
fit_output_textinput.value = fit.fit_report()
else:
plot_fit_source.data.update(x=[], y=[])
plot_bkg_source.data.update(x=[], y=[])
plot_peak_source.data.update(xs=[], ys=[])
fit_output_textinput.value = ""
# Main plot
plot = Plot(
x_range=DataRange1d(),
y_range=DataRange1d(only_visible=True),
plot_height=470,
plot_width=700,
)
plot.add_layout(LinearAxis(axis_label="Counts"), place="left")
plot.add_layout(LinearAxis(axis_label="Scan motor"), place="below")
plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
plot_scatter_source = ColumnDataSource(
dict(x=[0], y=[0], y_upper=[0], y_lower=[0]))
plot_scatter = plot.add_glyph(
plot_scatter_source, Scatter(x="x", y="y", line_color="steelblue"))
plot.add_layout(
Whisker(source=plot_scatter_source,
base="x",
upper="y_upper",
lower="y_lower"))
plot_fit_source = ColumnDataSource(dict(x=[0], y=[0]))
plot_fit = plot.add_glyph(plot_fit_source, Line(x="x", y="y"))
plot_bkg_source = ColumnDataSource(dict(x=[0], y=[0]))
plot_bkg = plot.add_glyph(
plot_bkg_source,
Line(x="x", y="y", line_color="green", line_dash="dashed"))
plot_peak_source = ColumnDataSource(dict(xs=[[0]], ys=[[0]]))
plot_peak = plot.add_glyph(
plot_peak_source,
MultiLine(xs="xs", ys="ys", line_color="red", line_dash="dashed"))
fit_from_span = Span(location=None, dimension="height", line_dash="dashed")
plot.add_layout(fit_from_span)
fit_to_span = Span(location=None, dimension="height", line_dash="dashed")
plot.add_layout(fit_to_span)
plot.add_layout(
Legend(
items=[
("data", [plot_scatter]),
("best fit", [plot_fit]),
("peak", [plot_peak]),
("linear", [plot_bkg]),
],
location="top_left",
click_policy="hide",
))
plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
plot.toolbar.logo = None
# Scan select
def scan_table_select_callback(_attr, old, new):
if not new:
# skip empty selections
return
# Avoid selection of multiple indicies (via Shift+Click or Ctrl+Click)
if len(new) > 1:
# drop selection to the previous one
scan_table_source.selected.indices = old
return
if len(old) > 1:
# skip unnecessary update caused by selection drop
return
_update_plot(det_data[new[0]])
def scan_table_source_callback(_attr, _old, _new):
_update_preview()
scan_table_source = ColumnDataSource(
dict(scan=[], hkl=[], fit=[], export=[]))
scan_table_source.on_change("data", scan_table_source_callback)
scan_table = DataTable(
source=scan_table_source,
columns=[
TableColumn(field="scan", title="Scan", width=50),
TableColumn(field="hkl", title="hkl", width=100),
TableColumn(field="fit", title="Fit", width=50),
TableColumn(field="export",
title="Export",
editor=CheckboxEditor(),
width=50),
],
width=310, # +60 because of the index column
height=350,
autosize_mode="none",
editable=True,
)
scan_table_source.selected.on_change("indices", scan_table_select_callback)
def _get_selected_scan():
return det_data[scan_table_source.selected.indices[0]]
merge_from_select = Select(title="scan:", width=145)
def merge_button_callback():
scan_into = _get_selected_scan()
scan_from = det_data[int(merge_from_select.value)]
if scan_into is scan_from:
print("WARNING: Selected scans for merging are identical")
return
pyzebra.merge_scans(scan_into, scan_from)
_update_plot(_get_selected_scan())
merge_button = Button(label="Merge into current", width=145)
merge_button.on_click(merge_button_callback)
def restore_button_callback():
pyzebra.restore_scan(_get_selected_scan())
_update_plot(_get_selected_scan())
restore_button = Button(label="Restore scan", width=145)
restore_button.on_click(restore_button_callback)
def fit_from_spinner_callback(_attr, _old, new):
fit_from_span.location = new
fit_from_spinner = Spinner(title="Fit from:", width=145)
fit_from_spinner.on_change("value", fit_from_spinner_callback)
def fit_to_spinner_callback(_attr, _old, new):
fit_to_span.location = new
fit_to_spinner = Spinner(title="to:", width=145)
fit_to_spinner.on_change("value", fit_to_spinner_callback)
def fitparams_add_dropdown_callback(click):
# bokeh requires (str, str) for MultiSelect options
new_tag = f"{click.item}-{fitparams_select.tags[0]}"
fitparams_select.options.append((new_tag, click.item))
fit_params[new_tag] = fitparams_factory(click.item)
fitparams_select.tags[0] += 1
fitparams_add_dropdown = Dropdown(
label="Add fit function",
menu=[
("Linear", "linear"),
("Gaussian", "gaussian"),
("Voigt", "voigt"),
("Pseudo Voigt", "pvoigt"),
# ("Pseudo Voigt1", "pseudovoigt1"),
],
width=145,
)
fitparams_add_dropdown.on_click(fitparams_add_dropdown_callback)
def fitparams_select_callback(_attr, old, new):
# Avoid selection of multiple indicies (via Shift+Click or Ctrl+Click)
if len(new) > 1:
# drop selection to the previous one
fitparams_select.value = old
return
if len(old) > 1:
# skip unnecessary update caused by selection drop
return
if new:
fitparams_table_source.data.update(fit_params[new[0]])
else:
fitparams_table_source.data.update(
dict(param=[], value=[], vary=[], min=[], max=[]))
fitparams_select = MultiSelect(options=[], height=120, width=145)
fitparams_select.tags = [0]
fitparams_select.on_change("value", fitparams_select_callback)
def fitparams_remove_button_callback():
if fitparams_select.value:
sel_tag = fitparams_select.value[0]
del fit_params[sel_tag]
for elem in fitparams_select.options:
if elem[0] == sel_tag:
fitparams_select.options.remove(elem)
break
fitparams_select.value = []
fitparams_remove_button = Button(label="Remove fit function", width=145)
fitparams_remove_button.on_click(fitparams_remove_button_callback)
def fitparams_factory(function):
if function == "linear":
params = ["slope", "intercept"]
elif function == "gaussian":
params = ["amplitude", "center", "sigma"]
elif function == "voigt":
params = ["amplitude", "center", "sigma", "gamma"]
elif function == "pvoigt":
params = ["amplitude", "center", "sigma", "fraction"]
elif function == "pseudovoigt1":
params = ["amplitude", "center", "g_sigma", "l_sigma", "fraction"]
else:
raise ValueError("Unknown fit function")
n = len(params)
fitparams = dict(
param=params,
value=[None] * n,
vary=[True] * n,
min=[None] * n,
max=[None] * n,
)
if function == "linear":
fitparams["value"] = [0, 1]
fitparams["vary"] = [False, True]
fitparams["min"] = [None, 0]
elif function == "gaussian":
fitparams["min"] = [0, None, None]
return fitparams
fitparams_table_source = ColumnDataSource(
dict(param=[], value=[], vary=[], min=[], max=[]))
fitparams_table = DataTable(
source=fitparams_table_source,
columns=[
TableColumn(field="param", title="Parameter"),
TableColumn(field="value", title="Value", editor=NumberEditor()),
TableColumn(field="vary", title="Vary", editor=CheckboxEditor()),
TableColumn(field="min", title="Min", editor=NumberEditor()),
TableColumn(field="max", title="Max", editor=NumberEditor()),
],
height=200,
width=350,
index_position=None,
editable=True,
auto_edit=True,
)
# start with `background` and `gauss` fit functions added
fitparams_add_dropdown_callback(types.SimpleNamespace(item="linear"))
fitparams_add_dropdown_callback(types.SimpleNamespace(item="gaussian"))
fitparams_select.value = ["gaussian-1"] # add selection to gauss
fit_output_textinput = TextAreaInput(title="Fit results:",
width=750,
height=200)
def proc_all_button_callback():
for scan, export in zip(det_data, scan_table_source.data["export"]):
if export:
pyzebra.fit_scan(scan,
fit_params,
fit_from=fit_from_spinner.value,
fit_to=fit_to_spinner.value)
pyzebra.get_area(
scan,
area_method=AREA_METHODS[area_method_radiobutton.active],
lorentz=lorentz_checkbox.active,
)
_update_plot(_get_selected_scan())
_update_table()
proc_all_button = Button(label="Process All",
button_type="primary",
width=145)
proc_all_button.on_click(proc_all_button_callback)
def proc_button_callback():
scan = _get_selected_scan()
pyzebra.fit_scan(scan,
fit_params,
fit_from=fit_from_spinner.value,
fit_to=fit_to_spinner.value)
pyzebra.get_area(
scan,
area_method=AREA_METHODS[area_method_radiobutton.active],
lorentz=lorentz_checkbox.active,
)
_update_plot(scan)
_update_table()
proc_button = Button(label="Process Current", width=145)
proc_button.on_click(proc_button_callback)
area_method_div = Div(text="Intensity:", margin=(5, 5, 0, 5))
area_method_radiobutton = RadioGroup(labels=["Function", "Area"],
active=0,
width=145)
lorentz_checkbox = CheckboxGroup(labels=["Lorentz Correction"],
width=145,
margin=(13, 5, 5, 5))
export_preview_textinput = TextAreaInput(title="Export file preview:",
width=500,
height=400)
def _update_preview():
with tempfile.TemporaryDirectory() as temp_dir:
temp_file = temp_dir + "/temp"
export_data = []
for s, export in zip(det_data, scan_table_source.data["export"]):
if export:
export_data.append(s)
pyzebra.export_1D(
export_data,
temp_file,
export_target_select.value,
hkl_precision=int(hkl_precision_select.value),
)
exported_content = ""
file_content = []
for ext in EXPORT_TARGETS[export_target_select.value]:
fname = temp_file + ext
if os.path.isfile(fname):
with open(fname) as f:
content = f.read()
exported_content += f"{ext} file:\n" + content
else:
content = ""
file_content.append(content)
js_data.data.update(content=file_content)
export_preview_textinput.value = exported_content
def export_target_select_callback(_attr, _old, new):
js_data.data.update(ext=EXPORT_TARGETS[new])
_update_preview()
export_target_select = Select(title="Export target:",
options=list(EXPORT_TARGETS.keys()),
value="fullprof",
width=80)
export_target_select.on_change("value", export_target_select_callback)
js_data.data.update(ext=EXPORT_TARGETS[export_target_select.value])
def hkl_precision_select_callback(_attr, _old, _new):
_update_preview()
hkl_precision_select = Select(title="hkl precision:",
options=["2", "3", "4"],
value="2",
width=80)
hkl_precision_select.on_change("value", hkl_precision_select_callback)
save_button = Button(label="Download File(s)",
button_type="success",
width=200)
save_button.js_on_click(
CustomJS(args={"js_data": js_data}, code=javaScript))
fitpeak_controls = row(
column(fitparams_add_dropdown, fitparams_select,
fitparams_remove_button),
fitparams_table,
Spacer(width=20),
column(fit_from_spinner, lorentz_checkbox, area_method_div,
area_method_radiobutton),
column(fit_to_spinner, proc_button, proc_all_button),
)
scan_layout = column(
scan_table,
row(monitor_spinner, column(Spacer(height=19), restore_button)),
row(column(Spacer(height=19), merge_button), merge_from_select),
)
import_layout = column(
proposal_textinput,
file_select,
row(file_open_button, file_append_button),
upload_div,
upload_button,
append_upload_div,
append_upload_button,
)
export_layout = column(
export_preview_textinput,
row(export_target_select, hkl_precision_select,
column(Spacer(height=19), row(save_button))),
)
tab_layout = column(
row(import_layout, scan_layout, plot, Spacer(width=30), export_layout),
row(fitpeak_controls, fit_output_textinput),
)
return Panel(child=tab_layout, title="ccl integrate")
#players = {
# 'cristiano-ronaldo': {
# 'position': 'Forward',
# 'title': 'Cristiano Ronaldo',
# },
# 'alvaro-morata': {
# 'position': 'forward',
# 'title': 'Alvaro Morata',
# }
#}
#player_select = Select(value=player, title='Player', options=sorted(players.keys()))
player_select = MultiSelect(value=player,
title='Player',
options=sorted(players.keys()),
size=30)
#distribution_select = Select(value=distribution, title='Distribution', options=['Discrete', 'Smoothed'])
player_select.on_change('value', update_plot)
print("PlayerSelect: " + str(player_select.value))
#source=d1.data
controls = column(player_select)
curdoc().add_root(row(p, controls))
curdoc().title = "Player Analysis"
# Add a periodic callback to be run every 500 milliseconds
curdoc().add_periodic_callback(update_data, 6000)
def __init__(self):
### Methods
self.args = Settings()
self.index = None
self.data_getter = None
self.filter = None
self._data = None
self._model_type = None
self._model_dir = self.args.models_path + 'unique_object/'
self.controls = {}
self.scene_plotter = ScenePlotter(self._set_head_selection)
### initialization of the interface
## Model type selector
def update_select_net():
if self._model_dir is not None:
file_list = [fn for fn in os.listdir(self._model_dir) if os.path.isfile(os.path.join(self._model_dir, fn))]
file_list.sort(key=lambda fn: os.stat(os.path.join(self._model_dir, fn)).st_mtime, reverse=True)
file_list = [os.path.splitext(fn)[0] for fn in file_list]
self.controls['net'].options = file_list
# print(self._model_dir)
# print('file_list')
# print(file_list)
if len(file_list) > 0:
self.controls['net'].value = file_list[0]
else:
self.controls['net'].value = None
def update_model_type():
if self.controls['multi_mono_object'].active == 0:
self._model_type = 'mono'
self._model_dir = self.args.models_path + 'unique_object/'
elif self.controls['multi_mono_object'].active == 1:
self._model_type = 'multi_obj'
self._model_dir = self.args.models_path + 'multi_objects/'
elif self.controls['multi_mono_object'].active == 2:
self._model_type = 'multi_pred'
self._model_dir = self.args.models_path + 'multi_pred/'
model_types = ['CV', 'CA', 'Bicycle', 'CV_LSTM', 'CA_LSTM', 'Bicycle_LSTM', 'nn_attention']
existing_types = [type for type in model_types if os.path.isdir(self._model_dir + type)]
self.controls['model_sub_type'].options = existing_types
print('existing types')
print(existing_types)
if len(existing_types) > 0 and not self.controls['model_sub_type'].value in existing_types:
self.controls['model_sub_type'].value = existing_types[0]
return
set_model_sub_type()
update_select_net()
def set_model_sub_type():
if self.controls['model_sub_type'].value is not None:
self._model_dir = self._model_dir + self.controls['model_sub_type'].value + '/'
self.args.model_type = self.controls['model_sub_type'].value
else:
self._model_dir = None
def update_multi_mono_object(attr, old, new):
update_model_type()
print(self._model_dir)
self._set_data_getter()
print('___')
multi_mono_object = RadioButtonGroup(labels=["Mono-object", "Multi-objects", "Multi-pred"], active=1)
self.controls['multi_mono_object'] = multi_mono_object
multi_mono_object.on_change('active', update_multi_mono_object)
## Model sub type selector
model_types = ['CV', 'CA', 'Bicycle', 'CV_LSTM', 'CA_LSTM', 'Bicycle_LSTM', 'nn_attention']
model_sub_type = Select(title='Select model type:', value=model_types[3], options=model_types)
self.controls['model_sub_type'] = model_sub_type
model_sub_type.on_change('value', lambda att, old, new: update_model_type())
## Model selector
select = Select(title="Select parameter file:", value=None, options=[])
self.controls['net'] = select
select.on_change('value', lambda att, old, new: self._set_data_getter())
## Select dataset to use
dataset_list = ['Argoverse', 'Fusion', 'NGSIM']
select = Select(title='Dataset:', value=dataset_list[0], options=dataset_list)
self.controls['dataset'] = select
select.on_change('value', lambda att, old, new: self._set_data_getter(change_index=True))
## Set what to draw
checkbox_group = CheckboxGroup(
labels=['Draw lanes', 'Draw history', 'Draw true future', 'Draw forecast', 'Draw forecast covariance'],
active=[0, 1, 2, 3, 4])
self.controls['check_box'] = checkbox_group
checkbox_group.on_change('active',
lambda att, old, new: (self._update_cov(), self._update_lanes(), self._update_path()))
## Set the number of pred
n_pred = Slider(start=1, end=6, step=1, value=1, title='Number of prediction hypothesis')
self.controls['n_pred'] = n_pred
n_pred.on_change('value', lambda att, old, new: (self._update_cov(), self._update_path()))
## Sequence ID input
text_input = TextInput(title="Sequence ID to plot:", value="Random")
self.controls['sequence_id'] = text_input
## Head selection input
multi_select_head = MultiSelect(title='Attention head multiple selection:',
value=[], options=[])
self.controls['Head_selection'] = multi_select_head
multi_select_head.on_change('value', self.scene_plotter.set_active_heads)
## Refresh all sample
button = Button(label="Refresh", button_type="success")
self.controls['refresh'] = button
button.on_click(
lambda event: (self._set_index(), self._set_data()))
# button.js_on_click(CustomJS(args=dict(p=self.image), code="""p.reset.emit()"""))
update_multi_mono_object(None, None, None)
## Set the interface layout
inputs = column(*(self.controls.values()), width=320, height=1000)
inputs.sizing_mode = "fixed"
lay = layout([[inputs, self.scene_plotter.get_image()]], sizing_mode="scale_both")
curdoc().add_root(lay)
self.scene_plotter._tap_on_veh('selected', [], [0])
highlight_alpha = np.ones((25, )) * 0.25
highlight_alpha[word_path] = 1
dice_src.data["fill_alpha"] = highlight_alpha.reshape(5, 5)
# reset highlights
else:
dice_src.data["fill_alpha"] = np.ones((5, 5))
# Set up callbacks
shuffle_button.on_click(shuffle)
start_button.on_click(start_game)
stop_button.on_click(stop_timer)
show_words_button.on_click(show_word_list)
show_words_options.on_change("active", sort_word_list)
word_select.on_change("value", show_word)
# Set up layouts and add to document
inputs = column(
special_toggle,
angle_toggle,
shuffle_button,
time_slider,
start_button,
stop_button,
timer,
p,
)
rhs = column(show_words_button,
show_words_options,
word_select,
def create_world_cases_time_series_tab():
## Data Sources
source_df, source_CDS = get_country_cases_vs_time()
## Line Plots
line_figure = figure(
x_axis_type='datetime',
y_axis_type='log',
title='World Confirmed Cases by Region',
x_axis_label='Date',
y_axis_label='Number of Confirmed Cases (Logarithmic Scale)',
active_scroll='wheel_zoom')
starting_regions = ['China', 'US', 'Italy']
excluded_columns_set = {'index', 'date'}
doubling_lines_props = {
'alpha': 0.6,
'muted_alpha': 0.2,
'line_width': 3,
'source': source_CDS,
'x': 'date',
'visible': True
}
for number, text, color in zip([4, 7, 14], ['four', 'seven', 'fourteen'],
gray(6)[2:5]):
column_name = f'{text}_day_doubling'
excluded_columns_set.add(column_name)
source_CDS.data[column_name] = 2**(
np.arange(len(source_CDS.data['index'])) / number)
line_figure.line(y=column_name,
legend_label=f'{number}-day Doubling Time',
line_color=color,
name=column_name,
**doubling_lines_props)
line_params = {
'x': 'date',
'source': source_CDS,
'line_width': 4,
'alpha': 0.6
}
lines = {
key: line_figure.line(y=key, name=key, line_color=color, **line_params)
for key, color in zip(starting_regions, viridis(len(starting_regions)))
}
line_figure.legend.location = 'top_left'
line_figure.legend.click_policy = 'hide'
hover_tool = HoverTool(
tooltips=[('Date', '@date{%F}'), ('Region', '$name'),
('Number of Cases', '@$name{0,0}')],
formatters={
'@date': 'datetime',
},
renderers=[*line_figure.renderers],
# mode='vline'
)
line_figure.add_tools(hover_tool)
## Region Selector
labels = [
key for key in source_CDS.data.keys()
if key not in excluded_columns_set
]
def region_select_callback(attr, old, new):
new_lines = set(new) - set(old)
old_lines = set(old) - set(new)
for key in old_lines:
lines[key].visible = False
for key in new_lines:
if key in lines.keys():
lines[key].visible = True
else:
lines[key] = line_figure.line(
y=key,
name=key,
line_color=np.random.choice(Viridis256),
**line_params)
hover_tool.renderers = [*hover_tool.renderers, lines[key]]
region_select = MultiSelect(title='Select Regions to Show',
value=starting_regions,
options=labels,
sizing_mode='stretch_height')
region_select.on_change('value', region_select_callback)
## Create Layout
child = row([
column([line_figure]),
column([region_select]),
])
return Panel(child=child, title='World Cases Time Series')
def do_layout(self):
"""
generates the overall layout by creating all the widgets, buttons etc and arranges
them in rows and columns
:return: None
"""
self.source = self.generate_source()
tab_plot = self.generate_plot(self.source)
multi_select = MultiSelect(title="Option (Multiselect Ctrl+Click):",
value=self.active_country_list,
options=countries,
height=500)
multi_select.on_change('value', self.update_data)
tab_plot.on_change('active', self.update_tab)
radio_button_group_per_capita = RadioButtonGroup(
labels=["Total Cases", "Cases per Million"],
active=0 if not self.active_per_capita else 1)
radio_button_group_per_capita.on_click(self.update_capita)
radio_button_group_scale = RadioButtonGroup(
labels=[Scale.log.name.title(),
Scale.linear.name.title()],
active=self.active_y_axis_type.value)
radio_button_group_scale.on_click(self.update_scale_button)
radio_button_group_df = RadioButtonGroup(labels=[
Prefix.confirmed.name.title(),
Prefix.deaths.name.title(),
Prefix.recovered.name.title(),
],
active=int(
self.active_case_type))
radio_button_group_df.on_click(self.update_data_frame)
refresh_button = Button(label="Refresh Data",
button_type="default",
width=150)
refresh_button.on_click(load_data_frames)
export_button = Button(label="Export Url",
button_type="default",
width=150)
export_button.on_click(self.export_url)
slider = Slider(start=1,
end=30,
value=self.active_window_size,
step=1,
title="Window Size for rolling average")
slider.on_change('value', self.update_window_size)
radio_button_average = RadioButtonGroup(
labels=[Average.mean.name.title(),
Average.median.name.title()],
active=self.active_average)
radio_button_average.on_click(self.update_average_button)
plot_variables = [
self.active_plot_raw, self.active_plot_average,
self.active_plot_trend
]
plots_button_group = CheckboxButtonGroup(
labels=["Raw", "Averaged", "Trend"],
active=[i for i, x in enumerate(plot_variables) if x])
plots_button_group.on_click(self.update_shown_plots)
world_map = self.create_world_map()
link_div = Div(
name="URL",
text=
fr'Link <a target="_blank" href="{self.url}">Link to this Plot</a>.',
width=300,
height=10,
align='center')
footer = Div(
text=
"""Covid-19 Dashboard created by Andreas Weichslgartner in April 2020 with python, bokeh, pandas,
numpy, pyproj, and colorcet. Source Code can be found at
<a href="https://github.com/weichslgartner/covid_dashboard/">Github</a>.""",
width=1600,
height=10,
align='center')
self.generate_table_cumulative()
columns = [
TableColumn(field="name", title="Country"),
TableColumn(field="number_rolling",
title="daily avg",
formatter=NumberFormatter(format="0.")),
TableColumn(field="number_daily",
title="daily raw",
formatter=NumberFormatter(format="0."))
]
top_top_14_new_header = Div(text="Highest confirmed (daily)",
align='center')
top_top_14_new = DataTable(source=self.top_new_source,
name="Highest confirmed(daily)",
columns=columns,
width=300,
height=380)
self.generate_table_new()
columns = [
TableColumn(field="name", title="Country"),
TableColumn(field="number",
title="confirmed(cumulative)",
formatter=NumberFormatter(format="0."))
]
top_top_14_cum_header = Div(text="Highest confirmed (cumulative)",
align='center')
top_top_14_cum = DataTable(source=self.top_total_source,
name="Highest confirmed(cumulative)",
columns=columns,
width=300,
height=380)
self.layout = layout([
row(
column(tab_plot, world_map),
column(top_top_14_new_header, top_top_14_new,
top_top_14_cum_header, top_top_14_cum),
column(link_div, row(refresh_button, export_button),
radio_button_group_df, radio_button_group_per_capita,
plots_button_group, radio_button_group_scale, slider,
radio_button_average, multi_select),
),
row(footer)
])
curdoc().add_root(self.layout)
curdoc().title = "Bokeh Covid-19 Dashboard"
def selected_from_plot(attr, old, new):
src_table.selected.indices = new
def selected_from_table(attr, old, new):
btl_sal.data_source.selected.indices = new
# set up change callbacks
parameter.on_change("value", lambda attr, old, new: update_selectors())
station.on_change("value", lambda attr, old, new: update_selectors())
flag_list.on_change("value", lambda attr, old, new: update_selectors())
flag_button.on_click(apply_flag)
comment_button.on_click(apply_comment)
save_button.on_click(save_data)
src_table.on_change("data", lambda attr, old, new: edit_flag())
src_table.selected.on_change("indices", selected_from_table)
btl_sal.data_source.selected.on_change("indices", selected_from_plot)
# build data tables
columns = []
fields = ["SSSCC", "SAMPNO", "CTDPRS", "CTDSAL", "SALNTY", "diff", "flag", "Comments"]
titles = ["SSSCC", "Bottle", "CTDPRS", "CTDSAL", "SALNTY", "Residual", "Flag", "Comments"]
widths = [40, 20, 75, 65, 65, 65, 15, 135]
for (field, title, width) in zip(fields, titles, widths):
if field == "flag":
def create():
det_data = []
fit_params = {}
js_data = ColumnDataSource(data=dict(content=["", ""], fname=["", ""]))
def proposal_textinput_callback(_attr, _old, new):
proposal = new.strip()
for zebra_proposals_path in pyzebra.ZEBRA_PROPOSALS_PATHS:
proposal_path = os.path.join(zebra_proposals_path, proposal)
if os.path.isdir(proposal_path):
# found it
break
else:
raise ValueError(f"Can not find data for proposal '{proposal}'.")
file_list = []
for file in os.listdir(proposal_path):
if file.endswith((".ccl", ".dat")):
file_list.append((os.path.join(proposal_path, file), file))
file_select.options = file_list
file_open_button.disabled = False
file_append_button.disabled = False
proposal_textinput = TextInput(title="Proposal number:", width=210)
proposal_textinput.on_change("value", proposal_textinput_callback)
def _init_datatable():
scan_list = [s["idx"] for s in det_data]
file_list = []
for scan in det_data:
file_list.append(os.path.basename(scan["original_filename"]))
scan_table_source.data.update(
file=file_list,
scan=scan_list,
param=[None] * len(scan_list),
fit=[0] * len(scan_list),
export=[True] * len(scan_list),
)
scan_table_source.selected.indices = []
scan_table_source.selected.indices = [0]
scan_motor_select.options = det_data[0]["scan_motors"]
scan_motor_select.value = det_data[0]["scan_motor"]
param_select.value = "user defined"
file_select = MultiSelect(title="Available .ccl/.dat files:",
width=210,
height=250)
def file_open_button_callback():
nonlocal det_data
det_data = []
for f_name in file_select.value:
with open(f_name) as file:
base, ext = os.path.splitext(f_name)
if det_data:
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data,
monitor_spinner.value)
det_data.extend(append_data)
else:
det_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(det_data, monitor_spinner.value)
js_data.data.update(
fname=[base + ".comm", base + ".incomm"])
_init_datatable()
append_upload_button.disabled = False
file_open_button = Button(label="Open New", width=100, disabled=True)
file_open_button.on_click(file_open_button_callback)
def file_append_button_callback():
for f_name in file_select.value:
with open(f_name) as file:
_, ext = os.path.splitext(f_name)
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data, monitor_spinner.value)
det_data.extend(append_data)
_init_datatable()
file_append_button = Button(label="Append", width=100, disabled=True)
file_append_button.on_click(file_append_button_callback)
def upload_button_callback(_attr, _old, new):
nonlocal det_data
det_data = []
for f_str, f_name in zip(new, upload_button.filename):
with io.StringIO(base64.b64decode(f_str).decode()) as file:
base, ext = os.path.splitext(f_name)
if det_data:
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data,
monitor_spinner.value)
det_data.extend(append_data)
else:
det_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(det_data, monitor_spinner.value)
js_data.data.update(
fname=[base + ".comm", base + ".incomm"])
_init_datatable()
append_upload_button.disabled = False
upload_div = Div(text="or upload new .ccl/.dat files:",
margin=(5, 5, 0, 5))
upload_button = FileInput(accept=".ccl,.dat", multiple=True, width=200)
upload_button.on_change("value", upload_button_callback)
def append_upload_button_callback(_attr, _old, new):
for f_str, f_name in zip(new, append_upload_button.filename):
with io.StringIO(base64.b64decode(f_str).decode()) as file:
_, ext = os.path.splitext(f_name)
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data, monitor_spinner.value)
det_data.extend(append_data)
_init_datatable()
append_upload_div = Div(text="append extra files:", margin=(5, 5, 0, 5))
append_upload_button = FileInput(accept=".ccl,.dat",
multiple=True,
width=200,
disabled=True)
append_upload_button.on_change("value", append_upload_button_callback)
def monitor_spinner_callback(_attr, _old, new):
if det_data:
pyzebra.normalize_dataset(det_data, new)
_update_plot()
monitor_spinner = Spinner(title="Monitor:",
mode="int",
value=100_000,
low=1,
width=145)
monitor_spinner.on_change("value", monitor_spinner_callback)
def scan_motor_select_callback(_attr, _old, new):
if det_data:
for scan in det_data:
scan["scan_motor"] = new
_update_plot()
scan_motor_select = Select(title="Scan motor:", options=[], width=145)
scan_motor_select.on_change("value", scan_motor_select_callback)
def _update_table():
fit_ok = [(1 if "fit" in scan else 0) for scan in det_data]
scan_table_source.data.update(fit=fit_ok)
def _update_plot():
_update_single_scan_plot(_get_selected_scan())
_update_overview()
def _update_single_scan_plot(scan):
scan_motor = scan["scan_motor"]
y = scan["counts"]
x = scan[scan_motor]
plot.axis[0].axis_label = scan_motor
plot_scatter_source.data.update(x=x,
y=y,
y_upper=y + np.sqrt(y),
y_lower=y - np.sqrt(y))
fit = scan.get("fit")
if fit is not None:
x_fit = np.linspace(x[0], x[-1], 100)
plot_fit_source.data.update(x=x_fit, y=fit.eval(x=x_fit))
x_bkg = []
y_bkg = []
xs_peak = []
ys_peak = []
comps = fit.eval_components(x=x_fit)
for i, model in enumerate(fit_params):
if "linear" in model:
x_bkg = x_fit
y_bkg = comps[f"f{i}_"]
elif any(val in model
for val in ("gaussian", "voigt", "pvoigt")):
xs_peak.append(x_fit)
ys_peak.append(comps[f"f{i}_"])
plot_bkg_source.data.update(x=x_bkg, y=y_bkg)
plot_peak_source.data.update(xs=xs_peak, ys=ys_peak)
fit_output_textinput.value = fit.fit_report()
else:
plot_fit_source.data.update(x=[], y=[])
plot_bkg_source.data.update(x=[], y=[])
plot_peak_source.data.update(xs=[], ys=[])
fit_output_textinput.value = ""
def _update_overview():
xs = []
ys = []
param = []
x = []
y = []
par = []
for s, p in enumerate(scan_table_source.data["param"]):
if p is not None:
scan = det_data[s]
scan_motor = scan["scan_motor"]
xs.append(scan[scan_motor])
x.extend(scan[scan_motor])
ys.append(scan["counts"])
y.extend([float(p)] * len(scan[scan_motor]))
param.append(float(p))
par.extend(scan["counts"])
if det_data:
scan_motor = det_data[0]["scan_motor"]
ov_plot.axis[0].axis_label = scan_motor
ov_param_plot.axis[0].axis_label = scan_motor
ov_plot_mline_source.data.update(xs=xs,
ys=ys,
param=param,
color=color_palette(len(xs)))
if y:
mapper["transform"].low = np.min([np.min(y) for y in ys])
mapper["transform"].high = np.max([np.max(y) for y in ys])
ov_param_plot_scatter_source.data.update(x=x, y=y, param=par)
if y:
interp_f = interpolate.interp2d(x, y, par)
x1, x2 = min(x), max(x)
y1, y2 = min(y), max(y)
image = interp_f(
np.linspace(x1, x2, ov_param_plot.inner_width // 10),
np.linspace(y1, y2, ov_param_plot.inner_height // 10),
assume_sorted=True,
)
ov_param_plot_image_source.data.update(image=[image],
x=[x1],
y=[y1],
dw=[x2 - x1],
dh=[y2 - y1])
else:
ov_param_plot_image_source.data.update(image=[],
x=[],
y=[],
dw=[],
dh=[])
def _update_param_plot():
x = []
y = []
fit_param = fit_param_select.value
for s, p in zip(det_data, scan_table_source.data["param"]):
if "fit" in s and fit_param:
x.append(p)
y.append(s["fit"].values[fit_param])
param_plot_scatter_source.data.update(x=x, y=y)
# Main plot
plot = Plot(
x_range=DataRange1d(),
y_range=DataRange1d(only_visible=True),
plot_height=450,
plot_width=700,
)
plot.add_layout(LinearAxis(axis_label="Counts"), place="left")
plot.add_layout(LinearAxis(axis_label="Scan motor"), place="below")
plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
plot_scatter_source = ColumnDataSource(
dict(x=[0], y=[0], y_upper=[0], y_lower=[0]))
plot_scatter = plot.add_glyph(
plot_scatter_source, Scatter(x="x", y="y", line_color="steelblue"))
plot.add_layout(
Whisker(source=plot_scatter_source,
base="x",
upper="y_upper",
lower="y_lower"))
plot_fit_source = ColumnDataSource(dict(x=[0], y=[0]))
plot_fit = plot.add_glyph(plot_fit_source, Line(x="x", y="y"))
plot_bkg_source = ColumnDataSource(dict(x=[0], y=[0]))
plot_bkg = plot.add_glyph(
plot_bkg_source,
Line(x="x", y="y", line_color="green", line_dash="dashed"))
plot_peak_source = ColumnDataSource(dict(xs=[[0]], ys=[[0]]))
plot_peak = plot.add_glyph(
plot_peak_source,
MultiLine(xs="xs", ys="ys", line_color="red", line_dash="dashed"))
fit_from_span = Span(location=None, dimension="height", line_dash="dashed")
plot.add_layout(fit_from_span)
fit_to_span = Span(location=None, dimension="height", line_dash="dashed")
plot.add_layout(fit_to_span)
plot.add_layout(
Legend(
items=[
("data", [plot_scatter]),
("best fit", [plot_fit]),
("peak", [plot_peak]),
("linear", [plot_bkg]),
],
location="top_left",
click_policy="hide",
))
plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
plot.toolbar.logo = None
# Overview multilines plot
ov_plot = Plot(x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=450,
plot_width=700)
ov_plot.add_layout(LinearAxis(axis_label="Counts"), place="left")
ov_plot.add_layout(LinearAxis(axis_label="Scan motor"), place="below")
ov_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
ov_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
ov_plot_mline_source = ColumnDataSource(
dict(xs=[], ys=[], param=[], color=[]))
ov_plot.add_glyph(ov_plot_mline_source,
MultiLine(xs="xs", ys="ys", line_color="color"))
hover_tool = HoverTool(tooltips=[("param", "@param")])
ov_plot.add_tools(PanTool(), WheelZoomTool(), hover_tool, ResetTool())
ov_plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
ov_plot.toolbar.logo = None
# Overview perams plot
ov_param_plot = Plot(x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=450,
plot_width=700)
ov_param_plot.add_layout(LinearAxis(axis_label="Param"), place="left")
ov_param_plot.add_layout(LinearAxis(axis_label="Scan motor"),
place="below")
ov_param_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
ov_param_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
ov_param_plot_image_source = ColumnDataSource(
dict(image=[], x=[], y=[], dw=[], dh=[]))
ov_param_plot.add_glyph(
ov_param_plot_image_source,
Image(image="image", x="x", y="y", dw="dw", dh="dh"))
ov_param_plot_scatter_source = ColumnDataSource(dict(x=[], y=[], param=[]))
mapper = linear_cmap(field_name="param", palette=Turbo256, low=0, high=50)
ov_param_plot.add_glyph(
ov_param_plot_scatter_source,
Scatter(x="x", y="y", line_color=mapper, fill_color=mapper, size=10),
)
ov_param_plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
ov_param_plot.toolbar.logo = None
# Parameter plot
param_plot = Plot(x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=400,
plot_width=700)
param_plot.add_layout(LinearAxis(axis_label="Fit parameter"), place="left")
param_plot.add_layout(LinearAxis(axis_label="Parameter"), place="below")
param_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
param_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
param_plot_scatter_source = ColumnDataSource(dict(x=[], y=[]))
param_plot.add_glyph(param_plot_scatter_source, Scatter(x="x", y="y"))
param_plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
param_plot.toolbar.logo = None
def fit_param_select_callback(_attr, _old, _new):
_update_param_plot()
fit_param_select = Select(title="Fit parameter", options=[], width=145)
fit_param_select.on_change("value", fit_param_select_callback)
# Plot tabs
plots = Tabs(tabs=[
Panel(child=plot, title="single scan"),
Panel(child=ov_plot, title="overview"),
Panel(child=ov_param_plot, title="overview map"),
Panel(child=column(param_plot, row(fit_param_select)),
title="parameter plot"),
])
# Scan select
def scan_table_select_callback(_attr, old, new):
if not new:
# skip empty selections
return
# Avoid selection of multiple indicies (via Shift+Click or Ctrl+Click)
if len(new) > 1:
# drop selection to the previous one
scan_table_source.selected.indices = old
return
if len(old) > 1:
# skip unnecessary update caused by selection drop
return
_update_plot()
def scan_table_source_callback(_attr, _old, _new):
_update_preview()
scan_table_source = ColumnDataSource(
dict(file=[], scan=[], param=[], fit=[], export=[]))
scan_table_source.on_change("data", scan_table_source_callback)
scan_table = DataTable(
source=scan_table_source,
columns=[
TableColumn(field="file", title="file", width=150),
TableColumn(field="scan", title="scan", width=50),
TableColumn(field="param",
title="param",
editor=NumberEditor(),
width=50),
TableColumn(field="fit", title="Fit", width=50),
TableColumn(field="export",
title="Export",
editor=CheckboxEditor(),
width=50),
],
width=410, # +60 because of the index column
editable=True,
autosize_mode="none",
)
def scan_table_source_callback(_attr, _old, _new):
if scan_table_source.selected.indices:
_update_plot()
scan_table_source.selected.on_change("indices", scan_table_select_callback)
scan_table_source.on_change("data", scan_table_source_callback)
def _get_selected_scan():
return det_data[scan_table_source.selected.indices[0]]
def param_select_callback(_attr, _old, new):
if new == "user defined":
param = [None] * len(det_data)
else:
param = [scan[new] for scan in det_data]
scan_table_source.data["param"] = param
_update_param_plot()
param_select = Select(
title="Parameter:",
options=["user defined", "temp", "mf", "h", "k", "l"],
value="user defined",
width=145,
)
param_select.on_change("value", param_select_callback)
def fit_from_spinner_callback(_attr, _old, new):
fit_from_span.location = new
fit_from_spinner = Spinner(title="Fit from:", width=145)
fit_from_spinner.on_change("value", fit_from_spinner_callback)
def fit_to_spinner_callback(_attr, _old, new):
fit_to_span.location = new
fit_to_spinner = Spinner(title="to:", width=145)
fit_to_spinner.on_change("value", fit_to_spinner_callback)
def fitparams_add_dropdown_callback(click):
# bokeh requires (str, str) for MultiSelect options
new_tag = f"{click.item}-{fitparams_select.tags[0]}"
fitparams_select.options.append((new_tag, click.item))
fit_params[new_tag] = fitparams_factory(click.item)
fitparams_select.tags[0] += 1
fitparams_add_dropdown = Dropdown(
label="Add fit function",
menu=[
("Linear", "linear"),
("Gaussian", "gaussian"),
("Voigt", "voigt"),
("Pseudo Voigt", "pvoigt"),
# ("Pseudo Voigt1", "pseudovoigt1"),
],
width=145,
)
fitparams_add_dropdown.on_click(fitparams_add_dropdown_callback)
def fitparams_select_callback(_attr, old, new):
# Avoid selection of multiple indicies (via Shift+Click or Ctrl+Click)
if len(new) > 1:
# drop selection to the previous one
fitparams_select.value = old
return
if len(old) > 1:
# skip unnecessary update caused by selection drop
return
if new:
fitparams_table_source.data.update(fit_params[new[0]])
else:
fitparams_table_source.data.update(
dict(param=[], value=[], vary=[], min=[], max=[]))
fitparams_select = MultiSelect(options=[], height=120, width=145)
fitparams_select.tags = [0]
fitparams_select.on_change("value", fitparams_select_callback)
def fitparams_remove_button_callback():
if fitparams_select.value:
sel_tag = fitparams_select.value[0]
del fit_params[sel_tag]
for elem in fitparams_select.options:
if elem[0] == sel_tag:
fitparams_select.options.remove(elem)
break
fitparams_select.value = []
fitparams_remove_button = Button(label="Remove fit function", width=145)
fitparams_remove_button.on_click(fitparams_remove_button_callback)
def fitparams_factory(function):
if function == "linear":
params = ["slope", "intercept"]
elif function == "gaussian":
params = ["amplitude", "center", "sigma"]
elif function == "voigt":
params = ["amplitude", "center", "sigma", "gamma"]
elif function == "pvoigt":
params = ["amplitude", "center", "sigma", "fraction"]
elif function == "pseudovoigt1":
params = ["amplitude", "center", "g_sigma", "l_sigma", "fraction"]
else:
raise ValueError("Unknown fit function")
n = len(params)
fitparams = dict(
param=params,
value=[None] * n,
vary=[True] * n,
min=[None] * n,
max=[None] * n,
)
if function == "linear":
fitparams["value"] = [0, 1]
fitparams["vary"] = [False, True]
fitparams["min"] = [None, 0]
elif function == "gaussian":
fitparams["min"] = [0, None, None]
return fitparams
fitparams_table_source = ColumnDataSource(
dict(param=[], value=[], vary=[], min=[], max=[]))
fitparams_table = DataTable(
source=fitparams_table_source,
columns=[
TableColumn(field="param", title="Parameter"),
TableColumn(field="value", title="Value", editor=NumberEditor()),
TableColumn(field="vary", title="Vary", editor=CheckboxEditor()),
TableColumn(field="min", title="Min", editor=NumberEditor()),
TableColumn(field="max", title="Max", editor=NumberEditor()),
],
height=200,
width=350,
index_position=None,
editable=True,
auto_edit=True,
)
# start with `background` and `gauss` fit functions added
fitparams_add_dropdown_callback(types.SimpleNamespace(item="linear"))
fitparams_add_dropdown_callback(types.SimpleNamespace(item="gaussian"))
fitparams_select.value = ["gaussian-1"] # add selection to gauss
fit_output_textinput = TextAreaInput(title="Fit results:",
width=750,
height=200)
def proc_all_button_callback():
for scan, export in zip(det_data, scan_table_source.data["export"]):
if export:
pyzebra.fit_scan(scan,
fit_params,
fit_from=fit_from_spinner.value,
fit_to=fit_to_spinner.value)
pyzebra.get_area(
scan,
area_method=AREA_METHODS[area_method_radiobutton.active],
lorentz=lorentz_checkbox.active,
)
_update_plot()
_update_table()
for scan in det_data:
if "fit" in scan:
options = list(scan["fit"].params.keys())
fit_param_select.options = options
fit_param_select.value = options[0]
break
_update_param_plot()
proc_all_button = Button(label="Process All",
button_type="primary",
width=145)
proc_all_button.on_click(proc_all_button_callback)
def proc_button_callback():
scan = _get_selected_scan()
pyzebra.fit_scan(scan,
fit_params,
fit_from=fit_from_spinner.value,
fit_to=fit_to_spinner.value)
pyzebra.get_area(
scan,
area_method=AREA_METHODS[area_method_radiobutton.active],
lorentz=lorentz_checkbox.active,
)
_update_plot()
_update_table()
for scan in det_data:
if "fit" in scan:
options = list(scan["fit"].params.keys())
fit_param_select.options = options
fit_param_select.value = options[0]
break
_update_param_plot()
proc_button = Button(label="Process Current", width=145)
proc_button.on_click(proc_button_callback)
area_method_div = Div(text="Intensity:", margin=(5, 5, 0, 5))
area_method_radiobutton = RadioGroup(labels=["Function", "Area"],
active=0,
width=145)
lorentz_checkbox = CheckboxGroup(labels=["Lorentz Correction"],
width=145,
margin=(13, 5, 5, 5))
export_preview_textinput = TextAreaInput(title="Export file preview:",
width=450,
height=400)
def _update_preview():
with tempfile.TemporaryDirectory() as temp_dir:
temp_file = temp_dir + "/temp"
export_data = []
for s, export in zip(det_data, scan_table_source.data["export"]):
if export:
export_data.append(s)
# pyzebra.export_1D(export_data, temp_file, "fullprof")
exported_content = ""
file_content = []
for ext in (".comm", ".incomm"):
fname = temp_file + ext
if os.path.isfile(fname):
with open(fname) as f:
content = f.read()
exported_content += f"{ext} file:\n" + content
else:
content = ""
file_content.append(content)
js_data.data.update(content=file_content)
export_preview_textinput.value = exported_content
save_button = Button(label="Download File(s)",
button_type="success",
width=220)
save_button.js_on_click(
CustomJS(args={"js_data": js_data}, code=javaScript))
fitpeak_controls = row(
column(fitparams_add_dropdown, fitparams_select,
fitparams_remove_button),
fitparams_table,
Spacer(width=20),
column(fit_from_spinner, lorentz_checkbox, area_method_div,
area_method_radiobutton),
column(fit_to_spinner, proc_button, proc_all_button),
)
scan_layout = column(scan_table,
row(monitor_spinner, scan_motor_select, param_select))
import_layout = column(
proposal_textinput,
file_select,
row(file_open_button, file_append_button),
upload_div,
upload_button,
append_upload_div,
append_upload_button,
)
export_layout = column(export_preview_textinput, row(save_button))
tab_layout = column(
row(import_layout, scan_layout, plots, Spacer(width=30),
export_layout),
row(fitpeak_controls, fit_output_textinput),
)
return Panel(child=tab_layout, title="param study")
class Selector:
def __init__(
self,
name="Specials",
descr="Choose one",
kind="specials",
css_classes=[],
entries={},
default="",
title=None,
none_allowed=False,
):
self.name = name
self.descr = descr
self.entries = entries
self.kind = kind
self.css_classes = css_classes
options = sorted(entries.keys())
if none_allowed:
options = ["None"] + options
if title is None:
title = "."
css_classes = ["deli-selector", "hide-title"]
else:
css_classes = ["deli-selector"]
self.widget = MultiSelect(
options=options,
value=[default],
# height=150,
size=8,
name="deli-selector",
title=title,
css_classes=css_classes,
)
# HACK: force MultiSelect to only have 1 value selected
def multi_select_hack(attr, old, new):
if len(new) > 1:
self.widget.value = old
self.widget.on_change("value", multi_select_hack)
@property
def value(self):
# HACK: This is because we are useing MultiSelect instead of Select
return self.widget.value[0]
def layout(self, additional_widgets=[], width=None):
title = Div(
text="""<h2>{0}</h2><h3>{1}</h3>""".format(self.name, self.descr),
css_classes=["controls"],
)
footer = Div(
text="""<a href="#">About the {0}</a>""".format(self.kind),
css_classes=["controls", "controls-footer"],
)
if width is None:
width = 160 * (1 + len(additional_widgets))
return column(
title,
row(
self.widget,
*additional_widgets,
width=width,
css_classes=["controls"],
),
footer,
css_classes=self.css_classes,
)
select_concurrency = Select(title="Concurrency Factor:",
value=options[0],
options=options)
select_concurrency.on_change('value', update)
labels = list(dataframe['database'].sort_values().apply(str).unique())
checkbox_database = CheckboxGroup(
labels=labels, active=[index for index, item in enumerate(labels)])
checkbox_database.on_change('active', update)
options = list(dataframe['query_id'].sort_values().apply(str).unique())
multiselect_query_id = MultiSelect(title="Query ID:",
value=options,
options=list(zip(options, options)),
size=5)
multiselect_query_id.on_change('value', update)
end = dataframe['rows'].max()
slider_rows = RangeSlider(start=0,
end=end,
range=(0, end),
step=end // 100,
title="Rows")
slider_rows.on_change('range', update)
end = dataframe['time'].max()
slider_time = RangeSlider(start=0,
end=end,
range=(0, end),
step=end // 100,
title="Time")
def create_logistic_growth_subtab(params_getter, time_series_getter,
starting_regions, tab_title):
# Data Sources
params_df, params_CDS = params_getter()
time_series_df, time_series_CDS = time_series_getter()
dates = np.arange(START_DATE_STRING, '2020-07-01', dtype='datetime64[D]')
bands_CDS = ColumnDataSource({'date': dates})
lines_CDS = ColumnDataSource({'date': dates})
# Set up Figure
plot = figure(x_axis_type='datetime',
x_axis_label='Date',
y_axis_label='Number of Cases',
title='Logistic Growth Modeling',
active_scroll='wheel_zoom')
plot.yaxis.formatter.use_scientific = False
lines = {} # region, line pairs housing which have been drawn already
bands = {} # region, band pairs housing which have been drawn already
def logistic_function(x, L, x0, k):
return L / (1 + np.exp(-k * (x - x0)))
def z_star(conf_level):
return st.norm.ppf(1 - (1 - conf_level) / 2)
def get_offset_from_start_date(region):
subdf = time_series_df[region]
nonzero_subdf = subdf[subdf > 0]
offset = (nonzero_subdf.index[0] - pd.to_datetime(START_DATE)).days
return offset
def draw_prediction_line(region, **kwargs):
plot_params = {
'line_width': 4,
'line_alpha': 0.4,
'line_dash': 'dashed'
}
plot_params.update(kwargs)
L, x0, k, L_std, x0_std, k_std = params_CDS.data[region]
xs = np.arange(lines_CDS.data['date'].size)
offset = get_offset_from_start_date(region)
line = logistic_function(xs, L, x0 + offset, k)
lines_CDS.data[region] = line
lines[f'{region}_prediction'] = plot.line(x='date',
y=region,
source=lines_CDS,
name=region,
**plot_params)
def draw_prediction_band(region, conf_level, **kwargs):
plot_params = {'line_alpha': 0, 'fill_alpha': 0.4}
plot_params.update(kwargs)
L, x0, k, L_std, x0_std, k_std = params_CDS.data[region]
xs = np.arange(lines_CDS.data['date'].size)
offset = get_offset_from_start_date(region)
bands_CDS.data[f'{region}_lower'], bands_CDS.data[
f'{region}_upper'] = (logistic_function(
xs, L - L_std * z_star(conf_level), x0 + offset, k),
logistic_function(
xs, L + L_std * z_star(conf_level),
x0 + offset, k))
bands[region] = Band(base='date',
lower=f'{region}_lower',
upper=f'{region}_upper',
source=bands_CDS,
level='underlay',
**plot_params)
plot.add_layout(bands[region])
def draw_data_line(region, **kwargs):
plot_params = {
'line_width': 4,
}
plot_params.update(kwargs)
lines[region] = plot.line(x='date',
y=region,
source=time_series_CDS,
name=region,
**plot_params)
confidence_level = 0.95
for region, color in zip(starting_regions, viridis(len(starting_regions))):
color = RGB(*hex_string_to_rgb(color))
darkened_color = color.darken(.15)
lightened_color = color.lighten(.15)
# draw prediction band
draw_prediction_band(region, confidence_level, fill_color=color)
# draw prediction line
draw_prediction_line(region, line_color=darkened_color)
# draw data line
draw_data_line(region, line_color=color)
# Hover Tool
hover_tool = HoverTool(tooltips=[('Date', '@date{%F}'),
('Region', '$name'),
('Num. Cases', '@$name{0,0}')],
formatters={
'@date': 'datetime',
})
plot.add_tools(hover_tool)
hover_tool.renderers = list(lines.values())
# Legend
prediction_line_glyph = plot.line(line_color='black',
line_dash='dashed',
name='prediction_line_glyph',
line_width=4)
prediction_line_glyph.visible = False
data_line_glyph = plot.line(line_color='black',
name='data_line_glyph',
line_width=4)
data_line_glyph.visible = False
confidence_interval_glyph = plot.patch(
[0, 0, 1, 1],
[0, 1, 1, 0],
name='confidence_interval_glyph',
line_color='black',
line_width=1,
fill_alpha=0.3,
fill_color='black',
)
confidence_interval_glyph.visible = False
legend = Legend(items=[
LegendItem(label="Data",
renderers=[plot.select_one({'name': 'data_line_glyph'})]),
LegendItem(
label="Prediction",
renderers=[plot.select_one({'name': 'prediction_line_glyph'})]),
LegendItem(
label="95% Confidence Interval",
renderers=[plot.select_one({'name': 'confidence_interval_glyph'})])
],
location='top_left')
plot.add_layout(legend)
## Prevent legend glyphs from affecting plotting ranges
def fit_to_visible_lines():
plot.x_range.renderers = list(
filter(lambda line: line.visible, lines.values()))
plot.y_range.renderers = list(
filter(lambda line: line.visible, lines.values()))
# Region Selector
excluded_columns_set = {'index', 'parameters'}
labels = [
key for key in params_CDS.data.keys()
if key not in excluded_columns_set
]
def region_select_callback(attr, old, new):
new_lines = set(new) - set(old)
old_lines = set(old) - set(new)
for key in old_lines:
lines[key].visible = False
lines[f'{key}_prediction'].visible = False
bands[key].visible = False
for key in new_lines:
if key in lines.keys():
lines[key].visible = True
lines[f'{key}_prediction'].visible = True
bands[key].visible = True
else:
color = RGB(*hex_string_to_rgb(np.random.choice(Viridis256)))
darkened_color = color.darken(.15)
lightened_color = color.lighten(.15)
draw_prediction_line(key, line_color=darkened_color)
draw_prediction_band(key,
conf_level=confidence_level,
fill_color=lightened_color)
draw_data_line(key, line_color=color)
hover_tool.renderers = [
*hover_tool.renderers, lines[key],
lines[f'{key}_prediction']
]
region_select = MultiSelect(title='Select Regions to Show',
value=starting_regions,
options=labels,
sizing_mode='stretch_height')
region_select.on_change('value', region_select_callback)
# Final Setup
fit_to_visible_lines()
child = row(column([plot]), column([region_select]))
return Panel(child=child, title=tab_title)
def modify_doc(doc):
####### plot tab3 (birds location) #####
fig = figure(title="Birds recording in Lekagul Roadways",
x_range=(0, 250),
y_range=(0, 200),
tools=TOOLS,
width=900)
fig.xaxis.visible = False
fig.yaxis.visible = False
# Display the 32-bit RGBA image
fig.image_rgba(image=[img], x=0, y=0, dw=200, dh=200)
name_birds = birds['English_name'].unique()
source = ColumnDataSource(birds)
fig.circle('X',
'Y',
source=source,
size=8,
color='colors',
fill_alpha=0.7,
legend='English_name')
fig.asterisk(test_birds[' X'].tolist(),
test_birds[' Y'].tolist(),
size=20,
line_color="black",
line_width=3,
fill_color="blue",
fill_alpha=0.5)
rangeslider = RangeSlider(title="Date Range",
start=1983,
end=2018,
value=(1983, 2018),
step=1,
callback_policy='mouseup')
multiselect = MultiSelect(title="Select your species : ",
value=list(name_birds),
options=list(name_birds))
def callback(attr, old, new):
date_tuple = rangeslider.value
birds_name = multiselect.value
dataset = birds[
(birds['year'] <= date_tuple[1]) & (birds['year'] >= date_tuple[0])
& (birds['English_name'].apply(lambda x: in_name(x, birds_name)))]
new_data = ColumnDataSource(dataset)
source.data = new_data.data
multiselect.on_change('value', callback)
rangeslider.on_change('value', callback)
inputs = widgetbox(rangeslider, multiselect)
####### main tab ######
fig_1 = figure(title="Birds recording in Lekagul Roadways",
x_range=(0, 200),
y_range=(0, 200),
tools=TOOLS,
width=600,
height=500)
fig_1.xaxis.visible = False
fig_1.yaxis.visible = False
fig_1.image_rgba(image=[img], x=0, y=0, dw=200, dh=200)
source_1 = ColumnDataSource(birds)
source_2 = ColumnDataSource(test_birds)
fig_1.circle_cross(x=148, y=160, color='black', size=30, fill_alpha=0.3)
fig_1.circle('X',
'Y',
source=source_1,
size=8,
color='colors',
fill_alpha=0.7)
fig_1.asterisk(' X',
' Y',
source=source_2,
size=20,
line_color="black",
line_width=3,
fill_color="blue",
fill_alpha=0.5)
radio_button_group_test = RadioButtonGroup(labels=list(
test_birds.ID.apply(str)),
active=None,
width=1300)
rangeslider_2 = RangeSlider(title="Date Range",
start=1983,
end=2018,
value=(1983, 2018),
step=1,
callback_policy='mouseup')
button_pred = Button(label='Predicted bird', button_type='danger')
def update_1(attr, old, new):
test_bird_num = radio_button_group_test.active
if test_bird_num is not None:
#button_pred.label=str(list_prediction[test_bird_num])
dataset = test_birds[test_birds['ID'] == test_bird_num + 1]
new_data = ColumnDataSource(dataset)
source_2.data = new_data.data
button_pred.label = list_ordered_name[np.argmax(
array_pred[test_bird_num, :])]
list_map = list(
map(lambda x, y: x + ' ------ ' + str(y), list_ordered_name,
array_pred[test_bird_num, :]))
list_change = []
for name_bird in name_birds:
list_change.append(
list_map[list_ordered_name.index(name_bird)])
radio_button_group_train._property_values[
'labels'][:] = list_change
plot_amplitude_test = create_signal_plot(
list_df_test_signals[test_bird_num],
'test %s' % (test_bird_num + 1))
plot_amplitude_test.x_range = Range1d(0, 60)
plot_spectrum_test = create_signal_plot(
list_df_test_spectrum[test_bird_num],
'test %s' % (test_bird_num + 1), "Fréquence", "Magnitude",
"Freq", "Magnitude")
plot_spectrum_test.x_range = Range1d(0, 0.02)
signals_plot.children[0] = plot_amplitude_test
signals_plot.children[2] = plot_spectrum_test
radio_button_group_test.on_change('active', update_1)
radio_button_group_train = RadioGroup(labels=list(name_birds), active=0)
def update_2(attr, old, new):
idx_name_bird = radio_button_group_train.active
date_tuple = rangeslider_2.value
if idx_name_bird is not None:
dataset = birds[(birds['year'] <= date_tuple[1])
& (birds['year'] >= date_tuple[0]) &
(birds['English_name']
== name_birds[idx_name_bird])]
new_data = ColumnDataSource(dataset)
source_1.data = new_data.data
if new in range(15):
plot_amplitude_test = create_signal_plot(
dict_df_signals[name_birds[idx_name_bird]],
name_birds[idx_name_bird])
plot_amplitude_test.x_range = Range1d(0, 60)
plot_spectrum_test = create_signal_plot(
dict_df_spectrum[name_birds[idx_name_bird]],
name_birds[idx_name_bird], "Fréquence", "Magnitude",
"Freq", "Magnitude")
plot_spectrum_test.x_range = Range1d(0, 0.02)
signals_plot.children[1] = plot_amplitude_test
signals_plot.children[3] = plot_spectrum_test
else:
pass
radio_button_group_train.on_change('active', update_2)
rangeslider_2.on_change('value', update_2)
plot_amplitude_train = create_signal_plot(dict_df_signals[name_birds[0]],
name_birds[0])
plot_amplitude_train.x_range = Range1d(0, 60)
plot_amplitude_test = create_signal_plot(list_df_test_signals[0], 'test 1')
plot_amplitude_test.x_range = Range1d(0, 60)
plot_spectrum_train = create_signal_plot(dict_df_spectrum[name_birds[0]],
name_birds[0], "Fréquence",
"Magnitude", "Freq", "Magnitude")
plot_spectrum_train.x_range = Range1d(0, 0.02)
plot_spectrum_test = create_signal_plot(list_df_test_spectrum[0], 'test 1',
"Fréquence", "Magnitude", "Freq",
"Magnitude")
plot_spectrum_test.x_range = Range1d(0, 0.02)
signals_plot = column([
plot_amplitude_test, plot_amplitude_train, plot_spectrum_test,
plot_spectrum_train
])
row_map_signal = row([
radio_button_group_train,
column([fig_1, rangeslider_2]), signals_plot
])
column_main = column([
row([
widgetbox([PreText(text="Prediction : "),
radio_button_group_test]), button_pred
]), row_map_signal
])
tab1 = Panel(child=column_main, title="Main view")
tab2 = Panel(child=column(inputs, fig), title="Birds location")
##### tab2 : birds call /song evolution
birds_call_song = birds.copy()
birds_call_song['Vocalization_type'] = birds_call_song[
'Vocalization_type'].apply(lambda x: x.lower().strip())
birds_call_song.dropna(inplace=True)
fig_3 = figure(title="Birds recording in Lekagul Roadways",
x_range=(0, 250),
y_range=(0, 200),
tools=TOOLS,
width=900)
fig_3.xaxis.visible = False
fig_3.yaxis.visible = False
fig_3.image_rgba(image=[img], x=0, y=0, dw=200, dh=200)
birds_call = birds_call_song[birds_call_song['Vocalization_type'] ==
'call']
birds_song = birds_call_song[birds_call_song['Vocalization_type'] ==
'song']
birds_call_song = birds_call_song[birds_call_song['Vocalization_type'] ==
'call, song']
source_call = ColumnDataSource(birds_call)
source_song = ColumnDataSource(birds_song)
source_call_song = ColumnDataSource(birds_call_song)
fig_3.circle_cross(x=148, y=160, color='black', size=30, fill_alpha=0.3)
fig_3.circle('X',
'Y',
source=source_call,
size=16,
color='colors',
fill_alpha=0.7,
legend='Vocalization_type')
fig_3.triangle('X',
'Y',
source=source_song,
size=16,
color='colors',
fill_alpha=0.7,
legend='Vocalization_type')
fig_3.square('X',
'Y',
source=source_call_song,
size=16,
color='colors',
fill_alpha=0.7,
legend='Vocalization_type')
fig_3.asterisk(test_birds[' X'].tolist(),
test_birds[' Y'].tolist(),
size=20,
line_color="black",
line_width=3,
fill_color="blue",
fill_alpha=0.5)
slider = Slider(title="Select the year:",
start=1983,
end=2018,
value=2017,
step=1)
select = Select(title="Select your specie : ",
value='Rose-crested Blue Pipit',
options=list(name_birds))
stacked_bar = figure(plot_height=250,
title="Vocalization type percentage by year",
toolbar_location=None)
source_stacked_bar = ColumnDataSource(
dict_stacked_df_percentage['Rose-crested Blue Pipit'])
stacked_bar.vbar_stack(['call', 'song'],
x='Year',
width=0.9,
source=source_stacked_bar,
color=['silver', 'lightblue'],
legend=[value(x) for x in ['call', 'song']])
stacked_bar.legend.location = "bottom_left"
stacked_bar_count = figure(plot_height=250,
title="Vocalization type count by year",
toolbar_location=None)
source_stacked_bar_count = ColumnDataSource(
dict_stacked_df_count['Rose-crested Blue Pipit'])
stacked_bar_count.vbar_stack(['call', 'song'],
x='Year',
width=0.9,
source=source_stacked_bar_count,
color=['silver', 'lightblue'],
legend=[value(x) for x in ['call', 'song']])
stacked_bar_count.legend.location = "bottom_left"
def callback_call(attr, old, new):
year = slider.value
birds_name = select.value
dataset_call = birds_call[(birds_call['year'] == year)
& (birds_call['English_name'] == birds_name)]
new_data = ColumnDataSource(dataset_call)
source_call.data = new_data.data
dataset_song = birds_song[(birds_song['year'] == year)
& (birds_song['English_name'] == birds_name)]
new_data = ColumnDataSource(dataset_song)
source_song.data = new_data.data
dataset_call_song = birds_call_song[
(birds_call_song['year'] == year)
& (birds_call_song['English_name'] == birds_name)]
new_data = ColumnDataSource(dataset_call_song)
source_call_song.data = new_data.data
new_data = ColumnDataSource(dict_stacked_df_percentage[birds_name])
source_stacked_bar.data = new_data.data
new_data = ColumnDataSource(dict_stacked_df_count[birds_name])
source_stacked_bar_count.data = new_data.data
select.on_change('value', callback_call)
slider.on_change('value', callback_call)
tab3 = Panel(child=column(
select, row([fig_3, column(stacked_bar, stacked_bar_count)]), slider),
title="Birds Evolution")
#### regroup the tabs into one dashboard #####
tabs = Tabs(tabs=[tab1, tab3, tab2])
doc.add_root(tabs)
def nuts_app(doc):
import app.bokeh.salt_config as cfg
# TODO: abstract parts of this to a separate file
# TODO: following above, make parts reusable?
# load continuous CTD data and make into a dict (only ~20MB)
file_list = sorted(
glob.glob("/Users/mkovatch/Documents/ctdcal/data/pressure/*.csv"))
ssscc_list = [
ssscc.strip("/Users/mkovatch/Documents/ctdcal/data/pressure/")[:5]
for ssscc in file_list
]
ctd_data = []
for f in file_list:
df = pd.read_csv(f,
header=12,
skiprows=[13],
skipfooter=1,
engine="python")
df["SSSCC"] = f.strip(
"/Users/mkovatch/Documents/ctdcal/data/pressure/")[:5]
ctd_data.append(df)
ctd_data = pd.concat(ctd_data, axis=0, sort=False)
# load bottle trip file
file_list = sorted(
glob.glob("/Users/mkovatch/Documents/ctdcal/data/bottle/*.pkl"))
ssscc_list = [
ssscc.strip("/Users/mkovatch/Documents/ctdcal/data/bottle/")[:5]
for ssscc in file_list
]
upcast_data = []
for f in file_list:
with open(f, "rb") as x:
df = pickle.load(x)
df["SSSCC"] = f.strip(
"/Users/mkovatch/Documents/ctdcal/data/bottle/")[:5]
# change to secondary if that is what's used
upcast_data.append(df[["SSSCC", "CTDCOND1", "CTDTMP1", "CTDPRS"]])
upcast_data = pd.concat(upcast_data, axis=0, sort=False)
upcast_data["CTDSAL"] = gsw.SP_from_C(upcast_data["CTDCOND1"],
upcast_data["CTDTMP1"],
upcast_data["CTDPRS"])
# load salt file (adapted from compare_salinities.ipynb)
file_list = sorted(
glob.glob("/Users/mkovatch/Documents/ctdcal/data/salt/*.csv"))
ssscc_list = [
ssscc.strip("/Users/mkovatch/Documents/ctdcal/data/salt/")[:5]
for ssscc in file_list
]
salt_data = []
for f in file_list:
df = pd.read_csv(f, usecols=["STNNBR", "CASTNO", "SAMPNO", "SALNTY"])
df["SSSCC"] = f.strip(
"/Users/mkovatch/Documents/ctdcal/data/salt/")[:5]
salt_data.append(df)
salt_data = pd.concat(salt_data, axis=0, sort=False)
salt_data["SALNTY"] = salt_data["SALNTY"].round(4)
if "SALNTY_FLAG_W" not in salt_data.columns:
salt_data["SALNTY_FLAG_W"] = 2
# load ctd btl data
df_ctd_btl = pd.read_csv(
"/Users/mkovatch/Documents/ctdcal/data/scratch_folder/ctd_to_bottle.csv",
skiprows=[1],
skipfooter=1,
engine="python",
)
df_btl_all = pd.merge(df_ctd_btl,
salt_data,
on=["STNNBR", "CASTNO", "SAMPNO"])
btl_data = df_btl_all.loc[:, [
"SSSCC",
"SAMPNO",
"CTDPRS",
"CTDTMP",
"REFTMP",
"CTDSAL",
"SALNTY",
"SALNTY_FLAG_W",
], ]
btl_data["Residual"] = btl_data["SALNTY"] - btl_data["CTDSAL"]
btl_data[["CTDPRS", "Residual"]] = btl_data[["CTDPRS",
"Residual"]].round(4)
btl_data["Comments"] = ""
btl_data["New Flag"] = btl_data["SALNTY_FLAG_W"].copy()
# update with old handcoded flags if file exists
handcoded_file = "salt_flags_handcoded.csv"
if glob.glob(handcoded_file):
handcodes = pd.read_csv(handcoded_file,
dtype={"SSSCC": str},
keep_default_na=False)
handcodes = handcodes.rename(columns={
"salinity_flag": "New Flag"
}).drop(columns="diff")
# there's gotta be a better way. but this is good enough for now
btl_data = btl_data.merge(handcodes,
on=["SSSCC", "SAMPNO"],
how="left")
merge_rows = ~btl_data["New Flag_y"].isnull(
) | ~btl_data["Comments_y"].isnull()
btl_data.loc[merge_rows, "New Flag_x"] = btl_data.loc[merge_rows,
"New Flag_y"]
btl_data.loc[merge_rows, "Comments_x"] = btl_data.loc[merge_rows,
"Comments_y"]
btl_data = btl_data.rename(columns={
"New Flag_x": "New Flag",
"Comments_x": "Comments"
}).drop(columns=["New Flag_y", "Comments_y"])
# intialize widgets
save_button = Button(label="Save flagged data", button_type="success")
parameter = Select(title="Parameter",
options=["CTDSAL", "CTDTMP"],
value="CTDTMP")
ref_param = Select(title="Reference", options=["SALNTY"], value="SALNTY")
# ref_param.options = ["foo","bar"] # can dynamically change dropdowns
station = Select(title="Station", options=ssscc_list, value=ssscc_list[0])
# explanation of flags:
# https://cchdo.github.io/hdo-assets/documentation/manuals/pdf/90_1/chap4.pdf
flag_list = MultiSelect(
title="Plot data flagged as:",
value=["1", "2", "3"],
options=cfg.flag_options,
) # returns list of select options, e.g., ['2'] or ['1','2']
flag_input = Select(
title="Flag:",
options=cfg.flag_options,
value="3",
)
comment_box = TextInput(value="", title="Comment:")
# button_type: default, primary, success, warning or danger
flag_button = Button(label="Apply to selected", button_type="primary")
comment_button = Button(label="Apply to selected", button_type="warning")
vspace = Div(text=""" """, width=200, height=65)
bulk_flag_text = Div(
text="""<br><br>
<b>Bulk Bottle Flagging:</b><br>
Select multiple bottles using the table
(with shift/control) or the 'Box Select' tool on the plot.""",
width=150,
height=135,
)
# set up plot datasources
src_plot_trace = ColumnDataSource(data=dict(x=[], y=[]))
src_plot_ctd = ColumnDataSource(data=dict(x=[], y=[]))
src_plot_upcast = ColumnDataSource(data=dict(x=[], y=[]))
src_plot_btl = ColumnDataSource(data=dict(x=[], y=[]))
# set up plots
fig = figure(
title="{} vs CTDPRS [Station {}]".format(parameter.value,
station.value),
tools="pan,box_zoom,wheel_zoom,box_select,reset",
y_axis_label="Pressure (dbar)",
)
fig.select(BoxSelectTool).select_every_mousemove = False
fig.y_range.flipped = True # invert y-axis
fig.line(
"x",
"y",
line_color="#000000",
line_width=2,
source=src_plot_trace,
legend_label="CTD Trace",
)
btl_sal = fig.asterisk(
"x",
"y",
size=12,
line_width=1.5,
color="#0033CC",
source=src_plot_btl,
legend_label="Bottle sample",
)
ctd_sal = fig.circle(
"x",
"y",
size=7,
color="#BB0000",
source=src_plot_ctd,
legend_label="Downcast CTD sample",
)
upcast_sal = fig.triangle(
"x",
"y",
size=7,
color="#00BB00",
source=src_plot_upcast,
legend_label="Upcast CTD sample",
)
fig.legend.location = "bottom_left"
fig.legend.border_line_width = 3
fig.legend.border_line_alpha = 1
btl_sal.nonselection_glyph.line_alpha = 0.2
ctd_sal.nonselection_glyph.fill_alpha = 1 # makes CTDSAL *not* change on select
upcast_sal.nonselection_glyph.fill_alpha = 1 # makes CTDSAL *not* change on select
# define callback functions
def update_selectors():
print("exec update_selectors()")
ctd_rows = ctd_data["SSSCC"] == station.value
table_rows = btl_data["SSSCC"] == station.value
btl_rows = (btl_data["New Flag"].isin(
flag_list.value)) & (btl_data["SSSCC"] == station.value)
# update table data
current_table = btl_data[table_rows].reset_index()
src_table.data = { # this causes edit_flag() to execute
"SSSCC": current_table["SSSCC"],
"SAMPNO": current_table["SAMPNO"],
"CTDPRS": current_table["CTDPRS"],
"CTDSAL": current_table["CTDSAL"],
"SALNTY": current_table["SALNTY"],
"diff": current_table["Residual"],
"flag": current_table["New Flag"],
"Comments": current_table["Comments"],
}
# update plot data
src_plot_trace.data = {
"x": ctd_data.loc[ctd_rows, parameter.value],
"y": ctd_data.loc[ctd_rows, "CTDPRS"],
}
src_plot_ctd.data = {
"x": btl_data.loc[table_rows, parameter.value],
"y": btl_data.loc[table_rows, "CTDPRS"],
}
src_plot_upcast.data = {
"x": upcast_data.loc[upcast_data["SSSCC"] == station.value,
"CTDSAL"],
"y": upcast_data.loc[upcast_data["SSSCC"] == station.value,
"CTDPRS"],
}
src_plot_btl.data = {
"x": btl_data.loc[btl_rows, "SALNTY"],
"y": btl_data.loc[btl_rows, "CTDPRS"],
}
# update plot labels/axlims
fig.title.text = "{} vs CTDPRS [Station {}]".format(
parameter.value, station.value)
fig.xaxis.axis_label = parameter.value
# deselect all datapoints
btl_sal.data_source.selected.indices = []
src_table.selected.indices = []
def edit_flag():
print("exec edit_flag()")
btl_data.loc[btl_data["SSSCC"] == src_table.data["SSSCC"].values[0],
"New Flag", ] = src_table.data["flag"].values
btl_data.loc[btl_data["SSSCC"] == src_table.data["SSSCC"].values[0],
"Comments", ] = src_table.data["Comments"].values
edited_rows = (btl_data["New Flag"].isin(
[3, 4])) | (btl_data["Comments"] != "")
src_table_changed.data = {
"SSSCC": btl_data.loc[edited_rows, "SSSCC"],
"SAMPNO": btl_data.loc[edited_rows, "SAMPNO"],
"diff": btl_data.loc[edited_rows, "Residual"],
"flag_old": btl_data.loc[edited_rows, "SALNTY_FLAG_W"],
"flag_new": btl_data.loc[edited_rows, "New Flag"],
"Comments": btl_data.loc[edited_rows, "Comments"],
}
def apply_flag():
print("Applying flags")
table_rows = btl_data["SSSCC"] == station.value
selected_rows = src_table.selected.indices
# update table data
current_table = btl_data[table_rows].reset_index()
current_table.loc[selected_rows, "New Flag"] = int(flag_input.value)
src_table.data = { # this causes edit_flag() to execute
"SSSCC": current_table["SSSCC"],
"SAMPNO": current_table["SAMPNO"],
"CTDPRS": current_table["CTDPRS"],
"CTDSAL": current_table["CTDSAL"],
"SALNTY": current_table["SALNTY"],
"diff": current_table["Residual"],
"flag": current_table["New Flag"],
"Comments": current_table["Comments"],
}
def apply_comment():
print("Applying Comments")
table_rows = btl_data["SSSCC"] == station.value
selected_rows = src_table.selected.indices
# update table data
current_table = btl_data[table_rows].reset_index()
current_table.loc[selected_rows, "Comments"] = comment_box.value
src_table.data = { # this causes edit_flag() to execute
"SSSCC": current_table["SSSCC"],
"SAMPNO": current_table["SAMPNO"],
"CTDPRS": current_table["CTDPRS"],
"CTDSAL": current_table["CTDSAL"],
"SALNTY": current_table["SALNTY"],
"diff": current_table["Residual"],
"flag": current_table["New Flag"],
"Comments": current_table["Comments"],
}
def save_data():
print("Saving flagged data...")
# get data from table
df_out = pd.DataFrame.from_dict(src_table_changed.data)
# minor changes to columns/names/etc.
df_out = df_out.rename(columns={
"flag_new": "salinity_flag"
}).drop(columns="flag_old")
# save it
df_out.to_csv("salt_flags_handcoded.csv", index=None)
def selected_from_plot(attr, old, new):
src_table.selected.indices = new
def selected_from_table(attr, old, new):
btl_sal.data_source.selected.indices = new
# set up DataTables
src_table = ColumnDataSource(data=dict())
src_table_changed = ColumnDataSource(data=dict())
columns = bk.build_columns(
cfg.cast_columns["fields"],
cfg.cast_columns["titles"],
cfg.cast_columns["widths"],
)
columns_changed = bk.build_columns(
cfg.changes_columns["fields"],
cfg.changes_columns["titles"],
cfg.changes_columns["widths"],
)
data_table = DataTable(
source=src_table,
columns=columns,
index_width=20,
width=480 + 20, # sum of col widths + idx width
height=600,
editable=True,
fit_columns=True,
sortable=False,
)
data_table_changed = DataTable(
source=src_table_changed,
columns=columns_changed,
index_width=20,
width=480 + 20, # sum of col widths + idx width
height=200,
editable=False,
fit_columns=True,
sortable=False,
)
data_table_title = Div(text="""<b>All Station Data:</b>""",
width=200,
height=15)
data_table_changed_title = Div(text="""<b>Flagged Data:</b>""",
width=200,
height=15)
# set up change callbacks
parameter.on_change("value", lambda attr, old, new: update_selectors())
station.on_change("value", lambda attr, old, new: update_selectors())
flag_list.on_change("value", lambda attr, old, new: update_selectors())
flag_button.on_click(apply_flag)
comment_button.on_click(apply_comment)
save_button.on_click(save_data)
src_table.on_change("data", lambda attr, old, new: edit_flag())
src_table.selected.on_change("indices", selected_from_table)
btl_sal.data_source.selected.on_change("indices", selected_from_plot)
# format document
controls = column(
parameter,
ref_param,
station,
flag_list,
bulk_flag_text,
flag_input,
flag_button,
comment_box,
comment_button,
vspace,
save_button,
width=170,
)
tables = column(data_table_title, data_table, data_table_changed_title,
data_table_changed)
doc.add_root(row(controls, tables, fig))
doc.theme = Theme(filename="theme.yaml")
update_selectors()
def wcushow(doc):
'''
bokeh function server
:param doc: bokeh document
:return: updated document
'''
doc.theme = settings.colortheme
TOOLTIPS = [
("(x,y)", "($x, $y)"),
]
renderer = 'webgl'
graphTools = 'pan,wheel_zoom,box_zoom,zoom_in,zoom_out,hover,crosshair,undo,redo,reset,save'
cabecalho = ''
channelCounter = 0
WCUconfig = pd.read_csv('./projectfolder/configuration/dataWCU.csv',
sep=';',
index_col=False)
CANconfig = pd.read_csv('./projectfolder/configuration/configCAN.csv',
sep=';',
index_col=False)
for channel in WCUconfig['channel']:
channelCounter = channelCounter + 1
if (len(WCUconfig) > channelCounter):
cabecalho = cabecalho + channel + ','
else:
cabecalho = cabecalho + channel
# test temp data folders for wcu
if os.path.isdir('./_wcu_cacheFiles_'):
print('./_wcu_cacheFiles_ ok')
else:
os.mkdir('./_wcu_cacheFiles_')
#set COM port baudrate
baudrate = settings.boudrateselected
wcuUpdateTimer = 1 / 5 #second -> 1/FPS
portWCU = settings.port
#conect to WDU and clean garbage
comport = connectSerial(portWCU, baudrate)
cleanCOMPORT(comport=comport)
#create csv file for writing WCU data
global wcufilename
wcufilename = createCSV(cabecalho)
time.sleep(
2
) #to start serial, requires an delay to arduino load data at the buffer
#get WDU data
global data, lastupdate
lastupdate = '0.0'
for channel in CANconfig['Channel']:
lastupdate = lastupdate + ',0.0'
wcufile = open(wcufilename, "at")
data, lastupdate = updateWCUcsv(seconds=wcuUpdateTimer,
wcufile=wcufile,
comport=comport,
header=cabecalho,
canconfig=CANconfig,
laststr=lastupdate)
source = ColumnDataSource(data=data)
# Start Gauges
gg = 0
#start variables for an array of gauges
plot = []
linemin = []
linemax = []
valueglyph = []
#for each channel listed in the wcucsv that have an 'true' marked on the display column,
# there will be a plot
text_data_s = []
text_unit_s = []
text_name_s = []
text_color_s = []
texts = []
texplot = figure()
for channel in WCUconfig['channel']:
if (len(data[channel]) > 3): #remove errors
line = WCUconfig.iloc[gg]
if (line['display'] == 'gauge'):
dataValue = pd.to_numeric(data[channel])[len(data[channel]) -
1]
plt, mx, mi, vg = plotGauge(dataValue,
unit=line['unit'],
name=channel,
color=line['color'],
offset=line['minvalue'],
maxValue=line['maxvalue'],
major_step=line['majorstep'],
minor_step=line['minorstep'])
plot.append(plt)
linemax.append(mx)
linemin.append(mi)
valueglyph.append(vg)
if (line['display'] == 'text'):
dataValue = pd.to_numeric(data[channel])[len(data[channel]) -
1]
text_data_s.append(dataValue)
text_unit_s.append(line['unit'])
text_name_s.append(channel)
text_color_s.append(line['color'])
texplot, texts = plot_text_data(text_data_s,
unit=text_unit_s,
name=text_name_s,
color=text_color_s)
gg = gg + 1
# Other main plots
#GPS:
track, points, livepoint = gps(data, singleplot=True, H=300, W=300)
gpssource = points.data_source
livesource = livepoint.data_source
#Steering for steering angle
steering, steering_image, steering_text = plot_angle_image()
# line plots: secondary tabs
renderer = 'webgl'
p = figure(
plot_height=300,
plot_width=1000,
y_range=(0, 13000),
title='RPM',
x_axis_label='s',
y_axis_label='rpm',
toolbar_location="below",
tooltips=TOOLTIPS,
output_backend=renderer,
tools=graphTools,
)
g = p.line(x='time', y='RPM', color='red', source=source)
p.toolbar.logo = None
#function for update all live gauges and graphics
global wcufileglobal
try:
wcufile.close()
wcufile = open(wcufilename, "at")
except:
pass
def update_data():
#t1_start = process_time()
global data, lastupdate, lastline
lastline = data.iloc[[data.ndim - 1]].to_csv(header=False,
index=False).strip('\r\n')
data, lastupdate = updateWCUcsv(seconds=wcuUpdateTimer,
wcufile=wcufile,
comport=comport,
header=cabecalho,
canconfig=CANconfig,
laststr=lastupdate)
#t1_stop = process_time()
#print("HEY: {:.9f}".format((t1_stop - t1_start)))
def update_source():
#t1_start = process_time()
global data
data = wcu_equations(data)
source.data = data
#t1_stop = process_time()
#print("HEYHEYHEYHEYHEYHEY: {:.9f}".format((t1_stop - t1_start)))
def callback():
'''
callback function to update bokeh server
:return: none
'''
#df = source.to_df()
#lastline = data.iloc[[df.ndim - 1]].to_csv(header=False, index=False).strip('\r\n')
#lastupdate = ',' + ','.join(lastline.split(',')[(-len(CANconfig['Channel'])):])
us = partial(update_source)
doc.add_next_tick_callback(us)
ud = partial(update_data)
doc.add_next_tick_callback(ud)
global data, lastupdate, lastline
#alternative method
#data = source.to_df()
#lastline = df.iloc[[df.ndim - 1]].to_csv(header=False, index=False).strip('\r\n')
#lastupdate = ',' + ','.join(lastline.split(',')[(-len(CANconfig['Channel'])):])
gg = 0
linectr = 0
text_values_update = []
text_unit_s_update = []
text_name_s_update = []
for channel in WCUconfig['channel']:
line = WCUconfig.iloc[gg]
if (line['display'] == 'gauge'):
dataValue = pd.to_numeric(data[channel])[len(data[channel]) -
1]
angle = speed_to_angle(dataValue,
offset=line['minvalue'],
max_value=line['maxvalue'])
linemax[linectr].update(angle=angle)
linemin[linectr].update(angle=angle - pi)
valueglyph[linectr].update(
text=[str(round(dataValue, 1)) + ' ' + line['unit']])
linectr = linectr + 1
if (line['display'] == 'text'):
text_values_update.append(
pd.to_numeric(data[channel])[len(data[channel]) - 1])
text_unit_s_update.append(line['unit'])
text_name_s_update.append(channel)
gg = gg + 1
for i in range(0, len(texts)):
texts[i].update(text=[
text_name_s_update[i] + ': ' + str(text_values_update[i]) +
text_unit_s_update[i]
])
steeringangle = pd.to_numeric(
data['SteeringAngle'])[len(data['SteeringAngle']) - 1]
#steering_image.update(angle = steeringangle)
steering_text.update(
text=['Steering Angle' + ': ' + str(steeringangle) + 'deg'])
lat, long = latlong(data)
gpssource.data.update(x=lat, y=long)
livesource.data.update(x=lat.iloc[-1:], y=long.iloc[-1:])
global per_call
#per_call = doc.add_periodic_callback(update_source, wcuUpdateTimer*1001)
per_call = doc.add_periodic_callback(callback, wcuUpdateTimer * 1000)
'''
#Button to stop the server
def exit_callback():
doc.remove_periodic_callback(per_call)
endwculog(wcufilename)
button = Button(label="Stop", button_type="success")
button.on_click(exit_callback)
doc.add_root(button)
'''
#pre = PreText(text="""Select Witch Channels to Watch""",width=500, height=100)
global type_graph_option, graph_points_size
graph_points_size = 2
type_graph_option = 0
def addGraph(attrname, old, new):
'''
callback function to add graphs figure in the tab area for plotting
:param attrname:
:param old: old user selection
:param new: new user selected channels
:return: none
'''
global old_ch, new_ch
if len(old) < 5:
old_ch = old
new_ch = new
else:
new_ch = new
if len(new) < 5:
uptab = doc.get_model_by_name('graphtab')
for channel in old_ch:
tb = doc.get_model_by_name('graphtab')
if channel != '':
tb.child.children.remove(
tb.child.children[len(tb.child.children) - 1])
for channel in new_ch:
plot = figure(plot_height=300,
plot_width=1300,
title=channel,
x_axis_label='s',
y_axis_label=channel,
toolbar_location="below",
tooltips=TOOLTIPS,
output_backend=renderer,
tools=graphTools,
name=channel)
global type_graph_option, graph_points_size
if type_graph_option == 0:
plot.line(x='time', y=channel, color='red', source=source)
if type_graph_option == 1:
plot.circle(x='time',
y=channel,
color='red',
source=source,
size=graph_points_size)
plot.toolbar.logo = None
uptab.child.children.append(plot)
else:
error_2_wcu()
def radio_group_options(attrname, old, new):
global type_graph_option
type_graph_option = new
OPTIONS_LABEL = ["Line Graph", "Circle Points"]
radio_group = RadioGroup(labels=OPTIONS_LABEL, active=0)
radio_group.on_change("active", radio_group_options)
OPTIONS = cabecalho.split(',')
multi_select = MultiSelect(value=[''],
options=OPTIONS,
title='Select Channels',
width=300,
height=300)
multi_select.on_change("value", addGraph)
def update_datapoints(attrname, old, new):
settings.telemetry_points = new
if (new > 5000):
warning_1_wcu()
datasize_spinner = Spinner(title="Data Points Size",
low=1000,
high=10000,
step=1000,
value=1000,
width=80)
datasize_spinner.on_change("value", update_datapoints)
def update_graph_points_size(attrname, old, new):
global graph_points_size
graph_points_size = new
graph_points_size_spinner = Spinner(title="Circle Size",
low=1,
high=10,
step=1,
value=graph_points_size,
width=80)
datasize_spinner.on_change("value", update_graph_points_size)
#make the grid plot of all gauges at the main tab
Gauges = gridplot([[plot[0], plot[1], plot[4], plot[3]],
[plot[6], plot[2], plot[5], plot[7]],
[plot[8], plot[9], plot[10], plot[11]]],
toolbar_options={'logo': None})
#addGraph()
Graphs = (p)
layoutGraphs = layout(
row(Graphs, multi_select,
column(datasize_spinner, radio_group, graph_points_size_spinner)))
layoutGauges = layout(row(Gauges, column(track, steering, texplot)))
Gauges = Panel(child=layoutGauges, title="Gauges", closable=True)
Graphs = Panel(child=layoutGraphs,
title="Graphs",
closable=True,
name='graphtab')
def cleanup_session(session_context):
'''
This function is called when a session is closed: callback function to end WCU Bokeh server
:return: none
'''
endWCU()
#activate callback to detect when browser is closed
doc.on_session_destroyed(cleanup_session)
tabs = Tabs(tabs=[
Gauges,
Graphs,
], name='WCU TABS')
doc.add_root(tabs)
doc.title = 'WCU SCREEN'
return doc
def xrayvis_app(doc):
def load_wav_cb(attr, old, new):
'''Handle selection of audio file to be loaded.'''
if new == '':
return
global wavname
global snd
spkr, fname = os.path.split(new)
wavname = get_cached_fname(
fname,
f'https://linguistics.berkeley.edu/phonapps/xray_microbeam_database/{spkr}',
spkr)
# wavname = new
if not wavname.endswith('.wav'):
return
snd = parselmouth.Sound(wavname)
srcdf = pd.DataFrame(
dict(
seconds=snd.ts().astype(np.float32),
ch0=snd.values[0, :].astype(np.float32),
))
#! TODO: do file caching
phdf = allphdf.loc[allphdf.wavpath == new, :].copy()
phdf['t1'] = phdf['t1'].astype(np.float32)
wddf = allwddf.loc[allwddf.wavpath == new, :].copy()
wddf['t1'] = wddf['t1'].astype(np.float32)
uttdiv.text = '<b>Utterance:</b> ' + ' '.join(
wddf.word.str.replace('sp', '')).strip()
phwddf = pd.merge_asof(phdf[['t1', 'phone']],
wddf[['t1', 'word']],
on='t1',
suffixes=['_ph', '_wd'])
# TODO: separate t1_ph and t1_wd columns
srcdf = pd.merge_asof(srcdf, phwddf, left_on='seconds', right_on='t1')
srcdf[['phone', 'word']] = srcdf[['phone', 'word']].fillna('')
srcdf = srcdf.drop('t1', axis='columns')
dfs['srcdf'] = srcdf
source.data = srcdf
tngsource.data = {'x': [], 'y': []}
othsource.data = {'x': [], 'y': []}
timesource.data = {k: [] for k in timesource.data.keys()}
lasttngtimesource.data = {'x': [], 'y': []}
lastothtimesource.data = {'x': [], 'y': []}
playvisbtn.channels = channels
playvisbtn.disabled = False
playselbtn.channels = channels
playselbtn.disabled = False
playvisbtn.fs = snd.sampling_frequency
playvisbtn.start = snd.start_time
playvisbtn.end = snd.end_time
playselbtn.fs = snd.sampling_frequency
playselbtn.start = 0.0
playselbtn.end = 0.0
selbox.left = 0.0
selbox.right = 0.0
selbox.visible = False
cursor.location = 0.0
cursor.visible = False
ch0.visible = True
update_sgram()
load_artic()
set_limits(0.0, srcdf['seconds'].max())
def load_artic():
'''Load articulation data.'''
trace.title.text = 'Static trace'
traj.title.text = 'Trajectories'
tngfile = os.path.splitext(wavname)[0] + '.txy'
palfile = os.path.join(os.path.dirname(wavname), 'PAL.DAT')
phafile = os.path.join(os.path.dirname(wavname), 'PHA.DAT')
tngdf = pd.read_csv(tngfile,
sep='\t',
names=[
'sec', 'ULx', 'ULy', 'LLx', 'LLy', 'T1x',
'T1y', 'T2x', 'T2y', 'T3x', 'T3y', 'T4x',
'T4y', 'MIx', 'MIy', 'MMx', 'MMy'
])
# Convert to seconds
tngdf['sec'] = tngdf['sec'] / 1e6
tngdf = tngdf.set_index(['sec'])
# Convert to mm
tngdf[[
'ULx', 'ULy', 'LLx', 'LLy', 'T1x', 'T1y', 'T2x', 'T2y', 'T3x',
'T3y', 'T4x', 'T4y', 'MIx', 'MIy', 'MMx', 'MMy'
]] = tngdf[[
'ULx', 'ULy', 'LLx', 'LLy', 'T1x', 'T1y', 'T2x', 'T2y', 'T3x',
'T3y', 'T4x', 'T4y', 'MIx', 'MIy', 'MMx', 'MMy'
]] * 1e-3
# Find global x/y max/min in this recording to set axis limits.
# Exclude bad values (1000000 in data file; 1000 mm in scaled dataframe).
cmpdf = tngdf[tngdf < badval]
xmax = np.max(
np.max(
cmpdf[['ULx', 'LLx', 'T1x', 'T2x', 'T3x', 'T4x', 'MIx',
'MMx']]))
xmin = np.min(
np.min(
cmpdf[['ULx', 'LLx', 'T1x', 'T2x', 'T3x', 'T4x', 'MIx',
'MMx']]))
ymax = np.max(
np.max(
cmpdf[['ULy', 'LLy', 'T1y', 'T2y', 'T3y', 'T4y', 'MIy',
'MMy']]))
ymin = np.min(
np.min(
cmpdf[['ULy', 'LLy', 'T1y', 'T2y', 'T3y', 'T4y', 'MIy',
'MMy']]))
paldf = pd.read_csv(palfile, sep='\s+', header=None, names=['x', 'y'])
paldf = paldf * 1e-3
palsource.data = {'x': paldf['x'], 'y': paldf['y']}
phadf = pd.read_csv(phafile, sep='\s+', header=None, names=['x', 'y'])
phadf = phadf * 1e-3
phasource.data = {'x': phadf['x'], 'y': phadf['y']}
xmin = np.min([xmin, np.min(paldf['x']), np.min(phadf['x'])])
xmax = np.max([xmax, np.max(paldf['x']), np.max(phadf['x'])])
ymin = np.min([ymin, np.min(paldf['y']), np.min(phadf['y'])])
ymax = np.max([ymax, np.max(paldf['y']), np.max(phadf['y'])])
xsz = xmax - xmin
ysz = ymax - ymin
xrng = [xmin - (xsz * 0.05), xmax + (xsz * 0.05)]
yrng = [ymin - (ysz * 0.05), ymax + (ysz * 0.05)]
dfs['tngdf'] = tngdf
dfs['paldf'] = paldf
dfs['phadf'] = phadf
def update_sgram():
'''Update spectrogram based on current values.'''
if snd.end_time < 15:
sgrams[0] = snd2specgram(snd, 0.005)
specsource.data = dict(
sgram0=[sgrams[0].values.astype(np.float32)])
spec0img.glyph.dw = sgrams[0].x_grid().max()
spec0img.glyph.dh = sgrams[0].y_grid().max()
spec0cmap.low = _low_thresh()
spec0.visible = True
else:
specsource.data = dict(sgram0=[])
spec0.visible = False
def update_trace():
'''Update the static trace at the cursor time.'''
trace.title.text = f'Static trace ({cursor.location:0.4f})'
tidx = dfs['tngdf'].index.get_loc(cursor.location, method='nearest')
row = dfs['tngdf'].iloc[tidx]
tngsource.data = {
'x': [row.T1x, row.T2x, row.T3x, row.T4x],
'y': [row.T1y, row.T2y, row.T3y, row.T4y]
}
othsource.data = {
'x': [row.ULx, row.LLx, row.MIx, row.MMx],
'y': [row.ULy, row.LLy, row.MIy, row.MMy]
}
def update_traj():
'''Update the trajectories during the selected time range.'''
traj.title.text = f'Trajectories ({selbox.left:0.4f} - {selbox.right:0.4f})'
seldf = dfs['tngdf'].loc[(dfs['tngdf'].index >= selbox.left)
& (dfs['tngdf'].index <= selbox.right)]
dfs['seldf'] = seldf
pts = ('T1x', 'T1y', 'T2x', 'T2y', 'T3x', 'T3y', 'T4x', 'T4y', 'ULx',
'ULy', 'LLx', 'LLy', 'MIx', 'MIy', 'MMx', 'MMy')
# Create a list of line segments for each tracked element.
newdata = {
pt: list(np.squeeze(np.dstack((seldf[pt].iloc[:-1], seldf[pt].iloc[1:])))) \
for pt in pts
}
newdata['color'] = np.arange(1, len(seldf))
newdata['sec'] = seldf.index[1:]
timesource.data = newdata
anim_slider.start = seldf.index[0]
anim_slider.end = seldf.index[-1]
anim_slider.step = np.diff(newdata['sec']).mean()
anim_slider.value = anim_slider.end
anim_slider.disabled = False
anim_btn.disabled = False
lastrow = seldf.iloc[-1]
lasttngtimesource.data = {
'x': [lastrow.T1x, lastrow.T2x, lastrow.T3x, lastrow.T4x],
'y': [lastrow.T1y, lastrow.T2y, lastrow.T3y, lastrow.T4y]
}
lastothtimesource.data = {
'x': [lastrow.ULx, lastrow.LLx, lastrow.MIx, lastrow.MMx],
'y': [lastrow.ULy, lastrow.LLy, lastrow.MIy, lastrow.MMy]
}
# TODO: this is a workaround until we can set x_range, y_range directly
# See https://github.com/bokeh/bokeh/issues/4014
def set_limits(xstart, xend):
'''Set axis limits.'''
ch0.x_range.start = xstart
ch0.x_range.end = xend
ch0.axis[0].bounds = (xstart, xend)
def update_select_widgets(clicked_x=None):
'''Update widgets based on current selection. Use the clicked_x param to
designate the cursor location if this function is called as the result of
a Tap event. If clicked_x is None, then use the existing cursor location
to set the center of the selection.'''
mode = selmodebtn.labels[selmodebtn.active]
if clicked_x is None and cursor.visible:
x_loc = cursor.location
elif clicked_x is not None:
x_loc = clicked_x
else:
return
if mode == '200ms':
start = x_loc - 0.100
end = x_loc + 0.100
cursor.location = x_loc
else: # 'word' or 'phone'
idx = np.abs(source.data['seconds'] - x_loc).argmin()
# TODO: clean up the redundancy
fld = {'word': 'word', 'phone': 'phone'}[mode]
label = source.data[fld][idx]
indexes = nonzero_groups(source.data[fld] == label,
include_any=idx)
secs = source.data['seconds'][indexes]
start = secs.min()
end = secs.max()
cursor.location = secs.mean()
playselbtn.start = start
playselbtn.end = end
selbox.left = start
selbox.right = end
selbox.visible = True
cursor.visible = True
def spkr_select_cb(attr, old, new):
'''Respond to changes in speaker multiselect.'''
try:
spkrs = demo[
(demo.sex.isin(sex_select.value) \
& demo.dialect_base_state.isin(state_select.value) \
& (demo.dialect_base_city.isin(city_select.value)))
].subject.unique()
new_opts = [''] + [(f.value, f.label) for f in fileoptsdf[
fileoptsdf.speaker.isin(spkrs)].itertuples()]
fselect.options = new_opts
fselect.value = ''
except NameError as e:
pass # Values not set yet, so ignore
def cursor_cb(e):
'''Handle cursor mouse click in the waveform.'''
update_select_widgets(clicked_x=e.x)
update_trace()
update_traj()
def x_range_cb(attr, old, new):
'''Handle change of x range in waveform/spectrogram.'''
if attr == 'start':
playvisbtn.start = new
elif attr == 'end':
playvisbtn.end = new
def selection_cb(e):
'''Handle data range selection event.'''
#! TODO: handle situation in which selection is too short, i.e. len(seldf) <= 1
cursor.location = (e.geometry['x0'] + e.geometry['x1']) / 2
cursor.visible = True
playselbtn.start = e.geometry['x0']
playselbtn.end = e.geometry['x1']
selbox.left = e.geometry['x0']
selbox.right = e.geometry['x1']
selbox.visible = True
update_trace()
update_traj()
def selmode_cb(attr, old, new):
'''Handle change in click selection value.'''
update_select_widgets(clicked_x=None)
def anim_cb(attr, old, new):
'''Handle change in the animation slider.'''
idx = np.argmin(np.abs(timesource.data['sec'] - new))
n = len(timesource.data['color'])
active = np.arange(n - idx, n + 1)
timesource.data['color'] = np.pad(active, (0, n - len(active)),
constant_values=0)
anim_cmap = LinearColorMapper(palette=r_Greys256,
low=1,
high=n + 1,
low_color='white')
for tag, palette in (('anim_tng', r_Reds9), ('anim_oth', r_Greens9)):
for a in find(traj.references(), {'tags': tag}):
a.line_color = linear_cmap('color',
palette,
low=1,
high=n + 1,
low_color='white')
lasttngtimesource.data = {
'x': [
timesource.data[pt][idx][1]
for pt in ('T1x', 'T2x', 'T3x', 'T4x')
],
'y': [
timesource.data[pt][idx][1]
for pt in ('T1y', 'T2y', 'T3y', 'T4y')
]
}
lastothtimesource.data = {
'x': [
timesource.data[pt][idx][1]
for pt in ('ULx', 'LLx', 'MIx', 'MMx')
],
'y': [
timesource.data[pt][idx][1]
for pt in ('ULy', 'LLy', 'MIy', 'MMy')
]
}
def anim_btn_cb():
'''Handle click of anim_btn animate trajectories of selected audio.'''
values = np.linspace(anim_slider.start, anim_slider.end,
len(timesource.data['T1x']))
for v in values:
anim_slider.value = v
def low_thresh_cb(attr, old, new):
'''Handle change in threshold slider to fade out low spectrogram values.'''
params['low_thresh_power'] = new
lt = _low_thresh()
spec0cmap.low = lt
def _low_thresh():
return sgrams[0].values.min() \
+ sgrams[0].values.std()**params['low_thresh_power']
step = None
rate = orig_rate = None
# dfs = {}
xrng = []
yrng = []
width = 1000
height = 200
cutoff = 50
order = 3
tngcolor = 'DarkRed'
othcolor = 'Indigo'
fselect = Select(options=[], value='')
fselect.on_change('value', load_wav_cb)
sex_select = MultiSelect(options=[('F', 'female'), ('M', 'male')],
value=['F', 'M'])
state_select = MultiSelect(
options=list(demo.dialect_base_state.cat.categories),
value=list(demo.dialect_base_state.cat.categories))
city_select = MultiSelect(
options=list(demo.dialect_base_city.cat.categories),
value=list(demo.dialect_base_city.cat.categories))
sex_select.on_change('value', spkr_select_cb)
state_select.on_change('value', spkr_select_cb)
city_select.on_change('value', spkr_select_cb)
spkr_select_cb('', '', '')
source = ColumnDataSource(data=dict(seconds=[], ch0=[]))
channels = ['ch0']
playvisbtn = AudioButton(label='Play visible signal',
source=source,
channels=channels,
width=120,
disabled=True)
playselbtn = AudioButton(label='Play selected signal',
source=source,
channels=channels,
width=120,
disabled=True)
selmodebtn = RadioButtonGroup(labels=['200ms', 'word', 'phone'], active=1)
selmodebtn.on_change('active', selmode_cb)
# Instantiate and share specific select/zoom tools so that
# highlighting is synchronized on all plots.
boxsel = BoxSelectTool(dimensions='width')
spboxsel = BoxSelectTool(dimensions='width')
boxzoom = BoxZoomTool(dimensions='width')
zoomin = ZoomInTool(dimensions='width')
zoomout = ZoomOutTool(dimensions='width')
crosshair = CrosshairTool(dimensions='height')
shared_tools = [
'xpan', boxzoom, boxsel, crosshair, zoomin, zoomout, 'reset'
]
uttdiv = Div(text='')
figargs = dict(tools=shared_tools, )
cursor = Span(dimension='height',
line_color='red',
line_dash='dashed',
line_width=1)
wavspec_height = 280
ch0 = figure(name='ch0',
tooltips=[('time', '$x{0.0000}'), ('word', '@word'),
('phone', '@phone')],
height=wavspec_height,
**figargs)
ch0.toolbar.logo = None
ch0.line(x='seconds', y='ch0', source=source, nonselection_line_alpha=0.6)
# Link pan, zoom events for plots with x_range.
ch0.x_range.on_change('start', x_range_cb)
ch0.x_range.on_change('end', x_range_cb)
ch0.on_event(SelectionGeometry, selection_cb)
ch0.on_event(Tap, cursor_cb)
ch0.add_layout(cursor)
wavtab = Panel(child=ch0, title='Waveform')
selbox = BoxAnnotation(name='selbox',
left=None,
right=None,
fill_color='green',
fill_alpha=0.1,
line_color='green',
line_width=1.5,
line_dash='dashed',
visible=False)
ch0.add_layout(selbox)
sgrams = [np.ones((1, 1))]
specsource = ColumnDataSource(data=dict(sgram0=[sgrams[0]]))
spec0 = figure(
name='spec0',
x_range=ch0.x_range, # Keep times synchronized
tooltips=[("time", "$x{0.0000}"), ("freq", "$y{0.0000}"),
("value", "@sgram0{0.000000}")],
height=wavspec_height,
**figargs)
spec0.toolbar.logo = None
spec0.x_range.on_change('start', x_range_cb)
spec0.x_range.on_change('end', x_range_cb)
spec0.on_event(SelectionGeometry, selection_cb)
spec0.on_event(Tap, cursor_cb)
spec0.add_layout(cursor)
spec0.x_range.range_padding = spec0.y_range.range_padding = 0
spec0cmap = LogColorMapper(palette=r_Greys256,
low_color=params['low_thresh_color'])
low_thresh_slider = Slider(start=1.0,
end=12.0,
step=0.03125,
value=params['low_thresh_power'],
title='Spectrogram threshold')
low_thresh_slider.on_change('value', low_thresh_cb)
spec0img = spec0.image(image='sgram0',
x=0,
y=0,
color_mapper=spec0cmap,
level='image',
source=specsource)
spec0.grid.grid_line_width = 0.0
spec0.add_layout(selbox)
sgramtab = Panel(child=spec0, title='Spectrogram')
tngsource = ColumnDataSource(data={'x': [], 'y': []})
othsource = ColumnDataSource(data={'x': [], 'y': []})
timesource = ColumnDataSource({
'T1x': [],
'T1y': [],
'T2x': [],
'T2y': [],
'T3x': [],
'T3y': [],
'T4x': [],
'T4y': [],
'ULx': [],
'ULy': [],
'LLx': [],
'LLy': [],
'MIx': [],
'MIy': [],
'MMx': [],
'MMy': [],
'color': [],
'sec': []
})
lasttngtimesource = ColumnDataSource(data={'x': [], 'y': []})
lastothtimesource = ColumnDataSource(data={'x': [], 'y': []})
palsource = ColumnDataSource(pd.DataFrame({'x': [], 'y': []}))
phasource = ColumnDataSource(pd.DataFrame({'x': [], 'y': []}))
trace = figure(width=500,
height=300,
title='Static trace',
x_range=(-100.0, 25.0),
y_range=(-37.650, 37.650),
tools=[],
tags=['xray', 'static_fig'])
trace.toolbar.logo = None
trace.circle('x',
'y',
source=tngsource,
size=3,
color=tngcolor,
tags=['update_xray'])
trace.circle('x',
'y',
source=othsource,
size=3,
color=othcolor,
tags=['update_xray'])
trace.line('x',
'y',
source=tngsource,
color=tngcolor,
tags=['update_xray'])
trace.line('x', 'y', source=palsource, color='black')
trace.line('x', 'y', source=phasource, color='black')
traj = figure(width=500,
height=300,
title='Trajectories',
x_range=(-100.0, 25.0),
y_range=(-37.650, 37.650),
tools=[],
tags=['xray', 'trajectory_fig'])
traj.toolbar.logo = None
traj.multi_line('T1x', 'T1y', source=timesource, tags=['anim_tng'])
traj.multi_line('T2x', 'T2y', source=timesource, tags=['anim_tng'])
traj.multi_line('T3x', 'T3y', source=timesource, tags=['anim_tng'])
traj.multi_line('T4x', 'T4y', source=timesource, tags=['anim_tng'])
traj.multi_line('ULx', 'ULy', source=timesource, tags=['anim_oth'])
traj.multi_line('LLx', 'LLy', source=timesource, tags=['anim_oth'])
traj.multi_line('MIx', 'MIy', source=timesource, tags=['anim_oth'])
traj.multi_line('MMx', 'MMy', source=timesource, tags=['anim_oth'])
traj.circle('x', 'y', source=lasttngtimesource, color=tngcolor)
traj.circle('x', 'y', source=lastothtimesource, color=othcolor)
traj.line('x', 'y', source=lasttngtimesource, color='lightgray')
traj.line('x', 'y', source=palsource, color='black')
traj.line('x', 'y', source=phasource, color='black')
anim_slider = Slider(start=0,
end=100,
step=1,
value=0,
width=240,
format='0.000f',
title='Selected trajectory',
orientation='horizontal',
disabled=True)
anim_slider.on_change('value', anim_cb)
anim_btn = Button(label='Animate', width=120, disabled=True)
anim_btn.on_click(anim_btn_cb)
audtabs = Tabs(tabs=[wavtab, sgramtab])
mainLayout = column(
row(sex_select, state_select, city_select),
row(fselect),
row(
column(uttdiv, audtabs),
column(
#! row(anim_slider, anim_btn),
column(Div(text='Click selection mode:'), selmodebtn,
low_thresh_slider),
row(playvisbtn, playselbtn))),
row(trace, traj),
name='mainLayout')
doc.add_root(mainLayout)
return doc
update_kinf()
update_actinides()
update_fission()
update_gd_istps()
#def printout():
#*****************************************************************
# Below are the widgets that appear in left column of interface
#*****************************************************************
data_table = DataTable(source = data_table_source, columns = columns, width=300, height=350)
data_table_source.on_change('selected', table_select_callback)
voids = MultiSelect(title="At what void[s]", value=["0% void"], options=sorted(void_axis_map.keys()))
voids.on_change('value', lambda attr, old, new: update_all())
actinides = MultiSelect(title="Choose Actinide[s]", value=["U-235"], options=open(join(dirname(__file__), 'widget_files/actinides.txt')).read().split())
actinides.on_change('value', lambda attr, old, new: update_actinides())
fission_prdts = MultiSelect(title="Choose Fission Product[s]", value=["Tc-99"], options=open(join(dirname(__file__), 'widget_files/fission_products.txt')).read().split())
fission_prdts.on_change('value', lambda attr, old, new: update_fission())
gd_istps = MultiSelect(title="Choose Gadolinium Isotope[s]", value=["Gd-154"], options=open(join(dirname(__file__), 'widget_files/gd_istps.txt')).read().split())
gd_istps.on_change('value', lambda attr, old, new: update_gd_istps())
rings = Select(title="Choose at what ring Gd Isotope[s] are measured", value="1", options=open(join(dirname(__file__), 'widget_files/rings.txt')).read().split())
rings.on_change('value', lambda attr, old, new: update_gd_istps())
#button = Button(label="Download", button_type="success")
#button.on_click(printout)
#*****************************************************************
middle_row = row(cap_plot, energy_plot)
bottom_row = row(cost_plot)
layout = column(title, top_row, middle_row, bottom_row)
# Set up tabs
tab1 = Panel(child=layout, title='General')
storage_dummy = PreText(text='storage summary here, incl. duration', width=600)
policy_dummy = PreText(text='policy summary here, including duals', width=600)
inputs_dummy = PreText(
text=
'inputs summary here, e.g. loads (profile charts, min, max, avg), costs',
width=600)
tab2 = Panel(child=storage_dummy, title='Storage')
tab3 = Panel(child=policy_dummy, title='Policy Targets')
tab4 = Panel(child=inputs_dummy, title='Inputs')
tabs = Tabs(tabs=[tab1, tab2, tab3, tab4]) # Put all tabs in one application
# Update Plots based on selected values
update_plots(attr="", old="", new="")
# Set up callback behavior (update plots if user changes selection)
scenario_select.on_change('value', update_plots)
period_select.on_change('value', update_plots)
stage_select.on_change('value', update_plots)
zone_select.on_change('value', update_plots)
capacity_select.on_change('value', update_plots)
# Set up curdoc
curdoc().add_root(tabs)
curdoc().title = "Dashboard"
value='Building Size [sq.ft.]',
options=sorted(axis_map.keys()))
y_axis = Select(title='Y-Axis',
value='Energy Consumption [MBTU]',
options=sorted(axis_map.keys()))
#%% Generate Plot Variables
inds = np.asarray([], dtype=int)
update_data()
p, r = create_scatter(plot_source)
px, xh = create_x_histogram(plot_source)
py, yh = create_y_histogram(plot_source)
m = create_map(map_source)
#%% Generate Layout
sizing_mode = 'fixed'
widgets = widgetbox([x_axis, y_axis], sizing_mode=sizing_mode)
layout = row(
column(row(widgets, Spacer(width=50), stats),
column(row(p, py), row(px, Spacer(width=300, height=300)))),
column(multi_select, m))
curdoc().add_root(layout)
curdoc().title = "Selection Histogram"
x_axis.on_change('value', update_x_axis)
y_axis.on_change('value', update_y_axis)
multi_select.on_change('value', update_map_selection)
plot_source.on_change('selected', update_plot_selection)
class Trends:
"""Trends layout
"""
def __init__(self, palette=Purples[3]):
self.cases = LinePlot(ARIMA_CASES_TABLE)
self.cases.render_figure()
self.cases.title("Cumulative Cases by State")
self.cases.axis_label('Date', 'Cases')
self.cases.color_palette(palette)
LOG.debug('state cases')
self.deaths = LinePlot(ARIMA_DEATHS_TABLE)
self.deaths.render_figure()
self.deaths.title("Cumulative Deaths by State")
self.deaths.axis_label('Date', 'Deaths')
self.deaths.color_palette(palette)
LOG.debug('state deaths')
self.multiselect = None
self._add_multiselect()
self.multiselect.value = ['12', '34', '36']
LOG.debug('render default states')
def _add_multiselect(self):
self.multiselect = MultiSelect(title='States:', value=['01'],
options=self.cases.options)
self.multiselect.max_width = 170
self.multiselect.min_height = 500 - 47
self.multiselect.on_change('value', self._callback_cases)
self.multiselect.on_change('value', self._callback_deaths)
def _callback_cases(self, _attr, _old, new):
for _id, _ in list(self.multiselect.options):
if self.cases.actual[_id].visible:
self.cases.actual[_id].visible = False
self.cases.predict[_id].visible = False
self.cases.lower[_id].visible = False
self.cases.upper[_id].visible = False
self.cases.area[_id].visible = False
for _id in new:
if not self.cases.actual[_id].visible:
_slice = self.cases.data.loc[_id, :]
self.cases.source[_id].data = ColumnDataSource.from_df(data=_slice)
self.cases.actual[_id].visible = True
self.cases.predict[_id].visible = True
self.cases.lower[_id].visible = True
self.cases.upper[_id].visible = True
self.cases.area[_id].visible = True
def _callback_deaths(self, _attr, _old, new):
for _id, _ in list(self.multiselect.options):
if self.deaths.actual[_id].visible:
self.deaths.actual[_id].visible = False
self.deaths.predict[_id].visible = False
self.deaths.lower[_id].visible = False
self.deaths.upper[_id].visible = False
self.deaths.area[_id].visible = False
for _id in new:
if not self.deaths.actual[_id].visible:
_slice = self.deaths.data.loc[_id, :]
self.deaths.source[_id].data = ColumnDataSource.from_df(data=_slice)
self.deaths.actual[_id].visible = True
self.deaths.predict[_id].visible = True
self.deaths.lower[_id].visible = True
self.deaths.upper[_id].visible = True
self.deaths.area[_id].visible = True
def layout(self):
"""Build trend layout
Returns:
Bokeh Layout -- layout with cases, deaths and state selection
"""
_graphs = gridplot([self.cases.plot, self.deaths.plot], ncols=1,
plot_width=800 - self.multiselect.max_width,
plot_height=250, toolbar_location=None)
_layout = row(_graphs, self.multiselect)
return _layout
import panel as pn
from bokeh.models import MultiSelect, CustomJS
from pcp.models.multiselect import PCPMultiSelect
from bokeh.sampledata.airports import data
options = [(str(i), str(n)) for i, n in zip(data["name"].index, data["name"])]
m1 = PCPMultiSelect(options=options, height=80, width=400)
m2 = MultiSelect(options=options)
t = pn.widgets.Toggle(name="search")
t.jslink(m1, value="searchbox")
# m1.on_change("value", lambda attr, old, new: setattr(m2,"value", new))
cb1 = CustomJS(args=dict(m2=m2), code="m2.value = cb_obj.value")
m1.js_on_change("value", cb1)
m1.on_change("value", lambda attr, old, new: print("m1", old, new))
cb2 = CustomJS(args=dict(m1=m1), code="m1.value = cb_obj.value")
# m2.on_change("value", lambda attr, old, new: setattr(m1,"value", new))
m2.js_on_change("value", cb2)
m2.on_change("value", lambda attr, old, new: print("m2", old, new))
pn.Row(m1, m2, t).show()
def bkapp(doc):
### Functions ###
# functions for user dialogs
def open_file(ftype):
root = Tk()
root.withdraw()
file = askopenfilename(filetypes=ftype,
title='Open File',
initialdir=os.getcwd())
root.destroy()
return file
def choose_directory():
root = Tk()
root.withdraw()
out_dir = askdirectory()
root.destroy()
return out_dir
def write_output_directory(output_dir):
root = Tk()
root.withdraw()
makeDir = askquestion('Make Directory','Output directory not set. Make directory: '
+output_dir+'? If not, you\'ll be prompted to change directories.',icon = 'warning')
root.destroy()
return makeDir
def overwrite_file():
root = Tk()
root.withdraw()
overwrite = askquestion('Overwrite File','File already exits. Do you want to overwrite?',icon = 'warning')
root.destroy()
return overwrite
def update_filename():
filetype = [("Video files", "*.mp4")]
fname = open_file(filetype)
if fname:
#print('Successfully loaded file: '+fname)
load_data(filename=fname)
def change_directory():
out_dir = choose_directory()
if out_dir:
source.data["output_dir"] = [out_dir]
outDir.text = out_dir
return out_dir
# load data from file
def load_data(filename):
vidcap = cv2.VideoCapture(filename)
success,frame = vidcap.read()
img_tmp,_,__ = cv2.split(frame)
h,w = np.shape(img_tmp)
img = np.flipud(img_tmp)
radio_button_gp.active = 0
fname = os.path.split(filename)[1]
input_dir = os.path.split(filename)[0]
if source.data['output_dir'][0]=='':
output_dir = os.path.join(input_dir,fname.split('.')[0])
else:
output_dir = source.data['output_dir'][0]
if not os.path.isdir(output_dir):
makeDir = write_output_directory(output_dir)
if makeDir=='yes':
os.mkdir(output_dir)
else:
output_dir = change_directory()
if output_dir:
source.data = dict(image_orig=[img], image=[img], bin_img=[0],
x=[0], y=[0], dw=[w], dh=[h], num_contours=[0], roi_coords=[0],
img_name=[fname], input_dir=[input_dir], output_dir=[output_dir])
curr_img = p.select_one({'name':'image'})
if curr_img:
p.renderers.remove(curr_img)
p.image(source=source, image='image', x='x', y='y', dw='dw', dh='dh', color_mapper=cmap,level='image',name='image')
p.plot_height=int(h/2)
p.plot_width=int(w/2)
#p.add_tools(HoverTool(tooltips=IMG_TOOLTIPS))
inFile.text = fname
outDir.text = output_dir
else:
print('Cancelled. To continue please set output directory.{:<100}'.format(' '),end="\r")
# resetting sources for new data or new filters/contours
def remove_plot():
source.data["num_contours"]=[0]
contours_found.text = 'Droplets Detected: 0'
source_contours.data = dict(xs=[], ys=[])
source_label.data = dict(x=[], y=[], label=[])
# apply threshold filter and display binary image
def apply_filter():
if source.data['input_dir'][0] == '':
print('No image loaded! Load image first.{:<100}'.format(' '),end="\r")
else:
img = np.squeeze(source.data['image_orig'])
# remove contours if present
if source.data["num_contours"]!=[0]:
remove_plot()
if radio_button_gp.active == 1:
thresh = filters.threshold_otsu(img)
binary = img > thresh
bin_img = binary*255
source.data["bin_img"] = [bin_img]
elif radio_button_gp.active == 2:
thresh = filters.threshold_isodata(img)
binary = img > thresh
bin_img = binary*255
source.data["bin_img"] = [bin_img]
elif radio_button_gp.active == 3:
thresh = filters.threshold_mean(img)
binary = img > thresh
bin_img = binary*255
source.data["bin_img"] = [bin_img]
elif radio_button_gp.active == 4:
thresh = filters.threshold_li(img)
binary = img > thresh
bin_img = binary*255
source.data["bin_img"] = [bin_img]
elif radio_button_gp.active == 5:
thresh = filters.threshold_yen(img)
binary = img > thresh
bin_img = binary*255
source.data["bin_img"] = [bin_img]
elif radio_button_gp.active == 6:
off = offset_spinner.value
block_size = block_spinner.value
thresh = filters.threshold_local(img,block_size,offset=off)
binary = img > thresh
bin_img = binary*255
source.data["bin_img"] = [bin_img]
else:
bin_img = img
source.data['image'] = [bin_img]
# image functions for adjusting the binary image
def close_img():
if source.data["num_contours"]!=[0]:
remove_plot()
if radio_button_gp.active == 0:
print("Please Select Filter for Threshold{:<100}".format(' '),end="\r")
else:
source.data["image"] = source.data["bin_img"]
img = np.squeeze(source.data['bin_img'])
closed_img = binary_closing(255-img)*255
source.data['image'] = [255-closed_img]
source.data['bin_img'] = [255-closed_img]
def dilate_img():
if source.data["num_contours"]!=[0]:
remove_plot()
if radio_button_gp.active == 0:
print("Please Select Filter for Threshold{:<100}".format(' '),end="\r")
else:
img = np.squeeze(source.data['bin_img'])
dilated_img = binary_dilation(255-img)*255
source.data['image'] = [255-dilated_img]
source.data['bin_img'] = [255-dilated_img]
def erode_img():
if source.data["num_contours"]!=[0]:
remove_plot()
if radio_button_gp.active == 0:
print("Please Select Filter for Threshold{:<100}".format(' '),end="\r")
else:
img = np.squeeze(source.data['bin_img'])
eroded_img = binary_erosion(255-img)*255
source.data['image'] = [255-eroded_img]
source.data['bin_img'] = [255-eroded_img]
# the function for identifying closed contours in the image
def find_contours(level=0.8):
min_drop_size = contour_rng_slider.value[0]
max_drop_size = contour_rng_slider.value[1]
min_dim = 20
max_dim = 200
if radio_button_gp.active == 0:
print("Please Select Filter for Threshold{:<100}".format(' '),end="\r")
elif source.data['input_dir'][0] == '':
print('No image loaded! Load image first.{:<100}'.format(' '),end="\r")
else:
img = np.squeeze(source.data['bin_img'])
h,w = np.shape(img)
contours = measure.find_contours(img, level)
length_cnt_x = [cnt[:,1] for cnt in contours if np.shape(cnt)[0] < max_drop_size
and np.shape(cnt)[0] > min_drop_size]
length_cnt_y = [cnt[:,0] for cnt in contours if np.shape(cnt)[0] < max_drop_size
and np.shape(cnt)[0] > min_drop_size]
matched_cnt_x = []
matched_cnt_y = []
roi_coords = []
label_text = []
label_y = np.array([])
count=0
for i in range(len(length_cnt_x)):
cnt_x = length_cnt_x[i]
cnt_y = length_cnt_y[i]
width = np.max(cnt_x)-np.min(cnt_x)
height = np.max(cnt_y)-np.min(cnt_y)
if width>min_dim and width<max_dim and height>min_dim and height<max_dim:
matched_cnt_x.append(cnt_x)
matched_cnt_y.append(cnt_y)
roi_coords.append([round(width),round(height),round(np.min(cnt_x)),round(h-np.max(cnt_y))])
label_text.append(str(int(count)+1))
label_y = np.append(label_y,np.max(cnt_y))
count+=1
curr_contours = p.select_one({'name':'overlay'})
if curr_contours:
p.renderers.remove(curr_contours)
#if source.data["num_contours"]==[0]:
#remove_plot()
#p.image(source=source, image='image_orig', x=0, y=0, dw=w, dh=h, color_mapper=cmap, name='overlay',level='underlay')
source.data["image"] = source.data["image_orig"]
source.data["num_contours"] = [count]
#source.data["cnt_x"] = [matched_cnt_x]
#source.data["cnt_y"] = [matched_cnt_y]
source.data["roi_coords"] = [roi_coords]
source_contours.data = dict(xs=matched_cnt_x, ys=matched_cnt_y)
p.multi_line(xs='xs',ys='ys',source=source_contours, color=(255,127,14),line_width=2, name="contours",level='glyph')
if len(np.array(roi_coords).shape)<2:
if len(np.array(roi_coords)) <4:
print('No contours found. Try adjusting parameters or filter for thresholding.{:<100}'.format(' '),end="\r")
source_label.data = dict(x=[],y=[],label=[])
else:
source_label.data = dict(x=np.array(roi_coords)[2], y=label_y, label=label_text)
else:
source_label.data = dict(x=np.array(roi_coords)[:,2], y=label_y, label=label_text)
contours_found.text = 'Droplets Detected: '+str(len(matched_cnt_x))
# write the contours and parameters to files
def export_ROIs():
if source.data["num_contours"]==[0]:
print("No Contours Found! Find contours first.{:<100}".format(' '),end="\r")
else:
hdr = 'threshold filter,contour min,contour max'
thresh_filter = radio_button_gp.active
cnt_min = contour_rng_slider.value[0]
cnt_max = contour_rng_slider.value[1]
params = [thresh_filter,cnt_min,cnt_max]
if radio_button_gp.active == 6:
off = offset_spinner.value
block_size = block_spinner.value
hdr = hdr + ',local offset,local block size'
params.append(off,block_size)
params_fname = 'ContourParams.csv'
params_out = os.path.join(source.data['output_dir'][0],params_fname)
overwrite = 'no'
if os.path.exists(params_out):
overwrite = overwrite_file()
if overwrite == 'yes' or not os.path.exists(params_out):
np.savetxt(params_out,np.array([params]),delimiter=',',header=hdr,comments='')
roi_coords = np.array(source.data["roi_coords"][0])
out_fname = 'ROI_coords.csv'
out_fullpath = os.path.join(source.data['output_dir'][0],out_fname)
if overwrite == 'yes' or not os.path.exists(out_fullpath):
hdr = 'width,height,x,y'
np.savetxt(out_fullpath,roi_coords,delimiter=',',header=hdr,comments='')
print('Successfully saved ROIs coordinates as: '+out_fullpath,end='\r')
source.data['roi_coords'] = [roi_coords]
# function for loading previously made files or error handling for going out of order
def check_ROI_files():
coords_file = os.path.join(source.data["output_dir"][0],'ROI_coords.csv')
n_cnt_curr = source.data["num_contours"][0]
roi_coords_curr = source.data["roi_coords"][0]
if os.path.exists(coords_file):
df_tmp=pd.read_csv(coords_file, sep=',')
roi_coords = np.array(df_tmp.values)
n_cnt = len(roi_coords)
if n_cnt_curr != n_cnt or not np.array_equal(roi_coords_curr,roi_coords):
print('Current ROIs are different from saved ROI file! ROIs from saved file will be used instead and plot updated.',end="\r")
source.data["num_contours"] = [n_cnt]
source.data["roi_coords"] = [roi_coords]
params_file = os.path.join(source.data['output_dir'][0],'ContourParams.csv')
params_df = pd.read_csv(params_file)
thresh_ind = params_df["threshold filter"].values[0]
radio_button_gp.active = int(thresh_ind)
if thresh_ind == 6:
offset_spinner.value = int(params_df["local offset"].values[0])
block_spinner.value = int(params_df["local block size"].values[0])
contour_rng_slider.value = tuple([int(params_df["contour min"].values[0]),int(params_df["contour max"].values[0])])
find_contours()
else:
print("ROI files not found! Check save directory or export ROIs.{:<100}".format(' '),end="\r")
# use FFMPEG to crop out regions from original mp4 and save to file
def create_ROI_movies():
if source.data['input_dir'][0] == '':
print('No image loaded! Load image first.{:<100}'.format(' '),end="\r")
else:
check_ROI_files()
side = 100 # for square ROIs, replace first two crop parameters with side & uncomment
padding = 20
roi_coords_file = os.path.join(source.data['output_dir'][0],'ROI_coords.csv')
if source.data["num_contours"]==[0]:
print("No contours found! Find contours first.{:<100}".format(' '),end="\r")
elif not os.path.exists(roi_coords_file):
print("ROI file does not exist! Check save directory or export ROIs.{:<100}".format(' '),end="\r")
else:
print('Creating Movies...{:<100}'.format(' '),end="\r")
pbar = tqdm(total=source.data["num_contours"][0])
for i in range(source.data["num_contours"][0]):
roi_coords = np.array(source.data["roi_coords"][0])
inPath = os.path.join(source.data['input_dir'][0],source.data['img_name'][0])
#out_fname = source.data['filename'][0].split('.')[0] +'_ROI'+str(i+1)+'.mp4'
out_fname = 'ROI'+str(i+1)+'.mp4'
outPath = os.path.join(source.data['output_dir'][0],out_fname)
#command = f"ffmpeg -i \'{(inPath)}\' -vf \"crop={(roi_coords[i,0]+padding*2)}:{(roi_coords[i,1]+padding*2)}:{(roi_coords[i,2]-padding)}:{(roi_coords[i,3]+padding)}\" -y \'{(outPath)}\'"
command = f"ffmpeg -i \'{(inPath)}\' -vf \"crop={side}:{side}:{(roi_coords[i,2]-padding)}:{(roi_coords[i,3]-padding)}\" -y \'{(outPath)}\'"
overwrite = 'no'
if os.path.exists(outPath):
overwrite = overwrite_file()
if overwrite == 'yes' or not os.path.exists(outPath):
saved = subprocess.check_call(command,shell=True)
if saved != 0:
print('An error occurred while creating movies! Check terminal window.{:<100}'.format(' '),end="\r")
pbar.update()
# change the display range on images from slider values
def update_image():
cmap.low = display_range_slider.value[0]
cmap.high = display_range_slider.value[1]
# create statistics files for each mp4 region specific file
def process_ROIs():
if source.data['input_dir'][0] == '':
print('No image loaded! Load image first.{:<100}'.format(' '),end="\r")
else:
check_ROI_files()
hdr = 'roi,time,area,mean,variance,min,max,median,skewness,kurtosis,rawDensity,COMx,COMy'
cols = hdr.split(',')
all_stats = np.zeros((0,13))
n_cnt = source.data["num_contours"][0]
if n_cnt == 0:
print("No contours found! Find contours first.{:<100}".format(' '),end="\r")
for i in range(n_cnt):
#in_fname = source.data['filename'][0].split('.')[0] +'_ROI'+str(i+1)+'.mp4'
in_fname = 'ROI'+str(i+1)+'.mp4'
inPath = os.path.join(source.data['output_dir'][0],in_fname)
#out_fname = source.data['filename'][0].split('.')[0] +'_ROI'+str(i+1)+'_stats.csv'
out_fname = 'stats_ROI'+str(i+1)+'.csv'
outPath = os.path.join(source.data['output_dir'][0],out_fname)
if not os.path.exists(inPath):
print('ROI movie not found! Create ROI movie first.{:<100}'.format(' '),end="\r")
break
vidcap = cv2.VideoCapture(inPath)
last_frame = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
if i==0:
pbar = tqdm(total=last_frame*n_cnt)
success,frame = vidcap.read()
img_tmp,_,__ = cv2.split(frame)
h,w = np.shape(img_tmp)
img = np.zeros((last_frame,h,w))
img_stats = np.zeros((last_frame,13))
stats = describe(img_tmp,axis=None)
median = np.median(img_tmp)
density = np.sum(img_tmp)
cx, cy = center_of_mass(img_tmp)
img_stats[0,0:13] = [i,0,stats.nobs,stats.mean,stats.variance,
stats.minmax[0],stats.minmax[1],median,stats.skewness,
stats.kurtosis,density,cx,cy]
img[0,:,:] = np.flipud(img_tmp)
pbar.update()
overwrite = 'no'
if os.path.exists(outPath):
overwrite = overwrite_file()
if overwrite=='no':
pbar.update(last_frame-1)
if overwrite == 'yes' or not os.path.exists(outPath):
for j in range(1,last_frame):
vidcap.set(1, j)
success,frame = vidcap.read()
img_tmp,_,__ = cv2.split(frame)
stats = describe(img_tmp,axis=None)
t = j*5/60
density = np.sum(img_tmp)
cx, cy = center_of_mass(img_tmp)
median = np.median(img_tmp)
img_stats[j,0:13] = [i,t,stats.nobs,stats.mean,stats.variance,
stats.minmax[0],stats.minmax[1],median,stats.skewness,
stats.kurtosis,density,cx,cy]
img[j,:,:] = np.flipud(img_tmp)
pbar.update(1)
all_stats = np.append(all_stats,img_stats,axis=0)
np.savetxt(outPath,img_stats,delimiter=',',header=hdr,comments='')
if i==(n_cnt-1):
df = pd.DataFrame(all_stats,columns=cols)
group = df.groupby('roi')
for i in range(len(group)):
sources_stats[i] = ColumnDataSource(group.get_group(i))
# load statistics CSVs and first ROI mp4 files and display in plots
def load_ROI_files():
if source.data['input_dir'][0] == '':
print('No image loaded! Load image first.{:<100}'.format(' '),end="\r")
else:
check_ROI_files()
n_cnt = source.data["num_contours"][0]
basepath = os.path.join(source.data["output_dir"][0],'stats')
all_files = [basepath+'_ROI'+str(i+1)+'.csv' for i in range(n_cnt)]
files_exist = [os.path.exists(f) for f in all_files]
if all(files_exist) and n_cnt != 0:
df = pd.concat((pd.read_csv(f) for f in all_files), ignore_index=True)
group = df.groupby('roi')
OPTIONS = []
LABELS = []
pbar = tqdm(total=len(stats)*len(group))
j=0
colors_ordered = list(Category20[20])
idx_reorder = np.append(np.linspace(0,18,10),np.linspace(1,19,10))
idx = idx_reorder.astype(int)
colors = [colors_ordered[i] for i in idx]
for roi, df_roi in group:
sources_stats[roi] = ColumnDataSource(df_roi)
OPTIONS.append([str(int(roi)+1),'ROI '+(str(int(roi)+1))])
LABELS.append('ROI '+str(int(roi)+1))
color = colors[j]
j+=1
if j>=20:
j=0
for i in range(3,len(df.columns)):
name = 'ROI '+str(int(roi)+1)
plot_check = p_stats[i-3].select_one({'name':name})
if not plot_check:
p_stats[i-3].line(x='time',y=str(df.columns[i]),source=sources_stats[roi],
name=name,visible=False,line_color=color)
p_stats[i-3].xaxis.axis_label = "Time [h]"
p_stats[i-3].yaxis.axis_label = str(df.columns[i])
p_stats[i-3].add_tools(HoverTool(tooltips=TOOLTIPS))
p_stats[i-3].toolbar_location = "left"
pbar.update(1)
ROI_multi_select.options = OPTIONS
ROI_multi_select.value = ["1"]
ROI_movie_radio_group.labels = LABELS
ROI_movie_radio_group.active = 0
else:
print('Not enough files! Check save directory or calculate new stats.{:<100}'.format(' '),end="\r")
# show/hide curves from selected/deselected labels for ROIs in statistics plots
def update_ROI_plots():
n_cnt = source.data["num_contours"][0]
pbar = tqdm(total=len(stats)*n_cnt)
for j in range(n_cnt):
for i in range(len(stats)):
name = 'ROI '+str(int(j)+1)
glyph = p_stats[i].select_one({'name': name})
if str(j+1) in ROI_multi_select.value:
glyph.visible = True
else:
glyph.visible = False
pbar.update(1)
# load and display the selected ROI's mp4
def load_ROI_movie():
idx = ROI_movie_radio_group.active
in_fname = 'ROI'+str(idx+1)+'.mp4'
inPath = os.path.join(source.data['output_dir'][0],in_fname)
if not os.path.exists(inPath):
print('ROI movie not found! Check save directory or create ROI movie.',end="\r")
else:
old_plot = p_ROI.select_one({'name': sourceROI.data['img_name'][0]})
if old_plot:
p_ROI.renderers.remove(old_plot)
vidcap = cv2.VideoCapture(inPath)
last_frame = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
ROI_movie_slider.end = (last_frame-1)*5/60
ROI_movie_slider.value = 0
vidcap.set(1, 0)
success,frame = vidcap.read()
img_tmp,_,__ = cv2.split(frame)
h,w = np.shape(img_tmp)
img = np.flipud(img_tmp)
name = 'ROI'+str(idx+1)
sourceROI.data = dict(image=[img],x=[0], y=[0], dw=[w], dh=[h],
img_name=[name])
p_ROI.image(source=sourceROI, image='image', x='x', y='y',
dw='dw', dh='dh', color_mapper=cmap, name='img_name')
# change the displayed frame from slider movement
def update_ROI_movie():
frame_idx = round(ROI_movie_slider.value*60/5)
in_fname = sourceROI.data['img_name'][0]+'.mp4'
inPath = os.path.join(source.data['output_dir'][0],in_fname)
vidcap = cv2.VideoCapture(inPath)
vidcap.set(1, frame_idx)
success,frame = vidcap.read()
img_tmp,_,__ = cv2.split(frame)
img = np.flipud(img_tmp)
sourceROI.data['image'] = [img]
# the following 2 functions are used to animate the mp4
def update_ROI_slider():
time = ROI_movie_slider.value + 5/60
end = ROI_movie_slider.end
if time > end:
animate_ROI_movie()
else:
ROI_movie_slider.value = time
return callback_id
def animate_ROI_movie():
global callback_id
if ROI_movie_play_button.label == '► Play':
ROI_movie_play_button.label = '❚❚ Pause'
callback_id = curdoc().add_periodic_callback(update_ROI_slider, 10)
else:
ROI_movie_play_button.label = '► Play'
curdoc().remove_periodic_callback(callback_id)
return callback_id
### Application Content ###
# main plot for segmentation and contour finding
cmap = LinearColorMapper(palette="Greys256", low=0, high=255)
TOOLS = "pan,wheel_zoom,box_zoom,reset,save,box_select,lasso_select"
IMG_TOOLTIPS = [('name', "@img_name"),("x", "$x"),("y", "$y"),("value", "@image")]
source = ColumnDataSource(data=dict(image=[0],bin_img=[0],image_orig=[0],
x=[0], y=[0], dw=[0], dh=[0], num_contours=[0], roi_coords=[0],
input_dir=[''],output_dir=[''],img_name=['']))
source_label = ColumnDataSource(data=dict(x=[0], y=[0], label=['']))
source_contours = ColumnDataSource(data=dict(xs=[0], ys=[0]))
roi_labels = LabelSet(x='x', y='y', text='label',source=source_label,
level='annotation',text_color='white',text_font_size='12pt')
# create a new plot and add a renderer
p = figure(tools=TOOLS, toolbar_location=("right"))
p.add_layout(roi_labels)
p.x_range.range_padding = p.y_range.range_padding = 0
# turn off gridlines
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None
p.axis.visible = False
# ROI plots
sourceROI = ColumnDataSource(data=dict(image=[0],
x=[0], y=[0], dw=[0], dh=[0], img_name=[0]))
sources_stats = {}
TOOLTIPS = [('name','$name'),('time', '@time'),('stat', "$y")]
stats = np.array(['mean','var','min','max','median','skew','kurt','rawDensity','COMx','COMy'])
p_stats = []
tabs = []
for i in range(len(stats)):
p_stats.append(figure(tools=TOOLS, plot_height=300, plot_width=600))
p_stats[i].x_range.range_padding = p_stats[i].y_range.range_padding = 0
tabs.append(Panel(child=p_stats[i], title=stats[i]))
# create a new plot and add a renderer
p_ROI = figure(tools=TOOLS, toolbar_location=("right"), plot_height=300, plot_width=300)
p_ROI.x_range.range_padding = p_ROI.y_range.range_padding = 0
# turn off gridlines
p_ROI.xgrid.grid_line_color = p_ROI.ygrid.grid_line_color = None
p_ROI.axis.visible = False
# Widgets - Buttons, Sliders, Text, Etc.
intro = Div(text="""<h2>Droplet Recognition and Analysis with Bokeh</h2>
This application is designed to help segment a grayscale image into
regions of interest (ROIs) and perform analysis on those regions.<br>
<h4>How to Use This Application:</h4>
<ol>
<li>Load in a grayscale mp4 file and choose a save directory.</li>
<li>Apply various filters for thresholding. Use <b>Close</b>, <b>Dilate</b>
and <b>Erode</b> buttons to adjust each binary image further.</li>
<li>Use <b>Find Contours</b> button to search the image for closed shape.
The <b>Contour Size Range</b> slider will change size of the perimeter to
be identified. You can apply new thresholds and repeat until satisfied with
the region selection. Total regions detected is displayed next to
the button.</li>
<li>When satisfied, use <b>Export ROIs</b> to write ROI locations and
contour finding parameters to file.</li>
<li><b>Create ROI Movies</b> to write mp4s of the selected regions.</li>
<li>Use <b>Calculate ROI Stats</b> to perform calculations on the
newly created mp4 files.</li>
<li>Finally, use <b>Load ROI Files</b> to load in the data that you just
created and view the plots. The statistics plots can be overlaid by
selecting multiple labels. Individual ROI mp4s can be animated or you can
use the slider to move through the frames.</li>
</ol>
Note: messages and progress bars are displayed below the GUI.""",
style={'font-size':'10pt'},width=1000)
file_button = Button(label="Choose File",button_type="primary")
file_button.on_click(update_filename)
inFile = PreText(text='Input File:\n'+source.data["img_name"][0], background=(255,255,255,0.5), width=500)
filter_LABELS = ["Original","OTSU", "Isodata", "Mean", "Li","Yen","Local"]
radio_button_gp = RadioButtonGroup(labels=filter_LABELS, active=0, width=600)
radio_button_gp.on_change('active', lambda attr, old, new: apply_filter())
offset_spinner = Spinner(low=0, high=500, value=1, step=1, width=100, title="Local: Offset",
background=(255,255,255,0.5))
offset_spinner.on_change('value', lambda attr, old, new: apply_filter())
block_spinner = Spinner(low=1, high=101, value=25, step=2, width=100, title="Local: Block Size",
background=(255,255,255,0.5))
block_spinner.on_change('value', lambda attr, old, new: apply_filter())
closing_button = Button(label="Close",button_type="default", width=100)
closing_button.on_click(close_img)
dilation_button = Button(label="Dilate",button_type="default", width=100)
dilation_button.on_click(dilate_img)
erosion_button = Button(label="Erode",button_type="default", width=100)
erosion_button.on_click(erode_img)
contour_rng_slider = RangeSlider(start=10, end=500, value=(200,350), step=1, width=300,
title="Contour Size Range", background=(255,255,255,0.5), bar_color='gray')
contour_button = Button(label="Find Contours", button_type="success")
contour_button.on_click(find_contours)
contours_found = PreText(text='Droplets Detected: '+str(source.data["num_contours"][0]), background=(255,255,255,0.5))
exportROIs_button = Button(label="Export ROIs", button_type="success", width=200)
exportROIs_button.on_click(export_ROIs)
changeDir_button = Button(label="Change Directory",button_type="primary", width=150)
changeDir_button.on_click(change_directory)
outDir = PreText(text='Save Directory:\n'+source.data["output_dir"][0], background=(255,255,255,0.5), width=500)
create_ROIs_button = Button(label="Create ROI Movies",button_type="success", width=200)
create_ROIs_button.on_click(create_ROI_movies)
process_ROIs_button = Button(label="Calculate ROI Stats",button_type="success")
process_ROIs_button.on_click(process_ROIs)
display_rng_text = figure(title="Display Range", title_location="left",
width=40, height=300, toolbar_location=None, min_border=0,
outline_line_color=None)
display_rng_text.title.align="center"
display_rng_text.title.text_font_size = '10pt'
display_rng_text.x_range.range_padding = display_rng_text.y_range.range_padding = 0
display_range_slider = RangeSlider(start=0, end=255, value=(0,255), step=1,
orientation='vertical', direction='rtl',
bar_color='gray', width=40, height=300, tooltips=True)
display_range_slider.on_change('value', lambda attr, old, new: update_image())
load_ROIfiles_button = Button(label="Load ROI Files",button_type="primary")
load_ROIfiles_button.on_click(load_ROI_files)
ROI_multi_select = MultiSelect(value=[], width=100, height=300)
ROI_multi_select.on_change('value', lambda attr, old, new: update_ROI_plots())
ROI_movie_radio_group = RadioGroup(labels=[],width=60)
ROI_movie_radio_group.on_change('active', lambda attr, old, new: load_ROI_movie())
ROI_movie_slider = Slider(start=0,end=100,value=0,step=5/60,title="Time [h]", width=280)
ROI_movie_slider.on_change('value', lambda attr, old, new: update_ROI_movie())
callback_id = None
ROI_movie_play_button = Button(label='► Play',width=50)
ROI_movie_play_button.on_click(animate_ROI_movie)
# initialize some data without having to choose file
# fname = os.path.join(os.getcwd(),'data','Droplets.mp4')
# load_data(filename=fname)
### Layout & Initialize application ###
ROI_layout = layout([
[ROI_movie_radio_group, p_ROI],
[ROI_movie_slider,ROI_movie_play_button]
])
app = layout(children=[
[intro],
[file_button,inFile],
[radio_button_gp, offset_spinner, block_spinner],
[closing_button, dilation_button, erosion_button],
[contour_rng_slider, contour_button, contours_found],
[exportROIs_button, outDir, changeDir_button],
[create_ROIs_button, process_ROIs_button],
[display_rng_text, display_range_slider, p],
[load_ROIfiles_button],
[ROI_layout, ROI_multi_select, Tabs(tabs=tabs)]
])
doc.add_root(app)
line_width=2,
source=source_plot_ssscc,
)
plot_all.scatter(
"x",
"y",
fill_color="#999999",
line_color="#000000",
size=10,
line_width=2,
source=source_plot_all,
)
parameter.on_change("value", lambda attr, old, new: update())
station.on_change("value", lambda attr, old, new: update())
flag_list.on_change("value", lambda attr, old, new: update())
def update():
print("exec update()")
table_rows = df_edited["SSSCC_rinko"] == int(station.value)
plot_rows = (df_edited["SSSCC_rinko"] == int(
station.value)) & (df_edited["Flag New"].isin(flag_list.value))
# breakpoint()
current_table = df_edited[table_rows].reset_index()
current_plot = df_edited[plot_rows]
# can this be split off into separate updates? might improve speed
def create():
doc = curdoc()
det_data = {}
cami_meta = {}
def proposal_textinput_callback(_attr, _old, new):
nonlocal cami_meta
proposal = new.strip()
for zebra_proposals_path in pyzebra.ZEBRA_PROPOSALS_PATHS:
proposal_path = os.path.join(zebra_proposals_path, proposal)
if os.path.isdir(proposal_path):
# found it
break
else:
raise ValueError(f"Can not find data for proposal '{proposal}'.")
file_list = []
for file in os.listdir(proposal_path):
if file.endswith(".hdf"):
file_list.append((os.path.join(proposal_path, file), file))
file_select.options = file_list
cami_meta = {}
proposal_textinput = TextInput(title="Proposal number:", width=210)
proposal_textinput.on_change("value", proposal_textinput_callback)
def upload_button_callback(_attr, _old, new):
nonlocal cami_meta
with io.StringIO(base64.b64decode(new).decode()) as file:
cami_meta = pyzebra.parse_h5meta(file)
file_list = cami_meta["filelist"]
file_select.options = [(entry, os.path.basename(entry))
for entry in file_list]
upload_div = Div(text="or upload .cami file:", margin=(5, 5, 0, 5))
upload_button = FileInput(accept=".cami", width=200)
upload_button.on_change("value", upload_button_callback)
def update_image(index=None):
if index is None:
index = index_spinner.value
current_image = det_data["data"][index]
proj_v_line_source.data.update(x=np.arange(0, IMAGE_W) + 0.5,
y=np.mean(current_image, axis=0))
proj_h_line_source.data.update(x=np.mean(current_image, axis=1),
y=np.arange(0, IMAGE_H) + 0.5)
image_source.data.update(
h=[np.zeros((1, 1))],
k=[np.zeros((1, 1))],
l=[np.zeros((1, 1))],
)
image_source.data.update(image=[current_image])
if main_auto_checkbox.active:
im_min = np.min(current_image)
im_max = np.max(current_image)
display_min_spinner.value = im_min
display_max_spinner.value = im_max
image_glyph.color_mapper.low = im_min
image_glyph.color_mapper.high = im_max
if "mf" in det_data:
metadata_table_source.data.update(mf=[det_data["mf"][index]])
else:
metadata_table_source.data.update(mf=[None])
if "temp" in det_data:
metadata_table_source.data.update(temp=[det_data["temp"][index]])
else:
metadata_table_source.data.update(temp=[None])
gamma, nu = calculate_pol(det_data, index)
omega = np.ones((IMAGE_H, IMAGE_W)) * det_data["omega"][index]
image_source.data.update(gamma=[gamma], nu=[nu], omega=[omega])
def update_overview_plot():
h5_data = det_data["data"]
n_im, n_y, n_x = h5_data.shape
overview_x = np.mean(h5_data, axis=1)
overview_y = np.mean(h5_data, axis=2)
overview_plot_x_image_source.data.update(image=[overview_x],
dw=[n_x],
dh=[n_im])
overview_plot_y_image_source.data.update(image=[overview_y],
dw=[n_y],
dh=[n_im])
if proj_auto_checkbox.active:
im_min = min(np.min(overview_x), np.min(overview_y))
im_max = max(np.max(overview_x), np.max(overview_y))
proj_display_min_spinner.value = im_min
proj_display_max_spinner.value = im_max
overview_plot_x_image_glyph.color_mapper.low = im_min
overview_plot_y_image_glyph.color_mapper.low = im_min
overview_plot_x_image_glyph.color_mapper.high = im_max
overview_plot_y_image_glyph.color_mapper.high = im_max
frame_range.start = 0
frame_range.end = n_im
frame_range.reset_start = 0
frame_range.reset_end = n_im
frame_range.bounds = (0, n_im)
scan_motor = det_data["scan_motor"]
overview_plot_y.axis[1].axis_label = f"Scanning motor, {scan_motor}"
var = det_data[scan_motor]
var_start = var[0]
var_end = var[-1] + (var[-1] - var[0]) / (n_im - 1)
scanning_motor_range.start = var_start
scanning_motor_range.end = var_end
scanning_motor_range.reset_start = var_start
scanning_motor_range.reset_end = var_end
# handle both, ascending and descending sequences
scanning_motor_range.bounds = (min(var_start,
var_end), max(var_start, var_end))
def file_select_callback(_attr, old, new):
nonlocal det_data
if not new:
# skip empty selections
return
# Avoid selection of multiple indicies (via Shift+Click or Ctrl+Click)
if len(new) > 1:
# drop selection to the previous one
file_select.value = old
return
if len(old) > 1:
# skip unnecessary update caused by selection drop
return
det_data = pyzebra.read_detector_data(new[0])
if cami_meta and "crystal" in cami_meta:
det_data["ub"] = cami_meta["crystal"]["UB"]
index_spinner.value = 0
index_spinner.high = det_data["data"].shape[0] - 1
index_slider.end = det_data["data"].shape[0] - 1
zebra_mode = det_data["zebra_mode"]
if zebra_mode == "nb":
metadata_table_source.data.update(geom=["normal beam"])
else: # zebra_mode == "bi"
metadata_table_source.data.update(geom=["bisecting"])
update_image(0)
update_overview_plot()
file_select = MultiSelect(title="Available .hdf files:",
width=210,
height=250)
file_select.on_change("value", file_select_callback)
def index_callback(_attr, _old, new):
update_image(new)
index_slider = Slider(value=0, start=0, end=1, show_value=False, width=400)
index_spinner = Spinner(title="Image index:", value=0, low=0, width=100)
index_spinner.on_change("value", index_callback)
index_slider.js_link("value_throttled", index_spinner, "value")
index_spinner.js_link("value", index_slider, "value")
plot = Plot(
x_range=Range1d(0, IMAGE_W, bounds=(0, IMAGE_W)),
y_range=Range1d(0, IMAGE_H, bounds=(0, IMAGE_H)),
plot_height=IMAGE_PLOT_H,
plot_width=IMAGE_PLOT_W,
toolbar_location="left",
)
# ---- tools
plot.toolbar.logo = None
# ---- axes
plot.add_layout(LinearAxis(), place="above")
plot.add_layout(LinearAxis(major_label_orientation="vertical"),
place="right")
# ---- grid lines
plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- rgba image glyph
image_source = ColumnDataSource(
dict(
image=[np.zeros((IMAGE_H, IMAGE_W), dtype="float32")],
h=[np.zeros((1, 1))],
k=[np.zeros((1, 1))],
l=[np.zeros((1, 1))],
gamma=[np.zeros((1, 1))],
nu=[np.zeros((1, 1))],
omega=[np.zeros((1, 1))],
x=[0],
y=[0],
dw=[IMAGE_W],
dh=[IMAGE_H],
))
h_glyph = Image(image="h", x="x", y="y", dw="dw", dh="dh", global_alpha=0)
k_glyph = Image(image="k", x="x", y="y", dw="dw", dh="dh", global_alpha=0)
l_glyph = Image(image="l", x="x", y="y", dw="dw", dh="dh", global_alpha=0)
gamma_glyph = Image(image="gamma",
x="x",
y="y",
dw="dw",
dh="dh",
global_alpha=0)
nu_glyph = Image(image="nu",
x="x",
y="y",
dw="dw",
dh="dh",
global_alpha=0)
omega_glyph = Image(image="omega",
x="x",
y="y",
dw="dw",
dh="dh",
global_alpha=0)
plot.add_glyph(image_source, h_glyph)
plot.add_glyph(image_source, k_glyph)
plot.add_glyph(image_source, l_glyph)
plot.add_glyph(image_source, gamma_glyph)
plot.add_glyph(image_source, nu_glyph)
plot.add_glyph(image_source, omega_glyph)
image_glyph = Image(image="image", x="x", y="y", dw="dw", dh="dh")
plot.add_glyph(image_source, image_glyph, name="image_glyph")
# ---- projections
proj_v = Plot(
x_range=plot.x_range,
y_range=DataRange1d(),
plot_height=150,
plot_width=IMAGE_PLOT_W,
toolbar_location=None,
)
proj_v.add_layout(LinearAxis(major_label_orientation="vertical"),
place="right")
proj_v.add_layout(LinearAxis(major_label_text_font_size="0pt"),
place="below")
proj_v.add_layout(Grid(dimension=0, ticker=BasicTicker()))
proj_v.add_layout(Grid(dimension=1, ticker=BasicTicker()))
proj_v_line_source = ColumnDataSource(dict(x=[], y=[]))
proj_v.add_glyph(proj_v_line_source,
Line(x="x", y="y", line_color="steelblue"))
proj_h = Plot(
x_range=DataRange1d(),
y_range=plot.y_range,
plot_height=IMAGE_PLOT_H,
plot_width=150,
toolbar_location=None,
)
proj_h.add_layout(LinearAxis(), place="above")
proj_h.add_layout(LinearAxis(major_label_text_font_size="0pt"),
place="left")
proj_h.add_layout(Grid(dimension=0, ticker=BasicTicker()))
proj_h.add_layout(Grid(dimension=1, ticker=BasicTicker()))
proj_h_line_source = ColumnDataSource(dict(x=[], y=[]))
proj_h.add_glyph(proj_h_line_source,
Line(x="x", y="y", line_color="steelblue"))
# add tools
hovertool = HoverTool(tooltips=[
("intensity", "@image"),
("gamma", "@gamma"),
("nu", "@nu"),
("omega", "@omega"),
("h", "@h"),
("k", "@k"),
("l", "@l"),
])
box_edit_source = ColumnDataSource(dict(x=[], y=[], width=[], height=[]))
box_edit_glyph = Rect(x="x",
y="y",
width="width",
height="height",
fill_alpha=0,
line_color="red")
box_edit_renderer = plot.add_glyph(box_edit_source, box_edit_glyph)
boxedittool = BoxEditTool(renderers=[box_edit_renderer], num_objects=1)
def box_edit_callback(_attr, _old, new):
if new["x"]:
h5_data = det_data["data"]
x_val = np.arange(h5_data.shape[0])
left = int(np.floor(new["x"][0]))
right = int(np.ceil(new["x"][0] + new["width"][0]))
bottom = int(np.floor(new["y"][0]))
top = int(np.ceil(new["y"][0] + new["height"][0]))
y_val = np.sum(h5_data[:, bottom:top, left:right], axis=(1, 2))
else:
x_val = []
y_val = []
roi_avg_plot_line_source.data.update(x=x_val, y=y_val)
box_edit_source.on_change("data", box_edit_callback)
wheelzoomtool = WheelZoomTool(maintain_focus=False)
plot.add_tools(
PanTool(),
BoxZoomTool(),
wheelzoomtool,
ResetTool(),
hovertool,
boxedittool,
)
plot.toolbar.active_scroll = wheelzoomtool
# shared frame ranges
frame_range = Range1d(0, 1, bounds=(0, 1))
scanning_motor_range = Range1d(0, 1, bounds=(0, 1))
det_x_range = Range1d(0, IMAGE_W, bounds=(0, IMAGE_W))
overview_plot_x = Plot(
title=Title(text="Projections on X-axis"),
x_range=det_x_range,
y_range=frame_range,
extra_y_ranges={"scanning_motor": scanning_motor_range},
plot_height=400,
plot_width=IMAGE_PLOT_W - 3,
)
# ---- tools
wheelzoomtool = WheelZoomTool(maintain_focus=False)
overview_plot_x.toolbar.logo = None
overview_plot_x.add_tools(
PanTool(),
BoxZoomTool(),
wheelzoomtool,
ResetTool(),
)
overview_plot_x.toolbar.active_scroll = wheelzoomtool
# ---- axes
overview_plot_x.add_layout(LinearAxis(axis_label="Coordinate X, pix"),
place="below")
overview_plot_x.add_layout(LinearAxis(axis_label="Frame",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
overview_plot_x.add_layout(Grid(dimension=0, ticker=BasicTicker()))
overview_plot_x.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- rgba image glyph
overview_plot_x_image_source = ColumnDataSource(
dict(image=[np.zeros((1, 1), dtype="float32")],
x=[0],
y=[0],
dw=[IMAGE_W],
dh=[1]))
overview_plot_x_image_glyph = Image(image="image",
x="x",
y="y",
dw="dw",
dh="dh")
overview_plot_x.add_glyph(overview_plot_x_image_source,
overview_plot_x_image_glyph,
name="image_glyph")
det_y_range = Range1d(0, IMAGE_H, bounds=(0, IMAGE_H))
overview_plot_y = Plot(
title=Title(text="Projections on Y-axis"),
x_range=det_y_range,
y_range=frame_range,
extra_y_ranges={"scanning_motor": scanning_motor_range},
plot_height=400,
plot_width=IMAGE_PLOT_H + 22,
)
# ---- tools
wheelzoomtool = WheelZoomTool(maintain_focus=False)
overview_plot_y.toolbar.logo = None
overview_plot_y.add_tools(
PanTool(),
BoxZoomTool(),
wheelzoomtool,
ResetTool(),
)
overview_plot_y.toolbar.active_scroll = wheelzoomtool
# ---- axes
overview_plot_y.add_layout(LinearAxis(axis_label="Coordinate Y, pix"),
place="below")
overview_plot_y.add_layout(
LinearAxis(
y_range_name="scanning_motor",
axis_label="Scanning motor",
major_label_orientation="vertical",
),
place="right",
)
# ---- grid lines
overview_plot_y.add_layout(Grid(dimension=0, ticker=BasicTicker()))
overview_plot_y.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- rgba image glyph
overview_plot_y_image_source = ColumnDataSource(
dict(image=[np.zeros((1, 1), dtype="float32")],
x=[0],
y=[0],
dw=[IMAGE_H],
dh=[1]))
overview_plot_y_image_glyph = Image(image="image",
x="x",
y="y",
dw="dw",
dh="dh")
overview_plot_y.add_glyph(overview_plot_y_image_source,
overview_plot_y_image_glyph,
name="image_glyph")
roi_avg_plot = Plot(
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=150,
plot_width=IMAGE_PLOT_W,
toolbar_location="left",
)
# ---- tools
roi_avg_plot.toolbar.logo = None
# ---- axes
roi_avg_plot.add_layout(LinearAxis(), place="below")
roi_avg_plot.add_layout(LinearAxis(major_label_orientation="vertical"),
place="left")
# ---- grid lines
roi_avg_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
roi_avg_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
roi_avg_plot_line_source = ColumnDataSource(dict(x=[], y=[]))
roi_avg_plot.add_glyph(roi_avg_plot_line_source,
Line(x="x", y="y", line_color="steelblue"))
cmap_dict = {
"gray": Greys256,
"gray_reversed": Greys256[::-1],
"plasma": Plasma256,
"cividis": Cividis256,
}
def colormap_callback(_attr, _old, new):
image_glyph.color_mapper = LinearColorMapper(palette=cmap_dict[new])
overview_plot_x_image_glyph.color_mapper = LinearColorMapper(
palette=cmap_dict[new])
overview_plot_y_image_glyph.color_mapper = LinearColorMapper(
palette=cmap_dict[new])
colormap = Select(title="Colormap:",
options=list(cmap_dict.keys()),
width=210)
colormap.on_change("value", colormap_callback)
colormap.value = "plasma"
STEP = 1
def main_auto_checkbox_callback(state):
if state:
display_min_spinner.disabled = True
display_max_spinner.disabled = True
else:
display_min_spinner.disabled = False
display_max_spinner.disabled = False
update_image()
main_auto_checkbox = CheckboxGroup(labels=["Main Auto Range"],
active=[0],
width=145,
margin=[10, 5, 0, 5])
main_auto_checkbox.on_click(main_auto_checkbox_callback)
def display_max_spinner_callback(_attr, _old_value, new_value):
display_min_spinner.high = new_value - STEP
image_glyph.color_mapper.high = new_value
display_max_spinner = Spinner(
low=0 + STEP,
value=1,
step=STEP,
disabled=bool(main_auto_checkbox.active),
width=100,
height=31,
)
display_max_spinner.on_change("value", display_max_spinner_callback)
def display_min_spinner_callback(_attr, _old_value, new_value):
display_max_spinner.low = new_value + STEP
image_glyph.color_mapper.low = new_value
display_min_spinner = Spinner(
low=0,
high=1 - STEP,
value=0,
step=STEP,
disabled=bool(main_auto_checkbox.active),
width=100,
height=31,
)
display_min_spinner.on_change("value", display_min_spinner_callback)
PROJ_STEP = 0.1
def proj_auto_checkbox_callback(state):
if state:
proj_display_min_spinner.disabled = True
proj_display_max_spinner.disabled = True
else:
proj_display_min_spinner.disabled = False
proj_display_max_spinner.disabled = False
update_overview_plot()
proj_auto_checkbox = CheckboxGroup(labels=["Projections Auto Range"],
active=[0],
width=145,
margin=[10, 5, 0, 5])
proj_auto_checkbox.on_click(proj_auto_checkbox_callback)
def proj_display_max_spinner_callback(_attr, _old_value, new_value):
proj_display_min_spinner.high = new_value - PROJ_STEP
overview_plot_x_image_glyph.color_mapper.high = new_value
overview_plot_y_image_glyph.color_mapper.high = new_value
proj_display_max_spinner = Spinner(
low=0 + PROJ_STEP,
value=1,
step=PROJ_STEP,
disabled=bool(proj_auto_checkbox.active),
width=100,
height=31,
)
proj_display_max_spinner.on_change("value",
proj_display_max_spinner_callback)
def proj_display_min_spinner_callback(_attr, _old_value, new_value):
proj_display_max_spinner.low = new_value + PROJ_STEP
overview_plot_x_image_glyph.color_mapper.low = new_value
overview_plot_y_image_glyph.color_mapper.low = new_value
proj_display_min_spinner = Spinner(
low=0,
high=1 - PROJ_STEP,
value=0,
step=PROJ_STEP,
disabled=bool(proj_auto_checkbox.active),
width=100,
height=31,
)
proj_display_min_spinner.on_change("value",
proj_display_min_spinner_callback)
def hkl_button_callback():
index = index_spinner.value
h, k, l = calculate_hkl(det_data, index)
image_source.data.update(h=[h], k=[k], l=[l])
hkl_button = Button(label="Calculate hkl (slow)", width=210)
hkl_button.on_click(hkl_button_callback)
def events_list_callback(_attr, _old, new):
doc.events_list_spind.value = new
events_list = TextAreaInput(rows=7, width=830)
events_list.on_change("value", events_list_callback)
doc.events_list_hdf_viewer = events_list
def add_event_button_callback():
diff_vec = []
p0 = [1.0, 0.0, 1.0]
maxfev = 100000
wave = det_data["wave"]
ddist = det_data["ddist"]
gamma = det_data["gamma"][0]
omega = det_data["omega"][0]
nu = det_data["nu"][0]
chi = det_data["chi"][0]
phi = det_data["phi"][0]
scan_motor = det_data["scan_motor"]
var_angle = det_data[scan_motor]
x0 = int(np.floor(det_x_range.start))
xN = int(np.ceil(det_x_range.end))
y0 = int(np.floor(det_y_range.start))
yN = int(np.ceil(det_y_range.end))
fr0 = int(np.floor(frame_range.start))
frN = int(np.ceil(frame_range.end))
data_roi = det_data["data"][fr0:frN, y0:yN, x0:xN]
cnts = np.sum(data_roi, axis=(1, 2))
coeff, _ = curve_fit(gauss,
range(len(cnts)),
cnts,
p0=p0,
maxfev=maxfev)
m = cnts.mean()
sd = cnts.std()
snr_cnts = np.where(sd == 0, 0, m / sd)
frC = fr0 + coeff[1]
var_F = var_angle[math.floor(frC)]
var_C = var_angle[math.ceil(frC)]
frStep = frC - math.floor(frC)
var_step = var_C - var_F
var_p = var_F + var_step * frStep
if scan_motor == "gamma":
gamma = var_p
elif scan_motor == "omega":
omega = var_p
elif scan_motor == "nu":
nu = var_p
elif scan_motor == "chi":
chi = var_p
elif scan_motor == "phi":
phi = var_p
intensity = coeff[1] * abs(
coeff[2] * var_step) * math.sqrt(2) * math.sqrt(np.pi)
projX = np.sum(data_roi, axis=(0, 1))
coeff, _ = curve_fit(gauss,
range(len(projX)),
projX,
p0=p0,
maxfev=maxfev)
x_pos = x0 + coeff[1]
projY = np.sum(data_roi, axis=(0, 2))
coeff, _ = curve_fit(gauss,
range(len(projY)),
projY,
p0=p0,
maxfev=maxfev)
y_pos = y0 + coeff[1]
ga, nu = pyzebra.det2pol(ddist, gamma, nu, x_pos, y_pos)
diff_vector = pyzebra.z1frmd(wave, ga, omega, chi, phi, nu)
d_spacing = float(pyzebra.dandth(wave, diff_vector)[0])
diff_vector = diff_vector.flatten() * 1e10
dv1, dv2, dv3 = diff_vector
diff_vec.append(diff_vector)
if events_list.value and not events_list.value.endswith("\n"):
events_list.value = events_list.value + "\n"
events_list.value = (
events_list.value +
f"{x_pos} {y_pos} {intensity} {snr_cnts} {dv1} {dv2} {dv3} {d_spacing}"
)
add_event_button = Button(label="Add spind event")
add_event_button.on_click(add_event_button_callback)
metadata_table_source = ColumnDataSource(
dict(geom=[""], temp=[None], mf=[None]))
num_formatter = NumberFormatter(format="0.00", nan_format="")
metadata_table = DataTable(
source=metadata_table_source,
columns=[
TableColumn(field="geom", title="Geometry", width=100),
TableColumn(field="temp",
title="Temperature",
formatter=num_formatter,
width=100),
TableColumn(field="mf",
title="Magnetic Field",
formatter=num_formatter,
width=100),
],
width=300,
height=50,
autosize_mode="none",
index_position=None,
)
# Final layout
import_layout = column(proposal_textinput, upload_div, upload_button,
file_select)
layout_image = column(
gridplot([[proj_v, None], [plot, proj_h]], merge_tools=False))
colormap_layout = column(
colormap,
main_auto_checkbox,
row(display_min_spinner, display_max_spinner),
proj_auto_checkbox,
row(proj_display_min_spinner, proj_display_max_spinner),
)
layout_controls = column(
row(metadata_table, index_spinner,
column(Spacer(height=25), index_slider)),
row(add_event_button, hkl_button),
row(events_list),
)
layout_overview = column(
gridplot(
[[overview_plot_x, overview_plot_y]],
toolbar_options=dict(logo=None),
merge_tools=True,
toolbar_location="left",
), )
tab_layout = row(
column(import_layout, colormap_layout),
column(layout_overview, layout_controls),
column(roi_avg_plot, layout_image),
)
return Panel(child=tab_layout, title="hdf viewer")
u'primary visual cortex (striate cortex, area V1/17)',
u'striatum',
u'primary motor cortex (area M1, area 4)',
u'posteroventral (inferior) parietal cortex',
u'primary somatosensory cortex (area S1, areas 3,1,2)',
u'cerebellum',
u'cerebellar cortex',
u'mediodorsal nucleus of thalamus']
df1, df2 = get_dataframes(gene, structures)
source1 = ColumnDataSource(df1)
source2 = ColumnDataSource(df2)
source3, source4 = get_boxplot_data(df1)
age_plot = expression_by_age(source1, source2)
structure_plot = structure_boxplot(source3, source4)
source5 = get_heatmap_data(genes1)
heatmap = plot_heatmap(source5, genes1)
plot = vplot(structure_plot,age_plot, heatmap)
multi_select = MultiSelect(title="Brain Regions:", value=structures,
options=structures)
multi_select.on_change('value', update_plot)
select = Select(title="Gene:", value=genes[2], options=genes)
select.on_change('value', update_plot)
curdoc().add_root(vplot(select, plot, multi_select))
plot_figure = figure(title='Multi-Select',
height=450,
width=600,
tools="save,reset",
toolbar_location="below")
plot_figure.scatter('x', 'y', color='label', source=source, size=10)
multi_select = MultiSelect(title="Filter Plot by color:",
value=["Red", "Orange"],
options=[("Red", "Red"), ("Orange", "Orange")])
def multiselect_click(attr, old, new):
active_mselect = multi_select.value ##Getting multi-select value
selected_df = df[df['label'].isin(
active_mselect)] #filter the dataframe with value in multi-select
source.data = dict(x=selected_df.x,
y=selected_df.y,
label=selected_df.label)
multi_select.on_change('value', multiselect_click)
layout = row(multi_select, plot_figure)
curdoc().add_root(layout)
curdoc().title = "Multi-Select Bokeh Server"
def build_lockdown_tab():
# Get data for lockdown tab
lockdown_data = get_lockdown_data()
lockdown_data = prep_lockdown_data(lockdown_data)
source_gantt = ColumnDataSource(lockdown_data.dropna())
source_points = ColumnDataSource(lockdown_data)
# Create lockdown figure
lockdown_fig = figure(
y_range=FactorRange(factors=lockdown_data.Country.unique().tolist()),
x_axis_type='datetime',
title="Lockdown Status by Nation",
x_range=(lockdown_data['start_date'].min() - timedelta(days=3),
lockdown_data['end_date'].max() + timedelta(days=3)),
outline_line_color=None,
width=1000,
height=650)
# Adding hbar glyph of lockdown dates
gantt_plot = lockdown_fig.hbar(y="Country",
left='start_date',
right='end_date',
height=0.4,
source=source_gantt,
color='color')
# Adding start point circle glyph
start_point = lockdown_fig.circle(x='start_date',
y='Country',
source=source_points,
size=5,
line_color='blue',
fill_color='white')
# Adding end point circle glyph
end_point = lockdown_fig.circle(x='end_date',
y='Country',
source=source_points,
size=5,
line_color='blue',
fill_color='white')
# Formatting x-axis
lockdown_fig.xaxis.axis_label = "Date"
lockdown_fig.xaxis.formatter = DatetimeTickFormatter(days='%d/%m/%Y')
lockdown_fig.xaxis.ticker = DaysTicker(days=np.arange(5, 365, 7))
lockdown_fig.xaxis.major_label_orientation = math.pi / 6
# Formatting y-axis
lockdown_fig.yaxis.axis_label = "Country"
lockdown_fig.yaxis.major_label_orientation = math.pi / 12
lockdown_fig.yaxis.major_label_text_font_size = "7pt"
lockdown_fig.yaxis.major_label_text_font_style = "bold"
lockdown_fig.ygrid.grid_line_color = None
lockdown_fig.y_range.range_padding = 0.05
# Align grid and axis tickers
lockdown_fig.xgrid[0].ticker = lockdown_fig.xaxis[0].ticker
# Adding hover tools
gantt_hover = HoverTool(renderers=[gantt_plot],
tooltips=[('Country', '@Country'),
('Start Date', '@start_date{%d/%m/%Y}'),
('End Date', '@end_date{%d/%m/%Y}'),
('Length', '@length{%d days}')],
formatters={
'@start_date': 'datetime',
'@end_date': 'datetime',
'@length': 'printf'
})
start_hover = HoverTool(renderers=[start_point],
tooltips=[('Country', '@Country'),
('Start Date', '@start_date{%d/%m/%Y}')],
formatters={'@start_date': 'datetime'})
end_hover = HoverTool(renderers=[end_point],
tooltips=[('Country', '@Country'),
('End Date', '@end_date{%d/%m/%Y}')],
formatters={'@end_date': 'datetime'})
lockdown_fig.add_tools(gantt_hover, start_hover, end_hover)
# Adding vertical span for today's date
today_date_span = Span(location=datetime.today(),
dimension='height',
line_color='blue',
line_width=3,
line_dash=[6, 6])
lockdown_fig.add_layout(today_date_span)
# Labelling span
span_label = Label(x=datetime.today() + timedelta(hours=12),
y=-1.2,
y_units='screen',
text='Current Date',
text_font_size='12pt')
lockdown_fig.add_layout(span_label)
# Adding CheckboxButtonGroup for continents
continents = lockdown_data['continent'].unique().tolist()
continent_rbg = RadioButtonGroup(labels=continents, active=None, width=500)
def continent_rbg_callback(attr, old, new):
if continent_rbg.active != None:
selected_continent = continent_rbg.labels[continent_rbg.active]
filtered_df = lockdown_data.loc[lockdown_data.continent ==
selected_continent]
gantt_cds = ColumnDataSource(filtered_df.dropna())
points_cds = ColumnDataSource(filtered_df)
source_gantt.data.update(gantt_cds.data)
source_points.data.update(points_cds.data)
lockdown_fig.y_range.factors = filtered_df.Country.unique().tolist(
)
# Synchronise country filter
country_multiselect.options = filtered_df['Country'].unique(
).tolist()
country_multiselect.value = filtered_df['Country'].unique().tolist(
)
continent_rbg.on_change('active', continent_rbg_callback)
# Adding MultiSelect Widget for filtering by country
countries = lockdown_data['Country'].unique().tolist()
country_multiselect = MultiSelect(title='Countries:',
options=countries,
value=countries,
height=500)
def country_multiselect_callback(attr, old, new):
selected_countries = country_multiselect.value
filter_condition = lockdown_data.Country.isin(selected_countries)
filtered_df = lockdown_data.loc[filter_condition]
gantt_cds = ColumnDataSource(filtered_df.dropna())
points_cds = ColumnDataSource(filtered_df)
source_gantt.data.update(gantt_cds.data)
source_points.data.update(points_cds.data)
lockdown_fig.y_range.factors = filtered_df.Country.unique().tolist()
# Synchronise continent filter
continent_rbg.active = None
country_multiselect.on_change('value', country_multiselect_callback)
# Creating Clear All, Select All Buttons
def clear_button_callback(event):
country_multiselect.options = countries
country_multiselect.value = []
continent_rbg.active = None
lockdown_fig.y_range.factors = lockdown_data.Country.unique().tolist()
def select_all_button_callback(event):
country_multiselect.options = countries
country_multiselect.value = countries
continent_rbg.active = None
lockdown_fig.y_range.factors = lockdown_data.Country.unique().tolist()
clear_button = Button(label="Clear Selection", button_type="success")
clear_button.on_click(clear_button_callback)
select_all_button = Button(label="Select All", button_type="success")
select_all_button.on_click(select_all_button_callback)
# Add the plot to the current document and add a title
layout = row(
widgetbox(clear_button, select_all_button, continent_rbg,
country_multiselect), lockdown_fig)
layout.sizing_mode = 'scale_width'
# Creating lockdown tab
lockdown_tab = Panel(child=layout, title='Lockdown')
return (lockdown_tab)
data_substantiated.fillna('')
fruits = list(dataz['Site'])
years = [str(i) for i in data.columns.values]
palette = list(itertools.islice(palette, len(data.columns.values)))
# this creates [ ("Apples", "2015"), ("Apples", "2016"), ("Apples", "2017"), ("Pears", "2015), ... ]
years_widget = RangeSlider(start=2013,
end=2017,
value=(2015, 2017),
step=1,
title="Year[s]")
site_widget = MultiSelect(title="Site[s]",
value=["ARKANSAS 1 & 2"],
options=open(join(dirname(__file__),
'sites.txt')).read().split('\n'))
site_widget.on_change('value', lambda attr, old, new: update)
# this creates [ ("Apples", "2015"), ("Apples", "2016"), ("Apples", "2017"), ("Pears", "2015), ... ]
x = [(fruit, year) for fruit in fruits for year in years]
source = ColumnDataSource(data=dict(x=[], counts=[]))
source2 = ColumnDataSource(data=dict(x=[], counts=[]))
p = figure(x_range=data.index.values.tolist(),
plot_height=350,
title="Allegations Received from All Sources",
toolbar_location=None,
tools="")
p.vbar(x='x',
top='counts',
width=0.9,
source=source,
In our case, it is a list
new (type of the attribute): New value of the changed attributes.
In our case, it is a list
"""
value = new
if len(value) != 0:
doc.add_next_tick_callback(partial(enable_btn, btn=btn_waveform))
else:
doc.add_next_tick_callback(partial(disable_btn, btn=btn_waveform))
# A multi select box to choose events from
multi_select = MultiSelect(title="Selected Events:", value=[], options=[])
multi_select.height = 100
multi_select.sizing_mode = "scale_width"
multi_select.on_change("value", callback_value_selected)
# Text box to indicate run
text_input = TextInput(value=run_id, title="Events for Run: ")
# add a btn to view waveform
code = """
var events = ms.value;
console.log(events);
for (event of events) {
console.log(event);
var url = `${app}/waveform/${run}/${event}`;
console.log(url);
window.open(url)
}
"""
