class ReferenceWidget:
"Dropdown for choosing the reference"
__files: FileList
__theme: ReferenceWidgetTheme
__widget: Dropdown
def __init__(self, ctrl, model) -> None:
self.__theme = ctrl.theme.swapmodels(ReferenceWidgetTheme())
self.__model = model
self.__files = FileList(ctrl)
def addtodoc(self, mainview, ctrl, *_) -> List[Widget]:
"creates the widget"
self.__widget = Dropdown(name='HS:reference',
width=self.__theme.width,
height=self.__theme.height,
**self.__data())
@mainview.actionifactive(ctrl)
def _py_cb(new):
inew = int(new.item)
val = None if inew < 0 else [i for _, i in self.__files()][inew]
self.__model.fittoreference.reference = val
def _observe(old=None, **_):
if 'reference' in old and mainview.isactive():
data = self.__data()
mainview.calllater(lambda: self.__widget.update(**data))
ctrl.display.observe(FitToReferenceStore().name, _observe)
self.__widget.on_click(_py_cb)
return [self.__widget]
def reset(self, resets):
"updates the widget"
resets[self.__widget].update(**self.__data())
@property
def widget(self):
"returns the widget"
return self.__widget
def __data(self) -> dict:
lst = list(self.__files())
menu: list = [(j, str(i)) for i, j in enumerate(i for i, _ in lst)]
menu += [None, (self.__theme.title, '-1')]
key = self.__model.fittoreference.reference
index = -1 if key is None else [i for _, i in lst].index(key)
return dict(menu=menu, label=menu[index][0], value=str(index))
menu.append((key,key))
plot=p.circle(x='xdata', y='ydata', source= source, size='size',color='ColorList', alpha=0.6, line_color="black", line_width = 1)
ds = plot.data_source
dropdownX = Dropdown(label="X value", button_type="default", menu=menu)
def dropdownX_handler(new):
dictionary['xdata'] = dictionary[new.item]
ds.data = dictionary
print(dictionary['xdata'])
print(new.item)
p.xaxis.axis_label = new.item
dropdownX.on_click(dropdownX_handler)
dropdownY = Dropdown(label="Y value", button_type="default", menu=menu)
def dropdownY_handler(new):
dictionary['ydata'] = dictionary[new.item]
ds.data = dictionary
p.yaxis.axis_label = new.item
dropdownY.on_click(dropdownY_handler)
dropdownR = Dropdown(label="Radius", button_type="default", menu=menu)
def dropdownR_handler(new):
temp = dictionary[new.item]
sizelist = [float(i - min(temp)) / float(max(temp) - min(temp)) * 29 + 5 for i in temp]
dictionary['size'] = sizelist
class SupervisableDataset(Loggable):
"""
Feature-agnostic class for a dataset open to supervision.
Raw -- piecewise annoatation -> Gold -> Dev/Test
Raw -- batch annotation -> Noisy -> Train
Keeping a DataFrame form and a list-of-dicts ("dictl") form, with the intention that
- the DataFrame form supports most kinds of operations;
- the list-of-dicts form could be useful for manipulations outside the scope of pandas;
- synchronization between the two forms should be called sparingly.
"""
# 'scratch': intended to be directly editable by other objects, i.e. Explorers
# labels will be stored but not used for information in hover itself
SCRATCH_SUBSETS = tuple(["raw"])
# non-'scratch': intended to be read-only outside of the class
# 'public': labels will be considered as part of the classification task and will be used for built-in supervision
PUBLIC_SUBSETS = tuple(["train", "dev"])
# 'private': labels will be considered as part of the classification task and will NOT be used for supervision
PRIVATE_SUBSETS = tuple(["test"])
FEATURE_KEY = "feature"
def __init__(
self,
raw_dictl,
train_dictl=None,
dev_dictl=None,
test_dictl=None,
feature_key="feature",
label_key="label",
):
"""
Initialize the dataset with dictl and df forms; initialize the mapping between categorical-int and string labels.
- param raw_dictl: a list of dicts holding the raw data that DO NOT have annotation.
- param train_dictl: a list of dicts holding the batch-annotated noisy train set.
- param dev_dictl: a list of dicts holding the gold dev set.
- param test_dictl: a list of dicts holding the gold test set.
- param feature_key: key in each piece of dict mapping to the feature.
- param label_key: key in each piece of dict mapping to the ground truth in STRING form.
"""
self._info("Initializing...")
def dictl_transform(dictl, labels=True):
"""
Burner function to transform the input list of dictionaries into standard format.
"""
# edge case when dictl is empty or None
if not dictl:
return []
# transform the feature and possibly the label
key_transform = {feature_key: self.__class__.FEATURE_KEY}
if labels:
key_transform[label_key] = "label"
def burner(d):
"""
Burner function to transform a single dict.
"""
if labels:
assert label_key in d, f"Expected dict key {label_key}"
trans_d = {
key_transform.get(_k, _k): _v
for _k, _v in d.items()
}
if not labels:
trans_d["label"] = module_config.ABSTAIN_DECODED
return trans_d
return [burner(_d) for _d in dictl]
self.dictls = {
"raw": dictl_transform(raw_dictl, labels=False),
"train": dictl_transform(train_dictl),
"dev": dictl_transform(dev_dictl),
"test": dictl_transform(test_dictl),
}
self.synchronize_dictl_to_df()
self.df_deduplicate()
self.synchronize_df_to_dictl()
self.setup_widgets()
# self.setup_label_coding() # redundant if setup_pop_table() immediately calls this again
self.setup_pop_table(width_policy="fit", height_policy="fit")
self._good("Finished initialization.")
def copy(self, use_df=True):
"""
Create another instance, carrying over the data entries.
"""
if use_df:
self.synchronize_df_to_dictl()
return self.__class__(
raw_dictl=self.dictls["raw"],
train_dictl=self.dictls["train"],
dev_dictl=self.dictls["dev"],
test_dictl=self.dictls["test"],
feature_key=self.__class__.FEATURE_KEY,
label_key="label",
)
def setup_widgets(self):
"""
Critical widgets for interactive data management.
"""
self.update_pusher = Button(label="Push",
button_type="success",
height_policy="fit",
width_policy="min")
self.data_committer = Dropdown(
label="Commit",
button_type="warning",
menu=[
*self.__class__.PUBLIC_SUBSETS, *self.__class__.PRIVATE_SUBSETS
],
height_policy="fit",
width_policy="min",
)
self.dedup_trigger = Button(
label="Dedup",
button_type="warning",
height_policy="fit",
width_policy="min",
)
def commit_base_callback():
"""
COMMIT creates cross-duplicates between subsets.
- PUSH shall be blocked until DEDUP is executed.
"""
self.dedup_trigger.disabled = False
self.update_pusher.disabled = True
def dedup_base_callback():
"""
DEDUP re-creates dfs with different indices than before.
- COMMIT shall be blocked until PUSH is executed.
"""
self.update_pusher.disabled = False
self.data_committer.disabled = True
self.df_deduplicate()
def push_base_callback():
"""
PUSH enforces df consistency with all linked explorers.
- DEDUP could be blocked because it stays trivial until COMMIT is executed.
"""
self.data_committer.disabled = False
self.dedup_trigger.disabled = True
self.update_pusher.on_click(push_base_callback)
self.data_committer.on_click(commit_base_callback)
self.dedup_trigger.on_click(dedup_base_callback)
self.help_div = dataset_help_widget()
def view(self):
"""
Defines the layout of bokeh models.
"""
# local import to avoid naming confusion/conflicts
from bokeh.layouts import row, column
return column(
self.help_div,
row(self.update_pusher, self.data_committer, self.dedup_trigger),
self.pop_table,
)
def subscribe_update_push(self, explorer, subset_mapping):
"""
Enable pushing updated DataFrames to explorers that depend on them.
Note: the reason we need this is due to `self.dfs[key] = ...`-like assignments. If DF operations were all in-place, then the explorers could directly access the updates through their `self.dfs` references.
"""
# local import to avoid import cycles
from hover.core.explorer import BokehForLabeledText
assert isinstance(explorer, BokehForLabeledText)
def callback_push():
df_dict = {_v: self.dfs[_k] for _k, _v in subset_mapping.items()}
explorer._setup_dfs(df_dict)
explorer._update_sources()
self.update_pusher.on_click(callback_push)
self._good(
f"Subscribed {explorer.__class__.__name__} to dataset pushes: {subset_mapping}"
)
def subscribe_data_commit(self, explorer, subset_mapping):
"""
Enable committing data across subsets, specified by a selection in an explorer and a dropdown widget of the dataset.
"""
def callback_commit(event):
for sub_k, sub_v in subset_mapping.items():
sub_to = event.item
selected_idx = explorer.sources[sub_v].selected.indices
if not selected_idx:
self._warn(
f"Attempting data commit: did not select any data points in subset {sub_v}."
)
return
# take selected slice, ignoring ABSTAIN'ed rows
sel_slice = self.dfs[sub_k].iloc[selected_idx]
valid_slice = sel_slice[
sel_slice["label"] != module_config.ABSTAIN_DECODED]
# concat to the end and do some accounting
size_before = self.dfs[sub_to].shape[0]
self.dfs[sub_to] = pd.concat(
[self.dfs[sub_to], valid_slice],
axis=0,
sort=False,
ignore_index=True,
)
size_mid = self.dfs[sub_to].shape[0]
self.dfs[sub_to].drop_duplicates(
subset=[self.__class__.FEATURE_KEY],
keep="last",
inplace=True)
size_after = self.dfs[sub_to].shape[0]
self._info(
f"Committed {valid_slice.shape[0]} (valid out of {sel_slice.shape[0]} selected) entries from {sub_k} to {sub_to} ({size_before} -> {size_after} with {size_mid-size_after} overwrites)."
)
self.data_committer.on_click(callback_commit)
self._good(
f"Subscribed {explorer.__class__.__name__} to dataset commits: {subset_mapping}"
)
def setup_label_coding(self, verbose=True, debug=False):
"""
Auto-determine labels in the dataset, then create encoder/decoder in lexical order.
Add ABSTAIN as a no-label placeholder.
"""
all_labels = set()
for _key in [
*self.__class__.PUBLIC_SUBSETS, *self.__class__.PRIVATE_SUBSETS
]:
_df = self.dfs[_key]
_found_labels = set(_df["label"].tolist())
all_labels = all_labels.union(_found_labels)
# exclude ABSTAIN from self.classes, but include it in the encoding
all_labels.discard(module_config.ABSTAIN_DECODED)
self.classes = sorted(all_labels)
self.label_encoder = {
**{_label: _i
for _i, _label in enumerate(self.classes)},
module_config.ABSTAIN_DECODED: module_config.ABSTAIN_ENCODED,
}
self.label_decoder = {_v: _k for _k, _v in self.label_encoder.items()}
if verbose:
self._good(
f"Set up label encoder/decoder with {len(self.classes)} classes."
)
if debug:
self.validate_labels()
def validate_labels(self, raise_exception=True):
"""
Check that every label is in the encoder.
"""
for _key in [
*self.__class__.PUBLIC_SUBSETS, *self.__class__.PRIVATE_SUBSETS
]:
_invalid_indices = None
assert "label" in self.dfs[_key].columns
_mask = self.dfs[_key]["label"].apply(
lambda x: x in self.label_encoder)
_invalid_indices = np.where(_mask == False)[0].tolist()
if _invalid_indices:
self._fail(f"Subset {_key} has invalid labels:")
self._print({self.dfs[_key].loc[_invalid_indices]})
if raise_exception:
raise ValueError("invalid labels")
def setup_pop_table(self, **kwargs):
subsets = [
*self.__class__.SCRATCH_SUBSETS,
*self.__class__.PUBLIC_SUBSETS,
*self.__class__.PRIVATE_SUBSETS,
]
pop_source = ColumnDataSource(dict())
pop_columns = [
TableColumn(field="label", title="label"),
*[
TableColumn(field=f"count_{_subset}", title=_subset)
for _subset in subsets
],
]
self.pop_table = DataTable(source=pop_source,
columns=pop_columns,
**kwargs)
def update_population():
"""
Callback function.
"""
# make sure that the label coding is correct
self.setup_label_coding()
# re-compute label population
eff_labels = [module_config.ABSTAIN_DECODED, *self.classes]
pop_data = dict(label=eff_labels)
for _subset in subsets:
_subpop = self.dfs[_subset]["label"].value_counts()
pop_data[f"count_{_subset}"] = [
_subpop.get(_label, 0) for _label in eff_labels
]
# push results to bokeh data source
pop_source.data = pop_data
self._good(
f"Pop updater: latest population with {len(self.classes)} classes"
)
update_population()
self.data_committer.on_click(update_population)
self.dedup_trigger.on_click(update_population)
def df_deduplicate(self):
"""
Cross-deduplicate data entries by feature between subsets.
"""
self._info("Deduplicating...")
# for data entry accounting
before, after = dict(), dict()
# deduplicating rule: entries that come LATER are of higher priority
ordered_subsets = [
*self.__class__.SCRATCH_SUBSETS,
*self.__class__.PUBLIC_SUBSETS,
*self.__class__.PRIVATE_SUBSETS,
]
# keep track of which df has which columns and which rows came from which subset
columns = dict()
for _key in ordered_subsets:
before[_key] = self.dfs[_key].shape[0]
columns[_key] = self.dfs[_key].columns
self.dfs[_key]["__subset"] = _key
# concatenate in order and deduplicate
overall_df = pd.concat([self.dfs[_key] for _key in ordered_subsets],
axis=0,
sort=False)
overall_df.drop_duplicates(subset=[self.__class__.FEATURE_KEY],
keep="last",
inplace=True)
overall_df.reset_index(drop=True, inplace=True)
# cut up slices
for _key in ordered_subsets:
self.dfs[_key] = overall_df[
overall_df["__subset"] == _key].reset_index(
drop=True, inplace=False)[columns[_key]]
after[_key] = self.dfs[_key].shape[0]
self._info(
f"--subset {_key} rows: {before[_key]} -> {after[_key]}.")
def synchronize_dictl_to_df(self):
"""
Re-make dataframes from lists of dictionaries.
"""
self.dfs = dict()
for _key, _dictl in self.dictls.items():
if _dictl:
_df = pd.DataFrame(_dictl)
assert self.__class__.FEATURE_KEY in _df.columns
assert "label" in _df.columns
else:
_df = pd.DataFrame(
columns=[self.__class__.FEATURE_KEY, "label"])
self.dfs[_key] = _df
def synchronize_df_to_dictl(self):
"""
Re-make lists of dictionaries from dataframes.
"""
self.dictls = dict()
for _key, _df in self.dfs.items():
self.dictls[_key] = _df.to_dict(orient="records")
def compute_2d_embedding(self, vectorizer, method, **kwargs):
"""
Get embeddings in the xy-plane and return the reducer.
"""
from hover.core.representation.reduction import DimensionalityReducer
# prepare input vectors to manifold learning
fit_subset = [
*self.__class__.SCRATCH_SUBSETS, *self.__class__.PUBLIC_SUBSETS
]
trans_subset = [*self.__class__.PRIVATE_SUBSETS]
assert not set(fit_subset).intersection(
set(trans_subset)), "Unexpected overlap"
# compute vectors and keep track which where to slice the array for fitting
feature_inp = []
for _key in fit_subset:
feature_inp += self.dfs[_key][self.__class__.FEATURE_KEY].tolist()
fit_num = len(feature_inp)
for _key in trans_subset:
feature_inp += self.dfs[_key][self.__class__.FEATURE_KEY].tolist()
trans_arr = np.array([vectorizer(_inp) for _inp in tqdm(feature_inp)])
# initialize and fit manifold learning reducer using specified subarray
self._info(
f"Fit-transforming {method.upper()} on {fit_num} samples...")
reducer = DimensionalityReducer(trans_arr[:fit_num])
fit_embedding = reducer.fit_transform(method, **kwargs)
# compute embedding of the whole dataset
self._info(
f"Transforming {method.upper()} on {trans_arr.shape[0]-fit_num} samples..."
)
trans_embedding = reducer.transform(trans_arr[fit_num:], method)
# assign x and y coordinates to dataset
start_idx = 0
for _subset, _embedding in [
(fit_subset, fit_embedding),
(trans_subset, trans_embedding),
]:
# edge case: embedding is too small
if _embedding.shape[0] < 1:
for _key in _subset:
assert (self.dfs[_key].shape[0] == 0
), "Expected empty df due to empty embedding"
continue
for _key in _subset:
_length = self.dfs[_key].shape[0]
self.dfs[_key]["x"] = pd.Series(
_embedding[start_idx:(start_idx + _length), 0])
self.dfs[_key]["y"] = pd.Series(
_embedding[start_idx:(start_idx + _length), 1])
start_idx += _length
return reducer
def loader(self, key, vectorizer, batch_size=64, smoothing_coeff=0.0):
"""
Prepare a Torch Dataloader for training or evaluation.
- param key(str): the subset of dataset to use.
- param vectorizer(callable): callable that turns a string into a vector.
- param smoothing_coeff: the smoothing coeffient for soft labels.
- type smoothing_coeff: float
"""
# lazy import: missing torch should not break the rest of the class
from hover.utils.torch_helper import vector_dataloader, one_hot, label_smoothing
# take the slice that has a meaningful label
df = self.dfs[key][
self.dfs[key]["label"] != module_config.ABSTAIN_DECODED]
# edge case: valid slice is too small
if df.shape[0] < 1:
raise ValueError(
f"Subset {key} has too few samples ({df.shape[0]})")
batch_size = min(batch_size, df.shape[0])
labels = df["label"].apply(lambda x: self.label_encoder[x]).tolist()
features = df[self.__class__.FEATURE_KEY].tolist()
output_vectors = one_hot(labels, num_classes=len(self.classes))
self._info(f"Preparing {key} input vectors...")
input_vectors = [vectorizer(_f) for _f in tqdm(features)]
if smoothing_coeff > 0.0:
output_vectors = label_smoothing(output_vectors,
coefficient=smoothing_coeff)
self._info(f"Preparing {key} data loader...")
loader = vector_dataloader(input_vectors,
output_vectors,
batch_size=batch_size)
self._good(
f"Prepared {key} loader consisting of {len(features)} examples with batch size {batch_size}"
)
return loader
class SequencePathWidget:
"Dropdown for choosing a fasta file"
_dialog: FileDialog
_widget: Dropdown
_theme: SequencePathTheme
_model: SequencePlotModelAccess
def __init__(self, ctrl, **kwa):
self._theme = ctrl.theme.swapmodels(SequencePathTheme(**kwa))
self._model = SequencePlotModelAccess()
self._model.swapmodels(ctrl)
def addtodoc(self, mainview, ctrl, *_) -> List[Widget]:
"creates the widget"
self._widget = Dropdown(name='Cycles:Sequence',
width=self._theme.width,
height=self._theme.height,
**self._data())
mainview.differedobserver(self._data, self._widget, ctrl.theme,
self._model.sequencemodel.config,
ctrl.display,
self._model.sequencemodel.display)
self._widget.on_click(ctrl.action(self._onclick))
return [self._widget]
def observe(self, ctrl):
"sets-up config observers"
self._dialog = FileDialog(ctrl,
storage="sequence",
title=self._theme.dlgtitle,
filetypes='fasta|txt|*')
def reset(self, resets):
"updates the widget"
resets[self._widget].update(**self._data())
@property
def widget(self):
"returns the widget"
return self._widget
def callbacks(self, hover: SequenceHoverMixin, tick1: SequenceTicker):
"sets-up callbacks for the tooltips and grids"
if hover is not None:
jsc = CustomJS(
code=("if(Object.keys(src.data).indexOf(cb_obj.value) > -1)"
"{ cb_obj.label = cb_obj.value;"
" tick1.key = cb_obj.value;"
" tick2.key = cb_obj.value;"
" src.data['text'] = src.data[cb_obj.value];"
" src.change.emit(); }"),
args=dict(tick1=tick1, tick2=tick1.axis, src=hover.source))
self._widget.js_on_change('value', jsc)
return self._widget
def _data(self) -> dict:
lst = sorted(self._model.sequences(...))
key = self._model.sequencemodel.currentkey
val = key if key in lst else None
label = self._theme.missingkey if val is None else key
menu: List[Optional[Tuple[str, str]]] = [(i, i) for i in lst]
menu += [
None if len(menu) else ('', '→'), (self._theme.missingpath, '←')
]
return dict(menu=menu, label=label, value='→' if val is None else val)
def _onclick(self, new):
if new.item == '←':
path = self._dialog.open()
self._widget.value = '→'
if self._model.setnewsequencepath(path):
if path is not None:
raise IOError("Could not find any sequence in the file")
elif new.item != '→':
self._model.setnewsequencekey(new.item)
lb = event.item
y1 = cnthr[int(lb)]
fig.line(x=x_always,
y=y1,
line_width=2,
legend_label=lb,
line_color='pink')
def handler2(event):
lb = event.item
y2 = cnthr[int(lb)]
fig.line(x_always, y2, line_width=2, legend_label=lb, line_color='blue')
dd1.on_click(handler1)
dd2.on_click(handler2)
fig = figure(x_range=[
'Jan', 'Feb', 'Mar', 'Apr', 'May', 'June', 'July', 'Aug', 'Sept', 'Oct',
'Nov', 'Dec'
],
y_range=(0, 1000),
title="NYC 311 Average Response Time 2020",
x_axis_label='month',
y_axis_label='Average response time in hour')
fig.line(x_always,
all_avg,
line_width=2,
legend_label='all',
line_color='grey')
curdoc().add_root(column(dd1, dd2, fig))
def create(palm):
fit_max = 1
fit_min = 0
doc = curdoc()
# THz calibration plot
scan_plot = Plot(
title=Title(text="THz calibration"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
scan_plot.toolbar.logo = None
scan_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(), ResetTool())
# ---- axes
scan_plot.add_layout(LinearAxis(axis_label="Stage delay motor"), place="below")
scan_plot.add_layout(
LinearAxis(axis_label="Energy shift, eV", major_label_orientation="vertical"), place="left"
)
# ---- grid lines
scan_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
scan_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- circle cluster glyphs
scan_circle_source = ColumnDataSource(dict(x=[], y=[]))
scan_plot.add_glyph(scan_circle_source, Circle(x="x", y="y", line_alpha=0, fill_alpha=0.5))
# ---- circle glyphs
scan_avg_circle_source = ColumnDataSource(dict(x=[], y=[]))
scan_plot.add_glyph(
scan_avg_circle_source, Circle(x="x", y="y", line_color="purple", fill_color="purple")
)
# ---- line glyphs
fit_line_source = ColumnDataSource(dict(x=[], y=[]))
scan_plot.add_glyph(fit_line_source, Line(x="x", y="y", line_color="purple"))
# THz calibration folder path text input
def path_textinput_callback(_attr, _old_value, _new_value):
update_load_dropdown_menu()
path_periodic_update()
path_textinput = TextInput(
title="THz calibration path:", value=os.path.join(os.path.expanduser("~")), width=510
)
path_textinput.on_change("value", path_textinput_callback)
# THz calibration eco scans dropdown
def scans_dropdown_callback(event):
scans_dropdown.label = event.item
scans_dropdown = Dropdown(label="ECO scans", button_type="default", menu=[])
scans_dropdown.on_click(scans_dropdown_callback)
# ---- eco scans periodic update
def path_periodic_update():
new_menu = []
if os.path.isdir(path_textinput.value):
for entry in os.scandir(path_textinput.value):
if entry.is_file() and entry.name.endswith(".json"):
new_menu.append((entry.name, entry.name))
scans_dropdown.menu = sorted(new_menu, reverse=True)
doc.add_periodic_callback(path_periodic_update, 5000)
# Calibrate button
def calibrate_button_callback():
palm.calibrate_thz(path=os.path.join(path_textinput.value, scans_dropdown.label))
fit_max_spinner.value = np.ceil(palm.thz_calib_data.index.values.max())
fit_min_spinner.value = np.floor(palm.thz_calib_data.index.values.min())
update_calibration_plot()
def update_calibration_plot():
scan_plot.xaxis.axis_label = f"{palm.thz_motor_name}, {palm.thz_motor_unit}"
scan_circle_source.data.update(
x=np.repeat(
palm.thz_calib_data.index, palm.thz_calib_data["peak_shift"].apply(len)
).tolist(),
y=np.concatenate(palm.thz_calib_data["peak_shift"].values).tolist(),
)
scan_avg_circle_source.data.update(
x=palm.thz_calib_data.index.tolist(), y=palm.thz_calib_data["peak_shift_mean"].tolist()
)
x = np.linspace(fit_min, fit_max, 100)
y = palm.thz_slope * x + palm.thz_intersect
fit_line_source.data.update(x=np.round(x, decimals=5), y=np.round(y, decimals=5))
calib_const_div.text = f"""
thz_slope = {palm.thz_slope}
"""
calibrate_button = Button(label="Calibrate THz", button_type="default", width=250)
calibrate_button.on_click(calibrate_button_callback)
# THz fit maximal value text input
def fit_max_spinner_callback(_attr, old_value, new_value):
nonlocal fit_max
if new_value > fit_min:
fit_max = new_value
palm.calibrate_thz(
path=os.path.join(path_textinput.value, scans_dropdown.label),
fit_range=(fit_min, fit_max),
)
update_calibration_plot()
else:
fit_max_spinner.value = old_value
fit_max_spinner = Spinner(title="Maximal fit value:", value=fit_max, step=0.1)
fit_max_spinner.on_change("value", fit_max_spinner_callback)
# THz fit maximal value text input
def fit_min_spinner_callback(_attr, old_value, new_value):
nonlocal fit_min
if new_value < fit_max:
fit_min = new_value
palm.calibrate_thz(
path=os.path.join(path_textinput.value, scans_dropdown.label),
fit_range=(fit_min, fit_max),
)
update_calibration_plot()
else:
fit_min_spinner.value = old_value
fit_min_spinner = Spinner(title="Minimal fit value:", value=fit_min, step=0.1)
fit_min_spinner.on_change("value", fit_min_spinner_callback)
# Save calibration button
def save_button_callback():
palm.save_thz_calib(path=path_textinput.value)
update_load_dropdown_menu()
save_button = Button(label="Save", button_type="default", width=250)
save_button.on_click(save_button_callback)
# Load calibration button
def load_dropdown_callback(event):
new_value = event.item
palm.load_thz_calib(os.path.join(path_textinput.value, new_value))
update_calibration_plot()
def update_load_dropdown_menu():
new_menu = []
calib_file_ext = ".palm_thz"
if os.path.isdir(path_textinput.value):
for entry in os.scandir(path_textinput.value):
if entry.is_file() and entry.name.endswith((calib_file_ext)):
new_menu.append((entry.name[: -len(calib_file_ext)], entry.name))
load_dropdown.button_type = "default"
load_dropdown.menu = sorted(new_menu, reverse=True)
else:
load_dropdown.button_type = "danger"
load_dropdown.menu = new_menu
doc.add_next_tick_callback(update_load_dropdown_menu)
doc.add_periodic_callback(update_load_dropdown_menu, 5000)
load_dropdown = Dropdown(label="Load", menu=[], width=250)
load_dropdown.on_click(load_dropdown_callback)
# Calibration constants
calib_const_div = Div(
text=f"""
thz_slope = {0}
"""
)
# assemble
tab_layout = column(
row(
scan_plot,
Spacer(width=30),
column(
path_textinput,
scans_dropdown,
calibrate_button,
fit_max_spinner,
fit_min_spinner,
row(save_button, load_dropdown),
calib_const_div,
),
)
)
return Panel(child=tab_layout, title="THz Calibration")
u_input = TextInput(value=odesystem_settings.sample_system_functions[
odesystem_settings.init_fun_key][0],
title="u(x,y):")
v_input = TextInput(value=odesystem_settings.sample_system_functions[
odesystem_settings.init_fun_key][1],
title="v(x,y):")
# dropdown menu for selecting one of the sample functions
sample_fun_input = Dropdown(
label="choose a sample function pair or enter one below",
menu=odesystem_settings.sample_system_names)
# Interactor for entering starting point of initial condition
interactor = my_bokeh_utils.Interactor(plot)
# initialize callback behaviour
sample_fun_input.on_click(sample_fun_change)
u_input.on_change('value', ode_change)
v_input.on_change('value', ode_change)
interactor.on_click(initial_value_change)
# calculate data
init_data()
# lists all the controls in our app associated with the default_funs panel
function_controls = widgetbox(sample_fun_input, u_input, v_input, width=400)
# refresh quiver field and streamline all 100ms
curdoc().add_periodic_callback(refresh_user_view, 100)
# make layout
curdoc().add_root(row(function_controls, plot))
dictionary['xdata3'] = dictionary[new.item]
dict['x3'] = new.item
ds3.data = dictionary
plot3.xaxis.axis_label = new.item
update_tooltip()
def dropdownX_handler4(new):
dictionary['xdata4'] = dictionary[new.item]
dict['x4'] = new.item
ds4.data = dictionary
plot4.xaxis.axis_label = new.item
update_tooltip()
dropdownX1.on_click(dropdownX_handler1)
dropdownX2.on_click(dropdownX_handler2)
dropdownX3.on_click(dropdownX_handler3)
dropdownX4.on_click(dropdownX_handler4)
dropdownY1 = Dropdown(label="Y value1", button_type="default", menu=menu)
dropdownY2 = Dropdown(label="Y value2", button_type="default", menu=menu)
dropdownY3 = Dropdown(label="Y value3", button_type="default", menu=menu)
dropdownY4 = Dropdown(label="Y value4", button_type="default", menu=menu)
def dropdownY_handler1(new):
dictionary['ydata1'] = dictionary[new.item]
dict['y1'] = new.item
ds1.data = dictionary
plot1.yaxis.axis_label = new.item
def create():
det_data = []
fit_params = {}
js_data = ColumnDataSource(data=dict(content=["", ""], fname=["", ""]))
def proposal_textinput_callback(_attr, _old, new):
proposal = new.strip()
for zebra_proposals_path in pyzebra.ZEBRA_PROPOSALS_PATHS:
proposal_path = os.path.join(zebra_proposals_path, proposal)
if os.path.isdir(proposal_path):
# found it
break
else:
raise ValueError(f"Can not find data for proposal '{proposal}'.")
file_list = []
for file in os.listdir(proposal_path):
if file.endswith((".ccl", ".dat")):
file_list.append((os.path.join(proposal_path, file), file))
file_select.options = file_list
file_open_button.disabled = False
file_append_button.disabled = False
proposal_textinput = TextInput(title="Proposal number:", width=210)
proposal_textinput.on_change("value", proposal_textinput_callback)
def _init_datatable():
scan_list = [s["idx"] for s in det_data]
file_list = []
for scan in det_data:
file_list.append(os.path.basename(scan["original_filename"]))
scan_table_source.data.update(
file=file_list,
scan=scan_list,
param=[None] * len(scan_list),
fit=[0] * len(scan_list),
export=[True] * len(scan_list),
)
scan_table_source.selected.indices = []
scan_table_source.selected.indices = [0]
scan_motor_select.options = det_data[0]["scan_motors"]
scan_motor_select.value = det_data[0]["scan_motor"]
param_select.value = "user defined"
file_select = MultiSelect(title="Available .ccl/.dat files:",
width=210,
height=250)
def file_open_button_callback():
nonlocal det_data
det_data = []
for f_name in file_select.value:
with open(f_name) as file:
base, ext = os.path.splitext(f_name)
if det_data:
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data,
monitor_spinner.value)
det_data.extend(append_data)
else:
det_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(det_data, monitor_spinner.value)
js_data.data.update(
fname=[base + ".comm", base + ".incomm"])
_init_datatable()
append_upload_button.disabled = False
file_open_button = Button(label="Open New", width=100, disabled=True)
file_open_button.on_click(file_open_button_callback)
def file_append_button_callback():
for f_name in file_select.value:
with open(f_name) as file:
_, ext = os.path.splitext(f_name)
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data, monitor_spinner.value)
det_data.extend(append_data)
_init_datatable()
file_append_button = Button(label="Append", width=100, disabled=True)
file_append_button.on_click(file_append_button_callback)
def upload_button_callback(_attr, _old, new):
nonlocal det_data
det_data = []
for f_str, f_name in zip(new, upload_button.filename):
with io.StringIO(base64.b64decode(f_str).decode()) as file:
base, ext = os.path.splitext(f_name)
if det_data:
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data,
monitor_spinner.value)
det_data.extend(append_data)
else:
det_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(det_data, monitor_spinner.value)
js_data.data.update(
fname=[base + ".comm", base + ".incomm"])
_init_datatable()
append_upload_button.disabled = False
upload_div = Div(text="or upload new .ccl/.dat files:",
margin=(5, 5, 0, 5))
upload_button = FileInput(accept=".ccl,.dat", multiple=True, width=200)
upload_button.on_change("value", upload_button_callback)
def append_upload_button_callback(_attr, _old, new):
for f_str, f_name in zip(new, append_upload_button.filename):
with io.StringIO(base64.b64decode(f_str).decode()) as file:
_, ext = os.path.splitext(f_name)
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data, monitor_spinner.value)
det_data.extend(append_data)
_init_datatable()
append_upload_div = Div(text="append extra files:", margin=(5, 5, 0, 5))
append_upload_button = FileInput(accept=".ccl,.dat",
multiple=True,
width=200,
disabled=True)
append_upload_button.on_change("value", append_upload_button_callback)
def monitor_spinner_callback(_attr, _old, new):
if det_data:
pyzebra.normalize_dataset(det_data, new)
_update_plot()
monitor_spinner = Spinner(title="Monitor:",
mode="int",
value=100_000,
low=1,
width=145)
monitor_spinner.on_change("value", monitor_spinner_callback)
def scan_motor_select_callback(_attr, _old, new):
if det_data:
for scan in det_data:
scan["scan_motor"] = new
_update_plot()
scan_motor_select = Select(title="Scan motor:", options=[], width=145)
scan_motor_select.on_change("value", scan_motor_select_callback)
def _update_table():
fit_ok = [(1 if "fit" in scan else 0) for scan in det_data]
scan_table_source.data.update(fit=fit_ok)
def _update_plot():
_update_single_scan_plot(_get_selected_scan())
_update_overview()
def _update_single_scan_plot(scan):
scan_motor = scan["scan_motor"]
y = scan["counts"]
x = scan[scan_motor]
plot.axis[0].axis_label = scan_motor
plot_scatter_source.data.update(x=x,
y=y,
y_upper=y + np.sqrt(y),
y_lower=y - np.sqrt(y))
fit = scan.get("fit")
if fit is not None:
x_fit = np.linspace(x[0], x[-1], 100)
plot_fit_source.data.update(x=x_fit, y=fit.eval(x=x_fit))
x_bkg = []
y_bkg = []
xs_peak = []
ys_peak = []
comps = fit.eval_components(x=x_fit)
for i, model in enumerate(fit_params):
if "linear" in model:
x_bkg = x_fit
y_bkg = comps[f"f{i}_"]
elif any(val in model
for val in ("gaussian", "voigt", "pvoigt")):
xs_peak.append(x_fit)
ys_peak.append(comps[f"f{i}_"])
plot_bkg_source.data.update(x=x_bkg, y=y_bkg)
plot_peak_source.data.update(xs=xs_peak, ys=ys_peak)
fit_output_textinput.value = fit.fit_report()
else:
plot_fit_source.data.update(x=[], y=[])
plot_bkg_source.data.update(x=[], y=[])
plot_peak_source.data.update(xs=[], ys=[])
fit_output_textinput.value = ""
def _update_overview():
xs = []
ys = []
param = []
x = []
y = []
par = []
for s, p in enumerate(scan_table_source.data["param"]):
if p is not None:
scan = det_data[s]
scan_motor = scan["scan_motor"]
xs.append(scan[scan_motor])
x.extend(scan[scan_motor])
ys.append(scan["counts"])
y.extend([float(p)] * len(scan[scan_motor]))
param.append(float(p))
par.extend(scan["counts"])
if det_data:
scan_motor = det_data[0]["scan_motor"]
ov_plot.axis[0].axis_label = scan_motor
ov_param_plot.axis[0].axis_label = scan_motor
ov_plot_mline_source.data.update(xs=xs,
ys=ys,
param=param,
color=color_palette(len(xs)))
if y:
mapper["transform"].low = np.min([np.min(y) for y in ys])
mapper["transform"].high = np.max([np.max(y) for y in ys])
ov_param_plot_scatter_source.data.update(x=x, y=y, param=par)
if y:
interp_f = interpolate.interp2d(x, y, par)
x1, x2 = min(x), max(x)
y1, y2 = min(y), max(y)
image = interp_f(
np.linspace(x1, x2, ov_param_plot.inner_width // 10),
np.linspace(y1, y2, ov_param_plot.inner_height // 10),
assume_sorted=True,
)
ov_param_plot_image_source.data.update(image=[image],
x=[x1],
y=[y1],
dw=[x2 - x1],
dh=[y2 - y1])
else:
ov_param_plot_image_source.data.update(image=[],
x=[],
y=[],
dw=[],
dh=[])
def _update_param_plot():
x = []
y = []
fit_param = fit_param_select.value
for s, p in zip(det_data, scan_table_source.data["param"]):
if "fit" in s and fit_param:
x.append(p)
y.append(s["fit"].values[fit_param])
param_plot_scatter_source.data.update(x=x, y=y)
# Main plot
plot = Plot(
x_range=DataRange1d(),
y_range=DataRange1d(only_visible=True),
plot_height=450,
plot_width=700,
)
plot.add_layout(LinearAxis(axis_label="Counts"), place="left")
plot.add_layout(LinearAxis(axis_label="Scan motor"), place="below")
plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
plot_scatter_source = ColumnDataSource(
dict(x=[0], y=[0], y_upper=[0], y_lower=[0]))
plot_scatter = plot.add_glyph(
plot_scatter_source, Scatter(x="x", y="y", line_color="steelblue"))
plot.add_layout(
Whisker(source=plot_scatter_source,
base="x",
upper="y_upper",
lower="y_lower"))
plot_fit_source = ColumnDataSource(dict(x=[0], y=[0]))
plot_fit = plot.add_glyph(plot_fit_source, Line(x="x", y="y"))
plot_bkg_source = ColumnDataSource(dict(x=[0], y=[0]))
plot_bkg = plot.add_glyph(
plot_bkg_source,
Line(x="x", y="y", line_color="green", line_dash="dashed"))
plot_peak_source = ColumnDataSource(dict(xs=[[0]], ys=[[0]]))
plot_peak = plot.add_glyph(
plot_peak_source,
MultiLine(xs="xs", ys="ys", line_color="red", line_dash="dashed"))
fit_from_span = Span(location=None, dimension="height", line_dash="dashed")
plot.add_layout(fit_from_span)
fit_to_span = Span(location=None, dimension="height", line_dash="dashed")
plot.add_layout(fit_to_span)
plot.add_layout(
Legend(
items=[
("data", [plot_scatter]),
("best fit", [plot_fit]),
("peak", [plot_peak]),
("linear", [plot_bkg]),
],
location="top_left",
click_policy="hide",
))
plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
plot.toolbar.logo = None
# Overview multilines plot
ov_plot = Plot(x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=450,
plot_width=700)
ov_plot.add_layout(LinearAxis(axis_label="Counts"), place="left")
ov_plot.add_layout(LinearAxis(axis_label="Scan motor"), place="below")
ov_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
ov_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
ov_plot_mline_source = ColumnDataSource(
dict(xs=[], ys=[], param=[], color=[]))
ov_plot.add_glyph(ov_plot_mline_source,
MultiLine(xs="xs", ys="ys", line_color="color"))
hover_tool = HoverTool(tooltips=[("param", "@param")])
ov_plot.add_tools(PanTool(), WheelZoomTool(), hover_tool, ResetTool())
ov_plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
ov_plot.toolbar.logo = None
# Overview perams plot
ov_param_plot = Plot(x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=450,
plot_width=700)
ov_param_plot.add_layout(LinearAxis(axis_label="Param"), place="left")
ov_param_plot.add_layout(LinearAxis(axis_label="Scan motor"),
place="below")
ov_param_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
ov_param_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
ov_param_plot_image_source = ColumnDataSource(
dict(image=[], x=[], y=[], dw=[], dh=[]))
ov_param_plot.add_glyph(
ov_param_plot_image_source,
Image(image="image", x="x", y="y", dw="dw", dh="dh"))
ov_param_plot_scatter_source = ColumnDataSource(dict(x=[], y=[], param=[]))
mapper = linear_cmap(field_name="param", palette=Turbo256, low=0, high=50)
ov_param_plot.add_glyph(
ov_param_plot_scatter_source,
Scatter(x="x", y="y", line_color=mapper, fill_color=mapper, size=10),
)
ov_param_plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
ov_param_plot.toolbar.logo = None
# Parameter plot
param_plot = Plot(x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=400,
plot_width=700)
param_plot.add_layout(LinearAxis(axis_label="Fit parameter"), place="left")
param_plot.add_layout(LinearAxis(axis_label="Parameter"), place="below")
param_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
param_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
param_plot_scatter_source = ColumnDataSource(dict(x=[], y=[]))
param_plot.add_glyph(param_plot_scatter_source, Scatter(x="x", y="y"))
param_plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
param_plot.toolbar.logo = None
def fit_param_select_callback(_attr, _old, _new):
_update_param_plot()
fit_param_select = Select(title="Fit parameter", options=[], width=145)
fit_param_select.on_change("value", fit_param_select_callback)
# Plot tabs
plots = Tabs(tabs=[
Panel(child=plot, title="single scan"),
Panel(child=ov_plot, title="overview"),
Panel(child=ov_param_plot, title="overview map"),
Panel(child=column(param_plot, row(fit_param_select)),
title="parameter plot"),
])
# Scan select
def scan_table_select_callback(_attr, old, new):
if not new:
# skip empty selections
return
# Avoid selection of multiple indicies (via Shift+Click or Ctrl+Click)
if len(new) > 1:
# drop selection to the previous one
scan_table_source.selected.indices = old
return
if len(old) > 1:
# skip unnecessary update caused by selection drop
return
_update_plot()
def scan_table_source_callback(_attr, _old, _new):
_update_preview()
scan_table_source = ColumnDataSource(
dict(file=[], scan=[], param=[], fit=[], export=[]))
scan_table_source.on_change("data", scan_table_source_callback)
scan_table = DataTable(
source=scan_table_source,
columns=[
TableColumn(field="file", title="file", width=150),
TableColumn(field="scan", title="scan", width=50),
TableColumn(field="param",
title="param",
editor=NumberEditor(),
width=50),
TableColumn(field="fit", title="Fit", width=50),
TableColumn(field="export",
title="Export",
editor=CheckboxEditor(),
width=50),
],
width=410, # +60 because of the index column
editable=True,
autosize_mode="none",
)
def scan_table_source_callback(_attr, _old, _new):
if scan_table_source.selected.indices:
_update_plot()
scan_table_source.selected.on_change("indices", scan_table_select_callback)
scan_table_source.on_change("data", scan_table_source_callback)
def _get_selected_scan():
return det_data[scan_table_source.selected.indices[0]]
def param_select_callback(_attr, _old, new):
if new == "user defined":
param = [None] * len(det_data)
else:
param = [scan[new] for scan in det_data]
scan_table_source.data["param"] = param
_update_param_plot()
param_select = Select(
title="Parameter:",
options=["user defined", "temp", "mf", "h", "k", "l"],
value="user defined",
width=145,
)
param_select.on_change("value", param_select_callback)
def fit_from_spinner_callback(_attr, _old, new):
fit_from_span.location = new
fit_from_spinner = Spinner(title="Fit from:", width=145)
fit_from_spinner.on_change("value", fit_from_spinner_callback)
def fit_to_spinner_callback(_attr, _old, new):
fit_to_span.location = new
fit_to_spinner = Spinner(title="to:", width=145)
fit_to_spinner.on_change("value", fit_to_spinner_callback)
def fitparams_add_dropdown_callback(click):
# bokeh requires (str, str) for MultiSelect options
new_tag = f"{click.item}-{fitparams_select.tags[0]}"
fitparams_select.options.append((new_tag, click.item))
fit_params[new_tag] = fitparams_factory(click.item)
fitparams_select.tags[0] += 1
fitparams_add_dropdown = Dropdown(
label="Add fit function",
menu=[
("Linear", "linear"),
("Gaussian", "gaussian"),
("Voigt", "voigt"),
("Pseudo Voigt", "pvoigt"),
# ("Pseudo Voigt1", "pseudovoigt1"),
],
width=145,
)
fitparams_add_dropdown.on_click(fitparams_add_dropdown_callback)
def fitparams_select_callback(_attr, old, new):
# Avoid selection of multiple indicies (via Shift+Click or Ctrl+Click)
if len(new) > 1:
# drop selection to the previous one
fitparams_select.value = old
return
if len(old) > 1:
# skip unnecessary update caused by selection drop
return
if new:
fitparams_table_source.data.update(fit_params[new[0]])
else:
fitparams_table_source.data.update(
dict(param=[], value=[], vary=[], min=[], max=[]))
fitparams_select = MultiSelect(options=[], height=120, width=145)
fitparams_select.tags = [0]
fitparams_select.on_change("value", fitparams_select_callback)
def fitparams_remove_button_callback():
if fitparams_select.value:
sel_tag = fitparams_select.value[0]
del fit_params[sel_tag]
for elem in fitparams_select.options:
if elem[0] == sel_tag:
fitparams_select.options.remove(elem)
break
fitparams_select.value = []
fitparams_remove_button = Button(label="Remove fit function", width=145)
fitparams_remove_button.on_click(fitparams_remove_button_callback)
def fitparams_factory(function):
if function == "linear":
params = ["slope", "intercept"]
elif function == "gaussian":
params = ["amplitude", "center", "sigma"]
elif function == "voigt":
params = ["amplitude", "center", "sigma", "gamma"]
elif function == "pvoigt":
params = ["amplitude", "center", "sigma", "fraction"]
elif function == "pseudovoigt1":
params = ["amplitude", "center", "g_sigma", "l_sigma", "fraction"]
else:
raise ValueError("Unknown fit function")
n = len(params)
fitparams = dict(
param=params,
value=[None] * n,
vary=[True] * n,
min=[None] * n,
max=[None] * n,
)
if function == "linear":
fitparams["value"] = [0, 1]
fitparams["vary"] = [False, True]
fitparams["min"] = [None, 0]
elif function == "gaussian":
fitparams["min"] = [0, None, None]
return fitparams
fitparams_table_source = ColumnDataSource(
dict(param=[], value=[], vary=[], min=[], max=[]))
fitparams_table = DataTable(
source=fitparams_table_source,
columns=[
TableColumn(field="param", title="Parameter"),
TableColumn(field="value", title="Value", editor=NumberEditor()),
TableColumn(field="vary", title="Vary", editor=CheckboxEditor()),
TableColumn(field="min", title="Min", editor=NumberEditor()),
TableColumn(field="max", title="Max", editor=NumberEditor()),
],
height=200,
width=350,
index_position=None,
editable=True,
auto_edit=True,
)
# start with `background` and `gauss` fit functions added
fitparams_add_dropdown_callback(types.SimpleNamespace(item="linear"))
fitparams_add_dropdown_callback(types.SimpleNamespace(item="gaussian"))
fitparams_select.value = ["gaussian-1"] # add selection to gauss
fit_output_textinput = TextAreaInput(title="Fit results:",
width=750,
height=200)
def proc_all_button_callback():
for scan, export in zip(det_data, scan_table_source.data["export"]):
if export:
pyzebra.fit_scan(scan,
fit_params,
fit_from=fit_from_spinner.value,
fit_to=fit_to_spinner.value)
pyzebra.get_area(
scan,
area_method=AREA_METHODS[area_method_radiobutton.active],
lorentz=lorentz_checkbox.active,
)
_update_plot()
_update_table()
for scan in det_data:
if "fit" in scan:
options = list(scan["fit"].params.keys())
fit_param_select.options = options
fit_param_select.value = options[0]
break
_update_param_plot()
proc_all_button = Button(label="Process All",
button_type="primary",
width=145)
proc_all_button.on_click(proc_all_button_callback)
def proc_button_callback():
scan = _get_selected_scan()
pyzebra.fit_scan(scan,
fit_params,
fit_from=fit_from_spinner.value,
fit_to=fit_to_spinner.value)
pyzebra.get_area(
scan,
area_method=AREA_METHODS[area_method_radiobutton.active],
lorentz=lorentz_checkbox.active,
)
_update_plot()
_update_table()
for scan in det_data:
if "fit" in scan:
options = list(scan["fit"].params.keys())
fit_param_select.options = options
fit_param_select.value = options[0]
break
_update_param_plot()
proc_button = Button(label="Process Current", width=145)
proc_button.on_click(proc_button_callback)
area_method_div = Div(text="Intensity:", margin=(5, 5, 0, 5))
area_method_radiobutton = RadioGroup(labels=["Function", "Area"],
active=0,
width=145)
lorentz_checkbox = CheckboxGroup(labels=["Lorentz Correction"],
width=145,
margin=(13, 5, 5, 5))
export_preview_textinput = TextAreaInput(title="Export file preview:",
width=450,
height=400)
def _update_preview():
with tempfile.TemporaryDirectory() as temp_dir:
temp_file = temp_dir + "/temp"
export_data = []
for s, export in zip(det_data, scan_table_source.data["export"]):
if export:
export_data.append(s)
# pyzebra.export_1D(export_data, temp_file, "fullprof")
exported_content = ""
file_content = []
for ext in (".comm", ".incomm"):
fname = temp_file + ext
if os.path.isfile(fname):
with open(fname) as f:
content = f.read()
exported_content += f"{ext} file:\n" + content
else:
content = ""
file_content.append(content)
js_data.data.update(content=file_content)
export_preview_textinput.value = exported_content
save_button = Button(label="Download File(s)",
button_type="success",
width=220)
save_button.js_on_click(
CustomJS(args={"js_data": js_data}, code=javaScript))
fitpeak_controls = row(
column(fitparams_add_dropdown, fitparams_select,
fitparams_remove_button),
fitparams_table,
Spacer(width=20),
column(fit_from_spinner, lorentz_checkbox, area_method_div,
area_method_radiobutton),
column(fit_to_spinner, proc_button, proc_all_button),
)
scan_layout = column(scan_table,
row(monitor_spinner, scan_motor_select, param_select))
import_layout = column(
proposal_textinput,
file_select,
row(file_open_button, file_append_button),
upload_div,
upload_button,
append_upload_div,
append_upload_button,
)
export_layout = column(export_preview_textinput, row(save_button))
tab_layout = column(
row(import_layout, scan_layout, plots, Spacer(width=30),
export_layout),
row(fitpeak_controls, fit_output_textinput),
)
return Panel(child=tab_layout, title="param study")
def bkapp(doc):
def fitButtonCallback(new):
x_range = [p.x_range.start, p.x_range.end]
y_range = [p.y_range.start, p.y_range.end]
x_inds = np.where(np.logical_and(bin_mids >= x_range[0], bin_mids <= x_range[1]), bin_mids, np.nan)
x = x_inds[np.isfinite(x_inds)]
y = hist[np.isfinite(x_inds)]
x_fit = np.linspace(x_range[0], x_range[1], 10000)
p0 = [y.max(), x[np.argmax(y)], x[np.argmax(y)], 0]
p_fit = curve_fit(_gaus, x, y, p0, ftol=1e-2)[0]
y_fit = _gaus(x_fit, *p_fit)
# p.line(x_fit, y_fit, legend_label='Amp. = %5.2f, Mean = %5.2f, Std. = %5.2f, Base = %5.2f' % p_fit)
fitDataSource.patch(dict(x=[(slice(0, len(x_fit)), x_fit)], y=[(slice(0, len(y_fit)), y_fit)]))
fit_str = 'Amp. = %5.2f, Mean = %5.2f, Std. = %5.2f, Base = %5.2f' % (p_fit[0], p_fit[1], p_fit[2], p_fit[3])
#p.line(x=x_fit, y=y_fit, legend_label=new_data['legend_label'], color='black')
p.x_range = Range1d(bins[0], bins[-1])
p.y_range = Range1d(0, hists[0].max())
p.legend.items = [(settings['sz_key'], [hist_line]), (fit_str, [fit_line])]
def SzDropDownCallback(new):
settings['sz_key'] = new.item
bins = np.linspace(0, 130000, 1000)
size_datasets = pysp2.util.calc_diams_masses(my_cal_data[settings['sz_key']])
hist, bins = np.histogram(size_datasets[settings['variable']].where(size_datasets.ScatRejectKey == 0).values,
bins=bins)
histDataSource.patch(dict(x=[(slice(0, len(bin_mids)), bin_mids)], y=[(slice(0, len(hist)), hist)]))
p.legend.items = [(settings['sz_key'], [hist_line]), ("", [fit_line])]
def VariableCallback(new):
settings['variable'] = new.item
bins = np.linspace(0, 130000, 1000)
size_datasets = pysp2.util.calc_diams_masses(my_cal_data[settings['sz_key']])
hist, bins = np.histogram(size_datasets[settings['variable']].where(size_datasets.ScatRejectKey == 0).values,
bins=bins)
histDataSource.patch(dict(x=[(slice(0, len(bin_mids)), bin_mids)], y=[(slice(0, len(hist)), hist)]))
p.legend.items = [(settings['sz_key'], [hist_line]), ("", [fit_line])]
settings = {'sz_key':'scat_200', 'variable': 'PkHt_ch0'}
sizes = np.array([350, 300, 269, 220, 200, 170])
scat_mean = np.ones_like(sizes, dtype=float)
size_datasets = {}
i = 0
# bins = np.logspace(-13, -10, 120)
bins = np.linspace(0, 130000, 1000)
hists = np.zeros((len(sizes), len(bins) - 1))
size_datasets = pysp2.util.calc_diams_masses(my_cal_data[settings['sz_key']])
hist, bins = np.histogram(size_datasets[settings['variable']].where(size_datasets.ScatRejectKey == 0).values,
bins=bins)
p = figure(tools="pan,box_zoom,reset,save", x_range=(bins[0], bins[-1]), y_range=(0, hist.max()))
bin_mids = (bins[:-1] + bins[1:])/2.0
histDataSource = ColumnDataSource(data=dict(x=bin_mids, y=hist))
x_fit = np.linspace(0, 130000, 10000)
fitDataSource = ColumnDataSource(data=dict(x=x_fit, y=np.nan*np.ones_like(x_fit)))
hist_line = p.line('x', 'y', source=histDataSource, legend_label=settings['sz_key'])
fit_line = p.line('x', 'y', source=fitDataSource, legend_label="")
button = Button(label="Curve fit")
button.on_click(fitButtonCallback)
size_menu = []
for keys in my_cal_data.keys():
size_menu.append((keys, keys))
SzDropDown = Dropdown(label="Calibration data", menu=size_menu)
SzDropDown.on_click(SzDropDownCallback)
vars = ["PkHt_ch0", "PkHt_ch1", "PkHt_ch4", "PkHt_ch5", "FtAmp_ch0", 'FtAmp_ch4']
vars = [(x, x) for x in vars]
VarDropDown = Dropdown(label="Variable", menu=vars)
VarDropDown.on_click(VariableCallback)
doc.add_root(column(button, SzDropDown, VarDropDown, p))
class BokehCorpusAnnotator(BokehCorpusExplorer):
"""
Annoate text data points via callbacks.
Features:
- alter values in the 'label' column through the widgets.
- **SERVER ONLY**: only works in a setting that allows Python callbacks.
"""
DATA_KEY_TO_KWARGS = {
_key: {
"constant": {
"line_alpha": 0.3
},
"search": {
"size": ("size", 10, 5, 7),
"fill_alpha": ("fill_alpha", 0.3, 0.1, 0.2),
},
}
for _key in ["raw", "train", "dev", "test"]
}
def __init__(self, df_dict, **kwargs):
"""Conceptually the same as the parent method."""
super().__init__(df_dict, **kwargs)
def _layout_widgets(self):
"""Define the layout of widgets."""
layout_rows = (
row(self.search_pos, self.search_neg),
row(self.data_key_button_group),
row(self.annotator_input, self.annotator_apply,
self.annotator_export),
)
return column(*layout_rows)
def _setup_widgets(self):
"""
Create annotator widgets and assign Python callbacks.
"""
from bokeh.models import TextInput, Button, Dropdown
super()._setup_widgets()
self.annotator_input = TextInput(title="Label:")
self.annotator_apply = Button(
label="Apply",
button_type="primary",
height_policy="fit",
width_policy="min",
)
self.annotator_export = Dropdown(
label="Export",
button_type="warning",
menu=["Excel", "CSV", "JSON", "pickle"],
height_policy="fit",
width_policy="min",
)
def callback_apply():
"""
A callback on clicking the 'self.annotator_apply' button.
Update labels in the source.
"""
label = self.annotator_input.value
selected_idx = self.sources["raw"].selected.indices
if not selected_idx:
self._warn(
"Attempting annotation: did not select any data points. Eligible subset is 'raw'."
)
return
example_old = self.dfs["raw"].at[selected_idx[0], "label"]
self.dfs["raw"].at[selected_idx, "label"] = label
example_new = self.dfs["raw"].at[selected_idx[0], "label"]
self._good(
f"Applied {len(selected_idx)} annotations: {label} (e.g. {example_old} -> {example_new}"
)
self._update_sources()
self.plot()
self._good(f"Updated annotator plot at {current_time()}")
def callback_export(event, path_root=None):
"""
A callback on clicking the 'self.annotator_export' button.
Saves the dataframe to a pickle.
"""
export_format = event.item
# auto-determine the export path root
if path_root is None:
timestamp = current_time("%Y%m%d%H%M%S")
path_root = f"hover-annotated-df-{timestamp}"
export_df = pd.concat(self.dfs,
axis=0,
sort=False,
ignore_index=True)
if export_format == "Excel":
export_path = f"{path_root}.xlsx"
export_df.to_excel(export_path, index=False)
elif export_format == "CSV":
export_path = f"{path_root}.csv"
export_df.to_csv(export_path, index=False)
elif export_format == "JSON":
export_path = f"{path_root}.json"
export_df.to_json(export_path, orient="records")
elif export_format == "pickle":
export_path = f"{path_root}.pkl"
export_df.to_pickle(export_path)
else:
raise ValueError(f"Unexpected export format {export_format}")
self._good(f"Saved DataFrame to {export_path}")
# keep the references to the callbacks
self._callback_apply = callback_apply
self._callback_export = callback_export
# assign callbacks
self.annotator_apply.on_click(self._callback_apply)
self.annotator_export.on_click(self._callback_export)
def plot(self):
"""
Re-plot with the new labels.
Overrides the parent method.
Determines the label->color mapping dynamically.
"""
labels, cmap = self.auto_labels_cmap()
for _key, _source in self.sources.items():
self.figure.circle(
"x",
"y",
name=_key,
color=factor_cmap("label", cmap, labels),
legend_group="label",
source=_source,
**self.glyph_kwargs[_key],
)
self._good(
f"Plotted subset {_key} with {self.dfs[_key].shape[0]} points")
self.auto_legend_correction()
source_function1 = ColumnDataSource(data=dict(x=[], y=[]))
source_function2 = ColumnDataSource(data=dict(x=[], y=[]))
source_result = ColumnDataSource(data=dict(x=[], y=[]))
source_convolution = ColumnDataSource(data=dict(x=[], y=[]))
source_xmarker = ColumnDataSource(data=dict(x=[], y=[]))
source_overlay = ColumnDataSource(data=dict(x=[], y=[], y_neg=[], y_pos=[]))
# initialize properties
update_is_enabled = True
# initialize controls
# dropdown menu for sample functions
function_type = Dropdown(
label="choose a sample function pair or enter one below",
menu=convolution_settings.sample_function_names)
function_type.on_click(function_pair_input_change)
# slider controlling the evaluated x value of the convolved function
x_value_input = Slider(title="x value",
name='x value',
value=convolution_settings.x_value_init,
start=convolution_settings.x_value_min,
end=convolution_settings.x_value_max,
step=convolution_settings.x_value_step)
x_value_input.on_change('value', input_change)
# text input for the first function to be convolved
function1_input = TextInput(value=convolution_settings.function1_input_init,
title="my first function:")
function1_input.on_change('value', input_change)
# text input for the second function to be convolved
function2_input = TextInput(value=convolution_settings.function1_input_init,
def create():
det_data = {}
fit_params = {}
js_data = ColumnDataSource(
data=dict(content=["", ""], fname=["", ""], ext=["", ""]))
def proposal_textinput_callback(_attr, _old, new):
proposal = new.strip()
for zebra_proposals_path in pyzebra.ZEBRA_PROPOSALS_PATHS:
proposal_path = os.path.join(zebra_proposals_path, proposal)
if os.path.isdir(proposal_path):
# found it
break
else:
raise ValueError(f"Can not find data for proposal '{proposal}'.")
file_list = []
for file in os.listdir(proposal_path):
if file.endswith((".ccl", ".dat")):
file_list.append((os.path.join(proposal_path, file), file))
file_select.options = file_list
file_open_button.disabled = False
file_append_button.disabled = False
proposal_textinput = TextInput(title="Proposal number:", width=210)
proposal_textinput.on_change("value", proposal_textinput_callback)
def _init_datatable():
scan_list = [s["idx"] for s in det_data]
hkl = [f'{s["h"]} {s["k"]} {s["l"]}' for s in det_data]
export = [s.get("active", True) for s in det_data]
scan_table_source.data.update(
scan=scan_list,
hkl=hkl,
fit=[0] * len(scan_list),
export=export,
)
scan_table_source.selected.indices = []
scan_table_source.selected.indices = [0]
merge_options = [(str(i), f"{i} ({idx})")
for i, idx in enumerate(scan_list)]
merge_from_select.options = merge_options
merge_from_select.value = merge_options[0][0]
file_select = MultiSelect(title="Available .ccl/.dat files:",
width=210,
height=250)
def file_open_button_callback():
nonlocal det_data
det_data = []
for f_name in file_select.value:
with open(f_name) as file:
base, ext = os.path.splitext(f_name)
if det_data:
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data,
monitor_spinner.value)
pyzebra.merge_datasets(det_data, append_data)
else:
det_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(det_data, monitor_spinner.value)
pyzebra.merge_duplicates(det_data)
js_data.data.update(fname=[base, base])
_init_datatable()
append_upload_button.disabled = False
file_open_button = Button(label="Open New", width=100, disabled=True)
file_open_button.on_click(file_open_button_callback)
def file_append_button_callback():
for f_name in file_select.value:
with open(f_name) as file:
_, ext = os.path.splitext(f_name)
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data, monitor_spinner.value)
pyzebra.merge_datasets(det_data, append_data)
_init_datatable()
file_append_button = Button(label="Append", width=100, disabled=True)
file_append_button.on_click(file_append_button_callback)
def upload_button_callback(_attr, _old, new):
nonlocal det_data
det_data = []
for f_str, f_name in zip(new, upload_button.filename):
with io.StringIO(base64.b64decode(f_str).decode()) as file:
base, ext = os.path.splitext(f_name)
if det_data:
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data,
monitor_spinner.value)
pyzebra.merge_datasets(det_data, append_data)
else:
det_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(det_data, monitor_spinner.value)
pyzebra.merge_duplicates(det_data)
js_data.data.update(fname=[base, base])
_init_datatable()
append_upload_button.disabled = False
upload_div = Div(text="or upload new .ccl/.dat files:",
margin=(5, 5, 0, 5))
upload_button = FileInput(accept=".ccl,.dat", multiple=True, width=200)
upload_button.on_change("value", upload_button_callback)
def append_upload_button_callback(_attr, _old, new):
for f_str, f_name in zip(new, append_upload_button.filename):
with io.StringIO(base64.b64decode(f_str).decode()) as file:
_, ext = os.path.splitext(f_name)
append_data = pyzebra.parse_1D(file, ext)
pyzebra.normalize_dataset(append_data, monitor_spinner.value)
pyzebra.merge_datasets(det_data, append_data)
_init_datatable()
append_upload_div = Div(text="append extra files:", margin=(5, 5, 0, 5))
append_upload_button = FileInput(accept=".ccl,.dat",
multiple=True,
width=200,
disabled=True)
append_upload_button.on_change("value", append_upload_button_callback)
def monitor_spinner_callback(_attr, old, new):
if det_data:
pyzebra.normalize_dataset(det_data, new)
_update_plot(_get_selected_scan())
monitor_spinner = Spinner(title="Monitor:",
mode="int",
value=100_000,
low=1,
width=145)
monitor_spinner.on_change("value", monitor_spinner_callback)
def _update_table():
fit_ok = [(1 if "fit" in scan else 0) for scan in det_data]
scan_table_source.data.update(fit=fit_ok)
def _update_plot(scan):
scan_motor = scan["scan_motor"]
y = scan["counts"]
x = scan[scan_motor]
plot.axis[0].axis_label = scan_motor
plot_scatter_source.data.update(x=x,
y=y,
y_upper=y + np.sqrt(y),
y_lower=y - np.sqrt(y))
fit = scan.get("fit")
if fit is not None:
x_fit = np.linspace(x[0], x[-1], 100)
plot_fit_source.data.update(x=x_fit, y=fit.eval(x=x_fit))
x_bkg = []
y_bkg = []
xs_peak = []
ys_peak = []
comps = fit.eval_components(x=x_fit)
for i, model in enumerate(fit_params):
if "linear" in model:
x_bkg = x_fit
y_bkg = comps[f"f{i}_"]
elif any(val in model
for val in ("gaussian", "voigt", "pvoigt")):
xs_peak.append(x_fit)
ys_peak.append(comps[f"f{i}_"])
plot_bkg_source.data.update(x=x_bkg, y=y_bkg)
plot_peak_source.data.update(xs=xs_peak, ys=ys_peak)
fit_output_textinput.value = fit.fit_report()
else:
plot_fit_source.data.update(x=[], y=[])
plot_bkg_source.data.update(x=[], y=[])
plot_peak_source.data.update(xs=[], ys=[])
fit_output_textinput.value = ""
# Main plot
plot = Plot(
x_range=DataRange1d(),
y_range=DataRange1d(only_visible=True),
plot_height=470,
plot_width=700,
)
plot.add_layout(LinearAxis(axis_label="Counts"), place="left")
plot.add_layout(LinearAxis(axis_label="Scan motor"), place="below")
plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
plot_scatter_source = ColumnDataSource(
dict(x=[0], y=[0], y_upper=[0], y_lower=[0]))
plot_scatter = plot.add_glyph(
plot_scatter_source, Scatter(x="x", y="y", line_color="steelblue"))
plot.add_layout(
Whisker(source=plot_scatter_source,
base="x",
upper="y_upper",
lower="y_lower"))
plot_fit_source = ColumnDataSource(dict(x=[0], y=[0]))
plot_fit = plot.add_glyph(plot_fit_source, Line(x="x", y="y"))
plot_bkg_source = ColumnDataSource(dict(x=[0], y=[0]))
plot_bkg = plot.add_glyph(
plot_bkg_source,
Line(x="x", y="y", line_color="green", line_dash="dashed"))
plot_peak_source = ColumnDataSource(dict(xs=[[0]], ys=[[0]]))
plot_peak = plot.add_glyph(
plot_peak_source,
MultiLine(xs="xs", ys="ys", line_color="red", line_dash="dashed"))
fit_from_span = Span(location=None, dimension="height", line_dash="dashed")
plot.add_layout(fit_from_span)
fit_to_span = Span(location=None, dimension="height", line_dash="dashed")
plot.add_layout(fit_to_span)
plot.add_layout(
Legend(
items=[
("data", [plot_scatter]),
("best fit", [plot_fit]),
("peak", [plot_peak]),
("linear", [plot_bkg]),
],
location="top_left",
click_policy="hide",
))
plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
plot.toolbar.logo = None
# Scan select
def scan_table_select_callback(_attr, old, new):
if not new:
# skip empty selections
return
# Avoid selection of multiple indicies (via Shift+Click or Ctrl+Click)
if len(new) > 1:
# drop selection to the previous one
scan_table_source.selected.indices = old
return
if len(old) > 1:
# skip unnecessary update caused by selection drop
return
_update_plot(det_data[new[0]])
def scan_table_source_callback(_attr, _old, _new):
_update_preview()
scan_table_source = ColumnDataSource(
dict(scan=[], hkl=[], fit=[], export=[]))
scan_table_source.on_change("data", scan_table_source_callback)
scan_table = DataTable(
source=scan_table_source,
columns=[
TableColumn(field="scan", title="Scan", width=50),
TableColumn(field="hkl", title="hkl", width=100),
TableColumn(field="fit", title="Fit", width=50),
TableColumn(field="export",
title="Export",
editor=CheckboxEditor(),
width=50),
],
width=310, # +60 because of the index column
height=350,
autosize_mode="none",
editable=True,
)
scan_table_source.selected.on_change("indices", scan_table_select_callback)
def _get_selected_scan():
return det_data[scan_table_source.selected.indices[0]]
merge_from_select = Select(title="scan:", width=145)
def merge_button_callback():
scan_into = _get_selected_scan()
scan_from = det_data[int(merge_from_select.value)]
if scan_into is scan_from:
print("WARNING: Selected scans for merging are identical")
return
pyzebra.merge_scans(scan_into, scan_from)
_update_plot(_get_selected_scan())
merge_button = Button(label="Merge into current", width=145)
merge_button.on_click(merge_button_callback)
def restore_button_callback():
pyzebra.restore_scan(_get_selected_scan())
_update_plot(_get_selected_scan())
restore_button = Button(label="Restore scan", width=145)
restore_button.on_click(restore_button_callback)
def fit_from_spinner_callback(_attr, _old, new):
fit_from_span.location = new
fit_from_spinner = Spinner(title="Fit from:", width=145)
fit_from_spinner.on_change("value", fit_from_spinner_callback)
def fit_to_spinner_callback(_attr, _old, new):
fit_to_span.location = new
fit_to_spinner = Spinner(title="to:", width=145)
fit_to_spinner.on_change("value", fit_to_spinner_callback)
def fitparams_add_dropdown_callback(click):
# bokeh requires (str, str) for MultiSelect options
new_tag = f"{click.item}-{fitparams_select.tags[0]}"
fitparams_select.options.append((new_tag, click.item))
fit_params[new_tag] = fitparams_factory(click.item)
fitparams_select.tags[0] += 1
fitparams_add_dropdown = Dropdown(
label="Add fit function",
menu=[
("Linear", "linear"),
("Gaussian", "gaussian"),
("Voigt", "voigt"),
("Pseudo Voigt", "pvoigt"),
# ("Pseudo Voigt1", "pseudovoigt1"),
],
width=145,
)
fitparams_add_dropdown.on_click(fitparams_add_dropdown_callback)
def fitparams_select_callback(_attr, old, new):
# Avoid selection of multiple indicies (via Shift+Click or Ctrl+Click)
if len(new) > 1:
# drop selection to the previous one
fitparams_select.value = old
return
if len(old) > 1:
# skip unnecessary update caused by selection drop
return
if new:
fitparams_table_source.data.update(fit_params[new[0]])
else:
fitparams_table_source.data.update(
dict(param=[], value=[], vary=[], min=[], max=[]))
fitparams_select = MultiSelect(options=[], height=120, width=145)
fitparams_select.tags = [0]
fitparams_select.on_change("value", fitparams_select_callback)
def fitparams_remove_button_callback():
if fitparams_select.value:
sel_tag = fitparams_select.value[0]
del fit_params[sel_tag]
for elem in fitparams_select.options:
if elem[0] == sel_tag:
fitparams_select.options.remove(elem)
break
fitparams_select.value = []
fitparams_remove_button = Button(label="Remove fit function", width=145)
fitparams_remove_button.on_click(fitparams_remove_button_callback)
def fitparams_factory(function):
if function == "linear":
params = ["slope", "intercept"]
elif function == "gaussian":
params = ["amplitude", "center", "sigma"]
elif function == "voigt":
params = ["amplitude", "center", "sigma", "gamma"]
elif function == "pvoigt":
params = ["amplitude", "center", "sigma", "fraction"]
elif function == "pseudovoigt1":
params = ["amplitude", "center", "g_sigma", "l_sigma", "fraction"]
else:
raise ValueError("Unknown fit function")
n = len(params)
fitparams = dict(
param=params,
value=[None] * n,
vary=[True] * n,
min=[None] * n,
max=[None] * n,
)
if function == "linear":
fitparams["value"] = [0, 1]
fitparams["vary"] = [False, True]
fitparams["min"] = [None, 0]
elif function == "gaussian":
fitparams["min"] = [0, None, None]
return fitparams
fitparams_table_source = ColumnDataSource(
dict(param=[], value=[], vary=[], min=[], max=[]))
fitparams_table = DataTable(
source=fitparams_table_source,
columns=[
TableColumn(field="param", title="Parameter"),
TableColumn(field="value", title="Value", editor=NumberEditor()),
TableColumn(field="vary", title="Vary", editor=CheckboxEditor()),
TableColumn(field="min", title="Min", editor=NumberEditor()),
TableColumn(field="max", title="Max", editor=NumberEditor()),
],
height=200,
width=350,
index_position=None,
editable=True,
auto_edit=True,
)
# start with `background` and `gauss` fit functions added
fitparams_add_dropdown_callback(types.SimpleNamespace(item="linear"))
fitparams_add_dropdown_callback(types.SimpleNamespace(item="gaussian"))
fitparams_select.value = ["gaussian-1"] # add selection to gauss
fit_output_textinput = TextAreaInput(title="Fit results:",
width=750,
height=200)
def proc_all_button_callback():
for scan, export in zip(det_data, scan_table_source.data["export"]):
if export:
pyzebra.fit_scan(scan,
fit_params,
fit_from=fit_from_spinner.value,
fit_to=fit_to_spinner.value)
pyzebra.get_area(
scan,
area_method=AREA_METHODS[area_method_radiobutton.active],
lorentz=lorentz_checkbox.active,
)
_update_plot(_get_selected_scan())
_update_table()
proc_all_button = Button(label="Process All",
button_type="primary",
width=145)
proc_all_button.on_click(proc_all_button_callback)
def proc_button_callback():
scan = _get_selected_scan()
pyzebra.fit_scan(scan,
fit_params,
fit_from=fit_from_spinner.value,
fit_to=fit_to_spinner.value)
pyzebra.get_area(
scan,
area_method=AREA_METHODS[area_method_radiobutton.active],
lorentz=lorentz_checkbox.active,
)
_update_plot(scan)
_update_table()
proc_button = Button(label="Process Current", width=145)
proc_button.on_click(proc_button_callback)
area_method_div = Div(text="Intensity:", margin=(5, 5, 0, 5))
area_method_radiobutton = RadioGroup(labels=["Function", "Area"],
active=0,
width=145)
lorentz_checkbox = CheckboxGroup(labels=["Lorentz Correction"],
width=145,
margin=(13, 5, 5, 5))
export_preview_textinput = TextAreaInput(title="Export file preview:",
width=500,
height=400)
def _update_preview():
with tempfile.TemporaryDirectory() as temp_dir:
temp_file = temp_dir + "/temp"
export_data = []
for s, export in zip(det_data, scan_table_source.data["export"]):
if export:
export_data.append(s)
pyzebra.export_1D(
export_data,
temp_file,
export_target_select.value,
hkl_precision=int(hkl_precision_select.value),
)
exported_content = ""
file_content = []
for ext in EXPORT_TARGETS[export_target_select.value]:
fname = temp_file + ext
if os.path.isfile(fname):
with open(fname) as f:
content = f.read()
exported_content += f"{ext} file:\n" + content
else:
content = ""
file_content.append(content)
js_data.data.update(content=file_content)
export_preview_textinput.value = exported_content
def export_target_select_callback(_attr, _old, new):
js_data.data.update(ext=EXPORT_TARGETS[new])
_update_preview()
export_target_select = Select(title="Export target:",
options=list(EXPORT_TARGETS.keys()),
value="fullprof",
width=80)
export_target_select.on_change("value", export_target_select_callback)
js_data.data.update(ext=EXPORT_TARGETS[export_target_select.value])
def hkl_precision_select_callback(_attr, _old, _new):
_update_preview()
hkl_precision_select = Select(title="hkl precision:",
options=["2", "3", "4"],
value="2",
width=80)
hkl_precision_select.on_change("value", hkl_precision_select_callback)
save_button = Button(label="Download File(s)",
button_type="success",
width=200)
save_button.js_on_click(
CustomJS(args={"js_data": js_data}, code=javaScript))
fitpeak_controls = row(
column(fitparams_add_dropdown, fitparams_select,
fitparams_remove_button),
fitparams_table,
Spacer(width=20),
column(fit_from_spinner, lorentz_checkbox, area_method_div,
area_method_radiobutton),
column(fit_to_spinner, proc_button, proc_all_button),
)
scan_layout = column(
scan_table,
row(monitor_spinner, column(Spacer(height=19), restore_button)),
row(column(Spacer(height=19), merge_button), merge_from_select),
)
import_layout = column(
proposal_textinput,
file_select,
row(file_open_button, file_append_button),
upload_div,
upload_button,
append_upload_div,
append_upload_button,
)
export_layout = column(
export_preview_textinput,
row(export_target_select, hkl_precision_select,
column(Spacer(height=19), row(save_button))),
)
tab_layout = column(
row(import_layout, scan_layout, plot, Spacer(width=30), export_layout),
row(fitpeak_controls, fit_output_textinput),
)
return Panel(child=tab_layout, title="ccl integrate")
def create(palm):
doc = curdoc()
# Calibration averaged waveforms per photon energy
waveform_plot = Plot(
title=Title(text="eTOF calibration waveforms"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=760,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
waveform_plot.toolbar.logo = None
waveform_plot_hovertool = HoverTool(
tooltips=[("energy, eV", "@en"), ("eTOF bin", "$x{0.}")])
waveform_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool(), waveform_plot_hovertool)
# ---- axes
waveform_plot.add_layout(LinearAxis(axis_label="eTOF time bin"),
place="below")
waveform_plot.add_layout(LinearAxis(axis_label="Intensity",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
waveform_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
waveform_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- multiline glyphs
waveform_ref_source = ColumnDataSource(dict(xs=[], ys=[], en=[]))
waveform_ref_multiline = waveform_plot.add_glyph(
waveform_ref_source, MultiLine(xs="xs", ys="ys", line_color="blue"))
waveform_str_source = ColumnDataSource(dict(xs=[], ys=[], en=[]))
waveform_str_multiline = waveform_plot.add_glyph(
waveform_str_source, MultiLine(xs="xs", ys="ys", line_color="red"))
# ---- legend
waveform_plot.add_layout(
Legend(items=[(
"reference",
[waveform_ref_multiline]), ("streaked",
[waveform_str_multiline])]))
waveform_plot.legend.click_policy = "hide"
# ---- vertical spans
photon_peak_ref_span = Span(location=0,
dimension="height",
line_dash="dashed",
line_color="blue")
photon_peak_str_span = Span(location=0,
dimension="height",
line_dash="dashed",
line_color="red")
waveform_plot.add_layout(photon_peak_ref_span)
waveform_plot.add_layout(photon_peak_str_span)
# Calibration fit plot
fit_plot = Plot(
title=Title(text="eTOF calibration fit"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
fit_plot.toolbar.logo = None
fit_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(), ResetTool())
# ---- axes
fit_plot.add_layout(LinearAxis(axis_label="Photoelectron peak shift"),
place="below")
fit_plot.add_layout(LinearAxis(axis_label="Photon energy, eV",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
fit_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
fit_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- circle glyphs
fit_ref_circle_source = ColumnDataSource(dict(x=[], y=[]))
fit_ref_circle = fit_plot.add_glyph(
fit_ref_circle_source, Circle(x="x", y="y", line_color="blue"))
fit_str_circle_source = ColumnDataSource(dict(x=[], y=[]))
fit_str_circle = fit_plot.add_glyph(fit_str_circle_source,
Circle(x="x", y="y", line_color="red"))
# ---- line glyphs
fit_ref_line_source = ColumnDataSource(dict(x=[], y=[]))
fit_ref_line = fit_plot.add_glyph(fit_ref_line_source,
Line(x="x", y="y", line_color="blue"))
fit_str_line_source = ColumnDataSource(dict(x=[], y=[]))
fit_str_line = fit_plot.add_glyph(fit_str_line_source,
Line(x="x", y="y", line_color="red"))
# ---- legend
fit_plot.add_layout(
Legend(items=[
("reference", [fit_ref_circle, fit_ref_line]),
("streaked", [fit_str_circle, fit_str_line]),
]))
fit_plot.legend.click_policy = "hide"
# Calibration results datatables
def datatable_ref_source_callback(_attr, _old_value, new_value):
for en, ps, use in zip(new_value["energy"], new_value["peak_pos_ref"],
new_value["use_in_fit"]):
palm.etofs["0"].calib_data.loc[
en, "calib_tpeak"] = ps if ps != "NaN" else np.nan
palm.etofs["0"].calib_data.loc[en, "use_in_fit"] = use
calib_res = {}
for etof_key in palm.etofs:
calib_res[etof_key] = palm.etofs[etof_key].fit_calibration_curve()
update_calibration_plot(calib_res)
datatable_ref_source = ColumnDataSource(
dict(energy=["", "", ""],
peak_pos_ref=["", "", ""],
use_in_fit=[True, True, True]))
datatable_ref_source.on_change("data", datatable_ref_source_callback)
datatable_ref = DataTable(
source=datatable_ref_source,
columns=[
TableColumn(field="energy",
title="Photon Energy, eV",
editor=IntEditor()),
TableColumn(field="peak_pos_ref",
title="Reference Peak",
editor=IntEditor()),
TableColumn(field="use_in_fit",
title=" ",
editor=CheckboxEditor(),
width=80),
],
index_position=None,
editable=True,
height=300,
width=250,
)
def datatable_str_source_callback(_attr, _old_value, new_value):
for en, ps, use in zip(new_value["energy"], new_value["peak_pos_str"],
new_value["use_in_fit"]):
palm.etofs["1"].calib_data.loc[
en, "calib_tpeak"] = ps if ps != "NaN" else np.nan
palm.etofs["1"].calib_data.loc[en, "use_in_fit"] = use
calib_res = {}
for etof_key in palm.etofs:
calib_res[etof_key] = palm.etofs[etof_key].fit_calibration_curve()
update_calibration_plot(calib_res)
datatable_str_source = ColumnDataSource(
dict(energy=["", "", ""],
peak_pos_str=["", "", ""],
use_in_fit=[True, True, True]))
datatable_str_source.on_change("data", datatable_str_source_callback)
datatable_str = DataTable(
source=datatable_str_source,
columns=[
TableColumn(field="energy",
title="Photon Energy, eV",
editor=IntEditor()),
TableColumn(field="peak_pos_str",
title="Streaked Peak",
editor=IntEditor()),
TableColumn(field="use_in_fit",
title=" ",
editor=CheckboxEditor(),
width=80),
],
index_position=None,
editable=True,
height=350,
width=250,
)
# eTOF calibration folder path text input
def path_textinput_callback(_attr, _old_value, _new_value):
path_periodic_update()
update_load_dropdown_menu()
path_textinput = TextInput(title="eTOF calibration path:",
value=os.path.join(os.path.expanduser("~")),
width=510)
path_textinput.on_change("value", path_textinput_callback)
# eTOF calibration eco scans dropdown
def scans_dropdown_callback(event):
scans_dropdown.label = event.item
scans_dropdown = Dropdown(label="ECO scans",
button_type="default",
menu=[])
scans_dropdown.on_click(scans_dropdown_callback)
# ---- etof scans periodic update
def path_periodic_update():
new_menu = []
if os.path.isdir(path_textinput.value):
for entry in os.scandir(path_textinput.value):
if entry.is_file() and entry.name.endswith(".json"):
new_menu.append((entry.name, entry.name))
scans_dropdown.menu = sorted(new_menu, reverse=True)
doc.add_periodic_callback(path_periodic_update, 5000)
path_tab = Panel(child=column(
path_textinput,
scans_dropdown,
),
title="Path")
upload_div = Div(text="Upload ECO scan (top) and all hdf5 files (bottom):")
# ECO scan upload FileInput
def eco_fileinput_callback(_attr, _old, new):
with io.BytesIO(base64.b64decode(new)) as eco_scan:
data = json.load(eco_scan)
print(data)
eco_fileinput = FileInput(accept=".json", disabled=True)
eco_fileinput.on_change("value", eco_fileinput_callback)
# HDF5 upload FileInput
def hdf5_fileinput_callback(_attr, _old, new):
for base64_str in new:
with io.BytesIO(base64.b64decode(base64_str)) as hdf5_file:
with h5py.File(hdf5_file, "r") as h5f:
print(h5f.keys())
hdf5_fileinput = FileInput(accept=".hdf5,.h5",
multiple=True,
disabled=True)
hdf5_fileinput.on_change("value", hdf5_fileinput_callback)
upload_tab = Panel(child=column(upload_div, eco_fileinput, hdf5_fileinput),
title="Upload")
# Calibrate button
def calibrate_button_callback():
try:
palm.calibrate_etof_eco(eco_scan_filename=os.path.join(
path_textinput.value, scans_dropdown.label))
except Exception:
palm.calibrate_etof(folder_name=path_textinput.value)
datatable_ref_source.data.update(
energy=palm.etofs["0"].calib_data.index.tolist(),
peak_pos_ref=palm.etofs["0"].calib_data["calib_tpeak"].tolist(),
use_in_fit=palm.etofs["0"].calib_data["use_in_fit"].tolist(),
)
datatable_str_source.data.update(
energy=palm.etofs["0"].calib_data.index.tolist(),
peak_pos_str=palm.etofs["1"].calib_data["calib_tpeak"].tolist(),
use_in_fit=palm.etofs["1"].calib_data["use_in_fit"].tolist(),
)
def update_calibration_plot(calib_res):
etof_ref = palm.etofs["0"]
etof_str = palm.etofs["1"]
shift_val = 0
etof_ref_wf_shifted = []
etof_str_wf_shifted = []
for wf_ref, wf_str in zip(etof_ref.calib_data["waveform"],
etof_str.calib_data["waveform"]):
shift_val -= max(wf_ref.max(), wf_str.max())
etof_ref_wf_shifted.append(wf_ref + shift_val)
etof_str_wf_shifted.append(wf_str + shift_val)
waveform_ref_source.data.update(
xs=len(etof_ref.calib_data) *
[list(range(etof_ref.internal_time_bins))],
ys=etof_ref_wf_shifted,
en=etof_ref.calib_data.index.tolist(),
)
waveform_str_source.data.update(
xs=len(etof_str.calib_data) *
[list(range(etof_str.internal_time_bins))],
ys=etof_str_wf_shifted,
en=etof_str.calib_data.index.tolist(),
)
photon_peak_ref_span.location = etof_ref.calib_t0
photon_peak_str_span.location = etof_str.calib_t0
def plot_fit(time, calib_a, calib_b):
time_fit = np.linspace(np.nanmin(time), np.nanmax(time), 100)
en_fit = (calib_a / time_fit)**2 + calib_b
return time_fit, en_fit
def update_plot(calib_results, circle, line):
(a, c), x, y = calib_results
x_fit, y_fit = plot_fit(x, a, c)
circle.data.update(x=x, y=y)
line.data.update(x=x_fit, y=y_fit)
update_plot(calib_res["0"], fit_ref_circle_source, fit_ref_line_source)
update_plot(calib_res["1"], fit_str_circle_source, fit_str_line_source)
calib_const_div.text = f"""
a_str = {etof_str.calib_a:.2f}<br>
b_str = {etof_str.calib_b:.2f}<br>
<br>
a_ref = {etof_ref.calib_a:.2f}<br>
b_ref = {etof_ref.calib_b:.2f}
"""
calibrate_button = Button(label="Calibrate eTOF",
button_type="default",
width=250)
calibrate_button.on_click(calibrate_button_callback)
# Photon peak noise threshold value text input
def phot_peak_noise_thr_spinner_callback(_attr, old_value, new_value):
if new_value > 0:
for etof in palm.etofs.values():
etof.photon_peak_noise_thr = new_value
else:
phot_peak_noise_thr_spinner.value = old_value
phot_peak_noise_thr_spinner = Spinner(title="Photon peak noise threshold:",
value=1,
step=0.1)
phot_peak_noise_thr_spinner.on_change(
"value", phot_peak_noise_thr_spinner_callback)
# Electron peak noise threshold value text input
def el_peak_noise_thr_spinner_callback(_attr, old_value, new_value):
if new_value > 0:
for etof in palm.etofs.values():
etof.electron_peak_noise_thr = new_value
else:
el_peak_noise_thr_spinner.value = old_value
el_peak_noise_thr_spinner = Spinner(title="Electron peak noise threshold:",
value=10,
step=0.1)
el_peak_noise_thr_spinner.on_change("value",
el_peak_noise_thr_spinner_callback)
# Save calibration button
def save_button_callback():
palm.save_etof_calib(path=path_textinput.value)
update_load_dropdown_menu()
save_button = Button(label="Save", button_type="default", width=250)
save_button.on_click(save_button_callback)
# Load calibration button
def load_dropdown_callback(event):
new_value = event.item
if new_value:
palm.load_etof_calib(os.path.join(path_textinput.value, new_value))
datatable_ref_source.data.update(
energy=palm.etofs["0"].calib_data.index.tolist(),
peak_pos_ref=palm.etofs["0"].calib_data["calib_tpeak"].tolist(
),
use_in_fit=palm.etofs["0"].calib_data["use_in_fit"].tolist(),
)
datatable_str_source.data.update(
energy=palm.etofs["0"].calib_data.index.tolist(),
peak_pos_str=palm.etofs["1"].calib_data["calib_tpeak"].tolist(
),
use_in_fit=palm.etofs["1"].calib_data["use_in_fit"].tolist(),
)
def update_load_dropdown_menu():
new_menu = []
calib_file_ext = ".palm_etof"
if os.path.isdir(path_textinput.value):
for entry in os.scandir(path_textinput.value):
if entry.is_file() and entry.name.endswith((calib_file_ext)):
new_menu.append(
(entry.name[:-len(calib_file_ext)], entry.name))
load_dropdown.button_type = "default"
load_dropdown.menu = sorted(new_menu, reverse=True)
else:
load_dropdown.button_type = "danger"
load_dropdown.menu = new_menu
doc.add_next_tick_callback(update_load_dropdown_menu)
doc.add_periodic_callback(update_load_dropdown_menu, 5000)
load_dropdown = Dropdown(label="Load", menu=[], width=250)
load_dropdown.on_click(load_dropdown_callback)
# eTOF fitting equation
fit_eq_div = Div(
text="""Fitting equation:<br><br><img src="/palm/static/5euwuy.gif">"""
)
# Calibration constants
calib_const_div = Div(text=f"""
a_str = {0}<br>
b_str = {0}<br>
<br>
a_ref = {0}<br>
b_ref = {0}
""")
# assemble
tab_layout = column(
row(
column(waveform_plot, fit_plot),
Spacer(width=30),
column(
Tabs(tabs=[path_tab, upload_tab]),
calibrate_button,
phot_peak_noise_thr_spinner,
el_peak_noise_thr_spinner,
row(save_button, load_dropdown),
row(datatable_ref, datatable_str),
calib_const_div,
fit_eq_div,
),
))
return Panel(child=tab_layout, title="eTOF Calibration")
class SupervisableDataset(Loggable):
"""
???+ note "Feature-agnostic class for a dataset open to supervision."
Keeping a DataFrame form and a list-of-dicts ("dictl") form, with the intention that
- the DataFrame form supports most kinds of operations;
- the list-of-dicts form could be useful for manipulations outside the scope of pandas;
- synchronization between the two forms should be called sparingly.
"""
# 'scratch': intended to be directly editable by other objects, i.e. Explorers
# labels will be stored but not used for information in hover itself
SCRATCH_SUBSETS = tuple(["raw"])
# non-'scratch': intended to be read-only outside of the class
# 'public': labels will be considered as part of the classification task and will be used for built-in supervision
PUBLIC_SUBSETS = tuple(["train", "dev"])
# 'private': labels will be considered as part of the classification task and will NOT be used for supervision
PRIVATE_SUBSETS = tuple(["test"])
FEATURE_KEY = "feature"
def __init__(
self,
raw_dictl,
train_dictl=None,
dev_dictl=None,
test_dictl=None,
feature_key="feature",
label_key="label",
):
"""
???+ note "Create (1) dictl and df forms and (2) the mapping between categorical and string labels."
| Param | Type | Description |
| :------------ | :----- | :----------------------------------- |
| `raw_dictl` | `list` | list of dicts holding the **to-be-supervised** raw data |
| `train_dictl` | `list` | list of dicts holding any **supervised** train data |
| `dev_dictl` | `list` | list of dicts holding any **supervised** dev data |
| `test_dictl` | `list` | list of dicts holding any **supervised** test data |
| `feature_key` | `str` | the key for the feature in each piece of data |
| `label_key` | `str` | the key for the `**str**` label in supervised data |
"""
self._info("Initializing...")
def dictl_transform(dictl, labels=True):
"""
Burner function to transform the input list of dictionaries into standard format.
"""
# edge case when dictl is empty or None
if not dictl:
return []
# transform the feature and possibly the label
key_transform = {feature_key: self.__class__.FEATURE_KEY}
if labels:
key_transform[label_key] = "label"
def burner(d):
"""
Burner function to transform a single dict.
"""
if labels:
assert label_key in d, f"Expected dict key {label_key}"
trans_d = {
key_transform.get(_k, _k): _v
for _k, _v in d.items()
}
if not labels:
trans_d["label"] = module_config.ABSTAIN_DECODED
return trans_d
return [burner(_d) for _d in dictl]
self.dictls = {
"raw": dictl_transform(raw_dictl, labels=False),
"train": dictl_transform(train_dictl),
"dev": dictl_transform(dev_dictl),
"test": dictl_transform(test_dictl),
}
self.synchronize_dictl_to_df()
self.df_deduplicate()
self.synchronize_df_to_dictl()
self.setup_widgets()
# self.setup_label_coding() # redundant if setup_pop_table() immediately calls this again
self.setup_file_export()
self.setup_pop_table(width_policy="fit", height_policy="fit")
self.setup_sel_table(width_policy="fit", height_policy="fit")
self._good(f"{self.__class__.__name__}: finished initialization.")
def copy(self, use_df=True):
"""
???+ note "Create another instance, copying over the data entries."
| Param | Type | Description |
| :------- | :----- | :----------------------------------- |
| `use_df` | `bool` | whether to use the df or dictl form |
"""
if use_df:
self.synchronize_df_to_dictl()
return self.__class__(
raw_dictl=self.dictls["raw"],
train_dictl=self.dictls["train"],
dev_dictl=self.dictls["dev"],
test_dictl=self.dictls["test"],
feature_key=self.__class__.FEATURE_KEY,
label_key="label",
)
def to_pandas(self, use_df=True):
"""
???+ note "Export to a pandas DataFrame."
| Param | Type | Description |
| :------- | :----- | :----------------------------------- |
| `use_df` | `bool` | whether to use the df or dictl form |
"""
if not use_df:
self.synchronize_dictl_to_df()
dfs = []
for _subset in ["raw", "train", "dev", "test"]:
_df = self.dfs[_subset].copy()
_df[DATASET_SUBSET_FIELD] = _subset
dfs.append(_df)
return pd.concat(dfs, axis=0)
@classmethod
def from_pandas(cls, df, **kwargs):
"""
???+ note "Import from a pandas DataFrame."
| Param | Type | Description |
| :------- | :----- | :----------------------------------- |
| `df` | `DataFrame` | with a "SUBSET" field dividing subsets |
"""
SUBSETS = cls.SCRATCH_SUBSETS + cls.PUBLIC_SUBSETS + cls.PRIVATE_SUBSETS
if DATASET_SUBSET_FIELD not in df.columns:
raise ValueError(
f"Expecting column '{DATASET_SUBSET_FIELD}' in the DataFrame which takes values from {SUBSETS}"
)
dictls = {}
for _subset in ["raw", "train", "dev", "test"]:
_sub_df = df[df[DATASET_SUBSET_FIELD] == _subset]
dictls[_subset] = _sub_df.to_dict(orient="records")
return cls(
raw_dictl=dictls["raw"],
train_dictl=dictls["train"],
dev_dictl=dictls["dev"],
test_dictl=dictls["test"],
**kwargs,
)
def setup_widgets(self):
"""
???+ note "Create `bokeh` widgets for interactive data management."
"""
self.update_pusher = Button(label="Push",
button_type="success",
height_policy="fit",
width_policy="min")
self.data_committer = Dropdown(
label="Commit",
button_type="warning",
menu=[
*self.__class__.PUBLIC_SUBSETS, *self.__class__.PRIVATE_SUBSETS
],
height_policy="fit",
width_policy="min",
)
self.dedup_trigger = Button(
label="Dedup",
button_type="warning",
height_policy="fit",
width_policy="min",
)
self.selection_viewer = Button(
label="View Selected",
button_type="primary",
height_policy="fit",
width_policy="min",
)
def commit_base_callback():
"""
COMMIT creates cross-duplicates between subsets.
- PUSH shall be blocked until DEDUP is executed.
"""
self.dedup_trigger.disabled = False
self.update_pusher.disabled = True
def dedup_base_callback():
"""
DEDUP re-creates dfs with different indices than before.
- COMMIT shall be blocked until PUSH is executed.
"""
self.update_pusher.disabled = False
self.data_committer.disabled = True
self.df_deduplicate()
def push_base_callback():
"""
PUSH enforces df consistency with all linked explorers.
- DEDUP could be blocked because it stays trivial until COMMIT is executed.
"""
self.data_committer.disabled = False
self.dedup_trigger.disabled = True
self.update_pusher.on_click(push_base_callback)
self.data_committer.on_click(commit_base_callback)
self.dedup_trigger.on_click(dedup_base_callback)
self.help_div = dataset_help_widget()
def view(self):
"""
???+ note "Defines the layout of `bokeh` objects when visualized."
"""
# local import to avoid naming confusion/conflicts
from bokeh.layouts import row, column
return column(
self.help_div,
row(
self.update_pusher,
self.data_committer,
self.dedup_trigger,
self.selection_viewer,
self.file_exporter,
),
self.pop_table,
self.sel_table,
)
def subscribe_update_push(self, explorer, subset_mapping):
"""
???+ note "Enable pushing updated DataFrames to explorers that depend on them."
| Param | Type | Description |
| :--------------- | :----- | :------------------------------------- |
| `explorer` | `BokehBaseExplorer` | the explorer to register |
| `subset_mapping` | `dict` | `dataset` -> `explorer` subset mapping |
Note: the reason we need this is due to `self.dfs[key] = ...`-like assignments. If DF operations were all in-place, then the explorers could directly access the updates through their `self.dfs` references.
"""
# local import to avoid import cycles
from hover.core.explorer.base import BokehBaseExplorer
assert isinstance(explorer, BokehBaseExplorer)
def callback_push():
df_dict = {_v: self.dfs[_k] for _k, _v in subset_mapping.items()}
explorer._setup_dfs(df_dict)
explorer._update_sources()
self.update_pusher.on_click(callback_push)
self._good(
f"Subscribed {explorer.__class__.__name__} to dataset pushes: {subset_mapping}"
)
def subscribe_data_commit(self, explorer, subset_mapping):
"""
???+ note "Enable committing data across subsets, specified by a selection in an explorer and a dropdown widget of the dataset."
| Param | Type | Description |
| :--------------- | :----- | :------------------------------------- |
| `explorer` | `BokehBaseExplorer` | the explorer to register |
| `subset_mapping` | `dict` | `dataset` -> `explorer` subset mapping |
"""
def callback_commit(event):
for sub_k, sub_v in subset_mapping.items():
sub_to = event.item
selected_idx = explorer.sources[sub_v].selected.indices
if not selected_idx:
self._warn(
f"Attempting data commit: did not select any data points in subset {sub_v}."
)
return
# take selected slice, ignoring ABSTAIN'ed rows
# CAUTION: applying selected_idx from explorer.source to self.df
# this assumes that the source and the df have consistent entries.
# Consider this:
# keep_cols = self.dfs[sub_k].columns
# sel_slice = explorer.dfs[sub_v].iloc[selected_idx][keep_cols]
sel_slice = self.dfs[sub_k].iloc[selected_idx]
valid_slice = sel_slice[
sel_slice["label"] != module_config.ABSTAIN_DECODED]
# concat to the end and do some accounting
size_before = self.dfs[sub_to].shape[0]
self.dfs[sub_to] = pd.concat(
[self.dfs[sub_to], valid_slice],
axis=0,
sort=False,
ignore_index=True,
)
size_mid = self.dfs[sub_to].shape[0]
self.dfs[sub_to].drop_duplicates(
subset=[self.__class__.FEATURE_KEY],
keep="last",
inplace=True)
size_after = self.dfs[sub_to].shape[0]
self._info(
f"Committed {valid_slice.shape[0]} (valid out of {sel_slice.shape[0]} selected) entries from {sub_k} to {sub_to} ({size_before} -> {size_after} with {size_mid-size_after} overwrites)."
)
# chain another callback
self._callback_update_population()
self.data_committer.on_click(callback_commit)
self._good(
f"Subscribed {explorer.__class__.__name__} to dataset commits: {subset_mapping}"
)
def subscribe_selection_view(self, explorer, subsets):
"""
???+ note "Enable viewing groups of data entries, specified by a selection in an explorer."
| Param | Type | Description |
| :--------------- | :----- | :------------------------------------- |
| `explorer` | `BokehBaseExplorer` | the explorer to register |
| `subsets` | `list` | subset selections to consider |
"""
assert (isinstance(subsets, list)
and len(subsets) > 0), "Expected a non-empty list of subsets"
def callback_view():
sel_slices = []
for subset in subsets:
selected_idx = explorer.sources[subset].selected.indices
sub_slice = explorer.dfs[subset].iloc[selected_idx]
sel_slices.append(sub_slice)
selected = pd.concat(sel_slices, axis=0)
# replace this with an actual display (and analysis)
self._callback_update_selection(selected)
self.selection_viewer.on_click(callback_view)
self._good(
f"Subscribed {explorer.__class__.__name__} to selection view: {subsets}"
)
def setup_label_coding(self, verbose=True, debug=False):
"""
???+ note "Auto-determine labels in the dataset, then create encoder/decoder in lexical order."
Add `"ABSTAIN"` as a no-label placeholder which gets ignored categorically.
| Param | Type | Description |
| :-------- | :----- | :--------------------------------- |
| `verbose` | `bool` | whether to log verbosely |
| `debug` | `bool` | whether to enable label validation |
"""
all_labels = set()
for _key in [
*self.__class__.PUBLIC_SUBSETS, *self.__class__.PRIVATE_SUBSETS
]:
_df = self.dfs[_key]
_found_labels = set(_df["label"].tolist())
all_labels = all_labels.union(_found_labels)
# exclude ABSTAIN from self.classes, but include it in the encoding
all_labels.discard(module_config.ABSTAIN_DECODED)
self.classes = sorted(all_labels)
self.label_encoder = {
**{_label: _i
for _i, _label in enumerate(self.classes)},
module_config.ABSTAIN_DECODED: module_config.ABSTAIN_ENCODED,
}
self.label_decoder = {_v: _k for _k, _v in self.label_encoder.items()}
if verbose:
self._good(
f"Set up label encoder/decoder with {len(self.classes)} classes."
)
if debug:
self.validate_labels()
def validate_labels(self, raise_exception=True):
"""
???+ note "Assert that every label is in the encoder."
| Param | Type | Description |
| :---------------- | :----- | :---------------------------------- |
| `raise_exception` | `bool` | whether to raise errors when failed |
"""
for _key in [
*self.__class__.PUBLIC_SUBSETS, *self.__class__.PRIVATE_SUBSETS
]:
_invalid_indices = None
assert "label" in self.dfs[_key].columns
_mask = self.dfs[_key]["label"].apply(
lambda x: x in self.label_encoder)
_invalid_indices = np.where(_mask is False)[0].tolist()
if _invalid_indices:
self._fail(f"Subset {_key} has invalid labels:")
self._print({self.dfs[_key].loc[_invalid_indices]})
if raise_exception:
raise ValueError("invalid labels")
def setup_file_export(self):
self.file_exporter = Dropdown(
label="Export",
button_type="warning",
menu=["Excel", "CSV", "JSON", "pickle"],
height_policy="fit",
width_policy="min",
)
def callback_export(event, path_root=None):
"""
A callback on clicking the 'self.annotator_export' button.
Saves the dataframe to a pickle.
"""
export_format = event.item
# auto-determine the export path root
if path_root is None:
timestamp = current_time("%Y%m%d%H%M%S")
path_root = f"hover-dataset-export-{timestamp}"
export_df = self.to_pandas(use_df=True)
if export_format == "Excel":
export_path = f"{path_root}.xlsx"
export_df.to_excel(export_path, index=False)
elif export_format == "CSV":
export_path = f"{path_root}.csv"
export_df.to_csv(export_path, index=False)
elif export_format == "JSON":
export_path = f"{path_root}.json"
export_df.to_json(export_path, orient="records")
elif export_format == "pickle":
export_path = f"{path_root}.pkl"
export_df.to_pickle(export_path)
else:
raise ValueError(f"Unexpected export format {export_format}")
self._good(f"Saved DataFrame to {export_path}")
# assign the callback, keeping its reference
self._callback_export = callback_export
self.file_exporter.on_click(self._callback_export)
def setup_pop_table(self, **kwargs):
"""
???+ note "Set up a bokeh `DataTable` widget for monitoring subset data populations."
| Param | Type | Description |
| :--------- | :----- | :--------------------------- |
| `**kwargs` | | forwarded to the `DataTable` |
"""
subsets = [
*self.__class__.SCRATCH_SUBSETS,
*self.__class__.PUBLIC_SUBSETS,
*self.__class__.PRIVATE_SUBSETS,
]
pop_source = ColumnDataSource(dict())
pop_columns = [
TableColumn(field="label", title="label"),
*[
TableColumn(field=f"count_{_subset}", title=_subset)
for _subset in subsets
],
TableColumn(
field="color",
title="color",
formatter=HTMLTemplateFormatter(template=COLOR_GLYPH_TEMPLATE),
),
]
self.pop_table = DataTable(source=pop_source,
columns=pop_columns,
**kwargs)
def update_population():
"""
Callback function.
"""
# make sure that the label coding is correct
self.setup_label_coding()
# re-compute label population
eff_labels = [module_config.ABSTAIN_DECODED, *self.classes]
color_dict = auto_label_color(self.classes)
eff_colors = [color_dict[_label] for _label in eff_labels]
pop_data = dict(color=eff_colors, label=eff_labels)
for _subset in subsets:
_subpop = self.dfs[_subset]["label"].value_counts()
pop_data[f"count_{_subset}"] = [
_subpop.get(_label, 0) for _label in eff_labels
]
# push results to bokeh data source
pop_source.data = pop_data
self._good(
f"Population updater: latest population with {len(self.classes)} classes."
)
update_population()
self.dedup_trigger.on_click(update_population)
# store the callback so that it can be referenced by other methods
self._callback_update_population = update_population
def setup_sel_table(self, **kwargs):
"""
???+ note "Set up a bokeh `DataTable` widget for viewing selected data points."
| Param | Type | Description |
| :--------- | :----- | :--------------------------- |
| `**kwargs` | | forwarded to the `DataTable` |
"""
def auto_columns(df):
return [TableColumn(field=_col, title=_col) for _col in df.columns]
sel_source = ColumnDataSource(dict())
sel_columns = auto_columns(self.dfs["train"])
self.sel_table = DataTable(source=sel_source,
columns=sel_columns,
**kwargs)
def update_selection(selected_df):
"""
Callback function.
"""
# push results to bokeh data source
self.sel_table.columns = auto_columns(selected_df)
sel_source.data = selected_df.to_dict(orient="list")
self._good(
f"Selection table: latest selection with {selected_df.shape[0]} entries."
)
self._callback_update_selection = update_selection
def df_deduplicate(self):
"""
???+ note "Cross-deduplicate data entries by feature between subsets."
"""
self._info("Deduplicating...")
# for data entry accounting
before, after = dict(), dict()
# deduplicating rule: entries that come LATER are of higher priority
ordered_subsets = [
*self.__class__.SCRATCH_SUBSETS,
*self.__class__.PUBLIC_SUBSETS,
*self.__class__.PRIVATE_SUBSETS,
]
# keep track of which df has which columns and which rows came from which subset
columns = dict()
for _key in ordered_subsets:
before[_key] = self.dfs[_key].shape[0]
columns[_key] = self.dfs[_key].columns
self.dfs[_key]["__subset"] = _key
# concatenate in order and deduplicate
overall_df = pd.concat([self.dfs[_key] for _key in ordered_subsets],
axis=0,
sort=False)
overall_df.drop_duplicates(subset=[self.__class__.FEATURE_KEY],
keep="last",
inplace=True)
overall_df.reset_index(drop=True, inplace=True)
# cut up slices
for _key in ordered_subsets:
self.dfs[_key] = overall_df[
overall_df["__subset"] == _key].reset_index(
drop=True, inplace=False)[columns[_key]]
after[_key] = self.dfs[_key].shape[0]
self._info(
f"--subset {_key} rows: {before[_key]} -> {after[_key]}.")
def synchronize_dictl_to_df(self):
"""
???+ note "Re-make dataframes from lists of dictionaries."
"""
self.dfs = dict()
for _key, _dictl in self.dictls.items():
if _dictl:
_df = pd.DataFrame(_dictl)
assert self.__class__.FEATURE_KEY in _df.columns
assert "label" in _df.columns
else:
_df = pd.DataFrame(
columns=[self.__class__.FEATURE_KEY, "label"])
self.dfs[_key] = _df
def synchronize_df_to_dictl(self):
"""
???+ note "Re-make lists of dictionaries from dataframes."
"""
self.dictls = dict()
for _key, _df in self.dfs.items():
self.dictls[_key] = _df.to_dict(orient="records")
def compute_2d_embedding(self, vectorizer, method, **kwargs):
"""
???+ note "Get embeddings in the xy-plane and return the dimensionality reducer."
Reference: [`DimensionalityReducer`](https://github.com/phurwicz/hover/blob/main/hover/core/representation/reduction.py)
| Param | Type | Description |
| :----------- | :--------- | :--------------------------------- |
| `vectorizer` | `callable` | the feature -> vector function |
| `method` | `str` | arg for `DimensionalityReducer` |
| `**kwargs` | | kwargs for `DimensionalityReducer` |
"""
from hover.core.representation.reduction import DimensionalityReducer
# prepare input vectors to manifold learning
fit_subset = [
*self.__class__.SCRATCH_SUBSETS, *self.__class__.PUBLIC_SUBSETS
]
trans_subset = [*self.__class__.PRIVATE_SUBSETS]
assert not set(fit_subset).intersection(
set(trans_subset)), "Unexpected overlap"
# compute vectors and keep track which where to slice the array for fitting
feature_inp = []
for _key in fit_subset:
feature_inp += self.dfs[_key][self.__class__.FEATURE_KEY].tolist()
fit_num = len(feature_inp)
for _key in trans_subset:
feature_inp += self.dfs[_key][self.__class__.FEATURE_KEY].tolist()
trans_arr = np.array([vectorizer(_inp) for _inp in tqdm(feature_inp)])
# initialize and fit manifold learning reducer using specified subarray
self._info(
f"Fit-transforming {method.upper()} on {fit_num} samples...")
reducer = DimensionalityReducer(trans_arr[:fit_num])
fit_embedding = reducer.fit_transform(method, **kwargs)
# compute embedding of the whole dataset
self._info(
f"Transforming {method.upper()} on {trans_arr.shape[0]-fit_num} samples..."
)
trans_embedding = reducer.transform(trans_arr[fit_num:], method)
# assign x and y coordinates to dataset
start_idx = 0
for _subset, _embedding in [
(fit_subset, fit_embedding),
(trans_subset, trans_embedding),
]:
# edge case: embedding is too small
if _embedding.shape[0] < 1:
for _key in _subset:
assert (self.dfs[_key].shape[0] == 0
), "Expected empty df due to empty embedding"
continue
for _key in _subset:
_length = self.dfs[_key].shape[0]
self.dfs[_key]["x"] = pd.Series(
_embedding[start_idx:(start_idx + _length), 0])
self.dfs[_key]["y"] = pd.Series(
_embedding[start_idx:(start_idx + _length), 1])
start_idx += _length
return reducer
def loader(self, key, *vectorizers, batch_size=64, smoothing_coeff=0.0):
"""
???+ note "Prepare a torch `Dataloader` for training or evaluation."
| Param | Type | Description |
| :------------ | :------------ | :--------------------------------- |
| `key` | `str` | subset of data, e.g. `"train"` |
| `vectorizers` | `callable`(s) | the feature -> vector function(s) |
| `batch_size` | `int` | size per batch |
| `smoothing_coeff` | `float` | portion of probability to equally split between classes |
"""
# lazy import: missing torch should not break the rest of the class
from hover.utils.torch_helper import (
VectorDataset,
MultiVectorDataset,
one_hot,
label_smoothing,
)
# take the slice that has a meaningful label
df = self.dfs[key][
self.dfs[key]["label"] != module_config.ABSTAIN_DECODED]
# edge case: valid slice is too small
if df.shape[0] < 1:
raise ValueError(
f"Subset {key} has too few samples ({df.shape[0]})")
batch_size = min(batch_size, df.shape[0])
# prepare output vectors
labels = df["label"].apply(lambda x: self.label_encoder[x]).tolist()
output_vectors = one_hot(labels, num_classes=len(self.classes))
if smoothing_coeff > 0.0:
output_vectors = label_smoothing(output_vectors,
coefficient=smoothing_coeff)
# prepare input vectors
assert len(vectorizers) > 0, "Expected at least one vectorizer"
multi_flag = len(vectorizers) > 1
features = df[self.__class__.FEATURE_KEY].tolist()
input_vector_lists = []
for _vec_func in vectorizers:
self._info(f"Preparing {key} input vectors...")
_input_vecs = [_vec_func(_f) for _f in tqdm(features)]
input_vector_lists.append(_input_vecs)
self._info(f"Preparing {key} data loader...")
if multi_flag:
assert len(
input_vector_lists) > 1, "Expected multiple lists of vectors"
loader = MultiVectorDataset(
input_vector_lists,
output_vectors).loader(batch_size=batch_size)
else:
assert len(
input_vector_lists) == 1, "Expected only one list of vectors"
input_vectors = input_vector_lists[0]
loader = VectorDataset(
input_vectors, output_vectors).loader(batch_size=batch_size)
self._good(
f"Prepared {key} loader with {len(features)} examples; {len(vectorizers)} vectors per feature, batch size {batch_size}"
)
return loader
dict["timestamp_"+attr] = []
dict[attr] = []
# The data sources for the main plot and the task boxes
source = ColumnDataSource(dict)
task_source = ColumnDataSource({"tasks" : [], "time" : []})
# The data source for the plo displaying the active tasks over time
session_source = ColumnDataSource({"xss": [], "yss": [], "colors": [], "tasktype": [], "running_tasks": []})
source.data, task_source.data, session_source.data = query_events(current_run)
# a dropdown menu to select a run for which data shall be visualized
run_menu = [(run_format(k, v['numLogEntries'], v['tstart']), add_prefix(k)) for k, v in run_map.iteritems() if v['tstart'] is not None] # use None for separator
dropdown = Dropdown(label="run", button_type="warning", menu=run_menu)
dropdown.on_click(lambda newValue: select_run(rem_prefix(newValue)))
manualLegendBox = Div(text=legendFormat(task_types), width=PLOT_WIDTH, height=80)
# info boxes to display the number of log messages in this run and the elapsed wall clock time
numMessages = Div(text=infoBoxFormat("Messages", 0), width=200, height=100)
runID = Div(text=infoBoxFormat("run", str(current_run)), width=400, height=100)
startTime = Div(text=infoBoxFormat("start time [ms]", str(general_info["start_time"])), width=200, height=100)
# Set the properties of the first plot
p = figure(plot_height=PLOT_HEIGHT, plot_width=PLOT_WIDTH,
#tools="pan,xpan,wheel_zoom,xwheel_zoom,ywheel_zoom,xbox_zoom,reset",
tools="xpan,xwheel_zoom,xbox_zoom,reset,save",
toolbar_location="above",
x_axis_type="linear", y_axis_location="right", y_axis_type=None,
webgl=True)
x, y = get_graph_df(df_sub)
source_sub1.data = {'month': x, 'duration': y}
#p.line(x='month',y='duration', line_width=2,color='blue',source=source_sub,legend_label='2020 data in zipcode 1')
# callback function for dropdown button 1
def draw_line_zip2(new):
zc = int(new.item)
df_sub = df[df['zip_code'] == zc]
#source_sub2 = get_graph_df(df_sub)
x, y = get_graph_df(df_sub)
source_sub2.data = {'month': x, 'duration': y}
#p.line(x='month',y='duration', line_width=2,color='green',source=source_sub,legend_label='2020 data in zipcode 2')
d1.on_click(draw_line_zip1)
d2.on_click(draw_line_zip2)
# input: df output: df for graph (two cols: month,duration)
def get_graph_df(df):
months = df['month'].unique()
months_list = list(months)
months_list.sort()
avg_dur = []
# iterate through months, calculate mean in duration for each
for i in months_list:
df_filter = df[df['month'] == i]
dur_i = df_filter['duration'].mean()
# initialize data source
source_function1 = ColumnDataSource(data=dict(x=[], y=[]))
source_function2 = ColumnDataSource(data=dict(x=[], y=[]))
source_result = ColumnDataSource(data=dict(x=[], y=[]))
source_convolution = ColumnDataSource(data=dict(x=[], y=[]))
source_xmarker = ColumnDataSource(data=dict(x=[], y=[]))
source_overlay = ColumnDataSource(data=dict(x=[], y=[], y_neg=[], y_pos=[]))
# initialize properties
update_is_enabled = True
# initialize controls
# dropdown menu for sample functions
function_type = Dropdown(label="choose a sample function pair or enter one below",
menu=convolution_settings.sample_function_names)
function_type.on_click(function_pair_input_change)
# slider controlling the evaluated x value of the convolved function
x_value_input = Slider(title="x value", name='x value', value=convolution_settings.x_value_init,
start=convolution_settings.x_value_min, end=convolution_settings.x_value_max,
step=convolution_settings.x_value_step)
x_value_input.on_change('value', input_change)
# text input for the first function to be convolved
function1_input = TextInput(value=convolution_settings.function1_input_init, title="my first function:")
function1_input.on_change('value', input_change)
# text input for the second function to be convolved
function2_input = TextInput(value=convolution_settings.function1_input_init, title="my second function:")
function2_input.on_change('value', input_change)
# initialize plot
toolset = "crosshair,pan,reset,resize,save,wheel_zoom"
plot.scatter('x', 'y', source=source_critical_pts, color='red', legend='critical pts')
plot.multi_line('x_ls', 'y_ls', source=source_critical_lines, color='red', legend='critical lines')
# initialize controls
# text input for input of the ode system [u,v] = [x',y']
u_input = TextInput(value=odesystem_settings.sample_system_functions[odesystem_settings.init_fun_key][0], title="u(x,y):")
v_input = TextInput(value=odesystem_settings.sample_system_functions[odesystem_settings.init_fun_key][1], title="v(x,y):")
# dropdown menu for selecting one of the sample functions
sample_fun_input = Dropdown(label="choose a sample function pair or enter one below",
menu=odesystem_settings.sample_system_names)
# Interactor for entering starting point of initial condition
interactor = my_bokeh_utils.Interactor(plot)
# initialize callback behaviour
sample_fun_input.on_click(sample_fun_change)
u_input.on_change('value', ode_change)
v_input.on_change('value', ode_change)
interactor.on_click(initial_value_change)
# calculate data
init_data()
# lists all the controls in our app associated with the default_funs panel
function_controls = widgetbox(sample_fun_input, u_input, v_input,width=400)
# refresh quiver field and streamline all 100ms
curdoc().add_periodic_callback(refresh_user_view, 100)
# make layout
curdoc().add_root(column(plot, function_controls))
bar_chart.visible = False
bar_chart1.visible = False
bar_chart420.visible = True
bar_chart330.visible = False
elif new == '3e Klass':
bar_chart.visible = False
bar_chart1.visible = False
bar_chart420.visible = False
bar_chart330.visible = True
# Create a dropdown Select widget: select
selectRegio = Dropdown(label="Maak keuze uit de klasse", menu=["All", "1e Klass", "2e Klass", "3e Klass"])
# Attach the update_plot callback to the 'value' property of select
selectRegio.on_click(update_bar_chart)
# Hoofdstuk 3 Scatter en 2-D visualisatie
titel4 = Div(text="<h2"">Hoofdstuk 3: Scatterplots en 2-D visualisaties""</h2>", width=800, height=50)
text4 = Div(text="<h4"">Hieronder volgt scatterplot van prijs per ticket tegenover de GDP per capita""</h4>", width=800,
height=50)
# dataframes
FareVsGDP = pd.read_csv('Data K\FareVsGDP.csv')
FareVsGDP1 = pd.read_csv('Data K\FareVsGDP1eKlass.csv')
FareVsGDP2 = pd.read_csv('Data K\FareVsGDP2eKlass.csv')
FareVsGDP3 = pd.read_csv('Data K\FareVsGDP3eKlass.csv')
# dropdown
menu = [("Class 1", "Class_1"), ("Class 2", "Class_2"), None, ("Class 3", "Class_3")]
# Create ColumnDataSource: source
def create(palm):
energy_min = palm.energy_range.min()
energy_max = palm.energy_range.max()
energy_npoints = palm.energy_range.size
current_results = (0, 0, 0, 0)
doc = curdoc()
# Streaked and reference waveforms plot
waveform_plot = Plot(
title=Title(text="eTOF waveforms"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
waveform_plot.toolbar.logo = None
waveform_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool())
# ---- axes
waveform_plot.add_layout(LinearAxis(axis_label="Photon energy, eV"),
place="below")
waveform_plot.add_layout(LinearAxis(axis_label="Intensity",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
waveform_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
waveform_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- line glyphs
waveform_source = ColumnDataSource(
dict(x_str=[], y_str=[], x_ref=[], y_ref=[]))
waveform_ref_line = waveform_plot.add_glyph(
waveform_source, Line(x="x_ref", y="y_ref", line_color="blue"))
waveform_str_line = waveform_plot.add_glyph(
waveform_source, Line(x="x_str", y="y_str", line_color="red"))
# ---- legend
waveform_plot.add_layout(
Legend(items=[("reference",
[waveform_ref_line]), ("streaked",
[waveform_str_line])]))
waveform_plot.legend.click_policy = "hide"
# Cross-correlation plot
xcorr_plot = Plot(
title=Title(text="Waveforms cross-correlation"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
xcorr_plot.toolbar.logo = None
xcorr_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool())
# ---- axes
xcorr_plot.add_layout(LinearAxis(axis_label="Energy shift, eV"),
place="below")
xcorr_plot.add_layout(LinearAxis(axis_label="Cross-correlation",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
xcorr_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
xcorr_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- line glyphs
xcorr_source = ColumnDataSource(dict(lags=[], xcorr1=[], xcorr2=[]))
xcorr_plot.add_glyph(
xcorr_source,
Line(x="lags", y="xcorr1", line_color="purple", line_dash="dashed"))
xcorr_plot.add_glyph(xcorr_source,
Line(x="lags", y="xcorr2", line_color="purple"))
# ---- vertical span
xcorr_center_span = Span(location=0, dimension="height")
xcorr_plot.add_layout(xcorr_center_span)
# Delays plot
pulse_delay_plot = Plot(
title=Title(text="Pulse delays"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
pulse_delay_plot.toolbar.logo = None
pulse_delay_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool())
# ---- axes
pulse_delay_plot.add_layout(LinearAxis(axis_label="Pulse number"),
place="below")
pulse_delay_plot.add_layout(
LinearAxis(axis_label="Pulse delay (uncalib), eV",
major_label_orientation="vertical"),
place="left",
)
# ---- grid lines
pulse_delay_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
pulse_delay_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- line glyphs
pulse_delay_source = ColumnDataSource(dict(pulse=[], delay=[]))
pulse_delay_plot.add_glyph(
pulse_delay_source, Line(x="pulse", y="delay", line_color="steelblue"))
# ---- vertical span
pulse_delay_plot_span = Span(location=0, dimension="height")
pulse_delay_plot.add_layout(pulse_delay_plot_span)
# Pulse lengths plot
pulse_length_plot = Plot(
title=Title(text="Pulse lengths"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
pulse_length_plot.toolbar.logo = None
pulse_length_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool())
# ---- axes
pulse_length_plot.add_layout(LinearAxis(axis_label="Pulse number"),
place="below")
pulse_length_plot.add_layout(
LinearAxis(axis_label="Pulse length (uncalib), eV",
major_label_orientation="vertical"),
place="left",
)
# ---- grid lines
pulse_length_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
pulse_length_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- line glyphs
pulse_length_source = ColumnDataSource(dict(x=[], y=[]))
pulse_length_plot.add_glyph(pulse_length_source,
Line(x="x", y="y", line_color="steelblue"))
# ---- vertical span
pulse_length_plot_span = Span(location=0, dimension="height")
pulse_length_plot.add_layout(pulse_length_plot_span)
# Folder path text input
def path_textinput_callback(_attr, _old_value, new_value):
save_textinput.value = new_value
path_periodic_update()
path_textinput = TextInput(title="Folder Path:",
value=os.path.join(os.path.expanduser("~")),
width=510)
path_textinput.on_change("value", path_textinput_callback)
# Saved runs dropdown menu
def h5_update(pulse, delays, debug_data):
prep_data, lags, corr_res_uncut, corr_results = debug_data
waveform_source.data.update(
x_str=palm.energy_range,
y_str=prep_data["1"][pulse, :],
x_ref=palm.energy_range,
y_ref=prep_data["0"][pulse, :],
)
xcorr_source.data.update(lags=lags,
xcorr1=corr_res_uncut[pulse, :],
xcorr2=corr_results[pulse, :])
xcorr_center_span.location = delays[pulse]
pulse_delay_plot_span.location = pulse
pulse_length_plot_span.location = pulse
# this placeholder function should be reassigned in 'saved_runs_dropdown_callback'
h5_update_fun = lambda pulse: None
def update_saved_runs(saved_run):
new_value = saved_run
if new_value != "Saved Runs":
nonlocal h5_update_fun, current_results
saved_runs_dropdown.label = new_value
filepath = os.path.join(path_textinput.value, new_value)
tags, delays, lengths, debug_data = palm.process_hdf5_file(
filepath, debug=True)
current_results = (new_value, tags, delays, lengths)
if autosave_checkbox.active:
save_button_callback()
pulse_delay_source.data.update(pulse=np.arange(len(delays)),
delay=delays)
pulse_length_source.data.update(x=np.arange(len(lengths)),
y=lengths)
h5_update_fun = partial(h5_update,
delays=delays,
debug_data=debug_data)
pulse_slider.end = len(delays) - 1
pulse_slider.value = 0
h5_update_fun(0)
def saved_runs_dropdown_callback(event):
update_saved_runs(event.item)
saved_runs_dropdown = Dropdown(label="Saved Runs",
button_type="primary",
menu=[])
saved_runs_dropdown.on_click(saved_runs_dropdown_callback)
# ---- saved run periodic update
def path_periodic_update():
new_menu = []
if os.path.isdir(path_textinput.value):
for entry in os.scandir(path_textinput.value):
if entry.is_file() and entry.name.endswith((".hdf5", ".h5")):
new_menu.append((entry.name, entry.name))
saved_runs_dropdown.menu = sorted(new_menu, reverse=True)
doc.add_periodic_callback(path_periodic_update, 5000)
# Pulse number slider
def pulse_slider_callback(_attr, _old_value, new_value):
h5_update_fun(pulse=new_value)
pulse_slider = Slider(
start=0,
end=99999,
value_throttled=0,
step=1,
title="Pulse ID",
)
pulse_slider.on_change("value_throttled", pulse_slider_callback)
# Energy maximal range value text input
def energy_max_spinner_callback(_attr, old_value, new_value):
nonlocal energy_max
if new_value > energy_min:
energy_max = new_value
palm.energy_range = np.linspace(energy_min, energy_max,
energy_npoints)
update_saved_runs(saved_runs_dropdown.label)
else:
energy_max_spinner.value = old_value
energy_max_spinner = Spinner(title="Maximal Energy, eV:",
value=energy_max,
step=0.1)
energy_max_spinner.on_change("value", energy_max_spinner_callback)
# Energy minimal range value text input
def energy_min_spinner_callback(_attr, old_value, new_value):
nonlocal energy_min
if new_value < energy_max:
energy_min = new_value
palm.energy_range = np.linspace(energy_min, energy_max,
energy_npoints)
update_saved_runs(saved_runs_dropdown.label)
else:
energy_min_spinner.value = old_value
energy_min_spinner = Spinner(title="Minimal Energy, eV:",
value=energy_min,
step=0.1)
energy_min_spinner.on_change("value", energy_min_spinner_callback)
# Energy number of interpolation points text input
def energy_npoints_spinner_callback(_attr, old_value, new_value):
nonlocal energy_npoints
if new_value > 1:
energy_npoints = new_value
palm.energy_range = np.linspace(energy_min, energy_max,
energy_npoints)
update_saved_runs(saved_runs_dropdown.label)
else:
energy_npoints_spinner.value = old_value
energy_npoints_spinner = Spinner(title="Number of interpolation points:",
value=energy_npoints)
energy_npoints_spinner.on_change("value", energy_npoints_spinner_callback)
# Save location
save_textinput = TextInput(title="Save Folder Path:",
value=os.path.join(os.path.expanduser("~")))
# Autosave checkbox
autosave_checkbox = CheckboxButtonGroup(labels=["Auto Save"],
active=[],
width=250)
# Save button
def save_button_callback():
if current_results[0]:
filename, tags, delays, lengths = current_results
save_filename = os.path.splitext(filename)[0] + ".csv"
df = pd.DataFrame({
"pulse_id": tags,
"pulse_delay": delays,
"pulse_length": lengths
})
df.to_csv(os.path.join(save_textinput.value, save_filename),
index=False)
save_button = Button(label="Save Results",
button_type="default",
width=250)
save_button.on_click(save_button_callback)
# assemble
tab_layout = column(
row(
column(waveform_plot, xcorr_plot),
Spacer(width=30),
column(
path_textinput,
saved_runs_dropdown,
pulse_slider,
Spacer(height=30),
energy_min_spinner,
energy_max_spinner,
energy_npoints_spinner,
Spacer(height=30),
save_textinput,
autosave_checkbox,
save_button,
),
),
row(pulse_delay_plot, Spacer(width=10), pulse_length_plot),
)
return Panel(child=tab_layout, title="HDF5 File")
