def test_stream_good_data(self):
ds = ColumnDataSource(data=dict(a=[10], b=[20]))
ds._document = "doc"
stuff = {}
def mock(*args, **kw):
stuff['args'] = args
stuff['kw'] = kw
ds.data._stream = mock
ds.stream(dict(a=[11, 12], b=[21, 22]), "foo")
self.assertEqual(stuff['args'], ("doc", ds, dict(a=[11, 12], b=[21, 22]), "foo"))
self.assertEqual(stuff['kw'], {})
class BokehVisuals:
def create_testing_charts(self):
# create charts for testing headset
self.figs = [
figure(plot_width=200, plot_height=125, toolbar_location=None)
for i in range(self.channels_number)
]
for f in self.figs:
f.extra_y_ranges = {"foo": Range1d(start=-100, end=100)}
[
f.add_layout(LinearAxis(y_range_name="foo"), "right")
for f in self.figs
]
self.cds = ColumnDataSource(
data={
i: []
for i in [
item for sublist in [["y" + str(i), "f" + str(i)]
for i in range(self.channels_number)]
for item in sublist
] + ["x"]
})
[(
self.figs[i].line("x",
"y" + str(i),
source=self.cds,
line_alpha=0.3),
self.figs[i].line(
"x",
"f" + str(i),
source=self.cds,
line_color="red",
y_range_name="foo",
line_width=1,
),
) for i in range(self.channels_number)]
for f in self.figs:
f.axis.visible = False
f.xgrid.grid_line_color = None
f.ygrid.grid_line_color = None
def create_features_charts(self):
# create charts for visualising features
cds_feats_data = {"x": [], "y": []}
for i in range(self.channels_number):
cds_feats_data["value_" + str(i)] = []
self.cds_feats = ColumnDataSource(data=cds_feats_data)
colors = [
"#75968f",
"#a5bab7",
"#c9d9d3",
"#e2e2e2",
"#dfccce",
"#ddb7b1",
"#cc7878",
"#933b41",
"#550b1d",
]
mapper = LinearColorMapper(palette=colors, low=10, high=70)
self.figs_feats = [
figure(plot_width=400, plot_height=120, toolbar_location=None)
for i in range(self.channels_number)
]
for f in self.figs_feats:
f.axis.visible = False
f.xgrid.grid_line_color = None
f.ygrid.grid_line_color = None
[
self.figs_feats[i].rect(
x="x",
y="y",
width=1,
height=1,
source=self.cds_feats,
line_color=None,
fill_color={
"field": "value_" + str(i),
"transform": mapper
},
) for i in range(self.channels_number)
]
def create_forecast_status_charts(self):
self.forecast_status_cds = ColumnDataSource(data={
"x": [],
"stage": [],
"prediction": []
})
self.forecast_status_figure = figure(plot_width=400,
plot_height=100,
toolbar_location=None)
self.forecast_status_figure.line(
"x",
"stage",
source=self.forecast_status_cds,
line_color="red",
line_alpha=0.5,
)
self.forecast_status_figure.line("x",
"prediction",
source=self.forecast_status_cds,
line_color="black")
self.forecast_status_figure.axis.visible = False
self.forecast_status_figure.xgrid.grid_line_color = None
self.forecast_status_figure.ygrid.grid_line_color = None
# figure for status
self.status_text_cds = ColumnDataSource(data={
"x": [0],
"y": [0],
"text": ["Calibration"]
})
self.status_text_figure = figure(plot_width=200,
plot_height=100,
toolbar_location=None)
self.status_text_figure.axis.visible = False
self.status_text_figure.xgrid.grid_line_color = None
self.status_text_figure.ygrid.grid_line_color = None
status_text = Text(x="x", y="y", text="text", text_color="black")
self.status_text_figure.add_glyph(self.status_text_cds, status_text)
self.update_button = Button(label="Start session")
def __init__(self, helmet):
self.channels_number = helmet.channels_number
self.create_testing_charts()
self.create_features_charts()
self.create_forecast_status_charts()
self.doc = curdoc()
# todo add port selector for real data
def button_link(self, button_f):
# todo add stop button
self.update_button.on_click(button_f)
def add_layout(self):
rows = []
for i in range(int(self.channels_number) / 2):
rows.append(
row(self.figs[i * 2:i * 2 + 2] +
self.figs_feats[i * 2:i * 2 + 2]))
# todo add option for uneven channel count
rows.append(
row(
widgetbox([self.update_button], width=200),
self.status_text_figure,
self.forecast_status_figure,
))
layout = column(*rows)
self.doc.add_root(layout)
def update_tuning(self, new_data, new_filtered_data, filtered_data):
lsd = len(filtered_data)
update_data = {"x": np.arange(lsd - len(new_data[:, 0]), lsd, 1)}
for i in range(self.channels_number):
update_data["f" + str(i)] = new_filtered_data[:, i]
update_data["y" + str(i)] = new_data[:, i]
self.cds.stream(update_data, 1000)
def update_protocol(self, new_features_data, current_state,
current_prediction):
update_data = {
"x": new_features_data.index.get_level_values(0).values,
"y": new_features_data.index.get_level_values(1).values,
}
for i in range(self.channels_number):
update_data["value_" + str(i)] = new_features_data[i].values
self.cds_feats.stream(
update_data,
len(new_features_data.columns) * self.channels_number * 100)
if current_state[0] == "target":
stage = 1
else:
stage = 0
self.forecast_status_cds.stream(
{
"x": [int(time.time()) - 1],
"stage": [stage],
"prediction": [current_prediction],
},
1000,
)
if current_state[1] == "calibration":
self.status_text_cds.stream(
{
"x": [0],
"y": [0],
"text": ["calibration"]
}, 1)
else:
self.status_text_cds.stream(
{
"x": [0],
"y": [0],
"text": ["feedback"]
}, 1)
def test_stream_bad_data(self):
ds = ColumnDataSource(data=dict(a=[10], b=[20]))
with self.assertRaises(ValueError) as cm:
ds.stream(dict())
self.assertEqual(str(cm.exception), "Must stream updates to all existing columns (missing: a, b)")
with self.assertRaises(ValueError) as cm:
ds.stream(dict(a=[10]))
self.assertEqual(str(cm.exception), "Must stream updates to all existing columns (missing: b)")
with self.assertRaises(ValueError) as cm:
ds.stream(dict(a=[10], b=[10], x=[10]))
self.assertEqual(str(cm.exception), "Must stream updates to all existing columns (extra: x)")
with self.assertRaises(ValueError) as cm:
ds.stream(dict(a=[10], x=[10]))
self.assertEqual(str(cm.exception), "Must stream updates to all existing columns (missing: b, extra: x)")
with self.assertRaises(ValueError) as cm:
ds.stream(dict(a=[10], b=[10, 20]))
self.assertEqual(str(cm.exception), "All streaming column updates must be the same length")
with self.assertRaises(ValueError) as cm:
ds.stream(dict(a=[10], b=np.ones((1,1))))
self.assertTrue(
str(cm.exception).startswith("stream(...) only supports 1d sequences, got ndarray with size (")
)
class Tabulator(Widget):
"""The Tabulator Pane wraps the [Tabulator](http://tabulator.info/) table to provide an
awesome interative table.
You can
- Specify a `configuration` dictionary at instantation. See http://tabulator.info/.
- Provide an initial `value` as a Pandas DataFrame or Bokeh ColumnDataSource.
- `stream` (append) to the `value`.
- `patch` (update) the `value`.
Example: Data specified in configuration
>>> from awesome_panel_extensions.widgets.tabulator import Tabulator
>>> configuration = {
... "layout": "fitColumns",
... "data": [
... {"x": [1], "y": 'a'},
... {"x": [2], "y": 'b'}
... ],
... "initialSort":[
... {"column":"y", "dir":"desc"},
... ],
... "columns":[
... {"title": "Value", "field":"x"},
... {"title": "Item", "field":"y", "hozAlign":"right", "formatter":"money"}
... ],
... }
>>> Tabulator(configuration=configuration)
Tabulator(...)
Example: Data specified as Pandas.DataFrame value
>>> import pandas as pd
>>> configuration = {
... "layout": "fitColumns",
... "initialSort":[
... {"column":"y", "dir":"desc"},
... ],
... "columns":[
... {"title": "Value", "field":"x"},
... {"title": "Item", "field":"y", "hozAlign":"right", "formatter":"money"}
... ],
... }
>>> value = pd.DataFrame([
... {"x": [1], "y": 'a'},
... {"x": [2], "y": 'b'}
... ])
>>> Tabulator(configuration=configuration, value=value)
Tabulator(...)
Example: Data specified as Bokeh ColumnDataSource value
>>> configuration = {
... "layout": "fitColumns",
... "initialSort":[
... {"column":"y", "dir":"desc"},
... ],
... "columns":[
... {"title": "Value", "field":"x"},
... {"title": "Item", "field":"y", "hozAlign":"right", "formatter":"money"}
... ],
... }
>>> value = ColumnDataSource({"x": [1,2], "y": ["a", "b"]})
>>> Tabulator(configuration=configuration, value=value)
Tabulator(...)
"""
value = param.Parameter(
doc="""One of pandas.DataFrame or bokeh.models.ColumnDataSource.
Please note when specifying a Pandas.Dataframe we currently have some narrow requirements
- The index should be a single index which should be called 'index' and be unique. To make
sure things work we suggest you `.reset_index()` before usage.
""")
selection = param.List(
doc="The list of selected row indexes. For example [1,4]. Default is []"
)
configuration = param.Dict(
constant=True,
doc=
"""The initial Tabulator configuration. See https://tabulator.info for lots of
examples.
If None is provided at instantiation, then {'autocolumns': True} is added
Please note that in order to get things working we add the below to the configuration
on the js side.
{
"rowSelectionChanged": rowSelectionChanged,
"cellEdited": cellEdited,
"index": "index",
}
""",
)
height = param.Integer(
default=300,
bounds=(0, None),
doc=
"""The height of the Tabulator table. Specifying a height is mandatory.""",
)
_source = param.ClassSelector(
class_=ColumnDataSource,
doc="Used to transfer the `value` efficiently to frontend")
_cell_change = param.Dict(
doc=
"""Changed whenever the user updates a cell in the client. Sends something like
{"c": "<COLUMN NAME>", "i": "<INDEX>", "v": "<NEW VALUE>"}. Used to transfer the change
efficiently and update the DataFrame as I could not find a similar property/ event on the
ColumnDataSource""")
_rename = {
"value": None,
"selection": None,
"_source": "source",
}
_widget_type = _BkTabulator
def __init__(self, **params):
"""The Tabulator Pane wraps the [Tabulator](http://tabulator.info/) table to provide an
awesome interative table.
You can
- Specify a `configuration` dictionary at instantation. See http://tabulator.info/.
- Provide an initial `value` as a Pandas DataFrame or Bokeh ColumnDataSource.
- `stream` (append) to the `value`.
- `patch` (update) the `value`.
Example: Data specified in configuration
>>> from awesome_panel_extensions.widgets.tabulator import Tabulator
>>> configuration = {
... "layout": "fitColumns",
... "data": [
... {"x": [1], "y": 'a'},
... {"x": [2], "y": 'b'}
... ],
... "initialSort":[
... {"column":"y", "dir":"desc"},
... ],
... "columns":[
... {"title": "Value", "field":"x"},
... {"title": "Item", "field":"y", "hozAlign":"right", "formatter":"money"}
... ],
... }
>>> Tabulator(configuration=configuration)
Tabulator(...)
Example: Data specified as Pandas.DataFrame value
>>> import pandas as pd
>>> configuration = {
... "layout": "fitColumns",
... "initialSort":[
... {"column":"y", "dir":"desc"},
... ],
... "columns":[
... {"title": "Value", "field":"x"},
... {"title": "Item", "field":"y", "hozAlign":"right", "formatter":"money"}
... ],
... }
>>> value = pd.DataFrame([
... {"x": [1], "y": 'a'},
... {"x": [2], "y": 'b'}
... ])
>>> Tabulator(configuration=configuration, value=value)
Tabulator(...)
Example: Data specified as Bokeh ColumnDataSource value
>>> configuration = {
... "layout": "fitColumns",
... "initialSort":[
... {"column":"y", "dir":"desc"},
... ],
... "columns":[
... {"title": "Value", "field":"x"},
... {"title": "Item", "field":"y", "hozAlign":"right", "formatter":"money"}
... ],
... }
>>> value = ColumnDataSource({"x": [1,2], "y": ["a", "b"]})
>>> Tabulator(configuration=configuration, value=value)
Tabulator(...)
"""
if "configuration" not in params:
params["configuration"] = _DEFAULT_CONFIGURATION.copy()
if "selection" not in params:
params["selection"] = []
super().__init__(**params)
self._pause_cds_updates = False
self._update_column_data_source()
@param.depends("value", watch=True)
def _update_column_data_source(self, *_):
if self._pause_cds_updates:
return
if self.value is None:
self._source = ColumnDataSource({})
elif isinstance(self.value, pd.DataFrame):
if (not isinstance(self.value.index, pd.RangeIndex)
or self.value.index.start != 0
or self.value.index.step != 1):
raise ValueError(
"Please provide a DataFrame with RangeIndex starting at 0 and with step 1"
)
if self._source:
self._source.data = self.value
else:
self._source = ColumnDataSource(self.value)
elif isinstance(self.value, ColumnDataSource):
self._source = self.value
else:
raise ValueError("The `data` provided is not of a supported type!")
@param.depends("_cell_change", watch=True)
def _update_value_with_cell_change(self):
if isinstance(self.value, pd.DataFrame):
column = self._cell_change["c"] # pylint: disable=unsubscriptable-object
index = self._cell_change["i"] # pylint: disable=unsubscriptable-object
new_value = self._cell_change["v"] # pylint: disable=unsubscriptable-object
self._pause_cds_updates = True
self.value.at[index, column] = new_value
self.param.trigger("value")
self._pause_cds_updates = False
def stream(self,
stream_value: Union[pd.DataFrame, pd.Series, Dict],
reset_index: bool = True):
"""Streams (appends) the `stream_value` provided to the existing value in an efficient
manner.
Args:
stream_value (Union[pd.DataFrame, pd.Series, Dict]): The new value(s) to append to the
existing value.
reset_index (bool, optional): If the stream_value is a DataFrame and `reset_index` is
True then the index of it is reset if True. Helps to keep the index unique and
named `index`. Defaults to True.
Raises:
ValueError: Raised if the stream_value is not a supported type.
Example: Stream a Series to a DataFrame
>>> value = pd.DataFrame({"x": [1, 2], "y": ["a", "b"]})
>>> tabulator = Tabulator(value=value)
>>> stream_value = pd.Series({"x": 4, "y": "d"})
>>> tabulator.stream(stream_value)
>>> tabulator.value.to_dict("list")
{'x': [1, 2, 4], 'y': ['a', 'b', 'd']}
Example: Stream a Dataframe to a Dataframe
>>> value = pd.DataFrame({"x": [1, 2], "y": ["a", "b"]})
>>> tabulator = Tabulator(value=value)
>>> stream_value = pd.DataFrame({"x": [3, 4], "y": ["c", "d"]})
>>> tabulator.stream(stream_value)
>>> tabulator.value.to_dict("list")
{'x': [1, 2, 3, 4], 'y': ['a', 'b', 'c', 'd']}
Example: Stream a Dictionary row to a DataFrame
>>> value = pd.DataFrame({"x": [1, 2], "y": ["a", "b"]})
>>> tabulator = Tabulator(value=value)
>>> stream_value = {"x": 4, "y": "d"}
>>> tabulator.stream(stream_value)
>>> tabulator.value.to_dict("list")
{'x': [1, 2, 4], 'y': ['a', 'b', 'd']}
Example: Stream a Dictionary of Columns to a Dataframe
>>> value = pd.DataFrame({"x": [1, 2], "y": ["a", "b"]})
>>> tabulator = Tabulator(value=value)
>>> stream_value = {"x": [3, 4], "y": ["c", "d"]}
>>> tabulator.stream(stream_value)
>>> tabulator.value.to_dict("list")
{'x': [1, 2, 3, 4], 'y': ['a', 'b', 'c', 'd']}
"""
if isinstance(self.value, pd.DataFrame):
value_index_start = self.value.index.max() + 1
if isinstance(stream_value, pd.DataFrame):
if reset_index:
stream_value = stream_value.reset_index(drop=True)
stream_value.index += value_index_start
self._pause_cds_updates = True
self.value = pd.concat([self.value, stream_value])
self._source.stream(stream_value)
self._pause_cds_updates = False
elif isinstance(stream_value, pd.Series):
self._pause_cds_updates = True
self.value.loc[value_index_start] = stream_value
self._source.stream(stream_value)
self.param.trigger("value")
self._pause_cds_updates = False
elif isinstance(stream_value, dict):
if stream_value:
try:
stream_value = pd.DataFrame(stream_value)
except ValueError:
stream_value = pd.Series(stream_value)
self.stream(stream_value)
else:
raise ValueError(
"The patch value provided is not a DataFrame, Series or Dict!"
)
else:
self._pause_cds_updates = True
self._source.stream(stream_value)
self.param.trigger("value")
self._pause_cds_updates = False
def patch(self, patch_value: Union[pd.DataFrame, pd.Series, Dict]):
"""Patches (updates) the existing value with the `patch_value` in an efficient manner.
Args:
patch_value (Union[pd.DataFrame, pd.Series, Dict]): The value(s) to patch the
existing value with.
Raises:
ValueError: Raised if the patch_value is not a supported type.
Example: Patch a DataFrame with a Dictionary row.
>>> value = pd.DataFrame({"x": [1, 2], "y": ["a", "b"]})
>>> tabulator = Tabulator(value=value)
>>> patch_value = {"x": [(0, 3)]}
>>> tabulator.patch(patch_value)
>>> tabulator.value.to_dict("list")
{'x': [3, 2], 'y': ['a', 'b']}
Example: Patch a Dataframe with a Dictionary of Columns.
>>> value = pd.DataFrame({"x": [1, 2], "y": ["a", "b"]})
>>> tabulator = Tabulator(value=value)
>>> patch_value = {"x": [(slice(2), (3,4))], "y": [(1,'d')]}
>>> tabulator.patch(patch_value)
>>> tabulator.value.to_dict("list")
{'x': [3, 4], 'y': ['a', 'd']}
Example: Patch a DataFrame with a Series. Please note the index is used in the update
>>> value = pd.DataFrame({"x": [1, 2], "y": ["a", "b"]})
>>> tabulator = Tabulator(value=value)
>>> patch_value = pd.Series({"index": 1, "x": 4, "y": "d"})
>>> tabulator.patch(patch_value)
>>> tabulator.value.to_dict("list")
{'x': [1, 4], 'y': ['a', 'd']}
Example: Patch a Dataframe with a Dataframe. Please note the index is used in the update.
>>> value = pd.DataFrame({"x": [1, 2], "y": ["a", "b"]})
>>> tabulator = Tabulator(value=value)
>>> patch_value = pd.DataFrame({"x": [3, 4], "y": ["c", "d"]})
>>> tabulator.patch(patch_value)
>>> tabulator.value.to_dict("list")
{'x': [3, 4], 'y': ['c', 'd']}
"""
if isinstance(self.value, pd.DataFrame):
if isinstance(patch_value, pd.DataFrame):
patch_value_dict = self._patch_dataframe_to_dict(patch_value)
self.patch(patch_value_dict)
elif isinstance(patch_value, pd.Series):
patch_value_dict = self._patch_series_to_dict(patch_value)
self.patch(patch_value_dict)
elif isinstance(patch_value, dict):
self._patch_from_dict(patch_value)
else:
raise ValueError(
f"""Patching a patch_value of type {type(patch_value)} is not supported.
Please provide a DataFrame, Series or Dict""")
else:
if isinstance(patch_value, dict):
self._pause_cds_updates = True
self._source.patch(patch_value)
self.param.trigger("value")
self._pause_cds_updates = False
else:
raise ValueError(
f"""Patching a patch_value of type {type(patch_value)} is not supported.
Please provide a dict""")
@staticmethod
def _patch_dataframe_to_dict(patch_value: pd.DataFrame) -> Dict:
patch_value_dict: Dict[Any, Any] = {}
for column in patch_value.columns:
patch_value_dict[column] = []
for index in patch_value.index:
patch_value_dict[column].append(
(index, patch_value.loc[index, column]))
return patch_value_dict
@staticmethod
def _patch_series_to_dict(patch_value: pd.Series) -> Dict:
if "index" in patch_value: # Series orient is row
patch_value_dict = {
k: [(patch_value["index"], v)]
for k, v in patch_value.items()
}
patch_value_dict.pop("index")
else: # Series orient is column
patch_value_dict = {patch_value.name: list(patch_value.items())}
return patch_value_dict
def _patch_from_dict(self, patch_value: Dict):
self._pause_cds_updates = True
for key, value in patch_value.items():
for update in value:
self.value.loc[update[0], key] = update[1]
self._source.patch(patch_value)
self.param.trigger("value")
self._pause_cds_updates = False
@classmethod
def to_columns_configuration(
cls, value: Union[pd.DataFrame,
ColumnDataSource]) -> List[Dict[str, str]]:
"""Returns a nice starter `columns` dictionary from the specified `value`.
Args:
value (Union[pd.DataFrame, ColumnDataSource]): The data source to transform.
Returns:
Dict: The columns configuration
Example:
>>> import pandas as pd
>>> value = {"name": ["python", "panel"]}
>>> df = pd.DataFrame(value)
>>> Tabulator.to_columns_configuration(df)
[{'title': 'Name', 'field': 'name', 'sorter': 'string', 'formatter': 'plaintext', \
'hozAlign': 'left'}]
"""
col_conf = []
for field in value.columns:
dtype = str(value.dtypes[field])
conf = cls._core(field=field, dtype=dtype)
col_conf.append(conf)
return col_conf
@classmethod
def _core(cls, field: str, dtype: str) -> Dict[str, str]:
dtype_str = str(dtype)
return {
"title": cls._to_title(field),
"field": field,
"sorter": _SORTERS.get(dtype_str, "string"),
"formatter": _FORMATTERS.get(dtype_str, "plaintext"),
"hozAlign": _HOZ_ALIGNS.get(dtype_str, "left"),
}
@staticmethod
def _to_title(field: str) -> str:
return field.replace("_", " ").title()
@staticmethod
def config(css: Optional[str] = "default"):
"""Adds the specified css theme to pn.config.css_files
Args:
css (Optional[str], optional): [description]. Defaults to "default".
"""
if css:
href = CSS_HREFS[css]
if href not in pn.config.css_files:
pn.config.css_files.append(href)
@property
def selected_values(self) -> Union[pd.DataFrame, ColumnDataSource, None]:
"""Returns the selected rows of the data based
Raises:
ValueError: If the value is not of the supported type.
Returns:
Union[pd.DataFrame, ColumnDataSource, None]: The selected values of the same type as
value. Based on the the current selection.
"""
# Selection is a list of row indices. For example [0,2]
if self.value is None:
return None
if isinstance(self.value, pd.DataFrame):
return self.value.iloc[self.selection, ]
if isinstance(self.value, ColumnDataSource):
# I could not find a direct way to get a selected ColumnDataSource
selected_data = self.value.to_df().iloc[self.selection, ]
return ColumnDataSource(selected_data)
raise ValueError("The value is not of a supported type!")
@param.depends("selection", watch=True)
def _update_source_selected_indices(self, *_):
self._source.selected.indices = self.selection
def _get_model(self, doc, root=None, parent=None, comm=None):
model = super()._get_model(doc=doc,
root=root,
parent=parent,
comm=comm)
if root is None:
root = model
self._link_props(model.source.selected, ["indices"], doc, root, comm)
return model
def _process_events(self, events):
if "indices" in events:
self.selection = events.pop("indices")
super()._process_events(events)
class Plot:
WIDTH = 800
HEIGHT = 600
def __init__(self, shifts, multiplicities, deviations):
self.shifts = shifts
self.multiplicities = multiplicities
self.deviations = deviations
xr = Range1d(start=220, end=-20)
self.plot = figure(x_axis_label="ppm", x_range=xr, tools="save,reset", plot_width=self.WIDTH, plot_height=self.HEIGHT)
self.lineSource = ColumnDataSource(data=dict(x=[0], y=[0]))
self.plot.line('x', 'y', source=self.lineSource, line_width=2)
# Remove grid from plot
self.plot.xgrid.grid_line_color = None
self.plot.ygrid.grid_line_color = None
# Remove bokeh logo
self.plot.toolbar.logo = None
horizontalBoxZoomTool = HorizontalBoxZoomTool()
self.plot.add_tools(horizontalBoxZoomTool)
fixedZoomOutTool = FixedZoomOutTool(factor=0.4)
self.plot.add_tools(fixedZoomOutTool)
self.plot.extra_y_ranges['box'] = Range1d(start=0, end=0.1)
self.selectionSource = ColumnDataSource(data=dict(left=[], right=[]), callback=CustomJS(code="""
if ('data' in this.selected) {
window.top.location.href = "http://localhost:8080?" + this.selected.data.query + "style=plot";
}
"""))
self.selectionSource.on_change('selected', lambda attr, old, new: self.delete(new['1d']['indices']))
rect = HBar(
left='left',
right='right',
y=0.5,
height=1,
line_alpha=0.2,
fill_alpha=0.2,
line_color="red",
fill_color="red"
)
renderer = self.plot.add_glyph(self.selectionSource, rect)
renderer.y_range_name = "box"
self.labelSource = ColumnDataSource(data=dict(x=[], y=[], text=[]))
label = Text(
x='x',
y='y',
text='text',
text_align='center',
text_font_size="10pt"
)
renderer = self.plot.add_glyph(self.labelSource, label)
renderer.y_range_name = "box"
removeTool = CustomTapTool()
self.plot.add_tools(removeTool)
self.plot.toolbar.active_tap = None
callback = CustomJS(args=dict(), code="""
/// get BoxSelectTool dimensions from cb_data parameter of Callback
var geometry = cb_data['geometry'];
var shift = (geometry['x0'] + geometry['x1']) / 2;
var deviation = Math.abs(geometry['x1'] - shift);
var query = cb_data['query'] + "shift%5B%5D=" + shift + '&multiplicity%5B%5D=any&deviation%5B%5D=' + deviation + '&style=plot';
window.top.location.href = "http://localhost:8080?" + query;
""")
selectTool = CustomBoxSelectTool(
tool_name="Select Area",
dimensions = "width",
callback = callback,
query=self.paramsToQuery()
)
self.plot.add_tools(selectTool)
self.initSelect()
self.drawButton = CustomButton(id='myButton')
self.drawButton.on_click(self.drawPlot)
curdoc().add_root(
row(
column(row(self.plot)),
column(CustomRow(column(self.drawButton), hide=True))
)
)
def paramsToQuery(self):
query = ""
for (shift, multiplicity, deviation) in zip(self.shifts, self.multiplicities, self.deviations):
query += "shift%5B%5D={}&multiplicity%5B%5D={}&deviation%5B%5D={}&".format(shift, multiplicity, deviation)
return query
def initSelect(self):
left, right = [], []
labelX, labelY, labelText = [], [], []
for (shift, multiplicity, deviation) in zip(self.shifts, self.multiplicities, self.deviations):
left.append(shift - deviation)
right.append(shift + deviation)
labelX.append(shift)
labelY.append(0.08)
labelText.append(multiplicity)
self.selectionSource.stream({
'left': left,
'right': right
})
self.labelSource.stream({
'x': labelX,
'y': labelY,
'text': labelText
})
def drawPlot(self, data):
dic, _ = ng.bruker.read("../data/{}".format(data['id']))
_, pdata = ng.bruker.read_pdata("../data/{}/pdata/1/".format(data['id']))
udic = ng.bruker.guess_udic(dic, pdata)
uc = ng.fileiobase.uc_from_udic(udic)
ppmScale = uc.ppm_scale()
self.plot.y_range = None
self.lineSource.data = {
'x': ppmScale,
'y': pdata
}
def delete(self, ids):
if ids:
left = list(self.selectionSource.data['left'])
right = list(self.selectionSource.data['right'])
for i in ids:
left.pop(i)
right.pop(i)
self.shifts.pop(i)
self.multiplicities.pop(i)
self.deviations.pop(i)
self.selectionSource.data = {
'left': left,
'right': right
}
# Deselect all
self.selectionSource.selected = {
'0d': {'glyph': None, 'indices': []},
'1d': {'indices': []},
'2d': {'indices': {}},
'data': {'query': self.paramsToQuery()}
}
def selectArea(self, dimensions):
patch = {
'left': [dimensions['x0']],
'right': [dimensions['x1']]
}
self.selectionSource.stream(patch)
class Controller:
params = {}
terminating = False
def __init__(self, params={}) -> None:
self.params = {}
for param in PARAMETERS:
self.update_parameter(param, PARAMETERS[param][1])
for param in params:
self.update_parameter(param, params[param])
self.engine = self.make_engine()
data = {}
data['x'], data['y'], data['color'] = trisect(
self.engine.agents)
self.visualisation_source = ColumnDataSource(data)
self.names = [status.name for status in AgentStatus][::-1]
self.status_source = ColumnDataSource(pd.DataFrame(
np.zeros((1, len(self.names))), columns=self.names))
self.controls = Controls(self)
self.start()
def update_parameter(self, key, value):
try:
self.params[key] = cajole(
float(value), PARAMETERS[key][0], PARAMETERS[key][2])
except:
self.params[key] = PARAMETERS[key][1]
if key in ('agents', 'sickness_proximity', 'sickness_duration', 'quarantine_delay'):
self.params[key] == int(self.params[key])
elif key == 'initial_immunity':
self.params[key] /= 100
elif key == 'distancing_factor':
self.params['distancing_factor'] /= 100
elif key == 'quarantining':
self.params['quarantining'] = self.params['quarantining'] == 1
if 'quarantining' in self.params and not self.params['quarantining']:
self.params['quarantine_delay'] = self.params['sickness_duration'] + 1
def make_engine(self) -> Engine:
return Engine(n=int(self.params['agents']), SICKNESS_PROXIMITY=int(self.params['sickness_proximity']),
SICKNESS_DURATION=int(self.params['sickness_duration']), DISTANCING_FACTOR=self.params['distancing_factor'],
QUARANTINE_DELAY=int(self.params['quarantine_delay']), INITIAL_IMMUNITY=self.params['initial_immunity'])
def show(self) -> None:
self.visualisation = get_visualisation(self.visualisation_source)
self.population_health_graph = get_population_health_graph(
self.names, self.status_source, self.params['agents'])
curdoc().add_root(gridplot([
[self.visualisation, self.population_health_graph],
[self.controls.get_controls(), get_about_us()]
], toolbar_location="left", toolbar_options={'logo': None}))
def start(self) -> None:
self.update_callback = curdoc().add_periodic_callback(
self.update, 1000 // TICKS_PER_SECOND)
def update(self) -> None:
self.engine.tick()
s = slice(self.engine.agent_count)
x, y, color = trisect(self.engine.agents)
self.visualisation_source.patch({
'x': [(s, x)],
'y': [(s, y)],
'color': [(s, color)]
})
self.status_source.stream({
"index": [self.engine.ticks],
AgentStatus.DEAD.name: [self.engine.stats.get(AgentStatus.DEAD.name, 0)],
AgentStatus.IMMUNE.name: [self.engine.stats.get(AgentStatus.IMMUNE.name, 0)],
AgentStatus.INFECTIOUS.name: [self.engine.stats.get(AgentStatus.INFECTIOUS.name, 0)],
AgentStatus.SUSCEPTIBLE.name: [self.engine.stats.get(AgentStatus.SUSCEPTIBLE.name, 0)],
})
if self.engine.stats.get(AgentStatus.INFECTIOUS.name, 0) == 0 and not self.terminating:
curdoc().add_timeout_callback(self.terminate, 8000)
def terminate(self) -> None:
if self.terminating:
return
self.terminating = True
try:
curdoc().remove_periodic_callback(self.update_callback)
except ValueError:
pass
def reset(self) -> None:
self.terminate()
self.engine = self.make_engine()
data = {}
data['x'], data['y'], data['color'] = trisect(
self.engine.agents)
self.visualisation_source.data = data
self.status_source.data = {
"index": [],
AgentStatus.DEAD.name: [],
AgentStatus.IMMUNE.name: [],
AgentStatus.INFECTIOUS.name: [],
AgentStatus.SUSCEPTIBLE.name: [],
}
self.terminating = False
curdoc().add_next_tick_callback(self.start)
def test_stream_bad_data(self):
ds = ColumnDataSource(data=dict(a=[10], b=[20]))
with pytest.raises(ValueError, match=r"Must stream updates to all existing columns \(missing: a, b\)"):
ds.stream(dict())
with pytest.raises(ValueError, match=r"Must stream updates to all existing columns \(missing: b\)"):
ds.stream(dict(a=[10]))
with pytest.raises(ValueError, match=r"Must stream updates to all existing columns \(extra: x\)"):
ds.stream(dict(a=[10], b=[10], x=[10]))
with pytest.raises(ValueError, match=r"Must stream updates to all existing columns \(missing: b, extra: x\)"):
ds.stream(dict(a=[10], x=[10]))
with pytest.raises(ValueError, match=r"All streaming column updates must be the same length"):
ds.stream(dict(a=[10], b=[10, 20]))
with pytest.raises(ValueError, match=r"stream\(...\) only supports 1d sequences, got ndarray with size \(.*"):
ds.stream(dict(a=[10], b=np.ones((1,1))))
class BokehVisuals:
# todo takes helmet as input to define layout
def __init__(self):
# todo add port selector for real data
# create charts for testing headset
self.figs = [
figure(plot_width=200, plot_height=125, toolbar_location=None)
for i in range(8)
]
for f in self.figs:
f.extra_y_ranges = {"foo": Range1d(start=-100, end=100)}
[
f.add_layout(LinearAxis(y_range_name="foo"), 'right')
for f in self.figs
]
self.cds = ColumnDataSource(
data={
i: []
for i in [
item for sublist in [['y' + str(i), 'f' + str(i)]
for i in range(8)] for item in sublist
] + ['x']
})
[(self.figs[i].line('x', 'y' + str(i), source=self.cds,
line_alpha=0.3),
self.figs[i].line('x',
'f' + str(i),
source=self.cds,
line_color='red',
y_range_name="foo",
line_width=1)) for i in range(8)]
for f in self.figs:
f.axis.visible = False
f.xgrid.grid_line_color = None
f.ygrid.grid_line_color = None
# create charts for visualising features
cds_feats_data = {'x': [], 'y': []}
for i in range(8):
cds_feats_data['value_' + str(i)] = []
self.cds_feats = ColumnDataSource(data=cds_feats_data)
colors = [
"#75968f", "#a5bab7", "#c9d9d3", "#e2e2e2", "#dfccce", "#ddb7b1",
"#cc7878", "#933b41", "#550b1d"
]
mapper = LinearColorMapper(palette=colors, low=10, high=70)
self.figs_feats = [
figure(plot_width=400, plot_height=120, toolbar_location=None)
for i in range(8)
]
for f in self.figs_feats:
f.axis.visible = False
f.xgrid.grid_line_color = None
f.ygrid.grid_line_color = None
[
self.figs_feats[i].rect(x="x",
y="y",
width=1,
height=1,
source=self.cds_feats,
line_color=None,
fill_color={
'field': 'value_' + str(i),
'transform': mapper
}) for i in range(8)
]
# create figure for forecast and status
self.forecast_status_cds = ColumnDataSource(data={
'x': [],
'stage': [],
'prediction': []
})
self.forecast_status_figure = figure(plot_width=400,
plot_height=100,
toolbar_location=None)
self.forecast_status_figure.line('x',
'stage',
source=self.forecast_status_cds,
line_color='red',
line_alpha=0.5)
self.forecast_status_figure.line('x',
'prediction',
source=self.forecast_status_cds,
line_color='black')
self.forecast_status_figure.axis.visible = False
self.forecast_status_figure.xgrid.grid_line_color = None
self.forecast_status_figure.ygrid.grid_line_color = None
# figure for status
self.status_text_cds = ColumnDataSource(data={
'x': [0],
'y': [0],
'text': ['Calibration']
})
self.status_text_figure = figure(plot_width=200,
plot_height=100,
toolbar_location=None)
self.status_text_figure.axis.visible = False
self.status_text_figure.xgrid.grid_line_color = None
self.status_text_figure.ygrid.grid_line_color = None
status_text = Text(x="x", y="y", text="text", text_color='black')
self.status_text_figure.add_glyph(self.status_text_cds, status_text)
self.update_button = Button(label="Start session")
self.doc = curdoc()
def button_link(self, button_f):
# todo add stop button
self.update_button.on_click(button_f)
def add_layout(self):
layout = column(
row(self.figs[0:2] + self.figs_feats[0:2]),
row(self.figs[2:4] + self.figs_feats[2:4]),
row(self.figs[4:6] + self.figs_feats[4:6]),
row(self.figs[6:] + self.figs_feats[6:]),
row(widgetbox([self.update_button], width=200),
self.status_text_figure, self.forecast_status_figure))
self.doc.add_root(layout)
# todo updates
# self.visuals.update_tuning(new_data, new_filtered_data, self.filtered_data)
# self.visuals.update_protocol(self.new_features_data)
def update_tuning(self, new_data, new_filtered_data, filtered_data):
lsd = len(filtered_data)
update_data = {'x': np.arange(lsd - len(new_data[:, 0]), lsd, 1)}
for i in range(8):
update_data['f' + str(i)] = new_filtered_data[:, i]
update_data['y' + str(i)] = new_data[:, i]
self.cds.stream(update_data, 1000)
def update_protocol(self, new_features_data, current_state,
current_prediction):
update_data = {
'x': new_features_data.index.get_level_values(0).values,
'y': new_features_data.index.get_level_values(1).values
}
for i in range(8):
update_data['value_' + str(i)] = new_features_data[i].values
self.cds_feats.stream(update_data,
len(new_features_data.columns) * 8 * 1000)
if current_state[0] == 'target':
stage = 1
else:
stage = 0
self.forecast_status_cds.stream(
{
'x': [int(time.time()) - 1],
'stage': [stage],
'prediction': [current_prediction]
}, 1000)
if current_state[1] == 'calibration':
self.status_text_cds.stream(
{
'x': [0],
'y': [0],
'text': ['calibration']
}, 1)
else:
self.status_text_cds.stream(
{
'x': [0],
'y': [0],
'text': ['feedback']
}, 1)
class Visualizer():
def __init__(self, best_ind, avg_ind):
self.p = figure(title="evolution",
x_axis_label="gen",
y_axis_label="energy",
plot_width=400,
plot_height=400)
self.bar = figure(title="popul",
x_axis_label="inds",
y_axis_label="energy",
plot_width=400,
plot_height=400)
best_energies = [best_ind]
avg_energies = [avg_ind]
index = [0]
inds = [0 for x in range(0, 1)]
index = [0]
self.inds_source = ColumnDataSource(data=dict(x=index, y=inds))
self.bar.vbar(x="x", top="y", source=self.inds_source, width=0.9)
data = {'x': index, 'y': best_energies, 'y2': avg_energies}
self.source = ColumnDataSource(data)
self.p.line(x='x', y='y', color="red", source=self.source)
self.p.line(x='x', y='y2', color="green", source=self.source)
self.doc = curdoc()
self.layout = row(self.p, self.bar)
#curdoc().add_periodic_callback(update, 50)
self.doc.add_root(self.layout)
@gen.coroutine
def update(self, x, y, y2, inds):
if (x == 1):
self.source.data = dict(x=x, y=y, y2=y2)
else:
self.source.stream(dict(x=x, y=y, y2=y2))
index = list(range(0, len(inds)))
self.inds_source.data = dict(x=index, y=inds)
def render_gen(self, gen, popul):
scores = [x.score for x in popul]
individuals = [x.genotype for x in popul]
best_ind = np.min(scores)
avg_ind = np.average(scores)
new_list = [best_ind]
new_avg_list = [avg_ind]
data = {'x': [gen], 'y': new_list, 'y2': new_avg_list}
self.doc.add_next_tick_callback(
partial(self.update,
x=data['x'],
y=data['y'],
y2=data['y2'],
inds=scores))
def update_gen(self, gen, best_ind, avg_ind, popul):
new_list = [best_ind]
new_avg_list = [avg_ind]
data = {'x': [gen], 'y': new_list, 'y2': new_avg_list}
self.doc.add_next_tick_callback(
partial(self.update,
x=data['x'],
y=data['y'],
y2=data['y2'],
inds=popul))
def test_run(self, source):
init_b_values = source.data["y"]
init_avg_values = source.data["y2"]
while True:
new_list = [init_b_values[-1] - 1]
new_avg_list = [init_avg_values[-1] - 1]
data = {
'x': [len(init_b_values) + 1],
'y': new_list,
'y2': new_avg_list
}
self.doc.add_next_tick_callback(
partial(self.update, x=data['x'], y=data['y'], y2=data['y2']))
init_b_values = init_b_values + new_list
init_avg_values = init_avg_values + new_avg_list
time.sleep(5)
futures = [c.submit(rosenbrock, (x, y)) for x, y in zip(startx, starty)]
iterator = as_completed(futures)
# TODO(holden): non-blocking?
for res in iterator:
# take a completed point, is it an improvement?
point, score = res.result()
if score < best_score:
best_score, best_point = score, point
print(score, point)
x, y = best_point
newx, newy = (x + uniform(-scale, scale), y + uniform(-scale, scale))
# update plot
source.stream({'x': [newx], 'y': [newy], 'c': ['grey']}, rollover=20)
push_notebook(document=t)
# add new point, dynamically, to work on the cluster
new_point = c.submit(rosenbrock, (newx, newy))
iterator.add(new_point) # Start tracking new task as well
# Narrow search and consider stopping
scale *= 0.99
if scale < 0.001:
break
point
# In[37]:
