def test_pipe_stream(self):
stream = Pipe(data=self.df)
plot = self.df.hvplot('x', 'y', stream=stream)
pd.testing.assert_frame_equal(plot[()].data, self.df)
new_df = pd.DataFrame([[7, 8], [9, 10]], columns=['x', 'y'])
stream.send(new_df)
pd.testing.assert_frame_equal(plot[()].data, new_df)
def init_plot(self):
hv.extension("plotly")
self.best_pipe = Pipe(data=[])
self.best_dmap = hv.DynamicMap(hv.Scatter3D, streams=[self.best_pipe])
self.best_dmap = self.best_dmap.opts(
xlim=(-2, 2),
ylim=(-2, 4),
color="red",
alpha=0.7,
# height=600, width=600,
xaxis=None,
yaxis=None,
title="Best solution",
)
self.label_pipe = Pipe(data=[])
self.label_dmap = hv.DynamicMap(hv.Labels, streams=[self.label_pipe])
self.label_dmap = self.label_dmap.opts(
# height=200, width=400,
xaxis=None,
yaxis=None,
title="Best solution",
)
example = pd.DataFrame({"reward": []})
self.stream = Stream()
self.buffer_df = DataFrame(stream=self.stream, example=example)
self.score_dmap = self.buffer_df.hvplot(
y=["reward"]).opts( # height=200, width=400,
title="Best value found")
def _get_params(self):
df = self.get_data()
if self.streaming:
from holoviews.streams import Pipe
self._stream = Pipe(data=df)
return dict(object=self.get_plot(df),
sizing_mode='stretch_both') # todo update sizing mode
def __init__(self, plot_func, dynamic_range=True, plot_every=1):
import holoviews as hv
from holoviews.streams import Pipe
# Store the user-defined plot function
self.plot_func = plot_func
self.dynamic_range = dynamic_range
self.pipe = Pipe(data=[])
self.data_has_been_sent = False
self.dmap = hv.DynamicMap(self.plot_wrapper, streams=[self.pipe])
self.plot_every = plot_every
self.plot_count = 0
def init_plot(self):
self.frame_pipe = Pipe(data=[])
self.frame_dmap = hv.DynamicMap(hv.RGB, streams=[self.frame_pipe])
self.frame_dmap = self.frame_dmap.opts(xlim=(-0.5, 0.5),
ylim=(-0.5, 0.5),
xaxis=None,
yaxis=None,
title="Game screen")
example = pd.DataFrame({"reward": []})
self.stream = Stream()
self.buffer_df = DataFrame(stream=self.stream, example=example)
self.score_dmap = self.buffer_df.hvplot(y=["reward"]).opts(
height=200, width=500, title="Game score")
def _get_params(self):
df = self.get_data()
if df is None:
df = self.empty_df
self._stream = Pipe(data=df)
return dict(object=self.get_plot(),
sizing_mode="stretch_both") # todo update sizing mode
def new_game_of_life_graph(game_size, plot_size):
pipe = Pipe(data=[])
dmap = hv.DynamicMap(hv.Points, streams=[pipe])
cell_size = round(plot_size / (2 * game_size))
dmap.opts(color='Grey',
xlim=(0, game_size),
ylim=(0, game_size),
size=cell_size,
width=plot_size,
height=plot_size)
return dmap
def __init__(self, mapdata):
'''initialize a SubwayMap object
Args:
mapdata (SubwayMapData): container-class for stations and lines dataframes with implemented observer pattern.
This is necessary for data binding of the view to the viewmodel.
'''
Stream.__init__(self)
#create an initial map
stations, lines = mapdata.stationsdf, mapdata.linesdf
self.pipe = Pipe(data=[])
self.subway_map = gv.Path(lines, vdims=['color']).opts(
projection=crs.LambertConformal(),
height=800,
width=800,
color='color') * gv.DynamicMap(self.callback, streams=[self.pipe])
self.pipe.send(stations)
#bind changes in the stationsdf to pipe.send
mapdata.bind_to_stationsdf(self.pipe.send)
self.mapdata = mapdata
def init_plot(self):
self.frame_pipe = Pipe(data=[])
self.frame_dmap = hv.DynamicMap(hv.Labels, streams=[self.frame_pipe])
self.frame_dmap = self.frame_dmap.opts(
xlim=(-10, 10),
ylim=(0.5, 2.5),
height=200,
width=500,
xaxis=None,
yaxis=None,
title="Best solution",
)
example = pd.DataFrame({"reward": []})
self.stream = Stream()
self.buffer_df = DataFrame(stream=self.stream, example=example)
self.score_dmap = self.buffer_df.hvplot(y=["reward"]).opts(
height=200, width=400, title="Best value found")
def new_game_of_life_graph(game_size,
plot_size,
bgcolor=default_bgcolor,
cmap=default_cmap,
use_fixed_cell_sizes=default_use_fixed_cell_sizes,
max_cell_age=default_max_cell_age):
"""
The input points will be [x,y,z]. For plotting, we clip z to be no larger
than max_cell_age. That affects how many colors are chosen from the specified
color palette. A smaller value of max_cell_size produces faster changes, since
most lifetimes are low, except for large game sizes, where some patterns can
be semi-permanent. If use_fixed_cell_sizes=False, then the points grow in size
from zero (dead) to the clipped size. The log(z)+1 is calculated as an alternative
for mapping to colors and sizes, but it doesn't work that well for small
lifetimes. See comment below inline.
"""
def new_points(data):
data2 = data
if len(data) > 0:
x = np.array(data[0])
y = np.array(data[1])
z = np.array(data[2])
data2 = np.c_[x, y, z, np.log1p(z), np.clip(z, 0, max_cell_age)]
return hv.Points(data2,
kdims=['x', 'y'],
vdims=['z', 'z_log1p', 'z_clipped'])
pipe = Pipe(data=[])
dmap = hv.DynamicMap(new_points, streams=[pipe])
cell_sizes = 'z_clipped' # or use z_log1p or even try z.
if use_fixed_cell_sizes:
cell_sizes = fixed_cell_size(plot_size, game_size)
dmap.opts(
color='z_clipped', # or use z_log1p or z.
xlim=(0, game_size),
ylim=(0, game_size),
size=cell_sizes,
width=plot_size,
height=plot_size,
bgcolor=bgcolor,
cmap=cmap)
return dmap
_SQSbuffer__trainId__pnccdDetect__last_hit = online.DataBuffer(5)
#pnccd_dist_px, pnccd_rightLeft_offset_px
for k in range(5):
_SQSbuffer__pnCCD_hits(
bin2_image(
np.zeros(shape=(pnccd_dims[0] + pnccd_dist_px, pnccd_dims[1] +
2 * abs(pnccd_rightLeft_offset_px)))))
_SQSbuffer__pnCCD_hits_tids((0))
_SQSbuffer__counter = online.DataBuffer(buffer_length)
print("...2")
# Data pipes and buffers for plots
## pipes provide a full update of data to the underlying object eg. plot
## buffers add only a single value to the plot and may kick one out when number of elements in the buffer has reached the length/size of the buffer
_pipe__TOF_single = Pipe(data=[])
#_buffer__TOF_integral = Buffer(pd.DataFrame({'x':[],'y':[]}, columns=['x','y']), length=100, index=False)
_pipe__TOF_integral = Pipe(data=[])
_pipe__TOF_height = Pipe(data=[])
_pipe__pnCCD_single = Pipe(data=[])
_pipe__pnCCD_integral = Pipe(data=[])
_pipe__GMD_history = Pipe(data=[])
_pipe__pnCCD_hitrate = Pipe(data=[])
_pipe__pnCCD_hitrate_2 = Pipe(data=[])
_pipe__pnCCD_hits_tids = Pipe(data=[])
_pipe__pnCCD_hits_list = list()
for i in range(5):
_pipe__pnCCD_hits_list.append(Pipe(data=[]))
_pipe__pnCCD_hits__last_hit = Pipe(data=[])
_pipe__TOF_hits__last_hit = Pipe(data=[])
_pipe__trainId__last_hit = Pipe(data=[])
_SQSbuffer__TOF_integral = online.DataBuffer(buffer_len)
_SQSbuffer__TOF_integral_1 = online.DataBuffer(buffer_len)
_SQSbuffer__TOF_integral_2 = online.DataBuffer(buffer_len)
_SQSbuffer__TOF_integral_3 = online.DataBuffer(buffer_len)
_SQSbuffer__TOF_avg = online.DataBuffer(100)
_SQSbuffer__TOF_hit_trace = online.DataBuffer(1)
_SQSbuffer__TOF_hits = online.DataBuffer(1000)
_SQSbuffer__GMD_history = online.DataBuffer(buffer_len)
_SQSbuffer__TOF_hits = online.DataBuffer(10)
_SQSbuffer__counter = online.DataBuffer(buffer_len)
print("...2")
# Data pipes and buffers for plots
## pipes provide a full update of data to the underlying object eg. plot
## buffers add only a single value to the plot and may kick one out when number of elements in the buffer has reached the length/size of the buffer
_pipe__TOF_single = Pipe(data=[])
_pipe__TOF_hit_trace = Pipe(data=[])
_pipe__TOF_avg = Pipe(data=[])
_pipe__TOF_integral = Pipe(data=[])
_pipe__TOF_integral_1 = Pipe(data=[])
_pipe__TOF_integral_2 = Pipe(data=[])
_pipe__TOF_integral_3 = Pipe(data=[])
_pipe__GMD_history = Pipe(data=[])
# SETUP PLOTS
print("...3")
# example for coupled plots
# layout = hv.Layout(largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE") + largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE 2", cmap=['red'])).cols(1)
## TOF
bokeh_live_tof = largeData_line_plot(_pipe__TOF_single,
title="TOF single shots - LIVE",
buffer_length)
_SQSbuffer__pnCCD_mean_1_integral_proc_right = online.DataBuffer(buffer_length)
_SQSbuffer__pnCCD_mean_1_integral_proc_left = online.DataBuffer(buffer_length)
_SQSbuffer__pnCCD_mean_2_integral_proc = online.DataBuffer(buffer_length)
_SQSbuffer__pnCCD_mean_helper_1 = online.DataBuffer(mean_1_length)
_SQSbuffer__pnCCD_mean_helper_2 = online.DataBuffer(mean_2_length)
_SQSbuffer__xgm_mean_helper_1 = online.DataBuffer(mean_1_length)
_SQSbuffer__xgm_mean_helper_2 = online.DataBuffer(mean_2_length)
_SQSbuffer__counter = online.DataBuffer(buffer_length)
_SQSbuffer__counter_time = online.DataBuffer(buffer_length)
print("...2")
# Data pipes and buffers for plots
## pipes provide a full update of data to the underlying object eg. plot
## buffers add only a single value to the plot and may kick one out when number of elements in the buffer has reached the length/size of the buffer
_pipe__pnCCD_single = Pipe(data=[])
_pipe__pnCCD_mean_1 = Pipe(data=[])
_pipe__pnCCD_mean_2 = Pipe(data=[])
_pipe__pnCCD_integral = Pipe(data=[])
_pipe__pnCCD_mean_1_integral = Pipe(data=[])
_pipe__pnCCD_mean_2_integral = Pipe(data=[])
_pipe__pnCCD_integral_by_gmd = Pipe(data=[])
_pipe__pnCCD_mean_1_integral_by_gmd = Pipe(data=[])
_pipe__pnCCD_mean_1_integral_by_gmd_top = Pipe(data=[])
_pipe__pnCCD_mean_1_integral_by_gmd_bottom = Pipe(data=[])
_pipe__pnCCD_mean_1_integral_by_gmd_right = Pipe(data=[])
_pipe__pnCCD_mean_1_integral_by_gmd_left = Pipe(data=[])
_pipe__pnCCD_mean_2_integral_by_gmd = Pipe(data=[])
_pipe__GMD_history = Pipe(data=[])
_pipe__pnCCD_hits__last_hit = Pipe(data=[])
_pipe__trainId__last_hit = Pipe(data=[])
return sqs_bk.hv_to_bokeh_obj( TOF_dmap.opts(width=width,height=height,ylim=ylim,xlim=xlim, xlabel=xlabel, ylabel=ylabel, title = title), renderer)
def start_stop_dataThread():
global makeBigData_stop
if not makeBigData_stop:
makeBigData_stop = True
else:
thread.start()
# Data buffers for live stream
print("...2")
# Data pipes and buffers for plots
## pipes provide a full update of data to the underlying object eg. plot
## buffers add only a single value to the plot and may kick one out when number of elements in the buffer has reached the length/size of the buffer
_pipe__TOF_single = Pipe(data=[])
# SETUP PLOTS
print("...3")
# example for coupled plots
# layout = hv.Layout(largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE") + largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE 2", cmap=['red'])).cols(1)
## TOF
bokeh_live_tof = largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE", width = 1900, height=900)
## SET UP Additional Widgets
bokeh_button_StartStop = Button(label = "Start / Stop", button_type="success")
bokeh_button_StartStop.on_click(start_stop_dataThread)
# SET UP BOKEH LAYOUT
#
bokeh_row_1 = row(bokeh_live_tof)
bokeh_row_interact = bokeh_button_StartStop
bokeh_layout = column(bokeh_row_1, bokeh_row_interact)
class SubwayMap(Stream):
'''subway map (holoviews dynamic map) including pyviz.param-based control interface.
inherited from a holoviews stream object.'''
#class variables
direction_list = ['North', 'South']
lines_list = [
'All', '1', '2', '3', '4', '5', '6', '7', 'A', 'B', 'C', 'D', 'E', 'F',
'G', 'H', 'J', 'L', 'M', 'N', 'Q', 'R', 'SI', 'W'
]
display_list = ['Time since last train', 'Probability of train delay']
#Selector class variables (parameters) for holoviews panel
direction = param.ObjectSelector(default='North', objects=direction_list)
line = param.ObjectSelector(default='All', objects=lines_list)
display = param.ObjectSelector(default='Probability of train delay',
objects=display_list)
def callback(self, data):
if (self.display == 'Probability of train delay'):
layout = gv.Points(data,
vdims=[
'color', 'displaysize', 'name',
'waittime_str', 'delay_prob', 'MTAdelay',
'inboundtrain', 'inbound_from'
]).opts(tools=[SubwayMap.hover_delays],
size='displaysize',
color='color')
#elif(self.display == 'Time since last train'):
else:
layout = gv.Points(data,
vdims=[
'waittimecolor', 'waittimedisplaysize',
'name', 'waittime_str', 'delay_prob',
'MTAdelay'
]).opts(tools=[SubwayMap.hover_waittime],
size='waittimedisplaysize',
color='waittimecolor')
return layout
def __init__(self, mapdata):
'''initialize a SubwayMap object
Args:
mapdata (SubwayMapData): container-class for stations and lines dataframes with implemented observer pattern.
This is necessary for data binding of the view to the viewmodel.
'''
Stream.__init__(self)
#create an initial map
stations, lines = mapdata.stationsdf, mapdata.linesdf
self.pipe = Pipe(data=[])
self.subway_map = gv.Path(lines, vdims=['color']).opts(
projection=crs.LambertConformal(),
height=800,
width=800,
color='color') * gv.DynamicMap(self.callback, streams=[self.pipe])
self.pipe.send(stations)
#bind changes in the stationsdf to pipe.send
mapdata.bind_to_stationsdf(self.pipe.send)
self.mapdata = mapdata
hover_delays = HoverTool(
tooltips=[("station", "@name"), (
"incoming train",
"@inboundtrain"), ("from station", "@inbound_from"),
("probability that incoming train is delayed",
"@delay_prob"), ("MTA reports delay?", "@MTAdelay")])
hover_waittime = HoverTool(
tooltips=[("station",
"@name"), ("time since last train", "@waittime_str")])
def view(self):
return self.subway_map
#https://panel.pyviz.org/user_guide/Param.html
@param.depends('direction', 'line', watch=True)
def update(self):
self.mapdata.selected_dir = self.direction
self.mapdata.selected_line = self.line
def _process_data(self, kind, data, x, y, by, groupby, row, col, use_dask,
persist, backlog, label, value_label, hover_cols, kwds):
gridded = kind in self._gridded_types
gridded_data = False
# Validate DataSource
self.data_source = data
self.is_series = is_series(data)
if self.is_series:
data = data.to_frame()
if is_intake(data):
data = process_intake(data, use_dask or persist)
if groupby is not None and not isinstance(groupby, list):
groupby = [groupby]
if by is not None and not isinstance(by, list):
by = [by]
streaming = False
if isinstance(data, pd.DataFrame):
self.data = data
if is_geopandas(data) and kind is None:
geom_types = set(
[gt[5:] if 'Multi' in gt else gt for gt in data.geom_type])
if len(geom_types) > 1:
raise ValueError(
'The GeopandasInterface can only read dataframes which '
'share a common geometry type')
geom_type = list(geom_types)[0]
if geom_type == 'Point':
kind = 'points'
elif geom_type == 'Polygon':
kind = 'polygons'
elif geom_type in ('LineString', 'LineRing'):
kind = 'paths'
elif is_dask(data):
self.data = data.persist() if persist else data
elif is_streamz(data):
self.data = data.example
self.stream_type = data._stream_type
streaming = True
self.cb = data
if data._stream_type == 'updating':
self.stream = Pipe(data=self.data)
else:
self.stream = Buffer(data=self.data,
length=backlog,
index=False)
data.stream.gather().sink(self.stream.send)
elif is_xarray(data):
import xarray as xr
z = kwds.get('z')
if z is None and isinstance(data, xr.Dataset):
z = list(data.data_vars)[0]
if gridded and isinstance(data,
xr.Dataset) and not isinstance(z, list):
data = data[z]
ignore = (groupby or []) + (by or [])
dims = [
c for c in data.coords
if data[c].shape != () and c not in ignore
]
if kind is None and (not (x or y) or all(c in data.coords
for c in (x, y))):
if len(dims) == 1:
kind = 'line'
elif len(dims) == 2 or (x and y):
kind = 'image'
gridded = True
else:
kind = 'hist'
if gridded:
gridded_data = True
data, x, y, by_new, groupby_new = process_xarray(
data, x, y, by, groupby, use_dask, persist, gridded, label,
value_label)
if kind not in self._stats_types:
if by is None: by = by_new
if groupby is None: groupby = groupby_new
if groupby:
groupby = [g for g in groupby if g not in (row, col)]
self.data = data
else:
raise ValueError('Supplied data type %s not understood' %
type(data).__name__)
# Validate data and arguments
if by is None: by = []
if groupby is None: groupby = []
if gridded:
if not gridded_data:
raise ValueError('%s plot type requires gridded data, '
'e.g. a NumPy array or xarray Dataset, '
'found %s type' %
(kind, type(self.data).__name__))
not_found = [g for g in groupby if g not in data.coords]
data_vars = list(data.data_vars) if isinstance(
data, xr.Dataset) else [data.name]
indexes = list(data.coords)
self.variables = list(data.coords) + data_vars
if groupby and not_found:
raise ValueError('The supplied groupby dimension(s) %s '
'could not be found, expected one or '
'more of: %s' %
(not_found, list(data.coords)))
else:
# Determine valid indexes
if isinstance(self.data, pd.DataFrame):
if self.data.index.names == [None]:
indexes = [self.data.index.name or 'index']
else:
indexes = list(self.data.index.names)
else:
indexes = [
c for c in self.data.reset_index().columns
if c not in self.data.columns
]
if len(indexes) == 2 and not (x or y or by):
if kind == 'heatmap':
x, y = indexes
elif kind in ('bar', 'barh'):
x, by = indexes
# Rename non-string columns
renamed = {
c: str(c)
for c in data.columns if not isinstance(c, hv.util.basestring)
}
if renamed:
self.data = self.data.rename(columns=renamed)
self.variables = indexes + list(self.data.columns)
# Reset groupby dimensions
groupby_index = [g for g in groupby if g in indexes]
if groupby_index:
self.data = self.data.reset_index(groupby_index)
not_found = [
g for g in groupby
if g not in list(self.data.columns) + indexes
]
if groupby and not_found:
raise ValueError('The supplied groupby dimension(s) %s '
'could not be found, expected one or '
'more of: %s' %
(not_found, list(self.data.columns)))
# Set data-level options
self.x = x
self.y = y
self.kind = kind or 'line'
self.gridded = gridded
self.use_dask = use_dask
self.indexes = indexes
if isinstance(by, (np.ndarray, pd.Series)):
self.data['by'] = by
self.by = ['by']
elif not by:
self.by = []
else:
self.by = by if isinstance(by, list) else [by]
self.groupby = groupby
self.streaming = streaming
self.hover_cols = hover_cols
class hvRectangleAppView(View):
opts = param.Dict(default={}, doc="HoloViews option to apply on the plot.")
view_type = 'rectangles'
streaming = param.Boolean(default=False,
doc="""
Whether to stream new data to the plot or rerender the plot.""")
def __init__(self, **params):
# import hvplot.pandas # noqa
# if 'dask' in sys.modules:
# try:
# import hvplot.dask # noqa
# except Exception:
# pass
self._stream = None
self._linked_objs = []
super().__init__(**params)
def get_panel(self):
kwargs = self._get_params()
#interactive? https://github.com/holoviz/panel/issues/1824
return pn.pane.HoloViews(**kwargs)
def get_plot(self, df):
"""
Dataframe df must have columns x0, y0, x1, y1 (in this order) for coordinates
bottom-left (x0, y0) and top right (x1, y1). Optionally a fifth value-column can be provided for colors
Parameters
----------
df
Returns
-------
"""
# processed = {}
# for k, v in self.kwargs.items():
# if k.endswith('formatter') and isinstance(v, str) and '%' not in v:
# v = NumeralTickFormatter(format=v)
# processed[k] = v
# if self.streaming:
# processed['stream'] = self._stream
#hvplots stream? https://holoviews.org/user_guide/Streaming_Data.html
# plot = hv.Rectangles([(0, 0, 1, 1), (2, 3, 4, 6), (0.5, 2, 1.5, 4), (2, 1, 3.5, 2.5)])
processed = {}
for k, v in self.kwargs.items():
if k.endswith('formatter') and isinstance(v, str) and '%' not in v:
v = NumeralTickFormatter(format=v)
processed[k] = v
if self.streaming:
#processed['stream'] = self._stream
plot = hv.DynamicMap(hv.Rectangles, streams=[self._stream])
plot = plot.apply.opts(**self.opts) if self.opts else plot
else:
plot = hv.Rectangles(df)
plot.opts(**self.opts) if self.opts else plot
if self.selection_group or 'selection_expr' in self._param_watchers:
plot = self._link_plot(plot)
return plot
def _get_params(self):
df = self.get_data()
if self.streaming:
from holoviews.streams import Pipe
self._stream = Pipe(data=df)
return dict(object=self.get_plot(df),
sizing_mode='stretch_both') # todo update sizing mode
def get_data(self):
#todo uniformify this method for all views
try:
return super().get_data()
except (KeyError, ValueError) as e:
print(f'Empty data in {self.__class__}: {e}')
return self.empty_df
def update(self, *events, invalidate_cache=True):
"""
Triggers an update in the View.
Parameters
----------
events: tuple
param events that may trigger an update.
invalidate_cache : bool
Whether to clear the View's cache.
Returns
-------
stale : bool
Whether the panel on the View is stale and needs to be
rerendered.
"""
# Skip events triggered by a parameter change on this View
own_parameters = [self.param[p] for p in self.param]
own_events = events and all(
isinstance(e.obj, ParamFilter) and
(e.obj.parameter in own_parameters
or e.new is self._ls.selection_expr) for e in events)
if own_events:
return False
if invalidate_cache:
self._cache = None
if not self.streaming or self._stream is None:
upd = self._update_panel()
return upd
self._stream.send(self.get_data())
return False
@property
def empty_df(self):
return pd.DataFrame([[0] * 5],
columns=['x0', 'x1', 'y0', 'y1', 'value'])
class DistributedSwarm:
def __init__(
self,
swarm: Callable,
n_swarms: int,
n_param_servers: int,
max_iters_ray: int = 10,
log_every: int = 100,
n_comp_add: int = 5,
minimize: bool = False,
ps_maxlen: int = 100,
init_reward: float = None,
log_reward: bool = False,
):
self.n_swarms = n_swarms
self.minimize = minimize
self.log = log_reward
self.init_reward = (init_reward if init_reward is not None else
(np.inf if minimize else -np.inf))
self.log_every = log_every
self.param_servers = [
ParamServer.remote(minimize=minimize, maxlen=ps_maxlen)
for _ in range(n_param_servers)
]
self.swarms = [
RemoteSwarm.remote(copy.copy(swarm),
int(n_comp_add),
minimize=minimize) for _ in range(self.n_swarms)
]
self.max_iters_ray = max_iters_ray
self.frame_pipe: Pipe = None
self.stream = None
self.buffer_df = None
self.score_dmap = None
self.frame_dmap = None
self.init_plot()
self.n_iters = 0
self.best = (None, None, None)
def init_plot(self):
self.frame_pipe = Pipe(data=[])
self.frame_dmap = hv.DynamicMap(hv.RGB, streams=[self.frame_pipe])
self.frame_dmap = self.frame_dmap.opts(xlim=(-0.5, 0.5),
ylim=(-0.5, 0.5),
xaxis=None,
yaxis=None,
title="Game screen")
example = pd.DataFrame({"reward": []})
self.stream = Stream()
self.buffer_df = DataFrame(stream=self.stream, example=example)
self.score_dmap = self.buffer_df.hvplot(y=["reward"]).opts(
height=200, width=500, title="Game score")
def plot(self):
return self.frame_dmap + self.score_dmap
def stream_progress(self, state, observation, reward):
example = pd.DataFrame({"reward": [reward]},
index=[self.n_iters // self.n_swarms])
self.stream.emit(example)
obs = observation.reshape((210, 160, 3)).astype(np.uint8)
self.frame_pipe.send(obs)
def run_swarm(self):
self.n_iters = 0
best_ids = [s.reset.remote() for s in self.swarms]
steps = {}
param_servers = deque([])
for worker, best in zip(self.swarms, best_ids):
steps[worker.make_iteration.remote(best)] = worker
bests = []
for ps, walker in zip(self.param_servers,
list(steps.keys())[:len(self.param_servers)]):
bests.append(ps.exchange_walker.remote(walker))
param_servers.append(ps)
ray.get(bests)
for i in range(self.max_iters_ray * len(self.swarms)):
self.n_iters += 1
ready_bests, _ = ray.wait(list(steps))
ready_best_id = ready_bests[0]
worker = steps.pop(ready_best_id)
ps = param_servers.popleft()
new_best = ps.exchange_walker.remote(ready_best_id)
param_servers.append(ps)
steps[worker.make_iteration.remote(new_best)] = worker
if i % (self.log_every * len(self.swarms)) == 0:
id_, _ = ray.wait([param_servers[-1].get_best.remote()])
(state, best_obs, best_reward) = ray.get(id_)[0]
if state is not None:
self.best = (state, best_obs, float(best_reward))
if ((best_reward > self.init_reward) if self.minimize else
(best_reward < self.init_reward)):
best_reward = self.init_reward
best_reward = np.log(
best_reward) if self.log else best_reward
self.stream_progress(state, best_obs, best_reward)
else:
print("skipping, not ready")
def _get_params(self):
df = self.get_data()
if self.streaming:
from holoviews.streams import Pipe
self._stream = Pipe(data=df)
return dict(object=self.get_plot(df))
class hvPlotView(View):
"""
The hvPlotView renders the queried data as a bokeh plot generated
with hvPlot. hvPlot allows for a concise declaration of a plot via
its simple API.
"""
kind = param.String(doc="The kind of plot, e.g. 'scatter' or 'line'.")
x = param.Selector(doc="The column to render on the x-axis.")
y = param.Selector(doc="The column to render on the y-axis.")
by = param.ListSelector(doc="The column(s) to facet the plot by.")
groupby = param.ListSelector(doc="The column(s) to group by.")
opts = param.Dict(default={},
doc="HoloViews options to apply on the plot.")
streaming = param.Boolean(default=False,
doc="""
Whether to stream new data to the plot or rerender the plot.""")
selection_expr = param.Parameter(doc="""
A selection expression caputirng the current selection applied
on the plot.""")
view_type = 'hvplot'
_field_params = ['x', 'y', 'by', 'groupby']
_supports_selections = True
def __init__(self, **params):
import hvplot.pandas # noqa
if 'dask' in sys.modules:
try:
import hvplot.dask # noqa
except Exception:
pass
if 'by' in params and isinstance(params['by'], str):
params['by'] = [params['by']]
if 'groupby' in params and isinstance(params['groupby'], str):
params['groupby'] = [params['groupby']]
self._stream = None
self._linked_objs = []
super().__init__(**params)
def get_plot(self, df):
processed = {}
for k, v in self.kwargs.items():
if k.endswith('formatter') and isinstance(v, str) and '%' not in v:
v = NumeralTickFormatter(format=v)
processed[k] = v
if self.streaming:
processed['stream'] = self._stream
plot = df.hvplot(kind=self.kind, x=self.x, y=self.y, **processed)
plot = plot.opts(**self.opts) if self.opts else plot
if self.selection_group or 'selection_expr' in self._param_watchers:
plot = self._link_plot(plot)
return plot
def _link_plot(self, plot):
self._init_link_selections()
linked_objs = list(self._ls._plot_reset_streams)
plot = self._ls(plot)
self._linked_objs += [
o for o in self._ls._plot_reset_streams if o not in linked_objs
]
return plot
def _cleanup(self):
if self._ls is None:
return
for obj in self._linked_objs:
reset = self._ls._plot_reset_streams.pop(obj)
sel_expr = self._ls._selection_expr_streams.pop(obj)
self._ls._cross_filter_stream.input_streams.remove(sel_expr)
sel_expr.clear()
sel_expr.source = None
reset.clear()
reset.source = None
self._linked_objs = []
def get_panel(self):
return pn.pane.HoloViews(**self._get_params())
def _get_params(self):
df = self.get_data()
if self.streaming:
from holoviews.streams import Pipe
self._stream = Pipe(data=df)
return dict(object=self.get_plot(df))
def update(self, *events, invalidate_cache=True):
"""
Triggers an update in the View.
Parameters
----------
events: tuple
param events that may trigger an update.
invalidate_cache : bool
Whether to clear the View's cache.
Returns
-------
stale : bool
Whether the panel on the View is stale and needs to be
rerendered.
"""
# Skip events triggered by a parameter change on this View
own_parameters = [self.param[p] for p in self.param]
own_events = events and all(
isinstance(e.obj, ParamFilter) and
(e.obj.parameter in own_parameters
or e.new is self._ls.selection_expr) for e in events)
if own_events:
return False
if invalidate_cache:
self._cache = None
if not self.streaming or self._stream is None:
return self._update_panel()
self._stream.send(self.get_data())
return False
# Data buffers for live stream
_SQSbuffer__TOF_integral = online.DataBuffer(100)
_SQSbuffer__GMD_history = online.DataBuffer(100)
_SQSbuffer__pnCCD_integral = online.DataBuffer(100)
_SQSbuffer__pnCCD_hits = online.DataBuffer(10)
for k in range(5):
_SQSbuffer__pnCCD_hits(np.zeros(shape=(1024,1024)))
_SQSbuffer__counter = online.DataBuffer(100)
print("...2")
# Data pipes and buffers for plots
## pipes provide a full update of data to the underlying object eg. plot
## buffers add only a single value to the plot and may kick one out when number of elements in the buffer has reached the length/size of the buffer
_pipe__TOF_single = Pipe(data=[])
#_buffer__TOF_integral = Buffer(pd.DataFrame({'x':[],'y':[]}, columns=['x','y']), length=100, index=False)
_pipe__TOF_integral = Pipe(data=[])
_pipe__pnCCD_single = Pipe(data=[])
_pipe__pnCCD_integral = Pipe(data=[])
_pipe__GMD_history = Pipe(data=[])
_pipe__pnCCD_hits_list = list()
for i in range(5):
_pipe__pnCCD_hits_list.append(Pipe(data=[]))
# SETUP PLOTS
print("...3")
# example for coupled plots
# layout = hv.Layout(largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE") + largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE 2", cmap=['red'])).cols(1)
## TOF
bokeh_live_tof = largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE", width = 500, height=500)
class Animator:
"""
Creates animated plots with holoviews.
Constructor Args:
dynamic_range: bool = if set to True, auto-scale the chart each time it is drawn
# ------------ Simple Example -------------------
# Define a function to make your plots.
# Data will be whatever object is passed to animator.send()
def myplot(data):
return hv.Curve(data)
# Pass your plotting function to the animator constructor
animator = Animator(myplot)
# Simply send the animator updated versions of your data
# to update the plot
t0 = np.linspace(0, 6., 100)
for delta in np.linspace(0, 3, 30):
t = t0 - delta
y = 2 + np.sin(t)
animator.send((t, y))
# ------------ Advanced Example -------------------
def myplot(data):
# First two elements of data are to be ploted
# Third element of data is going to update a label
c = hv.Curve(data[:2], 'a', 'b', label=f'hello {data[-1]}').options(color=ezr.cc.c, logy=True)
c *= hv.Scatter((data[0], 2 * data[1])).options(color=ezr.cc.d)
return c
# Send data for animation
for ind, delta in enumerate(np.linspace(0, 3, 300)):
t = t0 - delta
y = 2 + np.sin(t)
# Can control when animations are drawn with this
if ind % 1 == 0:
animator.send((t, y))
"""
def __init__(self, plot_func, dynamic_range=True, plot_every=1):
import holoviews as hv
from holoviews.streams import Pipe
# Store the user-defined plot function
self.plot_func = plot_func
self.dynamic_range = dynamic_range
self.pipe = Pipe(data=[])
self.data_has_been_sent = False
self.dmap = hv.DynamicMap(self.plot_wrapper, streams=[self.pipe])
self.plot_every = plot_every
self.plot_count = 0
def plot_wrapper(self, *args, **kwargs):
data = kwargs.get('data', ([0], [0]))
hv_obj = self.plot_func(data)
if self.dynamic_range:
hv_obj = hv_obj.opts(norm=dict(framewise=True))
return hv_obj
def send(self, data):
from IPython.display import display
if self.plot_count % self.plot_every == 0:
self.pipe.send(data)
if not self.data_has_been_sent:
display(self.dmap)
self.data_has_been_sent = True
self.plot_count += 1
def fit(self,
*,
x=None,
y=None,
weights=None,
model=None,
cost=None,
plot_every=None,
algorithm='fmin',
verbose=True):
"""
The method to use for training the fitter.
Args:
x: the indepenant data
y: the depenant data
model: a model function to fit the data to
cost: a custom cost function (defaults to least squares)
plot_every: Plot solution in real time every this number of iterations
algorithm: Scipy optimization routine to use. Enter nonsense to see list of valid.
verbose: Print convergence information
"""
import numpy as np
from scipy import optimize
import holoviews as hv
from holoviews.streams import Pipe
from IPython.display import display
self.plot_every = plot_every
if algorithm not in self.OPTIMIZER_NAMES:
raise ValueError(
f'Invalid optimizer {algorithm}. Choose one of {self.OPTIMIZER_NAMES}'
)
givens = dict(x=x, y=y)
if weights is not None:
givens.update({'weights': weights})
givens.update(self._givens)
self._params.given(**givens)
# This stuff only needs to happen if we are iteratively plotting fits
if plot_every is not None:
x, y = self._params.x, self._params.y
xmin, xmax = np.min(x), np.max(x)
ymin, ymax = np.min(y), np.max(y)
scale = .1
delta_x = scale * (xmax - xmin)
delta_y = scale * (ymax - ymin)
xmin, xmax = xmin - delta_x, xmax + delta_x
ymin, ymax = ymin - delta_y, ymax + delta_y
xlim = (xmin, xmax)
ylim = (ymin, ymax)
self.pipe = Pipe(data=[])
try:
dmap = hv.DynamicMap(self._plotter, streams=[self.pipe])
dmap.opts(hv.opts.Overlay(xlim=xlim, ylim=ylim))
display(dmap)
except AttributeError:
raise RuntimeError(
'You must import holoviews and set bokeh backround for plotting to work'
)
if model is None and cost is None:
raise ValueError(
'You must supply either a model function or a cost function')
self._raw_model = model
self._model = self._model_wrapper(self._raw_model)
a0 = self._params.array
if cost is not None:
self._cost = cost
optimizer = getattr(optimize, algorithm)
self._optimizer_kwargs.update(disp=verbose)
a_fit = optimizer(self._cost_wrapper,
a0,
args=(self._params, ),
**self._optimizer_kwargs)
a_fit = np.array(a_fit, ndmin=1)
self._params.ingest(a_fit)
return self
