def test_pipe_stream(self):
stream = Pipe(data=self.df)
plot = self.df.hvplot('x', 'y', stream=stream)
pd.testing.assert_frame_equal(plot[()].data, self.df)
new_df = pd.DataFrame([[7, 8], [9, 10]], columns=['x', 'y'])
stream.send(new_df)
pd.testing.assert_frame_equal(plot[()].data, new_df)
class SubwayMap(Stream):
'''subway map (holoviews dynamic map) including pyviz.param-based control interface.
inherited from a holoviews stream object.'''
#class variables
direction_list = ['North', 'South']
lines_list = [
'All', '1', '2', '3', '4', '5', '6', '7', 'A', 'B', 'C', 'D', 'E', 'F',
'G', 'H', 'J', 'L', 'M', 'N', 'Q', 'R', 'SI', 'W'
]
display_list = ['Time since last train', 'Probability of train delay']
#Selector class variables (parameters) for holoviews panel
direction = param.ObjectSelector(default='North', objects=direction_list)
line = param.ObjectSelector(default='All', objects=lines_list)
display = param.ObjectSelector(default='Probability of train delay',
objects=display_list)
def callback(self, data):
if (self.display == 'Probability of train delay'):
layout = gv.Points(data,
vdims=[
'color', 'displaysize', 'name',
'waittime_str', 'delay_prob', 'MTAdelay',
'inboundtrain', 'inbound_from'
]).opts(tools=[SubwayMap.hover_delays],
size='displaysize',
color='color')
#elif(self.display == 'Time since last train'):
else:
layout = gv.Points(data,
vdims=[
'waittimecolor', 'waittimedisplaysize',
'name', 'waittime_str', 'delay_prob',
'MTAdelay'
]).opts(tools=[SubwayMap.hover_waittime],
size='waittimedisplaysize',
color='waittimecolor')
return layout
def __init__(self, mapdata):
'''initialize a SubwayMap object
Args:
mapdata (SubwayMapData): container-class for stations and lines dataframes with implemented observer pattern.
This is necessary for data binding of the view to the viewmodel.
'''
Stream.__init__(self)
#create an initial map
stations, lines = mapdata.stationsdf, mapdata.linesdf
self.pipe = Pipe(data=[])
self.subway_map = gv.Path(lines, vdims=['color']).opts(
projection=crs.LambertConformal(),
height=800,
width=800,
color='color') * gv.DynamicMap(self.callback, streams=[self.pipe])
self.pipe.send(stations)
#bind changes in the stationsdf to pipe.send
mapdata.bind_to_stationsdf(self.pipe.send)
self.mapdata = mapdata
hover_delays = HoverTool(
tooltips=[("station", "@name"), (
"incoming train",
"@inboundtrain"), ("from station", "@inbound_from"),
("probability that incoming train is delayed",
"@delay_prob"), ("MTA reports delay?", "@MTAdelay")])
hover_waittime = HoverTool(
tooltips=[("station",
"@name"), ("time since last train", "@waittime_str")])
def view(self):
return self.subway_map
#https://panel.pyviz.org/user_guide/Param.html
@param.depends('direction', 'line', watch=True)
def update(self):
self.mapdata.selected_dir = self.direction
self.mapdata.selected_line = self.line
class hvRectangleAppView(View):
opts = param.Dict(default={}, doc="HoloViews option to apply on the plot.")
view_type = 'rectangles'
streaming = param.Boolean(default=False,
doc="""
Whether to stream new data to the plot or rerender the plot.""")
def __init__(self, **params):
# import hvplot.pandas # noqa
# if 'dask' in sys.modules:
# try:
# import hvplot.dask # noqa
# except Exception:
# pass
self._stream = None
self._linked_objs = []
super().__init__(**params)
def get_panel(self):
kwargs = self._get_params()
#interactive? https://github.com/holoviz/panel/issues/1824
return pn.pane.HoloViews(**kwargs)
def get_plot(self, df):
"""
Dataframe df must have columns x0, y0, x1, y1 (in this order) for coordinates
bottom-left (x0, y0) and top right (x1, y1). Optionally a fifth value-column can be provided for colors
Parameters
----------
df
Returns
-------
"""
# processed = {}
# for k, v in self.kwargs.items():
# if k.endswith('formatter') and isinstance(v, str) and '%' not in v:
# v = NumeralTickFormatter(format=v)
# processed[k] = v
# if self.streaming:
# processed['stream'] = self._stream
#hvplots stream? https://holoviews.org/user_guide/Streaming_Data.html
# plot = hv.Rectangles([(0, 0, 1, 1), (2, 3, 4, 6), (0.5, 2, 1.5, 4), (2, 1, 3.5, 2.5)])
processed = {}
for k, v in self.kwargs.items():
if k.endswith('formatter') and isinstance(v, str) and '%' not in v:
v = NumeralTickFormatter(format=v)
processed[k] = v
if self.streaming:
#processed['stream'] = self._stream
plot = hv.DynamicMap(hv.Rectangles, streams=[self._stream])
plot = plot.apply.opts(**self.opts) if self.opts else plot
else:
plot = hv.Rectangles(df)
plot.opts(**self.opts) if self.opts else plot
if self.selection_group or 'selection_expr' in self._param_watchers:
plot = self._link_plot(plot)
return plot
def _get_params(self):
df = self.get_data()
if self.streaming:
from holoviews.streams import Pipe
self._stream = Pipe(data=df)
return dict(object=self.get_plot(df),
sizing_mode='stretch_both') # todo update sizing mode
def get_data(self):
#todo uniformify this method for all views
try:
return super().get_data()
except (KeyError, ValueError) as e:
print(f'Empty data in {self.__class__}: {e}')
return self.empty_df
def update(self, *events, invalidate_cache=True):
"""
Triggers an update in the View.
Parameters
----------
events: tuple
param events that may trigger an update.
invalidate_cache : bool
Whether to clear the View's cache.
Returns
-------
stale : bool
Whether the panel on the View is stale and needs to be
rerendered.
"""
# Skip events triggered by a parameter change on this View
own_parameters = [self.param[p] for p in self.param]
own_events = events and all(
isinstance(e.obj, ParamFilter) and
(e.obj.parameter in own_parameters
or e.new is self._ls.selection_expr) for e in events)
if own_events:
return False
if invalidate_cache:
self._cache = None
if not self.streaming or self._stream is None:
upd = self._update_panel()
return upd
self._stream.send(self.get_data())
return False
@property
def empty_df(self):
return pd.DataFrame([[0] * 5],
columns=['x0', 'x1', 'y0', 'y1', 'value'])
class hvPlotView(View):
"""
The hvPlotView renders the queried data as a bokeh plot generated
with hvPlot. hvPlot allows for a concise declaration of a plot via
its simple API.
"""
kind = param.String(doc="The kind of plot, e.g. 'scatter' or 'line'.")
x = param.Selector(doc="The column to render on the x-axis.")
y = param.Selector(doc="The column to render on the y-axis.")
by = param.ListSelector(doc="The column(s) to facet the plot by.")
groupby = param.ListSelector(doc="The column(s) to group by.")
opts = param.Dict(default={},
doc="HoloViews options to apply on the plot.")
streaming = param.Boolean(default=False,
doc="""
Whether to stream new data to the plot or rerender the plot.""")
selection_expr = param.Parameter(doc="""
A selection expression caputirng the current selection applied
on the plot.""")
view_type = 'hvplot'
_field_params = ['x', 'y', 'by', 'groupby']
_supports_selections = True
def __init__(self, **params):
import hvplot.pandas # noqa
if 'dask' in sys.modules:
try:
import hvplot.dask # noqa
except Exception:
pass
if 'by' in params and isinstance(params['by'], str):
params['by'] = [params['by']]
if 'groupby' in params and isinstance(params['groupby'], str):
params['groupby'] = [params['groupby']]
self._stream = None
self._linked_objs = []
super().__init__(**params)
def get_plot(self, df):
processed = {}
for k, v in self.kwargs.items():
if k.endswith('formatter') and isinstance(v, str) and '%' not in v:
v = NumeralTickFormatter(format=v)
processed[k] = v
if self.streaming:
processed['stream'] = self._stream
plot = df.hvplot(kind=self.kind, x=self.x, y=self.y, **processed)
plot = plot.opts(**self.opts) if self.opts else plot
if self.selection_group or 'selection_expr' in self._param_watchers:
plot = self._link_plot(plot)
return plot
def _link_plot(self, plot):
self._init_link_selections()
linked_objs = list(self._ls._plot_reset_streams)
plot = self._ls(plot)
self._linked_objs += [
o for o in self._ls._plot_reset_streams if o not in linked_objs
]
return plot
def _cleanup(self):
if self._ls is None:
return
for obj in self._linked_objs:
reset = self._ls._plot_reset_streams.pop(obj)
sel_expr = self._ls._selection_expr_streams.pop(obj)
self._ls._cross_filter_stream.input_streams.remove(sel_expr)
sel_expr.clear()
sel_expr.source = None
reset.clear()
reset.source = None
self._linked_objs = []
def get_panel(self):
return pn.pane.HoloViews(**self._get_params())
def _get_params(self):
df = self.get_data()
if self.streaming:
from holoviews.streams import Pipe
self._stream = Pipe(data=df)
return dict(object=self.get_plot(df))
def update(self, *events, invalidate_cache=True):
"""
Triggers an update in the View.
Parameters
----------
events: tuple
param events that may trigger an update.
invalidate_cache : bool
Whether to clear the View's cache.
Returns
-------
stale : bool
Whether the panel on the View is stale and needs to be
rerendered.
"""
# Skip events triggered by a parameter change on this View
own_parameters = [self.param[p] for p in self.param]
own_events = events and all(
isinstance(e.obj, ParamFilter) and
(e.obj.parameter in own_parameters
or e.new is self._ls.selection_expr) for e in events)
if own_events:
return False
if invalidate_cache:
self._cache = None
if not self.streaming or self._stream is None:
return self._update_panel()
self._stream.send(self.get_data())
return False
class Animator:
"""
Creates animated plots with holoviews.
Constructor Args:
dynamic_range: bool = if set to True, auto-scale the chart each time it is drawn
# ------------ Simple Example -------------------
# Define a function to make your plots.
# Data will be whatever object is passed to animator.send()
def myplot(data):
return hv.Curve(data)
# Pass your plotting function to the animator constructor
animator = Animator(myplot)
# Simply send the animator updated versions of your data
# to update the plot
t0 = np.linspace(0, 6., 100)
for delta in np.linspace(0, 3, 30):
t = t0 - delta
y = 2 + np.sin(t)
animator.send((t, y))
# ------------ Advanced Example -------------------
def myplot(data):
# First two elements of data are to be ploted
# Third element of data is going to update a label
c = hv.Curve(data[:2], 'a', 'b', label=f'hello {data[-1]}').options(color=ezr.cc.c, logy=True)
c *= hv.Scatter((data[0], 2 * data[1])).options(color=ezr.cc.d)
return c
# Send data for animation
for ind, delta in enumerate(np.linspace(0, 3, 300)):
t = t0 - delta
y = 2 + np.sin(t)
# Can control when animations are drawn with this
if ind % 1 == 0:
animator.send((t, y))
"""
def __init__(self, plot_func, dynamic_range=True, plot_every=1):
import holoviews as hv
from holoviews.streams import Pipe
# Store the user-defined plot function
self.plot_func = plot_func
self.dynamic_range = dynamic_range
self.pipe = Pipe(data=[])
self.data_has_been_sent = False
self.dmap = hv.DynamicMap(self.plot_wrapper, streams=[self.pipe])
self.plot_every = plot_every
self.plot_count = 0
def plot_wrapper(self, *args, **kwargs):
data = kwargs.get('data', ([0], [0]))
hv_obj = self.plot_func(data)
if self.dynamic_range:
hv_obj = hv_obj.opts(norm=dict(framewise=True))
return hv_obj
def send(self, data):
from IPython.display import display
if self.plot_count % self.plot_every == 0:
self.pipe.send(data)
if not self.data_has_been_sent:
display(self.dmap)
self.data_has_been_sent = True
self.plot_count += 1
class DistributedSwarm:
def __init__(
self,
swarm: Callable,
n_swarms: int,
n_param_servers: int,
max_iters_ray: int = 10,
log_every: int = 100,
n_comp_add: int = 5,
minimize: bool = False,
ps_maxlen: int = 100,
init_reward: float = None,
log_reward: bool = False,
):
self.n_swarms = n_swarms
self.minimize = minimize
self.log = log_reward
self.init_reward = (init_reward if init_reward is not None else
(np.inf if minimize else -np.inf))
self.log_every = log_every
self.param_servers = [
ParamServer.remote(minimize=minimize, maxlen=ps_maxlen)
for _ in range(n_param_servers)
]
self.swarms = [
RemoteSwarm.remote(copy.copy(swarm),
int(n_comp_add),
minimize=minimize) for _ in range(self.n_swarms)
]
self.max_iters_ray = max_iters_ray
self.frame_pipe: Pipe = None
self.stream = None
self.buffer_df = None
self.score_dmap = None
self.frame_dmap = None
self.init_plot()
self.n_iters = 0
self.best = (None, None, None)
def init_plot(self):
self.frame_pipe = Pipe(data=[])
self.frame_dmap = hv.DynamicMap(hv.RGB, streams=[self.frame_pipe])
self.frame_dmap = self.frame_dmap.opts(xlim=(-0.5, 0.5),
ylim=(-0.5, 0.5),
xaxis=None,
yaxis=None,
title="Game screen")
example = pd.DataFrame({"reward": []})
self.stream = Stream()
self.buffer_df = DataFrame(stream=self.stream, example=example)
self.score_dmap = self.buffer_df.hvplot(y=["reward"]).opts(
height=200, width=500, title="Game score")
def plot(self):
return self.frame_dmap + self.score_dmap
def stream_progress(self, state, observation, reward):
example = pd.DataFrame({"reward": [reward]},
index=[self.n_iters // self.n_swarms])
self.stream.emit(example)
obs = observation.reshape((210, 160, 3)).astype(np.uint8)
self.frame_pipe.send(obs)
def run_swarm(self):
self.n_iters = 0
best_ids = [s.reset.remote() for s in self.swarms]
steps = {}
param_servers = deque([])
for worker, best in zip(self.swarms, best_ids):
steps[worker.make_iteration.remote(best)] = worker
bests = []
for ps, walker in zip(self.param_servers,
list(steps.keys())[:len(self.param_servers)]):
bests.append(ps.exchange_walker.remote(walker))
param_servers.append(ps)
ray.get(bests)
for i in range(self.max_iters_ray * len(self.swarms)):
self.n_iters += 1
ready_bests, _ = ray.wait(list(steps))
ready_best_id = ready_bests[0]
worker = steps.pop(ready_best_id)
ps = param_servers.popleft()
new_best = ps.exchange_walker.remote(ready_best_id)
param_servers.append(ps)
steps[worker.make_iteration.remote(new_best)] = worker
if i % (self.log_every * len(self.swarms)) == 0:
id_, _ = ray.wait([param_servers[-1].get_best.remote()])
(state, best_obs, best_reward) = ray.get(id_)[0]
if state is not None:
self.best = (state, best_obs, float(best_reward))
if ((best_reward > self.init_reward) if self.minimize else
(best_reward < self.init_reward)):
best_reward = self.init_reward
best_reward = np.log(
best_reward) if self.log else best_reward
self.stream_progress(state, best_obs, best_reward)
else:
print("skipping, not ready")