def test_pipe_stream(self): stream = Pipe(data=self.df) plot = self.df.hvplot('x', 'y', stream=stream) pd.testing.assert_frame_equal(plot[()].data, self.df) new_df = pd.DataFrame([[7, 8], [9, 10]], columns=['x', 'y']) stream.send(new_df) pd.testing.assert_frame_equal(plot[()].data, new_df)
def init_plot(self): hv.extension("plotly") self.best_pipe = Pipe(data=[]) self.best_dmap = hv.DynamicMap(hv.Scatter3D, streams=[self.best_pipe]) self.best_dmap = self.best_dmap.opts( xlim=(-2, 2), ylim=(-2, 4), color="red", alpha=0.7, # height=600, width=600, xaxis=None, yaxis=None, title="Best solution", ) self.label_pipe = Pipe(data=[]) self.label_dmap = hv.DynamicMap(hv.Labels, streams=[self.label_pipe]) self.label_dmap = self.label_dmap.opts( # height=200, width=400, xaxis=None, yaxis=None, title="Best solution", ) example = pd.DataFrame({"reward": []}) self.stream = Stream() self.buffer_df = DataFrame(stream=self.stream, example=example) self.score_dmap = self.buffer_df.hvplot( y=["reward"]).opts( # height=200, width=400, title="Best value found")
def _get_params(self): df = self.get_data() if self.streaming: from holoviews.streams import Pipe self._stream = Pipe(data=df) return dict(object=self.get_plot(df), sizing_mode='stretch_both') # todo update sizing mode
def __init__(self, plot_func, dynamic_range=True, plot_every=1): import holoviews as hv from holoviews.streams import Pipe # Store the user-defined plot function self.plot_func = plot_func self.dynamic_range = dynamic_range self.pipe = Pipe(data=[]) self.data_has_been_sent = False self.dmap = hv.DynamicMap(self.plot_wrapper, streams=[self.pipe]) self.plot_every = plot_every self.plot_count = 0
def init_plot(self): self.frame_pipe = Pipe(data=[]) self.frame_dmap = hv.DynamicMap(hv.RGB, streams=[self.frame_pipe]) self.frame_dmap = self.frame_dmap.opts(xlim=(-0.5, 0.5), ylim=(-0.5, 0.5), xaxis=None, yaxis=None, title="Game screen") example = pd.DataFrame({"reward": []}) self.stream = Stream() self.buffer_df = DataFrame(stream=self.stream, example=example) self.score_dmap = self.buffer_df.hvplot(y=["reward"]).opts( height=200, width=500, title="Game score")
def _get_params(self): df = self.get_data() if df is None: df = self.empty_df self._stream = Pipe(data=df) return dict(object=self.get_plot(), sizing_mode="stretch_both") # todo update sizing mode
def new_game_of_life_graph(game_size, plot_size): pipe = Pipe(data=[]) dmap = hv.DynamicMap(hv.Points, streams=[pipe]) cell_size = round(plot_size / (2 * game_size)) dmap.opts(color='Grey', xlim=(0, game_size), ylim=(0, game_size), size=cell_size, width=plot_size, height=plot_size) return dmap
def __init__(self, mapdata): '''initialize a SubwayMap object Args: mapdata (SubwayMapData): container-class for stations and lines dataframes with implemented observer pattern. This is necessary for data binding of the view to the viewmodel. ''' Stream.__init__(self) #create an initial map stations, lines = mapdata.stationsdf, mapdata.linesdf self.pipe = Pipe(data=[]) self.subway_map = gv.Path(lines, vdims=['color']).opts( projection=crs.LambertConformal(), height=800, width=800, color='color') * gv.DynamicMap(self.callback, streams=[self.pipe]) self.pipe.send(stations) #bind changes in the stationsdf to pipe.send mapdata.bind_to_stationsdf(self.pipe.send) self.mapdata = mapdata
def init_plot(self): self.frame_pipe = Pipe(data=[]) self.frame_dmap = hv.DynamicMap(hv.Labels, streams=[self.frame_pipe]) self.frame_dmap = self.frame_dmap.opts( xlim=(-10, 10), ylim=(0.5, 2.5), height=200, width=500, xaxis=None, yaxis=None, title="Best solution", ) example = pd.DataFrame({"reward": []}) self.stream = Stream() self.buffer_df = DataFrame(stream=self.stream, example=example) self.score_dmap = self.buffer_df.hvplot(y=["reward"]).opts( height=200, width=400, title="Best value found")
def new_game_of_life_graph(game_size, plot_size, bgcolor=default_bgcolor, cmap=default_cmap, use_fixed_cell_sizes=default_use_fixed_cell_sizes, max_cell_age=default_max_cell_age): """ The input points will be [x,y,z]. For plotting, we clip z to be no larger than max_cell_age. That affects how many colors are chosen from the specified color palette. A smaller value of max_cell_size produces faster changes, since most lifetimes are low, except for large game sizes, where some patterns can be semi-permanent. If use_fixed_cell_sizes=False, then the points grow in size from zero (dead) to the clipped size. The log(z)+1 is calculated as an alternative for mapping to colors and sizes, but it doesn't work that well for small lifetimes. See comment below inline. """ def new_points(data): data2 = data if len(data) > 0: x = np.array(data[0]) y = np.array(data[1]) z = np.array(data[2]) data2 = np.c_[x, y, z, np.log1p(z), np.clip(z, 0, max_cell_age)] return hv.Points(data2, kdims=['x', 'y'], vdims=['z', 'z_log1p', 'z_clipped']) pipe = Pipe(data=[]) dmap = hv.DynamicMap(new_points, streams=[pipe]) cell_sizes = 'z_clipped' # or use z_log1p or even try z. if use_fixed_cell_sizes: cell_sizes = fixed_cell_size(plot_size, game_size) dmap.opts( color='z_clipped', # or use z_log1p or z. xlim=(0, game_size), ylim=(0, game_size), size=cell_sizes, width=plot_size, height=plot_size, bgcolor=bgcolor, cmap=cmap) return dmap
_SQSbuffer__trainId__pnccdDetect__last_hit = online.DataBuffer(5) #pnccd_dist_px, pnccd_rightLeft_offset_px for k in range(5): _SQSbuffer__pnCCD_hits( bin2_image( np.zeros(shape=(pnccd_dims[0] + pnccd_dist_px, pnccd_dims[1] + 2 * abs(pnccd_rightLeft_offset_px))))) _SQSbuffer__pnCCD_hits_tids((0)) _SQSbuffer__counter = online.DataBuffer(buffer_length) print("...2") # Data pipes and buffers for plots ## pipes provide a full update of data to the underlying object eg. plot ## buffers add only a single value to the plot and may kick one out when number of elements in the buffer has reached the length/size of the buffer _pipe__TOF_single = Pipe(data=[]) #_buffer__TOF_integral = Buffer(pd.DataFrame({'x':[],'y':[]}, columns=['x','y']), length=100, index=False) _pipe__TOF_integral = Pipe(data=[]) _pipe__TOF_height = Pipe(data=[]) _pipe__pnCCD_single = Pipe(data=[]) _pipe__pnCCD_integral = Pipe(data=[]) _pipe__GMD_history = Pipe(data=[]) _pipe__pnCCD_hitrate = Pipe(data=[]) _pipe__pnCCD_hitrate_2 = Pipe(data=[]) _pipe__pnCCD_hits_tids = Pipe(data=[]) _pipe__pnCCD_hits_list = list() for i in range(5): _pipe__pnCCD_hits_list.append(Pipe(data=[])) _pipe__pnCCD_hits__last_hit = Pipe(data=[]) _pipe__TOF_hits__last_hit = Pipe(data=[]) _pipe__trainId__last_hit = Pipe(data=[])
_SQSbuffer__TOF_integral = online.DataBuffer(buffer_len) _SQSbuffer__TOF_integral_1 = online.DataBuffer(buffer_len) _SQSbuffer__TOF_integral_2 = online.DataBuffer(buffer_len) _SQSbuffer__TOF_integral_3 = online.DataBuffer(buffer_len) _SQSbuffer__TOF_avg = online.DataBuffer(100) _SQSbuffer__TOF_hit_trace = online.DataBuffer(1) _SQSbuffer__TOF_hits = online.DataBuffer(1000) _SQSbuffer__GMD_history = online.DataBuffer(buffer_len) _SQSbuffer__TOF_hits = online.DataBuffer(10) _SQSbuffer__counter = online.DataBuffer(buffer_len) print("...2") # Data pipes and buffers for plots ## pipes provide a full update of data to the underlying object eg. plot ## buffers add only a single value to the plot and may kick one out when number of elements in the buffer has reached the length/size of the buffer _pipe__TOF_single = Pipe(data=[]) _pipe__TOF_hit_trace = Pipe(data=[]) _pipe__TOF_avg = Pipe(data=[]) _pipe__TOF_integral = Pipe(data=[]) _pipe__TOF_integral_1 = Pipe(data=[]) _pipe__TOF_integral_2 = Pipe(data=[]) _pipe__TOF_integral_3 = Pipe(data=[]) _pipe__GMD_history = Pipe(data=[]) # SETUP PLOTS print("...3") # example for coupled plots # layout = hv.Layout(largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE") + largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE 2", cmap=['red'])).cols(1) ## TOF bokeh_live_tof = largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE",
buffer_length) _SQSbuffer__pnCCD_mean_1_integral_proc_right = online.DataBuffer(buffer_length) _SQSbuffer__pnCCD_mean_1_integral_proc_left = online.DataBuffer(buffer_length) _SQSbuffer__pnCCD_mean_2_integral_proc = online.DataBuffer(buffer_length) _SQSbuffer__pnCCD_mean_helper_1 = online.DataBuffer(mean_1_length) _SQSbuffer__pnCCD_mean_helper_2 = online.DataBuffer(mean_2_length) _SQSbuffer__xgm_mean_helper_1 = online.DataBuffer(mean_1_length) _SQSbuffer__xgm_mean_helper_2 = online.DataBuffer(mean_2_length) _SQSbuffer__counter = online.DataBuffer(buffer_length) _SQSbuffer__counter_time = online.DataBuffer(buffer_length) print("...2") # Data pipes and buffers for plots ## pipes provide a full update of data to the underlying object eg. plot ## buffers add only a single value to the plot and may kick one out when number of elements in the buffer has reached the length/size of the buffer _pipe__pnCCD_single = Pipe(data=[]) _pipe__pnCCD_mean_1 = Pipe(data=[]) _pipe__pnCCD_mean_2 = Pipe(data=[]) _pipe__pnCCD_integral = Pipe(data=[]) _pipe__pnCCD_mean_1_integral = Pipe(data=[]) _pipe__pnCCD_mean_2_integral = Pipe(data=[]) _pipe__pnCCD_integral_by_gmd = Pipe(data=[]) _pipe__pnCCD_mean_1_integral_by_gmd = Pipe(data=[]) _pipe__pnCCD_mean_1_integral_by_gmd_top = Pipe(data=[]) _pipe__pnCCD_mean_1_integral_by_gmd_bottom = Pipe(data=[]) _pipe__pnCCD_mean_1_integral_by_gmd_right = Pipe(data=[]) _pipe__pnCCD_mean_1_integral_by_gmd_left = Pipe(data=[]) _pipe__pnCCD_mean_2_integral_by_gmd = Pipe(data=[]) _pipe__GMD_history = Pipe(data=[]) _pipe__pnCCD_hits__last_hit = Pipe(data=[]) _pipe__trainId__last_hit = Pipe(data=[])
return sqs_bk.hv_to_bokeh_obj( TOF_dmap.opts(width=width,height=height,ylim=ylim,xlim=xlim, xlabel=xlabel, ylabel=ylabel, title = title), renderer) def start_stop_dataThread(): global makeBigData_stop if not makeBigData_stop: makeBigData_stop = True else: thread.start() # Data buffers for live stream print("...2") # Data pipes and buffers for plots ## pipes provide a full update of data to the underlying object eg. plot ## buffers add only a single value to the plot and may kick one out when number of elements in the buffer has reached the length/size of the buffer _pipe__TOF_single = Pipe(data=[]) # SETUP PLOTS print("...3") # example for coupled plots # layout = hv.Layout(largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE") + largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE 2", cmap=['red'])).cols(1) ## TOF bokeh_live_tof = largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE", width = 1900, height=900) ## SET UP Additional Widgets bokeh_button_StartStop = Button(label = "Start / Stop", button_type="success") bokeh_button_StartStop.on_click(start_stop_dataThread) # SET UP BOKEH LAYOUT # bokeh_row_1 = row(bokeh_live_tof) bokeh_row_interact = bokeh_button_StartStop bokeh_layout = column(bokeh_row_1, bokeh_row_interact)
class SubwayMap(Stream): '''subway map (holoviews dynamic map) including pyviz.param-based control interface. inherited from a holoviews stream object.''' #class variables direction_list = ['North', 'South'] lines_list = [ 'All', '1', '2', '3', '4', '5', '6', '7', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'L', 'M', 'N', 'Q', 'R', 'SI', 'W' ] display_list = ['Time since last train', 'Probability of train delay'] #Selector class variables (parameters) for holoviews panel direction = param.ObjectSelector(default='North', objects=direction_list) line = param.ObjectSelector(default='All', objects=lines_list) display = param.ObjectSelector(default='Probability of train delay', objects=display_list) def callback(self, data): if (self.display == 'Probability of train delay'): layout = gv.Points(data, vdims=[ 'color', 'displaysize', 'name', 'waittime_str', 'delay_prob', 'MTAdelay', 'inboundtrain', 'inbound_from' ]).opts(tools=[SubwayMap.hover_delays], size='displaysize', color='color') #elif(self.display == 'Time since last train'): else: layout = gv.Points(data, vdims=[ 'waittimecolor', 'waittimedisplaysize', 'name', 'waittime_str', 'delay_prob', 'MTAdelay' ]).opts(tools=[SubwayMap.hover_waittime], size='waittimedisplaysize', color='waittimecolor') return layout def __init__(self, mapdata): '''initialize a SubwayMap object Args: mapdata (SubwayMapData): container-class for stations and lines dataframes with implemented observer pattern. This is necessary for data binding of the view to the viewmodel. ''' Stream.__init__(self) #create an initial map stations, lines = mapdata.stationsdf, mapdata.linesdf self.pipe = Pipe(data=[]) self.subway_map = gv.Path(lines, vdims=['color']).opts( projection=crs.LambertConformal(), height=800, width=800, color='color') * gv.DynamicMap(self.callback, streams=[self.pipe]) self.pipe.send(stations) #bind changes in the stationsdf to pipe.send mapdata.bind_to_stationsdf(self.pipe.send) self.mapdata = mapdata hover_delays = HoverTool( tooltips=[("station", "@name"), ( "incoming train", "@inboundtrain"), ("from station", "@inbound_from"), ("probability that incoming train is delayed", "@delay_prob"), ("MTA reports delay?", "@MTAdelay")]) hover_waittime = HoverTool( tooltips=[("station", "@name"), ("time since last train", "@waittime_str")]) def view(self): return self.subway_map #https://panel.pyviz.org/user_guide/Param.html @param.depends('direction', 'line', watch=True) def update(self): self.mapdata.selected_dir = self.direction self.mapdata.selected_line = self.line
def _process_data(self, kind, data, x, y, by, groupby, row, col, use_dask, persist, backlog, label, value_label, hover_cols, kwds): gridded = kind in self._gridded_types gridded_data = False # Validate DataSource self.data_source = data self.is_series = is_series(data) if self.is_series: data = data.to_frame() if is_intake(data): data = process_intake(data, use_dask or persist) if groupby is not None and not isinstance(groupby, list): groupby = [groupby] if by is not None and not isinstance(by, list): by = [by] streaming = False if isinstance(data, pd.DataFrame): self.data = data if is_geopandas(data) and kind is None: geom_types = set( [gt[5:] if 'Multi' in gt else gt for gt in data.geom_type]) if len(geom_types) > 1: raise ValueError( 'The GeopandasInterface can only read dataframes which ' 'share a common geometry type') geom_type = list(geom_types)[0] if geom_type == 'Point': kind = 'points' elif geom_type == 'Polygon': kind = 'polygons' elif geom_type in ('LineString', 'LineRing'): kind = 'paths' elif is_dask(data): self.data = data.persist() if persist else data elif is_streamz(data): self.data = data.example self.stream_type = data._stream_type streaming = True self.cb = data if data._stream_type == 'updating': self.stream = Pipe(data=self.data) else: self.stream = Buffer(data=self.data, length=backlog, index=False) data.stream.gather().sink(self.stream.send) elif is_xarray(data): import xarray as xr z = kwds.get('z') if z is None and isinstance(data, xr.Dataset): z = list(data.data_vars)[0] if gridded and isinstance(data, xr.Dataset) and not isinstance(z, list): data = data[z] ignore = (groupby or []) + (by or []) dims = [ c for c in data.coords if data[c].shape != () and c not in ignore ] if kind is None and (not (x or y) or all(c in data.coords for c in (x, y))): if len(dims) == 1: kind = 'line' elif len(dims) == 2 or (x and y): kind = 'image' gridded = True else: kind = 'hist' if gridded: gridded_data = True data, x, y, by_new, groupby_new = process_xarray( data, x, y, by, groupby, use_dask, persist, gridded, label, value_label) if kind not in self._stats_types: if by is None: by = by_new if groupby is None: groupby = groupby_new if groupby: groupby = [g for g in groupby if g not in (row, col)] self.data = data else: raise ValueError('Supplied data type %s not understood' % type(data).__name__) # Validate data and arguments if by is None: by = [] if groupby is None: groupby = [] if gridded: if not gridded_data: raise ValueError('%s plot type requires gridded data, ' 'e.g. a NumPy array or xarray Dataset, ' 'found %s type' % (kind, type(self.data).__name__)) not_found = [g for g in groupby if g not in data.coords] data_vars = list(data.data_vars) if isinstance( data, xr.Dataset) else [data.name] indexes = list(data.coords) self.variables = list(data.coords) + data_vars if groupby and not_found: raise ValueError('The supplied groupby dimension(s) %s ' 'could not be found, expected one or ' 'more of: %s' % (not_found, list(data.coords))) else: # Determine valid indexes if isinstance(self.data, pd.DataFrame): if self.data.index.names == [None]: indexes = [self.data.index.name or 'index'] else: indexes = list(self.data.index.names) else: indexes = [ c for c in self.data.reset_index().columns if c not in self.data.columns ] if len(indexes) == 2 and not (x or y or by): if kind == 'heatmap': x, y = indexes elif kind in ('bar', 'barh'): x, by = indexes # Rename non-string columns renamed = { c: str(c) for c in data.columns if not isinstance(c, hv.util.basestring) } if renamed: self.data = self.data.rename(columns=renamed) self.variables = indexes + list(self.data.columns) # Reset groupby dimensions groupby_index = [g for g in groupby if g in indexes] if groupby_index: self.data = self.data.reset_index(groupby_index) not_found = [ g for g in groupby if g not in list(self.data.columns) + indexes ] if groupby and not_found: raise ValueError('The supplied groupby dimension(s) %s ' 'could not be found, expected one or ' 'more of: %s' % (not_found, list(self.data.columns))) # Set data-level options self.x = x self.y = y self.kind = kind or 'line' self.gridded = gridded self.use_dask = use_dask self.indexes = indexes if isinstance(by, (np.ndarray, pd.Series)): self.data['by'] = by self.by = ['by'] elif not by: self.by = [] else: self.by = by if isinstance(by, list) else [by] self.groupby = groupby self.streaming = streaming self.hover_cols = hover_cols
class hvRectangleAppView(View): opts = param.Dict(default={}, doc="HoloViews option to apply on the plot.") view_type = 'rectangles' streaming = param.Boolean(default=False, doc=""" Whether to stream new data to the plot or rerender the plot.""") def __init__(self, **params): # import hvplot.pandas # noqa # if 'dask' in sys.modules: # try: # import hvplot.dask # noqa # except Exception: # pass self._stream = None self._linked_objs = [] super().__init__(**params) def get_panel(self): kwargs = self._get_params() #interactive? https://github.com/holoviz/panel/issues/1824 return pn.pane.HoloViews(**kwargs) def get_plot(self, df): """ Dataframe df must have columns x0, y0, x1, y1 (in this order) for coordinates bottom-left (x0, y0) and top right (x1, y1). Optionally a fifth value-column can be provided for colors Parameters ---------- df Returns ------- """ # processed = {} # for k, v in self.kwargs.items(): # if k.endswith('formatter') and isinstance(v, str) and '%' not in v: # v = NumeralTickFormatter(format=v) # processed[k] = v # if self.streaming: # processed['stream'] = self._stream #hvplots stream? https://holoviews.org/user_guide/Streaming_Data.html # plot = hv.Rectangles([(0, 0, 1, 1), (2, 3, 4, 6), (0.5, 2, 1.5, 4), (2, 1, 3.5, 2.5)]) processed = {} for k, v in self.kwargs.items(): if k.endswith('formatter') and isinstance(v, str) and '%' not in v: v = NumeralTickFormatter(format=v) processed[k] = v if self.streaming: #processed['stream'] = self._stream plot = hv.DynamicMap(hv.Rectangles, streams=[self._stream]) plot = plot.apply.opts(**self.opts) if self.opts else plot else: plot = hv.Rectangles(df) plot.opts(**self.opts) if self.opts else plot if self.selection_group or 'selection_expr' in self._param_watchers: plot = self._link_plot(plot) return plot def _get_params(self): df = self.get_data() if self.streaming: from holoviews.streams import Pipe self._stream = Pipe(data=df) return dict(object=self.get_plot(df), sizing_mode='stretch_both') # todo update sizing mode def get_data(self): #todo uniformify this method for all views try: return super().get_data() except (KeyError, ValueError) as e: print(f'Empty data in {self.__class__}: {e}') return self.empty_df def update(self, *events, invalidate_cache=True): """ Triggers an update in the View. Parameters ---------- events: tuple param events that may trigger an update. invalidate_cache : bool Whether to clear the View's cache. Returns ------- stale : bool Whether the panel on the View is stale and needs to be rerendered. """ # Skip events triggered by a parameter change on this View own_parameters = [self.param[p] for p in self.param] own_events = events and all( isinstance(e.obj, ParamFilter) and (e.obj.parameter in own_parameters or e.new is self._ls.selection_expr) for e in events) if own_events: return False if invalidate_cache: self._cache = None if not self.streaming or self._stream is None: upd = self._update_panel() return upd self._stream.send(self.get_data()) return False @property def empty_df(self): return pd.DataFrame([[0] * 5], columns=['x0', 'x1', 'y0', 'y1', 'value'])
class DistributedSwarm: def __init__( self, swarm: Callable, n_swarms: int, n_param_servers: int, max_iters_ray: int = 10, log_every: int = 100, n_comp_add: int = 5, minimize: bool = False, ps_maxlen: int = 100, init_reward: float = None, log_reward: bool = False, ): self.n_swarms = n_swarms self.minimize = minimize self.log = log_reward self.init_reward = (init_reward if init_reward is not None else (np.inf if minimize else -np.inf)) self.log_every = log_every self.param_servers = [ ParamServer.remote(minimize=minimize, maxlen=ps_maxlen) for _ in range(n_param_servers) ] self.swarms = [ RemoteSwarm.remote(copy.copy(swarm), int(n_comp_add), minimize=minimize) for _ in range(self.n_swarms) ] self.max_iters_ray = max_iters_ray self.frame_pipe: Pipe = None self.stream = None self.buffer_df = None self.score_dmap = None self.frame_dmap = None self.init_plot() self.n_iters = 0 self.best = (None, None, None) def init_plot(self): self.frame_pipe = Pipe(data=[]) self.frame_dmap = hv.DynamicMap(hv.RGB, streams=[self.frame_pipe]) self.frame_dmap = self.frame_dmap.opts(xlim=(-0.5, 0.5), ylim=(-0.5, 0.5), xaxis=None, yaxis=None, title="Game screen") example = pd.DataFrame({"reward": []}) self.stream = Stream() self.buffer_df = DataFrame(stream=self.stream, example=example) self.score_dmap = self.buffer_df.hvplot(y=["reward"]).opts( height=200, width=500, title="Game score") def plot(self): return self.frame_dmap + self.score_dmap def stream_progress(self, state, observation, reward): example = pd.DataFrame({"reward": [reward]}, index=[self.n_iters // self.n_swarms]) self.stream.emit(example) obs = observation.reshape((210, 160, 3)).astype(np.uint8) self.frame_pipe.send(obs) def run_swarm(self): self.n_iters = 0 best_ids = [s.reset.remote() for s in self.swarms] steps = {} param_servers = deque([]) for worker, best in zip(self.swarms, best_ids): steps[worker.make_iteration.remote(best)] = worker bests = [] for ps, walker in zip(self.param_servers, list(steps.keys())[:len(self.param_servers)]): bests.append(ps.exchange_walker.remote(walker)) param_servers.append(ps) ray.get(bests) for i in range(self.max_iters_ray * len(self.swarms)): self.n_iters += 1 ready_bests, _ = ray.wait(list(steps)) ready_best_id = ready_bests[0] worker = steps.pop(ready_best_id) ps = param_servers.popleft() new_best = ps.exchange_walker.remote(ready_best_id) param_servers.append(ps) steps[worker.make_iteration.remote(new_best)] = worker if i % (self.log_every * len(self.swarms)) == 0: id_, _ = ray.wait([param_servers[-1].get_best.remote()]) (state, best_obs, best_reward) = ray.get(id_)[0] if state is not None: self.best = (state, best_obs, float(best_reward)) if ((best_reward > self.init_reward) if self.minimize else (best_reward < self.init_reward)): best_reward = self.init_reward best_reward = np.log( best_reward) if self.log else best_reward self.stream_progress(state, best_obs, best_reward) else: print("skipping, not ready")
def _get_params(self): df = self.get_data() if self.streaming: from holoviews.streams import Pipe self._stream = Pipe(data=df) return dict(object=self.get_plot(df))
class hvPlotView(View): """ The hvPlotView renders the queried data as a bokeh plot generated with hvPlot. hvPlot allows for a concise declaration of a plot via its simple API. """ kind = param.String(doc="The kind of plot, e.g. 'scatter' or 'line'.") x = param.Selector(doc="The column to render on the x-axis.") y = param.Selector(doc="The column to render on the y-axis.") by = param.ListSelector(doc="The column(s) to facet the plot by.") groupby = param.ListSelector(doc="The column(s) to group by.") opts = param.Dict(default={}, doc="HoloViews options to apply on the plot.") streaming = param.Boolean(default=False, doc=""" Whether to stream new data to the plot or rerender the plot.""") selection_expr = param.Parameter(doc=""" A selection expression caputirng the current selection applied on the plot.""") view_type = 'hvplot' _field_params = ['x', 'y', 'by', 'groupby'] _supports_selections = True def __init__(self, **params): import hvplot.pandas # noqa if 'dask' in sys.modules: try: import hvplot.dask # noqa except Exception: pass if 'by' in params and isinstance(params['by'], str): params['by'] = [params['by']] if 'groupby' in params and isinstance(params['groupby'], str): params['groupby'] = [params['groupby']] self._stream = None self._linked_objs = [] super().__init__(**params) def get_plot(self, df): processed = {} for k, v in self.kwargs.items(): if k.endswith('formatter') and isinstance(v, str) and '%' not in v: v = NumeralTickFormatter(format=v) processed[k] = v if self.streaming: processed['stream'] = self._stream plot = df.hvplot(kind=self.kind, x=self.x, y=self.y, **processed) plot = plot.opts(**self.opts) if self.opts else plot if self.selection_group or 'selection_expr' in self._param_watchers: plot = self._link_plot(plot) return plot def _link_plot(self, plot): self._init_link_selections() linked_objs = list(self._ls._plot_reset_streams) plot = self._ls(plot) self._linked_objs += [ o for o in self._ls._plot_reset_streams if o not in linked_objs ] return plot def _cleanup(self): if self._ls is None: return for obj in self._linked_objs: reset = self._ls._plot_reset_streams.pop(obj) sel_expr = self._ls._selection_expr_streams.pop(obj) self._ls._cross_filter_stream.input_streams.remove(sel_expr) sel_expr.clear() sel_expr.source = None reset.clear() reset.source = None self._linked_objs = [] def get_panel(self): return pn.pane.HoloViews(**self._get_params()) def _get_params(self): df = self.get_data() if self.streaming: from holoviews.streams import Pipe self._stream = Pipe(data=df) return dict(object=self.get_plot(df)) def update(self, *events, invalidate_cache=True): """ Triggers an update in the View. Parameters ---------- events: tuple param events that may trigger an update. invalidate_cache : bool Whether to clear the View's cache. Returns ------- stale : bool Whether the panel on the View is stale and needs to be rerendered. """ # Skip events triggered by a parameter change on this View own_parameters = [self.param[p] for p in self.param] own_events = events and all( isinstance(e.obj, ParamFilter) and (e.obj.parameter in own_parameters or e.new is self._ls.selection_expr) for e in events) if own_events: return False if invalidate_cache: self._cache = None if not self.streaming or self._stream is None: return self._update_panel() self._stream.send(self.get_data()) return False
# Data buffers for live stream _SQSbuffer__TOF_integral = online.DataBuffer(100) _SQSbuffer__GMD_history = online.DataBuffer(100) _SQSbuffer__pnCCD_integral = online.DataBuffer(100) _SQSbuffer__pnCCD_hits = online.DataBuffer(10) for k in range(5): _SQSbuffer__pnCCD_hits(np.zeros(shape=(1024,1024))) _SQSbuffer__counter = online.DataBuffer(100) print("...2") # Data pipes and buffers for plots ## pipes provide a full update of data to the underlying object eg. plot ## buffers add only a single value to the plot and may kick one out when number of elements in the buffer has reached the length/size of the buffer _pipe__TOF_single = Pipe(data=[]) #_buffer__TOF_integral = Buffer(pd.DataFrame({'x':[],'y':[]}, columns=['x','y']), length=100, index=False) _pipe__TOF_integral = Pipe(data=[]) _pipe__pnCCD_single = Pipe(data=[]) _pipe__pnCCD_integral = Pipe(data=[]) _pipe__GMD_history = Pipe(data=[]) _pipe__pnCCD_hits_list = list() for i in range(5): _pipe__pnCCD_hits_list.append(Pipe(data=[])) # SETUP PLOTS print("...3") # example for coupled plots # layout = hv.Layout(largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE") + largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE 2", cmap=['red'])).cols(1) ## TOF bokeh_live_tof = largeData_line_plot(_pipe__TOF_single, title="TOF single shots - LIVE", width = 500, height=500)
class Animator: """ Creates animated plots with holoviews. Constructor Args: dynamic_range: bool = if set to True, auto-scale the chart each time it is drawn # ------------ Simple Example ------------------- # Define a function to make your plots. # Data will be whatever object is passed to animator.send() def myplot(data): return hv.Curve(data) # Pass your plotting function to the animator constructor animator = Animator(myplot) # Simply send the animator updated versions of your data # to update the plot t0 = np.linspace(0, 6., 100) for delta in np.linspace(0, 3, 30): t = t0 - delta y = 2 + np.sin(t) animator.send((t, y)) # ------------ Advanced Example ------------------- def myplot(data): # First two elements of data are to be ploted # Third element of data is going to update a label c = hv.Curve(data[:2], 'a', 'b', label=f'hello {data[-1]}').options(color=ezr.cc.c, logy=True) c *= hv.Scatter((data[0], 2 * data[1])).options(color=ezr.cc.d) return c # Send data for animation for ind, delta in enumerate(np.linspace(0, 3, 300)): t = t0 - delta y = 2 + np.sin(t) # Can control when animations are drawn with this if ind % 1 == 0: animator.send((t, y)) """ def __init__(self, plot_func, dynamic_range=True, plot_every=1): import holoviews as hv from holoviews.streams import Pipe # Store the user-defined plot function self.plot_func = plot_func self.dynamic_range = dynamic_range self.pipe = Pipe(data=[]) self.data_has_been_sent = False self.dmap = hv.DynamicMap(self.plot_wrapper, streams=[self.pipe]) self.plot_every = plot_every self.plot_count = 0 def plot_wrapper(self, *args, **kwargs): data = kwargs.get('data', ([0], [0])) hv_obj = self.plot_func(data) if self.dynamic_range: hv_obj = hv_obj.opts(norm=dict(framewise=True)) return hv_obj def send(self, data): from IPython.display import display if self.plot_count % self.plot_every == 0: self.pipe.send(data) if not self.data_has_been_sent: display(self.dmap) self.data_has_been_sent = True self.plot_count += 1
def fit(self, *, x=None, y=None, weights=None, model=None, cost=None, plot_every=None, algorithm='fmin', verbose=True): """ The method to use for training the fitter. Args: x: the indepenant data y: the depenant data model: a model function to fit the data to cost: a custom cost function (defaults to least squares) plot_every: Plot solution in real time every this number of iterations algorithm: Scipy optimization routine to use. Enter nonsense to see list of valid. verbose: Print convergence information """ import numpy as np from scipy import optimize import holoviews as hv from holoviews.streams import Pipe from IPython.display import display self.plot_every = plot_every if algorithm not in self.OPTIMIZER_NAMES: raise ValueError( f'Invalid optimizer {algorithm}. Choose one of {self.OPTIMIZER_NAMES}' ) givens = dict(x=x, y=y) if weights is not None: givens.update({'weights': weights}) givens.update(self._givens) self._params.given(**givens) # This stuff only needs to happen if we are iteratively plotting fits if plot_every is not None: x, y = self._params.x, self._params.y xmin, xmax = np.min(x), np.max(x) ymin, ymax = np.min(y), np.max(y) scale = .1 delta_x = scale * (xmax - xmin) delta_y = scale * (ymax - ymin) xmin, xmax = xmin - delta_x, xmax + delta_x ymin, ymax = ymin - delta_y, ymax + delta_y xlim = (xmin, xmax) ylim = (ymin, ymax) self.pipe = Pipe(data=[]) try: dmap = hv.DynamicMap(self._plotter, streams=[self.pipe]) dmap.opts(hv.opts.Overlay(xlim=xlim, ylim=ylim)) display(dmap) except AttributeError: raise RuntimeError( 'You must import holoviews and set bokeh backround for plotting to work' ) if model is None and cost is None: raise ValueError( 'You must supply either a model function or a cost function') self._raw_model = model self._model = self._model_wrapper(self._raw_model) a0 = self._params.array if cost is not None: self._cost = cost optimizer = getattr(optimize, algorithm) self._optimizer_kwargs.update(disp=verbose) a_fit = optimizer(self._cost_wrapper, a0, args=(self._params, ), **self._optimizer_kwargs) a_fit = np.array(a_fit, ndmin=1) self._params.ingest(a_fit) return self