def plot_output_vs_time(self, bnum_it=None):
import holoviews as hv
from holoviews.operation import decimate
import pandas as pd, numpy as np
def byte_to_color(idx):
return hv.Palette.colormaps['Category20'](idx / 16.0)
if bnum_it is None:
bnum_it = range(16)
a = []
b = []
hv.extension('bokeh')
for i in range(16):
data = self.output_vs_time(i)
a.append(np.array(data[1]))
b.append(np.array(data[2]))
b.append(np.array(data[3]))
#pda = pd.DataFrame(a).transpose().rename(str, axis='columns')
#pdb = pd.DataFrame(b).transpose().rename(str, axis='columns')
curve = hv.Curve(b[i], "Sample").options(color='black').options(
title="Correlation Vs. Time")
for i in range(1, len(b)):
curve *= hv.Curve(b[i]).options(color='black',
width=900,
height=600)
for i in bnum_it:
curve *= hv.Curve(a[i]).options(color=byte_to_color(i),
width=900,
height=600)
return decimate(curve)
def __call__(self, dset, **params):
self.p = ParamOverrides(self, params)
if self.p.vdim is None:
vdim = dset.vdims[0].name
else:
vdim = self.p.vdim
pts = hv.util.Dynamic(dset, operation=skypoints,
streams=[self.p.filter_stream])
if self.p.aggregator == 'mean':
aggregator = ds.mean(vdim)
elif self.p.aggregator == 'std':
aggregator = ds.std(vdim)
elif self.p.aggregator == 'count':
aggregator = ds.count()
decimate_opts = dict(plot={'tools': ['hover',
'box_select']},
style={'alpha': 0,
'size': self.p.decimate_size,
'nonselection_alpha': 0})
decimated = decimate(pts).opts(**decimate_opts)
raster_ = rasterize(pts, aggregator=aggregator)
color_gadget = raster_.opts(cmap=Viridis[256], colorbar=True, alpha=0)
sky_shaded = shade(raster_, cmap=viridis)
plot = dynspread(sky_shaded) * decimated * color_gadget
return plot.options(bgcolor="black", responsive=True, min_height=100)
def box_chooser(self, **kwargs):
'''
Visual tool for choosing rectangle shaped blocks in mask.
:param kwargs: array, if you want to pre select pixels,
initialize_mask, True if you want to initialize mask.
True also Overrides array.
False does nothing and overrides
nothing
initialize_value, what value you want to use in
initialisation. Used if initialize_mask
is True. Fallback value is 0.
:return: hv.Object
'''
# cube.test_for_not_none()
wid = len(self._obj.coords[self._obj.M.dims[1]])
hei = len(self._obj.coords[self._obj.M.dims[0]])
# dep = len(cube.band)
self.__do_mask_changes(**kwargs)
all_pixels = np.array(list(itertools.product(range(wid), range(hei))))
mask_dims = self._obj.M.dims.copy()
mask_dims.reverse()
points = hv.Points(all_pixels, kdims=mask_dims)
# print(points)
ds_attrs = self._make_dataset_opts()
dataset3d = hv.Dataset(**ds_attrs)
box = hv.streams.BoundsXY(source=points, bounds=(0, 0, 0, 0))
bounds = hv.DynamicMap(lambda bounds: hv.Bounds(bounds), streams=[box])
third_dim_list = list(self._obj.dims)
third_dim_list.remove(mask_dims[0])
third_dim_list.remove(mask_dims[1])
layout = decimate(points) * \
dataset3d.to(hv.Image,
mask_dims,
'Value',
third_dim_list) * \
bounds + \
decimate(
hv.DynamicMap(
lambda bounds: hv.Points(
self._record_selections(bounds, points),
kdims=mask_dims
),
streams=[box]
)
)
return layout
def jtdp(self, x_range, y_range, **kwargs):
pointdec = hv.Points(df,
kdims=['X_CORD', 'Y_CORD'],
vdims=['EVENT_COUNT', 'FEEDER_ID'])
selecteddec = pointdec.select(EventType=self.plot,
EVENT_COUNT=self.numEvents)
dm2 = decimate(
selecteddec, x_range=x_range, y_range=y_range, dynamic=False
).opts(
style={'Points': dict(alpha=0.0, color='blue', size=self.maxpix)})
return dm2
def update_map(attr, old, new):
global selection, vizual, gvplot, hvplot, heatmap, feats, points, world_map, range_slider, controls3, max_cur_feature, min_cur_feature, temp_feats
max_cur_feature = temp_feats[choice.value].max()
min_cur_feature = temp_feats[choice.value].min()
range_slider = RangeSlider(start=min_cur_feature,
end=max_cur_feature,
value=(min_cur_feature, max_cur_feature),
step=(max_cur_feature - min_cur_feature) / 20,
title="Feature_range")
range_slider.on_change('value', update_map_val)
controls3 = widgetbox([range_slider], width=250)
new_loc_feats = temp_feats.loc[
(temp_feats[choice.value] < range_slider.value[1])
& (temp_feats[choice.value] > range_slider.value[0])]
feats = gv.Dataset(new_loc_feats, kdims=['Longitude', 'Latitude', new])
points = feats.to(gv.Points, ['Longitude', 'Latitude'], [new])
if len(new_loc_feats) <= 20000:
world_map = gv.Overlay(tiles * points).options(
'Points',
size=5,
cmap='viridis',
colorbar=True,
tools=TOOLS,
color_index=2,
width=900,
height=800,
global_extent=True,
colorbar_opts={'scale_alpha': 0.5},
fill_alpha=0.5,
line_alpha=0.5)
else:
world_map = decimate(gv.Points(points), max_samples=20000).options(
'Points',
size=5,
cmap='viridis',
colorbar=True,
tools=TOOLS,
color_index=2,
width=900,
height=800,
global_extent=True,
colorbar_opts={'scale_alpha': 0.5},
fill_alpha=0.5,
line_alpha=0.5)
selection = hv.streams.Selection1D(source=world_map)
heatmap = hv.DynamicMap(selected_points, streams=[selection])
zoom = hv.DynamicMap(test_bounds, streams=[box])
hvplot = renderer.get_plot(heatmap, curdoc())
gvplot = renderer.get_plot(world_map, curdoc())
bvplot = renderer.get_plot(zoom, curdoc())
vizual.children[1].children = [gvplot.state, hvplot.state, bvplot.state]
def gen_real_hover(self, resetting, x_range, y_range):
if resetting:
x_range = self.xrange
y_range = self.yrange1
opts_hover_time = dict(alpha=0,
hover_alpha=0.2,
fill_alpha=0,
framewise=True,
color='r')
hover = self.time_hover_gen(self.time_df,
vdims='Real',
x_range=x_range)
return decimate(hover, max_samples=self.max_samples,
dynamic=False).opts(**opts_hover_time)
def gen_imag_hover(self, resetting, x_range, y_range):
# print("gen_imag_hover {}".format(resetting))
if resetting:
x_range = self.xrange
y_range = self.yrange1
opts_hover_time = dict(alpha=0,
hover_alpha=0.2,
fill_alpha=0,
framewise=True,
color='r')
hover = self.time_hover_gen(self.time_df,
vdims='Imag',
x_range=x_range)
return decimate(
hover, max_samples=self.max_samples,
dynamic=False).opts(**opts_hover_time) #, dynamic=True)
def update_map_val(attr, old, new):
global selection, vizual, gvplot, hvplot, heatmap, feats, points, world_map, temp_feats
max_cur_feature = loc_feats[choice.value].max()
min_cur_feature = loc_feats[choice.value].min()
new_loc_feats = temp_feats.loc[(temp_feats[choice.value] <= new[1])
& (temp_feats[choice.value] >= new[0])]
feats = gv.Dataset(new_loc_feats,
kdims=['Longitude', 'Latitude', choice.value])
points = feats.to(gv.Points, ['Longitude', 'Latitude'], [choice.value])
if len(new_loc_feats) <= 20000:
world_map = gv.Points(points).options(
'Points',
size=5,
cmap='viridis',
colorbar=True,
tools=TOOLS,
color_index=2,
width=900,
height=800,
colorbar_opts={'scale_alpha': 0.5},
fill_alpha=0.5,
line_alpha=0.5)
else:
world_map = decimate(gv.Points(points), max_samples=20000).options(
'Points',
size=5,
cmap='viridis',
colorbar=True,
tools=TOOLS,
color_index=2,
width=900,
height=800,
colorbar_opts={'scale_alpha': 0.5},
fill_alpha=0.5,
line_alpha=0.5)
selection = hv.streams.Selection1D(source=world_map)
heatmap = hv.DynamicMap(selected_points, streams=[selection])
box = hv.streams.BoundsXY(source=world_map)
zoom = hv.DynamicMap(test_bounds, streams=[box])
hvplot = renderer.get_plot(heatmap, curdoc())
gvplot = renderer.get_plot(world_map, curdoc())
bvplot = renderer.get_plot(zoom, curdoc())
vizual.children[1].children = [gvplot.state, hvplot.state, bvplot.state]
def point_chooser(self, **kwargs):
'''
Visual tool for choosing points in the mask.
:param kwargs: array, if you want to pre select pixels,
initialize_mask, True if you want to initialize mask.
True also Overrides array.
False does nothing and overrides
nothing
initialize_value, what value you want to use in
initialisation. Used if initialize_mask
is True. Fallback value is 0.
:return: hv.Object
'''
self.__do_mask_changes(**kwargs)
tap = streams.Tap(rename={
'x': 'first_coord',
'y': 'second_coord'
},
transient=True)
ds_attrs = self._make_dataset_opts()
dataset3d = hv.Dataset(**ds_attrs)
mask_dims = self._obj.M.dims.copy()
mask_dims.reverse()
layout = dataset3d.to(hv.Image,
mask_dims,
'Value',
self._obj.M.no_mask_dims) * \
decimate(
hv.DynamicMap(
lambda first_coord, second_coord:
hv.Points(self._record_taps(first_coord,
second_coord),
kdims=mask_dims),
streams=[tap]
)
)
return layout
def __call__(self, dset, **params):
self.p = ParamOverrides(self, params)
if self.p.vdim is None:
vdim = dset.vdims[0].name
else:
vdim = self.p.vdim
pts = hv.util.Dynamic(dset,
operation=skypoints,
streams=[self.p.filter_stream])
if self.p.aggregator == 'mean':
aggregator = ds.mean(vdim)
elif self.p.aggregator == 'std':
aggregator = ds.std(vdim)
elif self.p.aggregator == 'count':
aggregator = ds.count()
kwargs = dict(cmap=cc.palette[self.p.cmap], aggregator=aggregator)
if self.p.width is not None:
kwargs.update(width=self.p.width, height=self.p.height)
# streams=[hv.streams.RangeXY])
decimate_opts = dict(plot={'tools': ['hover', 'box_select']},
style={
'alpha': 0,
'size': self.p.decimate_size,
'nonselection_alpha': 0
})
decimated = decimate(pts).opts(**decimate_opts)
sky_shaded = datashade(pts, **kwargs)
return dynspread(sky_shaded) * decimated
def produce_timehistory(doc, peak, peakTS):
# Streams
# See if you can limit the buffer
b_th_peak = Buffer(pd.DataFrame({
'peak': [],
'lowerbound': [],
'higherbound': []
}),
length=1000000)
b_th_peak_std = Buffer(pd.DataFrame({
'peak': [],
'lowerbound': [],
'higherbound': []
}),
length=1000000)
s_spikes = Buffer(pd.DataFrame({'y': []}), length=1000000)
s_labels = Buffer(pd.DataFrame({'x': [], 'y': []}), length=1000000)
s_spikes_end = Buffer(pd.DataFrame({'y': []}), length=1000000)
#s_spikes = hv.streams.Stream.define('df', df=pd.DataFrame({'location':[], 'y':[]}))()
#s_labels = hv.streams.Stream.define('df', df=pd.DataFrame({'x':[], 'y':[], 'labels':[]}))()
# Generate dynamic map
plot_peak_b = hv.DynamicMap(
partial(hv.Points, kdims=['index', 'peak']),
streams=[b_th_peak
]).options(width=1000,
finalize_hooks=[apply_formatter
]).redim.label(index='Time in UTC')
# HoloViews seems to currently have a bug with hv.Spread with buffers, once it is fixed
# we can try to implement hv.spread instead of this
plot_peak_std_low = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'lowerbound']),
streams=[b_th_peak
]).options(line_alpha=0.5,
line_color='gray').redim.label(index='Time in UTC')
plot_peak_std_high = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'higherbound']),
streams=[b_th_peak
]).options(line_alpha=0.5,
line_color='gray').redim.label(index='Time in UTC')
#hvSpikes = hv.DynamicMap(gen_spikes, streams=[s_spikes])
#hvLabels = hv.DynamicMap(gen_labels, streams=[s_labels])
hvSpikes = hv.DynamicMap(partial(hv.Spikes),
streams=[s_spikes]).options(apply_ranges=False,
color='green')
hvSpikes_end = hv.DynamicMap(hv.Spikes,
streams=[s_spikes_end
]).options(apply_ranges=False,
color='red')
hvLabels = hv.DynamicMap(partial(hv.Labels, kdims=['x', 'y']),
streams=[s_labels]).options(apply_ranges=False)
#testing = datashade(plot_ipm_b).options(width = 1000)
test1 = datashade(plot_peak_std_low,
streams=[hv.streams.PlotSize],
normalization='linear').options(
width=1000, finalize_hooks=[apply_formatter])
#.opts(norm=dict(framewise=True))
test2 = datashade(plot_peak_std_high,
streams=[hv.streams.PlotSize],
normalization='linear')
#.opts(norm=dict(framewise=True))
pointTest = decimate(plot_peak_b,
streams=[hv.streams.PlotSize],
normalization='linear')
# Scrolling after pausing the graph seems to cause Parameter name clashes for keys: {'height', 'width', 'scale'} error!
#plot = (plot_ipm_b*plot_ipm_std_low*plot_ipm_std_high)
plot = (pointTest * test1 * test2 * hvSpikes * hvLabels * hvSpikes_end)
#plot=hvSpikes*hvLabels
# Use bokeh to render plot
hvplot = renderer.get_plot(plot, doc)
switch_key = 'peak_8'
start = [1534280820000, 1534271880000]
end = [1534279380000, 1534272000000]
labels = ['Test 1', 'Test 2']
medianCheck = 0
# For pushing in data, maybe cut off first 119 points to get rid of those weird extremes
def push_data(stream):
TS_key = switch_key + '_TS'
data = list(peak[switch_key])
timestamp = list(peakTS[TS_key])
times = [1000 * time for time in timestamp]
dataSeries = pd.Series(data, index=times)
zipped = basic_event_builder(peak=dataSeries)
median = zipped.rolling(120, min_periods=1).median()
std = zipped.rolling(120, min_periods=1).std()
lowerbound = median - std
higherbound = median + std
df = pd.DataFrame({
'peak': median['peak'],
'lowerbound': lowerbound['peak'],
'higherbound': higherbound['peak']
})
stream.send(df)
def push_spikes(stream, position):
#start = [1534279200000, 1534201350000]
doubledData = [val for val in position for _ in (0, 1)]
height = [1000, -1000]
heightList = height * len(position)
df = pd.DataFrame({'location': doubledData, 'y': heightList})
df = df.set_index('location')
df.index.name = None
stream.send(df)
def push_labels(stream, labels, start):
nonlocal medianCheck
# Need to clear the buffers somehow to stop weird overlapping
height = [1]
data = list(peak[switch_key])
dataSeries = pd.Series(data)
median = [dataSeries.median()]
if median == medianCheck:
pass
else:
heightList = median * len(start)
df = pd.DataFrame({'x': start, 'y': heightList, 'labels': labels})
df = df.set_index('labels')
df.index.name = None
with util.disable_constant(s_labels):
s_labels.data = s_labels.data.iloc[:0]
stream.send(pd.DataFrame({'x': [], 'y': []}))
stream.send(df)
medianCheck = median
def clear_buffer():
"""
Modified version of hv.buffer.clear() since original appears to be
buggy
Clear buffer/graph whenever switch is toggled
"""
nonlocal b_th_peak, b_th_peak_std, s_labels
with util.disable_constant(b_th_peak) and util.disable_constant(
b_th_peak_std) and util.disable_constant(s_labels):
b_th_peak.data = b_th_peak.data.iloc[:0]
b_th_peak_std.data = b_th_peak_std.data.iloc[:0]
s_labels.data = s_labels.data.iloc[:0]
b_th_peak.send(
pd.DataFrame({
'peak': [],
'lowerbound': [],
'higherbound': []
}))
b_th_peak_std.send(
pd.DataFrame({
'peak': [],
'lowerbound': [],
'higherbound': []
}))
s_labels.send(pd.DataFrame({'x': [], 'y': []}))
def switch(attr, old, new):
"""
Update drop down menu value
"""
nonlocal switch_key
switch_key = select.value
clear_buffer()
print("Yes!")
def play_graph():
"""
Provide play and pause functionality to the graph
"""
nonlocal callback_id_th_b, cb_id_spikes, cb_id_labels, cb_id_spikes_end
if startButton.label == '► Play':
startButton.label = '❚❚ Pause'
callback_id_th_b = doc.add_periodic_callback(
partial(push_data, stream=b_th_peak), 1000)
cb_id_spikes = doc.add_periodic_callback(
partial(push_spikes, stream=s_spikes, position=start), 1000)
cb_id_labels = doc.add_periodic_callback(
partial(push_labels,
stream=s_labels,
labels=labels,
start=start), 1000)
cb_id_spikes_end = doc.add_periodic_callback(
partial(push_spikes, stream=s_spikes_end, position=end), 1000)
else:
startButton.label = '► Play'
doc.remove_periodic_callback(callback_id_th_b)
doc.remove_periodic_callback(cb_id_spikes)
doc.remove_periodic_callback(cb_id_labels)
doc.remove_periodic_callback(cb_id_spikes_end)
select = Select(title="Peak:", value="peak_8", options=list(peak))
select.on_change('value', switch)
startButton = Button(label='❚❚ Pause')
startButton.on_click(play_graph)
callback_id_th_b = doc.add_periodic_callback(
partial(push_data, stream=b_th_peak), 1000)
cb_id_labels = doc.add_periodic_callback(
partial(push_labels, stream=s_labels, labels=labels, start=start),
1000)
cb_id_spikes = doc.add_periodic_callback(
partial(push_spikes, stream=s_spikes, position=start), 1000)
cb_id_spikes_end = doc.add_periodic_callback(
partial(push_spikes, stream=s_spikes_end, position=end), 1000)
plot = column(select, startButton, hvplot.state)
doc.title = "Time History Graphs"
doc.add_root(plot)
def test_bounds(bounds):
global selection, vizual, gvplot, hvplot, heatmap, feats, points, world_map, range_slider, controls3, max_cur_feature, min_cur_feature, temp_feats
if bounds != None:
min_long = min(bounds[0], bounds[2])
max_long = max(bounds[0], bounds[2])
min_lat = min(bounds[1], bounds[3])
max_lat = max(bounds[1], bounds[3])
downscale_feats = loc_feats.loc[(loc_feats['Longitude'] >= min_long)
& (loc_feats['Longitude'] <= max_long)
& (loc_feats['Latitude'] >= min_lat) &
(loc_feats['Latitude'] <= max_lat)]
temp_feats = downscale_feats
max_cur_feature = temp_feats[choice.value].max()
min_cur_feature = temp_feats[choice.value].min()
range_slider = RangeSlider(start=min_cur_feature,
end=max_cur_feature,
value=(min_cur_feature, max_cur_feature),
step=(max_cur_feature - min_cur_feature) /
20,
title="Feature_range")
range_slider.on_change('value', update_map_val)
controls3 = widgetbox([range_slider], width=250)
new_loc_feats = temp_feats.loc[
(temp_feats[choice.value] <= range_slider.value[1])
& (temp_feats[choice.value] >= range_slider.value[0])]
feats = gv.Dataset(new_loc_feats,
kdims=['Longitude', 'Latitude', choice.value])
points = feats.to(gv.Points, ['Longitude', 'Latitude'], [choice.value])
if len(new_loc_feats <= 20000):
world_map = gv.Points(points).options(
'Points',
size=5,
cmap='viridis',
colorbar=True,
tools=TOOLS,
color_index=2,
width=900,
height=800,
colorbar_opts={'scale_alpha': 0.5},
fill_alpha=0.5,
line_alpha=0.5)
else:
world_map = decimate(gv.Points(points), max_samples=20000).options(
'Points',
size=5,
cmap='viridis',
colorbar=True,
tools=TOOLS,
color_index=2,
width=900,
height=800,
colorbar_opts={'scale_alpha': 0.5},
fill_alpha=0.5,
line_alpha=0.5)
selection = hv.streams.Selection1D(source=world_map)
heatmap = hv.DynamicMap(selected_points, streams=[selection])
box = hv.streams.BoundsXY(source=world_map)
zoom = hv.DynamicMap(test_bounds, streams=[box])
hvplot = renderer.get_plot(heatmap, curdoc())
gvplot = renderer.get_plot(world_map, curdoc())
bvplot = renderer.get_plot(zoom, curdoc())
vizual.children[1].children = [
gvplot.state, hvplot.state, bvplot.state
]
vizual.children[0].children[2] = controls3
else:
bounds = (0, 0, 0, 0)
return hv.Bounds(bounds).options(show_grid=False,
height=0,
width=0,
xaxis=None,
yaxis=None,
default_tools=[],
show_frame=False,
toolbar=None)
cur_var = 'Risk Score'
temp_feats = loc_feats
feats = gv.Dataset(temp_feats, kdims=['Longitude', 'Latitude', cur_var])
points = feats.to(gv.Points, ['Longitude', 'Latitude'], [cur_var])
tiles = gv.tile_sources.CartoEco
cur_size = 5
max_cur_feature = temp_feats['Risk Score'].max()
min_cur_feature = temp_feats['Risk Score'].min()
world_map = decimate(gv.Points(points), max_samples=20000).options(
'Points',
size=cur_size,
cmap='viridis',
colorbar=True,
tools=TOOLS,
color_index=2,
width=900,
height=800,
global_extent=True,
colorbar_opts={'scale_alpha': 0.5},
fill_alpha=0.5,
line_alpha=0.4)
def update_map(attr, old, new):
global selection, vizual, gvplot, hvplot, heatmap, feats, points, world_map, range_slider, controls3, max_cur_feature, min_cur_feature, temp_feats
max_cur_feature = temp_feats[choice.value].max()
min_cur_feature = temp_feats[choice.value].min()
range_slider = RangeSlider(start=min_cur_feature,
end=max_cur_feature,
value=(min_cur_feature, max_cur_feature),
def heatmap(adata,
genes,
groups=None,
compare='genes',
agg_fns=['mean', 'var'],
use_raw=False,
order_keys=[],
hover=True,
show_highlight=False,
show_scatter=False,
subsample=None,
keep_frac=0.2,
seed=None,
xrotation=90,
yrotation=0,
colorbar=True,
cont_cmap=None,
height=200,
width=600,
save=None,
**scatter_kwargs):
'''
Plot a heatmap with groups selected from a drop-down menu.
If `show_highlight=True` and `show_scatterplot=True`, additional
interaction occurrs when clicking on the highlighted heatmap.
Params
-------
adata: anndata.AnnData
adata object
genes: List[Str]
genes in `adata.var_names`
groups: List[Str], optional (default: `None`)
categorical observation in `adata.obs`,
if `None`, get all groups from `adata.obs`
compare: Str, optional (default: `'genes'`)
only used when `show_scatterplot=True`,
creates a drop-down:
if `'genes'`:
drow-down menu will contain values from `genes` and clicking
a gene in highlighted heatmap will plot scatterplot of the 2
genes with groups colored in
if `'basis'`:
drow-down menu will contain available bases and clicking
a gene in highlighted heatmap will plot the gene in the selected
embedding with its expression colored in
if `'order'`:
drop-down menu will contain values from `order_keys`,
and clicking on a gene in highlighted heatmap will plot its expression
in selected order
agg_fns: List[Str], optional (default: `['mean', 'var']`
names of pandas' aggregation functions, such `'min'`, ...
the first function specified is mapped to colors
use_raw: Bool, optional (default: `False`)
whether to use `.raw` for gene expression
order_keys: List[Str], optional (default: `None`)
keys in `adata.obs`, used when `compare='order'`
hover: Bool, optional (default: `True`)
whether to display hover information over the heatmap
show_highlight: Bool, optional (default: `False`)
whether to show when using boxselect
show_scatter: Bool, optional (default: `False`)
whether to show a scatterplot,
if `True`, overrides `show_highlight=False`
subsample: Str, optional (default: `'decimate'`)
subsampling strategy for large data
possible values are `None, 'none', 'decimate'`
keep_frac: Float, optional (default: `0.2`)
fraction of cells to keep, used when `subsample='decimate'`
seed: Union[Float, NoneType], optional (default: `None`)
random seed, used when `subsample='decimate'`
xrotation: Int, optional (default: `90`)
rotation of labels on x-axis
yrotation: Int, optional (default: `0`)
rotation of labels on y-axis
colorbar: Bool, optional (default: `True`)
whether to show colorbar
cont_cmap: Union[List[Str], NoneType], optional (default, `None`)
colormap of the heatmap,
if `None`, use `Viridis256`
height: Int, optional (default: `200`)
height of the heatmap
width: Int, optional (default: `600`)
width of the heatmap
save: Union[os.PathLike, Str, NoneType], optional (default: `None`)
path where to save the plot
**scatter_kwargs:
additional argument for `ipl.experimental.scatter`,
only used when `show_scatter=True`
Returns
-------
holoviews plot
'''
def _highlight(group, index):
original = hm[group]
if not index:
return original
return original.iloc[sorted(index)]
def _scatter(group, which, gwise, x, y):
indices = adata.obs[group] == y if gwise else np.isin(
adata.obs[group], highlight[group].data['y'])
# this is necessary
indices = np.where(indices)[0]
if is_ordered:
scatter_kwargs['order_key'] = which
x, y = None, x
elif f'X_{which}' in adata.obsm:
group = x
x, y = which, which
else:
x, y = x, which
return scatter2(adata,
x=x,
y=y,
color=group,
indices=indices,
**scatter_kwargs).opts(axiswise=True, framewise=True)
assert keep_frac >= 0 and keep_frac <= 1, f'`keep_perc` must be in interval `[0, 1]`, got `{keep_frac}`.'
assert subsample in (None, 'none','decimate'), f'Invalid subsampling strategy `{subsample}`. ' \
'Possible values are `None, \'none\', \'decimate\'`.'
assert compare in (
'genes', 'basis', 'order'
), f'`compare` must be one of `\'genes\', \'basis\', \'order\'`.'
if cont_cmap is None:
cont_cmap = Viridis256
is_ordered = False
scatter_kwargs['use_original_limits'] = True
scatter_kwargs['subsample'] = None
if 'plot_width' not in scatter_kwargs:
scatter_kwargs['plot_width'] = 300
if 'plot_height' not in scatter_kwargs:
scatter_kwargs['plot_height'] = 300
if groups is not None:
assert len(groups) > 0, f'Number of groups `> 1`.'
else:
groups = [k for k in adata.obs.keys() if is_categorical(adata.obs[k])]
kdims = [hv.Dimension('Group', values=groups, default=groups[0])]
hm = hv.DynamicMap(lambda g: _heatmap(adata,
genes,
agg_fns=agg_fns,
group=g,
hover=hover,
use_raw=use_raw,
cmap=cont_cmap,
xrotation=xrotation,
yrotation=yrotation,
colorbar=colorbar),
kdims=kdims).opts(frame_height=height,
frame_width=width)
if not show_highlight and not show_scatter:
return hm
highlight = hv.DynamicMap(_highlight,
kdims=kdims,
streams=[Selection1D(source=hm)])
if not show_scatter:
return (hm + highlight).cols(1)
if compare == 'basis':
basis = [b.lstrip('X_') for b in adata.obsm.keys()]
kdims += [hv.Dimension('Components', values=basis, default=basis[0])]
elif compare == 'genes':
kdims += [hv.Dimension('Genes', values=genes, default=genes[0])]
else:
is_ordered = True
k = scatter_kwargs.pop('order_key', None)
assert k is not None or order_keys != [], f'No order keys specified.'
if k is not None and k not in order_keys:
order_keys.append(k)
for k in order_keys:
assert k in adata.obs, f'Order key `{k}` not found in `adata.obs`.'
kdims += [hv.Dimension('Order', values=order_keys)]
kdims += [
hv.Dimension('Groupwise',
type=bool,
values=[True, False],
default=True)
]
scatter_stream = hv.streams.Tap(source=highlight,
x=genes[0],
y=adata.obs[groups[0]].values[0])
scatter = hv.DynamicMap(_scatter, kdims=kdims, streams=[scatter_stream])
if subsample == 'decimate':
scatter = decimate(scatter,
max_samples=int(adata.n_obs * keep_frac),
streams=[hv.streams.RangeXY(transient=True)],
random_seed=seed)
plot = (hm + highlight + scatter).cols(1)
if save is not None:
hv.rendered('bokeh').save(plot, save)
return plot
def _scatter(adata,
x,
y,
condition,
by=None,
subsample='datashade',
steps=40,
keep_frac=0.2,
seed=None,
legend_loc='top_right',
size=4,
xlabel=None,
ylabel=None,
title=None,
use_raw=True,
hover=None,
hover_width=10,
hover_height=10,
kde=None,
density=None,
density_size=150,
jitter=None,
perc=None,
xlim=None,
ylim=None,
cmap=None,
show_legend=True,
plot_height=400,
plot_width=400):
_sentinel = object()
def create_density_plots(df, density, kdims, cmap):
cm = {}
if density == 'all':
dfs = {_sentinel: df}
elif density == 'group':
if 'z' not in df.columns:
warnings.warn(
f'`density=\'groups\' was specified, but no group found. Did you specify `color=...`?'
)
dfs = {_sentinel: df}
elif not is_categorical(df['z']):
warnings.warn(
f'`density=\'groups\' was specified, but column `{condition}` is not categorical.'
)
dfs = {_sentinel: df}
else:
dfs = {k: v for k, v in df.groupby('z')}
cm = cmap
else:
raise ValueError(
f'Invalid `density` type: \'`{density}`\'. Possible values are `\'all\'`, `\'group\'`.'
)
# assumes x, y order in kdims
return [
hv.Overlay([
hv.Distribution(df, kdims=dim).opts(color=cm.get(k, 'black'),
framewise=True)
for k, df in dfs.items()
]) for dim in kdims
]
assert keep_frac >= 0 and keep_frac <= 1, f'`keep_perc` must be in interval `[0, 1]`, got `{keep_frac}`.'
adata_mraw = get_mraw(adata, use_raw)
if subsample == 'uniform':
cb_kwargs = {'steps': steps}
elif subsample == 'density':
cb_kwargs = {'size': int(keep_frac * adata.n_obs), 'seed': seed}
else:
cb_kwargs = {}
categorical = False
if condition is None:
cmap = ['black'] * len(x) if subsample == 'datashade' else 'black'
elif is_categorical(condition):
categorical = True
cmap = Sets1to3 if cmap is None else cmap
cmap = odict(
zip(condition.cat.categories, adata.uns.get(f'{by}_colors', cmap)))
else:
cmap = Viridis256 if cmap is None else cmap
jitter_x, jitter_y = None, None
if isinstance(jitter, (tuple, list)):
assert len(
jitter
) == 2, f'`jitter` must be of length `2`, found `{len(jitter)}`.'
jitter_x, jitter_y = jitter
elif jitter is not None:
jitter_x, jitter_y = jitter, jitter
if jitter_x is not None:
x += np.random.normal(0, jitter_x, size=x.shape)
if jitter_y is not None:
y += np.random.normal(0, jitter_y, size=y.shape)
data = {'x': x, 'y': y, 'z': condition}
vdims = ['z']
hovertool = None
if hover is not None:
for k, dt in hover.items():
vdims.append(k)
data[k] = dt
hovertool = HoverTool(
tooltips=[(key.capitalize(), f'@{key}') for key in (
['index'] if subsample == 'datashade' else hover.keys())])
data = pd.DataFrame(data)
if categorical:
data['z'] = data['z'].astype('category')
if not vdims:
vdims = None
if xlim is None:
xlim = pad(*minmax(x))
if ylim is None:
ylim = pad(*minmax(y))
kdims = [('x', 'x' if xlabel is None else xlabel),
('y', 'y' if ylabel is None else ylabel)]
scatter = hv.Scatter(data, kdims=kdims, vdims=vdims).sort('z')
scatter = scatter.opts(size=size, xlim=xlim, ylim=ylim)
kde_plot= None if kde is None else \
hv.Bivariate(scatter).opts(bandwidth=kde, show_legend=False, line_width=2)
xdist, ydist = (None, None) if density is None else create_density_plots(
data, density, kdims, cmap)
if categorical:
scatter = scatter.opts(cmap=cmap,
color='z',
show_legend=show_legend,
legend_position=legend_loc)
elif 'z' in data:
scatter = scatter.opts(cmap=cmap,
color='z',
clim=tuple(map(float, minmax(data['z'], perc))),
colorbar=True,
colorbar_opts={'width': 20})
else:
scatter = scatter.opts(color='black')
legend = None
if subsample == 'datashade':
subsampled = dynspread(datashade(
scatter,
aggregator=(ds.count_cat('z') if categorical else ds.mean('z'))
if vdims is not None else None,
color_key=cmap,
cmap=cmap,
streams=[hv.streams.RangeXY(transient=True), hv.streams.PlotSize],
min_alpha=255).opts(axiswise=True, framewise=True),
threshold=0.8,
max_px=5)
if show_legend and categorical:
legend = hv.NdOverlay({
k: hv.Points([0, 0], label=str(k)).opts(size=0, color=v)
for k, v in cmap.items()
})
if hover is not None:
t = hv.util.Dynamic(rasterize(scatter, width=hover_width, height=hover_height, streams=[hv.streams.RangeXY],
aggregator=ds.reductions.min('index')), operation=hv.QuadMesh)\
.opts(tools=[hovertool], axiswise=True, framewise=True,
alpha=0, hover_alpha=0.25,
height=plot_height, width=plot_width)
scatter = t * subsampled
else:
scatter = subsampled
elif subsample == 'decimate':
scatter = decimate(scatter,
max_samples=int(adata.n_obs * keep_frac),
streams=[hv.streams.RangeXY(transient=True)],
random_seed=seed)
if legend is not None:
scatter = (scatter * legend).opts(legend_position=legend_loc)
if kde_plot is not None:
scatter *= kde_plot
scatter = scatter.opts(height=plot_height, width=plot_width)
scatter = scatter.opts(hv.opts.Scatter(
tools=[hovertool])) if hovertool is not None else scatter
if xdist is not None and ydist is not None:
scatter = (scatter << ydist.opts(width=density_size)) << xdist.opts(
height=density_size)
return scatter.opts(title=title if title is not None else '')
def load_raster(self):
nr_units = get_number_of_units_for_patient(self.patient_id)
neural_rec_time = get_neural_rectime_of_patient(
self.patient_id, self.session_nr) / 1000
data_array = []
for i in range(nr_units):
spikes = get_spiking_activity(self.patient_id, self.session_nr, i)
# delete downloaded file from working directory
if os.path.exists("neural_rec_time.npy"):
os.remove("neural_rec_time.npy")
data_array.append(list(np.array(spikes) - neural_rec_time[0]))
ret = []
for i in range(len(data_array)):
# i is unit ID
for j in data_array[i]:
# j is spike time
ret.append((j, i))
scatter = hv.Scatter(ret)
# Toggle variable decimate_plot to specify whether you'd like to use decimate
# Decimate only plots a maximum number of elements at each zoom step, this way the plot is much faster
# If decimate is not activated, the plot is clean, but very slow
decimate_plot = True
if decimate_plot:
scatter = scatter.opts(
color='blue',
marker='dash',
size=12,
alpha=1,
line_width=0.6,
angle=90,
xlabel='Time from beginning of recording in milliseconds',
ylabel='Unit ID')
# adjust the max_samples parameter if the plot is too slow or if you think it can handle even more spikes
raster = decimate(scatter, max_samples=40000).opts(
width=1500, height=800) * boxes
else:
scatter = scatter.opts(color='blue',
marker='dash',
size=12,
alpha=1,
line_width=0.2,
angle=90)
raster = scatter.opts(
width=1500,
height=800,
xlabel='Time from beginning of recording in milliseconds',
ylabel='Unit ID') * boxes
# extracting necessary information from the database
start_times_pauses = \
(MoviePauses() & "patient_id={}".format(self.patient_id) & "session_nr={}".format(self.session_nr)).fetch(
"start_times")[0]
stop_times_pauses = \
(MoviePauses() & "patient_id={}".format(self.patient_id) & "session_nr={}".format(self.session_nr)).fetch(
"stop_times")[0]
start_times_pauses = start_times_pauses - neural_rec_time[0]
stop_times_pauses = stop_times_pauses - neural_rec_time[0]
start_times_skips = \
(MovieSkips() & "patient_id={}".format(self.patient_id) & "session_nr={}".format(self.session_nr)).fetch("start_times")[0]
stop_times_skips = \
(MovieSkips() & "patient_id={}".format(self.patient_id) & "session_nr={}".format(self.session_nr)).fetch("stop_times")[0]
start_times_skips = start_times_skips - neural_rec_time[0]
stop_times_skips = stop_times_skips - neural_rec_time[0]
# The user can select a region which should be highlighted on top of the raster plot.
# Here every option of the highlights variable from above has to be implemented
if self.highlight == "Pauses":
ov = hv.NdOverlay({
i: hv.VSpan(start_times_pauses[i],
stop_times_pauses[i]).opts(color='orange',
alpha=0.4)
for i in range(len(start_times_pauses))
})
if self.highlight == "Skips":
ov = hv.NdOverlay({
i: hv.VSpan(start_times_skips[i],
stop_times_skips[i]).opts(color='green', alpha=0.4)
for i in range(len(start_times_skips))
})
if self.highlight == 'None':
ov = hv.NdOverlay({
i: hv.VSpan(0, 0).opts(color='white', alpha=0)
for i in range(1)
})
return raster * ov.opts(framewise=True)
def produce_timehistory(self, context, doc):
"""
Create timetool data timehistory
Parameters
----------
context = zmq.Context()
Creates zmq socket to receive data
doc: bokeh.document (I think)
Bokeh document to be displayed on webpage
"""
# Port to connect to master
port = 5000
socket = context.socket(zmq.REQ)
# MUST BE FROM SAME MACHINE, CHANGE IF NECESSARY!!!
socket.connect("tcp://localhost:%d" % port)
# Dynamic Maps
plot_peak_b = hv.DynamicMap(partial(hv.Curve, kdims=['index', 'peak']),
streams=[self.b_th_peak]).options(
width=1000,
finalize_hooks=[
apply_formatter
]).redim.label(index='Time in UTC')
plot_peak_std_low = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'lowerbound']),
streams=[self.b_th_peak]).options(
line_alpha=0.5,
width=1000,
line_color='gray',
finalize_hooks=[apply_formatter
]).redim.label(index='Time in UTC')
plot_peak_std_high = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'higherbound']),
streams=[self.b_th_peak]).options(
line_alpha=0.5, width=1000,
line_color='gray').redim.label(index='Time in UTC')
# Decimate and datashade
pointTest = decimate(plot_peak_b, streams=[hv.streams.PlotSize])
test1 = datashade(plot_peak_std_low,
streams=[hv.streams.PlotSize],
normalization='linear').options(
width=1000, finalize_hooks=[apply_formatter])
test2 = datashade(plot_peak_std_high,
streams=[hv.streams.PlotSize],
normalization='linear')
plot = (test1 * test2 * pointTest)
# Use bokeh to render plot
hvplot = renderer.get_plot(plot, doc)
def push_data(stream):
"""
Push data to timetool time history graph
"""
median = pd.DataFrame({'peak': []})
lowerbound = pd.DataFrame({'peak': []})
higherbound = pd.DataFrame({'peak': []})
# Request
socket.send_string("Hello")
print("Oof")
data_dict = socket.recv_pyobj()
peakDict = data_dict['peakDict']
peakTSDict = data_dict['peakTSDict']
TS_key = self.switch_key + '_TS'
data = list(peakDict[self.switch_key])
timestamp = list(peakTSDict[TS_key])
times = [1000 * time for time in timestamp]
dataSeries = pd.Series(data, index=times)
zipped = basic_event_builder(peak=dataSeries)
median = zipped.rolling(120, min_periods=1).median()
std = zipped.rolling(120, min_periods=1).std()
lowerbound = median - std
higherbound = median + std
df = pd.DataFrame({
'peak': median['peak'],
'lowerbound': lowerbound['peak'],
'higherbound': higherbound['peak']
})
stream.send(df)
def switch(attr, old, new):
"""
Update drop down menu value
"""
self.switch_key = select.value
self.clear_buffer()
print("Yes!")
def play_graph():
"""
Provide play and pause functionality to the graph
"""
if startButton.label == '► Play':
startButton.label = '❚❚ Pause'
self.callback_id_th_b = doc.add_periodic_callback(
partial(push_data, stream=self.b_th_peak), 1000)
else:
startButton.label = '► Play'
doc.remove_periodic_callback(self.callback_id_th_b)
peak_list = [
'peak_8', 'peak_9', 'peak_10', 'peak_11', 'peak_12', 'peak_13',
'peak_14', 'peak_15'
]
select = Select(title='Peak:', value='peak_8', options=peak_list)
select.on_change('value', switch)
startButton = Button(label='❚❚ Pause')
startButton.on_click(play_graph)
self.callback_id_th_b = doc.add_periodic_callback(
partial(push_data, stream=self.b_th_peak), 1000)
plot = column(select, startButton, hvplot.state)
doc.title = "Time History Graphs"
doc.add_root(plot)
if front: #we want "seq" to be in the front
#so append current column to the end of the list
cols.append(x)
else:
#we want "seq" to be last, so insert this
#column in the front of the new column list
#"cols" we are building:
cols.insert(0, x)
return dataframe[cols]
pd_nodes = pd.read_pickle('nodes.pkl')
pd_edges = pd.read_pickle('edges-bundled.pkl')
r_nodes = hv.Points(set_column_sequence(pd_nodes, ['x', 'y']), label='Nodes')
r_edges = hv.Curve(pd_edges, label='Bundled')
hv_layout=datashade(r_edges, **sz) * \
decimate(hv.Points(r_nodes),max_samples=1000)
# Opts
hv.opts(
'RGB [xaxis=None yaxis=None show_grid=False bgcolor="black" width={width} height={height}]'
.format(**sz))
hv.opts('Points [tools=["hover"]] (color="cyan", size=3)')
hv_plot = renderer.get_plot(hv_layout, curdoc())
bk_layout = layout([[hv_plot.state]], sizing_mode='fixed')
curdoc().add_root(bk_layout)