def __init__(self):
self.switchButton = 'ipm2__sum'
self.maxlen = 1000000
# Initialize buffers
self.b_timetool = Buffer(pd.DataFrame({
'timestamp': [],
'timetool': []
}),
length=40000)
self.b_IpmAmp = Buffer(pd.DataFrame({
'timetool': [],
'ipm': []
}),
length=1000)
self.b_corr_timehistory = Buffer(pd.DataFrame({
'timestamp': [],
'correlation': []
}),
length=40000)
# Initialize callbacks
self.cb_id_timetool = None
self.cb_id_amp_ipm = None
self.cb_id_corr_timehistory = None
def test_buffer_stream_following(self):
stream = Buffer(data={
'x': np.array([1]),
'y': np.array([1])
},
following=True)
dmap = DynamicMap(Curve, streams=[stream])
plot = bokeh_renderer.get_plot(dmap)
x_range = plot.handles['x_range']
y_range = plot.handles['y_range']
self.assertEqual(x_range.start, 0)
self.assertEqual(x_range.end, 2)
self.assertEqual(y_range.start, 0)
self.assertEqual(y_range.end, 2)
stream.send({'x': np.array([2]), 'y': np.array([-1])})
self.assertEqual(x_range.start, 1)
self.assertEqual(x_range.end, 2)
self.assertEqual(y_range.start, -1)
self.assertEqual(y_range.end, 1)
stream.following = False
stream.send({'x': np.array([3]), 'y': np.array([3])})
self.assertEqual(x_range.start, 1)
self.assertEqual(x_range.end, 2)
self.assertEqual(y_range.start, -1)
self.assertEqual(y_range.end, 1)
def test_buffer_stream(self):
stream = Buffer(data=self.df, index=False)
plot = self.df.hvplot('x', 'y', stream=stream)
pd.testing.assert_frame_equal(plot[()].data, self.df)
new_df = pd.DataFrame([[7, 8], [9, 10]], columns=['x', 'y'])
stream.send(new_df)
pd.testing.assert_frame_equal(plot[()].data, pd.concat([self.df, new_df]))
def test_spread_stream_data(self):
buffer = Buffer({'y': np.array([]), 'yerror': np.array([]), 'x': np.array([])})
dmap = DynamicMap(Spread, streams=[buffer])
plot = bokeh_renderer.get_plot(dmap)
buffer.send({'y': [1, 2, 1, 4], 'yerror': [.5, .2, .1, .5], 'x': [0,1,2,3]})
cds = plot.handles['cds']
self.assertEqual(cds.data['x'], np.array([0., 1., 2., 3., 3., 2., 1., 0.]))
self.assertEqual(cds.data['y'], np.array([0.5, 1.8, 0.9, 3.5, 4.5, 1.1, 2.2, 1.5]))
def test_dynamic_groupby_kdims_and_streams(self):
def plot_function(mydim, data):
return Scatter(data[data[:, 2]==mydim])
buff = Buffer(data=np.empty((0, 3)))
dmap = DynamicMap(plot_function, streams=[buff], kdims='mydim').redim.values(mydim=[0, 1, 2])
ndlayout = dmap.groupby('mydim', container_type=NdLayout)
self.assertIsInstance(ndlayout[0], DynamicMap)
data = np.array([(0, 0, 0), (1, 1, 1), (2, 2, 2)])
buff.send(data)
self.assertEqual(ndlayout[0][()], Scatter([(0, 0)]))
self.assertEqual(ndlayout[1][()], Scatter([(1, 1)]))
self.assertEqual(ndlayout[2][()], Scatter([(2, 2)]))
def __init__(self):
self.switch_key = 'peak_8'
self.maxlen = 1000000
# Initialize buffers
self.b_th_peak = Buffer(pd.DataFrame({
'peak': [],
'lowerbound': [],
'higherbound': []
}),
length=1000000)
# Initialize callbacks
self.callback_id_th_b = None
def initialize_loc_history_replay(self):
self._loc_hist_replay_stream = Buffer(pd.DataFrame({
"x": pd.Series([], dtype=float),
"y": pd.Series([], dtype=float),
"Name": pd.Series([], dtype=str),
"Date": pd.Series([], dtype=str)
}), length=100, index=False)
loc_dmap = hv.DynamicMap(partial(hv.Points, vdims=["Name", "Date"]), streams=[self._loc_hist_replay_stream])
trace_dmap = hv.DynamicMap(partial(hv.Curve), streams=[self._loc_hist_replay_stream])
# title_dmap = hv.DynamicMap(partial(hv.Text, x=13, y=13, text="Hello", vdims=["Date", "Name", "timestamp"], streams=[self._loc_hist_replay_stream]))
print("Now call start_loc_history_replay() whenever you are ready")
return (loc_dmap * trace_dmap). opts(ylim=(0, self.cc.breadth + 2), xlim=(0, self.cc.length + 2),
show_legend=False).\
opts(opts.Points(size=6, tools=["hover"], color="Date", cmap="Blues"))
def produce_timehistory(doc, ipm2List, ipm3List, ebeamList, ipm2TS, ipm3TS,
ebeamTS):
# Streams
# See if you can limit the buffer
b_th_ipm = Buffer(pd.DataFrame({'ipm': []}), length=1200)
b_th_ipm_std = Buffer(pd.DataFrame({
'lowerbound': [],
'higherbound': []
}),
length=1200)
switch_key = 'ipm2'
# Generate dynamic map
plot_ipm_b = hv.DynamicMap(hv.Scatter, streams=[b_th_ipm]).options(
width=1000,
finalize_hooks=[apply_formatter]).redim.label(index='Time in UTC',
ipm='ipm data')
plot_ipm_std_low = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'lowerbound']),
streams=[b_th_ipm_std
]).options(line_alpha=0.5,
line_color='gray').redim.label(index='Time in UTC')
plot_ipm_std_high = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'higherbound']),
streams=[b_th_ipm_std
]).options(line_alpha=0.5,
line_color='gray').redim.label(index='Time in UTC')
plot = (plot_ipm_b * plot_ipm_std_low * plot_ipm_std_high)
# Use bokeh to render plot
hvplot = renderer.get_plot(plot, doc)
# For pushing in data, maybe cut off first 119 points to get rid of those weird extremes
def push_data(stream):
if switch_key == 'ipm2':
ipm_plot = list(ipm2List)
timestamp_plot = list(ipm2TS)
elif switch_key == 'ipm3':
ipm_plot = list(ipm3List)
timestamp_plot = list(ipm3TS)
times = [1000 * time for time in timestamp_plot]
ipmData = pd.Series(ipm_plot, index=times)
zipped = basic_event_builder(ipm=ipmData)
median = zipped.rolling(120, min_periods=1).median()
# This might be making it take a long time to switch
if type(stream) == hv.streams.Buffer:
if len(median) > 1000:
counter = 0
pos = 0
send = pd.DataFrame({'ipm': []})
while counter < 1000:
divide = len(median) / 1000.
test = int(pos)
send = send.append(median.iloc[[test]])
#print(send)
#print(type(median.iloc[[test]]))
pos += divide
counter += 1
#print(len(send))
#print(len(stream.data))
stream.send(send)
#print(stream.data[-10:])
print("Done")
else:
stream.send(median)
#stream.send(median)
elif len(median) > 100:
stream.event(df=median)
def push_std(stream):
if switch_key == 'ipm2':
ipm_plot = list(ipm2List)
timestamp_plot = list(ipm2TS)
elif switch_key == 'ipm3':
ipm_plot = list(ipm3List)
timestamp_plot = list(ipm3TS)
times = [1000 * time for time in timestamp_plot]
ipmData = pd.Series(ipm_plot, index=times)
zipped = basic_event_builder(ipm=ipmData)
median = zipped.rolling(120, min_periods=1).median()
std = zipped.rolling(120, min_periods=1).std()
lowerbound = median - std
higherbound = median + std
df = pd.DataFrame({
'lowerbound': lowerbound['ipm'],
'higherbound': higherbound['ipm']
})
if len(df) > 1000:
counter = 0
pos = 0
send = pd.DataFrame({
'lowerbound': lowerbound['ipm'],
'higherbound': higherbound['ipm']
})
while counter < 1000:
divide = len(df) / 1000.
test = int(pos)
send = send.append(df.iloc[[test]])
pos += divide
counter += 1
#print(divide)
stream.send(send)
else:
stream.send(df)
def clear_buffer():
"""
Modified version of hv.buffer.clear() since original appears to be
buggy
Clear buffer/graph whenever switch is toggled
"""
nonlocal b_th_ipm, b_th_ipm_std
with util.disable_constant(b_th_ipm) and util.disable_constant(
b_th_ipm_std):
b_th_ipm.data = b_th_ipm.data.iloc[:0]
b_th_ipm_std.data = b_th_ipm_std.data.iloc[:0]
b_th_ipm.send(pd.DataFrame({'ipm': []}))
b_th_ipm_std.send(pd.DataFrame({
'lowerbound': [],
'higherbound': []
}), )
def switch(attr, old, new):
"""
Update drop down menu value
"""
nonlocal switch_key
switch_key = select.value
clear_buffer()
print("Yes!")
select = Select(title="ipm value:", value="ipm2", options=["ipm2", "ipm3"])
select.on_change('value', switch)
callback_id_th_b = doc.add_periodic_callback(
partial(push_data, stream=b_th_ipm), 1000)
cb_id_low_std_th = doc.add_periodic_callback(
partial(push_std, stream=b_th_ipm_std), 1000)
cb_id_high_std_th = doc.add_periodic_callback(
partial(push_std, stream=b_th_ipm_std), 1000)
plot = column(select, hvplot.state)
doc.title = "Time History Graphs"
doc.add_root(plot)
def __init__(self,
data,
kind=None,
by=None,
width=700,
height=300,
shared_axes=False,
columns=None,
grid=False,
legend=True,
rot=None,
title=None,
xlim=None,
ylim=None,
xticks=None,
yticks=None,
fontsize=None,
colormap=None,
stacked=False,
logx=False,
logy=False,
loglog=False,
hover=True,
style_opts={},
plot_opts={},
use_index=False,
value_label='value',
group_label='Group',
colorbar=False,
streaming=False,
backlog=1000,
timeout=1000,
persist=False,
use_dask=False,
**kwds):
# Validate DataSource
if not isinstance(data, DataSource):
raise TypeError('Can only plot intake DataSource types')
elif data.container != 'dataframe':
raise NotImplementedError('Plotting interface currently only '
'supports DataSource objects with '
'dataframe container.')
self.data_source = data
self.streaming = streaming
self.use_dask = use_dask
if streaming:
self.data = data.read()
self.stream = Buffer(self.data, length=backlog)
if gen is None:
raise ImportError('Streaming support requires tornado.')
@gen.coroutine
def f():
self.stream.send(data.read())
self.cb = PeriodicCallback(f, timeout)
elif use_dask and dd is not None:
ddf = data.to_dask()
self.data = ddf.persist() if persist else ddf
else:
self.data = data.read()
# High-level options
self.by = by or []
self.columns = columns
self.stacked = stacked
self.use_index = use_index
self.kwds = kwds
self.value_label = value_label
self.group_label = group_label
# Process style options
if 'cmap' in kwds and colormap:
raise TypeError("Only specify one of `cmap` and `colormap`.")
elif 'cmap' in kwds:
cmap = kwds.pop('cmap')
else:
cmap = colormap
self._style_opts = dict(**style_opts)
if cmap:
self._style_opts['cmap'] = cmap
if 'size' in kwds:
self._style_opts['size'] = kwds.pop('size')
if 'alpha' in kwds:
self._style_opts['alpha'] = kwds.pop('alpha')
# Process plot options
plot_options = dict(plot_opts)
plot_options['logx'] = logx or loglog
plot_options['logy'] = logy or loglog
plot_options['show_grid'] = grid
plot_options['shared_axes'] = shared_axes
plot_options['show_legend'] = legend
if xticks:
plot_options['xticks'] = xticks
if yticks:
plot_options['yticks'] = yticks
if width:
plot_options['width'] = width
if height:
plot_options['height'] = height
if fontsize:
plot_options['fontsize'] = fontsize
if colorbar:
plot_options['colorbar'] = colorbar
if self.kwds.get('vert', False):
plot_options['invert_axes'] = True
if rot:
if (kind == 'barh' or kwds.get('orientation') == 'horizontal'
or kwds.get('vert')):
axis = 'yrotation'
else:
axis = 'xrotation'
plot_options[axis] = rot
if hover:
plot_options['tools'] = ['hover']
self._hover = hover
self._plot_opts = plot_options
self._relabel = {'label': title}
self._dim_ranges = {
'x': xlim or (None, None),
'y': ylim or (None, None)
}
self._norm_opts = {'framewise': True}
def _process_data(self, kind, data, x, y, by, groupby, row, col, use_dask,
persist, backlog, label, value_label, hover_cols, kwds):
gridded = kind in self._gridded_types
gridded_data = False
# Validate DataSource
self.data_source = data
self.is_series = is_series(data)
if self.is_series:
data = data.to_frame()
if is_intake(data):
data = process_intake(data, use_dask or persist)
if groupby is not None and not isinstance(groupby, list):
groupby = [groupby]
if by is not None and not isinstance(by, list):
by = [by]
streaming = False
if isinstance(data, pd.DataFrame):
self.data = data
if is_geopandas(data) and kind is None:
geom_types = set(
[gt[5:] if 'Multi' in gt else gt for gt in data.geom_type])
if len(geom_types) > 1:
raise ValueError(
'The GeopandasInterface can only read dataframes which '
'share a common geometry type')
geom_type = list(geom_types)[0]
if geom_type == 'Point':
kind = 'points'
elif geom_type == 'Polygon':
kind = 'polygons'
elif geom_type in ('LineString', 'LineRing'):
kind = 'paths'
elif is_dask(data):
self.data = data.persist() if persist else data
elif is_streamz(data):
self.data = data.example
self.stream_type = data._stream_type
streaming = True
self.cb = data
if data._stream_type == 'updating':
self.stream = Pipe(data=self.data)
else:
self.stream = Buffer(data=self.data,
length=backlog,
index=False)
data.stream.gather().sink(self.stream.send)
elif is_xarray(data):
import xarray as xr
z = kwds.get('z')
if z is None and isinstance(data, xr.Dataset):
z = list(data.data_vars)[0]
if gridded and isinstance(data,
xr.Dataset) and not isinstance(z, list):
data = data[z]
ignore = (groupby or []) + (by or [])
dims = [
c for c in data.coords
if data[c].shape != () and c not in ignore
]
if kind is None and (not (x or y) or all(c in data.coords
for c in (x, y))):
if len(dims) == 1:
kind = 'line'
elif len(dims) == 2 or (x and y):
kind = 'image'
gridded = True
else:
kind = 'hist'
if gridded:
gridded_data = True
data, x, y, by_new, groupby_new = process_xarray(
data, x, y, by, groupby, use_dask, persist, gridded, label,
value_label)
if kind not in self._stats_types:
if by is None: by = by_new
if groupby is None: groupby = groupby_new
if groupby:
groupby = [g for g in groupby if g not in (row, col)]
self.data = data
else:
raise ValueError('Supplied data type %s not understood' %
type(data).__name__)
# Validate data and arguments
if by is None: by = []
if groupby is None: groupby = []
if gridded:
if not gridded_data:
raise ValueError('%s plot type requires gridded data, '
'e.g. a NumPy array or xarray Dataset, '
'found %s type' %
(kind, type(self.data).__name__))
not_found = [g for g in groupby if g not in data.coords]
data_vars = list(data.data_vars) if isinstance(
data, xr.Dataset) else [data.name]
indexes = list(data.coords)
self.variables = list(data.coords) + data_vars
if groupby and not_found:
raise ValueError('The supplied groupby dimension(s) %s '
'could not be found, expected one or '
'more of: %s' %
(not_found, list(data.coords)))
else:
# Determine valid indexes
if isinstance(self.data, pd.DataFrame):
if self.data.index.names == [None]:
indexes = [self.data.index.name or 'index']
else:
indexes = list(self.data.index.names)
else:
indexes = [
c for c in self.data.reset_index().columns
if c not in self.data.columns
]
if len(indexes) == 2 and not (x or y or by):
if kind == 'heatmap':
x, y = indexes
elif kind in ('bar', 'barh'):
x, by = indexes
# Rename non-string columns
renamed = {
c: str(c)
for c in data.columns if not isinstance(c, hv.util.basestring)
}
if renamed:
self.data = self.data.rename(columns=renamed)
self.variables = indexes + list(self.data.columns)
# Reset groupby dimensions
groupby_index = [g for g in groupby if g in indexes]
if groupby_index:
self.data = self.data.reset_index(groupby_index)
not_found = [
g for g in groupby
if g not in list(self.data.columns) + indexes
]
if groupby and not_found:
raise ValueError('The supplied groupby dimension(s) %s '
'could not be found, expected one or '
'more of: %s' %
(not_found, list(self.data.columns)))
# Set data-level options
self.x = x
self.y = y
self.kind = kind or 'line'
self.gridded = gridded
self.use_dask = use_dask
self.indexes = indexes
if isinstance(by, (np.ndarray, pd.Series)):
self.data['by'] = by
self.by = ['by']
elif not by:
self.by = []
else:
self.by = by if isinstance(by, list) else [by]
self.groupby = groupby
self.streaming = streaming
self.hover_cols = hover_cols
# ### By Driver
#
# Chart laptimes for each driver.
#
# TO DO - how do we plot different traces on the chart?
#
# At the moment, perhaps just filter on a single driver.
#
# TO DO: better to do it on a car...
# +
# The buffer length is the max width of the chart?
# Do we need a buffer for each trace?
ncols = 2
buffer = Buffer(np.zeros((0, ncols)), length=50)
#Callback period in milliseconds
period = 20000
@gen.coroutine
def f():
currdb = getDriverLatestInDb()
tables = get_data(url)
dr = tables[DRIVERS_RESULT][['driverID', 'laps']]
print('s', set([tuple(x) for x in currdb.values]), 'x',
set([tuple(x) for x in dr.values]))
def produce_graphs(context, doc):
port = 5006
socket = context.socket(zmq.SUB)
# MUST BE FROM SAME MACHINE, CHANGE IF NECESSARY!!!
socket.connect("tcp://psanagpu114:%d" % port)
socket.setsockopt(zmq.SUBSCRIBE, b"")
b_scatter = Buffer(pd.DataFrame({'timetool': []}), length=40000)
b_IpmAmp = Buffer(pd.DataFrame({'ipm': []}), length=1000)
b_timehistory = Buffer(pd.DataFrame({'correlation': []}), length=40000)
hvScatter = hv.DynamicMap(hv.Points, streams=[b_scatter]).options(
width=1000, finalize_hooks=[apply_formatter],
xrotation=45).redim.label(index='Time in UTC')
hvIpmAmp = hv.DynamicMap(hv.Scatter, streams=[
b_IpmAmp
]).options(width=500).redim.label(index='Timetool Data')
hvTimeHistory = hv.DynamicMap(hv.Scatter, streams=[b_timehistory]).options(
width=500, finalize_hooks=[apply_formatter],
xrotation=45).redim.label(time='Time in UTC')
layout = (hvIpmAmp + hvTimeHistory + hvScatter).cols(2)
hvplot = renderer.get_plot(layout)
cb_id_scatter = None
cb_id_amp_ipm = None
cb_id_timehistory = None
def push_data_scatter(buffer):
timetool_d = deque(maxlen=1000000)
timetool_t = deque(maxlen=1000000)
# Current bug, may need to have continuous stream of data?
if socket.poll(timeout=0):
stuff = socket.recv_pyobj()
timetool_d = stuff['tt__FLTPOS_PS']
timeData = deque(maxlen=1000000)
for time in stuff['event_time']:
num1 = str(time[0])
num2 = str(time[1])
fullnum = num1 + "." + num2
timeData.append(float(fullnum))
print("Scatter")
timetool_t = timeData
timeStuff = list(timetool_t)
# Convert time to seconds so bokeh formatter can get correct datetime
times = [1000 * time for time in timeStuff]
data = pd.DataFrame({'timestamp': times, 'timetool': timetool_d})
data = data.set_index('timestamp')
data.index.name = None
buffer.send(data)
print("Boop")
def push_data_amp_ipm(buffer):
timetool_d = deque(maxlen=1000000)
ipm2_d = deque(maxlen=1000000)
if socket.poll(timeout=0):
stuff = socket.recv_pyobj()
timetool_d = stuff['tt__AMPL']
ipm2_d = stuff['ipm2__sum']
data = pd.DataFrame({'timetool': timetool_d, 'ipm': ipm2_d})
data = data.set_index('timetool')
data.index.name = None
buffer.send(data)
print("Gottem")
def push_data_correlation_time_history(buffer):
timetool_d = deque(maxlen=1000000)
timetool_t = deque(maxlen=1000000)
ipm2_d = deque(maxlen=1000000)
if socket.poll(timeout=0):
stuff = socket.recv_pyobj()
timetool_d = stuff['tt__FLTPOS_PS']
ipm2_d = stuff['ipm2__sum']
timeData = deque(maxlen=1000000)
for time in stuff['event_time']:
num1 = str(time[0])
num2 = str(time[1])
fullnum = num1 + "." + num2
timeData.append(float(fullnum))
timetool_t = timeData
timeStuff = list(timetool_t)
# Convert time to seconds so bokeh formatter can get correct datetime
times = [1000 * time for time in timeStuff]
data = pd.DataFrame({'timetool': timetool_d, 'ipm': ipm2_d})
data_list = data['timetool'].rolling(window=120).corr(
other=data['ipm'])
final_df = pd.DataFrame({
'time': times[119:],
'correlation': data_list[119:]
})
final_df = final_df.set_index('time')
final_df.index.name = None
buffer.send(final_df)
print("Heh")
# def switch(attr, old, new):
# """
# Update drop down menu value
# """
# No non local in python 2!
# global switch_key, b_timehistory, b_IpmAmp
# switch_key = select.value
# b_timehistory.clear()
# b_IpmAmp.clear()
cb_id_scatter = doc.add_periodic_callback(
partial(push_data_scatter, buffer=b_scatter), 1000)
cb_id_amp_ipm = doc.add_periodic_callback(
partial(push_data_amp_ipm, buffer=b_IpmAmp), 1000)
cb_id_timehistory = doc.add_periodic_callback(
partial(push_data_correlation_time_history, buffer=b_timehistory),
1000)
plot = hvplot.state
doc.add_root(plot)
def produce_correlation_graphs(doc, diode_t_dict, diode_dict):
"""
Produce correlation graphs and push them onto the web page document.
Parameters
----------
doc: bokeh.document (I think)
Bokeh document to be displayed on webpage
diode_t_dict: dictionary
diciontary with deques containing timestamps of the diode readings
diode_dict: dictionary
dictionary with deques containing diode readins
"""
# Initialize formatting variables
buffer_length = 40000
width = 500
# Initialize Streams
b_dcc_dco = Buffer(pd.DataFrame({
'x_diode': [],
'y_diode': []
}),
length=buffer_length)
b_t4d_dd = Buffer(pd.DataFrame({
'x_diode': [],
'y_diode': []
}),
length=buffer_length)
b_do_di = Buffer(pd.DataFrame({
'x_diode': [],
'y_diode': []
}),
length=buffer_length)
b_t4d_dco = Buffer(pd.DataFrame({
'x_diode': [],
'y_diode': []
}),
length=buffer_length)
# Initialize dynamic maps
hvPoint_dcc_dco = hv.DynamicMap(
partial(hv.Scatter, kdims=['x_diode', 'y_diode'], group='DCC vs DCO'),
streams=[b_dcc_dco]).options(width=width).redim.label(x_diode='DCC',
y_diode='DCO')
hvPoint_t4d_dd = hv.DynamicMap(
partial(hv.Scatter, kdims=['x_diode', 'y_diode'], group='T4D vs DD'),
streams=[b_t4d_dd]).options(width=width).redim.label(x_diode='T4D',
y_diode='DD')
hvPoint_do_di = hv.DynamicMap(
partial(hv.Scatter, kdims=['x_diode', 'y_diode'], group='DO vs DI'),
streams=[b_do_di]).options(width=width).redim.label(x_diode='DO',
y_diode='DI')
hvPoint_t4d_dco = hv.DynamicMap(
partial(hv.Scatter, kdims=['x_diode', 'y_diode'], group='T4D vs DCO'),
streams=[b_t4d_dco]).options(width=width).redim.label(x_diode='T4D',
y_diode='DCO')
#plots_col = (hvPoint_dcc_dco + hvPoint_t4d_dco + hvPoint_do_di + hvPoint_t4d_dd).cols(2)
# Render plot with bokeh
hvplot = renderer.get_plot(plots_col, doc)
# Initialize callbacks
cb_id_dcc_dco = None
cb_id_t4d_dd = None
cb_id_do_di = None
cb_id_t4d_dco = None
# Push data into buffers
def push_data(x_diode, y_diode, x_diode_t, y_diode_t, buffer):
"""
Push data from x and y diode into buffer to be graphed.
"""
x_diode_data = pd.Series(x_diode, index=x_diode_t)
y_diode_data = pd.Series(y_diode, index=y_diode_t)
zipped = basic_event_builder(x_diode=x_diode_data,
y_diode=y_diode_data)
buffer.send(zipped)
#
def play_graph():
"""
Provide play and pause functionality to the graph
"""
nonlocal cb_id_dcc_dco, cb_id_t4d_dd, cb_id_do_di, cb_id_t4d_dco
cb_time = 1000
if startButton.label == '► Play':
startButton.label = '❚❚ Pause'
cb_id_dcc_dco = doc.add_periodic_callback(
partial(push_data,
x_diode=diode_dict['dcc_d'],
y_diode=diode_dict['dco_d'],
x_diode_t=diode_t_dict['dcc_t'],
y_diode_t=diode_t_dict['dco_t'],
buffer=b_dcc_dco), cb_time)
cb_id_t4d_dd = doc.add_periodic_callback(
partial(push_data,
x_diode=diode_dict['t4d_d'],
y_diode=diode_dict['dd_d'],
x_diode_t=diode_t_dict['t4d_t'],
y_diode_t=diode_t_dict['dd_t'],
buffer=b_t4d_dd), cb_time)
cb_id_do_di = doc.add_periodic_callback(
partial(push_data,
x_diode=diode_dict['do_d'],
y_diode=diode_dict['di_d'],
x_diode_t=diode_t_dict['do_t'],
y_diode_t=diode_t_dict['di_t'],
buffer=b_do_di), cb_time)
cb_id_t4d_dco = doc.add_periodic_callback(
partial(push_data,
x_diode=diode_dict['t4d_d'],
y_diode=diode_dict['dco_d'],
x_diode_t=diode_t_dict['t4d_t'],
y_diode_t=diode_t_dict['dco_t'],
buffer=b_t4d_dco), cb_time)
else:
startButton.label = '► Play'
doc.remove_periodic_callback(cb_id_dcc_dco)
doc.remove_periodic_callback(cb_id_t4d_dd)
doc.remove_periodic_callback(cb_id_do_di)
doc.remove_periodic_callback(cb_id_t4d_dco)
# Create widgets
startButton = Button(label='► Play')
startButton.on_click(play_graph)
plot = layout([startButton, row([hvplot.state])])
doc.title = "Correlation Graphs"
doc.add_root(plot)
def produce_curve(doc, diode_t_dict, diode_dict):
"""
Produce time history graphs and push them onto the web page document.
Parameters
----------
doc: bokeh.document (I think)
Bokeh document to be displayed on webpage
diode_t_dict: dictionary
diciontary with deques containing timestamps of the diode readings
diode_dict: dictionary
dictionary with deques containing diode readings
"""
# Initialize formatting variables
buffer_length = 40000
width = 500
xrotation = 45
transparent_line_value = 0.1
# Initialize streams
buffer_dcc = Buffer(pd.DataFrame({'diode': []}), length=buffer_length)
buffer_dci = Buffer(pd.DataFrame({'diode': []}), length=buffer_length)
buffer_dco = Buffer(pd.DataFrame({'diode': []}), length=buffer_length)
buffer_dd = Buffer(pd.DataFrame({'diode': []}), length=buffer_length)
buffer_di = Buffer(pd.DataFrame({'diode': []}), length=buffer_length)
buffer_do = Buffer(pd.DataFrame({'diode': []}), length=buffer_length)
buffer_t1d = Buffer(pd.DataFrame({'diode': []}), length=buffer_length)
buffer_t4d = Buffer(pd.DataFrame({'diode': []}), length=buffer_length)
b_dcc_std = Buffer(pd.DataFrame({
'lowerbound': [],
'higherbound': []
}),
length=buffer_length)
b_dci_std = Buffer(pd.DataFrame({
'lowerbound': [],
'higherbound': []
}),
length=buffer_length)
b_dco_std = Buffer(pd.DataFrame({
'lowerbound': [],
'higherbound': []
}),
length=buffer_length)
b_dd_std = Buffer(pd.DataFrame({
'lowerbound': [],
'higherbound': []
}),
length=buffer_length)
b_di_std = Buffer(pd.DataFrame({
'lowerbound': [],
'higherbound': []
}),
length=buffer_length)
b_do_std = Buffer(pd.DataFrame({
'lowerbound': [],
'higherbound': []
}),
length=buffer_length)
b_t1d_std = Buffer(pd.DataFrame({
'lowerbound': [],
'higherbound': []
}),
length=buffer_length)
b_t4d_std = Buffer(pd.DataFrame({
'lowerbound': [],
'higherbound': []
}),
length=buffer_length)
# Weird bug where if you leave the beginning of the Curve graph in buffer zone,
# there will be lines connecting to beginning
# Generate dynamic map for medians
hvPoint_dcc = hv.DynamicMap(partial(hv.Points, group='Diode', label='DCC'),
streams=[buffer_dcc]).options(
width=width,
finalize_hooks=[apply_formatter],
xrotation=xrotation).redim.label(
index='Time in UTC',
diode='DCC Reading')
hvPoint_dci = hv.DynamicMap(partial(hv.Points, group='Diode', label='DCI'),
streams=[buffer_dci]).options(
width=width,
finalize_hooks=[apply_formatter],
xrotation=xrotation).redim.label(
index='Time in UTC',
diode='DCI Reading')
hvPoint_dco = hv.DynamicMap(partial(hv.Points, group='Diode', label='DCO'),
streams=[buffer_dco]).options(
width=width,
finalize_hooks=[apply_formatter],
xrotation=xrotation).redim.label(
index='Time in UTC',
diode='DCO Reading')
hvPoint_dd = hv.DynamicMap(partial(hv.Points, group='Diode', label='DD'),
streams=[buffer_dd]).options(
width=width,
finalize_hooks=[apply_formatter],
xrotation=xrotation).redim.label(
index='Time in UTC', diode='DD Reading')
hvPoint_di = hv.DynamicMap(partial(hv.Points, group='Diode', label='DI'),
streams=[buffer_di]).options(
width=width,
finalize_hooks=[apply_formatter],
xrotation=xrotation).redim.label(
index='Time in UTC', diode='DI Reading')
hvPoint_do = hv.DynamicMap(partial(hv.Points, group='Diode', label='DO'),
streams=[buffer_do]).options(
width=width,
finalize_hooks=[apply_formatter],
xrotation=xrotation).redim.label(
index='Time in UTC', diode='DO Reading')
hvPoint_t1d = hv.DynamicMap(partial(hv.Points, group='Diode', label='T1D'),
streams=[buffer_t1d]).options(
width=width,
finalize_hooks=[apply_formatter],
xrotation=xrotation).redim.label(
index='Time in UTC',
diode='T1D Reading')
hvPoint_t4d = hv.DynamicMap(partial(hv.Points, group='Diode', label='T4D'),
streams=[buffer_t4d]).options(
width=width,
finalize_hooks=[apply_formatter],
xrotation=xrotation).redim.label(
index='Time in UTC',
diode='T4D Reading')
# Same bug with weird connecting lines, but doesn't happen for std Curve graphs (happens for Area graphs)
# Generate dynamic map for standard deviation
hvStd_dcc_low = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'lowerbound']),
streams=[b_dcc_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_dcc_high = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'higherbound']),
streams=[b_dcc_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_dci_low = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'lowerbound']),
streams=[b_dci_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_dci_high = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'higherbound']),
streams=[b_dci_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_dco_low = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'lowerbound']),
streams=[b_dco_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_dco_high = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'higherbound']),
streams=[b_dco_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_dd_low = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'lowerbound']),
streams=[b_dd_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_dd_high = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'higherbound']),
streams=[b_dd_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_di_low = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'lowerbound']),
streams=[b_di_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_di_high = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'higherbound']),
streams=[b_di_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_do_low = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'lowerbound']),
streams=[b_do_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_do_high = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'higherbound']),
streams=[b_do_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_t1d_low = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'lowerbound']),
streams=[b_t1d_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_t1d_high = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'higherbound']),
streams=[b_t1d_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_t4d_low = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'lowerbound']),
streams=[b_t4d_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
hvStd_t4d_high = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'higherbound']),
streams=[b_t4d_std
]).options(width=width,
line_alpha=transparent_line_value,
line_color='red').redim.label(index='Time in UTC')
# Ideally, we would have wanted to plot the standard deviations as an area curve
# I didn't do it beceause there's a bug, but this would theoretically be the code
# to make a dynamic map with hv.Area
# hvArea = hv.DynamicMap(partial(
# hv.Area, vdims=['lowerbound', 'higherbound']), streams=[buffer_std_test]).options(
# width=1000).redim.label(index='Time in UTC')
# Put plots into two columns in layout
plots = (hvPoint_dcc * hvStd_dcc_low * hvStd_dcc_high +
hvPoint_dci * hvStd_dci_low * hvStd_dci_high +
hvPoint_dco * hvStd_dco_low * hvStd_dco_high +
hvPoint_dd * hvStd_dd_low * hvStd_dd_high +
hvPoint_di * hvStd_di_low * hvStd_di_high +
hvPoint_do * hvStd_do_low * hvStd_do_high +
hvPoint_t1d * hvStd_t1d_low * hvStd_t1d_high +
hvPoint_t4d * hvStd_t4d_low * hvStd_t4d_high).cols(2)
# Use bokeh to render the plot
hvplot = renderer.get_plot(plots, doc)
# Initialize callbacks
cb_id_dcc = None
cb_id_dci = None
cb_id_dco = None
cb_id_dd = None
cb_id_di = None
cb_id_do = None
cb_id_t1d = None
cb_id_t4d = None
cb_id_dcc_std = None
cb_id_dci_std = None
cb_id_dco_std = None
cb_id_dd_std = None
cb_id_di_std = None
cb_id_do_std = None
cb_id_t1d_std = None
cb_id_t4d_std = None
def push_data_median(diode_t, diode, buffer):
"""
Push rolling median of diode readings and push resulting list into buffer
to be graphed.
"""
timeStuff = list(diode_t)
# Convert time to seconds so bokeh formatter can get correct datetime
times = [1000 * time for time in timeStuff]
diodeData = pd.Series(diode, index=times)
zipped = basic_event_builder(diode=diodeData)
median = zipped.rolling(120, min_periods=1).median()
# Exclude first 119 points because of binning issues and sparsing the data (Doesn't seem to really work though)
buffer.send(median[119::2])
zipped.to_csv('testData2.csv')
def push_data_std(diode_t, diode, buffer):
"""
Calculate rolling standard deviation of diode readings. Generate lists
containing values one standard deviation away from the median (lower and higher)
and push resulting list into buffer to be graphed.
"""
timeStuff = list(diode_t)
times = [
1000 * time for time in timeStuff
] # Convert time to seconds so bokeh formatter can get correct datetime
diodeData = pd.Series(diode, index=times)
zipped = basic_event_builder(diode=diodeData)
median = zipped.rolling(120, min_periods=1).median()
std = zipped.rolling(120, min_periods=1).std()
lowerbound = median - std
higherbound = median + std
df = pd.DataFrame({
'lowerbound': lowerbound['diode'],
'higherbound': higherbound['diode']
})
buffer.send(df[119::2])
def play_graph():
"""
Provide play and pause functionality to the graph
"""
nonlocal cb_id_dcc, cb_id_dci, cb_id_dco, cb_id_dd, cb_id_di, cb_id_do, cb_id_t1d, cb_id_t4d
nonlocal cb_id_dcc_std, cb_id_dci_std, cb_id_dco_std, cb_id_dd_std
nonlocal cb_id_di_std, cb_id_do_std, cb_id_t1d_std, cb_id_t4d_std
if startButton.label == '► Play':
startButton.label = '❚❚ Pause'
cb_time = 3000
# Callbacks for median lines
cb_id_dcc = doc.add_periodic_callback(
partial(push_data_median,
diode_t=diode_t_dict['dcc_t'],
diode=diode_dict['dcc_d'],
buffer=buffer_dcc), cb_time)
cb_id_dci = doc.add_periodic_callback(
partial(push_data_median,
diode_t=diode_t_dict['dci_t'],
diode=diode_dict['dci_d'],
buffer=buffer_dci), cb_time)
cb_id_dco = doc.add_periodic_callback(
partial(push_data_median,
diode_t=diode_t_dict['dco_t'],
diode=diode_dict['dco_d'],
buffer=buffer_dco), cb_time)
cb_id_dd = doc.add_periodic_callback(
partial(push_data_median,
diode_t=diode_t_dict['dd_t'],
diode=diode_dict['dd_d'],
buffer=buffer_dd), cb_time)
cb_id_di = doc.add_periodic_callback(
partial(push_data_median,
diode_t=diode_t_dict['di_t'],
diode=diode_dict['di_d'],
buffer=buffer_di), cb_time)
cb_id_do = doc.add_periodic_callback(
partial(push_data_median,
diode_t=diode_t_dict['do_t'],
diode=diode_dict['do_d'],
buffer=buffer_do), cb_time)
cb_id_t1d = doc.add_periodic_callback(
partial(push_data_median,
diode_t=diode_t_dict['t1d_t'],
diode=diode_dict['t1d_d'],
buffer=buffer_t1d), cb_time)
cb_id_t4d = doc.add_periodic_callback(
partial(push_data_median,
diode_t=diode_t_dict['t4d_t'],
diode=diode_dict['t4d_d'],
buffer=buffer_t4d), cb_time)
# Callbacks for std lines
cb_id_dcc_std = doc.add_periodic_callback(
partial(push_data_std,
diode_t=diode_t_dict['dcc_t'],
diode=diode_dict['dcc_d'],
buffer=b_dcc_std), cb_time)
cb_id_dci_std = doc.add_periodic_callback(
partial(push_data_std,
diode_t=diode_t_dict['dci_t'],
diode=diode_dict['dci_d'],
buffer=b_dci_std), cb_time)
cb_id_dco_std = doc.add_periodic_callback(
partial(push_data_std,
diode_t=diode_t_dict['dco_t'],
diode=diode_dict['dco_d'],
buffer=b_dco_std), cb_time)
cb_id_dd_std = doc.add_periodic_callback(
partial(push_data_std,
diode_t=diode_t_dict['dd_t'],
diode=diode_dict['dd_d'],
buffer=b_dd_std), cb_time)
cb_id_di_std = doc.add_periodic_callback(
partial(push_data_std,
diode_t=diode_t_dict['di_t'],
diode=diode_dict['di_d'],
buffer=b_di_std), cb_time)
cb_id_do_std = doc.add_periodic_callback(
partial(push_data_std,
diode_t=diode_t_dict['do_t'],
diode=diode_dict['do_d'],
buffer=b_do_std), cb_time)
cb_id_t1d_std = doc.add_periodic_callback(
partial(push_data_std,
diode_t=diode_t_dict['t1d_t'],
diode=diode_dict['t1d_d'],
buffer=b_t1d_std), cb_time)
cb_id_t4d_std = doc.add_periodic_callback(
partial(push_data_std,
diode_t=diode_t_dict['t4d_t'],
diode=diode_dict['t4d_d'],
buffer=b_t4d_std), cb_time)
else:
startButton.label = '► Play'
doc.remove_periodic_callback(cb_id_dcc)
doc.remove_periodic_callback(cb_id_dci)
doc.remove_periodic_callback(cb_id_dco)
doc.remove_periodic_callback(cb_id_dd)
doc.remove_periodic_callback(cb_id_di)
doc.remove_periodic_callback(cb_id_do)
doc.remove_periodic_callback(cb_id_t1d)
doc.remove_periodic_callback(cb_id_t4d)
doc.remove_periodic_callback(cb_id_dcc_std)
doc.remove_periodic_callback(cb_id_dci_std)
doc.remove_periodic_callback(cb_id_dco_std)
doc.remove_periodic_callback(cb_id_dd_std)
doc.remove_periodic_callback(cb_id_di_std)
doc.remove_periodic_callback(cb_id_do_std)
doc.remove_periodic_callback(cb_id_t1d_std)
doc.remove_periodic_callback(cb_id_t4d_std)
# Create widgets
startButton = Button(label='► Play')
startButton.on_click(play_graph)
plot = layout([startButton, hvplot.state])
doc.title = "Time History"
doc.add_root(plot)
def keyword_stream(doc):
mykeyword = ktl.cache(stream_server, stream_keyword)
def convert_time(timestamp):
return datetime.fromtimestamp(
timestamp) # .strptime('%Y-%m-%d %H:%M:%S.%f')
global example
mydata = generate_history(stream_server, stream_keyword, '30 minutes ago')
#example = pd.DataFrame(
# {'x': [convert_time(time.time())],
# 'y': [initial_y]},
# columns=['x', 'y'])
example = mydata
dfstream = Buffer(example, length=100, index=False)
curve_dmap = hv.DynamicMap(hv.Points,
streams=[dfstream]).options(color='red',
line_width=5,
width=800,
height=500,
xrotation=90)
# doc = curdoc()
@gen.coroutine
def update(x, y):
print("Update called")
global example
example = example.append({'time': x, 'value': y}, ignore_index=True)
# html = file_html(hvplot, CDN, "Plot: %s from %s" % (keyword, server))
# return html
dfstream.send(example)
# print(example.head())
def callback(keyword):
print("call back called")
y = keyword.binary
print("y:" + str(y))
time_now = keyword.timestamp
print("timestamp:" + str(time_now))
last_time_stamp = example.iloc[-1]['time']
print(last_time_stamp, convert_time(time_now))
# if str(last_time_stamp) == convert_time(time_now):
# print("time did not change, update not called")
# return
print("calling update")
# update(convert_time(time_now), float(y))
doc.add_next_tick_callback(
partial(update, x=convert_time(time_now), y=float(y)))
def start_monitor():
global example
global stop_signal
print("Monitor started")
mykeyword.callback(callback)
mykeyword.monitor()
while True:
print(example)
print("Thread is running")
if stop_signal:
print("stopping monitoring")
mykeyword.monitor(start=False)
stop_signal = False
break
time.sleep(10)
renderer = hv.renderer('bokeh')
hvplot = renderer.get_plot(curve_dmap).state
doc.add_root(hvplot)
# doc = renderer.server_doc(curve_dmap)
# start_monitor()
thread = Thread(name='Keyword_monitor', target=start_monitor)
thread.start()
def produce_timehistory(doc, peak, peakTS):
# Streams
# See if you can limit the buffer
b_th_peak = Buffer(pd.DataFrame({
'peak': [],
'lowerbound': [],
'higherbound': []
}),
length=1000000)
b_th_peak_std = Buffer(pd.DataFrame({
'peak': [],
'lowerbound': [],
'higherbound': []
}),
length=1000000)
s_spikes = Buffer(pd.DataFrame({'y': []}), length=1000000)
s_labels = Buffer(pd.DataFrame({'x': [], 'y': []}), length=1000000)
s_spikes_end = Buffer(pd.DataFrame({'y': []}), length=1000000)
#s_spikes = hv.streams.Stream.define('df', df=pd.DataFrame({'location':[], 'y':[]}))()
#s_labels = hv.streams.Stream.define('df', df=pd.DataFrame({'x':[], 'y':[], 'labels':[]}))()
# Generate dynamic map
plot_peak_b = hv.DynamicMap(
partial(hv.Points, kdims=['index', 'peak']),
streams=[b_th_peak
]).options(width=1000,
finalize_hooks=[apply_formatter
]).redim.label(index='Time in UTC')
# HoloViews seems to currently have a bug with hv.Spread with buffers, once it is fixed
# we can try to implement hv.spread instead of this
plot_peak_std_low = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'lowerbound']),
streams=[b_th_peak
]).options(line_alpha=0.5,
line_color='gray').redim.label(index='Time in UTC')
plot_peak_std_high = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'higherbound']),
streams=[b_th_peak
]).options(line_alpha=0.5,
line_color='gray').redim.label(index='Time in UTC')
#hvSpikes = hv.DynamicMap(gen_spikes, streams=[s_spikes])
#hvLabels = hv.DynamicMap(gen_labels, streams=[s_labels])
hvSpikes = hv.DynamicMap(partial(hv.Spikes),
streams=[s_spikes]).options(apply_ranges=False,
color='green')
hvSpikes_end = hv.DynamicMap(hv.Spikes,
streams=[s_spikes_end
]).options(apply_ranges=False,
color='red')
hvLabels = hv.DynamicMap(partial(hv.Labels, kdims=['x', 'y']),
streams=[s_labels]).options(apply_ranges=False)
#testing = datashade(plot_ipm_b).options(width = 1000)
test1 = datashade(plot_peak_std_low,
streams=[hv.streams.PlotSize],
normalization='linear').options(
width=1000, finalize_hooks=[apply_formatter])
#.opts(norm=dict(framewise=True))
test2 = datashade(plot_peak_std_high,
streams=[hv.streams.PlotSize],
normalization='linear')
#.opts(norm=dict(framewise=True))
pointTest = decimate(plot_peak_b,
streams=[hv.streams.PlotSize],
normalization='linear')
# Scrolling after pausing the graph seems to cause Parameter name clashes for keys: {'height', 'width', 'scale'} error!
#plot = (plot_ipm_b*plot_ipm_std_low*plot_ipm_std_high)
plot = (pointTest * test1 * test2 * hvSpikes * hvLabels * hvSpikes_end)
#plot=hvSpikes*hvLabels
# Use bokeh to render plot
hvplot = renderer.get_plot(plot, doc)
switch_key = 'peak_8'
start = [1534280820000, 1534271880000]
end = [1534279380000, 1534272000000]
labels = ['Test 1', 'Test 2']
medianCheck = 0
# For pushing in data, maybe cut off first 119 points to get rid of those weird extremes
def push_data(stream):
TS_key = switch_key + '_TS'
data = list(peak[switch_key])
timestamp = list(peakTS[TS_key])
times = [1000 * time for time in timestamp]
dataSeries = pd.Series(data, index=times)
zipped = basic_event_builder(peak=dataSeries)
median = zipped.rolling(120, min_periods=1).median()
std = zipped.rolling(120, min_periods=1).std()
lowerbound = median - std
higherbound = median + std
df = pd.DataFrame({
'peak': median['peak'],
'lowerbound': lowerbound['peak'],
'higherbound': higherbound['peak']
})
stream.send(df)
def push_spikes(stream, position):
#start = [1534279200000, 1534201350000]
doubledData = [val for val in position for _ in (0, 1)]
height = [1000, -1000]
heightList = height * len(position)
df = pd.DataFrame({'location': doubledData, 'y': heightList})
df = df.set_index('location')
df.index.name = None
stream.send(df)
def push_labels(stream, labels, start):
nonlocal medianCheck
# Need to clear the buffers somehow to stop weird overlapping
height = [1]
data = list(peak[switch_key])
dataSeries = pd.Series(data)
median = [dataSeries.median()]
if median == medianCheck:
pass
else:
heightList = median * len(start)
df = pd.DataFrame({'x': start, 'y': heightList, 'labels': labels})
df = df.set_index('labels')
df.index.name = None
with util.disable_constant(s_labels):
s_labels.data = s_labels.data.iloc[:0]
stream.send(pd.DataFrame({'x': [], 'y': []}))
stream.send(df)
medianCheck = median
def clear_buffer():
"""
Modified version of hv.buffer.clear() since original appears to be
buggy
Clear buffer/graph whenever switch is toggled
"""
nonlocal b_th_peak, b_th_peak_std, s_labels
with util.disable_constant(b_th_peak) and util.disable_constant(
b_th_peak_std) and util.disable_constant(s_labels):
b_th_peak.data = b_th_peak.data.iloc[:0]
b_th_peak_std.data = b_th_peak_std.data.iloc[:0]
s_labels.data = s_labels.data.iloc[:0]
b_th_peak.send(
pd.DataFrame({
'peak': [],
'lowerbound': [],
'higherbound': []
}))
b_th_peak_std.send(
pd.DataFrame({
'peak': [],
'lowerbound': [],
'higherbound': []
}))
s_labels.send(pd.DataFrame({'x': [], 'y': []}))
def switch(attr, old, new):
"""
Update drop down menu value
"""
nonlocal switch_key
switch_key = select.value
clear_buffer()
print("Yes!")
def play_graph():
"""
Provide play and pause functionality to the graph
"""
nonlocal callback_id_th_b, cb_id_spikes, cb_id_labels, cb_id_spikes_end
if startButton.label == '► Play':
startButton.label = '❚❚ Pause'
callback_id_th_b = doc.add_periodic_callback(
partial(push_data, stream=b_th_peak), 1000)
cb_id_spikes = doc.add_periodic_callback(
partial(push_spikes, stream=s_spikes, position=start), 1000)
cb_id_labels = doc.add_periodic_callback(
partial(push_labels,
stream=s_labels,
labels=labels,
start=start), 1000)
cb_id_spikes_end = doc.add_periodic_callback(
partial(push_spikes, stream=s_spikes_end, position=end), 1000)
else:
startButton.label = '► Play'
doc.remove_periodic_callback(callback_id_th_b)
doc.remove_periodic_callback(cb_id_spikes)
doc.remove_periodic_callback(cb_id_labels)
doc.remove_periodic_callback(cb_id_spikes_end)
select = Select(title="Peak:", value="peak_8", options=list(peak))
select.on_change('value', switch)
startButton = Button(label='❚❚ Pause')
startButton.on_click(play_graph)
callback_id_th_b = doc.add_periodic_callback(
partial(push_data, stream=b_th_peak), 1000)
cb_id_labels = doc.add_periodic_callback(
partial(push_labels, stream=s_labels, labels=labels, start=start),
1000)
cb_id_spikes = doc.add_periodic_callback(
partial(push_spikes, stream=s_spikes, position=start), 1000)
cb_id_spikes_end = doc.add_periodic_callback(
partial(push_spikes, stream=s_spikes_end, position=end), 1000)
plot = column(select, startButton, hvplot.state)
doc.title = "Time History Graphs"
doc.add_root(plot)
def produce_timehistory(doc, ipm2List, ipm3List, ebeamList, ipm2TS, ipm3TS,
ebeamTS):
# Streams
# See if you can limit the buffer
b_th_ipm2 = Buffer(pd.DataFrame({'ipm': []}), length=1200)
b_th_ipm3 = Buffer(pd.DataFrame({'ipm': []}), length=1200)
streamTH2 = hv.streams.Stream.define('df',
df=pd.DataFrame({
'timestamp': [],
'ipm': []
}))()
streamTH3 = hv.streams.Stream.define('df',
df=pd.DataFrame({
'timestamp': [],
'ipm': []
}))()
# Generate dynamic map
# bottom plot
# blue data
plot_ipm2_b = hv.DynamicMap(hv.Scatter, streams=[b_th_ipm2]).options(
width=1001, finalize_hooks=[apply_formatter])
# red data
plot_ipm3_b = hv.DynamicMap(hv.Scatter, streams=[b_th_ipm3]).options(
color='red', finalize_hooks=[apply_formatter])
# top plot
# blue plot
# plot_ipm2 = hv.DynamicMap(
# gen_timehistory,
# streams=[streamTH2]).options(
# width=1000, finalize_hooks=[apply_formatter]).redim.label(
# index='test')
# # red plot
# plot_ipm3 = hv.DynamicMap(
# gen_timehistory,
# streams=[streamTH3]).options(
# color='red', finalize_hooks=[apply_formatter]).redim.label(
# index='test')
#plot = (plot_ipm2*plot_ipm3)# + plot_ipm2_b*plot_ipm3_b).cols(1)
plot = plot_ipm2_b * plot_ipm3_b
# Use bokeh to render plot
hvplot = renderer.get_plot(plot, doc)
callback_id_th2 = None
callback_id_th3 = None
def push_data(ipm, timestamp, stream):
"""
Push data into stream to be ploted on hextiles plot
"""
ipm_plot = list(ipm)
timestamp_plot = list(timestamp)
times = [1000 * time for time in timestamp_plot]
ipmData = pd.Series(ipm_plot, index=times)
zipped = basic_event_builder(ipm=ipmData)
median = zipped.rolling(120, min_periods=1).median()
if type(stream) == hv.streams.Buffer:
if len(median) > 1000:
counter = 0
pos = 0
send = pd.DataFrame({'ipm': []})
while counter < 1000:
divide = len(median) / 1000.
test = int(pos)
send = send.append(median.iloc[[test]])
#print(send)
#print(type(median.iloc[[test]]))
pos += divide
counter += 1
#print(len(send))
#print(len(stream.data))
stream.send(send)
#print(stream.data[-10:])
print("Done")
else:
stream.send(median)
#stream.send(median)
elif len(median) > 100:
stream.event(df=median)
#print(len(stream.data))
#stream.send(median)
#stream.event(df=median)
#print(type(stream))
#print(len(median))
# ipmData = pd.DataFrame({'timestamp': times, 'ipm': ipm_plot})
# stream.event(df=ipmData)
# callback_id_th2 = doc.add_periodic_callback(
# partial(push_data, ipm=ipm2List, timestamp=ipm2TS, stream=streamTH2),
# 1000)
# callback_id_th3 = doc.add_periodic_callback(
# partial(push_data, ipm=ipm3List, timestamp=ipm3TS, stream=streamTH3),
# 1000)
callback_id_th2_b = doc.add_periodic_callback(
partial(push_data, ipm=ipm2List, timestamp=ipm2TS, stream=b_th_ipm2),
1000)
callback_id_th3_b = doc.add_periodic_callback(
partial(push_data, ipm=ipm3List, timestamp=ipm3TS, stream=b_th_ipm3),
1000)
plot = hvplot.state
doc.title = "Time History Graphs"
doc.add_root(plot)
def produce_graphs(doc, timetool_d, timetool_t, ipm2_d, ipm2_t, ipm3_d,
ipm3_t):
switch_key = 'ipm2'
b_scatter = Buffer(pd.DataFrame({'timetool_data': []}), length=40000)
b_IpmAmp = Buffer(pd.DataFrame({'timetool': [], 'ipm': []}), length=1000)
b_timehistory = Buffer(pd.DataFrame({
'time': [],
'correlation': []
}),
length=40000)
hvScatter = hv.DynamicMap(partial(hv.Points), streams=[b_scatter]).options(
width=1000, finalize_hooks=[apply_formatter],
xrotation=45).redim.label(index='Time in UTC')
hvIpmAmp = hv.DynamicMap(
partial(hv.Scatter, kdims=['timetool', 'ipm']),
streams=[b_IpmAmp]).options(width=500).redim.label(time='Time in UTC')
hvTimeHistory = hv.DynamicMap(
partial(hv.Scatter, kdims=['time', 'correlation']),
streams=[b_timehistory
]).options(width=500,
finalize_hooks=[apply_formatter],
xrotation=45).redim.label(time='Time in UTC')
layout = (hvIpmAmp + hvTimeHistory + hvScatter).cols(2)
hvplot = renderer.get_plot(layout, doc)
cb_id_scatter = None
cb_id_amp_ipm = None
cb_id_timehistory = None
def push_data_scatter(timetool_d, timetool_t, buffer):
timeStuff = list(timetool_t)
# Convert time to seconds so bokeh formatter can get correct datetime
times = [1000 * time for time in timeStuff]
edgePos = []
for array in timetool_d:
edgePos.append(array[1])
edgePos_data = pd.Series(edgePos, index=times)
zipped = basic_event_builder(timetool_data=edgePos_data)
buffer.send(zipped)
def push_data_amp_ipm(timetool_d, timetool_t, ipm2_d, ipm2_t, ipm3_d,
ipm3_t, buffer):
timeStuff = list(timetool_t)
# Convert time to seconds so bokeh formatter can get correct datetime
times = [1000 * time for time in timeStuff]
tt_d = list(timetool_d)
tt_t = list(timetool_t)
i2_d = list(ipm2_d)
i2_t = list(ipm2_t)
i3_d = list(ipm3_d)
i3_t = list(ipm3_t)
ipmValue = i2_d
ipmTime = i2_t
if switch_key == 'ipm2':
ipmValue = i2_d
ipmTime = i2_t
elif switch_key == 'ipm3':
ipmValue = i3_d
ipmTime = i3_t
if len(tt_d) > len(ipmValue):
tt_d = tt_d[:len(ipmValue)]
tt_t = tt_t[:len(ipmValue)]
elif len(tt_d) < len(ipmValue):
ipmValue = ipmValue[:len(tt_d)]
ipmTime = ipmTime[:len(tt_d)]
edgeAmp = []
for array in tt_d:
edgeAmp.append(array[2])
data = pd.DataFrame({'timetool': edgeAmp, 'ipm': ipmValue})
buffer.send(data)
def push_data_correlation_time_history(timetool_d, timetool_t, ipm2_d,
ipm2_t, ipm3_d, ipm3_t, buffer):
timeStuff = list(timetool_t)
# Convert time to seconds so bokeh formatter can get correct datetime
times = [1000 * time for time in timeStuff]
tt_d = list(timetool_d)
tt_t = list(timetool_t)
i2_d = list(ipm2_d)
i2_t = list(ipm2_t)
i3_d = list(ipm3_d)
i3_t = list(ipm3_t)
ipmValue = i2_d
ipmTime = i2_t
if switch_key == 'ipm2':
ipmValue = i2_d
ipmTime = i2_t
elif switch_key == 'ipm3':
ipmValue = i3_d
ipmTime = i3_t
if len(tt_d) > len(ipmValue):
tt_d = tt_d[:len(ipmValue)]
tt_t = tt_t[:len(ipmValue)]
elif len(tt_d) < len(ipmValue):
ipmValue = ipmValue[:len(tt_d)]
ipmTime = ipmTime[:len(tt_d)]
edgeAmp = []
for array in tt_d:
edgeAmp.append(array[2])
data = pd.DataFrame({'timetool': edgeAmp, 'ipm': ipmValue})
data_list = data['timetool'].rolling(window=120).corr(
other=data['ipm'])
final_df = pd.DataFrame({
'time': times[119:],
'correlation': data_list[119:]
})
#print(final_df)
buffer.send(final_df)
def switch(attr, old, new):
"""
Update drop down menu value
"""
nonlocal switch_key, b_timehistory, b_IpmAmp
switch_key = select.value
b_timehistory.clear()
b_IpmAmp.clear()
print(switch_key)
select = Select(title='ipm value:', value='ipm2', options=['ipm2', 'ipm3'])
select.on_change('value', switch)
cb_id_scatter = doc.add_periodic_callback(
partial(push_data_scatter,
timetool_d=timetool_d,
timetool_t=timetool_t,
buffer=b_scatter), 1000)
cb_id_amp_ipm = doc.add_periodic_callback(
partial(push_data_amp_ipm,
timetool_d=timetool_d,
timetool_t=timetool_t,
ipm2_d=ipm2_d,
ipm2_t=ipm2_t,
ipm3_d=ipm3_d,
ipm3_t=ipm3_t,
buffer=b_IpmAmp), 1000)
cb_id_timehistory = doc.add_periodic_callback(
partial(push_data_correlation_time_history,
timetool_d=timetool_d,
timetool_t=timetool_t,
ipm2_d=ipm2_d,
ipm2_t=ipm2_t,
ipm3_d=ipm3_d,
ipm3_t=ipm3_t,
buffer=b_timehistory), 1000)
plot = column(select, hvplot.state)
doc.add_root(plot)
('MaxInt', '@MaxInt'), ('StdInt', '@StdInt')]
############### intensity box plot setup using holoviews in bokeh
renderer = hv.renderer('bokeh')
def boxInts(data):
p1hv = hv.BoxWhisker(data, ['clsLabel'],
'meanInt',
label="mean intensity per cluster").sort()
p1hv.opts(show_legend=False, width=500, cmap='Set1', ylim=(-1, 40))
# p1hv.opts(show_legend=False, width=500)
return p1hv
mem_stream = Buffer(sample)
p1hv_dmap = hv.DynamicMap(boxInts, kdims=[], streams=[mem_stream])
p1 = hv.render(p1hv_dmap)
# p1 = renderer(p1hv_dmap)
################################## plot for umap and transfetection color grouping
p = figure(tools="tap,reset", tooltips=hover.tooltips)
p.circle(x='one',
y='two',
source=source,
size=2,
color='color',
legend='Label')
p.title.text = 'UMAP - Applied on WT and Mutant transfected and untrasfected single cells'
p.xaxis.axis_label = 'one'
def get_stream(data):
return Buffer(data, length=buffer_length, index=index)
def produce_timehistory(doc, ipm2List, ipm3List, ebeamList, ipm2TS, ipm3TS,
ebeamTS):
# Streams
# See if you can limit the buffer
b_th_ipm = Buffer(pd.DataFrame({'ipm': []}), length=1200)
b_th_ipm_std = Buffer(pd.DataFrame({
'lowerbound': [],
'higherbound': []
}),
length=1000000)
switch_key = 'ipm2'
# Generate dynamic map
plot_ipm_b = hv.DynamicMap(hv.Points, streams=[b_th_ipm]).options(
width=1000,
finalize_hooks=[apply_formatter]).redim.label(index='Time in UTC',
ipm='ipm data')
# HoloViews seems to currently have a bug with hv.Spread with buffers, once it is fixed
# we can try to implement hv.spread instead of this
plot_ipm_std_low = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'lowerbound']),
streams=[b_th_ipm_std
]).options(line_alpha=0.5,
line_color='gray').redim.label(index='Time in UTC')
plot_ipm_std_high = hv.DynamicMap(
partial(hv.Curve, kdims=['index', 'higherbound']),
streams=[b_th_ipm_std
]).options(line_alpha=0.5,
line_color='gray').redim.label(index='Time in UTC')
#testing = datashade(plot_ipm_b).options(width = 1000)
test1 = datashade(plot_ipm_std_low,
streams=[hv.streams.PlotSize],
normalization='linear')
#.opts(norm=dict(framewise=True))
test2 = datashade(plot_ipm_std_high,
streams=[hv.streams.PlotSize],
normalization='linear')
#.opts(norm=dict(framewise=True))
# Scrolling after pausing the graph seems to cause Parameter name clashes for keys: {'height', 'width', 'scale'} error!
#plot = (plot_ipm_b*plot_ipm_std_low*plot_ipm_std_high)
plot = (plot_ipm_b * test1 * test2)
# Use bokeh to render plot
hvplot = renderer.get_plot(plot, doc)
# For pushing in data, maybe cut off first 119 points to get rid of those weird extremes
def push_data(stream):
if switch_key == 'ipm2':
ipm_plot = list(ipm2List)
timestamp_plot = list(ipm2TS)
elif switch_key == 'ipm3':
ipm_plot = list(ipm3List)
timestamp_plot = list(ipm3TS)
times = [1000 * time for time in timestamp_plot]
ipmData = pd.Series(ipm_plot, index=times)
zipped = basic_event_builder(ipm=ipmData)
median = zipped.rolling(120, min_periods=1).median()
#This might be making it take a long time to switch
if type(stream) == hv.streams.Buffer:
if len(median) > 1000:
counter = 0
pos = 0
send = pd.DataFrame({'ipm': []})
while counter < 1000:
divide = len(median) / 1000.
test = int(pos)
send = send.append(median.iloc[[test]])
#print(send)
#print(type(median.iloc[[test]]))
pos += divide
counter += 1
#print(len(send))
#print(len(stream.data))
stream.send(send)
#print(stream.data[-10:])
print("Done")
else:
stream.send(median)
#stream.send(median)
elif len(median) > 100:
stream.event(df=median)
def push_std(stream):
if switch_key == 'ipm2':
ipm_plot = list(ipm2List)
timestamp_plot = list(ipm2TS)
elif switch_key == 'ipm3':
ipm_plot = list(ipm3List)
timestamp_plot = list(ipm3TS)
times = [1000 * time for time in timestamp_plot]
ipmData = pd.Series(ipm_plot, index=times)
zipped = basic_event_builder(ipm=ipmData)
median = zipped.rolling(120, min_periods=1).median()
std = zipped.rolling(120, min_periods=1).std()
lowerbound = median - std
higherbound = median + std
df = pd.DataFrame({
'lowerbound': lowerbound['ipm'],
'higherbound': higherbound['ipm']
})
stream.send(df)
# if len(df) > 1000:
# counter = 0
# pos = 0
# send = pd.DataFrame({'lowerbound':lowerbound['ipm'], 'higherbound':higherbound['ipm']})
# while counter < 1000:
# divide = len(df)/1000.
# test = int(pos)
# send = send.append(df.iloc[[test]])
# pos += divide
# counter += 1
# #print(divide)
# stream.send(send)
# else:
# stream.send(df)
def clear_buffer():
"""
Modified version of hv.buffer.clear() since original appears to be
buggy
Clear buffer/graph whenever switch is toggled
"""
nonlocal b_th_ipm, b_th_ipm_std
with util.disable_constant(b_th_ipm) and util.disable_constant(
b_th_ipm_std):
b_th_ipm.data = b_th_ipm.data.iloc[:0]
b_th_ipm_std.data = b_th_ipm_std.data.iloc[:0]
b_th_ipm.send(pd.DataFrame({'ipm': []}))
b_th_ipm_std.send(pd.DataFrame({
'lowerbound': [],
'higherbound': []
}), )
def switch(attr, old, new):
"""
Update drop down menu value
"""
nonlocal switch_key
switch_key = select.value
clear_buffer()
print("Yes!")
def play_graph():
"""
Provide play and pause functionality to the graph
"""
nonlocal callback_id_th_b, cb_id_low_std_th, cb_id_high_std_th
if startButton.label == '► Play':
startButton.label = '❚❚ Pause'
callback_id_th_b = doc.add_periodic_callback(
partial(push_data, stream=b_th_ipm), 1000)
cb_id_low_std_th = doc.add_periodic_callback(
partial(push_std, stream=b_th_ipm_std), 1000)
cb_id_high_std_th = doc.add_periodic_callback(
partial(push_std, stream=b_th_ipm_std), 1000)
else:
startButton.label = '► Play'
doc.remove_periodic_callback(callback_id_th_b)
doc.remove_periodic_callback(cb_id_low_std_th)
doc.remove_periodic_callback(cb_id_high_std_th)
select = Select(title="ipm value:", value="ipm2", options=["ipm2", "ipm3"])
select.on_change('value', switch)
startButton = Button(label='❚❚ Pause')
startButton.on_click(play_graph)
callback_id_th_b = doc.add_periodic_callback(
partial(push_data, stream=b_th_ipm), 1000)
cb_id_low_std_th = doc.add_periodic_callback(
partial(push_std, stream=b_th_ipm_std), 1000)
cb_id_high_std_th = doc.add_periodic_callback(
partial(push_std, stream=b_th_ipm_std), 1000)
plot = column(startButton, select, hvplot.state)
doc.title = "Time History Graphs"
doc.add_root(plot)
ebeamStuff = actualData['ebeam'].astype(float)
ipm2Stuff = actualData['ipm2'].astype(float)
@gen.coroutine
def f():
global index
if index < len(actualData.index) + 100:
dfstream.send(pd.DataFrame({'ebeam':ebeamStuff[index:index+100],
'ipm2':ipm2Stuff[index:index+100]}))
index += 100
else:
#cb.stop()
# Needs to stop loop
# Starting data set
testData = pd.DataFrame({'ebeam':ebeamL3[:100],'ipm2':ipm2[:100]})
# Filtering starting data set
testData = testData[testData["ipm2"] < a]
testData = testData[testData["ebeam"] < b]
testData = testData[testData["ipm2"] > c]
testData = testData[testData["ebeam"] > d]
dfstream = Buffer(testData, length=100000, index=False) # Length is long to keep previous points plotted
index = 100
PeriodicCallback(f, 1).start()
doc = renderer.server_doc(hv.DynamicMap(hv.Points, streams=[dfstream]))
doc.title = 'HoloViews App'
