class DateRangeWidget:
def __init__(self, min_date: date, max_date: date):
self._min_date = min_date
self._max_date = max_date
self.slider = DateRangeSlider(
title="Time Period",
start=self._min_date,
end=self._max_date,
value=(self._min_date, self._max_date),
step=int(timedelta(days=1).total_seconds()) * 1000,
sizing_mode="stretch_width",
)
self.widget = row(self.slider)
def on_change(self, *callbacks: Callable[[str, object, object],
None]) -> None:
self.slider.on_change("value_throttled", *callbacks)
def set_enabled(self, enabled: bool) -> None:
self.slider.disabled = not enabled
@property
def min_and_max_date(self) -> Tuple[date, date]:
return self.slider.value_as_date
def modify_doc(doc):
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1], val=["a", "b"]))
plot = Plot(height=400,
width=400,
x_range=Range1d(0, 1),
y_range=Range1d(0, 1),
min_border=0)
plot.add_glyph(source, Circle(x='x', y='y', size=20))
plot.add_tools(
CustomAction(callback=CustomJS(args=dict(s=source),
code=RECORD("data", "s.data"))))
slider = DateRangeSlider(start=start,
end=end,
value=value,
css_classes=["foo"],
width=300)
def cb(attr, old, new):
source.data['val'] = [
slider.value_as_date[0].isoformat(),
slider.value_as_date[1].isoformat()
]
slider.on_change('value', cb)
doc.add_root(column(slider, plot))
def _timeseries_widgets(col_1, col_2, col_max, col_min, n_bins, aggregate,
callback):
col_1_time = pd.to_datetime(col_1)
if col_max is None:
col_max = col_1_time.max()
if col_min is None:
col_min = col_1_time.min()
slider = Slider(start=1, end=100, value=n_bins, step=1, title="Bins")
slider.on_change('value', callback)
range_select = DateRangeSlider(start=col_1_time.min(),
end=col_1_time.max(),
value=(col_min, col_max),
step=1,
title='Range',
format='%R %F')
range_select.on_change('value', callback)
dropdown = Dropdown(value=aggregate,
label=aggregate,
button_type="default",
menu=[('mean', 'mean'), ('count', 'count'),
('sum', 'sum'), ('max', 'max'), ('min', 'min')])
dropdown.on_change('value', callback)
return slider, range_select, dropdown
def modify_doc(doc):
plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
slider = DateRangeSlider(start=start, end=end, value=value, css_classes=["foo"], width=300, bar_color="red")
def cb(attr, old, new):
slider.bar_color = "rgba(255, 255, 0, 1)"
slider.on_change('value', cb)
doc.add_root(column(slider, plot))
def plot_with_slider(dataframe, country_name, y_axis_name):
""""
this function takes a dataframe, y-axis name and country as a parameter,
creates a plot with a slider and returns the plot.
"""
# create a figure object with width and height
plot = figure(x_axis_type="datetime",
width=1000,
height=400,
sizing_mode='fixed')
# creating columnDataSource object, for the dataframes
source = ColumnDataSource(dataframe)
# initialize the min and max value of the date
init_value = (dataframe['Date'].min(), dataframe['Date'].max())
# configuring date range slider with start date end date and value
date_range_slider = DateRangeSlider(start=init_value[0],
end=init_value[1],
value=init_value)
date_filter = BooleanFilter(booleans=[True] * dataframe.shape[0])
# not use scientific numbers on Y-axis
plot.yaxis.formatter = BasicTickFormatter(use_scientific=False)
# whenever a slider value updates. The date range sliders, value changes.
date_range_slider.js_on_change(
"value",
CustomJS(args=dict(f=date_filter, cases_source=source),
code="""\
const [start, end] = cb_obj.value;
f.booleans = Array.from(cases_source.data['Date']).map(d => (d >= start
&& d <= end));
// Needed because of https://github.com/bokeh/bokeh/issues/7273
cases_source.change.emit();
"""))
# add a circle renderer using the source's two columns.
plot.circle(x='Date',
y=country_name,
source=source,
view=CDSView(source=source, filters=[date_filter]),
color='Pink',
line_width=0.5)
# name and field pairs for the Hover tool
tooltips = [('Date', '@Date{%F}'), (country_name, "$y{int}")]
# formatting scheme of date column
formatters = {'@Date': 'datetime'}
# create a Hover tool for the figure with the tooltips and specify the formatting scheme
plot.add_tools(
HoverTool(tooltips=tooltips, formatters=formatters, mode='vline'))
plot.title.text_color = "midnightblue"
plot.title.text_font_size = "25px"
plot.toolbar.active_drag = None
plot.toolbar_location = None
plot.xaxis.axis_label = 'Date'
plot.yaxis.axis_label = y_axis_name
return column(plot, date_range_slider)
def __init__(self, min_date: date, max_date: date):
self._min_date = min_date
self._max_date = max_date
self.slider = DateRangeSlider(
title="Time Period",
start=self._min_date,
end=self._max_date,
value=(self._min_date, self._max_date),
step=int(timedelta(days=1).total_seconds()) * 1000,
sizing_mode="stretch_width",
)
self.widget = row(self.slider)
def test_js_on_change_executes(self, bokeh_model_page) -> None:
slider = DateRangeSlider(start=start, end=end, value=value, css_classes=["foo"], width=300)
slider.js_on_change('value', CustomJS(code=RECORD("value", "cb_obj.value")))
page = bokeh_model_page(slider)
drag_range_slider(page.driver, ".foo", "lower", 50)
results = page.results
assert datetime.fromtimestamp(results['value'][0]/1000) > datetime(*date.fromisoformat("2017-08-04").timetuple()[:3])
drag_range_slider(page.driver, ".foo", "upper", -70)
assert datetime.fromtimestamp(results['value'][1]/1000) < datetime(*date.fromisoformat("2017-08-09").timetuple()[:3])
assert page.has_no_console_errors()
def test_server_bar_color_updates(
self, bokeh_server_page: BokehServerPage) -> None:
slider = DateRangeSlider(start=start,
end=end,
value=value,
width=300,
bar_color="red")
def modify_doc(doc):
plot = Plot(height=400,
width=400,
x_range=Range1d(0, 1),
y_range=Range1d(0, 1),
min_border=0)
def cb(attr, old, new):
slider.bar_color = "rgba(255, 255, 0, 1)"
slider.on_change('value', cb)
doc.add_root(column(slider, plot))
page = bokeh_server_page(modify_doc)
drag_range_slider(page.driver, slider, "lower", 150)
sleep(1) # noUiSlider does a transition that takes some time
assert get_slider_bar_color(page.driver,
slider) == "rgba(255, 255, 0, 1)"
def test_display(self, bokeh_model_page: BokehModelPage) -> None:
slider = DateRangeSlider(start=start, end=end, value=value, width=300)
page = bokeh_model_page(slider)
children = find_elements_for(page.driver, slider,
"div.bk-input-group > div")
assert len(children) == 2
assert page.has_no_console_errors()
def test_display(self, bokeh_model_page) -> None:
slider = DateRangeSlider(start=start, end=end, value=value, css_classes=["foo"], width=300)
page = bokeh_model_page(slider)
el = page.driver.find_element_by_css_selector('.foo')
children = el.find_elements_by_css_selector('div.bk-input-group > div')
assert len(children) == 2
assert page.has_no_console_errors()
def test_displays_bar_color(self, bokeh_model_page) -> None:
slider = DateRangeSlider(start=start, end=end, value=value, css_classes=["foo"], width=300, bar_color="red")
page = bokeh_model_page(slider)
el = page.driver.find_element_by_css_selector('.foo')
assert len(el.find_elements_by_css_selector('div.bk-input-group > div')) == 2
assert get_slider_bar_color(page.driver, ".foo") == "rgba(255, 0, 0, 1)"
assert page.has_no_console_errors()
def test_displays_title(self, bokeh_model_page) -> None:
slider = DateRangeSlider(start=start, end=end, value=value, css_classes=["foo"], width=300)
page = bokeh_model_page(slider)
el = page.driver.find_element_by_css_selector('.foo')
assert len(el.find_elements_by_css_selector('div.bk-input-group > div')) == 2
assert get_slider_title_text(page.driver, ".foo") == '04 Aug 2017 .. 09 Aug 2017'
assert get_slider_title_value(page.driver, ".foo") == '04 Aug 2017 .. 09 Aug 2017'
assert page.has_no_console_errors()
def test_title_updates(self, bokeh_model_page) -> None:
slider = DateRangeSlider(start=start, end=end, value=value, css_classes=["foo"], width=300)
page = bokeh_model_page(slider)
assert get_slider_title_value(page.driver, ".foo") == "04 Aug 2017 .. 09 Aug 2017"
drag_range_slider(page.driver, ".foo", "lower", 50)
val = get_slider_title_value(page.driver, ".foo").split(" .. ")[0]
assert val > "04 Aug 2017"
drag_range_slider(page.driver, ".foo", "lower", -70)
val = get_slider_title_value(page.driver, ".foo").split(" .. ")[0]
assert val == "03 Aug 2017"
assert page.has_no_console_errors()
def test_server_on_change_round_trip(
self, bokeh_server_page: BokehServerPage) -> None:
slider = DateRangeSlider(start=start, end=end, value=value, width=300)
def modify_doc(doc):
source = ColumnDataSource(dict(x=[1, 2], y=[1, 1], val=["a", "b"]))
plot = Plot(height=400,
width=400,
x_range=Range1d(0, 1),
y_range=Range1d(0, 1),
min_border=0)
plot.add_glyph(source, Circle(x='x', y='y', size=20))
plot.tags.append(
CustomJS(name="custom-action",
args=dict(s=source),
code=RECORD("data", "s.data")))
def cb(attr, old, new):
source.data['val'] = [
slider.value_as_date[0].isoformat(),
slider.value_as_date[1].isoformat()
]
slider.on_change('value', cb)
doc.add_root(column(slider, plot))
page = bokeh_server_page(modify_doc)
drag_range_slider(page.driver, slider, "lower", 50)
page.eval_custom_action()
results = page.results
new = results['data']['val']
assert new[0] > '2017-08-04'
drag_range_slider(page.driver, slider, "upper", -50)
page.eval_custom_action()
results = page.results
new = results['data']['val']
assert new[1] < '2017-08-09'
inc = df.close > df.open
dec = df.open > df.close
w = 12 * 60 * 60 * 1000 # half day in ms
TOOLS = "pan,wheel_zoom,box_zoom,reset,save"
p = figure(x_axis_type="datetime",
tools=TOOLS,
plot_width=1000,
title="MSFT Candlestick")
p.xaxis.major_label_orientation = pi / 4
p.grid.grid_line_alpha = 0.3
range_slider = DateRangeSlider(start=df['shortened_date'].min(),
end=df['shortened_date'].max(),
value=(date(2016, 2, 3), date(2016, 3, 3)),
step=1,
title="Date Range")
# def candle_plot(stock_range: Tuple[float,float], df : pd.DataFrame = df_apple):
# df = df_apple[]
p.segment('shortened_date',
'high',
'shortened_date',
'low',
color="black",
source=data)
# p.vbar(data.shortened_date[inc], w, data.open[inc], data.close[inc], fill_color="#D5E1DD", line_color="black")
# p.vbar(data.shortened_date[dec], w, data.open[dec], data.close[dec], fill_color="#F2583E", line_color="black")
def callback(attr, old, new):
def bokeh_plots_by_lemma(lemma, df=df):
source = ColumnDataSource(df[colsbylemma(lemma)])
hover = HoverTool()
# WIDGET pickup
global dr_sl, sl
global dr_sl_callback, sl_callback
dr_sl = DateRangeSlider(value=(date(2021,1,1),date(2021,1,20)),
start=date(2020, 4, 20), end=datetime.today()) #start_date, end_date
dr_sl_callback = CustomJS(args=dict(source=source, data=df.reset_index().to_dict()),
code="""
console.log(data)
data.change.emit()
""")
dr_sl.js_on_change("value",dr_sl_callback)
sl = Select(title="Option:", value=lemma, options=list(L.keys()))
#sl.on_change("value",sl_callback)
# THEME, OUTPUT
curdoc().theme = "night_sky" # caliber, dark_minimal, light_minimal, night_sky, contrast
output_file("/tmp/output_bokeh.html", title="Output Bokeh")
# figure PARAMETERS
bokeh_figure_params = dict(tools="pan,tap,box_zoom,reset,save", title="(lemma:) {}".format(lemma.upper()),
plot_height=650,plot_width=1300,
x_axis_type="datetime",
sizing_mode="scale_both",
outline_line_color="navy", outline_line_width = 7, outline_line_alpha = 0.3)
p = figure(output_backend="canvas", **bokeh_figure_params)
p.xaxis.axis_label, p.yaxis.axis_label = ("Date", "Cases")
# outer PARAMETERS
selected_circle = Circle(fill_alpha=1,fill_color="red")
nonselected_circle = Circle(fill_alpha=0.1, fill_color="blue",line_color="firebrick")
bokeh_plotparams= dict(x="date", y=None, source=source, muted_alpha=0.1)
it_legend = []
# RUN!
for field in colsbylemma(lemma):
color_i = Category20_20[randint(0,19)]
bokeh_plotparams.update(dict(y=field,
muted_color=color_i))
# line (renderer)
rl = p.line(name="line",
line_color=color_i, line_alpha=0.4, line_cap="round",
**bokeh_plotparams)
# circle (renderer)
rcirc = p.circle(name="circle", size=10,
fill_color=color_i, fill_alpha=0.4,
**bokeh_plotparams)
rcirc.selection_glyph = selected_circle
rcirc.nonselection_glyph = nonselected_circle
it_legend.append(LegendItem(label=field, renderers=[rl,rcirc]))
# LEGEND
global leg
leg_kwargs = {"background_fill_color":"#33ee11",
"title":"Legend:",
"title_text_color":"#00aaff",
"title_text_font_size":"0.5cm",
"title_text_font_style":"bold",
"label_text_font":"Roboto",
"click_policy" : "mute"}
leg = Legend(items=it_legend, **leg_kwargs)
p.add_layout(leg, place="right")
p.legend.glyph_height, p.legend.glyph_width = (10, 10)
# SHOW
plotall = True
if plotall:
all_things_together = (dr_sl,sl,p)
show(column(*all_things_together), **dict(browser=None, notebook_handle=False))
else:
show(p, **dict(browser=None, notebook_handle=False)) # only the plot
gc.collect()
return p
def direction_plot(dataframe: object, ti: str) -> object:
"""
:param dataframe: pandas dataframe
:param ti: string defining 'week', 'month'...
:return:
"""
# use data in percent => transform db_data to percent
df = (dataframe.transpose().iloc[2:, 0:] / dataframe['sum']) * 100
df.columns = dataframe['tstamp']
db_datadictstr = {
str(int(time.mktime(item.timetuple()) * 1000)): list(df[item])
for item in df.columns
}
maxlist = df.max(1)
hist = df.mean(1)
# maxhist = sorted(maxlist)[-3]
maxhist = mean(
maxlist
) * 0.8 # don't use real max of dataset, too many discordant values
sumhist = sum(hist)
start = [-radians((i * 10) - 85) for i in list(range(0, 36))]
end = [-radians((i * 10) - 75) for i in list(range(0, 36))]
pdstart = [-radians((i * 10) - 95) for i in list(range(0, 36))]
pdend = start
# pdend = [-radians((i * 10) - 85) for i in list(range(0, 36))]
labeltext = ("Selection of " + ti + "ly histograms")
titletext = (ti + 'ly median and sum of all histograms').capitalize()
# need two different sources for callback in Bokeh
pdsource = ColumnDataSource(
data=dict(radius=hist, start=pdstart, end=pdend))
jssource = ColumnDataSource(data=db_datadictstr)
mainplot = figure(title=titletext,
plot_width=400,
plot_height=400,
x_axis_type=None,
y_axis_type=None,
tools="save",
min_border=0,
outline_line_color=None)
mainplot.title.text_font_size = "14pt"
# simple rose plot
mainplot.wedge(radius=hist,
start_angle=start,
end_angle=end,
x=0,
y=0,
direction='clock',
line_color='blue',
fill_color='lightblue',
alpha=0.5,
legend_label='Whole dataset')
# plot connected to slider
mainplot.wedge(radius='radius',
start_angle='start',
end_angle='end',
source=pdsource,
x=0,
y=0,
alpha=0.5,
direction='clock',
line_color='darkred',
fill_color='lightsalmon',
legend_label=labeltext)
# create slider
day = 1000 * 3600 * 24
stepsize = day
if ti == 'week':
stepsize = day
elif ti == 'month':
stepsize = 7 * day
elif ti == 'year':
stepsize = 30 * day
slider = DateSlider(start=min(df.columns),
end=max(df.columns),
value=min(df.columns),
step=stepsize,
title="date within histogram")
callback = CustomJS(args=dict(source=pdsource, data=jssource, slid=slider),
code="""
const S = slid.value;
let radius = source.data.radius;
const radii = Object.keys(data.data)
let slidestop = radii.reduce(function(prev, curr) {
(Math.abs(curr - S) < Math.abs(prev - S) ? curr : prev)
return (Math.abs(curr - S) < Math.abs(prev - S) ? curr : prev);
});
source.data.radius = data.data[slidestop]
source.change.emit();
""")
slider.js_on_change('value', callback)
# create range slider
rslider = DateRangeSlider(start=min(df.columns),
end=max(df.columns),
value=(min(df.columns), max(df.columns)),
step=stepsize,
title="Data within date range from ")
rcallback = CustomJS(args=dict(source=pdsource,
data=jssource,
rslid=rslider),
code="""
const smin = rslid.value[0]
const smax = rslid.value[1]
let radius = source.data.radius;
const radii = Object.keys(data.data)
let lstop = radii.reduce(function(prev, curr) {
return (Math.abs(curr - smin) < Math.abs(prev - smin) ? curr : prev);
});
let rstop = radii.reduceRight(function(prev, curr) {
return (Math.abs(curr - smax) < Math.abs(prev - smax) ? curr : prev);
});
let keylist = [];
for (let k in data.data) keylist.push(k);
let fromkey = keylist.indexOf(lstop);
let tokey = keylist.indexOf(rstop);
let rangekeys = keylist.slice(fromkey, tokey)
var dataavg = Array(36).fill(0)
var count = 0;
for (let k of rangekeys) {
dataavg = dataavg.map(function (num, idx) {return num + data.data[k][idx];});
count += 1
}
dataavg = dataavg.map(function (num, idx) {return num/count;});
source.data.radius = dataavg;
source.change.emit();
""")
rslider.js_on_change('value', rcallback)
# create grid
rund_perc = ceil(maxhist / sumhist * 100)
labels = list(range(0, rund_perc, 2))
labels.append(rund_perc)
rad_pos = [i * sumhist / 100 for i in labels]
out_rim = rad_pos[-1]
label_pos = [sqrt(((i - 1)**2) / 2) for i in rad_pos]
mainplot.text(label_pos[1:],
label_pos[1:], [str(r) + ' %' for r in labels[1:]],
text_font_size="10pt",
text_align="left",
text_baseline="top")
for rad in rad_pos:
mainplot.circle(x=0,
y=0,
radius=rad,
fill_color=None,
line_color='grey',
line_width=0.5,
line_alpha=0.8)
diagonal = sqrt((out_rim**2) / 2)
mainplot.multi_line(xs=[[diagonal, -diagonal], [-diagonal, diagonal],
[-out_rim, out_rim], [0, 0]],
ys=[[diagonal, -diagonal], [diagonal, -diagonal],
[0, 0], [-out_rim, out_rim]],
line_color="grey",
line_width=0.5,
line_alpha=0.8)
mainplot.x_range = Range1d(-out_rim * 1.1, out_rim * 1.1)
mainplot.y_range = Range1d(-out_rim * 1.1, out_rim * 1.1)
mainplot.legend.location = "top_left"
mainplot.legend.click_policy = "hide"
# plot bars for the number of values in each group as secondary 'by' plot
mapper = linear_cmap(field_name='count',
palette=Oranges9,
low=0,
high=max(dataframe['sum']))
bin_width = df.columns[0] - df.columns[1]
source = ColumnDataSource({
'date': dataframe['tstamp'],
'count': dataframe['sum']
})
distriplot = distribution_plot(source, mapper, bin_width,
'Number of values per cluster', 400)
script, div = components(column(distriplot, mainplot, slider, rslider),
wrap_script=False)
return {'div': div, 'script': script}
text_area = TextAreaInput(placeholder="Enter text ...", cols=20, rows=10, value="uuu")
select = Select(options=["Option 1", "Option 2", "Option 3"])
multi_select = MultiSelect(options=["Option %d" % (i+1) for i in range(16)], size=6)
multi_choice = MultiChoice(options=["Option %d" % (i+1) for i in range(16)])
slider = Slider(value=10, start=0, end=100, step=0.5)
range_slider = RangeSlider(value=[10, 90], start=0, end=100, step=0.5)
date_slider = DateSlider(value=date(2016, 1, 1), start=date(2015, 1, 1), end=date(2017, 12, 31))
date_range_slider = DateRangeSlider(value=(date(2016, 1, 1), date(2016, 12, 31)), start=date(2015, 1, 1), end=date(2017, 12, 31))
spinner = Spinner(value=100)
color_picker = ColorPicker(color="red", title="Choose color:")
date_picker = DatePicker(value=date(2017, 8, 1))
paragraph = Paragraph(text="some text")
div = Div(text="some <b>text</b>")
pre_text = PreText(text="some text")
def mk_tab(color):
plot = figure(width=300, height=300)
color='success',
total=f"{sum(df_recovered.iloc[-1, 1:]):n}"),
width_policy="max")
# widgets
case_select = RadioButtonGroup(
labels=["Confirmed", "Recovered", "Death", "All(Beta)"],
active=0,
name="case_select")
region_select = Select(value=region,
title='Country/Region',
options=list(regions_confirmed),
name="region_select")
range_slider = DateRangeSlider(start=slider_value[0],
end=slider_value[1],
value=(0, slider_value[1]),
title='Date',
name="range_slider")
button = Button(label="Change theme", button_type="success")
# onchange
region_select.on_change('value', handle_region_change)
code = """
var death = document.getElementById("total-Region-Death");
var recovered = document.getElementById("total-Region-Recovered");
var confirmed = document.getElementById("total-Region-Confirmed");
var region = cb_obj.value
death.innerHTML = death_case[region].toLocaleString('id')
recovered.innerHTML = recovered_case[region].toLocaleString('id')
confirmed.innerHTML = confirmed_case[region].toLocaleString('id')
'Bar' : data.loc[start:end].Bar.values,
'Colors' : data.loc[start:end].Colors,
'trendUpIndColor' : data.loc[start:end].trendUpIndColor,
'trendDownIndColor': data.loc[start:end].trendDownIndColor,
'emotionUpIndColor' : data.loc[start:end].emotionUpIndColor,
'emotionDownIndColor': data.loc[start:end].emotionDownIndColor,
'extremeUpIndColor' : data.loc[start:end].extremeUpIndColor,
'extremeDownIndColor': data.loc[start:end].extremeDownIndColor,
'sentimentUpIndColor' : data.loc[start:end].sentimentUpIndColor,
'sentimentDownIndColor': data.loc[start:end].sentimentDownIndColor
})
source.data.update(new_data.data)
# Make a slider object: slider
slider = DateRangeSlider(title="Date Range: ", start=min(data.index), end=max(data.index),
value=(start,end), step=1, width=1000)
# Create the figure for the candle stick
p = figure(title='Line chart', x_axis_label='Dates', y_axis_label='Closing Price',
x_axis_type='datetime', plot_height=400, plot_width=1300)
p.segment('Bar', 'High', 'Bar', 'Low', source=source, line_width=1, color='black') # plot the wicks
p.vbar('Bar', 0.7, 'Close', 'Open', source=source, line_color='black', fill_color = 'Colors')
# Creating the figure for symphonie indicators
p1 = figure(title='Symphonie indicators', x_axis_type='datetime', plot_height=150, plot_width=1300)
p1.vbar('Bar', 0.7, 1, 2, source=source, line_color='White', fill_color = 'trendDownIndColor')
p1.vbar('Bar', 0.7, 2, 3, source=source, line_color='White', fill_color = 'trendUpIndColor')
p1.vbar('Bar', 0.7, 5, 6, source=source, line_color='White', fill_color = 'emotionDownIndColor')
p1.vbar('Bar', 0.7, 6, 7, source=source, line_color='White', fill_color = 'emotionUpIndColor')
'open_dec': data.open[dec],
'close_dec': data.close[dec]})
return stock, stock_day
def yeardata(year:int = year):
unique_stocks = df['symbol'].unique()
year_data = df.loc[df['shortened_date'].dt.year==year]
return year_data, unique_stocks
#* Get data for year
year_data, unique_stocks = yeardata(year=year)
w = 12*60*60*1000 # half day in ms
# w = 0.5
range_slider = DateRangeSlider(start=year_data['shortened_date'].min(), end=year_data['shortened_date'].max(),
value=(date(2016,2,3),date(2016,3,3)), step=1, title="From to")
#Filtering for chosen stock
Select1 = Select(title='Compare:', value='AAL', options=list(unique_stocks))
Select2 = Select(title='To:', value='GD', options=list(unique_stocks))
TOOLS = "pan,wheel_zoom,box_zoom,reset,save"
plot = figure(title='Stock Prices',
x_axis_type="datetime",
tools=TOOLS,
plot_width=1000,
y_axis_label='Price in $USD',
x_axis_label='Date'
)
end_dt = start + timedelta(hours=24)
curr_dt = start
#print(start, end, curr_dt)
t1 = end - start
y1 = int((t1.total_seconds() / 3600) / 10)
y2 = int((t1.total_seconds() / 3600) / 4)
sldr_rate = Slider(start=1,
end=y2,
value=y1,
step=1,
title="Rate of change (in hours)")
date_range_slider = DateRangeSlider(title="Date Range: ",
start=start.date(),
end=end.date(),
value=(start.date(), end.date()),
step=1000000)
date_range_slider.on_change('value', cb_sldr_time)
button_play = Button(label="Play", button_type="success")
button_play.on_click(bt_play_click)
cur_hours = (curr_dt - start) / 3600
hours = int(cur_hours.total_seconds() / 3600)
dur_text = PreText(text=str(hours) + " hr from start", height=50)
######################## Word clouds for most relevant words
if len(relevantWords.keys()) > 40:
relevantWords = dict(
sorted(relevantWords.items(), key=operator.itemgetter(1),
def make_trendplots_2020(Data_2020):
Tooltips1 = [("Index", "$index"), ("Time:", "$x"), ("Flow: ", "$y")]
plot1 = figure(title='Flow Rate (m3/h)',
y_axis_label="F (m3/h)",
plot_width=330,
plot_height=320,
tooltips=Tooltips1,
sizing_mode='scale_width')
plot1.line(Data_2020['Time'],
Data_2020['F (m3/h)'],
line_alpha=0.8,
line_width=2,
color="#118DFF")
plot1.xaxis.formatter = DatetimeTickFormatter(
hours=["%d %b %Y"],
days=["%d %b %Y"],
months=["%d %b %Y"],
years=["%d %b %Y"],
)
plot1.xgrid.visible = False
plot1.ygrid.visible = False
plot1.xaxis.minor_tick_line_color = None
plot1.yaxis.minor_tick_line_color = None
plot1.outline_line_color = None
plot1.xaxis.major_label_orientation = math.pi / 4
Tooltips2 = [("Index", "$index"), ("Time:", "$x"), ("Flow: ", "$y")]
plot2 = figure(title='Inlet Pressure (kPa)',
x_range=plot1.x_range,
y_axis_label="IP (kPa)",
plot_width=330,
plot_height=320,
tooltips=Tooltips2,
sizing_mode='scale_width')
plot2.line(Data_2020['Time'],
Data_2020['IP (kPa)'],
line_alpha=0.8,
line_width=2,
color="#118DFF")
plot2.xaxis.formatter = DatetimeTickFormatter(
hours=["%d %b %Y"],
days=["%d %b %Y"],
months=["%d %b %Y"],
years=["%d %b %Y"],
)
plot2.xgrid.visible = False
plot2.ygrid.visible = False
plot2.xaxis.minor_tick_line_color = None
plot2.yaxis.minor_tick_line_color = None
plot2.outline_line_color = None
plot2.xaxis.major_label_orientation = math.pi / 4
Tooltips3 = [("Index", "$index"), ("Time:", "$x"), ("Flow: ", "$y")]
plot3 = figure(title='Outlet Pressure (kPa)',
x_range=plot1.x_range,
y_axis_label="OP (kPa)",
plot_width=330,
plot_height=320,
tooltips=Tooltips3,
sizing_mode='scale_width')
plot3.line(Data_2020['Time'],
Data_2020['OP (kPa)'],
line_alpha=0.8,
line_width=2,
color="#118DFF")
plot3.background_fill_color = None
plot3.xaxis.formatter = DatetimeTickFormatter(
hours=["%d %b %Y"],
days=["%d %b %Y"],
months=["%d %b %Y"],
years=["%d %b %Y"],
)
plot3.xgrid.visible = False
plot3.ygrid.visible = False
plot3.xaxis.minor_tick_line_color = None
plot3.yaxis.minor_tick_line_color = None
plot3.outline_line_color = None
plot3.xaxis.major_label_orientation = math.pi / 4
p = gridplot([[plot1, plot2, plot3]],
toolbar_location='right',
sizing_mode='stretch_both')
callback1 = CustomJS(args=dict(plot=plot1),
code="""
var a = cb_obj.value;
plot.x_range.start = a[0];
plot.x_range.end = a[1];
""")
slider = DateRangeSlider(start=date(2020, 1, 1),
end=date(2020, 9, 30),
value=(date(2020, 1, 1), date(2020, 9, 30)),
step=30,
format="%d, %b, %Y")
slider.js_on_change('value', callback1)
layout = column(p, slider)
return layout
def plot_trends_scatter_bokeh(self):
lon = self.lon
lat = self.lat
dw = lon[-1] - lon[0]
dh = lat[0] - lat[-1]
self.dfs = pd.DataFrame.from_dict(
data=dict(LONGITUDE=[], LATITUDE=[], NAME=[], slope=[], id2=[]))
p = figure(plot_width=int(400. * dw / dh),
plot_height=400,
match_aspect=True,
tools="pan,wheel_zoom,box_zoom,tap,reset",
output_backend="webgl")
##--- Create a modified version of seismic colormap
from bokeh.models import LinearColorMapper, ColorBar
import matplotlib.cm as mcm
import matplotlib.colors as mcol
fcmap = mcm.get_cmap('seismic')
cmap_mod = [fcmap(i) for i in np.linspace(0, 1, 15)]
cmap_mod[7] = mcm.get_cmap('RdYlGn')(
0.5) # replace white in the middle by the yellow of RdYlGn
scmap = mcol.LinearSegmentedColormap.from_list(
"", cmap_mod) # recreate a colormap
## Extract 256 colors from the new colormap and convert them to hex
cmap_mod = [scmap(i) for i in np.linspace(0, 1, 256)]
cmap_mod = [
"#%02x%02x%02x" % (int(255 * r), int(255 * g), int(255 * b))
for r, g, b, _ in cmap_mod
]
## Make a fake colormapper to start
## based on the previous 256 colors (needed because it does not make linear interpolation between colors)
self.sn_max = 0.001
color_mapper = LinearColorMapper(palette=cmap_mod,
low=-self.sn_max,
high=self.sn_max)
##--- Select CSV file to read
def upload_input_csv(attr, old, new):
## Read, decode and save input data to tmp file
print("Data upload succeeded")
print("file_input.filename=", file_input.filename)
data = base64.b64decode(file_input.value).decode('utf8')
with open(self.app_dir / 'data/tmp_input.csv', 'w') as f:
f.write(data)
## Get csv meta data and init plot
meta = {
l.split(':')[0]: l.split(':')[1]
for l in data.split('\n') if l.startswith('#')
}
self.hf = h5py.File(
self.app_dir / 'data' / meta['#input_extract_cache_file'], 'r')
if '#input_breaks_pickle_file' in meta.keys():
self.b_breaks = True
self.df_breaks = pd.read_pickle(
self.app_dir / 'data' / meta['#input_breaks_pickle_file'])
## Init line to display timeseries segment
# timeseries segment
segment_line = p2.line(x='dates',
y='var',
source=segment_source,
line_color='red')
# vertical lines for breaks
p2.segment(x0="x",
y0="y0",
x1="x",
y1="y1",
line_color="black",
line_dash='dashed',
line_width=2,
source=breaks_source)
# Add bottom horizontal line
#p2.line(x="x", y="y0", line_color="#fb8072", line_width=2, source=breaks_source)
#p2.diamond(x="x", y="y0", color="#fb8072", size=12, source=breaks_source)
else:
self.b_breaks = False
# Get date range from h5 file
self.dates = self.hf['meta/ts_dates'][:].view(
'datetime64[s]').tolist()
self.point_names = [
i.decode('utf8') for i in self.hf['meta/point_names'][:]
]
d_init = [
d for d in self.dates if d.year != 1970
] # Some date are set to 1970 (ie stored as 0 ? to be checked)
ts_source.data = dict(dates=d_init, var=np.zeros_like(d_init))
## Read tmp file and update select widget with available variables
df = pd.read_csv(self.app_dir / 'data/tmp_input.csv',
sep=';',
comment='#')
in_var = [i for i in df.columns if i.endswith('_sn')]
df = df.dropna(subset=in_var)
in_var = [i.replace('_sn', '') for i in in_var]
print(in_var)
select.disabled = False
select.options = in_var
select.value = in_var[0]
## If there is only one variable in the csv, plot it directly
if len(in_var) == 1:
read_data_for_plotting(in_var[0])
file_input = FileInput(
accept=".plot.csv") # comma separated list if any
file_input.on_change('value', upload_input_csv)
## Add variable selection
def select_variable(attr, old, new):
read_data_for_plotting(new)
select = Select(title="Variable in csv:", disabled=True)
select.on_change('value', select_variable)
##--- Add land mask
# must give a vector of image data for image parameter
mask = self.rebin(
self.mask,
(int(self.mask.shape[0] / 5), int(self.mask.shape[1] / 5)))
#p.image(image=[np.flipud(self.mask[::20,::20])],
p.image(image=[np.flipud(mask)],
x=lon[0],
y=lat[-1],
dw=dw,
dh=dh,
palette=('#FFFFFF', '#EEEEEE', '#DDDDDD', '#CCCCCC', '#BBBBBB',
'#AAAAAA', '#999999', '#888888'),
level="image")
p.grid.grid_line_width = 0.5
##--- Read selected data, filter and convert to ColumnDataSource
def read_data_for_plotting(var):
## Get the variable from the input h5 cache file
self.ts = self.hf['vars/' + var][:, 0, :].T
## Get data from input csv
var = var + '_sn'
if self.b_breaks:
df = pd.read_csv(self.app_dir / 'data/tmp_input.csv',
sep=';',
comment='#',
parse_dates=['start_date', 'end_date'])
else:
df = pd.read_csv(self.app_dir / 'data/tmp_input.csv',
sep=';',
comment='#')
id_sites_in_cache_file = {
s: i
for i, s in enumerate(self.point_names)
}
df['id2'] = df['NAME'].map(id_sites_in_cache_file)
df = df.dropna(subset=[var])
if self.b_breaks:
# better use loc[] to select part of a df that will be modified afterward to be sure to have a copy
self.dfs = df.loc[:, [
'LONGITUDE', 'LATITUDE', 'NAME', var, 'id2', 'lvl',
'start_date', 'end_date'
]]
else:
self.dfs = df.loc[:, [
'LONGITUDE', 'LATITUDE', 'NAME', var, 'id2'
]]
self.dfs['lvl'] = np.zeros_like(self.dfs[var])
self.dfs['start_date'] = np.zeros_like(self.dfs[var])
self.dfs['end_date'] = np.zeros_like(self.dfs[var])
self.dfs = self.dfs.rename(columns={var: 'slope'})
source.data = ColumnDataSource.from_df(self.dfs)
self.sn_max = np.abs(np.nanmax(self.dfs['slope']))
color_mapper.low = -self.sn_max
color_mapper.high = self.sn_max
slider.end = self.sn_max * 1000.
slider.step = self.sn_max * 1000. / 20.
slider.value = (0.0, self.sn_max * 1000.)
#slider.disabled=False
#slider.bar_color='#e6e6e6'
if self.b_breaks:
slider_date.start = self.dates[0]
slider_date.end = self.dates[-1]
slider_date.value = (self.dates[0], self.dates[-1])
slider_date.visible = True
##--- Add scatter
## Create source that will be populated according to slider
source = ColumnDataSource(data=dict(LONGITUDE=[],
LATITUDE=[],
NAME=[],
slope=[],
id2=[],
lvl=[],
start_date=[],
end_date=[]))
#source = ColumnDataSource(dfs)
scatter_renderer = p.scatter(x='LONGITUDE',
y='LATITUDE',
size=12,
color={
'field': 'slope',
'transform': color_mapper
},
source=source)
color_bar = ColorBar(color_mapper=color_mapper, label_standoff=12)
p.add_layout(color_bar, 'right')
## Add hover tool that only act on scatter and not on the background land mask
p.add_tools(
HoverTool(
#tooltips=[("A", "@A"), ("B", "@B"), ("C", "@C")], mode = "vline"
renderers=[scatter_renderer],
mode='mouse'))
##--- Add slider
slider = RangeSlider(start=0.0,
end=self.sn_max * 1000.,
value=(0.0, self.sn_max * 1000.),
step=self.sn_max * 1000. / 20.,
title="Trend threshold [10e-3]")
slider_date = DateRangeSlider(title="Date range: ",
start=dt.date(1981, 1, 1),
end=dt.date.today(),
value=(dt.date(1981, 1,
1), dt.date.today()),
step=1,
visible=False)
## Slider Python callback
def update_scatter(attr, old, new):
# new = new slider value
#source.data = ColumnDataSource.from_df(self.dfs.loc[ (np.abs(self.dfs['slope']) >= 0.001*new[0]) &
#(np.abs(self.dfs['slope']) <= 0.001*new[1]) ])
if self.b_breaks:
slope_sel = slider.value
date_sel = [
pd.to_datetime(d, unit='ms') for d in slider_date.value
]
source.data = ColumnDataSource.from_df(self.dfs.loc[
(np.abs(self.dfs['slope']) >= 0.001 * slope_sel[0])
& (np.abs(self.dfs['slope']) <= 0.001 * slope_sel[1]) &
(self.dfs['start_date'] >= date_sel[0]) &
(self.dfs['end_date'] <= date_sel[1])])
else:
slope_sel = slider.value
source.data = ColumnDataSource.from_df(self.dfs.loc[
(np.abs(self.dfs['slope']) >= 0.001 * slope_sel[0])
& (np.abs(self.dfs['slope']) <= 0.001 * slope_sel[1])])
slider.on_change('value', update_scatter)
slider_date.on_change('value', update_scatter)
##--- Add time series of selected point
pw = int(400. * dw / dh)
ph = 200
p2 = figure(plot_width=pw,
plot_height=ph,
tools="pan,wheel_zoom,box_zoom,reset",
output_backend="webgl",
x_axis_type="datetime",
title='---')
p2.add_tools(
HoverTool(
tooltips=[
("Date",
"@dates{%Y-%m-%d}"), # must specify desired format here
("Value", "@var")
],
formatters={"@dates": "datetime"},
mode='vline'))
## Create source and plot it
#ts_source = ColumnDataSource(data=dict(dates=[], var=[]))
d_init = [dt.datetime(1981, 9, 20), dt.datetime(2020, 6, 30)]
ts_source = ColumnDataSource(
data=dict(dates=d_init, var=np.zeros_like(d_init)))
segment_source = ColumnDataSource(
data=dict(dates=d_init, var=np.zeros_like(d_init)))
breaks_source = ColumnDataSource(data=dict(x=[], y0=[], y1=[]))
# Full timeseries line
p2.line(x='dates', y='var', source=ts_source)
## Add satellite periods
# Sensor dates
sensor_dates = []
sensor_dates.append(['NOAA7', ('20-09-1981', '31-12-1984')])
sensor_dates.append(['NOAA9', ('20-03-1985', '10-11-1988')])
sensor_dates.append(['NOAA11', ('30-11-1988', '20-09-1994')])
sensor_dates.append(['NOAA14', ('10-02-1995', '10-03-2001')])
sensor_dates.append(['NOAA16', ('20-03-2001', '10-09-2002')])
sensor_dates.append(['NOAA17', ('20-09-2002', '31-12-2005')])
sensor_dates.append(['VGT1', ('10-04-1998', '31-01-2003')])
sensor_dates.append(['VGT2', ('31-01-2003', '31-05-2014')])
sensor_dates.append(['PROBAV', ('31-10-2013', '30-06-2020')])
sensor_dates = [[
v[0], [dt.datetime.strptime(i, "%d-%m-%Y") for i in v[1]]
] for v in sensor_dates]
import itertools
from bokeh.palettes import Category10 as palette
colors = itertools.cycle(palette[10])
top_ba = []
bottom_ba = []
for v, color in zip(sensor_dates, colors):
if 'VGT' not in v[0]:
top_ba.append(
BoxAnnotation(top=ph,
top_units='screen',
bottom=int(ph / 2),
bottom_units='screen',
left=v[1][0],
right=v[1][1],
fill_alpha=0.2,
fill_color=color))
else:
bottom_ba.append(
BoxAnnotation(top=int(ph / 2),
top_units='screen',
bottom=0,
bottom_units='screen',
left=v[1][0],
right=v[1][1],
fill_alpha=0.2,
fill_color=color))
for ba in top_ba:
p2.add_layout(ba)
for ba in bottom_ba:
p2.add_layout(ba)
def update_ts(attr, old, new):
"""
attr: 'indices'
old (list): the previous selected indices
new (list): the new selected indices
"""
if 0:
print(p2.width, p2.height)
print(p2.frame_width, p2.frame_height)
print(p2.inner_width, p2.inner_height)
print(p2.x_range.start, p2.x_range.end, p2.x_scale)
print(p2.y_range.start, p2.y_range.end, p2.y_scale)
if len(new) > 0:
## Update line with the last index because this is the last drawn point that is visible
site_id = int(source.data['id2'][new[-1]])
ts_source.data = dict(dates=self.dates, var=self.ts[site_id])
if self.b_breaks:
## Add segment
multi_idx = (source.data['NAME'][new[-1]],
str(source.data['lvl'][new[-1]]))
segment_slice = self.df_breaks.loc[multi_idx]['x'].astype(
'int')
segment_source.data = dict(
dates=[self.dates[i] for i in segment_slice],
var=self.ts[site_id][segment_slice])
## Add breaks
xb = [
pd.to_datetime(i) for i in self.df_breaks.loc[
source.data['NAME'][new[-1]]]['bp_date'].values
if not pd.isnull(i)
]
# Add first and last dates
xb = [pd.to_datetime(self.dates[0])
] + xb + [pd.to_datetime(self.dates[-1])]
y0b = np.nanmin(self.ts[site_id]) * np.ones(len(xb))
y1b = np.nanmax(self.ts[site_id]) * np.ones(len(xb))
breaks_source.data = dict(x=xb, y0=y0b, y1=y1b)
## Update BoxAnnotation
ph = p2.inner_height
for ba in top_ba:
ba.top = ph
ba.bottom = int(ph / 2)
for ba in bottom_ba:
ba.top = int(ph / 2)
ba.bottom = 0
## Update p2 title text with the name of the site
p2.title.text = 'SITE : {} (#{})'.format(
source.data['NAME'][new[-1]], source.data['id2'][new[-1]])
source.selected.on_change('indices', update_ts)
##--- Save html file
#save(column(slider, p, p2))
#save(p)
##--- Serve the file
curdoc().add_root(
column(file_input, select, slider, slider_date, p, p2))
curdoc().title = "Quality monitoring"
def bargraph_tab(nyc_311_calls):
#dataset for bar graph given boroughs, categorical variable, start date, and end date
#only top 15 records are shown
def make_dataset(boroughs, category, start_date, end_date):
date_filter = nyc_311_calls[
(nyc_311_calls['created_mdy'] >= start_date)
& (nyc_311_calls['created_mdy'] <= end_date)]
borough_filter = date_filter[date_filter['borough'].isin(boroughs)]
df = pd.DataFrame(
borough_filter.groupby([category,
'borough'])['count'].sum()).reset_index()
df_pivot = df.pivot_table(values='count',
index=category,
columns='borough')
df_pivot['sum'] = df_pivot.sum(axis=1)
df_sorted = df_pivot.sort_values('sum', ascending=False).fillna(0)[:15]
return ColumnDataSource(df_sorted)
def style(p):
p.title.align = 'center'
p.title.text_font_size = '19pt'
p.axis.axis_label_text_font_size = '12pt'
p.axis.major_label_text_font_size = '10pt'
p.title.text_font = 'avenir'
p.axis.axis_label_text_font = 'avenir'
p.axis.major_label_text_font = 'avenir'
p.legend.label_text_font = 'avenir'
p.title.text_color = 'dimgray'
p.axis.major_label_text_color = 'dimgray'
p.axis.axis_label_text_color = 'dimgray'
p.xaxis.axis_label = 'Calls'
p.title.text_font_style = 'normal'
p.axis.axis_label_text_font_style = 'normal'
p.axis.major_label_text_font_style = 'normal'
p.legend.label_text_font_style = 'normal'
p.toolbar_location = None
p.xaxis.formatter = NumeralTickFormatter(format="0,0")
p.legend.location = "bottom_right"
return p
#horizontal stacked bar graph: y-axis is unique category values, bars are split by boroughs
def make_plot(src, title):
active_category_values = list(reversed(src.data[active_category]))
boroughs = [
x for x in list(src.data.keys()) if x in available_boroughs
]
colors = brewer['YlGnBu'][len(boroughs)]
p = figure(y_range=active_category_values,
title=title,
plot_height=700,
plot_width=1100)
p.hbar_stack(boroughs,
y=active_category,
height=0.9,
source=src,
color=colors,
legend=[x.lower() for x in boroughs],
fill_alpha=0.8)
category_value = f'@{active_category}'
#format number values in hover tool annotations as '10,000'
hover = HoverTool(
tooltips=[(display_category,
category_value), (
'Brooklyn',
'@Brooklyn{0,0}'), ('Bronx', '@Bronx{0,0}'),
('Staten Island',
'@Staten_Island{0,0}'), ('Manhattan',
'@Manhattan{0,0}'),
('Queens',
'@Queens{0,0}'), ('Unspecified', '@Unspecified{0,0}')])
p.add_tools(hover)
p = style(p)
return p
def update(attr, old, new):
#set new categorical variable, boroughs, and colors to plot
category_to_plot = labels_lookup[category_select.value]
boroughs_to_plot = [
borough_selection.labels[i] for i in borough_selection.active
]
colors = brewer['BuPu'][len(boroughs_to_plot)]
#convert date range slider values to timestamp, given dtype of returned value
if isinstance(date_range_slider.value[0], (int, float)):
start_date = pd.Timestamp(float(date_range_slider.value[0]) * 1e6)
end_date = pd.Timestamp(float(date_range_slider.value[1]) * 1e6)
else:
start_date = pd.Timestamp(date_range_slider.value[0])
end_date = pd.Timestamp(date_range_slider.value[1])
new_src = make_dataset(boroughs_to_plot, category_to_plot, start_date,
end_date)
src.data.update(new_src.data)
category_to_plot_values = list(src.data[category_to_plot])
p = figure(y_range=category_to_plot_values,
title=category_to_plot,
plot_height=700,
plot_width=1100)
p.hbar_stack(boroughs_to_plot,
y=category_to_plot,
height=0.9,
source=src,
color=colors,
legend=[x.lower() for x in boroughs_to_plot])
p.xaxis.axis_label = category_to_plot
p.title.text = display_category
print(
f'new category: {category_to_plot}, new boroughs: {boroughs_to_plot}, start: {start_date}, end: {end_date}'
)
#set boroughs available for selection
available_boroughs = list(set(nyc_311_calls['borough']))
available_boroughs.sort()
#checkbox for boroughs
borough_selection = CheckboxGroup(labels=available_boroughs,
active=[0, 1, 2, 3, 4, 5])
borough_selection.on_change('active', update)
#slider for date range
date_range_slider = DateRangeSlider(title="Date Range: ",
start=date(2020, 1, 1),
end=date.today(),
value=(date(2020, 1, 1), date.today()),
step=10,
bar_color='#8c96c6',
tooltips=True)
date_range_slider.on_change('value', update)
#dropdown for which category to plot
display_labels = [
'Agency', 'City', 'Descriptor', 'Location Type', 'Status', 'Zip Code'
]
actual_labels = [
'agency_name', 'cleaned_city', 'cleaned_descriptor',
'cleaned_location_type', 'status', 'incident_zip'
]
labels_lookup = {
display: actual
for display, actual in zip(display_labels, actual_labels)
}
category_select = Select(title="Category:",
value='Agency',
options=display_labels)
category_select.on_change('value', update)
#divider text for borough checkbox
div = Div(text="""Borough:""", width=200, height=15)
#set initial dataset params
display_category = category_select.value
active_category = labels_lookup[display_category]
initial_boroughs = [
borough_selection.labels[i] for i in borough_selection.active
]
start_date = pd.to_datetime(date_range_slider.value[0])
end_date = pd.to_datetime(date_range_slider.value[1])
#create initial plot
src = make_dataset(initial_boroughs, active_category, start_date, end_date)
p = make_plot(src, f'Calls by {display_category}')
controls = WidgetBox(date_range_slider, category_select, div,
borough_selection)
layout = row(controls, p)
tab = Panel(child=layout, title='Calls by Category')
tabs = Tabs(tabs=[tab])
return tab
@lru_cache()
def get_data(company):
dtf = data[data.Symbol == company].set_index('date')
for name, color in zip(company_list, turbo(n)):
if name == company:
dtf['color'] = color
return dtf
# set up widgets widgets
ticker = Select(value='AMCR', options=company_list, title="Company:")
range_slider = DateRangeSlider(start=startdate,
end=enddate,
value=(startdate, enddate),
step=1,
title='Date Range')
# Box selection
box = BoxAnnotation(fill_alpha=0.5,
line_alpha=0.5,
level='underlay',
left=startdate,
right=enddate)
# set up plots Main Figure
source = ColumnDataSource(data=dict(date=[],
open=[],
high=[],
low=[],
}
}
console.log("filter completed");
return indices;
''',
args={'speed_slider': speed_slider})
speed_slider.js_on_change(
"value",
CustomJS(code="source.change.emit();", args=dict(source=source_visible)))
# range slider for time
start = datetime.datetime.fromtimestamp(df.timestamp.min())
end = datetime.datetime.fromtimestamp(df.timestamp.max())
time_slider = DateRangeSlider(title="Промежуток времени",
value=(start, end),
start=start,
end=end,
format="%H:%M",
step=10 * 60 * 1000)
time_filter = CustomJSFilter(code='''
var min_timestamp = time_slider.value[0]/1000 - 3600
var max_timestamp = time_slider.value[1]/1000 - 3600
console.log(min_timestamp, max_timestamp);
var indices = [];
var column = source.data['timestamp'];
// iterate through rows of data source and see if each satisfies some constraint
for (var i = 0; i < column.length; i++){
if(column[i]>min_timestamp && column[i]<max_timestamp){
indices.push(true);
} else {
def SIF_Explorer_tab():
#################################
# Initialize all layers
#################################
# Load from a save file
if path.exists(LAYERS_FILE):
with open(LAYERS_FILE, 'rb') as layers_file:
us_county_layer, us_state_layer, world_grid_2_degree_layer = pickle.load(layers_file)
# Load from scratch
else:
us_county_layer = US_County_Layer()
us_state_layer = US_State_Layer()
world_grid_2_degree_layer = US_World_Grid_2_Degree_Layer()
# Save the layers to file
with open(LAYERS_FILE, 'wb') as layers_file:
layers = (us_county_layer, \
us_state_layer, world_grid_2_degree_layer)
pickle.dump(layers, layers_file)
# Want the custom layer to be new every time
custom_shapes_layer = Custom_Shapes_Layer()
world_grid_dynamic_layer = World_Grid_Dynamic_Layer()
# Set the active layer to be the county layer
active_layer = us_county_layer
#################################
# Set up layer selector
#################################
# Once a new layer is selected, use that layer to refresh the page
# and all data
def refresh_page():
# Get initial map details
xs, ys, names = active_layer.get_map_details()
if type(active_layer) != Custom_Shapes_Layer:
# Obtain and update date boundaries
start_date, end_date = active_layer.get_date_range()
date_range_slider.start = start_date
date_range_slider.end = end_date
# Unpack the current range
range_start, range_end = date_range_slider.value
# Convert to SQL format
range_start = utc_from_timestamp(range_start)
range_end = utc_from_timestamp(range_end)
# Get the initial sif values
sifs = active_layer.get_data_for_date_range(range_start,
range_end)
# Dictionary to hold the data
new_source_dict = dict(
x= xs, y= ys,
name= np.array(names), sifs= np.array(sifs))
# Update all source data values
source.data = new_source_dict
# Turn off custom layer
custom_data_source.data = dict(x= np.array([]), y= np.array([]),
name = np.array([]))
else:
# Turn off the other layers
source.data = dict(x= np.array([]), y= np.array([]),
name= np.array([]), sifs= np.array([]))
# Safeguard - that at least one custom shape is drawn
if (xs.size != 0):
# Dictionary to hold the selected shape data
new_source_dict = dict(
x= xs, y= ys,
name = np.array(names))
custom_data_source.data = new_source_dict
# Trigger for when a new layer is selected
def layer_selected(new):
# We want to modify the overall active layer (not local to this func)
nonlocal active_layer
# Simple dictionary to switch out the active layer
switcher = {
0 : custom_shapes_layer,
1 : us_state_layer,
2 : us_county_layer,
3 : world_grid_2_degree_layer,
4 : world_grid_dynamic_layer,
}
# Swap out the active layer
active_layer = switcher.get(new, active_layer)
# Fetch new dates, shapes, names, etc. and refresh the page
refresh_page()
# Define selection labels
layer_selector = RadioButtonGroup(
labels=["Custom", "US States", "US Counties",
"World", "World (Dynamic)"], active=2)
# Set up layer selection callback
layer_selector.on_click(layer_selected)
#################################
# Set up date range slider
#################################
# Obtain date boundaries
start_date, end_date = active_layer.get_date_range()
# create a callback for when the date slider is changed
def date_range_selected(attr, old, new):
t0 = time.time()
# Unpack the new range
range_start, range_end = new
# Convert to SQL format
range_start = utc_from_timestamp(range_start)
range_end = utc_from_timestamp(range_end)
# Get the new day's data
sifs = active_layer.get_data_for_date_range(range_start, range_end)
# Update the sif values
source.data["sifs"] = np.array(sifs)
# Set the title of the map to reflect the selected date range
p.title.text = "SIF Average: %s to %s" % (range_start, range_end)
print("Took " + str(time.time() - t0) + " seconds to update")
# Create the date slider
date_range_slider = DateRangeSlider(title="Date Range: ",
start=start_date, end=end_date,
value=(START_DATE_INIT,
END_DATE_INIT), step=1)
# Assign the callback for when the date slider changes
date_range_slider.callback_policy = "throttle"
date_range_slider.callback_throttle = 200
date_range_slider.on_change('value_throttled', date_range_selected)
#################################
# Set up the map and its source
#################################
# Get initial map details
xs, ys, names = active_layer.get_map_details()
# Get the initial sif values
sifs = active_layer.get_data_for_date_range(START_DATE_INIT,
END_DATE_INIT)
# Dictionary to hold the data
source=ColumnDataSource(data = dict(
x= np.array(xs),
y= np.array(ys),
name= np.array(names),
sifs= np.array(sifs))
)
# Which tools should be available to the user
TOOLS = "pan,wheel_zoom,reset,hover,save,tap"
# Obtain map provider
tile_provider = get_provider(Vendors.CARTODBPOSITRON_RETINA)
# tile_options = {}
# tile_options['url'] = 'https://mt1.google.com/vt/lyrs=m&x={x}&y={y}&z={z}'
# tile_options['attribution'] = """
# Map tiles by <a href="http://stamen.com">Stamen Design</a>, under
# <a href="http://creativecommons.org/licenses/by/3.0">CC BY 3.0</a>.
# Data by <a href="http://openstreetmap.org">OpenStreetMap</a>,
# under <a href="http://www.openstreetmap.org/copyright">ODbL</a>.
# """
# mq_tile_source = WMTSTileSource(**tile_options)
# Don't want the map to wrap around
tile_provider.wrap_around = False
# Configure the figure
p = figure(
title="SIF Average by County: %s" % end_date, tools=TOOLS,
active_scroll = "wheel_zoom",
x_axis_location=None, y_axis_location=None,
tooltips=[
("Name", "@name"),
("Average SIF", "@sifs")
],
x_axis_type='mercator',
y_axis_type='mercator',
plot_height = 900,
plot_width = 1100,
output_backend="webgl",
x_range=Range1d(-20000000, 20000000, bounds = 'auto'))
# Add the map!
p.add_tile(tile_provider)
p.lod_threshold = None # No downsampling
#p.lod_interval = 150
p.toolbar.logo = None # No logo
p.grid.grid_line_color = None # No grid
# Policy for hovering
p.hover.point_policy = "follow_mouse"
# Color mapper
color_mapper = LinearColorMapper(palette=palette,
low = SIF_MIN, high = SIF_MAX)
color_transform = {'field': 'sifs', 'transform': color_mapper}
# Patch all the information onto the map
patch_renderer = p.patches('x', 'y', source=source,
fill_color=color_transform,
fill_alpha=0.9, line_color="white", line_width=0.1,
selection_fill_alpha = 1.0,
selection_fill_color = color_transform,
nonselection_line_color="black",
nonselection_fill_alpha=0.7,
nonselection_fill_color= color_transform)
p.title.text_font_size = '16pt'
# Add a color bar
ticker = FixedTicker(ticks=[-1,0,1,2])
color_bar = ColorBar(color_mapper=color_mapper, ticker = ticker,
label_standoff=12, border_line_color=None, location=(0,0))
p.add_layout(color_bar, 'right')
# Add zoom in / out tools
zoom_in = ZoomInTool(factor=ZOOM_FACTOR)
zoom_out = ZoomOutTool(factor=ZOOM_FACTOR)
def range_changed(param, old, new):
if (type(active_layer) == World_Grid_Dynamic_Layer):
x1 = p.x_range.start
x2 = p.x_range.end
y1 = p.y_range.start
y2 = p.y_range.end
new_x1, new_y1 = to_lat_lon(y1, x1)
new_x2, new_y2 = to_lat_lon(y2, x2)
if (active_layer.range_changed(new_x1, new_y1, new_x2, new_y2)):
refresh_page()
#convert_shapes_to_mercator(np.array([[x1], [x2]]),
#np.array([[y1], [y2]]))
#print("(%s, %s) to (%s, %s)" % (x1, y1, x2, y2))
#40000000
#
# p.callback_policy = "throttle"
# p.callback_throttle = 200
p.x_range.on_change('end', range_changed)
p.add_tools(zoom_in)
p.add_tools(zoom_out)
#################################
# Set up custom plot data
#################################
# Data source for custom shapes
custom_data_source = ColumnDataSource(data = \
dict(x= np.array([]), y= np.array([]),
name=np.array([])))
# Patch the custom data onto the map
p.patches('x', 'y', source=custom_data_source,
line_color="darkslategray", line_width=1,
fill_alpha=0.3, fill_color="lightgray")
# On geometry selection
# zoom = WheelZoomTool()
# p.on_event()
# p.add_tools(lasso)
# p.on_event(SelectionGeometry, shape_drawn)
#################################
# Set up time series
#################################
def color_gen():
yield from itertools.cycle(time_pltt[10])
color = color_gen()
def shape_drawn(event):
# Check if more than one point
# Otherwise a tap triggers this function
if (type(event.geometry['x']) != float):
# Notify the custom selection layer that this
# shape was selected and obtain relevant info
# custom_shapes_layer.patch_selected(event)
# Change to the custom layer
layer_selector.active = 0
layer_selected(0)
# Notify the layer that this patch was created.
active_layer.patch_created(event)
# Clear all time series
sif_series.renderers = []
# Update the title and get new data from the active layer
sif_series.title.text, time_srs_src_list, names = \
active_layer.patch_clicked(source, None)
# Plot each series returned
for i in range(len(time_srs_src_list)):
sif_series.scatter('date', 'sif',
source=time_srs_src_list[i],
color=time_pltt[10][i])
# Make sure the current shape is drawn
refresh_page()
def patch_clicked(event):
""" When a patch is clicked, update the time series chart. """
# Clear all time series
sif_series.renderers = []
# Update the title and get new data from the active layer
sif_series.title.text, time_srs_src_list, names = \
active_layer.patch_clicked(source, event)
# Plot each series returned
for i in range(len(time_srs_src_list)):
sif_series.scatter('date', 'sif',
source=time_srs_src_list[i],
color=time_pltt[10][i])
# Which tools should be available to the user for the timer series
TOOLS = "pan,wheel_zoom,reset,hover,save"
# Figure that holds the time-series
sif_series = figure(plot_width=750, plot_height=400, x_axis_type='datetime',
tools=TOOLS,
title= "SIF Time-Series (Select a county...)",
active_scroll = "wheel_zoom",
tooltips=[
("Day", "@date"),
("Average SIF", "@sif")
],
x_axis_label = 'Date',
y_axis_label = 'SIF Average',
y_range = (SIF_MIN, SIF_MAX))
# Some font choices
sif_series.title.text_font_size = '16pt'
sif_series.xaxis.axis_label_text_font_size = "12pt"
sif_series.yaxis.axis_label_text_font_size = "12pt"
# Policy for hovering
sif_series.hover.point_policy = "follow_mouse"
# No logo
sif_series.toolbar.logo = None
# When a patch is selected, trigger the patch_time_series function
p.on_event(Tap, patch_clicked)
# On geometry selection
lasso = LassoSelectTool(select_every_mousemove = False)
p.add_tools(lasso)
p.on_event(SelectionGeometry, shape_drawn)
#################################
# TODO: Set up download area
#################################
# def save_data():
# active_layer.save_data()
# print("Button Clicked")
# callback = active_layer.get_save_data_js_callback()
button = Button(label="Save Data", button_type="success")
# button.on_click(active_layer.save_data)
#button.js_on_event(ButtonClick, callback)
#################################
# Set up tab
#################################
# The layout of the view
layout = row(column(p, date_range_slider, layer_selector),
column(sif_series, row(column(), button)))
# Create tab using layout
tab = Panel(child=layout, title = 'US Visualization')
# Return the created tab
return tab
def make_multi_yaxis_plot(Data_2020_Original):
Tooltips = [("Index", "$index"), ("Time", "$x"), ("Flow: ", "$y")]
plot = figure(x_axis_label="Time",
y_axis_label="F (m3/h)",
plot_width=880,
plot_height=400,
tooltips=Tooltips,
sizing_mode='scale_width')
plot.yaxis.axis_line_color = 'orange'
plot.yaxis.axis_label_text_color = 'orange'
plot.yaxis.major_label_text_color = 'orange'
plot.yaxis.major_tick_line_color = 'orange'
# Setting the second y axis range name and range
plot.extra_y_ranges = {
"foo":
Range1d(start=Data_2020_Original['OP (kPa)'].min(),
end=Data_2020_Original['OP (kPa)'].max()),
"bar":
Range1d(start=Data_2020_Original['IP (kPa)'].min(),
end=Data_2020_Original['IP (kPa)'].max()),
"cat":
Range1d(start=Data_2020_Original['Anomaly Score (SPE) %'].min(),
end=Data_2020_Original['Anomaly Score (SPE) %'].max())
}
# Adding the second axis to the plot.
plot.add_layout(
LinearAxis(y_range_name="foo",
axis_label="OP (kPa)",
axis_line_color='blue',
axis_label_text_color='blue',
major_label_text_color='blue',
major_tick_line_color='blue'), 'left')
# Adding the second axis to the plot.
plot.add_layout(
LinearAxis(y_range_name="bar",
axis_label="IP (kPa)",
axis_line_color='green',
axis_label_text_color='green',
major_label_text_color='green',
major_tick_line_color='green'), 'left')
# Adding the second axis to the plot.
plot.add_layout(
LinearAxis(y_range_name="cat",
axis_label="Anomaly (kPa)",
axis_line_color='red',
axis_label_text_color='red',
major_label_text_color='red',
major_tick_line_color='red'), 'left')
plot.line(Data_2020_Original['Time'],
Data_2020_Original['F (m3/h)'],
line_alpha=0.6,
legend_label='Flow',
line_width=2,
color='orange')
plot.line(Data_2020_Original['Time'],
Data_2020_Original['OP (kPa)'],
line_alpha=0.6,
legend_label='Outlet Pressure',
line_width=2,
color='blue',
y_range_name="foo")
plot.line(Data_2020_Original['Time'],
Data_2020_Original['IP (kPa)'],
line_alpha=0.6,
legend_label='Inlet Pressure',
line_width=2,
color='green',
y_range_name="bar")
plot.line(Data_2020_Original['Time'],
Data_2020_Original['Anomaly Score (SPE) %'],
line_alpha=0.8,
legend_label='Anomaly',
line_width=2,
color='red',
y_range_name="cat")
plot.add_layout(plot.legend[0], 'right')
plot.xaxis.formatter = DatetimeTickFormatter(
hours=["%d %b %Y"],
days=["%d %b %Y"],
months=["%d %b %Y"],
years=["%d %b %Y"],
)
callback1 = CustomJS(args=dict(plot=plot),
code="""
var a = cb_obj.value;
plot.x_range.start = a[0];
plot.x_range.end = a[1];
""")
slider = DateRangeSlider(start=date(2020, 1, 1),
end=date(2020, 9, 30),
value=(date(2020, 1, 1), date(2020, 9, 30)),
step=30,
format="%d, %b, %Y")
slider.js_on_change('value', callback1)
layout = column(plot, slider)
return layout
# class RhmAnalyzer:
# def processData():
# Flow = pd.read_csv('./static/data/Flow.csv', nrows=5)
# # OP = pd.read_csv('./data/OP.csv')
# # IP = pd.read_csv('./data/IP.csv')
# Flow = Flow[['Time', 'Value', 'Average', 'Minimum', 'Maximum']]
# print(Flow.head())
def daily_stats_tab(convoStats, convoSelection):
# Daily by-party and total message counts
def make_timeseries_datasets(convoTitle, startDate=None, endDate=None):
convo: analyser.ConvoStats = next(
(x for x in convoStats if x.title == convoTitle))
participants = convo.participants
participantToId = {x: i for i, x in enumerate(participants)}
totalsId = len(participants)
participantToId['Total'] = totalsId
xs = [[] for _ in participants] + [[]]
ys = [[] for _ in participants] + [[]]
color = Category10_7 if len(participants) < 7 else Turbo256
colors = [color[i] for i in range(len(participants) + 1)]
labels = sorted(participants) + ['Total']
for date in convo.dailyCountsBySender.keys():
convertedDate = pd.to_datetime(date)
if startDate is not None and endDate is not None and (
convertedDate < startDate or convertedDate > endDate):
continue
for i, (sender, count) in enumerate(
convo.dailyCountsBySender[date].items()):
participantId = participantToId[sender]
xs[participantId].append(convertedDate)
ys[participantId].append(count)
xs[totalsId].append(convertedDate)
ys[totalsId].append(sum(convo.dailyCountsBySender[date].values()))
# I need an invisible scatterplot for nice tooltips, because multiline tooltips don't work well
totalX = list(chain.from_iterable(xs))
totalY = list(chain.from_iterable(ys))
totalLabels = list(
chain.from_iterable([[x] * len(xs[participantToId[x]])
for x in labels]))
return (ColumnDataSource(data={
'x': xs,
'y': ys,
'color': colors,
'label': labels
}),
ColumnDataSource(data={
'x': totalX,
'y': totalY,
'label': totalLabels
}))
def make_piechart_dataset(convoTitle, startDate=None, endDate=None):
convo: analyser.ConvoStats = next(
(x for x in convoStats if x.title == convoTitle))
df = pd.DataFrame(columns=[
'sender', 'messageCount', 'messageCountAngle', 'f_messageCount',
'wordCount', 'wordCountAngle', 'f_wordCount', 'initiationCount',
'initiationCountAngle', 'f_initiationCount', 'color'
])
color = Category10_7 if len(convo.participants) <= 7 else Turbo256
allMessages = convo.messages
if startDate is not None and endDate is not None:
allMessages = list(
filter(
lambda m: m.datetime.date() >= startDate and m.datetime.
date() <= endDate, allMessages))
totalWordCount = sum(len(x.content.split()) for x in allMessages)
participantCount = len(convo.participants)
initiationsBySender = defaultdict(int)
curConvoParticipants = set()
lastMessage = ''
for i, message in enumerate(allMessages):
if i == 0:
# first message, so conversation initiated
initiationsBySender[message.sender] += 1
else:
timeDiff = message.datetime - allMessages[i - 1].datetime
# It is assumed that if 4h passed since last message, a new conversation has been initiated
hoursPassed = timeDiff.total_seconds() // (60 * 60)
# Extra conditions: if the last convo only had one participant or the last message was a question, don't count a new initiation
# TODO: Perhaps I should apply the same checks when calculating conversation stats, though for durations between messages etc just the time check is probably better
if hoursPassed >= 4 and '?' not in lastMessage and len(
curConvoParticipants) > 1:
initiationsBySender[message.sender] += 1
curConvoParticipants = set()
lastMessage = message.content
curConvoParticipants |= {message.sender}
totalInitiationCount = sum(initiationsBySender.values())
for i, participant in enumerate(sorted(convo.participants)):
messages = list(
filter(lambda m: m.sender == participant, allMessages))
tdf = pd.DataFrame()
tdf['sender'] = [participant]
tdf['messageCount'] = [len(messages)]
# The +1/+2 is to avoid division by zero if no messages are present in the interval
# TODO: Investigate whether I need to care about div by 0 here and in other places
tdf['messageCountAngle'] = [
(len(messages) + 1) / (len(allMessages) + participantCount) *
2 * pi
]
tdf['f_messageCount'] = [
f'{len(messages)} messages ({len(messages)/len(allMessages)*100:.2f}%)'
]
tdf['wordCount'] = [sum(len(x.content.split()) for x in messages)]
tdf['wordCountAngle'] = [
(tdf['wordCount'][0] + 1) /
(totalWordCount + participantCount) * 2 * pi
]
tdf['f_wordCount'] = [
f'{tdf["wordCount"][0]} words ({tdf["wordCount"][0]/totalWordCount*100:.2f}%)'
]
tdf['initiationCount'] = [initiationsBySender[participant]]
tdf['initiationCountAngle'] = [
initiationsBySender[participant] / totalInitiationCount * 2 *
pi
]
tdf['f_initiationCount'] = f'{tdf["initiationCount"][0]} initations ({tdf["initiationCount"][0]/totalInitiationCount*100:.2f}%)'
tdf['color'] = color[i]
df = df.append(tdf)
return ColumnDataSource(df)
def make_messages_display(convoTitle, startDate=None, endDate=None):
convo: analyser.ConvoStats = next(
(x for x in convoStats if x.title == convoTitle))
allMessages = convo.messages
if startDate is not None and endDate is not None:
allMessages = list(
filter(
lambda m: m.datetime.date() >= startDate and m.datetime.
date() <= endDate, allMessages))
# TODO: A single long word will make the div ignore width settings and overflow the window
rez = '<p style="overflow-wrap:break-word;width:95%;">'
for i, message in enumerate(allMessages):
if i > 500:
break
rez += f'<b>{message.sender}</b> <i>({message.datetime.strftime("%Y/%m/%d %H:%M")})</i>: {message.content} </br>'
rez += '</p>'
return Div(text=rez, sizing_mode='stretch_width')
# Statistics for the conversations in the selected date range like average message length
def make_stats_text(convoTitle, startDate=None, endDate=None):
convo: analyser.ConvoStats = next(
(x for x in convoStats if x.title == convoTitle))
allMessages = convo.messages
if startDate is not None and endDate is not None:
allMessages = list(
filter(
lambda m: m.datetime.date() >= startDate and m.datetime.
date() <= endDate, allMessages))
totalMessageLensWords = defaultdict(int)
messageCountsByParticipant = defaultdict(int)
convoDurationSum = 0
convoLenWordsSum = 0
pauseBetweenConvosDurationSum = 0
pauseBetweenMessagesInConvoSum = 0
lastConvoStart = allMessages[0].datetime
convoWordCount = len(allMessages[0].content.split())
convoCount = 0
convoMessageCount = 1
convoPauseBetweenMessagesSum = 0
for i, message in enumerate(allMessages):
wordCount = len(message.content.split())
totalMessageLensWords[message.sender] += wordCount
messageCountsByParticipant[message.sender] += 1
totalMessageLensWords['total'] += wordCount
if i != 0:
timeDiff = message.datetime - allMessages[i - 1].datetime
hoursPassed = timeDiff.total_seconds() // (60 * 60)
if hoursPassed >= 4:
# A new conversation has begun
convoDuration = (allMessages[i - 1].datetime -
lastConvoStart).total_seconds()
convoDurationSum += convoDuration
convoCount += 1
convoLenWordsSum += convoWordCount
pauseDuration = timeDiff.total_seconds()
pauseBetweenConvosDurationSum += pauseDuration
pauseBetweenMessagesInConvoSum += convoPauseBetweenMessagesSum / convoMessageCount
convoMessageCount = 1
convoPauseBetweenMessagesSum = 0
convoWordCount = wordCount
lastConvoStart = message.datetime
else:
convoWordCount += wordCount
convoPauseBetweenMessagesSum += timeDiff.total_seconds()
convoMessageCount += 1
# In some edge cases there may be no messages sent to the participant
if convoCount == 0:
return ''
hours, minutes = divmod(convoDurationSum / convoCount, 60 * 60)
rez = '<p style="width:95%;">'
rez += f'Average conversation duration: {hours:.0f} h {minutes // 60:.0f} min</br>'
if convoCount > 2:
hours, minutes = divmod(
pauseBetweenConvosDurationSum / (convoCount - 1), 60 * 60)
rez += f'Average pause between conversations duration: {hours:.0f} h {minutes // 60:.0f} min</br>'
rez += f'Average conversation length: {len(allMessages) / convoCount:.1f} messages, {convoLenWordsSum // convoCount} words</br>'
minutes, seconds = divmod(convoPauseBetweenMessagesSum / convoCount,
60)
rez += f'Average time between messages in a conversation: {minutes:.1f} min {seconds:.1f} s</br>'
rez += f'Average message length: {totalMessageLensWords["total"] // len(allMessages)} words</br>'
for participant in convo.participants:
if messageCountsByParticipant[participant] == 0:
continue
rez += f'Average length of messages from {participant}: {totalMessageLensWords[participant] // messageCountsByParticipant[participant]} words</br>'
rez += '</p>'
return rez
def make_timeseries_plot(src, tooltipSrc):
p = figure(plot_width=600,
plot_height=600,
title='Daily message counts by date',
x_axis_type='datetime',
x_axis_label='Date',
y_axis_label='Message count')
p.multi_line(xs='x',
ys='y',
source=src,
color='color',
line_width=3,
legend_field='label',
line_alpha=0.4)
tooltipScatter = p.scatter('x', 'y', source=tooltipSrc, alpha=0)
hover = HoverTool(
tooltips=[('Message count', '@y'), ('Details', '@x{%F}, @label')],
formatters={'@x': 'datetime'},
mode='vline'
) # vline means that tooltip will be shown when mouse is in a vertical line above glyph
hover.renderers = [tooltipScatter]
p.add_tools(hover)
return p
def _make_piechart(src, startAngle, endAngle, title, bottomTitle,
tooltips):
p = figure(plot_height=200,
plot_width=280,
toolbar_location=None,
title=title)
p.wedge(x=0,
y=1,
radius=0.5,
start_angle=startAngle,
end_angle=endAngle,
line_color='white',
fill_color='color',
source=src)
p.axis.axis_label = None
p.axis.visible = False
p.grid.grid_line_color = None
hover = HoverTool(tooltips=tooltips)
p.add_tools(hover)
p.add_layout(Title(text=bottomTitle, align="center"), "below")
return p
def make_piechart_plots(src):
totalMessages = sum(src.data["messageCount"])
p1 = _make_piechart(src, cumsum('messageCountAngle',
include_zero=True),
cumsum('messageCountAngle'),
'Messages sent by participant',
f'Total messages: {totalMessages}',
[('Participant', '@sender'),
('Message count', '@f_messageCount')])
totalWords = sum(src.data["wordCount"])
p2 = _make_piechart(src, cumsum('wordCountAngle', include_zero=True),
cumsum('wordCountAngle'),
'Word counts by participant',
f'Total words: {totalWords}',
[('Participant', '@sender'),
('Word count', '@f_wordCount')])
totalInitiations = sum(src.data["initiationCount"])
p3 = _make_piechart(
src, cumsum('initiationCountAngle', include_zero=True),
cumsum('initiationCountAngle'),
'Conversations initiated by participant',
f'Total conversations: {totalInitiations}',
[('Participant', '@sender'),
('Conversations initiated', '@f_initiationCount')])
return column(p1, p2, p3)
def _update_pie_bottom_labels():
# Update the bottom titles of the piecharts
for i, pie in enumerate(piePlots.children):
# This is a bit hack-ish, but don't know a better way to do it
if i == 0:
totalMessages = sum(pieSrc.data["messageCount"])
pie.below[1].text = f'Total messages: {totalMessages}'
elif i == 1:
totalWords = sum(pieSrc.data["wordCount"])
pie.below[1].text = f'Total words: {totalWords}'
elif i == 2:
totalInitiations = sum(pieSrc.data["initiationCount"])
pie.below[1].text = f'Total conversations: {totalInitiations}'
def on_conversation_changed(attr, oldValue, newValue):
convo: analyser.ConvoStats = next(
(x for x in convoStats if x.title == newValue))
# When switching to a new convo, update the date range slider to match convo data ranges
initialDates = list(convo.dailyCountsBySender.keys())
start = pd.to_datetime(initialDates[0]).date()
end = pd.to_datetime(initialDates[-1]).date()
dateSlider.start = start
dateSlider.end = end
dateSlider.value = (start, end)
# TODO: There is some black magic going on here, find if there is a proper way to do this
newScr, newTooltipSrc = make_timeseries_datasets(newValue)
src.data.update(newScr.data)
tooltipSrc.data.update(newTooltipSrc.data)
newPieSrc = make_piechart_dataset(newValue)
pieSrc.data.update(newPieSrc.data)
_update_pie_bottom_labels()
messageColumn.children = [make_messages_display(newValue)]
statsDisplay.text = make_stats_text(newValue)
def on_date_range_changed(attr, old, new):
convoToPlot = convoSelection.value
startDate, endDate = dateSlider.value_as_date
# TODO: There is some black magic going on here, find if there is a proper way to do this
new_src, newTooltipSrc = make_timeseries_datasets(
convoToPlot, startDate, endDate)
src.data.update(new_src.data)
tooltipSrc.data.update(newTooltipSrc.data)
newPieSrc = make_piechart_dataset(convoToPlot, startDate, endDate)
pieSrc.data.update(newPieSrc.data)
_update_pie_bottom_labels()
messageColumn.children = [
make_messages_display(convoToPlot, startDate, endDate)
]
statsDisplay.text = make_stats_text(convoToPlot, startDate, endDate)
# A dropdown list to select a conversation
conversationTitles = sorted([x.title for x in convoStats])
convoSelection.on_change('value', on_conversation_changed)
# A slider to select a date range for the analysis
initialConvo: analyser.ConvoStats = next(
(x for x in convoStats if x.title == conversationTitles[0]))
initialDates = list(initialConvo.dailyCountsBySender.keys())
start = pd.to_datetime(initialDates[0]).date()
end = pd.to_datetime(initialDates[-1]).date()
dateSlider = DateRangeSlider(title='Date interval',
start=start,
end=date.today(),
value=(start, end),
step=24 * 60 * 60 * 1000)
dateSlider.on_change('value_throttled', on_date_range_changed)
src, tooltipSrc = make_timeseries_datasets(conversationTitles[0], start,
end)
p = make_timeseries_plot(src, tooltipSrc)
p = style(p)
pieSrc = make_piechart_dataset(conversationTitles[0], start, end)
piePlots = make_piechart_plots(pieSrc)
messageContents = [
make_messages_display(conversationTitles[0], start, end)
]
messageColumn = column(children=messageContents,
height=670,
css_classes=['scrollable'],
sizing_mode='stretch_width')
statsDisplay = Div(text=make_stats_text(conversationTitles[0], start, end))
statsColumn = column(children=[statsDisplay],
height=540,
css_classes=['scrollable'])
# create layout
leftColumn = column(convoSelection, dateSlider, statsColumn)
layout = row(leftColumn, p, piePlots, messageColumn)
tab = Panel(child=layout, title='Daily statistics')
return tab
