def layout():
date_select = DateRangeSlider(name="period",
title="Period:",
value=(start, end),
bounds=(bounds_start, bounds_end),
value_labels='show',
range=(dict(days=1), None))
date_select.on_change('value', on_date_change)
country_select = Select(title="Host Site Country:",
value="World",
options=country_choices)
country_select.on_change('value', on_country_change)
controls = VBoxModelForm(_children=[
Paragraph(text="Date Range"),
Paragraph(text=""), # spacing hack
Paragraph(text=""),
date_select,
country_select
])
vboxsmall = VBoxModelForm(_children=[controls, source_par])
#hbox1 = HBox(children=[job_loc_plot_builder(), vboxsmall])
#hbox2 = HBox(children=[weekday_builder(), jobtype_builder()])
#layout = VBox(children=[hbox1, hbox2])
layout = VBox(children=[vboxsmall])
return layout
def assemble_precision_recall_tab(self):
dropdown_value = self.model_dropdown.value
threshold_value = self.threshold_slider.value
precision_and_recall = self.daily_precision_and_recall_for_threshold(
threshold_value, self.model_probas[dropdown_value],
self.test_labels)
daily_metrics = self.daily_metrics[dropdown_value]
vis_x_axis = daily_metrics.index
min_date, max_date = vis_x_axis.min(), vis_x_axis.max()
time_period_slider = DateRangeSlider(title="Time Period",
value=(min_date, max_date),
start=min_date,
end=max_date,
step=1)
precision_recall_f = figure(plot_width=700,
plot_height=400,
x_axis_type="datetime",
title=dropdown_value)
precision_recall_f.xaxis.axis_label = "Date"
precision_recall_f.yaxis.axis_label = "Precision"
prec_source = ColumnDataSource(precision_and_recall["precision"])
rec_source = ColumnDataSource(precision_and_recall["recall"])
precision_recall_f.line("date",
"precision",
source=prec_source,
color="orange",
legend="Precision")
precision_recall_f.line("date",
"recall",
source=rec_source,
color="green",
legend="Recall")
precision_recall_f.legend.location = "bottom_right"
def update(attr, old, new):
model = self.model_dropdown.value
threshold = self.threshold_slider.value
date_start, date_end = time_period_slider.value_as_datetime
precision_recall_f.title.text = model
precision_and_recall = self.daily_precision_and_recall_for_threshold(
threshold, self.model_probas[model], self.test_labels)
prec_metrics = precision_and_recall["precision"][
date_start:date_end]
recall_metrics = precision_and_recall["recall"][
date_start:date_end]
prec_source.data = ColumnDataSource.from_df(prec_metrics)
rec_source.data = ColumnDataSource.from_df(recall_metrics)
self.model_dropdown.on_change('value', update)
time_period_slider.on_change('value', update)
self.threshold_slider.on_change('value', update)
layout = column(time_period_slider, precision_recall_f)
return layout
def layout():
date_select = DateRangeSlider(
name="period",
title="Period:",
value=(start, end),
bounds=(bounds_start, bounds_end),
value_labels='show',
range=(dict(days=1), None))
date_select.on_change('value', on_date_change)
country_select = Select(title="Host Site Country:", value="World",
options=country_choices)
country_select.on_change('value', on_country_change)
controls = VBoxModelForm(_children=[Paragraph(text="Date Range"),
Paragraph(text=""), # spacing hack
Paragraph(text=""),
date_select, country_select])
vboxsmall = VBoxModelForm(_children=[controls, source_par])
#hbox1 = HBox(children=[job_loc_plot_builder(), vboxsmall])
#hbox2 = HBox(children=[weekday_builder(), jobtype_builder()])
#layout = VBox(children=[hbox1, hbox2])
layout = VBox(children=[vboxsmall])
return layout
def bokeh_main_2():
import sys
src = Path(__file__).absolute().parent / 'src'
sys.path.insert(0, str(src))
from bokeh.layouts import layout # type: ignore
from bokeh.models.widgets import Tabs, Panel # type: ignore
from bokeh.plotting import figure # type: ignore
from dashboard.core.bokeh import test_scatter_matrix
import holoviews as hv
f1 = hv.render(test_scatter_matrix())
f2 = figure()
f2.circle([0, 0, 2], [4, -1, 1])
# todo need something more lazy?
from dashboard.data import sleep_dataframe
from dashboard.sleep import plot_sleep
# TODO would be cool to display logs in the frontend and also currently evaluated function?
# although pehaps it's easier to just always keep logs open?
from bokeh.models.widgets import DateRangeSlider # type: ignore
from datetime import date
drs = DateRangeSlider(
title="Date Range: ",
start=date(2017, 1, 1),
end=date.today(),
value=(date(2017, 9, 7), date(2017, 10, 15)),
step=1,
)
def update(attr, old, new):
print(attr, old, new)
from bokeh.models import CustomJS # type: ignore
# todo see https://docs.bokeh.org/en/latest/docs/gallery/slider.html
update_js = CustomJS(args=dict(drs=drs),
code='''
console.log("HIIII");
''')
drs.on_change('value', update)
drs.js_on_change('value', update_js)
l1 = layout([[f1, drs]], sizing_mode='fixed')
l2 = layout([[f2]], sizing_mode='fixed')
tabs = Tabs(tabs=[
Panel(child=l1, title='This is Tab 1'),
Panel(child=l2, title='This is Tab 2'),
# Panel(child=layout([[sp]]), title='Sleep dataframe'),
])
curdoc().add_root(tabs)
def plot(df, feauture):
def create_plot(source,country, feauture):
hover = HoverTool(tooltips=[("Date", "@date{%F}"), (feauture, str('@'+feauture))],
formatters = {'@date':'datetime'})
p = figure(plot_width=900, plot_height=500,x_axis_type="datetime",
title="Total Covid-19 {} in {} by date".format(feauture, country),
toolbar_location='above')
p.line(x='date', y=feauture, line_width=1, source=source,
line_color="black")
p.vbar(x='date', top=feauture, line_width=1, source=source, fill_color='orange', hover_fill_color='grey',
hover_fill_alpha=1)
p.add_tools(hover)
p.xaxis.axis_label = 'Date'
p.yaxis.axis_label = 'Number of {}'.format(feauture)
p.axis.axis_label_text_font_style = 'bold'
p.yaxis.formatter = NumeralTickFormatter(format = '0.0a')
p.background_fill_color = "beige"
return p
def update(attr, old, new):
country = country_select.value
new_df = df[df['location'] == country]
start = datetime.fromtimestamp(date_select.value[0] / 1000)
end = datetime.fromtimestamp(date_select.value[1] / 1000)
new_df = new_df[new_df['date'] >= start]
new_df = new_df[new_df['date'] <= end]
new_src = ColumnDataSource(new_df)
source.data.update(new_src.data)
p.title.text = 'Total covid-19 {} in {}'.format(feauture, country)
countries = list(df['location'].unique())
country_select = Select(title = 'Country', value = 'world', options = countries)
country_select.on_change('value', update)
date_select = DateRangeSlider(title = 'Date Range', start = min(df['date']), end = max(df['date']),
value = (min(df['date']), max(df['date'])), step = 15)
date_select.on_change('value', update)
initial_country = country_select.value
source = ColumnDataSource(df[df['location'] == initial_country])
p = create_plot(source, initial_country, feauture)
controls = Column(country_select, date_select)
layout = row(controls, p)
return layout
def source_filters(sources):
def _handle_state_filter():
for fullname in sources['dataframes']:
df = sources['dataframes'][fullname]
df = df[(df.index >= _state['date_range'][0]) & (df.index <= _state['date_range'][1])]
units = UNIT_CONVERSION_MAP[_state['units']]
df = df * UNIT_CONVERSION[units]
sources['sources'][fullname].data = ColumnDataSource(df).data
for callback in ELEMENT_CALLBACKS:
callback(sources=sources, **_state)
def _date_slider_handler(attr, old, new):
start_time, end_time = new
start_date = dt.datetime.fromtimestamp(start_time / 1000)
end_date = dt.datetime.fromtimestamp(end_time / 1000)
_state['date_range'] = (start_date, end_date)
_handle_state_filter()
start_date = dt.datetime(year=2018, month=12, day=1)
end_date = dt.datetime(year=2018, month=12, day=1)
date_slider = DateRangeSlider(start=_state['date_range'][0], end=_state['date_range'][1], value=_state['date_range'], step=1, title="Date Range", height=100)
date_slider.on_change('value', _date_slider_handler)
def _radio_button_group_handler(attr, old, new):
_state['units'] = int(new)
_handle_state_filter()
radio_button_group = RadioButtonGroup(labels=UNIT_CONVERSION_MAP, active=0, width=700, height=100)
radio_button_group.on_change('active', _radio_button_group_handler)
div_button_group_selection = Div(text=f'<img src="/visualization/static/images/{UNIT_CONVERSION_MAP[_state["units"]]}.svg" height="100px"/>', height=100)
def _update_div_button_group(**kwargs):
units = UNIT_CONVERSION_MAP[kwargs['units']]
div_button_group_selection.text = f'<img src="/visualization/static/images/{units}.svg" height="100px"/>'
ELEMENT_CALLBACKS.append(_update_div_button_group)
return [date_slider, radio_button_group, div_button_group_selection]
def month_range_options(self):
"""Returns gridplot (bokeh layout or Element) defining elements for the User to manipulate Data shown in
other views.
Function first checks if the months variables are initialized and if not, initializes them. Then one
bokeh widget - DateRangeSlider is created.
DateRangeSlider will allow User to filter data to Months between two "borders" of the Slider. Even though
DateRangeSlider defines step: 1 (so all days from the months are shown), formatter applied will only show
month-year values ("%b-%Y).
Additionally, defined callback checks months of start-stop values and compares them to months from
old values. This way, callbacks aren't triggered for every slight change in the Slider and change in
.chosen_months is only triggered when the actual change in months happens.
Values for the Slider are also taken from instance attributes - this way User's previous choice is
remembered and can be shown upon coming back to Settings View.
Returns DateRangeSlider.
"""
if self.is_month_range_initialized is False:
self.__initialize_months()
sld = DateRangeSlider(start=self.all_months[0], end=self.all_months[-1], step=1,
value=(self.chosen_months[0], self.chosen_months[-1]),
format="%b-%Y", title="Chosen Month Range: ",
css_classes=["month_range_slider"])
def month_range_callback(attr, old, new):
formatting = "%Y-%m"
old_str = self.__create_timetuple_string_from_timestamp(old, formatting)
new_str = self.__create_timetuple_string_from_timestamp(new, formatting)
if old_str != new_str:
self.__update_chosen_months(new)
sld.on_change("value", month_range_callback)
return sld
# Get the current slider values
x_start = datetime.fromtimestamp(date_slider.value[0] / 1000)
x_end = datetime.fromtimestamp(date_slider.value[1] / 1000)
x_start = pd.to_datetime(x_start)
x_end = pd.to_datetime(x_end)
#print(x_start)
#print(x_end)
# Generate new data
new_df = df[(df['x'] >= x_start) & (df['x'] <= x_end)]
new_df.loc[:, 'port'] = (new_df['port'].pct_change().fillna(0) +
1).cumprod() * 100
new_df.loc[:, 'bm'] = (new_df['bm'].pct_change().fillna(0) +
1).cumprod() * 100
new_df.loc[:, 'longOnly'] = (new_df['longOnly'].pct_change().fillna(0) +
1).cumprod() * 100
new_df.loc[:, 'ER_port'] = new_df['port'] - new_df['bm']
new_df.loc[:, 'ER_long'] = new_df['port'] - new_df['longOnly']
new_df.loc[:, 'dd'] = drawdown(new_df['port'].values).values
new_df = new_df.reset_index().iloc[:, 1:]
newdata = ColumnDataSource(new_df)
source.data = newdata.data
date_slider.on_change('value', update_data)
plots = column(p1, p2, date_slider)
panel_1 = Panel(child=plots, title='Panel 1')
tabs = Tabs(tabs=[panel_1, panel_2])
curdoc().add_root(tabs)
curdoc().title = "DateRangeSlider Example"
def create_tab(data, name):
def make_dataset(metric_fun, metric_sentiment, month_start, month_end,
editorial, category, social_network, product):
"""Constrói os datasets para cada tipo de gráfico utilizado no dashboard
Parâmetros
----------
data : DataFrame
Pandas DataFrame expandido com dados do Sprinklr
metric_fun : FUN
Função para calcular métrica específica selecionada no widget
metric_sentiment : str
Sentimento relacionado à métrica escolhida (Positividade: positivo, Gradiente: negativo, Crise: negativo, Saúde do post: positivo)
[restante] : str
Valaores selecionados nas opções de filtros nos widgets
Retorna
-------
dict
Dicionário com três chaves, correspondentes aos três gráficos apresentados. Cada chave é relacionada ao nome o gráfico
e os valores são datasets no formato column data source
"""
month_start = pd.Timestamp(month_start)
month_end = pd.Timestamp(month_end)
# Filtragem dos dados com base nas seleções dos widgets
filters = {
'Editoria': editorial,
'Categoria': category,
'SocialNetwork': social_network,
'Produto': product
}
filtered_data = filter_data(data, filters)
# Gera datasets para cada gráfico
ts_data = metric_fun(filtered_data)
ts_data = ts_data[(ts_data.time >= month_start)
& (ts_data.time <= month_end)]
donut_data = filtered_data[(filtered_data.Month >= month_start)
& (filtered_data.Month <= month_end)]
donut_data = percent(donut_data)
donut_data['angle'] = donut_data['value'] / sum(
donut_data['value']) * 2 * pi
donut_data['color'] = Category20c[donut_data.shape[0]]
avg_donut_data = percent_avg(filtered_data)
avg_donut_data = avg_donut_data[avg_donut_data.Month == month_end][[
'Sentimento', 'MAVG'
]]
avg_donut_data.columns = ['label', 'value']
avg_donut_data['angle'] = avg_donut_data['value'] / sum(
avg_donut_data['value']) * 2 * pi
avg_donut_data['color'] = Category20c[avg_donut_data.shape[0]]
top_data = filter_sentiment(filtered_data, metric_sentiment)
top_data = top_data[(top_data.Month >= month_start)
& (top_data.Month <= month_end)]
top_data = brand_health_txengages(top_data)
avg_top_data = round(top_data.score.mean(), 2)
top_data = top_data.sort_values('score', ascending=False).iloc[:10]
top_data = top_data.sort_values('score')
top_data['recorte'] = [
'1', '2', '3', '4', '5', '6', '7', '8', '9', '10'
]
# Converte dataframes em column data source
datasets = {
'ts': ColumnDataSource(ts_data),
'donut': ColumnDataSource(donut_data),
'avg_donut': ColumnDataSource(avg_donut_data),
'top': ColumnDataSource(top_data),
'avg_top': avg_top_data
}
return datasets
def update(attr, old, new):
"""Constrói os datasets para cada tipo de gráfico utilizado no dashboard
Parâmetros
----------
old : ColumnDataSource
Dataframe antigo relacionado aos filtros antigos
new : ColumnDataSource
Dataframe novo, com linhas filtradas de acordo com seleções mais recentes
"""
month_start = month_select.value_as_date[0]
month_end = month_select.value_as_date[1]
editorial = editorial_select.value
category = category_select.value
product = product_select.value
social_network = [
social_network_select.labels[i]
for i in social_network_select.active
]
metric = metric_select.value
metric_attr = get_metric_attr(metric)
metric_fun = metric_attr['fun']
metric_sentiment = metric_attr['sentiment']
new_src = make_dataset(metric_fun=metric_fun,
metric_sentiment=metric_sentiment,
month_start=month_start,
month_end=month_end,
editorial=editorial,
category=category,
social_network=social_network,
product=product)
src['ts'].data.update(new_src['ts'].data)
src['top'].data.update(new_src['top'].data)
src['avg_top'] = new_src['avg_top']
src['donut'].data.update(new_src['donut'].data)
src['avg_donut'].data.update(new_src['avg_donut'].data)
networks = data.SocialNetwork.unique().tolist()
editorials = get_multselect_options(data, 'Editoria')
categories = get_multselect_options(data, 'Categoria')
products = get_multselect_options(data, 'Produto')
month_select = DateRangeSlider(start=date(2019, 1, 1),
end=date(2019, 8, 1),
value=(date(2019, 1, 1), date(2019, 8, 1)),
step=1,
format="%b %Y")
metric_select = Select(value="gradiente",
options=[("velocity", "Parâmetro de Crise"),
("positivity", "Grau de Positividade"),
("gradiente", "Grau de Negatividade"),
("brand_health", "Saúde da Marca"),
("post_health", "Saúde do Post")])
product_select = MultiSelect(value=['Todos'], options=products)
category_select = MultiSelect(value=['Todos'], options=categories)
editorial_select = MultiSelect(value=['Todos'], options=editorials)
social_network_select = CheckboxGroup(labels=networks,
active=list(range(len(networks))))
metric_select.on_change('value', update)
month_select.on_change('value', update)
editorial_select.on_change('value', update)
category_select.on_change('value', update)
product_select.on_change('value', update)
social_network_select.on_change('active', update)
initial_metric_attr = get_metric_attr(metric_select.value)
metric_sentiment = initial_metric_attr['sentiment']
initial_networks = [
social_network_select.labels[i] for i in social_network_select.active
]
src = make_dataset(metric_fun=initial_metric_attr['fun'],
metric_sentiment=metric_sentiment,
month_start=month_select.value_as_date[0],
month_end=month_select.value_as_date[1],
editorial=editorial_select.value,
category=category_select.value,
social_network=initial_networks,
product=product_select.value)
p_ts = make_plot_ts(src['ts'], 'Evolução', metric_sentiment)
p_top = make_dotplot(src['top'])
avg_top = src['avg_top']
avg_top = create_div_title(f'Escore Médio: {avg_top}')
p_donut = make_plot_donut(src['donut'], 'Percentual')
p_avg_donut = make_plot_donut(src['avg_donut'], 'Norma Percentual')
metric_title = create_div_title('MÉTRICA')
month_title = create_div_title('PERÍODO')
network_title = create_div_title('REDE SOCIAL')
category_title = create_div_title('CATEGORIA')
editorial_title = create_div_title('EDITORIA')
product_title = create_div_title('PRODUTO')
controls = WidgetBox(
column(metric_title,
metric_select,
Div(height=5),
month_title,
month_select,
Div(height=5),
editorial_title,
editorial_select,
Div(height=5),
category_title,
category_select,
Div(height=5),
product_title,
product_select,
Div(height=5),
network_title,
social_network_select,
width=250))
plots = column(p_ts, Div(height=20), row(p_donut, p_avg_donut))
layout = row(controls, Div(width=50), plots)
layout = column(Div(text="", height=5), layout, Div(width=20), avg_top,
p_top)
tab = Panel(child=layout, title=name)
return tab
def second_tab_create(filterData):
#all_min_date = filterData.groupby('dataid').agg(min)["time"]
#all_max_date = filterData.groupby('dataid').agg(max)["time"]
dummy_daterange = ['2019-05-01', '2019-08-20']
dummy_granularity = '15 Minutes'
dummy_house = 5679
dummy_pi_u = .20
dummy_pi_nm = .05
dummy_mode = 1
dummy_community = 'NY'
def plot3_data(daterange=dummy_daterange,
xaxis=dummy_granularity,
community=dummy_community):
houseData = filterData[filterData['state'] == community]
houseData = houseData[['time', 'grid']]
houseData = houseData.sort_values('time', ascending=True)
houseData.index = houseData['time']
houseData = houseData.loc[daterange[0]:
daterange[1], :] # cut to the days requested
if xaxis == '15 Minutes':
houseData = houseData.drop(columns="time")
houseData['grid'] = houseData['grid'] * 60 * 15 / 3600 # kWh
if xaxis == 'Hour':
houseData['grid'] = houseData['grid'] * 60 * 15 / 3600 # kWh
houseData = houseData.resample('1h').sum()
if xaxis == 'Day':
houseData['grid'] = houseData['grid'] * 60 * 15 / 3600 # kWh
houseData = houseData.resample('1d').sum()
if xaxis == 'Week':
houseData['grid'] = houseData['grid'] * 60 * 15 / 3600 # kWh
houseData = houseData.resample('1w').sum()
if xaxis == 'Month':
houseData['grid'] = houseData['grid'] * 60 * 15 / 3600 # kWh
houseData = houseData.resample('1m').sum()
houseData['time'] = houseData.index
netLoad = houseData.groupby(houseData['time'])['grid'].sum() # L - G
# note that all of the houses do not have data taken for the same range
# This affects the summation excluding some houses at different points in time
price = netLoad > 0
price = pd.DataFrame(data=price)
return ColumnDataSource(price)
def barPlot_data(daterange=dummy_daterange,
house=dummy_house,
pi_u=dummy_pi_u,
pi_nm=dummy_pi_nm,
mode=dummy_mode,
community=dummy_community):
sortedData = filterData[filterData['state'] == community]
sortedData = sortedData[[
'time', 'grid', 'PV_+_Battery(Discharge)', 'dataid'
]].sort_values('time', ascending=True)
sortedData.index = sortedData['time']
sortedData = sortedData.loc[daterange[0]:daterange[1], :]
sortedData['grid'] = sortedData['grid'] * 60 * 15 / 3600 # kWh
sortedData['PV_+_Battery(Discharge)'] = sortedData[
'PV_+_Battery(Discharge)'] * 60 * 15 / 3600 # kWh
houseData = sortedData[sortedData['dataid'] == house]
L = houseData['grid'] + houseData[
'PV_+_Battery(Discharge)'] # (L-S) + S = L
S = houseData['PV_+_Battery(Discharge)'] # S
Sbar = (S < L) * S + (S > L) * L # Sbar
solarAtDiscount = S - Sbar
load = L.sum() # blue plot no share
selfSolarSum = Sbar.sum() # green plot no share
discountSum = solarAtDiscount.sum() # red plot no share
houseAgg = sortedData.groupby(
sortedData['time'])['grid', 'PV_+_Battery(Discharge)'].sum()
loadAgg = houseAgg['grid'] + houseAgg['PV_+_Battery(Discharge)']
solarAgg = houseAgg['PV_+_Battery(Discharge)']
sumL = loadAgg.cumsum() # Over all of the houses
sumS = solarAgg.cumsum()
surplus_agg = (solarAgg - loadAgg) * (
solarAgg > loadAgg
) #total surplus of solar energy in the community by timestamp (positive part)
deficit_agg = (loadAgg - solarAgg) * (
solarAgg < loadAgg
) #total deficit of solar energy in the community by timestamp (positive part)
sum_deficit = deficit_agg.cumsum()
sum_surplus = surplus_agg.cumsum()
X = (L - S) * (L > S) * (sum_surplus - ((sumS > sumL) *
(sumS - sumL))) / sum_deficit
Shat = (S > L) * L + (S < L) * (S + X)
solar_sold_shared = (S - L) * (S > L) / sum_surplus * (
(sum_surplus - sum_deficit) * (sum_surplus > sum_deficit))
communitySolarSum = Shat.sum(
) # green plot share, consumed solar for one home in the sharing situation
discountShareSum = solar_sold_shared.sum(
) # red plot share # S - Shat Solar sold to the grid by one home in sharing situation
loadCost = load * pi_u # A $
solarCost = selfSolarSum * pi_u # B $
solarSoldCost = discountSum * pi_nm # C $
netBill = loadCost - solarCost - solarSoldCost # No Sharing $
communitySolarCost = communitySolarSum * pi_u # $
solarSoldCostShare = discountShareSum * pi_nm # $
# total_solar_sold_values = (S<L)*(S*pi_u) + (S>L)*((sum_surplus<sum_deficit)*S*pi_u + (sum_surplus>sum_deficit)*(S-solar_sold_shared)*pi_u + (solar_sold_shared*pi_nm))
total_solar_sold_values = (sum_surplus < sum_deficit) * (
sumS * pi_u) + (sum_surplus > sum_deficit) * ((
(sumS > sumL) * (sumS - sumL)) * pi_nm + sumL * pi_u)
total_solar_sold_values_sum = total_solar_sold_values.sum()
print(total_solar_sold_values)
print("_-----------------------------------------------------")
print(total_solar_sold_values_sum)
total_S = 0
total_noshare_solar_value = 0
for iter_home in np.unique(sortedData['dataid'].values):
# print(iter_home)
iter_home_data = sortedData[sortedData['dataid'] == iter_home]
iter_L = iter_home_data['grid'] + iter_home_data[
'PV_+_Battery(Discharge)'] # (L-S) + S = L
iter_S = iter_home_data['PV_+_Battery(Discharge)'] # S
one_home_consumed_solar_value = (
iter_S < iter_L) * iter_S * pi_u + (iter_S > iter_L) * (
iter_L * pi_u + (iter_S - iter_L) * pi_nm)
one_home_consumed_solar_value_sum = one_home_consumed_solar_value.sum(
)
total_S = total_S + iter_S.sum()
print("total S:")
print(total_S)
print(one_home_consumed_solar_value_sum)
total_noshare_solar_value = total_noshare_solar_value + one_home_consumed_solar_value_sum
print(total_noshare_solar_value)
print("not shared:")
print(total_noshare_solar_value)
print("shared:")
print(total_solar_sold_values_sum)
netBillShare = loadCost - communitySolarCost - solarSoldCostShare
pi_sns = round((total_noshare_solar_value) * 100 / total_S, 2)
pi_ss = round((total_solar_sold_values_sum) * 100 / sumS.sum(), 2)
if mode == 1:
d = {
'axis': [0, 1, 2, 3, 4, 5],
'colors': ['blue', 'green', 'red', 'blue', 'green', 'red'],
'data': [
load, selfSolarSum, discountSum, load, communitySolarSum,
discountShareSum
]
}
if mode == 2:
d = {
'axis': [0, 1, 2, 3, 4, 5, 6, 7],
'colors': [
'blue', 'green', 'red', 'orange', 'blue', 'green', 'red',
'orange'
],
'data': [
loadCost, solarCost, solarSoldCost, netBill, loadCost,
communitySolarCost, solarSoldCostShare, netBillShare
]
}
if mode == 3:
d = {'Normal': [pi_sns], 'Sharing': [pi_ss]}
df = pd.DataFrame(data=d)
return ColumnDataSource(data=df)
def plot3_plot(src):
plot3 = figure(title='Equilibrium Price',
x_axis_type="datetime",
x_axis_label="Time",
y_axis_label="Price")
plot3.line('time', 'grid', source=src)
plot3.plot_width = 1400
plot3.plot_height = 300
plot3.yaxis.ticker = [0, 1]
plot3.yaxis.major_label_overrides = {0: '5 ¢', 1: '20 ¢'}
return plot3
def plot4_plot(src):
plot4 = figure(title='Sharing Market Energy Effects of Home 5679',
x_axis_label='No Sharing / Sharing Energy Consumption',
y_axis_label='Energy [kWh]')
plot4.plot_width = 700
plot4.plot_height = 300
plot4.vbar(x='axis', top='data', color='colors', width=1, source=src)
plot4.xgrid.grid_line_color = None
plot4.xaxis.ticker = [0, 1, 2, 3, 4, 5]
plot4.xaxis.major_label_overrides = {
0: 'Load',
1: 'Consumed Solar',
2: 'Solar Sold',
3: 'Load',
4: 'Consumed Solar',
5: 'Solar Sold'
}
return plot4
def plot5_plot(src):
plot5 = figure(title='Sharing Market Effects on the Bill of Home 5679',
x_axis_label='No Sharing Bill / Sharing Bill',
y_axis_label='Dollar Cost [$]')
plot5.plot_width = 700
plot5.plot_height = 300
plot5.vbar(x='axis', top='data', color='colors', width=1, source=src)
plot5.xgrid.grid_line_color = None
plot5.xaxis.ticker = [0, 1, 2, 3, 4, 5, 6, 7]
plot5.xaxis.major_label_overrides = {
0: 'Load',
1: 'Consumed Solar',
2: 'Solar Sold',
3: 'Net Bill',
4: 'Load',
5: 'Consumed Solar',
6: 'Solar Sold',
7: 'Net Bill'
}
return plot5
def update(attr, old, new):
global home_id_to_plot
granularity_to_plot = granularity_1.labels[granularity_1.active]
pi_u_to_plot = int(pi_u_input.value) / 100
pi_nm_to_plot = int(pi_nm_input.value) / 100
## Update the country dropdown
country_selector.label = country_selector.value
## Update the state dropdown
states_available = np.unique(filterData[
filterData['country'] == country_selector.value]["state"])
states_available = states_available.tolist()
state_selector.menu = states_available
state_selector.label = state_selector.value
## Update Homes Available
## Home Updates
home_ids = np.unique(
filterData[filterData['state'] == state_selector.value]['dataid'])
home_ids_available = list(map(str, home_ids))
home_id_selector.menu = home_ids_available
home_id_selector.label = home_id_selector.value
new_home_id_to_plot = int(home_id_selector.value)
## DateRange updates
startDate = filterData[filterData['dataid'] ==
new_home_id_to_plot]['time'].dt.date.iloc[0]
endDate = filterData[filterData['dataid'] ==
new_home_id_to_plot]['time'].dt.date.iloc[-1]
date_range_slider.start = startDate
date_range_slider.end = endDate
if new_home_id_to_plot != home_id_to_plot:
date_range_slider.value = (startDate, endDate)
home_id_to_plot = new_home_id_to_plot
daterange_raw = list(date_range_slider.value_as_datetime)
daterange_to_plot = [
daterange_raw[0].strftime("%Y-%m-%d"),
daterange_raw[1].strftime("%Y-%m-%d")
]
## Plot Updates
plot4.title.text = f'Sharing Market Energy Effects of Home {home_id_to_plot}'
plot5.title.text = f'Sharing Market Effects on the Bill of Home {home_id_to_plot}'
plot3.yaxis.major_label_overrides = {
0: f'{pi_nm_input.value} ¢',
1: f'{pi_u_input.value} ¢'
}
## SRC Updates
new_src3 = plot3_data(daterange=daterange_to_plot,
xaxis=granularity_to_plot,
community=state_selector.value)
new_src4 = barPlot_data(daterange=daterange_to_plot,
house=home_id_to_plot,
pi_u=pi_u_to_plot,
pi_nm=pi_nm_to_plot,
mode=1,
community=state_selector.value)
new_src5 = barPlot_data(daterange=daterange_to_plot,
house=home_id_to_plot,
pi_u=pi_u_to_plot,
pi_nm=pi_nm_to_plot,
mode=2,
community=state_selector.value)
new_src6 = barPlot_data(daterange=daterange_to_plot,
house=home_id_to_plot,
pi_u=pi_u_to_plot,
pi_nm=pi_nm_to_plot,
mode=3,
community=state_selector.value)
print(state_selector.value)
print(home_id_to_plot)
src3.data.update(new_src3.data)
src4.data.update(new_src4.data)
src5.data.update(new_src5.data)
src6.data.update(new_src6.data)
## Granularity Button
granularity_1 = RadioButtonGroup(
labels=["15 Minutes", "Hour", "Day", "Week", "Month"],
active=0,
max_width=100)
granularity_1.on_change('active', update)
## Daterange Slider Button
startDate = filterData[filterData['dataid'] ==
5679]['time'].dt.date.iloc[0]
endDate = filterData[filterData['dataid'] == 5679]['time'].dt.date.iloc[-1]
date_range_slider = DateRangeSlider(title="Date Range: ",
start=startDate,
end=endDate,
value=(startDate, endDate),
step=1,
callback_policy='mouseup',
width=1400)
date_range_slider.on_change("value_throttled", update)
## Country Selector
countries_available = np.unique(filterData['country'])
countries_available = countries_available.tolist()
country_selector = Dropdown(label="Country",
button_type="warning",
menu=countries_available,
value="USA",
max_height=150,
width=300)
country_selector.on_change('value', update)
## State Selector
states_available = np.unique(
filterData[filterData['country'] == "USA"]["state"])
states_available = states_available.tolist()
state_selector = Dropdown(label="Region",
button_type="warning",
menu=states_available,
value="NY",
max_height=150,
width=300)
state_selector.on_change('value', update)
## Home Selector
home_ids_available = np.unique(
filterData[filterData['state'] == 'NY']['dataid'])
home_ids_available = list(map(str, home_ids_available))
home_id_selector = Dropdown(label="Home ID",
button_type="warning",
menu=home_ids_available,
value="5679",
max_height=150,
width=300)
home_id_selector.on_change('value', update)
## Text input
pi_u_input = TextInput(value="20",
title="Utility Rate [¢/kWh]:",
max_width=175,
max_height=50)
pi_u_input.on_change('value', update)
pi_nm_input = TextInput(value="5",
title="Net Metering Rate [¢/kWh]:",
max_width=175,
max_height=50)
pi_nm_input.on_change('value', update)
text_input = WidgetBox(row(pi_u_input, pi_nm_input))
## Initialize src and plot
src3 = plot3_data(['2019-05-01', '2019-08-20'], '15 Minutes', 'NY')
src4 = barPlot_data(['2019-05-01', '2019-08-20'], 5679, .20, .05, 1, 'NY')
src5 = barPlot_data(['2019-05-01', '2019-08-20'], 5679, .20, .05, 2, 'NY')
src6 = barPlot_data(['2019-05-01', '2019-08-20'], 5679, .20, .05, 3, 'NY')
plot3 = plot3_plot(src3)
plot4 = plot4_plot(src4)
plot5 = plot5_plot(src5)
## Table
columns = [
TableColumn(field='Sharing', title='Sharing Market'),
TableColumn(field='Normal', title='No Sharing'),
]
data_table = DataTable(source=src6, columns=columns, width=350, height=50)
table_title = Paragraph(text='Value of Solar Energy [¢/kWh]',
width=350,
max_height=50)
# Create Layout
controls_row3 = column(text_input, table_title, data_table)
row1 = row(country_selector,
state_selector,
home_id_selector,
controls_row3,
sizing_mode="scale_height")
row2 = row(granularity_1)
row3 = row(date_range_slider)
row4 = row(plot4, plot5)
row5 = row(plot3)
layout = column(row1, row2, row3, row4, row5)
# Make a tab with the layout
tab = Panel(child=layout, title='Market Analysis')
return tab
x_start = datetime.fromtimestamp(date_slider.value[0]/1000)
x_end = datetime.fromtimestamp(date_slider.value[1]/1000)
x_start = pd.to_datetime(x_start)
x_end = pd.to_datetime(x_end)
#print(x_start)
#print(x_end)
# Generate new data
new_df = df[(df['x']>= x_start) & (df['x'] <= x_end)]
new_df.loc[:,'y1'] = (new_df['y1'].pct_change().fillna(0) + 1).cumprod() * 100
new_df.loc[:,'y2'] = (new_df['y2'].pct_change().fillna(0) + 1).cumprod() * 100
new_df.loc[:,'dd'] = drawdown(new_df['y1'].values).values
new_df = new_df.reset_index().iloc[:,1:]
newdata = ColumnDataSource(new_df)
source.data = newdata.data
date_slider.on_change('value', update_data)
plots = column(p1, p2, date_slider)
panel_1 = Panel(child = plots, title = sheets[0])
##############################################################################
# Macro Cycle Plot (OECD CLI) - 2번 시트
##############################################################################
macroData = pd.read_excel(excelFile, sheet_name = sheets[1]).set_index('Date')[['OECD_CLI','ESI']]
numIndicator = len(macroData.columns)
for i in range(numIndicator):
colName_3 = macroData.columns[i] + '_3MA'
colName_12 = macroData.columns[i] + '_12MA'
data_src.on_change('data', source_py_callback)
# date_widget_js_callback = CustomJS(code="""
# var d0 = new Date(cb_obj.value[0]),
# d1 = new Date(cb_obj.value[1]);
# console.log(d0, d1);
# """)
# date_widget.js_on_change('value', date_widget_js_callback)
def date_widget_py_callback(attr, old, new):
global DT_RANGE
_dt_range = []
for dt in new:
if not isinstance(dt, datetime.datetime):
dt = datetime.datetime.utcfromtimestamp(dt / 1000.)
dt = dt.replace(tzinfo=None)
_dt_range.append(dt)
DT_RANGE = tuple(_dt_range)
p1.x_range.start = _dt_range[0]
p1.x_range.end = _dt_range[1]
date_widget.on_change('value', date_widget_py_callback)
wid_box = widgetbox(btn, date_widget, sizing_mode='scale_width')
pl_row = row(p1, p2, sizing_mode='stretch_both')
curdoc().add_root(column(pl_row, wid_box, sizing_mode='stretch_both'))
def Electron_Energy_Graph_Old(conn):
############################################################################
#################### CREATE THE DATA FOR THE GRAPH #########################
output_file(
"Electron_Output_Graph.html"
) #????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????
# Use the connection passed to the function to read the data into a
# dataframe via an SQL query.
df = pd.read_sql('SELECT * FROM [eEnergyICP]', conn)
print(df)
# Delete cells where 'protocol id' is empty
df = df.dropna(subset=['protocol id'])
# With any luck this can be removed after chatting to AW and MB ?????????????????????????????????????????????????????????????????????????????????
# Get the date and machinename from the protocol id' field
# Seperate on the first '_'
df_left = df['protocol id'].str.partition(sep='_')
# Seperate on the last '_'
df_right = df['protocol id'].str.rpartition(sep='_')
# From these sperated dataframes add the appropriate columns back into the
# main dataframe.
df.loc[:, 'adate'] = df_left[0]
df.loc[:, 'machinename'] = df_right[2]
# Turn 'adate' into datetime. Problems with this function as it assumes american date formats over british. ?????????????????????????????????????????????????????????????????????????????????
# Talk to AW and MB about getting date from other tables in the database and pulling them into the query. ???????????????????????????????????????????????????????????????????????????????????
# This way the date should be in a set format that the datetime function can be told, which should resolve this issue. ??????????????????????????????????????????????????????????????????????
#
# Need to turn the date fields into a Dateime object (either 'adate'
# (protons) or the newly created 'adate' (photons)). The date field should
# always be named 'adate' for consistency.
df.loc[:, 'adate'] = pd.to_datetime(df.loc[:, 'adate'])
# When starting a new graph can be useful to print the dataframe after any
# manipulations to make sure the code has done what you expected.
print(df)
# Create a list of the fields using the dataframe
TableFields = (list(df.columns))
############################################################################
############################################################################
############################################################################
################ CREATE THE DATAFRAME FOR THE TOLERANCES ###################
# The purpose of this plot is generally to be as general as possible but
# there are only a few parameters that will have defined tolerances.
# Therefore the tolerance section can be a bit more specific and a dataframe
# containing tolereances can be manually created for many cases and
# extracted from the database in others (in a manner similar to above but
# calling from a different table/query with the SQL statement)
#
# The format of the dataframe should be the first line being the x_axis
# (with some values taken from the main dataframe to get the right
# formatting). The subsequent columns are the tolerances [low, high].
# NB: column names should match those from the main dataframe.
df_tol1 = pd.read_sql('SELECT * FROM [ElectronFWHMLimits]', conn)
print(df_tol1)
df_tol1 = df_tol1.set_index('class')
print(df_tol1)
df_tol_TB = pd.DataFrame({
'adate': [df['adate'].max(), df['adate'].max()],
'6fwhm':
[df_tol1.loc['TBUCLH', 'lower6'], df_tol1.loc['TBUCLH', 'upper6']],
'9fwhm':
[df_tol1.loc['TBUCLH', 'lower9'], df_tol1.loc['TBUCLH', 'upper9']],
'12fwhm':
[df_tol1.loc['TBUCLH', 'lower12'], df_tol1.loc['TBUCLH', 'upper12']],
'15fwhm':
[df_tol1.loc['TBUCLH', 'lower15'], df_tol1.loc['TBUCLH', 'upper15']]
})
print(df_tol_TB)
df_tol_Classic = pd.DataFrame({
'adate': [df['adate'].max(), df['adate'].max()],
'6fwhm':
[df_tol1.loc['Classic', 'lower6'], df_tol1.loc['Classic', 'upper6']],
'9fwhm':
[df_tol1.loc['Classic', 'lower9'], df_tol1.loc['Classic', 'upper9']],
'12fwhm':
[df_tol1.loc['Classic', 'lower12'], df_tol1.loc['Classic', 'upper12']],
'16fwhm':
[df_tol1.loc['Classic', 'lower16'], df_tol1.loc['Classic', 'upper16']],
'20fwhm':
[df_tol1.loc['Classic', 'lower20'], df_tol1.loc['Classic', 'upper20']]
})
print(df_tol_Classic)
############################################################################
############################################################################
##########################################################################
################### CREATE THE COLUMNS FOR THE LEGEND ######################
# NB: The following section has been designed to be as general as possible
# but in reality it might be preferable to more manually choose the markers
# and colors based on optimising the most likey things to be plotted.
#
# This code is a way of creating a legend with markers based on one
# parameter (e.g. machine name) and color on another parameter (e.g. energy)
######### Colors:
# Create a sorted list of the unique values in a dataframe column that the
# colors will be based on.
list_forcolor = sorted(df['machinename'].unique().tolist())
# If the length of the list is <9 then we can use the colorblind palette,
# which contains 8 colors. This should be the default for accessability
# reasons unless there are compeling reasons otherwise.
if len(list_forcolor) < 9:
color_palette = Colorblind[len(list_forcolor)]
# If not <9 then we can use the much larger Turbo palette which contains
# 256 colors. Will check if there are more than 256 options though and
# throw an error if so.
elif len(list_forcolor) > 256:
print( 'Error - Name of Function: >256 unique energies in database ' \
'causing failure of the turbo color palette function (only ' \
'256 availible colors.' )
exit()
# If it passes the check then use the built in turbo function that splits
# the turbo palette into roughly equal sections based on a supplied integer
# number.
else:
color_palette = turbo(len(list_forcolor))
######### Markers:
# Doesn't seem to be a way to create a simple list of all the Bokeh marker
# options so just do this manually. May want to re-order to improve
# visibility of commonly used options.
markers = [
'asterisk', 'circle', 'circle_cross', 'circle_x', 'cross', 'dash',
'diamond', 'diamond_cross', 'hex', 'inverted_triangle', 'square',
'square_cross', 'square_x', 'triangle', 'x'
]
# Create a sorted list of the unique values in a dataframe column that the
# markers will be based on.
list_formarker = sorted(df['machinename'].unique().tolist())
# Check that there are enough markers to give a unique marker to each option
# but otherwise throw an error.
if len(list_formarker) > len(markers):
print( 'Error - Name of Function: Not enough markers to assign a ' \
'unique marker to each option.' )
exit()
######### Legend Key:
# Create a function that will be used to run through the dataframe looking
# at the energy and machine column and creating a new column that will have
# values for both seperated by a '_', stored as a string.
def add_legend(row):
return str(str(row['machinename']))
# Run the function and also copy the other columns into new columns so that
# when ther are renamed to 'x' and 'y' later they are still availible for
# the legend if needed.
df.loc[:, 'legend'] = df.apply(lambda row: add_legend(row), axis=1)
df.loc[:, 'machinename1'] = df.loc[:, 'machinename']
print(df)
############################################################################
############################################################################
############################################################################
################## FORMATTING AND CREATING A BASIC PLOT ####################
############################################################################
############################# USER INPUTS ##################################
# Decide what the default viewing option is going to be. (i.e. the fields to
# be plotted on the x and y axis when the graph is opened, the plot size
# etc.).
# From the legend defined above give the values that will be pre-ticked when
# the plot is opened
color_to_plot = ['TrueBeam B', 'TrueBeam C']
marker_to_plot = color_to_plot
# Decide on what data to plot on the x/y axis when opened.
x_data1 = 'adate'
y_data1 = '6fwhm'
# Decide what the plot formatting will be, inluding the plot title, axis
# titles and the size of the plot.
plot_title1 = 'Electron Energy'
x_axis_title1 = x_data1
y_axis_title1 = y_data1
plot_size_height1 = 450
plot_size_width1 = 800
legend_location = 'bottom_left'
# Create a list of the plot parameters that will be used as input to a
# function later.
list_plot_parameters = [
x_data1, y_data1, plot_title1, x_axis_title1, y_axis_title1,
plot_size_height1, plot_size_width1, legend_location
]
############################################################################
############################################################################
############################################################################
########################### CREATE THE PLOT ################################
# Create the actual ploy. Generally it's a good idea to do this by defining
# functions as they can then be used in the callbacks later without having
# a lot of redundant very similar code.
######### Make Dataset:
# Define a make dataset function that can be used now but also called later
# in the callback functions to save re-writing similar code later.
def make_dataset(color_to_plot, marker_to_plot, x_data1, y_data1):
# Create a sub dataframe
Sub_df1 = df.copy()
# Delete any rows in the sub-dataframes that do not exist in the
# checkboxes/default user choices. (e.g. if you've selected 6MV in the
# checkbox this will remove any rows that have something other than 6MV)
Sub_df1 = Sub_df1[Sub_df1['machinename'].isin(color_to_plot)]
Sub_df1 = Sub_df1[Sub_df1['machinename'].isin(marker_to_plot)]
# Search for the columns with the x_data and y_data names and replace
# them with 'x' and 'y'. Unless plotting the same data on both in which
# case add an extra column for 'y' that's a copy of 'x'
if x_data1 == y_data1:
Sub_df1.rename(columns={x_data1: 'x'}, inplace=True)
Sub_df1.loc[:, 'y'] = Sub_df1.loc[:, 'x']
else:
Sub_df1.rename(columns={x_data1: 'x'}, inplace=True)
Sub_df1.rename(columns={y_data1: 'y'}, inplace=True)
# Return the newly created Sub_df1
return Sub_df1
# Run the make_dataset function to create a sub dataframe that the plot will
# be made from.
Sub_df1 = make_dataset(color_to_plot, marker_to_plot, x_data1, y_data1)
# Create a Column Data Source. This is important as it is the data format
# needed for Bokeh. When making this it is useful to convert the dataframe
# into a dictionary, which seems to help with the callback function (see
# 'Common Issues' for details).
src1 = ColumnDataSource(Sub_df1.to_dict(orient='list'))
######### Make Plot:
# Create an empty plot (plot parameters will be applied later in a way that
# can be manipulated in the callbacks)
p1 = figure()
# Create a scatter plot.
p1.scatter( # source = The ColumnDataSource made above.
source=src1,
# x/y = 'x'/'y' which are fields that were renamed as such in
# the make_dataset function
x='x',
y='y',
# Some general parameters about marker size. These seem like
# reasonable values but possible could alter these in a
# callback?
fill_alpha=0.4,
size=12,
# Create the legend using the created fields added in the legend
# section. Use the factor_mark and factor_cmap functions to
# match the colors/markers to the right lists.
# NB: Always use legend_field for this not legend_group as the
# former acts on the javascript side but the latter the Python
# side. Therefore the former will update automatically when the
# plot is changed with no need for a callback.
legend_field='legend',
marker=factor_mark('machinename1', markers, list_formarker),
color=factor_cmap('machinename1', color_palette, list_forcolor))
######### Add plot parameters:
# Define a define plot parameters factor that can be used now but also
# called later in the callback functions.
def define_plot_parameters(list):
# Input is a List of format:
# list_plot_parameters = [ x_data1, y_data1,
# plot_title1, x_axis_title1, y_axis_title1,
# plot_size_height1, plot_size_width1,
# legend_location ]
# The parameters have to be controlled like this in a callback to allow
# for them to be adjusted. Otherwise the plot parameters are not
# interactive.
# Yes! - p1.xaxis.axis_label = 'X_axis_title'
# No! - p1 = figure(x_axis_label = 'X_axis_title')
p1.title.text = list[2]
p1.xaxis.axis_label = list[3]
p1.yaxis.axis_label = list[4]
p1.plot_height = list[5]
p1.plot_width = list[6]
p1.legend.location = list[7]
# If the user wants to plot an axis as datetime then the axis needs to
# be reformatted. Will do this by checking if the x_data1/y_data1 is
# =='adate'.
# NB: This only works if 'adate' is used as the name for the date column
# and also that this is the only date column.
if list[0] == 'adate':
p1.xaxis.formatter = DatetimeTickFormatter(days=['%d/%m', '%a%d'])
else:
p1.xaxis.formatter = BasicTickFormatter()
if list[1] == 'adate':
p1.yaxis.formatter = DatetimeTickFormatter(days=['%d/%m', '%a%d'])
else:
p1.yaxis.formatter = BasicTickFormatter()
return
# Run the define_plot_parameters function to format the plot.
define_plot_parameters(list_plot_parameters)
############################################################################
############################################################################
############################################################################
############################################################################
############################################################################
############################ ADD TOLERANCES ################################
# We defined the tolerances further up and now want to add the correct ones
# to the plot (having created the plot above). Again this will be done with
# functions and in a way that the functions can be used in the callbacks
# later.
#
# NB: At the moment this is still a bit of a work in progress and shows the
# way to add line tolerances. Another option might be to add colorblocks
# using varea and/or varea_stack.
#
# NB: Also this funcion assumes that tolerances will all be against one
# x_axis value (e.g. date). This is probably the majority of use cases but
# probably relatively trivial to add further toleraces against other x_axis
# data.
# Create a function that will create a dataframe that can be used to supply
# a plot of two tolerance lines. This will including 'appearing' and
# 'disappearing' depending on whether tolerances are defined or not.
def tolerances(x_data1, y_data1, Sub_df1, df_tol1):
# Get a list of the column headers from the tolerance table defined
# earlier.
headers1 = df_tol1.columns.values.tolist()
# Check if the xdata is what is in the df_tol1 as the x_axis (if not no
# point plotting tolerances as all tolerances are vs this tolerance).
if x_data1 != headers1[0]:
# x_data1 doesn't match so going to output something that should
# basically just not plot but also won't throw the viewing range.
data = {
'x': [Sub_df1['x'].max(), Sub_df1['x'].max()],
'y_low': [Sub_df1['y'].max(), Sub_df1['y'].max()],
'y_high': [Sub_df1['y'].max(), Sub_df1['y'].max()]
}
Sub_df1_tol1 = pd.DataFrame(data)
return Sub_df1_tol1
# Otherwise we have the right x_data1 so now just check if it's datetime
# or not.
if x_data1 == 'adate':
# It has the format 'adate' so should be datetime. So find the max
# min dates in the Sub_df1 and add a couple of weeks either side so
# that it plots the full range (plus a little bit for visualisation
# reasons).
max_x = Sub_df1['x'].max() + pd.DateOffset(weeks=2)
min_x = Sub_df1['x'].min() + pd.DateOffset(weeks=-2)
else:
# If it's not datetime then just add about 5% of the range to
# either side to make the plot look nicer.
# NB: This has not been checked extensively as most tolerances are
# vs. time.
max_x = Sub_df1['x'].max()
min_x = Sub_df1['x'].min()
range = max_x - min_x
max_x = max_x + (range / 20)
min_x = min_x - (range / 20)
# Used the x part so now remove the element from the list. This will
# help for the small case where x_data1 == ydata1.
headers1.remove(x_data1)
if y_data1 in headers1:
# If y_data1 is in the list then loop through to find out where and
# get the data from the tolerance dataframe.
for x in headers1:
if y_data1 == x:
# When the loop has found where it is then can output a
# dataframe of the form:
# x = [far left of plot, far right of plot]
# y_low = [low_tolerance, low_tolerance]
# y_high = [high_tolerance, high_tolerance]
data = {
'x': [min_x, max_x],
'y_low': [df_tol1[x][0], df_tol1[x][0]],
'y_high': [df_tol1[x][1], df_tol1[x][1]]
}
Sub_df1_tol1 = pd.DataFrame(data)
else:
# If y_data1 is not in the headers1 list then there are no
# tolerances to plot so going to output something that should
# basically just not plot but also won't throw the viewing range.
data = {
'x': [Sub_df1['x'].max(), Sub_df1['x'].max()],
'y_low': [Sub_df1['y'].max(), Sub_df1['y'].max()],
'y_high': [Sub_df1['y'].max(), Sub_df1['y'].max()]
}
Sub_df1_tol1 = pd.DataFrame(data)
return Sub_df1_tol1
return Sub_df1_tol1
def choose_tolerances(x_data1, y_data1, Sub_df1, color_to_plot):
if any(item in color_to_plot for item in
['TrueBeam B', 'TrueBeam C', 'TrueBeam D', 'TrueBeam F']):
# If this is true then will need to run the df_tol_TB tolerances
Sub_df1_tol_TB = tolerances(x_data1, y_data1, Sub_df1, df_tol_TB)
else:
data = {
'x': [Sub_df1['x'].max(), Sub_df1['x'].max()],
'y_low': [Sub_df1['y'].max(), Sub_df1['y'].max()],
'y_high': [Sub_df1['y'].max(), Sub_df1['y'].max()]
}
Sub_df1_tol_TB = pd.DataFrame(data)
if any(item in color_to_plot
for item in ['Linac B', 'Linac C', 'Linac D', 'Linac E']):
# If this is true then will need to run the df_tol_TB tolerances
Sub_df1_tol_Classic = tolerances(x_data1, y_data1, Sub_df1,
df_tol_Classic)
else:
data = {
'x': [Sub_df1['x'].max(), Sub_df1['x'].max()],
'y_low': [Sub_df1['y'].max(), Sub_df1['y'].max()],
'y_high': [Sub_df1['y'].max(), Sub_df1['y'].max()]
}
Sub_df1_tol_Classic = pd.DataFrame(data)
return Sub_df1_tol_TB, Sub_df1_tol_Classic
# Run the tolerances function to output the new dataframe
Sub_df1_tol_TB, Sub_df1_tol_Classic = choose_tolerances(
x_data1, y_data1, Sub_df1, color_to_plot)
# Turn the dataframe into a new ColumnDataSource (again turning it into a
# dictionary)
src1_tol_TB = ColumnDataSource(Sub_df1_tol_TB.to_dict(orient='list'))
src1_tol_Classic = ColumnDataSource(
Sub_df1_tol_Classic.to_dict(orient='list'))
# Add two lines to the plot using the new ColumnDataSource as the source,
# one line for low tolerance and one line for high.
p1.line(source=src1_tol_TB, x='x', y='y_low', color='firebrick')
p1.line(source=src1_tol_TB, x='x', y='y_high', color='firebrick')
p1.line(source=src1_tol_Classic, x='x', y='y_low', color='hotpink')
p1.line(source=src1_tol_Classic, x='x', y='y_high', color='hotpink')
############################################################################
############################################################################
############################################################################
################## ADD MORE COMPLEX TOOLS TO THE PLOT ######################
# Create tools here that will allow for some manipulation or inspection of
# plotted data.
#
# As an example a 'HoverTool' will be added to the plot.
#
# Other useful tools and details of the syntax can be found here:
# https://docs.bokeh.org/en/latest/docs/user_guide/tools.html
# Create the hover tool (see website above for syntax/details).
# This example creates a hover tool that displays:
# Date: The value of the data-point as measued on the x-axis
# (formatted for datetime)
# Y-Axis: The value of the data-point as measued on the y-axis
# (x,y): The x and y co-ordinates in plot space
# Chamber Comb.: The data stored under the 'Chamber' column for that
# data-point.
# Comments: The data stored under the 'comments' column for that
# data-point.
hover = HoverTool(tooltips=[('Date', '@x{%F}'), ('Y-Axis', '@y'),
('(x,y)', '($x, $y)'),
('Chamber Comb.', '@Chamber'),
('Comments', '@comments')],
formatters={'x': 'datetime'})
# Add the newly created tool to the plot.
p1.add_tools(hover)
############################################################################
############################################################################
############################################################################
################# CREATE WIDGETS TO BE ADDED TO THE PLOT ###################
# Create widgets here that will allow for some manipulation of the plotted
# data. These widgets provide an interactive ability that can alter the data
# that is plotted, provide update fnctions and call other programmatic
# functions. This is done either using built in Bokeh functionality or
# using more powerful but complex python and javascript based callbacks.
#
# As an example some 'Select' widgets, 'Checkbox' widgets and 'RangeSliders'
# will be added to the plot.
#
# Other useful widgets and details of the syntax can be found here:
# https://docs.bokeh.org/en/latest/docs/user_guide/interaction/widgets.html
######## 1)
# Create the select widget (see website above for syntax/details). This
# widget will be used for the callback example later to change data plotted
# on the x/y-axis.
# This example creates a select tool that displays:
# Dropdown list containing a list of every field that was downloaded from
# the database.
# NB: When making a list it may be worth manually creating it to limit
# it to the fields that can be plotted (e.g. not including fields
# like 'Comments'). This will shorten the dropdown list but you
# should err on the side of inclusion to make the final plot as
# flexible as possible.
#
# Create a list of the availible options
menu_axis = []
for field in TableFields:
menu_axis.append(field)
menu_axis = sorted(menu_axis)
# Select tool needs inputs for the title, a starting value and the just
# created list to supply the available options.
select_xaxis = Select(title='X-Axis Fields Available:',
value=x_axis_title1,
options=menu_axis)
select_yaxis = Select(title='Y-Axis Fields Available:',
value=y_axis_title1,
options=menu_axis)
######## 2)
# This select widget will be made in the same way and used to create a
# dropdown list to change the legend position.
#
# Create a list of the availible options
menu_legend = [
'top_left', 'top_center', 'top_right', 'center_left', 'center',
'center_right', 'bottom_left', 'bottom_center', 'bottom_right'
]
# Create the select tool as above
select_legend = Select(title='Legend Position',
value=legend_location,
options=menu_legend)
######## 3)
# These checkbox widgets will be used to create a tool to select the
# values that are being plotted from the fields that the legend is based on.
#
# NB: There is some built in Bokeh functionality for interavtive legends
# that can fulfill some of the same goals where the number of options is
# limited to something that can display on a reasonably sized legend. May
# be a better and more robust solution where possible.
# Create a list of all unique names in the column chosen to be matched to
# markers (sorted).
options_marker = sorted(df['machinename'].unique().tolist())
# Create an index list for all of the values that should be pre-ticked.
index_marker = [
i for i in range(len(options_marker))
if options_marker[i] in marker_to_plot
]
# Create the checkbox, providing the list of availible options and a list
# of what should be active (pre-ticked).
checkbox_marker = CheckboxGroup(labels=options_marker,
active=index_marker,
visible=False)
# Do the same for the column that was matched to colors.
options_color = sorted(df['machinename'].unique().tolist())
index_color = [
i for i in range(len(options_color))
if options_color[i] in color_to_plot
]
checkbox_color = CheckboxGroup(labels=options_color, active=index_color)
######## 4)
# Make some range sliders that will be used to manipulate the x-axis and
# y-axis range.
# Most of the manipulation will be done using a later function but will need
# to create the bare minimum rangeslider first that can later be manipulated
# (This seems to be the minimum number of parameters needed to create these
# widgets). Note that a RangeSliders AND a DateRangeSlider needs to be
# created for each axis.
range_slider_x = RangeSlider(title='X-Axis Range',
start=0,
end=1,
value=(0, 1),
step=0.1)
range_slider_y = RangeSlider(title='Y-Axis Range',
start=0,
end=1,
value=(0, 1),
step=0.1)
range_slider_xdate = DateRangeSlider(title='X-Axis Range (Date)',
start=date(2017, 1, 1),
end=date(2017, 1, 2),
value=(date(2017, 1,
1), date(2017, 1, 2)),
step=1)
range_slider_ydate = DateRangeSlider(title='Y-Axis Range (Date)',
start=date(2017, 1, 1),
end=date(2017, 1, 2),
value=(date(2017, 1,
1), date(2017, 1, 2)),
step=1)
# Define the function that will be used now and also in the callbacks later.
# This will allow the range_sliders to adjust to match any changes in the
# data being plotted on the x/y axis.
def range_slider(x_data1, y_data1, Sub_df1):
# Start with the y-axis.
# First need to check if 'adate' and if so edit the date range slider
# but otherwise edit the normal slider.
if y_data1 == 'adate':
# Set the start, end and value fields to the full range.
range_slider_ydate.start = Sub_df1['y'].min()
range_slider_ydate.end = Sub_df1['y'].max()
range_slider_ydate.value = (Sub_df1['y'].min(), Sub_df1['y'].max())
# Step to 1 works for DateRangeSlider
range_slider_ydate.step = 1
# Make the DateRangeSlider visible and hide the normal RangeSlider
range_slider_ydate.visible = True
range_slider_y.visible = False
else:
# Set the start, end and value fields to the full range.
range_slider_y.start = Sub_df1['y'].min()
range_slider_y.end = Sub_df1['y'].max()
range_slider_y.value = (Sub_df1['y'].min(), Sub_df1['y'].max())
# Step to range/10000 should give sufficient granularity
range_slider_y.step = (Sub_df1['y'].max() -
Sub_df1['y'].min()) / 100000
# Make the normal RangeSlider visible and hide the DateRangeSlider
range_slider_y.visible = True
range_slider_ydate.visible = False
# Do the same for the x-axis
if x_data1 == 'adate':
range_slider_xdate.start = Sub_df1['x'].min()
range_slider_xdate.end = Sub_df1['x'].max()
range_slider_xdate.value = (Sub_df1['x'].min(), Sub_df1['x'].max())
range_slider_xdate.step = 1
range_slider_xdate.visible = True
range_slider_x.visible = False
else:
range_slider_x.start = Sub_df1['x'].min()
range_slider_x.end = Sub_df1['x'].max()
range_slider_x.value = (Sub_df1['x'].min(), Sub_df1['x'].max())
range_slider_x.step = (Sub_df1['x'].max() -
Sub_df1['x'].min()) / 100000
range_slider_x.visible = True
range_slider_xdate.visible = False
return
# Run the function.
range_slider(x_data1, y_data1, Sub_df1)
############################################################################
############################################################################
############################################################################
########################### CREATE A LAYOUT ################################
# Create a layout to add widgets and arrange the display. This simple layout
# displays the select widgets above the plot with the checkboxes to the
# right (one above the other).
#
# More details can be found at:
# https://docs.bokeh.org/en/latest/docs/user_guide/layout.html
#
# NB: More work to do here to make plots responsive to browser window size
# (e.g. using sizing_mode = scale_both) but need to invstigate with/without
# remote desktops.
layout_checkbox = column([checkbox_marker, checkbox_color])
layout_plots = column([
select_xaxis, select_yaxis, select_legend, range_slider_x,
range_slider_y, range_slider_xdate, range_slider_ydate, p1
])
tab_layout = row([layout_plots, layout_checkbox])
############################################################################
############################################################################
############################################################################
####################### CREATE CALLBACK FUNCTIONS ##########################
# CAVEAT: Callback functions are very complex and below is my (CB) rough
# understanding of how they function based mainly on experience/trial and
# error while writting these functions for other graphs. It should be taken
# as a starting point but not as a definitive user guide.
#
# Callback functions are very powerful and can be based off of javascript or
# python. The example presented here uses python but in future a javascript
# copy should also be added.
######## 1)
# This callback is designed to take inputs from the select and checkbox
# widgets update the graph to plot the new data requested by the user.
#
# Syntax:
# attr = The value passed from the on_change function before the callback
# was named (e.g. in this example attr = 'value')
# old = The value of the widget before it was changed (I.e. If a select
# widget is changed from 'Output' to 'T/P Correction', then
# old = 'Output'
# new = The value of the widget after it was changed (I.e. If a select
# widget is changed from 'Output' to 'T/P Correction', then
# old = 'T/P Correction'
#
# NB: In general seen little need to use these inputs as you can generally
# access the value of the widgets directly which seems to be more powerful
# and flexible
#
# First define the callback function.
def callback(attr, old, new):
# Want to acquire the current values of all of the checkboxes and select
# widgets to provide as inputs for the re-plot. For the checkboxes this
# means itterating through the active list and outputting the labels
# that are active
color_to_plot = [
checkbox_color.labels[i] for i in checkbox_color.active
]
marker_to_plot = color_to_plot
plot1_xdata_to_plot = select_xaxis.value
plot1_ydata_to_plot = select_yaxis.value
legend_location = select_legend.value
# Use the pre-defined make_dataset function with these new inputs to
# create a new version of the sub dataframe.
Sub_df1 = make_dataset(color_to_plot, marker_to_plot,
plot1_xdata_to_plot, plot1_ydata_to_plot)
# Use the pre-defined define_plot_parameters function with these new
# inputs to update the plot parameters.
x_axis_title1 = plot1_xdata_to_plot
y_axis_title1 = plot1_ydata_to_plot
define_plot_parameters([
plot1_xdata_to_plot, plot1_ydata_to_plot, plot_title1,
x_axis_title1, y_axis_title1, plot_size_height1, plot_size_width1,
legend_location
])
# Use the pre-defined tolerances function with these new inputs to
# make a new version of the tolerances sub dataframe.
Sub_df1_tol_TB, Sub_df1_tol_Classic = choose_tolerances(
plot1_xdata_to_plot, plot1_ydata_to_plot, Sub_df1, color_to_plot)
# Use the pre-defined range_slider function with these new inputs to
# update the range sliders (this will make sure that the range sliders
# start/end etc. match up with what's being plotted, as well as
# displaying/hiding the RangeSlider/DateRangeSlider as needed
range_slider(plot1_xdata_to_plot, plot1_ydata_to_plot, Sub_df1)
# Update the ColumnDataSources using the newly created dataframes. The
# plots look to these as the source so this changes what is being
# plotted.
src1.data = Sub_df1.to_dict(orient='list')
src1_tol_TB.data = Sub_df1_tol_TB.to_dict(orient='list')
src1_tol_Classic.data = Sub_df1_tol_Classic.to_dict(orient='list')
return
# Use the on_change function to call the now defined callback function
# whenever the user changes the value in the widget.
# NB: Other functions such as on_click are availible for other widgets.
# Syntax:
# First argument is passed to the callback as attr (see callback section
# above)
# Second argument is the name of the callback function to be called.
select_xaxis.on_change('value', callback)
select_yaxis.on_change('value', callback)
select_legend.on_change('value', callback)
checkbox_color.on_change('active', callback)
checkbox_marker.on_change('active', callback)
######## 2)
# This callback is designed to take inputs from the range sliders to change
# visible range
def callback_range(attr, old, new):
# Check what is currently being plotted. Need this to know whether to
# look for the values from the DateRangeSlider or the RangeSlider
plot1_xdata_to_plot = select_xaxis.value
plot1_ydata_to_plot = select_yaxis.value
# Start with the x-axis
if plot1_xdata_to_plot == 'adate':
# If it's 'adate' then need to look at the DateRangeSlider and
# update the start and end values of the range using the values from
# the slider.
# NB: range_slider.value = left_value, right_value
p1.x_range.start, p1.x_range.end = range_slider_xdate.value
else:
# If it's not 'adate' then need to look at the normal RangeSlider
p1.x_range.start, p1.x_range.end = range_slider_x.value
# Do the same for the y-axis
if plot1_ydata_to_plot == 'adate':
p1.y_range.start, p1.y_range.end = range_slider_ydate.value
else:
p1.y_range.start, p1.y_range.end = range_slider_y.value
return
# Use the on_change function to call the now defined callback function
# whenever the user changes the value in the widget.
range_slider_x.on_change('value', callback_range)
range_slider_y.on_change('value', callback_range)
range_slider_xdate.on_change('value', callback_range)
range_slider_ydate.on_change('value', callback_range)
############################################################################
############################################################################
############################################################################
####################### RETURN TO THE MAIN SCRIPT ##########################
# Now that the script is finished and the plot created we can return to the
# main script.
#
# To pass back the data for the tab we need to return a Panel with:
# child = layout (the one that we made earlier with the widget and plot)
# title = 'Something that makes sense as a tab label for the user'
return Panel(child=tab_layout, title='Electron Energy')
def first_tab_create(filterData):
########method1: create data source for plots
# dummy data that will be replaced by button values once we get those implemented (right now only granulaity button is implemented)
all_min_date = filterData.groupby('dataid').agg(min)["time"]
all_max_date = filterData.groupby('dataid').agg(max)["time"]
dummy_daterange = ['2019-05-01', '2019-08-20']
dummy_home_id = 27
dummy_data_type = 'car1'
dummy_granularity = '15 Minutes'
def plot1_data(house, daterange=dummy_daterange, data=dummy_data_type, xaxis=dummy_granularity):
# house is an integer number ex. 27
# daterange is an array with 2 strings, start date and end date. ex. ['2019-05-01','2019-08-09']
# weekdays is an ordered list 0-6 of integers ex. [1,4,6] (these are the days we want to exclude)
# data is a string ex. 'car1'
# xaxis is also a string ex. 'hour'
houseData = filterData[filterData['dataid'] == house].sort_values('time', ascending=True)[[data, 'time']]
# that cuts the house, sorts by ascending time, and pulls out only the type of data that was requested
houseData.index = houseData['time'] # reindex by the datetime
houseData = houseData.loc[daterange[0]:daterange[1], :] # cut to the days requested
# Now we get into the xaxis user options #
if xaxis == '15 Minutes':
houseData = houseData.drop(columns="time")
if xaxis == 'Hour':
houseData = houseData.resample('1h').mean()
houseData[data]=houseData[data]*3600/3600
if xaxis == 'Day':
houseData = houseData.resample('1d').mean()
houseData[data] = houseData[data] * 3600 / 3600 * 24
if xaxis == 'Week':
houseData = houseData.resample('1w').mean()
houseData[data] = houseData[data] * 3600 / 3600 *24 * 7
if xaxis == 'Month':
houseData = houseData.resample('1m').mean()
houseData[data] = houseData[data] * 3600 / 3600 *24 * 7 * 30 ############this computation is wrong because n stadard month but just go with it for now
# if none of these, 15 Minutes is implied and passed through
return ColumnDataSource(houseData)
def plot2_data(house,daterange=dummy_daterange,weekdays = [],data=dummy_data_type,xaxis=dummy_granularity):
houseData = filterData[filterData['dataid'] == house].sort_values('time', ascending = True)[[data,'time']]
# that cuts the house, sorts by ascending time, and pulls out only the type of data that was requested
houseData.index = houseData['time'] # reindex by the datetime
houseData = houseData.loc[daterange[0]:daterange[1],:] # cut to the days requested
for i in weekdays:
houseData = houseData[houseData['time'].dt.dayofweek != i] # cut out days we dont want
if xaxis == 'avgday':
houseData = houseData.resample('1d').sum() ####chjange to mean
houseData['time'] = houseData.index
houseData = houseData.groupby(houseData['time'].dt.dayofweek)[data].mean()
houseData.index = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']
if xaxis == 'avghour':
houseData = houseData.resample('1h').mean()
houseData['time'] = houseData.index
houseData = houseData.groupby(houseData['time'].dt.hour)[data].mean() # Does not account for UTC change!
houseData = pd.DataFrame(data = houseData)
houseData['axis'] = houseData.index
return ColumnDataSource(houseData)
# now we need to set up our plot axis
##########method2: create plots
def plot1_plot(src):
# create the plot every time a change occurs
plot1 = figure(title="Energy Consumption Per Period", x_axis_type="datetime", x_axis_label="Date",
y_axis_label="Energy Consumption [kWh]")
plot1.line('time', "grid", source=src, ) # simple line plot #data_type_available[data_type_selector.active]
return plot1 # plot object type
def plot2_plot(src):
plot2 = figure(x_range = (0,23), y_range=(-7,2), title='Average Power Consumption For Hours in a Day') # gotta do the ranges as mins and maxes
plot2.vbar(x='axis', top = 'grid', width=1, source=src)
plot2.yaxis.axis_label = 'Power Consumption [kW]'
plot2.xaxis.axis_label = 'Hour of the Day'
#plot2 = figure(title = 'PLOT2')
#plot2.line('axis','grid', source = src)
return plot2
#########Method3: Update App
def update(attr, old, new): # still a little unsure how the update function gets values passed in implicitly
# these values to be replaced with button/user inputs
home_id_to_plot = 27
daterange_to_plot = ['2019-05-01', '2019-08-20']
data_type_to_plot = 'grid'
exclude_days_to_plot = []
avg_to_plot = 'avghour'
# home_id_to_plot = int(home_id_selector.value)
# print(attr, old, new)
# if home_id_selector.value == new:
# daterange_to_plot = [all_min_date[home_id_to_plot].strftime("%Y-%m-%d"), all_max_date[home_id_to_plot].strftime("%Y-%m-%d")]
# date_range_slider.start=all_min_date[home_id_to_plot]
# date_range_slider.end=all_max_date[home_id_to_plot]
# date_range_slider.value=(all_min_date[home_id_to_plot],all_max_date[home_id_to_plot])
# print("if 1 truth")
# else:
daterange_raw = list(date_range_slider.value_as_datetime)
daterange_to_plot = [daterange_raw[0].strftime("%Y-%m-%d"), daterange_raw[1].strftime("%Y-%m-%d")]
# print("falsyy")
# date_range_slider.update(start=all_min_date[home_id_to_plot], end=all_max_date[home_id_to_plot],
# value=(all_min_date[home_id_to_plot], all_max_date[home_id_to_plot]))
# print("after:",new)
# print(attr,old,new)
# daterange_to_plot=list(string_date1,string_date2)
# daterange_to_plot=dummy_daterange
data_type_to_plot = "grid" # data_type_available[data_type_selector.active]
# only working button call so far
granularity_to_plot = granularity_1.labels[granularity_1.active]
# granularity_to_plot= [granularity_1.labels[i] for i in granularity_1.active]
print("before data create")
# create a new data source
new_src1 = plot1_data(home_id_to_plot, daterange=daterange_to_plot, data=data_type_to_plot,
xaxis=granularity_to_plot)
new_src2 = plot2_data(home_id_to_plot, daterange=daterange_to_plot,
weekdays=exclude_days_to_plot, data=data_type_to_plot,xaxis=avg_to_plot)
print("before data update")
# push new data to the source data the rest of the app is usig for plot1
src1.data.update(new_src1.data)
src2.data.update(new_src2.data)
print("updated data")
# def new_home(attr,old,new):
# return
############# Add widgets
# only the granularity implemented so far
granularity_1 = RadioGroup(
labels=["15 Minutes", "Hour", "Day", "Week", "Month"], active=0)
granularity_1.on_change('active',
update) # not sure exactly how this works but runs update on the change of the button and passes through the value of the button
home_ids_available = np.unique(filterData['dataid'])
home_ids_available= list(map(str, home_ids_available))
home_id_selector = Dropdown(label="Home ID to Plot", button_type="warning", menu=home_ids_available)
home_ids_available = list(map(str, home_ids_available))
home_id_selector = Dropdown(label="Home ID to Plot", button_type="warning", menu=home_ids_available, value="27")
# home_id_selector.on_change('value', update) #############put back in later!!!!1
date_range_slider = DateRangeSlider(title="Date Range: ", start=date(2019, 5, 1), end=date(2019, 8, 20),
value=(date(2019, 5, 1), date(2019, 8, 20)), step=1, callback_policy = 'mouseup')
date_range_slider.on_change("value", update)
# data_type_selector = RadioButtonGroup(labels=["Net Home Consumption","Solar Generation","EV Consumption"],active=0)
# data_type_selector.on_change('active',update)
# data_type_available=["grid","solar","car1"]
############ Initialize opening plot and data
src1 = plot1_data(int(home_ids_available[0]), ['2019-05-01', '2019-08-20'], 'grid',
'15 Minutes') # start with a data range we know is correct
plot1 = plot1_plot(src1)
src2 = plot2_data(27,['2019-05-01', '2019-08-20'],[],'grid','avghour')
plot2 = plot2_plot(src2)
##### Formatting of the app screen
# Put controls in a single element (add more later to format)
controls = WidgetBox(granularity_1, home_id_selector, date_range_slider) # data_type_selector)
# Create a row layout
layout = row(controls, column(plot1,plot2))
# Make a tab with the layout
tab = Panel(child=layout, title='First Tab')
return tab
x_start = datetime.fromtimestamp(date_slider.value[0]/1000)
x_end = datetime.fromtimestamp(date_slider.value[1]/1000)
x_start = pd.to_datetime(x_start)
x_end = pd.to_datetime(x_end)
#print(x_start)
#print(x_end)
# Generate new data
new_df = df[(df['x']>= x_start) & (df['x'] <= x_end)]
new_df.loc[:,'y1'] = (new_df['y1'].pct_change().fillna(0) + 1).cumprod() * 100
new_df.loc[:,'y2'] = (new_df['y2'].pct_change().fillna(0) + 1).cumprod() * 100
new_df.loc[:,'dd'] = drawdown(new_df['y1'].values).values
new_df = new_df.reset_index().iloc[:,1:]
newdata = ColumnDataSource(new_df)
source.data = newdata.data
date_slider.on_change('value', update_data)
plots = column(p1, p2, date_slider)
panel_1 = Panel(child = plots, title='Portfolio Performance')
# 히트맵 그림 그리기
try:
from functools import lru_cache
except ImportError:
# Python 2 does stdlib does not have lru_cache so let's just
# create a dummy decorator to avoid crashing
print ("WARNING: Cache for this example is available on Python 3 only.")
def lru_cache():
def dec(f):
def _(*args, **kws):
class SoftFocus(object):
"""class to view and process bokeh sample data using a bokeh server.
When within its parent folder, open your terminal and write:
bokeh serve softfocus
see module doc
"""
def __init__(self):
# put the controls and the table in a layout and add to the document
self.document = curdoc()
#following method parses arguments and create the layout
# (in self.layout) with the main tab
self.create()
logger.info('layout created')
#add the layout to the document
self.document.title = "Soft Focus"
self.document.add_root(self.layout)
#show main table
# self.update()
# logger.info('table shown')
#add a callback called every hour to delete excessive amount of xlsx
# in the /static/uploads folder, where uploads are
self.document.add_periodic_callback(self._delete_excess_xlsx,3600000)
#variable holding app status
self.sel_csv = None#selected row from main table
#dicts hold data from all opened tabs
self.plot_dfs = dict()
def create(self):
"""parse the bokeh serve arguments then create the main layout
To create the main layout, the method _create_folder is called. Other
methods could be called depending on the argument if we want to fetch
data with different methods, e.g. _create_sql
"""
if len(sys.argv)>2:
print('Syntax for default Bokeh sampledata'
' folder: bokeh serve {}'.format(sys.argv[0]))
print('Syntax for own folder: bokeh serve'
' {} --args <folder/>'.format(sys.argv[0]))
sys.exit(0)
elif len(sys.argv)==1:
data_dir = os.path.join(CURRENT_DIR,'..','tests')
if (not os.path.exists(data_dir)
or (len(os.listdir(data_dir))<1)
):
logger.info('Creating new test folder...')
logger.info('{0}'.format(data_dir))
os.mkdir(data_dir)
from create_random import create_random
create_random(data_dir)
elif len(sys.argv)==2:
data_dir = sys.argv[1]
if not os.path.isdir(data_dir):
print("fpath must be a string indicating"
" a directory path")
sys.exit(0)
#other arguments could be processed to call different methods
self._create_folder(data_dir)
def _create_folder(self,data_dir):
"""
create softfocus instance based on folder data
"""
#list only csv files, populate a dict with general info about the files
logger.info('Database in a csv folder: {0}'.format(data_dir))
list_dir = os.listdir(data_dir)
csv_dic = {'CSV': [csv for csv in list_dir if csv.endswith('.csv')],
'size (kB)':[],
'last modification':[],
'number of columns':[],
}
if len(csv_dic)<1:
logger.warning("no csv file found in folder. Exit")
sys.exit(0)
for csv in csv_dic['CSV']:
csv_stat = os.stat(os.path.join(data_dir,csv))
csv_dic['size (kB)'].append(csv_stat.st_size/1024)
csv_dic['last modification'].append(
date.fromtimestamp(csv_stat.st_mtime)
)
with open(os.path.join(data_dir,csv),'rb') as f:
csv_dic['number of columns'].append(
len(f.readline().decode().split(','))
)
#make bokeh source from the dic
self.df = pd.DataFrame(csv_dic)
self.main_source = ColumnDataSource(self.df)
#### some widgets to filter the table ####
#date selector
last_date = self.df['last modification'].max()
first_date = self.df['last modification'].min()
if last_date == first_date:
last_date = first_date + timedelta(days=1)
self.date_slider = DateRangeSlider(title='Start date',
start=first_date,
end=last_date,
value=(first_date,last_date),
step=1)
self.date_slider.on_change('value',
lambda attr, old, new: self.update())
#byte size selection through text input
self.size_inputtext = TextInput(title='size in kbytes')
self.size_inputtext.value = "fmt: '100' or '10..200'"
self.size_inputtext.on_change('value',
lambda attr, old, new: self.update())
#filter by file name
self.csvname_text = TextInput(title='Testname')
self.csvname_text.on_change('value',
lambda attr, old, new: self.update())
#button to plot
self.plot_button = Button(label="Plot", button_type="success")
self.plot_button.on_click(self.add_plot_tab)
self.plot_button.disabled = True#active only when csv is selected
#make table widget
#table formatting
columns = []
for c in self.df.columns.tolist():
if c in ['last modification']:
columns.append(TableColumn(field=c,title=c,
formatter=DateFormatter(format="ISO-8601")))
else:
columns.append(TableColumn(field=c,title=c))
self.data_table = DataTable(source=self.main_source,
columns=columns,
width=800,
index_position=None,
editable=False,
)
self.data_table.source.on_change('selected',self.sel_table)
#controls in a box
controls = widgetbox(self.date_slider,
self.plot_button,
self.size_inputtext,
self.csvname_text,
)
#data table in its own box
table = widgetbox(self.data_table)
#insert all widgets in a Panel
tab1 = Panel(child=row(controls, table),title="CSVs",closable=False)
#single tab for now
self.tabs = Tabs(tabs=[tab1],
sizing_mode = 'stretch_both')
#need to add this callback otherwise the table will turn invisible
#after coming back to this main tab
self.tabs.on_change('active',
self.changed_tab_cb)
#add a status text above all tabs
self.info_text = Div(text='<font color="green">ready.</font>',
sizing_mode= "stretch_both",
height=25)
#main layout
self.layout = column([self.info_text,self.tabs])
# main data folder
self.data_dir = data_dir
def _create_sql(self,fpath=r"sql_db.ini"):
"""NOT IMPLEMENTED, called to link to an SQL database"""
pass
def _wait_message_decorator(f):
"""prints loading status during loading time
Add this decorator before any methods used as callbacks
This will indicate the user to wait or outputs errors
"""
#https://stackoverflow.com/questions/1263451/python-decorators-in-classes
def wait_please(*args,**kwargs):
self = args[0]
self.info_text.text = '<font color="orange">loading, please wait...</font>'
try:
r = f(*args,**kwargs)
except:
import traceback
err, val, tb = sys.exc_info()
logger.error(("Unexpected error:{0}\n"
"Error value: {1}\n"
"Error traceback: {2}\n"
"In function {3}").format(err,
val,
''.join(traceback.format_tb(tb)),
f))
self.info_text.text = (
'<font color="red">'
'Error: {0}'
'</font>').format(traceback.format_exception_only(err,val)[0])
return
self.info_text.text = '<font color="green">ready.</font>'
return r
return wait_please
def changed_tab_cb(self, attr, old, new):
"""
Callback called when another tab is selected
"""
if new ==0:#main tab
self.update()
#call function when selection on table
def sel_table(self, attr, old, new):
"""
Selection of a cell/row in a tab
"""
sels = self.data_table.source.selected['1d']['indices']
if sels:#if not empty
self.plot_button.disabled = False
self.sel_csv = self.main_source.data['CSV'][sels[0]]
else:
self.sel_csv = None
self.plot_button.disabled = True
#define callback function to show new table
@_wait_message_decorator
def update(self, attr=None, old=None, new=None):
"""
Callback function to show the main table with all tests
"""
df = self.df
filt = ((df['last modification']
>= self.date_slider.value_as_datetime[0])
& (df['last modification']
<= self.date_slider.value_as_datetime[1]))
try:
szfilt = [int(i) for i in self.size_inputtext.value.split('..')]
if len(szfilt)==2:
szfilt_max = max(szfilt)
szfilt_min = min(szfilt)
filt &= ((df['size(kB)'] >= szfilt_min)
&(df['size(kB)'] <= szfilt_max))
elif len(szfilt)==1:
szfilt = szfilt[0]
filt &= (df['size(kB)'] == szfilt)
else:
self.size_inputtext.value = "fmt: '100' or '98..102'"
except:
self.size_inputtext.value = "fmt: '100' or '98..102'"
try:
filt &= df['CSV'].str.contains(self.csvname_text.value,na=False)
except:
self.csvname_text.value = ''
current = df[filt]
current = current.fillna('NaN')
self.main_source.data = current.to_dict('list')
#callback function to add a plot tab
@_wait_message_decorator
def add_plot_tab(self):
"""
Callback function to add a new tab with a plot.
Each tab is differenciated by its name. The name is the csv file name
"""
#check if at least one line is selected
if not self.sel_csv:
self.sel_table(None,None,None)
return
#plot controls
logger.info("adding plot of {0}".format(self.sel_csv))
plot_df = pd.read_csv(os.path.join(self.data_dir, self.sel_csv),
parse_dates=True,
infer_datetime_format=True)
self.plot_dfs[self.sel_csv] = plot_df
cols = plot_df.columns.tolist()
x_sel = Select(title='X-Axis',
value=cols[0],
options=cols,
name='x_sel')
y_sel = Select(title='Y-Axis',value=cols[1],options=cols,
name='y_sel')
y_sel2 = Select(title='Y-Axis 2',value='None',options=cols+['None'],
name='y_sel2')
#exit button
exit_b = Button(label="Exit", button_type="success")
exit_b.on_click(self.remove_current_tab)
#download button
download_b = Button(label="Download", button_type="success",
name='download_b')
download_b.on_click(self.download)
download_b.tags = [0]
#plot button
plot_b = Button(label="Plot", button_type="success",name='plot_b')
plot_b.on_click(self.update_plot)
#text to indicate widgets manipulating the plot only
plot_group_text = Div(text='<b>Plot properties</b>')
#dummy idea from https://stackoverflow.com/questions/44212250/bokeh-widgets-call-customjs-and-python-callback-for-single-event
#the javascript callback is linked to the tag attribute of the download
#button (download_b.tag).
#To activate the download, download_b.tag needs to change, then
#./static/uploads/sessionid_output.xlsx is downloaded, where sessionid
#is the id of the current session.
JScode_fetch = """
var filename = t.name;//file name on client side
var get_path = '/softfocus/static/uploads/';//file path on server side
var session_id = t.tags[0];
get_path = get_path.concat(session_id);
get_path = get_path.concat('_output.xlsx')
filename = filename.concat('.xlsx');
fetch(get_path, {cache: "no-store"}).then(response => response.blob())
.then(blob => {
//addresses IE
if (navigator.msSaveBlob) {
navigator.msSaveBlob(blob, filename);
}
else {
var link = document.createElement("a");
link = document.createElement('a')
link.href = URL.createObjectURL(blob);
window.open(link.href, '_blank');
link.download = filename
link.target = "_blank";
link.style.visibility = 'hidden';
link.dispatchEvent(new MouseEvent('click'))
URL.revokeObjectURL(url);
}
return response.text();
});
"""
#plot controls together in a box
controls = widgetbox(plot_group_text,x_sel,y_sel,y_sel2,plot_b)
#tab panel for this plot, differenciated with its name
plot_tab = Panel(child=row(column(controls,download_b,exit_b),
Spacer(height=600,
width=600)
),
title="Plot {}".format(self.sel_csv),
closable=True,
name=str(self.sel_csv))#name of tab is csv filename
session_id= str(self.document.session_context._id)
plot_tab.tags = [session_id]
download_b.js_on_change('tags',CustomJS(args=dict(t=plot_tab),
code=JScode_fetch))
self.tabs.tabs.append(plot_tab)
self.create_plot_figure(plot_tab)
@_wait_message_decorator
def update_plot_source(self, attr=None, old=None, new=None):
"""
filter source
Not implemented yet
"""
tab_ix = self.tabs.active
test = self.tabs.tabs[tab_ix].name
source = self.ly[test].data_source
pass
@_wait_message_decorator
def update_plot(self):
"""
Get active tab then create/update its plot
"""
tab_ix = self.tabs.active
active_tab = self.tabs.tabs[tab_ix]
#col of widgets in place 0, plot in place 1
self.create_plot_figure(active_tab)
def create_plot_figure(self, active_tab):
"""
create a new plot and insert it in given tab.
"""
#find table name of active tab and its bokeh instances
test = active_tab.name#contains csv filename
x_sel=active_tab.select_one({'name':'x_sel'})
y_sel=active_tab.select_one({'name':'y_sel'})
y_sel2=active_tab.select_one({'name':'y_sel2'})
plot_df = self.plot_dfs[test]
source = ColumnDataSource(plot_df)
#Replace entirely p with a new plot
p = Plot(
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=600,
plot_width=600,
title=Title(text=self.sel_csv),
name='plot')
p.add_tools(BoxZoomTool(),
SaveTool(),
ResetTool(),
PanTool(),
HoverTool(tooltips=[('x','$x'),
('y','$y')]))
#see https://bokeh.github.io/blog/2017/7/5/idiomatic_bokeh/
x_axis = LinearAxis(
axis_label = x_sel.value,
ticker=BasicTicker(desired_num_ticks =10),
name='x_axis')
y_axis = LinearAxis(
axis_label = y_sel.value,
ticker=BasicTicker(desired_num_ticks =10),
name='y_axis')
#primary y-axis
ly = p.add_glyph(source,
Line(x=x_sel.value,
y=y_sel.value,
line_width=2,
line_color='black'),
name = 'ly'
)
p.add_layout(x_axis,'below')
p.add_layout(y_axis,'left')
p.y_range.renderers = [ly]
#secondary y-axis
if y_sel2.value.strip() != 'None':#secondary y-axis
y_axis2 = LinearAxis(
axis_label = y_sel2.value,
ticker=BasicTicker(desired_num_ticks=10),
name='y_axis2',
y_range_name='right_axis')
p.add_layout(y_axis2,'right')
p.extra_y_ranges = {"right_axis": DataRange1d()}
ly2 = p.add_glyph(source,
Line(x=x_sel.value,
y=y_sel2.value,
line_width=2,
line_color='red'),
y_range_name='right_axis',
name = 'ly2'
)
p.extra_y_ranges['right_axis'].renderers = [ly2]
leg_items = [LegendItem(label=y_sel.value,
renderers=[ly]),
LegendItem(label=y_sel2.value,
renderers=[ly2])]
else:
leg_items = [LegendItem(label=y_sel.value,
renderers=[ly])]
p.add_layout(Legend(items=leg_items,
location='top_right')
)
active_tab.child.children[1] = p
return p
#callback function to remove a tab
def remove_current_tab(self):
"""
Callback function to remove a tab
"""
tab_ix = self.tabs.active
if tab_ix == 0:
return#do nothing if main tab where all tests are
#self.tabs.tabs.pop(tab_ix)
del self.tabs.tabs[tab_ix]
@_wait_message_decorator
def download(self):
tab_ix = self.tabs.active
active_tab = self.tabs.tabs[tab_ix]
test = self.tabs.tabs[tab_ix].name#contains csv filename
download_b = active_tab.select_one({'name':'download_b'})
p=active_tab.select_one({'name':'plot'})
session_id= str(self.document.session_context._id)
ly = p.select_one({'name':'ly'})
data = pd.DataFrame(ly.data_source.data)
dirpath = os.path.join(os.path.dirname(__file__),'static','uploads')
if not os.path.exists(dirpath):
os.makedirs(dirpath)
xlsxpath = os.path.join(dirpath,session_id+'_output.xlsx')
if os.path.exists(xlsxpath):
os.remove(xlsxpath)
writer = pd.ExcelWriter(xlsxpath,
engine='xlsxwriter')
logger.info('Test name: {0}'.format(test))
data.to_excel(writer,'data'+test)
# infos.to_excel(writer,'info'+infos['Testname'])
writer.close()
#change tag to activate JS_fetch callback
download_b.tags = [download_b.tags[0]
+ pd.np.random.choice([-1,1],size=1)[0]]
# @gen.coroutine
def _delete_excess_xlsx(self):
"""deletes all xlsx files in the upload static folder older than 24h"""
dirpath = os.path.join(os.path.dirname(__file__),'static','uploads')
now = time.time()
for dirpath, dirnames, filenames in os.walk(dirpath,topdown=False):
for fname in filenames:
fpath = os.path.join(dirpath,fname)
file_age = (now - os.path.getatime(fpath))/3600
if ((file_age>24) and fpath.endswith('output.xlsx')):
os.remove(fpath)
class CityBikeAnalysis:
def __init__(self,
log_file,
city='Oslo',
coord_mapping_file='',
altitude_file=''):
self.city = city
self.station_info_file = 'station_info.csv'
self.df_s_info = pd.read_csv(self.station_info_file, index_col=0)
self.df_s_info.index = self.df_s_info.index.astype(str)
self.tz_offset = pd.Timedelta('0h')
self.log_file = log_file
self.df_log = pd.read_csv(log_file)
self.re_index()
self.stations = [
str(s_id) for s_id in np.sort(self.df_log['station_id'].unique())
]
self.df_bikes_avail = pd.DataFrame()
self.df_docs_avail = pd.DataFrame()
self.pivot_df()
self.df_bikes_source = pd.DataFrame()
self.df_docks_source = pd.DataFrame()
if os.path.isfile(altitude_file):
self.df_altitude = pd.read_csv(altitude_file, index_col=0)
self.id_coord_mapping_df = pd.DataFrame()
if os.path.isfile(coord_mapping_file):
s_mod = os.path.getmtime(coord_mapping_file)
s_now = datetime.datetime.now().timestamp()
if s_mod < (s_now - 3600):
self.id_coord_mapping_df = self.make_station_coord_transformed_df(
self.get_station_info())
with open(coord_mapping_file, 'wb') as handle:
pickle.dump(self.id_coord_mapping_df,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
# TODO Do not overwrite?
else:
with open('mapping_df.pickle', 'rb') as handle:
self.id_coord_mapping_df = pickle.load(handle)
if city == 'Oslo':
self.mid_lat, self.mid_lon = pyproj.transform(
pyproj.Proj(init='epsg:4326'), pyproj.Proj(init='epsg:3857'),
10.735, 59.928) # Center of map location
self.coord_range_delta = 7500
elif city == 'NewYorkCity':
self.mid_lat, self.mid_lon = pyproj.transform(
pyproj.Proj(init='epsg:4326'), pyproj.Proj(init='epsg:3857'),
-73.973, 40.736)
self.coord_range_delta = 12500
# Colors
self.empty_color = "#7e7e7e"
self.avail_color = "#87e03e"
self.full_color = "#ff2332"
x_mul_sel_opt = [(s, s + ' | ' + name) for s, name in zip(
self.stations,
self.df_s_info.reindex(self.stations, fill_value='Unknown')
['name'])]
self.x_mul_select = MultiSelect(
title="Stations",
value=[self.stations[0], self.stations[1]],
options=x_mul_sel_opt,
size=8)
self.x_mul_select.on_change('value', self.x_mul_select_callback)
s_date = datetime.date.fromtimestamp(
self.df_bikes_avail.index[0].value / 10**9)
e_date = datetime.date.fromtimestamp(
self.df_bikes_avail.index[-1].value / 10**9)
e_date += datetime.timedelta(days=1)
self.dr_slider = DateRangeSlider(
title='Date Range',
start=s_date,
end=e_date,
step=24,
value=(e_date - datetime.timedelta(days=5), e_date),
callback_throttle=500)
self.dr_slider.on_change('value', self.date_rage_slider_callback)
self.range_slider = RangeSlider(title='Time of day',
start=5,
end=25,
step=1,
value=(5, 25))
self.range_slider.on_change('value', self.hour_rage_slider_callback)
self.histogram_policy = RadioButtonGroup(
labels=["Aggregate", "Flatten"], active=0)
self.histogram_policy.on_change('active',
self.histogram_policy_callback)
self.x_select = Select(title="X-axis",
value=self.stations[0],
options=self.stations)
self.x_select.on_change('value', self.x_select_callback)
self.y_select = Select(title="Y-axis",
value=self.stations[1],
options=self.stations)
self.y_select.on_change('value', self.y_select_callback)
self.create_sliced_df()
self.station_history_fig, self.station_history_fig_source = self.create_station_history_fig(
)
self.station_avail_histogram, self.station_avail_histogram_source = self.create_station_avail_histogram(
)
self.extremes_annulus, self.extremes_annulus_source = self.create_extremes_figure(
)
self.station_map_fig, self.station_map_fig_source = self.create_station_map(
)
self.station_map_fig_source.selected.indices = [
self.id_coord_mapping_df.index.get_loc(s)
for s in self.x_mul_select.value
]
self._layout = column(
row(
column(self.x_mul_select, self.dr_slider, self.range_slider,
self.histogram_policy, self.extremes_annulus),
self.station_avail_histogram, self.station_map_fig),
row(self.station_history_fig))
# row(column(self.x_select, self.y_select), self.two_station_heatmap()))
@property
def layout(self):
return self._layout
def re_index(self):
self.df_log.index = pd.to_datetime(10**9 *
self.df_log['last_reported'])
self.df_log.drop(['last_reported'], axis=1, inplace=True)
self.df_log.index.tz_localize('UTC').tz_convert('Europe/Oslo')
def pivot_df(self):
def get_column_mapper(df):
d = {}
for col in df.columns.values:
d[col] = str(col)
return d
self.df_bikes_avail = self.df_log.pivot_table(
index='last_reported',
columns='station_id',
values='num_bikes_available')
self.df_bikes_avail.rename(columns=get_column_mapper(
self.df_bikes_avail),
inplace=True)
self.df_docs_avail = self.df_log.pivot_table(
index='last_reported',
columns='station_id',
values='num_docks_available')
self.df_docs_avail.rename(columns=get_column_mapper(
self.df_docs_avail),
inplace=True)
def create_sliced_df(self):
s_hour = self.range_slider.value[0]
e_hour = self.range_slider.value[1]
if e_hour == 25:
hours_of_day = self.df_bikes_avail.index.hour.isin(
np.append([0], range(s_hour, e_hour - 1)))
else:
hours_of_day = self.df_bikes_avail.index.hour.isin(
range(s_hour, e_hour))
df_bikes_sliced = self.df_bikes_avail.loc[
hours_of_day][self.dr_slider.value_as_datetime[0]:self.dr_slider.
value_as_datetime[1]]
df_docks_sliced = self.df_docs_avail.loc[
hours_of_day][self.dr_slider.value_as_datetime[0]:self.dr_slider.
value_as_datetime[1]]
self.df_bikes_source = df_bikes_sliced.loc[:, [
val for val in self.x_mul_select.value
]].fillna(0) # TODO Filling nan with 0 might not be best solution
self.df_docks_source = df_docks_sliced.loc[:, [
val for val in self.x_mul_select.value
]].fillna(0) #
def create_station_history_fig(self):
source_dict = self.create_station_history_fig_source_dict()
source = ColumnDataSource(source_dict)
fig = figure(plot_width=800, plot_height=600, x_axis_type='datetime')
colors = Category20[20][::2]
for val, color in zip(self.x_mul_select.value, colors):
fig.step('last_reported',
val,
color=color,
legend=value(val),
source=source)
return fig, source
def create_station_history_fig_source_dict(self):
df_sliced = self.df_bikes_avail[self.dr_slider.value_as_datetime[0]:
self.dr_slider.value_as_datetime[1]]
df_source = df_sliced.loc[:, [val for val in self.x_mul_select.value]]
df_source.index = df_source.index + self.tz_offset
df_source = df_source.reset_index()
return df_source.to_dict(orient='list')
def create_station_avail_histogram(self):
d = self.create_station_avail_histogram_source_dict()
source = ColumnDataSource(d)
fig = figure(plot_width=800,
plot_height=600,
title='Histogram of availability',
background_fill_color="#fafafa")
fig.quad(top='top',
bottom=0,
left='left',
right='right',
color="colors",
line_color="white",
alpha=0.9,
source=source)
fig.y_range.start = 0
# fig.legend.location = "center_right"
# fig.legend.background_fill_color = "#fefefe"
x_labl = 'x, Available bikes in station' if len(
self.x_mul_select.value) == 1 else 'x, Available bikes in stations'
fig.xaxis.axis_label = x_labl
fig.yaxis.axis_label = 'P(x)'
fig.grid.grid_line_color = "white"
return fig, source
def create_station_avail_histogram_source_dict(self):
if self.histogram_policy.labels[
self.histogram_policy.active] == 'Flatten':
df_bikes_source = self.df_bikes_source
df_docks_source = self.df_docks_source
elif self.histogram_policy.labels[
self.histogram_policy.active] == 'Aggregate':
df_bikes_source = self.df_bikes_source.sum(axis=1)
df_docks_source = self.df_docks_source.sum(axis=1)
else:
df_bikes_source = self.df_bikes_source
df_docks_source = self.df_docks_source
b = np.array((range(-1, int(df_bikes_source.max().max()) + 1))) + 0.5
hist, edges = np.histogram(df_bikes_source, density=True, bins=b)
colors = np.array([self.avail_color] * (len(hist)))
if np.any(df_bikes_source.values == 0):
colors[0] = self.empty_color
if np.any(df_docks_source.values == 0):
colors[-1] = self.full_color # TODO Add for all
d = dict(top=hist, left=edges[:-1], right=edges[1:], colors=colors)
return d
def create_extremes_figure(self):
fig = figure(title='Availability summary',
width=300,
height=300,
x_range=(-100, 100),
y_range=(-100, 100),
tools='')
fig.axis.visible = False
fig.grid.visible = False
fig.outline_line_alpha = 0.0
source = ColumnDataSource(self.create_extremes_figure_source_dict())
fig.annular_wedge(x=0,
y=0,
inner_radius=50,
outer_radius=75,
start_angle=0,
end_angle='end_angle_empty',
color=self.empty_color,
source=source) # Totally empty
fig.annular_wedge(x=0,
y=0,
inner_radius=50,
outer_radius=75,
start_angle='end_angle_empty',
end_angle='end_angle_mid',
color=self.avail_color,
source=source) # Not empty, not full
fig.annular_wedge(x=0,
y=0,
inner_radius=50,
outer_radius=75,
start_angle='end_angle_mid',
end_angle=2 * np.pi,
color=self.full_color,
source=source) # Totally full
fig.circle(x=0, y=0, radius=50, color="#fafafa")
fig.text(x=0,
y=14,
text='empty_percent',
text_color=self.empty_color,
text_baseline='bottom',
text_align='center',
text_font_size='16pt',
text_font_style='bold',
source=source)
fig.text(x=0,
y=0,
text='avail_percent',
text_color=self.avail_color,
text_baseline='middle',
text_align='center',
text_font_size='21pt',
text_font_style='bold',
source=source)
fig.text(x=0,
y=-14,
text='full_percent',
text_color=self.full_color,
text_baseline='top',
text_align='center',
text_font_size='16pt',
text_font_style='bold',
source=source)
return fig, source
def create_extremes_figure_source_dict(self):
df_bikes_source = self.df_bikes_source.sum(axis=1)
df_docks_source = self.df_docks_source.sum(axis=1)
d = dict(end_angle_empty=[
2 * np.pi * np.count_nonzero(df_bikes_source.values == 0) /
len(df_bikes_source.values)
],
end_angle_mid=[
2 * np.pi *
(1 - (np.count_nonzero(df_docks_source.values == 0) /
len(df_docks_source.values)))
])
d['empty_percent'] = [
f"{100 * (np.abs(d['end_angle_empty'][0]) / (2 * np.pi)):2.1f}% "
] # TODO deal with nan
d['avail_percent'] = [
f"{100*(np.abs(d['end_angle_mid'][0] - d['end_angle_empty'][0])/(2 * np.pi)):.1f}%"
] # TODO deal with nan
d['full_percent'] = [
f"{100 * (np.abs(2 * np.pi - d['end_angle_mid'][0]) / (2 * np.pi)):.1f}%"
] # TODO deal with nan
return d
def create_station_map(self):
fig = figure(
plot_width=780,
plot_height=600,
title='Stations map and selector',
tools=['pan', 'box_zoom', 'wheel_zoom', 'lasso_select', 'reset'],
x_range=(self.mid_lat - self.coord_range_delta,
self.mid_lat + self.coord_range_delta),
y_range=(self.mid_lon - self.coord_range_delta,
self.mid_lon + self.coord_range_delta),
x_axis_type="mercator",
y_axis_type="mercator")
fig.add_tile(CARTODBPOSITRON)
lst = fig.select(dict(type=LassoSelectTool))[0]
lst.select_every_mousemove = False
# # Bikes available
# fig.annular_wedge(x=status_df['x_proj'], y=status_df['y_proj'], color=self.avail_color,
# inner_radius=np.zeros(len(status_df)), outer_radius=25 * np.sqrt(status_df['num_docs']),
# start_angle=(np.pi / 2 + np.zeros(len(status_df))),
# end_angle=(np.pi / 2 - status_df['docks_start_ang']), direction='clock')
# # Docks available
# fig.annular_wedge(x=status_df['x_proj'], y=status_df['y_proj'], color="#ea6d3f",
# inner_radius=np.zeros(len(status_df)), outer_radius=25 * np.sqrt(status_df['num_docs']),
# start_angle=(np.pi / 2 - status_df['docks_start_ang']),
# end_angle=(10 ** (-3) + np.pi / 2 * np.ones(len(status_df))), direction='clock')
#
# fig.text(x=status_df['x_proj'], y=status_df['y_proj'], text=status_df.index)
source = ColumnDataSource(self.id_coord_mapping_df)
c = fig.circle(x='x_proj',
y='y_proj',
size=10,
color="navy",
source=source)
fig.text(x='x_proj', y='y_proj', text='station_id', source=source)
c.data_source.selected.on_change('indices', self.lasso_select_callback)
return fig, source
def two_station_heatmap(self):
fig = figure(width=700, match_aspect=True, tools='')
fig.xgrid.grid_line_color = None
fig.ygrid.grid_line_color = None
s_id1, s_id2 = self.x_select.value, self.y_select.value
df_counts = self.df_bikes_avail.groupby(
[s_id1, s_id2]).size().reset_index(name='counts')
df_counts.rename(columns={s_id1: s_id1, s_id2: s_id2}, inplace=True)
source = ColumnDataSource(df_counts)
pallette = [
'#084594', '#2171b5', '#4292c6', '#6baed6', '#9ecae1', '#c6dbef',
'#deebf7', '#f7fbff'
][::-1]
mapper = LinearColorMapper(palette=pallette,
low=0,
high=df_counts.counts.max())
color_bar = ColorBar(color_mapper=mapper)
fig.rect(x=str(s_id1),
y=str(s_id2),
width=1.0,
height=1.0,
line_alpha=0.0,
fill_color=transform('counts', mapper),
source=source)
fig.add_layout(color_bar, 'right')
return fig
def lasso_select_callback(self, attr, old, new):
# print(self.id_coord_mapping_df.iloc[new].index.values)
if new:
self.x_mul_select.value = list(
self.id_coord_mapping_df.iloc[new].index.values)
def get_station_info(self):
if self.city == 'Oslo':
json_url = "https://gbfs.urbansharing.com/oslobysykkel.no/station_information.json"
elif self.city == 'NewYorkCity':
json_url = "https://gbfs.citibikenyc.com/gbfs/es/station_information.json"
with urllib.request.urlopen(json_url) as url:
station_info = json.loads(url.read().decode())
return station_info
# @staticmethod
def make_station_coord_transformed_df(self, station_info_in):
def apply_transfomation(lon_in, lat_in):
return pyproj.transform(pyproj.Proj(init='epsg:4326'),
pyproj.Proj(init='epsg:3857'), lon_in,
lat_in)
lon = [s['lon'] for s in station_info_in['data']['stations']]
lat = [s['lat'] for s in station_info_in['data']['stations']]
x_proj, y_proj = zip(*map(apply_transfomation, lon, lat))
index = [s['station_id'] for s in station_info_in['data']['stations']]
df = pd.DataFrame(data={
'x_proj': x_proj,
'y_proj': y_proj
},
index=index)
df.index.name = 'station_id'
return df
def update(self):
t_start = time()
self.station_history_fig, self.station_history_fig_source = self.create_station_history_fig(
)
self.station_avail_histogram, self.station_avail_histogram_source = self.create_station_avail_histogram(
)
print(f"Used: {(time() - t_start) * 1000} ms to regenerate figures")
t_start = time()
self._layout.children[0].children[1] = self.station_avail_histogram
self._layout.children[1].children[0] = self.station_history_fig
print(f"Used: {(time() - t_start)*1000} ms to update layout")
# self._layout.children[1].children[1] = self.two_station_heatmap()
def y_select_callback(self, attr, old, new):
self.update()
def x_select_callback(self, attr, old, new):
self.update()
def x_mul_select_callback(self, attr, old, new):
self.create_sliced_df()
self.station_history_fig, self.station_history_fig_source = self.create_station_history_fig(
)
self.station_avail_histogram, self.station_avail_histogram_source = self.create_station_avail_histogram(
)
self.extremes_annulus_source.data = self.create_extremes_figure_source_dict(
)
self.station_map_fig_source.selected.indices = [
self.id_coord_mapping_df.index.get_loc(s)
for s in self.x_mul_select.value
]
self._layout.children[0].children[1] = self.station_avail_histogram
self._layout.children[1].children[0] = self.station_history_fig
def date_rage_slider_callback(self, attr, old, new):
t_start = time()
self.create_sliced_df()
self.station_history_fig_source.data = self.create_station_history_fig_source_dict(
)
self.station_avail_histogram_source.data = self.create_station_avail_histogram_source_dict(
)
self.extremes_annulus_source.data = self.create_extremes_figure_source_dict(
)
print(f"Used: {(time() - t_start) * 1000} ms to calculate sources")
def hour_rage_slider_callback(self, attr, old, new):
t_start = time()
# self.station_history_fig_source.data = self.create_station_history_fig_source_dict()
self.create_sliced_df()
self.extremes_annulus_source.data = self.create_extremes_figure_source_dict(
)
self.station_avail_histogram_source.data = self.create_station_avail_histogram_source_dict(
)
print(f"Used: {(time() - t_start) * 1000} ms to calculate sources")
def histogram_policy_callback(self, attr, old, new):
self.station_avail_histogram_source.data = self.create_station_avail_histogram_source_dict(
)
class PollViewer():
"""Shows the polls in the system for the selected election."""
# %%
def __init__(self, controller):
"""Initialize object.
First part of two-part initialization.
Put initialization code here that's very unlikely to fail.
"""
self.controller = controller
self.polls = None
# Table
# =====
# Stub code for DataTable setup.
_df = pd.DataFrame(
{'State name': ['Alaska'],
'Start date': ['2020-01-01'],
'End date': ['2020-01-10'],
'Polling company': ['Good Polls Inc'],
'Poll ID': [123456],
'Sample size': [1000],
'Democratic %': [44.9],
'Republican %': [45.1]})
_df['Start date'] = pd.to_datetime(_df['Start date'])
_df['End date'] = pd.to_datetime(_df['End date'])
self.pollsource = ColumnDataSource(_df)
columns = [TableColumn(field='State name', title='State name'),
TableColumn(field='Start date',
title='State date',
formatter=DateFormatter()),
TableColumn(field='End date',
title='End date',
formatter=DateFormatter()),
TableColumn(field='Polling company',
title='Polling company'),
TableColumn(field='Poll ID', title='Poll ID'),
TableColumn(field='Sample size', title='Sample size'),
TableColumn(field='Democratic %', title='Democratic %'),
TableColumn(field='Republican %', title='Republican %')]
# Opinion polls in the system.
self.opinionpolls = DataTable(
source=self.pollsource,
columns=columns,
index_position=None,
sizing_mode="""stretch_both""")
# Other widgets
# =============
# Date range
self.choosedates = DateRangeSlider(
title="""Choose the date for display""",
start="""2018-11-13T20:20:39+00:00""",
end="""2025-11-13T20:20:39+00:00""",
step=24*60*60*1000,
value=("""2018-11-13T20:20:39+00:00""",
"""2025-11-13T20:20:39+00:00"""),
sizing_mode="stretch_width")
# State
self.selectstate = Select(
title="""State""",
options=['dummy1', 'dummy2', 'dummy3'],
value="""dummy1""",
sizing_mode="stretch_width")
# Layout the widgets
# ==================
row1 = row(children=[self.choosedates,
Spacer(width=50),
self.selectstate])
layout = column(children=[self.opinionpolls,
row1,
Spacer(height=75,
sizing_mode='scale_width')],
sizing_mode='stretch_both')
self.panel = Panel(child=layout,
title='Poll viewer')
# %%
def setup(self):
"""Set up object.
Second part of two-part initialization.
Place initialization code here that's more likely to fail.
"""
# Setup the callbacks.
self.choosedates.on_change("value", self.callback_choosedates)
self.selectstate.on_change("value", self.callback_selectstate)
# %%
def update(self, polls):
"""Update view object."""
self.polls = polls
_states = sorted(self.polls['State name'].unique().tolist())
self.selectstate.options = _states
self.selectstate.value = sample(_states, 1)[0]
self.choosedates.start = self.polls['start_date'].min()
self.choosedates.end = self.polls['end_date'].max()
self.choosedates.value = (
self.polls['start_date'].min(),
self.polls['end_date'].max())
self._update_table()
# %%
def _update_table(self):
"""Update table."""
_slice = self.polls[
(self.polls['State name'] == self.selectstate.value)
& (self.polls['start_date']
>= self.choosedates.value_as_datetime[0])
& (self.polls['end_date']
<= self.choosedates.value_as_datetime[1])
].sort_values(['start_date', 'end_date'])
self.pollsource.data = {
'State name': _slice['State name'].to_list(),
'Start date': _slice['start_date'].to_list(),
'End date': _slice['end_date'].to_list(),
'Polling company': _slice['pollster'].to_list(),
'Poll ID': _slice['poll_id'].to_list(),
'Sample size': _slice['sample_size'].to_list(),
'Democratic %': _slice['Democratic'].to_list(),
'Republican %': _slice['Republican'].to_list()}
# %%
def callback_choosedates(self, attrname, old, new):
"""Execute callback for self.callback_choosedates."""
# pylint: disable=W0613
self._update_table()
# %%
def callback_selectstate(self, attrname, old, new):
"""Execute callback for self.callback_selectstate."""
# pylint: disable=W0613
self._update_table()
if selected:
num_shootings = [
line_source.data["num_shootings"][i] for i in selected
]
num_offenses = [line_source.data["num_offenses"][i] for i in selected]
mean_line.location = mean(num_offenses)
mean_shootings.location = mean(num_shootings)
def update_graphs():
update_heatmap()
update_top10()
update_line()
selected_months.on_change('value_throttled',
lambda attr, old, new: update_graphs())
selected_districts.on_change('active', lambda attr, old, new: update_graphs())
line_source.selected.on_change('indices',
lambda attr, old, new: update_selection())
update_graphs()
controls = row(widgetbox(selected_months),
widgetbox(selected_districts),
sizing_mode='stretch_width')
curdoc().add_root(controls)
curdoc().add_root(
layout([[num_offenses, num_shootings], [top10, heatmap]],
sizing_mode='stretch_width'))
values = list(values)
for i in all_gate_names:
if i not in labels:
labels.append(i)
values.append(0)
zipped = sorted(zip(labels, values))
labels, values = zip(*zipped)
labels = np.asarray(labels)
values = np.asarray(values)
source.data[car_type] = values
p = figure(plot_width=plot_width,
plot_height=plot_height,
x_range=labels,
tools=[hover, 'box_zoom', 'reset'])
p.vbar_stack(all_car_types,
x='Gate Names',
width=width,
source=source,
color=color,
line_color='white',
legend=[value(x) for x in legend_var],
muted_color=color,
muted_alpha=0.25)
p.xaxis.major_label_orientation = 1.2
p.legend.location = 'top_left'
p.legend.click_policy = 'mute'
date_slider.on_change('value', date_range_update)
curdoc().add_root(column(p, date_slider))
# states_selection = CheckboxGroup(labels=STATES, active = [0,1])
# states_selection.on_change('active', update)
thresh_select = Slider(start=0,
end=1000,
step=1,
value=0,
title='Case Count Minimum')
thresh_select.on_change('value', update)
range_select = DateRangeSlider(start=dt.date(2020, 1, 21),
end=dt.date.today(),
value=(dt.date(2020, 1, 21), dt.date.today()),
step=1,
title='Date Range')
range_select.on_change('value', update)
select_all = Button(label="select all")
select_all.on_click(activate_all_update)
unselect_all = Button(label="unselect all")
unselect_all.on_click(deactivate_all_update)
# Initialize source
initial_states = [states_select1.labels[i] for i in states_select1.active] + \
[states_select2.labels[i] for i in states_select2.active] + \
[states_select3.labels[i] for i in states_select3.active]
src = make_dataset(initial_states,
start=range_select.value[0],
end=range_select.value[1],
thresh=thresh_select.value)
end=VARS['global_end'],
start=VARS['global_start'],
step=1,
value=(
VARS['time_range'][0],
VARS['time_range'][1],
))
single_day_selector = DateSlider(
title="Select Day",
end=VARS['global_end'],
start=VARS['global_start'],
step=1,
value=VARS['selected_day'],
)
date_selector.on_change('value', update_time_range)
single_day_selector.on_change('value', update_highlighted_day)
##################################################################
# Bokeh Plots
##################################################################
TOOLS = "pan,wheel_zoom,box_select,lasso_select,reset"
##########
# Stage Time Series
##########
hydrograph = figure(plot_width=1000,
plot_height=350,
tools=TOOLS + ',box_zoom,hover',
toolbar_location="above",
def explore_tab(df):
def get_dataset(src, name, words, start, end):
df = src[src.user == name].copy()
mask = (df['timestamp'] > start) & (df['timestamp'] <= end)
df = df[mask]
words = [str(i) for i in words.split()]
safe_words = []
for word in words:
word = re.escape(word)
word = "(?=.*{})".format(word)
safe_words.append(word)
df = df[df['texts'].str.contains(''.join(safe_words))]
source = ColumnDataSource(data=dict())
cols = ['texts', 'displaySource', 'source']
df[cols] = df[cols].replace({',': '', ',,': '', ';': ''}, regex=True)
source.data = {
# 'index': df.index,
'impressionTime': df.impressionTime,
'impressionOrder': df.impressionOrder,
'source': df.source,
'fblinktype': df.fblinktype,
'texts': df.texts,
'textsize': df.textsize,
'publicationTime': df.publicationTime,
'permaLink': df.permaLink,
'nature': df.nature,
'ANGRY': df.ANGRY,
'HAHA': df.HAHA,
'LIKE': df.LIKE,
'LOVE': df.LOVE,
'SAD': df.SAD,
'WOW': df.WOW,
'displaySource': df.displaySource,
'id': df.id,
'timestamp': df.timestamp,
# 'images': df.images,
# 'opengraph': df.opengraph,
'postId': df.postId,
# 'semanticCount': df.semanticCount,
# 'semanticId': df.semanticId,
'sourceLink': df.sourceLink,
'timeline': df.timeline,
'user': df.user,
# 'videoautoplay': df.videoautoplay
}
return source
def make_table(source):
# Columns of tablem
table_columns = [
TableColumn(field='impressionTime', title='Time'),
TableColumn(field='impressionOrder', title='Order'),
TableColumn(field='source', title='Source'),
TableColumn(field='fblinktype', title='Type'),
TableColumn(field='texts', title='Text'),
TableColumn(field='textsize', title='Text Size'),
TableColumn(field='publicationTime', title='Publication Time'),
TableColumn(field='permaLink', title='Link'),
TableColumn(field='nature', title='Nature'),
TableColumn(field='ANGRY', title='Angry'),
TableColumn(field='HAHA', title='Haha'),
TableColumn(field='LIKE', title='Like'),
TableColumn(field='LOVE', title='Love'),
TableColumn(field='SAD', title='Sad'),
TableColumn(field='WOW', title='Wow')
]
user_table = DataTable(source=source,
columns=table_columns,
width=1400)
return user_table
def update(attrname, old, new):
name = name_select.value
text_filter = text_input.value
start = date_slider.value[0]
end = date_slider.value[1]
src = get_dataset(df, name, text_filter, start, end)
source.data.update(src.data)
name = df.user.iloc[0]
words = ''
names = df.user.unique()
start = df.timestamp.min()
end = df.timestamp.max()
name_select = Select(value=name, title='User', options=sorted(names))
text_input = TextInput(value="", title="Filter text:")
date_slider = DateRangeSlider(title="Date Range: ",
start=df.timestamp.min(),
end=date.today(),
value=(df.timestamp.min(), date.today()),
step=1,
callback_policy='mouseup')
button = Button(label="Download", button_type="success")
source = get_dataset(df, name, words, start, end)
table = make_table(source)
name_select.on_change('value', update)
text_input.on_change('value', update)
date_slider.on_change('value', update)
button.js_on_click(
CustomJS(args=dict(source=source),
code=open(join(dirname(__file__), "download.js")).read()))
controls = column(name_select, date_slider, text_input, button)
tab = Panel(child=row(table, controls), title='Explore')
return tab
def modify_doc(doc):
# function to make a dataset for histogram based on a list of set filters
valid_bin_widths = ['day', 'week', 'month']
default_bin_width='week'
slider_date_end = datetime.date.today()
slider_date_start = slider_date_end - relativedelta(months=6, day=1) # at most 2 months ago
# return delta and align for a range according to bin_width
# bin_width is one of 'week', 'month', 'day'
# delta can be used to move a date to the next bin, align to
# snap back a range the the current bin start
def align_range(bin_width):
if bin_width == 'week':
delta = relativedelta(weeks=1)
align = relativedelta(weekday=Monday(-1))
elif bin_width == 'month':
delta = relativedelta(months=1)
align = relativedelta(day=1)
else:
#nothing special to do for 'day'
delta = relativedelta(days=1)
align = relativedelta()
return delta, align
def make_dataset(endpoint, borough_list, date_start, date_end, bin_width):
delta, align = align_range(bin_width)
date_start += align
date_end += align + delta
df = query_dates(endpoint, date_start, date_end)
def histograms():
prev_buckets = None
for i, borough_name in enumerate(borough_list):
subset = df [df['borough'] == borough_name]
edges = list(time_range(date_start, date_end, delta))
buckets = subset['estimated_job_costs'].groupby(lambda x: x - align)\
.agg(sum=np.sum,
mean=np.mean,
amax=np.max,
len=len)
max_subset = subset.groupby(lambda x: x-align)\
.apply(lambda rows: rows.iloc[np.argmax(rows['estimated_job_costs'].values)])
# it is possible that buckets do not cover the full range, so we create
# another data frame for the full range and fill it with 0
tmp=pd.DataFrame(index=edges, columns=buckets.columns)
tmp.fillna(0, inplace=True)
# then we copy the subset shared with the other dataframe
tmp.loc[buckets.index & tmp.index ] = buckets.loc[buckets.index & tmp.index]
buckets = tmp
# extend edges with an extra 'after-the-end' element
edges = edges + [edges[-1] + delta]
buckets.sort_index()
# groupby.agg creates one column per aggregate
buckets['sum'] /= 10**6
buckets['mean'] /= 1000
buckets['amax'] /= 1000
# nothing to do with buckets['len']
buckets['left'] = edges[:-1]
buckets['right'] = edges[1:]
buckets['color'] = Category20_16[i]
buckets['name'] = borough_name
for c, format in col_meta.items():
if prev_buckets is not None:
buckets[c + '_top'] = buckets[c] + prev_buckets[c + '_top']
buckets[c + '_bottom'] = prev_buckets[c + '_top']
else:
buckets[c + '_top'] = buckets[c]
buckets[c + '_bottom'] = 0
buckets['f_' + c] = buckets[c].apply(lambda x: format%(x))
buckets['f_period'] = buckets.index.map(lambda x: '{} - {}'.format(x.date(), (x+delta).date()))
def f_address(rows):
addr = '{street_name} {house_no} {work_on_floor}'.format(**rows.to_dict())
return addr
buckets['f_address'] = max_subset.apply(f_address, axis=1)
buckets['f_job_description'] = max_subset['job_description']
prev_buckets = buckets
yield buckets.reset_index()
#Dataframe to hold information
by_borough = pd.DataFrame()
# Overall dataframe
all_buckets = list(histograms())
by_borough = by_borough.append(all_buckets, sort=False)
by_borough.sort_values(['name', 'left'], inplace=True)
return ColumnDataSource(by_borough)
def make_plot(src, title, y_label, tooltip, column):
# Blank plot with correct labels
p = figure(plot_width = 500, plot_height = 500,
title = title,
x_axis_type='datetime',
sizing_mode='stretch_both',
x_axis_label = 'Date', y_axis_label = y_label)
# Quad glyphs to create a histogram
p.quad(source = src, bottom = column +'_bottom', top = column + '_top', left = 'left', right = 'right',
color = 'color', fill_alpha = 0.7, hover_fill_color = 'color', legend_label = 'name',
hover_fill_alpha = 1.0, line_color = 'black')
if column == 'amax':
tooltips = [('Period:','@f_period'),
('Borough', '@name'),
('Address', '@f_address'),
('Description', '@f_job_description'),
('cost', '@f_amax')
]
else:
tooltips = [('Period:','@f_period'),
('Borough', '@name'),
(tooltip, '@f_'+column)
]
# Hover tool with vline mode
hover = HoverTool(tooltips=tooltips)
p.add_tools(hover)
# Styling
p = style(p, col_meta[column])
return p
def style(p, y_format):
# Title
p.title.align = 'center'
p.title.text_font_size = '20pt'
p.title.text_font = 'serif'
# Axis titles
p.xaxis.axis_label_text_font_size = '14pt'
p.xaxis.axis_label_text_font_style = 'bold'
p.yaxis.axis_label_text_font_size = '14pt'
p.yaxis.axis_label_text_font_style = 'bold'
p.yaxis.formatter = PrintfTickFormatter (format=y_format)
# Tick labels
p.xaxis.major_label_text_font_size = '12pt'
p.yaxis.major_label_text_font_size = '12pt'
return p
src = ColumnDataSource()
old_params = [None]
def do_update():
try:
new_params = (approval_res,
[borough_selection.labels[i] for i in borough_selection.active],
fixup_date(date_select.value[0]),
fixup_date(date_select.value[1]),
valid_bin_widths[binwidth_select.active])
if new_params != old_params[0]:
show_spinner()
new_data = make_dataset(*new_params)
old_params[0] = new_params
src.data.update(new_data.data)
except Exception:
print(traceback.print_exc())
def update(attr, old, new):
do_update()
# DateRangeSlider mouseup is broken, do nothing on change and use a timer
slow_update=[time.time()]
def update_no_op(attr, old, new):
show_spinner()
if time.time()-slow_update[0] < .5:
return
slow_update[0] = time.time()
update(attr, old, new)
def time_update():
#return
slow_update[0] = time.time()
do_update()
hide_spinner()
spinner_text = """
<!-- https://www.w3schools.com/howto/howto_css_loader.asp -->
<div class="loader" >
<style scoped>
.loader {
border: 16px solid #f3f3f3; /* Light grey */
border-top: 16px solid #3498db; /* Blue */
border-radius: 50%;
margin: auto;
width: 100px;
height: 100px;
animation: spin 2s linear infinite;
}
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}
</style>
</div>
"""
div_spinner = Div(text="",width=120,height=120)
def show_spinner():
div_spinner.text = spinner_text
def hide_spinner():
div_spinner.text = ""
binwidth_select = RadioButtonGroup(labels=valid_bin_widths,
active=valid_bin_widths.index(default_bin_width), #index of 'week', i.e. 0
sizing_mode='stretch_both')
binwidth_select.on_change('active', update)
date_default_end= slider_date_end
date_default_start = date_default_end - relativedelta(months=1)
date_select = DateRangeSlider(start=slider_date_start,
end=slider_date_end,
value=(date_default_start,date_default_end),
callback_policy='mouseup', # do not start untill mouse released
step=1,
callback_throttle=1000,
sizing_mode='stretch_both') # this is slow, so calls at most every 2000ms
date_select.on_change('value', update_no_op)
available_boroughs = ['QUEENS', 'MANHATTAN', 'STATEN ISLAND', 'BROOKLYN', 'BRONX']
borough_selection = CheckboxGroup(labels=available_boroughs, active = list(range(0, len(available_boroughs))),
sizing_mode='stretch_both')
borough_selection.on_change('active', update)
initial_borough = [borough_selection.labels[i] for i in borough_selection.active]
# Put controls in a single element
controls = layout([[borough_selection, binwidth_select, date_select, div_spinner]] , width=500)
col_meta = {
'len': '%d',
'mean': '%dl',
'sum': '%dM',
'amax': '%dk'
}
data = [ ('Number of Projects', 'Total projects', 'counts', 'len'),
('Most Expensive Project', 'Max cost', 'cost', 'amax'),
('Total Project Cost', 'Total project cost', 'cost', 'sum'),
('Mean Project Cost', 'Median project cost', 'cost', 'mean') ]
do_update()
plots = [ make_plot(src, *args) for args in data ]
# Create a row layout
lyt = layout([controls, plots[3]],
plots[0:3])
# Make a tab with the layout
tab = Panel(child=lyt, title = 'Histogram')
tabs = Tabs(tabs=[tab])
doc.add_periodic_callback(time_update, 1000)
doc.add_root(tabs)
search_fig.y_range.end,
search_fig.y_range.end,
search_fig.y_range.start,
],
})
search_fig.patch(x="x", y="y", source=patch_src, alpha=0.5, line_width=2)
period_slider = DateRangeSlider(
value=(data.timeseries.start_datetime, data.timeseries.end_datetime),
start=data.timeseries.search_start_datetime,
end=data.timeseries.search_end_datetime,
)
period_slider.format = "%d-%m-%Y"
period_slider.on_change("value", update_on_period)
period_slider.js_link("value_throttled",
time_figs_x_range,
"start",
attr_selector=0)
period_slider.js_link("value_throttled",
time_figs_x_range,
"end",
attr_selector=1)
#period_slider.show_value=True
# %% define layout
width = 1920 * 0.82
height = 1080 * 0.82
select_locations.size = 10
#map_fig.sizing_mode = "stretch_width"
p.extra_y_ranges['Avg'].end = 1.05 * np.max(
[src.data['avgc'], src.data['avgd']])
# set up the controls
sel_country = Select(value="India", options=sorted_by_cases, width=220)
sel_chart = Select(value='Day by Day',
options=['Day by Day', 'Cumulative'],
width=120)
dateslider = DateRangeSlider(start=datemin,
end=datemax,
value=(datemin, datemax),
title='Date Range',
sizing_mode="scale_width")
# set up the event handlers
sel_chart.on_change('value', update)
sel_country.on_change('value', update)
dateslider.on_change('value', update)
# create the dataset and the plot
src = make_dataset(sel_country.value,
sel_chart.value,
range_start=datemin,
range_end=datemax)
p = make_plot(src)
# set up the layout of the plot
controls = row(sel_country, sel_chart, dateslider)
layout = column(controls, p)
curdoc().add_root(layout)
new_sorted_acts.sort()
indices = [i for i in range(len(new_sorted_acts)) if new_sorted_acts[i] in old_acts]
checkbox_group.labels = new_sorted_acts
checkbox_group.active = indices
update_x, update_y = reduce_data_set_by_indices(checkbox_group.active, x, y, new_sorted_acts)
# This is a workaround for strange bokeh behaviour
if len(update_x) != 0:
p.x_range.factors = update_x
source.data = dict(x=update_x, counts=update_y)
else:
# I don't like these gymnastics, but it makes the interface more intuitive.
placeholder_goals = {y for (x,y) in p.x_range.factors}
placeholder_indices = [i for i in range(len(new_sorted_acts)) if new_sorted_acts[i] in placeholder_goals]
checkbox_group.active = placeholder_indices
checkbox_group.on_change('active',lambda attr, old, new: update_checkbox())
date_slider.on_change('value',lambda attr, old, new: update_slider())
controls = column(checkbox_group,date_slider,width=300)
curdoc().add_root(row(p,controls))
#show(row(p,controls))
def first_tab_create(filterData):
# all_min_date = filterData.groupby('dataid').agg(min)["time"]
# all_max_date = filterData.groupby('dataid').agg(max)["time"]
dummy_daterange = ['2019-05-01', '2019-08-20']
dummy_home_id = 27
dummy_data_type = 'car1'
dummy_granularity = '15 Minutes'
dummy_analysis = 'avgday'
def plot1_data(houseData, data=dummy_data_type, xaxis=dummy_granularity):
# house is an integer number ex. 27
# daterange is an array with 2 strings, start date and end date. ex. ['2019-05-01','2019-08-09']
# weekdays is an ordered list 0-6 of integers ex. [1,4,6] (these are the days we want to exclude)
# data is a string ex. 'car1'
# xaxis is also a string ex. 'hour'
# that cuts the house, sorts by ascending time, and pulls out only the type of data that was requested
# houseData.index = houseData['time'] # reindex by the datetime
# houseData = houseData.loc[daterange[0]:daterange[1], :] # cut to the days requested
if xaxis == '15 Minutes':
houseData = houseData.drop(columns="time")
houseData[data] = houseData[data] * 60 * 15 / 3600 # kWh
if xaxis == 'Hour':
houseData[data] = houseData[data] * 60 * 15 / 3600 # kWh
houseData = houseData.resample('1h').sum()
if xaxis == 'Day':
houseData[data] = houseData[data] * 60 * 15 / 3600 # kWh
houseData = houseData.resample('1d').sum()
if xaxis == 'Week':
houseData[data] = houseData[data] * 60 * 15 / 3600 # kWh
houseData = houseData.resample('1w').sum()
if xaxis == 'Month':
houseData[data] = houseData[data] * 60 * 15 / 3600 # kWh
houseData = houseData.resample('1m').sum()
houseData['data'] = houseData[data]
houseData = houseData.drop(columns=data)
return ColumnDataSource(houseData)
def plot2_data(houseData,
weekdays=[],
data=dummy_data_type,
xaxis=dummy_analysis):
#communityData = filterData[filterData['state'] == filterData[filterData['dataid'] == house]['state'].iloc[0]]
# houseData = filterData[filterData['dataid'] == house].sort_values('time', ascending = True)[[data,'time']]
# that cuts the house, sorts by ascending time, and pulls out only the type of data that was requested
# houseData.index = houseData['time'] # reindex by the datetime
# houseData = houseData.loc[daterange[0]:daterange[1],:] # cut to the days requested
for i in weekdays:
houseData = houseData[houseData['time'].dt.dayofweek !=
i] # cut out days we dont want
houseData[data] = houseData[data] * 60 * 15 # kilojoules every 15 min
houseData = houseData.resample('1h').sum() # kJ every hour
houseData[data] = houseData[data] / 3600 # kilojoules to kWh
if xaxis == 'avgday':
houseData = houseData.resample('1d').sum() # net daily sum
houseData['axis'] = houseData.index
houseData = houseData.groupby(
houseData['axis'].dt.dayofweek)[data].mean()
#houseData.index = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']
if xaxis == 'avghour':
houseData['axis'] = houseData.index
houseData = houseData.groupby(
houseData['axis'].dt.hour)[data].mean(
) # Does not account for UTC change!
houseData = pd.DataFrame(
data=houseData
) # shold figure out a way without needing to make dummy columns
houseData['data'] = houseData[data]
houseData = houseData.drop(columns=data)
return ColumnDataSource(houseData)
# now we need to set up our plot axis
##########method2: create plots
def plot1_plot(src):
plot1 = figure(plot_width=1000,
plot_height=340,
title="Net Load Profile of Home 27",
x_axis_type="datetime",
x_axis_label="Time",
y_axis_label="Net Load [kWh]")
plot1.line('time', 'data', source=src) # simple line plot
return plot1 # plot object type
def plot2_plot(src):
plot2 = figure(
title='Plot 2: Average Weekly Net Load Profile of Home 27',
x_axis_label='Day of the week',
y_axis_label='Net Load [kWh]')
plot2.plot_width = 1000
plot2.plot_height = 400
plot2.vbar(x='axis', top='data', width=1, source=src)
return plot2
## Update Function
def update(
attr, old, new
): # still a little unsure how the update function gets values passed in implicitly
global home_to_plot
# global state_selector
data_type_to_plot = 'grid'
exclude_days_to_plot = [0, 1, 2, 3, 4, 5, 6]
avg_to_plot = 'avgday'
granularity_to_plot = granularity_1.labels[granularity_1.active]
new_home_to_plot = int(home_id_selector.value)
data_selector = data_type_selector.labels[data_type_selector.active]
## Update the country dropdown
country_selector.label = country_selector.value
## Update the state dropdown
states_available = np.unique(filterData[
filterData['country'] == country_selector.value]["state"])
states_available = states_available.tolist()
state_selector.menu = states_available
state_selector.label = state_selector.value
## Update Homes Available
home_ids = np.unique(
filterData[filterData['state'] == state_selector.value]['dataid'])
home_ids_available = list(map(str, home_ids))
home_id_selector.menu = home_ids_available
## Update Aggregate selector titles
aggregate_selector.labels = [
f'Aggregate By Country: {country_selector.value}',
f'Aggregate By Region: {state_selector.value}',
f'Aggregate By Home: {home_id_selector.value}'
]
## plot updates:
if data_selector == 'Net Load':
data_type_to_plot = 'grid'
plot2.yaxis.axis_label = 'Net Load [kWh]'
plot1.yaxis.axis_label = 'Net Load [kWh]'
if data_selector == 'Load + Battery(Charging)':
data_type_to_plot = 'Load_+_Battery(Charging)'
plot2.yaxis.axis_label = 'Load [kWh]'
plot1.yaxis.axis_label = 'Load [kWh]'
if data_selector == "Electric Vehicle Consumption":
data_type_to_plot = 'car1'
plot2.yaxis.axis_label = 'Consumption [kWh]'
plot1.yaxis.axis_label = 'Consumption [kWh]'
if data_selector == "PV Generation + Battery(Discharge)":
data_type_to_plot = 'PV_+_Battery(Discharge)'
plot2.yaxis.axis_label = 'Generation [kWh]'
plot1.yaxis.axis_label = 'Generation [kWh]'
avg_selector = analysis.labels[analysis.active]
if avg_selector == 'Weekly Pattern':
avg_to_plot = 'avgday'
if avg_selector == 'Daily Pattern':
avg_to_plot = 'avghour'
include_days_to_plot = weekdays_checkbox.active # wish they had an inactive :/
for i in include_days_to_plot:
exclude_days_to_plot.remove(i) # lame way
## Create the dataset to plot (houseDate) based on aggregate or non aggregate selection
if aggregate_selector.active == 0:
houseData = filterData.loc[filterData['country'] ==
country_selector.value, :]
houseData = houseData.groupby(
'time').mean() # same question as above
# houseData = filterData.groupby(filterData[filterData['country']==country_selector.value]['time']).mean() #should i use mean or sum here?
# print(country_selector.value)
# print(houseData.head(15))
houseData = houseData[[data_type_to_plot]]
# print(houseData.head(15))
houseData['time'] = houseData.index
startDate = houseData.index[0]
endDate = houseData.index[-1]
new_home_to_plot = country_selector.value
elif aggregate_selector.active == 1:
houseData = filterData.loc[filterData['state'] ==
state_selector.value, :]
houseData = houseData.groupby(
'time').mean() # same question as above
# print(state_selector.value)
# print(houseData.head(15))
houseData = houseData[[data_type_to_plot]]
# print(houseData.head(15))
houseData['time'] = houseData.index
startDate = houseData.index[0]
endDate = houseData.index[-1]
new_home_to_plot = state_selector.value
else:
houseData = filterData[
filterData['dataid'] == new_home_to_plot].sort_values(
'time', ascending=True)[[data_type_to_plot, 'time']]
houseData.index = houseData['time']
startDate = houseData.index[0]
endDate = houseData.index[-1]
# ## Update DateRange Slider
if new_home_to_plot != home_to_plot:
startDate = str(startDate)[0:10]
endDate = str(endDate)[0:10]
date_slider.start = date(int(startDate[0:4]), int(startDate[5:7]),
int(startDate[8:10]))
date_slider.end = date(int(endDate[0:4]), int(endDate[5:7]),
int(endDate[8:10]))
date_slider.value = (date(int(startDate[0:4]), int(startDate[5:7]),
int(startDate[8:10])),
date(int(endDate[0:4]), int(endDate[5:7]),
int(endDate[8:10])))
home_to_plot = new_home_to_plot
daterange_raw = list(date_slider.value_as_datetime)
daterange_to_plot = [
daterange_raw[0].strftime("%Y-%m-%d"),
daterange_raw[1].strftime("%Y-%m-%d")
]
##Edit bounds of data we are plotting
houseData = houseData.loc[
daterange_to_plot[0]:
daterange_to_plot[1], :] # cut to the days requested
## change plot titles to fit current data
plot1.title.text = f'{data_selector} Profile of {new_home_to_plot}'
## plot 2 updates:
if avg_to_plot == 'avgday':
plot2.title.text = f'Average Weekly {data_selector} Profile of {new_home_to_plot}'
plot2.xaxis.axis_label = 'Day of the week'
if avg_to_plot == 'avghour':
plot2.title.text = f'Average Hourly {data_selector} Profile of {new_home_to_plot}'
plot2.xaxis.axis_label = 'Hours of Day'
## SRC Updates
new_src1 = plot1_data(houseData,
data=data_type_to_plot,
xaxis=granularity_to_plot)
new_src2 = plot2_data(houseData,
weekdays=exclude_days_to_plot,
data=data_type_to_plot,
xaxis=avg_to_plot)
src1.data.update(new_src1.data)
src2.data.update(new_src2.data)
## Widgets ##
## Granularity
granularity_1 = RadioGroup(
labels=["15 Minutes", "Hour", "Day", "Week", "Month"],
active=0,
max_width=125)
granularity_1.on_change(
'active', update
) # not sure exactly how this works but runs update on the change of the button and passes through the value of the button
## Analysis button
analysis = RadioGroup(
labels=['Weekly Pattern', 'Daily Pattern'],
active=0,
)
analysis.on_change('active', update)
## Weekday Checkbox
weekdays_checkbox = CheckboxGroup(
labels=[
'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday',
'Sunday'
],
active=[0, 1, 2, 3, 4, 5, 6],
)
weekdays_checkbox.on_change('active', update) # Run the whole update
## Country Selector
countries_available = np.unique(filterData['country'])
countries_available = countries_available.tolist()
country_selector = Dropdown(label="Country",
button_type="warning",
menu=countries_available,
value="USA",
max_height=150,
width=400)
country_selector.on_change('value', update)
## State Selector
states_available = np.unique(
filterData[filterData['country'] == "USA"]["state"])
states_available = states_available.tolist()
state_selector = Dropdown(label="Region",
button_type="warning",
menu=states_available,
value="NY",
max_height=150,
width=500)
state_selector.on_change('value', update)
## Home Selector
home_ids_available = np.unique(
filterData[filterData['state'] == 'NY']['dataid'])
home_ids_available = list(map(str, home_ids_available))
home_id_selector = Dropdown(label="Home ID",
button_type="warning",
menu=home_ids_available,
value="27",
max_height=150,
width=600)
home_id_selector.on_change('value', update)
##Aggregate Country/State/ home Button
aggregate_selector = RadioButtonGroup(labels=[
f'Aggregate By Country: {country_selector.value}',
f'Aggregate By Region: {state_selector.value}',
f'Aggregate By Home: {home_id_selector.value}'
],
active=2)
aggregate_selector.on_change('active', update)
## Date Range Selector
date_slider = DateRangeSlider(title="Date Range: ",
start=date(2019, 5, 1),
end=date(2019, 8, 20),
value=(date(2019, 5, 1), date(2019, 8, 20)),
step=1,
callback_policy='mouseup',
width=1000)
date_slider.on_change("value_throttled", update)
## Data Options
data_type_selector = RadioButtonGroup(labels=[
"Net Load", "Load + Battery(Charging)",
"PV Generation + Battery(Discharge)", "Electric Vehicle Consumption"
],
active=0,
max_height=150)
data_type_selector.on_change('active', update)
## Initialize opening plot and data
xaxis = '15 Minutes'
initial_home_id = 27
initial_daterange_to_plot = ['2019-05-01', '2019-08-20']
initial_data_type_to_plot = "grid"
initial_day_type = "avgday"
initial_data = filterData[
filterData['dataid'] == initial_home_id].sort_values(
'time', ascending=True)[[initial_data_type_to_plot, 'time']]
initial_data.index = initial_data['time']
initial_data = initial_data.loc[
initial_daterange_to_plot[0]:
initial_daterange_to_plot[1], :] # cut to the days requested
src1 = plot1_data(initial_data, initial_data_type_to_plot,
xaxis) # start with a data range we know is correct
plot1 = plot1_plot(src1)
src2 = plot2_data(initial_data, [], initial_data_type_to_plot,
initial_day_type)
plot2 = plot2_plot(src2)
## Put controls in a single element (add more later to format)
controls_plot1_text = Paragraph(text='Sampling Rate')
controls_plot1 = WidgetBox(
controls_plot1_text, granularity_1,
sizing_mode="scale_width") # data_type_selector)
controls_plot2_text = Paragraph(text='Pattern Type')
controls_plot2_text2 = Paragraph(text='Days Included')
controls_plot2 = WidgetBox(controls_plot2_text,
analysis,
controls_plot2_text2,
weekdays_checkbox,
sizing_mode="scale_width")
row1 = row(country_selector,
state_selector,
home_id_selector,
aggregate_selector,
sizing_mode="scale_height")
row2 = row(data_type_selector)
row3 = row(date_slider)
row4 = row(plot1, controls_plot1)
row5 = row(plot2, controls_plot2)
## Create a row layout
layout = column(row1, row2, row3, row4, row5)
## Make a tab with the layout
tab = Panel(child=layout, title='Input Data')
return tab
def update_stats():
stats_plot.x_range.factors = data_provider.dispatch_types
def update():
"""Periodic callback."""
data_provider.fetch_data()
update_stats()
def update_date(attr, old, new):
if new != old:
data_provider.set_date(new)
update_stats()
def update_casualties(attr, old, new):
if new != old:
data_provider.set_casualties(new)
update_stats()
date_slider.on_change("value_throttled", update_date)
casualties_slider.on_change("value_throttled", update_casualties)
curdoc().add_root(main_map)
curdoc().add_root(full_table)
curdoc().add_root(stats_plot)
curdoc().add_root(date_slider)
curdoc().add_root(casualties_slider)
#curdoc().add_periodic_callback(update, cfg.UPDATE_INTERVAL)
def timeseries_tab(dataframe):
"""
return a tab showing steps vs time for each person
"""
def make_dataset(name_list,
range_start='2019-01-01',
range_end='2019-12-31'):
"""
Filter the full dataset by name and by date range,
and return it as a Bokeh ColumnDataSource
"""
## why do I have to do this? What is the point of a DateRangeSlider???
if isinstance(range_start, int):
range_start = datetime.fromtimestamp(range_start / 1000)
if isinstance(range_end, int):
range_end = datetime.fromtimestamp(range_end / 1000)
# filtered_df = dataframe[name_list].loc[range_start: range_end]
filtered_df = dataframe.loc[range_start:range_end]
source = ColumnDataSource(filtered_df)
return source
def style(p):
# Title
p.title.align = 'center'
p.title.text_font_size = '20pt'
p.title.text_font = 'serif'
# Axis titles
p.xaxis.axis_label_text_font_size = '14pt'
p.xaxis.axis_label_text_font_style = 'bold'
p.yaxis.axis_label_text_font_size = '14pt'
p.yaxis.axis_label_text_font_style = 'bold'
# Tick labels
p.xaxis.major_label_text_font_size = '12pt'
p.yaxis.major_label_text_font_size = '12pt'
return p
def make_plot(source):
"""
create a Bokeh figure with the selected data
"""
names_to_plot = source.column_names[1:]
time_series = figure(x_axis_type="datetime",
plot_width=700,
plot_height=550)
hover = HoverTool(tooltips=[("Name", "$name"), ("Date", "@Date{%F}"),
("Steps", "$y")],
formatters={'Date': 'datetime'})
time_series = style(time_series)
time_series.add_tools(hover)
time_series.legend.location = "top_left"
for i, name in enumerate(names_to_plot):
time_series.line("Date",
name,
source=source,
line_color=Category20_16[i],
line_width=2,
name=name,
legend_label=name)
return time_series
def update(attr, old, new):
"""
Update data source when something is changed.
"""
name_list = [name_selection.labels[i] for i in name_selection.active]
new_src = make_dataset(name_list, date_slider.value[0],
date_slider.value[1])
cds.data.update(new_src.data)
def update_lines(attr, old, new):
"""
Hide selected lines
"""
names_to_plot = [
name_selection.labels[i] for i in name_selection.active
]
for name in names:
if name in names_to_plot:
p.select_one({"name": name}).visible = True
else:
p.select_one({"name": name}).visible = False
### back to the timeseries_tab function
names = list(dataframe.columns)
names.sort()
# widgets to allow user to configure the plot
name_selection = CheckboxGroup(labels=names,
active=list(range(len(names))))
name_selection.on_change('active', update_lines)
date_slider = DateRangeSlider(title="Date range",
start=date(2019, 1, 1),
end=date(2019, 12, 31),
value=(date(2019, 1, 1), date(2019, 12, 31)),
step=1)
date_slider.on_change('value', update)
initial_names = [name_selection.labels[i] for i in name_selection.active]
cds = make_dataset(initial_names, date_slider.value[0],
date_slider.value[1])
p = make_plot(cds)
controls = WidgetBox(name_selection, date_slider)
layout = row(controls, p)
tab = Panel(child=layout, title="Time series")
return tab
def overview_tab(df, x_name, date_name, measure):
def make_dataset(category_list, range_start, range_end):
reduced = pd.DataFrame()
for cat in category_list:
subset = df[df[x_name] == cat]
if (type(range_start) == int) or (type(range_start) == float):
range_start = datetime.fromtimestamp(range_start / 1000)
range_end = datetime.fromtimestamp(range_end / 1000)
print(range_start, range_end)
subset = subset[(subset[date_name] > range_start) & (subset[date_name] < range_end)]
reduced = reduced.append(subset)
temp = pd.DataFrame(zip(viridis(len(category_list)), category_list), columns=['color', x_name])
reduced = pd.merge(reduced, temp, on = x_name)
reduced = reduced.sort_values(by=date_name, ascending=True)
return ColumnDataSource(reduced)
def style(p):
# Title
p.title.align = 'center'
p.title.text_font_size = '20pt'
p.title.text_font = 'serif'
# Axis titles
p.xaxis.axis_label_text_font_size = '14pt'
p.xaxis.axis_label_text_font_style = 'bold'
p.yaxis.axis_label_text_font_size = '14pt'
p.yaxis.axis_label_text_font_style = 'bold'
# Tick labels
p.xaxis.major_label_text_font_size = '12pt'
p.yaxis.major_label_text_font_size = '12pt'
p.xaxis.formatter = DatetimeTickFormatter(
hours=["%d %B %Y"],
days=["%d %B %Y"],
months=["%d %B %Y"],
years=["%d %B %Y"]
)
# Names in Angles
p.xaxis.major_label_orientation = np.pi / 4
return p
def make_plot(src):
# Create the blank plot
p = figure(plot_width=1200, plot_height=700,
title=measure + ' vs ' + date_name,
x_axis_label=date_name, y_axis_label=measure)
# p.line(src.data[date_name], src.data[measure], line_width=0.9, line_color='grey')
# p.circle(src.data[date_name], src.data[measure], fill_color='grey', line_color= 'black')
p.line(x = date_name, y = measure, source = src, line_width=0.9, line_color='grey')
p.circle(x = date_name, y = measure, source = src, size = 10, fill_color='color', fill_alpha=0.75, line_color='black',
hover_fill_alpha=1.0, hover_fill_color='green')
# Hover tool referring to our own data field using @ and
# a position on the graph using $
h = HoverTool(tooltips=[(x_name, '@' + x_name),
(measure, '@' + measure)])
# Add the hover tool to the graph
p.add_tools(h)
# Style the plot
p = style(p)
return p
def update(attr, old, new):
categories_to_plot = [category_selection.labels[i] for i in category_selection.active]
new_src = make_dataset(categories_to_plot, range_select.value[0], range_select.value[1])
src.data.update(new_src.data)
df = df.sort_values(by = date_name, ascending= True)
available_categories = list(set(df[x_name]))
category_selection = CheckboxGroup(labels=available_categories,
active=[i for i in range(len(available_categories))])
category_selection.on_change('active', update)
# Initial categories and data source
initial_categories = [category_selection.labels[i] for i in category_selection.active]
# categories and colors
range_select = DateRangeSlider(start=df[date_name].iloc[0], end=df[date_name].iloc[-1],
value=(df[date_name].iloc[0], df[date_name].iloc[-1]),
step=1, title='Time Window')
range_select.on_change('value', update)
# Initial categories and data source
src = make_dataset(initial_categories, range_start = range_select.value[0], range_end = range_select.value[1])
p = make_plot(src)
# Put controls in a single element
controls = WidgetBox(category_selection, range_select)
# Create a row layout
layout = row(controls, p)
# Make a tab with the layout
tab = Panel(child = layout, title = 'Overview')
return tab