class DataGrab:
"""
provides a list of available experimental runs.
Sensitive to the specific folder structure of the e11 lab
"""
def __init__(self,ROOTFOLDER):
self.ROOTFOLDER = ROOTFOLDER
self.date_picker =DatePicker(title='Select date of experiment',min_date=date(2017,1,1),max_date=date.today())
self.date_picker.on_change('value',self._date_callback)
self.file_data = dict(
runid=[],
address=[],
filename=[]
)
self.file_source = ColumnDataSource(self.file_data)
self.file_columns = [
TableColumn(field="runid", title="Run ID"),
TableColumn(field="filename", title="file name")
]
self.data_table = DataTable(source=self.file_source, columns=self.file_columns,width =300,height=500)
def _date_callback(self,attr,old,new):
"""
on change of data, update the datatable showing all runs which can be plotted
"""
date_string = date_to_file(self.date_picker.value)
self.data_table.source.data = get_files(self.ROOTFOLDER,date_string[0],date_string[1],date_string[2])
def show(self):
return column(self.date_picker,self.data_table)
def modify_doc(doc):
crnt_date = dt.now()
dt_pckr_strt = DatePicker(title='Select start of sync date',
min_date=date(2017, 1, 1),
max_date=date.today())
def callback(attr, old, new):
print(type(old))
print('old was {} and new is {}'.format(old, new))
dt_pckr_strt.on_change('value', callback)
doc.add_root(bokehCol(dt_pckr_strt))
class Protocol:
def __init__(self):
note = Div(
text="<b>NOTE</b>: Each plan may only have one protocol assigned. "
"Updating database will overwrite any existing data.",
width=700)
self.update_checkbox = CheckboxGroup(
labels=["Only update plans in table."], active=[0])
self.toxicity = [] # Will be used to link to Toxicity tab
self.source = ColumnDataSource(data=dict(mrn=['']))
self.source.selected.on_change('indices', self.source_listener)
self.clear_source_selection_button = Button(label='Clear Selection',
button_type='primary',
width=150)
self.clear_source_selection_button.on_click(
self.clear_source_selection)
self.protocol = Select(value='', options=[''], title='Protocols:')
self.physician = Select(value='',
options=[''],
title='Physician:',
width=150)
self.date_filter_by = Select(
value='None',
options=['None', 'sim_study_date', 'import_time_stamp'],
title='Date Filter Type:',
width=150)
self.date_filter_by.on_change('value', self.date_ticker)
self.date_start = DatePicker(title='Start Date:', width=200)
self.date_start.on_change('value', self.date_ticker)
self.date_end = DatePicker(title='End Date:', width=200)
self.date_end.on_change('value', self.date_ticker)
self.update_protocol_options()
self.update_physician_options()
self.protocol_input = TextInput(value='',
title='Protocol for MRN Input:')
self.update_button = Button(label='Need MRNs to Update',
button_type='default',
width=150)
self.update_button.on_click(self.update_db)
self.mrn_input = TextAreaInput(value='',
title='MRN Input:',
rows=30,
cols=25,
max_length=2000)
self.mrn_input.on_change('value', self.mrn_input_ticker)
self.columns = [
'mrn', 'protocol', 'physician', 'tx_site', 'sim_study_date',
'import_time_stamp', 'toxicity_grades'
]
relative_widths = [1, 0.8, 0.5, 1, 0.75, 1, 0.8]
column_widths = [int(250. * rw) for rw in relative_widths]
table_columns = [
TableColumn(field=c, title=c, width=column_widths[i])
for i, c in enumerate(self.columns)
]
self.table = DataTable(source=self.source,
columns=table_columns,
width=800,
editable=True,
height=600)
self.protocol.on_change('value', self.protocol_ticker)
self.physician.on_change('value', self.physician_ticker)
self.update_source()
self.layout = column(
row(self.protocol, self.physician),
row(self.date_filter_by, Spacer(width=30), self.date_start,
Spacer(width=30), self.date_end), note,
row(
self.table, Spacer(width=30),
column(self.update_checkbox,
row(self.protocol_input, self.update_button),
self.clear_source_selection_button, self.mrn_input)))
def source_listener(self, attr, old, new):
mrns = [self.source.data['mrn'][x] for x in new]
self.mrn_input.value = '\n'.join(mrns)
def update_source(self):
condition = []
if self.protocol.value not in self.protocol.options[0:3]:
condition.append("protocol = '%s'" % self.protocol.value)
elif self.protocol.value == self.protocol.options[1]:
condition.append("protocol != ''")
elif self.protocol.value == self.protocol.options[2]:
condition.append("protocol = ''")
if self.physician.value != self.physician.options[0]:
condition.append("physician = '%s'" % self.physician.value)
if self.date_filter_by.value != 'None':
if self.date_start.value:
condition.append(
"%s >= '%s'::date" %
(self.date_filter_by.value, self.date_start.value))
if self.date_end.value:
condition.append(
"%s >= '%s'::date" %
(self.date_filter_by.value, self.date_end.value))
condition = ' AND '.join(condition)
columns = ', '.join(self.columns + ['study_instance_uid'])
data = DVH_SQL().query('Plans',
columns,
condition,
order_by='mrn',
bokeh_cds=True)
self.source.data = data
def update_protocol_options(self):
options = ['All Data', 'Any Protocol', 'No Protocol'
] + self.get_protocols()
self.protocol.options = options
if self.protocol.value not in options:
self.protocol.value = options[0]
@property
def mrns_to_add(self):
return parse_text_area_input_to_list(self.mrn_input.value,
delimeter=None)
@staticmethod
def get_protocols(condition=None):
return DVH_SQL().get_unique_values('Plans',
'protocol',
condition,
ignore_null=True)
def update_physician_options(self):
physicians = ['Any'] + DVH_SQL().get_unique_values(
'Plans', 'physician')
self.physician.options = physicians
if self.physician.value not in physicians:
self.physician.value = physicians[0]
def protocol_ticker(self, attr, old, new):
self.update_source()
def physician_ticker(self, attr, old, new):
self.update_source()
def date_ticker(self, attr, old, new):
self.update_source()
def update_db(self):
if 0 in self.update_checkbox.active:
condition = "mrn in ('%s')" % "', '".join(self.mrns_to_add)
else:
uids = []
for i, mrn in enumerate(self.mrns_to_add):
if mrn in self.source.data['mrn']:
index = self.source.data['mrn'].index(mrn)
uids.append(self.source.data['study_instance_uid'][index])
condition = "study_instance_uid in ('%s')" % "', '".join(uids)
DVH_SQL().update(
'Plans', 'protocol',
self.protocol_input.value.replace("'",
"").replace("\"", "").replace(
"\\", ""), condition)
self.update_source()
self.clear_source_selection()
self.protocol_input.value = ''
self.update_protocol_options()
self.toxicity.update_protocol_options()
self.toxicity.update_source()
def clear_source_selection(self):
self.source.selected.indices = []
def add_toxicity_tab_link(self, toxicity):
self.toxicity = toxicity
def mrn_input_ticker(self, attr, old, new):
self.update_update_button_status()
def update_update_button_status(self):
if self.mrns_to_add:
self.update_button.label = 'Update'
self.update_button.button_type = 'primary'
else:
self.update_button.label = 'Need MRNs to Update'
self.update_button.button_type = 'default'
radius_select = TextInput(value='5', title='Select Radius (Miles)')
radius_select.on_change('value', update)
solution_select = TextInput(value='0', title='Select Solution')
solution_select.on_change('value', update)
tran_type_select = CheckboxGroup(labels=['Sale', 'Return'],
active=[0, 1],
name='Transaction Type')
tran_type_select.on_change('active', update)
min_date_select = DatePicker(title='Start Date',
value=min_date1,
min_date=min_date1,
max_date=max_date1)
min_date_select.on_change('value', update)
max_date_select = DatePicker(title='End Date',
value=max_date1,
min_date=min_date1,
max_date=max_date1)
max_date_select.on_change('value', update)
guideline = Div(text="""<center>
<h2>Dashboard Guideline</h2>
</center><center>
<h3>Purpose</h3>
</center>
<p style="margin-left: 25px;">This dashboard shows the best in-state location covering the most unreached PUMA ecommerce transactions, which are not in vicinity of a physical store.</p>
<center>
<h3>Parameters</h3>
</center>
columns_tb3 = []
for c in positions.column_names:
if c in ['IDs', 'cs_level_1', 'wind_level_1', 'name', 'list_date']:
columns_tb3.append(TableColumn(field=c, title=c))
else:
columns_tb3.append(TableColumn(field=c, title=c, formatter=NumberFormatter(format='0.00%')))
table_tb3 = DataTable(source=positions, columns=columns_tb3, width=1000, height=600)
def update_data_tb3():
strategy = strategy_select_tb3.value
date = as_timestamp(datepicker.value)
data = strategy_data_provider.load_positions(date, strategy)
# print(data)
positions.data = positions.from_df(data)
datepicker.on_change('value', lambda attr, old, new: update_data_tb3())
strategy_select_tb3.on_change('value', lambda attr, old, new: update_data_tb3())
update_data_tb3()
controls_tb3 = widgetbox(datepicker, strategy_select_tb3)
tab3 = Panel(child=column(controls_tb3, widgetbox(table_tb3)), title='POSITIONS')
# Tab Four
all_benchmarks = {MARKET_INDEX_WINDCODE_REVERSE[x]: x[:6] for x in index_weights}
def max_date_tab4():
risk_ds = RiskDataSource('xy')
return as_timestamp(risk_ds.max_date_of_factor)
# test for paragraph at the top of the page. stored in HTML.
description = Div(text=open(join(dirname(__file__), "description.html")).read(), width=800)
''' --- Left side of page: Onset and demographic data --- '''
## Onset Panel
# date onset
def dt_date_onset_handler(attr, old, new):
print(type(old))
print(type(new))
print('old was {} and new is {}'.format(old,new))
dt_date_onset=DatePicker(title="Date of ALS Onset",
min_date=date(1990,1,1),
max_date=date.today(),
value=date(2015,3,1))
dt_date_onset.on_change("value", dt_date_onset_handler)
# subject id
def subject_id_handler(attr, old, new):
print("Previous label: " + tpye(old))
print("Updated label: " + new)
subject_id = TextInput(title="Subject ID:", value="ref number")
#subject_id.on_change("value", subject_id_handler)
# age at onset
def age_onset_handler(attr, old, new):
print("Previous label: " + type(old))
print("Updated label: " + new)
age_onset = TextInput(title="Age at ALS Onset (years)", value="")
#age_onset.on_change("value", age_onset_handler)
def blockminer_tab(page_width):
# source for top N table
topN_src = ColumnDataSource(
data=dict(percentage=[], address=[], block_count=[]))
class This_tab(Mytab):
def __init__(self, table, cols, dedup_cols):
Mytab.__init__(self, table, cols, dedup_cols)
self.table = table
self.df2 = None
self.key_tab = 'blockminer'
self.n = 20
# ------- DIVS setup begin
self.page_width = page_width
txt = """<hr/>
<div style="text-align:center;width:{}px;height:{}px;
position:relative;background:black;margin-bottom:200px">
<h1 style="color:#fff;margin-bottom:300px">{}</h1>
</div>""".format(self.page_width, 50, 'Welcome')
self.notification_div = {
'top': Div(text=txt, width=self.page_width, height=20),
'bottom': Div(text=txt, width=self.page_width, height=10),
}
self.section_divider = '-----------------------------------'
self.section_headers = {}
# ----- UPDATED DIVS END
# ---------------------- DIVS ----------------------------
def section_header_div(self,
text,
html_header='h2',
width=600,
margin_top=150,
margin_bottom=-150):
text = """<div style="margin-top:{}px;margin-bottom:-{}px;"><{} style="color:#4221cc;">{}</{}></div>""" \
.format(margin_top, margin_bottom, html_header, text, html_header)
return Div(text=text, width=width, height=15)
def load_this_data(self, start_date, end_date):
end_date = datetime.combine(end_date, datetime.min.time())
start_date = datetime.combine(start_date, datetime.min.time())
logger.warning('load_data start date:%s', start_date)
logger.warning('load_data end date:%s', end_date)
# load only mined blocks and remove the double entry
supplemental_where = "AND update_type = 'mined_block' AND amount >= 0"
self.df_load(start_date,
end_date,
supplemental_where=supplemental_where,
cols=['address', 'amount', 'block_time'])
logger.warning('after load:%s', self.df.head(30))
return self.prep_dataset(start_date, end_date)
def prep_dataset(self, start_date, end_date):
try:
logger.warning("prep dataset start date:%s", start_date)
self.df1 = self.df1[['address', 'block_time']]
self.df1['address'] = self.df1['address']\
.map(self.poolname_verbose_trun)
self.df1 = self.df1.groupby(['address'
]).agg({'block_time': 'count'})
self.df1 = self.df1.reset_index()
self.df1 = self.df1.rename(
columns={'block_time': 'block_count'})
self.df1['percentage'] = 100*self.df1['block_count']\
/self.df1['block_count'].sum()
self.df1['percentage'] = self.df1['percentage'].map(
lambda x: round(x, 1))
self.df1 = self.df1.reset_index()
logger.warning("topN column:%s", self.df1.columns.tolist())
#logger.warning('END prep dataset DF1:%s', self.df1.head())
return self.df1.hvplot.bar(
'address',
'block_count',
rot=90,
height=600,
width=self.page_width,
title='# of blocks mined by miner address',
hover_cols=['percentage'])
except Exception:
logger.error('prep dataset:', exc_info=True)
def view_topN(self):
logger.warning("top n called:%s", self.n)
# change n from string to int
try:
#table_n = df1.hvplot.table(columns=['address','percentage'],
#title=title, width=400)
logger.warning('top N:%s', self.n)
df2 = self.df1.nlargest(self.n, 'percentage')
df2 = df2.compute()
logger.warning('in view top n :%s', df2.head(10))
new_data = dict(percentage=df2.percentage.tolist(),
address=df2.address.tolist(),
block_count=df2.block_count.tolist())
topN_src.stream(new_data, rollover=self.n)
columns = [
TableColumn(field="address", title="Address"),
TableColumn(field="percentage", title="percentage"),
TableColumn(field="block_count", title="# of blocks")
]
table_n = DataTable(source=topN_src,
columns=columns,
width=300,
height=600)
gc.collect()
return table_n
except Exception:
logger.error('view_topN:', exc_info=True)
def set_n(self, n):
if isinstance(n, int):
pass
else:
try:
self.n = int(n)
except Exception:
logger.error('set_n', exc_info=True)
# ####################################################
# UTILITY DIVS
def results_div(self, text, width=600, height=300):
div = Div(text=text, width=width, height=height)
return div
def title_div(self, text, width=700):
text = '<h2 style="color:green;">{}</h2>'.format(text)
return Div(text=text, width=width, height=20)
def notification_updater_2(self, text):
self.notification_div.text = '<h3 style="color:red">{}</h3>'.format(
text)
def spacing_div(self, width=20, height=100):
return Div(text='', width=width, height=height)
def spacing_paragraph(self, width=20, height=100):
return Paragraph(text='', width=width, height=height)
def update(attrname, old, new):
this_tab.notification_updater(
"Calculations underway. Please be patient")
stream_start_date.event(start_date=datepicker_start.value)
stream_end_date.event(end_date=datepicker_end.value)
this_tab.set_n(topN_select.value)
this_tab.view_topN()
this_tab.notification_updater("ready")
# update based on selected top n
def update_topN():
this_tab.notification_updater("Calculations in progress! Please wait.")
logger.warning('topN selected value:%s', topN_select.value)
this_tab.set_n(topN_select.value)
this_tab.view_topN()
this_tab.notification_updater("ready")
try:
# create class and get date range
cols = ['address', 'block_timestamp', 'block_time']
this_tab = This_tab('account_ext_warehouse', cols, [])
#STATIC DATES
first_date_range = "2018-04-23 00:00:00"
first_date_range = datetime.strptime(first_date_range,
"%Y-%m-%d %H:%M:%S")
last_date_range = datetime.now().date()
last_date = last_date_range
first_date = datetime_to_date(last_date - timedelta(days=60))
# STREAMS Setup
# date comes out stream in milliseconds
stream_start_date = streams.Stream.define('Start_date',
start_date=first_date)()
stream_end_date = streams.Stream.define('End_date',
end_date=last_date)()
# create a text widget for top N
topN_select = Select(title='Top N',
value=str(this_tab.n),
options=menu)
datepicker_start = DatePicker(title="Start",
min_date=first_date_range,
max_date=last_date_range,
value=first_date)
datepicker_end = DatePicker(title="End",
min_date=first_date_range,
max_date=last_date_range,
value=last_date)
# ALL MINERS.
# --------------------- ALL MINERS ----------------------------------
hv_bar_plot = hv.DynamicMap(
this_tab.load_this_data,
streams=[stream_start_date, stream_end_date],
datashade=True)
renderer = hv.renderer('bokeh')
bar_plot = renderer.get_plot(hv_bar_plot)
# --------------------- TOP N MINERS -----------------------------------
# set up data source for the ton N miners table
this_tab.view_topN()
columns = [
TableColumn(field="address", title="Address"),
TableColumn(field="percentage", title="percentage"),
TableColumn(field="block_count", title="# of blocks")
]
topN_table = DataTable(source=topN_src,
columns=columns,
width=400,
height=600)
# add callbacks
datepicker_start.on_change('value', update)
datepicker_end.on_change('value', update)
topN_select.on_change("value", lambda attr, old, new: update_topN())
download_button = Button(label='Save Table to CSV',
button_type="success")
download_button.callback = CustomJS(
args=dict(source=topN_src),
code=open(
join(dirname(__file__),
"../../../static/js/topN_download.js")).read())
# put the controls in a single element
controls = WidgetBox(datepicker_start, datepicker_end, download_button,
topN_select)
# create the dashboards
grid = gridplot([[this_tab.notification_div['top']],
[Spacer(width=20, height=70)], [
topN_table,
controls,
], [bar_plot.state]])
# Make a tab with the layout
tab = Panel(child=grid, title='miners: blocks')
return tab
except Exception:
logger.error("Blockminer", exc_info=True)
return tab_error_flag('miners: blocks')
# linacs.insert(0, 'All')
# linacs.append('None')
# select_linac = {key: Select(title='Linac %s:' % key, value='All', options=['All'], width=250) for key in [1, 2]}
# select_linac[2].value = 'None'
# select_linac[1].on_change('value', plot.update_source)
# select_linac[2].on_change('value', plot.update_source)
avg_len_input = TextInput(title='Avg. Len:', value='10', width=100)
avg_len_input.on_change('value', plot.update_source)
percentile_input = TextInput(title='Percentile:', value='90', width=100)
percentile_input.on_change('value', plot.update_source)
start_date_picker = DatePicker(title='Start Date:', value=plot.x[0])
end_date_picker = DatePicker(title='End Date:', value=plot.x[-1])
start_date_picker.on_change('value', plot.update_source)
end_date_picker.on_change('value', plot.update_source)
gamma_options = ['5.0%/3.0mm', '3.0%/3.0mm', '3.0%/2.0mm', 'Any']
checkbox_button_group = CheckboxButtonGroup(labels=gamma_options, active=[3])
checkbox_button_group.on_change('active', plot.update_source)
text = {key: Div() for key in [1, 2]}
plot.update_source(None, None, None)
layout = column(row(select_y, avg_len_input, percentile_input),
row(start_date_picker, end_date_picker),
row(Div(text='Gamma Criteria: '), checkbox_button_group),
text[1], text[2], row(Spacer(width=10), plot.fig),
Spacer(height=50), row(Spacer(width=10), plot.histogram),
class StateDisplay:
def __init__(self, dataset=STATES):
self.dataset = dataset
self.state_selection = MultiSelect(
title="States:",
options=self.dataset,
value=["New York", "Texas"],
sizing_mode="stretch_both",
)
self.per_capita = RadioGroup(
labels=["Total", "Per Capita"],
active=0,
sizing_mode="stretch_width",
)
self.data_getter = RadioGroup(
labels=[
"Cases",
"Deaths",
"Positivity",
"Testing",
"Constant Positivity",
"Constant Testing",
],
active=0,
sizing_mode="stretch_width",
)
self.plot_type = RadioGroup(
labels=["Linear", "Logarithmic"],
active=0,
sizing_mode="stretch_width",
)
self.constant_date = DatePicker(
title="Constant Date",
value=(datetime.today() - timedelta(days=1)).date(),
sizing_mode="stretch_width",
)
self.show_total = CheckboxGroup(
labels=["Show total"],
sizing_mode="stretch_width",
)
self.total_only = CheckboxGroup(
labels=["Total only"],
sizing_mode="stretch_width",
)
self.src = None
self.p = None
self.logp = None
self.tooltips = [("State", "@state")]
def make_dataset(self, state_list):
by_state = {
"avg_date": [],
"avg_data": [],
"state": [],
"color": [],
"line-width": [],
}
color_cycle = cycle(Category20_20)
palette = [next(color_cycle) for _ in self.dataset]
show_total = self.show_total.active == [0]
total_only = self.total_only.active == [0]
totals = None
totals_denom = None
for state_name in state_list:
per_capita = self.per_capita.active == 1
data_getter = self.data_getter.labels[
self.data_getter.active].lower()
constant_date = self.constant_date.value
(
dates,
avg_dates,
data,
avg_data,
test_data,
label,
tot_positive,
tot_testing,
) = get_data(state_name, per_capita, data_getter, constant_date)
if tot_positive is None and tot_testing is None:
subtotal = pd.Series(avg_data.values[7:])
subtotal.index = pd.DatetimeIndex(avg_dates.values[7:])
subtotal_denom = None
else:
subtotal = pd.Series(tot_positive.values[7:])
subtotal.index = pd.DatetimeIndex(avg_dates.values[7:])
subtotal_denom = pd.Series(tot_testing.values[7:])
subtotal_denom.index = pd.DatetimeIndex(avg_dates.values[7:])
idx = pd.date_range(subtotal.index.min(), subtotal.index.max())
subtotal = subtotal.reindex(idx)
subtotal.interpolate(method="time", inplace=True)
if subtotal_denom is not None:
subtotal_denom = subtotal_denom.reindex(idx)
subtotal_denom.interpolate(method="time", inplace=True)
if totals is None:
totals = subtotal
if subtotal_denom is not None:
totals_denom = subtotal_denom
else:
idx = pd.date_range(
min(subtotal.index.min(), totals.index.min()),
max(subtotal.index.max(), totals.index.max()),
)
totals = totals.reindex(idx, fill_value=0)
subtotal = subtotal.reindex(idx, fill_value=0)
totals += subtotal
if subtotal_denom is not None:
totals_denom = totals_denom.reindex(idx, fill_value=0)
subtotal_denom = subtotal_denom.reindex(idx, fill_value=0)
totals_denom += subtotal_denom
if len(state_list) == 1 or not show_total or not total_only:
by_state["avg_date"].append(avg_dates.values)
by_state["avg_data"].append(avg_data.values)
by_state["state"].append(state_name)
by_state["color"].append(
palette[self.dataset.index(state_name)])
by_state["line-width"].append(1)
if totals_denom is not None:
totals /= totals_denom
if show_total:
by_state["avg_date"].append(totals.index.values)
by_state["avg_data"].append(totals.values)
by_state["state"].append("Total")
by_state["color"].append("black")
by_state["line-width"].append(2)
return label, ColumnDataSource(by_state)
def make_plot(self):
self.p = figure(
x_axis_label="Date",
x_axis_type="datetime",
y_axis_label="Total Cases",
width=900,
)
self.p.multi_line(
source=self.src,
xs="avg_date",
ys="avg_data",
legend_field="state",
color="color",
line_width="line-width",
)
self.p.add_tools(HoverTool(tooltips=self.tooltips))
self.p.legend.location = "top_left"
self.logp = figure(
x_axis_label="Date",
x_axis_type="datetime",
y_axis_label="Total Cases",
y_axis_type="log",
width=900,
)
self.logp.multi_line(
source=self.src,
xs="avg_date",
ys="avg_data",
legend_field="state",
color="color",
line_width="line-width",
)
self.logp.add_tools(HoverTool(tooltips=self.tooltips))
self.logp.legend.location = "top_left"
def update_data(self, label, src):
if self.src is None:
self.src = src
self.make_plot()
else:
self.src.data.update(src.data)
if self.plot_type.active == 0:
self.p.visible = True
self.logp.visible = False
else:
self.p.visible = False
self.logp.visible = True
self.p.yaxis.axis_label = label
self.logp.yaxis.axis_label = label
if len(self.data_getter.labels) == 1:
self.data_getter.visible = False
data_getter = self.data_getter.labels[self.data_getter.active].lower()
self.constant_date.visible = data_getter in (
"constant positivity",
"constant testing",
)
def update(self, attr, old, new):
states_to_plot = sorted(self.state_selection.value)
label, new_src = self.make_dataset(states_to_plot)
self.update_data(label, new_src)
self.show_total.visible = len(states_to_plot) != 1
self.total_only.visible = self.show_total.active == [0]
def run(self):
self.state_selection.on_change("value", self.update)
self.per_capita.on_change("active", self.update)
self.data_getter.on_change("active", self.update)
self.plot_type.on_change("active", self.update)
self.constant_date.on_change("value", self.update)
self.show_total.on_change("active", self.update)
self.total_only.on_change("active", self.update)
controls = column(
[
self.state_selection,
self.per_capita,
self.data_getter,
self.plot_type,
self.constant_date,
self.show_total,
self.total_only,
],
sizing_mode="fixed",
width=300,
height=600,
)
self.update(None, None, None)
plots = column(self.p, self.logp)
return row(controls, plots, sizing_mode="stretch_both")
code_text = TextInput(value=None, title="代码:", width=420)
code_text.on_change('value', update_stock)
msource = ColumnDataSource(dict(code=list(), pday=list(), profit=list()))
mmap_title = Div(text="股票分析",
width=120,
height=40,
margin=[25, 0, 0, 0],
style={
'font-size': '150%',
'color': 'blue'
})
mmap_pckr = DatePicker(title='开始日期',
value=date.today(),
min_date=date(2000, 1, 1),
max_date=date.today())
mmap_pckr.on_change('value', update_mmap)
mmap_select_row = row(mmap_title, mmap_pckr)
tsource = ColumnDataSource(dict(code=list(), pday=list(), profit=list()))
source_code = """
row = cb_obj.indices[0]
text_row.value = String(source.data['code'][row]);
"""
callback = CustomJS(args=dict(source=tsource, text_row=code_text),
code=source_code)
tsource.selected.js_on_change('indices', callback)
columns = [
TableColumn(field="code", title="代码"),
TableColumn(field="pday", title="牛熊天数", sortable=True),
TableColumn(field="profit", title="牛熊程度", sortable=True)
]
mtable = DataTable(source=tsource, columns=columns, width=1300, height=200)
class MapBase:
def __init__(self):
self.per_capita = RadioGroup(
labels=["Total", "Per Capita", "Logarithmic"],
active=0,
sizing_mode="stretch_width",
)
self.data_getter = RadioGroup(
labels=["Cases", "Deaths", "Positivity"],
active=0,
sizing_mode="stretch_width",
)
self.date = DatePicker(title="Date", sizing_mode="stretch_width")
self.save_files = CheckboxGroup(labels=["Save files"],
sizing_mode="stretch_width")
self.button = Button(label="► Play", sizing_mode="stretch_width")
self.tooltips = [("Name", "@name"), ("Value", "@value")]
self.src = None
self.p = None
self.callback = None
self.counter = None
self.tempdir = None
self.filenames = None
def make_dataset(self):
raise NotImplementedError
def make_plot(self, maxval):
color_mapper = LinearColorMapper(palette=PALETTE, low=0, high=maxval)
color_bar = ColorBar(
color_mapper=color_mapper,
ticker=BasicTicker(),
label_standoff=12,
border_line_color=None,
location=(0, 0),
)
self.p = figure(
toolbar_location="left",
tooltips=self.tooltips,
width=1000,
aspect_ratio=1.8,
)
self.p.patches(
source=self.src,
xs="lons",
ys="lats",
fill_color="color",
line_color="white",
line_width=0.5,
)
self.p.axis.visible = False
self.p.grid.visible = False
self.p.outline_line_color = None
self.p.add_layout(color_bar, "right")
def update(self, attr, old, new):
label, maxval, new_src = self.make_dataset()
if self.src is None:
self.src = new_src
self.make_plot(maxval)
else:
self.src.data.update(new_src.data)
strdate = date.fromisoformat(self.date.value).strftime("%B %d, %Y")
self.p.title.text = f"{label} on {strdate}"
color_mapper = LogColorMapper
self.p.right[0].color_mapper = color_mapper(palette=PALETTE,
low=0,
high=maxval)
self.p.right[0].ticker = BasicTicker()
def animate_update(self):
self.counter += 1
if self.save_files.active == [0]:
filename = os.path.join(
self.tempdir,
f"{self.__class__.__name__}_plot_{self.counter}.png",
)
export_png(self.p, filename=filename)
self.filenames.append(filename)
new_date = date.fromisoformat(self.date.value) + timedelta(days=1)
self.date.value = new_date.isoformat()
if new_date > self.date.enabled_dates[0][1] - timedelta(days=1):
self.animate()
def animate(self):
if self.button.label == "► Play":
self.button.label = "❚❚ Pause"
self.counter = 0
if self.save_files.active == [0]:
self.tempdir = tempfile.mkdtemp()
self.filenames = []
self.callback = curdoc().add_periodic_callback(
self.animate_update, 200)
else:
self.button.label = "► Play"
curdoc().remove_periodic_callback(self.callback)
if self.save_files.active == [0]:
with imageio.get_writer(f"{self.__class__.__name__}_plot.gif",
mode="I") as writer:
for filename in self.filenames:
image = imageio.imread(filename)
writer.append_data(image)
shutil.rmtree(self.tempdir)
def run(self):
self.per_capita.on_change("active", self.update)
self.data_getter.on_change("active", self.update)
self.date.on_change("value", self.update)
self.button.on_click(self.animate)
self.update(None, None, None)
controls = column(
[
self.per_capita,
self.data_getter,
self.date,
self.save_files,
self.button,
],
sizing_mode="fixed",
width=300,
height=600,
)
return row(controls, self.p)
def kpi_bcc_rentals_visitor_tab(panel_title):
class Thistab(KPI):
def __init__(self, table, cols=[]):
KPI.__init__(self, table,name='rentals',cols=cols)
self.table = table
self.df = None
self.pym = PythonMongo('aion')
self.checkboxgroup = {
'category': [],
'item' : [],
'area':[],
'visit_duration':[]
}
self.multiline_vars = {
'y' : 'visit_duration'
}
self.groupby_dict = {
'item':'count',
'area':'count',
'category':'count',
'status':'count',
'gender':'count',
'visit_duration':'sum'
}
# setup selects
self.select_values = {}
self.select_menus = {}
for item in ['area', 'item', 'category', 'status', 'gender']:
self.select_values[item] = 'all'
self.select_menus[item] = ['all']
self.select = {}
for item in ['area', 'item', 'category', 'status', 'gender']:
self.select[item] = Select(title='Select ' + item, value='all',
options=self.select_menus[item])
self.timestamp_col = 'visit_start'
self.variable = 'item'
self.multiline_resample_period = 'M'
self.ptd_startdate = datetime(datetime.today().year, 1, 1, 0, 0, 0)
# cards
self.KPI_card_div = self.initialize_cards(self.page_width, height=350)
# ------- DIVS setup begin
self.page_width = 1200
txt = """<hr/><div style="text-align:center;width:{}px;height:{}px;
position:relative;background:black;margin-bottom:200px">
<h1 style="color:#fff;margin-bottom:300px">{}</h1>
</div>""".format(self.page_width, 50, 'Welcome')
self.notification_div = {
'top': Div(text=txt, width=self.page_width, height=20),
'bottom': Div(text=txt, width=self.page_width, height=10),
}
self.section_divider = '-----------------------------------'
self.section_headers = {
'cards': self.section_header_div(text='Period to date:{}'.format(self.section_divider),
width=600, html_header='h2', margin_top=5,
margin_bottom=-155),
'pop': self.section_header_div(text='Period over period:{}'.format(self.section_divider),
width=600, html_header='h2', margin_top=5, margin_bottom=-155),
'dow': self.section_header_div(text='Compare days of the week:'.format(self.section_divider),
width=600, html_header='h2', margin_top=5, margin_bottom=-155),
}
# ---------------------- DIVS ----------------------------
def section_header_div(self, text, html_header='h2', width=600, margin_top=150, margin_bottom=-150):
text = """<div style="margin-top:{}px;margin-bottom:-{}px;"><{} style="color:#4221cc;">{}</{}></div>""" \
.format(margin_top, margin_bottom, html_header, text, html_header)
return Div(text=text, width=width, height=15)
# ---------------------- DIVS ----------------------------
def reset_checkboxes(self, value='all',checkboxgroup=''):
try:
self.checkboxgroup[checkboxgroup].value = value
except Exception:
logger.error('reset checkboxes', exc_info=True)
def information_div(self, width=400, height=300):
div_style = """
style='width:350px;margin-right:-800px;
border:1px solid #ddd;border-radius:3px;background:#efefef50;'
"""
txt = """
<div {}>
<h4 {}>How to interpret relationships </h4>
<ul style='margin-top:-10px;'>
<li>
</li>
<li>
</li>
<li>
</li>
<li>
e7 </li>
<li>
</li>
<li>
</li>
</ul>
</div>
""".format(div_style, self.header_style)
div = Div(text=txt, width=width, height=height)
return div
# ------------------- LOAD AND SETUP DATA -----------------------
def filter_df(self, df):
try:
for item in self.select_values.keys():
if self.select_values[item] != 'all':
df = df[df[item] == self.select_values[item]]
return df
except Exception:
logger.error('filters', exc_info=True)
def set_select_menus(self,df):
try:
for item in self.select.keys():
if item in df.columns and len(df) > 0:
logger.warning('LINE 151: item: %s', item)
lst = list(set(df[item].values))
lst.append('all')
sorted(lst)
logger.warning('LINE 157: LIST: %s',lst)
self.select[item].options = lst
except Exception:
logger.error('set filters menus', exc_info=True)
def load_df_pym(self, req_startdate, req_enddate, cols, timestamp_col):
try:
# get min and max of loaded df
if self.df is not None:
loaded_min = self.df[timestamp_col].min()
loaded_max = self.df[timestamp_col].max()
if loaded_min <= req_startdate and loaded_max >= req_enddate:
df = self.df[(self.df[timestamp_col] >= req_startdate) &
(self.df[timestamp_col] <= req_enddate)]
df = self.filter_df(df)
else:
df = self.pym.load_df(req_startdate, req_enddate, table=self.table,
cols=cols, timestamp_col=timestamp_col)
else:
df = self.pym.load_df(req_startdate, req_enddate, table=self.table,
cols=cols, timestamp_col=timestamp_col)
df = self.filter_df(df)
#logger.warning('LINE 185: item: %s', df.head())
self.set_select_menus(df)
return df
except Exception:
logger.error('load_df', exc_info=True)
# -------------------- CARDS -----------------------------------------
def initialize_cards(self, width, height=250):
try:
txt = ''
for period in ['year', 'quarter', 'month', 'week']:
design = random.choice(list(KPI_card_css.keys()))
txt += self.card(title='', data='', card_design=design)
text = """<div style="margin-top:100px;display:flex; flex-direction:row;">
{}
</div>""".format(txt)
div = Div(text=text, width=width, height=height)
return div
except Exception:
logger.error('initialize cards', exc_info=True)
# -------------------- GRAPHS -------------------------------------------
def graph_periods_to_date(self, df1, timestamp_filter_col, variable):
try:
dct = {}
for idx, period in enumerate(['week', 'month', 'quarter', 'year']):
df = self.period_to_date(df1, timestamp=dashboard_config['dates']['last_date'],
timestamp_filter_col=timestamp_filter_col, period=period)
# get unique instances
df = df[[variable]]
df = df.drop_duplicates(keep='first')
#logger.warning('post duplicates dropped:%s', df.head(10))
data = 0
if self.groupby_dict[variable] == 'sum':
data = int(df[variable].sum())
elif self.groupby_dict[variable] == 'mean':
data = "{}%".format(round(df[variable].mean(),3))
else:
data = int(df[variable].count())
del df
gc.collect()
dct[period] = data
self.update_cards(dct)
except Exception:
logger.error('graph periods to date', exc_info=True)
def period_over_period(self, df, start_date, end_date, period,
history_periods=2, timestamp_col='timestamp_of_first_event'):
try:
# filter cols if necessary
string = '0 {}(s) prev(current)'.format(period)
# filter out the dates greater than today
df_current = df.copy()
df_current['period'] = string
# label the days being compared with the same label
df_current = self.label_dates_pop(df_current, period, timestamp_col)
#logger.warning('LINE 244:%s', df_current.head(15))
# zero out time information
start = datetime(start_date.year, start_date.month, start_date.day, 0, 0, 0)
end = datetime(end_date.year, end_date.month, end_date.day, 0, 0, 0)
cols = list(df.columns)
counter = 1
if isinstance(history_periods, str):
history_periods = int(history_periods)
# make dataframes for request no. of periods
start, end = self.shift_period_range(period, start, end)
while counter < history_periods and start >= self.initial_date:
# load data
if period == 'quarter':
logger.warning('start:end %s:%s', start, end)
df_temp = self.load_df_pym(start, end, cols, timestamp_col)
df_temp[timestamp_col] = pd.to_datetime(df_temp[timestamp_col])
if df_temp is not None:
if len(df_temp) > 1:
string = '{} {}(s) prev'.format(counter, period)
# label period
df_temp[period] = string
# relabel days to get matching day of week,doy, dom, for different periods
df_temp = self.label_dates_pop(df_temp, period, timestamp_col)
# logger.warning('df temp loaded for %s previous: %s',counter,len(df_temp))
df_current = pd.concat([df_current, df_temp])
del df_temp
gc.collect()
# shift the loading window
counter += 1
start, end = self.shift_period_range(period, start, end)
return df_current
except Exception:
logger.error('period over period', exc_info=True)
def graph_period_over_period(self,period):
try:
periods = [period]
start_date = self.pop_start_date
end_date = self.pop_end_date
if isinstance(start_date,date):
start_date = datetime.combine(start_date,datetime.min.time())
if isinstance(end_date,date):
end_date = datetime.combine(end_date,datetime.min.time())
#cols = [self.variable, self.timestamp_col, 'day']
cols = [self.variable, self.timestamp_col]
df = self.load_df_pym(start_date,end_date,cols=cols,
timestamp_col=self.timestamp_col)
for idx,period in enumerate(periods):
df_period = self.period_over_period(df, start_date=start_date, end_date=end_date,
period=period, history_periods=self.pop_history_periods,
timestamp_col=self.timestamp_col)
logger.warning('LINE 274 start:end=%s:%s,%s,%s len(Df),df.head',start_date,end_date,len(df),df.head())
groupby_cols = ['dayset','period']
if len(df_period) > 0:
df_period = df_period.groupby(groupby_cols).agg({self.variable:'count'})
df_period = df_period.reset_index()
else:
df_period = df_period.rename(index=str,columns={'day':'dayset'})
prestack_cols = list(df_period.columns)
logger.warning('Line 179:%s', df_period.head(10))
df_period = self.split_period_into_columns(df_period,col_to_split='period',
value_to_copy=self.variable)
logger.warning('line 180 df_period columns:%s',df_period.head(50))
poststack_cols = list(df_period.columns)
title = "{} over {}".format(period,period)
plotcols = list(np.setdiff1d(poststack_cols, prestack_cols))
df_period,plotcols = self.pop_include_zeros(df_period=df_period,plotcols=plotcols,period=period)
if 'dayset' not in df_period.columns:
leng = len(df_period)
if leng > 0:
df_period['dayset'] = 0
else:
df_period['dayset'] = ''
if idx == 0:
p = df_period.hvplot.bar('dayset',plotcols,rot=45,title=title,
stacked=False)
else:
p += df_period.hvplot.bar('dayset',plotcols,rot=45,title=title,
stacked=False)
return p
except Exception:
logger.error('period over period to date', exc_info=True)
def pop_week(self, launch=-1):
try:
return self.graph_period_over_period('week')
except Exception:
logger.error('pop week', exc_info=True)
def pop_month(self, launch=-1):
try:
return self.graph_period_over_period('month')
except Exception:
logger.error('pop month', exc_info=True)
def pop_quarter(self, launch=-1):
try:
return self.graph_period_over_period('quarter')
except Exception:
logger.error('pop quarter', exc_info=True)
def pop_year(self, launch=-1):
try:
return self.graph_period_over_period('year')
except Exception:
logger.error('pop year', exc_info=True)
def multiline_dow(self, launch=1):
try:
df = self.df.copy()
dct = {
'Y':'year',
'M':'month',
'W':'week',
'Q':'Qtr'
}
resample_period = dct[self.multiline_resample_period]
yvar = self.multiline_vars['y']
xvar = 'day_of_week'
df[resample_period] = df[self.timestamp_col].dt.to_period(self.multiline_resample_period)
df[xvar] = df[self.timestamp_col].dt.day_name()
df = df.groupby([xvar,resample_period]).agg({yvar: 'mean'})
df = df.reset_index()
p = df.hvplot.line(x=self.timestamp_col, y=yvar,groupby=resample_period, width=1200, height=500)
return p
except Exception:
logger.error('multiline plot', exc_info=True)
def update(attrname, old, new):
thistab.notification_updater("Calculations underway. Please be patient")
for item in ['area','category','gender','item']:
thistab.select_values[item] = thistab.select[item].value
thistab.graph_periods_to_date(thistab.df,thistab.timestamp_col)
thistab.section_header_updater('cards')
thistab.section_header_updater('pop')
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("ready")
def update_variable(attrname, old, new):
thistab.notification_updater("Calculations underway. Please be patient")
thistab.variable = variable_select.value
thistab.graph_periods_to_date(thistab.df,'block_timestamp',thistab.variable)
thistab.section_header_updater('cards',label='')
thistab.section_header_updater('pop',label='')
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("ready")
def update_period_over_period(attrname, old, new):
thistab.notification_updater("Calculations underway. Please be patient")
thistab.pop_history_periods = history_periods_select.value
thistab.pop_start_date=datepicker_period_start.value # trigger period over period
thistab.pop_end_date=datepicker_period_end.value # trigger period
thistab.trigger +=1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("ready")
def update_history_periods(attrname, old, new):
thistab.notification_updater("Calculations underway. Please be patient")
thistab.pop_history_periods = pop_number_select.value
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("ready")
def update_multiline(attrname, old, new):
thistab.notification_updater("Calculations in progress! Please wait.")
thistab.multiline_vars['y'] = multiline_y_select.value
thistab.multiline_resample_period = multiline_resample_period_select.value
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("Ready!")
try:
table = 'bcc_composite'
cols = cols_to_load['guest'] + cols_to_load['rental']
thistab = Thistab(table, cols)
# ------------------------------------- SETUP ----------------------------
# format dates
first_date_range = thistab.initial_date
last_date_range = datetime.now().date()
last_date = dashboard_config['dates']['last_date']
first_date = datetime(2014,1,1,0,0,0)
thistab.df = thistab.load_df_pym(first_date, last_date,cols,thistab.timestamp_col)
thistab.graph_periods_to_date(thistab.df,timestamp_filter_col=thistab.timestamp_col,
variable=thistab.variable)
thistab.section_header_updater('cards')
thistab.section_header_updater('pop')
'''
df_temp = thistab.df[(thistab.df['visit_start'].dt.year == 2019) & (thistab.df['visit_start'].dt.month == 8)]
logger.warning('LINE 416: df_temp:%s,%s',len(df_temp),df_temp.head(30))
'''
# MANAGE STREAM
# date comes out stream in milliseconds
# --------------------------------CREATE WIDGETS ---------------------------------
datepicker_start = DatePicker(title="Start", min_date=first_date_range,
max_date=last_date_range, value=first_date)
datepicker_end = DatePicker(title="End", min_date=first_date_range,
max_date=last_date_range, value=last_date)
thistab.pop_end_date = datetime.now().date()
daynum = thistab.pop_end_date.day
if daynum < 3:
thistab.pop_end_date = datetime.now().date() - timedelta(days=daynum)
thistab.pop_start_date = thistab.pop_end_date - timedelta(days=7)
else:
thistab.pop_start_date = thistab.first_date_in_period(thistab.pop_end_date, 'week')
stream_launch = streams.Stream.define('Launch',launch=-1)()
datepicker_period_start = DatePicker(title="Period start", min_date=first_date_range,
max_date=last_date_range, value=thistab.pop_start_date)
datepicker_period_end = DatePicker(title="Period end", min_date=first_date_range,
max_date=last_date_range, value=thistab.pop_end_date)
history_periods_select = Select(title='Select # of comparative periods',
value='2',
options=thistab.menus['history_periods'])
datepicker_pop_start = DatePicker(title="Period start", min_date=first_date_range,
max_date=last_date_range, value=thistab.pop_start_date)
datepicker_pop_end = DatePicker(title="Period end", min_date=first_date_range,
max_date=last_date_range, value=thistab.pop_end_date)
pop_number_select = Select(title='Select # of comparative periods',
value=str(5),
options=thistab.menus['history_periods'])
pop_button = Button(label="Select dates/periods, then click me!", width=15, button_type="success")
variable_select = Select(title='Select variable', value=thistab.variable,
options=thistab.menus['bcc']['rental'])
multiline_y_select = Select(title='Select comparative DV(y)',
value=thistab.multiline_vars['y'],
options=['price', 'amount', 'visit_duration'])
multiline_resample_period_select = Select(title='Select comparative DV(y)',
value=thistab.multiline_resample_period,
options=['W','M','Q','Y'])
# --------------------------------- GRAPHS ---------------------------
hv_pop_week = hv.DynamicMap(thistab.pop_week, streams=[stream_launch])
pop_week = renderer.get_plot(hv_pop_week)
hv_pop_month = hv.DynamicMap(thistab.pop_month, streams=[stream_launch])
pop_month = renderer.get_plot(hv_pop_month)
hv_pop_quarter = hv.DynamicMap(thistab.pop_quarter, streams=[stream_launch])
pop_quarter = renderer.get_plot(hv_pop_quarter)
hv_multiline_dow = hv.DynamicMap(thistab.multiline_dow, streams=[stream_launch])
multiline_dow = renderer.get_plot(hv_multiline_dow)
# -------------------------------- CALLBACKS ------------------------
#datepicker_start.on_change('value', update)
#datepicker_end.on_change('value', update)
for item in ['area','category','gender','item']:
thistab.select[item].on_change('value',update)
history_periods_select.on_change('value',update_period_over_period)
datepicker_period_start.on_change('value',update_period_over_period)
datepicker_period_end.on_change('value',update_period_over_period)
pop_number_select.on_change('value',update_history_periods)
variable_select.on_change('value', update_variable)
multiline_y_select.on_change('value', update_multiline)
multiline_resample_period_select.on_change('value', update_multiline)
# -----------------------------------LAYOUT ----------------------------
# put the controls in a single element
controls = WidgetBox(datepicker_start,datepicker_end,
thistab.select['area'],
thistab.select['category'],
thistab.select['item'],
thistab.select['gender'],
thistab.select['status'],
)
controls_pop = WidgetBox(datepicker_pop_start,
datepicker_pop_end,
history_periods_select,
pop_button)
controls_multiline = WidgetBox(multiline_y_select,
multiline_resample_period_select)
# create the dashboards
grid = gridplot([
[thistab.notification_div['top']],
[Spacer(width=20, height=70)],
[thistab.section_headers['cards']],
[Spacer(width=20, height=2)],
[thistab.KPI_card_div,controls],
[thistab.section_headers['pop']],
[Spacer(width=20, height=25)],
[pop_week.state,controls_pop],
[pop_month.state],
[pop_quarter.state],
[thistab.section_headers['dow']],
[Spacer(width=20, height=30)],
[multiline_dow.state, controls_multiline],
[thistab.notification_div['bottom']]
])
# Make a tab with the layout
tab = Panel(child=grid, title=panel_title)
return tab
except Exception:
logger.error('rendering err:', exc_info=True)
return tab_error_flag(panel_title)
class TrendingDashboard:
def __init__(self, file_path, day_first=False):
self.data = import_csv(file_path, day_first=day_first)
self.__create_sources()
self.__set_x()
self.__create_figures()
self.__set_properties()
self.__create_divs()
self.__add_plot_data()
self.__add_histogram_data()
self.__add_ichart_data()
self.__add_hover()
self.__add_legend()
self.__create_widgets()
self.__bind_widgets()
self.__do_layout()
self.update()
def __create_sources(self):
self.source = {grp: {'plot': ColumnDataSource(data={key: [] for key in MAIN_PLOT_KEYS}),
'trend': ColumnDataSource(data=dict(x=[], y=[])),
'bound': ColumnDataSource(data=dict(x=[], y=[])),
'patch': ColumnDataSource(data=dict(x=[], y=[])),
'hist': ColumnDataSource(data=dict(x=[], y=[]))} for grp in GROUPS}
self.ichart_source = {grp: {'plot': ColumnDataSource(data=dict(x=[], y=[], mrn=[], color=[], alpha=[], dates=[],
gamma_index=[], daily_corr=[], gamma_crit=[],
dta=[])),
'center_line': ColumnDataSource(data=dict(x=[], y=[], mrn=[])),
'ucl_line': ColumnDataSource(data=dict(x=[], y=[], mrn=[])),
'lcl_line': ColumnDataSource(data=dict(x=[], y=[], mrn=[])),
'bound': ColumnDataSource(data=dict(x=[], mrn=[], upper=[], avg=[], lower=[])),
'patch': ColumnDataSource(data=dict(x=[], y=[]))} for grp in GROUPS}
def __set_x(self):
self.x = self.data['date_time_obj']
def __create_figures(self):
self.fig = figure(plot_width=1000, plot_height=375, x_axis_type='datetime')
self.histogram = figure(tools="", plot_width=1000, plot_height=275)
self.ichart = figure(plot_width=1000, plot_height=375)
def __set_properties(self):
self.fig.xaxis.axis_label_text_font_size = "17pt"
self.fig.yaxis.axis_label_text_font_size = "17pt"
self.fig.xaxis.major_label_text_font_size = "15pt"
self.fig.yaxis.major_label_text_font_size = "15pt"
self.histogram.xaxis.axis_label_text_font_size = "17pt"
self.histogram.yaxis.axis_label_text_font_size = "17pt"
self.histogram.xaxis.major_label_text_font_size = "15pt"
self.histogram.yaxis.major_label_text_font_size = "15pt"
self.ichart.xaxis.axis_label = "Study #"
self.ichart.xaxis.axis_label_text_font_size = "17pt"
self.ichart.yaxis.axis_label_text_font_size = "17pt"
self.ichart.xaxis.major_label_text_font_size = "15pt"
self.ichart.yaxis.major_label_text_font_size = "15pt"
def __add_plot_data(self):
self.plot_data = {grp: self.fig.circle('x', 'y', source=self.source[grp]['plot'],
color=COLORS[grp], size=4, alpha=0.4) for grp in GROUPS}
self.plot_trend = {grp: self.fig.line('x', 'y', source=self.source[grp]['trend'],
line_color='black', line_width=4) for grp in GROUPS}
self.plot_avg = {grp: self.fig.line('x', 'avg', source=self.source[grp]['bound'],
line_color='black') for grp in GROUPS}
self.plot_patch = {grp: self.fig.patch('x', 'y', source=self.source[grp]['patch'],
color=COLORS[grp], alpha=0.2) for grp in GROUPS}
def __add_histogram_data(self):
self.vbar = {grp: self.histogram.vbar(x='x', width='width', bottom=0, top='top',
source=self.source[grp]['hist'], alpha=0.5, color=COLORS[grp])
for grp in GROUPS}
self.histogram.xaxis.axis_label = ""
self.histogram.yaxis.axis_label = "Frequency"
def __add_ichart_data(self):
self.ichart_data = {grp: self.ichart.circle('x', 'y', source=self.ichart_source[grp]['plot'],
size=4, color='color', alpha='alpha') for grp in GROUPS}
self.ichart_data_line = {grp: self.ichart.line('x', 'y', source=self.ichart_source[grp]['plot'],
color=COLORS[grp], line_dash='solid') for grp in GROUPS}
self.ichart_patch = {grp: self.ichart.patch('x', 'y', color=COLORS[grp],
source=self.ichart_source[grp]['patch'],
alpha=0.1) for grp in GROUPS}
self.ichart_center_line = {grp: self.ichart.line('x', 'y', source=self.ichart_source[grp]['center_line'],
alpha=1, color='black', line_dash='solid') for grp in GROUPS}
self.ichart_lcl_line = {grp: self.ichart.line('x', 'y', source=self.ichart_source[grp]['lcl_line'], alpha=1,
color='red', line_dash='dashed') for grp in GROUPS}
self.ichart_ucl_line = {grp: self.ichart.line('x', 'y', source=self.ichart_source[grp]['ucl_line'], alpha=1,
color='red', line_dash='dashed') for grp in GROUPS}
def __add_legend(self):
# Main TrendingDashboard
group_items = {grp: [("Data %s " % grp, [self.plot_data[grp]]),
("Avg %s " % grp, [self.plot_avg[grp]]),
("Rolling Avg %s " % grp, [self.plot_trend[grp]]),
("Percentile Region %s " % grp, [self.plot_patch[grp]])] for grp in GROUPS}
items = group_items[GROUPS[0]]
if len(GROUPS) > 1:
for grp in GROUPS[1:]:
items.extend(group_items[grp])
legend_plot = Legend(items=items, orientation='horizontal')
self.fig.add_layout(legend_plot, 'above')
self.fig.legend.click_policy = "hide"
# Control Chart
group_items = {grp: [("Value %s " % grp, [self.ichart_data[grp]]),
("Line %s" % grp, [self.ichart_data_line[grp]]),
('Center %s' % grp, [self.ichart_center_line[grp]]),
('UCL %s' % grp, [self.ichart_ucl_line[grp]]),
('LCL %s' % grp, [self.ichart_lcl_line[grp]]),
('In Ctrl %s' % grp, [self.ichart_patch[grp]])] for grp in GROUPS}
items = group_items[GROUPS[0]]
if len(GROUPS) > 1:
for grp in GROUPS[1:]:
items.extend(group_items[grp])
legend_ichart = Legend(items=items, orientation='horizontal')
self.ichart.add_layout(legend_ichart, 'above')
self.ichart.legend.click_policy = "hide"
def __add_hover(self):
self.fig.add_tools(HoverTool(tooltips=[("Plan Date", "@x{%F}"),
("Patient", "@id"),
("y", "@y"),
('Gamma Crit', "@gamma_crit"),
('Gamma Pass', '@gamma_index'),
('DTA', '@dta'),
('Daily Corr', '@daily_corr'),
('file', '@file_name')],
formatters={'x': 'datetime'},
renderers=[self.plot_data[grp] for grp in GROUPS]))
self.histogram.add_tools(HoverTool(show_arrow=True, line_policy='next', mode='vline',
tooltips=[("Bin Center", "@x"),
('Frequency', '@top')],
renderers=[self.vbar[grp] for grp in GROUPS]))
self.ichart.add_tools(HoverTool(show_arrow=True,
tooltips=[('ID', '@mrn'),
('Date', '@dates{%F}'),
('Study', '@x'),
('Value', '@y{0.2f}'),
("y", "@y"),
('Gamma Crit', "@gamma_crit"),
('Gamma Pass', '@gamma_index'),
('DTA', '@dta'),
('Daily Corr', '@daily_corr'),
('file', '@file_name')
],
formatters={'dates': 'datetime'},
renderers=[self.ichart_data[grp] for grp in GROUPS]))
def __create_divs(self):
self.div_summary = {grp: Div() for grp in GROUPS}
self.div_center_line = {grp: Div(text='', width=175) for grp in GROUPS}
self.div_ucl = {grp: Div(text='', width=175) for grp in GROUPS}
self.div_lcl = {grp: Div(text='', width=175) for grp in GROUPS}
def __create_widgets(self):
ignored_y = ['Patient Name', 'Patient ID', 'Plan Date', 'Radiation Dev', 'Energy', 'file_name', 'date_time_obj']
y_options = [option for option in list(self.data) if option not in ignored_y]
self.select_y = Select(title='Y-variable:', value='Dose Dev', options=y_options)
linacs = list(set(self.data['Radiation Dev']))
linacs.sort()
linacs.insert(0, 'All')
linacs.append('None')
self.select_linac = {grp: Select(title='Linac %s:' % grp, value='All', options=linacs, width=250)
for grp in GROUPS}
self.select_linac[2].value = 'None'
energies = list(set(self.data['Energy']))
energies.sort()
energies.insert(0, 'Any')
self.select_energies = {grp: Select(title='Energy %s:' % grp, value='Any', options=energies, width=250)
for grp in GROUPS}
self.avg_len_input = TextInput(title='Avg. Len:', value='10', width=100)
self.percentile_input = TextInput(title='Percentile:', value='90', width=100)
self.bins_input = TextInput(title='Bins:', value='20', width=100)
self.start_date_picker = DatePicker(title='Start Date:', value=self.x[0])
self.end_date_picker = DatePicker(title='End Date:', value=self.x[-1])
self.gamma_options = ['5.0%/3.0mm', '3.0%/3.0mm', '3.0%/2.0mm', 'Any']
self.checkbox_button_group = CheckboxButtonGroup(labels=self.gamma_options, active=[3])
def __bind_widgets(self):
self.select_y.on_change('value', self.update_source_ticker)
for grp in GROUPS:
self.select_linac[grp].on_change('value', self.update_source_ticker)
self.select_energies[grp].on_change('value', self.update_source_ticker)
self.avg_len_input.on_change('value', self.update_source_ticker)
self.percentile_input.on_change('value', self.update_source_ticker)
self.bins_input.on_change('value', self.update_source_ticker)
self.start_date_picker.on_change('value', self.update_source_ticker)
self.end_date_picker.on_change('value', self.update_source_ticker)
self.checkbox_button_group.on_change('active', self.update_source_ticker)
def __do_layout(self):
# TODO: Generalize for 1 or 2 groups
self.layout = column(row(self.select_y, self.select_linac[1], self.select_linac[2], self.avg_len_input,
self.percentile_input, self.bins_input),
row(self.select_energies[1], self.select_energies[2]),
row(self.start_date_picker, self.end_date_picker),
row(Div(text='Gamma Criteria: '), self.checkbox_button_group),
self.div_summary[1],
self.div_summary[2],
row(Spacer(width=10), self.fig),
Spacer(height=50),
row(Spacer(width=10), self.histogram),
Spacer(height=50),
row(Spacer(width=10), self.ichart),
row(self.div_center_line[1], self.div_ucl[1], self.div_lcl[1]),
row(self.div_center_line[2], self.div_ucl[2], self.div_lcl[2]))
def update_source_ticker(self, attr, old, new):
self.update()
def update(self):
for grp in GROUPS:
new_data = {key: [] for key in MAIN_PLOT_KEYS}
active_gamma = [self.gamma_options[a] for a in self.checkbox_button_group.active]
if self.select_linac[grp] != 'None':
for i in range(len(self.x)):
if self.select_linac[grp].value == 'All' or \
self.data['Radiation Dev'][i] == self.select_linac[grp].value:
if self.end_date_picker.value > self.x[i] > self.start_date_picker.value:
gamma_crit = "%s%%/%smm" % (self.data['Gamma Dose Criteria'][i],
self.data['Gamma Dist Criteria'][i])
if 'Any' in active_gamma or gamma_crit in active_gamma:
if 'Any' == self.select_energies[grp].value or \
self.data['Energy'][i] == self.select_energies[grp].value:
try:
new_data['y'].append(float(self.data[self.select_y.value][i]))
except ValueError:
continue
new_data['x'].append(self.x[i])
new_data['id'].append(self.data['Patient ID'][i])
new_data['gamma_crit'].append(gamma_crit)
new_data['file_name'].append(self.data['file_name'][i])
new_data['gamma_index'].append('%s%%' % self.data['Gamma-Index'][i])
new_data['daily_corr'].append(self.data['Daily Corr'][i])
new_data['dta'].append('%s%%' % self.data['DTA'][i])
try:
y = new_data['y']
self.div_summary[grp].text = "<b>Linac %s</b>: <b>Min</b>: %0.3f | <b>Low</b>: %0.3f | " \
"<b>Mean</b>: %0.3f | <b>Median</b>: %0.3f | <b>Upper</b>: %0.3f | " \
"<b>Max</b>: %0.3f" % \
(grp, np.min(y), np.percentile(y, 25), np.sum(y)/len(y),
np.percentile(y, 50), np.percentile(y, 75), np.max(y))
except:
self.div_summary[grp].text = "<b>Linac %s</b>" % grp
self.source[grp]['plot'].data = new_data
self.fig.yaxis.axis_label = self.select_y.value
self.fig.xaxis.axis_label = 'Plan Date'
self.update_histogram(grp)
self.update_trend(grp, int(float(self.avg_len_input.value)), float(self.percentile_input.value))
self.update_ichart()
def update_histogram(self, group):
width_fraction = 0.9
try:
bin_size = int(self.bins_input.value)
except ValueError:
bin_size = 20
self.bins_input.value = str(bin_size)
hist, bins = np.histogram(self.source[group]['plot'].data['y'], bins=bin_size)
width = [width_fraction * (bins[1] - bins[0])] * bin_size
center = (bins[:-1] + bins[1:]) / 2.
if set(hist) != {0}:
self.source[group]['hist'].data = {'x': center, 'top': hist, 'width': width}
else:
self.source[group]['hist'].data = {'x': [], 'top': [], 'width': []}
self.histogram.xaxis.axis_label = self.select_y.value
def update_trend(self, source_key, avg_len, percentile):
x = self.source[source_key]['plot'].data['x']
y = self.source[source_key]['plot'].data['y']
if x and y:
data_collapsed = collapse_into_single_dates(x, y)
x_trend, y_trend = moving_avg(data_collapsed, avg_len)
y_np = np.array(self.source[source_key]['plot'].data['y'])
upper_bound = float(np.percentile(y_np, 50. + percentile / 2.))
average = float(np.percentile(y_np, 50))
lower_bound = float(np.percentile(y_np, 50. - percentile / 2.))
self.source[source_key]['trend'].data = {'x': x_trend, 'y': y_trend, 'mrn': ['Avg'] * len(x_trend)}
self.source[source_key]['bound'].data = {'x': [x[0], x[-1]],
'mrn': ['Series Avg'] * 2,
'upper': [upper_bound] * 2,
'avg': [average] * 2,
'lower': [lower_bound] * 2,
'y': [average] * 2}
self.source[source_key]['patch'].data = {'x': [x[0], x[-1], x[-1], x[0]],
'y': [upper_bound, upper_bound, lower_bound, lower_bound]}
else:
self.source[source_key]['trend'].data = {'x': [], 'y': [], 'mrn': []}
self.source[source_key]['bound'].data = {'x': [], 'mrn': [], 'upper': [], 'avg': [], 'lower': [], 'y': []}
self.source[source_key]['patch'].data = {'x': [], 'y': []}
def update_ichart(self):
self.ichart.yaxis.axis_label = self.select_y.value
for grp in GROUPS:
y = self.source[grp]['plot'].data['y']
mrn = self.source[grp]['plot'].data['id']
dates = self.source[grp]['plot'].data['x']
gamma_crit = self.source[grp]['plot'].data['gamma_crit']
gamma_index = self.source[grp]['plot'].data['gamma_index']
daily_corr = self.source[grp]['plot'].data['daily_corr']
dta = self.source[grp]['plot'].data['dta']
file_name = self.source[grp]['plot'].data['file_name']
x = list(range(len(dates)))
center_line, ucl, lcl = get_control_limits(y)
if self.select_y.value in ['Gamma-Index', 'DTA'] and ucl > 100:
ucl = 100
colors = ['red', 'blue']
alphas = [0.3, 0.4]
color = [colors[ucl >= value >= lcl] for value in y]
alpha = [alphas[ucl >= value >= lcl] for value in y]
self.ichart_source[grp]['plot'].data = {'x': x, 'y': y, 'mrn': mrn, 'gamma_crit': gamma_crit,
'gamma_index': gamma_index, 'daily_corr': daily_corr, 'dta': dta,
'color': color, 'alpha': alpha, 'dates': dates,
'file_name': file_name}
if len(x) > 1:
self.ichart_source[grp]['patch'].data = {'x': [x[0], x[-1], x[-1], x[0]],
'y': [ucl, ucl, lcl, lcl]}
self.ichart_source[grp]['center_line'].data = {'x': [min(x), max(x)],
'y': [center_line] * 2,
'mrn': ['center line'] * 2}
self.ichart_source[grp]['lcl_line'].data = {'x': [min(x), max(x)],
'y': [lcl] * 2,
'mrn': ['center line'] * 2}
self.ichart_source[grp]['ucl_line'].data = {'x': [min(x), max(x)],
'y': [ucl] * 2,
'mrn': ['center line'] * 2}
self.div_center_line[grp].text = "<b>Center line</b>: %0.3f" % center_line
self.div_ucl[grp].text = "<b>UCL</b>: %0.3f" % ucl
self.div_lcl[grp].text = "<b>LCL</b>: %0.3f" % lcl
else:
self.ichart_source[grp]['patch'].data = {'x': [], 'y': []}
self.ichart_source[grp]['center_line'].data = {'x': [], 'y': [], 'mrn': []}
self.ichart_source[grp]['lcl_line'].data = {'x': [], 'y': [], 'mrn': []}
self.ichart_source[grp]['ucl_line'].data = {'x': [], 'y': [], 'mrn': []}
self.div_center_line[grp].text = "<b>Center line</b>:"
self.div_ucl[grp].text = "<b>UCL</b>:"
self.div_lcl[grp].text = "<b>LCL</b>:"
def date_range_update(attrname, old, new):
update()
# Set up widgets
yaxis_select = Select(value="Outside Temperature",
title='Y-axis Data',
options=columns_list)
start_date_pick = DatePicker(title="Start date",
min_date=start_date,
max_date=end_date,
value=start_date)
end_date_pick = DatePicker(title="End date",
min_date=start_date,
max_date=end_date,
value=end_date)
temp_unit_button = RadioButtonGroup(labels=["°F", "°C"], active=0)
# hook up events
temp_unit_button.on_change('active', temp_unit_update)
yaxis_select.on_change('value', yaxis_select_update)
start_date_pick.on_change('value', date_range_update)
end_date_pick.on_change('value', date_range_update)
inputs = column(yaxis_select, temp_unit_button, start_date_pick, end_date_pick)
plots = column(plot, select)
curdoc().add_root(row(inputs, plots, width=1200))
curdoc().title = "Observatory Data Plot"
def account_activity_tab(DAYS_TO_LOAD=90,panel_title=None):
class Thistab(Mytab):
def __init__(self, table,cols=[], dedup_cols=[]):
Mytab.__init__(self, table, cols, dedup_cols,panel_title=panel_title)
self.table = table
self.cols = cols
self.period = menus['period'][0]
self.update_type = menus['update_type'][0]
self.status = menus['status'][0]
self.account_type = menus['account_type'][0]
self.trigger = 0
self.df_warehouse = None
# correlation
self.variable = 'aion_fork'
self.strong_thresh = .65
self.mod_thresh = 0.4
self.weak_thresh = 0.25
self.corr_df = None
self.div_style = """ style='width:350px; margin-left:25px;
border:1px solid #ddd;border-radius:3px;background:#efefef50;'
"""
self.header_style = """ style='color:blue;text-align:center;' """
self.feature_list = hyp_variables.copy()
self.groupby_dict = groupby_dict
self.pym = PythonMongo('aion')
# ------- DIVS setup begin
self.page_width = 1200
txt = """<hr/><div style="text-align:center;width:{}px;height:{}px;
position:relative;background:black;margin-bottom:200px">
<h1 style="color:#fff;margin-bottom:300px">{}</h1>
</div>""".format(self.page_width, 50, 'Welcome')
self.notification_div = {
'top': Div(text=txt, width=self.page_width, height=20),
'bottom': Div(text=txt, width=self.page_width, height=10),
}
self.section_divider = '-----------------------------------'
self.section_headers = {
'account activity': self.section_header_div(text='Account activity:{}'.format(self.section_divider),
width=600, html_header='h2', margin_top=5, margin_bottom=-155),
'relationships': self.section_header_div(text='Relationships:{}'.format(self.section_divider),
width=600, html_header='h2', margin_top=5, margin_bottom=-155),
}
# ---------------------- DIVS ----------------------------
def section_header_div(self, text, html_header='h2', width=600, margin_top=150, margin_bottom=-150):
text = """<div style="margin-top:{}px;margin-bottom:-{}px;"><{} style="color:#4221cc;">{}</{}></div>""" \
.format(margin_top, margin_bottom, html_header, text, html_header)
return Div(text=text, width=width, height=15)
def clean_data(self, df):
df = df.fillna(0)
df[df == -inf] = 0
df[df == inf] = 0
return df
def load_df(self,start_date, end_date,cols,timestamp_col):
try:
# make timestamp into index
self.df_load(start_date, end_date,cols=cols,timestamp_col='block_timestamp')
#logger.warning('df loaded:%s',self.df.head())
except Exception:
logger.warning('load df',exc_info=True)
def prep_data(self):
try:
#self.df = dd.dataframe.from_pandas(self.df,npartitions=10)
# make timestamp into index
logger.warning('%s',self.df['block_timestamp'].head())
self.df1 = self.df.set_index('block_timestamp')
except Exception:
logger.warning('load df',exc_info=True)
def plot_account_activity(self,launch=-1):
try:
df = self.df1
if self.update_type != 'all':
df = df[df['update_type'] == self.update_type]
if self.account_type != 'all':
df = df[df['account_type'] == self.account_type]
logger.warning('df columns:%s',df.columns)
df = df[df.amount >= 0]
#logger.warning('line 100 df:%s',df.head(30))
df = df.resample(self.period).agg({'address':'count'})
df = df.reset_index()
df = df.compute()
df = df.rename(index=str,columns={'address':'period_activity'})
df['activity_delta(%)'] = df['period_activity'].pct_change(fill_method='ffill')
df['activity_delta(%)'] = df['activity_delta(%)'].multiply(100)
df = df.fillna(0)
logger.warning('df in balance after resample:%s',df.tail(10))
# make timestamp into index
return df.hvplot.line(x='block_timestamp', y=['period_activity'],
title='# of transactions')+\
df.hvplot.line(x='block_timestamp', y=['activity_delta(%)'],
title='% change in # of transactions')
# make timestamp into index
except Exception:
logger.warning('plot account activity',exc_info=True)
def plot_account_status(self, launch=-1):
try:
state = self.status
#logger.warning('df1 head:%s',self.df1.columns)
df = self.df1
if self.account_type != 'all':
df = self.df1[self.df1['account_type'] == self.account_type]
df = df[df['status'] == state]
df = df.resample(self.period).agg({'status': 'count'})
df = df.reset_index()
df = df.compute()
df['perc_change'] = df['status'].pct_change(fill_method='ffill')
df.perc_change = df.perc_change.multiply(100)
df = df.fillna(0)
# df = self.clean_data(df)
# make timestamp into index
value_label = '# '+state
gc.collect()
title1 = 'accounts {} by period'.format(state)
title2 = 'percentage {} change by period'.format(state)
return df.hvplot.line(x='block_timestamp', y=['status'], value_label=value_label,
title=title1) + \
df.hvplot.line(x='block_timestamp', y=['perc_change'], value_label='%',
title=title2)
except Exception:
logger.error('plot account status', exc_info=True)
def title_div(self, text, width=700):
text = '<h2 style="color:#4221cc;">{}</h2>'.format(text)
return Div(text=text, width=width, height=15)
def corr_information_div(self, width=400, height=300):
div_style = """
style='width:350px; margin-left:-500px;
border:1px solid #ddd;border-radius:3px;background:#efefef50;'
"""
txt = """
<div {}>
<h4 {}>How to interpret relationships </h4>
<ul style='margin-top:-10px;'>
<li>
Positive: as variable 1 increases, so does variable 2.
</li>
<li>
Negative: as variable 1 increases, variable 2 decreases.
</li>
<li>
Strength: decisions can be made on the basis of strong and moderate relationships.
</li>
<li>
No relationship/not significant: no statistical support for decision making.
</li>
<li>
The scatter graphs (below) are useful for visual confirmation.
</li>
<li>
The histogram (right) shows the distribution of the variable.
</li>
</ul>
</div>
""".format(div_style, self.header_style)
div = Div(text=txt, width=width, height=height)
return div
def hist(self,launch):
try:
return self.corr_df.hvplot.hist(
y=self.variable, bins=50, alpha=0.3,width=350,xaxis=False)
except Exception:
logger.warning('histogram', exc_info=True)
def correlation_table(self,launch):
try:
corr_dict = {
'Variable 1':[],
'Variable 2':[],
'Relationship':[],
'r':[],
'p-value':[]
}
df = self.corr_df
logger.warning(' df:%s',df.head(10))
a = df[self.variable].tolist()
for col in df.columns.tolist():
logger.warning('col :%s', col)
if col != self.variable:
logger.warning('%s:%s', col, self.variable)
b = df[col].tolist()
slope, intercept, rvalue, pvalue, std_err = linregress(a, b)
logger.warning('slope:%s,intercept:%s,rvalue:%s,pvalue:%s,std_err:%s',
slope, intercept, rvalue, pvalue, std_err)
if pvalue < 0.05:
if abs(rvalue) <= self.weak_thresh:
txt = 'none'
else:
strength = 'weak'
if rvalue > 0:
direction = 'positive'
if rvalue < 0:
direction = 'negative'
if abs(rvalue) > self.mod_thresh:
strength = 'moderate'
if abs(rvalue) > self.strong_thresh:
strength = 'strong'
txt = "{} {}".format(strength,direction)
else:
txt = 'Not significant'
corr_dict['Variable 1'].append(self.variable)
corr_dict['Variable 2'].append(col)
corr_dict['Relationship'].append(txt)
corr_dict['r'].append(round(rvalue,4))
corr_dict['p-value'].append(round(pvalue,4))
df = pd.DataFrame(
{
'Variable 1': corr_dict['Variable 1'],
'Variable 2': corr_dict['Variable 2'],
'Relationship': corr_dict['Relationship'],
'r':corr_dict['r'],
'p-value':corr_dict['p-value']
})
logger.warning('df:%s',df.head(23))
return df.hvplot.table(columns=['Variable 1', 'Variable 2','Relationship','r','p-value'],
width=700,height=400,title='Correlation between variables')
except Exception:
logger.warning('correlation table', exc_info=True)
def matrix_plot(self,launch=-1):
try:
#logger.warning('line 306 self.feature list:%s',self.feature_list)
if self.update_type != 'all':
df = self.df1[self.df1['update_type'] == self.update_type]
else:
df = self.df1
#df = df[self.feature_list]
# get difference for money columns
#logger.warning('line 282 df; %s', list(df.columns))
df = df.resample(self.period).mean()
#logger.warning('line 285 df; %s', self.groupby_dict)
df = df.reset_index()
#logger.warning('line 286 df; %s', df.head())
df = df.drop('block_timestamp',axis=1)
df = df.fillna(0)
df = df.compute()
df['aion_close'] = df['aion_close']
df['aion_market_cap'] = df['aion_market_cap']
df['bitcoin_close'] = df['bitcoin_close']
df['ethereum_close'] = df['ethereum_close']
df['bitcoin_market_cap'] = df['aion_market_cap']
df['ethereum_market_cap'] = df['aion_market_cap']
df = df.fillna(0)
#logger.warning('line 302. df: %s',df.head(10))
self.corr_df = df.copy()
cols_lst = df.columns.tolist()
cols_temp = cols_lst.copy()
if self.variable in cols_temp:
cols_temp.remove(self.variable)
variable_select.options = cols_lst
logger.warning('line 305 cols temp:%s',cols_temp)
logger.warning('line 306 self.variable:%s',self.variable)
logger.warning('line 307 df columns:%s',df.columns)
p = df.hvplot.scatter(x=self.variable,y=cols_temp,width=400,
subplots=True,shared_axes=False,xaxis=False).cols(3)
return p
except Exception:
logger.error('matrix plot', exc_info=True)
def update(attrname, old, new):
thistab.notification_updater("Calculations in progress! Please wait.")
thistab.load_df(datepicker_start.value,datepicker_end.value)
thistab.prep_data()
thistab.update_type = update_type_select.value
thistab.status = status_select.value
thistab.account_type = account_type_select.value
thistab.variable = variable_select.value
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
stream_launch_matrix.event(launch=thistab.trigger)
thistab.notification_updater("Ready.")
def update_resample(attr,old,new):
thistab.notification_updater("Calculations in progress! Please wait.")
thistab.prep_data()
thistab.period = new
thistab.update_type = update_type_select.value
thistab.status = status_select.value
thistab.account_type = account_type_select.value
thistab.variable = variable_select.value
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
stream_launch_matrix.event(launch=thistab.trigger)
stream_launch_corr.event(launch=thistab.trigger)
thistab.notification_updater("Ready!")
def update_account_type(attr, old, new):
thistab.notification_updater("Calculations in progress! Please wait.")
thistab.prep_data()
thistab.account_type = new
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
stream_launch_matrix.event(launch=thistab.trigger)
stream_launch_corr.event(launch=thistab.trigger)
thistab.notification_updater("Ready!")
def update_update_type(attr, old, new):
thistab.notification_updater("Calculations in progress! Please wait.")
thistab.prep_data()
thistab.update_type = new
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
stream_launch_matrix.event(launch=thistab.trigger)
stream_launch_corr.event(launch=thistab.trigger)
thistab.notification_updater("Ready!")
def update_variable(attr, old, new):
thistab.notification_updater("Calculations in progress! Please wait.")
thistab.prep_data()
thistab.variable = new
thistab.trigger += 1
stream_launch_matrix.event(launch=thistab.trigger)
stream_launch_corr.event(launch=thistab.trigger)
thistab.notification_updater("Ready!")
def update_status(attr, old, new):
thistab.notification_updater("Calculations in progress! Please wait.")
thistab.prep_data()
thistab.status = new
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
stream_launch_matrix.event(launch=thistab.trigger)
stream_launch_corr.event(launch=thistab.trigger)
thistab.notification_updater("Ready!")
try:
cols = list(set(hyp_variables + ['address','update_type','account_type','balance',
'status','block_timestamp','timestamp_of_first_event']))
thistab = Thistab(table='account_ext_warehouse',cols=cols)
# STATIC DATES
# format dates
first_date_range = "2018-04-25 00:00:00"
first_date_range = datetime.strptime(first_date_range, thistab.DATEFORMAT)
last_date_range = datetime.now().date()
last_date = dashboard_config['dates']['last_date']
first_date = datetime_to_date(last_date - timedelta(days=DAYS_TO_LOAD))
'''
thistab.df = thistab.pym.load_df(start_date=first_date, end_date=last_date,
cols=cols,table='account_ext_warehouse',timestamp_col='block_timestamp')
'''
thistab.load_df(start_date=first_date, end_date=last_date,cols=cols,
timestamp_col='block_timestamp')
thistab.prep_data()
# MANAGE STREAM
# date comes out stream in milliseconds
stream_launch = streams.Stream.define('Launch',launch=-1)()
stream_launch_matrix = streams.Stream.define('Launch_matrix',launch=-1)()
stream_launch_corr = streams.Stream.define('Launch_corr',launch=-1)()
# CREATE WIDGETS
datepicker_start = DatePicker(title="Start", min_date=first_date_range,
max_date=last_date_range, value=first_date)
datepicker_end = DatePicker(title="End", min_date=first_date_range,
max_date=last_date_range, value=last_date)
period_select = Select(title='Select aggregation period',
value=thistab.period,
options=menus['period'])
variable_select = Select(title='Select variable',
value='aion_fork',
options=sorted(hyp_variables))
status_select = Select(title='Select account status',
value=thistab.status,
options=menus['status'])
account_type_select = Select(title='Select account type',
value=thistab.account_type,
options=menus['account_type'])
update_type_select = Select(title='Select transfer type',
value=thistab.update_type,
options=menus['update_type'])
# --------------------- PLOTS----------------------------------
width = 800
hv_account_status = hv.DynamicMap(thistab.plot_account_status,
streams=[stream_launch]).opts(plot=dict(width=width, height=400))
hv_account_activity = hv.DynamicMap(thistab.plot_account_activity,
streams=[stream_launch]).opts(plot=dict(width=width, height=400))
hv_matrix_plot = hv.DynamicMap(thistab.matrix_plot,
streams=[stream_launch_matrix])
hv_corr_table = hv.DynamicMap(thistab.correlation_table,
streams=[stream_launch_corr])
hv_hist_plot = hv.DynamicMap(thistab.hist,streams=[stream_launch_corr])
account_status = renderer.get_plot(hv_account_status)
account_activity = renderer.get_plot(hv_account_activity)
matrix_plot = renderer.get_plot(hv_matrix_plot)
corr_table = renderer.get_plot(hv_corr_table)
hist_plot = renderer.get_plot(hv_hist_plot)
# handle callbacks
datepicker_start.on_change('value', update)
datepicker_end.on_change('value', update)
period_select.on_change('value',update_resample)
update_type_select.on_change('value',update_update_type)
account_type_select.on_change('value',update_account_type)
variable_select.on_change('value',update_variable)
status_select.on_change('value',update_status)
# COMPOSE LAYOUT
# put the controls in a single element
controls = WidgetBox(
datepicker_start,
datepicker_end,
period_select,
update_type_select,
account_type_select,
status_select,
variable_select)
# create the dashboards
grid = gridplot([
[thistab.notification_div['top']],
[Spacer(width=20, height=50)],
[thistab.section_headers['relationships']],
[Spacer(width=20, height=30)],
[matrix_plot.state,controls],
[corr_table.state, thistab.corr_information_div()],
[hist_plot.state],
[thistab.section_headers['account activity']],
[Spacer(width=20, height=30)],
[account_status.state],
[account_activity.state],
[thistab.notification_div['bottom']]
])
# Make a tab with the layout
tab = Panel(child=grid, title=thistab.panel_title)
return tab
except Exception:
logger.error('rendering err:', exc_info=True)
return tab_error_flag(thistab.panel_title)
def barchart_datepicker(df):
def make_dataset(df,
usageType='gastotalusage',
start=datetime.date(2015, 1, 1),
stop=datetime.date.today()):
if type(start) == str:
start = datetime.datetime.strptime(start, '%Y-%m-%d').date()
if type(stop) == str:
stop = datetime.datetime.strptime(stop, '%Y-%m-%d').date()
start_milliseconds = (
(start - datetime.date(1970, 1, 1)).total_seconds() * 1000)
stop_milliseconds = (
(stop - datetime.date(1970, 1, 1)).total_seconds() * 1000) + (
86400000 - 1
) # Plus 1 dag minus - milliseconde = einde van die dag
if usageType == 'powertotalusage': # select both tariff1totalusage and tariff2totalusage
df_verwerkt1 = df.loc[(df['attributeName'] == 'tariff1totalusage')
& (df['time'] >= start_milliseconds) &
(df['time'] <= stop_milliseconds)]
df_verwerkt1 = df_verwerkt1.sort_values('time')
df_verwerkt1 = df_verwerkt1.replace(0, np.NaN)
df_verwerkt1 = df_verwerkt1.fillna(method='ffill')
df_verwerkt1 = df_verwerkt1.fillna(
method='bfill') # nodig om eerste rij te fixen.
df_verwerkt1['time'] = pd.to_datetime(df_verwerkt1['time'],
unit='ms',
utc=True)
df_verwerkt1['time'] = df_verwerkt1['time'].apply(
lambda x: x.astimezone(timezone('Europe/Amsterdam')))
# df_verwerkt1['hour'] = df_verwerkt1['time'].dt.hour
df_verwerkt1['minutes'] = (df_verwerkt1['time'].dt.hour *
60) + df_verwerkt1['time'].dt.minute
# df_verwerkt1['dayofweek'] = df_verwerkt1['time'].dt.dayofweek
df_verwerkt1['difference'] = df_verwerkt1['value'].diff()
df_verwerkt1['difference'] = df_verwerkt1['difference'].fillna(0)
df_verwerkt1[df_verwerkt1['difference'] >
1000] = 0 # sprong naar Nederhoven uitfilteren
df_verwerkt1 = df_verwerkt1[df_verwerkt1['difference'] > 0]
# df_verwerkt1 = remove_outlier(df_verwerkt1, 'difference')
df_verwerkt2 = df.loc[(df['attributeName'] == 'tariff2totalusage')
& (df['time'] >= start_milliseconds) &
(df['time'] <= stop_milliseconds)]
df_verwerkt2 = df_verwerkt2.sort_values('time')
df_verwerkt2 = df_verwerkt2.replace(0, np.NaN)
df_verwerkt2 = df_verwerkt2.fillna(method='ffill')
df_verwerkt2 = df_verwerkt2.fillna(
method='bfill') # nodig om eerste rij te fixen.
df_verwerkt2['time'] = pd.to_datetime(df_verwerkt2['time'],
unit='ms',
utc=True)
df_verwerkt2['time'] = df_verwerkt2['time'].apply(
lambda x: x.astimezone(timezone('Europe/Amsterdam')))
# df_verwerkt2['hour'] = df_verwerkt2['time'].dt.hour
df_verwerkt2['minutes'] = (df_verwerkt2['time'].dt.hour *
60) + df_verwerkt2['time'].dt.minute
# df_verwerkt2['dayofweek'] = df_verwerkt2['time'].dt.dayofweek
df_verwerkt2['difference'] = df_verwerkt2['value'].diff()
df_verwerkt2['difference'] = df_verwerkt2['difference'].fillna(0)
df_verwerkt2[df_verwerkt2['difference'] >
1000] = 0 # sprong naar Nederhoven uitfilteren
df_verwerkt2 = df_verwerkt2[df_verwerkt2['difference'] > 0]
# df_verwerkt2 = remove_outlier(df_verwerkt2, 'difference')
df_verwerkt = pd.concat([df_verwerkt1, df_verwerkt2],
ignore_index=True)
groupby = df_verwerkt.groupby('minutes').sum()
else:
df_verwerkt = df.loc[(df['attributeName'] == usageType)
& (df['time'] >= start_milliseconds) &
(df['time'] <= stop_milliseconds)]
df_verwerkt = df_verwerkt.sort_values('time')
df_verwerkt = df_verwerkt.replace(0, np.NaN)
df_verwerkt = df_verwerkt.fillna(method='ffill')
df_verwerkt = df_verwerkt.fillna(
method='bfill') # nodig om eerste rij te fixen.
df_verwerkt['time'] = pd.to_datetime(df_verwerkt['time'],
unit='ms',
utc=True)
df_verwerkt['time'] = df_verwerkt['time'].apply(
lambda x: x.astimezone(timezone('Europe/Amsterdam')))
# df_verwerkt['hour'] = df_verwerkt['time'].dt.hour
df_verwerkt['minutes'] = (df_verwerkt['time'].dt.hour *
60) + df_verwerkt['time'].dt.minute
# df_verwerkt['dayofweek'] = df_verwerkt['time'].dt.dayofweek
df_verwerkt['difference'] = df_verwerkt['value'].diff()
df_verwerkt['difference'] = df_verwerkt['difference'].fillna(0)
df_verwerkt[df_verwerkt['difference'] >
1000] = 0 # sprong naar Nederhoven uitfilteren
df_verwerkt = df_verwerkt[df_verwerkt['difference'] > 0]
# df_verwerkt = remove_outlier(df_verwerkt, 'difference')
groupby = df_verwerkt.groupby('minutes').sum()
print('startdatum zoals verwerkt: ' + str(start))
print('einddatum zoals verwerkt: ' + str(stop))
return ColumnDataSource(data=groupby)
def make_plot(src):
# Deze regel maakt een dict met de vorm {0: '00:00', 1: '00:01', ... t/m 1440 voor alle minuten van de dag.
# In feite zet deze regel alle minuten van 0 t/m 1440 om in een string met tijd voor de x-as.
d = {
i: f"{floor(i/60):02d}" + ":" + f"{int(i%60):02d}"
for i in range(1440)
}
p = figure(title='Bar chart',
x_range=(0, 1440),
tools=TOOLS,
background_fill_color="#fafafa")
p.sizing_mode = 'stretch_both' # https://docs.bokeh.org/en/latest/docs/user_guide/layout.html
p.vbar(x='minutes',
bottom=0,
top='difference',
source=src,
width=0.9,
color={
'field': 'difference',
'transform': color_mapper
})
p.y_range.start = 0
p.xaxis.axis_label = 'Tijd'
p.xaxis.major_label_overrides = d
p.yaxis.axis_label = 'Verbruik'
p.grid.grid_line_color = "white"
return p
def update_radios(attr, old, new):
# Get the selected items for the graph
# ...
selected = radio_button_group.active
global usageType
# Koppel de selectie aan de juiste gegevens uit het DataFrame
if selected == 0:
usageType = 'gastotalusage'
p.title.text = 'Gasverbruik per minuut'
elif selected == 1:
usageType = 'tariff1totalusage'
p.title.text = 'Stroomtarief 1 verbruik'
elif selected == 2:
usageType = 'tariff2totalusage'
p.title.text = 'Stroomtarief 2 verbruik'
elif selected == 3:
usageType = 'powertotalusage'
p.title.text = 'Stroomverbruik totaal'
print('Update usageType: ' + str(usageType))
# update data
new_src = make_dataset(df, usageType=usageType, start=start, stop=stop)
src.data.update(new_src.data)
def update_start_date(attr, old, new):
print('New start date: ' + str(new))
global start
start = new
# update data
new_src = make_dataset(df, usageType=usageType, start=new, stop=stop)
src.data.update(new_src.data)
def update_end_date(attr, old, new):
print('New stop date: ' + str(new))
global stop
stop = new
# update data
new_src = make_dataset(df, usageType=usageType, start=start, stop=new)
src.data.update(new_src.data)
radio_button_group = RadioButtonGroup(
labels=["Gas", "Tarief 1", "Tarief 2", "Stroom totaal"], active=0)
radio_button_group.on_change('active', update_radios)
datepicker_start = DatePicker(title='Startdatum',
min_date=date(2015, 1, 1),
max_date=date.today())
datepicker_stop = DatePicker(title='Einddatum',
min_date=date(2015, 1, 1),
max_date=date.today())
datepicker_start.on_change('value', update_start_date)
datepicker_stop.on_change('value', update_end_date)
# initial execution
src = make_dataset(df, 'gastotalusage')
p = make_plot(src)
# make a grid
grid = gridplot([[p], [radio_button_group], [datepicker_start],
[datepicker_stop]])
grid.sizing_mode = 'scale_width'
tab = Panel(child=grid, title='Bar chart')
print("barchart() uitgevoerd")
return tab
value=instrum,
options=sorted(config.instruments()))
instrum_select.on_change('value', refresh_plot)
## start time delta
start_date = DatePicker(title='start',
width=140,
min_date=datetime.date(2020, 1, 1),
max_date=today,
value=today - datetime.timedelta(days=7))
start_hour = select = Select(title='hour', value="0", options=hours, width=70)
start_minute = select = Select(title='min',
value="0",
options=minutes,
width=70)
start_date.on_change('value', refresh_plot)
start_hour.on_change('value', refresh_plot)
start_minute.on_change('value', refresh_plot)
end_date = DatePicker(title='end',
width=140,
min_date=datetime.date(2020, 1, 1),
max_date=today,
value=today)
end_date.on_change('value', refresh_plot)
end_hour = select = Select(title='hour', value="23", options=hours, width=70)
end_minute = select = Select(title='min',
value="59",
options=minutes,
width=70)
end_hour.on_change('value', refresh_plot)
y_max = 1
for lst in y_list:
if max(lst) > y_max:
y_max = max(lst)
y_max += (y_max * 0.1)
plot.y_range.end = y_max
source.data = dict(date=x_list,
freq=y_list,
group=group_list,
color=color_list)
phrase.on_change('value', update_data)
datepicker1.on_change('value', update_data)
datepicker2.on_change('value', update_data)
checkbox_group.on_change('active', update_data)
def normalize(atrr, old, new):
d1 = datetime.datetime.combine(datepicker1.value, datetime.time())
d2 = datetime.datetime.combine(datepicker2.value, datetime.time())
wd1 = d1.weekday()
wd2 = d2.weekday()
d1 = d1 - datetime.timedelta(wd1)
d2 = d2 - datetime.timedelta(wd2)
t = list(norm.keys())
t.sort()
new = []
else:
new_data2['total'] = new_joined['sum'] * 0
src_cat.data = new_data2
plot2.x_range = new_factors
plot2.y_range.start = 0
plot2.y_range.end = max(rk_data.loc[d_to:d_from].groupby('Type').count()
[default_y_axis]) * 1.02
menu1 = Select(options=my_num_axes,
value=default_y_axis,
title='Select plot to draw (numerals)')
menu2 = DatePicker(title='From:', value=src_nm.data['x_ax'].min())
menu3 = DatePicker(title='To:', value=src_nm.data['x_ax'].max())
menu1.on_change('value', update_plot1)
menu2.on_change('value', update_plot1)
menu3.on_change('value', update_plot1)
dates = column(
widgetbox(menu2),
widgetbox(menu3),
plot2,
)
layout = row(column(widgetbox(menu1), dates), plot1)
# show(layout)
curdoc().add_root(layout)
def KPI_user_adoption_tab(DAYS_TO_LOAD=90):
class Thistab(KPI):
def __init__(self, table, cols=[]):
KPI.__init__(self, table, name='social_media', cols=cols)
self.table = table
self.df = None
self.checkboxgroup = {'account_type': [], 'update_type': []}
self.KPI_card_div = self.initialize_cards(self.page_width,
height=350)
# ------- DIVS setup begin
self.page_width = 1200
txt = """<hr/><div style="text-align:center;width:{}px;height:{}px;
position:relative;background:black;margin-bottom:200px">
<h1 style="color:#fff;margin-bottom:300px">{}</h1>
</div>""".format(self.page_width, 50, 'Welcome')
self.notification_div = {
'top': Div(text=txt, width=self.page_width, height=20),
'bottom': Div(text=txt, width=self.page_width, height=10),
}
self.section_divider = '-----------------------------------'
self.section_headers = {
'cards':
self.section_header_div(text='Period to date:{}'.format(
self.section_divider),
width=600,
html_header='h2',
margin_top=5,
margin_bottom=-155),
'pop':
self.section_header_div(text='Period over period:{}'.format(
self.section_divider),
width=600,
html_header='h2',
margin_top=5,
margin_bottom=-155),
}
# ---------------------- DIVS ----------------------------
def section_header_div(self,
text,
html_header='h2',
width=600,
margin_top=150,
margin_bottom=-150):
text = """<div style="margin-top:{}px;margin-bottom:-{}px;"><{} style="color:#4221cc;">{}</{}></div>""" \
.format(margin_top, margin_bottom, html_header, text, html_header)
return Div(text=text, width=width, height=15)
# ---------------------- DIVS ----------------------------
def reset_checkboxes(self, value='all', checkboxgroup=''):
try:
self.checkboxgroup[checkboxgroup].value = value
except Exception:
logger.error('reset checkboxes', exc_info=True)
def information_div(self, width=400, height=300):
div_style = """
style='width:350px;margin-right:-800px;
border:1px solid #ddd;border-radius:3px;background:#efefef50;'
"""
txt = """
<div {}>
<h4 {}>How to interpret relationships </h4>
<ul style='margin-top:-10px;'>
<li>
</li>
<li>
</li>
<li>
</li>
<li>
</li>
<li>
</li>
<li>
</li>
</ul>
</div>
""".format(div_style, self.header_style)
div = Div(text=txt, width=width, height=height)
return div
# -------------------- CARDS -----------------------------------------
def initialize_cards(self, width, height=250):
try:
txt = ''
for period in ['year', 'quarter', 'month', 'week']:
design = random.choice(list(KPI_card_css.keys()))
txt += self.card(title='', data='', card_design=design)
text = """<div style="margin-top:100px;display:flex; flex-direction:row;">
{}
</div>""".format(txt)
div = Div(text=text, width=width, height=height)
return div
except Exception:
logger.error('initialize cards', exc_info=True)
# -------------------- GRAPHS -------------------------------------------
def graph_periods_to_date(self, df1, filter_col):
try:
if self.account_type != 'all':
df1 = df1[df1.account_type == self.account_type]
dct = {}
for idx, period in enumerate(
['week', 'month', 'quarter', 'year']):
df = self.period_to_date(
df1,
timestamp=dashboard_config['dates']['last_date'],
timestamp_filter_col=filter_col,
period=period)
# get unique instances
df = df[['address']]
df = df.compute()
df = df.drop_duplicates(keep='first')
#logger.warning('post duplicates dropped:%s', df.head(10))
data = len(df)
del df
gc.collect()
dct[period] = data
self.update_cards(dct)
except Exception:
logger.error('graph periods to date', exc_info=True)
def graph_period_over_period(self, period):
try:
periods = [period]
start_date = self.pop_start_date
end_date = self.pop_end_date
if isinstance(start_date, date):
start_date = datetime.combine(start_date,
datetime.min.time())
if isinstance(end_date, date):
end_date = datetime.combine(end_date, datetime.min.time())
cols = ['account_type', 'timestamp_of_first_event', 'day']
df = self.load_df(start_date=start_date,
end_date=end_date,
cols=cols,
timestamp_col='timestamp_of_first_event')
if abs(start_date - end_date).days > 7:
if 'week' in periods:
periods.remove('week')
if abs(start_date - end_date).days > 31:
if 'month' in periods:
periods.remove('month')
if abs(start_date - end_date).days > 90:
if 'quarter' in periods:
periods.remove('quarter')
if self.account_type != 'all':
df = df[df.account_type == self.account_type]
# col for when list is empty
self.variable = 'account_type'
for idx, period in enumerate(periods):
df_period = self.period_over_period(
df,
start_date=start_date,
end_date=end_date,
period=period,
history_periods=self.pop_history_periods,
timestamp_col='timestamp_of_first_event')
groupby_cols = ['dayset', 'period']
if len(df_period) > 0:
df_period = df_period.groupby(groupby_cols).agg(
{'account_type': 'count'})
df_period = df_period.reset_index()
df_period = df_period.compute()
else:
df_period = df_period.compute()
df_period = df_period.rename(index=str,
columns={'day': 'dayset'})
prestack_cols = list(df_period.columns)
logger.warning('Line 179:%s', df_period.head(10))
df_period = self.split_period_into_columns(
df_period,
col_to_split='period',
value_to_copy='account_type')
logger.warning('line 180 df_period columns:%s',
df_period.head(50))
poststack_cols = list(df_period.columns)
title = "{} over {}".format(period, period)
plotcols = list(np.setdiff1d(poststack_cols,
prestack_cols))
df_period, plotcols = self.pop_include_zeros(
df_period=df_period, plotcols=plotcols, period=period)
if idx == 0:
p = df_period.hvplot.bar('dayset',
plotcols,
rot=45,
title=title,
stacked=False)
else:
p += df_period.hvplot.bar('dayset',
plotcols,
rot=45,
title=title,
stacked=False)
return p
except Exception:
logger.error('period over period to date', exc_info=True)
def update_account(attrname, old, new):
thistab.notification_updater(
"Calculations underway. Please be patient")
thistab.account_type = new
thistab.graph_periods_to_date(thistab.df, 'timestamp_of_first_event')
thistab.section_header_updater('cards')
thistab.section_header_updater('pop')
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("ready")
def update_period_over_period(attrname, old, new):
thistab.notification_updater(
"Calculations underway. Please be patient")
thistab.pop_history_periods = history_periods_select.value
thistab.pop_start_date = datepicker_period_start.value # trigger period over period
thistab.pop_end_date = datepicker_period_end.value # trigger period
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("ready")
def update_history_periods(attrname, old, new):
thistab.notification_updater(
"Calculations underway. Please be patient")
thistab.pop_history_periods = pop_number_select.value
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("ready")
try:
cols = [
'address', 'account_type', 'update_type', 'balance',
'timestamp_of_first_event'
]
thistab = Thistab(table='account_ext_warehouse', cols=cols)
# ------------------------------------- SETUP ----------------------------
# format dates
first_date_range = thistab.initial_date
last_date_range = datetime.now().date()
last_date = dashboard_config['dates']['last_date']
first_date = datetime(last_date.year, 1, 1, 0, 0, 0)
thistab.df = thistab.load_df(first_date, last_date, cols,
'timestamp_of_first_event')
thistab.graph_periods_to_date(thistab.df,
filter_col='timestamp_of_first_event')
thistab.section_header_updater('cards')
thistab.section_header_updater('pop')
# MANAGE STREAM
# date comes out stream in milliseconds
# --------------------------------CREATE WIDGETS ---------------------------------
datepicker_start = DatePicker(title="Start",
min_date=first_date_range,
max_date=last_date_range,
value=first_date)
datepicker_end = DatePicker(title="End",
min_date=first_date_range,
max_date=last_date_range,
value=last_date)
thistab.pop_end_date = last_date
thistab.pop_start_date = thistab.first_date_in_period(
thistab.pop_end_date, 'week')
stream_launch = streams.Stream.define('Launch', launch=-1)()
datepicker_period_start = DatePicker(title="Period start",
min_date=first_date_range,
max_date=last_date_range,
value=thistab.pop_start_date)
datepicker_period_end = DatePicker(title="Period end",
min_date=first_date_range,
max_date=last_date_range,
value=thistab.pop_end_date)
history_periods_select = Select(
title='Select # of comparative periods',
value='2',
options=thistab.menus['history_periods'])
account_type_select = Select(title='Select account type',
value='all',
options=thistab.menus['account_type'])
pop_number_select = Select(title='Select # of comparative periods',
value=str(thistab.pop_history_periods),
options=thistab.menus['history_periods'])
# --------------------------------- GRAPHS ---------------------------
hv_pop_week = hv.DynamicMap(thistab.pop_week, streams=[stream_launch])
pop_week = renderer.get_plot(hv_pop_week)
hv_pop_month = hv.DynamicMap(thistab.pop_month,
streams=[stream_launch])
pop_month = renderer.get_plot(hv_pop_month)
hv_pop_quarter = hv.DynamicMap(thistab.pop_quarter,
streams=[stream_launch])
pop_quarter = renderer.get_plot(hv_pop_quarter)
# -------------------------------- CALLBACKS ------------------------
#datepicker_start.on_change('value', update)
#datepicker_end.on_change('value', update)
account_type_select.on_change('value', update_account)
history_periods_select.on_change('value', update_period_over_period)
datepicker_period_start.on_change('value', update_period_over_period)
datepicker_period_end.on_change('value', update_period_over_period)
pop_number_select.on_change('value', update_history_periods)
# -----------------------------------LAYOUT ----------------------------
# put the controls in a single element
controls = WidgetBox(datepicker_start, datepicker_end,
account_type_select)
controls_pop = WidgetBox(datepicker_period_start,
datepicker_period_end, history_periods_select)
# create the dashboards
grid = gridplot([[thistab.notification_div['top']],
[Spacer(width=20, height=70)],
[thistab.section_headers['cards']],
[Spacer(width=20, height=2)],
[thistab.KPI_card_div, controls],
[thistab.section_headers['pop']],
[Spacer(width=20, height=25)],
[pop_week.state, controls_pop], [pop_month.state],
[pop_quarter.state],
[thistab.notification_div['bottom']]])
# Make a tab with the layout
tab = Panel(child=grid, title='KPI: user adoption')
return tab
except Exception:
logger.error('rendering err:', exc_info=True)
return tab_error_flag('KPI accounts')
search_start_date_picker = DatePicker(
title="start datum",
value=data.search_start_datetime.strftime("%Y-%m-%d"),
min_date=data.first_value_datetime.strftime("%Y-%m-%d"),
max_date=data.search_start_datetime.strftime("%Y-%m-%d"),
)
search_end_date_picker = DatePicker(
title="eind datum",
value=data.now.strftime("%Y-%m-%d"),
min_date=data.search_start_datetime.strftime("%Y-%m-%d"),
max_date=data.now.strftime("%Y-%m-%d"),
)
search_start_date_picker.on_change("value", update_search_range)
search_end_date_picker.on_change("value", update_search_range)
search_button = Button(label="update zoekgrafiek", button_type="success")
search_button.on_click(update_search_fig)
select_search_timeseries = Select(title="Zoektijdserie:",
value=None,
options=[])
select_search_timeseries.on_change("value", update_on_search_select)
# %% define empty time_figs and fig-handlers
centre_datetime = (
data.timeseries.start_datetime +
(data.timeseries.end_datetime - data.timeseries.start_datetime) / 2)
# IndexSelecter
index_codes = list(ct.INDEX_DICT.keys())
index_codes.append('880883')
index_type_list = ['K线图', '牛熊股比']
index_select = Select(value='000001', title='指数代码', options=sorted(index_codes), height=50)
index_type_select = Select(value='牛熊股比', title='数据类型', options=sorted(index_type_list), height=50)
# DatePciekr
dt_pckr_start = DatePicker(title='开始日期', value = date.today() - timedelta(days = 200), min_date = date(2004,1,1), max_date = date.today())
dt_pckr_end = DatePicker(title='结束日期', value = date.today(), min_date = date(2004,1,1), max_date = date.today())
index_select_row = row(index_select, index_type_select, dt_pckr_start, dt_pckr_end)
index_layout = column(index_select_row, create_index_figure_column(index_select.value, index_type_select.value, dt_pckr_start.value, dt_pckr_end.value))
index_select.on_change('value', update_index)
index_type_select.on_change('value', update_index)
dt_pckr_start.on_change('value', update_index)
dt_pckr_end.on_change('value', update_index)
# Add Stock Analysis
stock_auto_input = AutocompleteInput(value = '601318', completions = initializer.update_code_list(), title = '股票代码')
# DatePciekr
stock_pckr_start = DatePicker(title='开始日期', value = date.today() - timedelta(days = 100), min_date = date(2000,1,1), max_date = date.today())
stock_pckr_end = DatePicker(title='股票日期', value = date.today(), min_date = date(2000,1,1), max_date = date.today())
stock_select_row = row(stock_auto_input, stock_pckr_start, stock_pckr_end)
stock_layout = column(stock_select_row, create_stock_figure_column(stock_auto_input.value, stock_pckr_start.value, stock_pckr_end.value))
stock_auto_input.on_change('value', update_stock)
stock_pckr_start.on_change('value', update_stock)
stock_pckr_end.on_change('value', update_stock)
groups = pd.Categorical(df[color.value])
c = [COLORS[xx] for xx in groups.codes]
p.circle(x=xs, y=ys, color=c, size=sz, line_color="white", alpha=0.6, hover_color='white', hover_alpha=0.5)
return p
def update(attr, old, new):
layout.children[1].children[0] = GetPlot2(new)
layout.children[1].children[1] = GetPlot1()
def update_date_start(attr, old, new):
layout.children[3] = Div(text = 'Количество брошенных корзин за выбранный период: ' + str(Report3(date_start.value,date_finish.value)))
def update_date_finish(attr, old, new):
layout.children[3] = Div(text = 'Количество брошенных корзин за выбранный период: ' + str(Report3(date_start.value,date_finish.value)))
current_category = Select(title='Выберите категорию продукта:', value='fresh_fish', options= ["fresh_fish", "canned_food", "semi_manufactures", "caviar","frozen_fish"])
current_category.on_change('value', update)
date_start = DatePicker(title = 'Начало периода', value = '2018-08-01')
date_finish = DatePicker(title = 'Конец периода',value = '2018-08-14')
date_start.on_change('value', update_date_start)
date_finish.on_change('value', update_date_finish)
count_carts = Div(text = 'Количество брошенных корзин за выбранный период: ' + str(Report3(date_start.value,date_finish.value)))
#controls = widgetbox([current_category], width=300)
layout = layout([current_category, [GetPlot2("fresh_fish"), GetPlot1()],[date_start,date_finish],count_carts])
curdoc().add_root(layout)
curdoc().title = "Dashboard"
