def __init__(self):
self.per_capita = RadioGroup(
labels=["Total", "Per Capita", "Logarithmic"],
active=0,
sizing_mode="stretch_width",
)
self.data_getter = RadioGroup(
labels=["Cases", "Deaths", "Positivity"],
active=0,
sizing_mode="stretch_width",
)
self.date = DatePicker(title="Date", sizing_mode="stretch_width")
self.save_files = CheckboxGroup(labels=["Save files"],
sizing_mode="stretch_width")
self.button = Button(label="► Play", sizing_mode="stretch_width")
self.tooltips = [("Name", "@name"), ("Value", "@value")]
self.src = None
self.p = None
self.callback = None
self.counter = None
self.tempdir = None
self.filenames = None
class DataGrab:
"""
provides a list of available experimental runs.
Sensitive to the specific folder structure of the e11 lab
"""
def __init__(self,ROOTFOLDER):
self.ROOTFOLDER = ROOTFOLDER
self.date_picker =DatePicker(title='Select date of experiment',min_date=date(2017,1,1),max_date=date.today())
self.date_picker.on_change('value',self._date_callback)
self.file_data = dict(
runid=[],
address=[],
filename=[]
)
self.file_source = ColumnDataSource(self.file_data)
self.file_columns = [
TableColumn(field="runid", title="Run ID"),
TableColumn(field="filename", title="file name")
]
self.data_table = DataTable(source=self.file_source, columns=self.file_columns,width =300,height=500)
def _date_callback(self,attr,old,new):
"""
on change of data, update the datatable showing all runs which can be plotted
"""
date_string = date_to_file(self.date_picker.value)
self.data_table.source.data = get_files(self.ROOTFOLDER,date_string[0],date_string[1],date_string[2])
def show(self):
return column(self.date_picker,self.data_table)
def __create_widgets(self):
ignored_y = ['Patient Name', 'Patient ID', 'Plan Date', 'Radiation Dev', 'Energy', 'file_name', 'date_time_obj']
y_options = [option for option in list(self.data) if option not in ignored_y]
self.select_y = Select(title='Y-variable:', value='Dose Dev', options=y_options)
linacs = list(set(self.data['Radiation Dev']))
linacs.sort()
linacs.insert(0, 'All')
linacs.append('None')
self.select_linac = {grp: Select(title='Linac %s:' % grp, value='All', options=linacs, width=250)
for grp in GROUPS}
self.select_linac[2].value = 'None'
energies = list(set(self.data['Energy']))
energies.sort()
energies.insert(0, 'Any')
self.select_energies = {grp: Select(title='Energy %s:' % grp, value='Any', options=energies, width=250)
for grp in GROUPS}
self.avg_len_input = TextInput(title='Avg. Len:', value='10', width=100)
self.percentile_input = TextInput(title='Percentile:', value='90', width=100)
self.bins_input = TextInput(title='Bins:', value='20', width=100)
self.start_date_picker = DatePicker(title='Start Date:', value=self.x[0])
self.end_date_picker = DatePicker(title='End Date:', value=self.x[-1])
self.gamma_options = ['5.0%/3.0mm', '3.0%/3.0mm', '3.0%/2.0mm', 'Any']
self.checkbox_button_group = CheckboxButtonGroup(labels=self.gamma_options, active=[3])
def __init__(self):
dateNow = datetime.now()
self.startDay = DatePicker(title="Start Date",
min_date=datetime(2000, 1, 1),
max_date=datetime(2030, 12, 31),
value=datetime(dateNow.year, dateNow.month,
dateNow.day))
self.endDay = DatePicker(title="End Date",
min_date=datetime(2000, 1, 1),
max_date=datetime(2030, 12, 31),
value=datetime(dateNow.year, dateNow.month,
dateNow.day))
# snapshot time hours minutes
self.timeStep = Slider(title="Time step [min]",
value=15.0,
start=1.0,
end=60.0,
step=1.0)
self.PVsystem = Select(title="Select PV system", value="", options=[])
self.runSimulation = Button(label="Run simulation",
button_type="success")
r1 = row(self.startDay, self.endDay, self.timeStep, self.PVsystem,
self.runSimulation)
self.final_layout = r1
return
def __init__(self, dataset=STATES):
self.dataset = dataset
self.state_selection = MultiSelect(
title="States:",
options=self.dataset,
value=["New York", "Texas"],
sizing_mode="stretch_both",
)
self.per_capita = RadioGroup(
labels=["Total", "Per Capita"],
active=0,
sizing_mode="stretch_width",
)
self.data_getter = RadioGroup(
labels=[
"Cases",
"Deaths",
"Positivity",
"Testing",
"Constant Positivity",
"Constant Testing",
],
active=0,
sizing_mode="stretch_width",
)
self.plot_type = RadioGroup(
labels=["Linear", "Logarithmic"],
active=0,
sizing_mode="stretch_width",
)
self.constant_date = DatePicker(
title="Constant Date",
value=(datetime.today() - timedelta(days=1)).date(),
sizing_mode="stretch_width",
)
self.show_total = CheckboxGroup(
labels=["Show total"],
sizing_mode="stretch_width",
)
self.total_only = CheckboxGroup(
labels=["Total only"],
sizing_mode="stretch_width",
)
self.src = None
self.p = None
self.logp = None
self.tooltips = [("State", "@state")]
def modify_doc(doc):
crnt_date = dt.now()
dt_pckr_strt = DatePicker(title='Select start of sync date',
min_date=date(2017, 1, 1),
max_date=date.today())
def callback(attr, old, new):
print(type(old))
print('old was {} and new is {}'.format(old, new))
dt_pckr_strt.on_change('value', callback)
doc.add_root(bokehCol(dt_pckr_strt))
def analytics_tab():
panel_title = Div(text="Highway Performance",
css_classes=["panel-title", "text-center"])
panel_text = Div(
text=
"""Lorem Ipsum is simply dummy text of the printing and typesetting industry.
Lorem Ipsum has been the industry's standard dummy text ever since the 1500s,
when an unknown printer took a galley of type and scrambled it to make a type
specimen book.""",
css_classes=["panel-content"])
#Date Range
date_range = Div(text="Data References:<br>MMDDYYYY - MMDDYYYY",
css_classes=["panel-content", "text-center"])
#Panel Buttons
period_select = Select(options=['test 1', 'test 2', 'test 3'],
title='Time Period:',
height=60)
date_picker_start = DatePicker(min_date=date(2015, 1, 1),
max_date=date(2017, 12, 31),
css_classes=["panel-content", "col-lg-4"],
title="Start Date:",
height=60)
month = Select(options=[
"January", "February", "March", "April", "May", "June", "July",
"August", "September", "October", "November", "December"
],
title="Month:",
css_classes=["panel-content", "col-lg-4"],
height=60)
day_of_week = Select(
options=["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"],
title="Day of Week:",
height=60)
time_of_day = RangeSlider(start=1,
end=24,
step=1,
value=(1, 7),
title="Time of Day:",
bar_color="black",
height=60)
l1 = Div(text="test", css_classes=["text-center"])
return row(column(panel_title,
panel_text,
date_range,
period_select,
month,
day_of_week,
time_of_day,
height=700,
css_classes=["panel", "col-lg-4"]),
column(l1, css_classes=["container-fluid", "col-lg-8"]),
css_classes=["container-fluid", "col-lg-12"])
def set_up_widgets(p, source, source1, source2, source3, df, text_dsn, table):
dsn = text_dsn.value
# Set up widgets
text_title = TextInput(title="Title:", value="{} Data".format(dsn))
text_save = TextInput(title="Save As:", value=dsn)
max_for_dsn = df.row_timestamp.max()
min_day, max_day = day_limits(table)
calendar = DatePicker(title="Day:",
value=date(max_for_dsn.year, max_for_dsn.month,
max_for_dsn.day + 1),
max_date=date(max_day.year, max_day.month,
max_day.day + 1),
min_date=date(min_day.year, min_day.month,
min_day.day + 1))
button = Button(label="Update", button_type="success")
# Set up callbacks
def update_title(attrname, old, new):
p.title.text = text_title.value
def update_save(attrname, old, new):
p.tools[0].save_name = text_save.value
def update():
text_dsn.value = text_dsn.value.strip(" ") # Get rid of extra space
# Make sure time is valid
date_start = "{} 00:00:00".format(calendar.value)
date_end = "{} 23:59:59".format(calendar.value)
df_new = update_data(p,
text_title,
text_save,
source,
source1,
source2,
source3,
text_dsn.value,
table,
dates=[date_start, date_end])
# if df_new is going to be used, make sure it's not empty:
# if df_new is empty...
# day = df_new.row_timestamp.max()
# min_day = df_new.row_timestamp.min()
# calendar.value = date(day.year, day.month, day.day+1)
# calendar.max_date = date(day.year, day.month, day.day+1)
# calendar.min_date = date(min_day.year, min_day.month,min_day.day+1)
text_title.on_change('value', update_title)
text_save.on_change('value', update_save)
button.on_click(update)
button.js_on_click(CustomJS(args=dict(p=p), code="""p.reset.emit()"""))
# Set up layouts and add to document
inputs = widgetbox(text_title, text_save, calendar, button)
curdoc().add_root(row(inputs, p, width=1300))
def __init__(self):
dateNow = datetime.now()
Results = [
'Ptotal', 'Qtotal', 'LMP', 'Vmax', 'Vmin', 'PlTotal', 'QlTotal',
'Pelm', 'Qelm', 'VelmMax', 'VelmMin', 'SOC'
]
self.startDay = DatePicker(title="Start Date",
min_date=datetime(2000, 1, 1),
max_date=datetime(2030, 12, 31),
value=datetime(dateNow.year, 1, 2).date())
self.endDay = DatePicker(title="End Date",
min_date=datetime(2000, 1, 1),
max_date=datetime(2030, 12, 31),
value=datetime(dateNow.year + 1, 1, 1).date())
self.timeStep = Slider(title="Time step [min]",
value=60.0,
start=1.0,
end=60.0,
step=1.0)
self.Vul = Slider(title="Voltage upper limit [p.u]",
value=1.05,
start=0.9,
end=1.1,
step=0.005)
self.Vll = Slider(title="Voltage lower limit [p.u]",
value=0.95,
start=0.9,
end=1.1,
step=0.005)
self.Class = Select(title="Select class",
value="",
options=['', 'PVsystem', 'Storage'])
self.Element = Select(title="Select element", value="", options=[])
self.PptyCombo = Select(title="Plot type",
value='Ptotal',
options=Results)
self.runSimulation = Button(label="Run simulation",
button_type="success")
r1 = row(self.startDay, self.endDay, self.timeStep, self.runSimulation)
r2 = row(self.Class, self.Element, self.PptyCombo, self.Vll, self.Vul)
C = column(r1, r2)
self.final_layout = C
return
def __init__(self,ROOTFOLDER):
self.ROOTFOLDER = ROOTFOLDER
self.date_picker =DatePicker(title='Select date of experiment',min_date=date(2017,1,1),max_date=date.today())
self.date_picker.on_change('value',self._date_callback)
self.file_data = dict(
runid=[],
address=[],
filename=[]
)
self.file_source = ColumnDataSource(self.file_data)
self.file_columns = [
TableColumn(field="runid", title="Run ID"),
TableColumn(field="filename", title="file name")
]
self.data_table = DataTable(source=self.file_source, columns=self.file_columns,width =300,height=500)
def __init__(self, table,cols=[]):
KPI.__init__(self, table,name='developer',cols=cols)
self.table = table
self.df = None
self.checkboxgroup = {}
self.period_to_date_cards = {
'year': self.card('',''),
'quarter': self.card('', ''),
'month': self.card('', ''),
'week': self.card('', '')
}
self.ptd_startdate = datetime(datetime.today().year,1,1,0,0,0)
self.timestamp_col = 'block_timestamp'
self.variable = self.menus['developer_adoption_DVs'][0]
self.datepicker_pop_start = DatePicker(
title="Period start", min_date=self.initial_date,
max_date=dashboard_config['dates']['last_date'], value=dashboard_config['dates']['last_date'])
# ------- DIVS setup begin
self.page_width = page_width
self.KPI_card_div = self.initialize_cards(width=self.page_width,height=1000)
txt = """<hr/><div style="text-align:center;width:{}px;height:{}px;
position:relative;background:black;margin-bottom:200px">
<h1 style="color:#fff;margin-bottom:300px">{}</h1>
</div>""".format(self.page_width, 50, 'Welcome')
self.notification_div = {
'top': Div(text=txt, width=self.page_width, height=20),
'bottom': Div(text=txt, width=self.page_width, height=10),
}
self.section_divider = '-----------------------------------'
self.section_headers = {
'cards': self.section_header_div(
text='Period to date:{}'.format(self.section_divider),
width=int(self.page_width*.5), html_header='h2', margin_top=5,margin_bottom=-155),
'pop': self.section_header_div(
text='Period over period:{}'.format(self.section_divider),
width=int(self.page_width*.5), html_header='h2', margin_top=5, margin_bottom=-155),
'sig_ratio': self.section_header_div(
text='Time series of ratio of DV to significant IVs'.format(self.section_divider),
width=int(self.page_width*.5), html_header='h2', margin_top=5, margin_bottom=-155),
}
def __init__(self):
dateNow = datetime.now()
self.startDay = DatePicker(title="Start Date",
min_date=datetime(2000, 1, 1),
max_date=datetime(2030, 12, 31),
value=datetime(dateNow.year, dateNow.month,
dateNow.day))
# snapshot time hours minutes
self.Hour = Slider(title="Time of day [h]",
value=1.0,
start=1.0,
end=24.0,
step=0.25)
self.timeStep = Slider(title="Time step [min]",
value=15.0,
start=1.0,
end=60.0,
step=1.0)
self.timeStep.on_change('value', self.updateHourSlider)
r1 = row(self.startDay, self.Hour, self.timeStep)
self.Vul = Slider(title="Voltage upper limit [p.u]",
value=1.05,
start=0.9,
end=1.1,
step=0.005)
self.Vll = Slider(title="Voltage lower limit [p.u]",
value=0.95,
start=0.9,
end=1.1,
step=0.005)
self.Lul = Slider(title="Loading upper limit [%]",
value=90,
start=0,
end=120,
step=1)
r2 = row(self.Vll, self.Vul, self.Lul)
self.final_layout = column(r1, r2)
return
def KPI_user_adoption_tab(DAYS_TO_LOAD=90):
class Thistab(KPI):
def __init__(self, table, cols=[]):
KPI.__init__(self, table, name='social_media', cols=cols)
self.table = table
self.df = None
self.checkboxgroup = {'account_type': [], 'update_type': []}
self.KPI_card_div = self.initialize_cards(self.page_width,
height=350)
# ------- DIVS setup begin
self.page_width = 1200
txt = """<hr/><div style="text-align:center;width:{}px;height:{}px;
position:relative;background:black;margin-bottom:200px">
<h1 style="color:#fff;margin-bottom:300px">{}</h1>
</div>""".format(self.page_width, 50, 'Welcome')
self.notification_div = {
'top': Div(text=txt, width=self.page_width, height=20),
'bottom': Div(text=txt, width=self.page_width, height=10),
}
self.section_divider = '-----------------------------------'
self.section_headers = {
'cards':
self.section_header_div(text='Period to date:{}'.format(
self.section_divider),
width=600,
html_header='h2',
margin_top=5,
margin_bottom=-155),
'pop':
self.section_header_div(text='Period over period:{}'.format(
self.section_divider),
width=600,
html_header='h2',
margin_top=5,
margin_bottom=-155),
}
# ---------------------- DIVS ----------------------------
def section_header_div(self,
text,
html_header='h2',
width=600,
margin_top=150,
margin_bottom=-150):
text = """<div style="margin-top:{}px;margin-bottom:-{}px;"><{} style="color:#4221cc;">{}</{}></div>""" \
.format(margin_top, margin_bottom, html_header, text, html_header)
return Div(text=text, width=width, height=15)
# ---------------------- DIVS ----------------------------
def reset_checkboxes(self, value='all', checkboxgroup=''):
try:
self.checkboxgroup[checkboxgroup].value = value
except Exception:
logger.error('reset checkboxes', exc_info=True)
def information_div(self, width=400, height=300):
div_style = """
style='width:350px;margin-right:-800px;
border:1px solid #ddd;border-radius:3px;background:#efefef50;'
"""
txt = """
<div {}>
<h4 {}>How to interpret relationships </h4>
<ul style='margin-top:-10px;'>
<li>
</li>
<li>
</li>
<li>
</li>
<li>
</li>
<li>
</li>
<li>
</li>
</ul>
</div>
""".format(div_style, self.header_style)
div = Div(text=txt, width=width, height=height)
return div
# -------------------- CARDS -----------------------------------------
def initialize_cards(self, width, height=250):
try:
txt = ''
for period in ['year', 'quarter', 'month', 'week']:
design = random.choice(list(KPI_card_css.keys()))
txt += self.card(title='', data='', card_design=design)
text = """<div style="margin-top:100px;display:flex; flex-direction:row;">
{}
</div>""".format(txt)
div = Div(text=text, width=width, height=height)
return div
except Exception:
logger.error('initialize cards', exc_info=True)
# -------------------- GRAPHS -------------------------------------------
def graph_periods_to_date(self, df1, filter_col):
try:
if self.account_type != 'all':
df1 = df1[df1.account_type == self.account_type]
dct = {}
for idx, period in enumerate(
['week', 'month', 'quarter', 'year']):
df = self.period_to_date(
df1,
timestamp=dashboard_config['dates']['last_date'],
timestamp_filter_col=filter_col,
period=period)
# get unique instances
df = df[['address']]
df = df.compute()
df = df.drop_duplicates(keep='first')
#logger.warning('post duplicates dropped:%s', df.head(10))
data = len(df)
del df
gc.collect()
dct[period] = data
self.update_cards(dct)
except Exception:
logger.error('graph periods to date', exc_info=True)
def graph_period_over_period(self, period):
try:
periods = [period]
start_date = self.pop_start_date
end_date = self.pop_end_date
if isinstance(start_date, date):
start_date = datetime.combine(start_date,
datetime.min.time())
if isinstance(end_date, date):
end_date = datetime.combine(end_date, datetime.min.time())
cols = ['account_type', 'timestamp_of_first_event', 'day']
df = self.load_df(start_date=start_date,
end_date=end_date,
cols=cols,
timestamp_col='timestamp_of_first_event')
if abs(start_date - end_date).days > 7:
if 'week' in periods:
periods.remove('week')
if abs(start_date - end_date).days > 31:
if 'month' in periods:
periods.remove('month')
if abs(start_date - end_date).days > 90:
if 'quarter' in periods:
periods.remove('quarter')
if self.account_type != 'all':
df = df[df.account_type == self.account_type]
# col for when list is empty
self.variable = 'account_type'
for idx, period in enumerate(periods):
df_period = self.period_over_period(
df,
start_date=start_date,
end_date=end_date,
period=period,
history_periods=self.pop_history_periods,
timestamp_col='timestamp_of_first_event')
groupby_cols = ['dayset', 'period']
if len(df_period) > 0:
df_period = df_period.groupby(groupby_cols).agg(
{'account_type': 'count'})
df_period = df_period.reset_index()
df_period = df_period.compute()
else:
df_period = df_period.compute()
df_period = df_period.rename(index=str,
columns={'day': 'dayset'})
prestack_cols = list(df_period.columns)
logger.warning('Line 179:%s', df_period.head(10))
df_period = self.split_period_into_columns(
df_period,
col_to_split='period',
value_to_copy='account_type')
logger.warning('line 180 df_period columns:%s',
df_period.head(50))
poststack_cols = list(df_period.columns)
title = "{} over {}".format(period, period)
plotcols = list(np.setdiff1d(poststack_cols,
prestack_cols))
df_period, plotcols = self.pop_include_zeros(
df_period=df_period, plotcols=plotcols, period=period)
if idx == 0:
p = df_period.hvplot.bar('dayset',
plotcols,
rot=45,
title=title,
stacked=False)
else:
p += df_period.hvplot.bar('dayset',
plotcols,
rot=45,
title=title,
stacked=False)
return p
except Exception:
logger.error('period over period to date', exc_info=True)
def update_account(attrname, old, new):
thistab.notification_updater(
"Calculations underway. Please be patient")
thistab.account_type = new
thistab.graph_periods_to_date(thistab.df, 'timestamp_of_first_event')
thistab.section_header_updater('cards')
thistab.section_header_updater('pop')
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("ready")
def update_period_over_period(attrname, old, new):
thistab.notification_updater(
"Calculations underway. Please be patient")
thistab.pop_history_periods = history_periods_select.value
thistab.pop_start_date = datepicker_period_start.value # trigger period over period
thistab.pop_end_date = datepicker_period_end.value # trigger period
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("ready")
def update_history_periods(attrname, old, new):
thistab.notification_updater(
"Calculations underway. Please be patient")
thistab.pop_history_periods = pop_number_select.value
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("ready")
try:
cols = [
'address', 'account_type', 'update_type', 'balance',
'timestamp_of_first_event'
]
thistab = Thistab(table='account_ext_warehouse', cols=cols)
# ------------------------------------- SETUP ----------------------------
# format dates
first_date_range = thistab.initial_date
last_date_range = datetime.now().date()
last_date = dashboard_config['dates']['last_date']
first_date = datetime(last_date.year, 1, 1, 0, 0, 0)
thistab.df = thistab.load_df(first_date, last_date, cols,
'timestamp_of_first_event')
thistab.graph_periods_to_date(thistab.df,
filter_col='timestamp_of_first_event')
thistab.section_header_updater('cards')
thistab.section_header_updater('pop')
# MANAGE STREAM
# date comes out stream in milliseconds
# --------------------------------CREATE WIDGETS ---------------------------------
datepicker_start = DatePicker(title="Start",
min_date=first_date_range,
max_date=last_date_range,
value=first_date)
datepicker_end = DatePicker(title="End",
min_date=first_date_range,
max_date=last_date_range,
value=last_date)
thistab.pop_end_date = last_date
thistab.pop_start_date = thistab.first_date_in_period(
thistab.pop_end_date, 'week')
stream_launch = streams.Stream.define('Launch', launch=-1)()
datepicker_period_start = DatePicker(title="Period start",
min_date=first_date_range,
max_date=last_date_range,
value=thistab.pop_start_date)
datepicker_period_end = DatePicker(title="Period end",
min_date=first_date_range,
max_date=last_date_range,
value=thistab.pop_end_date)
history_periods_select = Select(
title='Select # of comparative periods',
value='2',
options=thistab.menus['history_periods'])
account_type_select = Select(title='Select account type',
value='all',
options=thistab.menus['account_type'])
pop_number_select = Select(title='Select # of comparative periods',
value=str(thistab.pop_history_periods),
options=thistab.menus['history_periods'])
# --------------------------------- GRAPHS ---------------------------
hv_pop_week = hv.DynamicMap(thistab.pop_week, streams=[stream_launch])
pop_week = renderer.get_plot(hv_pop_week)
hv_pop_month = hv.DynamicMap(thistab.pop_month,
streams=[stream_launch])
pop_month = renderer.get_plot(hv_pop_month)
hv_pop_quarter = hv.DynamicMap(thistab.pop_quarter,
streams=[stream_launch])
pop_quarter = renderer.get_plot(hv_pop_quarter)
# -------------------------------- CALLBACKS ------------------------
#datepicker_start.on_change('value', update)
#datepicker_end.on_change('value', update)
account_type_select.on_change('value', update_account)
history_periods_select.on_change('value', update_period_over_period)
datepicker_period_start.on_change('value', update_period_over_period)
datepicker_period_end.on_change('value', update_period_over_period)
pop_number_select.on_change('value', update_history_periods)
# -----------------------------------LAYOUT ----------------------------
# put the controls in a single element
controls = WidgetBox(datepicker_start, datepicker_end,
account_type_select)
controls_pop = WidgetBox(datepicker_period_start,
datepicker_period_end, history_periods_select)
# create the dashboards
grid = gridplot([[thistab.notification_div['top']],
[Spacer(width=20, height=70)],
[thistab.section_headers['cards']],
[Spacer(width=20, height=2)],
[thistab.KPI_card_div, controls],
[thistab.section_headers['pop']],
[Spacer(width=20, height=25)],
[pop_week.state, controls_pop], [pop_month.state],
[pop_quarter.state],
[thistab.notification_div['bottom']]])
# Make a tab with the layout
tab = Panel(child=grid, title='KPI: user adoption')
return tab
except Exception:
logger.error('rendering err:', exc_info=True)
return tab_error_flag('KPI accounts')
"""
data = query(df,
symbol=select_symbols.value,
date=date_picker.value,
expiry=expiry_select.value)
x_range = [(str(int(a)), str(b))
for a, b in zip(data.strike_pr, data.option_typ)]
p1.x_range.factors = x_range
src = ColumnDataSource({'x': x_range, 'values': data.open_int.values})
source.data = src.data
# Create widgets
select_symbols = Select(title='Pick a symbol', options=symbols)
expiry_select = Select(title='Select an expiry date', options=None)
date_picker = DatePicker(title='Select a date',
value=datetime.datetime.today().date())
button = Button(label="Update", button_type="success")
# Create charts
q = query(df,
symbol=select_symbols.value,
date=date_picker.value,
expiry=expiry_select.value)
q = query(df, symbol='BANKBARODA')
x = [(str(int(a)), str(b)) for a, b in zip(q.strike_pr, q.option_typ)]
source = ColumnDataSource(data=dict(x=x, values=q.open_int.values))
TOOLTIPS = [('index', '$index'), ('open_interest', '@values'),
('option', '@x')]
p1 = figure(x_range=FactorRange(),
title="Open Interest",
tooltips=TOOLTIPS,
width=rectWidth,
height=1,
source=plot_data,
fill_color={
'field': 'level',
'transform': mapper
},
line_color={
'field': 'level',
'transform': mapper
})
# create widgets
datePickStart = DatePicker(name="StartD",
title="Start Date:",
max_date=df['Time'].iloc[-1].date(),
min_date=df['Time'][0].date(),
value=df['Time'][0].date(),
width=140)
datePickEnd = DatePicker(name="EndD",
title="End Date:",
max_date=df['Time'].iloc[-1].date(),
min_date=df['Time'][0].date(),
value=df['Time'].iloc[-1].date(),
width=140)
areaList = []
for n in an['Name']:
areaList.append((n, n))
area_select = MultiSelect(title="Select area:", options=areaList)
channel_select = MultiSelect(title="Select channel:", options=[])
plotDates = Button(label="Plot between start and end dates",
def update_tfl():
date = datetime.strptime(datePicker_tfl.value, "%Y-%m-%d")
params = [[int(newCasesWorld_tfl.value), int(mobility_tfl.value)]]
predict_tfl = vd.predict_transport_use("TfL Tube", params)
predictedDataSource_tfl.stream(dict(Date=[date], prediction=predict_tfl))
newCasesWorld_tfl = Slider(title="New Cases World",
value=10,
start=0,
end=700000,
step=1)
mobility_tfl = Slider(title="Mobility", value=0, start=-400, end=400, step=1)
datePicker_tfl = DatePicker(title="Date",
min_date=date(2020, 3, 1),
max_date=date.today())
preidctToggle_tfl = Button(label="Predict TfL Tube Demand")
preidctToggle_tfl.on_click(lambda event: update_tfl())
def update_rail():
date = datetime.strptime(datePicker_rail.value, "%Y-%m-%d")
params = [[int(newCasesWorld_rail.value), int(mobility_rail.value)]]
predict_rail = vd.predict_transport_use("National Rail", params)
predictedDataSource_rail.stream(dict(Date=[date], prediction=predict_rail))
newCasesWorld_rail = Slider(title="New Cases World",
value=10,
start=0,
def DateWidget(*args, **kw):
kw['min_date'] = kw.pop('start')
kw['max_date'] = kw.pop('end')
return DatePicker(*args,**kw)
text_input = TextInput(placeholder="Enter value ...")
autocomplete_input = AutocompleteInput()
select = Select(options=["Option 1", "Option 2", "Option 3"])
multi_select = MultiSelect(options=["Option %d" % (i + 1) for i in range(16)],
size=6)
slider = Slider(value=10, start=0, end=100)
range_slider = RangeSlider()
#date_range_slider = DateRangeSlider(value=(date(2016, 1, 1), date(2016, 12, 31)))
date_picker = DatePicker()
paragraph = Paragraph(text="some text")
div = Div(text="some <b>text</b>")
pre_text = PreText(text="some text")
def mk_tab(color):
plot = figure(width=300, height=300)
plot.scatter(flowers["petal_length"],
flowers["petal_width"],
color=color,
fill_alpha=0.2,
size=12)
def __init__(self, search_term='beiber'):
# TWITTER SETUP
self.api = None
self.topic = search_term
self.options = {'messages': [str(x) for x in range(10, 1000, 50)]}
self.limits = {
'messages': int(self.options['messages'][0]),
}
self.hidden_path = dashboard_config['hidden_path']
self.DATEFORMAT = "%Y-%d-%m %H:%M:%S"
self.df = None
min_date = datetime.today() - timedelta(days=7)
print(min_date)
self.selects = {
'window':
Select(title='Select rolling mean window',
value='1',
options=[str(x) for x in range(1, 20, 2)]),
'date_since':
DatePicker(title="Tweets since:",
min_date=min_date,
max_date=datetime.today(),
value=min_date)
}
self.selects_values = {
'window': int(self.selects['window'].value),
'date_since': self.selects['date_since'].value
}
self.resample_period = {'menu': []}
for val in range(30, 350, 30):
self.resample_period['menu'].append(str(val) + 'Min')
self.resample_period['value'] = self.resample_period['menu'][0]
# DIV VISUAL SETUP
self.trigger = -1
self.html_header = 'h2'
self.margin_top = 150
self.margin_bottom = -150
self.div_style = """
style='width:350px; margin-left:25px;
border:1px solid #ddd;border-radius:3px;background:#efefef50;'
"""
self.header_style = """ style='color:blue;text-align:center;' """
self.page_width = 1250
txt = """<hr/>
<div style="text-align:center;width:{}px;height:{}px;
position:relative;background:black;margin-bottom:200px">
<h1 style="color:#fff;margin-bottom:300px">{}</h1>
</div>""".format(self.page_width, 50, 'Welcome')
self.notification_div = {
'top': Div(text=txt, width=self.page_width, height=20),
'bottom': Div(text=txt, width=self.page_width, height=10),
}
self.section_divider = '-----------------------------------'
self.section_headers = {
'twitter':
self.section_header_div(text='Twitter search results:',
width=600,
html_header='h2',
margin_top=155,
margin_bottom=-155),
}
row = df.to_dict('records')[0]
overview_dict[name]['value'] = round(row['close'], 2)
overview_dict[name]['change'] = round(row['pchange'], 2)
return overview_dict
cdoc = curdoc()
dboard = DashBoard()
# IndexSelecter
index_codes = list(ct.INDEX_DICT.keys())
index_codes.append('880883')
index_type_list = ['K线图', '牛熊股比']
index_select = Select(value='000001', title='指数代码', options=sorted(index_codes), height=50)
index_type_select = Select(value='牛熊股比', title='数据类型', options=sorted(index_type_list), height=50)
# DatePciekr
dt_pckr_start = DatePicker(title='开始日期', value = date.today() - timedelta(days = 200), min_date = date(2004,1,1), max_date = date.today())
dt_pckr_end = DatePicker(title='结束日期', value = date.today(), min_date = date(2004,1,1), max_date = date.today())
index_select_row = row(index_select, index_type_select, dt_pckr_start, dt_pckr_end)
index_layout = column(index_select_row, create_index_figure_column(index_select.value, index_type_select.value, dt_pckr_start.value, dt_pckr_end.value))
index_select.on_change('value', update_index)
index_type_select.on_change('value', update_index)
dt_pckr_start.on_change('value', update_index)
dt_pckr_end.on_change('value', update_index)
# Add Stock Analysis
stock_auto_input = AutocompleteInput(value = '601318', completions = initializer.update_code_list(), title = '股票代码')
# DatePciekr
stock_pckr_start = DatePicker(title='开始日期', value = date.today() - timedelta(days = 100), min_date = date(2000,1,1), max_date = date.today())
stock_pckr_end = DatePicker(title='股票日期', value = date.today(), min_date = date(2000,1,1), max_date = date.today())
def KPI_projects_tab(panel_title, DAYS_TO_LOAD=90):
timeline_source = ColumnDataSource(data=dict(
Item=[], Start=[], End=[], Color=[], start=[], end=[], ID=[], ID1=[]))
class Thistab(KPI):
def __init__(self, table, cols=[]):
KPI.__init__(self, table, name='project', cols=cols)
self.table = table
self.df = None
self.df_pop = None
self.checkboxgroup = {}
self.period_to_date_cards = {}
self.ptd_startdate = datetime(datetime.today().year, 1, 1, 0, 0, 0)
self.timestamp_col = 'project_startdate_actual'
self.pym = PythonMongo('aion')
self.groupby_dict = {
'project': 'sum',
'project_duration': 'sum',
'project_start_delay': 'mean',
'project_end_delay': ' mean',
'milestone': 'sum',
'milestone_duration': 'sum',
'milestone_start_delay': 'mean',
'milestone_end_delay': ' mean',
'task': 'sum',
'task_duration': 'sum',
'task_start_delay': 'mean',
'task_end_delay': ' mean',
}
self.menus = {
'status': ['all', 'open', 'closed'],
'type': [
'all', 'research', 'reconciliation', 'audit', 'innovation',
'construction', 'manufacturing', 'conference'
],
'gender': ['all', 'male', 'female'],
'variables':
list(self.groupby_dict.keys()),
'history_periods':
['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'],
}
self.status = 'all'
self.pm_gender = 'all'
self.m_gender = 'all'
self.t_gender = 'all'
self.type = 'all'
self.variables = sorted(list(self.groupby_dict.keys()))
self.variable = self.variables[0]
self.groupby_var = 'project'
self.chord_data = {
'rename': {
'project_owner': 'source',
'milestone_owner': 'target',
'remuneration': 'value'
},
'percentile_threshold': .75,
}
self.percentile_threshold = 10
# ------- DIVS setup begin
self.page_width = 1200
txt = """<hr/><div style="text-align:center;width:{}px;height:{}px;
position:relative;background:black;margin-bottom:200px">
<h1 style="color:#fff;margin-bottom:300px">{}</h1>
</div>""".format(self.page_width, 50,
'Welcome')
self.notification_div = {
'top': Div(text=txt, width=self.page_width, height=20),
'bottom': Div(text=txt, width=self.page_width, height=10),
}
self.section_divider = '-----------------------------------'
self.section_headers = {
'cards':
self.section_header_div(text='Period to date:{}'.format(
self.section_divider),
width=1000,
html_header='h2',
margin_top=50,
margin_bottom=5),
'pop':
self.section_header_div(text='Period over period:{}'.format(
self.section_divider),
width=600,
html_header='h2',
margin_top=5,
margin_bottom=-155),
'chord':
self.section_header_div(text='Relationships:{}'.format(
self.section_divider),
width=600,
html_header='h2',
margin_top=5,
margin_bottom=-155),
'timeline':
self.section_header_div(text='Project timeline:{}'.format(
self.section_divider),
width=600,
html_header='h2',
margin_top=5,
margin_bottom=-155),
}
self.KPI_card_div = self.initialize_cards(self.page_width,
height=350)
start = datetime(2014, 1, 1, 0, 0, 0)
end = datetime(2019, 5, 15, 0, 0, 0)
self.tools = [
BoxZoomTool(),
ResetTool(),
PanTool(),
SaveTool(),
WheelZoomTool()
]
self.timeline_vars = {
'projects':
'',
'project':
'',
'types': ['all', 'milestone', 'task', 'project'],
'type':
'all',
'DF':
None,
'G':
figure(title=None,
x_axis_type='datetime',
width=1200,
height=900,
y_range=[],
x_range=Range1d(start, end),
toolbar_location=None),
'toolbar_box':
ToolbarBox()
}
# ----- UPDATED DIVS END
# ---------------------- DIVS ----------------------------
def section_header_div(self,
text,
html_header='h2',
width=600,
margin_top=150,
margin_bottom=-150):
text = """<div style="margin-top:{}px;margin-bottom:-{}px;"><{} style="color:#4221cc;">{}</{}></div>"""\
.format(margin_top,margin_bottom,html_header, text, html_header)
return Div(text=text, width=width, height=15)
def information_div(self, width=400, height=300):
txt = """
<div {}>
<h4 {}>How to interpret sentiment score</h4>
<ul style='margin-top:-10px;'>
<li>
</li>
<li>
</li>
<li>
</li>
<li>
</li>
</ul>
</div>
""".format(self.div_style, self.header_style)
div = Div(text=txt, width=width, height=height)
return div
def initialize_cards(self, width, height=250):
try:
txt = ''
for period in ['year', 'quarter', 'month', 'week']:
design = random.choice(list(KPI_card_css.keys()))
txt += self.card(title='', data='', card_design=design)
text = """<div style="margin-top:100px;display:flex; flex-direction:row;">
{}
</div>""".format(txt)
div = Div(text=text, width=width, height=height)
return div
except Exception:
logger.error('initialize cards', exc_info=True)
def load_df(self, req_startdate, req_enddate, table, cols,
timestamp_col):
try:
# get min and max of loaded df
if self.df is not None:
loaded_min = self.df[timestamp_col].min()
loaded_max = self.df[timestamp_col].max()
if loaded_min <= req_startdate and loaded_max >= req_enddate:
df = self.df[(self.df[timestamp_col] >= req_startdate)
& (self.df[timestamp_col] <= req_enddate)]
return df
return self.pym.load_df(req_startdate,
req_enddate,
table=table,
cols=cols,
timestamp_col=timestamp_col)
except Exception:
logger.error('load_df', exc_info=True)
def filter_df(self, df1):
if self.status != 'all':
df1 = df1[df1.status == self.status]
if self.pm_gender != 'all':
df1 = df1[df1.project_owner_gender == self.pm_gender]
if self.m_gender != 'all':
df1 = df1[df1.milestone_owner_gender == self.m_gender]
if self.t_gender != 'all':
df1 = df1[df1.task_owner_gender == self.t_gender]
if self.type != 'all':
df1 = df1[df1.type == self.type]
return df1
def period_to_date(self,
df,
timestamp=None,
timestamp_filter_col=None,
cols=[],
period='week'):
try:
if timestamp is None:
timestamp = datetime.now()
timestamp = datetime(timestamp.year, timestamp.month,
timestamp.day, timestamp.hour, 0, 0)
start = self.first_date_in_period(timestamp, period)
# filter
df[timestamp_filter_col] = pd.to_datetime(
df[timestamp_filter_col], format=self.DATEFORMAT_PTD)
#logger.warning('df:%s', df[self.timestamp_col])
df = df[(df[timestamp_filter_col] >= start)
& (df[timestamp_filter_col] <= timestamp)]
if len(cols) > 0:
df = df[cols]
return df
except Exception:
logger.error('period to date', exc_info=True)
def period_over_period(self,
df,
start_date,
end_date,
period,
history_periods=2,
timestamp_col='timestamp_of_first_event'):
try:
# filter cols if necessary
string = '0 {}(s) prev(current)'.format(period)
# filter out the dates greater than today
df_current = df.copy()
df_current = self.filter_df(df_current)
logger.warning('df current:%s', df_current.head(10))
df_current['period'] = string
# label the days being compared with the same label
if len(df_current) > 0:
df_current = self.label_dates_pop(df_current, period,
timestamp_col)
cols = [self.variable, 'period', 'dayset']
if 'project' in self.variable:
if self.variable != 'project':
df_current = df_current[[
self.variable, 'period', 'dayset', 'project'
]]
elif 'milestone' in self.variable:
if self.variable != 'milestone':
df_current = df_current[[
self.variable, 'period', 'dayset', 'milestone',
'project'
]]
elif 'task' in self.variable:
if self.variable != 'task':
df_current = df_current[[
self.variable, 'period', 'dayset', 'task',
'milestone', 'project'
]]
# zero out time information
start = datetime(start_date.year, start_date.month,
start_date.day, 0, 0, 0)
end = datetime(end_date.year, end_date.month, end_date.day, 0,
0, 0)
cols = list(df.columns)
counter = 1
if isinstance(history_periods, str):
history_periods = int(history_periods)
# make dataframes for request no. of periods
start, end = self.shift_period_range(period, start, end)
while counter < history_periods and start >= self.initial_date:
# load data
df_temp = self.load_df(start,
end,
table=self.table,
cols=[],
timestamp_col=timestamp_col)
df_temp = self.filter_df(df_temp)
if df_temp is not None:
if len(df_temp) > 1:
df_temp[timestamp_col] = pd.to_datetime(
df_temp[timestamp_col])
string = '{} {}(s) prev'.format(counter, period)
# label period
df_temp['period'] = string
# relabel days to get matching day of week,doy, dom, for different periods
df_temp = self.label_dates_pop(
df_temp, period, timestamp_col)
df_temp = df_temp[cols]
# logger.warning('df temp loaded for %s previous: %s',counter,len(df_temp))
df_current = pd.concat([df_current, df_temp])
del df_temp
gc.collect()
# shift the loading window
counter += 1
start, end = self.shift_period_range(period, start, end)
return df_current
except Exception:
logger.error('period over period', exc_info=True)
# label dates for period over period (pop)
def pop_include_zeros(self, df_period, plotcols, period):
try:
# check for no data on original dates
tmp_title = '0 {}(s) prev(current)'.format(period)
if tmp_title not in plotcols:
df_period[tmp_title] = [0] * len(df_period)
plotcols.append(tmp_title)
logger.warning('line 218 cols to plot:%s', plotcols)
# do other periods
tmp = plotcols[0]
txt = tmp[1:]
if isinstance(self.pop_history_periods, str):
self.pop_history_periods = int(self.pop_history_periods)
for i in range(1, self.pop_history_periods):
tmp_txt = str(i) + txt
if tmp_txt not in plotcols:
df_period[tmp_txt] = [0] * len(df_period)
plotcols.append(tmp_txt)
clean_plotcols = []
for idx, col in enumerate(plotcols):
if 'prev' in col or 'curr' in col:
clean_plotcols.append(col)
logger.warning(
'LINE 340 plotcols at end of pop include zeros:%s',
clean_plotcols)
return df_period, sorted(clean_plotcols)
except Exception:
logger.error('pop include zeros', exc_info=True)
def label_dates_pop(self, df, period, timestamp_col):
#df[timestamp_col] = pd.to_datetime(df[timestamp_col])
def label_qtr_pop(y):
try:
curr_quarter = int((y.month - 1) / 3 + 1)
start = datetime(y.year, 3 * curr_quarter - 2, 1)
return abs((start - y).days)
except Exception:
logger.error('df label quarter', exc_info=True)
try:
logger.warning('df columns:%s', list(df.columns))
if period == 'week':
df['dayset'] = df[timestamp_col].dt.dayofweek
elif period == 'month':
df['dayset'] = df[timestamp_col].dt.day
elif period == 'year':
#logger.warning('LINE 218:%s', df.head(5))
df['dayset'] = df[timestamp_col].dt.dayofyear
elif period == 'quarter':
df['dayset'] = df[timestamp_col].apply(
lambda x: label_qtr_pop(x))
return df
except Exception:
logger.error('label data ', exc_info=True)
def get_groupby_pop_df(self, df, variable, groupby_cols):
try:
if df is not None:
if len(df) > 0:
if 'dayset' in df.columns:
if variable in ['project']:
df = df.groupby(groupby_cols).agg(
{variable: 'count'})
df = df.reset_index()
#logger.warning('LINE 286 df:%s',df)
elif variable in ['milestone']:
df = df.groupby(groupby_cols).agg(
{variable: 'count'})
df = df.reset_index()
#logger.warning('LINE 291 df:%s', df)
elif variable in ['task']:
df = df.groupby(groupby_cols).agg(
{variable: 'count'})
df = df.reset_index()
elif variable in ['remuneration']:
df = df.groupby(groupby_cols).agg(
{variable: 'sum'})
df = df.reset_index()
else:
#logger.warning('LINE 259:df:%s',df.head())
df = df.groupby(groupby_cols).agg(
{variable: 'mean'})
df = df.reset_index()
# clean up
if self.groupby_var in df.columns and self.variable != self.groupby_var:
df = df.drop([self.groupby_var], axis=1)
return df
except Exception:
logger.error('get groupby card data', exc_info=True)
def get_groupby_card_data(self, df, variable):
try:
if variable in ['project']:
data = len(df[variable].unique())
data = "{} {}s".format(data, variable)
elif variable in ['milestone']:
df = df.groupby(['project']).agg({variable: 'nunique'})
data = df[variable].sum()
data = "{} {}s".format(data, variable)
elif variable in ['task']:
df = df.groupby(['project',
'milestone']).agg({variable: 'count'})
data = df[variable].sum()
data = "{} {}s".format(data, variable)
elif variable in ['project_duration'] or 'delay' in variable:
df = df.groupby([self.groupby_var]).agg({variable: 'mean'})
df = df.reset_index()
data = "{} days".format(round(df[variable].sum(), 2))
elif variable in ['milestone_duration']:
df = df.groupby([self.groupby_var,
'project']).agg({variable: 'mean'})
df = df.reset_index()
data = "{} days".format(round(df[variable].sum(), 2))
elif variable in [
'task_duration', 'task_start_delay', 'task_start_end'
]:
df = df.groupby([self.groupby_var, 'project',
'milestone']).agg({variable: 'mean'})
df = df.reset_index()
data = "{} hours".format(round(df[variable].sum(), 2))
elif variable in ['remuneration']:
data = df[variable].sum()
data = "${:,.2f}".format(data)
return data
except Exception:
logger.error('get groupby card data', exc_info=True)
# -------------------- GRAPHS -------------------------------------------
def graph_periods_to_date(self, df2, timestamp_filter_col, variable):
df1 = df2.copy()
#self.section_header_updater(section='cards',label=variable,margin_top=159,html_header='h2')
try:
df1 = self.filter_df(df1)
dct = {}
for idx, period in enumerate(
['week', 'month', 'quarter', 'year']):
df = self.period_to_date(
df1,
timestamp=dashboard_config['dates']['last_date'],
timestamp_filter_col=timestamp_filter_col,
period=period)
df = df.drop_duplicates(keep='first')
# groupby to eliminate repetition
data = self.get_groupby_card_data(df, variable)
del df
gc.collect()
dct[period] = data
#title = "{} to date".format(period)
#p = self.card(title=title, data=data, card_design=random.choice(list(self.KPI_card_css.keys())))
#self.period_to_date_cards[period].text = p.text
self.update_cards(dct)
except Exception:
logger.error('graph periods to date', exc_info=True)
def graph_period_over_period(self, period):
try:
periods = [period]
start_date = self.pop_start_date
end_date = self.pop_end_date
if isinstance(start_date, date):
start_date = datetime.combine(start_date,
datetime.min.time())
if isinstance(end_date, date):
end_date = datetime.combine(end_date, datetime.min.time())
today = datetime.combine(datetime.today().date(),
datetime.min.time())
df = self.df_pop.copy()
df = self.filter_df(df)
#logger.warning('LINE 363 -df:%s',df.head())
cols = [self.variable, self.timestamp_col]
if self.variable != 'project':
cols.append('project')
if abs(start_date - end_date).days > 7:
if 'week' in periods:
periods.remove('week')
if abs(start_date - end_date).days > 31:
if 'month' in periods:
periods.remove('month')
if abs(start_date - end_date).days > 90:
if 'quarter' in periods:
periods.remove('quarter')
for idx, period in enumerate(periods):
df_period = self.period_over_period(
df,
start_date=start_date,
end_date=end_date,
period=period,
history_periods=self.pop_history_periods,
timestamp_col=self.timestamp_col)
groupby_cols = ['dayset', 'period']
if len(df_period) > 0:
logger.warning('LINE 473:%s', list(df_period.columns))
df_period = self.get_groupby_pop_df(
df_period,
variable=self.variable,
groupby_cols=groupby_cols)
df_period = df_period.reset_index()
else:
if not 'day' in df_period.columns:
df_period['dayset'] = ""
else:
df_period = df_period.rename(
index=str, columns={'day': 'dayset'})
logger.warning('LINE 478:%s', list(df_period.columns))
prestack_cols = list(df_period.columns)
df_period = self.split_period_into_columns(
df_period,
col_to_split='period',
value_to_copy=self.variable)
# short term fix: filter out the unnecessary first day added by a corrupt quarter functionality
if period == 'quarter':
if 'dayset' in df_period.columns:
min_day = df_period['dayset'].min()
df_period = df_period[
df_period['dayset'] > min_day]
poststack_cols = list(df_period.columns)
title = "{} over {}".format(period, period)
plotcols = list(np.setdiff1d(poststack_cols,
prestack_cols))
# include current period if not extant
df_period, plotcols = self.pop_include_zeros(
df_period, plotcols=plotcols, period=period)
if self.variable in [
'task_start_delay', 'task_end_delay',
'task_duration'
]:
ylabel = 'hours'
elif self.variable in [
'project_duration', 'milestone_duration',
'project_start_delay', 'project_end_delay',
'milestone_start_delay', 'milestone_end_delay'
]:
ylabel = 'days'
elif self.variable in ['project', 'task', 'milestone']:
ylabel = '#'
elif self.variable == 'remuneration':
ylabel = '$'
if 'dayset' not in df_period.columns:
leng = len(df_period)
if leng > 0:
df_period['dayset'] = 0
logger.warning('LINE 549')
else:
logger.warning('LINE 551')
df_period['dayset'] = ''
logger.warning('LINE 552: df columns:%s',
list(df_period.columns))
if idx == 0:
p = df_period.hvplot.bar('dayset',
plotcols,
rot=45,
title=title,
stacked=False,
width=1200,
height=400,
value_label=ylabel)
else:
p += df_period.hvplot.bar('dayset',
plotcols,
rot=45,
title=title,
stacked=False,
width=1200,
height=400,
value_label=ylabel)
return p
except Exception:
logger.error('period over period to date', exc_info=True)
def chord_diagram(self, launch):
try:
def normalize_value(x, total):
x = int((x / total) * 1000)
if x <= 0:
return 1
return x
df = self.df.copy()
# -------------- nodes
data = {}
data['nodes'] = []
source_list = df['milestone_owner'].tolist()
names = list(set(source_list))
person_type_dict = dict(zip(df.milestone_owner, df.type))
type_dict = {}
types = list(set(df['type'].tolist()))
name_dict = {}
for idx, name in enumerate(names):
name_dict[name] = idx
for idx, name in enumerate(names):
type_tmp = person_type_dict[name]
index = name_dict[name]
data['nodes'].append({
'OwnerID': index,
'index': idx,
'Type': type_tmp
})
nodes = hv.Dataset(pd.DataFrame(data['nodes']), 'index')
# --------- make the links
data['links'] = []
for idx, row in df.iterrows():
src = name_dict[row['project_owner']]
tgt = name_dict[row['milestone_owner']]
val = row['remuneration']
data['links'].append({
'source': src,
'target': tgt,
'value': val
})
links = pd.DataFrame(data['links'])
# get the individual links
links = links.groupby(['source', 'target'])['value'].sum()
links = links.reset_index()
total = links['value'].sum()
links['value'] = links['value'].apply(
lambda x: normalize_value(x, total))
# filter for top percentile
quantile_val = links['value'].quantile(
self.chord_data['percentile_threshold'])
links = links[links['value'] >= quantile_val]
#logger.warning('after quantile filter:%s',len(links))
chord_ = hv.Chord((links, nodes), ['source', 'target'],
['value'])
chord_.opts(
opts.Chord(cmap='Category20',
edge_cmap='Category20',
edge_color=dim('source').str(),
labels='Type',
node_color=dim('index').str(),
width=1000,
height=1000))
return chord_
except Exception:
logger.error('chord diagram', exc_info=True)
def timeline(self, project, type='milestone'):
try:
DF = self.df.copy()
if type != project:
DF = DF[DF['project'] == project]
if type == 'all':
rename_dct = {
'milestone_enddate_proposed': 'milestone_enddate',
'milestone_startdate_proposed': 'milestone_startdate',
'task_enddate_proposed': 'task_enddate',
'task_startdate_proposed': 'task_startdate',
}
DF = DF.rename(index=str, columns=rename_dct)
DF = DF.groupby(['milestone', 'task']).agg({
'milestone_startdate':
'min',
'milestone_enddate':
'max',
'task_startdate':
'min',
'task_enddate':
'max',
})
DF = DF.reset_index()
# melt to get milestone and task into one column
df = pd.melt(DF,
value_vars=['milestone', 'task'],
id_vars=[
'milestone_startdate',
'milestone_enddate', 'task_startdate',
'task_enddate'
],
value_name='Item',
var_name='type')
df = df.groupby(['Item', 'type']).agg({
'milestone_startdate':
'min',
'milestone_enddate':
'max',
'task_startdate':
'min',
'task_enddate':
'max'
}).reset_index()
df = pd.melt(
df,
id_vars=[
'Item', 'type', 'milestone_startdate',
'task_startdate'
],
value_vars=['milestone_enddate', 'task_enddate'],
value_name='End',
var_name='enddate_type')
# filter out where tasks label dates and vice versa
df1 = df[(df['type'] == 'task')
& (df['enddate_type'] == 'task_enddate')]
df = df[(df['type'] == 'milestone')
& (df['enddate_type'] == 'milestone_enddate')]
df = pd.concat([df1, df])
df = df.drop('enddate_type', axis=1)
# do startdate
df = pd.melt(
df,
id_vars=['Item', 'type', 'End'],
value_vars=['milestone_startdate', 'task_startdate'],
value_name='Start',
var_name='startdate_type')
# filter out where tasks label dates and vice versa
df1 = df[(df['type'] == 'task')
& (df['startdate_type'] == 'task_startdate')]
df = df[(df['type'] == 'milestone')
& (df['startdate_type'] == 'milestone_startdate')]
df = pd.concat([df1, df])
df = df.drop('startdate_type', axis=1)
# label colors
df['Color'] = df['type'].apply(
lambda x: 'black' if x == 'milestone' else 'green')
# organize by milestone and tasks belonging to milestone
df = df.sort_values(by=['Start']).reset_index()
df = df.drop('index', axis=1)
#logger.warning('LINE 605 - df:%s',df.head(50))
DF = df
print(
'##################################################################################'
)
else:
start_str = type + '_startdate_proposed'
end_str = type + '_enddate_proposed'
# group milestone
rename_dct = {
start_str: 'Start',
end_str: 'End',
type: 'Item'
}
DF = DF.rename(index=str, columns=rename_dct)
DF = DF[['Item', 'Start', 'End']]
DF = DF.groupby(['Item']).agg({
'Start': 'min',
'End': 'max'
})
DF = DF.reset_index()
color_list = []
for item in DF.Item.tolist():
color_list.append(
random.choice(dashboard_config['colors']))
DF['Color'] = np.array(color_list)
DF['start'] = DF['Start'].dt.strftime('%Y-%m-%d')
DF['end'] = DF['End'].dt.strftime('%Y-%m-%d')
DF['ID'] = DF.index + 0.6
DF['ID1'] = DF.index + 1.4
logger.warning('LINE 648 %s', DF)
self.timeline_vars['DF'] = DF
# update source
data = dict(Item=DF.Item.tolist(),
Start=DF.Start.tolist(),
End=DF.End.tolist(),
Color=DF.Color.tolist(),
start=DF.start.tolist(),
end=DF.end.tolist(),
ID=DF.ID.tolist(),
ID1=DF.ID1.tolist())
# <-- This is the trick, make the x_rage empty first, before assigning new value
self.timeline_vars['G'].y_range.factors = []
self.timeline_vars['G'].y_range.factors = DF.Item.tolist()
#self.timeline_vars['G'].x_range.factors = []
#self.timeline_vars['G'].x_range.factors = sorted(DF.Start.tolist())
timeline_source.data = data
except Exception:
logger.error('timeline', exc_info=True)
def timeline_plot(self, DF):
try:
hover = HoverTool(tooltips="Task: @Item<br>\
Start: @start<br>\
End: @end")
self.timeline_vars['G'].quad(left='Start',
right='End',
bottom='ID',
top='ID1',
source=timeline_source,
color="Color")
self.tools = [hover] + self.tools
self.timeline_vars['G'].tools = self.tools
self.timeline_vars['toolbar_box'] = ToolbarBox()
self.timeline_vars['toolbar_box'].toolbar = Toolbar(
tools=self.tools)
self.timeline_vars['toolbar_box'].toolbar_location = "above"
self.timeline_vars['G'].x_range.start = DF.Start.min(
) - timedelta(days=10)
self.timeline_vars['G'].x_range.start = DF.End.max(
) + timedelta(days=10)
return self.timeline_vars['G']
except Exception:
logger.error('timeline', exc_info=True)
def update(attrname, old, new):
thistab.notification_updater(
"Calculations underway. Please be patient")
thistab.pm_gender = pm_gender_select.value
thistab.m_gender = m_gender_select.value
thistab.t_gender = t_gender_select.value
thistab.type = type_select.value
thistab.variable = variable_select.value
if 'project' in thistab.variable:
thistab.groupby_var = 'project'
elif 'milestone' in thistab.variable:
thistab.groupby_var = 'milestone'
elif 'task' in thistab.variable:
thistab.groupby_var = 'task'
thistab.status = status_select.value
thistab.graph_periods_to_date(thistab.df,
thistab.timestamp_col,
variable=thistab.variable)
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("ready")
def update_pop_dates():
thistab.notification_updater(
"Calculations underway. Please be patient")
thistab.pop_history_periods = pop_number_select.value
thistab.pop_start_date = datepicker_pop_start.value # trigger period over period
thistab.pop_end_date = datepicker_pop_end.value
thistab.df_pop = thistab.pym.load_df(start_date=thistab.pop_start_date,
end_date=thistab.pop_end_date,
cols=[],
table=thistab.table,
timestamp_col='startdate_actual')
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("ready")
def update_history_periods(attrname, old, new):
thistab.notification_updater(
"Calculations underway. Please be patient")
thistab.pop_history_periods = pop_number_select.value
thistab.trigger += 1
stream_launch.event(launch=thistab.trigger)
thistab.notification_updater("ready")
def update_timeline(attrname, old, new):
thistab.notification_updater(
"Calculations underway. Please be patient")
thistab.timeline_vars['project'] = timeline_project_select.value
thistab.timeline_vars['type'] = timeline_type_select.value
thistab.timeline(thistab.timeline_vars['project'],
thistab.timeline_vars['type'])
thistab.notification_updater("ready")
try:
cols = []
thistab = Thistab(table='project_composite', cols=cols)
# ------------------------------------- SETUP ----------------------------
# format dates
first_date_range = thistab.initial_date
last_date_range = datetime.now().date()
last_date = dashboard_config['dates']['last_date']
first_date = datetime(last_date.year, 4, 1, 0, 0, 0)
thistab.df = thistab.pym.load_df(start_date=first_date,
end_date=last_date,
table=thistab.table,
cols=[],
timestamp_col=thistab.timestamp_col)
thistab.graph_periods_to_date(
thistab.df,
timestamp_filter_col=thistab.timestamp_col,
variable=thistab.variable)
thistab.pop_end_date = last_date
thistab.pop_start_date = last_date - timedelta(days=5)
thistab.df_pop = thistab.pym.load_df(
start_date=thistab.pop_start_date,
end_date=thistab.pop_end_date,
cols=[],
table=thistab.table,
timestamp_col=thistab.timestamp_col)
thistab.timeline_vars['projects'] = sorted(
list(set(thistab.df['project'].tolist())))
thistab.timeline_vars['project'] = thistab.timeline_vars['projects'][0]
# MANAGE STREAM
# date comes out stream in milliseconds
# --------------------------------CREATE WIDGETS ---------------------------------
stream_launch = streams.Stream.define('Launch', launch=-1)()
datepicker_pop_start = DatePicker(title="Period start",
min_date=first_date_range,
max_date=last_date_range,
value=thistab.pop_start_date)
datepicker_pop_end = DatePicker(title="Period end",
min_date=first_date_range,
max_date=last_date_range,
value=thistab.pop_end_date)
pop_number_select = Select(title='Select # of comparative periods',
value=str(thistab.pop_history_periods),
options=thistab.menus['history_periods'])
pop_dates_button = Button(label="Select dates, then click me!",
width=15,
button_type="success")
type_select = Select(title='Select project type',
value=thistab.type,
options=thistab.menus['type'])
status_select = Select(title='Select project status',
value=thistab.status,
options=thistab.menus['status'])
pm_gender_select = Select(title="Select project owner's gender",
value=thistab.pm_gender,
options=thistab.menus['gender'])
m_gender_select = Select(title="Select milestone owner's gender",
value=thistab.m_gender,
options=thistab.menus['gender'])
t_gender_select = Select(title="Select task owner's gender",
value=thistab.t_gender,
options=thistab.menus['gender'])
variable_select = Select(title='Select variable of interest',
value=thistab.variable,
options=thistab.menus['variables'])
timeline_project_select = Select(
title='Select project',
value=thistab.timeline_vars['project'],
options=thistab.timeline_vars['projects'])
timeline_type_select = Select(title='Select granularity',
value='all',
options=thistab.timeline_vars['types'])
# --------------------------------- GRAPHS ---------------------------
hv_pop_week = hv.DynamicMap(thistab.pop_week, streams=[stream_launch])
pop_week = renderer.get_plot(hv_pop_week)
hv_pop_month = hv.DynamicMap(thistab.pop_month,
streams=[stream_launch])
pop_month = renderer.get_plot(hv_pop_month)
hv_pop_quarter = hv.DynamicMap(thistab.pop_quarter,
streams=[stream_launch])
pop_quarter = renderer.get_plot(hv_pop_quarter)
hv_pop_year = hv.DynamicMap(thistab.pop_year, streams=[stream_launch])
pop_year = renderer.get_plot(hv_pop_year)
hv_chord = hv.DynamicMap(thistab.chord_diagram,
streams=[stream_launch])
chord = renderer.get_plot(hv_chord)
thistab.timeline(thistab.timeline_vars['project'],
thistab.timeline_vars['type'])
timeline = thistab.timeline_plot(DF=thistab.timeline_vars['DF'])
# -------------------------------- CALLBACKS ------------------------
type_select.on_change('value', update)
pop_dates_button.on_click(update_pop_dates) # lags array
status_select.on_change('value', update)
pm_gender_select.on_change('value', update)
m_gender_select.on_change('value', update)
t_gender_select.on_change('value', update)
variable_select.on_change('value', update)
pop_number_select.on_change('value', update_history_periods)
timeline_project_select.on_change('value', update_timeline)
timeline_type_select.on_change('value', update_timeline)
# -----------------------------------LAYOUT ----------------------------
# put the controls in a single element
controls_top = WidgetBox(
variable_select,
type_select,
status_select,
pm_gender_select,
m_gender_select,
t_gender_select,
)
controls_pop = WidgetBox(datepicker_pop_start, datepicker_pop_end,
pop_number_select)
controls_timeline = WidgetBox(thistab.timeline_vars['toolbar_box'],
timeline_project_select,
timeline_type_select)
grid = gridplot([[thistab.notification_div['top']],
[Spacer(width=20, height=70)],
[thistab.section_headers['cards']],
[thistab.KPI_card_div, controls_top],
[thistab.section_headers['pop']],
[Spacer(width=20, height=25)],
[pop_week.state, controls_pop], [pop_month.state],
[pop_quarter.state], [pop_year.state],
[thistab.section_headers['chord']],
[Spacer(width=20, height=25)], [chord.state],
[thistab.section_headers['timeline']],
[Spacer(width=20, height=25)],
[timeline, controls_timeline],
[thistab.notification_div['bottom']]])
# Make a tab with the layout
tab = Panel(child=grid, title=panel_title)
return tab
except Exception:
logger.error('rendering err:', exc_info=True)
return tab_error_flag(panel_title)
def selection_tab(rtdap_data):
def make_base_map(tile_map=CARTODBPOSITRON_RETINA,
map_width=800,
map_height=500,
xaxis=None,
yaxis=None,
xrange=(-9990000, -9619944),
yrange=(5011119, 5310000),
plot_tools="pan,wheel_zoom,reset,save"):
p = figure(tools=plot_tools,
width=map_width,
height=map_height,
x_axis_location=xaxis,
y_axis_location=yaxis,
x_range=xrange,
y_range=yrange,
toolbar_location="above")
p.grid.grid_line_color = None
#p.background_fill_color = None
p.background_fill_alpha = 0.5
p.border_fill_color = None
p.add_tile(tile_map)
return p
def rtdap_avg(df, corr, value):
"""
returns mean for specificed attribute by highway corridor
Keyword arguments:
df -- dataframe to filter by corridor and calculate mean
corr -- corridor name
value -- dataframe column name to calculate mean
"""
df_corr = df.loc[df['corridor'] == corr]
mean_value = df_corr[value].mean()
return mean_value
def filter_selection(df, corr, date_s, date_e, weekday, tod):
"""
returns subset of data based on corridor and time selections
Keyword arguments:
df -- dataframe to filter by corridor and time selections
corr -- corridor name
date_s -- start date
date_e -- end date
weekday -- day of week (Monday - Friday)
tod -- time of day (8 tod time periods)
"""
tod_start = tod[0]
tod_end = tod[1]
date_start = datetime.strptime(date_s, '%Y-%m-%d')
date_end = datetime.strptime(date_e, '%Y-%m-%d')
if weekday == 'All':
weekday = df['dow'].drop_duplicates().values.tolist()
else:
weekday = [weekday]
select_df = df.loc[(df['corridor'] == corr) &\
(df['date']>=date_start) & (df['date']<=date_end) &\
(df['dow'].isin(weekday)) &\
(df['hour']>=tod_start) & (df['hour']<=tod_end)]
return select_df
def summarize_metrics(df, corr, group, avg, select, label, missing):
"""
return a summary of frequency, mean, mean difference, and count of missing values
Keyword arguments:
df -- dataframe to summarize
corr -- corridor name
group -- dataframe column name used to group and summarize data
avg -- mean value derived from rtdap_avg(), used calculate mean diff
select -- dateframe column name to calculate mean
label -- name for values being calculate (ie Speed, Volumne, Time etc)
missing -- dataframe column name of missing values
"""
df['freq'] = 1
df_groupby = df.groupby(group).agg({
'freq': 'count',
select: np.mean,
missing: sum
}).reset_index()
df_groupby.loc[:,
'Mean Diff'] = (avg -
df_groupby[select]) / df_groupby[select]
df_groupby.loc[:, group] = label
df_groupby.columns = [
corr, 'Frequency', 'Mean', 'Missing Values', 'Mean Diff'
]
df_groupby = df_groupby.set_index(corr)
return df_groupby[['Frequency', 'Mean', 'Mean Diff', 'Missing Values']]
def vbar_chart(full_df, df):
"""
returns bokeh horizontal barchart representing mean % diff
Keyword arguments:
df -- dataframe to derive content of barchart
col -- column name for values to diplay in graph
"""
df_avg = full_df.groupby('FieldDeviceID').agg({'avgSpeed': np.mean})
df_select = df.groupby('FieldDeviceID').agg({'avgSpeed': np.mean})
df_select.columns = ['speed']
diff = df_avg.merge(df_select,
how='left',
left_index=True,
right_index=True).fillna(0)
diff_no_zero = diff.loc[(diff['avgSpeed'] > 0) & (diff['speed'] > 0)]
diff_no_zero['speed_difference'] = diff_no_zero[
'avgSpeed'] - diff_no_zero['speed']
diff_no_zero = diff_no_zero.reset_index()
diff_no_zero['bins'] = pd.cut(diff_no_zero['speed_difference'],
bins=list(range(-20, 22)),
labels=list(range(-20, 22))[:-1])
source = ColumnDataSource(
data=diff_no_zero.groupby('bins').agg({
'speed_difference': 'count'
}).reset_index())
p = figure(plot_width=1000,
plot_height=150,
title="Speed Difference Distribution",
toolbar_location="above")
p.vbar(x='bins',
top='speed_difference',
width=1,
color='navy',
alpha=0.5,
source=source)
#p.yaxis.visible = False
#p.xaxis.formatter = NumeralTickFormatter(format="0.f%")
p.xgrid.visible = False
p.ygrid.visible = False
#p.background_fill_color = None
p.background_fill_alpha = 0.5
p.border_fill_color = None
return p
def hbar_chart(df, col):
"""
returns bokeh horizontal barchart representing mean % diff
Keyword arguments:
df -- dataframe to derive content of barchart
col -- column name for values to diplay in graph
"""
df_src = df[[col]]
df_src['type'] = df_src.index
df_src['order'] = 0
df_src['order'] = np.where(df_src.index == 'Speed', 2, df_src['order'])
df_src['order'] = np.where(df_src.index == 'Occupancy', 1,
df_src['order'])
df_src['order'] = np.where(df_src.index == 'Volume', 0,
df_src['order'])
df_src['color'] = '#C0C0C0'
df_src['color'] = np.where(df_src['Mean Diff'] < -.05, '#FF0000',
df_src['color'])
df_src['color'] = np.where(df_src['Mean Diff'] > .05, '#008000',
df_src['color'])
source = ColumnDataSource(data=df_src.sort_values(by='order'))
hover = HoverTool(tooltips=[
("Corridor Attribute", "@type"),
("% Difference", "@{%s}" % (col) + '{%0.2f}'),
])
tools = ['reset', 'save', hover]
p = figure(plot_width=400,
plot_height=175,
toolbar_location="above",
title='Mean Difference',
tools=tools)
p.hbar(y='order',
height=0.5,
left=0,
fill_color='color',
line_color=None,
right=col,
color="navy",
source=source)
p.yaxis.visible = False
p.xaxis.formatter = NumeralTickFormatter(format="0.f%")
p.xgrid.visible = False
p.ygrid.visible = False
#p.background_fill_color = None
p.background_fill_alpha = 0.5
p.border_fill_color = None
return source, p
def scatter_plot(title_text):
rng = np.random.RandomState(0)
x = rng.randn(100)
y = rng.randn(100)
source = ColumnDataSource(data=dict(
x=rng.randn(100),
y=rng.randn(100),
desc=['A', 'b', 'C', 'd', 'E'] * 20,
))
hover = HoverTool(tooltips=[
("index", "$index"),
("(x,y)", "($x, $y)"),
("desc", "@desc"),
])
p = figure(plot_width=300,
plot_height=250,
tools=[hover, 'box_select'],
toolbar_location="above",
title=title_text)
p.circle('x', 'y', size=5, source=source)
#p.background_fill_color = None
p.background_fill_alpha = 0.5
p.border_fill_color = None
return p
"""
return selection tab contents
Keyword arguments:
rtdap_data - dataframe containing rtdap vds detail data
"""
#-----------------------------------------------------------------------------------------------------------------
#submit_selection -- Data Selection Update Function
def submit_selection():
"""
python callback to update table and visual content based on
user selections in the data review panel
"""
avgs_speed = rtdap_avg(rtdap_data, corridor_select.value, 'avgSpeed')
avgs_occ = rtdap_avg(rtdap_data, corridor_select.value, 'avgOccupancy')
avgs_volume = rtdap_avg(rtdap_data, corridor_select.value, 'avgVolume')
filtered_data = filter_selection(rtdap_data, corridor_select.value,
str(date_picker_start.value),
str(date_picker_end.value),
day_of_week.value, time_of_day.value)
speed = summarize_metrics(filtered_data, corridor_select.value,
'corridor', avgs_speed, 'avgSpeed', 'Speed',
'missing_speed')
occ = summarize_metrics(filtered_data, corridor_select.value,
'corridor', avgs_occ, 'avgOccupancy',
'Occupancy', 'missing_occ')
volume = summarize_metrics(filtered_data, corridor_select.value,
'corridor', avgs_volume, 'avgVolume',
'Volume', 'missing_vol')
summary_df = speed.append(occ)
summary_df = summary_df.append(volume)
summary_title.text = "<h1>" + corridor_select.value + " Summary</h1>"
summary_df_tbl = summary_df.copy()
summary_df_tbl['Mean Diff'] = summary_df_tbl['Mean Diff'] * 100
summary_df_tbl = summary_df_tbl.reset_index()
summary_table.text = str(
summary_df_tbl.fillna(0).to_html(
index=False,
formatters=[
str, '{:20,}'.format, '{:20,.1f}'.format,
'{:20,.1f}%'.format, '{:20,}'.format
],
classes=["w3-table", "w3-hoverable", "w3-small"]))
if len(summary_df) > 0:
new_df = summary_df.fillna(0)
bar_viz_new = hbar_chart(new_df, 'Mean Diff')[0]
bar_viz_src.data.update(bar_viz_new.data)
#-----------------------------------------------------------------------------------------------------------------
#Data Review Panel
panel_title = Div(
text="Data Review",
css_classes=["panel-heading", "text-center", "w3-text-white"])
panel_text = Div(
text=
"""Lorem Ipsum is simply dummy text of the printing and typesetting industry.
Lorem Ipsum has been the industry's standard dummy text ever since the 1500s,
when an unknown printer took a galley of type and scrambled it to make a type
specimen book.""",
css_classes=["panel-content", "w3-text-white"])
#Panel Buttons
corridor_select = Select(
options=rtdap_data['corridor'].drop_duplicates().values.tolist(),
title='Corridor:',
height=60,
value='Dan Ryan Express Lane',
css_classes=["panel-content"])
date_picker_start = DatePicker(min_date=date(2015, 1, 1),
max_date=date(2018, 12, 31),
css_classes=["panel-content"],
title="Start Date:",
height=60,
value=date(2015, 12, 31))
date_picker_end = DatePicker(min_date=date(2015, 1, 1),
max_date=date(2018, 12, 31),
css_classes=["panel-content"],
title="End Date:",
height=60,
value=date(2017, 12, 31))
time_of_day = RangeSlider(start=1,
end=8,
step=1,
value=(1, 2),
title="Time of Day:",
bar_color="black",
css_classes=["panel-content"])
tod_description = Div(
text="""Time of Day Categories:<br>
<ol>
<li>8pm-6am</li>
<li>6pm-7am</li>
<li>7am-8am</li>
<li>9am-10am</li>
<li>10am-2pm</li>
<li>2pm-4pm</li>
<li>4pm-6pm</li>
<li>6pm-8pm</li>
</ol>""",
css_classes=["panel-content", "caption", "w3-text-white"])
day_of_week = Select(options=['All'] +
rtdap_data['dow'].drop_duplicates().values.tolist(),
title="Day of Week:",
css_classes=["panel-content"],
height=60,
value="All")
select_data = Button(label="Select Subset",
css_classes=["panel-content"],
height=60)
select_data.on_click(submit_selection)
#-----------------------------------------------------------------------------------------------------------------
#-----------------------------------------------------------------------------------------------------------------
#Create initial content
avgs_speed = rtdap_avg(rtdap_data, corridor_select.value, 'avgSpeed')
avgs_occ = rtdap_avg(rtdap_data, corridor_select.value, 'avgOccupancy')
avgs_volume = rtdap_avg(rtdap_data, corridor_select.value, 'avgVolume')
filtered_data = filter_selection(rtdap_data, corridor_select.value,
str(date_picker_start.value),
str(date_picker_end.value),
day_of_week.value, time_of_day.value)
speed = summarize_metrics(filtered_data, corridor_select.value, 'corridor',
avgs_speed, 'avgSpeed', 'Speed', 'missing_speed')
occ = summarize_metrics(filtered_data, corridor_select.value, 'corridor',
avgs_occ, 'avgOccupancy', 'Occupancy',
'missing_occ')
volume = summarize_metrics(filtered_data, corridor_select.value,
'corridor', avgs_volume, 'avgVolume', 'Volume',
'missing_vol')
summary_df = speed.append(occ)
summary_df = summary_df.append(volume)
summary_df_tbl = summary_df.copy()
summary_df_tbl['Mean Diff'] = summary_df_tbl['Mean Diff'] * 100
summary_df_tbl = summary_df_tbl.reset_index()
summary_title = Div(text="<h1>" + corridor_select.value + " Summary</h1>",
width=2000,
css_classes=["w3-panel", "w3-white"])
summary_table = Div(text="", width=550, height=150)
summary_table.text = str(
summary_df_tbl.fillna(0).to_html(
index=False,
formatters=[
str, '{:20,}'.format, '{:20,.1f}'.format, '{:20,.1f}%'.format,
'{:20,}'.format
],
classes=["w3-table", "w3-hoverable", "w3-small"]))
line = Div(text="<hr>", css_classes=["w3-container"], width=1000)
#-----------------------------------------------------------------------------------------------------------------
#-----------------------------------------------------------------------------------------------------------------
#Create initial graphics
'''#horizontal bar chart
p = figure(plot_width=300, plot_height=100)
p.hbar(y=[1, 2, 3], height=0.5, left=0,
right=[1.2, 2.5, 3.7], color="navy")
p.yaxis.visible = False
p.xaxis.formatter = NumeralTickFormatter(format="0.0f%")'''
bar_viz = hbar_chart(summary_df.fillna(0), 'Mean Diff')
bar_viz_src = bar_viz[0]
bar_viz_chart = bar_viz[1]
volume_scatter = scatter_plot('Volumes')
time_scatter = scatter_plot('Time')
corr_df = rtdap_data.loc[rtdap_data['corridor'] == corridor_select.value]
speed_diff_vbar = (vbar_chart(corr_df, filtered_data))
occ_diff_vbar = (vbar_chart(corr_df, filtered_data))
volume_diff_vbar = (vbar_chart(corr_df, filtered_data))
base_map = make_base_map(map_width=450,
map_height=960,
xaxis=None,
yaxis=None,
xrange=(-9990000, -9619944),
yrange=(5011119, 5310000),
plot_tools="pan,wheel_zoom,reset,save")
select_content = row(
#PANEL
column(panel_title,
panel_text,
corridor_select,
date_picker_start,
date_picker_end,
day_of_week,
time_of_day,
tod_description,
select_data,
height=1000,
css_classes=["w3-sidebar", "w3-bar-block", "w3-darkgrey"]),
column(css_classes=["w3-col"], width=275),
#CONTENT
column(summary_title,
row(
Spacer(width=20),
column(
Spacer(height=10),
row(summary_table,
Spacer(width=50),
bar_viz_chart,
css_classes=["w3-panel", "w3-white", "w3-card-4"]),
Spacer(height=10),
row(volume_scatter,
Spacer(width=10),
time_scatter,
css_classes=["w3-panel", "w3-white", "w3-card-4"],
width=650),
Spacer(height=10),
row(column(speed_diff_vbar, occ_diff_vbar,
volume_diff_vbar),
css_classes=["w3-panel", "w3-white", "w3-card-4"],
width=1050),
),
row(
Spacer(width=20),
column(
Spacer(height=10),
column(base_map,
css_classes=[
"w3-panel", "w3-white", "w3-card-4"
],
width=500)))),
css_classes=["w3-container", "w3-row-padding"]),
css_classes=["w3-container", "w3-light-grey"],
width=2000,
height=1200)
return select_content
# hitter seartch
hfirstname = TextInput(placeholder='First Name',
title='Batter Search',
width=200)
hlastname = TextInput(placeholder='Last Name', title=' ', width=200)
hsearch = Button(label='Search', button_type='primary', width=200)
hitterselect = Select(title='Batter', options=[])
hitterselect.on_change('value', selection_update)
# button actions
psearch.on_click(pitcher)
hsearch.on_click(hitter)
# date selectors
start_date = DatePicker(title='Start Date',
min_date=datetime(2017, 4, 1),
max_date=datetime.now(),
value=datetime.now())
end_date = DatePicker(title='End Date',
min_date=datetime(2017, 4, 1),
max_date=datetime.now(),
value=datetime.now())
run_button = Button(label='Run', button_type='primary', width=20)
run_button.on_click(retrieve_data)
# define the figure that shows each pitch location
plot = figure(width=400, height=500, title='Pitches')
# add strike zone lines
plot.line(x='x', y='top', line_width=3, color='grey', source=strike_zone_cds)
plot.line(x='x',
y='bottom',
# linacs = list(set(data['Radiation Dev']))
# linacs.sort()
# linacs.insert(0, 'All')
# linacs.append('None')
# select_linac = {key: Select(title='Linac %s:' % key, value='All', options=['All'], width=250) for key in [1, 2]}
# select_linac[2].value = 'None'
# select_linac[1].on_change('value', plot.update_source)
# select_linac[2].on_change('value', plot.update_source)
avg_len_input = TextInput(title='Avg. Len:', value='10', width=100)
avg_len_input.on_change('value', plot.update_source)
percentile_input = TextInput(title='Percentile:', value='90', width=100)
percentile_input.on_change('value', plot.update_source)
start_date_picker = DatePicker(title='Start Date:', value=plot.x[0])
end_date_picker = DatePicker(title='End Date:', value=plot.x[-1])
start_date_picker.on_change('value', plot.update_source)
end_date_picker.on_change('value', plot.update_source)
gamma_options = ['5.0%/3.0mm', '3.0%/3.0mm', '3.0%/2.0mm', 'Any']
checkbox_button_group = CheckboxButtonGroup(labels=gamma_options, active=[3])
checkbox_button_group.on_change('active', plot.update_source)
text = {key: Div() for key in [1, 2]}
plot.update_source(None, None, None)
layout = column(row(select_y, avg_len_input, percentile_input),
row(start_date_picker, end_date_picker),
row(Div(text='Gamma Criteria: '), checkbox_button_group),
# bokeh file
output_file("prediction_ui.html")
# test for paragraph at the top of the page. stored in HTML.
description = Div(text=open(join(dirname(__file__), "description.html")).read(), width=800)
''' --- Left side of page: Onset and demographic data --- '''
## Onset Panel
# date onset
def dt_date_onset_handler(attr, old, new):
print(type(old))
print(type(new))
print('old was {} and new is {}'.format(old,new))
dt_date_onset=DatePicker(title="Date of ALS Onset",
min_date=date(1990,1,1),
max_date=date.today(),
value=date(2015,3,1))
dt_date_onset.on_change("value", dt_date_onset_handler)
# subject id
def subject_id_handler(attr, old, new):
print("Previous label: " + tpye(old))
print("Updated label: " + new)
subject_id = TextInput(title="Subject ID:", value="ref number")
#subject_id.on_change("value", subject_id_handler)
# age at onset
def age_onset_handler(attr, old, new):
print("Previous label: " + type(old))
print("Updated label: " + new)
age_onset = TextInput(title="Age at ALS Onset (years)", value="")
hover_line_alpha=1.0)
hover = HoverTool(tooltips=[('Date', '$x{%F}'), ('Frequency', '$y{0}')],
formatters={'$x': 'datetime'},
mode='mouse',
line_policy='nearest')
plot.add_tools(hover)
data_table = DataTable(source=table_source, columns=columns, width=600)
# Set up widgets
phrase = TextInput(title="Graph these comma-separated phrases:", value='')
datepicker1 = DatePicker(title="Start Date:",
min_date=datetime.datetime(2018, 4, 1),
max_date=datetime.datetime.now(),
value=datetime.datetime(2018, 4, 15))
datepicker2 = DatePicker(title="End Date:",
min_date=datetime.datetime(2018, 4, 16),
max_date=datetime.datetime.now(),
value=datetime.datetime.now())
checkbox_group = CheckboxGroup(labels=["Normalize"])
button = Button(label="Update Corpus")
def update_data(attrname, old, new):
d1 = datetime.datetime.combine(datepicker1.value, datetime.time())
d2 = datetime.datetime.combine(datepicker2.value, datetime.time())
hex_filter_no.title = "hex_filter by number"
else:
hex_filter_no.visible = True
hex_filter_no.title = "hex_filter by threshold"
hex_filter_select.on_change('value', hex_filter_callback)
#p.toolbar.active_inspect
#2019-08-12 00:00:01
Min_date = datetime.date(2020, 3, 16)
Max_date = datetime.date(2020, 3, 16)
drs_start = DatePicker(title='Date',
min_date=datetime.date(2020, 3, 15),
max_date=datetime.date(2020, 6, 1),
value=Min_date)
drs_start_hour = Select(title='Hour of the day',
value='0',
options=[str(x) for x in list(range(0, 24))])
drs_end = DatePicker(title='Date',
min_date=datetime.date(2020, 3, 15),
max_date=datetime.date(2020, 6, 1),
value=Max_date)
drs_end_hour = Select(title='Hour of the day',
value='6',
options=[str(x) for x in list(range(0, 24))])
dre_p1 = Panel(child=row(drs_start, drs_start_hour, width=280), title="From")
dre_p1 = Tabs(tabs=[dre_p1])
import os.path
from bokeh.layouts import row,column,layout,widgetbox
from bokeh.models import Button,ColumnDataSource,HoverTool,Span
from bokeh.models.widgets import TextInput,DataTable,TableColumn,Div,DateFormatter,NumberFormatter,Paragraph,DatePicker,CheckboxButtonGroup,Select
from bokeh.plotting import figure,curdoc
from datetime import datetime as DT
from datetime import date, timedelta
log.getLogger().setLevel(log.WARN)
# page elements
ww=700
hh=70
searchBox = TextInput(title="filter news", value='',width=ww,height=hh)
datePick1=DatePicker(title='start date',min_date=date(2005,1,1),max_date=date(2007,12,31),value=date(2007,2,14))
datePick2=DatePicker(title='end date',min_date=date(2005,1,1),max_date=date(2007,12,31),value=date(2007,2,18))
#def setDate(attr,old,new): inputDate=new
#datePick1.on_change('value',setDate)
submit1 = Button(label='Submit',width=20,height=hh)
submit2 = Button(label='Submit',width=20,height=hh)
exeBox = TextInput(title="function",width=ww,height=hh,css_classes=['itext'])
errBox = Div(text="",width=ww//3,height=hh,style={'overflow-y':'scroll','height':'150px'})
emptyResult = data=dict(headline=[],prevHeadline=[],symbols=[],gmtstamp=[],ret=[])
searchResultTable = ColumnDataSource(data=emptyResult)
searchResultColumns = [
TableColumn(field='prevHeadline',title='Past Headline',width=400),
TableColumn(field='headline',title='Headline',width=400),
TableColumn(field='sym',title='Symbols',width=70),
TableColumn(field='gmtstamp',title='GMT Timestamp',formatter=DateFormatter(format="%F %T"),width=180),
TableColumn(field='date',title='Date',formatter=DateFormatter(format="%Y-%m-%d"),width=120),
def set_up_widgets(p, source, source1, source2, source3, source4, source5, df,
all_dates, text_dsn):
# print("set_up_widgets")
'''Set up widgets needed after an initial dsn is entered'''
dsn = text_dsn.value
# Set up widgets
text_title = TextInput(title="Title:", value="{} Data".format(dsn))
text_save = TextInput(title="Save As:", value=dsn)
min_day = date_of_file(all_dates[0])
max_day = date_of_file(all_dates[-1])
plus_one = max_day + timedelta(days=1)
calendar = DatePicker(title="Day:",
value=date(plus_one.year, plus_one.month,
plus_one.day),
max_date=max_day,
min_date=min_day)
button = Button(label="Update", button_type="success")
# columns = [
# TableColumn(field="day", title="Date", formatter=DateFormatter(format="%m/%d/%Y")),
# TableColumn(field="hr", title="Avg HR", formatter=NumberFormatter(format="0.0")),
# TableColumn(field="o2", title="Avg O2", formatter=NumberFormatter(format="0.0")),
# TableColumn(field="temp", title="Avg Temp", formatter=NumberFormatter(format="0.0"))
# ]
# table_title = Div(text="""Daily Averages:""", width=200)
# daily_avg = get_daily_avgs(df_full)
# data = {
# 'day' : daily_avg.index,
# 'hr' : daily_avg.heart_rate_avg,
# 'o2' : daily_avg.oxygen_avg,
# 'temp' : daily_avg.skin_temperature
# }
# table_source = ColumnDataSource(data=data)
# data_table = DataTable(source=table_source, columns=columns, width=280, height=180, index_position=None)
# export_png(data_table, filename="table.png")
# save_table = Button(label='Save Daily Averages Table', button_type="primary")
# Set up callbacks
def update_title(attrname, old, new):
p.title.text = text_title.value
def update_save(attrname, old, new):
p.tools[0].save_name = text_save.value
def update():
text_dsn.value = text_dsn.value.strip(" ") # Get rid of extra space
update_data(p,
source,
source1,
source2,
source3,
source4,
source5,
df,
text_dsn.value,
all_dates,
date=str(calendar.value))
# Title/save update dsn
text_title.value = text_dsn.value + " Data"
text_save.value = text_dsn.value
# def save():
# export_png(data_table, filename="{}_averages.png".format(text_dsn.value))
text_title.on_change('value', update_title)
text_save.on_change('value', update_save)
button.on_click(update)
# save_table.on_click(save)
# Set up layouts and add to document
inputs = widgetbox(text_title, text_save, calendar,
button) #, table_title, data_table, save_table)
curdoc().add_root(row(inputs, p, width=1300))
def blockminer_tab(page_width):
# source for top N table
topN_src = ColumnDataSource(
data=dict(percentage=[], address=[], block_count=[]))
class This_tab(Mytab):
def __init__(self, table, cols, dedup_cols):
Mytab.__init__(self, table, cols, dedup_cols)
self.table = table
self.df2 = None
self.key_tab = 'blockminer'
self.n = 20
# ------- DIVS setup begin
self.page_width = page_width
txt = """<hr/>
<div style="text-align:center;width:{}px;height:{}px;
position:relative;background:black;margin-bottom:200px">
<h1 style="color:#fff;margin-bottom:300px">{}</h1>
</div>""".format(self.page_width, 50, 'Welcome')
self.notification_div = {
'top': Div(text=txt, width=self.page_width, height=20),
'bottom': Div(text=txt, width=self.page_width, height=10),
}
self.section_divider = '-----------------------------------'
self.section_headers = {}
# ----- UPDATED DIVS END
# ---------------------- DIVS ----------------------------
def section_header_div(self,
text,
html_header='h2',
width=600,
margin_top=150,
margin_bottom=-150):
text = """<div style="margin-top:{}px;margin-bottom:-{}px;"><{} style="color:#4221cc;">{}</{}></div>""" \
.format(margin_top, margin_bottom, html_header, text, html_header)
return Div(text=text, width=width, height=15)
def load_this_data(self, start_date, end_date):
end_date = datetime.combine(end_date, datetime.min.time())
start_date = datetime.combine(start_date, datetime.min.time())
logger.warning('load_data start date:%s', start_date)
logger.warning('load_data end date:%s', end_date)
# load only mined blocks and remove the double entry
supplemental_where = "AND update_type = 'mined_block' AND amount >= 0"
self.df_load(start_date,
end_date,
supplemental_where=supplemental_where,
cols=['address', 'amount', 'block_time'])
logger.warning('after load:%s', self.df.head(30))
return self.prep_dataset(start_date, end_date)
def prep_dataset(self, start_date, end_date):
try:
logger.warning("prep dataset start date:%s", start_date)
self.df1 = self.df1[['address', 'block_time']]
self.df1['address'] = self.df1['address']\
.map(self.poolname_verbose_trun)
self.df1 = self.df1.groupby(['address'
]).agg({'block_time': 'count'})
self.df1 = self.df1.reset_index()
self.df1 = self.df1.rename(
columns={'block_time': 'block_count'})
self.df1['percentage'] = 100*self.df1['block_count']\
/self.df1['block_count'].sum()
self.df1['percentage'] = self.df1['percentage'].map(
lambda x: round(x, 1))
self.df1 = self.df1.reset_index()
logger.warning("topN column:%s", self.df1.columns.tolist())
#logger.warning('END prep dataset DF1:%s', self.df1.head())
return self.df1.hvplot.bar(
'address',
'block_count',
rot=90,
height=600,
width=self.page_width,
title='# of blocks mined by miner address',
hover_cols=['percentage'])
except Exception:
logger.error('prep dataset:', exc_info=True)
def view_topN(self):
logger.warning("top n called:%s", self.n)
# change n from string to int
try:
#table_n = df1.hvplot.table(columns=['address','percentage'],
#title=title, width=400)
logger.warning('top N:%s', self.n)
df2 = self.df1.nlargest(self.n, 'percentage')
df2 = df2.compute()
logger.warning('in view top n :%s', df2.head(10))
new_data = dict(percentage=df2.percentage.tolist(),
address=df2.address.tolist(),
block_count=df2.block_count.tolist())
topN_src.stream(new_data, rollover=self.n)
columns = [
TableColumn(field="address", title="Address"),
TableColumn(field="percentage", title="percentage"),
TableColumn(field="block_count", title="# of blocks")
]
table_n = DataTable(source=topN_src,
columns=columns,
width=300,
height=600)
gc.collect()
return table_n
except Exception:
logger.error('view_topN:', exc_info=True)
def set_n(self, n):
if isinstance(n, int):
pass
else:
try:
self.n = int(n)
except Exception:
logger.error('set_n', exc_info=True)
# ####################################################
# UTILITY DIVS
def results_div(self, text, width=600, height=300):
div = Div(text=text, width=width, height=height)
return div
def title_div(self, text, width=700):
text = '<h2 style="color:green;">{}</h2>'.format(text)
return Div(text=text, width=width, height=20)
def notification_updater_2(self, text):
self.notification_div.text = '<h3 style="color:red">{}</h3>'.format(
text)
def spacing_div(self, width=20, height=100):
return Div(text='', width=width, height=height)
def spacing_paragraph(self, width=20, height=100):
return Paragraph(text='', width=width, height=height)
def update(attrname, old, new):
this_tab.notification_updater(
"Calculations underway. Please be patient")
stream_start_date.event(start_date=datepicker_start.value)
stream_end_date.event(end_date=datepicker_end.value)
this_tab.set_n(topN_select.value)
this_tab.view_topN()
this_tab.notification_updater("ready")
# update based on selected top n
def update_topN():
this_tab.notification_updater("Calculations in progress! Please wait.")
logger.warning('topN selected value:%s', topN_select.value)
this_tab.set_n(topN_select.value)
this_tab.view_topN()
this_tab.notification_updater("ready")
try:
# create class and get date range
cols = ['address', 'block_timestamp', 'block_time']
this_tab = This_tab('account_ext_warehouse', cols, [])
#STATIC DATES
first_date_range = "2018-04-23 00:00:00"
first_date_range = datetime.strptime(first_date_range,
"%Y-%m-%d %H:%M:%S")
last_date_range = datetime.now().date()
last_date = last_date_range
first_date = datetime_to_date(last_date - timedelta(days=60))
# STREAMS Setup
# date comes out stream in milliseconds
stream_start_date = streams.Stream.define('Start_date',
start_date=first_date)()
stream_end_date = streams.Stream.define('End_date',
end_date=last_date)()
# create a text widget for top N
topN_select = Select(title='Top N',
value=str(this_tab.n),
options=menu)
datepicker_start = DatePicker(title="Start",
min_date=first_date_range,
max_date=last_date_range,
value=first_date)
datepicker_end = DatePicker(title="End",
min_date=first_date_range,
max_date=last_date_range,
value=last_date)
# ALL MINERS.
# --------------------- ALL MINERS ----------------------------------
hv_bar_plot = hv.DynamicMap(
this_tab.load_this_data,
streams=[stream_start_date, stream_end_date],
datashade=True)
renderer = hv.renderer('bokeh')
bar_plot = renderer.get_plot(hv_bar_plot)
# --------------------- TOP N MINERS -----------------------------------
# set up data source for the ton N miners table
this_tab.view_topN()
columns = [
TableColumn(field="address", title="Address"),
TableColumn(field="percentage", title="percentage"),
TableColumn(field="block_count", title="# of blocks")
]
topN_table = DataTable(source=topN_src,
columns=columns,
width=400,
height=600)
# add callbacks
datepicker_start.on_change('value', update)
datepicker_end.on_change('value', update)
topN_select.on_change("value", lambda attr, old, new: update_topN())
download_button = Button(label='Save Table to CSV',
button_type="success")
download_button.callback = CustomJS(
args=dict(source=topN_src),
code=open(
join(dirname(__file__),
"../../../static/js/topN_download.js")).read())
# put the controls in a single element
controls = WidgetBox(datepicker_start, datepicker_end, download_button,
topN_select)
# create the dashboards
grid = gridplot([[this_tab.notification_div['top']],
[Spacer(width=20, height=70)], [
topN_table,
controls,
], [bar_plot.state]])
# Make a tab with the layout
tab = Panel(child=grid, title='miners: blocks')
return tab
except Exception:
logger.error("Blockminer", exc_info=True)
return tab_error_flag('miners: blocks')
