def plot_similarity(self, result, parameter):
source1 = column_source(result, "FDA")
source2 = column_source(result, "PPI")
source3 = column_source(result, "MACRO")
source4 = column_source(result, "NP")
source5 = column_source(result, "FDA PEP")
source6 = column_source(result, "linear")
source7 = column_source(result, "cyclic")
hover = HoverTool(tooltips=[
("sim", "$x"),
("CDF", "$y"),
])
p = figure(
title="CDF of Tanimoto Similarity, based on: " + parameter[0],
x_axis_label="Similarity",
y_axis_label="Cumulative Distribution Function",
x_range=(0, 1),
y_range=(0, 1),
tools=[hover],
plot_width=1000,
plot_height=800,
)
p.add_tools(LassoSelectTool(), ZoomInTool(), ZoomOutTool(), SaveTool(),
PanTool())
FDA_plot = p.line(x="x",
y="y",
source=source1,
color="darkslateblue",
line_width=3)
PPI_plot = p.line(x="x",
y="y",
source=source2,
color="yellowgreen",
line_width=3)
MACRO_plot = p.line(x="x",
y="y",
source=source3,
color="lightsteelblue",
line_width=3)
NP_plot = p.line(x="x",
y="y",
source=source4,
color="olive",
line_width=3)
PEP_FDA_plot = p.line(x="x",
y="y",
source=source5,
color="darkslategray",
line_width=3)
LIN_plot = p.line(x="x",
y="y",
source=source6,
color="teal",
line_width=3)
CYC_plot = p.line(x="x",
y="y",
source=source7,
color="mediumvioletred",
line_width=3)
legend = Legend(
items=[
("FDA", [FDA_plot]),
("PPI", [PPI_plot]),
("MACRO", [MACRO_plot]),
("NP", [NP_plot]),
("PEP FDA", [PEP_FDA_plot]),
("LIN", [LIN_plot]),
("CYC", [CYC_plot]),
],
location="center",
orientation="vertical",
click_policy="hide",
)
p.add_layout(legend, place="right")
p.xaxis.axis_label_text_font_size = "20pt"
p.yaxis.axis_label_text_font_size = "20pt"
p.xaxis.axis_label_text_color = "black"
p.yaxis.axis_label_text_color = "black"
p.xaxis.major_label_text_font_size = "18pt"
p.yaxis.major_label_text_font_size = "18pt"
p.title.text_font_size = "22pt"
return p
'@y1')])
hover_pie = HoverTool(tooltips=[('Percentage', '@ratio')])
select_day = Select(options=[
'Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday',
'Saturday'
],
value='Monday',
title='Day of the Week')
chosen_tools = [
hover,
ResetTool(),
BoxSelectTool(),
SaveTool(),
PanTool(),
UndoTool(),
RedoTool(),
ZoomInTool(),
ZoomOutTool()
]
#data
data = pd.read_csv(r'~/Desktop/yelp/yelp_business.csv',
index_col='business_id')
hour = pd.read_csv(r'~/Desktop/yelp/yelp_business_hours.csv',
index_col='business_id')
whole = pd.merge(data, hour, left_index=True, right_index=True)
def large_plot(n: int) -> Tuple[Model, Set[Model]]:
from bokeh.models import (
BoxSelectTool,
BoxZoomTool,
Column,
ColumnDataSource,
DataRange1d,
GlyphRenderer,
Grid,
Line,
LinearAxis,
PanTool,
Plot,
ResetTool,
SaveTool,
WheelZoomTool,
ZoomInTool,
ZoomOutTool,
)
col = Column()
objects: Set[Model] = {col}
for i in range(n):
source = ColumnDataSource(data=dict(x=[0, i + 1], y=[0, i + 1]))
xdr = DataRange1d()
ydr = DataRange1d()
plot = Plot(x_range=xdr, y_range=ydr)
xaxis = LinearAxis()
plot.add_layout(xaxis, "below")
yaxis = LinearAxis()
plot.add_layout(yaxis, "left")
xgrid = Grid(dimension=0)
plot.add_layout(xgrid, "center")
ygrid = Grid(dimension=1)
plot.add_layout(ygrid, "center")
tickers = [
xaxis.ticker, xaxis.formatter, yaxis.ticker, yaxis.formatter
]
glyph = Line(x='x', y='y')
renderer = GlyphRenderer(data_source=source, glyph=glyph)
plot.renderers.append(renderer)
pan = PanTool()
zoom_in = ZoomInTool()
zoom_out = ZoomOutTool()
wheel_zoom = WheelZoomTool()
box_zoom = BoxZoomTool()
box_select = BoxSelectTool()
save = SaveTool()
reset = ResetTool()
tools = [
pan, zoom_in, zoom_out, wheel_zoom, box_zoom, box_select, save,
reset
]
plot.add_tools(*tools)
col.children.append(plot)
objects |= set([
xdr,
ydr,
xaxis,
xaxis.major_label_policy,
yaxis,
yaxis.major_label_policy,
xgrid,
ygrid,
renderer,
renderer.view,
renderer.view.filter,
glyph,
source,
source.selected,
source.selection_policy,
plot,
plot.x_scale,
plot.y_scale,
plot.toolbar,
plot.title,
box_zoom.overlay,
box_select.overlay,
] + tickers + tools)
return col, objects
def generate_plot(self):
"""Iteratively reformats and plots self.data for each cell+model combo.
TODO: Finish this doc
"""
plots = []
modelnames = self.data.index.levels[0].tolist()
# Iterate over a list of tuples representing all unique pairs of models.
for pair in list(itertools.combinations(modelnames,2)):
tools = [
PanTool(), SaveTool(), WheelZoomTool(),
ResetTool(), self.create_hover(),
]
modelX = pair[0]
modelY = pair[1]
dataX = self.data.loc[modelX]
dataY = self.data.loc[modelY]
# Only necessary b/c bokeh's $index identifier for HoverTool()
# is pulling an integer index despite data being indexed by cellid.
# If cellid strings can be pulled from index instead,
# this code will no longer be needed.
cells = []
cellsX = list(set(dataX.index.values.tolist()))
cellsY = list(set(dataY.index.values.tolist()))
if self.fair:
# cellsX and cellsY should be the same if fair was checked
cells = cellsX
if cells != cellsY:
self.script = 'Problem with form_data_array:'
self.div = 'Model x: ' + modelX + 'and Model y: ' + modelY\
+ ' applied to different cells despite fair check.'
return
else:
# If fair wasn't checked, use the longer list to avoid errors.
if len(cellsX) >= len(cellsY):
cells = cellsX
else:
cells = cellsY
x_mean = np.mean(dataX[self.measure[0]])
x_median = np.median(dataX[self.measure[0]])
y_mean = np.mean(dataY[self.measure[0]])
y_median = np.median(dataY[self.measure[0]])
x_label = (
"{0}, mean: {1:5.4f}, median: {2:5.4f}"
.format(modelX, x_mean, x_median)
)
y_label = (
"{0}, mean: {1:5.4f}, median: {2:5.4f}"
.format(modelY, y_mean, y_median)
)
data = pd.DataFrame({
'x_values':dataX[self.measure[0]],
'y_values':dataY[self.measure[0]],
'cellid':cells,
})
dat_source = ColumnDataSource(data)
p = figure(
x_range=[0,1], y_range=[0,1],
x_axis_label=x_label, y_axis_label=y_label,
title=("{0}, prefix: {1}, suffix: {2}"
.format(self.measure[0], self.pre, self.suf)),
tools=tools, responsive=True,
toolbar_location=TOOL_LOC, toolbar_sticky=TOOL_STICK,
output_backend="svg"
)
glyph = Circle(
x='x_values', y='y_values', size=CIRCLE_SIZE,
fill_color=CIRCLE_FILL, fill_alpha=CIRCLE_ALPHA,
)
p.add_glyph(dat_source, glyph)
p.line([0,1], [0,1], line_width=1, color='black')
plots.append(p)
# If more than one plot was made (i.e. 2 or more models were selected),
# put them in a grid.
#if len(plots) == 1:
# singleplot = plots[0]
# self.script,self.div = components(singleplot)
# return
#elif len(plots) > 1:
grid = gridplot(
plots, ncols=GRID_COLS, responsive=True,
)
self.script,self.div = components(grid)
if not plots:
self.script, self.div = (
'Error, no plots to display.',
'Make sure you selected two models.'
)
def generate_plot(self):
"""TODO: write this doc."""
tools = [
PanTool(), SaveTool(), WheelZoomTool(),
ResetTool(),
]
x_values = []
y_values = []
std_errors = []
models = self.data.index.levels[0].tolist()
for model in models:
values = self.data[self.measure[0]].loc[model].values
mean = np.mean(values)
stderr = np.around(st.sem(values, nan_policy='omit'), 5)
n_parms = self.data['n_parms'].loc[model].values[0]
x_values.append(n_parms)
y_values.append(mean)
std_errors.append(stderr)
newData = pd.DataFrame.from_dict({
'stderr':std_errors, 'mean':y_values, 'n_parms':x_values,
'modelname':models,
})
# Drop any models with NaN values, since that means they had no
# performance data for one or more columns.
newData.dropna(axis=0, how='any', inplace=True)
if newData.size == 0:
self.script,self.div = (
"Error, no plot to display.",
"None of the models contained valid performance data."
)
return
dat_source = ColumnDataSource(newData)
p = figure(
tools=tools,
x_axis_label=("N Parms, model prefix: {0}, "
"suffix: {1}".format(self.pre, self.suf)),
y_axis_label=("Mean {0}, +/- Standard Error"
.format(self.measure[0])),
title="Mean {0} per Model vs Complexity".format(self.measure[0]),
output_backend="svg", sizing_mode='scale_width',
)
circles = Circle(
x='n_parms', y='mean', size=6, fill_color="navy",
fill_alpha=0.7,
)
circle_renderer = p.add_glyph(dat_source, circles)
hover = self.create_hover()
hover.renderers = [circle_renderer]
p.add_tools(hover)
#p.circle(x_values, y_values, size=6, color="navy", alpha=0.7)
error_bars_x = []
error_bars_y = []
for i, std in enumerate(std_errors):
error_bars_x.append([x_values[i], x_values[i]])
error_bars_y.append([y_values[i] - std, y_values[i] + std])
p.multi_line(
error_bars_x, error_bars_y, color="firebrick",
alpha=0.4, line_width=2,
)
# workaround to prevent title and toolbar from overlapping
grid = gridplot(
[p], ncols=GRID_COLS, sizing_mode='scale_width'
)
self.script, self.div = components(grid)
__author__ = "Anton Suchaneck"
__email__ = "*****@*****.**"
"""
TODO:
* check for NaN when sorting?
* tick formatter in coffeescript with percent
minauc:
* minauc could be generally on x=pos+a*tot, y=pos+b*tot (?) -> prec would still be slope, only recall would be less visible
* use ordered string values?
"""
heatmap_cmap = plt.cm.YlOrRd
DEFAULT_MAX_PTS = 10000
DEFAULT_BOKEH_TOOLS = [BoxZoomTool(), PanTool(), ResetTool(), SaveTool()]
def thin_out_idxs(x, max_len):
"""
:param x: sorted numpy array
:param max_len: number of elements to sample
:return: indices of selected elements
Returns at most max_len indices 0...len(x)-1 such that the subsampled x are evenly spread
"""
if len(x) <= max_len:
return np.arange(len(x))
val_min = x[0]
val_max = x[-1]
def main():
DATASET = "_jobsv1_all" # 10k jobs, but also some low quality ones
# OPT number of topics = ?
#DATASET = "_jobsv1_goodq" # 5k jobs, selecting only higher quality ones
# OPT number of topics = ?
#DATASET = "_stackv1_graphicdesignqa_sub20k" # 20k subsets of q'n'a from graphic design
# OPT number of topics = ?
import numpy as np
# < HAX numpy version
# save np.load
np_load_old = np.load
# modify the default parameters of np.load
np.load = lambda *a, **k: np_load_old(*a, allow_pickle=True, **k)
# call load_data with allow_pickle implicitly set to true
texts = np.load("data/texts" + DATASET + ".npz")['a']
data = np.load("data/documents" + DATASET + ".npz")['a']
titles = np.load("data/titles" + DATASET + ".npz")['a']
# restore np.load for future normal usage
np.load = np_load_old
# HAX numpy version />
plot_dir = "plots" + DATASET + "/"
import os
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
import pandas as pd
from pprint import pprint
import gensim
import gensim.corpora as corpora
from gensim.models import CoherenceModel
import pyLDAvis
import pyLDAvis.gensim # don't skip this
import matplotlib.pyplot as plt
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.ERROR)
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
### Settings:
METAOPT_num_of_topics = True
NAME_METAOPT_plot = plot_dir + "LDA_best_number_of_topics_"
LOAD_lda_model = False
LDA_tryMallet = False # doesn't have the same support as generic LDA tho - convert differently?
LDA_number_of_topics = 13 # Wait for metaopt!
CALC_coherence = True
VIZ_hist = True
VIZ_wordclouds = True
VIZ_TSNE = True
VIZ_html_interactive = True #SLOW and last
NAME_hist = plot_dir + "hist_topics.png"
NAME_tsne = plot_dir + "tsne.html"
NAME_wordclouds = plot_dir + "wordclouds_" # +i+.png
NAME_html_interactive = plot_dir + "LDA_Visualization.html" #"vis.html"
DEBUG_print_docs = False
DEBUG_print_topics = False
SAVE_doc2topic = False # don't really need
NAME_doc2topic = plot_dir + "doc2topic.csv"
SAVE_topic2docs = True
NAME_topic2docs = plot_dir + "topic2docs.csv"
SAVE_topic2num = True
NAME_topic2num = plot_dir + "topic2num.csv"
# GENSIM analysis
id2word = corpora.Dictionary(texts)
corpus = [id2word.doc2bow(text) for text in texts]
if DEBUG_print_docs:
print("document in vector format:", corpus[:1])
print("readable form:",
[[(id2word[id], freq) for id, freq in cp] for cp in corpus[:1]])
print()
if METAOPT_num_of_topics:
# Can take a long time to run.
topics_start = 5
topics_end = 15 # non exclusive
topics_step = 1
plot_name = NAME_METAOPT_plot + str(topics_start) + "TO" + str(
topics_end) + "BY" + str(topics_step) + ".png"
LDA_best_number_of_topics(id2word,
corpus,
texts,
topics_start,
topics_end,
topics_step,
mallet_lda=LDA_tryMallet,
plot_name=plot_name)
print("Analyze the results of the meta-optimalization ^^^")
assert False
# Build LDA model
print("Building/Loading LDA model (takes time)")
if LOAD_lda_model:
lda_model = gensim.models.LdaModel.load('data/model_LDAEXAMPLE.lda')
else:
if LDA_tryMallet:
# mallet LDA
mallet_path = '../mallet-2.0.8/bin/mallet' # update this path
lda_model = gensim.models.wrappers.LdaMallet(
mallet_path,
corpus=corpus,
num_topics=LDA_number_of_topics,
id2word=id2word)
# convert
lda_model = gensim.models.wrappers.ldamallet.malletmodel2ldamodel(
lda_model)
else:
# normal LDA
lda_model = gensim.models.ldamodel.LdaModel(
corpus=corpus,
id2word=id2word,
num_topics=LDA_number_of_topics,
random_state=100,
update_every=1,
chunksize=100,
passes=10,
alpha='auto',
per_word_topics=True)
lda_model.save('data/model_LDAEXAMPLE.lda')
if DEBUG_print_topics:
print("Topics:")
pprint(
lda_model.print_topics(num_topics=LDA_number_of_topics,
num_words=5))
doc_lda = lda_model[corpus]
# Evaluation - Perplexity, Coherence Score
print('\nPerplexity: ', lda_model.log_perplexity(
corpus)) # a measure of how good the model is. lower the better.
if CALC_coherence:
coherence_model_lda = CoherenceModel(model=lda_model,
texts=texts,
dictionary=id2word,
coherence='c_v')
coherence_lda = coherence_model_lda.get_coherence()
print('\nCoherence Score: ', coherence_lda)
######################################################################################################
# Dominant topics per document
if SAVE_doc2topic or SAVE_topic2docs or SAVE_topic2num or VIZ_hist:
df_topic_sents_keywords, dominant_topics_as_arr = format_topics_sentences(
ldamodel=lda_model, corpus=corpus, texts=data)
df_dominant_topic = df_topic_sents_keywords.reset_index()
df_dominant_topic.columns = [
'Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Keywords',
'Text'
]
if SAVE_doc2topic:
df_dominant_topic.to_csv(NAME_doc2topic, index=True)
# Topic 2 representative documents
if SAVE_topic2docs:
sent_topics_sorteddf_mallet = pd.DataFrame()
sent_topics_outdf_grpd = df_topic_sents_keywords.groupby(
'Dominant_Topic')
for i, grp in sent_topics_outdf_grpd:
sent_topics_sorteddf_mallet = pd.concat([
sent_topics_sorteddf_mallet,
grp.sort_values(['Perc_Contribution'], ascending=[0]).head(1)
],
axis=0)
sent_topics_sorteddf_mallet.reset_index(drop=True, inplace=True)
sent_topics_sorteddf_mallet.columns = [
'Topic_Num', "Topic_Perc_Contrib", "Keywords", "Text"
]
sent_topics_sorteddf_mallet.to_csv(NAME_topic2docs, index=True)
# Topic 2 number of docs
if SAVE_topic2num:
topic_counts = df_topic_sents_keywords['Dominant_Topic'].value_counts()
topic_contribution = round(topic_counts / topic_counts.sum(), 4)
topic_num_keywords = df_topic_sents_keywords[[
'Dominant_Topic', 'Topic_Keywords'
]]
df_dominant_topics = pd.concat(
[topic_num_keywords, topic_counts, topic_contribution], axis=1)
df_dominant_topics.columns = [
'Dominant_Topic', 'Topic_Keywords', 'Num_Documents',
'Perc_Documents'
]
df_dominant_topics.to_csv(NAME_topic2num, index=True)
# As a plot:
if VIZ_hist:
print("Saving histogram into >", NAME_hist)
hist_tmp = {}
for val in dominant_topics_as_arr:
if val not in hist_tmp:
hist_tmp[val] = 0
else:
hist_tmp[val] += 1
print("My own hist:", hist_tmp)
xs = list(range(LDA_number_of_topics))
ys = [0] * LDA_number_of_topics
for topic_num, val in hist_tmp.items():
wp = lda_model.show_topic(topic_num)
topic_keywords = ", ".join([word for word, prop in wp
]) # removed probabilities
print("topic", topic_num, ":", val, "time(s) , keywords = ",
topic_keywords)
ys[topic_num] = val
plt.bar(xs, ys)
plt.xticks(np.arange(LDA_number_of_topics),
np.arange(LDA_number_of_topics))
plt.savefig(NAME_hist)
plt.close()
import matplotlib.colors as mcolors
colors_topics = [
color for name, color in mcolors.TABLEAU_COLORS.items()
] + [color for name, color in mcolors.XKCD_COLORS.items()]
# Wordclouds of Top N words in each topic
if VIZ_wordclouds:
print("Saving wordclouds into >", NAME_wordclouds, "...")
from matplotlib import pyplot as plt
from wordcloud import WordCloud
from nltk.corpus import stopwords
stop_words = stopwords.words('english')
stop_words.extend(['from', 'subject', 're', 'edu', 'use'])
stoplist = set(
', . : / ( ) [ ] - _ ; * & ? ! – a b c d e t i p an us on 000 if it ll to as are then '
'they our the you we s in if a m I x re to this at ref do and'.
split())
stop_words.extend(stoplist)
stoplist = set(
'experience job ensure able working join key apply strong recruitment work team successful '
'paid contact email role skills company day good high time required want right success'
'ideal needs feel send yes no arisen arise title true'.split())
stop_words.extend(stoplist)
stoplist = set(
'work experience role application process contract interested touch'
.split())
stop_words.extend(stoplist)
#topics = lda_model.show_topics(formatted=False)
topics = lda_model.show_topics(num_topics=LDA_number_of_topics,
formatted=False)
for i_t, topic in enumerate(topics):
topic_i = topic[0]
topic_words = topic[1]
print("topic", topic_i, "===", topic_words)
fig = plt.figure()
topic_words = dict(topic_words)
cloud = WordCloud(
stopwords=stop_words,
background_color='white',
width=2500,
height=1800,
max_words=10,
colormap='tab10',
color_func=lambda *args, **kwargs: colors_topics[topic_i],
prefer_horizontal=1.0)
cloud.generate_from_frequencies(topic_words, max_font_size=300)
plt.gca().imshow(cloud)
plt.gca().set_title('Topic ' + str(topic_i),
fontdict=dict(size=16))
plt.axis('off')
plt.margins(x=0, y=0)
plt.tight_layout()
plt.savefig(NAME_wordclouds + str(topic_i).zfill(2) + ".png")
plt.close()
# T-SNE
if VIZ_TSNE:
print("Saving TSNE visualization into >", NAME_tsne)
from sklearn.manifold import TSNE
from bokeh.plotting import figure, output_file, show, save, ColumnDataSource
# Get topic weights
doc_features = []
doc_titles = []
doc_dominanttopics = []
for i, row_list in enumerate(lda_model[corpus]):
# What we have in the encoding:
# row_list[0] = Document topics: [(0, 0.87507219282484316), (1, 0.12492780717515681)]
# row_list[1] = Word topics: [(0, [0, 1]), (3, [0, 1]), (4, [0, 1]), (7, [0, 1])]
# row_list[2] = Phi values: [(0, [(0, 0.9783234200583657), (1, 0.021676579941634355)]), (3, [(0, 0.93272653621872503), (1, 0.067273463781275009)]), (4, [(0, 0.98919912227661466), (1, 0.010800877723385368)]), (7, [(0, 0.97541896333079636), (1, 0.024581036669203641)])]
# row_list[0] has the weights to topics
# This means that one document was encoded into the LDA_number_of_topics topics we chose
tmp = np.zeros(LDA_number_of_topics)
max_w = -1
max_w_idx = -1
for j, w in row_list[0]:
tmp[j] = w
if max_w < w:
max_w_idx = j
max_w = w
doc_features.append(tmp)
doc_dominanttopics.append(max_w_idx)
doc_titles.append(titles[i])
arr = pd.DataFrame(doc_features).fillna(0).values
# tSNE Dimension Reduction
tsne_model = TSNE(n_components=2,
verbose=1,
random_state=0,
angle=.99,
init='pca')
tsne_lda = tsne_model.fit_transform(arr)
TOOLTIPS = [
("index", "$index"),
("(x,y)", "($x, $y)"),
("desc", "@desc"),
]
mycolors = np.array(colors_topics)
from bokeh.models import HoverTool, WheelZoomTool, PanTool, BoxZoomTool, ResetTool, SaveTool
hover = HoverTool(tooltips=TOOLTIPS)
tools = [
hover,
WheelZoomTool(),
PanTool(),
BoxZoomTool(),
ResetTool(),
SaveTool()
]
#plot = figure(title="t-SNE Clustering of {} LDA Topics".format(LDA_number_of_topics),
# tooltips=TOOLTIPS, plot_width=900, plot_height=700)
plot = figure(title="t-SNE Clustering of {} LDA Topics".format(
LDA_number_of_topics),
tools=tools,
plot_width=900,
plot_height=700)
source = ColumnDataSource(data=dict(
x=tsne_lda[:, 0],
y=tsne_lda[:, 1],
desc=doc_titles,
color=mycolors[doc_dominanttopics],
))
plot.scatter(x='x', y='y', source=source, color='color')
output_file(NAME_tsne)
save(plot)
#show(plot)
if VIZ_html_interactive:
# Takes forever!
print("creating visualization...")
vis = pyLDAvis.gensim.prepare(lda_model, corpus, id2word)
print("saving it...")
pyLDAvis.save_html(vis, NAME_html_interactive)
print("done")
def buildPlot():
#####################Setup
# Grab graph colors, pop undesireable ones
colors = SEABORN_PALETTES['bright']
#Grab and sort the FQs
quals = fruit_df.reset_index()
quals = quals['FruitQuality'].unique().tolist()
for idx, i in enumerate(list(quals)):
if type(i) == type(0.5):
quals.pop(idx)
unique_FQs = quals
#a little math to get the epoch time to set the initial x range
minDate = ts_to_epoch(fruit_df['Date'].min())
maxDate = ts_to_epoch(fruit_df['Date'].max())
###########Create and format the plot
plot = figure(
x_axis_type="datetime",
plot_width=600,
plot_height=400,
tools=[PanTool(),
WheelZoomTool(),
SaveTool(),
BoxZoomTool()],
x_range=DataRange1d(
start=minDate, end=maxDate
), #sets the initial date range to the limits of the data
y_range=DataRange1d(start=0, end=1),
name='the_plot',
toolbar_location='above')
#some styling
plot.title.text = "Historical Volatility"
plot.xaxis.axis_label = "Trade Date"
plot.yaxis.axis_label = "Vol"
plot.background_fill_color = '#EAEBF0'
plot.xgrid.grid_line_color = 'white'
plot.ygrid.grid_line_color = 'white'
plot.xaxis.axis_line_color = 'white'
plot.xaxis.major_tick_line_color = 'white'
plot.xaxis.minor_tick_line_color = 'white'
plot.yaxis.axis_line_color = 'white'
plot.yaxis.major_tick_line_color = 'white'
plot.yaxis.minor_tick_line_color = 'white'
plot.toolbar.logo = None
#a list for all of the lines to reside in
lines = []
legends = []
##############Create the widgets
#a console style window to show debug messages TODO: add on/off functionality
debug = PreText(text="", width=1200, height=500)
#echos the debug in a place more visiable for the user
user_message = Paragraph(text='')
#Asset_Class, Product, and From dropdown boxes. Sets dropdown's initial value.
asCls = Select(title="Asset Class",
options=ddOpts['Asset_Class'].unique().tolist())
asCls.value = asCls.options[0]
prod = Select(title="Products",
options=ddOpts[ddOpts['Asset_Class'] == asCls.value]
['Product'].unique().tolist())
prod.value = prod.options[0]
whereFrom = Select(title="From",
options=ddOpts[(ddOpts['Asset_Class'] == asCls.value)
& (ddOpts['Product'] == prod.value)]
['From'].unique().tolist())
whereFrom.value = whereFrom.options[0]
FQslider = Slider(title='Fruit Quality',
start=min(unique_FQs),
end=max(unique_FQs),
step=1)
#the amount of days back to look for the data
days_back = TextInput(title='Days ago', value='365')
days_back_buttons = RadioButtonGroup(
labels=['10', '30', '90', '180', '365', '730'], active=4)
#the date to linear fit to
fixed_date_buttons = RadioButtonGroup(
labels=['30', '60', '90', '120', '180', '365'], active=2)
fixed_date = TextInput(title='Days to Exp', value='90')
#the amount of days with which to calculate the rolling mean
rolling_days_buttons = RadioButtonGroup(labels=['1', '2', '5', '10'],
active=0)
rolling_days = TextInput(title='Rolling Mean Days', value='1')
#a dynamically resizing checkbox group that allows for the changing of the visablity of any line on the plot
line_onOff = CheckboxGroup(width=400, name='line_onOff')
#the associated colors to act as a legend for line_onOff
legendDiv = Div(width=50)
#button to add a line
addLine = Button(label="Add Line")
#an html rendered visualization of the data for each line
descriptions = Div(text='', width=500)
#resizes the plot
rszButton = Button(label='resize')
##########Define functions associated with the widgets
#concats any dubug call to the end of the current debug text, and changes the user message
def updateDebug(inString):
inString = str(inString)
user_message.text = inString
oldText = debug.text
newText = ("*- " + str(datetime.now()) + " : " + inString)
debug.text = oldText + '\n' + newText
#changes the potential products and contract categories to match the user selected asset class
def asClsChange(attrname, old, new):
prod.options = ddOpts[ddOpts['Asset_Class'] ==
asCls.value]['Product'].unique().tolist()
prod.value = prod.options[0]
#changes the potential contract categories to match the user selected product
def prodChange(attrname, old, new):
whereFrom.options = ddOpts[(ddOpts['Asset_Class'] == asCls.value) & (
ddOpts['Product'] == prod.value)]['From'].unique().tolist()
whereFrom.value = whereFrom.options[0]
#links the days back button and text box
def days_back_buttonChange(attrname, old, new):
days_back.value = days_back_buttons.labels[days_back_buttons.active]
#checks that the users input is an int
def days_backChange(attrname, old, new):
try:
days_back.value = str(int(days_back.value))
except ValueError:
days_back.value = '0'
updateDebug('please type an integer')
#links the fixed date button and text box
def fixed_date_buttonChange(attrname, old, new):
fixed_date.value = fixed_date_buttons.labels[fixed_date_buttons.active]
#checks that the users input is an int
def fixed_dateChange(attrname, old, new):
try:
fixed_date.value = str(int(fixed_date.value))
except ValueError:
fixed_date.value = '0'
updateDebug('please type an integer')
#links the rolling days button and text box
def rolling_days_buttonsChange(attrname, old, new):
rolling_days.value = rolling_days_buttons.labels[
rolling_days_buttons.active]
#checks that the users input is an int
def rolling_daysChange(attrname, old, new):
try:
rolling_days.value = str(int(rolling_days.value))
except ValueError:
rolling_days.value = '0'
updateDebug('please type an integer')
#fits the plot to the currently visiable lines
def resize():
if len(line_onOff.active) == 0 or len(line_onOff.labels) == 0:
plot.x_range.start = ts_to_epoch(fruit_df['Date'].min())
plot.x_range.end = ts_to_epoch(fruit_df['Date'].max())
plot.y_range.start = 0
plot.y_range.end = 100
else:
xmin, xmax, ymin, ymax = calc_range(lines)
plot.x_range.start = xmin
plot.x_range.end = xmax
plot.y_range.start = ymin
plot.y_range.end = ymax
#turn lines on or off
def line_onOffChange(attrname, old, new):
for i in range(len(line_onOff.labels)):
if i in line_onOff.active:
lines[i].glyph.visible = True
else:
lines[i].glyph.visible = False
legendDiv.text = '<div>'
for line in lines:
legendDiv.text += '<br><div style="background-color: %s; float:up; padding: 4px 4px 4px 4px"></div><br>' % line.glyph.line_color
legendDiv.text += '</div>'
resize()
#adds a line to the graph
def grphUpdt():
#adds some debug messages, grabs the current time as to later show the total time taken to calculate
updateDebug("Starting")
updateDebug("total dataframe size: " + str(fruit_df.shape))
stTime = datetime.now()
#the value to linear fit to
fit_to = int(fixed_date.value)
#instiantiate an empty dataframe that will eventually contain the graphs data
graphData = pd.DataFrame({
'Date': [],
'PriceVolatility': [],
'Days_to_Exp': []
})
#grab the appropriate subset of the whole dataframe based on the users input into the widgets
updateDebug("querying the data..")
try:
workingDf = fruit_df.loc[asCls.value, prod.value, whereFrom.value]
except KeyError:
updateDebug(
'no data with that combination of Asset Class, Product, From')
return
try:
workingDf = workingDf[[
'Date', 'PriceVolatility', 'Days_to_Exp'
]][(workingDf['Date'] >
(date.today() - timedelta(days=int(days_back.value))))]
except KeyError:
updateDebug(
'no data with that combination of Asset Class, Product, From, and days back'
)
return
updateDebug("done breaking down df")
#a hook in the case that the users inputs resulted in an empty dataframe
if (workingDf.empty):
updateDebug(
'no data with that combination of Asset Class, Product, From, and days back'
)
return
#widdle down the database to only contain the user specified FQ
try:
graphData = workingDf.loc[int(FQslider.value)].copy()
except KeyError:
updateDebug('no data with that FQ')
#another empty graph hook
if (graphData.empty):
updateDebug(
'no data with that combination of Asset Class, Product, Contract Category, FQ, and days back'
)
return
updateDebug('grabed correct FQs')
#calculate linear fit on the current subset
updateDebug('calculating linear fit...')
graphData = mu.linearFit(fit_to=fit_to,
group_on_column='Date',
df=graphData,
fit_column='Days_to_Exp',
on_columns=['PriceVolatility'])
updateDebug('finished with linear fit')
# a few more debug messages
updateDebug(
"working df qry: Asset_Class = %s and Product = %s and From = %s and Date > %s "
% (asCls.value, prod.value, whereFrom.value,
str(date.today() - timedelta(days=int(days_back.value)))))
updateDebug("graph data shape: " + str(workingDf.shape))
#makes sure graph data has at least 5 rows, so that rolling mean can be calculated
if graphData.shape[0] > int(rolling_days.value):
#make the graph legend, based on if there's a denominator specified or not
this_legend = '%s - %s FQ: %s Days to Exp: %s From: %s Rolling Days: %s' % (
prod.value, whereFrom.value, int(
FQslider.value), fixed_date.value,
str(date.today() - timedelta(days=int(days_back.value))),
rolling_days.value)
#add a new line to the graph, and add the accosiated GlyphRenderer created by adding the line to the lines list.
#Set the legend to the previously calculated legend, and set the color to the next color in the current theme (if there are more lines than colors, there will be multiple lines with the same color)
#Calculates a 5 day rolling mean on the y values. Maybe add a slider/text box/other widget so the user can set the rolling mean themselves
updateDebug('adding line to plot')
lines.append(
plot.line(graphData.index.values[int(rolling_days.value) - 1:],
graphData['PriceVolatility'].rolling(
window=int(rolling_days.value)).mean()
[int(rolling_days.value) - 1:],
line_width=3,
color=colors[len(lines) % len(colors)]))
legends.append(this_legend)
updateDebug("updated graph")
global descDf
#either creates, or adds to, a dataframe containing statistics about the data. stats come from pandas DataFrame.describe.
if descDf is None:
graphData[this_legend] = graphData['PriceVolatility']
descDf = graphData[[
this_legend
]].rolling(window=int(rolling_days.value)).mean(
)[int(rolling_days.value) -
1:].describe(percentiles=[]).transpose().copy()
else:
graphData[this_legend] = graphData['PriceVolatility']
descDf = pd.concat([
descDf, graphData[[
this_legend
]].rolling(window=int(rolling_days.value)).mean()
[int(rolling_days.value) -
1:].describe(percentiles=[]).transpose().copy()
])
descDf = descDf.round(1)
descriptions.text = descDf.to_html().replace('\\n', '')
graphData.drop(this_legend, 1, inplace=True)
#add the name of the line to the checkbox so that it can be turned off and o
line_onOff.labels.append(this_legend)
line_onOff.active.append(len(line_onOff.labels) - 1)
legendDiv.text = '<div>'
for line in lines:
legendDiv.text += '<br><div style="background-color: %s; float:up; padding: 4px 4px 4px 4px"></div><br>' % line.glyph.line_color
legendDiv.text += '</div>'
##leaving this in case we get around to figuring out the hover tool
##formats the date values for the hover tool, currently commented out until we, or bokeh, fix the hover tool for multiple lines
#formDates= pd.to_datetime(graphData['Date'] ,format="%m-%d-%Y")
#lines[-1].data_source.data['formDates'] = formDates.apply(lambda x: x.strftime('%m-%d-%Y'))
##Displays the amout of time it took to draw the line, as well as the number of points in the graph
updateDebug("updated y vals, with rolling mean calculated")
updateDebug(
str(datetime.now() - stTime) + " FOR " +
str(len(lines[-1].data_source.data['x'])) + " points")
else:
updateDebug("There's no data to display")
del graphData
del workingDf
#######Link widgets to their associated functions
asCls.on_change('value', asClsChange)
prod.on_change('value', prodChange)
days_back_buttons.on_change('active', days_back_buttonChange)
days_back.on_change('value', days_backChange)
fixed_date_buttons.on_change('active', fixed_date_buttonChange)
fixed_date.on_change('value', fixed_dateChange)
rolling_days_buttons.on_change('active', rolling_days_buttonsChange)
rolling_days.on_change('value', rolling_daysChange)
line_onOff.on_change('active', line_onOffChange)
addLine.on_click(grphUpdt)
rszButton.on_click(resize)
#Formatting
fixed_date_box = WidgetBox(fixed_date, fixed_date_buttons)
days_back_box = WidgetBox(days_back, days_back_buttons)
rolling_days_box = WidgetBox(rolling_days, rolling_days_buttons)
widgets = [
asCls, prod, whereFrom, FQslider, days_back_box, fixed_date_box,
rolling_days_box, addLine, rszButton, user_message
]
plot_w_description = VBox(plot, descriptions, width=700)
pwd_w_leg = HBox(plot_w_description,
VBox(legendDiv),
VBox(line_onOff),
width=plot_w_description.width + line_onOff.width + 100,
name='div_to_save')
input_box = VBox(*widgets, width=400, height=1200)
total_box = HBox(VBox(input_box),
VBox(pwd_w_leg),
width=input_box.width + pwd_w_leg.width + 100,
height=1200)
tot_w_debug = VBox(total_box, VBox(HBox(debug)))
resize()
return tot_w_debug
def plot_tsne(self, parameter):
result = self.result
source1 = column_source(result, 'INACTIVO')
source2 = column_source(result, 'PQSR-AGONISTA')
source3 = column_source(result, 'RHLR-ANTAGONISTA')
source4 = column_source(result, 'LASR-AGONISTA')
source5 = column_source(result, 'PQSR-ANTAGONISTA')
source6 = column_source(result, 'BIOFACQUIM_V2')
source7 = column_source(result, 'LASR-ANTAGONISTA')
source8 = column_source(result, 'QUIPRONAB')
source9 = column_source(result, 'NuBBE')
source10 = column_source(result, 'RHLR-AGONISTA')
hover = HoverTool(tooltips=[
("PCA 1", "$x"),
("PCA 2", "$y"),
("NAME", "@N"),
])
p = figure(
title="tSNE based on " + parameter,
x_axis_label="PC 1",
y_axis_label="PC 2",
x_range=(-7, 7),
y_range=(-7, 7),
tools=[hover],
plot_width=1000,
plot_height=800,
)
p.add_tools(LassoSelectTool(), ZoomInTool(), ZoomOutTool(), SaveTool(),
PanTool())
INACTIVO_plot = p.circle(x="x",
y="y",
source=source1,
color="indigo",
size=5)
PQSR_AGONISTA_plot = p.circle(x="x",
y="y",
source=source2,
color="hotpink",
size=5)
RHLR_ANTAGONISTA_plot = p.circle(x="x",
y="y",
source=source3,
color="navy",
size=5)
LASR_AGONISTA_plot = p.circle(x="x",
y="y",
source=source4,
color="dodgerblue",
size=5)
PQSR_ANTAGONISTA_plot = p.circle(x="x",
y="y",
source=source5,
color="lightcoral",
size=5)
BIOFACQUIM_V2_plot = p.circle(x="x",
y="y",
source=source6,
color="gold",
size=5)
LASR_ANTAGONISTA_plot = p.circle(x="x",
y="y",
source=source7,
color="yellowgreen",
size=5)
QUIPRONAB_plot = p.circle(x="x",
y="y",
source=source8,
color="green",
size=5)
NuBBE_plot = p.circle(x="x",
y="y",
source=source9,
color="orangered",
size=5)
RHLR_AGONISTA_plot = p.circle(x="x",
y="y",
source=source10,
color="mediumvioletred",
size=5)
legend = Legend(
items=[
('INACTIVO', [INACTIVO_plot]),
('PQSR-AGONISTA', [PQSR_AGONISTA_plot]),
('RHLR-ANTAGONISTA', [RHLR_ANTAGONISTA_plot]),
('LASR-AGONISTA', [LASR_AGONISTA_plot]),
('PQSR-ANTAGONISTA', [PQSR_ANTAGONISTA_plot]),
('BIOFACQUIM_V2', [BIOFACQUIM_V2_plot]),
('LASR-ANTAGONISTA', [LASR_ANTAGONISTA_plot]),
('QUIPRONAB', [QUIPRONAB_plot]),
('NuBBE', [NuBBE_plot]),
('RHLR-AGONISTA', [RHLR_AGONISTA_plot]),
],
location="center",
orientation="vertical",
click_policy="hide",
)
p.add_layout(legend, place="right")
p.xaxis.axis_label_text_font_size = "20pt"
p.yaxis.axis_label_text_font_size = "20pt"
p.xaxis.axis_label_text_color = "black"
p.yaxis.axis_label_text_color = "black"
p.xaxis.major_label_text_font_size = "18pt"
p.yaxis.major_label_text_font_size = "18pt"
p.title.text_font_size = "22pt"
show(p)
graph_circle.edge_renderer.glyph.line_width = {'field': 'weight'}
graph_circle.edge_renderer.data_source.data['color'] = colors
#graph_circle.edge_renderer.glyph.line_color = {'field': 'color'}
graph_circle.selection_policy = NodesAndLinkedEdges()
graph_circle.inspection_policy = NodesAndLinkedEdges()
# !!! Hover the node attributes !!!
node_hover = HoverTool(tooltips=[('Name', '@index'), ('Degree', '@degree')])
plot_circle.add_tools(node_hover)
plot_circle.add_tools(WheelZoomTool())
plot_circle.add_tools(ResetTool())
plot_circle.add_tools(PanTool())
plot_circle.add_tools(TapTool())
plot_circle.add_tools(SaveTool())
plot_circle.add_tools(BoxSelectTool())
plot_circle.add_tools(LassoSelectTool())
plot_circle.renderers.append(graph_circle)
# spring layout
plot_spring = Plot(plot_width=1000, plot_height=715,
x_range=Range1d(-1.1, 1.1), y_range=Range1d(-1.1, 1.1))
my_points= nx.spring_layout(g, iterations = 50)
my_colors = []
for key, value in my_points.items():
#print (key, 'corresponds to', value)
#print(' X: ', value[0] )
def __init__(self,
nplots,
plot_height=350,
plot_width=700,
lower=0,
upper=1000,
nbins=100):
"""Initialize histogram plots.
Args:
nplots (int): Number of histogram plots that will share common controls.
plot_height (int, optional): Height of plot area in screen pixels. Defaults to 350.
plot_width (int, optional): Width of plot area in screen pixels. Defaults to 700.
lower (int, optional): Initial lower range of the bins. Defaults to 0.
upper (int, optional): Initial upper range of the bins. Defaults to 1000.
nbins (int, optional): Initial number of the bins. Defaults to 100.
"""
# Histogram plots
self.plots = []
self._plot_sources = []
for ind in range(nplots):
plot = Plot(
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=plot_height,
plot_width=plot_width,
toolbar_location="left",
)
# ---- tools
plot.toolbar.logo = None
# share 'pan', 'boxzoom', and 'wheelzoom' tools between all plots
if ind == 0:
pantool = PanTool()
boxzoomtool = BoxZoomTool()
wheelzoomtool = WheelZoomTool()
plot.add_tools(pantool, boxzoomtool, wheelzoomtool, SaveTool(),
ResetTool())
# ---- axes
plot.add_layout(LinearAxis(), place="below")
plot.add_layout(LinearAxis(major_label_orientation="vertical"),
place="left")
# ---- grid lines
plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- quad (single bin) glyph
plot_source = ColumnDataSource(dict(left=[], right=[], top=[]))
plot.add_glyph(
plot_source,
Quad(left="left",
right="right",
top="top",
bottom=0,
fill_color="steelblue"),
)
self.plots.append(plot)
self._plot_sources.append(plot_source)
self._counts = []
self._empty_counts()
# Histogram controls
# ---- histogram range toggle button
def auto_toggle_callback(state):
if state: # Automatic
lower_spinner.disabled = True
upper_spinner.disabled = True
else: # Manual
lower_spinner.disabled = False
upper_spinner.disabled = False
auto_toggle = CheckboxGroup(labels=["Auto Hist Range"],
active=[0],
default_size=145)
auto_toggle.on_click(auto_toggle_callback)
self.auto_toggle = auto_toggle
# ---- histogram lower range
def lower_spinner_callback(_attr, _old_value, new_value):
self.upper_spinner.low = new_value + STEP
self._empty_counts()
lower_spinner = Spinner(
title="Lower Range:",
high=upper - STEP,
value=lower,
step=STEP,
disabled=bool(auto_toggle.active),
default_size=145,
)
lower_spinner.on_change("value", lower_spinner_callback)
self.lower_spinner = lower_spinner
# ---- histogram upper range
def upper_spinner_callback(_attr, _old_value, new_value):
self.lower_spinner.high = new_value - STEP
self._empty_counts()
upper_spinner = Spinner(
title="Upper Range:",
low=lower + STEP,
value=upper,
step=STEP,
disabled=bool(auto_toggle.active),
default_size=145,
)
upper_spinner.on_change("value", upper_spinner_callback)
self.upper_spinner = upper_spinner
# ---- histogram number of bins
def nbins_spinner_callback(_attr, _old_value, _new_value):
self._empty_counts()
nbins_spinner = Spinner(title="Number of Bins:",
low=1,
value=nbins,
default_size=145)
nbins_spinner.on_change("value", nbins_spinner_callback)
self.nbins_spinner = nbins_spinner
# ---- histogram log10 of counts toggle button
def log10counts_toggle_callback(state):
self._empty_counts()
for plot in self.plots:
if state:
plot.yaxis[0].axis_label = "log⏨(Counts)"
else:
plot.yaxis[0].axis_label = "Counts"
log10counts_toggle = CheckboxGroup(labels=["log⏨(Counts)"],
default_size=145)
log10counts_toggle.on_click(log10counts_toggle_callback)
self.log10counts_toggle = log10counts_toggle
#make fit visible
if dropdown_marker_fit.value!='none':
renderers_plot_fit[dropdown_marker_fit.value].visible=True
if dropdown_linedash_fit.value=='none':
renderers_plot_fit['line'].visible=False
else:
renderers_plot_fit['line'].visible=True
###########################################################################################################################################
###########################################################plot tab########################################################################
###########################################################################################################################################
################
#create figure#
##############
fig_plot = figure(toolbar_location='right', toolbar_sticky=False, tools=[PanTool(), BoxZoomTool(), WheelZoomTool(), BoxSelectTool(), TapTool(), ResetTool(), SaveTool()],output_backend='webgl')
fig_plot.plot_width = 800
fig_plot.plot_height = 600
fig_plot.toolbar.logo = None
############################################################################
#create all selectable renderers and set the circle and line renderer true#
##########################################################################
renderers_plot = {rn: getattr(fig_plot, rn)(x='x', y='y', source=source_plot,
**extra, visible=False)
for rn, extra in [('line', dict(line_width=__linewidth_plot, line_color=__linecolor_plot, line_alpha=__linealpha_plot, line_dash=dict_linedash[__linedash_plot])),
('circle', dict(size=__size_plot, line_width=__linewidth_plot, line_color=__linecolor_plot, fill_color=__fillcolor_plot, line_alpha=__linealpha_plot, fill_alpha=__fillalpha_plot, line_dash=dict_linedash[__linedash_plot])),
('diamond', dict(size=__size_plot, line_width=__linewidth_plot, line_color=__linecolor_plot, fill_color=__fillcolor_plot, line_alpha=__linealpha_plot, fill_alpha=__fillalpha_plot, line_dash=dict_linedash[__linedash_plot])),
('square', dict(size=__size_plot, line_width=__linewidth_plot, line_color=__linecolor_plot, fill_color=__fillcolor_plot, line_alpha=__linealpha_plot, fill_alpha=__fillalpha_plot, line_dash=dict_linedash[__linedash_plot])),
('triangle', dict(size=__size_plot, line_width=__linewidth_plot, line_color=__linecolor_plot, fill_color=__fillcolor_plot, line_alpha=__linealpha_plot, fill_alpha=__fillalpha_plot, line_dash=dict_linedash[__linedash_plot])),
('asterisk', dict(size=__size_plot, line_width=__linewidth_plot, line_color=__linecolor_plot, fill_color=__fillcolor_plot, line_alpha=__linealpha_plot, fill_alpha=__fillalpha_plot, line_dash=dict_linedash[__linedash_plot])),
def make_document(doc):
base = "http://192.168.50.62/get?gyrX=%s|gyr_time&gyr_time=%s&gyrY=%s|gyr_time&gyrZ=%s|gyr_time"
data = ColumnDataSource(
dict(
time=[],
# display_time=[],
x=[],
y=[],
z=[],
))
def get_last():
global current_time
# print(current_time)
raw = requests.get(
base % (current_time, current_time, current_time, current_time))
j = raw.json()
ln = min(
len(j['buffer']["gyr_time"]["buffer"]),
len(j['buffer']["gyrX"]["buffer"]),
len(j['buffer']["gyrY"]["buffer"]),
len(j['buffer']["gyrZ"]["buffer"]),
)
prices_df = pd.DataFrame.from_dict({
"time":
j['buffer']["gyr_time"]["buffer"][:ln],
"x":
j['buffer']["gyrX"]["buffer"][:ln],
"y":
j['buffer']["gyrY"]["buffer"][:ln],
"z":
j['buffer']["gyrZ"]["buffer"][:ln],
})
if len(j['buffer']["gyr_time"]["buffer"]) == 0:
current_time = 0
else:
current_time = j['buffer']["gyr_time"]["buffer"][-1]
# prices_df["time"] = pd.to_datetime(prices_df["time"], unit="ms")
# prices_df["display_time"] = prices_df["time"].dt.strftime("%m-%d-%Y %H:%M:%S.%f")
return prices_df
def update_price():
new_price = get_last()
print(len(new_price['time']), end='\n\n')
data.stream(
dict(time=new_price["time"],
x=new_price["x"],
y=new_price["y"],
z=new_price["z"]), 1000)
return
hover = HoverTool(
tooltips=[("Time", "@display_time"), ("IEX Real-Time Price",
"@price")])
fig_x = figure(plot_width=800,
plot_height=400,
x_axis_type='datetime',
tools=[WheelZoomTool(),
ResetTool(),
PanTool(),
SaveTool()],
title="Real-Time Price Plot")
fig_x.line(source=data, color="navy", x='time', y='x')
fig_x.xaxis.axis_label = "Time"
fig_x.yaxis.axis_label = "IEX Real-Time Price"
fig_x.title.text = "IEX Real Time Price: "
fig_y = figure(plot_width=800,
plot_height=400,
x_axis_type='datetime',
tools=[WheelZoomTool(),
ResetTool(),
PanTool(),
SaveTool()],
title="Real-Time Price Plot",
x_range=fig_x.x_range,
y_range=fig_x.y_range)
fig_y.line(source=data, color="navy", x='time', y='y')
fig_y.xaxis.axis_label = "Time"
fig_y.yaxis.axis_label = "IEX Real-Time Price"
fig_y.title.text = "IEX Real Time Price: "
fig_z = figure(plot_width=800,
plot_height=400,
x_axis_type='datetime',
tools=[WheelZoomTool(),
ResetTool(),
PanTool(),
SaveTool()],
title="Real-Time Price Plot",
x_range=fig_x.x_range,
y_range=fig_x.y_range)
fig_z.line(source=data, color="navy", x='time', y='z')
fig_z.xaxis.axis_label = "Time"
fig_z.yaxis.axis_label = "IEX Real-Time Price"
fig_z.title.text = "IEX Real Time Price: "
ticker_textbox = TextInput(placeholder="Ticker")
update = Button(label="Update")
inputs = widgetbox([ticker_textbox, update], width=200)
doc.add_root(column(fig_x, fig_y, fig_z, width=1600))
doc.title = "Real-Time Price Plot from IEX"
doc.add_periodic_callback(update_price, 30)
def bokeh_plot(import_df):
import pandas as pd
import numpy as np
from bokeh.plotting import figure, show
from bokeh.layouts import layout, widgetbox, row, column, gridplot
from bokeh.models import ColumnDataSource, HoverTool, BoxZoomTool, ResetTool, PanTool, CustomJS, PrintfTickFormatter, WheelZoomTool, SaveTool, LassoSelectTool, NumeralTickFormatter
from bokeh.models.widgets import Slider, Select, TextInput, Div, Tabs, Panel, DataTable, DateFormatter, TableColumn, PreText, NumberFormatter, RangeSlider
from bokeh.io import curdoc
from functools import lru_cache
from bokeh.transform import dodge
from os.path import dirname, join
from bokeh.core.properties import value
#load plotting data here
@lru_cache()
def load_data():
df = import_df
df.dropna(how='all', axis=0)
#Northest=['3229','3277','3276','3230','3259','All_Stores_NE']
df.location_reference_id = df.location_reference_id.astype(str)
#df['region'] = ['Northeast' if x in Northest else 'Midwest' for x in df['location_reference_id']]
df['date'] = pd.to_datetime(df['date'])
df[[
'BOH_gt_Shelf_Capacity', 'OTL_gt_Shelf_Capacity',
'Ideal_BOH_gt_Shelf_Capacity', 'BOH_lt_Ideal', 'BOH_eq_Ideal',
'BOH_gt_Ideal', 'Demand_Fulfilled', 'Fill_Rate', 'Backroom_OH',
'Total_OH', 'Prop_OH_in_Backroom', 'Never_Q98_gt_POG',
'Never_Ideal_BOH_gt_POG', 'Sometimes_OTL_Casepack_1_gt_POG',
'Always_OTL_Casepack_1_le_POG', 'Non_POG'
]] = df[[
'BOH > Shelf Capacity', 'OTL > Shelf Capacity',
'Ideal BOH > Shelf Capacity', 'BOH < Ideal', 'BOH = Ideal',
'BOH > Ideal', 'Demand Fulfilled', 'Fill Rate', 'Backroom_OH',
'Total OH', 'Prop OH in Backroom', 'Never: Q98 > POG',
'Never: Ideal BOH > POG', 'Sometimes: OTL+Casepack-1 > POG',
'Always: OTL+Casepack-1 <= POG', 'Non-POG'
]]
df['date_bar'] = df['date']
df['date_bar'] = df['date_bar'].astype(str)
return df
#Filter data source for "All" stores OR data agrregation on DC level
df_agg = load_data().groupby(['location_reference_id'],
as_index=False).sum()
source1 = ColumnDataSource(data=df_agg)
sdate = min(load_data()['date'])
edate = max(load_data()['date'])
nodes = len(list(load_data().location_reference_id.unique()))
days = len(list(load_data().date.unique()))
policy = "Prod"
#list of dates for vbar charts
x_range_list = list(load_data().date_bar.unique())
#direct access to number of location_reference_idand region
all_locations1 = list(load_data().location_reference_id.unique())
#agg_value=['All']
#all location_reference_idfrom csv file along with an option for agg data "All"
#all_locations=all_locations1+agg_value
#all_regions = ['Northeast', 'Midwest']
all_regions = list(load_data().region.unique())
desc = Div(text="All locations", width=230)
pre = Div(text="_", width=230)
location = Select(title="Location",
options=all_locations1,
value="All_Stores_NE")
region = Select(title="Region", options=all_regions, value="NE")
text_input = TextInput(value="default", title="Search Location:")
#full data set from load_data(df=df_import)
source = ColumnDataSource(data=load_data())
original_source = ColumnDataSource(data=load_data())
#plotting starts........... here are total 8 graphs for each Metric.
#Back room on hand
hover = HoverTool(
tooltips=[("Location", "@location_reference_id"), (
"Date", "@date_bar"), ("Backroom_OH", "@Backroom_OH{0,0.00}")])
TOOLS = [
hover,
BoxZoomTool(),
LassoSelectTool(),
WheelZoomTool(),
PanTool(),
ResetTool(),
SaveTool()
]
p = figure(x_range=x_range_list,
plot_width=1000,
plot_height=525,
title="Backroom On hand by store",
tools=TOOLS,
toolbar_location='above',
x_axis_label="Date",
y_axis_label="Backroom OH")
p.background_fill_color = "#e6e9ed"
p.background_fill_alpha = 0.5
p.vbar(x=dodge('date_bar', -0.25, range=p.x_range),
top='Backroom_OH',
hover_alpha=0.5,
hover_line_color='black',
width=0.8,
source=source,
color="#718dbf")
p.xaxis.major_label_orientation = 1
p.legend.border_line_width = 3
p.legend.border_line_color = None
p.legend.border_line_alpha = 0.5
p.title.text_color = "olive"
#inbound outbound
hover_m = HoverTool(
tooltips=[("Location", "@location_reference_id"), (
"Date", "@date_bar"), (
"Inbound",
"@Inbound{0,0.00}"), ("Outbound", "@Outbound{0,0.00}")])
TOOLS_m = [
hover_m,
BoxZoomTool(),
LassoSelectTool(),
WheelZoomTool(),
PanTool(),
ResetTool(),
SaveTool()
]
m = figure(plot_height=525,
plot_width=1000,
x_range=x_range_list,
title="Inbound/Outbound by store",
tools=TOOLS_m,
toolbar_location='above',
x_axis_label="Date",
y_axis_label="Units")
m.background_fill_color = "#e6e9ed"
m.background_fill_alpha = 0.5
m.vbar(x=dodge('date_bar', -0.25, range=m.x_range),
top='Inbound',
hover_alpha=0.5,
hover_line_color='black',
width=0.4,
source=source,
color="#718dbf",
legend=value("Inbound"))
m.vbar(x=dodge('date_bar', 0.25, range=m.x_range),
top='Outbound',
hover_alpha=0.5,
hover_line_color='black',
width=0.4,
source=source,
color="#e84d60",
legend=value("Outbound"))
m.xaxis.major_label_orientation = 1
m.legend.border_line_width = 3
m.legend.border_line_color = None
m.legend.border_line_alpha = 0.5
m.title.text_color = "olive"
#Stockout
hover_s = HoverTool(
tooltips=[("Location", "@location_reference_id"), (
"Date", "@date_bar"), (
"BOH_OOS",
"@BOH_OOS{0,0.000}"), ("EOH_OOS", "@EOH_OOS{0,0.000}")])
TOOLS_s = [
hover_s,
BoxZoomTool(),
LassoSelectTool(),
WheelZoomTool(),
PanTool(),
ResetTool(),
SaveTool()
]
s = figure(plot_height=525,
plot_width=1000,
title="Stockouts by store",
x_axis_type="datetime",
toolbar_location='above',
tools=TOOLS_s,
x_axis_label="Date",
y_axis_label="Prop Stockout")
s.background_fill_color = "#e6e9ed"
s.background_fill_alpha = 0.5
s.circle(x='date',
y='EOH_OOS',
source=source,
fill_color=None,
line_color="#4375c6")
s.line(x='date',
y='EOH_OOS',
source=source,
hover_alpha=0.5,
hover_line_color='black',
line_width=2,
line_color='navy',
legend=value("EOH OOS"))
s.circle(x='date',
y='BOH_OOS',
source=source,
fill_color=None,
line_color="#4375c6")
s.line(x='date',
y='BOH_OOS',
source=source,
hover_alpha=0.5,
hover_line_color='black',
line_width=2,
line_color='red',
legend=value("BOH OOS"))
s.legend.border_line_width = 3
s.legend.border_line_color = None
s.legend.border_line_alpha = 0.5
s.title.text_color = "olive"
#Fill rate
hover_t = HoverTool(
tooltips=[("Location", "@location_reference_id"), (
"Date", "@date_bar"), ("Fill Rate", "@Fill_Rate{0,0.00}")])
TOOLS_t = [
hover_t,
BoxZoomTool(),
LassoSelectTool(),
WheelZoomTool(),
PanTool(),
ResetTool(),
SaveTool()
]
t = figure(plot_height=525,
x_range=x_range_list,
plot_width=1000,
title="Fill rates by store",
tools=TOOLS_t,
toolbar_location='above',
x_axis_label="Date",
y_axis_label="Fill rate")
t.background_fill_color = "#e6e9ed"
t.background_fill_alpha = 0.5
t.vbar(x=dodge('date_bar', -0.25, range=t.x_range),
top='Fill Rate',
hover_alpha=0.5,
hover_line_color='black',
width=0.8,
source=source,
color="#718dbf")
t.xaxis.major_label_orientation = 1
t.legend.border_line_width = 3
t.legend.border_line_color = None
t.legend.border_line_alpha = 0.5
t.title.text_color = "olive"
# % Backroom spillover
hover_w = HoverTool(
tooltips=[("Location", "@location_reference_id"), ("Date",
"@date_bar"),
("Prop OH in Backroom", "@Prop_OH_in_Backroom{0,0.00}")])
TOOLS_w = [
hover_w,
BoxZoomTool(),
LassoSelectTool(),
WheelZoomTool(),
PanTool(),
ResetTool(),
SaveTool()
]
w = figure(plot_height=525,
plot_width=1000,
title="Prop OH in Backroom by store",
x_axis_type="datetime",
tools=TOOLS_w,
toolbar_location='above',
x_axis_label="Date",
y_axis_label=" % Backroom spillover")
w.background_fill_color = "#e6e9ed"
w.background_fill_alpha = 0.5
w.circle(x='date',
y='Prop OH in Backroom',
source=source,
fill_color=None,
line_color="#4375c6")
w.line(x='date',
y='Prop OH in Backroom',
source=source,
hover_alpha=0.5,
hover_line_color='black',
line_width=2,
line_color='navy')
w.title.text_font_style = "bold"
w.title.text_color = "olive"
w.legend.click_policy = "hide"
w.yaxis[0].formatter = NumeralTickFormatter(format="0.0%")
#BOH vs Ideal
hover_f = HoverTool(
tooltips=[("Location", "@location_reference_id"), (
"Date",
"@date_bar"), ('BOH < Ideal', "@BOH_lt_Ideal{0,0.00}"
), ('BOH > Ideal', "@BOH_gt_Ideal{0,0.00}"
), ('BOH = Ideal', "@BOH_eq_Ideal{0,0.00}")])
TOOLS_f = [
hover_f,
BoxZoomTool(),
LassoSelectTool(),
WheelZoomTool(),
PanTool(),
ResetTool(),
SaveTool()
]
colors = ["#c9d9d3", "#718dbf", "#e84d60"]
BOH_vs_ideal = ['BOH < Ideal', 'BOH > Ideal', 'BOH = Ideal']
f = figure(x_range=x_range_list,
plot_height=525,
plot_width=1000,
title="BOH vs Ideal by store",
toolbar_location='above',
x_axis_label="Date",
y_axis_label="Prop",
tools=TOOLS_f)
f.vbar_stack(BOH_vs_ideal,
x='date_bar',
width=0.9,
color=colors,
source=source,
legend=[value(x) for x in BOH_vs_ideal],
name=BOH_vs_ideal)
f.xaxis.major_label_orientation = 1
f.legend.border_line_width = 3
f.legend.border_line_color = None
f.legend.border_line_alpha = 0.5
f.title.text_color = "olive"
#Pog Fit
hover_g = HoverTool(
tooltips=[("Location", "@location_reference_id"), (
"Date",
"@date_bar"), ('Never: Q98 > POG', "@Never_Q98_gt_POG{0,0.00}"),
("Never: Ideal BOH > POG",
"@Never_Ideal_BOH_gt_POG{0,0.00}"),
("Sometimes: OTL+Casepack-1 > POG",
"@Sometimes_OTL_Casepack_1_gt_POG{0,0.00}"),
("Always: OTL+Casepack-1 <= POG",
"@Always_OTL_Casepack_1_le_POG{0,0.00}"
), ("Non-POG'", "@Non_POG{0,0.00}")])
TOOLS_g = [
hover_g,
BoxZoomTool(),
LassoSelectTool(),
WheelZoomTool(),
PanTool(),
ResetTool(),
SaveTool()
]
colors2 = ['#79D151', "#718dbf", '#29788E', '#fc8d59', '#d53e4f']
pog_fit = [
'Never: Q98 > POG', 'Never: Ideal BOH > POG',
'Sometimes: OTL+Casepack-1 > POG', 'Always: OTL+Casepack-1 <= POG',
'Non-POG'
]
g = figure(x_range=x_range_list,
plot_height=525,
plot_width=1200,
title="Pog Fit by store",
toolbar_location='above',
x_axis_label="Date",
y_axis_label="Counts",
tools=TOOLS_g)
g.vbar_stack(pog_fit,
x='date_bar',
width=0.9,
color=colors2,
source=source,
legend=[value(x) for x in pog_fit],
name=pog_fit)
g.xaxis.major_label_orientation = 1
g.legend.border_line_width = 3
g.legend.border_line_color = None
g.legend.border_line_alpha = 0.5
g.title.text_color = "olive"
g.legend.location = "top_right"
# BOH vs Pog
colors3 = ["#c9d9d3", "#718dbf", "#e84d60"]
shelf = [
'BOH > Shelf Capacity', 'OTL > Shelf Capacity',
'Ideal BOH > Shelf Capacity'
]
hover_h = HoverTool(
tooltips=[("Location", "@location_reference_id"), ("Date",
"@date_bar"),
("OTL > Shelf Capacity", "@OTL_gt_Shelf_Capacity{0,0.00}"
), ("BOH > Shelf Capacity",
"@BOH_gt_Shelf_Capacity{0,0.00}"),
("Ideal BOH > Shelf Capacity",
"@Ideal_BOH_gt_Shelf_Capacity{0,0.00}")])
TOOLS_h = [
hover_h,
BoxZoomTool(),
LassoSelectTool(),
WheelZoomTool(),
PanTool(),
ResetTool(),
SaveTool()
]
h = figure(plot_height=525,
plot_width=1000,
title="BOH vs Pog by store",
x_axis_type="datetime",
toolbar_location='above',
tools=TOOLS_h,
x_axis_label="Date",
y_axis_label="Prop")
h.background_fill_color = "#e6e9ed"
h.background_fill_alpha = 0.5
h.circle(x='date',
y='BOH > Shelf Capacity',
source=source,
fill_color=None,
line_color="#4375c6")
h.line(x='date',
y='BOH > Shelf Capacity',
source=source,
hover_alpha=0.5,
hover_line_color='black',
line_width=2,
line_color='navy',
legend=value("BOH > Shelf Capacity"))
h.circle(x='date',
y='OTL > Shelf Capacity',
source=source,
fill_color=None,
line_color="#4375c6")
h.line(x='date',
y='OTL > Shelf Capacity',
source=source,
hover_alpha=0.5,
hover_line_color='black',
line_width=2,
line_color="green",
legend=value("OTL > Shelf Capacity"))
h.circle(x='date',
y='Ideal BOH > Shelf Capacity',
source=source,
fill_color=None,
line_color="#4375c6")
h.line(x='date',
y='Ideal BOH > Shelf Capacity',
source=source,
hover_alpha=0.5,
hover_line_color='black',
line_width=2,
line_color="#e84d60",
legend=value("Ideal BOH > Shelf Capacity"))
h.legend.border_line_width = 3
h.legend.border_line_color = None
h.legend.border_line_alpha = 0.5
h.title.text_color = "olive"
h.legend.click_policy = "mute"
# Inventory
hover_j = HoverTool(
tooltips=[("Location", "@location_reference_id"), (
"Date", "@date_bar"), ("DFE_Q98", "@DFE_Q98{0,0.00}"),
("OTL",
"@OTL{0,0.00}"), ("EOH",
"@EOH{0,0.00}"), ("BOH", "@BOH{0,0.00}")])
TOOLS_j = [
hover_j,
BoxZoomTool(),
LassoSelectTool(),
WheelZoomTool(),
PanTool(),
ResetTool(),
SaveTool()
]
j = figure(plot_height=525,
plot_width=1200,
x_range=x_range_list,
title="Inbound/Outbound by store",
tools=TOOLS_j,
toolbar_location='above',
x_axis_label="Date",
y_axis_label="Units")
j.background_fill_color = "#e6e9ed"
j.background_fill_alpha = 0.5
j.vbar(x=dodge('date_bar', -0.40, range=j.x_range),
top='DFE_Q98',
hover_alpha=0.3,
hover_line_color='black',
width=0.2,
source=source,
color="#FBA40A",
legend=value("DFE_Q98"))
j.vbar(x=dodge('date_bar', -0.20, range=j.x_range),
top='OTL',
hover_alpha=0.3,
hover_line_color='black',
width=0.2,
source=source,
color="#4292c6",
legend=value("OTL"))
j.vbar(x=dodge('date_bar', 0.00, range=j.x_range),
top='EOH',
hover_alpha=0.3,
hover_line_color='black',
width=0.2,
source=source,
color='#a1dab4',
legend=value("EOH"))
j.vbar(x=dodge('date_bar', 0.20, range=j.x_range),
top='BOH',
hover_alpha=0.3,
hover_line_color='black',
width=0.2,
source=source,
color="#DC5039",
legend=value("BOH"))
j.xaxis.major_label_orientation = 1
j.legend.border_line_width = 3
j.legend.border_line_color = None
j.legend.border_line_alpha = 0.5
j.title.text_color = "olive"
j.legend.location = "top_left"
j.legend.click_policy = "mute"
#desc.text = " <br > <b> Region:</b> <i> </i> <br /> "
pre.text = " <b>Start date:</b> <i>{}</i> <br /> <b>End date:</b> <i>{}</i> <br /> <b>Time period:</b> <i>{}</i> days <br /> <b> Total Number of Nodes:</b> <i>{}</i> <br /> <b>Policy</b> = <i>{}</i><br /> ".format(
sdate, edate, days, nodes, policy)
#fuction to update data on selection
callback = CustomJS(args=dict(source=source,
original_source=original_source,
location_select_obj=location,
region_select_obj=region,
div=desc,
text_input=text_input),
code="""
var data = source.get('data');
var original_data = original_source.get('data');
var loc = location_select_obj.get('value');
var reg = region_select_obj.get('value');
var line = " <br /> <b> Region:</b>"+ reg + "<br /> <b>Location:</b> " + loc;
var text_input =text_input.get('value');
div.text=line;
for (var key in original_data) {
data[key] = [];
for (var i = 0; i < original_data['location_reference_id'].length; ++i) {
if ((original_data['location_reference_id'][i] === loc) && (original_data['region'][i] === reg) ) {
data[key].push(original_data[key][i]);
} } }
source.trigger('change');
""")
#controls = [location, region]
#for control in controls:
#control.js_on_change("value", callback)
#source.js_on_change("value", callback)
desc.js_on_event('event', callback)
location.js_on_change('value', callback)
region.js_on_change('value', callback)
text_input.js_on_change('value', callback)
#inputs = widgetbox(*controls, sizing_mode="fixed")
#inputs = widgetbox(*controls,width=220,height=500)
inputs = widgetbox(location, region, desc, pre, width=220, height=500)
# controls number of tabs
tab1 = Panel(child=p, title='Backroom OH')
tab2 = Panel(child=s, title='Stockouts')
tab3 = Panel(child=f, title='BOH vs Ideal')
tab4 = Panel(child=g, title='Pog Fit')
tab5 = Panel(child=m, title='Inbound/Outbound')
tab6 = Panel(child=h, title='BOH vs POG')
tab7 = Panel(child=t, title='Fill Rate')
tab8 = Panel(child=j, title='Inventory')
tab9 = Panel(child=w, title='Prop OH in Backroom')
#data table columns to summarize data
columns = [
TableColumn(field="location_reference_id", title="Location"),
TableColumn(field="Backroom_OH",
title="Backroom_OH",
formatter=NumberFormatter(format="0,0")),
TableColumn(field="Outbound",
title="Outbound",
formatter=NumberFormatter(format="0,0")),
TableColumn(field="Inbound",
title="Inbound",
formatter=NumberFormatter(format="0,0")),
TableColumn(field="OTL",
title="OTL",
formatter=NumberFormatter(format="0,0")),
TableColumn(field="DFE_Q98",
title="DFE_Q98",
formatter=NumberFormatter(format="0,0")),
TableColumn(field="BOH",
title="BOH",
formatter=NumberFormatter(format="0,0")),
TableColumn(field="EOH",
title="EOH",
formatter=NumberFormatter(format="0,0")),
TableColumn(field="BOH_OOS",
title="BOH_OOS",
formatter=NumberFormatter(format="0,0")),
TableColumn(field="EOH_OOS",
title="EOH_OOS",
formatter=NumberFormatter(format="0,0"))
]
data_table = DataTable(source=source1, columns=columns, width=1250)
tab10 = Panel(child=data_table, title='Summary Table')
view = Tabs(
tabs=[tab1, tab2, tab5, tab8, tab6, tab3, tab7, tab4, tab9, tab10])
layout_text = column(inputs)
layout1 = row(layout_text, view)
#laying out plot
layout2 = layout(children=[[layout_text, view]],
sizing_mode='scale_height')
#update plots
return layout2
def main():
"""Invoke when run directly as a program."""
args = parse_arguments()
df = pd.read_csv(args.infile, sep=r"\s+", names=["amplicon", "meancov", "gene"])
df["offsetcov"] = df["meancov"] + 0.1 # shift zero values by 0.1
df = df.dropna().reset_index(drop=True)
# Make a hover (show amplicon name on mouse-over)
hover = HoverTool(tooltips=[("Amplicon", "@names"), ("Y value", "$y")])
tools = [
PanTool(),
BoxZoomTool(),
WheelZoomTool(),
RedoTool(),
UndoTool(),
ResetTool(),
hover,
SaveTool(),
]
# Produce plot
output_file(args.outfile)
fig = figure(tools=tools, width=1200, height=600, y_axis_type="log")
# Fill plot with one point for each amplicon:
xvals, yvals, labels, colors = [], [], [], []
for i, (name, group) in enumerate(df.groupby("gene", sort=False)):
xvals.extend(list(group.index))
yvals.extend(list(group.offsetcov))
labels.extend(list(group.amplicon))
# Elements in the same group should have the same color. Cycle between colors in COLOR_CYCLE:
colors.extend([COLOR_CYCLE[i % len(COLOR_CYCLE)]] * len(list(group.index)))
data = ColumnDataSource(data=dict(x=xvals, y=yvals, names=labels, colors=colors))
fig.circle(x="x", y="y", color="colors", size=10, source=data)
# Make span lines on 0.05, 0.1, 0.2, 1 and 5 mutiples of mean amplicon coverage:
mean_coverage = df.offsetcov.mean()
span_lines = [
(5.0, "Blue"),
(1.0, "Green"),
(0.2, "Red"),
(0.1, "Purple"),
(0.05, "Magenta"),
]
xmin, xmax = min(xvals) - 1, max(xvals) + 1
for ratio, color in span_lines:
fig.line(
[xmin, xmax],
[mean_coverage * ratio] * 2,
line_color=color,
line_dash="dashed",
legend="{:.0f} % of mean coverage".format(ratio * 100),
)
# Customize plot:
ymax = 2.0 * max(df.offsetcov.max() + 1000, mean_coverage * 5)
ymin = 0.2
fig.y_range = Range1d(ymin, ymax)
fig.x_range = Range1d(xmin, xmax)
fig.xaxis.major_tick_line_color = None # Turn off x-axis major ticks
fig.xaxis.minor_tick_line_color = None # Turn off x-axis minor ticks
fig.xaxis.major_label_text_font_size = "0pt" # Hack to remove tick labels
fig.xaxis.axis_label = "Amplicon"
fig.yaxis.axis_label = "Log10 (Amplicon coverage)"
fig.legend.location = "bottom_right"
script, div = components(layouts.row(fig))
with open(os.path.join(os.path.dirname(args.outfile), "plot.js"), "wt") as jfile:
jfile.write("\n".join(script.split("\n")[2:-1]))
with open(args.outfile, "wt") as ofile:
env = Environment(loader=FileSystemLoader(os.path.dirname(args.template)))
page_template = env.get_template(os.path.basename(args.template))
html_text = page_template.render({"bokeh_div": div})
ofile.write(html_text)
# Definition of the protocol values and their actual names
protocols_path = "{}doc/protocols".format(potiron_path)
protocols = potiron.define_protocols(protocols_path)
# Define the strings used for legends, titles, etc. concerning fields
field_string, field_in_file_name = field2string(field, potiron_path)
field_data = create_dict(field, potiron_path)
# Creation of the figure and the tools used on it
namefile=output_name(source,field_in_file_name,fieldvalues,date,outputdir)
output_file("{}.html".format(namefile), title=namefile.split("/")[-1])
hover = HoverTool(tooltips = [('count','@y'),('protocol','@protocol')])
taptool = TapTool()
TOOLS = [hover,PanTool(),BoxZoomTool(),WheelZoomTool(), taptool, SaveTool(), ResetTool()]
p = figure(width=plot_width,height=plot_height,tools=TOOLS)
# Definition of some variables which will be used and modified with the iterations
at_least_one = False
days = calendar.monthrange(int(date[0:4]),int(date[4:6]))[1]
maxVal = 0
minVal = sys.maxsize
maxDay = 0
vlength = len(fieldvalues)
actual_values = []
nbLine = 0
# For each selected field or occurrence
for v in range(vlength):
value = fieldvalues[v].split('-')
actual_field = value[0]
#
def dashboard(df,
sim_time=180,
lat=43.223628,
lon=-90.294633,
outfile='dashboard.html'):
#Isolate the individual housing states
df_onlyState = df[[
'inspection_get', 'claim_get', 'fema_get', 'sba_get', 'permit_get',
'repair_get', 'home_get', 'occupy_get'
]]
#list of statuses in correct order for future reference
statuses = [
'inspection_get', 'claim_get', 'fema_get', 'sba_get', 'permit_get',
'repair_get', 'home_get', 'occupy_get'
]
NUM_HOMES = df.shape[0] #the total number of entities
NUM_CAT = len(statuses) #the total number of categories
# assign colors to the sequential housing statuses.
# colors are map-optimised, from ColorBrewer
# palettes will scale to size automatically, max size is 20
colors_only = bokeh.palettes.d3['Category20'][NUM_CAT]
colors = {}
for i in range(0, len(statuses)):
colors[statuses[i]] = colors_only[i]
# create a list to hold the status of all homes by day
home_status_list = []
for i in range(0, sim_time):
single_home_status = np.empty(shape=[2860, 1], dtype=object)
curr_max = i
curr = 0
for row in df_onlyState.itertuples(index=False):
#convert the row into a dictionary for key-value analysis
row_asDict = row._asdict()
#find the most recent status time of the home within currMax. ignores None and nan
try:
mostRecentTime = max(value
for name, value in row_asDict.items()
if value is not None and value < curr_max)
#reverse key-value to determine actual status
key = next(key for key, value in row_asDict.items()
if value == mostRecentTime)
except ValueError:
key = 'no_status'
single_home_status[curr] = key
curr += 1
home_status_list.append(
pd.Series(data=single_home_status.ravel(), name=i))
# create a single DataFrame for the home statuses at every unit of simulation time
home_status = pd.concat(home_status_list, axis=1)
# dataframe for number of homes with a given status at every point of simulation time
status_count_list = []
for time in range(0, sim_time):
status_count_list.append(
pd.Series(data=home_status[time].value_counts(), name=str(time)))
#concatenate and fill NaN with zeroes. re-index rows to correct order
status_count_df = pd.concat(status_count_list,
axis=1).fillna(value=0).reindex(statuses)
#create a new dataframe with just the colors for the map
#current colors: white to dark green, where dark green = home_get
home_status_colors = home_status.replace(colors)
# Interactive Barplot
# Standalone HTML file using CustomJS
#wrangle the data into a data source for the ColumnDataSource to work properly with Javascript
per_day = status_count_df.transpose().values.tolist()
data = dict({str(i): v for i, v in enumerate(per_day)})
data['x'] = statuses #add the statuses to the data source
data['y'] = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0] #dummy column for CustomJS to overwrite
data['colors'] = colors_only
source = ColumnDataSource(data)
output_file(outfile)
#plot setup
barplot = figure(
plot_width=800,
plot_height=600,
tools='pan, wheel_zoom, reset, save',
x_axis_label='Status',
x_range=source.data['x'],
y_range=ranges.Range1d(start=0, end=len(data['y'])),
title="Number of Households by Status at Specified Time (Slider)")
barplot.vbar(source=source, x='x', top='y', color='colors', width=0.6)
bar_hover = HoverTool(tooltips=[('num', '@y')])
barplot.add_tools(bar_hover)
lat = 43.223628
lon = -90.294633
#Map Setup
map_options = GMapOptions(lat=lat,
lng=lon,
scale_control=True,
map_type="roadmap",
zoom=16)
mapplot = GMapPlot(x_range=Range1d(),
y_range=Range1d(),
map_options=map_options)
mapplot.title.text = "Locations of Modeled Homes (Marker Size Proportional to Original Damage State; Colors Defined Below)"
# Set up tool tip with household stories
hover = HoverTool()
hover.tooltips = [("Household's Story:", "@story")]
mapplot.add_tools(PanTool(), WheelZoomTool(), ResetTool(), hover,
SaveTool())
mapplot.min_border_top = 50
mapplot.min_border_bottom = 0
#set Google Maps API key
mapplot.api_key = "AIzaSyDedFXgNZ71_xOGmVE_eAhElFJOuVweB1Y"
#data wrangling for JS interaction
home_status_formap = pd.concat(
[home_status_colors.copy(), df['latitude'], df['longitude']], axis=1)
home_status_formap['y'] = np.nan #dummy column
home_status_formap['story'] = df['story']
home_status_formap.columns = home_status_formap.columns.astype(str)
# Reclassify damage states at start of simulation for each household to set
# marker/circle size for mapping
damage_size = dict({
'None': 10,
'Slight': 15,
'Moderate': 20,
'Extensive': 25,
'Complete': 30
})
home_status_formap['circle_size'] = df['damage_state_start'].replace(
list(damage_size.keys()), list(damage_size.values()))
mapsource = ColumnDataSource(home_status_formap)
circle = Circle(x="longitude",
y="latitude",
size='circle_size',
fill_color="y",
fill_alpha=0.8,
line_color='black')
mapplot.add_glyph(mapsource, circle)
# Javascript callback to enable and link interactivity between the two plots.
callback = CustomJS(args=dict(s1=source, s2=mapsource),
code="""
console.log(' changed selected time', cb_obj.value);
var data = s1.data;
var data2 = s2.data;
data['y'] = data[cb_obj.value];
data2['y'] = data2[cb_obj.value];
s1.change.emit()
s2.change.emit()
""")
#Line Graph setup
line_plot = figure(
title='Number of Households By Status vs. Simulated Time',
y_range=ranges.Range1d(start=int(NUM_HOMES * 0.1),
end=int(NUM_HOMES * 1.5)),
tools='pan, wheel_zoom, reset, save')
all_line_data = status_count_df.values.tolist()
day_range = np.linspace(0, sim_time, num=sim_time).tolist()
# Generate a vertical bar to indicate current time within the line graph
# Line is generated to 10% above the number of homes and 10% below zero
currtime_list = {
'x': [0, 0],
'y': [int(NUM_HOMES * 1.1),
int(NUM_HOMES * -0.1)]
} #dummy column for js callback
for i in range(0, sim_time):
currtime_list[str(i)] = [i, i]
currtime_source = ColumnDataSource(currtime_list)
# Create set of line plots with ability to turn individual lines on/off
for data, name, color in zip(all_line_data, statuses, colors_only):
line_data = pd.DataFrame(data).values.tolist()
line_plot.line(day_range,
line_data,
color=color,
alpha=0.8,
legend=name,
line_width=2)
line_plot.line(x='x', y='y', source=currtime_source, line_color='red')
line_plot.legend.location = "top_left"
line_plot.legend.click_policy = "hide"
# Requires Bokeh 0.12.7
# Javascript callback to enable and link interactivity between the two plots.
callback = CustomJS(args=dict(s1=source, s2=mapsource, s3=currtime_source),
code="""
console.log(' changed selected time', cb_obj.value);
var data = s1.data;
var data2 = s2.data;
var data3 = s3.data;
data['y'] = data[cb_obj.value];
data2['y'] = data2[cb_obj.value];
data3['x'] = data3[cb_obj.value];
s1.change.emit();
s2.change.emit();
s3.change.emit();
""")
#slider
time_slider = Slider(start=0,
end=sim_time,
value=0,
step=1,
callback=callback,
title='DAY')
#show entire layout
show(
layout([[mapplot], [line_plot, barplot], time_slider],
sizing_mode='stretch_both'))
def drawgraph(self, ds, filterMin, filterMax, layout):
G = nx.Graph() # create an empty graph with no nodes and no edges
# nodes = []
# for i in range(len(ds.getNodes())):
# nodes.append([])
# # print(ds.getNodes()[i].getName())
# for j in range(len(ds.getNodes())):
# nodes[i].append(ds.getNodes()[i].getLinks()[j][1])
# # print(" " + str(ds.getNodes()[i].getLinks()[j][1]))
# ds.toMinSpanTree()
nodes = ds.getDoubleList(filterMin, filterMax, False)
x = filterMin
adj = np.array(nodes)
G = nx.from_numpy_matrix(adj)
for i in range(len(G.node)):
G.node[i]['name'] = ds.getNames()[i]
#pos = nx.drawing.layout.circular_layout(G, 1, None, 2)
#nx.draw_networkx(G, pos, with_labels=True)
# pt.show()
#plt.show()
plot = Plot(plot_width=500, plot_height=500,
x_range=Range1d(-1.1, 1.1), y_range=Range1d(-1.1, 1.1))
node_hover_tool = HoverTool(tooltips=[("Name of this node", "@name")])
# plot.add_tools(node_hover_tool, BoxZoomTool(), ResetTool())
plot.add_tools(node_hover_tool, TapTool(), BoxSelectTool(), BoxZoomTool(), UndoTool(), RedoTool(), SaveTool(),
ResetTool())
plot.toolbar_location = 'left'
if layout == 0:
graph_renderer = from_networkx(G, nx.circular_layout, scale=1, center=(0, 0))
elif layout == 1:
graph_renderer = from_networkx(G, nx.spring_layout, scale=1, center=(0, 0))
else:
graph_renderer = from_networkx(G, nx.random_layout)
graph_renderer.node_renderer.glyph = Circle(size=15, fill_color=Spectral4[0])
graph_renderer.node_renderer.selection_glyph = Circle(size=15, fill_color=Spectral4[2])
graph_renderer.node_renderer.hover_glyph = Circle(size=15, fill_color=Spectral4[1])
graph_renderer.edge_renderer.glyph = MultiLine(line_color="#CCCCCC", line_alpha=0.8, line_width=5)
graph_renderer.edge_renderer.selection_glyph = MultiLine(line_color=Spectral4[2], line_width=5)
graph_renderer.edge_renderer.hover_glyph = MultiLine(line_color=Spectral4[1], line_width=5)
graph_renderer.selection_policy = NodesAndLinkedEdges()
#graph_renderer.inspection_policy = EdgesAndLinkedNodes()
plot.renderers.append(graph_renderer)
output_file("interactive_graphs.html")
return plot
x_range=xdr, y_range=ydr,
plot_width=1000, plot_height=600,
min_border=0,
toolbar_location=None,
background_fill_color='#F0F0F0',
border_fill_color='lightgray',
)
line_glyph = Line(x="x", y="y", line_color="navy", line_width=2, line_dash="dashed")
line = plot.add_glyph(source, line_glyph)
circle = Circle(x="x", y="y2", size=6, line_color="red", fill_color="orange", fill_alpha=0.6)
circle = plot.add_glyph(source, circle)
pan = PanTool()
wheel_zoom = WheelZoomTool()
preview_save = SaveTool()
plot.add_tools(pan, wheel_zoom, preview_save)
# Add axes (Note it's important to add these before adding legends in side panels)
plot.add_layout(LinearAxis(), 'below')
plot.add_layout(LinearAxis(), 'left')
plot.add_layout(LinearAxis(), 'right')
def add_legend(location, orientation, side):
legend = Legend(
# TODO: title
items=[("line", [line]), ("circle", [circle])],
location=location, orientation=orientation,
border_line_color="black",
)
def generate_graph(self):
"""
Generate the graph; return a 2-tuple of strings, script to place in the
head of the HTML document and div content for the graph itself.
:return: 2-tuple (script, div)
:rtype: tuple
"""
logger.debug('Generating graph for %s', self._graph_id)
# tools to use
tools = [
PanTool(),
BoxZoomTool(),
WheelZoomTool(),
SaveTool(),
ResetTool(),
ResizeTool()
]
# generate the stacked area graph
try:
g = Area(
self._data, x='Date', y=self._y_series_names,
title=self._title, stack=True, xlabel='Date',
ylabel='Downloads', tools=tools,
# note the width and height will be set by JavaScript
plot_height=400, plot_width=800,
toolbar_location='above', legend=False
)
except Exception as ex:
logger.error("Error generating %s graph", self._graph_id)
logger.error("Data: %s", self._data)
logger.error("y=%s", self._y_series_names)
raise ex
lines = []
legend_parts = []
# add a line at the top of each Patch (stacked area) for hovertool
for renderer in g.select(GlyphRenderer):
if not isinstance(renderer.glyph, Patches):
continue
series_name = renderer.data_source.data['series'][0]
logger.debug('Adding line for Patches %s (series: %s)', renderer,
series_name)
line = self._line_for_patches(self._data, g, renderer, series_name)
if line is not None:
lines.append(line)
legend_parts.append((series_name, [line]))
# add the Hovertool, specifying only our line glyphs
g.add_tools(
HoverTool(
tooltips=[
(self._y_name, '@SeriesName'),
('Date', '@FmtDate'),
('Downloads', '@Downloads'),
],
renderers=lines,
line_policy='nearest'
)
)
# legend outside chart area
legend = Legend(legends=legend_parts, location=(0, 0))
g.add_layout(legend, 'right')
return components(g)
def body_figs(data, height=500, width=1000):
#Data setup
data['date_str'] = data['date'].map(str)
ma_cds_working = ColumnDataSource(dict(date=data['date'], date_str=data['date_str'], bpm=data['bpm'], hrv=data['hrv'], scaled_hrv=data['scaled_hrv'], sleep_overall_q=data['sleep_overall_q'], sleep_onset=data['sleep_onset'], sleep_duration=data['sleep_duration'], sleep_how_much_more=data['sleep_how_much_more'], sleep_how_deep=data['sleep_how_deep'], sleep_interruptions=data['sleep_interruptions'], glucose=data['glucose'], ketones=data['ketones'], weight=data['weight'], bodyfat=data['bodyfat']))
ma_cds_static = ColumnDataSource(dict(date=data['date'], date_str=data['date_str'], bpm=data['bpm'], hrv=data['hrv'], scaled_hrv=data['scaled_hrv'], sleep_overall_q=data['sleep_overall_q'], sleep_onset=data['sleep_onset'], sleep_duration=data['sleep_duration'], sleep_how_much_more=data['sleep_how_much_more'], sleep_how_deep=data['sleep_how_deep'], sleep_interruptions=data['sleep_interruptions'], glucose=data['glucose'], ketones=data['ketones'], weight=data['weight'], bodyfat=data['bodyfat']))
#Plot tools configuration
wz = WheelZoomTool(dimensions='width')
plt_biomarkers_tools = [HoverTool(tooltips=[("Date", "@date_str"), ("RHR", "@bpm{0,0} bpm"), ("Sleep Q", "@sleep_overall_q"), ("Glucose", "@glucose{0,0} mg/dl"), ("Ketones", "@ketones{1.11} mmol/L")],names=["bpm", "glucose"],mode='vline'), PanTool(dimensions='width'), wz, ResetTool(), SaveTool()]
wz2 = WheelZoomTool(dimensions='width')
plt_bcomp_tools = [HoverTool(tooltips=[("Date", "@date_str"), ("Weight", "@weight{1.1} lbs"), ("BF", "@bodyfat{1.1}%")],mode='vline'), PanTool(dimensions='width'), wz2, ResetTool(), SaveTool()]
wz3 = WheelZoomTool(dimensions='width')
plt_sleep_tools = [HoverTool(tooltips=[("Date", "@date_str"),("Sleep Quality", "@sleep_overall_q"),("Duration", "@sleep_duration"),("Satisfaction", "@sleep_how_much_more"),("Depth", "@sleep_how_deep"),("Interruptions", "@sleep_interruptions")],names=["sleep_overall_q"],mode='vline'), PanTool(dimensions='width'), wz3, ResetTool(), SaveTool()]
#Plot Blood (glucose and ketones)
plot_blood = figure(x_axis_type="datetime", title="Biomarkers (Various)", h_symmetry=False, v_symmetry=False, min_border=0, plot_height=height, y_range=[40, 140], plot_width=int(width/2 - 50), toolbar_location="above", outline_line_color="#666666", tools=plt_biomarkers_tools, active_scroll=wz)
plot_blood.extra_y_ranges = {"ketones_range": Range1d(start=0, end=7)}
plot_blood.add_layout(LinearAxis(y_range_name="ketones_range"), 'right')
plot_blood.line('date', 'glucose', name="glucose", source=ma_cds_working, line_color="#FF7700", line_width=3, line_alpha=0.6, legend="Blood Glucose")
plot_blood.cross('date', 'ketones', source=ma_cds_working, line_color="#C74D56", line_alpha=0.6, legend="Blood Ketones", y_range_name="ketones_range")
plot_blood.ray(x=data['date'][1195],y=.5, length=0, angle=0, line_color="#C74D56", line_width=1, y_range_name="ketones_range")
plot_blood.ray(x=data['date'][1195],y=1, length=0, angle=0, line_color="#C74D56", line_width=1, y_range_name="ketones_range")
plot_blood.legend.location = "top_left"
plot_blood.legend.click_policy="hide"
#Plot Heartrate
plot_rhr = figure(x_axis_type="datetime", title="Morning Resting HR", h_symmetry=False, v_symmetry=False, min_border=0, plot_height=int(height/2), plot_width=int(width/2), x_range=plot_blood.x_range, outline_line_color="#666666")
plot_rhr.line('date', 'bpm', name="bpm", source=ma_cds_working, line_color="#8B0A50", line_width=3, line_alpha=0.6, legend="BPM")
plot_rhr.line('date', 'hrv', name="hrv", source=ma_cds_working, line_color="#0a8b45", line_width=3, line_alpha=0.6, legend="HRV")
plot_rhr.line('date', 'scaled_hrv', name="scaled_hrv", source=ma_cds_working, line_color="#333366", line_width=3, line_alpha=0.6, y_range_name="scaled_hrv_range", legend="HRV (Scaled)")
#***TODO*** Add SNS/PNS indicator
plot_rhr.extra_y_ranges = {"scaled_hrv_range": Range1d(start=1, end=10)}
plot_rhr.add_layout(LinearAxis(y_range_name="scaled_hrv_range"), 'right')
plot_rhr.legend.location = "bottom_left"
plot_rhr.legend.click_policy="hide"
plot_rhr.toolbar_location = None
#Plot sleep quality (single indicator)
plot_osq = figure(x_axis_type="datetime", title="Overall Sleep Quality", h_symmetry=False, v_symmetry=False, min_border=0, plot_height=int(height/2), plot_width=int(width/2), y_range=[1, 9], x_range=plot_blood.x_range, outline_line_color="#666666")
plot_osq.line('date', 'sleep_overall_q', source=ma_cds_working, line_color="#333366", line_width=3, line_alpha=0.6)
plot_osq.toolbar_location = None
#Plot body compostion (weight and bodyfat)
plot_composition = figure(x_axis_type="datetime", title="Body Composition", h_symmetry=False, v_symmetry=False, min_border=0, plot_height=height, plot_width=width, y_range=[120,180], toolbar_location="above", outline_line_color="#666666", tools=plt_bcomp_tools, active_scroll=wz2)
plot_composition.extra_y_ranges = {"bodyfat_range": Range1d(start=5, end=15)}
plot_composition.add_layout(LinearAxis(y_range_name="bodyfat_range"), 'right')
plot_composition.line('date', 'weight', name="weight", source=ma_cds_working, line_color="#FF7700", line_width=3, line_alpha=0.6, legend="Weight")
plot_composition.line('date', 'bodyfat', source=ma_cds_working, line_color="#333366", line_width=3, line_alpha=0.6, legend="Bodyfat", y_range_name="bodyfat_range")
plot_composition.legend.location = "top_left"
plot_composition.legend.click_policy="hide"
#Plot sleep (all indicators)
plot_sleep = figure(x_axis_type="datetime", title="Sleep", h_symmetry=False, v_symmetry=False, min_border=0, plot_height=height, plot_width=width, y_range=[1,9], x_range=plot_blood.x_range, toolbar_location="above", outline_line_color="#666666", tools=plt_sleep_tools, active_scroll=wz3)
plot_sleep.extra_y_ranges = {"sleep_range": Range1d(start=0, end=12)}
plot_sleep.add_layout(LinearAxis(y_range_name="sleep_range"), 'right')
plot_sleep.line('date', 'sleep_overall_q', name='sleep_overall_q', source=ma_cds_working, line_color="#8B0A50", line_width=2, line_alpha=0.6, legend="Sleep Quality")
plot_sleep.line('date', 'sleep_how_much_more', source=ma_cds_working, line_color="#FF7700", line_width=2, line_alpha=0.6, legend="Satisfaction")
plot_sleep.line('date', 'sleep_how_deep', source=ma_cds_working, line_color="#C74D56", line_width=2, line_alpha=0.6, legend="Depth")
plot_sleep.line('date', 'sleep_interruptions', source=ma_cds_working, line_color="#0a8b45", line_width=2, line_alpha=0.6, legend="Interruptions")
plot_sleep.line('date', 'sleep_duration', source=ma_cds_working, line_color="#333366", line_width=5, line_alpha=0.6, legend="Duration (hrs)", y_range_name="sleep_range")
plot_sleep.legend.location = "top_left"
plot_sleep.legend.click_policy="hide"
#Statistics divs for the control panel
div_days = Div()
div_avg_bg = Div()
div_avg_rhr = Div()
div_avg_slp_dur = Div()
div_avg_slp_q = Div()
div_days_bc = Div()
div_avg_wt = Div()
div_avg_bf = Div()
#Callbacks
ma_cb = CustomJS(args=dict(w=ma_cds_working, s=ma_cds_static), code=MA_SLIDER_CODE)
plot_blood.x_range.callback = CustomJS(args=dict(d_d=div_days, d_a_bg=div_avg_bg, d_a_r=div_avg_rhr, d_a_s_d=div_avg_slp_dur, d_a_s_q=div_avg_slp_q, s=ma_cds_static), code=BODY_STATS_CODE)
plot_composition.x_range.callback = CustomJS(args=dict(d_d_bc=div_days_bc, d_a_wt=div_avg_wt, d_a_bf=div_avg_bf, s=ma_cds_static), code=BODY_COMP_STATS_CODE)
ma_slider = Slider(start=1, end=30, value=7, step=1, title="Moving Average", callback=ma_cb)
return components((div_days, div_avg_bg, div_avg_rhr, div_avg_slp_dur, div_avg_slp_q, div_days_bc, div_avg_wt, div_avg_bf, plot_blood, plot_rhr, plot_osq, plot_composition, plot_sleep, ma_slider))
doc.clear()
doc.title = 'Homework 4 Question 2'
source = ColumnDataSource(
data=dict(x=[], y=[], radius=[], top=[], bottom=[], left=[], right=[]))
p = figure(plot_width=800,
plot_height=800,
x_range=(-20, 20),
y_range=(-20, 20),
tools=[
BoxSelectTool(),
BoxZoomTool(),
ResetTool(),
WheelZoomTool(),
SaveTool(),
PanTool()
],
x_axis_label='Units',
y_axis_label="Units",
toolbar_location="right")
circ = p.circle(x='x',
y='y',
radius='radius',
color="firebrick",
alpha=0.8,
source=source,
radius_dimension='max')
squa = p.quad(top='top',
bottom='bottom',
def generate_plot(self):
"""Calculates mean and standard deviation for measure(s) by model,
then generates a bar plot of model vs mean performance.
TODO: Finish this doc.
"""
# Use this for a built-in bar plot instead,
# but doesn't work with custom hover tool
#p = Bar(self.data,label='modelname',values=self.measure,agg='mean',\
# title='Mean %s Performance By Model'%self.measure,legend=None,\
# tools=tools, color='modelname')
#self.script,self.div = components(p)
#return
# TODO: hardcoded self.measure[0] for now, but should incorporate
# a for loop somewhere to subplot for each selected measure
# TODO: add significance information (see plot_bar_pretty
# and randttest in narf_analysis)
# build new pandas series of stdev values to be added to dataframe
# if want to show more info on tooltip in the future, just need
# to build an appropriate series to add and then build its tooltip
#in the create_hover function
modelnames = self.data.index.levels[0].tolist()
stdev_col = pd.Series(index=modelnames)
mean_col = pd.Series(index=modelnames)
median_col = pd.Series(index=modelnames)
n_cells_col = pd.Series(index=modelnames)
#for each model, find the stdev and mean over the measure values, then
#assign those values to new Series objects to use for the plot
for model in modelnames:
values = self.data[self.measure[0]].loc[model]
stdev = values.std(skipna=True)
mean = values.mean(skipna=True)
median = values.median(skipna=True)
if (math.isnan(stdev)) or (math.isnan(mean)) or (math.isnan(median)):
# If either statistic comes out as NaN, entire column was NaN,
# so model doesn't have the necessary data.
continue
stdev_col.at[model] = stdev
mean_col.at[model] = mean
median_col.at[model] = median
n_cells_col.at[model] = values.count()
newData = pd.DataFrame.from_dict({
'stdev':stdev_col, 'mean':mean_col, 'median':median_col,
'n_cells':n_cells_col,
})
# Drop any models with NaN values, since that means they had no
# performance data for one or more columns.
newData.dropna(axis=0, how='any', inplace=True)
if newData.size == 0:
self.script,self.div = (
"Error, no plot to display.",
"None of the models contained valid performance data."
)
return
dat_source = ColumnDataSource(newData)
tools = [
PanTool(), SaveTool(), WheelZoomTool(),
ResetTool(), self.create_hover()
]
xrange = FactorRange(factors=modelnames)
yrange = Range1d(
start=0,
end=(max(newData['mean'])*1.5)
)
p = figure(
x_range=xrange, x_axis_label=(
"Modelname, prefix: {0}, suffix: {1}"
.format(self.pre, self.suf)
),
y_range=yrange, y_axis_label='Mean %s'%self.measure[0],
title="Mean %s Performance By Model"%self.measure[0],
tools=tools, responsive=True, toolbar_location=TOOL_LOC,
toolbar_sticky=TOOL_STICK,
output_backend="svg"
)
p.xaxis.major_label_orientation=-(np.pi/4)
glyph = VBar(
x='index', top='mean', bottom=0, width=VBAR_WIDTH,
fill_color=VBAR_FILL, line_color='black'
)
p.add_glyph(dat_source,glyph)
# workaround to prevent title and toolbar from overlapping
grid = gridplot(
[p], ncols=GRID_COLS, responsive=True,
)
self.script, self.div = components(grid)
def bubble_plot_tabs(dataframes):
dataframes = dataframes.copy()
# convert asset dicts to pandas dataframes
base_df = pd.DataFrame.from_dict(dataframes['base_output_by_asset'])
reform_df = pd.DataFrame.from_dict(dataframes['reform_output_by_asset'])
change_df = pd.DataFrame.from_dict(dataframes['changed_output_by_asset'])
list_df = [base_df, change_df, reform_df]
list_string = ['base', 'change', 'reform']
data_sources = {}
for i, df in enumerate(list_df):
# remove data from Intellectual Property, Land, and Inventories Categories
df = df[~df['asset_category'].
isin(['Intellectual Property', 'Land', 'Inventories'])].copy()
df = df.dropna()
# define the size DataFrame, if change, use base sizes
if list_string[i] == 'base':
SIZES = list(range(20, 80, 15))
size = pd.qcut(df['assets_c'].values, len(SIZES), labels=SIZES)
size_c = pd.qcut(df['assets_c'].values, len(SIZES), labels=SIZES)
size_nc = pd.qcut(df['assets_nc'].values, len(SIZES), labels=SIZES)
df['size'] = size
df['size_c'] = size_c
df['size_nc'] = size_nc
else:
df['size'] = size
df['size_c'] = size_c
df['size_nc'] = size_nc
# form the two Categories: Equipment and Structures
equipment_df = df[(~df.asset_category.str.contains('Structures'))
& (~df.asset_category.str.contains('Buildings'))]
structure_df = df[(df.asset_category.str.contains('Structures')) |
(df.asset_category.str.contains('Buildings'))]
format_fields = [
'metr_c', 'metr_nc', 'metr_c_d', 'metr_nc_d', 'metr_c_e',
'metr_nc_e', 'mettr_c', 'mettr_nc', 'mettr_c_d', 'mettr_nc_d',
'mettr_c_e', 'mettr_nc_e', 'rho_c', 'rho_nc', 'rho_c_d',
'rho_nc_d', 'rho_c_e', 'rho_nc_e', 'z_c', 'z_nc', 'z_c_d',
'z_nc_d', 'z_c_e', 'z_nc_e'
]
# Make short category
make_short = {
'Instruments and Communications Equipment':
'Instruments and Communications',
'Office and Residential Equipment': 'Office and Residential',
'Other Equipment': 'Other',
'Transportation Equipment': 'Transportation',
'Other Industrial Equipment': 'Other Industrial',
'Nonresidential Buildings': 'Nonresidential Bldgs',
'Residential Buildings': 'Residential Bldgs',
'Mining and Drilling Structures': 'Mining and Drilling',
'Other Structures': 'Other',
'Computers and Software': 'Computers and Software',
'Industrial Machinery': 'Industrial Machinery'
}
equipment_df['short_category'] = equipment_df['asset_category'].map(
make_short)
structure_df['short_category'] = structure_df['asset_category'].map(
make_short)
# Add the Reform and the Baseline to Equipment Asset
for f in format_fields:
equipment_copy = equipment_df.copy()
equipment_copy['rate'] = equipment_copy[f]
equipment_copy['hover'] = equipment_copy.apply(
lambda x: "{0:.1f}%".format(x[f] * 100), axis=1)
simple_equipment_copy = equipment_copy.filter(items=[
'size', 'size_c', 'size_nc', 'rate', 'hover', 'short_category',
'Asset'
])
data_sources[list_string[i] + '_equipment_' +
f] = ColumnDataSource(simple_equipment_copy)
# Add the Reform and the Baseline to Structures Asset
for f in format_fields:
structure_copy = structure_df.copy()
structure_copy['rate'] = structure_copy[f]
structure_copy['hover'] = structure_copy.apply(
lambda x: "{0:.1f}%".format(x[f] * 100), axis=1)
simple_structure_copy = structure_copy.filter(items=[
'size', 'size_c', 'size_nc', 'rate', 'hover', 'short_category',
'Asset'
])
data_sources[list_string[i] + '_structure_' +
f] = ColumnDataSource(simple_structure_copy)
# Create initial data sources to plot on load
if list_string[i] == 'base':
equipment_copy = equipment_df.copy()
equipment_copy['rate'] = equipment_copy['mettr_c']
equipment_copy['hover'] = equipment_copy.apply(
lambda x: "{0:.1f}%".format(x['mettr_c'] * 100), axis=1)
simple_equipment_copy = equipment_copy.filter(items=[
'size', 'size_c', 'size_nc', 'rate', 'hover', 'short_category',
'Asset'
])
data_sources['equip_source'] = ColumnDataSource(
simple_equipment_copy)
structure_copy = structure_df.copy()
structure_copy['rate'] = structure_copy['mettr_c']
structure_copy['hover'] = structure_copy.apply(
lambda x: "{0:.1f}%".format(x['mettr_c'] * 100), axis=1)
simple_structure_copy = structure_copy.filter(items=[
'size', 'size_c', 'size_nc', 'rate', 'hover', 'short_category',
'Asset'
])
data_sources['struc_source'] = ColumnDataSource(
simple_structure_copy)
# Define categories for Equipments assets
equipment_assets = [
'Computers and Software', 'Instruments and Communications',
'Office and Residential', 'Transportation', 'Industrial Machinery',
'Other Industrial', 'Other'
]
# Define categories for Structures assets
structure_assets = [
'Residential Bldgs', 'Nonresidential Bldgs', 'Mining and Drilling',
'Other'
]
# Equipment plot
p = figure(
plot_height=540,
plot_width=990,
y_range=list(reversed(equipment_assets)),
tools='hover',
background_fill_alpha=0,
title=
'Marginal Effective Total Tax Rates on Corporate Investments in Equipment'
)
p.title.align = 'center'
p.title.text_color = '#6B6B73'
hover = p.select(dict(type=HoverTool))
hover.tooltips = [('Asset', ' @Asset (@hover)')]
p.xaxis.axis_label = "Marginal effective total tax rate"
p.xaxis[0].formatter = NumeralTickFormatter(format="0.1%")
p.toolbar_location = None
p.min_border_right = 5
p.outline_line_width = 5
p.border_fill_alpha = 0
p.xaxis.major_tick_line_color = "firebrick"
p.xaxis.major_tick_line_width = 3
p.xaxis.minor_tick_line_color = "orange"
p.outline_line_width = 1
p.outline_line_alpha = 1
p.outline_line_color = "black"
p.circle(x='rate',
y='short_category',
color=BLUE,
size='size',
line_color="#333333",
fill_alpha=.4,
source=data_sources['equip_source'],
alpha=.4)
# Style the tools
p.add_tools(WheelZoomTool(), ResetTool(), SaveTool())
p.toolbar_location = "right"
p.toolbar.logo = None
# Define and add a legend
legend_cds = ColumnDataSource({
'size': SIZES,
'label': ['<$20B', '', '', '<$1T'],
'x': [0, .15, .35, .6]
})
p_legend = figure(height=150,
width=480,
x_range=(-0.075, .75),
title='Asset Amount')
p_legend.circle(y=None,
x='x',
size='size',
source=legend_cds,
color=BLUE,
fill_alpha=.4,
alpha=.4,
line_color="#333333")
l = LabelSet(y=None,
x='x',
text='label',
x_offset=-20,
y_offset=-50,
source=legend_cds)
p_legend.add_layout(l)
p_legend.axis.visible = False
p_legend.grid.grid_line_color = None
p_legend.toolbar.active_drag = None
data_sources['equip_plot'] = p
# Structures plot
p2 = figure(
plot_height=540,
plot_width=990,
y_range=list(reversed(structure_assets)),
tools='hover',
background_fill_alpha=0,
title=
'Marginal Effective Total Tax Rates on Corporate Investments in Structures'
)
p2.title.align = 'center'
p2.title.text_color = '#6B6B73'
hover = p2.select(dict(type=HoverTool))
hover.tooltips = [('Asset', ' @Asset (@hover)')]
p2.xaxis.axis_label = "Marginal effective total tax rate"
p2.xaxis[0].formatter = NumeralTickFormatter(format="0.1%")
p2.toolbar_location = None
p2.min_border_right = 5
p2.outline_line_width = 0
p2.border_fill_alpha = 0
p2.xaxis.major_tick_line_color = "firebrick"
p2.xaxis.major_tick_line_width = 3
p2.xaxis.minor_tick_line_color = "orange"
p2.circle(x='rate',
y='short_category',
color=RED,
size='size',
line_color="#333333",
fill_alpha=.4,
source=data_sources['struc_source'],
alpha=.4)
p2.outline_line_width = 1
p2.outline_line_alpha = 1
p2.outline_line_color = "black"
# Style the tools
p2.add_tools(WheelZoomTool(), ResetTool(), SaveTool())
p2.toolbar_location = "right"
p2.toolbar.logo = None
# Define and add a legend
p2_legend = figure(height=150,
width=380,
x_range=(-0.075, .75),
title='Asset Amount')
p2_legend.circle(y=None,
x='x',
size='size',
source=legend_cds,
color=RED,
fill_alpha=.4,
alpha=.4,
line_color="#333333")
l2 = LabelSet(y=None,
x='x',
text='label',
x_offset=-20,
y_offset=-50,
source=legend_cds)
p2_legend.add_layout(l2)
p2_legend.axis.visible = False
p2_legend.grid.grid_line_color = None
p2_legend.toolbar.active_drag = None
data_sources['struc_plot'] = p2
# add buttons
controls_callback = CustomJS(args=data_sources,
code=CONTROLS_CALLBACK_SCRIPT)
c_nc_buttons = RadioButtonGroup(labels=['Corporate', 'Noncorporate'],
active=0,
callback=controls_callback)
controls_callback.args['c_nc_buttons'] = c_nc_buttons
format_buttons = RadioButtonGroup(labels=['Baseline', 'Reform', 'Change'],
active=0,
callback=controls_callback)
controls_callback.args['format_buttons'] = format_buttons
interest_buttons = RadioButtonGroup(
labels=['METTR', 'METR', 'Cost of Capital', 'Depreciation'],
active=0,
width=700,
callback=controls_callback)
controls_callback.args['interest_buttons'] = interest_buttons
type_buttons = RadioButtonGroup(
labels=['Typically Financed', 'Equity Financed', 'Debt Financed'],
active=0,
width=700,
callback=controls_callback)
controls_callback.args['type_buttons'] = type_buttons
# Create Tabs
tab = Panel(child=column([p, p_legend]), title='Equipment')
tab2 = Panel(child=column([p2, p2_legend]), title='Structures')
tabs = Tabs(tabs=[tab, tab2])
layout = gridplot(children=[[tabs], [c_nc_buttons, interest_buttons],
[format_buttons, type_buttons]])
# Create components
js, div = components(layout)
cdn_js = CDN.js_files[0]
cdn_css = CDN.css_files[0]
return js, div, cdn_js, cdn_css
def plot(self, source1, source2, source3, source4, source5, source6,
source7, source8, source9):
a = self.a
b = self.b
hover = HoverTool(tooltips=[
("PCA1", "($x)"),
("PCA2", "($y)"),
("NAME", "(@N)"),
])
p = figure(title="CHEMICAL SPACE BY MORGAN2 FP",
x_axis_label="PC 1 " + "(" + str(a) + "%)",
y_axis_label="PC 2 " + "(" + str(b) + "%)",
x_range=(-7, 7),
y_range=(-7, 7),
tools=[hover],
plot_width=1000,
plot_height=800)
FDA_plot = p.circle(x="x",
y="y",
source=source1,
color="darkslateblue",
size=5)
PPI_plot = p.circle(x="x",
y="y",
source=source2,
color="yellowgreen",
size=5)
MACRO_plot = p.circle(x="x",
y="y",
source=source3,
color="lightsteelblue",
size=5)
NP_plot = p.circle(x="x", y="y", source=source4, color="olive", size=5)
PEP_FDA_plot = p.circle(x="x",
y="y",
source=source5,
color="darkslategray",
size=5)
LIN_plot = p.circle(x="x",
y="y",
source=source6,
color="aquamarine",
size=5)
LIN_NM_plot = p.circle(x="x",
y="y",
source=source7,
color="teal",
size=5)
CYC_plot = p.circle(x="x",
y="y",
source=source8,
color="lightpink",
size=5)
CYC_NM_plot = p.circle(x="x",
y="y",
source=source9,
color="mediumvioletred",
size=5)
p.add_tools(LassoSelectTool(), ZoomInTool(), ZoomOutTool(), SaveTool(),
PanTool())
legend = Legend(items=[
("FDA", [FDA_plot]),
("PPI", [PPI_plot]),
("MACRO", [MACRO_plot]),
("NP", [NP_plot]),
("PEP FDA", [PEP_FDA_plot]),
("LIN", [LIN_plot]),
("LIN NM", [LIN_NM_plot]),
("CYC", [CYC_plot]),
("CYC NM", [CYC_NM_plot]),
],
location="center",
orientation="vertical",
click_policy="hide")
p.add_layout(legend, place='right')
p.xaxis.axis_label_text_font_size = "20pt"
p.yaxis.axis_label_text_font_size = "20pt"
p.xaxis.axis_label_text_color = "black"
p.yaxis.axis_label_text_color = "black"
p.xaxis.major_label_text_font_size = "18pt"
p.yaxis.major_label_text_font_size = "18pt"
p.title.text_font_size = "22pt"
return p
def add_graph(self,
field_names,
legends,
window='hann',
window_length=256,
noverlap=128):
""" add a spectrogram plot to the graph
field_names: can be a list of fields from the data set, or a list of
functions with the data set as argument and returning a tuple of
(field_name, data)
legends: description for the field_names that will appear in the title of the plot
window: the type of window to use for the frequency analysis. check scipy documentation for available window types.
window_length: length of the analysis window in samples.
noverlap: number of overlapping samples between windows.
"""
if self._had_error: return
try:
data_set = {}
data_set['timestamp'] = self._cur_dataset.data['timestamp']
# calculate the sampling frequency
# (Note: logging dropouts are not taken into account here)
delta_t = ((data_set['timestamp'][-1] - data_set['timestamp'][0]) *
1.0e-6) / len(data_set['timestamp'])
sampling_frequency = int(1.0 / delta_t)
if sampling_frequency < 100: # require min sampling freq
self._had_error = True
return
field_names_expanded = self._expand_field_names(
field_names, data_set)
# calculate the spectrogram
psd = dict()
for key in field_names_expanded:
frequency, time, psd[key] = scipy.signal.spectrogram(
data_set[key],
fs=sampling_frequency,
window=window,
nperseg=window_length,
noverlap=noverlap,
scaling='density')
# sum all psd's
key_it = iter(psd)
sum_psd = psd[next(key_it)]
for key in key_it:
sum_psd += psd[key]
# offset = int(((1024/2.0)/250.0)*1e6)
# scale time to microseconds and add start time as offset
time = time * 1.0e6 + self._cur_dataset.data['timestamp'][0]
color_mapper = LinearColorMapper(palette=viridis(256),
low=-80,
high=0)
image = [10 * np.log10(sum_psd)]
title = self.title
for legend in legends:
title += " " + legend
title += " [dB]"
# assume maximal data points per pixel at full resolution
max_num_data_points = 2.0 * self._config['plot_width']
if len(time) > max_num_data_points:
step_size = int(len(time) / max_num_data_points)
time = time[::step_size]
image[0] = image[0][:, ::step_size]
self._p.y_range = Range1d(frequency[0], frequency[-1])
self._p.toolbar_location = 'above'
self._p.image(image=image,
x=time[0],
y=frequency[0],
dw=(time[-1] - time[0]),
dh=(frequency[-1] - frequency[0]),
color_mapper=color_mapper)
color_bar = ColorBar(color_mapper=color_mapper,
major_label_text_font_size="5pt",
ticker=BasicTicker(desired_num_ticks=5),
formatter=PrintfTickFormatter(format="%f"),
title='[dB]',
label_standoff=6,
border_line_color=None,
location=(0, 0))
self._p.add_layout(color_bar, 'right')
# add plot zoom tool that only zooms in time axis
wheel_zoom = WheelZoomTool()
self._p.toolbar.tools = [
PanTool(), wheel_zoom,
BoxZoomTool(dimensions="width"),
ResetTool(),
SaveTool()
] # updated_tools
self._p.toolbar.active_scroll = wheel_zoom
except (KeyError, IndexError, ValueError, ZeroDivisionError) as error:
print(type(error), "(" + self._data_name + "):", error)
self._had_error = True
<div>
<img
src="@imgs" height="@imgs_ht" alt="@imgs" width="@imgs_wd"
style="float: left; margin: 0px 15px 15px 0px;"
border="1"
></img>
</div>
<div>
<img
src="@flag" height="@flag_ht" alt="@imgs" width="@flag_wd"
></img>
<span style="font-size: 14px; font-weight: bold;">@name</span>
</div>
<div>
<span style="font-size: 10px: bold;">@addr</span>
</div>
</div>
""")
wheel_zoom = WheelZoomTool()
tap = TapTool()
tap.renderers = []
tap.callback = OpenURL(url='@embassy_url')
plot.add_tools(PanTool(), wheel_zoom, hover, tap, ResetTool(), SaveTool())
plot.toolbar.active_scroll = wheel_zoom
### Export
show(plot)
def modify_doc(doc):
print("Someone is viewing the Graph")
with open(sys.argv[3] + 'lossdump.csv', 'r') as f:
reader = csv.reader(f)
array = np.array(list(reader))
iterationarray = (array[:, 0]).tolist()
lossarray = (array[:, 1]).tolist()
learningratearray = (array[:, 2]).tolist()
imgprocessarray = (array[:, 3]).tolist()
for i in range(0, len(iterationarray)):
iterationarray[i] = int(float(iterationarray[i].split(':')[1]))
lossarray[i] = float(lossarray[i].split(':')[1])
learningratearray[i] = float(learningratearray[i].split(':')[1])
imgprocessarray[i] = int(imgprocessarray[i].split(':')[1])
hover = HoverTool(tooltips=[
("index", "$index"),
("(x,y)", "(@x,@y)"),
])
# tck = interpolate.UnivariateSpline(imgprocessarray, lossarray)
# xnew = np.linspace(min(lossarray), max(lossarray), num=len(lossarray), endpoint=True)
# check = tck(xnew)
# print(type(check))
# print(check.shape)
fig1 = figure(title='Training Loss Plot',
width=1500,
height=400,
x_axis_label='Images Processed',
y_axis_label='Training Loss',
tools=[BoxZoomTool(),
ResetTool(), hover,
SaveTool()])
fig1.line(x=imgprocessarray, y=lossarray, line_width=1)
# fig1.line(x=imgprocessarray, y=check.tolist(),line_width=1,line_color="#f46d43" )
fig1.xaxis[0].formatter.use_scientific = False
doc.add_root(fig1)
fig2 = figure(title='Learning Rate Plot',
width=1500,
height=400,
x_axis_label='Images Processed',
y_axis_label='Learning Rate Loss',
tools=[BoxZoomTool(),
ResetTool(), hover,
SaveTool()])
fig2.line(x=imgprocessarray, y=learningratearray, line_width=1)
fig2.xaxis[0].formatter.use_scientific = False
doc.add_root(fig2)
cfg_file = sys.argv[2]
with open(sys.argv[2]) as file:
losslines = file.read().splitlines()
subdivision = 0
if ("[yolo]") in losslines:
num_of_yolo_layers = losslines.count("[yolo]")
else:
print("Change the Detection Layer from YOLO to REGION/DETECTION")
print("Classification Models Visualization not supported now")
exit(1)
list2d = []
AvgIOU, Class, Obj, NoObj, FIFTYR, SEVENFIVER = [[], []], [[], []], [
[], []
], [[], []], [[], []], [[], []]
count = 0
linenumber = 0
tempavgiou, tempclass, tempobj, tempnoobj, temp50R, temp75R = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
for index in range(0, len(losslines)):
if losslines[index].find(
"subdivision") != -1 and losslines[index].find("#") == -1:
subdivision = losslines[index]
subdivision = int(re.findall(r'\b\d+\b', subdivision)[0])
if subdivision == 0:
print("Check the subdivisions of your CFG File")
exit(1)
with open(sys.argv[3] + "detectiondump.csv") as f:
lines = f.readlines()
linenumber = 0
batchcount = 0
counter = 0
tempavgiou,tempclass,tempobj,tempnoobj,temp50R,temp75R = [],[],[],[],[],[]
AvgIOU, Class, Obj, NoObj, FIFTYR, SEVENFIVER = [], [], [], [], [], []
num_lines = len(lines)
for index in range(0, num_of_yolo_layers):
tempavgiou.append([0.0])
tempclass.append([0.0])
tempobj.append([0.0])
tempnoobj.append([0.0])
temp50R.append([0.0])
temp75R.append([0.0])
AvgIOU.append([])
Class.append([])
Obj.append([])
NoObj.append([])
FIFTYR.append([])
SEVENFIVER.append([])
while linenumber < num_lines - 1:
for index in range(0, num_of_yolo_layers):
currentline = lines[linenumber].strip()
if currentline.find("Images_Processed") != -1:
for index_inner in range(0, num_of_yolo_layers):
AvgIOU[index_inner].append(tempavgiou[index_inner][0] * 2 /
batchcount)
Class[index_inner].append(tempclass[index_inner][0] * 2 /
batchcount)
Obj[index_inner].append(tempobj[index_inner][0] * 2 /
batchcount)
NoObj[index_inner].append(tempnoobj[index_inner][0] * 2 /
batchcount)
FIFTYR[index_inner].append(temp50R[index_inner][0] * 2 /
batchcount)
SEVENFIVER[index_inner].append(temp75R[index_inner][0] *
2 / batchcount)
linenumber += 1
batchcount = 0
tempavgiou,tempclass,tempobj,tempnoobj,temp50R,temp75R = [],[],[],[],[],[]
for index in range(0, num_of_yolo_layers):
tempavgiou.append([0.0])
tempclass.append([0.0])
tempobj.append([0.0])
tempnoobj.append([0.0])
temp50R.append([0.0])
temp75R.append([0.0])
else:
if currentline.split(',')[1].split(':')[1] != "-nan":
tempavgiou[index][0] = tempavgiou[index][0] + float(
currentline.split(',')[1].split(':')[1])
tempclass[index][0] = tempclass[index][0] + float(
currentline.split(',')[2].split(':')[1])
tempobj[index][0] = tempobj[index][0] + float(
currentline.split(',')[3].split(':')[1])
tempnoobj[index][0] = tempnoobj[index][0] + float(
currentline.split(',')[4].split(':')[1])
temp50R[index][0] = temp50R[index][0] + float(
currentline.split(',')[5].split(':')[1])
temp75R[index][0] = temp75R[index][0] + float(
currentline.split(',')[6].split(':')[1])
linenumber += 1
batchcount += 1
for index in range(0, num_of_yolo_layers):
fig3 = figure(title='IOU Yolo Layer ' + str(index + 1),
width=1500,
height=400,
x_axis_label='Images Processed',
y_axis_label='IOU of Yolo Layer ' + str(index + 1),
tools=[BoxZoomTool(),
ResetTool(), hover,
SaveTool()])
fig3.line(x=imgprocessarray, y=AvgIOU[index], line_width=1)
fig3.xaxis[0].formatter.use_scientific = False
doc.add_root(fig3)
for index in range(0, num_of_yolo_layers):
fig4 = figure(title='Class Yolo Layer ' + str(index + 1),
width=1500,
height=400,
x_axis_label='Images Processed',
y_axis_label='Class of Yolo Layer' + str(index + 1),
tools=[BoxZoomTool(),
ResetTool(), hover,
SaveTool()])
fig4.line(x=imgprocessarray, y=Class[index], line_width=1)
fig4.xaxis[0].formatter.use_scientific = False
doc.add_root(fig4)
for index in range(0, num_of_yolo_layers):
fig5 = figure(title='Objectness Yolo Layer ' + str(index + 1),
width=1500,
height=400,
x_axis_label='Images Processed',
y_axis_label='Objectness of Yolo Layer' + str(index + 1),
tools=[BoxZoomTool(),
ResetTool(), hover,
SaveTool()])
fig5.line(x=imgprocessarray, y=Obj[index], line_width=1)
fig5.xaxis[0].formatter.use_scientific = False
doc.add_root(fig5)
for index in range(0, num_of_yolo_layers):
fig6 = figure(title='No Objectness Yolo Layer ' + str(index + 1),
width=1500,
height=400,
x_axis_label='Images Processed',
y_axis_label='No Objectness of Yolo Layer' +
str(index + 1),
tools=[BoxZoomTool(),
ResetTool(), hover,
SaveTool()])
fig6.line(x=imgprocessarray, y=NoObj[index], line_width=1)
fig6.xaxis[0].formatter.use_scientific = False
doc.add_root(fig6)
for index in range(0, num_of_yolo_layers):
fig7 = figure(title='.5 IOU Recall Yolo Layer ' + str(index + 1),
width=1500,
height=400,
x_axis_label='Images Processed',
y_axis_label='.5 IOU Recall of Yolo Layer' +
str(index + 1),
tools=[BoxZoomTool(),
ResetTool(), hover,
SaveTool()])
fig7.line(x=imgprocessarray, y=FIFTYR[index], line_width=1)
fig7.xaxis[0].formatter.use_scientific = False
doc.add_root(fig7)
for index in range(0, num_of_yolo_layers):
fig8 = figure(title='.75 IOU Recall Yolo Layer ' + str(index + 1),
width=1500,
height=400,
x_axis_label='Images Processed',
y_axis_label='.75 IOU Recall of Yolo Layer' +
str(index + 1),
tools=[BoxZoomTool(),
ResetTool(), hover,
SaveTool()])
fig8.line(x=imgprocessarray, y=SEVENFIVER[index], line_width=1)
fig8.xaxis[0].formatter.use_scientific = False
doc.add_root(fig8)
doc.title = "Darknet Visualization!"
def large_plot(n):
from bokeh.models import (Plot, LinearAxis, Grid, GlyphRenderer,
ColumnDataSource, DataRange1d, PanTool,
ZoomInTool, ZoomOutTool, WheelZoomTool,
BoxZoomTool, BoxSelectTool, SaveTool, ResetTool)
from bokeh.models.layouts import Column
from bokeh.models.glyphs import Line
col = Column()
objects = set([col])
for i in xrange(n):
source = ColumnDataSource(data=dict(x=[0, i + 1], y=[0, i + 1]))
xdr = DataRange1d()
ydr = DataRange1d()
plot = Plot(x_range=xdr, y_range=ydr)
xaxis = LinearAxis(plot=plot)
yaxis = LinearAxis(plot=plot)
xgrid = Grid(plot=plot, dimension=0)
ygrid = Grid(plot=plot, dimension=1)
tickers = [
xaxis.ticker, xaxis.formatter, yaxis.ticker, yaxis.formatter
]
glyph = Line(x='x', y='y')
renderer = GlyphRenderer(data_source=source, glyph=glyph)
plot.renderers.append(renderer)
pan = PanTool()
zoom_in = ZoomInTool()
zoom_out = ZoomOutTool()
wheel_zoom = WheelZoomTool()
box_zoom = BoxZoomTool()
box_select = BoxSelectTool()
save = SaveTool()
reset = ResetTool()
tools = [
pan, zoom_in, zoom_out, wheel_zoom, box_zoom, box_select, save,
reset
]
plot.add_tools(*tools)
col.children.append(plot)
objects |= set([
xdr,
ydr,
xaxis,
yaxis,
xgrid,
ygrid,
renderer,
renderer.view,
glyph,
source,
source.selected,
source.selection_policy,
plot,
plot.x_scale,
plot.y_scale,
plot.toolbar,
plot.title,
box_zoom.overlay,
box_select.overlay,
] + tickers + tools)
return col, objects
# Hover
hover = HoverTool(tooltips=[
("||", "@orient"),
("Fe", "Fo# @fonum{0.0} @buffer"),
("Heat", "@hours hrs at @celsius C "),
("log10D", "@logD{0.0} in m2/s"),
("Type", "@exper, @mech"),
("Source", "@author @year @name"),
])
# The figure
left, right, bottom, top = 6, 10, -16, -8
p = figure(plot_width=500,
plot_height=500,
tools=[BoxZoomTool(), hover,
SaveTool(), ResetTool()],
title="H diffusion in olivine",
x_range=Range1d(left, right),
y_range=Range1d(bottom, top))
p.circle('x', 'y', size=10, source=source, color='color', alpha='alpha')
p.xaxis.axis_label = "1e4 / Temperature (K)"
p.yaxis.axis_label = "log10 Diffusivity (m2/s)"
# background area fills
wD = 1.8
leftD = -10.4
rightD = -12.8
pvd = wD
p.patch([6, 6, 10, 10], [leftD, leftD + wD, rightD + wD, rightD],
alpha=0.075,
line_width=0,