def bokeh_plot(dataframe, export_dir, name, plot_size=850):
png_path = os.path.join(export_dir, 't-SNE_' + name + '_vectors.png')
html_path = os.path.join(export_dir, 't-SNE_' + name + '_vectors.html')
title = '[t-SNE] Java/StackOverflow ' + name + ' vectors in 2D space'
output_file(html_path, title=title)
# plot tools
hover = HoverTool(tooltips = '@tag')
tools = [hover, WheelZoomTool(), BoxZoomTool(), ResetTool(), PanTool()]
#tools = [WheelZoomTool(), BoxZoomTool(), ResetTool(), PanTool()]
# add DataFrame as a ColumnDataSource for Bokeh
plot_data = ColumnDataSource(dataframe)
# create the plot and configure the title, dimensions, and tools
tsne_plot = figure(
title=title,
tools=tools,
plot_width = plot_size,
plot_height = plot_size)
tsne_plot.circle(
'x_coord', 'y_coord',
source=plot_data, color='color', legend='legend',
line_alpha=0.2, fill_alpha=0.5, size=10, hover_line_color='black')
# configure visual elements of the plot
tsne_plot.title.text_font_size = value('16pt')
tsne_plot.xaxis.visible = False
tsne_plot.yaxis.visible = False
tsne_plot.grid.grid_line_color = None
tsne_plot.outline_line_color = None
# engage!
show(tsne_plot)
def plot(self, backlog=1000, width=800, height=300, **kwargs):
""" Plot streaming dataframe as Bokeh plot
This is fragile. It only works in the classic Jupyter Notebook. It
only works on numeric data. It assumes that the index is a datetime
index
"""
from bokeh.palettes import Category10
from bokeh.io import output_notebook, push_notebook, show
from bokeh.models import value
from bokeh.plotting import figure, ColumnDataSource
output_notebook()
sdf = self.to_frame()
colors = Category10[max(3, min(10, len(sdf.columns)))]
data = {c: [] for c in sdf.columns}
data['index'] = []
cds = ColumnDataSource(data)
if ('x_axis_type' not in kwargs
and np.issubdtype(self.index.dtype, np.datetime64)):
kwargs['x_axis_type'] = 'datetime'
fig = figure(width=width, height=height, **kwargs)
for i, column in enumerate(sdf.columns):
color = colors[i % len(colors)]
fig.line(source=cds,
x='index',
y=column,
color=color,
legend=value(column))
fig.legend.click_policy = 'hide'
fig.min_border_left = 30
fig.min_border_bottom = 30
result = show(fig, notebook_handle=True)
from tornado.ioloop import IOLoop
loop = IOLoop.current()
def push_data(df):
df = df.reset_index()
d = {c: df[c] for c in df.columns}
def _():
cds.stream(d, backlog)
push_notebook(handle=result)
loop.add_callback(_)
return {
'figure': fig,
'cds': cds,
'stream': sdf.stream.gather().map(push_data)
}
def word2vec_tsne_vis(word_vectors,
fpath=fpathroot + fpathappend,
dims=[800, 800],
colors='blue',
topn=5000,
stopwords=STOP_WORDS):
"""
Have not finished yet
Takes word_vectors dataframe (output from _word2vec_dataframe_)
and outputs tsne representation of w2v terms in 2D.
"""
from bokeh.plotting import figure, show, output_notebook
from bokeh.models import HoverTool, ColumnDataSource, value
tsne_input = word_vectors.drop(stopwords, errors=u'ignore')
tsne_input = tsne_input.head(topn)
tsne = TSNE()
tsne_vectors = tsne.fit_transform(tsne_input.values)
tsne_filepath = fpathroot + fpathappend + '_tsne_model'
tsne_vectors_filepath = fpathroot + fpathappend + 'tsne_vectors.npy'
with open(tsne_filepath, 'w') as f:
pickle.dump(tsne, f)
pd.np.save(tsne_vectors_filepath, tsne_vectors)
tsne_vectors = pd.DataFrame(tsne_vectors,
index=pd.Index(tsne_input.index),
columns=[u'x_coord', u'y_coord'])
tsne_vectors[u'word'] = tsne_vectors.index
output_notebook()
# add our DataFrame as a ColumnDataSource for Bokeh
plot_data = ColumnDataSource(tsne_vectors)
# create the plot and configure the
# title, dimensions, and tools
w, h = dims
tsne_plot = figure(title=u't-SNE Word Embeddings',
plot_width=w,
plot_height=h,
tools=(u'pan, wheel_zoom, box_zoom,'
u'box_select, resize, reset'),
active_scroll=u'wheel_zoom')
# add a hover tool to display words on roll-over
tsne_plot.add_tools(HoverTool(tooltips=u'@word'))
# draw the words as circles on the plot
tsne_plot.circle(u'x_coord',
u'y_coord',
source=plot_data,
color=colors,
line_alpha=0.2,
fill_alpha=0.1,
size=10,
hover_line_color=u'black')
# configure visual elements of the plot
tsne_plot.title.text_font_size = value(u'16pt')
tsne_plot.xaxis.visible = False
tsne_plot.yaxis.visible = False
tsne_plot.grid.grid_line_color = None
tsne_plot.outline_line_color = None
# engage!
show(tsne_plot)
def __init__(self, **kwargs):
data = progress_quads(dict(all={}, memory={}, erred={}, released={}))
self.source = ColumnDataSource(data=data)
x_range = DataRange1d()
y_range = Range1d(-8, 0)
self.root = Plot(id='bk-task-progress-plot',
x_range=x_range,
y_range=y_range,
toolbar_location=None,
**kwargs)
self.root.add_glyph(
self.source,
Quad(top='top',
bottom='bottom',
left='left',
right='right',
fill_color="#aaaaaa",
line_color="#aaaaaa",
fill_alpha=0.2))
self.root.add_glyph(
self.source,
Quad(top='top',
bottom='bottom',
left='left',
right='released-loc',
fill_color="color",
line_color="color",
fill_alpha=0.6))
self.root.add_glyph(
self.source,
Quad(top='top',
bottom='bottom',
left='released-loc',
right='memory-loc',
fill_color="color",
line_color="color",
fill_alpha=1.0))
self.root.add_glyph(
self.source,
Quad(top='top',
bottom='bottom',
left='erred-loc',
right='erred-loc',
fill_color='#000000',
line_color='#000000',
fill_alpha=0.3))
self.root.add_glyph(
self.source,
Text(text='show-name',
y='bottom',
x='left',
x_offset=5,
text_font_size=value('10pt')))
self.root.add_glyph(
self.source,
Text(text='done',
y='bottom',
x='right',
x_offset=-5,
text_align='right',
text_font_size=value('10pt')))
hover = HoverTool(point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 14px; font-weight: bold;">Name:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">All:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@all</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Memory:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@memory</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Erred:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@erred</span>
</div>
""")
self.root.add_tools(hover)
def __init__(self, scheduler, **kwargs):
self.scheduler = scheduler
ps = [p for p in scheduler.plugins if isinstance(p, AllProgress)]
if ps:
self.plugin = ps[0]
else:
self.plugin = AllProgress(scheduler)
data = progress_quads(
dict(all={}, memory={}, erred={}, released={}, processing={}))
self.source = ColumnDataSource(data=data)
x_range = DataRange1d(range_padding=0)
y_range = Range1d(-8, 0)
self.root = figure(id='bk-task-progress-plot',
title='Progress',
x_range=x_range,
y_range=y_range,
toolbar_location=None,
**kwargs)
self.root.line( # just to define early ranges
x=[0, 0.9],
y=[-1, 0],
line_color="#FFFFFF",
alpha=0.0)
self.root.quad(
source=self.source,
top='top',
bottom='bottom',
left='left',
right='right',
fill_color="#aaaaaa",
line_color='#aaaaaa',
fill_alpha=0.1,
line_alpha=0.3,
)
self.root.quad(source=self.source,
top='top',
bottom='bottom',
left='left',
right='released-loc',
fill_color="color",
line_color="color",
fill_alpha=0.6)
self.root.quad(source=self.source,
top='top',
bottom='bottom',
left='released-loc',
right='memory-loc',
fill_color="color",
line_color="color",
fill_alpha=1.0)
self.root.quad(
source=self.source,
top='top',
bottom='bottom',
left='memory-loc',
right='erred-loc',
fill_color='black',
fill_alpha=0.5,
line_alpha=0,
)
self.root.quad(
source=self.source,
top='top',
bottom='bottom',
left='erred-loc',
right='processing-loc',
fill_color='gray',
fill_alpha=0.35,
line_alpha=0,
)
self.root.text(source=self.source,
text='show-name',
y='bottom',
x='left',
x_offset=5,
text_font_size=value('10pt'))
self.root.text(source=self.source,
text='done',
y='bottom',
x='right',
x_offset=-5,
text_align='right',
text_font_size=value('10pt'))
self.root.ygrid.visible = False
self.root.yaxis.minor_tick_line_alpha = 0
self.root.yaxis.visible = False
self.root.xgrid.visible = False
self.root.xaxis.minor_tick_line_alpha = 0
self.root.xaxis.visible = False
hover = HoverTool(point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 14px; font-weight: bold;">Name:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">All:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@all</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Memory:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@memory</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Erred:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@erred</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Ready:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@processing</span>
</div>
""")
self.root.add_tools(hover)
plot_width=700,
plot_height=700,
tools=('pan, wheel_zoom, box_zoom, box_select, reset'),
active_scroll=u'wheel_zoom')
bokeh_figure.add_tools(
HoverTool(tooltips=OrderedDict([('pod_name',
'@pod_name'), ('cluster', '@cluster')])))
# draw the subreddits as circles on the plot
bokeh_figure.circle(u'x',
u'y',
source=plot_data,
line_color=None,
fill_alpha='fill_alpha',
size=10,
hover_line_color=u'black')
# bokeh_figure.x_range.callback = CustomJS(args=dict(source=plot_data), code=jscode)
# bokeh_figure.y_range.callback = CustomJS(args=dict(source=plot_data), code=jscode)
# configure visual elements of the plot
bokeh_figure.title.text_font_size = value('18pt')
bokeh_figure.title.align = 'center'
bokeh_figure.xaxis.visible = False
bokeh_figure.yaxis.visible = False
bokeh_figure.grid.grid_line_color = None
bokeh_figure.outline_line_color = '#222222'
# st.write(bokeh_figure)
def draw_chart(df):
driver = webdriver.Chrome(os.path.join(BASE_DIR, "chromedriver"))
X = df["wv"].to_list()
y = df["cluster"].to_list()
tsne_filepath = "tsne3000.pkl"
if not os.path.exists(tsne_filepath):
tsne = TSNE(random_state=42)
tsne_points = tsne.fit_transform(X)
with open(tsne_filepath, "wb+") as f:
pickle.dump(tsne_points, f)
else: # Cache Hits!
with open(tsne_filepath, "rb") as f:
tsne_points = pickle.load(f)
tsne_df = pd.DataFrame(
tsne_points, index=range(len(X)), columns=["x_coord", "y_coord"]
)
tsne_df["title"] = df["title"].to_list()
tsne_df["tokens_len"] = df["tokens_len"].to_list()
tsne_df["cluster_no"] = y
colormap = {0: "#ffee33", 1: "#00a152", 2: "#2979ff", 3: "#d500f9"}
colors = [colormap[x] for x in tsne_df["cluster_no"]]
tsne_df["color"] = colors
normalized = min_max_normalize(tsne_df.tokens_len.to_list())
tsne_df["radius"] = [5 + x * 10 for x in normalized]
print(tsne_df.to_dict(orient="list"))
plot_data = ColumnDataSource(data=tsne_df.to_dict(orient="list"))
print(plot_data)
tsne_plot = figure(
# title='TSNE Twitter BIO Embeddings',
plot_width=1200,
plot_height=1200,
active_scroll="wheel_zoom",
output_backend="svg",
)
tsne_plot.add_tools(HoverTool(tooltips="@title"))
tsne_plot.circle(
source=plot_data,
x="x_coord",
y="y_coord",
line_alpha=0.6,
fill_alpha=0.6,
size="radius",
fill_color="color",
line_color="color",
)
tsne_plot.title.text_font_size = value("16pt")
tsne_plot.xaxis.visible = True
tsne_plot.yaxis.visible = True
tsne_plot.background_fill_color = None
tsne_plot.border_fill_color = None
tsne_plot.grid.grid_line_color = None
tsne_plot.outline_line_color = None
# tsne_plot.grid.grid_line_color = None
# tsne_plot.outline_line_color = None
show(tsne_plot)
tsne_plot.toolbar.logo = None
tsne_plot.toolbar_location = None
export_svg(
tsne_plot, filename=f"cluster.svg", webdriver=driver,
)
export_png(
tsne_plot, filename=f"cluster.png", webdriver=driver,
)
def __init__(self, **kwargs):
data = progress_quads(dict(all={}, memory={}, erred={}, released={}))
self.source = ColumnDataSource(data=data)
x_range = DataRange1d()
y_range = Range1d(-8, 0)
self.root = Plot(
id='bk-task-progress-plot',
x_range=x_range, y_range=y_range, toolbar_location=None, **kwargs
)
self.root.add_glyph(
self.source,
Quad(top='top', bottom='bottom', left='left', right='right',
fill_color="#aaaaaa", line_color="#aaaaaa", fill_alpha=0.2)
)
self.root.add_glyph(
self.source,
Quad(top='top', bottom='bottom', left='left', right='released-loc',
fill_color="color", line_color="color", fill_alpha=0.6)
)
self.root.add_glyph(
self.source,
Quad(top='top', bottom='bottom', left='released-loc',
right='memory-loc', fill_color="color", line_color="color",
fill_alpha=1.0)
)
self.root.add_glyph(
self.source,
Quad(top='top', bottom='bottom', left='erred-loc',
right='erred-loc', fill_color='#000000', line_color='#000000',
fill_alpha=0.3)
)
self.root.add_glyph(
self.source,
Text(text='show-name', y='bottom', x='left', x_offset=5,
text_font_size=value('10pt'))
)
self.root.add_glyph(
self.source,
Text(text='done', y='bottom', x='right', x_offset=-5,
text_align='right', text_font_size=value('10pt'))
)
hover = HoverTool(
point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 14px; font-weight: bold;">Name:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">All:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@all</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Memory:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@memory</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Erred:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@erred</span>
</div>
"""
)
self.root.add_tools(hover)
def progress_plot(**kwargs):
with log_errors():
from ..diagnostics.progress_stream import progress_quads
data = progress_quads({
'all': {},
'memory': {},
'erred': {},
'released': {}
})
y_range = Range1d(-8, 0)
source = ColumnDataSource(data)
fig = figure(tools='',
toolbar_location=None,
y_range=y_range,
id='bk-progress-plot',
**kwargs)
fig.quad(source=source,
top='top',
bottom='bottom',
left='left',
right='right',
color='#aaaaaa',
alpha=0.2)
fig.quad(source=source,
top='top',
bottom='bottom',
left='left',
right='released-loc',
color='color',
alpha=0.6)
fig.quad(source=source,
top='top',
bottom='bottom',
left='released-loc',
right='memory-loc',
color='color',
alpha=1)
fig.quad(source=source,
top='top',
bottom='bottom',
left='erred-loc',
right='erred-loc',
color='#000000',
alpha=0.3)
fig.text(source=source,
text='show-name',
y='bottom',
x='left',
x_offset=5,
text_font_size=value('10pt'))
fig.text(source=source,
text='done',
y='bottom',
x='right',
x_offset=-5,
text_align='right',
text_font_size=value('10pt'))
fig.xaxis.visible = False
fig.yaxis.visible = False
fig.grid.grid_line_alpha = 0
hover = HoverTool()
fig.add_tools(hover)
hover = fig.select(HoverTool)
hover.tooltips = """
<div>
<span style="font-size: 14px; font-weight: bold;">Name:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">All:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@all</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Memory:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@memory</span>
</div>
<div>
<span style="font-size: 14px; font-weight: bold;">Erred:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@erred</span>
</div>
"""
hover.point_policy = 'follow_mouse'
return source, fig
