def image_figure(key,
source,
im_size=(240, 320),
axes=True,
tools='',
gray=False,
figure=None):
h, w = im_size
if figure is None:
f = bokeh_figure(x_range=(0, w),
y_range=(0, h),
plot_width=w,
plot_height=h,
tools=tools,
tooltips=[("x", "$x"), ("y", "$y"),
("value", "@image")])
else:
f = figure
f.toolbar.logo = None
if not axes:
f.xaxis[0].visible = False
f.yaxis[0].visible = False
# f.image_rgba(key, x=0, y=0, dw=w, dh=h, source=source)
if gray:
f.image(key, x=0, y=0, dw=w, dh=h, source=source)
else:
f.image_rgba(key, x=0, y=0, dw=w, dh=h, source=source)
return f
def make_chart_bokeh(model_name, chart_id, options):
points = make_points(model_name, chart_id)
xx = [point[0] for point in points]
yy = [point[1] for point in points]
if options.get('color') == options.get('bg_color'):
color_to_contrast = options.get('color', '#292928')
options['bg_color'], options['color'] = get_contrasted_colors(
color_to_contrast)
fig_kwargs = dict()
fig_kwargs['title'] = options.get('title', 'Bokeh')
fig_kwargs['width'] = 500
fig_kwargs['height'] = 500
fig_kwargs['x_axis_label'] = options.get('x_label', 'x')
fig_kwargs['y_axis_label'] = options.get('y_label', 'y')
fig_kwargs['toolbar_location'] = None
fig_kwargs['min_border_left'] = 0
fig_kwargs['min_border_right'] = 45
fig_kwargs['min_border_top'] = 0
fig_kwargs['min_border_bottom'] = 0
fig_kwargs['background_fill_color'] = options.get('bg_color', '#cccccc')
flag_scatter_plot = options.get('flag_scatter_plot', False)
flag_show_grid = options.get('flag_show_grid', True)
flag_logscale_x = options.get('flag_logscale_x', False)
flag_logscale_y = options.get('flag_logscale_y', False)
if flag_logscale_x:
fig_kwargs[
'x_axis_type'] = "log" # linear, datetime ,mercator <------------------
else:
fig_kwargs['x_axis_type'] = "linear"
if flag_logscale_y:
fig_kwargs['y_axis_type'] = "log"
else:
fig_kwargs['y_axis_type'] = "linear"
bokeh_chart = bokeh_figure(**fig_kwargs)
bokeh_chart.xgrid.visible = flag_show_grid
bokeh_chart.ygrid.visible = flag_show_grid
plot_kwargs = dict()
plot_kwargs['color'] = options.get(
'color', 'black') # many colors... <------------------
plot_kwargs['line_width'] = options.get('line_width', 2)
plot_kwargs['line_dash'] = options.get(
'line_style', 'solid'
) # solid' 'dashed' 'dotted' 'dotdash' 'dashdot' <------------------
if flag_scatter_plot:
bokeh_chart.scatter(xx, yy, **plot_kwargs)
else:
bokeh_chart.line(xx, yy, **plot_kwargs)
return bokeh_chart
def plot(self, figure=None, figure_kwargs=None, text_kwargs=None):
if figure_kwargs is None:
figure_kwargs = dict()
if text_kwargs is None:
text_kwargs = dict()
if figure is None:
figure = bokeh_figure(**figure_kwargs)
self.layout = figure
self.text = self.layout.text(**text_kwargs, )
def make_image_figure(im_size=(240, 320), axes=True):
w, h = im_size
f = bokeh_figure(x_range=(0, w),
y_range=(0, h),
plot_width=w,
plot_height=h,
tools='',
tooltips=[("x", "$x"), ("y", "$y"), ("value", "@image")])
f.toolbar.logo = None
if not axes:
f.xaxis[0].visible = False
f.yaxis[0].visible = False
return f
def plot(self, figure=None, figure_kwargs=None, segment_kwargs=None):
if figure_kwargs is None:
figure_kwargs = dict()
if segment_kwargs is None:
segment_kwargs = dict()
if figure is None:
figure = bokeh_figure(**figure_kwargs)
self.layout = figure
self.segment = self.layout.segment(
source=self.data_source,
**segment_kwargs,
)
def figure(**kwargs):
fig = bokeh_figure(width=1200,
height=500,
toolbar_location=None,
x_range=(
np.datetime64(date(2020, 3, 1)),
np.datetime64(date.today() + timedelta(days=1)),
),
sizing_mode="scale_width",
tools="",
**kwargs)
add_interventions(fig)
fig.yaxis.formatter = NumeralTickFormatter(format="0,0")
fig.xaxis.formatter = DatetimeTickFormatter(days="%d %b")
fig.xgrid.visible = False
fig.y_range.start = 0
return fig
def plot(self, figure=None, figure_kwargs=None, scatter_kwargs=None):
if figure_kwargs is None:
figure_kwargs = dict()
if scatter_kwargs is None:
scatter_kwargs = dict()
if figure is None:
figure = bokeh_figure(**figure_kwargs)
self.layout = figure
source = dict()
if self.data_source is not None:
source = {'source': self.data_source}
self.scatter = self.layout.circle(
**source,
**scatter_kwargs,
)
def make_image_figure(im_size=(240, 320), axes=False):
w, h = max(im_size), min(im_size)
f = bokeh_figure(x_range=(0, w - 1),
y_range=(0, h - 1),
plot_width=w,
plot_height=h,
title='',
tooltips=[("x", "$x"), ("y", "$y"), ("value", "@image")])
# f.x_range.range_padding = f.y_range.range_padding = 0
f.toolbar.logo = None
f.toolbar_location = None
f.axis.visible = False
f.grid.visible = False
f.min_border = 0
f.outline_line_width = 0
f.outline_line_color = None
f.background_fill_color = None
f.border_fill_color = None
return f
def age_heatmap(age_rate):
age_rate = age_rate.copy()
# age_rate.index = [datetime.combine(date.fromisocalendar(2020, week, 7), datetime.min.time()) for week in age_rate.index]
age_rate.index.name = "Week Ending"
age_rate.columns.name = "Age"
fig = bokeh_figure(
width=1200,
height=300,
title="Infection rate in England by age",
tools="",
toolbar_location=None,
x_range=[age_rate.index[0], age_rate.index[-1]],
y_range=list(age_rate.columns),
)
fig.add_tools(
HoverTool(
tooltips=[
("Week", "@{Week Ending}"),
("Age range", "@Age"),
("Cases", "@rate{0.00} per 100,000"),
],
toggleable=False,
))
df = pd.DataFrame(age_rate.stack(), columns=["rate"]).reset_index()
ds = ColumnDataSource(df)
fig.rect(
"Week Ending",
"Age",
dilate=True,
width=1.001,
height=1.001,
source=ds,
line_color=None,
fill_color=linear_cmap("rate", palette=colorcet.kbc, low=1, high=180),
)
fig.xaxis.axis_label = "Week number"
return fig
def figure(**kwargs):
if "x_range" not in kwargs:
kwargs["x_range"] = (
np.datetime64(date(2020, 3, 1)),
np.datetime64(date.today() + timedelta(days=1)),
)
fig = bokeh_figure(width=1200,
height=500,
toolbar_location="right",
sizing_mode="scale_width",
tools="reset,box_zoom",
**kwargs)
fig.toolbar.logo = None
add_interventions(fig)
legend = Legend()
legend.click_policy = "hide"
fig.add_layout(legend)
fig.xaxis.formatter = DatetimeTickFormatter(days="%d %b", months="%d %b")
fig.xgrid.visible = False
fig.y_range.start = 0
return fig
def content(request):
if request.param == "matplotlib":
f = plt.figure()
yield (f, SVGContentConfiguration)
f.clf()
del f
if request.param == "odd":
yield (OddContent(), FallbackContentConfiguration)
if request.param == "altair":
chart = (altair.Chart(
pd.DataFrame({
"a": ["A", "B", "C", "D", "E", "F", "G", "H", "I"],
"b": [28, 55, 43, 91, 81, 53, 19, 87, 52],
})).mark_bar().encode(x="a", y="b"))
yield (chart, AltairContentConfiguration)
if request.param == "bokeh":
plot = bokeh_figure(plot_width=400, plot_height=300)
plot.circle([random.random() for _ in range(100)],
[random.random() for _ in range(100)])
yield (plot, BokehContentConfiguration)
if request.param == "code":
yield (
CodeContent(code="lambda x: x", language="python"),
CodeContentConfiguration,
)
if request.param == "latex":
yield (
LatexContent(text="$$x = {-b \pm \sqrt{b^2-4ac} \over 2a}.$$"),
LatexContentConfiguration,
)
if request.param == "mathjax":
yield (
MathJaxContent(text="$$x = {-b \pm \sqrt{b^2-4ac} \over 2a}.$$"),
MathJaxContentConfiguration,
)
if request.param == "custom":
yield (
MathJaxContent(text="$$x = {-b \pm \sqrt{b^2-4ac} \over 2a}.$$"),
MathJaxContentConfiguration,
)
def __init__(self, thelenota, figsize=(800, 600)):
# DIMRED widgets
self.thelenota = thelenota
self.figsize = figsize
self.counts = None
#widgets
self.w = DUMMY()
current_cluster_labels = None
wstyle = widgets.Layout(width='200px') #, max_width='120px',
# min_width='120px')
self.w.drfilter_lower_perc_switch = ccwidget(
self.thelenota,
'drfilter_lower_perc_switch',
'bool',
self.dr_name_simple,
False,
lwidth='40px')
self.w.drfilter_log_switch = ccwidget(self.thelenota,
'drfilter_log_switch',
'bool',
self.dr_name_simple,
False,
lwidth='40px')
self.w.drfilter_lower_percentage = ccwidget(
self.thelenota,
'drfilter_lower_percentage',
'float',
self.dr_name_simple,
0.2,
min=0.01,
max=1,
value=0.2)
self.w.drmethod = ccwidget(
self.thelenota,
'dimred_method',
'dropdown',
self.dr_name_simple,
'tsne',
options='pca ica scorpius KernelPCA tsne nmf'.split())
self.w.drperplex = cache_widget_value(widgets.IntSlider(value=67,
min=2,
max=99,
step=5),
67,
self.thelenota,
'dimred_perplexity',
self.dr_name_simple,
fmt='int')
self.w.drangle = cache_widget_value(widgets.FloatSlider(value=0.5,
min=0.05,
max=0.95,
step=0.05),
0.5,
self.thelenota,
'dimred_angle',
self.dr_name_simple,
fmt='float')
self.w.drlrate = cache_widget_value(widgets.IntSlider(value=920,
min=20,
max=10000,
step=50),
920,
self.thelenota,
'dimred_learning_rate',
self.dr_name_simple,
fmt='int')
self.w.drearly = cache_widget_value(widgets.FloatSlider(value=3.5,
min=1,
max=20,
step=0.5),
3.5,
self.thelenota,
'dimred_early_exag',
self.dr_name_simple,
fmt='float')
self.w.dr_tsne_pcavar_cutoff = ccwidget(self.thelenota,
'tsne_pca_var_cutoff',
'float',
self.dr_name_simple,
0.05,
min=0.01,
max=0.2,
step=0.01)
self.w.drrun = widgets.Button(description='GO!')
self.w.drforce = widgets.Checkbox(description='force',
layout=widgets.Layout(width='300px'))
@dcache(self.thelenota, 'dimred_correlating_genes', 'tsv',
self.dr_name)
def get_dr_correlating(*_):
stats, trans = run_dr_2()
c1 = self.thelenota.counttable.apply(
lambda x: pearsonr(x, trans.iloc[:, 0])[0], axis=1)
c2 = self.thelenota.counttable.apply(
lambda x: pearsonr(x, trans.iloc[:, 1])[0], axis=1)
return pd.DataFrame({0: c1, 1: c2})
def get_top_correlating(*_):
d = get_dr_correlating().abs().sum(1).sort_values(ascending=False)
d = d.head(40)
d = d.reset_index()
d = d.apply(lambda x: '{} ({:.3g})'.format(x.iloc[0], x.iloc[1]),
axis=1)
return list(d)
tcolormap = cmapper.CMAPPER(self.thelenota,
extra_intrinsic_methods={
"top DR correlate":
(get_top_correlating,
cmapper.intrinsic_gene_score)
})
self.w.sl_group_name = cache_widget_value(widgets.Text(layout=wstyle),
'self.thelenota',
self.thelenota,
'group_define_name',
self.dr_name_simple)
self.w.sl_group_set = widgets.Text(layout=wstyle)
self.w.sl_group_set_go = widgets.Button(description='set',
layout=wstyle)
self.w.sl_groupextractname = widgets.Text(layout=wstyle)
self.w.sl_group_extract_go = widgets.Button(description='extract')
self.w.clu_method = ccwidget(self.thelenota,
'cluster_method',
'dropdown',
self.dr_name_simple,
'dbscan',
options=['dbscan'])
self.w.clu_dbscan_eps = ccwidget(self.thelenota,
'clu_dbscan_eps_w',
'float',
self.dr_name_simple,
2.5,
min=0.1,
max=10.0,
step=0.1)
self.w.clu_go = widgets.Button(description='Cluster!')
self.w.clu_name = ccwidget(self.thelenota, "cluster_name", "text",
self.dr_name_simple, 'cluster')
self.w.clu_store_go = widgets.Button(description='save')
plotinfo = {}
self.w.html = widgets.HTML()
# data!
samples = self.thelenota.counttable.columns
nosamples = len(samples)
color_mapper = LinearColorMapper(palette="Inferno256",
low=-0.3,
high=2.5)
topgene = self.thelenota.counttable.std(1).sort_values().tail(
1).index[0]
colv = list(self.thelenota.counttable.loc[topgene])
pdata = ColumnDataSource(
dict(x=[random.uniform(-10, 10) for x in range(nosamples)],
y=[random.uniform(-10, 10) for x in range(nosamples)],
desc=list(samples),
size=[0.3] * nosamples,
score=colv))
self.thelenota._pdata = pdata
# Clustering
def run_clustering(*args):
method = self.w.clu_method.value
stats, trans = run_dr_2()
if method == 'dbscan':
labels = run_dbscan(trans, self.w.clu_dbscan_eps.value)
else:
lg.warning('Not implemented cluster method: {}'.format(method))
return
self.thelenota._CLUSTER_LABELS[self.dr_name()] = labels
colv = labels
color_mapper = CategoricalColorMapper(
palette=bokeh.palettes.Category20[20],
factors=list(colv.value_counts().index))
colorbar_mapper = LinearColorMapper(palette='Inferno256',
low=0,
high=0)
bcolorbar.color_mapper = colorbar_mapper
if not bfigure.legend[0].items:
bfigure.legend[0].items.append(blegend)
bplot.data_source.data['score'] = colv
bplot.glyph.fill_color['transform'] = color_mapper
bplot.glyph.line_width = 0
bplot.glyph.line_alpha = 0
bokeh_io.push_notebook(handle=bhandle)
def store_current_cluster(*args):
labels = self.thelenota._CLUSTER_LABELS.get(self.dr_name())
if labels is None:
self.w.html.value = "no cluster defined"
return
cname = self.w.clu_name.value
labels = labels.apply(lambda x: '{}_{}'.format(cname, x))
outfile = self.thelenota.metadata_dir / '{}.tsv'.format(cname)
moutfile = self.thelenota.metadata_dir / '{}.meta.tsv'.format(
cname)
tmeta = pd.DataFrame({cname: labels})
tmeta.to_csv(outfile, sep="\t")
with open(moutfile, 'w') as F:
F.write("{}\tcategorical\n".format(cname))
self.w.html.value = 'saved cluster to {}'.format(outfile)
self.w.clu_store_go.on_click(store_current_cluster)
self.w.clu_go.on_click(run_clustering)
# GROUP SET
def define_group(*args):
groupset = self.w.sl_group_name.value
groupname = self.w.sl_group_set.value
self.thelenota.selected = list(
self.thelenota.counttable.columns.to_series()\
.iloc[list(self.thelenota._DR_INDICI_SEL_)])
if groupset in self.thelenota.metadata_info.index:
tmeta = self.thelenota.get_metadata(groupset)
else:
tmeta = pd.Series('na',
index=self.thelenota.counttable.columns)
tmeta.loc[self.thelenota.selected] = groupname
self.thelenota.metadata_dir.makedirs_p()
outfile = self.thelenota.metadata_dir / '{}.tsv'.format(groupset)
moutfile = self.thelenota.metadata_dir / '{}.meta.tsv'.format(
groupset)
tmeta = pd.DataFrame({groupset: tmeta})
tmeta.to_csv(outfile, sep="\t")
with open(moutfile, 'w') as F:
F.write("{}\tcategorical\n".format(groupset))
run_dr_color()
self.w.sl_group_set_go.on_click(define_group)
# GROUP EXTRACT
#sl_groupextractname_w
def extract_group(*args):
groupname = self.w.sl_groupextractname.value
self.thelenota.selected = list(
self.thelenota.counttable.columns.to_series()\
.iloc[list(self.thelenota._DR_INDICI_SEL_)])
outfile = self.thelenota.counttable_file.dirname() \
/ '{}__{}.tsv'.format(self.thelenota.counttable_name, groupname)
newcounts = self.thelenota.counttable[self.thelenota.selected]
newcounts.to_csv(outfile, sep="\t")
self.w.html.value = "save new count table to {}".format(outfile)
self.w.sl_group_extract_go.on_click(extract_group)
def force_refresh():
"Force to refresh calculations? Or can we use the cache??"
return self.w.drforce.value
@dcache(self.thelenota, 'filtered', 'pickle', self.dr_name_filter,
force_refresh)
def filter_counts(*_):
counts = self.thelenota.counttable
if self.w.drfilter_log_switch.value:
minval = 0.5 * counts[counts > 0].min().min()
lg.warning('log tranform log10(x+{:.4g})'.format(minval))
counts = np.log10(counts + minval)
if self.w.drfilter_lower_perc_switch.value:
perc = self.w.drfilter_lower_percentage.value
csum = counts.sum(1)
counts = counts[csum >= csum.quantile(perc)]
return counts
@dcache(self.thelenota, 'dimred_table', 'mtsv', self.dr_name,
force_refresh)
def run_dr_2(*_):
param = self.get_dr_param()
method = param['method']
del param['method']
if method == 'pca':
stats, trans = qrtask.pca.sync(self.counts)
elif method == 'ica':
stats, trans = rqtask.ica.sync(self.counts)
elif method == 'KernelPCA':
stats, trans = rqtask.kernelPCA.sync(self.counts)
elif method == 'scorpius':
stats, trans = rqtask.scorpius_dr.sync(self.counts)
elif method == 'nmf':
stats, trans = rqtask.nmf.sync(self.counts)
elif method == 'tsne':
stats, trans = rqtask.tsne.sync(self.counts,
self.get_dr_param())
else:
raise NotImplemented
return stats, trans
def set_color_scale():
score = tcolormap.score
method = tcolormap.method
self.warn('set color scale {}/{}'.format(method, tcolormap.value))
color_mapper = tcolormap.colormapper
bplot.glyph.fill_color['transform'] = color_mapper
bplot.data_source.data['score'] = score
if not tcolormap.discrete:
bcolorbar.color_mapper = color_mapper
if tcolormap.discrete:
if not bfigure.legend[0].items:
bfigure.legend[0].items.append(blegend)
else:
if bfigure.legend[0].items:
bfigure.legend[0].items.pop()
bplot.glyph.line_width = 0
bplot.glyph.line_alpha = 0
def _create_title():
cmethod = '{}/{}'.format(tcolormap.method, tcolormap.value)
if self.w.drfilter_lower_perc_switch.value:
cmethod += '/low%{}'.format(
self.w.drfilter_lower_percentage.value)
if self.w.drfilter_log_switch.value:
cmethod += '/log'
cmethod += '/{}/{}'.format(*self.counts.shape)
return '{}/{}'.format(self.thelenota.counttable_name, cmethod)
def _create_scatter_plot(only_color=False):
if (self.w.drfilter_lower_perc_switch.value) or (
self.w.drfilter_log_switch.value):
self.counts = filter_counts()
else:
self.counts = self.thelenota.counttable
self.warn("count table: {}".format(str(self.counts.shape)))
if not only_color:
meta, trans = run_dr_2()
self.thelenota.dr = trans
self.thelenota._dr_meta = meta
self.thelenota._dr_trans = trans
col1, col2 = trans.columns[:2]
d1 = trans[col1]
d2 = trans[col2]
title = self.dr_name().replace('_', ' ')
m = max(d2.max() - d2.min(), d1.max() - d1.min())
bplot.data_source.data['size'] = [0.008 * m] * nosamples
bplot.data_source.data['x'] = d1
bplot.data_source.data['y'] = d2
bfigure.title.text = _create_title()
set_color_scale()
bokeh_io.push_notebook(handle=bhandle)
def run_dr_color(*_):
""" refresh dr scatter plot - color only """
self.w.drrun.button_style = 'info'
try:
_create_scatter_plot(only_color=True)
except:
self.w.drrun.button_style = 'danger'
raise
self.w.drrun.button_style = ''
def run_dr(*_):
self.w.drrun.button_style = 'info'
try:
_create_scatter_plot()
except:
self.w.drrun.button_style = 'danger'
raise
self.w.drrun.button_style = ''
tcolormap.on_change = run_dr_color
self.w.drrun.on_click(run_dr)
# clgenesetchoice_w.observe(run_dr_color, 'value')
# clmetadata_w.observe(run_dr_color, 'value')
# set up bokeh
select_callback = CustomJS(args={'dsource': pdata},
code="""
var indici = dsource.selected['1d'].indices;
console.log(indici);
IPython.notebook.kernel.execute(
'T._DR_INDICI_SEL_ = ' + indici);
""")
bokeh_tools = [
bmodels.HoverTool(tooltips=[
("(x,y)", "($x, $y)"),
("score", "$score"),
("desc", "@desc"),
]),
bmodels.BoxSelectTool(callback=select_callback),
bmodels.PanTool(),
bmodels.WheelZoomTool(),
bmodels.BoxZoomTool(),
bmodels.LassoSelectTool(callback=select_callback),
bmodels.SaveTool(),
bmodels.ResetTool(),
bmodels.HelpTool(),
]
bfigure = bokeh_figure(plot_width=figsize[0],
plot_height=figsize[1],
tools=bokeh_tools,
toolbar_sticky=False,
toolbar_location='left',
title='dimredplot')
bfigure.title.text_color = 'darkgrey'
bfigure.title.text_font_style = 'normal'
bfigure.title.text_font_size = "12px"
self.thelenota._bfigure = bfigure
bplot = bfigure.circle(x='x',
y='y',
radius='size',
source=pdata,
legend='score',
color=dict(field='score',
transform=color_mapper))
self.thelenota._bplot = bplot
bcolorbar = ColorBar(color_mapper=tcolormap.colormapper,
ticker=BasicTicker(),
formatter=BasicTickFormatter(precision=1),
label_standoff=10,
border_line_color=None,
location=(0, 0))
bfigure.add_layout(bcolorbar, 'right')
blegend = bfigure.legend[0].items[0]
bhandle = bokeh_io.show(bfigure, notebook_handle=True)
tab_children = []
tab_children.append(
widgets.VBox([
ilabel('filter sum%', self.w.drfilter_lower_perc_switch,
self.w.drfilter_lower_percentage),
ilabel('log tranform', self.w.drfilter_log_switch),
ilabel('method', self.w.drmethod),
ilabel('perplexity', self.w.drperplex),
ilabel('angle', self.w.drangle),
ilabel('learning rate', self.w.drlrate),
ilabel('early exagg.', self.w.drearly),
ilabel('PCA var. cutoff', self.w.dr_tsne_pcavar_cutoff),
widgets.HBox([self.w.drrun, self.w.drforce]),
]))
tab_children.append(widgets.VBox([tcolormap.prepare_display()]))
tab_children.append(
widgets.VBox([
ilabel('method', self.w.clu_method),
ilabel('dbscan:eps', self.w.clu_dbscan_eps),
ilabel('store', self.w.clu_name, self.w.clu_store_go),
self.w.clu_go
]))
tab_children.append(
widgets.VBox([
ilabel('group define', self.w.sl_group_name,
self.w.sl_group_set, self.w.sl_group_set_go),
ilabel('count extract', self.w.sl_groupextractname,
self.w.sl_group_extract_go),
]))
tabs = widgets.Tab(children=tab_children)
tabs.set_title(0, 'DimRed')
tabs.set_title(1, 'Color')
tabs.set_title(2, 'Cluster')
tabs.set_title(3, 'Select')
tabs.selected_index = 0
display(tabs)
display(self.w.html)
#run a few on_change functions so that all is in sync
#on_colormethod_change()
run_dr()
import random
from bokeh.plotting import figure as bokeh_figure
from vizno.report import Report
r = Report(title="My bokeh", datetime="The datetime", description="a very basic report")
plot = bokeh_figure(plot_width=400, plot_height=300)
plot.circle(
[random.random() for _ in range(100)], [random.random() for _ in range(100)]
)
r.widget(plot, name="A bokeh widget")
def la_rate_plot(data, names, region, rolling_days=7):
data = (data["cases_norm"].resample(date="1D").nearest(
tolerance="1D").ffill("date").fillna(0).diff("date")[:, :-4].rolling(
date=rolling_days, center=True).sum().dropna("date"))
palette = [
(0, "#f1f1f1"),
(5, "#fef0d9"),
(10, "#fdd49e"),
(25, "#fdbb84"),
(50, "#fc8d59"),
(100, "#e34a33"),
(100000, "#b30000"),
]
def colour_val(cases):
for threshold, colour in palette:
if cases <= threshold:
return colour
return palette[-1][1]
colours = []
y_range = []
yname = []
xname = []
for la in names.index:
if la not in data["gss_code"]:
continue
name = names[la]
y_range.append(name)
xname += list(data["date"].values)
for val in data.sel(gss_code=la).values * 100000 * (7 / rolling_days):
yname.append(name)
colours.append(colour_val(val))
x_range = [data["date"].min().values, data["date"].max().values]
height = len(y_range) * 12 + 100
fig = bokeh_figure(
y_range=y_range,
x_range=x_range,
width=1200,
height=height,
title=region,
sizing_mode="scale_width",
tools="",
toolbar_location=None,
)
data_source = {"y": yname, "x": xname, "colours": colours}
fig.rect(
"x",
"y",
1.01 * (60 * 60 * 24 * 1000),
0.85,
source=data_source,
color="colours",
line_color=None,
dilate=True,
)
fig.axis.axis_line_color = None
fig.grid.grid_line_color = None
fig.yaxis.major_tick_line_color = None
fig.add_layout(DatetimeAxis(), "above")
fig.xaxis.formatter = DatetimeTickFormatter(days="%d %b", months="%d %b")
return fig
def GENE(self):
def gcw(*args, **kwargs):
return self.ccwidget('geneview_' + args[0],
*args[1:],
**kwargs,
setnamer='gene')
#widgets
w = dict(
gene=gcw('gene_name', 'text'),
warn=widgets.HTML(),
)
# data
samples = self.counttable.columns
nosamples = len(samples)
#gene mapper
gmapper = CMAPPER(self, name='gene')
gmapper.method = 'gene'
defgene = self.counttable.std(1).sort_values().tail(1).index[0]
gmapper.value = defgene
w['gene'].value = defgene
#color mapper
cmapper = CMAPPER(self, name='color')
cmapper.method = 'metadata'
cmapper.value = 'plate'
#sort order
smapper = CMAPPER(self, name='sort')
smapper2 = CMAPPER(self, name='sort 2')
def get_sort_order():
sdf = pd.DataFrame({
1: smapper.score.sort_values(),
2: smapper2.score.sort_values()
})
sdf = sdf.sort_values(by=[1, 2])
return sdf.index
sort_order = get_sort_order()
pdata = ColumnDataSource(
dict(
x=pd.Series(range(nosamples), index=sort_order),
y=gmapper.score.loc[sort_order],
score=cmapper.score.loc[sort_order],
))
pdata2 = pd.DataFrame(
dict(
x=pd.Series(range(nosamples), index=sort_order),
y=gmapper.score.loc[sort_order],
score=cmapper.score.loc[sort_order],
))
bfigure = bokeh_figure(
plot_width=FIGSIZE[0],
plot_height=int(FIGSIZE[1] * 0.8),
# tools = bokeh_tools,
y_range=bmodels.Range1d(gmapper.min(), gmapper.max()),
toolbar_sticky=False,
toolbar_location='left',
title='geneplot')
#bbar = bokeh_chart.Bar(pdata2, 'x', values='y', group='plate')
bfigure.title.text_color = 'darkgrey'
bfigure.title.text_font_style = 'normal'
bfigure.title.text_font_size = "12px"
bplot = bfigure.vbar(x='x',
width=0.5,
bottom=0,
top='y',
source=pdata,
legend='score',
color=dict(field='score',
transform=cmapper.colormapper))
blegend = bfigure.legend[0].items[0]
bcolorbar = ColorBar(color_mapper=gmapper.colormapper,
ticker=BasicTicker(),
formatter=BasicTickFormatter(precision=1),
label_standoff=10,
border_line_color=None,
location=(0, 0))
null_colorbar_mapper = LinearColorMapper(palette='Inferno256',
low=0,
high=0)
if cmapper.discrete:
#remove ColorBar
bcolorbar.color_mapper = null_colorbar_mapper
else:
#remove legend
bfigure.legend[0].items.pop(
) #remove legend - we can add this later again
bfigure.add_layout(bcolorbar, 'right')
# # display widgets
display(ilabel('gene', w['gene']))
cmapper.display()
smapper.display()
smapper2.display()
display(w['warn'])
#for k, v in bplot.data_source.data.items():
# print(k, v.shape, v.dropna().shape)
bhandle = bokeh_io.show(bfigure, notebook_handle=True)
#bhandle = bokeh_io.show(bbar, notebook_handle=True)
def warn(message):
w['warn'].value = '<b>{}</b>'.format(message)
def on_gene_change(*args):
gene = w['gene'].value
if not gene in self.counttable.index:
warn("gene {} is not in current counttable".format(gene))
return
sortorder = get_sort_order()
gmapper.value = gene
yval = gmapper.score.loc[sortorder]
bplot.data_source.data['y'] = yval
bokeh_io.push_notebook(handle=bhandle)
def on_sort_change(*args):
order = get_sort_order()
d = bplot.data_source.data
d['x'].index = order
d['y'] = d['y'].loc[order]
d['score'] = d['score'].loc[order]
bokeh_io.push_notebook(handle=bhandle)
def on_color_change(*args):
order = get_sort_order()
score = cmapper.score
score = score.loc[order]
bplot.data_source.data['score'] = score
bplot.glyph.fill_color['transform'] = cmapper.colormapper
cm = cmapper.colormapper
self._cm = cm
if cmapper.discrete:
warn('discrete')
bcolorbar.color_mapper = null_colorbar_mapper
if not bfigure.legend[0].items:
bfigure.legend[0].items.append(blegend)
else:
warn('cont')
bcolorbar.color_mapper = cmapper.colormapper
if bfigure.legend[0].items:
bfigure.legend[0].items.pop()
bokeh_io.push_notebook(handle=bhandle)
smapper.on_change = on_sort_change
smapper2.on_change = on_sort_change
cmapper.on_change = on_color_change
w['gene'].on_submit(on_gene_change)
on_gene_change
on_color_change
on_sort_change
def DIFX(self):
# define widgets
cat_meta_grp = list(self.get_metadata_info('categorical').index)
wstyle = widgets.Layout(width='200')
sl_group_a = self.ccwidget("diffx_group_a",
"dropdown",
lambda: 'difx',
'thelenota',
options=cat_meta_grp)
sl_set_a = self.ccwidget("diffx_group_a_set", "dropdown",
lambda: 'difx', None)
sl_group_b = self.ccwidget("diffx_group_b",
"dropdown",
lambda: 'difx',
'thelenota',
options=cat_meta_grp)
sl_set_b = self.ccwidget("diffx_group_b_set", "dropdown",
lambda: 'difx', None)
self.DIFX_stats = None
butlay = widgets.Layout(width="120px")
sl_go = widgets.Button(description='Go', layout=butlay)
sl_check = widgets.Button(description='Check', layout=butlay)
sl_save_set = widgets.Button(description='Save', layout=butlay)
sl_enrichr_link = widgets.Button(description='S&Enrichr',
layout=butlay)
sl_set_name = self.ccwidget('diffx_setname', 'text', lambda: 'difx',
"set_name")
sl_norm = widgets.Checkbox(value=False)
html_w = widgets.HTML()
html_link_w = widgets.HTML()
nogenes = self.counttable.shape[0]
colv = [1] * nogenes
color_mapper = LinearColorMapper(palette="Inferno256",
low=-0.3,
high=2.5)
pdata = ColumnDataSource(
dict(x=[random.uniform(-10, 10) for x in range(nogenes)],
y=[random.uniform(-10, 10) for x in range(nogenes)],
mean_a=[3.1] * nogenes,
mean_b=[-3.1] * nogenes,
size=[0.1] * nogenes,
desc=list(self.counttable.index),
score=colv))
self._fdata = pdata
select_callback = CustomJS(args={'dsource': pdata},
code="""
var indici = dsource.selected['1d'].indices;
console.log(indici);
IPython.notebook.kernel.execute(
'T._DF_INDICI_SEL_ = ' + indici);
""")
bokeh_tools = [
bmodels.HoverTool(tooltips=[
("(mean,lfc)", "($x, $y)"),
("desc", "@desc"),
('mean a', "@mean_a"),
('mean b', "@mean_b"),
]),
bmodels.BoxSelectTool(callback=select_callback),
bmodels.PanTool(),
bmodels.WheelZoomTool(),
bmodels.BoxZoomTool(),
bmodels.LassoSelectTool(callback=select_callback),
bmodels.SaveTool(),
bmodels.ResetTool(),
bmodels.HelpTool(),
]
bfigure = bokeh_figure(plot_width=FIGSIZE[0],
plot_height=FIGSIZE[1],
tools=bokeh_tools,
toolbar_sticky=False,
toolbar_location='left',
title='diffexplot')
bfigure.title.text_color = 'darkgrey'
bfigure.title.text_font_style = 'normal'
bfigure.title.text_font_size = "12px"
self._ffigure = bfigure
bplot = bfigure.circle(x='x',
y='y',
radius='size',
source=pdata,
legend='score',
color=dict(field='score',
transform=color_mapper))
self._fplot = bplot
bhandle = bokeh_io.show(bfigure, notebook_handle=True)
#def save_geneset
def go_enrichr(*args):
idx = list(self._DF_INDICI_SEL_)
if len(idx) == 0:
return
genes = list(self.counttable.index.to_series()\
.iloc[idx])
if len(genes) == 0:
return
setname = sl_set_name.value
self.save_geneset(setname, genes)
genes_str = "\n".join(genes)
description = setname
if not description:
description = 'a set'
payload = {
'list': (None, genes_str),
'description': (None, description)
}
ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/addList'
response = requests.post(ENRICHR_URL, files=payload)
if not response.ok:
#print(response)
raise Exception('Error analyzing gene list')
data = json.loads(response.text)
shortid = data['shortId']
newurl = 'http://amp.pharm.mssm.edu/Enrichr/enrich?dataset='
newurl += shortid
js = '<script>window.open("{}", "_blank")</script>'.format(newurl)
html_link_w.value = js
sl_enrichr_link.on_click(go_enrichr)
# group defintion logic
def update_groups(*args):
#fill the menus with he correct values
group_a = sl_group_a.value
group_b = sl_group_b.value
meta_a = self.get_metadata(group_a)
meta_b = self.get_metadata(group_b) \
if group_b != group_a else meta_a
valc_a = meta_a.value_counts().sort_values(ascending=False)
valc_b = meta_b.value_counts().sort_values(ascending=False)
sl_set_a.options = \
['<all> -- {}'.format(len(meta_a))] + \
['{} -- {}'.format(a, b)
for (a,b) in valc_a.items()]
sl_set_b.options = \
['<all> -- {}'.format(len(meta_a))] + \
['{} -- {}'.format(a, b)
for (a,b) in valc_b.items()]
sl_set_a.value = sl_set_a.options[1]
update_groups()
sl_group_a.observe(update_groups, 'value')
sl_group_b.observe(update_groups, 'value')
def run(*args):
title = 'QDDE'
all_samples_set = set(self.counttable.columns)
logdata = pd.Series()
normalize = sl_norm.value
logdata['total cells'] = len(all_samples_set)
group_a = sl_group_a.value
group_b = sl_group_b.value
set_a = sl_set_a.value.split('--')[0].strip()
set_b = sl_set_b.value.split('--')[0].strip()
title += ' A:{}/{} B:{}/{}'.format(group_a, set_a, group_b, set_b)
meta_a = self.get_metadata(group_a)
meta_b = self.get_metadata(group_b) \
if group_b != group_a else meta_a
logdata['group a'] = set_a
logdata['group b'] = set_b
sample_a = copy.copy(all_samples_set) \
if set_a == '<all>' else set (meta_a[meta_a == set_a].index)
sample_b = copy.copy(all_samples_set) \
if set_b == '<all>' else set (meta_b[meta_b == set_b].index)
sample_a &= all_samples_set #ensure overlap with this count table
sample_b &= all_samples_set
sample_b -= sample_a # so we don't have any duplicates
logdata['cells in a'] = len(sample_a)
logdata['cells in b'] = len(sample_b)
cnts = self.counttable
if normalize:
cnts = 1e6 * cnts / cnts.sum()
if cnts.min().min() < 0:
#assume this is in log_space
logdata['assuming log space'] = True
cnts = 10**cnts
cnts_a = cnts.loc[:, sample_a]
cnts_b = cnts.loc[:, sample_b]
if cnts_a.shape[1] < 1: return
if cnts_b.shape[1] < 1: return
html_w.value = pd.DataFrame(logdata).to_html(header=False)
stats = pd.DataFrame(
dict(
mean_a=cnts_a.mean(1),
mean_b=cnts_b.mean(1),
mean_all=np.log10(cnts.mean(1)),
))
stats['a/b'] = stats['mean_a'] / stats['mean_b']
stats['lfc'] = np.log2(stats['a/b'])
# lfc_l = stats['lfc']
stats['no_cells_in_a'] = len(sample_a)
stats['no_cells_in_b'] = len(sample_b)
stats['name_a'] = '{}/{}'.format(group_a, set_a)
stats['name_b'] = '{}/{}'.format(group_b, set_b)
#print(stats.head())
#stats = stats.sort_values(by='lfc', ascending=False)
#bplot.data_source.data['x'] = stats['mean_a']
#bplot.data_source.data['y'] = stats['mean_b']
bplot.data_source.data['x'] = stats['mean_all']
bplot.data_source.data['y'] = stats['lfc']
bplot.data_source.data['mean_a'] = stats['mean_a']
bplot.data_source.data['mean_b'] = stats['mean_b']
m = stats['mean_all'].max()
bfigure.title.text = title
self.DIFX_stats = stats
bplot.data_source.data['size'] = [0.01 * m] * nogenes
bokeh_io.push_notebook(handle=bhandle)
sl_go.on_click(run)
run()
# display interface
display(ilabel('Group A', sl_group_a, sl_set_a))
display(ilabel('Group B (-A)', sl_group_b, sl_set_b))
display(ilabel('TPM normalize', sl_norm))
display(ilabel('Set Name', sl_set_name))
# tabs = widgets.Tab(children=tab_children)
# tabs.set_title(0, 'Define sets')
# display(tabs) sl_enrichr_link html_link_w
display(widgets.HBox([sl_go, sl_check, sl_save_set, sl_enrichr_link]))
display(html_w)
display(html_link_w)
def scatter_plot(successes, failures, label, min_time=None, max_time=None):
"""
Scatter plot the successes/failures for a particular request type.
"""
# SUCCESSES
# Convert all success response timestamps to numpy datetimes.
all_success_timestamps = np.array([
np.datetime64(datetime.datetime.isoformat(s['timestamp']))
for s in successes
])
# Start the bins with the initial response time.
min_time, max_time = [
np.datetime64(datetime.datetime.isoformat(x))
for x in [min_time, max_time]
]
num_bins = 50
bin_interval = (max_time - min_time) / num_bins
# Make regular time-intervaled bins until all response times are covered.
time_bins = []
for i in range(int(num_bins)):
time_bins.append(min_time + bin_interval * i)
# Convert the timestamps into a comparable data type.
all_success_timestamps_i8 = all_success_timestamps.view('i8')
time_bins_i8 = np.array(time_bins).view('i8')
# Aggregate each response time into the appropriate time bin.
binned = np.digitize(all_success_timestamps_i8, time_bins_i8)
vals_per_interval = [[] for x in xrange(time_bins_i8.size)]
for i, n in enumerate(binned):
vals_per_interval[n - 1].append(successes[i]['response_time'])
# Calculate the mean for each time bin.
means_per_interval = [
np.mean(x) if len(x) else 0.0 for x in vals_per_interval
]
# Extract the max response time per interval.
max_per_interval = [max(x) if len(x) else 0 for x in vals_per_interval]
# FAILURES
# Convert all failure response timestamps to numpy datetimes.
all_failure_timestamps = np.array([
np.datetime64(datetime.datetime.isoformat(s['timestamp']))
for s in failures
])
all_failure_timestamps_i8 = all_failure_timestamps.view('i8')
# Convert each failure response time into a plottable point against its time bin.
binned = []
if len(all_failure_timestamps_i8):
binned = np.digitize(all_failure_timestamps_i8, time_bins_i8)
failure_timestamps = []
failure_resp_times = []
for i, n in enumerate(binned):
failure_timestamps.append(time_bins[n - 1])
failure_resp_times.append(failures[i]['response_time'])
TOOLS = "resize,crosshair,pan,wheel_zoom,box_zoom,reset,box_select"
# Find the maximum y-value.
y_max = max(max_per_interval)
if len(failure_resp_times):
y_max = max(y_max, max(failure_resp_times))
# Create a new plot with the tools above, and set axis labels.
graph_plot = bokeh_figure(title=label,
tools=TOOLS,
x_range=(min_time, max_time),
y_range=(0, y_max))
graph_plot.xaxis.axis_label = "Time"
graph_plot.yaxis.axis_label = "Mean Response Time (ms)"
# Scatter-plot the mean response time data for successes.
graph_plot.circle(x=np.array(time_bins),
y=means_per_interval,
fill_color='blue')
# Scatter-plot the failure response times for all failures.
graph_plot.x(x=np.array(failure_timestamps),
y=failure_resp_times,
line_color='red')
return graph_plot
def __plot_heat_image(map, method, title, list_name):
print "=================================================================="
print " Plotting heat image..."
print "=================================================================="
# matplotlib only used for saving file!
fig = plt.figure()
ax = fig.add_subplot(111)
TOOLS = "pan,wheel_zoom,box_zoom,reset,hover,save"
bokeh_plot = bokeh_figure(width=1400, height=1000, tools=TOOLS)
min_val = 0
max_val = 255
my_cmap, norm = __define_heat_map_values(min_val=min_val, max_val=max_val)
num_slabs = 255 # number of color steps
jet_100 = [colors.rgb2hex(m) for m in my_cmap(np.arange(0, my_cmap.N, my_cmap.N / (num_slabs - 1)))]
for low_energy, values in map.iteritems():
high_energy, start_time_list, end_time_list, current_fits_list, map_position_list, std_fits_list, mean_fits_list = values
# Plot rectangle
# Source: http://matthiaseisen.com/pp/patterns/p0203/
for i, (current_fits, map_position) in enumerate(zip(locals()[list_name], map_position_list)):
color_i = my_cmap(norm(current_fits)) # returns a rgba value
rect = patches.Rectangle( # make your rectangle
(map_position, low_energy), # (x,y)
1, # width
high_energy - low_energy, # height
edgecolor=None,
color=color_i)
ax.add_patch(rect)
if method == "Original":
source = ColumnDataSource(data=dict(
file=[map_position],
start_time=[start_time_list[i]],
end_time=[end_time_list[i]],
))
else:
source = ColumnDataSource(data=dict(
file=[map_position],
start_time_first=[start_time_list[i]],
end_time_first=[end_time_list[i]],
start_time_second=[start_time_list[i + 1]],
end_time_second=[end_time_list[i + 1]]
))
bokeh_plot.rect(x=map_position + 0.5, y=low_energy + (high_energy - low_energy) / 2, width=1,
height=high_energy - low_energy,
color=rgb2hex(color_i),
source=source)
# width_units="screen", height_units="screen")
plt.title(title)
hover = bokeh_plot.select_one(HoverTool)
hover.point_policy = "follow_mouse"
if method == "Original":
hover.tooltips = [
("File", "@file"),
("Start time:", "@start_time"),
("End time:", "@end_time")
]
else:
hover.tooltips = [
("File", "@file"),
("Start time first:", "@start_time_first"),
("End time first:", "@end_time_first"),
("Start time second:", "@start_time_second"),
("End time second:", "@end_time_second")
]
pcb = bokeh_figure(plot_width=80, plot_height=1000, x_range=[0, 1], y_range=[0, max_val], min_border_right=10)
pcb.image(image=[np.linspace(min_val, max_val, 100).reshape(100, 1)], x=[0], y=[0], dw=[1], dh=[max_val - min_val],
palette=jet_100)
pcb.xaxis.major_label_text_color = None
pcb.xaxis.major_tick_line_color = None
pcb.xaxis.minor_tick_line_color = None
pgrid = bokeh_gridplot([[bokeh_plot, pcb]]) # this places the colorbar next to the image
bokeh_output_file(__save_path + title + ".html", title=title)
bokeh_show(pgrid)
__save_file(__save_path, title + ".png", fig=fig)
plt.clf()
plt.close()
print "=================================================================="
print " Plotting heat image, done!"
print "=================================================================="
def scatter_plot(successes, failures, label, min_time=None, max_time=None):
"""
Scatter plot the successes/failures for a particular request type.
"""
# SUCCESSES
# Convert all success response timestamps to numpy datetimes.
all_success_timestamps = np.array(
[np.datetime64(datetime.datetime.isoformat(s['timestamp'])) for s in successes]
)
# Start the bins with the initial response time.
min_time, max_time = [np.datetime64(datetime.datetime.isoformat(x)) for x in [min_time, max_time]]
num_bins = 50
bin_interval = (max_time - min_time) / num_bins
# Make regular time-intervaled bins until all response times are covered.
time_bins = []
for i in range(int(num_bins)):
time_bins.append(min_time + bin_interval * i)
# Convert the timestamps into a comparable data type.
all_success_timestamps_i8 = all_success_timestamps.view('i8')
time_bins_i8 = np.array(time_bins).view('i8')
# Aggregate each response time into the appropriate time bin.
binned = np.digitize(all_success_timestamps_i8, time_bins_i8)
vals_per_interval = [[] for x in xrange(time_bins_i8.size)]
for i, n in enumerate(binned):
vals_per_interval[n - 1].append(successes[i]['response_time'])
# Calculate the mean for each time bin.
means_per_interval = [np.mean(x) if len(x) else 0.0 for x in vals_per_interval]
# Extract the max response time per interval.
max_per_interval = [max(x) if len(x) else 0 for x in vals_per_interval]
# FAILURES
# Convert all failure response timestamps to numpy datetimes.
all_failure_timestamps = np.array(
[np.datetime64(datetime.datetime.isoformat(s['timestamp'])) for s in failures]
)
all_failure_timestamps_i8 = all_failure_timestamps.view('i8')
# Convert each failure response time into a plottable point against its time bin.
binned = []
if len(all_failure_timestamps_i8):
binned = np.digitize(all_failure_timestamps_i8, time_bins_i8)
failure_timestamps = []
failure_resp_times = []
for i, n in enumerate(binned):
failure_timestamps.append(time_bins[n - 1])
failure_resp_times.append(failures[i]['response_time'])
TOOLS = "resize,crosshair,pan,wheel_zoom,box_zoom,reset,box_select"
# Find the maximum y-value.
y_max = max(max_per_interval)
if len(failure_resp_times):
y_max = max(y_max, max(failure_resp_times))
# Create a new plot with the tools above, and set axis labels.
graph_plot = bokeh_figure(
title=label,
tools=TOOLS,
x_range=(min_time, max_time),
y_range=(0, y_max)
)
graph_plot.xaxis.axis_label = "Time"
graph_plot.yaxis.axis_label = "Mean Response Time (ms)"
# Scatter-plot the mean response time data for successes.
graph_plot.circle(x=np.array(time_bins), y=means_per_interval, fill_color='blue')
# Scatter-plot the failure response times for all failures.
graph_plot.x(x=np.array(failure_timestamps), y=failure_resp_times, line_color='red')
return graph_plot