def new_plot(self, title, units, *, line_width=None, precision=2):
"""
Creates a blank line plot for with timestamps on the x-axis and
a line for each data series on the y-axis.
"""
plot = plotting.figure(title=title, tools=[])
self.active_plot = plot
self.plots.append(plot)
self.colors = list(colors)
self.units = units
self.line_width = line_width or self.line_width
plot.plot_width = self.width
plot.plot_height = self.height
plot.x_range = self.x_range
datetime_tick_formats = {
key: ["%a %b %d %H:%M:%S"]
for key in ("seconds", "minsec", "minutes", "hourmin", "hours", "days")
}
plot.xaxis.formatter = models.DatetimeTickFormatter(**datetime_tick_formats)
# https://bokeh.pydata.org/en/latest/docs/reference/models/formatters.html
# plot.yaxis.formatter = models.NumeralTickFormatter(format="0a")
plot.yaxis.formatter = models.NumeralTickFormatter(format="0,0.00 a")
# With default precision level 2 (decimal places)
# The units_formats = "@y{0,0.00}"
# and the value would look like 1,234,567.89
units_formats = f"@y{{0,0.{'0' * precision}}}"
hover = models.HoverTool(
# mode = vline would be nice to use,
# but then separate hovers block each when lines are too close.
# Would like a single hover box with time once, and a value per line
# perhaps this will help acheive that:
# https://stackoverflow.com/questions/29435200/bokeh-plotting-enable-tooltips-for-only-some-glyphs
mode="mouse", # other optins: vline
line_policy="nearest", # other optins: prev, next, nearest, interp, none
tooltips=[
("Name", "$name"),
("Time", "@x{%a %m/%d %H:%M:%S}"),
(self.units, units_formats),
],
formatters={"x": "datetime", "Time": "datetime", "@x": "datetime"},
)
plot.add_tools(hover)
plot.add_tools(models.BoxZoomTool())
plot.add_tools(models.HelpTool())
plot.add_tools(models.PanTool())
plot.add_tools(models.WheelZoomTool(dimensions="width"))
plot.toolbar.active_scroll = plot.select_one(models.WheelZoomTool)
plot.add_tools(models.WheelZoomTool(dimensions="height"))
plot.add_tools(models.UndoTool())
plot.add_tools(models.RedoTool())
plot.add_tools(models.ResetTool())
plot.add_tools(models.SaveTool())
def plot_bokeh(self, labels, output_file=None, node_size=4,
node_color=None, edge_color=None, width=None, **kwargs):
# Unfortunately, nodes in Bokeh have to be strings or ints
plot_d = nx.relabel.relabel_nodes(self, labels)
tooltips = []
for node, data in plot_d.nodes(data=True):
for key in data:
tooltips += [(key, f"@{key}")]
break
if node_color is None:
node_color = {labels[node]: "green"
if node in self.path else "lightgray"
for node in self.nodes}
nx.set_node_attributes(plot_d, node_color, "node_color")
fill_color = "node_color"
if edge_color is None:
edge_color = {(labels[edge[0]], labels[edge[1]]): "limegreen"
if edge in self.solution
or edge[::-1] in self.solution else "gray"
for edge in self.edges}
nx.set_edge_attributes(plot_d, edge_color, "edge_color")
line_color = "edge_color"
if width is None:
width = {(labels[edge[0]], labels[edge[1]]): 2
if edge in self.solution else 0.1
for edge in self.edges}
nx.set_edge_attributes(plot_d, width, "edge_width")
line_width = "edge_width"
graph = bkplt.from_networkx(plot_d, nx.circular_layout)
graph.node_renderer.glyph = mod.Circle(size=node_size,
line_color=fill_color,
fill_color=fill_color)
graph.edge_renderer.glyph = mod.MultiLine(line_color=line_color,
line_width=line_width)
plot = mod.Plot()
plot.renderers.append(graph)
tooltips = [("Label", "@index")] + tooltips
node_hover_tool = mod.HoverTool(tooltips=tooltips)
plot.add_tools(node_hover_tool, mod.PanTool(), mod.BoxZoomTool(),
mod.WheelZoomTool(), mod.SaveTool(), mod.ResetTool())
if output_file:
bkplt.output_file(output_file)
bkplt.show(plot)
def get_map(self):
import bokeh.plotting as bplt
import bokeh.models as bm
wheel_zoom = bm.WheelZoomTool()
self.hover = bm.HoverTool(tooltips=[
("Navn", "@kommuner"),
#("(Long, Lat)", "($x, $y)"),
("Dato", "@kommuner_dates"),
("stemme pct", "@stemme_pct %"),
])
self.hover.point_policy = "follow_mouse"
tools = [
bm.PanTool(),
bm.BoxZoomTool(), wheel_zoom,
bm.SaveTool(),
bm.ResetTool(),
bm.UndoTool(),
bm.RedoTool(),
bm.CrosshairTool(), self.hover
]
fig = bplt.figure(title="Test",
tools=tools,
x_axis_location=None,
y_axis_location=None,
match_aspect=True)
# Activate scrool
fig.toolbar.active_scroll = wheel_zoom
# Remove grid lines
fig.grid.grid_line_color = None
# Check if source exists
if not hasattr(self, 'source'):
self.make_map_source()
# Make color mapper
#from bokeh.palettes import Viridis6 as palette
#from bokeh.palettes import Spectral11 as palette
from bokeh.palettes import RdYlGn11 as palette
palette.reverse()
#color_mapper = bm.LogColorMapper(palette=palette, high=90., low=50.)
color_mapper = bm.LinearColorMapper(palette=palette, high=90., low=50.)
# Plot
fig.patches(xs='x_lon',
ys='y_lat',
source=self.source,
fill_color={
'field': 'stemme_pct',
'transform': color_mapper
},
fill_alpha=0.7,
line_color="white",
line_width=0.5)
return fig
def get_fig(self):
import bokeh.plotting as bplt
# Make tools
wheel_zoom = bm.WheelZoomTool()
self.hover = bm.HoverTool(tooltips=[
#("index", "$index"),
("(x,y)", "($x, $y)"),
("int", "@intensity"),
("VOL", "@VOL"),
#("fill color", "$color[hex, swatch]:fill_color"),
#("Color", "@line_color"),
])
tools = [bm.PanTool(), bm.BoxZoomTool(), wheel_zoom, bm.SaveTool(), bm.ResetTool(), bm.UndoTool(), bm.RedoTool(), bm.CrosshairTool(), self.hover]
# Make figure
if self.dic:
self.fig = bplt.figure(plot_width=400,plot_height=400, x_range=(self.x0_ppm, self.x1_ppm), y_range=(self.y0_ppm, self.y1_ppm), tools=tools)
else:
self.fig = bplt.figure(plot_width=400,plot_height=400, tools=tools)
# Activate scrool
self.fig.toolbar.active_scroll = wheel_zoom
# If not ColumnDataSource exists, then create
if not self.ColumnDataSource:
self.create_ColumnDataSource()
# Create figure
self.fig_multi = self.fig.multi_line(xs='xs', ys='ys', line_color='line_color', source=self.ColumnDataSource, legend="Contours")
# Possible for text: angle, angle_units, js_event_callbacks, js_property_callbacks, name,
# subscribed_events, tags, text, text_align, text_alpha, text_baseline, text_color, text_font, text_font_size,
# text_font_style, x, x_offset, y or y_offset
#fig.text(x='xt',y='yt',text='text', source=self.ColumnDataSourceText,
# text_baseline='middle', text_align='center', text_font_size="10px", legend="Text")
# Hide glyphs in Interactive Legends
self.fig.legend.click_policy="hide" # "mute"
# Set label
if self.dic:
self.fig.xaxis.axis_label = self.udic[1]['label'] + ' ppm'
self.fig.yaxis.axis_label = self.udic[0]['label'] + ' ppm'
return self.fig
def __init__(self, thelenota, figsize=(800, 600)):
# DIMRED widgets
self.thelenota = thelenota
self.figsize = figsize
self.counts = None
#widgets
self.w = DUMMY()
current_cluster_labels = None
wstyle = widgets.Layout(width='200px') #, max_width='120px',
# min_width='120px')
self.w.drfilter_lower_perc_switch = ccwidget(
self.thelenota,
'drfilter_lower_perc_switch',
'bool',
self.dr_name_simple,
False,
lwidth='40px')
self.w.drfilter_log_switch = ccwidget(self.thelenota,
'drfilter_log_switch',
'bool',
self.dr_name_simple,
False,
lwidth='40px')
self.w.drfilter_lower_percentage = ccwidget(
self.thelenota,
'drfilter_lower_percentage',
'float',
self.dr_name_simple,
0.2,
min=0.01,
max=1,
value=0.2)
self.w.drmethod = ccwidget(
self.thelenota,
'dimred_method',
'dropdown',
self.dr_name_simple,
'tsne',
options='pca ica scorpius KernelPCA tsne nmf'.split())
self.w.drperplex = cache_widget_value(widgets.IntSlider(value=67,
min=2,
max=99,
step=5),
67,
self.thelenota,
'dimred_perplexity',
self.dr_name_simple,
fmt='int')
self.w.drangle = cache_widget_value(widgets.FloatSlider(value=0.5,
min=0.05,
max=0.95,
step=0.05),
0.5,
self.thelenota,
'dimred_angle',
self.dr_name_simple,
fmt='float')
self.w.drlrate = cache_widget_value(widgets.IntSlider(value=920,
min=20,
max=10000,
step=50),
920,
self.thelenota,
'dimred_learning_rate',
self.dr_name_simple,
fmt='int')
self.w.drearly = cache_widget_value(widgets.FloatSlider(value=3.5,
min=1,
max=20,
step=0.5),
3.5,
self.thelenota,
'dimred_early_exag',
self.dr_name_simple,
fmt='float')
self.w.dr_tsne_pcavar_cutoff = ccwidget(self.thelenota,
'tsne_pca_var_cutoff',
'float',
self.dr_name_simple,
0.05,
min=0.01,
max=0.2,
step=0.01)
self.w.drrun = widgets.Button(description='GO!')
self.w.drforce = widgets.Checkbox(description='force',
layout=widgets.Layout(width='300px'))
@dcache(self.thelenota, 'dimred_correlating_genes', 'tsv',
self.dr_name)
def get_dr_correlating(*_):
stats, trans = run_dr_2()
c1 = self.thelenota.counttable.apply(
lambda x: pearsonr(x, trans.iloc[:, 0])[0], axis=1)
c2 = self.thelenota.counttable.apply(
lambda x: pearsonr(x, trans.iloc[:, 1])[0], axis=1)
return pd.DataFrame({0: c1, 1: c2})
def get_top_correlating(*_):
d = get_dr_correlating().abs().sum(1).sort_values(ascending=False)
d = d.head(40)
d = d.reset_index()
d = d.apply(lambda x: '{} ({:.3g})'.format(x.iloc[0], x.iloc[1]),
axis=1)
return list(d)
tcolormap = cmapper.CMAPPER(self.thelenota,
extra_intrinsic_methods={
"top DR correlate":
(get_top_correlating,
cmapper.intrinsic_gene_score)
})
self.w.sl_group_name = cache_widget_value(widgets.Text(layout=wstyle),
'self.thelenota',
self.thelenota,
'group_define_name',
self.dr_name_simple)
self.w.sl_group_set = widgets.Text(layout=wstyle)
self.w.sl_group_set_go = widgets.Button(description='set',
layout=wstyle)
self.w.sl_groupextractname = widgets.Text(layout=wstyle)
self.w.sl_group_extract_go = widgets.Button(description='extract')
self.w.clu_method = ccwidget(self.thelenota,
'cluster_method',
'dropdown',
self.dr_name_simple,
'dbscan',
options=['dbscan'])
self.w.clu_dbscan_eps = ccwidget(self.thelenota,
'clu_dbscan_eps_w',
'float',
self.dr_name_simple,
2.5,
min=0.1,
max=10.0,
step=0.1)
self.w.clu_go = widgets.Button(description='Cluster!')
self.w.clu_name = ccwidget(self.thelenota, "cluster_name", "text",
self.dr_name_simple, 'cluster')
self.w.clu_store_go = widgets.Button(description='save')
plotinfo = {}
self.w.html = widgets.HTML()
# data!
samples = self.thelenota.counttable.columns
nosamples = len(samples)
color_mapper = LinearColorMapper(palette="Inferno256",
low=-0.3,
high=2.5)
topgene = self.thelenota.counttable.std(1).sort_values().tail(
1).index[0]
colv = list(self.thelenota.counttable.loc[topgene])
pdata = ColumnDataSource(
dict(x=[random.uniform(-10, 10) for x in range(nosamples)],
y=[random.uniform(-10, 10) for x in range(nosamples)],
desc=list(samples),
size=[0.3] * nosamples,
score=colv))
self.thelenota._pdata = pdata
# Clustering
def run_clustering(*args):
method = self.w.clu_method.value
stats, trans = run_dr_2()
if method == 'dbscan':
labels = run_dbscan(trans, self.w.clu_dbscan_eps.value)
else:
lg.warning('Not implemented cluster method: {}'.format(method))
return
self.thelenota._CLUSTER_LABELS[self.dr_name()] = labels
colv = labels
color_mapper = CategoricalColorMapper(
palette=bokeh.palettes.Category20[20],
factors=list(colv.value_counts().index))
colorbar_mapper = LinearColorMapper(palette='Inferno256',
low=0,
high=0)
bcolorbar.color_mapper = colorbar_mapper
if not bfigure.legend[0].items:
bfigure.legend[0].items.append(blegend)
bplot.data_source.data['score'] = colv
bplot.glyph.fill_color['transform'] = color_mapper
bplot.glyph.line_width = 0
bplot.glyph.line_alpha = 0
bokeh_io.push_notebook(handle=bhandle)
def store_current_cluster(*args):
labels = self.thelenota._CLUSTER_LABELS.get(self.dr_name())
if labels is None:
self.w.html.value = "no cluster defined"
return
cname = self.w.clu_name.value
labels = labels.apply(lambda x: '{}_{}'.format(cname, x))
outfile = self.thelenota.metadata_dir / '{}.tsv'.format(cname)
moutfile = self.thelenota.metadata_dir / '{}.meta.tsv'.format(
cname)
tmeta = pd.DataFrame({cname: labels})
tmeta.to_csv(outfile, sep="\t")
with open(moutfile, 'w') as F:
F.write("{}\tcategorical\n".format(cname))
self.w.html.value = 'saved cluster to {}'.format(outfile)
self.w.clu_store_go.on_click(store_current_cluster)
self.w.clu_go.on_click(run_clustering)
# GROUP SET
def define_group(*args):
groupset = self.w.sl_group_name.value
groupname = self.w.sl_group_set.value
self.thelenota.selected = list(
self.thelenota.counttable.columns.to_series()\
.iloc[list(self.thelenota._DR_INDICI_SEL_)])
if groupset in self.thelenota.metadata_info.index:
tmeta = self.thelenota.get_metadata(groupset)
else:
tmeta = pd.Series('na',
index=self.thelenota.counttable.columns)
tmeta.loc[self.thelenota.selected] = groupname
self.thelenota.metadata_dir.makedirs_p()
outfile = self.thelenota.metadata_dir / '{}.tsv'.format(groupset)
moutfile = self.thelenota.metadata_dir / '{}.meta.tsv'.format(
groupset)
tmeta = pd.DataFrame({groupset: tmeta})
tmeta.to_csv(outfile, sep="\t")
with open(moutfile, 'w') as F:
F.write("{}\tcategorical\n".format(groupset))
run_dr_color()
self.w.sl_group_set_go.on_click(define_group)
# GROUP EXTRACT
#sl_groupextractname_w
def extract_group(*args):
groupname = self.w.sl_groupextractname.value
self.thelenota.selected = list(
self.thelenota.counttable.columns.to_series()\
.iloc[list(self.thelenota._DR_INDICI_SEL_)])
outfile = self.thelenota.counttable_file.dirname() \
/ '{}__{}.tsv'.format(self.thelenota.counttable_name, groupname)
newcounts = self.thelenota.counttable[self.thelenota.selected]
newcounts.to_csv(outfile, sep="\t")
self.w.html.value = "save new count table to {}".format(outfile)
self.w.sl_group_extract_go.on_click(extract_group)
def force_refresh():
"Force to refresh calculations? Or can we use the cache??"
return self.w.drforce.value
@dcache(self.thelenota, 'filtered', 'pickle', self.dr_name_filter,
force_refresh)
def filter_counts(*_):
counts = self.thelenota.counttable
if self.w.drfilter_log_switch.value:
minval = 0.5 * counts[counts > 0].min().min()
lg.warning('log tranform log10(x+{:.4g})'.format(minval))
counts = np.log10(counts + minval)
if self.w.drfilter_lower_perc_switch.value:
perc = self.w.drfilter_lower_percentage.value
csum = counts.sum(1)
counts = counts[csum >= csum.quantile(perc)]
return counts
@dcache(self.thelenota, 'dimred_table', 'mtsv', self.dr_name,
force_refresh)
def run_dr_2(*_):
param = self.get_dr_param()
method = param['method']
del param['method']
if method == 'pca':
stats, trans = qrtask.pca.sync(self.counts)
elif method == 'ica':
stats, trans = rqtask.ica.sync(self.counts)
elif method == 'KernelPCA':
stats, trans = rqtask.kernelPCA.sync(self.counts)
elif method == 'scorpius':
stats, trans = rqtask.scorpius_dr.sync(self.counts)
elif method == 'nmf':
stats, trans = rqtask.nmf.sync(self.counts)
elif method == 'tsne':
stats, trans = rqtask.tsne.sync(self.counts,
self.get_dr_param())
else:
raise NotImplemented
return stats, trans
def set_color_scale():
score = tcolormap.score
method = tcolormap.method
self.warn('set color scale {}/{}'.format(method, tcolormap.value))
color_mapper = tcolormap.colormapper
bplot.glyph.fill_color['transform'] = color_mapper
bplot.data_source.data['score'] = score
if not tcolormap.discrete:
bcolorbar.color_mapper = color_mapper
if tcolormap.discrete:
if not bfigure.legend[0].items:
bfigure.legend[0].items.append(blegend)
else:
if bfigure.legend[0].items:
bfigure.legend[0].items.pop()
bplot.glyph.line_width = 0
bplot.glyph.line_alpha = 0
def _create_title():
cmethod = '{}/{}'.format(tcolormap.method, tcolormap.value)
if self.w.drfilter_lower_perc_switch.value:
cmethod += '/low%{}'.format(
self.w.drfilter_lower_percentage.value)
if self.w.drfilter_log_switch.value:
cmethod += '/log'
cmethod += '/{}/{}'.format(*self.counts.shape)
return '{}/{}'.format(self.thelenota.counttable_name, cmethod)
def _create_scatter_plot(only_color=False):
if (self.w.drfilter_lower_perc_switch.value) or (
self.w.drfilter_log_switch.value):
self.counts = filter_counts()
else:
self.counts = self.thelenota.counttable
self.warn("count table: {}".format(str(self.counts.shape)))
if not only_color:
meta, trans = run_dr_2()
self.thelenota.dr = trans
self.thelenota._dr_meta = meta
self.thelenota._dr_trans = trans
col1, col2 = trans.columns[:2]
d1 = trans[col1]
d2 = trans[col2]
title = self.dr_name().replace('_', ' ')
m = max(d2.max() - d2.min(), d1.max() - d1.min())
bplot.data_source.data['size'] = [0.008 * m] * nosamples
bplot.data_source.data['x'] = d1
bplot.data_source.data['y'] = d2
bfigure.title.text = _create_title()
set_color_scale()
bokeh_io.push_notebook(handle=bhandle)
def run_dr_color(*_):
""" refresh dr scatter plot - color only """
self.w.drrun.button_style = 'info'
try:
_create_scatter_plot(only_color=True)
except:
self.w.drrun.button_style = 'danger'
raise
self.w.drrun.button_style = ''
def run_dr(*_):
self.w.drrun.button_style = 'info'
try:
_create_scatter_plot()
except:
self.w.drrun.button_style = 'danger'
raise
self.w.drrun.button_style = ''
tcolormap.on_change = run_dr_color
self.w.drrun.on_click(run_dr)
# clgenesetchoice_w.observe(run_dr_color, 'value')
# clmetadata_w.observe(run_dr_color, 'value')
# set up bokeh
select_callback = CustomJS(args={'dsource': pdata},
code="""
var indici = dsource.selected['1d'].indices;
console.log(indici);
IPython.notebook.kernel.execute(
'T._DR_INDICI_SEL_ = ' + indici);
""")
bokeh_tools = [
bmodels.HoverTool(tooltips=[
("(x,y)", "($x, $y)"),
("score", "$score"),
("desc", "@desc"),
]),
bmodels.BoxSelectTool(callback=select_callback),
bmodels.PanTool(),
bmodels.WheelZoomTool(),
bmodels.BoxZoomTool(),
bmodels.LassoSelectTool(callback=select_callback),
bmodels.SaveTool(),
bmodels.ResetTool(),
bmodels.HelpTool(),
]
bfigure = bokeh_figure(plot_width=figsize[0],
plot_height=figsize[1],
tools=bokeh_tools,
toolbar_sticky=False,
toolbar_location='left',
title='dimredplot')
bfigure.title.text_color = 'darkgrey'
bfigure.title.text_font_style = 'normal'
bfigure.title.text_font_size = "12px"
self.thelenota._bfigure = bfigure
bplot = bfigure.circle(x='x',
y='y',
radius='size',
source=pdata,
legend='score',
color=dict(field='score',
transform=color_mapper))
self.thelenota._bplot = bplot
bcolorbar = ColorBar(color_mapper=tcolormap.colormapper,
ticker=BasicTicker(),
formatter=BasicTickFormatter(precision=1),
label_standoff=10,
border_line_color=None,
location=(0, 0))
bfigure.add_layout(bcolorbar, 'right')
blegend = bfigure.legend[0].items[0]
bhandle = bokeh_io.show(bfigure, notebook_handle=True)
tab_children = []
tab_children.append(
widgets.VBox([
ilabel('filter sum%', self.w.drfilter_lower_perc_switch,
self.w.drfilter_lower_percentage),
ilabel('log tranform', self.w.drfilter_log_switch),
ilabel('method', self.w.drmethod),
ilabel('perplexity', self.w.drperplex),
ilabel('angle', self.w.drangle),
ilabel('learning rate', self.w.drlrate),
ilabel('early exagg.', self.w.drearly),
ilabel('PCA var. cutoff', self.w.dr_tsne_pcavar_cutoff),
widgets.HBox([self.w.drrun, self.w.drforce]),
]))
tab_children.append(widgets.VBox([tcolormap.prepare_display()]))
tab_children.append(
widgets.VBox([
ilabel('method', self.w.clu_method),
ilabel('dbscan:eps', self.w.clu_dbscan_eps),
ilabel('store', self.w.clu_name, self.w.clu_store_go),
self.w.clu_go
]))
tab_children.append(
widgets.VBox([
ilabel('group define', self.w.sl_group_name,
self.w.sl_group_set, self.w.sl_group_set_go),
ilabel('count extract', self.w.sl_groupextractname,
self.w.sl_group_extract_go),
]))
tabs = widgets.Tab(children=tab_children)
tabs.set_title(0, 'DimRed')
tabs.set_title(1, 'Color')
tabs.set_title(2, 'Cluster')
tabs.set_title(3, 'Select')
tabs.selected_index = 0
display(tabs)
display(self.w.html)
#run a few on_change functions so that all is in sync
#on_colormethod_change()
run_dr()
def return_graph(title, graph_dataset, columns="price,bmi"):
# First, let's convert colors
print(" - Converting colors...", end="")
colors = list(graph_dataset.colors)
for i, color in enumerate(graph_dataset.colors):
r, g, b = COLOR_ID_TO_COLOR_RGB[color]
colors[i] = bokeh.colors.RGB(r, g, b)
graph_dataset.colors = colors
columns = columns.split(",")
print("\r\033[K - Creating figure...", end="")
# Create hover tool
tooltips = [("Country", "@names"), ("Region", "@regions")]
if "products" in graph_dataset:
tooltips.append(("Product", "@products"))
products = graph_dataset["products"]
hover = bkm.HoverTool(tooltips=tooltips)
# Create figure
f = plt.figure(title=title,
x_axis_label=columns[0],
y_axis_label=columns[1],
tools=[
hover,
bkm.WheelZoomTool(),
bkm.BoxZoomTool(),
bkm.PanTool(),
bkm.SaveTool(),
bkm.ResetTool()
],
width=900)
# Plot
# First, make the graph_dataset unique
x_list = graph_dataset[columns[0]]
y_list = graph_dataset[columns[1]]
names = graph_dataset["names"]
regions = graph_dataset["regions"]
colors = graph_dataset["colors"]
#for i in range(len(graph_dataset[columns[0]])):
# elem1 = graph_dataset[columns[0]][i]
# elem2 = graph_dataset[columns[1]][i]
# if elem1 not in x_list and elem2 not in y_list:
# x_list.append(elem1)
# y_list.append(elem2)
# names.append(graph_dataset["names"][i])
# regions.append(graph_dataset["regions"][i])
# colors.append(graph_dataset["colors"][i])
print("\r\033[K - Plotting...", end="")
data = {
columns[0]: x_list,
columns[1]: y_list,
"names": names,
"regions": regions,
"colors": colors
}
if "products" in graph_dataset:
data["products"] = products
source = bkm.ColumnDataSource(data=data)
f.scatter(columns[0],
columns[1],
source=source,
color="colors",
muted_color="colors",
muted_alpha=0.1,
size=10)
#f.x_range = bkm.DataRange1d(start=-1, end=3)
#f.y_range = bkm.DataRange1d(start=20, end=30)
# Add legend & save
print("\r\033[K - Done")
return f
for i in range(len(x["x"])):
xyz.append(str(x["x"][i]) + "," + str(x["y"][i]) + "," + str(x["z"][i]))
data = {
"x": x_embedded[:, 0],
"y": x_embedded[:, 1],
"color": x["color"],
"xyz": xyz,
"lbl": x["lbl"]
}
# Create figure
f = plt.figure(title="A",
x_axis_label="B",
y_axis_label="C",
tools=[
hover,
bkm.WheelZoomTool(),
bkm.BoxZoomTool(),
bkm.PanTool(),
bkm.SaveTool(),
bkm.ResetTool()
],
width=900)
f.scatter("x",
"y",
source=bokeh.models.ColumnDataSource(data=data),
color="color",
size=10)
plt.show(f)
"""# Make bokeh graph"""
wheel_zoom = bm.WheelZoomTool()
hover = bm.HoverTool(tooltips=[
("Navn", "@KOMNAVN"),
("KOMKODE", "@KOMKODE"),
("(Long, Lat)", "($x, $y)"),
("index", "@index"),
("REGIONNAVN", "@REGIONNAVN"),
#("REGIONKODE", "@REGIONKODE"),
("AREAL", "@AREAL"),
])
hover.point_policy = "follow_mouse"
tools = [
bm.PanTool(),
bm.BoxZoomTool(), wheel_zoom,
bm.SaveTool(),
bm.ResetTool(),
bm.UndoTool(),
bm.RedoTool(),
bm.CrosshairTool(), hover
]
fig = bplt.figure(title="Test",
tools=tools,
x_axis_location=None,
y_axis_location=None,
match_aspect=False)
# Activate scrool
fig.toolbar.active_scroll = wheel_zoom
# Remove grid lines
def DIFX(self):
# define widgets
cat_meta_grp = list(self.get_metadata_info('categorical').index)
wstyle = widgets.Layout(width='200')
sl_group_a = self.ccwidget("diffx_group_a",
"dropdown",
lambda: 'difx',
'thelenota',
options=cat_meta_grp)
sl_set_a = self.ccwidget("diffx_group_a_set", "dropdown",
lambda: 'difx', None)
sl_group_b = self.ccwidget("diffx_group_b",
"dropdown",
lambda: 'difx',
'thelenota',
options=cat_meta_grp)
sl_set_b = self.ccwidget("diffx_group_b_set", "dropdown",
lambda: 'difx', None)
self.DIFX_stats = None
butlay = widgets.Layout(width="120px")
sl_go = widgets.Button(description='Go', layout=butlay)
sl_check = widgets.Button(description='Check', layout=butlay)
sl_save_set = widgets.Button(description='Save', layout=butlay)
sl_enrichr_link = widgets.Button(description='S&Enrichr',
layout=butlay)
sl_set_name = self.ccwidget('diffx_setname', 'text', lambda: 'difx',
"set_name")
sl_norm = widgets.Checkbox(value=False)
html_w = widgets.HTML()
html_link_w = widgets.HTML()
nogenes = self.counttable.shape[0]
colv = [1] * nogenes
color_mapper = LinearColorMapper(palette="Inferno256",
low=-0.3,
high=2.5)
pdata = ColumnDataSource(
dict(x=[random.uniform(-10, 10) for x in range(nogenes)],
y=[random.uniform(-10, 10) for x in range(nogenes)],
mean_a=[3.1] * nogenes,
mean_b=[-3.1] * nogenes,
size=[0.1] * nogenes,
desc=list(self.counttable.index),
score=colv))
self._fdata = pdata
select_callback = CustomJS(args={'dsource': pdata},
code="""
var indici = dsource.selected['1d'].indices;
console.log(indici);
IPython.notebook.kernel.execute(
'T._DF_INDICI_SEL_ = ' + indici);
""")
bokeh_tools = [
bmodels.HoverTool(tooltips=[
("(mean,lfc)", "($x, $y)"),
("desc", "@desc"),
('mean a', "@mean_a"),
('mean b', "@mean_b"),
]),
bmodels.BoxSelectTool(callback=select_callback),
bmodels.PanTool(),
bmodels.WheelZoomTool(),
bmodels.BoxZoomTool(),
bmodels.LassoSelectTool(callback=select_callback),
bmodels.SaveTool(),
bmodels.ResetTool(),
bmodels.HelpTool(),
]
bfigure = bokeh_figure(plot_width=FIGSIZE[0],
plot_height=FIGSIZE[1],
tools=bokeh_tools,
toolbar_sticky=False,
toolbar_location='left',
title='diffexplot')
bfigure.title.text_color = 'darkgrey'
bfigure.title.text_font_style = 'normal'
bfigure.title.text_font_size = "12px"
self._ffigure = bfigure
bplot = bfigure.circle(x='x',
y='y',
radius='size',
source=pdata,
legend='score',
color=dict(field='score',
transform=color_mapper))
self._fplot = bplot
bhandle = bokeh_io.show(bfigure, notebook_handle=True)
#def save_geneset
def go_enrichr(*args):
idx = list(self._DF_INDICI_SEL_)
if len(idx) == 0:
return
genes = list(self.counttable.index.to_series()\
.iloc[idx])
if len(genes) == 0:
return
setname = sl_set_name.value
self.save_geneset(setname, genes)
genes_str = "\n".join(genes)
description = setname
if not description:
description = 'a set'
payload = {
'list': (None, genes_str),
'description': (None, description)
}
ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/addList'
response = requests.post(ENRICHR_URL, files=payload)
if not response.ok:
#print(response)
raise Exception('Error analyzing gene list')
data = json.loads(response.text)
shortid = data['shortId']
newurl = 'http://amp.pharm.mssm.edu/Enrichr/enrich?dataset='
newurl += shortid
js = '<script>window.open("{}", "_blank")</script>'.format(newurl)
html_link_w.value = js
sl_enrichr_link.on_click(go_enrichr)
# group defintion logic
def update_groups(*args):
#fill the menus with he correct values
group_a = sl_group_a.value
group_b = sl_group_b.value
meta_a = self.get_metadata(group_a)
meta_b = self.get_metadata(group_b) \
if group_b != group_a else meta_a
valc_a = meta_a.value_counts().sort_values(ascending=False)
valc_b = meta_b.value_counts().sort_values(ascending=False)
sl_set_a.options = \
['<all> -- {}'.format(len(meta_a))] + \
['{} -- {}'.format(a, b)
for (a,b) in valc_a.items()]
sl_set_b.options = \
['<all> -- {}'.format(len(meta_a))] + \
['{} -- {}'.format(a, b)
for (a,b) in valc_b.items()]
sl_set_a.value = sl_set_a.options[1]
update_groups()
sl_group_a.observe(update_groups, 'value')
sl_group_b.observe(update_groups, 'value')
def run(*args):
title = 'QDDE'
all_samples_set = set(self.counttable.columns)
logdata = pd.Series()
normalize = sl_norm.value
logdata['total cells'] = len(all_samples_set)
group_a = sl_group_a.value
group_b = sl_group_b.value
set_a = sl_set_a.value.split('--')[0].strip()
set_b = sl_set_b.value.split('--')[0].strip()
title += ' A:{}/{} B:{}/{}'.format(group_a, set_a, group_b, set_b)
meta_a = self.get_metadata(group_a)
meta_b = self.get_metadata(group_b) \
if group_b != group_a else meta_a
logdata['group a'] = set_a
logdata['group b'] = set_b
sample_a = copy.copy(all_samples_set) \
if set_a == '<all>' else set (meta_a[meta_a == set_a].index)
sample_b = copy.copy(all_samples_set) \
if set_b == '<all>' else set (meta_b[meta_b == set_b].index)
sample_a &= all_samples_set #ensure overlap with this count table
sample_b &= all_samples_set
sample_b -= sample_a # so we don't have any duplicates
logdata['cells in a'] = len(sample_a)
logdata['cells in b'] = len(sample_b)
cnts = self.counttable
if normalize:
cnts = 1e6 * cnts / cnts.sum()
if cnts.min().min() < 0:
#assume this is in log_space
logdata['assuming log space'] = True
cnts = 10**cnts
cnts_a = cnts.loc[:, sample_a]
cnts_b = cnts.loc[:, sample_b]
if cnts_a.shape[1] < 1: return
if cnts_b.shape[1] < 1: return
html_w.value = pd.DataFrame(logdata).to_html(header=False)
stats = pd.DataFrame(
dict(
mean_a=cnts_a.mean(1),
mean_b=cnts_b.mean(1),
mean_all=np.log10(cnts.mean(1)),
))
stats['a/b'] = stats['mean_a'] / stats['mean_b']
stats['lfc'] = np.log2(stats['a/b'])
# lfc_l = stats['lfc']
stats['no_cells_in_a'] = len(sample_a)
stats['no_cells_in_b'] = len(sample_b)
stats['name_a'] = '{}/{}'.format(group_a, set_a)
stats['name_b'] = '{}/{}'.format(group_b, set_b)
#print(stats.head())
#stats = stats.sort_values(by='lfc', ascending=False)
#bplot.data_source.data['x'] = stats['mean_a']
#bplot.data_source.data['y'] = stats['mean_b']
bplot.data_source.data['x'] = stats['mean_all']
bplot.data_source.data['y'] = stats['lfc']
bplot.data_source.data['mean_a'] = stats['mean_a']
bplot.data_source.data['mean_b'] = stats['mean_b']
m = stats['mean_all'].max()
bfigure.title.text = title
self.DIFX_stats = stats
bplot.data_source.data['size'] = [0.01 * m] * nogenes
bokeh_io.push_notebook(handle=bhandle)
sl_go.on_click(run)
run()
# display interface
display(ilabel('Group A', sl_group_a, sl_set_a))
display(ilabel('Group B (-A)', sl_group_b, sl_set_b))
display(ilabel('TPM normalize', sl_norm))
display(ilabel('Set Name', sl_set_name))
# tabs = widgets.Tab(children=tab_children)
# tabs.set_title(0, 'Define sets')
# display(tabs) sl_enrichr_link html_link_w
display(widgets.HBox([sl_go, sl_check, sl_save_set, sl_enrichr_link]))
display(html_w)
display(html_link_w)
def main(input_path, input_path_bmi, average, year_break):
# Welcoming message
print("\n############################")
print("## LINEAIR REGRESSION ##")
print("## v1.1 ##")
print("############################\n")
# Show the paths
print("USING PATHS:")
print(" - Price database: {}".format(input_path))
print(" - BMI database: {}".format(input_path_bmi))
# Read database
print("\nReading database...")
db_price = pd.read_csv(input_path)
print("Done, reading BMI database...")
db_BMI = pd.read_csv(input_path_bmi)
print("Done")
# Fill NaN
db_price = db_price.fillna(0)
db_BMI = db_BMI.fillna(0)
# Custom
print("Collecting graphs...")
price, BMI, price_years, BMI_years, total_things = collect_graphs(
db_price, db_BMI)
print("\nDone")
# Now plot price VS time
print("Plotting...")
# Create hover tool
hover = bkm.HoverTool(tooltips=[("Country",
"@country"), ("Year",
"@year"), ("Value",
"@value")])
# Create figure
f_price = plt.figure(title="Price per country, with Lineair Regression",
x_axis_label="Years",
y_axis_label="Price",
tools=[
hover,
bkm.WheelZoomTool(),
bkm.BoxZoomTool(),
bkm.PanTool(),
bkm.SaveTool(),
bkm.ResetTool()
],
width=900)
f_BMI = plt.figure(title="BMI per country, with Lineair Regression",
x_axis_label="Years",
y_axis_label="BMI",
tools=[
hover,
bkm.WheelZoomTool(),
bkm.BoxZoomTool(),
bkm.PanTool(),
bkm.SaveTool(),
bkm.ResetTool()
],
width=900)
# Generate random RGB list
RGBs = []
N = 8
for r in range(N):
for g in range(N):
for b in range(N):
if r == g == b == N:
# NO WHITE
continue
# Add to the RGBs list
RGBs.append(
bokeh.colors.RGB(int((r / N) * 255), int((g / N) * 255),
int((b / N) * 255)))
legend_list_price = []
legend_list_BMI = []
total_x_price = []
total_x_BMI = []
total_y_price = []
total_y_BMI = []
progress_bar = ProgressBar(max_amount=len(price))
for country in price:
price_list = price[country]
BMI_list = BMI[country]
price_year_list = price_years[country]
BMI_year_list = BMI_years[country]
# Now plot the graph
line_elem, circle_elem, new_RGBs = plot_list(f_price, price_year_list,
price_list, country, RGBs)
RGBs = list(new_RGBs)
legend_list_price.append(
(country + " (price)", [line_elem, circle_elem]))
# Do the same for BMI
line_elem, circle_elem, new_RGBs = plot_list(f_BMI, BMI_year_list,
BMI_list, country, RGBs)
RGBs = list(new_RGBs)
legend_list_BMI.append((country + " (BMI)", [line_elem, circle_elem]))
# Add to the total_x and total_y
total_x_price += price_list
total_x_BMI += BMI_list
total_y_price += price_year_list
total_y_BMI += BMI_year_list
progress_bar.update()
print("Done")
# Alright, we're nearly done: only add lineair regression
print("Doing lineair regression...")
if year_break > -1:
# Use this year to break
old_y = list(total_y_price)
old_x = list(total_x_price)
total_y_price = []
total_x_price = []
for i, year in enumerate(old_y):
if year >= year_break:
total_x_price.append(old_x[i])
total_y_price.append(year)
old_y = list(total_y_BMI)
old_x = list(total_x_BMI)
total_y_BMI = []
total_x_BMI = []
for i, year in enumerate(old_y):
if year >= year_break:
total_x_BMI.append(old_x[i])
total_y_BMI.append(year)
legend_list_price.append(
plot_lineair_regression(f_price, total_y_price, total_x_price))
legend_list_BMI.append(
plot_lineair_regression(f_BMI, total_y_BMI, total_x_BMI))
# Do legend and show
legend_price = bkm.Legend(items=legend_list_price,
location=(0, 0),
click_policy="mute")
legend_BMI = bkm.Legend(items=legend_list_BMI,
location=(0, 0),
click_policy="mute")
f_price.add_layout(legend_price, "right")
f_BMI.add_layout(legend_BMI, "right")
print("Done")
plt.output_file("lineair_regression.html", mode="inline")
layout = bokeh.layouts.Column(f_price, f_BMI)
plt.show(layout)
print("\nDone.")
FIGURES.remove('AdjointLayout') # bugs out (I think width/height)
DEFAULT_LABEL_SIZES = dict(
title='1.65em',
labels='1.2em',
ticks='1.05em',
legend='1.05em',
legend_title='0.9em',
colorbar='1.05em',
)
DEFAULT_TOOLS = dict(
pan=models.PanTool(),
box_zoom=models.BoxZoomTool(),
wheel_zoom=models.WheelZoomTool(),
save=models.SaveTool(),
reset=models.ResetTool()
)
class Mod(object):
def __init__(
self,
title_format=None,
title=None,
xlabel=None,
ylabel=None,
neaten_io=True,
width=None,
height=None,
autosize=True,
def do_graph_animation(graph_product, title, path, progress_bar):
# The total graph info
plt.output_file(path)
hover = bkm.HoverTool(tooltips=[("Country", "@names"), ("Region",
"@region")])
f = plt.figure(title=title,
x_axis_label=XLABEL,
y_axis_label=YLABEL,
tools=[
hover,
bkm.WheelZoomTool(),
bkm.BoxZoomTool(),
bkm.PanTool(),
bkm.SaveTool(),
bkm.ResetTool()
],
width=900)
legend_list = []
sources = []
data = {}
country_data = {}
lowest_year = float("inf")
highest_year = -float("inf")
# Real quick, determine the boundries
for region in graph_product:
for timestamp in graph_product[region]:
if timestamp > highest_year:
highest_year = timestamp
if timestamp < lowest_year:
lowest_year = timestamp
# Construct the new plot
for region in graph_product:
if region not in country_data:
country_data[region] = {}
lowest_region_year = float("inf")
for timestamp in graph_product[region]:
x_list, y_list = graph_product[region][timestamp]
# Create the dataset
data = {"x": [], "y": [], "names": [], "region": []}
if timestamp < lowest_region_year:
lowest_region_year = timestamp
for country in x_list:
country_price = x_list[country]
country_BMI = y_list[country]
# Save the data
data["x"].append(country_price)
data["y"].append(country_BMI)
data["names"].append(country)
data["region"].append(region)
# Append country_data
country_data[region][timestamp] = data
# Now create a nice source of the data
if lowest_region_year == lowest_year:
data = country_data[region][lowest_year]
else:
data = {
"x": [0] * len(country_data[region][lowest_region_year]["x"]),
"y": [0] * len(country_data[region][lowest_region_year]["x"]),
"names": country_data[region][lowest_region_year]["names"],
"region": country_data[region][lowest_region_year]["region"]
}
data_source = bokeh.models.ColumnDataSource(data=data)
sources.append([data_source, region])
# Now plot it
region_color = REGION_NAME_2_COLOR[region]
elem = f.scatter("x",
"y",
source=data_source,
color=region_color,
muted_color=region_color,
muted_alpha=0.1,
size=10)
# Add elem to the legends
legend_list.append((region, [elem]))
# Do dat progress bar
progress_bar.update()
first_time = False
legend = bokeh.models.Legend(items=legend_list,
location=(0, 0),
click_policy="mute")
f.x_range = bkm.DataRange1d(start=0, end=3.6)
f.y_range = bkm.DataRange1d(start=19, end=32)
# Make the slider
callback = bokeh.models.CustomJS(args=dict(sources=sources,
all_data=country_data),
code=callback_code)
slider = bokeh.models.Slider(start=lowest_year,
end=highest_year,
value=lowest_year,
step=1,
title="Year")
slider.js_on_change('value', callback)
f.add_layout(legend, 'right')
layout = bokeh.layouts.column(bokeh.layouts.widgetbox(slider), f)
plt.save(layout)