def update_table():
item_list = main_scatter_source.data['id']
if isinstance(item_list, pandas.core.series.Series):
item_list = item_list.values.tolist()
if len(current_indices)>0 and len(item_list)>0:
selected_id = list()
for i in current_indices:
selected_item_id = item_list[i]
selected_id.append(selected_item_id)
new_df = df[(df.attr_name==table_selected_attribute) & (df.id.isin(selected_id))].groupby(['attr_value']).agg({'id':'count'}).sort_values(by='id', ascending=False).rename(columns={'id':'n'})
joined_df = calculate_ratio(new_df)
table_source.data = ColumnDataSource._data_from_df(joined_df)
else:
new_df = df[(df.attr_name==table_selected_attribute)].groupby(['attr_value']).agg({'id':'count'}).sort_values(by='id', ascending=False).rename(columns={'id':'n'})
joined_df = calculate_ratio(new_df)
table_source.data = ColumnDataSource._data_from_df(joined_df)
def update_select(select_sec, app):
# TODO: Definitely needs a better design to avoid using globals...
global source
global csource
global df
global index
ss = app.objects['select_sec'].value
df = load_symbol(ss)
index = df.pop('dt')
df = df[num_columns]
df['ind'] = list(range(len(index)))
for name in num_columns:
stdv = df[name].std()
values = [stdv for x in df.index]
df['%s_std_dv' % name] = values
source = ColumnDataSource(df)
source.tags = ['main_source']
ndt = DataTable(source=source, columns=[], editable=True, width=500)
charts_box = AppVBox(app=app, children=[], name='charts_box')
templ = """FOUND %s CRASHES FOR SYMBOL %s"""
crashes_info = check_for_crash(ss)
crashes_source = ColumnDataSource(crashes_info)
ccolumns = [TableColumn(field=x, title=x, editor=NumberEditor()) for x in crashes_info.columns]
txt = PreText(text=templ % (len(crashes_info), ss), width=500, height=100)
crashes_dt = DataTable(source=crashes_source, columns=ccolumns, editable=True, width=500)
crashes_box = AppVBox(app=app, children=[txt, crashes_dt], name='crash_stats')
new_ms = MultiSelect.create(options=[], name='ms')
return {
'charts_box': charts_box,
'dt': ndt,
'crash_stats': crashes_box,
'ms': new_ms,
}
def candle_stix(dataframe):
'''
This function takes in dataframes and makes a candlesticks plot using bokeh
Parameters
----------
dataframe Stock dataframe
This frame has elements: Open, Volume, Close, High, Low, Symbol and Date
'''
# Develop Aspects and Calculations
mids = (dataframe["Open"] + dataframe["Close"])/2
spans = abs(dataframe.Close-dataframe.Open)
inc = dataframe.Close > dataframe.Open
dec = dataframe.Open > dataframe.Close
w = 12*60*60*1000 #width about a half day in width
# Define Hover tool tips
Gain = ColumnDataSource(ColumnDataSource.from_df(dataframe[inc]))
Loss = ColumnDataSource(ColumnDataSource.from_df(dataframe[dec]))
hover = HoverTool(
tooltips=[
("Price","$y"),
("High","@High"),
("Low","@Low"),
("Open","@Open"),
("Close","@Close"),
("Volume(M)","@Volume")
]
)
# Build Plot
output_file("candlestix.html", title="Candlestix")
TOOLS = "pan,wheel_zoom,box_zoom"
p = figure(x_axis_type="datetime",tools=[TOOLS,hover],plot_width=800,toolbar_location="left")
p.segment(dataframe.Time, dataframe.Low ,dataframe.Time, dataframe.High , color="black")
p.rect(dataframe.Time[inc], mids[inc], w, spans[inc], source=Gain, fill_color="white", line_color="black")
p.rect(dataframe.Time[dec], mids[dec], w, spans[dec], source=Loss, fill_color="black", line_color="black")
p.title = str(str(dataframe["Symbol"][1]))
show(p)
def create_objects(cls, securities):
ranks = utils.get_pumps_rank()
quotient_metrics = utils.get_quotient_metrics()
ranks['quotient'] = quotient_metrics['quotient']
foo = lambda x: utils.spam_counts.get(x, 0)
ranks['spams'] = map(foo, ranks['symbol'])
ranks = ranks.sort('quotient', ascending=False)
cls._pre_filtered_ranks = {
'All': {k: ranks[k] for k in ranks.columns},
'Stocks with Spam': dict(ranks[ranks['symbol'].
isin(plugins.spam_counts.keys())].
sort('quotient', ascending=False)),
'Stocks without Spam': dict(ranks[~ranks['symbol'].
isin(plugins.spam_counts.keys())].
sort('quotient', ascending=False)),
}
source_stocks_rank = ColumnDataSource(cls._pre_filtered_ranks['Stocks with Spam'])
table_stocks_rank = DataTable(
source=source_stocks_rank, width=785, height=400,
selectable=True, editable=True,
columns=[
TableColumn(field='symbol', title='symbol', editor=StringEditor()),
TableColumn(field='quotient', title='quotient', editor=StringEditor()),
TableColumn(field='rank', title='rank', editor=StringEditor()),
TableColumn(field='spams', title='spams', editor=StringEditor()),
])
callback = Callback(
args={'tr': table_stocks_rank, 'sr': source_stocks_rank},
code=callbacks.source_stocks_rank
)
source_stocks_rank.callback = callback
return locals()
def set_sources_heatmap(self):
from bokeh.plotting import ColumnDataSource
# initialize empty source
if not hasattr(self, '_source'):
self._source = ColumnDataSource(data=dict())
# remove mask from data -> not necessary soon? / convert to nan?
var = np.ma.getdata(self._heatmap_var)
self._source.add([var], 'image')
self._source.add([self._data_xmin], 'x')
self._source.add([self._data_ymin], 'y')
self._source.add([self._data_xmax - self._data_xmin], 'dw')
self._source.add([self._data_ymax - self._data_ymin], 'dh')
def make_dashboard(motors, xy_train, operating_earnings, maintenance_cost, repair_cost, run_interval):
#calculate revenue vs time dataframe
print '...generating dashboard...'
events = get_events(motors)
events['earnings'] = 0.0
events.loc[events.state == 'operating', 'earnings'] = operating_earnings
events['expenses'] = 0.0
events.loc[events.state == 'maintenance', 'expenses'] = maintenance_cost
events.loc[events.state == 'repair', 'expenses'] = repair_cost
money = events.groupby('Time').sum()[['earnings', 'expenses']]
money['revenue'] = money.earnings - money.expenses
money['cumulative_earnings'] = money.earnings.cumsum()
money['cumulative_expenses'] = money.expenses.cumsum()
money['cumulative_revenue'] = money.revenue.cumsum()
#map the (P,T) decision surface
T_min = 50
T_max = 150
P_min = 0
P_max = 100
T_axis = np.arange(T_min, T_max, 0.5)
P_axis = np.arange(P_min, P_max, 0.5)
x, y = np.meshgrid(T_axis, P_axis)
ttf = np.zeros((len(P_axis), len(T_axis)))
import copy
m = copy.deepcopy(motors[0])
for p_idx in np.arange(len(P_axis)):
for t_idx in np.arange(len(T_axis)):
m.Temp = T_axis[t_idx]
m.Pressure = P_axis[p_idx]
ttf[p_idx, t_idx] = m.predicted_time_to_fail()
#plot decision surface
from bokeh.plotting import figure, show, output_file, ColumnDataSource, vplot
from bokeh.models import HoverTool, Callback
from bokeh.io import vform
output_file('dashboard.html', title='Smart Maintenance Dashboard')
source = ColumnDataSource(
data=dict(
x = xy_train.Temp,
y = xy_train.Pressure,
ttf = xy_train.time_to_fail,
size = 0.6*xy_train.time_to_fail,
)
)
dec_fig = figure(x_range=[T_min, T_max], y_range=[P_min, P_max], title='SVM Decision Surface',
x_axis_label='Temperature', y_axis_label='Pressure', tools='box_zoom,reset,hover,crosshair',
width=600, plot_height=600)
dec_fig.title_text_font_size = '18pt'
dec_fig.xaxis.axis_label_text_font_size = '14pt'
dec_fig.yaxis.axis_label_text_font_size = '14pt'
dec_fig.image(image=[-ttf], x=[T_min], y=[P_min], dw=[T_max - T_min], dh=[P_max - P_min],
palette='RdYlGn8')
dec_fig.x('x', 'y', size='size', source=source, fill_alpha=0.5, fill_color='navy',
line_color='navy', line_width=1, line_alpha=0.5)
hover = dec_fig.select(dict(type=HoverTool))
hover.tooltips = [
("Temperature", "@x"),
("Pressure", "@y"),
("measured lifetime", "@ttf"),
]
#plot earnings vs time
source = ColumnDataSource(
data=dict(
t = money.index,
earnings = money.cumulative_earnings/1.e6,
expenses = money.cumulative_expenses/1.e6,
revenue = money.cumulative_revenue/1.e6,
zero = money.cumulative_revenue*0,
)
)
earn_fig = figure(title='Cumulative Earnings & Expenses', x_axis_label='Time',
y_axis_label='Earnings & Expenses (M$)', tools='box_zoom,reset,hover,crosshair',
width=1000, plot_height=300, x_range=[0, 1200], y_range=[0, 120])
earn_fig.title_text_font_size = '15pt'
earn_fig.xaxis.axis_label_text_font_size = '11pt'
earn_fig.yaxis.axis_label_text_font_size = '11pt'
earn_fig.line('t', 'earnings', color='blue', source=source, line_width=5, legend='earnings')
earn_fig.line('t', 'expenses', color='red', source=source, line_width=5, legend='expenses')
earn_fig.legend.orientation = "bottom_right"
earn_fig.patch([0, 200, 200, 0], [0, 0, 120, 120], color='lightsalmon', alpha=0.35,
line_width=0)
earn_fig.patch([200, 400, 400, 200], [0, 0, 120, 120], color='gold', alpha=0.35,
line_width=0)
earn_fig.patch([400, 1200, 1200, 400], [0, 0, 120, 120], color='darkseagreen',
alpha=0.35, line_width=0)
earn_fig.text([45], [101], ['run-to-fail'])
earn_fig.text([245], [101], ['scheduled'])
earn_fig.text([245], [90], ['maintenance'])
earn_fig.text([445], [101], ['predictive'])
earn_fig.text([445], [90], ['maintenance'])
hover = earn_fig.select(dict(type=HoverTool))
hover.tooltips = [
(" Time", "@t"),
(" earning (M$)", "@earnings"),
("expenses (M$)", "@expenses"),
]
#plot revenue vs time
rev_fig = figure(title='Cumulative Revenue', x_axis_label='Time',
y_axis_label='Revenue (M$)', tools='box_zoom,reset,hover,crosshair',
width=1000, plot_height=300, x_range=[0, 1200], y_range=[-15, 10])
rev_fig.title_text_font_size = '15pt'
rev_fig.xaxis.axis_label_text_font_size = '11pt'
rev_fig.yaxis.axis_label_text_font_size = '11pt'
rev_fig.line('t', 'revenue', color='green', source=source, line_width=5, legend='revenue')
rev_fig.line('t', 'zero', color='purple', source=source, line_width=3, alpha=0.5,
line_dash=[10, 5])
rev_fig.legend.orientation = "bottom_right"
rev_fig.patch([0, 200, 200, 0], [-15, -15, 10, 10], color='lightsalmon', alpha=0.35,
line_width=0)
rev_fig.patch([200, 400, 400, 200], [-15, -15, 10, 10], color='gold', alpha=0.35,
line_width=0)
rev_fig.patch([400, 1200, 1200, 400], [-15, -15, 10, 10], color='darkseagreen',
alpha=0.35, line_width=0)
hover = rev_fig.select(dict(type=HoverTool))
hover.tooltips = [
(" Time", "@t"),
(" revenue (M$)", "@revenue"),
]
#plot number of motors vs time
N = events.groupby(['Time', 'state']).count().unstack()['id'].reset_index()
N.fillna(value=0, inplace=True)
N['total'] = N.maintenance + N.operating + N.repair
s1 = ColumnDataSource(
data=dict(
Time = N.Time,
operating = N.operating,
maintenance = N.maintenance,
repair = N.repair,
total = N.total,
)
)
motor_fig = figure(title='Number of Motors', x_axis_label='Time',
y_axis_label='Number of motors', tools='box_zoom,reset,hover,crosshair',
width=1000, plot_height=300, x_range=[0, 1200], y_range=[-10, 210])
motor_fig.title_text_font_size = '15pt'
motor_fig.xaxis.axis_label_text_font_size = '11pt'
motor_fig.yaxis.axis_label_text_font_size = '11pt'
motor_fig.line('Time', 'total', color='blue', source=s1, line_width=3, legend='total')
motor_fig.line('Time', 'operating', color='green', source=s1, line_width=3, legend='operating')
motor_fig.line('Time', 'maintenance', color='orange', source=s1, line_width=3, legend='maintenance')
motor_fig.line('Time', 'repair', color='red', source=s1, line_width=3, legend='repair')
motor_fig.legend.orientation = "top_right"
motor_fig.patch([0, 200, 200, 0], [-10, -10, 210, 210], color='lightsalmon', alpha=0.35,
line_width=0)
motor_fig.patch([200, 400, 400, 200], [-10, -10, 210, 210], color='gold', alpha=0.35,
line_width=0)
motor_fig.patch([400, 1200, 1200, 400], [-10, -10, 210, 210], color='darkseagreen',
alpha=0.35, line_width=0)
#display N table
from bokeh.models.widgets import DataTable, TableColumn
from bokeh.io import vform
columns = [
TableColumn(field='Time', title='Time'),
TableColumn(field='operating', title='operating'),
TableColumn(field='maintenance', title='maintenance'),
TableColumn(field='repair', title='repair'),
TableColumn(field='total', title='total'),
]
s2 = s1.clone()
N_table = DataTable(source=s2, columns=columns, width=600, height=300)
s1.callback = Callback(args=dict(s2=s2), code="""
var inds = cb_obj.get('selected')['1d'].indices;
var d1 = cb_obj.get('data');
var d2 = s2.get('data');
d2['Time'] = []
d2['operating'] = []
d2['maintenance'] = []
d2['repair'] = []
d2['total'] = []
for (i = 0; i < inds.length; i++) {
d2['Time'].push(d1['Time'][inds[i]])
d2['operating'].push(d1['operating'][inds[i]])
d2['maintenance'].push(d1['maintenance'][inds[i]])
d2['repair'].push(d1['repair'][inds[i]])
d2['total'].push(d1['total'][inds[i]])
}
s2.trigger('change');
""")
#export plot to html and return
plot_grid = vplot(dec_fig, earn_fig, rev_fig, motor_fig, vform(N_table))
show(plot_grid, new='tab')
return money, events,N
def create_objects(cls, symbol, df, securities):
descr_box = Paragraph(text='content loading...')
btn_close_loading = Button(label='Close Loading')
dialog_loading = Dialog(
title='loading', content=vplot(descr_box), name='loading_dialog',
buttons=[btn_close_loading], visible=False)
source_data = dict(df)
main_source = ColumnDataSource(dict(df))
source = ColumnDataSource(source_data)
# TODO: REMOVE THIS COMMENTED CODE! IT'S JUST THE PREVIOUS
# VERSION USED BEFORE NEW P&D Cached results and algorithm
# get the cached results of the P&D algorithm computed with the
# "default" configuration
# intervals = utils.cached_pumps.get(symbol, pumps.to_dicts(((),(),(),(),(),())))
# intervals['bottom'] = [0] * len(intervals['start'])
# intervals['values'] = [max(df['price'])] * len(intervals['start'])
#
# intervals = pd.DataFrame(intervals)
# new version
stats = utils.get_symbols_cached_stats()[symbol]
intervals = pd.DataFrame(stats)
intervals['bottom'] = [0] * len(intervals['start'])
intervals['values'] = [max(df['price'])] * len(intervals['start'])
conv = lambda x: utils.to_seconds(pd.to_datetime(x))
intervals = intervals[
(pd.to_datetime(intervals['start']) > conv(config.date_range[0])) &
(pd.to_datetime(intervals['start']) < conv(config.date_range[1]))
]
# Create P&Ds intervals DataSource
intervals_source = ColumnDataSource(intervals)
source.tags = ['main_source']
trends = utils.load_trends_data(symbol, start_date=min(df['dt']))
trends_source = ColumnDataSource(trends)
trades = Slider(
title="trades", name='trades',
value=0, start=0, end=124, step=1
)
# Selectors
symbol = Select.create(
options=securities, value=symbol, name='symbol', title=""
)
window_selector = Select.create(
options=['---'], name='period_selector', title="Search intervals with:"
)
symbol_filter = Select.create(
options=['All', 'Stocks with Spam', 'Stocks without Spam'],
name='symbol_filter', title="Filter Symbols:",
value='Stocks with Spam'
)
callback = Callback(
args={'symbol_filter': symbol_filter,
'dialog_loading': dialog_loading},
code=callbacks.symbol_filter
)
symbol_filter.callback = callback
btn_detect_pumps = Button(label='Configure P&D Detection', name='config_pumps')
main_tab = Panel(title="Main")
tabs = Tabs()
# Create STOCKS TABLE
ranks = utils.get_pumps_rank()
# quotient_metrics = utils.get_quotient_metrics()
# ranks['quotient'] = quotient_metrics['quotient']
foo = lambda x: utils.spams_count.get(x, 0)
ranks['spams'] = map(foo, ranks['symbol'])
ranks = ranks.sort(['spams', 'vol_quotient'], ascending=False)
cls._pre_filtered_ranks = {
'All': {k: ranks[k] for k in ranks.columns},
'Stocks with Spam': dict(ranks[ranks['spams'] > 0].
sort('vol_quotient', ascending=False)),
'Stocks without Spam': dict(ranks[ranks['spams'] == 0].
sort('vol_quotient', ascending=False)),
}
source_stocks_rank = ColumnDataSource(cls._pre_filtered_ranks['All'])
table_stocks_rank = DataTable(
source=source_stocks_rank, width=560, height=450,
selectable=True, editable=True,
columns=[
TableColumn(field='symbol', title='symbol', width=130, editor=StringEditor()),
TableColumn(field='vol_quotient', title='volume ratio', editor=StringEditor(),
default_sort='descending'),
TableColumn(field='risk_score', title='risk', width=100, editor=StringEditor(),
default_sort='descending'),
TableColumn(field='spams', title='spams', width=130, editor=StringEditor(),
default_sort='descending'),
])
callback = Callback(args={'tr': table_stocks_rank, 'sr': source_stocks_rank, 'symb': symbol,
'dialog_loading': dialog_loading},
code=callbacks.source_stocks_rank)
source_stocks_rank.callback = callback
return locals()
def low_state_bars(death_types, death_type_state, states_dict):
output_file("low_bar_state.html")
high_low_states = [
'Hawaii', 'Rhode Island', 'New Hampshire', 'Massachusetts'
]
all_types_list = []
for word in death_types:
type_list = []
for state in high_low_states:
type_list.append(death_type_state[state][word])
all_types_list.append(type_list)
colors = [
"#2ad123", "#1b7c17", "#1518d8", "#4e50e5", "#4d71e5", "#4d9be5",
"#f9ed3b", "#bc7f14", "#b74207", "#e51010", "#ff8c1a"
]
data = {
'States': high_low_states,
death_types[0]: all_types_list[0],
death_types[1]: all_types_list[1],
death_types[2]: all_types_list[2],
death_types[3]: all_types_list[3],
death_types[4]: all_types_list[4],
death_types[5]: all_types_list[5],
death_types[6]: all_types_list[6],
death_types[7]: all_types_list[7],
death_types[8]: all_types_list[8],
death_types[9]: all_types_list[9],
death_types[10]: all_types_list[10]
}
source = ColumnDataSource(data=data)
TOOLS = "pan,wheel_zoom,reset,save"
p = figure(
x_range=high_low_states,
plot_height=400,
plot_width=600,
title="Causes of death in states with lowest amount of incidents",
toolbar_location=None,
tools=TOOLS)
rs = p.vbar_stack(death_types,
x='States',
width=0.9,
color=colors,
source=source)
p.y_range.start = 0
p.x_range.range_padding = 0.1
p.xgrid.grid_line_color = None
p.axis.minor_tick_line_color = None
p.outline_line_color = None
p.xaxis.major_label_orientation = "vertical"
p.yaxis[0].formatter = NumeralTickFormatter(format='0 %')
legend = Legend(items=[(death, [r])
for (death, r) in zip(death_types, rs)],
location=(0, 30))
p.add_layout(legend, 'right')
def make_html_plot(d,
output_dir=os.path.join(my_config.figure_location,
'mouseover'),
n=30000,
num_imgs=2000,
title=""):
"""Make an HTML tooltip mouse-over plot."""
required_fields = ['image', 'embedding']
d = update_data(d, required_fields, n=n)
embedding = d['embedding']
specs = d['image']
# Write the tooltip images.
if not os.path.exists(output_dir):
os.makedirs(output_dir)
num_imgs = min(len(specs), num_imgs)
for i in tqdm(range(num_imgs)):
save_image(specs[i], os.path.join(output_dir, str(i) + '.jpg'))
output_file(os.path.join(output_dir, "mouseover.html"))
source = ColumnDataSource(data=dict(
x=embedding[:num_imgs, 0],
y=embedding[:num_imgs, 1],
imgs=[
os.path.join(output_dir,
str(i) + '.jpg') for i in range(num_imgs)
],
))
source2 = ColumnDataSource(data=dict(
x=embedding[num_imgs:, 0],
y=embedding[num_imgs:, 1],
))
p = figure(plot_width=800, plot_height=600, title=title)
p.scatter('x',
'y',
size=3,
fill_color='blue',
fill_alpha=0.1,
source=source2)
tooltip_points = p.scatter('x',
'y',
size=5,
fill_color='red',
source=source)
hover = HoverTool(renderers=[tooltip_points],
tooltips="""
<div>
<div>
<img
src="@imgs" height="128" alt="@imgs" width="128"
style="float: left; margin: 0px 0px 0px 0px;"
border="1"
></img>
</div>
</div>
""")
p.add_tools(hover)
p.title.align = "center"
p.title.text_font_size = "25px"
p.axis.visible = False
p.xgrid.visible = False
p.ygrid.visible = False
show(p)
from bokeh.plotting import ColumnDataSource
# Import output_file and show from bokeh.io
from bokeh.io import output_file, show
N = 4
state = "NY"
# Create a ColumnDataSource: source
source = ColumnDataSource(
data={
'positive':
df_full[df_full.state == state].iloc[:(N + 2)].positive.tolist(),
'new_cases':
df_full[df_full.state == state].iloc[:(N + 2)].new_cases.tolist(),
"date_string":
df_full[df_full.state == state].iloc[:(N + 2)].date_string.tolist(),
"week_number":
df_full[df_full.state == state].iloc[:(N + 2)].week_number.tolist(),
"death":
df_full[df_full.state == state].iloc[:(N + 2)].death.tolist(),
"color_test":
df_full[df_full.state == state].iloc[:(N + 2)].color.tolist()
})
# Create a figure with x_axis_type='datetime': p
p = figure(tools="",
title="COVID-19 development in each US state",
plot_width=700 * 0.65,
plot_height=400 * 0.65,
x_axis_type='log',
y_axis_type='log',
f = open("mappingdf.pkl", 'rb')
df = pickle.load(f)
f.close
reset_output()
def display_data(selectedYear):
yr = selectedYear
df_yr = df[df['year_of_sale'] == yr]
df_yr = df_yr[df_yr['sale_price'] >= 100000]
return df_yr
source = ColumnDataSource(data=display_data(2017))
pal = RdYlGn[6]
mapper = log_cmap(field_name="sale_price",
palette=pal,
low=100000,
low_color='green',
high=23000000)
tooltips = [("Price", "@sale_price"), ("Address", "@address"),
("Neighborhood", "@neighborhood")]
slider = Slider(start=2003, end=2017, step=1, value=2017, title='Year')
fig = figure(x_axis_type='mercator',
y_axis_type='mercator',
tooltips=tooltips,
def plot_with_bokeh(self, figure_width=5):
"""Plot the graphic record using Bokeh.
Examples
--------
>>>
"""
if not BOKEH_AVAILABLE:
raise ImportError("``plot_with_bokeh`` requires Bokeh installed.")
if not PANDAS_AVAILABLE:
raise ImportError("``plot_with_bokeh`` requires Pandas installed.")
ax, (features_levels, plot_data) = self.plot(figure_width=figure_width)
width, height = [int(100 * e) for e in ax.figure.get_size_inches()]
plt.close(ax.figure)
max_y = max([data["annotation_y"] for f, data in plot_data.items()] +
list(features_levels.values()))
hover = HoverTool(tooltips="@hover_html")
p = figure(plot_width=width,
plot_height=height,
tools=[hover, "xpan,xwheel_zoom,reset,tap"],
x_range=Range1d(0, self.sequence_length),
y_range=Range1d(-1, max_y + 1))
p.patches(xs='xs',
ys='ys',
color='color',
line_color="#000000",
source=ColumnDataSource(
pd.DataFrame.from_records([
bokeh_feature_patch(
self,
feature.start,
feature.end,
feature.strand,
level=level,
color=feature.color,
label=feature.label,
hover_html=(feature.html if feature.html
is not None else feature.label))
for feature, level in features_levels.items()
])))
if plot_data != {}:
p.text(x='x',
y='y',
text='text',
text_align="center",
text_font_size='12px',
text_font="arial",
text_font_style="normal",
source=ColumnDataSource(
pd.DataFrame.from_records([
dict(x=feature.x_center,
y=pdata["annotation_y"],
text=feature.label,
color=feature.color)
for feature, pdata in plot_data.items()
])))
p.segment(x0='x0',
x1='x1',
y0='y0',
y1='y1',
line_width=0.5,
color="#000000",
source=ColumnDataSource(
pd.DataFrame.from_records([
dict(x0=feature.x_center,
x1=feature.x_center,
y0=pdata["annotation_y"],
y1=pdata["feature_y"])
for feature, pdata in plot_data.items()
])))
p.yaxis.visible = False
p.outline_line_color = None
p.grid.grid_line_color = None
p.toolbar.logo = None
return p
def dataGatherer(m, qdata):
"""
Instrument/plot/query specific contortions needed to make the
bulk of the plot code generic and abstract. I feel ok
hardcoding stuff in here at least, since this will always be namespace
protected and unambigious (e.g. instrumentTelem.dataGatherer).
"""
pdata = OrderedDict()
for qtag in m.queries.keys():
pdata.update({qtag: qdata[qtag]})
r = pdata['q_tcssv']
r2 = pdata['q_tcslois']
r3 = pdata['q_cubeinstcover']
r4 = pdata['q_cubefolds']
r5 = pdata['q_aossv']
# Common "now" time to compare everything against
now = np.datetime64(dt.datetime.utcnow())
# These are from other data sources, so get their values too
domeshut = bplot.getLast(r2, "DomeShutter", compTime=now)
mirrorcov = bplot.getLast(r, "MirrorCover", compTime=now)
# We use the nullVal parameter for these so we can catch the
# .likelyInvalid parameters in the final table data collection since
# they have special logic a little down below
instcover = bplot.getLast(r3, "InstCover", compTime=now, nullVal=-1)
portT = bplot.getLast(r4, 'PortThru', compTime=now, nullVal=-1)
portA = bplot.getLast(r4, 'PortA', compTime=now, nullVal=-1)
portB = bplot.getLast(r4, 'PortB', compTime=now, nullVal=-1)
portC = bplot.getLast(r4, 'PortC', compTime=now, nullVal=-1)
portD = bplot.getLast(r4, 'PortD', compTime=now, nullVal=-1)
m2piston = bplot.getLast(r5,
'M2PistonDemand',
label="Demand M2 Piston",
compTime=now,
scaleFactor=1e6,
fstr="%.3f")
totfocus = bplot.getLast(r5,
'totalFocusOffset',
label="Total Focus Offset",
compTime=now,
scaleFactor=1e6,
fstr="%.3f")
# Finally done! Now put it all into a list so it can be passed
# back a little easier and taken from there
tableDat = [
domeshut, mirrorcov, instcover, m2piston, totfocus, portT, portA,
portB, portC, portD
]
values = []
labels = []
tooold = []
for each in tableDat:
if each.label == "InstCover":
# Special conversion to text for this one
if each.value == 0:
values.append("Closed")
elif each.value == -1:
values.append(bplot.funnyValues())
else:
values.append("Open")
labels.append(each.label)
elif each.label.startswith("Port"):
if each.value == 0:
values.append("Inactive")
elif each.value == -1:
values.append(bplot.funnyValues())
else:
values.append("Active")
labels.append(each.label)
else:
values.append(each.value)
labels.append(each.label)
# Rather than put this in each elif, I'll just do it here.
# Add in our age comparison column, for color/styling later
tooold.append(each.tooOld)
mds = dict(labels=labels, values=values, ageStatement=tooold)
cds = ColumnDataSource(mds)
return cds
def calculate_proxy_svg(snp, pop, request, r2_d="r2"):
# Set data directories using config.yml
with open('config.yml', 'r') as f:
config = yaml.load(f)
vcf_dir = config['data']['vcf_dir']
tmp_dir = "./tmp/"
# Ensure tmp directory exists
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
if request is False:
request = str(time.strftime("%I%M%S"))
# Create JSON output
# Find coordinates (GRCh37/hg19) for SNP RS number
# Connect to Mongo snp database
client = MongoClient(
'mongodb://' + username + ':' + password + '@localhost/admin', port)
db = client["LDLink"]
def get_coords(db, rsid):
rsid = rsid.strip("rs")
query_results = db.dbsnp151.find_one({"id": rsid})
query_results_sanitized = json.loads(json_util.dumps(query_results))
return query_results_sanitized
# Query genomic coordinates
def get_rsnum(db, coord):
temp_coord = coord.strip("chr").split(":")
chro = temp_coord[0]
pos = temp_coord[1]
query_results = db.dbsnp151.find({
"chromosome":
chro.upper() if chro == 'x' or chro == 'y' else chro,
"position":
pos
})
query_results_sanitized = json.loads(json_util.dumps(query_results))
return query_results_sanitized
# Replace input genomic coordinates with variant ids (rsids)
def replace_coord_rsid(db, snp):
if snp[0:2] == "rs":
return snp
else:
snp_info_lst = get_rsnum(db, snp)
print("snp_info_lst")
print(snp_info_lst)
if snp_info_lst != None:
if len(snp_info_lst) > 1:
var_id = "rs" + snp_info_lst[0]['id']
ref_variants = []
for snp_info in snp_info_lst:
if snp_info['id'] == snp_info['ref_id']:
ref_variants.append(snp_info['id'])
if len(ref_variants) > 1:
var_id = "rs" + ref_variants[0]
elif len(ref_variants) == 0 and len(snp_info_lst) > 1:
var_id = "rs" + snp_info_lst[0]['id']
else:
var_id = "rs" + ref_variants[0]
return var_id
elif len(snp_info_lst) == 1:
var_id = "rs" + snp_info_lst[0]['id']
return var_id
else:
return snp
else:
return snp
return snp
snp = replace_coord_rsid(db, snp)
# Find RS number in snp database
snp_coord = get_coords(db, snp)
# Get population ids from LDproxy.py tmp output files
pop_list = open(tmp_dir + "pops_" + request + ".txt").readlines()
ids = []
for i in range(len(pop_list)):
ids.append(pop_list[i].strip())
pop_ids = list(set(ids))
# Extract query SNP phased genotypes
vcf_file = vcf_dir + \
snp_coord['chromosome'] + ".phase3_shapeit2_mvncall_integrated_v5.20130502.genotypes.vcf.gz"
tabix_snp_h = "tabix -H {0} | grep CHROM".format(vcf_file)
proc_h = subprocess.Popen(tabix_snp_h, shell=True, stdout=subprocess.PIPE)
head = [x.decode('utf-8')
for x in proc_h.stdout.readlines()][0].strip().split()
tabix_snp = "tabix {0} {1}:{2}-{2} | grep -v -e END > {3}".format(
vcf_file, snp_coord['chromosome'], snp_coord['position'],
tmp_dir + "snp_no_dups_" + request + ".vcf")
subprocess.call(tabix_snp, shell=True)
# Check SNP is in the 1000G population, has the correct RS number, and not
# monoallelic
vcf = open(tmp_dir + "snp_no_dups_" + request + ".vcf").readlines()
if len(vcf) == 0:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
return None
elif len(vcf) > 1:
geno = []
for i in range(len(vcf)):
if vcf[i].strip().split()[2] == snp:
geno = vcf[i].strip().split()
if geno == []:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf",
shell=True)
return None
else:
geno = vcf[0].strip().split()
if geno[2] != snp:
snp = geno[2]
if "," in geno[3] or "," in geno[4]:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
return None
index = []
for i in range(9, len(head)):
if head[i] in pop_ids:
index.append(i)
genotypes = {"0": 0, "1": 0}
for i in index:
sub_geno = geno[i].split("|")
for j in sub_geno:
if j in genotypes:
genotypes[j] += 1
else:
genotypes[j] = 1
if genotypes["0"] == 0 or genotypes["1"] == 0:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
return None
# Define window of interest around query SNP
window = 500000
coord1 = int(snp_coord['position']) - window
if coord1 < 0:
coord1 = 0
coord2 = int(snp_coord['position']) + window
# Calculate proxy LD statistics in parallel
threads = 4
block = (2 * window) // 4
commands = []
for i in range(threads):
if i == min(range(threads)) and i == max(range(threads)):
command = "python LDproxy_sub.py " + "True " + snp + " " + \
snp_coord['chromosome'] + " " + str(coord1) + " " + \
str(coord2) + " " + request + " " + str(i)
elif i == min(range(threads)):
command = "python LDproxy_sub.py " + "True " + snp + " " + \
snp_coord['chromosome'] + " " + str(coord1) + " " + \
str(coord1 + block) + " " + request + " " + str(i)
elif i == max(range(threads)):
command = "python LDproxy_sub.py " + "True " + snp + " " + snp_coord[
'chromosome'] + " " + str(
coord1 + (block * i) +
1) + " " + str(coord2) + " " + request + " " + str(i)
else:
command = "python LDproxy_sub.py " + "True " + snp + " " + snp_coord[
'chromosome'] + " " + str(
coord1 + (block * i) +
1) + " " + str(coord1 +
(block *
(i + 1))) + " " + request + " " + str(i)
commands.append(command)
processes = [
subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
for command in commands
]
# collect output in parallel
def get_output(process):
return process.communicate()[0].splitlines()
if not hasattr(threading.current_thread(), "_children"):
threading.current_thread()._children = weakref.WeakKeyDictionary()
pool = Pool(len(processes))
out_raw = pool.map(get_output, processes)
pool.close()
pool.join()
# Aggregate output
out_prox = []
for i in range(len(out_raw)):
for j in range(len(out_raw[i])):
col = out_raw[i][j].decode('utf-8').strip().split("\t")
col[6] = int(col[6])
col[7] = float(col[7])
col[8] = float(col[8])
col.append(abs(int(col[6])))
out_prox.append(col)
# Sort output
if r2_d not in ["r2", "d"]:
r2_d = "r2"
out_dist_sort = sorted(out_prox, key=operator.itemgetter(14))
if r2_d == "r2":
out_ld_sort = sorted(out_dist_sort,
key=operator.itemgetter(8),
reverse=True)
else:
out_ld_sort = sorted(out_dist_sort,
key=operator.itemgetter(7),
reverse=True)
# Organize scatter plot data
q_rs = []
q_allele = []
q_coord = []
q_maf = []
p_rs = []
p_allele = []
p_coord = []
p_maf = []
dist = []
d_prime = []
d_prime_round = []
r2 = []
r2_round = []
corr_alleles = []
regdb = []
funct = []
color = []
size = []
for i in range(len(out_ld_sort)):
q_rs_i, q_allele_i, q_coord_i, p_rs_i, p_allele_i, p_coord_i, dist_i, d_prime_i, r2_i, corr_alleles_i, regdb_i, q_maf_i, p_maf_i, funct_i, dist_abs = out_ld_sort[
i]
if float(r2_i) > 0.01:
q_rs.append(q_rs_i)
q_allele.append(q_allele_i)
q_coord.append(float(q_coord_i.split(":")[1]) / 1000000)
q_maf.append(str(round(float(q_maf_i), 4)))
if p_rs_i == ".":
p_rs_i = p_coord_i
p_rs.append(p_rs_i)
p_allele.append(p_allele_i)
p_coord.append(float(p_coord_i.split(":")[1]) / 1000000)
p_maf.append(str(round(float(p_maf_i), 4)))
dist.append(str(round(dist_i / 1000000.0, 4)))
d_prime.append(float(d_prime_i))
d_prime_round.append(str(round(float(d_prime_i), 4)))
r2.append(float(r2_i))
r2_round.append(str(round(float(r2_i), 4)))
corr_alleles.append(corr_alleles_i)
# Correct Missing Annotations
if regdb_i == ".":
regdb_i = ""
regdb.append(regdb_i)
if funct_i == ".":
funct_i = ""
if funct_i == "NA":
funct_i = "none"
funct.append(funct_i)
# Set Color
if i == 0:
color_i = "blue"
elif funct_i != "none" and funct_i != "":
color_i = "red"
else:
color_i = "orange"
color.append(color_i)
# Set Size
size_i = 9 + float(p_maf_i) * 14.0
size.append(size_i)
# Begin Bokeh Plotting
from collections import OrderedDict
from bokeh.embed import components, file_html
from bokeh.layouts import gridplot
from bokeh.models import HoverTool, LinearAxis, Range1d
from bokeh.plotting import ColumnDataSource, curdoc, figure, output_file, reset_output, save
from bokeh.resources import CDN
from bokeh.io import export_svgs
import svgutils.compose as sg
reset_output()
# Proxy Plot
x = p_coord
if r2_d == "r2":
y = r2
else:
y = d_prime
whitespace = 0.01
xr = Range1d(start=coord1 / 1000000.0 - whitespace,
end=coord2 / 1000000.0 + whitespace)
yr = Range1d(start=-0.03, end=1.03)
sup_2 = "\u00B2"
proxy_plot = figure(
title="Proxies for " + snp + " in " + pop,
min_border_top=2,
min_border_bottom=2,
min_border_left=60,
min_border_right=60,
h_symmetry=False,
v_symmetry=False,
plot_width=900,
plot_height=600,
x_range=xr,
y_range=yr,
tools="hover,tap,pan,box_zoom,box_select,undo,redo,reset,previewsave",
logo=None,
toolbar_location="above")
proxy_plot.title.align = "center"
# Get recomb from LDproxy.py tmp output files
filename = tmp_dir + "recomb_" + request + ".txt"
recomb_raw = open(filename).readlines()
recomb_x = []
recomb_y = []
for i in range(len(recomb_raw)):
chr, pos, rate = recomb_raw[i].strip().split()
recomb_x.append(int(pos) / 1000000.0)
recomb_y.append(float(rate) / 100.0)
data = {
'x': x,
'y': y,
'qrs': q_rs,
'q_alle': q_allele,
'q_maf': q_maf,
'prs': p_rs,
'p_alle': p_allele,
'p_maf': p_maf,
'dist': dist,
'r': r2_round,
'd': d_prime_round,
'alleles': corr_alleles,
'regdb': regdb,
'funct': funct,
'size': size,
'color': color
}
source = ColumnDataSource(data)
proxy_plot.line(recomb_x, recomb_y, line_width=1, color="black", alpha=0.5)
proxy_plot.circle(x='x',
y='y',
size='size',
color='color',
alpha=0.5,
source=source)
hover = proxy_plot.select(dict(type=HoverTool))
hover.tooltips = OrderedDict([
("Query Variant", "@qrs @q_alle"),
("Proxy Variant", "@prs @p_alle"),
("Distance (Mb)", "@dist"),
("MAF (Query,Proxy)", "@q_maf,@p_maf"),
("R" + sup_2, "@r"),
("D\'", "@d"),
("Correlated Alleles", "@alleles"),
("RegulomeDB", "@regdb"),
("Functional Class", "@funct"),
])
proxy_plot.text(x,
y,
text=regdb,
alpha=1,
text_font_size="7pt",
text_baseline="middle",
text_align="center",
angle=0)
if r2_d == "r2":
proxy_plot.yaxis.axis_label = "R" + sup_2
else:
proxy_plot.yaxis.axis_label = "D\'"
proxy_plot.extra_y_ranges = {"y2_axis": Range1d(start=-3, end=103)}
proxy_plot.add_layout(
LinearAxis(y_range_name="y2_axis",
axis_label="Combined Recombination Rate (cM/Mb)"), "right")
# Rug Plot
y2_ll = [-0.03] * len(x)
y2_ul = [1.03] * len(x)
yr_rug = Range1d(start=-0.03, end=1.03)
data_rug = {
'x': x,
'y': y,
'y2_ll': y2_ll,
'y2_ul': y2_ul,
'qrs': q_rs,
'q_alle': q_allele,
'q_maf': q_maf,
'prs': p_rs,
'p_alle': p_allele,
'p_maf': p_maf,
'dist': dist,
'r': r2_round,
'd': d_prime_round,
'alleles': corr_alleles,
'regdb': regdb,
'funct': funct,
'size': size,
'color': color
}
source_rug = ColumnDataSource(data_rug)
rug = figure(x_range=xr,
y_range=yr_rug,
border_fill_color='white',
y_axis_type=None,
title="",
min_border_top=2,
min_border_bottom=2,
min_border_left=60,
min_border_right=60,
h_symmetry=False,
v_symmetry=False,
plot_width=900,
plot_height=50,
tools="xpan,tap",
logo=None)
rug.segment(x0='x',
y0='y2_ll',
x1='x',
y1='y2_ul',
source=source_rug,
color='color',
alpha=0.5,
line_width=1)
rug.toolbar_location = None
# Gene Plot
# Get genes from LDproxy.py tmp output files
filename = tmp_dir + "genes_" + request + ".txt"
genes_raw = open(filename).readlines()
genes_plot_start = []
genes_plot_end = []
genes_plot_y = []
genes_plot_name = []
exons_plot_x = []
exons_plot_y = []
exons_plot_w = []
exons_plot_h = []
exons_plot_name = []
exons_plot_id = []
exons_plot_exon = []
lines = [0]
gap = 80000
tall = 0.75
if genes_raw != None:
for i in range(len(genes_raw)):
bin, name_id, chrom, strand, txStart, txEnd, cdsStart, cdsEnd, exonCount, exonStarts, exonEnds, score, name2, cdsStartStat, cdsEndStat, exonFrames = genes_raw[
i].strip().split()
name = name2
id = name_id
e_start = exonStarts.split(",")
e_end = exonEnds.split(",")
# Determine Y Coordinate
i = 0
y_coord = None
while y_coord == None:
if i > len(lines) - 1:
y_coord = i + 1
lines.append(int(txEnd))
elif int(txStart) > (gap + lines[i]):
y_coord = i + 1
lines[i] = int(txEnd)
else:
i += 1
genes_plot_start.append(int(txStart) / 1000000.0)
genes_plot_end.append(int(txEnd) / 1000000.0)
genes_plot_y.append(y_coord)
genes_plot_name.append(name + " ")
for i in range(len(e_start) - 1):
if strand == "+":
exon = i + 1
else:
exon = len(e_start) - 1 - i
width = (int(e_end[i]) - int(e_start[i])) / 1000000.0
x_coord = int(e_start[i]) / 1000000.0 + (width / 2)
exons_plot_x.append(x_coord)
exons_plot_y.append(y_coord)
exons_plot_w.append(width)
exons_plot_h.append(tall)
exons_plot_name.append(name)
exons_plot_id.append(id)
exons_plot_exon.append(exon)
n_rows = len(lines)
genes_plot_yn = [n_rows - x + 0.5 for x in genes_plot_y]
exons_plot_yn = [n_rows - x + 0.5 for x in exons_plot_y]
yr2 = Range1d(start=0, end=n_rows)
data_gene_plot = {
'exons_plot_x': exons_plot_x,
'exons_plot_yn': exons_plot_yn,
'exons_plot_w': exons_plot_w,
'exons_plot_h': exons_plot_h,
'exons_plot_name': exons_plot_name,
'exons_plot_id': exons_plot_id,
'exons_plot_exon': exons_plot_exon
}
source_gene_plot = ColumnDataSource(data_gene_plot)
if len(lines) < 3:
plot_h_pix = 150
else:
plot_h_pix = 150 + (len(lines) - 2) * 50
gene_plot = figure(
x_range=xr,
y_range=yr2,
border_fill_color='white',
title="",
min_border_top=2,
min_border_bottom=2,
min_border_left=60,
min_border_right=60,
h_symmetry=False,
v_symmetry=False,
plot_width=900,
plot_height=plot_h_pix,
tools="hover,tap,xpan,box_zoom,undo,redo,reset,previewsave",
logo=None)
gene_plot.segment(genes_plot_start,
genes_plot_yn,
genes_plot_end,
genes_plot_yn,
color="black",
alpha=1,
line_width=2)
gene_plot.rect(x='exons_plot_x',
y='exons_plot_yn',
width='exons_plot_w',
height='exons_plot_h',
source=source_gene_plot,
fill_color="grey",
line_color="grey")
gene_plot.xaxis.axis_label = "Chromosome " + \
snp_coord['chromosome'] + " Coordinate (Mb)(GRCh37)"
gene_plot.yaxis.axis_label = "Genes"
gene_plot.ygrid.grid_line_color = None
gene_plot.yaxis.axis_line_color = None
gene_plot.yaxis.minor_tick_line_color = None
gene_plot.yaxis.major_tick_line_color = None
gene_plot.yaxis.major_label_text_color = None
hover = gene_plot.select(dict(type=HoverTool))
hover.tooltips = OrderedDict([
("Gene", "@exons_plot_name"),
("ID", "@exons_plot_id"),
("Exon", "@exons_plot_exon"),
])
gene_plot.text(genes_plot_start,
genes_plot_yn,
text=genes_plot_name,
alpha=1,
text_font_size="7pt",
text_font_style="bold",
text_baseline="middle",
text_align="right",
angle=0)
gene_plot.toolbar_location = "below"
# Change output backend to SVG temporarily for headless export
# Will be changed back to canvas in LDlink.js
proxy_plot.output_backend = "svg"
rug.output_backend = "svg"
gene_plot.output_backend = "svg"
export_svgs(proxy_plot,
filename=tmp_dir + "proxy_plot_1_" + request + ".svg")
export_svgs(gene_plot,
filename=tmp_dir + "gene_plot_1_" + request + ".svg")
# 1 pixel = 0.0264583333 cm
svg_height = str(20.00 + (0.0264583333 * plot_h_pix)) + "cm"
svg_height_scaled = str(100.00 + (0.1322916665 * plot_h_pix)) + "cm"
# Concatenate svgs
sg.Figure("24.59cm", svg_height,
sg.SVG(tmp_dir + "proxy_plot_1_" + request + ".svg"),
sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").move(
0, 630)).save(tmp_dir + "proxy_plot_" + request + ".svg")
sg.Figure(
"122.95cm", svg_height_scaled,
sg.SVG(tmp_dir + "proxy_plot_1_" + request + ".svg").scale(5),
sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").scale(5).move(
0, 3150)).save(tmp_dir + "proxy_plot_scaled_" + request + ".svg")
# Export to PDF
subprocess.call("phantomjs ./rasterize.js " + tmp_dir + "proxy_plot_" +
request + ".svg " + tmp_dir + "proxy_plot_" + request +
".pdf",
shell=True)
# Export to PNG
subprocess.call("phantomjs ./rasterize.js " + tmp_dir +
"proxy_plot_scaled_" + request + ".svg " + tmp_dir +
"proxy_plot_" + request + ".png",
shell=True)
# Export to JPEG
subprocess.call("phantomjs ./rasterize.js " + tmp_dir +
"proxy_plot_scaled_" + request + ".svg " + tmp_dir +
"proxy_plot_" + request + ".jpeg",
shell=True)
# Remove individual SVG files after they are combined
subprocess.call("rm " + tmp_dir + "proxy_plot_1_" + request + ".svg",
shell=True)
subprocess.call("rm " + tmp_dir + "gene_plot_1_" + request + ".svg",
shell=True)
# Remove scaled SVG file after it is converted to png and jpeg
subprocess.call("rm " + tmp_dir + "proxy_plot_scaled_" + request + ".svg",
shell=True)
reset_output()
# Remove temporary files
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt", shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
subprocess.call("rm " + tmp_dir + "genes_" + request + ".txt", shell=True)
subprocess.call("rm " + tmp_dir + "recomb_" + request + ".txt", shell=True)
# Return plot output
return None
def station_bar_plot(summary_csv,
layer,
out_dir=None,
x_label=None,
y_label=None,
title=None,
subtitle=None,
year_subtitle=True):
"""
Produce an interactive bar chart comparing multiple climate stations to each
other for a particular variable, e.g. bias ratios or interpolated residuals.
Arguments:
summary_csv (str): path to summary CSV produced by either :func:`gridwxcomp.calc_bias_ratios`
or by :func:`gridwxcomp.interpolate`. Should contain ``layer``
data for plot.
layer (str): name of variable to plot.
Keyword Arguments:
out_dir (str or None): default None. Output directory path, default is
'station_bar_plots' in parent directory of ``summary_csv``.
x_label (str or None): default None. Label for x-axis.
y_label (str or None): default None. Label for y-axis, defaults to
``layer``.
title (str or None): default None. Title of plot.
subtitle (str, list, or None): default None. Additional subtitle(s)
for plot.
year_subtitle (bool): default True. If true print subtitle on plot with
the max year range used for station data, e.g. 'years: 1995-2005'
Example:
Let's say we want to compare the mean growing seasion bias ratios of
reference evapotranspiration (ETr) for the selection of stations we
used to calculate bias ratios. The summary CSV file containing the
ratios should be first created using :func:`gridwxcomp.calc_bias_ratios`.
>>> from gridwxcomp.plot import station_bar_plot
>>> # path to summary CSV with station data
>>> in_file = 'monthly_ratios/etr_mm_summary_all_yrs.csv'
>>> layer = 'growseason_mean'
>>> station_bar_plot(in_file, layer)
The resulting file will be saved using the layer name as a file name::
'monthly_ratios/station_bar_plots/growseason_mean.html'
The plot file will contain the mean growing season bias ratios
of ETr for each station, sorted from smallest to largest values.
This function may also be used for any numerical data in the summary CSV
files that are created by :func:`gridwxcomp.interpolate` in addition to
those created by :func:`gridwxcomp.calc_bias_ratios`. The main
requirement is that ``summary_csv`` must contain the column 'STATION_ID'
and the ``layer`` keyword argument.
Raises:
FileNotFoundError: if ``summary_csv`` is not found.
KeyError: if ``layer`` does not exist as a column name in ``summary_csv``.
"""
if not Path(summary_csv).is_file():
err_msg = '\n{} is not a valid path to a summary CSV file!'.\
format(summary_csv)
raise FileNotFoundError(err_msg)
df = pd.read_csv(summary_csv, na_values=[-999])
if not layer in df.columns:
err_msg = '\nColumn {} was not found in {}'.format(layer, summary_csv)
raise KeyError(err_msg)
df.sort_values(layer, inplace=True)
source = ColumnDataSource(df)
# hover tooltip with station and value
tooltips = [
("station", "@STATION_ID"),
("value", "@{}".format(layer)),
]
hover = models.HoverTool(tooltips=tooltips)
if not y_label:
y_label = layer
# save to working directory in 'station_bar_plots' if not specified
if not out_dir:
out_dir = Path(summary_csv).parent / 'station_bar_plots'
else:
out_dir = Path(out_dir)
if not out_dir.is_dir():
print('\n{}\nDoes not exist, making directory'.format(
out_dir.absolute()))
out_dir.mkdir(parents=True, exist_ok=True)
out_file = out_dir / '{}.html'.format(layer)
print('\nCreating station bar plot for variable: ', layer,
'\nUsing data from file: ',
Path(summary_csv).absolute())
output_file(out_file)
p = figure(x_range=df.STATION_ID, y_axis_label=y_label, title=title)
p.vbar(x='STATION_ID', top=layer, width=0.8, source=source)
p.xaxis.major_label_orientation = pi / 2
p.add_tools(hover, models.BoxSelectTool())
if year_subtitle:
# add data range (years start to end) as subtitle
min_yr = df.start_year.min().astype(int)
max_yr = df.end_year.max().astype(int)
if min_yr == max_yr:
year_str = 'year: {}'.format(min_yr)
else:
year_str = 'years: {}-{}'.format(min_yr, max_yr)
# caution note if not all stations use full year range
if not (df.end_year == max_yr).all() or not (df.start_year
== min_yr).all():
year_str = '{} (less years exist for some stations)'.\
format(year_str)
p.add_layout(models.Title(text=year_str, text_font_style="italic"),
'above')
# add arbitrary number of custom subtitles as lines above plot
if isinstance(subtitle, (list, tuple)):
for st in subtitle:
p.add_layout(models.Title(text=st, text_font_style="italic"),
'above')
elif subtitle:
p.add_layout(models.Title(text=subtitle, text_font_style="italic"),
'above')
save(p)
print('\nPlot saved to: ', out_file.absolute())
def main():
# file where .html should be saved
output_file('docs/amount_staff_scatter.html',
title='Scatterplot: Amount of Students per Staff Member')
# reads df from file
df = pd.read_csv('../../university_ranking.csv', index_col=0)
colormap = {2016: 'red', 2017: 'green', 2018: 'blue'}
years = [2016, 2017, 2018]
# gets top 800 for every year and puts into list
dfs = [df.loc[df['year'] == year].head(200) for year in years]
df = dfs[0].append(dfs[1].append(dfs[2]))
# creates all data we need
year_list = []
# creates list of years for every datapoint
for year in years:
for i in range(200):
year_list.append(year)
# creates list of colors for every datapoint
colors = [colormap[x] for x in df['year']]
# all data collected in a dictionary
data = {
'ranking': df['ranking'],
'no_student_p_staff': df['no_student_p_staff'],
'years': year_list,
'color': colors,
'university': df['university_name']
}
m, b = best_fit_line(df['ranking'], df['no_student_p_staff'])
m = round(m, 4)
b = round(b, 4)
x = [i for i in range(201)]
y = [m * x + b for x in range(len(x))]
source = ColumnDataSource(data)
hover = HoverTool(tooltips=[('Year', '@years'), ('Ranking', '@ranking'),
('Amount of Students per Staff',
'@no_student_p_staff'),
('University', '@university')],
names=['scatter'])
p = figure(tools=[hover],
title="Scatterplot: Amount of Students per Staff Member")
p.xaxis.axis_label = 'International Ranking'
p.yaxis.axis_label = 'Amount of Students per Staff Member'
p.scatter('ranking',
'no_student_p_staff',
source=source,
color='color',
name='scatter',
legend='years')
r1 = p.line(x, y, line_width=2, color='black')
legend = Legend(items=[("{0}x + {1}".format(m, b), [r1])])
p.add_layout(legend, 'above')
show(p)
# Define parameter vector to be included for the similrity calculation
# or we can use all cap parameters for this claculation (?)
ShortParList = [
'sac_UZTWM', 'sac_UZFWM', 'sac_LZTWM', 'sac_LZFSM', 'sac_LZFPM',
'sac_ZPERC', 'sac_UZK', 'sac_REXP', 'sac_PFREE', 'sac_LZSK', 'sac_LZPK'
]
# cell 13 #############################################################
# Read Calibrated Parameter Sets for the basins to be regionalized
results = {}
sources = {}
for Basin in Basins:
print Basin
results[Basin] = genResult(Basin, NASA_RUNS)
sources[Basin] = ColumnDataSource(results[Basin])
print results, sources
# cell 14 ###############################################################
"""
Que
- what is 'F2'? why set 20 as the value
ans: F2 penalty function. default 20
Note
- Normalize F1,F2,F3 result
- calculate the mean of the normalized F1-F3 result: results[Basin]['Obj']
- rank the mean of normalized F1-F3 result: results[Basin]['obj_rank']
"""
# Normalize objective functions, calculate rank of mean normalized objective functions
colors = ["green", "red", "cyan", "yellow", "blue"]
color_src = []
for i in range(len(x_corr)):
color_src.append(colors[labels[i]])
from bokeh.plotting import figure, output_file, show, ColumnDataSource
from bokeh.models import HoverTool #,BoxZoomTool, ResetTool,ResizeTool,WheelZoomTool
output_file("toolbar.html")
TOOLS = "resize,crosshair,pan,wheel_zoom,box_zoom,reset,tap,previewsave,box_select,poly_select,lasso_select"
source = ColumnDataSource(data=dict(
x=x_corr,
y=y_corr,
desc=label,
# colors=color_src,
))
hover = HoverTool(tooltips="""
<div>
<div>
<span style="font-size: 17px; font-weight: bold;">@desc</span>
<span style="font-size: 15px; color: #966;">[$index]</span>
</div>
<div>
<span style="font-size: 15px;">Location</span>
<span style="font-size: 10px; color: #696;">($x, $y)</span>
</div>
</div>
""")
from bokeh.plotting import figure, ColumnDataSource
def resample_wrapper():
numpoints = 1
source.data = resample(numpoints, covariance_slider.value)
source3.data = dict(xs=[n + 1 for n in range(numpoints + 1)],
ys=[source.data['x1'][0], source.data['x2'][0]])
def update_all():
source2.data = drawContour(covariance_slider.value)
resample_wrapper()
source = ColumnDataSource(data=dict(x1=[0], x2=[0]))
source2 = ColumnDataSource(data=dict(xs=[0], ys=[0]))
source3 = ColumnDataSource(data=dict(xs=[0], ys=[0]))
plot = figure(y_range=(-3, 3),
x_range=(-3, 3),
plot_width=400,
plot_height=400)
plot.scatter('x1', 'x2', source=source, line_width=3, line_alpha=0.6)
plot.multi_line(xs='xs', ys='ys', line_color='line_color', source=source2)
plot2 = figure(y_range=(-3, 3),
x_range=(0, 3),
plot_width=400,
plot_height=400,
x_axis_type=None,
from bokeh.models.widgets import RangeSlider
import pandas
from bokeh.models.tools import Action
# Spans (line-type annotations) have a single dimension (width or height) and extend to the edge of the plot area.
from bokeh.models import Span
import numpy as np
import copy
import csv
# from bokeh.io import show
# from bokeh.events import ButtonClick
# from bokeh.models.widgets import Button
#Read in CSV
df = pandas.read_csv("Data/Cs6s.csv")
#Create ColumnDataSource from Data Frame - ColumnDataSource gives more features like hover,tooltips
source = ColumnDataSource(df)
df2 = pandas.read_csv("Data/cs7p3m3.csv")
source2 = ColumnDataSource(df2)
# dividing data to get line graph from separate parts
dfdiff = pandas.read_csv('Data/cs7p3m1.csv').sort_values(by=['wavelength'],
ignore_index=True)
# sorted_diff = dfdiff.sort_values(by=['wavelength'])
print(dfdiff)
# there is dissociation where y==0
def data_function(input_data):
data = []
data2 = []
def figure_as_html(builds_data, nodes=None, title=None, with_labels=False):
''' For every build in builds_data draw rectangle with data:
:center_x: = left(=timestamp) + width/2
:center y: = builtOn
:width: = duration
:height: = 100
:color: = result_to_color(result)
:label: = fullDisplayName
'''
# Data
center_x = []
center_y = []
width = []
left = []
height = 0.95
label = []
color = []
max_time = datetime_to_timestamp(datetime.datetime.now())
for build_data in builds_data:
logger.debug("Processing build %s" % build_data)
build_left = build_data['timestamp']
if build_data['duration'] == 0 and not build_data['result']:
logger.debug("Build %s is still running" % build_data['fullDisplayName'])
duration = max_time - build_data['timestamp']
build_width = duration
else:
build_width = build_data['duration']
build_label = build_data['fullDisplayName']
build_color = get_build_color(build_data)
left.append(build_left)
width.append(build_width)
label.append(build_label)
center_x.append(build_left + build_width/2)
center_y.append(build_data['builtOn'])
color.append(build_color)
source = ColumnDataSource(
data=dict(
x=center_x,
y=center_y,
width=width,
left=left,
color=color,
label=label,
)
)
if not nodes:
nodes = sorted(list(set(center_y)))
# Plot properties
TOOLS="pan,xwheel_zoom,box_zoom,reset,previewsave,hover,resize"
p = figure(x_axis_type="datetime", y_range=nodes, width=900,
tools=TOOLS,
title=title,
toolbar_location="below",
height=max(min(len(nodes)* 40, 800),100),
)
hover = p.select(dict(type=HoverTool))
hover.tooltips = '<font color="@color">•</font> @label'
# Draw data
p.rect('x', 'y', 'width', height, color='color', source=source,
line_width=1,
line_color='white',
line_alpha=0.4,
fill_alpha=0.4,
)
if with_labels:
# Add labels layer
p.text('left', 'y', 'label',
source=source,
angle=0.3,
name="Labels",
text_font_size='6pt',
text_baseline='middle',
text_alpha=0.8,
x_offset=map(lambda x: x % 3 * 20, xrange(len(label))),
y_offset=map(lambda y: y % 2 * 10 -5 , xrange(len(label))),
)
# To HTML
script, div = components(p)
return script + div
#import numpy as np
import pandas as pd
#from bokeh.charts import Bar
#from bokeh.plotting import hbar
from bokeh.plotting import ColumnDataSource, figure
from bokeh.io import output_file, show
output_file('pop-life.html')
file = 'country-pops.csv'
countries = pd.read_csv(file, nrows=5)
countries_array = np.array(countries.head)
#print(countries_array)
#bar_chart = hbar(countries, 'Country_English', values='Population', title='Population', legend=False)
#show(bar_chart)
country_data = ColumnDataSource(countries)
plot = figure(x_axis_label='Population', y_axis_label='Life Expectancy')
plot.circle(x='Population', y='Life_expectancy', source=country_data, size=15)
show(plot)
def main():
shp_link = "data/LSOA_2011_London_gen_MHW.shp"
# Read in LSOA shapefile
try:
shp = gpd.read_file(shp_link, crs="+init=epsg:4326")
# Set CRS for later mapping
shp = shp.to_crs(epsg=3857)
except:
# Unzip file if not done already
zip = ZipFile('data.zip')
zip.extractall()
shp = gpd.read_file(shp_link, crs="+init=epsg:4326")
# Set CRS for later mapping
shp = shp.to_crs(epsg=3857)
# Read PTAL table
PTAL_AV = pd.read_csv("data/AVPTAL2015_LSOA2011.csv")
# Rename column for merge
PTAL_AV = PTAL_AV.rename(columns = {'LSOA2011':'LSOA11CD'})
PTAL = shp.merge(PTAL_AV, on='LSOA11CD')
# Rename average PTAL col to easier name
PTAL = PTAL.rename(columns = {'AvPTAI2015':'AV_PTAL'})
# Read in bokeh modules
from collections import OrderedDict
from bokeh.plotting import figure, show, output_notebook, ColumnDataSource
from bokeh.models import HoverTool
from bokeh.io import output_file, show #Scatter
# Convert geomertries from gpd to bokeh format
lons, lats = gpd_bokeh(PTAL)
# Create PTAL Rate quantile bins in pysal
bins_q5 = ps.Quantiles(PTAL.AV_PTAL, k=5)
# Creat colour classes from bins
bwr = plt.cm.get_cmap('Reds')
bwr(.76)
c5 = [bwr(c) for c in [0.2, 0.4, 0.6, 0.7, 1.0]]
classes = bins_q5.yb
colors = [c5[i] for i in classes]
colors5 = ["#F1EEF6", "#D4B9DA", "#C994C7", "#DF65B0", "#DD1C77"]
colors = [colors5[i] for i in classes]
p = figure(title="London PTAI 2015 Quintiles", toolbar_location='left',
plot_width=1100, plot_height=700)
p.patches(lons, lats, fill_alpha=0.7, fill_color=colors,
line_color="#884444", line_width=2, line_alpha=0.3)
# Now for interactive plot
from bokeh.models import HoverTool
from bokeh.plotting import figure, show, output_file, ColumnDataSource
from bokeh.tile_providers import STAMEN_TERRAIN, CARTODBPOSITRON_RETINA
source = ColumnDataSource(data=dict(
x=lons,
y=lats,
color=colors,
name=PTAL.LSOA11NM,
rate=PTAL.AV_PTAL
))
# Add bokeh tools for interactive mapping
TOOLS = "pan, wheel_zoom, box_zoom, reset, hover, save"
p = figure(title="London Average PTAL Index ", tools=TOOLS,
plot_width=900, plot_height=900)
p.patches('x', 'y', source=source,
fill_color='color', fill_alpha=0.7,
line_color='white', line_width=0.5)
# Define what info shows with mouse hover
hover = p.select_one(HoverTool)
hover.point_policy = 'follow_mouse'
hover.tooltips = [
("Name", "@name"),
("PTAL rank", "@rate"),
]
# Turn off axis
p.axis.visible = False
# Add basemap
p.add_tile(CARTODBPOSITRON_RETINA)
# Output file to html
output_file("London_PTAL2015.html", title="Average_PTAL_2015")
show(p)
def _set_logo_CDS(self):
data = dict(url=[self.url_descr_map['init']], x=[0])
source = ColumnDataSource(data)
return source
def calculate_assoc_svg(file, region, pop, request, genome_build, myargs,
myargsName, myargsOrigin):
# Set data directories using config.yml
with open('config.yml', 'r') as yml_file:
config = yaml.load(yml_file)
env = config['env']
api_mongo_addr = config['api']['api_mongo_addr']
data_dir = config['data']['data_dir']
tmp_dir = config['data']['tmp_dir']
genotypes_dir = config['data']['genotypes_dir']
aws_info = config['aws']
mongo_username = config['database']['mongo_user_readonly']
mongo_password = config['database']['mongo_password']
mongo_port = config['database']['mongo_port']
num_subprocesses = config['performance']['num_subprocesses']
export_s3_keys = retrieveAWSCredentials()
# Ensure tmp directory exists
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
chrs = [
"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13",
"14", "15", "16", "17", "18", "19", "20", "21", "22", "X", "Y"
]
# Define parameters for --variant option
if region == "variant":
if myargsOrigin == "None":
return None
if myargsOrigin != "None":
# Find coordinates (GRCh37/hg19) or (GRCh38/hg38) for SNP RS number
if myargsOrigin[0:2] == "rs":
snp = myargsOrigin
# Connect to Mongo snp database
if env == 'local':
mongo_host = api_mongo_addr
else:
mongo_host = 'localhost'
client = MongoClient(
'mongodb://' + mongo_username + ':' + mongo_password + '@' +
mongo_host + '/admin', mongo_port)
db = client["LDLink"]
def get_coords_var(db, rsid):
rsid = rsid.strip("rs")
query_results = db.dbsnp.find_one({"id": rsid})
query_results_sanitized = json.loads(
json_util.dumps(query_results))
return query_results_sanitized
# Find RS number in snp database
var_coord = get_coords_var(db, snp)
if var_coord == None:
return None
elif myargsOrigin.split(":")[0].strip("chr") in chrs and len(
myargsOrigin.split(":")) == 2:
snp = myargsOrigin
#var_coord=[None,myargsOrigin.split(":")[0].strip("chr"),myargsOrigin.split(":")[1]]
var_coord = {
'chromosome': myargsOrigin.split(":")[0].strip("chr"),
'position': myargsOrigin.split(":")[1]
}
else:
return None
chromosome = var_coord['chromosome']
org_coord = var_coord[genome_build_vars[genome_build]['position']]
# Open Association Data
header_list = []
header_list.append(myargs['chr'])
header_list.append(myargs['bp'])
header_list.append(myargs['pval'])
# Load input file
with open(file) as fp:
header = fp.readline().strip().split()
first = fp.readline().strip().split()
if len(header) != len(first):
return None
# Check header
for item in header_list:
if item not in header:
return None
len_head = len(header)
chr_index = header.index(myargs['chr'])
pos_index = header.index(myargs['bp'])
p_index = header.index(myargs['pval'])
# Define window of interest around query SNP
if myargs['window'] == None:
if region == "variant":
window = 500000
elif region == "gene":
window = 100000
else:
window = 0
else:
window = myargs['window']
if region == "variant":
coord1 = int(org_coord) - window
if coord1 < 0:
coord1 = 0
coord2 = int(org_coord) + window
elif region == "gene":
if myargsName == "None":
return None
def get_coords_gene(gene_raw, db):
gene = gene_raw.upper()
mongoResult = db.genes_name_coords.find_one({"name": gene})
#format mongo output
if mongoResult != None:
geneResult = [
mongoResult["name"],
mongoResult[genome_build_vars[genome_build]['chromosome']],
mongoResult[genome_build_vars[genome_build]['gene_begin']],
mongoResult[genome_build_vars[genome_build]['gene_end']]
]
return geneResult
else:
return None
# Find RS number in snp database
gene_coord = get_coords_gene(myargsName, db)
if gene_coord == None or gene_coord[2] == 'NA' or gene_coord == 'NA':
return None
# Define search coordinates
coord1 = int(gene_coord[2]) - window
if coord1 < 0:
coord1 = 0
coord2 = int(gene_coord[3]) + window
# Run with --origin option
if myargsOrigin != "None":
if gene_coord[1] != chromosome:
return None
if coord1 > int(org_coord) or int(org_coord) > coord2:
return None
else:
chromosome = gene_coord[1]
elif region == "region":
if myargs['start'] == None:
return None
if myargs['end'] == None:
return None
# Parse out chr and positions for --region option
if len(myargs['start'].split(":")) != 2:
return None
if len(myargs['end'].split(":")) != 2:
return None
chr_s = myargs['start'].strip("chr").split(":")[0]
coord_s = myargs['start'].split(":")[1]
chr_e = myargs['end'].strip("chr").split(":")[0]
coord_e = myargs['end'].split(":")[1]
if chr_s not in chrs:
return None
if chr_e not in chrs:
return None
if chr_s != chr_e:
return None
if coord_s >= coord_e:
return None
coord1 = int(coord_s) - window
if coord1 < 0:
coord1 = 0
coord2 = int(coord_e) + window
# Run with --origin option
if myargsOrigin != "None":
if chr_s != chromosome:
return None
if coord1 > int(org_coord) or int(org_coord) > coord2:
return None
else:
chromosome = chr_s
# Generate coordinate list and P-value dictionary
max_window = 3000000
if coord2 - coord1 > max_window:
return None
assoc_coords = []
a_pos = []
assoc_dict = {}
assoc_list = []
with open(file) as fp:
for line in fp:
col = line.strip().split()
if len(col) == len_head:
if col[chr_index].strip("chr") == chromosome:
try:
int(col[pos_index])
except ValueError:
continue
else:
if coord1 <= int(col[pos_index]) <= coord2:
try:
float(col[p_index])
except ValueError:
continue
else:
coord_i = genome_build_vars[genome_build][
'1000G_chr_prefix'] + col[chr_index].strip(
"chr") + ":" + col[
pos_index] + "-" + col[pos_index]
assoc_coords.append(coord_i)
a_pos.append(col[pos_index])
assoc_dict[coord_i] = [col[p_index]]
assoc_list.append(
[coord_i, float(col[p_index])])
# Coordinate list checks
if len(assoc_coords) == 0:
return None
# Get population ids from population output file from LDassoc.py
pop_list = open(tmp_dir + "pops_" + request + ".txt").readlines()
ids = []
for i in range(len(pop_list)):
ids.append(pop_list[i].strip())
pop_ids = list(set(ids))
# Define LD origin coordinate
try:
org_coord
except NameError:
for var_p in sorted(assoc_list, key=operator.itemgetter(1)):
snp = "chr" + var_p[0].split("-")[0]
# Extract lowest P SNP phased genotypes
vcf_filePath = "%s/%s%s/%s" % (
config['aws']['data_subfolder'], genotypes_dir,
genome_build_vars[genome_build]["1000G_dir"],
genome_build_vars[genome_build]["1000G_file"] % (chromosome))
vcf_file = "s3://%s/%s" % (config['aws']['bucket'], vcf_filePath)
checkS3File(aws_info, config['aws']['bucket'], vcf_filePath)
tabix_snp_h = export_s3_keys + " cd {1}; tabix -HD {0} | grep CHROM".format(
vcf_file, data_dir + genotypes_dir +
genome_build_vars[genome_build]['1000G_dir'])
head = [
x.decode('utf-8') for x in subprocess.Popen(
tabix_snp_h, shell=True,
stdout=subprocess.PIPE).stdout.readlines()
][0].strip().split()
# Check lowest P SNP is in the 1000G population and not monoallelic from LDassoc.py output file
vcf = open(tmp_dir + "snp_no_dups_" + request + ".vcf").readlines()
if len(vcf) == 0:
continue
elif len(vcf) > 1:
geno = vcf[0].strip().split()
geno[0] = geno[0].lstrip('chr')
else:
geno = vcf[0].strip().split()
geno[0] = geno[0].lstrip('chr')
if "," in geno[3] or "," in geno[4]:
continue
index = []
for i in range(9, len(head)):
if head[i] in pop_ids:
index.append(i)
genotypes = {"0": 0, "1": 0}
for i in index:
sub_geno = geno[i].split("|")
for j in sub_geno:
if j in genotypes:
genotypes[j] += 1
else:
genotypes[j] = 1
if genotypes["0"] == 0 or genotypes["1"] == 0:
continue
org_coord = var_p[0].split("-")[1]
break
else:
if genome_build_vars[genome_build][
'1000G_chr_prefix'] + chromosome + ":" + org_coord + "-" + org_coord not in assoc_coords:
return None
# Extract query SNP phased genotypes
vcf_filePath = "%s/%s%s/%s" % (
config['aws']['data_subfolder'], genotypes_dir,
genome_build_vars[genome_build]["1000G_dir"],
genome_build_vars[genome_build]["1000G_file"] % (chromosome))
vcf_file = "s3://%s/%s" % (config['aws']['bucket'], vcf_filePath)
checkS3File(aws_info, config['aws']['bucket'], vcf_filePath)
tabix_snp_h = export_s3_keys + " cd {1}; tabix -HD {0} | grep CHROM".format(
vcf_file, data_dir + genotypes_dir +
genome_build_vars[genome_build]['1000G_dir'])
head = [
x.decode('utf-8') for x in
subprocess.Popen(tabix_snp_h, shell=True,
stdout=subprocess.PIPE).stdout.readlines()
][0].strip().split()
# Check query SNP is in the 1000G population, has the correct RS number, and not monoallelic
vcf = open(tmp_dir + "snp_no_dups_" + request + ".vcf").readlines()
if len(vcf) == 0:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf",
shell=True)
return None
elif len(vcf) > 1:
geno = []
for i in range(len(vcf)):
if vcf[i].strip().split()[2] == snp:
geno = vcf[i].strip().split()
geno[0] = geno[0].lstrip('chr')
if geno == []:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf",
shell=True)
return None
else:
geno = vcf[0].strip().split()
geno[0] = geno[0].lstrip('chr')
if geno[2] != snp and snp[0:2] == "rs" and "rs" in geno[2]:
snp = geno[2]
if "," in geno[3] or "," in geno[4]:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf",
shell=True)
return None
index = []
for i in range(9, len(head)):
if head[i] in pop_ids:
index.append(i)
genotypes = {"0": 0, "1": 0}
for i in index:
sub_geno = geno[i].split("|")
for j in sub_geno:
if j in genotypes:
genotypes[j] += 1
else:
genotypes[j] = 1
if genotypes["0"] == 0 or genotypes["1"] == 0:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf",
shell=True)
return None
# Calculate proxy LD statistics in parallel
if len(assoc_coords) < 60:
num_subprocesses = 1
# else:
# threads=4
assoc_coords_subset_chunks = np.array_split(assoc_coords, num_subprocesses)
# block=len(assoc_coords) // num_subprocesses
commands = []
# for i in range(num_subprocesses):
# if i==min(range(num_subprocesses)) and i==max(range(num_subprocesses)):
# command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords)+" "+request+" "+str(i)
# elif i==min(range(num_subprocesses)):
# command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[:block])+" "+request+" "+str(i)
# elif i==max(range(num_subprocesses)):
# command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:])+" "+request+" "+str(i)
# else:
# command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:block*(i+1)])+" "+request+" "+str(i)
# commands.append(command)
for subprocess_id in range(num_subprocesses):
subprocessArgs = " ".join([
str(snp),
str(chromosome),
str("_".join(assoc_coords_subset_chunks[subprocess_id])),
str(request),
str(genome_build),
str(subprocess_id)
])
commands.append("python3 LDassoc_sub.py " + subprocessArgs)
processes = [
subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
for command in commands
]
# collect output in parallel
def get_output(process):
return process.communicate()[0].splitlines()
pool = Pool(len(processes))
out_raw = pool.map(get_output, processes)
pool.close()
pool.join()
# Aggregate output
out_prox = []
for i in range(len(out_raw)):
for j in range(len(out_raw[i])):
col = out_raw[i][j].decode('utf-8').strip().split("\t")
col[6] = int(col[6])
col[7] = float(col[7])
col[8] = float(col[8])
col.append(abs(int(col[6])))
pos_i_j = col[5].split(":")[1]
coord_i_j = genome_build_vars[genome_build][
'1000G_chr_prefix'] + chromosome + ":" + pos_i_j + "-" + pos_i_j
if coord_i_j in assoc_dict:
col.append(float(assoc_dict[coord_i_j][0]))
out_prox.append(col)
out_dist_sort = sorted(out_prox, key=operator.itemgetter(14))
out_p_sort = sorted(out_dist_sort,
key=operator.itemgetter(15),
reverse=False)
# Organize scatter plot data
q_rs = []
q_allele = []
q_coord = []
q_maf = []
p_rs = []
p_allele = []
p_coord = []
p_pos = []
p_maf = []
dist = []
d_prime = []
d_prime_round = []
r2 = []
r2_round = []
corr_alleles = []
regdb = []
funct = []
color = []
alpha = []
size = []
p_val = []
neg_log_p = []
for i in range(len(out_p_sort)):
q_rs_i, q_allele_i, q_coord_i, p_rs_i, p_allele_i, p_coord_i, dist_i, d_prime_i, r2_i, corr_alleles_i, regdb_i, q_maf_i, p_maf_i, funct_i, dist_abs, p_val_i = out_p_sort[
i]
q_rs.append(q_rs_i)
q_allele.append(q_allele_i)
q_coord.append(float(q_coord_i.split(":")[1]) / 1000000)
q_maf.append(str(round(float(q_maf_i), 4)))
if p_rs_i == ".":
p_rs_i = p_coord_i
p_rs.append(p_rs_i)
p_allele.append(p_allele_i)
p_coord.append(float(p_coord_i.split(":")[1]) / 1000000)
p_pos.append(p_coord_i.split(":")[1])
p_maf.append(str(round(float(p_maf_i), 4)))
dist.append(str(round(dist_i / 1000000.0, 4)))
d_prime.append(float(d_prime_i))
d_prime_round.append(str(round(float(d_prime_i), 4)))
r2.append(float(r2_i))
r2_round.append(str(round(float(r2_i), 4)))
corr_alleles.append(corr_alleles_i)
# P-value
p_val.append(p_val_i)
neg_log_p.append(-log10(p_val_i))
# Correct Missing Annotations
if regdb_i == ".":
regdb_i = ""
regdb.append(regdb_i)
if funct_i == ".":
funct_i = ""
if funct_i == "NA":
funct_i = "none"
funct.append(funct_i)
# Set Color
reds = [
"#FFCCCC", "#FFCACA", "#FFC8C8", "#FFC6C6", "#FFC4C4", "#FFC2C2",
"#FFC0C0", "#FFBEBE", "#FFBCBC", "#FFBABA", "#FFB8B8", "#FFB6B6",
"#FFB4B4", "#FFB1B1", "#FFAFAF", "#FFADAD", "#FFABAB", "#FFA9A9",
"#FFA7A7", "#FFA5A5", "#FFA3A3", "#FFA1A1", "#FF9F9F", "#FF9D9D",
"#FF9B9B", "#FF9999", "#FF9797", "#FF9595", "#FF9393", "#FF9191",
"#FF8F8F", "#FF8D8D", "#FF8B8B", "#FF8989", "#FF8787", "#FF8585",
"#FF8383", "#FF8181", "#FF7E7E", "#FF7C7C", "#FF7A7A", "#FF7878",
"#FF7676", "#FF7474", "#FF7272", "#FF7070", "#FF6E6E", "#FF6C6C",
"#FF6A6A", "#FF6868", "#FF6666", "#FF6464", "#FF6262", "#FF6060",
"#FF5E5E", "#FF5C5C", "#FF5A5A", "#FF5858", "#FF5656", "#FF5454",
"#FF5252", "#FF5050", "#FF4E4E", "#FF4B4B", "#FF4949", "#FF4747",
"#FF4545", "#FF4343", "#FF4141", "#FF3F3F", "#FF3D3D", "#FF3B3B",
"#FF3939", "#FF3737", "#FF3535", "#FF3333", "#FF3131", "#FF2F2F",
"#FF2D2D", "#FF2B2B", "#FF2929", "#FF2727", "#FF2525", "#FF2323",
"#FF2121", "#FF1F1F", "#FF1D1D", "#FF1B1B", "#FF1818", "#FF1616",
"#FF1414", "#FF1212", "#FF1010", "#FF0E0E", "#FF0C0C", "#FF0A0A",
"#FF0808", "#FF0606", "#FF0404", "#FF0202", "#FF0000"
]
if q_coord_i == p_coord_i:
color_i = "#0000FF"
alpha_i = 0.7
else:
if myargs['dprime'] == True:
color_i = reds[int(d_prime_i * 100.0)]
alpha_i = 0.7
elif myargs['dprime'] == False:
color_i = reds[int(r2_i * 100.0)]
alpha_i = 0.7
color.append(color_i)
alpha.append(alpha_i)
# Set Size
size_i = 9 + float(p_maf_i) * 14.0
size.append(size_i)
# Pull out SNPs from association file not found in 1000G
p_plot_pos = []
p_plot_pval = []
p_plot_pos2 = []
p_plot_pval2 = []
p_plot_dist = []
index_var_pos = float(q_coord_i.split(":")[1]) / 1000000
for input_pos in a_pos:
if input_pos not in p_pos:
p_plot_pos.append(float(input_pos) / 1000000)
p_plot_pval.append(-log10(
float(assoc_dict[chromosome + ":" + input_pos + "-" +
input_pos][0])))
p_plot_pos2.append("chr" + chromosome + ":" + input_pos)
p_plot_pval2.append(
float(assoc_dict[chromosome + ":" + input_pos + "-" +
input_pos][0]))
p_plot_dist.append(
str(round(float(input_pos) / 1000000 - index_var_pos, 4)))
# Begin Bokeh Plotting
from collections import OrderedDict
from bokeh.embed import components, file_html
from bokeh.layouts import gridplot
from bokeh.models import HoverTool, LinearAxis, Range1d
from bokeh.plotting import ColumnDataSource, curdoc, figure, output_file, reset_output, save
from bokeh.resources import CDN
from bokeh.io import export_svgs
import svgutils.compose as sg
reset_output()
data_p = {
'p_plot_posX': p_plot_pos,
'p_plot_pvalY': p_plot_pval,
'p_plot_pos2': p_plot_pos2,
'p_plot_pval2': p_plot_pval2,
'p_plot_dist': p_plot_dist
}
source_p = ColumnDataSource(data_p)
# Assoc Plot
x = p_coord
y = neg_log_p
data = {
'x': x,
'y': y,
'qrs': q_rs,
'q_alle': q_allele,
'q_maf': q_maf,
'prs': p_rs,
'p_alle': p_allele,
'p_maf': p_maf,
'dist': dist,
'r': r2_round,
'd': d_prime_round,
'alleles': corr_alleles,
'regdb': regdb,
'funct': funct,
'p_val': p_val,
'size': size,
'color': color,
'alpha': alpha
}
source = ColumnDataSource(data)
whitespace = 0.01
xr = Range1d(start=coord1 / 1000000.0 - whitespace,
end=coord2 / 1000000.0 + whitespace)
yr = Range1d(start=-0.03, end=max(y) * 1.03)
sup_2 = "\u00B2"
assoc_plot = figure(
title="P-values and Regional LD for " + snp + " in " + pop,
min_border_top=2,
min_border_bottom=2,
min_border_left=60,
min_border_right=60,
h_symmetry=False,
v_symmetry=False,
plot_width=900,
plot_height=600,
x_range=xr,
y_range=yr,
tools=
"tap,pan,box_zoom,wheel_zoom,box_select,undo,redo,reset,previewsave",
logo=None,
toolbar_location="above")
assoc_plot.title.align = "center"
# Add recombination rate from LDassoc.py output file
recomb_file = tmp_dir + "recomb_" + request + ".json"
recomb_raw = open(recomb_file).readlines()
recomb_x = []
recomb_y = []
for recomb_raw_obj in recomb_raw:
recomb_obj = json.loads(recomb_raw_obj)
recomb_x.append(
int(recomb_obj[genome_build_vars[genome_build]['position']]) /
1000000.0)
recomb_y.append(float(recomb_obj['rate']) / 100 * max(y))
assoc_plot.line(recomb_x, recomb_y, line_width=1, color="black", alpha=0.5)
# Add genome-wide significance
a = [coord1 / 1000000.0 - whitespace, coord2 / 1000000.0 + whitespace]
b = [-log10(0.00000005), -log10(0.00000005)]
assoc_plot.line(a, b, color="blue", alpha=0.5)
assoc_points_not1000G = assoc_plot.circle(x='p_plot_posX',
y='p_plot_pvalY',
size=9 + float("0.25") * 14.0,
source=source_p,
line_color="gray",
fill_color="white")
assoc_points = assoc_plot.circle(x='x',
y='y',
size='size',
color='color',
alpha='alpha',
source=source)
assoc_plot.add_tools(
HoverTool(renderers=[assoc_points_not1000G],
tooltips=OrderedDict([("Variant", "@p_plot_pos2"),
("P-value", "@p_plot_pval2"),
("Distance (Mb)", "@p_plot_dist")])))
hover = HoverTool(renderers=[assoc_points])
hover.tooltips = OrderedDict([
("Variant", "@prs @p_alle"),
("P-value", "@p_val"),
("Distance (Mb)", "@dist"),
("MAF", "@p_maf"),
("R" + sup_2 + " (" + q_rs[0] + ")", "@r"),
("D\' (" + q_rs[0] + ")", "@d"),
("Correlated Alleles", "@alleles"),
("RegulomeDB", "@regdb"),
("Functional Class", "@funct"),
])
assoc_plot.add_tools(hover)
# Annotate RebulomeDB scores
if myargs['annotate'] == True:
assoc_plot.text(x,
y,
text=regdb,
alpha=1,
text_font_size="7pt",
text_baseline="middle",
text_align="center",
angle=0)
assoc_plot.yaxis.axis_label = "-log10 P-value"
assoc_plot.extra_y_ranges = {"y2_axis": Range1d(start=-3, end=103)}
assoc_plot.add_layout(
LinearAxis(y_range_name="y2_axis",
axis_label="Combined Recombination Rate (cM/Mb)"),
"right") ## Need to confirm units
# Rug Plot
y2_ll = [-0.03] * len(x)
y2_ul = [1.03] * len(x)
yr_rug = Range1d(start=-0.03, end=1.03)
data_rug = {
'x': x,
'y': y,
'y2_ll': y2_ll,
'y2_ul': y2_ul,
'qrs': q_rs,
'q_alle': q_allele,
'q_maf': q_maf,
'prs': p_rs,
'p_alle': p_allele,
'p_maf': p_maf,
'dist': dist,
'r': r2_round,
'd': d_prime_round,
'alleles': corr_alleles,
'regdb': regdb,
'funct': funct,
'p_val': p_val,
'size': size,
'color': color,
'alpha': alpha
}
source_rug = ColumnDataSource(data_rug)
rug = figure(x_range=xr,
y_range=yr_rug,
border_fill_color='white',
y_axis_type=None,
title="",
min_border_top=2,
min_border_bottom=2,
min_border_left=60,
min_border_right=60,
h_symmetry=False,
v_symmetry=False,
plot_width=900,
plot_height=50,
tools="xpan,tap,wheel_zoom",
logo=None)
rug.segment(x0='x',
y0='y2_ll',
x1='x',
y1='y2_ul',
source=source_rug,
color='color',
alpha='alpha',
line_width=1)
rug.toolbar_location = None
# Gene Plot (All Transcripts)
if myargs['transcript'] == True:
# Get genes from LDassoc.py output file
genes_file = tmp_dir + "genes_" + request + ".json"
genes_raw = open(genes_file).readlines()
genes_plot_start = []
genes_plot_end = []
genes_plot_y = []
genes_plot_name = []
exons_plot_x = []
exons_plot_y = []
exons_plot_w = []
exons_plot_h = []
exons_plot_name = []
exons_plot_id = []
exons_plot_exon = []
message = ["Too many genes to plot."]
lines = [0]
gap = 80000
tall = 0.75
if genes_raw != None and len(genes_raw) > 0:
for gene_raw_obj in genes_raw:
gene_obj = json.loads(gene_raw_obj)
bin = gene_obj["bin"]
name_id = gene_obj["name"]
chrom = gene_obj["chrom"]
strand = gene_obj["strand"]
txStart = gene_obj["txStart"]
txEnd = gene_obj["txEnd"]
cdsStart = gene_obj["cdsStart"]
cdsEnd = gene_obj["cdsEnd"]
exonCount = gene_obj["exonCount"]
exonStarts = gene_obj["exonStarts"]
exonEnds = gene_obj["exonEnds"]
score = gene_obj["score"]
name2 = gene_obj["name2"]
cdsStartStat = gene_obj["cdsStartStat"]
cdsEndStat = gene_obj["cdsEndStat"]
exonFrames = gene_obj["exonFrames"]
name = name2
id = name_id
e_start = exonStarts.split(",")
e_end = exonEnds.split(",")
# Determine Y Coordinate
i = 0
y_coord = None
while y_coord == None:
if i > len(lines) - 1:
y_coord = i + 1
lines.append(int(txEnd))
elif int(txStart) > (gap + lines[i]):
y_coord = i + 1
lines[i] = int(txEnd)
else:
i += 1
genes_plot_start.append(int(txStart) / 1000000.0)
genes_plot_end.append(int(txEnd) / 1000000.0)
genes_plot_y.append(y_coord)
genes_plot_name.append(name + " ")
for i in range(len(e_start) - 1):
if strand == "+":
exon = i + 1
else:
exon = len(e_start) - 1 - i
width = (int(e_end[i]) - int(e_start[i])) / 1000000.0
x_coord = int(e_start[i]) / 1000000.0 + (width / 2)
exons_plot_x.append(x_coord)
exons_plot_y.append(y_coord)
exons_plot_w.append(width)
exons_plot_h.append(tall)
exons_plot_name.append(name)
exons_plot_id.append(id)
exons_plot_exon.append(exon)
n_rows = len(lines)
genes_plot_yn = [n_rows - x + 0.5 for x in genes_plot_y]
exons_plot_yn = [n_rows - x + 0.5 for x in exons_plot_y]
yr2 = Range1d(start=0, end=n_rows)
data_gene_plot = {
'exons_plot_x': exons_plot_x,
'exons_plot_yn': exons_plot_yn,
'exons_plot_w': exons_plot_w,
'exons_plot_h': exons_plot_h,
'exons_plot_name': exons_plot_name,
'exons_plot_id': exons_plot_id,
'exons_plot_exon': exons_plot_exon
}
source_gene_plot = ColumnDataSource(data_gene_plot)
max_genes = 40
# if len(lines) < 3 or len(genes_raw) > max_genes:
if len(lines) < 3:
plot_h_pix = 250
else:
plot_h_pix = 250 + (len(lines) - 2) * 50
gene_plot = figure(
min_border_top=2,
min_border_bottom=0,
min_border_left=100,
min_border_right=5,
x_range=xr,
y_range=yr2,
border_fill_color='white',
title="",
h_symmetry=False,
v_symmetry=False,
logo=None,
plot_width=900,
plot_height=plot_h_pix,
tools=
"hover,xpan,box_zoom,wheel_zoom,tap,undo,redo,reset,previewsave")
# if len(genes_raw) <= max_genes:
gene_plot.segment(genes_plot_start,
genes_plot_yn,
genes_plot_end,
genes_plot_yn,
color="black",
alpha=1,
line_width=2)
gene_plot.rect(x='exons_plot_x',
y='exons_plot_yn',
width='exons_plot_w',
height='exons_plot_h',
source=source_gene_plot,
fill_color="grey",
line_color="grey")
gene_plot.text(genes_plot_start,
genes_plot_yn,
text=genes_plot_name,
alpha=1,
text_font_size="7pt",
text_font_style="bold",
text_baseline="middle",
text_align="right",
angle=0)
hover = gene_plot.select(dict(type=HoverTool))
hover.tooltips = OrderedDict([
("Gene", "@exons_plot_name"),
("Transcript ID", "@exons_plot_id"),
("Exon", "@exons_plot_exon"),
])
# else:
# x_coord_text = coord1/1000000.0 + (coord2/1000000.0 - coord1/1000000.0) / 2.0
# gene_plot.text(x_coord_text, n_rows / 2.0, text=message, alpha=1,
# text_font_size="12pt", text_font_style="bold", text_baseline="middle", text_align="center", angle=0)
gene_plot.xaxis.axis_label = "Chromosome " + chromosome + " Coordinate (Mb)(" + genome_build_vars[
genome_build]['title'] + ")"
gene_plot.yaxis.axis_label = "Genes (All Transcripts)"
gene_plot.ygrid.grid_line_color = None
gene_plot.yaxis.axis_line_color = None
gene_plot.yaxis.minor_tick_line_color = None
gene_plot.yaxis.major_tick_line_color = None
gene_plot.yaxis.major_label_text_color = None
gene_plot.toolbar_location = "below"
# Change output backend to SVG temporarily for headless export
assoc_plot.output_backend = "svg"
rug.output_backend = "svg"
gene_plot.output_backend = "svg"
export_svgs(assoc_plot,
filename=tmp_dir + "assoc_plot_1_" + request + ".svg")
export_svgs(gene_plot,
filename=tmp_dir + "gene_plot_1_" + request + ".svg")
# 1 pixel = 0.0264583333 cm
svg_height = str(20.00 + (0.0264583333 * plot_h_pix)) + "cm"
svg_height_scaled = str(100.00 + (0.1322916665 * plot_h_pix)) + "cm"
# Concatenate svgs
sg.Figure(
"24.59cm", svg_height,
sg.SVG(tmp_dir + "assoc_plot_1_" + request + ".svg"),
sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").move(
-40, 630)).save(tmp_dir + "assoc_plot_" + request + ".svg")
sg.Figure(
"122.95cm", svg_height_scaled,
sg.SVG(tmp_dir + "assoc_plot_1_" + request + ".svg").scale(5),
sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").scale(5).move(
-200,
3150)).save(tmp_dir + "assoc_plot_scaled_" + request + ".svg")
# Export to PDF
subprocess.call("phantomjs ./rasterize.js " + tmp_dir + "assoc_plot_" +
request + ".svg " + tmp_dir + "assoc_plot_" + request +
".pdf",
shell=True)
# Export to PNG
subprocess.call("phantomjs ./rasterize.js " + tmp_dir +
"assoc_plot_scaled_" + request + ".svg " + tmp_dir +
"assoc_plot_" + request + ".png",
shell=True)
# Export to JPEG
subprocess.call("phantomjs ./rasterize.js " + tmp_dir +
"assoc_plot_scaled_" + request + ".svg " + tmp_dir +
"assoc_plot_" + request + ".jpeg",
shell=True)
# Remove individual SVG files after they are combined
subprocess.call("rm " + tmp_dir + "assoc_plot_1_" + request + ".svg",
shell=True)
subprocess.call("rm " + tmp_dir + "gene_plot_1_" + request + ".svg",
shell=True)
# Remove scaled SVG file after it is converted to png and jpeg
subprocess.call("rm " + tmp_dir + "assoc_plot_scaled_" + request +
".svg",
shell=True)
# Gene Plot (Collapsed)
else:
# Get genes from LDassoc.py output file
genes_c_file = tmp_dir + "genes_c_" + request + ".json"
genes_c_raw = open(genes_c_file).readlines()
genes_c_plot_start = []
genes_c_plot_end = []
genes_c_plot_y = []
genes_c_plot_name = []
exons_c_plot_x = []
exons_c_plot_y = []
exons_c_plot_w = []
exons_c_plot_h = []
exons_c_plot_name = []
exons_c_plot_id = []
message_c = ["Too many genes to plot."]
lines_c = [0]
gap = 80000
tall = 0.75
if genes_c_raw != None and len(genes_c_raw) > 0:
for gene_raw_obj in genes_c_raw:
gene_c_obj = json.loads(gene_raw_obj)
chrom = gene_c_obj["chrom"]
txStart = gene_c_obj["txStart"]
txEnd = gene_c_obj["txEnd"]
exonStarts = gene_c_obj["exonStarts"]
exonEnds = gene_c_obj["exonEnds"]
name2 = gene_c_obj["name2"]
transcripts = gene_c_obj["transcripts"]
name = name2
e_start = exonStarts.split(",")
e_end = exonEnds.split(",")
e_transcripts = transcripts.split(",")
# Determine Y Coordinate
i = 0
y_coord = None
while y_coord == None:
if i > len(lines_c) - 1:
y_coord = i + 1
lines_c.append(int(txEnd))
elif int(txStart) > (gap + lines_c[i]):
y_coord = i + 1
lines_c[i] = int(txEnd)
else:
i += 1
genes_c_plot_start.append(int(txStart) / 1000000.0)
genes_c_plot_end.append(int(txEnd) / 1000000.0)
genes_c_plot_y.append(y_coord)
genes_c_plot_name.append(name + " ")
# for i in range(len(e_start)):
for i in range(len(e_start) - 1):
width = (int(e_end[i]) - int(e_start[i])) / 1000000.0
x_coord = int(e_start[i]) / 1000000.0 + (width / 2)
exons_c_plot_x.append(x_coord)
exons_c_plot_y.append(y_coord)
exons_c_plot_w.append(width)
exons_c_plot_h.append(tall)
exons_c_plot_name.append(name)
exons_c_plot_id.append(e_transcripts[i].replace("-", ","))
n_rows_c = len(lines_c)
genes_c_plot_yn = [n_rows_c - x + 0.5 for x in genes_c_plot_y]
exons_c_plot_yn = [n_rows_c - x + 0.5 for x in exons_c_plot_y]
yr2_c = Range1d(start=0, end=n_rows_c)
data_gene_c_plot = {
'exons_c_plot_x': exons_c_plot_x,
'exons_c_plot_yn': exons_c_plot_yn,
'exons_c_plot_w': exons_c_plot_w,
'exons_c_plot_h': exons_c_plot_h,
'exons_c_plot_name': exons_c_plot_name,
'exons_c_plot_id': exons_c_plot_id
}
source_gene_c_plot = ColumnDataSource(data_gene_c_plot)
max_genes_c = 40
# if len(lines_c) < 3 or len(genes_c_raw) > max_genes_c:
if len(lines_c) < 3:
plot_c_h_pix = 250
else:
plot_c_h_pix = 250 + (len(lines_c) - 2) * 50
gene_c_plot = figure(
min_border_top=2,
min_border_bottom=0,
min_border_left=100,
min_border_right=5,
x_range=xr,
y_range=yr2_c,
border_fill_color='white',
title="",
h_symmetry=False,
v_symmetry=False,
logo=None,
plot_width=900,
plot_height=plot_c_h_pix,
tools=
"hover,xpan,box_zoom,wheel_zoom,tap,undo,redo,reset,previewsave")
# if len(genes_c_raw) <= max_genes_c:
gene_c_plot.segment(genes_c_plot_start,
genes_c_plot_yn,
genes_c_plot_end,
genes_c_plot_yn,
color="black",
alpha=1,
line_width=2)
gene_c_plot.rect(x='exons_c_plot_x',
y='exons_c_plot_yn',
width='exons_c_plot_w',
height='exons_c_plot_h',
source=source_gene_c_plot,
fill_color="grey",
line_color="grey")
gene_c_plot.text(genes_c_plot_start,
genes_c_plot_yn,
text=genes_c_plot_name,
alpha=1,
text_font_size="7pt",
text_font_style="bold",
text_baseline="middle",
text_align="right",
angle=0)
hover = gene_c_plot.select(dict(type=HoverTool))
hover.tooltips = OrderedDict([
("Gene", "@exons_c_plot_name"),
("Transcript IDs", "@exons_c_plot_id"),
])
# else:
# x_coord_text = coord1/1000000.0 + (coord2/1000000.0 - coord1/1000000.0) / 2.0
# gene_c_plot.text(x_coord_text, n_rows_c / 2.0, text=message_c, alpha=1,
# text_font_size="12pt", text_font_style="bold", text_baseline="middle", text_align="center", angle=0)
gene_c_plot.xaxis.axis_label = "Chromosome " + chromosome + " Coordinate (Mb)(" + genome_build_vars[
genome_build]['title'] + ")"
gene_c_plot.yaxis.axis_label = "Genes (Transcripts Collapsed)"
gene_c_plot.ygrid.grid_line_color = None
gene_c_plot.yaxis.axis_line_color = None
gene_c_plot.yaxis.minor_tick_line_color = None
gene_c_plot.yaxis.major_tick_line_color = None
gene_c_plot.yaxis.major_label_text_color = None
gene_c_plot.toolbar_location = "below"
# Change output backend to SVG temporarily for headless export
assoc_plot.output_backend = "svg"
rug.output_backend = "svg"
gene_c_plot.output_backend = "svg"
export_svgs(assoc_plot,
filename=tmp_dir + "assoc_plot_1_" + request + ".svg")
export_svgs(gene_c_plot,
filename=tmp_dir + "gene_plot_1_" + request + ".svg")
# 1 pixel = 0.0264583333 cm
svg_height = str(20.00 + (0.0264583333 * plot_c_h_pix)) + "cm"
svg_height_scaled = str(100.00 + (0.1322916665 * plot_c_h_pix)) + "cm"
# Concatenate svgs
sg.Figure(
"24.59cm", svg_height,
sg.SVG(tmp_dir + "assoc_plot_1_" + request + ".svg"),
sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").move(
-40, 630)).save(tmp_dir + "assoc_plot_" + request + ".svg")
sg.Figure(
"122.95cm", svg_height_scaled,
sg.SVG(tmp_dir + "assoc_plot_1_" + request + ".svg").scale(5),
sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").scale(5).move(
-200,
3150)).save(tmp_dir + "assoc_plot_scaled_" + request + ".svg")
# Export to PDF
subprocess.call("phantomjs ./rasterize.js " + tmp_dir + "assoc_plot_" +
request + ".svg " + tmp_dir + "assoc_plot_" + request +
".pdf",
shell=True)
# Export to PNG
subprocess.call("phantomjs ./rasterize.js " + tmp_dir +
"assoc_plot_scaled_" + request + ".svg " + tmp_dir +
"assoc_plot_" + request + ".png",
shell=True)
# Export to JPEG
subprocess.call("phantomjs ./rasterize.js " + tmp_dir +
"assoc_plot_scaled_" + request + ".svg " + tmp_dir +
"assoc_plot_" + request + ".jpeg",
shell=True)
# Remove individual SVG files after they are combined
subprocess.call("rm " + tmp_dir + "assoc_plot_1_" + request + ".svg",
shell=True)
subprocess.call("rm " + tmp_dir + "gene_plot_1_" + request + ".svg",
shell=True)
# Remove scaled SVG file after it is converted to png and jpeg
subprocess.call("rm " + tmp_dir + "assoc_plot_scaled_" + request +
".svg",
shell=True)
reset_output()
# Remove temporary files
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt", shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
subprocess.call("rm " + tmp_dir + "genes_*" + request + "*.json",
shell=True)
subprocess.call("rm " + tmp_dir + "recomb_" + request + ".json",
shell=True)
subprocess.call("rm " + tmp_dir + "assoc_args" + request + ".json",
shell=True)
print("Bokeh high quality image export complete!")
# Return plot output
return None
def _init_all_descr_CDS(self):
data = dict(descr=list(self.descr_key_title_map.values()),
score=["0" for __ in range(5)])
source = ColumnDataSource(data)
return source
def stacked_chart(death_types, death_list):
if len(death_list[0]) > 6:
x_axis_labels = [
'Jan', 'Feb', 'Mrt', 'Apr', 'Mei', 'Juni', 'Jul', 'Aug', 'Sep',
'Okt', 'Nov', 'Dec'
]
output_file("stacked_months.html")
elif len(death_list[0]) < 6:
x_axis_labels = [
"male victim", "male suspect", 'female victim', "female suspect"
]
output_file("stacked_gender.html")
else:
output_file("stacked_years.html")
x_axis_labels = ['2013', '2014', '2015', '2016', '2017', '2018']
colors = [
"#2ad123", "#1b7c17", "#1518d8", "#4e50e5", "#4d71e5", "#4d9be5",
"#f9ed3b", "#bc7f14", "#b74207", "#e51010", "#ff8c1a"
]
data = {
'Months': x_axis_labels,
death_types[0]: death_list[0],
death_types[1]: death_list[1],
death_types[2]: death_list[2],
death_types[3]: death_list[3],
death_types[4]: death_list[4],
death_types[5]: death_list[5],
death_types[6]: death_list[6],
death_types[7]: death_list[7],
death_types[8]: death_list[8],
death_types[9]: death_list[9],
death_types[10]: death_list[10]
}
source = ColumnDataSource(data=data)
p = figure(x_range=x_axis_labels,
plot_height=400,
title="Causes of deaths",
toolbar_location=None,
tools="")
rs = p.vbar_stack(death_types,
x='Months',
width=0.9,
color=colors,
source=source)
p.y_range.start = 0
p.x_range.range_padding = 0.1
p.xgrid.grid_line_color = None
p.axis.minor_tick_line_color = None
p.outline_line_color = None
p.legend.location = "top_right"
p.legend.orientation = "horizontal"
p.yaxis[0].formatter = NumeralTickFormatter(format='0 %')
legend = Legend(items=[(death, [r])
for (death, r) in zip(death_types, rs)],
location=(0, 30))
p.add_layout(legend, 'right')
#show(p)
return (p)
def _init_input_res_CDS(self):
data = dict(resno=["" for __ in range(31)],
match_vals=["" for __ in range(31)])
source = ColumnDataSource(data)
return source
def plot_3DModel_bokeh(filename, map_data_all_slices, map_depth_all_slices, \
color_range_all_slices, profile_data_all, boundary_data, \
style_parameter):
'''
Plot shear velocity maps and velocity profiles using bokeh
Input:
filename is the filename of the resulting html file
map_data_all_slices contains the velocity model parameters saved for map view plots
map_depth_all_slices is a list of depths
color_range_all_slices is a list of color ranges
profile_data_all constains the velocity model parameters saved for profile plots
boundary_data is a list of boundaries
style_parameter contains plotting parameters
Output:
None
'''
xlabel_fontsize = style_parameter['xlabel_fontsize']
#
colorbar_data_all_left = []
colorbar_data_all_right = []
map_view_ndepth = style_parameter['map_view_ndepth']
ncolor = len(palette)
colorbar_top = [0.1 for i in range(ncolor)]
colorbar_bottom = [0 for i in range(ncolor)]
map_data_all_slices_depth = []
for idepth in range(map_view_ndepth):
color_min = color_range_all_slices[idepth][0]
color_max = color_range_all_slices[idepth][1]
color_step = (color_max - color_min)*1./ncolor
colorbar_left = np.linspace(color_min,color_max-color_step,ncolor)
colorbar_right = np.linspace(color_min+color_step,color_max,ncolor)
colorbar_data_all_left.append(colorbar_left)
colorbar_data_all_right.append(colorbar_right)
map_depth = map_depth_all_slices[idepth]
map_data_all_slices_depth.append('Depth: {0:8.1f} km'.format(map_depth))
#
palette_r = palette[::-1]
# data for the colorbar
colorbar_data_one_slice = {}
colorbar_data_one_slice['colorbar_left'] = colorbar_data_all_left[style_parameter['map_view_default_index']]
colorbar_data_one_slice['colorbar_right'] = colorbar_data_all_right[style_parameter['map_view_default_index']]
colorbar_data_one_slice_bokeh = ColumnDataSource(data=dict(colorbar_top=colorbar_top,colorbar_bottom=colorbar_bottom,\
colorbar_left=colorbar_data_one_slice['colorbar_left'],\
colorbar_right=colorbar_data_one_slice['colorbar_right'],\
palette_r=palette_r))
colorbar_data_all_slices_bokeh = ColumnDataSource(data=dict(colorbar_data_all_left=colorbar_data_all_left,\
colorbar_data_all_right=colorbar_data_all_right))
#
map_view_label_lon = style_parameter['map_view_depth_label_lon']
map_view_label_lat = style_parameter['map_view_depth_label_lat']
map_data_one_slice_depth = map_data_all_slices_depth[style_parameter['map_view_default_index']]
map_data_one_slice_depth_bokeh = ColumnDataSource(data=dict(lat=[map_view_label_lat], lon=[map_view_label_lon],
map_depth=[map_data_one_slice_depth],
left=[style_parameter['profile_plot_xmin']], \
right=[style_parameter['profile_plot_xmax']]))
#
map_view_default_index = style_parameter['map_view_default_index']
#map_data_one_slice = map_data_all_slices[map_view_default_index]
#
map_color_all_slices = []
for i in range(len(map_data_all_slices)):
vmin, vmax = color_range_all_slices[i]
map_color = val_to_rgb(map_data_all_slices[i], palette_r, vmin, vmax)
map_color_all_slices.append(map_color)
map_color_one_slice = map_color_all_slices[map_view_default_index]
#
map_data_one_slice_bokeh = ColumnDataSource(data=dict(x=[style_parameter['map_view_image_lon_min']],\
y=[style_parameter['map_view_image_lat_min']],dw=[style_parameter['nlon']*style_parameter['dlon']],\
dh=[style_parameter['nlat']*style_parameter['dlat']],map_data_one_slice=[map_color_one_slice]))
map_data_all_slices_bokeh = ColumnDataSource(data=dict(map_data_all_slices=map_color_all_slices,\
map_data_all_slices_depth=map_data_all_slices_depth))
# ------------------------------
nprofile = len(profile_data_all)
grid_lat_list = []
grid_lon_list = []
width_list = []
height_list = []
for iprofile in range(nprofile):
aprofile = profile_data_all[iprofile]
grid_lat_list.append(aprofile['lat'])
grid_lon_list.append(aprofile['lon'])
width_list.append(style_parameter['map_view_grid_width'])
height_list.append(style_parameter['map_view_grid_height'])
grid_data_bokeh = ColumnDataSource(data=dict(lon=grid_lon_list,lat=grid_lat_list,\
width=width_list, height=height_list))
profile_default_index = style_parameter['profile_default_index']
selected_dot_on_map_bokeh = ColumnDataSource(data=dict(lat=[grid_lat_list[profile_default_index]], \
lon=[grid_lon_list[profile_default_index]], \
width=[style_parameter['map_view_grid_width']],\
height=[style_parameter['map_view_grid_height']],\
index=[profile_default_index]))
# ------------------------------
profile_vs_all = []
profile_depth_all = []
profile_ndepth = style_parameter['profile_ndepth']
profile_lat_label_list = []
profile_lon_label_list = []
for iprofile in range(nprofile):
aprofile = profile_data_all[iprofile]
vs_raw = aprofile['vs']
top_raw = aprofile['top']
profile_lat_label_list.append('Lat: {0:12.1f}'.format(aprofile['lat']))
profile_lon_label_list.append('Lon: {0:12.1f}'.format(aprofile['lon']))
vs_plot = []
depth_plot = []
for idepth in range(profile_ndepth):
vs_plot.append(vs_raw[idepth])
depth_plot.append(top_raw[idepth])
vs_plot.append(vs_raw[idepth])
depth_plot.append(top_raw[idepth+1])
profile_vs_all.append(vs_plot)
profile_depth_all.append(depth_plot)
profile_data_all_bokeh = ColumnDataSource(data=dict(profile_vs_all=profile_vs_all, \
profile_depth_all=profile_depth_all))
selected_profile_data_bokeh = ColumnDataSource(data=dict(vs=profile_vs_all[profile_default_index],\
depth=profile_depth_all[profile_default_index]))
selected_profile_lat_label_bokeh = ColumnDataSource(data=\
dict(x=[style_parameter['profile_lat_label_x']], y=[style_parameter['profile_lat_label_y']],\
lat_label=[profile_lat_label_list[profile_default_index]]))
selected_profile_lon_label_bokeh = ColumnDataSource(data=\
dict(x=[style_parameter['profile_lon_label_x']], y=[style_parameter['profile_lon_label_y']],\
lon_label=[profile_lon_label_list[profile_default_index]]))
all_profile_lat_label_bokeh = ColumnDataSource(data=dict(profile_lat_label_list=profile_lat_label_list))
all_profile_lon_label_bokeh = ColumnDataSource(data=dict(profile_lon_label_list=profile_lon_label_list))
#
button_ndepth = style_parameter['button_ndepth']
button_data_all_vs = []
button_data_all_vp = []
button_data_all_rho = []
button_data_all_top = []
for iprofile in range(nprofile):
aprofile = profile_data_all[iprofile]
button_data_all_vs.append(aprofile['vs'][:button_ndepth])
button_data_all_vp.append(aprofile['vp'][:button_ndepth])
button_data_all_rho.append(aprofile['rho'][:button_ndepth])
button_data_all_top.append(aprofile['top'][:button_ndepth])
button_data_all_bokeh = ColumnDataSource(data=dict(button_data_all_vs=button_data_all_vs,\
button_data_all_vp=button_data_all_vp,\
button_data_all_rho=button_data_all_rho,\
button_data_all_top=button_data_all_top))
# ==============================
map_view = Figure(plot_width=style_parameter['map_view_plot_width'], plot_height=style_parameter['map_view_plot_height'], \
tools=style_parameter['map_view_tools'], title=style_parameter['map_view_title'], \
y_range=[style_parameter['map_view_figure_lat_min'], style_parameter['map_view_figure_lat_max']],\
x_range=[style_parameter['map_view_figure_lon_min'], style_parameter['map_view_figure_lon_max']])
#
map_view.image_rgba('map_data_one_slice',x='x',\
y='y',dw='dw',dh='dh',
source=map_data_one_slice_bokeh, level='image')
depth_slider_callback = CustomJS(args=dict(map_data_one_slice_bokeh=map_data_one_slice_bokeh,\
map_data_all_slices_bokeh=map_data_all_slices_bokeh,\
colorbar_data_all_slices_bokeh=colorbar_data_all_slices_bokeh,\
colorbar_data_one_slice_bokeh=colorbar_data_one_slice_bokeh,\
map_data_one_slice_depth_bokeh=map_data_one_slice_depth_bokeh), code="""
var d_index = Math.round(cb_obj.value)
var map_data_all_slices = map_data_all_slices_bokeh.data
map_data_one_slice_bokeh.data['map_data_one_slice'] = [map_data_all_slices['map_data_all_slices'][d_index]]
map_data_one_slice_bokeh.change.emit()
var color_data_all_slices = colorbar_data_all_slices_bokeh.data
colorbar_data_one_slice_bokeh.data['colorbar_left'] = color_data_all_slices['colorbar_data_all_left'][d_index]
colorbar_data_one_slice_bokeh.data['colorbar_right'] = color_data_all_slices['colorbar_data_all_right'][d_index]
colorbar_data_one_slice_bokeh.change.emit()
map_data_one_slice_depth_bokeh.data['map_depth'] = [map_data_all_slices['map_data_all_slices_depth'][d_index]]
map_data_one_slice_depth_bokeh.change.emit()
""")
depth_slider = Slider(start=0, end=style_parameter['map_view_ndepth']-1, \
value=map_view_default_index, step=1, \
width=style_parameter['map_view_plot_width'],\
title=style_parameter['depth_slider_title'], height=50, \
callback=depth_slider_callback)
# ------------------------------
# add boundaries to map view
# country boundaries
map_view.multi_line(boundary_data['country']['longitude'],\
boundary_data['country']['latitude'],color='black',\
line_width=2, level='underlay',nonselection_line_alpha=1.0,\
nonselection_line_color='black')
# marine boundaries
map_view.multi_line(boundary_data['marine']['longitude'],\
boundary_data['marine']['latitude'],color='black',\
level='underlay',nonselection_line_alpha=1.0,\
nonselection_line_color='black')
# shoreline boundaries
map_view.multi_line(boundary_data['shoreline']['longitude'],\
boundary_data['shoreline']['latitude'],color='black',\
line_width=2, level='underlay',nonselection_line_alpha=1.0,\
nonselection_line_color='black')
# state boundaries
map_view.multi_line(boundary_data['state']['longitude'],\
boundary_data['state']['latitude'],color='black',\
level='underlay',nonselection_line_alpha=1.0,\
nonselection_line_color='black')
# ------------------------------
# add depth label
map_view.rect(style_parameter['map_view_depth_box_lon'], style_parameter['map_view_depth_box_lat'], \
width=style_parameter['map_view_depth_box_width'], height=style_parameter['map_view_depth_box_height'], \
width_units='screen',height_units='screen', color='#FFFFFF', line_width=1., line_color='black', level='underlay')
map_view.text('lon', 'lat', 'map_depth', source=map_data_one_slice_depth_bokeh,\
text_font_size=style_parameter['annotating_text_font_size'],text_align='left',level='underlay')
# ------------------------------
map_view.rect('lon', 'lat', width='width', \
width_units='screen', height='height', \
height_units='screen', line_color='gray', line_alpha=0.5, \
selection_line_color='gray', selection_line_alpha=0.5, selection_fill_color=None,\
nonselection_line_color='gray',nonselection_line_alpha=0.5, nonselection_fill_color=None,\
source=grid_data_bokeh, color=None, line_width=1, level='glyph')
map_view.rect('lon', 'lat',width='width', \
width_units='screen', height='height', \
height_units='screen', line_color='#00ff00', line_alpha=1.0, \
source=selected_dot_on_map_bokeh, fill_color=None, line_width=3.,level='glyph')
# ------------------------------
grid_data_js = CustomJS(args=dict(selected_dot_on_map_bokeh=selected_dot_on_map_bokeh, \
grid_data_bokeh=grid_data_bokeh,\
profile_data_all_bokeh=profile_data_all_bokeh,\
selected_profile_data_bokeh=selected_profile_data_bokeh,\
selected_profile_lat_label_bokeh=selected_profile_lat_label_bokeh,\
selected_profile_lon_label_bokeh=selected_profile_lon_label_bokeh, \
all_profile_lat_label_bokeh=all_profile_lat_label_bokeh, \
all_profile_lon_label_bokeh=all_profile_lon_label_bokeh, \
), code="""
var inds = cb_obj.indices
var grid_data = grid_data_bokeh.data
selected_dot_on_map_bokeh.data['lat'] = [grid_data['lat'][inds]]
selected_dot_on_map_bokeh.data['lon'] = [grid_data['lon'][inds]]
selected_dot_on_map_bokeh.data['index'] = [inds]
selected_dot_on_map_bokeh.change.emit()
var profile_data_all = profile_data_all_bokeh.data
selected_profile_data_bokeh.data['vs'] = profile_data_all['profile_vs_all'][inds]
selected_profile_data_bokeh.data['depth'] = profile_data_all['profile_depth_all'][inds]
selected_profile_data_bokeh.change.emit()
var all_profile_lat_label = all_profile_lat_label_bokeh.data['profile_lat_label_list']
var all_profile_lon_label = all_profile_lon_label_bokeh.data['profile_lon_label_list']
selected_profile_lat_label_bokeh.data['lat_label'] = [all_profile_lat_label[inds]]
selected_profile_lon_label_bokeh.data['lon_label'] = [all_profile_lon_label[inds]]
selected_profile_lat_label_bokeh.change.emit()
selected_profile_lon_label_bokeh.change.emit()
""")
grid_data_bokeh.selected.js_on_change('indices', grid_data_js)
# ------------------------------
# change style
map_view.title.text_font_size = style_parameter['title_font_size']
map_view.title.align = 'center'
map_view.title.text_font_style = 'normal'
map_view.xaxis.axis_label = style_parameter['map_view_xlabel']
map_view.xaxis.axis_label_text_font_style = 'normal'
map_view.xaxis.axis_label_text_font_size = xlabel_fontsize
map_view.xaxis.major_label_text_font_size = xlabel_fontsize
map_view.yaxis.axis_label = style_parameter['map_view_ylabel']
map_view.yaxis.axis_label_text_font_style = 'normal'
map_view.yaxis.axis_label_text_font_size = xlabel_fontsize
map_view.yaxis.major_label_text_font_size = xlabel_fontsize
map_view.xgrid.grid_line_color = None
map_view.ygrid.grid_line_color = None
map_view.toolbar.logo = None
map_view.toolbar_location = 'above'
map_view.toolbar_sticky = False
# ==============================
# plot colorbar
colorbar_fig = Figure(tools=[], y_range=(0,0.1),plot_width=style_parameter['map_view_plot_width'], \
plot_height=style_parameter['colorbar_plot_height'],title=style_parameter['colorbar_title'])
colorbar_fig.toolbar_location=None
colorbar_fig.quad(top='colorbar_top',bottom='colorbar_bottom',left='colorbar_left',right='colorbar_right',\
color='palette_r',source=colorbar_data_one_slice_bokeh)
colorbar_fig.yaxis[0].ticker=FixedTicker(ticks=[])
colorbar_fig.xgrid.grid_line_color = None
colorbar_fig.ygrid.grid_line_color = None
colorbar_fig.xaxis.axis_label_text_font_size = xlabel_fontsize
colorbar_fig.xaxis.major_label_text_font_size = xlabel_fontsize
colorbar_fig.xaxis[0].formatter = PrintfTickFormatter(format="%5.2f")
colorbar_fig.title.text_font_size = xlabel_fontsize
colorbar_fig.title.align = 'center'
colorbar_fig.title.text_font_style = 'normal'
# ==============================
profile_xrange = Range1d(start=style_parameter['profile_plot_xmin'], end=style_parameter['profile_plot_xmax'])
profile_yrange = Range1d(start=style_parameter['profile_plot_ymax'], end=style_parameter['profile_plot_ymin'])
profile_fig = Figure(plot_width=style_parameter['profile_plot_width'], plot_height=style_parameter['profile_plot_height'],\
x_range=profile_xrange, y_range=profile_yrange, tools=style_parameter['profile_tools'],\
title=style_parameter['profile_title'])
profile_fig.line('vs','depth',source=selected_profile_data_bokeh, line_width=2, line_color='black')
# ------------------------------
# add lat, lon
profile_fig.rect([style_parameter['profile_label_box_x']], [style_parameter['profile_label_box_y']],\
width=style_parameter['profile_label_box_width'], height=style_parameter['profile_label_box_height'],\
width_units='screen', height_units='screen', color='#FFFFFF', line_width=1., line_color='black',\
level='underlay')
profile_fig.text('x','y','lat_label', source=selected_profile_lat_label_bokeh)
profile_fig.text('x','y','lon_label', source=selected_profile_lon_label_bokeh)
# ------------------------------
# change style
profile_fig.xaxis.axis_label = style_parameter['profile_xlabel']
profile_fig.xaxis.axis_label_text_font_style = 'normal'
profile_fig.xaxis.axis_label_text_font_size = xlabel_fontsize
profile_fig.xaxis.major_label_text_font_size = xlabel_fontsize
profile_fig.yaxis.axis_label = style_parameter['profile_ylabel']
profile_fig.yaxis.axis_label_text_font_style = 'normal'
profile_fig.yaxis.axis_label_text_font_size = xlabel_fontsize
profile_fig.yaxis.major_label_text_font_size = xlabel_fontsize
profile_fig.xgrid.grid_line_dash = [4, 2]
profile_fig.ygrid.grid_line_dash = [4, 2]
profile_fig.title.text_font_size = style_parameter['title_font_size']
profile_fig.title.align = 'center'
profile_fig.title.text_font_style = 'normal'
profile_fig.toolbar_location = 'above'
profile_fig.toolbar_sticky = False
profile_fig.toolbar.logo = None
# ==============================
profile_slider_callback = CustomJS(args=dict(selected_dot_on_map_bokeh=selected_dot_on_map_bokeh,\
grid_data_bokeh=grid_data_bokeh, \
profile_data_all_bokeh=profile_data_all_bokeh, \
selected_profile_data_bokeh=selected_profile_data_bokeh,\
selected_profile_lat_label_bokeh=selected_profile_lat_label_bokeh,\
selected_profile_lon_label_bokeh=selected_profile_lon_label_bokeh, \
all_profile_lat_label_bokeh=all_profile_lat_label_bokeh, \
all_profile_lon_label_bokeh=all_profile_lon_label_bokeh), code="""
var p_index = Math.round(cb_obj.value)
var grid_data = grid_data_bokeh.data
selected_dot_on_map_bokeh.data['lat'] = [grid_data['lat'][p_index]]
selected_dot_on_map_bokeh.data['lon'] = [grid_data['lon'][p_index]]
selected_dot_on_map_bokeh.data['index'] = [p_index]
selected_dot_on_map_bokeh.change.emit()
var profile_data_all = profile_data_all_bokeh.data
selected_profile_data_bokeh.data['vs'] = profile_data_all['profile_vs_all'][p_index]
selected_profile_data_bokeh.data['depth'] = profile_data_all['profile_depth_all'][p_index]
selected_profile_data_bokeh.change.emit()
var all_profile_lat_label = all_profile_lat_label_bokeh.data['profile_lat_label_list']
var all_profile_lon_label = all_profile_lon_label_bokeh.data['profile_lon_label_list']
selected_profile_lat_label_bokeh.data['lat_label'] = [all_profile_lat_label[p_index]]
selected_profile_lon_label_bokeh.data['lon_label'] = [all_profile_lon_label[p_index]]
selected_profile_lat_label_bokeh.change.emit()
selected_profile_lon_label_bokeh.change.emit()
""")
profile_slider = Slider(start=0, end=nprofile-1, value=style_parameter['profile_default_index'], \
step=1, title=style_parameter['profile_slider_title'], \
width=style_parameter['profile_plot_width'], height=50,\
callback=profile_slider_callback)
profile_slider_callback.args['profile_index'] = profile_slider
# ==============================
simple_text_button_callback = CustomJS(args=dict(button_data_all_bokeh=button_data_all_bokeh,\
selected_dot_on_map_bokeh=selected_dot_on_map_bokeh), \
code="""
var index = selected_dot_on_map_bokeh.data['index']
var button_data_vs = button_data_all_bokeh.data['button_data_all_vs'][index]
var button_data_vp = button_data_all_bokeh.data['button_data_all_vp'][index]
var button_data_rho = button_data_all_bokeh.data['button_data_all_rho'][index]
var button_data_top = button_data_all_bokeh.data['button_data_all_top'][index]
var csvContent = "data:text;charset=utf-8,"
var i = 0
var temp = csvContent
temp += "# Layer Top (km) Vs(km/s) Vp(km/s) Rho(g/cm^3) \\n"
while(button_data_vp[i]) {
temp+=button_data_top[i].toPrecision(6) + " " + button_data_vs[i].toPrecision(4) + " " + \
button_data_vp[i].toPrecision(4) + " " + button_data_rho[i].toPrecision(4) + "\\n"
i = i + 1
}
var encodedUri = encodeURI(temp)
link = document.createElement('a');
link.setAttribute('href', encodedUri);
link.setAttribute('download', 'vel_model.txt');
link.click();
""")
simple_text_button = Button(label=style_parameter['simple_text_button_label'], button_type='default', width=style_parameter['button_width'],\
callback=simple_text_button_callback)
# ------------------------------
model96_button_callback = CustomJS(args=dict(button_data_all_bokeh=button_data_all_bokeh,\
selected_dot_on_map_bokeh=selected_dot_on_map_bokeh), \
code="""
var index = selected_dot_on_map_bokeh.data['index']
var lat = selected_dot_on_map_bokeh.data['lat']
var lon = selected_dot_on_map_bokeh.data['lon']
var button_data_vs = button_data_all_bokeh.data['button_data_all_vs'][index]
var button_data_vp = button_data_all_bokeh.data['button_data_all_vp'][index]
var button_data_rho = button_data_all_bokeh.data['button_data_all_rho'][index]
var button_data_top = button_data_all_bokeh.data['button_data_all_top'][index]
var csvContent = "data:text;charset=utf-8,"
var i = 0
var temp = csvContent
temp += "MODEL." + index + " \\n"
temp += "ShearVelocityModel Lat: "+ lat +" Lon: " + lon + "\\n"
temp += "ISOTROPIC \\n"
temp += "KGS \\n"
temp += "SPHERICAL EARTH \\n"
temp += "1-D \\n"
temp += "CONSTANT VELOCITY \\n"
temp += "LINE08 \\n"
temp += "LINE09 \\n"
temp += "LINE10 \\n"
temp += "LINE11 \\n"
temp += " H(KM) VP(KM/S) VS(KM/S) RHO(GM/CC) QP QS ETAP ETAS FREFP FREFS \\n"
while(button_data_vp[i+1]) {
var thickness = button_data_top[i+1] - button_data_top[i]
temp+=" " +thickness.toPrecision(6) + " " + button_data_vp[i].toPrecision(4) + " " + button_data_vs[i].toPrecision(4) \
+ " " + button_data_rho[i].toPrecision(4) + " 0.00 0.00 0.00 0.00 1.00 1.00" + "\\n"
i = i + 1
}
var encodedUri = encodeURI(temp)
link = document.createElement('a');
link.setAttribute('href', encodedUri);
link.setAttribute('download', 'vel_model96.txt');
link.click();
""")
model96_button = Button(label=style_parameter['model96_button_label'], button_type='default', width=style_parameter['button_width'],\
callback=model96_button_callback)
# ==============================
# annotating text
annotating_fig01 = Div(text=style_parameter['annotating_html01'], \
width=style_parameter['annotation_plot_width'], height=style_parameter['annotation_plot_height'])
annotating_fig02 = Div(text=style_parameter['annotating_html02'],\
width=style_parameter['annotation_plot_width'], height=style_parameter['annotation_plot_height'])
# ==============================
output_file(filename,title=style_parameter['html_title'], mode=style_parameter['library_source'])
left_column = Column(depth_slider, map_view, colorbar_fig, annotating_fig01, width=style_parameter['left_column_width'])
button_pannel = Row(simple_text_button, model96_button)
right_column = Column(profile_slider, profile_fig, button_pannel, annotating_fig02, width=style_parameter['right_column_width'])
layout = Row(left_column, right_column)
save(layout)
def plot():
if request.method == 'POST':
symbol = request.form['symbol']
# Get Stock DataFrame
# msft = yf.Ticker("MSFT")
msft = yf.Ticker(symbol)
hist = msft.history(period='max')
# Define constants
W_PLOT = 1000
H_PLOT = 400
TOOLS = 'pan,wheel_zoom,hover,reset'
VBAR_WIDTH = 0.2
RED = Category20[7][6]
GREEN = Category20[5][4]
BLUE = Category20[3][0]
BLUE_LIGHT = Category20[3][1]
ORANGE = Category20[3][2]
PURPLE = Category20[9][8]
BROWN = Category20[11][10]
def get_symbol_df(symbol=None):
df = pd.DataFrame(hist)[-50:]
df.reset_index(inplace=True)
df["Date"] = pd.to_datetime(df["Date"])
return df
def plot_stock_price(stock):
p = figure(plot_width=W_PLOT,
plot_height=H_PLOT,
tools=TOOLS,
title="Stock price",
toolbar_location='above')
inc = stock.data['Close'] > stock.data['Open']
dec = stock.data['Open'] > stock.data['Close']
view_inc = CDSView(source=stock, filters=[BooleanFilter(inc)])
view_dec = CDSView(source=stock, filters=[BooleanFilter(dec)])
# map dataframe indices to date strings and use as label overrides
p.xaxis.major_label_overrides = {
i + int(stock.data['index'][0]): date.strftime('%b %d')
for i, date in enumerate(pd.to_datetime(stock.data["Date"]))
}
p.xaxis.bounds = (stock.data['index'][0], stock.data['index'][-1])
p.segment(x0='index',
x1='index',
y0='Low',
y1='High',
color=RED,
source=stock,
view=view_inc)
p.segment(x0='index',
x1='index',
y0='Low',
y1='High',
color=GREEN,
source=stock,
view=view_dec)
p.vbar(x='index',
width=VBAR_WIDTH,
top='Open',
bottom='Close',
fill_color=BLUE,
line_color=BLUE,
source=stock,
view=view_inc,
name="price")
p.vbar(x='index',
width=VBAR_WIDTH,
top='Open',
bottom='Close',
fill_color=RED,
line_color=RED,
source=stock,
view=view_dec,
name="price")
p.legend.location = "top_left"
p.legend.border_line_alpha = 0
p.legend.background_fill_alpha = 0
p.legend.click_policy = "mute"
p.yaxis.formatter = NumeralTickFormatter(format='$ 0,0[.]000')
p.x_range.range_padding = 0.05
p.xaxis.ticker.desired_num_ticks = 40
p.xaxis.major_label_orientation = 3.14 / 4
# Select specific tool for the plot
price_hover = p.select(dict(type=HoverTool))
# Choose, which glyphs are active by glyph name
price_hover.names = ["price"]
# Creating tooltips
price_hover.tooltips = [("Datetime", "@Date{%Y-%m-%d}"),
("Open", "@Open{$0,0.00}"),
("Close", "@Close{$0,0.00}"),
("Volume", "@Volume{($ 0.00 a)}")]
price_hover.formatters = {"Date": 'datetime'}
return p
stock = ColumnDataSource(
data=dict(Date=[], Open=[], Close=[], High=[], Low=[], index=[]))
#stock = AjaxDataSource(data=dict(Date=[], Open=[], Close=[], High=[], Low=[],index=[]),data_url='http://127.0.0.1:5000/plot',polling_interval=1000,mode='append')
#symbol = 'msft'
df = get_symbol_df(symbol)
stock.data = stock.from_df(df)
elements = list()
# update_plot()
p_stock = plot_stock_price(stock)
elements.append(p_stock)
curdoc().add_root(column(elements))
curdoc().title = 'Bokeh stocks historical prices'
#show(p_stock)
script, div = components(p_stock)
kwargs = {'script': script, 'div': div}
#kwargs['title'] = 'bokeh-with-flask'
return render_template('index.html', **kwargs)
#return redirect(url_for('index', **kwargs))
#return kwargs
#return json.dumps(json_item(p_stock,"myplot"))
#return redirect(url_for('index'))
return "OK"
# Specify the name of the output file and show the result
output_file('auto-df.html')
show(p)
####################################
#The Bokeh ColumnDataSource
####################################
# Import the ColumnDataSource class from bokeh.plotting
from bokeh.plotting import ColumnDataSource
# Create a ColumnDataSource: source
source = ColumnDataSource(df)
# Add circle glyphs to the figure p
p.circle(x='Year', y='Time', color='color', size=8, source=source)
# Specify the name of the output file and show the result
output_file('sprint.html')
show(p)
###################################
#Selection and non-selection glyphs
####################################
# Create a figure with the "box_select" tool: p
class PlotterBokeh(Plotter):
def __init__(self, ds, x, y, z=None, **kwargs):
"""
"""
# bokeh custom options / defaults
kwargs['return_fig'] = kwargs.pop('return_fig', False)
self._interactive = kwargs.pop('interactive', False)
super().__init__(ds, x, y, z, **kwargs, backend='BOKEH')
def prepare_axes(self):
"""Make the bokeh plot figure and set options.
"""
from bokeh.plotting import figure
if self.add_to_axes is not None:
self._plot = self.add_to_axes
else:
# Currently axes scale type must be set at figure creation?
self._plot = figure(
# convert figsize to roughly matplotlib dimensions
width=int(self.figsize[0] * 80 +
(100 if self._use_legend else 0) +
(20 if self._ytitle else 0) +
(20 if not self.yticklabels_hide else 0)),
height=int(self.figsize[1] * 80 +
(20 if self.title else 0) +
(20 if self._xtitle else 0) +
(20 if not self.xticklabels_hide else 0)),
x_axis_type=('log' if self.xlog else 'linear'),
y_axis_type=('log' if self.ylog else 'linear'),
y_axis_location=('right' if self.ytitle_right else 'left'),
title=self.title,
toolbar_location="above",
toolbar_sticky=False,
active_scroll="wheel_zoom",
)
def set_axes_labels(self):
"""Set the labels on the axes.
"""
if self._xtitle:
self._plot.xaxis.axis_label = self._xtitle
if self._ytitle:
self._plot.yaxis.axis_label = self._ytitle
def set_axes_range(self):
"""Set the plot ranges of the axes, and the panning limits.
"""
from bokeh.models import DataRange1d
self.calc_data_range()
# plt_x_centre = (self._data_xmax + self._data_xmin) / 2
# plt_x_range = self._data_xmax - self._data_xmin
# xbounds = (plt_x_centre - plt_x_range, plt_x_centre + plt_x_range)
xbounds = None
self._plot.x_range = (DataRange1d(start=self._xlims[0],
end=self._xlims[1],
bounds=xbounds) if self._xlims else
DataRange1d(bounds=xbounds))
# plt_y_centre = (self._data_ymax + self._data_ymin) / 2
# plt_y_range = abs(self._data_ymax - self._data_ymin)
# ybounds = (plt_y_centre - plt_y_range, plt_y_centre + plt_y_range)
ybounds = None
self._plot.y_range = (DataRange1d(start=self._ylims[0],
end=self._ylims[1],
bounds=ybounds) if self._ylims else
DataRange1d(bounds=ybounds))
def set_spans(self):
"""Set custom horizontal and verical line spans.
"""
from bokeh.models import Span
span_opts = {
'level': 'glyph',
'line_dash': 'dashed',
'line_color': (127, 127, 127),
'line_width': self.span_width,
}
if self.hlines:
for hl in self.hlines:
self._plot.add_layout(Span(
location=hl, dimension='width', **span_opts))
if self.vlines:
for vl in self.vlines:
self._plot.add_layout(Span(
location=vl, dimension='height', **span_opts))
def set_gridlines(self):
"""Set whether to use gridlines or not.
"""
if not self.gridlines:
self._plot.xgrid.visible = False
self._plot.ygrid.visible = False
else:
self._plot.xgrid.grid_line_dash = self.gridline_style
self._plot.ygrid.grid_line_dash = self.gridline_style
def set_tick_marks(self):
"""Set custom locations for the tick marks.
"""
from bokeh.models import FixedTicker
if self.xticks:
self._plot.xaxis[0].ticker = FixedTicker(ticks=self.xticks)
if self.yticks:
self._plot.yaxis[0].ticker = FixedTicker(ticks=self.yticks)
if self.xticklabels_hide:
self._plot.xaxis.major_label_text_font_size = '0pt'
if self.yticklabels_hide:
self._plot.yaxis.major_label_text_font_size = '0pt'
def set_sources_heatmap(self):
from bokeh.plotting import ColumnDataSource
# initialize empty source
if not hasattr(self, '_source'):
self._source = ColumnDataSource(data=dict())
# remove mask from data -> not necessary soon? / convert to nan?
var = np.ma.getdata(self._heatmap_var)
self._source.add([var], 'image')
self._source.add([self._data_xmin], 'x')
self._source.add([self._data_ymin], 'y')
self._source.add([self._data_xmax - self._data_xmin], 'dw')
self._source.add([self._data_ymax - self._data_ymin], 'dh')
def set_sources(self):
"""Set the source dictionaries to be used by the plotter functions.
This is seperate to allow interactive updates of the data only.
"""
from bokeh.plotting import ColumnDataSource
# check if heatmap
if hasattr(self, '_heatmap_var'):
return self.set_sources_heatmap()
# 'copy' the zlabels iterator into src_zlbs
self._zlbls, src_zlbs = itertools.tee(self._zlbls)
# Initialise with empty dicts
if not hasattr(self, "_sources"):
self._sources = [ColumnDataSource(dict())
for _ in range(len(self._z_vals))]
# range through all data and update the sources
for i, (zlabel, data) in enumerate(zip(src_zlbs, self._gen_xy())):
self._sources[i].add(data['x'], 'x')
self._sources[i].add(data['y'], 'y')
self._sources[i].add([zlabel] * len(data['x']), 'z_coo')
# check for color for scatter plot
if 'c' in data:
self._sources[i].add(data['c'], 'c')
# check if should set y_err as well
if 'ye' in data:
y_err_p = data['y'] + data['ye']
y_err_m = data['y'] - data['ye']
self._sources[i].add(
list(zip(data['x'], data['x'])), 'y_err_xs')
self._sources[i].add(list(zip(y_err_p, y_err_m)), 'y_err_ys')
# check if should set x_err as well
if 'xe' in data:
x_err_p = data['x'] + data['xe']
x_err_m = data['x'] - data['xe']
self._sources[i].add(
list(zip(data['y'], data['y'])), 'x_err_ys')
self._sources[i].add(list(zip(x_err_p, x_err_m)), 'x_err_xs')
def plot_legend(self, legend_items=None):
"""Add a legend to the plot.
"""
if self._use_legend:
from bokeh.models import Legend
loc = {'best': 'top_left'}.get(self.legend_loc, self.legend_loc)
where = {None: 'right'}.get(self.legend_where, self.legend_where)
# might be manually specified, e.g. from multiplot
if legend_items is None:
legend_items = self._lgnd_items
lg = Legend(items=legend_items)
lg.location = loc
lg.click_policy = 'hide'
self._plot.add_layout(lg, where)
# Don't repeatedly redraw legend
self._use_legend = False
def set_mappable(self):
from bokeh.models import LogColorMapper, LinearColorMapper
import matplotlib as plt
mappr_fn = (LogColorMapper if self.colormap_log else LinearColorMapper)
bokehpalette = [plt.colors.rgb2hex(m) for m in self.cmap(range(256))]
self.mappable = mappr_fn(palette=bokehpalette,
low=self._zmin, high=self._zmax)
def plot_colorbar(self):
if self._use_colorbar:
where = {None: 'right'}.get(self.legend_where, self.legend_where)
from bokeh.models import ColorBar, LogTicker, BasicTicker
ticker = LogTicker if self.colormap_log else BasicTicker
color_bar = ColorBar(color_mapper=self.mappable, location=(0, 0),
ticker=ticker(desired_num_ticks=6),
title=self._ctitle)
self._plot.add_layout(color_bar, where)
def set_tools(self):
"""Set which tools appear for the plot.
"""
from bokeh.models import HoverTool
self._plot.add_tools(HoverTool(tooltips=[
("({}, {})".format(self.x_coo, self.y_coo
if isinstance(self.y_coo, str) else None),
"(@x, @y)"), (self.z_coo, "@z_coo")]))
def update(self):
from bokeh.io import push_notebook
self.set_sources()
push_notebook()
def show(self, **kwargs):
"""Show the produced figure.
"""
if self.return_fig:
return self._plot
bshow(self._plot, **kwargs)
return self
def prepare_plot(self):
self.prepare_axes()
self.set_axes_labels()
self.set_axes_range()
self.set_spans()
self.set_gridlines()
self.set_tick_marks()
self.set_sources()
df.to_excel("test.xlsx", sheet_name="sheet1", index=False)
########################################################################
# REVIEW restart here!
source_song = ColumnDataSource(
data=dict(
x=embedding[voc_size:, 0],
y=embedding[voc_size:, 1],
size=[start_size] * (lyrics_size),
title=title_list,
album=album_list,
artist=artist_list,
ids=sorted_ids,
# voc = voc_list,
alpha=[start_alpha] * (lyrics_size),
lyrics=lyrics_list
# SorV = [0]*(voc_size)+[1]*(lyrics_size)
)
)
lyrics_data.lyricsinfos[0].voc_dict()
title_list[0]
album_list[0]
[voc_list[e] for e in lyrics_list[0].keys()]
# TODO change the source_voc to the displaying property, and elsewhere for storing DF TF
source_voc = ColumnDataSource(
data=dict(
x=embedding[:voc_size, 0],
title_list = [""]*voc_size+title_list
album_list = [""]*voc_size+album_list
artist_list = [""]*voc_size+artist_list
len(voc_list)
len(title_list)
len(album_list)
len(artist_list)
#REVIEW restart here!
source_song = ColumnDataSource(
data=dict(
x=embedding_matrix[:,0],
y=embedding_matrix[:,1],
size = [0]*(voc_size)+[1]*(lyrics_size),
title=title_list,
album=album_list,
artist=artist_list,
voc = voc_list,
alpha=[0.1]*(lyrics_size+voc_size),
color=colors,
SorV = [0]*(voc_size)+[1]*(lyrics_size)
)
)
source_voc = ColumnDataSource(
data=dict(
x=embedding_matrix[:,0],
y=embedding_matrix[:,1],
size = [1]*(voc_size)+[0]*(lyrics_size),
title=title_list,
album=album_list,
artist=artist_list,
voc = voc_list,
for i in path_n:
img_list_n.append(os.path.abspath(i))
img_list_ne = []
for i in path_ne:
img_list_ne.append(os.path.abspath(i))
img_list_se = []
for i in path_se:
img_list_se.append(os.path.abspath(i))
# Set up interactive plot
tile_providers= get_provider(Vendors.STAMEN_TERRAIN)
source_sw = ColumnDataSource(data=dict(
x=xList_sw,
y=yList_sw,
desc = nameList_sw,
imgs=img_list_sw,
))
source_nw = ColumnDataSource(data=dict(
x=xList_nw,
y=yList_nw,
desc = nameList_nw,
imgs=img_list_nw,
))
source_n = ColumnDataSource(data=dict(
x=xList_n,
y=yList_n,
desc = nameList_n,
imgs=img_list_n,
def _line_for_patches(self, data, chart, renderer, series_name):
"""
Add a line along the top edge of a Patch in a stacked Area Chart; return
the new Glyph for addition to HoverTool.
:param data: original data for the graph
:type data: dict
:param chart: Chart to add the line to
:type chart: bokeh.charts.Chart
:param renderer: GlyphRenderer containing one Patches glyph, to draw
the line for
:type renderer: bokeh.models.renderers.GlyphRenderer
:param series_name: the data series name this Patches represents
:type series_name: str
:return: GlyphRenderer for a Line at the top edge of this Patch
:rtype: bokeh.models.renderers.GlyphRenderer
"""
# @TODO this method needs a major refactor
# get the original x and y values, and color
xvals = deepcopy(renderer.data_source.data['x_values'][0])
yvals = deepcopy(renderer.data_source.data['y_values'][0])
line_color = renderer.glyph.fill_color
# save original values for logging if needed
orig_xvals = [x for x in xvals]
orig_yvals = [y for y in yvals]
# get a list of the values
new_xvals = [x for x in xvals]
new_yvals = [y for y in yvals]
# so when a Patch is made, the first point is (0,0); trash it
xvals = new_xvals[1:]
yvals = new_yvals[1:]
# then, we can tell the last point in the "top" line because it will be
# followed by a point with the same x value and a y value of 0.
last_idx = None
for idx, val in enumerate(xvals):
if yvals[idx+1] == 0 and xvals[idx+1] == xvals[idx]:
last_idx = idx
break
if last_idx is None:
logger.error('Unable to find top line of patch (x_values=%s '
'y_values=%s', orig_xvals, orig_yvals)
return None
# truncate our values to just what makes up the top line
xvals = xvals[:last_idx+1]
yvals = yvals[:last_idx+1]
# Currently (bokeh 0.12.1) HoverTool won't show the tooltip for the last
# point in our line. As a hack for this, add a point with the same Y value
# and an X slightly before it.
lastx = xvals[-1]
xvals[-1] = lastx - 1000 # 1000 nanoseconds
xvals.append(lastx)
yvals.append(yvals[-1])
# get the actual download counts from the original data
download_counts = [data[series_name][y] for y in range(0, len(yvals) - 1)]
download_counts.append(download_counts[-1])
# create a ColumnDataSource for the new overlay line
data2 = {
'x': xvals, # Date/x values are numpy.datetime64
'y': yvals,
# the following are hacks for data that we want in the HoverTool tooltip
'SeriesName': [series_name for _ in yvals],
# formatted date
'FmtDate': [self.datetime64_to_formatted_date(x) for x in xvals],
# to show the exact value, not where the pointer is
'Downloads': download_counts
}
# set the formatted date for our hacked second-to-last point to the
# same value as the last point
data2['FmtDate'][-2] = data2['FmtDate'][-1]
# create the CloumnDataSource, then the line for it, then the Glyph
line_ds = ColumnDataSource(data2)
line = Line(x='x', y='y', line_color=line_color)
lineglyph = chart.add_glyph(line_ds, line)
return lineglyph
def selection_plot(response):
# Let's move these into a settings file somewhere?
# height/width could potentially be driven by the request?
# Include color data in the ColumnDataSource to allow for changing the color on
# the client side.
urls = [x[0] for x in response["pages"]]
xdata = [x[1] for x in response["pages"]]
ydata = [x[2] for x in response["pages"]]
tags = [x[3] for x in response["pages"]]
color = []
custom_tags = ["Custom tags"]
for tag in tags:
custom = False
if tag:
for t in tag:
if t not in ["Relevant", "Irrelevant", ""]:
custom = True
if t not in custom_tags:
custom_tags.append(t)
if not custom:
color.append(colormap(tag[0]))
else:
color.append(colormap("Custom"))
else:
color.append(colormap(None))
source = ColumnDataSource(
data=dict(
x=xdata,
y=ydata,
urls=urls,
tags=tags,
color=color,
)
)
# Callback code for CDS.
source.callback = CustomJS(code="""
var inds = cb_obj.get('selected')["1d"].indices;
var data = cb_obj.get('data');
BokehPlots.showPages(inds);
""")
# Create the figure with FIGURE_WIDTH and FIGURE_HEIGHT
p = figure(
tools="hover,wheel_zoom,reset",
width=FIGURE_WIDTH,
responsive=True,
tags=["clusterPlot"],
)
# Ensure that the lasso only selects with mouseup, not mousemove.
p.add_tools(
LassoSelectTool(select_every_mousemove=False),
)
# These turn off the x/y axis ticks
p.axis.visible = None
# These turn the major grid off
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None
# Plot non-selected circles with a particular style using CIRCLE_SIZE and
# 'color' list
p.circle("x", "y", size=13, line_width=2, line_alpha=1,
line_color=None, fill_alpha=1, color='color', source=source,
name="urls")
nonselected_circle = Circle(fill_alpha=0.1, line_alpha=0.1, fill_color='color',
line_color='color')
renderer = p.select(name="urls")
renderer.nonselection_glyph = nonselected_circle
# Create buttons and their callbacks, use button_code string for callbacks.
button_code = """
event.preventDefault();
var inds = source.get('selected')["1d"].indices;
var data = source.get('data');
var selected = [];
tag = "%s";
for(var i = 0; i < inds.length; i++){
selected.push({
x: data.x[inds[i]],
y: data.y[inds[i]],
url: data.urls[inds[i]],
tags: data.tags[inds[i]],
selected: true,
possible: false,
});
data["color"][inds[i]] = "%s";
}
BokehPlots.updateTags(selected, tag, "Apply");
source.trigger("change");
"""
textinput_code = """
event.preventDefault();
var inds = source.get('selected')["1d"].indices;
var data = source.get('data');
var selected = [];
var tag = cb_obj.get("value");
// Reinitialise to the default value
cb_obj.set("value", "Add custom tag...")
if(tag.indexOf("Add custom tag...") < 0) {
//Update the custom tags selection list
var options = custom_tags_select.get("options");
if(options.indexOf(tag) < 0){
options.push(tag);
custom_tags_select.set("options", options);
}
for(var i = 0; i < inds.length; i++){
selected.push({
x: data.x[inds[i]],
y: data.y[inds[i]],
url: data.urls[inds[i]],
tags: data.tags[inds[i]],
selected: true,
possible: false,
});
data["color"][inds[i]] = "%s";
}
BokehPlots.updateTags(selected, tag, "Apply");
source.trigger("change");
}
"""
selectinput_code = """
event.preventDefault();
var inds = source.get('selected')["1d"].indices;
var data = source.get('data');
var selected = [];
var tag = cb_obj.get("value");
cb_obj.set("value", "Enter tags...")
if(tag.indexOf("Add custom tag...") < 0) {
for(var i = 0; i < inds.length; i++){
selected.push({
x: data.x[inds[i]],
y: data.y[inds[i]],
url: data.urls[inds[i]],
tags: data.tags[inds[i]],
selected: true,
possible: false,
});
data["color"][inds[i]] = "%s";
}
BokehPlots.updateTags(selected, tag, "Apply");
source.trigger("change");
}
"""
# Create buttons and their callbacks, use button_code string for callbacks.
crawl_code = """
event.preventDefault();
var inds = source.get('selected')["1d"].indices;
var data = source.get('data');
var selected = [];
var crawl = '%s';
for(var i = 0; i < inds.length; i++){
selected.push(data.urls[inds[i]]);
}
BokehPlots.crawlPages(selected, crawl);
source.trigger("change");
"""
# Supply color with print formatting.
but_relevant = Button(label="Relevant", type="success")
but_relevant.callback = CustomJS(args=dict(source=source),
code=button_code % ("Relevant", POSITIVE_COLOR))
but_irrelevant = Button(label="Irrelevant", type="success")
but_irrelevant.callback = CustomJS(args=dict(source=source),
code=button_code % ("Irrelevant", NEGATIVE_COLOR))
but_neutral = Button(label="Neutral", type="success")
but_neutral.callback = CustomJS(args=dict(source=source),
code=button_code % ("Neutral", NEUTRAL_COLOR))
custom_tag_input = TextInput(value="Add custom tag...")
custom_tag_input.callback = CustomJS(args=dict(source=source),
code=textinput_code % (CUSTOM_COLOR))
custom_tag_select = Select(value="Custom tags", options=custom_tags)
custom_tag_select.callback = CustomJS(args=dict(source=source),
code=selectinput_code % (CUSTOM_COLOR))
custom_tag_input.callback.args["custom_tags_select"] = custom_tag_select
but_backward_crawl = Button(label="Backlinks", type="success")
but_backward_crawl.callback = CustomJS(args=dict(source=source),
code=crawl_code % ("backward"))
but_forward_crawl = Button(label="Forwardlinks", type="success")
but_forward_crawl.callback = CustomJS(args=dict(source=source),
code=crawl_code % ("forward"))
# Adjust what attributes are displayed by the HoverTool
hover = p.select(dict(type=HoverTool))
hover.tooltips = [
("urls", "@urls"),
]
tags = hplot(custom_tag_input, custom_tag_select, but_neutral, but_relevant, but_irrelevant)
tags_crawl = hplot(but_backward_crawl, but_forward_crawl)
layout = vplot(p, tags, tags_crawl)
# Combine script and div into a single string.
plot_code = components(layout)
return plot_code[0] + plot_code[1]
sizing_mode="scale_both")
print((nominal_image_stack.max()))
layer = Slider(start=0,
end=len(nominal_image_stack),
value=0,
step=1,
title="z layer")
threshold = Slider(start=0,
end=nominal_image_stack.max(),
value=nominal_image_stack.mean(),
step=1,
title="threshold")
prev_threshold = nominal_image_stack.mean()
segmented_image_source = ColumnDataSource(data=dict(
image=[segmented_image_stack[0]]))
nominal_image_source = ColumnDataSource(data=dict(
image=[nominal_image_stack[0]]))
main_fig.image(source=nominal_image_source,
image="image",
x=0,
y=0,
dw=10,
dh=10,
level="image",
palette=RGreens7)
main_fig.image(
source=segmented_image_source,
image="image",
x=0,
def selection_plot(response):
# Let's move these into a settings file somewhere?
# height/width could potentially be driven by the request?
# Include color data in the ColumnDataSource to allow for changing the color on
# the client side.
urls = [x[0] for x in response["pages"]]
xdata = [x[1] for x in response["pages"]]
ydata = [x[2] for x in response["pages"]]
tags = [x[3] for x in response["pages"]]
color = [colormap(x[0]) if x else colormap(None) for x in tags]
source = ColumnDataSource(
data=dict(
x=xdata,
y=ydata,
urls=urls,
tags=tags,
color=color,
)
)
# Callback code for CDS.
source.callback = CustomJS(code="""
var inds = cb_obj.get('selected')["1d"].indices;
var data = cb_obj.get('data');
BokehPlots.showPages(inds);
""")
# Create the figure with FIGURE_WIDTH and FIGURE_HEIGHT
p = figure(tools="hover", width=FIGURE_WIDTH,
toolbar_location=None, responsive=True, tags=["clusterPlot"])
# Ensure that the lasso only selects with mouseup, not mousemove.
p.add_tools(LassoSelectTool(select_every_mousemove=False))
# These turn off the x/y axis ticks
p.axis.visible = None
# These turn the major grid off
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None
# Plot non-selected circles with a particular style using CIRCLE_SIZE and
# 'color' list
p.circle("x", "y", size=13, line_width=2, line_alpha=1,
line_color=None, fill_alpha=1, color='color', source=source,
name="urls")
nonselected_circle = Circle(fill_alpha=0.1, line_alpha=0.1, fill_color='color',
line_color='color')
renderer = p.select(name="urls")
renderer.nonselection_glyph = nonselected_circle
# Create buttons and their callbacks, use button_code string for callbacks.
button_code = """
event.preventDefault();
var inds = source.get('selected')["1d"].indices;
var data = source.get('data');
var selected = [];
tag = "%s";
for(var i = 0; i < inds.length; i++){
selected.push({
x: data.x[inds[i]],
y: data.y[inds[i]],
url: data.urls[inds[i]],
tags: data.tags[inds[i]],
selected: true,
possible: false,
});
data["color"][inds[i]] = "%s";
}
BokehPlots.updateTags(selected, tag, inds);
source.trigger("change");
"""
# Supply color with print formatting.
button1 = Button(label="Relevant", type="success")
button1.callback = CustomJS(args=dict(source=source),
code=button_code % ("Relevant", POSITIVE_COLOR))
button2 = Button(label="Irrelevant", type="success")
button2.callback = CustomJS(args=dict(source=source),
code=button_code % ("Irrelevant", NEGATIVE_COLOR))
button3 = Button(label="Neutral", type="success")
button3.callback = CustomJS(args=dict(source=source),
code=button_code % ("Neutral", NEUTRAL_COLOR))
# Adjust what attributes are displayed by the HoverTool
hover = p.select(dict(type=HoverTool))
hover.tooltips = [
("urls", "@urls"),
]
layout = vform(p, button1, button2, button3)
# Combine script and div into a single string.
plot_code = components(layout)
return plot_code[0] + plot_code[1]
md_data = md_data.interpolate(method='linear', axis=0).ffill().bfill()
# Tell Bokeh where to save the interactive chart
output_file('us_marriages_divorces_per_capita.html',
title='144 years of marriage and divorce in the U.S.A.')
# Set up the data sources for the lines we'll be plotting.
# We need separate data sources for each line because we're
# displaying different data in the hover tool.
source_marriages = ColumnDataSource(data=dict(
# x-axis (Years) for the chart
x=md_data.Year.values,
# y-axis (Marriages per capita) for the chart
y=md_data.Marriages_per_1000.values,
# The string version of the y-value that is displayed in the hover box
y_text=md_data.Marriages_per_1000.apply(
lambda x: '{}'.format(round(x, 1))),
# Extra descriptive text that is displayed in the hover box
desc=['marriages per 1,000 people'] * len(md_data),
))
source_divorces = ColumnDataSource(data=dict(
# x-axis (Years) for the chart
x=md_data.Year.values,
# y-axis (Marriages per capita) for the chart
y=md_data.Divorces_per_1000.values,
# The string version of the y-value that is displayed in the hover box
y_text=md_data.Divorces_per_1000.apply(lambda x: '{}'.format(round(x, 1))),
# Extra descriptive text that is displayed in the hover box
desc=['divorces and annulments per 1,000 people'] * len(md_data),
def compare():
# if proj among arguments, show this tree first.
try:
proj_a = int(request.args.get('proj_a', None))
proj_b = int(request.args.get('proj_b', None))
except (TypeError, ValueError):
proj_a = proj_b = None
include = request.args.get('include', None)
# list of projects (and proj_names) used to create dropdown project selector
upload_list = UserFile.query.filter_by(user_id=current_user.id).\
filter_by(run_complete=True).order_by(UserFile.file_id).all()
if len(upload_list) > 1:
# Use specified project from args or highest file_id as CURRENT PROJECT
current_proj = upload_list[-1] # override if valid proj specified
if proj_a and proj_b:
current_temp_a = [u for u in upload_list if u.file_id == proj_a]
current_temp_b = [u for u in upload_list if u.file_id == proj_b]
# if not among user's finished projects, use highest file_id
if len(current_temp_a) == 1 and len(current_temp_b) == 1:
current_proj_a = current_temp_a[0]
current_proj_b = current_temp_b[0]
else:
current_proj_a = upload_list[-2]
current_proj_b = upload_list[-1]
else:
current_proj_a = upload_list[-2]
current_proj_b = upload_list[-1]
detail_path1 = naming_rules.get_detailed_path(current_proj_a)
detail_path2 = naming_rules.get_detailed_path(current_proj_b)
js_name1 = naming_rules.get_js_name(current_proj_a)
js_name2 = naming_rules.get_js_name(current_proj_b)
xlabel = u"Effect size ({})".format(current_proj_a.get_fancy_filename())
ylabel = u"Effect size ({})".format(current_proj_b.get_fancy_filename())
# load pathways with 1+ mutation in 1+ patients,
# ignoring ones with 'cancer' etc in name
all_paths1 = load_pathway_list_from_file(detail_path1)
all_paths2 = load_pathway_list_from_file(detail_path2)
# IDs with p<0.05 and +ve effect
sig_p = OrderedDict(
[(i.path_id, i.nice_name) for i in all_paths1 if i.gene_set])
sig_pids1 = [i for i in sig_p]
sig_p2 = OrderedDict(
[(i.path_id, i.nice_name) for i in all_paths2 if i.gene_set])
sig_pids2 = [i for i in sig_p2]
sig_p.update(sig_p2) # ORDERED by proj1 effect size
# BUILD DATAFRAME WITH ALL sig PATHWAYS, proj1 object order.
pway_names = sig_p.values() # order important
columns = ['path_id', 'pname', 'ind1', 'ind2', 'e1', 'e2', 'e1_only',
'e2_only', 'q1', 'q2']
df = pd.DataFrame(index=sig_p.keys(), data={'pname': pway_names},
columns=columns)
for path_group, evar, qvar, ind, sigs in \
[(all_paths1, 'e1', 'q1', 'ind1', sig_pids1),
(all_paths2, 'e2', 'q2', 'ind2', sig_pids2)]:
for path in path_group:
path_id = path.path_id
if path_id not in sig_p:
continue
df.loc[path_id, evar] = get_effect(path)
df.loc[path_id, qvar] = get_q(path)
temp_ind = sigs.index(path_id) if path_id in sigs else -1
df.loc[path_id, ind] = temp_ind
df.ind1.fillna(-1, inplace=True)
df.ind2.fillna(-1, inplace=True)
df.e1_only = df.where(df.e2.isnull())['e1']
df.e2_only = df.where(df.e1.isnull())['e2']
inds1 = list(df.ind1)
inds2 = list(df.ind2)
source = ColumnDataSource(data=df)
source_full = ColumnDataSource(data=df)
source.name, source_full.name = 'data_visible', 'data_full'
# SET UP FIGURE
minx = df.e1.min()
minx *= 1 - minx / abs(minx) * 0.2
miny = df.e2.min()
miny *= 1 - miny/abs(miny) * 0.2
maxx = df.e1.max() * 1.2
maxy = df.e2.max() * 1.2
TOOLS = "lasso_select,box_select,hover,crosshair,pan,wheel_zoom,"\
"box_zoom,reset,tap,help" # poly_select,lasso_select, previewsave
# SUPLOTS
p = figure(plot_width=DIM_COMP_W, plot_height=DIM_COMP_H, tools=TOOLS,
title=None, logo=None, toolbar_location="above",
x_range=Range1d(minx, maxx), y_range=Range1d(miny, maxy),
x_axis_type="log", y_axis_type="log"
)
pb = figure(plot_width=DIM_COMP_SM, plot_height=DIM_COMP_H, tools=TOOLS,
y_range=p.y_range, x_axis_type="log", y_axis_type="log")
pa = figure(plot_width=DIM_COMP_W, plot_height=DIM_COMP_SM, tools=TOOLS,
x_range=p.x_range, x_axis_type="log", y_axis_type="log")
pp = figure(plot_width=DIM_COMP_SM, plot_height=DIM_COMP_SM,
tools=TOOLS, outline_line_color=None)
# SPANS
p.add_layout(plot_fns.get_span(1, 'height'))
p.add_layout(plot_fns.get_span(1, 'width'))
pa.add_layout(plot_fns.get_span(1, 'height'))
pb.add_layout(plot_fns.get_span(1, 'width'))
# STYLE
for ax in [p, pa, pb]:
ax.grid.visible = False
ax.outline_line_width = 2
ax.background_fill_color = 'whitesmoke'
for ax in [pa, pb]:
ax.xaxis.visible = False
ax.yaxis.visible = False
pa.title.text = xlabel
pb.title.text = ylabel
pa.title_location, pa.title.align = 'below', 'center'
pb.title_location, pb.title.align = 'left', 'center'
# WIDGETS
q_input = TextInput(value='', title="P* cutoff",
placeholder='e.g. 0.05')
gene_input = TextInput(value='', title="Gene list",
placeholder='e.g. TP53,BRAF')
radio_include = RadioGroup(labels=["Include", "Exclude"], active=0)
widgets = widgetbox(q_input, gene_input, radio_include, width=200,
css_classes=['widgets_sg'])
grid = gridplot([[pb, p, widgets],
[Spacer(width=DIM_COMP_SM), pa, Spacer()]],
sizing_mode='fixed')
cb_inclusion = CustomJS(args=dict(genes=gene_input), code="""
var gene_str = genes.value
if (!gene_str)
return;
var include = cb_obj.active == 0 ? true : false
selectPathwaysByGenes(gene_str, include);
""")
cb_genes = CustomJS(args=dict(radio=radio_include), code="""
var gene_str = cb_obj.value
if (!gene_str)
return;
var include = radio.active == 0 ? true : false
selectPathwaysByGenes(gene_str, include);
""")
radio_include.js_on_change('active', cb_inclusion)
gene_input.js_on_change('value', cb_genes)
# SCATTER
p.circle("e1", "e2", source=source, **SCATTER_KW)
pa.circle('e1_only', 1, source=source, **SCATTER_KW)
pb.circle(1, 'e2_only', source=source, **SCATTER_KW)
# HOVER
for hover in grid.select(dict(type=HoverTool)):
hover.tooltips = OrderedDict([
("name", "@pname"),
("effects", "(@e1, @e2)"),
("P*", ("(@q1, @q2)"))
])
# ADD Q FILTERING CALLBACK
callback = CustomJS(args=dict(source=source, full=source_full), code="""
// get old selection indices, if any
var prv_selected = source.selected['1d'].indices;
var prv_select_full = []
for(var i=0; i<prv_selected.length; i++){
prv_select_full.push(scatter_array[prv_selected[i]])
}
var new_selected = []
var q_val = cb_obj.value;
if(q_val == '')
q_val = 1
var fullset = full.data;
var n_total = fullset['e1'].length;
// Convert float64arrays to array
var col_names = %s ;
col_names.forEach(function(col_name){
source.data[col_name] = [].slice.call(source.data[col_name])
source.data[col_name].length = 0
})
scatter_array.length = 0;
var j = -1; // new glyph indices
for (i = 0; i < n_total; i++) {
this_q1 = fullset['q1'][i];
this_q2 = fullset['q2'][i];
if(this_q1 <= q_val || this_q2 <= q_val){
j++; // preserve previous selection if still visible
col_names.forEach(function(col){
source.data[col].push(fullset[col][i]);
})
scatter_array.push(i)
if($.inArray(i, prv_select_full) > -1){
new_selected.push(j);
}
}
}
source.selected['1d'].indices = new_selected;
source.trigger('change');
updateIfSelectionChange_afterWait();
""" % columns)
q_input.js_on_change('value', callback)
script, div = plot_fns.get_bokeh_components(grid)
proj_dir_a = naming_rules.get_project_folder(current_proj_a)
proj_dir_b = naming_rules.get_project_folder(current_proj_b)
if os.path.exists(os.path.join(proj_dir_a, 'matrix_svg_cnv')) and \
os.path.exists(os.path.join(proj_dir_b, 'matrix_svg_cnv')):
has_cnv = True
else:
has_cnv = False
else: # not enough projects yet!
flash("Two completed projects are required for a comparison.",
"warning")
return redirect(url_for('.index'))
return render_template('pway/compare.html',
current_projs=[current_proj_a, current_proj_b],
inds_use=[inds1, inds2],
has_cnv=has_cnv,
js_name_a=js_name1,
js_name_b=js_name2,
projects=upload_list,
bokeh_script=script,
bokeh_div=div, include_genes=include,
resources=plot_fns.resources)
import numpy as np
from bokeh.io import vform
from bokeh.models import CustomJS, Slider
from bokeh.plotting import figure, hplot, output_file, show, ColumnDataSource
x = np.linspace(0, 10, 500)
y = np.sin(x)
source = ColumnDataSource(data=dict(x=x, y=y))
plot = figure(y_range=(-10, 10), plot_width=400, plot_height=400)
plot.line('x', 'y', source=source, line_width=3, line_alpha=0.6)
callback = CustomJS(args=dict(source=source),
code="""
var data = source.get('data');
var A = amp.get('value')
var k = freq.get('value')
var phi = phase.get('value')
var B = offset.get('value')
x = data['x']
y = data['y']
for (i = 0; i < x.length; i++) {
y[i] = B + A*Math.sin(k*x[i]+phi);
}
source.trigger('change');
""")
amp_slider = Slider(start=0.1,
def dashboard():
"""
Creates the main app objects
"""
# TODO: Should find a better design and remove those global objs...
global source
global csource
source = ColumnDataSource(df)
source.tags = ['main_source']
csource = ColumnDataSource(cdf)
csource.tags = ['crash_source']
select_sec = Select.create(options=SECURITIES, name='select_sec', title="")
asel = Select.create(options=df.columns, name='field_to_add', title="")
chart_sel = Select.create(options=chart_types, name='chart_to_add', title="")
msel = MultiSelect.create(options=[], name='ms')
abutton = Button(label="add", type="success", name='add_button')
rbutton = Button(label="remove", type="danger", name='remove_button')
add_chart_button = Button(label="add chart", type="success", name='add_chart')
data_table = DataTable(source=source, editable=True, width=500, height=400)
ccolumns = [TableColumn(field=x, title=x, editor=NumberEditor()) for x in cdf.columns]
crashes_table = DataTable(source=csource, editable=True, width=1200, height=400, columns=ccolumns)
charts_box = AppVBox(children=[], name='charts_box')
counts = cdf['crashes']
crashes_hist = charts.Histogram({'crash_counts': counts}, bins=20, height=300, title="# Crashes per Symbol")
crashes_scatter = create_crashes_scatter()
main_tab = VBox(children=[], name='d_box')
dlg = Dialog(buttons=[], content=main_tab , visible=False)
options = CheckboxGroup(
labels=['Show peaks', 'Show Std. Dev.', 'Bolinger Bands'],
name='new_chart_options'
)
select_palette = Select.create(options=brewer.keys(), name='select_palette', title="Palette")
return {
# security tab
'select_sec': select_sec,
'field_to_add': asel,
'chart_to_add': chart_sel,
'ms': msel,
'dt': data_table,
'add_button': abutton,
'remove_button': rbutton,
'add_chart': add_chart_button,
'charts_box': charts_box,
'dlg': dlg,
# Crashes tab objs
'crashes_hist': crashes_hist,
'crashes_scatter': crashes_scatter,
'crashes_table': crashes_table,
'crash_stats': PreText(text="NO CRASHES FOR SYMBOL %s" % SYMBOL, width=300),
#
'chart_values': MultiSelect.create(options=df.columns, name="chart_values", title="Values"),
'new_chart_index': Select.create(options=['---']+list(df.columns), title="Chart index", name="new_chart_index"),
'new_chart_title': TextInput(title="Chart Title", name='new_chart_title', value=''),
'new_chart_options': options,
'select_palette': select_palette,
# ANNOTATIONS
"annotations": MultiSelect.create(options=[], name="annotations", title="Annotations"),
"add_annotation": Button(label="add", type="success", name='add_annotation'),
"delete_annotation": Button(label="remove", type="danger", name='delete_annotation'),
"show_annotation": Button(label="Show", type="link", name='show_annotation'),
'new_annotation_dlg': Dialog(buttons=[], content=main_tab , visible=False),
'annotation_txt': TextInput(title="New Annotation", name='annotation_txt', value=''),
'new_annotation_options': CheckboxGroup(labels=[], name='new_annotation_options'),
}
'cause'] = 'Judy Garland stars as Dorothy in the Wizard of Oz movie (1939)'
data.loc[
data.Name == 'Whitney',
'cause'] = 'The singer Whitney Houston was No. 1 artist of the year and her album was the No. 1 album of the year on the 1986 Billboard year-end charts.'
data.loc[
data.Name == 'Ashanti',
'cause'] = 'In April 2002 the singer Ashanti released her eponymous debut album, which featured the hit song "Foolish", and sold over 503,000 copies in its first week of release throughout the U.S.'
data.loc[data.Name == 'Woodrow',
'cause'] = 'Woodrow Wilson ran for president of the USA in 1912.'
data.loc[data.Name == 'Jacqueline',
'cause'] = 'Jacqueline Kennedy becomes First Lady'
data.loc[data.cause != '', 'text_color'] = data['color']
# In[ ]:
source_noexpl = ColumnDataSource(data=data.loc[data.cause == ''])
source_expl = ColumnDataSource(data=data.loc[data.cause != ''])
hover = HoverTool(tooltips="""
<div>
<div>
<span style="font-size: 17px; font-weight: bold;">@Name</span>
<span style="font-size: 15px; color: #966;">@BestYearPopDiff</span>
</div>
<div style='max-width: 300px'>
<span style="font-size: 15px;">@cause</span>
</div>
</div>
""")
p = figure(plot_width=800,
from bokeh.plotting import ColumnDataSource, figure, gridplot, output_file, show
from bokeh.sampledata.autompg import autompg
output_file("brushing.html")
# Load some Automobile data into a data source. Interesting columns are:
# "yr" - Year manufactured
# "mpg" - miles per gallon
# "displ" - engine displacement
# "hp" - engine horsepower
# "cyl" - number of cylinders
source = ColumnDataSource(autompg.to_dict("list"))
source.add(autompg["yr"], name="yr")
# define some tools to add
TOOLS = "pan,wheel_zoom,box_zoom,box_select,lasso_select"# Let's set up some plot options in a dict that we can re-use on multiple plots
# Let's set up some plot options in a dict that we can re-use on multiple plots
plot_config = dict(plot_width=300, plot_height=300, tools=TOOLS)
# First let's plot the "yr" vs "mpg" using the plot config above
# Note that we are supplying our our data source to the renderer explicitly
p1 = figure(title="MPG by Year", **plot_config)
p1.circle("yr", "mpg", color="blue", source=source)
# EXERCISE: make another figure p2 with circle renderer, for "hp" vs "displ" with
# color "green". This renderer should use the same data source as the renderer
# above, that is what will cause the plots selections to be linked
p2 = figure(title="HP vs. Displacement", **plot_config)
p2.circle("hp", "displ", color="green", source=source)
def _heatmap_summary(pvals, coefs, plot_width=1200, plot_height=400):
""" Plots heatmap of coefficients colored by pvalues
Parameters
----------
pvals : pd.DataFrame
Table of pvalues where rows are balances and columns are
covariates.
coefs : pd.DataFrame
Table of coefficients where rows are balances and columns are
covariates.
plot_width : int, optional
Width of plot.
plot_height : int, optional
Height of plot.
Returns
-------
bokeh.charts.Heatmap
Heatmap summarizing the regression statistics.
"""
c = coefs.reset_index()
c = c.rename(columns={'index': 'balance'})
# fix alpha in fdr to account for the number of covariates
def fdr(x):
return multipletests(x, method='fdr_bh',
alpha=0.05 / pvals.shape[1])[1]
cpvals = pvals.apply(fdr, axis=0)
# log scale for coloring
log_p = -np.log10(cpvals+1e-200)
log_p = log_p.reset_index()
log_p = log_p.rename(columns={'index': 'balance'})
p = pvals.reset_index()
p = p.rename(columns={'index': 'balance'})
cp = cpvals.reset_index()
cp = cp.rename(columns={'index': 'balance'})
cm = pd.melt(c, id_vars='balance', var_name='Covariate',
value_name='Coefficient')
pm = pd.melt(p, id_vars='balance', var_name='Covariate',
value_name='Pvalue')
cpm = pd.melt(cp, id_vars='balance', var_name='Covariate',
value_name='Corrected_Pvalue')
logpm = pd.melt(log_p, id_vars='balance', var_name='Covariate',
value_name='log_Pvalue')
m = pd.merge(cm, pm,
left_on=['balance', 'Covariate'],
right_on=['balance', 'Covariate'])
m = pd.merge(m, logpm,
left_on=['balance', 'Covariate'],
right_on=['balance', 'Covariate'])
m = pd.merge(m, cpm,
left_on=['balance', 'Covariate'],
right_on=['balance', 'Covariate'])
hover = HoverTool(
tooltips=[("Pvalue", "@Pvalue"),
("Corrected Pvalue", "@Corrected_Pvalue"),
("Coefficient", "@Coefficient")]
)
N, _min, _max = len(palette), m.log_Pvalue.min(), m.log_Pvalue.max()
X = pd.Series(np.arange(len(pvals.index)), index=pvals.index)
Y = pd.Series(np.arange(len(pvals.columns)), index=pvals.columns)
m['X'] = [X.loc[i] for i in m.balance]
m['Y'] = [Y.loc[i] for i in m.Covariate]
# fill in nans with zero. Sometimes the pvalue calculation fails.
m = m.fillna(0)
for i in m.index:
x = m.loc[i, 'log_Pvalue']
ind = int(np.floor((x - _min) / (_max - _min) * (N - 1)))
m.loc[i, 'color'] = palette[ind]
source = ColumnDataSource(ColumnDataSource.from_df(m))
hm = figure(title='Regression Coefficients Summary',
plot_width=1200, plot_height=400,
tools=[hover, PanTool(), BoxZoomTool(),
WheelZoomTool(), ResetTool(),
SaveTool()])
hm.rect(x='X', y='Y', width=1, height=1,
fill_color='color', line_color="white", source=source)
Xlabels = pd.Series(pvals.index, index=np.arange(len(pvals.index)))
Ylabels = pd.Series(pvals.columns, index=np.arange(len(pvals.columns)), )
hm.xaxis[0].ticker = FixedTicker(ticks=Xlabels.index)
hm.xaxis.formatter = FuncTickFormatter(code="""
var labels = %s;
return labels[tick];
""" % Xlabels.to_dict())
hm.yaxis[0].ticker = FixedTicker(ticks=Ylabels.index)
hm.yaxis.formatter = FuncTickFormatter(code="""
var labels = %s;
return labels[tick];
""" % Ylabels.to_dict())
return hm
