def plotting(self):
#Tools = [hover, TapTool(), BoxZoomTool(), BoxSelectTool(), PreviewSaveTool(), ResetTool()]
TOOLS = "crosshair,pan,wheel_zoom,box_zoom,reset,hover,previewsave"
tab_plots = []
#output_file("test.html")
self.all_elements = []
self.elements_comparison = []
for attr_id, i in zip(self.attribute_ids,
range(len(self.attribute_ids))):
"""
create plots for each datafile and put them in a tab.
"""
list_of_datasets = getattr(self, attr_id)
y_axis_units = [x["y_unit"] for x in list_of_datasets]
x_axis_units = [x["x_unit"] for x in list_of_datasets]
figure_obj = figure(plot_width=1000,
plot_height=800,
y_axis_type="log",
title=attr_id,
tools=TOOLS)
#figure_obj.axes.major_label_text_font_size("12pt")
#figure_obj.major_label_text_font_size("12pt")
setattr(self, attr_id + "_" + "figure_obj", figure_obj)
figure_obj.yaxis.axis_label = y_axis_units[0]
figure_obj.xaxis.axis_label = x_axis_units[0]
if not all(x == y_axis_units[0] for x in y_axis_units):
for unit, data in zip(y_axis_units, list_of_datasets):
if not unit == y_axis_units[0]:
figure_obj.extra_y_ranges = {
"foo":
Range1d(start=np.amin(data["data"]["y"]),
end=np.amax(data["data"]["y"]))
}
figure_obj.add_layout(
LogAxis(y_range_name="foo", axis_label=unit),
"right")
break
if not all(x == x_axis_units[0] for x in x_axis_units):
for unit, data in zip(x_axis_units, list_of_datasets):
if not unit == x_axis_units[0]:
figure_obj.extra_x_ranges = {
"bar":
Range1d(start=np.amin(data["data"]["x"]),
end=np.amax(data["data"]["x"]))
}
figure_obj.add_layout(
LinearAxis(x_range_name="bar", axis_label=unit),
"above")
break
figure_obj.xaxis.axis_label = list_of_datasets[0]["x_unit"]
colour_list = Spectral11 + RdPu9 + Oranges9
colour_indices = [0, 2, 8, 10, 12, 14, 20, 22, 1, 3, 9, 11, 13, 15]
list_of_elements = []
for dataset, color_index in zip(list_of_datasets, colour_indices):
self.all_elements.append(
dataset["sample element"]) #strip isotope number
color = colour_list[color_index]
source = ColumnDataSource(
data=dataset["data"]
) #Datastructure for source of plotting
setattr(self,
attr_id + "_" + dataset["sample element"] + "_source",
source) #Source element generalized for all plotting
list_of_elements.append(dataset["sample element"])
figure_obj.line(
"x",
"y",
source=getattr(
self,
attr_id + "_" + dataset["sample element"] + "_source"),
line_width=2,
line_color=color,
legend=dataset["sample element"],
name=dataset["sample element"],
)
hover = figure_obj.select_one(HoverTool).tooltips = [
("element", "@element"), ("(x,y)", "($x, $y)")
]
radio_group = RadioGroup(labels=list_of_elements, active=0)
"""
Need to fetch default variables from input file and replace DEFAULT
Block of code produces the layout of buttons and callbacks
"""
text_input_rsf = TextInput(value="default",
title="RSF (at/cm^3): ")
text_input_sputter = TextInput(value="default",
title="Sputter speed: float unit")
text_input_crater_depth = TextInput(
value="default", title="Depth of crater in: float")
radio_group.on_change("active", lambda attr, old, new: None)
text_input_xval_integral = TextInput(
value="0", title="x-value for calibration integral ")
text_input_yval_integral = TextInput(
value="0", title="y-value for calibration integral ")
do_integral_button = Button(label="Calibration Integral")
save_flexDPE_button = Button(label="Save element for FlexPDE")
save_all_flexDPE_button = Button(
label="Save all elements for FlexPDE")
do_integral_button.on_click(
lambda identity=self.attribute_ids[i], radio=radio_group, x_box
=text_input_xval_integral, y_box=text_input_yval_integral: self
.integrate(identity, radio, x_box, y_box))
save_flexDPE_button.on_click(lambda identity=self.attribute_ids[
i], radio=radio_group: self.write_to_flexPDE(identity, radio))
save_all_flexDPE_button.on_click(
lambda identity=self.attribute_ids[i], radio=radio_group: self.
write_all_to_flexPDE(identity, radio))
text_input_rsf.on_change(
"value",
lambda attr, old, new, radio=radio_group, identity=self.
attribute_ids[i], text_input=text_input_rsf, which="rsf": self.
update_data(identity, radio, text_input, new, which))
text_input_sputter.on_change(
"value",
lambda attr, old, new, radio=radio_group, identity=self.
attribute_ids[i], text_input=
text_input_sputter, which="sputter": self.update_data(
identity, radio, text_input, new, which))
text_input_crater_depth.on_change(
"value",
lambda attr, old, new, radio=radio_group, identity=self.
attribute_ids[i], text_input=text_input_crater_depth, which=
"crater_depth": self.update_data(identity, radio, text_input,
new, which))
tab_plots.append(
Panel(child=hplot(
figure_obj, vform(radio_group, save_flexDPE_button),
vform(text_input_rsf, text_input_sputter,
text_input_crater_depth),
vform(text_input_xval_integral, text_input_yval_integral,
do_integral_button)),
title=attr_id))
"""
Check to see if one or more element exists in the samples and creat a comparison plot for each
of those elements.
"""
for element in self.all_elements:
checkers = list(self.all_elements)
checkers.remove(element)
if element in checkers and not element in self.elements_comparison:
self.elements_comparison.append(element)
"""create plots for each element that is to be compared """
for comparison_element in self.elements_comparison:
colour_list = Spectral11 + RdPu9 + Oranges9
colour_indices = [0, 2, 8, 10, 12, 14, 20, 22, 1, 3, 9, 11, 13, 15]
figure_obj = figure(plot_width=1000,
plot_height=800,
y_axis_type="log",
title=comparison_element,
tools=TOOLS)
#figure_obj.xaxis.major_label_text_font_size("12pt")
#figure_obj.yaxis.major_label_text_font_size("12pt")
y_axis_units = []
x_axis_units = []
comparison_datasets = []
for attr_id, color_index in zip(self.attribute_ids,
colour_indices):
list_of_datasets = getattr(self, attr_id)
for dataset in list_of_datasets:
if dataset["sample element"] == comparison_element:
comparison_datasets.append(dataset)
y_axis_units.append(dataset["y_unit"])
x_axis_units.append(dataset["x_unit"])
figure_obj.xaxis.axis_label = comparison_datasets[-1]["x_unit"]
figure_obj.yaxis.axis_label = comparison_datasets[-1]["y_unit"]
if not all(x == y_axis_units[-1] for x in y_axis_units):
for unit, data in zip(y_axis_units, comparison_datasets):
if not unit == y_axis_units[-1]:
figure_obj.extra_y_ranges = {
"foo":
Range1d(start=np.amin(data["data"]["y"]),
end=np.amax(data["data"]["y"]))
}
figure_obj.add_layout(
LogAxis(y_range_name="foo", axis_label=unit),
"right")
break
if not all(x == x_axis_units[-1] for x in x_axis_units):
for unit, data in zip(x_axis_units, comparison_datasets):
if not unit == x_axis_units[-1]:
figure_obj.extra_x_ranges = {
"bar":
Range1d(start=np.amin(data["data"]["x"]),
end=np.amax(data["data"]["x"]))
}
figure_obj.add_layout(
LinearAxis(x_range_name="bar", axis_label=unit),
"above")
break
for attr_id, color_index in zip(self.attribute_ids,
colour_indices):
list_of_datasets = getattr(self, attr_id)
for dataset in list_of_datasets:
if dataset["sample element"] == comparison_element:
color = colour_list[color_index]
"""
Logic that ensures that plots get put with correspoinding axes.
"""
if dataset["x_unit"] != x_axis_units[-1] or dataset[
"y_unit"] != y_axis_units[-1]:
if dataset["x_unit"] != x_axis_units[
-1] and dataset["y_unit"] != y_axis_units[
-1]:
figure_obj.line(
"x",
"y",
source=getattr(
self, attr_id + "_" +
dataset["sample element"] + "_source"),
line_width=2,
line_color=color,
legend=attr_id,
x_range_name="bar",
y_range_name="foo")
elif dataset["x_unit"] != x_axis_units[-1]:
figure_obj.line(
"x",
"y",
source=getattr(
self, attr_id + "_" +
dataset["sample element"] + "_source"),
line_width=2,
line_color=color,
legend=attr_id,
x_range_name="bar")
else:
figure_obj.line(
"x",
"y",
source=getattr(
self, attr_id + "_" +
dataset["sample element"] + "_source"),
line_width=2,
line_color=color,
legend=attr_id,
y_range_name="foo")
else:
figure_obj.line(
"x",
"y",
source=getattr(
self, attr_id + "_" +
dataset["sample element"] + "_source"),
line_width=2,
line_color=color,
legend=attr_id)
tab_plots.append(Panel(child=figure_obj, title=comparison_element))
tabs = Tabs(tabs=tab_plots)
session = push_session(curdoc())
session.show()
session.loop_until_closed()
def plot_news_sentiment(log_path, base_path, base_name):
'''
Plot news sentiment analysis of Chronicling America dataset and contrast
with another dataset. This creates two plots: one of the volume of news
mentions in the data against a base comparison, and another of the share of
negative polarity in news mentions against the base comparison.
log_path (str): path to log CSV file generated by chronicling_america.py.
base_path (str): path to base comparison CSV data.
base_name (str): name of base comparison data to label plots.
NB: base comparison data must have two columns: "year" and "count".
Create two Bokeh plots as HTML files.
'''
# Load and prepare data for Bokeh.
log = pd.read_csv(log_path)
news_sentiment = prepare_news_sentiment(log)
news_volume = prepare_news_volume(log, news_sentiment)
news_y_min, news_y_max = calculate_axis_range(news_volume["w_volume"])
negative_polarity = prepare_negative_polarity(news_sentiment)
base_comparison = pd.read_csv(base_path)
base_y_min, base_y_max = calculate_axis_range(base_comparison["count"])
# Plot news mentions with base comparison.
figure_a = figure(toolbar_location=None,
tools="",
plot_height=600,
plot_width=1000,
x_axis_label="Year",
y_axis_label="News Mentions",
y_range=(Range1d(start=news_y_min, end=news_y_max)))
figure_a.title.text = "News Mentions & " + base_name
figure_a.title.text_font_size = "40px"
figure_a.title.align = "center"
figure_a.line(legend="News Mentions of " + base_name,
x=news_volume["year"],
y=news_volume["w_volume"],
line_width=2,
color="red",
alpha=1)
figure_a.circle(x=news_volume["year"],
y=news_volume["w_volume"],
size=6,
line_color="red",
fill_color="white",
alpha=1)
figure_a.extra_y_ranges = {
"base_comparison": Range1d(start=base_y_min, end=base_y_max)
}
figure_a.line(legend=base_name,
x=base_comparison["year"],
y=base_comparison["count"],
y_range_name="base_comparison",
line_width=2,
color="navy",
alpha=1)
figure_a.circle(x=base_comparison["year"],
y=base_comparison["count"],
y_range_name="base_comparison",
size=6,
line_color="navy",
fill_color="white",
alpha=1)
figure_a.add_layout(LinearAxis(y_range_name="base_comparison", \
axis_label=base_name), "right")
with open("news_mentions.html", "w") as figure_a_html:
figure_a_html.write(file_html(figure_a, CDN, "News Mentions"))
# Plot negative polarity as a share of news mentions with base comparison.
figure_b = figure(
toolbar_location=None,
tools="",
plot_height=600,
plot_width=1000,
x_axis_label="Year",
y_axis_label="Share of News Mentions with Negative Polarity",
y_range=(Range1d(start=0, end=1)))
figure_b.title.text = "News Polarity & " + base_name
figure_b.title.text_font_size = "40px"
figure_b.title.align = "center"
figure_b.line(legend="Share of Negative Polarity in News Mentions of " +
base_name,
x=negative_polarity["year"],
y=negative_polarity["polarity"],
line_width=2,
color="black",
alpha=1)
figure_b.circle(x=negative_polarity["year"],
y=negative_polarity["polarity"],
size=6,
line_color="black",
fill_color="white",
alpha=1)
figure_b.extra_y_ranges = {
"base_comparison": Range1d(start=base_y_min, end=base_y_max)
}
figure_b.line(legend=base_name,
x=base_comparison["year"],
y=base_comparison["count"],
y_range_name="base_comparison",
line_width=2,
color="red",
alpha=1)
figure_b.circle(x=base_comparison["year"],
y=base_comparison["count"],
y_range_name="base_comparison",
size=6,
line_color="red",
fill_color="white",
alpha=1)
figure_b.add_layout(LinearAxis(y_range_name="base_comparison", \
axis_label=base_name), "right")
with open("negative_polarity.html", "w") as figure_b_html:
figure_b_html.write(file_html(figure_b, CDN, "Negative Polarity"))
def comparison(request, stock_id1, stock_id2):
stock1 = get_object_or_404(Stock, pk=stock_id1)
stock2 = get_object_or_404(Stock, pk=stock_id2)
update(stock1)
update(stock2)
# Graph
# Last known weekday
current_day = weekday().isoformat()
# Retrieve live data YYYY-MM-DD
historical_price1 = ystockquote.get_historical_prices(
stock1, '2013-01-24', current_day)
correct_order1 = sorted(historical_price1)
stock_prices1 = []
dates1 = []
for values in correct_order1:
stock_prices1.append(historical_price1[values]['Adj Close'])
dates1.append(values)
# Convert to Float
for p in range(len(stock_prices1)):
stock_prices1[p] = float(stock_prices1[p])
# Convert to Datetime Format
dates_objects1 = []
for d in dates1:
dates_objects1.append(datetime.strptime(d, '%Y-%m-%d'))
# Retrieve live data YYYY-MM-DD
historical_price2 = ystockquote.get_historical_prices(
stock2, '2013-01-24', current_day)
correct_order2 = sorted(historical_price2)
stock_prices2 = []
dates2 = []
for values in correct_order2:
stock_prices2.append(historical_price2[values]['Adj Close'])
dates2.append(values)
# Convert to Float
for p in range(len(stock_prices2)):
stock_prices2[p] = float(stock_prices2[p])
# Convert to Datetime Format
dates_objects2 = []
for d in dates2:
dates_objects2.append(datetime.strptime(d, '%Y-%m-%d'))
p1 = figure(x_axis_type="datetime",
responsive=True,
plot_height=250,
toolbar_location=None)
# Multiple Axises
min1 = min(stock_prices1)
max1 = max(stock_prices1)
min2 = min(stock_prices2)
max2 = max(stock_prices2)
p1.y_range = Range1d(start=min1 - (min1 / 10), end=max1 + (min1 / 10))
p1.extra_y_ranges = {
'range2': Range1d(start=min2 - (min2 / 10), end=max2 + (min2 / 10))
}
p1.add_layout(LinearAxis(y_range_name="range2"), 'right')
p1.line(np.array(dates_objects1, dtype=np.datetime64),
stock_prices1,
color='#b41f2e',
legend=stock1.ticker)
p1.line(np.array(dates_objects2, dtype=np.datetime64),
stock_prices2,
y_range_name='range2',
color='#1F78B4',
legend=stock2.ticker)
p1.grid.grid_line_alpha = 0.3
p1.xaxis.axis_label = 'Date'
p1.yaxis.axis_label = 'Price'
script, div = components(p1)
context = {
'stock1': stock1,
'stock2': stock2,
'the_script': script,
'the_div': div,
}
return render(request, 'stocktracker/comparison.html', context)
def __init__(self, **kwargs):
self.source = ColumnDataSource(
data={
'time': [],
'cpu': [],
'memory_percent': [],
'network-send': [],
'network-recv': []
})
x_range = DataRange1d(follow='end',
follow_interval=30000,
range_padding=0)
resource_plot = Plot(x_range=x_range,
y_range=Range1d(start=0, end=1),
toolbar_location=None,
min_border_bottom=10,
**kwargs)
line_opts = dict(line_width=2, line_alpha=0.8)
g1 = resource_plot.add_glyph(
self.source,
Line(x='time',
y='memory_percent',
line_color="#33a02c",
**line_opts))
g2 = resource_plot.add_glyph(
self.source,
Line(x='time', y='cpu', line_color="#1f78b4", **line_opts))
resource_plot.add_layout(
LinearAxis(formatter=NumeralTickFormatter(format="0 %")), 'left')
legend_opts = dict(location='top_left',
orientation='horizontal',
padding=5,
margin=5,
label_height=5)
resource_plot.add_layout(
Legend(items=[('Memory', [g1]), ('CPU', [g2])], **legend_opts))
network_plot = Plot(x_range=x_range,
y_range=DataRange1d(start=0),
toolbar_location=None,
**kwargs)
g1 = network_plot.add_glyph(
self.source,
Line(x='time', y='network-send', line_color="#a6cee3",
**line_opts))
g2 = network_plot.add_glyph(
self.source,
Line(x='time', y='network-recv', line_color="#b2df8a",
**line_opts))
network_plot.add_layout(DatetimeAxis(axis_label="Time"), "below")
network_plot.add_layout(LinearAxis(axis_label="MB/s"), 'left')
network_plot.add_layout(
Legend(items=[('Network Send', [g1]), ('Network Recv', [g2])],
**legend_opts))
tools = [
PanTool(dimensions='width'),
WheelZoomTool(dimensions='width'),
BoxZoomTool(),
ResetTool()
]
if 'sizing_mode' in kwargs:
sizing_mode = {'sizing_mode': kwargs['sizing_mode']}
else:
sizing_mode = {}
combo_toolbar = ToolbarBox(tools=tools,
logo=None,
toolbar_location='right',
**sizing_mode)
self.root = row(column(resource_plot, network_plot, **sizing_mode),
column(combo_toolbar, **sizing_mode),
id='bk-resource-profiles-plot',
**sizing_mode)
# Required for update callback
self.resource_index = [0]
minor_tick_out=None,
major_tick_in=None,
major_label_text_font_size="10pt",
major_label_text_font_style="normal",
axis_label_text_font_size="10pt",
axis_line_color='#AAAAAA',
major_tick_line_color='#AAAAAA',
major_label_text_color='#666666',
major_tick_line_cap="round",
axis_line_cap="round",
axis_line_width=1,
major_tick_line_width=1,
)
xaxis = LinearAxis(ticker=SingleIntervalTicker(interval=1),
axis_label="Children per woman (total fertility)",
**AXIS_FORMATS)
yaxis = LinearAxis(ticker=SingleIntervalTicker(interval=20),
axis_label="Life expectancy at birth (years)",
**AXIS_FORMATS)
plot.add_layout(xaxis, 'below')
plot.add_layout(yaxis, 'left')
# ### Add the background year text
# We add this first so it is below all the other glyphs
text_source = ColumnDataSource({'year': ['%s' % years[0]]})
text = Text(x=2,
y=35,
text='year',
text_font_size='150pt',
text_color='#EEEEEE')
def construct_total_pnl_figure(self, x, y, t):
str_total_pnl = "Total Pnl " + POUND_SYMBOL
# workaround to format date in the hover tool at the moment bokeh does not supported in the tool tips
time = [e.strftime('%d %b %Y') for e in x]
source_total_pnl = ColumnDataSource(data=dict(x=x, y=y, time=time))
tooltips_total_pnl = [
("Date", "@time"),
("Total Pnl", "@y{0.00}"),
]
tooltips_capital = [
("Date", "@time"),
("Capital", "@y{0.00}"),
]
# create a new pnl plot
p2 = Figure(x_axis_type="datetime", title="Total Pnl/Capital Allocated " + POUND_SYMBOL,
toolbar_location="above", tools=['box_zoom, box_select, crosshair, resize, reset, save, wheel_zoom'])
# add renderers
r1 = p2.circle(x, y, size=8, color='black', alpha=0.2, legend=str_total_pnl, source=source_total_pnl)
r11 = p2.line(x, y, color='navy', legend=str_total_pnl, source=source_total_pnl)
# add renderers to the HoverTool instead of to the figure so we can have different tooltips for each glyph
p2.add_tools(HoverTool(renderers=[r1, r11], tooltips=tooltips_total_pnl))
max_total_pnl = max(y)
min_total_pnl = min(y)
# offset to adjust the plot so the max and min ranges are visible
offset = (max(abs(max_total_pnl), abs(min_total_pnl))) * 0.10
p2.y_range = Range1d(min_total_pnl - offset, max_total_pnl + offset)
# NEW: customize by setting attributes
# p2.title = "Total Pnl/Capital Allocated " + POUND_SYMBOL
p2.legend.location = "top_left"
p2.grid.grid_line_alpha = 0
p2.xaxis.axis_label = 'Date'
p2.yaxis.axis_label = str_total_pnl
p2.ygrid.band_fill_color = "olive"
p2.ygrid.band_fill_alpha = 0.1
p2.xaxis.formatter = DatetimeTickFormatter(formats={'days': ['%d %b'], 'months': ['%b %Y']})
# formatter without exponential notation
p2.yaxis.formatter = PrintfTickFormatter(format="%.0f")
# secondary axis
max_capital = max(t)
min_capital = min(t)
# offset to adjust the plot so the max and min ranges are visible
offset = (max(abs(max_capital), abs(min_capital))) * 0.10
p2.extra_y_ranges = {"capital": Range1d(start=min_capital - offset, end=max_capital + offset)}
# formatter without exponential notation
formatter = PrintfTickFormatter()
formatter.format = "%.0f"
# formatter=NumeralTickFormatter(format="0,0"))
p2.add_layout(LinearAxis(y_range_name="capital", axis_label="Capital allocated " + POUND_SYMBOL,
formatter=formatter), 'right')
# create plot for capital series
source_capital = ColumnDataSource(data=dict(x=x, t=t, time=time))
r2 = p2.square(x, t, size=8, color='green', alpha=0.2, legend="Capital " + POUND_SYMBOL, y_range_name="capital",
source=source_capital)
r22 = p2.line(x, t, color='green', legend="Capital " + POUND_SYMBOL, y_range_name="capital", source=source_capital)
# add renderers to the HoverTool instead of to the figure so we can have different tooltips for each glyph
p2.add_tools(HoverTool(renderers=[r2, r22], tooltips=tooltips_capital))
return p2
def calculate_proxy_svg(snp, pop, request, r2_d="r2"):
# Set data directories using config.yml
with open('config.yml', 'r') as f:
config = yaml.load(f)
vcf_dir = config['data']['vcf_dir']
tmp_dir = "./tmp/"
# Ensure tmp directory exists
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
if request is False:
request = str(time.strftime("%I%M%S"))
# Create JSON output
# Find coordinates (GRCh37/hg19) for SNP RS number
# Connect to Mongo snp database
client = MongoClient(
'mongodb://' + username + ':' + password + '@localhost/admin', port)
db = client["LDLink"]
def get_coords(db, rsid):
rsid = rsid.strip("rs")
query_results = db.dbsnp151.find_one({"id": rsid})
query_results_sanitized = json.loads(json_util.dumps(query_results))
return query_results_sanitized
# Query genomic coordinates
def get_rsnum(db, coord):
temp_coord = coord.strip("chr").split(":")
chro = temp_coord[0]
pos = temp_coord[1]
query_results = db.dbsnp151.find({
"chromosome":
chro.upper() if chro == 'x' or chro == 'y' else chro,
"position":
pos
})
query_results_sanitized = json.loads(json_util.dumps(query_results))
return query_results_sanitized
# Replace input genomic coordinates with variant ids (rsids)
def replace_coord_rsid(db, snp):
if snp[0:2] == "rs":
return snp
else:
snp_info_lst = get_rsnum(db, snp)
print("snp_info_lst")
print(snp_info_lst)
if snp_info_lst != None:
if len(snp_info_lst) > 1:
var_id = "rs" + snp_info_lst[0]['id']
ref_variants = []
for snp_info in snp_info_lst:
if snp_info['id'] == snp_info['ref_id']:
ref_variants.append(snp_info['id'])
if len(ref_variants) > 1:
var_id = "rs" + ref_variants[0]
elif len(ref_variants) == 0 and len(snp_info_lst) > 1:
var_id = "rs" + snp_info_lst[0]['id']
else:
var_id = "rs" + ref_variants[0]
return var_id
elif len(snp_info_lst) == 1:
var_id = "rs" + snp_info_lst[0]['id']
return var_id
else:
return snp
else:
return snp
return snp
snp = replace_coord_rsid(db, snp)
# Find RS number in snp database
snp_coord = get_coords(db, snp)
# Get population ids from LDproxy.py tmp output files
pop_list = open(tmp_dir + "pops_" + request + ".txt").readlines()
ids = []
for i in range(len(pop_list)):
ids.append(pop_list[i].strip())
pop_ids = list(set(ids))
# Extract query SNP phased genotypes
vcf_file = vcf_dir + \
snp_coord['chromosome'] + ".phase3_shapeit2_mvncall_integrated_v5.20130502.genotypes.vcf.gz"
tabix_snp_h = "tabix -H {0} | grep CHROM".format(vcf_file)
proc_h = subprocess.Popen(tabix_snp_h, shell=True, stdout=subprocess.PIPE)
head = [x.decode('utf-8')
for x in proc_h.stdout.readlines()][0].strip().split()
tabix_snp = "tabix {0} {1}:{2}-{2} | grep -v -e END > {3}".format(
vcf_file, snp_coord['chromosome'], snp_coord['position'],
tmp_dir + "snp_no_dups_" + request + ".vcf")
subprocess.call(tabix_snp, shell=True)
# Check SNP is in the 1000G population, has the correct RS number, and not
# monoallelic
vcf = open(tmp_dir + "snp_no_dups_" + request + ".vcf").readlines()
if len(vcf) == 0:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
return None
elif len(vcf) > 1:
geno = []
for i in range(len(vcf)):
if vcf[i].strip().split()[2] == snp:
geno = vcf[i].strip().split()
if geno == []:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf",
shell=True)
return None
else:
geno = vcf[0].strip().split()
if geno[2] != snp:
snp = geno[2]
if "," in geno[3] or "," in geno[4]:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
return None
index = []
for i in range(9, len(head)):
if head[i] in pop_ids:
index.append(i)
genotypes = {"0": 0, "1": 0}
for i in index:
sub_geno = geno[i].split("|")
for j in sub_geno:
if j in genotypes:
genotypes[j] += 1
else:
genotypes[j] = 1
if genotypes["0"] == 0 or genotypes["1"] == 0:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
return None
# Define window of interest around query SNP
window = 500000
coord1 = int(snp_coord['position']) - window
if coord1 < 0:
coord1 = 0
coord2 = int(snp_coord['position']) + window
# Calculate proxy LD statistics in parallel
threads = 4
block = (2 * window) // 4
commands = []
for i in range(threads):
if i == min(range(threads)) and i == max(range(threads)):
command = "python LDproxy_sub.py " + "True " + snp + " " + \
snp_coord['chromosome'] + " " + str(coord1) + " " + \
str(coord2) + " " + request + " " + str(i)
elif i == min(range(threads)):
command = "python LDproxy_sub.py " + "True " + snp + " " + \
snp_coord['chromosome'] + " " + str(coord1) + " " + \
str(coord1 + block) + " " + request + " " + str(i)
elif i == max(range(threads)):
command = "python LDproxy_sub.py " + "True " + snp + " " + snp_coord[
'chromosome'] + " " + str(
coord1 + (block * i) +
1) + " " + str(coord2) + " " + request + " " + str(i)
else:
command = "python LDproxy_sub.py " + "True " + snp + " " + snp_coord[
'chromosome'] + " " + str(
coord1 + (block * i) +
1) + " " + str(coord1 +
(block *
(i + 1))) + " " + request + " " + str(i)
commands.append(command)
processes = [
subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
for command in commands
]
# collect output in parallel
def get_output(process):
return process.communicate()[0].splitlines()
if not hasattr(threading.current_thread(), "_children"):
threading.current_thread()._children = weakref.WeakKeyDictionary()
pool = Pool(len(processes))
out_raw = pool.map(get_output, processes)
pool.close()
pool.join()
# Aggregate output
out_prox = []
for i in range(len(out_raw)):
for j in range(len(out_raw[i])):
col = out_raw[i][j].decode('utf-8').strip().split("\t")
col[6] = int(col[6])
col[7] = float(col[7])
col[8] = float(col[8])
col.append(abs(int(col[6])))
out_prox.append(col)
# Sort output
if r2_d not in ["r2", "d"]:
r2_d = "r2"
out_dist_sort = sorted(out_prox, key=operator.itemgetter(14))
if r2_d == "r2":
out_ld_sort = sorted(out_dist_sort,
key=operator.itemgetter(8),
reverse=True)
else:
out_ld_sort = sorted(out_dist_sort,
key=operator.itemgetter(7),
reverse=True)
# Organize scatter plot data
q_rs = []
q_allele = []
q_coord = []
q_maf = []
p_rs = []
p_allele = []
p_coord = []
p_maf = []
dist = []
d_prime = []
d_prime_round = []
r2 = []
r2_round = []
corr_alleles = []
regdb = []
funct = []
color = []
size = []
for i in range(len(out_ld_sort)):
q_rs_i, q_allele_i, q_coord_i, p_rs_i, p_allele_i, p_coord_i, dist_i, d_prime_i, r2_i, corr_alleles_i, regdb_i, q_maf_i, p_maf_i, funct_i, dist_abs = out_ld_sort[
i]
if float(r2_i) > 0.01:
q_rs.append(q_rs_i)
q_allele.append(q_allele_i)
q_coord.append(float(q_coord_i.split(":")[1]) / 1000000)
q_maf.append(str(round(float(q_maf_i), 4)))
if p_rs_i == ".":
p_rs_i = p_coord_i
p_rs.append(p_rs_i)
p_allele.append(p_allele_i)
p_coord.append(float(p_coord_i.split(":")[1]) / 1000000)
p_maf.append(str(round(float(p_maf_i), 4)))
dist.append(str(round(dist_i / 1000000.0, 4)))
d_prime.append(float(d_prime_i))
d_prime_round.append(str(round(float(d_prime_i), 4)))
r2.append(float(r2_i))
r2_round.append(str(round(float(r2_i), 4)))
corr_alleles.append(corr_alleles_i)
# Correct Missing Annotations
if regdb_i == ".":
regdb_i = ""
regdb.append(regdb_i)
if funct_i == ".":
funct_i = ""
if funct_i == "NA":
funct_i = "none"
funct.append(funct_i)
# Set Color
if i == 0:
color_i = "blue"
elif funct_i != "none" and funct_i != "":
color_i = "red"
else:
color_i = "orange"
color.append(color_i)
# Set Size
size_i = 9 + float(p_maf_i) * 14.0
size.append(size_i)
# Begin Bokeh Plotting
from collections import OrderedDict
from bokeh.embed import components, file_html
from bokeh.layouts import gridplot
from bokeh.models import HoverTool, LinearAxis, Range1d
from bokeh.plotting import ColumnDataSource, curdoc, figure, output_file, reset_output, save
from bokeh.resources import CDN
from bokeh.io import export_svgs
import svgutils.compose as sg
reset_output()
# Proxy Plot
x = p_coord
if r2_d == "r2":
y = r2
else:
y = d_prime
whitespace = 0.01
xr = Range1d(start=coord1 / 1000000.0 - whitespace,
end=coord2 / 1000000.0 + whitespace)
yr = Range1d(start=-0.03, end=1.03)
sup_2 = "\u00B2"
proxy_plot = figure(
title="Proxies for " + snp + " in " + pop,
min_border_top=2,
min_border_bottom=2,
min_border_left=60,
min_border_right=60,
h_symmetry=False,
v_symmetry=False,
plot_width=900,
plot_height=600,
x_range=xr,
y_range=yr,
tools="hover,tap,pan,box_zoom,box_select,undo,redo,reset,previewsave",
logo=None,
toolbar_location="above")
proxy_plot.title.align = "center"
# Get recomb from LDproxy.py tmp output files
filename = tmp_dir + "recomb_" + request + ".txt"
recomb_raw = open(filename).readlines()
recomb_x = []
recomb_y = []
for i in range(len(recomb_raw)):
chr, pos, rate = recomb_raw[i].strip().split()
recomb_x.append(int(pos) / 1000000.0)
recomb_y.append(float(rate) / 100.0)
data = {
'x': x,
'y': y,
'qrs': q_rs,
'q_alle': q_allele,
'q_maf': q_maf,
'prs': p_rs,
'p_alle': p_allele,
'p_maf': p_maf,
'dist': dist,
'r': r2_round,
'd': d_prime_round,
'alleles': corr_alleles,
'regdb': regdb,
'funct': funct,
'size': size,
'color': color
}
source = ColumnDataSource(data)
proxy_plot.line(recomb_x, recomb_y, line_width=1, color="black", alpha=0.5)
proxy_plot.circle(x='x',
y='y',
size='size',
color='color',
alpha=0.5,
source=source)
hover = proxy_plot.select(dict(type=HoverTool))
hover.tooltips = OrderedDict([
("Query Variant", "@qrs @q_alle"),
("Proxy Variant", "@prs @p_alle"),
("Distance (Mb)", "@dist"),
("MAF (Query,Proxy)", "@q_maf,@p_maf"),
("R" + sup_2, "@r"),
("D\'", "@d"),
("Correlated Alleles", "@alleles"),
("RegulomeDB", "@regdb"),
("Functional Class", "@funct"),
])
proxy_plot.text(x,
y,
text=regdb,
alpha=1,
text_font_size="7pt",
text_baseline="middle",
text_align="center",
angle=0)
if r2_d == "r2":
proxy_plot.yaxis.axis_label = "R" + sup_2
else:
proxy_plot.yaxis.axis_label = "D\'"
proxy_plot.extra_y_ranges = {"y2_axis": Range1d(start=-3, end=103)}
proxy_plot.add_layout(
LinearAxis(y_range_name="y2_axis",
axis_label="Combined Recombination Rate (cM/Mb)"), "right")
# Rug Plot
y2_ll = [-0.03] * len(x)
y2_ul = [1.03] * len(x)
yr_rug = Range1d(start=-0.03, end=1.03)
data_rug = {
'x': x,
'y': y,
'y2_ll': y2_ll,
'y2_ul': y2_ul,
'qrs': q_rs,
'q_alle': q_allele,
'q_maf': q_maf,
'prs': p_rs,
'p_alle': p_allele,
'p_maf': p_maf,
'dist': dist,
'r': r2_round,
'd': d_prime_round,
'alleles': corr_alleles,
'regdb': regdb,
'funct': funct,
'size': size,
'color': color
}
source_rug = ColumnDataSource(data_rug)
rug = figure(x_range=xr,
y_range=yr_rug,
border_fill_color='white',
y_axis_type=None,
title="",
min_border_top=2,
min_border_bottom=2,
min_border_left=60,
min_border_right=60,
h_symmetry=False,
v_symmetry=False,
plot_width=900,
plot_height=50,
tools="xpan,tap",
logo=None)
rug.segment(x0='x',
y0='y2_ll',
x1='x',
y1='y2_ul',
source=source_rug,
color='color',
alpha=0.5,
line_width=1)
rug.toolbar_location = None
# Gene Plot
# Get genes from LDproxy.py tmp output files
filename = tmp_dir + "genes_" + request + ".txt"
genes_raw = open(filename).readlines()
genes_plot_start = []
genes_plot_end = []
genes_plot_y = []
genes_plot_name = []
exons_plot_x = []
exons_plot_y = []
exons_plot_w = []
exons_plot_h = []
exons_plot_name = []
exons_plot_id = []
exons_plot_exon = []
lines = [0]
gap = 80000
tall = 0.75
if genes_raw != None:
for i in range(len(genes_raw)):
bin, name_id, chrom, strand, txStart, txEnd, cdsStart, cdsEnd, exonCount, exonStarts, exonEnds, score, name2, cdsStartStat, cdsEndStat, exonFrames = genes_raw[
i].strip().split()
name = name2
id = name_id
e_start = exonStarts.split(",")
e_end = exonEnds.split(",")
# Determine Y Coordinate
i = 0
y_coord = None
while y_coord == None:
if i > len(lines) - 1:
y_coord = i + 1
lines.append(int(txEnd))
elif int(txStart) > (gap + lines[i]):
y_coord = i + 1
lines[i] = int(txEnd)
else:
i += 1
genes_plot_start.append(int(txStart) / 1000000.0)
genes_plot_end.append(int(txEnd) / 1000000.0)
genes_plot_y.append(y_coord)
genes_plot_name.append(name + " ")
for i in range(len(e_start) - 1):
if strand == "+":
exon = i + 1
else:
exon = len(e_start) - 1 - i
width = (int(e_end[i]) - int(e_start[i])) / 1000000.0
x_coord = int(e_start[i]) / 1000000.0 + (width / 2)
exons_plot_x.append(x_coord)
exons_plot_y.append(y_coord)
exons_plot_w.append(width)
exons_plot_h.append(tall)
exons_plot_name.append(name)
exons_plot_id.append(id)
exons_plot_exon.append(exon)
n_rows = len(lines)
genes_plot_yn = [n_rows - x + 0.5 for x in genes_plot_y]
exons_plot_yn = [n_rows - x + 0.5 for x in exons_plot_y]
yr2 = Range1d(start=0, end=n_rows)
data_gene_plot = {
'exons_plot_x': exons_plot_x,
'exons_plot_yn': exons_plot_yn,
'exons_plot_w': exons_plot_w,
'exons_plot_h': exons_plot_h,
'exons_plot_name': exons_plot_name,
'exons_plot_id': exons_plot_id,
'exons_plot_exon': exons_plot_exon
}
source_gene_plot = ColumnDataSource(data_gene_plot)
if len(lines) < 3:
plot_h_pix = 150
else:
plot_h_pix = 150 + (len(lines) - 2) * 50
gene_plot = figure(
x_range=xr,
y_range=yr2,
border_fill_color='white',
title="",
min_border_top=2,
min_border_bottom=2,
min_border_left=60,
min_border_right=60,
h_symmetry=False,
v_symmetry=False,
plot_width=900,
plot_height=plot_h_pix,
tools="hover,tap,xpan,box_zoom,undo,redo,reset,previewsave",
logo=None)
gene_plot.segment(genes_plot_start,
genes_plot_yn,
genes_plot_end,
genes_plot_yn,
color="black",
alpha=1,
line_width=2)
gene_plot.rect(x='exons_plot_x',
y='exons_plot_yn',
width='exons_plot_w',
height='exons_plot_h',
source=source_gene_plot,
fill_color="grey",
line_color="grey")
gene_plot.xaxis.axis_label = "Chromosome " + \
snp_coord['chromosome'] + " Coordinate (Mb)(GRCh37)"
gene_plot.yaxis.axis_label = "Genes"
gene_plot.ygrid.grid_line_color = None
gene_plot.yaxis.axis_line_color = None
gene_plot.yaxis.minor_tick_line_color = None
gene_plot.yaxis.major_tick_line_color = None
gene_plot.yaxis.major_label_text_color = None
hover = gene_plot.select(dict(type=HoverTool))
hover.tooltips = OrderedDict([
("Gene", "@exons_plot_name"),
("ID", "@exons_plot_id"),
("Exon", "@exons_plot_exon"),
])
gene_plot.text(genes_plot_start,
genes_plot_yn,
text=genes_plot_name,
alpha=1,
text_font_size="7pt",
text_font_style="bold",
text_baseline="middle",
text_align="right",
angle=0)
gene_plot.toolbar_location = "below"
# Change output backend to SVG temporarily for headless export
# Will be changed back to canvas in LDlink.js
proxy_plot.output_backend = "svg"
rug.output_backend = "svg"
gene_plot.output_backend = "svg"
export_svgs(proxy_plot,
filename=tmp_dir + "proxy_plot_1_" + request + ".svg")
export_svgs(gene_plot,
filename=tmp_dir + "gene_plot_1_" + request + ".svg")
# 1 pixel = 0.0264583333 cm
svg_height = str(20.00 + (0.0264583333 * plot_h_pix)) + "cm"
svg_height_scaled = str(100.00 + (0.1322916665 * plot_h_pix)) + "cm"
# Concatenate svgs
sg.Figure("24.59cm", svg_height,
sg.SVG(tmp_dir + "proxy_plot_1_" + request + ".svg"),
sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").move(
0, 630)).save(tmp_dir + "proxy_plot_" + request + ".svg")
sg.Figure(
"122.95cm", svg_height_scaled,
sg.SVG(tmp_dir + "proxy_plot_1_" + request + ".svg").scale(5),
sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").scale(5).move(
0, 3150)).save(tmp_dir + "proxy_plot_scaled_" + request + ".svg")
# Export to PDF
subprocess.call("phantomjs ./rasterize.js " + tmp_dir + "proxy_plot_" +
request + ".svg " + tmp_dir + "proxy_plot_" + request +
".pdf",
shell=True)
# Export to PNG
subprocess.call("phantomjs ./rasterize.js " + tmp_dir +
"proxy_plot_scaled_" + request + ".svg " + tmp_dir +
"proxy_plot_" + request + ".png",
shell=True)
# Export to JPEG
subprocess.call("phantomjs ./rasterize.js " + tmp_dir +
"proxy_plot_scaled_" + request + ".svg " + tmp_dir +
"proxy_plot_" + request + ".jpeg",
shell=True)
# Remove individual SVG files after they are combined
subprocess.call("rm " + tmp_dir + "proxy_plot_1_" + request + ".svg",
shell=True)
subprocess.call("rm " + tmp_dir + "gene_plot_1_" + request + ".svg",
shell=True)
# Remove scaled SVG file after it is converted to png and jpeg
subprocess.call("rm " + tmp_dir + "proxy_plot_scaled_" + request + ".svg",
shell=True)
reset_output()
# Remove temporary files
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt", shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
subprocess.call("rm " + tmp_dir + "genes_" + request + ".txt", shell=True)
subprocess.call("rm " + tmp_dir + "recomb_" + request + ".txt", shell=True)
# Return plot output
return None
def plot_volume(self, data, alpha=1.0, extra_axis=False):
'''
Plot method for volume
extra_axis: displays a second axis (for overlay on data plotting)
'''
source_id = get_source_id(data)
self.set_cds_col(source_id + 'volume')
# create color columns
volup = convert_color(self._scheme.volup)
voldown = convert_color(self._scheme.voldown)
self.set_cds_col((source_id + 'colors_volume', source_id + 'open',
source_id + 'close',
partial(cds_op_color,
color_up=volup,
color_down=voldown)))
# prepare bar kwargs
kwargs = {
'x': 'index',
'width': self._bar_width,
'top': source_id + 'volume',
'bottom': 0,
'fill_color': source_id + 'colors_volume',
'line_color': source_id + 'colors_volume',
'fill_alpha': alpha,
'line_alpha': alpha,
'name': 'Volume',
'legend_label': 'Volume'
}
# set axis
ax_formatter = NumeralTickFormatter(format=self._scheme.number_format)
if extra_axis:
source_data_axis = 'axvol'
# use colorup
ax_color = convert_color(self._scheme.volup)
# use only one additional axis to prevent multiple axis being added
# to a single figure
ax = self.figure.select_one({'name': source_data_axis})
if ax is None:
# create new axis if not already available
self.figure.extra_y_ranges = {
source_data_axis:
DataRange1d(range_padding=1.0 / self._scheme.volscaling,
start=0)
}
ax = LinearAxis(name=source_data_axis,
y_range_name=source_data_axis,
formatter=ax_formatter,
axis_label_text_color=ax_color,
axis_line_color=ax_color,
major_label_text_color=ax_color,
major_tick_line_color=ax_color,
minor_tick_line_color=ax_color)
self.figure.add_layout(ax, self._scheme.vol_axis_location)
kwargs['y_range_name'] = source_data_axis
else:
self.figure.yaxis.formatter = ax_formatter
# append renderer
self._figure_append_renderer(self.figure.vbar, **kwargs)
# set hover label
self._fp.hover.add_hovertip(
'Volume', f'@{source_id}volume{{({self._scheme.number_format})}}',
data)
def plot_grouped_barchart(df):
from bokeh.models import FactorRange # 设置x坐标轴
from bokeh.models import LinearAxis, Range1d # 设置双y轴
from bokeh.io import output_notebook, show
from bokeh.models import ColumnDataSource # 整合数据
from bokeh.transform import factor_cmap # 颜色
from bokeh.models import HoverTool
from bokeh.plotting import figure, ColumnDataSource
# x坐标轴坐标两个参数
months = list(df['Month'][1:])
depart_list = ['SAC', 'ICB', 'SPU']
# y轴高度
sac_list = list(df['SAC'])[1:]
icb_list = list(df['ICB'])[1:]
spu_list = list(df['SPU'])[1:]
tot_target = df['TOTAL.1'][1:]
tot_real = df['TOTAL.2'][1:]
y2 = tot_real / tot_target * 100
data = {
'SAC': sac_list,
'ICB': icb_list,
'SPU': spu_list,
}
colors2 = ['salmon', 'darkseagreen', 'paleturquoise']
# 设置x坐标轴
x = [(month, dep) for month in months for dep in depart_list]
# 设置竖直
counts = sum(zip(data['SAC'], data['ICB'], data['SPU']), ())
source = ColumnDataSource(data=dict(
x=x,
counts=counts,
))
p = figure(x_range=FactorRange(*x), plot_height=450, plot_width=1000, title="2017年月度实际部门销售额度",
y_axis_label='销售额(万元)', x_axis_label='时间/部门')
p.vbar(x='x', top='counts', width=1, source=source, line_color='white',
fill_color=factor_cmap('x', palette=colors2, factors=depart_list, start=1, end=2))
# ["#FC9D9A", "#F9CDAD", "#C8C8A9"]
p.extra_y_ranges = {"foo": Range1d(start=0, end=150)}
p.line(months, y2, color='pink', y_range_name="foo")
p.circle(months, y2, color='red', y_range_name="foo")
p.add_layout(LinearAxis(y_range_name="foo"), "right")
p.y_range.start = 0
p.x_range.range_padding = 0.1
p.xaxis.major_label_orientation = 1
p.xgrid.grid_line_color = None
# set hover function
hover1 = HoverTool(tooltips=[("时间/部门", '@x'), ("销售额", "@counts" + "万元")],
mode='vline')
p.add_tools(hover1)
show(p)
def plot_graph_by_transaction(scenario_metrics_df, overall_ninety_fifth_df,
scenario):
# Remove $ from the names of column names of scenario_metrics_df and
# Rename the Transactions of overall_ninety_fifth_df
col_name_dict = remove_dollar_sign_and_get_column_names_dict(
scenario_metrics_df, overall_ninety_fifth_df)
# Define Y-Axis Range of the Graph
(left_y_range, right_y_range) = get_y_range_of_graph(scenario_metrics_df)
# Get the colors for the Lines of the Graph
color_palette = get_color_palette(scenario_metrics_df, scenario)
# Tools to be available in graph
tools_to_show = 'box_zoom,reset,save'
# create a new plot with a title and axis labels
scenario_graph = plot_new_graph('Time', 'datetime', 'Response Time (ms)',
1900, 400, left_y_range, 'below',
tools_to_show)
# Disabling Hover Tool
scenario_graph.toolbar.active_inspect = None
# Index to go through Color Palette
color_index = 0
# get all the legends in one list
legend_list = []
# Source of Graphs
source = ColumnDataSource(scenario_metrics_df)
# Plot graph transaction-wise
for col_name in scenario_metrics_df.columns:
# Ignore Column LocalTime
if col_name not in "LocalTime":
if col_name in "RPM":
# Get the legend name
legend_name = "Throughput (RPM)"
# Setting the second y axis range name and range
scenario_graph.extra_y_ranges = {
"RPM": Range1d(0, right_y_range)
}
# Adding the second axis to the plot.
scenario_graph.add_layout(
LinearAxis(y_range_name="RPM",
axis_label="Throughput (RPM)"), 'right')
# PlotGraph
plot_graph = scenario_graph.line('LocalTime',
col_name,
source=source,
line_width=2,
color="yellow",
y_range_name="RPM",
name=col_name)
else:
# Transaction 95th
col_ninety_fifth = int(overall_ninety_fifth_df.loc[
overall_ninety_fifth_df['Transaction'] == col_name,
'NinetyFifth'].item())
# Get the legend name along with Transaction's 95th
legend_name = col_name_dict[
col_name] + " (95th: {} ms)".format(col_ninety_fifth)
# PlotGraph
plot_graph = scenario_graph.line(
'LocalTime',
col_name,
source=source,
line_width=2,
color=color_palette[color_index],
name=col_name)
# increment through color palette
color_index = color_index + 1
# Append the legend
legend_list.append((legend_name, [plot_graph]))
# Append the graph in list which will be passed to "Column"
scenario_graph_final = set_graph_and_legend_properties(
scenario_graph, legend_list, scenario)
return scenario_graph_final
def make_yeartrendplot(result, top_result_time):
#import matplotlib.pyplot as plt
import numpy as np
from bokeh.plotting import figure
from bokeh.embed import components
from bokeh.plotting import figure
from bokeh.models import Range1d, LinearAxis
import datetime
visitors = np.exp(result['pred'])
#define different axis labs, for visibility
if max(visitors) > 500000:
breaks = [10000, 50000, 100000, 500000, 1000000]
overrides = {
10000: '10k',
50000: '50k',
100000: '100k',
500000: '500k',
1000000: '1 million'
}
elif max(visitors) > 100000:
breaks = [10000, 50000, 100000, 500000]
overrides = {
10000: '10k',
50000: '50k',
100000: '100k',
500000: '500k'
}
elif max(visitors) > 50000:
breaks = [10000, 25000, 50000, 75000, 100000]
overrides = {
10000: '10k',
25000: '25k',
50000: '50k',
75000: '75k',
100000: '100k'
}
elif max(visitors) <= 50000:
breaks = [1000, 5000, 10000, 25000, 50000]
overrides = {
1000: '1k',
5000: '5k',
10000: '10k',
25000: '25k',
50000: '50k'
}
dates_in_dt = np.array(result['date'], dtype=np.datetime64)
top_in_dt = np.array(top_result_time, dtype=np.datetime64)
# index top result for plotting
ti = np.where(dates_in_dt == top_in_dt)[0]
# def make_yeartrendplot(SELECTED_PARK):
# plot a yearly trend
plot = figure(y_range=(min(visitors) - 1000, max(visitors) + 1000),
x_axis_type='datetime',
plot_width=650,
plot_height=300)
plot.xaxis.axis_label = 'Month / Year'
plot.yaxis.axis_label = 'Number of Visitors'
plot.line(result['date'],
visitors,
color="black",
legend='visitors',
line_width=3)
plot.yaxis.ticker = breaks
plot.yaxis.major_label_overrides = overrides
# Create 2nd y-axis
plot.extra_y_ranges['temp'] = Range1d(start=0, end=110)
plot.add_layout(
LinearAxis(y_range_name='temp', axis_label='Temperature (°F)'),
'right')
plot.line(x=result['date'],
y=result['MaxT'],
legend='max temp.',
y_range_name='temp',
color='orangered',
line_width=3,
line_dash='dotted')
plot.line(x=result['date'],
y=result['MinT'],
legend='min temp.',
y_range_name='temp',
color='royalblue',
line_width=3,
line_dash='dotted')
plot.toolbar_location = 'above'
plot.quad(top=[max(visitors) + 1000],
bottom=[min(visitors) - 1000],
left=[dates_in_dt[ti][0]],
right=[dates_in_dt[ti + 1][0]],
color='#22A784',
fill_alpha=0.2)
plot.legend.border_line_width = 2
plot.legend.border_line_color = "black"
plot.legend.border_line_alpha = 0.5
script, div = components(plot)
return script, div
def plot(y: list,
x: list = None,
label: list = None,
width: int = 400,
height: int = 400,
gain: float = 0.4,
margin_x: int = 1,
title: str = "graph",
script_name: str = "",
slider_partitions: int = None,
multiple_axes=False):
"""Plots that represent data with sound and can be checked interactively
You can interactively check the data in graph form by moving the mouse cursor.
When you enter or leave the graph image, you will be notified by voice.
Also, when you move the mouse left or right on the graph image,
the y-axis value corresponding to that location will be expressed with a high or low tone.
A single click will read out the value corresponding to that location.
Also, double-clicking switches the group according to the label specified as an option.
Parameters
----------
y : list
A list of values to be graphed.
x : list
A list of x-axis values corresponding to y-axis values.
If not specified, it is substituted by the value of the equal interval. Optional.
label : list
A list of grouping numbers for each value, which must start with zero.
You can compare the graph data by sound, switching between each number. Optional.
width : int
Width of the graph image (in pixels). Optional.
height : int
Height of the graph image (in pixels). Optional.
title: str
Graph name to be read out. Optional.
multiple_axes: bool
Set to True if you want each label to have a separate y-axis. Optional.
Examples
--------
>>> plot([0, 1, 2])
<IPython.core.display.HTML object>
>>> plot(x=[0, 1, 2], y=[4, 5, 6], label=[0, 0, 1])
<IPython.core.display.HTML object>
"""
if type(y) == np.ndarray:
y = y.tolist()
if type(x) == np.ndarray:
x = x.tolist()
elif x == None:
x = np.arange(len(y)).tolist()
if type(label) == np.ndarray:
label = label.astype(int).tolist()
elif label == None:
label = np.zeros_like(y).astype(int).tolist()
if label:
assert max(label) < len(__COLORS), "max label must be lower {}".format(
len(__COLORS))
assert max(label) + 1 == len(set(
label)), "label should be in {} because max label is {}.".format(
list(range(max(label) + 1)), max(label))
if script_name == "":
__set_context()
output_notebook()
plot = figure(plot_width=width,
plot_height=height,
tools="",
toolbar_location=None)
colors = [__COLORS[c] for c in label]
if multiple_axes:
assert max(label) == 1, "The number of labels must be two kinds"
multi_axes_str = "true"
y_ranges = {}
for l in range(max(label) + 1):
__x = np.array(x)[np.array(label) == l].tolist()
__y = np.array(y)[np.array(label) == l].tolist()
__c = np.array(colors)[np.array(label) == l].tolist()
plot.scatter(__x,
__y,
line_color=__c,
fill_color=__c,
y_range_name=str(l))
if l == 1:
plot.add_layout(LinearAxis(y_range_name=str(l)), 'right')
y_ranges[str(l)] = Range1d(start=min(__y) - 1, end=max(__y) + 1)
plot.extra_y_ranges = y_ranges
else:
multi_axes_str = "false"
plot.scatter(x, y, line_color=colors, fill_color=colors)
sound_js = """
const multiAxes = %s;
%s
if(diff[nearestIdx] > marginX) {
return;
}
const gain = %s; // max: 1.0
osc.type = 'triangle'; // sine, square, sawtooth, triangle
osc.frequency.value = 261.626 + (nearestY - minY) / (maxY - minY) * 261.626 // Hz
audioGain.gain.linearRampToValueAtTime(gain, audioContext.currentTime + 0.2); // atack
audioGain.gain.setTargetAtTime(0, audioContext.currentTime + 0.2, 0.5); // decay, sustain
let pan = (nearestX - minX) / (maxX - minX) * 2 - 1;
panNode.pan.value = pan; // left:-1 ~ right:1
""" % (multi_axes_str, __FIND_NEAREST_JS, gain)
# Mouse hover on plot
hover_code = """
let marginX = %s;
let position = cb_data.geometry.x;
%s
""" % (margin_x, sound_js)
callback = CustomJS(args={"x": x, "y": y, "label": label}, code=hover_code)
plot.add_tools(HoverTool(tooltips=None, callback=callback))
# Single tap on plot
tap_code = """
let position = cb_obj.x;
const multiAxes = %s;
%s
%s
""" % (multi_axes_str, __FIND_NEAREST_JS,
__speak_js("`X is ${nearestX}. Y is ${nearestY}`"))
plot.js_on_event(
events.Tap,
CustomJS(args={
"x": x,
"y": y,
"label": label
}, code=tap_code))
if len(set(label)) > 1:
# Double tap on plot
double_tap_code = """
oscTarget = (oscTarget + 1) %% (maxLabel + 1);
%s
""" % (__speak_js("`label ${oscTarget} is selected`"))
plot.js_on_event(
events.DoubleTap,
CustomJS(args={"maxLabel": max(label)}, code=double_tap_code))
# Enter or leave on plot
read_label = (max(label) > 0)
plot.js_on_event(events.MouseEnter, __speak_inout(title, True, read_label))
plot.js_on_event(events.MouseLeave, __speak_inout(title, False,
read_label))
# slider for keyboard interaction
sliders = []
for l in range(max(label) + 1):
__x = np.array(x)[np.array(label) == l].tolist()
if slider_partitions is None:
slider_partitions = np.min([len(__x) - 1, 30])
if slider_partitions == 30:
print(
"The number of slider partitions has been reduced to 30 as the default limit. Please set slider_partitions as an argument if necessary."
)
slider_start = np.min(__x)
slider_end = np.max(__x)
if slider_start == slider_end:
slider_end += 1
slider_step = (slider_end - slider_start) / slider_partitions
slider_code = """
oscTarget = target;
let marginX = %s;
let position = slider.value;
%s
setTimeout(function(){%s}, 3000);
""" % (slider_step, sound_js,
__speak_js("`X is ${nearestX}. Y is ${nearestY}`"))
slider = Slider(start=slider_start,
end=slider_end,
value=slider_start,
step=slider_step,
title="label {}".format(l))
slider.js_on_change(
'value',
CustomJS(args={
"x": x,
"y": y,
"label": label,
"slider": slider,
"target": l
},
code=slider_code))
sliders.append(slider)
# layout
message1 = Div(text="""<h2>output of audio plot lib</h2>""")
message2 = Div(
text=
"""<p>There is a graph and a series of sliders to check the values. If you have a mouse, you can check the values by hovering over the graph. If you are using only a keyboard, you can move the slider to move the horizontal axis of the graph to check the value of the graph as a pitch according to the location.</p>"""
)
show(column(message1, message2, row(plot, column(sliders))))
if script_name != "":
from bs4 import BeautifulSoup
HTML = """
<button id="unmuteButton">Push here to unmute graph</button>
<script>
document.getElementById('unmuteButton').addEventListener('click', function() {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
audioGain = audioContext.createGain();
panNode = audioContext.createStereoPanner();
osc = audioContext.createOscillator();
osc.connect(panNode);
panNode.connect(audioGain);
audioGain.connect(audioContext.destination);
osc.start(audioContext.currentTime);
audioGain.gain.setValueAtTime(0, audioContext.currentTime);
oscTarget = 0;
})
</script>
"""
html_filename = script_name.replace(".py", ".html")
soup = BeautifulSoup(open(html_filename), 'html.parser')
soup.body.insert(0, BeautifulSoup(HTML, "html.parser")) # after body
with open(html_filename, "w") as file:
file.write(str(soup))
def plot_ncu_graph(title, labels, renderer, graphtools,
figure_plots,
source,
):
'''
create plots customized by programed and users files
:param title: plot title
:param labels: plot label in sequence [x,y,y2], can be just [x,y]
:param renderer: webgl, svg or canvas
:param graphtools: tools for user analysis the graph
:param figure_plots: dataframe containing all data for make the plot
:param source: ColumnDatSource object for bokeh struct
:return: plot
'''
#try:
lines = []
secondary_axis = False
for ax_ch in figure_plots['axis']:
if ax_ch == 'b':
secondary_axis = True
TOOLTIPS = [
("(x,y)", "($x, $y)"),
]
p = figure(title = title,
x_axis_label = labels[0],
y_axis_label = labels[1],
plot_width = 1400, plot_height = 300,
toolbar_location="below",
tooltips=TOOLTIPS,
output_backend=renderer,
tools=graphtools,
)
y_overlimit = 0.05 # show y axis below and above y min and max value
# FIRST AXIS
min_all = []
max_all = []
for index, figure_plot in figure_plots.iterrows():
min_all.append(source.data[figure_plot['channel']].min())
max_all.append(source.data[figure_plot['channel']].max())
if secondary_axis:
p.extra_y_ranges = {"b": Range1d( # 0,140
source.data[figure_plots.iloc[-1, 1]].min() * (1 - y_overlimit),
source.data[figure_plots.iloc[-1, 1]].max() * (1 + y_overlimit)
)}
p.add_layout(LinearAxis(y_range_name='b', axis_label=labels[2]), 'right')
i=0
for index, figure_plot in figure_plots.iterrows():
ch_max = ' | Max: '+ str(int(pd.to_numeric(source.data[figure_plot['channel']]).max()))
ch_min = ' -- Min: '+ str(int(pd.to_numeric(source.data[figure_plot['channel']]).min()))
ch_ave = ' | Average: '+ str(int(np.average(pd.to_numeric(source.data[figure_plot['channel']]))))
leg = figure_plot['legend'] #+ ch_min + ch_ave + ch_max
color = settings.channels_config_propertise.loc[figure_plot['channel'], 'Color']
if figure_plot['axis'] == 'b':
gli = p.line(x=figure_plot['x'], y=figure_plot['channel'], color=color, line_width=figure_plot['line_w'],
alpha=figure_plot['alpha'], y_range_name=figure_plot['axis'],
source=source,
legend_label = leg)
#lines.append() #, y_range_name=figure_plot['axis']
else:
gli = p.line(x=figure_plot['x'], y=figure_plot['channel'], color=color, line_width=figure_plot['line_w'],alpha=figure_plot['alpha'], source=source, legend_label = leg)
p.toolbar.logo = None
p.legend.location = "top_left"
p.legend.click_policy = "hide"
p.legend.label_text_font_size = '10pt'
return p
template = """
{% block preamble %}
<style>
body { background-color: lightblue; }
</style>
{% endblock %}
"""
plot = Plot(plot_width=600,
plot_height=600,
x_range=Range1d(0, 10),
y_range=Range1d(0, 10),
toolbar_location=None)
# This is the no-fill that we're testing
plot.background_fill_color = None
plot.border_fill_color = None
plot.add_glyph(Circle(x=3, y=3, size=50, fill_color='#ffffff'))
plot.add_glyph(Circle(x=6, y=6, size=50, fill_color='#ffffff'))
yaxis = LinearAxis(major_label_text_color='#ffffff',
major_label_text_font_size="30pt")
plot.add_layout(yaxis, 'left')
xaxis = LinearAxis(major_label_text_color='#ffffff',
major_label_text_font_size="30pt")
plot.add_layout(xaxis, 'below')
save(plot, template=template)
def StationPlot(ID):
now = datetime.datetime.now()
end = str(now.year) + fixdatestrings(now.month) + fixdatestrings(
now.day) + fixdatestrings(now.hour)
startdate = date.today() - timedelta(30)
start = str(startdate.year) + fixdatestrings(
startdate.month) + fixdatestrings(startdate.day)
# print(start)
# print (end)
# 201904240000
var = 'vars=pressure,air_temp,dew_point_temperature,relative_humidity,wind_speed,wind_direction,wind_gust,precip_accum,precip_accum_24_hour'
filename = 'https://api.mesowest.net/v2/stations/timeseries?&stid=' + ID + '&start=' + start + '0000&end=' + end + '00&token=demotoken&r&obtimezone=local&' + var + '&output=csv'
dat_df = read_files(filename) #load json and turn into dataframe.
dates = np.array(dat_df.Date_Time, dtype=np.datetime64)
dat_df['dates'] = np.array(dat_df.Date_Time, dtype=np.datetime64)
source = ColumnDataSource(data=dat_df)
# source = ColumnDataSource(data=dict(date=dates, temp=dat_df.air_temp_set_1, dew = dat_df.dew_point_temperature_set_1d))
# print(dat_df.head())
p = figure(plot_height=300,
plot_width=800,
title=ID,
x_axis_type="datetime",
x_axis_location="above",
background_fill_color="#efefef",
x_range=(dates[-200], dates[-1]))
# tools="xpan", toolbar_location=None
p.line('dates',
'air_temp_set_1',
source=source,
legend='Air Temp',
line_color="tomato")
p.line('dates',
'dew_point_temperature_set_1d',
source=source,
legend='Dew Point',
line_color="indigo")
p.y_range = Range1d(dat_df.dew_point_temperature_set_1d.min(),
dat_df.air_temp_set_1.max() + 2)
p.yaxis.axis_label = 'Celsius'
p.legend.location = "bottom_left"
p.legend.click_policy = "hide"
p.extra_y_ranges = {"hum": Range1d(start=0, end=100)}
p.add_layout(LinearAxis(y_range_name="hum", axis_label='%'), 'right')
p.line('dates',
'relative_humidity_set_1',
source=source,
legend='Rel Hum',
line_color="green",
y_range_name="hum")
p1 = figure(
plot_height=300,
plot_width=800,
x_range=p.x_range,
# title="Richardson @ Trims DOT (TRDA2)",
x_axis_type="datetime",
x_axis_location="above",
background_fill_color="#efefef")
# tools="xpan", toolbar_location=None
p1.line('dates',
'wind_speed_set_1',
source=source,
legend='Wind speed',
line_color="tomato")
p1.line('dates',
'wind_gust_set_1',
source=source,
legend='Gusts',
line_color="indigo")
p1.y_range = Range1d(start=0, end=dat_df.wind_gust_set_1.max() + 2)
p1.yaxis.axis_label = 'm/s'
p1.legend.location = "bottom_left"
p1.legend.click_policy = "hide"
p1.extra_y_ranges = {"dir": Range1d(start=0, end=360)}
p1.add_layout(LinearAxis(y_range_name="dir"), 'right')
p1.circle('dates',
'wind_direction_set_1',
source=source,
legend='Wind Dir',
line_color="black",
y_range_name="dir")
select = figure(title="Drag the selection box to change the range above",
plot_height=130,
plot_width=800,
y_range=p.y_range,
x_axis_type="datetime",
tools="",
toolbar_location=None,
background_fill_color="#efefef",
x_range=(dates[0], dates[-1]))
range_tool = RangeTool(x_range=p.x_range)
range_tool.overlay.fill_color = "navy"
range_tool.overlay.fill_alpha = 0.2
select.line('dates', 'air_temp_set_1', source=source)
select.extra_y_ranges = {"pcp": Range1d(start=0, end=5)}
select.yaxis.axis_label = 'C'
select.add_layout(LinearAxis(y_range_name="pcp", axis_label='mm'), 'right')
select.vbar(x='dates',
top='precip_accum_24_hour_set_1',
source=source,
width=1,
y_range_name="pcp")
select.ygrid.grid_line_color = None
select.add_tools(range_tool)
select.toolbar.active_multi = range_tool
return (column(p, p1, select))
title="Class",
editor=SelectEditor(options=classes)),
TableColumn(field="cty", title="City MPG", editor=IntEditor()),
TableColumn(field="hwy", title="Highway MPG", editor=IntEditor()),
]
data_table = DataTable(source=source, columns=columns, editable=True)
# TODO: rewrite this using plotting API {{{
xdr = DataRange1d(sources=[source.columns("index")])
ydr = DataRange1d(sources=[source.columns("cty"), source.columns("hwy")])
plot = Plot(title=None,
x_range=xdr,
y_range=ydr,
plot_width=1000,
plot_height=300)
xaxis = LinearAxis(plot=plot)
plot.below.append(xaxis)
yaxis = LinearAxis(plot=plot)
ygrid = Grid(plot=plot, dimension=1, ticker=yaxis.ticker)
plot.left.append(yaxis)
cty_glyph = Circle(x="index",
y="cty",
fill_color="#396285",
size=8,
fill_alpha=0.5,
line_alpha=0.5)
hwy_glyph = Circle(x="index",
y="hwy",
fill_color="#CE603D",
size=8,
fill_alpha=0.5,
x_range=(-3, 3),
plot_width=400,
plot_height=400)
plot.scatter('x1', 'x2', source=source, line_width=3, line_alpha=0.6)
plot.multi_line(xs='xs', ys='ys', line_color='line_color', source=source2)
plot2 = figure(y_range=(-3, 3),
x_range=(0, 3),
plot_width=400,
plot_height=400,
x_axis_type=None,
y_axis_label='y_i value')
plot2.line(x='xs', y='ys', source=source3)
plot2.scatter('xs', 'ys', source=source3)
ticker = SingleIntervalTicker(interval=1, num_minor_ticks=0)
xaxis = LinearAxis(ticker=ticker, axis_label='y index')
plot2.add_layout(xaxis, 'below')
resample_Button = Button(label="Resample")
covariance_slider = Slider(start=0.0,
end=.99,
value=0,
step=.01,
title="Covariance")
resample_Button.on_click(lambda: resample_wrapper())
covariance_slider.on_change('value', lambda attr, old, new: update_all())
controls = [covariance_slider, resample_Button]
inputs = widgetbox(*controls, sizing_mode='fixed')
l = gridplot([[inputs], [plot, plot2]])
def calculate_proxy(snp, pop, request, web, r2_d="r2"):
import csv
import json
import operator
import os
import sqlite3
import subprocess
import sys
import time
import threading
import weakref
import time
from multiprocessing.dummy import Pool
start_time = time.time()
# Set data directories
# data_dir = "/local/content/ldlink/data/"
# gene_dir = data_dir + "refGene/sorted_refGene.txt.gz"
# recomb_dir = data_dir + "recomb/genetic_map_autosomes_combined_b37.txt.gz"
# snp_dir = data_dir + "snp142/snp142_annot_2.db"
# pop_dir = data_dir + "1000G/Phase3/samples/"
# vcf_dir = data_dir + "1000G/Phase3/genotypes/ALL.chr"
# Set data directories using config.yml
with open('config.yml', 'r') as f:
config = yaml.load(f)
gene_dir=config['data']['gene_dir']
recomb_dir=config['data']['recomb_dir']
snp_dir=config['data']['snp_dir']
pop_dir=config['data']['pop_dir']
vcf_dir=config['data']['vcf_dir']
tmp_dir = "./tmp/"
# Ensure tmp directory exists
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
if request is False:
request = str(time.strftime("%I%M%S"))
# Create JSON output
out_json = open(tmp_dir + 'proxy' + request + ".json", "w")
output = {}
# Find coordinates (GRCh37/hg19) for SNP RS number
# Connect to snp database
conn = sqlite3.connect(snp_dir)
conn.text_factory = str
cur = conn.cursor()
def get_coords(rs):
id = rs.strip("rs")
t = (id,)
cur.execute("SELECT * FROM tbl_" + id[-1] + " WHERE id=?", t)
return cur.fetchone()
# Find RS number in snp database
snp_coord = get_coords(snp)
# Close snp connection
cur.close()
conn.close()
if snp_coord == None:
output["error"] = snp + " is not in dbSNP build " + config['data']['dbsnp_version'] + "."
json_output = json.dumps(output, sort_keys=True, indent=2)
print >> out_json, json_output
out_json.close()
return("", "")
raise
# Select desired ancestral populations
pops = pop.split("+")
pop_dirs = []
for pop_i in pops:
if pop_i in ["ALL", "AFR", "AMR", "EAS", "EUR", "SAS", "ACB", "ASW", "BEB", "CDX", "CEU", "CHB", "CHS", "CLM", "ESN", "FIN", "GBR", "GIH", "GWD", "IBS", "ITU", "JPT", "KHV", "LWK", "MSL", "MXL", "PEL", "PJL", "PUR", "STU", "TSI", "YRI"]:
pop_dirs.append(pop_dir + pop_i + ".txt")
else:
output["error"] = pop_i + " is not an ancestral population. Choose one of the following ancestral populations: AFR, AMR, EAS, EUR, or SAS; or one of the following sub-populations: ACB, ASW, BEB, CDX, CEU, CHB, CHS, CLM, ESN, FIN, GBR, GIH, GWD, IBS, ITU, JPT, KHV, LWK, MSL, MXL, PEL, PJL, PUR, STU, TSI, or YRI."
json_output = json.dumps(output, sort_keys=True, indent=2)
print >> out_json, json_output
out_json.close()
return("", "")
raise
get_pops = "cat " + " ".join(pop_dirs) + " > " + \
tmp_dir + "pops_" + request + ".txt"
subprocess.call(get_pops, shell=True)
# Get population ids
pop_list = open(tmp_dir + "pops_" + request + ".txt").readlines()
ids = []
for i in range(len(pop_list)):
ids.append(pop_list[i].strip())
pop_ids = list(set(ids))
# Extract query SNP phased genotypes
vcf_file = vcf_dir + \
snp_coord[
1] + ".phase3_shapeit2_mvncall_integrated_v5.20130502.genotypes.vcf.gz"
tabix_snp_h = "tabix -H {0} | grep CHROM".format(vcf_file)
proc_h = subprocess.Popen(tabix_snp_h, shell=True, stdout=subprocess.PIPE)
head = proc_h.stdout.readlines()[0].strip().split()
tabix_snp = "tabix {0} {1}:{2}-{2} | grep -v -e END > {3}".format(
vcf_file, snp_coord[1], snp_coord[2], tmp_dir + "snp_no_dups_" + request + ".vcf")
subprocess.call(tabix_snp, shell=True)
# Check SNP is in the 1000G population, has the correct RS number, and not
# monoallelic
vcf = open(tmp_dir + "snp_no_dups_" + request + ".vcf").readlines()
if len(vcf) == 0:
output["error"] = snp + " is not in 1000G reference panel."
json_output = json.dumps(output, sort_keys=True, indent=2)
print >> out_json, json_output
out_json.close()
subprocess.call("rm " + tmp_dir + "pops_" +
request + ".txt", shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
return("", "")
raise
elif len(vcf) > 1:
geno = []
for i in range(len(vcf)):
if vcf[i].strip().split()[2] == snp:
geno = vcf[i].strip().split()
if geno == []:
output["error"] = snp + " is not in 1000G reference panel."
json_output = json.dumps(output, sort_keys=True, indent=2)
print >> out_json, json_output
out_json.close()
subprocess.call("rm " + tmp_dir + "pops_" +
request + ".txt", shell=True)
subprocess.call("rm " + tmp_dir + "*" +
request + "*.vcf", shell=True)
return("", "")
raise
else:
geno = vcf[0].strip().split()
if geno[2] != snp:
output["warning"] = "Genomic position for query variant (" + snp + \
") does not match RS number at 1000G position (chr"+geno[0]+":"+geno[1]+")"
snp = geno[2]
if "," in geno[3] or "," in geno[4]:
output["error"] = snp + " is not a biallelic variant."
json_output = json.dumps(output, sort_keys=True, indent=2)
print >> out_json, json_output
out_json.close()
subprocess.call("rm " + tmp_dir + "pops_" +
request + ".txt", shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
return("", "")
raise
index = []
for i in range(9, len(head)):
if head[i] in pop_ids:
index.append(i)
genotypes = {"0": 0, "1": 0}
for i in index:
sub_geno = geno[i].split("|")
for j in sub_geno:
if j in genotypes:
genotypes[j] += 1
else:
genotypes[j] = 1
if genotypes["0"] == 0 or genotypes["1"] == 0:
output["error"] = snp + \
" is monoallelic in the " + pop + " population."
json_output = json.dumps(output, sort_keys=True, indent=2)
print >> out_json, json_output
out_json.close()
subprocess.call("rm " + tmp_dir + "pops_" +
request + ".txt", shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
return("", "")
raise
# Define window of interest around query SNP
window = 500000
coord1 = int(snp_coord[2]) - window
if coord1 < 0:
coord1 = 0
coord2 = int(snp_coord[2]) + window
print ""
# Calculate proxy LD statistics in parallel
threads = 4
block = (2 * window) / 4
commands = []
for i in range(threads):
if i == min(range(threads)) and i == max(range(threads)):
command = "python LDproxy_sub.py " + snp + " " + \
snp_coord[1] + " " + str(coord1) + " " + \
str(coord2) + " " + request + " " + str(i)
elif i == min(range(threads)):
command = "python LDproxy_sub.py " + snp + " " + \
snp_coord[1] + " " + str(coord1) + " " + \
str(coord1 + block) + " " + request + " " + str(i)
elif i == max(range(threads)):
command = "python LDproxy_sub.py " + snp + " " + snp_coord[1] + " " + str(
coord1 + (block * i) + 1) + " " + str(coord2) + " " + request + " " + str(i)
else:
command = "python LDproxy_sub.py " + snp + " " + snp_coord[1] + " " + str(coord1 + (
block * i) + 1) + " " + str(coord1 + (block * (i + 1))) + " " + request + " " + str(i)
commands.append(command)
processes = [subprocess.Popen(
command, shell=True, stdout=subprocess.PIPE) for command in commands]
# collect output in parallel
def get_output(process):
return process.communicate()[0].splitlines()
if not hasattr(threading.current_thread(), "_children"):
threading.current_thread()._children = weakref.WeakKeyDictionary()
pool = Pool(len(processes))
out_raw = pool.map(get_output, processes)
pool.close()
pool.join()
# Aggregate output
out_prox = []
for i in range(len(out_raw)):
for j in range(len(out_raw[i])):
col = out_raw[i][j].strip().split("\t")
col[6] = int(col[6])
col[7] = float(col[7])
col[8] = float(col[8])
col.append(abs(int(col[6])))
out_prox.append(col)
# Sort output
if r2_d not in ["r2", "d"]:
if "warning" in output:
output["warning"] = output["warning"] + ". " + r2_d + \
" is not an acceptable value for r2_d (r2 or d required). r2 is used by default"
else:
output["warning"] = r2_d + \
" is not an acceptable value for r2_d (r2 or d required). r2 is used by default"
r2_d = "r2"
out_dist_sort = sorted(out_prox, key=operator.itemgetter(14))
if r2_d == "r2":
out_ld_sort = sorted(
out_dist_sort, key=operator.itemgetter(8), reverse=True)
else:
out_ld_sort = sorted(
out_dist_sort, key=operator.itemgetter(7), reverse=True)
# Populate JSON and text output
outfile = open(tmp_dir + "proxy" + request + ".txt", "w")
header = ["RS_Number", "Coord", "Alleles", "MAF", "Distance",
"Dprime", "R2", "Correlated_Alleles", "RegulomeDB", "Function"]
print >> outfile, "\t".join(header)
ucsc_track = {}
ucsc_track["header"] = ["chr", "pos", "rsid", "stat"]
query_snp = {}
query_snp["RS"] = out_ld_sort[0][3]
query_snp["Alleles"] = out_ld_sort[0][1]
query_snp["Coord"] = out_ld_sort[0][2]
query_snp["Dist"] = out_ld_sort[0][6]
query_snp["Dprime"] = str(round(float(out_ld_sort[0][7]), 4))
query_snp["R2"] = str(round(float(out_ld_sort[0][8]), 4))
query_snp["Corr_Alleles"] = out_ld_sort[0][9]
query_snp["RegulomeDB"] = out_ld_sort[0][10]
query_snp["MAF"] = str(round(float(out_ld_sort[0][11]), 4))
query_snp["Function"] = out_ld_sort[0][13]
output["query_snp"] = query_snp
temp = [query_snp["RS"], query_snp["Coord"], query_snp["Alleles"], query_snp["MAF"], str(query_snp["Dist"]), str(
query_snp["Dprime"]), str(query_snp["R2"]), query_snp["Corr_Alleles"], query_snp["RegulomeDB"], query_snp["Function"]]
print >> outfile, "\t".join(temp)
chr, pos = query_snp["Coord"].split(':')
if r2_d == "r2":
temp2 = [chr, pos, query_snp["RS"], query_snp["R2"]]
else:
temp2 = [chr, pos, query_snp["RS"], query_snp["Dprime"]]
ucsc_track["query_snp"] = temp2
ucsc_track["0.8-1.0"] = []
ucsc_track["0.6-0.8"] = []
ucsc_track["0.4-0.6"] = []
ucsc_track["0.2-0.4"] = []
ucsc_track["0.0-0.2"] = []
proxies = {}
rows = []
digits = len(str(len(out_ld_sort)))
for i in range(1, len(out_ld_sort)):
if float(out_ld_sort[i][8]) > 0.01 and out_ld_sort[i][3] != snp:
proxy_info = {}
row = []
proxy_info["RS"] = out_ld_sort[i][3]
proxy_info["Alleles"] = out_ld_sort[i][4]
proxy_info["Coord"] = out_ld_sort[i][5]
proxy_info["Dist"] = out_ld_sort[i][6]
proxy_info["Dprime"] = str(round(float(out_ld_sort[i][7]), 4))
proxy_info["R2"] = str(round(float(out_ld_sort[i][8]), 4))
proxy_info["Corr_Alleles"] = out_ld_sort[i][9]
proxy_info["RegulomeDB"] = out_ld_sort[i][10]
proxy_info["MAF"] = str(round(float(out_ld_sort[i][12]), 4))
proxy_info["Function"] = out_ld_sort[i][13]
proxies["proxy_" + (digits - len(str(i))) *
"0" + str(i)] = proxy_info
chr, pos = proxy_info["Coord"].split(':')
# Adding a row for the Data Table
row.append(proxy_info["RS"])
row.append(chr)
row.append(pos)
row.append(proxy_info["Alleles"])
row.append(str(round(float(proxy_info["MAF"]), 4)))
row.append(abs(proxy_info["Dist"]))
row.append(str(round(float(proxy_info["Dprime"]), 4)))
row.append(str(round(float(proxy_info["R2"]), 4)))
row.append(proxy_info["Corr_Alleles"])
row.append(proxy_info["RegulomeDB"])
row.append("HaploReg link")
row.append(proxy_info["Function"])
rows.append(row)
temp = [proxy_info["RS"], proxy_info["Coord"], proxy_info["Alleles"], proxy_info["MAF"], str(proxy_info["Dist"]), str(
proxy_info["Dprime"]), str(proxy_info["R2"]), proxy_info["Corr_Alleles"], proxy_info["RegulomeDB"], proxy_info["Function"]]
print >> outfile, "\t".join(temp)
chr, pos = proxy_info["Coord"].split(':')
if r2_d == "r2":
temp2 = [chr, pos, proxy_info["RS"],
round(float(out_ld_sort[i][8]), 4)]
else:
temp2 = [chr, pos, proxy_info["RS"],
round(float(out_ld_sort[i][7]), 4)]
if 0.8 < temp2[3] <= 1.0:
ucsc_track["0.8-1.0"].append(temp2)
elif 0.6 < temp2[3] <= 0.8:
ucsc_track["0.6-0.8"].append(temp2)
elif 0.4 < temp2[3] <= 0.6:
ucsc_track["0.4-0.6"].append(temp2)
elif 0.2 < temp2[3] <= 0.4:
ucsc_track["0.2-0.4"].append(temp2)
else:
ucsc_track["0.0-0.2"].append(temp2)
track = open(tmp_dir + "track" + request + ".txt", "w")
print >> track, "browser position chr" + \
str(snp_coord[1]) + ":" + str(coord1) + "-" + str(coord2)
print >> track, ""
if r2_d == "r2":
print >> track, "track type=bedGraph name=\"R2 Plot\" description=\"Plot of R2 values\" color=50,50,50 visibility=full alwaysZero=on graphType=bar maxHeightPixels=60"
else:
print >> track, "track type=bedGraph name=\"D Prime Plot\" description=\"Plot of D prime values\" color=50,50,50 visibility=full alwaysZero=on graphType=bar maxHeightPixels=60"
print >> track, "\t".join(
[str(ucsc_track["query_snp"][i]) for i in [0, 1, 1, 3]])
if len(ucsc_track["0.8-1.0"]) > 0:
for var in ucsc_track["0.8-1.0"]:
print >> track, "\t".join([str(var[i]) for i in [0, 1, 1, 3]])
if len(ucsc_track["0.6-0.8"]) > 0:
for var in ucsc_track["0.6-0.8"]:
print >> track, "\t".join([str(var[i]) for i in [0, 1, 1, 3]])
if len(ucsc_track["0.4-0.6"]) > 0:
for var in ucsc_track["0.4-0.6"]:
print >> track, "\t".join([str(var[i]) for i in [0, 1, 1, 3]])
if len(ucsc_track["0.2-0.4"]) > 0:
for var in ucsc_track["0.2-0.4"]:
print >> track, "\t".join([str(var[i]) for i in [0, 1, 1, 3]])
if len(ucsc_track["0.0-0.2"]) > 0:
for var in ucsc_track["0.0-0.2"]:
print >> track, "\t".join([str(var[i]) for i in [0, 1, 1, 3]])
print >> track, ""
print >> track, "track type=bed name=\"" + snp + \
"\" description=\"Query Variant: " + snp + "\" color=108,108,255"
print >> track, "\t".join([ucsc_track["query_snp"][i]
for i in [0, 1, 1, 2]])
print >> track, ""
if len(ucsc_track["0.8-1.0"]) > 0:
if r2_d == "r2":
print >> track, "track type=bed name=\"0.8<R2<=1.0\" description=\"Proxy Variants with 0.8<R2<=1.0\" color=198,129,0"
else:
print >> track, "track type=bed name=\"0.8<D'<=1.0\" description=\"Proxy Variants with 0.8<D'<=1.0\" color=198,129,0"
for var in ucsc_track["0.8-1.0"]:
print >> track, "\t".join([var[i] for i in [0, 1, 1, 2]])
print >> track, ""
if len(ucsc_track["0.6-0.8"]) > 0:
if r2_d == "r2":
print >> track, "track type=bed name=\"0.6<R2<=0.8\" description=\"Proxy Variants with 0.6<R2<=0.8\" color=198,129,0"
else:
print >> track, "track type=bed name=\"0.6<D'<=0.8\" description=\"Proxy Variants with 0.6<D'<=0.8\" color=198,129,0"
for var in ucsc_track["0.6-0.8"]:
print >> track, "\t".join([var[i] for i in [0, 1, 1, 2]])
print >> track, ""
if len(ucsc_track["0.4-0.6"]) > 0:
if r2_d == "r2":
print >> track, "track type=bed name=\"0.4<R2<=0.6\" description=\"Proxy Variants with 0.4<R2<=0.6\" color=198,129,0"
else:
print >> track, "track type=bed name=\"0.4<D'<=0.6\" description=\"Proxy Variants with 0.4<D'<=0.6\" color=198,129,0"
for var in ucsc_track["0.4-0.6"]:
print >> track, "\t".join([var[i] for i in [0, 1, 1, 2]])
print >> track, ""
if len(ucsc_track["0.2-0.4"]) > 0:
if r2_d == "r2":
print >> track, "track type=bed name=\"0.2<R2<=0.4\" description=\"Proxy Variants with 0.2<R2<=0.4\" color=198,129,0"
else:
print >> track, "track type=bed name=\"0.2<D'<=0.4\" description=\"Proxy Variants with 0.2<D'<=0.4\" color=198,129,0"
for var in ucsc_track["0.2-0.4"]:
print >> track, "\t".join([var[i] for i in [0, 1, 1, 2]])
print >> track, ""
if len(ucsc_track["0.0-0.2"]) > 0:
if r2_d == "r2":
print >> track, "track type=bed name=\"0.0<R2<=0.2\" description=\"Proxy Variants with 0.0<R2<=0.2\" color=198,129,0"
else:
print >> track, "track type=bed name=\"0.0<D'<=0.2\" description=\"Proxy Variants with 0.0<D'<=0.2\" color=198,129,0"
for var in ucsc_track["0.0-0.2"]:
print >> track, "\t".join([var[i] for i in [0, 1, 1, 2]])
print >> track, ""
output["aaData"] = rows
output["proxy_snps"] = proxies
# Output JSON and text file
json_output = json.dumps(output, sort_keys=True, indent=2)
print >> out_json, json_output
out_json.close()
outfile.close()
track.close()
# Organize scatter plot data
q_rs = []
q_allele = []
q_coord = []
q_maf = []
p_rs = []
p_allele = []
p_coord = []
p_maf = []
dist = []
d_prime = []
d_prime_round = []
r2 = []
r2_round = []
corr_alleles = []
regdb = []
funct = []
color = []
size = []
for i in range(len(out_ld_sort)):
q_rs_i, q_allele_i, q_coord_i, p_rs_i, p_allele_i, p_coord_i, dist_i, d_prime_i, r2_i, corr_alleles_i, regdb_i, q_maf_i, p_maf_i, funct_i, dist_abs = out_ld_sort[
i]
if float(r2_i) > 0.01:
q_rs.append(q_rs_i)
q_allele.append(q_allele_i)
q_coord.append(float(q_coord_i.split(":")[1]) / 1000000)
q_maf.append(str(round(float(q_maf_i), 4)))
if p_rs_i == ".":
p_rs_i = p_coord_i
p_rs.append(p_rs_i)
p_allele.append(p_allele_i)
p_coord.append(float(p_coord_i.split(":")[1]) / 1000000)
p_maf.append(str(round(float(p_maf_i), 4)))
dist.append(str(round(dist_i / 1000000.0, 4)))
d_prime.append(float(d_prime_i))
d_prime_round.append(str(round(float(d_prime_i), 4)))
r2.append(float(r2_i))
r2_round.append(str(round(float(r2_i), 4)))
corr_alleles.append(corr_alleles_i)
# Correct Missing Annotations
if regdb_i == ".":
regdb_i = ""
regdb.append(regdb_i)
if funct_i == ".":
funct_i = ""
if funct_i == "NA":
funct_i = "none"
funct.append(funct_i)
# Set Color
if i == 0:
color_i = "blue"
elif funct_i != "none" and funct_i != "":
color_i = "red"
else:
color_i = "orange"
color.append(color_i)
# Set Size
size_i = 9 + float(p_maf_i) * 14.0
size.append(size_i)
# Begin Bokeh Plotting
from collections import OrderedDict
from bokeh.embed import components, file_html
from bokeh.layouts import gridplot
from bokeh.models import HoverTool, LinearAxis, Range1d
from bokeh.plotting import ColumnDataSource, curdoc, figure, output_file, reset_output, save
from bokeh.resources import CDN
reset_output()
# Proxy Plot
x = p_coord
if r2_d == "r2":
y = r2
else:
y = d_prime
whitespace = 0.01
xr = Range1d(start=coord1 / 1000000.0 - whitespace,
end=coord2 / 1000000.0 + whitespace)
yr = Range1d(start=-0.03, end=1.03)
sup_2 = u"\u00B2"
proxy_plot = figure(
title="Proxies for " + snp + " in " + pop,
min_border_top=2, min_border_bottom=2, min_border_left=60, min_border_right=60, h_symmetry=False, v_symmetry=False,
plot_width=900,
plot_height=600,
x_range=xr, y_range=yr,
tools="hover,tap,pan,box_zoom,box_select,undo,redo,reset,previewsave", logo=None,
toolbar_location="above")
proxy_plot.title.align = "center"
tabix_recomb = "tabix -fh {0} {1}:{2}-{3} > {4}".format(recomb_dir, snp_coord[
1], coord1 - whitespace, coord2 + whitespace, tmp_dir + "recomb_" + request + ".txt")
subprocess.call(tabix_recomb, shell=True)
filename = tmp_dir + "recomb_" + request + ".txt"
recomb_raw = open(filename).readlines()
recomb_x = []
recomb_y = []
for i in range(len(recomb_raw)):
chr, pos, rate = recomb_raw[i].strip().split()
recomb_x.append(int(pos) / 1000000.0)
recomb_y.append(float(rate) / 100.0)
data = {
'x': x,
'y': y,
'qrs': q_rs,
'q_alle': q_allele,
'q_maf': q_maf,
'prs': p_rs,
'p_alle': p_allele,
'p_maf': p_maf,
'dist': dist,
'r': r2_round,
'd': d_prime_round,
'alleles': corr_alleles,
'regdb': regdb,
'funct': funct,
'size': size,
'color': color
}
source = ColumnDataSource(data)
proxy_plot.line(recomb_x, recomb_y, line_width=1, color="black", alpha=0.5)
proxy_plot.circle(x='x', y='y', size='size', color='color', alpha=0.5, source=source)
hover = proxy_plot.select(dict(type=HoverTool))
hover.tooltips = OrderedDict([
("Query Variant", "@qrs @q_alle"),
("Proxy Variant", "@prs @p_alle"),
("Distance (Mb)", "@dist"),
("MAF (Query,Proxy)", "@q_maf,@p_maf"),
("R" + sup_2, "@r"),
("D\'", "@d"),
("Correlated Alleles", "@alleles"),
("RegulomeDB", "@regdb"),
("Functional Class", "@funct"),
])
proxy_plot.text(x, y, text=regdb, alpha=1, text_font_size="7pt",
text_baseline="middle", text_align="center", angle=0)
if r2_d == "r2":
proxy_plot.yaxis.axis_label = "R" + sup_2
else:
proxy_plot.yaxis.axis_label = "D\'"
proxy_plot.extra_y_ranges = {"y2_axis": Range1d(start=-3, end=103)}
proxy_plot.add_layout(LinearAxis(y_range_name="y2_axis",
axis_label="Combined Recombination Rate (cM/Mb)"), "right")
# Rug Plot
y2_ll = [-0.03] * len(x)
y2_ul = [1.03] * len(x)
yr_rug = Range1d(start=-0.03, end=1.03)
data_rug = {
'x': x,
'y': y,
'y2_ll': y2_ll,
'y2_ul': y2_ul,
'qrs': q_rs,
'q_alle': q_allele,
'q_maf': q_maf,
'prs': p_rs,
'p_alle': p_allele,
'p_maf': p_maf,
'dist': dist,
'r': r2_round,
'd': d_prime_round,
'alleles': corr_alleles,
'regdb': regdb,
'funct': funct,
'size': size,
'color': color
}
source_rug = ColumnDataSource(data_rug)
rug = figure(
x_range=xr, y_range=yr_rug, border_fill_color='white', y_axis_type=None,
title="", min_border_top=2, min_border_bottom=2, min_border_left=60, min_border_right=60, h_symmetry=False, v_symmetry=False,
plot_width=900, plot_height=50, tools="xpan,tap", logo=None)
rug.segment(x0='x', y0='y2_ll', x1='x', y1='y2_ul', source=source_rug,
color='color', alpha=0.5, line_width=1)
rug.toolbar_location = None
# Gene Plot
tabix_gene = "tabix -fh {0} {1}:{2}-{3} > {4}".format(
gene_dir, snp_coord[1], coord1, coord2, tmp_dir + "genes_" + request + ".txt")
subprocess.call(tabix_gene, shell=True)
filename = tmp_dir + "genes_" + request + ".txt"
genes_raw = open(filename).readlines()
genes_plot_start = []
genes_plot_end = []
genes_plot_y = []
genes_plot_name = []
exons_plot_x = []
exons_plot_y = []
exons_plot_w = []
exons_plot_h = []
exons_plot_name = []
exons_plot_id = []
exons_plot_exon = []
lines = [0]
gap = 80000
tall = 0.75
if genes_raw != None:
for i in range(len(genes_raw)):
bin, name_id, chrom, strand, txStart, txEnd, cdsStart, cdsEnd, exonCount, exonStarts, exonEnds, score, name2, cdsStartStat, cdsEndStat, exonFrames = genes_raw[
i].strip().split()
name = name2
id = name_id
e_start = exonStarts.split(",")
e_end = exonEnds.split(",")
# Determine Y Coordinate
i = 0
y_coord = None
while y_coord == None:
if i > len(lines) - 1:
y_coord = i + 1
lines.append(int(txEnd))
elif int(txStart) > (gap + lines[i]):
y_coord = i + 1
lines[i] = int(txEnd)
else:
i += 1
genes_plot_start.append(int(txStart) / 1000000.0)
genes_plot_end.append(int(txEnd) / 1000000.0)
genes_plot_y.append(y_coord)
genes_plot_name.append(name + " ")
for i in range(len(e_start) - 1):
if strand == "+":
exon = i + 1
else:
exon = len(e_start) - 1 - i
width = (int(e_end[i]) - int(e_start[i])) / 1000000.0
x_coord = int(e_start[i]) / 1000000.0 + (width / 2)
exons_plot_x.append(x_coord)
exons_plot_y.append(y_coord)
exons_plot_w.append(width)
exons_plot_h.append(tall)
exons_plot_name.append(name)
exons_plot_id.append(id)
exons_plot_exon.append(exon)
n_rows = len(lines)
genes_plot_yn = [n_rows - x + 0.5 for x in genes_plot_y]
exons_plot_yn = [n_rows - x + 0.5 for x in exons_plot_y]
yr2 = Range1d(start=0, end=n_rows)
data_gene_plot = {
'exons_plot_x': exons_plot_x,
'exons_plot_yn': exons_plot_yn,
'exons_plot_w': exons_plot_w,
'exons_plot_h': exons_plot_h,
'exons_plot_name': exons_plot_name,
'exons_plot_id': exons_plot_id,
'exons_plot_exon': exons_plot_exon
}
source_gene_plot = ColumnDataSource(data_gene_plot)
if len(lines) < 3:
plot_h_pix = 150
else:
plot_h_pix = 150 + (len(lines) - 2) * 50
gene_plot = figure(
x_range=xr, y_range=yr2, border_fill_color='white',
title="", min_border_top=2, min_border_bottom=2, min_border_left=60, min_border_right=60, h_symmetry=False, v_symmetry=False,
plot_width=900, plot_height=plot_h_pix, tools="hover,tap,xpan,box_zoom,undo,redo,reset,previewsave", logo=None)
gene_plot.segment(genes_plot_start, genes_plot_yn, genes_plot_end,
genes_plot_yn, color="black", alpha=1, line_width=2)
gene_plot.rect(x='exons_plot_x', y='exons_plot_yn', width='exons_plot_w', height='exons_plot_h',
source=source_gene_plot, fill_color="grey", line_color="grey")
gene_plot.xaxis.axis_label = "Chromosome " + \
snp_coord[1] + " Coordinate (Mb)(GRCh37)"
gene_plot.yaxis.axis_label = "Genes"
gene_plot.ygrid.grid_line_color = None
gene_plot.yaxis.axis_line_color = None
gene_plot.yaxis.minor_tick_line_color = None
gene_plot.yaxis.major_tick_line_color = None
gene_plot.yaxis.major_label_text_color = None
hover = gene_plot.select(dict(type=HoverTool))
hover.tooltips = OrderedDict([
("Gene", "@exons_plot_name"),
("ID", "@exons_plot_id"),
("Exon", "@exons_plot_exon"),
])
gene_plot.text(genes_plot_start, genes_plot_yn, text=genes_plot_name, alpha=1, text_font_size="7pt",
text_font_style="bold", text_baseline="middle", text_align="right", angle=0)
gene_plot.toolbar_location = "below"
# Combine plots into a grid
out_grid = gridplot(proxy_plot, rug, gene_plot, ncols=1,
toolbar_options=dict(logo=None))
# Generate high quality images only if accessed via web instance
if web:
# Open thread for high quality image exports
command = "python LDproxy_plot_sub.py " + snp + " " + pop + " " + request + " " + r2_d
subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
###########################
# Html output for testing #
###########################
#html=file_html(out_grid, CDN, "Test Plot")
# out_html=open("LDproxy.html","w")
#print >> out_html, html
# out_html.close()
out_script, out_div = components(out_grid, CDN)
reset_output()
# Print run time statistics
pop_list = open(tmp_dir + "pops_" + request + ".txt").readlines()
print "\nNumber of Individuals: " + str(len(pop_list))
print "SNPs in Region: " + str(len(out_prox))
duration = time.time() - start_time
print "Run time: " + str(duration) + " seconds\n"
# Remove temporary files
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt", shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
subprocess.call("rm " + tmp_dir + "genes_" + request + ".txt", shell=True)
subprocess.call("rm " + tmp_dir + "recomb_" + request + ".txt", shell=True)
# Return plot output
return(out_script, out_div)
def test_label(output_file_url, selenium, screenshot):
# Have to specify x/y range as labels aren't included in the plot area solver
plot = Plot(plot_height=HEIGHT,
plot_width=WIDTH,
x_range=Range1d(0, 10),
y_range=Range1d(0, 10),
toolbar_location=None)
label1 = Label(x=1,
y=6,
x_offset=25,
y_offset=25,
text=["Demo Label"],
text_font_size='38pt',
text_color='red',
text_alpha=0.9,
text_baseline='bottom',
text_align='left',
background_fill_color='green',
background_fill_alpha=0.2,
angle=15,
angle_units='deg',
render_mode='canvas')
label2 = Label(x=3,
y=5.5,
text=["(I'm Canvas)"],
text_font_size='20pt',
border_line_color='black',
border_line_width=2,
border_line_dash='8 4',
render_mode='canvas')
label3 = Label(x=1,
y=2,
x_offset=25,
y_offset=25,
text=["Demo Label"],
text_font_size='38pt',
text_color='red',
text_alpha=0.9,
text_baseline='bottom',
text_align='left',
background_fill_color='green',
background_fill_alpha=0.2,
angle=0.261,
angle_units='rad',
render_mode='css')
label4 = Label(x=3,
y=1.0,
text=["(I'm CSS)"],
text_font_size='20pt',
border_line_color='black',
border_line_width=2,
border_line_dash='8 4',
render_mode='css')
label_above = Label(
x=0,
y=0,
text=["Label in above panel"],
x_units='screen',
y_units='screen',
text_font_size='38pt',
text_color='firebrick',
text_alpha=0.9,
)
label_left = Label(
x=0,
y=0,
text=["Label in left panel"],
x_units='screen',
y_units='screen',
angle=90,
angle_units='deg',
text_font_size='18pt',
text_color='firebrick',
text_alpha=0.9,
background_fill_color='aliceblue',
text_baseline='top',
)
plot.add_layout(LinearAxis(), 'below')
plot.add_layout(LinearAxis(), 'left')
plot.add_layout(label1)
plot.add_layout(label2)
plot.add_layout(label3)
plot.add_layout(label4)
plot.add_layout(label_above, 'above')
plot.add_layout(label_left, 'left')
# Save the plot and start the test
save(plot)
selenium.get(output_file_url)
# Take screenshot
assert screenshot.is_valid()
def test_plot_add_layout_adds_axis_to_renderers_and_side_renderers():
plot = figure()
axis = LinearAxis()
plot.add_layout(axis, 'left')
assert axis in plot.left
x_axis_label='wavelength(nm)',
y_axis_label='polarizability',
# x_axis_type='linear',
# y_axis_type='linear',
x_axis_type=None,
y_axis_type=None,
tools="pan,wheel_zoom,box_zoom,zoom_in,zoom_out,save,hover,crosshair")
p.x_range = Range1d(1100, 1800)
p.y_range = Range1d(-5000, 5000)
# ticker = SingleIntervalTicker(interval=100, desired_num_ticks=40,num_minor_ticks=10)
ticker = BasicTicker(max_interval=100, min_interval=10, desired_num_ticks=10)
# ticker = ContinuousTicker(desired_num_ticks=20,num_minor_ticks=10)
xaxis = LinearAxis(ticker=ticker)
# p.xaxis.visible = True
xaxis.axis_label = "wavelength (nm)"
xaxis.axis_line_width = 1
xaxis.axis_label_text_font_style = "italic"
p.add_layout(xaxis, 'below')
# tickery = SingleIntervalTicker(interval=1000, num_minor_ticks=20)
tickery = BasicTicker(max_interval=1000,
min_interval=100,
desired_num_ticks=10)
yaxis = LinearAxis(ticker=tickery)
# Disable scientific notation on yaxis
yaxis.formatter.use_scientific = False
yaxis.axis_label = "polarizability (a.u.)"
def create():
det_data = {}
roi_selection = {}
upload_div = Div(text="Open .cami file:")
def upload_button_callback(_attr, _old, new):
with io.StringIO(base64.b64decode(new).decode()) as file:
h5meta_list = pyzebra.parse_h5meta(file)
file_list = h5meta_list["filelist"]
filelist.options = file_list
filelist.value = file_list[0]
upload_button = FileInput(accept=".cami")
upload_button.on_change("value", upload_button_callback)
def update_image(index=None):
if index is None:
index = index_spinner.value
current_image = det_data["data"][index]
proj_v_line_source.data.update(x=np.arange(0, IMAGE_W) + 0.5,
y=np.mean(current_image, axis=0))
proj_h_line_source.data.update(x=np.mean(current_image, axis=1),
y=np.arange(0, IMAGE_H) + 0.5)
image_source.data.update(
h=[np.zeros((1, 1))],
k=[np.zeros((1, 1))],
l=[np.zeros((1, 1))],
)
image_source.data.update(image=[current_image])
if auto_toggle.active:
im_max = int(np.max(current_image))
im_min = int(np.min(current_image))
display_min_spinner.value = im_min
display_max_spinner.value = im_max
image_glyph.color_mapper.low = im_min
image_glyph.color_mapper.high = im_max
def update_overview_plot():
h5_data = det_data["data"]
n_im, n_y, n_x = h5_data.shape
overview_x = np.mean(h5_data, axis=1)
overview_y = np.mean(h5_data, axis=2)
overview_plot_x_image_source.data.update(image=[overview_x], dw=[n_x])
overview_plot_y_image_source.data.update(image=[overview_y], dw=[n_y])
if frame_button_group.active == 0: # Frame
overview_plot_x.axis[1].axis_label = "Frame"
overview_plot_y.axis[1].axis_label = "Frame"
overview_plot_x_image_source.data.update(y=[0], dh=[n_im])
overview_plot_y_image_source.data.update(y=[0], dh=[n_im])
elif frame_button_group.active == 1: # Omega
overview_plot_x.axis[1].axis_label = "Omega"
overview_plot_y.axis[1].axis_label = "Omega"
om = det_data["rot_angle"]
om_start = om[0]
om_end = (om[-1] - om[0]) * n_im / (n_im - 1)
overview_plot_x_image_source.data.update(y=[om_start], dh=[om_end])
overview_plot_y_image_source.data.update(y=[om_start], dh=[om_end])
def filelist_callback(_attr, _old, new):
nonlocal det_data
det_data = pyzebra.read_detector_data(new)
index_spinner.value = 0
index_spinner.high = det_data["data"].shape[0] - 1
update_image(0)
update_overview_plot()
filelist = Select()
filelist.on_change("value", filelist_callback)
def index_spinner_callback(_attr, _old, new):
update_image(new)
index_spinner = Spinner(title="Image index:", value=0, low=0)
index_spinner.on_change("value", index_spinner_callback)
plot = Plot(
x_range=Range1d(0, IMAGE_W, bounds=(0, IMAGE_W)),
y_range=Range1d(0, IMAGE_H, bounds=(0, IMAGE_H)),
plot_height=IMAGE_H * 3,
plot_width=IMAGE_W * 3,
toolbar_location="left",
)
# ---- tools
plot.toolbar.logo = None
# ---- axes
plot.add_layout(LinearAxis(), place="above")
plot.add_layout(LinearAxis(major_label_orientation="vertical"),
place="right")
# ---- grid lines
plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- rgba image glyph
image_source = ColumnDataSource(
dict(
image=[np.zeros((IMAGE_H, IMAGE_W), dtype="float32")],
h=[np.zeros((1, 1))],
k=[np.zeros((1, 1))],
l=[np.zeros((1, 1))],
x=[0],
y=[0],
dw=[IMAGE_W],
dh=[IMAGE_H],
))
h_glyph = Image(image="h", x="x", y="y", dw="dw", dh="dh", global_alpha=0)
k_glyph = Image(image="k", x="x", y="y", dw="dw", dh="dh", global_alpha=0)
l_glyph = Image(image="l", x="x", y="y", dw="dw", dh="dh", global_alpha=0)
plot.add_glyph(image_source, h_glyph)
plot.add_glyph(image_source, k_glyph)
plot.add_glyph(image_source, l_glyph)
image_glyph = Image(image="image", x="x", y="y", dw="dw", dh="dh")
plot.add_glyph(image_source, image_glyph, name="image_glyph")
# ---- projections
proj_v = Plot(
x_range=plot.x_range,
y_range=DataRange1d(),
plot_height=200,
plot_width=IMAGE_W * 3,
toolbar_location=None,
)
proj_v.add_layout(LinearAxis(major_label_orientation="vertical"),
place="right")
proj_v.add_layout(LinearAxis(major_label_text_font_size="0pt"),
place="below")
proj_v.add_layout(Grid(dimension=0, ticker=BasicTicker()))
proj_v.add_layout(Grid(dimension=1, ticker=BasicTicker()))
proj_v_line_source = ColumnDataSource(dict(x=[], y=[]))
proj_v.add_glyph(proj_v_line_source,
Line(x="x", y="y", line_color="steelblue"))
proj_h = Plot(
x_range=DataRange1d(),
y_range=plot.y_range,
plot_height=IMAGE_H * 3,
plot_width=200,
toolbar_location=None,
)
proj_h.add_layout(LinearAxis(), place="above")
proj_h.add_layout(LinearAxis(major_label_text_font_size="0pt"),
place="left")
proj_h.add_layout(Grid(dimension=0, ticker=BasicTicker()))
proj_h.add_layout(Grid(dimension=1, ticker=BasicTicker()))
proj_h_line_source = ColumnDataSource(dict(x=[], y=[]))
proj_h.add_glyph(proj_h_line_source,
Line(x="x", y="y", line_color="steelblue"))
# add tools
hovertool = HoverTool(tooltips=[("intensity",
"@image"), ("h", "@h"), ("k",
"@k"), ("l",
"@l")])
box_edit_source = ColumnDataSource(dict(x=[], y=[], width=[], height=[]))
box_edit_glyph = Rect(x="x",
y="y",
width="width",
height="height",
fill_alpha=0,
line_color="red")
box_edit_renderer = plot.add_glyph(box_edit_source, box_edit_glyph)
boxedittool = BoxEditTool(renderers=[box_edit_renderer], num_objects=1)
def box_edit_callback(_attr, _old, new):
if new["x"]:
h5_data = det_data["data"]
x_val = np.arange(h5_data.shape[0])
left = int(np.floor(new["x"][0]))
right = int(np.ceil(new["x"][0] + new["width"][0]))
bottom = int(np.floor(new["y"][0]))
top = int(np.ceil(new["y"][0] + new["height"][0]))
y_val = np.sum(h5_data[:, bottom:top, left:right], axis=(1, 2))
else:
x_val = []
y_val = []
roi_avg_plot_line_source.data.update(x=x_val, y=y_val)
box_edit_source.on_change("data", box_edit_callback)
wheelzoomtool = WheelZoomTool(maintain_focus=False)
plot.add_tools(
PanTool(),
BoxZoomTool(),
wheelzoomtool,
ResetTool(),
hovertool,
boxedittool,
)
plot.toolbar.active_scroll = wheelzoomtool
# shared frame range
frame_range = DataRange1d()
det_x_range = DataRange1d()
overview_plot_x = Plot(
title=Title(text="Projections on X-axis"),
x_range=det_x_range,
y_range=frame_range,
plot_height=400,
plot_width=400,
toolbar_location="left",
)
# ---- tools
wheelzoomtool = WheelZoomTool(maintain_focus=False)
overview_plot_x.toolbar.logo = None
overview_plot_x.add_tools(
PanTool(),
BoxZoomTool(),
wheelzoomtool,
ResetTool(),
)
overview_plot_x.toolbar.active_scroll = wheelzoomtool
# ---- axes
overview_plot_x.add_layout(LinearAxis(axis_label="Coordinate X, pix"),
place="below")
overview_plot_x.add_layout(LinearAxis(axis_label="Frame",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
overview_plot_x.add_layout(Grid(dimension=0, ticker=BasicTicker()))
overview_plot_x.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- rgba image glyph
overview_plot_x_image_source = ColumnDataSource(
dict(image=[np.zeros((1, 1), dtype="float32")],
x=[0],
y=[0],
dw=[1],
dh=[1]))
overview_plot_x_image_glyph = Image(image="image",
x="x",
y="y",
dw="dw",
dh="dh")
overview_plot_x.add_glyph(overview_plot_x_image_source,
overview_plot_x_image_glyph,
name="image_glyph")
det_y_range = DataRange1d()
overview_plot_y = Plot(
title=Title(text="Projections on Y-axis"),
x_range=det_y_range,
y_range=frame_range,
plot_height=400,
plot_width=400,
toolbar_location="left",
)
# ---- tools
wheelzoomtool = WheelZoomTool(maintain_focus=False)
overview_plot_y.toolbar.logo = None
overview_plot_y.add_tools(
PanTool(),
BoxZoomTool(),
wheelzoomtool,
ResetTool(),
)
overview_plot_y.toolbar.active_scroll = wheelzoomtool
# ---- axes
overview_plot_y.add_layout(LinearAxis(axis_label="Coordinate Y, pix"),
place="below")
overview_plot_y.add_layout(LinearAxis(axis_label="Frame",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
overview_plot_y.add_layout(Grid(dimension=0, ticker=BasicTicker()))
overview_plot_y.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- rgba image glyph
overview_plot_y_image_source = ColumnDataSource(
dict(image=[np.zeros((1, 1), dtype="float32")],
x=[0],
y=[0],
dw=[1],
dh=[1]))
overview_plot_y_image_glyph = Image(image="image",
x="x",
y="y",
dw="dw",
dh="dh")
overview_plot_y.add_glyph(overview_plot_y_image_source,
overview_plot_y_image_glyph,
name="image_glyph")
def frame_button_group_callback(_active):
update_overview_plot()
frame_button_group = RadioButtonGroup(labels=["Frames", "Omega"], active=0)
frame_button_group.on_click(frame_button_group_callback)
roi_avg_plot = Plot(
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=IMAGE_H * 3,
plot_width=IMAGE_W * 3,
toolbar_location="left",
)
# ---- tools
roi_avg_plot.toolbar.logo = None
# ---- axes
roi_avg_plot.add_layout(LinearAxis(), place="below")
roi_avg_plot.add_layout(LinearAxis(major_label_orientation="vertical"),
place="left")
# ---- grid lines
roi_avg_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
roi_avg_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
roi_avg_plot_line_source = ColumnDataSource(dict(x=[], y=[]))
roi_avg_plot.add_glyph(roi_avg_plot_line_source,
Line(x="x", y="y", line_color="steelblue"))
cmap_dict = {
"gray": Greys256,
"gray_reversed": Greys256[::-1],
"plasma": Plasma256,
"cividis": Cividis256,
}
def colormap_callback(_attr, _old, new):
image_glyph.color_mapper = LinearColorMapper(palette=cmap_dict[new])
overview_plot_x_image_glyph.color_mapper = LinearColorMapper(
palette=cmap_dict[new])
overview_plot_y_image_glyph.color_mapper = LinearColorMapper(
palette=cmap_dict[new])
colormap = Select(title="Colormap:", options=list(cmap_dict.keys()))
colormap.on_change("value", colormap_callback)
colormap.value = "plasma"
radio_button_group = RadioButtonGroup(labels=["nb", "nb_bi"], active=0)
STEP = 1
# ---- colormap auto toggle button
def auto_toggle_callback(state):
if state:
display_min_spinner.disabled = True
display_max_spinner.disabled = True
else:
display_min_spinner.disabled = False
display_max_spinner.disabled = False
update_image()
auto_toggle = Toggle(label="Auto Range",
active=True,
button_type="default")
auto_toggle.on_click(auto_toggle_callback)
# ---- colormap display max value
def display_max_spinner_callback(_attr, _old_value, new_value):
display_min_spinner.high = new_value - STEP
image_glyph.color_mapper.high = new_value
display_max_spinner = Spinner(
title="Maximal Display Value:",
low=0 + STEP,
value=1,
step=STEP,
disabled=auto_toggle.active,
)
display_max_spinner.on_change("value", display_max_spinner_callback)
# ---- colormap display min value
def display_min_spinner_callback(_attr, _old_value, new_value):
display_max_spinner.low = new_value + STEP
image_glyph.color_mapper.low = new_value
display_min_spinner = Spinner(
title="Minimal Display Value:",
high=1 - STEP,
value=0,
step=STEP,
disabled=auto_toggle.active,
)
display_min_spinner.on_change("value", display_min_spinner_callback)
def hkl_button_callback():
index = index_spinner.value
setup_type = "nb_bi" if radio_button_group.active else "nb"
h, k, l = calculate_hkl(det_data, index, setup_type)
image_source.data.update(h=[h], k=[k], l=[l])
hkl_button = Button(label="Calculate hkl (slow)")
hkl_button.on_click(hkl_button_callback)
selection_list = TextAreaInput(rows=7)
def selection_button_callback():
nonlocal roi_selection
selection = [
int(np.floor(det_x_range.start)),
int(np.ceil(det_x_range.end)),
int(np.floor(det_y_range.start)),
int(np.ceil(det_y_range.end)),
int(np.floor(frame_range.start)),
int(np.ceil(frame_range.end)),
]
filename_id = filelist.value[-8:-4]
if filename_id in roi_selection:
roi_selection[f"{filename_id}"].append(selection)
else:
roi_selection[f"{filename_id}"] = [selection]
selection_list.value = str(roi_selection)
selection_button = Button(label="Add selection")
selection_button.on_click(selection_button_callback)
# Final layout
layout_image = column(
gridplot([[proj_v, None], [plot, proj_h]], merge_tools=False),
row(index_spinner))
colormap_layout = column(colormap, auto_toggle, display_max_spinner,
display_min_spinner)
hkl_layout = column(radio_button_group, hkl_button)
layout_overview = column(
gridplot(
[[overview_plot_x, overview_plot_y]],
toolbar_options=dict(logo=None),
merge_tools=True,
),
frame_button_group,
)
tab_layout = row(
column(
upload_div,
upload_button,
filelist,
layout_image,
row(colormap_layout, hkl_layout),
),
column(
roi_avg_plot,
layout_overview,
row(selection_button, selection_list),
),
)
return Panel(child=tab_layout, title="Data Viewer")
def __init__(self, **kwargs):
names = [
'processes', 'disk-read', 'cores', 'cpu', 'disk-write', 'memory',
'last-seen', 'memory_percent', 'host'
]
self.source = ColumnDataSource({k: [] for k in names})
columns = {
name: TableColumn(field=name, title=name.replace('_percent', ' %'))
for name in names
}
cnames = [
'host', 'cores', 'processes', 'memory', 'cpu', 'memory_percent'
]
formatters = {
'cpu': NumberFormatter(format='0.0 %'),
'memory_percent': NumberFormatter(format='0.0 %'),
'memory': NumberFormatter(format='0 b'),
'latency': NumberFormatter(format='0.00000'),
'last-seen': NumberFormatter(format='0.000'),
'disk-read': NumberFormatter(format='0 b'),
'disk-write': NumberFormatter(format='0 b'),
'net-send': NumberFormatter(format='0 b'),
'net-recv': NumberFormatter(format='0 b')
}
table = DataTable(
source=self.source,
columns=[columns[n] for n in cnames],
)
for name in cnames:
if name in formatters:
table.columns[cnames.index(name)].formatter = formatters[name]
mem_plot = Plot(title=Title(text="Memory Usage (%)"),
toolbar_location=None,
x_range=Range1d(start=0, end=1),
y_range=Range1d(start=-0.1, end=0.1),
**kwargs)
mem_plot.add_glyph(
self.source,
Circle(x='memory_percent', y=0, size=10, fill_alpha=0.5))
mem_plot.add_layout(LinearAxis(), 'below')
hover = HoverTool(point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 10px; font-family: Monaco, monospace;">@host: </span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@memory_percent</span>
</div>
""")
mem_plot.add_tools(hover, BoxSelectTool())
if 'sizing_mode' in kwargs:
sizing_mode = {'sizing_mode': kwargs['sizing_mode']}
else:
sizing_mode = {}
self.root = column(mem_plot,
table,
id='bk-worker-table',
**sizing_mode)
def create(self):
manufacturers = sorted(mpg["manufacturer"].unique())
models = sorted(mpg["model"].unique())
transmissions = sorted(mpg["trans"].unique())
drives = sorted(mpg["drv"].unique())
classes = sorted(mpg["class"].unique())
manufacturer_select = Select(title="Manufacturer:",
value="All",
options=["All"] + manufacturers)
manufacturer_select.on_change('value', self.on_manufacturer_change)
model_select = Select(title="Model:",
value="All",
options=["All"] + models)
model_select.on_change('value', self.on_model_change)
transmission_select = Select(title="Transmission:",
value="All",
options=["All"] + transmissions)
transmission_select.on_change('value', self.on_transmission_change)
drive_select = Select(title="Drive:",
value="All",
options=["All"] + drives)
drive_select.on_change('value', self.on_drive_change)
class_select = Select(title="Class:",
value="All",
options=["All"] + classes)
class_select.on_change('value', self.on_class_change)
columns = [
TableColumn(field="manufacturer",
title="Manufacturer",
editor=SelectEditor(options=manufacturers),
formatter=StringFormatter(font_style="bold")),
TableColumn(field="model",
title="Model",
editor=StringEditor(completions=models)),
TableColumn(field="displ",
title="Displacement",
editor=NumberEditor(step=0.1),
formatter=NumberFormatter(format="0.0")),
TableColumn(field="year", title="Year", editor=IntEditor()),
TableColumn(field="cyl", title="Cylinders", editor=IntEditor()),
TableColumn(field="trans",
title="Transmission",
editor=SelectEditor(options=transmissions)),
TableColumn(field="drv",
title="Drive",
editor=SelectEditor(options=drives)),
TableColumn(field="class",
title="Class",
editor=SelectEditor(options=classes)),
TableColumn(field="cty", title="City MPG", editor=IntEditor()),
TableColumn(field="hwy", title="Highway MPG", editor=IntEditor()),
]
data_table = DataTable(source=self.source,
columns=columns,
editable=True)
plot = Plot(title=None,
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_width=1000,
plot_height=300)
# Set up x & y axis
plot.add_layout(LinearAxis(), 'below')
yaxis = LinearAxis()
plot.add_layout(yaxis, 'left')
plot.add_layout(Grid(dimension=1, ticker=yaxis.ticker))
# Add Glyphs
cty_glyph = Circle(x="index",
y="cty",
fill_color="#396285",
size=8,
fill_alpha=0.5,
line_alpha=0.5)
hwy_glyph = Circle(x="index",
y="hwy",
fill_color="#CE603D",
size=8,
fill_alpha=0.5,
line_alpha=0.5)
cty = plot.add_glyph(self.source, cty_glyph)
hwy = plot.add_glyph(self.source, hwy_glyph)
# Add the tools
tooltips = [
("Manufacturer", "@manufacturer"),
("Model", "@model"),
("Displacement", "@displ"),
("Year", "@year"),
("Cylinders", "@cyl"),
("Transmission", "@trans"),
("Drive", "@drv"),
("Class", "@class"),
]
cty_hover_tool = HoverTool(renderers=[cty],
tooltips=tooltips + [("City MPG", "@cty")])
hwy_hover_tool = HoverTool(renderers=[hwy],
tooltips=tooltips +
[("Highway MPG", "@hwy")])
select_tool = BoxSelectTool(renderers=[cty, hwy], dimensions=['width'])
plot.add_tools(cty_hover_tool, hwy_hover_tool, select_tool)
controls = VBox(children=[
manufacturer_select, model_select, transmission_select,
drive_select, class_select
])
top_panel = HBox(children=[controls, plot])
layout = VBox(children=[top_panel, data_table])
return layout
def main_charts(save=False):
### --------- PRICE CHART
data = {
'time':
[t_s(t) for t, p in get_gecko(models()['epic']).data['price_7d']][::2],
'price':
[p for t, p in get_gecko(models()['epic']).data['price_7d']][::2],
}
df = pd.DataFrame(data)
source = ColumnDataSource(df)
DTF = DatetimeTickFormatter()
DTF.hours = ["%H:%M"]
DTF.days = ["%d/%m"]
DTF.months = ["%d/%m/%Y"]
DTF.years = ["%d/%m/%Y"]
TOOLS = "pan, wheel_zoom, box_zoom, reset, save"
p = figure(x_axis_type="datetime",
tools=TOOLS,
plot_width=900,
plot_height=300,
sizing_mode='scale_width')
p.line(x='time', y='price', source=source, line_width=2, color='gold')
p.varea(x='time',
y1=0,
y2='price',
source=source,
alpha=0.2,
fill_color='gold')
hover = HoverTool(tooltips=[("Date: ", "@time{%y-%m-%d}"),
("Time: ", "@time{%H:%M}"),
("Price: ", "@price{$0.00f}")],
formatters={
"@time": "datetime",
'price': 'printf'
},
mode='vline')
p.add_tools(hover)
p.xaxis.major_label_orientation = 3.14 / 4
p.yaxis.visible = False
# x stuff
p.xaxis.visible = True
p.xgrid.visible = False
p.xaxis.major_label_text_color = "#cccac4"
p.xaxis[0].formatter = DTF
# Y - PRICE
p.y_range = Range1d(min(data['price']) * 0.9, max(data['price']) * 1.1)
# p.yaxis.axis_label = "Price in USD"
p.add_layout(LinearAxis(), 'right')
p.yaxis[1].formatter = NumeralTickFormatter(format="$0.000")
p.yaxis.major_label_text_color = "gold"
p.ygrid.visible = False
p.background_fill_color = None
p.border_fill_color = None
p.outline_line_color = None
p.toolbar.autohide = True
epic_7d_price, created = Chart.objects.get_or_create(name='epic_7d_price',
coin=models()['epic'])
epic_7d_price.script, epic_7d_price.div = components(p)
epic_7d_price.updated = timezone.now()
epic_7d_price.save()
### VOLUME CHART
colors = ["#f2a900", "#459e86"]
piles = ['Total', 'Citex', 'Vitex']
targets = ['Bitcoin', 'USD(T)']
vol_data = {
'piles':
piles,
'Bitcoin': [
volumes()['total']['btc'],
volumes()['citex']['btc'],
volumes()['vitex']['btc'],
],
'USD(T)': [volumes()['total']['usdt'],
volumes()['citex']['usdt'], 0]
}
p1 = figure(x_range=piles,
plot_height=50,
plot_width=150,
toolbar_location=None,
tools="hover",
tooltips="$name: @$name{0 a}",
sizing_mode='scale_width')
p1.vbar_stack(targets,
x='piles',
width=0.7,
color=colors,
source=vol_data,
legend_label=targets)
labels = LabelSet(x='piles',
y=0,
text='targets',
level='glyph',
x_offset=5,
y_offset=5,
source=source)
p1.yaxis.formatter = NumeralTickFormatter(format="0,0")
p1.ygrid.visible = False
p1.xgrid.visible = False
p1.legend.orientation = "horizontal"
p1.legend.background_fill_alpha = 0
p1.legend.border_line_color = None
p1.background_fill_color = None
p1.border_fill_color = None
p1.outline_line_color = None
p1.toolbar.autohide = True
vol_24h, created = Chart.objects.get_or_create(name='vol_24h',
coin=models()['epic'])
vol_24h.script, vol_24h.div = components(p1)
vol_24h.updated = timezone.now()
vol_24h.save()
return f'Main charts updated successfully!'
def create(palm):
connected = False
current_message = None
stream_t = 0
doc = curdoc()
# Streaked and reference waveforms plot
waveform_plot = Plot(
title=Title(text="eTOF waveforms"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
waveform_plot.toolbar.logo = None
waveform_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool())
# ---- axes
waveform_plot.add_layout(LinearAxis(axis_label="Photon energy, eV"),
place="below")
waveform_plot.add_layout(LinearAxis(axis_label="Intensity",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
waveform_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
waveform_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- line glyphs
waveform_source = ColumnDataSource(
dict(x_str=[], y_str=[], x_ref=[], y_ref=[]))
waveform_ref_line = waveform_plot.add_glyph(
waveform_source, Line(x="x_ref", y="y_ref", line_color="blue"))
waveform_str_line = waveform_plot.add_glyph(
waveform_source, Line(x="x_str", y="y_str", line_color="red"))
# ---- legend
waveform_plot.add_layout(
Legend(items=[("reference",
[waveform_ref_line]), ("streaked",
[waveform_str_line])]))
waveform_plot.legend.click_policy = "hide"
# Cross-correlation plot
xcorr_plot = Plot(
title=Title(text="Waveforms cross-correlation"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
xcorr_plot.toolbar.logo = None
xcorr_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool())
# ---- axes
xcorr_plot.add_layout(LinearAxis(axis_label="Energy shift, eV"),
place="below")
xcorr_plot.add_layout(LinearAxis(axis_label="Cross-correlation",
major_label_orientation="vertical"),
place="left")
# ---- grid lines
xcorr_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
xcorr_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- line glyphs
xcorr_source = ColumnDataSource(dict(lags=[], xcorr=[]))
xcorr_plot.add_glyph(xcorr_source,
Line(x="lags", y="xcorr", line_color="purple"))
# ---- vertical span
xcorr_center_span = Span(location=0, dimension="height")
xcorr_plot.add_layout(xcorr_center_span)
# Delays plot
pulse_delay_plot = Plot(
title=Title(text="Pulse delays"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
pulse_delay_plot.toolbar.logo = None
pulse_delay_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool())
# ---- axes
pulse_delay_plot.add_layout(LinearAxis(axis_label="Pulse number"),
place="below")
pulse_delay_plot.add_layout(
LinearAxis(axis_label="Pulse delay (uncalib), eV",
major_label_orientation="vertical"),
place="left",
)
# ---- grid lines
pulse_delay_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
pulse_delay_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- line glyphs
pulse_delay_source = ColumnDataSource(dict(x=[], y=[]))
pulse_delay_plot.add_glyph(pulse_delay_source,
Line(x="x", y="y", line_color="steelblue"))
# Pulse lengths plot
pulse_length_plot = Plot(
title=Title(text="Pulse lengths"),
x_range=DataRange1d(),
y_range=DataRange1d(),
plot_height=PLOT_CANVAS_HEIGHT,
plot_width=PLOT_CANVAS_WIDTH,
toolbar_location="right",
)
# ---- tools
pulse_length_plot.toolbar.logo = None
pulse_length_plot.add_tools(PanTool(), BoxZoomTool(), WheelZoomTool(),
ResetTool())
# ---- axes
pulse_length_plot.add_layout(LinearAxis(axis_label="Pulse number"),
place="below")
pulse_length_plot.add_layout(
LinearAxis(axis_label="Pulse length (uncalib), eV",
major_label_orientation="vertical"),
place="left",
)
# ---- grid lines
pulse_length_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
pulse_length_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
# ---- line glyphs
pulse_length_source = ColumnDataSource(dict(x=[], y=[]))
pulse_length_plot.add_glyph(pulse_length_source,
Line(x="x", y="y", line_color="steelblue"))
# Image buffer slider
def buffer_slider_callback(_attr, _old, new):
message = receiver.data_buffer[new]
doc.add_next_tick_callback(partial(update, message=message))
buffer_slider = Slider(
start=0,
end=59,
value=0,
step=1,
title="Buffered Image",
callback_policy="throttle",
callback_throttle=500,
)
buffer_slider.on_change("value", buffer_slider_callback)
# Connect toggle button
def connect_toggle_callback(state):
nonlocal connected
if state:
connected = True
connect_toggle.label = "Connecting"
connect_toggle.button_type = "default"
else:
connected = False
connect_toggle.label = "Connect"
connect_toggle.button_type = "default"
connect_toggle = Toggle(label="Connect", button_type="default", width=250)
connect_toggle.on_click(connect_toggle_callback)
# Intensity stream reset button
def reset_button_callback():
nonlocal stream_t
stream_t = 1 # keep the latest point in order to prevent full axis reset
reset_button = Button(label="Reset", button_type="default", width=250)
reset_button.on_click(reset_button_callback)
# Stream update coroutine
async def update(message):
nonlocal stream_t
if connected and receiver.state == "receiving":
y_ref = message[receiver.reference].value[np.newaxis, :]
y_str = message[receiver.streaked].value[np.newaxis, :]
delay, length, debug_data = palm.process({
"0": y_ref,
"1": y_str
},
debug=True)
prep_data, lags, corr_res_uncut, _ = debug_data
waveform_source.data.update(
x_str=palm.energy_range,
y_str=prep_data["1"][0, :],
x_ref=palm.energy_range,
y_ref=prep_data["0"][0, :],
)
xcorr_source.data.update(lags=lags, xcorr=corr_res_uncut[0, :])
xcorr_center_span.location = delay[0]
pulse_delay_source.stream({
"x": [stream_t],
"y": [delay]
},
rollover=120)
pulse_length_source.stream({
"x": [stream_t],
"y": [length]
},
rollover=120)
stream_t += 1
# Periodic callback to fetch data from receiver
async def internal_periodic_callback():
nonlocal current_message
if waveform_plot.inner_width is None:
# wait for the initialization to finish, thus skip this periodic callback
return
if connected:
if receiver.state == "polling":
connect_toggle.label = "Polling"
connect_toggle.button_type = "warning"
elif receiver.state == "stopped":
connect_toggle.label = "Not available"
connect_toggle.button_type = "danger"
elif receiver.state == "receiving":
connect_toggle.label = "Receiving"
connect_toggle.button_type = "success"
# Set slider to the right-most position
if len(receiver.data_buffer) > 1:
buffer_slider.end = len(receiver.data_buffer) - 1
buffer_slider.value = len(receiver.data_buffer) - 1
if receiver.data_buffer:
current_message = receiver.data_buffer[-1]
doc.add_next_tick_callback(partial(update, message=current_message))
doc.add_periodic_callback(internal_periodic_callback, 1000)
# assemble
tab_layout = column(
row(
column(waveform_plot, xcorr_plot),
Spacer(width=30),
column(buffer_slider, row(connect_toggle, reset_button)),
),
row(pulse_delay_plot, Spacer(width=10), pulse_length_plot),
)
return Panel(child=tab_layout, title="Stream")
x = np.linspace(-2 * pi, 2 * pi, 1000)
y = sin(x)
z = cos(x)
source = ColumnDataSource(data=dict(x=x, y=y))
xdr = DataRange1d(sources=[source.columns("x")])
ydr = DataRange1d(sources=[source.columns("y")])
plot = Plot(x_range=xdr, y_range=ydr, min_border=50)
line_glyph = Line(x="x", y="y", line_color="blue")
plot.add_glyph(source, line_glyph)
plot.add_layout(LinearAxis(), 'below')
plot.add_layout(LinearAxis(), 'left')
pan = PanTool()
wheel_zoom = WheelZoomTool()
preview_save = PreviewSaveTool()
plot.add_tools(pan, wheel_zoom, preview_save)
doc = Document()
doc.add(plot)
if __name__ == "__main__":
filename = "line.html"
with open(filename, "w") as f:
f.write(file_html(doc, INLINE, "Line Glyph Example"))
msource = ColumnDataSource(data=dict(ticks=pos, cracked=freq, color=mcm))
p = figure(
x_range=(0, tick),
y_range=(0, 100.0 * total / sf),
plot_height=600,
title=
"Passwords by Time To Crack (number on left, proportion of total on right)",
toolbar_location=None,
tools="")
p.xaxis.axis_label = 'Ticks (10s each)'
p.yaxis.axis_label = 'Quantity'
p.extra_y_ranges = {"Percentage": Range1d(start=0, end=maxp)}
p.add_layout(LinearAxis(y_range_name="Percentage"), 'right')
p.vbar(x='ticks',
top='cracked',
width=1,
color='color',
legend=False,
source=msource)
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None
show(p)
#save(p)
from bokeh.models import ColumnDataSource, Grid, LinearAxis, Plot, Scatter
N = len(MarkerType)
x = np.linspace(-2, 2, N)
y = x**2
markers = list(MarkerType)
source = ColumnDataSource(dict(x=x, y=y, markers=markers))
plot = Plot(title=None,
plot_width=300,
plot_height=300,
min_border=0,
toolbar_location=None)
glyph = Scatter(x="x", y="y", size=20, fill_color="#74add1", marker="markers")
plot.add_glyph(source, glyph)
xaxis = LinearAxis()
plot.add_layout(xaxis, 'below')
yaxis = LinearAxis()
plot.add_layout(yaxis, 'left')
plot.add_layout(Grid(dimension=0, ticker=xaxis.ticker))
plot.add_layout(Grid(dimension=1, ticker=yaxis.ticker))
curdoc().add_root(plot)
show(plot)
def __init__(self, n_rectangles=1000, clear_interval=20000, **kwargs):
"""
kwargs are applied to the bokeh.models.plots.Plot constructor
"""
self.n_rectangles = n_rectangles
self.clear_interval = clear_interval
self.last = 0
self.source = ColumnDataSource(
data=dict(start=[time() - clear_interval],
duration=[0.1],
key=['start'],
name=['start'],
color=['white'],
worker=['foo'],
y=[0],
worker_thread=[1],
alpha=[0.0]))
x_range = DataRange1d(range_padding=0)
y_range = DataRange1d(range_padding=0)
self.root = Plot(title=Title(text="Task Stream"),
id='bk-task-stream-plot',
x_range=x_range,
y_range=y_range,
toolbar_location="above",
min_border_right=35,
**kwargs)
self.root.add_glyph(
self.source,
Rect(x="start",
y="y",
width="duration",
height=0.4,
fill_color="color",
line_color="color",
line_alpha=0.6,
fill_alpha="alpha",
line_width=3))
self.root.add_layout(DatetimeAxis(axis_label="Time"), "below")
ticker = BasicTicker(num_minor_ticks=0)
self.root.add_layout(
LinearAxis(axis_label="Worker Core", ticker=ticker), "left")
self.root.add_layout(
Grid(dimension=1, grid_line_alpha=0.4, ticker=ticker))
self.root.yaxis.major_label_text_alpha = 0
hover = HoverTool(point_policy="follow_mouse",
tooltips="""
<div>
<span style="font-size: 12px; font-weight: bold;">@name:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@duration</span>
<span style="font-size: 10px;">ms</span>
</div>
""")
self.root.add_tools(hover, BoxZoomTool(), ResetTool(reset_size=False),
PanTool(dimensions="width"),
WheelZoomTool(dimensions="width"))
if ExportTool:
export = ExportTool()
export.register_plot(self.root)
self.root.add_tools(export)
# Required for update callback
self.task_stream_index = [0]
