def get_data(username, docid, datasourceid):
bokehuser = bokeh_app.authentication.current_user()
request_username = bokehuser.username
# handle docid later...
clientdoc = bokeh_app.backbone_storage.get_document(docid)
prune(clientdoc)
init_bokeh(clientdoc)
serverdatasource = clientdoc._models[datasourceid]
parameters = json.loads(request.values.get('resample_parameters'))
plot_state = json.loads(request.values.get('plot_state'))
render_state = json.loads(request.values.get('render_state')) if 'render_state' in request.values else None
# TODO: Desserializing directly to ranges....awk-ward.
# There is probably a better way via the properties system that detects type...probably...
plot_state=dict([(k, Range1d.load_json(r)) for k,r in iteritems(plot_state)])
result = bokeh_app.datamanager.get_data(
request_username,
serverdatasource,
parameters,
plot_state,
render_state)
result = make_json(protocol.serialize_json(result))
return result
def test_init_with_float(self):
range1d = Range1d(start=-1.0, end=3.0)
assert range1d.start == -1.0
assert range1d.end == 3.0
assert range1d.bounds is None
def test_init_with_timedelta(self):
range1d = Range1d(start=-dt.timedelta(seconds=5),
end=dt.timedelta(seconds=3))
assert range1d.start == -dt.timedelta(seconds=5)
assert range1d.end == dt.timedelta(seconds=3)
assert range1d.bounds is None
def test_bounds_with_text_rejected_as_the_correct_value_error(self):
with pytest.raises(ValueError) as e:
Range1d(1, 2, bounds="21"
) # The string is indexable, so this may not fail properly
assert e.value.args[0].startswith('expected an element of either')
def plotOutlierHistogram(dataframe, maxOutliers, func,
statisticalOutlierThreshold, userLatencyThreshold,
averageDuration, maxDuration):
global pixelsForTitle
global pixelsPerHeightUnit
global plotWidth
global timeUnitString
cds = ColumnDataSource(dataframe)
figureTitle = "Occurrences of " + func + " that took longer than " \
+ statisticalOutlierThreshold + "."
hover = HoverTool(
tooltips=[("interval start",
"@lowerbound{0,0}"), ("interval end", "@upperbound{0,0}")])
TOOLS = [hover, "tap, reset"]
p = figure(title = figureTitle, plot_width = plotWidth,
plot_height = min(500, (max(5, (maxOutliers + 1)) \
* pixelsPerHeightUnit + \
pixelsForTitle)),
x_axis_label = "Execution timeline (" + timeUnitString + ")",
y_axis_label = "Number of outliers",
tools = TOOLS, toolbar_location="above")
y_ticker_max = p.plot_height // pixelsPerHeightUnit
y_ticker_step = max(1, (maxOutliers + 1) // y_ticker_max)
y_upper_bound = (maxOutliers // y_ticker_step + 2) * y_ticker_step
p.yaxis.ticker = FixedTicker(
ticks=list(range(0, y_upper_bound, y_ticker_step)))
p.ygrid.ticker = FixedTicker(
ticks=list(range(0, y_upper_bound, y_ticker_step)))
p.xaxis.formatter = NumeralTickFormatter(format="0,")
p.y_range = Range1d(0, y_upper_bound)
p.quad(left='lowerbound',
right='upperbound',
bottom='bottom',
top='height',
color=funcToColor[func],
source=cds,
nonselection_fill_color=funcToColor[func],
nonselection_fill_alpha=1.0,
line_color="lightgrey",
selection_fill_color=funcToColor[func],
selection_line_color="grey")
p.x(x='markerX',
y='markerY',
size='markersize',
color='navy',
line_width=1,
source=cds)
# Add an annotation to the chart
#
y_max = dataframe['height'].max()
text = "Average duration: " + '{0:,.0f}'.format(averageDuration) + " " + \
timeUnitString + \
". Maximum duration: " + '{0:,.0f}'.format(maxDuration) + " " + \
timeUnitString + ". "
if (userLatencyThreshold is not None):
text = text + \
"An \'x\' shows intervals with operations exceeding " + \
"a user-defined threshold of " + \
userLatencyThreshold + "."
mytext = Label(x=0,
y=y_upper_bound - y_ticker_step,
text=text,
text_color="grey",
text_font="helvetica",
text_font_size="10pt",
text_font_style="italic")
p.add_layout(mytext)
url = "@bucketfiles"
taptool = p.select(type=TapTool)
taptool.callback = OpenURL(url=url)
return p
def y_range(f, y, source, ylo=None, yhi=None):
ylo, yhi = source[y].dropna().min(
) if ylo is None else ylo, source[y].dropna().max() if yhi is None else yhi
dy = yhi - ylo
f.y_range = Range1d(start=ylo - 0.1 * dy, end=yhi + 0.1 * dy)
min_x = min([DATA[site][which[0]][0] for site in DATA]) max_x = max([DATA[site][which[0]][-1] for site in DATA]) min_y = min([min(DATA[site][which[1]]) for site in DATA]) max_y = max([max(DATA[site][which[1]]) for site in DATA]) # we will set the y axis range at +/- 10% of the data max amplitude ampli = max_y - min_y min_y = min_y - ampli*0.1/100 max_y = max_y + ampli*0.1/100 milestone = time.time() if type(min_x) == datetime: fig = figure(output_backend = "webgl", title = 'TCCON '+which[1]+' vs '+which[0], y_range=[min_y,max_y], plot_width = 900, plot_height = 650, tools = TOOLS, toolbar_location = 'above', x_axis_type='datetime', x_range = Range1d(min_x,max_x)) else: fig = figure(output_backend = "webgl", title = 'TCCON '+which[1]+' vs '+which[0], y_range=[min_y,max_y], plot_width = 900, plot_height = 650, tools = TOOLS, toolbar_location = 'above', x_range = Range1d(int(min_x),ceil(max_x))) plots=[] for site in lat_ordered_sites: plots.append( fig.scatter(x='x',y='y',color=DATA[site]['color'],alpha=0.5,source=sources[site]) ) N_plots = range(len(plots)) legend=Legend(items=[(lat_ordered_sites[i],[plots[i]]) for i in N_plots],location=(0,0)) fig.add_layout(legend,'right') fig.yaxis.axis_label = which[1] fig.xaxis.axis_label = which[0] checkbox = CheckboxGroup(labels=[site+' '+str(round(all_latitudes[lat_ordered_sites.index(site)],2)) for site in lat_ordered_sites],active=[],width=200)
####################################
####################################
# Figure:
####################################
####################################
bk.output_file("Berlin_dens_gmap.html", mode="cdn") # title="Berlin population")
source = bk.ColumnDataSource(data=dict( boroughsnames=boroughsnames,
population=population, area=area, density=density))
p = GMapPlot(title="", # False, # "Berlin population",
# x_axis_label='Longitude', y_axis_label='Latitude',
plot_width=570, plot_height=500,
x_range = Range1d(), y_range = Range1d(),
# border_fill = '#130f30',
map_options=GMapOptions(lat=52.521123, lng=13.407478, zoom=10))
# tools="pan,wheel_zoom,box_zoom,reset,hover,save")
p.map_options.map_type="satellite" # satellite, roadmap, terrain or hybrid
source_patches = bk.ColumnDataSource(data=dict( boroughs_xs=boroughs_xs, boroughs_ys=boroughs_ys,
boroughs_colors=boroughs_colors,
boroughsnames=boroughsnames,
population=population, area=area, density=density))
patches = Patches(xs="boroughs_xs", ys="boroughs_ys", fill_color="boroughs_colors",
fill_alpha=0.5, line_color="black", line_width=0.5)
patches_glyph = p.add_glyph(source_patches, patches)
p.add_tools(PanTool(), WheelZoomTool(), BoxSelectTool(), HoverTool(),
ResetTool(), PreviewSaveTool())
import networkx as nx
from bokeh.io import output_file, show
from bokeh.models import (BoxSelectTool, Circle, EdgesAndLinkedNodes,
HoverTool, MultiLine, Plot, Range1d, TapTool)
from bokeh.palettes import Spectral4
from bokeh.plotting import from_networkx
G=nx.karate_club_graph()
plot = Plot(width=400, height=400,
x_range=Range1d(-1.1,1.1), y_range=Range1d(-1.1,1.1))
plot.title.text = "Graph Interaction Demonstration"
plot.add_tools(HoverTool(tooltips=None), TapTool(), BoxSelectTool())
graph_renderer = from_networkx(G, nx.circular_layout, scale=1, center=(0,0))
graph_renderer.node_renderer.glyph = Circle(size=15, fill_color=Spectral4[0])
graph_renderer.node_renderer.selection_glyph = Circle(size=15, fill_color=Spectral4[2])
graph_renderer.node_renderer.hover_glyph = Circle(size=15, fill_color=Spectral4[1])
graph_renderer.edge_renderer.glyph = MultiLine(line_color="#CCCCCC", line_alpha=0.8, line_width=5)
graph_renderer.edge_renderer.selection_glyph = MultiLine(line_color=Spectral4[2], line_width=5)
graph_renderer.edge_renderer.hover_glyph = MultiLine(line_color=Spectral4[1], line_width=5)
graph_renderer.selection_policy = EdgesAndLinkedNodes()
graph_renderer.inspection_policy = EdgesAndLinkedNodes()
plot.renderers.append(graph_renderer)
from bokeh.client import push_session
from bokeh.document import Document
from bokeh.models import (Plot, DataRange1d, LinearAxis, Range1d,
ColumnDataSource, PanTool, WheelZoomTool, Line)
document = Document()
session = push_session(document)
x = linspace(-6 * pi, 6 * pi, 1000)
y = sin(x)
z = cos(x)
source = ColumnDataSource(data=dict(x=x, y=y, z=z))
plot = Plot(x_range=Range1d(-2 * pi, 2 * pi),
y_range=DataRange1d(),
min_border=50)
line_glyph = Line(x="x", y="y", line_color="blue")
plot.add_glyph(source, line_glyph)
line_glyph2 = Line(x="x", y="z", line_color="red")
plot.add_glyph(source, line_glyph2)
plot.add_layout(LinearAxis(), 'below')
plot.add_layout(LinearAxis(), 'left')
plot.add_tools(PanTool(), WheelZoomTool())
document.add_root(plot)
def plot(xaxis, yaxis, xaxis_name='Date', yaxis_name='', title='', format='line',
external_data=[], notebook=False, mission='', target='', yaxis_units='',
date_format='TDB', plot_width=975, plot_height=300,
fill_color=[], fill_alpha=0, background_image=False,
line_width=2):
if not isinstance(yaxis_name, list):
yaxis_name = [yaxis_name]
yaxis = [yaxis]
if not title:
title = '{} {}'.format(mission, yaxis_name).title().upper()
html_file_name = 'plot_{}_{}_{}-{}.html'.format('Time', yaxis_name,
mission,
target)
html_file_name = valid_url(html_file_name)
else:
title = title.upper()
if ' ' in title:
html_file_name = title.replace(' ', '_').upper()
else:
html_file_name = title
html_file_name = valid_url(html_file_name)
# TODO: Move this to the time object (convert to datatime)
# Function needs to be vectorised
# x = self.time.window
if xaxis_name == 'Date':
window_dt = []
window = xaxis
for element in window:
window_dt.append(et_to_datetime(element, date_format))
x = window_dt
else:
x = xaxis
y = yaxis
if notebook:
output_notebook()
else:
output_file(html_file_name + '.html')
if xaxis_name == 'Date':
x_axis_type = "datetime"
else:
x_axis_type = "auto"
p = figure(title=title,
plot_width=plot_width,
plot_height=plot_height,
x_axis_label=xaxis_name.upper(),
y_axis_label=yaxis_units,
x_axis_type=x_axis_type)
if xaxis_name == 'Date':
p.xaxis.formatter = DatetimeTickFormatter(
seconds=["%Y-%m-%d %H:%M:%S"],
minsec=["%Y-%m-%d %H:%M:%S"],
minutes=["%Y-%m-%d %H:%M:%S"],
hourmin=["%Y-%m-%d %H:%M:%S"],
hours=["%Y-%m-%d %H:%M:%S"],
days=["%Y-%m-%d %H:%M:%S"],
months=["%Y-%m-%d %H:%M:%S"],
years=["%Y-%m-%d %H:%M:%S"],
)
hover = HoverTool(
tooltips=[(xaxis_name, '@x{0.000}'),
(title, '@y{0.000}')],
formatters={xaxis_name: 'numeral',
title: 'numeral'})
p.add_tools(hover)
if external_data:
window_dt = []
window = external_data[0]
for element in window:
window_dt.append(et_to_datetime(element, 'TDB'))
x_ext = window_dt
y_ext = external_data[1]
if format == 'circle':
p.circle(x_ext, y_ext, size=5, color='red')
elif format == 'line':
p.line(x_ext, y_ext, line_width=2,
color='red')
# add a line renderer with legend and line thickness
color_list = ['red', 'green', 'blue', 'orange', "black", 'darkgoldenrod', 'chocolate', 'aqua', 'coral',
'darkcyan', 'cornflowerblue' 'aquamarine', 'darkturquoise', 'cornsilk']
index = 0
color_idx = 0
if background_image:
if 'TGO' in mission.upper() or 'MEX' in mission.upper():
image = 'Mars_Viking_MDIM21_ClrMosaic_global_1024.jpg'
else:
image = 'Earth_Contemporary_Basic.png'
p.image_url(url=[os.path.join(os.path.dirname(images.__file__), image)], x=-180, y=-90,
w=360, h=180, anchor="bottom_left", global_alpha=0.6)
left, right, bottom, top = -180, 180, -90, 90
p.x_range = Range1d(left, right)
p.y_range = Range1d(bottom, top)
if format == 'scatter':
is_multi_legend = len(y) <= len(yaxis_name)
for idx in range(len(y)):
p.circle([x[idx]], [y[idx]], size=3,
color=color_list[color_idx] if is_multi_legend else color_list[0],
legend=str(yaxis_name[idx]).upper() if is_multi_legend else str(yaxis_name[0]).upper())
color_idx = idx % len(color_list)
else:
for element in y:
if format == 'circle':
p.line(x, element, line_width=line_width, color=color_list[color_idx], legend=str(yaxis_name[index]).upper())
p.circle(x, element, fill_color="white", size=8)
if format == 'circle_only':
p.circle(x, element, size=3, color=color_list[color_idx], legend=str(yaxis_name[index]).upper())
elif format == 'line':
p.line(x, element, line_width=line_width, color=color_list[color_idx], legend=str(yaxis_name[index]).upper())
index += 1
color_idx = index % len(color_list)
p.legend.click_policy = "hide"
# show the results
show(p)
return
def create_bar_chart(data,
title,
x_name,
y_name,
hover_tool=None,
width=1200,
height=300):
"""Creates a bar chart plot, uses centcom dashboard styling.
Parameters:
- data, dict
- title, str
- names of x and y axes, str
- hover tool, html"""
source = ColumnDataSource(data) # Gets the data for the plot
# From the data gets the axis names
xdr = FactorRange(factors=data[x_name])
ydr = Range1d(start=0, end=max(data[y_name]) * 1.5)
tools = []
if hover_tool: # turns on the over tool if exists
tools = [
hover_tool,
]
# Defines the plot main figure
plot = figure(
title=title,
x_range=xdr, # value ranges
y_range=ydr,
plot_width=width,
plot_height=height,
h_symmetry=False,
v_symmetry=False,
min_border=0,
toolbar_location="above",
tools=tools,
# responsive=True,
outline_line_color="#666666")
# Defines the plot's shape as a vertical bar and sets its parameters
glyph = VBar(x=x_name,
top=y_name,
bottom=0,
width=.8,
fill_color="#e12127")
# Adds the glyph to the plot with the data
plot.add_glyph(source, glyph)
# Defines the x, y axes
xaxis = LinearAxis()
yaxis = LinearAxis()
plot.add_layout(Grid(dimension=0, ticker=xaxis.ticker))
plot.add_layout(Grid(dimension=1, ticker=yaxis.ticker))
plot.toolbar.logo = None
plot.min_border_top = 0
plot.xaxis.axis_label = "Days after app deployment"
plot.yaxis.axis_label = "Bugs found"
plot.xaxis.major_label_orientation = 1
plot.xgrid.grid_line_color = None
plot.ygrid.grid_line_color = "#999999"
plot.ygrid.grid_line_alpha = 0.1
return plot
# Test the model
model.eval()
with torch.no_grad():
correct = 0
total = 0
for images, labels in test_loader:
outputs = model(images)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
print('Test Accuracy of the model on the 10000 test images: {} %'.format(
(correct / total) * 100))
# Save the model and plot
torch.save(model.state_dict(), MODEL_STORE_PATH + 'conv_net_model.ckpt')
p = figure(y_axis_label='Loss',
width=850,
y_range=(0, 1),
title='PyTorch ConvNet results')
p.extra_y_ranges = {'Accuracy': Range1d(start=0, end=100)}
p.add_layout(LinearAxis(y_range_name='Accuracy', axis_label='Accuracy (%)'),
'right')
p.line(np.arange(len(loss_list)), loss_list)
p.line(np.arange(len(loss_list)),
np.array(acc_list) * 100,
y_range_name='Accuracy',
color='red')
show(p)
def analyze_bokeh(algo, title_suffix=None, title=None, show_trades=False):
"""
Draw charts for backtest results
Use Bokeh
:param title:
:param show_trades:
:return:
"""
# TODO Replace to YouEngine class
bokeh.plotting.output_file("generated/chart{}.html".format(title_suffix),
title=title)
p = bokeh.plotting.figure(x_axis_type="datetime",
plot_width=1000,
plot_height=400,
title=title)
p.grid.grid_line_alpha = 0.3
p.xaxis.axis_label = 'Date'
p.yaxis.axis_label = 'Equity'
if algo.records:
# Setting the second y axis range name and range
p.extra_y_ranges = {
"Records": Range1d(start=0, end=algo.data['close'].max())
}
# Adding the second axis to the plot.
p.add_layout(LinearAxis(y_range_name="Records"), 'right')
records = pd.DataFrame(algo.records)
for c in records.columns:
if c == 'date':
continue
p.line(records['date'],
records[c],
color='grey',
legend=c,
y_range_name="Records")
# print(algo.data[['date', 'close', 'sma50', 'sma150']])
p.line(algo.data.index,
algo.data['base_equity'],
color='#CAD8DE',
legend='Buy and Hold')
p.line(algo.data.index,
algo.data['equity'],
color='#49516F',
legend='Strategy')
p.legend.location = "top_left"
# check total trades
total_trades = len(algo.account.opened_trades) + len(
algo.account.closed_trades)
# Always disable show trade when trades more than 200 (Work too slow)
if total_trades > 200:
show_trades = False
logger.warning("Show trades disabled. Use analyze_mpl().")
if show_trades:
for trade in algo.account.opened_trades:
x = time.mktime(trade.date.timetuple()) * 1000
y = algo.data['equity'].loc[trade.date]
if trade.type_ == 'Long':
p.circle(x, y, size=6, color='green', alpha=0.5)
elif trade.type_ == 'Short':
p.circle(x, y, size=6, color='red', alpha=0.5)
for trade in algo.account.closed_trades:
x = time.mktime(trade.date.timetuple()) * 1000
y = algo.data['equity'].loc[trade.date]
if trade.type_ == 'Long':
p.circle(x, y, size=6, color='blue', alpha=0.5)
elif trade.type_ == 'Short':
p.circle(x, y, size=6, color='orange', alpha=0.5)
bokeh.plotting.show(p)
def make_interactive_network(G,
labels=False,
title='My Network',
color_palette=Blues8,
node_size='degree',
node_color='modularity_class'):
from networkx.algorithms import community
# Get network info
degrees = dict(networkx.degree(G))
networkx.set_node_attributes(G, name='degree', values=degrees)
betweenness_centrality = networkx.betweenness_centrality(G)
networkx.set_node_attributes(G,
name='betweenness',
values=betweenness_centrality)
communities = community.greedy_modularity_communities(G)
# Create empty dictionaries
modularity_color = {}
modularity_class = {}
#Loop through each community in the network
for community_number, community in enumerate(communities):
#For each member of the community, add their community number and a distinct color
for name in community:
modularity_color[name] = Spectral8[community_number]
modularity_class[name] = community_number
networkx.set_node_attributes(G, modularity_color, 'modularity_color')
networkx.set_node_attributes(G, modularity_class, 'modularity_class')
#Choose colors for node and edge highlighting
node_highlight_color = node_color
edge_highlight_color = 'black'
#Choose attributes from G network to size and color by — setting manual size (e.g. 10) or color (e.g. 'skyblue') also allowed
#Pick a color palette — Blues8, Reds8, Purples8, Oranges8, Viridis8
#color_palette = Blues8
#Choose a title!
title = title
#Establish which categories will appear when hovering over each node
HOVER_TOOLTIPS = [
("Character", "@index"),
("Degree", "@degree"),
("Modularity Class", "@modularity_class"),
("Modularity Color", "$color[swatch]:modularity_color"),
]
#Create a plot — set dimensions, toolbar, and title
plot = figure(tooltips=HOVER_TOOLTIPS,
tools="pan,wheel_zoom,save,reset",
active_scroll='wheel_zoom',
x_range=Range1d(-10.1, 10.1),
y_range=Range1d(-10.1, 10.1),
title=title)
#Create a network graph object
# https://networkx.github.io/documentation/networkx-1.9/reference/generated/networkx.drawing.layout.spring_layout.html
network_graph = from_networkx(G,
networkx.spring_layout,
scale=10,
center=(0, 0))
#Set node sizes and colors according to node degree (color as category from attribute)
if node_color == 'degree':
#Set node sizes and colors according to node degree (color as spectrum of color palette)
minimum_value_color = min(
network_graph.node_renderer.data_source.data[node_color])
maximum_value_color = max(
network_graph.node_renderer.data_source.data[node_color])
network_graph.node_renderer.glyph = Circle(
size=node_size,
fill_color=linear_cmap(node_color, color_palette,
minimum_value_color, maximum_value_color))
elif node_color == 'modularity_color':
#node_color='modularity_color'
network_graph.node_renderer.glyph = Circle(size=node_size,
fill_color=node_color)
#Set node highlight colors
network_graph.node_renderer.hover_glyph = Circle(
size=node_size, fill_color=node_highlight_color, line_width=2)
network_graph.node_renderer.selection_glyph = Circle(
size=node_size, fill_color=node_highlight_color, line_width=2)
#Set edge opacity and width
network_graph.edge_renderer.glyph = MultiLine(line_alpha=0.3, line_width=1)
#Set edge highlight colors
network_graph.edge_renderer.selection_glyph = MultiLine(
line_color=edge_highlight_color, line_width=2)
network_graph.edge_renderer.hover_glyph = MultiLine(
line_color=edge_highlight_color, line_width=2)
#Highlight nodes and edges
network_graph.selection_policy = NodesAndLinkedEdges()
network_graph.inspection_policy = NodesAndLinkedEdges()
plot.renderers.append(network_graph)
if labels == True:
#Add Labels
x, y = zip(*network_graph.layout_provider.graph_layout.values())
node_labels = list(G.nodes())
source = ColumnDataSource({
'x':
x,
'y':
y,
'name': [node_labels[i] for i in range(len(x))]
})
labels = LabelSet(x='x',
y='y',
text='name',
source=source,
background_fill_color='white',
text_font_size='10px',
background_fill_alpha=.7)
plot.renderers.append(labels)
show(plot)
#save(plot, filename=f"{title}.html")
from numpy import arange, linspace, pi, sin
from bokeh.models import LinearAxis, Range1d
from bokeh.plotting import figure, output_file, show
x = arange(-2 * pi, 2 * pi, 0.2)
y = sin(x)
y2 = linspace(0, 100, len(x))
p = figure(x_range=(-6.5, 6.5), y_range=(-1.1, 1.1), min_border=80)
p.background_fill_color = "#fafafa"
p.circle(x, y, color="crimson", size=8)
p.yaxis.axis_label = "red circles"
p.yaxis.axis_label_text_color = "crimson"
p.extra_y_ranges['foo'] = Range1d(0, 100)
p.circle(x, y2, color="navy", size=8, y_range_name="foo")
ax2 = LinearAxis(y_range_name="foo", axis_label="blue circles")
ax2.axis_label_text_color = "navy"
p.add_layout(ax2, 'left')
output_file("twin_axis.html", title="twin_axis.py example")
show(p)
height=100)
layout = row(y, div)
tab9 = Panel(child=layout, title="All India Tests over Time")
#Testing Growth Rate
cases_tests['timestamp'] = pd.to_datetime(cases_tests['timestamp'],
format=r'%Y-%m-%d')
cases_tests['daily_tests'] = cases_tests['totalSamplesTested'].diff(1)
cases_tests['daily_confirmed'] = cases_tests['totalPositiveCases'].diff(1)
z = figure(plot_width=1200,
plot_height=700,
sizing_mode="scale_both",
x_axis_type='datetime',
y_range=Range1d(start=0,
end=cases_tests['totalSamplesTested'].max()))
z.title.text = 'COVID19 Tests Growth Rate over Time'
z.title.align = 'center'
z.title.text_font_size = '17px'
z.xaxis.axis_label = 'Date'
z.yaxis.axis_label = 'Tests Count'
z.extra_y_ranges = {
'tests_growth_rate':
Range1d(start=cases_tests['totalSamplesTested'].pct_change().min(),
end=cases_tests['totalSamplesTested'].pct_change().max())
}
z.add_layout(LinearAxis(y_range_name='tests_growth_rate'), 'right')
sample_test_growth = z.vbar(x=cases_tests['timestamp'],
top=cases_tests['totalSamplesTested'].pct_change(),
def main():
global originator
parser = argparse.ArgumentParser(
description='based on prov json generate graph')
parser.add_argument('-f',
'--file',
dest='file_name',
required=True,
help='name of file need to be process')
args = parser.parse_args()
with open(args.file_name) as f:
data = json.load(f)
root_url = data['root']
bundle_data = data['bundle']
originator = find_originator(bundle_data)
#initial empty graph:
G = nx.Graph()
# add all nodes
node_types = [item for item in bundle_data if item in NODE_TYPES]
for node_type in node_types:
print("add all " + node_type + " nodes")
nodes = bundle_data[node_type]
for node_name in nodes:
node = nodes[node_name]
n_type = node[add_prefix_to_name('type')]
n_color, n_name = assign_color(n_type)
n_value = node[add_prefix_to_name(n_name)] if n_name else None
n_color = 'black' if n_value == root_url else n_color
G.add_node(extract_name(node_name),
type=n_type,
color=n_color,
value=n_value)
# add all edges
edge_types = [item for item in bundle_data if item not in NODE_TYPES]
for edge_type in edge_types:
print("add all " + edge_type + " edges")
edges = bundle_data[edge_type]
for edge_name in edges:
edge = edges[edge_name]
values = list(edge.values())
from_node = extract_name(values[0])
to_node = extract_name(values[1])
G.add_edge(from_node, to_node)
# Show with Bokeh
plot = Plot(plot_width=1200,
plot_height=750,
x_range=Range1d(-1.1, 1.1),
y_range=Range1d(-1.1, 1.1))
plot.title.text = "Provenance graph"
node_hover_tool = HoverTool(tooltips=[("type", "@type"), ("value",
"@value")])
plot.add_tools(node_hover_tool, BoxZoomTool(), ResetTool())
graph_renderer = from_networkx(G, nx.spring_layout, scale=1, center=(0, 0))
graph_renderer.node_renderer.glyph = Circle(size=15, fill_color='color')
graph_renderer.edge_renderer.glyph = MultiLine(line_alpha=0.8,
line_width=1)
plot.renderers.append(graph_renderer)
output_file("provenance_graphs_" + args.file_name + ".html")
show(plot)
def nb_view_patches(Yr, A, C, b, f, d1, d2, YrA = None, image_neurons=None, thr=0.99, denoised_color=None,cmap='jet'):
"""
Interactive plotting utility for ipython notebook
Parameters:
-----------
Yr: np.ndarray
movie
A,C,b,f: np.ndarrays
outputs of matrix factorization algorithm
d1,d2: floats
dimensions of movie (x and y)
YrA: np.ndarray
ROI filtered residual as it is given from update_temporal_components
If not given, then it is computed (K x T)
image_neurons: np.ndarray
image to be overlaid to neurons (for instance the average)
thr: double
threshold regulating the extent of the displayed patches
denoised_color: string or None
color name (e.g. 'red') or hex color code (e.g. '#F0027F')
cmap: string
name of colormap (e.g. 'viridis') used to plot image_neurons
"""
colormap = cm.get_cmap(cmap)
grayp = [mpl.colors.rgb2hex(m) for m in colormap(np.arange(colormap.N))]
nr, T = C.shape
nA2 = np.ravel(A.power(2).sum(0))
b = np.squeeze(b)
f = np.squeeze(f)
if YrA is None:
Y_r = np.array(spdiags(old_div(1, nA2), 0, nr, nr) *
(A.T * np.matrix(Yr) -
(A.T * np.matrix(b[:, np.newaxis])) * np.matrix(f[np.newaxis]) -
A.T.dot(A) * np.matrix(C)) + C)
else:
Y_r = C + YrA
x = np.arange(T)
z = old_div(np.squeeze(np.array(Y_r[:, :].T)), 100)
if image_neurons is None:
image_neurons = A.mean(1).reshape((d1, d2), order='F')
coors = get_contours(A, (d1, d2), thr)
cc1 = [cor['coordinates'][:, 0] for cor in coors]
cc2 = [cor['coordinates'][:, 1] for cor in coors]
c1 = cc1[0]
c2 = cc2[0]
# split sources up, such that Bokeh does not warn
# "ColumnDataSource's columns must be of the same length"
source = ColumnDataSource(data=dict(x=x, y=z[:, 0], y2=C[0] / 100))
source_ = ColumnDataSource(data=dict(z=z.T, z2=C / 100))
source2 = ColumnDataSource(data=dict(c1=c1, c2=c2))
source2_ = ColumnDataSource(data=dict(cc1=cc1, cc2=cc2))
callback = CustomJS(args=dict(source=source, source_=source_, source2=source2, source2_=source2_), code="""
var data = source.get('data')
var data_ = source_.get('data')
var f = cb_obj.get('value')-1
x = data['x']
y = data['y']
y2 = data['y2']
for (i = 0; i < x.length; i++) {
y[i] = data_['z'][i+f*x.length]
y2[i] = data_['z2'][i+f*x.length]
}
var data2_ = source2_.get('data');
var data2 = source2.get('data');
c1 = data2['c1'];
c2 = data2['c2'];
cc1 = data2_['cc1'];
cc2 = data2_['cc2'];
for (i = 0; i < c1.length; i++) {
c1[i] = cc1[f][i]
c2[i] = cc2[f][i]
}
source2.trigger('change')
source.trigger('change')
""")
plot = bpl.figure(plot_width=600, plot_height=300)
plot.line('x', 'y', source=source, line_width=1, line_alpha=0.6)
if denoised_color is not None:
plot.line('x', 'y2', source=source, line_width=1, line_alpha=0.6, color=denoised_color)
slider = bokeh.models.Slider(start=1, end=Y_r.shape[0], value=1, step=1,
title="Neuron Number", callback=callback)
xr = Range1d(start=0, end=image_neurons.shape[1])
yr = Range1d(start=image_neurons.shape[0], end=0)
plot1 = bpl.figure(x_range=xr, y_range=yr, plot_width=300, plot_height=300)
plot1.image(image=[image_neurons[::-1, :]], x=0,
y=image_neurons.shape[0], dw=d2, dh=d1, palette=grayp)
plot1.patch('c1', 'c2', alpha=0.6, color='purple', line_width=2, source=source2)
bpl.show(bokeh.layouts.layout([[slider], [bokeh.layouts.row(plot1, plot)]]))
return Y_r
from bokeh.models.widgets import Select, Slider
from bokeh.layouts import layout
#crate columndatasource
source_original = ColumnDataSource(
dict(average_grades=[7, 8, 10],
exam_grades=[6, 9, 8],
student_names=["Stephan", "Helder", "Riazudidn"]))
source = ColumnDataSource(
dict(average_grades=[7, 8, 10],
exam_grades=[6, 9, 8],
student_names=["Stephan", "Helder", "Riazudidn"]))
#create the figure
f = figure(x_range=Range1d(start=0, end=12), y_range=Range1d(start=0, end=12))
#add labels for glyphs
labels = LabelSet(x="average_grades",
y="exam_grades",
text="student_names",
x_offset=5,
y_offset=5,
source=source)
f.add_layout(labels)
#create glyphs
f.circle(x="average_grades", y="exam_grades", source=source, size=8)
#create filtering function
def nb_view_patches3d(Y_r, A, C, dims, image_type='mean', Yr=None,
max_projection=False, axis=0, thr=0.9, denoised_color=None,cmap='jet'):
"""
Interactive plotting utility for ipython notbook
Parameters:
-----------
Y_r: np.ndarray
residual of each trace
A,C,b,f: np.ndarrays
outputs of matrix factorization algorithm
dims: tuple of ints
dimensions of movie (x, y and z)
image_type: 'mean', 'max' or 'corr'
image to be overlaid to neurons
(average of shapes, maximum of shapes or nearest neigbor correlation of raw data)
Yr: np.ndarray
movie, only required if image_type=='corr' to calculate correlation image
max_projection: boolean
plot max projection along specified axis if True, plot layers if False
axis: int (0, 1 or 2)
axis along which max projection is performed or layers are shown
thr: scalar between 0 and 1
Energy threshold for computing contours
denoised_color: string or None
color name (e.g. 'red') or hex color code (e.g. '#F0027F')
cmap: string
name of colormap (e.g. 'viridis') used to plot image_neurons
Raise:
------
ValueError("image_type must be 'mean', 'max' or 'corr'")
"""
bokeh.io.curdoc().clear() # prune old orphaned models, otherwise filesize blows up
d = A.shape[0]
order = list(range(4))
order.insert(0, order.pop(axis))
Y_r = Y_r + C
index_permut = np.reshape(np.arange(d), dims, order='F').transpose(
order[:-1]).reshape(d, order='F')
A = csc_matrix(A)[index_permut, :]
dims = tuple(np.array(dims)[order[:3]])
d1, d2, d3 = dims
colormap = cm.get_cmap(cmap)
grayp = [mpl.colors.rgb2hex(m) for m in colormap(np.arange(colormap.N))]
nr, T = C.shape
x = np.arange(T)
source = ColumnDataSource(data=dict(x=x, y=Y_r[0] / 100, y2=C[0] / 100))
source_ = ColumnDataSource(data=dict(z=Y_r / 100, z2=C / 100))
sourceN = ColumnDataSource(data=dict(N=[nr], nan=np.array([np.nan])))
if max_projection:
if image_type == 'corr':
tmp = [(local_correlations(
Yr.reshape(dims + (-1,), order='F'))[:, ::-1]).max(i)
for i in range(3)]
elif image_type == 'mean':
tmp = [(np.array(A.mean(axis=1)).reshape(dims, order='F')[:, ::-1]).max(i)
for i in range(3)]
elif image_type == 'max':
tmp = [(A.max(axis=1).toarray().reshape(dims, order='F')[:, ::-1]).max(i)
for i in range(3)]
else:
raise ValueError("image_type must be 'mean', 'max' or 'corr'")
image_neurons = np.nan * np.ones((int(1.05 * (d1 + d2)), int(1.05 * (d1 + d3))))
image_neurons[:d2, -d3:] = tmp[0][::-1]
image_neurons[:d2, :d1] = tmp[2].T[::-1]
image_neurons[-d1:, -d3:] = tmp[1]
offset1 = image_neurons.shape[1] - d3
offset2 = image_neurons.shape[0] - d1
proj_ = [coo_matrix([A[:, nnrr].toarray().reshape(dims, order='F').max(
i).reshape(-1, order='F') for nnrr in range(A.shape[1])]) for i in range(3)]
proj_ = [pproj_.T for pproj_ in proj_]
coors = [get_contours(proj_[i], tmp[i].shape, thr=thr) for i in range(3)]
pl.close()
K = np.max([[len(cor['coordinates']) for cor in cc] for cc in coors])
cc1 = np.nan * np.zeros(np.shape(coors) + (K,))
cc2 = np.nan * np.zeros(np.shape(coors) + (K,))
for i, cor in enumerate(coors[0]):
cc1[0, i, :len(cor['coordinates'])] = cor['coordinates'][:, 0] + offset1
cc2[0, i, :len(cor['coordinates'])] = cor['coordinates'][:, 1]
for i, cor in enumerate(coors[2]):
cc1[1, i, :len(cor['coordinates'])] = cor['coordinates'][:, 1]
cc2[1, i, :len(cor['coordinates'])] = cor['coordinates'][:, 0]
for i, cor in enumerate(coors[1]):
cc1[2, i, :len(cor['coordinates'])] = cor['coordinates'][:, 0] + offset1
cc2[2, i, :len(cor['coordinates'])] = cor['coordinates'][:, 1] + offset2
c1x = cc1[0][0]
c2x = cc2[0][0]
c1y = cc1[1][0]
c2y = cc2[1][0]
c1z = cc1[2][0]
c2z = cc2[2][0]
source2_ = ColumnDataSource(data=dict(cc1=cc1, cc2=cc2))
source2 = ColumnDataSource(data=dict(c1x=c1x, c1y=c1y, c1z=c1z,
c2x=c2x, c2y=c2y, c2z=c2z))
callback = CustomJS(args=dict(source=source, source_=source_, sourceN=sourceN,
source2=source2, source2_=source2_), code="""
var data = source.get('data');
var data_ = source_.get('data');
var f = cb_obj.get('value')-1
x = data['x']
y = data['y']
y2 = data['y2']
for (i = 0; i < x.length; i++) {
y[i] = data_['z'][i+f*x.length]
y2[i] = data_['z2'][i+f*x.length]
}
var data2_ = source2_.get('data');
var data2 = source2.get('data');
c1x = data2['c1x'];
c2x = data2['c2x'];
c1y = data2['c1y'];
c2y = data2['c2y'];
c1z = data2['c1z'];
c2z = data2['c2z'];
cc1 = data2_['cc1'];
cc2 = data2_['cc2'];
var N = sourceN.get('data')['N'][0];
for (i = 0; i < c1x.length; i++) {
c1x[i] = cc1[f*c1x.length + i]
c2x[i] = cc2[f*c1x.length + i]
}
for (i = 0; i < c1x.length; i++) {
c1y[i] = cc1[N*c1y.length + f*c1y.length + i]
c2y[i] = cc2[N*c1y.length + f*c1y.length + i]
}
for (i = 0; i < c1x.length; i++) {
c1z[i] = cc1[2*N*c1z.length + f*c1z.length + i]
c2z[i] = cc2[2*N*c1z.length + f*c1z.length + i]
}
source2.trigger('change');
source.trigger('change');
""")
else:
if image_type == 'corr':
image_neurons = local_correlations(Yr.reshape(dims + (-1,), order='F'))[:-1, ::-1]
elif image_type == 'mean':
image_neurons = np.array(A.mean(axis=1)).reshape(dims, order='F')[:, ::-1]
elif image_type == 'max':
image_neurons = A.max(axis=1).toarray().reshape(dims, order='F')[:, ::-1]
else:
raise ValueError('image_type must be mean, max or corr')
cmap = bokeh.models.mappers.LinearColorMapper([mpl.colors.rgb2hex(m)
for m in colormap(np.arange(colormap.N))])
cmap.high = image_neurons.max()
coors = get_contours(A, dims, thr=thr)
pl.close()
cc1 = [[(l[:, 0]) for l in n['coordinates']] for n in coors]
cc2 = [[(l[:, 1]) for l in n['coordinates']] for n in coors]
length = np.ravel([list(map(len, cc)) for cc in cc1])
idx = np.cumsum(np.concatenate([[0], length[:-1]]))
cc1 = np.concatenate(list(map(np.concatenate, cc1)))
cc2 = np.concatenate(list(map(np.concatenate, cc2)))
linit = int(round(coors[0]['CoM'][0])) # pick initial layer in which first neuron lies
K = length.max()
c1 = np.nan * np.zeros(K)
c2 = np.nan * np.zeros(K)
c1[:length[linit]] = cc1[idx[linit]:idx[linit] + length[linit]]
c2[:length[linit]] = cc2[idx[linit]:idx[linit] + length[linit]]
source2 = ColumnDataSource(data=dict(c1=c1, c2=c2))
source2_ = ColumnDataSource(data=dict(cc1=cc1, cc2=cc2))
source2_idx = ColumnDataSource(data=dict(idx=idx, length=length))
source3 = ColumnDataSource(
data=dict(image=[image_neurons[linit]], im=[image_neurons],
x=[0], y=[d2], dw=[d3], dh=[d2]))
callback = CustomJS(args=dict(source=source, source_=source_, sourceN=sourceN,
source2=source2, source2_=source2_, source2_idx=source2_idx),
code="""
var data = source.data;
var data_ = source_.data;
var f = slider_neuron.value-1;
var l = slider_layer.value-1;
x = data['x']
y = data['y']
y2 = data['y2']
for (i = 0; i < x.length; i++) {
y[i] = data_['z'][i+f*x.length]
y2[i] = data_['z2'][i+f*x.length]
}
var data2 = source2.data;
var data2_ = source2_.data;
var data2_idx = source2_idx.data;
var idx = data2_idx['idx'];
c1 = data2['c1'];
c2 = data2['c2'];
var nz = idx.length / sourceN.data['N'][0];
var nan = sourceN.data['nan'][0];
for (i = 0; i < c1.length; i++) {
c1[i] = nan;
c2[i] = nan;
}
for (i = 0; i < data2_idx['length'][l+f*nz]; i++) {
c1[i] = data2_['cc1'][idx[l+f*nz] + i];
c2[i] = data2_['cc2'][idx[l+f*nz] + i];
}
source2.trigger('change');
source.trigger('change');
""")
callback_layer = CustomJS(args=dict(source=source3, sourceN=sourceN, source2=source2,
source2_=source2_, source2_idx=source2_idx), code="""
var f = slider_neuron.value-1;
var l = slider_layer.value-1;
var dh = source.data['dh'][0];
var dw = source.data['dw'][0];
var image = source.data['image'][0];
var images = source.data['im'][0];
for (var i = 0; i < x.length; i++) {
for (var j = 0; j < dw; j++){
image[i*dh+j] = images[l*dh*dw + i*dh + j];
}
}
var data2 = source2.data;
var data2_ = source2_.data;
var data2_idx = source2_idx.data;
var idx = data2_idx['idx']
c1 = data2['c1'];
c2 = data2['c2'];
var nz = idx.length / sourceN.data['N'][0];
var nan = sourceN.data['nan'][0];
for (i = 0; i < c1.length; i++) {
c1[i] = nan;
c2[i] = nan;
}
for (i = 0; i < data2_idx['length'][l+f*nz]; i++) {
c1[i] = data2_['cc1'][idx[l+f*nz] + i];
c2[i] = data2_['cc2'][idx[l+f*nz] + i];
}
source.trigger('change');
source2.trigger('change');
""")
plot = bpl.figure(plot_width=600, plot_height=300)
plot.line('x', 'y', source=source, line_width=1, line_alpha=0.6)
if denoised_color is not None:
plot.line('x', 'y2', source=source, line_width=1, line_alpha=0.6, color=denoised_color)
slider = bokeh.models.Slider(start=1, end=Y_r.shape[0], value=1, step=1,
title="Neuron Number", callback=callback)
xr = Range1d(start=0, end=image_neurons.shape[1] if max_projection else d3)
yr = Range1d(start=image_neurons.shape[0] if max_projection else d2, end=0)
plot1 = bpl.figure(x_range=xr, y_range=yr, plot_width=300, plot_height=300)
if max_projection:
plot1.image(image=[image_neurons[::-1, :]], x=0, y=image_neurons.shape[0],
dw=image_neurons.shape[1], dh=image_neurons.shape[0], palette=grayp)
plot1.patch('c1x', 'c2x', alpha=0.6, color='purple', line_width=2, source=source2)
plot1.patch('c1y', 'c2y', alpha=0.6, color='purple', line_width=2, source=source2)
plot1.patch('c1z', 'c2z', alpha=0.6, color='purple', line_width=2, source=source2)
layout = bokeh.layouts.layout([[slider], [bokeh.layouts.row(plot1, plot)]],
sizing_mode="scale_width")
else:
slider_layer = bokeh.models.Slider(start=1, end=d1, value=linit + 1, step=1,
title="Layer", callback=callback_layer)
callback.args['slider_neuron'] = slider
callback.args['slider_layer'] = slider_layer
callback_layer.args['slider_neuron'] = slider
callback_layer.args['slider_layer'] = slider_layer
plot1.image(image='image', x='x', y='y', dw='dw', dh='dh',
color_mapper=cmap, source=source3)
plot1.patch('c1', 'c2', alpha=0.6, color='purple', line_width=2, source=source2)
layout = bokeh.layouts.layout([[slider], [slider_layer], [bokeh.layouts.row(plot1, plot)]],
sizing_mode="scale_width")
bpl.show(layout)
return Y_r
return x, fy, ty
def update_data():
x, fy, ty = taylor(expr, xs, order, (-2 * sy.pi, 2 * sy.pi), 200)
plot.title.text = "%s vs. taylor(%s, n=%d)" % (expr, expr, order)
legend.items[0].label = value("%s" % expr)
legend.items[1].label = value("taylor(%s)" % expr)
source.data = dict(x=x, fy=fy, ty=ty)
slider.value = order
source = ColumnDataSource(data=dict(x=[], fy=[], ty=[]))
xdr = Range1d(-7, 7)
ydr = Range1d(-20, 200)
plot = Plot(x_range=xdr, y_range=ydr, plot_width=800, plot_height=400)
line_f = Line(x="x", y="fy", line_color="blue", line_width=2)
line_f_glyph = plot.add_glyph(source, line_f)
plot.add_layout(line_f_glyph)
line_t = Line(x="x", y="ty", line_color="red", line_width=2)
line_t_glyph = plot.add_glyph(source, line_t)
plot.add_layout(line_t_glyph)
xaxis = LinearAxis()
plot.add_layout(xaxis, 'below')
from bokeh.util.dependencies import import_required
pandas = import_required('pandas', 'get it suckah')
colormap = {
'chemical explosion': 'orange',
'quarry blast': 'green',
'earthquake': 'violet',
'sonic boom': 'magenta',
'explosion': 'blue',
'landslide': 'brown',
'mining explosion': 'black'
}
title = "QUAKIN' IN MAH BOOTS"
graph = figure(title=title)
graph.xaxis.axis_label = 'Magnitude'
graph.yaxis.axis_label = 'Depth'
quakes = pandas.read_csv(join(dirname(__file__), 'resource/earthquakes.csv'))
graph.circle(quakes['mag'],
quakes['depth'],
color=[colormap[x] for x in quakes['type']],
fill_alpha=0.2,
size=10)
graph.y_range = Range1d(700, -10)
output_file("Earthquake Visualization.html", title=title)
show(graph)
def test_with_max_bound_smaller_than_min_bounded_raises_valueerror(self):
with pytest.raises(ValueError):
Range1d(1, 2, bounds=(1, 0))
with pytest.raises(ValueError):
Range1d(1, 2, bounds=[1, 0])
def calculate_assoc_svg(file, region, pop, request, genome_build, myargs,
myargsName, myargsOrigin):
# Set data directories using config.yml
with open('config.yml', 'r') as yml_file:
config = yaml.load(yml_file)
env = config['env']
connect_external = config['database']['connect_external']
api_mongo_addr = config['database']['api_mongo_addr']
data_dir = config['data']['data_dir']
tmp_dir = config['data']['tmp_dir']
genotypes_dir = config['data']['genotypes_dir']
aws_info = config['aws']
mongo_username = config['database']['mongo_user_readonly']
mongo_password = config['database']['mongo_password']
mongo_port = config['database']['mongo_port']
num_subprocesses = config['performance']['num_subprocesses']
export_s3_keys = retrieveAWSCredentials()
# Ensure tmp directory exists
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
chrs = [
"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13",
"14", "15", "16", "17", "18", "19", "20", "21", "22", "X", "Y"
]
# Define parameters for --variant option
if region == "variant":
if myargsOrigin == "None":
return None
if myargsOrigin != "None":
# Find coordinates (GRCh37/hg19) or (GRCh38/hg38) for SNP RS number
if myargsOrigin[0:2] == "rs":
snp = myargsOrigin
# Connect to Mongo snp database
if env == 'local' or connect_external:
mongo_host = api_mongo_addr
else:
mongo_host = 'localhost'
client = MongoClient(
'mongodb://' + mongo_username + ':' + mongo_password + '@' +
mongo_host + '/admin', mongo_port)
db = client["LDLink"]
def get_coords_var(db, rsid):
rsid = rsid.strip("rs")
query_results = db.dbsnp.find_one({"id": rsid})
query_results_sanitized = json.loads(
json_util.dumps(query_results))
return query_results_sanitized
# Find RS number in snp database
var_coord = get_coords_var(db, snp)
if var_coord == None:
return None
elif myargsOrigin.split(":")[0].strip("chr") in chrs and len(
myargsOrigin.split(":")) == 2:
snp = myargsOrigin
#var_coord=[None,myargsOrigin.split(":")[0].strip("chr"),myargsOrigin.split(":")[1]]
var_coord = {
'chromosome': myargsOrigin.split(":")[0].strip("chr"),
'position': myargsOrigin.split(":")[1]
}
else:
return None
chromosome = var_coord['chromosome']
org_coord = var_coord[genome_build_vars[genome_build]['position']]
# Open Association Data
header_list = []
header_list.append(myargs['chr'])
header_list.append(myargs['bp'])
header_list.append(myargs['pval'])
# Load input file
with open(file) as fp:
header = fp.readline().strip().split()
first = fp.readline().strip().split()
if len(header) != len(first):
return None
# Check header
for item in header_list:
if item not in header:
return None
len_head = len(header)
chr_index = header.index(myargs['chr'])
pos_index = header.index(myargs['bp'])
p_index = header.index(myargs['pval'])
# Define window of interest around query SNP
if myargs['window'] == None:
if region == "variant":
window = 500000
elif region == "gene":
window = 100000
else:
window = 0
else:
window = myargs['window']
if region == "variant":
coord1 = int(org_coord) - window
if coord1 < 0:
coord1 = 0
coord2 = int(org_coord) + window
elif region == "gene":
if myargsName == "None":
return None
def get_coords_gene(gene_raw, db):
gene = gene_raw.upper()
mongoResult = db.genes_name_coords.find_one({"name": gene})
#format mongo output
if mongoResult != None:
geneResult = [
mongoResult["name"],
mongoResult[genome_build_vars[genome_build]['chromosome']],
mongoResult[genome_build_vars[genome_build]['gene_begin']],
mongoResult[genome_build_vars[genome_build]['gene_end']]
]
return geneResult
else:
return None
# Find RS number in snp database
gene_coord = get_coords_gene(myargsName, db)
if gene_coord == None or gene_coord[2] == 'NA' or gene_coord == 'NA':
return None
# Define search coordinates
coord1 = int(gene_coord[2]) - window
if coord1 < 0:
coord1 = 0
coord2 = int(gene_coord[3]) + window
# Run with --origin option
if myargsOrigin != "None":
if gene_coord[1] != chromosome:
return None
if coord1 > int(org_coord) or int(org_coord) > coord2:
return None
else:
chromosome = gene_coord[1]
elif region == "region":
if myargs['start'] == None:
return None
if myargs['end'] == None:
return None
# Parse out chr and positions for --region option
if len(myargs['start'].split(":")) != 2:
return None
if len(myargs['end'].split(":")) != 2:
return None
chr_s = myargs['start'].strip("chr").split(":")[0]
coord_s = myargs['start'].split(":")[1]
chr_e = myargs['end'].strip("chr").split(":")[0]
coord_e = myargs['end'].split(":")[1]
if chr_s not in chrs:
return None
if chr_e not in chrs:
return None
if chr_s != chr_e:
return None
if coord_s >= coord_e:
return None
coord1 = int(coord_s) - window
if coord1 < 0:
coord1 = 0
coord2 = int(coord_e) + window
# Run with --origin option
if myargsOrigin != "None":
if chr_s != chromosome:
return None
if coord1 > int(org_coord) or int(org_coord) > coord2:
return None
else:
chromosome = chr_s
# Generate coordinate list and P-value dictionary
max_window = 3000000
if coord2 - coord1 > max_window:
return None
assoc_coords = []
a_pos = []
assoc_dict = {}
assoc_list = []
with open(file) as fp:
for line in fp:
col = line.strip().split()
if len(col) == len_head:
if col[chr_index].strip("chr") == chromosome:
try:
int(col[pos_index])
except ValueError:
continue
else:
if coord1 <= int(col[pos_index]) <= coord2:
try:
float(col[p_index])
except ValueError:
continue
else:
coord_i = genome_build_vars[genome_build][
'1000G_chr_prefix'] + col[chr_index].strip(
"chr") + ":" + col[
pos_index] + "-" + col[pos_index]
assoc_coords.append(coord_i)
a_pos.append(col[pos_index])
assoc_dict[coord_i] = [col[p_index]]
assoc_list.append(
[coord_i, float(col[p_index])])
# Coordinate list checks
if len(assoc_coords) == 0:
return None
# Get population ids from population output file from LDassoc.py
pop_list = open(tmp_dir + "pops_" + request + ".txt").readlines()
ids = []
for i in range(len(pop_list)):
ids.append(pop_list[i].strip())
pop_ids = list(set(ids))
# Define LD origin coordinate
try:
org_coord
except NameError:
for var_p in sorted(assoc_list, key=operator.itemgetter(1)):
snp = "chr" + var_p[0].split("-")[0]
# Extract lowest P SNP phased genotypes
vcf_filePath = "%s/%s%s/%s" % (
config['aws']['data_subfolder'], genotypes_dir,
genome_build_vars[genome_build]["1000G_dir"],
genome_build_vars[genome_build]["1000G_file"] % (chromosome))
vcf_file = "s3://%s/%s" % (config['aws']['bucket'], vcf_filePath)
checkS3File(aws_info, config['aws']['bucket'], vcf_filePath)
tabix_snp_h = export_s3_keys + " cd {1}; tabix -HD {0} | grep CHROM".format(
vcf_file, data_dir + genotypes_dir +
genome_build_vars[genome_build]['1000G_dir'])
head = [
x.decode('utf-8') for x in subprocess.Popen(
tabix_snp_h, shell=True,
stdout=subprocess.PIPE).stdout.readlines()
][0].strip().split()
# Check lowest P SNP is in the 1000G population and not monoallelic from LDassoc.py output file
vcf = open(tmp_dir + "snp_no_dups_" + request + ".vcf").readlines()
if len(vcf) == 0:
continue
elif len(vcf) > 1:
geno = vcf[0].strip().split()
geno[0] = geno[0].lstrip('chr')
else:
geno = vcf[0].strip().split()
geno[0] = geno[0].lstrip('chr')
if "," in geno[3] or "," in geno[4]:
continue
index = []
for i in range(9, len(head)):
if head[i] in pop_ids:
index.append(i)
genotypes = {"0": 0, "1": 0}
for i in index:
sub_geno = geno[i].split("|")
for j in sub_geno:
if j in genotypes:
genotypes[j] += 1
else:
genotypes[j] = 1
if genotypes["0"] == 0 or genotypes["1"] == 0:
continue
org_coord = var_p[0].split("-")[1]
break
else:
if genome_build_vars[genome_build][
'1000G_chr_prefix'] + chromosome + ":" + org_coord + "-" + org_coord not in assoc_coords:
return None
# Extract query SNP phased genotypes
vcf_filePath = "%s/%s%s/%s" % (
config['aws']['data_subfolder'], genotypes_dir,
genome_build_vars[genome_build]["1000G_dir"],
genome_build_vars[genome_build]["1000G_file"] % (chromosome))
vcf_file = "s3://%s/%s" % (config['aws']['bucket'], vcf_filePath)
checkS3File(aws_info, config['aws']['bucket'], vcf_filePath)
tabix_snp_h = export_s3_keys + " cd {1}; tabix -HD {0} | grep CHROM".format(
vcf_file, data_dir + genotypes_dir +
genome_build_vars[genome_build]['1000G_dir'])
head = [
x.decode('utf-8') for x in
subprocess.Popen(tabix_snp_h, shell=True,
stdout=subprocess.PIPE).stdout.readlines()
][0].strip().split()
# Check query SNP is in the 1000G population, has the correct RS number, and not monoallelic
vcf = open(tmp_dir + "snp_no_dups_" + request + ".vcf").readlines()
if len(vcf) == 0:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf",
shell=True)
return None
elif len(vcf) > 1:
geno = []
for i in range(len(vcf)):
# if vcf[i].strip().split()[2] == snp:
geno = vcf[i].strip().split()
geno[0] = geno[0].lstrip('chr')
if geno == []:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf",
shell=True)
return None
else:
geno = vcf[0].strip().split()
geno[0] = geno[0].lstrip('chr')
if geno[2] != snp and snp[0:2] == "rs" and "rs" in geno[2]:
snp = geno[2]
if "," in geno[3] or "," in geno[4]:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf",
shell=True)
return None
index = []
for i in range(9, len(head)):
if head[i] in pop_ids:
index.append(i)
genotypes = {"0": 0, "1": 0}
for i in index:
sub_geno = geno[i].split("|")
for j in sub_geno:
if j in genotypes:
genotypes[j] += 1
else:
genotypes[j] = 1
if genotypes["0"] == 0 or genotypes["1"] == 0:
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt",
shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf",
shell=True)
return None
# Calculate proxy LD statistics in parallel
if len(assoc_coords) < 60:
num_subprocesses = 1
# else:
# threads=4
assoc_coords_subset_chunks = np.array_split(assoc_coords, num_subprocesses)
# block=len(assoc_coords) // num_subprocesses
commands = []
# for i in range(num_subprocesses):
# if i==min(range(num_subprocesses)) and i==max(range(num_subprocesses)):
# command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords)+" "+request+" "+str(i)
# elif i==min(range(num_subprocesses)):
# command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[:block])+" "+request+" "+str(i)
# elif i==max(range(num_subprocesses)):
# command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:])+" "+request+" "+str(i)
# else:
# command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:block*(i+1)])+" "+request+" "+str(i)
# commands.append(command)
for subprocess_id in range(num_subprocesses):
subprocessArgs = " ".join([
str(snp),
str(chromosome),
str("_".join(assoc_coords_subset_chunks[subprocess_id])),
str(request),
str(genome_build),
str(subprocess_id)
])
commands.append("python3 LDassoc_sub.py " + subprocessArgs)
processes = [
subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
for command in commands
]
# collect output in parallel
def get_output(process):
return process.communicate()[0].splitlines()
pool = Pool(len(processes))
out_raw = pool.map(get_output, processes)
pool.close()
pool.join()
# Aggregate output
out_prox = []
for i in range(len(out_raw)):
for j in range(len(out_raw[i])):
col = out_raw[i][j].decode('utf-8').strip().split("\t")
col[6] = int(col[6])
col[7] = float(col[7])
col[8] = float(col[8])
col.append(abs(int(col[6])))
pos_i_j = col[5].split(":")[1]
coord_i_j = genome_build_vars[genome_build][
'1000G_chr_prefix'] + chromosome + ":" + pos_i_j + "-" + pos_i_j
if coord_i_j in assoc_dict:
col.append(float(assoc_dict[coord_i_j][0]))
out_prox.append(col)
out_dist_sort = sorted(out_prox, key=operator.itemgetter(14))
out_p_sort = sorted(out_dist_sort,
key=operator.itemgetter(15),
reverse=False)
# Organize scatter plot data
q_rs = []
q_allele = []
q_coord = []
q_maf = []
p_rs = []
p_allele = []
p_coord = []
p_pos = []
p_maf = []
dist = []
d_prime = []
d_prime_round = []
r2 = []
r2_round = []
corr_alleles = []
regdb = []
funct = []
color = []
alpha = []
size = []
p_val = []
neg_log_p = []
for i in range(len(out_p_sort)):
q_rs_i, q_allele_i, q_coord_i, p_rs_i, p_allele_i, p_coord_i, dist_i, d_prime_i, r2_i, corr_alleles_i, regdb_i, q_maf_i, p_maf_i, funct_i, dist_abs, p_val_i = out_p_sort[
i]
q_rs.append(q_rs_i)
q_allele.append(q_allele_i)
q_coord.append(float(q_coord_i.split(":")[1]) / 1000000)
q_maf.append(str(round(float(q_maf_i), 4)))
if p_rs_i == ".":
p_rs_i = p_coord_i
p_rs.append(p_rs_i)
p_allele.append(p_allele_i)
p_coord.append(float(p_coord_i.split(":")[1]) / 1000000)
p_pos.append(p_coord_i.split(":")[1])
p_maf.append(str(round(float(p_maf_i), 4)))
dist.append(str(round(dist_i / 1000000.0, 4)))
d_prime.append(float(d_prime_i))
d_prime_round.append(str(round(float(d_prime_i), 4)))
r2.append(float(r2_i))
r2_round.append(str(round(float(r2_i), 4)))
corr_alleles.append(corr_alleles_i)
# P-value
p_val.append(p_val_i)
neg_log_p.append(-log10(p_val_i))
# Correct Missing Annotations
if regdb_i == ".":
regdb_i = ""
regdb.append(regdb_i)
if funct_i == ".":
funct_i = ""
if funct_i == "NA":
funct_i = "none"
funct.append(funct_i)
# Set Color
reds = [
"#FFCCCC", "#FFCACA", "#FFC8C8", "#FFC6C6", "#FFC4C4", "#FFC2C2",
"#FFC0C0", "#FFBEBE", "#FFBCBC", "#FFBABA", "#FFB8B8", "#FFB6B6",
"#FFB4B4", "#FFB1B1", "#FFAFAF", "#FFADAD", "#FFABAB", "#FFA9A9",
"#FFA7A7", "#FFA5A5", "#FFA3A3", "#FFA1A1", "#FF9F9F", "#FF9D9D",
"#FF9B9B", "#FF9999", "#FF9797", "#FF9595", "#FF9393", "#FF9191",
"#FF8F8F", "#FF8D8D", "#FF8B8B", "#FF8989", "#FF8787", "#FF8585",
"#FF8383", "#FF8181", "#FF7E7E", "#FF7C7C", "#FF7A7A", "#FF7878",
"#FF7676", "#FF7474", "#FF7272", "#FF7070", "#FF6E6E", "#FF6C6C",
"#FF6A6A", "#FF6868", "#FF6666", "#FF6464", "#FF6262", "#FF6060",
"#FF5E5E", "#FF5C5C", "#FF5A5A", "#FF5858", "#FF5656", "#FF5454",
"#FF5252", "#FF5050", "#FF4E4E", "#FF4B4B", "#FF4949", "#FF4747",
"#FF4545", "#FF4343", "#FF4141", "#FF3F3F", "#FF3D3D", "#FF3B3B",
"#FF3939", "#FF3737", "#FF3535", "#FF3333", "#FF3131", "#FF2F2F",
"#FF2D2D", "#FF2B2B", "#FF2929", "#FF2727", "#FF2525", "#FF2323",
"#FF2121", "#FF1F1F", "#FF1D1D", "#FF1B1B", "#FF1818", "#FF1616",
"#FF1414", "#FF1212", "#FF1010", "#FF0E0E", "#FF0C0C", "#FF0A0A",
"#FF0808", "#FF0606", "#FF0404", "#FF0202", "#FF0000"
]
if q_coord_i == p_coord_i:
color_i = "#0000FF"
alpha_i = 0.7
else:
if myargs['dprime'] == True:
color_i = reds[int(d_prime_i * 100.0)]
alpha_i = 0.7
elif myargs['dprime'] == False:
color_i = reds[int(r2_i * 100.0)]
alpha_i = 0.7
color.append(color_i)
alpha.append(alpha_i)
# Set Size
size_i = 9 + float(p_maf_i) * 14.0
size.append(size_i)
# Pull out SNPs from association file not found in 1000G
p_plot_pos = []
p_plot_pval = []
p_plot_pos2 = []
p_plot_pval2 = []
p_plot_dist = []
index_var_pos = float(q_coord_i.split(":")[1]) / 1000000
for input_pos in a_pos:
if input_pos not in p_pos:
p_plot_pos.append(float(input_pos) / 1000000)
p_plot_pval.append(-log10(
float(assoc_dict[chromosome + ":" + input_pos + "-" +
input_pos][0])))
p_plot_pos2.append("chr" + chromosome + ":" + input_pos)
p_plot_pval2.append(
float(assoc_dict[chromosome + ":" + input_pos + "-" +
input_pos][0]))
p_plot_dist.append(
str(round(float(input_pos) / 1000000 - index_var_pos, 4)))
# Begin Bokeh Plotting
from collections import OrderedDict
from bokeh.embed import components, file_html
from bokeh.layouts import gridplot
from bokeh.models import HoverTool, LinearAxis, Range1d
from bokeh.plotting import ColumnDataSource, curdoc, figure, output_file, reset_output, save
from bokeh.resources import CDN
from bokeh.io import export_svgs
import svgutils.compose as sg
reset_output()
data_p = {
'p_plot_posX': p_plot_pos,
'p_plot_pvalY': p_plot_pval,
'p_plot_pos2': p_plot_pos2,
'p_plot_pval2': p_plot_pval2,
'p_plot_dist': p_plot_dist
}
source_p = ColumnDataSource(data_p)
# Assoc Plot
x = p_coord
y = neg_log_p
data = {
'x': x,
'y': y,
'qrs': q_rs,
'q_alle': q_allele,
'q_maf': q_maf,
'prs': p_rs,
'p_alle': p_allele,
'p_maf': p_maf,
'dist': dist,
'r': r2_round,
'd': d_prime_round,
'alleles': corr_alleles,
'regdb': regdb,
'funct': funct,
'p_val': p_val,
'size': size,
'color': color,
'alpha': alpha
}
source = ColumnDataSource(data)
whitespace = 0.01
xr = Range1d(start=coord1 / 1000000.0 - whitespace,
end=coord2 / 1000000.0 + whitespace)
yr = Range1d(start=-0.03, end=max(y) * 1.03)
sup_2 = "\u00B2"
assoc_plot = figure(
title="P-values and Regional LD for " + snp + " in " + pop,
min_border_top=2,
min_border_bottom=2,
min_border_left=60,
min_border_right=60,
h_symmetry=False,
v_symmetry=False,
plot_width=900,
plot_height=600,
x_range=xr,
y_range=yr,
tools=
"tap,pan,box_zoom,wheel_zoom,box_select,undo,redo,reset,previewsave",
logo=None,
toolbar_location="above")
assoc_plot.title.align = "center"
# Add recombination rate from LDassoc.py output file
recomb_file = tmp_dir + "recomb_" + request + ".json"
recomb_raw = open(recomb_file).readlines()
recomb_x = []
recomb_y = []
for recomb_raw_obj in recomb_raw:
recomb_obj = json.loads(recomb_raw_obj)
recomb_x.append(
int(recomb_obj[genome_build_vars[genome_build]['position']]) /
1000000.0)
recomb_y.append(float(recomb_obj['rate']) / 100 * max(y))
assoc_plot.line(recomb_x, recomb_y, line_width=1, color="black", alpha=0.5)
# Add genome-wide significance
a = [coord1 / 1000000.0 - whitespace, coord2 / 1000000.0 + whitespace]
b = [-log10(0.00000005), -log10(0.00000005)]
assoc_plot.line(a, b, color="blue", alpha=0.5)
assoc_points_not1000G = assoc_plot.circle(x='p_plot_posX',
y='p_plot_pvalY',
size=9 + float("0.25") * 14.0,
source=source_p,
line_color="gray",
fill_color="white")
assoc_points = assoc_plot.circle(x='x',
y='y',
size='size',
color='color',
alpha='alpha',
source=source)
assoc_plot.add_tools(
HoverTool(renderers=[assoc_points_not1000G],
tooltips=OrderedDict([("Variant", "@p_plot_pos2"),
("P-value", "@p_plot_pval2"),
("Distance (Mb)", "@p_plot_dist")])))
hover = HoverTool(renderers=[assoc_points])
hover.tooltips = OrderedDict([
("Variant", "@prs @p_alle"),
("P-value", "@p_val"),
("Distance (Mb)", "@dist"),
("MAF", "@p_maf"),
("R" + sup_2 + " (" + q_rs[0] + ")", "@r"),
("D\' (" + q_rs[0] + ")", "@d"),
("Correlated Alleles", "@alleles"),
("RegulomeDB", "@regdb"),
("Functional Class", "@funct"),
])
assoc_plot.add_tools(hover)
# Annotate RebulomeDB scores
if myargs['annotate'] == True:
assoc_plot.text(x,
y,
text=regdb,
alpha=1,
text_font_size="7pt",
text_baseline="middle",
text_align="center",
angle=0)
assoc_plot.yaxis.axis_label = "-log10 P-value"
assoc_plot.extra_y_ranges = {"y2_axis": Range1d(start=-3, end=103)}
assoc_plot.add_layout(
LinearAxis(y_range_name="y2_axis",
axis_label="Combined Recombination Rate (cM/Mb)"),
"right") ## Need to confirm units
# Rug Plot
y2_ll = [-0.03] * len(x)
y2_ul = [1.03] * len(x)
yr_rug = Range1d(start=-0.03, end=1.03)
data_rug = {
'x': x,
'y': y,
'y2_ll': y2_ll,
'y2_ul': y2_ul,
'qrs': q_rs,
'q_alle': q_allele,
'q_maf': q_maf,
'prs': p_rs,
'p_alle': p_allele,
'p_maf': p_maf,
'dist': dist,
'r': r2_round,
'd': d_prime_round,
'alleles': corr_alleles,
'regdb': regdb,
'funct': funct,
'p_val': p_val,
'size': size,
'color': color,
'alpha': alpha
}
source_rug = ColumnDataSource(data_rug)
rug = figure(x_range=xr,
y_range=yr_rug,
border_fill_color='white',
y_axis_type=None,
title="",
min_border_top=2,
min_border_bottom=2,
min_border_left=60,
min_border_right=60,
h_symmetry=False,
v_symmetry=False,
plot_width=900,
plot_height=50,
tools="xpan,tap,wheel_zoom",
logo=None)
rug.segment(x0='x',
y0='y2_ll',
x1='x',
y1='y2_ul',
source=source_rug,
color='color',
alpha='alpha',
line_width=1)
rug.toolbar_location = None
# Gene Plot (All Transcripts)
if myargs['transcript'] == True:
# Get genes from LDassoc.py output file
genes_file = tmp_dir + "genes_" + request + ".json"
genes_raw = open(genes_file).readlines()
genes_plot_start = []
genes_plot_end = []
genes_plot_y = []
genes_plot_name = []
exons_plot_x = []
exons_plot_y = []
exons_plot_w = []
exons_plot_h = []
exons_plot_name = []
exons_plot_id = []
exons_plot_exon = []
message = ["Too many genes to plot."]
lines = [0]
gap = 80000
tall = 0.75
if genes_raw != None and len(genes_raw) > 0:
for gene_raw_obj in genes_raw:
gene_obj = json.loads(gene_raw_obj)
bin = gene_obj["bin"]
name_id = gene_obj["name"]
chrom = gene_obj["chrom"]
strand = gene_obj["strand"]
txStart = gene_obj["txStart"]
txEnd = gene_obj["txEnd"]
cdsStart = gene_obj["cdsStart"]
cdsEnd = gene_obj["cdsEnd"]
exonCount = gene_obj["exonCount"]
exonStarts = gene_obj["exonStarts"]
exonEnds = gene_obj["exonEnds"]
score = gene_obj["score"]
name2 = gene_obj["name2"]
cdsStartStat = gene_obj["cdsStartStat"]
cdsEndStat = gene_obj["cdsEndStat"]
exonFrames = gene_obj["exonFrames"]
name = name2
id = name_id
e_start = exonStarts.split(",")
e_end = exonEnds.split(",")
# Determine Y Coordinate
i = 0
y_coord = None
while y_coord == None:
if i > len(lines) - 1:
y_coord = i + 1
lines.append(int(txEnd))
elif int(txStart) > (gap + lines[i]):
y_coord = i + 1
lines[i] = int(txEnd)
else:
i += 1
genes_plot_start.append(int(txStart) / 1000000.0)
genes_plot_end.append(int(txEnd) / 1000000.0)
genes_plot_y.append(y_coord)
genes_plot_name.append(name + " ")
for i in range(len(e_start) - 1):
if strand == "+":
exon = i + 1
else:
exon = len(e_start) - 1 - i
width = (int(e_end[i]) - int(e_start[i])) / 1000000.0
x_coord = int(e_start[i]) / 1000000.0 + (width / 2)
exons_plot_x.append(x_coord)
exons_plot_y.append(y_coord)
exons_plot_w.append(width)
exons_plot_h.append(tall)
exons_plot_name.append(name)
exons_plot_id.append(id)
exons_plot_exon.append(exon)
n_rows = len(lines)
genes_plot_yn = [n_rows - x + 0.5 for x in genes_plot_y]
exons_plot_yn = [n_rows - x + 0.5 for x in exons_plot_y]
yr2 = Range1d(start=0, end=n_rows)
data_gene_plot = {
'exons_plot_x': exons_plot_x,
'exons_plot_yn': exons_plot_yn,
'exons_plot_w': exons_plot_w,
'exons_plot_h': exons_plot_h,
'exons_plot_name': exons_plot_name,
'exons_plot_id': exons_plot_id,
'exons_plot_exon': exons_plot_exon
}
source_gene_plot = ColumnDataSource(data_gene_plot)
max_genes = 40
# if len(lines) < 3 or len(genes_raw) > max_genes:
if len(lines) < 3:
plot_h_pix = 250
else:
plot_h_pix = 250 + (len(lines) - 2) * 50
gene_plot = figure(
min_border_top=2,
min_border_bottom=0,
min_border_left=100,
min_border_right=5,
x_range=xr,
y_range=yr2,
border_fill_color='white',
title="",
h_symmetry=False,
v_symmetry=False,
logo=None,
plot_width=900,
plot_height=plot_h_pix,
tools=
"hover,xpan,box_zoom,wheel_zoom,tap,undo,redo,reset,previewsave")
# if len(genes_raw) <= max_genes:
gene_plot.segment(genes_plot_start,
genes_plot_yn,
genes_plot_end,
genes_plot_yn,
color="black",
alpha=1,
line_width=2)
gene_plot.rect(x='exons_plot_x',
y='exons_plot_yn',
width='exons_plot_w',
height='exons_plot_h',
source=source_gene_plot,
fill_color="grey",
line_color="grey")
gene_plot.text(genes_plot_start,
genes_plot_yn,
text=genes_plot_name,
alpha=1,
text_font_size="7pt",
text_font_style="bold",
text_baseline="middle",
text_align="right",
angle=0)
hover = gene_plot.select(dict(type=HoverTool))
hover.tooltips = OrderedDict([
("Gene", "@exons_plot_name"),
("Transcript ID", "@exons_plot_id"),
("Exon", "@exons_plot_exon"),
])
# else:
# x_coord_text = coord1/1000000.0 + (coord2/1000000.0 - coord1/1000000.0) / 2.0
# gene_plot.text(x_coord_text, n_rows / 2.0, text=message, alpha=1,
# text_font_size="12pt", text_font_style="bold", text_baseline="middle", text_align="center", angle=0)
gene_plot.xaxis.axis_label = "Chromosome " + chromosome + " Coordinate (Mb)(" + genome_build_vars[
genome_build]['title'] + ")"
gene_plot.yaxis.axis_label = "Genes (All Transcripts)"
gene_plot.ygrid.grid_line_color = None
gene_plot.yaxis.axis_line_color = None
gene_plot.yaxis.minor_tick_line_color = None
gene_plot.yaxis.major_tick_line_color = None
gene_plot.yaxis.major_label_text_color = None
gene_plot.toolbar_location = "below"
# Change output backend to SVG temporarily for headless export
assoc_plot.output_backend = "svg"
rug.output_backend = "svg"
gene_plot.output_backend = "svg"
export_svgs(assoc_plot,
filename=tmp_dir + "assoc_plot_1_" + request + ".svg")
export_svgs(gene_plot,
filename=tmp_dir + "gene_plot_1_" + request + ".svg")
# 1 pixel = 0.0264583333 cm
svg_height = str(20.00 + (0.0264583333 * plot_h_pix)) + "cm"
svg_height_scaled = str(100.00 + (0.1322916665 * plot_h_pix)) + "cm"
# Concatenate svgs
sg.Figure(
"24.59cm", svg_height,
sg.SVG(tmp_dir + "assoc_plot_1_" + request + ".svg"),
sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").move(
-40, 630)).save(tmp_dir + "assoc_plot_" + request + ".svg")
sg.Figure(
"122.95cm", svg_height_scaled,
sg.SVG(tmp_dir + "assoc_plot_1_" + request + ".svg").scale(5),
sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").scale(5).move(
-200,
3150)).save(tmp_dir + "assoc_plot_scaled_" + request + ".svg")
# Export to PDF
subprocess.call("phantomjs ./rasterize.js " + tmp_dir + "assoc_plot_" +
request + ".svg " + tmp_dir + "assoc_plot_" + request +
".pdf",
shell=True)
# Export to PNG
subprocess.call("phantomjs ./rasterize.js " + tmp_dir +
"assoc_plot_scaled_" + request + ".svg " + tmp_dir +
"assoc_plot_" + request + ".png",
shell=True)
# Export to JPEG
subprocess.call("phantomjs ./rasterize.js " + tmp_dir +
"assoc_plot_scaled_" + request + ".svg " + tmp_dir +
"assoc_plot_" + request + ".jpeg",
shell=True)
# Remove individual SVG files after they are combined
subprocess.call("rm " + tmp_dir + "assoc_plot_1_" + request + ".svg",
shell=True)
subprocess.call("rm " + tmp_dir + "gene_plot_1_" + request + ".svg",
shell=True)
# Remove scaled SVG file after it is converted to png and jpeg
subprocess.call("rm " + tmp_dir + "assoc_plot_scaled_" + request +
".svg",
shell=True)
# Gene Plot (Collapsed)
else:
# Get genes from LDassoc.py output file
genes_c_file = tmp_dir + "genes_c_" + request + ".json"
genes_c_raw = open(genes_c_file).readlines()
genes_c_plot_start = []
genes_c_plot_end = []
genes_c_plot_y = []
genes_c_plot_name = []
exons_c_plot_x = []
exons_c_plot_y = []
exons_c_plot_w = []
exons_c_plot_h = []
exons_c_plot_name = []
exons_c_plot_id = []
message_c = ["Too many genes to plot."]
lines_c = [0]
gap = 80000
tall = 0.75
if genes_c_raw != None and len(genes_c_raw) > 0:
for gene_raw_obj in genes_c_raw:
gene_c_obj = json.loads(gene_raw_obj)
chrom = gene_c_obj["chrom"]
txStart = gene_c_obj["txStart"]
txEnd = gene_c_obj["txEnd"]
exonStarts = gene_c_obj["exonStarts"]
exonEnds = gene_c_obj["exonEnds"]
name2 = gene_c_obj["name2"]
transcripts = gene_c_obj["transcripts"]
name = name2
e_start = exonStarts.split(",")
e_end = exonEnds.split(",")
e_transcripts = transcripts.split(",")
# Determine Y Coordinate
i = 0
y_coord = None
while y_coord == None:
if i > len(lines_c) - 1:
y_coord = i + 1
lines_c.append(int(txEnd))
elif int(txStart) > (gap + lines_c[i]):
y_coord = i + 1
lines_c[i] = int(txEnd)
else:
i += 1
genes_c_plot_start.append(int(txStart) / 1000000.0)
genes_c_plot_end.append(int(txEnd) / 1000000.0)
genes_c_plot_y.append(y_coord)
genes_c_plot_name.append(name + " ")
# for i in range(len(e_start)):
for i in range(len(e_start) - 1):
width = (int(e_end[i]) - int(e_start[i])) / 1000000.0
x_coord = int(e_start[i]) / 1000000.0 + (width / 2)
exons_c_plot_x.append(x_coord)
exons_c_plot_y.append(y_coord)
exons_c_plot_w.append(width)
exons_c_plot_h.append(tall)
exons_c_plot_name.append(name)
exons_c_plot_id.append(e_transcripts[i].replace("-", ","))
n_rows_c = len(lines_c)
genes_c_plot_yn = [n_rows_c - x + 0.5 for x in genes_c_plot_y]
exons_c_plot_yn = [n_rows_c - x + 0.5 for x in exons_c_plot_y]
yr2_c = Range1d(start=0, end=n_rows_c)
data_gene_c_plot = {
'exons_c_plot_x': exons_c_plot_x,
'exons_c_plot_yn': exons_c_plot_yn,
'exons_c_plot_w': exons_c_plot_w,
'exons_c_plot_h': exons_c_plot_h,
'exons_c_plot_name': exons_c_plot_name,
'exons_c_plot_id': exons_c_plot_id
}
source_gene_c_plot = ColumnDataSource(data_gene_c_plot)
max_genes_c = 40
# if len(lines_c) < 3 or len(genes_c_raw) > max_genes_c:
if len(lines_c) < 3:
plot_c_h_pix = 250
else:
plot_c_h_pix = 250 + (len(lines_c) - 2) * 50
gene_c_plot = figure(
min_border_top=2,
min_border_bottom=0,
min_border_left=100,
min_border_right=5,
x_range=xr,
y_range=yr2_c,
border_fill_color='white',
title="",
h_symmetry=False,
v_symmetry=False,
logo=None,
plot_width=900,
plot_height=plot_c_h_pix,
tools=
"hover,xpan,box_zoom,wheel_zoom,tap,undo,redo,reset,previewsave")
# if len(genes_c_raw) <= max_genes_c:
gene_c_plot.segment(genes_c_plot_start,
genes_c_plot_yn,
genes_c_plot_end,
genes_c_plot_yn,
color="black",
alpha=1,
line_width=2)
gene_c_plot.rect(x='exons_c_plot_x',
y='exons_c_plot_yn',
width='exons_c_plot_w',
height='exons_c_plot_h',
source=source_gene_c_plot,
fill_color="grey",
line_color="grey")
gene_c_plot.text(genes_c_plot_start,
genes_c_plot_yn,
text=genes_c_plot_name,
alpha=1,
text_font_size="7pt",
text_font_style="bold",
text_baseline="middle",
text_align="right",
angle=0)
hover = gene_c_plot.select(dict(type=HoverTool))
hover.tooltips = OrderedDict([
("Gene", "@exons_c_plot_name"),
("Transcript IDs", "@exons_c_plot_id"),
])
# else:
# x_coord_text = coord1/1000000.0 + (coord2/1000000.0 - coord1/1000000.0) / 2.0
# gene_c_plot.text(x_coord_text, n_rows_c / 2.0, text=message_c, alpha=1,
# text_font_size="12pt", text_font_style="bold", text_baseline="middle", text_align="center", angle=0)
gene_c_plot.xaxis.axis_label = "Chromosome " + chromosome + " Coordinate (Mb)(" + genome_build_vars[
genome_build]['title'] + ")"
gene_c_plot.yaxis.axis_label = "Genes (Transcripts Collapsed)"
gene_c_plot.ygrid.grid_line_color = None
gene_c_plot.yaxis.axis_line_color = None
gene_c_plot.yaxis.minor_tick_line_color = None
gene_c_plot.yaxis.major_tick_line_color = None
gene_c_plot.yaxis.major_label_text_color = None
gene_c_plot.toolbar_location = "below"
# Change output backend to SVG temporarily for headless export
assoc_plot.output_backend = "svg"
rug.output_backend = "svg"
gene_c_plot.output_backend = "svg"
export_svgs(assoc_plot,
filename=tmp_dir + "assoc_plot_1_" + request + ".svg")
export_svgs(gene_c_plot,
filename=tmp_dir + "gene_plot_1_" + request + ".svg")
# 1 pixel = 0.0264583333 cm
svg_height = str(20.00 + (0.0264583333 * plot_c_h_pix)) + "cm"
svg_height_scaled = str(100.00 + (0.1322916665 * plot_c_h_pix)) + "cm"
# Concatenate svgs
sg.Figure(
"24.59cm", svg_height,
sg.SVG(tmp_dir + "assoc_plot_1_" + request + ".svg"),
sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").move(
-40, 630)).save(tmp_dir + "assoc_plot_" + request + ".svg")
sg.Figure(
"122.95cm", svg_height_scaled,
sg.SVG(tmp_dir + "assoc_plot_1_" + request + ".svg").scale(5),
sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").scale(5).move(
-200,
3150)).save(tmp_dir + "assoc_plot_scaled_" + request + ".svg")
# Export to PDF
subprocess.call("phantomjs ./rasterize.js " + tmp_dir + "assoc_plot_" +
request + ".svg " + tmp_dir + "assoc_plot_" + request +
".pdf",
shell=True)
# Export to PNG
subprocess.call("phantomjs ./rasterize.js " + tmp_dir +
"assoc_plot_scaled_" + request + ".svg " + tmp_dir +
"assoc_plot_" + request + ".png",
shell=True)
# Export to JPEG
subprocess.call("phantomjs ./rasterize.js " + tmp_dir +
"assoc_plot_scaled_" + request + ".svg " + tmp_dir +
"assoc_plot_" + request + ".jpeg",
shell=True)
# Remove individual SVG files after they are combined
subprocess.call("rm " + tmp_dir + "assoc_plot_1_" + request + ".svg",
shell=True)
subprocess.call("rm " + tmp_dir + "gene_plot_1_" + request + ".svg",
shell=True)
# Remove scaled SVG file after it is converted to png and jpeg
subprocess.call("rm " + tmp_dir + "assoc_plot_scaled_" + request +
".svg",
shell=True)
reset_output()
# Remove temporary files
subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt", shell=True)
subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True)
subprocess.call("rm " + tmp_dir + "genes_*" + request + "*.json",
shell=True)
subprocess.call("rm " + tmp_dir + "recomb_" + request + ".json",
shell=True)
subprocess.call("rm " + tmp_dir + "assoc_args" + request + ".json",
shell=True)
print("Bokeh high quality image export complete!")
# Return plot output
return None
def test_bounds_with_three_item_tuple_raises_valueerror(self):
with pytest.raises(ValueError):
Range1d(1, 2, bounds=(0, 1, 2))
plot_width=1300,
title="Quantized Circles of Confusion")
l1 = coc_plot.multi_line(xs='x',
ys='coc',
source=source,
line_width=2,
color='color',
legend='legend_label')
# coc_plot.legend.title = "CoCs"
# coc_plot.line('x', 'coc1', source=source, line_width=2, color='blue', legend=legend_label[0])
# coc_plot.line('x', 'coc2', source=source, line_width=2, color='green', legend=legend_label[1])
# coc_plot.line('x', 'coc_diff', source=source, line_width=2, color='black', line_dash=(2,2), legend=legend_label[2])
coc_plot.xaxis.axis_label = "Range (m)"
coc_plot.yaxis.axis_label = "Pixel Radius"
coc_plot.axis.axis_label_text_font_style = "bold"
coc_plot.x_range = Range1d(start=min_x_spin.value, end=max_x_spin.value)
coc_plot.y_range = Range1d(start=min_y_spin.value, end=max_y_spin.value)
# coc_plot.legend[0].location = (900, 200))
# legend = Legend(items=[(("fp 1", "fp 2", "CoC Difference"), [l1])], location=(0, -60))
# coc_plot.add_layout(legend, 'right')
coc_plot.add_tools(ht)
b1_data = np.full((1, blur_img_h * blur_img_w), 255, dtype=np.uint8)
b1_data[:, 0:(blur_img_w * 99)] = 0
tmp_dict = dict(source=source, b_source=b_source, b1_d=b1_data)
tmp_cb = CustomJS(args=tmp_dict,
code="""
var data = source.data;
//var b1_data = [];
//Object.assign(b1_data, b1_d);
def test_init_with_datetime(self):
range1d = Range1d(start=dt.datetime(2016, 4, 28, 2, 20, 50),
end=dt.datetime(2017, 4, 28, 2, 20, 50))
assert range1d.start == dt.datetime(2016, 4, 28, 2, 20, 50)
assert range1d.end == dt.datetime(2017, 4, 28, 2, 20, 50)
assert range1d.bounds is None
for year in years:
fertility = fertility_df[year]
fertility.name = 'fertility'
life = life_expectancy_df[year]
life.name = 'life'
population = population_df_size[year]
population.name = 'population'
new_df = pd.concat([fertility, life, population, region_name], axis=1)
sources['_' + str(year)] = ColumnDataSource(new_df)
dictionary_of_sources = dict(
zip([x for x in years], ['_%s' % x for x in years]))
js_source_array = str(dictionary_of_sources).replace("'", "")
xdr = Range1d(1, 9)
ydr = Range1d(20, 100)
plot = Plot(
x_range=xdr,
y_range=ydr,
title=Title(text=''),
plot_width=800,
plot_height=400,
outline_line_color=None,
toolbar_location=None,
min_border=20,
)
AXIS_FORMATS = dict(
minor_tick_in=None,
minor_tick_out=None,
def make_axis(axis,
size,
factors,
dim,
flip=False,
rotation=0,
label_size=None,
tick_size=None,
axis_height=35):
factors = list(map(dim.pprint_value, factors))
nchars = np.max([len(f) for f in factors])
ranges = FactorRange(factors=factors)
ranges2 = Range1d(start=0, end=1)
axis_label = dim_axis_label(dim)
reset = "range.setv({start: 0, end: range.factors.length})"
ranges.callback = CustomJS(args=dict(range=ranges), code=reset)
axis_props = {}
if label_size:
axis_props['axis_label_text_font_size'] = value(label_size)
if tick_size:
axis_props['major_label_text_font_size'] = value(tick_size)
tick_px = font_size_to_pixels(tick_size)
if tick_px is None:
tick_px = 8
label_px = font_size_to_pixels(label_size)
if label_px is None:
label_px = 10
rotation = np.radians(rotation)
if axis == 'x':
align = 'center'
# Adjust height to compensate for label rotation
height = int(axis_height + np.abs(np.sin(rotation)) *
((nchars * tick_px) * 0.82)) + tick_px + label_px
opts = dict(x_axis_type='auto',
x_axis_label=axis_label,
x_range=ranges,
y_range=ranges2,
plot_height=height,
plot_width=size)
else:
# Adjust width to compensate for label rotation
align = 'left' if flip else 'right'
width = int(axis_height + np.abs(np.cos(rotation)) *
((nchars * tick_px) * 0.82)) + tick_px + label_px
opts = dict(y_axis_label=axis_label,
x_range=ranges2,
y_range=ranges,
plot_width=width,
plot_height=size)
p = Figure(toolbar_location=None, **opts)
p.outline_line_alpha = 0
p.grid.grid_line_alpha = 0
if axis == 'x':
p.yaxis.visible = False
axis = p.xaxis[0]
if flip:
p.above = p.below
p.below = []
p.xaxis[:] = p.above
else:
p.xaxis.visible = False
axis = p.yaxis[0]
if flip:
p.right = p.left
p.left = []
p.yaxis[:] = p.right
axis.major_label_orientation = rotation
axis.major_label_text_align = align
axis.major_label_text_baseline = 'middle'
axis.update(**axis_props)
return p
def homepage(request):
xmax = ultima_fecha('ise2_infra')
df1, df2, df3, xmin_ise1_infra, xmax_ise1_infra, xmin_ise2_infra, xmax_ise2_infra, xmin_e1ms1, xmax_e1ms1 = get_data(
)
d = xmax - timedelta(seconds=30)
#d= datetime.now() - timedelta(hours=48)
#xmax =datetime.now()
plot1, script1, div1 = get_plot(df1, 'ise1_infra', xmin_ise1_infra,
xmax_ise1_infra)
plot2, script2, div2 = get_plot(df2, 'ise2_infra', xmin_ise2_infra,
xmax_ise2_infra)
plot3, script3, div3 = get_plot(df3, 'e1ms1', xmin_e1ms1, xmax_e1ms1)
xmin = xmax - timedelta(seconds=30)
# data_list = ultimos_datos()
# frame = pd.DataFrame(data_list)
# frame.columns= ['fecha_sistema','gxe','gye','gze','axe','aye','aze']
df = pd.DataFrame(
list(
ISE2_INFRA.objects.filter(fecha_recepcion__range=(d, xmax)).values(
'fecha_recepcion', 'infrasonido_1', 'infrasonido_2',
'infrasonido_3', 'infrasonido_4').order_by('fecha_recepcion')))
# df = pd.DataFrame(list(ISE2_INFRA.objects.filter(fecha_recepcion__range=(d,datetime.now())).values('fecha_recepcion','infrasonido_1','infrasonido_2','infrasonido_3','infrasonido_4').order_by('fecha_recepcion')))
# df = pd.DataFrame(list(ISE2_INFRA.objects.all().values('fecha_recepcion','infrasonido_1','infrasonido_2','infrasonido_3','infrasonido_4')))
source = ColumnDataSource(df)
p1 = figure(title='Data Cruda del Sensor 1 al {0}'.format(xmax),
x_axis_label='Tiempo',
y_axis_label='Cuentas',
x_axis_type='datetime',
y_axis_type='log',
plot_width=500,
plot_height=300)
p1.x_range = Range1d(xmin, xmax)
l1 = p1.line('fecha_recepcion',
'infrasonido_1',
source=source,
line_width=2,
line_alpha=1,
line_color="blue")
p2 = figure(title='Data Cruda del Sensor 2 al {0}'.format(xmax),
x_axis_label='Tiempo',
y_axis_label='Cuentas',
x_axis_type='datetime',
y_axis_type='log',
plot_width=500,
plot_height=300)
p2.x_range = Range1d(xmin, xmax)
l2 = p2.line('fecha_recepcion',
'infrasonido_2',
source=source,
line_width=2,
line_alpha=1,
line_color="red")
p3 = figure(title='Data Cruda del Sensor 3 al {0}'.format(xmax),
x_axis_label='Tiempo',
y_axis_label='Cuentas',
x_axis_type='datetime',
y_axis_type='log',
plot_width=500,
plot_height=300)
p3.x_range = Range1d(xmin, xmax)
l3 = p3.line('fecha_recepcion',
'infrasonido_3',
source=source,
line_width=2,
line_alpha=1,
line_color="green")
p4 = figure(title='Data Cruda del Sensor 4 al {0}'.format(xmax),
x_axis_label='Tiempo',
y_axis_label='Cuentas',
x_axis_type='datetime',
y_axis_type='log',
plot_width=500,
plot_height=300)
p4.x_range = Range1d(xmin, xmax)
l4 = p4.line('fecha_recepcion',
'infrasonido_4',
source=source,
line_width=2,
line_alpha=1,
line_color="orange")
#plot.xaxis.formatter=DatetimeTickFormatter(
#hours=["%d %B %Y"],
#minutes=["%d %B %Y"],
#days=["%d %B %Y"],
#months=["%d %B %Y"],
#years=["%d %B %Y"],
#)
#plot.xaxis.major_label_orientation = pi/4
#legend = Legend(items=[
#("Infrasonido 1", [l1]),
#("Infrasonido 2", [l2]),
#("Infrasonido 3", [l3]),
#("Infrasonido 4", [l4])
#], location=(0, -30))
#plot.add_layout(legend, 'right')
plot = gridplot([[p1], [p2], [p3], [p4]])
script, div = components(plot)
img1 = ultima_foto()
return render_to_response(
'polls/dash.html', {
'script1': script1,
'div1': div1,
'script2': script2,
'div2': div2,
'script3': script3,
'div3': div3,
'img1': img1
})
