def get_data(username, docid, datasourceid): bokehuser = bokeh_app.authentication.current_user() request_username = bokehuser.username # handle docid later... clientdoc = bokeh_app.backbone_storage.get_document(docid) prune(clientdoc) init_bokeh(clientdoc) serverdatasource = clientdoc._models[datasourceid] parameters = json.loads(request.values.get('resample_parameters')) plot_state = json.loads(request.values.get('plot_state')) render_state = json.loads(request.values.get('render_state')) if 'render_state' in request.values else None # TODO: Desserializing directly to ranges....awk-ward. # There is probably a better way via the properties system that detects type...probably... plot_state=dict([(k, Range1d.load_json(r)) for k,r in iteritems(plot_state)]) result = bokeh_app.datamanager.get_data( request_username, serverdatasource, parameters, plot_state, render_state) result = make_json(protocol.serialize_json(result)) return result
def test_init_with_float(self): range1d = Range1d(start=-1.0, end=3.0) assert range1d.start == -1.0 assert range1d.end == 3.0 assert range1d.bounds is None
def test_init_with_timedelta(self): range1d = Range1d(start=-dt.timedelta(seconds=5), end=dt.timedelta(seconds=3)) assert range1d.start == -dt.timedelta(seconds=5) assert range1d.end == dt.timedelta(seconds=3) assert range1d.bounds is None
def test_bounds_with_text_rejected_as_the_correct_value_error(self): with pytest.raises(ValueError) as e: Range1d(1, 2, bounds="21" ) # The string is indexable, so this may not fail properly assert e.value.args[0].startswith('expected an element of either')
def plotOutlierHistogram(dataframe, maxOutliers, func, statisticalOutlierThreshold, userLatencyThreshold, averageDuration, maxDuration): global pixelsForTitle global pixelsPerHeightUnit global plotWidth global timeUnitString cds = ColumnDataSource(dataframe) figureTitle = "Occurrences of " + func + " that took longer than " \ + statisticalOutlierThreshold + "." hover = HoverTool( tooltips=[("interval start", "@lowerbound{0,0}"), ("interval end", "@upperbound{0,0}")]) TOOLS = [hover, "tap, reset"] p = figure(title = figureTitle, plot_width = plotWidth, plot_height = min(500, (max(5, (maxOutliers + 1)) \ * pixelsPerHeightUnit + \ pixelsForTitle)), x_axis_label = "Execution timeline (" + timeUnitString + ")", y_axis_label = "Number of outliers", tools = TOOLS, toolbar_location="above") y_ticker_max = p.plot_height // pixelsPerHeightUnit y_ticker_step = max(1, (maxOutliers + 1) // y_ticker_max) y_upper_bound = (maxOutliers // y_ticker_step + 2) * y_ticker_step p.yaxis.ticker = FixedTicker( ticks=list(range(0, y_upper_bound, y_ticker_step))) p.ygrid.ticker = FixedTicker( ticks=list(range(0, y_upper_bound, y_ticker_step))) p.xaxis.formatter = NumeralTickFormatter(format="0,") p.y_range = Range1d(0, y_upper_bound) p.quad(left='lowerbound', right='upperbound', bottom='bottom', top='height', color=funcToColor[func], source=cds, nonselection_fill_color=funcToColor[func], nonselection_fill_alpha=1.0, line_color="lightgrey", selection_fill_color=funcToColor[func], selection_line_color="grey") p.x(x='markerX', y='markerY', size='markersize', color='navy', line_width=1, source=cds) # Add an annotation to the chart # y_max = dataframe['height'].max() text = "Average duration: " + '{0:,.0f}'.format(averageDuration) + " " + \ timeUnitString + \ ". Maximum duration: " + '{0:,.0f}'.format(maxDuration) + " " + \ timeUnitString + ". " if (userLatencyThreshold is not None): text = text + \ "An \'x\' shows intervals with operations exceeding " + \ "a user-defined threshold of " + \ userLatencyThreshold + "." mytext = Label(x=0, y=y_upper_bound - y_ticker_step, text=text, text_color="grey", text_font="helvetica", text_font_size="10pt", text_font_style="italic") p.add_layout(mytext) url = "@bucketfiles" taptool = p.select(type=TapTool) taptool.callback = OpenURL(url=url) return p
def y_range(f, y, source, ylo=None, yhi=None): ylo, yhi = source[y].dropna().min( ) if ylo is None else ylo, source[y].dropna().max() if yhi is None else yhi dy = yhi - ylo f.y_range = Range1d(start=ylo - 0.1 * dy, end=yhi + 0.1 * dy)
min_x = min([DATA[site][which[0]][0] for site in DATA]) max_x = max([DATA[site][which[0]][-1] for site in DATA]) min_y = min([min(DATA[site][which[1]]) for site in DATA]) max_y = max([max(DATA[site][which[1]]) for site in DATA]) # we will set the y axis range at +/- 10% of the data max amplitude ampli = max_y - min_y min_y = min_y - ampli*0.1/100 max_y = max_y + ampli*0.1/100 milestone = time.time() if type(min_x) == datetime: fig = figure(output_backend = "webgl", title = 'TCCON '+which[1]+' vs '+which[0], y_range=[min_y,max_y], plot_width = 900, plot_height = 650, tools = TOOLS, toolbar_location = 'above', x_axis_type='datetime', x_range = Range1d(min_x,max_x)) else: fig = figure(output_backend = "webgl", title = 'TCCON '+which[1]+' vs '+which[0], y_range=[min_y,max_y], plot_width = 900, plot_height = 650, tools = TOOLS, toolbar_location = 'above', x_range = Range1d(int(min_x),ceil(max_x))) plots=[] for site in lat_ordered_sites: plots.append( fig.scatter(x='x',y='y',color=DATA[site]['color'],alpha=0.5,source=sources[site]) ) N_plots = range(len(plots)) legend=Legend(items=[(lat_ordered_sites[i],[plots[i]]) for i in N_plots],location=(0,0)) fig.add_layout(legend,'right') fig.yaxis.axis_label = which[1] fig.xaxis.axis_label = which[0] checkbox = CheckboxGroup(labels=[site+' '+str(round(all_latitudes[lat_ordered_sites.index(site)],2)) for site in lat_ordered_sites],active=[],width=200)
#################################### #################################### # Figure: #################################### #################################### bk.output_file("Berlin_dens_gmap.html", mode="cdn") # title="Berlin population") source = bk.ColumnDataSource(data=dict( boroughsnames=boroughsnames, population=population, area=area, density=density)) p = GMapPlot(title="", # False, # "Berlin population", # x_axis_label='Longitude', y_axis_label='Latitude', plot_width=570, plot_height=500, x_range = Range1d(), y_range = Range1d(), # border_fill = '#130f30', map_options=GMapOptions(lat=52.521123, lng=13.407478, zoom=10)) # tools="pan,wheel_zoom,box_zoom,reset,hover,save") p.map_options.map_type="satellite" # satellite, roadmap, terrain or hybrid source_patches = bk.ColumnDataSource(data=dict( boroughs_xs=boroughs_xs, boroughs_ys=boroughs_ys, boroughs_colors=boroughs_colors, boroughsnames=boroughsnames, population=population, area=area, density=density)) patches = Patches(xs="boroughs_xs", ys="boroughs_ys", fill_color="boroughs_colors", fill_alpha=0.5, line_color="black", line_width=0.5) patches_glyph = p.add_glyph(source_patches, patches) p.add_tools(PanTool(), WheelZoomTool(), BoxSelectTool(), HoverTool(), ResetTool(), PreviewSaveTool())
import networkx as nx from bokeh.io import output_file, show from bokeh.models import (BoxSelectTool, Circle, EdgesAndLinkedNodes, HoverTool, MultiLine, Plot, Range1d, TapTool) from bokeh.palettes import Spectral4 from bokeh.plotting import from_networkx G=nx.karate_club_graph() plot = Plot(width=400, height=400, x_range=Range1d(-1.1,1.1), y_range=Range1d(-1.1,1.1)) plot.title.text = "Graph Interaction Demonstration" plot.add_tools(HoverTool(tooltips=None), TapTool(), BoxSelectTool()) graph_renderer = from_networkx(G, nx.circular_layout, scale=1, center=(0,0)) graph_renderer.node_renderer.glyph = Circle(size=15, fill_color=Spectral4[0]) graph_renderer.node_renderer.selection_glyph = Circle(size=15, fill_color=Spectral4[2]) graph_renderer.node_renderer.hover_glyph = Circle(size=15, fill_color=Spectral4[1]) graph_renderer.edge_renderer.glyph = MultiLine(line_color="#CCCCCC", line_alpha=0.8, line_width=5) graph_renderer.edge_renderer.selection_glyph = MultiLine(line_color=Spectral4[2], line_width=5) graph_renderer.edge_renderer.hover_glyph = MultiLine(line_color=Spectral4[1], line_width=5) graph_renderer.selection_policy = EdgesAndLinkedNodes() graph_renderer.inspection_policy = EdgesAndLinkedNodes() plot.renderers.append(graph_renderer)
from bokeh.client import push_session from bokeh.document import Document from bokeh.models import (Plot, DataRange1d, LinearAxis, Range1d, ColumnDataSource, PanTool, WheelZoomTool, Line) document = Document() session = push_session(document) x = linspace(-6 * pi, 6 * pi, 1000) y = sin(x) z = cos(x) source = ColumnDataSource(data=dict(x=x, y=y, z=z)) plot = Plot(x_range=Range1d(-2 * pi, 2 * pi), y_range=DataRange1d(), min_border=50) line_glyph = Line(x="x", y="y", line_color="blue") plot.add_glyph(source, line_glyph) line_glyph2 = Line(x="x", y="z", line_color="red") plot.add_glyph(source, line_glyph2) plot.add_layout(LinearAxis(), 'below') plot.add_layout(LinearAxis(), 'left') plot.add_tools(PanTool(), WheelZoomTool()) document.add_root(plot)
def plot(xaxis, yaxis, xaxis_name='Date', yaxis_name='', title='', format='line', external_data=[], notebook=False, mission='', target='', yaxis_units='', date_format='TDB', plot_width=975, plot_height=300, fill_color=[], fill_alpha=0, background_image=False, line_width=2): if not isinstance(yaxis_name, list): yaxis_name = [yaxis_name] yaxis = [yaxis] if not title: title = '{} {}'.format(mission, yaxis_name).title().upper() html_file_name = 'plot_{}_{}_{}-{}.html'.format('Time', yaxis_name, mission, target) html_file_name = valid_url(html_file_name) else: title = title.upper() if ' ' in title: html_file_name = title.replace(' ', '_').upper() else: html_file_name = title html_file_name = valid_url(html_file_name) # TODO: Move this to the time object (convert to datatime) # Function needs to be vectorised # x = self.time.window if xaxis_name == 'Date': window_dt = [] window = xaxis for element in window: window_dt.append(et_to_datetime(element, date_format)) x = window_dt else: x = xaxis y = yaxis if notebook: output_notebook() else: output_file(html_file_name + '.html') if xaxis_name == 'Date': x_axis_type = "datetime" else: x_axis_type = "auto" p = figure(title=title, plot_width=plot_width, plot_height=plot_height, x_axis_label=xaxis_name.upper(), y_axis_label=yaxis_units, x_axis_type=x_axis_type) if xaxis_name == 'Date': p.xaxis.formatter = DatetimeTickFormatter( seconds=["%Y-%m-%d %H:%M:%S"], minsec=["%Y-%m-%d %H:%M:%S"], minutes=["%Y-%m-%d %H:%M:%S"], hourmin=["%Y-%m-%d %H:%M:%S"], hours=["%Y-%m-%d %H:%M:%S"], days=["%Y-%m-%d %H:%M:%S"], months=["%Y-%m-%d %H:%M:%S"], years=["%Y-%m-%d %H:%M:%S"], ) hover = HoverTool( tooltips=[(xaxis_name, '@x{0.000}'), (title, '@y{0.000}')], formatters={xaxis_name: 'numeral', title: 'numeral'}) p.add_tools(hover) if external_data: window_dt = [] window = external_data[0] for element in window: window_dt.append(et_to_datetime(element, 'TDB')) x_ext = window_dt y_ext = external_data[1] if format == 'circle': p.circle(x_ext, y_ext, size=5, color='red') elif format == 'line': p.line(x_ext, y_ext, line_width=2, color='red') # add a line renderer with legend and line thickness color_list = ['red', 'green', 'blue', 'orange', "black", 'darkgoldenrod', 'chocolate', 'aqua', 'coral', 'darkcyan', 'cornflowerblue' 'aquamarine', 'darkturquoise', 'cornsilk'] index = 0 color_idx = 0 if background_image: if 'TGO' in mission.upper() or 'MEX' in mission.upper(): image = 'Mars_Viking_MDIM21_ClrMosaic_global_1024.jpg' else: image = 'Earth_Contemporary_Basic.png' p.image_url(url=[os.path.join(os.path.dirname(images.__file__), image)], x=-180, y=-90, w=360, h=180, anchor="bottom_left", global_alpha=0.6) left, right, bottom, top = -180, 180, -90, 90 p.x_range = Range1d(left, right) p.y_range = Range1d(bottom, top) if format == 'scatter': is_multi_legend = len(y) <= len(yaxis_name) for idx in range(len(y)): p.circle([x[idx]], [y[idx]], size=3, color=color_list[color_idx] if is_multi_legend else color_list[0], legend=str(yaxis_name[idx]).upper() if is_multi_legend else str(yaxis_name[0]).upper()) color_idx = idx % len(color_list) else: for element in y: if format == 'circle': p.line(x, element, line_width=line_width, color=color_list[color_idx], legend=str(yaxis_name[index]).upper()) p.circle(x, element, fill_color="white", size=8) if format == 'circle_only': p.circle(x, element, size=3, color=color_list[color_idx], legend=str(yaxis_name[index]).upper()) elif format == 'line': p.line(x, element, line_width=line_width, color=color_list[color_idx], legend=str(yaxis_name[index]).upper()) index += 1 color_idx = index % len(color_list) p.legend.click_policy = "hide" # show the results show(p) return
def create_bar_chart(data, title, x_name, y_name, hover_tool=None, width=1200, height=300): """Creates a bar chart plot, uses centcom dashboard styling. Parameters: - data, dict - title, str - names of x and y axes, str - hover tool, html""" source = ColumnDataSource(data) # Gets the data for the plot # From the data gets the axis names xdr = FactorRange(factors=data[x_name]) ydr = Range1d(start=0, end=max(data[y_name]) * 1.5) tools = [] if hover_tool: # turns on the over tool if exists tools = [ hover_tool, ] # Defines the plot main figure plot = figure( title=title, x_range=xdr, # value ranges y_range=ydr, plot_width=width, plot_height=height, h_symmetry=False, v_symmetry=False, min_border=0, toolbar_location="above", tools=tools, # responsive=True, outline_line_color="#666666") # Defines the plot's shape as a vertical bar and sets its parameters glyph = VBar(x=x_name, top=y_name, bottom=0, width=.8, fill_color="#e12127") # Adds the glyph to the plot with the data plot.add_glyph(source, glyph) # Defines the x, y axes xaxis = LinearAxis() yaxis = LinearAxis() plot.add_layout(Grid(dimension=0, ticker=xaxis.ticker)) plot.add_layout(Grid(dimension=1, ticker=yaxis.ticker)) plot.toolbar.logo = None plot.min_border_top = 0 plot.xaxis.axis_label = "Days after app deployment" plot.yaxis.axis_label = "Bugs found" plot.xaxis.major_label_orientation = 1 plot.xgrid.grid_line_color = None plot.ygrid.grid_line_color = "#999999" plot.ygrid.grid_line_alpha = 0.1 return plot
# Test the model model.eval() with torch.no_grad(): correct = 0 total = 0 for images, labels in test_loader: outputs = model(images) _, predicted = torch.max(outputs.data, 1) total += labels.size(0) correct += (predicted == labels).sum().item() print('Test Accuracy of the model on the 10000 test images: {} %'.format( (correct / total) * 100)) # Save the model and plot torch.save(model.state_dict(), MODEL_STORE_PATH + 'conv_net_model.ckpt') p = figure(y_axis_label='Loss', width=850, y_range=(0, 1), title='PyTorch ConvNet results') p.extra_y_ranges = {'Accuracy': Range1d(start=0, end=100)} p.add_layout(LinearAxis(y_range_name='Accuracy', axis_label='Accuracy (%)'), 'right') p.line(np.arange(len(loss_list)), loss_list) p.line(np.arange(len(loss_list)), np.array(acc_list) * 100, y_range_name='Accuracy', color='red') show(p)
def analyze_bokeh(algo, title_suffix=None, title=None, show_trades=False): """ Draw charts for backtest results Use Bokeh :param title: :param show_trades: :return: """ # TODO Replace to YouEngine class bokeh.plotting.output_file("generated/chart{}.html".format(title_suffix), title=title) p = bokeh.plotting.figure(x_axis_type="datetime", plot_width=1000, plot_height=400, title=title) p.grid.grid_line_alpha = 0.3 p.xaxis.axis_label = 'Date' p.yaxis.axis_label = 'Equity' if algo.records: # Setting the second y axis range name and range p.extra_y_ranges = { "Records": Range1d(start=0, end=algo.data['close'].max()) } # Adding the second axis to the plot. p.add_layout(LinearAxis(y_range_name="Records"), 'right') records = pd.DataFrame(algo.records) for c in records.columns: if c == 'date': continue p.line(records['date'], records[c], color='grey', legend=c, y_range_name="Records") # print(algo.data[['date', 'close', 'sma50', 'sma150']]) p.line(algo.data.index, algo.data['base_equity'], color='#CAD8DE', legend='Buy and Hold') p.line(algo.data.index, algo.data['equity'], color='#49516F', legend='Strategy') p.legend.location = "top_left" # check total trades total_trades = len(algo.account.opened_trades) + len( algo.account.closed_trades) # Always disable show trade when trades more than 200 (Work too slow) if total_trades > 200: show_trades = False logger.warning("Show trades disabled. Use analyze_mpl().") if show_trades: for trade in algo.account.opened_trades: x = time.mktime(trade.date.timetuple()) * 1000 y = algo.data['equity'].loc[trade.date] if trade.type_ == 'Long': p.circle(x, y, size=6, color='green', alpha=0.5) elif trade.type_ == 'Short': p.circle(x, y, size=6, color='red', alpha=0.5) for trade in algo.account.closed_trades: x = time.mktime(trade.date.timetuple()) * 1000 y = algo.data['equity'].loc[trade.date] if trade.type_ == 'Long': p.circle(x, y, size=6, color='blue', alpha=0.5) elif trade.type_ == 'Short': p.circle(x, y, size=6, color='orange', alpha=0.5) bokeh.plotting.show(p)
def make_interactive_network(G, labels=False, title='My Network', color_palette=Blues8, node_size='degree', node_color='modularity_class'): from networkx.algorithms import community # Get network info degrees = dict(networkx.degree(G)) networkx.set_node_attributes(G, name='degree', values=degrees) betweenness_centrality = networkx.betweenness_centrality(G) networkx.set_node_attributes(G, name='betweenness', values=betweenness_centrality) communities = community.greedy_modularity_communities(G) # Create empty dictionaries modularity_color = {} modularity_class = {} #Loop through each community in the network for community_number, community in enumerate(communities): #For each member of the community, add their community number and a distinct color for name in community: modularity_color[name] = Spectral8[community_number] modularity_class[name] = community_number networkx.set_node_attributes(G, modularity_color, 'modularity_color') networkx.set_node_attributes(G, modularity_class, 'modularity_class') #Choose colors for node and edge highlighting node_highlight_color = node_color edge_highlight_color = 'black' #Choose attributes from G network to size and color by — setting manual size (e.g. 10) or color (e.g. 'skyblue') also allowed #Pick a color palette — Blues8, Reds8, Purples8, Oranges8, Viridis8 #color_palette = Blues8 #Choose a title! title = title #Establish which categories will appear when hovering over each node HOVER_TOOLTIPS = [ ("Character", "@index"), ("Degree", "@degree"), ("Modularity Class", "@modularity_class"), ("Modularity Color", "$color[swatch]:modularity_color"), ] #Create a plot — set dimensions, toolbar, and title plot = figure(tooltips=HOVER_TOOLTIPS, tools="pan,wheel_zoom,save,reset", active_scroll='wheel_zoom', x_range=Range1d(-10.1, 10.1), y_range=Range1d(-10.1, 10.1), title=title) #Create a network graph object # https://networkx.github.io/documentation/networkx-1.9/reference/generated/networkx.drawing.layout.spring_layout.html network_graph = from_networkx(G, networkx.spring_layout, scale=10, center=(0, 0)) #Set node sizes and colors according to node degree (color as category from attribute) if node_color == 'degree': #Set node sizes and colors according to node degree (color as spectrum of color palette) minimum_value_color = min( network_graph.node_renderer.data_source.data[node_color]) maximum_value_color = max( network_graph.node_renderer.data_source.data[node_color]) network_graph.node_renderer.glyph = Circle( size=node_size, fill_color=linear_cmap(node_color, color_palette, minimum_value_color, maximum_value_color)) elif node_color == 'modularity_color': #node_color='modularity_color' network_graph.node_renderer.glyph = Circle(size=node_size, fill_color=node_color) #Set node highlight colors network_graph.node_renderer.hover_glyph = Circle( size=node_size, fill_color=node_highlight_color, line_width=2) network_graph.node_renderer.selection_glyph = Circle( size=node_size, fill_color=node_highlight_color, line_width=2) #Set edge opacity and width network_graph.edge_renderer.glyph = MultiLine(line_alpha=0.3, line_width=1) #Set edge highlight colors network_graph.edge_renderer.selection_glyph = MultiLine( line_color=edge_highlight_color, line_width=2) network_graph.edge_renderer.hover_glyph = MultiLine( line_color=edge_highlight_color, line_width=2) #Highlight nodes and edges network_graph.selection_policy = NodesAndLinkedEdges() network_graph.inspection_policy = NodesAndLinkedEdges() plot.renderers.append(network_graph) if labels == True: #Add Labels x, y = zip(*network_graph.layout_provider.graph_layout.values()) node_labels = list(G.nodes()) source = ColumnDataSource({ 'x': x, 'y': y, 'name': [node_labels[i] for i in range(len(x))] }) labels = LabelSet(x='x', y='y', text='name', source=source, background_fill_color='white', text_font_size='10px', background_fill_alpha=.7) plot.renderers.append(labels) show(plot) #save(plot, filename=f"{title}.html")
from numpy import arange, linspace, pi, sin from bokeh.models import LinearAxis, Range1d from bokeh.plotting import figure, output_file, show x = arange(-2 * pi, 2 * pi, 0.2) y = sin(x) y2 = linspace(0, 100, len(x)) p = figure(x_range=(-6.5, 6.5), y_range=(-1.1, 1.1), min_border=80) p.background_fill_color = "#fafafa" p.circle(x, y, color="crimson", size=8) p.yaxis.axis_label = "red circles" p.yaxis.axis_label_text_color = "crimson" p.extra_y_ranges['foo'] = Range1d(0, 100) p.circle(x, y2, color="navy", size=8, y_range_name="foo") ax2 = LinearAxis(y_range_name="foo", axis_label="blue circles") ax2.axis_label_text_color = "navy" p.add_layout(ax2, 'left') output_file("twin_axis.html", title="twin_axis.py example") show(p)
height=100) layout = row(y, div) tab9 = Panel(child=layout, title="All India Tests over Time") #Testing Growth Rate cases_tests['timestamp'] = pd.to_datetime(cases_tests['timestamp'], format=r'%Y-%m-%d') cases_tests['daily_tests'] = cases_tests['totalSamplesTested'].diff(1) cases_tests['daily_confirmed'] = cases_tests['totalPositiveCases'].diff(1) z = figure(plot_width=1200, plot_height=700, sizing_mode="scale_both", x_axis_type='datetime', y_range=Range1d(start=0, end=cases_tests['totalSamplesTested'].max())) z.title.text = 'COVID19 Tests Growth Rate over Time' z.title.align = 'center' z.title.text_font_size = '17px' z.xaxis.axis_label = 'Date' z.yaxis.axis_label = 'Tests Count' z.extra_y_ranges = { 'tests_growth_rate': Range1d(start=cases_tests['totalSamplesTested'].pct_change().min(), end=cases_tests['totalSamplesTested'].pct_change().max()) } z.add_layout(LinearAxis(y_range_name='tests_growth_rate'), 'right') sample_test_growth = z.vbar(x=cases_tests['timestamp'], top=cases_tests['totalSamplesTested'].pct_change(),
def main(): global originator parser = argparse.ArgumentParser( description='based on prov json generate graph') parser.add_argument('-f', '--file', dest='file_name', required=True, help='name of file need to be process') args = parser.parse_args() with open(args.file_name) as f: data = json.load(f) root_url = data['root'] bundle_data = data['bundle'] originator = find_originator(bundle_data) #initial empty graph: G = nx.Graph() # add all nodes node_types = [item for item in bundle_data if item in NODE_TYPES] for node_type in node_types: print("add all " + node_type + " nodes") nodes = bundle_data[node_type] for node_name in nodes: node = nodes[node_name] n_type = node[add_prefix_to_name('type')] n_color, n_name = assign_color(n_type) n_value = node[add_prefix_to_name(n_name)] if n_name else None n_color = 'black' if n_value == root_url else n_color G.add_node(extract_name(node_name), type=n_type, color=n_color, value=n_value) # add all edges edge_types = [item for item in bundle_data if item not in NODE_TYPES] for edge_type in edge_types: print("add all " + edge_type + " edges") edges = bundle_data[edge_type] for edge_name in edges: edge = edges[edge_name] values = list(edge.values()) from_node = extract_name(values[0]) to_node = extract_name(values[1]) G.add_edge(from_node, to_node) # Show with Bokeh plot = Plot(plot_width=1200, plot_height=750, x_range=Range1d(-1.1, 1.1), y_range=Range1d(-1.1, 1.1)) plot.title.text = "Provenance graph" node_hover_tool = HoverTool(tooltips=[("type", "@type"), ("value", "@value")]) plot.add_tools(node_hover_tool, BoxZoomTool(), ResetTool()) graph_renderer = from_networkx(G, nx.spring_layout, scale=1, center=(0, 0)) graph_renderer.node_renderer.glyph = Circle(size=15, fill_color='color') graph_renderer.edge_renderer.glyph = MultiLine(line_alpha=0.8, line_width=1) plot.renderers.append(graph_renderer) output_file("provenance_graphs_" + args.file_name + ".html") show(plot)
def nb_view_patches(Yr, A, C, b, f, d1, d2, YrA = None, image_neurons=None, thr=0.99, denoised_color=None,cmap='jet'): """ Interactive plotting utility for ipython notebook Parameters: ----------- Yr: np.ndarray movie A,C,b,f: np.ndarrays outputs of matrix factorization algorithm d1,d2: floats dimensions of movie (x and y) YrA: np.ndarray ROI filtered residual as it is given from update_temporal_components If not given, then it is computed (K x T) image_neurons: np.ndarray image to be overlaid to neurons (for instance the average) thr: double threshold regulating the extent of the displayed patches denoised_color: string or None color name (e.g. 'red') or hex color code (e.g. '#F0027F') cmap: string name of colormap (e.g. 'viridis') used to plot image_neurons """ colormap = cm.get_cmap(cmap) grayp = [mpl.colors.rgb2hex(m) for m in colormap(np.arange(colormap.N))] nr, T = C.shape nA2 = np.ravel(A.power(2).sum(0)) b = np.squeeze(b) f = np.squeeze(f) if YrA is None: Y_r = np.array(spdiags(old_div(1, nA2), 0, nr, nr) * (A.T * np.matrix(Yr) - (A.T * np.matrix(b[:, np.newaxis])) * np.matrix(f[np.newaxis]) - A.T.dot(A) * np.matrix(C)) + C) else: Y_r = C + YrA x = np.arange(T) z = old_div(np.squeeze(np.array(Y_r[:, :].T)), 100) if image_neurons is None: image_neurons = A.mean(1).reshape((d1, d2), order='F') coors = get_contours(A, (d1, d2), thr) cc1 = [cor['coordinates'][:, 0] for cor in coors] cc2 = [cor['coordinates'][:, 1] for cor in coors] c1 = cc1[0] c2 = cc2[0] # split sources up, such that Bokeh does not warn # "ColumnDataSource's columns must be of the same length" source = ColumnDataSource(data=dict(x=x, y=z[:, 0], y2=C[0] / 100)) source_ = ColumnDataSource(data=dict(z=z.T, z2=C / 100)) source2 = ColumnDataSource(data=dict(c1=c1, c2=c2)) source2_ = ColumnDataSource(data=dict(cc1=cc1, cc2=cc2)) callback = CustomJS(args=dict(source=source, source_=source_, source2=source2, source2_=source2_), code=""" var data = source.get('data') var data_ = source_.get('data') var f = cb_obj.get('value')-1 x = data['x'] y = data['y'] y2 = data['y2'] for (i = 0; i < x.length; i++) { y[i] = data_['z'][i+f*x.length] y2[i] = data_['z2'][i+f*x.length] } var data2_ = source2_.get('data'); var data2 = source2.get('data'); c1 = data2['c1']; c2 = data2['c2']; cc1 = data2_['cc1']; cc2 = data2_['cc2']; for (i = 0; i < c1.length; i++) { c1[i] = cc1[f][i] c2[i] = cc2[f][i] } source2.trigger('change') source.trigger('change') """) plot = bpl.figure(plot_width=600, plot_height=300) plot.line('x', 'y', source=source, line_width=1, line_alpha=0.6) if denoised_color is not None: plot.line('x', 'y2', source=source, line_width=1, line_alpha=0.6, color=denoised_color) slider = bokeh.models.Slider(start=1, end=Y_r.shape[0], value=1, step=1, title="Neuron Number", callback=callback) xr = Range1d(start=0, end=image_neurons.shape[1]) yr = Range1d(start=image_neurons.shape[0], end=0) plot1 = bpl.figure(x_range=xr, y_range=yr, plot_width=300, plot_height=300) plot1.image(image=[image_neurons[::-1, :]], x=0, y=image_neurons.shape[0], dw=d2, dh=d1, palette=grayp) plot1.patch('c1', 'c2', alpha=0.6, color='purple', line_width=2, source=source2) bpl.show(bokeh.layouts.layout([[slider], [bokeh.layouts.row(plot1, plot)]])) return Y_r
from bokeh.models.widgets import Select, Slider from bokeh.layouts import layout #crate columndatasource source_original = ColumnDataSource( dict(average_grades=[7, 8, 10], exam_grades=[6, 9, 8], student_names=["Stephan", "Helder", "Riazudidn"])) source = ColumnDataSource( dict(average_grades=[7, 8, 10], exam_grades=[6, 9, 8], student_names=["Stephan", "Helder", "Riazudidn"])) #create the figure f = figure(x_range=Range1d(start=0, end=12), y_range=Range1d(start=0, end=12)) #add labels for glyphs labels = LabelSet(x="average_grades", y="exam_grades", text="student_names", x_offset=5, y_offset=5, source=source) f.add_layout(labels) #create glyphs f.circle(x="average_grades", y="exam_grades", source=source, size=8) #create filtering function
def nb_view_patches3d(Y_r, A, C, dims, image_type='mean', Yr=None, max_projection=False, axis=0, thr=0.9, denoised_color=None,cmap='jet'): """ Interactive plotting utility for ipython notbook Parameters: ----------- Y_r: np.ndarray residual of each trace A,C,b,f: np.ndarrays outputs of matrix factorization algorithm dims: tuple of ints dimensions of movie (x, y and z) image_type: 'mean', 'max' or 'corr' image to be overlaid to neurons (average of shapes, maximum of shapes or nearest neigbor correlation of raw data) Yr: np.ndarray movie, only required if image_type=='corr' to calculate correlation image max_projection: boolean plot max projection along specified axis if True, plot layers if False axis: int (0, 1 or 2) axis along which max projection is performed or layers are shown thr: scalar between 0 and 1 Energy threshold for computing contours denoised_color: string or None color name (e.g. 'red') or hex color code (e.g. '#F0027F') cmap: string name of colormap (e.g. 'viridis') used to plot image_neurons Raise: ------ ValueError("image_type must be 'mean', 'max' or 'corr'") """ bokeh.io.curdoc().clear() # prune old orphaned models, otherwise filesize blows up d = A.shape[0] order = list(range(4)) order.insert(0, order.pop(axis)) Y_r = Y_r + C index_permut = np.reshape(np.arange(d), dims, order='F').transpose( order[:-1]).reshape(d, order='F') A = csc_matrix(A)[index_permut, :] dims = tuple(np.array(dims)[order[:3]]) d1, d2, d3 = dims colormap = cm.get_cmap(cmap) grayp = [mpl.colors.rgb2hex(m) for m in colormap(np.arange(colormap.N))] nr, T = C.shape x = np.arange(T) source = ColumnDataSource(data=dict(x=x, y=Y_r[0] / 100, y2=C[0] / 100)) source_ = ColumnDataSource(data=dict(z=Y_r / 100, z2=C / 100)) sourceN = ColumnDataSource(data=dict(N=[nr], nan=np.array([np.nan]))) if max_projection: if image_type == 'corr': tmp = [(local_correlations( Yr.reshape(dims + (-1,), order='F'))[:, ::-1]).max(i) for i in range(3)] elif image_type == 'mean': tmp = [(np.array(A.mean(axis=1)).reshape(dims, order='F')[:, ::-1]).max(i) for i in range(3)] elif image_type == 'max': tmp = [(A.max(axis=1).toarray().reshape(dims, order='F')[:, ::-1]).max(i) for i in range(3)] else: raise ValueError("image_type must be 'mean', 'max' or 'corr'") image_neurons = np.nan * np.ones((int(1.05 * (d1 + d2)), int(1.05 * (d1 + d3)))) image_neurons[:d2, -d3:] = tmp[0][::-1] image_neurons[:d2, :d1] = tmp[2].T[::-1] image_neurons[-d1:, -d3:] = tmp[1] offset1 = image_neurons.shape[1] - d3 offset2 = image_neurons.shape[0] - d1 proj_ = [coo_matrix([A[:, nnrr].toarray().reshape(dims, order='F').max( i).reshape(-1, order='F') for nnrr in range(A.shape[1])]) for i in range(3)] proj_ = [pproj_.T for pproj_ in proj_] coors = [get_contours(proj_[i], tmp[i].shape, thr=thr) for i in range(3)] pl.close() K = np.max([[len(cor['coordinates']) for cor in cc] for cc in coors]) cc1 = np.nan * np.zeros(np.shape(coors) + (K,)) cc2 = np.nan * np.zeros(np.shape(coors) + (K,)) for i, cor in enumerate(coors[0]): cc1[0, i, :len(cor['coordinates'])] = cor['coordinates'][:, 0] + offset1 cc2[0, i, :len(cor['coordinates'])] = cor['coordinates'][:, 1] for i, cor in enumerate(coors[2]): cc1[1, i, :len(cor['coordinates'])] = cor['coordinates'][:, 1] cc2[1, i, :len(cor['coordinates'])] = cor['coordinates'][:, 0] for i, cor in enumerate(coors[1]): cc1[2, i, :len(cor['coordinates'])] = cor['coordinates'][:, 0] + offset1 cc2[2, i, :len(cor['coordinates'])] = cor['coordinates'][:, 1] + offset2 c1x = cc1[0][0] c2x = cc2[0][0] c1y = cc1[1][0] c2y = cc2[1][0] c1z = cc1[2][0] c2z = cc2[2][0] source2_ = ColumnDataSource(data=dict(cc1=cc1, cc2=cc2)) source2 = ColumnDataSource(data=dict(c1x=c1x, c1y=c1y, c1z=c1z, c2x=c2x, c2y=c2y, c2z=c2z)) callback = CustomJS(args=dict(source=source, source_=source_, sourceN=sourceN, source2=source2, source2_=source2_), code=""" var data = source.get('data'); var data_ = source_.get('data'); var f = cb_obj.get('value')-1 x = data['x'] y = data['y'] y2 = data['y2'] for (i = 0; i < x.length; i++) { y[i] = data_['z'][i+f*x.length] y2[i] = data_['z2'][i+f*x.length] } var data2_ = source2_.get('data'); var data2 = source2.get('data'); c1x = data2['c1x']; c2x = data2['c2x']; c1y = data2['c1y']; c2y = data2['c2y']; c1z = data2['c1z']; c2z = data2['c2z']; cc1 = data2_['cc1']; cc2 = data2_['cc2']; var N = sourceN.get('data')['N'][0]; for (i = 0; i < c1x.length; i++) { c1x[i] = cc1[f*c1x.length + i] c2x[i] = cc2[f*c1x.length + i] } for (i = 0; i < c1x.length; i++) { c1y[i] = cc1[N*c1y.length + f*c1y.length + i] c2y[i] = cc2[N*c1y.length + f*c1y.length + i] } for (i = 0; i < c1x.length; i++) { c1z[i] = cc1[2*N*c1z.length + f*c1z.length + i] c2z[i] = cc2[2*N*c1z.length + f*c1z.length + i] } source2.trigger('change'); source.trigger('change'); """) else: if image_type == 'corr': image_neurons = local_correlations(Yr.reshape(dims + (-1,), order='F'))[:-1, ::-1] elif image_type == 'mean': image_neurons = np.array(A.mean(axis=1)).reshape(dims, order='F')[:, ::-1] elif image_type == 'max': image_neurons = A.max(axis=1).toarray().reshape(dims, order='F')[:, ::-1] else: raise ValueError('image_type must be mean, max or corr') cmap = bokeh.models.mappers.LinearColorMapper([mpl.colors.rgb2hex(m) for m in colormap(np.arange(colormap.N))]) cmap.high = image_neurons.max() coors = get_contours(A, dims, thr=thr) pl.close() cc1 = [[(l[:, 0]) for l in n['coordinates']] for n in coors] cc2 = [[(l[:, 1]) for l in n['coordinates']] for n in coors] length = np.ravel([list(map(len, cc)) for cc in cc1]) idx = np.cumsum(np.concatenate([[0], length[:-1]])) cc1 = np.concatenate(list(map(np.concatenate, cc1))) cc2 = np.concatenate(list(map(np.concatenate, cc2))) linit = int(round(coors[0]['CoM'][0])) # pick initial layer in which first neuron lies K = length.max() c1 = np.nan * np.zeros(K) c2 = np.nan * np.zeros(K) c1[:length[linit]] = cc1[idx[linit]:idx[linit] + length[linit]] c2[:length[linit]] = cc2[idx[linit]:idx[linit] + length[linit]] source2 = ColumnDataSource(data=dict(c1=c1, c2=c2)) source2_ = ColumnDataSource(data=dict(cc1=cc1, cc2=cc2)) source2_idx = ColumnDataSource(data=dict(idx=idx, length=length)) source3 = ColumnDataSource( data=dict(image=[image_neurons[linit]], im=[image_neurons], x=[0], y=[d2], dw=[d3], dh=[d2])) callback = CustomJS(args=dict(source=source, source_=source_, sourceN=sourceN, source2=source2, source2_=source2_, source2_idx=source2_idx), code=""" var data = source.data; var data_ = source_.data; var f = slider_neuron.value-1; var l = slider_layer.value-1; x = data['x'] y = data['y'] y2 = data['y2'] for (i = 0; i < x.length; i++) { y[i] = data_['z'][i+f*x.length] y2[i] = data_['z2'][i+f*x.length] } var data2 = source2.data; var data2_ = source2_.data; var data2_idx = source2_idx.data; var idx = data2_idx['idx']; c1 = data2['c1']; c2 = data2['c2']; var nz = idx.length / sourceN.data['N'][0]; var nan = sourceN.data['nan'][0]; for (i = 0; i < c1.length; i++) { c1[i] = nan; c2[i] = nan; } for (i = 0; i < data2_idx['length'][l+f*nz]; i++) { c1[i] = data2_['cc1'][idx[l+f*nz] + i]; c2[i] = data2_['cc2'][idx[l+f*nz] + i]; } source2.trigger('change'); source.trigger('change'); """) callback_layer = CustomJS(args=dict(source=source3, sourceN=sourceN, source2=source2, source2_=source2_, source2_idx=source2_idx), code=""" var f = slider_neuron.value-1; var l = slider_layer.value-1; var dh = source.data['dh'][0]; var dw = source.data['dw'][0]; var image = source.data['image'][0]; var images = source.data['im'][0]; for (var i = 0; i < x.length; i++) { for (var j = 0; j < dw; j++){ image[i*dh+j] = images[l*dh*dw + i*dh + j]; } } var data2 = source2.data; var data2_ = source2_.data; var data2_idx = source2_idx.data; var idx = data2_idx['idx'] c1 = data2['c1']; c2 = data2['c2']; var nz = idx.length / sourceN.data['N'][0]; var nan = sourceN.data['nan'][0]; for (i = 0; i < c1.length; i++) { c1[i] = nan; c2[i] = nan; } for (i = 0; i < data2_idx['length'][l+f*nz]; i++) { c1[i] = data2_['cc1'][idx[l+f*nz] + i]; c2[i] = data2_['cc2'][idx[l+f*nz] + i]; } source.trigger('change'); source2.trigger('change'); """) plot = bpl.figure(plot_width=600, plot_height=300) plot.line('x', 'y', source=source, line_width=1, line_alpha=0.6) if denoised_color is not None: plot.line('x', 'y2', source=source, line_width=1, line_alpha=0.6, color=denoised_color) slider = bokeh.models.Slider(start=1, end=Y_r.shape[0], value=1, step=1, title="Neuron Number", callback=callback) xr = Range1d(start=0, end=image_neurons.shape[1] if max_projection else d3) yr = Range1d(start=image_neurons.shape[0] if max_projection else d2, end=0) plot1 = bpl.figure(x_range=xr, y_range=yr, plot_width=300, plot_height=300) if max_projection: plot1.image(image=[image_neurons[::-1, :]], x=0, y=image_neurons.shape[0], dw=image_neurons.shape[1], dh=image_neurons.shape[0], palette=grayp) plot1.patch('c1x', 'c2x', alpha=0.6, color='purple', line_width=2, source=source2) plot1.patch('c1y', 'c2y', alpha=0.6, color='purple', line_width=2, source=source2) plot1.patch('c1z', 'c2z', alpha=0.6, color='purple', line_width=2, source=source2) layout = bokeh.layouts.layout([[slider], [bokeh.layouts.row(plot1, plot)]], sizing_mode="scale_width") else: slider_layer = bokeh.models.Slider(start=1, end=d1, value=linit + 1, step=1, title="Layer", callback=callback_layer) callback.args['slider_neuron'] = slider callback.args['slider_layer'] = slider_layer callback_layer.args['slider_neuron'] = slider callback_layer.args['slider_layer'] = slider_layer plot1.image(image='image', x='x', y='y', dw='dw', dh='dh', color_mapper=cmap, source=source3) plot1.patch('c1', 'c2', alpha=0.6, color='purple', line_width=2, source=source2) layout = bokeh.layouts.layout([[slider], [slider_layer], [bokeh.layouts.row(plot1, plot)]], sizing_mode="scale_width") bpl.show(layout) return Y_r
return x, fy, ty def update_data(): x, fy, ty = taylor(expr, xs, order, (-2 * sy.pi, 2 * sy.pi), 200) plot.title.text = "%s vs. taylor(%s, n=%d)" % (expr, expr, order) legend.items[0].label = value("%s" % expr) legend.items[1].label = value("taylor(%s)" % expr) source.data = dict(x=x, fy=fy, ty=ty) slider.value = order source = ColumnDataSource(data=dict(x=[], fy=[], ty=[])) xdr = Range1d(-7, 7) ydr = Range1d(-20, 200) plot = Plot(x_range=xdr, y_range=ydr, plot_width=800, plot_height=400) line_f = Line(x="x", y="fy", line_color="blue", line_width=2) line_f_glyph = plot.add_glyph(source, line_f) plot.add_layout(line_f_glyph) line_t = Line(x="x", y="ty", line_color="red", line_width=2) line_t_glyph = plot.add_glyph(source, line_t) plot.add_layout(line_t_glyph) xaxis = LinearAxis() plot.add_layout(xaxis, 'below')
from bokeh.util.dependencies import import_required pandas = import_required('pandas', 'get it suckah') colormap = { 'chemical explosion': 'orange', 'quarry blast': 'green', 'earthquake': 'violet', 'sonic boom': 'magenta', 'explosion': 'blue', 'landslide': 'brown', 'mining explosion': 'black' } title = "QUAKIN' IN MAH BOOTS" graph = figure(title=title) graph.xaxis.axis_label = 'Magnitude' graph.yaxis.axis_label = 'Depth' quakes = pandas.read_csv(join(dirname(__file__), 'resource/earthquakes.csv')) graph.circle(quakes['mag'], quakes['depth'], color=[colormap[x] for x in quakes['type']], fill_alpha=0.2, size=10) graph.y_range = Range1d(700, -10) output_file("Earthquake Visualization.html", title=title) show(graph)
def test_with_max_bound_smaller_than_min_bounded_raises_valueerror(self): with pytest.raises(ValueError): Range1d(1, 2, bounds=(1, 0)) with pytest.raises(ValueError): Range1d(1, 2, bounds=[1, 0])
def calculate_assoc_svg(file, region, pop, request, genome_build, myargs, myargsName, myargsOrigin): # Set data directories using config.yml with open('config.yml', 'r') as yml_file: config = yaml.load(yml_file) env = config['env'] connect_external = config['database']['connect_external'] api_mongo_addr = config['database']['api_mongo_addr'] data_dir = config['data']['data_dir'] tmp_dir = config['data']['tmp_dir'] genotypes_dir = config['data']['genotypes_dir'] aws_info = config['aws'] mongo_username = config['database']['mongo_user_readonly'] mongo_password = config['database']['mongo_password'] mongo_port = config['database']['mongo_port'] num_subprocesses = config['performance']['num_subprocesses'] export_s3_keys = retrieveAWSCredentials() # Ensure tmp directory exists if not os.path.exists(tmp_dir): os.makedirs(tmp_dir) chrs = [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "X", "Y" ] # Define parameters for --variant option if region == "variant": if myargsOrigin == "None": return None if myargsOrigin != "None": # Find coordinates (GRCh37/hg19) or (GRCh38/hg38) for SNP RS number if myargsOrigin[0:2] == "rs": snp = myargsOrigin # Connect to Mongo snp database if env == 'local' or connect_external: mongo_host = api_mongo_addr else: mongo_host = 'localhost' client = MongoClient( 'mongodb://' + mongo_username + ':' + mongo_password + '@' + mongo_host + '/admin', mongo_port) db = client["LDLink"] def get_coords_var(db, rsid): rsid = rsid.strip("rs") query_results = db.dbsnp.find_one({"id": rsid}) query_results_sanitized = json.loads( json_util.dumps(query_results)) return query_results_sanitized # Find RS number in snp database var_coord = get_coords_var(db, snp) if var_coord == None: return None elif myargsOrigin.split(":")[0].strip("chr") in chrs and len( myargsOrigin.split(":")) == 2: snp = myargsOrigin #var_coord=[None,myargsOrigin.split(":")[0].strip("chr"),myargsOrigin.split(":")[1]] var_coord = { 'chromosome': myargsOrigin.split(":")[0].strip("chr"), 'position': myargsOrigin.split(":")[1] } else: return None chromosome = var_coord['chromosome'] org_coord = var_coord[genome_build_vars[genome_build]['position']] # Open Association Data header_list = [] header_list.append(myargs['chr']) header_list.append(myargs['bp']) header_list.append(myargs['pval']) # Load input file with open(file) as fp: header = fp.readline().strip().split() first = fp.readline().strip().split() if len(header) != len(first): return None # Check header for item in header_list: if item not in header: return None len_head = len(header) chr_index = header.index(myargs['chr']) pos_index = header.index(myargs['bp']) p_index = header.index(myargs['pval']) # Define window of interest around query SNP if myargs['window'] == None: if region == "variant": window = 500000 elif region == "gene": window = 100000 else: window = 0 else: window = myargs['window'] if region == "variant": coord1 = int(org_coord) - window if coord1 < 0: coord1 = 0 coord2 = int(org_coord) + window elif region == "gene": if myargsName == "None": return None def get_coords_gene(gene_raw, db): gene = gene_raw.upper() mongoResult = db.genes_name_coords.find_one({"name": gene}) #format mongo output if mongoResult != None: geneResult = [ mongoResult["name"], mongoResult[genome_build_vars[genome_build]['chromosome']], mongoResult[genome_build_vars[genome_build]['gene_begin']], mongoResult[genome_build_vars[genome_build]['gene_end']] ] return geneResult else: return None # Find RS number in snp database gene_coord = get_coords_gene(myargsName, db) if gene_coord == None or gene_coord[2] == 'NA' or gene_coord == 'NA': return None # Define search coordinates coord1 = int(gene_coord[2]) - window if coord1 < 0: coord1 = 0 coord2 = int(gene_coord[3]) + window # Run with --origin option if myargsOrigin != "None": if gene_coord[1] != chromosome: return None if coord1 > int(org_coord) or int(org_coord) > coord2: return None else: chromosome = gene_coord[1] elif region == "region": if myargs['start'] == None: return None if myargs['end'] == None: return None # Parse out chr and positions for --region option if len(myargs['start'].split(":")) != 2: return None if len(myargs['end'].split(":")) != 2: return None chr_s = myargs['start'].strip("chr").split(":")[0] coord_s = myargs['start'].split(":")[1] chr_e = myargs['end'].strip("chr").split(":")[0] coord_e = myargs['end'].split(":")[1] if chr_s not in chrs: return None if chr_e not in chrs: return None if chr_s != chr_e: return None if coord_s >= coord_e: return None coord1 = int(coord_s) - window if coord1 < 0: coord1 = 0 coord2 = int(coord_e) + window # Run with --origin option if myargsOrigin != "None": if chr_s != chromosome: return None if coord1 > int(org_coord) or int(org_coord) > coord2: return None else: chromosome = chr_s # Generate coordinate list and P-value dictionary max_window = 3000000 if coord2 - coord1 > max_window: return None assoc_coords = [] a_pos = [] assoc_dict = {} assoc_list = [] with open(file) as fp: for line in fp: col = line.strip().split() if len(col) == len_head: if col[chr_index].strip("chr") == chromosome: try: int(col[pos_index]) except ValueError: continue else: if coord1 <= int(col[pos_index]) <= coord2: try: float(col[p_index]) except ValueError: continue else: coord_i = genome_build_vars[genome_build][ '1000G_chr_prefix'] + col[chr_index].strip( "chr") + ":" + col[ pos_index] + "-" + col[pos_index] assoc_coords.append(coord_i) a_pos.append(col[pos_index]) assoc_dict[coord_i] = [col[p_index]] assoc_list.append( [coord_i, float(col[p_index])]) # Coordinate list checks if len(assoc_coords) == 0: return None # Get population ids from population output file from LDassoc.py pop_list = open(tmp_dir + "pops_" + request + ".txt").readlines() ids = [] for i in range(len(pop_list)): ids.append(pop_list[i].strip()) pop_ids = list(set(ids)) # Define LD origin coordinate try: org_coord except NameError: for var_p in sorted(assoc_list, key=operator.itemgetter(1)): snp = "chr" + var_p[0].split("-")[0] # Extract lowest P SNP phased genotypes vcf_filePath = "%s/%s%s/%s" % ( config['aws']['data_subfolder'], genotypes_dir, genome_build_vars[genome_build]["1000G_dir"], genome_build_vars[genome_build]["1000G_file"] % (chromosome)) vcf_file = "s3://%s/%s" % (config['aws']['bucket'], vcf_filePath) checkS3File(aws_info, config['aws']['bucket'], vcf_filePath) tabix_snp_h = export_s3_keys + " cd {1}; tabix -HD {0} | grep CHROM".format( vcf_file, data_dir + genotypes_dir + genome_build_vars[genome_build]['1000G_dir']) head = [ x.decode('utf-8') for x in subprocess.Popen( tabix_snp_h, shell=True, stdout=subprocess.PIPE).stdout.readlines() ][0].strip().split() # Check lowest P SNP is in the 1000G population and not monoallelic from LDassoc.py output file vcf = open(tmp_dir + "snp_no_dups_" + request + ".vcf").readlines() if len(vcf) == 0: continue elif len(vcf) > 1: geno = vcf[0].strip().split() geno[0] = geno[0].lstrip('chr') else: geno = vcf[0].strip().split() geno[0] = geno[0].lstrip('chr') if "," in geno[3] or "," in geno[4]: continue index = [] for i in range(9, len(head)): if head[i] in pop_ids: index.append(i) genotypes = {"0": 0, "1": 0} for i in index: sub_geno = geno[i].split("|") for j in sub_geno: if j in genotypes: genotypes[j] += 1 else: genotypes[j] = 1 if genotypes["0"] == 0 or genotypes["1"] == 0: continue org_coord = var_p[0].split("-")[1] break else: if genome_build_vars[genome_build][ '1000G_chr_prefix'] + chromosome + ":" + org_coord + "-" + org_coord not in assoc_coords: return None # Extract query SNP phased genotypes vcf_filePath = "%s/%s%s/%s" % ( config['aws']['data_subfolder'], genotypes_dir, genome_build_vars[genome_build]["1000G_dir"], genome_build_vars[genome_build]["1000G_file"] % (chromosome)) vcf_file = "s3://%s/%s" % (config['aws']['bucket'], vcf_filePath) checkS3File(aws_info, config['aws']['bucket'], vcf_filePath) tabix_snp_h = export_s3_keys + " cd {1}; tabix -HD {0} | grep CHROM".format( vcf_file, data_dir + genotypes_dir + genome_build_vars[genome_build]['1000G_dir']) head = [ x.decode('utf-8') for x in subprocess.Popen(tabix_snp_h, shell=True, stdout=subprocess.PIPE).stdout.readlines() ][0].strip().split() # Check query SNP is in the 1000G population, has the correct RS number, and not monoallelic vcf = open(tmp_dir + "snp_no_dups_" + request + ".vcf").readlines() if len(vcf) == 0: subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt", shell=True) subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True) return None elif len(vcf) > 1: geno = [] for i in range(len(vcf)): # if vcf[i].strip().split()[2] == snp: geno = vcf[i].strip().split() geno[0] = geno[0].lstrip('chr') if geno == []: subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt", shell=True) subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True) return None else: geno = vcf[0].strip().split() geno[0] = geno[0].lstrip('chr') if geno[2] != snp and snp[0:2] == "rs" and "rs" in geno[2]: snp = geno[2] if "," in geno[3] or "," in geno[4]: subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt", shell=True) subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True) return None index = [] for i in range(9, len(head)): if head[i] in pop_ids: index.append(i) genotypes = {"0": 0, "1": 0} for i in index: sub_geno = geno[i].split("|") for j in sub_geno: if j in genotypes: genotypes[j] += 1 else: genotypes[j] = 1 if genotypes["0"] == 0 or genotypes["1"] == 0: subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt", shell=True) subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True) return None # Calculate proxy LD statistics in parallel if len(assoc_coords) < 60: num_subprocesses = 1 # else: # threads=4 assoc_coords_subset_chunks = np.array_split(assoc_coords, num_subprocesses) # block=len(assoc_coords) // num_subprocesses commands = [] # for i in range(num_subprocesses): # if i==min(range(num_subprocesses)) and i==max(range(num_subprocesses)): # command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords)+" "+request+" "+str(i) # elif i==min(range(num_subprocesses)): # command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[:block])+" "+request+" "+str(i) # elif i==max(range(num_subprocesses)): # command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:])+" "+request+" "+str(i) # else: # command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:block*(i+1)])+" "+request+" "+str(i) # commands.append(command) for subprocess_id in range(num_subprocesses): subprocessArgs = " ".join([ str(snp), str(chromosome), str("_".join(assoc_coords_subset_chunks[subprocess_id])), str(request), str(genome_build), str(subprocess_id) ]) commands.append("python3 LDassoc_sub.py " + subprocessArgs) processes = [ subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) for command in commands ] # collect output in parallel def get_output(process): return process.communicate()[0].splitlines() pool = Pool(len(processes)) out_raw = pool.map(get_output, processes) pool.close() pool.join() # Aggregate output out_prox = [] for i in range(len(out_raw)): for j in range(len(out_raw[i])): col = out_raw[i][j].decode('utf-8').strip().split("\t") col[6] = int(col[6]) col[7] = float(col[7]) col[8] = float(col[8]) col.append(abs(int(col[6]))) pos_i_j = col[5].split(":")[1] coord_i_j = genome_build_vars[genome_build][ '1000G_chr_prefix'] + chromosome + ":" + pos_i_j + "-" + pos_i_j if coord_i_j in assoc_dict: col.append(float(assoc_dict[coord_i_j][0])) out_prox.append(col) out_dist_sort = sorted(out_prox, key=operator.itemgetter(14)) out_p_sort = sorted(out_dist_sort, key=operator.itemgetter(15), reverse=False) # Organize scatter plot data q_rs = [] q_allele = [] q_coord = [] q_maf = [] p_rs = [] p_allele = [] p_coord = [] p_pos = [] p_maf = [] dist = [] d_prime = [] d_prime_round = [] r2 = [] r2_round = [] corr_alleles = [] regdb = [] funct = [] color = [] alpha = [] size = [] p_val = [] neg_log_p = [] for i in range(len(out_p_sort)): q_rs_i, q_allele_i, q_coord_i, p_rs_i, p_allele_i, p_coord_i, dist_i, d_prime_i, r2_i, corr_alleles_i, regdb_i, q_maf_i, p_maf_i, funct_i, dist_abs, p_val_i = out_p_sort[ i] q_rs.append(q_rs_i) q_allele.append(q_allele_i) q_coord.append(float(q_coord_i.split(":")[1]) / 1000000) q_maf.append(str(round(float(q_maf_i), 4))) if p_rs_i == ".": p_rs_i = p_coord_i p_rs.append(p_rs_i) p_allele.append(p_allele_i) p_coord.append(float(p_coord_i.split(":")[1]) / 1000000) p_pos.append(p_coord_i.split(":")[1]) p_maf.append(str(round(float(p_maf_i), 4))) dist.append(str(round(dist_i / 1000000.0, 4))) d_prime.append(float(d_prime_i)) d_prime_round.append(str(round(float(d_prime_i), 4))) r2.append(float(r2_i)) r2_round.append(str(round(float(r2_i), 4))) corr_alleles.append(corr_alleles_i) # P-value p_val.append(p_val_i) neg_log_p.append(-log10(p_val_i)) # Correct Missing Annotations if regdb_i == ".": regdb_i = "" regdb.append(regdb_i) if funct_i == ".": funct_i = "" if funct_i == "NA": funct_i = "none" funct.append(funct_i) # Set Color reds = [ "#FFCCCC", "#FFCACA", "#FFC8C8", "#FFC6C6", "#FFC4C4", "#FFC2C2", "#FFC0C0", "#FFBEBE", "#FFBCBC", "#FFBABA", "#FFB8B8", "#FFB6B6", "#FFB4B4", "#FFB1B1", "#FFAFAF", "#FFADAD", "#FFABAB", "#FFA9A9", "#FFA7A7", "#FFA5A5", "#FFA3A3", "#FFA1A1", "#FF9F9F", "#FF9D9D", "#FF9B9B", "#FF9999", "#FF9797", "#FF9595", "#FF9393", "#FF9191", "#FF8F8F", "#FF8D8D", "#FF8B8B", "#FF8989", "#FF8787", "#FF8585", "#FF8383", "#FF8181", "#FF7E7E", "#FF7C7C", "#FF7A7A", "#FF7878", "#FF7676", "#FF7474", "#FF7272", "#FF7070", "#FF6E6E", "#FF6C6C", "#FF6A6A", "#FF6868", "#FF6666", "#FF6464", "#FF6262", "#FF6060", "#FF5E5E", "#FF5C5C", "#FF5A5A", "#FF5858", "#FF5656", "#FF5454", "#FF5252", "#FF5050", "#FF4E4E", "#FF4B4B", "#FF4949", "#FF4747", "#FF4545", "#FF4343", "#FF4141", "#FF3F3F", "#FF3D3D", "#FF3B3B", "#FF3939", "#FF3737", "#FF3535", "#FF3333", "#FF3131", "#FF2F2F", "#FF2D2D", "#FF2B2B", "#FF2929", "#FF2727", "#FF2525", "#FF2323", "#FF2121", "#FF1F1F", "#FF1D1D", "#FF1B1B", "#FF1818", "#FF1616", "#FF1414", "#FF1212", "#FF1010", "#FF0E0E", "#FF0C0C", "#FF0A0A", "#FF0808", "#FF0606", "#FF0404", "#FF0202", "#FF0000" ] if q_coord_i == p_coord_i: color_i = "#0000FF" alpha_i = 0.7 else: if myargs['dprime'] == True: color_i = reds[int(d_prime_i * 100.0)] alpha_i = 0.7 elif myargs['dprime'] == False: color_i = reds[int(r2_i * 100.0)] alpha_i = 0.7 color.append(color_i) alpha.append(alpha_i) # Set Size size_i = 9 + float(p_maf_i) * 14.0 size.append(size_i) # Pull out SNPs from association file not found in 1000G p_plot_pos = [] p_plot_pval = [] p_plot_pos2 = [] p_plot_pval2 = [] p_plot_dist = [] index_var_pos = float(q_coord_i.split(":")[1]) / 1000000 for input_pos in a_pos: if input_pos not in p_pos: p_plot_pos.append(float(input_pos) / 1000000) p_plot_pval.append(-log10( float(assoc_dict[chromosome + ":" + input_pos + "-" + input_pos][0]))) p_plot_pos2.append("chr" + chromosome + ":" + input_pos) p_plot_pval2.append( float(assoc_dict[chromosome + ":" + input_pos + "-" + input_pos][0])) p_plot_dist.append( str(round(float(input_pos) / 1000000 - index_var_pos, 4))) # Begin Bokeh Plotting from collections import OrderedDict from bokeh.embed import components, file_html from bokeh.layouts import gridplot from bokeh.models import HoverTool, LinearAxis, Range1d from bokeh.plotting import ColumnDataSource, curdoc, figure, output_file, reset_output, save from bokeh.resources import CDN from bokeh.io import export_svgs import svgutils.compose as sg reset_output() data_p = { 'p_plot_posX': p_plot_pos, 'p_plot_pvalY': p_plot_pval, 'p_plot_pos2': p_plot_pos2, 'p_plot_pval2': p_plot_pval2, 'p_plot_dist': p_plot_dist } source_p = ColumnDataSource(data_p) # Assoc Plot x = p_coord y = neg_log_p data = { 'x': x, 'y': y, 'qrs': q_rs, 'q_alle': q_allele, 'q_maf': q_maf, 'prs': p_rs, 'p_alle': p_allele, 'p_maf': p_maf, 'dist': dist, 'r': r2_round, 'd': d_prime_round, 'alleles': corr_alleles, 'regdb': regdb, 'funct': funct, 'p_val': p_val, 'size': size, 'color': color, 'alpha': alpha } source = ColumnDataSource(data) whitespace = 0.01 xr = Range1d(start=coord1 / 1000000.0 - whitespace, end=coord2 / 1000000.0 + whitespace) yr = Range1d(start=-0.03, end=max(y) * 1.03) sup_2 = "\u00B2" assoc_plot = figure( title="P-values and Regional LD for " + snp + " in " + pop, min_border_top=2, min_border_bottom=2, min_border_left=60, min_border_right=60, h_symmetry=False, v_symmetry=False, plot_width=900, plot_height=600, x_range=xr, y_range=yr, tools= "tap,pan,box_zoom,wheel_zoom,box_select,undo,redo,reset,previewsave", logo=None, toolbar_location="above") assoc_plot.title.align = "center" # Add recombination rate from LDassoc.py output file recomb_file = tmp_dir + "recomb_" + request + ".json" recomb_raw = open(recomb_file).readlines() recomb_x = [] recomb_y = [] for recomb_raw_obj in recomb_raw: recomb_obj = json.loads(recomb_raw_obj) recomb_x.append( int(recomb_obj[genome_build_vars[genome_build]['position']]) / 1000000.0) recomb_y.append(float(recomb_obj['rate']) / 100 * max(y)) assoc_plot.line(recomb_x, recomb_y, line_width=1, color="black", alpha=0.5) # Add genome-wide significance a = [coord1 / 1000000.0 - whitespace, coord2 / 1000000.0 + whitespace] b = [-log10(0.00000005), -log10(0.00000005)] assoc_plot.line(a, b, color="blue", alpha=0.5) assoc_points_not1000G = assoc_plot.circle(x='p_plot_posX', y='p_plot_pvalY', size=9 + float("0.25") * 14.0, source=source_p, line_color="gray", fill_color="white") assoc_points = assoc_plot.circle(x='x', y='y', size='size', color='color', alpha='alpha', source=source) assoc_plot.add_tools( HoverTool(renderers=[assoc_points_not1000G], tooltips=OrderedDict([("Variant", "@p_plot_pos2"), ("P-value", "@p_plot_pval2"), ("Distance (Mb)", "@p_plot_dist")]))) hover = HoverTool(renderers=[assoc_points]) hover.tooltips = OrderedDict([ ("Variant", "@prs @p_alle"), ("P-value", "@p_val"), ("Distance (Mb)", "@dist"), ("MAF", "@p_maf"), ("R" + sup_2 + " (" + q_rs[0] + ")", "@r"), ("D\' (" + q_rs[0] + ")", "@d"), ("Correlated Alleles", "@alleles"), ("RegulomeDB", "@regdb"), ("Functional Class", "@funct"), ]) assoc_plot.add_tools(hover) # Annotate RebulomeDB scores if myargs['annotate'] == True: assoc_plot.text(x, y, text=regdb, alpha=1, text_font_size="7pt", text_baseline="middle", text_align="center", angle=0) assoc_plot.yaxis.axis_label = "-log10 P-value" assoc_plot.extra_y_ranges = {"y2_axis": Range1d(start=-3, end=103)} assoc_plot.add_layout( LinearAxis(y_range_name="y2_axis", axis_label="Combined Recombination Rate (cM/Mb)"), "right") ## Need to confirm units # Rug Plot y2_ll = [-0.03] * len(x) y2_ul = [1.03] * len(x) yr_rug = Range1d(start=-0.03, end=1.03) data_rug = { 'x': x, 'y': y, 'y2_ll': y2_ll, 'y2_ul': y2_ul, 'qrs': q_rs, 'q_alle': q_allele, 'q_maf': q_maf, 'prs': p_rs, 'p_alle': p_allele, 'p_maf': p_maf, 'dist': dist, 'r': r2_round, 'd': d_prime_round, 'alleles': corr_alleles, 'regdb': regdb, 'funct': funct, 'p_val': p_val, 'size': size, 'color': color, 'alpha': alpha } source_rug = ColumnDataSource(data_rug) rug = figure(x_range=xr, y_range=yr_rug, border_fill_color='white', y_axis_type=None, title="", min_border_top=2, min_border_bottom=2, min_border_left=60, min_border_right=60, h_symmetry=False, v_symmetry=False, plot_width=900, plot_height=50, tools="xpan,tap,wheel_zoom", logo=None) rug.segment(x0='x', y0='y2_ll', x1='x', y1='y2_ul', source=source_rug, color='color', alpha='alpha', line_width=1) rug.toolbar_location = None # Gene Plot (All Transcripts) if myargs['transcript'] == True: # Get genes from LDassoc.py output file genes_file = tmp_dir + "genes_" + request + ".json" genes_raw = open(genes_file).readlines() genes_plot_start = [] genes_plot_end = [] genes_plot_y = [] genes_plot_name = [] exons_plot_x = [] exons_plot_y = [] exons_plot_w = [] exons_plot_h = [] exons_plot_name = [] exons_plot_id = [] exons_plot_exon = [] message = ["Too many genes to plot."] lines = [0] gap = 80000 tall = 0.75 if genes_raw != None and len(genes_raw) > 0: for gene_raw_obj in genes_raw: gene_obj = json.loads(gene_raw_obj) bin = gene_obj["bin"] name_id = gene_obj["name"] chrom = gene_obj["chrom"] strand = gene_obj["strand"] txStart = gene_obj["txStart"] txEnd = gene_obj["txEnd"] cdsStart = gene_obj["cdsStart"] cdsEnd = gene_obj["cdsEnd"] exonCount = gene_obj["exonCount"] exonStarts = gene_obj["exonStarts"] exonEnds = gene_obj["exonEnds"] score = gene_obj["score"] name2 = gene_obj["name2"] cdsStartStat = gene_obj["cdsStartStat"] cdsEndStat = gene_obj["cdsEndStat"] exonFrames = gene_obj["exonFrames"] name = name2 id = name_id e_start = exonStarts.split(",") e_end = exonEnds.split(",") # Determine Y Coordinate i = 0 y_coord = None while y_coord == None: if i > len(lines) - 1: y_coord = i + 1 lines.append(int(txEnd)) elif int(txStart) > (gap + lines[i]): y_coord = i + 1 lines[i] = int(txEnd) else: i += 1 genes_plot_start.append(int(txStart) / 1000000.0) genes_plot_end.append(int(txEnd) / 1000000.0) genes_plot_y.append(y_coord) genes_plot_name.append(name + " ") for i in range(len(e_start) - 1): if strand == "+": exon = i + 1 else: exon = len(e_start) - 1 - i width = (int(e_end[i]) - int(e_start[i])) / 1000000.0 x_coord = int(e_start[i]) / 1000000.0 + (width / 2) exons_plot_x.append(x_coord) exons_plot_y.append(y_coord) exons_plot_w.append(width) exons_plot_h.append(tall) exons_plot_name.append(name) exons_plot_id.append(id) exons_plot_exon.append(exon) n_rows = len(lines) genes_plot_yn = [n_rows - x + 0.5 for x in genes_plot_y] exons_plot_yn = [n_rows - x + 0.5 for x in exons_plot_y] yr2 = Range1d(start=0, end=n_rows) data_gene_plot = { 'exons_plot_x': exons_plot_x, 'exons_plot_yn': exons_plot_yn, 'exons_plot_w': exons_plot_w, 'exons_plot_h': exons_plot_h, 'exons_plot_name': exons_plot_name, 'exons_plot_id': exons_plot_id, 'exons_plot_exon': exons_plot_exon } source_gene_plot = ColumnDataSource(data_gene_plot) max_genes = 40 # if len(lines) < 3 or len(genes_raw) > max_genes: if len(lines) < 3: plot_h_pix = 250 else: plot_h_pix = 250 + (len(lines) - 2) * 50 gene_plot = figure( min_border_top=2, min_border_bottom=0, min_border_left=100, min_border_right=5, x_range=xr, y_range=yr2, border_fill_color='white', title="", h_symmetry=False, v_symmetry=False, logo=None, plot_width=900, plot_height=plot_h_pix, tools= "hover,xpan,box_zoom,wheel_zoom,tap,undo,redo,reset,previewsave") # if len(genes_raw) <= max_genes: gene_plot.segment(genes_plot_start, genes_plot_yn, genes_plot_end, genes_plot_yn, color="black", alpha=1, line_width=2) gene_plot.rect(x='exons_plot_x', y='exons_plot_yn', width='exons_plot_w', height='exons_plot_h', source=source_gene_plot, fill_color="grey", line_color="grey") gene_plot.text(genes_plot_start, genes_plot_yn, text=genes_plot_name, alpha=1, text_font_size="7pt", text_font_style="bold", text_baseline="middle", text_align="right", angle=0) hover = gene_plot.select(dict(type=HoverTool)) hover.tooltips = OrderedDict([ ("Gene", "@exons_plot_name"), ("Transcript ID", "@exons_plot_id"), ("Exon", "@exons_plot_exon"), ]) # else: # x_coord_text = coord1/1000000.0 + (coord2/1000000.0 - coord1/1000000.0) / 2.0 # gene_plot.text(x_coord_text, n_rows / 2.0, text=message, alpha=1, # text_font_size="12pt", text_font_style="bold", text_baseline="middle", text_align="center", angle=0) gene_plot.xaxis.axis_label = "Chromosome " + chromosome + " Coordinate (Mb)(" + genome_build_vars[ genome_build]['title'] + ")" gene_plot.yaxis.axis_label = "Genes (All Transcripts)" gene_plot.ygrid.grid_line_color = None gene_plot.yaxis.axis_line_color = None gene_plot.yaxis.minor_tick_line_color = None gene_plot.yaxis.major_tick_line_color = None gene_plot.yaxis.major_label_text_color = None gene_plot.toolbar_location = "below" # Change output backend to SVG temporarily for headless export assoc_plot.output_backend = "svg" rug.output_backend = "svg" gene_plot.output_backend = "svg" export_svgs(assoc_plot, filename=tmp_dir + "assoc_plot_1_" + request + ".svg") export_svgs(gene_plot, filename=tmp_dir + "gene_plot_1_" + request + ".svg") # 1 pixel = 0.0264583333 cm svg_height = str(20.00 + (0.0264583333 * plot_h_pix)) + "cm" svg_height_scaled = str(100.00 + (0.1322916665 * plot_h_pix)) + "cm" # Concatenate svgs sg.Figure( "24.59cm", svg_height, sg.SVG(tmp_dir + "assoc_plot_1_" + request + ".svg"), sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").move( -40, 630)).save(tmp_dir + "assoc_plot_" + request + ".svg") sg.Figure( "122.95cm", svg_height_scaled, sg.SVG(tmp_dir + "assoc_plot_1_" + request + ".svg").scale(5), sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").scale(5).move( -200, 3150)).save(tmp_dir + "assoc_plot_scaled_" + request + ".svg") # Export to PDF subprocess.call("phantomjs ./rasterize.js " + tmp_dir + "assoc_plot_" + request + ".svg " + tmp_dir + "assoc_plot_" + request + ".pdf", shell=True) # Export to PNG subprocess.call("phantomjs ./rasterize.js " + tmp_dir + "assoc_plot_scaled_" + request + ".svg " + tmp_dir + "assoc_plot_" + request + ".png", shell=True) # Export to JPEG subprocess.call("phantomjs ./rasterize.js " + tmp_dir + "assoc_plot_scaled_" + request + ".svg " + tmp_dir + "assoc_plot_" + request + ".jpeg", shell=True) # Remove individual SVG files after they are combined subprocess.call("rm " + tmp_dir + "assoc_plot_1_" + request + ".svg", shell=True) subprocess.call("rm " + tmp_dir + "gene_plot_1_" + request + ".svg", shell=True) # Remove scaled SVG file after it is converted to png and jpeg subprocess.call("rm " + tmp_dir + "assoc_plot_scaled_" + request + ".svg", shell=True) # Gene Plot (Collapsed) else: # Get genes from LDassoc.py output file genes_c_file = tmp_dir + "genes_c_" + request + ".json" genes_c_raw = open(genes_c_file).readlines() genes_c_plot_start = [] genes_c_plot_end = [] genes_c_plot_y = [] genes_c_plot_name = [] exons_c_plot_x = [] exons_c_plot_y = [] exons_c_plot_w = [] exons_c_plot_h = [] exons_c_plot_name = [] exons_c_plot_id = [] message_c = ["Too many genes to plot."] lines_c = [0] gap = 80000 tall = 0.75 if genes_c_raw != None and len(genes_c_raw) > 0: for gene_raw_obj in genes_c_raw: gene_c_obj = json.loads(gene_raw_obj) chrom = gene_c_obj["chrom"] txStart = gene_c_obj["txStart"] txEnd = gene_c_obj["txEnd"] exonStarts = gene_c_obj["exonStarts"] exonEnds = gene_c_obj["exonEnds"] name2 = gene_c_obj["name2"] transcripts = gene_c_obj["transcripts"] name = name2 e_start = exonStarts.split(",") e_end = exonEnds.split(",") e_transcripts = transcripts.split(",") # Determine Y Coordinate i = 0 y_coord = None while y_coord == None: if i > len(lines_c) - 1: y_coord = i + 1 lines_c.append(int(txEnd)) elif int(txStart) > (gap + lines_c[i]): y_coord = i + 1 lines_c[i] = int(txEnd) else: i += 1 genes_c_plot_start.append(int(txStart) / 1000000.0) genes_c_plot_end.append(int(txEnd) / 1000000.0) genes_c_plot_y.append(y_coord) genes_c_plot_name.append(name + " ") # for i in range(len(e_start)): for i in range(len(e_start) - 1): width = (int(e_end[i]) - int(e_start[i])) / 1000000.0 x_coord = int(e_start[i]) / 1000000.0 + (width / 2) exons_c_plot_x.append(x_coord) exons_c_plot_y.append(y_coord) exons_c_plot_w.append(width) exons_c_plot_h.append(tall) exons_c_plot_name.append(name) exons_c_plot_id.append(e_transcripts[i].replace("-", ",")) n_rows_c = len(lines_c) genes_c_plot_yn = [n_rows_c - x + 0.5 for x in genes_c_plot_y] exons_c_plot_yn = [n_rows_c - x + 0.5 for x in exons_c_plot_y] yr2_c = Range1d(start=0, end=n_rows_c) data_gene_c_plot = { 'exons_c_plot_x': exons_c_plot_x, 'exons_c_plot_yn': exons_c_plot_yn, 'exons_c_plot_w': exons_c_plot_w, 'exons_c_plot_h': exons_c_plot_h, 'exons_c_plot_name': exons_c_plot_name, 'exons_c_plot_id': exons_c_plot_id } source_gene_c_plot = ColumnDataSource(data_gene_c_plot) max_genes_c = 40 # if len(lines_c) < 3 or len(genes_c_raw) > max_genes_c: if len(lines_c) < 3: plot_c_h_pix = 250 else: plot_c_h_pix = 250 + (len(lines_c) - 2) * 50 gene_c_plot = figure( min_border_top=2, min_border_bottom=0, min_border_left=100, min_border_right=5, x_range=xr, y_range=yr2_c, border_fill_color='white', title="", h_symmetry=False, v_symmetry=False, logo=None, plot_width=900, plot_height=plot_c_h_pix, tools= "hover,xpan,box_zoom,wheel_zoom,tap,undo,redo,reset,previewsave") # if len(genes_c_raw) <= max_genes_c: gene_c_plot.segment(genes_c_plot_start, genes_c_plot_yn, genes_c_plot_end, genes_c_plot_yn, color="black", alpha=1, line_width=2) gene_c_plot.rect(x='exons_c_plot_x', y='exons_c_plot_yn', width='exons_c_plot_w', height='exons_c_plot_h', source=source_gene_c_plot, fill_color="grey", line_color="grey") gene_c_plot.text(genes_c_plot_start, genes_c_plot_yn, text=genes_c_plot_name, alpha=1, text_font_size="7pt", text_font_style="bold", text_baseline="middle", text_align="right", angle=0) hover = gene_c_plot.select(dict(type=HoverTool)) hover.tooltips = OrderedDict([ ("Gene", "@exons_c_plot_name"), ("Transcript IDs", "@exons_c_plot_id"), ]) # else: # x_coord_text = coord1/1000000.0 + (coord2/1000000.0 - coord1/1000000.0) / 2.0 # gene_c_plot.text(x_coord_text, n_rows_c / 2.0, text=message_c, alpha=1, # text_font_size="12pt", text_font_style="bold", text_baseline="middle", text_align="center", angle=0) gene_c_plot.xaxis.axis_label = "Chromosome " + chromosome + " Coordinate (Mb)(" + genome_build_vars[ genome_build]['title'] + ")" gene_c_plot.yaxis.axis_label = "Genes (Transcripts Collapsed)" gene_c_plot.ygrid.grid_line_color = None gene_c_plot.yaxis.axis_line_color = None gene_c_plot.yaxis.minor_tick_line_color = None gene_c_plot.yaxis.major_tick_line_color = None gene_c_plot.yaxis.major_label_text_color = None gene_c_plot.toolbar_location = "below" # Change output backend to SVG temporarily for headless export assoc_plot.output_backend = "svg" rug.output_backend = "svg" gene_c_plot.output_backend = "svg" export_svgs(assoc_plot, filename=tmp_dir + "assoc_plot_1_" + request + ".svg") export_svgs(gene_c_plot, filename=tmp_dir + "gene_plot_1_" + request + ".svg") # 1 pixel = 0.0264583333 cm svg_height = str(20.00 + (0.0264583333 * plot_c_h_pix)) + "cm" svg_height_scaled = str(100.00 + (0.1322916665 * plot_c_h_pix)) + "cm" # Concatenate svgs sg.Figure( "24.59cm", svg_height, sg.SVG(tmp_dir + "assoc_plot_1_" + request + ".svg"), sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").move( -40, 630)).save(tmp_dir + "assoc_plot_" + request + ".svg") sg.Figure( "122.95cm", svg_height_scaled, sg.SVG(tmp_dir + "assoc_plot_1_" + request + ".svg").scale(5), sg.SVG(tmp_dir + "gene_plot_1_" + request + ".svg").scale(5).move( -200, 3150)).save(tmp_dir + "assoc_plot_scaled_" + request + ".svg") # Export to PDF subprocess.call("phantomjs ./rasterize.js " + tmp_dir + "assoc_plot_" + request + ".svg " + tmp_dir + "assoc_plot_" + request + ".pdf", shell=True) # Export to PNG subprocess.call("phantomjs ./rasterize.js " + tmp_dir + "assoc_plot_scaled_" + request + ".svg " + tmp_dir + "assoc_plot_" + request + ".png", shell=True) # Export to JPEG subprocess.call("phantomjs ./rasterize.js " + tmp_dir + "assoc_plot_scaled_" + request + ".svg " + tmp_dir + "assoc_plot_" + request + ".jpeg", shell=True) # Remove individual SVG files after they are combined subprocess.call("rm " + tmp_dir + "assoc_plot_1_" + request + ".svg", shell=True) subprocess.call("rm " + tmp_dir + "gene_plot_1_" + request + ".svg", shell=True) # Remove scaled SVG file after it is converted to png and jpeg subprocess.call("rm " + tmp_dir + "assoc_plot_scaled_" + request + ".svg", shell=True) reset_output() # Remove temporary files subprocess.call("rm " + tmp_dir + "pops_" + request + ".txt", shell=True) subprocess.call("rm " + tmp_dir + "*" + request + "*.vcf", shell=True) subprocess.call("rm " + tmp_dir + "genes_*" + request + "*.json", shell=True) subprocess.call("rm " + tmp_dir + "recomb_" + request + ".json", shell=True) subprocess.call("rm " + tmp_dir + "assoc_args" + request + ".json", shell=True) print("Bokeh high quality image export complete!") # Return plot output return None
def test_bounds_with_three_item_tuple_raises_valueerror(self): with pytest.raises(ValueError): Range1d(1, 2, bounds=(0, 1, 2))
plot_width=1300, title="Quantized Circles of Confusion") l1 = coc_plot.multi_line(xs='x', ys='coc', source=source, line_width=2, color='color', legend='legend_label') # coc_plot.legend.title = "CoCs" # coc_plot.line('x', 'coc1', source=source, line_width=2, color='blue', legend=legend_label[0]) # coc_plot.line('x', 'coc2', source=source, line_width=2, color='green', legend=legend_label[1]) # coc_plot.line('x', 'coc_diff', source=source, line_width=2, color='black', line_dash=(2,2), legend=legend_label[2]) coc_plot.xaxis.axis_label = "Range (m)" coc_plot.yaxis.axis_label = "Pixel Radius" coc_plot.axis.axis_label_text_font_style = "bold" coc_plot.x_range = Range1d(start=min_x_spin.value, end=max_x_spin.value) coc_plot.y_range = Range1d(start=min_y_spin.value, end=max_y_spin.value) # coc_plot.legend[0].location = (900, 200)) # legend = Legend(items=[(("fp 1", "fp 2", "CoC Difference"), [l1])], location=(0, -60)) # coc_plot.add_layout(legend, 'right') coc_plot.add_tools(ht) b1_data = np.full((1, blur_img_h * blur_img_w), 255, dtype=np.uint8) b1_data[:, 0:(blur_img_w * 99)] = 0 tmp_dict = dict(source=source, b_source=b_source, b1_d=b1_data) tmp_cb = CustomJS(args=tmp_dict, code=""" var data = source.data; //var b1_data = []; //Object.assign(b1_data, b1_d);
def test_init_with_datetime(self): range1d = Range1d(start=dt.datetime(2016, 4, 28, 2, 20, 50), end=dt.datetime(2017, 4, 28, 2, 20, 50)) assert range1d.start == dt.datetime(2016, 4, 28, 2, 20, 50) assert range1d.end == dt.datetime(2017, 4, 28, 2, 20, 50) assert range1d.bounds is None
for year in years: fertility = fertility_df[year] fertility.name = 'fertility' life = life_expectancy_df[year] life.name = 'life' population = population_df_size[year] population.name = 'population' new_df = pd.concat([fertility, life, population, region_name], axis=1) sources['_' + str(year)] = ColumnDataSource(new_df) dictionary_of_sources = dict( zip([x for x in years], ['_%s' % x for x in years])) js_source_array = str(dictionary_of_sources).replace("'", "") xdr = Range1d(1, 9) ydr = Range1d(20, 100) plot = Plot( x_range=xdr, y_range=ydr, title=Title(text=''), plot_width=800, plot_height=400, outline_line_color=None, toolbar_location=None, min_border=20, ) AXIS_FORMATS = dict( minor_tick_in=None, minor_tick_out=None,
def make_axis(axis, size, factors, dim, flip=False, rotation=0, label_size=None, tick_size=None, axis_height=35): factors = list(map(dim.pprint_value, factors)) nchars = np.max([len(f) for f in factors]) ranges = FactorRange(factors=factors) ranges2 = Range1d(start=0, end=1) axis_label = dim_axis_label(dim) reset = "range.setv({start: 0, end: range.factors.length})" ranges.callback = CustomJS(args=dict(range=ranges), code=reset) axis_props = {} if label_size: axis_props['axis_label_text_font_size'] = value(label_size) if tick_size: axis_props['major_label_text_font_size'] = value(tick_size) tick_px = font_size_to_pixels(tick_size) if tick_px is None: tick_px = 8 label_px = font_size_to_pixels(label_size) if label_px is None: label_px = 10 rotation = np.radians(rotation) if axis == 'x': align = 'center' # Adjust height to compensate for label rotation height = int(axis_height + np.abs(np.sin(rotation)) * ((nchars * tick_px) * 0.82)) + tick_px + label_px opts = dict(x_axis_type='auto', x_axis_label=axis_label, x_range=ranges, y_range=ranges2, plot_height=height, plot_width=size) else: # Adjust width to compensate for label rotation align = 'left' if flip else 'right' width = int(axis_height + np.abs(np.cos(rotation)) * ((nchars * tick_px) * 0.82)) + tick_px + label_px opts = dict(y_axis_label=axis_label, x_range=ranges2, y_range=ranges, plot_width=width, plot_height=size) p = Figure(toolbar_location=None, **opts) p.outline_line_alpha = 0 p.grid.grid_line_alpha = 0 if axis == 'x': p.yaxis.visible = False axis = p.xaxis[0] if flip: p.above = p.below p.below = [] p.xaxis[:] = p.above else: p.xaxis.visible = False axis = p.yaxis[0] if flip: p.right = p.left p.left = [] p.yaxis[:] = p.right axis.major_label_orientation = rotation axis.major_label_text_align = align axis.major_label_text_baseline = 'middle' axis.update(**axis_props) return p
def homepage(request): xmax = ultima_fecha('ise2_infra') df1, df2, df3, xmin_ise1_infra, xmax_ise1_infra, xmin_ise2_infra, xmax_ise2_infra, xmin_e1ms1, xmax_e1ms1 = get_data( ) d = xmax - timedelta(seconds=30) #d= datetime.now() - timedelta(hours=48) #xmax =datetime.now() plot1, script1, div1 = get_plot(df1, 'ise1_infra', xmin_ise1_infra, xmax_ise1_infra) plot2, script2, div2 = get_plot(df2, 'ise2_infra', xmin_ise2_infra, xmax_ise2_infra) plot3, script3, div3 = get_plot(df3, 'e1ms1', xmin_e1ms1, xmax_e1ms1) xmin = xmax - timedelta(seconds=30) # data_list = ultimos_datos() # frame = pd.DataFrame(data_list) # frame.columns= ['fecha_sistema','gxe','gye','gze','axe','aye','aze'] df = pd.DataFrame( list( ISE2_INFRA.objects.filter(fecha_recepcion__range=(d, xmax)).values( 'fecha_recepcion', 'infrasonido_1', 'infrasonido_2', 'infrasonido_3', 'infrasonido_4').order_by('fecha_recepcion'))) # df = pd.DataFrame(list(ISE2_INFRA.objects.filter(fecha_recepcion__range=(d,datetime.now())).values('fecha_recepcion','infrasonido_1','infrasonido_2','infrasonido_3','infrasonido_4').order_by('fecha_recepcion'))) # df = pd.DataFrame(list(ISE2_INFRA.objects.all().values('fecha_recepcion','infrasonido_1','infrasonido_2','infrasonido_3','infrasonido_4'))) source = ColumnDataSource(df) p1 = figure(title='Data Cruda del Sensor 1 al {0}'.format(xmax), x_axis_label='Tiempo', y_axis_label='Cuentas', x_axis_type='datetime', y_axis_type='log', plot_width=500, plot_height=300) p1.x_range = Range1d(xmin, xmax) l1 = p1.line('fecha_recepcion', 'infrasonido_1', source=source, line_width=2, line_alpha=1, line_color="blue") p2 = figure(title='Data Cruda del Sensor 2 al {0}'.format(xmax), x_axis_label='Tiempo', y_axis_label='Cuentas', x_axis_type='datetime', y_axis_type='log', plot_width=500, plot_height=300) p2.x_range = Range1d(xmin, xmax) l2 = p2.line('fecha_recepcion', 'infrasonido_2', source=source, line_width=2, line_alpha=1, line_color="red") p3 = figure(title='Data Cruda del Sensor 3 al {0}'.format(xmax), x_axis_label='Tiempo', y_axis_label='Cuentas', x_axis_type='datetime', y_axis_type='log', plot_width=500, plot_height=300) p3.x_range = Range1d(xmin, xmax) l3 = p3.line('fecha_recepcion', 'infrasonido_3', source=source, line_width=2, line_alpha=1, line_color="green") p4 = figure(title='Data Cruda del Sensor 4 al {0}'.format(xmax), x_axis_label='Tiempo', y_axis_label='Cuentas', x_axis_type='datetime', y_axis_type='log', plot_width=500, plot_height=300) p4.x_range = Range1d(xmin, xmax) l4 = p4.line('fecha_recepcion', 'infrasonido_4', source=source, line_width=2, line_alpha=1, line_color="orange") #plot.xaxis.formatter=DatetimeTickFormatter( #hours=["%d %B %Y"], #minutes=["%d %B %Y"], #days=["%d %B %Y"], #months=["%d %B %Y"], #years=["%d %B %Y"], #) #plot.xaxis.major_label_orientation = pi/4 #legend = Legend(items=[ #("Infrasonido 1", [l1]), #("Infrasonido 2", [l2]), #("Infrasonido 3", [l3]), #("Infrasonido 4", [l4]) #], location=(0, -30)) #plot.add_layout(legend, 'right') plot = gridplot([[p1], [p2], [p3], [p4]]) script, div = components(plot) img1 = ultima_foto() return render_to_response( 'polls/dash.html', { 'script1': script1, 'div1': div1, 'script2': script2, 'div2': div2, 'script3': script3, 'div3': div3, 'img1': img1 })