def make_plot(data):
# 设置图标悬浮工具及数据格式
hover = define_hover()
# 定义 长宽、X轴数据类型、悬浮工具、y轴范围、标题
x = data.date_id_format
y1 = data.uv_ol
y2 = data.uv_lt
title1 = str(data['rank'].iloc[0]) + " " + data.name.iloc[0]
# print(data.max_date.iloc[0])
title2 = str(
data['rank'].iloc[0]
) + " " + data.name.iloc[0] + " 最新rule：" + data.max_date.iloc[0]
p1 = figure(plot_width=400,
plot_height=250,
x_axis_type="datetime",
tools=[hover],
y_range=(0, max(y1)),
title=title1)
p1.yaxis.formatter = BasicTickFormatter(use_scientific=False)
p1.line(x, y1, color='green')
p2 = figure(plot_width=400,
plot_height=250,
x_axis_type="datetime",
tools=[hover],
y_range=(0, max(y2)),
title=title2)
p2.yaxis.formatter = BasicTickFormatter(use_scientific=False)
p2.line(x, y2, color='orange')
return p1, p2
def define_plot_parameters(list):
# Input is a List of format:
# list_plot_parameters = [ x_data1, y_data1,
# plot_title1, x_axis_title1, y_axis_title1,
# plot_size_height1, plot_size_width1,
# legend_location ]
# The parameters have to be controlled like this in a callback to allow
# for them to be adjusted. Otherwise the plot parameters are not
# interactive.
# Yes! - p1.xaxis.axis_label = 'X_axis_title'
# No! - p1 = figure(x_axis_label = 'X_axis_title')
p1.title.text = list[2]
p1.xaxis.axis_label = list[3]
p1.yaxis.axis_label = list[4]
p1.plot_height = list[5]
p1.plot_width = list[6]
p1.legend.location = list[7]
# If the user wants to plot an axis as datetime then the axis needs to
# be reformatted. Will do this by checking if the x_data1/y_data1 is
# =='adate'.
# NB: This only works if 'adate' is used as the name for the date column
# and also that this is the only date column.
if list[0] == 'adate':
p1.xaxis.formatter = DatetimeTickFormatter(days=['%d/%m', '%a%d'])
else:
p1.xaxis.formatter = BasicTickFormatter()
if list[1] == 'adate':
p1.yaxis.formatter = DatetimeTickFormatter(days=['%d/%m', '%a%d'])
else:
p1.yaxis.formatter = BasicTickFormatter()
return
def style(p):
# Title
p.title.align = 'center'
p.title.text_font_size = '18pt'
# Axis titles
p.xaxis.axis_label_text_font_size = '14pt'
p.xaxis.axis_label_text_font_style = 'bold'
p.yaxis.axis_label_text_font_size = '14pt'
p.yaxis.axis_label_text_font_style = 'bold'
# Tick labels
p.xaxis.major_label_text_font_size = '12pt'
p.yaxis.major_label_text_font_size = '12pt'
#p.yaxis.formatter = PrintfTickFormatter(format='%4.0e')
#p.yaxis.formatter = LogTickFormatter()
p.yaxis.formatter = BasicTickFormatter(use_scientific=True,
power_limit_low=1)
# Legend
p.legend.location = 'top_left'
p.legend.label_text_font_size = '8pt'
p.legend.spacing = -9
p.legend.title = 'states, in order of first case'
# Toolbar
p.toolbar.autohide = True
return p
def compare_countries_cumulative_per_million(cases_dataframe, country1,
country2):
"""
This function takes cases dataframe, country1, country2 as a parameter, creates a line plot for the cases of the
two countries and returns the plot.
"""
# convert the values of the dataframe to per million
converted_dataframe = convert_data_to_per_million(cases_dataframe)
# create a figure object with width and height
compare_countries_plot = figure(x_axis_type="datetime",
width=1000,
height=400,
sizing_mode='fixed')
# creating columnDataSource object, for the dataframes
cases_source = ColumnDataSource(converted_dataframe)
compare_countries_plot.yaxis.formatter = BasicTickFormatter(
use_scientific=False)
# add a circle renderers using the source's two columns.
compare_countries_plot.line(x='Date',
y=country1,
source=cases_source,
color='Green',
line_width=2,
legend_label=country1)
compare_countries_plot.line(x='Date',
y=country2,
source=cases_source,
color='purple',
line_width=2,
legend_label=country2)
compare_countries_plot.xaxis.axis_label = 'Date'
compare_countries_plot.yaxis.axis_label = 'Cases'
compare_countries_plot.legend.location = "top_left"
compare_countries_plot.toolbar_location = None
return compare_countries_plot
def size_complexity_plot():
output_file(f"plot_time_complexity_all_scaled.html")
p = figure(
title="Time vs size for Python3. Log scale",
x_axis_type="log",
y_axis_type="log")
p.xaxis.axis_label = 'Elapsed time (seconds)'
p.yaxis.axis_label = 'Problem size'
common_plot_cfg(p)
size_complexity_subplot(
p, vmin=0, vmax=7, index=1,
color="blue", legend="v00-v07 (size of N)")
size_complexity_subplot(
p, vmin=8, vmax=12, index=1048576,
color="red", legend="v08-v12 (x*N<=10^6)")
size_complexity_subplot(
p, vmin=13, vmax=14, index=1048576,
color="dimgrey", legend="v13-v14 (x*N<=10^6)")
p.xaxis.axis_label_text_font_size = '18pt'
p.yaxis.axis_label_text_font_size = '18pt'
p.yaxis.major_label_text_font_size = '15pt'
p.xaxis.major_label_text_font_size = '15pt'
p.xaxis.formatter = BasicTickFormatter(use_scientific=False)
p.legend.location = "top_center"
p.legend.label_text_font_size = '21pt'
p.legend.glyph_height = 45
p.legend.glyph_width = 45
show(p)
def _bokeh_heatmap(source,
colormapper,
x_label,
y_label,
x_range,
y_range,
x_log=False,
y_log=False):
p = figure(x_range=x_range,
y_range=y_range,
x_axis_type="log" if x_log else "linear",
y_axis_type="log" if y_log else "linear",
toolbar_location=None,
tools="")
p.rect(x=x_label,
y=y_label,
width=1,
height=1,
source=source,
fill_color=transform('zz', colormapper),
line_color=None)
color_bar = ColorBar(color_mapper=colormapper,
location=(0, 0),
ticker=BasicTicker(desired_num_ticks=20),
formatter=BasicTickFormatter(use_scientific=False))
p.add_layout(color_bar, 'right')
p.xaxis.axis_label = x_label
p.yaxis.axis_label = y_label
return p
def ETA_plot(vmin=0, vmax=7, eta="eta_MAXN_y", title_sufix="N=2^20",
unit="years", color=["red", "blue"], legend=["pypy", "python3"]):
output_file(f"plot_eta{vmax}.html")
timing_subset = {k: v for k, v in timing.items()
if vmin <= int(k[1:]) <= vmax}
timing_factors = []
timing_ETA = []
python3_line = []
for version, language in timing_subset.items():
for lang_name, times in language.items():
timing_factors.append((version, lang_name))
timing_ETA.append(times[eta])
if lang_name == "python3":
python3_line.append(times[eta])
p = figure(
title=f"Estimated time to compute {title_sufix}",
x_range=FactorRange(*timing_factors))
p.xaxis.axis_label = 'Code version'
p.yaxis.axis_label = f'Elapsed time ({unit})'
colors_by_lang = color * 10
p.vbar(x=timing_factors, top=timing_ETA, width=1, alpha=0.8,
color=colors_by_lang[:len(timing_factors)])
p.line(x=[f for f in timing_factors if f[1] == "python3"],
y=python3_line, color="black", line_width=6, line_dash='dashed')
common_plot_cfg(p, legend=legend, color=color)
p.yaxis.formatter = BasicTickFormatter(use_scientific=False)
show(p)
def plot_with_slider(dataframe, country_name, y_axis_name):
""""
this function takes a dataframe, y-axis name and country as a parameter,
creates a plot with a slider and returns the plot.
"""
# create a figure object with width and height
plot = figure(x_axis_type="datetime",
width=1000,
height=400,
sizing_mode='fixed')
# creating columnDataSource object, for the dataframes
source = ColumnDataSource(dataframe)
# initialize the min and max value of the date
init_value = (dataframe['Date'].min(), dataframe['Date'].max())
# configuring date range slider with start date end date and value
date_range_slider = DateRangeSlider(start=init_value[0],
end=init_value[1],
value=init_value)
date_filter = BooleanFilter(booleans=[True] * dataframe.shape[0])
# not use scientific numbers on Y-axis
plot.yaxis.formatter = BasicTickFormatter(use_scientific=False)
# whenever a slider value updates. The date range sliders, value changes.
date_range_slider.js_on_change(
"value",
CustomJS(args=dict(f=date_filter, cases_source=source),
code="""\
const [start, end] = cb_obj.value;
f.booleans = Array.from(cases_source.data['Date']).map(d => (d >= start
&& d <= end));
// Needed because of https://github.com/bokeh/bokeh/issues/7273
cases_source.change.emit();
"""))
# add a circle renderer using the source's two columns.
plot.circle(x='Date',
y=country_name,
source=source,
view=CDSView(source=source, filters=[date_filter]),
color='Pink',
line_width=0.5)
# name and field pairs for the Hover tool
tooltips = [('Date', '@Date{%F}'), (country_name, "$y{int}")]
# formatting scheme of date column
formatters = {'@Date': 'datetime'}
# create a Hover tool for the figure with the tooltips and specify the formatting scheme
plot.add_tools(
HoverTool(tooltips=tooltips, formatters=formatters, mode='vline'))
plot.title.text_color = "midnightblue"
plot.title.text_font_size = "25px"
plot.toolbar.active_drag = None
plot.toolbar_location = None
plot.xaxis.axis_label = 'Date'
plot.yaxis.axis_label = y_axis_name
return column(plot, date_range_slider)
def death_and_cases_plot(cases_dataframe, death_dataframe, country_name,
y_axis_type):
"""
This function takes in cases dataframes, deaths dataframe, country name, and y-axis type as a parameter.
It creates a line chart with cases and deaths of the country that is passed as a parameter. The plots y-axis type
will either be linear or log, depending on the parameter. Then returns the plot.
"""
# create a figure object with width and height
death_and_cases_fig = figure(x_axis_type="datetime",
y_axis_type=y_axis_type,
width=1000,
height=400,
sizing_mode='fixed')
# creating columnDataSource object, for the dataframes
cases_source = ColumnDataSource(cases_dataframe)
death_sources = ColumnDataSource(death_dataframe)
# not use scientific numbers on Y-axis
death_and_cases_fig.yaxis.formatter = BasicTickFormatter(
use_scientific=False)
# add a line renderer using the cases_source's two columns with a label, color and line width to the figure object
death_and_cases_fig.line(x='Date',
y=country_name,
source=cases_source,
color='Blue',
line_width=2,
legend_label="Cases")
# add another line renderer using the death_source's two columns with a label, color and line width.
death_and_cases_fig.line(x='Date',
y=country_name,
source=death_sources,
color='Red',
line_width=2,
legend_label="Deaths")
# name and field pairs for the Hover tool
tooltips = [('Date', '@Date{%F}'), (country_name, "$y{int}")]
# formatting scheme of date column
formatters = {'@Date': 'datetime'}
# create a Hover tool for the figure with the tooltips and specify the formatting scheme
death_and_cases_fig.add_tools(
HoverTool(tooltips=tooltips, formatters=formatters))
# get rid of the default toolbar
death_and_cases_fig.toolbar_location = None
death_and_cases_fig.title.text = 'Covid cases and deaths'
death_and_cases_fig.title.text_color = "midnightblue"
death_and_cases_fig.title.text_font_size = "25px"
death_and_cases_fig.xaxis.axis_label = 'Date'
death_and_cases_fig.yaxis.axis_label = 'Confirmed Cases'
death_and_cases_fig.legend.location = "top_left"
return death_and_cases_fig
def bokeh_line_uncertainty(values, mean, std, x_log, x_label, y_label,
inc_indices):
"""
Return a bokeh plot with a simple line plot and uncertainty marked as filled area above and below plot.
Parameters
----------
values: List
values for x-axis (same length as mean and std)
mean: List[float]
means (same length as values and std)
std: List[float]
uncertainty to be plotted around the mean (same length as values and mean)
x_log: bool
plot x-axis in log
x_label, y_label: str
axis-labels
inc_indices: List[int]
list of indices in values that are to be marked as incumbents
Returns
-------
plot: bokeh.plotting.figure
the bokeh figure
"""
lower_curve = mean - std
upper_curve = mean + std
p = figure(x_range=(values[0], values[-1]),
x_axis_type="log" if x_log else "linear",
height=350,
width=800)
p.line(values, mean)
band_x = np.append(values, values[::-1])
band_y = np.append(lower_curve, upper_curve[::-1])
p.patch(band_x, band_y, color='#7570B3', fill_alpha=0.2)
if len(inc_indices) > 0:
p.circle(list([values[idx] for idx in inc_indices]),
list([float(mean[idx]) for idx in inc_indices]),
name='incumbent',
color='black')
p.xaxis.axis_label = x_label
p.yaxis.axis_label = y_label
p.yaxis.formatter = BasicTickFormatter(use_scientific=False)
return p
def doAllTheWork(filename="GlobalLandTemperaturesByCountry.csv"):
L = getTemps(filename)
N = convertDate(L)
dateList = []
tList = []
for tup in N:
try:
tList.append(float(tup[1]))
dateList.append(tup[0])
except:
continue
output_file("pratplot.html")
p = figure(plot_width=500, plot_height=250, x_axis_type="datetime")
p.circle(dateList, tList, radius=0.1, alpha=0.5)
p.xaxis.formatter = BasicTickFormatter()
show(p)
def create_date_price_sum_chart():
groupByPrice = housing[['date', 'price']].groupby(['date']).sum()
groupByPrice['index'] = groupByPrice.index
p8 = figure(plot_height=300, x_axis_type='datetime')
p8.yaxis.formatter = BasicTickFormatter(use_scientific=False)
p8.line(groupByPrice['index'],
groupByPrice['price'],
line_width=2,
color="#33A02C",
legend='Earned money')
p8.add_tools(
HoverTool(tooltips=[("Date", "@x{%F}"), ('Income', '@y{0.0,0}$')],
formatters={'x': 'datetime'}))
p8.sizing_mode = 'scale_width'
# source = ColumnDataSource(data=dict(x=groupByPrice['date'], y=groupByPrice['price']))
tab8 = Panel(child=p8, title='Houses income')
return tab8
def bokeh_boxplot(labels, mean, std, x_label, y_label, runtime, inc_indices):
"""
Plot a boxplot in bokeh
Parameters
----------
labels: List[str]
labels for the x-axis (for the different boxes)
mean: List[float]
means, same length as labels
std: List[float]
stds, same length as labels
x_label, y_label: str
axis-labels
runtime: bool
whether runtime is analyzed, if so will limit y-axis to 0 for std
inc_indices: List[int]
list of indices in values that are to be marked as incumbents
Returns
-------
plot: bokeh.plotting.figure
the bokeh figure
"""
p = figure(x_range=labels, height=350, width=800)
upper = mean + std
lower = mean - std
if runtime:
lower = [0 if l < 0 else l for l in lower]
p.vbar(labels, 0.7, mean, upper, fill_color="#E08E79", line_color="black")
p.vbar(labels, 0.7, mean, lower, fill_color="#3B8686", line_color="black")
if len(inc_indices) > 0:
p.circle(list([labels[idx] for idx in inc_indices]),
list([float(mean[idx]) for idx in inc_indices]),
name='incumbent',
color='black')
p.xaxis.axis_label = x_label
p.yaxis.axis_label = y_label
p.yaxis.formatter = BasicTickFormatter(use_scientific=False)
return p
class TrendIndicator(HTMLBox):
"""
A Bokeh model indicating trends.
"""
description = String()
change_formatter = Instance(
TickFormatter, default=lambda: NumeralTickFormatter(format='0.00%'))
formatter = Instance(TickFormatter, default=lambda: BasicTickFormatter())
layout = String()
source = Instance(ColumnDataSource)
plot_x = String()
plot_y = String()
plot_color = String()
plot_type = String()
pos_color = String()
neg_color = String()
title = String()
value = Float()
value_change = Float()
def bokeh_choropleth(data_df, geojson, key_on, data, tooltips, color_mapper):
geojson2 = geojson.copy()
in_proj = Proj(init='epsg:4326')
out_proj = Proj(init='epsg:3857')
for f in geojson2['features']:
for col in data_df.columns:
f['properties'][col] = str(data_df[data_df[key_on[0]] == f['properties'][key_on[1]]][col].values[0])
if f['geometry']['type'] == 'MultiPolygon':
f['geometry']['coordinates'] = [[[[x for x in transform(in_proj, out_proj, p[0], p[1])] for p in c]] for c in f['geometry']['coordinates']]
else:
f['geometry']['coordinates'] = [[[x for x in transform(in_proj, out_proj, p[0], p[1])] for p in c] for c in f['geometry']['coordinates']]
geo_source = GeoJSONDataSource(geojson=json.dumps(geojson2, ensure_ascii=True))
TOOLS = "pan,wheel_zoom,hover,save"
choropleth = figure(tools=TOOLS,
x_axis_location=None, y_axis_location=None)
choropleth.patches('xs', 'ys',
source=geo_source,
line_color="black",
line_width=0.2,
fill_color={'field': data, 'transform': color_mapper},
fill_alpha=1)
choropleth.add_tile(STAMEN_TERRAIN)
color_bar = ColorBar(color_mapper=color_mapper, major_label_text_font_size="5pt",
ticker=BasicTicker(desired_num_ticks=len(color_mapper.palette)),
formatter=BasicTickFormatter(precision=2, use_scientific=False),
label_standoff=6, border_line_color=None, location=(0,0),
scale_alpha=1,
orientation='horizontal')
choropleth.add_layout(color_bar, 'above')
hover = choropleth.select_one(HoverTool)
hover.point_policy = "follow_mouse"
hover.tooltips = tooltips
return choropleth
def view_alignment(ids, seqs, chromPoints, side, fontsize="9pt", plot_width=800):
"""Bokeh sequence alignment view"""
text = [i for s in list(seqs) for i in s]
clrs = {'T':RGB(153, 204, 153), 'A':RGB(255, 153, 153), 'G':RGB(255, 219, 153), 'C':RGB(153, 153, 255), '-':'white', 'N':'white', '*':'white'}
colors = [clrs[i] for i in text]
N = len(seqs[0])/2
center = int(chromPoints[2])
x = np.arange(center-N,center+N)
y = np.arange(0,len(seqs),1)
xx, yy = np.meshgrid(x, y)
gx = xx.ravel()
gy = yy.flatten()
source = ColumnDataSource(dict(x=gx+0.5, y=gy, recty=gy+0.5, rectx1=gx, rectx2=gx+1, text=text, colors=colors))
plot_height = len(seqs)*20+200
view_range = (center-20, center+20)
p1 = figure(title=' '.join(chromPoints), plot_width=plot_width, plot_height=plot_height,
x_range=view_range, y_range=ids, tools="xpan,reset",
min_border_bottom=100, min_border_top=100, toolbar_location='below')#, lod_factor=1)
glyph = Text(x="x", y="y", text="text", text_align='center',text_color="black",
text_font="monospace",text_font_size=fontsize)
p1.segment(x0='rectx1', y0='recty', x1='rectx2',
y1='recty', color="colors", line_width=14, source=source)
p1.add_glyph(source, glyph)
breakLine = Span(location=int(chromPoints[2]), dimension='height', line_color='red', line_width=2)
p1.renderers.extend([breakLine])
p1.grid.visible = False
p1.add_layout(LinearAxis(formatter=BasicTickFormatter(use_scientific=False), major_label_orientation = pi/4), 'above')
p1.xaxis.major_label_text_font_style = "bold"
p1.yaxis.minor_tick_line_width = 0
p1.yaxis.major_tick_line_width = 0
p1.below[0].formatter.use_scientific = False
p1.xaxis.major_label_orientation = pi/4
if side == 'left': p1.name = 'pl'
else:
p1.name = 'pr'
p1.min_border_left = 10
p1.yaxis.visible=False
return p1, source
def updatePlot():
"""Read data and generate plot in HTML file via Bokeh"""
df = pandas.read_csv("data.csv")
x = [datetime.fromtimestamp(ts) for ts in df["timestamp"]]
y = df["concentration"]
output_file("plot.html")
fig1 = figure(toolbar_location='below',
x_axis_type='datetime',
tools="box_zoom,wheel_zoom,reset",
active_drag="box_zoom",
active_scroll=None,
active_tap=None,
toolbar_sticky=False,
y_range=(0, 1.5e6))
fig1.title.text_font_size = '24pt'
fig1.yaxis.axis_label = r"$$ Particle \ (>1 \ \mu m) \ counts/ ft^{3}$$"
fig1.yaxis.axis_label_text_font_size = '18pt'
fig1.yaxis.major_label_text_font_size = '16pt'
fig1.xaxis.axis_label_text_font_size = '18pt'
fig1.xaxis.major_label_text_font_size = '16pt'
fig1.background_fill_color = 'beige'
fig1.xaxis.formatter = DatetimeTickFormatter(days="%m/%d",
hours="%H:%M",
minutes="%H:%M")
fig1.yaxis.formatter = BasicTickFormatter(precision=1)
fig1.background_fill_alpha = 0.5
cr = fig1.circle(x,
y,
size=3,
fill_color='black',
hover_fill_color="firebrick",
line_color='black',
hover_line_color=None)
fig1.add_tools(HoverTool(tooltips=None, renderers=[cr], mode='hline'))
fig1.sizing_mode = 'stretch_both'
curdoc().add_root(fig1)
curdoc().title = "Harlem Air Quality"
save(fig1)
def plot_market_depth(buys, sells, pair_base, pair_quote):
'''
Expects buys and sells as ordered columndatasources
Both lists start at the midpoint and extend towards 0 and infinity respectively
'''
minx = buys.data['prices'][
0] * 0.98 #default xrange is 2% either side of buy/sell gap
maxx = sells.data['prices'][0] * 1.02
plot = figure(lod_threshold=None, x_range=(minx, maxx), plot_width=1600)
plot.axis.formatter = BasicTickFormatter(use_scientific=False)
plot.xaxis.axis_label = pair_base
plot.xaxis.ticker = BasicTicker(desired_num_ticks=11)
plot.depth('prices',
'amounts',
source=buys,
fill_color='#00a000',
line_alpha=0)
plot.depth('prices',
'amounts',
source=sells,
fill_color='#a00000',
line_alpha=0)
return plot
def make_week_bars(title: str, weeks: list, categories: list, legends: list, stats: list):
cur_data = {
'weeks': weeks
}
for category in categories:
cur_data[category] = []
for stat in stats:
cur_data[category].append(stat[category])
source = ColumnDataSource(data=cur_data)
bar_size = .8 / len(categories)
cur_fig = figure(
x_range=weeks,
y_axis_type='log',
plot_height=480,
title=title,
)
for idx, category in enumerate(categories):
cur_fig.vbar(
x=dodge('weeks', idx*bar_size-0.42, range=cur_fig.x_range),
top=category, bottom=1,
width=bar_size, source=source,
legend_label=legends[idx],
color=Accent[8][idx]
)
cur_fig.x_range.range_padding = 0.1
cur_fig.xgrid.grid_line_color = None
cur_fig.legend.location = "bottom_right"
cur_fig.legend.orientation = "horizontal"
cur_fig.yaxis.formatter = BasicTickFormatter(use_scientific=False)
return cur_fig
def parallel_plot(df, color=None, palette=None):
"""From a dataframe create a parallel coordinate plot
"""
npts = df.shape[0]
ndims = len(df.columns)
if color is None:
color = np.ones(npts)
if palette is None:
palette = ['#ff0000']
cmap = LinearColorMapper(high=color.min(),
low=color.max(),
palette=palette)
data_source = ColumnDataSource(dict(
xs=np.arange(ndims)[None, :].repeat(npts, axis=0).tolist(),
ys=np.array((df-df.min())/(df.max()-df.min())).tolist(),
color=color))
p = figure(x_range=(-1, ndims),
y_range=(0, 1),
width=1000,
tools="pan, box_zoom")
# Create x axis ticks from columns contained in dataframe
fixed_x_ticks = FixedTicker(
ticks=np.arange(ndims), minor_ticks=[])
formatter_x_ticks = FuncTickFormatter(
code="return columns[index]", args={"columns": df.columns})
p.xaxis.ticker = fixed_x_ticks
p.xaxis.formatter = formatter_x_ticks
p.yaxis.visible = False
p.y_range.start = 0
p.y_range.end = 1
p.y_range.bounds = (-0.1, 1.1) # add a little padding around y axis
p.xgrid.visible = False
p.ygrid.visible = False
# Create extra y axis for each dataframe column
tickformatter = BasicTickFormatter(precision=1)
for index, col in enumerate(df.columns):
start = df[col].min()
end = df[col].max()
bound_min = start + abs(end-start) * (p.y_range.bounds[0] - p.y_range.start)
bound_max = end + abs(end-start) * (p.y_range.bounds[1] - p.y_range.end)
p.extra_y_ranges.update(
{col: Range1d(start=bound_min, end=bound_max, bounds=(bound_min, bound_max))})
fixedticks = FixedTicker(
ticks=np.linspace(start, end, 8), minor_ticks=[])
p.add_layout(LinearAxis(fixed_location=index, y_range_name=col,
ticker=fixedticks, formatter=tickformatter), 'right')
# create the data renderer ( MultiLine )
# specify selected and non selected style
non_selected_line_style = dict(line_color='grey', line_width=0.1, line_alpha=0.5)
selected_line_style = dict(line_color={'field': 'color', 'transform': cmap}, line_width=1)
parallel_renderer = p.multi_line(
xs="xs", ys="ys", source=data_source, **non_selected_line_style)
# Specify selection style
selected_lines = MultiLine(**selected_line_style)
# Specify non selection style
nonselected_lines = MultiLine(**non_selected_line_style)
parallel_renderer.selection_glyph = selected_lines
parallel_renderer.nonselection_glyph = nonselected_lines
p.y_range.start = p.y_range.bounds[0]
p.y_range.end = p.y_range.bounds[1]
rect_source = ColumnDataSource({
'x': [], 'y': [], 'width': [], 'height': []
})
# add rectangle selections
selection_renderer = p.rect(x='x', y='y', width='width', height='height',
source=rect_source,
fill_alpha=0.7, fill_color='#009933')
selection_tool = ParallelSelectionTool(
renderer_select=selection_renderer, renderer_data=parallel_renderer,
box_width=10)
# custom resets (reset only axes not selections)
reset_axes = ParallelResetTool()
# add tools and activate selection ones
p.add_tools(selection_tool, reset_axes)
p.toolbar.active_drag = selection_tool
return p
def load_page(experiment_df, experiment_db):
########## bokeh plots ##########
# general plot tools
plot_tools = 'wheel_zoom, pan, reset, save, hover'
hover = HoverTool(tooltips=[("(x,y)", "($x, $y)")])
# progress curve plots
global raw_source
raw_source = ColumnDataSource(data=dict(x=[], y=[], yr=[], yfit=[]))
global raw
raw = figure(title="Initial Rate Fit",
x_axis_label="Time",
y_axis_label="Signal",
plot_width=350,
plot_height=300,
tools=plot_tools)
raw.circle('x',
'y',
size=2,
source=raw_source,
color='gray',
selection_color="black",
alpha=0.6,
nonselection_alpha=0.2,
selection_alpha=0.6)
raw.line('x', 'yfit', source=raw_source, color='red')
global warning_source
warning_source = ColumnDataSource(data=dict(
x=[0],
y=[0],
t=[
'Please enter transform equation! \nMust convert signal to [substrate] \nin Schnell-Mendoza mode (e.g. via \nx/6.22/0.45/0.001 for sample data). \nNote: this transform may need \nto be inverted through multiplying \nby -1 when analyzing experiments \nthat measure increasing product \nconcentration over time)'
]))
global warning
warning = raw.text(x='x',
y='y',
text='t',
text_font_size='12pt',
angle=0,
source=warning_source)
warning.visible = False
global circles_source
circles_source = ColumnDataSource(
data=dict(x=[-.05, -.05, 1.6, 1.6], y=[0, 0.6, 0, 0.6]))
global circles
circles = raw.circle(x='x', y='y', alpha=0., source=circles_source)
circles.visible = False
global resi
resi = figure(title="Initial Rate Fit Residuals",
x_axis_label="Time",
y_axis_label="Residual",
plot_width=700,
plot_height=200,
tools='wheel_zoom,pan,reset')
resi.yaxis.formatter = BasicTickFormatter(precision=2, use_scientific=True)
resi.circle('x', 'yr', size=5, source=raw_source, color='grey', alpha=0.6)
# model plot for titration experiments
global model_data_source
model_data_source = ColumnDataSource(
data=dict(xt=[], yt=[], n=[], ct=[], et=[]))
global model_plot_source
model_plot_source = ColumnDataSource(
data=dict(xp=[], yp=[], l=[], u=[], cp=[], ep=[]))
global model_fit_source
model_fit_source = ColumnDataSource(data=dict(x=[], y=[]))
global varea_source
varea_source = ColumnDataSource(data=dict(x=[], r1=[], r2=[]))
global model
model = figure(title='Model Fit',
x_axis_label='Concentration',
y_axis_label='Rate',
plot_width=350,
plot_height=300,
tools=plot_tools)
model.circle('xp',
'yp',
size=8,
source=model_plot_source,
color='cp',
alpha=0.6)
model.add_layout(
Whisker(source=model_plot_source, base='xp', upper='u', lower='l'))
model.line('x',
'y',
source=model_fit_source,
line_width=3,
color='black',
alpha=0.8)
model.varea('x', 'r1', 'r2', source=varea_source, color='grey', alpha=0.3)
########## bokeh widgets ##########
# button for selecting progress curve fitting routine
global fit_button
fit_button = RadioButtonGroup(labels=[
'Maximize Slope Magnitude', 'Linear Fit', 'Logarithmic Fit',
'Schnell-Mendoza'
],
active=0,
width=375)
fit_button.on_change('active', widget_callback)
# button for selecting progress curve fitting routine
global scalex_box
scalex_box = CheckboxButtonGroup(
labels=["transform x-axis to Log10 scale"], active=[])
scalex_box.on_change('active', widget_callback)
# dropdown menu for selecting titration experiment model
global model_select
model_select = Select(
title='Choose Model',
value='Michaelis-Menten',
options=['Michaelis-Menten', 'pEC50/pIC50', 'High-Throughput Screen'],
width=350)
model_select.on_change('value', widget_callback)
# dropdown menu for selecting blank sample to subtract from remaining titration samples
global subtract_select
subtract_select = Select(title='Select Blank Sample for Subtraction',
value='',
options=list(experiment_df)[1:] + [''],
width=350)
subtract_select.on_change('value', widget_callback)
# dropdown menu for selecting titration sample to plot in current view
global sample_select
sample_select = Select(title='Y Axis Sample',
value=list(experiment_df)[-1],
options=list(experiment_df)[1:],
width=350)
sample_select.on_change('value', sample_callback)
# text input box for transforming slopes to rates
global transform_input
transform_input = TextInput(value='',
title="Enter Transform Equation",
width=350)
transform_input.on_change('value', widget_callback)
# text input box for setting delay time in logarithmic progress curve fitting
global offset_input
offset_input = TextInput(value='',
title="Enter Time Between Mixing and First Read",
width=350)
offset_input.on_change('value', widget_callback)
# text input boxes for fixing EC/IC50 parameters
global bottom_fix
bottom_fix = TextInput(value='', title="Fix pIC50/pEC50 Bottom")
bottom_fix.on_change('value', widget_callback)
global top_fix
top_fix = TextInput(value='', title="Fix pIC50/pEC50 Top")
top_fix.on_change('value', widget_callback)
global slope_fix
slope_fix = TextInput(value='', title="Fix pIC50/pEC50 Hill Slope")
slope_fix.on_change('value', widget_callback)
# text input boxes for progress curve xrange selection
global start_time
start_time = TextInput(value=str(
experiment_df[list(experiment_df)[0]].values[0]),
title="Enter Start Time")
global end_time
end_time = TextInput(value=str(
experiment_df[list(experiment_df)[0]].values[-1]),
title='Enter End Time')
start_time.on_change('value', xbox_callback)
end_time.on_change('value', xbox_callback)
# range slider to select threshold for hit detection in HTS mode
global threshold_slider
threshold_slider = Slider(start=0,
end=5,
value=2,
step=0.1,
title='HTS Hit Threshold (Standard Deviation)',
width=350)
threshold_slider.on_change('value', threshold_callback)
# range slider to update plots according to progress cuve xrange selection
xmin = experiment_df[experiment_df.columns[0]].values[0]
xmax = experiment_df[experiment_df.columns[0]].values[-1]
global range_slider
range_slider = RangeSlider(
start=xmin,
end=xmax,
value=(xmin, xmax),
step=experiment_df[experiment_df.columns[0]].values[1] - xmin,
title='Fine Tune X-Axis Range',
width=650)
range_slider.on_change('value', slider_callback)
# button to upload local data file
global file_source
file_source = ColumnDataSource(data=dict(file_contents=[], file_name=[]))
file_source.on_change('data', file_callback)
try:
output_filename = file_source.data['file_name'] + '-out.csv'
except:
output_filename = 'output.csv'
global upload_button
upload_button = Button(label="Upload Local File",
button_type="success",
width=350)
upload_button.callback = CustomJS(args=dict(file_source=file_source),
code=open(
join(dirname(__file__),
"upload.js")).read())
# table containing rate fits and errors
template = """
<div style="background:<%=ct%>"; color="white";>
<%= value %></div>
"""
formatter = HTMLTemplateFormatter(template=template)
columns = [
TableColumn(field='n', title='Sample'),
TableColumn(field='yt',
title='Slope (Initial Rate)',
formatter=formatter),
TableColumn(field='et', title='Std. Error')
]
global rate_table
rate_table = DataTable(source=model_data_source,
columns=columns,
width=350,
height=250,
selectable=True,
editable=True)
# tables containing model fits and errors
global mm_source
mm_source = ColumnDataSource(dict(label=[], Km=[], Vmax=[]))
columns = [
TableColumn(field='label', title=''),
TableColumn(field='Vmax', title='Vmax'),
TableColumn(field='Km', title='Km')
]
global mm_table
mm_table = DataTable(source=mm_source,
columns=columns,
width=350,
height=75,
selectable=True,
editable=True)
global ic_source
ic_source = ColumnDataSource(
dict(label=[], Bottom=[], Top=[], Slope=[], p50=[]))
columns = [
TableColumn(field='label', title=''),
TableColumn(field='Bottom', title='Bottom'),
TableColumn(field='Top', title='Top'),
TableColumn(field='Slope', title='Slope'),
TableColumn(field='p50', title='pEC/IC50')
]
global ic_table
ic_table = DataTable(source=ic_source,
columns=columns,
width=350,
height=75,
selectable=True,
editable=True)
# button for copying rate data table to clipboard
global copy_button
copy_button = Button(label="Copy Table to Clipboard",
button_type="primary",
width=350)
copy_button.callback = CustomJS(args=dict(source=model_data_source),
code=open(
join(dirname(__file__),
"copy.js")).read())
# button for downloading rate data table to local csv file
global download_button
download_button = Button(label="Download Table to CSV",
button_type="primary",
width=350)
download_button.callback = CustomJS(args=dict(source=model_data_source,
file_name=output_filename),
code=open(
join(dirname(__file__),
"download.js")).read())
########## document formatting #########
desc = Div(text=open(join(dirname(__file__), "description.html")).read(),
width=1400)
advanced = Div(
text="""<strong>Advanced Settings for \npEC/IC50 Analysis</strong>""")
widgets = widgetbox(model_select, sample_select, subtract_select,
transform_input, offset_input, advanced, scalex_box,
bottom_fix, top_fix, slope_fix)
table = widgetbox(rate_table)
main_row = row(
column(upload_button, widgets),
column(fit_button, row(raw, model), resi, row(start_time, end_time),
range_slider),
column(download_button, copy_button, table, mm_table, ic_table,
threshold_slider))
sizing_mode = 'scale_width'
l = layout([[desc], [main_row]], sizing_mode=sizing_mode)
update()
curdoc().clear()
curdoc().add_root(l)
curdoc().title = "ICEKAT"
def density_handler(doc: Document) -> None:
"""
Handler function for the density application.
"""
task_id = doc.session_context.request.arguments.get('task_id')
username = doc.session_context.request.arguments.get('username')
user_dir = os.path.join(settings.USER_DATA, username)
path_to_db = os.path.join(user_dir, task_id)
try:
conn = os.path.join(path_to_db, task_id + '.csv')
df = pd.read_csv(conn)
proteins = list(df.Protein.unique())
lipids = list(df.Lipids.unique())
except:
proteins, lipids = [], []
for o in os.listdir(path_to_db):
key = o.split('/')[-1]
if key.endswith('.npy'):
if key.startswith('prot_'):
proteins.append('Protein(s)')
else:
lipid = key.split('_')[0]
lipids.append(lipid)
all_mpl_cmaps = [
'viridis',
'plasma',
'inferno',
'cividis',
'Greys',
'Purples',
'Blues',
'Greens',
'Oranges',
'Reds',
'RdYlBu',
'RdYlGn',
'Spectral',
'Spectral_r',
'coolwarm',
'coolwarm_r',
'seismic',
'YlOrBr',
'YlOrRd',
'OrRd',
'PuRd',
'RdPu',
'BuPu',
'GnBu',
'PuBu',
'YlGnBu',
'PuBuGn',
'BuGn',
'YlGn',
'PiYG',
'PRGn',
'BrBG',
'PuOr',
'RdGy',
'RdBu',
]
#TODO: globals declaration should not be necessary anymore.
global prot_xyz
prot_xyz = np.load(os.path.join(path_to_db, "prot_" + task_id + '.npy'))
global l_xyz
global x_min, x_max, y_min, y_max, z_min, z_max
l_xyz = np.load(
os.path.join(path_to_db, lipids[0] + "_" + task_id + '.npy'))
x_min, x_max = l_xyz[:, 0].min(), l_xyz[:, 0].max()
y_min, y_max = l_xyz[:, 1].min(), l_xyz[:, 1].max()
z_min, z_max = l_xyz[:, 2].min(), l_xyz[:, 2].max()
# widgets
color_map = Select(title="Colormap",
value="viridis",
options=all_mpl_cmaps,
width=120)
lipid = Select(title="Lipids", value=lipids[0], options=lipids, width=100)
denstype = Select(title="Density Type",
value="All Proteins",
options=["All Proteins", "Average"],
width=120)
number = Slider(title="Number of Bins",
value=380,
start=80,
end=500,
step=10,
width=200)
protein = Select(title="Show Protein",
value="No",
options=["Yes", "No"],
width=90)
gpcr = Select(title="Protein",
value=proteins[0],
options=proteins,
width=130)
zrange = RangeSlider(start=z_min,
end=z_max,
value=(z_min, z_max),
step=0.2,
title="Z-Axis Range",
callback_policy='mouseup',
width=352)
cbar_range = Slider(value=256,
start=0,
end=256,
step=1,
height=600,
orientation='vertical',
callback_policy='mouseup',
margin=(15, 0),
tooltips=False,
show_value=False)
# colorschemes
colormap = cm.get_cmap(color_map.value)
bokehpalette = [
mpl.colors.rgb2hex(m) for m in colormap(np.arange(colormap.N))
]
def lipid_density(array, bins=160):
"""Given a 2D array, containing the x, y values of a lipid,
return its histogram with edges."""
x = array[:, 0]
y = array[:, 1]
lipid, e1, e2 = np.histogram2d(x, y, density=True, bins=bins)
return lipid, e1, e2
# Plot histogram image
H, xe, ye = lipid_density(l_xyz, number.value)
minx = np.abs(xe.min())
miny = np.abs(ye.min())
p1 = figure(plot_height=640, plot_width=640, tools='')
image_source = ColumnDataSource(data=dict(image=[]))
img = p1.image(image="image",
x=xe[0],
y=ye[0],
dw=xe[-1] + minx,
dh=ye[-1] + miny,
palette=bokehpalette,
source=image_source)
circle_prot_source = ColumnDataSource(
data=dict(x1=prot_xyz[:, 0], y1=prot_xyz[:, 1]))
p1.circle(x="x1",
y="y1",
source=circle_prot_source,
size=2,
fill_alpha=0.2)
cb_palette = LinearColorMapper(palette=bokehpalette,
low=H.min(),
high=H.max())
color_bar = ColorBar(color_mapper=cb_palette,
width=8,
location=(0, 0),
label_standoff=10)
color_bar.formatter = BasicTickFormatter(use_scientific=False)
p1.add_layout(color_bar, 'right')
# Make graph pretty
p1.xgrid.grid_line_color = None
p1.ygrid.grid_line_color = None
p1.xaxis.major_tick_line_color = None
p1.xaxis.minor_tick_line_color = None
p1.yaxis.major_tick_line_color = None
p1.yaxis.minor_tick_line_color = None
p1.xaxis.major_label_text_font_size = '0pt'
p1.yaxis.major_label_text_font_size = '0pt'
p1.grid.visible = False
p1.toolbar.logo = None
p1.toolbar_location = None
def update_all(cond=False, cmap=False):
"""
Update the image showing all proteins.
"""
if cond:
# For efficiency execute only if GPCR structure changes
global l_xyz
global x_min, x_max, y_min, y_max, z_min, z_max
l_xyz = np.load(
os.path.join(path_to_db,
str(lipid.value) + "_" + task_id + '.npy'))
x_min, x_max = l_xyz[:, 0].min(), l_xyz[:, 0].max()
y_min, y_max = l_xyz[:, 1].min(), l_xyz[:, 1].max()
z_min, z_max = l_xyz[:, 2].min(), l_xyz[:, 2].max()
zrange.start = z_min
zrange.end = z_max
index = np.where((l_xyz[:, 2] > zrange.value[0])
& (l_xyz[:, 2] < zrange.value[1]))
l_xyz_new = l_xyz[index]
else:
l_xyz_new = l_xyz
if cmap:
# For efficiency execute only if image colormap changes
cb_cut_value = 256 - cbar_range.value
cmap = color_map.value
colormap = cm.get_cmap(cmap)
bokehpalette = [
mpl.colors.rgb2hex(m) for m in colormap(np.arange(colormap.N))
]
bp_i = 0
while bp_i < len(bokehpalette[:cb_cut_value]):
bokehpalette[bp_i] = '#ffffff'
bp_i += 1
img.glyph.color_mapper.palette = bokehpalette
color_bar.color_mapper.palette = bokehpalette
# Update histogram image
H, xe, ye = lipid_density(l_xyz_new, number.value)
minx = np.abs(xe.min())
miny = np.abs(ye.min())
img.glyph.dw = xe[-1] + minx
img.glyph.dh = ye[-1] + miny
# update image source
image_source.data = dict(image=[H])
def update_protein(cond=False):
"""
Update the protein representation.
"""
if cond:
# For efficiency execute only if GPCR structure changes
global prot_xyz
if protein.value == "Yes":
circle_prot_source.data = dict(x1=prot_xyz[:, 0],
y1=prot_xyz[:, 1])
elif protein.value == "No":
circle_prot_source.data = dict(x1=[], y1=[])
def update_cbar():
cb_cut_value = 256 - cbar_range.value
cmap = color_map.value
colormap = cm.get_cmap(cmap)
bokehpalette = [
mpl.colors.rgb2hex(m) for m in colormap(np.arange(colormap.N))
]
bp_i = 0
while bp_i < len(bokehpalette[:cb_cut_value]):
bokehpalette[bp_i] = '#ffffff'
bp_i += 1
img.glyph.color_mapper.palette = bokehpalette
color_bar.color_mapper.palette = bokehpalette
# event listeners
controls = [lipid, zrange, gpcr]
for control in controls:
control.on_change('value',
lambda attr, old, new: update_denstype(cond=True))
number.on_change('value', lambda attr, old, new: update_denstype())
color_map.on_change('value',
lambda attr, old, new: update_denstype(cmap=True))
protein.on_change('value', lambda attr, old, new: update_protein())
denstype.on_change('value', lambda attr, old, new: update_denstype())
cbar_range.on_change('value', lambda attr, old, new: update_cbar())
# deal with what gets updated and what not.
def update_denstype(cond=False, cmap=False):
update_all(cond, cmap)
update_protein(cond)
update_denstype()
input1 = row([gpcr, lipid, color_map, protein])
input2 = row([p1, cbar_range])
input3 = row([number, zrange])
input3 = column([input1, input2, input3])
l = layout([input3])
doc.add_root(l)
doc.title = "Density App"
def __init__(self, thelenota, figsize=(800, 600)):
# DIMRED widgets
self.thelenota = thelenota
self.figsize = figsize
self.counts = None
#widgets
self.w = DUMMY()
current_cluster_labels = None
wstyle = widgets.Layout(width='200px') #, max_width='120px',
# min_width='120px')
self.w.drfilter_lower_perc_switch = ccwidget(
self.thelenota,
'drfilter_lower_perc_switch',
'bool',
self.dr_name_simple,
False,
lwidth='40px')
self.w.drfilter_log_switch = ccwidget(self.thelenota,
'drfilter_log_switch',
'bool',
self.dr_name_simple,
False,
lwidth='40px')
self.w.drfilter_lower_percentage = ccwidget(
self.thelenota,
'drfilter_lower_percentage',
'float',
self.dr_name_simple,
0.2,
min=0.01,
max=1,
value=0.2)
self.w.drmethod = ccwidget(
self.thelenota,
'dimred_method',
'dropdown',
self.dr_name_simple,
'tsne',
options='pca ica scorpius KernelPCA tsne nmf'.split())
self.w.drperplex = cache_widget_value(widgets.IntSlider(value=67,
min=2,
max=99,
step=5),
67,
self.thelenota,
'dimred_perplexity',
self.dr_name_simple,
fmt='int')
self.w.drangle = cache_widget_value(widgets.FloatSlider(value=0.5,
min=0.05,
max=0.95,
step=0.05),
0.5,
self.thelenota,
'dimred_angle',
self.dr_name_simple,
fmt='float')
self.w.drlrate = cache_widget_value(widgets.IntSlider(value=920,
min=20,
max=10000,
step=50),
920,
self.thelenota,
'dimred_learning_rate',
self.dr_name_simple,
fmt='int')
self.w.drearly = cache_widget_value(widgets.FloatSlider(value=3.5,
min=1,
max=20,
step=0.5),
3.5,
self.thelenota,
'dimred_early_exag',
self.dr_name_simple,
fmt='float')
self.w.dr_tsne_pcavar_cutoff = ccwidget(self.thelenota,
'tsne_pca_var_cutoff',
'float',
self.dr_name_simple,
0.05,
min=0.01,
max=0.2,
step=0.01)
self.w.drrun = widgets.Button(description='GO!')
self.w.drforce = widgets.Checkbox(description='force',
layout=widgets.Layout(width='300px'))
@dcache(self.thelenota, 'dimred_correlating_genes', 'tsv',
self.dr_name)
def get_dr_correlating(*_):
stats, trans = run_dr_2()
c1 = self.thelenota.counttable.apply(
lambda x: pearsonr(x, trans.iloc[:, 0])[0], axis=1)
c2 = self.thelenota.counttable.apply(
lambda x: pearsonr(x, trans.iloc[:, 1])[0], axis=1)
return pd.DataFrame({0: c1, 1: c2})
def get_top_correlating(*_):
d = get_dr_correlating().abs().sum(1).sort_values(ascending=False)
d = d.head(40)
d = d.reset_index()
d = d.apply(lambda x: '{} ({:.3g})'.format(x.iloc[0], x.iloc[1]),
axis=1)
return list(d)
tcolormap = cmapper.CMAPPER(self.thelenota,
extra_intrinsic_methods={
"top DR correlate":
(get_top_correlating,
cmapper.intrinsic_gene_score)
})
self.w.sl_group_name = cache_widget_value(widgets.Text(layout=wstyle),
'self.thelenota',
self.thelenota,
'group_define_name',
self.dr_name_simple)
self.w.sl_group_set = widgets.Text(layout=wstyle)
self.w.sl_group_set_go = widgets.Button(description='set',
layout=wstyle)
self.w.sl_groupextractname = widgets.Text(layout=wstyle)
self.w.sl_group_extract_go = widgets.Button(description='extract')
self.w.clu_method = ccwidget(self.thelenota,
'cluster_method',
'dropdown',
self.dr_name_simple,
'dbscan',
options=['dbscan'])
self.w.clu_dbscan_eps = ccwidget(self.thelenota,
'clu_dbscan_eps_w',
'float',
self.dr_name_simple,
2.5,
min=0.1,
max=10.0,
step=0.1)
self.w.clu_go = widgets.Button(description='Cluster!')
self.w.clu_name = ccwidget(self.thelenota, "cluster_name", "text",
self.dr_name_simple, 'cluster')
self.w.clu_store_go = widgets.Button(description='save')
plotinfo = {}
self.w.html = widgets.HTML()
# data!
samples = self.thelenota.counttable.columns
nosamples = len(samples)
color_mapper = LinearColorMapper(palette="Inferno256",
low=-0.3,
high=2.5)
topgene = self.thelenota.counttable.std(1).sort_values().tail(
1).index[0]
colv = list(self.thelenota.counttable.loc[topgene])
pdata = ColumnDataSource(
dict(x=[random.uniform(-10, 10) for x in range(nosamples)],
y=[random.uniform(-10, 10) for x in range(nosamples)],
desc=list(samples),
size=[0.3] * nosamples,
score=colv))
self.thelenota._pdata = pdata
# Clustering
def run_clustering(*args):
method = self.w.clu_method.value
stats, trans = run_dr_2()
if method == 'dbscan':
labels = run_dbscan(trans, self.w.clu_dbscan_eps.value)
else:
lg.warning('Not implemented cluster method: {}'.format(method))
return
self.thelenota._CLUSTER_LABELS[self.dr_name()] = labels
colv = labels
color_mapper = CategoricalColorMapper(
palette=bokeh.palettes.Category20[20],
factors=list(colv.value_counts().index))
colorbar_mapper = LinearColorMapper(palette='Inferno256',
low=0,
high=0)
bcolorbar.color_mapper = colorbar_mapper
if not bfigure.legend[0].items:
bfigure.legend[0].items.append(blegend)
bplot.data_source.data['score'] = colv
bplot.glyph.fill_color['transform'] = color_mapper
bplot.glyph.line_width = 0
bplot.glyph.line_alpha = 0
bokeh_io.push_notebook(handle=bhandle)
def store_current_cluster(*args):
labels = self.thelenota._CLUSTER_LABELS.get(self.dr_name())
if labels is None:
self.w.html.value = "no cluster defined"
return
cname = self.w.clu_name.value
labels = labels.apply(lambda x: '{}_{}'.format(cname, x))
outfile = self.thelenota.metadata_dir / '{}.tsv'.format(cname)
moutfile = self.thelenota.metadata_dir / '{}.meta.tsv'.format(
cname)
tmeta = pd.DataFrame({cname: labels})
tmeta.to_csv(outfile, sep="\t")
with open(moutfile, 'w') as F:
F.write("{}\tcategorical\n".format(cname))
self.w.html.value = 'saved cluster to {}'.format(outfile)
self.w.clu_store_go.on_click(store_current_cluster)
self.w.clu_go.on_click(run_clustering)
# GROUP SET
def define_group(*args):
groupset = self.w.sl_group_name.value
groupname = self.w.sl_group_set.value
self.thelenota.selected = list(
self.thelenota.counttable.columns.to_series()\
.iloc[list(self.thelenota._DR_INDICI_SEL_)])
if groupset in self.thelenota.metadata_info.index:
tmeta = self.thelenota.get_metadata(groupset)
else:
tmeta = pd.Series('na',
index=self.thelenota.counttable.columns)
tmeta.loc[self.thelenota.selected] = groupname
self.thelenota.metadata_dir.makedirs_p()
outfile = self.thelenota.metadata_dir / '{}.tsv'.format(groupset)
moutfile = self.thelenota.metadata_dir / '{}.meta.tsv'.format(
groupset)
tmeta = pd.DataFrame({groupset: tmeta})
tmeta.to_csv(outfile, sep="\t")
with open(moutfile, 'w') as F:
F.write("{}\tcategorical\n".format(groupset))
run_dr_color()
self.w.sl_group_set_go.on_click(define_group)
# GROUP EXTRACT
#sl_groupextractname_w
def extract_group(*args):
groupname = self.w.sl_groupextractname.value
self.thelenota.selected = list(
self.thelenota.counttable.columns.to_series()\
.iloc[list(self.thelenota._DR_INDICI_SEL_)])
outfile = self.thelenota.counttable_file.dirname() \
/ '{}__{}.tsv'.format(self.thelenota.counttable_name, groupname)
newcounts = self.thelenota.counttable[self.thelenota.selected]
newcounts.to_csv(outfile, sep="\t")
self.w.html.value = "save new count table to {}".format(outfile)
self.w.sl_group_extract_go.on_click(extract_group)
def force_refresh():
"Force to refresh calculations? Or can we use the cache??"
return self.w.drforce.value
@dcache(self.thelenota, 'filtered', 'pickle', self.dr_name_filter,
force_refresh)
def filter_counts(*_):
counts = self.thelenota.counttable
if self.w.drfilter_log_switch.value:
minval = 0.5 * counts[counts > 0].min().min()
lg.warning('log tranform log10(x+{:.4g})'.format(minval))
counts = np.log10(counts + minval)
if self.w.drfilter_lower_perc_switch.value:
perc = self.w.drfilter_lower_percentage.value
csum = counts.sum(1)
counts = counts[csum >= csum.quantile(perc)]
return counts
@dcache(self.thelenota, 'dimred_table', 'mtsv', self.dr_name,
force_refresh)
def run_dr_2(*_):
param = self.get_dr_param()
method = param['method']
del param['method']
if method == 'pca':
stats, trans = qrtask.pca.sync(self.counts)
elif method == 'ica':
stats, trans = rqtask.ica.sync(self.counts)
elif method == 'KernelPCA':
stats, trans = rqtask.kernelPCA.sync(self.counts)
elif method == 'scorpius':
stats, trans = rqtask.scorpius_dr.sync(self.counts)
elif method == 'nmf':
stats, trans = rqtask.nmf.sync(self.counts)
elif method == 'tsne':
stats, trans = rqtask.tsne.sync(self.counts,
self.get_dr_param())
else:
raise NotImplemented
return stats, trans
def set_color_scale():
score = tcolormap.score
method = tcolormap.method
self.warn('set color scale {}/{}'.format(method, tcolormap.value))
color_mapper = tcolormap.colormapper
bplot.glyph.fill_color['transform'] = color_mapper
bplot.data_source.data['score'] = score
if not tcolormap.discrete:
bcolorbar.color_mapper = color_mapper
if tcolormap.discrete:
if not bfigure.legend[0].items:
bfigure.legend[0].items.append(blegend)
else:
if bfigure.legend[0].items:
bfigure.legend[0].items.pop()
bplot.glyph.line_width = 0
bplot.glyph.line_alpha = 0
def _create_title():
cmethod = '{}/{}'.format(tcolormap.method, tcolormap.value)
if self.w.drfilter_lower_perc_switch.value:
cmethod += '/low%{}'.format(
self.w.drfilter_lower_percentage.value)
if self.w.drfilter_log_switch.value:
cmethod += '/log'
cmethod += '/{}/{}'.format(*self.counts.shape)
return '{}/{}'.format(self.thelenota.counttable_name, cmethod)
def _create_scatter_plot(only_color=False):
if (self.w.drfilter_lower_perc_switch.value) or (
self.w.drfilter_log_switch.value):
self.counts = filter_counts()
else:
self.counts = self.thelenota.counttable
self.warn("count table: {}".format(str(self.counts.shape)))
if not only_color:
meta, trans = run_dr_2()
self.thelenota.dr = trans
self.thelenota._dr_meta = meta
self.thelenota._dr_trans = trans
col1, col2 = trans.columns[:2]
d1 = trans[col1]
d2 = trans[col2]
title = self.dr_name().replace('_', ' ')
m = max(d2.max() - d2.min(), d1.max() - d1.min())
bplot.data_source.data['size'] = [0.008 * m] * nosamples
bplot.data_source.data['x'] = d1
bplot.data_source.data['y'] = d2
bfigure.title.text = _create_title()
set_color_scale()
bokeh_io.push_notebook(handle=bhandle)
def run_dr_color(*_):
""" refresh dr scatter plot - color only """
self.w.drrun.button_style = 'info'
try:
_create_scatter_plot(only_color=True)
except:
self.w.drrun.button_style = 'danger'
raise
self.w.drrun.button_style = ''
def run_dr(*_):
self.w.drrun.button_style = 'info'
try:
_create_scatter_plot()
except:
self.w.drrun.button_style = 'danger'
raise
self.w.drrun.button_style = ''
tcolormap.on_change = run_dr_color
self.w.drrun.on_click(run_dr)
# clgenesetchoice_w.observe(run_dr_color, 'value')
# clmetadata_w.observe(run_dr_color, 'value')
# set up bokeh
select_callback = CustomJS(args={'dsource': pdata},
code="""
var indici = dsource.selected['1d'].indices;
console.log(indici);
IPython.notebook.kernel.execute(
'T._DR_INDICI_SEL_ = ' + indici);
""")
bokeh_tools = [
bmodels.HoverTool(tooltips=[
("(x,y)", "($x, $y)"),
("score", "$score"),
("desc", "@desc"),
]),
bmodels.BoxSelectTool(callback=select_callback),
bmodels.PanTool(),
bmodels.WheelZoomTool(),
bmodels.BoxZoomTool(),
bmodels.LassoSelectTool(callback=select_callback),
bmodels.SaveTool(),
bmodels.ResetTool(),
bmodels.HelpTool(),
]
bfigure = bokeh_figure(plot_width=figsize[0],
plot_height=figsize[1],
tools=bokeh_tools,
toolbar_sticky=False,
toolbar_location='left',
title='dimredplot')
bfigure.title.text_color = 'darkgrey'
bfigure.title.text_font_style = 'normal'
bfigure.title.text_font_size = "12px"
self.thelenota._bfigure = bfigure
bplot = bfigure.circle(x='x',
y='y',
radius='size',
source=pdata,
legend='score',
color=dict(field='score',
transform=color_mapper))
self.thelenota._bplot = bplot
bcolorbar = ColorBar(color_mapper=tcolormap.colormapper,
ticker=BasicTicker(),
formatter=BasicTickFormatter(precision=1),
label_standoff=10,
border_line_color=None,
location=(0, 0))
bfigure.add_layout(bcolorbar, 'right')
blegend = bfigure.legend[0].items[0]
bhandle = bokeh_io.show(bfigure, notebook_handle=True)
tab_children = []
tab_children.append(
widgets.VBox([
ilabel('filter sum%', self.w.drfilter_lower_perc_switch,
self.w.drfilter_lower_percentage),
ilabel('log tranform', self.w.drfilter_log_switch),
ilabel('method', self.w.drmethod),
ilabel('perplexity', self.w.drperplex),
ilabel('angle', self.w.drangle),
ilabel('learning rate', self.w.drlrate),
ilabel('early exagg.', self.w.drearly),
ilabel('PCA var. cutoff', self.w.dr_tsne_pcavar_cutoff),
widgets.HBox([self.w.drrun, self.w.drforce]),
]))
tab_children.append(widgets.VBox([tcolormap.prepare_display()]))
tab_children.append(
widgets.VBox([
ilabel('method', self.w.clu_method),
ilabel('dbscan:eps', self.w.clu_dbscan_eps),
ilabel('store', self.w.clu_name, self.w.clu_store_go),
self.w.clu_go
]))
tab_children.append(
widgets.VBox([
ilabel('group define', self.w.sl_group_name,
self.w.sl_group_set, self.w.sl_group_set_go),
ilabel('count extract', self.w.sl_groupextractname,
self.w.sl_group_extract_go),
]))
tabs = widgets.Tab(children=tab_children)
tabs.set_title(0, 'DimRed')
tabs.set_title(1, 'Color')
tabs.set_title(2, 'Cluster')
tabs.set_title(3, 'Select')
tabs.selected_index = 0
display(tabs)
display(self.w.html)
#run a few on_change functions so that all is in sync
#on_colormethod_change()
run_dr()
tools=TOOLS,
x_axis_type="datetime",
title="TCP sequence values in Honeypot {}".format(
title))
hoverseq = p_seq.select(dict(type=HoverTool))
hoverseq.tooltips = [("index", "$index"),
("timestamp", "@x{%F %H:%M:%S}"),
("number", "@y{0,0}"),
("dest port", "@dport"),
("src port", "@sport")]
hoverseq.formatters = {'x': 'datetime'}
seq_sourceplot = ColumnDataSource(data=dict(
x=x, y=y, dport=z_d, sport=z_s, colorseq=colorseq))
p_seq.xaxis.axis_label = "Time"
p_seq.yaxis.axis_label = "Sequence Numbers"
p_seq.yaxis[0].formatter = BasicTickFormatter(
use_scientific=False)
p_seq.scatter(x='x',
y='y',
color='colorseq',
legend="seq values",
alpha=0.5,
source=seq_sourceplot)
p_ack = figure(
width=1500,
height=700,
tools=TOOLS,
x_axis_type="datetime",
title="TCP acknowledgement values in Honeypot {}".
format(title))
hoverack = p_ack.select(dict(type=HoverTool))
hoverack.tooltips = [("index", "$index"),
def plot_frame(frame,
cols=None,
scale='linear',
trace_coeffs=None,
saturation=0.8,
title=None,
wavecal=None):
"""
Plot a SOSS frame
Parameters
----------
frame: sequence
The 2D frame to plot
cols: int, list, tuple
The 1D column(s) to plot
scale: str
Plot scale, ['linear', 'log']
trace_coeffs: sequence
Plot the traces for the given coefficients
saturation: float
The full-well fraction to be considered saturated, (0, 1)
title: str
A title for the plot
wavecal: np.ndarray
A wavelength calibration map for each pixel
"""
# Determine subarray
nrows, ncols = frame.shape
# Get data, snr, and saturation for plotting
dat = frame
snr = np.sqrt(frame)
fullWell = 65536.0
sat = dat > saturation * fullWell
sat = sat.astype(int)
dh, dw = dat.shape
# Fix if log scale
if scale == 'log':
dat[dat < 1.] = 1.
# Set the source data
source = dict(data=[dat], snr=[snr], saturation=[sat])
# Set the tooltips
tooltips = [("(x,y)", "($x{int}, $y{int})"), ("ADU/s", "@data"),
("SNR", "@snr"), ('Saturation', '@saturation')]
# Add wavelength calibration if possible
if isinstance(wavecal, np.ndarray):
if wavecal.shape == frame.shape:
source['wave1'] = [wavecal]
tooltips.append(("Wavelength", "@wave1"))
if wavecal.ndim == 3 and wavecal.shape[0] == 3:
source['wave1'] = [wavecal[0]]
source['wave2'] = [wavecal[1]]
source['wave3'] = [wavecal[2]]
tooltips.append(("Wave 1", "@wave1"))
tooltips.append(("Wave 2", "@wave2"))
tooltips.append(("Wave 3", "@wave3"))
# Set shared plot params
x_range = (0, dat.shape[1])
y_range = (0, dat.shape[0])
height = int(nrows / 2.) + 160
toolbar = 'above'
# Draw the figures
tabs = []
for pname, ptype in zip([
'Counts', 'SNR', 'Saturation ({}% Full Well)'.format(
saturation * 100)
], ['data', 'snr', 'saturation']):
# Make the figure
fig_title = '{} - {}'.format(title, pname)
fig = figure(x_range=x_range,
y_range=y_range,
tooltips=tooltips,
width=1024,
height=height,
title=fig_title,
toolbar_location=toolbar,
toolbar_sticky=True)
# Get the data
vals = source[ptype][0]
# Saturation plot is different
if ptype == 'saturation':
vmin = 0
vmax = 1
formatter = FuncTickFormatter(
code="""return {0: 'Unsaturated', 1: 'Saturated'}[tick]""")
color_map = ['#404387', '#FDE724']
ticker = FixedTicker(ticks=[vmin, vmax])
# Counts and SNR are similar plots
else:
vmin = int(np.nanmin(vals[vals >= 0]))
vmax = int(np.nanmax(vals[vals < np.inf]))
formatter = BasicTickFormatter()
color_map = 'Viridis256'
ticker = BasicTicker()
# Set the plot scale
if scale == 'log':
mapper = LogColorMapper(palette=color_map, low=vmin, high=vmax)
else:
mapper = LinearColorMapper(palette=color_map, low=vmin, high=vmax)
# Plot the frame
fig.image(source=source,
image=ptype,
x=0,
y=0,
dw=dw,
dh=dh,
color_mapper=mapper)
color_bar = ColorBar(color_mapper=mapper,
ticker=ticker,
formatter=formatter,
orientation="horizontal",
location=(0, 0))
# Plot the trace polynomials
if trace_coeffs is not None:
X = np.linspace(0, 2048, 2048)
# Check number of traces
if np.array(trace_coeffs).ndim == 1:
trace_coeffs = [trace_coeffs]
for coeffs in trace_coeffs:
Y = np.polyval(coeffs, X)
fig.line(X, Y, color='red', line_dash='dashed')
# Add the colorbar
fig.add_layout(color_bar, 'below')
# Plot the column data
col_fig = None
col_colors = ['blue', 'green', 'purple', 'cyan', 'orange']
if cols is not None:
# Make sure it's iterable
if isinstance(cols, int):
cols = [cols]
# Initialize column figure
col_fig = figure(width=1024,
height=300,
toolbar_location=toolbar,
toolbar_sticky=True)
for n, col in enumerate(cols):
col_color = col_colors[n]
col_fig.step(np.arange(256),
vals[:, col],
color=col_color,
legend='Col {}'.format(col))
col_fig.y_range = Range1d(vmin * 0.9, vmax * 1.1)
col_fig.x_range = Range1d(*y_range)
# Add line to image
fig.line([col, col], [0, 256], color=col_color, line_width=3)
# Update click policy
col_fig.legend.click_policy = 'hide'
if col_fig is not None:
# Add the figure to the tab list
tabs.append(Panel(child=column([fig, col_fig]), title=pname))
else:
# No column object
tabs.append(Panel(child=fig, title=pname))
# Make the final tabbed figure
final = Tabs(tabs=tabs)
return final
def plot_frames(data,
idx=0,
col=0,
scale='linear',
trace_coeffs=None,
saturation=0.8,
width=1000,
title=None,
wavecal=None):
"""
Plot a SOSS frame
Parameters
----------
data: sequence
The 3D data to plot
scale: str
Plot scale, ['linear', 'log']
trace_coeffs: sequence
Plot the traces for the given coefficients
saturation: float
The full-well fraction to be considered saturated, (0, 1)
title: str
A title for the plot
wavecal: np.ndarray
A wavelength calibration map for each pixel
"""
# Determine subarray
nframes, nrows, ncols = data.shape
# Remove infs
data[data == np.inf] = np.nan
# Get data, snr, and saturation for plotting
dat = data
snr = np.sqrt(data)
fullWell = 65536.0
sat = dat > saturation * fullWell
sat = sat.astype(int)
# Fix log scale plot values
if scale == 'log':
dat[dat < 1.] = 1.
snr[snr < 1.] = 1.
# Broadcast the data
frames = np.arange(nframes)
columns = np.arange(ncols)
rows = np.arange(nrows)
verticals = np.tile(np.arange(ncols), (nrows, 1)).T
# Wrap the data in ColumnDataSources
source_available = ColumnDataSource(data=dict(
**{'counts{}'.format(n): dat[n]
for n in frames}, **{'snr{}'.format(n): snr[n]
for n in frames}, **
{'saturation{}'.format(n): sat[n]
for n in frames}))
source_visible = ColumnDataSource(
data=dict(counts=[dat[idx]], snr=[snr[idx]], saturation=[sat[idx]]))
vertical_available = ColumnDataSource(data=dict(
**{'vertical{}'.format(n): vert
for n, vert in enumerate(verticals)}))
vertical_visible = ColumnDataSource(
data=dict(column=rows, vertical=verticals[col]))
col_visible = ColumnDataSource(data=dict(columns=rows,
counts=dat[0, :, col],
snr=snr[0, :, col],
saturation=sat[0, :, col]))
col_dict = {}
for fnum in frames:
for cnum in columns:
for datacube, pname in zip([dat, snr, sat],
['counts', 'snr', 'saturation']):
col_dict['{}{}_{}'.format(pname, cnum,
fnum)] = datacube[fnum, :, cnum]
col_available = ColumnDataSource(data=col_dict)
# Set the tooltips
tooltips = [("(x,y)", "($x{int}, $y{int})"), ("ADU/s", "@counts"),
("SNR", "@snr"), ('Saturation', '@saturation')]
col_color = 'blue'
# Add wavelength calibration if possible
if isinstance(wavecal, np.ndarray):
if wavecal.shape == dat[idx].shape:
source_visible.data['wave1'] = [wavecal]
tooltips.append(("Wavelength", "@wave1"))
if wavecal.ndim == 3 and wavecal.shape[0] == 3:
source_visible.data['wave1'] = [wavecal[0]]
source_visible.data['wave2'] = [wavecal[1]]
source_visible.data['wave3'] = [wavecal[2]]
tooltips.append(("Wave 1", "@wave1"))
tooltips.append(("Wave 2", "@wave2"))
tooltips.append(("Wave 3", "@wave3"))
# Set shared plot params
x_range = (0, ncols)
y_range = (0, nrows)
height = int(nrows / 2.) + 160
toolbar = 'right'
# Draw the figures
tabs = []
for pdata, pname, ptype, ylabel in zip(
[dat, snr, sat], [
'Counts', 'SNR', 'Saturation ({}% Full Well)'.format(
saturation * 100)
], ['counts', 'snr', 'saturation'],
['Count Rate [ADU/s]', 'SNR', 'Saturated']):
# Make the figure
fig_title = '{} - {}'.format(title, pname)
# Get min and max
vmin = np.nanmin(pdata)
vmax = np.nanmax(pdata)
# Saturation plot is different
if ptype == 'saturation':
formatter = FuncTickFormatter(
code="""return {0: 'Unsaturated', 1: 'Saturated'}[tick]""")
color_map = ['#404387', '#FDE724']
ticker = FixedTicker(ticks=[vmin, vmax])
# Counts and SNR are similar plots
else:
formatter = BasicTickFormatter()
color_map = 'Viridis256'
ticker = BasicTicker()
# Set the plot scale
if scale == 'log':
mapper = LogColorMapper(palette=color_map, low=vmin, high=vmax)
else:
mapper = LinearColorMapper(palette=color_map, low=vmin, high=vmax)
# Plot the image data
fig = figure(x_range=x_range,
y_range=y_range,
tooltips=tooltips,
width=width,
height=height,
title=fig_title,
toolbar_location=toolbar,
toolbar_sticky=True)
fig.image(source=source_visible,
image=ptype,
x=0,
y=0,
dw=ncols,
dh=nrows,
color_mapper=mapper)
# Plot the line indicating the column
fig.line('vertical',
'column',
source=vertical_visible,
color=col_color,
line_width=3)
# Add the colorbar
color_bar = ColorBar(color_mapper=mapper,
ticker=ticker,
formatter=formatter,
orientation="horizontal",
location=(0, 0))
fig.add_layout(color_bar, 'below')
# Plot the column data
col_fig = figure(width=width,
height=300,
toolbar_location=toolbar,
toolbar_sticky=True)
col_fig.step('columns', ptype, source=col_visible, color=col_color)
col_fig.xaxis.axis_label = 'Row'
col_fig.yaxis.axis_label = ylabel
col_fig.y_range = Range1d(vmin * 0.9, vmax * 1.1)
col_fig.x_range = Range1d(*y_range)
# Plot the trace polynomials
if trace_coeffs is not None:
for coeffs in trace_coeffs:
Y = np.polyval(coeffs, columns)
fig.line(columns, Y, color='red')
# Add the figure to the tab list
tabs.append(Panel(child=column(fig, col_fig), title=pname))
# Make the final tabbed figure
final = Tabs(tabs=tabs)
# Write JS code
code = """
var vis = visible.data;
var avail = available.data;
var frame = fr_slide.value.toString(10);
var viscol = col_vis.data;
var availcol = col_avail.data;
var col = col_slide.value.toString(10);
var visvert = vert_vis.data;
var availvert = vert_avail.data;
vis['counts'] = [avail['counts'.concat(frame)]];
vis['snr'] = [avail['snr'.concat(frame)]];
vis['saturation'] = [avail['saturation'.concat(frame)]];
viscol['counts'] = availcol['counts'.concat(col, '_', frame)];
viscol['snr'] = availcol['snr'.concat(col, '_', frame)];
viscol['saturation'] = availcol['saturation'.concat(col, '_', frame)];
visvert['vertical'] = availvert['vertical'.concat(col)];
visible.change.emit();
col_vis.change.emit();
vert_vis.change.emit();
"""
# Make the column slider
column_slider = Slider(title='Column',
value=col,
start=0,
end=ncols - 1,
step=1)
# Make the frame slider
if nframes - 1 > 0:
frame_slider = Slider(title='Frame',
value=idx,
start=0,
end=nframes - 1,
step=1)
else:
frame_slider = None
code = code.replace('fr_slide.value.toString(10);', '0')
# CustomJS callback to update the three plots on slider changes
callback = CustomJS(args=dict(visible=source_visible,
available=source_available,
col_vis=col_visible,
col_avail=col_available,
vert_vis=vertical_visible,
vert_avail=vertical_available,
fr_slide=frame_slider,
col_slide=column_slider),
code=code)
# Add callback to column slider
column_slider.js_on_change('value', callback)
# Add callback to frame slider
if frame_slider is not None:
frame_slider.js_on_change('value', callback)
return column(final, frame_slider, column_slider)
else:
return column(final, column_slider)
plot_raw.toolbar_location = None plot_raw.x_range = DataRange1d(0.1,0.3) plot_raw.x_range.follow = 'end' plot_raw.x_range.follow_interval = 100 plot_raw.x_range.range_padding = 0.1 plot_raw.xaxis.axis_label = "Voltage (Volts)" plot_raw.xaxis.axis_label_text_font_size = '12pt' plot_raw.xaxis.axis_label_text_font_style = "bold" plot_raw.xaxis.axis_line_width = 2 plot_raw.xaxis.major_label_text_font_size = "10pt" plot_raw.yaxis.axis_label = "Current (Amperes)" plot_raw.yaxis.axis_label_text_font_size = '12pt' plot_raw.yaxis.axis_label_text_font_style = "bold" plot_raw.yaxis.axis_line_width = 2 plot_raw.yaxis.major_label_text_font_size = "10pt" plot_raw.yaxis.formatter = BasicTickFormatter(precision=1, use_scientific=True) plot_raw.grid.grid_line_dash = 'dashed' plot_raw.title.align = 'center' plot_raw.title.text_font = 'arial' plot_raw.title.text_font_style = 'bold' plot_raw.title.text_font_size = '15px' #plot_raw.add_layout(LinearAxis(axis_label="LayoutRight"), place='right') plot_raw.axis.major_tick_line_width = 1.5 plot_raw.axis.major_tick_out = 6 plot_raw.axis.major_tick_in = 0 #FIXME The reason why circle is plotted instead of line, is that #information bits are getting "delayed". Solve this. #plot_raw.line(source=source, x='time', y='raw_data', alpha=0.2, # line_width=3, color='navy')
def process_isn(src_dir, source, output):
braces = "{}"
date = src_dir.split('/')[-4:-1]
# Tshark command part of the global command used
tshark = "parallel --gnu --line-buffer tshark -n -q -Tfields -e frame.time_epoch -e tcp.seq \
-e tcp.ack -e tcp.dstport -E separator=/s -o tcp.relative_sequence_numbers:FALSE -r {}".format(
braces)
TOOLS = "hover,crosshair,pan,wheel_zoom,zoom_in,zoom_out,box_zoom,undo,redo,reset,tap,save,box_select,poly_select,lasso_select,"
# For each hour of the day
for hours in range(0, 24):
# For each 10 minutes of the hour
for minutes in range(0, 6, 1):
h = format(hours, '02d')
w_input = []
x_input = []
y_input = []
z_input = []
# Beginning of the command, used to find and select the files corresponding to the parameters,
# to put as input files for the tshark command
cmd = "find {} -type f -print | sort | grep {}-{}{}{}{}{} | {}".format(
src_dir, source, date[0], date[1], date[2], h, minutes, tshark)
proc = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
lines = proc.stdout.readlines()
if not lines:
continue
for line in lines:
line = line[:-1].decode()
timestamp, iseq, iack, dport = line.split(' ')
a, _ = timestamp.split('.')
dobj = datetime.datetime.fromtimestamp(float(a))
stime = dobj.strftime("%Y-%m-%d %H:%M:%S")
x_input.append(stime)
y_input.append(iseq)
w_input.append(iack)
if dport == '':
dport = 0
z_input.append(int(dport))
proc.wait()
x = np.array(x_input, dtype=np.datetime64)
z = np.array(z_input)
y = np.array(y_input)
w = np.array(w_input)
colors = [
"#%02x%02x%02x" % (int(r), int(g), 150)
for r, g in zip(50 + z * 2, 30 + z * 2)
]
title = "{} collected on {}/{}/{} at {}".format(
source, date[0], date[1], date[2], h)
# Definition of the sequence numbers plot
p_seq = figure(
width=1500,
height=700,
tools=TOOLS,
x_axis_type="datetime",
title="TCP sequence values in Honeypot {}".format(title))
p_seq.xaxis.axis_label = "Time"
p_seq.yaxis[0].formatter = BasicTickFormatter(use_scientific=False)
p_seq.scatter(
x,
y,
color=colors,
legend="seq values",
alpha=0.5,
)
hoverseq = p_seq.select(dict(type=HoverTool))
hoverseq.tooltips = [("index", "$index"), ("timestamp", "@x"),
("number", "@y{0,0}")]
# Definition of the acknowledgement numbers plot
p_ack = figure(
width=1500,
height=700,
tools=TOOLS,
x_axis_type="datetime",
title="TCP acknowledgement values in Honeypot {}".format(
title))
p_ack.xaxis.axis_label = "Time"
p_ack.yaxis[0].formatter = BasicTickFormatter(use_scientific=False)
p_ack.scatter(
x,
w,
color=colors,
legend="ack values",
alpha=0.5,
)
hoverack = p_ack.select(dict(type=HoverTool))
hoverack.tooltips = [("index", "$index"), ("timestamp", "@x"),
("number", "@y{0,0}")]
output_name = "color_scatter_{}_{}{}{}_{}:{}0_syn+ack".format(
source, date[0], date[1], date[2], h, minutes)
output_dir = "{}{}/{}/{}/{}".format(output, date[0], date[1],
date[2], h)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
output_file("{}/{}.html".format(output_dir, output_name),
title="TCP ISN values in Honeypot",
mode='inline')
p = column(p_seq, p_ack)
save(p)
print("{}:{}0".format(h, minutes))
export_png(p, filename="{}/{}.png".format(output_dir, output_name))
'planets': solsys_planets,
'radii': solsys_radii,
'periods': 365 * np.array(solsys_periods),
'circlesize': 2 + np.array(solsys_radii)
}
source_solsys = ColumnDataSource(data=solsys_data)
plot = figure(x_range=PERIOD_RANGE,
y_range=RADIUS_RANGE,
x_axis_type="log",
y_axis_type="log",
plot_width=600,
plot_height=450)
plot.xaxis.ticker = FixedTicker(ticks=[0.1, 1, 10, 100, 365, 2000])
plot.xaxis.formatter = BasicTickFormatter()
plot.xaxis.axis_label = "Orbital period (days)"
plot.yaxis.ticker = FixedTicker(ticks=[0.3, 0.5, 1, 2, 4, 10, 30])
plot.yaxis.formatter = BasicTickFormatter()
plot.yaxis.axis_label = "Planet size (relative to Earth)"
color_mapper = CategoricalColorMapper(
palette=['#3498db', '#2ecc71', '#f1c40f', '#e74c3c'],
factors=['Earth-size', 'Super-Earth-size', 'Neptune-size', 'Jupiter-size'])
circles_gray = plot.circle(
x='fpl_orbper',
y='fpl_rade',
size='circlesize',
source=source_gray,
p = figure(plot_height=600,
plot_width=700,
x_axis_label='year',
y_axis_label='$')
p.tools.append(hover)
# add a line renderer with legend and line thickness
p.line('x', 'y', source=source, legend="daily compound", line_width=2)
p.line('x',
'y2',
source=source,
legend="no compound",
line_color='green',
line_width=2)
p.yaxis.formatter = BasicTickFormatter(use_scientific=False)
p.legend.location = "top_left"
# +
def year_to_date(yr):
return yr * 365
def update(principal, addon, r):
compound = principal * (1 + r)
return compound + addon
def compute_total_value_lost(hour_wasted_per_day, hourly_output, max_yr,