def create_figure():
xs, ys, colors, sizes = model.get_axes_values()
fig_args = dict(tools='pan', plot_height=600, plot_width=800)
if model.x_field in model.discrete_column_names and model.y_field in model.discrete_column_names:
figure = Figure(x_range=xs, y_range=ys, **fig_args)
figure.axis.major_label_orientation = math.pi / 4
elif model.x_field in model.discrete_column_names:
figure = Figure(x_range=xs, **fig_args)
figure.xaxis.major_label_orientation = math.pi / 4
elif model.y_field in model.discrete_column_names:
figure = Figure(y_range=ys, **fig_args)
figure.yaxis.major_label_orientation = math.pi / 4
else:
figure = Figure(**fig_args)
figure.circle(x=xs, y=ys, color=colors, size=sizes, line_color="white", alpha=0.8)
figure.toolbar_location = None
figure.xaxis.axis_label = model.x_field
figure.yaxis.axis_label = model.y_field
figure.background_fill_color = model.background_fill
figure.border_fill_color = model.background_fill
figure.axis.axis_line_color = "white"
figure.axis.axis_label_text_color = "white"
figure.axis.major_label_text_color = "white"
figure.axis.major_tick_line_color = "white"
figure.axis.minor_tick_line_color = "white"
figure.axis.minor_tick_line_color = "white"
figure.grid.grid_line_dash = [6, 4]
figure.grid.grid_line_alpha = .3
return figure
def _create_title_fig(self):
fig = Figure(sizing_mode='scale_width',
x_range=[0, 1], y_range=[0, 1], plot_height=50, tools="")
# Remove existing axes and grid
fig.xgrid.grid_line_color = None
fig.ygrid.grid_line_color = None
fig.axis.visible = False
fig.background_fill_color = None
fig.toolbar_location = None
text = Label(x=0.5, y=0.5,
text=self._title_text, render_mode='css',
background_fill_color='white', background_fill_alpha=1.0,
text_font_size='28pt', text_align='center', text_baseline='middle'
)
fig.add_layout(text)
return (fig)
def apply_default_style(p: Figure) -> None:
p.background_fill_color = BACKGROUND_COLOR
p.grid.grid_line_color = "white"
p.toolbar.logo = None
p.xaxis.axis_label_text_font_size = FONT_SIZE
p.yaxis.axis_label_text_font_size = FONT_SIZE
p.axis.axis_line_color = None
p.xaxis.major_tick_line_color = None # turn off x-axis major ticks
p.xaxis.minor_tick_line_color = None # turn off x-axis minor ticks
p.yaxis.major_tick_line_color = None # turn off y-axis major ticks
p.yaxis.minor_tick_line_color = None # turn off y-axis minor ticks
p.axis.major_label_standoff = 0
if p.legend:
p.legend.label_text_font_size = FONT_SIZE
p.legend.background_fill_alpha = 0.85
def make_plots(linesources, pointsources):
plots = []
i=0
for linesource, pointsource in zip(linesources, pointsources):
fig = Figure(title=None, toolbar_location=None, tools=[],
x_axis_type="datetime",
width=300, height=70)
fig.xaxis.visible = False
if i in [0, 9] :
fig.xaxis.visible = True
fig.height = 90
fig.yaxis.visible = False
fig.xgrid.visible = True
fig.ygrid.visible = False
fig.min_border_left = 10
fig.min_border_right = 10
fig.min_border_top = 5
fig.min_border_bottom = 5
if not i in [0, 9]:
fig.xaxis.major_label_text_font_size = "0pt"
#fig.yaxis.major_label_text_font_size = "0pt"
fig.xaxis.major_tick_line_color = None
fig.yaxis.major_tick_line_color = None
fig.xaxis.minor_tick_line_color = None
fig.yaxis.minor_tick_line_color = None
fig.background_fill_color = "whitesmoke"
fig.line(x='date', y="y", source=linesource)
fig.circle(x='date', y='y', size=5, source=pointsource)
fig.text(x='date', y='y', text='text', x_offset=5, y_offset=10, text_font_size='7pt', source=pointsource)
fig.title.align = 'left'
fig.title.text_font_style = 'normal'
plots.append(fig)
i+=1
return plots
def create_figure():
xs, ys, colors, sizes = model.get_axes_values()
fig_args = dict(tools='pan', plot_height=600, plot_width=800)
if model.x_field in model.discrete_column_names and model.y_field in model.discrete_column_names:
figure = Figure(x_range=xs, y_range=ys, **fig_args)
figure.axis.major_label_orientation = math.pi / 4
elif model.x_field in model.discrete_column_names:
figure = Figure(x_range=xs, **fig_args)
figure.xaxis.major_label_orientation = math.pi / 4
elif model.y_field in model.discrete_column_names:
figure = Figure(y_range=ys, **fig_args)
figure.yaxis.major_label_orientation = math.pi / 4
else:
figure = Figure(**fig_args)
figure.circle(x=xs,
y=ys,
color=colors,
size=sizes,
line_color="white",
alpha=0.8)
figure.toolbar_location = None
figure.xaxis.axis_label = model.x_field
figure.yaxis.axis_label = model.y_field
figure.background_fill_color = model.background_fill
figure.border_fill_color = model.background_fill
figure.axis.axis_line_color = "white"
figure.axis.axis_label_text_color = "white"
figure.axis.major_label_text_color = "white"
figure.axis.major_tick_line_color = "white"
figure.axis.minor_tick_line_color = "white"
figure.axis.minor_tick_line_color = "white"
figure.grid.grid_line_dash = [6, 4]
figure.grid.grid_line_alpha = .3
return figure
plotyrange = ColumnDataSource(data=dict(yrange=yrange))
textlabel = ColumnDataSource(data=dict(label=planet_label))
################################
# BOKEH PLOTTING
################################
#plots spectrum and exposure time
snr_plot = Figure(
plot_height=500,
plot_width=750,
tools="crosshair,pan,reset,resize,save,box_zoom,wheel_zoom,hover",
toolbar_location='right',
x_range=[0.4, 5.0],
y_range=[min(Fo) * 0.9, max(Fo) * 1.1])
snr_plot.background_fill_color = "beige"
snr_plot.background_fill_alpha = 0.5
snr_plot.yaxis.axis_label = 'Flux [W/m**2/s]'
snr_plot.xaxis.axis_label = 'Wavelength [micron]'
snr_plot.title.text = 'Object Spectrum: Sun' #initial spectrum is Sun at 10 pc
snr_plot.line('lam',
'Fo',
source=theobject,
line_width=2.0,
color="darkgreen",
alpha=0.7)
snr_plot.circle('lam',
'spec',
source=theobject,
fill_color='lightgreen',
<div>
<span style="font-size: 15px; font-weight: bold; color: #696">D = </span>
<span style="font-size: 15px; font-weight: bold; color: #696;">@r</span>
<span style="font-size: 15px; font-weight: bold; color: #696;"> pc</span>
</div>
<div>
<span style="font-size: 15px; font-weight: bold; color: #696">C = </span>
<span style="font-size: 15px; font-weight: bold; color: #696;">@complete</span>
</div>
</div>
""")
plot1.add_tools(hover)
hover = plot1.select(dict(type=HoverTool))
plot1.x_range = Range1d(-50, 50, bounds=(-50, 50))
plot1.y_range = Range1d(-50, 50, bounds=(-50, 50))
plot1.background_fill_color = "black"
plot1.background_fill_alpha = 1.0
plot1.yaxis.axis_label = 'Yield'
plot1.xaxis.axis_label = ' '
plot1.xaxis.axis_line_width = 0
plot1.yaxis.axis_line_width = 0
plot1.xaxis.axis_line_color = 'black'
plot1.yaxis.axis_line_color = 'black'
plot1.border_fill_color = "black"
plot1.min_border_left = 80
# main glyphs for planet circles
star_syms = plot1.circle('x', 'y', source=star_points, name="star_points_to_hover", \
fill_color='color', line_color='color', radius=0.5, line_alpha=0.5, fill_alpha=0.7)
star_syms.selection_glyph = Circle(fill_alpha=0.8,
fill_color="purple",
def plot_pulse_files(metafile, time=False):
'''
plot_pulse_files(metafile)
Helper function to plot a list of AWG files. A JS slider allows choice of sequence number.
'''
#If we only go one filename turn it into a list
with open(metafile, 'r') as FID:
meta_info = json.load(FID)
fileNames = []
for el in meta_info["instruments"].values():
# Accomodate seq_file per instrument and per channel
if isinstance(el, str):
fileNames.append(el)
elif isinstance(el, dict):
for file in el.values():
fileNames.append(file)
line_names, num_seqs, data_dicts = extract_waveforms(fileNames, time=time)
localname = os.path.split(fileNames[0])[1]
sequencename = localname.split('-')[0]
data = {
line_name: ColumnDataSource(data=dat)
for line_name, dat in data_dicts.items()
}
plot = Figure(title=sequencename, plot_width=1000)
if config.plotBackground:
plot.background_fill_color = config.plotBackground
if config.gridColor:
plot.xgrid.grid_line_color = config.gridColor
plot.ygrid.grid_line_color = config.gridColor
if len(line_names) < 10:
# Colobrewer2 qualitative Set1 (http://colorbrewer2.org)
colors = brewer['Set1'][max(3, len(line_names))]
elif 9 < len(line_names) < 19:
# d3 Category20 for up to 18 channels
colors = d3['Category20'][len(line_names)]
else:
# 256 palette
colors = [
Inferno[256][np.int(ct - 1)]
for ct in np.floor(np.linspace(0, 256, len(line_names)))
]
js_sources = data
for ct, k in enumerate(line_names):
k_ = k.replace("-", "_")
line = plot.line(data_dicts[k + "_1"]["x"],
data_dicts[k + "_1"]["y"],
color=colors[ct % len(colors)],
line_width=2,
legend=k.replace("_", "-"))
js_sources[k_] = line.data_source
code_template = Template("""
var seq_num = cb_obj.getv('value');
// console.log(seq_num);
var all_data = {
{% for trace in trace_names %}
'{{trace}}': {{trace}}.getv('data'),
{% endfor %}
};
{% for line in line_names %}
{{line}}['data'] = {'x': (all_data['{{line}}_'.concat(seq_num.toString())])['x'],
'y': (all_data['{{line}}_'.concat(seq_num.toString())])['y']};
{% endfor %}
""")
rendered = code_template.render(line_names=line_names,
trace_names=list(
data.keys())[:-len(line_names)])
callback = CustomJS(args=dict(**js_sources), code=rendered)
if num_seqs > 1:
slider = Slider(start=1,
end=num_seqs,
value=1,
step=1,
title="Sequence",
callback=callback)
layout = column(slider, plot)
show(layout)
else:
show(plot)
def create_figure(self):
analog_glyphs = {}
binary_glyphs = {}
multistates_glyphs = {}
multistates_labels = {}
virtuals_glyphs = {}
self._log.debug("Creating figure")
TOOLS = "pan,box_zoom,wheel_zoom,save,reset"
p = Figure(
x_axis_type="datetime",
x_axis_label="Time",
y_axis_label="Value",
title="Live trends",
tools=TOOLS,
plot_width=1200,
plot_height=800,
toolbar_location="right",
)
p.title.text_font_size = "24pt"
p.xaxis.axis_label_text_font_size = "18pt"
p.yaxis.axis_label_text_font_size = "18pt"
p.xaxis.axis_label_text_font_style = "normal"
p.yaxis.axis_label_text_font_style = "normal"
p.xaxis.major_label_text_font_size = "12pt"
p.yaxis.major_label_text_font_size = "12pt"
p.background_fill_color = "#f4f3ef"
p.border_fill_color = "#f4f3ef"
p.extra_y_ranges = {
"bool": Range1d(start=0, end=1.1),
"enum": Range1d(start=0, end=10),
}
p.add_layout(
LinearAxis(y_range_name="bool", axis_label="Binary",
visible=False), "left")
p.add_layout(
LinearAxis(
y_range_name="enum",
axis_label="Enumerated",
visible=False,
# ticker=list(range(11)),
),
"right",
)
hover_common = HoverTool(
tooltips=[
("name", "$name"),
("value", "$data_y"),
# ('state', "@$name_state"),
# ("units", "$tags"),
("time", "@time_s"),
],
renderers=[],
toggleable=False,
formatters={"@time_s": "datetime"},
mode="mouse",
)
hover_multi = {}
p.add_tools(hover_common)
for name in self._binary_name:
binary_glyphs[name] = p.step(
x="index",
y=name,
source=self.cds,
name=name,
color=self.color_mappers["binary"][name],
y_range_name="bool",
mode="after",
line_width=8,
visible=False,
tags=["unit", "description"],
)
# binary_glyphs[name].add_tool(hover_common)
hover_common.renderers.append(binary_glyphs[name])
for name in self._multistates_name:
multistates_glyphs[name] = p.step(
x="index",
y=name,
source=self.cds,
name=name,
color=self.color_mappers["multistates"][name],
y_range_name="enum",
line_dash="dashed",
line_width=7,
visible=False,
tags=["unit", "description"],
mode="after",
)
# for name in self._multistates_labels:
# multistates_labels[name] = LabelSet(x="index", y=name.split('_')[0], text=name, level='glyph',
# x_offset=0, y_offset=1, source=self.cds, render_mode='canvas', visible=False)
# p.add_layout(multistates_labels[name])
# for name in self._multistates_labels:
# _msname = name.split("_")[0]
# multistates_labels[name] = p.circle(
# x="index",
# y=_msname,
# source=self.cds,
# color=self.color_mappers["multistates"][_msname],
# size=10,
# alpha=0.1,
# y_range_name="enum",
# visible=False,
# )
# hover_multi[name] = HoverTool(
# tooltips=[
# ("name", "$name"),
# ("value", "@" + name),
# ("time", "@time_s"),
# ],
# mode="mouse",
# renderers=[multistates_labels[name]],
# toggleable=False,
# )
# p.add_tools(hover_multi[name])
for name in self._multistates_labels:
_msname = name.split("_")[0]
multistates_labels[name] = p.text(
x="index",
y=_msname,
text=name,
source=self.cds,
text_color=self.color_mappers["multistates"][_msname],
angle=0.7835,
# size=10,
# alpha=0.1,
y_range_name="enum",
visible=False,
)
hover_multi[name] = HoverTool(
tooltips=[
("name", "$name"),
("value", "@" + name),
("time", "@time_s"),
],
mode="mouse",
renderers=[multistates_labels[name]],
toggleable=False,
)
p.add_tools(hover_multi[name])
for name in self._analog_name:
analog_glyphs[name] = p.line(
x="index",
y=name,
source=self.cds,
name=name,
color=self.color_mappers["analog"][name],
line_width=2,
visible=False,
tags=["unit", "description"],
)
# analog_glyphs[name].add_tool(hover_common)
hover_common.renderers.append(analog_glyphs[name])
for name in self._virtuals_name:
virtuals_glyphs[name] = p.line(
x="index",
y=name,
source=self.cds,
name=name,
color=self.color_mappers["virtual"][name],
line_width=2,
visible=False,
tags=["unit", "description"],
)
# virtuals_glyphs[name].add_tool(hover_common)
hover_common.renderers.append(virtuals_glyphs[name])
self.glyphs = {
"analog": analog_glyphs,
"binary": binary_glyphs,
"multistates": multistates_glyphs,
"multistates_labels": multistates_labels,
"virtual": virtuals_glyphs,
}
legend = Legend(items=[])
legend.click_policy = "hide"
p.add_layout(legend, "below")
return p
def plot_pulse_files(fileNames):
'''
plot_pulse_files(fileNames)
Helper function to plot a list of AWG files. A JS slider allows choice of sequence number.
'''
#If we only go one filename turn it into a list
if isinstance(fileNames, str):
fileNames = [fileNames]
wfs = {}
dataDict = {}
lineNames = []
title = ""
translators = build_awg_translator_map()
num_seqs = 0
for fileName in sorted(fileNames):
#Assume a naming convention path/to/file/SequenceName-AWGName.h5
AWGName = (os.path.split(os.path.splitext(fileName)[0])[1]).split('-')[1]
#Strip any _ suffix
if '_' in AWGName:
AWGName = AWGName[:AWGName.index('_')]
title += os.path.split(os.path.splitext(fileName)[0])[1] + "; "
wfs[AWGName] = translators[AWGName].read_sequence_file(fileName)
for (k,seqs) in sorted(wfs[AWGName].items()):
if not all_zero_seqs(seqs):
num_seqs = max(num_seqs, len(seqs))
lineNames.append(AWGName + '-' + k)
k_ = lineNames[-1].replace("-", "_")
for ct,seq in enumerate(seqs):
dataDict[k_+"_x_{:d}".format(ct+1)] = np.arange(len(seqs[ct]))
dataDict[k_+"_y_{:d}".format(ct+1)] = seqs[ct] + 2*(len(lineNames)-1)
#Remove trailing semicolon from title
title = title[:-2]
all_data = ColumnDataSource(data=dataDict)
plot = Figure(title=title, plot_width=1000)
plot.background_fill_color = config.plotBackground
if config.gridColor:
plot.xgrid.grid_line_color = config.gridColor
plot.ygrid.grid_line_color = config.gridColor
# Colobrewer2 qualitative Set1 (http://colorbrewer2.org)
colours = [
"#e41a1c",
"#377eb8",
"#4daf4a",
"#984ea3",
"#ff7f00",
"#ffff33",
"#a65628",
"#f781bf",
"#999999"
]
js_sources = {}
js_sources["all_data"] = all_data
for ct,k in enumerate(lineNames):
k_ = k.replace("-", "_")
line = plot.line(dataDict[k_+"_x_1"], dataDict[k_+"_y_1"], color=colours[ct%len(colours)], line_width=2, legend=k)
js_sources[k_] = line.data_source
code_template = Template("""
var seq_num = cb_obj.get('value');
console.log(seq_num)
var all_data = all_data.get('data');
{% for line in lineNames %}
{{line}}.set('data', {'x':all_data['{{line}}_x_'.concat(seq_num.toString())], 'y':all_data['{{line}}_y_'.concat(seq_num.toString())]} );
{% endfor %}
console.log("Got here!")
""")
callback = CustomJS(args=js_sources, code=code_template.render(lineNames=[l.replace("-","_") for l in lineNames]))
slider = Slider(start=1, end=num_seqs, value=1, step=1, title="Sequence", callback=callback)
layout = vform(slider, plot)
show(layout)
def plot_pulse_files_compare(metafile1, metafile2, time=False):
'''
plot_pulse_files_compare(fileNames1, fileNames2)
Helper function to plot a list of AWG files. A JS slider allows choice of sequence number.
'''
#If we only go one filename turn it into a list
fileNames1 = []
fileNames2 = []
with open(metafile1, 'r') as FID:
meta_info1 = json.load(FID)
for el in meta_info1["instruments"].values():
# Accomodate seq_file per instrument and per channel
if isinstance(el, str):
fileNames1.append(el)
elif isinstance(el, dict):
for file in el.values():
fileNames1.append(file)
with open(metafile2, 'r') as FID:
meta_info2 = json.load(FID)
for el in meta_info2["instruments"].values():
# Accomodate seq_file per instrument and per channel
if isinstance(el, str):
fileNames2.append(el)
elif isinstance(el, dict):
for file in el.values():
fileNames2.append(file)
line_names1, num_seqs1, data_dicts1 = extract_waveforms(fileNames1,
'A',
time=time)
line_names2, num_seqs2, data_dicts2 = extract_waveforms(fileNames2,
'B',
time=time)
num_seqs = max(num_seqs1, num_seqs2)
data_dicts1.update(data_dicts2)
line_names = line_names1 + line_names2
localname = os.path.split(fileNames1[0])[1]
sequencename = localname.split('-')[0]
data = {
line_name: ColumnDataSource(data=dat)
for line_name, dat in data_dicts1.items()
}
plot = Figure(title=sequencename, plot_width=1000)
plot.background_fill_color = config.plotBackground
if config.gridColor:
plot.xgrid.grid_line_color = config.gridColor
plot.ygrid.grid_line_color = config.gridColor
# Colobrewer2 qualitative Set1 (http://colorbrewer2.org)
colours = [
"#e41a1c", "#377eb8", "#4daf4a", "#984ea3", "#ff7f00", "#ffff33",
"#a65628", "#f781bf", "#999999"
]
js_sources = data
for ct, k in enumerate(line_names):
k_ = k.replace("-", "_")
line = plot.line(data_dicts1[k + "_1"]["x"],
data_dicts1[k + "_1"]["y"],
color=colours[ct % len(colours)],
line_width=2,
legend=k.replace("_", "-"))
js_sources[k_] = line.data_source
code_template = Template("""
var seq_num = cb_obj.getv('value');
// console.log(seq_num);
var all_data = {
{% for trace in trace_names %}
'{{trace}}': {{trace}}.getv('data'),
{% endfor %}
};
{% for line in line_names %}
{{line}}['data'] = {'x': (all_data['{{line}}_'.concat(seq_num.toString())])['x'],
'y': (all_data['{{line}}_'.concat(seq_num.toString())])['y']};
{% endfor %}
""")
rendered = code_template.render(line_names=line_names,
trace_names=list(
data.keys())[:-len(line_names)])
callback = CustomJS(args=dict(**js_sources), code=rendered)
if num_seqs > 1:
slider = Slider(start=1,
end=num_seqs,
value=1,
step=1,
title="Sequence",
callback=callback)
layout = column(slider, plot)
show(layout)
else:
show(plot)
throttle = setTimeout(update_dims, 100, "replace");
"""
# Create plot -------------------------------
xmin = -8240227.037
ymin = 4974203.152
xmax = -8231283.905
ymax = 4979238.441
fig = Figure(x_range=(xmin, xmax),
y_range=(ymin, ymax),
plot_height=600,
plot_width=900,
tools='pan,wheel_zoom')
fig.background_fill_color = 'black'
fig.add_tile(get_provider("STAMEN_TONER"), alpha=.3)
fig.x_range.callback = CustomJS(code=dims_jscode,
args=dict(plot=fig, dims=dims))
fig.y_range.callback = CustomJS(code=dims_jscode,
args=dict(plot=fig, dims=dims))
fig.axis.visible = False
fig.grid.grid_line_alpha = 0
fig.min_border_left = 0
fig.min_border_right = 0
fig.min_border_top = 0
fig.min_border_bottom = 0
image_source = ColumnDataSource(dict(image=[], x=[], y=[], dw=[], dh=[]))
fig.image_rgba(source=image_source,
image='image',
srcData = ColumnDataSource(data=dict(x = x, y = y, err_x_up = err_x_up, err_y_up = err_y_up, err_x_down = err_x_down, err_y_down = err_y_down))
# create the scatter plot
TOOLS="pan,wheel_zoom,reset"
p = Figure(tools=TOOLS, plot_width=500, plot_height=500, min_border=10, min_border_left=50,
title="Inputted Data",
toolbar_location="above",
active_scroll='wheel_zoom',
active_drag = "pan",
x_axis_label = "x",
y_axis_label = "y"
)
p.background_fill_color = "#fafafa"
r = p.scatter(source = srcData, x='x', y='y', size=10, color="blue", alpha=0.6)
#y error bars:
p.add_layout(
Whisker(source = srcData, base="x", upper="err_y_up", lower="err_y_down")
)
#x error bars:
p.add_layout(
Whisker(source = srcData, base="y", upper="err_x_up", lower="err_x_down", dimension="width")
)
#JS callbacks
def main():
print('''Please select the CSV dataset you\'d like to use.
The dataset should contain these columns:
- metric to apply threshold to
- indicator of event to detect (e.g. malicious activity)
- Please label this as 1 or 0 (true or false);
This will not work otherwise!
''')
# Import the dataset
imported_data = None
while isinstance(imported_data, pd.DataFrame) == False:
file_path = input('Enter the path of your dataset: ')
imported_data = file_to_df(file_path)
time.sleep(1)
print(f'''\nGreat! Here is a preview of your data:
Imported fields:''')
# List headers by column index.
cols = list(imported_data.columns)
for index in range(len(cols)):
print(f'{index}: {cols[index]}')
print(f'Number of records: {len(imported_data.index)}\n')
# Preview the DataFrame
time.sleep(1)
print(imported_data.head(), '\n')
# Prompt for the metric and source of truth.
time.sleep(1)
metric_col, indicator_col = columns_picker(cols)
# User self-validation.
col_check = input('Can you confirm if this is correct? (y/n): ').lower()
# If it's wrong, let them try again
while col_check != 'y':
metric_col, indicator_col = columns_picker(cols)
col_check = input(
'Can you confirm if this is correct? (y/n): ').lower()
else:
print(
'''\nGreat! Thanks for your patience. Generating summary stats now..\n'''
)
# Generate summary stats.
time.sleep(1)
malicious, normal = classification_split(imported_data, metric_col,
indicator_col)
mal_mean = malicious.mean()
mal_stddev = malicious.std()
mal_count = malicious.size
mal_median = malicious.median()
norm_mean = normal.mean()
norm_stddev = normal.std()
norm_count = normal.size
norm_median = normal.median()
print(f'''Normal vs Malicious Summary (metric = {metric_col}):
Normal:
-----------------------------
Observations: {round(norm_count, 2)}
Average: {round(norm_mean, 2)}
Median: {round(norm_median, 2)}
Standard Deviation: {round(norm_stddev, 2)}
Malicious:
-----------------------------
Observations: {round(mal_count, 2)}
Average: {round(mal_mean, 2)}
Median: {round(mal_median, 2)}
Standard Deviation: {round(mal_stddev, 2)}
''')
# Insights and advisories
# Provide the accuracy metrics of a generic threshold at avg + 3 std deviations
generic_threshold = confusion_matrix(
malicious, normal, threshold_calc(norm_mean, norm_stddev, 3))
time.sleep(1)
print(
f'''A threshold at (average + 3x standard deviations) {metric_col} would result in:
- True Positives (correctly identified malicious events: {generic_threshold['TP']:,}
- False Positives (wrongly identified normal events: {generic_threshold['FP']:,}
- True Negatives (correctly identified normal events: {generic_threshold['TN']:,}
- False Negatives (wrongly identified malicious events: {generic_threshold['FN']:,}
Accuracy Metrics:
- Precision (what % of events above threshold are actually malicious): {round(generic_threshold['precision'] * 100, 1)}%
- Recall (what % of malicious events did we catch): {round(generic_threshold['recall'] * 100, 1)}%
- F1 Score (blends precision and recall): {round(generic_threshold['f1_score'] * 100, 1)}%'''
)
# Distribution skew check.
if norm_mean >= (norm_median * 1.1):
time.sleep(1)
print(
f'''\nYou may want to be cautious as your normal traffic\'s {metric_col}
has a long tail towards high values. The median is {round(norm_median, 2)}
compared to {round(norm_mean, 2)} for the average.''')
if mal_mean < threshold_calc(norm_mean, norm_stddev, 2):
time.sleep(1)
print(
f'''\nWarning: you may find it difficult to avoid false positives as the average
{metric_col} for malicious traffic is under the 95th percentile of the normal traffic.'''
)
# For fun/anticipation. Actually a nerd joke because of the method we'll be using.
if '-q' not in sys.argv[1:]:
time.sleep(1)
play_a_game.billy()
decision = input('yes/no: ').lower()
while decision != 'yes':
time.sleep(1)
print('...That\'s no fun...')
decision = input('Let\'s try that again: ').lower()
# Let's get to the simulations!
time.sleep(1)
print('''\nInstead of manually experimenting with threshold multipliers,
let\'s simulate a range of options and see what produces the best result.
This is similar to what is known as \"Monte Carlo simulation\".\n''')
# Initialize session name & create app folder if there isn't one.
time.sleep(1)
session_name = input('Please provide a name for this project/session: ')
session_folder = make_folder(session_name)
# Generate list of multipliers to iterate over.
time.sleep(1)
mult_start = float(
input(
'Please provide the minimum multiplier you want to start at. We recommend 2: '
))
# Set the max to how many std deviations away the sample max is.
mult_end = (imported_data[metric_col].max() - norm_mean) / norm_stddev
mult_interval = float(
input('Please provide the desired gap between multiplier options: '))
# range() only allows integers, let's manually populate a list
multipliers = []
mult_counter = mult_start
while mult_counter < mult_end:
multipliers.append(round(mult_counter, 2))
mult_counter += mult_interval
print('Generating simulations..\n')
# Run simulations using our multipliers.
simulations = monte_carlo(malicious, normal, norm_mean, norm_stddev,
multipliers)
print('Done!')
time.sleep(1)
# Save simulations as CSV for later use.
simulation_filepath = os.path.join(
session_folder, f'{session_name}_simulation_results.csv')
simulations.to_csv(simulation_filepath, index=False)
print(f'Saved results to: {simulation_filepath}')
# Find the first threshold with the highest F1 score.
# This provides a balanced approach between precision and recall.
f1_max = simulations[simulations.f1_score ==
simulations.f1_score.max()].head(1)
f1_max_mult = f1_max.squeeze()['multiplier']
time.sleep(1)
print(
f'''\nBased on the F1 score metric, setting a threshold at {round(f1_max_mult,1)} standard deviations
above the average magnitude might provide optimal results.\n''')
time.sleep(1)
print(f'''{f1_max}
We recommend that you skim the CSV and the following visualization outputs
to sanity check results and make your own judgement.
''')
# Now for the fun part..generating the visualizations via Bokeh.
# Header & internal CSS.
title_text = '''
<style>
@font-face {
font-family: RobotoBlack;
src: url(fonts/Roboto-Black.ttf);
font-weight: bold;
}
@font-face {
font-family: RobotoBold;
src: url(fonts/Roboto-Bold.ttf);
font-weight: bold;
}
@font-face {
font-family: RobotoRegular;
src: url(fonts/Roboto-Regular.ttf);
}
body {
background-color: #f2ebe6;
}
title_header {
font-size: 80px;
font-style: bold;
font-family: RobotoBlack, Helvetica;
font-weight: bold;
margin-bottom: -200px;
}
h1, h2, h3 {
font-family: RobotoBlack, Helvetica;
color: #313596;
}
p {
font-size: 12px;
font-family: RobotoRegular
}
b {
color: #58c491;
}
th, td {
text-align:left;
padding: 5px;
}
tr:nth-child(even) {
background-color: white;
opacity: .7;
}
.vertical {
border-left: 1px solid black;
height: 190px;
}
</style>
<title_header style="text-align:left; color: white;">
Cream.
</title_header>
<p style="font-family: RobotoBold, Helvetica;
font-size:18px;
margin-top: 0px;
margin-left: 5px;">
Because time is money, and <b style="font-size=18px;">"Cash Rules Everything Around Me"</b>.
</p>
</div>
'''
title_div = Div(text=title_text,
width=800,
height=160,
margin=(40, 0, 0, 70))
# Summary stats from earlier.
summary_text = f'''
<h1>Results Overview</h1>
<i>metric = magnitude</i>
<table style="width:100%">
<tr>
<th>Metric</th>
<th>Normal Events</th>
<th>Malicious Events</th>
</tr>
<tr>
<td>Observations</td>
<td>{norm_count:,}</td>
<td>{mal_count:,}</td>
</tr>
<tr>
<td>Average</td>
<td>{round(norm_mean, 2):,}</td>
<td>{round(mal_mean, 2):,}</td>
</tr>
<tr>
<td>Median</td>
<td>{round(norm_median, 2):,}</td>
<td>{round(mal_median, 2):,}</td>
</tr>
<tr>
<td>Standard Deviation</td>
<td>{round(norm_stddev, 2):,}</td>
<td>{round(mal_stddev, 2):,}</td>
</tr>
</table>
'''
summary_div = Div(text=summary_text,
width=470,
height=320,
margin=(3, 0, -70, 73))
# Results of the hypothetical threshold.
hypothetical = f'''
<h1>"Rule of thumb" Hypothetical Threshold</h1>
<p>A threshold at <i>(average + 3x standard deviations)</i> {metric_col} would result in:</p>
<ul>
<li>True Positives (correctly identified malicious events:
<b>{generic_threshold['TP']:,}</b></li>
<li>False Positives (wrongly identified normal events:
<b>{generic_threshold['FP']:,}</b></li>
<li>True Negatives (correctly identified normal events:
<b>{generic_threshold['TN']:,}</b></li>
<li>False Negatives (wrongly identified malicious events:
<b>{generic_threshold['FN']:,}</b></li>
</ul>
<h2>Accuracy Metrics</h2>
<ul>
<li>Precision (what % of events above threshold are actually malicious):
<b>{round(generic_threshold['precision'] * 100, 1)}%</b></li>
<li>Recall (what % of malicious events did we catch):
<b>{round(generic_threshold['recall'] * 100, 1)}%</b></li>
<li>F1 Score (blends precision and recall):
<b>{round(generic_threshold['f1_score'] * 100, 1)}%</b></li>
</ul>
'''
hypo_div = Div(text=hypothetical,
width=600,
height=320,
margin=(5, 0, -70, 95))
line = '''
<div class="vertical"></div>
'''
vertical_line = Div(text=line,
width=20,
height=320,
margin=(80, 0, -70, -10))
# Let's get the exploratory charts generated.
malicious_hist, malicious_edge = np.histogram(malicious, bins=100)
mal_hist_df = pd.DataFrame({
'metric': malicious_hist,
'left': malicious_edge[:-1],
'right': malicious_edge[1:]
})
normal_hist, normal_edge = np.histogram(normal, bins=100)
norm_hist_df = pd.DataFrame({
'metric': normal_hist,
'left': normal_edge[:-1],
'right': normal_edge[1:]
})
exploratory = figure(
plot_width=plot_width,
plot_height=plot_height,
sizing_mode='fixed',
title=f'{metric_col.capitalize()} Distribution (σ = std dev)',
x_axis_label=f'{metric_col.capitalize()}',
y_axis_label='Observations')
exploratory.title.text_font_size = title_font_size
exploratory.border_fill_color = cell_bg_color
exploratory.border_fill_alpha = cell_bg_alpha
exploratory.background_fill_color = cell_bg_color
exploratory.background_fill_alpha = plot_bg_alpha
exploratory.min_border_left = left_border
exploratory.min_border_right = right_border
exploratory.min_border_top = top_border
exploratory.min_border_bottom = bottom_border
exploratory.quad(bottom=0,
top=mal_hist_df.metric,
left=mal_hist_df.left,
right=mal_hist_df.right,
legend_label='malicious',
fill_color=malicious_color,
alpha=.85,
line_alpha=.35,
line_width=.5)
exploratory.quad(bottom=0,
top=norm_hist_df.metric,
left=norm_hist_df.left,
right=norm_hist_df.right,
legend_label='normal',
fill_color=normal_color,
alpha=.35,
line_alpha=.35,
line_width=.5)
exploratory.add_layout(
Arrow(end=NormalHead(fill_color=malicious_color, size=10,
line_alpha=0),
line_color=malicious_color,
x_start=mal_mean,
y_start=mal_count,
x_end=mal_mean,
y_end=0))
arrow_label = Label(x=mal_mean,
y=mal_count,
y_offset=5,
text='Malicious Events',
text_font_style='bold',
text_color=malicious_color,
text_font_size='10pt')
exploratory.add_layout(arrow_label)
exploratory.xaxis.formatter = NumeralTickFormatter(format='0,0')
exploratory.yaxis.formatter = NumeralTickFormatter(format='0,0')
# 3 sigma reference line
sigma_ref(exploratory, norm_mean, norm_stddev)
exploratory.legend.location = "top_right"
exploratory.legend.background_fill_alpha = .3
# Zoomed in version
overlap_view = figure(
plot_width=plot_width,
plot_height=plot_height,
sizing_mode='fixed',
title=f'Overlap Highlight',
x_axis_label=f'{metric_col.capitalize()}',
y_axis_label='Observations',
y_range=(0, mal_count * .33),
x_range=(norm_mean + (norm_stddev * 2.5), mal_mean + (mal_stddev * 3)),
)
overlap_view.title.text_font_size = title_font_size
overlap_view.border_fill_color = cell_bg_color
overlap_view.border_fill_alpha = cell_bg_alpha
overlap_view.background_fill_color = cell_bg_color
overlap_view.background_fill_alpha = plot_bg_alpha
overlap_view.min_border_left = left_border
overlap_view.min_border_right = right_border
overlap_view.min_border_top = top_border
overlap_view.min_border_bottom = bottom_border
overlap_view.quad(bottom=0,
top=mal_hist_df.metric,
left=mal_hist_df.left,
right=mal_hist_df.right,
legend_label='malicious',
fill_color=malicious_color,
alpha=.85,
line_alpha=.35,
line_width=.5)
overlap_view.quad(bottom=0,
top=norm_hist_df.metric,
left=norm_hist_df.left,
right=norm_hist_df.right,
legend_label='normal',
fill_color=normal_color,
alpha=.35,
line_alpha=.35,
line_width=.5)
overlap_view.xaxis.formatter = NumeralTickFormatter(format='0,0')
overlap_view.yaxis.formatter = NumeralTickFormatter(format='0,0')
sigma_ref(overlap_view, norm_mean, norm_stddev)
overlap_view.legend.location = "top_right"
overlap_view.legend.background_fill_alpha = .3
# Probability Density - bigger bins for sparser malicous observations
malicious_hist_dense, malicious_edge_dense = np.histogram(malicious,
density=True,
bins=50)
mal_hist_dense_df = pd.DataFrame({
'metric': malicious_hist_dense,
'left': malicious_edge_dense[:-1],
'right': malicious_edge_dense[1:]
})
normal_hist_dense, normal_edge_dense = np.histogram(normal,
density=True,
bins=100)
norm_hist_dense_df = pd.DataFrame({
'metric': normal_hist_dense,
'left': normal_edge_dense[:-1],
'right': normal_edge_dense[1:]
})
density = figure(plot_width=plot_width,
plot_height=plot_height,
sizing_mode='fixed',
title='Probability Density',
x_axis_label=f'{metric_col.capitalize()}',
y_axis_label='% of Group Total')
density.title.text_font_size = title_font_size
density.border_fill_color = cell_bg_color
density.border_fill_alpha = cell_bg_alpha
density.background_fill_color = cell_bg_color
density.background_fill_alpha = plot_bg_alpha
density.min_border_left = left_border
density.min_border_right = right_border
density.min_border_top = top_border
density.min_border_bottom = bottom_border
density.quad(bottom=0,
top=mal_hist_dense_df.metric,
left=mal_hist_dense_df.left,
right=mal_hist_dense_df.right,
legend_label='malicious',
fill_color=malicious_color,
alpha=.85,
line_alpha=.35,
line_width=.5)
density.quad(bottom=0,
top=norm_hist_dense_df.metric,
left=norm_hist_dense_df.left,
right=norm_hist_dense_df.right,
legend_label='normal',
fill_color=normal_color,
alpha=.35,
line_alpha=.35,
line_width=.5)
density.xaxis.formatter = NumeralTickFormatter(format='0,0')
density.yaxis.formatter = NumeralTickFormatter(format='0.000%')
sigma_ref(density, norm_mean, norm_stddev)
density.legend.location = "top_right"
density.legend.background_fill_alpha = .3
# Simulation Series to be used
false_positives = simulations.FP
false_negatives = simulations.FN
multiplier = simulations.multiplier
precision = simulations.precision
recall = simulations.recall
f1_score = simulations.f1_score
f1_max = simulations[simulations.f1_score == simulations.f1_score.max(
)].head(1).squeeze()['multiplier']
# False Positives vs False Negatives
errors = figure(plot_width=plot_width,
plot_height=plot_height,
sizing_mode='fixed',
x_range=(multiplier.min(), multiplier.max()),
y_range=(0, false_positives.max()),
title='False Positives vs False Negatives',
x_axis_label='Multiplier',
y_axis_label='Count')
errors.title.text_font_size = title_font_size
errors.border_fill_color = cell_bg_color
errors.border_fill_alpha = cell_bg_alpha
errors.background_fill_color = cell_bg_color
errors.background_fill_alpha = plot_bg_alpha
errors.min_border_left = left_border
errors.min_border_right = right_border
errors.min_border_top = top_border
errors.min_border_bottom = right_border
errors.line(multiplier,
false_positives,
legend_label='false positives',
line_width=2,
color=fp_color)
errors.line(multiplier,
false_negatives,
legend_label='false negatives',
line_width=2,
color=fn_color)
errors.yaxis.formatter = NumeralTickFormatter(format='0,0')
errors.extra_y_ranges = {"y2": Range1d(start=0, end=1.1)}
errors.add_layout(
LinearAxis(y_range_name="y2",
axis_label="Score",
formatter=NumeralTickFormatter(format='0.00%')), 'right')
errors.line(multiplier,
f1_score,
line_width=2,
color=f1_color,
legend_label='F1 Score',
y_range_name="y2")
# F1 Score Maximization point
f1_thresh = Span(location=f1_max,
dimension='height',
line_color=f1_color,
line_dash='dashed',
line_width=2)
f1_label = Label(x=f1_max + .05,
y=180,
y_units='screen',
text=f'F1 Max: {round(f1_max,2)}',
text_font_size='10pt',
text_font_style='bold',
text_align='left',
text_color=f1_color)
errors.add_layout(f1_thresh)
errors.add_layout(f1_label)
errors.legend.location = "top_right"
errors.legend.background_fill_alpha = .3
# False Negative Weighting.
# Intro.
weighting_intro = f'''
<h3>Error types differ in impact.</h3>
<p>In the case of security incidents, a false negative,
though possibly rarer than false positives, is likely more costly. For example, downtime suffered
from a DDoS attack (lost sales/customers) incurs more loss than time wasted chasing a false positive
(labor hours). </p>
<p>Try playing around with the slider to the right to see how your thresholding strategy might need to change
depending on the relative weight of false negatives to false positives. What does it look like at
1:1, 50:1, etc.?</p>
'''
weighting_div = Div(text=weighting_intro,
width=420,
height=180,
margin=(0, 75, 0, 0))
# Now for the weighted errors viz
default_weighting = 10
initial_fp_cost = 100
simulations['weighted_FN'] = simulations.FN * default_weighting
weighted_fn = simulations.weighted_FN
simulations[
'total_weighted_error'] = simulations.FP + simulations.weighted_FN
total_weighted_error = simulations.total_weighted_error
simulations['fp_cost'] = initial_fp_cost
fp_cost = simulations.fp_cost
simulations[
'total_estimated_cost'] = simulations.total_weighted_error * simulations.fp_cost
total_estimated_cost = simulations.total_estimated_cost
twe_min = simulations[simulations.total_weighted_error ==
simulations.total_weighted_error.min()].head(
1).squeeze()['multiplier']
twe_min_count = simulations[simulations.multiplier == twe_min].head(
1).squeeze()['total_weighted_error']
generic_twe = simulations[simulations.multiplier.apply(
lambda x: round(x, 2)) == 3.00].squeeze()['total_weighted_error']
comparison = f'''
<p>Based on your inputs, the optimal threshold is around <b>{twe_min}</b>.
This would result in an estimated <b>{int(twe_min_count):,}</b> total weighted errors and
<b>${int(twe_min_count * initial_fp_cost):,}</b> in losses.</p>
<p>The generic threshold of 3.0 standard deviations would result in <b>{int(generic_twe):,}</b>
total weighted errors and <b>${int(generic_twe * initial_fp_cost):,}</b> in losses.</p>
<p>Using the optimal threshold would save <b>${int((generic_twe - twe_min_count) * initial_fp_cost):,}</b>,
reducing costs by <b>{(generic_twe - twe_min_count) / generic_twe * 100:.1f}%</b>
(assuming near-future events are distributed similarly to those from the past).</p>
'''
comparison_div = Div(text=comparison,
width=420,
height=230,
margin=(0, 75, 0, 0))
loss_min = ColumnDataSource(data=dict(multiplier=multiplier,
fp=false_positives,
fn=false_negatives,
weighted_fn=weighted_fn,
twe=total_weighted_error,
fpc=fp_cost,
tec=total_estimated_cost,
precision=precision,
recall=recall,
f1=f1_score))
evaluation = Figure(plot_width=900,
plot_height=520,
sizing_mode='fixed',
x_range=(multiplier.min(), multiplier.max()),
title='Evaluation Metrics vs Total Estimated Cost',
x_axis_label='Multiplier',
y_axis_label='Cost')
evaluation.title.text_font_size = title_font_size
evaluation.border_fill_color = cell_bg_color
evaluation.border_fill_alpha = cell_bg_alpha
evaluation.background_fill_color = cell_bg_color
evaluation.background_fill_alpha = plot_bg_alpha
evaluation.min_border_left = left_border
evaluation.min_border_right = right_border
evaluation.min_border_top = top_border
evaluation.min_border_bottom = bottom_border
evaluation.line('multiplier',
'tec',
source=loss_min,
line_width=3,
line_alpha=0.6,
color=total_weighted_color,
legend_label='Total Estimated Cost')
evaluation.yaxis.formatter = NumeralTickFormatter(format='$0,0')
# Evaluation metrics on second right axis.
evaluation.extra_y_ranges = {"y2": Range1d(start=0, end=1.1)}
evaluation.add_layout(
LinearAxis(y_range_name="y2",
axis_label="Score",
formatter=NumeralTickFormatter(format='0.00%')), 'right')
evaluation.line('multiplier',
'precision',
source=loss_min,
line_width=3,
line_alpha=0.6,
color=precision_color,
legend_label='Precision',
y_range_name="y2")
evaluation.line('multiplier',
'recall',
source=loss_min,
line_width=3,
line_alpha=0.6,
color=recall_color,
legend_label='Recall',
y_range_name="y2")
evaluation.line('multiplier',
'f1',
source=loss_min,
line_width=3,
line_alpha=0.6,
color=f1_color,
legend_label='F1 score',
y_range_name="y2")
evaluation.legend.location = "bottom_right"
evaluation.legend.background_fill_alpha = .3
twe_thresh = Span(location=twe_min,
dimension='height',
line_color=total_weighted_color,
line_dash='dashed',
line_width=2)
twe_label = Label(x=twe_min - .05,
y=240,
y_units='screen',
text=f'Cost Min: {round(twe_min,2)}',
text_font_size='10pt',
text_font_style='bold',
text_align='right',
text_color=total_weighted_color)
evaluation.add_layout(twe_thresh)
evaluation.add_layout(twe_label)
# Add in same f1 thresh as previous viz
evaluation.add_layout(f1_thresh)
evaluation.add_layout(f1_label)
handler = CustomJS(args=dict(source=loss_min,
thresh=twe_thresh,
label=twe_label,
comparison=comparison_div),
code="""
var data = source.data
var ratio = cb_obj.value
var multiplier = data['multiplier']
var fp = data['fp']
var fn = data['fn']
var weighted_fn = data['weighted_fn']
var twe = data['twe']
var fpc = data['fpc']
var tec = data['tec']
var generic_twe = 0
function round(value, decimals) {
return Number(Math.round(value+'e'+decimals)+'e-'+decimals);
}
function comma_sep(x) {
return x.toString().replace(/\B(?<!\.\d*)(?=(\d{3})+(?!\d))/g, ",");
}
for (var i = 0; i < multiplier.length; i++) {
weighted_fn[i] = Math.round(fn[i] * ratio)
twe[i] = weighted_fn[i] + fp[i]
tec[i] = twe[i] * fpc[i]
if (round(multiplier[i],2) == 3.00) {
generic_twe = twe[i]
}
}
var min_loss = Math.min.apply(null,twe)
var new_thresh = 0
for (var i = 0; i < multiplier.length; i++) {
if (twe[i] == min_loss) {
new_thresh = multiplier[i]
thresh.location = new_thresh
thresh.change.emit()
label.x = new_thresh
label.text = `Cost Min: ${new_thresh}`
label.change.emit()
comparison.text = `
<p>Based on your inputs, the optimal threshold is around <b>${new_thresh}</b>.
This would result in an estimated <b>${comma_sep(round(min_loss,0))}</b> total weighted errors and
<b>$${comma_sep(round(min_loss * fpc[i],0))}</b> in losses.</p>
<p>The generic threshold of 3.0 standard deviations would result in <b>${comma_sep(round(generic_twe,0))}</b>
total weighted errors and <b>$${comma_sep(round(generic_twe * fpc[i],0))}</b> in losses.</p>
<p>Using the optimal threshold would save <b>$${comma_sep(round((generic_twe - min_loss) * fpc[i],0))}</b>,
reducing costs by <b>${comma_sep(round((generic_twe - min_loss) / generic_twe * 100,0))}%</b>
(assuming near-future events are distributed similarly to those from the past).</p>
`
comparison.change.emit()
}
}
source.change.emit();
""")
slider = Slider(start=1.0,
end=500,
value=default_weighting,
step=.25,
title="FN:FP Ratio",
bar_color='#FFD100',
height=50,
margin=(5, 0, 5, 0))
slider.js_on_change('value', handler)
cost_handler = CustomJS(args=dict(source=loss_min,
comparison=comparison_div),
code="""
var data = source.data
var new_cost = cb_obj.value
var multiplier = data['multiplier']
var fp = data['fp']
var fn = data['fn']
var weighted_fn = data['weighted_fn']
var twe = data['twe']
var fpc = data['fpc']
var tec = data['tec']
var generic_twe = 0
function round(value, decimals) {
return Number(Math.round(value+'e'+decimals)+'e-'+decimals);
}
function comma_sep(x) {
return x.toString().replace(/\B(?<!\.\d*)(?=(\d{3})+(?!\d))/g, ",");
}
for (var i = 0; i < multiplier.length; i++) {
fpc[i] = new_cost
tec[i] = twe[i] * fpc[i]
if (round(multiplier[i],2) == 3.00) {
generic_twe = twe[i]
}
}
var min_loss = Math.min.apply(null,twe)
var new_thresh = 0
for (var i = 0; i < multiplier.length; i++) {
if (twe[i] == min_loss) {
new_thresh = multiplier[i]
comparison.text = `
<p>Based on your inputs, the optimal threshold is around <b>${new_thresh}</b>.
This would result in an estimated <b>${comma_sep(round(min_loss,0))}</b> total weighted errors and
<b>$${comma_sep(round(min_loss * new_cost,0))}</b> in losses.</p>
<p>The generic threshold of 3.0 standard deviations would result in
<b>${comma_sep(round(generic_twe,0))}</b> total weighted errors and
<b>$${comma_sep(round(generic_twe * new_cost,0))}</b> in losses.</p>
<p>Using the optimal threshold would save
<b>$${comma_sep(round((generic_twe - min_loss) * new_cost,0))}</b>,
reducing costs by <b>${comma_sep(round((generic_twe - min_loss)/generic_twe * 100,0))}%</b>
(assuming near-future events are distributed similarly to those from the past).</p>
`
comparison.change.emit()
}
}
source.change.emit();
""")
cost_input = TextInput(value=f"{initial_fp_cost}",
title="How much a false positive costs:",
height=75,
margin=(20, 75, 20, 0))
cost_input.js_on_change('value', cost_handler)
# Include DataTable of simulation results
dt_columns = [
TableColumn(field="multiplier", title="Multiplier"),
TableColumn(field="fp",
title="False Positives",
formatter=NumberFormatter(format='0,0')),
TableColumn(field="fn",
title="False Negatives",
formatter=NumberFormatter(format='0,0')),
TableColumn(field="weighted_fn",
title="Weighted False Negatives",
formatter=NumberFormatter(format='0,0.00')),
TableColumn(field="twe",
title="Total Weighted Errors",
formatter=NumberFormatter(format='0,0.00')),
TableColumn(field="fpc",
title="Estimated FP Cost",
formatter=NumberFormatter(format='$0,0.00')),
TableColumn(field="tec",
title="Estimated Total Cost",
formatter=NumberFormatter(format='$0,0.00')),
TableColumn(field="precision",
title="Precision",
formatter=NumberFormatter(format='0.00%')),
TableColumn(field="recall",
title="Recall",
formatter=NumberFormatter(format='0.00%')),
TableColumn(field="f1",
title="F1 Score",
formatter=NumberFormatter(format='0.00%')),
]
data_table = DataTable(source=loss_min,
columns=dt_columns,
width=1400,
height=700,
sizing_mode='fixed',
fit_columns=True,
reorderable=True,
sortable=True,
margin=(30, 0, 20, 0))
# weighting_layout = column([weighting_div, evaluation, slider, data_table])
weighting_layout = column(
row(column(weighting_div, cost_input, comparison_div),
column(slider, evaluation), Div(text='', height=200, width=60)),
data_table)
# Initialize visualizations in browser
time.sleep(1.5)
layout = grid([
[title_div],
[row(summary_div, vertical_line, hypo_div)],
[
row(Div(text='', height=200, width=60), exploratory,
Div(text='', height=200, width=10), overlap_view,
Div(text='', height=200, width=40))
],
[Div(text='', height=10, width=200)],
[
row(Div(text='', height=200, width=60), density,
Div(text='', height=200, width=10), errors,
Div(text='', height=200, width=40))
],
[Div(text='', height=10, width=200)],
[
row(Div(text='', height=200, width=60), weighting_layout,
Div(text='', height=200, width=40))
],
])
# Generate html resources for dashboard
fonts = os.path.join(os.getcwd(), 'fonts')
if os.path.isdir(os.path.join(session_folder, 'fonts')):
shutil.rmtree(os.path.join(session_folder, 'fonts'))
shutil.copytree(fonts, os.path.join(session_folder, 'fonts'))
else:
shutil.copytree(fonts, os.path.join(session_folder, 'fonts'))
html = file_html(layout, INLINE, "Cream")
with open(os.path.join(session_folder, f'{session_name}.html'),
"w") as file:
file.write(html)
webbrowser.open("file://" +
os.path.join(session_folder, f'{session_name}.html'))
import numpy as np
import matplotlib.image as mpimg
from astropy.table import Table
from bokeh.plotting import Figure, gridplot, output_file, show
from bokeh.models import ColumnDataSource, HBox, VBoxForm, Paragraph, Range1d
from bokeh.io import curdoc
from bokeh.models.widgets import Select, CheckboxButtonGroup, CheckboxGroup
from bokeh.models import NumeralTickFormatter
from __future__ import print_function
mjup = 317.8
p1 = Figure(x_range=[0.003, 3000], y_range=[0.01, 30000], tools="pan,save,wheel_zoom,reset",toolbar_location='above', \
title=' ', y_axis_type="log", x_axis_type='log', plot_height=400, plot_width=800)
p1.background_fill_color = "white"
p1.patch([0.003, 1.0, 0.2, 0.003, 0.003], [1, 20, 30000., 30000., 1.],
alpha=0.5,
fill_color='pink',
line_width=0)
p1.patch([0.003, 0.04, 0.04, 0.003, 0.003], [100, 100, 30000., 30000., 10000.],
alpha=0.6,
fill_color='lightblue',
line_width=0)
p1.patch([3, 3000, 3000, 30, 3], [300, 300, 30000., 30000., 300.],
alpha=0.6,
fill_color='purple',
line_width=0)
p1.yaxis.axis_label = 'Planet Mass / Earth'
p1.xaxis.axis_label = 'Semi-major Axis / Snow Line'
p1.xaxis.axis_label_text_font_style = 'bold'
p1.yaxis.axis_label_text_font_style = 'bold'
sig= Cratio/SNR
# Add gaussian noise to flux ratio
spec = Cratio + np.random.randn(len(Cratio))*sig
planet = ColumnDataSource(data=dict(lam=lam, cratio=Cratio*1e9, spec=spec*1e9, downerr=(spec-sig)*1e9, uperr=(spec+sig)*1e9))
################################
# BOKEH PLOTTING
################################
snr_plot = Figure(plot_height=400, plot_width=750,
tools="crosshair,pan,reset,resize,save,box_zoom,wheel_zoom",
x_range=[0.3, 2.7], y_range=[0, 1], toolbar_location='right')
snr_plot.x_range = Range1d(0.3, 2.7, bounds=(0.3, 2.7))
snr_plot.y_range = Range1d(0.0, 1.2, bounds=(0.3, 5.0))
snr_plot.background_fill_color = "beige"
snr_plot.background_fill_alpha = 0.5
snr_plot.yaxis.axis_label='F_p/F_s (x10^9)'
snr_plot.xaxis.axis_label='Wavelength [micron]'
snr_plot.line('lam','cratio',source=planet,line_width=2.0, color="green", alpha=0.7)
snr_plot.circle('lam', 'spec', source=planet, fill_color='red', line_color='black', size=8)
snr_plot.segment('lam', 'downerr', 'lam', 'uperr', source=planet, line_width=1, line_color='grey', line_alpha=0.5)
def change_filename(attrname, old, new):
format_button_group.active = None
instruction0 = Div(text="""Choose a file rootname here
(no special characters):""", width=300, height=15)
text_input = TextInput(value="filename", title=" ", width=100)
def custom_reports(report_id):
if report_id == 'A':
# result = db_session.execute('''select ga_date,sum(page_views),floor(dbms_random.value(2000, 6000)) as sales
# from ga_sink
# group by ga_date''' ).fetchall()
result = db_session.execute(
'''select T1.ga_date,T1.page_views, T2.total_sale
from (select ga_date,sum(page_views) as page_views from ga_sink group by ga_date) T1
join (select sale_date,sum(amount) as total_sale from demo_sales group by sale_date) T2
on T1.ga_date=T2.sale_date''').fetchall(
)
# result = db_session.execute('''select T1."date",T1.page_views, T2.total_sale
# from (select "date",sum(page_views) as page_views from test group by "date") T1
# join (select sale_date,sum(amount) as total_sale from demo_sales group by sale_date) T2
# on T1."date"=T2.sale_date''' ).fetchall()
print(result)
test = pd.DataFrame(result,
columns=['date', 'page_views', 'total_sale'])
test['date'] = pd.to_datetime(test['date'])
test.set_index(keys=['date'], inplace=True)
test.sort_index(inplace=True)
cds = ColumnDataSource(test)
p = Figure(plot_width=1000,
plot_height=500,
title="Sales Vs Views",
y_range=Range1d(start=2500, end=33000),
x_axis_type='datetime',
x_axis_label='Date',
y_axis_label='Revenue($)')
l1 = p.line('date',
'page_views',
source=cds,
line_color=d3['Category10'][10][0],
line_width=5,
legend="Page Views")
l2 = p.line('date',
'total_sale',
source=cds,
line_color=d3['Category10'][10][1],
line_width=5,
legend="Revenue")
p.extra_y_ranges = {"foo": Range1d(start=0, end=6000)}
p.add_layout(
LinearAxis(y_range_name='foo', axis_label="Number of Views"),
'right')
p.legend.location = "bottom_right"
p.background_fill_color = "beige"
p.background_fill_alpha = 0.5
p.border_fill_color = "#F8F8FF"
p.add_tools(
HoverTool(
renderers=[l1],
tooltips=[
('date',
'@date{%F}'), # use @{ } for field names with spaces
('views', '@page_views'),
],
formatters={
'date':
'datetime', # use 'datetime' formatter for 'date' field
# use default 'numeral' formatter for other fields
},
# display a tooltip whenever the cursor is vertically in line with a glyph
mode='vline'))
p.add_tools(
HoverTool(
renderers=[l2],
tooltips=[
# ( 'date', '@date{%F}' ),
('revenue', '$@{total_sale}'
), # use @{ } for field names with spaces
],
formatters={
# 'date' : 'datetime', # use 'datetime' formatter for 'date' field
'revenue':
'printf', # use 'printf' formatter for 'adj close' field
# use default 'numeral' formatter for other fields
},
# display a tooltip whenever the cursor is vertically in line with a glyph
mode='vline'))
return json.dumps(json_item(p))
if report_id == "B":
result = db_session.execute(
'''select product_id,sum(page_views) as views
from ga_sink
group by product_id
order by views desc ''').fetchall()
# result = db_session.execute('''select product_id,sum(page_views) as views
# from test
# group by product_id
# order by views desc ''' ).fetchall()
test = pd.DataFrame(result, columns=['product_id', 'page_views'])
test.set_index(keys=['product_id'], inplace=True)
cds = ColumnDataSource(test)
p = Figure(x_range=cds.data['product_id'],
plot_height=350,
title="Top Products by Views",
tools="")
p.vbar(x='product_id',
top='page_views',
source=cds,
width=0.9,
fill_color=factor_cmap(field_name='product_id',
palette=d3['Category10'][10],
factors=cds.data['product_id']))
p.xgrid.grid_line_color = None
p.y_range.start = 0
p.background_fill_color = "beige"
p.background_fill_alpha = 0.5
p.border_fill_color = "#F8F8FF"
return json.dumps(json_item(p))
if report_id == "C":
# cdata= [{'product_id':'BGB-US-001','total_sale': random.randint(1000,8000)},
# {'product_id':'BGB-US-002','total_sale': random.randint(1000,8000)},
# {'product_id':'BGB-US-003','total_sale': random.randint(1000,8000)},
# {'product_id':'BGB-US-004','total_sale': random.randint(1000,8000)},
# {'product_id':'BGB-US-005','total_sale': random.randint(1000,8000)},
# {'product_id':'BGB-US-006','total_sale': random.randint(1000,8000)},
# {'product_id':'BGB-US-007','total_sale': random.randint(1000,8000)}]
cdata = db_session.execute('''select product_id,sum(amount)
from demo_sales
group by product_id''').fetchall()
c = pd.DataFrame(cdata, columns=['product_id', 'amount'])
c.rename(columns={"amount": "total_sale"}, inplace=True)
print(c)
c.set_index(keys=['product_id'], inplace=True)
c['angle'] = c['total_sale'] / c['total_sale'].sum() * 2 * pi
c['color'] = d3['Category10'][10][len(c) - 1::-1]
c['percent'] = round(c['total_sale'] / c['total_sale'].sum() * 100, 0)
cds = ColumnDataSource(c)
p = Figure(plot_height=350,
title="Revenue Breakdown by Product",
tools="hover",
tooltips="@product_id: @percent %",
x_range=(-0.5, 1.0))
p.wedge(x=0,
y=1,
radius=0.4,
start_angle=cumsum('angle', include_zero=True),
end_angle=cumsum('angle'),
line_color="white",
fill_color='color',
legend='product_id',
source=cds)
p.axis.axis_label = None
p.axis.visible = False
p.grid.grid_line_color = None
p.background_fill_color = "beige"
p.background_fill_alpha = 0.5
p.border_fill_color = "#F8F8FF"
return json.dumps(json_item(p))
flux_cut = spec_dict[template_to_start_with].flux
flux_cut[spec_dict[template_to_start_with].wave < lumos.lambda_range[0]] = -999.
flux_cut[spec_dict[template_to_start_with].wave > lumos.lambda_range[0]] = -999.
spectrum_template = ColumnDataSource(data=dict(w=spec_dict[template_to_start_with].wave, f=spec_dict[template_to_start_with].flux, \
w0=spec_dict[template_to_start_with].wave, f0=spec_dict[template_to_start_with].flux, \
flux_cut=flux_cut, sn=signal_to_noise))
# set up the flux plot
flux_plot = Figure(plot_height=400, plot_width=800,
tools="crosshair,hover,pan,reset,resize,save,box_zoom,wheel_zoom", outline_line_color='black',
x_range=[900, 2000], y_range=[0, 4e-16], toolbar_location='right')
flux_plot.x_range=Range1d(900,2000,bounds=(900,2000))
flux_plot.y_range=Range1d(0,4e-16,bounds=(0,None))
flux_plot.background_fill_color = "beige"
flux_plot.background_fill_alpha = 0.5
flux_plot.yaxis.axis_label = 'Flux'
flux_plot.xaxis.axis_label = 'Wavelength'
flux_plot.line('w', 'f', source=spectrum_template, line_width=3, line_color='firebrick', line_alpha=0.7, legend='Source Flux')
flux_plot.line(lumos.wave, lumos.bef, line_width=3, line_color='darksalmon', line_alpha=0.7, legend='Background')
# set up the flux plot
sn_plot = Figure(plot_height=400, plot_width=800,
tools="crosshair,hover,pan,reset,resize,save,box_zoom,wheel_zoom", outline_line_color='black',
x_range=[900, 2000], y_range=[0, 40], toolbar_location='right')
sn_plot.x_range=Range1d(900,2000,bounds=(900,2000))
sn_plot.y_range=Range1d(0,40,bounds=(0,None))
def plot_carto_single(self, data, frente, palette, path=FILE_OUT,
name_file="", low=0, high=100, show_plot=True):
"""
:param data: df loaded by data_load
:param frente: string, name of "partido" lowercase: diff, mas, cc, creemos, fpv, pan_bol
:param palette: ej: P_GRAD_CC
:param name_file: default:test
:param low: cmap low limit: default: -80
:param high: cmap high limit: defauilt: +80.
:param path: file out
:return: df
"""
da_col = ['HAB','PAIS','MUN','REC','X','Y','LAT','LON','x','y',
'r','r2','GX','GY'
]
cart_init_val = self.CART_SLIDER_INIT # add slider
self.process_data(cart_init_val, data)
if frente == "diff":
low = self.C_BAR_LOW
high = self.C_BAR_HIGH
frente = "d_mas_cc"
f1 = 'mas_o_cc'
f2 = 'ad_mas_cc'
_p = 'mas'
_p1 = 'cc'
da_col.append(frente)
da_col.append(f1)
da_col.append(f2)
da_col.append(_p)
da_col.append(_p1)
if frente == "d_mas_creemos":
low = self.C_BAR_LOW
high = self.C_BAR_HIGH
f1 = 'mas_o_creemos'
f2 = 'ad_mas_creemos'
da_col.append(frente)
da_col.append(f1)
da_col.append(f2)
da_col.append('mas')
da_col.append('creemos')
da_col.append(frente)
cm = linear_cmap(frente, palette=palette, low=low, high=high)
data = data[da_col]
source_master = ColumnDataSource(data)
source_red_map = ColumnDataSource({'gx': [], 'gy': []})
# la, lo = ebu.get_la_lo_bolivia()
# source_bol = ColumnDataSource({'la': la, 'lo': lo})
# source_red_car = ColumnDataSource({'lo': [], 'la': []})
# JS CODE
code_draw_red_map = """
const data = {'gx': [], 'gy': []}
const indices = cb_data.index.indices
for (var i = 0; i < indices.length; i++ ) {
data['gx'].push(source_master.data.GX[indices[i]])
data['gy'].push(source_master.data.GY[indices[i]])
}
source_red_map.data = data
"""
code_slider = """
var data = source.data;
var f = cb_obj.value
var x = data['x']
var y = data['y']
var Y = data['Y']
var X = data['X']
var lat = data['LAT']
var lon = data['LON']
for (var i = 0; i < x.length; i++) {
y[i] = (1-f)*lat[i] + f*Y[i]
x[i] = (1-f)*lon[i] + f*X[i]
}
source.change.emit();
"""
# FIGURES
curr_time = ebu.get_bolivian_time(-3)
pw = self.FIG_WIDTH
callback_red_map = CustomJS(
args={'source_master': source_master,
'source_red_map': source_red_map, },
code=code_draw_red_map)
hover_cart = bokeh.models.HoverTool(
tooltips=self.TOOL_TIP_DIC[frente],
callback=callback_red_map,
# renderers = [red_scat_car]
)
cart_fig = Figure(plot_width=pw, plot_height=pw,
output_backend="webgl", )
cart_fig.background_fill_color = "grey"
cart_fig.background_fill_alpha = .5
cart_fig.scatter('x', 'y', source=source_master, radius='r', color=cm)
cart_fig.add_tools(hover_cart, )
title = "Última actualización: " + curr_time["datetime_val"].strftime(
"%Y-%m-%d %H:%M") + "BOT"
map_fig = Figure(plot_width=pw, plot_height=pw,
x_axis_type='mercator',
y_axis_type='mercator',
output_backend="webgl",
title=title,
)
# cb_fig = bokeh.plotting.Figure(plot_height=pw,plot_width=)
# cb_fig.toolbar.logo = None
# cb_fig.toolbar_location = None
# SCATTER
# noinspection PyUnresolvedReferences
# add tiles
tile_provider = bokeh.tile_providers.get_provider(
bokeh.tile_providers.Vendors.CARTODBPOSITRON)
map_fig.add_tile(tile_provider)
# scatter in map
map_fig.scatter(
'GX', 'GY', source=source_master, size='r2',
color=cm
)
# todo if we wont use map then we nee to delete the source
# cart_fig.line('lo', 'la', source=source_bol, color='black')
# noinspection PyUnusedLocal
red_scat_map = map_fig.circle_cross('gx', 'gy',
source=source_red_map,
fill_color=None,
size=20,
line_color="white",
line_width=4
)
# noinspection PyUnusedLocal
red_scat_map = map_fig.circle_cross('gx', 'gy',
source=source_red_map,
fill_color=None,
size=20,
line_color="red",
line_width=1
)
# red_scat_car = cart_fig.scatter('lo', 'la',
# source=source_red_car, color='green')
# add a hover tool that sets the link data for a hovered circle
# callbacks
# code = code_merged)
# callback_red_car = CustomJS(
# args={'source_master': source_master, 'source_red_car': source_red_car},
# code=code_draw_red_car)
# tools
hover_map = bokeh.models.HoverTool(
tooltips=self.TOOL_TIP_DIC[frente],
# callback=callback_red_car,
# renderers = [red_scat_map]
)
map_fig.add_tools(hover_map, )
# slider
callback_slider = CustomJS(args=dict(source=source_master),
code=code_slider)
slider = Slider(start=0, end=1, value=cart_init_val, step=.02,
title="carto")
slider.js_on_change('value', callback_slider)
# COLOR BAR
ml = {int(i): str(np.abs(i)) for i in np.arange(-80, 81, 20)}
cb = bokeh.models.ColorBar(
color_mapper=cm['transform'],
# width=int(.9 * 450),
width='auto',
location=(0, 0),
# title="DEN (N/km^2)",
# title=(BAR_TITLE),
# margin=0,padding=0,
title_standoff=10,
# ticker=bokeh.models.LogTicker(),
orientation='horizontal',
major_label_overrides=ml
)
cart_fig.add_layout(cb, 'above')
# cb.title_text_align = 'left'
cart_fig.title.text = self.BAR_TITLE_DIC[frente]
cart_fig.title.align = 'center'
# layout = row(column(slider, cart_f),map_f)
layout = bokeh.layouts.gridplot(
[[slider, None], [cart_fig, map_fig]], sizing_mode='scale_width',
merge_tools=False)
layout.max_width = 1400
# layout = bokeh.layouts.column([slider, cart_fig])
cart_fig.x_range.start = self.CXS
cart_fig.x_range.end = self.CXE
cart_fig.y_range.start = self.CYS
cart_fig.y_range.end = self.CYE
_ll = ebu.lola_to_cart(lo=[self.MXS, self.MXE], la=[self.MYS, self.MYE])
map_fig.x_range.start = _ll[0][0]
map_fig.x_range.end = _ll[0][1]
map_fig.y_range.start = _ll[1][0]
map_fig.y_range.end = _ll[1][1]
cart_fig.xaxis.major_tick_line_color = None
# turn off x-axis major ticks
cart_fig.xaxis.minor_tick_line_color = None
# turn off x-axis minor ticks
cart_fig.yaxis.major_tick_line_color = None
# turn off y-axis major ticks
cart_fig.yaxis.minor_tick_line_color = None
cart_fig.xaxis.major_label_text_font_size = '0pt'
# turn off x-axis tick labels
cart_fig.yaxis.major_label_text_font_size = '0pt'
# turn off y-axis tick labels
nam = 'z037_' + frente + '_' + name_file + '.html'
nam_lat = 'z037_' + frente + '_' + 'latest' + '.html'
nam1 = os.path.join(path, nam)
nam2 = os.path.join(os.path.dirname(ebu.DIR), 'docs',
'graficas_htmls',
nam_lat)
# bokeh.plotting.output_file(nam2)
if show_plot:
bokeh.plotting.show(layout)
bokeh.plotting.save(layout, nam1)
bokeh.plotting.save(layout, nam2)
return data
from bokeh.models import ColumnDataSource, Slider, VBox, HBox, Select
from bokeh.plotting import ColumnDataSource, curdoc, Figure
from bokeh.driving import count
BUFSIZE = 200
MA12, MA26, EMA12, EMA26 = '12-tick Moving Avg', '26-tick Moving Avg', '12-tick EMA', '26-tick EMA'
source = ColumnDataSource(dict(
time=[], average=[], low=[], high=[], open=[], close=[],
ma=[], macd=[], macd9=[], macdh=[], color=[]
))
p = Figure(plot_width=1200, plot_height=600, toolbar_location="left")
p.x_range.follow = "end"
p.x_range.follow_interval = 100
p.background_fill_color = "#eeeeee"
p.line(x='time', y='average', alpha=0.2, line_width=3, color='navy', source=source)
p.line(x='time', y='ma', alpha=0.8, line_width=2, color='orange', source=source)
p.segment(x0='time', y0='low', x1='time', y1='high', source=source, color='black', line_width=2)
p.segment(x0='time', y0='open', x1='time', y1='close', line_width=8, color='color', source=source)
p2 = Figure(plot_width=1200, plot_height=250, toolbar_location="left", tools="pan", x_range=p.x_range)
p2.line(x='time', y='macd', color='red', source=source)
p2.line(x='time', y='macd9', color='blue', source=source)
p2.segment(x0='time', y0=0, x1='time', y1='macdh', line_width=6, color='black', alpha=0.5, source=source)
mean = Slider(title="mean", value=0, start=-0.01, end=0.01, step=0.001)
stddev = Slider(title="stddev", value=0.04, start=0.01, end=0.1, step=0.01)
mavg = Select(value=MA12, options=[MA12, MA26, EMA12, EMA26])
source.change.emit();
"""
# FIGURES
pw = 700
cart_fig = Figure(plot_width=pw + int(.2 * pw),
plot_height=pw,
output_backend="webgl")
map_fig = Figure(
plot_width=pw,
plot_height=pw,
x_axis_type='mercator',
y_axis_type='mercator',
output_backend="webgl",
)
cart_fig.background_fill_color = "grey"
cart_fig.background_fill_alpha = .5
# cb_fig = bokeh.plotting.Figure(plot_height=pw,plot_width=)
# cb_fig.toolbar.logo = None
# cb_fig.toolbar_location = None
# %%
# SCATTER
# noinspection PyUnresolvedReferences
# add tiles
tile_provider = bokeh.tile_providers.get_provider(
bokeh.tile_providers.Vendors.CARTODBPOSITRON)
map_fig.add_tile(tile_provider)
# scatter in map
def templateAcceptedLoanPerRegion():
LABELS = ["Central", "Mid - Atlantic", "NorthEast", "NorthWest", "South", "SouthEast", "SouthWest"]
colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#17becf']
data = pd.read_csv(DATA + 'perc_acc_loan_per_region_date_compact.csv')
central = data['Central'] / 100
midatl = data['Mid-Atlantic'] / 100
northe = data['Northeast'] / 100
northw = data['Northwest'] / 100
south = data['South'] / 100
southe = data['Southeast'] / 100
southw = data['Southwest'] / 100
date = data['date']
source = ColumnDataSource(data=dict(x=[datetime.strptime(d, '%b-%Y') for d in date.values],
Central=central,
MidAtlantic=midatl,
Northeast=northe,
Northwest=northw,
South=south,
Southeast=southe,
Southwest=southw
))
props = dict(line_width=4, line_alpha=0.8)
p = Figure(x_axis_type="datetime", width=1200, height=380)
p0 = p.line('x', 'Central', source=source, legend="Central", line_color=colors[0], **props)
p1 = p.line('x', 'MidAtlantic', source=source, legend="Mid - Atlantic", line_color=colors[1], **props)
p2 = p.line('x', 'Northeast', source=source, legend="NorthEast", line_color=colors[2], **props)
p3 = p.line('x', 'Northwest', source=source, legend="NorthWest", line_color=colors[3], **props)
p4 = p.line('x', 'South', source=source, legend="South", line_color=colors[4], **props)
p5 = p.line('x', 'Southeast', source=source, legend="SouthEast", line_color=colors[5], **props)
p6 = p.line('x', 'Southwest', source=source, legend="SouthWest", line_color=colors[6], **props)
p.yaxis.axis_label = 'Percentage of accepted loans'
p.yaxis[0].formatter = NumeralTickFormatter(format="0.0%")
p.border_fill_color = LIGHT_GREEN
p.background_fill_color = LIGHT_GREEN
p.legend.background_fill_color = LIGHT_GREEN
p.legend.background_fill_alpha = 0.5
checkbox = CheckboxGroup(
labels=LABELS,
inline=True,
active=[0, 1, 2, 3, 4, 5, 6],
width=800)
code = """
//console.log(cb_obj.active);
p0.visible = false;
p1.visible = false;
p2.visible = false;
p3.visible = false;
p4.visible = false;
p5.visible = false;
p6.visible = false;
for (i in checkbox.active) {
//console.log(cb_obj.active[i]);
if (checkbox.active[i] == 0) {
p0.visible = true;
} else if (checkbox.active[i] == 1) {
p1.visible = true;
} else if (checkbox.active[i] == 2) {
p2.visible = true;
} else if (checkbox.active[i] == 3) {
p3.visible = true;
} else if (checkbox.active[i] == 4) {
p4.visible = true;
} else if (checkbox.active[i] == 5) {
p5.visible = true;
} else if (checkbox.active[i] == 6) {
p6.visible = true;
}
}
"""
checkbox.callback = CustomJS(args=dict(p0=p0, p1=p1, p2=p2, p3=p3, p4=p4, p5=p5, p6=p6, checkbox=checkbox),
code=code)
boundaries = open(DATA + 'boundaries.json').read()
states = json.loads(boundaries)
region_state = pd.read_csv(DATA + 'region-state.csv', header=0)
region_state = region_state.set_index('state')
state_xs = [states[code]["lons"] for code in states]
state_ys = [states[code]["lats"] for code in states]
name = states.keys()
colors_state = []
for i in name:
if i != 'AK' and i != 'HI':
reg = region_state.loc[i]['region']
if reg == "Central":
colors_state.append(colors[0])
elif reg == "Mid-Atlantic":
colors_state.append(colors[1])
elif reg == "Northeast":
colors_state.append(colors[2])
elif reg == "Northwest":
colors_state.append(colors[3])
elif reg == "South":
colors_state.append(colors[4])
elif reg == "Southeast":
colors_state.append(colors[5])
elif reg == "Southwest":
colors_state.append(colors[6])
source = ColumnDataSource(data=dict(
x=state_xs,
y=state_ys,
name=name,
colors=colors_state,
))
q = figure(title="",
toolbar_location=None,
plot_width=300,
plot_height=160
)
q.xaxis.visible = False
q.yaxis.visible = False
q.xgrid.grid_line_color = None
q.ygrid.grid_line_color = None
q.min_border_left = False
q.min_border_right = False
q.min_border_top = False
q.min_border_bottom = False
q.border_fill_color = LIGHT_GREEN
q.background_fill_color = LIGHT_GREEN
q.patches('x', 'y', source=source,
fill_color='colors',
fill_alpha=0.9, line_color="white", line_width=0.1)
layout = VBox(q, checkbox, p)
# show(layout)
script, div = components(layout)
return script, div
from bokeh.io import curdoc
from bokeh.models import ColumnDataSource, Legend
from bokeh.plotting import Figure
POINTS_TO_KEEP = 50
doc = curdoc()
source = ColumnDataSource(dict(x=[], avg_fitness=[], min_fitness=[]))
fig = Figure(tools="",
title="Mean and minimum fitness value of each generation",
plot_width=800)
fig.title.text_font_size = "12pt"
fig.title.align = "center"
fig.background_fill_color = "beige"
fig.background_fill_alpha = 0.5
l1 = fig.line(source=source,
x="x",
y="avg_fitness",
line_width=2,
alpha=0.85,
color="red")
l2 = fig.line(
source=source,
x="x",
y="min_fitness",
line_width=2,
alpha=0.85,
color="blue",
)
for xx in np.arange(17) * 10. - 80:
print(xx)
plot1.line([xx, xx], [-50, 50], line_width=1, line_color='grey')
plot1.line([-75, 75], [xx, xx], line_width=1, line_color='grey')
# set up the cluster vmax vs. mass plot
plot2 = Figure(plot_height=400,
plot_width=450,
tools="pan,reset,wheel_zoom,hover,save",
outline_line_color='black',
x_range=[2, 7],
y_range=[80, 1500],
y_axis_type='log',
toolbar_location='right',
title=' Outflow Properties')
plot2.background_fill_color = "beige"
plot2.title.text_font_size = '14pt'
plot2.title.align = 'center'
plot2.background_fill_alpha = 0.5
plot2.yaxis.axis_label = 'Max Outflow Velocity [km/s]'
plot2.xaxis.axis_label = 'log(Cluster Mass)'
plot2.xaxis.axis_line_color = 'black'
plot2.yaxis.axis_line_color = 'black'
plot2.border_fill_color = "white"
plot2.min_border_left = 100
cluster_points = plot2.square('mass',
'vmax',
source=shutter_positions,
size=15,
color='grey',
"""
# Create plot -------------------------------
xmin = -8240227.037
ymin = 4974203.152
xmax = -8231283.905
ymax = 4979238.441
path = './data/projected.tif'
fig = Figure(x_range=(xmin, xmax),
y_range=(ymin, ymax),
plot_height=600,
plot_width=900,
tools='pan,wheel_zoom')
fig.background_fill_color = 'black'
fig.add_tile(STAMEN_TONER, alpha=0) # used to set axis ranges
fig.x_range.callback = CustomJS(code=dims_jscode, args=dict(plot=fig, dims=dims))
fig.y_range.callback = CustomJS(code=dims_jscode, args=dict(plot=fig, dims=dims))
fig.axis.visible = False
fig.grid.grid_line_alpha = 0
fig.min_border_left = 0
fig.min_border_right = 0
fig.min_border_top = 0
fig.min_border_bottom = 0
image_source = ColumnDataSource(dict(image=[], x=[], y=[], dw=[], dh=[]))
fig.image_rgba(source=image_source,
image='image',
x='x',
y='y',
def plot_carto_single(self,
data,
frente,
palette,
name_file="test.html",
low=0,
high=100):
"""
:param data: df loaded by data_load
:param frente: string, name of "partido" lowercase: diff, mas, cc, creemos, fpv, panbol
:param palette: ej: P_GRAD_CC
:param name_file: default:test
:param low: cmap low limit: default: -80
:param high: cmap high limit: defauilt: +80.
:return: df
"""
if frente == "diff":
low = self.C_BAR_LOW
high = self.C_BAR_HIGH
frente = "d_mas_cc"
bokeh.plotting.output_file(self.FILE_OUT + '_' + frente + '_' +
name_file)
cart_init_val = self.CART_SLIDER_INIT # add slider
data['x'] = data['LON'] * (1 -
cart_init_val) + data['X'] * cart_init_val
data['y'] = data['LAT'] * (1 -
cart_init_val) + data['Y'] * cart_init_val
cm = linear_cmap(frente, palette=palette, low=low, high=high)
data['mas'] = data['MAS'] / data['VV'] * 100
data['cc'] = data['CC'] / data['VV'] * 100
data['pdc'] = data['PDC'] / data['VV'] * 100
#data['creemos'] = data['CREEMOS'] / data['VV'] * 100
#data['fpv'] = data['FPV'] / data['VV'] * 100
#data['panbol'] = data['PANBOL'] / data['VV'] * 100
data['ad_mas_cc'] = data['d_mas_cc'].abs()
data['mas_o_cc'] = 'n'
data.loc[data['d_mas_cc'] >= 0, 'mas_o_cc'] = 'MAS'
data.loc[data['d_mas_cc'] < 0, 'mas_o_cc'] = 'CC'
source_master = ColumnDataSource(data)
source_red_map = ColumnDataSource({'gx': [], 'gy': []})
la, lo = ebu.get_la_lo_bolivia()
source_bol = ColumnDataSource({'la': la, 'lo': lo})
# source_red_car = ColumnDataSource({'lo': [], 'la': []})
# JS CODE
code_draw_red_map = """
const data = {'gx': [], 'gy': []}
const indices = cb_data.index.indices
for (var i = 0; i < indices.length; i++ ) {
data['gx'].push(source_master.data.GX[indices[i]])
data['gy'].push(source_master.data.GY[indices[i]])
}
source_red_map.data = data
"""
code_draw_red_car = """
const data = {'lo': [], 'la': []}
const indices = cb_data.index.indices
for (var i = 0; i < indices.length; i++) {
data['lo'].push(source_master.data.x[indices[i]])
data['la'].push(source_master.data.y[indices[i]])
}
source_red_car.data = data
"""
code_merged = """
const data_map = {'lo': [], 'la': []}
const data_car = {'gx': [], 'gy': []}
const indices = cb_data.index.indices
for (var i = 0; i < indices.length; i++) {
data_map['lo'].push(source_master.data.x[indices[i]])
data_map['la'].push(source_master.data.y[indices[i]])
data_car['gx'].push(source_master.data.GX[indices[i]])
data_car['gy'].push(source_master.data.GY[indices[i]])
}
source_red_car.data = data_car
source_red_map.data = data_map
"""
code_slider = """
var data = source.data;
var f = cb_obj.value
var x = data['x']
var y = data['y']
var Y = data['Y']
var X = data['X']
var lat = data['LAT']
var lon = data['LON']
for (var i = 0; i < x.length; i++) {
y[i] = (1-f)*lat[i] + f*Y[i]
x[i] = (1-f)*lon[i] + f*X[i]
}
source.change.emit();
"""
# FIGURES
pw = self.FIG_WIDTH
cart_fig = Figure(plot_width=pw,
plot_height=pw,
output_backend="webgl")
map_fig = Figure(
plot_width=pw,
plot_height=pw,
x_axis_type='mercator',
y_axis_type='mercator',
output_backend="webgl",
)
cart_fig.background_fill_color = "grey"
cart_fig.background_fill_alpha = .5
# cb_fig = bokeh.plotting.Figure(plot_height=pw,plot_width=)
# cb_fig.toolbar.logo = None
# cb_fig.toolbar_location = None
# SCATTER
# noinspection PyUnresolvedReferences
# add tiles
tile_provider = bokeh.tile_providers.get_provider(
bokeh.tile_providers.Vendors.CARTODBPOSITRON)
map_fig.add_tile(tile_provider)
# scatter in map
map_fig.scatter('GX', 'GY', source=source_master, size='r2', color=cm)
# todo if we wont use map then we nee to delete the source
# cart_fig.line('lo', 'la', source=source_bol, color='black')
cart_fig.scatter('x', 'y', source=source_master, radius='r', color=cm)
red_scat_map = map_fig.circle_cross(
'gx',
'gy',
source=source_red_map,
# color='red',
fill_color=None,
# line_color='green',
size=20,
line_color="white",
line_width=4)
red_scat_map = map_fig.circle_cross(
'gx',
'gy',
source=source_red_map,
# color='red',
fill_color=None,
# line_color='green',
size=20,
line_color="red",
line_width=1)
# red_scat_car = cart_fig.scatter('lo', 'la',
# source=source_red_car, color='green')
# add a hover tool that sets the link data for a hovered circle
# callbacks
callback_red_map = CustomJS(
args={
'source_master': source_master,
'source_red_map': source_red_map,
# 'source_red_car':source_red_car
},
code=code_draw_red_map)
# code = code_merged)
# callback_red_car = CustomJS(
# args={'source_master': source_master, 'source_red_car': source_red_car},
# code=code_draw_red_car)
# tools
hover_cart = bokeh.models.HoverTool(
tooltips=self.TOOL_TIP_DIC[frente],
callback=callback_red_map,
# renderers = [red_scat_car]
)
cart_fig.add_tools(hover_cart, )
hover_map = bokeh.models.HoverTool(
tooltips=self.TOOL_TIP_DIC[frente],
# callback=callback_red_car,
# renderers = [red_scat_map]
)
map_fig.add_tools(hover_map, )
# slider
callback_slider = CustomJS(args=dict(source=source_master),
code=code_slider)
slider = Slider(start=0,
end=1,
value=cart_init_val,
step=.02,
title="carto")
slider.js_on_change('value', callback_slider)
# COLOR BAR
ml = {int(i): str(np.abs(i)) for i in np.arange(-80, 81, 20)}
cb = bokeh.models.ColorBar(
color_mapper=cm['transform'],
width=int(.9 * self.FIG_WIDTH),
location=(0, 0),
# title="DEN (N/km^2)",
# title=(BAR_TITLE),
# margin=0,padding=0,
title_standoff=10,
# ticker=bokeh.models.LogTicker(),
orientation='horizontal',
major_label_overrides=ml)
cart_fig.add_layout(cb, 'above')
# cb.title_text_align = 'left'
cart_fig.title.text = self.BAR_TITLE_DIC[frente]
cart_fig.title.align = 'center'
# layout = row(column(slider, cart_f),map_f)
layout = bokeh.layouts.gridplot([[slider, None], [cart_fig, map_fig]],
merge_tools=False)
# layout = bokeh.layouts.column([slider, cart_fig])
cart_fig.x_range.start = self.CXS
cart_fig.x_range.end = self.CXE
cart_fig.y_range.start = self.CYS
cart_fig.y_range.end = self.CYE
_ll = ebu.lola_to_cart(lo=[self.MXS, self.MXE],
la=[self.MYS, self.MYE])
map_fig.x_range.start = _ll[0][0]
map_fig.x_range.end = _ll[0][1]
map_fig.y_range.start = _ll[1][0]
map_fig.y_range.end = _ll[1][1]
cart_fig.xaxis.major_tick_line_color = None # turn off x-axis major ticks
cart_fig.xaxis.minor_tick_line_color = None # turn off x-axis minor ticks
cart_fig.yaxis.major_tick_line_color = None # turn off y-axis major ticks
cart_fig.yaxis.minor_tick_line_color = None
cart_fig.xaxis.major_label_text_font_size = '0pt' # turn off x-axis tick labels
cart_fig.yaxis.major_label_text_font_size = '0pt' # turn off y-axis tick labels
bokeh.plotting.show(layout)
return data