def display_tf_code(func):
code = tf.autograph.to_code(func)
from IPython.display import display, Markdown
display(Markdown('```python\n{}\n```'.format(code)))
def show(self):
'''
'''
display(Markdown(self.body))
def markdown(self, line, cell):
"""Render the cell as Markdown text block"""
display(Markdown(cell))
def event_table(self, **_args):
"""Print out event table in Markdown format."""
return Markdown(self.event_table_text(**_args))
def show_response(self):
response_url = 'https://' + self.url + '/download/markdown'
display(Markdown(response_url))
def confusion_matrix_information(self, y_test, y_pred):
cm = pd.crosstab(y_test,
y_pred,
rownames=['Known OBEC URLs'],
colnames=['Predicted OBEC URLs'])
TP = cm.iloc[1][1]
TN = cm.iloc[0][0]
FP = cm.iloc[0][1]
FN = cm.iloc[1][0]
display(Markdown('**Confusion Matrix Information:**'))
print('OBEC URLs that are predicted as False and we do not ' \
'know (True Negatives):', TN)
print('OBEC URLs that are predicted as False and we do know ' \
'(False Negatives):', FN)
print('OBEC URLs that are predicted as True and we do not ' \
'know (False Positives):', FP)
print('OBEC URLs that are predicted as True and we do know ' \
'(True Positives):', TP)
print('Accuracy = (TP+TN)/(TP+TN+FP+FN):', '{0:.2}'.format(
(TP + TN) / (TP + TN + FP + FN)))
print('Precision = TP/(TP+FP):', '{0:.2}'.format(TP / (TP + FP)))
print('Recall = TP/(TP+FN):', '{0:.2}'.format(TP / (TP + FN)))
print('Specificity = TN/(TN+FP):', '{0:.2}'.format(TN / (TN + FP)))
print('F1 Score = 2*TP/(2*TP+FP+FN):',
'{0:.2}'.format(2 * TP / (2 * TP + FP + FN)))
print(
'MCC = (TP*TN-FP*FN)/math.sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN)):',
'{0:.2}'.format((TP * TN - FP * FN) / math.sqrt(
(TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))))
display(Markdown('**Confusion Matrix Explanations:**'))
print('Accuracy - This is simply equal to the proportion ' \
'of predictions that the model classified correctly.')
print('Precision - Precision is also known as positive ' \
'predictive value and is the proportion of relevant ' \
'instances among the retrieved instances. In other ' \
'words, it answers the question "What proportion of ' \
'positive identifications was actually correct?".')
print('Recall - Recall, also known as the sensitivity, hit ' \
'rate, or the true positive rate (TPR), is the ' \
'proportion of the total amount of relevant instances ' \
'that were actually retrieved. It answers the question ' \
'"What proportion of actual positives was identified ' \
'correctly?".')
print('Specificity - Specificity, also known as the true ' \
'negative rate (TNR), measures the proportion of actual ' \
'negatives that are correctly identified as such. It is ' \
'the opposite of recall.')
print('F1 Score - The F1 score is a measure of a test\'s ' \
'accuracy — it is the harmonic mean of precision and ' \
'recall. It can have a maximum score of 1 (perfect ' \
'precision and recall) and a minimum of 0. Overall, ' \
'it is a measure of the preciseness and robustness ' \
'of your model.')
print('Matthews correlation coefficient (MCC) - The MCC is ' \
'in essence a correlation coefficient between the ' \
'observed and predicted binary classifications; it ' \
'returns a value between -1 and +1. A coefficient ' \
'of +1 represents a perfect prediction, 0 no better ' \
'than random prediction and -1 indicates total ' \
'disagreement between prediction and observation. ' \
'The statistic is also known as the phi coefficient. ' \
'MCC is related to the chi-square statistic for a 2x2 ' \
'contingency table.')
def FlyingCameraNetworkDisplay():
import random
import pandas as pd
import pickle as pkl
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
from random import randrange
import random
from ipywidgets import interactive, interact , FloatSlider,widgets,VBox,HBox,interactive_output,Layout,Output,GridBox
from IPython.display import display,HTML,clear_output,Markdown
from scipy.interpolate import UnivariateSpline
clear_output()
print("loading...(Might take a minute)")
#Network layers
def hyperspectral(input_,n_o_input, keep_prob, filter_width = 1, stride_size =1, relu_alpha = 0.2):
n_o_strides = int((n_o_input-filter_width)/stride_size) +1 #round down
Hyper_layer = []
def dense(input_,start,keep_prob, filter_width, stride_size, relu_alpha):
nn_input = tf.slice(input_,[0,start],[-1,filter_width])
dropout1 = tf.nn.dropout(nn_input, keep_prob)
dense1 = tf.layers.dense(dropout1, 1)
relu1 = tf.maximum(relu_alpha * dense1, dense1)
return relu1
for step in range(n_o_strides):
start = step*stride_size
output = dense(input_,start,keep_prob, filter_width, stride_size, relu_alpha)
Hyper_layer.append(output)
if (n_o_input-filter_width)%stride_size>0:
start = n_o_input-filter_width
output = dense(input_,start,keep_prob, filter_width, stride_size, relu_alpha)
Hyper_layer.append(output)
Hyper_l_stacked = tf.concat(Hyper_layer,1)
return Hyper_l_stacked , n_o_strides
def Classifier(input_,n_o_class,n_o_input, keep_prob,relu_alpha = 0.2):
if n_o_input == 3:
is_RGB = True
elif n_o_input == 571:
is_RGB = False
else:
raise ValueError('A very specific bad thing happened.'+str(n_o_input))
if is_RGB:
dense0 = tf.layers.dense(input_, 3)
relu0 = tf.maximum(relu_alpha * dense0, dense0)
first_layer_out = tf.nn.dropout(relu0, keep_prob)
else:
first_layer_out,n_o_input= hyperspectral(input_,n_o_input, keep_prob, filter_width = 30, stride_size =1, relu_alpha = 0.2)
hidden_size = n_o_input*2/3
hidden_nodes = int(hidden_size)+1 # rounding
dense1 = tf.layers.dense(first_layer_out, hidden_nodes)
relu1 = tf.maximum(relu_alpha * dense1, dense1)
dropout1 = tf.nn.dropout(relu1, keep_prob)
class_logits = tf.layers.dense(dropout1, n_o_class)
return class_logits
def softmax(input_,m_class, n_class,n_o_input, keep_prob,relu_alpha = 0.2,sub_scaling = 1):
n_o_class = m_class+n_class
#raw output
logits= Classifier(input_,n_o_class,n_o_input, keep_prob,relu_alpha = 0.2)
subclass_softmax = tf.nn.softmax(logits)
#Reduce outputs from 8 subclasses to 2 main classes
m_class_logit, n_class_logit = tf.split(logits, [m_class, n_class], 1)
m_class_logit1 =tf.reduce_sum(m_class_logit,1, keepdims =True)
n_class_logit1 =tf.reduce_sum(n_class_logit,1, keepdims =True)
main_class_logits = tf.concat([m_class_logit1, n_class_logit1], 1)
main_class_softmax = tf.nn.softmax(main_class_logits)
return subclass_softmax,main_class_softmax
#Defining the training parameters
tf.reset_default_graph()
n_class = 3
m_class = 5
n_o_class = m_class+n_class
epochs = 10
keep_probability = 0.95
sub_scaling = 1
g1 = tf.Graph()
with g1.as_default() as g_1:
H_keep_prob = tf.placeholder(tf.float32)
H_n_o_input = 571
H_input_ = tf.placeholder(tf.float32, [None,H_n_o_input])
H_subclass_softmax,H_main_class_softmax =softmax(H_input_,m_class, n_class,H_n_o_input, H_keep_prob,relu_alpha = 0.2,sub_scaling = sub_scaling)
#Relectance spectrum
g2 = tf.Graph()
with g2.as_default() as g_2:
R_keep_prob = tf.placeholder(tf.float32)
R_n_o_input = 3
R_input_ = tf.placeholder(tf.float32, [None,R_n_o_input])
R_subclass_softmax,R_main_class_softmax =softmax(R_input_,m_class, n_class,R_n_o_input, R_keep_prob,relu_alpha = 0.2,sub_scaling = sub_scaling)
#Display network result
clear_output()
display(Markdown("## View Classification" ))
RGB = []
hyperspectral = []
box1 = Output(layout=Layout(width='100%',border ='1px solid #E0E0E0',padding = '0px 0px 0px 20px',fill = "black"))
box2 = Output(layout=Layout(width='100%',border ='1px solid #E0E0E0',padding = '0px 20px 0px 20px'))
box3 = Output(layout=Layout(width='100%',border ='1px solid #E0E0E0',padding = '0px 0px 0px 20px'))
box4 = Output(layout=Layout(width='100%',border ='1px solid #E0E0E0',padding = '0px 0px 0px 20px'))
box5 = Output(layout=Layout(width='100%',border ='1px solid #E0E0E0',padding = '0px 20px 0px 20px'))
box6 = Output(layout=Layout(width='100%',border ='1px solid #E0E0E0',padding = '0px 20px 0px 20px'))
markdown = Output(layout=Layout(width='95%'))
with markdown:
display(Markdown("#### Adjust The Wavelengths"))
display(Markdown("<sup>To Customise Your Own RGB & Hyperspectral Input</sup>"))
slider1 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='380nm',layout= Layout(width='99%'))
slider2 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='450nm',layout= Layout(width='99%'))
slider3 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='550nm',layout= Layout(width='99%'))
slider4 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='650nm',layout= Layout(width='99%'))
slider5 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='750nm',layout= Layout(width='99%'))
slider6 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='850nm',layout= Layout(width='99%'))
slider7 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='950nm',layout= Layout(width='99%'))
def click(event):
global hyperspectral
global RGB
classify(RGB,hyperspectral)
button = widgets.Button(
description='Run Classification',
layout={'width': '130px','margin':'0px 0px 10px 15%'}
)
button.on_click(click)
slider_list = [markdown,slider1,slider2,slider3,slider4,slider5,slider6,slider7,button]
for slider in slider_list:
with box3:
display(slider)
def classify(RGB,hyperspectral):
keep_probability = 1
def print_network_output(save_model_path,title,subclass_softmax,main_class_softmax,input_,reflectance_data,n_o_input,g,keep_prob):
clear_output(wait=True)
print("""Loading...(Might take a minute)
*The next classification doesn't load until this one is completed""")
with tf.Session(graph=g) as sess:
sess.run(tf.global_variables_initializer())
loader = tf.train.import_meta_graph(save_model_path + '.meta')
loader.restore(sess, save_model_path)
subclass_softmax_p,main_class_softmax_p= sess.run([subclass_softmax,main_class_softmax], feed_dict = {input_:reflectance_data,keep_prob:keep_probability})
clear_output(wait=True)
display(Markdown("#### "+title))
display(Markdown("##### Main Class"))
main_class_softmax_p = main_class_softmax_p[0]
display(Markdown("Non-scat: "+str(main_class_softmax_p[0])))
display(Markdown("Scat : "+str(main_class_softmax_p[1])))
display(Markdown("##### Subclass"))
subclass_softmax_p = subclass_softmax_p[0]
subclass_name = [ "Leaf Litter ",
"Ground ",
"Wood ",
"Bird Scat ",
"Mammal Scat ",
"Amphibian Scat",
"Reptile Scat ",
"Reptile Urea "]
for subclass,name in zip(subclass_softmax_p,subclass_name):
display(Markdown(name+": " +str(subclass)))
with box4:
print_network_output('../training_rgb',
"RGB Network Classification",
R_subclass_softmax,
R_main_class_softmax,
R_input_,[RGB],
3,
g_2,
R_keep_prob)
with box5:
print_network_output('../training_Hyperspectral',
"Hyperspectral Network Classification",
H_subclass_softmax,
H_main_class_softmax,
H_input_,
[hyperspectral],
571,
g_1,
H_keep_prob)
def f(z,a,b,c,d,e,f,g):
if z == "Select":
for slider in slider_list:
slider.layout.display = 'none'
return 0
data_path ="../Training data/averaged all.csv"
data = pd.read_csv(data_path)
data.head()
target_fields ='Class'
data = data.drop(["Class code"],axis=1)
features0, targets0 = data.drop(target_fields, axis=1), data[target_fields]
features, targets = np.array(features0) , np.array(targets0)
dic = {}
for feature,target in zip(features, targets):
dic[target]=list(feature)
if z == "Make your own":
x= [380,450,550,650,750,850,950]
y = [a,b,c,d,e,f,g]
plt.scatter(x, y)
plt.plot(x, y, 'o')
else:
x= [x for x in range(380,951)]
y = dic[z.replace("Average ","")]
for slider in slider_list:
slider.layout.display = 'none'
graph = UnivariateSpline(x, y, s=0)
xs = np.linspace(380, 950, 100)
ys = graph(xs)
with box2:
clear_output(wait=True)
display(Markdown("#### Hyperspectral Reflectance" ))
plt.ylim(0,1)
plt.xlim(380, 950)
plt.plot(xs, ys)
plt.title("Reflectance Intensity")
plt.show()
global hyperspectral
global RGB
hyperspectral = [ min(max(round(float(graph(x)),7),0),1) for x in range(380,951)]
red = min(max(round(float(graph(680)),4),0),1)
green = min(max(round(float(graph(530)),4),0),1)
blue = min(max(round(float(graph(465)),4),0),1)
RGB = [red,green,blue]
with box1:
clear_output(wait=True)
red_rgb = int(red*255)
green_rgb = int(green*255)
blue_rgb = int(blue*255)
display(Markdown("#### RGB Reflectance" ))
html = '<svg height="100" width="100%"><rect x="0" y="10px" width="100" height="100" fill="rgb('+ str(red_rgb)+","+str(green_rgb)+","+str(blue_rgb)+')" /></svg>'
display(HTML(html))
display(Markdown("R(680nm): "+ str(red) ))
display(Markdown("G(530nm): "+ str(green) ))
display(Markdown("B(465nm): "+ str(blue) ))
if z == "Make your own":
with box4:
clear_output(wait=False)
with box5:
clear_output(wait=False)
for slider in slider_list:
slider.layout.display = 'flex'
else:
classify(RGB,hyperspectral)
material = widgets.Dropdown(
options=["Select",
"Make your own",
'Average Amphibian Scat',
'Average Bird Scat',
'Average Ground',
"Average Leaf Litter",
"Average Mammal Scat",
"Average Reptile Scat",
"Average Reptile Urea",
"Average Wood"],
value='Select',
description='Material:',
)
output = interactive_output(f, {"z":material,'a':slider1,'b' :slider2, 'c':slider3,'d':slider4,'e' :slider5, 'f':slider6,'g':slider7})
display(material)
grid = GridBox(children=[box1,box2,box3,box4,box5,box6],
layout=Layout(
width='97%',
grid_template_columns='30% 30% 30%',
grid_template_rows='auto auto',
grid_gap='5px 15px',
margin = "10px 0px 0px 25px"
)
)
display(grid)
def _handleSuccess(resj):
display(Markdown('%s' % resj['result']))
def plot_fair_metrics(fair_metrics):
fig, ax = plt.subplots(figsize=(20, 4), ncols=5, nrows=1)
plt.subplots_adjust(left=0.125,
bottom=0.1,
right=0.9,
top=0.9,
wspace=.5,
hspace=1.1)
y_title_margin = 1.2
plt.suptitle("Fairness metrics", y=1.09, fontsize=20)
sns.set(style="dark")
cols = fair_metrics.columns.values
obj = fair_metrics.loc['objective']
size_rect = [0.2, 0.2, 0.2, 0.4, 0.25]
rect = [-0.1, -0.1, -0.1, 0.8, 0]
bottom = [-1, -1, -1, 0, 0]
top = [1, 1, 1, 2, 1]
bound = [[-0.1, 0.1], [-0.1, 0.1], [-0.1, 0.1], [0.8, 1.2], [0, 0.25]]
display(Markdown("### Check bias metrics :"))
#display(Markdown("A model can be considered bias if just one of these five metrics show that this model is biased."))
for attr in fair_metrics.index[1:len(fair_metrics)].values:
#display(Markdown("#### For the %s attribute :"%attr))
check = [
bound[i][0] < fair_metrics.loc[attr][i] < bound[i][1]
for i in range(0, 5)
]
#display(Markdown("With default thresholds, bias against unprivileged group detected in **%d** out of 5 metrics"%(5 - sum(check))))
for i in range(0, 5):
plt.subplot(1, 5, i + 1)
ax = sns.barplot(x=fair_metrics.index[1:len(fair_metrics)],
y=fair_metrics.iloc[1:len(fair_metrics)][cols[i]])
for j in range(0, len(fair_metrics) - 1):
a, val = ax.patches[j], fair_metrics.iloc[j + 1][cols[i]]
marg = -0.2 if val < 0 else 0.1
ax.text(a.get_x() + a.get_width() / 5,
a.get_y() + a.get_height() + marg,
round(val, 3),
fontsize=15,
color='black')
plt.ylim(bottom[i], top[i])
plt.setp(ax.patches, linewidth=0)
ax.add_patch(
patches.Rectangle((-5, rect[i]),
10,
size_rect[i],
alpha=0.3,
facecolor="green",
linewidth=1,
linestyle='solid'))
plt.axhline(obj[i], color='black', alpha=0.3)
plt.title(cols[i])
ax.set_ylabel('')
ax.set_xlabel('')
def pretty_print_answers():
for i, (q, a) in enumerate(answers.items()):
display(Markdown('#### ' + str(i + 1) + '. ' + q))
display(Markdown('*' + a['answer'] + '*'))
def _handleError(resj):
display(Markdown('%s' % resj['error']))
def guppi(self, line=''):
args = magic_arguments.parse_argstring(self.guppi, line)
if (len(args.arguments) > 0):
# switch service
if (args.arguments[0] == 'switch'):
if (len(args.arguments) < 3 and len(args.arguments) > 1):
i = 0
for file_name in self.python_files:
if (args.arguments[1] == file_name):
self.cloud_index = i
print('set cloud interactions to ' + file_name)
i = i + 1
else:
print(
"Not a valid command, type guppi switch followed by the service provider, valid services:"
)
for file_name in self.python_files:
print(file_name)
# cloud services
elif (args.arguments[0] == 'cloud'):
i = 1
# set serviceStr to cloud service
if len(args.arguments) > 1:
serviceStr = args.arguments[i]
i += 1
else:
print(
"Please enter a cloud service, the available services are:"
)
for file_name in self.python_files:
print(" -- " + file_name[0:-7].lower())
print("or choose multicloud to view all services")
return
# ssh service
if len(args.arguments) > 2:
if args.arguments[i] == 'ssh':
i += 1
verbose = False
if len(args.arguments) > 3:
# cloud ssh view mode
if args.arguments[i] == 'view':
if serviceStr.casefold() == 'multicloud':
user_interfaces.SshInterface.render_ssh_interface(
self.cloud_list, -1, verbose)
else:
index = 0
for i, cloudService in enumerate(
self.cloud_list):
if cloudService.name.casefold(
) == serviceStr.casefold():
index = i
user_interfaces.SshInterface.render_ssh_interface(
self.cloud_list, index, verbose)
return
# verbose flag for ssh non-view
if args.arguments[i] == 'v':
i += 1
verbose = True
# check for group name
if len(args.arguments) < i + 1:
print('Please provide group name')
print(
'Run %guppi help for the list of commands')
else:
#set group_name
group_name = args.arguments[i]
i += 1
# if multicloud
if serviceStr.casefold(
) == 'multicloud'.casefold():
noInstancesFlag = True
for cloudService in self.cloud_list:
cloudName = cloudService.name
instances = cloudService.get_instances_info(
)
sshInstances = []
for instance in instances:
if instance[
'Group Name'] == group_name:
if instance[
'State'] == 'running':
sshInstances.append(
instance)
commands = ' '.join(args.arguments[i:])
if len(sshInstances) > 0:
noInstancesFlag = False
print(cloudName.upper())
cloudService.ssh(
sshInstances, commands,
verbose)
if noInstancesFlag == True:
print('No instances in group \'' +
group_name + '\'')
else:
for cloudService in self.cloud_list:
if cloudService.name.casefold(
) == serviceStr.casefold():
instances = cloudService.get_instances_info(
)
sshInstances = []
for instance in instances:
if instance[
'Group Name'] == group_name:
if instance['State'] == 'running':
sshInstances.append(instance)
commands = ' '.join(args.arguments[i:])
if len(sshInstances) == 0:
print('No instances in group \'' +
group_name + '\'')
else:
for cloudService in self.cloud_list:
if cloudService.name.casefold(
) == serviceStr.casefold():
cloudService.ssh(
sshInstances, commands,
verbose)
else:
print("Please enter an ssh mode")
print('Run %guppi help for the list of commands')
# create instance
elif (args.arguments[i] == 'create'):
index = 0
for i, cloudService in enumerate(self.cloud_list):
if cloudService.name.casefold(
) == serviceStr.casefold():
index = i
user_interfaces.CreateInterface.render_create_interface(
self.cloud_list, index)
# view interface
elif (args.arguments[i] == 'view'):
if (args.arguments[i - 1] == "multicloud"):
user_interfaces.CloudInterface.render_cloud_interface(
self.cloud_list, -1)
else:
index = 0
for i, cloudService in enumerate(self.cloud_list):
if cloudService.name.casefold(
) == serviceStr.casefold():
index = i
user_interfaces.CloudInterface.render_cloud_interface(
self.cloud_list, index)
else:
print(
args.arguments[i] +
" is not a cloud command, for usage, use %guppi help"
)
else:
print(
"Please enter a cloud command to use. eg: view, ssh, create"
)
print('Run %guppi help for the list of commands')
#slack service
elif (args.arguments[0] == 'slack'):
if (len(args.arguments) > 1):
if (args.arguments[1] == 'view'):
if (len(args.arguments) < 3):
# user_interfaces.SlackInterface.render_slack_interface()
print("")
else:
print(
args.arguments[2] +
" is not a slack command, for usage, use %guppi help"
)
elif (args.arguments[1] == 'send'):
if (len(args.arguments) < 2):
print(
"Please enter a channel name: %guppi slack send <channel_name> <\"MESSAGE\">"
)
else:
if (len(args.arguments) < 4):
print(
"Please enter a message: %guppi slack send <channel_name> <\"MESSAGE\">"
)
print("Note: message must be in quotes")
elif (len(args.arguments) > 4):
print("Your message must be in quotes")
else:
# user_interfaces.SlackInterface.send_message(args.arguments[2],args.arguments[3])
print("")
else:
print(
args.arguments[1] +
" is not a slack command, for usage, use %guppi help"
)
else:
print("For Slack usage, use %guppi help ")
# github service
elif (args.arguments[0] == 'github'):
if (len(args.arguments) < 2):
user_interfaces.GitHubInterface.display_notifications(5)
else:
try:
num = int(args.arguments[1])
except ValueError:
print("Number of notifications must be an integer")
else:
user_interfaces.GitHubInterface.display_notifications(
num + 1)
elif (args.arguments[0] == 'help'):
fn = "guppihelp.md"
help_markdown = open(fn, 'r').read()
display(Markdown(help_markdown))
else:
print(args.arguments[0] +
" is not a guppi command, for usage, use %guppi help")
else:
print("For usage, use the %guppi help command")
def end_and_md_print(self):
from IPython.display import Markdown, display
string = "Time needed to run experiment: " + str(
np.round(time.time() - self.time, 3)) + " s"
display(Markdown(string))
def display_src(python_object):
display(Markdown("```Python\n"+inspect.getsource(python_object)+"\n```\n"))
def printmd(string):
display(Markdown(string))
def disp_markdown(*args):
""" A shortcut to display(Markdown(*args)) """
return display(Markdown(*args))
def drawTable(p):
pHTML, sizeLst = toHTMLlist(p)
display(Markdown("# Schedule table for tasks"))
display(Markdown(html_table(formatHTMLstr(pHTML, sizeLst))))
def print_markdown(string):
display(Markdown(string))
def classification_report(self, y_test, y_pred):
display(Markdown('**Classification Report:**'))
print(metrics.classification_report(y_test, y_pred))
training_data_schema(corrupt_data)
except pa.errors.SchemaError as exc:
print("Invalid Data:\n-------------")
print(exc.data.iloc[:, :5])
print("\nFailure Cases:\n--------------")
print(exc.failure_cases)
# %% [markdown] slideshow={"slide_type": "slide"}
# ### Summarize the Data
# %% [markdown]
# What percent of cases in the training data are "accidental"?
# %%
percent_accidental = fatal_encounters_clean.disposition_accidental.mean()
display(Markdown(f"{percent_accidental * 100:0.02f}%"))
# %% [markdown] slideshow={"slide_type": "subslide"}
# **Hypothesis**: "the `disposition_accidental` target has a
# class balance of ~2.75%"
# %%
from pandera import Hypothesis
# use the Column object as a stand-alone schema object
target_schema = Column(
pa.Bool,
name="disposition_accidental",
checks=Hypothesis.one_sample_ttest(
popmean=0.0275, relationship="equal", alpha=0.01
)
def f(z,a,b,c,d,e,f,g):
if z == "Select":
for slider in slider_list:
slider.layout.display = 'none'
return 0
data_path ="../Training data/averaged all.csv"
data = pd.read_csv(data_path)
data.head()
target_fields ='Class'
data = data.drop(["Class code"],axis=1)
features0, targets0 = data.drop(target_fields, axis=1), data[target_fields]
features, targets = np.array(features0) , np.array(targets0)
dic = {}
for feature,target in zip(features, targets):
dic[target]=list(feature)
if z == "Make your own":
x= [380,450,550,650,750,850,950]
y = [a,b,c,d,e,f,g]
plt.scatter(x, y)
plt.plot(x, y, 'o')
else:
x= [x for x in range(380,951)]
y = dic[z.replace("Average ","")]
for slider in slider_list:
slider.layout.display = 'none'
graph = UnivariateSpline(x, y, s=0)
xs = np.linspace(380, 950, 100)
ys = graph(xs)
with box2:
clear_output(wait=True)
display(Markdown("#### Hyperspectral Reflectance" ))
plt.ylim(0,1)
plt.xlim(380, 950)
plt.plot(xs, ys)
plt.title("Reflectance Intensity")
plt.show()
global hyperspectral
global RGB
hyperspectral = [ min(max(round(float(graph(x)),7),0),1) for x in range(380,951)]
red = min(max(round(float(graph(680)),4),0),1)
green = min(max(round(float(graph(530)),4),0),1)
blue = min(max(round(float(graph(465)),4),0),1)
RGB = [red,green,blue]
with box1:
clear_output(wait=True)
red_rgb = int(red*255)
green_rgb = int(green*255)
blue_rgb = int(blue*255)
display(Markdown("#### RGB Reflectance" ))
html = '<svg height="100" width="100%"><rect x="0" y="10px" width="100" height="100" fill="rgb('+ str(red_rgb)+","+str(green_rgb)+","+str(blue_rgb)+')" /></svg>'
display(HTML(html))
display(Markdown("R(680nm): "+ str(red) ))
display(Markdown("G(530nm): "+ str(green) ))
display(Markdown("B(465nm): "+ str(blue) ))
if z == "Make your own":
with box4:
clear_output(wait=False)
with box5:
clear_output(wait=False)
for slider in slider_list:
slider.layout.display = 'flex'
else:
classify(RGB,hyperspectral)
def check_data(
data,
columns=[],
missing=0.1,
cardinality=15,
float_frequency=30,
category_frequency=100,
outlier_function=quartile,
title=True,
):
"""
Checks all columns of the given pandas dataframe for data abnormalities
:param columns: a list of column names to analyze, defaults to all columns in the dataframe
:param missing: a cutoff point for high percentage of missing / zero values, defaults to 10%
:param cardinality: a cutoff point for high cardinality of a categorical column, defaults to 15
:param float_frequency: a cutoff point for high frequency of floating point numbers, defaults to 30
:param category_frequency: a cutoff point for low frequency of categories in categorical columns, defaults to 100
:param outlier_function: a function of the dataset and column name that returns the lower and upper limit for outliers, defaults to 1.5*IQR above the 3rd quartile or below the 1st quartile
"""
# default to all columns
if columns == []:
columns = data.columns
# abnormalities is a dictionary from column name to a list of abnormality statements
abnormalities = {}
for col in columns:
# initialize dictionary item
abnormalities[col] = []
"""
Check for high proportion of zeros or missing values
"""
proportion_zero = sum(data[col] == 0) / len(data)
if proportion_zero > missing:
ab = (
"high proportion of zero values at "
+ str(round(proportion_zero * 100, 2))
+ "%"
)
abnormalities[col].append(ab)
prop_null = sum(data[col].isnull()) / len(data)
if prop_null > missing:
ab = (
"high proportion of null values at "
+ str(round(prop_null * 100, 2))
+ "%"
)
abnormalities[col].append(ab)
"""
Check for high cardinality of categorical variables
"""
if data[col].dtype == "O":
num_unique = len(data[col].unique())
if num_unique > cardinality:
ab = (
"high cardinality with "
+ str(num_unique)
+ " unique values"
)
abnormalities[col].append(ab)
"""
Check for high frequency of floating point columns
"""
if data[col].dtype == "float64":
high_frequency_count = sum(
data.groupby(col)[col].count().values > float_frequency
)
if high_frequency_count > 0:
ab = (
"high frequency of floating point numbers with "
+ str(high_frequency_count)
+ " number(s) having frequency over "
+ str(float_frequency)
)
abnormalities[col].append(ab)
"""
Check for low frequency of categories of categorical variables
"""
if data[col].dtype == "O":
low_frequency_count = 0
for val in data.groupby(col)[col].count().values:
if val < category_frequency:
low_frequency_count += 1
if low_frequency_count > 0:
ab = (
"low frequncy in categories with "
+ str(low_frequency_count)
+ " categorie(s) having under "
+ str(category_frequency)
+ " observations"
)
abnormalities[col].append(ab)
"""
Check for outliers in quantitative data
"""
if data[col].dtype == "float64":
lower_limit, upper_limit = outlier_function(data, col)
upper_outlier_count = sum(data[col] > upper_limit)
lower_outlier_count = sum(data[col] < lower_limit)
if lower_outlier_count > 0 or upper_outlier_count > 0:
ab = (
"outliers are present with "
+ str(upper_outlier_count)
+ " high outliers and "
+ str(lower_outlier_count)
+ " low outliers"
)
abnormalities[col].append(ab)
has_abnormalities = False
for col in abnormalities:
if len(abnormalities[col]) > 0:
abs = ""
for ab in abnormalities[col]:
# title is true unless check_data is called by check_column
if title:
abs += (
"\n<div style= 'margin-left: 30px; text-indent:-1em;'>"
+ ab
+ "</div>"
)
else:
abs += (
"\n<div style = 'margin-left: 1em; text-indent:-1em;'>"
+ ab
+ "</div>"
)
if title:
display(Markdown("<div>" + col + ": </div>" + abs))
else:
display(Markdown(abs))
has_abnormalities = True
if not has_abnormalities:
display(Markdown("No abnormalities found"))
# a simplified environment for processing 1D Bruker FTICR datasets with `SPIKE`
#
# Run each python cell in sequence by using the ⇥Run button above (or typing *shift* Enter).
#
# Cells are meant to be used in order, taking you to the complete analysis, but you can go back at any time.
#
# %% [markdown]
# ### Initialization of the environment
# the following cell should be run only once, at the beginning of the processing
# %%
# load all python and interactive tools
from __future__ import print_function, division
from IPython.display import display, HTML, Markdown, Image
display(Markdown('## STARTING Environment...'))
# %matplotlib notebook
import os.path as op
import numpy as np
import spike
from spike.Interactive import INTER as I
from spike.Interactive import FTICR_INTER as FI
from spike.Interactive.ipyfilechooser import FileChooser
from spike.File import BrukerMS
display(Markdown('## ... program is Ready'))
I.hidecode()
# %% [markdown]
# ### Choose the file
# Use `FileChooser()` to choose a file on your disk - The optional `base` argument, starts the exploration on a given location.
#
def check_column(
data,
columns,
bins=False,
missing=0.1,
cardinality=15,
float_frequency=30,
category_frequency=100,
outlier_function=quartile,
):
"""
Presents a summary of given column(s) of the given pandas dataframe
including summary statistics, bar chart or histogram, and any abnormalities found
:param data: a pandas dataframe
:param columns: a single column name or a list of column names to analyze
:param bins: a boolean value or list of boolean values to determine whether to bin the histogram for each column
:param missing: a cutoff point for high percentage of missing / zero values, defaults to 10%
:param cardinality: a cutoff point for high cardinality of a categorical column, defaults to 15
:param float_frequency: a cutoff point for high frequency of floating point numbers, defaults to 30
:param category_frequency: a cutoff point for low frequency of categories in categorical columns, defaults to 100
:param outlier_function: a function of the dataset and column name that returns the lower and upper limit for outliers,
defaults to 1.5*IQR above the 3rd quartile or below the 1st quartile
"""
if isinstance(columns, str):
# with only one column, convert to lists
columns = [columns]
bins = [bins]
else:
if bins == False:
# with multiple columns and no bins,
# convert to list of correct length
bins = [False] * len(columns)
if isinstance(bins, int):
# with multiple columns and only one bin
# specification, convert to list of correct length
bins = [bins] * len(columns)
i = 0
for col in columns:
bin = bins[i]
i += 1
if data[col].dtype == "O":
# cannot bin categorical data
bin = False
if bin == False:
if data[col].dtype == "O":
chart = (
alt.Chart(data)
.mark_bar(color="#64b5f6")
.encode(
alt.X(
col,
axis=alt.Axis(title=col.title()),
sort=alt.SortField(
field="count()", order="descending", op="values"
),
),
alt.Y("count()"),
)
)
else:
chart = (
alt.Chart(data)
.mark_bar(color="#64b5f6")
.encode(
alt.X(
col,
axis=alt.Axis(title=col.title())
),
alt.Y("count()")
)
)
else:
chart = (
alt.Chart(data)
.mark_bar(color="#64b5f6")
.encode(
alt.X(
col,
bin=alt.Bin(maxbins=bin),
axis=alt.Axis(title=col.title())
),
alt.Y("count()"),
)
)
if data[col].dtype == "float64":
stats = data[col].describe()
else:
stats = data.groupby(col)[col].agg(["count"])
stats["prop"] = stats["count"] / len(data)
stats = pd.DataFrame(stats).T
display(Markdown("#### Column Summary: " + col.title()))
display(stats)
display(chart)
check_data(
data,
[col],
missing=missing,
cardinality=cardinality,
float_frequency=float_frequency,
category_frequency=category_frequency,
outlier_function=outlier_function,
title=False,
)
def show_prompt(self):
prompt_url = 'https://' + self.url + '?edit'
prompt_link = '[Excercise](' + prompt_url + ')'
display(Markdown(prompt_link))
def print_title(string):
display(Markdown('**' + string + '**'))
def show(self):
'''
'''
display(Markdown('<img src="./slides/' + self.name + '.png" />'))
async def async_execute_request(self, stream, ident, parent):
"""process an execution request, sending it on to the child kernel or launching one
if it has not been started.
"""
# Announce that we are busy handling this request
self._publish_status("busy")
# Only one kernel can be acquired at the same time
async with self.acquiring_kernel:
# Check for configuration code first
content = parent["content"]
code = content["code"]
config, kernel_name = self.parse_cell(content["code"])
has_config = bool(config)
if has_config:
# Report back that we'll be running the config code
# NOTE: We are, for the time being, ignoring the silent flag, store_history, etc.
self._publish_execute_input(code, parent, self.execution_count)
# User is attempting to run a cell with config after the kernel is started,
# so we inform them and bail
if has_config and self.child_kernel is not None:
self.display(
Markdown(f"""## Kernel already configured and launched.
You can only run the `%%kernel.{kernel_name}` cell at the top of your notebook and the
start of your session. Please **restart your kernel** and run the cell again if
you want to change configuration.
"""),
parent=parent,
)
# Complete the "execution request" so the jupyter client (e.g. the notebook) thinks
# execution is finished
reply_content = {
"status": "error",
# Since our result is not part of `In` or `Out`, ensure
# that the execution count is unset
"execution_count": None,
"user_expressions": {},
"payload": {},
}
metadata = {
"parametrized-kernel": True,
"status": reply_content["status"],
}
self.session.send(
stream,
"execute_reply",
reply_content,
parent,
metadata=metadata,
ident=ident,
)
self._publish_status("idle")
return
kernel_display_id = hexlify(os.urandom(8)).decode("ascii")
# Launching a custom kernel
if self.child_kernel is None and has_config:
# Start a kernel now
# If there's config set, we launch with that config
# If the user is requesting the kernel with config, launch it!
self.display(
Markdown("Launching customized runtime..."),
display_id=kernel_display_id,
parent=parent,
)
try:
self.child_kernel = await self.start_kernel(
kernel_name, config)
except PickRegistrationException as err:
# Get access to the exception info prior to doing any potential awaiting
exc_info = sys.exc_info()
self.log.info(exc_info)
separator = "\n"
self.display(
Markdown(
f"""## There is no kernel magic named `{kernel_name}`
These are the available kernels:
{separator.join([f"* `{kernel}`" for kernel in _subkernels.list_subkernels()])}
"""),
display_id=kernel_display_id,
update=True,
parent=parent,
)
self.log.error(err)
self._publish_execute_reply_error(exc_info,
ident=ident,
parent=parent)
self._publish_status("idle", parent=parent)
return
except Exception as err:
self.log.error(err)
exc_info = sys.exc_info()
self._publish_error(exc_info, parent=parent)
self._publish_execute_reply_error(exc_info,
ident=ident,
parent=parent)
self._publish_status("idle", parent=parent)
return
self.display(
Markdown("Runtime ready!"),
display_id=kernel_display_id,
parent=parent,
update=True,
)
# Complete the "execution request" so the jupyter client (e.g. the notebook) thinks
# execution is finished
reply_content = {
"status": "ok",
# Our kernel setup is always the zero-th execution (In[] starts at 1)
"execution_count": 0,
# Note: user_expressions are not supported on our kernel creation magic
"user_expressions": {},
"payload": {},
}
metadata = {
"parametrized-kernel": True,
"status": reply_content["status"],
}
self.session.send(
stream,
"execute_reply",
reply_content,
parent,
metadata=metadata,
ident=ident,
)
# With that, we're all done launching the customized kernel and
# pushing updates on the kernel to the user.
return
# Start the default kernel to run code
if self.child_kernel is None:
self.display(
Markdown("Preparing default kernel..."),
parent=parent,
display_id=kernel_display_id,
)
self.child_kernel = await self.start_kernel(
self.default_kernel, self.default_config)
self.display(
# Wipe out the previous message.
# NOTE: The Jupyter notebook frontend ignores the case of an empty output for
# an update_display_data so we have to publish some empty content instead.
Markdown(""),
parent=parent,
display_id=kernel_display_id,
update=True,
)
self.session.send(self.child_kernel.shell, parent, ident=ident)
{#fig:img}
In this version of Pandoc image caption @fig:img works.
"""
# %% {echo=True}
from IPython.display import Markdown
import pandas as pd
import numpy as np
import tabulatehelper as th
df = pd.DataFrame(np.random.random(16).reshape(4, 4))
Markdown(f'''
{th.md_table(df)}
: Table {{#tbl:table1}}
''')
# %%
"""
Text and @tbl:table1
"""
# %% {input=True, eval=False, echo=True}
import pandas as pd
import numpy as np
df = pd.DataFrame(np.random.random(16).reshape(4, 4))
df
# %% {r, echo=True} R cell:
"""
# data preprocessing and feature extraction
# Dataset: LibriSpeech for English - read speech, designed for training and evaluating models for ASR
# Features: 1. Spectrogram 2. MFCCs
from data_generator import vis_train_features
from IPython.display import Markdown, display
from IPython.display import audio
from data_generator import vis_train_features, plot_raw_audio, plot_spectrogram_feature, plot_mfcc_feature
# extract label and audio feature for a single training example
vis_text, vis_raw_audio, vis_mfcc_feature, vis_spectrogram_feature, vis_audio_path = vis_train_features()
plot_raw_audio(vis_raw_audio)
display(Markdown('** Shape of Audio Signal**:' + str(vis_raw_audio.shape)))
display(Markdow('**Transcript**:' + str(vis_text)))
# Step 1: Acoustic Feature for Speech Recognition
# Feature 1: Spectrogram
plot_spectrogram_feature(vis_spectrogram_feature)
# Feature 2: MFCC s
plot_mfcc_feature(vis_mfcc_feature)
