def display_tf_code(func): code = tf.autograph.to_code(func) from IPython.display import display, Markdown display(Markdown('```python\n{}\n```'.format(code)))
def show(self): ''' ''' display(Markdown(self.body))
def markdown(self, line, cell): """Render the cell as Markdown text block""" display(Markdown(cell))
def event_table(self, **_args): """Print out event table in Markdown format.""" return Markdown(self.event_table_text(**_args))
def show_response(self): response_url = 'https://' + self.url + '/download/markdown' display(Markdown(response_url))
def confusion_matrix_information(self, y_test, y_pred): cm = pd.crosstab(y_test, y_pred, rownames=['Known OBEC URLs'], colnames=['Predicted OBEC URLs']) TP = cm.iloc[1][1] TN = cm.iloc[0][0] FP = cm.iloc[0][1] FN = cm.iloc[1][0] display(Markdown('**Confusion Matrix Information:**')) print('OBEC URLs that are predicted as False and we do not ' \ 'know (True Negatives):', TN) print('OBEC URLs that are predicted as False and we do know ' \ '(False Negatives):', FN) print('OBEC URLs that are predicted as True and we do not ' \ 'know (False Positives):', FP) print('OBEC URLs that are predicted as True and we do know ' \ '(True Positives):', TP) print('Accuracy = (TP+TN)/(TP+TN+FP+FN):', '{0:.2}'.format( (TP + TN) / (TP + TN + FP + FN))) print('Precision = TP/(TP+FP):', '{0:.2}'.format(TP / (TP + FP))) print('Recall = TP/(TP+FN):', '{0:.2}'.format(TP / (TP + FN))) print('Specificity = TN/(TN+FP):', '{0:.2}'.format(TN / (TN + FP))) print('F1 Score = 2*TP/(2*TP+FP+FN):', '{0:.2}'.format(2 * TP / (2 * TP + FP + FN))) print( 'MCC = (TP*TN-FP*FN)/math.sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN)):', '{0:.2}'.format((TP * TN - FP * FN) / math.sqrt( (TP + FP) * (TP + FN) * (TN + FP) * (TN + FN)))) display(Markdown('**Confusion Matrix Explanations:**')) print('Accuracy - This is simply equal to the proportion ' \ 'of predictions that the model classified correctly.') print('Precision - Precision is also known as positive ' \ 'predictive value and is the proportion of relevant ' \ 'instances among the retrieved instances. In other ' \ 'words, it answers the question "What proportion of ' \ 'positive identifications was actually correct?".') print('Recall - Recall, also known as the sensitivity, hit ' \ 'rate, or the true positive rate (TPR), is the ' \ 'proportion of the total amount of relevant instances ' \ 'that were actually retrieved. It answers the question ' \ '"What proportion of actual positives was identified ' \ 'correctly?".') print('Specificity - Specificity, also known as the true ' \ 'negative rate (TNR), measures the proportion of actual ' \ 'negatives that are correctly identified as such. It is ' \ 'the opposite of recall.') print('F1 Score - The F1 score is a measure of a test\'s ' \ 'accuracy — it is the harmonic mean of precision and ' \ 'recall. It can have a maximum score of 1 (perfect ' \ 'precision and recall) and a minimum of 0. Overall, ' \ 'it is a measure of the preciseness and robustness ' \ 'of your model.') print('Matthews correlation coefficient (MCC) - The MCC is ' \ 'in essence a correlation coefficient between the ' \ 'observed and predicted binary classifications; it ' \ 'returns a value between -1 and +1. A coefficient ' \ 'of +1 represents a perfect prediction, 0 no better ' \ 'than random prediction and -1 indicates total ' \ 'disagreement between prediction and observation. ' \ 'The statistic is also known as the phi coefficient. ' \ 'MCC is related to the chi-square statistic for a 2x2 ' \ 'contingency table.')
def FlyingCameraNetworkDisplay(): import random import pandas as pd import pickle as pkl import numpy as np import tensorflow as tf import matplotlib.pyplot as plt from random import randrange import random from ipywidgets import interactive, interact , FloatSlider,widgets,VBox,HBox,interactive_output,Layout,Output,GridBox from IPython.display import display,HTML,clear_output,Markdown from scipy.interpolate import UnivariateSpline clear_output() print("loading...(Might take a minute)") #Network layers def hyperspectral(input_,n_o_input, keep_prob, filter_width = 1, stride_size =1, relu_alpha = 0.2): n_o_strides = int((n_o_input-filter_width)/stride_size) +1 #round down Hyper_layer = [] def dense(input_,start,keep_prob, filter_width, stride_size, relu_alpha): nn_input = tf.slice(input_,[0,start],[-1,filter_width]) dropout1 = tf.nn.dropout(nn_input, keep_prob) dense1 = tf.layers.dense(dropout1, 1) relu1 = tf.maximum(relu_alpha * dense1, dense1) return relu1 for step in range(n_o_strides): start = step*stride_size output = dense(input_,start,keep_prob, filter_width, stride_size, relu_alpha) Hyper_layer.append(output) if (n_o_input-filter_width)%stride_size>0: start = n_o_input-filter_width output = dense(input_,start,keep_prob, filter_width, stride_size, relu_alpha) Hyper_layer.append(output) Hyper_l_stacked = tf.concat(Hyper_layer,1) return Hyper_l_stacked , n_o_strides def Classifier(input_,n_o_class,n_o_input, keep_prob,relu_alpha = 0.2): if n_o_input == 3: is_RGB = True elif n_o_input == 571: is_RGB = False else: raise ValueError('A very specific bad thing happened.'+str(n_o_input)) if is_RGB: dense0 = tf.layers.dense(input_, 3) relu0 = tf.maximum(relu_alpha * dense0, dense0) first_layer_out = tf.nn.dropout(relu0, keep_prob) else: first_layer_out,n_o_input= hyperspectral(input_,n_o_input, keep_prob, filter_width = 30, stride_size =1, relu_alpha = 0.2) hidden_size = n_o_input*2/3 hidden_nodes = int(hidden_size)+1 # rounding dense1 = tf.layers.dense(first_layer_out, hidden_nodes) relu1 = tf.maximum(relu_alpha * dense1, dense1) dropout1 = tf.nn.dropout(relu1, keep_prob) class_logits = tf.layers.dense(dropout1, n_o_class) return class_logits def softmax(input_,m_class, n_class,n_o_input, keep_prob,relu_alpha = 0.2,sub_scaling = 1): n_o_class = m_class+n_class #raw output logits= Classifier(input_,n_o_class,n_o_input, keep_prob,relu_alpha = 0.2) subclass_softmax = tf.nn.softmax(logits) #Reduce outputs from 8 subclasses to 2 main classes m_class_logit, n_class_logit = tf.split(logits, [m_class, n_class], 1) m_class_logit1 =tf.reduce_sum(m_class_logit,1, keepdims =True) n_class_logit1 =tf.reduce_sum(n_class_logit,1, keepdims =True) main_class_logits = tf.concat([m_class_logit1, n_class_logit1], 1) main_class_softmax = tf.nn.softmax(main_class_logits) return subclass_softmax,main_class_softmax #Defining the training parameters tf.reset_default_graph() n_class = 3 m_class = 5 n_o_class = m_class+n_class epochs = 10 keep_probability = 0.95 sub_scaling = 1 g1 = tf.Graph() with g1.as_default() as g_1: H_keep_prob = tf.placeholder(tf.float32) H_n_o_input = 571 H_input_ = tf.placeholder(tf.float32, [None,H_n_o_input]) H_subclass_softmax,H_main_class_softmax =softmax(H_input_,m_class, n_class,H_n_o_input, H_keep_prob,relu_alpha = 0.2,sub_scaling = sub_scaling) #Relectance spectrum g2 = tf.Graph() with g2.as_default() as g_2: R_keep_prob = tf.placeholder(tf.float32) R_n_o_input = 3 R_input_ = tf.placeholder(tf.float32, [None,R_n_o_input]) R_subclass_softmax,R_main_class_softmax =softmax(R_input_,m_class, n_class,R_n_o_input, R_keep_prob,relu_alpha = 0.2,sub_scaling = sub_scaling) #Display network result clear_output() display(Markdown("## View Classification" )) RGB = [] hyperspectral = [] box1 = Output(layout=Layout(width='100%',border ='1px solid #E0E0E0',padding = '0px 0px 0px 20px',fill = "black")) box2 = Output(layout=Layout(width='100%',border ='1px solid #E0E0E0',padding = '0px 20px 0px 20px')) box3 = Output(layout=Layout(width='100%',border ='1px solid #E0E0E0',padding = '0px 0px 0px 20px')) box4 = Output(layout=Layout(width='100%',border ='1px solid #E0E0E0',padding = '0px 0px 0px 20px')) box5 = Output(layout=Layout(width='100%',border ='1px solid #E0E0E0',padding = '0px 20px 0px 20px')) box6 = Output(layout=Layout(width='100%',border ='1px solid #E0E0E0',padding = '0px 20px 0px 20px')) markdown = Output(layout=Layout(width='95%')) with markdown: display(Markdown("#### Adjust The Wavelengths")) display(Markdown("<sup>To Customise Your Own RGB & Hyperspectral Input</sup>")) slider1 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='380nm',layout= Layout(width='99%')) slider2 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='450nm',layout= Layout(width='99%')) slider3 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='550nm',layout= Layout(width='99%')) slider4 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='650nm',layout= Layout(width='99%')) slider5 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='750nm',layout= Layout(width='99%')) slider6 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='850nm',layout= Layout(width='99%')) slider7 = FloatSlider(min=0, max=1, step=0.01, value=0.5,description='950nm',layout= Layout(width='99%')) def click(event): global hyperspectral global RGB classify(RGB,hyperspectral) button = widgets.Button( description='Run Classification', layout={'width': '130px','margin':'0px 0px 10px 15%'} ) button.on_click(click) slider_list = [markdown,slider1,slider2,slider3,slider4,slider5,slider6,slider7,button] for slider in slider_list: with box3: display(slider) def classify(RGB,hyperspectral): keep_probability = 1 def print_network_output(save_model_path,title,subclass_softmax,main_class_softmax,input_,reflectance_data,n_o_input,g,keep_prob): clear_output(wait=True) print("""Loading...(Might take a minute) *The next classification doesn't load until this one is completed""") with tf.Session(graph=g) as sess: sess.run(tf.global_variables_initializer()) loader = tf.train.import_meta_graph(save_model_path + '.meta') loader.restore(sess, save_model_path) subclass_softmax_p,main_class_softmax_p= sess.run([subclass_softmax,main_class_softmax], feed_dict = {input_:reflectance_data,keep_prob:keep_probability}) clear_output(wait=True) display(Markdown("#### "+title)) display(Markdown("##### Main Class")) main_class_softmax_p = main_class_softmax_p[0] display(Markdown("Non-scat: "+str(main_class_softmax_p[0]))) display(Markdown("Scat : "+str(main_class_softmax_p[1]))) display(Markdown("##### Subclass")) subclass_softmax_p = subclass_softmax_p[0] subclass_name = [ "Leaf Litter ", "Ground ", "Wood ", "Bird Scat ", "Mammal Scat ", "Amphibian Scat", "Reptile Scat ", "Reptile Urea "] for subclass,name in zip(subclass_softmax_p,subclass_name): display(Markdown(name+": " +str(subclass))) with box4: print_network_output('../training_rgb', "RGB Network Classification", R_subclass_softmax, R_main_class_softmax, R_input_,[RGB], 3, g_2, R_keep_prob) with box5: print_network_output('../training_Hyperspectral', "Hyperspectral Network Classification", H_subclass_softmax, H_main_class_softmax, H_input_, [hyperspectral], 571, g_1, H_keep_prob) def f(z,a,b,c,d,e,f,g): if z == "Select": for slider in slider_list: slider.layout.display = 'none' return 0 data_path ="../Training data/averaged all.csv" data = pd.read_csv(data_path) data.head() target_fields ='Class' data = data.drop(["Class code"],axis=1) features0, targets0 = data.drop(target_fields, axis=1), data[target_fields] features, targets = np.array(features0) , np.array(targets0) dic = {} for feature,target in zip(features, targets): dic[target]=list(feature) if z == "Make your own": x= [380,450,550,650,750,850,950] y = [a,b,c,d,e,f,g] plt.scatter(x, y) plt.plot(x, y, 'o') else: x= [x for x in range(380,951)] y = dic[z.replace("Average ","")] for slider in slider_list: slider.layout.display = 'none' graph = UnivariateSpline(x, y, s=0) xs = np.linspace(380, 950, 100) ys = graph(xs) with box2: clear_output(wait=True) display(Markdown("#### Hyperspectral Reflectance" )) plt.ylim(0,1) plt.xlim(380, 950) plt.plot(xs, ys) plt.title("Reflectance Intensity") plt.show() global hyperspectral global RGB hyperspectral = [ min(max(round(float(graph(x)),7),0),1) for x in range(380,951)] red = min(max(round(float(graph(680)),4),0),1) green = min(max(round(float(graph(530)),4),0),1) blue = min(max(round(float(graph(465)),4),0),1) RGB = [red,green,blue] with box1: clear_output(wait=True) red_rgb = int(red*255) green_rgb = int(green*255) blue_rgb = int(blue*255) display(Markdown("#### RGB Reflectance" )) html = '<svg height="100" width="100%"><rect x="0" y="10px" width="100" height="100" fill="rgb('+ str(red_rgb)+","+str(green_rgb)+","+str(blue_rgb)+')" /></svg>' display(HTML(html)) display(Markdown("R(680nm): "+ str(red) )) display(Markdown("G(530nm): "+ str(green) )) display(Markdown("B(465nm): "+ str(blue) )) if z == "Make your own": with box4: clear_output(wait=False) with box5: clear_output(wait=False) for slider in slider_list: slider.layout.display = 'flex' else: classify(RGB,hyperspectral) material = widgets.Dropdown( options=["Select", "Make your own", 'Average Amphibian Scat', 'Average Bird Scat', 'Average Ground', "Average Leaf Litter", "Average Mammal Scat", "Average Reptile Scat", "Average Reptile Urea", "Average Wood"], value='Select', description='Material:', ) output = interactive_output(f, {"z":material,'a':slider1,'b' :slider2, 'c':slider3,'d':slider4,'e' :slider5, 'f':slider6,'g':slider7}) display(material) grid = GridBox(children=[box1,box2,box3,box4,box5,box6], layout=Layout( width='97%', grid_template_columns='30% 30% 30%', grid_template_rows='auto auto', grid_gap='5px 15px', margin = "10px 0px 0px 25px" ) ) display(grid)
def _handleSuccess(resj): display(Markdown('%s' % resj['result']))
def plot_fair_metrics(fair_metrics): fig, ax = plt.subplots(figsize=(20, 4), ncols=5, nrows=1) plt.subplots_adjust(left=0.125, bottom=0.1, right=0.9, top=0.9, wspace=.5, hspace=1.1) y_title_margin = 1.2 plt.suptitle("Fairness metrics", y=1.09, fontsize=20) sns.set(style="dark") cols = fair_metrics.columns.values obj = fair_metrics.loc['objective'] size_rect = [0.2, 0.2, 0.2, 0.4, 0.25] rect = [-0.1, -0.1, -0.1, 0.8, 0] bottom = [-1, -1, -1, 0, 0] top = [1, 1, 1, 2, 1] bound = [[-0.1, 0.1], [-0.1, 0.1], [-0.1, 0.1], [0.8, 1.2], [0, 0.25]] display(Markdown("### Check bias metrics :")) #display(Markdown("A model can be considered bias if just one of these five metrics show that this model is biased.")) for attr in fair_metrics.index[1:len(fair_metrics)].values: #display(Markdown("#### For the %s attribute :"%attr)) check = [ bound[i][0] < fair_metrics.loc[attr][i] < bound[i][1] for i in range(0, 5) ] #display(Markdown("With default thresholds, bias against unprivileged group detected in **%d** out of 5 metrics"%(5 - sum(check)))) for i in range(0, 5): plt.subplot(1, 5, i + 1) ax = sns.barplot(x=fair_metrics.index[1:len(fair_metrics)], y=fair_metrics.iloc[1:len(fair_metrics)][cols[i]]) for j in range(0, len(fair_metrics) - 1): a, val = ax.patches[j], fair_metrics.iloc[j + 1][cols[i]] marg = -0.2 if val < 0 else 0.1 ax.text(a.get_x() + a.get_width() / 5, a.get_y() + a.get_height() + marg, round(val, 3), fontsize=15, color='black') plt.ylim(bottom[i], top[i]) plt.setp(ax.patches, linewidth=0) ax.add_patch( patches.Rectangle((-5, rect[i]), 10, size_rect[i], alpha=0.3, facecolor="green", linewidth=1, linestyle='solid')) plt.axhline(obj[i], color='black', alpha=0.3) plt.title(cols[i]) ax.set_ylabel('') ax.set_xlabel('')
def pretty_print_answers(): for i, (q, a) in enumerate(answers.items()): display(Markdown('#### ' + str(i + 1) + '. ' + q)) display(Markdown('*' + a['answer'] + '*'))
def _handleError(resj): display(Markdown('%s' % resj['error']))
def guppi(self, line=''): args = magic_arguments.parse_argstring(self.guppi, line) if (len(args.arguments) > 0): # switch service if (args.arguments[0] == 'switch'): if (len(args.arguments) < 3 and len(args.arguments) > 1): i = 0 for file_name in self.python_files: if (args.arguments[1] == file_name): self.cloud_index = i print('set cloud interactions to ' + file_name) i = i + 1 else: print( "Not a valid command, type guppi switch followed by the service provider, valid services:" ) for file_name in self.python_files: print(file_name) # cloud services elif (args.arguments[0] == 'cloud'): i = 1 # set serviceStr to cloud service if len(args.arguments) > 1: serviceStr = args.arguments[i] i += 1 else: print( "Please enter a cloud service, the available services are:" ) for file_name in self.python_files: print(" -- " + file_name[0:-7].lower()) print("or choose multicloud to view all services") return # ssh service if len(args.arguments) > 2: if args.arguments[i] == 'ssh': i += 1 verbose = False if len(args.arguments) > 3: # cloud ssh view mode if args.arguments[i] == 'view': if serviceStr.casefold() == 'multicloud': user_interfaces.SshInterface.render_ssh_interface( self.cloud_list, -1, verbose) else: index = 0 for i, cloudService in enumerate( self.cloud_list): if cloudService.name.casefold( ) == serviceStr.casefold(): index = i user_interfaces.SshInterface.render_ssh_interface( self.cloud_list, index, verbose) return # verbose flag for ssh non-view if args.arguments[i] == 'v': i += 1 verbose = True # check for group name if len(args.arguments) < i + 1: print('Please provide group name') print( 'Run %guppi help for the list of commands') else: #set group_name group_name = args.arguments[i] i += 1 # if multicloud if serviceStr.casefold( ) == 'multicloud'.casefold(): noInstancesFlag = True for cloudService in self.cloud_list: cloudName = cloudService.name instances = cloudService.get_instances_info( ) sshInstances = [] for instance in instances: if instance[ 'Group Name'] == group_name: if instance[ 'State'] == 'running': sshInstances.append( instance) commands = ' '.join(args.arguments[i:]) if len(sshInstances) > 0: noInstancesFlag = False print(cloudName.upper()) cloudService.ssh( sshInstances, commands, verbose) if noInstancesFlag == True: print('No instances in group \'' + group_name + '\'') else: for cloudService in self.cloud_list: if cloudService.name.casefold( ) == serviceStr.casefold(): instances = cloudService.get_instances_info( ) sshInstances = [] for instance in instances: if instance[ 'Group Name'] == group_name: if instance['State'] == 'running': sshInstances.append(instance) commands = ' '.join(args.arguments[i:]) if len(sshInstances) == 0: print('No instances in group \'' + group_name + '\'') else: for cloudService in self.cloud_list: if cloudService.name.casefold( ) == serviceStr.casefold(): cloudService.ssh( sshInstances, commands, verbose) else: print("Please enter an ssh mode") print('Run %guppi help for the list of commands') # create instance elif (args.arguments[i] == 'create'): index = 0 for i, cloudService in enumerate(self.cloud_list): if cloudService.name.casefold( ) == serviceStr.casefold(): index = i user_interfaces.CreateInterface.render_create_interface( self.cloud_list, index) # view interface elif (args.arguments[i] == 'view'): if (args.arguments[i - 1] == "multicloud"): user_interfaces.CloudInterface.render_cloud_interface( self.cloud_list, -1) else: index = 0 for i, cloudService in enumerate(self.cloud_list): if cloudService.name.casefold( ) == serviceStr.casefold(): index = i user_interfaces.CloudInterface.render_cloud_interface( self.cloud_list, index) else: print( args.arguments[i] + " is not a cloud command, for usage, use %guppi help" ) else: print( "Please enter a cloud command to use. eg: view, ssh, create" ) print('Run %guppi help for the list of commands') #slack service elif (args.arguments[0] == 'slack'): if (len(args.arguments) > 1): if (args.arguments[1] == 'view'): if (len(args.arguments) < 3): # user_interfaces.SlackInterface.render_slack_interface() print("") else: print( args.arguments[2] + " is not a slack command, for usage, use %guppi help" ) elif (args.arguments[1] == 'send'): if (len(args.arguments) < 2): print( "Please enter a channel name: %guppi slack send <channel_name> <\"MESSAGE\">" ) else: if (len(args.arguments) < 4): print( "Please enter a message: %guppi slack send <channel_name> <\"MESSAGE\">" ) print("Note: message must be in quotes") elif (len(args.arguments) > 4): print("Your message must be in quotes") else: # user_interfaces.SlackInterface.send_message(args.arguments[2],args.arguments[3]) print("") else: print( args.arguments[1] + " is not a slack command, for usage, use %guppi help" ) else: print("For Slack usage, use %guppi help ") # github service elif (args.arguments[0] == 'github'): if (len(args.arguments) < 2): user_interfaces.GitHubInterface.display_notifications(5) else: try: num = int(args.arguments[1]) except ValueError: print("Number of notifications must be an integer") else: user_interfaces.GitHubInterface.display_notifications( num + 1) elif (args.arguments[0] == 'help'): fn = "guppihelp.md" help_markdown = open(fn, 'r').read() display(Markdown(help_markdown)) else: print(args.arguments[0] + " is not a guppi command, for usage, use %guppi help") else: print("For usage, use the %guppi help command")
def end_and_md_print(self): from IPython.display import Markdown, display string = "Time needed to run experiment: " + str( np.round(time.time() - self.time, 3)) + " s" display(Markdown(string))
def display_src(python_object): display(Markdown("```Python\n"+inspect.getsource(python_object)+"\n```\n"))
def printmd(string): display(Markdown(string))
def disp_markdown(*args): """ A shortcut to display(Markdown(*args)) """ return display(Markdown(*args))
def drawTable(p): pHTML, sizeLst = toHTMLlist(p) display(Markdown("# Schedule table for tasks")) display(Markdown(html_table(formatHTMLstr(pHTML, sizeLst))))
def print_markdown(string): display(Markdown(string))
def classification_report(self, y_test, y_pred): display(Markdown('**Classification Report:**')) print(metrics.classification_report(y_test, y_pred))
training_data_schema(corrupt_data) except pa.errors.SchemaError as exc: print("Invalid Data:\n-------------") print(exc.data.iloc[:, :5]) print("\nFailure Cases:\n--------------") print(exc.failure_cases) # %% [markdown] slideshow={"slide_type": "slide"} # ### Summarize the Data # %% [markdown] # What percent of cases in the training data are "accidental"? # %% percent_accidental = fatal_encounters_clean.disposition_accidental.mean() display(Markdown(f"{percent_accidental * 100:0.02f}%")) # %% [markdown] slideshow={"slide_type": "subslide"} # **Hypothesis**: "the `disposition_accidental` target has a # class balance of ~2.75%" # %% from pandera import Hypothesis # use the Column object as a stand-alone schema object target_schema = Column( pa.Bool, name="disposition_accidental", checks=Hypothesis.one_sample_ttest( popmean=0.0275, relationship="equal", alpha=0.01 )
def f(z,a,b,c,d,e,f,g): if z == "Select": for slider in slider_list: slider.layout.display = 'none' return 0 data_path ="../Training data/averaged all.csv" data = pd.read_csv(data_path) data.head() target_fields ='Class' data = data.drop(["Class code"],axis=1) features0, targets0 = data.drop(target_fields, axis=1), data[target_fields] features, targets = np.array(features0) , np.array(targets0) dic = {} for feature,target in zip(features, targets): dic[target]=list(feature) if z == "Make your own": x= [380,450,550,650,750,850,950] y = [a,b,c,d,e,f,g] plt.scatter(x, y) plt.plot(x, y, 'o') else: x= [x for x in range(380,951)] y = dic[z.replace("Average ","")] for slider in slider_list: slider.layout.display = 'none' graph = UnivariateSpline(x, y, s=0) xs = np.linspace(380, 950, 100) ys = graph(xs) with box2: clear_output(wait=True) display(Markdown("#### Hyperspectral Reflectance" )) plt.ylim(0,1) plt.xlim(380, 950) plt.plot(xs, ys) plt.title("Reflectance Intensity") plt.show() global hyperspectral global RGB hyperspectral = [ min(max(round(float(graph(x)),7),0),1) for x in range(380,951)] red = min(max(round(float(graph(680)),4),0),1) green = min(max(round(float(graph(530)),4),0),1) blue = min(max(round(float(graph(465)),4),0),1) RGB = [red,green,blue] with box1: clear_output(wait=True) red_rgb = int(red*255) green_rgb = int(green*255) blue_rgb = int(blue*255) display(Markdown("#### RGB Reflectance" )) html = '<svg height="100" width="100%"><rect x="0" y="10px" width="100" height="100" fill="rgb('+ str(red_rgb)+","+str(green_rgb)+","+str(blue_rgb)+')" /></svg>' display(HTML(html)) display(Markdown("R(680nm): "+ str(red) )) display(Markdown("G(530nm): "+ str(green) )) display(Markdown("B(465nm): "+ str(blue) )) if z == "Make your own": with box4: clear_output(wait=False) with box5: clear_output(wait=False) for slider in slider_list: slider.layout.display = 'flex' else: classify(RGB,hyperspectral)
def check_data( data, columns=[], missing=0.1, cardinality=15, float_frequency=30, category_frequency=100, outlier_function=quartile, title=True, ): """ Checks all columns of the given pandas dataframe for data abnormalities :param columns: a list of column names to analyze, defaults to all columns in the dataframe :param missing: a cutoff point for high percentage of missing / zero values, defaults to 10% :param cardinality: a cutoff point for high cardinality of a categorical column, defaults to 15 :param float_frequency: a cutoff point for high frequency of floating point numbers, defaults to 30 :param category_frequency: a cutoff point for low frequency of categories in categorical columns, defaults to 100 :param outlier_function: a function of the dataset and column name that returns the lower and upper limit for outliers, defaults to 1.5*IQR above the 3rd quartile or below the 1st quartile """ # default to all columns if columns == []: columns = data.columns # abnormalities is a dictionary from column name to a list of abnormality statements abnormalities = {} for col in columns: # initialize dictionary item abnormalities[col] = [] """ Check for high proportion of zeros or missing values """ proportion_zero = sum(data[col] == 0) / len(data) if proportion_zero > missing: ab = ( "high proportion of zero values at " + str(round(proportion_zero * 100, 2)) + "%" ) abnormalities[col].append(ab) prop_null = sum(data[col].isnull()) / len(data) if prop_null > missing: ab = ( "high proportion of null values at " + str(round(prop_null * 100, 2)) + "%" ) abnormalities[col].append(ab) """ Check for high cardinality of categorical variables """ if data[col].dtype == "O": num_unique = len(data[col].unique()) if num_unique > cardinality: ab = ( "high cardinality with " + str(num_unique) + " unique values" ) abnormalities[col].append(ab) """ Check for high frequency of floating point columns """ if data[col].dtype == "float64": high_frequency_count = sum( data.groupby(col)[col].count().values > float_frequency ) if high_frequency_count > 0: ab = ( "high frequency of floating point numbers with " + str(high_frequency_count) + " number(s) having frequency over " + str(float_frequency) ) abnormalities[col].append(ab) """ Check for low frequency of categories of categorical variables """ if data[col].dtype == "O": low_frequency_count = 0 for val in data.groupby(col)[col].count().values: if val < category_frequency: low_frequency_count += 1 if low_frequency_count > 0: ab = ( "low frequncy in categories with " + str(low_frequency_count) + " categorie(s) having under " + str(category_frequency) + " observations" ) abnormalities[col].append(ab) """ Check for outliers in quantitative data """ if data[col].dtype == "float64": lower_limit, upper_limit = outlier_function(data, col) upper_outlier_count = sum(data[col] > upper_limit) lower_outlier_count = sum(data[col] < lower_limit) if lower_outlier_count > 0 or upper_outlier_count > 0: ab = ( "outliers are present with " + str(upper_outlier_count) + " high outliers and " + str(lower_outlier_count) + " low outliers" ) abnormalities[col].append(ab) has_abnormalities = False for col in abnormalities: if len(abnormalities[col]) > 0: abs = "" for ab in abnormalities[col]: # title is true unless check_data is called by check_column if title: abs += ( "\n<div style= 'margin-left: 30px; text-indent:-1em;'>" + ab + "</div>" ) else: abs += ( "\n<div style = 'margin-left: 1em; text-indent:-1em;'>" + ab + "</div>" ) if title: display(Markdown("<div>" + col + ": </div>" + abs)) else: display(Markdown(abs)) has_abnormalities = True if not has_abnormalities: display(Markdown("No abnormalities found"))
# a simplified environment for processing 1D Bruker FTICR datasets with `SPIKE` # # Run each python cell in sequence by using the ⇥Run button above (or typing *shift* Enter). # # Cells are meant to be used in order, taking you to the complete analysis, but you can go back at any time. # # %% [markdown] # ### Initialization of the environment # the following cell should be run only once, at the beginning of the processing # %% # load all python and interactive tools from __future__ import print_function, division from IPython.display import display, HTML, Markdown, Image display(Markdown('## STARTING Environment...')) # %matplotlib notebook import os.path as op import numpy as np import spike from spike.Interactive import INTER as I from spike.Interactive import FTICR_INTER as FI from spike.Interactive.ipyfilechooser import FileChooser from spike.File import BrukerMS display(Markdown('## ... program is Ready')) I.hidecode() # %% [markdown] # ### Choose the file # Use `FileChooser()` to choose a file on your disk - The optional `base` argument, starts the exploration on a given location. #
def check_column( data, columns, bins=False, missing=0.1, cardinality=15, float_frequency=30, category_frequency=100, outlier_function=quartile, ): """ Presents a summary of given column(s) of the given pandas dataframe including summary statistics, bar chart or histogram, and any abnormalities found :param data: a pandas dataframe :param columns: a single column name or a list of column names to analyze :param bins: a boolean value or list of boolean values to determine whether to bin the histogram for each column :param missing: a cutoff point for high percentage of missing / zero values, defaults to 10% :param cardinality: a cutoff point for high cardinality of a categorical column, defaults to 15 :param float_frequency: a cutoff point for high frequency of floating point numbers, defaults to 30 :param category_frequency: a cutoff point for low frequency of categories in categorical columns, defaults to 100 :param outlier_function: a function of the dataset and column name that returns the lower and upper limit for outliers, defaults to 1.5*IQR above the 3rd quartile or below the 1st quartile """ if isinstance(columns, str): # with only one column, convert to lists columns = [columns] bins = [bins] else: if bins == False: # with multiple columns and no bins, # convert to list of correct length bins = [False] * len(columns) if isinstance(bins, int): # with multiple columns and only one bin # specification, convert to list of correct length bins = [bins] * len(columns) i = 0 for col in columns: bin = bins[i] i += 1 if data[col].dtype == "O": # cannot bin categorical data bin = False if bin == False: if data[col].dtype == "O": chart = ( alt.Chart(data) .mark_bar(color="#64b5f6") .encode( alt.X( col, axis=alt.Axis(title=col.title()), sort=alt.SortField( field="count()", order="descending", op="values" ), ), alt.Y("count()"), ) ) else: chart = ( alt.Chart(data) .mark_bar(color="#64b5f6") .encode( alt.X( col, axis=alt.Axis(title=col.title()) ), alt.Y("count()") ) ) else: chart = ( alt.Chart(data) .mark_bar(color="#64b5f6") .encode( alt.X( col, bin=alt.Bin(maxbins=bin), axis=alt.Axis(title=col.title()) ), alt.Y("count()"), ) ) if data[col].dtype == "float64": stats = data[col].describe() else: stats = data.groupby(col)[col].agg(["count"]) stats["prop"] = stats["count"] / len(data) stats = pd.DataFrame(stats).T display(Markdown("#### Column Summary: " + col.title())) display(stats) display(chart) check_data( data, [col], missing=missing, cardinality=cardinality, float_frequency=float_frequency, category_frequency=category_frequency, outlier_function=outlier_function, title=False, )
def show_prompt(self): prompt_url = 'https://' + self.url + '?edit' prompt_link = '[Excercise](' + prompt_url + ')' display(Markdown(prompt_link))
def print_title(string): display(Markdown('**' + string + '**'))
def show(self): ''' ''' display(Markdown('<img src="./slides/' + self.name + '.png" />'))
async def async_execute_request(self, stream, ident, parent): """process an execution request, sending it on to the child kernel or launching one if it has not been started. """ # Announce that we are busy handling this request self._publish_status("busy") # Only one kernel can be acquired at the same time async with self.acquiring_kernel: # Check for configuration code first content = parent["content"] code = content["code"] config, kernel_name = self.parse_cell(content["code"]) has_config = bool(config) if has_config: # Report back that we'll be running the config code # NOTE: We are, for the time being, ignoring the silent flag, store_history, etc. self._publish_execute_input(code, parent, self.execution_count) # User is attempting to run a cell with config after the kernel is started, # so we inform them and bail if has_config and self.child_kernel is not None: self.display( Markdown(f"""## Kernel already configured and launched. You can only run the `%%kernel.{kernel_name}` cell at the top of your notebook and the start of your session. Please **restart your kernel** and run the cell again if you want to change configuration. """), parent=parent, ) # Complete the "execution request" so the jupyter client (e.g. the notebook) thinks # execution is finished reply_content = { "status": "error", # Since our result is not part of `In` or `Out`, ensure # that the execution count is unset "execution_count": None, "user_expressions": {}, "payload": {}, } metadata = { "parametrized-kernel": True, "status": reply_content["status"], } self.session.send( stream, "execute_reply", reply_content, parent, metadata=metadata, ident=ident, ) self._publish_status("idle") return kernel_display_id = hexlify(os.urandom(8)).decode("ascii") # Launching a custom kernel if self.child_kernel is None and has_config: # Start a kernel now # If there's config set, we launch with that config # If the user is requesting the kernel with config, launch it! self.display( Markdown("Launching customized runtime..."), display_id=kernel_display_id, parent=parent, ) try: self.child_kernel = await self.start_kernel( kernel_name, config) except PickRegistrationException as err: # Get access to the exception info prior to doing any potential awaiting exc_info = sys.exc_info() self.log.info(exc_info) separator = "\n" self.display( Markdown( f"""## There is no kernel magic named `{kernel_name}` These are the available kernels: {separator.join([f"* `{kernel}`" for kernel in _subkernels.list_subkernels()])} """), display_id=kernel_display_id, update=True, parent=parent, ) self.log.error(err) self._publish_execute_reply_error(exc_info, ident=ident, parent=parent) self._publish_status("idle", parent=parent) return except Exception as err: self.log.error(err) exc_info = sys.exc_info() self._publish_error(exc_info, parent=parent) self._publish_execute_reply_error(exc_info, ident=ident, parent=parent) self._publish_status("idle", parent=parent) return self.display( Markdown("Runtime ready!"), display_id=kernel_display_id, parent=parent, update=True, ) # Complete the "execution request" so the jupyter client (e.g. the notebook) thinks # execution is finished reply_content = { "status": "ok", # Our kernel setup is always the zero-th execution (In[] starts at 1) "execution_count": 0, # Note: user_expressions are not supported on our kernel creation magic "user_expressions": {}, "payload": {}, } metadata = { "parametrized-kernel": True, "status": reply_content["status"], } self.session.send( stream, "execute_reply", reply_content, parent, metadata=metadata, ident=ident, ) # With that, we're all done launching the customized kernel and # pushing updates on the kernel to the user. return # Start the default kernel to run code if self.child_kernel is None: self.display( Markdown("Preparing default kernel..."), parent=parent, display_id=kernel_display_id, ) self.child_kernel = await self.start_kernel( self.default_kernel, self.default_config) self.display( # Wipe out the previous message. # NOTE: The Jupyter notebook frontend ignores the case of an empty output for # an update_display_data so we have to publish some empty content instead. Markdown(""), parent=parent, display_id=kernel_display_id, update=True, ) self.session.send(self.child_kernel.shell, parent, ident=ident)
![Sample image with cross-references.](https://avatars3.githubusercontent.com/u/19735117?s=460&v=4){#fig:img} In this version of Pandoc image caption @fig:img works. """ # %% {echo=True} from IPython.display import Markdown import pandas as pd import numpy as np import tabulatehelper as th df = pd.DataFrame(np.random.random(16).reshape(4, 4)) Markdown(f''' {th.md_table(df)} : Table {{#tbl:table1}} ''') # %% """ Text and @tbl:table1 """ # %% {input=True, eval=False, echo=True} import pandas as pd import numpy as np df = pd.DataFrame(np.random.random(16).reshape(4, 4)) df # %% {r, echo=True} R cell: """
# data preprocessing and feature extraction # Dataset: LibriSpeech for English - read speech, designed for training and evaluating models for ASR # Features: 1. Spectrogram 2. MFCCs from data_generator import vis_train_features from IPython.display import Markdown, display from IPython.display import audio from data_generator import vis_train_features, plot_raw_audio, plot_spectrogram_feature, plot_mfcc_feature # extract label and audio feature for a single training example vis_text, vis_raw_audio, vis_mfcc_feature, vis_spectrogram_feature, vis_audio_path = vis_train_features() plot_raw_audio(vis_raw_audio) display(Markdown('** Shape of Audio Signal**:' + str(vis_raw_audio.shape))) display(Markdow('**Transcript**:' + str(vis_text))) # Step 1: Acoustic Feature for Speech Recognition # Feature 1: Spectrogram plot_spectrogram_feature(vis_spectrogram_feature) # Feature 2: MFCC s plot_mfcc_feature(vis_mfcc_feature)