def load_mode(args): core = get_plugin(args.device, args.l, args.config) log.info('IR for {} : {}'.format(args.device, args.model)) log.info('Loading blob from {}'.format(args.load)) net = get_net(model=args.model, core=core) net_layers, net_inputs, net_outputs = get_model_info(net) out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers) print_input_layers(net_inputs) print_output_layers(out_layers) layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping, user_layers=out_layers) inputs = input_processing(args.model, net_inputs, args.input, layers_map) global_accuracy = [] loaded = load_dump(args.load) for out_layer in layers_map: ref_out_layer = layers_map[out_layer] if out_layer == ref_out_layer: log.info('Layer {} statistics'.format(out_layer)) else: log.info('Statistics \'{}\' vs \'{}\''.format(out_layer, ref_out_layer)) net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) if out_layer not in results: continue out_blob, pc = results[out_layer] if ref_out_layer not in loaded: continue ref_out_blob = loaded[ref_out_layer]['blob'] a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob) if 'pc' in loaded[ref_out_layer]: ref_pc = loaded[ref_out_layer]['pc'] performance_metrics(pc=pc, ref_pc=ref_pc) blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob) global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) print_all_over_the_net_metrics(global_accuracy=global_accuracy)
def one_ir_mode(args): net = get_net(model=args.model) net_layers, net_inputs, net_outputs, precision = get_model_info(net) log.info('{}:{} vs {}:{}'.format(args.device, precision, args.reference_device, precision)) log.info('The same IR on both devices: {}'.format(args.model)) out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers) print_input_layers(net_inputs) print_output_layers(out_layers) plugin = get_plugin(args.device, args.l, args.plugin_path, args.config) ref_plugin = get_plugin(args.reference_device, args.l, args.plugin_path, args.reference_config) global_accuracy = [] inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input) global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.model, out_layers=out_layers, ref_out_layers=out_layers, inputs=inputs, ref_inputs=inputs, plugin=plugin, ref_plugin=ref_plugin, layers=args.layers, num_of_iterations=args.num_of_iterations) for out_layer in out_layers: log.info('Layer {} statistics'.format(out_layer)) net_copy = get_net_copy_with_output(model=args.model, output=out_layer) results = infer(net=net_copy, plugin=plugin, inputs=inputs, output=[out_layer]) if out_layer not in results: continue out_blob, pc = results[out_layer] ref_results = infer(net=net_copy, plugin=ref_plugin, inputs=inputs, output=[out_layer]) if out_layer not in ref_results: continue ref_out_blob, ref_pc = ref_results[out_layer] a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob) performance_metrics(pc=pc, ref_pc=ref_pc) blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob) global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times, global_accuracy=global_accuracy)
def one_ir_mode(args): core = get_plugin(args.device, args.l, args.config) net = get_net(model=args.model, core=core) net_layers, net_inputs, net_outputs = get_model_info(net) log.info(f'{args.device} vs {args.reference_device}') log.info(f'The same IR on both devices: {args.model}') out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers) print_input_layers(net_inputs) print_output_layers(out_layers) ref_core = get_plugin(args.reference_device, args.l, args.reference_config) global_accuracy = [] inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input) global_times, ref_global_times = overall_accuracy_check( model=args.model, ref_model=args.model, out_layers=out_layers, ref_out_layers=out_layers, inputs=inputs, ref_inputs=inputs, core=core, device=args.device, ref_core=ref_core, ref_device=args.reference_device, layers=args.layers, num_of_iterations=args.num_of_iterations) for out_layer in out_layers: log.info(f'Layer {out_layer} statistics') net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) if out_layer not in results: continue out_blob, pc = results[out_layer] ref_results = infer(net=net_copy, core=ref_core, device=args.reference_device, inputs=inputs, output=[out_layer]) if out_layer not in ref_results: continue ref_out_blob, ref_pc = ref_results[out_layer] a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob) performance_metrics(pc=pc, ref_pc=ref_pc) blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob) global_accuracy = update_global_accuracy_matrics( global_accuracy=global_accuracy, current_accuracy=a_m) print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times, global_accuracy=global_accuracy)
def two_ir_mode(args): core = get_plugin(args.device, args.l, args.config) ref_core = get_plugin(args.reference_device, args.l, args.reference_config) net = get_net(model=args.model, core=core) net_layers, net_inputs, net_outputs = get_model_info(net) ref_net = get_net(model=args.reference_model, core=ref_core) ref_net_layers, ref_net_inputs, ref_net_outputs = get_model_info(ref_net) log.info('{} vs {}'.format(args.device, args.reference_device)) log.info('IR for {} : {}'.format(args.device, args.model)) log.info('IR for {} : {}'.format(args.reference_device, args.reference_model)) out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers) ref_out_layers = get_layers_list(ref_net_layers, ref_net_inputs, ref_net_outputs, args.layers) print_input_layers(net_inputs) print_output_layers(out_layers) layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping, user_layers=out_layers) inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input, layers_map=layers_map) ref_inputs = input_processing(model_path=args.reference_model, net_inputs=ref_net_inputs, input_file=args.input, layers_map=layers_map) global_accuracy = [] global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.reference_model, out_layers=out_layers, ref_out_layers=ref_out_layers, inputs=inputs, ref_inputs=ref_inputs, core=core, device=args.device, ref_core=ref_core, ref_device=args.reference_device, layers=args.layers, num_of_iterations=args.num_of_iterations) for out_layer in layers_map: ref_out_layer = layers_map[out_layer] if out_layer == ref_out_layer: log.info('Layer {} statistics'.format(out_layer)) else: log.info('Statistics \'{}\' vs \'{}\''.format(out_layer, ref_out_layer)) net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) ref_net_copy = get_net_copy_with_output(model=args.reference_model, output=ref_out_layer, core=ref_core) results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) if out_layer not in results: continue out_blob, pc = results[out_layer] ref_results = infer(net=ref_net_copy, core=ref_core, device=args.reference_device, inputs=ref_inputs, output=[ref_out_layer]) ref_out_blob, ref_pc = ref_results[ref_out_layer] if ref_out_layer not in ref_results: continue a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob) performance_metrics(pc=pc, ref_pc=ref_pc) blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob) global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times, global_accuracy=global_accuracy)
def majority_voting(scores_frame, true_labels_col, drop=False, t=0.5): # Extract scores and true labels true_labels = scores_frame[true_labels_col] if drop: scores = scores_frame.drop(labels=[true_labels_col] + drop, axis=1) else: scores = scores_frame.drop(labels=[true_labels_col], axis=1) # Change scores to predicted class preds = scores.copy() preds[preds < t] = 0 preds[preds >= t] = 1 # Get majority class majorities = [] for ID in preds.index: row = preds.loc[ID] mean = np.nanmean(row) if mean < t: majorities.append(0) else: majorities.append(1) #majority = pd.DataFrame(majorities, columns=['preds'], index=preds.index) # Get metrics auc_score, sensitivity, specificity, mcc, con = u.performance_metrics( y_true=true_labels, y_pred=majorities, threshold=t) # Get ROC_AUC parameters fpr, tpr, thresholds = roc_curve(y_true=true_labels, y_score=majorities) return pd.Series(majorities, index=scores.index, name='majority'), [ auc_score, sensitivity, specificity, mcc ], (fpr, tpr)
def confident_majority_voting(scores_frame, true_labels_col, confidence=0.6, drop=False, verbose=False, t=0.5): # Extract scores and true labels true_labels = scores_frame[true_labels_col] if drop: scores = scores_frame.drop(labels=[true_labels_col] + drop, axis=1) else: scores = scores_frame.drop(labels=[true_labels_col], axis=1) # Find confident scores for each sample confident_votes, labels, ids = [], [], [] no_con, no_cons = 0, {0: 0, 1: 0} for ID in scores.index: row = scores.loc[ID] con = [v for v in row if v >= confidence or v <= 1 - confidence] if len(con) < 1: #print(row, '\n') #con = [0.5] no_con += 1 if true_labels.loc[ID] == 1: no_cons[1] += 1 else: no_cons[0] += 1 else: labels.append(true_labels.loc[ID]) ids.append(ID) # Change scores to classes classes = [1 if c >= t else 0 for c in con] mean = np.nanmean(classes) if mean < t: confident_votes.append(0) else: confident_votes.append(1) # Look at samples with no confident scores if no_con > 0 and verbose: print('Fraction of samples with no confident votes: {0}/{1}'.format( no_con, len(scores.index))) print('Classbalance:', no_cons) # Get metrics auc_score, sensitivity, specificity, mcc, con = u.performance_metrics( y_true=labels, y_pred=confident_votes, threshold=t) # Get ROC_AUC parameters fpr, tpr, thresholds = roc_curve(y_true=labels, y_score=confident_votes) return pd.Series(confident_votes, index=ids, name='con_vot_' + str(round(confidence, 2))), [ auc_score, sensitivity, specificity, mcc ], (fpr, tpr)
def evaluate_performance(self): self.performance = {} for set in ["train", "test"]: auc, unc, fpr, tpr, tpr_unc = utils.performance_metrics( self.data[set]["label"], self.prediction[set], self.data[set]["evt_weight_"], 10) #auc, unc, fpr, tpr, thresh = utils.auc_and_unc(self.data[set]["label"], self.prediction[set], self.data[set]["evt_weight_"], 10) self.performance[set] = { "auc": auc, "auc_unc": unc, "fpr": fpr, "tpr": tpr, "tpr_unc": tpr_unc, } print( "[MVA_HELPER] ASSESSING PERFORMANCE -- %s set: AUC = %.3f +/- %.3f" % (set, self.performance[set]["auc"], self.performance[set]["auc_unc"])) self.make_plots() self.save_performance()
def average_scoring(scores_frame, true_labels_col, drop=False, t=0.5): # Extract scores and true labels true_labels = scores_frame[true_labels_col] if drop: scores = scores_frame.drop(labels=[true_labels_col] + drop, axis=1) else: scores = scores_frame.drop(labels=[true_labels_col], axis=1) # Get mean of each score means = np.nanmean(scores, axis=1) # Get metrics auc_score, sensitivity, specificity, mcc, con = u.performance_metrics( y_true=true_labels, y_pred=means, threshold=t) # Get ROC_AUC parameters fpr, tpr, thresholds = roc_curve(y_true=true_labels, y_score=means) return pd.Series(means, index=scores.index, name='mean'), [auc_score, sensitivity, specificity, mcc], (fpr, tpr)
def train_and_evaluate_classifier(name, clf, validation_data, validation_labels): clf.fit(validation_data) predictions = np.array(clf.predict(validation_data)) if name in PYOD_ALGOS: predictions = np.where(predictions == 1, -1, predictions) predictions = np.where(predictions == 0, 1, predictions) tp, tn, fp, fn, tpr, tnr, ppv, npv, ts, pt, acc, f1, mcc = performance_metrics( validation_labels, predictions) result = [name, tp, tn, fp, fn, tpr, tnr, ppv, npv, ts, pt, acc, f1, mcc] print("%s results:" % (name)) print( tabulate([result], [ "ALGO", "TP", "TN", "FP", "FN", "TPR", "TNR", "PPV", "NPV", "TS", "PT", "ACC", "F1", "MCC" ], tablefmt="grid")) return result
def sklearn_model(X_data, y_data, model, seed=seed, cv_splits=5, stratified=True, permute=False, verbose=False): """ Parameter optimization can be built on top of this function. X_data: Model data, input as pandas dataframe. y_data: True labels, input as numpy 1d array. model: Sklearn class classifier object with function ".predict_proba()". seed: Random seed, to make reproducable results. cv_splits: Number of crossvalidation splits, max = N. stratified: Whether or not to make k-fold splits stratified by target classes. Only used when cv_splits < N. verbose: if True, prints results. """ # Check IDs are in same order for X and y intersect = X_data.index.intersection(y_data.index) X_data, y_data = X_data.loc[intersect], y_data.loc[intersect] print(X_data.shape, y_data.shape) # Data shapes N, M = X_data.shape # If permutation test is enabled if permute: y_shuffle = y_data['y_shuffle'] y_data = y_data['y'] # Set cross-validation (partition vector) if cv_splits == N: part_vec = np.array(list(range(N))) else: if stratified: CV = StratifiedKFold(n_splits=cv_splits, shuffle=True, random_state=seed) # Split stratified by target value else: CV = KFold(n_splits=cv_splits, shuffle=True, random_state=seed) # Random split part_vec, k = np.zeros(N), 0 for train_index, test_index in CV.split(X_data, y_data): # y must here be made a 1d array if error occurs part_vec[test_index] = k k += 1 # Set lists and parameters train_scores, test_scores = [], [] train_pred, test_pred = [], [] y_target_train, y_target_test = [], [] ids_test, model_list = [], [] X, y = X_data.values, y_data.values.ravel() if permute: ys = y_shuffle.values.ravel() # Run through cross-validation for k in range(cv_splits): if verbose: print('\n# K-fold {0}/{1}'.format(k+1, cv_splits)) # Get training and test data train_index, test_index = np.where(part_vec != k), np.where(part_vec == k) X_train, X_test = X[train_index], X[test_index] print(X_train.shape, X_test.shape) if permute: y_train, y_test = y[train_index], ys[test_index] else: y_train, y_test = y[train_index], y[test_index] # Get the IDs of the test group test_ids = X_data.index[test_index] ids_test.append(test_ids) # Classbalance unique, counts = np.unique(y_train, return_counts=True) print('\tClassbalance in training is:\t', dict(zip(unique, counts))) unique, counts = np.unique(y_test, return_counts=True) print('\tClassbalance in test is:\t', dict(zip(unique, counts)), '\n') # Train model model.fit(X_train, y_train) # Calculate metrics if cv_splits == N: # Get prediction on training and test set pred_train = model.predict_proba(X_train)[:,1] pred_test = model.predict_proba(X_test)[:,1] else: # Get prediction on training and test set pred_train = model.predict_proba(X_train)[:,1] pred_test = model.predict_proba(X_test)[:,1] #auc.append(roc_auc_score(y_test, y_test_pred)) auc_train, sens_train, spec_train, mcc_train, con_train = u.performance_metrics(y_true=y_train, y_pred=pred_train) auc_test, sens_test, spec_test, mcc_test, con_test = u.performance_metrics(y_true=y_test, y_pred=pred_test) # Append metrics to lists if verbose: print('AUC train = {0}, AUC test = {1}'.format(auc_train, auc_test)) train_scores.append([auc_train, sens_train, spec_train, mcc_train]) test_scores.append([auc_test, sens_test, spec_test, mcc_test]) # Append labels and predictions #model_list.append(model) model_list.append(model.feature_importances_.tolist()) # save feature importances instead train_pred.append(pred_train) test_pred.append(pred_test) y_target_train.append(y_train) y_target_test.append(y_test) k += 1 # convert to long np array: y_train_pred=np.array(train_pred) y_test_pred=np.array(test_pred) y_target_train=np.array(y_target_train) y_target_test=np.array(y_target_test) ids_test=np.array(ids_test) # Flatten arrays y_train_pred=np.concatenate(y_train_pred).flatten() y_test_pred=np.concatenate(y_test_pred).flatten() y_target_train=np.concatenate(y_target_train).flatten() y_target_test=np.concatenate(y_target_test).flatten() ids_test=np.concatenate(ids_test).flatten() # Make dataframe for metrics auc_train, sens_train, spec_train, mcc_train, con_train = u.performance_metrics(y_true=y_target_train, y_pred=y_train_pred) auc_test, sens_test, spec_test, mcc_test, con_test = u.performance_metrics(y_true=y_target_test, y_pred=y_test_pred) metrics_train = pd.DataFrame(data=np.array([[auc_train], [sens_train], [spec_train], [mcc_train]]).T, columns=['auc', 'sens', 'spec', 'mcc'], index=[seed]) metrics_test = pd.DataFrame(data=np.array([[auc_test], [sens_test], [spec_test], [mcc_test]]).T, columns=['auc', 'sens', 'spec', 'mcc'], index=[seed]) metrics = (metrics_train, metrics_test) if verbose: print("\n# Train AUC: " + str(round(auc_train,4))) print("# Test AUC: " + str(round(auc_test,4))) # calculate false positive and true positive rate at different thresholds (ROC curve): fpr,tpr,thresholds = roc_curve(y_target_test, y_test_pred, drop_intermediate=False) return metrics, (fpr, tpr), ids_test, (y_test_pred, y_target_test), thresholds, model_list