def test_sensitivity_specificity_f_binary_single_class(): # Such a case may occur with non-stratified cross-validation assert_equal(1., sensitivity_score([1, 1], [1, 1])) assert_equal(0., specificity_score([1, 1], [1, 1])) assert_equal(0., sensitivity_score([-1, -1], [-1, -1])) assert_equal(0., specificity_score([-1, -1], [-1, -1]))
def test_sensitivity_specificity_f_binary_single_class(): # Such a case may occur with non-stratified cross-validation assert sensitivity_score([1, 1], [1, 1]) == 1. assert specificity_score([1, 1], [1, 1]) == 0. assert sensitivity_score([-1, -1], [-1, -1]) == 0. assert specificity_score([-1, -1], [-1, -1]) == 0.
def test_sensitivity_specificity_error_multilabels(): y_true = [1, 3, 3, 2] y_pred = [1, 1, 3, 2] y_true_bin = label_binarize(y_true, classes=np.arange(5)) y_pred_bin = label_binarize(y_pred, classes=np.arange(5)) with pytest.raises(ValueError): sensitivity_score(y_true_bin, y_pred_bin)
def test_sensitivity_specificity_error_multilabels(): y_true = [1, 3, 3, 2] y_pred = [1, 1, 3, 2] y_true_bin = label_binarize(y_true, classes=np.arange(5)) y_pred_bin = label_binarize(y_pred, classes=np.arange(5)) with pytest.raises(ValueError): sensitivity_score(y_true_bin, y_pred_bin)
def test_sensitivity_specificity_f_binary_single_class(): """Test sensitivity and specificity behave with a single positive or negative class""" # Such a case may occur with non-stratified cross-validation assert_equal(1., sensitivity_score([1, 1], [1, 1])) assert_equal(0., specificity_score([1, 1], [1, 1])) assert_equal(0., sensitivity_score([-1, -1], [-1, -1])) assert_equal(0., specificity_score([-1, -1], [-1, -1]))
def performance_summary( clf: OptimalSamplingClassifier, X: np.ndarray, y: np.ndarray, info: Optional[Dict[str, any]] = None, ) -> Dict[str, float]: predicted_proba = clf.predict_proba(X) predicted = clf.predict(X) nominal_proba = (y == clf.positive_class).mean() return dict(model=str(clf.estimator).replace("\n", "").replace(" ", ""), class_ratio=1 / nominal_proba, weight_ratio=clf.positive_weight / clf.negative_weight, sampling_probability=clf._sampling_proba, previous_probabilities=clf._prev_sampling_probas, cross_val_probabilities=clf._cross_val_sampling_probas, sampling_ratio=clf._sampling_proba / nominal_proba, iter_to_converge=clf._iter_count, accuracy=accuracy_score(y, predicted), sensitivity=sensitivity_score(y, predicted), specificity=specificity_score(y, predicted), precision=precision_score(y, predicted) if (predicted == clf.positive_class).sum() > 0 else None, recall=recall_score(y, predicted) if (predicted == clf.positive_class).sum() > 0 else None, f1_score=f1_score(y, predicted), geometric_mean_score=geometric_mean_score(y, predicted) if (predicted == clf.positive_class).sum() > 0 else None, roc_auc_score=roc_auc_score(y, predicted_proba), average_precision_score=average_precision_score( y, predicted_proba), weighted_loss=clf.weighted_loss(X, y).mean(), cost=clf.cost(X, y).mean(), **(info if info else {}))
def identification(method, rs_cnt, batch_size): out_file = open( os.path.join(os.path.abspath(""), "results", "{}_{}_vggface2.txt".format(method, rs_cnt)), "w") data_file = os.path.join(os.path.abspath(""), "data", "vggface2_{}_dataset.hdf5".format(method)) train, val, test = [ h5py.File(data_file, "r")[part] for part in ["train", "val", "test"] ] rs = np.load( os.path.join(os.path.abspath(""), "data", "rs_{}_feat.npz".format(method)))["arr_0"] mlp = get_mlp(512, np.unique(train[:, -1].astype(int) - 1).shape[0]) mlp = train_mlp(mlp, train, val, rs, rs_cnt, method, batch_size) y_prob = predict_mlp(mlp, test, rs, rs_cnt) y_pred = np.argmax(y_prob, axis=-1) y_true = test[:, -1].astype(int) - 1 out_file.write("ACC -- {:.6f}\n".format(accuracy_score(y_true, y_pred))) out_file.write("FPR -- {:.6f}\n".format( 1 - sensitivity_score(y_true, y_pred, average="micro"))) out_file.write("FRR -- {:.6f}\n".format( 1 - specificity_score(y_true, y_pred, average="micro"))) out_file.write("PRE -- {:.6f}\n".format( precision_score(y_true, y_pred, average="micro"))) out_file.write("REC -- {:.6f}\n".format( recall_score(y_true, y_pred, average="micro"))) out_file.write("F1 -- {:.6f}\n".format( f1_score(y_true, y_pred, average="micro"))) out_file.close()
def compute_metrics(y_test, y_pred, y_proba=None, average='weighted', return_index=False): """ Function computing metrics of interest for a sets of prediction :input y_test: pd.DataFrame or np.array of original label :input y_pred: pd.DataFrame or np.array of predicted label :output red: list of value for metrics, in order - Accuracy - Precision - Recall - F1 Score - Sensitivity - Specifity """ if return_index: return [ 'accuracy', 'precision', 'recall', 'f1_score', 'sensitivity_score', 'specificity_score', 'geometric_mean_score', 'average_precision_score' ] else: res = [] res.append(accuracy_score(y_test, y_pred)) res.append(precision_score(y_test, y_pred, average=average)) res.append(recall_score(y_test, y_pred, average=average)) res.append(f1_score(y_test, y_pred, average=average)) res.append(sensitivity_score(y_test, y_pred, average=average)) res.append(specificity_score(y_test, y_pred, average=average)) res.append(geometric_mean_score(y_test, y_pred, average=average)) if y_proba is not None: res.append( average_precision_score(y_test, y_proba, average=average)) return res
def print_metrics(model, test, cols): test_x = test.drop(['Label'], axis=1) test_y = test['Label'] y_probs = model.predict_proba(test_x[cols])[:, 1] y_test_predictions = np.where( model.predict_proba(test_x[cols])[:, 1] > 0.119, 2, 1) n_levels = test_y.value_counts().count() if (n_levels == 1): #print(test_x.shape) return y_test_predictions else: mcc = metrics.matthews_corrcoef(test_y, y_test_predictions) tn, fp, fn, tp = confusion_matrix(test_y, y_test_predictions).ravel() ppv = tp / (tp + fp) npv = tn / (tn + fn) sen = tp / (tp + fn) spe = tn / (tn + fp) score = ppv + npv + sen + spe #y_test_predictions=model.predict(test_x[cols]) sensi = sensitivity_score(test_y, y_test_predictions, pos_label=2) speci = specificity_score(test_y, y_test_predictions, pos_label=2) accu = accuracy_score(test_y, y_test_predictions) auro = roc_auc_score(test_y, y_test_predictions) #acc=accuracy_score(test_y,y_test_predictions) print("Composite Score for Martelotto et al.: ", score) return y_test_predictions
def calc_metrics(y_test, pred, auc, i): sen = metrics.sensitivity_score(y_test, pred, pos_label=1) spe = metrics.specificity_score(y_test, pred, pos_label=1) geo = metrics.geometric_mean_score(y_test, pred, pos_label=1) index = ['sm', 'b1', 'b2', 'enn', 'tom', 'ada', 'mnd'] metrics_list = [index[i], sen, spe, geo, auc] return metrics_list
def GetMetrics(self, y_test): tn, fp, fn, tp = confusion_matrix(y_test, self.y_pred).flatten() self.acc = '%.3f' % accuracy_score(y_test, self.y_pred) self.AUC = '%.3f' % roc_auc_score(y_test, self.y_pred_proba_pos) # self.sens = tp/(tp+fn) self.sens = '%.3f' % sensitivity_score(y_test, self.y_pred) # self.spec = '%.3f'%(tn/(tn+fp)) self.spec = '%.3f' % specificity_score(y_test, self.y_pred)
def classification_results(train,test): #Derivation of NBDriver using training data """ Arguments: train = feature matrix derived from Brown et al. test= feature matrix derived from Martelotto et al. Returns: best_model = Best ensemble model derived using the training data X_red= Dataframe derived after sampling that was used to train the model scores= probability based classification scores """ sen=[];spe=[];acc=[];auc=[];c=[];m=[];s=[] train_x=train.drop('Label',axis=1);train_y=train['Label']; test_x=test.drop('Label',axis=1);test_y=test['Label']; #Random undersampling to reduce the majority class size samp=RepeatedEditedNearestNeighbours(random_state=42) X_samp,y_samp=samp.fit_resample(train_x,train_y) X_samp = pd.DataFrame(X_samp, columns = train_x.columns) #Experimenting with different numbers of top features derived from the tree-based feature extraction method top_n_feats=[30,40,50,60,70] X_r=feature_reduction_using_trees(X_samp,y_samp) cols=X_r.columns for n in top_n_feats: print("For top: ",n," features") X_red=X_r[cols[0:n]] sv=SVC(kernel="linear",probability=True,C=0.01,random_state=42) #chosen from 5foldCV based grid search kde=KDEClassifier(bandwidth=1.27) #chosen from 5foldCV based grid search best_model = VotingClassifier(estimators=[('sv', sv), ('kde', kde)], voting='soft',weights=[4, 7]) #best combination of weights selected by a brute force search (possible weights 1-10) using a cross-validation approach on the training data best_model.fit(X_red,y_samp) y_probs = best_model.predict_proba(test_x[X_red.columns])[:,1] thresholds = arange(0, 1, 0.001) scores = [roc_auc_score(test_y, to_labels(y_probs, t)) for t in thresholds] ix= argmax(scores) y_test_predictions = np.where(best_model.predict_proba(test_x[X_red.columns])[:,1] > thresholds[ix], 2, 1) print("Thresh: ",thresholds[ix]) sensi= sensitivity_score(test_y, y_test_predictions, pos_label=2) speci=specificity_score(test_y,y_test_predictions,pos_label=2) accu=accuracy_score(test_y,y_test_predictions) auro=roc_auc_score(test_y,y_test_predictions) mcc=metrics.matthews_corrcoef(test_y,y_test_predictions) tn, fp, fn, tp = confusion_matrix(test_y, y_test_predictions).ravel() ppv=tp/(tp+fp) npv=tn/(tn+fn) sen=tp/(tp+fn) spe=tn/(tn+fp) score=ppv+npv+sen+spe print("For kmer size: ",len(train.columns[0])) print("for top ",n," features") print(list(X_red.columns.values),"\n") score_dict={"Sen":sen,"Spe":spe,"PPV":ppv,"NPV":npv,"AUC":auro,"MCC":mcc,"ACC":accu} print(score) print(score_dict) df=pd.DataFrame(y_test_predictions) y_samp = pd.DataFrame(y_samp, columns = ['x']) return best_model,X_red,scores
def compute_metrics(gt, pred, competition=True): """ Computes accuracy, precision, recall and F1-score from prediction scores. Args: gt: Pytorch tensor on GPU, shape = [n_samples, n_classes] true binary labels. pred: Pytorch tensor on GPU, shape = [n_samples, n_classes] can either be probability estimates of the positive class, confidence values, or binary decisions. competition: whether to use competition tasks. If False, use all tasks Returns: List of AUROCs of all classes. """ AUROCs, Accus, Senss, Recas, Specs = [], [], [], [], [] gt_np = gt.cpu().detach().numpy() # if cfg.uncertainty == 'U-Zeros': # gt_np[np.where(gt_np==-1)] = 0 # if cfg.uncertainty == 'U-Ones': # gt_np[np.where(gt_np==-1)] = 1 pred_np = pred.cpu().detach().numpy() THRESH = 0.18 # indexes = TARGET_INDEXES if competition else range(N_CLASSES) #indexes = range(n_classes) # pdb.set_trace() indexes = range(len(CLASS_NAMES)) for i, cls in enumerate(indexes): try: AUROCs.append(roc_auc_score(gt_np[:, i], pred_np[:, i])) except ValueError as error: print('Error in computing accuracy for {}.\n Error msg:{}'.format(i, error)) AUROCs.append(0) try: Accus.append(accuracy_score(gt_np[:, i], (pred_np[:, i]>=THRESH))) except ValueError as error: print('Error in computing accuracy for {}.\n Error msg:{}'.format(i, error)) Accus.append(0) try: Senss.append(sensitivity_score(gt_np[:, i], (pred_np[:, i]>=THRESH))) except ValueError: print('Error in computing precision for {}.'.format(i)) Senss.append(0) try: Specs.append(specificity_score(gt_np[:, i], (pred_np[:, i]>=THRESH))) except ValueError: print('Error in computing F1-score for {}.'.format(i)) Specs.append(0) return AUROCs, Accus, Senss, Specs
def calc_metrics(y_test, pred, i): sen = metrics.sensitivity_score(y_test, pred, average='micro') spe = metrics.specificity_score(y_test, pred, average='micro') geo = metrics.geometric_mean_score(y_test, pred, average='micro') f1 = f1_score(y_test, pred, average='micro') mcc = matthews_corrcoef(y_test, pred) index = ['sm', 'b1', 'b2', 'enn', 'tom', 'ada', 'mnd'] metrics_list = [index[i], sen, spe, geo, f1, mcc] return metrics_list
def calc_metrics(y_test, pred, auc, i): sen = metrics.sensitivity_score(y_test, pred, pos_label=1) spe = metrics.specificity_score(y_test, pred, pos_label=1) geo = metrics.geometric_mean_score(y_test, pred, pos_label=1) f1 = f1_score(y_test, pred, pos_label=1) mcc = matthews_corrcoef(y_test, pred) index = ['original', 'sm', 'b1', 'b2', 'enn', 'tom', 'ada', 'mnd'] metrics_list = [index[i], sen, spe, geo, f1, mcc, auc] return metrics_list
def train_and_predict(classifier, X_train, y_train, X_test, y_test): start = timer() classifier.fit(X_train, y_train) end = timer() predictions = classifier.predict(X_test) results = OrderedDict() results['recall'] = metrics.sensitivity_score(y_test, predictions, pos_label=1, average='binary') results['gmean'] = metrics.geometric_mean_score(y_test, predictions, pos_label=1, average='binary') results['fmeasure'] = f1_score(y_test, predictions, average=None)[1] results['auc'] = roc_auc_score(y_test, predictions, average=None) results['time'] = end-start return results
def metrique_classe(y_pred, y_true, xclass): from imblearn.metrics import specificity_score from imblearn.metrics import sensitivity_score from imblearn.metrics import geometric_mean_score # La sensibilité est le rapport où est le nombre de vrais positifs et le nombre de faux négatifs. # La sensibilité quantifie la capacité à éviter les faux négatifs.tp # estimator issu de quelques FIT log.traceLogInfo("Classe ", xclass) if xclass == 0: log.traceLogInfo("Classe 0") if xclass == 1: log.traceLogInfo("Classe 1") log.traceLogInfo("Sensibilité du re-equilibrage des données sur le TEST") #log.traceLogInfo("Binary ",sensitivity_score(y_true, y_pred, average='binary', pos_label=xclass)) log.traceLogInfo( "La spécificité est intuitivement la capacité du classificateur à trouver tous les échantillons positifs" ) log.traceLogInfo("Binary ") log.traceLogInfo( specificity_score(y_true, y_pred, labels=None, pos_label=xclass, average='binary', sample_weight=None)) print("\nCalculer la moyenne géométrique") print(geometric_mean_score(y_true, y_pred, labels=None, pos_label=xclass)) print("\n Calculer sensitivity score") print( "La sensibilité est le rapport où est le nombre de vrais positifs et le nombre de faux négatifs." ) print("La sensibilité quantifie la capacité à éviter les faux négatifs.") print( sensitivity_score(y_true, y_pred, labels=None, pos_label=xclass, average='binary'))
def fit(self, x, y, x_test, y_test): # xPos = [] xNeg = [] xTrain = [] yTrain = [] xlastTrain = [] ylastTrain = [] for i in range(0, len(y)): if y[i] == 1: xPos.append(x[i]) xlastTrain.append(x[i]) ylastTrain.append(y[i]) else: xNeg.append(x[i]) xTrain.append(x[i]) yTrain.append(y[i]) xNLSV = [] iterRecord = 0 for i in range(0, self.T): svc = SVC(C=self.C, class_weight=self.class_weight, degree=self.degree, gamma=self.gamma, kernel='linear') print(iterRecord) iterRecord += 1 svc.fit(xTrain, yTrain) sv = svc.support_ # This is support vector xTrain, yTrain, xNLSV, lastMar = self.rebuild( xTrain, yTrain, sv, xNLSV) # rebuild sample #print (lastMar) if lastMar < 0.1 * len(xPos): break for i in xNLSV: xlastTrain.append(i) ylastTrain.append(-1) self.allSVC.fit(xlastTrain, ylastTrain) y_pre = self.allSVC.predict(x_test) ACC = accuracy_score(y_test, y_pre) SN = sensitivity_score(y_test, y_pre) SP = specificity_score(y_test, y_pre) MCC = matthews_corrcoef(y_test, y_pre) return SN, SP, ACC, MCC
def sensitivity_weighted(output, target): """ Sensitivity = TP / (TP + FN) :param output: Batch x Channel x .... :param target: Batch x .... :return: """ with torch.no_grad(): if len(output.shape) == (len(target.shape) + 1): # reverse one-hot encode output output = torch.argmax(output, 1) assert output.shape == target.shape, "The output size should be the same or one dimension more than the shape of target." output = output.cpu().detach().numpy() target = target.cpu().detach().numpy() score = sensitivity_score(target, output, average='weighted') return score
def get_sensitivity_tc(self, y_true, y_pred): return imb_metrics.sensitivity_score(y_true, y_pred, average='binary') * 100
print('--- Extratrees model ---') extra = ExtraTreesClassifier(random_state=20) extra.fit(X_train, y_train) y_score = extra.predict(X_test) acc = accuracy_score(y_test, y_score) acc_list.append(acc) precision = precision_score(y_test, y_score, average='macro') precision_list.append(precision) recall = recall_score(y_test, y_score, average='macro') recall_list.append(recall) sensitivity = sensitivity_score(y_test, y_score, average='macro') sensitivity_list.append(sensitivity) specificity = specificity_score(y_test, y_score, average='macro') specificity_list.append(specificity) f1 = f1_score(y_test, y_score, average='macro') f1_list.append(f1) gmean = geometric_mean_score(y_test, y_score, average='macro') g_list.append(gmean) mat = confusion_matrix(y_test, y_score) # tn,fp,fn,tp=confusion_matrix(Y_test, Y_test_predict).ravel() # print(tn,fp,fn,tp)
def model_fitting(model_name, train_x, val_x, train_y, val_y, EPOCHS=20, BATCH_SIZE=32): global model_history, best_model_ind, val_conf_mat_tp, cross_val_prec_list, cross_val_rcl_list, scale_positive_weight global cross_val_f1_list, cross_val_rocauc_list, cross_val_sens_list, cross_val_spec_list, cross_val_gm_list global bal_tech if model_name.lower() == "lr": logging.info("Fitting a Logistic Regression Model...") if bal_tech == "weighted": scikit_log_reg = LogisticRegression(verbose=1, solver='liblinear', class_weight='balanced', max_iter=1000) else: scikit_log_reg = LogisticRegression(verbose=1, solver='liblinear', max_iter=1000) event_time = time.time() model = scikit_log_reg.fit(train_x, train_y) logging.info("Training time for LR model: %s", (time.time() - event_time)) elif model_name.lower() == 'gb': logging.info("Fitting a Gradient Boost Model...") if bal_tech == "weighted": scikit_gradboost = XGBClassifier( scale_pos_weight=scale_positive_weight) else: scikit_gradboost = XGBClassifier() event_time = time.time() model = scikit_gradboost.fit(train_x, train_y) logging.info("Training time for GB model: %s", (time.time() - event_time)) elif model_name.lower() == 'rf': logging.info("Fitting a Random Forest Model...") if bal_tech == "weighted": scikit_randomforest = RandomForestClassifier( class_weight='balanced_subsample') else: scikit_randomforest = RandomForestClassifier() event_time = time.time() model = scikit_randomforest.fit(train_x, train_y) logging.info("Training time for RF model: %s", (time.time() - event_time)) event_time = time.time() train_pred_y = model.predict(train_x) logging.info("Training-Prediction time: %s", (time.time() - event_time)) conf_mat = confusion_matrix(train_y, train_pred_y) logging.info("Confusion Matrix") logging.info("%s", conf_mat) logging.info("%s", classification_report(train_y, train_pred_y)) logging.info("Cross-Validating with validation data...") event_time = time.time() val_pred_y = model.predict(val_x) ##Sensitivity -- Recall of +ve class (in binary classification) ##Specificity -- Recall of -ve class (in binary classification) logging.info("Cross-Validation Prediction time: %s", (time.time() - event_time)) val_conf_mat = confusion_matrix(val_y, val_pred_y) logging.info("%s", val_conf_mat) val_conf_mat_tp.append(val_conf_mat[1][1]) logging.info("%s", classification_report(val_y, val_pred_y)) cross_val_prec_list.append( precision_score(val_y, val_pred_y, average='binary')) cross_val_rcl_list.append(recall_score(val_y, val_pred_y, average='binary')) cross_val_f1_list.append(f1_score(val_y, val_pred_y, average='binary')) cross_val_rocauc_list.append(roc_auc_score(val_y, val_pred_y)) cross_val_sens_list.append( specificity_score(val_y, val_pred_y, average='binary')) cross_val_spec_list.append( sensitivity_score(val_y, val_pred_y, average='binary')) cross_val_gm_list.append( geometric_mean_score(val_y, val_pred_y, average='binary')) logging.info("Precision: %s ", precision_score(val_y, val_pred_y, average='binary')) logging.info("Recall: %s ", recall_score(val_y, val_pred_y, average='binary')) logging.info("F1: %s ", f1_score(val_y, val_pred_y, average='binary')) logging.info("ROC-AUC: %s ", roc_auc_score(val_y, val_pred_y)) logging.info("Specificity: %s ", specificity_score(val_y, val_pred_y, average='binary')) logging.info("Sensitivity: %s ", sensitivity_score(val_y, val_pred_y, average='binary')) logging.info("Geometric Mean: %s", geometric_mean_score(val_y, val_pred_y, average='binary')) return model
def test_stats(model, test_inp, ground_truth_inp, smplng_strtgy_ky, tst_proj_ky, tst_df_cmnts): rslt = {} tst_df = pd.DataFrame() global tst_rslt_dic, mstr_tst_df event_time = time.time() y_class = model.predict(test_inp) y_pred_prob = model.predict_proba(test_inp)[:, 1] logging.info("-inference time- %s seconds ---" % (time.time() - event_time)) logging.info(classification_report(ground_truth_inp, y_class)) logging.info( precision_recall_fscore_support(ground_truth_inp, y_class, average='binary')) logging.info(confusion_matrix(ground_truth_inp, y_class)) rslt["prec"] = round( precision_score(ground_truth_inp, y_class, average='binary'), 3) rslt["rcl"] = round( recall_score(ground_truth_inp, y_class, average='binary'), 3) rslt["f1"] = round(f1_score(ground_truth_inp, y_class, average='binary'), 3) rslt["roc_auc"] = round(roc_auc_score(ground_truth_inp, y_class), 3) rslt["spec"] = round( specificity_score(ground_truth_inp, y_class, average='binary'), 3) rslt["sens"] = round( sensitivity_score(ground_truth_inp, y_class, average='binary'), 3) rslt["gm"] = round( geometric_mean_score(ground_truth_inp, y_class, average='binary'), 3) tst_rslt_dic[tst_proj_ky] = rslt tst_df_cmnts = tst_df_cmnts logging.info("type ground_truth_inp: %s,%s", type(ground_truth_inp), ground_truth_inp.shape) logging.info("type y_class: %s,%s", type(y_class), y_class.shape) logging.info("type y_pred_prob: %s,%s", type(y_pred_prob), y_pred_prob.shape) logging.info("type tst_df_cmnts: %s,%s", type(tst_df_cmnts), tst_df_cmnts.shape) if tst_proj_ky != 'overall_test': logging.info(classification_report(ground_truth_inp, y_class)) data = { 'Project': ground_truth_inp, 'Comments': tst_df_cmnts, 'Ground_Truth': ground_truth_inp, 'Predicted_Class': y_class, 'Predicted_Probability': y_pred_prob } tst_df = pd.DataFrame(data) tst_df['Project'] = tst_proj_ky logging.info("tst_df shape: %s", tst_df.shape) logging.info("tst_df : %s", tst_df) mstr_tst_df = mstr_tst_df.append(tst_df, ignore_index=True) logging.info("mstr_tst_df shape: %s", mstr_tst_df.shape) logging.info("%s : %s : Precision: %s ", smplng_strtgy_ky, tst_proj_ky, rslt["prec"]) logging.info("%s : %s : Recall: %s ", smplng_strtgy_ky, tst_proj_ky, rslt["rcl"]) logging.info("%s : %s : F1: %s ", smplng_strtgy_ky, tst_proj_ky, rslt["f1"]) logging.info("%s : %s : ROC-AUC: %s ", smplng_strtgy_ky, tst_proj_ky, rslt["roc_auc"]) logging.info("%s : %s : Specificity: %s ", smplng_strtgy_ky, tst_proj_ky, rslt["spec"]) logging.info("%s : %s : Sensitivity: %s ", smplng_strtgy_ky, tst_proj_ky, rslt["sens"]) logging.info("%s : %s : Geometric Mean: %s", smplng_strtgy_ky, tst_proj_ky, rslt["gm"])
# K-means with correct k num_clusters = 2 minority = (x_cluster[i]["Class"].values == 1).sum() min_class = [] labels = [] center = [] for i in range(0, 9): model = KMeans(n_clusters=num_clusters).fit(x_cluster[i]) labels.append(model.labels_) center.append(model.cluster_centers_) min_class.append(minority) # Assign X to Closest Cluster y_pred = model.predict(x_cluster[9]) score = geometric_mean_score(x_cluster[9]['Class'], y_pred, average=None) sensitivity = sensitivity_score(x_cluster[9]['Class'], y_pred, average='micro') specificity = specificity_score(x_cluster[9]['Class'], y_pred, average='micro') scores.append(score) sense.append(sensitivity) specs.append(specificity) youden.append((sensitivity + specificity - 1)) gmean.append(math.sqrt(specificity * sensitivity)) # Compute Metric Scores x_dp = sensitivity / (1 - sensitivity) y_dp = specificity / (1 - specificity) dp = (math.sqrt(3) / math.pi) print("Sensitivity ", sensitivity) print("Specificity ", specificity) print("Youden's Index Score: ", (sensitivity + specificity - 1)) print('Geometric Mean Score: ', math.sqrt(specificity * sensitivity))
def test_sensitivity_specificity_f_binary_single_class( y_pred, expected_sensitivity, expected_specificity): # Such a case may occur with non-stratified cross-validation assert sensitivity_score(*y_pred) == expected_sensitivity assert specificity_score(*y_pred) == expected_specificity
def run_repeatedCV(): sen_df = pd.DataFrame() spe_df = pd.DataFrame() auc_df = pd.DataFrame() mcc_df = pd.DataFrame() k = 0 pos_data, kmer_data = pd_read_pattern() new_kmer = shuffle_data(kmer_data) #KDE kmer using top 30 percentile features with tfidf scores for test derived from train using transform() for i in range(0, 10): ctr = 0 print("Window size: ", i + 1, "\n") if (i == 0): names2 = ['df_2_cv', 'df_3_cv', 'df_2_tf', 'df_3_tf'] else: names2 = [ 'df_2_cv', 'df_3_cv', 'df_4_cv', 'df_2_tf', 'df_3_tf', 'df_4_tf' ] X = new_kmer[i] y = new_kmer[i]['Label'] print("kmer size", names2[ctr]) ctr = ctr + 1 rskf = RepeatedStratifiedKFold(n_splits=10, n_repeats=3) for train_index, test_index in rskf.split(X, y): sen_kde = [] spe_kde = [] acc_kde = [] auc_kde = [] m_kde = [] c = [] k = k + 1 print(k, end=",") X_train, X_test = X.iloc[train_index], X.iloc[test_index] dat_train, dat_test, names = cv_tf_transformation(X_train, X_test) for j in range(0, len(dat_train)): dat_train[j]['Chr'] = dat_train[j]['Chr'].replace(['X'], '21') dat_test[j]['Chr'] = dat_test[j]['Chr'].replace(['X'], '21') train_x = dat_train[j].drop('Label', axis=1) train_y = dat_train[j]['Label'] test_x = dat_test[j].drop('Label', axis=1) test_y = dat_test[j]['Label'] X_red = feature_reduction_using_trees(train_x, train_y) rf = RandomForestClassifier() param_grid = { 'n_estimators': [50, 100, 200, 300, 400], 'max_features': ['auto', 'sqrt', 'log2'], 'max_depth': [2, 3, 5, 7], 'min_samples_leaf': [1, 3], 'min_samples_split': [2, 5, 10], } grid = GridSearchCV(rf, param_grid, cv=3) grid.fit(X_red, train_y.ravel()) best_model = grid.best_estimator_ best_model.fit(X_red, train_y.ravel()) y_probs = best_model.predict_proba(test_x[X_red.columns])[:, 1] thresholds = arange(0, 1, 0.001) scores = [ roc_auc_score(test_y, convert_to_labels(y_probs, t)) for t in thresholds ] ix = argmax(scores) y_test_predictions = np.where( best_model.predict_proba(test_x[X_red.columns])[:, 1] > thresholds[ix], 2, 1) sensi = sensitivity_score(test_y, y_test_predictions, pos_label=2) speci = specificity_score(test_y, y_test_predictions, pos_label=2) accu = accuracy_score(test_y, y_test_predictions) auro = roc_auc_score(test_y, y_test_predictions) mcc = metrics.matthews_corrcoef(test_y, y_test_predictions) c.append(X_red.columns) sen_kde.append(sensi) spe_kde.append(speci) acc_kde.append(accu) auc_kde.append(auro) m_kde.append(mcc) if (i == 0): sen_df = sen_df.append( { 'df_2_cv': sen_kde[0], 'df_3_cv': sen_kde[1], 'df_2_tf': sen_kde[2], 'df_3_tf': sen_kde[3] }, ignore_index=True) spe_df = spe_df.append( { 'df_2_cv': spe_kde[0], 'df_3_cv': spe_kde[1], 'df_2_tf': spe_kde[2], 'df_3_tf': spe_kde[3] }, ignore_index=True) auc_df = auc_df.append( { 'df_2_cv': auc_kde[0], 'df_3_cv': auc_kde[1], 'df_2_tf': auc_kde[2], 'df_3_tf': auc_kde[3] }, ignore_index=True) mcc_df = mcc_df.append( { 'df_2_cv': m_kde[0], 'df_3_cv': m_kde[1], 'df_2_tf': m_kde[2], 'df_3_tf': m_kde[3] }, ignore_index=True) else: sen_df = sen_df.append( { 'df_2_cv': sen_kde[0], 'df_3_cv': sen_kde[1], 'df_4_cv': sen_kde[2], 'df_2_tf': sen_kde[3], 'df_3_tf': sen_kde[4], 'df_4_tf': sen_kde[5] }, ignore_index=True) spe_df = spe_df.append( { 'df_2_cv': spe_kde[0], 'df_3_cv': spe_kde[1], 'df_4_cv': spe_kde[2], 'df_2_tf': spe_kde[3], 'df_3_tf': spe_kde[4], 'df_4_tf': spe_kde[5] }, ignore_index=True) auc_df = auc_df.append( { 'df_2_cv': auc_kde[0], 'df_3_cv': auc_kde[1], 'df_4_cv': auc_kde[2], 'df_2_tf': auc_kde[3], 'df_3_tf': auc_kde[4], 'df_4_tf': auc_kde[5] }, ignore_index=True) mcc_df = mcc_df.append( { 'df_2_cv': m_kde[0], 'df_3_cv': m_kde[1], 'df_4_cv': m_kde[2], 'df_2_tf': m_kde[3], 'df_3_tf': m_kde[4], 'df_4_tf': m_kde[5] }, ignore_index=True)
def evaluate(model, test_x, test_y, output_folder, title, class_specific=False, all_labels=None, weight_vector=None): # Ensure that labels is an np array if all_labels is None: labels = None else: labels = list(all_labels) if weight_vector is None: y_predicted = model.predict(test_x) y_predicted_max = np.argmax(y_predicted, axis=1) else: # Variant Output Shift y_predicted = model.predict(test_x) predicted_shift = list() for e in y_predicted: predicted_shift.append(shift_output(e, weight_vector)) y_predicted_max = np.argmax(predicted_shift, axis=1) y_test_max = np.argmax(test_y, axis=1) # Print classification report report = classification_report(y_test_max, y_predicted_max, labels=labels, output_dict=True, digits=5) report_df = pd.DataFrame(report) report_df.to_csv(os.path.join(output_folder, 'report_' + title + '.csv'), sep=' ', header=True, mode='a') # Print confusion matrix cm = confusion_matrix(y_test_max, y_predicted_max, labels=labels) cm_df = pd.DataFrame(cm) cm_df.to_csv(os.path.join(output_folder, 'cm_' + title + '.csv'), sep=' ', header=True, mode='a') metrics = dict() # Evaluate further metrics # ============================================================================= # Balanced Accuracy Score # ============================================================================= metrics['Balanced Accuracy Score'] = balanced_accuracy_score( y_test_max, y_predicted_max) # ============================================================================= # Cohen Kappa Score # ============================================================================= metrics['Cohen Kappa Score (No weighted)'] = cohen_kappa_score( y_predicted_max, y_test_max, weights=None) metrics['Cohen Kappa Score (Linear weighted)'] = cohen_kappa_score( y_predicted_max, y_test_max, weights='linear') metrics['Cohen Kappa Score (Quadratic weighted)'] = cohen_kappa_score( y_predicted_max, y_test_max, weights='quadratic') # ============================================================================= # Hinge Loss # ============================================================================= metrics['Hinge Loss'] = hinge_loss(y_test_max, y_predicted, labels=labels) # ============================================================================= # Matthews Correlation Coefficient # ============================================================================= metrics['Matthews Correlation Coefficient'] = matthews_corrcoef( y_test_max, y_predicted_max) # ============================================================================= # Top k Accuracy Score (does not work, To DO) # ============================================================================= # print("\n Top k Accuracy: ") # print(top_k_accuracy_score(y_test_max, y_predicted_max, k=5)) # ============================================================================= # The following also work in the multi label case # ============================================================================= # ============================================================================= # Accuracy Score # ============================================================================= metrics['Accuracy Score'] = accuracy_score(y_test_max, y_predicted_max) # ============================================================================= # F1 Score # ============================================================================= metrics['F Score (Micro)'] = f1_score(y_test_max, y_predicted_max, average='micro') metrics['F Score (Macro)'] = f1_score(y_test_max, y_predicted_max, average='macro') metrics['F Score (Weighted)'] = f1_score(y_test_max, y_predicted_max, average='weighted') if class_specific: metrics['F Score (None, i.e. for each class)'] = f1_score( y_test_max, y_predicted_max, average=None) # ============================================================================= # ROC AUC Score (in case of multi class sklearn only support macro and weighted averages) # ============================================================================= # ROC AUC only works if each label occurs at least one time. Hence, we need to catch a exception print(y_test_max) try: metrics['ROC AUC Score (OVR) Macro'] = roc_auc_score(y_test_max, y_predicted, multi_class='ovr', average='macro', labels=labels) metrics['ROC AUC Score (OVR) Weighted'] = roc_auc_score( y_test_max, y_predicted, multi_class='ovr', average='weighted', labels=labels) metrics['ROC AUC Score (OVO) Macro'] = roc_auc_score(y_test_max, y_predicted, multi_class='ovo', average='macro', labels=labels) metrics['ROC AUC Score (OVO) Weighted'] = roc_auc_score( y_test_max, y_predicted, multi_class='ovo', average='weighted', labels=labels) except: print("Cannot calculate ROC AUC Score!") pass # ============================================================================= # F Beta Score # ============================================================================= metrics['F Beta Score (Micro) b=0.5'] = fbeta_score(y_test_max, y_predicted_max, average='micro', beta=0.5) metrics['F Beta Score (Macro) b=0.5'] = fbeta_score(y_test_max, y_predicted_max, average='macro', beta=0.5) metrics['F Beta Score (Weighted) b=0.5'] = fbeta_score(y_test_max, y_predicted_max, average='weighted', beta=0.5) if class_specific: metrics[ 'F Beta Score (None, i.e. for each class) b=0.5'] = fbeta_score( y_test_max, y_predicted_max, average=None, beta=0.5) metrics['F Beta Score (Micro) b=1.5'] = fbeta_score(y_test_max, y_predicted_max, average='micro', beta=1.5) metrics['F Beta Score (Macro) b=1.5'] = fbeta_score(y_test_max, y_predicted_max, average='macro', beta=1.5) metrics['F Beta Score (Weighted) b=1.5'] = fbeta_score(y_test_max, y_predicted_max, average='weighted', beta=1.5) if class_specific: metrics[ 'F Beta Score (None, i.e. for each class) b=1.5'] = fbeta_score( y_test_max, y_predicted_max, average=None, beta=1.5) # ============================================================================= # Hamming Loss # ============================================================================= metrics['Hamming Loss'] = hamming_loss(y_test_max, y_predicted_max) # ============================================================================= # Jaccard Score # ============================================================================= metrics['Jaccard Score (Micro)'] = jaccard_score(y_test_max, y_predicted_max, average='micro') metrics['Jaccard Score (Macro)'] = jaccard_score(y_test_max, y_predicted_max, average='macro') metrics['Jaccard Score (Weighted)'] = jaccard_score(y_test_max, y_predicted_max, average='weighted') if class_specific: metrics['Jaccard Score (None, i.e. for each class)'] = jaccard_score( y_test_max, y_predicted_max, average=None) # ============================================================================= # Log Loss # ============================================================================= metrics['Logg Loss'] = log_loss(y_test_max, y_predicted, labels=labels) # ============================================================================= # Precision Score # ============================================================================= metrics['Precision Score (Micro)'] = precision_score(y_test_max, y_predicted_max, average='micro') metrics['Precision Score (Macro)'] = precision_score(y_test_max, y_predicted_max, average='macro') metrics['Precision Score (Weighted)'] = precision_score(y_test_max, y_predicted_max, average='weighted') if class_specific: metrics[ 'Precision Score (None, i.e. for each class)'] = precision_score( y_test_max, y_predicted_max, average=None) # ============================================================================= # Specificity Score # ============================================================================= metrics['Specificity Score (Micro)'] = specificity_score(y_test_max, y_predicted_max, average='micro') metrics['Specificity Score (Macro)'] = specificity_score(y_test_max, y_predicted_max, average='macro') metrics['Specificity Score (Weighted)'] = specificity_score( y_test_max, y_predicted_max, average='weighted') if class_specific: metrics[ 'Specificity Score (None, i.e. for each class)'] = specificity_score( y_test_max, y_predicted_max, average=None) # ============================================================================= # Recall Score (also named Sensitivity Score). Hence, the Sensitivity Score values # should be the same as the Recall Score values # ============================================================================= metrics['Recall Score (Micro)'] = recall_score(y_test_max, y_predicted_max, average='micro') metrics['Recall Score (Macro)'] = recall_score(y_test_max, y_predicted_max, average='macro') metrics['Recall Score (Weighted)'] = recall_score(y_test_max, y_predicted_max, average='weighted') if class_specific: metrics['Recall Score (None, i.e. for each class)'] = recall_score( y_test_max, y_predicted_max, average=None) metrics['Sensitivity Score (Micro)'] = sensitivity_score(y_test_max, y_predicted_max, average='micro') metrics['Sensitivity Score (Macro)'] = sensitivity_score(y_test_max, y_predicted_max, average='macro') metrics['Sensitivity Score (Weighted)'] = sensitivity_score( y_test_max, y_predicted_max, average='weighted') if class_specific: metrics[ 'Sensitivity Score (None, i.e. for each class)'] = sensitivity_score( y_test_max, y_predicted_max, average=None) # ============================================================================= # Geometric Mean Score # ============================================================================= metrics['Geometric Mean Score (Normal)'] = geometric_mean_score( y_test_max, y_predicted_max) metrics['Geometric Mean Score (Micro)'] = geometric_mean_score( y_test_max, y_predicted_max, average='micro') metrics['Geometric Mean Score (Macro)'] = geometric_mean_score( y_test_max, y_predicted_max, average='macro') metrics['Geometric Mean Score (Weighted)'] = geometric_mean_score( y_test_max, y_predicted_max, average='weighted') if class_specific: metrics[ 'Geometric Mean Score (None, i.e. for each class)'] = geometric_mean_score( y_test_max, y_predicted_max, average=None) # ============================================================================= # Zero one Loss # ============================================================================= metrics['Zero One Loss'] = zero_one_loss(y_test_max, y_predicted_max) # ============================================================================= # Make Index Balanced Accuracy with # ============================================================================= # print("\n MIBA with Matthews") # geo_mean = make_index_balanced_accuracy(alpha=0.5, squared=True)(hamming_loss) # print(geo_mean(y_test_max, y_predicted_max)) return metrics
def test_sensitivity_specificity_f_binary_single_class(y_pred, expected_sensitivity, expected_specificity): # Such a case may occur with non-stratified cross-validation assert sensitivity_score(*y_pred) == expected_sensitivity assert specificity_score(*y_pred) == expected_specificity
def main(file): """ """ N = re.findall('_(.*?)_final', file)[0] try: os.mkdir('./res_0314/{}'.format(N)) except FileExistsError: pass res = pd.DataFrame() res_fit = pd.DataFrame() metrics = pd.DataFrame(index=['SE', 'SP', 'ACC', 'AUC']) metrics_fit = pd.DataFrame(index=['SE', 'SP', 'ACC', 'AUC']) X = pd.read_csv(os.path.join('./docking_origin_data_ywl', file)) ## X = X.dropna(axis=0) X.drop(['mol', 'name'], axis=1, inplace=True) y = X.pop('Label') X = np.array(X) for turn in range(1, 11): proba = pd.DataFrame() proba_fit = pd.DataFrame() print('================ {} Turn: {} ================'.format(N, turn)) for X_train, X_test, y_train, y_test in GetKFold(X, y, Kfold=5): res_, res_fit_ = GetProba(X_train, X_test, y_train, y_test, loop=100) proba = pd.concat([proba, res_]) proba_fit = pd.concat([proba_fit, res_fit_]) proba_fit['Turn'] = turn proba.sort_index(inplace=True) res['Turn_{}'.format(turn)] = proba[0] # print(res) res_fit = pd.concat([res_fit, proba_fit]) # return res, res_fit for proba, turn in zip(res.iteritems(), range(1, 11)): y_pred = (proba[1] >= 0.5) + 0 acc = accuracy_score(y, y_pred) se = sensitivity_score(y, y_pred) sp = specificity_score(y, y_pred) auc = roc_auc_score(y, proba[1]) metrics['Turn_{}'.format(turn)] = [se, sp, acc, auc] y_pred_fit = (res_fit[res_fit.Turn == turn].Proba_fit >= 0.5) + 0 y_true_fit = res_fit[res_fit.Turn == turn].Label.values acc = accuracy_score(y_true_fit, y_pred_fit) se = sensitivity_score(y_true_fit, y_pred_fit) sp = specificity_score(y_true_fit, y_pred_fit) auc = roc_auc_score(y_pred_fit, res_fit[res_fit.Turn == turn].Proba_fit) metrics_fit['Turn_{}'.format(turn)] = [se, sp, acc, auc] metrics['Mean'] = metrics.loc[:, 'Turn_1':'Turn_10'].mean(axis=1) metrics['Std'] = metrics.loc[:, 'Turn_1':'Turn_10'].std(axis=1) metrics_fit['Mean'] = metrics_fit.loc[:, 'Turn_1':'Turn_10'].mean(axis=1) metrics_fit['Std'] = metrics_fit.loc[:, 'Turn_1':'Turn_10'].std(axis=1) res.insert(0, 'Label', y) metrics.to_csv(r'./res_0314/{}/Metrics_{}.csv'.format(N, N)) res.to_csv(r'./res_0314/{}/Proba_{}.csv'.format(N, N), index=False) metrics_fit.to_csv(r'./res_0314/{}/MetricsFit_{}.csv'.format(N, N)) res_fit.to_csv(r'./res_0314/{}/ProbaFit_{}.csv'.format(N, N), index=False)
def authentication(dataset, method, representation, thread_cnt): # load data and setup output file out_file = open( os.path.join( os.path.abspath(""), "results", "authentication_{}_{}_{}.txt".format(dataset, method, representation)), "w") feature_dataset = h5py.File( os.path.join(os.path.abspath(""), "data", method, "{}.hdf5".format(dataset)), "r") rs_dataset = h5py.File( os.path.join(os.path.abspath(""), "data", method, "rs.hdf5"), "r") X, y = feature_dataset["X"][:], feature_dataset["y"][:] y = LabelEncoder().fit_transform(np.array([label.decode() for label in y])) + 1 rs_feature = rs_dataset["X"][0] # remove subjects from lfw dataset with less than 5 images if dataset == "lfw": y_uni, y_cnt = np.unique(y, return_counts=True) mask = np.array([ idx for idx, label in enumerate(y) if label not in y_uni[y_cnt < 5] ]) X, y = X[mask], y[mask] for label_new, label in enumerate(np.unique(y)): y[y == label] = label_new + 1 # fuse features with rs to form biocapsules if representation == "biocapsule": bc_gen = BioCapsuleGenerator() X_train = bc_gen.biocapsule_batch(X, rs_feature) # 5-fold cross-validation experiment skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42) fps, fns, acc, pre, rec, f1s, fpr, frr, eer, aucs = [[] for _ in range(10)] for k, (train_idx, test_idx) in enumerate(skf.split(X, y)): # setup for training print("{} {} {} -- authentication -- fold {}".format( dataset, method, representation, k)) X_train, y_train = X[train_idx], y[train_idx] X_test, y_test = X[test_idx], y[test_idx] # split classes so we can spawn threads to train each binary classifier classes = np.unique(y_train) classes_split = [ classes[i:i + thread_cnt] for i in range(0, classes.shape[0], thread_cnt) ] # train binary classifiers y_prob = np.zeros((X_test.shape[0] * classes.shape[0], )) y_true = np.zeros((X_test.shape[0] * classes.shape[0], )) for li in classes_split: threads = [] queue = Queue() for c in li: threads.append( Thread(target=train_test_clf, args=(c, X_train, y_train, X_test, y_test, queue))) threads[-1].start() _ = [t.join() for t in threads] while not queue.empty(): (c_idx, true, prob) = queue.get() c_idx = int(c_idx - 1) * X_test.shape[0] y_true[c_idx:c_idx + X_test.shape[0]] = true y_prob[c_idx:c_idx + X_test.shape[0]] = prob del X_train, y_train del X_test, y_test y_pred = np.around(y_prob) _, fp, fn, _ = confusion_matrix(y_true, y_pred).ravel() fprf, tprf, thresholds = roc_curve(y_true, y_prob) fps.append(fp) fns.append(fn) acc.append(accuracy_score(y_true, y_pred)) pre.append(precision_score(y_true, y_pred)) rec.append(recall_score(y_true, y_pred)) f1s.append(f1_score(y_true, y_pred)) fpr.append(1. - sensitivity_score(y_true, y_pred)) frr.append(1. - specificity_score(y_true, y_pred)) eer.append(brentq(lambda x: 1. - x - interp1d(fprf, tprf)(x), 0., 1.)) aucs.append(auc(fprf, tprf)) out_file.write("Fold {}:\n".format(k)) out_file.write("FP -- {}, FN -- {}\n".format(fps[-1], fns[-1])) out_file.write("ACC -- {:.6f}\n".format(100. * acc[-1])) out_file.write("PRE -- {:.6f}\n".format(100. * pre[-1])) out_file.write("REC -- {:.6f}\n".format(100. * rec[-1])) out_file.write("F1 -- {:.6f}\n".format(f1s[-1])) out_file.write("FPR -- {:.6f}\n".format(100. * fpr[-1])) out_file.write("FRR -- {:.6f}\n".format(100. * frr[-1])) out_file.write("EER -- {:.6f}\n".format(100. * eer[-1])) out_file.write("AUC -- {:.6f}\n".format(aucs[-1])) out_file.flush() out_file.write("Overall:\n") out_file.write("FP -- {}, FN -- {}\n".format(np.sum(fps), np.sum(fns))) out_file.write("ACC -- {:.6f} (+/-{:.6f})\n".format( 100. * np.average(acc), 100. * np.std(acc))) out_file.write("PRE -- {:.6f} (+/-{:.6f})\n".format( 100. * np.average(pre), 100. * np.std(pre))) out_file.write("REC -- {:.6f} (+/-{:.6f})\n".format( 100. * np.average(rec), 100. * np.std(rec))) out_file.write("F1 -- {:.6f} (+/-{:.6f})\n".format( np.average(f1s), np.std(f1s))) out_file.write("FPR -- {:.6f} (+/-{:.6f})\n".format( 100. * np.average(fpr), 100. * np.std(fpr))) out_file.write("FRR -- {:.6f} (+/-{:.6f})\n".format( 100. * np.average(frr), 100. * np.std(frr))) out_file.write("EER -- {:.6f} (+/-{:.6f})\n".format( 100. * np.average(eer), 100. * np.std(eer))) out_file.write("AUC -- {:.6f} (+/-{:.6f})\n".format( np.average(aucs), np.std(aucs))) out_file.close()
def fit(self, x, y, x_test, y_test): # xPos = [] xNeg = [] xTrain = [] yTrain = [] xlastTrain = [] ylastTrain = [] for i in range(0, len(y)): if y[i] == 1: xPos.append(x[i]) xlastTrain.append(x[i]) ylastTrain.append(y[i]) else: xNeg.append(x[i]) xTrain.append(x[i]) yTrain.append(y[i]) xNLSV = [] iterRecord = 0 for i in range(0, self.T): print(iterRecord) knernel_train = self.get_weight_kernel(np.array(xTrain), np.array(xTrain), self.feature_id, self.gamma_list) # print(knernel_train.shape) iterRecord = iterRecord + 1 svc = SVC(C=self.C, kernel='precomputed', class_weight='balanced', degree=3) svc.fit(knernel_train, yTrain) sv = svc.support_ # This is support vector ne_sv = self.get_NLSV_sv(sv, yTrain) xTrain, yTrain, xNLSV, lastMar = self.rebuild( xTrain, yTrain, sv, xNLSV) # rebuild sample if lastMar < 1 * len(xPos): break for i in xNLSV: xlastTrain.append(i) ylastTrain.append(-1) Kernel_last_train = self.get_weight_kernel(np.array(xlastTrain), np.array(xlastTrain), self.feature_id, self.gamma_list) Kernel_last_test = self.get_weight_kernel(np.array(x_test), np.array(xlastTrain), self.feature_id, self.gamma_list) svc = SVC(C=self.C, kernel='precomputed', class_weight='balanced', degree=3) svc.fit(Kernel_last_train, ylastTrain) y_pre = svc.predict(Kernel_last_test) ACC = accuracy_score(y_test, y_pre) SN = sensitivity_score(y_test, y_pre) SP = specificity_score(y_test, y_pre) MCC = matthews_corrcoef(y_test, y_pre) return SN, SP, ACC, MCC
def test(cfg, model, test_loader, pos_classes=cfg.MODEL.MINORITY_CLASS, device='cpu'): print("Minority Classes: ", pos_classes) model = model.to(device) model.anchor = model.anchor.to(device) model.eval() with torch.no_grad(): """post-training step""" cls_embedding = [] labels = [] # load training dataset x, y = load_dataset(cfg) # randomly sample some samples and matins the class ratio in it classes = np.unique(y) cls_ratio = {} for c in classes: cls_ratio[c] = sum(y == c) / len(y) train_sample_size = cfg.TEST.TRAIN_SAMPLE_SIZE idx = np.arange(len(y)) remain_size = train_sample_size for c in classes: cls_idx = idx[y == c] np.random.shuffle(cls_idx) sampled_idx = cls_idx[:min( max(int(train_sample_size * cls_ratio[c]), 1), remain_size)] remain_size -= len(sampled_idx) # get embedding samples = x[sampled_idx] samples = torch.from_numpy(samples).to(device).float() embedding = model(samples) cls_embedding.append(embedding) labels.append(c) cls_embedding = torch.cat(cls_embedding, dim=0) """test""" # start evaluation on test dataset test_loader = tqdm.tqdm(test_loader) predictions = [] ground_truths = [] scores = [] for i, (x, y) in enumerate(test_loader): x, y = x.to(device).float(), y.to(device) emb = model(x) rel_score = torch.softmax(torch.matmul(emb, cls_embedding.t()), dim=1) predict_cls = labels[torch.argmax(rel_score, dim=1)] predictions.append(predict_cls.item()) ground_truths.append(y.item()) scores.append(rel_score[0, 1]) # compute metrics gmean = geometric_mean_score(ground_truths, predictions, average='binary') f1 = f1_score(ground_truths, predictions, average='macro') auc = roc_auc_score(ground_truths, scores, average='macro') specificity = specificity_score(ground_truths, predictions, average='binary') sensitivity = sensitivity_score(ground_truths, predictions, average='binary') print( "F1: {:.5f} | G-Mean:{:.5f} | AUC: {:.5f} | Spec: {:.5f} | Sens: {:.5f}" .format(f1, gmean, auc, specificity, sensitivity)) return f1, gmean, auc, specificity, sensitivity
tpr, color="blue", lw=2, label='ROC curve (area = {f:.2f})'.format(d=1, f=roc_auc)) plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('ROC curves') plt.legend(loc="lower right") plt.show() plot_confusion_matrix(y_test, y_pred) plot_roc_auc(y_test, y_pred) sensitivity = sensitivity_score(y_test, y_pred) specificity = specificity_score(y_test, y_pred) print("Melanoma Sensitivity:", sensitivity) print("Melanoma Specificity:", specificity) # In[24]: def plot_images(X_test, y_pred, y_test): predicted_class_names = np.array( [class_names[int(round(id))] for id in y_pred]) # some nice plotting plt.figure(figsize=(10, 9)) for n in range(30, 60): plt.subplot(6, 5, n - 30 + 1)