def test_sensitivity_specificity_f_binary_single_class(): # Such a case may occur with non-stratified cross-validation assert sensitivity_score([1, 1], [1, 1]) == 1. assert specificity_score([1, 1], [1, 1]) == 0. assert sensitivity_score([-1, -1], [-1, -1]) == 0. assert specificity_score([-1, -1], [-1, -1]) == 0.
def test_sensitivity_specificity_extra_labels(): y_true = [1, 3, 3, 2] y_pred = [1, 1, 3, 2] # No average: zeros in array actual = specificity_score(y_true, y_pred, labels=[0, 1, 2, 3, 4], average=None) assert_allclose([1., 0.67, 1., 1., 1.], actual, rtol=R_TOL) # Macro average is changed actual = specificity_score(y_true, y_pred, labels=[0, 1, 2, 3, 4], average='macro') assert_allclose(np.mean([1., 0.67, 1., 1., 1.]), actual, rtol=R_TOL) # Check for micro actual = specificity_score(y_true, y_pred, labels=[0, 1, 2, 3, 4], average='micro') assert_allclose(15. / 16., actual, rtol=R_TOL) # Check for weighted actual = specificity_score(y_true, y_pred, labels=[0, 1, 2, 3, 4], average='macro') assert_allclose(np.mean([1., 0.67, 1., 1., 1.]), actual, rtol=R_TOL)
def test_sensitivity_specificity_f_binary_single_class(): # Such a case may occur with non-stratified cross-validation assert_equal(1., sensitivity_score([1, 1], [1, 1])) assert_equal(0., specificity_score([1, 1], [1, 1])) assert_equal(0., sensitivity_score([-1, -1], [-1, -1])) assert_equal(0., specificity_score([-1, -1], [-1, -1]))
def test_sensitivity_specificity_extra_labels(): """Test handling of explicit additional (not in input) labels to SS""" y_true = [1, 3, 3, 2] y_pred = [1, 1, 3, 2] # No average: zeros in array actual = specificity_score(y_true, y_pred, labels=[0, 1, 2, 3, 4], average=None) assert_array_almost_equal([1., 0.67, 1., 1., 1.], actual, 2) # Macro average is changed actual = specificity_score(y_true, y_pred, labels=[0, 1, 2, 3, 4], average='macro') assert_array_almost_equal(np.mean([1., 0.67, 1., 1., 1.]), actual, 2) # Check for micro actual = specificity_score(y_true, y_pred, labels=[0, 1, 2, 3, 4], average='micro') assert_array_almost_equal(15. / 16., actual) # Check for weighted actual = specificity_score(y_true, y_pred, labels=[0, 1, 2, 3, 4], average='macro') assert_array_almost_equal(np.mean([1., 0.67, 1., 1., 1.]), actual, 2)
def specificity(self): """ Specificity is also known as True Negative Rate """ self.overall_specificity = specificity_score( self.y_true, self.y_pred, average=self.average_type).round(self.digits_count_fp) self.classes_specificity = specificity_score(self.y_true, self.y_pred, average=None).round( self.digits_count_fp)
def test_sensitivity_specificity_f_binary_single_class(): """Test sensitivity and specificity behave with a single positive or negative class""" # Such a case may occur with non-stratified cross-validation assert_equal(1., sensitivity_score([1, 1], [1, 1])) assert_equal(0., specificity_score([1, 1], [1, 1])) assert_equal(0., sensitivity_score([-1, -1], [-1, -1])) assert_equal(0., specificity_score([-1, -1], [-1, -1]))
def examine_model(prob_path, pred_path, data_path): prob_data = pd.read_csv(prob_path) pred_data = pd.read_csv(pred_path) label = prob_data['label'].values model_list = prob_data.columns[2:] dic = { 'NCP': 'non_contrast', 'CMP': 'arterial', 'NP': 'venous', 'NCP+CMP': ['non_contrast', 'arterial'], 'NCP+NP': ['non_contrast', 'venous'], 'CMP+NP': ['arterial', 'venous'], 'NCP+CMP+NP': ['non_contrast', 'arterial', 'venous'] } for model_name in model_list: period_list = dic[model_name] if isinstance(period_list, list): compute_ic(united_model_path, data_path, period_list) else: compute_period_ic(united_model_path, data_path, period_list) prob = prob_data[model_name] pred = pred_data[model_name] auc = roc_auc_score(label, prob) acc = accuracy_score(label, pred) sen = recall_score(label, pred) spe = specificity_score(label, pred) print('{}: auc:{:.3f},\tacc: {:.3f},\tsen: {:.3f},\tspe: {:.3f}\n'. format(model_name, auc, acc, sen, spe))
def test_sensitivity_specificity_extra_labels(average, expected_specificty): y_true = [1, 3, 3, 2] y_pred = [1, 1, 3, 2] actual = specificity_score( y_true, y_pred, labels=[0, 1, 2, 3, 4], average=average) assert_allclose(expected_specificty, actual, rtol=R_TOL)
def identification(method, rs_cnt, batch_size): out_file = open( os.path.join(os.path.abspath(""), "results", "{}_{}_vggface2.txt".format(method, rs_cnt)), "w") data_file = os.path.join(os.path.abspath(""), "data", "vggface2_{}_dataset.hdf5".format(method)) train, val, test = [ h5py.File(data_file, "r")[part] for part in ["train", "val", "test"] ] rs = np.load( os.path.join(os.path.abspath(""), "data", "rs_{}_feat.npz".format(method)))["arr_0"] mlp = get_mlp(512, np.unique(train[:, -1].astype(int) - 1).shape[0]) mlp = train_mlp(mlp, train, val, rs, rs_cnt, method, batch_size) y_prob = predict_mlp(mlp, test, rs, rs_cnt) y_pred = np.argmax(y_prob, axis=-1) y_true = test[:, -1].astype(int) - 1 out_file.write("ACC -- {:.6f}\n".format(accuracy_score(y_true, y_pred))) out_file.write("FPR -- {:.6f}\n".format( 1 - sensitivity_score(y_true, y_pred, average="micro"))) out_file.write("FRR -- {:.6f}\n".format( 1 - specificity_score(y_true, y_pred, average="micro"))) out_file.write("PRE -- {:.6f}\n".format( precision_score(y_true, y_pred, average="micro"))) out_file.write("REC -- {:.6f}\n".format( recall_score(y_true, y_pred, average="micro"))) out_file.write("F1 -- {:.6f}\n".format( f1_score(y_true, y_pred, average="micro"))) out_file.close()
def evaluate_prediction(y, pred): # Convert to numpy array pred = np.array(pred, dtype=np.float64) # Count null predictions n_null = np.isnan(pred).sum() #pred.isnull().sum() # Remove null predictions and convert to np.bool y = y[~np.isnan(pred)].astype(np.bool) pred = pred[~np.isnan(pred)].astype(np.bool) if y.all() or (~y).all(): return pd.Series(( 1.0 - n_null / (n_null + len(y)), metrics.accuracy_score(y, pred), np.nan, np.nan, np.nan, np.nan, np.nan, ), index=('coverage', 'accuracy', 'precision', 'specificity', 'sensitivity', 'auc', 'mcc')) return pd.Series(( 1.0 - n_null / (n_null + len(y)), metrics.accuracy_score(y, pred), metrics.precision_score(y, pred), specificity_score(y, pred), metrics.recall_score(y, pred), metrics.roc_auc_score(y, pred), metrics.matthews_corrcoef(y, pred), ), index=('coverage', 'accuracy', 'precision', 'specificity', 'sensitivity', 'auc', 'mcc'))
def print_metrics(model, test, cols): test_x = test.drop(['Label'], axis=1) test_y = test['Label'] y_probs = model.predict_proba(test_x[cols])[:, 1] y_test_predictions = np.where( model.predict_proba(test_x[cols])[:, 1] > 0.119, 2, 1) n_levels = test_y.value_counts().count() if (n_levels == 1): #print(test_x.shape) return y_test_predictions else: mcc = metrics.matthews_corrcoef(test_y, y_test_predictions) tn, fp, fn, tp = confusion_matrix(test_y, y_test_predictions).ravel() ppv = tp / (tp + fp) npv = tn / (tn + fn) sen = tp / (tp + fn) spe = tn / (tn + fp) score = ppv + npv + sen + spe #y_test_predictions=model.predict(test_x[cols]) sensi = sensitivity_score(test_y, y_test_predictions, pos_label=2) speci = specificity_score(test_y, y_test_predictions, pos_label=2) accu = accuracy_score(test_y, y_test_predictions) auro = roc_auc_score(test_y, y_test_predictions) #acc=accuracy_score(test_y,y_test_predictions) print("Composite Score for Martelotto et al.: ", score) return y_test_predictions
def calc_metrics(y_test, pred, auc, i): sen = metrics.sensitivity_score(y_test, pred, pos_label=1) spe = metrics.specificity_score(y_test, pred, pos_label=1) geo = metrics.geometric_mean_score(y_test, pred, pos_label=1) index = ['sm', 'b1', 'b2', 'enn', 'tom', 'ada', 'mnd'] metrics_list = [index[i], sen, spe, geo, auc] return metrics_list
def compute_metrics(y_test, y_pred, y_proba=None, average='weighted', return_index=False): """ Function computing metrics of interest for a sets of prediction :input y_test: pd.DataFrame or np.array of original label :input y_pred: pd.DataFrame or np.array of predicted label :output red: list of value for metrics, in order - Accuracy - Precision - Recall - F1 Score - Sensitivity - Specifity """ if return_index: return [ 'accuracy', 'precision', 'recall', 'f1_score', 'sensitivity_score', 'specificity_score', 'geometric_mean_score', 'average_precision_score' ] else: res = [] res.append(accuracy_score(y_test, y_pred)) res.append(precision_score(y_test, y_pred, average=average)) res.append(recall_score(y_test, y_pred, average=average)) res.append(f1_score(y_test, y_pred, average=average)) res.append(sensitivity_score(y_test, y_pred, average=average)) res.append(specificity_score(y_test, y_pred, average=average)) res.append(geometric_mean_score(y_test, y_pred, average=average)) if y_proba is not None: res.append( average_precision_score(y_test, y_proba, average=average)) return res
def performance_summary( clf: OptimalSamplingClassifier, X: np.ndarray, y: np.ndarray, info: Optional[Dict[str, any]] = None, ) -> Dict[str, float]: predicted_proba = clf.predict_proba(X) predicted = clf.predict(X) nominal_proba = (y == clf.positive_class).mean() return dict(model=str(clf.estimator).replace("\n", "").replace(" ", ""), class_ratio=1 / nominal_proba, weight_ratio=clf.positive_weight / clf.negative_weight, sampling_probability=clf._sampling_proba, previous_probabilities=clf._prev_sampling_probas, cross_val_probabilities=clf._cross_val_sampling_probas, sampling_ratio=clf._sampling_proba / nominal_proba, iter_to_converge=clf._iter_count, accuracy=accuracy_score(y, predicted), sensitivity=sensitivity_score(y, predicted), specificity=specificity_score(y, predicted), precision=precision_score(y, predicted) if (predicted == clf.positive_class).sum() > 0 else None, recall=recall_score(y, predicted) if (predicted == clf.positive_class).sum() > 0 else None, f1_score=f1_score(y, predicted), geometric_mean_score=geometric_mean_score(y, predicted) if (predicted == clf.positive_class).sum() > 0 else None, roc_auc_score=roc_auc_score(y, predicted_proba), average_precision_score=average_precision_score( y, predicted_proba), weighted_loss=clf.weighted_loss(X, y).mean(), cost=clf.cost(X, y).mean(), **(info if info else {}))
def classification_report_imbalanced_values( y_true, y_pred, labels, target_names=None, sample_weight=None, digits=2, alpha=0.1 ): """Copy of imblearn.metrics.classification_report_imbalanced to have access to the raw values. The code is mostly the same except the formatting code and generation of the report which haven removed. Copied from version 0.4.3. The original code is living here: https://github.com/scikit-learn-contrib/imbalanced-learn/blob/b861b3a8e3414c52f40a953f2e0feca5b32e7460/imblearn/metrics/_classification.py#L790 """ labels = np.asarray(labels) if target_names is None: target_names = [str(label) for label in labels] # Compute the different metrics # Precision/recall/f1 precision, recall, f1, support = precision_recall_fscore_support( y_true, y_pred, labels=labels, average=None, sample_weight=sample_weight ) # Specificity specificity = specificity_score( y_true, y_pred, labels=labels, average=None, sample_weight=sample_weight ) # Geometric mean geo_mean = geometric_mean_score( y_true, y_pred, labels=labels, average=None, sample_weight=sample_weight ) # Index balanced accuracy iba_gmean = make_index_balanced_accuracy(alpha=alpha, squared=True)( geometric_mean_score ) iba = iba_gmean( y_true, y_pred, labels=labels, average=None, sample_weight=sample_weight ) result = {"targets": {}} for i, label in enumerate(labels): result["targets"][target_names[i]] = { "precision": precision[i], "recall": recall[i], "specificity": specificity[i], "f1": f1[i], "geo_mean": geo_mean[i], "iba": iba[i], "support": support[i], } result["average"] = { "precision": np.average(precision, weights=support), "recall": np.average(recall, weights=support), "specificity": np.average(specificity, weights=support), "f1": np.average(f1, weights=support), "geo_mean": np.average(geo_mean, weights=support), "iba": np.average(iba, weights=support), "support": np.sum(support), } return result
def GetMetrics(self, y_test): tn, fp, fn, tp = confusion_matrix(y_test, self.y_pred).flatten() self.acc = '%.3f' % accuracy_score(y_test, self.y_pred) self.AUC = '%.3f' % roc_auc_score(y_test, self.y_pred_proba_pos) # self.sens = tp/(tp+fn) self.sens = '%.3f' % sensitivity_score(y_test, self.y_pred) # self.spec = '%.3f'%(tn/(tn+fp)) self.spec = '%.3f' % specificity_score(y_test, self.y_pred)
def classification_results(train,test): #Derivation of NBDriver using training data """ Arguments: train = feature matrix derived from Brown et al. test= feature matrix derived from Martelotto et al. Returns: best_model = Best ensemble model derived using the training data X_red= Dataframe derived after sampling that was used to train the model scores= probability based classification scores """ sen=[];spe=[];acc=[];auc=[];c=[];m=[];s=[] train_x=train.drop('Label',axis=1);train_y=train['Label']; test_x=test.drop('Label',axis=1);test_y=test['Label']; #Random undersampling to reduce the majority class size samp=RepeatedEditedNearestNeighbours(random_state=42) X_samp,y_samp=samp.fit_resample(train_x,train_y) X_samp = pd.DataFrame(X_samp, columns = train_x.columns) #Experimenting with different numbers of top features derived from the tree-based feature extraction method top_n_feats=[30,40,50,60,70] X_r=feature_reduction_using_trees(X_samp,y_samp) cols=X_r.columns for n in top_n_feats: print("For top: ",n," features") X_red=X_r[cols[0:n]] sv=SVC(kernel="linear",probability=True,C=0.01,random_state=42) #chosen from 5foldCV based grid search kde=KDEClassifier(bandwidth=1.27) #chosen from 5foldCV based grid search best_model = VotingClassifier(estimators=[('sv', sv), ('kde', kde)], voting='soft',weights=[4, 7]) #best combination of weights selected by a brute force search (possible weights 1-10) using a cross-validation approach on the training data best_model.fit(X_red,y_samp) y_probs = best_model.predict_proba(test_x[X_red.columns])[:,1] thresholds = arange(0, 1, 0.001) scores = [roc_auc_score(test_y, to_labels(y_probs, t)) for t in thresholds] ix= argmax(scores) y_test_predictions = np.where(best_model.predict_proba(test_x[X_red.columns])[:,1] > thresholds[ix], 2, 1) print("Thresh: ",thresholds[ix]) sensi= sensitivity_score(test_y, y_test_predictions, pos_label=2) speci=specificity_score(test_y,y_test_predictions,pos_label=2) accu=accuracy_score(test_y,y_test_predictions) auro=roc_auc_score(test_y,y_test_predictions) mcc=metrics.matthews_corrcoef(test_y,y_test_predictions) tn, fp, fn, tp = confusion_matrix(test_y, y_test_predictions).ravel() ppv=tp/(tp+fp) npv=tn/(tn+fn) sen=tp/(tp+fn) spe=tn/(tn+fp) score=ppv+npv+sen+spe print("For kmer size: ",len(train.columns[0])) print("for top ",n," features") print(list(X_red.columns.values),"\n") score_dict={"Sen":sen,"Spe":spe,"PPV":ppv,"NPV":npv,"AUC":auro,"MCC":mcc,"ACC":accu} print(score) print(score_dict) df=pd.DataFrame(y_test_predictions) y_samp = pd.DataFrame(y_samp, columns = ['x']) return best_model,X_red,scores
def test_sensitivity_specificity_extra_labels(average, expected_specificty): y_true = [1, 3, 3, 2] y_pred = [1, 1, 3, 2] actual = specificity_score(y_true, y_pred, labels=[0, 1, 2, 3, 4], average=average) assert_allclose(expected_specificty, actual, rtol=R_TOL)
def compute_confusion_matrix(pred_list, label_list, model_name='Unnamed'): acc = metrics.accuracy_score(label_list, pred_list) sen = metrics.recall_score(label_list, pred_list) spe = specificity_score(label_list, pred_list) ppv = metrics.precision_score(label_list, pred_list) youden_index = sen + spe - 1 print( '{}: \tacc: {:.3f},\tsen: {:.3f},\tspe: {:.3f}\n,\tppv:{:.3f},\tyouden index:{:.3f}' .format(model_name, acc, sen, spe, ppv, youden_index))
def compute_metrics(gt, pred, competition=True): """ Computes accuracy, precision, recall and F1-score from prediction scores. Args: gt: Pytorch tensor on GPU, shape = [n_samples, n_classes] true binary labels. pred: Pytorch tensor on GPU, shape = [n_samples, n_classes] can either be probability estimates of the positive class, confidence values, or binary decisions. competition: whether to use competition tasks. If False, use all tasks Returns: List of AUROCs of all classes. """ AUROCs, Accus, Senss, Recas, Specs = [], [], [], [], [] gt_np = gt.cpu().detach().numpy() # if cfg.uncertainty == 'U-Zeros': # gt_np[np.where(gt_np==-1)] = 0 # if cfg.uncertainty == 'U-Ones': # gt_np[np.where(gt_np==-1)] = 1 pred_np = pred.cpu().detach().numpy() THRESH = 0.18 # indexes = TARGET_INDEXES if competition else range(N_CLASSES) #indexes = range(n_classes) # pdb.set_trace() indexes = range(len(CLASS_NAMES)) for i, cls in enumerate(indexes): try: AUROCs.append(roc_auc_score(gt_np[:, i], pred_np[:, i])) except ValueError as error: print('Error in computing accuracy for {}.\n Error msg:{}'.format(i, error)) AUROCs.append(0) try: Accus.append(accuracy_score(gt_np[:, i], (pred_np[:, i]>=THRESH))) except ValueError as error: print('Error in computing accuracy for {}.\n Error msg:{}'.format(i, error)) Accus.append(0) try: Senss.append(sensitivity_score(gt_np[:, i], (pred_np[:, i]>=THRESH))) except ValueError: print('Error in computing precision for {}.'.format(i)) Senss.append(0) try: Specs.append(specificity_score(gt_np[:, i], (pred_np[:, i]>=THRESH))) except ValueError: print('Error in computing F1-score for {}.'.format(i)) Specs.append(0) return AUROCs, Accus, Senss, Specs
def calc_metrics(y_test, pred, i): sen = metrics.sensitivity_score(y_test, pred, average='micro') spe = metrics.specificity_score(y_test, pred, average='micro') geo = metrics.geometric_mean_score(y_test, pred, average='micro') f1 = f1_score(y_test, pred, average='micro') mcc = matthews_corrcoef(y_test, pred) index = ['sm', 'b1', 'b2', 'enn', 'tom', 'ada', 'mnd'] metrics_list = [index[i], sen, spe, geo, f1, mcc] return metrics_list
def calc_metrics(y_test, pred, auc, i): sen = metrics.sensitivity_score(y_test, pred, pos_label=1) spe = metrics.specificity_score(y_test, pred, pos_label=1) geo = metrics.geometric_mean_score(y_test, pred, pos_label=1) f1 = f1_score(y_test, pred, pos_label=1) mcc = matthews_corrcoef(y_test, pred) index = ['original', 'sm', 'b1', 'b2', 'enn', 'tom', 'ada', 'mnd'] metrics_list = [index[i], sen, spe, geo, f1, mcc, auc] return metrics_list
def measure(self, j_gamma, k_c, i_folder, Label_test, Label_predict): j = j_gamma k = k_c i = i_folder self.Accuracy[j, k, i] = metrics.accuracy_score(Label_test, Label_predict) self.Precision[j, k, i] = metrics.precision_score(Label_test, Label_predict) self.Recall[j, k, i] = metrics.recall_score(Label_test, Label_predict) self.Specificity[j, k, i] = specificity_score(Label_test, Label_predict) self.G_Mean[j, k, i] = geometric_mean_score(Label_test, Label_predict) self.F_Mean[j, k, i] = metrics.f1_score(Label_test, Label_predict)
def test_sensitivity_specificity_extra_labels(): y_true = [1, 3, 3, 2] y_pred = [1, 1, 3, 2] # No average: zeros in array actual = specificity_score( y_true, y_pred, labels=[0, 1, 2, 3, 4], average=None) assert_allclose([1., 0.67, 1., 1., 1.], actual, rtol=R_TOL) # Macro average is changed actual = specificity_score( y_true, y_pred, labels=[0, 1, 2, 3, 4], average='macro') assert_allclose(np.mean([1., 0.67, 1., 1., 1.]), actual, rtol=R_TOL) # Check for micro actual = specificity_score( y_true, y_pred, labels=[0, 1, 2, 3, 4], average='micro') assert_allclose(15. / 16., actual, rtol=R_TOL) # Check for weighted actual = specificity_score( y_true, y_pred, labels=[0, 1, 2, 3, 4], average='macro') assert_allclose(np.mean([1., 0.67, 1., 1., 1.]), actual, rtol=R_TOL)
def metrique_classe(y_pred, y_true, xclass): from imblearn.metrics import specificity_score from imblearn.metrics import sensitivity_score from imblearn.metrics import geometric_mean_score # La sensibilité est le rapport où est le nombre de vrais positifs et le nombre de faux négatifs. # La sensibilité quantifie la capacité à éviter les faux négatifs.tp # estimator issu de quelques FIT log.traceLogInfo("Classe ", xclass) if xclass == 0: log.traceLogInfo("Classe 0") if xclass == 1: log.traceLogInfo("Classe 1") log.traceLogInfo("Sensibilité du re-equilibrage des données sur le TEST") #log.traceLogInfo("Binary ",sensitivity_score(y_true, y_pred, average='binary', pos_label=xclass)) log.traceLogInfo( "La spécificité est intuitivement la capacité du classificateur à trouver tous les échantillons positifs" ) log.traceLogInfo("Binary ") log.traceLogInfo( specificity_score(y_true, y_pred, labels=None, pos_label=xclass, average='binary', sample_weight=None)) print("\nCalculer la moyenne géométrique") print(geometric_mean_score(y_true, y_pred, labels=None, pos_label=xclass)) print("\n Calculer sensitivity score") print( "La sensibilité est le rapport où est le nombre de vrais positifs et le nombre de faux négatifs." ) print("La sensibilité quantifie la capacité à éviter les faux négatifs.") print( sensitivity_score(y_true, y_pred, labels=None, pos_label=xclass, average='binary'))
def fit(self, x, y, x_test, y_test): # xPos = [] xNeg = [] xTrain = [] yTrain = [] xlastTrain = [] ylastTrain = [] for i in range(0, len(y)): if y[i] == 1: xPos.append(x[i]) xlastTrain.append(x[i]) ylastTrain.append(y[i]) else: xNeg.append(x[i]) xTrain.append(x[i]) yTrain.append(y[i]) xNLSV = [] iterRecord = 0 for i in range(0, self.T): svc = SVC(C=self.C, class_weight=self.class_weight, degree=self.degree, gamma=self.gamma, kernel='linear') print(iterRecord) iterRecord += 1 svc.fit(xTrain, yTrain) sv = svc.support_ # This is support vector xTrain, yTrain, xNLSV, lastMar = self.rebuild( xTrain, yTrain, sv, xNLSV) # rebuild sample #print (lastMar) if lastMar < 0.1 * len(xPos): break for i in xNLSV: xlastTrain.append(i) ylastTrain.append(-1) self.allSVC.fit(xlastTrain, ylastTrain) y_pre = self.allSVC.predict(x_test) ACC = accuracy_score(y_test, y_pre) SN = sensitivity_score(y_test, y_pre) SP = specificity_score(y_test, y_pre) MCC = matthews_corrcoef(y_test, y_pre) return SN, SP, ACC, MCC
def specificity(output, target): """ Specificity = TN / (TN + FP) :param output: Batch x Channel x .... :param target: Batch x .... :return: """ with torch.no_grad(): if len(output.shape) == (len(target.shape) + 1): # reverse one-hot encode output output = torch.argmax(output, 1) assert output.shape == target.shape, "The output size should be the same or one dimension more than the shape of target." output = output.cpu().detach().numpy() target = target.cpu().detach().numpy() score = specificity_score(target, output, average='micro') return score
def eval_(self, data, label): self.ROI = self.ROI.eval() gt = [] pre = [] total = int(len(label) / 240) with torch.no_grad(): for i in range(total): a = i * 240 b = a + 240 sin_x = torch.Tensor(data[a:b]).cuda() sin_x = sin_x.view(240, 1024, 15) sin_y = label[a:b] predict = self.ROI(sin_x) predict, index = torch.max(predict, 1) pre.extend(index.cpu().tolist()) gt.extend(sin_y) print("ppv:{}".format(metrics.precision_score(gt, pre, average='micro'))) print("spe:{}".format(specificity_score(gt, pre, average='micro'))) print("sen:{}".format(metrics.recall_score(gt, pre, average='micro')))
def measure(self, i_folder, Label_test, Label_predict, average): i = i_folder self.Accuracy[i] = metrics.accuracy_score(Label_test, Label_predict) # print(self.Accuracy) self.Precision[i] = metrics.precision_score(Label_test, Label_predict, average=average) # print(self.Precision) self.Recall[i] = metrics.recall_score(Label_test, Label_predict, average=average) # print(self.Recall) self.Specificity[i] = specificity_score(Label_test, Label_predict, average=average) # print(self.Specificity) self.G_Mean[i] = geometric_mean_score(Label_test, Label_predict, average=average) # print(self.G_Mean) self.F_Mean[i] = metrics.f1_score(Label_test, Label_predict, average=average)
print(mean_SP) print(mean_ACC) print(mean_MCC) print(mean_AUC) clf.fit(gram_train, y_train) y_score = clf.predict_proba(gram_test) y_score = get_y_score(y_score) precision, recall, thresholds = metrics.precision_recall_curve(y_test, y_score) y_pred = clf.predict(gram_test) ACC = metrics.accuracy_score(y_test, y_pred) precision = metrics.precision_score(y_test, y_pred) sensitivity = metrics.recall_score(y_test, y_pred) specificity = specificity_score(y_test, y_pred) AUC = metrics.roc_auc_score(y_test, y_score) MCC = metrics.matthews_corrcoef(y_test, y_pred) AUPR = get_AUPR(y_test, y_score) #print("===========================") #print('testing:') print(sensitivity) print(specificity) print(ACC) print(MCC) print(AUC) #print('AUPR =', AUPR)
def eval_specificity(gt, pred, average='macro'): if type(gt) is pd.core.frame.DataFrame or type(gt) is pd.core.frame.Series: return specificity_score(gt.fillna(0.0), pred.fillna(0.0), average=average) else: return specificity_score(gt, pred, average=average)
best = fmin( fn=svm_weight_ACC, space=space, algo=tpe.suggest, max_evals=100, trials=trials, ) C = best['C'] g = best['gamma'] prob = svm_problem(membership, y_svm, X_svm) param = svm_parameter('-t 2 -c ' + str(C) + ' -g ' + str(g) + ' -b 1') m = svm_train(prob, param) p_label, p_acc, p_val = svm_predict(y_test_svm, X_test_svm, m, '-b 1') y_prob = np.reshape([p_val[i][0] for i in range(np.shape(p_val)[0])], (np.shape(p_val)[0], 1)) ACC = metrics.accuracy_score(y_test, p_label) precision = metrics.precision_score(y_test, p_label) sensitivity = metrics.recall_score(y_test, p_label) specificity = specificity_score(y_test, p_label) AUC = metrics.roc_auc_score(y_test, y_prob) MCC = metrics.matthews_corrcoef(y_test, p_label) print('C =', C) print('g =', g) print('SN =', sensitivity) print('SP =', specificity) print('ACC =', p_acc[0]) print('MCC =', MCC) print('AUC =', AUC)
def test_sensitivity_specificity_f_binary_single_class(y_pred, expected_sensitivity, expected_specificity): # Such a case may occur with non-stratified cross-validation assert sensitivity_score(*y_pred) == expected_sensitivity assert specificity_score(*y_pred) == expected_specificity
def test_sensitivity_specificity_f_binary_single_class( y_pred, expected_sensitivity, expected_specificity): # Such a case may occur with non-stratified cross-validation assert sensitivity_score(*y_pred) == expected_sensitivity assert specificity_score(*y_pred) == expected_specificity