def test_sensitivity_specificity_f_binary_single_class():
# Such a case may occur with non-stratified cross-validation
assert_equal(1., sensitivity_score([1, 1], [1, 1]))
assert_equal(0., specificity_score([1, 1], [1, 1]))
assert_equal(0., sensitivity_score([-1, -1], [-1, -1]))
assert_equal(0., specificity_score([-1, -1], [-1, -1]))
def test_sensitivity_specificity_f_binary_single_class():
# Such a case may occur with non-stratified cross-validation
assert sensitivity_score([1, 1], [1, 1]) == 1.
assert specificity_score([1, 1], [1, 1]) == 0.
assert sensitivity_score([-1, -1], [-1, -1]) == 0.
assert specificity_score([-1, -1], [-1, -1]) == 0.
def test_sensitivity_specificity_error_multilabels():
y_true = [1, 3, 3, 2]
y_pred = [1, 1, 3, 2]
y_true_bin = label_binarize(y_true, classes=np.arange(5))
y_pred_bin = label_binarize(y_pred, classes=np.arange(5))
with pytest.raises(ValueError):
sensitivity_score(y_true_bin, y_pred_bin)
def test_sensitivity_specificity_error_multilabels():
y_true = [1, 3, 3, 2]
y_pred = [1, 1, 3, 2]
y_true_bin = label_binarize(y_true, classes=np.arange(5))
y_pred_bin = label_binarize(y_pred, classes=np.arange(5))
with pytest.raises(ValueError):
sensitivity_score(y_true_bin, y_pred_bin)
def test_sensitivity_specificity_f_binary_single_class():
"""Test sensitivity and specificity behave with a single positive or
negative class"""
# Such a case may occur with non-stratified cross-validation
assert_equal(1., sensitivity_score([1, 1], [1, 1]))
assert_equal(0., specificity_score([1, 1], [1, 1]))
assert_equal(0., sensitivity_score([-1, -1], [-1, -1]))
assert_equal(0., specificity_score([-1, -1], [-1, -1]))
def performance_summary(
clf: OptimalSamplingClassifier,
X: np.ndarray,
y: np.ndarray,
info: Optional[Dict[str, any]] = None,
) -> Dict[str, float]:
predicted_proba = clf.predict_proba(X)
predicted = clf.predict(X)
nominal_proba = (y == clf.positive_class).mean()
return dict(model=str(clf.estimator).replace("\n", "").replace(" ", ""),
class_ratio=1 / nominal_proba,
weight_ratio=clf.positive_weight / clf.negative_weight,
sampling_probability=clf._sampling_proba,
previous_probabilities=clf._prev_sampling_probas,
cross_val_probabilities=clf._cross_val_sampling_probas,
sampling_ratio=clf._sampling_proba / nominal_proba,
iter_to_converge=clf._iter_count,
accuracy=accuracy_score(y, predicted),
sensitivity=sensitivity_score(y, predicted),
specificity=specificity_score(y, predicted),
precision=precision_score(y, predicted) if
(predicted == clf.positive_class).sum() > 0 else None,
recall=recall_score(y, predicted) if
(predicted == clf.positive_class).sum() > 0 else None,
f1_score=f1_score(y, predicted),
geometric_mean_score=geometric_mean_score(y, predicted) if
(predicted == clf.positive_class).sum() > 0 else None,
roc_auc_score=roc_auc_score(y, predicted_proba),
average_precision_score=average_precision_score(
y, predicted_proba),
weighted_loss=clf.weighted_loss(X, y).mean(),
cost=clf.cost(X, y).mean(),
**(info if info else {}))
def identification(method, rs_cnt, batch_size):
out_file = open(
os.path.join(os.path.abspath(""), "results",
"{}_{}_vggface2.txt".format(method, rs_cnt)), "w")
data_file = os.path.join(os.path.abspath(""), "data",
"vggface2_{}_dataset.hdf5".format(method))
train, val, test = [
h5py.File(data_file, "r")[part] for part in ["train", "val", "test"]
]
rs = np.load(
os.path.join(os.path.abspath(""), "data",
"rs_{}_feat.npz".format(method)))["arr_0"]
mlp = get_mlp(512, np.unique(train[:, -1].astype(int) - 1).shape[0])
mlp = train_mlp(mlp, train, val, rs, rs_cnt, method, batch_size)
y_prob = predict_mlp(mlp, test, rs, rs_cnt)
y_pred = np.argmax(y_prob, axis=-1)
y_true = test[:, -1].astype(int) - 1
out_file.write("ACC -- {:.6f}\n".format(accuracy_score(y_true, y_pred)))
out_file.write("FPR -- {:.6f}\n".format(
1 - sensitivity_score(y_true, y_pred, average="micro")))
out_file.write("FRR -- {:.6f}\n".format(
1 - specificity_score(y_true, y_pred, average="micro")))
out_file.write("PRE -- {:.6f}\n".format(
precision_score(y_true, y_pred, average="micro")))
out_file.write("REC -- {:.6f}\n".format(
recall_score(y_true, y_pred, average="micro")))
out_file.write("F1 -- {:.6f}\n".format(
f1_score(y_true, y_pred, average="micro")))
out_file.close()
def compute_metrics(y_test,
y_pred,
y_proba=None,
average='weighted',
return_index=False):
"""
Function computing metrics of interest for a sets of prediction
:input y_test: pd.DataFrame or np.array of original label
:input y_pred: pd.DataFrame or np.array of predicted label
:output red: list of value for metrics, in order - Accuracy - Precision - Recall - F1 Score - Sensitivity - Specifity
"""
if return_index:
return [
'accuracy', 'precision', 'recall', 'f1_score', 'sensitivity_score',
'specificity_score', 'geometric_mean_score',
'average_precision_score'
]
else:
res = []
res.append(accuracy_score(y_test, y_pred))
res.append(precision_score(y_test, y_pred, average=average))
res.append(recall_score(y_test, y_pred, average=average))
res.append(f1_score(y_test, y_pred, average=average))
res.append(sensitivity_score(y_test, y_pred, average=average))
res.append(specificity_score(y_test, y_pred, average=average))
res.append(geometric_mean_score(y_test, y_pred, average=average))
if y_proba is not None:
res.append(
average_precision_score(y_test, y_proba, average=average))
return res
def print_metrics(model, test, cols):
test_x = test.drop(['Label'], axis=1)
test_y = test['Label']
y_probs = model.predict_proba(test_x[cols])[:, 1]
y_test_predictions = np.where(
model.predict_proba(test_x[cols])[:, 1] > 0.119, 2, 1)
n_levels = test_y.value_counts().count()
if (n_levels == 1):
#print(test_x.shape)
return y_test_predictions
else:
mcc = metrics.matthews_corrcoef(test_y, y_test_predictions)
tn, fp, fn, tp = confusion_matrix(test_y, y_test_predictions).ravel()
ppv = tp / (tp + fp)
npv = tn / (tn + fn)
sen = tp / (tp + fn)
spe = tn / (tn + fp)
score = ppv + npv + sen + spe
#y_test_predictions=model.predict(test_x[cols])
sensi = sensitivity_score(test_y, y_test_predictions, pos_label=2)
speci = specificity_score(test_y, y_test_predictions, pos_label=2)
accu = accuracy_score(test_y, y_test_predictions)
auro = roc_auc_score(test_y, y_test_predictions)
#acc=accuracy_score(test_y,y_test_predictions)
print("Composite Score for Martelotto et al.: ", score)
return y_test_predictions
def calc_metrics(y_test, pred, auc, i):
sen = metrics.sensitivity_score(y_test, pred, pos_label=1)
spe = metrics.specificity_score(y_test, pred, pos_label=1)
geo = metrics.geometric_mean_score(y_test, pred, pos_label=1)
index = ['sm', 'b1', 'b2', 'enn', 'tom', 'ada', 'mnd']
metrics_list = [index[i], sen, spe, geo, auc]
return metrics_list
def GetMetrics(self, y_test):
tn, fp, fn, tp = confusion_matrix(y_test, self.y_pred).flatten()
self.acc = '%.3f' % accuracy_score(y_test, self.y_pred)
self.AUC = '%.3f' % roc_auc_score(y_test, self.y_pred_proba_pos)
# self.sens = tp/(tp+fn)
self.sens = '%.3f' % sensitivity_score(y_test, self.y_pred)
# self.spec = '%.3f'%(tn/(tn+fp))
self.spec = '%.3f' % specificity_score(y_test, self.y_pred)
def classification_results(train,test):
#Derivation of NBDriver using training data
"""
Arguments:
train = feature matrix derived from Brown et al.
test= feature matrix derived from Martelotto et al.
Returns:
best_model = Best ensemble model derived using the training data
X_red= Dataframe derived after sampling that was used to train the model
scores= probability based classification scores
"""
sen=[];spe=[];acc=[];auc=[];c=[];m=[];s=[]
train_x=train.drop('Label',axis=1);train_y=train['Label'];
test_x=test.drop('Label',axis=1);test_y=test['Label'];
#Random undersampling to reduce the majority class size
samp=RepeatedEditedNearestNeighbours(random_state=42)
X_samp,y_samp=samp.fit_resample(train_x,train_y)
X_samp = pd.DataFrame(X_samp, columns = train_x.columns)
#Experimenting with different numbers of top features derived from the tree-based feature extraction method
top_n_feats=[30,40,50,60,70]
X_r=feature_reduction_using_trees(X_samp,y_samp)
cols=X_r.columns
for n in top_n_feats:
print("For top: ",n," features")
X_red=X_r[cols[0:n]]
sv=SVC(kernel="linear",probability=True,C=0.01,random_state=42) #chosen from 5foldCV based grid search
kde=KDEClassifier(bandwidth=1.27) #chosen from 5foldCV based grid search
best_model = VotingClassifier(estimators=[('sv', sv), ('kde', kde)],
voting='soft',weights=[4, 7]) #best combination of weights selected by a brute force search (possible weights 1-10) using a cross-validation approach on the training data
best_model.fit(X_red,y_samp)
y_probs = best_model.predict_proba(test_x[X_red.columns])[:,1]
thresholds = arange(0, 1, 0.001)
scores = [roc_auc_score(test_y, to_labels(y_probs, t)) for t in thresholds]
ix= argmax(scores)
y_test_predictions = np.where(best_model.predict_proba(test_x[X_red.columns])[:,1] > thresholds[ix], 2, 1)
print("Thresh: ",thresholds[ix])
sensi= sensitivity_score(test_y, y_test_predictions, pos_label=2)
speci=specificity_score(test_y,y_test_predictions,pos_label=2)
accu=accuracy_score(test_y,y_test_predictions)
auro=roc_auc_score(test_y,y_test_predictions)
mcc=metrics.matthews_corrcoef(test_y,y_test_predictions)
tn, fp, fn, tp = confusion_matrix(test_y, y_test_predictions).ravel()
ppv=tp/(tp+fp)
npv=tn/(tn+fn)
sen=tp/(tp+fn)
spe=tn/(tn+fp)
score=ppv+npv+sen+spe
print("For kmer size: ",len(train.columns[0]))
print("for top ",n," features")
print(list(X_red.columns.values),"\n")
score_dict={"Sen":sen,"Spe":spe,"PPV":ppv,"NPV":npv,"AUC":auro,"MCC":mcc,"ACC":accu}
print(score)
print(score_dict)
df=pd.DataFrame(y_test_predictions)
y_samp = pd.DataFrame(y_samp, columns = ['x'])
return best_model,X_red,scores
def compute_metrics(gt, pred, competition=True):
"""
Computes accuracy, precision, recall and F1-score from prediction scores.
Args:
gt: Pytorch tensor on GPU, shape = [n_samples, n_classes]
true binary labels.
pred: Pytorch tensor on GPU, shape = [n_samples, n_classes]
can either be probability estimates of the positive class,
confidence values, or binary decisions.
competition: whether to use competition tasks. If False,
use all tasks
Returns:
List of AUROCs of all classes.
"""
AUROCs, Accus, Senss, Recas, Specs = [], [], [], [], []
gt_np = gt.cpu().detach().numpy()
# if cfg.uncertainty == 'U-Zeros':
# gt_np[np.where(gt_np==-1)] = 0
# if cfg.uncertainty == 'U-Ones':
# gt_np[np.where(gt_np==-1)] = 1
pred_np = pred.cpu().detach().numpy()
THRESH = 0.18
# indexes = TARGET_INDEXES if competition else range(N_CLASSES)
#indexes = range(n_classes)
# pdb.set_trace()
indexes = range(len(CLASS_NAMES))
for i, cls in enumerate(indexes):
try:
AUROCs.append(roc_auc_score(gt_np[:, i], pred_np[:, i]))
except ValueError as error:
print('Error in computing accuracy for {}.\n Error msg:{}'.format(i, error))
AUROCs.append(0)
try:
Accus.append(accuracy_score(gt_np[:, i], (pred_np[:, i]>=THRESH)))
except ValueError as error:
print('Error in computing accuracy for {}.\n Error msg:{}'.format(i, error))
Accus.append(0)
try:
Senss.append(sensitivity_score(gt_np[:, i], (pred_np[:, i]>=THRESH)))
except ValueError:
print('Error in computing precision for {}.'.format(i))
Senss.append(0)
try:
Specs.append(specificity_score(gt_np[:, i], (pred_np[:, i]>=THRESH)))
except ValueError:
print('Error in computing F1-score for {}.'.format(i))
Specs.append(0)
return AUROCs, Accus, Senss, Specs
def calc_metrics(y_test, pred, i):
sen = metrics.sensitivity_score(y_test, pred, average='micro')
spe = metrics.specificity_score(y_test, pred, average='micro')
geo = metrics.geometric_mean_score(y_test, pred, average='micro')
f1 = f1_score(y_test, pred, average='micro')
mcc = matthews_corrcoef(y_test, pred)
index = ['sm', 'b1', 'b2', 'enn', 'tom', 'ada', 'mnd']
metrics_list = [index[i], sen, spe, geo, f1, mcc]
return metrics_list
def calc_metrics(y_test, pred, auc, i):
sen = metrics.sensitivity_score(y_test, pred, pos_label=1)
spe = metrics.specificity_score(y_test, pred, pos_label=1)
geo = metrics.geometric_mean_score(y_test, pred, pos_label=1)
f1 = f1_score(y_test, pred, pos_label=1)
mcc = matthews_corrcoef(y_test, pred)
index = ['original', 'sm', 'b1', 'b2', 'enn', 'tom', 'ada', 'mnd']
metrics_list = [index[i], sen, spe, geo, f1, mcc, auc]
return metrics_list
def train_and_predict(classifier, X_train, y_train, X_test, y_test):
start = timer()
classifier.fit(X_train, y_train)
end = timer()
predictions = classifier.predict(X_test)
results = OrderedDict()
results['recall'] = metrics.sensitivity_score(y_test, predictions, pos_label=1, average='binary')
results['gmean'] = metrics.geometric_mean_score(y_test, predictions, pos_label=1, average='binary')
results['fmeasure'] = f1_score(y_test, predictions, average=None)[1]
results['auc'] = roc_auc_score(y_test, predictions, average=None)
results['time'] = end-start
return results
def metrique_classe(y_pred, y_true, xclass):
from imblearn.metrics import specificity_score
from imblearn.metrics import sensitivity_score
from imblearn.metrics import geometric_mean_score
# La sensibilité est le rapport où est le nombre de vrais positifs et le nombre de faux négatifs.
# La sensibilité quantifie la capacité à éviter les faux négatifs.tp
# estimator issu de quelques FIT
log.traceLogInfo("Classe ", xclass)
if xclass == 0:
log.traceLogInfo("Classe 0")
if xclass == 1:
log.traceLogInfo("Classe 1")
log.traceLogInfo("Sensibilité du re-equilibrage des données sur le TEST")
#log.traceLogInfo("Binary ",sensitivity_score(y_true, y_pred, average='binary', pos_label=xclass))
log.traceLogInfo(
"La spécificité est intuitivement la capacité du classificateur à trouver tous les échantillons positifs"
)
log.traceLogInfo("Binary ")
log.traceLogInfo(
specificity_score(y_true,
y_pred,
labels=None,
pos_label=xclass,
average='binary',
sample_weight=None))
print("\nCalculer la moyenne géométrique")
print(geometric_mean_score(y_true, y_pred, labels=None, pos_label=xclass))
print("\n Calculer sensitivity score")
print(
"La sensibilité est le rapport où est le nombre de vrais positifs et le nombre de faux négatifs."
)
print("La sensibilité quantifie la capacité à éviter les faux négatifs.")
print(
sensitivity_score(y_true,
y_pred,
labels=None,
pos_label=xclass,
average='binary'))
def fit(self, x, y, x_test, y_test):
#
xPos = []
xNeg = []
xTrain = []
yTrain = []
xlastTrain = []
ylastTrain = []
for i in range(0, len(y)):
if y[i] == 1:
xPos.append(x[i])
xlastTrain.append(x[i])
ylastTrain.append(y[i])
else:
xNeg.append(x[i])
xTrain.append(x[i])
yTrain.append(y[i])
xNLSV = []
iterRecord = 0
for i in range(0, self.T):
svc = SVC(C=self.C,
class_weight=self.class_weight,
degree=self.degree,
gamma=self.gamma,
kernel='linear')
print(iterRecord)
iterRecord += 1
svc.fit(xTrain, yTrain)
sv = svc.support_ # This is support vector
xTrain, yTrain, xNLSV, lastMar = self.rebuild(
xTrain, yTrain, sv, xNLSV) # rebuild sample
#print (lastMar)
if lastMar < 0.1 * len(xPos):
break
for i in xNLSV:
xlastTrain.append(i)
ylastTrain.append(-1)
self.allSVC.fit(xlastTrain, ylastTrain)
y_pre = self.allSVC.predict(x_test)
ACC = accuracy_score(y_test, y_pre)
SN = sensitivity_score(y_test, y_pre)
SP = specificity_score(y_test, y_pre)
MCC = matthews_corrcoef(y_test, y_pre)
return SN, SP, ACC, MCC
def sensitivity_weighted(output, target):
"""
Sensitivity = TP / (TP + FN)
:param output: Batch x Channel x ....
:param target: Batch x ....
:return:
"""
with torch.no_grad():
if len(output.shape) == (len(target.shape) + 1):
# reverse one-hot encode output
output = torch.argmax(output, 1)
assert output.shape == target.shape, "The output size should be the same or one dimension more than the shape of target."
output = output.cpu().detach().numpy()
target = target.cpu().detach().numpy()
score = sensitivity_score(target, output, average='weighted')
return score
def get_sensitivity_tc(self, y_true, y_pred):
return imb_metrics.sensitivity_score(y_true, y_pred,
average='binary') * 100
print('--- Extratrees model ---')
extra = ExtraTreesClassifier(random_state=20)
extra.fit(X_train, y_train)
y_score = extra.predict(X_test)
acc = accuracy_score(y_test, y_score)
acc_list.append(acc)
precision = precision_score(y_test, y_score, average='macro')
precision_list.append(precision)
recall = recall_score(y_test, y_score, average='macro')
recall_list.append(recall)
sensitivity = sensitivity_score(y_test, y_score, average='macro')
sensitivity_list.append(sensitivity)
specificity = specificity_score(y_test, y_score, average='macro')
specificity_list.append(specificity)
f1 = f1_score(y_test, y_score, average='macro')
f1_list.append(f1)
gmean = geometric_mean_score(y_test, y_score, average='macro')
g_list.append(gmean)
mat = confusion_matrix(y_test, y_score)
# tn,fp,fn,tp=confusion_matrix(Y_test, Y_test_predict).ravel()
# print(tn,fp,fn,tp)
def model_fitting(model_name,
train_x,
val_x,
train_y,
val_y,
EPOCHS=20,
BATCH_SIZE=32):
global model_history, best_model_ind, val_conf_mat_tp, cross_val_prec_list, cross_val_rcl_list, scale_positive_weight
global cross_val_f1_list, cross_val_rocauc_list, cross_val_sens_list, cross_val_spec_list, cross_val_gm_list
global bal_tech
if model_name.lower() == "lr":
logging.info("Fitting a Logistic Regression Model...")
if bal_tech == "weighted":
scikit_log_reg = LogisticRegression(verbose=1,
solver='liblinear',
class_weight='balanced',
max_iter=1000)
else:
scikit_log_reg = LogisticRegression(verbose=1,
solver='liblinear',
max_iter=1000)
event_time = time.time()
model = scikit_log_reg.fit(train_x, train_y)
logging.info("Training time for LR model: %s",
(time.time() - event_time))
elif model_name.lower() == 'gb':
logging.info("Fitting a Gradient Boost Model...")
if bal_tech == "weighted":
scikit_gradboost = XGBClassifier(
scale_pos_weight=scale_positive_weight)
else:
scikit_gradboost = XGBClassifier()
event_time = time.time()
model = scikit_gradboost.fit(train_x, train_y)
logging.info("Training time for GB model: %s",
(time.time() - event_time))
elif model_name.lower() == 'rf':
logging.info("Fitting a Random Forest Model...")
if bal_tech == "weighted":
scikit_randomforest = RandomForestClassifier(
class_weight='balanced_subsample')
else:
scikit_randomforest = RandomForestClassifier()
event_time = time.time()
model = scikit_randomforest.fit(train_x, train_y)
logging.info("Training time for RF model: %s",
(time.time() - event_time))
event_time = time.time()
train_pred_y = model.predict(train_x)
logging.info("Training-Prediction time: %s", (time.time() - event_time))
conf_mat = confusion_matrix(train_y, train_pred_y)
logging.info("Confusion Matrix")
logging.info("%s", conf_mat)
logging.info("%s", classification_report(train_y, train_pred_y))
logging.info("Cross-Validating with validation data...")
event_time = time.time()
val_pred_y = model.predict(val_x)
##Sensitivity -- Recall of +ve class (in binary classification)
##Specificity -- Recall of -ve class (in binary classification)
logging.info("Cross-Validation Prediction time: %s",
(time.time() - event_time))
val_conf_mat = confusion_matrix(val_y, val_pred_y)
logging.info("%s", val_conf_mat)
val_conf_mat_tp.append(val_conf_mat[1][1])
logging.info("%s", classification_report(val_y, val_pred_y))
cross_val_prec_list.append(
precision_score(val_y, val_pred_y, average='binary'))
cross_val_rcl_list.append(recall_score(val_y, val_pred_y,
average='binary'))
cross_val_f1_list.append(f1_score(val_y, val_pred_y, average='binary'))
cross_val_rocauc_list.append(roc_auc_score(val_y, val_pred_y))
cross_val_sens_list.append(
specificity_score(val_y, val_pred_y, average='binary'))
cross_val_spec_list.append(
sensitivity_score(val_y, val_pred_y, average='binary'))
cross_val_gm_list.append(
geometric_mean_score(val_y, val_pred_y, average='binary'))
logging.info("Precision: %s ",
precision_score(val_y, val_pred_y, average='binary'))
logging.info("Recall: %s ",
recall_score(val_y, val_pred_y, average='binary'))
logging.info("F1: %s ", f1_score(val_y, val_pred_y, average='binary'))
logging.info("ROC-AUC: %s ", roc_auc_score(val_y, val_pred_y))
logging.info("Specificity: %s ",
specificity_score(val_y, val_pred_y, average='binary'))
logging.info("Sensitivity: %s ",
sensitivity_score(val_y, val_pred_y, average='binary'))
logging.info("Geometric Mean: %s",
geometric_mean_score(val_y, val_pred_y, average='binary'))
return model
def test_stats(model, test_inp, ground_truth_inp, smplng_strtgy_ky,
tst_proj_ky, tst_df_cmnts):
rslt = {}
tst_df = pd.DataFrame()
global tst_rslt_dic, mstr_tst_df
event_time = time.time()
y_class = model.predict(test_inp)
y_pred_prob = model.predict_proba(test_inp)[:, 1]
logging.info("-inference time- %s seconds ---" %
(time.time() - event_time))
logging.info(classification_report(ground_truth_inp, y_class))
logging.info(
precision_recall_fscore_support(ground_truth_inp,
y_class,
average='binary'))
logging.info(confusion_matrix(ground_truth_inp, y_class))
rslt["prec"] = round(
precision_score(ground_truth_inp, y_class, average='binary'), 3)
rslt["rcl"] = round(
recall_score(ground_truth_inp, y_class, average='binary'), 3)
rslt["f1"] = round(f1_score(ground_truth_inp, y_class, average='binary'),
3)
rslt["roc_auc"] = round(roc_auc_score(ground_truth_inp, y_class), 3)
rslt["spec"] = round(
specificity_score(ground_truth_inp, y_class, average='binary'), 3)
rslt["sens"] = round(
sensitivity_score(ground_truth_inp, y_class, average='binary'), 3)
rslt["gm"] = round(
geometric_mean_score(ground_truth_inp, y_class, average='binary'), 3)
tst_rslt_dic[tst_proj_ky] = rslt
tst_df_cmnts = tst_df_cmnts
logging.info("type ground_truth_inp: %s,%s", type(ground_truth_inp),
ground_truth_inp.shape)
logging.info("type y_class: %s,%s", type(y_class), y_class.shape)
logging.info("type y_pred_prob: %s,%s", type(y_pred_prob),
y_pred_prob.shape)
logging.info("type tst_df_cmnts: %s,%s", type(tst_df_cmnts),
tst_df_cmnts.shape)
if tst_proj_ky != 'overall_test':
logging.info(classification_report(ground_truth_inp, y_class))
data = {
'Project': ground_truth_inp,
'Comments': tst_df_cmnts,
'Ground_Truth': ground_truth_inp,
'Predicted_Class': y_class,
'Predicted_Probability': y_pred_prob
}
tst_df = pd.DataFrame(data)
tst_df['Project'] = tst_proj_ky
logging.info("tst_df shape: %s", tst_df.shape)
logging.info("tst_df : %s", tst_df)
mstr_tst_df = mstr_tst_df.append(tst_df, ignore_index=True)
logging.info("mstr_tst_df shape: %s", mstr_tst_df.shape)
logging.info("%s : %s : Precision: %s ", smplng_strtgy_ky, tst_proj_ky,
rslt["prec"])
logging.info("%s : %s : Recall: %s ", smplng_strtgy_ky, tst_proj_ky,
rslt["rcl"])
logging.info("%s : %s : F1: %s ", smplng_strtgy_ky, tst_proj_ky,
rslt["f1"])
logging.info("%s : %s : ROC-AUC: %s ", smplng_strtgy_ky, tst_proj_ky,
rslt["roc_auc"])
logging.info("%s : %s : Specificity: %s ", smplng_strtgy_ky, tst_proj_ky,
rslt["spec"])
logging.info("%s : %s : Sensitivity: %s ", smplng_strtgy_ky, tst_proj_ky,
rslt["sens"])
logging.info("%s : %s : Geometric Mean: %s", smplng_strtgy_ky, tst_proj_ky,
rslt["gm"])
# K-means with correct k
num_clusters = 2
minority = (x_cluster[i]["Class"].values == 1).sum()
min_class = []
labels = []
center = []
for i in range(0, 9):
model = KMeans(n_clusters=num_clusters).fit(x_cluster[i])
labels.append(model.labels_)
center.append(model.cluster_centers_)
min_class.append(minority)
# Assign X to Closest Cluster
y_pred = model.predict(x_cluster[9])
score = geometric_mean_score(x_cluster[9]['Class'], y_pred, average=None)
sensitivity = sensitivity_score(x_cluster[9]['Class'], y_pred, average='micro')
specificity = specificity_score(x_cluster[9]['Class'], y_pred, average='micro')
scores.append(score)
sense.append(sensitivity)
specs.append(specificity)
youden.append((sensitivity + specificity - 1))
gmean.append(math.sqrt(specificity * sensitivity))
# Compute Metric Scores
x_dp = sensitivity / (1 - sensitivity)
y_dp = specificity / (1 - specificity)
dp = (math.sqrt(3) / math.pi)
print("Sensitivity ", sensitivity)
print("Specificity ", specificity)
print("Youden's Index Score: ", (sensitivity + specificity - 1))
print('Geometric Mean Score: ', math.sqrt(specificity * sensitivity))
def test_sensitivity_specificity_f_binary_single_class(
y_pred, expected_sensitivity, expected_specificity):
# Such a case may occur with non-stratified cross-validation
assert sensitivity_score(*y_pred) == expected_sensitivity
assert specificity_score(*y_pred) == expected_specificity
def run_repeatedCV():
sen_df = pd.DataFrame()
spe_df = pd.DataFrame()
auc_df = pd.DataFrame()
mcc_df = pd.DataFrame()
k = 0
pos_data, kmer_data = pd_read_pattern()
new_kmer = shuffle_data(kmer_data)
#KDE kmer using top 30 percentile features with tfidf scores for test derived from train using transform()
for i in range(0, 10):
ctr = 0
print("Window size: ", i + 1, "\n")
if (i == 0):
names2 = ['df_2_cv', 'df_3_cv', 'df_2_tf', 'df_3_tf']
else:
names2 = [
'df_2_cv', 'df_3_cv', 'df_4_cv', 'df_2_tf', 'df_3_tf',
'df_4_tf'
]
X = new_kmer[i]
y = new_kmer[i]['Label']
print("kmer size", names2[ctr])
ctr = ctr + 1
rskf = RepeatedStratifiedKFold(n_splits=10, n_repeats=3)
for train_index, test_index in rskf.split(X, y):
sen_kde = []
spe_kde = []
acc_kde = []
auc_kde = []
m_kde = []
c = []
k = k + 1
print(k, end=",")
X_train, X_test = X.iloc[train_index], X.iloc[test_index]
dat_train, dat_test, names = cv_tf_transformation(X_train, X_test)
for j in range(0, len(dat_train)):
dat_train[j]['Chr'] = dat_train[j]['Chr'].replace(['X'], '21')
dat_test[j]['Chr'] = dat_test[j]['Chr'].replace(['X'], '21')
train_x = dat_train[j].drop('Label', axis=1)
train_y = dat_train[j]['Label']
test_x = dat_test[j].drop('Label', axis=1)
test_y = dat_test[j]['Label']
X_red = feature_reduction_using_trees(train_x, train_y)
rf = RandomForestClassifier()
param_grid = {
'n_estimators': [50, 100, 200, 300, 400],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth': [2, 3, 5, 7],
'min_samples_leaf': [1, 3],
'min_samples_split': [2, 5, 10],
}
grid = GridSearchCV(rf, param_grid, cv=3)
grid.fit(X_red, train_y.ravel())
best_model = grid.best_estimator_
best_model.fit(X_red, train_y.ravel())
y_probs = best_model.predict_proba(test_x[X_red.columns])[:, 1]
thresholds = arange(0, 1, 0.001)
scores = [
roc_auc_score(test_y, convert_to_labels(y_probs, t))
for t in thresholds
]
ix = argmax(scores)
y_test_predictions = np.where(
best_model.predict_proba(test_x[X_red.columns])[:, 1] >
thresholds[ix], 2, 1)
sensi = sensitivity_score(test_y,
y_test_predictions,
pos_label=2)
speci = specificity_score(test_y,
y_test_predictions,
pos_label=2)
accu = accuracy_score(test_y, y_test_predictions)
auro = roc_auc_score(test_y, y_test_predictions)
mcc = metrics.matthews_corrcoef(test_y, y_test_predictions)
c.append(X_red.columns)
sen_kde.append(sensi)
spe_kde.append(speci)
acc_kde.append(accu)
auc_kde.append(auro)
m_kde.append(mcc)
if (i == 0):
sen_df = sen_df.append(
{
'df_2_cv': sen_kde[0],
'df_3_cv': sen_kde[1],
'df_2_tf': sen_kde[2],
'df_3_tf': sen_kde[3]
},
ignore_index=True)
spe_df = spe_df.append(
{
'df_2_cv': spe_kde[0],
'df_3_cv': spe_kde[1],
'df_2_tf': spe_kde[2],
'df_3_tf': spe_kde[3]
},
ignore_index=True)
auc_df = auc_df.append(
{
'df_2_cv': auc_kde[0],
'df_3_cv': auc_kde[1],
'df_2_tf': auc_kde[2],
'df_3_tf': auc_kde[3]
},
ignore_index=True)
mcc_df = mcc_df.append(
{
'df_2_cv': m_kde[0],
'df_3_cv': m_kde[1],
'df_2_tf': m_kde[2],
'df_3_tf': m_kde[3]
},
ignore_index=True)
else:
sen_df = sen_df.append(
{
'df_2_cv': sen_kde[0],
'df_3_cv': sen_kde[1],
'df_4_cv': sen_kde[2],
'df_2_tf': sen_kde[3],
'df_3_tf': sen_kde[4],
'df_4_tf': sen_kde[5]
},
ignore_index=True)
spe_df = spe_df.append(
{
'df_2_cv': spe_kde[0],
'df_3_cv': spe_kde[1],
'df_4_cv': spe_kde[2],
'df_2_tf': spe_kde[3],
'df_3_tf': spe_kde[4],
'df_4_tf': spe_kde[5]
},
ignore_index=True)
auc_df = auc_df.append(
{
'df_2_cv': auc_kde[0],
'df_3_cv': auc_kde[1],
'df_4_cv': auc_kde[2],
'df_2_tf': auc_kde[3],
'df_3_tf': auc_kde[4],
'df_4_tf': auc_kde[5]
},
ignore_index=True)
mcc_df = mcc_df.append(
{
'df_2_cv': m_kde[0],
'df_3_cv': m_kde[1],
'df_4_cv': m_kde[2],
'df_2_tf': m_kde[3],
'df_3_tf': m_kde[4],
'df_4_tf': m_kde[5]
},
ignore_index=True)
def evaluate(model,
test_x,
test_y,
output_folder,
title,
class_specific=False,
all_labels=None,
weight_vector=None):
# Ensure that labels is an np array
if all_labels is None:
labels = None
else:
labels = list(all_labels)
if weight_vector is None:
y_predicted = model.predict(test_x)
y_predicted_max = np.argmax(y_predicted, axis=1)
else:
# Variant Output Shift
y_predicted = model.predict(test_x)
predicted_shift = list()
for e in y_predicted:
predicted_shift.append(shift_output(e, weight_vector))
y_predicted_max = np.argmax(predicted_shift, axis=1)
y_test_max = np.argmax(test_y, axis=1)
# Print classification report
report = classification_report(y_test_max,
y_predicted_max,
labels=labels,
output_dict=True,
digits=5)
report_df = pd.DataFrame(report)
report_df.to_csv(os.path.join(output_folder, 'report_' + title + '.csv'),
sep=' ',
header=True,
mode='a')
# Print confusion matrix
cm = confusion_matrix(y_test_max, y_predicted_max, labels=labels)
cm_df = pd.DataFrame(cm)
cm_df.to_csv(os.path.join(output_folder, 'cm_' + title + '.csv'),
sep=' ',
header=True,
mode='a')
metrics = dict()
# Evaluate further metrics
# =============================================================================
# Balanced Accuracy Score
# =============================================================================
metrics['Balanced Accuracy Score'] = balanced_accuracy_score(
y_test_max, y_predicted_max)
# =============================================================================
# Cohen Kappa Score
# =============================================================================
metrics['Cohen Kappa Score (No weighted)'] = cohen_kappa_score(
y_predicted_max, y_test_max, weights=None)
metrics['Cohen Kappa Score (Linear weighted)'] = cohen_kappa_score(
y_predicted_max, y_test_max, weights='linear')
metrics['Cohen Kappa Score (Quadratic weighted)'] = cohen_kappa_score(
y_predicted_max, y_test_max, weights='quadratic')
# =============================================================================
# Hinge Loss
# =============================================================================
metrics['Hinge Loss'] = hinge_loss(y_test_max, y_predicted, labels=labels)
# =============================================================================
# Matthews Correlation Coefficient
# =============================================================================
metrics['Matthews Correlation Coefficient'] = matthews_corrcoef(
y_test_max, y_predicted_max)
# =============================================================================
# Top k Accuracy Score (does not work, To DO)
# =============================================================================
# print("\n Top k Accuracy: ")
# print(top_k_accuracy_score(y_test_max, y_predicted_max, k=5))
# =============================================================================
# The following also work in the multi label case
# =============================================================================
# =============================================================================
# Accuracy Score
# =============================================================================
metrics['Accuracy Score'] = accuracy_score(y_test_max, y_predicted_max)
# =============================================================================
# F1 Score
# =============================================================================
metrics['F Score (Micro)'] = f1_score(y_test_max,
y_predicted_max,
average='micro')
metrics['F Score (Macro)'] = f1_score(y_test_max,
y_predicted_max,
average='macro')
metrics['F Score (Weighted)'] = f1_score(y_test_max,
y_predicted_max,
average='weighted')
if class_specific:
metrics['F Score (None, i.e. for each class)'] = f1_score(
y_test_max, y_predicted_max, average=None)
# =============================================================================
# ROC AUC Score (in case of multi class sklearn only support macro and weighted averages)
# =============================================================================
# ROC AUC only works if each label occurs at least one time. Hence, we need to catch a exception
print(y_test_max)
try:
metrics['ROC AUC Score (OVR) Macro'] = roc_auc_score(y_test_max,
y_predicted,
multi_class='ovr',
average='macro',
labels=labels)
metrics['ROC AUC Score (OVR) Weighted'] = roc_auc_score(
y_test_max,
y_predicted,
multi_class='ovr',
average='weighted',
labels=labels)
metrics['ROC AUC Score (OVO) Macro'] = roc_auc_score(y_test_max,
y_predicted,
multi_class='ovo',
average='macro',
labels=labels)
metrics['ROC AUC Score (OVO) Weighted'] = roc_auc_score(
y_test_max,
y_predicted,
multi_class='ovo',
average='weighted',
labels=labels)
except:
print("Cannot calculate ROC AUC Score!")
pass
# =============================================================================
# F Beta Score
# =============================================================================
metrics['F Beta Score (Micro) b=0.5'] = fbeta_score(y_test_max,
y_predicted_max,
average='micro',
beta=0.5)
metrics['F Beta Score (Macro) b=0.5'] = fbeta_score(y_test_max,
y_predicted_max,
average='macro',
beta=0.5)
metrics['F Beta Score (Weighted) b=0.5'] = fbeta_score(y_test_max,
y_predicted_max,
average='weighted',
beta=0.5)
if class_specific:
metrics[
'F Beta Score (None, i.e. for each class) b=0.5'] = fbeta_score(
y_test_max, y_predicted_max, average=None, beta=0.5)
metrics['F Beta Score (Micro) b=1.5'] = fbeta_score(y_test_max,
y_predicted_max,
average='micro',
beta=1.5)
metrics['F Beta Score (Macro) b=1.5'] = fbeta_score(y_test_max,
y_predicted_max,
average='macro',
beta=1.5)
metrics['F Beta Score (Weighted) b=1.5'] = fbeta_score(y_test_max,
y_predicted_max,
average='weighted',
beta=1.5)
if class_specific:
metrics[
'F Beta Score (None, i.e. for each class) b=1.5'] = fbeta_score(
y_test_max, y_predicted_max, average=None, beta=1.5)
# =============================================================================
# Hamming Loss
# =============================================================================
metrics['Hamming Loss'] = hamming_loss(y_test_max, y_predicted_max)
# =============================================================================
# Jaccard Score
# =============================================================================
metrics['Jaccard Score (Micro)'] = jaccard_score(y_test_max,
y_predicted_max,
average='micro')
metrics['Jaccard Score (Macro)'] = jaccard_score(y_test_max,
y_predicted_max,
average='macro')
metrics['Jaccard Score (Weighted)'] = jaccard_score(y_test_max,
y_predicted_max,
average='weighted')
if class_specific:
metrics['Jaccard Score (None, i.e. for each class)'] = jaccard_score(
y_test_max, y_predicted_max, average=None)
# =============================================================================
# Log Loss
# =============================================================================
metrics['Logg Loss'] = log_loss(y_test_max, y_predicted, labels=labels)
# =============================================================================
# Precision Score
# =============================================================================
metrics['Precision Score (Micro)'] = precision_score(y_test_max,
y_predicted_max,
average='micro')
metrics['Precision Score (Macro)'] = precision_score(y_test_max,
y_predicted_max,
average='macro')
metrics['Precision Score (Weighted)'] = precision_score(y_test_max,
y_predicted_max,
average='weighted')
if class_specific:
metrics[
'Precision Score (None, i.e. for each class)'] = precision_score(
y_test_max, y_predicted_max, average=None)
# =============================================================================
# Specificity Score
# =============================================================================
metrics['Specificity Score (Micro)'] = specificity_score(y_test_max,
y_predicted_max,
average='micro')
metrics['Specificity Score (Macro)'] = specificity_score(y_test_max,
y_predicted_max,
average='macro')
metrics['Specificity Score (Weighted)'] = specificity_score(
y_test_max, y_predicted_max, average='weighted')
if class_specific:
metrics[
'Specificity Score (None, i.e. for each class)'] = specificity_score(
y_test_max, y_predicted_max, average=None)
# =============================================================================
# Recall Score (also named Sensitivity Score). Hence, the Sensitivity Score values
# should be the same as the Recall Score values
# =============================================================================
metrics['Recall Score (Micro)'] = recall_score(y_test_max,
y_predicted_max,
average='micro')
metrics['Recall Score (Macro)'] = recall_score(y_test_max,
y_predicted_max,
average='macro')
metrics['Recall Score (Weighted)'] = recall_score(y_test_max,
y_predicted_max,
average='weighted')
if class_specific:
metrics['Recall Score (None, i.e. for each class)'] = recall_score(
y_test_max, y_predicted_max, average=None)
metrics['Sensitivity Score (Micro)'] = sensitivity_score(y_test_max,
y_predicted_max,
average='micro')
metrics['Sensitivity Score (Macro)'] = sensitivity_score(y_test_max,
y_predicted_max,
average='macro')
metrics['Sensitivity Score (Weighted)'] = sensitivity_score(
y_test_max, y_predicted_max, average='weighted')
if class_specific:
metrics[
'Sensitivity Score (None, i.e. for each class)'] = sensitivity_score(
y_test_max, y_predicted_max, average=None)
# =============================================================================
# Geometric Mean Score
# =============================================================================
metrics['Geometric Mean Score (Normal)'] = geometric_mean_score(
y_test_max, y_predicted_max)
metrics['Geometric Mean Score (Micro)'] = geometric_mean_score(
y_test_max, y_predicted_max, average='micro')
metrics['Geometric Mean Score (Macro)'] = geometric_mean_score(
y_test_max, y_predicted_max, average='macro')
metrics['Geometric Mean Score (Weighted)'] = geometric_mean_score(
y_test_max, y_predicted_max, average='weighted')
if class_specific:
metrics[
'Geometric Mean Score (None, i.e. for each class)'] = geometric_mean_score(
y_test_max, y_predicted_max, average=None)
# =============================================================================
# Zero one Loss
# =============================================================================
metrics['Zero One Loss'] = zero_one_loss(y_test_max, y_predicted_max)
# =============================================================================
# Make Index Balanced Accuracy with
# =============================================================================
# print("\n MIBA with Matthews")
# geo_mean = make_index_balanced_accuracy(alpha=0.5, squared=True)(hamming_loss)
# print(geo_mean(y_test_max, y_predicted_max))
return metrics
def test_sensitivity_specificity_f_binary_single_class(y_pred,
expected_sensitivity,
expected_specificity):
# Such a case may occur with non-stratified cross-validation
assert sensitivity_score(*y_pred) == expected_sensitivity
assert specificity_score(*y_pred) == expected_specificity
def main(file):
"""
"""
N = re.findall('_(.*?)_final', file)[0]
try:
os.mkdir('./res_0314/{}'.format(N))
except FileExistsError:
pass
res = pd.DataFrame()
res_fit = pd.DataFrame()
metrics = pd.DataFrame(index=['SE', 'SP', 'ACC', 'AUC'])
metrics_fit = pd.DataFrame(index=['SE', 'SP', 'ACC', 'AUC'])
X = pd.read_csv(os.path.join('./docking_origin_data_ywl', file)) ##
X = X.dropna(axis=0)
X.drop(['mol', 'name'], axis=1, inplace=True)
y = X.pop('Label')
X = np.array(X)
for turn in range(1, 11):
proba = pd.DataFrame()
proba_fit = pd.DataFrame()
print('================ {} Turn: {} ================'.format(N, turn))
for X_train, X_test, y_train, y_test in GetKFold(X, y, Kfold=5):
res_, res_fit_ = GetProba(X_train,
X_test,
y_train,
y_test,
loop=100)
proba = pd.concat([proba, res_])
proba_fit = pd.concat([proba_fit, res_fit_])
proba_fit['Turn'] = turn
proba.sort_index(inplace=True)
res['Turn_{}'.format(turn)] = proba[0]
# print(res)
res_fit = pd.concat([res_fit, proba_fit])
# return res, res_fit
for proba, turn in zip(res.iteritems(), range(1, 11)):
y_pred = (proba[1] >= 0.5) + 0
acc = accuracy_score(y, y_pred)
se = sensitivity_score(y, y_pred)
sp = specificity_score(y, y_pred)
auc = roc_auc_score(y, proba[1])
metrics['Turn_{}'.format(turn)] = [se, sp, acc, auc]
y_pred_fit = (res_fit[res_fit.Turn == turn].Proba_fit >= 0.5) + 0
y_true_fit = res_fit[res_fit.Turn == turn].Label.values
acc = accuracy_score(y_true_fit, y_pred_fit)
se = sensitivity_score(y_true_fit, y_pred_fit)
sp = specificity_score(y_true_fit, y_pred_fit)
auc = roc_auc_score(y_pred_fit,
res_fit[res_fit.Turn == turn].Proba_fit)
metrics_fit['Turn_{}'.format(turn)] = [se, sp, acc, auc]
metrics['Mean'] = metrics.loc[:, 'Turn_1':'Turn_10'].mean(axis=1)
metrics['Std'] = metrics.loc[:, 'Turn_1':'Turn_10'].std(axis=1)
metrics_fit['Mean'] = metrics_fit.loc[:, 'Turn_1':'Turn_10'].mean(axis=1)
metrics_fit['Std'] = metrics_fit.loc[:, 'Turn_1':'Turn_10'].std(axis=1)
res.insert(0, 'Label', y)
metrics.to_csv(r'./res_0314/{}/Metrics_{}.csv'.format(N, N))
res.to_csv(r'./res_0314/{}/Proba_{}.csv'.format(N, N), index=False)
metrics_fit.to_csv(r'./res_0314/{}/MetricsFit_{}.csv'.format(N, N))
res_fit.to_csv(r'./res_0314/{}/ProbaFit_{}.csv'.format(N, N), index=False)
def authentication(dataset, method, representation, thread_cnt):
# load data and setup output file
out_file = open(
os.path.join(
os.path.abspath(""), "results",
"authentication_{}_{}_{}.txt".format(dataset, method,
representation)), "w")
feature_dataset = h5py.File(
os.path.join(os.path.abspath(""), "data", method,
"{}.hdf5".format(dataset)), "r")
rs_dataset = h5py.File(
os.path.join(os.path.abspath(""), "data", method, "rs.hdf5"), "r")
X, y = feature_dataset["X"][:], feature_dataset["y"][:]
y = LabelEncoder().fit_transform(np.array([label.decode()
for label in y])) + 1
rs_feature = rs_dataset["X"][0]
# remove subjects from lfw dataset with less than 5 images
if dataset == "lfw":
y_uni, y_cnt = np.unique(y, return_counts=True)
mask = np.array([
idx for idx, label in enumerate(y) if label not in y_uni[y_cnt < 5]
])
X, y = X[mask], y[mask]
for label_new, label in enumerate(np.unique(y)):
y[y == label] = label_new + 1
# fuse features with rs to form biocapsules
if representation == "biocapsule":
bc_gen = BioCapsuleGenerator()
X_train = bc_gen.biocapsule_batch(X, rs_feature)
# 5-fold cross-validation experiment
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
fps, fns, acc, pre, rec, f1s, fpr, frr, eer, aucs = [[] for _ in range(10)]
for k, (train_idx, test_idx) in enumerate(skf.split(X, y)):
# setup for training
print("{} {} {} -- authentication -- fold {}".format(
dataset, method, representation, k))
X_train, y_train = X[train_idx], y[train_idx]
X_test, y_test = X[test_idx], y[test_idx]
# split classes so we can spawn threads to train each binary classifier
classes = np.unique(y_train)
classes_split = [
classes[i:i + thread_cnt]
for i in range(0, classes.shape[0], thread_cnt)
]
# train binary classifiers
y_prob = np.zeros((X_test.shape[0] * classes.shape[0], ))
y_true = np.zeros((X_test.shape[0] * classes.shape[0], ))
for li in classes_split:
threads = []
queue = Queue()
for c in li:
threads.append(
Thread(target=train_test_clf,
args=(c, X_train, y_train, X_test, y_test, queue)))
threads[-1].start()
_ = [t.join() for t in threads]
while not queue.empty():
(c_idx, true, prob) = queue.get()
c_idx = int(c_idx - 1) * X_test.shape[0]
y_true[c_idx:c_idx + X_test.shape[0]] = true
y_prob[c_idx:c_idx + X_test.shape[0]] = prob
del X_train, y_train
del X_test, y_test
y_pred = np.around(y_prob)
_, fp, fn, _ = confusion_matrix(y_true, y_pred).ravel()
fprf, tprf, thresholds = roc_curve(y_true, y_prob)
fps.append(fp)
fns.append(fn)
acc.append(accuracy_score(y_true, y_pred))
pre.append(precision_score(y_true, y_pred))
rec.append(recall_score(y_true, y_pred))
f1s.append(f1_score(y_true, y_pred))
fpr.append(1. - sensitivity_score(y_true, y_pred))
frr.append(1. - specificity_score(y_true, y_pred))
eer.append(brentq(lambda x: 1. - x - interp1d(fprf, tprf)(x), 0., 1.))
aucs.append(auc(fprf, tprf))
out_file.write("Fold {}:\n".format(k))
out_file.write("FP -- {}, FN -- {}\n".format(fps[-1], fns[-1]))
out_file.write("ACC -- {:.6f}\n".format(100. * acc[-1]))
out_file.write("PRE -- {:.6f}\n".format(100. * pre[-1]))
out_file.write("REC -- {:.6f}\n".format(100. * rec[-1]))
out_file.write("F1 -- {:.6f}\n".format(f1s[-1]))
out_file.write("FPR -- {:.6f}\n".format(100. * fpr[-1]))
out_file.write("FRR -- {:.6f}\n".format(100. * frr[-1]))
out_file.write("EER -- {:.6f}\n".format(100. * eer[-1]))
out_file.write("AUC -- {:.6f}\n".format(aucs[-1]))
out_file.flush()
out_file.write("Overall:\n")
out_file.write("FP -- {}, FN -- {}\n".format(np.sum(fps), np.sum(fns)))
out_file.write("ACC -- {:.6f} (+/-{:.6f})\n".format(
100. * np.average(acc), 100. * np.std(acc)))
out_file.write("PRE -- {:.6f} (+/-{:.6f})\n".format(
100. * np.average(pre), 100. * np.std(pre)))
out_file.write("REC -- {:.6f} (+/-{:.6f})\n".format(
100. * np.average(rec), 100. * np.std(rec)))
out_file.write("F1 -- {:.6f} (+/-{:.6f})\n".format(
np.average(f1s), np.std(f1s)))
out_file.write("FPR -- {:.6f} (+/-{:.6f})\n".format(
100. * np.average(fpr), 100. * np.std(fpr)))
out_file.write("FRR -- {:.6f} (+/-{:.6f})\n".format(
100. * np.average(frr), 100. * np.std(frr)))
out_file.write("EER -- {:.6f} (+/-{:.6f})\n".format(
100. * np.average(eer), 100. * np.std(eer)))
out_file.write("AUC -- {:.6f} (+/-{:.6f})\n".format(
np.average(aucs), np.std(aucs)))
out_file.close()
def fit(self, x, y, x_test, y_test):
#
xPos = []
xNeg = []
xTrain = []
yTrain = []
xlastTrain = []
ylastTrain = []
for i in range(0, len(y)):
if y[i] == 1:
xPos.append(x[i])
xlastTrain.append(x[i])
ylastTrain.append(y[i])
else:
xNeg.append(x[i])
xTrain.append(x[i])
yTrain.append(y[i])
xNLSV = []
iterRecord = 0
for i in range(0, self.T):
print(iterRecord)
knernel_train = self.get_weight_kernel(np.array(xTrain),
np.array(xTrain),
self.feature_id,
self.gamma_list)
# print(knernel_train.shape)
iterRecord = iterRecord + 1
svc = SVC(C=self.C,
kernel='precomputed',
class_weight='balanced',
degree=3)
svc.fit(knernel_train, yTrain)
sv = svc.support_ # This is support vector
ne_sv = self.get_NLSV_sv(sv, yTrain)
xTrain, yTrain, xNLSV, lastMar = self.rebuild(
xTrain, yTrain, sv, xNLSV) # rebuild sample
if lastMar < 1 * len(xPos):
break
for i in xNLSV:
xlastTrain.append(i)
ylastTrain.append(-1)
Kernel_last_train = self.get_weight_kernel(np.array(xlastTrain),
np.array(xlastTrain),
self.feature_id,
self.gamma_list)
Kernel_last_test = self.get_weight_kernel(np.array(x_test),
np.array(xlastTrain),
self.feature_id,
self.gamma_list)
svc = SVC(C=self.C,
kernel='precomputed',
class_weight='balanced',
degree=3)
svc.fit(Kernel_last_train, ylastTrain)
y_pre = svc.predict(Kernel_last_test)
ACC = accuracy_score(y_test, y_pre)
SN = sensitivity_score(y_test, y_pre)
SP = specificity_score(y_test, y_pre)
MCC = matthews_corrcoef(y_test, y_pre)
return SN, SP, ACC, MCC
def test(cfg,
model,
test_loader,
pos_classes=cfg.MODEL.MINORITY_CLASS,
device='cpu'):
print("Minority Classes: ", pos_classes)
model = model.to(device)
model.anchor = model.anchor.to(device)
model.eval()
with torch.no_grad():
"""post-training step"""
cls_embedding = []
labels = []
# load training dataset
x, y = load_dataset(cfg)
# randomly sample some samples and matins the class ratio in it
classes = np.unique(y)
cls_ratio = {}
for c in classes:
cls_ratio[c] = sum(y == c) / len(y)
train_sample_size = cfg.TEST.TRAIN_SAMPLE_SIZE
idx = np.arange(len(y))
remain_size = train_sample_size
for c in classes:
cls_idx = idx[y == c]
np.random.shuffle(cls_idx)
sampled_idx = cls_idx[:min(
max(int(train_sample_size * cls_ratio[c]), 1), remain_size)]
remain_size -= len(sampled_idx)
# get embedding
samples = x[sampled_idx]
samples = torch.from_numpy(samples).to(device).float()
embedding = model(samples)
cls_embedding.append(embedding)
labels.append(c)
cls_embedding = torch.cat(cls_embedding, dim=0)
"""test"""
# start evaluation on test dataset
test_loader = tqdm.tqdm(test_loader)
predictions = []
ground_truths = []
scores = []
for i, (x, y) in enumerate(test_loader):
x, y = x.to(device).float(), y.to(device)
emb = model(x)
rel_score = torch.softmax(torch.matmul(emb, cls_embedding.t()),
dim=1)
predict_cls = labels[torch.argmax(rel_score, dim=1)]
predictions.append(predict_cls.item())
ground_truths.append(y.item())
scores.append(rel_score[0, 1])
# compute metrics
gmean = geometric_mean_score(ground_truths,
predictions,
average='binary')
f1 = f1_score(ground_truths, predictions, average='macro')
auc = roc_auc_score(ground_truths, scores, average='macro')
specificity = specificity_score(ground_truths,
predictions,
average='binary')
sensitivity = sensitivity_score(ground_truths,
predictions,
average='binary')
print(
"F1: {:.5f} | G-Mean:{:.5f} | AUC: {:.5f} | Spec: {:.5f} | Sens: {:.5f}"
.format(f1, gmean, auc, specificity, sensitivity))
return f1, gmean, auc, specificity, sensitivity
tpr,
color="blue",
lw=2,
label='ROC curve (area = {f:.2f})'.format(d=1, f=roc_auc))
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC curves')
plt.legend(loc="lower right")
plt.show()
plot_confusion_matrix(y_test, y_pred)
plot_roc_auc(y_test, y_pred)
sensitivity = sensitivity_score(y_test, y_pred)
specificity = specificity_score(y_test, y_pred)
print("Melanoma Sensitivity:", sensitivity)
print("Melanoma Specificity:", specificity)
# In[24]:
def plot_images(X_test, y_pred, y_test):
predicted_class_names = np.array(
[class_names[int(round(id))] for id in y_pred])
# some nice plotting
plt.figure(figsize=(10, 9))
for n in range(30, 60):
plt.subplot(6, 5, n - 30 + 1)
