def compute_auc(test_gen, class_names, pred): Y_testc = to_categorical(test_gen.classes, num_classes=len(class_names)) fpr = dict() tpr = dict() roc_auc = dict() for i in range(len(class_names)): fpr[i], tpr[i], _ = roc_curve(Y_testc[:, i], pred[:, i]) roc_auc[i] = auc_roc(fpr[i], tpr[i]) # micro calculate metrics globally considerig each element of the label indicator matrix as a label. #fpr, tpr, _ = roc_curve(Y_test, predicted) fpr["micro"], tpr["micro"], _ = roc_curve(Y_testc.ravel(), pred.ravel()) roc_auc["micro"] = auc_roc(fpr["micro"], tpr["micro"]) auc_score = roc_auc_score(Y_testc[:, i], pred[:, i], average='weighted') return roc_auc, auc_score, fpr, tpr
all_losses.append(l) #testing test=MyDataset(bags_ts) loader_ts=DataLoader(test, batch_size=1) predictions=[] for param in mlp.parameters(): param.requires_grad =False for idx_ts, tsbag in enumerate(loader_ts): tsbag=tsbag.float() tsbag=cuda(Variable(tsbag)) scores=mlp.forward(tsbag[0]) predictions.append(float(torch.max(scores))) auc=auc_roc(y_ts, predictions) aucs.append(auc) print ('AUC=',auc) f, t, a=metrics.roc_curve(y_ts, predictions) AN=sum(x<0 for x in y_ts) AP=sum(x>0 for x in y_ts) TN=(1.0-f)*AN TP=t*AP Acc2=(TP+TN)/len(y_ts) acc=max(Acc2) print ('accuracy=',acc ) accs.append(acc)
#%% fold_auc = [] skf = StratifiedKFold(n_splits=5) skf.get_n_splits(data, lbl) fold = 0 for tr_idx, ts_idx in skf.split(data, lbl): fold += 1 ixtr, ixts, ytr, yts = data[tr_idx], data[ts_idx], lbl[tr_idx], lbl[ts_idx] test_auc = [] model = Net().cuda() #e.g. input_model model.load_state_dict(torch.load('model_name' + str(fold) + '.pth')) predictions = [] for ijk in range(len(ixts)): sc = [] tsbag = ixts[ijk] for p in range(len(tsbag)): patch = torch.FloatTensor(tsbag[p]) patch = patch.cuda() patch = Variable(patch) patch = patch.reshape([1, 3, 512, 512]) sc.append(mlp.forward(patch).item()) predictions.append(float(np.max(sc))) test_auc.append(auc_roc(yts, predictions)) fold_auc.append(max(test_auc)) print(fold_auc)
#%% from sklearn import svm fold_auc = [] skf = StratifiedKFold(n_splits=5) skf.get_n_splits(data, lbl) pr_auc = [] fold = 0 for tr_idx, ts_idx in skf.split(data, lbl): fold += 1 ixtr, ixts, ytr, yts, pxtr, pxts = data[tr_idx], data[ts_idx], lbl[ tr_idx], lbl[ts_idx], gene[tr_idx], gene[ts_idx] pmodel = svm.SVC(kernel='rbf', gamma='auto', C=100) pmodel.fit(pxtr, ytr) pr_auc.append(auc_roc(yts, pmodel.decision_function(pxts))) criterion = hinge test_auc = [] all_losses = [] t = 0.30 epochs = 150 lupi = Net() lupi.cuda() optimizer = optim.Adam(lupi.parameters(), lr=0.001) max_auc = 0 max_score = torch.FloatTensor([0]).cuda() for e in range(epochs): loss = 0.0
from sklearn.metrics import roc_auc_score as auc_roc import pandas as pd from sklearn.model_selection import StratifiedKFold file=pd.read_csv('./data_files/labels.csv') lbl=l=np.array(file.iloc[:,0],dtype=np.float64) file=pd.read_csv('./data_files/genes.csv') gene=np.array(file) #%% from sklearn import svm fold_auc=[] skf = StratifiedKFold(n_splits=5) skf.get_n_splits(gene,lbl) pr_auc=[] fold=0 for tr_idx, ts_idx in skf.split(gene,lbl): fold+=1 ytr, yts,pxtr,pxts=lbl[tr_idx],lbl[ts_idx],gene[tr_idx],gene[ts_idx] pmodel = svm.SVC(kernel='rbf',gamma='auto', C=100) pmodel.fit(pxtr,ytr) fold_auc.append(auc_roc(yts,pmodel.decision_function(pxts))) print(np.mean(fold_auc))
optimizer.zero_grad() loss.backward(retain_graph=True) optimizer.step() all_losses.append(float(loss)) y_val = [] val_pred = [] for batch_idx, (data, label) in enumerate(val_loader): y_val.append(float(float(label[0]) > 0)) valbag = data.float() valbag = Variable(valbag).type(torch.cuda.FloatTensor) scores = cnn.forward(valbag) val_pred.append(float(torch.max(scores))) val_auc = auc_roc(y_val, val_pred) if e == 0: best_auc = val_auc best_cnn = deepcopy(cnn) if val_auc > best_auc: best_auc = val_auc best_cnn = deepcopy(cnn) avg_loss = l / count # print ("current loss=",loss) print("best validation auc yet=", best_auc) for param in best_cnn.parameters(): param.requires_grad = False predictions = []