pd.get_dummies(perch.test['labels']).values, pred_ensemble_weighted)
# accuracy_weighted2=multiclass_acc(pd.get_dummies(perch.test['labels']).values,pred_ensemble_weighted2)
auc = np.mean(
multilabel_auc(pd.get_dummies(perch.test['labels']).values, pred_ensemble))
auc_weighted = np.mean(
multilabel_auc(
pd.get_dummies(perch.test['labels']).values, pred_ensemble_weighted))
# auc_weighted2=np.mean(multilabel_auc(pd.get_dummies(perch.test['labels']).values,pred_ensemble_weighted2))
#majority vote
pred_ensemble_cat = pred_stacked.argmax(axis=2)
pred_cat_prob = np.zeros((len(perch.test['labels']), 5))
for p in range(pred_cat_prob.shape[0]):
for c in range(5):
pred_cat_prob[p, c] = np.mean(pred_ensemble_cat[:, p] == c)
multiclass_acc(pd.get_dummies(perch.test['labels']).values, pred_cat_prob)
save_table3(
method="Multi-task learning:Ensemble",
chestray=True,
model=basemodel.name,
accuracy=accuracy,
auc=auc,
details=details_,
other=json.dumps({
'weighted': [accuracy_weighted, auc_weighted],
# 'weighted_model':[accuracy_weighted2,auc_weighted2],
'acc_cons_infil': accuracy_consolidation_infiltrates
}))
pd.get_dummies(perch.test['labels']).values, pred_ensemble_weighted)
auc = np.mean(
multilabel_auc(pd.get_dummies(perch.test['labels']).values, pred_ensemble))
auc_weighted = np.mean(
multilabel_auc(
pd.get_dummies(perch.test['labels']).values, pred_ensemble_weighted))
auc_arbitrators = np.mean(
multilabel_auc(
pd.get_dummies(perch.test['labels']).values,
pred_ensemble_arbitrators))
save_table3(method="Supervised Pretraining - Ensemble",
chestray=with_chestray,
model=basemodel.name,
accuracy=accuracy,
auc=auc,
details=details_,
other=json.dumps({
'weighted': [accuracy_weighted, auc_weighted],
'acc_cons_infil':
accuracy_consolidation_infiltrates,
'arbitrators': [accuracy_arbitrators, auc_arbitrators]
}))
save_table_reviewers(method="Supervised Pretraining - Ensemble",
chestray=with_chestray,
model=basemodel.name,
accuracy=rev_acc,
auc=rev_auc,
details=details_)
if display:
perch_confusion_matrix(perch.test['labels'].values,
pred_ensemble_weighted.argmax(axis=1))
'pred_class_3', 'pred_class_4'
]].values,
multi_class='ovr')
aucs = [
roc_auc_score(test_pred_agg['labels'].values == i,
test_pred_agg[f'pred_class_{i}'].values) for i in range(5)
]
accuracy = test_pred_agg['correct'].mean()
site_accuracies = test_pred_agg.groupby('SITE')[['correct']].mean()
age_accuracies = test_pred_agg.groupby('age12m')[['correct']].mean()
print(f"Accuracy: {accuracy:.2f}, AUC: {auc:.2f}")
save_table3(experiment=experiment,
initialize="Imagenet",
model=base_model,
accuracy=accuracy,
auc=auc,
config=json.dumps(best_config),
details=json.dumps({
'hostname': os.uname()[1],
'aucs': aucs
}))
torch.save(best_model.state_dict(),
os.path.join(result_dir, f"weights/{experiment}.pth"))
embeddings = best_model.embedding.weight.detach().cpu().numpy()
labelList = [
'REV1-SITE3', 'REV2-SITE2', 'REV3-SITE4', 'REV4-SITE7', 'REV5-SITE5',
'REV6-SITE6', 'REV7-SITE1', 'REV8-SITE3', 'REV9-SITE2', 'REV10-SITE4',
'REV11-SITE7', 'REV12-SITE5', 'REV13-SITE6', 'REV14-SITE1', 'ARB1', 'ARB2',
'ARB3', 'ARB4'
]
if display:
plt.show()
else:
plt.savefig("/home/pmwaniki/Dropbox/tmp/finetune_%s_%s.png" %
(data_name, details_))
pred_ens = []
for w in swa_callback.weight_list:
model2.set_weights(w)
pred_ens.append(model2.predict(test_dataset))
pred_ens = np.stack(pred_ens)
pred_ens_stacked = pred_ens.mean(axis=0)
sep_ens_acc = [
multiclass_acc(labs_test, pred_ens[i, :, :])
for i in range(pred_ens.shape[0])
]
sep_ens_auc = [
np.mean(multilabel_auc(labs_test, pred_ens[i, :, :]))
for i in range(pred_ens.shape[0])
]
auc = np.mean(multilabel_auc(labs_test, pred_ens_stacked))
acc = multiclass_acc(labs_test, pred_ens_stacked)
save_table3(method="Unsupervised Pretraining",
chestray=False if data_name == "perch" else True,
model=type_,
accuracy=acc,
auc=auc,
details=details_)
pred_ens_swa = model.predict(ds_test)
pred_ens_stacked = pred_ens.mean(axis=0)
sep_ens_acc = [
multiclass_acc(labs_test, pred_ens[i, :, :])
for i in range(pred_ens.shape[0])
]
sep_ens_auc = [
np.mean(multilabel_auc(labs_test, pred_ens[i, :, :]))
for i in range(pred_ens.shape[0])
]
auc = np.mean(multilabel_auc(labs_test, pred_ens_stacked))
acc = multiclass_acc(labs_test, pred_ens_stacked)
cons_infil_scores = consolidation_infiltrates_acc(labs_test, pred_ens_stacked)
multilabel_auc(labs_test, pred_ens_swa)
multiclass_acc(labs_test, pred_ens_swa)
if display:
perch_confusion_matrix(labs_test.argmax(axis=1),
pred_ens_stacked.argmax(axis=1))
save_table3(method="Supervised Pretraining",
chestray=with_chestray,
model=basemodel.name,
accuracy=acc,
auc=auc,
details=details_,
other=json.dumps({'acc_cons_infil': cons_infil_scores}))
pd.get_dummies(perch.test['labels']).values, pred_ensemble)
accuracy_weighted = multiclass_acc(
pd.get_dummies(perch.test['labels']).values, pred_ensemble_weighted)
accuracy_arbitrators = multiclass_acc(
pd.get_dummies(perch.test['labels']).values, pred_ensemble_arbitrators)
auc = np.mean(
multilabel_auc(pd.get_dummies(perch.test['labels']).values, pred_ensemble))
auc_weighted = np.mean(
multilabel_auc(
pd.get_dummies(perch.test['labels']).values, pred_ensemble_weighted))
auc_arbitrators = np.mean(
multilabel_auc(
pd.get_dummies(perch.test['labels']).values,
pred_ensemble_arbitrators))
accuracy_consolidation_infiltrates = consolidation_infiltrates_acc(
pd.get_dummies(perch.test['labels']).values, pred_ensemble)
save_table3(
method="Unsupervised Pretraining:Ensemble",
chestray=False if data_name == "perch" else True,
model=type_,
accuracy=accuracy,
auc=auc,
details=details_,
other=json.dumps({
'weighted': [accuracy_weighted, auc_weighted],
# 'weighted_model':[accuracy_weighted2,auc_weighted2],
'acc_cons_infil': accuracy_consolidation_infiltrates,
'arbitrators': [accuracy_arbitrators, auc_arbitrators]
}))
pred_ens.append(model.model_perch.predict(ds_test_perch))
pred_ens = np.stack(pred_ens)
pred_ens_stacked = pred_ens[0:5, :, :].mean(axis=0)
sep_ens_acc = [
multiclass_acc(labs_test_perch, pred_ens[i, :, :])
for i in range(pred_ens.shape[0])
]
sep_ens_auc = [
np.mean(multilabel_auc(labs_test_perch, pred_ens[i, :, :]))
for i in range(pred_ens.shape[0])
]
auc = np.mean(multilabel_auc(labs_test_perch, pred_ens_stacked))
acc = multiclass_acc(labs_test_perch, pred_ens_stacked)
cons_infil_scores = consolidation_infiltrates_acc(labs_test_perch,
pred_ens_stacked)
test_pred_class = pred_ens_stacked.argmax(axis=1)
test_y_class = labs_test_perch.argmax(axis=1)
confusion_matrix(test_y_class, test_pred_class)
save_table3(method="Multi-task learning",
chestray=True,
model=basemodel.name,
accuracy=acc,
auc=auc,
details=details_,
other=json.dumps({'acc_cons_infil': cons_infil_scores}))