rev_acc = perch_train.groupby('reviewer').apply(
lambda df: np.mean(df['labels'] == df['rev_label'])).to_dict()
rev_weights = {
k: v / np.sum(list(rev_acc.values()))
for k, v in rev_acc.items()
}
rev_weights = [rev_weights[i] for i in range(18)]
#accuracy for each rater
rev_acc = [
multiclass_acc(pd.get_dummies(perch.test['labels']).values, pred_all[i])
for i in range(18)
]
rev_auc = [
np.mean(
multilabel_auc(
pd.get_dummies(perch.test['labels']).values, pred_all[i]))
for i in range(18)
]
# aggregate prediction
pred_stacked = np.stack(pred_all)
pred_ensemble_arbitrators = pred_stacked[14:, :, :].mean(axis=0)
pred_ensemble = pred_stacked.mean(axis=0)
pred_ensemble_weighted = np.average(pred_stacked,
axis=0,
weights=np.array(rev_weights))
accuracy = multiclass_acc(
pd.get_dummies(perch.test['labels']).values, pred_ensemble)
accuracy_consolidation_infiltrates = consolidation_infiltrates_acc(
pd.get_dummies(perch.test['labels']).values, pred_ensemble)
accuracy_arbitrators = multiclass_acc(
weights_swa = []
for w2 in range(len(model.weights)):
w2_l = [w[w2] for w in swa_callback.weight_list[3:]]
w2_stacked = np.stack(w2_l)
w2_agg = w2_stacked.mean(axis=0)
weights_swa.append(w2_agg)
model.set_weights(weights_swa)
pred_ens_swa = model.predict(ds_test)
pred_ens_stacked = pred_ens.mean(axis=0)
sep_ens_acc = [
multilabel_acc(labs_test, pred_ens[i, :, :])
for i in range(pred_ens.shape[0])
]
sep_ens_auc = [
np.mean(multilabel_auc(labs_test, pred_ens[i, :, :]))
for i in range(pred_ens.shape[0])
]
best_model = np.where(sep_ens_auc == np.max(sep_ens_auc))[0][0]
model.set_weights(swa_callback.weight_list[best_model])
auc = np.mean(multilabel_auc(labs_test, pred_ens_stacked))
auc_swa = np.mean(multilabel_auc(labs_test, pred_ens_swa))
acc = multilabel_acc(labs_test, pred_ens_stacked)
multilabel_auc(labs_test, pred_ens_swa)
multilabel_acc(labs_test, pred_ens_swa)
model.save_weights()
# model.save_weights()
pred_ens = []
for w in weights_perch:
model.model_perch.set_weights(w)
pred_ens.append(model.model_perch.predict(ds_test_perch))
pred_ens = np.stack(pred_ens)
pred_ens_stacked = pred_ens[0:5, :, :].mean(axis=0)
sep_ens_acc = [
multiclass_acc(labs_test_perch, pred_ens[i, :, :])
for i in range(pred_ens.shape[0])
]
sep_ens_auc = [
np.mean(multilabel_auc(labs_test_perch, pred_ens[i, :, :]))
for i in range(pred_ens.shape[0])
]
auc = np.mean(multilabel_auc(labs_test_perch, pred_ens_stacked))
acc = multiclass_acc(labs_test_perch, pred_ens_stacked)
cons_infil_scores = consolidation_infiltrates_acc(labs_test_perch,
pred_ens_stacked)
test_pred_class = pred_ens_stacked.argmax(axis=1)
test_y_class = labs_test_perch.argmax(axis=1)
confusion_matrix(test_y_class, test_pred_class)
save_table3(method="Multi-task learning",
chestray=True,