map(partial(tf.multiply,y=2.0),num_parallel_calls=AUTOTUNE)
for i in range(18):
print("Making prediction for reviewer %d ..." % (i + 1))
rev_ = tf.data.Dataset.from_tensor_slices([i] * perch.test.shape[0])
ds_test_ = tf.data.Dataset.zip(
(final_test_im, rev_)).batch(50).prefetch(AUTOTUNE)
pred_ens = []
for w in weights_perch:
model.model_perch.set_weights(w)
p = [model.model_perch.predict(k) for k in ds_test_]
pred_ens.append(np.concatenate(p, axis=0))
pred_ens2 = np.stack(pred_ens)
pred_all.append(pred_ens2.mean(axis=0))
[
multiclass_acc(pd.get_dummies(perch.test['labels']).values, pred_all[i])
for i in range(18)
]
[
np.mean(
multilabel_auc(
pd.get_dummies(perch.test['labels']).values, pred_all[i]))
for i in range(18)
]
rev_acc = perch_train.groupby('reviewer').apply(
lambda df: np.mean(df['labels'] == df['rev_label'])).to_dict()
rev_weights = {
k: v / np.sum(list(rev_acc.values()))
for k, v in rev_acc.items()
}
distance_sort='descending',
show_leaf_counts=True)
plt.show()
#reviewer accuracy
rev_acc = perch_train.groupby('reviewer').apply(
lambda df: np.mean(df['labels'] == df['rev_label'])).to_dict()
rev_weights = {
k: v / np.sum(list(rev_acc.values()))
for k, v in rev_acc.items()
}
rev_weights = [rev_weights[i] for i in range(18)]
#accuracy for each rater
rev_acc = [
multiclass_acc(pd.get_dummies(perch.test['labels']).values, pred_all[i])
for i in range(18)
]
rev_auc = [
np.mean(
multilabel_auc(
pd.get_dummies(perch.test['labels']).values, pred_all[i]))
for i in range(18)
]
# aggregate prediction
pred_stacked = np.stack(pred_all)
pred_ensemble_arbitrators = pred_stacked[14:, :, :].mean(axis=0)
pred_ensemble = pred_stacked.mean(axis=0)
pred_ensemble_weighted = np.average(pred_stacked,
axis=0,
axs[1].set_title("accuracy")
if display:
plt.show()
else:
plt.savefig("/home/pmwaniki/Dropbox/tmp/finetune_%s_%s.png" %
(data_name, details_))
pred_ens = []
for w in swa_callback.weight_list:
model2.set_weights(w)
pred_ens.append(model2.predict(test_dataset))
pred_ens = np.stack(pred_ens)
pred_ens_stacked = pred_ens.mean(axis=0)
sep_ens_acc = [
multiclass_acc(labs_test, pred_ens[i, :, :])
for i in range(pred_ens.shape[0])
]
sep_ens_auc = [
np.mean(multilabel_auc(labs_test, pred_ens[i, :, :]))
for i in range(pred_ens.shape[0])
]
auc = np.mean(multilabel_auc(labs_test, pred_ens_stacked))
acc = multiclass_acc(labs_test, pred_ens_stacked)
save_table3(method="Unsupervised Pretraining",
chestray=False if data_name == "perch" else True,
model=type_,
accuracy=acc,
auc=auc,