map(partial(tf.multiply,y=2.0),num_parallel_calls=AUTOTUNE) for i in range(18): print("Making prediction for reviewer %d ..." % (i + 1)) rev_ = tf.data.Dataset.from_tensor_slices([i] * perch.test.shape[0]) ds_test_ = tf.data.Dataset.zip( (final_test_im, rev_)).batch(50).prefetch(AUTOTUNE) pred_ens = [] for w in weights_perch: model.model_perch.set_weights(w) p = [model.model_perch.predict(k) for k in ds_test_] pred_ens.append(np.concatenate(p, axis=0)) pred_ens2 = np.stack(pred_ens) pred_all.append(pred_ens2.mean(axis=0)) [ multiclass_acc(pd.get_dummies(perch.test['labels']).values, pred_all[i]) for i in range(18) ] [ np.mean( multilabel_auc( pd.get_dummies(perch.test['labels']).values, pred_all[i])) for i in range(18) ] rev_acc = perch_train.groupby('reviewer').apply( lambda df: np.mean(df['labels'] == df['rev_label'])).to_dict() rev_weights = { k: v / np.sum(list(rev_acc.values())) for k, v in rev_acc.items() }
distance_sort='descending', show_leaf_counts=True) plt.show() #reviewer accuracy rev_acc = perch_train.groupby('reviewer').apply( lambda df: np.mean(df['labels'] == df['rev_label'])).to_dict() rev_weights = { k: v / np.sum(list(rev_acc.values())) for k, v in rev_acc.items() } rev_weights = [rev_weights[i] for i in range(18)] #accuracy for each rater rev_acc = [ multiclass_acc(pd.get_dummies(perch.test['labels']).values, pred_all[i]) for i in range(18) ] rev_auc = [ np.mean( multilabel_auc( pd.get_dummies(perch.test['labels']).values, pred_all[i])) for i in range(18) ] # aggregate prediction pred_stacked = np.stack(pred_all) pred_ensemble_arbitrators = pred_stacked[14:, :, :].mean(axis=0) pred_ensemble = pred_stacked.mean(axis=0) pred_ensemble_weighted = np.average(pred_stacked, axis=0,
axs[1].set_title("accuracy") if display: plt.show() else: plt.savefig("/home/pmwaniki/Dropbox/tmp/finetune_%s_%s.png" % (data_name, details_)) pred_ens = [] for w in swa_callback.weight_list: model2.set_weights(w) pred_ens.append(model2.predict(test_dataset)) pred_ens = np.stack(pred_ens) pred_ens_stacked = pred_ens.mean(axis=0) sep_ens_acc = [ multiclass_acc(labs_test, pred_ens[i, :, :]) for i in range(pred_ens.shape[0]) ] sep_ens_auc = [ np.mean(multilabel_auc(labs_test, pred_ens[i, :, :])) for i in range(pred_ens.shape[0]) ] auc = np.mean(multilabel_auc(labs_test, pred_ens_stacked)) acc = multiclass_acc(labs_test, pred_ens_stacked) save_table3(method="Unsupervised Pretraining", chestray=False if data_name == "perch" else True, model=type_, accuracy=acc, auc=auc,