def eval(emb,
label_mat,
emb_IDmap,
label_IDmap,
n_splits=5,
random_state=None,
shuffle=False):
"""Evaluate predictions using auROC
Args:
emb(:obj:`np.ndarray`): embedding matrix
label_mat(:obj:`np.ndarray`): label matrix
emb_IDmap(dict of `str`:`int`): IDmap for embedding matrix
label_IDmap(dict of `str`:`int`): IDmap fro label matrix
n_splits(int): number folds in stratified k-fold cross validation
random_state(int): random state used to generate split
"""
y_true_all, y_pred_all = test(emb,
label_mat,
emb_IDmap,
label_IDmap,
n_splits=n_splits,
random_state=random_state,
shuffle=shuffle)
auroc_all = [
auroc(y_true, y_pred)
for y_true, y_pred in zip(y_true_all, y_pred_all)
]
return auroc_all
def _get_misclass_auroc(preds,
targets,
criterion,
topk=1,
expected_data_uncertainty_array=None):
"""
Get AUROC for Misclassification detection
:param preds: Prediction probabilities as numpy array
:param targets: Targets as numpy array
:param criterion: Criterion to use for scoring on misclassification detection.
:param topk: Top-kl class probabilities to consider while making predictions.
:param expected_data_uncertainty_array: Expected data uncertainty as numpy array
:return: AUROC on misclassification detection
"""
misclassification_targets = (1 - _misclass_tgt(preds, targets,
(topk, ))).astype(bool)
if criterion == 'entropy':
criterion_values = np.sum(-preds * np.log(preds), axis=1)
elif criterion == 'confidence':
criterion_values = -preds.max(axis=1)
elif criterion == 'model_uncertainty':
criterion_values = np.sum(-preds * np.log(preds),
axis=1) - expected_data_uncertainty_array
else:
raise NotImplementedError
return auroc(misclassification_targets, criterion_values)
def get_misclass_auroc(preds, targets, criterion, topk=1, **args):
misclassification_targets = (
1 - misclass_tgt(preds, targets, topk)).astype(bool)
if criterion == 'entropy':
criterion_values = np.sum(-preds * np.log(preds), axis=1)
elif criterion == 'confidence':
criterion_values = -preds.max(axis=1)
elif criterion == 'MI':
criterion_values = np.sum(-preds * np.log(preds),
axis=1) - args['mean_ens_entropy']
else:
raise NotImplementedError
return auroc(misclassification_targets, criterion_values)
self.w = np.linalg.inv(X.T.dot(X)+lambdaa).dot((X.T).dot(y))
def decision_function(self,X):
return X.dot(self.w)
if __name__=='__main__':
plt.close("all")
lambdaa = 0.0
X,y = getExamples()
ols = OLS(lambdaa = lambdaa)
ols.fit(X,y)
plt.figure()
e = plotit(X = X, Y = y, clf = ols.decision_function, conts =[-1,0,1])
plt.title("OLS"+" AUC:"+"{0:.2f}".format(auroc(y,ols.decision_function(X))))
plt.show()
clf = LinearSVC(C=1e1)
clf.fit(X,y)
plt.figure()
plotit(X = X, Y = y, clf = clf.decision_function, conts =[-1,0,1],extent = e)
plt.title("SVM"+" AUC:"+"{0:.2f}".format(auroc(y,clf.decision_function(X))))
plt.show()
X[0]=-1000*X[0] #Let's add some fun!
ols2 = OLS(lambdaa = lambdaa)
ols2.fit(X,y)
plt.figure()