def score(self,
actual: np.array,
predicted: np.array,
sample_weight: typing.Optional[np.array] = None,
labels: typing.Optional[np.array] = None) -> float:
if sample_weight is not None:
sample_weight = sample_weight.ravel()
enc_actual, enc_predicted, labels = prep_actual_predicted(
actual, predicted, labels)
cm_weights = sample_weight if sample_weight is not None else None
# multiclass
if enc_predicted.shape[1] > 1:
enc_predicted = enc_predicted.ravel()
enc_actual = label_binarize(enc_actual, labels).ravel()
cm_weights = np.repeat(
cm_weights,
predicted.shape[1]).ravel() if cm_weights is not None else None
assert enc_predicted.shape == enc_actual.shape
assert cm_weights is None or enc_predicted.shape == cm_weights.shape
cms = daicx.confusion_matrices(enc_actual.ravel(),
enc_predicted.ravel(),
sample_weight=cm_weights)
cms = cms.loc[cms[[
self.__class__._threshold_optimizer
]].idxmax()] # get row(s) for optimal metric defined above
cms['metric'] = cms[['tp', 'fp', 'tn',
'fn']].apply(lambda x: self.protected_metric(*x),
axis=1,
raw=True)
return cms['metric'].mean() # in case of ties
def score(self,
actual: np.array,
predicted: np.array,
sample_weight: typing.Optional[np.array] = None,
labels: typing.Optional[np.array] = None,
X: typing.Optional[dt.Frame] = None,
**kwargs) -> float:
# can compute arbitrary cost from all original features
if X is not None:
assert X.nrows == len(actual)
assert X.ncols >= 1
X_pd = X.to_pandas()
if sample_weight is not None:
sample_weight = sample_weight.ravel()
enc_actual, enc_predicted, labels = prep_actual_predicted(
actual, predicted, labels)
cm_weights = sample_weight if sample_weight is not None else None
# multiclass
if enc_predicted.shape[1] > 1:
enc_predicted = enc_predicted.ravel()
enc_actual = label_binarize(enc_actual, labels).ravel()
cm_weights = np.repeat(
cm_weights,
predicted.shape[1]).ravel() if cm_weights is not None else None
assert enc_predicted.shape == enc_actual.shape
assert cm_weights is None or enc_predicted.shape == cm_weights.shape
cms = daicx.confusion_matrices(enc_actual.ravel(),
enc_predicted.ravel(),
sample_weight=cm_weights)
cms = cms.loc[cms[[
self.__class__._threshold_optimizer
]].idxmax()] # get row(s) for optimal metric defined above
cms['metric'] = cms[['tp', 'fp', 'tn', 'fn']].apply(
lambda x: self.protected_metric(*x, X_pd), axis=1, raw=True)
return cms['metric'].mean() # in case of ties
def test_metrics():
actual=np.array([1,2,3,4])
predicted = np.array([1.1, 2, 3.5, 4])
score = daicx.ll(actual, predicted)
print("score")
print(score)
assert np.array_equal(score, np.array([ 9.9920072216264148e-16, -3.4539575992340879e+01, -6.9079151984681758e+01, -1.0361872797702264e+02])), "ll failed"
score = daicx.log_loss(actual, predicted)
print("score")
print(score)
assert score == -51.80936398851132, "logloss failed"
score = daicx.se(actual, predicted)
print("score")
print(score)
assert np.array_equal(score, np.array([0.010000000000000018, 0., 0.25, 0.])), "se failed"
score = daicx.mse(actual, predicted)
print("score")
print(score)
assert score == 0.065, "mse failed"
score = daicx.rmse(actual, predicted)
print("score")
print(score)
assert score == 0.25495097567963926, "rmse failed"
score = daicx.ce(actual, predicted)
print("score")
print(score)
assert score == 0.5, "ce failed"
score = daicx.ae(actual, predicted)
print("score")
print(score)
assert np.array_equal(score, np.array([0.10000000000000009, 0., 0.5, 0.])), "ae failed"
score = daicx.mae(actual, predicted)
print("score")
print(score)
assert score == 0.15000000000000002, "mae failed"
score = daicx.sle(actual, predicted)
print("score")
print(score)
assert np.allclose(score, np.array([0.002380480119680131, 0., 0.013872843488432929, 0.])), "sle failed"
score = daicx.msle(actual, predicted)
print("score")
print(score)
assert score == 0.004063330902028265, "msle failed"
score = daicx.rmsle(actual, predicted)
print("score")
print(score)
assert score == 0.06374426171843443, "rmsle failed"
score = daicx.auc(actual, predicted)
print("score")
print(score)
assert score == 0.0, "auc failed"
rank = daicx.tied_rank(actual)
print("rank")
print(rank)
assert np.array_equal(rank, np.array([1.0, 2.0, 3.0, 4.0])), "rank failed"
score = daicx.f05_opt(actual, predicted)
print("score")
print(score)
assert score == 3.2142857142857144, "f05_opt failed"
score = daicx.f1_opt(actual, predicted)
print("score")
print(score)
assert score == 1.6363636363636365, "f1_opt failed"
score = daicx.f2_opt(actual, predicted)
print("score")
print(score)
assert score == 1.1363636363636365, "f2_opt failed"
score = daicx.mcc_opt(actual, predicted)
print("score")
print(score)
assert score == 0.0, "mcc_opt failed"
score = daicx.acc_opt(actual, predicted)
print("score")
print(score)
assert score == 2.75, "acc_opt failed"
matrix = daicx.confusion_matrices(actual, predicted)
print("matrix")
print(matrix)#assert score == 1.0, "mcc_opt failed"