def test_log_loss():
y_true = np.array([0, 0, 1, 1])
y_pred = np.array([0.1, 0.4, 0.35, 0.8])
assert_almost_equal(log_loss(y_true, y_pred),
sklearn_log_loss(y_true, y_pred))
y_true = np.array([0, 0, 1, 1, 0])
y_pred = np.array([0.8, 0.4, 0.4, 0.8, 0.8])
assert_almost_equal(log_loss(y_true, y_pred),
sklearn_log_loss(y_true, y_pred))
def test_log_loss_at_limits():
y_true = np.array([0., 1., 2.], dtype=np.float)
y_pred = np.array([0., 0.5, 1.], dtype=np.float)
err_msg = ("The shape of y_pred doesn't " "match the number of classes")
with pytest.raises(ValueError, match=err_msg):
log_loss(y_true, y_pred)
y_true = np.array([0., 0.5, 1.0], dtype=np.float)
y_pred = np.array([0., 0.5, 1.], dtype=np.float)
err_msg = ("'y_true' can only have integer values")
with pytest.raises(ValueError, match=err_msg):
log_loss(y_true, y_pred)
def train_and_eval(X_param,
y_param,
penalty='l2',
C=1.0,
l1_ratio=None,
fit_intercept=True):
"""
Splits the given data into train and test split to train and evaluate the model
for the params parameters.
Params
______
X_param: DataFrame.
The data to use for training and testing.
y_param: Series.
The label for training
penalty, C, l1_ratio, fit_intercept: The parameter values for Logistic Regression.
Returns
score: log loss of the fitted model
"""
X_train, X_valid, y_train, y_valid = train_test_split(X_param,
y_param,
random_state=42)
classifier = LogisticRegression(penalty=penalty,
C=C,
l1_ratio=l1_ratio,
fit_intercept=fit_intercept)
classifier.fit(X_train, y_train)
y_pred = classifier.predict(X_valid)
score = log_loss(y_valid, y_pred)
return score
def test_log_loss_random(n_samples, dtype):
y_true, _, _, _ = generate_random_labels(
lambda rng: rng.randint(0, 10, n_samples).astype(dtype))
y_pred, _, _, _ = generate_random_labels(
lambda rng: rng.rand(n_samples, 10))
assert_almost_equal(log_loss(y_true, y_pred),
sklearn_log_loss(y_true, y_pred))
def _calc_score_cuml(y_true, y_preds, y_proba=None, metrics=('accuracy',), task=const.TASK_BINARY, pos_label=1,
classes=None, average=None):
if y_proba is None:
y_proba = y_preds
if len(y_proba.shape) == 2 and y_proba.shape[-1] == 1:
y_proba = y_proba.reshape(-1)
if len(y_preds.shape) == 2 and y_preds.shape[-1] == 1:
y_preds = y_preds.reshape(-1)
y_true = _to_dtype(y_true, 'float64')
y_preds = _to_dtype(y_preds, 'float64')
y_proba = _to_dtype(y_proba, 'float64')
if task == const.TASK_REGRESSION:
if isinstance(y_true, cudf.Series):
y_true = y_true.values
if isinstance(y_preds, cudf.Series):
y_preds = y_preds.values
if isinstance(y_proba, cudf.Series):
y_proba = y_proba.values
scores = {}
for metric in metrics:
if callable(metric):
scores[metric.__name__] = metric(y_true, y_preds)
else:
metric_lower = metric.lower()
if metric_lower == 'auc':
if len(y_proba.shape) == 2:
# if task == const.TASK_MULTICLASS:
# s = cu_metrics.roc_auc_score(y_true, y_proba, multi_class='ovo', labels=classes)
# else:
# s = cu_metrics.roc_auc_score(y_true, y_proba[:, 1])
s = cu_metrics.roc_auc_score(y_true, y_proba[:, 1])
else:
s = cu_metrics.roc_auc_score(y_true, y_proba)
elif metric_lower == 'accuracy':
if y_preds is None:
s = 0
else:
s = cu_metrics.accuracy_score(y_true, y_preds)
# elif metric_lower == 'recall':
# s = cu_metrics.recall_score(y_true, y_preds, **recall_options)
# elif metric_lower == 'precision':
# s = cu_metrics.precision_score(y_true, y_preds, **recall_options)
# elif metric_lower == 'f1':
# s = cu_metrics.f1_score(y_true, y_preds, **recall_options)
elif metric_lower == 'mse':
s = cu_metrics.mean_squared_error(y_true, y_preds)
elif metric_lower == 'mae':
s = cu_metrics.mean_absolute_error(y_true, y_preds)
elif metric_lower == 'msle':
s = cu_metrics.mean_squared_log_error(y_true, y_preds)
elif metric_lower in {'rmse', 'rootmeansquarederror', 'root_mean_squared_error'}:
s = cu_metrics.mean_squared_error(y_true, y_preds, squared=False)
elif metric_lower == 'r2':
s = cu_metrics.r2_score(y_true, y_preds)
elif metric_lower in {'logloss', 'log_loss'}:
# s = cu_metrics.log_loss(y_true, y_proba, labels=classes)
s = cu_metrics.log_loss(y_true, y_proba)
else:
logger.warning(f'unknown metric: {metric}')
continue
if isinstance(s, cp.ndarray):
s = float(cp.asnumpy(s))
scores[metric] = s
return scores
