def test_log_loss(): y_true = np.array([0, 0, 1, 1]) y_pred = np.array([0.1, 0.4, 0.35, 0.8]) assert_almost_equal(log_loss(y_true, y_pred), sklearn_log_loss(y_true, y_pred)) y_true = np.array([0, 0, 1, 1, 0]) y_pred = np.array([0.8, 0.4, 0.4, 0.8, 0.8]) assert_almost_equal(log_loss(y_true, y_pred), sklearn_log_loss(y_true, y_pred))
def test_log_loss_at_limits(): y_true = np.array([0., 1., 2.], dtype=np.float) y_pred = np.array([0., 0.5, 1.], dtype=np.float) err_msg = ("The shape of y_pred doesn't " "match the number of classes") with pytest.raises(ValueError, match=err_msg): log_loss(y_true, y_pred) y_true = np.array([0., 0.5, 1.0], dtype=np.float) y_pred = np.array([0., 0.5, 1.], dtype=np.float) err_msg = ("'y_true' can only have integer values") with pytest.raises(ValueError, match=err_msg): log_loss(y_true, y_pred)
def train_and_eval(X_param, y_param, penalty='l2', C=1.0, l1_ratio=None, fit_intercept=True): """ Splits the given data into train and test split to train and evaluate the model for the params parameters. Params ______ X_param: DataFrame. The data to use for training and testing. y_param: Series. The label for training penalty, C, l1_ratio, fit_intercept: The parameter values for Logistic Regression. Returns score: log loss of the fitted model """ X_train, X_valid, y_train, y_valid = train_test_split(X_param, y_param, random_state=42) classifier = LogisticRegression(penalty=penalty, C=C, l1_ratio=l1_ratio, fit_intercept=fit_intercept) classifier.fit(X_train, y_train) y_pred = classifier.predict(X_valid) score = log_loss(y_valid, y_pred) return score
def test_log_loss_random(n_samples, dtype): y_true, _, _, _ = generate_random_labels( lambda rng: rng.randint(0, 10, n_samples).astype(dtype)) y_pred, _, _, _ = generate_random_labels( lambda rng: rng.rand(n_samples, 10)) assert_almost_equal(log_loss(y_true, y_pred), sklearn_log_loss(y_true, y_pred))
def _calc_score_cuml(y_true, y_preds, y_proba=None, metrics=('accuracy',), task=const.TASK_BINARY, pos_label=1, classes=None, average=None): if y_proba is None: y_proba = y_preds if len(y_proba.shape) == 2 and y_proba.shape[-1] == 1: y_proba = y_proba.reshape(-1) if len(y_preds.shape) == 2 and y_preds.shape[-1] == 1: y_preds = y_preds.reshape(-1) y_true = _to_dtype(y_true, 'float64') y_preds = _to_dtype(y_preds, 'float64') y_proba = _to_dtype(y_proba, 'float64') if task == const.TASK_REGRESSION: if isinstance(y_true, cudf.Series): y_true = y_true.values if isinstance(y_preds, cudf.Series): y_preds = y_preds.values if isinstance(y_proba, cudf.Series): y_proba = y_proba.values scores = {} for metric in metrics: if callable(metric): scores[metric.__name__] = metric(y_true, y_preds) else: metric_lower = metric.lower() if metric_lower == 'auc': if len(y_proba.shape) == 2: # if task == const.TASK_MULTICLASS: # s = cu_metrics.roc_auc_score(y_true, y_proba, multi_class='ovo', labels=classes) # else: # s = cu_metrics.roc_auc_score(y_true, y_proba[:, 1]) s = cu_metrics.roc_auc_score(y_true, y_proba[:, 1]) else: s = cu_metrics.roc_auc_score(y_true, y_proba) elif metric_lower == 'accuracy': if y_preds is None: s = 0 else: s = cu_metrics.accuracy_score(y_true, y_preds) # elif metric_lower == 'recall': # s = cu_metrics.recall_score(y_true, y_preds, **recall_options) # elif metric_lower == 'precision': # s = cu_metrics.precision_score(y_true, y_preds, **recall_options) # elif metric_lower == 'f1': # s = cu_metrics.f1_score(y_true, y_preds, **recall_options) elif metric_lower == 'mse': s = cu_metrics.mean_squared_error(y_true, y_preds) elif metric_lower == 'mae': s = cu_metrics.mean_absolute_error(y_true, y_preds) elif metric_lower == 'msle': s = cu_metrics.mean_squared_log_error(y_true, y_preds) elif metric_lower in {'rmse', 'rootmeansquarederror', 'root_mean_squared_error'}: s = cu_metrics.mean_squared_error(y_true, y_preds, squared=False) elif metric_lower == 'r2': s = cu_metrics.r2_score(y_true, y_preds) elif metric_lower in {'logloss', 'log_loss'}: # s = cu_metrics.log_loss(y_true, y_proba, labels=classes) s = cu_metrics.log_loss(y_true, y_proba) else: logger.warning(f'unknown metric: {metric}') continue if isinstance(s, cp.ndarray): s = float(cp.asnumpy(s)) scores[metric] = s return scores