def score(self, X, y, sample_weight=None):
"""
Return the mean accuracy on the given test data and labels.
In multi-label classification, this is the subset accuracy which is a
harsh metric since you require for each sample that each label set be
correctly predicted.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Test samples.
y : array-like of shape (n_samples,) or (n_samples, n_outputs)
True labels for X.
sample_weight : array-like of shape (n_samples,), default=None
Sample weights. Currently, sample weight is ignored
Returns
-------
score : float Mean accuracy of self.predict(X) with respect to y.
"""
y_hat = self.predict(X)
return accuracy_score(y_hat, cp.asarray(y, dtype=y.dtype))
def get_loss(self, y, y_pred):
if self.metric == 'error':
return 1 - accuracy_score(y, y_pred)
elif self.metric == 'auc': # TODO: Add a warning checking if y_predict is all [0, 1], it should be probability
return 1 - roc_auc_score(y, y_pred)
else:
raise Exception("Not implemented yet.")
def score(self, y_test, predictions):
""" Score predictions vs ground truth labels on test data """
dataset_dtype = self.hpo_config.dataset_dtype
score = accuracy_score(y_test.astype(dataset_dtype),
predictions.astype(dataset_dtype))
hpo_log.info(f'score = {round(score,5)}')
self.cv_fold_scores.append(score)
return score
def train_and_eval(X_param, y_param, max_depth=16, n_estimators=100):
X_train, X_valid, y_train, y_valid = train_test_split(X_param,
y_param,
random_state=77)
classifier = RandomForestClassifier(max_depth=max_depth,
n_estimators=n_estimators)
classifier.fit(X_train, y_train)
y_pred = classifier.predict(X_valid)
score = accuracy_score(y_valid, y_pred)
return score
def score(self, y_test, predictions):
""" Score predictions vs ground truth labels on test data """
hpo_log.info('> score predictions')
y_test = y_test.compute()
score = accuracy_score(
y_test.astype(self.hpo_config.dataset_dtype),
predictions.astype(self.hpo_config.dataset_dtype))
hpo_log.info(f'\t score = {score}')
self.cv_fold_scores.append(score)
return score
def score(self, X, y, sample_weight=None) -> float:
"""
Return the mean accuracy on the given test data and labels.
In multi-label classification, this is the subset accuracy which is a
harsh metric since you require for each sample that each label set be
correctly predicted.
Currently, sample weight is ignored
"""
y_hat = self.predict(X)
return accuracy_score(y_hat, cp.asarray(y, dtype=y.dtype))
def cross_val_score(clf, X, y, cv=3, scoring=None):
from sklearn.model_selection import StratifiedKFold
# from sklearn.model_selection import KFold
from cupy import asnumpy
from cuml.metrics import accuracy_score
kf = StratifiedKFold(cv)
# kf = KFold(cv)
acc_scores = []
i = 0
# for train_index, test_index in kf.split(X):
for train_index, test_index in kf.split(X, asnumpy(y)):
X_train, X_test = X.iloc[train_index], X.iloc[test_index]
y_train, y_test = y.iloc[train_index], y.iloc[test_index]
# X_train, X_test = X[train_index], X[test_index]
# y_train, y_test = y[train_index], y[test_index]
try:
clf.fit(X_train, y_train, convert_dtype=True)
except:
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
acc_score = accuracy_score(y_test, y_pred)
acc_scores.append(acc_score)
i += 1
return acc_scores
def _calc_score_cuml(y_true, y_preds, y_proba=None, metrics=('accuracy',), task=const.TASK_BINARY, pos_label=1,
classes=None, average=None):
if y_proba is None:
y_proba = y_preds
if len(y_proba.shape) == 2 and y_proba.shape[-1] == 1:
y_proba = y_proba.reshape(-1)
if len(y_preds.shape) == 2 and y_preds.shape[-1] == 1:
y_preds = y_preds.reshape(-1)
y_true = _to_dtype(y_true, 'float64')
y_preds = _to_dtype(y_preds, 'float64')
y_proba = _to_dtype(y_proba, 'float64')
if task == const.TASK_REGRESSION:
if isinstance(y_true, cudf.Series):
y_true = y_true.values
if isinstance(y_preds, cudf.Series):
y_preds = y_preds.values
if isinstance(y_proba, cudf.Series):
y_proba = y_proba.values
scores = {}
for metric in metrics:
if callable(metric):
scores[metric.__name__] = metric(y_true, y_preds)
else:
metric_lower = metric.lower()
if metric_lower == 'auc':
if len(y_proba.shape) == 2:
# if task == const.TASK_MULTICLASS:
# s = cu_metrics.roc_auc_score(y_true, y_proba, multi_class='ovo', labels=classes)
# else:
# s = cu_metrics.roc_auc_score(y_true, y_proba[:, 1])
s = cu_metrics.roc_auc_score(y_true, y_proba[:, 1])
else:
s = cu_metrics.roc_auc_score(y_true, y_proba)
elif metric_lower == 'accuracy':
if y_preds is None:
s = 0
else:
s = cu_metrics.accuracy_score(y_true, y_preds)
# elif metric_lower == 'recall':
# s = cu_metrics.recall_score(y_true, y_preds, **recall_options)
# elif metric_lower == 'precision':
# s = cu_metrics.precision_score(y_true, y_preds, **recall_options)
# elif metric_lower == 'f1':
# s = cu_metrics.f1_score(y_true, y_preds, **recall_options)
elif metric_lower == 'mse':
s = cu_metrics.mean_squared_error(y_true, y_preds)
elif metric_lower == 'mae':
s = cu_metrics.mean_absolute_error(y_true, y_preds)
elif metric_lower == 'msle':
s = cu_metrics.mean_squared_log_error(y_true, y_preds)
elif metric_lower in {'rmse', 'rootmeansquarederror', 'root_mean_squared_error'}:
s = cu_metrics.mean_squared_error(y_true, y_preds, squared=False)
elif metric_lower == 'r2':
s = cu_metrics.r2_score(y_true, y_preds)
elif metric_lower in {'logloss', 'log_loss'}:
# s = cu_metrics.log_loss(y_true, y_proba, labels=classes)
s = cu_metrics.log_loss(y_true, y_proba)
else:
logger.warning(f'unknown metric: {metric}')
continue
if isinstance(s, cp.ndarray):
s = float(cp.asnumpy(s))
scores[metric] = s
return scores
def test_qn(loss, dtype, penalty, l1_strength, l2_strength, fit_intercept):
if penalty == "none" and (l1_strength > 0 or l2_strength > 0):
pytest.skip("`none` penalty does not take l1/l2_strength")
tol = 1e-6
qn = cuQN(loss=loss,
fit_intercept=fit_intercept,
l1_strength=l1_strength,
l2_strength=l2_strength,
tol=1e-8,
output_type="cupy")
if loss == 'softmax':
X, y = make_classification(n_samples=5000,
n_informative=10,
n_features=20,
n_classes=4,
dtype=dtype)
stratify = y.astype(dtype)
X_train, X_test, y_train, y_test = train_test_split(X.astype(dtype),
y.astype(dtype),
stratify=stratify)
most_class = cp.unique(y)[cp.argmax(cp.bincount(y))]
baseline_preds = cp.array([most_class] * y_test.shape[0], dtype=dtype)
baseline_score = accuracy_score(y_test, baseline_preds)
y_pred = qn.fit(X_train, y_train).predict(X_test)
cuml_score = accuracy_score(y_test, y_pred)
assert (cuml_score > baseline_score)
assert (cuml_score >= 0.50)
elif loss == 'sigmoid':
X = np.array(precomputed_X, dtype=dtype)
y = np.array(precomputed_y_log, dtype=dtype)
qn.fit(X, y)
print(qn.objective)
print(qn.coef_)
if penalty == 'none' and l1_strength == 0.0 and l2_strength == 0.0:
if fit_intercept:
assert (qn.objective - 0.40263831615448) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array([[-2.1088872],
[2.4812558]]),
decimal=3)
else:
assert (qn.objective - 0.4317452311515808) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array([[-2.120777],
[3.056865]]),
decimal=3)
elif penalty == 'l1' and l2_strength == 0.0:
if fit_intercept:
if l1_strength == 0.0:
assert (qn.objective - 0.40263831615448) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array(
[[-2.1088872],
[2.4812558]]),
decimal=3)
else:
assert (qn.objective - 0.44295936822891235) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array(
[[-1.6899368],
[1.9021575]]),
decimal=3)
else:
if l1_strength == 0.0:
assert (qn.objective - 0.4317452311515808) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array([[-2.120777],
[3.056865]
]),
decimal=3)
else:
assert (qn.objective - 0.4769895672798157) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array(
[[-1.6214856],
[2.3650239]]),
decimal=3)
# assert False
elif penalty == 'l2' and l1_strength == 0.0:
if fit_intercept:
if l2_strength == 0.0:
assert (qn.objective - 0.40263831615448) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array(
[[-2.1088872],
[2.4812558]]),
decimal=3)
else:
assert (qn.objective - 0.43780848383903503) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array(
[[-1.5337948],
[1.678699]]),
decimal=3)
else:
if l2_strength == 0.0:
assert (qn.objective - 0.4317452311515808) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array([[-2.120777],
[3.056865]
]),
decimal=3)
else:
assert (qn.objective - 0.4750209450721741) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array(
[[-1.3931049],
[2.0140104]]),
decimal=3)
if penalty == 'elasticnet':
if fit_intercept:
if l1_strength == 0.0 and l2_strength == 0.0:
assert (qn.objective - 0.40263831615448) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array(
[[-2.1088872],
[2.4812558]]),
decimal=3)
elif l1_strength == 0.0:
assert (qn.objective - 0.43780848383903503) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array(
[[-1.5337948],
[1.678699]]),
decimal=3)
elif l2_strength == 0.0:
assert (qn.objective - 0.44295936822891235) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array(
[[-1.6899368],
[1.9021575]]),
decimal=3)
else:
assert (qn.objective - 0.467987984418869) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array(
[[-1.3727235],
[1.4639963]]),
decimal=3)
else:
if l1_strength == 0.0 and l2_strength == 0.0:
assert (qn.objective - 0.4317452311515808) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array([[-2.120777],
[3.056865]
]),
decimal=3)
elif l1_strength == 0.0:
assert (qn.objective - 0.4750209450721741) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array(
[[-1.3931049],
[2.0140104]]),
decimal=3)
elif l2_strength == 0.0:
assert (qn.objective - 0.4769895672798157) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array(
[[-1.6214856],
[2.3650239]]),
decimal=3)
else:
assert (qn.objective - 0.5067970156669617) < tol
cp.testing.assert_array_almost_equal(qn.coef_,
np.array(
[[-1.2102532],
[1.752459]]),
decimal=3)
print()
def _accuracy(actuals, preds):
return accuracy_score(ground_truth=actuals, predictions=preds)
