def testAccuracyScore(self):
y_pred = [0, 2, 1, 3]
y_true = [0, 1, 2, 3]
score = accuracy_score(y_true, y_pred)
result = self.executor.execute_tileables([score])[0]
expected = sklearn_accuracy_score(y_true, y_pred)
self.assertAlmostEqual(result, expected)
score = accuracy_score(y_true, y_pred, normalize=False)
result = self.executor.execute_tileables([score])[0]
expected = sklearn_accuracy_score(y_true, y_pred, normalize=False)
self.assertAlmostEqual(result, expected)
y_pred = np.array([[0, 1], [1, 1]])
y_true = np.ones((2, 2))
score = accuracy_score(y_true, y_pred)
result = self.executor.execute_tileables([score])[0]
expected = sklearn_accuracy_score(y_true, y_pred)
self.assertAlmostEqual(result, expected)
sample_weight = [0.7, 0.3]
score = accuracy_score(y_true, y_pred, sample_weight=sample_weight)
result = self.executor.execute_tileables([score])[0]
expected = sklearn_accuracy_score(y_true, y_pred, sample_weight=sample_weight)
self.assertAlmostEqual(result, expected)
score = accuracy_score(mt.tensor(y_true), mt.tensor(y_pred),
sample_weight=mt.tensor(sample_weight), normalize=False)
result = self.executor.execute_tileables([score])[0]
expected = sklearn_accuracy_score(y_true, y_pred, sample_weight=sample_weight,
normalize=False)
self.assertAlmostEqual(result, expected)
def test_accuracy_score(setup):
y_pred = [0, 2, 1, 3]
y_true = [0, 1, 2, 3]
score = accuracy_score(y_true, y_pred)
result = score.execute().fetch()
expected = sklearn_accuracy_score(y_true, y_pred)
assert pytest.approx(result) == expected
score = accuracy_score(y_true, y_pred, normalize=False)
result = score.execute().fetch()
expected = sklearn_accuracy_score(y_true, y_pred, normalize=False)
assert pytest.approx(result) == expected
y_pred = np.array([[0, 1], [1, 1]])
y_true = np.ones((2, 2))
score = accuracy_score(y_true, y_pred)
result = score.execute().fetch()
expected = sklearn_accuracy_score(y_true, y_pred)
assert pytest.approx(result) == expected
sample_weight = [0.7, 0.3]
score = accuracy_score(y_true, y_pred, sample_weight=sample_weight)
result = score.execute().fetch()
expected = sklearn_accuracy_score(y_true, y_pred, sample_weight=sample_weight)
assert pytest.approx(result) == expected
score = accuracy_score(mt.tensor(y_true), mt.tensor(y_pred),
sample_weight=mt.tensor(sample_weight), normalize=False)
result = score.execute().fetch()
expected = sklearn_accuracy_score(y_true, y_pred, sample_weight=sample_weight,
normalize=False)
assert pytest.approx(result) == expected
def evaluate_test_perf(self, trained_model, X_test, y_test, threshold=0.5):
with PerfTimer(self, 'score_timer'):
if 'XGBoost' in self.model_type:
dtest = xgboost.dask.DaskDMatrix(self.client, X_test, y_test)
predictions = xgboost.dask.predict(self.client, trained_model,
dtest).compute()
predictions = np.where(
predictions >= threshold, 1,
0) # threshold returned probabilities into 0/1 labels
elif 'RandomForest' in self.model_type:
predictions = trained_model.predict(X_test)
if 'multi-CPU' not in self.compute_type:
predictions = predictions.compute()
if 'multi' in self.compute_type:
y_test = y_test.compute()
if 'GPU' in self.compute_type:
test_accuracy = cuml_accuracy_score(y_test, predictions)
elif 'CPU' in self.compute_type:
test_accuracy = sklearn_accuracy_score(y_test, predictions)
# accumulate internal list
return test_accuracy
def predict(self, trained_model, X_test, y_test, threshold=0.5):
""" Inference with the trained model on the unseen test data """
with PerfTimer(f'predict [ {self.model_type} ]'):
if 'XGBoost' in self.model_type:
if 'single' in self.compute_type:
dtest = xgboost.DMatrix(X_test, y_test)
predictions = trained_model.predict(dtest)
predictions = (predictions > threshold) * 1.0
elif 'multi' in self.compute_type:
dtest = xgboost.dask.DaskDMatrix(self.client, X_test,
y_test)
predictions = xgboost.dask.predict(self.client,
trained_model,
dtest).compute()
predictions = (predictions > threshold) * 1.0
y_test = y_test.compute()
if 'GPU' in self.compute_type:
test_accuracy = cuml_accuracy_score(y_test, predictions)
elif 'CPU' in self.compute_type:
test_accuracy = sklearn_accuracy_score(y_test, predictions)
elif 'RandomForest' in self.model_type:
if 'single' in self.compute_type:
test_accuracy = trained_model.score(X_test, y_test)
elif 'multi' in self.compute_type:
if 'GPU' in self.compute_type:
y_test = y_test.compute()
predictions = trained_model.predict(X_test).compute()
test_accuracy = cuml_accuracy_score(
y_test, predictions)
elif 'CPU' in self.compute_type:
test_accuracy = sklearn_accuracy_score(
y_test, trained_model.predict(X_test))
print(f' subfold score: {test_accuracy}\n')
self.cv_fold_scores += [test_accuracy]
return test_accuracy
def test_dataframe_accuracy_score(setup):
rs = np.random.RandomState(0)
raw = pd.DataFrame({'a': rs.randint(0, 10, (10,)),
'b': rs.randint(0, 10, (10,))})
df = md.DataFrame(raw)
y = df['a'].to_tensor().astype('int')
pred = df['b'].astype('int')
score = accuracy_score(y, pred)
expect = sklearn_accuracy_score(raw['a'].to_numpy().astype('int'),
raw['b'].to_numpy().astype('int'))
assert pytest.approx(score.fetch()) == expect
def predict ( self, trained_model, X_test, y_test, threshold = 0.5 ):
with PerfTimer(f'predict [ {self.model_type} ]'):
if 'XGBoost' in self.model_type:
if 'single' in self.compute_type:
dtest = xgboost.DMatrix( X_test, y_test)
predictions = trained_model.predict( dtest )
predictions = (predictions > threshold ) * 1.0
elif 'multi' in self.compute_type:
dtest = xgboost.dask.DaskDMatrix( self.client, X_test, y_test)
predictions = xgboost.dask.predict( self.client, trained_model, dtest).compute()
predictions = (predictions > threshold ) * 1.0
y_test = y_test.compute()
if 'GPU' in self.compute_type:
test_accuracy = cuml_accuracy_score ( y_test, predictions )
elif 'CPU' in self.compute_type:
test_accuracy = sklearn_accuracy_score ( y_test, predictions )
elif 'RandomForest' in self.model_type:
if 'single' in self.compute_type:
test_accuracy = trained_model.score( X_test, y_test )
elif 'multi' in self.compute_type:
if 'GPU' in self.compute_type:
y_test = y_test.compute()
predictions = trained_model.predict( X_test ).compute()
test_accuracy = cuml_accuracy_score ( y_test, predictions )
elif 'CPU' in self.compute_type:
test_accuracy = sklearn_accuracy_score ( y_test, trained_model.predict( X_test ) )
# accumulate internal list
self.scores += [ test_accuracy ]
return test_accuracy
def accuracy_score(y_pred, y_true, title):
# plot confusion matrix for classes
cnf_matrix = confusion_matrix(y_true, y_pred)
print(title, cnf_matrix)
np.savetxt(util.home_dir + '/' + title + '.csv', cnf_matrix, delimiter=',')
np.set_printoptions(precision=2)
plt.figure()
if np.unique(y_true).size > 2:
plot_confusion_matrix(cnf_matrix, classes=util.newsgroups, title=title)
else:
plot_confusion_matrix(cnf_matrix,
classes=['Liberal', 'Conservative'],
title=title)
confusion_matrices_pdf.savefig(plt.gcf())
return sklearn_accuracy_score(y_true, y_pred)
def scorer(y_true, y_pred, **kwargs) -> float:
"""Accuracy score based on true and predicted target values.
Parameters
----------
y_true : array-like
True labels.
y_pred : array-like
Predicted labels.
Returns
-------
score
float
"""
return sklearn_accuracy_score(y_true, y_pred, **kwargs)
def testAccuracyScore(self):
service_ep = 'http://127.0.0.1:' + self.web_port
timeout = 120 if 'CI' in os.environ else -1
with new_session(service_ep) as sess:
run_kwargs = {'timeout': timeout}
rs = np.random.RandomState(0)
raw = pd.DataFrame({
'a': rs.randint(0, 10, (10, )),
'b': rs.randint(0, 10, (10, ))
})
df = md.DataFrame(raw)
y = df['a'].to_tensor().astype('int')
pred = df['b'].astype('int')
score = accuracy_score(y,
pred,
session=sess,
run_kwargs=run_kwargs)
expect = sklearn_accuracy_score(raw['a'].to_numpy().astype('int'),
raw['b'].to_numpy().astype('int'))
self.assertAlmostEqual(score.fetch(session=sess), expect)