def test_new_metrics(n_samples=2000, knn=50):
X, y = generate_sample(n_samples=n_samples, n_features=10)
sample_weight = numpy.random.exponential(size=n_samples) ** 0.
predictions = numpy.random.random(size=[n_samples, 2])
predictions /= predictions.sum(axis=1, keepdims=True)
predictions *= 1000.
# Checking SDE
features = X.columns[:1]
sde_val1 = sde(y, predictions, X, uniform_variables=features, sample_weight=sample_weight, label=0, knn=knn)
sde2 = KnnBasedSDE(n_neighbours=knn, uniform_features=features, uniform_label=0, )
sde2.fit(X, y, sample_weight=sample_weight)
sde_val2 = sde2(y, predictions, sample_weight=sample_weight)
assert sde_val1 == sde_val2, 'SDE values are different'
# Checking CVM
features = X.columns[:1]
cvm_val1 = cvm_flatness(y, predictions, X, uniform_variables=features, sample_weight=sample_weight, label=0,
knn=knn)
cvm2 = KnnBasedCvM(n_neighbours=knn, uniform_features=features, uniform_label=0, )
cvm2.fit(X, y, sample_weight=sample_weight)
cvm_val2 = cvm2(y, predictions, sample_weight=sample_weight)
assert cvm_val1 == cvm_val2, 'CvM values are different'
def test_new_metrics(n_samples=2000, knn=50):
X, y = generate_sample(n_samples=n_samples, n_features=10)
sample_weight = numpy.random.exponential(size=n_samples)**0.
predictions = numpy.random.random(size=[n_samples, 2])
predictions /= predictions.sum(axis=1, keepdims=True)
predictions *= 1000.
# Checking SDE
features = X.columns[:1]
sde_val1 = sde(y,
predictions,
X,
uniform_variables=features,
sample_weight=sample_weight,
label=0,
knn=knn)
sde2 = KnnBasedSDE(
n_neighbours=knn,
uniform_features=features,
uniform_label=0,
)
sde2.fit(X, y, sample_weight=sample_weight)
sde_val2 = sde2(y, predictions, sample_weight=sample_weight)
assert sde_val1 == sde_val2, 'SDE values are different'
# Checking CVM
features = X.columns[:1]
cvm_val1 = cvm_flatness(y,
predictions,
X,
uniform_variables=features,
sample_weight=sample_weight,
label=0,
knn=knn)
cvm2 = KnnBasedCvM(
n_neighbours=knn,
uniform_features=features,
uniform_label=0,
)
cvm2.fit(X, y, sample_weight=sample_weight)
cvm_val2 = cvm2(y, predictions, sample_weight=sample_weight)
assert cvm_val1 == cvm_val2, 'CvM values are different'
def test_new_metrics(n_samples=2000, knn=50):
X, y = generate_sample(n_samples=n_samples, n_features=10)
sample_weight = numpy.random.exponential(size=n_samples) ** 0.
predictions_orig = numpy.random.random(size=[n_samples, 2])
for shift in [0.1, 0.2]:
predictions = predictions_orig.copy()
predictions[:, 1] += shift * y
predictions /= predictions.sum(axis=1, keepdims=True)
# Checking SDE
features = X.columns[:1]
sde_val1 = sde(y, predictions, X, uniform_features=features, sample_weight=sample_weight, label=0, knn=knn)
sde_metric = KnnBasedSDE(n_neighbours=knn, uniform_features=features, uniform_label=0, )
sde_metric.fit(X, y, sample_weight=sample_weight)
sde_val2 = sde_metric(y, predictions, sample_weight=sample_weight)
assert numpy.allclose(sde_val1, sde_val2), 'SDE values are different'
# Checking theil
theil_val1 = theil_flatness(y, predictions, X, uniform_features=features, sample_weight=sample_weight,
label=0, knn=knn)
theil_metric = KnnBasedTheil(n_neighbours=knn, uniform_features=features, uniform_label=0, )
theil_metric.fit(X, y, sample_weight=sample_weight)
theil_val2 = theil_metric(y, predictions, sample_weight=sample_weight)
print(theil_val1, theil_val2)
assert numpy.allclose(theil_val1, theil_val2), 'Theil values are different'
# Checking CVM
features = X.columns[:1]
cvm_val1 = cvm_flatness(y, predictions, X, uniform_features=features, sample_weight=sample_weight, label=0,
knn=knn)
cvm_metric = KnnBasedCvM(n_neighbours=knn, uniform_features=features, uniform_label=0, )
cvm_metric.fit(X, y, sample_weight=sample_weight)
cvm_val2 = cvm_metric(y, predictions, sample_weight=sample_weight)
assert numpy.allclose(cvm_val1, cvm_val2), 'CvM values are different'
