def test_new_metrics(n_samples=2000, knn=50): X, y = generate_sample(n_samples=n_samples, n_features=10) sample_weight = numpy.random.exponential(size=n_samples) ** 0. predictions = numpy.random.random(size=[n_samples, 2]) predictions /= predictions.sum(axis=1, keepdims=True) predictions *= 1000. # Checking SDE features = X.columns[:1] sde_val1 = sde(y, predictions, X, uniform_variables=features, sample_weight=sample_weight, label=0, knn=knn) sde2 = KnnBasedSDE(n_neighbours=knn, uniform_features=features, uniform_label=0, ) sde2.fit(X, y, sample_weight=sample_weight) sde_val2 = sde2(y, predictions, sample_weight=sample_weight) assert sde_val1 == sde_val2, 'SDE values are different' # Checking CVM features = X.columns[:1] cvm_val1 = cvm_flatness(y, predictions, X, uniform_variables=features, sample_weight=sample_weight, label=0, knn=knn) cvm2 = KnnBasedCvM(n_neighbours=knn, uniform_features=features, uniform_label=0, ) cvm2.fit(X, y, sample_weight=sample_weight) cvm_val2 = cvm2(y, predictions, sample_weight=sample_weight) assert cvm_val1 == cvm_val2, 'CvM values are different'
def test_new_metrics(n_samples=2000, knn=50): X, y = generate_sample(n_samples=n_samples, n_features=10) sample_weight = numpy.random.exponential(size=n_samples)**0. predictions = numpy.random.random(size=[n_samples, 2]) predictions /= predictions.sum(axis=1, keepdims=True) predictions *= 1000. # Checking SDE features = X.columns[:1] sde_val1 = sde(y, predictions, X, uniform_variables=features, sample_weight=sample_weight, label=0, knn=knn) sde2 = KnnBasedSDE( n_neighbours=knn, uniform_features=features, uniform_label=0, ) sde2.fit(X, y, sample_weight=sample_weight) sde_val2 = sde2(y, predictions, sample_weight=sample_weight) assert sde_val1 == sde_val2, 'SDE values are different' # Checking CVM features = X.columns[:1] cvm_val1 = cvm_flatness(y, predictions, X, uniform_variables=features, sample_weight=sample_weight, label=0, knn=knn) cvm2 = KnnBasedCvM( n_neighbours=knn, uniform_features=features, uniform_label=0, ) cvm2.fit(X, y, sample_weight=sample_weight) cvm_val2 = cvm2(y, predictions, sample_weight=sample_weight) assert cvm_val1 == cvm_val2, 'CvM values are different'
def test_new_metrics(n_samples=2000, knn=50): X, y = generate_sample(n_samples=n_samples, n_features=10) sample_weight = numpy.random.exponential(size=n_samples) ** 0. predictions_orig = numpy.random.random(size=[n_samples, 2]) for shift in [0.1, 0.2]: predictions = predictions_orig.copy() predictions[:, 1] += shift * y predictions /= predictions.sum(axis=1, keepdims=True) # Checking SDE features = X.columns[:1] sde_val1 = sde(y, predictions, X, uniform_features=features, sample_weight=sample_weight, label=0, knn=knn) sde_metric = KnnBasedSDE(n_neighbours=knn, uniform_features=features, uniform_label=0, ) sde_metric.fit(X, y, sample_weight=sample_weight) sde_val2 = sde_metric(y, predictions, sample_weight=sample_weight) assert numpy.allclose(sde_val1, sde_val2), 'SDE values are different' # Checking theil theil_val1 = theil_flatness(y, predictions, X, uniform_features=features, sample_weight=sample_weight, label=0, knn=knn) theil_metric = KnnBasedTheil(n_neighbours=knn, uniform_features=features, uniform_label=0, ) theil_metric.fit(X, y, sample_weight=sample_weight) theil_val2 = theil_metric(y, predictions, sample_weight=sample_weight) print(theil_val1, theil_val2) assert numpy.allclose(theil_val1, theil_val2), 'Theil values are different' # Checking CVM features = X.columns[:1] cvm_val1 = cvm_flatness(y, predictions, X, uniform_features=features, sample_weight=sample_weight, label=0, knn=knn) cvm_metric = KnnBasedCvM(n_neighbours=knn, uniform_features=features, uniform_label=0, ) cvm_metric.fit(X, y, sample_weight=sample_weight) cvm_val2 = cvm_metric(y, predictions, sample_weight=sample_weight) assert numpy.allclose(cvm_val1, cvm_val2), 'CvM values are different'