def test_custom_metric_api_experiment_with_kappa_filename():
"""Test API with metric defined in a file named kappa"""
# register a dummy metric that just returns 1 from
# a file called 'kappa.py'
input_dir = join(_my_dir, "other")
custom_metrics_file = join(input_dir, "kappa.py")
register_custom_metric(custom_metrics_file, "dummy_metric")
# read in some train/test data
train_file = join(input_dir, "examples_train.jsonlines")
test_file = join(input_dir, "examples_test.jsonlines")
train_fs = NDJReader.for_path(train_file).read()
test_fs = NDJReader.for_path(test_file).read()
# set up a learner to tune using our usual kappa metric
# and evaluate it using the dummy metric we loaded
# this should work as there should be no confict between
# the two "kappa" names
learner = Learner("LogisticRegression")
_ = learner.train(train_fs, grid_objective="unweighted_kappa")
results = learner.evaluate(
test_fs,
grid_objective="unweighted_kappa",
output_metrics=["balanced_accuracy", "dummy_metric"])
test_objective_value = results[-2]
test_output_metrics_dict = results[-1]
test_accuracy_value = test_output_metrics_dict["balanced_accuracy"]
test_dummy_metric_value = test_output_metrics_dict["dummy_metric"]
# check that the values are as expected
assert_almost_equal(test_objective_value, 0.9699, places=4)
assert_almost_equal(test_accuracy_value, 0.9792, places=4)
eq_(test_dummy_metric_value, 1.0)
def test_custom_metric_api_experiment():
"""Test API with custom metrics"""
# register two different metrics from two files
input_dir = join(_my_dir, "other")
custom_metrics_file1 = join(input_dir, "custom_metrics.py")
register_custom_metric(custom_metrics_file1, "f075_macro")
custom_metrics_file2 = join(input_dir, "custom_metrics2.py")
register_custom_metric(custom_metrics_file2, "f06_micro")
# read in some train/test data
train_file = join(input_dir, "examples_train.jsonlines")
test_file = join(input_dir, "examples_test.jsonlines")
train_fs = NDJReader.for_path(train_file).read()
test_fs = NDJReader.for_path(test_file).read()
# set up a learner to tune using one of the custom metrics
# and evaluate it using the other one
learner = Learner("LogisticRegression")
_ = learner.train(train_fs, grid_objective="f075_macro")
results = learner.evaluate(
test_fs,
grid_objective="f075_macro",
output_metrics=["balanced_accuracy", "f06_micro"])
test_objective_value = results[-2]
test_output_metrics_dict = results[-1]
test_accuracy_value = test_output_metrics_dict["balanced_accuracy"]
test_f06_micro_value = test_output_metrics_dict["f06_micro"]
# check that the values are as expected
assert_almost_equal(test_objective_value, 0.9785, places=4)
assert_almost_equal(test_accuracy_value, 0.9792, places=4)
assert_almost_equal(test_f06_micro_value, 0.98, places=4)
