data_set.predictions = predict(data_set, model)
# calculate metrics
from mercury_ml.common import CustomMetrics
from mercury_ml.common.tasks import evaluate_metrics
custom_metrics = ["evaluate_numpy_micro_auc", "evaluate_numpy_macro_auc"]
custom_metrics_dict = {
custom_metric_name: getattr(CustomMetrics, custom_metric_name)
for custom_metric_name in custom_metrics
}
print("Metric functions")
for metric_name, metric_function in custom_metrics_dict.items():
print(metric_name, metric_function)
print("")
metrics = evaluate_metrics(data_set, custom_metrics_dict)
import json
print("Metrics:")
print(json.dumps(metrics, indent=2), "\n")
# calculate label metrics
from mercury_ml.common import CustomLabelMetrics
from mercury_ml.common.tasks import evaluate_label_metrics
custom_label_metrics = ["evaluate_numpy_auc", "evaluate_numpy_accuracy"]
custom_label_metrics_dict = {
custom_label_metric_name: getattr(CustomLabelMetrics,
custom_label_metric_name)
for custom_label_metric_name in custom_label_metrics
}
print("Label Metric functions")
print("Data transformed with following parameters: \n")
print_dict(config["exec"]["evaluate_custom_metrics"].get(
"pre_execution_transformation"))
print("Transformed data_bunch consists of: \n")
print_data_bunch(data_bunch_custom_metrics)
# ##### Calculate custom metrics
#
custom_metrics = {}
for data_set_name in config["exec"]["evaluate_custom_metrics"][
"data_set_names"]:
data_set = getattr(data_bunch_custom_metrics, data_set_name)
custom_metrics[data_set_name] = tasks.evaluate_metrics(
data_set, custom_metrics_dict)
print("Resulting custom metrics: \n")
print_dict(custom_metrics)
# ##### Calculate custom label metrics
custom_label_metrics = {}
for data_set_name in config["exec"]["evaluate_custom_label_metrics"][
"data_set_names"]:
data_set = getattr(data_bunch_custom_metrics, data_set_name)
custom_label_metrics[data_set_name] = tasks.evaluate_label_metrics(
data_set, custom_label_metrics_dict)
print("Resulting custom label metrics: \n")
print_dict(custom_label_metrics)
def test_evaluate_metrics(data_set, custom_metrics_dict, expected_dict):
metrics = evaluate_metrics(data_set, custom_metrics_dict)
for metric_name, expected_value in expected_dict.items():
assert metrics[metric_name] == expected_value