def pr_curve(precision, recall, average_precision=None):
return V1EventCurve(
kind=V1EventCurveKind.PR,
x=precision,
y=recall,
annotation=str(average_precision) if average_precision else None,
)
def curve(x, y, annotation=None):
return V1EventCurve(
kind=V1EventCurveKind.CUSTOM,
x=x,
y=y,
annotation=str(annotation) if annotation else None,
)
def sklearn_pr_curve(y_preds, y_targets, pos_label=None):
try:
from sklearn.metrics import average_precision_score, precision_recall_curve
except ImportError:
logger.warning(SKLEARN_ERROR_MESSAGE)
try:
y_true = y_targets.numpy()
except AttributeError:
y_true = y_targets
try:
y_pred = y_preds.numpy()
except AttributeError:
y_pred = y_preds
precision, recall, _ = precision_recall_curve(y_true,
y_pred,
pos_label=pos_label)
ap = average_precision_score(y_true, y_pred)
return V1EventCurve(
kind=V1EventCurveKind.PR,
x=precision,
y=recall,
annotation=str(ap),
)
def roc_auc_curve(fpr, tpr, auc=None):
return V1EventCurve(
kind=V1EventCurveKind.ROC,
x=fpr,
y=tpr,
annotation=str(auc) if auc else None,
)
def test_curves_read_yaml(self):
values = [
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
curve=V1EventCurve(kind="roc",
x=[1.1, 3.1, 5.1],
y=[0.1, 0.3, 0.4],
annotation="0.1"),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:25:57"),
curve=V1EventCurve(kind="pr",
x=[1.1, 3.1, 5.1],
y=[0.1, 0.3, 0.4],
annotation="0.21"),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:26:57"),
curve=V1EventCurve(
kind="custom",
x=[1.1, 3.1, 5.1],
y=[0.1, 0.3, 0.4],
annotation="0.1",
),
step=14,
),
]
events = V1Events.read(
name="foo",
kind="curve",
data=os.path.abspath(
"tests/fixtures/polyboard/curve/curve_events.plx"),
)
assert events.name == "foo"
assert len(events.df.values) == 3
for i in range(3):
assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_curve(self):
events = LoggedEventListSpec(
name="foo",
kind="curve",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
curve=V1EventCurve(
kind="roc",
x=[1.1, 3.1, 5.1],
y=[0.1, 0.3, 0.4],
annotation="0.1",
),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
curve=V1EventCurve(
kind="pr",
x=[1.1, 3.1, 5.1],
y=[0.1, 0.3, 0.4],
annotation="0.21",
),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
curve=V1EventCurve(
kind="custom",
x=[1.1, 3.1, 5.1],
y=[0.1, 0.3, 0.4],
annotation="0.1",
),
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def sklearn_pr_curve(y_preds, y_targets):
try:
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score
except ImportError:
logger.warning(SKLEARN_ERROR_MESSAGE)
y_true = y_targets.numpy()
y_pred = y_preds.numpy()
precision, recall, _ = precision_recall_curve(y_true, y_pred)
ap = average_precision_score(y_true, y_pred)
return V1EventCurve(
kind=V1EventCurveKind.PR,
x=precision,
y=recall,
annotation=str(ap),
)
def sklearn_roc_auc_curve(y_preds, y_targets):
try:
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
except ImportError:
logger.warning(SKLEARN_ERROR_MESSAGE)
y_true = y_targets.numpy()
y_pred = y_preds.numpy()
fpr, tpr, _ = roc_curve(y_true, y_pred)
auc = roc_auc_score(y_true, y_pred)
return V1EventCurve(
kind=V1EventCurveKind.ROC,
x=fpr,
y=tpr,
annotation=str(auc),
)
def sklearn_roc_auc_curve(y_preds, y_targets, pos_label=None):
try:
from sklearn.metrics import auc, roc_curve
except ImportError:
logger.warning(SKLEARN_ERROR_MESSAGE)
try:
y_true = y_targets.numpy()
except AttributeError:
y_true = y_targets
try:
y_pred = y_preds.numpy()
except AttributeError:
y_pred = y_preds
fpr, tpr, _ = roc_curve(y_true, y_pred, pos_label=pos_label)
auc_score = auc(fpr, tpr)
return V1EventCurve(
kind=V1EventCurveKind.ROC,
x=fpr,
y=tpr,
annotation=str(auc_score),
)