def test_passing_kwarg(self):
with self.subTest("illegal kwargs"):
with self.assertRaises(ValueError):
# `y_pred` is illegal karg
trace = MCC(true_key="label",
pred_key="pred",
output_name=self.mcc_key,
y_pred=None)
with self.subTest("check if kwargs pass to matthews_corrcoef"):
with unittest.mock.patch(
"fastestimator.trace.metric.mcc.matthews_corrcoef"
) as fake:
kwargs = {"e1": "extra1", "e2": "extra2"}
trace = MCC(true_key="label",
pred_key="pred",
output_name=self.mcc_key,
**kwargs)
batch = {"label": tf.constant([0, 1, 0, 1])}
pred = {
"pred": tf.constant([[0.2], [0.6], [0.8], [0.1]])
} # [[0], [1], [1], [0]]
run = TraceRun(trace=trace, batch=batch, prediction=pred)
run.run_trace()
fake_kwargs = fake.call_args[1]
for key, val in kwargs.items():
self.assertTrue(key in fake_kwargs)
self.assertEqual(val, fake_kwargs[key])
def test_torch_binary_class(self):
with self.subTest("ordinal label"):
trace = MCC(true_key="label",
pred_key="pred",
output_name=self.mcc_key)
batch = {"label": torch.tensor([0, 1, 0, 1])}
pred = {
"pred": torch.tensor([[0.2], [0.6], [0.8], [0.1]])
} # [[0], [1], [1], [0]]
run = TraceRun(trace=trace, batch=batch, prediction=pred)
run.run_trace()
tp, tn, fp, fn = [1, 1, 1, 1]
ans = mcc_func(tp, tn, fp, fn)
self.assertEqual(run.data_on_epoch_end[self.mcc_key], ans)
with self.subTest("one-hot label"):
trace = MCC(true_key="label",
pred_key="pred",
output_name=self.mcc_key)
batch = {
"label": torch.tensor([[1, 0], [0, 1], [0, 1], [0, 1]])
} # [0, 1, 1, 1]
pred = {
"pred": torch.tensor([[0.2], [0.6], [0.8], [0.1]])
} # [[0], [1], [1], [0]]
run = TraceRun(trace=trace, batch=batch, prediction=pred)
run.run_trace()
tp, tn, fp, fn = [2, 1, 0, 1]
ans = mcc_func(tp, tn, fp, fn)
self.assertEqual(run.data_on_epoch_end[self.mcc_key], ans)
def setUpClass(cls):
x = np.array([[1, 2], [3, 4]])
x_pred = np.array([[1, 5, 3], [2, 1, 0]])
x_1d = np.array([2.5])
x_pred_1d = np.array([1])
cls.data = Data({'x': x, 'x_pred': x_pred})
cls.data_1d = Data({'x': x_1d, 'x_pred': x_pred_1d})
cls.mcc = MCC(true_key='x', pred_key='x_pred')
def get_estimator(epochs=50,
batch_size=128,
max_train_steps_per_epoch=None,
max_eval_steps_per_epoch=None,
save_dir=tempfile.mkdtemp()):
# step 1
train_data, eval_data = cifair100.load_data()
# Add label noise to simulate real-world labeling problems
corrupt_dataset(train_data)
test_data = eval_data.split(range(len(eval_data) // 2))
pipeline = fe.Pipeline(
train_data=train_data,
eval_data=eval_data,
test_data=test_data,
batch_size=batch_size,
ops=[
Normalize(inputs="x", outputs="x", mean=(0.4914, 0.4822, 0.4465), std=(0.2471, 0.2435, 0.2616)),
PadIfNeeded(min_height=40, min_width=40, image_in="x", image_out="x", mode="train"),
RandomCrop(32, 32, image_in="x", image_out="x", mode="train"),
Sometimes(HorizontalFlip(image_in="x", image_out="x", mode="train")),
CoarseDropout(inputs="x", outputs="x", mode="train", max_holes=1),
ChannelTranspose(inputs="x", outputs="x")
])
# step 2
model = fe.build(model_fn=big_lenet, optimizer_fn='adam')
network = fe.Network(ops=[
ModelOp(model=model, inputs="x", outputs="y_pred"),
SuperLoss(CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), output_confidence="confidence"),
UpdateOp(model=model, loss_name="ce")
])
# step 3
traces = [
MCC(true_key="y", pred_key="y_pred"),
BestModelSaver(model=model, save_dir=save_dir, metric="mcc", save_best_mode="max", load_best_final=True),
LabelTracker(metric="confidence",
label="data_labels",
label_mapping={
"Normal": 0, "Corrupted": 1
},
mode="train",
outputs="label_confidence"),
ImageSaver(inputs="label_confidence", save_dir=save_dir, mode="train"),
]
estimator = fe.Estimator(pipeline=pipeline,
network=network,
epochs=epochs,
traces=traces,
max_train_steps_per_epoch=max_train_steps_per_epoch,
max_eval_steps_per_epoch=max_eval_steps_per_epoch)
return estimator
def test_torch_multi_class(self):
with self.subTest("ordinal label"):
trace = MCC(true_key="label",
pred_key="pred",
output_name=self.mcc_key)
batch = {"label": torch.tensor([0, 0, 0, 1, 1, 2])}
pred = {
"pred":
torch.tensor([[0.2, 0.1, -0.6], [0.6, 2.0,
0.1], [0.1, 0.1, 0.8],
[0.4, 0.1, -0.3], [0.2, 0.7, 0.1],
[0.3, 0.6,
1.5]]) # [[0], [1], [2], [0], [1], [2]]
}
run = TraceRun(trace=trace, batch=batch, prediction=pred)
run.run_trace()
self.assertEqual(run.data_on_epoch_end[self.mcc_key],
0.26111648393354675)
with self.subTest("one-hot label"):
trace = MCC(true_key="label",
pred_key="pred",
output_name=self.mcc_key)
batch = {
"label":
torch.tensor([[1, 0, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0],
[0, 1, 0], [0, 0, 1]])
} # [0, 0, 0, 1, 1, 2]
pred = {
"pred":
torch.tensor([[0.2, 0.1, -0.6], [0.6, 2.0,
0.1], [0.1, 0.1, 0.8],
[0.4, 0.1, -0.3], [0.2, 0.7, 0.1],
[0.3, 0.6,
1.5]]) # [[0], [1], [2], [0], [1], [2]]
}
run = TraceRun(trace=trace, batch=batch, prediction=pred)
run.run_trace()
self.assertEqual(run.data_on_epoch_end[self.mcc_key],
0.26111648393354675)