def test_rle_encode_decode():
train_pred = auto_file(
'Oct10_20_28_dpn_128_medium_wonderful_goldberg_val_lb.pth_train_predictions.npz'
)
train_pred = np.load(train_pred)
train_ids = D.all_train_ids()
true_masks = D.read_train_masks(train_ids)
pred_masks = np.array([train_pred[id] for id in train_ids])
pred_masks = (pred_masks > 0.45).astype(np.uint8)
submit = create_submission(train_ids, pred_masks)
submit.to_csv('test_rle_encode_decode.csv.gz',
compression='gzip',
index=False)
decoded_ids, decoded_masks = decode_submission(
'test_rle_encode_decode.csv.gz')
decoded_masks = dict(zip(decoded_ids, decoded_masks))
assert set(decoded_ids) == set(train_ids)
decoded_masks = np.array([decoded_masks[id] for id in train_ids])
p1, r1, _ = do_kaggle_metric(pred_masks, true_masks)
p2, r2, _ = do_kaggle_metric(decoded_masks, true_masks)
assert np.array_equal(p1, p2)
assert np.array_equal(r1, r2)
print(np.mean(p1), np.mean(p2))
def test_kaggle_metric():
ids = D.all_train_ids()
y_true = D.read_train_masks(ids)
# y_pred = np.load('experiments/Sep14_18_14_ternaus_netv3_naughty_roentgen/Sep14_18_14_ternaus_netv3_naughty_roentgen_best_lb.pth_train_predictions.npz')
# y_pred = np.array([y_pred[x] for x in ids])
# y_pred = y_true.copy()
# print(y_pred.min(), y_pred.max())
# print(np.count_nonzero(y_pred > 0), np.count_nonzero(y_true))
# print(np.sum(y_true == (y_pred > 0)) / float(np.prod(y_true.shape)))
precision, result, threshold = do_kaggle_metric(y_true, y_true, 0.5)
print(np.mean(precision))
def process_epoch(model,
criterions: dict,
criterion_weights: dict,
optimizer,
dataloader,
epoch: int,
is_train,
summary_writer,
tag=None) -> dict:
avg_loss = AverageMeter()
metrics = {
'iou': JaccardIndex(0.5),
'acc': PixelAccuracy(),
}
if tag is None:
tag = 'train' if is_train else 'val'
epoch_ids = []
epoch_image = []
epoch_mask_labels = []
epoch_mask_pred = []
epoch_mask_true = []
epoch_losses = {}
for key, _ in criterions.items():
epoch_losses[key] = []
with torch.set_grad_enabled(is_train):
if is_train:
model.train()
else:
model.eval()
n_batches = len(dataloader)
with tqdm(total=n_batches) as tq:
tq.set_description(f'{tag} epoch %d' % epoch)
batch = None
total_loss = None
for batch_index, batch in enumerate(dataloader):
epoch_ids.extend(batch['id'])
epoch_mask_true.extend(to_numpy(batch['mask']))
epoch_mask_labels.extend(compute_mask_class(batch['mask']))
epoch_image.extend(to_numpy(batch['image'])[:, 0:1, :, :])
# Move all data to GPU
for key, value in batch.items():
if isinstance(value, torch.Tensor):
batch[key] = value.cuda(non_blocking=True)
if is_train:
with torch.autograd.detect_anomaly():
optimizer.zero_grad()
predictions = model(batch)
losses = dict((key, criterions[key](
predictions[key], batch[target_for_dsv(key)]))
for key in predictions.keys())
total_loss = compute_total_loss(
losses, criterion_weights)
total_loss.mean().backward()
optimizer.step()
else:
predictions = model(batch)
losses = dict((key, criterions[key](
predictions[key], batch[target_for_dsv(key)]))
for key in predictions.keys())
total_loss = compute_total_loss(losses, criterion_weights)
mask_pred_activate = logit_to_prob(predictions['mask'],
criterions['mask'])
epoch_mask_pred.extend(to_numpy(mask_pred_activate))
# Add losses
for loss_name in predictions.keys():
epoch_losses[loss_name].extend(to_numpy(losses[loss_name]))
# Log metrics
for name, metric in metrics.items():
metric.update(mask_pred_activate, batch['mask'])
if is_train and batch_index == 0:
# Log gradients at the first batch of epoch
for name, param in model.named_parameters():
if param.grad is not None:
summary_writer.add_histogram(
f'{tag}/grad/{name}', to_numpy(param.grad),
epoch)
avg_loss.extend(to_numpy(total_loss))
tq.set_postfix(loss='{:.3f}'.format(avg_loss.avg))
tq.update()
del batch, total_loss
for key, metric in metrics.items():
metric.log_to_tensorboard(summary_writer, f'{tag}/epoch/' + key, epoch)
epoch_ids = np.array(epoch_ids)
epoch_image = np.array(epoch_image)
epoch_mask_true = np.array(epoch_mask_true)
epoch_mask_pred = np.array(epoch_mask_pred)
# End of train epoch
# Log losses
for loss_name, epoch_losses in epoch_losses.items():
if len(epoch_losses):
summary_writer.add_scalar(f'{tag}/loss/{loss_name}',
np.mean(epoch_losses), epoch)
summary_writer.add_histogram(f'{tag}/loss/{loss_name}/histogram',
np.array(epoch_losses), epoch)
# epoch_mask_labels = np.array(epoch_mask_labels)
# for cls in np.unique(epoch_mask_labels):
# summary_writer.add_scalar(f'{tag}/epoch/seg_loss_class_{cls}', np.mean(epoch_losses[epoch_mask_labels == cls]), epoch)
# Mine thresholds on val
if not is_train:
thresholds, scores = threshold_mining(epoch_mask_pred, epoch_mask_true)
i = np.argmax(scores)
optimal_threshold = float(thresholds[i])
lb_at_optimal_threshold = float(scores[i])
summary_writer.add_scalar(f'{tag}/epoch/lb/optimal_threshold',
optimal_threshold, epoch)
summary_writer.add_scalar(f'{tag}/epoch/lb/optimal_score',
lb_at_optimal_threshold, epoch)
# Compute LB metric
precision, result, threshold = do_kaggle_metric(epoch_mask_pred,
epoch_mask_true, 0.50)
lb_50 = np.mean(precision)
summary_writer.add_scalar(f'{tag}/epoch/lb', lb_50, epoch)
# Plot negative examples (LB)
iou_metric_asc = np.argsort(precision)[:64]
iou_negatives = pd.DataFrame.from_dict({
'id':
epoch_ids[iou_metric_asc],
'iou_score':
precision[iou_metric_asc]
})
summary_writer.add_image(
f'{tag}/hard_negatives/lb/image',
make_grid(torch.from_numpy(epoch_image[iou_metric_asc]),
nrow=4,
normalize=True), epoch)
summary_writer.add_image(
f'{tag}/hard_negatives/lb/y_true',
make_grid(torch.from_numpy(epoch_mask_true[iou_metric_asc]),
normalize=False,
nrow=4), epoch)
summary_writer.add_image(
f'{tag}/hard_negatives/lb/y_pred',
make_grid(torch.from_numpy(epoch_mask_pred[iou_metric_asc]),
normalize=False,
nrow=4), epoch)
summary_writer.add_text(f'{tag}/hard_negatives/lb/ids',
'```' + iou_negatives.to_csv(index=False) + '```',
epoch)
if is_train:
# Plot histogram of parameters after each epoch
for name, param in model.named_parameters():
if param.grad is not None:
summary_writer.add_histogram('model/' + name,
to_numpy(param.data), epoch)
metric_scores = {f'{tag}_lb': lb_50, f'{tag}_loss': avg_loss.avg}
for key, metric in metrics.items():
metric_scores[f'{tag}_{key}'] = metric.value()
return metric_scores
def process_epoch(model,
seg_criterion,
optimizer,
dataloader,
epoch: int,
is_train,
summary_writer,
mask_postprocess=noop,
tag=None) -> dict:
avg_seg_loss = AverageMeter()
metrics = {
'iou': JaccardIndex(0.5),
'acc': PixelAccuracy(),
}
if tag is None:
tag = 'train' if is_train else 'val'
epoch_ids = []
epoch_seg_losses = []
epoch_msk_labels = []
epoch_image = []
epoch_mask_pred = []
epoch_mask_true = []
with torch.set_grad_enabled(is_train):
if is_train:
model.train()
else:
model.eval()
n_batches = len(dataloader)
with tqdm(total=n_batches) as tq:
tq.set_description(f'{tag} epoch %d' % epoch)
image = None
mask_true = None
msk_pred = None
seg_loss = None
for batch_index, (image, mask_true,
sample_ids) in enumerate(dataloader):
mask_true = mask_postprocess(mask_true)
epoch_ids.extend(sample_ids)
epoch_image.extend(image.detach().numpy()[:, 0:1, :, :])
epoch_mask_true.extend(mask_true.detach().numpy())
mask_class_labels = compute_mask_class(mask_true)
image, mask_true = image.cuda(
non_blocking=True), mask_true.cuda(non_blocking=True)
if isinstance(seg_criterion, CELoss):
mask_true = mask_true.long().squeeze()
if is_train:
with torch.autograd.detect_anomaly():
optimizer.zero_grad()
msk_pred = mask_postprocess(model(image))
seg_loss = seg_criterion(msk_pred, mask_true)
seg_loss.mean().backward()
optimizer.step()
else:
msk_pred = mask_postprocess(model(image))
seg_loss = seg_criterion(msk_pred, mask_true)
mask_pred_activate = logit_to_prob(msk_pred, seg_criterion)
seg_loss_np = seg_loss.detach().cpu().numpy()
epoch_mask_pred.extend(mask_pred_activate.cpu().numpy())
epoch_seg_losses.extend(seg_loss_np)
epoch_msk_labels.extend(mask_class_labels.numpy())
# Log metrics
for name, metric in metrics.items():
metric.update(mask_pred_activate, mask_true)
if is_train and batch_index == 0:
# Log gradients at the first batch of epoch
for name, param in model.named_parameters():
if param.grad is not None:
summary_writer.add_histogram(
f'{tag}/grad/{name}', param.grad.cpu(), epoch)
avg_seg_loss.extend(seg_loss_np)
tq.set_postfix(seg_loss='{:.3f}'.format(avg_seg_loss.avg))
tq.update()
del image, mask_true, msk_pred, seg_loss
for key, metric in metrics.items():
metric.log_to_tensorboard(summary_writer, f'{tag}/epoch/' + key, epoch)
epoch_ids = np.array(epoch_ids)
epoch_image = np.array(epoch_image)
epoch_mask_true = np.array(epoch_mask_true)
epoch_mask_pred = np.array(epoch_mask_pred)
epoch_seg_losses = np.array(epoch_seg_losses)
epoch_msk_labels = np.array(epoch_msk_labels)
# End of train epoch
# Mine thresholds on val
if not is_train:
thresholds, scores = threshold_mining(epoch_mask_pred, epoch_mask_true)
i = np.argmax(scores)
optimal_threshold = float(thresholds[i])
lb_at_optimal_threshold = float(scores[i])
summary_writer.add_scalar(f'{tag}/epoch/lb/optimal_threshold',
optimal_threshold, epoch)
summary_writer.add_scalar(f'{tag}/epoch/lb/optimal_score',
lb_at_optimal_threshold, epoch)
precision, result, threshold = do_kaggle_metric(epoch_mask_pred,
epoch_mask_true, 0.50)
lb_50 = np.mean(precision)
summary_writer.add_scalar(f'{tag}/epoch/lb', lb_50, epoch)
# Log losses
summary_writer.add_scalar(f'{tag}/epoch/seg_loss', epoch_seg_losses.mean(),
epoch)
summary_writer.add_histogram(f'{tag}/epoch/seg_loss/histogram',
epoch_seg_losses, epoch)
for cls in np.unique(epoch_msk_labels):
summary_writer.add_scalar(
f'{tag}/epoch/seg_loss_class_{cls}',
np.mean(epoch_seg_losses[epoch_msk_labels == cls]), epoch)
# Plot segmentation negatives (loss)
seg_losses_desc = np.argsort(-epoch_seg_losses)[:64]
seg_negatives = pd.DataFrame.from_dict({
'id':
epoch_ids[seg_losses_desc],
'seg_loss':
epoch_seg_losses[seg_losses_desc]
})
summary_writer.add_image(
f'{tag}/hard_negatives/loss/image',
make_grid(torch.from_numpy(epoch_image[seg_losses_desc]),
nrow=4,
normalize=True), epoch)
summary_writer.add_image(
f'{tag}/hard_negatives/loss/y_true',
make_grid(torch.from_numpy(epoch_mask_true[seg_losses_desc]),
normalize=False,
nrow=4), epoch)
summary_writer.add_image(
f'{tag}/hard_negatives/loss/y_pred',
make_grid(torch.from_numpy(epoch_mask_pred[seg_losses_desc]),
normalize=False,
nrow=4), epoch)
summary_writer.add_text(f'{tag}/hard_negatives/loss/ids',
'```' + seg_negatives.to_csv(index=False) + '```',
epoch)
# Plot negative examples (LB)
iou_losses_desc = np.argsort(precision)[:64]
iou_negatives = pd.DataFrame.from_dict({
'id':
epoch_ids[iou_losses_desc],
'iou_score':
epoch_seg_losses[iou_losses_desc]
})
summary_writer.add_image(
f'{tag}/hard_negatives/lb/image',
make_grid(torch.from_numpy(epoch_image[iou_losses_desc]),
nrow=4,
normalize=True), epoch)
summary_writer.add_image(
f'{tag}/hard_negatives/lb/y_true',
make_grid(torch.from_numpy(epoch_mask_true[iou_losses_desc]),
normalize=False,
nrow=4), epoch)
summary_writer.add_image(
f'{tag}/hard_negatives/lb/y_pred',
make_grid(torch.from_numpy(epoch_mask_pred[iou_losses_desc]),
normalize=False,
nrow=4), epoch)
summary_writer.add_text(f'{tag}/hard_negatives/lb/ids',
'```' + iou_negatives.to_csv(index=False) + '```',
epoch)
if is_train:
# Plot histogram of parameters after each epoch
for name, param in model.named_parameters():
if param.grad is not None:
param_data = param.data.cpu().numpy()
summary_writer.add_histogram('model/' + name, param_data,
epoch)
metric_scores = {
f'{tag}_seg_loss': epoch_seg_losses.mean(),
f'{tag}_lb': lb_50
}
for key, metric in metrics.items():
metric_scores[f'{tag}_{key}'] = metric.value()
return metric_scores
def test_inspect_train_predictions():
train_ids = D.all_train_ids()
train_images = D.read_train_images(train_ids)
train_masks = D.read_train_masks(train_ids)
print(train_ids.shape, train_images.shape, train_masks.shape)
CONFIG = auto_file('wonderful_goldberg.json')
WEIGHT_TRAIN = auto_file(
'Oct10_20_28_dpn_128_medium_wonderful_goldberg_val_lb.pth_train_predictions.npz'
)
WEIGHT_TEST = auto_file(
'Oct10_20_28_dpn_128_medium_wonderful_goldberg_val_lb.pth_test_predictions.npz'
)
convert_predictions_to_images(WEIGHT_TEST,
os.path.join('test', 'test_predictions'))
convert_predictions_to_images(WEIGHT_TRAIN,
os.path.join('test', 'train_predictions'))
train_predictions = auto_file(WEIGHT_TRAIN)
train_predictions = np.load(train_predictions)
# image = train_predictions['0aab0afa9c']
train_predictions = np.array([train_predictions[id] for id in train_ids])
print(train_predictions.shape)
threshold, lb_score = threshold_mining(train_predictions,
train_masks,
min_threshold=0.15,
max_threshold=0.85,
step=0.005)
plt.figure()
plt.plot(threshold, lb_score)
plt.tight_layout()
i = np.argmax(lb_score)
best_threshold, best_lb_score = float(threshold[i]), float(lb_score[i])
print(best_threshold, best_lb_score)
config_file = auto_file(CONFIG)
config = json.load(open(config_file))
valid_ids = np.array(config['valid_set'])
valid_mask = D.get_selection_mask(train_ids, valid_ids)
val_threshold, val_lb_score = threshold_mining(
train_predictions[valid_mask],
train_masks[valid_mask],
min_threshold=0.15,
max_threshold=0.85,
step=0.005)
plt.figure()
plt.plot(val_threshold, val_lb_score)
plt.tight_layout()
plt.show()
val_i = np.argmax(val_lb_score)
val_th = val_threshold[val_i]
print(val_threshold[val_i], val_lb_score[val_i])
precision, result, threshold = do_kaggle_metric(train_predictions,
train_masks, val_th)
x = []
y = []
for prec, true_mask in zip(precision, train_masks):
x.append(prec)
y.append(cv2.countNonZero(true_mask))
plt.figure()
plt.scatter(x, y)
plt.tight_layout()
plt.show()