def test_crf():
loaded = np.load('/mnt/g/ship/tmp/val_out.npz')
outputs = loaded['outputs']
y_true = loaded['y_true']
print(outputs.shape)
print(y_true.shape)
tgt_size = (settings.ORIG_H, settings.ORIG_W)
resized = list(map(lambda x: resize(x, tgt_size), outputs))
print(resized[0].shape, len(resized))
y_pred = list(map(lambda x: (x > 0.5).astype(np.uint8), resized))
print(y_pred[0].shape, len(y_pred))
iou_score = intersection_over_union(y_true, y_pred)
iout_score = intersection_over_union_thresholds(y_true, y_pred)
print(iou_score, iout_score)
_, val_meta = get_train_val_meta(True)
img_ids = val_meta['ImageId'].values.tolist()
crf_imgs = []
for i, img_id in enumerate(img_ids):
orig_img = imread('/mnt/g/ship/train_v2/{}'.format(img_id))
crf_output = crf(orig_img, y_pred[i])
crf_imgs.append(crf_output)
if i % 100 == 0:
print(i)
iou_score = intersection_over_union(y_true, crf_imgs)
iout_score = intersection_over_union_thresholds(y_true, crf_imgs)
print(iou_score, iout_score)
def do_eval(net, dataset):
net.set_mode('eval')
probs = np.zeros((len(dataset), 101, 101))
truths = np.zeros((len(dataset), 101, 101))
for i in range(len(dataset)):
with torch.no_grad():
index, image, y_mask, _ = dataset[i]
hflip_image = np.array(image)[:, ::-1]
images = np.array([image, hflip_image])
images = torch.Tensor(images).cuda()
logit = net(images)
prob = logit.sigmoid()
prob = prob.cpu().data.numpy().squeeze()
mask = prob[0]
hflip_mask = prob[1][:, ::-1]
prob = (mask + hflip_mask) / 2
prob = prob[13:128 - 14, 13:128 - 14]
probs[i, :, :] = prob
truths[i, :, :] = y_mask
iou = intersection_over_union_thresholds(np.int32(truths > 0.5),
np.int32(probs > 0.5))
return iou
def _evaluate_pipeline(pipeline_name, validation_size):
meta = pd.read_csv(os.path.join(params.meta_dir, 'stage1_metadata.csv'))
meta_train = meta[meta['is_train'] == 1]
valid_ids = eval(params.valid_category_ids)
meta_train_split, meta_valid_split = train_valid_split(meta_train, validation_size, valid_category_ids=valid_ids)
data = {'input': {'meta': meta_valid_split,
'meta_valid': None,
'train_mode': False,
'target_sizes': meta_valid_split[SIZE_COLUMNS].values
},
}
y_true = read_masks(meta_valid_split[Y_COLUMNS_SCORING].values)
pipeline = PIPELINES[pipeline_name]['inference'](SOLUTION_CONFIG)
pipeline.clean_cache()
output = pipeline.transform(data)
pipeline.clean_cache()
y_pred = output['y_pred']
logger.info('Calculating IOU and IOUT Scores')
iou_score = intersection_over_union(y_true, y_pred)
logger.info('IOU score on validation is {}'.format(iou_score))
ctx.channel_send('IOU Score', 0, iou_score)
iout_score = intersection_over_union_thresholds(y_true, y_pred)
logger.info('IOUT score on validation is {}'.format(iout_score))
ctx.channel_send('IOUT Score', 0, iout_score)
def validate(args, model, val_loader, epoch=0, threshold=0.5):
model.eval()
outputs = []
focal_loss, lovaz_loss, salt_loss, w_loss = 0, 0, 0, 0
with torch.no_grad():
for img, target, salt_target in val_loader:
if args.depths:
add_depth_channel(img, args.pad_mode)
img, target, salt_target = img.cuda(), target.cuda(
), salt_target.cuda()
output, salt_out = model(img)
_, floss, lovaz, _salt_loss, _w_loss = weighted_loss(
args, (output, salt_out), (target, salt_target), epoch=epoch)
focal_loss += floss
lovaz_loss += lovaz
salt_loss += _salt_loss
w_loss += _w_loss
output = torch.sigmoid(output)
for o in output.cpu():
outputs.append(o.squeeze().numpy())
n_batches = val_loader.num // args.batch_size if val_loader.num % args.batch_size == 0 else val_loader.num // args.batch_size + 1
# y_pred, list of np array, each np array's shape is 101,101
y_pred = generate_preds(args, outputs, (settings.ORIG_H, settings.ORIG_W),
threshold)
iou_score = intersection_over_union(val_loader.y_true, y_pred)
iout_score = intersection_over_union_thresholds(val_loader.y_true, y_pred)
return iout_score, iou_score, focal_loss / n_batches, lovaz_loss / n_batches, salt_loss / n_batches, iout_score * 4 - w_loss
def do_eval(net, dataset, mode='padding'):
net.set_mode('eval')
probs = np.zeros((len(dataset), 101, 101))
truths = np.zeros((len(dataset), 101, 101))
for i in range(len(dataset)):
with torch.no_grad():
index, image, y_mask, y_label = dataset[i]
hflip_image = np.array(image)[:, ::-1]
images = np.array([image, hflip_image])
images = torch.Tensor(images).cuda()
logit_fuse, logit_pixel, logit_image = net(images)
prob = logit_fuse.sigmoid()
prob = prob.cpu().data.numpy().squeeze()
mask = prob[0]
hflip_mask = prob[1][:, ::-1]
prob = (mask + hflip_mask) / 2
if mode == 'padding':
prob = prob[13:128 - 14, 13:128 - 14]
else:
prob = cv2.resize(prob, (101, 101))
probs[i, :, :] = prob
truths[i, :, :] = y_mask
iou = intersection_over_union_thresholds(np.int32(truths >= 0.5),
np.int32(probs >= 0.5))
return probs, iou
def validate(args, model, val_loader, epoch=0, threshold=0.5):
model.eval()
print('validating...')
outputs = []
val_loss = 0
with torch.no_grad():
for img, target, salt_target in val_loader:
img, target, salt_target = img.cuda(), target.cuda(
), salt_target.cuda()
output, salt_out = model(img)
#print(output.size(), salt_out.size())
loss = weighted_loss((output, salt_out), (target, salt_target),
epoch=epoch)
val_loss += loss.item()
output = torch.sigmoid(output)
for o in output.cpu():
outputs.append(o.squeeze().numpy())
n_batches = val_loader.num // args.batch_size if val_loader.num % args.batch_size == 0 else val_loader.num // batch_size + 1
# y_pred, list of 400 np array, each np array's shape is 101,101
y_pred = generate_preds_softmax(outputs,
(settings.ORIG_H, settings.ORIG_W),
threshold)
print(y_pred[0].shape)
print('Validation loss: {:.4f}'.format(val_loss / n_batches))
iou_score = intersection_over_union(val_loader.y_true, y_pred)
iout_score = intersection_over_union_thresholds(val_loader.y_true, y_pred)
print('IOU score on validation is {:.4f}'.format(iou_score))
print('IOUT score on validation is {:.4f}'.format(iout_score))
return iout_score, iou_score, val_loss / n_batches
def test_bbox():
tgt_size = (settings.ORIG_H, settings.ORIG_W)
outputs, y_true = get_val_result(16)
resized = list(map(lambda x: resize_image(x, tgt_size), outputs))
print(resized[0].shape, len(resized))
y_pred = list(map(lambda x: (x > 0.5).astype(np.uint8), resized))
print(y_pred[0].shape, len(y_pred))
iou_score = intersection_over_union(y_true, y_pred)
iout_score = intersection_over_union_thresholds(y_true, y_pred)
print(iou_score, iout_score)
processed = list(map(lambda x: masks_to_bounding_boxes(x), y_pred))
iou_score = intersection_over_union(y_true, processed)
iout_score = intersection_over_union_thresholds(y_true, processed)
print(iou_score, iout_score)
def do_eval_with_pros(probs, dataset):
truths = np.zeros((len(dataset), 101, 101))
for i in range(len(dataset)):
index, image, y_mask, y_labels = dataset[i]
truths[i, :, :] = y_mask
iou = intersection_over_union_thresholds(np.int32(truths >= 0.5),
np.int32(probs >= 0.5))
return iou
def _get_validation_loss(self):
output, epoch_loss = self._transform()
y_pred = self._generate_prediction(output)
logger.info('Calculating IOU and IOUT Scores')
iou_score = intersection_over_union(self.y_true, y_pred)
iout_score = intersection_over_union_thresholds(self.y_true, y_pred)
logger.info('IOU score on validation is {}'.format(iou_score))
logger.info('IOUT score on validation is {}'.format(iout_score))
if not self.transformer.validation_loss:
self.transformer.validation_loss = {}
self.transformer.validation_loss.setdefault(
self.epoch_id, {
'sum': epoch_loss,
'iou': Variable(torch.Tensor([iou_score])),
'iout': Variable(torch.Tensor([iout_score]))
})
return self.transformer.validation_loss[self.epoch_id]
def validate(args,
model,
val_loader,
epoch=0,
threshold=0.5,
cls_threshold=0.5):
model.eval()
#print('validating...')
outputs = []
cls_preds = []
total_num = 0
cls_corrects = 0
focal_loss, lovaz_loss, bce_loss, ship_loss = 0, 0, 0, 0
with torch.no_grad():
for img, target, ship_target in val_loader:
img, target, ship_target = img.cuda(), target.cuda(
), ship_target.cuda()
#output, ship_out = model(img)
output, logit_pixel, ship_out = model(img)
#print(output.size(), salt_out.size())
ship_pred = (torch.sigmoid(ship_out) > cls_threshold).byte()
total_num += len(img)
cls_corrects += ship_pred.eq(ship_target.byte()).sum().item()
_, floss, lovaz, _bce_loss, _ship_loss = criterion(
args, (output, ship_out), (target, ship_target), epoch=epoch)
focal_loss += floss
lovaz_loss += lovaz
ship_loss += _ship_loss
bce_loss += _bce_loss
output = torch.sigmoid(output)
for o in output.cpu():
outputs.append(o.squeeze().numpy())
cls_preds.extend(ship_pred.cpu().squeeze().numpy().tolist())
n_batches = val_loader.num // args.batch_size if val_loader.num % args.batch_size == 0 else val_loader.num // args.batch_size + 1
# y_pred, list of 400 np array, each np array's shape is 101,101
y_pred = generate_preds(args, outputs, cls_preds,
(settings.ORIG_H, settings.ORIG_W), threshold)
#draw
if args.dev_mode:
for p, y in zip(y_pred, val_loader.y_true):
print(p.shape, y.shape)
#objs = split_mask(p, threshold_obj=30, threshold=None)
#if objs:
# if False:
# objs = map(lambda x: mask_to_bbox(x), objs)
# cv2.imshow('image', np.hstack([*objs, y])*255)
#else:
#bb_img = masks_to_bounding_boxes(p)
#bb_img = (bb_img > 0).astype(np.uint8)
#print(bb_img.max())
cv2.imshow('image', np.hstack([p, y]) * 255)
cv2.waitKey(0)
iou_score = intersection_over_union(val_loader.y_true, y_pred)
iout_score = intersection_over_union_thresholds(val_loader.y_true, y_pred)
#print('IOU score on validation is {:.4f}'.format(iou_score))
#print('IOUT score on validation is {:.4f}'.format(iout_score))
cls_acc = cls_corrects / total_num
return iout_score, iou_score, focal_loss / n_batches, lovaz_loss / n_batches, bce_loss / n_batches, ship_loss / n_batches, cls_acc
