def test_crf(): loaded = np.load('/mnt/g/ship/tmp/val_out.npz') outputs = loaded['outputs'] y_true = loaded['y_true'] print(outputs.shape) print(y_true.shape) tgt_size = (settings.ORIG_H, settings.ORIG_W) resized = list(map(lambda x: resize(x, tgt_size), outputs)) print(resized[0].shape, len(resized)) y_pred = list(map(lambda x: (x > 0.5).astype(np.uint8), resized)) print(y_pred[0].shape, len(y_pred)) iou_score = intersection_over_union(y_true, y_pred) iout_score = intersection_over_union_thresholds(y_true, y_pred) print(iou_score, iout_score) _, val_meta = get_train_val_meta(True) img_ids = val_meta['ImageId'].values.tolist() crf_imgs = [] for i, img_id in enumerate(img_ids): orig_img = imread('/mnt/g/ship/train_v2/{}'.format(img_id)) crf_output = crf(orig_img, y_pred[i]) crf_imgs.append(crf_output) if i % 100 == 0: print(i) iou_score = intersection_over_union(y_true, crf_imgs) iout_score = intersection_over_union_thresholds(y_true, crf_imgs) print(iou_score, iout_score)
def do_eval(net, dataset): net.set_mode('eval') probs = np.zeros((len(dataset), 101, 101)) truths = np.zeros((len(dataset), 101, 101)) for i in range(len(dataset)): with torch.no_grad(): index, image, y_mask, _ = dataset[i] hflip_image = np.array(image)[:, ::-1] images = np.array([image, hflip_image]) images = torch.Tensor(images).cuda() logit = net(images) prob = logit.sigmoid() prob = prob.cpu().data.numpy().squeeze() mask = prob[0] hflip_mask = prob[1][:, ::-1] prob = (mask + hflip_mask) / 2 prob = prob[13:128 - 14, 13:128 - 14] probs[i, :, :] = prob truths[i, :, :] = y_mask iou = intersection_over_union_thresholds(np.int32(truths > 0.5), np.int32(probs > 0.5)) return iou
def _evaluate_pipeline(pipeline_name, validation_size): meta = pd.read_csv(os.path.join(params.meta_dir, 'stage1_metadata.csv')) meta_train = meta[meta['is_train'] == 1] valid_ids = eval(params.valid_category_ids) meta_train_split, meta_valid_split = train_valid_split(meta_train, validation_size, valid_category_ids=valid_ids) data = {'input': {'meta': meta_valid_split, 'meta_valid': None, 'train_mode': False, 'target_sizes': meta_valid_split[SIZE_COLUMNS].values }, } y_true = read_masks(meta_valid_split[Y_COLUMNS_SCORING].values) pipeline = PIPELINES[pipeline_name]['inference'](SOLUTION_CONFIG) pipeline.clean_cache() output = pipeline.transform(data) pipeline.clean_cache() y_pred = output['y_pred'] logger.info('Calculating IOU and IOUT Scores') iou_score = intersection_over_union(y_true, y_pred) logger.info('IOU score on validation is {}'.format(iou_score)) ctx.channel_send('IOU Score', 0, iou_score) iout_score = intersection_over_union_thresholds(y_true, y_pred) logger.info('IOUT score on validation is {}'.format(iout_score)) ctx.channel_send('IOUT Score', 0, iout_score)
def validate(args, model, val_loader, epoch=0, threshold=0.5): model.eval() outputs = [] focal_loss, lovaz_loss, salt_loss, w_loss = 0, 0, 0, 0 with torch.no_grad(): for img, target, salt_target in val_loader: if args.depths: add_depth_channel(img, args.pad_mode) img, target, salt_target = img.cuda(), target.cuda( ), salt_target.cuda() output, salt_out = model(img) _, floss, lovaz, _salt_loss, _w_loss = weighted_loss( args, (output, salt_out), (target, salt_target), epoch=epoch) focal_loss += floss lovaz_loss += lovaz salt_loss += _salt_loss w_loss += _w_loss output = torch.sigmoid(output) for o in output.cpu(): outputs.append(o.squeeze().numpy()) n_batches = val_loader.num // args.batch_size if val_loader.num % args.batch_size == 0 else val_loader.num // args.batch_size + 1 # y_pred, list of np array, each np array's shape is 101,101 y_pred = generate_preds(args, outputs, (settings.ORIG_H, settings.ORIG_W), threshold) iou_score = intersection_over_union(val_loader.y_true, y_pred) iout_score = intersection_over_union_thresholds(val_loader.y_true, y_pred) return iout_score, iou_score, focal_loss / n_batches, lovaz_loss / n_batches, salt_loss / n_batches, iout_score * 4 - w_loss
def do_eval(net, dataset, mode='padding'): net.set_mode('eval') probs = np.zeros((len(dataset), 101, 101)) truths = np.zeros((len(dataset), 101, 101)) for i in range(len(dataset)): with torch.no_grad(): index, image, y_mask, y_label = dataset[i] hflip_image = np.array(image)[:, ::-1] images = np.array([image, hflip_image]) images = torch.Tensor(images).cuda() logit_fuse, logit_pixel, logit_image = net(images) prob = logit_fuse.sigmoid() prob = prob.cpu().data.numpy().squeeze() mask = prob[0] hflip_mask = prob[1][:, ::-1] prob = (mask + hflip_mask) / 2 if mode == 'padding': prob = prob[13:128 - 14, 13:128 - 14] else: prob = cv2.resize(prob, (101, 101)) probs[i, :, :] = prob truths[i, :, :] = y_mask iou = intersection_over_union_thresholds(np.int32(truths >= 0.5), np.int32(probs >= 0.5)) return probs, iou
def validate(args, model, val_loader, epoch=0, threshold=0.5): model.eval() print('validating...') outputs = [] val_loss = 0 with torch.no_grad(): for img, target, salt_target in val_loader: img, target, salt_target = img.cuda(), target.cuda( ), salt_target.cuda() output, salt_out = model(img) #print(output.size(), salt_out.size()) loss = weighted_loss((output, salt_out), (target, salt_target), epoch=epoch) val_loss += loss.item() output = torch.sigmoid(output) for o in output.cpu(): outputs.append(o.squeeze().numpy()) n_batches = val_loader.num // args.batch_size if val_loader.num % args.batch_size == 0 else val_loader.num // batch_size + 1 # y_pred, list of 400 np array, each np array's shape is 101,101 y_pred = generate_preds_softmax(outputs, (settings.ORIG_H, settings.ORIG_W), threshold) print(y_pred[0].shape) print('Validation loss: {:.4f}'.format(val_loss / n_batches)) iou_score = intersection_over_union(val_loader.y_true, y_pred) iout_score = intersection_over_union_thresholds(val_loader.y_true, y_pred) print('IOU score on validation is {:.4f}'.format(iou_score)) print('IOUT score on validation is {:.4f}'.format(iout_score)) return iout_score, iou_score, val_loss / n_batches
def test_bbox(): tgt_size = (settings.ORIG_H, settings.ORIG_W) outputs, y_true = get_val_result(16) resized = list(map(lambda x: resize_image(x, tgt_size), outputs)) print(resized[0].shape, len(resized)) y_pred = list(map(lambda x: (x > 0.5).astype(np.uint8), resized)) print(y_pred[0].shape, len(y_pred)) iou_score = intersection_over_union(y_true, y_pred) iout_score = intersection_over_union_thresholds(y_true, y_pred) print(iou_score, iout_score) processed = list(map(lambda x: masks_to_bounding_boxes(x), y_pred)) iou_score = intersection_over_union(y_true, processed) iout_score = intersection_over_union_thresholds(y_true, processed) print(iou_score, iout_score)
def do_eval_with_pros(probs, dataset): truths = np.zeros((len(dataset), 101, 101)) for i in range(len(dataset)): index, image, y_mask, y_labels = dataset[i] truths[i, :, :] = y_mask iou = intersection_over_union_thresholds(np.int32(truths >= 0.5), np.int32(probs >= 0.5)) return iou
def _get_validation_loss(self): output, epoch_loss = self._transform() y_pred = self._generate_prediction(output) logger.info('Calculating IOU and IOUT Scores') iou_score = intersection_over_union(self.y_true, y_pred) iout_score = intersection_over_union_thresholds(self.y_true, y_pred) logger.info('IOU score on validation is {}'.format(iou_score)) logger.info('IOUT score on validation is {}'.format(iout_score)) if not self.transformer.validation_loss: self.transformer.validation_loss = {} self.transformer.validation_loss.setdefault( self.epoch_id, { 'sum': epoch_loss, 'iou': Variable(torch.Tensor([iou_score])), 'iout': Variable(torch.Tensor([iout_score])) }) return self.transformer.validation_loss[self.epoch_id]
def validate(args, model, val_loader, epoch=0, threshold=0.5, cls_threshold=0.5): model.eval() #print('validating...') outputs = [] cls_preds = [] total_num = 0 cls_corrects = 0 focal_loss, lovaz_loss, bce_loss, ship_loss = 0, 0, 0, 0 with torch.no_grad(): for img, target, ship_target in val_loader: img, target, ship_target = img.cuda(), target.cuda( ), ship_target.cuda() #output, ship_out = model(img) output, logit_pixel, ship_out = model(img) #print(output.size(), salt_out.size()) ship_pred = (torch.sigmoid(ship_out) > cls_threshold).byte() total_num += len(img) cls_corrects += ship_pred.eq(ship_target.byte()).sum().item() _, floss, lovaz, _bce_loss, _ship_loss = criterion( args, (output, ship_out), (target, ship_target), epoch=epoch) focal_loss += floss lovaz_loss += lovaz ship_loss += _ship_loss bce_loss += _bce_loss output = torch.sigmoid(output) for o in output.cpu(): outputs.append(o.squeeze().numpy()) cls_preds.extend(ship_pred.cpu().squeeze().numpy().tolist()) n_batches = val_loader.num // args.batch_size if val_loader.num % args.batch_size == 0 else val_loader.num // args.batch_size + 1 # y_pred, list of 400 np array, each np array's shape is 101,101 y_pred = generate_preds(args, outputs, cls_preds, (settings.ORIG_H, settings.ORIG_W), threshold) #draw if args.dev_mode: for p, y in zip(y_pred, val_loader.y_true): print(p.shape, y.shape) #objs = split_mask(p, threshold_obj=30, threshold=None) #if objs: # if False: # objs = map(lambda x: mask_to_bbox(x), objs) # cv2.imshow('image', np.hstack([*objs, y])*255) #else: #bb_img = masks_to_bounding_boxes(p) #bb_img = (bb_img > 0).astype(np.uint8) #print(bb_img.max()) cv2.imshow('image', np.hstack([p, y]) * 255) cv2.waitKey(0) iou_score = intersection_over_union(val_loader.y_true, y_pred) iout_score = intersection_over_union_thresholds(val_loader.y_true, y_pred) #print('IOU score on validation is {:.4f}'.format(iou_score)) #print('IOUT score on validation is {:.4f}'.format(iout_score)) cls_acc = cls_corrects / total_num return iout_score, iou_score, focal_loss / n_batches, lovaz_loss / n_batches, bce_loss / n_batches, ship_loss / n_batches, cls_acc