def train_epoch(current_epoch, seg_loss, ce_loss, seg_seesaw, model, optimizer, scheduler, train_data_loader): losses = AverageMeter() losses1 = AverageMeter() dices = AverageMeter() iterator = tqdm(train_data_loader) model.train() for i, sample in enumerate(iterator): imgs = sample["img"].cuda(non_blocking=True) msks = sample["msk"].cuda(non_blocking=True) lbl_msk = sample["lbl_msk"].cuda(non_blocking=True) out = model(imgs) # loss0 = seg_loss(out[:, 0, ...], msks[:, 0, ...]) # loss1 = seg_loss(out[:, 1, ...], msks[:, 1, ...]) # loss2 = seg_loss(out[:, 2, ...], msks[:, 2, ...]) # loss3 = seg_loss(out[:, 3, ...], msks[:, 3, ...]) # loss4 = seg_loss(out[:, 4, ...], msks[:, 4, ...]) loss5 = ce_loss(out, lbl_msk) #loss5 = seg_seesaw(out, lbl_msk) loss = loss5 #loss = 0.1 * loss0 + 0.1 * loss1 + 0.3 * loss2 + 0.3 * loss3 + 0.2 * loss4 + loss5 * 2 with torch.no_grad(): _probs = 1 - torch.sigmoid(out[:, 0, ...]) dice_sc = 1 - dice_round(_probs, 1 - msks[:, 0, ...]) losses.update(loss.item(), imgs.size(0)) losses1.update(loss5.item(), imgs.size(0)) dices.update(dice_sc, imgs.size(0)) iterator.set_description( "epoch: {}; lr {:.7f}; Loss {loss.val:.4f} ({loss.avg:.4f}); cce_loss {loss1.val:.4f} ({loss1.avg:.4f}); Dice {dice.val:.4f} ({dice.avg:.4f})" .format(current_epoch, scheduler.get_lr()[-1], loss=losses, loss1=losses1, dice=dices)) optimizer.zero_grad() # loss.backward() with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), 0.999) optimizer.step() scheduler.step() print( "epoch: {}; lr {:.7f}; Loss {loss.avg:.4f}; CCE_loss {loss1.avg:.4f}; Dice {dice.avg:.4f}" .format(current_epoch, scheduler.get_lr()[-1], loss=losses, loss1=losses1, dice=dices))
def validate(net, data_loader, predictions_dir, visualizer_path, data_dir, truth_csv="oof_gt.txt"): os.makedirs(predictions_dir, exist_ok=True) dices = [] with torch.no_grad(): for sample in tqdm(data_loader): imgs = sample["image"].cuda().float() mask = sample["mask"].cuda().float() output = net(imgs) pred = torch.sigmoid(output) d = dice_round(pred[:, 10:11, ...], mask[:, 10:11, ...], t=0.5).cpu().numpy() for i in range(d.shape[0]): dices.append(d[i]) cv2.imwrite( os.path.join( predictions_dir, sample["img_name"][i][:-4].replace("MS", "RGB") + ".png"), pred[i, 10].cpu().numpy() * 255) pred_csv = os.path.join( Path(predictions_dir).parent, os.path.basename(predictions_dir) + ".txt") vectorize_dir(predictions_dir, pred_csv) apls = calculate_visualizer(visualizer_path, truth_csv=truth_csv, pred_path=pred_csv, img_dir=data_dir) return np.mean(dices), apls
def validate(net, data_loader, predictions_dir, data_dir, visualizer_path): os.makedirs(predictions_dir, exist_ok=True) preds_dir = predictions_dir + "/predictions" os.makedirs(preds_dir, exist_ok=True) dices = [] with torch.no_grad(): for sample in tqdm(data_loader): imgs = sample["image"].cuda().float() mask = sample["mask"].cuda().float() output = net(imgs) binary_pred = torch.sigmoid(output) for i in range(output.shape[0]): d = dice_round(binary_pred[:, 0:1, :], mask[:, 0:1, ...], t=0.5).item() dices.append(d) cv2.imwrite(os.path.join(preds_dir, sample["img_name"][i] + ".png"), (np.moveaxis(binary_pred[i].cpu().numpy(), 0, -1)[..., :3] * 255)) f_score = calculate_metrics(fold_dir=predictions_dir, visualizer_path=visualizer_path, truth_csv=os.path.join(data_dir, "SummaryData/SN6_Train_AOI_11_Rotterdam_Buildings.csv"), img_dir=os.path.join(data_dir, "SAR-Intensity/"), sar_orientations_csv=os.path.join(data_dir, "SummaryData/SAR_orientations.txt")) return np.mean(dices), f_score
def train_epoch(current_epoch, seg_loss, ce_loss, mse_loss, model, optimizer, scheduler, train_data_loader): losses = AverageMeter() losses1 = AverageMeter() losses2 = AverageMeter() losses3 = AverageMeter() losses4 = AverageMeter() dices = AverageMeter() iterator = tqdm(train_data_loader) model.train() scheduler.step(current_epoch) for i, sample in enumerate(iterator): imgs = sample["img"].cuda(non_blocking=True) msks = sample["msk"].cuda(non_blocking=True) msks_speed = sample["msk_speed"].cuda(non_blocking=True) lbls_speed = sample["lbl_speed"].cuda(non_blocking=True) msks_speed_cont = sample["msk_speed_cont"].cuda(non_blocking=True) out = model(imgs) loss1 = seg_loss(out[:, 0, ...], msks[:, 0, ...]) loss2 = seg_loss(out[:, 1, ...], msks[:, 1, ...]) loss3 = ce_loss(out[:, 3:, ...], lbls_speed) loss4 = mse_loss(out[:, 2:3, ...], msks_speed_cont) loss = 1.2 * loss1 + 0.05 * loss2 + 0.2 * loss3 + 0.1 * loss4 for _i in range(3, 13): loss += 0.03 * seg_loss(out[:, _i, ...], msks_speed[:, _i-3, ...]) with torch.no_grad(): _probs = torch.sigmoid(out[:, 0, ...]) dice_sc = 1 - dice_round(_probs, msks[:, 0, ...]) losses.update(loss.item(), imgs.size(0)) losses1.update(loss1.item(), imgs.size(0)) losses2.update(loss2.item(), imgs.size(0)) losses3.update(loss3.item(), imgs.size(0)) losses4.update(loss4.item(), imgs.size(0)) dices.update(dice_sc, imgs.size(0)) iterator.set_description( "epoch: {}; lr {:.7f}; Loss {loss.val:.4f} ({loss.avg:.4f}); Loss1 {loss1.val:.4f} ({loss1.avg:.4f}); Loss2 {loss2.val:.4f} ({loss2.avg:.4f}); Loss3 {loss3.val:.4f} ({loss3.avg:.4f}); Loss4 {loss4.val:.4f} ({loss4.avg:.4f}); Dice {dice.val:.4f} ({dice.avg:.4f})".format( current_epoch, scheduler.get_lr()[-1], loss=losses, loss1=losses1, loss2=losses2, loss3=losses3, loss4=losses4, dice=dices)) optimizer.zero_grad() loss.backward() # with amp.scale_loss(loss, optimizer) as scaled_loss: # scaled_loss.backward() optimizer.step() # scheduler.step() print("epoch: {}; lr {:.7f}; Loss {loss.avg:.4f}; Loss1 {loss1.avg:.4f}; Loss2 {loss2.avg:.4f}; Loss3 {loss3.avg:.4f}; Loss4 {loss4.avg:.4f}; Dice {dice.avg:.4f}".format( current_epoch, scheduler.get_lr()[-1], loss=losses, loss1=losses1, loss2=losses2, loss3=losses3, loss4=losses4, dice=dices))
def train_epoch(current_epoch, loss_functions, model, optimizer, scheduler, train_data_loader, summary_writer, conf, local_rank): losses = AverageMeter() dices = AverageMeter() iterator = tqdm(train_data_loader) model.train() if conf["optimizer"]["schedule"]["mode"] == "epoch": scheduler.step(current_epoch) for i, sample in enumerate(iterator): imgs = sample["image"].cuda()[:, :3, :, :] masks = sample["mask"].cuda().float() out_mask = model(imgs) mask_band = 4 with torch.no_grad(): pred = torch.sigmoid(out_mask) d = dice_round(pred, masks[:, mask_band:, ...], t=0.5).item() dices.update(d, imgs.size(0)) mask_loss = loss_functions["mask_loss"](out_mask, masks[:, mask_band:, ...].contiguous()) loss = mask_loss losses.update(loss.item(), imgs.size(0)) iterator.set_description( "epoch: {}; lr {:.7f}; Loss ({loss.avg:.4f}); dice ({dice.avg:.4f}); " .format(current_epoch, scheduler.get_lr()[-1], loss=losses, dice=dices)) optimizer.zero_grad() if conf['fp16']: with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() else: loss.backward() torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), 1) optimizer.step() torch.cuda.synchronize() if conf["optimizer"]["schedule"]["mode"] in ("step", "poly"): scheduler.step(i + current_epoch * len(train_data_loader)) if local_rank == 0: for idx, param_group in enumerate(optimizer.param_groups): lr = param_group['lr'] summary_writer.add_scalar('group{}/lr'.format(idx), float(lr), global_step=current_epoch) summary_writer.add_scalar('train/loss', float(losses.avg), global_step=current_epoch)
def train_epoch(current_epoch, loss_function, l1_loss, model, optimizer, scheduler, train_data_loader): losses = AverageMeter() losses2 = AverageMeter() losses3 = AverageMeter() losses4 = AverageMeter() nadir_losses = AverageMeter() dices = AverageMeter() iterator = tqdm(train_data_loader) model.train() scheduler.step(current_epoch) for i, sample in enumerate(iterator): imgs = sample["img"].cuda(non_blocking=True) cat_inp = sample["cat_inp"].cuda(non_blocking=True) coord_inp = sample["coord_inp"].cuda(non_blocking=True) masks = sample["mask"].cuda(non_blocking=True) nadir = sample["nadir"].cuda(non_blocking=True) out, nadir_pred = model(imgs, nadir, cat_inp, coord_inp) loss1 = loss_function(out[:, 0, ...], masks[:, 0, ...]) loss2 = loss_function(out[:, 1, ...], masks[:, 1, ...]) loss3 = loss_function(out[:, 2, ...], masks[:, 2, ...]) loss4 = loss_function(out[:, 3, ...], masks[:, 3, ...]) nadir_loss = l1_loss(nadir_pred, nadir) loss = loss1 + 0.4 * loss2 + 0.05 * loss3 + 0.005 * loss4 + 0.002 * nadir_loss # with torch.no_grad(): _probs = torch.sigmoid(out[:, 0, ...]) dice_sc = 1 - dice_round(_probs, masks[:, 0, ...]) losses.update(loss1.item(), imgs.size(0)) losses2.update(loss2.item(), imgs.size(0)) losses3.update(loss3.item(), imgs.size(0)) losses4.update(loss4.item(), imgs.size(0)) nadir_losses.update(nadir_loss.item(), imgs.size(0)) dices.update(dice_sc, imgs.size(0)) iterator.set_description( "epoch: {}; lr {:.7f}; Loss {loss.val:.4f} ({loss.avg:.4f}); Loss2 {loss2.val:.4f} ({loss2.avg:.4f}); Loss3 {loss3.val:.4f} ({loss3.avg:.4f}); Loss4 {loss4.val:.4f} ({loss4.avg:.4f}); Dice {dice.val:.4f} ({dice.avg:.4f}); Nadir {nadir_loss.val:.4f} ({nadir_loss.avg:.4f})".format( current_epoch, scheduler.get_lr()[-1], loss=losses, loss2=losses2, loss3=losses3, loss4=losses4, dice=dices, nadir_loss=nadir_losses)) optimizer.zero_grad() loss.backward() optimizer.step() print("epoch: {}; lr {:.7f}; Loss {loss.avg:.4f}; Loss2 {loss2.avg:.4f}; Loss3 {loss3.avg:.4f}; Loss4 {loss4.avg:.4f}; Dice {dice.avg:.4f}; Nadir {nadir_loss.avg:.4f}".format( current_epoch, scheduler.get_lr()[-1], loss=losses, loss2=losses2, loss3=losses3, loss4=losses4, dice=dices, nadir_loss=nadir_losses))
def train_epoch(current_epoch, seg_loss, model, optimizer, scheduler, train_data_loader): losses = AverageMeter() dices = AverageMeter() iterator = tqdm(train_data_loader) model.train() for i, sample in enumerate(iterator): imgs = sample["img"].cuda(non_blocking=True) msks = sample["msk"].cuda(non_blocking=True) out = model(imgs) loss = seg_loss(out, msks) with torch.no_grad(): _probs = torch.sigmoid(out[:, 0, ...]) dice_sc = 1 - dice_round(_probs, msks[:, 0, ...]) losses.update(loss.item(), imgs.size(0)) dices.update(dice_sc, imgs.size(0)) iterator.set_description( "epoch: {}; lr {:.7f}; Loss {loss.val:.4f} ({loss.avg:.4f}); Dice {dice.val:.4f} ({dice.avg:.4f})" .format(current_epoch, scheduler.get_lr()[-1], loss=losses, dice=dices)) optimizer.zero_grad() # loss.backward() with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), 1.1) optimizer.step() scheduler.step(current_epoch) print("epoch: {}; lr {:.7f}; Loss {loss.avg:.4f}; Dice {dice.avg:.4f}". format(current_epoch, scheduler.get_lr()[-1], loss=losses, dice=dices))
def validate(net, data_loader, predictions_dir): os.makedirs(predictions_dir, exist_ok=True) preds_dir = predictions_dir + "/predictions" os.makedirs(preds_dir, exist_ok=True) dices = [] with torch.no_grad(): for sample in tqdm(data_loader): imgs = sample["image"].cuda().float()[:, :3, :, :] mask = sample["mask"].cuda().float() output = net(imgs) binary_pred = torch.sigmoid(output) for i in range(output.shape[0]): d = dice_round(binary_pred, mask[:, 4:, ...], t=0.5).item() dices.append(d) cv2.imwrite( os.path.join( preds_dir, "test_localization_" + sample["img_name"][i] + "_prediction.png"), (binary_pred[i, 0].cpu().numpy() > 0.5) * 1) return np.mean(dices)
def train_epoch(current_epoch, combo_loss, model, optimizer, scaler, train_data_loader): losses = AverageMeter() losses2 = AverageMeter() losses3 = AverageMeter() dices = AverageMeter() dices2 = AverageMeter() dices3 = AverageMeter() if args.local_rank == 0: iterator = tqdm(train_data_loader) else: iterator = train_data_loader model.train() _lr = optimizer.param_groups[0]['lr'] for i, sample in enumerate(iterator): with torch.cuda.amp.autocast(): imgs = sample["img"].cuda(non_blocking=True) otps = sample["msk"].cuda(non_blocking=True) res = model(imgs) loss1 = combo_loss(res[:, 0, ...], otps[:, 0, ...]) loss2 = combo_loss(res[:, 1, ...], otps[:, 1, ...]) loss3 = combo_loss(res[:, 2, ...], otps[:, 2, ...]) loss = loss2 + loss1 * 0.25 + 0.25 * loss3 with torch.no_grad(): _probs = torch.sigmoid(res[:, 0, ...]) dice_sc = 1 - dice_round(_probs, otps[:, 0, ...]) _probs = torch.sigmoid(res[:, 1, ...]) dice_sc2 = 1 - dice_round(_probs, otps[:, 1, ...]) _probs = torch.sigmoid(res[:, 2, ...]) dice_sc3 = 1 - dice_round(_probs, otps[:, 2, ...]) if i % 5 == 0: if args.distributed: reduced_loss1 = reduce_tensor(loss1.data) reduced_loss2 = reduce_tensor(loss2.data) reduced_loss3 = reduce_tensor(loss3.data) reduced_dice = reduce_tensor(dice_sc) reduced_dice2 = reduce_tensor(dice_sc2) reduced_dice3 = reduce_tensor(dice_sc3) else: reduced_loss1 = loss1.data reduced_loss2 = loss2.data reduced_loss3 = loss3.data reduced_dice = dice_sc reduced_dice2 = dice_sc reduced_dice3 = dice_sc # to_python_float incurs a host<->device sync losses.update(to_python_float(reduced_loss1), imgs.size(0)) losses2.update(to_python_float(reduced_loss2), imgs.size(0)) losses3.update(to_python_float(reduced_loss3), imgs.size(0)) dices.update(reduced_dice, imgs.size(0)) dices2.update(reduced_dice2, imgs.size(0)) dices3.update(reduced_dice3, imgs.size(0)) if args.local_rank == 0: iterator.set_description( "epoch: {}; lr {:.7f}; Loss {loss.val:.4f} ({loss.avg:.4f}); Loss2 {loss2.val:.4f} ({loss2.avg:.4f}); Loss3 {loss3.val:.4f} ({loss3.avg:.4f}); dice {dices.val:.4f} ({dices.avg:.4f}); dice2 {dices2.val:.4f} ({dices2.avg:.4f}); dice3 {dices3.val:.4f} ({dices3.avg:.4f});".format( current_epoch, _lr, loss=losses, loss2=losses2, loss3=losses3, dices=dices, dices2=dices2, dices3=dices3)) optimizer.zero_grad() scaler.scale(loss).backward() scaler.unscale_(optimizer) torch.nn.utils.clip_grad_norm_(model.parameters(), 1.5) scaler.step(optimizer) scaler.update() torch.cuda.synchronize() if args.local_rank == 0: print("epoch: {}; lr {:.7f}; Loss {loss.avg:.4f}; Loss2 {loss2.avg:.4f}; Loss3 {loss3.avg:.4f}; Dice {dices.avg:.4f}; Dice2 {dices2.avg:.4f}; Dice3 {dices3.avg:.4f}".format( current_epoch, _lr, loss=losses, loss2=losses2, loss3=losses3, dices=dices, dices2=dices2, dices3=dices3))