def evaluate(model, valid_dataloader, ids_valid):
print("| -- Get Best Threshold from Validation Dataset -- |")
model.eval()
valid_preds = []
valid_masks = []
valid_indexs = []
for inputs, labels, indexs in tqdm(valid_dataloader):
inputs = cuda(inputs)
labels = cuda(labels)
logits = model(inputs) # no torch.sigmoid
logits = to_numpy(logits)
valid_preds.append(logits)
valid_masks.append(labels)
valid_indexs.append(list(indexs))
valid_preds = np.vstack(valid_preds)[:, 0, :, :]
valid_masks = np.vstack(valid_masks)[:, 0, :, :]
valid_preds = un_puzzle_pad(valid_preds)
valid_masks = un_puzzle_pad(valid_masks)
# all_valid_preds = np.vstack(all_valid_preds)
# all_valid_masks = np.vstack(all_valid_masks)
# all_valid_indexs = sum(all_valid_indexs, [])
# print(all_valid_preds.shape)
# print(all_valid_masks.shape)
# print(set(all_valid_indexs) - set(ids_trains))
best_threshold = get_best_threshold(valid_preds, valid_masks, ids_valid)
return best_threshold
def evaluate2(model, valid_dataloader, ids_valid):
print("| -- Get Best Threshold from Validation Dataset -- |")
model.eval()
valid_preds = []
valid_masks = []
valid_indexs = []
for inputs, labels, nemptys, indexs in tqdm(valid_dataloader):
inputs = cuda(inputs)
labels = cuda(labels)
logit_fuse, logit_pixel, logit_image = model(inputs)
logit_fuse = to_numpy(logit_fuse)
valid_preds.append(logit_fuse)
valid_masks.append(labels)
valid_indexs.append(list(indexs))
valid_preds = np.vstack(valid_preds)[:, 0, :, :]
valid_masks = np.vstack(valid_masks)[:, 0, :, :]
valid_preds = un_puzzle_pad(valid_preds)
valid_masks = un_puzzle_pad(valid_masks)
best_threshold = get_best_threshold(valid_preds, valid_masks, ids_valid)
return best_threshold
def train2(model, train_dataloader, epoch, num_epochs, optimizer):
model.train()
train_loss = []
train_iout = []
train_iout2 = []
train_iout3 = []
for batch_idx, (inputs, labels, nemptys,
indexs) in enumerate(train_dataloader):
inputs = cuda(inputs)
with torch.no_grad():
nemptys = Variable(nemptys).type(torch.cuda.FloatTensor)
labels = cuda(labels)
optimizer.zero_grad()
logit_fuse, logit_pixel, logit_image = model(inputs)
preds_fuse = torch.sigmoid(logit_fuse)
preds_image = torch.sigmoid(logit_image)
truth_pixel, truth_image = labels, nemptys
loss_fuse, loss_pixel, loss_image = dscriterion(
logit_fuse, logit_pixel, logit_image, truth_pixel, truth_image)
preds_image = preds_image.view(-1, 1, 1, 1)
# preds_pixel = torch.mul(logit_pixel, (preds_image > 0.5).float())
preds_fuse2 = torch.mul(preds_fuse, (preds_image > 0.5).float())
iout = get_iou_vector((logit_fuse > 0), labels) # TODO
# iout2 = get_iou_vector((preds_pixel > 0), labels)
iout2 = get_iou_vector((preds_fuse > 0.5), labels)
iout3 = get_iou_vector((preds_fuse2 > 0.5), labels)
loss = loss_fuse + loss_pixel + loss_image
loss.backward()
# loss_pixel.backward(retain_graph=True)
# loss_image.backward()
optimizer.step()
train_loss.append(to_item(loss))
train_iout.append(to_item(iout))
train_iout2.append(to_item(iout2))
train_iout3.append(to_item(iout3))
sys.stdout.write('\r')
sys.stdout.write(
f'| Epoch [{epoch:3d}/{num_epochs:3d}] '
f'Iter[{(batch_idx + 1):3d}/{len(train_dataloader):3d}]\t\t'
f'Loss: {np.mean(train_loss):.4f} '
f'{np.mean(train_iout):.4f} '
f'{np.mean(train_iout2):.4f} '
f'{np.mean(train_iout3):.4f}')
sys.stdout.flush()
print()
metrics = {
'train_loss': np.mean(train_loss),
'train_iout': np.mean(train_iout)
}
return metrics
def evaluate(model, valid_dataloader, epoch, criterion, is_finetune):
with torch.no_grad():
model.eval()
valid_loss = []
valid_iout = []
for inputs, labels, nemptys, indexs in valid_dataloader:
inputs = cuda(inputs)
labels = cuda(labels)
logits = model(inputs)
preds = torch.sigmoid(logits)
loss = criterion(logits, labels)
if not is_finetune:
iout = get_iou_vector((preds > 0.5), labels)
else:
iout = get_iou_vector((logits > 0), labels)
valid_loss.append(to_item(loss))
valid_iout.append(to_item(iout))
print(f'| Validation Epoch #{epoch}\t\t\t'
f'Valid Loss: {np.mean(valid_loss):.4f} '
f'Valid IoUT: {np.mean(valid_iout):.4f}')
metrics = {
'valid_loss': np.mean(valid_loss),
'valid_iout': np.mean(valid_iout)
}
return metrics
def evaluate2(model, valid_dataloader, epoch):
with torch.no_grad():
model.eval()
valid_loss = []
valid_iout = []
valid_iout2 = []
valid_iout3 = []
for inputs, labels, nemptys, indexs in valid_dataloader:
inputs = cuda(inputs)
nemptys = Variable(nemptys).type(torch.cuda.FloatTensor)
labels = cuda(labels)
logit_fuse, logit_pixel, logit_image = model(inputs)
preds_fuse = torch.sigmoid(logit_fuse)
preds_image = torch.sigmoid(logit_image)
truth_pixel, truth_image = labels, nemptys
loss_fuse, loss_pixel, loss_image = dscriterion(
logit_fuse, logit_pixel, logit_image, truth_pixel, truth_image)
preds_image = preds_image.view(-1, 1, 1, 1)
# preds_pixel = torch.mul(logit_pixel, (preds_image > 0.5).float())
preds_fuse2 = torch.mul(preds_fuse, (preds_image > 0.5).float())
iout = get_iou_vector((logit_fuse > 0), labels)
# iout2 = get_iou_vector((preds_pixel > 0), labels)
iout2 = get_iou_vector((preds_fuse > 0.5), labels)
iout3 = get_iou_vector((preds_fuse2 > 0.5), labels)
loss = loss_fuse + loss_pixel + loss_image
valid_loss.append(to_item(loss))
valid_iout.append(to_item(iout))
valid_iout2.append(to_item(iout2))
valid_iout3.append(to_item(iout3))
print(f'| Validation Epoch #{epoch}\t\t\t'
f'Valid Loss: {np.mean(valid_loss):.4f} '
f'Valid IoUT: {np.mean(valid_iout):.4f} '
f'Valid IoUT2: {np.mean(valid_iout2):.4f} '
f'Valid IoUT3: {np.mean(valid_iout3):.4f}')
metrics = {
'valid_loss': np.mean(valid_loss),
'valid_iout': np.mean(valid_iout)
}
return metrics
def train_fold(train_df, ids_trains, train_dir, batch_size, n_fold, phase):
# Create train/validation split stratified by salt coverage
skf = StratifiedKFold(n_splits=8, shuffle=True, random_state=1234)
for fold, (train_idx, valid_idx) in enumerate(
skf.split(ids_trains, train_df.coverage_class)):
ids_train, ids_valid = ids_trains[train_idx], ids_trains[valid_idx]
if n_fold != -1 and fold != n_fold:
continue
histall = histcoverage(train_df.coverage_class[ids_train].values)
histall_valid = histcoverage(train_df.coverage_class[ids_valid].values)
print(f"fold: {fold}\n"
f"train size: {len(ids_train)}\n"
f"number of each mask class: {histall}\n"
f"valid size: {len(ids_valid)}\n"
f"number of each mask class: {histall_valid}")
train_dataset = TGSSaltDataset(
train_dir, ids_train, transform, mode='train')
valid_dataset = TGSSaltDataset(
train_dir, ids_valid, None, mode='train')
train_dataloader = data.DataLoader(
train_dataset,
batch_size=batch_size,
shuffle=True,
drop_last=True,
num_workers=0,
pin_memory=True)
valid_dataloader = data.DataLoader(
valid_dataset,
batch_size=batch_size,
shuffle=False,
drop_last=False,
num_workers=0,
pin_memory=True)
# imshow_example(train_dataloader)
# exit()
model = unet_models.DeepSupervision50(pretrained=True)
print(sum(p.numel() for p in model.parameters()))
model = cuda(model)
if phase == "train2":
train_model2(model, train_dataloader, valid_dataloader, fold=fold)
elif phase == "train":
train_model(model, train_dataloader, valid_dataloader, fold=fold)
elif phase == "finetune":
finetune(model, train_dataloader, valid_dataloader, fold=fold)
elif phase == "finetune2":
finetune2(model, train_dataloader, valid_dataloader, fold=fold)
def predict(model, test_dataloader):
print("| -- Predict on Test Dataset -- |")
model.eval()
test_preds = []
test_indexs = []
for inputs, indexs in tqdm(test_dataloader):
inputs = cuda(inputs)
logits1 = model(inputs)
logits2 = flip_tensor_lr(model(flip_tensor_lr(inputs))) # TTA
logits = [logits1, logits2]
logits = torch.mean(torch.stack(logits, 0), 0)
logits = to_numpy(logits)
test_preds.append(logits)
test_indexs.append(indexs)
test_preds = np.vstack(test_preds)[:, 0, :, :]
test_preds = un_puzzle_pad(test_preds)
return test_preds
def infer(ids_test, test_dir, batch_size):
print(f"ids_test: {len(ids_test)}\n{ids_test[:10]}")
test_dataset = TGSSaltDataset(test_dir, ids_test, None, mode='test')
test_dataloader = data.DataLoader(
test_dataset,
batch_size=batch_size,
shuffle=False,
drop_last=False,
num_workers=0,
pin_memory=True)
def get_all_test_preds(model, best_model_paths, file_path):
if os.path.exists(file_path):
print(f"Load {file_path}")
all_test_preds = np.load(file_path)
return all_test_preds
n = len(best_model_paths)
all_test_preds = np.zeros((n, 18000, img_size_ori, img_size_ori))
for i, path in enumerate(best_model_paths):
model, _, _ = load_state(model, path)
# best_threshold = evaluate2(model, valid_dataloader, ids_valid)
test_preds = predict2(model, test_dataloader) # predict
all_test_preds[i] = test_preds
all_test_preds = all_test_preds.mean(axis=0)
np.save(file_path, all_test_preds)
return all_test_preds
model = unet_models.DeepSupervision50(pretrained=True)
model = cuda(model)
all_test_preds = []
# load save npys
best_mean_npys = [
"result/0929/finetune_meanfold.npy",
# "result/0929/finetune2_fold0.npy",
# "result/0929/finetune2_fold1.npy",
# "result/0929/finetune2_fold2.npy",
# "result/0929/finetune2_fold3.npy",
# "result/0929/finetune2_fold4.npy",
"result/0929/finetune2_01234.npy",
"result/1002/finetune_meanfold.npy",
# "result/1002/finetune2_fold0.npy",
# "result/1002/finetune2_fold1.npy",
# "result/1002/finetune2_fold2.npy",
# "result/1002/finetune2_fold3.npy",
# "result/1002/finetune2_fold4.npy",
# "result/1002/finetune2_fold5.npy",
# "result/1002/finetune2_fold6.npy",
# "result/1002/finetune2_fold8.npy",
# "result/1002/finetune2_fold9.npy",
"result/1002/finetune2_012345689.npy",
"result/1009/best_model_0.npy",
"result/1009/best_model_1.npy",
"result/1009/best_model_2.npy",
"result/1009/best_model_3.npy",
"result/1009/best_model_5.npy",
"result/1009/best_model_6.npy",
"result/1009/best_model_7.npy",
]
for npy in best_mean_npys:
test_preds = np.load(npy)
all_test_preds.append(test_preds)
# load and save train2 model
# best_model_paths = []
# for fold in [0, 1, 2, 3, 5, 6, 7]:
# best_model_paths.append(f"model/best_model_{fold}.pt")
# file_path = f"result/best_model_{fold}.npy"
# all_test_preds.append(
# get_all_test_preds(model, best_model_paths, file_path))
# load and save finetune model
# best_model_paths = []
# for fold in range(4):
# best_model_paths.append(f"model/best_model_{fold}_finetune.pt")
# file_path = "result/finetune_meanfold.npy"
# all_test_preds.append(
# get_all_test_preds(model, best_model_paths, file_path))
# load and save finetune2 model
# for fold in [4, 5]:
# best_model_paths = []
# for i in range(6):
# best_model_paths.append(
# f"model/best_model_{fold}_finetune2_{i}.pt")
# file_path = f"result/finetune2_fold{fold}.npy"
# all_test_preds.append(
# get_all_test_preds(model, best_model_paths, file_path))
# mean
all_test_preds = np.array(all_test_preds)
print(all_test_preds.shape)
all_test_preds = all_test_preds.mean(axis=0)
# np.save("result/1002/finetune2_012345689.npy", all_test_preds)
return all_test_preds
def train(model,
train_dataloader,
epoch,
num_epochs,
criterion,
optimizer,
is_finetune,
scheduler=None,
writer=None):
model.train()
# # do
# def set_bn_eval(model):
# for name, module in model.named_children():
# if isinstance(module, torch.nn.BatchNorm2d):
# module.eval()
# module.weight.requires_grad = False
# module.bias.requires_grad = False
# else:
# set_bn_eval(module)
# # do
# for name, module in model.named_children():
# if "encoder" in name:
# set_bn_eval(module)
train_loss = []
train_iout = []
for batch_idx, (inputs, labels, nemptys,
indexs) in enumerate(train_dataloader):
if scheduler is not None:
scheduler.step()
lr = optimizer.param_groups[0]['lr']
step = (epoch - 1) * len(train_dataloader) + batch_idx
writer.add_scalar('lr/lr', lr, step)
inputs = cuda(inputs)
with torch.no_grad():
labels = cuda(labels)
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
logits = model(inputs)
preds = torch.sigmoid(logits)
loss = criterion(logits, labels)
if not is_finetune:
iout = get_iou_vector((preds > 0.5), labels)
else:
iout = get_iou_vector((logits > 0), labels)
loss.backward()
optimizer.step()
train_loss.append(to_item(loss))
train_iout.append(to_item(iout))
sys.stdout.write('\r')
sys.stdout.write(
f'| Epoch [{epoch:3d}/{num_epochs:3d}] '
f'Iter[{(batch_idx + 1):3d}/{len(train_dataloader):3d}]\t\t'
f'Loss: {np.mean(train_loss):.4f} '
f'IoUT: {np.mean(train_iout):.4f}')
sys.stdout.flush()
print()
metrics = {
'train_loss': np.mean(train_loss),
'train_iout': np.mean(train_iout)
}
return metrics
