def main(argv=None):
image_dir = '/mnt/nas/data/denoise/LRDE/image/'
mask_dir = '/mnt/nas/data/denoise/LRDE/mask/'
overlap = 30. / 100. # 30. / 100. -> 317,750
imgh = 256
imgw = 256
image_save_dir = '/data/denoise/LRDE/image_patches'
mask_save_dir = '/data/denoise/LRDE/mask_patches'
os.makedirs(image_save_dir, exist_ok=True)
os.makedirs(mask_save_dir, exist_ok=True)
image_pathes = os.listdir(image_dir)
for image_path in image_pathes:
if not check_is_image(image_path):
print('not image', image_path)
continue
image_name = image_path.split('.')[0]
mask = cv2.imread(mask_dir + image_path, cv2.IMREAD_GRAYSCALE)
image = cv2.imread(image_dir + image_path)
print('processing the image:', image_path)
image_patches, _ = get_image_patch(image,
imgh,
imgw,
overlap=overlap,
is_mask=False)
mask_patches, poslist = get_image_patch(mask,
imgh,
imgw,
overlap=overlap,
is_mask=True)
print('get patches: %d' % len(image_patches))
for idx in range(len(image_patches)):
img_color = image_patches[idx]
img_gray = cv2.cvtColor(img_color, cv2.COLOR_BGR2GRAY)
mask_gray = mask_patches[idx]
img_color_tmp = img_color
mask_gray_tmp = mask_gray
cv2.imwrite('%s/%s_i%dh0.png' % (image_save_dir, image_name, idx),
img_color_tmp)
cv2.imwrite('%s/%s_i%dh0.png' % (mask_save_dir, image_name, idx),
mask_gray_tmp)
# horizontal axis
img_color_tmp = np.flipud(img_color)
mask_gray_tmp = np.flipud(mask_gray)
cv2.imwrite('%s/%s_i%dh1.png' % (image_save_dir, image_name, idx),
img_color_tmp)
cv2.imwrite('%s/%s_i%dh1.png' % (mask_save_dir, image_name, idx),
mask_gray_tmp)
def unet_train(epochs, gpu, base_model_name, encoder_weights, generator_lr,
discriminator_lr, lambda_bce, threshold, batch_size,
image_train_dir, mask_train_dir, original_dir, fold_num,
fold_total):
# make save directory
weight_path = (
'./step1_label%d_' % fold_num) + base_model_name + '_' + str(
int(lambda_bce)) + '_' + str(generator_lr) + '_' + str(threshold)
image_save_path = weight_path + '/images'
os.makedirs(weight_path, exist_ok=True)
os.makedirs(image_save_path, exist_ok=True)
# rgb , preprocess input
imagenet_mean = np.array([0.485, 0.456, 0.406])
imagenet_std = np.array([0.229, 0.224, 0.225])
train_data_set = Dataset_Return_Four(image_train_dir,
mask_train_dir,
os.path.join(original_dir, 'image'),
base_model_name,
encoder_weights,
threshold=threshold,
is_train=True,
fold_num=fold_num,
fold_total=fold_total)
train_loader = DataLoader(train_data_set,
batch_size=batch_size,
num_workers=4,
shuffle=True)
device = torch.device("cuda:%s" % gpu)
image_test_list = []
test_image_pathes = os.listdir(os.path.join(original_dir, 'image'))
for test_image_path in test_image_pathes:
if not check_is_image(test_image_path):
print(test_image_path, 'not image')
continue
image_test_list.append(
(os.path.join(original_dir, 'image', test_image_path),
os.path.join(original_dir, 'mask', test_image_path)))
image_test_list = image_test_list[fold_num::fold_total]
print('test len:', len(image_test_list))
preprocess_input = get_preprocessing_fn(base_model_name,
pretrained=encoder_weights)
models = []
optimizers = []
for channel in range(4):
models.append(
smp.Unet(base_model_name,
encoder_weights=encoder_weights,
in_channels=3))
models[channel].to(device)
optimizers.append(
optim.Adam(models[channel].parameters(),
lr=generator_lr,
betas=(0.5, 0.999)))
discriminator = Discriminator(in_channels=4)
discriminator.to(device)
optimizer_discriminator = optim.Adam(discriminator.parameters(),
lr=discriminator_lr,
betas=(0.5, 0.999))
criterion = nn.BCEWithLogitsLoss()
channel_dict = {0: 'blue', 1: 'green', 2: 'red', 3: 'gray'}
value = int(256 * 0.5)
best_fmeasure = 0.0
lambda_gp = 10.0
epoch_start_time = time.time()
for epoch in range(epochs):
# train
for channel in range(4):
models[channel].train()
models[channel].requires_grad_(False)
for idx, (images, masks) in enumerate(train_loader):
# for sample image
test_masks_pred_list = []
test_masks_list = []
test_images_list = []
for channel in range(4):
images[channel] = images[channel].to(device)
masks[channel] = masks[channel].to(device)
models[channel].requires_grad_(True)
masks_pred = models[channel](images[channel])
# discriminator
discriminator.requires_grad_(True)
# Fake
fake_AB = torch.cat((images[channel], masks_pred), 1).detach()
pred_fake = discriminator(fake_AB)
# Real
real_AB = torch.cat((images[channel], masks[channel]), 1)
pred_real = discriminator(real_AB)
gradient_penalty = compute_gradient_penalty(
discriminator, real_AB, fake_AB, device)
discriminator_loss = -torch.mean(pred_real) + torch.mean(
pred_fake) + lambda_gp * gradient_penalty
optimizer_discriminator.zero_grad()
discriminator_loss.backward()
optimizer_discriminator.step()
discriminator.requires_grad_(False)
# end discriminator
# generator
fake_AB = torch.cat((images[channel], masks_pred), 1)
pred_fake = discriminator(fake_AB)
generator_loss = -torch.mean(pred_fake)
bce_loss = criterion(masks_pred, masks[channel])
total_loss = generator_loss + bce_loss * lambda_bce
optimizers[channel].zero_grad()
total_loss.backward()
optimizers[channel].step()
models[channel].requires_grad_(False)
# end generator
if idx % 2000 == 0:
print(
'channel %s train step[%d/%d] discriminator loss: %.5f, total loss: %.5f, generator loss: %.5f, bce loss: %.5f, time: %.2f'
% (channel_dict[channel], idx, len(train_loader),
discriminator_loss.item(), total_loss.item(),
generator_loss.item(), bce_loss.item(),
time.time() - epoch_start_time))
# for sample images
rand_idx_start = randrange(masks[channel].size(0) - 2)
rand_idx_end = rand_idx_start + 2
test_masks_pred = torch.sigmoid(
masks_pred[rand_idx_start:rand_idx_end]).detach().cpu(
)
test_masks_pred = test_masks_pred.permute(
0, 2, 3, 1).numpy().astype(np.float32)
test_masks_pred = np.squeeze(test_masks_pred, axis=-1)
test_masks_pred_list.extend(test_masks_pred)
test_masks = masks[
channel][rand_idx_start:rand_idx_end].permute(
0, 2, 3, 1).cpu().numpy().astype(np.float32)
test_masks = np.squeeze(test_masks, axis=-1)
test_masks_list.extend(test_masks)
test_images = images[channel][
rand_idx_start:rand_idx_end].permute(0, 2, 3,
1).cpu().numpy()
test_images = test_images * imagenet_std + imagenet_mean
test_images = np.maximum(test_images, 0.0)
test_images = np.minimum(test_images, 1.0)
test_images_list.extend(test_images)
if idx % 2000 == 0:
sample_images(epoch, idx, test_images_list, test_masks_list,
test_masks_pred_list, image_save_path)
# break
# eval
for channel in range(4):
models[channel].eval()
total_fmeasure = 0.0
total_image_number = 0
for eval_idx, (image_test, mask_test) in enumerate(image_test_list):
image = cv2.imread(image_test)
h, w, _ = image.shape
image_name = image_test.split('/')[-1].split('.')[0]
# print('eval the image:', image_name)
gt_mask = cv2.imread(mask_test, cv2.IMREAD_GRAYSCALE)
gt_mask = np.expand_dims(gt_mask, axis=-1)
image_patches, poslist = get_image_patch(image,
256,
256,
overlap=0.5,
is_mask=False)
# random_number = randrange(10)
for channel in range(4):
color_patches = []
for patch in image_patches:
tmp = patch.astype(np.float32)
if channel != 3:
color_patches.append(
preprocess_input(tmp[:, :, channel:channel + 1]))
else:
color_patches.append(
preprocess_input(
np.expand_dims(cv2.cvtColor(
tmp, cv2.COLOR_BGR2GRAY),
axis=-1)))
step = 0
preds = []
with torch.no_grad():
while step < len(image_patches):
ps = step
pe = step + batch_size
if pe >= len(image_patches):
pe = len(image_patches)
target = torch.from_numpy(
np.array(color_patches[ps:pe])).permute(
0, 3, 1, 2).float()
pred = torch.sigmoid(models[channel](
target.to(device))).cpu()
preds.extend(pred)
step += batch_size
# handling overlap
out_img = np.ones((h, w, 1)) * 255
for i in range(len(image_patches)):
patch = preds[i].permute(1, 2, 0).numpy() * 255
start_h, start_w, end_h, end_w, h_shift, w_shift = poslist[
i]
h_cut = end_h - start_h
w_cut = end_w - start_w
tmp = np.minimum(
out_img[start_h:end_h, start_w:end_w],
patch[h_shift:h_shift + h_cut,
w_shift:w_shift + w_cut])
out_img[start_h:end_h, start_w:end_w] = tmp
out_img = out_img.astype(np.uint8)
out_img[out_img > value] = 255
out_img[out_img <= value] = 0
# if random_number == 0:
# cv2.imwrite('%s/%d_%d_%s.png' % (image_save_path, epoch, channel, image_name), out_img)
# f_measure
# background 1, text 0
gt_mask[gt_mask > 0] = 1
out_img[out_img > 0] = 1
# true positive
tp = np.zeros(gt_mask.shape, np.uint8)
tp[(out_img == 0) & (gt_mask == 0)] = 1
numtp = tp.sum()
# false positive
fp = np.zeros(gt_mask.shape, np.uint8)
fp[(out_img == 0) & (gt_mask == 1)] = 1
numfp = fp.sum()
# false negative
fn = np.zeros(gt_mask.shape, np.uint8)
fn[(out_img == 1) & (gt_mask == 0)] = 1
numfn = fn.sum()
precision = numtp / float(numtp + numfp)
recall = numtp / float(numtp + numfn)
fmeasure = 100. * (2. * recall * precision) / (
recall + precision) # percent
total_fmeasure += fmeasure
total_image_number += 4
# break
total_fmeasure /= total_image_number
if best_fmeasure < total_fmeasure:
best_fmeasure = total_fmeasure
print('epoch[%d/%d] fmeasure: %.4f, best_fmeasure: %.4f, time: %.2f' %
(epoch + 1, epochs, total_fmeasure, best_fmeasure,
time.time() - epoch_start_time))
print()
for channel in range(4):
torch.save(
models[channel].state_dict(), weight_path +
'/unet_%d_%d_%.4f.pth' % (channel, epoch + 1, total_fmeasure))
torch.save(discriminator.state_dict(),
weight_path + '/dis_%d_%.4f.pth' % (epoch + 1, total_fmeasure))
def main(argv=None):
image_dir = '/mnt/nas/data/denoise/Label_data/image'
mask_dir = '/mnt/nas/data/denoise/Label_data/mask'
overlap = 30. / 100. # 30. / 100. -> 65,247
imgh = 256
imgw = 256
scale_list = [
0.75, 1.00, 1.25, 1.50
] # sample patches with the scale factor and resize patches to 256 * 256 // 192, 256, 384
resize_size = (imgh, imgw)
image_save_dir = '/data/denoise/Label_patch/image_patches'
mask_save_dir = '/data/denoise/Label_patch/mask_patches'
os.makedirs(image_save_dir, exist_ok=True)
os.makedirs(mask_save_dir, exist_ok=True)
image_pathes = os.listdir(image_dir)
for image_path in image_pathes:
if not check_is_image(image_path):
print('not image', image_path)
continue
image_name = image_path.split('.')[0]
image = cv2.imread(os.path.join(image_dir, image_path))
# find and read mask file
if not os.path.isfile(os.path.join(mask_dir, image_path)):
print(image_path, 'no mask')
exit(1)
mask = cv2.imread(os.path.join(mask_dir, image_path),
cv2.IMREAD_GRAYSCALE)
mask[mask <= 128] = 0
mask[mask > 128] = 255
if image.shape[:2] != mask.shape[:2]:
print(image_path, 'size mismatch')
exit(1)
print('processing the image:', image_path)
scale_cnt = 0
for scale in scale_list:
# (patches, 256, 256, 3)
crpW = int(scale * imgw)
crpH = int(scale * imgh)
image_patches, poslist = get_image_patch(image, crpH, crpW,
overlap, False)
mask_patches, poslist = get_image_patch(mask, crpH, crpW, overlap,
True)
print('get patches: %d' % len(image_patches))
for idx in range(len(image_patches)):
img_color = image_patches[idx]
img_color = cv2.resize(img_color,
dsize=resize_size,
interpolation=cv2.INTER_NEAREST)
img_gray = cv2.cvtColor(img_color, cv2.COLOR_BGR2GRAY)
mask_gray = mask_patches[idx]
mask_gray = cv2.resize(mask_gray,
dsize=resize_size,
interpolation=cv2.INTER_NEAREST)
cv2.imwrite(
'%s/%s_s%di%d.png' %
(image_save_dir, image_name, scale_cnt, idx), img_color)
cv2.imwrite(
'%s/%s_s%di%d.png' %
(mask_save_dir, image_name, scale_cnt, idx), mask_gray)
scale_cnt += 1
image = cv2.imread(os.path.join(root_image_path, image_path))
# find and read mask file
if not os.path.isfile(os.path.join(root_mask_path, image_path)):
print(image_path, 'no mask')
exit(1)
mask = cv2.imread(os.path.join(root_mask_path, image_path),
cv2.IMREAD_GRAYSCALE)
mask[mask <= 128] = 0
mask[mask > 128] = 255
h, w, _ = image.shape
image_patches, poslist = get_image_patch(image,
crop_h,
crop_w,
overlap=predict_overlap_ratio,
is_mask=False)
merge_img = np.ones((h, w, 3))
out_imgs = []
for channel in range(4):
color_patches = []
for patch in image_patches:
tmp = patch.astype(np.float32)
if channel != 3:
color_patches.append(
preprocess_input(tmp[:, :, channel:channel + 1]))
else:
color_patches.append(
save_csv_file.writerow(csv_tmp)
save_csv_file.writerow([])
prev_dibco_year = dibco_year
# prepare ground truth mask for f-measure
gt_mask = cv2.imread(test_mask, cv2.IMREAD_GRAYSCALE)
gt_mask[gt_mask > 0] = 1
# end ground truth mask
print('processing the image:', img_name)
h, w, _ = image.shape
# start step1
image_patches, poslist = get_image_patch(image,
256,
256,
overlap=0.1,
is_mask=False)
merge_img = np.ones((h, w, 3))
out_imgs = []
for channel in range(4):
color_patches = []
for patch in image_patches:
tmp = patch.astype(np.float32)
if channel != 3:
color_patches.append(
preprocess_input(tmp[:, :, channel:channel + 1]))
else:
color_patches.append(
preprocess_input(
def unet_train(epochs, gpu, base_model_name, encoder_weights, generator_lr, discriminator_lr, lambda_bce,
batch_size, image_train_dir, mask_train_dir, image_test_dir, original_dir, fold_num, fold_total):
image_train_dir = image_train_dir.replace('%d', str(fold_num))
mask_train_dir = mask_train_dir.replace('%d', str(fold_num))
image_test_dir = image_test_dir.replace('%d', str(fold_num))
weight_path = ('./step2_label%d_' % fold_num) + base_model_name + '_' + str(int(lambda_bce)) + '_' + str(generator_lr)
image_save_path = weight_path + '/images'
os.makedirs(weight_path, exist_ok=True)
os.makedirs(image_save_path, exist_ok=True)
# rgb , preprocess input
imagenet_mean = np.array( [0.485, 0.456, 0.406] )
imagenet_std = np.array( [0.229, 0.224, 0.225] )
# patch data loader
patch_train_data_set = Dataset_Return_One(image_train_dir, mask_train_dir, os.path.join(original_dir, 'image'),
base_model_name, encoder_weights, is_train=True, fold_num=0, fold_total=5)
patch_train_loader = DataLoader(patch_train_data_set, batch_size=batch_size, num_workers=4, shuffle=True)
device = torch.device("cuda:%s" % gpu)
# step2 patch unet
patch_model = smp.Unet(base_model_name, encoder_weights=encoder_weights, in_channels=3)
patch_model.to(device)
optimizer_patch_generator = optim.Adam(patch_model.parameters(), lr=generator_lr, betas=(0.5, 0.999))
discriminator = Discriminator(in_channels=4)
discriminator.to(device)
optimizer_discriminator = optim.Adam(discriminator.parameters(), lr=discriminator_lr, betas=(0.5, 0.999))
criterion = nn.BCEWithLogitsLoss()
image_test_pathes = os.listdir(image_test_dir)
image_test_list = []
for image_test_path in image_test_pathes:
if not check_is_image(image_test_path):
print(image_test_path, 'not image')
continue
image_test_list.append( (os.path.join(image_test_dir, image_test_path),
os.path.join(original_dir, 'mask', image_test_path)) )
print('test len:', len(image_test_list))
preprocess_input = get_preprocessing_fn(base_model_name, pretrained=encoder_weights)
lambda_gp = 10.0
threshold_value = int(256 * 0.5)
patch_best_fmeasure = 0.0
epoch_start_time = time.time()
for epoch in range(epochs):
# train
patch_model.train()
for idx, (images, masks) in enumerate(patch_train_loader):
# train discriminator with patch
images = images.to(device)
masks = masks.to(device)
masks_pred = patch_model(images)
# discriminator
discriminator.requires_grad_(True)
# Fake
fake_AB = torch.cat((images, masks_pred), 1).detach()
pred_fake = discriminator(fake_AB)
# Real
real_AB = torch.cat((images, masks), 1)
pred_real = discriminator(real_AB)
gradient_penalty = compute_gradient_penalty(discriminator, real_AB, fake_AB, device)
discriminator_loss = -torch.mean(pred_real) + torch.mean(pred_fake) + lambda_gp * gradient_penalty
optimizer_discriminator.zero_grad()
discriminator_loss.backward()
optimizer_discriminator.step()
if idx % 5 == 0:
discriminator.requires_grad_(False)
# generator
fake_AB = torch.cat((images, masks_pred), 1)
pred_fake = discriminator(fake_AB)
generator_loss = -torch.mean(pred_fake)
bce_loss = criterion(masks_pred, masks)
total_loss = generator_loss + bce_loss * lambda_bce
optimizer_patch_generator.zero_grad()
total_loss.backward()
optimizer_patch_generator.step()
if idx % 2000 == 0:
print('train step[%d/%d] patch discriminator loss: %.5f, total loss: %.5f, generator loss: %.5f, bce loss: %.5f, time: %.2f' %
(idx, len(patch_train_loader), discriminator_loss.item(), total_loss.item(), generator_loss.item(), bce_loss.item(), time.time() - epoch_start_time))
rand_idx_start = randrange(masks.size(0) - 3)
rand_idx_end = rand_idx_start + 3
test_masks_pred = torch.sigmoid(masks_pred[rand_idx_start:rand_idx_end]).detach().cpu()
test_masks_pred = test_masks_pred.permute(0, 2, 3, 1).numpy().astype(np.float32)
test_masks_pred = np.squeeze(test_masks_pred, axis=-1)
test_masks = masks[rand_idx_start:rand_idx_end].permute(0, 2, 3, 1).cpu().numpy().astype(np.float32)
test_masks = np.squeeze(test_masks, axis=-1)
test_images = images[rand_idx_start:rand_idx_end].permute(0, 2, 3, 1).cpu().numpy()
test_images = test_images * imagenet_std + imagenet_mean
test_images = np.maximum(test_images, 0.0)
test_images = np.minimum(test_images, 1.0)
sample_images(epoch, idx, test_images, test_masks, test_masks_pred, image_save_path)
break
# eval
patch_model.eval()
print('eval patch')
total_fmeasure = 0.0
total_image_number = 0
# random_number = randrange(len(patch_image_test_list))
for eval_idx, (image_test, mask_test) in enumerate(image_test_list):
image = cv2.imread(image_test)
h, w, _ = image.shape
image_name = image_test.split('/')[-1].split('.')[0]
gt_mask = cv2.imread(mask_test, cv2.IMREAD_GRAYSCALE)
gt_mask = np.expand_dims(gt_mask, axis=-1)
image_patches, poslist = get_image_patch(image, 256, 256, overlap=0.5, is_mask=False)
color_patches = []
for patch in image_patches:
color_patches.append(preprocess_input(patch.astype(np.float32), input_space="BGR"))
step = 0
preds = []
with torch.no_grad():
while step < len(image_patches):
ps = step
pe = step + batch_size
if pe >= len(image_patches):
pe = len(image_patches)
images_global = torch.from_numpy(np.array(color_patches[ps:pe])).permute(0, 3, 1, 2).float().to(device)
preds.extend( torch.sigmoid(patch_model(images_global)).cpu() )
step += batch_size
# handling overlap
out_img = np.ones((h, w, 1)) * 255
for i in range(len(image_patches)):
patch = preds[i].permute(1, 2, 0).numpy() * 255
start_h, start_w, end_h, end_w, h_shift, w_shift = poslist[i]
h_cut = end_h - start_h
w_cut = end_w - start_w
tmp = np.minimum(out_img[start_h:end_h, start_w:end_w], patch[h_shift:h_shift+h_cut, w_shift:w_shift+w_cut])
out_img[start_h:end_h, start_w:end_w] = tmp
out_img = out_img.astype(np.uint8)
out_img[out_img > threshold_value] = 255
out_img[out_img <= threshold_value] = 0
# if random_number == eval_idx:
# cv2.imwrite('%s/patch_%d_%s.png' % (image_save_path, epoch, image_name), out_img)
# f_measure
# background 1, text 0
gt_mask[gt_mask > 0] = 1
out_img[out_img > 0] = 1
# true positive
tp = np.zeros(gt_mask.shape, np.uint8)
tp[(out_img==0) & (gt_mask==0)] = 1
numtp = tp.sum()
# false positive
fp = np.zeros(gt_mask.shape, np.uint8)
fp[(out_img==0) & (gt_mask==1)] = 1
numfp = fp.sum()
# false negative
fn = np.zeros(gt_mask.shape, np.uint8)
fn[(out_img==1) & (gt_mask==0)] = 1
numfn = fn.sum()
precision = (numtp) / float(numtp + numfp)
recall = (numtp) / float(numtp + numfn)
fmeasure = 100. * (2. * recall * precision) / (recall + precision) # percent
total_fmeasure += fmeasure
total_image_number += 1
break
total_fmeasure /= total_image_number
if patch_best_fmeasure < total_fmeasure:
patch_best_fmeasure = total_fmeasure
print('epoch[%d/%d] patch fmeasure: %.4f, best_fmeasure: %.4f, time: %.2f'
% (epoch + 1, epochs, total_fmeasure, patch_best_fmeasure, time.time() - epoch_start_time))
print()
torch.save(patch_model.state_dict(), weight_path + '/unet_patch_%d_%.4f.pth' % (epoch + 1, total_fmeasure))
torch.save(discriminator.state_dict(), weight_path + '/dis_%d_%.4f.pth' % (epoch + 1, total_fmeasure))
mask = cv2.imread(os.path.join(mask_train_dir, image_name + '.png'),
cv2.IMREAD_GRAYSCALE)
elif os.path.isfile(os.path.join(mask_train_dir, image_name + '.bmp')):
mask = cv2.imread(os.path.join(mask_train_dir, image_name + '.bmp'),
cv2.IMREAD_GRAYSCALE)
else:
print(img, 'no mask')
exit(1)
mask[mask < 190] = 0
mask[mask >= 190] = 255
h, w, _ = image.shape
image_patches, poslist = get_image_patch(image,
crop_h,
crop_w,
overlap=predict_overlap_ratio,
is_mask=False)
merge_img = np.ones((h, w, 3))
out_imgs = []
for channel in range(4):
color_patches = []
for patch in image_patches:
tmp = patch.astype(np.float32)
if channel != 3:
color_patches.append(
preprocess_input(tmp[:, :, channel:channel + 1]))
else:
color_patches.append(