def __getitem__(self, index):
file_index = index
aug_num = 0
if self.augment:
file_index = index // self.factor
aug_num = int(index % self.factor)
load_dir = self.image_files[file_index]
data = sio.loadmat(load_dir)
ms = np.array(data['ms'][...], dtype=np.float32)
lms = np.array(data['ms_bicubic'][...], dtype=np.float32)
gt = np.array(data['gt'][...], dtype=np.float32)
ms, lms, gt = utils.data_augmentation(ms, mode=aug_num), utils.data_augmentation(lms, mode=aug_num), \
utils.data_augmentation(gt, mode=aug_num)
'''
# add Laplacian pyramid
gp = []
for _ in range(3):
gp.append(ms.copy())
ms = cv2.pyrDown(ms)
ss = []
for lvl in gp:
ss.append(torch.from_numpy(lvl).permute(2, 0, 1))
'''
ms = torch.from_numpy(ms.copy()).permute(2, 0, 1)
lms = torch.from_numpy(lms.copy()).permute(2, 0, 1)
gt = torch.from_numpy(gt.copy()).permute(2, 0, 1)
return ms, lms, gt
def main():
global args, config, last_epoch, best_prec, writer
writer = SummaryWriter(log_dir=args.work_path + '/event')
# 加载配置文件
with open(args.work_path + '/config.yaml') as f:
config = yaml.load(f)
config = easydict.EasyDict(config)
logger.info((config))
# 获取模型
net = get_model(config)
logger.info(net)
logger.info("=====total parameters:" + str(utils.count_parameters(net)))
device = 'cuda' if config.use_gpu else 'cpu'
if device == 'cuda':
net = torch.nn.DataParallel(net)
cudnn.benchmark = True
net.to(device)
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(net.parameters(),
config.lr_scheduler.base_lr,
momentum=config.optimize.momentum,
weight_decay=config.optimize.weight_decay,
nesterov=config.optimize.nesterov)
last_epoch = -1
best_prec = 0
# 加载训练过的模型继续训练
if args.work_path:
ckpt_file_name = args.work_path + '/' + config.ckpt_name + '.pth.tar'
if args.resume:
best_prec, last_epoch = utils.load_checkpoint(ckpt_file_name,
net,
optimizer=optimizer)
# 设置数据的格式转换
transform_train = transforms.Compose(utils.data_augmentation(config))
transform_test = transforms.Compose(
utils.data_augmentation(config, is_train=False))
train_loader, test_loader = utils.get_data_loader(transform_train,
transform_test, config)
logger.info("==============trian-test-file-pathL{}".format(config.dataset))
logger.info(" ======= Training =======\n")
for epoch in range(last_epoch + 1, config.epochs):
lr = utils.adjust_learning_rate(optimizer, epoch, config)
writer.add_scalar('learning_rate', lr, epoch)
train(train_loader, net, criterion, optimizer, epoch, device)
if epoch == 0 or (
epoch +
1) % config.eval_freq == 0 or epoch == config.epochs - 1:
test(test_loader, net, criterion, optimizer, epoch, device)
writer.close()
logger.info(
"======== Training Finished. best_test_acc: {:.3f}% ========".format(
best_prec))
def prepare_train_data(data_path_A, data_path_B, patch_size=60, stride=10, aug_times=1, if_reseize=True):
# train
print('process training data')
scales = [1]#, 0.9, 0.8, 0.7]
files_A = glob.glob(os.path.join('datasets', data_path_A, '*.*'))
files_A.sort()
files_B = glob.glob(os.path.join('datasets', data_path_B, '*.*'))
files_B.sort()
# assume all images in a single set have the same size
lenA = len(files_A)
lenScale = len(scales)
h_a, w_a, _ = cv2.imread(files_A[0]).shape
h_b, w_b, _ = cv2.imread(files_B[0]).shape
dataLength = aug_times * lenA
print(dataLength)
data = np.empty(shape=(2,dataLength, h_a, w_a))
train_num = 0
for i in range(lenA):
img_A = cv2.imread(files_A[i])
img_B = cv2.imread(files_B[i])
if if_reseize == True:
img_B = cv2.resize(img_B, (h_a, w_a), interpolation=cv2.INTER_CUBIC)
for k in range(lenScale):
Img_A = cv2.resize(img_A, (int(h_a*scales[k]), int(w_a*scales[k])), interpolation=cv2.INTER_CUBIC)
Img_A = np.expand_dims(Img_A[:,:,0].copy(), 0)
Img_A = np.float32(normalize(Img_A))
Img_B = cv2.resize(img_B, (int(h_b*scales[k]), int(w_b*scales[k])), interpolation=cv2.INTER_CUBIC)
Img_B = np.expand_dims(Img_B[:,:,0].copy(), 0)
Img_B = np.float32(normalize(Img_B))
data_AB= [Img_A, Img_B]
data[:, train_num,:,:] = data_AB
train_num += 1
for m in range(aug_times-1):
rand = np.random.randint(1,8)
data_aug_A = data_augmentation(Img_A, rand)
data_aug_B = data_augmentation(Img_B, rand)
data_aug_AB= [data_aug_A, data_aug_B]
data[:, train_num,:,:] = data_aug_AB
train_num += 1
print('training set, # samples %d\n' % train_num)
return data
def bal_aug_patches3(percent, patch_size, patches_img, patches_ref):
patches_images = []
patches_labels = []
print('bal_aug_patches')
print(len(patches_img))
for i in range(0, len(patches_img)):
patch = patches_ref[i]
class1 = patch[patch == 1]
# o que é esse cálculo
#print('class1')
# print(len(class1))
# print(int((patch_size**2)*(percent/100)))
if len(class1) >= int(
(patch_size**2) * (percent / 100)) and np.all(patch != -1) == True:
patch_img = patches_img[i]
patch_label = patches_ref[i]
img_aug, label_aug = data_augmentation(patch_img, patch_label)
patches_images.append(img_aug)
patches_labels.append(label_aug)
print(len(patches_images))
patches_bal = np.concatenate(patches_images).astype(np.float32)
labels_bal = np.concatenate(patches_labels).astype(np.float32)
return patches_bal, labels_bal
def bal_aug_patches2(percent, patch_size, patches_img, patches_ref):
patches_images = []
patches_labels = []
print('bal_aug_patches')
print(len(patches_img))
print(patches_img.shape)
for i in range(0, len(patches_img)):
patch = patches_ref[i]
class1 = patch[patch == 1]
#print('class1')
# print(len(class1))
# print(int((patch_size**2)*(percent/100)))
#if len(class1) >= int((patch_size**2)*(percent/100)):
patch_img = patches_img[i]
patch_label = patches_ref[i]
# print(patch_label.shape)
# print(patch_img.shape)
img_aug, label_aug = data_augmentation(patch_img, patch_label)
# print(img_aug.shape)
# print(label_aug.shape)
patches_images.append(img_aug)
patches_labels.append(label_aug)
print(len(patches_images))
patches_bal = np.concatenate(patches_images).astype(np.float32)
labels_bal = np.concatenate(patches_labels).astype(np.float32)
return patches_bal, labels_bal
def read_iris_data(self, img_paths, margin=5, is_augment=False):
batch_imgs = np.zeros((len(img_paths), self.input_img_shape[1], self.input_img_shape[1], 1), dtype=np.float32)
batch_labels = np.zeros((len(img_paths), 1), dtype=np.uint8)
for i, img_path in enumerate(img_paths):
mask = np.zeros((self.img_shape[0], self.img_shape[1], 3), dtype=np.uint8)
# Extract Iris part
img_combine = cv2.imread(img_path)
img = img_combine[:, :self.img_shape[1], :]
seg = img_combine[:, self.img_shape[1]:, :]
if is_augment is True:
# Data augmentation: random brightness + random rotation
img_aug, seg_aug = utils.data_augmentation(img, seg)
mask[:, :, :][seg_aug[:, :, 1] == 204] = 1
img = img_aug * mask
else:
mask[:, :, :][seg[:, :, 1] == 204] = 1
img = img * mask
# Cropping iris part
x, y, w, h = cv2.boundingRect(mask[:, :, 1])
new_x = np.maximum(0, x - margin)
new_y = np.maximum(0, y - margin)
crop_img = img[new_y:new_y + h + margin, new_x:new_x + w + margin, 1] # Extract more bigger area
# Padding to the required size by preserving ratio of height and width
batch_imgs[i, :, :, 0] = utils.padding(crop_img)
batch_labels[i] = self.convert_to_cls(img_path)
return batch_imgs, batch_labels
def main():
data_augmentation = utils.data_augmentation(ToTensor=True,
HFlip=0.5,
VFlip=0.5)
x_train_dir, y_train_dir, x_valid_dir, y_valid_dir, x_test_dir, y_test_dir = _get_dir(
)
# Get train and val dataset instances
train_dataset = RoadsDataset(
x_train_dir,
y_train_dir,
Numclass=2,
augmentation=data_augmentation,
)
valid_dataset = RoadsDataset(
x_valid_dir,
y_valid_dir,
Numclass=2,
augmentation=data_augmentation,
)
# 展示一个看看
image, mask = train_dataset[0]
train_dataset.visu()
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=16,
shuffle=True,
num_workers=4)
valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=1,
shuffle=False,
num_workers=4)
return train_loader, valid_loader, train_dataset, valid_dataset
def prepare_data(data_path, patch_size, stride, aug_times=1):
# train
print('process training data')
scales = [1]
files = glob.glob(os.path.join('D:', 'DIV2K_train_HR', '*.png'))
files.sort()
h5f = h5py.File('D:/train.h5', 'w')
train_num = 0
for i in range(len(files)):
Img = cv2.imread(files[i])
h, w, c = Img.shape
for k in range(len(scales)):
# Img = cv2.resize(img, (int(h*scales[k]), int(w*scales[k])), interpolation=cv2.INTER_CUBIC)
Img = np.expand_dims(Img[:, :, 0].copy(), 0)
Img = np.float32(normalize(Img))
patches = Im2Patch(Img, win=patch_size, stride=stride)
print("file: %s scale %.1f # samples: %d" %
(files[i], scales[k], patches.shape[3] * aug_times))
for n in range(patches.shape[3]):
data = patches[:, :, :, n].copy()
h5f.create_dataset(str(train_num), data=data)
train_num += 1
for m in range(aug_times - 1):
data_aug = data_augmentation(data, np.random.randint(1, 8))
h5f.create_dataset(str(train_num) + "_aug_%d" % (m + 1),
data=data_aug)
train_num += 1
h5f.close()
'''
def infer(data_path, model):
psnr = utils.AvgrageMeter()
ssim = utils.AvgrageMeter()
model.eval()
transforms = torchvision.transforms.Compose(
[torchvision.transforms.ToTensor()])
with torch.no_grad():
for step, pt in enumerate(glob.glob(data_path)):
image = np.array(Image.open(pt))
clear_image = utils.crop_img(image[:, :image.shape[1] // 2, :],
base=args.patch_size)
rain_image = utils.crop_img(image[:, image.shape[1] // 2:, :],
base=args.patch_size)
# # Test on whole image
# input = transforms(rain_image).unsqueeze(dim=0).cuda()
# target = transforms(clear_image).unsqueeze(dim=0).cuda(async=True)
# logits = model(input)
# n = input.size(0)
# Test on whole image with data augmentation
target = transforms(clear_image).unsqueeze(dim=0).cuda()
for i in range(8):
im = utils.data_augmentation(rain_image, i)
input = transforms(im.copy()).unsqueeze(dim=0).cuda()
begin_time = time.time()
if i == 0:
logits = utils.inverse_augmentation(
model(input).cpu().numpy().transpose(0, 2, 3, 1)[0], i)
else:
logits = logits + utils.inverse_augmentation(
model(input).cpu().numpy().transpose(0, 2, 3, 1)[0], i)
end_time = time.time()
n = input.size(0)
logits = transforms(logits / 8).unsqueeze(dim=0).cuda()
# # Test on patches2patches
# noise_patches = utils.slice_image2patches(rain_image, patch_size=args.patch_size)
# image_patches = utils.slice_image2patches(clear_image, patch_size=args.patch_size)
# input = torch.tensor(noise_patches.transpose(0,3,1,2)/255.0, dtype=torch.float32).cuda()
# target = torch.tensor(image_patches.transpose(0,3,1,2)/255.0, dtype=torch.float32).cuda()
# logits = model(input)
# n = input.size(0)
s = pytorch_ssim.ssim(torch.clamp(logits, 0, 1), target)
p = utils.compute_psnr(
np.clip(logits.detach().cpu().numpy(), 0, 1),
target.detach().cpu().numpy())
psnr.update(p, n)
ssim.update(s, n)
print('psnr:%6f ssim:%6f' % (p, s))
# Image.fromarray(rain_image).save(args.save+'/'+str(step)+'_noise.png')
# Image.fromarray(np.clip(logits[0].cpu().numpy().transpose(1,2,0)*255, 0, 255).astype(np.uint8)).save(args.save+'/'+str(step)+'_denoised.png')
return psnr.avg, ssim.avg
def getitem(phase, data_names, item, patch_size):
if phase == 'train':
low_im = load_images(data_names[0][item])
high_im = load_images(data_names[1][item])
h, w, _ = low_im.shape
x = random.randint(0, h - patch_size)
y = random.randint(0, w - patch_size)
rand_mode = random.randint(0, 7)
low_im = data_augmentation(
low_im[x:x + patch_size, y:y + patch_size, :], rand_mode)
high_im = data_augmentation(
high_im[x:x + patch_size, y:y + patch_size, :], rand_mode)
return low_im, high_im
elif phase == 'eval':
low_im = load_images(data_names[item])
return low_im
def prepare_data(data_path,
patch_size,
stride,
aug_times=1,
grayscale=True,
scales_bool=False):
# train
print('process training data')
if scales_bool:
scales = [1, 0.9, 0.8, 0.7]
else:
scales = [1]
if grayscale:
files = glob.glob(os.path.join(data_path, 'BSD400', '*.png'))
files.sort()
train_file_name = 'train_%d.h5' % aug_times
if scales_bool:
train_file_name = 'train_%d_scales.h5' % aug_times
else:
files = glob.glob(os.path.join(data_path, 'CBSD432', '*.jpg'))
files.sort()
train_file_name = 'color_train_%d.h5' % aug_times
if scales_bool:
train_file_name = 'color_train_%d_scales.h5' % aug_times
h5f = h5py.File(train_file_name, 'w')
train_num = 0
for i in range(len(files)):
img = cv2.imread(files[i])
h, w, c = img.shape
for k in range(len(scales)):
Img = cv2.resize(img, (int(h * scales[k]), int(w * scales[k])),
interpolation=cv2.INTER_CUBIC)
if grayscale:
Img = np.expand_dims(Img[:, :, 0].copy(), 0)
else:
Img = np.rollaxis(Img, axis=2, start=0)
Img = np.float32(normalize(Img))
patches = Im2Patch(Img, win=patch_size, stride=stride)
print("file: %s scale %.1f # samples: %d" %
(files[i], scales[k], patches.shape[3] * aug_times))
for n in range(patches.shape[3]):
data = patches[:, :, :, n].copy()
h5f.create_dataset(str(train_num), data=data)
train_num += 1
for m in range(aug_times - 1):
data_aug = data_augmentation(data, np.random.randint(1, 8))
h5f.create_dataset(str(train_num) + "_aug_%d" % (m + 1),
data=data_aug)
train_num += 1
h5f.close()
print('training set, # samples %d\n' % train_num)
def prepare_data(data_path, patch_size, stride, aug_times=1, debug='N'):
# train
print('process training data')
scales = [1, 0.9, 0.8, 0.7]
if debug == 'Y':
train_dir = 'train_small'
train_file_out = '../train_small.h5'
else:
train_dir = 'train'
train_file_out = '../train.h5'
files = glob.glob(os.path.join(data_path, train_dir,
'*.png')) # 用来匹配所有的png
files.sort()
h5f = h5py.File(train_file_out, 'w')
train_num = 0
for i in range(len(files)):
img = cv2.imread(files[i])
h, w, c = img.shape
for k in range(len(scales)):
Img = cv2.resize(img, (int(h * scales[k]), int(w * scales[k])),
interpolation=cv2.INTER_CUBIC) # 构造不同的清晰度
Img = np.expand_dims(Img[:, :, 0].copy(), 0)
Img = np.float32(normalize(Img)) # 归一化为小数
patches = Im2Patch(Img, win=patch_size, stride=stride)
print("file: %s scale %.1f # samples: %d" %
(files[i], scales[k], patches.shape[3] * aug_times))
for n in range(patches.shape[3]):
data = patches[:, :, :, n].copy()
h5f.create_dataset(str(train_num), data=data)
train_num += 1
for m in range(aug_times - 1):
data_aug = data_augmentation(data, np.random.randint(1, 8))
h5f.create_dataset(str(train_num) + "_aug_%d" % (m + 1),
data=data_aug)
train_num += 1
h5f.close()
print('\nprocess validation data test')
files.clear()
files = glob.glob(os.path.join(data_path, 'test', '*.png'))
files.sort()
h5f = h5py.File('../val.h5', 'w')
val_num = 0
for i in range(len(files)):
print("file: %s" % files[i])
img = cv2.imread(files[i])
img = np.expand_dims(img[:, :, 0], 0)
img = np.float32(normalize(img))
h5f.create_dataset(str(val_num), data=img)
val_num += 1
h5f.close()
print('training set, # samples %d\n' % train_num)
print('testing set, # samples %d\n' % val_num)
def prepare_data(data_path, patch_size, stride, aug_times=1):
# train
print('process training data')
scales = [1, 0.9, 0.8, 0.7]
files = glob.glob(os.path.join(data_path, 'Train', '*', '*.jpg'))
train_files = []
for i in range(500):
idx = random.randrange(0, len(files))
train_files.append(files.pop(idx))
test_files = []
for i in range(100):
idx = random.randrange(0, len(files))
test_files.append(files.pop(idx))
train_files.sort()
h5f = h5py.File('NWPU_train.h5', 'w')
train_num = 0
for i in range(len(train_files)):
img = cv2.imread(train_files[i])
h, w, c = img.shape
for k in range(len(scales)):
Img = cv2.resize(img, (int(h * scales[k]), int(w * scales[k])),
interpolation=cv2.INTER_CUBIC)
#Img = np.expand_dims(Img[:,:,:].copy(), 0)
Img = np.transpose(Img, (2, 0, 1))
Img = np.float32(normalize(Img))
patches = Im2Patch(Img, win=patch_size, stride=stride)
print("file: %s scale %.1f # samples: %d" %
(train_files[i], scales[k], patches.shape[3] * aug_times))
for n in range(patches.shape[3]):
data = patches[:, :, :, n].copy()
h5f.create_dataset(str(train_num), data=data)
train_num += 1
for m in range(aug_times - 1):
data_aug = data_augmentation(data, np.random.randint(1, 8))
h5f.create_dataset(str(train_num) + "_aug_%d" % (m + 1),
data=data_aug)
train_num += 1
h5f.close()
# val
# print('\nprocess validation data')
# files.clear()
# files = glob.glob(os.path.join(data_path, 'Set12', '*.png'))
test_files.sort()
h5f = h5py.File('NWPU_val.h5', 'w')
val_num = 0
for i in range(len(test_files)):
print("file: %s" % test_files[i])
img = cv2.imread(test_files[i])
img = np.transpose(img, (2, 0, 1))
img = np.float32(normalize(img))
h5f.create_dataset(str(val_num), data=img)
val_num += 1
h5f.close()
print('training set, # samples %d\n' % train_num)
print('val set, # samples %d\n' % val_num)
def __getitem__(self, index):
file_index = index
aug_num = 0
if self.augment:
file_index = index // self.factor
aug_num = int(index % self.factor)
load_dir = self.image_files[file_index]
data = sio.loadmat(load_dir)
ms = np.array(data['ms'][...], dtype=np.float32)
lms = np.array(data['ms_bicubic'][...], dtype=np.float32)
gt = np.array(data['gt'][...], dtype=np.float32)
ms, lms, gt = utils.data_augmentation(ms, mode=aug_num), utils.data_augmentation(lms, mode=aug_num), \
utils.data_augmentation(gt, mode=aug_num)
if self.use_3Dconv:
ms, lms, gt = ms[np.newaxis, :, :, :], lms[
np.newaxis, :, :, :], gt[np.newaxis, :, :, :]
ms = torch.from_numpy(ms.copy()).permute(0, 3, 1, 2)
lms = torch.from_numpy(lms.copy()).permute(0, 3, 1, 2)
gt = torch.from_numpy(gt.copy()).permute(0, 3, 1, 2)
else:
ms = torch.from_numpy(ms.copy()).permute(2, 0, 1)
lms = torch.from_numpy(lms.copy()).permute(2, 0, 1)
gt = torch.from_numpy(gt.copy()).permute(2, 0, 1)
return ms, lms, gt
def prepare_data(data_path, patch_size, stride, aug_times=1):
# train
print('process training data')
scales = [1, 0.9, 0.8, 0.7]
#files = glob.glob(os.path.join(data_path, 'grayimages', '*'))
files = glob.glob(os.path.join(data_path, 'pristine_images_gray', '*'))
files.sort()
h5f = h5py.File('train.h5', 'w')
train_num = 0
for i in range(len(files)):
img = cv2.imread(files[i])
h, w, c = img.shape
for k in range(len(scales)):
Img = cv2.resize(img, (int(h * scales[k]), int(w * scales[k])),
interpolation=cv2.INTER_CUBIC)
Img = np.expand_dims(Img[:, :, 0].copy(), 0)
Img = np.float32(normalize(Img))
patches = Im2Patch(Img, win=patch_size, stride=stride)
print("file: %s scale %.1f # samples: %d" %
(files[i], scales[k], patches.shape[3] * aug_times))
for n in range(patches.shape[3]):
data = patches[:, :, :, n].copy()
h5f.create_dataset(str(train_num), data=data)
train_num += 1
for m in range(aug_times - 1):
data_aug = data_augmentation(data, np.random.randint(1, 8))
h5f.create_dataset(str(train_num) + "_aug_%d" % (m + 1),
data=data_aug)
train_num += 1
h5f.close()
# val
print('\nprocess validation data')
#files.clear()
files = glob.glob(os.path.join(data_path, 'Set68', '*.png'))
files.sort()
h5f = h5py.File('val.h5', 'w')
val_num = 0
for i in range(len(files)):
print("file: %s" % files[i])
img = cv2.imread(files[i])
img = np.expand_dims(img[:, :, 0], 0)
img = np.float32(normalize(img))
h5f.create_dataset(str(val_num), data=img)
val_num += 1
h5f.close()
print('training set, # samples %d\n' % train_num)
print('val set, # samples %d\n' % val_num)
def read_data(self, index):
"""This function is used to read the data with the index
:param index: the index of the data you want to get.
"""
# if this is for training, just load the the from training list
if self.training:
x1 = self.train_images[index] # the first list of images (ADC)
x2 = self.train_images[index] # the second list of images (T2WI)
y = self.train_labels[index] # the list of labels
else: # if this is for testing, just load the the from testing list
x1 = self.test_images[index] # the first list of images (ADC)
x2 = self.test_images[index] # the second list of images (T2WI)
y = self.test_labels[index] # the list of labels
height, width = x1.shape # get the size of the image
x1 = normalize(
x1.reshape(height, width,
1)) # apply the normalization (norm to range [0, 1])
x1 = standardize(x1) # apply the standardization (reshape the data)
x2 = normalize(
x2.reshape(height, width,
1)) # apply the normalization (norm to range [0, 1])
x2 = standardize(x2) # apply the standardization (reshape the data)
# apply data augmentation
augmented_data = data_augmentation(np.concatenate([x1, x2], axis=2),
use_rigid=self.use_rigid,
use_non_rigid=self.use_non_rigid)
# NOTE: because the data I used has multiple classes, so I have to modified it a bit. Remove the following line (just one line)
y = (y != 1).astype(np.uint8) # remove this
return augmented_data[:, :, :, :
3], augmented_data[:, :, :,
3:], tf.keras.utils.to_categorical(
y,
num_classes=2,
dtype='float32')
def main():
global args, config, last_epoch, best_prec, writer
writer = SummaryWriter(log_dir=args.work_path + "/event")
# read config from yaml file
with open(args.work_path + "/config.yaml") as f:
config = yaml.load(f)
# convert to dict
config = EasyDict(config)
logger.info(config)
# define netowrk
net = get_model(config)
logger.info(net)
logger.info(" == total parameters: " + str(count_parameters(net)))
# CPU or GPU
device = "cuda" if config.use_gpu else "cpu"
# data parallel for multiple-GPU
if device == "cuda":
net = torch.nn.DataParallel(net)
cudnn.benchmark = True
net.to(device)
# define loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(
net.parameters(),
config.lr_scheduler.base_lr,
momentum=config.optimize.momentum,
weight_decay=config.optimize.weight_decay,
nesterov=config.optimize.nesterov,
)
# resume from a checkpoint
last_epoch = -1
best_prec = 0
if args.work_path:
ckpt_file_name = args.work_path + "/" + config.ckpt_name + ".pth.tar"
if args.resume:
best_prec, last_epoch = load_checkpoint(ckpt_file_name,
net,
optimizer=optimizer)
# load training data, do data augmentation and get data loader
transform_train = transforms.Compose(data_augmentation(config))
transform_test = transforms.Compose(
data_augmentation(config, is_train=False))
train_loader, test_loader = get_data_loader(transform_train,
transform_test, config)
logger.info(" ======= Training =======\n")
for epoch in range(last_epoch + 1, config.epochs):
lr = adjust_learning_rate(optimizer, epoch, config)
writer.add_scalar("learning_rate", lr, epoch)
train(train_loader, net, criterion, optimizer, epoch, device)
if (epoch == 0 or (epoch + 1) % config.eval_freq == 0
or epoch == config.epochs - 1):
test(test_loader, net, criterion, optimizer, epoch, device)
writer.close()
logger.info(
"======== Training Finished. best_test_acc: {:.3f}% ========".format(
best_prec))
os.makedirs(os.path.join(folder_path, 'labels/seg'))
os.makedirs(os.path.join(folder_path, 'labels/bound'))
os.makedirs(os.path.join(folder_path, 'labels/dist'))
os.makedirs(os.path.join(folder_path, 'labels/color'))
def filename(i):
return f'patch_{i}.npy'
print(f'Number of patches: {len(patches_tr)}')
if args.data_aug:
print(f'Number of patches expected: {len(patches_tr)*5}')
for i in tqdm(range(len(patches_tr))):
if args.data_aug:
img_aug, label_aug = data_augmentation(patches_tr[i],
patches_tr_ref[i])
else:
img_aug, label_aug = np.expand_dims(
patches_tr[i], axis=0), np.expand_dims(patches_tr_ref[i], axis=0)
label_aug_h = tf.keras.utils.to_categorical(label_aug, args.num_classes)
for j in range(len(img_aug)):
# Input image RGB
# Float32 its need to train the model
img_float = img_aug[j].astype(np.float32)
img_normalized = normalize_rgb(img_float, norm_type=args.norm_type)
np.save(os.path.join(folder_path, 'train', filename(i * 5 + j)),
img_normalized)
# All multitasking labels are saved in one-hot
# Segmentation
np.save(os.path.join(folder_path, 'labels/seg', filename(i * 5 + j)),
label_aug_h[j].astype(np.float32))
def preprocess(index, x, y):
return (index,
data_augmentation(tf.add(x, -self.mean) / 255,
224), tf.one_hot(y, self.n_classes))
def gen_data(args):
"""
生成训练测试数据集
"""
data_path = args.data_path
save_path = args.save_path
train = args.train
test = args.test
size = args.size
stride = args.stride
aug_times = args.aug_times
gray_mode = args.gray_mode
pic_type = args.pic_type
train_path = Path(data_path).joinpath("Train")
val_data_path = Path(data_path).joinpath("Test")
if save_path is not None:
save_path = Path(save_path)
if not save_path.exists():
save_path.mkdir()
files_train = {}
files_test = {}
for x in train_path.glob("*"):
if x.is_dir():
file_list_train = [
str(f_train.absolute().resolve())
for f_train in x.glob(f"*.{pic_type}")
]
files_train[x.name] = []
files_train[x.name].extend(file_list_train)
for y in val_data_path.glob("*"):
if y.is_dir():
file_list_test = [
str(f_test.absolute().resolve())
for f_test in y.glob(f"*.{pic_type}")
]
files_test[y.name] = []
files_test[y.name].extend(file_list_test)
if gray_mode:
train_h5 = 'train_gray.h5'
train_h5_label = 'train_gray_label.h5'
val_h5 = 'val_gray.h5'
val_h5_label = 'val_gray_label.h5'
else:
train_h5 = 'train_rgb.h5'
train_h5_label = 'train_rgb_label.h5'
val_h5 = 'val_rgb.h5'
val_h5_label = 'val_rgb_label.h5'
if train:
# 读取训练图片，并成生数据集
f_train = h5py.File(save_path.joinpath(train_h5), 'w')
f_train_label = h5py.File(save_path.joinpath(train_h5_label), 'w')
train_num = 0
# k->label v->filename list
for k, v in files_train.items():
print(k)
print(v)
if len(v) == 0:
continue
# 读取每一张大图
for f in v:
if gray_mode:
# H * W * C
t_pic = cv2.imread(f, cv2.IMREAD_GRAYSCALE)
else:
t_pic = cv2.imread(f, cv2.IMREAD_COLOR)
# BRG -> RGB
t_pic = t_pic[:, :, ::-1]
# HWC -> CHW
t_pic = np.transpose(t_pic, (2, 0, 1))
t_pic = normalize(t_pic)
# CHW * patch_size
patches = img_2_patches(t_pic, size, stride)
# 控制样本数量
patches = patches[:, :, :, :2400]
# 处理每一张小图
print(f"训练文件:{f} --> ##{patches.shape[3]}##样本")
for nx in range(patches.shape[3]):
data = patches[:, :, :, nx]
f_train.create_dataset(str(train_num), data=data)
f_train_label.create_dataset(str(train_num),
data=np.array(
get_label(int(k))))
train_num += 1
# 数据增广
for mx in range(aug_times):
data_aug = data_augmentation(
patches[:, :, :, nx].copy(),
np.random.randint(1, 8))
f_train.create_dataset(str(train_num), data=data_aug)
f_train_label.create_dataset(str(train_num),
data=np.array(
get_label(int(k))))
train_num += 1
f_train.close()
f_train_label.close()
print(f"训练集图片数量:{train_num}")
if test:
# Gen Test Data
f_test = h5py.File(save_path.joinpath(val_h5), 'w')
f_test_label = h5py.File(save_path.joinpath(val_h5_label), 'w')
# k->label v->filename list
val_num = 0
for k, v in files_test.items():
print(k)
print(v)
if len(v) == 0:
continue
# 读取每一张大图
for f in v:
if gray_mode:
# H * W * C
t_pic = cv2.imread(f, cv2.IMREAD_GRAYSCALE)
else:
t_pic = cv2.imread(f, cv2.IMREAD_COLOR)
# BRG -> RGB
t_pic = t_pic[:, :, ::-1]
# HWC -> CHW
t_pic = np.transpose(t_pic, (2, 0, 1))
t_pic = normalize(t_pic)
# CHW * patch_size
patches = img_2_patches(t_pic, size, stride)
# 处理每一张小图
print(f"测试文件:{f} --> ##{patches.shape[3]}##样本")
for nx in range(patches.shape[3]):
data = patches[:, :, :, nx]
f_test.create_dataset(str(val_num), data=data)
f_test_label.create_dataset(str(val_num),
data=np.array(get_label(
int(k))))
val_num += 1
f_test.close()
f_test_label.close()
print(f"测试集图片数量:{val_num}")
def train(config, fold, model_1, model_2, dict_loader, optimizer, scheduler, list_dir_save_model, dir_pyplot, Validation=True, Test_flag = True):
train_loader = dict_loader['train']
val_loader = dict_loader['val']
test_loader = dict_loader['test']
""" loss """
# criterion_cls = nn.CrossEntropyLoss()
# criterion_cls = ut.FocalLoss(gamma=st.focal_gamma, alpha=st.focal_alpha, size_average=True)
criterion_cls = nn.BCELoss()
# criterion = nn.L1Loss(reduction='mean').cuda()
criterion = nn.MSELoss(reduction='mean').cuda()
# criterion_gdl = gdl_loss(pNorm=2).cuda()
EMS = ut.eval_metric_storage()
list_selected_EMS = []
list_ES = []
for i_tmp in range(len(st.list_standard_eval_dir)):
list_selected_EMS.append(ut.eval_selected_metirc_storage())
list_ES.append(ut.EarlyStopping(delta=0, patience=st.early_stopping_patience, verbose=True))
print('training')
""" epoch """
ut.model_freeze(model_2, requires_grad=False)
num_data = len(train_loader.dataset)
for epoch in range(config.num_epochs):
epoch = epoch + 1 # increase the # of the epoch
print(" ")
print("--------------- epoch {} ----------------".format(epoch))
torch.cuda.empty_cache()
""" print learning rate """
for param_group in optimizer.param_groups:
print('current LR : {}'.format(param_group['lr']))
""" batch """
for i, data_batch in enumerate(train_loader):
# start = time.time()
model_1.train()
model_2.eval()
EMS.total_train_step += 1
with torch.no_grad():
""" input"""
datas = Variable(data_batch['data'].float()).cuda()
# labels = Variable(data_batch['label'].long()).cuda()
labels = Variable(data_batch['label'].float()).cuda()
""" minmax norm"""
if st.list_data_norm_type[st.data_norm_type_num] == 'minmax':
tmp_datas = datas.view(datas.size(0), -1)
tmp_datas -= tmp_datas.min(1, keepdim=True)[0]
tmp_datas /= tmp_datas.max(1, keepdim=True)[0]
datas = tmp_datas.view_as(datas)
""" data augmentation """
##TODO : flip
# flip_flag_list = np.random.normal(size=datas.shape[0])>0
# datas[flip_flag_list] = datas[flip_flag_list].flip(-3)
##TODO : translation, cropping
dict_result = ut.data_augmentation(datas=datas, cur_epoch=epoch)
datas = dict_result['datas']
# aug_dict_result = ut.data_augmentation(datas=aug_datas, cur_epoch=epoch)
# aug_datas = aug_dict_result['datas']
""" gaussain noise """
# Gaussian_dist = torch.distributions.normal.Normal(loc=torch.tensor([0.0]), scale=torch.tensor([0.01]))
# Gaussian_dist = torch.distributions.normal.Normal(loc=torch.tensor([0.0]), scale=torch.FloatTensor(1).uniform_(0, 0.01))
# Gaussian_noise = Gaussian_dist.sample(datas.size()).squeeze(-1)
# datas = datas + Gaussian_noise.cuda()
""" model 1 forward """
dict_result = model_2(datas)
output_3 = dict_result['logitMap']
""" forward propagation """
dict_result = model_1(output_3.detach())
output_1 = dict_result['logits']
output_2 = dict_result['Aux_logits']
output_3 = dict_result['logitMap']
""" classification """
loss_list_1 = []
loss_2 = criterion_cls(output_1, labels)
loss_list_1.append(loss_2)
EMS.train_aux_loss_1.append(loss_2.data.cpu().numpy())
loss = sum(loss_list_1)
optimizer.zero_grad()
loss.backward()
optimizer.step()
""" print the train loss and tensorboard"""
if (EMS.total_train_step) % 10 == 0 :
# print('time : ', time.time() - start)
print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f'
%(epoch, config.num_epochs, i + 1, (round(num_data / config.batch_size)), loss.data.cpu().numpy()))
torch.cuda.empty_cache()
""" pyplot """
EMS.train_loss.append(loss.data.cpu().numpy())
EMS.train_step.append(EMS.total_train_step)
""" val """
if Validation == True:
print("------------------ val --------------------------")
dict_result = ut.eval_classification_model_2(config, fold, val_loader, model_1, model_2, criterion_cls)
val_loss = dict_result['Loss']
acc = dict_result['Acc']
auc = dict_result['AUC']
print('Fold : %d, Epoch [%d/%d] val Loss = %f val Acc = %f' % (fold, epoch, config.num_epochs, val_loss, acc))
torch.cuda.empty_cache()
""" save the metric """
EMS.dict_val_metric['val_loss'].append(val_loss)
EMS.dict_val_metric['val_acc'].append(acc)
EMS.dict_val_metric['val_auc'].append(auc)
EMS.val_step.append(EMS.total_train_step)
""" save model """
for i_tmp in range(len(list_selected_EMS)):
save_flag = ut.model_save_through_validation(fold, epoch, EMS=EMS,
selected_EMS=list_selected_EMS[i_tmp],
ES=list_ES[i_tmp],
model=model_1,
dir_save_model=list_dir_save_model[i_tmp],
metric_1=st.list_standard_eval[i_tmp], metric_2='',
save_flag=False)
if Test_flag== True:
print("------------------ test _ test dataset --------------------------")
""" load data """
dict_result = ut.eval_classification_model_2(config, fold, test_loader, model_1, model_2, criterion_cls)
test_loss = dict_result['Loss']
acc = dict_result['Acc']
test_loss = dict_result['Loss']
""" pyplot """
EMS.test_acc.append(acc)
EMS.test_loss.append(test_loss)
EMS.test_step.append(EMS.total_train_step)
print('number of test samples : {}'.format(len(test_loader.dataset)))
print('Fold : %d, Epoch [%d/%d] test Loss = %f test Acc = %f' % (fold, epoch, config.num_epochs, test_loss, acc))
torch.cuda.empty_cache()
""" learning rate decay"""
EMS.LR.append(optimizer.param_groups[0]['lr'])
scheduler.step()
# scheduler.step(val_loss)
""" plot the chat """
if epoch % 10 == 0:
ut.plot_training_info_1(fold, dir_pyplot, EMS, flag='percentile', flag_match=False)
##TODO : early stop only if all of metric has been stopped
tmp_count = 0
for i in range(len(list_ES)):
if list_ES[i].early_stop == True:
tmp_count += 1
if tmp_count == len(list_ES):
break
""" release the model """
del model_1, EMS
torch.cuda.empty_cache()
def prepare_train_data(data_path_A,
data_path_B,
patch_size_dn=15,
patch_size_sr=60,
stride=10,
aug_times=1,
if_reseize=True):
# train
print('process training data')
scales = [1, 0.9, 0.8, 0.7]
files_A = glob.glob(os.path.join('datasets', data_path_A, '*.*'))
files_A.sort()
files_B = glob.glob(os.path.join('datasets', data_path_B, '*.*'))
files_B.sort()
# assume all images in a single set have the same size
lenA = len(files_A)
lenScale = len(scales)
lenPatch = num_of_patch(cv2.imread(files_A[0]),
scales=scales,
win=patch_size_sr,
stride=stride * 4)
h_a, w_a, _ = cv2.imread(files_A[0]).shape
h_b, w_b, _ = cv2.imread(files_B[0]).shape
dataLength = (aug_times) * lenA * lenPatch
print(lenPatch)
data_dn = np.empty(shape=(2, dataLength, patch_size_dn, patch_size_dn))
data_sr = np.empty(shape=(1, dataLength, patch_size_sr, patch_size_sr))
train_num = 0
for i in range(lenA):
img_A = cv2.imread(files_A[i])
img_B = cv2.imread(files_B[i])
if if_reseize == True:
img_L = cv2.resize(img_A, (h_b, w_b),
interpolation=cv2.INTER_CUBIC)
for k in range(lenScale):
Img_A = cv2.resize(img_A,
(int(h_a * scales[k]), int(w_a * scales[k])),
interpolation=cv2.INTER_CUBIC)
Img_A = np.expand_dims(Img_A[:, :, 0].copy(), 0)
Img_A = np.float32(normalize(Img_A))
Img_L = cv2.resize(img_L,
(int(h_b * scales[k]), int(w_b * scales[k])),
interpolation=cv2.INTER_CUBIC)
Img_L = np.expand_dims(Img_L[:, :, 0].copy(), 0)
Img_L = np.float32(normalize(Img_L))
Img_B = cv2.resize(img_B,
(int(h_b * scales[k]), int(w_b * scales[k])),
interpolation=cv2.INTER_CUBIC)
Img_B = np.expand_dims(Img_B[:, :, 0].copy(), 0)
Img_B = np.float32(normalize(Img_B))
patches_A = Im2Patch(Img_A,
win=patch_size_sr,
winscale=4,
stride=stride * 4)
patches_L = Im2Patch(Img_L,
win=patch_size_dn,
winscale=1,
stride=stride)
patches_B = Im2Patch(Img_B,
win=patch_size_dn,
winscale=1,
stride=stride)
#print("file: %s scale %.1f # samples: %d" % (files_A[i], scales[k], patches_A.shape[3]*aug_times))
#print("file: %s scale %.1f # samples: %d" % (files_B[i], scales[k], patches_A.shape[3]*aug_times))
for n in range(patches_A.shape[3]):
data_A = patches_A[:, :, :, n].copy()
data_L = patches_L[:, :, :, n].copy()
data_B = patches_B[:, :, :, n].copy()
data_LB = [data_L, data_B]
data_dn[:, train_num, :, :] = data_LB
data_sr[:, train_num, :, :] = [data_A]
#data_sr = np.append(data_sr, [data_A], axis=1)
#data_dn = np.append(data_dn, data_LB, axis=1)
train_num += 1
for m in range(aug_times - 1):
rand = np.random.randint(1, 8)
data_aug_A = data_augmentation(data_A, rand)
data_aug_L = data_augmentation(data_L, rand)
data_aug_B = data_augmentation(data_B, rand)
data_aug_LB = [data_aug_L, data_aug_B]
data_dn[:, train_num, :, :] = data_aug_LB
data_sr[:, train_num, :, :] = [data_aug_A]
#data_dn = np.append(data_dn, data_aug_LB, axis=1)
#data_sr = np.append(data_sr, [data_aug_A], axis=1)
train_num += 1
print('training set, # samples %d\n' % train_num)
return data_sr, data_dn
def train(config,
fold,
model,
dict_loader,
optimizer,
scheduler,
list_dir_save_model,
dir_pyplot,
Validation=True,
Test_flag=True):
train_loader = dict_loader['train']
val_loader = dict_loader['val']
test_loader = dict_loader['test']
""" loss """
# criterion_cls = nn.CrossEntropyLoss()
# criterion_cls = ut.FocalLoss(gamma=st.focal_gamma, alpha=st.focal_alpha, size_average=True)
# kdloss = ut.KDLoss(4.0)
criterion_KL = nn.KLDivLoss(reduction="sum")
criterion_cls = nn.BCELoss()
# criterion_L1 = nn.L1Loss(reduction='sum').cuda()
# criterion_L2 = nn.MSELoss(reduction='mean').cuda()
# criterion_gdl = gdl_loss(pNorm=2).cuda()
EMS = ut.eval_metric_storage()
list_selected_EMS = []
list_ES = []
for i_tmp in range(len(st.list_standard_eval_dir)):
list_selected_EMS.append(ut.eval_selected_metirc_storage())
list_ES.append(
ut.EarlyStopping(delta=0,
patience=st.early_stopping_patience,
verbose=True))
loss_tmp = [0] * 5
loss_tmp_total = 0
print('training')
optimizer.zero_grad()
optimizer.step()
""" epoch """
num_data = len(train_loader.dataset)
for epoch in range(1, config.num_epochs + 1):
scheduler.step()
print(" ")
print("--------------- epoch {} ----------------".format(epoch))
""" print learning rate """
for param_group in optimizer.param_groups:
print('current LR : {}'.format(param_group['lr']))
""" batch """
for i, data_batch in enumerate(train_loader):
# start = time.time()
model.train()
with torch.no_grad():
""" input"""
datas = Variable(data_batch['data'].float()).cuda()
# labels = Variable(data_batch['label'].long()).cuda()
labels = Variable(data_batch['label'].float()).cuda()
""" data augmentation """
##TODO : flip
# flip_flag_list = np.random.normal(size=datas.shape[0])>0
# datas[flip_flag_list] = datas[flip_flag_list].flip(-3)
##TODO : translation, cropping
dict_result = ut.data_augmentation(datas=datas,
cur_epoch=epoch)
datas = dict_result['datas']
translation_list = dict_result['translation_list']
# aug_dict_result = ut.data_augmentation(datas=aug_datas, cur_epoch=epoch)
# aug_datas = aug_dict_result['datas']
""" minmax norm"""
if st.list_data_norm_type[st.data_norm_type_num] == 'minmax':
tmp_datas = datas.view(datas.size(0), -1)
tmp_datas -= tmp_datas.min(1, keepdim=True)[0]
tmp_datas /= tmp_datas.max(1, keepdim=True)[0]
datas = tmp_datas.view_as(datas)
""" gaussain noise """
# Gaussian_dist = torch.distributions.normal.Normal(loc=torch.tensor([0.0]), scale=torch.tensor([0.01]))
# Gaussian_dist = torch.distributions.normal.Normal(loc=torch.tensor([0.0]), scale=torch.FloatTensor(1).uniform_(0, 0.01))
# Gaussian_noise = Gaussian_dist.sample(datas.size()).squeeze(-1)
# datas = datas + Gaussian_noise.cuda()
""" forward propagation """
dict_result = model(datas, translation_list)
output_1 = dict_result['logits']
output_2 = dict_result['Aux_logits']
output_3 = dict_result['logitMap']
output_4 = dict_result['l1_norm']
#
loss_list_1 = []
count_loss = 0
if fst.flag_loss_1 == True:
s_labels = ut.smooth_one_hot(labels,
config.num_classes,
smoothing=st.smoothing_img)
loss_2 = criterion_cls(
output_1,
s_labels) * st.lambda_major[0] / st.iter_to_update
loss_list_1.append(loss_2)
loss_tmp[count_loss] += loss_2.data.cpu().numpy()
if (EMS.total_train_iter + 1) % st.iter_to_update == 0:
EMS.train_aux_loss[count_loss].append(loss_tmp[count_loss])
loss_tmp[count_loss] = 0
count_loss += 1
if fst.flag_loss_2 == True:
for i_tmp in range(len(output_2)):
s_labels = ut.smooth_one_hot(labels,
config.num_classes,
smoothing=st.smoothing_roi)
loss_2 = criterion_cls(
output_2[i_tmp],
s_labels) * st.lambda_aux[i_tmp] / st.iter_to_update
loss_list_1.append(loss_2)
loss_tmp[count_loss] += loss_2.data.cpu().numpy()
if (EMS.total_train_iter + 1) % st.iter_to_update == 0:
EMS.train_aux_loss[count_loss].append(
loss_tmp[count_loss])
loss_tmp[count_loss] = 0
count_loss += 1
if fst.flag_loss_3 == True:
# patch
list_loss_tmp = []
for tmp_j in range(len(output_4)): # type i.e., patch, roi
loss_2 = 0
for tmp_i in range(len(output_4[tmp_j])): # batch
tmp_shape = output_4[tmp_j][tmp_i].shape
logits = output_4[tmp_j][tmp_i].view(
tmp_shape[0], tmp_shape[1], -1)
# loss_2 += torch.norm(logits, p=1)
loss_2 += torch.norm(logits,
p=1) / (logits.view(-1).size(0))
list_loss_tmp.append(
(loss_2 / len(output_4[tmp_j]) * st.l1_reg_norm) /
st.iter_to_update)
loss_list_1.append(sum(list_loss_tmp))
loss_tmp[count_loss] += sum(list_loss_tmp).data.cpu().numpy()
if (EMS.total_train_iter + 1) % st.iter_to_update == 0:
EMS.train_aux_loss[count_loss].append(loss_tmp[count_loss])
loss_tmp[count_loss] = 0
count_loss += 1
""" L1 reg"""
# norm = torch.FloatTensor([0]).cuda()
# for parameter in model.parameters():
# norm += torch.norm(parameter, p=1)
# loss_list_1.append(norm * st.l1_reg)
loss = sum(loss_list_1)
loss.backward()
torch.cuda.empty_cache()
loss_tmp_total += loss.data.cpu().numpy()
#TODO : optimize the model param
if (EMS.total_train_iter + 1) % st.iter_to_update == 0:
optimizer.step()
optimizer.zero_grad()
""" pyplot """
EMS.total_train_step += 1
EMS.train_step.append(EMS.total_train_step)
EMS.train_loss.append(loss_tmp_total)
""" print the train loss and tensorboard"""
if (EMS.total_train_step) % 10 == 0:
# print('time : ', time.time() - start)
print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f' %
(epoch, config.num_epochs, (i + 1),
(num_data // (config.batch_size)), loss_tmp_total))
loss_tmp_total = 0
EMS.total_train_iter += 1
# scheduler.step(epoch + i / len(train_loader))
""" val """
if Validation == True:
print("------------------ val --------------------------")
if fst.flag_cropping == True and fst.flag_eval_cropping == True:
dict_result = ut.eval_classification_model_cropped_input(
config, fold, val_loader, model, criterion_cls)
elif fst.flag_translation == True and fst.flag_eval_translation == True:
dict_result = ut.eval_classification_model_esemble(
config, fold, val_loader, model, criterion_cls)
elif fst.flag_MC_dropout == True:
dict_result = ut.eval_classification_model_MC_dropout(
config, fold, val_loader, model, criterion_cls)
else:
dict_result = ut.eval_classification_model(
config, fold, val_loader, model, criterion_cls)
val_loss = dict_result['Loss']
acc = dict_result['Acc']
auc = dict_result['AUC']
print('Fold : %d, Epoch [%d/%d] val Loss = %f val Acc = %f' %
(fold, epoch, config.num_epochs, val_loss, acc))
""" save the metric """
EMS.dict_val_metric['val_loss'].append(val_loss)
EMS.dict_val_metric['val_acc'].append(acc)
if fst.flag_loss_2 == True:
for tmp_i in range(len(st.lambda_aux)):
EMS.dict_val_metric['val_acc_aux'][tmp_i].append(
dict_result['Acc_aux'][tmp_i])
EMS.dict_val_metric['val_auc'].append(auc)
EMS.val_step.append(EMS.total_train_step)
n_stacking_loss_for_selection = 5
if len(EMS.dict_val_metric['val_loss_queue']
) > n_stacking_loss_for_selection:
EMS.dict_val_metric['val_loss_queue'].popleft()
EMS.dict_val_metric['val_loss_queue'].append(val_loss)
EMS.dict_val_metric['val_mean_loss'].append(
np.mean(EMS.dict_val_metric['val_loss_queue']))
""" save model """
for i_tmp in range(len(list_selected_EMS)):
save_flag = ut.model_save_through_validation(
fold,
epoch,
EMS=EMS,
selected_EMS=list_selected_EMS[i_tmp],
ES=list_ES[i_tmp],
model=model,
dir_save_model=list_dir_save_model[i_tmp],
metric_1=st.list_standard_eval[i_tmp],
metric_2='',
save_flag=False)
if Test_flag == True:
print(
"------------------ test _ test dataset --------------------------"
)
""" load data """
if fst.flag_cropping == True and fst.flag_eval_cropping == True:
print("eval : cropping")
dict_result = ut.eval_classification_model_cropped_input(
config, fold, test_loader, model, criterion_cls)
elif fst.flag_translation == True and fst.flag_eval_translation == True:
print("eval : assemble")
dict_result = ut.eval_classification_model_esemble(
config, fold, test_loader, model, criterion_cls)
elif fst.flag_MC_dropout == True:
dict_result = ut.eval_classification_model_MC_dropout(
config, fold, test_loader, model, criterion_cls)
else:
print("eval : whole image")
dict_result = ut.eval_classification_model(
config, fold, test_loader, model, criterion_cls)
acc = dict_result['Acc']
test_loss = dict_result['Loss']
""" pyplot """
EMS.test_acc.append(acc)
if fst.flag_loss_2 == True:
for tmp_i in range(len(st.lambda_aux)):
EMS.test_acc_aux[tmp_i].append(
dict_result['Acc_aux'][tmp_i])
EMS.test_loss.append(test_loss)
EMS.test_step.append(EMS.total_train_step)
print('number of test samples : {}'.format(len(
test_loader.dataset)))
print('Fold : %d, Epoch [%d/%d] test Loss = %f test Acc = %f' %
(fold, epoch, config.num_epochs, test_loss, acc))
""" learning rate decay"""
EMS.LR.append(optimizer.param_groups[0]['lr'])
# scheduler.step()
# scheduler.step(val_loss)
""" plot the chat """
if epoch % 1 == 0:
ut.plot_training_info_1(fold,
dir_pyplot,
EMS,
flag='percentile',
flag_match=False)
##TODO : early stop only if all of metric has been stopped
tmp_count = 0
for i in range(len(list_ES)):
if list_ES[i].early_stop == True:
tmp_count += 1
if tmp_count == len(list_ES):
break
""" release the model """
del model, EMS
torch.cuda.empty_cache()
def prepare_data(data_path_A,
data_path_B,
data_path_val_A,
data_path_val_B,
patch_size,
stride,
aug_times=1,
if_reseize=False):
# train
print('process training data')
scales = [1, 0.9, 0.8, 0.7]
files_A = glob.glob(os.path.join('datasets', data_path_A, '*.*'))
files_A.sort()
files_B = glob.glob(os.path.join('datasets', data_path_B, '*.*'))
files_B.sort()
h5f = h5py.File('train.h5', 'w')
train_num = 0
for i in range(len(files_A)):
img_A = cv2.imread(files_A[i])
img_B = cv2.imread(files_B[i])
h, w, c = img_A.shape
if if_reseize == True:
img_B = cv2.resize(img_B, (h, w), interpolation=cv2.INTER_CUBIC)
#h, w, c = img_B.shape
for k in range(len(scales)):
Img_A = cv2.resize(img_A, (int(h * scales[k]), int(w * scales[k])),
interpolation=cv2.INTER_CUBIC)
Img_A = np.expand_dims(Img_A[:, :, 0].copy(), 0)
Img_A = np.float32(normalize(Img_A))
Img_B = cv2.resize(img_B, (int(h * scales[k]), int(w * scales[k])),
interpolation=cv2.INTER_CUBIC)
Img_B = np.expand_dims(Img_B[:, :, 0].copy(), 0)
Img_B = np.float32(normalize(Img_B))
patches_A = Im2Patch(Img_A, win=patch_size, stride=stride)
patches_B = Im2Patch(Img_B, win=patch_size, stride=stride)
print("file: %s scale %.1f # samples: %d" %
(files_A[i], scales[k], patches_A.shape[3] * aug_times))
print("file: %s scale %.1f # samples: %d" %
(files_B[i], scales[k], patches_A.shape[3] * aug_times))
for n in range(patches_A.shape[3]):
data_A = patches_A[:, :, :, n].copy()
data_B = patches_B[:, :, :, n].copy()
h5f.create_dataset(str(train_num), data=[data_A, data_B])
train_num += 1
for m in range(aug_times - 1):
rand = np.random.randint(1, 8)
data_aug_A = data_augmentation(data_A, rand)
data_aug_B = data_augmentation(data_B, rand)
h5f.create_dataset(str(train_num) + "_aug_%d" % (m + 1),
data=[data_aug_A, data_aug_B])
train_num += 1
h5f.close()
# val
print('\nprocess validation data')
files_A.clear()
files_A = glob.glob(os.path.join('datasets', data_path_val_A, '*.*'))
files_A.sort()
files_B.clear()
files_B = glob.glob(os.path.join('datasets', data_path_val_B, '*.*'))
files_B.sort()
h5f = h5py.File('val.h5', 'w')
val_num = 0
for i in range(len(files_A)):
print("file: %s" % files_A[i])
img_A = cv2.imread(files_A[i])
img_A = np.expand_dims(img_A[:, :, 0], 0)
img_A = np.float32(normalize(img_A))
img_B = cv2.imread(files_B[i])
if if_reseize == True:
h, w, c = img_B.shape
img_B = cv2.resize(img_B, (h * 4, w * 4),
interpolation=cv2.INTER_CUBIC)
img_B = np.expand_dims(img_B[:, :, 0], 0)
img_B = np.float32(normalize(img_B))
h5f.create_dataset(str(val_num), data=[img_A, img_B])
val_num += 1
h5f.close()
print('training set, # samples %d\n' % train_num)
print('val set, # samples %d\n' % val_num)
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
# (train_img, train_lab),(test_img, test_lab) = utils.data_loader("CIFAR10")
model = san.san(sa_type=1, layers=(2, 1, 2, 4, 1), kernels=[3, 7, 7, 7, 7])
model.build(input_shape=(config.BATCH_SIZE, config.channels,
config.image_height, config.image_width))
model.summary()
train_img, train_lab, test_img, test_lab = utils.read_train_test_data(
"/Users/hamnamoieez/Desktop/Projects/self-attention-image-recognition/dataset"
)
train_img = utils.data_preprocess(train_img)
train_lab = utils.one_hot_encoder(train_lab)
X_train, X_val, y_train, y_val = utils.validation_data(train_img, train_lab)
train_generator, val_generator = utils.data_augmentation(
X_train, y_train, X_val, y_val)
# define loss and optimizer
loss_object = tf.keras.losses.CategoricalCrossentropy()
optimizer = tf.keras.optimizers.Adam()
train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.CategoricalAccuracy(name='train_accuracy')
valid_loss = tf.keras.metrics.Mean(name='valid_loss')
valid_accuracy = tf.keras.metrics.CategoricalAccuracy(name='valid_accuracy')
# @tf.function
# def get_optimizer(lr):
# return tf.keras.optimizers.Adam(learning_rate=lr)
def prepare_data(data_path, \
val_data_path, \
patch_size, \
stride, \
max_num_patches=None, \
aug_times=1, \
gray_mode=False):
r"""Builds the training and validations datasets by scanning the
corresponding directories for images and extracting patches from them.
Args:
data_path: path containing the training image dataset
val_data_path: path containing the validation image dataset
patch_size: size of the patches to extract from the images
stride: size of stride to extract patches
stride: size of stride to extract patches
max_num_patches: maximum number of patches to extract
aug_times: number of times to augment the available data minus one
gray_mode: build the databases composed of grayscale patches
"""
# training database
print('> Training database')
scales = [1, 0.9, 0.8, 0.7]
types = ('*.bmp', '*.png')
files = []
for tp in types:
files.extend(glob.glob(os.path.join(data_path, tp)))
files.sort()
if gray_mode:
traindbf = 'train_gray.h5'
valdbf = 'val_gray.h5'
else:
traindbf = 'train_rgb.h5'
valdbf = 'val_rgb.h5'
if max_num_patches is None:
max_num_patches = 5000000
print("\tMaximum number of patches not set")
else:
print("\tMaximum number of patches set to {}".format(max_num_patches))
train_num = 0
i = 0
with h5py.File(traindbf, 'w') as h5f:
while i < len(files) and train_num < max_num_patches:
imgor = cv2.imread(files[i])
# h, w, c = img.shape
for sca in scales:
img = cv2.resize(imgor, (0, 0), fx=sca, fy=sca, \
interpolation=cv2.INTER_CUBIC)
if not gray_mode:
# CxHxW RGB image
img = (cv2.cvtColor(img,
cv2.COLOR_BGR2RGB)).transpose(2, 0, 1)
else:
# CxHxW grayscale image (C=1)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img = np.expand_dims(img, 0)
img = normalize(img)
patches = img_to_patches(img, win=patch_size, stride=stride)
print("\tfile: %s scale %.1f # samples: %d" % \
(files[i], sca, patches.shape[3] * aug_times))
for nx in range(patches.shape[3]):
data = data_augmentation(patches[:, :, :, nx].copy(), \
np.random.randint(0, 7))
h5f.create_dataset(str(train_num), data=data)
train_num += 1
for mx in range(aug_times - 1):
data_aug = data_augmentation(data,
np.random.randint(1, 4))
h5f.create_dataset(str(train_num) + "_aug_%d" %
(mx + 1),
data=data_aug)
train_num += 1
i += 1
# validation database
print('\n> Validation database')
files = []
for tp in types:
files.extend(glob.glob(os.path.join(val_data_path, tp)))
files.sort()
h5f = h5py.File(valdbf, 'w')
val_num = 0
for i, item in enumerate(files):
print("\tfile: %s" % item)
img = cv2.imread(item)
if not gray_mode:
# C. H. W, RGB image
img = (cv2.cvtColor(img, cv2.COLOR_BGR2RGB)).transpose(2, 0, 1)
else:
# C, H, W grayscale image (C=1)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img = np.expand_dims(img, 0)
img = normalize(img)
h5f.create_dataset(str(val_num), data=img)
val_num += 1
h5f.close()
print('\n> Total')
print('\ttraining set, # samples %d' % train_num)
print('\tvalidation set, # samples %d\n' % val_num)
def prepare_data(data_path, patch_size, stride, aug_times=1):
# '''
# train
print('process training data')
scales = [1]
files = glob.glob(os.path.join('D:', 'NH-HAZE_train', 'HAZY', '*.png'))
# mix = list(range(len(files)))
# random.shuffle(mix)
# mix_train = mix[:int(len(files)*0.96)]
# mix_val = mix[int(len(files)*0.96):]
files.sort()
h5f = h5py.File('D:/train_input.h5', 'w')
train_num = 0
for i in range(len(files)):
Img = cv2.imread(files[i])
h, w, c = Img.shape
for k in range(len(scales)):
# Img = cv2.resize(img, (int(h*scales[k]), int(w*scales[k])), interpolation=cv2.INTER_CUBIC)
# Img = np.expand_dims(Img[:, :, :].copy(), 0)
Img = np.swapaxes(Img, 0, 2)
Img = np.swapaxes(Img, 1, 2)
Img = np.float32(normalize(Img))
# print(Img.shape)
patches = Im2Patch(Img, patch_size, stride)
# print(i)
print("file: %s scale %.1f # samples: %d" %
(files[i], scales[k], aug_times * patches.shape[3]))
for n in range(patches.shape[3]):
data = patches[:, :, :, n].copy()
# print(data.shape)
h5f.create_dataset(str(train_num), data=data)
train_num += 1
for m in range(aug_times - 1):
data_aug = data_augmentation(data, np.random.randint(1, 8))
h5f.create_dataset(str(train_num) + "_aug_%d" % (m + 1),
data=data_aug)
train_num += 1
h5f.close()
print('process training gt')
scales = [1]
files = glob.glob(os.path.join('D:', 'NH-HAZE_train', 'GT', '*.png'))
files.sort()
h5f = h5py.File('D:/train_gt.h5', 'w')
train_num = 0
for i in range(len(files)):
Img = cv2.imread(files[i])
h, w, c = Img.shape
for k in range(len(scales)):
# Img = cv2.resize(img, (int(h*scales[k]), int(w*scales[k])), interpolation=cv2.INTER_CUBIC)
# Img = np.expand_dims(Img[:, :, :].copy(), 0)
Img = np.swapaxes(Img, 0, 2)
Img = np.swapaxes(Img, 1, 2)
Img = np.float32(normalize(Img))
patches = Im2Patch(Img, patch_size, stride)
# print(i)
print("file: %s scale %.1f # samples: %d" %
(files[i], scales[k], aug_times * patches.shape[3]))
for n in range(patches.shape[3]):
data = patches[:, :, :, n].copy()
# print(data.shape)
h5f.create_dataset(str(train_num), data=data)
train_num += 1
for m in range(aug_times - 1):
data_aug = data_augmentation(data, np.random.randint(1, 8))
h5f.create_dataset(str(train_num) + "_aug_%d" % (m + 1),
data=data_aug)
train_num += 1
h5f.close()
# val
print('\nprocess validation data')
# files.clear()
files = glob.glob(os.path.join('D:', 'NH-HAZE_validation', 'HAZY',
'*.png'))
files.sort()
h5f = h5py.File('D:/val_input.h5', 'w')
val_num = 0
for i in range(len(files)):
print("file: %s" % files[i])
img = cv2.imread(files[i])
# img = np.expand_dims(img[:, :, :], 0)
img = np.swapaxes(img, 0, 2)
img = np.swapaxes(img, 1, 2)
img = np.float32(normalize(img))
# print(i)
# print(img.shape)
h5f.create_dataset(str(val_num), data=img)
val_num += 1
h5f.close()
# '''
print('\nprocess validation gt')
# files.clear()
files = glob.glob(os.path.join('D:', 'NH-HAZE_validation', 'GT', '*.png'))
files.sort()
h5f = h5py.File('D:/val_gt.h5', 'w')
val_num = 0
for i in range(len(files)):
print("file: %s" % files[i])
img = cv2.imread(files[i])
# img = np.expand_dims(img[:, :, :], 0)
img = np.swapaxes(img, 0, 2)
img = np.swapaxes(img, 1, 2)
img = np.float32(normalize(img))
# print(i)
# print(img.shape)
h5f.create_dataset(str(val_num), data=img)
val_num += 1
h5f.close()
# print('training set, # samples %d\n' % train_num)
print('val set, # samples %d\n' % val_num)
def gen_data(args):
"""
generate datasets for training and validation
"""
data_path = args.data_path
save_path = args.save_path
train = args.train
test = args.test
size = args.size
stride = args.stride
aug_times = args.aug_times
gray_mode = args.gray_mode
pic_type = args.pic_type
train_path = Path(data_path).joinpath("Train")
val_data_path = Path(data_path).joinpath("Test")
if save_path is not None:
save_path = Path(save_path)
if not save_path.exists():
save_path.mkdir()
files_train = {}
files_test = {}
for x in train_path.glob("*"):
if x.is_dir():
file_list_train = [str(f_train.absolute().resolve()) for f_train in x.glob(f"*.{pic_type}")]
files_train[x.name] = []
files_train[x.name].extend(file_list_train)
for y in val_data_path.glob("*"):
if y.is_dir():
file_list_test = [str(f_test.absolute().resolve()) for f_test in y.glob(f"*.{pic_type}")]
files_test[y.name] = []
files_test[y.name].extend(file_list_test)
if gray_mode:
train_h5 = 'train_gray.h5'
train_h5_label = 'train_gray_label.h5'
val_h5 = 'val_gray.h5'
val_h5_label = 'val_gray_label.h5'
else:
train_h5 = 'train_rgb.h5'
train_h5_label = 'train_rgb_label.h5'
val_h5 = 'val_rgb.h5'
val_h5_label = 'val_rgb_label.h5'
if train:
# load the training img and generate the dataset
f_train = h5py.File(save_path.joinpath(train_h5), 'w')
f_train_label = h5py.File(save_path.joinpath(train_h5_label), 'w')
train_num = 0
# k->label v->filename list
for k, v in files_train.items():
print(k)
print(v)
if len(v) == 0:
continue
# load a full size image
for f in v:
if gray_mode:
# H * W * C
t_pic = cv2.imread(f, cv2.IMREAD_GRAYSCALE)
else:
t_pic = cv2.imread(f, cv2.IMREAD_COLOR)
# BRG -> RGB
t_pic = t_pic[:, :, ::-1]
# HWC -> CHW
t_pic = np.transpose(t_pic, (2, 0, 1))
t_pic = normalize(t_pic)
# CHW * patch_size
patches = img_2_patches(t_pic, size, stride)
# Control the maximum sample from a single image
patches = patches[:, :, :, :2400]
# dealing with every patch
print(f"training file:{f} --> ##{patches.shape[3]}##sample")
for nx in range(patches.shape[3]):
data = patches[:, :, :, nx]
f_train.create_dataset(str(train_num), data=data)
f_train_label.create_dataset(str(train_num), data=np.array(get_label(int(k))))
train_num += 1
# data augmentation
for mx in range(aug_times):
data_aug = data_augmentation(patches[:, :, :, nx].copy(), np.random.randint(1, 8))
f_train.create_dataset(str(train_num), data=data_aug)
f_train_label.create_dataset(str(train_num), data=np.array(get_label(int(k))))
train_num += 1
f_train.close()
f_train_label.close()
print(f"the number of training images:{train_num}")
if test:
# Gen Test Data
f_test = h5py.File(save_path.joinpath(val_h5), 'w')
f_test_label = h5py.File(save_path.joinpath(val_h5_label), 'w')
# k->label v->filename list
val_num = 0
for k, v in files_test.items():
print(k)
print(v)
if len(v) == 0:
continue
# load full size image
for f in v:
if gray_mode:
# H * W * C
t_pic = cv2.imread(f, cv2.IMREAD_GRAYSCALE)
else:
t_pic = cv2.imread(f, cv2.IMREAD_COLOR)
# BRG -> RGB
t_pic = t_pic[:, :, ::-1]
# HWC -> CHW
t_pic = np.transpose(t_pic, (2, 0, 1))
t_pic = normalize(t_pic)
# CHW * patch_size
patches = img_2_patches(t_pic, size, stride)
# dealing with every patch
print(f"validation file:{f} --> ##{patches.shape[3]}##sample")
for nx in range(patches.shape[3]):
data = patches[:, :, :, nx]
f_test.create_dataset(str(val_num), data=data)
f_test_label.create_dataset(str(val_num), data=np.array(get_label(int(k))))
val_num += 1
f_test.close()
f_test_label.close()
print(f"the number of validation images:{val_num}")
def main():
global args, config, last_epoch, best_prec, writer
writer = SummaryWriter(log_dir=args.work_path + '/event')
# read config from yaml file
with open(args.work_path + '/config.yaml') as f:
config = yaml.load(f)
# convert to dict
config = EasyDict(config)
logger.info(config)
# define netowrk
net = get_model(config)
logger.info(net)
logger.info(" == total parameters: " + str(count_parameters(net)))
# CPU or GPU
device = 'cuda' if config.use_gpu else 'cpu'
# data parallel for multiple-GPU
if device == 'cuda':
net = torch.nn.DataParallel(net)
cudnn.benchmark = True
net.to(device)
#smart noise
layer_inputs = []
layer_outputs = []
grad_inputs = []
grad_outputs = []
def forward_hook(module, input, output):
layer_inputs.append(input[0].detach())
layer_outputs.append(output.detach())
def backward_hook(module, grad_input, grad_output):
grad_inputs.append(grad_input[0].detach())
grad_outputs.append(grad_output[0].detach())
def input_hook(grad):
grad_inputs.append(grad)
def output_hook(grad):
grad_outputs.append(grad)
# for p in net.modules():
# if isinstance(p, nn.Conv2d):
# p.register_forward_hook(forward_hook)
# p.register_backward_hook(backward_hook)
layers = []
index = 0
# for p in net.modules():
# if isinstance(p, nn.Conv2d):
# in_planes = p.in_channels
# planes = p.out_channels
# kernel_size = p.kernel_size[0]
# padding = p.padding[0]
# stride = p.stride[0]
# # if index ==0:
# # layer = Conv1(in_planes, planes, kernel_size, stride, padding)
# # else:
# layer = Conv(in_planes, planes, kernel_size, stride, padding)
# layers.append(layer)
# index +=1
groups = []
group_index = 0
mults = 1
index_list = [5, 10, 15, 20, 25, 30, 35, 40, 45, 50]
for p in net.modules():
if isinstance(p, BasicBlock):
# group.load_state_dict(p.state_dict())
# group.load_state_dict(p.state_dict())
downsample = None
if p.downsample is not None:
downsample = copy.deepcopy(p.downsample)
# tmp = copy.deepcopy(p.downsample)
group = BasicGroup(p.conv_1.in_channels,
p.conv_1.out_channels,
stride=p.stride,
downsample=downsample)
group.to(device)
if group_index in index_list:
mults *= 2
group.eps *= mults
groups.append(group)
group_index += 1
# print('yes')
crit = Layer_loss()
# define loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(net.parameters(),
config.lr_scheduler.base_lr,
momentum=config.optimize.momentum,
weight_decay=config.optimize.weight_decay,
nesterov=config.optimize.nesterov)
# resume from a checkpoint
last_epoch = -1
best_prec = 0
if args.work_path:
ckpt_file_name = args.work_path + '/' + config.ckpt_name + '.pth.tar'
if args.resume:
best_prec, last_epoch = load_checkpoint(ckpt_file_name,
net,
optimizer=optimizer)
# load training data, do data augmentation and get data loader
transform_train = transforms.Compose(data_augmentation(config))
transform_test = transforms.Compose(
data_augmentation(config, is_train=False))
train_loader, test_loader = get_data_loader(transform_train,
transform_test, config)
logger.info(" ======= Training =======\n")
for epoch in range(last_epoch + 1, config.epochs):
lr = adjust_learning_rate(optimizer, epoch, config)
writer.add_scalar('learning_rate', lr, epoch)
train(train_loader, net, criterion, optimizer, epoch, device,\
layer_inputs, layer_outputs, grad_inputs, grad_outputs, layers, crit, groups)
if epoch == 0 or (
epoch +
1) % config.eval_freq == 0 or epoch == config.epochs - 1:
test(test_loader, net, criterion, optimizer, epoch, device,\
layer_inputs, layer_outputs, grad_inputs, grad_outputs)
writer.close()
logger.info(
"======== Training Finished. best_test_acc: {:.3f}% ========".format(
best_prec))
def augmentation(phase, image, resize, crop_height=None, crop_width=None):
image_aug = data_augmentation(phase, image, resize, crop_height,
crop_width)
# image_aug = Image.fromarray(image_aug)
return image_aug
