def yield_data(aug_name = ['gaussian_blur'], aug_proportion = [0.2], search = False): batch_image_numpy = np.zeros([4, 256, 256, 3]) batch_label_numpy = np.zeros([4, 256, 256, 4]) if search: read_data_list = np.array([[1,0,1017],[2,0,543],[3,0,85],[4,0,92]]) npy_dir = '../../final_dataset/valid_search/class' else: read_data_list = np.array([[1,0,2445],[2,0,1588],[3,0,401],[4,0,369]]) npy_dir = '../../final_dataset/train/class' while True: permutation = np.random.permutation(4) for i in range(4): read_list = read_data_list[i] batch_image_numpy[permutation[i]], batch_label_numpy[permutation[i]]\ = data_aug(np.load(npy_dir+str(read_list[0])+'_image/'+str(read_list[1])+'.npy'), np.load(npy_dir+str(read_list[0])+'_label/'+str(read_list[1])+'.npy').astype(np.float32), aug_name, aug_proportion) read_data_list[:,1] = (read_data_list[:,1]+1)%read_data_list[:,2] yield batch_image_numpy, batch_label_numpy
def _load_imgs(self):
"""Load imgs"""
for index in range(len(self.landmarks)):
# load img
if self.color:
img = cv2.imread(self.img_paths[index])
else:
img = cv2.imread(self.img_paths[index], cv2.IMREAD_GRAYSCALE)
if img is None:
logger("{} img read error".format(self.img_paths[index]))
continue
bbox = [int(_) for _ in self.bboxes[index]]
# normalize landmark
landmark = self.landmarks[index]
landmark_normalized = normalize_data(landmark)
# data augment
face = cv2.resize(get_face(img, bbox),
(self.img_size, self.img_size))
name = self.img_paths[index].lstrip(
data_param['img_root_dir']).replace("/", "_")
faces, landmarks, occlusions, names = \
data_aug(face=face, landmark=landmark_normalized, name=name)
self.faces.extend(faces)
self.aug_landmarks.extend(landmarks)
self.occlusions.extend(occlusions)
self.names.extend(names)
if self.print_debug and (index + 1) % 500 == 0:
logger("processed {} images".format(index + 1))
self.data_size = len(self.occlusions)
for index in range(len(self.aug_landmarks)):
self.aug_landmarks[index] = np.multiply(
self.aug_landmarks[index],
np.array([self.img_size, self.img_size]))
del self.landmarks
def yield_data(dataset_path,
aug_name=['gaussian_blur'],
aug_proportion=[0.2],
search=False):
'''
we empirically found that add same number of images for each class in one batch can significantly increase the accuracy
the batch size in the searching process is 4, and 1 for each class (totally 4 classes)
Args:
dataset_path: dictionary path of the dataset
aug_name: add augmentation name in the list, the supported methods including 'random_rotate', 'random_flip'
'gaussian_blur', 'median_blur' and 'elastic_transform'
aug_proportion: the probability to do each of the augmentation method, the length of this list must be the
same as the aug_name
search: if this flag is True, the data is generated for searching, if False, the data is generated for rebuilding
'''
batch_image_numpy = np.zeros([4, 256, 256, 3])
batch_label_numpy = np.zeros([4, 256, 256, 4])
if search:
read_data_list = np.array([[1, 0, 1017], [2, 0, 543], [3, 0, 85],
[4, 0, 92]])
# 1017 is searching process's training image number in class 1, and so on
npy_dir = dataset_path + '/valid_search/class'
else:
read_data_list = np.array([[1, 0, 2445], [2, 0, 1588], [3, 0, 401],
[4, 0, 369]])
# 2445 is rebuilding process's training image number in class 1, and so on
npy_dir = dataset_path + '/train/class'
while True:
permutation = np.random.permutation(
4) # randomly shuffle the order of 4 class images
for i in range(4):
read_list = read_data_list[i]
batch_image_numpy[permutation[i]], batch_label_numpy[permutation[i]]\
= data_aug(np.load(npy_dir+str(read_list[0])+'_image/'+str(read_list[1])+'.npy'),
np.load(npy_dir+str(read_list[0])+'_label/'+str(read_list[1])+'.npy').astype(np.float32),
aug_name, aug_proportion)
read_data_list[:,
1] = (read_data_list[:, 1] + 1) % read_data_list[:, 2]
yield batch_image_numpy, batch_label_numpy
def __init__(self, config, model, model2, device, train_loader, val_loader,
accuracy_criterion, loss_criterion, optimizer):
self.config = config
self.max_val_iter = self.config.patience
self.model = model
self.model2 = model2
self.device = device
self.output_device = self.device
self.train_loader = train_loader
self.val_loader = val_loader
self.accuracy_criterion = accuracy_criterion
self.loss_criterion = loss_criterion
self.optimizer = optimizer
self.train_loss = utils.RunningAverage()
self.train_accuracy_label1 = utils.RunningAverage()
self.train_accuracy_label2 = utils.RunningAverage()
self.print_term = 20
self.flag_reduce_lr = False
self.data_aug = utils.data_aug()
self.num_params_model = self.get_num_learnable_params(self.model)
print("========> The number of learnable parameters for model1 is {}".
format(self.num_params_model))
self.num_params_model2 = self.get_num_learnable_params(self.model2)
print("========> The number of learnable parameters for model2 is {}".
format(self.num_params_model2))
self.writer = SummaryWriter(
os.path.join(self.config.checkpoint_dir, 'logs'))
# load the saved checkpoint - update model2 parameters (no need to update model parameters since they are fixed)
if self.config.consume:
self.state = utils.load_checkpoint(
os.path.join(self.config.checkpoint_dir,
"last_checkpoint.pytorch"), self.output_device,
self.model2, self.optimizer)
self.num_epoch = self.state['epochs']
self.num_iter = self.state['iterations']
self.best_val_accuracy = self.state['best_val_accuracy']
self.config = self.state['config']
self.config.consume = True
self.val_iter = self.state['val_iter']
self.lr_record = self.state['lr_record']
self.dict_train_loss = self.state['train_loss']
self.dict_train_accuracy_label1 = self.state[
'train_accuracy_label1']
self.dict_train_accuracy_label2 = self.state[
'train_accuracy_label2']
self.dict_val_loss = self.state['val_loss']
self.dict_val_accuracy_label1 = self.state['val_accuracy_label1']
self.dict_val_accuracy_label2 = self.state['val_accuracy_label2']
else:
self.num_epoch = 0
self.num_iter = 0
self.best_val_accuracy = 0.
self.val_iter = 0
self.lr_record = self.config.learning_rate
self.dict_train_loss = {}
self.dict_train_accuracy_label1 = {}
self.dict_train_accuracy_label2 = {}
self.dict_val_loss = {}
self.dict_val_accuracy_label1 = {}
self.dict_val_accuracy_label2 = {}
if self.config.lr_decay_mode == 'accumulated':
assert self.config.lr_decay_epoch < self.max_val_iter
print(self.model2)
def train(self, sess, opt):
# set hyper-parameters
self.sess = sess
model_name = opt.model_name
patch_size = opt.patch_size
batch_size = opt.batch_size
nEpoch = opt.epochs
lr = opt.lr
lr_decay = opt.lr_decay
weight_decay = opt.weight_decay
train_path = opt.train_path
sigma = opt.sigma
self.c_dim = opt.c_dim
# build network(s)
self.inputs = tf.placeholder(tf.float32, [None, None, None, self.c_dim], name='inputs')
self.labels = tf.placeholder(tf.float32, [None, None, None, self.c_dim], name='labels')
self.results = self._build_model(weight_decay)
self.lr = tf.placeholder(tf.float32, name='learning_rate') # to add decay
# self.loss = (0.5 / batch_size) * tf.nn.l2_loss(self.results - self.labels)
self.loss = (0.5 / batch_size) * tf.reduce_sum(tf.square(self.results - self.labels))
self.optimizer = tf.train.AdamOptimizer(self.lr, name='AdamOptimizer')
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self.train_op = self.optimizer.minimize(self.loss)
# load data
if not os.path.exists(train_path):
print("File: \"%s\" not found." % train_path)
exit()
# read train dataset from tfrecord
def datamap(record):
keys_to_feature = {
'inputs': tf.FixedLenFeature([], tf.string),
}
tf_features = tf.parse_single_example(record,features=keys_to_feature)
inputs = tf.decode_raw(tf_features['inputs'], tf.uint8)
inputs = tf.reshape(inputs, [patch_size, patch_size])
return inputs
self.sess.run(tf.global_variables_initializer())
tf.train.start_queue_runners(sess=self.sess)
losses = []
losses_aver = []
# validate related
validate_set = opt.validate_set
validate_dir = opt.validate_dir
test_datas = []
test_labels = []
test_names = []
path = validate_dir + str(sigma) + '/' + validate_set # file folder
files = os.listdir(path)
for file in files:
imageName = os.path.splitext(file)[0]
mat = scipy.io.loadmat(path + "/" + imageName + ".mat")
original = mat['img']
input_ = mat['noisyimg']
original = original.astype(np.float32)
input_ = input_.astype(np.float32)
input_ = input_ / 255.0
label_ = original / 255.0
test_datas.append(input_)
test_labels.append(label_)
test_names.append(imageName)
psnr_summary = []
ssim_summary = []
# training
flag = False
saver = tf.train.Saver(tf.global_variables(), max_to_keep=50)
for epoch in range(1, nEpoch+1):
dataset = tf.data.TFRecordDataset([train_path], num_parallel_reads=4)\
.map(datamap, num_parallel_calls=batch_size)\
.shuffle(buffer_size=batch_size*4*patch_size**2, reshuffle_each_iteration=True)\
.batch(batch_size)\
.repeat(1)
iterator = dataset.make_one_shot_iterator()
inputs_batch = iterator.get_next()
step = 0
start_time = time.time()
total_loss = 0
try:
step = 0
while True:
inputs_val = self.sess.run(inputs_batch)
mini_batch = np.array(inputs_val, dtype=np.float32)/255
mini_batch = np.reshape(mini_batch, [mini_batch.shape[0], patch_size, patch_size, 1])
rnd_aug = np.random.randint(8,size=mini_batch.shape[0])
for i in range(mini_batch.shape[0]):
mini_batch[i,:,:,:] = np.reshape(data_aug(
np.reshape(mini_batch[i,:,:,:], [patch_size,patch_size]),
rnd_aug[i]),[1, patch_size, patch_size, 1])
label_b = sigma / 255.0 * np.random.normal(size=np.shape(mini_batch))
input_b = mini_batch + label_b
if epoch < lr_decay:
_, loss, result = self.sess.run([self.train_op, self.loss, self.results], feed_dict={
self.inputs:input_b, self.labels: label_b,
self.lr:lr, self.is_training: True})
else:
_, loss, result = self.sess.run([self.train_op, self.loss, self.results], feed_dict={
self.inputs:input_b, self.labels: label_b,
self.lr:lr/10, self.is_training: True})
if flag:
for ind in range(batch_size):
tmppath = './tmp/'
if not os.path.exists(tmppath):
os.mkdir(tmppath)
scipy.misc.imsave(tmppath+str(ind)+'_in.png', np.squeeze(input_b[ind,:,:,0]))
scipy.misc.imsave(tmppath+str(ind)+'_resi_l.png', np.squeeze(label_b[ind,:,:,0]))
scipy.misc.imsave(tmppath+str(ind)+'_resi.png', np.squeeze(result[ind,:,:,0]))
scipy.misc.imsave(tmppath+str(ind)+'_label.png', (np.squeeze(input_b[ind,:,:,0]) - np.squeeze(label_b[ind,:,:,0])))
res = np.squeeze(input_b[ind,:,:,0]) - np.squeeze(result[ind,:,:,0])
res[res < 0] = 0
res[res > 1.] = 1.
scipy.misc.imsave(tmppath+str(ind)+'.png', (res))
flag = False
# print('iter: [%2d] Time: %4.2 Loss: %.6f\n' % (iter, time.time() - start_time, loss))
total_loss = total_loss + loss
step = step + 1
if step%500 == 0:
flag = True
if step%50 == 0:
losses.append(loss)
print("Epoch: [{}] Iterations: [{}] Time: {} Loss: {}".format(epoch, step, time.time() - start_time, loss))
except tf.errors.OutOfRangeError:
losses_aver.append(total_loss/step)
print('Done training for %d epochs, %d steps. Time: %f, AverLoss: %f' % (epoch, step, time.time() - start_time, total_loss/step))
checkpoint = opt.checkpoint_path
if not os.path.exists(checkpoint):
os.mkdir(checkpoint)
print("[*] Saving model...{}".format(epoch))
saver.save(self.sess, os.path.join(checkpoint, opt.model_name), global_step=epoch)
psnr_aver, ssim_aver = self.validate(validate_set, test_datas, test_labels, test_names,epoch, model_name)
psnr_summary.append(np.mean(psnr_aver))
ssim_summary.append(np.mean(ssim_aver))
print("model {} PSNR={} SSIM={}".format(epoch, np.mean(psnr_aver), np.mean(ssim_aver)))
sio.savemat(model_name + '_'+validate_set+'_validate.mat', {'psnr': np.array(psnr_summary), 'ssim':np.array(ssim_summary)})
def test_train(self, sess, opt):
self.sess = sess
sigma = opt.sigma
model_start = opt.model_start
model_stop = opt.model_stop
checkpoint = opt.checkpoint_dir
# test_set = opt.test_set
test_set = 'trainset'
test_dir = opt.test_dir
self.c_dim = opt.c_dim
self.inputs = tf.placeholder(tf.float32, [None, None, None, self.c_dim], name='inputs')
# self.labels = tf.placeholder(tf.float32, [None, None, None, self.c_dim], name='labels')
self.results = self._build_model()
self.sess.run(tf.global_variables_initializer())
# load test data
# test_datas = []
# test_labels = []
# test_names = []
# path = test_dir + str(sigma) + '/' + test_set # file folder
# files = os.listdir(path)
# for file in files:
# imageName = os.path.splitext(file)[0]
# mat = scipy.io.loadmat(path + "/" + imageName + ".mat")
# original = mat['img']
# input_ = mat['noisyimg']
# original = original.astype(np.float32)
# input_ = input_.astype(np.float32)
# input_ = input_ / 255.0
# label_ = original / 255.0
# test_datas.append(input_)
# test_labels.append(label_)
# test_names.append(imageName)
patch_size = 40
batch_size = 64
def datamap(record):
keys_to_feature = {
'inputs': tf.FixedLenFeature([], tf.string),
}
tf_features = tf.parse_single_example(record,features=keys_to_feature)
inputs = tf.decode_raw(tf_features['inputs'], tf.uint8)
inputs = tf.reshape(inputs, [patch_size, patch_size])
return inputs
dataset = tf.data.TFRecordDataset(['./data/imdb_40_128_V1.tfrecords'], num_parallel_reads=4)\
.map(datamap, num_parallel_calls=batch_size)\
.shuffle(buffer_size=batch_size*4*patch_size**2, reshuffle_each_iteration=True)\
.batch(batch_size)\
.repeat(1)
iterator = dataset.make_one_shot_iterator()
inputs_batch = iterator.get_next()
inputs_val = self.sess.run(inputs_batch)
mini_batch = np.array(inputs_val, dtype=np.float32)/255
mini_batch = np.reshape(mini_batch, [mini_batch.shape[0], patch_size, patch_size, 1])
rnd_aug = np.random.randint(8,size=mini_batch.shape[0])
for i in range(mini_batch.shape[0]):
mini_batch[i,:,:,:] = np.reshape(data_aug(
np.reshape(mini_batch[i,:,:,:], [patch_size,patch_size]),
rnd_aug[i]),[1, patch_size, patch_size, 1])
label_b = sigma / 255.0 * np.random.normal(size=np.shape(mini_batch))
input_b = mini_batch + label_b
# validate
# saver = tf.train.import_meta_graph(checkpoint+'/DnCNN-'+str(model_stop)+'.meta')
saver = tf.train.Saver()
psnr_summary = []
ssim_summary = []
for model_id in range(model_start,model_stop+1):
# load pre-trained model
print("[*] Reading checkpoint... [%d]" %model_id)
saver.restore(self.sess, checkpoint+'/DnCNN-'+str(model_id))
out_path = './results_DnCNN/'+test_set+'/'+str(model_id)
if not os.path.exists(out_path):
os.makedirs(out_path)
psnr_aver = []
ssim_aver = []
for i in range(batch_size):
input_data = np.squeeze(input_b[i, ...])
label_data = np.squeeze(label_b[i, ...])
imageName = str(i)
result = self.sess.run([self.results], feed_dict={
self.inputs:input_data[np.newaxis,:,:, np.newaxis],
self.is_training: False})
result = np.squeeze(result)
# residual = input_data - result
residual = result
scipy.misc.imsave(out_path+'/'+imageName+'_resi.png', residual)
input_= input_data
input_[input_ < 0] = 0
input_[input_ > 1.] = 1.
scipy.misc.imsave(out_path+'/'+imageName+'_in.png', input_)
scipy.misc.imsave(out_path+'/'+imageName+'_label.png', label_data)
result = input_data-result
result[result < 0] = 0
result[result > 1.] = 1.
psnr_aver.append(c_psnr(result, label_data))
ssim_aver.append(c_ssim(result, label_data))
scipy.misc.imsave(out_path+'/'+imageName+'.png', result)
print("model {} PSNR={} SSIM={}".format(model_id, np.mean(psnr_aver), np.mean(ssim_aver)))
psnr_summary.append(np.mean(psnr_aver))
ssim_summary.append(np.mean(ssim_aver))
sio.savemat('DnCNN_'+test_set+'_resArray.mat', {'psnr': np.array(psnr_summary), 'ssim':np.array(ssim_summary)})
return