def denoiser_train(denoiser, lr):
noisy_eval_files = glob('./data/train/noisy/*.png')
noisy_eval_files = sorted(noisy_eval_files)
print(noisy_eval_files)
eval_files = glob('./data/train/original/*.png')
eval_files = sorted(eval_files)
denoiser.train(eval_files, noisy_eval_files, batch_size=args.batch_size, ckpt_dir=args.ckpt_dir, epoch=args.epoch, lr=lr)
def denoiser_train(denoiser, lr):
#
# load noisy and clean data
#
print("[*] Loading data...")
#--------------------------------------------------------------------------
# apply log(x+1) to the raw value
#--------------------------------------------------------------------------
noisy_data = np.load(
'../../model_input/rand2d/rand2d_noisy_pats_1e+05.npy')
clean_data = np.load(
'../../model_input/rand2d/rand2d_clean_pats_1e+05.npy')
# print noisy_data.shape , clean_data.shape
noisy_data = np.log(noisy_data + 1.)
clean_data = np.log(clean_data + 1.)
denoiser.train(noisy_data,
clean_data,
batch_size=args.batch_size,
ckpt_dir=args.ckpt_dir,
epoch=args.epoch,
lr=lr)
def denoiser_train(denoiser, lr):
eval_files = natsort.natsorted(
glob('./data/test/{}/*.png'.format(args.eval_set)))
denoiser.train(args.data,
eval_files,
batch_size=args.batch_size,
epoch=args.epoch,
lr=lr)
def denoiser_train(denoiser, lr):
with load_data(filepath='./data/img_clean_pats.npy') as data:
# if there is a small memory, please comment this line and uncomment the line99 in model.py
data = data.astype(np.float32) / 255.0 # normalize the data to 0-1
eval_files = glob('./data/test/{}/*.png'.format(args.eval_set))
eval_data = load_images(eval_files) # list of array of different size, 4-D, pixel value range is 0-255
denoiser.train(data, eval_data, batch_size=args.batch_size, ckpt_dir=args.ckpt_dir, epoch=args.epoch, lr=lr,
sample_dir=args.sample_dir)
def denoiser_train(denoiser, lr):
input_data = np.load('./data/input_data_pats.npy')
label_data = np.load('./data/label_data_pats.npy')
denoiser.train(input_data,
label_data,
batch_size=args.batch_size,
ckpt_dir=args.ckpt_dir,
epoch=args.epoch,
lr=lr)
def denoiser_train(denoiser, lr):
#
# load noisy and clean data
#
print("[*] Loading data...")
#--------------------------------------------------------------------------
# apply log(x+1) to the raw value, and select maxV to normalize
#--------------------------------------------------------------------------
noisy_data = np.load(
'../../model_input/rand2d_homo_partial/rand2d_noisy_pats_1e+04.npy')
clean_data = np.load(
'../../model_input/rand2d_homo_partial/rand2d_clean_pats_1e+04.npy')
print noisy_data.shape, clean_data.shape
#--- max value ---#
print "\nprev log(x+1)"
print "noisy_max \t clean_max \t max"
max_noisy, max_clean = np.amax(noisy_data), np.amax(clean_data)
maxV = max(max_noisy, max_clean)
print max_noisy, max_clean, maxV
#--- apply log(x + 1) ---#
noisy_data = np.log(noisy_data + 1.)
clean_data = np.log(clean_data + 1.)
print "\nafter log(x+1)"
print "noisy_max \t clean_max \t max"
max_noisy, max_clean = np.amax(noisy_data), np.amax(clean_data)
maxV = max(max_noisy, max_clean)
print max_noisy, max_clean, maxV
maxV = float(int(maxV) + 3) # add extra 3 for upper bound
if maxV <= 25.0:
maxV = 25.0
print "Using %f for maxV (after log)" % maxV
print "\nSaving maxV to use in matlab..."
sio.savemat('maxV.mat', dict(maxV=maxV))
print "\nDone!"
#--- normalize with maxV ---#
noisy_data = noisy_data / maxV
clean_data = clean_data / maxV
#--- run training ---#
denoiser.train(noisy_data,
clean_data,
batch_size=args.batch_size,
ckpt_dir=args.ckpt_dir,
epoch=args.epoch,
lr=lr)
def denoiser_train(denoiser, lr):
#
# load noisy and clean data
#
print("[*] Loading data...")
#--------------------------------------------------------------------------
# apply log(x+1) to the raw value, and select maxV to normalize
#--------------------------------------------------------------------------
noisy_data = np.load('../../model_input/osa/osa_noisy_pats.npy')
clean_data = np.load('../../model_input/osa/osa_clean_pats.npy')
print noisy_data.shape , clean_data.shape
#--- max value ---#
print "\nprev log(x+1)"
print "noisy_max \t clean_max \t max"
max_noisy, max_clean = np.amax(noisy_data), np.amax(clean_data)
maxV = max(max_noisy, max_clean)
print max_noisy, max_clean, maxV
#--- apply log(x + 1) ---#
noisy_data = np.log(noisy_data + 1.)
clean_data = np.log(clean_data + 1.)
#print "\nafter log(x+1)"
#print "noisy_max \t clean_max \t max"
#max_noisy, max_clean = np.amax(noisy_data), np.amax(clean_data)
## maxV = max(max_noisy, max_clean)
#print max_noisy, max_clean
#maxV = 25.
#print "Using %f for maxV (after log)" % maxV
#print "\nSaving maxV to use in matlab..."
#sio.savemat('maxV.mat', dict(maxV=maxV))
#print "\nDone!"
##--- normalize with maxV ---#
#noisy_data = noisy_data / maxV
#clean_data = clean_data / maxV
#--- run training ---#
denoiser.train(noisy_data,
clean_data,
batch_size=args.batch_size,
ckpt_dir=args.ckpt_dir,
epoch=args.epoch,
lr=lr)
def denoiser_train(denoiser, lr):
with load_data(filepath='./noisydata/ndct/train/raw_float_pats.npy') as ndct_data, load_data(filepath='./noisydata/sparse/train/raw_float_pats.npy') as ldct_data:
# if there is a small memory, please comment this line and uncomment the line99 in model.py
ldct_eval_files = sorted(glob('./noisydata/sparse/train/*img.flt'.format(args.eval_set)))
print(ldct_eval_files)
ldct_eval_data = load_floats(ldct_eval_files) # list of array of different size, 4-D, pixel value range is 0-255
ndct_eval_files = sorted(glob('./noisydata/ndct/train/*img.flt'.format(args.eval_set)))
print(ndct_eval_files)
ndct_eval_data = load_floats(ndct_eval_files) # list of array of different size, 4-D, pixel value range is 0-255
denoiser.train(ndct_data, ldct_data, ndct_eval_data, ldct_eval_data, batch_size=args.batch_size, ckpt_dir=args.ckpt_dir, epoch=args.epoch, lr=lr,
sample_dir=args.sample_dir)
def conv_denoise_train(denoiser, server, task_index, lr):
denoise_files = glob('./data/test/{}/*.jpg'.format(args.denoise_set))
noise_files = glob('./data/test/{}/*.jpg'.format(args.denoise_set +
'_nodenoise'))
denoiser.train(server,
denoise_files,
noise_files,
width=args.img_width,
height=args.img_height,
ckpt_dir=args.ckpt_dir,
epoch=args.epoch,
lr=lr,
task_index=task_index)
def denoiser_train(denoiser, lr):
noisy_eval_files = glob(f'{args.dir}/train/noisy/*.png')
noisy_eval_files = sorted(noisy_eval_files)
print(noisy_eval_files)
eval_files = glob(f'{args.dir}/train/original/*.png')
eval_files = sorted(eval_files)
denoiser.train(eval_files,
noisy_eval_files,
batch_size=args.batch_size,
ckpt_dir=os.path.join(args.dir, args.ckpt_dir),
epoch=args.epoch,
lr=lr,
evaluate_files=args.evaluate_files)
def denoiser_train(denoiser, lr):
train_files = glob('./data/{}/*.png'.format('Train400'))
eval_files = glob('./data/{}/*.png'.format(args.eval_set))
data = np.asarray(load_images(train_files))
eval_data = load_images(eval_files)
denoiser.train(data,
eval_data,
batch_size=args.batch_size,
ckpt_dir=args.ckpt_dir,
epoch=args.epoch,
lr=lr,
sample_dir=args.sample_dir)
'''
def denoiser_train(denoiser, lr):
with load_data(filepath='./data/img_clean_pats.npy') as data:
# if there is a small memory, please comment this line and uncomment the line99 in model.py
data = data.astype(np.float32) / 255.0 # normalize the data to 0-1
eval_files = glob('./data/test/{}/*.png'.format(args.eval_set))
eval_data = load_images(
eval_files
) # list of array of different size, 4-D, pixel value range is 0-255
denoiser.train(data,
eval_data,
batch_size=args.batch_size,
ckpt_dir=args.ckpt_dir,
epoch=args.epoch,
lr=lr,
sample_dir=args.sample_dir)
def denoiser_train(denoiser, lr):
ndct_train = sorted(glob(args.ndct_train_file_path))
ldct_train = sorted(glob(args.ldct_train_file_path))
ndct_eval_data = sorted(glob(args.ndct_test_file_path))
ldct_eval_data = sorted(glob(args.ldct_test_file_path))
denoiser.train(ndct_train,
ldct_train,
ndct_eval_data,
ldct_eval_data,
lr,
ckpt_dir=args.ckpt_dir,
num_epochs=args.num_epochs,
sample_dir=args.sample_dir,
buffer_size=args.buffer_size)
def denoiser_train(denoiser):
print(args.save_dir + '/' + args.results_clean + '_' + database_output +
'.npy')
print(args.save_dir + '/' + args.results_noisy + '_' + database_output +
'.npy')
with load_data(filepath=args.save_dir + '/' + args.results_clean + '_' +
database_output + '.npy') as data_clean:
with load_data(filepath=args.save_dir + '/' + args.results_noisy +
'_' + database_output + '.npy') as data_noisy:
# if there is a small memory, please comment this line and uncomment the line99 in model.py
data_clean = data_clean.astype(
np.float32) / 255.0 # normalize the data to 0-1
data_noisy = data_noisy.astype(
np.float32) / 255.0 # normalize the data to 0-1
eval_noisy_files = sorted(
glob('./data/test/{}/*.png'.format(args.eval_noisy_set)))
eval_clean_files = sorted(
glob('./data/test/{}/*.png'.format(args.eval_clean_set)))
eval_data_noisy = load_images(
eval_noisy_files
) # list of array of different size, 4-D, pixel value range is 0-255
eval_data_clean = load_images(
eval_clean_files
) # list of array of different size, 4-D, pixel value range is 0-255
print("work")
numBatch = int(data_clean.shape[0] / args.batch_size)
max_iter_number = 51200
epoches = args.epoch
if numBatch * epoches < max_iter_number:
epoches = round(max_iter_number / numBatch)
lr = args.lr * np.ones([epoches])
lr[30:] = lr[0] / 10.0
denoiser.train(data_clean,
data_noisy,
eval_data_clean,
eval_data_noisy,
batch_size=args.batch_size,
ckpt_dir=checkpoint_dir,
epoch=args.epoch,
lr=lr,
sample_dir=sample_dir,
logs_dir=logs_dir)
def conv_patch_denoise_train(denoiser, server=None, task_index=0):
with load_data(filepath='./data/img_denoise_pats.npy',
rand=False) as data_denoise:
with load_data(filepath='./data/img_noise_pats.npy',
rand=False) as data_noise:
data_denoise = data_denoise.astype(
np.float32) / 255.0 # normalize the data to 0-1
data_noise = data_noise.astype(
np.float32) / 255.0 # normalize the data to 0-1
denoiser.train(server,
data_denoise,
data_noise,
'./data/test/{}/*.jpg'.format(args.test_set),
args.test_dir,
128,
ckpt_dir=args.ckpt_dir,
epoch=args.epoch,
task_index=task_index)
def cmp_denoise_train(server, task_index, num_worker=7):
with load_data(filepath='./data/img_denoise_pats.npy',
rand=False) as data_denoise:
with load_data(filepath='./data/img_noise_pats.npy',
rand=False) as data_noise:
denoiser = cmpdenoiser(num_workers=num_worker,
is_chief=FLAGS.task_index == 0)
print(len(data_denoise))
data_denoise = data_denoise.astype(
np.float32) / 255.0 # normalize the data to 0-1
data_noise = data_noise.astype(
np.float32) / 255.0 # normalize the data to 0-1
denoiser.train(server,
data_denoise,
data_noise,
batch_size=args.batch_size,
ckpt_dir=args.ckpt_dir,
epoch=args.epoch,
task_index=task_index)
def denoiser_train(denoiser, lr):
#
# load noisy and clean data
#
print("[*] Loading data...")
#--------------------------------------------------------------------------
# apply log(x+1) to the raw value, and select maxV to normalize
#--------------------------------------------------------------------------
#
# step 2: train DAE with noisy images
noisy_data = np.load('../../model_input/srcpos_hetero/noisy_pats.npy')
#noisy_data = np.load('../../model_input/srcpos_hetero/clean_pats.npy')
clean_data = np.load('../../model_input/srcpos_hetero/clean_pats.npy')
print noisy_data.shape , clean_data.shape
#--- max value ---#
print "\nprev log(x+1)"
print "noisy_max \t clean_max \t max"
max_noisy, max_clean = np.amax(noisy_data), np.amax(clean_data)
maxV = max(max_noisy, max_clean)
print("{}\t{}\t{}".format(max_noisy, max_clean, maxV))
print "noisy_min \t clean_min "
min_noisy, min_clean = np.amin(noisy_data), np.amin(clean_data)
print("{}\t{}".format(min_noisy, min_clean))
#--- apply log(x + 1) ---#
noisy_data = np.log(noisy_data + 1.)
clean_data = np.log(clean_data + 1.)
print "\nafter log(x+1)"
print "noisy_max \t clean_max \t max"
max_noisy, max_clean = np.amax(noisy_data), np.amax(clean_data)
maxV = max(max_noisy, max_clean)
print max_noisy, max_clean, maxV
maxV = float(int(maxV) + 3) # add extra 3 for upper bound
# if maxV <= 25 use 25, else use maxV
if maxV <= 25.0:
maxV = 25.0
print "Using %f for maxV (after log)" % maxV
print "\nSaving maxV to use in matlab..."
sio.savemat('maxV.mat', dict(maxV=maxV))
print "\nDone!"
#--- normalize with maxV ---#
noisy_data = noisy_data / maxV
clean_data = clean_data / maxV
print noisy_data.shape
print clean_data.shape
#
# mcx images is 100x100 as default, here we resize to 128 x 128
#
noisy_data_resize = transform_resize(noisy_data)
clean_data_resize = transform_resize(clean_data)
print noisy_data_resize.shape
print clean_data_resize.shape
##--- run training ---#
denoiser.train(noisy_data_resize, clean_data_resize, batch_size=args.batch_size, ckpt_dir=args.ckpt_dir, epoch=args.epoch, lr=lr)
def denoiser_train(denoiser, lr):
#
# load noisy and clean data
#
print("[*] Loading data...")
#--------------------------------------------------------------------------
# apply log(x+1) to the raw value, and select maxV to normalize
#--------------------------------------------------------------------------
# 1) h**o: around 12K images: osa_img_noisy_pats_1e+05.npy
#
# the original data is located at: data/osa/1e+05
#
# 2) hete: around 8K images: rand2d/rand2d_noisy_pats_1e+05.npy
#
# To generate rand2d() heteregeneous data: prepare_data/rand2d_old
#
# the original data is located at : data/rand2d/1e+05
noisy_data = np.load('../../model_input/homo_hetero/osa_rand2d_noisy.npy')
clean_data = np.load('../../model_input/homo_hetero/osa_rand2d_clean.npy')
print noisy_data.shape , clean_data.shape
#--- max value ---#
print "\nprev log(x+1)"
print "noisy_max \t clean_max \t max"
max_noisy, max_clean = np.amax(noisy_data), np.amax(clean_data)
maxV = max(max_noisy, max_clean)
print max_noisy, max_clean, maxV
#--- apply log(x + 1) ---#
noisy_data = np.log(noisy_data + 1.)
clean_data = np.log(clean_data + 1.)
print "\nafter log(x+1)"
print "noisy_max \t clean_max \t max"
max_noisy, max_clean = np.amax(noisy_data), np.amax(clean_data)
maxV = max(max_noisy, max_clean)
print max_noisy, max_clean, maxV
#maxV = float(int(maxV) + 3) # add extra 3 for upper bound
maxV = 25.
print "Using %f for maxV (after log)" % maxV
print "\nSaving maxV to use in matlab..."
sio.savemat('maxV.mat', dict(maxV=maxV))
print "\nDone!"
#--- normalize with maxV ---#
noisy_data = noisy_data / maxV
clean_data = clean_data / maxV
#--- run training ---#
denoiser.train(noisy_data,
clean_data,
batch_size=args.batch_size,
ckpt_dir=args.ckpt_dir,
epoch=args.epoch,
lr=lr)
def denoiser_train(denoiser, lr):
noisy_eval_files = glob('/home/mli/tomograms/pycharm_demos/single_density/without_Gaussian_shapes/normal_test/with_noise/1024*1024_with_oversampling=2/phase_contrast_images/Gaussian_noise_PPSNR=12dB/npy_format/phase_contrast_image_distance_4_in_npy_format/validate/*.npy')
noisy_eval_files = sorted(noisy_eval_files)
eval_files = glob('/home/mli/tomograms/pycharm_demos/single_density/without_Gaussian_shapes/normal_test/without_noise/1024*1024_with_oversampling=2/phase_contrast_images_1_without_elliptical_cylinder/nd_zoom_order=1/npy_format/phase_contrast_image_distance_4_in_npy_format/validate/*.npy')
eval_files = sorted(eval_files)
denoiser.train(eval_files, noisy_eval_files, batch_size=args.batch_size, ckpt_dir=args.ckpt_dir, epoch=args.epoch, lr=lr)
def denoiser_train(denoiser, lr):
#
# load noisy and clean data
#
print("[*] Loading data...")
#--------------------------------------------------------------------------
# apply log(x+1) to the raw value, and select maxV to normalize
#--------------------------------------------------------------------------
#noisy_data = np.load('../../model_input/spie2d/noisy1e4.npy')
#clean_data = np.load('../../model_input/spie2d/clean1e7.npy')
#noisy_data = np.load('../../model_input/spie2d/journal2_hom_pack1_p5.npy')
#clean_data = np.load('../../model_input/spie2d/journal2_hom_pack1_p8.npy')
noisy_data = np.load('../../model_input/spie_2d_hetgrid_p567/hetgrid_p5.npy')
clean_data = np.load('../../model_input/spie_2d_hetgrid_p567/hetgrid_p7.npy')
print noisy_data.shape
print clean_data.shape
#--- max value ---#
print "\nprev log(x+1)"
print "noisy_max \t clean_max \t max"
max_noisy, max_clean = np.amax(noisy_data), np.amax(clean_data)
maxV = max(max_noisy, max_clean)
print max_noisy, max_clean, maxV
#--- apply log(x + 1) ---#
noisy_data = np.log(noisy_data + 1.)
clean_data = np.log(clean_data + 1.)
print "\nafter log(x+1)"
print "noisy_max \t clean_max \t max"
max_noisy, max_clean = np.amax(noisy_data), np.amax(clean_data)
maxV = max(max_noisy, max_clean)
print max_noisy, max_clean, maxV
#maxV = float(int(maxV) + 3) # add extra 3 for upper bound
maxV = 25.
print "Using %f for maxV (after log)" % maxV
print "\nSaving maxV to use in matlab..."
sio.savemat('maxV.mat', dict(maxV=maxV))
print "\nDone!"
#--- normalize with maxV ---#
noisy_data = noisy_data / maxV
clean_data = clean_data / maxV
#--- run training ---#
denoiser.train(noisy_data,
clean_data,
batch_size=args.batch_size,
ckpt_dir=args.ckpt_dir,
epoch=args.epoch,
lr=lr)
