def IHS(pan, hs):
M, N, c = pan.shape
m, n, C = hs.shape
ratio = int(np.round(M / m))
print('get sharpening ratio: ', ratio)
assert int(np.round(M / m)) == int(np.round(N / n))
#upsample
u_hs = upsample_interp23(hs, ratio)
I = np.mean(u_hs, axis=-1, keepdims=True)
P = (pan - np.mean(pan)) * np.std(I, ddof=1) / np.std(pan,
ddof=1) + np.mean(I)
I_IHS = u_hs + np.tile(P - I, (1, 1, C))
#adjustment
I_IHS[I_IHS < 0] = 0
I_IHS[I_IHS > 1] = 1
return np.uint8(I_IHS * 255)
def Wavelet(pan, hs):
M, N, c = pan.shape
m, n, C = hs.shape
ratio = int(np.round(M / m))
print('get sharpening ratio: ', ratio)
assert int(np.round(M / m)) == int(np.round(N / n))
#upsample
u_hs = upsample_interp23(hs, ratio)
pan = np.squeeze(pan)
pc = pywt.wavedec2(pan, 'haar', level=2)
rec = []
for i in range(C):
temp_dec = pywt.wavedec2(u_hs[:, :, i], 'haar', level=2)
pc[0] = temp_dec[0]
temp_rec = pywt.waverec2(pc, 'haar')
temp_rec = np.expand_dims(temp_rec, -1)
rec.append(temp_rec)
I_Wavelet = np.concatenate(rec, axis=-1)
#adjustment
I_Wavelet[I_Wavelet < 0] = 0
I_Wavelet[I_Wavelet > 1] = 1
return np.uint8(I_Wavelet * 255)
def Brovey(pan, hs):
M, N, c = pan.shape
m, n, C = hs.shape
ratio = int(np.round(M/m))
print('get sharpening ratio: ', ratio)
assert int(np.round(M/m)) == int(np.round(N/n))
#upsample
u_hs = upsample_interp23(hs, ratio)
I = np.mean(u_hs, axis=-1)
image_hr = (pan-np.mean(pan))*(np.std(I, ddof=1)/np.std(pan, ddof=1))+np.mean(I)
image_hr = np.squeeze(image_hr)
I_Brovey=[]
for i in range(C):
temp = image_hr*u_hs[:, :, i]/(I+1e-8)
temp = np.expand_dims(temp, axis=-1)
I_Brovey.append(temp)
I_Brovey = np.concatenate(I_Brovey, axis=-1)
#adjustment
I_Brovey[I_Brovey<0]=0
I_Brovey[I_Brovey>1]=1
return np.uint8(I_Brovey*255)
def GFPCA(pan, hs):
M, N, c = pan.shape
m, n, C = hs.shape
ratio = int(np.round(M / m))
print('get sharpening ratio: ', ratio)
assert int(np.round(M / m)) == int(np.round(N / n))
p = princomp(n_components=C)
pca_hs = p.fit_transform(np.reshape(hs, (m * n, C)))
pca_hs = np.reshape(pca_hs, (m, n, C))
pca_hs = upsample_interp23(pca_hs, ratio)
gp_hs = []
for i in range(C):
temp = guidedFilter(np.float32(pan),
np.float32(np.expand_dims(pca_hs[:, :, i], -1)),
8,
eps=0.001**2)
temp = np.expand_dims(temp, axis=-1)
gp_hs.append(temp)
gp_hs = np.concatenate(gp_hs, axis=-1)
I_GFPCA = p.inverse_transform(gp_hs)
#adjustment
I_GFPCA[I_GFPCA < 0] = 0
I_GFPCA[I_GFPCA > 1] = 1
return np.uint8(I_GFPCA * 255)
def GS(pan, hs):
M, N, c = pan.shape
m, n, C = hs.shape
ratio = int(np.round(M / m))
print('get sharpening ratio: ', ratio)
assert int(np.round(M / m)) == int(np.round(N / n))
#upsample
u_hs = upsample_interp23(hs, ratio)
#remove means from u_hs
means = np.mean(u_hs, axis=(0, 1))
image_lr = u_hs - means
#sintetic intensity
I = np.mean(u_hs, axis=2, keepdims=True)
I0 = I - np.mean(I)
image_hr = (pan - np.mean(pan)) * (np.std(I0, ddof=1) /
np.std(pan, ddof=1)) + np.mean(I0)
#computing coefficients
g = []
g.append(1)
for i in range(C):
temp_h = image_lr[:, :, i]
c = np.cov(np.reshape(I0, (-1, )), np.reshape(temp_h, (-1, )), ddof=1)
g.append(c[0, 1] / np.var(I0))
g = np.array(g)
#detail extraction
delta = image_hr - I0
deltam = np.tile(delta, (1, 1, C + 1))
#fusion
V = np.concatenate((I0, image_lr), axis=-1)
g = np.expand_dims(g, 0)
g = np.expand_dims(g, 0)
g = np.tile(g, (M, N, 1))
V_hat = V + g * deltam
I_GS = V_hat[:, :, 1:]
I_GS = I_GS - np.mean(I_GS, axis=(0, 1)) + means
#adjustment
I_GS[I_GS < 0] = 0
I_GS[I_GS > 1] = 1
return np.uint8(I_GS * 255)
def SFIM(pan, hs):
M, N, c = pan.shape
m, n, C = hs.shape
ratio = int(np.round(M / m))
print('get sharpening ratio: ', ratio)
assert int(np.round(M / m)) == int(np.round(N / n))
#upsample
u_hs = upsample_interp23(hs, ratio)
if np.mod(ratio, 2) == 0:
ratio = ratio + 1
pan = np.tile(pan, (1, 1, C))
pan = (pan - np.mean(pan, axis=(0, 1))) * (
np.std(u_hs, axis=(0, 1), ddof=1) /
np.std(pan, axis=(0, 1), ddof=1)) + np.mean(u_hs, axis=(0, 1))
kernel = np.ones((ratio, ratio))
kernel = kernel / np.sum(kernel)
I_SFIM = np.zeros((M, N, C))
for i in range(C):
lrpan = signal.convolve2d(pan[:, :, i],
kernel,
mode='same',
boundary='wrap')
I_SFIM[:, :, i] = u_hs[:, :, i] * pan[:, :, i] / (lrpan + 1e-8)
#adjustment
I_SFIM[I_SFIM < 0] = 0
I_SFIM[I_SFIM > 1] = 1
return np.uint8(I_SFIM * 255)
def PCA(pan, hs):
M, N, c = pan.shape
m, n, C = hs.shape
ratio = int(np.round(M / m))
print('get sharpening ratio: ', ratio)
assert int(np.round(M / m)) == int(np.round(N / n))
image_hr = pan
#upsample
u_hs = upsample_interp23(hs, ratio)
p = princomp(n_components=C)
pca_hs = p.fit_transform(np.reshape(u_hs, (M * N, C)))
pca_hs = np.reshape(pca_hs, (M, N, C))
I = pca_hs[:, :, 0]
image_hr = (image_hr - np.mean(image_hr)) * np.std(I, ddof=1) / np.std(
image_hr, ddof=1) + np.mean(I)
pca_hs[:, :, 0] = image_hr[:, :, 0]
I_PCA = p.inverse_transform(pca_hs)
#equalization
I_PCA = I_PCA - np.mean(I_PCA, axis=(0, 1)) + np.mean(u_hs)
#adjustment
I_PCA[I_PCA < 0] = 0
I_PCA[I_PCA > 1] = 1
return np.uint8(I_PCA * 255)
def MTF_GLP(pan, hs, sensor='gaussian'):
M, N, c = pan.shape
m, n, C = hs.shape
ratio = int(np.round(M/m))
print('get sharpening ratio: ', ratio)
assert int(np.round(M/m)) == int(np.round(N/n))
#upsample
u_hs = upsample_interp23(hs, ratio)
#equalization
image_hr = np.tile(pan, (1, 1, C))
image_hr = (image_hr - np.mean(image_hr, axis=(0,1)))*(np.std(u_hs, axis=(0, 1), ddof=1)/np.std(image_hr, axis=(0, 1), ddof=1))+np.mean(u_hs, axis=(0,1))
# print(image_hr.shape)
pan_lp = np.zeros_like(u_hs)
N =31
fcut = 1/ratio
match = 0
if sensor == 'gaussian':
sig = (1/(2*(2.772587)/ratio**2))**0.5
kernel = np.multiply(cv2.getGaussianKernel(9, sig), cv2.getGaussianKernel(9,sig).T)
t=[]
for i in range(C):
temp = signal.convolve2d(image_hr[:, :, i], kernel, mode='same', boundary = 'wrap')
temp = temp[0::ratio, 0::ratio]
temp = np.expand_dims(temp, -1)
t.append(temp)
t = np.concatenate(t, axis=-1)
pan_lp = upsample_interp23(t, ratio)
elif sensor == None:
match=1
GNyq = 0.3*np.ones((C,))
elif sensor=='QB':
match=1
GNyq = np.asarray([0.34, 0.32, 0.30, 0.22],dtype='float32') # Band Order: B,G,R,NIR
elif sensor=='IKONOS':
match=1 #MTF usage
GNyq = np.asarray([0.26,0.28,0.29,0.28],dtype='float32') # Band Order: B,G,R,NIR
elif sensor=='GeoEye1':
match=1 # MTF usage
GNyq = np.asarray([0.23,0.23,0.23,0.23],dtype='float32') # Band Order: B,G,R,NIR
elif sensor=='WV2':
match=1 # MTF usage
GNyq = [0.35,0.35,0.35,0.35,0.35,0.35,0.35,0.27]
elif sensor=='WV3':
match=1 #MTF usage
GNyq = 0.29 * np.ones(8)
if match==1:
t = []
for i in range(C):
alpha = np.sqrt(N*(fcut/2)**2/(-2*np.log(GNyq)))
H = np.multiply(cv2.getGaussianKernel(N, alpha[i]), cv2.getGaussianKernel(N, alpha[i]).T)
HD = H/np.max(H)
h = fir_filter_wind(HD, kaiser2d(N, 0.5))
temp = signal.convolve2d(image_hr[:, :, i], np.real(h), mode='same', boundary = 'wrap')
temp = temp[0::ratio, 0::ratio]
temp = np.expand_dims(temp, -1)
t.append(temp)
t = np.concatenate(t, axis=-1)
pan_lp = upsample_interp23(t, ratio)
I_MTF_GLP = u_hs + image_hr - pan_lp
#adjustment
I_MTF_GLP[I_MTF_GLP<0]=0
I_MTF_GLP[I_MTF_GLP>1]=1
return np.uint8(I_MTF_GLP*255)
def PNN(hrms, lrhs, sensor = None):
"""
this is an zero-shot learning method with deep learning (PNN)
hrms: numpy array with MXNXc
lrhs: numpy array with mxnxC
"""
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.set_session(sess)
M, N, c = hrms.shape
m, n, C = lrhs.shape
stride = 8
training_size=32#training patch size
testing_size=400#testing patch size
reconstructing_size=320#reconstructing patch size
left_pad = (testing_size-reconstructing_size)//2
'''
testing
---------------
| rec |
| ------- |
| | | |
| | | |
| ------- |
| |
---------------
|pad|
'''
ratio = int(np.round(M/m))
print('get sharpening ratio: ', ratio)
assert int(np.round(M/m)) == int(np.round(N/n))
train_hrhs_all = []
train_hrms_all = []
train_lrhs_all = []
used_hrhs = lrhs
used_lrhs = lrhs
used_lrhs, used_hrms = downgrade_images(used_lrhs, hrms, ratio, sensor=sensor)
print(used_lrhs.shape, used_hrms.shape)
used_lrhs = upsample_interp23(used_lrhs, ratio)
"""crop images"""
print('croping images...')
for j in range(0, used_hrms.shape[0]-training_size, stride):
for k in range(0, used_hrms.shape[1]-training_size, stride):
temp_hrhs = used_hrhs[j:j+training_size, k:k+training_size, :]
temp_hrms = used_hrms[j:j+training_size, k:k+training_size, :]
temp_lrhs = used_lrhs[j:j+training_size, k:k+training_size, :]
train_hrhs_all.append(temp_hrhs)
train_hrms_all.append(temp_hrms)
train_lrhs_all.append(temp_lrhs)
train_hrhs_all = np.array(train_hrhs_all, dtype='float16')
train_hrms_all = np.array(train_hrms_all, dtype='float16')
train_lrhs_all = np.array(train_lrhs_all, dtype='float16')
index = [i for i in range(train_hrhs_all.shape[0])]
# random.seed(2020)
random.shuffle(index)
train_hrhs = train_hrhs_all[index, :, :, :]
train_hrms= train_hrms_all[index, :, :, :]
train_lrhs = train_lrhs_all[index, :, :, :]
print(train_hrhs.shape, train_hrms.shape, train_lrhs.shape)
"""train net"""
print('training...')
def lr_schedule(epoch):
"""Learning Rate Schedule
# Arguments
epoch (int): The number of epochs
# Returns
lr (float32): learning rate
"""
lr = 5e-4
if epoch > 40:
lr *= 1e-2
elif epoch > 20:
lr *= 1e-1
return lr
lr_scheduler = LearningRateScheduler(lr_schedule, verbose=1)
checkpoint = ModelCheckpoint(filepath='./weights/PNN_model.h5',
monitor='val_psnr',
mode='max',
verbose=1,
save_best_only=True)
callbacks = [lr_scheduler, checkpoint]
model = pnn_net(lrhs_size=(training_size, training_size, C), hrms_size=(training_size, training_size, c))
model.fit( x=[train_lrhs, train_hrms],
y=train_hrhs,
validation_split=0.33,
batch_size=32,
epochs=50,
verbose=1,
callbacks=callbacks)
model = pnn_net(lrhs_size=(testing_size, testing_size, C), hrms_size=(testing_size, testing_size, c))
model.load_weights('./weights/PNN_model.h5')
"""eval"""
print('evaling...')
new_M = min(M, m*ratio)
new_N = min(N, n*ratio)
print('output image size:', new_M, new_N)
test_label = np.zeros((new_M, new_N, C), dtype = 'uint8')
used_lrhs = lrhs[:new_M//ratio, :new_N//ratio, :]
used_hrms = hrms[:new_M, :new_N, :]
used_lrhs = upsample_interp23(used_lrhs, ratio)
used_lrhs = np.expand_dims(used_lrhs, 0)
used_hrms = np.expand_dims(used_hrms, 0)
used_lrhs = np.pad(used_lrhs, ((0, 0), (left_pad, testing_size), (left_pad, testing_size), (0, 0)), mode='symmetric')
used_hrms = np.pad(used_hrms, ((0, 0), (left_pad, testing_size), (left_pad, testing_size), (0, 0)), mode='symmetric')
for h in tqdm(range(0, new_M, reconstructing_size)):
for w in range(0, new_N, reconstructing_size):
temp_lrhs = used_lrhs[:, h:h+testing_size, w:w+testing_size, :]
temp_hrms = used_hrms[:, h:h+testing_size, w:w+testing_size, :]
fake = model.predict([temp_lrhs, temp_hrms])
fake = np.clip(fake, 0, 1)
fake.shape=(testing_size, testing_size, C)
fake = fake[left_pad:(testing_size-left_pad), left_pad:(testing_size-left_pad)]
fake = np.uint8(fake*255)
if h+testing_size>new_M:
fake = fake[:new_M-h, :, :]
if w+testing_size>new_N:
fake = fake[:, :new_N-w, :]
test_label[h:h+reconstructing_size, w:w+reconstructing_size]=fake
K.clear_session()
gc.collect()
del model
return np.uint8(test_label)
def GSA(pan, hs):
M, N, c = pan.shape
m, n, C = hs.shape
ratio = int(np.round(M / m))
print('get sharpening ratio: ', ratio)
assert int(np.round(M / m)) == int(np.round(N / n))
#upsample
u_hs = upsample_interp23(hs, ratio)
#remove means from u_hs
means = np.mean(u_hs, axis=(0, 1))
image_lr = u_hs - means
#remove means from hs
image_lr_lp = hs - np.mean(hs, axis=(0, 1))
#sintetic intensity
image_hr = pan - np.mean(pan)
image_hr0 = cv2.resize(image_hr, (n, m), cv2.INTER_CUBIC)
image_hr0 = np.expand_dims(image_hr0, -1)
alpha = estimation_alpha(image_hr0,
np.concatenate((image_lr_lp, np.ones((m, n, 1))),
axis=-1),
mode='global')
I = np.dot(np.concatenate((image_lr, np.ones((M, N, 1))), axis=-1), alpha)
I0 = I - np.mean(I)
#computing coefficients
g = []
g.append(1)
for i in range(C):
temp_h = image_lr[:, :, i]
c = np.cov(np.reshape(I0, (-1, )), np.reshape(temp_h, (-1, )), ddof=1)
g.append(c[0, 1] / np.var(I0))
g = np.array(g)
#detail extraction
delta = image_hr - I0
deltam = np.tile(delta, (1, 1, C + 1))
#fusion
V = np.concatenate((I0, image_lr), axis=-1)
g = np.expand_dims(g, 0)
g = np.expand_dims(g, 0)
g = np.tile(g, (M, N, 1))
V_hat = V + g * deltam
I_GSA = V_hat[:, :, 1:]
I_GSA = I_GSA - np.mean(I_GSA, axis=(0, 1)) + means
#adjustment
I_GSA[I_GSA < 0] = 0
I_GSA[I_GSA > 1] = 1
return np.uint8(I_GSA * 255)
