def upscale_alg(self, im_l_y, s):
# assert im_l_y.ndim == 2, 'Input image has to be a grayscale image!'
if im_l_y.ndim == 2: # input is single frame
im_l_y = utils.ExtendBorder(im_l_y, self.params['border_size'])
self.model.blobs[self.input_blob].data[...] = np.reshape(
im_l_y, (1, 1) + im_l_y.shape)
else: # input is multiple frames
im_tmp = np.empty([
im_l_y.shape[0],
im_l_y.shape[1] + 2 * self.params['border_size'],
im_l_y.shape[2] + 2 * self.params['border_size']
])
for j in range(im_l_y.shape[0]):
im_tmp[j, ...] = utils.ExtendBorder(np.squeeze(im_l_y[j, ...]),
self.params['border_size'])
im_l_y = im_tmp
self.model.blobs[self.input_blob].data[...] = np.expand_dims(
im_l_y, axis=0)
t_start = time.time()
fea = self.model.forward(blobs=[self.output_blob])
t_end = time.time()
im_h_y = fea[self.output_blob][0, 0, :, :]
print 'caffe time: {}'.format(t_end - t_start)
return im_h_y
def upscale_alg(self, im_l_y, s):
h_gt, w_gt = im_l_y.shape[0]*s, im_l_y.shape[1]*s
# hpsz = self.params['patch_size']/2
itr_all = int(np.ceil(np.log(s)/np.log(self.params['mdl_scale'])))
indata = np.zeros((self.params['input_size']**2, self.params['batch']), dtype=np.float32)
outdata = np.zeros((self.params['output_size']**2, self.params['batch']), dtype=np.float32)
idxdata = np.zeros((2, self.params['batch']), dtype=np.float32)
ftrs = np.zeros((self.params['batch'], self.params['output_size']**2), dtype=np.float32)
for itr in range(itr_all):
print 'itr:', itr
if self.params['matlab_bic']==1:
im_y = utils.imresize_bic2(im_l_y, self.params['mdl_scale'])
else:
im_y = utils.imresize(im_l_y, self.params['mdl_scale'])
im_y = utils.ExtendBorder(im_y, self.params['border_size'])
h, w = im_y.shape
Height_idx = range(0, h-self.params['input_size'], self.params['output_size'])
Width_idx = range(0, w-self.params['input_size'], self.params['output_size'])
Height_idx += [h-self.params['input_size']]
Width_idx += [w-self.params['input_size']]
bcnt = 0
im_h_y = np.zeros((h, w), dtype=np.float32)
t0 = time.time()
for i in Height_idx:
for j in Width_idx:
idxdata[0, bcnt] = i
idxdata[1, bcnt] = j
tmp = im_y[i:i+self.params['input_size'], j:j+self.params['input_size']]
indata[:, bcnt] = np.reshape(tmp, (indata.shape[0], ))
bcnt += 1
if bcnt==self.params['batch'] or (i==Height_idx[-1] and j==Width_idx[-1]):
self.model.do_write_one_feature([indata, outdata], ftrs, self.params['layer_idx'])
for b in range(bcnt):
si = idxdata[0, b]+self.params['border_size']
sj = idxdata[1, b]+self.params['border_size']
im_h_y[si:si+self.params['output_size'], sj:sj+self.params['output_size']] = \
np.reshape(ftrs[b, :], (self.params['output_size'], self.params['output_size']))
bcnt = 0
t1 = time.time()
print 'convnet time: {}'.format(t1-t0)
im_h_y = im_h_y[self.params['border_size']:-self.params['border_size'], self.params['border_size']:-self.params['border_size']]
im_l_y = im_h_y
# shrink size to gt
if (im_h_y.shape[0]>h_gt):
print 'downscale from {} to {}'.format(im_h_y.shape, (h_gt, w_gt))
if self.params['matlab_bic']==1:
im_h_y = utils.imresize_bic2(im_h_y, 1.0*h_gt/im_h_y.shape[0])
else:
im_h_y = utils.imresize(im_h_y, 1.0*h_gt/im_h_y.shape[0])
assert(im_h_y.shape[1]==w_gt)
return im_h_y
def upscale_alg(self, im_l_y, s):
h_gt, w_gt = im_l_y.shape[0] * s, im_l_y.shape[1] * s
hpsz = self.PATCH_SIZE / 2
itr_all = int(np.ceil(np.log(s) / np.log(self.MDL_SCALE)))
for itr in range(itr_all):
print 'itr:', itr
im_y = utils.imresize(im_l_y, self.MDL_SCALE)
im_y = utils.ExtendBorder(im_y, self.BORDER_SIZE)
mdl = self.mdls[itr]
# extract gradient features
convfea = utils.ExtrConvFea(im_y, mdl['conv'])
im_mean = utils.ExtrConvFea(im_y, mdl['mean2'])
diffms = utils.ExtrConvFea(im_y, mdl['diffms'])
# matrix operation
h, w, c = convfea.shape
convfea = convfea.reshape([h * w, c])
convfea_norm = np.linalg.norm(convfea, axis=1)
convfea = (convfea.T / convfea_norm).T
wd = np.dot(convfea, mdl['wd'])
z0 = utils.ShLU(wd, 1)
z = utils.ShLU(np.dot(z0, mdl['usd1']) + wd, 1) #sparse code
hPatch = np.dot(z, mdl['ud'])
hNorm = np.linalg.norm(hPatch, axis=1)
diffms = diffms.reshape([h * w, diffms.shape[2]])
mNorm = np.linalg.norm(diffms, axis=1)
hPatch = (hPatch.T / hNorm * mNorm).T * self.SCALE_Y
hPatch = hPatch * mdl['addp'].flatten()
hPatch = hPatch.reshape([h, w, hPatch.shape[1]])
im_h_y = im_mean[:, :, 0]
h, w = im_h_y.shape
cnt = 0
for ii in range(self.PATCH_SIZE - 1, -1, -1):
for jj in range(self.PATCH_SIZE - 1, -1, -1):
im_h_y = im_h_y + hPatch[jj:(jj + h), ii:(ii + w), cnt]
cnt = cnt + 1
im_l_y = im_h_y
# shrink size to gt
if (im_h_y.shape[0] > h_gt):
print 'downscale from {} to {}'.format(im_h_y.shape, (h_gt, w_gt))
im_h_y = utils.imresize(im_h_y, 1.0 * h_gt / im_h_y.shape[0])
assert (im_h_y.shape[1] == w_gt)
return im_h_y
def upscale_alg(self, im_l_y, s):
h_gt, w_gt = im_l_y.shape[0]*s, im_l_y.shape[1]*s
# hpsz = self.params['patch_size']/2
itr_all = int(np.ceil(np.log(s)/np.log(self.params['mdl_scale'])))
idxdata = np.zeros((2, self.params['batch']), dtype=np.float32)
indata = np.zeros(self.model.blobs[self.input_blob].data.shape, dtype=np.float32)
for itr in range(itr_all):
print 'itr:', itr
if self.params['matlab_bic']==1:
im_y = utils.imresize_bic2(im_l_y, self.params['mdl_scale'])
else:
im_y = utils.imresize(im_l_y, self.params['mdl_scale'])
im_y = utils.ExtendBorder(im_y, self.params['border_size'])
# im_y = utils.RepeatBorder(im_y, self.params['border_size'])
h, w = im_y.shape
Height_idx = range(0, h-self.params['input_size'], self.params['output_size'])
Width_idx = range(0, w-self.params['input_size'], self.params['output_size'])
Height_idx += [h-self.params['input_size']]
Width_idx += [w-self.params['input_size']]
bcnt = 0
im_h_y = np.zeros((h, w), dtype=np.float32)
t0 = time.time()
#fea = self.model.forward(data=indata, end=self.params['layer'], blobs=[self.params['layer'], 'data'])
for i in Height_idx:
for j in Width_idx:
idxdata[0, bcnt] = i
idxdata[1, bcnt] = j
tmp = im_y[i:i+self.params['input_size'], j:j+self.params['input_size']]
indata[bcnt, 0, :, :] = np.reshape(tmp, (1, 1)+tmp.shape)
bcnt += 1
if bcnt==self.params['batch'] or (i==Height_idx[-1] and j==Width_idx[-1]):
# fea = self.model.forward(data=indata, end=self.params['layer'], blobs=[self.params['layer'], 'data'])
self.model.blobs[self.input_blob].data[...] = indata
fea = self.model.forward(blobs=[self.output_blob])
for b in range(bcnt):
si = idxdata[0, b]+self.params['border_size']
sj = idxdata[1, b]+self.params['border_size']
im_h_y[si:si+self.params['output_size'], sj:sj+self.params['output_size']] = \
np.reshape(fea[self.output_blob][b, 0, :, :], (self.params['output_size'], self.params['output_size']))
bcnt = 0
t1 = time.time()
print 'caffe time: {}'.format(t1-t0)
im_h_y = im_h_y[self.params['border_size']:-self.params['border_size'], self.params['border_size']:-self.params['border_size']]
im_l_y = im_h_y
# shrink size to gt
if (im_h_y.shape[0]>h_gt):
print 'downscale from {} to {}'.format(im_h_y.shape, (h_gt, w_gt))
# if self.params['matlab_bic']==1:
if 0:
im_h_y = utils.imresize_bic2(im_h_y, 1.0*h_gt/im_h_y.shape[0])
else:
im_h_y = utils.imresize(im_h_y, 1.0*h_gt/im_h_y.shape[0])
assert(im_h_y.shape[1]==w_gt)
return im_h_y
def upscale_alg(self, im_l_y, s):
# h_gt, w_gt = im_l_y.shape[0]*s, im_l_y.shape[1]*s
if len(im_l_y.shape)==2:
im_l_y = im_l_y[None,...]
h_gt, w_gt = im_l_y.shape[1], im_l_y.shape[2]
itr_all = int(np.ceil(np.log(s)/np.log(self.params['mdl_scale'])))
idxdata = np.zeros((2, self.params['batch']), dtype=np.float32)
indata = np.zeros(self.model.blobs[self.input_blob].data.shape, dtype=np.float32)
# print 'indata shape: ', indata.shape
for itr in range(itr_all):
print 'itr:', itr
"""
if self.params['matlab_bic']==1:
im_y = utils.imresize_bic2(im_l_y, self.params['mdl_scale'])
else:
im_y = utils.imresize(im_l_y, self.params['mdl_scale'])
"""
im_y = im_l_y
im_y_tmp = np.empty([im_y.shape[0], im_y.shape[1]+2*self.params['border_size'], im_y.shape[2]+2*self.params['border_size']])
for j in range(im_y.shape[0]):
im_y_tmp[j,...] = utils.ExtendBorder( np.squeeze(im_y[j,...]), self.params['border_size'])
im_y = im_y_tmp
# if run comparison experiments
if 0:
im_y = im_y[im_y.shape[0]/2,...][None,...]
# im_y = utils.RepeatBorder(im_y, self.params['border_size'])
depth, h, w = im_y.shape
# print 'im_y shape: ', im_y.shape
Height_idx = range(0, h-self.params['input_size'], self.params['output_size'])
Width_idx = range(0, w-self.params['input_size'], self.params['output_size'])
Height_idx += [h-self.params['input_size']]
Width_idx += [w-self.params['input_size']]
bcnt = 0
im_h_y = np.zeros((h, w), dtype=np.float32)
t0 = time.time()
#fea = self.model.forward(data=indata, end=self.params['layer'], blobs=[self.params['layer'], 'data'])
for i in Height_idx:
for j in Width_idx:
idxdata[0, bcnt] = i
idxdata[1, bcnt] = j
# tmp = im_y[i:i+self.params['input_size'], j:j+self.params['input_size']]
tmp = im_y[:, i:i+self.params['input_size'], j:j+self.params['input_size']]
# indata[bcnt, 0, :, :] = np.reshape(tmp, (1, 1)+tmp.shape)
indata[bcnt, :, :, :] = np.reshape(tmp, tmp[None,...].shape)
bcnt += 1
if bcnt==self.params['batch'] or (i==Height_idx[-1] and j==Width_idx[-1]):
# fea = self.model.forward(data=indata, end=self.params['layer'], blobs=[self.output_blob, self.input_blob])
self.model.blobs[self.input_blob].data[...] = indata
fea = self.model.forward(blobs=[self.output_blob])
for b in range(bcnt):
si = idxdata[0, b]+self.params['border_size']
sj = idxdata[1, b]+self.params['border_size']
im_h_y[si:si+self.params['output_size'], sj:sj+self.params['output_size']] = \
np.reshape(fea[self.output_blob][b, 0, :, :], (self.params['output_size'], self.params['output_size']))
# np.reshape(fea[self.params['layer']][b, 0, :, :], (self.params['output_size'], self.params['output_size']))
bcnt = 0
t1 = time.time()
print 'caffe time: {}'.format(t1-t0)
im_h_y = im_h_y[self.params['border_size']:-self.params['border_size'], self.params['border_size']:-self.params['border_size']]
im_l_y = im_h_y
# print 'im_h_y shape: ', im_h_y.shape
# shrink size to gt
if (im_h_y.shape[0]>h_gt):
print 'downscale from {} to {}'.format(im_h_y.shape, (h_gt, w_gt))
if self.params['matlab_bic']==1:
im_h_y = utils.imresize_bic2(im_h_y, 1.0*h_gt/im_h_y.shape[0])
else:
im_h_y = utils.imresize(im_h_y, 1.0*h_gt/im_h_y.shape[0])
assert(im_h_y.shape[1]==w_gt)
return im_h_y