def render(img0, img1, v, alpha=0.5, w =None):
X, Y = np.meshgrid(np.arange(img0.shape[1]), np.arange(img0.shape[0]))
Y = Y[:, :, np.newaxis]
X = X[:, :, np.newaxis]
q = np.concatenate([Y, X], axis=2)
vp = get_color(v, q)
if w is not None:
wp = get_color(w, q)
dampening = 0.8
for i in range(20):
if w is None:
p = q - (2.0 * alpha - 1.0) * vp
else:
p = q - (2.0 * alpha - 1.0) * vp - 4 * alpha * (1 - alpha) * wp
wp = get_color(w, p)
new_vp = get_color(v, p)
vp = dampening * new_vp + (1 - dampening) * vp
if np.linalg.norm(vp-new_vp) < 1e-5:
break
c0 = get_color(img0, p - vp)
c1 = get_color(img1, p + vp)
morphed = (1 - alpha) * c0 + alpha * c1
if morphed.std() > 1:
morphed = morphed.astype(np.uint8)
if GPU:
morphed = np.asnumpy(morphed)
return morphed
def plot_sim(x0, y0):
plt.figure(figsize=(10,10))
for i in range(16):
idx = np.random.randint(x0.shape[0])
im0 = x0[idx]
im1 = y0[idx]
plt.subplot(4, 4, i+1)
plt.grid(b=False)
img = np.concatenate([im0, im1], axis=1)
sim = SIM(im0, im1)
plt.imshow(img[:,:,::-1])
plt.title('SIM: {:.2f}'.format(sim))
plt.show()
def resize_v(size, v, size_x=None):
if GPU:
v = np.asnumpy(v)
vx = v[:, :, 0]
vy = v[:, :, 1]
if size_x is None:
vx_low = cv2.resize(vx, (size, size))
vy_low = cv2.resize(vy, (size, size))
else:
vx_low = cv2.resize(vx, (size, size_x))
vy_low = cv2.resize(vy, (size, size_x))
if GPU:
vx_low = np.asarray(vx_low)
vy_low = np.asarray(vy_low)
vx_low = vx_low[:, :, np.newaxis]
vy_low = vy_low[:, :, np.newaxis]
v_low = np.concatenate([vx_low, vy_low], axis=2)
if size_x is None:
v_low = v_low / v.shape[0] * size
else:
v_low[:, :, 0] = v_low[:, :, 0] / v.shape[0] * size
v_low[:, :, 1] = v_low[:, :, 1] / v.shape[1] * size_x
return v_low
def E_UI(v, p0, p1):
if v.ndim == 3:
p = (p1 + p0) / 2
p = p[:, np.newaxis, :]
v_true = (p1 - p0) / 2
tmp = np.array([
[0., 0.],
[1., 0.],
[0., 1.],
[1., 1.]])
tmp = tmp[np.newaxis, :, :]
p_around = tmp + np.floor(p)
tmp = np.array([
[1., 1.],
[0., 0.]])
tmp = tmp[np.newaxis, :, :] # (1, 2, 2)
p_dif = p - np.floor(p) # (n, 1, 2)
p_dif = np.concatenate([-p_dif, p_dif], axis=1) # (n, 2, 2)
p_dif = p_dif + tmp # (10, 2, 2)
# if original p[0] is [2.2, 5.3]
# p_dif[0] is [[0.8, 0.7],[0.2, 0.3]]
p_dif_x = p_dif[:, :, 0] # p_dif_x[0] is [0.8, 0.2]
p_dif_y = p_dif[:, :, 1] # p_dif_x[1] is [0.7, 0.3]
p_dif_x = p_dif_x[:, np.newaxis, :]
p_dif_y = p_dif_y[:, :, np.newaxis]
b = np.matmul(p_dif_y, p_dif_x) # b[0] is [[0.56, 0.14], [0.24, 0.06]]
b = b.reshape(-1, 4, 1)
v_around = get_v(v, p_around)
v_true = v_true[:, np.newaxis, :]
ui = b * (v_true - v_around) ** 2
return ui.mean(axis=1).sum()
elif v.ndim == 4:
p = (p1 + p0) / 2
p = p[:, np.newaxis, :] # (n , 1, 2)
v_true = (p1 - p0) / 2
tmp = np.array([
[0., 0.],
[1., 0.],
[0., 1.],
[1., 1.]])
tmp = tmp[np.newaxis, :, :] # (1, 4, 2)
p_around = tmp + np.floor(p) # (n, 4, 2)
tmp = np.array([
[1., 1.],
[0., 0.]])
tmp = tmp[np.newaxis, :, :] # (1, 2, 2)
p_dif = p - np.floor(p) # (n, 1, 2)
p_dif = np.concatenate([-p_dif, p_dif], axis=1) # (n, 2, 2)
p_dif = p_dif + tmp # (n, 2, 2)
# if original p[0] is [2.2, 5.3]
# p_dif[0] is [[0.8, 0.7],[0.2, 0.3]]
p_dif_x = p_dif[:, :, 0] # p_dif_x[0] is [0.8, 0.2]
p_dif_y = p_dif[:, :, 1] # p_dif_x[1] is [0.7, 0.3]
p_dif_x = p_dif_x[:, np.newaxis, :]
p_dif_y = p_dif_y[:, :, np.newaxis]
b = np.matmul(p_dif_y, p_dif_x) # b[0] is [[0.56, 0.14], [0.24, 0.06]]
b = b.reshape(1, -1, 4, 1) # (1, n, 4, 1)
p_around.shape
v.shape
v_around = get_v(v, p_around) # (N, n, 4, 2)
v_true = v_true[np.newaxis, :, np.newaxis, :] # (1, n, 1, 2)
ui = b * (v_true - v_around) ** 2 # (N, n, 4, 2)
return ui.mean(axis=2).sum(axis=(1, 2))
def E_SIM(v, img0, img1, p=None):
'''
if p is None, compute and return sum of E_SIM(p)
'''
if v.ndim == 2 and v.shape[0] == v.shape[1]:
if v.shape[0] < 16 * 16:
return 0
Y, X = np.meshgrid(np.arange(-2,3), np.arange(-2,3))
Y = Y[:, :, np.newaxis]
X = X[:, :, np.newaxis]
grid = np.concatenate([Y, X], axis=2)
grid = grid[np.newaxis, :, :, :] # (1, 5, 5, 2)
X, Y = np.meshgrid(range(2, img0.shape[0]-2), range(2, img0.shape[1] -2))
Y = Y[:, :, np.newaxis]
X = X[:, :, np.newaxis]
p_all = np.concatenate([Y, X], axis=2)
p_all = p_all.reshape([-1, 2])
p_all = p_all[:, np.newaxis, np.newaxis, :] # (num of p, 1, 1, 2)
grids = grid + p_all # (1, num of p, 5, 5, 2)
size = int((v.size / 4) ** 0.25)
v = v.reshape(-1, size, size, 2) # (N, size, size, 2)
grids_v = get_color(v, grids) # (N, num of p, 5, 5, 2)
grids_img0 = get_color(img0, grids - grids_v)
grids_img1 = get_color(img1, grids + grids_v) # (N, num of p, 5, 5, 3)
sim = SIM(grids_img0, grids_img1)
return -1 * sim
elif p is not None:
Y, X = np.meshgrid(np.arange(-2,3), np.arange(-2,3))
Y = Y[:, :, np.newaxis]
X = X[:, :, np.newaxis]
grid = np.concatenate([Y, X], axis=2)
grid += p
# now grid is 5*5 points around p
grid_v = get_v(v, grid)
grid_img0 = get_color(img0, grid - grid_v)
grid_img1 = get_color(img1, grid + grid_v)
sim = SIM(grid_img0, grid_img1)
return -1 * sim
else:
if v.shape[0] <= 16 or v.shape[1] <= 16:
return 0
Y, X = np.meshgrid(np.arange(-2,3), np.arange(-2,3))
Y = Y[:, :, np.newaxis]
X = X[:, :, np.newaxis]
grid = np.concatenate([Y, X], axis=2)
grid = grid[np.newaxis, :, :, :] # (1, 5, 5, 2)
X, Y = np.meshgrid(np.arange(2, img0.shape[0]-2), np.arange(2, img0.shape[1] -2))
Y = Y[:, :, np.newaxis]
X = X[:, :, np.newaxis]
p_all = np.concatenate([Y, X], axis=2)
p_all = p_all.reshape([-1, 2])
p_all = p_all[:, np.newaxis, np.newaxis, :] # (num of p, 1, 1, 2)
grids = grid + p_all # (num of p, 5, 5, 2)
grids_v = get_v(v, grids)
grids_img0 = get_color(img0, grids - grids_v)
grids_img1 = get_color(img1, grids + grids_v)
sim = SIM(grids_img0, grids_img1)
return -1 * sim