def plot_correspondances(img0, img1, p0, p1):
if GPU:
img0, img1 = np.asnumpy(img0), np.asnumpy(img1)
p0, p1 = np.asnumpy(p0), np.asnumpy(p1)
img0_point = img0 // 3
img1_point = img1 // 3
img0_point[p0[:, 0].astype(np.int), p0[:, 1].astype(np.int), 0:2] = 255
img1_point[p1[:, 0].astype(np.int), p1[:, 1].astype(np.int), 0:2] = 255
plt.figure(figsize=(10, 5))
plt.subplot(1, 2, 1)
plt.imshow(img0_point[:, :, ::-1])
plt.subplot(1, 2, 2)
plt.imshow(img1_point[:, :, ::-1])
plt.show()
def resize_img(size, img0, img1):
if GPU:
img0 = np.asnumpy(img0)
img1 = np.asnumpy(img1)
img0_low = cv2.resize(img0, (size, size))
img1_low = cv2.resize(img1, (size, size))
if GPU:
img0_low = np.asarray(img0_low)
img1_low = np.asarray(img1_low)
if img0_low.std() > 1:
img0_low = img0_low / 255.0
if img1_low.std() > 1:
img1_low = img1_low / 255.0
return img0_low, img1_low
def render(img0, img1, v, alpha=0.5, w =None):
X, Y = np.meshgrid(np.arange(img0.shape[1]), np.arange(img0.shape[0]))
Y = Y[:, :, np.newaxis]
X = X[:, :, np.newaxis]
q = np.concatenate([Y, X], axis=2)
vp = get_color(v, q)
if w is not None:
wp = get_color(w, q)
dampening = 0.8
for i in range(20):
if w is None:
p = q - (2.0 * alpha - 1.0) * vp
else:
p = q - (2.0 * alpha - 1.0) * vp - 4 * alpha * (1 - alpha) * wp
wp = get_color(w, p)
new_vp = get_color(v, p)
vp = dampening * new_vp + (1 - dampening) * vp
if np.linalg.norm(vp-new_vp) < 1e-5:
break
c0 = get_color(img0, p - vp)
c1 = get_color(img1, p + vp)
morphed = (1 - alpha) * c0 + alpha * c1
if morphed.std() > 1:
morphed = morphed.astype(np.uint8)
if GPU:
morphed = np.asnumpy(morphed)
return morphed
def render_animation(img0, img1, v, steps=30, save=True, file_name='animation.mov', time=1, w=None):
alpha_list = np.arange(0, 1.0 + 1e-5, 1.0/steps)
if GPU:
alpha_list = np.asnumpy(alpha_list)
img0 = np.asarray(img0)
img1 = np.asarray(img1)
v = np.asarray(v)
if w is not None:
w = np.asarray(w)
imgs = []
print('Start Rendering')
for alpha in alpha_list:
# print('Rendering: {:.1f} %'.format(alpha*100))
img = render(img0, img1, v, alpha=alpha, w=w)
if img.std() < 1:
img = (img * 255).astype(np.uint8)
else:
img = img.astype(np.uint8)
imgs.append(img)
if save:
if os.path.exists(file_name):
os.remove(file_name)
save_animation(imgs, file_name=file_name, time=time)
print('Rendering finished!')
return imgs
def save_animation(imgs, file_name='animation.mov', time=1):
fps = len(imgs) // time
fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 's')
out = cv2.VideoWriter(file_name, int(fourcc), fps,
(imgs[0].shape[0], imgs[0].shape[1]))
for img in imgs:
if GPU:
img = np.asnumpy(img)
out.write(img)
def imshow(img):
if GPU:
img = np.asnumpy(img)
if img.std() < 1:
img = img * 255
if img.ndim == 2:
plt.imshow(img.astype(np.uint8))
else:
plt.imshow(img[:, :, ::-1].astype(np.uint8))
def describe(x):
print('*** Numpy array info ***')
print('type : ' + str(type(x)))
if GPU:
x = np.asnumpy(x)
print('shape : ' + str(x.shape))
print('mean : {:.4f}'.format(x.mean()))
print('std : {:.4f}'.format(x.std()))
print('min : {:.4f}, argmin : {}'.format(
x.min(), str(np.unravel_index(x.argmin(), x.shape))))
print('max : {:.4f}, argmax : {}'.format(
x.max(), str(np.unravel_index(x.argmax(), x.shape))))
def log_cb(vk):
global _iter, _args, _prev_time, _log_interbal, _elapsed_time
now = time.time()
_log_interbal += now - _prev_time
_elapsed_time += now - _prev_time
if _log_interbal > _LOGINTERBAL or _iter == 0:
if GPU:
ev = np.asnumpy(E(vk, *_args))
else:
ev = E(vk, *_args)
print('iter {:4d}, E: {:.4f}, time: {:.1f} s'.format(
_iter, ev, _elapsed_time))
_log_interbal = 0
img0, _, _, _, _, _ = _args
v = vk.reshape([img0.shape[0], img0.shape[1], 2])
np.save('.cache/v.npy', v)
_prev_time = now
_iter += 1
def image_morphing(img0_path, img1_path, p0_path, p1_path, vmax_size=32, render_name='animation.mov',
lr_v=7e-2, tol_v=1e-1, lr_w=7e-2, tol_w=1e-3, lambda_tps=1e-3, gamma_ui=1e2,
tol_count_v=20, tol_count_w=3, render=False, render_steps=60, render_time=1,
save_dir='.cache'):
img0_src = cv2.imread(img0_path)
img1_src = cv2.imread(img1_path)
p0_src = load_points(p0_path)
p1_src = load_points(p1_path)
size = 8
v = np.random.randn(size, size, 2)
w = np.random.randn(size, size, 2)
# sizes = np.arange(8, vmax_size + 1, 8)
sizes = 2 ** np.arange(3, 10)
sizes = sizes[sizes <= vmax_size]
if GPU:
sizes = np.asnumpy(sizes)
for size in sizes:
print('\nOptimization size {:3d} start.'.format(size))
name = os.path.join(save_dir, 'v{:03d}'.format(size))
if os.path.exists(name):
v = np.load(name)
else:
print('Optimization of v start.')
v = adam(size, img0_src, img1_src, v, p0_src, p1_src, lr=lr_v, tol=tol_v,
render=render, tol_count=tol_count_v, lambda_tps=lambda_tps,
gamma_ui=gamma_ui, save_dir=save_dir)
name = os.path.join(save_dir, 'w{:03d}'.format(size))
if os.path.exists(name):
w = np.load(w)
else:
print('Optimization of w start.')
w = adam_w(size, w, v, lr=lr_w, tol=tol_w, tol_count=tol_count_w,
save_dir=save_dir)
v_final = resize_v(v=v, size=img0_src.shape[0], size_x=img0_src.shape[1])
w_final = resize_v(v=w, size=img0_src.shape[0], size_x=img0_src.shape[1])
img1 = cv2.resize(img1_src, (img0_src.shape[0], img0_src.shape[1]))
render_path = os.path.join(save_dir, render_name)
render_animation(img0_src, img1, v_final, w=w_final, steps=render_steps,
time=render_time, file_name=render_path)
def sgd(size, img0_src, img1_src, v_src, p0_src, p1_src, tol=1e-1, lambda_tps=1e-3, gamma_ui=1e2, lr=1e-2, return_half=False):
img0, img1, v, p0, p1 = resize_all(size, img0_src, img1_src, v_src, p0_src, p1_src)
args = (img0, img1, p0, p1, lambda_tps, gamma_ui)
iter = 0
start = prev = time.time()
log_interbal = 0
prev_e = 1e10
while 1:
dedv, e = dEdv(v, *args)
# describe(dedv)
v = v - lr * dedv
iter += 1
log_interbal += time.time() - prev
prev = time.time()
if log_interbal > _LOGINTERBAL:
if GPU:
ev = np.asnumpy(E(v, *args))
else:
ev = E(v, *args)
print('iter {:4d}, E: {:.4f}, time: {:.1f} s'.format(
iter, ev, time.time() - start))
log_interbal = 0
if abs(prev_e - e) < tol:
break
prev_e = e
end = time.time()
print('Time: {:.3f}'.format(end - start))
name = '.cache/v{:03d}_{}'.format(size, datetime.now().strftime('%m%d%H%M'))
img0_256, img1_256, v256, _, _ = resize_all(256, img0_src, img1_src, v, p0_src, p1_src)
render_animation(img0_256, img1_256, v256, file_name=name+'.mov')
np.save(name + '.npy', v)
if return_half:
half = render(img0, img1, v)
return v, half
else:
return v
def resize_v(size, v, size_x=None):
if GPU:
v = np.asnumpy(v)
vx = v[:, :, 0]
vy = v[:, :, 1]
if size_x is None:
vx_low = cv2.resize(vx, (size, size))
vy_low = cv2.resize(vy, (size, size))
else:
vx_low = cv2.resize(vx, (size, size_x))
vy_low = cv2.resize(vy, (size, size_x))
if GPU:
vx_low = np.asarray(vx_low)
vy_low = np.asarray(vy_low)
vx_low = vx_low[:, :, np.newaxis]
vy_low = vy_low[:, :, np.newaxis]
v_low = np.concatenate([vx_low, vy_low], axis=2)
if size_x is None:
v_low = v_low / v.shape[0] * size
else:
v_low[:, :, 0] = v_low[:, :, 0] / v.shape[0] * size
v_low[:, :, 1] = v_low[:, :, 1] / v.shape[1] * size_x
return v_low
def adam(size, img0_src, img1_src, v_src, p0_src, p1_src, tol=1e-1, tol_count=20,
lambda_tps=1e-3, gamma_ui=1e2, lr=1e-4, beta1=0.9, beta2=0.999, eps=1e-8,
render=True, save_dir='.cache'):
img0, img1, v, p0, p1 = resize_all(size, img0_src, img1_src, v_src, p0_src, p1_src)
args = (img0, img1, p0, p1, lambda_tps, gamma_ui)
iter = 0
start = prev = time.time()
log_interbal = 0
prev_e = 1e10
count = 0
m = np.zeros_like(v)
vt = np.zeros_like(v)
try:
while 1:
grads, e = dEdv(v, *args)
iter += 1
lr_t = lr * np.sqrt(1.0 - beta2**iter) / (1.0 - beta1**iter)
m += (1 - beta1) * (grads - m)
vt += (1 - beta2) * (grads**2 - vt)
v -= lr_t * m / (np.sqrt(vt) + 1e-7)
#unbias_m += (1 - beta1) * (grads - m) # correct bias
#unbisa_b += (1 - beta2) * (grads*grads - vt) # correct bias
#params += lr * unbias_m / (np.sqrt(unbisa_b) + 1e-7)
log_interbal += time.time() - prev
prev = time.time()
if log_interbal > _LOGINTERBAL or iter == 1:
if GPU:
ev = np.asnumpy(E(v, *args))
else:
ev = E(v, *args)
print('iter {:4d}, E: {:.4f}, time: {:.1f} s'.format(
iter, ev, time.time() - start))
log_interbal = 0
if abs(prev_e - e) < tol and e < prev_e:
break
elif prev_e < e:
count += 1
if count >= tol_count:
break
prev_e = e
except KeyboardInterrupt:
print('Interrupted')
name = os.path.join(save_dir, 'v{:03d}'.format(size))
img0_256, img1_256, v256, _, _ = resize_all(256, img0_src, img1_src, v, p0_src, p1_src)
if render:
render_animation(img0_256, img1_256, v256, file_name=name+'.mov')
np.save(name + '.npy', v)
return v
end = time.time()
if GPU:
ev = np.asnumpy(E(v, *args))
else:
ev = E(v, *args)
print('iter {:4d}, E: {:.4f}, time: {:.1f} s'.format(
iter, ev, end - start))
print('Optimization of v finished!')
name = os.path.join(save_dir, 'v{:03d}'.format(size))
img0_256, img1_256, v256, _, _ = resize_all(256, img0_src, img1_src, v, p0_src, p1_src)
if render:
render_animation(img0_256, img1_256, v256, file_name=name+'.mov')
np.save(name + '.npy', v)
return v