def _runSFS(self, C0_32F, A_8U, N0_32F, AN_8U):
I0_32F = luminance(C0_32F)
L = lightEstimation(I0_32F, N0_32F, A_8U)
LdN = LdotN(L, N0_32F)
layer_area = A_8U > 0.5 * np.max(A_8U)
Cs = C0_32F[layer_area].reshape(-1, 3)
Is = LdN[layer_area].flatten()
M = ColorMapEstimation(Cs, Is)
I_const = M.illumination(Cs)
I_32F = np.array(I0_32F)
I_32F[layer_area] = I_const
I_32F[A_8U < 0.5 * np.max(A_8U)] = np.min(I_const)
toon_sfs = ToonSFS(L, C0_32F, A_8U)
toon_sfs.setInitialNormal(N0_32F)
toon_sfs.run()
N_32F = toon_sfs.normal()
LdN_recovered = LdotN(L, N_32F)
LdN_recovered[A_8U < 0.5 * np.max(A_8U)] = np.min(LdN_recovered)
Is = LdN_recovered[layer_area].flatten()
Cs_recovered = M.shading(Is)
C_32F = np.array(C0_32F)
C_32F[layer_area, :] = Cs_recovered
return N_32F, L, C_32F, M
def _relightingImages(self, N_32F, A_8U, M):
Ls = self._lightAnimation()
h, w = N_32F.shape[:2]
images = []
for L in Ls:
LdN = LdotN(L, N_32F)
C = M.shading(LdN.flatten()).reshape(h, w, -1)
C[A_8U < 0.5 * np.max(A_8U), :] = 0
images.append(C)
return images
def _relightingImages(self, N_32F, A_8U, M):
Ls = self._lightAnimation()
h, w = N_32F.shape[:2]
images = []
for L in Ls:
LdN = LdotN(L, N_32F)
C = M.shading(LdN.flatten()).reshape(h, w, -1)
C[A_8U < 0.5 * np.max(A_8U), :] = 0
images.append(C)
return images
def estimatedReflectance(C0_32F, L, N_32F, A_8U):
LdN = LdotN(L, N_32F)
layer_area = A_8U > 0.5 * np.max(A_8U)
Cs = C0_32F[layer_area].reshape(-1, 3)
Is = LdN[layer_area].flatten()
M = ColorMapEstimation(Cs, Is)
return M
def computeErrors(L, C0_32F, C_32F, Ng_32F, N_32F, A_8U):
h, w = A_8U.shape
N_error = angleErros(N_32F.reshape(-1, 3),
Ng_32F.reshape(-1, 3)).reshape(h, w)
N_error[A_8U < 0.5 * np.max(A_8U)] = 0.0
N_error = trim(N_error, A_8U)
C_error = normVectors(C0_32F.reshape(-1, 3) -
C_32F.reshape(-1, 3)).reshape(h, w)
C_error[A_8U < 0.5 * np.max(A_8U)] = 0.0
C_error = trim(C_error, A_8U)
I_32F = np.clip(LdotN(L, N_32F), 0.0, 1.0)
Ig_32F = np.clip(LdotN(L, Ng_32F), 0.0, 1.0)
I_error = np.abs(I_32F - Ig_32F)
I_error[A_8U < 0.5 * np.max(A_8U)] = 0.0
I_error = trim(I_error, A_8U)
return C_error, N_error, I_error
def _relightingOffsetImages(self, L0, C0_32F, N_32F, A_8U, M):
Cs = C0_32F.reshape(-1, 3)
I0 = M.illumination(Cs)
I0_32F = np.float32(I0.reshape(C0_32F.shape[:2]))
I0_32F = cv2.bilateralFilter(I0_32F, 0, 0.1, 3)
I0 = I0_32F.flatten()
LdN0 = LdotN(L0, N_32F).flatten()
dI = I0 - LdN0
Ls = self._lightAnimation()
h, w = N_32F.shape[:2]
images = []
for L in Ls:
LdN = LdotN(L, N_32F).flatten()
C = M.shading(LdN + dI).reshape(h, w, -1)
C[A_8U < 0.5 * np.max(A_8U), :] = 0
images.append(C)
return images
def diffuseTerm(self, L, N_32F):
L = normalizeVector(L)
LdN = LdotN(L, N_32F)
I_32F = np.clip(LdN, 0.0, 1.0)
return np.float32(I_32F)
def computeIllumination(L, N_32F, A_8U):
I_32F = np.clip(LdotN(L, N_32F), 0.0, 1.0)
I_32F[A_8U < 0.5 * np.max(A_8U)] = 0.0
I_32F = setAlpha(gray2rgb(to8U(I_32F)), A_8U)
I_32F = trim(I_32F, A_8U)
return I_32F
def reflectanceMatching(C0_32F, L, N_32F, A_8U):
LdN = LdotN(L, N_32F)
M = estimatedReflectance(C0_32F, L, N_32F, A_8U)
C_32F = M.shading(LdN.flatten()).reshape(C0_32F.shape)
return C_32F
def diffuseTerm(self, L, N_32F):
LdN = LdotN(L, N_32F)
I_32F = 0.5 * LdN + 0.5
I_32F = np.clip(I_32F, 0.0, 1.0)
return np.float32(I_32F)
def overviewFigure():
cmap_id = 10
colormap_file = colorMapFile(cmap_id)
num_rows = 1
num_cols = 5
w = 10
h = w * num_rows / num_cols
fig, axes = plt.subplots(figsize=(w, h))
font_size = 15
fig.subplots_adjust(left=0.02, right=0.98, top=0.96, bottom=0.02, hspace=0.05, wspace=0.05)
fig.suptitle("", fontsize=font_size)
plot_grid = SubplotGrid(num_rows, num_cols)
L = normalizeVector(np.array([-0.4, 0.6, 0.6]))
L_img = lightSphere(L)
shape_name = "ThreeBox"
Ng_data = shapeFile(shape_name)
Ng_data = loadNormal(Ng_data)
Ng_32F, A_8U = Ng_data
N0_file = shapeResultFile(result_name="InitialNormal", data_name=shape_name)
N0_data = loadNormal(N0_file)
N0_32F, A_8U = N0_data
M_32F = loadColorMap(colormap_file)
Cg_32F = ColorMapShader(M_32F).diffuseShading(L, Ng_32F)
borders=[0.6, 0.8, 0.92]
colors = [np.array([0.2, 0.2, 0.4]),
np.array([0.3, 0.3, 0.6]),
np.array([0.4, 0.4, 0.8]),
np.array([0.5, 0.5, 1.0])]
#Cg_32F = ToonShader(borders, colors).diffuseShading(L, Ng_32F)
#Cg_32F = cv2.GaussianBlur(Cg_32F, (0,0), 2.0)
sfs_method = ToonSFS(L, Cg_32F, A_8U)
sfs_method.setInitialNormal(N0_32F)
sfs_method.setNumIterations(iterations=40)
sfs_method.setWeights(w_lap=10.0)
sfs_method.run()
N_32F = sfs_method.normal()
I_32F = np.float32(np.clip(LdotN(L, N_32F), 0.0, 1.0))
I0_32F = np.float32(np.clip(LdotN(L, N0_32F), 0.0, 1.0))
C_32F = sfs_method.shading()
C0_32F = sfs_method.initialShading()
M_32F = sfs_method.colorMap().mapImage()
L1 = normalizeVector(np.array([0.0, 0.6, 0.6]))
L1_img = lightSphere(L1)
C1_32F = sfs_method.relighting(L1)
L2 = normalizeVector(np.array([0.5, 0.8, 0.6]))
L2_img = lightSphere(L2)
C2_32F = sfs_method.relighting(L2)
N_sil = silhouetteNormal(A_8U, sigma=7.0)
N_sil[:, :, 2] = N_sil[:, :, 2] ** 10.0
N_sil = normalizeImage(N_sil)
A_sil = 1.0 - N_sil[:, :, 2]
A_sil = to8U(A_sil)
N_xy = N_sil[:, :, 0] ** 2 + N_sil[:, :, 1] ** 2
A_sil[N_xy < 0.1] = 0
title = ""
plot_grid.showImage(setAlpha(Cg_32F, to32F(A_8U)), title)
plot_grid.showImage(normalToColor(N0_32F, A_8U), title)
plot_grid.showImage(setAlpha(C0_32F, to32F(A_8U)), title)
plot_grid.showImage(normalToColor(N_32F, A_8U), title)
plot_grid.showImage(setAlpha(C_32F, to32F(A_8U)), title)
# plot_grid.showImage(normalToColor(Ng_32F, A_8U), title)
#showMaximize()
file_path = shapeResultFile("Overview", "Overview")
fig.savefig(file_path, transparent=True)
file_path = shapeResultFile("Overview", "Cg")
saveRGBA(file_path, setAlpha(Cg_32F, to32F(A_8U)))
file_path = shapeResultFile("Overview", "L")
saveRGB(file_path, gray2rgb(to8U(L_img)))
file_path = shapeResultFile("Overview", "L1")
saveRGB(file_path, gray2rgb(to8U(L1_img)))
file_path = shapeResultFile("Overview", "L2")
saveRGB(file_path, gray2rgb(to8U(L2_img)))
file_path = shapeResultFile("Overview", "N0")
saveNormal(file_path, N0_32F, A_8U)
file_path = shapeResultFile("Overview", "N_sil")
saveNormal(file_path, N_sil, A_sil)
file_path = shapeResultFile("Overview", "N")
saveNormal(file_path, N_32F, A_8U)
file_path = shapeResultFile("Overview", "C0")
saveRGBA(file_path, setAlpha(C0_32F, to32F(A_8U)))
file_path = shapeResultFile("Overview", "C")
saveRGBA(file_path, setAlpha(C_32F, to32F(A_8U)))
file_path = shapeResultFile("Overview", "C1")
saveRGBA(file_path, setAlpha(C1_32F, to32F(A_8U)))
file_path = shapeResultFile("Overview", "C2")
saveRGBA(file_path, setAlpha(C2_32F, to32F(A_8U)))
file_path = shapeResultFile("Overview", "I")
saveRGBA(file_path, setAlpha(gray2rgb(I_32F), to32F(A_8U)))
file_path = shapeResultFile("Overview", "I0")
saveRGBA(file_path, setAlpha(gray2rgb(I0_32F), to32F(A_8U)))
file_path = shapeResultFile("Overview", "M")
saveRGB(file_path, M_32F)
def reflectanceMatching(C0_32F, L, N_32F, A_8U):
LdN = LdotN(L, N_32F)
M = estimatedReflectance(C0_32F, L, N_32F, A_8U)
C_32F = M.shading(LdN.flatten()).reshape(C0_32F.shape)
return C_32F
def relightingVideo(shape_name="Ogre", cmap_id=3):
num_methods = 3
num_rows = 1
num_cols = num_methods + 2
num_lights = 120
w = 10
h = 5
fig, axes = plt.subplots(figsize=(w, h))
font_size = 15
fig.subplots_adjust(left=0.02,
right=0.98,
top=0.96,
bottom=0.04,
hspace=0.15,
wspace=0.1)
fig.suptitle("Shading Analysis", fontsize=font_size)
plot_grid = SubplotGrid(num_rows, num_cols)
Lg = normalizeVector(np.array([-0.2, 0.3, 0.5]))
Lg_img = lightSphere(Lg)
L1 = normalizeVector(np.array([0.5, 0.5, 0.6]))
Ls = [
normalizeVector(Lg * (1.0 - t) + t * L1)
for t in np.linspace(0.0, 1.0, num_lights)
]
# Ls = [normalizeVector(Lg + 1.0 * np.cos(t) * np.array([1, 0, 0]) + 1.0 * np.sin(t) * np.array([0, 1, 0])) for t in np.linspace(0.0, 1.0, num_lights) ]
Ng_data = shapeFile(shape_name)
Ng_data = loadNormal(Ng_data)
Ng_32F, A_8U = Ng_data
N0_file = shapeResultFile(result_name="InitialNormal",
data_name=shape_name)
N0_data = loadNormal(N0_file)
N0_32F, A_8U = N0_data
A_8U = cv2.bilateralFilter(A_8U, 0, 5, 2)
colormap_file = colorMapFile(cmap_id)
M_32F = loadColorMap(colormap_file)
C0_32F = ColorMapShader(M_32F).diffuseShading(Lg, Ng_32F)
toon_sfs = ToonSFS(Lg, C0_32F, A_8U)
toon_sfs.setInitialNormal(N0_32F)
toon_sfs.setNumIterations(iterations=100)
toon_sfs.setWeights(w_lap=0.2)
toon_sfs.run()
N_toon = toon_sfs.normal()
C_toon = toon_sfs.shading()
C_lumo, N_lumo = lumoSFS(C0_32F, Lg, N0_32F, A_8U)
C_wu, N_wu = wuSFS(C0_32F, Lg, N0_32F, A_8U)
M_lumo = estimatedReflectance(C0_32F, Lg, N_lumo, A_8U)
M_wu = estimatedReflectance(C0_32F, Lg, N_wu, A_8U)
plot_grid.showImage(Lg_img, "Light direction")
plot_grid.showImage(setAlpha(C0_32F, to32F(A_8U)), "Ground-truth")
title = ""
plot_grid.showImage(setAlpha(C_lumo, to32F(A_8U)), "Lumo")
#plot_grid.showColorMap(C_error_lumo, title, v_min=0, v_max=0.1, with_colorbar=True)
plot_grid.showImage(setAlpha(C_wu, to32F(A_8U)), "Lambert assumption")
#plot_grid.showColorMap(C_error_wu, title, v_min=0, v_max=0.1, with_colorbar=True)
plot_grid.showImage(setAlpha(C_toon, to32F(A_8U)), "Our result")
#plot_grid.showColorMap(C_error_toon, title, v_min=0, v_max=0.1, with_colorbar=True)
images = []
for i in xrange(48):
images.append(figure2numpy(fig))
for li, L in enumerate(Ls):
print li
fig.clear()
fig.suptitle("Relighting", fontsize=font_size)
plot_grid.setPlot(1, 1)
C1 = ColorMapShader(M_32F).diffuseShading(L, Ng_32F)
C1_lumo = M_lumo.shading(LdotN(L,
N_lumo).flatten()).reshape(C0_32F.shape)
C1_wu = M_wu.shading(LdotN(L, N_wu).flatten()).reshape(C0_32F.shape)
C1_toon = toon_sfs.relighting(L)
plot_grid.showImage(lightSphere(L), "Light direction")
plot_grid.showImage(setAlpha(C1, to32F(A_8U)), "Ground-truth")
title = ""
plot_grid.showImage(setAlpha(C1_lumo, to32F(A_8U)), "Lumo")
#plot_grid.showColorMap(C_error_lumo, title, v_min=0, v_max=0.1, with_colorbar=True)
plot_grid.showImage(setAlpha(C1_wu, to32F(A_8U)), "Lambert assumption")
#plot_grid.showColorMap(C_error_wu, title, v_min=0, v_max=0.1, with_colorbar=True)
plot_grid.showImage(setAlpha(C1_toon, to32F(A_8U)), "Our result")
images.append(figure2numpy(fig))
file_path = shapeResultFile("Relighting",
"Relighting_%s_%s" % (shape_name, cmap_id),
file_ext=".wmv")
saveVideo(file_path, images)
def relightingFigure(shape_name="Vase", cmap_id=3):
num_methods = 3
num_lights = 2
num_rows = num_lights + 1
num_cols = num_methods + 2
w = 15
h = w * num_rows / num_cols
fig, axes = plt.subplots(figsize=(w, h))
font_size = 15
fig.subplots_adjust(left=0.02,
right=0.98,
top=0.96,
bottom=0.04,
hspace=0.15,
wspace=0.1)
fig.suptitle("", fontsize=font_size)
plot_grid = SubplotGrid(num_rows, num_cols)
Lg = normalizeVector(np.array([-0.2, 0.3, 0.5]))
Lg_img = lightSphere(Lg)
L1 = normalizeVector(np.array([0.0, 0.7, 0.6]))
L2 = normalizeVector(np.array([0.3, 0.5, 0.6]))
# Ls = [normalizeVector(Lg * (1.0 - t) + t * L1) for t in np.linspace(0.0, 1.0, num_lights) ]
Ls = [L1, L2]
Ng_data = shapeFile(shape_name)
Ng_data = loadNormal(Ng_data)
Ng_32F, A_8U = Ng_data
N0_file = shapeResultFile(result_name="InitialNormal",
data_name=shape_name)
N0_data = loadNormal(N0_file)
N0_32F, A_8U = N0_data
A_8U = cv2.bilateralFilter(A_8U, 0, 5, 2)
colormap_file = colorMapFile(cmap_id)
M_32F = loadColorMap(colormap_file)
C0_32F = ColorMapShader(M_32F).diffuseShading(Lg, Ng_32F)
toon_sfs = ToonSFS(Lg, C0_32F, A_8U)
toon_sfs.setInitialNormal(N0_32F)
toon_sfs.setNumIterations(iterations=50)
toon_sfs.setWeights(w_lap=0.2)
toon_sfs.run()
N_toon = toon_sfs.normal()
C_toon = toon_sfs.shading()
C_lumo, N_lumo = lumoSFS(C0_32F, Lg, N0_32F, A_8U)
C_wu, N_wu = wuSFS(C0_32F, Lg, N0_32F, A_8U)
M_lumo = estimatedReflectance(C0_32F, Lg, N_lumo, A_8U)
M_wu = estimatedReflectance(C0_32F, Lg, N_wu, A_8U)
plot_grid.showImage(Lg_img, "Light direction")
plot_grid.showImage(setAlpha(C0_32F, to32F(A_8U)), "Ground-truth")
title = ""
plot_grid.showImage(setAlpha(C_lumo, to32F(A_8U)), "Lumo")
#plot_grid.showColorMap(C_error_lumo, title, v_min=0, v_max=0.1, with_colorbar=True)
plot_grid.showImage(setAlpha(C_wu, to32F(A_8U)), "Lambert assumption")
#plot_grid.showColorMap(C_error_wu, title, v_min=0, v_max=0.1, with_colorbar=True)
plot_grid.showImage(setAlpha(C_toon, to32F(A_8U)), "Our result")
#plot_grid.showColorMap(C_error_toon, title, v_min=0, v_max=0.1, with_colorbar=True)
for L in Ls:
C1 = ColorMapShader(M_32F).diffuseShading(L, Ng_32F)
C1_lumo = M_lumo.shading(LdotN(L,
N_lumo).flatten()).reshape(C0_32F.shape)
C1_wu = M_wu.shading(LdotN(L, N_wu).flatten()).reshape(C0_32F.shape)
C1_toon = toon_sfs.relighting(L)
plot_grid.showImage(lightSphere(L), "")
plot_grid.showImage(setAlpha(C1, to32F(A_8U)), "")
title = ""
plot_grid.showImage(setAlpha(C1_lumo, to32F(A_8U)), "")
#plot_grid.showColorMap(C_error_lumo, title, v_min=0, v_max=0.1, with_colorbar=True)
plot_grid.showImage(setAlpha(C1_wu, to32F(A_8U)), "")
#plot_grid.showColorMap(C_error_wu, title, v_min=0, v_max=0.1, with_colorbar=True)
plot_grid.showImage(setAlpha(C1_toon, to32F(A_8U)), "")
# showMaximize()
file_path = shapeResultFile("Relighting",
"RelightingComparison",
file_ext=".png")
fig.savefig(file_path, transparent=True)
def _optimize(self):
L = self._L
N0_32F = self._N0_32F
C0_32F = self._C0_32F
LdN = LdotN(L, N0_32F)
A_8U = self._A_8U
layer_area = A_8U > 0.5 * np.max(A_8U)
Cs = C0_32F[layer_area].reshape(-1, 3)
Is = LdN[layer_area].flatten()
M = ColorMapEstimation(Cs, Is)
self._Cini_32F = M.shading(LdN.flatten()).reshape(self._C0_32F.shape)
I_recovered = M.illumination(Cs)
I_32F = np.zeros(LdN.shape)
I_32F[:] = np.min(I_recovered)
I_32F[layer_area] = I_recovered
I_dog = DoG(I_32F, sigma=2.0)
sigma = 2.0
I_32F = np.float32(cv2.GaussianBlur(I_32F, (0, 0), sigma))
I_lap = -cv2.Laplacian(np.float32(I_32F), cv2.CV_32F, ksize=1)
I_lap = np.abs(I_lap)
I_lap_median = np.median(I_lap[layer_area])
print "I_lap_mean", np.mean(I_lap[layer_area])
print "I_lap_median", np.median(I_lap[layer_area])
sigma = 0.05
epsilon = 0.0 * I_lap_median
w_min = 0.05
w_max = 1.0
W_32F = w_min + (w_max - w_min) * (1.0 - np.exp(- (I_lap - epsilon) ** 2 / (sigma ** 2)))
W_32F = w_min * np.ones(I_32F.shape[:2])
W_32F[I_lap < epsilon] = w_min
W_32F[I_lap > epsilon] = w_max
W_32F[:, :] = 1.0
constraints = []
#W_32F = np.ones(N0_32F.shape[:2])
#constraints.append(image_constraints.normalConstraints(W_32F, N0_32F, w_c=0.05))
w_lap = self._w_lap
constraints.append(image_constraints.laplacianConstraints(w_c=w_lap))
constraints.append(image_constraints.brightnessConstraints(L, I_32F, w_c=1.0))
constraints.append(image_constraints.gradientConstraints(L, I_32F, w_c=0.2))
w_sil = 0.4 * w_lap
constraints.append(image_constraints.silhouetteConstraints(A_8U, w_c=w_sil))
N_32F = np.array(N0_32F, dtype=np.float64)
solver_iter = image_solver.solveIterator(constraints,
[image_constraints.postNormalize(th=0.0)])
#[image_constraints.postComputeNz()])
N_32F = image_solver.solveMG(N_32F, solver_iter, iterations=self._iterations)
self._N_32F = N_32F
LdN = LdotN(L, N_32F)
I_32F[layer_area] = LdN[layer_area]
self._C_32F = M.shading(I_32F.flatten()).reshape(self._C0_32F.shape)
self._M = M
def relighting(self, L):
N_32F = self._N_32F
LdN = LdotN(L, N_32F)
return self._M.shading(LdN.flatten()).reshape(self._C0_32F.shape)