def build_inv_delta_C_paddle(self, C):
""" Return inv_delta_C which is needed to calculate T """
F = self.F
hat_eye = paddle.eye(F, dtype='float64') # F x F
hat_C = paddle.norm(C.reshape([1, F, 2]) - C.reshape([F, 1, 2]),
axis=2) + hat_eye
hat_C = (hat_C**2) * paddle.log(hat_C)
delta_C = paddle.concat( # F+3 x F+3
[
paddle.concat([paddle.ones((F, 1), dtype='float64'), C, hat_C],
axis=1), # F x F+3
paddle.concat([
paddle.zeros((2, 3), dtype='float64'),
paddle.transpose(C, perm=[1, 0])
],
axis=1), # 2 x F+3
paddle.concat([
paddle.zeros((1, 3), dtype='float64'),
paddle.ones((1, F), dtype='float64')
],
axis=1) # 1 x F+3
],
axis=0)
inv_delta_C = paddle.inverse(delta_C)
return inv_delta_C # F+3 x F+3
def create_sparse_motions(self, source_image, kp_driving, kp_source):
"""
Eq 4. in the paper T_{s<-d}(z)
"""
bs, _, h, w = source_image.shape
identity_grid = make_coordinate_grid((h, w),
type=kp_source['value'].dtype)
identity_grid = identity_grid.reshape([1, 1, h, w, 2])
coordinate_grid = identity_grid - kp_driving['value'].reshape(
[bs, self.num_kp, 1, 1, 2])
if 'jacobian' in kp_driving:
jacobian = paddle.matmul(kp_source['jacobian'],
paddle.inverse(kp_driving['jacobian']))
jacobian = jacobian.unsqueeze(-3).unsqueeze(-3)
jacobian = paddle.tile(jacobian, [1, 1, h, w, 1, 1])
coordinate_grid = paddle.matmul(jacobian,
coordinate_grid.unsqueeze(-1))
coordinate_grid = coordinate_grid.squeeze(-1)
driving_to_source = coordinate_grid + kp_source['value'].reshape(
[bs, self.num_kp, 1, 1, 2])
#adding background feature
identity_grid = paddle.tile(identity_grid, (bs, 1, 1, 1, 1))
sparse_motions = paddle.concat([identity_grid, driving_to_source],
axis=1)
return sparse_motions
def normalize_kp(kp_source,
kp_driving,
kp_driving_initial,
adapt_movement_scale=False,
use_relative_movement=False,
use_relative_jacobian=False):
if adapt_movement_scale:
source_area = ConvexHull(kp_source['value'][0].numpy()).volume
driving_area = ConvexHull(kp_driving_initial['value'][0].numpy()).volume
adapt_movement_scale = np.sqrt(source_area) / np.sqrt(driving_area)
else:
adapt_movement_scale = 1
kp_new = {k: v for k, v in kp_driving.items()}
if use_relative_movement:
kp_value_diff = (kp_driving['value'] - kp_driving_initial['value'])
kp_value_diff *= adapt_movement_scale
kp_new['value'] = kp_value_diff + kp_source['value']
if use_relative_jacobian:
jacobian_diff = paddle.matmul(
kp_driving['jacobian'],
paddle.inverse(kp_driving_initial['jacobian']))
kp_new['jacobian'] = paddle.matmul(jacobian_diff,
kp_source['jacobian'])
return kp_new
def build_inv_delta_C_paddle(self, C):
""" Return inv_delta_C which is needed to calculate T """
F = self.F
hat_C = paddle.zeros((F, F), dtype='float32') # F x F
for i in range(0, F):
for j in range(i, F):
if i == j:
hat_C[i, j] = 1
else:
r = paddle.norm(C[i] - C[j])
hat_C[i, j] = r
hat_C[j, i] = r
hat_C = (hat_C**2) * paddle.log(hat_C)
delta_C = paddle.concat( # F+3 x F+3
[
paddle.concat([paddle.ones(
(F, 1)), C, hat_C], axis=1), # F x F+3
paddle.concat(
[paddle.zeros((2, 3)),
paddle.transpose(C, perm=[1, 0])],
axis=1), # 2 x F+3
paddle.concat([paddle.zeros(
(1, 3)), paddle.ones((1, F))],
axis=1) # 1 x F+3
],
axis=0)
inv_delta_C = paddle.inverse(delta_C)
return inv_delta_C # F+3 x F+3
def create_sparse_motions(self, source_image, kp_driving, kp_source):
"""
Eq 4. in the paper T_{s<-d}(z)
"""
bs, _, h, w = source_image.shape
identity_grid = make_coordinate_grid((h, w))
identity_grid = identity_grid.reshape((1, 1, h, w, 2))
coordinate_grid = identity_grid - kp_driving['value'].reshape(
(bs, self.num_kp, 1, 1, 2))
if 'jacobian' in kp_driving:
jacobian = paddle.matmul(kp_source['jacobian'],
paddle.inverse(kp_driving['jacobian']))
dim_1, dim_2, *else_dim = jacobian.shape
jacobian = jacobian.reshape((-1, *else_dim))
jacobian = jacobian.unsqueeze(-3).unsqueeze(-3)
jacobian = jacobian.tile((1, h, w, 1, 1))
_, _, *dimm = coordinate_grid.shape
coordinate_grid = coordinate_grid.reshape((-1, *dimm))
coordinate_grid = paddle.matmul(jacobian,
coordinate_grid.unsqueeze(-1))
coordinate_grid = coordinate_grid.squeeze(-1)
coordinate_grid = coordinate_grid.reshape(
(dim_1, dim_2, *(coordinate_grid.shape[1:])))
driving_to_source = coordinate_grid + kp_source['value'].reshape(
(bs, self.num_kp, 1, 1, 2))
# adding background feature
identity_grid = identity_grid.tile((bs, 1, 1, 1, 1))
sparse_motions = paddle.concat([identity_grid, driving_to_source],
axis=1)
return sparse_motions
def create_sparse_motions(self, source_image, kp_driving, kp_source):
def _inverse(x):
assert x.shape[-1] == 2 and x.shape[-2] == 2
buf = fluid.layers.reshape(x, (-1, *(x.shape[-2:])))
a, b, c, d = buf[:, 0, 0], buf[:, 1, 1], buf[:, 0, 1], buf[:, 1, 0]
div = fluid.layers.reshape(a * b - c * d, (-1, 1, 1))
n_a, n_b, n_c, n_d = b, a, -c, -d
colum_0 = fluid.layers.stack([n_a, n_d], axis=-1)
colum_1 = fluid.layers.stack([n_c, n_b], axis=-1)
mat = fluid.layers.stack([colum_0, colum_1], axis=-1) / div
return fluid.layers.reshape(mat, x.shape)
"""
Eq 4. in the paper T_{s<-d}(z)
"""
bs, _, h, w = source_image.shape
identity_grid = make_coordinate_grid_cpu((h, w), np.float32)
identity_grid = identity_grid.reshape(1, 1, h, w, 2)
identity_grid = dygraph.to_variable(identity_grid)
_buf = fluid.layers.reshape(kp_driving['value'],
(bs, self.num_kp, 1, 1, 2))
max_shape = np.array(
[max(i, j) for i, j in zip(identity_grid.shape, _buf.shape)])
coordinate_grid = fluid.layers.expand(
identity_grid, (max_shape / np.array(identity_grid.shape)).astype(
np.uint8).tolist()) - fluid.layers.expand(
_buf, (max_shape / np.array(_buf.shape)).astype(
np.uint8).tolist())
if 'jacobian' in kp_driving:
if PP_v2:
jacobian = fluid.layers.matmul(
kp_source['jacobian'],
paddle.inverse(kp_driving['jacobian']))
else:
jacobian = fluid.layers.matmul(
kp_source['jacobian'], _inverse(kp_driving['jacobian']))
dim_1, dim_2, *else_dim = jacobian.shape
jacobian = fluid.layers.reshape(jacobian, (-1, *else_dim))
jacobian = fluid.layers.unsqueeze(
fluid.layers.unsqueeze(jacobian, [-3]), [-3])
jacobian = _repeat(jacobian, (1, h, w, 1, 1))
_, _, *dimm = coordinate_grid.shape
coordinate_grid = fluid.layers.reshape(coordinate_grid,
(-1, *dimm))
_right = fluid.layers.unsqueeze(coordinate_grid, [-1])
coordinate_grid = fluid.layers.matmul(jacobian, _right)
coordinate_grid = fluid.layers.squeeze(coordinate_grid, [-1])
coordinate_grid = fluid.layers.reshape(
coordinate_grid, (dim_1, dim_2, *(coordinate_grid.shape[1:])))
driving_to_source = coordinate_grid + fluid.layers.reshape(
kp_source['value'], (bs, self.num_kp, 1, 1, 2))
# adding background feature
identity_grid = _repeat(identity_grid, (bs, 1, 1, 1, 1))
sparse_motions = fluid.layers.concat(
[identity_grid, driving_to_source], axis=1)
return sparse_motions
def test_dygraph(self):
for place in self.places:
with fluid.dygraph.guard(place):
input_np = np.random.random([4, 4]).astype("float64")
input = fluid.dygraph.to_variable(input_np)
result = paddle.inverse(input)
self.assertTrue(
np.allclose(result.numpy(), np.linalg.inv(input_np)))
def __init__(self,
output_image_size=None,
num_control_points=None,
margins=None):
super(TPSSpatialTransformer, self).__init__()
self.output_image_size = output_image_size
self.num_control_points = num_control_points
self.margins = margins
self.target_height, self.target_width = output_image_size
target_control_points = build_output_control_points(
num_control_points, margins)
N = num_control_points
# create padded kernel matrix
forward_kernel = paddle.zeros(shape=[N + 3, N + 3])
target_control_partial_repr = compute_partial_repr(
target_control_points, target_control_points)
target_control_partial_repr = paddle.cast(target_control_partial_repr,
forward_kernel.dtype)
forward_kernel[:N, :N] = target_control_partial_repr
forward_kernel[:N, -3] = 1
forward_kernel[-3, :N] = 1
target_control_points = paddle.cast(target_control_points,
forward_kernel.dtype)
forward_kernel[:N, -2:] = target_control_points
forward_kernel[-2:, :N] = paddle.transpose(target_control_points,
perm=[1, 0])
# compute inverse matrix
inverse_kernel = paddle.inverse(forward_kernel)
# create target cordinate matrix
HW = self.target_height * self.target_width
target_coordinate = list(
itertools.product(range(self.target_height),
range(self.target_width)))
target_coordinate = paddle.to_tensor(target_coordinate) # HW x 2
Y, X = paddle.split(target_coordinate,
target_coordinate.shape[1],
axis=1)
Y = Y / (self.target_height - 1)
X = X / (self.target_width - 1)
target_coordinate = paddle.concat(
[X, Y], axis=1) # convert from (y, x) to (x, y)
target_coordinate_partial_repr = compute_partial_repr(
target_coordinate, target_control_points)
target_coordinate_repr = paddle.concat([
target_coordinate_partial_repr,
paddle.ones(shape=[HW, 1]), target_coordinate
],
axis=1)
# register precomputed matrices
self.inverse_kernel = inverse_kernel
self.padding_matrix = paddle.zeros(shape=[3, 2])
self.target_coordinate_repr = target_coordinate_repr
self.target_control_points = target_control_points
def test_dygraph(self):
for place in self.places:
with fluid.dygraph.guard(place):
input_np = np.ones([4, 4]).astype("float64")
input = fluid.dygraph.to_variable(input_np)
try:
result = paddle.inverse(input)
except fluid.core.EnforceNotMet as ex:
print("The mat is singular")
pass
def check_static_result(self, place):
with fluid.program_guard(fluid.Program(), fluid.Program()):
input = fluid.data(name="input", shape=[4, 4], dtype="float64")
result = paddle.inverse(x=input)
input_np = np.random.random([4, 4]).astype("float64")
result_np = np.linalg.inv(input_np)
exe = fluid.Executor(place)
fetches = exe.run(fluid.default_main_program(),
feed={"input": input_np},
fetch_list=[result])
self.assertTrue(np.allclose(fetches[0], np.linalg.inv(input_np)))
def check_static_result(self, place):
with fluid.program_guard(fluid.Program(), fluid.Program()):
input = fluid.data(name="input", shape=[4, 4], dtype="float64")
result = paddle.inverse(x=input)
input_np = np.zeros([4, 4]).astype("float64")
exe = fluid.Executor(place)
try:
fetches = exe.run(fluid.default_main_program(),
feed={"input": input_np},
fetch_list=[result])
except fluid.core.EnforceNotMet as ex:
print("The mat is singular")
pass
def forward(self, x, discriminator):
kp_source = self.kp_extractor(x['source'])
kp_driving = self.kp_extractor(x['driving'])
generated = self.generator(x['source'],
kp_source=kp_source,
kp_driving=kp_driving)
generated.update({'kp_source': kp_source, 'kp_driving': kp_driving})
loss_values = {}
pyramide_real = self.pyramid(x['driving'])
pyramide_generated = self.pyramid(generated['prediction'])
# VGG19 perceptual Loss
if sum(self.loss_weights['perceptual']) != 0:
value_total = 0
for scale in self.scales:
x_vgg = self.vgg(pyramide_generated['prediction_' +
str(scale)])
y_vgg = self.vgg(pyramide_real['prediction_' + str(scale)])
for i, weight in enumerate(self.loss_weights['perceptual']):
value = paddle.abs(x_vgg[i] - y_vgg[i].detach()).mean()
value_total += self.loss_weights['perceptual'][i] * value
loss_values['perceptual'] = value_total
# Generator Loss
if self.loss_weights['generator_gan'] != 0:
discriminator_maps_generated = discriminator(
pyramide_generated, kp=detach_kp(kp_driving))
discriminator_maps_real = discriminator(pyramide_real,
kp=detach_kp(kp_driving))
value_total = 0
for scale in self.disc_scales:
key = 'prediction_map_%s' % scale
value = ((1 - discriminator_maps_generated[key])**2).mean()
value_total += self.loss_weights['generator_gan'] * value
loss_values['gen_gan'] = value_total
# Feature matching Loss
if sum(self.loss_weights['feature_matching']) != 0:
value_total = 0
for scale in self.disc_scales:
key = 'feature_maps_%s' % scale
for i, (a, b) in enumerate(
zip(discriminator_maps_real[key],
discriminator_maps_generated[key])):
if self.loss_weights['feature_matching'][i] == 0:
continue
value = paddle.abs(a - b).mean()
value_total += self.loss_weights['feature_matching'][
i] * value
loss_values['feature_matching'] = value_total
if (self.loss_weights['equivariance_value'] +
self.loss_weights['equivariance_jacobian']) != 0:
transform = Transform(x['driving'].shape[0],
**self.train_params['transform_params'])
transformed_frame = transform.transform_frame(x['driving'])
transformed_kp = self.kp_extractor(transformed_frame)
generated['transformed_frame'] = transformed_frame
generated['transformed_kp'] = transformed_kp
# Value loss part
if self.loss_weights['equivariance_value'] != 0:
value = paddle.abs(kp_driving['value'] -
transform.warp_coordinates(
transformed_kp['value'])).mean()
loss_values['equivariance_value'] = self.loss_weights[
'equivariance_value'] * value
# jacobian loss part
if self.loss_weights['equivariance_jacobian'] != 0:
jacobian_transformed = paddle.matmul(
*broadcast(transform.jacobian(transformed_kp['value']),
transformed_kp['jacobian']))
normed_driving = paddle.inverse(kp_driving['jacobian'])
normed_transformed = jacobian_transformed
value = paddle.matmul(
*broadcast(normed_driving, normed_transformed))
eye = paddle.tensor.eye(2, dtype='float32').reshape(
(1, 1, 2, 2))
value = paddle.abs(eye - value).mean()
loss_values['equivariance_jacobian'] = self.loss_weights[
'equivariance_jacobian'] * value
return loss_values, generated
def camera_to_lidar(points, r_rect, velo2cam):
num_points = points.shape[0]
points = paddle.concat(
[points, paddle.ones((num_points, 1)).astype(points.dtype)], axis=-1)
lidar_points = points @ paddle.inverse((r_rect @ velo2cam).t())
return lidar_points[..., :3]
def inverse(self, x):
return paddle.inverse(x)
