def build_inv_delta_C_paddle(self, C): """ Return inv_delta_C which is needed to calculate T """ F = self.F hat_eye = paddle.eye(F, dtype='float64') # F x F hat_C = paddle.norm(C.reshape([1, F, 2]) - C.reshape([F, 1, 2]), axis=2) + hat_eye hat_C = (hat_C**2) * paddle.log(hat_C) delta_C = paddle.concat( # F+3 x F+3 [ paddle.concat([paddle.ones((F, 1), dtype='float64'), C, hat_C], axis=1), # F x F+3 paddle.concat([ paddle.zeros((2, 3), dtype='float64'), paddle.transpose(C, perm=[1, 0]) ], axis=1), # 2 x F+3 paddle.concat([ paddle.zeros((1, 3), dtype='float64'), paddle.ones((1, F), dtype='float64') ], axis=1) # 1 x F+3 ], axis=0) inv_delta_C = paddle.inverse(delta_C) return inv_delta_C # F+3 x F+3
def create_sparse_motions(self, source_image, kp_driving, kp_source): """ Eq 4. in the paper T_{s<-d}(z) """ bs, _, h, w = source_image.shape identity_grid = make_coordinate_grid((h, w), type=kp_source['value'].dtype) identity_grid = identity_grid.reshape([1, 1, h, w, 2]) coordinate_grid = identity_grid - kp_driving['value'].reshape( [bs, self.num_kp, 1, 1, 2]) if 'jacobian' in kp_driving: jacobian = paddle.matmul(kp_source['jacobian'], paddle.inverse(kp_driving['jacobian'])) jacobian = jacobian.unsqueeze(-3).unsqueeze(-3) jacobian = paddle.tile(jacobian, [1, 1, h, w, 1, 1]) coordinate_grid = paddle.matmul(jacobian, coordinate_grid.unsqueeze(-1)) coordinate_grid = coordinate_grid.squeeze(-1) driving_to_source = coordinate_grid + kp_source['value'].reshape( [bs, self.num_kp, 1, 1, 2]) #adding background feature identity_grid = paddle.tile(identity_grid, (bs, 1, 1, 1, 1)) sparse_motions = paddle.concat([identity_grid, driving_to_source], axis=1) return sparse_motions
def normalize_kp(kp_source, kp_driving, kp_driving_initial, adapt_movement_scale=False, use_relative_movement=False, use_relative_jacobian=False): if adapt_movement_scale: source_area = ConvexHull(kp_source['value'][0].numpy()).volume driving_area = ConvexHull(kp_driving_initial['value'][0].numpy()).volume adapt_movement_scale = np.sqrt(source_area) / np.sqrt(driving_area) else: adapt_movement_scale = 1 kp_new = {k: v for k, v in kp_driving.items()} if use_relative_movement: kp_value_diff = (kp_driving['value'] - kp_driving_initial['value']) kp_value_diff *= adapt_movement_scale kp_new['value'] = kp_value_diff + kp_source['value'] if use_relative_jacobian: jacobian_diff = paddle.matmul( kp_driving['jacobian'], paddle.inverse(kp_driving_initial['jacobian'])) kp_new['jacobian'] = paddle.matmul(jacobian_diff, kp_source['jacobian']) return kp_new
def build_inv_delta_C_paddle(self, C): """ Return inv_delta_C which is needed to calculate T """ F = self.F hat_C = paddle.zeros((F, F), dtype='float32') # F x F for i in range(0, F): for j in range(i, F): if i == j: hat_C[i, j] = 1 else: r = paddle.norm(C[i] - C[j]) hat_C[i, j] = r hat_C[j, i] = r hat_C = (hat_C**2) * paddle.log(hat_C) delta_C = paddle.concat( # F+3 x F+3 [ paddle.concat([paddle.ones( (F, 1)), C, hat_C], axis=1), # F x F+3 paddle.concat( [paddle.zeros((2, 3)), paddle.transpose(C, perm=[1, 0])], axis=1), # 2 x F+3 paddle.concat([paddle.zeros( (1, 3)), paddle.ones((1, F))], axis=1) # 1 x F+3 ], axis=0) inv_delta_C = paddle.inverse(delta_C) return inv_delta_C # F+3 x F+3
def create_sparse_motions(self, source_image, kp_driving, kp_source): """ Eq 4. in the paper T_{s<-d}(z) """ bs, _, h, w = source_image.shape identity_grid = make_coordinate_grid((h, w)) identity_grid = identity_grid.reshape((1, 1, h, w, 2)) coordinate_grid = identity_grid - kp_driving['value'].reshape( (bs, self.num_kp, 1, 1, 2)) if 'jacobian' in kp_driving: jacobian = paddle.matmul(kp_source['jacobian'], paddle.inverse(kp_driving['jacobian'])) dim_1, dim_2, *else_dim = jacobian.shape jacobian = jacobian.reshape((-1, *else_dim)) jacobian = jacobian.unsqueeze(-3).unsqueeze(-3) jacobian = jacobian.tile((1, h, w, 1, 1)) _, _, *dimm = coordinate_grid.shape coordinate_grid = coordinate_grid.reshape((-1, *dimm)) coordinate_grid = paddle.matmul(jacobian, coordinate_grid.unsqueeze(-1)) coordinate_grid = coordinate_grid.squeeze(-1) coordinate_grid = coordinate_grid.reshape( (dim_1, dim_2, *(coordinate_grid.shape[1:]))) driving_to_source = coordinate_grid + kp_source['value'].reshape( (bs, self.num_kp, 1, 1, 2)) # adding background feature identity_grid = identity_grid.tile((bs, 1, 1, 1, 1)) sparse_motions = paddle.concat([identity_grid, driving_to_source], axis=1) return sparse_motions
def create_sparse_motions(self, source_image, kp_driving, kp_source): def _inverse(x): assert x.shape[-1] == 2 and x.shape[-2] == 2 buf = fluid.layers.reshape(x, (-1, *(x.shape[-2:]))) a, b, c, d = buf[:, 0, 0], buf[:, 1, 1], buf[:, 0, 1], buf[:, 1, 0] div = fluid.layers.reshape(a * b - c * d, (-1, 1, 1)) n_a, n_b, n_c, n_d = b, a, -c, -d colum_0 = fluid.layers.stack([n_a, n_d], axis=-1) colum_1 = fluid.layers.stack([n_c, n_b], axis=-1) mat = fluid.layers.stack([colum_0, colum_1], axis=-1) / div return fluid.layers.reshape(mat, x.shape) """ Eq 4. in the paper T_{s<-d}(z) """ bs, _, h, w = source_image.shape identity_grid = make_coordinate_grid_cpu((h, w), np.float32) identity_grid = identity_grid.reshape(1, 1, h, w, 2) identity_grid = dygraph.to_variable(identity_grid) _buf = fluid.layers.reshape(kp_driving['value'], (bs, self.num_kp, 1, 1, 2)) max_shape = np.array( [max(i, j) for i, j in zip(identity_grid.shape, _buf.shape)]) coordinate_grid = fluid.layers.expand( identity_grid, (max_shape / np.array(identity_grid.shape)).astype( np.uint8).tolist()) - fluid.layers.expand( _buf, (max_shape / np.array(_buf.shape)).astype( np.uint8).tolist()) if 'jacobian' in kp_driving: if PP_v2: jacobian = fluid.layers.matmul( kp_source['jacobian'], paddle.inverse(kp_driving['jacobian'])) else: jacobian = fluid.layers.matmul( kp_source['jacobian'], _inverse(kp_driving['jacobian'])) dim_1, dim_2, *else_dim = jacobian.shape jacobian = fluid.layers.reshape(jacobian, (-1, *else_dim)) jacobian = fluid.layers.unsqueeze( fluid.layers.unsqueeze(jacobian, [-3]), [-3]) jacobian = _repeat(jacobian, (1, h, w, 1, 1)) _, _, *dimm = coordinate_grid.shape coordinate_grid = fluid.layers.reshape(coordinate_grid, (-1, *dimm)) _right = fluid.layers.unsqueeze(coordinate_grid, [-1]) coordinate_grid = fluid.layers.matmul(jacobian, _right) coordinate_grid = fluid.layers.squeeze(coordinate_grid, [-1]) coordinate_grid = fluid.layers.reshape( coordinate_grid, (dim_1, dim_2, *(coordinate_grid.shape[1:]))) driving_to_source = coordinate_grid + fluid.layers.reshape( kp_source['value'], (bs, self.num_kp, 1, 1, 2)) # adding background feature identity_grid = _repeat(identity_grid, (bs, 1, 1, 1, 1)) sparse_motions = fluid.layers.concat( [identity_grid, driving_to_source], axis=1) return sparse_motions
def test_dygraph(self): for place in self.places: with fluid.dygraph.guard(place): input_np = np.random.random([4, 4]).astype("float64") input = fluid.dygraph.to_variable(input_np) result = paddle.inverse(input) self.assertTrue( np.allclose(result.numpy(), np.linalg.inv(input_np)))
def __init__(self, output_image_size=None, num_control_points=None, margins=None): super(TPSSpatialTransformer, self).__init__() self.output_image_size = output_image_size self.num_control_points = num_control_points self.margins = margins self.target_height, self.target_width = output_image_size target_control_points = build_output_control_points( num_control_points, margins) N = num_control_points # create padded kernel matrix forward_kernel = paddle.zeros(shape=[N + 3, N + 3]) target_control_partial_repr = compute_partial_repr( target_control_points, target_control_points) target_control_partial_repr = paddle.cast(target_control_partial_repr, forward_kernel.dtype) forward_kernel[:N, :N] = target_control_partial_repr forward_kernel[:N, -3] = 1 forward_kernel[-3, :N] = 1 target_control_points = paddle.cast(target_control_points, forward_kernel.dtype) forward_kernel[:N, -2:] = target_control_points forward_kernel[-2:, :N] = paddle.transpose(target_control_points, perm=[1, 0]) # compute inverse matrix inverse_kernel = paddle.inverse(forward_kernel) # create target cordinate matrix HW = self.target_height * self.target_width target_coordinate = list( itertools.product(range(self.target_height), range(self.target_width))) target_coordinate = paddle.to_tensor(target_coordinate) # HW x 2 Y, X = paddle.split(target_coordinate, target_coordinate.shape[1], axis=1) Y = Y / (self.target_height - 1) X = X / (self.target_width - 1) target_coordinate = paddle.concat( [X, Y], axis=1) # convert from (y, x) to (x, y) target_coordinate_partial_repr = compute_partial_repr( target_coordinate, target_control_points) target_coordinate_repr = paddle.concat([ target_coordinate_partial_repr, paddle.ones(shape=[HW, 1]), target_coordinate ], axis=1) # register precomputed matrices self.inverse_kernel = inverse_kernel self.padding_matrix = paddle.zeros(shape=[3, 2]) self.target_coordinate_repr = target_coordinate_repr self.target_control_points = target_control_points
def test_dygraph(self): for place in self.places: with fluid.dygraph.guard(place): input_np = np.ones([4, 4]).astype("float64") input = fluid.dygraph.to_variable(input_np) try: result = paddle.inverse(input) except fluid.core.EnforceNotMet as ex: print("The mat is singular") pass
def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): input = fluid.data(name="input", shape=[4, 4], dtype="float64") result = paddle.inverse(x=input) input_np = np.random.random([4, 4]).astype("float64") result_np = np.linalg.inv(input_np) exe = fluid.Executor(place) fetches = exe.run(fluid.default_main_program(), feed={"input": input_np}, fetch_list=[result]) self.assertTrue(np.allclose(fetches[0], np.linalg.inv(input_np)))
def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): input = fluid.data(name="input", shape=[4, 4], dtype="float64") result = paddle.inverse(x=input) input_np = np.zeros([4, 4]).astype("float64") exe = fluid.Executor(place) try: fetches = exe.run(fluid.default_main_program(), feed={"input": input_np}, fetch_list=[result]) except fluid.core.EnforceNotMet as ex: print("The mat is singular") pass
def forward(self, x, discriminator): kp_source = self.kp_extractor(x['source']) kp_driving = self.kp_extractor(x['driving']) generated = self.generator(x['source'], kp_source=kp_source, kp_driving=kp_driving) generated.update({'kp_source': kp_source, 'kp_driving': kp_driving}) loss_values = {} pyramide_real = self.pyramid(x['driving']) pyramide_generated = self.pyramid(generated['prediction']) # VGG19 perceptual Loss if sum(self.loss_weights['perceptual']) != 0: value_total = 0 for scale in self.scales: x_vgg = self.vgg(pyramide_generated['prediction_' + str(scale)]) y_vgg = self.vgg(pyramide_real['prediction_' + str(scale)]) for i, weight in enumerate(self.loss_weights['perceptual']): value = paddle.abs(x_vgg[i] - y_vgg[i].detach()).mean() value_total += self.loss_weights['perceptual'][i] * value loss_values['perceptual'] = value_total # Generator Loss if self.loss_weights['generator_gan'] != 0: discriminator_maps_generated = discriminator( pyramide_generated, kp=detach_kp(kp_driving)) discriminator_maps_real = discriminator(pyramide_real, kp=detach_kp(kp_driving)) value_total = 0 for scale in self.disc_scales: key = 'prediction_map_%s' % scale value = ((1 - discriminator_maps_generated[key])**2).mean() value_total += self.loss_weights['generator_gan'] * value loss_values['gen_gan'] = value_total # Feature matching Loss if sum(self.loss_weights['feature_matching']) != 0: value_total = 0 for scale in self.disc_scales: key = 'feature_maps_%s' % scale for i, (a, b) in enumerate( zip(discriminator_maps_real[key], discriminator_maps_generated[key])): if self.loss_weights['feature_matching'][i] == 0: continue value = paddle.abs(a - b).mean() value_total += self.loss_weights['feature_matching'][ i] * value loss_values['feature_matching'] = value_total if (self.loss_weights['equivariance_value'] + self.loss_weights['equivariance_jacobian']) != 0: transform = Transform(x['driving'].shape[0], **self.train_params['transform_params']) transformed_frame = transform.transform_frame(x['driving']) transformed_kp = self.kp_extractor(transformed_frame) generated['transformed_frame'] = transformed_frame generated['transformed_kp'] = transformed_kp # Value loss part if self.loss_weights['equivariance_value'] != 0: value = paddle.abs(kp_driving['value'] - transform.warp_coordinates( transformed_kp['value'])).mean() loss_values['equivariance_value'] = self.loss_weights[ 'equivariance_value'] * value # jacobian loss part if self.loss_weights['equivariance_jacobian'] != 0: jacobian_transformed = paddle.matmul( *broadcast(transform.jacobian(transformed_kp['value']), transformed_kp['jacobian'])) normed_driving = paddle.inverse(kp_driving['jacobian']) normed_transformed = jacobian_transformed value = paddle.matmul( *broadcast(normed_driving, normed_transformed)) eye = paddle.tensor.eye(2, dtype='float32').reshape( (1, 1, 2, 2)) value = paddle.abs(eye - value).mean() loss_values['equivariance_jacobian'] = self.loss_weights[ 'equivariance_jacobian'] * value return loss_values, generated
def camera_to_lidar(points, r_rect, velo2cam): num_points = points.shape[0] points = paddle.concat( [points, paddle.ones((num_points, 1)).astype(points.dtype)], axis=-1) lidar_points = points @ paddle.inverse((r_rect @ velo2cam).t()) return lidar_points[..., :3]
def inverse(self, x): return paddle.inverse(x)