def __init__(self, radius, num_inputs, num_outputs, config):
super(KPConvLayer, self).__init__()
self.radius = radius
self.num_inputs = num_inputs
self.num_outputs = num_outputs
self.config = config
self.extent = self.config.NETWORK.KP_EXTENT * self.radius /\
self.config.NETWORK.DENSITY_PARAMETER
# Initial kernel extent for this layer
K_radius = 1.5 * self.extent
K_points_numpy = create_kernel_points(
K_radius,
self.config.NETWORK.NUM_KERNEL_POINTS,
num_kernels=1,
dimension=self.config.INPUT.POINTS_DIM,
fixed=self.config.NETWORK.FIXED_KERNEL_POINTS)
self.K_points = Parameter(torch.from_numpy(K_points_numpy.reshape((
self.config.NETWORK.NUM_KERNEL_POINTS,
self.config.INPUT.POINTS_DIM))).to(torch.float),
requires_grad=False)
self.weight = Parameter(
weight_variable([self.config.NETWORK.NUM_KERNEL_POINTS,
self.num_inputs,
self.num_outputs]))
def KPConv(query_points,
support_points,
neighbors_indices,
features,
K_values,
fixed='center',
KP_extent=1.0,
KP_influence='linear',
aggregation_mode='sum'):
"""
This function initiates the kernel point disposition before building KPConv graph ops
:param query_points: float32[n_points, dim] - input query points (center of neighborhoods)
:param support_points: float32[n0_points, dim] - input support points (from which neighbors are taken)
:param neighbors_indices: int32[n_points, n_neighbors] - indices of neighbors of each point
:param features: float32[n_points, in_fdim] - input features
:param K_values: float32[n_kpoints, in_fdim, out_fdim] - weights of the kernel
:param fixed: string in ('none', 'center' or 'verticals') - fix position of certain kernel points
:param KP_extent: float32 - influence radius of each kernel point
:param KP_influence: string in ('constant', 'linear', 'gaussian') - influence function of the kernel points
:param aggregation_mode: string in ('closest', 'sum') - whether to sum influences, or only keep the closest
:return: output_features float32[n_points, out_fdim]
"""
# Initial kernel extent for this layer
K_radius = 1.5 * KP_extent
# Number of kernel points
num_kpoints = int(K_values.shape[0])
# Check point dimension (currently only 3D is supported)
points_dim = int(query_points.shape[1])
# Create one kernel disposition (as numpy array). Choose the KP distance to center thanks to the KP extent
K_points_numpy = create_kernel_points(K_radius,
num_kpoints,
num_kernels=1,
dimension=points_dim,
fixed=fixed)
K_points_numpy = K_points_numpy.reshape((num_kpoints, points_dim))
# Create the tensorflow variable
K_points = tf.Variable(K_points_numpy.astype(np.float32),
name='kernel_points',
trainable=False,
dtype=tf.float32)
return KPConv_ops(query_points,
support_points,
neighbors_indices,
features,
K_points,
K_values,
KP_extent,
KP_influence,
aggregation_mode)
def __init__(self, radius, num_inputs,
num_outputs, config, version=0, modulated=False):
"""
it doesn't work yet :
"""
super(DeformableKPConvLayer, self).__init__()
self.radius = radius
self.num_inputs = num_inputs
self.num_outputs = num_outputs
self.config = config
self.extent = self.config.NETWORK.KP_EXTENT * self.radius /\
self.config.NETWORK.DENSITY_PARAMETER
self.version = version
self.modulated = modulated
# Initial kernel extent for this layer
K_radius = 1.5 * self.extent
K_points_numpy = create_kernel_points(
K_radius,
self.config.NETWORK.NUM_KERNEL_POINTS,
num_kernels=1,
dimension=self.config.INPUT.POINTS_DIM,
fixed=self.config.NETWORK.FIXED_KERNEL_POINTS)
self.K_points = Parameter(torch.from_numpy(K_points_numpy.reshape((
self.config.NETWORK.NUM_KERNEL_POINTS,
self.config.INPUT.POINTS_DIM))).to(torch.float),
requires_grad=False)
# Parameter of the deformable convolution
self.weight = Parameter(
weight_variable([self.config.NETWORK.NUM_KERNEL_POINTS,
self.num_inputs,
self.num_outputs]))
if(self.modulated):
offset_dim = (self.config.INPUT.POINTS_DIM+1) * (self.config.NETWORK.NUM_KERNEL_POINTS -1)
else:
offset_dim = (self.config.INPUT.POINTS_DIM) * (self.config.NETWORK.NUM_KERNEL_POINTS -1)
if(self.version == 0):
# kp conv to estimate the offset
self.deformable_weight = Parameter(
weight_variable([self.config.NETWORK.NUM_KERNEL_POINTS,
self.num_inputs,
offset_dim]))
elif(self.version == 1):
# MLP to estimate the offset
self.deformable_weight = Parameter(
weight_variable([self.num_inputs,
offset_dim]))
self.bias = torch.nn.Parameter(
torch.zeros(offset_dim, dtype=torch.float32), requires_grad=True)
def KPConv_deformable_v2(query_points,
support_points,
neighbors_indices,
features,
K_values,
fixed='center',
KP_extent=1.0,
KP_influence='linear',
aggregation_mode='sum',
modulated=False):
"""
This alternate version uses a pointwise MLP instead of KPConv to get the offset. It has thus less parameters.
It also fixes the center point to remain in the center in any case. This definition offers similar performances
:param query_points: float32[n_points, dim] - input query points (center of neighborhoods)
:param support_points: float32[n0_points, dim] - input support points (from which neighbors are taken)
:param neighbors_indices: int32[n_points, n_neighbors] - indices of neighbors of each point
:param features: float32[n_points, in_fdim] - input features
:param K_values: float32[n_kpoints, in_fdim, out_fdim] - weights of the kernel
:param fixed: string in ('none', 'center' or 'verticals') - fix position of certain kernel points
:param KP_extent: float32 - influence radius of each kernel point
:param KP_influence: string in ('constant', 'linear', 'gaussian') - influence function of the kernel points
:param aggregation_mode: string in ('closest', 'sum') - behavior of the convolution
:param modulated: bool - If deformable conv should be modulated
:return: output_features float32[n_points, out_fdim]
"""
############
# Parameters
############
# Check point dimension (currently only 3D is supported)
points_dim = int(query_points.shape[1])
# Number of kernel points
num_kpoints = int(K_values.shape[0])
#################
# MLP for offsets
#################
# Create independant weight for the first convolution and a bias term as no batch normalization happen
if modulated:
offset_dim = (points_dim + 1) * (num_kpoints - 1)
else:
offset_dim = points_dim * (num_kpoints - 1)
shape0 = K_values.shape.as_list()
w0 = tf.Variable(tf.zeros([shape0[1], offset_dim], dtype=tf.float32),
name='offset_mlp_weights')
b0 = tf.Variable(tf.zeros([offset_dim], dtype=tf.float32),
name='offset_mlp_bias')
# Get features from mlp
features0 = unary_convolution(features, w0) + b0
if modulated:
# Get offset (in normalized scale) from features
offsets = features0[:, :points_dim * (num_kpoints - 1)]
offsets = tf.reshape(offsets, [-1, (num_kpoints - 1), points_dim])
# Get modulations
modulations = 2 * tf.sigmoid(features0[:, points_dim *
(num_kpoints - 1):])
# No offset for the first Kernel points
offsets = tf.concat([tf.zeros_like(offsets[:, :1, :]), offsets],
axis=1)
modulations = tf.concat(
[tf.zeros_like(modulations[:, :1]), modulations], axis=1)
else:
# Get offset (in normalized scale) from features
offsets = tf.reshape(features0, [-1, (num_kpoints - 1), points_dim])
# No offset for the first Kernel points
offsets = tf.concat([tf.zeros_like(offsets[:, :1, :]), offsets],
axis=1)
# No modulations
modulations = None
# Rescale offset for this layer
offsets *= KP_extent
#################################
# Initiate kernel point positions
#################################
# Radius of the initial positions of the kernel points
K_radius = 1.5 * KP_extent
# Create one kernel disposition (as numpy array). Choose the KP distance to center thanks to the KP extent
K_points_numpy = create_kernel_points(K_radius,
num_kpoints,
num_kernels=1,
dimension=points_dim,
fixed=fixed)
K_points_numpy = K_points_numpy.reshape((num_kpoints, points_dim))
# Create the tensorflow variable
K_points = tf.Variable(K_points_numpy.astype(np.float32),
name='kernel_points',
trainable=False,
dtype=tf.float32)
###############################
# Build deformable KPConv graph
###############################
# Apply deformed convolution
return KPConv_deform_ops(query_points, support_points, neighbors_indices,
features, K_points, offsets, modulations,
K_values, KP_extent, KP_influence,
aggregation_mode)
def KPConv_deformable(query_points,
support_points,
neighbors_indices,
features,
K_values,
fixed='center',
KP_extent=1.0,
KP_influence='linear',
aggregation_mode='sum',
modulated=False):
"""
This function initiates the kernel point disposition before building deformable KPConv graph ops
:param query_points: float32[n_points, dim] - input query points (center of neighborhoods)
:param support_points: float32[n0_points, dim] - input support points (from which neighbors are taken)
:param neighbors_indices: int32[n_points, n_neighbors] - indices of neighbors of each point
:param features: float32[n_points, in_fdim] - input features
:param K_values: float32[n_kpoints, in_fdim, out_fdim] - weights of the kernel
:param fixed: string in ('none', 'center' or 'verticals') - fix position of certain kernel points
:param KP_extent: float32 - influence radius of each kernel point
:param KP_influence: string in ('constant', 'linear', 'gaussian') - influence function of the kernel points
:param aggregation_mode: string in ('closest', 'sum') - behavior of the convolution
:param modulated: bool - If deformable conv should be modulated
:return: output_features float32[n_points, out_fdim]
"""
############
# Parameters
############
# Radius of the initial positions of the kernel points
K_radius = 1.5 * KP_extent
# Number of kernel points
num_kpoints = int(K_values.shape[0])
# Check point dimension (currently only 3D is supported)
points_dim = int(query_points.shape[1])
#################################
# Initiate kernel point positions
#################################
# Create one kernel disposition (as numpy array). Choose the KP distance to center thanks to the KP extent
K_points_numpy = create_kernel_points(K_radius,
num_kpoints,
num_kernels=1,
dimension=points_dim,
fixed=fixed)
K_points_numpy = K_points_numpy.reshape((num_kpoints, points_dim))
# Create the tensorflow variable
K_points = tf.Variable(K_points_numpy.astype(np.float32),
name='kernel_points',
trainable=False,
dtype=tf.float32)
#############################
# Standard KPConv for offsets
#############################
# Create independant weight for the first convolution and a bias term as no batch normalization happen
if modulated:
offset_dim = (points_dim + 1) * num_kpoints
else:
offset_dim = points_dim * num_kpoints
shape0 = K_values.shape.as_list()
shape0[-1] = offset_dim
K_values0 = tf.Variable(tf.zeros(shape0, dtype=tf.float32),
name='offset_conv_weights')
b0 = tf.Variable(tf.zeros(offset_dim, dtype=tf.float32),
name='offset_conv_bias')
# Get features from standard convolution
features0 = KPConv_ops(query_points, support_points, neighbors_indices,
features, K_points, K_values0, KP_extent,
KP_influence, aggregation_mode) + b0
if modulated:
# Get offset (in normalized scale) from features
offsets = features0[:, :points_dim * num_kpoints]
offsets = tf.reshape(offsets, [-1, num_kpoints, points_dim])
# Get modulations
modulations = 2 * tf.sigmoid(features0[:, points_dim * num_kpoints:])
else:
# Get offset (in normalized scale) from features
offsets = tf.reshape(features0, [-1, num_kpoints, points_dim])
# No modulations
modulations = None
# Rescale offset for this layer
offsets *= KP_extent
###############################
# Build deformable KPConv graph
###############################
# Apply deformed convolution
return KPConv_deform_ops(query_points, support_points, neighbors_indices,
features, K_points, offsets, modulations,
K_values, KP_extent, KP_influence,
aggregation_mode)
def KPConv_deformable(query_points,
support_points,
neighbors_indices,
features,
K_values,
fixed='center',
KP_extent=1.0,
KP_influence='linear',
aggregation_mode='sum',
modulated=False):
"""
This function initiates the kernel point disposition before building deformable KPConv graph ops
:param query_points: float32[n_points, dim] - input query points (center of neighborhoods)
:param support_points: float32[n0_points, dim] - input support points (from which neighbors are taken)
:param neighbors_indices: int32[n_points, n_neighbors] - indices of neighbors of each point
:param features: float32[n_points, in_fdim] - input features
:param K_values: float32[n_kpoints, in_fdim, out_fdim] - weights of the kernel
:param fixed: string in ('none', 'center' or 'verticals') - fix position of certain kernel points
:param KP_extent: float32 - influence radius of each kernel point
:param KP_influence: string in ('constant', 'linear', 'gaussian') - influence function of the kernel points
:param aggregation_mode: string in ('closest', 'sum') - behavior of the convolution
:param modulated: bool - If deformable conv should be modulated
:return: output_features float32[n_points, out_fdim]
"""
############
# Parameters
############
#####Change####
indices = pf.knn_indices_general(qrs, pts, K, True)
nn_pts = tf.gather_nd(pts, indices, name=tag + 'nn_pts') # (N, P, K, 3)
nn_pts_center = tf.expand_dims(qrs, axis=2,
name=tag + 'nn_pts_center') # (N, P, 1, 3)
nn_pts_local = tf.subtract(nn_pts,
nn_pts_center,
name=tag + 'nn_pts_local') # (N, P, K, 3)
[N, P, K, dim] = nn_pts_local.shape # (N, P, K, 3)
nn_fts_local = None
C_pts_fts = 64
if with_local:
nn_fts_local = dense(nn_pts_local,
C_pts_fts // 2,
is_training,
tag + 'nn_fts_from_pts_0',
bn_decay=bn_decay)
nn_fts_local = dense(nn_fts_local,
C_pts_fts,
is_training,
tag + 'nn_fts_from_pts',
bn_decay=bn_decay)
else:
nn_fts_local = nn_pts_local
if fts_prev is not None:
fts_prev = tf.gather_nd(fts_prev, indices,
name=tag + 'fts_prev') # (N, P, K, 3)
pts_X_0 = tf.concat([nn_fts_local, fts_prev], axis=-1)
else:
pts_X_0 = nn_fts_local
s = int(K.value / D) # no. of divisions
feat_max = tf.layers.max_pooling2d(pts_X_0, [1, s],
strides=[1, s],
padding='valid',
name=tag + 'maxpool_0')
fts_X = conv2d(feat_max,
C,
name=tag + 'conv',
is_training=is_training,
kernel_size=[1, feat_max.shape[-2].value])
fts_X = tf.squeeze(fts_X, axis=-2)
###Change####
# Radius of the initial positions of the kernel points
K_radius = 1.5 * KP_extent
# Number of kernel points
num_kpoints = int(K_values.shape[0])
# Check point dimension (currently only 3D is supported)
points_dim = int(query_points.shape[1])
#################################
# Initiate kernel point positions
#################################
# Create one kernel disposition (as numpy array). Choose the KP distance to center thanks to the KP extent
K_points_numpy = create_kernel_points(K_radius,
num_kpoints,
num_kernels=1,
dimension=points_dim,
fixed=fixed)
K_points_numpy = K_points_numpy.reshape((num_kpoints, points_dim))
# Create the tensorflow variable
K_points = tf.Variable(K_points_numpy.astype(np.float32),
name='kernel_points',
trainable=False,
dtype=tf.float32)
#############################
# Standard KPConv for offsets
#############################
# Create independant weight for the first convolution and a bias term as no batch normalization happen
if modulated:
offset_dim = (points_dim + 1) * num_kpoints
else:
offset_dim = points_dim * num_kpoints
shape0 = K_values.shape.as_list()
shape0[-1] = offset_dim
K_values0 = tf.Variable(tf.zeros(shape0, dtype=tf.float32),
name='offset_conv_weights')
b0 = tf.Variable(tf.zeros(offset_dim, dtype=tf.float32),
name='offset_conv_bias')
# Get features from standard convolution
features0 = KPConv_ops(query_points, support_points, neighbors_indices,
features, K_points, K_values0, KP_extent,
KP_influence, aggregation_mode) + b0
if modulated:
# Get offset (in normalized scale) from features
offsets = features0[:, :points_dim * num_kpoints]
offsets = tf.reshape(offsets, [-1, num_kpoints, points_dim])
# Get modulations
modulations = 2 * tf.sigmoid(features0[:, points_dim * num_kpoints:])
else:
# Get offset (in normalized scale) from features
offsets = tf.reshape(features0, [-1, num_kpoints, points_dim])
# No modulations
modulations = None
# Rescale offset for this layer
offsets *= KP_extent
###############################
# Build deformable KPConv graph
###############################
# Apply deformed convolution
return KPConv_deform_ops(query_points, support_points, neighbors_indices,
features, K_points, offsets, modulations,
K_values, KP_extent, KP_influence,
aggregation_mode)