def ADGM(features, is_training, fc_layers=[8, 8], scope=''): # tadpole: [32,16]
net = features
print(net.shape)
for i, fc in enumerate(fc_layers):
net = tf_util.conv2d(net, fc, [1, 1], padding='VALID', stride=[1, 1],
bn=False, is_training=is_training, scope='adgm%d' % i, is_dist=True)
# print(net.shape)
# net = tf.nn.dropout(net,keep_prob=1-0.2*tf.cast(is_training,'float'))
# part 1: Graph representation feature learning
X_hat = net
# Probabilistic graph generator
A = -tf_util.pairwise_distance(X_hat)
print('here:', 1 - tf.eye(A.get_shape().as_list()[-1]))
temp = (1 + tf_util._variable_with_weight_decay(scope + 'temp', [1], 1e-1, None, use_xavier=False))
# th = (5 + tf_util._variable_with_weight_decay(scope+'th', [X_hat.shape[1],1], 1e-6, None, use_xavier=False))
th = (5 + tf_util._variable_with_weight_decay(scope + 'th', [1, 1], 1e-1, None, use_xavier=False))
print('temp:',temp)
print('theta:',th)
A = tf.nn.sigmoid(temp * A + th)
A = A * (1 - tf.eye(A.get_shape().as_list()[-1])) + tf.eye(A.get_shape().as_list()[-1])
A = A / tf.reduce_sum(A, -1)[..., None]
return A, th
def __init__(self,
input_channels,
block_elemnts,
out_channels,
scope,
is_training,
use_xavier=True,
l2_regularizer=1.0e-3):
self.input_channels = input_channels
self.block_elemnts = block_elemnts
self.out_channels = out_channels
self.scope = scope
self.use_xavier = use_xavier
self.is_training = is_training
self.weight_decay = 0.0
self.l2_regularizer = l2_regularizer
with tf.variable_scope(self.scope) as sc:
self.k_tensor = tf_util._variable_with_weight_decay(
'weights',
shape=[
self.out_channels, self.input_channels,
self.block_elemnts.num_of_translations
],
use_xavier=self.use_xavier,
stddev=0.1,
wd=self.weight_decay)
def sample_without_replacement(P, K, is_training, scope=''):
temp = (1 + tf_util._variable_with_weight_decay(scope + '_temp', [1], 1e-6, None, use_xavier=False))
P = P * (tf.abs(temp) + 1e-8)
P = tf.identity(P, name=scope + '_P')
b, n, f = P.get_shape().as_list()
# apply gumble top-k trick
q = tf.random_uniform(tf.shape(P), 0, 1) + 1e-15 # q=u
Pq = (P - tf.log(-tf.log(q)))
_, indices = tf.nn.top_k(Pq, K)
# Rearranging the indices
cols = tf.reshape(tf.tile(tf.tile(tf.range(K)[None, :], (n, 1))[None, :], (b, 1, 1)),
[-1]) # fetch the prob for the sampled edges
rows = tf.tile(tf.tile(tf.range(n)[:, None], (1, K))[None, :], (b, 1, 1))
rows = tf.reshape((rows + n * np.arange(b)[:, None, None]), [-1]) # .flatten()
ndindices = tf.concat((rows[:, None], tf.reshape(indices, (-1, 1))), -1)
samp_logprobs = tf.reshape(tf.gather_nd(tf.reshape(P, (-1, f)), ndindices),
(b, n, -1)) # samp_logprobs = log(p(e_ij))
return indices, tf.exp(samp_logprobs)
def ADGM(features, edges, k, conv_func, is_training, scope=''):
X_hat = conv_func(features, edges)
# Probabilistic graph generator
A = -tf_util.pairwise_distance(X_hat)
# if temp is None:
temp = (1 + tf_util._variable_with_weight_decay(scope + 'temp', [1], 1e-6, None, use_xavier=False))
# print('TEMP: ', temp)
f = features.get_shape().as_list()[-1]
th = (1 + tf_util._variable_with_weight_decay(scope + 'th', [1, ], 1e-6, None, use_xavier=False))
A = tf.nn.sigmoid(temp * (A + tf.abs(th)))
# A = A * (1 - tf.eye(A.get_shape().as_list()[-1]))
A = A / tf.reduce_sum(A, -1)[..., None]
return X_hat, A, A
def get_model(point_cloud, cls_label, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = tf.slice(point_cloud, [0, 0, 0], [-1, -1, 3])
l0_points = tf.slice(point_cloud, [0, 0, 3], [-1, -1, 3])
farthest_distance = 0.15
num_neighbors = 4
#######Contextual representation
new_xyz = l0_xyz # (batch_size, npoint, 3)
idx, pts_cnt = query_ball_point(farthest_distance, num_neighbors, l0_xyz,
new_xyz)
neighbor_xyz = group_point(l0_xyz, idx)
neighbor_xyz -= tf.tile(tf.expand_dims(new_xyz, 2),
[1, 1, num_neighbors, 1])
neighbor_points = group_point(l0_points, idx)
neighbor_representation = tf.concat([neighbor_xyz, neighbor_points],
axis=-1)
neighbor_representation = tf.reshape(neighbor_representation,
(batch_size, num_point, -1))
num_channel = neighbor_representation.get_shape()[2].value
points = tf_util.conv1d(point_cloud,
num_channel,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='points_fc',
bn_decay=bn_decay)
neighbor_representation_gp = gating_process(
neighbor_representation,
num_channel,
padding='VALID',
is_training=is_training,
scope='neighbor_representation_gp',
bn_decay=bn_decay)
points_gp = gating_process(points,
num_channel,
padding='VALID',
is_training=is_training,
scope='points_gp',
bn_decay=bn_decay)
l0_points_CR = tf.concat([
neighbor_representation_gp * points,
points_gp * neighbor_representation
],
axis=-1)
l0_points = l0_points_CR
########## Positional Representation
#num_channel=K
idx, pts_cnt = query_ball_point(0.2, 16, l0_xyz, l0_xyz)
neighbor_xyz = group_point(l0_xyz, idx)
# neighbor_xyz = self.gather_neighbour(xyz, neigh_idx)
xyz_tile = tf.tile(tf.expand_dims(l0_xyz, axis=2),
[1, 1, tf.shape(idx)[-1], 1])
relative_xyz = xyz_tile - neighbor_xyz
#relative_xyz =neighbor_xyz
relative_dis = tf.reduce_sum(tf.square(relative_xyz),
axis=-1,
keepdims=True)
encoded_position = tf.concat(
[relative_dis, relative_xyz, xyz_tile, neighbor_xyz], axis=-1)
encoded_position = tf_util.conv2d(encoded_position,
16, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv011',
bn_decay=bn_decay)
encoded_neighbours = group_point(l0_points, idx)
positional_representation = tf.concat(
[encoded_neighbours, encoded_position], axis=-1)
positional_representation = tf.reduce_mean(positional_representation,
axis=[2],
keep_dims=True,
name='avgpool')
points = tf_util.conv2d(tf.concat(
[positional_representation,
tf.expand_dims(l0_points_CR, 2)], axis=-1),
num_channel * 2, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='attp',
bn_decay=bn_decay)
points = tf.squeeze(points, [2])
end_points['points'] = points
# Set abstraction layers
l1_xyz, l1_points = pointnet_sa_module_msg(
l0_xyz,
l0_points,
512, [0.1, 0.2, 0.4], [32, 64, 128],
[[32, 32, 64], [64, 64, 128], [64, 96, 128]],
is_training,
bn_decay,
scope='layer1')
l2_xyz, l2_points = pointnet_sa_module_msg(
l1_xyz,
l1_points,
128, [0.4, 0.8], [64, 128], [[128, 128, 256], [128, 196, 256]],
is_training,
bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=None,
radius=None,
nsample=None,
mlp=[256, 512, 1024],
mlp2=None,
group_all=True,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
# Feature propagation layers
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer2')
cls_label_one_hot = tf.one_hot(cls_label,
depth=NUM_CATEGORIES,
on_value=1.0,
off_value=0.0)
cls_label_one_hot = tf.reshape(cls_label_one_hot,
[batch_size, 1, NUM_CATEGORIES])
cls_label_one_hot = tf.tile(cls_label_one_hot, [1, num_point, 1])
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
tf.concat(
[cls_label_one_hot, l0_xyz, l0_points],
axis=-1),
l1_points, [128, 128],
is_training,
bn_decay,
scope='fp_layer3')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
########## Channel-wise Attention
input = net
output_a = tf_util.conv2d(tf.expand_dims(input, 1),
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv_output_a',
bn_decay=bn_decay)
output_b = tf_util.conv2d(tf.expand_dims(input, 1),
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv_output_b',
bn_decay=bn_decay)
output_b = tf.transpose(output_b, [0, 1, 3, 2])
output_a = tf.squeeze(output_a, [1])
output_b = tf.squeeze(output_b, [1])
energy = tf.matmul(output_b, output_a)
D = tf.reduce_max(energy, -1)
D = tf.expand_dims(D, -1)
energy_new = tf.tile(D, multiples=[1, 1, energy.shape[2]]) - energy
attention = tf.nn.softmax(energy_new, axis=-1)
output_d = tf_util.conv2d(tf.expand_dims(input, 1),
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv_output_d',
bn_decay=bn_decay)
output_d = tf.squeeze(output_d, [1])
output_CA = tf.matmul(output_d, attention)
gamma2 = tf_util._variable_with_weight_decay('weightsgamma2m',
shape=[1],
use_xavier=True,
stddev=1e-3,
wd=0.0)
output_CA = output_CA * gamma2 + input
########## Squeeze-and-Excitation
ex1 = tf.reduce_mean(input, axis=[1], keep_dims=True, name='avgpool1')
print(ex1.get_shape())
ex1 = tf_util.conv1d(ex1,
64,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='ex1',
bn_decay=bn_decay)
print(ex1.get_shape())
ex1 = tf_util.conv1d(ex1,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='ex2',
bn_decay=bn_decay)
print(ex1.get_shape())
output2 = input * ex1
output = tf.concat([output_CA, output2], axis=-1)
end_points['feats'] = output
net = tf_util.dropout(net,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
50,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def group_conv_cayley(inputs,
scope,
kernel_size=3,
stride=[1, 1, 1],
padding='VALID',
stddev=1e-3,
use_xavier=True,
weight_decay=0.0,
activation_fn=tf.nn.relu,
bn=True,
bn_decay=None,
is_training=None):
'''
group convolution defined on symmetry group p4m
Args:
inputs : (Batch_size, D, W, H, l)
num_input_channels : l ( i.e. the length of latent vector in RBF-VAE module)
num_output_channels : 1
kernel_size : odd number
'''
num_in_channels = inputs.get_shape()[-1].value
weight_shape = (kernel_size, kernel_size, kernel_size, num_in_channels, 1)
weight = tf_util._variable_with_weight_decay(name=scope + '/gconv_weights',
shape=weight_shape,
stddev=stddev,
wd=weight_decay,
use_xavier=use_xavier)
weight_list = tf.split(weight, num_in_channels, axis=-2)
weight_list = [tf.squeeze(ww) for ww in weight_list]
# weight_list : list of (ksize, ksize, ksize) tensor
permute_order = permutation.VVInt()
permute_in = permutation.VInt([cnt for cnt in range(kernel_size)])
permutation.permute(permute_in, kernel_size, permute_order)
permute_order = list(list(pp) for pp in permute_order)
gconv_kernel = []
for pp in permute_order:
gconv_kernel.append([
cayley(tf.transpose(weight_list[cnt], pp))
for cnt in range(num_in_channels)
])
# gconv_kernel -> list [6][num_in_channels][(ksize, ksize, ksize, #cayley diagram)]
gconv_kernel = [
tf.stack(gconv_kernel[cnt], axis=-2)
for cnt in range(len(permute_order))
]
# gconv_kernel -> list [6][(ksize, ksize, ksize,num_in_channels, #cayley diagram)]
gconv_kernel = tf.concat(gconv_kernel, axis=-1)
# gconv_kernel -> tensor of shape (ksize, ksize, num_in_channels, 2 * #cayley diagram)
bias = tf.get_variable(name=scope + "/bias",
shape=[1],
initializer=tf.constant_initializer(0.0))
bias_share = tf.reshape(tf.stack([bias for cnt in range(2 * 8)]), [-1])
output = tf.nn.conv3d(input=inputs,
filter=gconv_kernel,
strides=(1, stride[0], stride[1], stride[2], 1),
padding=padding)
if bn:
output = tf_util.batch_norm_for_conv3d(output,
is_training=is_training,
bn_decay=bn_decay,
scope="bn")
output = tf.nn.bias_add(output, bias_share)
if activation_fn is not None:
output = activation_fn(output)
return output
def get_model(point_cloud,
is_training,
num_neighbors,
farthest_distance,
bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx9, output Bxnum_class """
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value
end_points = {}
l0_xyz = point_cloud[:, :, 0:3]
l0_points = point_cloud[:, :, 3:9]
"""Point Enrichment """
new_xyz = l0_xyz # (batch_size, npoint, 3)
idx, pts_cnt = query_ball_point(farthest_distance, num_neighbors, l0_xyz,
new_xyz)
neighbor_xyz = group_point(l0_xyz, idx)
neighbor_xyz -= tf.tile(tf.expand_dims(new_xyz, 2),
[1, 1, num_neighbors, 1])
neighbor_points = group_point(l0_points, idx)
neighbor_representation = tf.concat([neighbor_xyz, neighbor_points],
axis=-1)
neighbor_representation = tf.reshape(neighbor_representation,
(batch_size, num_point, -1))
num_channel = neighbor_representation.get_shape()[2].value
points = tf_util.conv1d(point_cloud,
num_channel,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='points_fc',
bn_decay=bn_decay)
neighbor_representation_gp = gating_process(
neighbor_representation,
num_channel,
padding='VALID',
is_training=is_training,
scope='neighbor_representation_gp',
bn_decay=bn_decay)
points_gp = gating_process(points,
num_channel,
padding='VALID',
is_training=is_training,
scope='points_gp',
bn_decay=bn_decay)
l0_points = tf.concat([
neighbor_representation_gp * points,
points_gp * neighbor_representation
],
axis=-1)
# Layer 1
l1_xyz, l1_points, l1_indices = pointnet_sa_module_withgab(
l0_xyz,
l0_points,
npoint=1024,
radius=0.1,
nsample=32,
mlp=[32, 32, 64],
mlp2=[64, 64],
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1',
gab=True)
l2_xyz, l2_points, l2_indices = pointnet_sa_module_withgab(
l1_xyz,
l1_points,
npoint=256,
radius=0.2,
nsample=32,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module_withgab(
l2_xyz,
l2_points,
npoint=64,
radius=0.4,
nsample=32,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module_withgab(
l3_xyz,
l3_points,
npoint=16,
radius=0.8,
nsample=32,
mlp=[256, 256, 512],
mlp2=None,
group_all=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [128, 128, 128],
is_training,
bn_decay,
scope='fa_layer4')
# FC layers
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
"""Spatial-wise Attention"""
input = net
output_a = tf_util.conv2d(tf.expand_dims(input, 1),
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv_output_a',
bn_decay=bn_decay)
output_b = tf_util.conv2d(tf.expand_dims(input, 1),
64, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv_output_b',
bn_decay=bn_decay)
output_b = tf.transpose(output_b, [0, 1, 3, 2])
output_a = tf.squeeze(output_a)
output_b = tf.squeeze(output_b)
energy = tf.matmul(output_a, output_b)
attention = tf.nn.softmax(energy, axis=-1)
output_d = tf_util.conv2d(tf.expand_dims(input, 1),
128, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv_output_d',
bn_decay=bn_decay)
output_d = tf.squeeze(output_d)
gamma = tf_util._variable_with_weight_decay('weight_patial',
shape=[1],
use_xavier=True,
stddev=1e-3,
wd=0.0)
output_SA = tf.matmul(attention, output_d)
output_SA = output_SA * gamma + tf.squeeze(input)
"""Channel-wise Attention"""
output_f = tf.transpose(input, [0, 2, 1])
energy = tf.matmul(output_f, input)
D = tf.reduce_max(energy, -1)
D = tf.expand_dims(D, -1)
energy_new = tf.tile(D, multiples=[1, 1, energy.shape[2]]) - energy
attention = tf.nn.softmax(energy_new, axis=-1)
output_CA = tf.matmul(input, attention)
gamma2 = tf_util._variable_with_weight_decay('weightsgamma2m',
shape=[1],
use_xavier=True,
stddev=1e-3,
wd=0.0)
output_CA = output_CA * gamma2 + input
output = output_SA + output_CA
end_points['feats'] = output
net = tf_util.dropout(output,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
13,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
def edge_unit_with_ec(point_cloud,
mask,
pooling,
neighbornum,
outchannel,
scope,
bn=False,
activation_fn=tf.nn.relu,
bn_decay=None,
is_training=None):
"""
:param point_cloud: B N C*K
:param mask: tensor
:param pooling: String
:return: Variable tensor
"""
batch_size = point_cloud.get_shape()[0]
point_num = point_cloud.get_shape()[1]
coordinate_length = neighbornum # adj points number * 4, will change
#input_image = tf.expand_dims(point_cloud, -1) # B N 1 C*K
mask = tf.expand_dims(mask, -1)
ec = econ.create_ec(point_cloud, mask)
ec_length = ec.get_shape()[3].value
ec = tf.reshape(ec, [batch_size, point_num, -1])
ec = tf.expand_dims(ec, axis=3)
#ww = point_cloud.get_shape()[2].value / coordinate_length
kernel_1 = tf_util._variable_with_weight_decay(
name='weights_1',
shape=[1, ec_length, 1, outchannel],
use_xavier=True,
stddev=0.001,
wd=0.1) # kernel_h, kernel_w, num_in_channels, output
biases_1 = tf_util._variable_on_cpu('biases_1', [outchannel],
tf.constant_initializer(0.0))
outputs = tf.nn.conv2d(
ec,
kernel_1,
[1, 1, ec_length, 1], # [1, stride_h, stride_w, 1]
padding='VALID') # 4 -> 1
outputs = tf.nn.bias_add(outputs, biases_1)
if bn:
outputs = tf_util.batch_norm_for_conv2d(outputs,
is_training,
bn_decay=bn_decay,
scope='bn')
if activation_fn is not None:
outputs = activation_fn(outputs)
outputs = outputs * mask
#for i in range(100000):
# print(tf.shape(outputs))
max_index = tf.squeeze(tf.argmax(tf.squeeze(outputs, -1)), -1)
if pooling == 'max':
outputs = tf.nn.max_pool(outputs,
ksize=[1, 1, coordinate_length, 1],
strides=[1, 1, 1, 1],
padding='VALID')
elif pooling == 'avg':
outputs = tf.nn.avg_pool(outputs,
ksize=[1, 1, coordinate_length, 1],
strides=[1, 1, 1, 1],
padding='VALID')
return outputs, kernel_1
def edge_unit_without_pooling(data,
mask,
pooling,
neighbornum,
outchannel,
scope,
bn=False,
activation_fn=tf.nn.relu,
bn_decay=None,
is_training=None):
"""
:param point_cloud: B N C*K
:param mask: tensor
:param pooling: String
:return: Variable tensor
"""
batch_size = data.get_shape()[0]
point_num = data.get_shape()[1]
mask = tf.expand_dims(mask, -1)
ww = data.get_shape()[2].value / neighbornum
data = tf.reshape(data, [batch_size, point_num, -1])
data = tf.expand_dims(data, -1)
kernel_1 = tf_util._variable_with_weight_decay(
name='weights_1',
shape=[1, ww, 1, outchannel],
use_xavier=True,
stddev=0.001,
wd=0.1) # kernel_h, kernel_w, num_in_channels, output
biases_1 = tf_util._variable_on_cpu('biases_1', [outchannel],
tf.constant_initializer(0.0))
outputs = tf.nn.conv2d(
data,
kernel_1,
[1, 1, ww, 1], # [1, stride_h, stride_w, 1]
padding='VALID') # 4 -> 1
outputs = tf.nn.bias_add(outputs, biases_1)
if bn:
outputs = tf_util.batch_norm_for_conv2d(
outputs, is_training, bn_decay=bn_decay,
scope='bn') # none values not supported
if activation_fn is not None:
outputs = activation_fn(outputs)
outputs = outputs * mask
#
#
# outputs = tf_util.conv2d(outputs, 32, [1, 1],
# padding='VALID', stride=[1, 1],
# bn=True, is_training=is_training,
# scope='ec_conv2', bn_decay=bn_decay)
# outputs = tf.reshape(outputs, [batch_size, -1])
# outputs = tf_util.fully_connected(outputs, point_num*neighbornum*16, bn=True, is_training=is_training,
# scope='tfc1', bn_decay=bn_decay)
# outputs = tf_util.fully_connected(outputs, point_num*neighbornum*8, bn=True, is_training=is_training,
# scope='tfc2', bn_decay=bn_decay)
# outputs = tf_util.fully_connected(outputs, point_num*neighbornum*7, bn=True, is_training=is_training,
# scope='tfc3', bn_decay=bn_decay)
tmp = tf.zeros([1, neighbornum * 7])
for i in range(batch_size):
#for j in range(batch_size):
for j in range(point_num):
edges = outputs[i, j, :]
edges = tf.reshape(edges, [1, -1])
with tf.variable_scope('ec_weights_%d_%d' % (i, j)) as sc:
edges = tf_util.fully_connected(edges,
neighbornum * 32,
bn=True,
is_training=is_training,
scope='tfc1',
bn_decay=bn_decay)
rst = tf_util.fully_connected(edges,
neighbornum * 7,
bn=True,
is_training=is_training,
scope='tfc2',
bn_decay=bn_decay)
tmp = tf.concat([tmp, rst], axis=0)
a = 1
outputs = tmp[1:, ]
outputs = tf.reshape(outputs, [batch_size, point_num, neighbornum, -1])
return outputs
def edge_unit(point_cloud,
mask,
pooling,
neighbornum,
outchannel,
scope,
bn=False,
activation_fn=tf.nn.relu,
bn_decay=None,
is_training=None):
"""
:param point_cloud: tensor
:param mask: tensor
:param pooling: String
:return: Variable tensor
"""
coordinate_length = neighbornum # adj points number * 4, will change
input_image = tf.expand_dims(point_cloud, -1) # B N C*K 1
#masked_result = mask
mask = tf.expand_dims(mask, 0) # 1 32 512 16
mask = tf.tile(mask, [outchannel, 1, 1, 1])
#print(mask.shape)
#print(mask.shape)
ww = point_cloud.get_shape()[2].value / coordinate_length
kernel_1 = tf_util._variable_with_weight_decay(
name='weights_1',
shape=[1, ww, 1, outchannel],
use_xavier=True,
stddev=0.001,
wd=0.1) # kernel_h, kernel_w, num_in_channels, output
biases_1 = tf_util._variable_on_cpu('biases_1', [outchannel],
tf.constant_initializer(0.0))
outputs = tf.nn.conv2d(
input_image,
kernel_1,
[1, 1, ww, 1], # [1, stride_h, stride_w, 1]
padding='VALID') # 4 -> 1
outputs = tf.nn.bias_add(outputs, biases_1)
if bn:
outputs = tf_util.batch_norm_for_conv2d(outputs,
is_training,
bn_decay=bn_decay,
scope='bn')
if activation_fn is not None:
outputs = activation_fn(outputs)
#masked_result = mask
outputs = tf.transpose(outputs, [3, 0, 1, 2]) # 32 32 512 16
outputs = tf.multiply(outputs, mask) # 32 32 512 16
#print(outputs.shape)
#max_index = tf.argmax(outputs,)
#print(tf.shape(outputs))
outputs = tf.transpose(outputs, [1, 2, 3, 0]) # 32 512 16 32
masked_result = outputs
print(masked_result.shape)
#print(masked_result.shape)
max_index_local = tf.squeeze(tf.argmax(outputs, 2))
if pooling == 'max':
outputs = tf.nn.max_pool(outputs,
ksize=[1, 1, coordinate_length, 1],
strides=[1, 1, 1, 1],
padding='VALID')
elif pooling == 'avg':
outputs = tf.nn.avg_pool(outputs,
ksize=[1, 1, coordinate_length, 1],
strides=[1, 1, 1, 1],
padding='VALID')
return outputs, kernel_1, max_index_local, masked_result
def get_model(point_cloud, num_frames, is_training, bn_decay=None):
""" Semantic segmentation PointNet, input is BxNx3, output Bxnum_class """
end_points = {}
batch_size = point_cloud.get_shape()[0].value
num_point = point_cloud.get_shape()[1].value // num_frames
l0_xyz = point_cloud[:, :, 0:3]
#l0_time = tf.concat([tf.ones([batch_size, num_point, 1]) * i for i in range(num_frames)], \
# axis=-2)
#l0_points = tf.concat([point_cloud[:, :, 3:], l0_time], axis=-1)
l0_points = point_cloud[:, :, 3:]
#######Contextual representation
farthest_distance = 0.6
num_neighbors = 4
new_xyz = l0_xyz # (batch_size, npoint, 3)
idx, pts_cnt = query_ball_point(farthest_distance, num_neighbors, l0_xyz,
new_xyz)
neighbor_xyz = group_point(l0_xyz, idx)
neighbor_xyz -= tf.tile(tf.expand_dims(new_xyz, 2),
[1, 1, num_neighbors, 1])
neighbor_points = group_point(l0_points, idx)
neighbor_representation = tf.concat([neighbor_xyz, neighbor_points],
axis=-1)
neighbor_representation = tf.reshape(
neighbor_representation,
(batch_size, point_cloud.get_shape()[1].value, -1))
num_channel = neighbor_representation.get_shape()[2].value
points = tf_util.conv1d(point_cloud,
num_channel,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='points_fc',
bn_decay=bn_decay)
neighbor_representation_gp = gating_process(
neighbor_representation,
num_channel,
padding='VALID',
is_training=is_training,
scope='neighbor_representation_gp',
bn_decay=bn_decay)
points_gp = gating_process(points,
num_channel,
padding='VALID',
is_training=is_training,
scope='points_gp',
bn_decay=bn_decay)
l0_points_CR = tf.concat([
neighbor_representation_gp * points,
points_gp * neighbor_representation
],
axis=-1)
########## Positional Representation
idx, pts_cnt = query_ball_point(0.6, 32, l0_xyz, l0_xyz)
neighbor_xyz = group_point(l0_xyz, idx)
# neighbor_xyz = self.gather_neighbour(xyz, neigh_idx)
xyz_tile = tf.tile(tf.expand_dims(l0_xyz, axis=2),
[1, 1, tf.shape(idx)[-1], 1])
relative_xyz = xyz_tile - neighbor_xyz
#relative_xyz =neighbor_xyz
relative_dis = tf.reduce_sum(tf.square(relative_xyz),
axis=-1,
keepdims=True)
encoded_position = tf.concat(
[relative_dis, relative_xyz, xyz_tile, neighbor_xyz], axis=-1)
encoded_position = tf_util.conv2d(encoded_position,
num_channel * 2, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='conv011',
bn_decay=bn_decay)
positional_representation = tf.reduce_mean(encoded_position,
axis=[2],
keep_dims=True,
name='avgpool')
l0_points = tf_util.conv2d(tf.concat(
[positional_representation,
tf.expand_dims(l0_points_CR, 2)], axis=-1),
num_channel * 2, [1, 1],
padding='VALID',
stride=[1, 1],
bn=True,
is_training=is_training,
scope='attp',
bn_decay=bn_decay)
l0_points = tf.squeeze(l0_points, [2])
l1_xyz, l1_points, l1_indices = pointnet_sa_module_withgab(
l0_xyz,
l0_points,
npoint=2048,
radius=1.0,
nsample=32,
mlp=[32, 32, 64],
mlp2=None,
group_all=False,
knn=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer1',
gab=True)
l2_xyz, l2_points, l2_indices = pointnet_sa_module(l1_xyz,
l1_points,
npoint=512,
radius=2.0,
nsample=32,
mlp=[64, 64, 128],
mlp2=None,
group_all=False,
knn=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer2')
l3_xyz, l3_points, l3_indices = pointnet_sa_module(l2_xyz,
l2_points,
npoint=128,
radius=4.0,
nsample=32,
mlp=[128, 128, 256],
mlp2=None,
group_all=False,
knn=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer3')
l4_xyz, l4_points, l4_indices = pointnet_sa_module(l3_xyz,
l3_points,
npoint=64,
radius=8.0,
nsample=32,
mlp=[256, 256, 512],
mlp2=None,
group_all=False,
knn=False,
is_training=is_training,
bn_decay=bn_decay,
scope='layer4')
# Feature Propagation layers
l3_points = pointnet_fp_module(l3_xyz,
l4_xyz,
l3_points,
l4_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer1')
l2_points = pointnet_fp_module(l2_xyz,
l3_xyz,
l2_points,
l3_points, [256, 256],
is_training,
bn_decay,
scope='fa_layer2')
l1_points = pointnet_fp_module(l1_xyz,
l2_xyz,
l1_points,
l2_points, [256, 128],
is_training,
bn_decay,
scope='fa_layer3')
l0_points = pointnet_fp_module(l0_xyz,
l1_xyz,
l0_points,
l1_points, [128, 128],
is_training,
bn_decay,
scope='fa_layer4')
##### debug
net = tf_util.conv1d(l0_points,
128,
1,
padding='VALID',
bn=True,
is_training=is_training,
scope='fc1',
bn_decay=bn_decay)
########## Channel-wise Attention
input = net
output_f = tf.transpose(input, [0, 2, 1])
energy = tf.matmul(output_f, input)
D = tf.reduce_max(energy, -1)
D = tf.expand_dims(D, -1)
energy_new = tf.tile(D, multiples=[1, 1, energy.shape[2]]) - energy
attention = tf.nn.softmax(energy_new, axis=-1)
output_CA = tf.matmul(input, attention)
gamma2 = tf_util._variable_with_weight_decay('weightsgamma2m',
shape=[1],
use_xavier=True,
stddev=1e-3,
wd=0.0)
output_CA = output_CA * gamma2 + input
output_CA = tf_util.conv1d(output_CA,
2,
1,
padding='VALID',
activation_fn=None,
scope='cpm')
end_points['feats'] = output_CA
########## Squeeze-and-Excitation
ex1 = tf.reduce_mean(input, axis=[1], keep_dims=True, name='avgpool1')
print(ex1.get_shape())
ex1 = tf_util.conv1d(ex1, 64, 1, padding='VALID', scope='ex1')
print(ex1.get_shape())
ex1 = tf_util.conv1d(ex1, 128, 1, padding='VALID', scope='ex2')
print(ex1.get_shape())
output = input * ex1
net = tf_util.dropout(output,
keep_prob=0.5,
is_training=is_training,
scope='dp1')
net = tf_util.conv1d(net,
12,
1,
padding='VALID',
activation_fn=None,
scope='fc2')
return net, end_points
