def get_model(net, is_training, bn_decay=None, separately=False):
""" NVIDIA regression model, input is BxWxHx3, output Bx2"""
batch_size = net[0].get_shape()[0].value
img_net, pt_net = net[0], net[1]
for i, dim in enumerate([24, 36, 48, 64, 64]):
scope = "conv" + str(i + 1)
img_net = tf_util.conv2d(img_net, dim, [5, 5],
padding='VALID', stride=[1, 1],
bn=True, is_training=is_training,
scope=scope, bn_decay=bn_decay)
img_net = tf.reshape(img_net, [batch_size, -1])
img_net = tf_util.fully_connected(img_net, 256, bn=True,
is_training=is_training,
scope='img_fc0',
bn_decay=bn_decay)
with tf.variable_scope('pointnet'):
pt_net = pointnet.get_model(pt_net, tf.constant(True))
net = tf.reshape(tf.stack([img_net, pt_net], axis=2), [batch_size, 512])
for i, dim in enumerate([256, 128, 16]):
fc_scope = "fc" + str(i + 1)
dp_scope = "dp" + str(i + 1)
net = tf_util.fully_connected(net, dim, bn=True,
is_training=is_training,
scope=fc_scope,
bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.7,
is_training=is_training,
scope=dp_scope)
net = tf_util.fully_connected(net, 2, activation_fn=None, scope='fc5')
return net
def get_model(net,
is_training,
add_lstm=False,
bn_decay=None,
separately=False):
""" Densenet169 regression model, input is BxWxHx3, output Bx2"""
batch_size = net[0].get_shape()[0].value
img_net, pt_net = net[0], net[1]
img_net = get_densenet(299, 299)(img_net)
with tf.variable_scope('pointnet'):
pt_net = pointnet.get_model(pt_net, tf.constant(True))
net = tf.reshape(tf.stack([img_net, pt_net], axis=2), [batch_size, -1])
if not add_lstm:
for i, dim in enumerate([256, 128, 16]):
fc_scope = "fc" + str(i + 1)
dp_scope = "dp" + str(i + 1)
net = tf_util.fully_connected(net,
dim,
bn=True,
is_training=is_training,
scope=fc_scope,
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope=dp_scope)
net = tf_util.fully_connected(net, 2, activation_fn=None, scope='fc4')
else:
fc_scope = "fc1"
net = tf_util.fully_connected(net,
784,
bn=True,
is_training=is_training,
scope=fc_scope,
bn_decay=bn_decay)
net = tf_util.dropout(net,
keep_prob=0.7,
is_training=is_training,
scope="dp1")
net = cnn_lstm_block(net)
return net
def get_model(point_cloud, is_training, group_cate_num=50, m=10., bn_decay=None):
#input: point_cloud: BxNx9 (XYZ, RGB, NormalizedXYZ)
batch_size = point_cloud.get_shape()[0].value
print(point_cloud.get_shape())
F = pointnet.get_model(point_cloud, is_training, bn=True, bn_decay=bn_decay)
# Semantic prediction
Fsem = tf_util.conv2d(F, 128, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='Fsem')
ptssemseg_logits = tf_util.conv2d(Fsem, group_cate_num, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='ptssemseg_logits')
ptssemseg_logits = tf.squeeze(ptssemseg_logits, [2])
ptssemseg = tf.nn.softmax(ptssemseg_logits, name="ptssemseg")
# Similarity matrix
Fsim = tf_util.conv2d(F, 128, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='Fsim')
Fsim = tf.squeeze(Fsim, [2])
r = tf.reduce_sum(Fsim * Fsim, 2)
r = tf.reshape(r, [batch_size, -1, 1])
print(r.get_shape(),Fsim.get_shape())
D = r - 2 * tf.matmul(Fsim, tf.transpose(Fsim, perm=[0, 2, 1])) + tf.transpose(r, perm=[0, 2, 1])
# simmat_logits = tf.maximum(D, 0.)
simmat_logits = tf.maximum(m * D, 0.)
# Confidence Map
Fconf = tf_util.conv2d(F, 128, [1, 1], padding='VALID', stride=[1, 1], bn=False, is_training=is_training, scope='Fsconf')
conf_logits = tf_util.conv2d(Fconf, 1, [1, 1], padding='VALID', stride=[1, 1], activation_fn=None, scope='conf_logits')
conf_logits = tf.squeeze(conf_logits, [2])
conf = tf.nn.sigmoid(conf_logits, name="confidence")
return {'semseg': ptssemseg,
'semseg_logits': ptssemseg_logits,
'simmat': simmat_logits,
'conf': conf,
'conf_logits': conf_logits}