def pose_discriminator(point_cloud, point_cloud_gt, mode_node, batch_size,
grid_size_gt):
point_cloud = tf.reshape(point_cloud, (batch_size, -1))
point_cloud_gt = tf.reshape(point_cloud_gt, (batch_size, -1))
point_cloud_batch = tf.concat((point_cloud, point_cloud_gt), axis=0)
labels = tf.concat((tf.zeros([
batch_size,
], tf.int32), tf.ones([
batch_size,
], tf.int32)),
axis=0)
current = tf.nn.dropout(
cell1D(point_cloud_batch, 16, mode_node, SCOPE='fc1', with_act=True),
1.0)
# current = tf.nn.dropout(cell1D(current ,16, mode_node, SCOPE='fc2', with_act=True) ,1.0)
# current = tf.nn.dropout(cell1D(current ,16, mode_node, SCOPE='fc3', with_act=False),1.0)
current = cell1D(current,
2,
mode_node,
SCOPE='logits',
with_act=False,
with_bn=False)
batch = {}
# batch['logits_g'] = tf.slice(current,[0,0],[batch_size,-1])
# batch['logits_d'] = tf.slice(current,[batch_size,0],[batch_size,-1])
batch['logits'] = current
batch['labels'] = labels
return batch
def model_3(data_node, mode_node, batch_size, vox_params, in_size=44):
"""The Model definition."""
ch_in = data_node.get_shape().as_list()[-1]
with tf.variable_scope("input"):
print(data_node.get_shape())
conv1_w, conv1_b = CONV3D([2, 2, 2, ch_in, 256])
conv1 = tf.nn.conv3d(data_node,
conv1_w,
strides=[1, 1, 1, 1, 1],
padding='SAME')
current = tf.nn.bias_add(conv1, conv1_b)
print(current.get_shape())
with tf.variable_scope("residuals"):
# for layer in range(4):
current = cell3D_res(current, 2, 256, mode_node, 1, 'layer_' + str(1))
current = cell3D_res(current, 2, 256, mode_node, 1, 'layer_' + str(2))
current = cell3D_res(current, 2, 256, mode_node, 1, 'layer_' + str(3))
current = cell3D_res(current, 2, 256, mode_node, 1, 'layer_' + str(4))
current = cell3D_res(current, 2, 256, mode_node, 2, 'layer_' + str(5))
size_out = current.get_shape().as_list()[1]
with tf.variable_scope("logits"):
features = tf.nn.avg_pool3d(current, [1, 2, 2, 2, 1], [1, 2, 2, 2, 1],
padding='SAME')
features = tf.reshape(current, (batch_size, -1))
logits = cell1D(features,
40,
mode_node,
SCOPE='logits',
with_act=False,
with_bn=False)
return logits
def multiplexer(data_node, mode_node):
with tf.variable_scope("Multiplexer"):
current = cell1D(data_node,64, mode_node, SCOPE='l1', with_act=False, with_bn=False)
# current = cell1D(current,256, mode_node, SCOPE='l2', with_act=True, with_bn=False)
# current = cell1D(current,256, mode_node, SCOPE='l3', with_act=False, with_bn=False)
current = tf.tanh(current)
return current
def model_2(data_node, mode_node, batch_size, vox_params, in_size=44):
"""The Model definition."""
ch_in = data_node.get_shape().as_list()[-1]
with tf.variable_scope("input"):
print(data_node.get_shape())
conv1_w, conv1_b = CONV3D([5, 5, 5, ch_in, 128])
conv1 = tf.nn.conv3d(data_node,
conv1_w,
strides=[1, 1, 1, 1, 1],
padding='SAME')
current = tf.nn.bias_add(conv1, conv1_b)
print(current.get_shape())
with tf.variable_scope("residuals"):
num_layers = len(vox_params['filter'])
for layer in range(num_layers):
current = cell3D_res_gated(current, 3, vox_params['filter'][layer],
mode_node,
vox_params['downsample'][layer],
'layer_' + str(layer))
size_out = current.get_shape().as_list()[1]
with tf.variable_scope("logits"):
features = tf.nn.avg_pool3d(current,
[1, size_out, size_out, size_out, 1],
[1, size_out, size_out, size_out, 1],
padding='SAME')
features = tf.reshape(features, (batch_size, -1))
logits = cell1D(features,
40,
mode_node,
SCOPE='logits',
with_act=False,
with_bn=False)
return logits
def resnet(data_node, mode_node, layers):
shape = data_node.get_shape().as_list()
weights = []
ch_in = shape[-1]
data_node_wrap = tf.concat(
(data_node, tf.slice(data_node, [0, 0, 0, 0], [-1, -1, 1, -1])),
axis=2)
# data_node_wrap = tf.concat((data_node_wrap,tf.slice(data_node_wrap,[0,0,0,0],[-1,1,-1,-1])),axis=1)
with tf.variable_scope("input") as SCOPE:
conv1_w, conv1_b = CONV2D([2, 2, ch_in, 32])
c1 = tf.nn.conv2d(data_node_wrap,
conv1_w,
strides=[1, 1, 1, 1],
padding='VALID')
c1 = tf.nn.bias_add(c1, conv1_b)
with tf.variable_scope("residuals") as SCOPE:
current = cell2D_res(c1, 3, 32, 32, mode_node, 2, 'r1')
current = cell2D_res(current, 3, 32, 64, mode_node, 2, 'r2')
current = cell2D_res(current, 3, 64, 64, mode_node, 2, 'r3')
current = cell2D_res(current, 3, 64, 128, mode_node, 2, 'r4')
current = cell2D_res(current, 3, 128, 256, mode_node, 2, 'r5')
current = BatchNorm(current, mode_node, SCOPE)
current = tf.nn.avg_pool(current, [1, 7, 7, 1], [1, 1, 1, 1],
padding='SAME')
with tf.variable_scope("fully") as SCOPE:
features = tf.reshape(current, (shape[0], -1))
for ii in range(len(layers) - 1):
layer_in = layers[ii]
layer_out = layers[ii + 1]
w = tf.reshape(
cell1D(features,
layer_in * layer_out,
mode_node,
SCOPE='w' + str(ii),
with_act=False,
with_bn=False), (shape[0], layer_in, layer_out))
b = tf.reshape(
cell1D(features,
layer_out,
mode_node,
SCOPE='b' + str(ii),
with_act=False,
with_bn=False), (shape[0], 1, layer_out))
weights.append({'w': w, 'b': b})
return weights
def model_1(data_node, mode_node, batch_size, vox_params):
"""The Model definition."""
ch_in = data_node.get_shape().as_list()[-1]
etaiFlag = True
# mean, var = tf.nn.moments(data_node, axes=[0],keep_dims=True)
# data_node = (data_node-mean)/tf.sqrt(var)
# data_node = tf.stop_gradient(data_node)
with tf.variable_scope("input"):
print(data_node.get_shape())
conv1_w, conv1_b = CONV3D([5, 5, 5, ch_in, 128])
conv1 = tf.nn.conv3d(data_node,
conv1_w,
strides=[1, 1, 1, 1, 1],
padding='SAME')
current = tf.nn.bias_add(conv1, conv1_b)
if etaiFlag:
with tf.variable_scope('reg'):
reg_etai(current, 10, eps_init=vox_params['eps'])
print(current.get_shape())
with tf.variable_scope("residuals"):
num_layers = len(vox_params['filter'])
for layer in range(num_layers):
current = cell3D_res(current,
3,
vox_params['filter'][layer],
mode_node,
vox_params['downsample'][layer],
'layer_' + str(layer),
bn=False,
etaiFlag=etaiFlag,
eps_init=vox_params['eps'])
size_out = current.get_shape().as_list()[1]
with tf.variable_scope("logits"):
features = tf.nn.avg_pool3d(current,
[1, size_out, size_out, size_out, 1],
[1, size_out, size_out, size_out, 1],
padding='SAME')
features = tf.reshape(features, (batch_size, -1))
logits = cell1D(features,
40,
mode_node,
SCOPE='logits',
with_act=False,
with_bn=False)
return logits
def fc_IG(in_node, in_node2, mode_node, batch_size, grid_size, grid_size_gt):
with tf.variable_scope('fully'):
current = cell1D(in_node,
512,
mode_node,
SCOPE='fc1',
with_act=True,
with_bn=False)
features = cell1D(current,
512,
mode_node,
SCOPE='fc2',
with_act=True,
with_bn=False)
with tf.variable_scope("Kinematics"):
num_joints = 21
pose_limits_init = np.ones((1, 3, 21), dtype=np.float32)
pose_limits_init[:,
2, :] = 0.3 * pose_limits_init[:, 2, :] #Z rotations
pose_limits_init[:, 1, :] = 0.7 * pose_limits_init[:, 1, :] #
pose_limits_init[:, 0, :] = 0.2 * pose_limits_init[:, 0, :]
pose_limits_init[:, :, 0] = 1
pose_limits = tf.constant(pose_limits_init, dtype=tf.float32)
bones = 0.5 * tf.tanh(
tf.expand_dims(
cell1D(features,
21,
mode_node,
SCOPE='Bones',
with_act=False,
with_bn=False), 1)) + 1
pose = pose_limits * tf.tanh(
tf.reshape(
cell1D(features,
63,
mode_node,
SCOPE='Pose',
with_act=False,
with_bn=False), (batch_size, 3, 21)))
camera = tf.tanh(
tf.expand_dims(
cell1D(features,
3,
mode_node,
SCOPE='Camera',
with_act=False,
with_bn=False), -1))
KL = KINETIC.Kinetic(batch_size, num_joints=num_joints)
pred_cloud, _ = KL.apply_trans(bones, pose, camera)
marks = tf.transpose(pred_cloud, (0, 2, 1))
marks_norm = (marks / 2 + 0.5) * (grid_size_gt - 1)
with tf.variable_scope("xentropy"):
gt_meshgrid = np.meshgrid(
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt))
grid_x = (gt_meshgrid[1]).astype(np.float32)
grid_y = (gt_meshgrid[0]).astype(np.float32)
grid_z = (gt_meshgrid[2]).astype(np.float32)
grid_x_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='x',
initializer=grid_x,
trainable=False,
dtype='float32'), 0), -1)
grid_y_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='y',
initializer=grid_y,
trainable=False,
dtype='float32'), 0), -1)
grid_z_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='z',
initializer=grid_z,
trainable=False,
dtype='float32'), 0), -1)
pred_x = grid_x_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 0, 0], [-1, 1, -1]), 1), 1)
pred_y = grid_y_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 1, 0], [-1, 1, -1]), 1), 1)
pred_z = grid_z_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 2, 0], [-1, 1, -1]), 1), 1)
logits = -1 * (tf.pow(pred_x, 2) + tf.pow(pred_y, 2) +
tf.pow(pred_z, 2))
init_scale = np.zeros((1, 1, 1, 1, 21), dtype='float32')
init_bias = np.zeros((1, 1, 1, 1, 21), dtype='float32')
# init_scale = np.zeros((1),dtype='float32')
# init_bias = np.zeros((1),dtype='float32')
scale = tf.get_variable(name='scale',
initializer=init_scale,
trainable=True,
dtype='float32')
bias = tf.get_variable(name='bias',
initializer=init_bias,
trainable=True,
dtype='float32')
logits = logits * scale + bias
pred_cloud = tf.transpose(marks, (0, 2, 1))
return logits, pred_cloud
def fc(in_node, in_node2, mode_node, batch_size, grid_size, grid_size_gt):
with tf.variable_scope('fully'):
current = cell1D(in_node,
512,
mode_node,
SCOPE='fc1',
with_act=True,
with_bn=False)
current = cell1D(current,
256,
mode_node,
SCOPE='fc2',
with_act=True,
with_bn=False)
current = cell1D(current,
63,
mode_node,
SCOPE='logits',
with_act=False,
with_bn=False)
marks = tf.tanh(tf.reshape(current, (batch_size, 3, 21)))
marks_norm = (marks / 2 + 0.5) * (grid_size_gt - 1)
with tf.variable_scope("xentropy"):
gt_meshgrid = np.meshgrid(
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt))
grid_x = (gt_meshgrid[1]).astype(np.float32)
grid_y = (gt_meshgrid[0]).astype(np.float32)
grid_z = (gt_meshgrid[2]).astype(np.float32)
grid_x_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='x',
initializer=grid_x,
trainable=False,
dtype='float32'), 0), -1)
grid_y_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='y',
initializer=grid_y,
trainable=False,
dtype='float32'), 0), -1)
grid_z_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='z',
initializer=grid_z,
trainable=False,
dtype='float32'), 0), -1)
pred_x = grid_x_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 0, 0], [-1, 1, -1]), 1), 1)
pred_y = grid_y_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 1, 0], [-1, 1, -1]), 1), 1)
pred_z = grid_z_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 2, 0], [-1, 1, -1]), 1), 1)
logits = -1 * (tf.pow(pred_x, 2) + tf.pow(pred_y, 2) +
tf.pow(pred_z, 2))
init_scale = np.zeros((1, 1, 1, 1, 21), dtype='float32')
init_bias = np.zeros((1, 1, 1, 1, 21), dtype='float32')
# init_scale = np.zeros((1),dtype='float32')
# init_bias = np.zeros((1),dtype='float32')
scale = tf.get_variable(name='scale',
initializer=init_scale,
trainable=True,
dtype='float32')
bias = tf.get_variable(name='bias',
initializer=init_bias,
trainable=True,
dtype='float32')
logits = logits * scale + bias
pred_cloud = tf.transpose(marks, (0, 2, 1))
return logits, pred_cloud
def deep_prior4_IG(in_node, in_node2, mode_node, batch_size, grid_size,
grid_size_gt):
in_node = tf.reduce_max(in_node, axis=3, keep_dims=True)
in_node = tf.slice(in_node, [0, 0, 0, 0, 2], [-1, -1, -1, -1, 1])
in_node = tf.squeeze(in_node, axis=3)
ch_in = in_node.get_shape().as_list()[-1]
with tf.variable_scope("Input") as SCOPE:
conv1_w, conv1_b = CONV2D([5, 5, ch_in, 32])
c1 = tf.nn.conv2d(in_node,
conv1_w,
strides=[1, 1, 1, 1],
padding='SAME')
c1 = tf.nn.bias_add(c1, conv1_b)
with tf.variable_scope("Residuals") as SCOPE:
current = cell2D_res(c1, 3, 32, 64, mode_node, 2, 'r1')
current = cell2D_res(current, 3, 64, 64, mode_node, 1, 'r2')
current = cell2D_res(current, 3, 64, 128, mode_node, 2, 'r3')
current = cell2D_res(current, 3, 128, 128, mode_node, 1, 'r4')
current = cell2D_res(current, 3, 128, 256, mode_node, 2, 'r5')
current = cell2D_res(current, 3, 256, 256, mode_node, 1, 'r6')
current = cell2D_res(current, 3, 256, 512, mode_node, 2, 'r7')
current = BatchNorm(current, mode_node, SCOPE)
current = tf.nn.avg_pool(current, [1, 6, 6, 1], [1, 1, 1, 1],
padding='SAME')
with tf.variable_scope("Features") as SCOPE:
features = tf.reshape(current, (batch_size, -1))
with tf.variable_scope("Kinematics") as SCOPE:
num_joints = 21
KL = KINETIC.Kinetic(batch_size, num_joints=num_joints)
with tf.variable_scope("Bones_GT") as SCOPE:
bones_gt = bone_logits(in_node2)
bones_gt = tf.transpose(bones_gt, (0, 2, 1))
bones_gt_scale = tf.div(
bones_gt, tf.slice(KL.model['Bones'], [0, 0, 1], [-1, 1, -1]))
bones_gt_scale = tf.concat((tf.zeros(
(batch_size, 1, 1)), bones_gt_scale),
axis=2)
pose_limits_scale = 180 * np.ones((1, 3, 21), dtype=np.float32)
# X axis
pose_limits_scale[:, 0, (4, 8, 12, 16, 20)] = 5 # Tip bones
pose_limits_scale[:, 0, (3, 7, 11, 15, 19)] = 5 #
pose_limits_scale[:, 0, (2, 6, 10, 14,
18)] = 40 # Most of the rotational freedom
pose_limits_scale[:, 0, (1, 5, 9, 13, 17)] = 30 # Metacarpal
# Z axis
pose_limits_scale[:, 2, (4, 8, 12, 16, 20)] = 2 # Tip bones
pose_limits_scale[:, 2, (3, 7, 11, 15, 19)] = 2
pose_limits_scale[:, 2, (2, 6, 10, 14,
18)] = 50 # Most of the rotational freedom
pose_limits_scale[:, 2, (1, 5, 9, 13, 17)] = 30 # Metacarpal
# Y axis
pose_limits_scale[:, 1, (4, 8, 12, 16, 20)] = 80 # Tip bones
pose_limits_scale[:, 1, (3, 7, 11, 15, 19)] = 80
pose_limits_scale[:, 1, (2, 6, 10, 14,
18)] = 80 # Most of the rotational freedom
pose_limits_scale[:, 1, (1, 5, 9, 13, 17)] = 50 # Metacarpal
# Wrist
pose_limits_scale[:, :, 0] = 190
pose_limits_scale_tf = tf.constant(pose_limits_scale / 180.,
dtype=tf.float32)
pose = pose_limits_scale_tf * tf.tanh(
tf.reshape(
cell1D(features,
63,
mode_node,
SCOPE='Pose',
with_act=False,
with_bn=False), (batch_size, 3, 21)))
# bones = 1. + 0.5*tf.tanh(tf.expand_dims(cell1D(features,21, mode_node, SCOPE='Bones', with_act=False, with_bn=False),1))
camera = tf.tanh(
tf.expand_dims(
cell1D(features,
3,
mode_node,
SCOPE='Camera',
with_act=False,
with_bn=False), -1))
pred_cloud, _ = KL.apply_trans(bones_gt_scale, pose, camera)
marks = tf.transpose(pred_cloud, (0, 2, 1))
marks_norm = (marks / 2 + 0.5) * (grid_size_gt - 1)
with tf.variable_scope("xentropy") as SCOPE:
gt_meshgrid = np.meshgrid(
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt))
grid_x = (gt_meshgrid[1]).astype(np.float32)
grid_y = (gt_meshgrid[0]).astype(np.float32)
grid_z = (gt_meshgrid[2]).astype(np.float32)
grid_x_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='x',
initializer=grid_x,
trainable=False,
dtype='float32'), 0), -1)
grid_y_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='y',
initializer=grid_y,
trainable=False,
dtype='float32'), 0), -1)
grid_z_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='z',
initializer=grid_z,
trainable=False,
dtype='float32'), 0), -1)
pred_x = grid_x_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 0, 0], [-1, 1, -1]), 1), 1)
pred_y = grid_y_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 1, 0], [-1, 1, -1]), 1), 1)
pred_z = grid_z_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 2, 0], [-1, 1, -1]), 1), 1)
logits = -1 * (tf.pow(pred_x, 2) + tf.pow(pred_y, 2) +
tf.pow(pred_z, 2))
init_scale = np.zeros((1, 1, 1, 1, 21), dtype='float32')
init_bias = np.zeros((1, 1, 1, 1, 21), dtype='float32')
# init_scale = np.zeros((1),dtype='float32')
# init_bias = np.zeros((1),dtype='float32')
scale = tf.get_variable(name='scale',
initializer=init_scale,
trainable=True,
dtype='float32')
bias = tf.get_variable(name='bias',
initializer=init_bias,
trainable=True,
dtype='float32')
logits = logits * scale + bias
return logits, pred_cloud
def deep_prior4(in_node, in_node2, mode_node, batch_size, grid_size,
grid_size_gt):
in_node = tf.reduce_max(in_node, axis=3, keep_dims=True)
in_node = tf.slice(in_node, [0, 0, 0, 0, 2], [-1, -1, -1, -1, 1])
in_node = tf.squeeze(in_node, axis=3)
# in_node_max = tf.reduce_max(in_node,axis=(1,2,3),keep_dims=True)
# in_node = (tf.div(in_node,in_node_max) - 0.5)*2
ch_in = in_node.get_shape().as_list()[-1]
num_params = 63
with tf.variable_scope("input") as SCOPE:
conv1_w, conv1_b = CONV2D([5, 5, ch_in, 32])
c1 = tf.nn.conv2d(in_node,
conv1_w,
strides=[1, 1, 1, 1],
padding='SAME')
c1 = tf.nn.bias_add(c1, conv1_b)
with tf.variable_scope("residuals") as SCOPE:
current = cell2D_res(c1, 3, 32, 64, mode_node, 2, 'r1')
current = cell2D_res(current, 3, 64, 64, mode_node, 1, 'r2')
current = cell2D_res(current, 3, 64, 128, mode_node, 2, 'r3')
current = cell2D_res(current, 3, 128, 128, mode_node, 1, 'r4')
current = cell2D_res(current, 3, 128, 256, mode_node, 2, 'r5')
current = cell2D_res(current, 3, 256, 256, mode_node, 1, 'r6')
current = cell2D_res(current, 3, 256, 512, mode_node, 2, 'r7')
current = BatchNorm(current, mode_node, SCOPE)
current = tf.nn.avg_pool(current, [1, 6, 6, 1], [1, 1, 1, 1],
padding='SAME')
with tf.variable_scope("fully") as SCOPE:
features = tf.reshape(current, (batch_size, -1))
current = cell1D(features,
num_params,
mode_node,
SCOPE='logits',
with_act=False,
with_bn=False)
marks = tf.tanh(tf.reshape(current, (batch_size, 3, 21)))
pred_cloud = tf.transpose(marks, (0, 2, 1))
marks_norm = (marks / 2 + 0.5) * (grid_size_gt - 1)
with tf.variable_scope("xentropy") as SCOPE:
gt_meshgrid = np.meshgrid(
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt))
grid_x = (gt_meshgrid[1]).astype(np.float32)
grid_y = (gt_meshgrid[0]).astype(np.float32)
grid_z = (gt_meshgrid[2]).astype(np.float32)
grid_x_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='x',
initializer=grid_x,
trainable=False,
dtype='float32'), 0), -1)
grid_y_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='y',
initializer=grid_y,
trainable=False,
dtype='float32'), 0), -1)
grid_z_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='z',
initializer=grid_z,
trainable=False,
dtype='float32'), 0), -1)
pred_x = grid_x_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 0, 0], [-1, 1, -1]), 1), 1)
pred_y = grid_y_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 1, 0], [-1, 1, -1]), 1), 1)
pred_z = grid_z_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 2, 0], [-1, 1, -1]), 1), 1)
logits = -1 * (tf.pow(pred_x, 2) + tf.pow(pred_y, 2) +
tf.pow(pred_z, 2))
init_scale = np.zeros((1, 1, 1, 1, 21), dtype='float32')
init_bias = np.zeros((1, 1, 1, 1, 21), dtype='float32')
# init_scale = np.zeros((1),dtype='float32')
# init_bias = np.zeros((1),dtype='float32')
scale = tf.get_variable(name='scale',
initializer=init_scale,
trainable=True,
dtype='float32')
bias = tf.get_variable(name='bias',
initializer=init_bias,
trainable=True,
dtype='float32')
logits = logits * scale + bias
return logits, pred_cloud
def point_transformer(self, point_cloud, mode_node, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
with tf.variable_scope('point_transformer'):
point_cloud = tf.expand_dims(point_cloud, -1)
# TRANSFORM to feature space []
net = cell2D(point_cloud,
1,
3,
1,
64,
mode_node,
1,
'pointconv1',
padding='VALID',
bn=True)
net = cell2D(net,
1,
1,
64,
128,
mode_node,
1,
'pointconv2',
padding='VALID',
bn=True)
net = cell2D(net,
1,
1,
128,
1024,
mode_node,
1,
'pointconv3',
padding='VALID',
bn=True)
# AUGMENT features
net_ = tf.reduce_max(net, axis=1, keep_dims=True)
# MLP 2
net = cell2D(net_,
1,
1,
1024,
512,
mode_node,
1,
'pointconv4',
padding='VALID',
bn=True)
net = cell2D(net,
1,
1,
512,
256,
mode_node,
1,
'pointconv5',
padding='VALID',
bn=True)
# FINAL MAXPOOL
features = tf.squeeze(net, axis=(1, 2))
params = cell1D(features,
1,
mode_node,
SCOPE='pred',
with_act=False,
with_bn=False)
# rot_mat = self.rotation_matrix(params)
rot_mat = self.rotation_matrix_1d(params)
return rot_mat
def build(self, inputs, mode_node, vox_params):
# POINTS
point_cloud = inputs[0]
# Voxels
if self.rep_size == 128:
# # BEST ETAI (elu) 87.5% relu (87.3% ) train 96.7%
# activation = tf.nn.elu
# point_cloud_rot = tf.expand_dims(point_cloud,-1)
## point_cloud_rot,rot_mat = self.transform_points(point_cloud,mode_node,scope='T1')
## T1 = tf.expand_dims(point_cloud_rot,-1)
## mlp1 = self.mlp_etai(point_cloud_rot,[64,64],mode_node,act_type=activation, eps=eps, scope='mlp1')
## T2 = self.transform_features(mlp1,mode_node,scope='T2')
# mlp2 = self.mlp_etai(point_cloud_rot,[64,128,1024],mode_node,scope='mlp1',prob=1,act_type=activation, eps_init=vox_params['eps'])
# pooled = tf.reduce_max(mlp2,axis=1,keep_dims=True)
# mlp3 = self.mlp_etai(pooled,[512,256],mode_node,scope='mlp2',prob=[0.8,0.8],act_type=activation, eps_init=vox_params['eps'])
# features_vox = tf.squeeze(mlp3,axis=(1,2))
# features_vox = cell1D(features_vox,40, mode_node, SCOPE='logits', with_act=False, with_bn=False)
# voxels = features_vox
## # Etai baseline 87.86% train 97.8%
# point_cloud_rot = tf.expand_dims(point_cloud,-1)
## point_cloud_rot,rot_mat = self.transform_points(point_cloud,mode_node,scope='T1')
## T1 = tf.expand_dims(point_cloud_rot,-1)
## mlp1 = self.mlp_etai(point_cloud_rot,[64,64],mode_node,act_type=activation, eps=eps, scope='mlp1')
## T2 = self.transform_features(mlp1,mode_node,scope='T2')
# mlp2 = self.mlp(point_cloud_rot,[64,128,1024],mode_node,scope='mlp2')
# pooled = tf.reduce_max(mlp2,axis=1,keep_dims=True)
# mlp3 = self.mlp(pooled,[512,256],mode_node,scope='mlp3',prob=[0.8,0.8])
# features_vox = tf.squeeze(mlp3,axis=(1,2))
# features_vox = cell1D(features_vox,40, mode_node, SCOPE='logits', with_act=False, with_bn=False)
# voxels = features_vox
# BEST yet 88.63 97.5
act_type = tf.nn.relu
scale_ratio = 0.9
point_cloud = tf.expand_dims(point_cloud, -1)
rot_mat = tf.expand_dims(self.rotations(), axis=1)
point_cloud_tiled = tf.tile(point_cloud, (1, 1, 1, 3))
point_cloud_rot = tf.reduce_sum(point_cloud_tiled * rot_mat,
axis=2)
point_cloud_rot = tf.expand_dims(point_cloud_rot, -1)
scale = tf.random_uniform((self.batch_size, 1, 1, 1), scale_ratio,
1 / scale_ratio)
point_cloud_rot_scaled = point_cloud_rot * scale
with tf.variable_scope('main_branch') as scope:
mlp1 = self.permute_eq(point_cloud, [128, 256, 512, 1024],
mode_node,
scope='mlp1',
act_type=act_type)
pooled_1 = tf.reduce_max(mlp1, axis=1, keep_dims=True)
mlp2 = self.mlp(pooled_1, [512, 256],
mode_node,
scope='mlp2',
prob=[0.6, 0.5],
act_type=act_type)
scope.reuse_variables()
mlp1_ = self.permute_eq(point_cloud_rot_scaled,
[128, 256, 512, 1024],
mode_node,
scope='mlp1',
act_type=act_type)
pooled_1_ = tf.reduce_max(mlp1_, axis=1, keep_dims=True)
# mlp2_ = self.mlp(pooled_1_,[512,256],mode_node,scope='mlp2',prob=[0.6,1],act_type=act_type)
diff = self.correlation_loss(pooled_1, pooled_1_)
tf.add_to_collection('rot_inv', diff)
# diff2 = self.correlation_loss(mlp2, mlp2_)
# tf.add_to_collection('rot_inv',diff2)
features_vox = tf.squeeze(mlp2, axis=(1, 2))
features_vox = cell1D(features_vox,
40,
mode_node,
SCOPE='logits',
with_act=False,
with_bn=False)
voxels = features_vox
count = features_vox
## # BEST yet 87.3 96.2
# point_cloud_rot = tf.expand_dims(point_cloud,-1)
# mlp1 = self.permute_eq(point_cloud_rot,[128,256,1024],mode_node,scope='mlp1',act_type='relu')
# pooled_1 = tf.reduce_max(mlp1,axis=1,keep_dims=True)
# mlp2 = self.mlp(pooled_1,[512,256],mode_node,scope='mlp2',prob=[0.9,0.9],act_type='relu')
# features_vox = tf.squeeze(mlp2,axis=(1,2))
# features_vox = cell1D(features_vox,40, mode_node, SCOPE='logits', with_act=False, with_bn=False)
# voxels = features_vox
# count = features_vox
## # BEST yet 85 90
# set_size = 3
# num_centers = set_size*set_size*set_size
# point_cloud_rot = tf.expand_dims(point_cloud,-1)
# mlp1 = self.permute_eq(point_cloud_rot,[128,256,1024],mode_node,scope='mlp1',act_type='relu')
# pooled_1 = tf.reduce_max(mlp1,axis=1,keep_dims=True)
# mlp2 = self.mlp(pooled_1,[512,512],mode_node,scope='mlp2',prob=[0.9,0.9],act_type='relu')
# centers = tf.squeeze(mlp2,axis=(1,2))
# centers = cell1D(centers,num_centers*3, mode_node, SCOPE='logits', with_act=False, with_bn=False)
# centers = tf.reshape(centers,(self.batch_size,1,3,num_centers))
# point_cloud_aug = point_cloud_rot-centers
# features_vox = []
# with tf.variable_scope('mini_nets'): #, reuse=tf.AUTO_REUSE
# for ii in np.arange(0,num_centers):
# with tf.variable_scope('mini_net_'+str(ii)):
# points = tf.slice(point_cloud_aug,[0,0,0,ii],[-1,-1,-1,1])
# points_ = tf.pow(points,2)
# points = tf.concat((points,points_),axis=2)
# mlp1_mini = self.permute_eq(points,[256,256],mode_node,scope='mlp1_mini',act_type='relu')
# pooled_1_mini = tf.reduce_max(mlp1_mini,axis=1,keep_dims=True)
# mlp2_mini = self.mlp(pooled_1_mini,[128],mode_node,scope='mlp2_mini',prob=1,act_type='relu')
# features_vox.append(mlp2_mini)
# features_vox = tf.reshape(tf.concat(features_vox,axis=1),(self.batch_size,set_size,set_size,set_size,-1) )
# voxels = centers
# count = centers
# # # BEST yet 87 92
# set_size = 3
# num_centers = set_size*set_size*set_size
# point_cloud_rot = tf.expand_dims(point_cloud,-1)
# mlp1 = self.permute_eq(point_cloud_rot,[128,256,1024],mode_node,scope='mlp1',act_type='relu')
# pooled_1 = tf.reduce_max(mlp1,axis=1,keep_dims=True)
# mlp2 = self.mlp(pooled_1,[512,512],mode_node,scope='mlp2',prob=[0.9,0.9],act_type='relu')
# centers = tf.squeeze(mlp2,axis=(1,2))
# centers = cell1D(centers,num_centers*3, mode_node, SCOPE='logits', with_act=False, with_bn=False)
# centers = 3*tf.tanh(tf.reshape(centers,(self.batch_size,1,3,num_centers)))
# point_cloud_aug = point_cloud_rot-centers
# features_vox = []
# with tf.variable_scope('mini_nets'): #, reuse=tf.AUTO_REUSE
# for ii in np.arange(0,num_centers):
# with tf.variable_scope('mini_net_'+str(ii)):
# points = tf.slice(point_cloud_aug,[0,0,0,ii],[-1,-1,-1,1])
# r = tf.reduce_sum(tf.pow(points,2),axis=2,keep_dims=True)
# points = tf.concat((points,r),axis=2)
# mlp1_mini = self.mlp(points,[54,128,512],mode_node,scope='mlp1_mini',prob=1,act_type='relu')
# pooled_1_mini = tf.reduce_max(mlp1_mini,axis=1,keep_dims=True)
# mlp2_mini = self.mlp(pooled_1_mini,[256],mode_node,scope='mlp2_mini',prob=1,act_type='relu')
# features_vox.append(mlp2_mini)
# features_vox = tf.reshape(tf.concat(features_vox,axis=1),(self.batch_size,set_size,set_size,set_size,-1) )
# voxels = centers
# count = centers
# point_cloud_rot = tf.expand_dims(point_cloud,-1)
# mlp = self.mlp(point_cloud_rot,[64,128,512,4096],mode_node,scope='mlp',prob=1,act_type='relu')
# mlp = tf.reduce_max(mlp,axis=1,keep_dims=True)
# features_vox = tf.squeeze(mlp,axis=(1,2))
# voxels = tf.reshape(features_vox,(self.batch_size,self.grid_size,self.grid_size))*self.grid_size
# count = voxels
else:
point_cloud_rot = point_cloud
indices, idx, count, count_normalized, point_cloud_4d = self.get_centers(
point_cloud_rot)
features_vox = tf.cast(indices, tf.float32) / (self.grid_size - 1)
features_vox = tf.slice(features_vox, [0, 2], [-1, 1])
features_vox = tf.expand_dims(tf.cast(count, dtype=tf.float32), -1)
features_vox = features_vox / features_vox
voxels = tf.scatter_nd(indices, features_vox, self.voxel_shape)
voxels = tf.stop_gradient(voxels)
batch_ = []
batch_.append(point_cloud)
batch_.append(point_cloud_rot)
batch_.append(features_vox)
batch_.append(voxels)
return batch_
