def __init__(self,
cls="Car",
batch_size=1,
learning_rate=0.001,
max_gradient_norm=5.0,
alpha=1.5,
beta=1,
training=True):
super(RPN3D, self).__init__()
self.cls = cls
self.batch_size = batch_size
#self.learning_rate = tf.Variable(float(learning_rate), trainable=False, dtype=tf.float32)
self.learning_rate = learning_rate
self.global_step = tf.Variable(0, trainable=False)
#self.global_step = 1
self.epoch = tf.Variable(0, trainable=False)
#self.epoch = 0
#self.epoch_add_op = self.epoch.assign(self.epoch + 1)
self.alpha = alpha
self.max_gradient_norm = max_gradient_norm
self.beta = beta
boundaries = [80, 120]
values = [
self.learning_rate, self.learning_rate * 0.1,
self.learning_rate * 0.01
]
self.lr = tf.compat.v1.train.piecewise_constant(
self.epoch, boundaries, values)
#build graph
self.is_train = training
self.vox_feature = []
self.vox_number = []
self.vox_coordinate = []
self.targets = []
self.pos_equal_one = []
self.pos_equal_one_sum = []
self.pos_equal_one_for_reg = []
self.neg_equal_one = []
self.neg_equal_one_sum = []
self.trainable_params = []
#self.delta_output = []
#self.prob_output = []
self.opt = tf.keras.optimizers.Adam(learning_rate=self.lr)
#self.gradient_norm = []
#self.tower_grads = []
self.rpn = MiddleAndRPN()
self.feature = FeatureNet()
self.ckpt = tf.train.Checkpoint(step=self.global_step,
optimizer=self.opt,
model=self)
def __init__(self, cls = 'Car', alpha = 1.5, beta = 1, sigma = 3):
super(RPN3D, self).__init__()
self.cls = cls
self.alpha = alpha
self.beta = beta
self.sigma = sigma
self.feature = FeatureNet()
self.rpn = MiddleAndRpn()
self.anchors = cal_anchors()
self.rpn_output_shape = self.rpn.output_shape
def __init__(self, cls = 'Car', alpha = 1.5, beta = 1, sigma = 3):
super(RPN3D, self).__init__()
self.cls = cls
self.alpha = alpha
self.beta = beta
self.sigma = sigma
self.feature = FeatureNet()
self.rpn = MiddleAndRPN()
# Generate anchors
self.anchors = cal_anchors() # [cfg.FEATURE_HEIGHT, cfg.FEATURE_WIDTH, 2, 7]; 2 means two rotations; 7 means (cx, cy, cz, h, w, l, r)
self.rpn_output_shape = self.rpn.output_shape
def __init__(self,
cls='Car',
single_batch_size=2,
learning_rate=0.001,
max_gradient_norm=5.0,
alpha=1.5,
beta=1,
avail_gpus=['0']):
self.cls = cls
self.single_batch_size = single_batch_size
self.learning_rate = tf.Variable(float(learning_rate),
trainable=False,
dtype=tf.float32)
self.global_step = tf.Variable(1, trainable=False)
self.epoch = tf.Variable(0, trainable=False)
self.epoch_add_op = self.epoch.assign(self.epoch + 1)
self.alpha = alpha
self.beta = beta
self.avail_gpus = avail_gpus
# 走到一定步长时更改学习率
boundaries = [80, 120]
values = [
self.learning_rate, self.learning_rate * 0.1,
self.learning_rate * 0.01
]
lr = tf.train.piecewise_constant(self.epoch, boundaries, values)
self.is_train = tf.placeholder(tf.bool, name='phase')
self.vox_feature = []
self.vox_number = []
self.vox_coordinate = []
self.targets = []
self.pos_equal_one = []
self.pos_equal_one_sum = []
self.pos_equal_one_for_reg = []
self.neg_equal_one = []
self.neg_equal_one_sum = []
self.delta_output = []
self.prob_output = []
self.opt = tf.train.AdamOptimizer(lr)
self.gradient_norm = []
self.tower_grads = []
with tf.variable_scope(tf.get_variable_scope()):
for idx, dev in enumerate(self.avail_gpus):
with tf.device('/gpu:{}'.format(dev)), tf.name_scope(
'gpu_{}'.format(dev)):
# 生成要进行中间卷积的特征图
feature = FeatureNet(training=self.is_train,
batch_size=self.single_batch_size)
rpn = MiddleAndRPN(input=feature.outputs,
alpha=self.alpha,
beta=self.beta,
training=self.is_train)
tf.get_variable_scope().reuse_variables()
# 输入
self.vox_feature.append(feature.feature)
self.vox_number.append(feature.number)
self.vox_coordinate.append(feature.coordinate)
self.targets.append(rpn.targets)
self.pos_equal_one.append(rpn.pos_equal_one)
self.pos_equal_one_sum.append(rpn.pos_equal_one_sum)
self.pos_equal_one_for_reg.append(
rpn.pos_equal_one_for_reg)
self.neg_equal_one.append(rpn.neg_equal_one)
self.neg_equal_one_sum.append(rpn.neg_equal_one_sum)
# 输出
feature_output = feature.outputs
delta_output = rpn.delta_output
prob_output = rpn.prob_output
# 损失 梯度
if idx == 0:
self.extra_update_ops = tf.get_collection(
tf.GraphKeys.UPDATE_OPS)
self.loss = rpn.loss
self.reg_loss = rpn.reg_loss
self.cls_loss = rpn.cls_loss
self.cls_pos_loss = rpn.cls_pos_loss_rec
self.cls_neg_loss = rpn.cls_neg_loss_rec
self.params = tf.trainable_variables()
gradients = tf.gradients(self.loss, self.params)
clipped_gradients, gradient_norm = tf.clip_by_global_norm(
gradients, max_gradient_norm)
self.delta_output.append(delta_output)
self.prob_output.append(prob_output)
self.tower_grads.append(clipped_gradients)
self.gradient_norm.append(gradient_norm)
self.rpn_output_shape = rpn.output_shape
self.vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
with tf.device('/gpu:{}'.format(self.avail_gpus[0])):
self.grads = average_gradients(self.tower_grads)
self.update = [
self.opt.apply_gradients(zip(self.grads, self.params),
global_step=self.global_step)
]
self.gradient_norm = tf.group(*self.gradient_norm)
self.update.extend(self.extra_update_ops)
self.update = tf.group(*self.update)
self.delta_output = tf.concat(self.delta_output, axis=0)
self.prob_output = tf.concat(self.prob_output, axis=0)
self.anchors = cal_anchors()
# 预测
self.rgb = tf.placeholder(tf.uint8,
[None, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH, 3])
self.bv = tf.placeholder(
tf.uint8,
[ #4
None, cfg.BV_LOG_FACTOR * cfg.INPUT_HEIGHT,
cfg.BV_LOG_FACTOR * cfg.INPUT_WIDTH, 3
])
self.bv_heatmap = tf.placeholder(tf.uint8, [
None, cfg.BV_LOG_FACTOR * cfg.FEATURE_HEIGHT,
cfg.BV_LOG_FACTOR * cfg.FEATURE_WIDTH, 3
])
self.boxes2d = tf.placeholder(tf.float32, [None, 4])
self.boxes2d_scores = tf.placeholder(tf.float32, [None])
# NMS(2D) 非极大值抑制
with tf.device('/gpu:{}'.format(self.avail_gpus[0])):
self.box2d_ind_after_nms = tf.image.non_max_suppression(
self.boxes2d,
self.boxes2d_scores,
max_output_size=cfg.RPN_NMS_POST_TOPK,
iou_threshold=cfg.RPN_NMS_THRESH)
# 保存
self.saver = tf.train.Saver(write_version=tf.train.SaverDef.V2,
max_to_keep=10,
pad_step_number=True,
keep_checkpoint_every_n_hours=1.0)
self.train_summary = tf.summary.merge([
tf.summary.scalar('train/loss', self.loss),
tf.summary.scalar('train/reg_loss', self.reg_loss),
tf.summary.scalar('train/cls_loss', self.cls_loss),
tf.summary.scalar('train/cls_pos_loss', self.cls_pos_loss),
tf.summary.scalar('train/cls_neg_loss', self.cls_neg_loss), *[
tf.summary.histogram(each.name, each)
for each in self.vars + self.params
]
])
self.validate_summary = tf.summary.merge([
tf.summary.scalar('validate/loss', self.loss),
tf.summary.scalar('validate/reg_loss', self.reg_loss),
tf.summary.scalar('validate/cls_loss', self.cls_loss),
tf.summary.scalar('validate/cls_pos_loss', self.cls_pos_loss),
tf.summary.scalar('validate/cls_neg_loss', self.cls_neg_loss)
])
self.predict_summary = tf.summary.merge([
tf.summary.image('predict/bird_view_lidar', self.bv),
tf.summary.image('predict/bird_view_heatmap', self.bv_heatmap),
tf.summary.image('predict/front_view_rgb', self.rgb),
])
def __init__(self,
cls='Car',
single_batch_size=2, # batch_size_per_gpu
learning_rate=0.001,
max_gradient_norm=5.0,
alpha=1.5,
beta=1,
is_train=True,
avail_gpus=['0']):
# hyper parameters and status
self.cls = cls
self.single_batch_size = single_batch_size
self.learning_rate = tf.Variable(
float(learning_rate), trainable=False, dtype=tf.float32)
self.global_step = tf.Variable(1, trainable=False)
self.epoch = tf.Variable(0, trainable=False)
self.epoch_add_op = self.epoch.assign(self.epoch + 1)
self.alpha = alpha
self.beta = beta
self.avail_gpus = avail_gpus
lr = tf.train.exponential_decay(
self.learning_rate, self.global_step, 10000, 0.96)
# build graph
# input placeholders
self.vox_feature = []
self.vox_number = []
self.vox_coordinate = []
self.targets = []
self.pos_equal_one = []
self.pos_equal_one_sum = []
self.pos_equal_one_for_reg = []
self.neg_equal_one = []
self.neg_equal_one_sum = []
self.delta_output = []
self.prob_output = []
self.opt = tf.train.AdamOptimizer(lr)
self.gradient_norm = []
self.tower_grads = []
with tf.variable_scope(tf.get_variable_scope()):
for idx, dev in enumerate(self.avail_gpus):
with tf.device('/gpu:{}'.format(dev)), tf.name_scope('gpu_{}'.format(dev)):
# must use name scope here since we do not want to create new variables
# graph
feature = FeatureNet(
training=is_train, batch_size=self.single_batch_size)
rpn = MiddleAndRPN(
input=feature.outputs, alpha=self.alpha, beta=self.beta, training=is_train)
tf.get_variable_scope().reuse_variables()
# input
self.vox_feature.append(feature.feature)
self.vox_number.append(feature.number)
self.vox_coordinate.append(feature.coordinate)
self.targets.append(rpn.targets)
self.pos_equal_one.append(rpn.pos_equal_one)
self.pos_equal_one_sum.append(rpn.pos_equal_one_sum)
self.pos_equal_one_for_reg.append(
rpn.pos_equal_one_for_reg)
self.neg_equal_one.append(rpn.neg_equal_one)
self.neg_equal_one_sum.append(rpn.neg_equal_one_sum)
# output
feature_output = feature.outputs
delta_output = rpn.delta_output
prob_output = rpn.prob_output
# loss and grad
self.loss = rpn.loss
self.reg_loss = rpn.reg_loss
self.cls_loss = rpn.cls_loss
self.params = tf.trainable_variables()
gradients = tf.gradients(self.loss, self.params)
clipped_gradients, gradient_norm = tf.clip_by_global_norm(
gradients, max_gradient_norm)
self.delta_output.append(delta_output)
self.prob_output.append(prob_output)
self.tower_grads.append(clipped_gradients)
self.gradient_norm.append(gradient_norm)
self.rpn_output_shape = rpn.output_shape
# loss and optimizer
# self.xxxloss is only the loss for the lowest tower
with tf.device('/gpu:{}'.format(self.avail_gpus[0])):
self.grads = average_gradients(self.tower_grads)
self.update = self.opt.apply_gradients(
zip(self.grads, self.params), global_step=self.global_step)
self.gradient_norm = tf.group(*self.gradient_norm)
self.delta_output = tf.concat(self.delta_output, axis=0)
self.prob_output = tf.concat(self.prob_output, axis=0)
self.anchors = cal_anchors()
# for predict and image summary
self.rgb = tf.placeholder(
tf.uint8, [None, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH, 3])
self.bv = tf.placeholder(tf.uint8, [
None, cfg.BV_LOG_FACTOR * cfg.INPUT_HEIGHT, cfg.BV_LOG_FACTOR * cfg.INPUT_WIDTH, 3])
self.bv_heatmap = tf.placeholder(tf.uint8, [
None, cfg.BV_LOG_FACTOR * cfg.FEATURE_HEIGHT, cfg.BV_LOG_FACTOR * cfg.FEATURE_WIDTH, 3])
self.boxes2d = tf.placeholder(tf.float32, [None, 4])
self.boxes2d_scores = tf.placeholder(tf.float32, [None])
# NMS(2D)
with tf.device('/gpu:{}'.format(self.avail_gpus[0])):
self.box2d_ind_after_nms = tf.image.non_max_suppression(
self.boxes2d, self.boxes2d_scores, max_output_size=cfg.RPN_NMS_POST_TOPK, iou_threshold=cfg.RPN_NMS_THRESH)
# summary and saver
self.saver = tf.train.Saver(write_version=tf.train.SaverDef.V2,
max_to_keep=10, pad_step_number=True, keep_checkpoint_every_n_hours=1.0)
self.train_summary = tf.summary.merge([
tf.summary.scalar('train/loss', self.loss),
tf.summary.scalar('train/reg_loss', self.reg_loss),
tf.summary.scalar('train/cls_loss', self.cls_loss),
*[tf.summary.histogram(each.name, each) for each in self.params]
])
self.validate_summary = tf.summary.merge([
tf.summary.scalar('validate/loss', self.loss),
tf.summary.scalar('validate/reg_loss', self.reg_loss),
tf.summary.scalar('validate/cls_loss', self.cls_loss)
])
# TODO: bird_view_summary and front_view_summary
self.predict_summary = tf.summary.merge([
tf.summary.image('predict/bird_view_lidar', self.bv),
tf.summary.image('predict/bird_view_heatmap', self.bv_heatmap),
tf.summary.image('predict/front_view_rgb', self.rgb),
])
def __init__(
self,
cls='Car',
single_batch_size=2, # batch_size_per_gpu
learning_rate=0.001,
max_gradient_norm=5.0,
alpha=1.5,
beta=1,
avail_gpus=['0']):
# hyper parameters and status
self.cls = cls
self.single_batch_size = single_batch_size
self.learning_rate = tf.Variable(float(learning_rate),
trainable=False,
dtype=tf.float32)
self.global_step = tf.Variable(1, trainable=False)
self.epoch = tf.Variable(0, trainable=False)
self.epoch_add_op = self.epoch.assign(self.epoch + 1)
self.alpha = alpha
self.beta = beta
self.avail_gpus = avail_gpus
boundaries = [80, 120]
values = [
self.learning_rate, self.learning_rate * 0.1,
self.learning_rate * 0.01
]
lr = tf.compat.v1.train.piecewise_constant(self.epoch, boundaries,
values)
# build graph
# input placeholders
self.is_train = tf.compat.v1.placeholder(tf.bool, name='phase')
self.vox_feature = []
self.vox_number = []
self.vox_coordinate = []
self.targets = []
self.pos_equal_one = []
self.pos_equal_one_sum = []
self.pos_equal_one_for_reg = []
self.neg_equal_one = []
self.neg_equal_one_sum = []
self.delta_output = []
self.prob_output = []
self.opt = tf.compat.v1.train.AdamOptimizer(lr)
self.gradient_norm = []
self.tower_grads = []
with tf.compat.v1.variable_scope(tf.compat.v1.get_variable_scope()):
for idx, dev in enumerate(self.avail_gpus):
with tf.device('/gpu:{}'.format(dev)), tf.name_scope(
'gpu_{}'.format(dev)):
# must use name scope here since we do not want to create new variables
# graph
feature = FeatureNet(training=self.is_train,
batch_size=self.single_batch_size)
rpn = MiddleAndRPN(input=feature.outputs,
alpha=self.alpha,
beta=self.beta,
training=self.is_train)
tf.compat.v1.get_variable_scope().reuse_variables()
# input
self.vox_feature.append(feature.feature)
self.vox_number.append(feature.number)
self.vox_coordinate.append(feature.coordinate)
self.targets.append(rpn.targets)
self.pos_equal_one.append(rpn.pos_equal_one)
self.pos_equal_one_sum.append(rpn.pos_equal_one_sum)
self.pos_equal_one_for_reg.append(
rpn.pos_equal_one_for_reg)
self.neg_equal_one.append(rpn.neg_equal_one)
self.neg_equal_one_sum.append(rpn.neg_equal_one_sum)
# output
feature_output = feature.outputs
delta_output = rpn.delta_output
prob_output = rpn.prob_output
# loss and grad
if idx == 0:
self.extra_update_ops = tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.UPDATE_OPS)
self.loss = rpn.loss
self.reg_loss = rpn.reg_loss
self.cls_loss = rpn.cls_loss
self.cls_pos_loss = rpn.cls_pos_loss_rec
self.cls_neg_loss = rpn.cls_neg_loss_rec
self.params = tf.compat.v1.trainable_variables()
gradients = tf.gradients(self.loss, self.params)
clipped_gradients, gradient_norm = tf.clip_by_global_norm(
gradients, max_gradient_norm)
self.delta_output.append(delta_output)
self.prob_output.append(prob_output)
self.tower_grads.append(clipped_gradients)
self.gradient_norm.append(gradient_norm)
self.rpn_output_shape = rpn.output_shape
self.vars = tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.GLOBAL_VARIABLES)
# loss and optimizer
# self.xxxloss is only the loss for the lowest tower
with tf.device('/gpu:{}'.format(self.avail_gpus[0])):
self.grads = average_gradients(self.tower_grads)
self.update = [
self.opt.apply_gradients(zip(self.grads, self.params),
global_step=self.global_step)
]
self.gradient_norm = tf.group(*self.gradient_norm)
self.update.extend(self.extra_update_ops)
self.update = tf.group(*self.update)
self.delta_output = tf.concat(self.delta_output, axis=0)
self.prob_output = tf.concat(self.prob_output, axis=0)
self.anchors = cal_anchors()
# for predict and image summary
self.rgb = tf.compat.v1.placeholder(
tf.uint8, [None, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH, 3])
self.bv = tf.compat.v1.placeholder(tf.uint8, [
None, cfg.BV_LOG_FACTOR * cfg.INPUT_HEIGHT,
cfg.BV_LOG_FACTOR * cfg.INPUT_WIDTH, 3
])
self.bv_heatmap = tf.compat.v1.placeholder(tf.uint8, [
None, cfg.BV_LOG_FACTOR * cfg.FEATURE_HEIGHT,
cfg.BV_LOG_FACTOR * cfg.FEATURE_WIDTH, 3
])
self.boxes2d = tf.compat.v1.placeholder(tf.float32, [None, 4])
self.boxes2d_scores = tf.compat.v1.placeholder(tf.float32, [None])
# NMS(2D)
with tf.device('/gpu:{}'.format(self.avail_gpus[0])):
self.box2d_ind_after_nms = tf.image.non_max_suppression(
self.boxes2d,
self.boxes2d_scores,
max_output_size=cfg.RPN_NMS_POST_TOPK,
iou_threshold=cfg.RPN_NMS_THRESH)
# summary and saver
self.saver = tf.compat.v1.train.Saver(
write_version=tf.compat.v1.train.SaverDef.V2,
max_to_keep=10,
pad_step_number=True,
keep_checkpoint_every_n_hours=1.0)
def __init__(self,
cls='Car',
single_batch_size=2, # batch_size_per_gpu
learning_rate=0.001,
max_gradient_norm=5.0,
alpha=1.5,
beta=1,
avail_gpus=['0']):
# hyper parameters and status
self.cls = cls
self.single_batch_size = single_batch_size
self.learning_rate = tf.Variable(
float(learning_rate), trainable=False, dtype=tf.float32)
self.global_step = tf.Variable(1, trainable=False)
self.epoch = tf.Variable(0, trainable=False)
self.epoch_add_op = self.epoch.assign(self.epoch + 1)
self.alpha = alpha
self.beta = beta
self.avail_gpus = avail_gpus
boundaries = [80, 120]
values = [ self.learning_rate, self.learning_rate * 0.1, self.learning_rate * 0.01 ]
lr = tf.train.piecewise_constant(self.epoch, boundaries, values)
# build graph
# input placeholders
self.is_train = tf.placeholder(tf.bool, name='phase')
self.vox_feature = []
self.vox_number = []
self.vox_coordinate = []
self.vox_mask = []
self.total_voxcnt = []
self.targets = []
self.pos_equal_one = []
self.pos_equal_one_sum = []
self.pos_equal_one_for_reg = []
self.neg_equal_one = []
self.neg_equal_one_sum = []
self.delta_output = []
self.prob_output = []
self.opt = tf.train.AdamOptimizer(lr)
self.gradient_norm = []
self.tower_grads = []
with tf.variable_scope(tf.get_variable_scope()):
for idx, dev in enumerate(self.avail_gpus):
with tf.device('/gpu:{}'.format(dev)), tf.name_scope('gpu_{}'.format(dev)):
# must use name scope here since we do not want to create new variables
# graph
if cfg.FEATURE_NET_TYPE == 'FeatureNet':
from model.group_pointcloud import FeatureNet
feature = FeatureNet(training=self.is_train, batch_size=self.single_batch_size)
elif cfg.FEATURE_NET_TYPE == 'FeatureNetSIFT':
from model.group_pointcloud_sift import FeatureNetSIFT
feature = FeatureNetSIFT(training=self.is_train, batch_size=self.single_batch_size)
elif cfg.FEATURE_NET_TYPE == 'FeatureNet_PntNet':
from model.group_pointcloud_pntnet import FeatureNet_PntNet
feature = FeatureNet_PntNet(training=self.is_train, batch_size=self.single_batch_size)
elif cfg.FEATURE_NET_TYPE == 'FeatureNet_PntNet1':
from model.group_pointcloud_pntnet1 import FeatureNet_PntNet1
feature = FeatureNet_PntNet1(training=self.is_train, batch_size=self.single_batch_size)
elif cfg.FEATURE_NET_TYPE == 'FeatureNet_Simple':
from model.group_pointcloud_simple import FeatureNet_Simple
feature = FeatureNet_Simple(training=self.is_train, batch_size=self.single_batch_size)
elif cfg.FEATURE_NET_TYPE == 'FeatureNet_AE':
from model.group_pointcloud_ae import FeatureNet_AE
feature = FeatureNet_AE(training=self.is_train, batch_size=self.single_batch_size, trainable=True)
elif cfg.FEATURE_NET_TYPE == 'FeatureNet_VAE':
from model.group_pointcloud_vae import FeatureNet_VAE
feature = FeatureNet_VAE(training=self.is_train, batch_size=self.single_batch_size, trainable=True)
#
rpn = MiddleAndRPN(input_data=feature.outputs, alpha=self.alpha, beta=self.beta, training=self.is_train)
#
tf.get_variable_scope().reuse_variables()
# input
self.vox_feature.append(feature.feature_pl)
self.vox_number.append(feature.number_pl)
self.vox_coordinate.append(feature.coordinate_pl)
self.vox_mask.append(feature.mask_pl)
self.total_voxcnt.append(feature.voxcnt_pl)
self.targets.append(rpn.targets)
self.pos_equal_one.append(rpn.pos_equal_one)
self.pos_equal_one_sum.append(rpn.pos_equal_one_sum)
self.pos_equal_one_for_reg.append(
rpn.pos_equal_one_for_reg)
self.neg_equal_one.append(rpn.neg_equal_one)
self.neg_equal_one_sum.append(rpn.neg_equal_one_sum)
# output
feature_output = feature.outputs
delta_output = rpn.delta_output
prob_output = rpn.prob_output
# loss and grad
if idx == 0:
self.extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
self.loss = rpn.loss
if cfg.L2_LOSS:
train_vars = tf.trainable_variables()
self.l2_loss = tf.add_n([tf.nn.l2_loss(v) for v in train_vars if 'bias' not in v.name]) * cfg.L2_LOSS_ALPHA
self.loss = self.loss + self.l2_loss
if cfg.L1_LOSS:
train_vars = tf.trainable_variables()
self.l1_loss = tf.add_n([tf.reduce_sum(tf.abs(v)) for v in train_vars if 'bias' not in v.name]) * cfg.L1_LOSS_ALPHA
self.loss = self.loss + self.l1_loss
self.reg_loss = rpn.reg_loss
self.cls_loss = rpn.cls_loss
self.cls_pos_loss = rpn.cls_pos_loss_rec
self.cls_neg_loss = rpn.cls_neg_loss_rec
self.params = tf.trainable_variables()
gradients = tf.gradients(self.loss, self.params)
clipped_gradients, gradient_norm = tf.clip_by_global_norm(
gradients, max_gradient_norm)
self.delta_output.append(delta_output)
self.prob_output.append(prob_output)
self.tower_grads.append(clipped_gradients)
self.gradient_norm.append(gradient_norm)
self.rpn_output_shape = rpn.output_shape
self.vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
# loss and optimizer
# self.xxxloss is only the loss for the lowest tower
with tf.device('/gpu:{}'.format(self.avail_gpus[0])):
self.grads = average_gradients(self.tower_grads)
self.update = [self.opt.apply_gradients(
zip(self.grads, self.params), global_step=self.global_step)]
self.gradient_norm = tf.group(*self.gradient_norm)
self.update.extend(self.extra_update_ops)
self.update = tf.group(*self.update)
self.delta_output = tf.concat(self.delta_output, axis=0)
self.prob_output = tf.concat(self.prob_output, axis=0)
self.anchors = cal_anchors()
# for predict and image summary
self.rgb = tf.placeholder(
tf.uint8, [None, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH, 3])
self.bv = tf.placeholder(tf.uint8, [
None, cfg.BV_LOG_FACTOR * cfg.INPUT_HEIGHT, cfg.BV_LOG_FACTOR * cfg.INPUT_WIDTH, 3])
self.bv_heatmap = tf.placeholder(tf.uint8, [
None, cfg.BV_LOG_FACTOR * cfg.FEATURE_HEIGHT, cfg.BV_LOG_FACTOR * cfg.FEATURE_WIDTH, 3])
self.boxes2d = tf.placeholder(tf.float32, [None, 4])
self.boxes2d_scores = tf.placeholder(tf.float32, [None])
# NMS(2D)
with tf.device('/gpu:{}'.format(self.avail_gpus[0])):
self.box2d_ind_after_nms = tf.image.non_max_suppression(
self.boxes2d, self.boxes2d_scores, max_output_size=cfg.RPN_NMS_POST_TOPK, iou_threshold=cfg.RPN_NMS_THRESH)
# summary and saver
self.saver = tf.train.Saver(max_to_keep=5, pad_step_number=True, keep_checkpoint_every_n_hours=1.0)
train_summary_list = [
tf.summary.scalar('train/loss', self.loss),
tf.summary.scalar('train/reg_loss', self.reg_loss),
tf.summary.scalar('train/cls_loss', self.cls_loss),
tf.summary.scalar('train/cls_pos_loss', self.cls_pos_loss),
tf.summary.scalar('train/cls_neg_loss', self.cls_neg_loss)
]
if cfg.L2_LOSS:
train_summary_list.append(tf.summary.scalar('train/l2_loss', self.l2_loss))
if cfg.L1_LOSS:
train_summary_list.append(tf.summary.scalar('train/l1_loss', self.l1_loss))
if cfg.SUMMART_ALL_VARS:
train_summary_list.append(*[tf.summary.histogram(each.name, each) for each in self.vars + self.params])
self.train_summary = tf.summary.merge(train_summary_list)
self.validate_summary = tf.summary.merge([
tf.summary.scalar('validate/loss', self.loss),
tf.summary.scalar('validate/reg_loss', self.reg_loss),
tf.summary.scalar('validate/cls_loss', self.cls_loss),
tf.summary.scalar('validate/cls_pos_loss', self.cls_pos_loss),
tf.summary.scalar('validate/cls_neg_loss', self.cls_neg_loss)
])
# TODO: bird_view_summary and front_view_summary
self.predict_summary = tf.summary.merge([
tf.summary.image('predict/bird_view_lidar', self.bv),
tf.summary.image('predict/bird_view_heatmap', self.bv_heatmap),
tf.summary.image('predict/front_view_rgb', self.rgb),
])
def __init__(
self,
cls='Car',
single_batch_size=2, # batch_size_per_gpu
learning_rate=0.001,
max_gradient_norm=5.0,
alpha=1.5,
beta=1,
avail_gpus=['0']):
# hyper parameters and status
self.cls = cls
self.single_batch_size = single_batch_size
self.learning_rate = tf.compat.v1.Variable(float(learning_rate),
trainable=False,
dtype=tf.compat.v1.float32)
self.global_step = tf.compat.v1.Variable(1, trainable=False)
self.epoch = tf.compat.v1.Variable(0, trainable=False)
self.epoch_add_op = self.epoch.assign(self.epoch + 1)
self.alpha = alpha
self.beta = beta
self.avail_gpus = avail_gpus
boundaries = [80, 120]
values = [
self.learning_rate, self.learning_rate * 0.1,
self.learning_rate * 0.01
] #changed the learning rate
lr = tf.compat.v1.train.piecewise_constant(self.epoch, boundaries,
values)
# build graph
# input placeholders
self.is_train = tf.compat.v1.placeholder(tf.compat.v1.bool,
name='phase')
self.vox_feature = []
self.vox_number = []
self.vox_coordinate = []
#pedestrian
self.targets = [[] for i in range(2)]
self.pos_equal_one = [[] for i in range(2)]
self.pos_equal_one_sum = [[] for i in range(2)]
self.pos_equal_one_for_reg = [[] for i in range(2)]
self.neg_equal_one = [[] for i in range(2)]
self.neg_equal_one_sum = [[] for i in range(2)]
self.anchors = [[] for i in range(2)]
self.delta_output = [[] for i in range(2)]
self.prob_output = [[] for i in range(2)]
self.opt = tf.compat.v1.train.AdamOptimizer(lr)
self.gradient_norm = []
self.tower_grads = []
with tf.compat.v1.variable_scope(tf.compat.v1.get_variable_scope()):
for idx, dev in enumerate(self.avail_gpus):
with tf.compat.v1.device(
'/gpu:{}'.format(dev)), tf.compat.v1.name_scope(
'gpu_{}'.format(dev)):
#must use name scope here since we do not want to create new variables
#graph
feature = FeatureNet(training=self.is_train,
batch_size=self.single_batch_size)
rpn = MiddleAndRPN(input=feature.outputs,
alpha=self.alpha,
beta=self.beta,
training=self.is_train)
tf.compat.v1.get_variable_scope().reuse_variables()
#input
self.vox_feature.append(feature.feature)
self.vox_number.append(feature.number)
self.vox_coordinate.append(feature.coordinate)
#pedestrian
self.targets[0].append(rpn.targets_p)
self.pos_equal_one[0].append(rpn.pos_equal_one_p)
self.pos_equal_one_sum[0].append(rpn.pos_equal_one_sum_p)
self.pos_equal_one_for_reg[0].append(
rpn.pos_equal_one_for_reg_p)
self.neg_equal_one[0].append(rpn.neg_equal_one_p)
self.neg_equal_one_sum[0].append(rpn.neg_equal_one_sum_p)
#cyclist
self.targets[1].append(rpn.targets_c)
self.pos_equal_one[1].append(rpn.pos_equal_one_c)
self.pos_equal_one_sum[1].append(rpn.pos_equal_one_sum_c)
self.pos_equal_one_for_reg[1].append(
rpn.pos_equal_one_for_reg_c)
self.neg_equal_one[1].append(rpn.neg_equal_one_c)
self.neg_equal_one_sum[1].append(rpn.neg_equal_one_sum_c)
# output
feature_output = feature.outputs
delta_output_p = rpn.delta_output_p
prob_output_p = rpn.prob_output_p
delta_output_c = rpn.delta_output_c
prob_output_c = rpn.prob_output_c
# loss and grad
if idx == 0:
self.extra_update_ops = tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.UPDATE_OPS)
self.loss = rpn.loss
self.reg_loss = rpn.reg_loss
self.cls_loss = rpn.cls_loss
self.cls_pos_loss_p = rpn.cls_pos_loss_rec_p
self.cls_neg_loss_p = rpn.cls_neg_loss_rec_p
self.cls_pos_loss_c = rpn.cls_pos_loss_rec_c
self.cls_neg_loss_c = rpn.cls_neg_loss_rec_c
self.reg_loss_p = rpn.reg_loss_p
self.reg_loss_c = rpn.reg_loss_c
self.params = tf.compat.v1.trainable_variables()
gradients = tf.compat.v1.gradients(self.loss, self.params)
clipped_gradients, gradient_norm = tf.compat.v1.clip_by_global_norm(
gradients, max_gradient_norm)
self.delta_output[0].append(delta_output_p)
self.prob_output[0].append(prob_output_p)
self.delta_output[1].append(delta_output_c)
self.prob_output[1].append(prob_output_c)
self.tower_grads.append(clipped_gradients)
self.gradient_norm.append(gradient_norm)
self.rpn_output_shape = rpn.output_shape
self.vars = tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.GLOBAL_VARIABLES)
# loss and optimizer
# self.xxxloss is only the loss for the lowest tower
with tf.compat.v1.device('/gpu:{}'.format(self.avail_gpus[0])):
self.grads = average_gradients(self.tower_grads)
self.update = [
self.opt.apply_gradients(zip(self.grads, self.params),
global_step=self.global_step)
]
self.gradient_norm = tf.compat.v1.group(*self.gradient_norm)
self.update.extend(self.extra_update_ops)
self.update = tf.compat.v1.group(*self.update)
self.delta_output[0] = tf.compat.v1.concat(self.delta_output[0],
axis=0)
self.prob_output[0] = tf.compat.v1.concat(self.prob_output[0], axis=0)
self.delta_output[1] = tf.compat.v1.concat(self.delta_output[1],
axis=0)
self.prob_output[1] = tf.compat.v1.concat(self.prob_output[1], axis=0)
# concatenate both cyclist and pedestrian tensors here
# concatenate prob_map of cyclist and pedestrian and reg_map of cyclist and pedestrian
self.anchors[0] = cal_anchors(0.8) #pedestrian
self.anchors[1] = cal_anchors(1.7) #cyclist
# for predict and image summary
self.rgb = tf.compat.v1.placeholder(
tf.compat.v1.uint8, [None, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH, 3])
self.bv = tf.compat.v1.placeholder(tf.compat.v1.uint8, [
None, cfg.BV_LOG_FACTOR * cfg.INPUT_HEIGHT,
cfg.BV_LOG_FACTOR * cfg.INPUT_WIDTH, 3
])
self.bv_heatmap = tf.compat.v1.placeholder(tf.compat.v1.uint8, [
None, cfg.BV_LOG_FACTOR * cfg.FEATURE_HEIGHT,
cfg.BV_LOG_FACTOR * cfg.FEATURE_WIDTH, 3
])
self.boxes2d = tf.compat.v1.placeholder(tf.compat.v1.float32,
[None, 4],
name="boxes2d")
self.boxes2d_scores = tf.compat.v1.placeholder(tf.compat.v1.float32,
[None],
name="boxes2d_scores")
# NMS(2D) write 3D NMS
with tf.compat.v1.device('/gpu:{}'.format(self.avail_gpus[0])):
self.box2d_ind_after_nms = tf.compat.v1.image.non_max_suppression(
self.boxes2d,
self.boxes2d_scores,
max_output_size=cfg.RPN_NMS_POST_TOPK,
iou_threshold=cfg.RPN_NMS_THRESH)
with tf.compat.v1.device('/gpu:{}'.format(self.avail_gpus[0])):
self.box2d_ind_after_nms = tf.compat.v1.image.non_max_suppression(
self.boxes2d,
self.boxes2d_scores,
max_output_size=cfg.RPN_NMS_POST_TOPK,
iou_threshold=cfg.RPN_NMS_THRESH)
# summary and saver
self.saver = tf.compat.v1.train.Saver(
write_version=tf.compat.v1.train.SaverDef.V2,
max_to_keep=10,
pad_step_number=True,
keep_checkpoint_every_n_hours=1.0)
self.train_summary = tf.compat.v1.summary.merge([
tf.compat.v1.summary.scalar('train/loss', self.loss),
tf.compat.v1.summary.scalar('train/reg_loss', self.reg_loss),
tf.compat.v1.summary.scalar('train/cls_loss', self.cls_loss),
tf.compat.v1.summary.scalar('train/cls_pos_loss_p',
self.cls_pos_loss_p),
tf.compat.v1.summary.scalar('train/cls_neg_loss_p',
self.cls_neg_loss_p),
tf.compat.v1.summary.scalar('train/cls_pos_loss_c',
self.cls_pos_loss_c),
tf.compat.v1.summary.scalar('train/cls_neg_loss_c',
self.cls_neg_loss_c),
tf.compat.v1.summary.scalar('train/reg_loss_p', self.reg_loss_p),
tf.compat.v1.summary.scalar('train/reg_loss_c', self.reg_loss_c),
*[
tf.compat.v1.summary.histogram(each.name, each)
for each in self.vars + self.params
]
])
self.validate_summary = tf.compat.v1.summary.merge([
tf.compat.v1.summary.scalar('validate/loss', self.loss),
tf.compat.v1.summary.scalar('validate/reg_loss', self.reg_loss),
tf.compat.v1.summary.scalar('validate/cls_loss', self.cls_loss),
tf.compat.v1.summary.scalar('validate/cls_pos_loss_p',
self.cls_pos_loss_p),
tf.compat.v1.summary.scalar('validate/cls_neg_loss_p',
self.cls_neg_loss_p),
tf.compat.v1.summary.scalar('validate/cls_pos_loss_c',
self.cls_pos_loss_c),
tf.compat.v1.summary.scalar('validate/cls_neg_loss_c',
self.cls_neg_loss_c),
tf.compat.v1.summary.scalar('train/reg_loss_p', self.reg_loss_p),
tf.compat.v1.summary.scalar('train/reg_loss_c', self.reg_loss_c)
])
# TODO: bird_view_summary and front_view_summary
self.predict_summary = tf.compat.v1.summary.merge([
tf.compat.v1.summary.image('predict/bird_view_lidar', self.bv),
tf.compat.v1.summary.image('predict/bird_view_heatmap',
self.bv_heatmap),
tf.compat.v1.summary.image('predict/front_view_rgb', self.rgb),
])