def loss(self):
# matching loss
self.labels, self.weights, self.det_gt_matching = \
matching_module.detection_matching(
self.det_anno_iou, self.prediction, self.gt_crowd)
# class weighting
if class_weights is None:
class_weights = np.ones((num_classes + 1), dtype=np.float32)
self.class_weights = tf.constant(class_weights, dtype=tf.float32)
det_crowd = tf.cond(
tf.shape(self.gt_crowd)[0] > 0, lambda: tf.gather(
self.gt_crowd, tf.maximum(self.det_gt_matching, 0)),
lambda: tf.zeros(tf.shape(self.labels), dtype=tf.bool))
det_class = tf.cond(
tf.shape(self.gt_crowd)[0] > 0,
lambda: tf.gather(tf.cast(self.gt_classes, tf.int32),
tf.maximum(self.det_gt_matching, 0)),
lambda: tf.zeros(tf.shape(self.labels), dtype=tf.int32))
det_class = tf.where(
tf.logical_and(self.det_gt_matching >= 0,
tf.logical_not(det_crowd)), det_class,
tf.zeros_like(det_class))
sample_weight = tf.gather(self.class_weights, det_class)
self.weights = self.weights * sample_weight
sample_losses = tf.nn.sigmoid_cross_entropy_with_logits(
self.prediction, self.labels)
weighted_losses = sample_losses * self.weights
self.loss_unnormed = tf.reduce_sum(weighted_losses,
name='cls_loss_unnormed')
self.loss_unnormed.set_shape([])
self.loss_normed = tf.reduce_mean(weighted_losses,
name='cls_loss_normed')
self.loss_normed.set_shape([])
if cfg.train.normalize_loss:
self.loss = self.loss_normed
else:
self.loss = self.loss_unnormed
tf.contrib.losses.add_loss(self.loss)
def __init__(self,
num_classes,
class_weights=None,
batch=None,
weight_reg=None,
reuse=False):
self.num_classes = num_classes
self.multiclass = num_classes > 1
# inputs
if batch is None:
for name, (dtype,
shape) in self.get_batch_spec(num_classes).items():
setattr(self, name, tf.placeholder(dtype, shape=shape))
else:
for name, (dtype,
shape) in self.get_batch_spec(num_classes).items():
batch[name].set_shape(shape)
setattr(self, name, batch[name])
self._ignore_prefixes = []
if cfg.gnet.imfeats:
self.imfeats, stride, self._ignore_prefixes, self.resnet_endpoints = get_resnet(
self.image, reuse=reuse)
with tf.variable_scope('gnet', reuse=reuse):
with tf.variable_scope('preprocessing'):
# generate useful box transformations (once)
self.dets_boxdata = self._xyxy_to_boxdata(self.dets)
self.gt_boxdata = self._xyxy_to_boxdata(self.gt_boxes)
# overlaps
self.det_anno_iou = self._iou(self.dets_boxdata,
self.gt_boxdata, self.gt_crowd)
self.det_det_iou = self._iou(self.dets_boxdata,
self.dets_boxdata)
if self.multiclass:
# set overlaps of detection and annotations to 0 if they
# have different classes, so they don't get matched in the
# loss
print('doing multiclass NMS')
same_class = tf.equal(
tf.reshape(self.det_classes, [-1, 1]),
tf.reshape(self.gt_classes, [1, -1]))
zeros = tf.zeros_like(self.det_anno_iou)
self.det_anno_iou = tf.where(same_class, self.det_anno_iou,
zeros)
else:
print('doing single class NMS')
# find neighbors
self.neighbor_pair_idxs = tf.where(
tf.greater_equal(self.det_det_iou,
cfg.gnet.neighbor_thresh))
pair_c_idxs = self.neighbor_pair_idxs[:, 0]
pair_n_idxs = self.neighbor_pair_idxs[:, 1]
# generate handcrafted pairwise features
self.num_dets = tf.shape(self.dets)[0]
pw_feats = (self._geometry_feats(pair_c_idxs, pair_n_idxs) *
cfg.gnet.pw_feat_multiplyer)
# initializers for network weights
if cfg.gnet.weight_init == 'xavier':
weights_init = tf.contrib.layers.xavier_initializer(
seed=cfg.random_seed)
elif cfg.gnet.weight_init == 'caffe':
weights_init = tf.contrib.layers.variance_scaling_initializer(
factor=1.0, mode='FAN_IN', uniform=True)
elif cfg.gnet.weight_init == 'msra':
weights_init = tf.contrib.layers.variance_scaling_initializer(
factor=2.0, mode='FAN_IN', uniform=False)
else:
raise ValueError('unknown weight init {}'.format(
cfg.gnet.weight_init))
biases_init = tf.constant_initializer(cfg.gnet.bias_const_init)
if cfg.gnet.num_pwfeat_fc > 0:
with tf.variable_scope('pw_feats'):
pw_feats = self._pw_feats_fc(pw_feats, weights_init,
biases_init, weight_reg)
self.pw_feats = pw_feats
if cfg.gnet.imfeats:
self.roifeats, self.frcn_boxes = crop_windows(
self.imfeats, self.dets_boxdata, stride)
self.det_imfeats = tf.contrib.layers.flatten(self.roifeats)
with tf.variable_scope('reduce_imfeats'):
if cfg.gnet.imfeat_dim > 0:
self.det_imfeats = tf.contrib.layers.fully_connected(
inputs=self.det_imfeats,
num_outputs=cfg.gnet.imfeat_dim,
activation_fn=tf.nn.relu,
weights_initializer=weights_init,
weights_regularizer=weight_reg,
biases_initializer=biases_init)
start_feat = tf.contrib.layers.fully_connected(
inputs=self.det_imfeats,
num_outputs=cfg.gnet.shortcut_dim,
activation_fn=tf.nn.relu,
weights_initializer=weights_init,
weights_regularizer=weight_reg,
biases_initializer=biases_init)
else:
with tf.variable_scope('gnet'):
shortcut_shape = tf.stack(
[self.num_dets, cfg.gnet.shortcut_dim])
start_feat = tf.zeros(shortcut_shape, dtype=tf.float32)
self.block_feats = []
self.block_feats.append(start_feat)
# stack the blocks
for block_idx in range(1, cfg.gnet.num_blocks + 1):
outfeats = self._block(block_idx, self.block_feats[-1],
weights_init, biases_init, pair_c_idxs,
pair_n_idxs, pw_feats, weight_reg)
#outfeats = tf.Print(outfeats, [outfeats, tf.reduce_max(outfeats), tf.reduce_mean(outfeats)], summarize=20, message='block_{}'.format(block_idx))
self.block_feats.append(outfeats)
# do prediction
feats = self.block_feats[-1]
for i in range(1, cfg.gnet.num_predict_fc):
with tf.variable_scope('predict/fc{}'.format(i)):
feats = tf.contrib.layers.fully_connected(
inputs=feats,
num_outputs=cfg.gnet.predict_fc_dim,
activation_fn=None,
weights_initializer=weights_init,
biases_initializer=biases_init)
with tf.variable_scope('predict/logits'):
prediction = tf.contrib.layers.fully_connected(
inputs=feats,
num_outputs=1,
activation_fn=None,
weights_initializer=weights_init,
biases_initializer=biases_init)
self.prediction = tf.reshape(prediction, [-1])
with tf.variable_scope('loss'):
# matching loss
self.labels, self.weights, self.det_gt_matching = \
matching_module.detection_matching(
self.det_anno_iou, self.prediction, self.gt_crowd)
# class weighting
if class_weights is None:
class_weights = np.ones((num_classes + 1),
dtype=np.float32)
self.class_weights = tf.constant(class_weights,
dtype=tf.float32)
det_crowd = tf.cond(
tf.shape(self.gt_crowd)[0] > 0, lambda: tf.gather(
self.gt_crowd, tf.maximum(self.det_gt_matching, 0)),
lambda: tf.zeros(tf.shape(self.labels), dtype=tf.bool))
det_class = tf.cond(
tf.shape(self.gt_crowd)[0] > 0,
lambda: tf.gather(tf.cast(self.gt_classes, tf.int32),
tf.maximum(self.det_gt_matching, 0)),
lambda: tf.zeros(tf.shape(self.labels), dtype=tf.int32))
det_class = tf.where(
tf.logical_and(self.det_gt_matching >= 0,
tf.logical_not(det_crowd)), det_class,
tf.zeros_like(det_class))
sample_weight = tf.gather(self.class_weights, det_class)
self.weights = self.weights * sample_weight
sample_losses = tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.prediction, labels=self.labels)
weighted_losses = sample_losses * self.weights
self.loss_unnormed = tf.reduce_sum(weighted_losses,
name='cls_loss_unnormed')
self.loss_unnormed.set_shape([])
self.loss_normed = tf.reduce_mean(weighted_losses,
name='cls_loss_normed')
self.loss_normed.set_shape([])
if cfg.train.normalize_loss:
self.loss = self.loss_normed * cfg.train.loss_multiplyer
else:
self.loss = self.loss_unnormed * cfg.train.loss_multiplyer
tf.contrib.losses.add_loss(self.loss)
# collect trainable variables
tvars = tf.trainable_variables()
self.trainable_variables = [
var for var in tvars
if (var.name.startswith('gnet') or var.name.startswith('resnet'))
and not any(
var.name.startswith(pref) for pref in self._ignore_prefixes)
]