def loss(self): # matching loss self.labels, self.weights, self.det_gt_matching = \ matching_module.detection_matching( self.det_anno_iou, self.prediction, self.gt_crowd) # class weighting if class_weights is None: class_weights = np.ones((num_classes + 1), dtype=np.float32) self.class_weights = tf.constant(class_weights, dtype=tf.float32) det_crowd = tf.cond( tf.shape(self.gt_crowd)[0] > 0, lambda: tf.gather( self.gt_crowd, tf.maximum(self.det_gt_matching, 0)), lambda: tf.zeros(tf.shape(self.labels), dtype=tf.bool)) det_class = tf.cond( tf.shape(self.gt_crowd)[0] > 0, lambda: tf.gather(tf.cast(self.gt_classes, tf.int32), tf.maximum(self.det_gt_matching, 0)), lambda: tf.zeros(tf.shape(self.labels), dtype=tf.int32)) det_class = tf.where( tf.logical_and(self.det_gt_matching >= 0, tf.logical_not(det_crowd)), det_class, tf.zeros_like(det_class)) sample_weight = tf.gather(self.class_weights, det_class) self.weights = self.weights * sample_weight sample_losses = tf.nn.sigmoid_cross_entropy_with_logits( self.prediction, self.labels) weighted_losses = sample_losses * self.weights self.loss_unnormed = tf.reduce_sum(weighted_losses, name='cls_loss_unnormed') self.loss_unnormed.set_shape([]) self.loss_normed = tf.reduce_mean(weighted_losses, name='cls_loss_normed') self.loss_normed.set_shape([]) if cfg.train.normalize_loss: self.loss = self.loss_normed else: self.loss = self.loss_unnormed tf.contrib.losses.add_loss(self.loss)
def __init__(self, num_classes, class_weights=None, batch=None, weight_reg=None, reuse=False): self.num_classes = num_classes self.multiclass = num_classes > 1 # inputs if batch is None: for name, (dtype, shape) in self.get_batch_spec(num_classes).items(): setattr(self, name, tf.placeholder(dtype, shape=shape)) else: for name, (dtype, shape) in self.get_batch_spec(num_classes).items(): batch[name].set_shape(shape) setattr(self, name, batch[name]) self._ignore_prefixes = [] if cfg.gnet.imfeats: self.imfeats, stride, self._ignore_prefixes, self.resnet_endpoints = get_resnet( self.image, reuse=reuse) with tf.variable_scope('gnet', reuse=reuse): with tf.variable_scope('preprocessing'): # generate useful box transformations (once) self.dets_boxdata = self._xyxy_to_boxdata(self.dets) self.gt_boxdata = self._xyxy_to_boxdata(self.gt_boxes) # overlaps self.det_anno_iou = self._iou(self.dets_boxdata, self.gt_boxdata, self.gt_crowd) self.det_det_iou = self._iou(self.dets_boxdata, self.dets_boxdata) if self.multiclass: # set overlaps of detection and annotations to 0 if they # have different classes, so they don't get matched in the # loss print('doing multiclass NMS') same_class = tf.equal( tf.reshape(self.det_classes, [-1, 1]), tf.reshape(self.gt_classes, [1, -1])) zeros = tf.zeros_like(self.det_anno_iou) self.det_anno_iou = tf.where(same_class, self.det_anno_iou, zeros) else: print('doing single class NMS') # find neighbors self.neighbor_pair_idxs = tf.where( tf.greater_equal(self.det_det_iou, cfg.gnet.neighbor_thresh)) pair_c_idxs = self.neighbor_pair_idxs[:, 0] pair_n_idxs = self.neighbor_pair_idxs[:, 1] # generate handcrafted pairwise features self.num_dets = tf.shape(self.dets)[0] pw_feats = (self._geometry_feats(pair_c_idxs, pair_n_idxs) * cfg.gnet.pw_feat_multiplyer) # initializers for network weights if cfg.gnet.weight_init == 'xavier': weights_init = tf.contrib.layers.xavier_initializer( seed=cfg.random_seed) elif cfg.gnet.weight_init == 'caffe': weights_init = tf.contrib.layers.variance_scaling_initializer( factor=1.0, mode='FAN_IN', uniform=True) elif cfg.gnet.weight_init == 'msra': weights_init = tf.contrib.layers.variance_scaling_initializer( factor=2.0, mode='FAN_IN', uniform=False) else: raise ValueError('unknown weight init {}'.format( cfg.gnet.weight_init)) biases_init = tf.constant_initializer(cfg.gnet.bias_const_init) if cfg.gnet.num_pwfeat_fc > 0: with tf.variable_scope('pw_feats'): pw_feats = self._pw_feats_fc(pw_feats, weights_init, biases_init, weight_reg) self.pw_feats = pw_feats if cfg.gnet.imfeats: self.roifeats, self.frcn_boxes = crop_windows( self.imfeats, self.dets_boxdata, stride) self.det_imfeats = tf.contrib.layers.flatten(self.roifeats) with tf.variable_scope('reduce_imfeats'): if cfg.gnet.imfeat_dim > 0: self.det_imfeats = tf.contrib.layers.fully_connected( inputs=self.det_imfeats, num_outputs=cfg.gnet.imfeat_dim, activation_fn=tf.nn.relu, weights_initializer=weights_init, weights_regularizer=weight_reg, biases_initializer=biases_init) start_feat = tf.contrib.layers.fully_connected( inputs=self.det_imfeats, num_outputs=cfg.gnet.shortcut_dim, activation_fn=tf.nn.relu, weights_initializer=weights_init, weights_regularizer=weight_reg, biases_initializer=biases_init) else: with tf.variable_scope('gnet'): shortcut_shape = tf.stack( [self.num_dets, cfg.gnet.shortcut_dim]) start_feat = tf.zeros(shortcut_shape, dtype=tf.float32) self.block_feats = [] self.block_feats.append(start_feat) # stack the blocks for block_idx in range(1, cfg.gnet.num_blocks + 1): outfeats = self._block(block_idx, self.block_feats[-1], weights_init, biases_init, pair_c_idxs, pair_n_idxs, pw_feats, weight_reg) #outfeats = tf.Print(outfeats, [outfeats, tf.reduce_max(outfeats), tf.reduce_mean(outfeats)], summarize=20, message='block_{}'.format(block_idx)) self.block_feats.append(outfeats) # do prediction feats = self.block_feats[-1] for i in range(1, cfg.gnet.num_predict_fc): with tf.variable_scope('predict/fc{}'.format(i)): feats = tf.contrib.layers.fully_connected( inputs=feats, num_outputs=cfg.gnet.predict_fc_dim, activation_fn=None, weights_initializer=weights_init, biases_initializer=biases_init) with tf.variable_scope('predict/logits'): prediction = tf.contrib.layers.fully_connected( inputs=feats, num_outputs=1, activation_fn=None, weights_initializer=weights_init, biases_initializer=biases_init) self.prediction = tf.reshape(prediction, [-1]) with tf.variable_scope('loss'): # matching loss self.labels, self.weights, self.det_gt_matching = \ matching_module.detection_matching( self.det_anno_iou, self.prediction, self.gt_crowd) # class weighting if class_weights is None: class_weights = np.ones((num_classes + 1), dtype=np.float32) self.class_weights = tf.constant(class_weights, dtype=tf.float32) det_crowd = tf.cond( tf.shape(self.gt_crowd)[0] > 0, lambda: tf.gather( self.gt_crowd, tf.maximum(self.det_gt_matching, 0)), lambda: tf.zeros(tf.shape(self.labels), dtype=tf.bool)) det_class = tf.cond( tf.shape(self.gt_crowd)[0] > 0, lambda: tf.gather(tf.cast(self.gt_classes, tf.int32), tf.maximum(self.det_gt_matching, 0)), lambda: tf.zeros(tf.shape(self.labels), dtype=tf.int32)) det_class = tf.where( tf.logical_and(self.det_gt_matching >= 0, tf.logical_not(det_crowd)), det_class, tf.zeros_like(det_class)) sample_weight = tf.gather(self.class_weights, det_class) self.weights = self.weights * sample_weight sample_losses = tf.nn.sigmoid_cross_entropy_with_logits( logits=self.prediction, labels=self.labels) weighted_losses = sample_losses * self.weights self.loss_unnormed = tf.reduce_sum(weighted_losses, name='cls_loss_unnormed') self.loss_unnormed.set_shape([]) self.loss_normed = tf.reduce_mean(weighted_losses, name='cls_loss_normed') self.loss_normed.set_shape([]) if cfg.train.normalize_loss: self.loss = self.loss_normed * cfg.train.loss_multiplyer else: self.loss = self.loss_unnormed * cfg.train.loss_multiplyer tf.contrib.losses.add_loss(self.loss) # collect trainable variables tvars = tf.trainable_variables() self.trainable_variables = [ var for var in tvars if (var.name.startswith('gnet') or var.name.startswith('resnet')) and not any( var.name.startswith(pref) for pref in self._ignore_prefixes) ]