def _loss_ops(self):
class_labels = []
pairwise_dt_gt_coords = spatial.construct_pairwise_features_tf(
self.dt_coords, self.gt_coords)
dt_gt_iou = tf.squeeze(
spatial.compute_pairwise_spatial_features_iou_tf(
pairwise_dt_gt_coords), 2)
for class_id in range(0, self.n_classes):
gt_per_label = losses.construct_ground_truth_per_label_tf(
dt_gt_iou, self.gt_labels, class_id)
# self.gt_per_labels.append(gt_per_label)
class_labels.append(
losses.compute_match_gt_net_per_label_tf(
self.class_scores, gt_per_label, class_id))
labels = tf.stack(class_labels, axis=1)
if self.use_hinge_loss:
loss = slim.losses.hinge_loss(self.logits, labels)
else:
loss = tf.nn.weighted_cross_entropy_with_logits(
self.logits, labels, pos_weight=self.pos_weight)
# hard_indices_tf = misc.data_subselection_hard_negative_tf(
# self.dt_labels, cross_entropy, n_neg_examples=self.n_neg_examples)
#
# loss_hard_tf = tf.gather(cross_entropy, hard_indices_tf)
loss_final = tf.reduce_mean(loss)
return labels, loss_final
def _detection_loss_ops(self):
classes_labels_independent = []
classes_labels_final = []
pairwise_dt_gt_coords = spatial.construct_pairwise_features_tf(
self.dt_coords, self.gt_coords)
dt_gt_iou = tf.squeeze(
spatial.compute_pairwise_spatial_features_iou_tf(
pairwise_dt_gt_coords), 2)
for class_id in range(0, self.n_classes):
class_labels_independent = losses.construct_independent_labels(
dt_gt_iou,
self.gt_labels,
class_id,
iou_threshold=self.gt_match_iou_thr)
classes_labels_independent.append(class_labels_independent)
gt_per_label = losses.construct_ground_truth_per_label_tf(
dt_gt_iou,
self.gt_labels,
class_id,
iou_threshold=self.gt_match_iou_thr)
classes_labels_final.append(
losses.compute_match_gt_net_per_label_tf(
self.dt_probs_ini, gt_per_label, class_id))
# self.classes_labels_independent = tf.stack(classes_labels_independent, axis=1)
self.classes_labels_final = tf.stack(classes_labels_final, axis=1)
# self.filter_labels = tf.to_float(tf.equal(self.classes_labels_independent, self.classes_labels_final))
labels = self.classes_labels_final
if self.class_scores_func == 'softmax':
loss = tf.nn.softmax_cross_entropy_with_logits(labels=labels,
logits=self.logits)
else:
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels,
logits=self.logits)
self.det_loss_elementwise = loss
# weighted_loss = tf.multiply(loss, 10*self.dt_probs_ini)
# det_loss_final_weighted = tf.reduce_mean(weighted_loss, name='detection_loss_weighted')
det_loss_final = tf.reduce_mean(loss, name='detection_loss')
return labels, det_loss_final
def _inference_ops(self):
# we'll do one iteration of fc layers to prepare input for knet
dt_features_ini = self._fc_layer_chain(
input_tensor=self.dt_features,
layer_size=self.fc_ini_layer_size,
n_layers=self.fc_ini_layers_cnt,
scope='fc_ini_layer')
# we also reduce dimensionality of our features before passing it through knet
dt_features_pre_knet = self._fc_layer_chain(
input_tensor=self.dt_features,
layer_size=self.fc_pre_layer_size,
n_layers=self.fc_pre_layers_cnt,
scope='fc_pre_layer_knet')
pairwise_coords_features = spatial.construct_pairwise_features_tf(
self.dt_coords)
spatial_features_list = []
n_spatial_features = 0
iou_feature = spatial.compute_pairwise_spatial_features_iou_tf(
pairwise_coords_features)
if self.use_iou_features:
spatial_features_list.append(iou_feature)
n_spatial_features += 1
pairwise_obj_features = spatial.construct_pairwise_features_tf(
dt_features_pre_knet)
if self.use_object_features:
spatial_features_list.append(pairwise_obj_features)
n_spatial_features += dt_features_pre_knet.get_shape().as_list(
)[1] * 2
if self.use_coords_features:
spatial_features_list.append(pairwise_coords_features)
n_spatial_features += self.n_dt_coords * 2
spatial_features = tf.concat(axis=2, values=spatial_features_list)
diagonals = []
for i in range(0, n_spatial_features):
d = tf.expand_dims(tf.diag(tf.diag_part(spatial_features[:, :,
i])),
axis=2)
diagonals.append(d)
diag = tf.concat(axis=2, values=diagonals)
spatial_features = spatial_features - diag
spatial_features = tf.reshape(
spatial_features,
[self.n_bboxes, n_spatial_features * self.n_bboxes])
input_features = tf.concat(axis=1,
values=[self.dt_features, spatial_features])
fc_chain = self._fc_layer_chain(input_tensor=input_features,
layer_size=512,
n_layers=3,
scope='fc_spatial')
fc_chain = tf.nn.dropout(fc_chain, self.keep_prob)
logits = slim.layers.fully_connected(fc_chain,
self.n_classes,
activation_fn=None)
class_scores = tf.nn.sigmoid(logits)
return input_features, iou_feature, logits, class_scores
def _nms_loss_ops(self):
if self.n_classes == 1:
self.pairwise_probs_features = spatial.construct_pairwise_features_tf(
self.dt_probs_ini)
suppression_map = self.pairwise_probs_features[:, :, 1] > \
self.pairwise_probs_features[:, :, 0]
iou_map = self.iou_feature[:, :, 0] > self.nms_label_iou
nms_pairwise_labels = tf.to_float(
tf.logical_and(suppression_map, iou_map))
nms_labels = 1 - tf.reshape(
tf.reduce_max(nms_pairwise_labels, axis=1), [self.n_bboxes, 1])
else:
self.pairwise_probs_features = spatial.construct_pairwise_features_tf(
self.dt_probs_ini)
nms_labels = []
# background_class_labels = tf.ones([self.n_bboxes, 1])
# nms_labels.append(background_class_labels)
for class_id in range(0, self.n_classes):
suppression_map = self.pairwise_probs_features[:, :, class_id + self.n_classes] >\
self.pairwise_probs_features[:, :, class_id]
iou_map = self.iou_feature[:, :, 0] > self.nms_label_iou
nms_pairwise_labels = tf.to_float(
tf.logical_and(suppression_map, iou_map))
class_nms_labels = 1 - tf.reshape(
tf.reduce_max(nms_pairwise_labels, axis=1),
[self.n_bboxes, 1])
nms_labels.append(class_nms_labels)
nms_labels = tf.squeeze(tf.stack(nms_labels, axis=1), axis=2)
# suppression_map = self.pairwise_obj_features[:, :,
# self.n_dt_features+1] > self.pairwise_obj_features[:, :, 1]
# self.nms_pairwise_labels = nms_pairwise_labels
nms_elementwise_loss = tf.nn.sigmoid_cross_entropy_with_logits(
labels=nms_labels, logits=self.logits)
nms_loss_final = tf.reduce_mean(nms_elementwise_loss, name='nms_loss')
# weighted_loss = tf.multiply(nms_elementwise_loss, 10*self.dt_probs_ini)
#
# nms_loss_final_weighted = tf.reduce_mean(weighted_loss, name='nms_loss_weighted')
return nms_labels, nms_loss_final
def _inference_ops_top_k(self):
if self.n_classes == 1:
highest_prob = tf.reduce_max(self.dt_probs_ini, axis=1)
else:
# we are considering all classes, skipping the backgorund class
highest_prob = tf.reduce_max(self.dt_probs_ini[:, 1:], axis=1)
_, top_ix = tf.nn.top_k(highest_prob, k=self.top_k_hypotheses)
pairwise_coords_features = spatial.construct_pairwise_features_tf(
self.dt_coords)
pairwise_coords_features_top_k = spatial.construct_pairwise_features_tf(
self.dt_coords, tf.gather(self.dt_coords, top_ix))
spatial_features_list = []
n_pairwise_features = 0
iou_feature = spatial.compute_pairwise_spatial_features_iou_tf(
pairwise_coords_features)
iou_feature_top_k = spatial.compute_pairwise_spatial_features_iou_tf(
pairwise_coords_features_top_k)
if self.use_iou_features:
spatial_features_list.append(iou_feature_top_k)
n_pairwise_features += 1
if self.loss_type == 'detection':
misc_spatial_features = spatial.compute_misc_pairwise_spatial_features_tf(
pairwise_coords_features)
spatial_features_list.append(misc_spatial_features)
n_pairwise_features += 5
pairwise_obj_features_top_k = spatial.construct_pairwise_features_tf(
self.dt_features_merged, tf.gather(self.dt_features_merged,
top_ix))
if self.use_object_features:
spatial_features_list.append(pairwise_obj_features_top_k)
n_pairwise_features += self.dt_features_merged.get_shape().as_list(
)[1] * 2
score_diff_sign_feature = tf.sign(
pairwise_obj_features_top_k[:, :, 0:self.n_dt_features] -
pairwise_obj_features_top_k[:, :, self.n_dt_features:])
score_diff_feature = pairwise_obj_features_top_k[:, :, 0:self.n_dt_features] - \
pairwise_obj_features_top_k[:, :, self.n_dt_features:]
spatial_features_list.append(score_diff_sign_feature)
spatial_features_list.append(score_diff_feature)
n_pairwise_features += self.dt_features_merged.get_shape().as_list(
)[1] * 2
pairwise_features = tf.concat(axis=2, values=spatial_features_list)
self.pairwise_obj_features = pairwise_features
kernel_features = self._kernel(pairwise_features,
n_pairwise_features,
hlayer_size=self.knet_hlayer_size,
n_kernels=self.n_kernels)
kernel_features_sigmoid = tf.nn.sigmoid(kernel_features)
kernel_max = tf.reshape(tf.reduce_max(kernel_features_sigmoid, axis=1),
[self.n_bboxes, self.n_kernels])
kernel_sum = tf.reshape(tf.reduce_sum(kernel_features_sigmoid, axis=1),
[self.n_bboxes, self.n_kernels])
object_and_context_features = tf.concat(
axis=1, values=[self.dt_features_merged, kernel_max, kernel_sum])
self.object_and_context_features = object_and_context_features
fc1 = slim.layers.fully_connected(object_and_context_features,
self.fc_apres_layer_size,
activation_fn=tf.nn.relu)
fc2 = slim.layers.fully_connected(fc1,
self.fc_apres_layer_size,
activation_fn=tf.nn.relu)
fc2_drop = tf.nn.dropout(fc2, self.keep_prob)
logits = slim.fully_connected(fc2_drop,
self.n_classes,
activation_fn=None)
if self.class_scores_func == 'softmax':
class_scores = tf.nn.softmax(logits)
elif self.class_scores_func == 'sigmoid':
class_scores = tf.nn.sigmoid(logits)
else:
class_scores = tf.nn.sigmoid(logits)
return iou_feature, logits, class_scores
def _inference_ops(self):
if self.n_classes == 1:
highest_prob = tf.reduce_max(self.dt_probs_ini, axis=1)
else:
# we are considering all classes, skip the backgorund class
highest_prob = tf.reduce_max(self.dt_probs_ini[:, 1:], axis=1)
_, top_ix = tf.nn.top_k(highest_prob, k=self.top_k_hypotheses)
pairwise_coords_features = spatial.construct_pairwise_features_tf(
self.dt_coords)
spatial_features_list = []
n_pairwise_features = 0
iou_feature = spatial.compute_pairwise_spatial_features_iou_tf(
pairwise_coords_features)
if self.use_iou_features:
spatial_features_list.append(iou_feature)
n_pairwise_features += 1
pairwise_obj_features = spatial.construct_pairwise_features_tf(
self.dt_features_merged)
if self.use_object_features:
spatial_features_list.append(pairwise_obj_features)
n_pairwise_features += self.dt_features_merged.get_shape().as_list(
)[1] * 2
score_diff_sign_feature = tf.sign(
pairwise_obj_features[:, :, 0:self.n_dt_features] -
pairwise_obj_features[:, :, self.n_dt_features:])
score_diff_feature = pairwise_obj_features[:, :, 0:self.n_dt_features] -\
pairwise_obj_features[:, :, self.n_dt_features:]
spatial_features_list.append(score_diff_sign_feature)
spatial_features_list.append(score_diff_feature)
n_pairwise_features += self.dt_features_merged.get_shape().as_list(
)[1] * 2
pairwise_features = tf.concat(axis=2, values=spatial_features_list)
diagonals = []
for i in range(0, n_pairwise_features):
d = tf.expand_dims(tf.diag(tf.diag_part(pairwise_features[:, :,
i])),
axis=2)
diagonals.append(d)
diag = tf.concat(axis=2, values=diagonals)
pairwise_features = pairwise_features - diag
self.pairwise_obj_features = pairwise_features
kernel_features = self._kernel(pairwise_features,
n_pairwise_features,
hlayer_size=self.knet_hlayer_size,
n_kernels=self.n_kernels)
kernel_features_sigmoid = tf.nn.sigmoid(kernel_features)
kernel_max = tf.reshape(tf.reduce_max(kernel_features_sigmoid, axis=1),
[self.n_bboxes, self.n_kernels])
kernel_sum = tf.reshape(tf.reduce_sum(kernel_features_sigmoid, axis=1),
[self.n_bboxes, self.n_kernels])
object_and_context_features = tf.concat(
axis=1, values=[self.dt_features_merged, kernel_max, kernel_sum])
self.object_and_context_features = object_and_context_features
fc1 = slim.layers.fully_connected(object_and_context_features,
self.fc_apres_layer_size,
activation_fn=tf.nn.relu)
fc2 = slim.layers.fully_connected(fc1,
self.fc_apres_layer_size,
activation_fn=tf.nn.relu)
fc2_drop = tf.nn.dropout(fc2, self.keep_prob)
logits = slim.fully_connected(fc2_drop,
self.n_classes,
activation_fn=None)
class_scores = tf.nn.sigmoid(logits)
return iou_feature, logits, class_scores