def _negative_weighted_ce_loss(y_true, y_pred): """ Compute the focal loss given the target tensor and the predicted tensor. As defined in https://arxiv.org/abs/1708.02002 Args y_true: Tensor of target data from the generator with shape (B, N, num_classes). y_pred: Tensor of predicted data from the network with shape (B, N, num_classes). Returns The focal loss of y_pred w.r.t. y_true. """ labels = y_true[:, :, :-1] anchor_state = y_true[:, :, -1] # -1 for ignore, 0 for background, 1 for object classification = y_pred # filter out "ignore" anchors indices = backend.where(keras.backend.not_equal(anchor_state, -1)) labels = backend.gather_nd(labels, indices) classification = backend.gather_nd(classification, indices) cls_loss = binary_crossentropy(labels, classification) # compute the normalizer: the number of positive anchors normalizer = backend.where(keras.backend.equal(anchor_state, 1)) normalizer = keras.backend.cast(keras.backend.shape(normalizer)[0], keras.backend.floatx()) normalizer = keras.backend.maximum(keras.backend.cast_to_floatx(1.0), normalizer) return keras.backend.sum(cls_loss) / normalizer
def _smooth_l1(y_true, y_pred): """ Compute the smooth L1 loss of y_pred w.r.t. y_true. Args y_true: Tensor from the generator of shape (B, N, 5). The last value for each box is the state of the anchor (ignore, negative, positive). y_pred: Tensor from the network of shape (B, N, 4). Returns The smooth L1 loss of y_pred w.r.t. y_true. """ # separate target and state regression = y_pred regression_target = y_true[:, :, :-1] anchor_state = y_true[:, :, -1] # filter out "ignore" anchors indices = backend.where(keras.backend.equal(anchor_state, 1)) regression = backend.gather_nd(regression, indices) regression_target = backend.gather_nd(regression_target, indices) # compute smooth L1 loss # f(x) = 0.5 * (sigma * x)^2 if |x| < 1 / sigma / sigma # |x| - 0.5 / sigma / sigma otherwise regression_diff = regression - regression_target regression_diff = keras.backend.abs(regression_diff) regression_loss = backend.where( keras.backend.less(regression_diff, 1.0 / sigma_squared), 0.5 * sigma_squared * keras.backend.pow(regression_diff, 2), regression_diff - 0.5 / sigma_squared) # compute the normalizer: the number of positive anchors normalizer = keras.backend.maximum(1, keras.backend.shape(indices)[0]) normalizer = keras.backend.cast(normalizer, dtype=keras.backend.floatx()) return r_weight * (keras.backend.sum(regression_loss) / normalizer)
def _smooth_l1(y_true, y_pred): # separate target and state regression = y_pred regression_target = y_true[:, :, :4] anchor_state = y_true[:, :, 4] # compute smooth L1 loss # f(x) = 0.5 * (sigma * x)^2 if |x| < 1 / sigma / sigma # |x| - 0.5 / sigma / sigma otherwise regression_diff = regression - regression_target regression_diff = keras.backend.abs(regression_diff) regression_loss = backend.where( keras.backend.less(regression_diff, 1.0 / sigma_squared), 0.5 * sigma_squared * keras.backend.pow(regression_diff, 2), regression_diff - 0.5 / sigma_squared ) # filter out "ignore" anchors indices = backend.where(keras.backend.equal(anchor_state, 1)) regression_loss = backend.gather_nd(regression_loss, indices) # compute the normalizer: the number of positive anchors normalizer = keras.backend.maximum(1, keras.backend.shape(indices)[0]) normalizer = keras.backend.cast(keras.backend.maximum(1, normalizer), dtype=keras.backend.floatx()) return keras.backend.sum(regression_loss) / normalizer
def _focal(y_true, y_pred): labels = y_true classification = y_pred # compute the divisor: for each image in the batch, we want the number of positive anchors # override the -1 labels, since we treat values -1 and 0 the same way for determining the divisor divisor = backend.where(keras.backend.less_equal(labels, 0), keras.backend.zeros_like(labels), labels) divisor = keras.backend.max(divisor, axis=2, keepdims=True) divisor = keras.backend.cast(divisor, keras.backend.floatx()) # compute the number of positive anchors divisor = keras.backend.sum(divisor, axis=1, keepdims=True) # ensure we do not divide by 0 divisor = keras.backend.maximum(1.0, divisor) # compute the focal loss alpha_factor = keras.backend.ones_like(labels) * alpha alpha_factor = backend.where(keras.backend.equal(labels, 1), alpha_factor, 1 - alpha_factor) focal_weight = backend.where(keras.backend.equal(labels, 1), 1 - classification, classification) focal_weight = alpha_factor * focal_weight**gamma cls_loss = focal_weight * keras.backend.binary_crossentropy( labels, classification) # normalise by the number of positive anchors for each entry in the minibatch cls_loss = cls_loss / divisor # filter out "ignore" anchors anchor_state = keras.backend.max( labels, axis=2) # -1 for ignore, 0 for background, 1 for object indices = backend.where(keras.backend.not_equal(anchor_state, -1)) cls_loss = backend.gather_nd(cls_loss, indices) # divide by the size of the minibatch return keras.backend.sum(cls_loss) / keras.backend.cast( keras.backend.shape(labels)[0], keras.backend.floatx())
def _focal(y_true, y_pred): """!@brief Compute the focal loss given the target tensor and the predicted tensor. As defined in https://arxiv.org/abs/1708.02002 @param y_true : Tensor of target data from the generator with shape (B, N, num_classes). @param y_pred : Tensor of predicted data from the network with shape (B, N, num_classes). @return The focal loss of y_pred w.r.t. y_true. """ labels = y_true[:, :, :-1] anchor_state = y_true[:, :, -1] # -1 for ignore, 0 for background, 1 for object classification = y_pred # filter out "ignore" anchors indices = backend.where(keras.backend.not_equal(anchor_state, -1)) labels = backend.gather_nd(labels, indices) classification = backend.gather_nd(classification, indices) # compute the focal loss alpha_factor = keras.backend.ones_like(labels) * alpha alpha_factor = backend.where(keras.backend.equal(labels, 1), alpha_factor, 1 - alpha_factor) focal_weight = backend.where(keras.backend.equal(labels, 1), 1 - classification, classification) focal_weight = alpha_factor * focal_weight**gamma cls_loss = focal_weight * keras.backend.binary_crossentropy( labels, classification) # compute the normalizer: the number of positive anchors normalizer = backend.where(keras.backend.equal(anchor_state, 1)) normalizer = keras.backend.cast( keras.backend.shape(normalizer)[0], keras.backend.floatx()) normalizer = keras.backend.maximum(keras.backend.cast_to_floatx(1.0), normalizer) return keras.backend.sum(cls_loss) / normalizer
def _smooth_l1(y_true, y_pred): # separate target and state regression = y_pred regression_target = y_true[:, :, :4] anchor_state = y_true[:, :, 4] # compute the divisor: for each image in the batch, we want the number of positive and negative anchors divisor = backend.where(keras.backend.equal(anchor_state, 1), keras.backend.ones_like(anchor_state), keras.backend.zeros_like(anchor_state)) divisor = keras.backend.sum(divisor, axis=1, keepdims=True) divisor = keras.backend.maximum(1.0, divisor) # pad the tensor to have shape (batch_size, 1, 1) for future division divisor = keras.backend.expand_dims(divisor, axis=2) # compute smooth L1 loss # f(x) = 0.5 * (sigma * x)^2 if |x| < 1 / sigma / sigma # |x| - 0.5 / sigma / sigma otherwise regression_diff = regression - regression_target regression_diff = keras.backend.abs(regression_diff) regression_loss = backend.where( keras.backend.less(regression_diff, 1.0 / sigma_squared), 0.5 * sigma_squared * keras.backend.pow(regression_diff, 2), regression_diff - 0.5 / sigma_squared) # normalise by the number of positive and negative anchors for each entry in the minibatch regression_loss = regression_loss / divisor # filter out "ignore" anchors indices = backend.where(keras.backend.equal(anchor_state, 1)) regression_loss = backend.gather_nd(regression_loss, indices) # divide by the size of the minibatch regression_loss = keras.backend.sum( regression_loss) / keras.backend.cast( keras.backend.shape(y_true)[0], keras.backend.floatx()) return regression_loss
def _focal(y_true, y_pred): labels = y_true classification = y_pred # compute the focal loss alpha_factor = keras.backend.ones_like(labels) * alpha alpha_factor = backend.where(keras.backend.equal(labels, 1), alpha_factor, 1 - alpha_factor) focal_weight = backend.where(keras.backend.equal(labels, 1), 1 - classification, classification) focal_weight = alpha_factor * focal_weight ** gamma cls_loss = focal_weight * keras.backend.binary_crossentropy(labels, classification) # filter out "ignore" anchors anchor_state = keras.backend.max(labels, axis=2) # -1 for ignore, 0 for background, 1 for object indices = backend.where(keras.backend.not_equal(anchor_state, -1)) cls_loss = backend.gather_nd(cls_loss, indices) # compute the normalizer: the number of positive anchors normalizer = backend.where(keras.backend.equal(anchor_state, 1)) normalizer = keras.backend.cast(keras.backend.shape(normalizer)[0], keras.backend.floatx()) normalizer = keras.backend.maximum(1.0, normalizer) return keras.backend.sum(cls_loss) / normalizer
def _filter_detections(scores, labels): # threshold based on score indices = backend.where(keras.backend.greater(scores, score_threshold)) if nms: filtered_boxes = backend.gather_nd(boxes, indices) filtered_scores = keras.backend.gather(scores, indices)[:, 0] # perform NMS nms_indices = backend.non_max_suppression( filtered_boxes, filtered_scores, max_output_size=max_detections, iou_threshold=nms_threshold) # filter indices based on NMS indices = keras.backend.gather(indices, nms_indices) # add indices to list of all indices labels = backend.gather_nd(labels, indices) indices = keras.backend.stack([indices[:, 0], labels], axis=1) return indices
def _focal(y_true, y_pred): """ Compute the focal loss given the target tensor and the predicted tensor. As defined in https://arxiv.org/abs/1708.02002 Args y_true: Tensor of target data from the generator with shape (B, N, num_classes). y_pred: Tensor of predicted data from the network with shape (B, N, num_classes). Returns The focal loss of y_pred w.r.t. y_true. """ labels = y_true[:, :, :-1] anchor_state = y_true[:, :, -1] # -1 for ignore, 0 for background, 1 for object classification = y_pred # filter out "ignore" anchors indices = backend.where(keras.backend.not_equal(anchor_state, -1)) labels = backend.gather_nd(labels, indices) classification = backend.gather_nd(classification, indices) if weights_list is not None: # adding my own weights if keras.backend.image_data_format() == 'channels_first': axis = 1 else: axis = -1 classSelectors = keras.backend.argmax(labels, axis=axis) classSelectors = [ keras.backend.equal(np.int64(i), classSelectors) for i in range(len(weights_list)) ] classSelectors = [ keras.backend.cast(x, keras.backend.floatx()) for x in classSelectors ] weights = [sel * w for sel, w in zip(classSelectors, weights_list)] weightMultiplier = weights[0] for i in range(1, len(weights)): weightMultiplier = weightMultiplier + weights[i] weightMultiplier = keras.backend.expand_dims(weightMultiplier, 1) weightMultiplier = keras.backend.tile(weightMultiplier, [1, 8]) """ weights_array = np.ones((8,8)) i = 0 for w in weights_list: weights_array[i,:] = w i += 1 nb_cl = len(weights_array) final_mask = keras.backend.zeros_like(classification[:, 0]) y_pred_max = keras.backend.max(classification, axis=1) y_pred_max = keras.backend.reshape(y_pred_max, (keras.backend.shape(classification)[0], 1)) #y_pred_max = keras.backend.expand_dims(y_pred_max, 1) y_pred_max_mat = keras.backend.equal(classification, y_pred_max) for c_p, c_t in product(range(nb_cl), range(nb_cl)): final_mask += (keras.backend.cast(weights_array[c_t, c_p],tf.float32) * keras.backend.cast(y_pred_max_mat[:, c_p] ,tf.float32)* keras.backend.cast(labels[:, c_t],tf.float32)) """ # add my own weights #if weight_list: #weight_factor = keras.backend.zeros_like(labels) # weight_factor = keras.backend.ones_like(labels) # for i in range(len(weight_list)): # weight_factor = backend.scatter_update(weight_factor[:,:]) # weight_np[:,:,i] = weight_list[i] # weight_factor = keras.backend.variable(weight_np) #else: # weight_factor = keras.backend.ones_like(labels) # compute the focal loss alpha_factor = keras.backend.ones_like(labels) * alpha alpha_factor = backend.where(keras.backend.equal(labels, 1), alpha_factor, 1 - alpha_factor) focal_weight = backend.where(keras.backend.equal(labels, 1), 1 - classification, classification) focal_weight = alpha_factor * focal_weight**gamma #cls_loss = weight_factor * focal_weight * keras.backend.binary_crossentropy(labels, classification) if weights_list is not None: cls_loss = focal_weight * keras.backend.binary_crossentropy( labels, classification) * weightMultiplier else: cls_loss = focal_weight * keras.backend.binary_crossentropy( labels, classification) # compute the normalizer: the number of positive anchors normalizer = backend.where(keras.backend.equal(anchor_state, 1)) normalizer = keras.backend.cast( keras.backend.shape(normalizer)[0], keras.backend.floatx()) normalizer = keras.backend.maximum(keras.backend.cast_to_floatx(1.0), normalizer) return c_weight * (keras.backend.sum(cls_loss) / normalizer)
def filter_detections(boxes, classification, other=[], nms=True, score_threshold=0.05, max_detections=300, nms_threshold=0.5): """ Filter detections using the boxes and classification values. Args boxes : Tensor of shape (num_boxes, 4) containing the boxes in (x1, y1, x2, y2) format. classification : Tensor of shape (num_boxes, num_classes) containing the classification scores. other : List of tensors of shape (num_boxes, ...) to filter along with the boxes and classification scores. nms : Flag to enable/disable non maximum suppression. score_threshold : Threshold used to prefilter the boxes with. max_detections : Maximum number of detections to keep. nms_threshold : Threshold for the IoU value to determine when a box should be suppressed. Returns A list of [boxes, scores, labels, other[0], other[1], ...]. boxes is shaped (max_detections, 4) and contains the (x1, y1, x2, y2) of the non-suppressed boxes. scores is shaped (max_detections,) and contains the scores of the predicted class. labels is shaped (max_detections,) and contains the predicted label. other[i] is shaped (max_detections, ...) and contains the filtered other[i] data. In case there are less than max_detections detections, the tensors are padded with -1's. """ all_boxes = [] all_scores = [] all_labels = [] all_other = [] # perform per class filtering for c in range(int(classification.shape[1])): scores = classification[:, c] # threshold based on score score_indices = backend.where( keras.backend.greater(scores, score_threshold)) score_indices = keras.backend.cast(score_indices, 'int32') filtered_boxes = backend.gather_nd(boxes, score_indices) filtered_scores = keras.backend.gather(scores, score_indices)[:, 0] filtered_other = [backend.gather_nd(o, score_indices) for o in other] if nms: # perform NMS nms_indices = backend.non_max_suppression( filtered_boxes, filtered_scores, max_output_size=max_detections, iou_threshold=nms_threshold) # filter NMS detections filtered_boxes = keras.backend.gather(filtered_boxes, nms_indices) filtered_scores = keras.backend.gather(filtered_scores, nms_indices) filtered_other = [ keras.backend.gather(o, nms_indices) for o in filtered_other ] # labels is a vector of the current class label filtered_labels = c * keras.backend.ones( (keras.backend.shape(filtered_scores)[0], ), dtype='int32') # append to lists all_boxes.append(filtered_boxes) all_scores.append(filtered_scores) all_labels.append(filtered_labels) all_other.append(filtered_other) # concatenate outputs to single tensors boxes = keras.backend.concatenate(all_boxes, axis=0) scores = keras.backend.concatenate(all_scores, axis=0) labels = keras.backend.concatenate(all_labels, axis=0) other_ = [ keras.backend.concatenate([o[i] for o in all_other], axis=0) for i in range(len(other)) ] # select top k scores, top_indices = backend.top_k(scores, k=keras.backend.minimum( max_detections, keras.backend.shape(scores)[0])) boxes = keras.backend.gather(boxes, top_indices) labels = keras.backend.gather(labels, top_indices) other_ = [keras.backend.gather(o, top_indices) for o in other_] # zero pad the outputs pad_size = keras.backend.maximum( 0, max_detections - keras.backend.shape(scores)[0]) boxes = backend.pad(boxes, [[0, pad_size], [0, 0]], constant_values=-1) scores = backend.pad(scores, [[0, pad_size]], constant_values=-1) labels = backend.pad(labels, [[0, pad_size]], constant_values=-1) labels = keras.backend.cast(labels, 'int32') other_ = [ backend.pad(o, [[0, pad_size]] + [[0, 0] for _ in range(1, len(o.shape))], constant_values=-1) for o in other_ ] # set shapes, since we know what they are boxes.set_shape([max_detections, 4]) scores.set_shape([max_detections]) labels.set_shape([max_detections]) for o, s in zip(other_, [list(keras.backend.int_shape(o)) for o in other]): o.set_shape([max_detections] + s[1:]) return [boxes, scores, labels] + other_