def _negative_weighted_ce_loss(y_true, y_pred):
""" Compute the focal loss given the target tensor and the predicted tensor.
As defined in https://arxiv.org/abs/1708.02002
Args
y_true: Tensor of target data from the generator with shape (B, N, num_classes).
y_pred: Tensor of predicted data from the network with shape (B, N, num_classes).
Returns
The focal loss of y_pred w.r.t. y_true.
"""
labels = y_true[:, :, :-1]
anchor_state = y_true[:, :, -1] # -1 for ignore, 0 for background, 1 for object
classification = y_pred
# filter out "ignore" anchors
indices = backend.where(keras.backend.not_equal(anchor_state, -1))
labels = backend.gather_nd(labels, indices)
classification = backend.gather_nd(classification, indices)
cls_loss = binary_crossentropy(labels, classification)
# compute the normalizer: the number of positive anchors
normalizer = backend.where(keras.backend.equal(anchor_state, 1))
normalizer = keras.backend.cast(keras.backend.shape(normalizer)[0], keras.backend.floatx())
normalizer = keras.backend.maximum(keras.backend.cast_to_floatx(1.0), normalizer)
return keras.backend.sum(cls_loss) / normalizer
def _smooth_l1(y_true, y_pred):
""" Compute the smooth L1 loss of y_pred w.r.t. y_true.
Args
y_true: Tensor from the generator of shape (B, N, 5). The last value for each box is the state of the anchor (ignore, negative, positive).
y_pred: Tensor from the network of shape (B, N, 4).
Returns
The smooth L1 loss of y_pred w.r.t. y_true.
"""
# separate target and state
regression = y_pred
regression_target = y_true[:, :, :-1]
anchor_state = y_true[:, :, -1]
# filter out "ignore" anchors
indices = backend.where(keras.backend.equal(anchor_state, 1))
regression = backend.gather_nd(regression, indices)
regression_target = backend.gather_nd(regression_target, indices)
# compute smooth L1 loss
# f(x) = 0.5 * (sigma * x)^2 if |x| < 1 / sigma / sigma
# |x| - 0.5 / sigma / sigma otherwise
regression_diff = regression - regression_target
regression_diff = keras.backend.abs(regression_diff)
regression_loss = backend.where(
keras.backend.less(regression_diff, 1.0 / sigma_squared),
0.5 * sigma_squared * keras.backend.pow(regression_diff, 2),
regression_diff - 0.5 / sigma_squared)
# compute the normalizer: the number of positive anchors
normalizer = keras.backend.maximum(1, keras.backend.shape(indices)[0])
normalizer = keras.backend.cast(normalizer,
dtype=keras.backend.floatx())
return r_weight * (keras.backend.sum(regression_loss) / normalizer)
def _smooth_l1(y_true, y_pred):
# separate target and state
regression = y_pred
regression_target = y_true[:, :, :4]
anchor_state = y_true[:, :, 4]
# compute smooth L1 loss
# f(x) = 0.5 * (sigma * x)^2 if |x| < 1 / sigma / sigma
# |x| - 0.5 / sigma / sigma otherwise
regression_diff = regression - regression_target
regression_diff = keras.backend.abs(regression_diff)
regression_loss = backend.where(
keras.backend.less(regression_diff, 1.0 / sigma_squared),
0.5 * sigma_squared * keras.backend.pow(regression_diff, 2),
regression_diff - 0.5 / sigma_squared
)
# filter out "ignore" anchors
indices = backend.where(keras.backend.equal(anchor_state, 1))
regression_loss = backend.gather_nd(regression_loss, indices)
# compute the normalizer: the number of positive anchors
normalizer = keras.backend.maximum(1, keras.backend.shape(indices)[0])
normalizer = keras.backend.cast(keras.backend.maximum(1, normalizer), dtype=keras.backend.floatx())
return keras.backend.sum(regression_loss) / normalizer
def _focal(y_true, y_pred):
"""!@brief
Compute the focal loss given the target tensor and the predicted tensor.
As defined in https://arxiv.org/abs/1708.02002
@param y_true : Tensor of target data from the generator with shape (B, N, num_classes).
@param y_pred : Tensor of predicted data from the network with shape (B, N, num_classes).
@return
The focal loss of y_pred w.r.t. y_true.
"""
labels = y_true[:, :, :-1]
anchor_state = y_true[:, :,
-1] # -1 for ignore, 0 for background, 1 for object
classification = y_pred
# filter out "ignore" anchors
indices = backend.where(keras.backend.not_equal(anchor_state, -1))
labels = backend.gather_nd(labels, indices)
classification = backend.gather_nd(classification, indices)
# compute the focal loss
alpha_factor = keras.backend.ones_like(labels) * alpha
alpha_factor = backend.where(keras.backend.equal(labels, 1),
alpha_factor, 1 - alpha_factor)
focal_weight = backend.where(keras.backend.equal(labels, 1),
1 - classification, classification)
focal_weight = alpha_factor * focal_weight**gamma
cls_loss = focal_weight * keras.backend.binary_crossentropy(
labels, classification)
# compute the normalizer: the number of positive anchors
normalizer = backend.where(keras.backend.equal(anchor_state, 1))
normalizer = keras.backend.cast(
keras.backend.shape(normalizer)[0], keras.backend.floatx())
normalizer = keras.backend.maximum(keras.backend.cast_to_floatx(1.0),
normalizer)
return keras.backend.sum(cls_loss) / normalizer
def _filter_detections(scores, labels):
# threshold based on score
indices = backend.where(keras.backend.greater(scores, score_threshold))
if nms:
filtered_boxes = backend.gather_nd(boxes, indices)
filtered_scores = keras.backend.gather(scores, indices)[:, 0]
# perform NMS
nms_indices = backend.non_max_suppression(
filtered_boxes,
filtered_scores,
max_output_size=max_detections,
iou_threshold=nms_threshold)
# filter indices based on NMS
indices = keras.backend.gather(indices, nms_indices)
# add indices to list of all indices
labels = backend.gather_nd(labels, indices)
indices = keras.backend.stack([indices[:, 0], labels], axis=1)
return indices
def _focal(y_true, y_pred):
labels = y_true
classification = y_pred
# compute the divisor: for each image in the batch, we want the number of positive anchors
# override the -1 labels, since we treat values -1 and 0 the same way for determining the divisor
divisor = backend.where(keras.backend.less_equal(labels, 0),
keras.backend.zeros_like(labels), labels)
divisor = keras.backend.max(divisor, axis=2, keepdims=True)
divisor = keras.backend.cast(divisor, keras.backend.floatx())
# compute the number of positive anchors
divisor = keras.backend.sum(divisor, axis=1, keepdims=True)
# ensure we do not divide by 0
divisor = keras.backend.maximum(1.0, divisor)
# compute the focal loss
alpha_factor = keras.backend.ones_like(labels) * alpha
alpha_factor = backend.where(keras.backend.equal(labels, 1),
alpha_factor, 1 - alpha_factor)
focal_weight = backend.where(keras.backend.equal(labels, 1),
1 - classification, classification)
focal_weight = alpha_factor * focal_weight**gamma
cls_loss = focal_weight * keras.backend.binary_crossentropy(
labels, classification)
# normalise by the number of positive anchors for each entry in the minibatch
cls_loss = cls_loss / divisor
# filter out "ignore" anchors
anchor_state = keras.backend.max(
labels, axis=2) # -1 for ignore, 0 for background, 1 for object
indices = backend.where(keras.backend.not_equal(anchor_state, -1))
cls_loss = backend.gather_nd(cls_loss, indices)
# divide by the size of the minibatch
return keras.backend.sum(cls_loss) / keras.backend.cast(
keras.backend.shape(labels)[0], keras.backend.floatx())
def _smooth_l1(y_true, y_pred):
# separate target and state
regression = y_pred
regression_target = y_true[:, :, :4]
anchor_state = y_true[:, :, 4]
# compute the divisor: for each image in the batch, we want the number of positive and negative anchors
divisor = backend.where(keras.backend.equal(anchor_state, 1),
keras.backend.ones_like(anchor_state),
keras.backend.zeros_like(anchor_state))
divisor = keras.backend.sum(divisor, axis=1, keepdims=True)
divisor = keras.backend.maximum(1.0, divisor)
# pad the tensor to have shape (batch_size, 1, 1) for future division
divisor = keras.backend.expand_dims(divisor, axis=2)
# compute smooth L1 loss
# f(x) = 0.5 * (sigma * x)^2 if |x| < 1 / sigma / sigma
# |x| - 0.5 / sigma / sigma otherwise
regression_diff = regression - regression_target
regression_diff = keras.backend.abs(regression_diff)
regression_loss = backend.where(
keras.backend.less(regression_diff, 1.0 / sigma_squared),
0.5 * sigma_squared * keras.backend.pow(regression_diff, 2),
regression_diff - 0.5 / sigma_squared)
# normalise by the number of positive and negative anchors for each entry in the minibatch
regression_loss = regression_loss / divisor
# filter out "ignore" anchors
indices = backend.where(keras.backend.equal(anchor_state, 1))
regression_loss = backend.gather_nd(regression_loss, indices)
# divide by the size of the minibatch
regression_loss = keras.backend.sum(
regression_loss) / keras.backend.cast(
keras.backend.shape(y_true)[0], keras.backend.floatx())
return regression_loss
def _focal(y_true, y_pred):
labels = y_true
classification = y_pred
# compute the focal loss
alpha_factor = keras.backend.ones_like(labels) * alpha
alpha_factor = backend.where(keras.backend.equal(labels, 1), alpha_factor, 1 - alpha_factor)
focal_weight = backend.where(keras.backend.equal(labels, 1), 1 - classification, classification)
focal_weight = alpha_factor * focal_weight ** gamma
cls_loss = focal_weight * keras.backend.binary_crossentropy(labels, classification)
# filter out "ignore" anchors
anchor_state = keras.backend.max(labels, axis=2) # -1 for ignore, 0 for background, 1 for object
indices = backend.where(keras.backend.not_equal(anchor_state, -1))
cls_loss = backend.gather_nd(cls_loss, indices)
# compute the normalizer: the number of positive anchors
normalizer = backend.where(keras.backend.equal(anchor_state, 1))
normalizer = keras.backend.cast(keras.backend.shape(normalizer)[0], keras.backend.floatx())
normalizer = keras.backend.maximum(1.0, normalizer)
return keras.backend.sum(cls_loss) / normalizer
def filter_detections(boxes,
classification,
other=[],
class_specific_filter=True,
nms=True,
score_threshold=0.05,
max_detections=300,
nms_threshold=0.5):
""" Filter detections using the boxes and classification values.
Args
boxes : Tensor of shape (num_boxes, 4) containing the boxes in (x1, y1, x2, y2) format.
classification : Tensor of shape (num_boxes, num_classes) containing the classification scores.
other : List of tensors of shape (num_boxes, ...) to filter along with the boxes and classification scores.
class_specific_filter : Whether to perform filtering per class, or take the best scoring class and filter those.
nms : Flag to enable/disable non maximum suppression.
score_threshold : Threshold used to prefilter the boxes with.
max_detections : Maximum number of detections to keep.
nms_threshold : Threshold for the IoU value to determine when a box should be suppressed.
Returns
A list of [boxes, scores, labels, other[0], other[1], ...].
boxes is shaped (max_detections, 4) and contains the (x1, y1, x2, y2) of the non-suppressed boxes.
scores is shaped (max_detections,) and contains the scores of the predicted class.
labels is shaped (max_detections,) and contains the predicted label.
other[i] is shaped (max_detections, ...) and contains the filtered other[i] data.
In case there are less than max_detections detections, the tensors are padded with -1's.
"""
def _filter_detections(scores, labels):
# threshold based on score
indices = backend.where(keras.backend.greater(scores, score_threshold))
if nms:
filtered_boxes = backend.gather_nd(boxes, indices)
filtered_scores = keras.backend.gather(scores, indices)[:, 0]
# perform NMS
nms_indices = backend.non_max_suppression(
filtered_boxes,
filtered_scores,
max_output_size=max_detections,
iou_threshold=nms_threshold)
# filter indices based on NMS
indices = keras.backend.gather(indices, nms_indices)
# add indices to list of all indices
labels = backend.gather_nd(labels, indices)
indices = keras.backend.stack([indices[:, 0], labels], axis=1)
return indices
if class_specific_filter:
all_indices = []
# perform per class filtering
for c in range(int(classification.shape[1])):
scores = classification[:, c]
labels = c * backend.ones(
(keras.backend.shape(scores)[0], ), dtype='int64')
all_indices.append(_filter_detections(scores, labels))
# concatenate indices to single tensor
indices = keras.backend.concatenate(all_indices, axis=0)
else:
scores = keras.backend.max(classification, axis=1)
labels = keras.backend.argmax(classification, axis=1)
indices = _filter_detections(scores, labels)
# select top k
scores = backend.gather_nd(classification, indices)
labels = indices[:, 1]
scores, top_indices = backend.top_k(scores,
k=keras.backend.minimum(
max_detections,
keras.backend.shape(scores)[0]))
# filter input using the final set of indices
indices = keras.backend.gather(indices[:, 0], top_indices)
boxes = keras.backend.gather(boxes, indices)
labels = keras.backend.gather(labels, top_indices)
other_ = [keras.backend.gather(o, indices) for o in other]
# zero pad the outputs
pad_size = keras.backend.maximum(
0, max_detections - keras.backend.shape(scores)[0])
boxes = backend.pad(boxes, [[0, pad_size], [0, 0]], constant_values=-1)
scores = backend.pad(scores, [[0, pad_size]], constant_values=-1)
labels = backend.pad(labels, [[0, pad_size]], constant_values=-1)
labels = keras.backend.cast(labels, 'int32')
other_ = [
backend.pad(o, [[0, pad_size]] + [[0, 0]
for _ in range(1, len(o.shape))],
constant_values=-1) for o in other_
]
# set shapes, since we know what they are
boxes.set_shape([max_detections, 4])
scores.set_shape([max_detections])
labels.set_shape([max_detections])
for o, s in zip(other_, [list(keras.backend.int_shape(o)) for o in other]):
o.set_shape([max_detections] + s[1:])
return [boxes, scores, labels] + other_
def _focal(y_true, y_pred):
""" Compute the focal loss given the target tensor and the predicted tensor.
As defined in https://arxiv.org/abs/1708.02002
Args
y_true: Tensor of target data from the generator with shape (B, N, num_classes).
y_pred: Tensor of predicted data from the network with shape (B, N, num_classes).
Returns
The focal loss of y_pred w.r.t. y_true.
"""
labels = y_true[:, :, :-1]
anchor_state = y_true[:, :,
-1] # -1 for ignore, 0 for background, 1 for object
classification = y_pred
# filter out "ignore" anchors
indices = backend.where(keras.backend.not_equal(anchor_state, -1))
labels = backend.gather_nd(labels, indices)
classification = backend.gather_nd(classification, indices)
if weights_list is not None:
# adding my own weights
if keras.backend.image_data_format() == 'channels_first':
axis = 1
else:
axis = -1
classSelectors = keras.backend.argmax(labels, axis=axis)
classSelectors = [
keras.backend.equal(np.int64(i), classSelectors)
for i in range(len(weights_list))
]
classSelectors = [
keras.backend.cast(x, keras.backend.floatx())
for x in classSelectors
]
weights = [sel * w for sel, w in zip(classSelectors, weights_list)]
weightMultiplier = weights[0]
for i in range(1, len(weights)):
weightMultiplier = weightMultiplier + weights[i]
weightMultiplier = keras.backend.expand_dims(weightMultiplier, 1)
weightMultiplier = keras.backend.tile(weightMultiplier, [1, 8])
"""
weights_array = np.ones((8,8))
i = 0
for w in weights_list:
weights_array[i,:] = w
i += 1
nb_cl = len(weights_array)
final_mask = keras.backend.zeros_like(classification[:, 0])
y_pred_max = keras.backend.max(classification, axis=1)
y_pred_max = keras.backend.reshape(y_pred_max, (keras.backend.shape(classification)[0], 1))
#y_pred_max = keras.backend.expand_dims(y_pred_max, 1)
y_pred_max_mat = keras.backend.equal(classification, y_pred_max)
for c_p, c_t in product(range(nb_cl), range(nb_cl)):
final_mask += (keras.backend.cast(weights_array[c_t, c_p],tf.float32) * keras.backend.cast(y_pred_max_mat[:, c_p] ,tf.float32)* keras.backend.cast(labels[:, c_t],tf.float32))
"""
# add my own weights
#if weight_list:
#weight_factor = keras.backend.zeros_like(labels)
# weight_factor = keras.backend.ones_like(labels)
# for i in range(len(weight_list)):
# weight_factor = backend.scatter_update(weight_factor[:,:])
# weight_np[:,:,i] = weight_list[i]
# weight_factor = keras.backend.variable(weight_np)
#else:
# weight_factor = keras.backend.ones_like(labels)
# compute the focal loss
alpha_factor = keras.backend.ones_like(labels) * alpha
alpha_factor = backend.where(keras.backend.equal(labels, 1),
alpha_factor, 1 - alpha_factor)
focal_weight = backend.where(keras.backend.equal(labels, 1),
1 - classification, classification)
focal_weight = alpha_factor * focal_weight**gamma
#cls_loss = weight_factor * focal_weight * keras.backend.binary_crossentropy(labels, classification)
if weights_list is not None:
cls_loss = focal_weight * keras.backend.binary_crossentropy(
labels, classification) * weightMultiplier
else:
cls_loss = focal_weight * keras.backend.binary_crossentropy(
labels, classification)
# compute the normalizer: the number of positive anchors
normalizer = backend.where(keras.backend.equal(anchor_state, 1))
normalizer = keras.backend.cast(
keras.backend.shape(normalizer)[0], keras.backend.floatx())
normalizer = keras.backend.maximum(keras.backend.cast_to_floatx(1.0),
normalizer)
return c_weight * (keras.backend.sum(cls_loss) / normalizer)
def filter_detections(boxes,
classification,
other=[],
nms=True,
score_threshold=0.05,
max_detections=300,
nms_threshold=0.5):
""" Filter detections using the boxes and classification values.
Args
boxes : Tensor of shape (num_boxes, 4) containing the boxes in (x1, y1, x2, y2) format.
classification : Tensor of shape (num_boxes, num_classes) containing the classification scores.
other : List of tensors of shape (num_boxes, ...) to filter along with the boxes and classification scores.
nms : Flag to enable/disable non maximum suppression.
score_threshold : Threshold used to prefilter the boxes with.
max_detections : Maximum number of detections to keep.
nms_threshold : Threshold for the IoU value to determine when a box should be suppressed.
Returns
A list of [boxes, scores, labels, other[0], other[1], ...].
boxes is shaped (max_detections, 4) and contains the (x1, y1, x2, y2) of the non-suppressed boxes.
scores is shaped (max_detections,) and contains the scores of the predicted class.
labels is shaped (max_detections,) and contains the predicted label.
other[i] is shaped (max_detections, ...) and contains the filtered other[i] data.
In case there are less than max_detections detections, the tensors are padded with -1's.
"""
all_boxes = []
all_scores = []
all_labels = []
all_other = []
# perform per class filtering
for c in range(int(classification.shape[1])):
scores = classification[:, c]
# threshold based on score
score_indices = backend.where(
keras.backend.greater(scores, score_threshold))
score_indices = keras.backend.cast(score_indices, 'int32')
filtered_boxes = backend.gather_nd(boxes, score_indices)
filtered_scores = keras.backend.gather(scores, score_indices)[:, 0]
filtered_other = [backend.gather_nd(o, score_indices) for o in other]
if nms:
# perform NMS
nms_indices = backend.non_max_suppression(
filtered_boxes,
filtered_scores,
max_output_size=max_detections,
iou_threshold=nms_threshold)
# filter NMS detections
filtered_boxes = keras.backend.gather(filtered_boxes, nms_indices)
filtered_scores = keras.backend.gather(filtered_scores,
nms_indices)
filtered_other = [
keras.backend.gather(o, nms_indices) for o in filtered_other
]
# labels is a vector of the current class label
filtered_labels = c * keras.backend.ones(
(keras.backend.shape(filtered_scores)[0], ), dtype='int32')
# append to lists
all_boxes.append(filtered_boxes)
all_scores.append(filtered_scores)
all_labels.append(filtered_labels)
all_other.append(filtered_other)
# concatenate outputs to single tensors
boxes = keras.backend.concatenate(all_boxes, axis=0)
scores = keras.backend.concatenate(all_scores, axis=0)
labels = keras.backend.concatenate(all_labels, axis=0)
other_ = [
keras.backend.concatenate([o[i] for o in all_other], axis=0)
for i in range(len(other))
]
# select top k
scores, top_indices = backend.top_k(scores,
k=keras.backend.minimum(
max_detections,
keras.backend.shape(scores)[0]))
boxes = keras.backend.gather(boxes, top_indices)
labels = keras.backend.gather(labels, top_indices)
other_ = [keras.backend.gather(o, top_indices) for o in other_]
# zero pad the outputs
pad_size = keras.backend.maximum(
0, max_detections - keras.backend.shape(scores)[0])
boxes = backend.pad(boxes, [[0, pad_size], [0, 0]], constant_values=-1)
scores = backend.pad(scores, [[0, pad_size]], constant_values=-1)
labels = backend.pad(labels, [[0, pad_size]], constant_values=-1)
labels = keras.backend.cast(labels, 'int32')
other_ = [
backend.pad(o, [[0, pad_size]] + [[0, 0]
for _ in range(1, len(o.shape))],
constant_values=-1) for o in other_
]
# set shapes, since we know what they are
boxes.set_shape([max_detections, 4])
scores.set_shape([max_detections])
labels.set_shape([max_detections])
for o, s in zip(other_, [list(keras.backend.int_shape(o)) for o in other]):
o.set_shape([max_detections] + s[1:])
return [boxes, scores, labels] + other_
