def yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False):
"""Convert final layer features to bounding box parameters."""
num_anchors = anchors_per_level
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2])
grid_shape = K.shape(feats)[1:3] # height, width
grid_y = K.tile(
tf.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1],
name='yolo_head/tile/reshape/grid_y'),
[1, grid_shape[1], 1, 1])
grid_x = K.tile(
tf.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1],
name='yolo_head/tile/reshape/grid_x'),
[grid_shape[0], 1, 1, 1])
grid = tf.concat([grid_x, grid_y],
axis=-1,
name='yolo_head/concatenate/grid')
grid = K.cast(grid, K.dtype(feats))
feats = tf.reshape(feats, [
-1, grid_shape[0], grid_shape[1], num_anchors,
num_classes + 5 + NUM_ANGLES3
],
name='yolo_head/reshape/feats')
# Adjust predictions to each spatial grid point and anchor size.
box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(
grid_shape[..., ::-1], K.dtype(feats))
box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(
input_shape[..., ::-1], K.dtype(feats))
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.sigmoid(feats[..., 5:5 + num_classes])
polygons_confidence = K.sigmoid(feats[..., 5 + num_classes + 2:5 +
num_classes + NUM_ANGLES3:3])
polygons_x = K.exp(feats[...,
5 + num_classes:num_classes + 5 + NUM_ANGLES3:3])
dx = K.square(anchors_tensor[..., 0:1] / 2)
dy = K.square(anchors_tensor[..., 1:2] / 2)
d = K.cast(K.sqrt(dx + dy), K.dtype(polygons_x))
a = K.pow(input_shape[..., ::-1], 2)
a = K.cast(a, K.dtype(feats))
b = K.sum(a)
diagonal = K.cast(K.sqrt(b), K.dtype(feats))
polygons_x = polygons_x * d / diagonal
polygons_y = feats[...,
5 + num_classes + 1:num_classes + 5 + NUM_ANGLES3:3]
polygons_y = K.sigmoid(polygons_y)
if calc_loss == True:
return grid, feats, box_xy, box_wh, polygons_confidence
return box_xy, box_wh, box_confidence, box_class_probs, polygons_x, polygons_y, polygons_confidence
def full_affinity(input_x, scale):
"""Calculates the symmetrized full Gaussian affinity matrix, scaled by a provided scale.
Args:
input_x: input dataset of size n x d
scale: provided scale
Returns:
n x n affinity matrix
"""
sigma = K.variable(scale)
dist_x = squared_distance(input_x)
sigma_squared = K.expand_dims(K.pow(sigma, 2), -1)
weight_mat = K.exp(-dist_x / (2 * sigma_squared))
return weight_mat
def __center_loss(y_true, y_pred, centers):
y_true_value = K.argmax(y_true)
loss = K.variable(0.0)
for label in range(CONFIG["num_classes"]):
center = centers[label]
indices = tf.where(tf.equal(y_true_value, label))
pred_per_class = tf.gather_nd(y_pred, indices=indices)
diff = tf.subtract(pred_per_class, center)
square_diff = K.pow(diff, 2)
sum = K.sum(K.sum(square_diff, axis=-1), axis=-1)
loss = tf.add(loss, sum)
return loss
def full_affinity(X, scale):
'''
Calculates the symmetrized full Gaussian affinity matrix, scaled
by a provided scale
X: input dataset of size n
scale: provided scale
returns: n x n affinity matrix
'''
sigma = K.variable(scale)
Dx = squared_distance(X)
sigma_squared = K.pow(sigma, 2)
sigma_squared = K.expand_dims(sigma_squared, -1)
Dx_scaled = Dx / (2 * sigma_squared)
W = K.exp(-Dx_scaled)
return W
def yolo_loss(args, anchors, num_classes, ignore_thresh=.5):
"""Return yolo_loss tensor
Parameters
----------
yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body
y_true: list of array, the output of preprocess_true_boxes
anchors: array, shape=(N, 2), wh
num_classes: integer
ignore_thresh: float, the iou threshold whether to ignore object confidence loss
Returns
-------
loss: tensor, shape=(1,)
"""
num_layers = 1
yolo_outputs = args[:num_layers]
y_true = args[num_layers:]
g_y_true = y_true
input_shape = K.cast(
K.shape(yolo_outputs[0])[1:3] * grid_size_multiplier,
K.dtype(y_true[0]))
grid_shapes = [
K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0]))
for l in range(num_layers)
]
loss = 0
m = K.shape(yolo_outputs[0])[0] # batch size, tensor
mf = K.cast(m, K.dtype(yolo_outputs[0]))
for layer in range(num_layers):
object_mask = y_true[layer][..., 4:5]
vertices_mask = y_true[layer][..., 5 + num_classes + 2:5 +
num_classes + NUM_ANGLES3:3]
true_class_probs = y_true[layer][..., 5:5 + num_classes]
grid, raw_pred, pred_xy, pred_wh, pol_cnf = yolo_head(
yolo_outputs[layer],
anchors[anchor_mask[layer]],
num_classes,
input_shape,
calc_loss=True)
pred_box = K.concatenate([pred_xy, pred_wh])
raw_true_xy = y_true[layer][..., :2] * grid_shapes[layer][
..., ::-1] - grid
raw_true_polygon0 = y_true[layer][..., 5 + num_classes:5 +
num_classes + NUM_ANGLES3]
raw_true_wh = K.log(y_true[layer][..., 2:4] /
anchors[anchor_mask[layer]] *
input_shape[..., ::-1])
raw_true_wh = K.switch(object_mask, raw_true_wh,
K.zeros_like(raw_true_wh)) # avoid log(0)=-inf
raw_true_polygon_x = raw_true_polygon0[..., ::3]
raw_true_polygon_y = raw_true_polygon0[..., 1::3]
dx = K.square(anchors[anchor_mask[layer]][..., 0:1] / 2)
dy = K.square(anchors[anchor_mask[layer]][..., 1:2] / 2)
d = K.cast(K.sqrt(dx + dy), K.dtype(raw_true_polygon_x))
diagonal = K.sqrt(
K.pow(input_shape[..., ::-1][0], 2) +
K.pow(input_shape[..., ::-1][1], 2))
raw_true_polygon_x = K.log(raw_true_polygon_x / d * diagonal)
raw_true_polygon_x = K.switch(vertices_mask, raw_true_polygon_x,
K.zeros_like(raw_true_polygon_x))
box_loss_scale = 2 - y_true[layer][..., 2:3] * y_true[layer][..., 3:4]
# Find ignore mask, iterate over each of batch.
ignore_mask = tf.TensorArray(K.dtype(y_true[0]),
size=1,
dynamic_size=True)
object_mask_bool = K.cast(object_mask, 'bool')
def loop_body(b, ignore_mask):
true_box = tf.boolean_mask(y_true[layer][b, ..., 0:4],
object_mask_bool[b, ..., 0])
iou = box_iou(pred_box[b], true_box)
best_iou = K.max(iou, axis=-1)
ignore_mask = ignore_mask.write(
b, K.cast(best_iou < ignore_thresh, K.dtype(true_box)))
return b + 1, ignore_mask
_, ignore_mask = tf.while_loop(lambda b, *args: b < m, loop_body,
[0, ignore_mask])
ignore_mask = ignore_mask.stack()
ignore_mask = K.expand_dims(ignore_mask, -1)
# K.binary_crossentropy is helpful to avoid exp overflow.
xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(
raw_true_xy, raw_pred[..., 0:2], from_logits=True)
wh_loss = object_mask * box_loss_scale * 0.5 * K.square(
raw_true_wh - raw_pred[..., 2:4])
confidence_loss = object_mask * K.binary_crossentropy(
object_mask, raw_pred[..., 4:5], from_logits=True
) + (1 - object_mask) * K.binary_crossentropy(
object_mask, raw_pred[..., 4:5], from_logits=True) * ignore_mask
class_loss = object_mask * K.binary_crossentropy(
true_class_probs,
raw_pred[..., 5:5 + num_classes],
from_logits=True)
polygon_loss_x = object_mask * vertices_mask * box_loss_scale * 0.5 * K.square(
raw_true_polygon_x -
raw_pred[..., 5 + num_classes:5 + num_classes + NUM_ANGLES3:3])
polygon_loss_y = object_mask * vertices_mask * box_loss_scale * K.binary_crossentropy(
raw_true_polygon_y,
raw_pred[..., 5 + num_classes + 1:5 + num_classes + NUM_ANGLES3:3],
from_logits=True)
vertices_confidence_loss = object_mask * K.binary_crossentropy(
vertices_mask,
raw_pred[..., 5 + num_classes + 2:5 + num_classes + NUM_ANGLES3:3],
from_logits=True)
xy_loss = K.sum(xy_loss) / mf
wh_loss = K.sum(wh_loss) / mf
class_loss = K.sum(class_loss) / mf
confidence_loss = K.sum(confidence_loss) / mf
vertices_confidence_loss = K.sum(vertices_confidence_loss) / mf
polygon_loss = K.sum(polygon_loss_x) / mf + K.sum(polygon_loss_y) / mf
diou_loss = K.sum(
object_mask * box_loss_scale *
(1 - box_diou(pred_box, y_true[layer][..., 0:4]))) / mf
loss += (xy_loss + wh_loss + confidence_loss + class_loss +
0.2 * polygon_loss + 0.2 * vertices_confidence_loss) / (
K.sum(object_mask) + 1) * mf
return loss
def __euclidean(x, y):
diff = tf.subtract(x, y)
square_diff = K.pow(diff, 2)
d = K.sqrt(K.sum(square_diff, axis=-1))
return d
