def yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False):
"""Convert final layer features to bounding box parameters."""
num_anchors = anchors_per_level
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2])
grid_shape = K.shape(feats)[1:3] # height, width
grid_y = K.tile(
tf.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1],
name='yolo_head/tile/reshape/grid_y'),
[1, grid_shape[1], 1, 1])
grid_x = K.tile(
tf.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1],
name='yolo_head/tile/reshape/grid_x'),
[grid_shape[0], 1, 1, 1])
grid = tf.concat([grid_x, grid_y],
axis=-1,
name='yolo_head/concatenate/grid')
grid = K.cast(grid, K.dtype(feats))
feats = tf.reshape(feats, [
-1, grid_shape[0], grid_shape[1], num_anchors,
num_classes + 5 + NUM_ANGLES3
],
name='yolo_head/reshape/feats')
# Adjust predictions to each spatial grid point and anchor size.
box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(
grid_shape[..., ::-1], K.dtype(feats))
box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(
input_shape[..., ::-1], K.dtype(feats))
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.sigmoid(feats[..., 5:5 + num_classes])
polygons_confidence = K.sigmoid(feats[..., 5 + num_classes + 2:5 +
num_classes + NUM_ANGLES3:3])
polygons_x = K.exp(feats[...,
5 + num_classes:num_classes + 5 + NUM_ANGLES3:3])
dx = K.square(anchors_tensor[..., 0:1] / 2)
dy = K.square(anchors_tensor[..., 1:2] / 2)
d = K.cast(K.sqrt(dx + dy), K.dtype(polygons_x))
a = K.pow(input_shape[..., ::-1], 2)
a = K.cast(a, K.dtype(feats))
b = K.sum(a)
diagonal = K.cast(K.sqrt(b), K.dtype(feats))
polygons_x = polygons_x * d / diagonal
polygons_y = feats[...,
5 + num_classes + 1:num_classes + 5 + NUM_ANGLES3:3]
polygons_y = K.sigmoid(polygons_y)
if calc_loss == True:
return grid, feats, box_xy, box_wh, polygons_confidence
return box_xy, box_wh, box_confidence, box_class_probs, polygons_x, polygons_y, polygons_confidence
def euclidean_distance(vects):
'''
Computes the euclidean distances between vects[0] and vects[1]
'''
x, y = vects
return K.sqrt(
K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))
def pearson_correlation_metric_fn(
y_true: tf.Tensor,
y_pred: tf.Tensor,
) -> tf.Tensor:
"""
Pearson correlation metric function.
https://github.com/WenYanger/Keras_Metrics
Args:
y_true (tf.Tensor): y_true
y_pred (tf.Tensor): y_pred
Returns:
tf.contrib.metrics: pearson correlation
"""
x = y_true
y = y_pred
mx = K.mean(x, axis=0)
my = K.mean(y, axis=0)
xm, ym = x - mx, y - my
r_num = K.sum(xm * ym)
x_square_sum = K.sum(xm * xm)
y_square_sum = K.sum(ym * ym)
r_den = K.sqrt(x_square_sum * y_square_sum) + 1e-12
r = r_num / r_den
return K.mean(r)
def call(self, QKVs):
"""Core logic of multi-head self attention.
Args:
QKVs (list): inputs of multi-head self attention i.e. query, key and value.
Returns:
object: ouput tensors.
"""
if len(QKVs) == 3:
Q_seq, K_seq, V_seq = QKVs
Q_len, V_len = None, None
elif len(QKVs) == 5:
Q_seq, K_seq, V_seq, Q_len, V_len = QKVs
Q_seq = K.dot(Q_seq, self.WQ)
Q_seq = K.reshape(Q_seq,
shape=(-1, K.shape(Q_seq)[1], self.multiheads,
self.head_dim))
Q_seq = K.permute_dimensions(Q_seq, pattern=(0, 2, 1, 3))
K_seq = K.dot(K_seq, self.WK)
K_seq = K.reshape(K_seq,
shape=(-1, K.shape(K_seq)[1], self.multiheads,
self.head_dim))
K_seq = K.permute_dimensions(K_seq, pattern=(0, 2, 1, 3))
V_seq = K.dot(V_seq, self.WV)
V_seq = K.reshape(V_seq,
shape=(-1, K.shape(V_seq)[1], self.multiheads,
self.head_dim))
V_seq = K.permute_dimensions(V_seq, pattern=(0, 2, 1, 3))
A = einsum("abij, abkj -> abik", Q_seq, K_seq) / K.sqrt(
K.cast(self.head_dim, dtype="float32"))
A = K.permute_dimensions(
A, pattern=(0, 3, 2, 1)
) # A.shape=[batch_size,K_sequence_length,Q_sequence_length,self.multiheads]
A = self.Mask(A, V_len, "add")
A = K.permute_dimensions(A, pattern=(0, 3, 2, 1))
if self.mask_right:
ones = K.ones_like(A[:1, :1])
lower_triangular = K.tf.matrix_band_part(ones,
num_lower=-1,
num_upper=0)
mask = (ones - lower_triangular) * 1e12
A = A - mask
A = K.softmax(A)
O_seq = einsum("abij, abjk -> abik", A, V_seq)
O_seq = K.permute_dimensions(O_seq, pattern=(0, 2, 1, 3))
O_seq = K.reshape(O_seq,
shape=(-1, K.shape(O_seq)[1], self.output_dim))
O_seq = self.Mask(O_seq, Q_len, "mul")
return O_seq
def get_log_probability_density(pred, y):
mu_and_sigma = pred
mu = mu_and_sigma[:, :2]
sigma = mu_and_sigma[:, 2:]
variance = K.square(sigma)
pdf = 1. / K.sqrt(2. * np.pi * variance) * K.exp(-K.square(y - mu) /
(2. * variance))
log_pdf = K.log(pdf + K.epsilon())
return log_pdf
def squared_distance(input_x, input_y=None, weight=None):
"""Calculates the pairwise distance between points in X and Y.
Args:
input_x: n x d matrix
input_y: m x d matrix
weight: affinity n x m -- if provided, we normalize the distance
Returns:
n x m matrix of all pairwise squared Euclidean distances
"""
if input_y is None:
input_y = input_x
sum_dimensions = list(range(2, K.ndim(input_x) + 1))
input_x = K.expand_dims(input_x, axis=1)
if weight is not None:
# if weight provided, we normalize input_x and input_y by weight
d_diag = K.expand_dims(K.sqrt(K.sum(weight, axis=1)), axis=1)
input_x /= d_diag
input_y /= d_diag
squared_difference = K.square(input_x - input_y)
distance = K.sum(squared_difference, axis=sum_dimensions)
return distance
def squared_distance(X, Y=None, W=None):
'''
Calculates the pairwise distance between points in X and Y
X: n x d matrix
Y: m x d matrix
W: affinity -- if provided, we normalize the distance
returns: n x m matrix of all pairwise squared Euclidean distances
'''
if Y is None:
Y = X
# distance = squaredDistance(X, Y)
sum_dimensions = list(range(2, K.ndim(X) + 1))
X = K.expand_dims(X, axis=1)
if W is not None:
# if W provided, we normalize X and Y by W
D_diag = K.expand_dims(K.sqrt(K.sum(W, axis=1)), axis=1)
X /= D_diag
Y /= D_diag
squared_difference = K.square(X - Y)
distance = K.sum(squared_difference, axis=sum_dimensions)
return distance
def yolo_loss(args, anchors, num_classes, ignore_thresh=.5):
"""Return yolo_loss tensor
Parameters
----------
yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body
y_true: list of array, the output of preprocess_true_boxes
anchors: array, shape=(N, 2), wh
num_classes: integer
ignore_thresh: float, the iou threshold whether to ignore object confidence loss
Returns
-------
loss: tensor, shape=(1,)
"""
num_layers = 1
yolo_outputs = args[:num_layers]
y_true = args[num_layers:]
g_y_true = y_true
input_shape = K.cast(
K.shape(yolo_outputs[0])[1:3] * grid_size_multiplier,
K.dtype(y_true[0]))
grid_shapes = [
K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0]))
for l in range(num_layers)
]
loss = 0
m = K.shape(yolo_outputs[0])[0] # batch size, tensor
mf = K.cast(m, K.dtype(yolo_outputs[0]))
for layer in range(num_layers):
object_mask = y_true[layer][..., 4:5]
vertices_mask = y_true[layer][..., 5 + num_classes + 2:5 +
num_classes + NUM_ANGLES3:3]
true_class_probs = y_true[layer][..., 5:5 + num_classes]
grid, raw_pred, pred_xy, pred_wh, pol_cnf = yolo_head(
yolo_outputs[layer],
anchors[anchor_mask[layer]],
num_classes,
input_shape,
calc_loss=True)
pred_box = K.concatenate([pred_xy, pred_wh])
raw_true_xy = y_true[layer][..., :2] * grid_shapes[layer][
..., ::-1] - grid
raw_true_polygon0 = y_true[layer][..., 5 + num_classes:5 +
num_classes + NUM_ANGLES3]
raw_true_wh = K.log(y_true[layer][..., 2:4] /
anchors[anchor_mask[layer]] *
input_shape[..., ::-1])
raw_true_wh = K.switch(object_mask, raw_true_wh,
K.zeros_like(raw_true_wh)) # avoid log(0)=-inf
raw_true_polygon_x = raw_true_polygon0[..., ::3]
raw_true_polygon_y = raw_true_polygon0[..., 1::3]
dx = K.square(anchors[anchor_mask[layer]][..., 0:1] / 2)
dy = K.square(anchors[anchor_mask[layer]][..., 1:2] / 2)
d = K.cast(K.sqrt(dx + dy), K.dtype(raw_true_polygon_x))
diagonal = K.sqrt(
K.pow(input_shape[..., ::-1][0], 2) +
K.pow(input_shape[..., ::-1][1], 2))
raw_true_polygon_x = K.log(raw_true_polygon_x / d * diagonal)
raw_true_polygon_x = K.switch(vertices_mask, raw_true_polygon_x,
K.zeros_like(raw_true_polygon_x))
box_loss_scale = 2 - y_true[layer][..., 2:3] * y_true[layer][..., 3:4]
# Find ignore mask, iterate over each of batch.
ignore_mask = tf.TensorArray(K.dtype(y_true[0]),
size=1,
dynamic_size=True)
object_mask_bool = K.cast(object_mask, 'bool')
def loop_body(b, ignore_mask):
true_box = tf.boolean_mask(y_true[layer][b, ..., 0:4],
object_mask_bool[b, ..., 0])
iou = box_iou(pred_box[b], true_box)
best_iou = K.max(iou, axis=-1)
ignore_mask = ignore_mask.write(
b, K.cast(best_iou < ignore_thresh, K.dtype(true_box)))
return b + 1, ignore_mask
_, ignore_mask = tf.while_loop(lambda b, *args: b < m, loop_body,
[0, ignore_mask])
ignore_mask = ignore_mask.stack()
ignore_mask = K.expand_dims(ignore_mask, -1)
# K.binary_crossentropy is helpful to avoid exp overflow.
xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(
raw_true_xy, raw_pred[..., 0:2], from_logits=True)
wh_loss = object_mask * box_loss_scale * 0.5 * K.square(
raw_true_wh - raw_pred[..., 2:4])
confidence_loss = object_mask * K.binary_crossentropy(
object_mask, raw_pred[..., 4:5], from_logits=True
) + (1 - object_mask) * K.binary_crossentropy(
object_mask, raw_pred[..., 4:5], from_logits=True) * ignore_mask
class_loss = object_mask * K.binary_crossentropy(
true_class_probs,
raw_pred[..., 5:5 + num_classes],
from_logits=True)
polygon_loss_x = object_mask * vertices_mask * box_loss_scale * 0.5 * K.square(
raw_true_polygon_x -
raw_pred[..., 5 + num_classes:5 + num_classes + NUM_ANGLES3:3])
polygon_loss_y = object_mask * vertices_mask * box_loss_scale * K.binary_crossentropy(
raw_true_polygon_y,
raw_pred[..., 5 + num_classes + 1:5 + num_classes + NUM_ANGLES3:3],
from_logits=True)
vertices_confidence_loss = object_mask * K.binary_crossentropy(
vertices_mask,
raw_pred[..., 5 + num_classes + 2:5 + num_classes + NUM_ANGLES3:3],
from_logits=True)
xy_loss = K.sum(xy_loss) / mf
wh_loss = K.sum(wh_loss) / mf
class_loss = K.sum(class_loss) / mf
confidence_loss = K.sum(confidence_loss) / mf
vertices_confidence_loss = K.sum(vertices_confidence_loss) / mf
polygon_loss = K.sum(polygon_loss_x) / mf + K.sum(polygon_loss_y) / mf
diou_loss = K.sum(
object_mask * box_loss_scale *
(1 - box_diou(pred_box, y_true[layer][..., 0:4]))) / mf
loss += (xy_loss + wh_loss + confidence_loss + class_loss +
0.2 * polygon_loss + 0.2 * vertices_confidence_loss) / (
K.sum(object_mask) + 1) * mf
return loss
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)))
def __euclidean(x, y):
diff = tf.subtract(x, y)
square_diff = K.pow(diff, 2)
d = K.sqrt(K.sum(square_diff, axis=-1))
return d
