def build(self, input_shape=None):
self.input_spec = InputSpec(shape=input_shape)
if not self.layer.built:
self.layer.build(input_shape)
self.layer.built = True
super(ConcreteDropout, self).build(
) # this is very weird.. we must call super before we add new losses
# initialise p
self.p_logit = self.layer.add_weight(name='p_logit',
shape=(1, ),
initializer=RandomUniform(
self.init_min, self.init_max),
trainable=True)
self.p = K.sigmoid(self.p_logit[0])
# initialise regulariser / prior KL term
input_dim = np.prod(input_shape[1:]) # we drop only last dim
weight = self.layer.kernel
kernel_regularizer = self.weight_regularizer * K.sum(
K.square(weight)) / (1. - self.p)
dropout_regularizer = self.p * K.log(self.p)
dropout_regularizer += (1. - self.p) * K.log(1. - self.p)
dropout_regularizer *= self.dropout_regularizer * input_dim
regularizer = K.sum(kernel_regularizer + dropout_regularizer)
self.layer.add_loss(regularizer)
def loss(y_true, y_pred):
# scale predictions so that the class probas of each sample sum to 1
y_pred /= K.sum(y_pred, axis=-1, keepdims=True)
# clip to prevent NaN's and Inf's
y_pred = K.clip(y_pred, K.epsilon(), 1 - K.epsilon())
# calc
loss = y_true * K.log(y_pred) * weights
loss = -K.sum(loss, -1)
return loss
def concrete_dropout(self, x):
'''
Concrete dropout - used at training time (gradients can be propagated)
:param x: input
:return: approx. dropped out input
'''
eps = K.cast_to_floatx(K.epsilon())
temp = 0.1
unif_noise = K.random_uniform(shape=K.shape(x))
drop_prob = (K.log(self.p + eps) - K.log(1. - self.p + eps) +
K.log(unif_noise + eps) - K.log(1. - unif_noise + eps))
drop_prob = K.sigmoid(drop_prob / temp)
random_tensor = 1. - drop_prob
retain_prob = 1. - self.p
x *= random_tensor
x /= retain_prob
return x
def get_log_probability_density(pred, y):
mu_and_sigma = pred
mu = mu_and_sigma[:, :2]
sigma = mu_and_sigma[:, 2:]
variance = K.square(sigma)
pdf = 1. / K.sqrt(2. * np.pi * variance) * K.exp(-K.square(y - mu) /
(2. * variance))
log_pdf = K.log(pdf + K.epsilon())
return log_pdf
def loss(y_true, y_pred):
PPO_LOSS_CLIPPING = 0.2
PPO_ENTROPY_LOSS = 5 * 1e-3 # Does not converge without entropy penalty
log_pdf_new = get_log_probability_density(y_pred, y_true)
log_pdf_old = get_log_probability_density(old_prediction, y_true)
ratio = K.exp(log_pdf_new - log_pdf_old)
surrogate1 = ratio * advantage
clip_ratio = K.clip(ratio,
min_value=(1 - PPO_LOSS_CLIPPING),
max_value=(1 + PPO_LOSS_CLIPPING))
surrogate2 = clip_ratio * advantage
loss_actor = -K.mean(K.minimum(surrogate1, surrogate2))
sigma = y_pred[:, 2:]
variance = K.square(sigma)
loss_entropy = PPO_ENTROPY_LOSS * K.mean(
-(K.log(2 * np.pi * variance) + 1) / 2)
return loss_actor + loss_entropy
def __softmax_cosine_loss(y_true, y_pred, centers):
y_true_value = K.argmax(y_true)
base_loss = K.variable(0.0)
for label in range(CONFIG["num_classes"]):
center = centers[label]
indices = tf.where(tf.equal(y_true_value, label))
pred_per_class = tf.gather_nd(y_pred, indices=indices)
distances_per_class = __cosine(pred_per_class, center)
sum = K.sum(distances_per_class, axis=-1)
base_loss = tf.add(base_loss, sum)
base_distances = K.zeros_like(y_true_value, dtype='float32')
for label in range(CONFIG["num_classes"]):
center = centers[label]
distances_with_all_classes = __cosine(y_pred, center)
exp_distances = K.exp(-distances_with_all_classes)
base_distances = tf.add(base_distances, exp_distances)
log_distances = K.log(base_distances)
sum_on_batch = K.sum(log_distances)
return base_loss + sum_on_batch
def mean_pred(y_true, y_pred):
return -K.mean(y_true * K.log(y_pred + 1.e-7) +
(1 - y_true) * K.log(1 - y_pred + 1.e-7)) * 10
def yolo_loss(args, anchors, num_classes, ignore_thresh=.5):
"""Return yolo_loss tensor
Parameters
----------
yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body
y_true: list of array, the output of preprocess_true_boxes
anchors: array, shape=(N, 2), wh
num_classes: integer
ignore_thresh: float, the iou threshold whether to ignore object confidence loss
Returns
-------
loss: tensor, shape=(1,)
"""
num_layers = 1
yolo_outputs = args[:num_layers]
y_true = args[num_layers:]
g_y_true = y_true
input_shape = K.cast(
K.shape(yolo_outputs[0])[1:3] * grid_size_multiplier,
K.dtype(y_true[0]))
grid_shapes = [
K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0]))
for l in range(num_layers)
]
loss = 0
m = K.shape(yolo_outputs[0])[0] # batch size, tensor
mf = K.cast(m, K.dtype(yolo_outputs[0]))
for layer in range(num_layers):
object_mask = y_true[layer][..., 4:5]
vertices_mask = y_true[layer][..., 5 + num_classes + 2:5 +
num_classes + NUM_ANGLES3:3]
true_class_probs = y_true[layer][..., 5:5 + num_classes]
grid, raw_pred, pred_xy, pred_wh, pol_cnf = yolo_head(
yolo_outputs[layer],
anchors[anchor_mask[layer]],
num_classes,
input_shape,
calc_loss=True)
pred_box = K.concatenate([pred_xy, pred_wh])
raw_true_xy = y_true[layer][..., :2] * grid_shapes[layer][
..., ::-1] - grid
raw_true_polygon0 = y_true[layer][..., 5 + num_classes:5 +
num_classes + NUM_ANGLES3]
raw_true_wh = K.log(y_true[layer][..., 2:4] /
anchors[anchor_mask[layer]] *
input_shape[..., ::-1])
raw_true_wh = K.switch(object_mask, raw_true_wh,
K.zeros_like(raw_true_wh)) # avoid log(0)=-inf
raw_true_polygon_x = raw_true_polygon0[..., ::3]
raw_true_polygon_y = raw_true_polygon0[..., 1::3]
dx = K.square(anchors[anchor_mask[layer]][..., 0:1] / 2)
dy = K.square(anchors[anchor_mask[layer]][..., 1:2] / 2)
d = K.cast(K.sqrt(dx + dy), K.dtype(raw_true_polygon_x))
diagonal = K.sqrt(
K.pow(input_shape[..., ::-1][0], 2) +
K.pow(input_shape[..., ::-1][1], 2))
raw_true_polygon_x = K.log(raw_true_polygon_x / d * diagonal)
raw_true_polygon_x = K.switch(vertices_mask, raw_true_polygon_x,
K.zeros_like(raw_true_polygon_x))
box_loss_scale = 2 - y_true[layer][..., 2:3] * y_true[layer][..., 3:4]
# Find ignore mask, iterate over each of batch.
ignore_mask = tf.TensorArray(K.dtype(y_true[0]),
size=1,
dynamic_size=True)
object_mask_bool = K.cast(object_mask, 'bool')
def loop_body(b, ignore_mask):
true_box = tf.boolean_mask(y_true[layer][b, ..., 0:4],
object_mask_bool[b, ..., 0])
iou = box_iou(pred_box[b], true_box)
best_iou = K.max(iou, axis=-1)
ignore_mask = ignore_mask.write(
b, K.cast(best_iou < ignore_thresh, K.dtype(true_box)))
return b + 1, ignore_mask
_, ignore_mask = tf.while_loop(lambda b, *args: b < m, loop_body,
[0, ignore_mask])
ignore_mask = ignore_mask.stack()
ignore_mask = K.expand_dims(ignore_mask, -1)
# K.binary_crossentropy is helpful to avoid exp overflow.
xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(
raw_true_xy, raw_pred[..., 0:2], from_logits=True)
wh_loss = object_mask * box_loss_scale * 0.5 * K.square(
raw_true_wh - raw_pred[..., 2:4])
confidence_loss = object_mask * K.binary_crossentropy(
object_mask, raw_pred[..., 4:5], from_logits=True
) + (1 - object_mask) * K.binary_crossentropy(
object_mask, raw_pred[..., 4:5], from_logits=True) * ignore_mask
class_loss = object_mask * K.binary_crossentropy(
true_class_probs,
raw_pred[..., 5:5 + num_classes],
from_logits=True)
polygon_loss_x = object_mask * vertices_mask * box_loss_scale * 0.5 * K.square(
raw_true_polygon_x -
raw_pred[..., 5 + num_classes:5 + num_classes + NUM_ANGLES3:3])
polygon_loss_y = object_mask * vertices_mask * box_loss_scale * K.binary_crossentropy(
raw_true_polygon_y,
raw_pred[..., 5 + num_classes + 1:5 + num_classes + NUM_ANGLES3:3],
from_logits=True)
vertices_confidence_loss = object_mask * K.binary_crossentropy(
vertices_mask,
raw_pred[..., 5 + num_classes + 2:5 + num_classes + NUM_ANGLES3:3],
from_logits=True)
xy_loss = K.sum(xy_loss) / mf
wh_loss = K.sum(wh_loss) / mf
class_loss = K.sum(class_loss) / mf
confidence_loss = K.sum(confidence_loss) / mf
vertices_confidence_loss = K.sum(vertices_confidence_loss) / mf
polygon_loss = K.sum(polygon_loss_x) / mf + K.sum(polygon_loss_y) / mf
diou_loss = K.sum(
object_mask * box_loss_scale *
(1 - box_diou(pred_box, y_true[layer][..., 0:4]))) / mf
loss += (xy_loss + wh_loss + confidence_loss + class_loss +
0.2 * polygon_loss + 0.2 * vertices_confidence_loss) / (
K.sum(object_mask) + 1) * mf
return loss
def yolo_loss(args, anchors, num_classes, ignore_thresh=.5):
'''Return yolo_loss tensor
Parameters
----------
yolo_outputs: list of tensor, the output of yolo_body
y_true: list of array, the output of preprocess_true_boxes
anchors: array, shape=(T, 2), wh
num_classes: integer
ignore_thresh: float, the iou threshold whether to ignore object confidence loss
Returns
-------
loss: tensor, shape=(1,)
'''
yolo_outputs = args[:3]
y_true = args[3:]
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]
input_shape = K.cast(
K.shape(yolo_outputs[0])[1:3] * 32, K.dtype(y_true[0]))
grid_shapes = [
K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0]))
for l in range(3)
]
loss = 0
m = K.shape(yolo_outputs[0])[0]
for l in range(3):
object_mask = y_true[l][..., 4:5]
true_class_probs = y_true[l][..., 5:]
pred_xy, pred_wh, pred_confidence, pred_class_probs = yolo_head(
yolo_outputs[l], anchors[anchor_mask[l]], num_classes, input_shape)
pred_box = K.concatenate([pred_xy, pred_wh])
# Darknet box loss.
xy_delta = (y_true[l][..., :2] - pred_xy) * grid_shapes[l][::-1]
wh_delta = K.log(y_true[l][..., 2:4]) - K.log(pred_wh)
# Avoid log(0)=-inf.
wh_delta = K.switch(object_mask, wh_delta, K.zeros_like(wh_delta))
box_delta = K.concatenate([xy_delta, wh_delta], axis=-1)
box_delta_scale = 2 - y_true[l][..., 2:3] * y_true[l][..., 3:4]
# Find ignore mask, iterate over each of batch.
ignore_mask = tf.TensorArray(K.dtype(y_true[0]),
size=1,
dynamic_size=True)
object_mask_bool = K.cast(object_mask, 'bool')
def loop_body(b, ignore_mask):
true_box = tf.boolean_mask(y_true[l][b, ..., 0:4],
object_mask_bool[b, ..., 0])
iou = box_iou(pred_box[b], true_box)
best_iou = K.max(iou, axis=-1)
ignore_mask = ignore_mask.write(
b, K.cast(best_iou < ignore_thresh, K.dtype(true_box)))
return b + 1, ignore_mask
_, ignore_mask = K.control_flow_ops.while_loop(lambda b, *args: b < m,
loop_body,
[0, ignore_mask])
ignore_mask = ignore_mask.stack()
ignore_mask = K.expand_dims(ignore_mask, -1)
box_loss = object_mask * K.square(box_delta * box_delta_scale)
confidence_loss = object_mask * K.square(1-pred_confidence) + \
(1-object_mask) * K.square(0-pred_confidence) * ignore_mask
class_loss = object_mask * K.square(true_class_probs -
pred_class_probs)
loss += K.sum(box_loss) + K.sum(confidence_loss) + K.sum(class_loss)
return loss / K.cast(m, K.dtype(loss))
def yolo_loss(args, anchors, num_classes, ignore_thresh=.5, print_loss=False):
'''Return yolo_loss tensor
Parameters
----------
yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body
y_true: list of array, the output of preprocess_true_boxes
anchors: array, shape=(N, 2), wh
num_classes: integer
ignore_thresh: float, the iou threshold whether to ignore object confidence loss
Returns
-------
loss: tensor, shape=(1,)
'''
num_layers = len(anchors) // 3 # default setting
yolo_outputs = args[:num_layers]
y_true = args[num_layers:]
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]
] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]]
input_shape = K.cast(
K.shape(yolo_outputs[0])[1:3] * 32, K.dtype(y_true[0]))
grid_shapes = [
K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0]))
for l in range(num_layers)
]
loss = 0
m = K.shape(yolo_outputs[0])[0] # batch size, tensor
mf = K.cast(m, K.dtype(yolo_outputs[0]))
for l in range(num_layers):
object_mask = y_true[l][..., 4:5]
true_class_probs = y_true[l][..., 5:]
grid, raw_pred, pred_xy, pred_wh = yolo_head(yolo_outputs[l],
anchors[anchor_mask[l]],
num_classes,
input_shape,
calc_loss=True)
pred_box = K.concatenate([pred_xy, pred_wh])
# Darknet raw box to calculate loss.
raw_true_xy = y_true[l][..., :2] * grid_shapes[l][::-1] - grid
raw_true_wh = K.log(y_true[l][..., 2:4] / anchors[anchor_mask[l]] *
input_shape[::-1])
raw_true_wh = K.switch(object_mask, raw_true_wh,
K.zeros_like(raw_true_wh)) # avoid log(0)=-inf
box_loss_scale = 2 - y_true[l][..., 2:3] * y_true[l][..., 3:4]
# Find ignore mask, iterate over each of batch.
ignore_mask = tf.TensorArray(K.dtype(y_true[0]),
size=1,
dynamic_size=True)
object_mask_bool = K.cast(object_mask, 'bool')
def loop_body(b, ignore_mask):
true_box = tf.boolean_mask(y_true[l][b, ..., 0:4],
object_mask_bool[b, ..., 0])
iou = box_iou(pred_box[b], true_box)
best_iou = K.max(iou, axis=-1)
ignore_mask = ignore_mask.write(
b, K.cast(best_iou < ignore_thresh, K.dtype(true_box)))
return b + 1, ignore_mask
_, ignore_mask = K.control_flow_ops.while_loop(lambda b, *args: b < m,
loop_body,
[0, ignore_mask])
ignore_mask = ignore_mask.stack()
ignore_mask = K.expand_dims(ignore_mask, -1)
# K.binary_crossentropy is helpful to avoid exp overflow.
xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(
raw_true_xy, raw_pred[..., 0:2], from_logits=True)
wh_loss = object_mask * box_loss_scale * 0.5 * K.square(
raw_true_wh - raw_pred[..., 2:4])
confidence_loss = object_mask * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True)+ \
(1-object_mask) * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True) * ignore_mask
class_loss = object_mask * K.binary_crossentropy(
true_class_probs, raw_pred[..., 5:], from_logits=True)
xy_loss = K.sum(xy_loss) / mf
wh_loss = K.sum(wh_loss) / mf
confidence_loss = K.sum(confidence_loss) / mf
class_loss = K.sum(class_loss) / mf
loss += xy_loss + wh_loss + confidence_loss + class_loss
if print_loss:
loss = tf.Print(loss, [
loss, xy_loss, wh_loss, confidence_loss, class_loss,
K.sum(ignore_mask)
],
message='loss: ')
return loss
