def test_cce_one_hot():
y_a = K.variable(np.random.randint(0, 7, (5, 6)))
y_b = K.variable(np.random.random((5, 6, 7)))
objective_output = sparse_categorical_crossentropy(y_a, y_b)
assert K.eval(objective_output).shape == (5, 6)
y_a = K.variable(np.random.randint(0, 7, (6,)))
y_b = K.variable(np.random.random((6, 7)))
assert K.eval(sparse_categorical_crossentropy(y_a, y_b)).shape == (6,)
def transformer_loss(target_waveform, pred_waveform, target_genres,
pred_genres):
waveform_loss = sparse_categorical_crossentropy(target_waveform,
pred_waveform,
from_logits=True)
genre_loss = sparse_categorical_crossentropy(target_genres,
pred_genres,
from_logits=True)
return tf.reduce_sum(waveform_loss,
axis=-1) - 0.01 * 44100 * genre_loss
def call(self, inputs):
# update key extractor weights
k_weights = [self.m*w_k + (1-self.m)*w_q for w_k, w_q \
in zip(self.k_enc.get_weights(), self.q_enc.get_weights())]
self.k_enc.set_weights(k_weights)
# get two versions of same batch data
x_q, x_k = self.rand_aug(inputs)
# save the key and query
# sample_q = tf.io.encode_jpeg(tf.cast(x_q[0]*255, tf.uint8))
# ts = ''.join(map(str, list(time.localtime())))
# tf.io.write_file(f'{ts}_sample_q.jpeg', sample_q)
# sample_k = tf.io.encode_jpeg(tf.cast(x_k[0]*255, tf.uint8))
# tf.io.write_file(f'{ts}_sample_k.jpeg', sample_k)
# forward
q = self.g(self.q_enc(x_q))
q = tf.reshape(q, (tf.shape(q)[0], 1, -1))
k = self.g(self.k_enc(x_k))
k = tf.reshape(k, (tf.shape(k)[0], -1, 1))
l_pos = tf.squeeze(tf.matmul(q, k), axis=-1)
l_neg = tf.matmul(tf.squeeze(q), tf.transpose(self.queue))
# logits = softmax(tf.concat([l_pos, l_neg], axis=1))
logits = tf.concat([l_pos, l_neg], axis=1)
self.queue_them(tf.squeeze(k))
###### keras-fashion version ######
# return logits
###### gradient-tape version ######
labels = tf.zeros(tf.shape(inputs)[0])
loss = K.mean(sparse_categorical_crossentropy(labels, logits, from_logits=True))
l2 = tf.reduce_mean(tf.math.l2_normalize(q))
# print(K.max(logits, axis=1).numpy())
hits = tf.equal(tf.argmax(logits, axis=1), tf.cast(labels, 'int64'))
acc = tf.reduce_mean(tf.cast(hits, 'float64'))
return loss + 0.1 * l2, acc
def step(self, x_true, y_true):
with tf.GradientTape() as tape:
pred = self.model(x_true)
loss = sparse_categorical_crossentropy(y_true, pred)
grads = tape.gradient(loss, self.model.trainable_variables)
self.opt.apply_gradients(zip(grads, self.model.trainable_variables))
def custom_loss(y_true, y_pred):
loss = sparse_categorical_crossentropy(y_true, y_pred)
if extra_losses is not None:
for fn in extra_losses:
loss += fn(model)
return loss
def yolo_loss(y_true, y_pred):
# 1. transform all pred outputs
# y_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...cls))
pred_box, pred_obj, pred_class, pred_xywh = yolo_boxes(
y_pred, anchors, classes)
pred_xy = pred_xywh[..., 0:2]
pred_wh = pred_xywh[..., 2:4]
# 2. transform all true outputs
# y_true: (batch_size, grid, grid, anchors, (x1, y1, x2, y2, obj, cls))
true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1),
axis=-1)
true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2
true_wh = true_box[..., 2:4] - true_box[..., 0:2]
# give higher weights to small boxes
box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1]
# 3. inverting the pred box equations
grid_size = tf.shape(y_true)[1]
grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size))
grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)
true_xy = true_xy * tf.cast(grid_size, tf.float32) - \
tf.cast(grid, tf.float32)
true_wh = tf.math.log(true_wh / anchors)
true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh),
true_wh)
# 4. calculate all masks
obj_mask = tf.squeeze(true_obj, -1)
# ignore false positive when iou is over threshold
best_iou = tf.map_fn(
lambda x: tf.reduce_max(broadcast_iou(
x[0], tf.boolean_mask(x[1], tf.cast(x[2], tf.bool))),
axis=-1), (pred_box, true_box, obj_mask),
tf.float32)
ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32)
# 5. calculate all losses
xy_loss = obj_mask * box_loss_scale * \
tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1)
wh_loss = obj_mask * box_loss_scale * \
tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1)
obj_loss = binary_crossentropy(true_obj, pred_obj)
obj_loss = obj_mask * obj_loss + \
(1 - obj_mask) * ignore_mask * obj_loss
# TODO: use binary_crossentropy instead
if FLAGS.num_classes == 1:
class_loss = obj_mask * binary_crossentropy(
true_class_idx, pred_class, from_logits=True)
else:
class_loss = obj_mask * sparse_categorical_crossentropy(
true_class_idx, pred_class, from_logits=True)
# 6. sum over (batch, gridx, gridy, anchors) => (batch, 1)
xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3))
wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3))
obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3))
class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3))
return xy_loss + wh_loss + obj_loss + class_loss
def yolo_loss(y_true, y_pred):
""" Code from: https://github.com/zzh8829/yolov3-tf2/blob/master/yolov3_tf2/dataset.py """
# 1. transform all pred outputs
# y_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...cls))
pred_box, pred_obj, pred_class, pred_xywh = self.extract_from_predictions(
y_pred, scale_index)
pred_xy = pred_xywh[..., 0:2]
pred_wh = pred_xywh[..., 2:4]
# 2. transform all true outputs
# y_true: (batch_size, grid, grid, anchors, (x1, y1, x2, y2, obj, cls))
true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1),
axis=-1)
true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2
true_wh = true_box[..., 2:4] - true_box[..., 0:2]
# give higher weights to small boxes
box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1]
# 3. inverting the pred box equations
grid_size = tf.shape(y_true)[1]
grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size))
grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)
true_xy = true_xy * tf.cast(grid_size, tf.float32) - tf.cast(
grid, tf.float32)
true_wh = tf.math.log(true_wh /
self.anchors[self.masks[scale_index]])
true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh),
true_wh)
# 4. calculate all masks
obj_mask = tf.squeeze(true_obj, -1)
# ignore false positive when iou is over threshold
true_box_flat = tf.boolean_mask(true_box,
tf.cast(obj_mask, tf.bool))
best_iou = tf.reduce_max(helpers.broadcast_iou(
pred_box, true_box_flat),
axis=-1)
ignore_mask = tf.cast(best_iou < self.iou_threshold, tf.float32)
# 5. calculate all losses
xy_loss = obj_mask * box_loss_scale * tf.reduce_sum(
tf.square(true_xy - pred_xy), axis=-1)
wh_loss = obj_mask * box_loss_scale * tf.reduce_sum(
tf.square(true_wh - pred_wh), axis=-1)
obj_loss = binary_crossentropy(true_obj, pred_obj)
obj_loss = obj_mask * obj_loss + (
1 - obj_mask) * ignore_mask * obj_loss
class_loss = obj_mask * sparse_categorical_crossentropy(
true_class_idx, pred_class)
# 6. sum over (batch, gridx, gridy, anchors) => (batch, 1)
xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3))
wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3))
obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3))
class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3))
return xy_loss + wh_loss + obj_loss + class_loss
def _build_net(self):
input = layers.Input(shape=(self.n_features,))
x = layers.Dense(10, activation="tanh")(input)
self.all_act = layers.Dense(self.n_actions)(x)
loss = losses.sparse_categorical_crossentropy()
rmsprop = optimizers.RMSprop(lr=self.learning_rate)
self.model = models.Model(input, self.all_act)
self.model.compile(loss=loss, optimizer=rmsprop, metrics=['accuracy'])
def compute_loss(y_true, y_pred):
M = tf.convert_to_tensor(y_true[:, :, :, :, 0])
W = tf.convert_to_tensor(y_true[:, :, :, :, 1])
y_pred = tf.convert_to_tensor(y_pred)
y1 = sparse_categorical_crossentropy(M, y_pred)
y2 = tf.math.multiply(W, y1)
y1 = tf.reduce_mean(y1)
y2 = tf.reduce_mean(y2)
y = y1 + alpha * y2
return y
def compute_loss(y_true, y_pred):
"""
计算loss
:param y_true: 模型
:param y_pred:
:return:
"""
mse = losses.sparse_categorical_crossentropy(y_true, y_pred, from_logits=True)
return mse
def yolo_loss(y_true, y_pred):
# y_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...cls))
pred_box, pred_obj, pred_class, pred_xywh = yoloBoxes(
y_pred, anchors, classes)
pred_xy = pred_xywh[..., 0:2]
pred_wh = pred_xywh[..., 2:4]
# PASCAL VOC -> COCO
# xmin, ymin, xmax, ymax -> x, y, w, h
# y_true: (batch_size, grid, grid, anchors, (x1, y1, x2, y2, obj, cls))
true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1),
axis=-1)
true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2
true_wh = true_box[..., 2:4] - true_box[..., 0:2]
# give higher weights to small boxes
# true_wh[..., 0] * true_wh[..., 1] : box area
box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1]
grid_size = tf.shape(y_true)[1]
grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size))
grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)
true_xy = true_xy * tf.cast(grid_size, tf.float32) - tf.cast(
grid, tf.float32)
true_wh = tf.math.log(true_wh / anchors)
true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh),
true_wh)
obj_mask = tf.squeeze(true_obj, -1)
best_iou = tf.map_fn(
lambda x: tf.reduce_max(broadcast_iou(
x[0], tf.boolean_mask(x[1], tf.cast(x[2], tf.bool))),
axis=-1), (pred_box, true_box, obj_mask),
tf.float32)
ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32)
xy_loss = obj_mask * box_loss_scale * tf.reduce_sum(
tf.square(true_xy - pred_xy), axis=-1)
wh_loss = obj_mask * box_loss_scale * tf.reduce_sum(
tf.square(true_wh - pred_wh), axis=-1)
obj_loss = binary_crossentropy(true_obj, pred_obj)
obj_loss = obj_mask * obj_loss + (1 -
obj_mask) * ignore_mask * obj_loss
class_loss = obj_mask * sparse_categorical_crossentropy(
true_class_idx, pred_class)
xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3))
wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3))
obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3))
class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3))
return xy_loss + wh_loss + obj_loss + class_loss
def custom_loss(y_true, y_pred):
# y_true[i] = (row, col, height, width,
# p_class1, p_class2, p_class3,
# p(object_appeared|img))
# the bounding box loss:
bce_1 = binary_crossentropy(y_true[:, :4], y_pred[:, :4])
# the object class prediction loss:
scce = sparse_categorical_crossentropy(y_true[:, 4], y_pred[:, 4:-1])
# the binary prediction (about an object being present in an image) loss:
bce_2 = binary_crossentropy(y_true[:, -1], y_pred[:, -1])
return ALPHA * y_true[:,
-1] * bce_1 + BETA * y_true[:,
-1] * scce + GAMMA * bce_2
def do_test_forward(self, sequence):
with self.device:
self.test_metric.reset_states()
for inputs, labels in sequence:
x, adj, index, _ = inputs
logit = self.propagation(x, adj, training=False)
output = tf.gather(logit, index)
output = softmax(output)
loss = tf.reduce_mean(sparse_categorical_crossentropy(labels, output))
self.test_metric.update_state(labels, output)
return loss, self.test_metric.result()
def yolo_loss(y_true, y_pred):
# 1. transform all pred outputs
# y_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...cls))
pred_box, pred_obj, pred_class = yolo_boxes(y_pred,
anchors,
classes,
calc_loss=True)
pred_xy = (pred_box[..., 0:2] + pred_box[..., 2:4]) / 2
pred_wh = pred_box[..., 2:4] - pred_box[..., 0:2]
# 2. transform all true outputs
# y_true: (batch_size, grid, grid, anchors, (x1, y1, x2, y2, obj, cls))
true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1),
axis=-1)
true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2
true_wh = true_box[..., 2:4] - true_box[..., 0:2]
# give higher weights to small boxes
box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1]
# 4. calculate all masks
obj_mask = tf.squeeze(true_obj, -1)
# ignore false positive when iou is over threshold
best_iou = tf.map_fn(
lambda x: tf.reduce_max(broadcast_iou(
x[0], tf.boolean_mask(x[1], tf.cast(x[2], tf.bool))),
axis=-1), (pred_box, true_box, obj_mask),
tf.float32)
ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32)
# 5. calculate all losses
xy_loss = obj_mask * box_loss_scale * \
tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1)
wh_loss = obj_mask * box_loss_scale * \
tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1)
obj_loss = binary_crossentropy(true_obj, pred_obj, from_logits=True)
obj_loss = obj_mask * obj_loss + \
(1 - obj_mask) * ignore_mask * obj_loss
# TODO: use binary_crossentropy instead
class_loss = obj_mask * sparse_categorical_crossentropy(
true_class_idx, pred_class, from_logits=True)
# 6. sum over (batch, gridx, gridy, anchors) => (batch, 1)
xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3))
wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3))
obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3))
class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3))
return xy_loss + wh_loss + obj_loss + class_loss
def call(self, inputs: list[tf.Tensor], *args, **kwargs) -> tf.Tensor:
# (batch_size, label_seq_len)
# (batch_size, label_seq_len, vocab_size)
# (batch_size, label_seq_len)
token_ids, logits, mask = inputs
loss = sparse_categorical_crossentropy(token_ids, logits, from_logits=True)
mask = tf.cast(mask, loss.dtype)
self.add_loss(tf.reduce_sum(loss * mask) / tf.reduce_sum(mask))
predicted_token_ids = tf.math.argmax(logits, axis=-1)
is_equal = tf.cast(tf.equal(token_ids, predicted_token_ids), loss.dtype)
lm_accuracy = tf.reduce_sum(is_equal * mask) / tf.reduce_sum(mask)
self.add_metric(lm_accuracy, name="lm_accuracy")
boolean_mask = tf.cast(mask, tf.bool)
return tf.ragged.boolean_mask(predicted_token_ids, boolean_mask)
def yoloLoss(y_true, y_pred):
pred_box, pred_obj, pred_class, pred_xywh = yolo_boxes(
y_pred, anchors, class_num)
pred_xy = pred_xywh[..., 0:2] # 取出偏移量
pred_wh = pred_xywh[..., 2:4]
true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1),
axis=-1)
true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2
true_wh = true_box[..., 2:4] - true_box[..., 0:2]
box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1]
grid_size = tf.shape(y_true)[1]
grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size))
grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)
true_xy = true_xy * tf.cast(grid_size, tf.float32) - \
tf.cast(grid, tf.float32)
true_wh = tf.math.log(true_wh / anchors)
true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh),
true_wh) # 将真实坐标转变为偏移量用于计算loss
obj_mask = tf.squeeze(true_obj, -1)
# ignore false positive when iou is over threshold
best_iou = tf.map_fn(
lambda x: tf.reduce_max(broadcast_iou(
x[0], tf.boolean_mask(x[1], tf.cast(x[2], tf.bool))),
axis=-1), (pred_box, true_box, obj_mask),
tf.float32)
ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32)
xy_loss = obj_mask * box_loss_scale * \
tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1)
wh_loss = obj_mask * box_loss_scale * \
tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1)
obj_loss = binary_crossentropy(true_obj, pred_obj)
obj_loss = obj_mask * obj_loss + \
(1 - obj_mask) * ignore_mask * obj_loss
# TODO: use binary_crossentropy instead
class_loss = obj_mask * sparse_categorical_crossentropy(
true_class_idx, pred_class)
xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3))
wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3))
obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3))
class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3))
return xy_loss + wh_loss + obj_loss + class_loss
def yolo_loss(y_true, y_pred):
pred_box, pred_obj, pred_class, pred_xywh = yolo_boxes(
y_pred, anchors, classes)
pred_xy = pred_xywh[..., 0:2]
pred_wh = pred_xywh[..., 2:4]
true_box, true_obj, true_class_idx = tf.split(
y_true, (4, 1, 1), axis=-1)
true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2
true_wh = true_box[..., 2:4] - true_box[..., 0:2]
box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1]
grid_size = tf.shape(y_true)[1]
grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size))
grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)
true_xy = true_xy * tf.cast(grid_size, tf.float32) - \
tf.cast(grid, tf.float32)
true_wh = tf.math.log(true_wh / anchors)
true_wh = tf.where(tf.math.is_inf(true_wh),
tf.zeros_like(true_wh), true_wh)
obj_mask = tf.squeeze(true_obj, -1)
# ignore when Intersection Over Union is over threshold
true_box_flat = tf.boolean_mask(true_box, tf.cast(obj_mask, tf.bool))
best_iou = tf.reduce_max(intersectionOverUnion(
pred_box, true_box_flat), axis=-1)
ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32)
xy_loss = obj_mask * box_loss_scale * \
tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1)
wh_loss = obj_mask * box_loss_scale * \
tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1)
obj_loss = binary_crossentropy(true_obj, pred_obj)
obj_loss = obj_mask * obj_loss + \
(1 - obj_mask) * ignore_mask * obj_loss
class_loss = obj_mask * sparse_categorical_crossentropy(
true_class_idx, pred_class)
xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3))
wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3))
obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3))
class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3))
return xy_loss + wh_loss + obj_loss + class_loss
def mask_sparse_cross_entropy(y_true=None, y_pred=None, mask=0):
# y_true: [None, steps]
# y_pred: [None, steps, num_classes+1], +1 for pad to mask
# loss: [None, steps]
loss = sparse_categorical_crossentropy(y_true, y_pred)
# masks: [None, steps]
masks = tf.cast(tf.not_equal(y_true, mask), tf.float32)
# masked-loss: [None, steps]
loss = tf.multiply(loss, masks)
# reduce_mean: shape=()
loss = tf.cast(tf.reduce_mean(loss), tf.float32)
return loss
def yolo_loss(y_true, y_pred):
p_bboxes, p_objectness, p_class, p_original = make_bboxs(y_pred, anchors, classes)
p_xy, p_wh = tf.split(p_original, (2, 2), axis=-1)
t_xy = (y_true[..., 0:2] + y_true[..., 2:4]) / 2 # Center of each coord
t_wh = y_true[..., 2:4] - y_true[..., 0:2] # Width and height for each
# Give higher weights to small boxes
l_tune = 2 - t_wh[..., 0] * t_wh[..., 1]
# Inverting the pred box equations
grid_size = tf.shape(y_true)[1]
grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size))
grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)
t_xy = t_xy * tf.cast(grid_size, tf.float32) - tf.cast(grid, tf.float32)
t_wh = tf.math.log(t_wh / anchors)
t_wh = tf.where(tf.math.is_inf(t_wh), tf.zeros_like(t_wh), t_wh)
# Calculate masks
obj_mask = tf.squeeze(y_true[..., 4:5], -1)
gather_mask = tf.where(tf.not_equal(obj_mask, 0.))
buffer = [0] * 7
for i, el in enumerate([t_wh, p_wh, t_xy, p_xy, y_true[..., 5:6], p_class, l_tune]):
buffer[i] = tf.gather_nd(el, gather_mask)
t_wh, p_wh, t_xy, p_xy, t_class, p_class, l_tune = buffer
# Calculate best IOU between predictions and true objects
best_iou = bbox_iou(p_bboxes, tf.boolean_mask(y_true[..., :4], tf.cast(obj_mask, tf.bool)))
obj_loss = losses.binary_crossentropy(y_true[..., 4:5], p_objectness)
obj_loss = obj_mask * obj_loss + (1 - obj_mask) * obj_loss * tf.cast(best_iou < iou_thresh, tf.float32)
# Losses
wh_loss = l_tune * tf.reduce_sum(tf.square(t_wh - p_wh), axis=-1)
xy_loss = l_tune * tf.reduce_sum(tf.square(t_xy - p_xy), axis=-1)
class_loss = losses.sparse_categorical_crossentropy(t_class, p_class) * 20
total_loss = 0
for loss in [xy_loss, wh_loss, obj_loss, class_loss]:
total_loss += tf.reduce_sum(loss)
return total_loss
def yolo_loss(y_true, y_pred):
pred_box, pred_obj, pred_class, pred_xywh = get_boxes(
y_pred, anchors, classes)
pred_xy = pred_xywh[..., 0:2]
pred_wh = pred_xywh[..., 2:4]
true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1),
axis=-1)
true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2
true_wh = true_box[..., 2:4] - true_box[..., 0:2]
box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1]
grid_size = tf.shape(y_true)[1]
grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size))
grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)
true_xy = true_xy * tf.cast(grid_size, tf.float32) - tf.cast(
grid, tf.float32)
true_wh = tf.math.log(true_wh / anchors)
true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh),
true_wh)
obj_mask = tf.squeeze(true_obj, -1)
best_iou = tf.map_fn(
lambda x: tf.reduce_max(
broadcast_iou(x[0],
tf.boolean_mask(x[1], tf.cast(x[2], tf.bool))),
axis=-1,
),
(pred_box, true_box, obj_mask),
fn_output_signature=tf.float32,
)
ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32)
xy_loss = (obj_mask * box_loss_scale *
tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1))
wh_loss = (obj_mask * box_loss_scale *
tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1))
obj_loss = binary_crossentropy(true_obj, pred_obj)
obj_loss = obj_mask * obj_loss + (1 -
obj_mask) * ignore_mask * obj_loss
class_loss = obj_mask * sparse_categorical_crossentropy(
true_class_idx, pred_class)
xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3))
wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3))
obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3))
class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3))
return xy_loss + wh_loss + obj_loss + class_loss
def custom_loss(y_true, y_pred):
y_pred = tf.reshape(y_pred, (-1, n_objects, 5 + n_classes))
pred_obj, pred_xywh, pred_class = tf.split(y_pred, [1, 4, n_classes], axis=-1)
y_true = tf.reshape(y_true, (-1, n_objects, 6))
true_obj, true_xywh, true_class = tf.split(y_true, [1, 4, 1], axis=-1)
# Cross entropy loss for score
obj_loss = binary_crossentropy(true_obj, pred_obj)
obj_loss = tf.reduce_sum(obj_loss, axis=1)
# MSE loss for the BB
bb_loss = mean_squared_error(true_xywh, pred_xywh)
bb_loss = tf.reduce_sum(bb_loss, axis=1)
# We have to convert the class into a vector
class_loss = sparse_categorical_crossentropy(true_class, pred_class)
class_loss = tf.reduce_sum(class_loss, axis=1)
return obj_loss + bb_loss + class_loss
def crf_loss(y_true, y_pred):
"""General CRF loss function depending on the learning mode.
Args:
y_true: tensor with true targets.
y_pred: tensor with predicted targets.
Returns:
If the CRF layer is being trained in the join mode, returns the negative
log-likelihood. Otherwise returns the categorical crossentropy implemented
by the underlying Keras backend.
"""
crf, idx = y_pred._keras_history[:2]
if crf.learn_mode == 'join':
return crf_nll(y_true, y_pred)
else:
if crf.sparse_target:
return sparse_categorical_crossentropy(y_true, y_pred)
else:
return categorical_crossentropy(y_true, y_pred)
def crf_loss(y_true, y_pred):
"""General CRF loss function depending on the learning mode.
# Arguments
y_true: tensor with true targets.
y_pred: tensor with predicted targets.
# Returns
If the CRF layer is being trained in the join mode, returns the negative
log-likelihood. Otherwise returns the categorical crossentropy implemented
by the underlying Keras backend.
# About GitHub
If you open an issue or a pull request about CRF, please
add `cc @lzfelix` to notify Luiz Felix.
"""
crf, idx = y_pred._keras_history[:2]
if crf.learn_mode == 'join':
return crf_nll(y_true, y_pred)
else:
if crf.sparse_target:
return sparse_categorical_crossentropy(y_true, y_pred)
else:
return categorical_crossentropy(y_true, y_pred)
def step(X, y):
# get the weights
weights = phase2Kernel.forward()
conv_bias = model.layers[0].get_weights()[1]
model.layers[0].set_weights([weights, conv_bias])
# keep track of our gradients
with tf.GradientTape() as tape:
# make a prediction using the model and then calculate the
# loss
pred = model(X)
print("Pred shape:", pred.shape)
loss = sparse_categorical_crossentropy(y, pred)
accuracy = accuracy_score(y, np.argmax(pred, axis=1))
print("loss", tf.math.reduce_mean(loss))
print("accuracy:", accuracy)
grads = tape.gradient(loss, model.trainable_variables)
print("gradients shapes: ", grads[0].shape, grads[1].shape)
final_grad = phase2Kernel.backward(grads[0])
phase2Kernel.phi -= PhaseLR * cp.asnumpy(final_grad)
opt.apply_gradients(zip(grads, model.trainable_variables))
def do_train_forward(self, sequence):
with self.device:
self.train_metric.reset_states()
for inputs, labels in sequence:
x, adj, index, adv_mask = inputs
with tf.GradientTape() as tape:
logit = self.propagation(x, adj)
output = tf.gather(logit, index)
output = softmax(output)
loss = tf.reduce_mean(sparse_categorical_crossentropy(labels, output))
entropy_loss = entropy_y_x(logit)
vat_loss = self.virtual_adversarial_loss(x, adj, logit=logit, adv_mask=adv_mask)
loss += self.p1 * vat_loss + self.p2 * entropy_loss
self.train_metric.update_state(labels, output)
trainable_variables = self.model.trainable_variables
gradients = tape.gradient(loss, trainable_variables)
self.optimizer.apply_gradients(zip(gradients, trainable_variables))
return loss, self.train_metric.result()
def yolo_loss(detector_mask, matching_gt_boxes, matching_classes_oh,
gt_boxes_grid, y_pred):
# detector_mask: [b,16,16,5,1]
# matching_gt_boxes: [b,16,16,5,5] x-y-w-h-l
# matching_classes_oh: [b,16,16,5,2] l1-l2
# gt_boxes_grid: [b,40,5] x-y-wh-l
# y_pred: [b,16,16,5,7] x-y-w-h-conf-l0-l1
IMGSZ = cfg.TRAIN.IMGSZ
GRIDSZ = cfg.TRAIN.GRIDSZ
ANCHORS = cfg.TRAIN.ANCHORS
anchors = np.array(ANCHORS).reshape(5, 2)
# create starting position for each grid anchors
# [16,16]
x_grid = tf.tile(tf.range(GRIDSZ), [GRIDSZ])
# [1,16,16,1,1]
# [b,16,16,5,2]
x_grid = tf.reshape(x_grid, (1, GRIDSZ, GRIDSZ, 1, 1))
x_grid = tf.cast(x_grid, tf.float32)
# [b,16_1,16_2,1,1]=>[b,16_2,16_1,1,1]
y_grid = tf.transpose(x_grid, (0, 2, 1, 3, 4))
xy_grid = tf.concat([x_grid, y_grid], axis=-1)
# [1,16,16,1,2]=> [b,16,16,5,2]
xy_grid = tf.tile(xy_grid, [y_pred.shape[0], 1, 1, 5, 1])
# [b,16,16,5,7] x-y-w-h-conf-l1-l2
pred_xy = tf.sigmoid(y_pred[..., 0:2])
pred_xy = pred_xy + xy_grid
# [b,16,16,5,2]
pred_wh = tf.exp(y_pred[..., 2:4])
# [b,16,16,5,2] * [5,2] => [b,16,16,5,2]
pred_wh = pred_wh * anchors
n_detector_mask = tf.reduce_sum(tf.cast(detector_mask > 0., tf.float32))
# [b,16,16,5,1] * [b,16,16,5,2]
#
xy_loss = detector_mask * tf.square(matching_gt_boxes[..., :2] -
pred_xy) / (n_detector_mask + 1e-6)
xy_loss = tf.reduce_sum(xy_loss)
wh_loss = detector_mask * tf.square(tf.sqrt(matching_gt_boxes[..., 2:4]) - \
tf.sqrt(pred_wh)) / (n_detector_mask + 1e-6)
wh_loss = tf.reduce_sum(wh_loss)
# 4.1 coordinate loss
coord_loss = xy_loss + wh_loss
# 4.2 class loss
# [b,16,16,5,2]
pred_box_class = y_pred[..., 5:]
# [b,16,16,5]
true_box_class = tf.argmax(matching_classes_oh, -1)
# [b,16,16,5] vs [b,16,16,5,2]
class_loss = losses.sparse_categorical_crossentropy( \
true_box_class, pred_box_class, from_logits=True)
# [b,16,16,5] => [b,16,16,5,1]* [b,16,16,5,1]
class_loss = tf.expand_dims(class_loss, -1) * detector_mask
class_loss = tf.reduce_sum(class_loss) / (n_detector_mask + 1e-6)
# 4.3 object loss
# nonobject_mask
# iou done!
# [b,16,16,5]
x1, y1, w1, h1 = matching_gt_boxes[..., 0], matching_gt_boxes[..., 1], \
matching_gt_boxes[..., 2], matching_gt_boxes[..., 3]
# [b,16,16,5]
x2, y2, w2, h2 = pred_xy[..., 0], pred_xy[..., 1], pred_wh[...,
0], pred_wh[...,
1]
ious = compute_iou(x1, y1, w1, h1, x2, y2, w2, h2)
# [b,16,16,5,1]
ious = tf.expand_dims(ious, axis=-1)
# [b,16,16,5,1]
pred_conf = tf.sigmoid(y_pred[..., 4:5])
# [b,16,16,5,2] => [b,16,16,5, 1, 2]
pred_xy = tf.expand_dims(pred_xy, axis=4)
# [b,16,16,5,2] => [b,16,16,5, 1, 2]
pred_wh = tf.expand_dims(pred_wh, axis=4)
pred_wh_half = pred_wh / 2.
pred_xymin = pred_xy - pred_wh_half
pred_xymax = pred_xy + pred_wh_half
# [b, 40, 5] => [b, 1, 1, 1, 40, 5]
true_boxes_grid = tf.reshape(gt_boxes_grid, [
gt_boxes_grid.shape[0], 1, 1, 1, gt_boxes_grid.shape[1],
gt_boxes_grid.shape[2]
])
true_xy = true_boxes_grid[..., 0:2]
true_wh = true_boxes_grid[..., 2:4]
true_wh_half = true_wh / 2.
true_xymin = true_xy - true_wh_half
true_xymax = true_xy + true_wh_half
# predxymin, predxymax, true_xymin, true_xymax
# [b,16,16,5,1,2] vs [b,1,1,1,40,2]=> [b,16,16,5,40,2]
intersectxymin = tf.maximum(pred_xymin, true_xymin)
# [b,16,16,5,1,2] vs [b,1,1,1,40,2]=> [b,16,16,5,40,2]
intersectxymax = tf.minimum(pred_xymax, true_xymax)
# [b,16,16,5,40,2]
intersect_wh = tf.maximum(intersectxymax - intersectxymin, 0.)
# [b,16,16,5,40] * [b,16,16,5,40]=>[b,16,16,5,40]
intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1]
# [b,16,16,5,1]
pred_area = pred_wh[..., 0] * pred_wh[..., 1]
# [b,1,1,1,40]
true_area = true_wh[..., 0] * true_wh[..., 1]
# [b,16,16,5,1]+[b,1,1,1,40]-[b,16,16,5,40]=>[b,16,16,5,40]
union_area = pred_area + true_area - intersect_area
# [b,16,16,5,40]
iou_score = intersect_area / union_area
# [b,16,16,5]
best_iou = tf.reduce_max(iou_score, axis=4)
# [b,16,16,5,1]
best_iou = tf.expand_dims(best_iou, axis=-1)
nonobj_detection = tf.cast(best_iou < 0.6, tf.float32)
nonobj_mask = nonobj_detection * (1 - detector_mask)
# nonobj counter
n_nonobj = tf.reduce_sum(tf.cast(nonobj_mask > 0., tf.float32))
nonobj_loss = tf.reduce_sum(nonobj_mask * tf.square(-pred_conf)) \
/ (n_nonobj + 1e-6)
obj_loss = tf.reduce_sum(detector_mask * tf.square(ious - pred_conf)) \
/ (n_detector_mask + 1e-6)
loss = coord_loss + class_loss + nonobj_loss + 5 * obj_loss
return loss, [nonobj_loss + 5 * obj_loss, class_loss, coord_loss]
def yolo_loss(y_true, y_pred):
#tf.print("true", y_true.shape)
# 1. transform all pred outputs
# y_pred: (batch_size, grid_y, grid_x, anchors, (x, y, w, h, obj, ...cls))
pred_box, pred_obj, pred_class, pred_xywh = yolo_boxes(
y_pred, anchors, classes)
pred_xy = pred_xywh[..., 0:2]
pred_wh = pred_xywh[..., 2:4]
#tf.print("before", y_pred[...][0, tf.random.uniform(shape=[], minval=0, maxval=16, dtype=tf.int64), tf.random.uniform(shape=[], minval=0, maxval=16, dtype=tf.int64), tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int64), :])
# 2. transform all true outputs
# y_true: (batch_size, grid_y, grid_x, anchors, (x1, y1, x2, y2, obj, cls))
true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1),
axis=-1)
true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2
true_wh = true_box[..., 2:4] - true_box[..., 0:2]
# give higher weights to small boxes
box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1]
# 3. inverting the pred box equations
grid_size = tf.shape(y_true)[1:3]
grid = tf.meshgrid(tf.range(grid_size[1]), tf.range(grid_size[0]))
grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)
true_xy = true_xy * tf.cast((grid_size[1], grid_size[0]), tf.float32) - \
tf.cast(grid, tf.float32)
true_wh = tf.math.log(true_wh / anchors)
true_wh = tf.where(tf.math.is_inf(true_wh),
tf.ones_like(true_wh) * -20, true_wh)
# 4. calculate all masks
obj_mask = tf.squeeze(true_obj, -1)
mask = tf.not_equal(obj_mask, 0)
not_mask = tf.logical_not(mask)
summarize = 20
#tf.print("a", pred_xy.shape)
#tf.print("b", obj_mask.shape)
#tf.print("c", mask.shape)
#tf.print("d", tf.boolean_mask(true_obj, mask), summarize=summarize)
#tf.print("e", tf.boolean_mask(pred_obj, mask), summarize=summarize)
# ignore false positive when iou is over threshold
best_iou = tf.map_fn(
lambda x: tf.reduce_max(broadcast_iou(
x[0], tf.boolean_mask(x[1], tf.cast(x[2], tf.bool))),
axis=-1), (pred_box, true_box, obj_mask),
tf.float32)
ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32)
# 5. calculate all losses
xy_loss = obj_mask * box_loss_scale * \
tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1)
wh_loss = obj_mask * box_loss_scale * \
tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1)
#tf.print("losses 1", tf.math.reduce_sum(xy_loss))
#tf.print("losses 2", tf.math.reduce_sum(wh_loss))
#tf.print("true_obj shape", true_obj.shape, true_obj.dtype)
#tf.print("pred_obj shape", pred_obj.shape, pred_obj.dtype)
obj_loss = binary_crossentropy(true_obj, pred_obj)
#tf.print("obj_loss shape", obj_loss.shape)
#tf.print("f", tf.boolean_mask(obj_loss, mask), summarize=summarize)
obj_loss = obj_mask * obj_loss + \
(1 - obj_mask) * ignore_mask * obj_loss
#tf.print("losses 3", tf.math.reduce_sum(obj_loss))
#tf.print("g", tf.boolean_mask(y_pred, mask), summarize=summarize)
#tf.print("h", tf.boolean_mask(y_pred, not_mask), summarize=summarize)
#tf.print("g_true", tf.boolean_mask(y_true, mask), summarize=summarize)
#tf.print("h_true", tf.boolean_mask(y_true, not_mask), summarize=summarize)
#tf.cond(tf.logical_or(tf.math.is_nan(tf.math.reduce_sum(obj_loss)), tf.math.is_inf(tf.math.reduce_sum(obj_loss))), lambda: [tf.print(y_pred), y_pred][-1], lambda: y_pred)
# TODO: use binary_crossentropy instead
class_loss = obj_mask * sparse_categorical_crossentropy(
true_class_idx, pred_class)
# 6. sum over (batch, gridx, gridy, anchors) => (batch, 1)
xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3))
wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3))
obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3))
class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3))
loss = xy_loss + wh_loss + obj_loss + class_loss
loss_mask = tf.logical_or(tf.math.is_nan(loss), tf.math.is_nan(loss))
#tf.print("xy_loss", tf.boolean_mask(xy_loss, loss_mask))
#tf.print("wh_loss", tf.boolean_mask(wh_loss, loss_mask))
#tf.print("obj_loss", tf.boolean_mask(obj_loss, loss_mask))
#tf.print("class_loss", tf.boolean_mask(class_loss, loss_mask))
#tf.print("loss", tf.boolean_mask(loss, loss_mask))
#tf.print("non entries", tf.boolean_mask(y_pred, loss_mask))
#tf.print("tru entries", tf.boolean_mask(y_true, loss_mask))
#tf.print("LOSS", xy_loss + wh_loss + obj_loss + class_loss)
return xy_loss + wh_loss + obj_loss + class_loss
def sparse_cat_loss(y_true, y_pred): #wrapper
return sparse_categorical_crossentropy(
y_true, y_pred, from_logits=True) #for one-hot-encoded features
def yolo_loss(y_true, y_pred):
# Part 1 - Transform all pred outputs:
# y_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...cls))
pred_box, pred_obj, pred_class, pred_xywh = yolo_boxes(pred=y_pred,
anchors=anchors,
classes=classes)
# Split the predicted box shape lines:
pred_xy = pred_xywh[..., 0:2]
pred_wh = pred_xywh[..., 2:4]
# Part 2 - Transform all true outputs
# y_true: (batch_size, grid, grid, anchors, (x1, y1, x2, y2, obj, cls))
true_box, true_obj, true_class_idx = tf.split(value=y_true,
num_or_size_splits=(4, 1,
1),
axis=-1)
# Split the Ground truth box shape lines:
true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2
true_wh = true_box[..., 2:4] - true_box[..., 0:2]
# Allow for higher weights to small boxes:
box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1]
# Part 3 - Inverting the pred box equations:
grid_size = tf.shape(y_true)[1]
grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size))
grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)
# Convert true_xy to match grid size:
true_xy = true_xy * tf.cast(grid_size, tf.float32) - tf.cast(
grid, tf.float32)
true_wh = tf.math.log(true_wh / anchors)
true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh),
true_wh)
# Part 4 - Compute all masks:
obj_mask = tf.squeeze(input=true_obj, axis=-1)
# Ignore when IoU is over the threshold stated:
true_box_flat = tf.boolean_mask(tensor=true_box,
mask=tf.cast(obj_mask, tf.bool))
# Find the Best IoU:
best_iou = tf.reduce_max(intersectionOverUnion(box1=pred_box,
box2=true_box_flat),
axis=-1)
# Ignore IoU:
ignore_mask = tf.cast(best_iou < ignore_threshold, tf.float32)
# Part 5 - Compute all losses:
xy_loss = obj_mask * box_loss_scale * tf.reduce_sum(
input_tensor=tf.square(true_xy - pred_xy), axis=-1)
wh_loss = obj_mask * box_loss_scale * tf.reduce_sum(
input_tensor=tf.square(true_wh - pred_wh), axis=-1)
# Using binary_crossentropy for the object loss:
obj_loss = binary_crossentropy(y_true=true_obj, y_pred=pred_obj)
obj_loss = obj_mask * obj_loss + (1 -
obj_mask) * ignore_mask * obj_loss
# Using sparse_categorical_crossentropy instead for the classes loss:
class_loss = obj_mask * sparse_categorical_crossentropy(
y_true=true_class_idx, y_pred=pred_class)
# Part 6 - Sum over (batch, gridx, gridy, anchors) => (batch, 1)
xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3))
wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3))
obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3))
class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3))
return xy_loss + wh_loss + obj_loss + class_loss
def sparse_categorical_crossentropy(y_true, y_pred):
y_true = tf.cast(y_true, 'float32')
y_pred = tf.cast(y_pred, 'float32')
return losses.sparse_categorical_crossentropy(y_true, y_pred)